diff --git a/.github/workflows/bot-pr-new.yaml b/.github/workflows/bot-pr-new.yaml new file mode 100644 index 00000000000..13724cc14f0 --- /dev/null +++ b/.github/workflows/bot-pr-new.yaml @@ -0,0 +1,73 @@ +# Greet new pull requests with a welcome comment and apply labels. +# This workflow must initiate from an authenticated bot repo collaborator. +# Webhook events: Pull requests +name: New pull request +on: + repository_dispatch: + types: [opened, reopened] + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + comment-welcome: + permissions: + contents: read # to fetch code (actions/checkout) + pull-requests: write # to comment on pull-request + + if: ${{ github.actor == 'tfdocsbot' }} + runs-on: ubuntu-latest + steps: + - name: Fetch pull request branch + uses: actions/checkout@v2 + with: + repository: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + ref: ${{ github.event.client_payload.pull_request.head.sha }} + - name: Fetch base master branch + run: git fetch -u "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" master:master + - name: Create message + env: + HEAD_REPOSITORY: ${{ github.event.client_payload.pull_request.head.repo.full_name }} + HEAD_REF: ${{ github.event.client_payload.pull_request.head.ref }} + PR_NUM: ${{ github.event.client_payload.pull_request.number }} + run: | + # Preview links and tool usage only needed for notebook changes. + readarray -t changed_notebooks < <(git diff --name-only master | grep '\.ipynb$' || true) + if [[ ${#changed_notebooks[@]} == 0 ]]; then + echo "No notebooks modified in this pull request." + else + msg="

Preview

\n" + msg+="Preview and run these notebook edits with Google Colab:\n\n" + + reviewnb_url="https://app.reviewnb.com/${GITHUB_REPOSITORY}/pull/${PR_NUM}/files/" + msg+="Rendered notebook diffs available on ReviewNB.com.\n" + + msg+="

Format and style

\n" + msg+="Use the TensorFlow docs notebook tools to format for consistent source diffs and lint for style:\n" + msg+="
\n$ python3 -m pip install -U --user git+https://github.com/tensorflow/docs\n
"
+          msg+="$ python3 -m tensorflow_docs.tools.nbfmt notebook.ipynb\n
"
+          msg+="$ python3 -m tensorflow_docs.tools.nblint --arg=repo:tensorflow/docs notebook.ipynb\n
\n" + + msg+="If commits are added to the pull request, synchronize your local branch: git pull origin $HEAD_REF\n" + fi + echo "MESSAGE=$msg" >> $GITHUB_ENV + - name: Post comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ISSUE_URL: ${{ github.event.client_payload.pull_request.issue_url }} + run: | + # Env var defined in previous step. Escape string for JSON. + body="$(echo -n -e $MESSAGE | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))')" + # Add comment to pull request. + curl -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + "${ISSUE_URL}/comments" \ + --data "{\"body\": $body}" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000000..d50aa5d37bb --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,101 @@ +# Pass/fail checks for continuous integration testing. +# Webhook events: Pull requests +name: CI +on: + pull_request: + paths: + - "site/en/**" + +jobs: + nbfmt: + name: Notebook format + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 + - name: Fetch master branch + run: git fetch -u origin master:master + - name: Install tensorflow-docs + run: python3 -m pip install -U git+https://github.com/tensorflow/docs + - name: Check notebook formatting + run: | + # Only check notebooks modified in this pull request. + readarray -t changed_notebooks < <(git diff --name-only master | grep '\.ipynb$' || true) + if [[ ${#changed_notebooks[@]} == 0 ]]; then + echo "No notebooks modified in this pull request." + exit 0 + else + echo "Check formatting with nbfmt:" + python3 -m tensorflow_docs.tools.nbfmt --test "${changed_notebooks[@]}" + fi + + nblint: + name: Notebook lint + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 + - name: Fetch master branch + run: git fetch -u origin master:master + - name: Install tensorflow-docs + run: python3 -m pip install -U git+https://github.com/tensorflow/docs + - name: Lint notebooks + run: | + # Only check notebooks modified in this pull request. + readarray -t changed_notebooks < <(git diff --name-only master | grep '\.ipynb$' || true) + if [[ ${#changed_notebooks[@]} == 0 ]]; then + echo "No notebooks modified in this pull request." + exit 0 + else + echo "Lint check with nblint:" + python3 -m tensorflow_docs.tools.nblint \ + --arg=repo:tensorflow/docs "${changed_notebooks[@]}" + fi + + outputs-removed: + name: Notebook outputs removed + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Fetch master branch + run: git fetch -u origin master:master + - name: Check for output cells + run: | + # Notebooks that are allowed to save outputs and excluded from test. + EXCLUDED_PATHS=( + site/en/guide/gpu.ipynb + ) + # Only check notebooks modified in this pull request. + readarray -t changed_notebooks < <(git diff --name-only master | grep '\.ipynb$' || true) + if [[ ${#changed_notebooks[@]} == 0 ]]; then + echo "No notebooks modified in this pull request." + exit 0 + fi + # Remove notebooks excluded from test. + declare -a tested_notebooks + for fp in "${changed_notebooks[@]}"; do + is_excluded=0 + for excluded_fp in "${EXCLUDED_PATHS[@]}"; do + if [[ "$fp" == "$excluded_fp" ]]; then + is_excluded=1 + break + fi + done + if [[ $is_excluded == 0 ]]; then + tested_notebooks+=("$fp") + fi + done + # Test notebooks for output cells. + status_code=0 + for fp in "${tested_notebooks[@]}"; do + # Output cells use the "output_type" property. + if grep --quiet "\"output_type\":" "$fp"; then + echo "[${GITHUB_WORKFLOW}] Remove output cells from: $fp" >&2 + status_code=1 + fi + done + if [[ "$status_code" != 0 ]]; then + echo "To remove notebook outputs:" >&2 + echo "$ python3 -m tensorflow_docs.tools.nbfmt --remove_outputs notebook.ipynb" >&2 + fi + exit "$status_code" diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml new file mode 100644 index 00000000000..0ca76b0677e --- /dev/null +++ b/.github/workflows/stale.yaml @@ -0,0 +1,49 @@ +# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. +# +# You can adjust the behavior by modifying this file. +# For more information, see: +# https://github.com/actions/stale +name: Mark stale issues and pull requests + +on: + schedule: + # Scheduled to run at 1.30 UTC everyday + - cron: '30 1 * * *' + workflow_dispatch: + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v9 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-issue-stale: 14 + days-before-issue-close: 14 + stale-issue-label: "status:stale" + close-issue-reason: not_planned + any-of-labels: "awaiting-contributor-response,cla:no" + stale-issue-message: > + Marking this issue as stale since it has been open for 14 days with no activity. + This issue will be closed if no further activity occurs. + close-issue-message: > + This issue was closed because it has been inactive for 28 days. + Please post a new issue if you need further assistance. Thanks! + days-before-pr-stale: 14 + days-before-pr-close: 14 + stale-pr-label: "status:stale" + stale-pr-message: > + Marking this pull request as stale since it has been open for 14 days with no activity. + This PR will be closed if no further activity occurs. + close-pr-message: > + This pull request was closed because it has been inactive for 28 days. + Please open a new pull request if you need further assistance. Thanks! + # Label that can be assigned to issues to exclude them from being marked as stale + exempt-issue-labels: 'override-stale' + # Label that can be assigned to PRs to exclude them from being marked as stale + exempt-pr-labels: "override-stale" diff --git a/CODEOWNERS b/CODEOWNERS index 090999b3db1..d4d2932d8bc 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,13 +1,14 @@ # https://help.github.com/articles/about-codeowners/ -# Last matching pattern takes preecedence. +# Last matching pattern takes precedence. # Default owners for everything in repo. -* @lamberta @MarkDaoust +* @tensorflow/docs-team -# Docs -/site/en/guide/ @lamberta @MarkDaoust @yashk2810 -/site/en/tutorials/ @lamberta @MarkDaoust @yashk2810 +# Install +/site/en/install/ @haifeng-jin @MarkDaoust @8bitmp3 # Community -/site/en/community/ @ewilderj @lamberta -/site/en/community/groups.md @soonson +/site/en/community/ @ewilderj @theadactyl @joanafilipa + +# Hub +/site/en/hub @gustheman \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1559b721f51..6f301eab782 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,9 +6,7 @@ This guide shows how to make contributions to [tensorflow.org](https://www.tenso See the [TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs) -for guidance. For questions, the -[docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs) -mailing list is available. +for guidance. For questions, check out [TensorFlow Forum](https://discuss.tensorflow.org/). Questions about TensorFlow usage are better addressed on [Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow) or the diff --git a/LICENSE b/LICENSE index 4862420c023..08026f1ac8e 100644 --- a/LICENSE +++ b/LICENSE @@ -201,3 +201,28 @@ Copyright 2018 The TensorFlow Authors. All rights reserved. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + +--------------------------- + +Where indicated, some files are also distributed under the MIT License: + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000000..c3f79ccabe2 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +global-include **/templates/* +global-include *.sh \ No newline at end of file diff --git a/README.md b/README.md index b9441cad824..66b6d3fb065 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,29 @@

-This is the TensorFlow documentation for [tensorflow.org](https://www.tensorflow.org). -To contribute, see [CONTRIBUTING.md](CONTRIBUTING.md) and the -[docs contributor guide](https://www.tensorflow.org/community/contribute/docs). +These are the source files for the guide and tutorials on +[tensorflow.org](https://www.tensorflow.org/overview). -To file a docs issue, use the tracker in the +To contribute to the TensorFlow documentation, please read +[CONTRIBUTING.md](CONTRIBUTING.md), the +[TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs), +and the [style guide](https://www.tensorflow.org/community/contribute/docs_style). + +To file a docs issue, use the issue tracker in the [tensorflow/tensorflow](https://github.com/tensorflow/tensorflow/issues/new?template=20-documentation-issue.md) repo. And join the TensorFlow documentation contributors on the -[docs@tensorflow.org mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). +[TensorFlow Forum](https://discuss.tensorflow.org/). + +## Community translations + +[Community translations](https://www.tensorflow.org/community/contribute/docs#community_translations) +are located in the +[tensorflow/docs-l10n](https://github.com/tensorflow/docs-l10n) repo. These docs +are contributed, reviewed, and maintained by the community as *best-effort*. To +participate as a translator or reviewer, see the `site//README.md`, join +the language mailing list, and submit a pull request. ## License [Apache License 2.0](LICENSE) - diff --git a/setup.py b/setup.py index 54ed487c537..404479668b3 100644 --- a/setup.py +++ b/setup.py @@ -15,46 +15,51 @@ """tensorflow_docs is a package for generating python api-reference docs.""" import datetime +import subprocess import sys from setuptools import find_packages from setuptools import setup -nightly = False -if '--nightly' in sys.argv: - nightly = True - sys.argv.remove('--nightly') - project_name = 'tensorflow-docs' -version = '0.0.0' -if nightly: - project_name = 'tfds-nightly' - datestring = datetime.datetime.now().strftime('%Y%m%d%H%M') - version = '%s-dev%s' % (version, datestring) + + +def get_version() -> str: + ts = int( + subprocess.check_output(['git', 'log', '-1', '--format=%ct', 'tools']) + .decode('utf-8') + .strip() + ) + dt = datetime.datetime.utcfromtimestamp(ts) + sec = 60 * 60 * dt.hour + 60 * dt.minute + dt.second + + # calver.org + return f'{dt.year}.{dt.month}.{dt.day}.{sec}' + + +version = get_version() DOCLINES = __doc__.split('\n') REQUIRED_PKGS = [ 'astor', 'absl-py', - 'six', - 'pathlib2', + 'jinja2', + 'nbformat', + 'protobuf>=3.12', 'pyyaml', ] -VIS_REQURE = [ +VIS_REQUIRE = [ 'numpy', 'PILLOW', 'webp', ] -if sys.version_info < (3, 4): - # enum introduced in Python 3.4 - REQUIRED_PKGS.append('enum34') - # https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords setup( name=project_name, + python_requires='>=3.9', version=version, description=DOCLINES[0], long_description='\n'.join(DOCLINES[2:]), @@ -67,7 +72,7 @@ package_dir={'': 'tools'}, scripts=[], install_requires=REQUIRED_PKGS, - extras_require={'vis': VIS_REQURE}, + extras_require={'vis': VIS_REQUIRE}, classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', @@ -75,4 +80,7 @@ 'Topic :: Scientific/Engineering :: Artificial Intelligence', ], keywords='tensorflow api reference', + # Include_package_data is required for setup.py to recognize the MANIFEST.in + # https://python-packaging.readthedocs.io/en/latest/non-code-files.html + include_package_data=True, ) diff --git a/site/README.md b/site/README.md deleted file mode 100644 index c3ce21564cb..00000000000 --- a/site/README.md +++ /dev/null @@ -1,12 +0,0 @@ -These are the source files for the guides and tutorials on -[tensorflow.org](https://www.tensorflow.org). - -The [en/](./en) directory is the *source-of-truth*. Read the -[documentation contributor guide](https://www.tensorflow.org/community/contribute/docs) -for guidance. - -Additional language directories are -[community translations](https://www.tensorflow.org/community/contribute/docs#community_translations) -contributed by the community and maintained as *best-effort*. To participate as -a translator or reviewer, see the `site//README.md`, join the language -mailing list, and submit a pull request. diff --git a/site/en/README.md b/site/en/README.md new file mode 100644 index 00000000000..28dc0cce7d4 --- /dev/null +++ b/site/en/README.md @@ -0,0 +1,50 @@ +# TensorFlow docs + +These are the source files for the core TensorFlow +[guide](https://www.tensorflow.org/guide), +[tutorials](https://www.tensorflow.org/tutorials), and other technical docs. +Please read the +[contributor guide](https://www.tensorflow.org/community/contribute) +to submit patches to the TensorFlow documentation and code. + +## TensorFlow ecosystem projects + +In addition to the core TensorFlow docs, +[tensorflow.org](https://www.tensorflow.org) hosts documentation for many +[libraries and extensions](https://www.tensorflow.org/resources/libraries-extensions). +These docs are sourced from separate project repos and where pull requests can +be sent. The following is a list of TensorFlow documentation projects published +on the website and a link to their source files: + +tensorflow.org project | GitHub docs location +-----------------------|--------------------- +[/addons](https://www.tensorflow.org/addons) | https://github.com/tensorflow/addons/tree/master/docs +[/agents](https://www.tensorflow.org/agents) | https://github.com/tensorflow/agents/tree/master/docs +[/cloud](https://www.tensorflow.org/cloud) | https://github.com/tensorflow/cloud/tree/master/g3doc +[/datasets](https://www.tensorflow.org/datasets) | https://github.com/tensorflow/datasets/tree/master/docs +[/decision_forests](https://www.tensorflow.org/decision_forests) | https://github.com/tensorflow/decision-forests/tree/main/documentation +[/federated](https://www.tensorflow.org/federated) | https://github.com/tensorflow/federated/tree/main/docs +[/graphics](https://www.tensorflow.org/graphics) | https://github.com/tensorflow/graphics/tree/master/tensorflow_graphics/g3doc +[/hub](https://www.tensorflow.org/hub) | https://github.com/tensorflow/hub/tree/master/docs +[/io](https://www.tensorflow.org/io) | https://github.com/tensorflow/io/tree/master/docs/ +[/js](https://www.tensorflow.org/js) | https://github.com/tensorflow/tfjs-website/tree/master/docs +[/jvm](https://www.tensorflow.org/jvm) | https://github.com/tensorflow/java/tree/master/docs +[/lattice](https://www.tensorflow.org/lattice) | https://github.com/tensorflow/lattice/tree/master/docs +[/lite](https://www.tensorflow.org/lite) | https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/g3doc +[/mlir](https://www.tensorflow.org/mlir) | https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/mlir/g3doc +[/model_optimization](https://www.tensorflow.org/model_optimization) | https://github.com/tensorflow/model-optimization/tree/master/tensorflow_model_optimization/g3doc +[/neural_structured_learning](https://www.tensorflow.org/neural_structured_learning) | https://github.com/tensorflow/neural-structured-learning/tree/master/g3doc +[/probability](https://www.tensorflow.org/probability) | https://github.com/tensorflow/probability/tree/main/tensorflow_probability/g3doc +[/quantum](https://www.tensorflow.org/quantum) | https://github.com/tensorflow/quantum/tree/master/docs +[/ranking](https://www.tensorflow.org/ranking) | https://github.com/tensorflow/ranking/tree/master/docs +[/recommenders](https://www.tensorflow.org/recommenders) | https://github.com/tensorflow/recommenders/tree/main/docs +[/responsible_ai/fairness_indicators](https://www.tensorflow.org/responsible_ai/fairness_indicators/guide) | https://github.com/tensorflow/fairness-indicators/tree/master/g3doc +[/responsible_ai/model_card_toolkit](https://www.tensorflow.org/responsible_ai/model_card_toolkit/guide) | https://github.com/tensorflow/model-card-toolkit/tree/main/model_card_toolkit/documentation +[/responsible_ai/model_remediation](https://www.tensorflow.org/responsible_ai/model_remediation) | https://github.com/tensorflow/model-remediation/tree/master/docs +[/responsible_ai/privacy](https://www.tensorflow.org/responsible_ai/privacy/guide) | https://github.com/tensorflow/privacy/tree/master/g3doc +[/tensorboard](https://www.tensorflow.org/tensorboard) | https://github.com/tensorflow/tensorboard/tree/master/docs +[/guide/keras](https://www.tensorflow.org/guide/keras/) | https://github.com/keras-team/keras-io/tree/master/guides +[/text](https://www.tensorflow.org/text) | https://github.com/tensorflow/text/tree/master/docs +[/tfx](https://www.tensorflow.org/tfx) | https://github.com/tensorflow/tfx/tree/master/docs +[/tfx/guide/serving](https://www.tensorflow.org/tfx/guide/serving) | https://github.com/tensorflow/serving/tree/master/tensorflow_serving/g3doc +[/xla](https://www.tensorflow.org/xla) | https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/xla/g3doc diff --git a/site/en/about/_menu_toc.yaml b/site/en/about/_menu_toc.yaml index 61a4035fe17..8f34ca8089d 100644 --- a/site/en/about/_menu_toc.yaml +++ b/site/en/about/_menu_toc.yaml @@ -5,5 +5,3 @@ toc: path: /about/ - label: "Case studies" path: /about/case-studies/ - - label: "Trusted Partner Program" - path: /trusted-partners/ diff --git a/site/en/about/bib.md b/site/en/about/bib.md index 031b5bc3983..16da75adc3e 100644 --- a/site/en/about/bib.md +++ b/site/en/about/bib.md @@ -1,6 +1,9 @@ -# TensorFlow White Papers +# Citing TensorFlow -This document identifies white papers about TensorFlow. +TensorFlow publishes a DOI for the open-source code base using Zenodo.org: +[10.5281/zenodo.4724125](https://doi.org/10.5281/zenodo.4724125) + +TensorFlow's white papers are listed for citation below. ## Large-Scale Machine Learning on Heterogeneous Distributed Systems @@ -40,7 +43,7 @@ title={ {TensorFlow}: Large-Scale Machine Learning on Heterogeneous Systems}, url={https://www.tensorflow.org/}, note={Software available from tensorflow.org}, author={ - Mart\'{\i}n~Abadi and + Mart\'{i}n~Abadi and Ashish~Agarwal and Paul~Barham and Eugene~Brevdo and @@ -113,7 +116,7 @@ uses dataflow graphs to represent computation, shared state, and the operations that mutate that state. It maps the nodes of a dataflow graph across many machines in a cluster, and within a machine across multiple computational -devices, including multicore CPUs, generalpurpose +devices, including multicore CPUs, general purpose GPUs, and custom-designed ASICs known as Tensor Processing Units (TPUs). This architecture gives flexibility to the application developer: whereas in previous diff --git a/site/en/api_docs/_book.yaml b/site/en/api_docs/_book.yaml deleted file mode 100644 index 1704826f0c2..00000000000 --- a/site/en/api_docs/_book.yaml +++ /dev/null @@ -1,109 +0,0 @@ -upper_tabs: -- include: /_upper_tabs_left.yaml - -- name: API - path: /api - menu: - - include: /api_docs/_menu_toc.yaml - lower_tabs: - other: - - name: Overview - contents: - - title: Overview - path: {VERSION_ROOT}/ - - - heading: Implementations - - title: "TensorFlow.js" - path: https://js.tensorflow.org/api/latest/ - - title: "Swift" - path: /swift/api_docs/ - - title: "Go" - path: https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go - status: external - - title: "Haskell" - path: https://github.com/tensorflow/haskell - status: external - - title: Community Supported - section: - - title: "C#" - path: https://github.com/migueldeicaza/TensorFlowSharp - - title: "Julia" - path: https://github.com/malmaud/TensorFlow.jl - status: external - - title: "Ruby" - path: https://github.com/somaticio/tensorflow.rb - status: external - - title: "Rust" - path: https://github.com/tensorflow/rust - status: external - - title: "Scala" - path: https://github.com/eaplatanios/tensorflow_scala - status: external - - - heading: "Resources" - - title: "Addons" - path: /addons/api_docs/python/tfa - - title: "Datasets" - path: /datasets/api_docs/python/tfds - - title: "Federated" - path: /federated/api_docs/python/tff - - title: "Graphics" - path: /graphics/api_docs/python/tgf - - title: "Hub" - path: /hub/api_docs/python/hub - - title: "Model Optimization" - path: /model_optimization/api_docs/python/tfmot - - title: "Neural Structured Learning" - path: /neural_structured_learning/api_docs/python/nsl - - title: "Probability" - path: /probability/api_docs/python/tfp - - - heading: "Lite" - - title: "Python" - path: /api_docs/python/tf/lite - - title: "C++" - path: /lite/api_docs/cc - - - heading: "TFX" - - title: "TFX" - path: /tfx/api_docs/python/tfx - - title: "Data Validation" - path: /tfx/data_validation/api_docs/python/tfdv - - title: "Model Analysis" - path: /tfx/model_analysis/api_docs/python/tfma - - title: "Transform" - path: /tfx/transform/api_docs/python/tft - - title: Serving - path: /tfx/serving/api_docs/cc - - - name: Python - skip_translation: true - contents: - - title: Overview - path: {VERSION_ROOT}/python/tf - - title: All Symbols - path: {VERSION_ROOT}/python/ - - title: Python {VERSION} - - include: {VERSION_ROOT}/python/_toc.yaml - - - name: JavaScript - contents: - - title: JavaScript - path: https://js.tensorflow.org/api/latest/ - - - name: C++ - skip_translation: true - contents: - - title: C++ - path: {VERSION_ROOT}/cc/ - - include: {VERSION_ROOT}/cc/_doxygen.yaml - - - name: Java - skip_translation: true - contents: - - title: Java - path: {VERSION_ROOT}/java/reference/org/tensorflow/package-summary.html - - include: {VERSION_ROOT}/java/reference/_toc.yaml - -- include: /resources/_upper_tabs_resources.yaml -- include: /_upper_tabs_right.yaml diff --git a/site/en/api_docs/_menu_toc.yaml b/site/en/api_docs/_menu_toc.yaml deleted file mode 100644 index 9127ce79062..00000000000 --- a/site/en/api_docs/_menu_toc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -toc: -- column: - - heading: API - links: - - label: r2.0 (stable) - path: /api/stable - - label: r2.1 (rc) - path: /api/r2.1 -- column: - - heading: API r1 - links: - - label: r1.15 - path: /api/r1.15 - - label: More… - path: /versions/ diff --git a/site/en/api_docs/_project.yaml b/site/en/api_docs/_project.yaml deleted file mode 100644 index b28d9a13682..00000000000 --- a/site/en/api_docs/_project.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: TensorFlow Core {VERSION} -home_url: {VERSION_ROOT} -parent_project_metadata_path: /_project.yaml -description: > - An open source machine learning library for research and production. -use_site_branding: true -hide_from_products_list: true -content_license: cc-apache -buganizer_id: 162698 -include: /_project_included.yaml diff --git a/site/en/api_docs/_upper_tabs_api.yaml b/site/en/api_docs/_upper_tabs_api.yaml deleted file mode 100644 index e5675096ddd..00000000000 --- a/site/en/api_docs/_upper_tabs_api.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Upper tab dropdown menu used across site. -toc: -- name: API - path: /api - skip_translation: true - menu: - - include: /api_docs/_menu_toc.yaml - lower_tabs: - guides: - # should not display - - title: Versions - path: /versions diff --git a/site/en/api_docs/index.md b/site/en/api_docs/index.md deleted file mode 100644 index 25f4bcf5999..00000000000 --- a/site/en/api_docs/index.md +++ /dev/null @@ -1,33 +0,0 @@ -# API Documentation - -TensorFlow has APIs available in several languages both for constructing and -executing a TensorFlow graph. The Python API is at present the most complete -and the easiest to use, but other language APIs may be easier to integrate -into projects and may offer some performance advantages in graph execution. - -A word of caution: the APIs in languages other than Python are not yet -covered by the [API stability promises](https://www.tensorflow.org/guide/versions). - -* [Python](https://www.tensorflow.org/api_docs/python/) -* [JavaScript](https://js.tensorflow.org/api/latest/) -* [C++](https://www.tensorflow.org/api_docs/cc/) -* [Java](https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/package-summary) -* [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go) -* [Swift (Early Release)](https://www.tensorflow.org/swift) - - -We encourage the community to develop and maintain support for other languages -with the [approach recommended by the TensorFlow maintainers](https://github.com/tensorflow/docs/blob/master/site/en/r1/guide/extend/bindings.md). -For example, see the bindings for: - -* C#: [TensorFlowSharp](https://github.com/migueldeicaza/TensorFlowSharp) and [TensorFlow.NET](https://github.com/SciSharp/TensorFlow.NET), -* [Haskell](https://github.com/tensorflow/haskell), -* [Julia](https://github.com/malmaud/TensorFlow.jl), -* [MATLAB](https://github.com/asteinh/tensorflow.m), -* [Ruby](https://github.com/somaticio/tensorflow.rb), -* [Rust](https://github.com/tensorflow/rust), and -* [Scala](https://github.com/eaplatanios/tensorflow_scala). - -We also provide the C++ API reference for TensorFlow Serving: - -* [TensorFlow Serving](https://www.tensorflow.org/tfx/guide/serving) diff --git a/site/en/community/_toc.yaml b/site/en/community/_toc.yaml index 31fd56b4ea7..1a81d38cb01 100644 --- a/site/en/community/_toc.yaml +++ b/site/en/community/_toc.yaml @@ -1,16 +1,14 @@ toc: -- name: "Forums & user groups" +- name: "Groups" contents: - - title: "Forums" - path: /community/forums - title: "User groups" path: /community/groups + - title: "Mailing lists" + path: /community/mailing-lists - heading: "Community resources" - title: "SIG playbook" path: /community/sig_playbook - - title: "Roadmap" - path: /community/roadmap - name: "Contribute" contents: @@ -20,6 +18,8 @@ toc: - heading: "Code" - title: "Contribute to code" path: /community/contribute/code + - title: "Contribute tests" + path: /community/contribute/tests - title: "Code style" path: /community/contribute/code_style @@ -34,5 +34,7 @@ toc: - heading: "Community" - title: "Contribute to the community" path: /community/contribute/community + - title: "Contribute to SIGs" + path: /community/contribute/sigs - title: "RFC process" path: /community/contribute/rfc_process diff --git a/site/en/community/contribute/code.md b/site/en/community/contribute/code.md index 9f33166f121..2f71f12d7fe 100644 --- a/site/en/community/contribute/code.md +++ b/site/en/community/contribute/code.md @@ -2,31 +2,50 @@ Whether you are adding a loss function, improving test coverage, or writing an RFC for a major design change, this portion of the contributor guide will help -you get started. Thank you for work and interest in improving TensorFlow. +you get started. Thank you for your work and interest in improving TensorFlow. ## Before you get started -Before you contribute source code to a TensorFlow project, please review the `CONTRIBUTING.md` file in the GitHub repo of the project. (For example, see the -[CONTRIBUTING.md file for the core TensorFlow repo](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md).) All code contributors are required to sign a [Contributor License Agreement](https://cla.developers.google.com/clas) (CLA). - -To avoid duplicating work, please review [current RFCs](https://github.com/tensorflow/community/tree/master/rfcs) and contact the developers on the TensorFlow forums before you start work on a non-trivial feature. We are somewhat selective when deciding to add new functionality, and the best way to contribute and help the project is to work on known issues. +Before you contribute source code to a TensorFlow project, please review the +`CONTRIBUTING.md` file in the GitHub repo of the project. For example, see the +[CONTRIBUTING.md](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md) +file in the core TensorFlow repo. All code contributors are required to sign a +[Contributor License Agreement](https://cla.developers.google.com/clas) (CLA). + +To avoid duplicating work, please review +[current](https://github.com/tensorflow/community/tree/master/rfcs) or +[proposed](https://github.com/tensorflow/community/labels/RFC%3A%20Proposed) +RFCs and contact the developers on the TensorFlow forums +([developers@tensorflow.org](https://groups.google.com/u/1/a/tensorflow.org/g/developers)) +before you start work on a non-trivial feature. We are somewhat selective when +deciding to add new functionality, and the best way to contribute and help the +project is to work on known issues. ## Issues for new contributors -New contributors should look for the following tags when searching for a first contribution to the TensorFlow code base. We strongly recommend that new contributors tackle “easy” and "good first issue" projects first; this helps the contributor become familiar with the contribution workflow, and for the core devs to become acquainted with the contributor. - -- `good first issue` -- `easy` -- `contributions welcome` +New contributors should look for the following tags when searching for a first +contribution to the TensorFlow code base. We strongly recommend that new +contributors tackle “good first issue” and "contributions welcome" projects +first; this helps the contributor become familiar with the contribution +workflow, and for the core devs to become acquainted with the contributor. -If you are interested in recruiting a team to help tackle a large-scale problem or a new feature, please email the [developers@ group](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers) and review our current list of RFCs. +- [good first issue](https://github.com/tensorflow/tensorflow/labels/good%20first%20issue) +- [contributions welcome](https://github.com/tensorflow/tensorflow/labels/stat%3Acontributions%20welcome) +If you are interested in recruiting a team to help tackle a large-scale problem +or a new feature, please email the +[developers@ group](https://groups.google.com/a/tensorflow.org/g/developers) +and review our current list of RFCs. ## Code review -New features, bug fixes, and any other changes to the code base are subject to code review. +New features, bug fixes, and any other changes to the code base are subject to +code review. -Reviewing code contributed to the project as pull requests is a crucial component of TensorFlow development. We encourage anyone to start reviewing code submitted by other developers, especially if the feature is something that you are likely to use. +Reviewing code contributed to the project as pull requests is a crucial +component of TensorFlow development. We encourage anyone to start reviewing code +submitted by other developers, especially if the feature is something that you +are likely to use. Here are some questions to keep in mind during the code review process: @@ -41,36 +60,47 @@ Here are some questions to keep in mind during the code review process: ## Test and improve test coverage -High-quality unit testing is a corner-stone of the TensorFlow development process. For this purpose, we use Docker images. The test functions are appropriately named, and are responsible for checking the validity of algorithms as well as different options of the code. +High-quality unit testing is a corner-stone of the TensorFlow development +process. For this purpose, we use Docker images. The test functions are +appropriately named, and are responsible for checking the validity of algorithms +as well as different options of the code. -All new features and bug fixes *must* include adequate test coverage. We also welcome contributions of new test cases or improvements to existing tests. If you discover that our existing tests are not complete — even if that is not currently causing a bug — please file an issue and, if possible, a pull request. +All new features and bug fixes *must* include adequate test coverage. We also +welcome contributions of new test cases or improvements to existing tests. If +you discover that our existing tests are not complete — even if that is not +currently causing a bug — please file an issue and, if possible, a pull request. -For the specific details of testing procedures in each TensorFlow project, see the `README.md` and `CONTRIBUTING.md` files in the project repo on GitHub. +For the specific details of testing procedures in each TensorFlow project, see +the `README.md` and `CONTRIBUTING.md` files in the project repo on GitHub. Of particular concerns in *adequate testing*: -* Is *every public function and class* tested? -* Are a *reasonable set of parameters*, their values, value types, and combinations tested? -* Do the tests validate that the *code is correct*, and that it is *doing what the documentation says* the code is intended to do? +* Is *every public function and class* tested? +* Are a *reasonable set of parameters*, their values, value types, and + combinations tested? +* Do the tests validate that the *code is correct*, and that it is *doing what + the documentation says* the code is intended to do? * If the change is a bug fix, is a *non-regression test* included? * Do the tests *pass the continuous integration* build? -* Do the tests *cover every line of code?* If not, are the exceptions reasonable and explicit? - -If you find any problems, please consider helping the contributor understand those problems and resolve them. +* Do the tests *cover every line of code?* If not, are the exceptions + reasonable and explicit? +If you find any problems, please consider helping the contributor understand +those problems and resolve them. ## Improve error messages or logs -We welcome contributions that improve error messages and logging. - +We welcome contributions that improve error messages and logging. ## Contribution workflow -Code contributions—bug fixes, new development, test improvement—all follow a GitHub-centered workflow. To participate in TensorFlow development, set up a GitHub account. Then: +Code contributions—bug fixes, new development, test improvement—all follow a +GitHub-centered workflow. To participate in TensorFlow development, set up a +GitHub account. Then: -1. Fork the repo you plan to work on. - Go to the project repo page and use the *Fork* button. This will create a copy of the - repo, under your username. (For more details on how to fork a repository see +1. Fork the repo you plan to work on. Go to the project repo page and use the + *Fork* button. This will create a copy of the repo, under your username. + (For more details on how to fork a repository see [this guide](https://help.github.com/articles/fork-a-repo/).) 2. Clone down the repo to your local system. @@ -85,7 +115,7 @@ Code contributions—bug fixes, new development, test improvement—all follow a 5. Commit your changes. - `$ git add -a` + `$ git add -A` `$ git commit -m "commit message here"` @@ -93,44 +123,54 @@ Code contributions—bug fixes, new development, test improvement—all follow a `$ git push origin branch-name` -7. Open a *Pull Request* (PR). Go to the original project repo on GitHub. There will be a message about your recently pushed branch, asking if you would like to open a pull request. Follow the prompts, *compare across repositories*, and submit the PR. This will send an email to the committers. You may want to consider sending an email to the mailing list for more visibility. (For more details, see the [GitHub guide on PRs](https://help.github.com/articles/creating-a-pull-request-from-a-fork). +7. Open a *Pull Request* (PR). Go to the original project repo on GitHub. There + will be a message about your recently pushed branch, asking if you would + like to open a pull request. Follow the prompts, *compare across + repositories*, and submit the PR. This will send an email to the committers. + You may want to consider sending an email to the mailing list for more + visibility. (For more details, see the + [GitHub guide on PRs](https://help.github.com/articles/creating-a-pull-request-from-a-fork). -8. Maintainers and other contributors will *review your PR*. Please participate in the conversation, and try to *make any requested changes*. Once the PR is approved, the code will be merged. +8. Maintainers and other contributors will *review your PR*. Please participate + in the conversation, and try to *make any requested changes*. Once the PR is + approved, the code will be merged. -*Before working on your next contribution*, make sure your local repository is up to date. +*Before working on your next contribution*, make sure your local repository is +up to date. -1. Set the upstream remote. (You only have to do this once per project, not every time.) +1. Set the upstream remote. (You only have to do this once per project, not + every time.) `$ git remote add upstream git@github.com:tensorflow/project-repo-name` -2. Switch to the local master branch. +2. Switch to the local master branch. `$ git checkout master` -3. Pull down the changes from upstream. +3. Pull down the changes from upstream. `$ git pull upstream master` -4. Push the changes to your GitHub account. (Optional, but a good practice.) +4. Push the changes to your GitHub account. (Optional, but a good practice.) `$ git push origin master` -5. Create a new branch if you are starting new work. +5. Create a new branch if you are starting new work. `$ git checkout -b branch-name` Additional `git` and GitHub resources: * [Git documentation](https://git-scm.com/documentation) -* [Git development workflow](https://docs.scipy.org/doc/numpy/dev/gitwash/development_workflow.html) +* [Git development workflow](https://docs.scipy.org/doc/numpy/dev/development_workflow.html) * [Resolving merge conflicts](https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line/). ## Contributor checklist -* Read contributing guidelines. -* Read the Code of Conduct. -* Ensure you have signed the Contributor License Agreement (CLA). -* Check if your changes are consistent with the guidelines. -* Check if your changes are consistent with the TensorFlow coding style. -* Run unit tests. +* Read the [contributing guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md). +* Read the [Code of Conduct](https://github.com/tensorflow/tensorflow/blob/master/CODE_OF_CONDUCT.md). +* Ensure you have signed the [Contributor License Agreement (CLA)](https://cla.developers.google.com/). +* Check if your changes are consistent with the [guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md#general-guidelines-and-philosophy-for-contribution). +* Check if your changes are consistent with the [TensorFlow coding style](https://www.tensorflow.org/community/contribute/code_style). +* [Run the unit tests](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md#running-unit-tests). diff --git a/site/en/community/contribute/code_style.md b/site/en/community/contribute/code_style.md index 833e2178900..74e3194ca33 100644 --- a/site/en/community/contribute/code_style.md +++ b/site/en/community/contribute/code_style.md @@ -11,63 +11,40 @@ and use [pylint](https://www.pylint.org/) to check your Python changes. ### pylint -To install `pylint` and retrieve TensorFlow's custom style definition: +To install `pylint`: ```bash - $ pip install pylint -$ wget -O /tmp/pylintrc https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/tools/ci_build/pylintrc - ``` -To check a file with `pylint`: +To check a file with `pylint` from the TensorFlow source code root directory: ```bash -$ pylint --rcfile=/tmp/pylintrc myfile.py +$ pylint --rcfile=tensorflow/tools/ci_build/pylintrc tensorflow/python/keras/losses.py ``` ### Supported Python versions -TensorFlow supports Python 2.7 and Python >= 3.4. See the -[installation guide](https://www.tensorflow.org/install) for details. +For supported Python versions, see the TensorFlow +[installation guide](https://www.tensorflow.org/install). See the TensorFlow [continuous build status](https://github.com/tensorflow/tensorflow/blob/master/README.md#continuous-build-status) for official and community supported builds. -#### Legacy Python compatibility - -TensorFlow will support Legacy Python (Python 2.7) until -[January 1, 2020](https://groups.google.com/a/tensorflow.org/forum/#!searchin/announce/python$202.7%7Csort:date/announce/gVwS5RC8mds/dCt1ka2XAAAJ). -Until that time, all code will need to be compatible with the Python versions -listed above. - -These lines should be present in every Python file: - - -```python -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -``` - -Use `six` to write compatible code (for example, `six.moves.range`). - ## C++ coding style Changes to TensorFlow C++ code should conform to the [Google C++ Style -Guide](https://google.github.io/styleguide/cppguide.html). Use `clang-tidy` to -check your C/C++ changes. - -To install `clang-tidy` on Ubuntu 16+, do: +Guide](https://google.github.io/styleguide/cppguide.html) and [TensorFlow specific style details](https://github.com/tensorflow/community/blob/master/governance/cpp-style.md). Use `clang-format` to check your C/C++ changes. +To install on Ubuntu 16+, do: ```bash -$ apt-get install -y clang-tidy +$ apt-get install -y clang-format ``` -You can check a C/C++ file by using the following: +You can check the format of a C/C++ file with the following: ```bash $ clang-format --style=google > /tmp/my_cc_file.cc @@ -86,23 +63,10 @@ $ diff /tmp/my_cc_file.cc ## TensorFlow conventions and special uses -### Tensors - -* Operations that deal with batches may assume that the **first dimension** of - a Tensor is the batch dimension. -* In most models, the **last dimension** is the number of _channels_. -* Dimensions excluding the first and last usually make up the _space_ - dimensions: sequence-length, or image-size. -* When possible, use a Tensor's overloaded operators rather than TensorFlow - functions. For example, we prefer `**`, `+`, `/`, `*`, `-`, `and/or` over - `tf.pow`, `tf.add`, `tf.divide`, `tf.multiply`, `tf.subtract`, and `tf.logical_*` — - unless a specific name for the operation is desired. - - ### Python operations -A _Python operation_ is a function that, given input tensors and parameters, -creates a part of the graph and returns output tensors. +A TensorFlow _operation_ is a function that, given input tensors returns output +tensors (or adds an op to a graph when building graphs). * The first argument should be tensors, followed by basic Python parameters. The last argument is `name` with a default value of `None`. diff --git a/site/en/community/contribute/community.md b/site/en/community/contribute/community.md index b2c589c4665..bb2b2035d1b 100644 --- a/site/en/community/contribute/community.md +++ b/site/en/community/contribute/community.md @@ -2,12 +2,28 @@ An open source project isn't just about the code, it's also about the community of users, developers, writers, researchers, and other contributors. You can help grow and support this community. +Please read the TensorFlow [Code and Collaboration governance](https://github.com/tensorflow/community/blob/master/governance/code-and-collaboration.md). + ## Community support -Many people [ask questions about TensorFlow on Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow). Answering those questions and pointing people to the relevant documentation is a great service to the community. +Many people [ask questions about TensorFlow on the TensorFlow Forum](https://discuss.tensorflow.org/). Answering those questions and pointing people to the relevant documentation is a great service to the community. Some users also ask support questions as GitHub issues. We try to discourage this, as GitHub issues are not the best place to ask for technical support. However, if you notice these issues, you are encouraged to answer them and point people to the relevant documentation. +### TensorFlow Forum + +The [TensorFlow Forum](https://discuss.tensorflow.org/) is a central platform for community discussion and support. It brings our community together to share ideas, best practices and use cases related to TensorFlow. We foster an open and welcoming environment according to the [TensorFlow Code of Conduct](https://discuss.tensorflow.org/faq). + +The TensorFlow Forum is organized by categories, subcategories and tags. We encourage you to create an account and follow categories and tags of interest. When you create a new post, select the most appropriate [category or subcategory](https://discuss.tensorflow.org/categories) and [tags](https://discuss.tensorflow.org/tags) to help other users find your topic. + +For more information on Discourse features, read the [Discourse New User Guide](https://meta.discourse.org/t/discourse-new-user-guide/96331). + +### Become a Forum expert + +Discourse uses [trust levels](https://blog.discourse.org/2018/06/understanding-discourse-trust-levels/) to reward increasing levels of participation in the forum. The Forum facilitates learning by doing, letting you to collect [badges](https://discuss.tensorflow.org/badges) that are displayed on your profile. This is a great way to be recognized for helping fellow community members. The more you invest in helping community members, the more badges and forum tools you will unlock. + +Certain groups, such as TensorFlow Team members and Machine Learning GDEs, display a special icon for easier identification. + ## Communication The TensorFlow community has a number of formal and informal ways of keeping in touch. @@ -16,99 +32,33 @@ The TensorFlow community has a number of formal and informal ways of keeping in The primary communication about work on TensorFlow happens in the [TensorFlow repositories on GitHub](https://github.com/tensorflow). This is the place to discuss bugs, new features, and in-progress work. - - ### Mailing lists -Mailing lists are reserved for announcements and contributor conversation. They are not intended to provide technical support. - -#### General TensorFlow lists +Most communication happens on the TensorFlow Forum. The following mailing lists are still used for announcements and contributor conversations. Note that they are not intended to provide technical support. * [announce@tensorflow.org](mailto:announce@tensorflow.org) — All major releases and important announcements are sent to this mailing group. We recommend that you join this list if you depend on TensorFlow in any way. -* [discuss@tensorflow.org](mailto:discuss@tensorflow.org) — General discussion about TensorFlow development and direction. * [developers@tensorflow.org](mailto:developers@tensorflow.org) — Discussion for developers who are contributing to TensorFlow. +For more information on project-specific communication, visit the [Contribute to SIGs](https://tensorflow.org/community/contribute/sigs) page. + +### Blog and social media + +The [TensorFlow Blog](http://blog.tensorflow.org/) is full of great content both from our team at Google and the broader community. We'd love to hear what you have to say, so if you would like to submit an article for review, please contact us at tensorflow-blog@google.com. Note that we receive many great submissions, and setting expectations, we can only publish a few. + +On [Twitter](https://twitter.com/tensorflow) we share the latest and greatest from our community, and our [YouTube channel](https://www.youtube.com/tensorflow) has free educational content to help you create, understand and deploy models for a variety of applications. + +## TensorFlow Community Spotlight + +The TensorFlow Community Spotlight Program provides an opportunity to showcase your passion projects using TensorFlow. [Submit your project](https://services.google.com/fb/forms/tensorflowprojectrecognitionform/) for a chance to be featured and recognized on TensorFlow’s Twitter account. + +Follow the [#TFCommunitySpotlight](https://twitter.com/hashtag/TFCommunitySpotlight?src=hashtag_click) hashtag and find out more about past winners [here](https://blog.tensorflow.org/2020/11/tensorflow-community-spotlight-program-update.html). + +## User groups + +[TensorFlow User Groups](https://www.tensorflow.org/community/groups) (or TFUGs, for short) are local communities of developers and researchers around the world. If you don’t have a TFUG in your country or city, we encourage you to start one by reaching out to [tfug-help@tensorflow.org](mailto:tfug-help@tensorflow.org). + +## Events + +The TensorFlow team hosts and supports events all around the world! If your TFUG is planning an upcoming event or meetup, please let our Community know by posting about it on the TensorFlow Forum under the [Events category](https://discuss.tensorflow.org/c/events/27). -#### Project-specific lists - -* [docs@tensorflow.org](mailto:docs@tensorflow.org) — If you are interested in contributing to the TensorFlow documentation, join this mailing list. -* [hub@tensorflow.org](mailto:hub@tensorflow.org) — Discussion and collaboration around TensorFlow Hub. -* [magenta-discuss@tensorflow.org](mailto:magenta-discuss@tensorflow.org) — General discussion about Magenta development and direction. -* [swift@tensorflow.org](mailto:swift@tensorflow.org) — Community and collaboration around Swift for TensorFlow. -* [tensor2tensor@tensorflow.org](mailto:tensor2tensor@tensorflow.org) — Discussion and peer support for Tensor2Tensor. -* [tfjs-announce@tensorflow.org](mailto:tfjs-announce@tensorflow.org) — Announcements of new TensorFlow.js releases. -* [tfjs@tensorflow.org](mailto:tfjs@tensorflow.org) — Discussion and peer support for TensorFlow.js. -* [tflite@tensorflow.org](mailto:tflite@tensorflow.org) — Discussion and peer support for TensorFlow Lite. -* [tfprobability@tensorflow.org](mailto:tfprobability@tensorflow.org) — Discussion and peer support for TensorFlow Probability. -* [tpu-users@tensorflow.org](mailto:tpu-users@tensorflow.org) — Community discussion and support for TPU users. - - -### Blog - -We post regularly to the [TensorFlow Blog](http://blog.tensorflow.org/), with content sourced from both TensorFlow developers and the broader community. If you would like to submit an article for review, please contact the TensorFlow Developer Relations team. - -### Social media - -For news and updates from around the universe of TensorFlow projects, follow [@tensorflow](https://twitter.com/tensorflow) on Twitter. To watch TensorFlow-related content, check out our [YouTube](http://youtube.com/tensorflow/) channel. - -### Development roadmap - -The [Roadmap](https://www.tensorflow.org/community/roadmap) summarizes plans for upcoming additions to TensorFlow. - -### User groups - -TensorFlow has many communities all over the world! For a complete listing, please refer to the [Community](https://www.tensorflow.org/community/groups) section on the TensorFlow website. - -### Special Interest Groups (SIGs) - -To enable focused collaboration on particular areas of TensorFlow, we host Special Interest Groups (SIGs). SIGs do their work in public. If you want to join and contribute, review the work of the group, and get in touch with the relevant SIG leader. Membership policies vary on a per-SIG basis. - - -#### Current SIGs - -As of January 2019, the current TF-SIGs are: - - - - - - - - - - - - - - - - - - - - - - - - - - -
SIG Addons - Maintains a repository of contributions that conform to well-established API patterns, but implement new functionality not available in core TensorFlow. -
SIG Build - Focuses on issues surrounding building, packaging, and distribution of TensorFlow. -
SIG IO - Works on support for file systems and formats other than those in core TensorFlow (such as Apache Ignite FS, or Apache Hadoop SequenceFile), as subclasses of tf.data.Dataset and TensorFlow filesystems. -
SIG Networking - Maintains network fabrics and protocols not available in core TensorFlow. -
SIG TensorBoard - Furthers the development and direction of TensorBoard and its plug-ins. -
SIG Rust - Collaborates on the development of TensorFlow's Rust bindings. -
- -If you believe there is a strong need for a new SIG, -please read the [SIG playbook]() and get in touch with -the TensorFlow Developer Relations Team. +If you’ve already held your event, please share feedback with us [here](https://docs.google.com/forms/d/e/1FAIpQLSdvb8c2ZFXxS05aX6dpUVZlfYA0WsFFq-sUAzjiohVKAQ1RLw/viewform)! Feel free to share recaps and recordings on the Forum as well. diff --git a/site/en/community/contribute/docs.md b/site/en/community/contribute/docs.md index 56e7976a2e0..34b1619ca5d 100644 --- a/site/en/community/contribute/docs.md +++ b/site/en/community/contribute/docs.md @@ -24,13 +24,15 @@ To participate in the TensorFlow docs community: * Watch the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -* Subscribe to [docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). -* Join the [Gitter chat room](https://gitter.im/tensorflow/docs). +* Follow the [docs](https://discuss.tensorflow.org/tag/docs) tag on the + [TensorFlow Forum](https://discuss.tensorflow.org/). + ## API reference -To update reference documentation, find the -[source file](https://www.tensorflow.org/code/tensorflow/python/) +For details, use the [TensorFlow API docs contributor guide](docs_ref.md). This +shows you how to find the +[source file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/) and edit the symbol's docstring. Many API reference pages on tensorflow.org include a link to the source file @@ -39,9 +41,6 @@ where the symbol is defined. Docstrings support and can be (approximately) previewed using any Markdown previewer. -For reference documentation quality and how to get involved with doc sprints and -the community, see the -[TensorFlow 2.0 API Docs advice](https://docs.google.com/document/d/1e20k9CuaZ_-hp25-sSd8E8qldxKPKQR-SkwojYr_r-U/preview). ### Versions and branches @@ -54,9 +53,9 @@ main tensorflow/tensorflow repo. The reference documentation is generated from code comments and docstrings in the source code for -Python, -C++, and -Java. +Python, +C++, and +Java. Previous versions of the TensorFlow documentation are available as [rX.x branches](https://github.com/tensorflow/docs/branches) in the TensorFlow @@ -77,7 +76,7 @@ install: pip install git+https://github.com/tensorflow/docs -To generate the TensorFlow 2.0 reference docs, use the +To generate the TensorFlow 2 reference docs, use the `tensorflow/tools/docs/generate2.py` script: 
@@ -109,11 +108,12 @@ branches.
 
 ### Simple changes
 
-The easiest way to make straightforward documentation updates and fixes is to
-use GitHub's
-web-based file editor.
-Browse the [tensorflow/docs](https://github.com/tensorflow/docs/tree/master/site/en)
-repository to find the Markdown or notebook file that roughly corresponds to the
+The easiest way to make straightforward documentation updates to Markdown files
+is to use GitHub's
+web-based
+file editor. Browse the
+[tensorflow/docs](https://github.com/tensorflow/docs/tree/master/site/en)
+repository to find the Markdown that roughly corresponds to the
 tensorflow.org URL structure. In the
 upper right corner of the file view, click the pencil icon
 
@@ -167,21 +167,21 @@ when you submit your pull request.
 Add a remote:
 
 
-git remote add upstream git@github.com:tensorflow/docs.git
+git remote add upstream git@github.com:tensorflow/docs.git
 
 # View remote repos
 git remote -v
 origin    git@github.com:username/docs.git (fetch)
 origin    git@github.com:username/docs.git (push)
-upstream  git@github.com:tensorflow/docs.git (fetch)
-upstream  git@github.com:tensorflow/docs.git (push)
+upstream  git@github.com:tensorflow/docs.git (fetch)
+upstream  git@github.com:tensorflow/docs.git (push)
 

 
 To update:
 
 
 git checkout master
-git pull upstream master
+git pull upstream master
 
 git push  # Push changes to your GitHub account (defaults to origin)
 

@@ -257,9 +257,10 @@ is a hosted notebook environment that makes it easy to edit—and run—notebook
 documentation. Notebooks in GitHub are loaded in Google Colab by passing the
 path to the Colab URL, for example,
 the notebook located in GitHub here:
-https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/classification.ipynb

+https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/classification.ipynb

 can be loaded into Google Colab at this URL:
 https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/keras/classification.ipynb
+
 
 There is an
 Open in Colab
@@ -267,6 +268,32 @@ Chrome extension that performs this URL substitution when browsing a notebook on
 GitHub. This is useful when opening a notebook in your repo fork, because the
 top buttons always link to the TensorFlow Docs `master` branch.
 
+### Notebook formatting
+
+A notebook formatting tool makes Jupyter notebook source diffs consistent and
+easier to review. Since notebook authoring environments differ with regards to
+file output, indentation, metadata and other non-specified fields; `nbfmt` uses
+opinionated defaults with a preference for the TensorFlow docs Colab workflow.
+To format a notebook, install the
+TensorFlow
+docs notebook tools and run the `nbfmt` tool:
+
+```
+# Install the tensorflow-docs package:
+$ python3 -m pip install -U [--user] git+https://github.com/tensorflow/docs
+
+$ python3 -m tensorflow_docs.tools.nbfmt [options] notebook.ipynb [...]
+```
+
+For TensorFlow docs projects, notebooks *without* output cells are executed and
+tested; notebooks *with* saved output cells are published as-is. `nbfmt`
+respects the notebook state and uses the `--remove_outputs` option to explicitly
+remove output cells.
+
+To create a new notebook, copy and edit the
+TensorFlow
+docs notebook template.
+
 ### Edit in Colab
 
 Within the Google Colab environment, double-click cells to edit text and code
@@ -301,68 +328,31 @@ edit and update your forked GitHub repo directly from Google Colab:
 
 Success: Your changes have been accepted to the TensorFlow documentation.
 
-
-## Community translations
-
-Community translations are a great way to make TensorFlow accessible all over
-the world. To update a translation, find or add a file in the
-[language directory](https://github.com/tensorflow/docs/tree/master/site) that
-matches the same directory structure of the `en/` directory. The English docs
-are the *source-of-truth* and translations should follow these guides as close
-as possible. That said, translations are written for the communities they serve.
-If the English terminology, phrasing, style, or tone does not translate to
-another language, please use a translation appropriate for the reader.
-
-Note: The API reference is *not* translated for tensorflow.org.
-
-There are language-specific docs groups that make it easier for translation
-contributors to organize. Please join if you're an author, reviewer, or just
-interested in building out TensorFlow.org content for the community:
-
-* Chinese (Simplified): [docs-zh-cn@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)
-* Japanese: [docs-ja@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)
-* Korean: [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)
-* Russian: [docs-ru@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)
-* Turkish: [docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr)
-
-### Review notifications
-
-All documentation updates require a review. To collaborate more efficiently with
-the TensorFlow translation communities, here are some ways to keep on top of
-language-specific activity:
-
-* Join a language group listed above to receive an email for any *created* pull
-  request that touches the site/lang
-  directory for that language.
-* Add your GitHub username to the `site//REVIEWERS` file to get
-  automatically comment-tagged in a pull request. When comment-tagged, GitHub
-  will send you notifications for all changes and discussion in that pull
-  request.
-
-### Keep code up-to-date in translations
-
-For an open source project like TensorFlow, keeping documentation up-to-date is
-challenging. After talking with the community, readers of translated content
-will tolerate text that is a little out-of-date, but out-of-date code is
-frustrating. To make it easier to keep the code in sync, use the
-[nb-code-sync](https://github.com/tensorflow/docs/blob/master/tools/nb_code_sync.py)
-tool for the translated notebooks:
-
-
-./tools/nb_code_sync.py [--lang=en] site/lang/notebook.ipynb
-

-
-This script reads the code cells of a language notebook and check it against the
-English version. After stripping the comments, it compares the code blocks and
-updates the language notebook if they are different. This tool is particularly
-useful with an interactive git workflow to selectively add hunks of the file to
-the commit using: `git add --patch site/lang/notebook.ipynb`
-
-## Docs sprint
-
-Attend one of the
-[TensorFlow 2.0 Global Docs Sprint](https://www.google.com/maps/d/viewer?mid=1FmxIWZBXi4cvSy6gJUW9WRPfvVRbievf)
-events near you, or join remotely. Follow along with this
-[blog post](https://medium.com/tensorflow/https-medium-com-margaretmz-tf-docs-sprint-cheatsheet-7cb1dfd3e8b5?linkId=68384164).
-These events are a great way to get started contributing to the TensorFlow documentation.
+## Translations
+
+The TensorFlow team works with the community and vendors to provide translations
+for tensorflow.org. Translations of notebooks and other technical content are
+located in the
+tensorflow/docs-l10n
+GitHub repo. Please submit pull requests through the
+TensorFlow
+GitLocalize project.
+
+The English docs are the *source-of-truth* and translations should follow these
+guides as close as possible. That said, translations are written for the
+communities they serve. If the English terminology, phrasing, style, or tone
+does not translate to another language, please use a translation appropriate for
+the reader.
+
+Language support is determined by a number of factors including—but not limited
+to—site metrics and demand, community support,
+English
+proficiency, audience preference, and other indicators. Since each supported
+language incurs a cost, unmaintained languages are removed. Support for new
+languages will be announced on the
+TensorFlow blog or
+Twitter.
+
+If your preferred language is not supported, you are welcome to maintain a
+community fork for open source contributors. These are not published to
+tensorflow.org.
diff --git a/site/en/community/contribute/docs_ref.md b/site/en/community/contribute/docs_ref.md
index c1a12b26295..41fce4dde40 100644
--- a/site/en/community/contribute/docs_ref.md
+++ b/site/en/community/contribute/docs_ref.md
@@ -1,12 +1,14 @@
 # Contribute to the TensorFlow API documentation
 
+
+
 ## Testable docstrings
 
 TensorFlow uses [DocTest](https://docs.python.org/3/library/doctest.html) to
 test code snippets in Python docstrings. The snippet must be executable Python
 code. To enable testing, prepend the line with `>>>` (three left-angle
 brackets). For example, here's a excerpt from the `tf.concat` function in the
-[array_ops.py](https://www.tensorflow.org/code/tensorflow/python/ops/array_ops.py)
+[array_ops.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/array_ops.py)
 source file:
 
 ```
@@ -43,17 +45,42 @@ def concat(values, axis, name="concat"):
 
 Note: TensorFlow DocTest uses TensorFlow 2 and Python 3.
 
+To assess reference documentation quality, see the example section of the
+[TensorFlow 2 API Docs advice](https://docs.google.com/document/d/1e20k9CuaZ_-hp25-sSd8E8qldxKPKQR-SkwojYr_r-U/preview).
+(Be aware that the Task Tracker on this sheet is no longer in use.)
+
+
+### Make the code testable with DocTest
+
 Currently, many docstrings use backticks (```) to identify code. To make the
 code testable with DocTest:
 
 *   Remove the backticks (```) and use the left-brackets (>>>) in front of each
     line. Use (...) in front of continued lines.
-*   (```) can still be used for non-Python code or code that cannot be tested
-    for some reason.
-*   Add a newline as a separation between each DocTest and Markdown text.
+*   Add a newline to separate DocTest snippets from Markdown text to
+    render properly on tensorflow.org.
+
+### Customizations
+
+TensorFlow uses a few customizations to the builtin doctest logic:
+
+*   It does not compare float values as text: Float values are extracted from
+    the text and compared using `allclose` with _liberal `atol` and `rtol`
+    tolerences_. This allows :
+    *   Clearer docs - Authors don't need to include all decimal places.
+    *   More robust tests - Numerical changes in the underlying implementation
+        should never cause a doctest to fail.
+*   It only checks the output if the author includes output for a line. This
+    allows for clearer docs because authors usually don't need to capture
+    irrelevant intermediate values to prevent them from being printed.
 
 ### Docstring considerations
 
+*   *Overall*: The goal of doctest is to provide documentation, and confirm that
+    the documentation works. This is different from unit-testing. So:
+    *   Keep examples simple.
+    *   Avoid long or complicated outputs.
+    *   Use round numbers if possible.
 *   *Output format*: The output of the snippet needs to be directly beneath the
     code that’s generating the output. Also, the output in the docstring has to
     be exactly equal to what the output would be after the code is executed. See
@@ -62,17 +89,18 @@ code testable with DocTest:
     DocTest documentation. If the output exceeds the 80 line limit, you can put
     the extra output on the new line and DocTest will recognize it. For example,
     see multi-line blocks below.
-*   *Globals*: The `tf`, `np` and `os` modules are always available in
-    TensorFlow's DocTest.
+*   *Globals*: The `tf`, `np` and `os` modules are always
+    available in TensorFlow's DocTest.
 *   *Use symbols*: In DocTest you can directly access symbols defined in the
     same file. To use a symbol that’s not defined in the current file, please
     use TensorFlow’s public API `tf.xxx` instead of `xxx`. As you can see in the
-    example below, `random.normal` is accessed via `tf.random.normal`. This is
-    because `random.normal` is not visible in `NewLayer`.
+    example below, `random.normal` is accessed via
+    `tf.random.normal`. This is because
+    `random.normal` is not visible in `NewLayer`.
 
     ```
     def NewLayer():
-      “””This layer does cool stuff.
+      """This layer does cool stuff.
 
       Example usage:
 
@@ -80,14 +108,14 @@ code testable with DocTest:
       >>> new_layer = NewLayer(x)
       >>> new_layer
       
-      “””
+      """
     ```
 
 *   *Floating point values*: The TensorFlow doctest extracts float values from
     the result strings, and compares using `np.allclose` with reasonable
-    tolerances (`atol=1e-6`, `rtol=1e-6`). This way authors do not need to
-    worry about overly precise docstrings causing failures due to numerical
-    issues. Simply paste in the expected value.
+    tolerances (`atol=1e-6`, `rtol=1e-6`). This way authors do not need to worry
+    about overly precise docstrings causing failures due to numerical issues.
+    Simply paste in the expected value.
 
 *   *Non-deterministic output*: Use ellipsis(`...`) for the uncertain parts and
     DocTest will ignore that substring.
@@ -122,13 +150,35 @@ code testable with DocTest:
     ValueError: Unexpectedly found an instance of type ``.
     ```
 
+### Use a project-local copy of tf-doctest.
+
+Note: The tf-doctest utility is only setup to test source files within the
+`tensorflow` repository. If the files you are editing are in TensorFlow you can
+skip to the next section. Otherwise keep reading this section.
+
+Some API's in TensorFlow come from an external project:
+
+*   `tf.estimator` (from
+    [tensorflow_estimator](https://github.com/tensorflow/estimator))
+*   `tf.summary` [tensorboard](https://github.com/tensorflow/tensorboard))
+*   `tf.keras.preprocessing` (from
+    [keras-preprocessing](https://github.com/keras-team/keras-preprocessing))
+
+If you're working on an external project, or on TensorFlow APIs that are housed
+in an external project, these instructions won't work unless that project has
+its own local copy of `tf_doctest`, and you use that copy instead of
+TensorFlow's.
+
+For example:
+[tf_estimator_doctest.py](https://github.com/tensorflow/estimator/python/estimator/tf_estimator_doctest.py).
+
 ### Test on your local machine
 
 There are two ways to test the code in the docstring locally:
 
 *   If you are only changing the docstring of a class/function/method, then you
     can test it by passing that file's path to
-    [tf_doctest.py](https://www.tensorflow.org/code/tensorflow/tools/docs/tf_doctest.py).
+    [tf_doctest.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/docs/tf_doctest.py).
     For example:
 
     
diff --git a/site/en/community/contribute/docs_style.md b/site/en/community/contribute/docs_style.md
index 2edbed398ae..10f18e52699 100644
--- a/site/en/community/contribute/docs_style.md
+++ b/site/en/community/contribute/docs_style.md
@@ -12,12 +12,11 @@
 
 ## Markdown
 
-With a few exceptions, TensorFlow uses a Markdown syntax similiar to
+With a few exceptions, TensorFlow uses a Markdown syntax similar to
 [GitHub Flavored Markdown](https://guides.github.com/features/mastering-markdown/)
 (GFM). This section explains differences between GFM Markdown syntax and the
 Markdown used for TensorFlow documentation.
 
-
 ### Write about code
 
 #### Inline mentions of code
@@ -49,50 +48,94 @@ language after the first backtick group, for example:
 ```
 

 
-### Links in Markdown
+### Links in Markdown and notebooks
 
-#### Links between files in this repository
+#### Links between files in a repository
 
-Use relative links between files in a repository. This works on
-[tensorflow.org](https://www.tensorflow.org) and
-[GitHub](https://github.com/tensorflow/docs/tree/master/site/en):

-\[Custom layers\]\(../tutorials/eager/custom_layers.ipynb\) produces
-[Custom layers](https://www.tensorflow.org/tutorials/eager/custom_layers) on the
-site.
+Use relative links between files in a single GitHub repository. Include the file
+extension.
 
-#### Links to API documentation
+For example, **this file you're reading** is from the
+[https://github.com/tensorflow/docs](https://github.com/tensorflow/docs)
+repository. Therefore, it can use relative paths to link to other files in the same
+repository like this:
 
-API links are converted when the site is published. To link to a symbol's API
-reference page, enclose the full symbol path in backticks:
+* \[Basics\]\(../../guide/basics.ipynb\) produces
+[Basics](../../guide/basics.ipynb).
 
-*   `tf.data.Dataset` produces
-    [`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
-
-For the C++ API, use the namespace path:
+This is the preferred approach because this way the links on
+[tensorflow.org](https://www.tensorflow.org),
+[GitHub](https://github.com/tensorflow/docs) and
+[Colab](https://github.com/tensorflow/docs/tree/master/site/en/guide/bazics.ipynb)
+all work. Also, the reader stays in the same site when they click a link.
 
-*   `tensorflow::Tensor` produces
-    [tensorflow::Tensor](https://www.tensorflow.org/api_docs/cc/class/tensorflow/tensor)
+Note: You should include the file extension—such as `.ipynb` or `.md`—for
+relative links. It will rendered on `tensorflow.org` without an extension.
 
 #### External links
 
-For external links, including files on https://www.tensorflow.org
-that are not in the `tensorflow/docs` repository, use standard Markdown links
-with the full URI.
+For links to files that are not in the current repository, use standard Markdown
+links with the full URI. Prefer to link to the
+[tensorflow.org](https://www.tensorflow.org) URI if it's available.
 
 To link to source code, use a link starting with
 https://www.github.com/tensorflow/tensorflow/blob/master/, followed
 by the file name starting at the GitHub root.
 
-This URI naming scheme ensures that https://www.tensorflow.org can
-forward the link to the branch of the code corresponding to the version of the
-documentation you're viewing.
+When linking off of [tensorflow.org](https://www.tensorflow.org), include a
+`` on the Markdown link so that the "external link" symbol is shown.
+
+* `[GitHub](https://github.com/tensorflow/docs)` produces
+  [GitHub](https://github.com/tensorflow/docs)
+
+Do not include URI query parameters in the link:
+
+* Use: `https://www.tensorflow.org/guide/data`
+* Not: `https://www.tensorflow.org/guide/data?hl=en`
+
+
+#### Images
+
+The advice in the previous section is for links to pages. Images are handled
+differently.
+
+Generally, you should not check in images, and instead add the
+[TensorFlow-Docs team](https://github.com/tensorflow/docs) to your PR, and ask
+them to host the images on [tensorflow.org](https://www.tensorflow.org).
+This helps keep the size of your repository down.
+
+If you do submit images to your repository, note that some systems do not handle
+relative paths to images. Prefer to use a full URL pointing to the image's
+eventual location on [tensorflow.org](https://www.tensorflow.org).
 
-Do not include URI query parameters in the link.
+#### Links to API documentation
+
+API links are converted when the site is published. To link to a symbol's API
+reference page, enclose the symbol path in backticks:
+
+*   `tf.data.Dataset` produces
+    [`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
+
+Full paths are slightly preferred except for long paths. Paths
+can be abbreviated by dropping the leading path components. Partial paths will
+be converted to links if:
 
-File paths use underscores for spaces, for example, `custom_layers.ipynb`.
+*   There is at least one `.` in the path, and
+*   The partial path is unique within the project.
+
+API paths are linked **for every project** with a Python API published on
+[tensorflow.org](https://www.tensorflow.org). You can easily link to multiple
+subprojects from a single file by wrapping the API names with backticks.
+For example:
+
+*   `tf.metrics`, `tf_agents.metrics`,
+    `text.metrics` produces: `tf.metrics`,
+    `tf_agents.metrics`, `text.metrics`.
+
+For symbols with multiple path aliases there is a slight preference for the
+path that matches the API-page on [tensorflow.org](https://www.tensorflow.org).
+All aliases will redirect to the correct page.
 
-Include the file extension in links to use on the site *and* GitHub, for example,

-\[Custom layers\]\(../tutorials/eager/custom_layers.ipynb\).
 
 ### Math in Markdown
 
@@ -102,24 +145,29 @@ following:
 *   MathJax renders properly on [tensorflow.org](https://www.tensorflow.org).
 *   MathJax does not render properly on GitHub.
 *   This notation can be off-putting to unfamiliar developers.
+*   For consistency [tensorflow.org](https://www.tensorflow.org) follows the
+    same  rules as Jupyter/Colab.
 
 Use $$ around a block of MathJax:
 
-
$$
+
$$
 E=\frac{1}{2n}\sum_x\lVert (y(x)-y'(x)) \rVert^2
-$$

+$$

 
 $$
 E=\frac{1}{2n}\sum_x\lVert (y(x)-y'(x)) \rVert^2
 $$
 
-Wrap inline MathJax expressions with \\( ... \\):
+Wrap inline MathJax expressions with $ ... $:
 
 

-This is an example of an inline MathJax expression: \\( 2 \times 2 = 4 \\)
+This is an example of an inline MathJax expression: $ 2 \times 2 = 4 $
 

 
-This is an example of an inline MathJax expression: \\( 2 \times 2 = 4 \\)
+This is an example of an inline MathJax expression: $ 2 \times 2 = 4 $
+
+\\( ... \\) delimiters also work for inline math,
+but the \$ form is sometimes more readable.
 
 Note: If you need to use a dollar sign in text or MathJax expressions, escape it
 with a leading slash: `\$`. Dollar signs within code blocks (such as Bash
@@ -161,14 +209,21 @@ not the technical content.
 
 ### Ops
 
-Use `# ⇒` instead of a single equal sign when you want to show what an op
-returns.
+In markdown files, use `# ⇒` instead of a single equal sign when you want to
+show what an op returns.
 
 ```python
-# 'input' is a tensor of shape [2, 3, 5] 
-(tf.expand_dims(input, 0))  # ⇒ [1, 2, 3, 5]
+# 'input' is a tensor of shape [2, 3, 5]
+tf.expand_dims(input, 0)  # ⇒ [1, 2, 3, 5]
 ```
 
+In notebooks, display the result instead of adding a comment (If the last
+expression in a notebook cell is not assigned to a variable, it is automatically
+displayed.)
+
+In API reference docs prefer using [doctest](docs_ref.md#doctest) to show
+results.
+
 ### Tensors
 
 When you're talking about a tensor in general, don't capitalize the word
@@ -180,10 +235,15 @@ Don't use the word *Tensors* (plural) to describe multiple `Tensor` objects
 unless you really are talking about a `Tensors` object. Instead, say "a list (or
 collection) of `Tensor` objects".
 
-Use the word *shape* to detail the dimensions of a tensor, and show the shape in
+Use the word *shape* to detail the axes of a tensor, and show the shape in
 square brackets with backticks. For example:
 
 

-If `input` is a three-dimensional tensor with shape `[3, 4, 3]`, this operation
-returns a three-dimensional tensor with shape `[6, 8, 6]`.
+If `input` is a three-axis `Tensor` with shape `[3, 4, 3]`, this operation
+returns a three-axis `Tensor` with shape `[6, 8, 6]`.
 

+
+As above, prefer "axis" or "index" over "dimension" when talking about the
+elements of a `Tensor`'s shape. Otherwise it's easy to confuse "dimension" with
+the dimension of a vector space. A "three-dimensional vector" has a single axis
+with length 3.
diff --git a/site/en/community/contribute/index.md b/site/en/community/contribute/index.md
index 84c9ba02d42..028389b68ff 100644
--- a/site/en/community/contribute/index.md
+++ b/site/en/community/contribute/index.md
@@ -37,11 +37,12 @@ This guide provides everything you need to get started. Our most common
 contributions include *code*, *documentation*, and *community support*.
 
 - [Write code](code.md).
+- [Improve tests](tests.md).
 - [Improve documentation](docs.md).
 - Answer questions on
   [Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow).
-- Participate in the discussion on the
-  [TensorFlow forums](https://www.tensorflow.org/community/forums).
+- Participate in the discussion on our
+  [mailing lists](../mailing-lists.md).
 - Contribute [example notebooks](http://www.github.com/tensorflow/examples).
 - Investigate [bugs and issues](https://github.com/tensorflow/tensorflow/issues)
   on GitHub.
diff --git a/site/en/community/contribute/rfc_process.md b/site/en/community/contribute/rfc_process.md
index b37c1a8a46c..8d34b3932a2 100644
--- a/site/en/community/contribute/rfc_process.md
+++ b/site/en/community/contribute/rfc_process.md
@@ -25,6 +25,7 @@ experts, and communicating design changes broadly.
 
 2.  Draft your RFC.
 
+    *   Read the [design review criteria](https://github.com/tensorflow/community/blob/master/governance/design-reviews.md)
     *   Follow the
         [RFC template](https://github.com/tensorflow/community/blob/master/rfcs/yyyymmdd-rfc-template.md).
     *   Name your RFC file `YYYYMMDD-descriptive-name.md`, where `YYYYMMDD` is
diff --git a/site/en/community/contribute/sigs.md b/site/en/community/contribute/sigs.md
new file mode 100644
index 00000000000..b736ec5919a
--- /dev/null
+++ b/site/en/community/contribute/sigs.md
@@ -0,0 +1,97 @@
+# Contribute to TensorFlow Special Interest Groups (SIGs)
+
+The TensorFlow Special Interest Groups (TF SIGs) organize community contributions to key parts of the TensorFlow ecosystem. SIG leads and members work together to build and support important TensorFlow use cases.
+
+SIGs are led by members of the open source community, including industry collaborators and [Machine Learning Google Developer Experts](https://developers.google.com/community/experts) (ML GDEs). TensorFlow's success is due in large part to their hard work and contributions.
+
+We encourage you to join a SIG working on the area of TensorFlow's ecosystem you care most about. Not all SIGs will have the same level of energy, breadth of scope, or governance models — browse our [SIG charters](https://github.com/tensorflow/community/tree/master/sigs) to learn more. Stay connected with SIG leads and members on the [TensorFlow Forum](https://discuss.tensorflow.org/c/special-interest-groups/8), where you can subscribe to preferred [tags](https://discuss.tensorflow.org/tags) and learn more about the regular SIG meetings.
+
+## SIG Addons
+
+SIG Addons builds and maintains a repository of community contributions that conform to well-established API patterns, but implement new functionality not available in core TensorFlow. 
+
+TensorFlow natively supports a large number of operators, layers, metrics, losses, optimizers, and more. However, in a fast-moving field like ML, there are many new developments that cannot be integrated into core TensorFlow (because their broad applicability is not yet clear, or it is mostly used by a smaller subset of the community). SIG Addons enables users to introduce new extensions to the TensorFlow ecosystem in a sustainable manner.
+
+SIG Addons on GitHub Contributing Discuss on the Forum
+
+## SIG Build
+
+SIG Build improves and extends the TensorFlow build process. SIG Build maintains a repository showcasing resources, guides, tools, and builds contributed by the community, for the community.
+
+SIG Build on GitHub Contributing Discuss on the Forum
+
+## SIG IO
+
+SIG IO maintains TensorFlow I/O, a collection of file systems and file formats that are not available in TensorFlow's built-in support.
+
+SIG IO on GitHub Contributing Discuss on the Forum
+
+## SIG JVM
+
+SIG JVM maintains the TF Java bindings to let users use JVM for building, training and running machine learning models.
+
+Java and other JVM languages, such as Scala or Kotlin, are frequently used in small-to-large enterprises all over the world, which makes TensorFlow a strategic choice for adopting machine learning at a large scale.
+
+SIG JVM on GitHub Contributing Discuss on the Forum
+
+## SIG Models
+
+SIG Models focuses on enabling contributions to the state-of-the-art model implementation in TensorFlow 2, and sharing best practices of using TensorFlow 2 for state-of-the-art research. Subgroups orient around different machine learning applications (Vision, NLP, etc.).
+
+SIG Models host discussions and collaborations around the [TensorFlow Model Garden](https://github.com/tensorflow/models) and [TensorFlow Hub](https://tfhub.dev). Learn how to contribute on GitHub below, or discuss [Research & Models](https://discuss.tensorflow.org/c/research-models/26) on the Forum.
+
+TensorFlow Model Garden on GitHub Contributing 
+
+TensorFlow Hub on GitHub Contributing 
+
+## SIG Micro
+
+SIG Micro discusses and shares updates on [TensorFlow Lite for Microcontrollers](https://www.tensorflow.org/lite/microcontrollers), a port of TensorFlow Lite designed to run machine learning models on DSPs, microcontrollers and other devices with limited memory.
+
+TensorFlow Lite Micro on GitHub Contributing Discuss on the Forum
+
+## SIG MLIR
+
+SIG MLIR maintains [MLIR](https://mlir.llvm.org/) dialects and utilities for TensorFlow, XLA and TF Lite, providing high performance compilers and optimization techniques that can be applied to TensorFlow graphs and code generation. Their overarching goal is to create common intermediate representation (IR) that reduces the cost to bring up new hardware, and improve usability for existing TensorFlow users.
+
+SIG MLIR on GitHub Contributing Discuss on the Forum
+
+## SIG Networking
+
+SIG Networking maintains the TensorFlow Networking repository for platform-specific networking extensions to core TensorFlow and related utilities.
+
+SIG Networking on GitHub Discuss on the Forum
+
+## SIG Recommenders
+
+SIG Recommenders maintains a collection of projects related to large-scale recommendation systems built upon TensorFlow contributed and maintained by the community. Those contributions are complementary to [TensorFlow Core](https://www.tensorflow.org/overview) and [TensorFlow Recommenders](https://www.tensorflow.org/recommenders).
+
+SIG Recommenders on GitHub Contributing Discuss on the Forum
+
+## SIG Rust
+
+SIG Rust maintains idiomatic Rust language bindings for TensorFlow.
+
+SIG Rust on GitHub Contributing Discuss on the Forum
+
+## SIG TensorBoard
+
+SIG TensorBoard facilitates discussion around [TensorBoard](https://www.tensorflow.org/tensorboard)—a suite of tools for inspecting, debugging and optimizing TensorFlow programs.
+
+TensorBoard on GitHub Contributing Discuss on the Forum
+
+## SIG TF.js
+
+SIG TF.js facilitates community-contributed components to [TensorFlow.js](https://www.tensorflow.org/js) and offers project support through the SIG.
+
+TensorFlow.js on GitHub Contributing Discuss on the Forum
+
+## SIG TFX-Addons
+
+SIG TFX-Addons accelerates the sharing of customizations and additions to meet the needs of production ML, expand the vision, and help drive new directions for [TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx) and the ML community.
+
+SIG TFX-Addons on GitHub Contributing Discuss on the Forum
+
+## New SIGs
+
+Didn't find what you were looking for? If you believe there is a strong need for a new TensorFlow SIG, please read the [SIG playbook](https://www.tensorflow.org/community/sig_playbook) and follow instructions on how to propose it to our contributor community.
diff --git a/site/en/community/contribute/tests.md b/site/en/community/contribute/tests.md
new file mode 100644
index 00000000000..d0763159728
--- /dev/null
+++ b/site/en/community/contribute/tests.md
@@ -0,0 +1,200 @@
+# TensorFlow testing best practices
+
+These are the recommended practices for testing code in the
+[TensorFlow repository](https://github.com/tensorflow/tensorflow).
+
+## Before you get started
+
+Before you contribute source code to a TensorFlow project, please review the
+`CONTRIBUTING.md` file in the GitHub repo of the project. (For example, see the
+[CONTRIBUTING.md file for the core TensorFlow repo](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md).)
+All code contributors are required to sign a
+[Contributor License Agreement](https://cla.developers.google.com/clas) (CLA).
+
+## General principles
+
+### Only depend on what you use in your BUILD rules
+
+TensorFlow is a large library, and depending on the full package when
+writing a unit test for its submodules has been a common practice. However, this
+disables the `bazel` dependency-based analysis. This means that continuous
+integration systems cannot intelligently eliminate unrelated tests for
+presubmit/postsubmit runs. If you only depend on the submodules that you are
+testing in your `BUILD` file, you will save time for all TensorFlow developers,
+and a lot of valuable computation power.
+
+However, modifying your build dependency to omit the full TF targets brings some
+limitations for what you can import in your Python code. You will not be able to
+use the `import tensorflow as tf` statement in your unit tests anymore. But this
+is a worthwhile tradeoff since as it saves all developers from running thousands
+of unnecessary tests.
+
+### All code should have unit tests
+
+For any code you write, you should also write its unit tests. If you write a new
+file `foo.py`, you should place its unit tests in `foo_test.py` and submit it
+within the same change. Aim for >90% incremental test coverage for all your
+code.
+
+### Avoid using native bazel test rules in TF
+
+TF has a lot of subtleties when running tests. We have worked to hide all of
+those complexities in our bazel macros. To avoid having to deal with those, use
+the following instead of the native test rules. Note that all of these are
+defined in `tensorflow/tensorflow.bzl`
+For CC tests, use `tf_cc_test`, `tf_gpu_cc_test`, `tf_gpu_only_cc_test`.
+For python tests, use `tf_py_test` or `gpu_py_test`.
+If you need something really close to the native `py_test` rule, please use the
+one defined in tensorflow.bzl instead. You just need to add the following line
+at the top of the BUILD file: `load(“tensorflow/tensorflow.bzl”, “py_test”)`
+
+### Be aware where the test executes
+
+When you write a test, our test infra can take care of running your tests on
+CPU, GPU and accelerators if you write them accordingly. We have automated tests
+that run on Linux, macos, windows, that have systems with or without GPUs. You
+simply need to pick one of the macros listed above, and then use tags to limit
+where they are executed.
+
+* `manual` tag will exclude your test from running anywhere. This includes
+manual test executions that use patterns such as `bazel test tensorflow/…`
+
+* `no_oss` will exclude your test from running in the official TF OSS test
+infrastructure.
+
+* `no_mac` or `no_windows` tags can be used to exclude your test from relevant
+operating system test suites.
+* `no_gpu` tag can be used to exclude your test from running in GPU test suites.
+
+### Verify tests run in expected test suites
+
+TF has quite a few test suites. Sometimes, they may be confusing to set up.
+There might be different problems that cause your tests to be omitted from
+continuous builds. Thus, you should verify your tests are executing as expected.
+To do this:
+
+* Wait for your presubmits on your Pull Request(PR) to run to completion.
+* Scroll to the bottom of your PR to see the status checks.
+* Click the “Details” link at the right side of any Kokoro check.
+* Check the “Targets” list to find your newly added targets.
+
+### Each class/unit should have its own unit test file
+
+Separate test classes help us better isolate failures and resources. They lead
+to much shorter and easier to read test files. Therefore, all your Python files
+should have at least one corresponding test file (For each `foo.py`, it should
+have `foo_test.py`). For more elaborate tests, such as integration tests that
+require different setups, it is fine to add more test files.
+
+## Speed and running times
+
+### Sharding should be used as little as possible
+
+Instead of sharding please consider:
+* Making your tests smaller
+* If the above is not possible, split the tests up
+
+Sharding helps reduce the overall latency of a test, but the same can be
+achieved by breaking up tests to smaller targets. Splitting tests gives us a
+finer level of control on each test, minimizing unnecessary presubmit runs and
+reducing the coverage loss from a buildcop disabling an entire target due to a
+misbehaving testcase. Moreover, sharding incurs hidden costs that are not so
+obvious, such as running all test initialization code for all shards. This issue
+has been escalated to us by infra teams as a source that creates extra load.
+
+### Smaller tests are better
+
+The quicker your tests run, the more likely people will be to run your tests.
+One extra second for your test can accumulate to hours of extra time spent
+running your test by developers and our infrastructure. Try to make your tests
+run under 30 seconds (in non-opt mode!), and make them small. Only mark your
+tests as medium as a last resort. The infra does not run any large tests as
+presubmits or postsubmits! Therefore, only write a large test if you are going
+to arrange where it is going to run. Some tips to make tests run faster:
+
+* Run less iterations of training in your test
+* Consider using dependency injection to replace heavy dependencies of system
+under test with simple fakes.
+* Consider using smaller input data in unit tests
+* If nothing else works, try splitting up your test file.
+
+### Test times should aim for half of test size timeout to avoid flakes
+
+With `bazel` test targets, small tests have 1 minute timeouts. Medium test
+timeouts are 5 minutes. Large tests are just not executed by the TensorFlow test
+infra. However, many tests are not deterministic in the amount of time they
+take. For various reasons your tests might take more time every now and then.
+And, if you mark a test that runs for 50 seconds on the average as small, your
+test will flake if it schedules on a machine with an old CPU. Therefore, aim for
+30 second average running time for small tests. Aim for 2 minutes 30 seconds of
+average running time for medium tests.
+
+### Reduce the number of samples and increase tolerances for training
+
+Slow running tests deter contributors. Running training in tests can be very
+slow. Prefer higher tolerances to be able to use less samples in your tests to
+keep your tests sufficiently fast (2.5 minutes max).
+
+## Eliminate non-determinism and flakes
+
+### Write deterministic tests
+
+Unit tests should always be deterministic. All tests running on TAP and guitar
+should run the same way every single time, if there is no code change affecting
+them. To ensure this, below are some points to consider.
+
+### Always seed any source of stochasticity
+
+Any random number generator, or any other sources of stochasticity can cause
+flakiness. Therefore, each of these must be seeded. In addition to making tests
+less flaky, this makes all tests reproducible. Different ways to set some seeds
+you may need to set in TF tests are:
+
+```python
+# Python RNG
+import random
+random.seed(42)
+
+# Numpy RNG
+import numpy as np
+np.random.seed(42)
+
+# TF RNG
+from tensorflow.python.framework import random_seed
+random_seed.set_seed(42)
+```
+
+### Avoid using `sleep` in multithreaded tests
+
+Using `sleep` function in tests can be a major cause of flakiness. Especially
+when using multiple threads, using sleep to wait for another thread will never
+be determistic. This is due to system not being able to guarantee any ordering
+of execution of different threads or processes. Therefore, prefer deterministic
+synchronization constructs such as mutexes.
+
+### Check if the test is flaky
+
+Flakes cause buildcops and developers to lose many hours. They are difficult to
+detect, and they are difficult to debug. Even though there are automated systems
+to detect flakiness, they need to accumulate hundreds of test runs before they
+can accurately denylist tests. Even when they detect, they denylist your tests
+and test coverage is lost. Therefore, test authors should check if their tests
+are flaky when writing tests. This can be easily done by running your test with
+the flag: `--runs_per_test=1000`
+
+### Use TensorFlowTestCase
+
+TensorFlowTestCase takes necessary precautions such as seeding all random number
+generators used to reduce flakiness as much as possible. As we discover and fix
+more flakiness sources, these all will be added to TensorFlowTestCase.
+Therefore, you should use TensorFlowTestCase when writing tests for tensorflow.
+TensorFlowTestCase is defined here: `tensorflow/python/framework/test_util.py`
+
+### Write hermetic tests
+
+Hermetic tests do not need any outside resources. They are packed with
+everything they need, and they just start any fake services they might need. Any
+services other than your tests are sources for non determinism. Even with 99%
+availability of other services, network can flake, rpc response can be delayed,
+and you might end up with an inexplicable error message.
+Outside services may be, but not limited to, GCS, S3 or any website.
diff --git a/site/en/community/forums.md b/site/en/community/forums.md
deleted file mode 100644
index 0d04b0e8502..00000000000
--- a/site/en/community/forums.md
+++ /dev/null
@@ -1,76 +0,0 @@
-# Forums
-
-As a community, we do much of our collaboration on public mailing lists. Please
-note that if you're looking for help using TensorFlow,
-[Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow) and
-[GitHub issues](https://github.com/tensorflow/tensorflow/issues) are the best
-initial places to look. For more information, see
-[how to get help](/community/#get_help).
-
-## General TensorFlow lists
-
-* [announce](https://groups.google.com/a/tensorflow.org/d/forum/announce) - Low-volume announcements of new releases.
-* [discuss](https://groups.google.com/a/tensorflow.org/d/forum/discuss) - General community discussion around TensorFlow.
-* [developers](https://groups.google.com/a/tensorflow.org/d/forum/developers) - Discussion for developers contributing to TensorFlow.
-* [documentation](https://groups.google.com/a/tensorflow.org/d/forum/docs) - Discussion for contributing to TensorFlow documentation.
-* [testing](https://groups.google.com/a/tensorflow.org/d/forum/testing) - Discussion and questions on TensorFlow 2.0 testing.
-
-## Project-specific lists
-
-These projects inside the TensorFlow GitHub organization have lists dedicated to their communities:
-
-*   [hub](https://groups.google.com/a/tensorflow.org/d/forum/hub) - Discussion
-    and collaboration around
-    [TensorFlow Hub](https://github.com/tensorflow/hub).
-*   [magenta-discuss](https://groups.google.com/a/tensorflow.org/d/forum/magenta-discuss) -
-    General discussion about [Magenta](https://magenta.tensorflow.org/)
-    development and directions.
-*   [tensor2tensor](https://groups.google.com/d/forum/tensor2tensor) -
-    Discussion and peer support for Tensor2Tensor.
-*   [tfjs-announce](https://groups.google.com/a/tensorflow.org/d/forum/tfjs-announce) -
-    Announcements of new TensorFlow.js releases.
-*   [tfjs](https://groups.google.com/a/tensorflow.org/d/forum/tfjs) - Discussion
-    and peer support for TensorFlow.js.
-*   [tflite](https://groups.google.com/a/tensorflow.org/d/forum/tflite) -
-    Discussion and peer support for TensorFlow Lite.
-*   [tfprobability](https://groups.google.com/a/tensorflow.org/d/forum/tfprobability) -
-    Discussion and peer support for TensorFlow Probability.
-*   [tfx](https://groups.google.com/a/tensorflow.org/forum/#!forum/tfx) -
-    Discussion and collaboration around [TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx/).
-*   [tpu-users](https://groups.google.com/a/tensorflow.org/d/forum/tpu-users) -
-    Community discussion and support for TPU users.
-*   [xla-dev](https://groups.google.com/forum/#!forum/xla-dev) - Discussion for
-    developers contributing to the [XLA](https://www.tensorflow.org/xla)
-    compiler.
-
-## Special Interest Groups
-
-TensorFlow's
-[Special Interest Groups](https://github.com/tensorflow/community/tree/master/sigs) (SIGs)
-support community collaboration on particular project focuses. Members of these
-groups work together to build and support TensorFlow related projects. While their
-archives are public, different SIGs have their own membership policies.
-
-* [addons](https://groups.google.com/a/tensorflow.org/d/forum/addons) -
-  Supporting SIG Addons, for extensions to TensorFlow that confirm to the stable
-  API.
-* [build](https://groups.google.com/a/tensorflow.org/d/forum/build) - Supporting
-  SIG Build, for build, distribution and packaging of TensorFlow.
-* [io](https://groups.google.com/a/tensorflow.org/d/forum/io) - Supporting SIG
-  IO, for file systems and formats not available in core TensorFlow.
-* [jvm](https://groups.google.com/a/tensorflow.org/d/forum/jvm) -
-  Supporting SIG JVM, building Java and JVM support for TensorFlow.
-* [keras](https://groups.google.com/forum/#!forum/keras-users) - Keras users
-  mailing list, for design reviews and discussions relating to SIG Keras.
-* [micro](https://groups.google.com/a/tensorflow.org/d/forum/micro) -
-  Supporting SIG Micro, focusing on low power TF Lite deployment.
-* [mlir](https://groups.google.com/a/tensorflow.org/d/forum/mlir) - Supporting
-  SIG MLIR, collaboration around MLIR, Multi-Level Intermediate Representation.
-* [networking](https://groups.google.com/a/tensorflow.org/d/forum/networking) -
-  Supporting SIG Networking, for adding network protocols other than gRPC.
-* [rust](https://groups.google.com/a/tensorflow.org/d/forum/rust) -
-  Supporting SIG Rust, for the Rust language bindings.
-* [swift](https://groups.google.com/a/tensorflow.org/d/forum/swift) - Supporting
-  SIG Swift, developing Swift for TensorFlow.
-* [tensorboard](https://groups.google.com/a/tensorflow.org/d/forum/sig-tensorboard) -
-  Supporting SIG TensorBoard, for plugin development and other contribution.
diff --git a/site/en/community/groups.md b/site/en/community/groups.md
deleted file mode 100644
index e021e0cae90..00000000000
--- a/site/en/community/groups.md
+++ /dev/null
@@ -1,80 +0,0 @@
-# User groups
-
-TensorFlow has communities around the world. [Add your community!](https://github.com/tensorflow/docs/blob/master/site/en/community/groups.md)
-
-## Africa
-
-* [TensorFlow Abuja](https://www.meetup.com/Tensorflow-Abuja/events/)
-* [TensorFlow Abidjan](https://www.meetup.com/TensorFlow-Abidjan/)
-* [TensorFlow Bauchi](https://www.meetup.com/TensorFlow-Bauchi/)
-* [TensorFlow Cotonou](https://www.meetup.com/TensorFlow-Cotonou/)
-* [TensorFlow Harare](https://www.meetup.com/TensorFlow-Harare/)
-* [TensorFlow Ibadan](https://www.meetup.com/TensorFlow-Ibadan/)
-* [TensorFlow Nairobi](https://www.meetup.com/TensorFlow-Nairobi/)
-* [TensorFlow Lagos](https://www.meetup.com/tf-lagos/)
-* [TensorFlow Tunis Meetup](https://www.meetup.com/TensorFlow-Tunis-Meetup/)
-* [TensorFlow Egypt](https://www.facebook.com/groups/546277615773342/)
-
-## Americas
-
-* [Austin, TX TensorFlow User Group](https://www.meetup.com/Austin-TensorFlow-Meetup-Group/)
-* [Houston TensorFlow & Applied AI/ML user group](https://www.meetup.com/Houston-AI/)
-* [TensorFlow Buenos Aires](https://www.meetup.com/TensorFlow-Buenos-Aires/)
-* [TensorFlow-Northwest (Portland)](https://www.meetup.com/TensorFlow-Northwest/)
-* [TensorFlow São Paulo](https://www.meetup.com/TensorFlowSP/)
-
-## Asia
-
-* [TensorFlow China community](https://www.tensorflowers.cn)
-* [TensorFlow User Group Beijing](https://mp.weixin.qq.com/s/IftQLGI9qHM5zvlxSD756Q)
-* [TensorFlow User Group Hangzhou](https://mp.weixin.qq.com/s/EIyGkk5ctE-gVsdYoqVEww)
-* [TensorFlow User Group Shanghai](https://mp.weixin.qq.com/s/9iQ0vzYjZdZXyi3LGN5fRQ)
-* [TensorFlow User Group Shenzhen](https://mp.weixin.qq.com/s/cHDNye-WOmMU60iqO88OYw)
-* [TensorFlow User Group Guangzhou](https://mp.weixin.qq.com/s/JD-_RlBcXpx83NmNnJvnNA)
-* [TensorFlow User Group Chengdu](https://mp.weixin.qq.com/s/4V_eZN7--waIDb3T19hqpA)
-* [TensorFlow User Group Xiamen](https://mp.weixin.qq.com/s/Kd6oJ1aae2nLTdHul-klqQ)
-* [TensorFlow User Group Xi'an](https://mp.weixin.qq.com/s/3czrE99NSWtYqSYLQcUDnQ)
-* [TensorFlow User Group Zhuhai](https://mp.weixin.qq.com/s/cqdQSksbzAqkID6iF2vnxA)
-* [TensorFlow User Group Hefei](https://mp.weixin.qq.com/s/6UhgbKPh3_5S7d_cuAuRcw)
-* [TensorFlow User Group Wuhan](https://mp.weixin.qq.com/s/7GrFWghNMpKBmwabAL0fOQ)
-* [TensorFlow User Group Nanjing](https://mp.weixin.qq.com/s/8mnSmZAmECZqZX12ulCBzA)
-* [TensorFlow User Group Dalian](https://mp.weixin.qq.com/s/LzL99TJHMGByGvefzM5T4A)
-* [TensorFlow User Group Harbin](https://mp.weixin.qq.com/s/E4vjAcVzCDS7uRm5OgBmxA)
-* [TensorFlow User Group Zhengzhou](https://mp.weixin.qq.com/s/-xCo0mvn9TxAszxA6iu13g)
-* [TensorFlow User Group Taipei](https://www.meetup.com/TensorFlow-User-Group-Taipei/)
-* [TensorFlow Hong Kong](https://fb.me/tensorflowhk)
-* [TensorFlow User Group Tokyo](https://tfug-tokyo.connpass.com/)
-* [TensorFlow User Group Utsunomiya](https://tfug-utsunomiya.connpass.com/)
-* [TensorFlow User Group Aizu](https://tfug-aizu.connpass.com/)
-* [TensorFlow User Group Niigata](https://tfug-niigata.connpass.com/)
-* [TensorFlow User Group Kansai](https://tfug-kansai.connpass.com/)
-* [TensorFlow User Group Fukuoka](https://tfugfuk.connpass.com/)
-* [TensorFlow User Group Kagoshima](https://tfug-kagoshima.connpass.com/)
-* [TensorFlow User Group Okinawa](https://tfug-okinawa.connpass.com/)
-* [TensorFlow Korea (TF-KR) User Group](https://www.facebook.com/groups/TensorFlowKR/)
-* [TensorFlow Philippines Community](https://www.facebook.com/groups/TensorFlowPH/)
-* [TensorFlow and Deep Learning Singapore](https://www.meetup.com/TensorFlow-and-Deep-Learning-Singapore/)
-* [Tensorflow Community Sri Lanka](https://www.facebook.com/groups/150363118841298/)
-* [TensorFlow Việt Nam](https://www.facebook.com/gdgtensorflowvietnam)
-* [TF Nepal](https://www.facebook.com/groups/tfnepal/)
-* [TensorFlow UserGroup Mumbai](https://t.me/tfugmumbai/)
-
-
-## Europe
-
-* [TensorFlow Barcelona](https://www.meetup.com/Barcelona-Machine-Learning-Meetup/)
-* [TensorFlow Berlin](https://www.meetup.com/Tensorflow-in-Berlin/)
-* [TensorFlow Madrid](https://www.meetup.com/TensorFlow-Madrid/)
-* [Tensorflow Belgium](https://www.meetup.com/TensorFlow-Belgium)
-* [TensorFlow x Rome Meetup](https://www.meetup.com/it-IT/TensorFlow-x-Rome-Meetup)
-* [TensorFlow London](https://www.meetup.com/TensorFlow-London/)
-* [London TensorFlow Meetup](https://www.meetup.com/London-TensorFlow-Meetup/)
-* [TensorFlow Edinburgh](https://www.meetup.com/tensorflow-edinburgh/)
-* [Kraków TensorFlow User Group](https://www.facebook.com/groups/370356090255755/)
-* [TensorFlow Turkey](https://kommunity.com/tensorflow-turkey/)
-* [Paris TensorFlow User Group](https://www.meetup.com/Paris-Tensorflow-User-Group/)
-
-## Oceania
-
-* [Melbourne TensorFlow Meetup](https://www.meetup.com/Melbourne-TensorFlow-Meetup)
-* [Sydney TensorFlow Meetup](https://www.meetup.com/Sydney-TensorFlow-Meetup)
diff --git a/site/en/community/mailing-lists.md b/site/en/community/mailing-lists.md
new file mode 100644
index 00000000000..35bfb218ba1
--- /dev/null
+++ b/site/en/community/mailing-lists.md
@@ -0,0 +1,84 @@
+# Mailing lists
+
+As a community, we do much of our collaboration on public mailing lists. Please
+note that if you're looking for help using TensorFlow,
+[TensorFlow Forum](https://discuss.tensorflow.org/),
+[Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow), and
+[GitHub issues](https://github.com/tensorflow/tensorflow/issues) are the best
+initial places to look. To receive a roundup of updates from the TensorFlow team each quarter, subscribe to the [TensorFlow newsletter](https://services.google.com/fb/forms/tensorflow/).
+
+## General TensorFlow lists and forums
+
+*   [announce](https://groups.google.com/a/tensorflow.org/d/forum/announce) -
+    Low-volume announcements of new releases.
+*   [discuss](https://groups.google.com/a/tensorflow.org/d/forum/discuss) -
+    General community discussion around TensorFlow.
+*   [developers](https://groups.google.com/a/tensorflow.org/d/forum/developers) -
+    Discussion for developers contributing to TensorFlow.
+*   [documentation](https://discuss.tensorflow.org/tag/docs) -
+    Discussion for contributing to TensorFlow documentation. See
+    [community translations](https://www.tensorflow.org/community/contribute/docs#community_translations)
+    for language-specific docs lists.
+*   [testing](https://groups.google.com/a/tensorflow.org/d/forum/testing) -
+    Discussion and questions on TensorFlow 2 testing.
+
+## Project-specific lists
+
+These projects inside the TensorFlow GitHub organization have lists dedicated to their communities:
+
+*   [hub](https://groups.google.com/a/tensorflow.org/d/forum/hub) - Discussion
+    and collaboration around
+    [TensorFlow Hub](https://github.com/tensorflow/hub).
+*   [magenta-discuss](https://groups.google.com/a/tensorflow.org/d/forum/magenta-discuss) -
+    General discussion about [Magenta](https://magenta.tensorflow.org/)
+    development and directions.
+*   [tensor2tensor](https://groups.google.com/d/forum/tensor2tensor) -
+    Discussion and peer support for Tensor2Tensor.
+*   [tfjs-announce](https://groups.google.com/a/tensorflow.org/d/forum/tfjs-announce) -
+    Announcements of new TensorFlow.js releases.
+*   [tfjs](https://groups.google.com/a/tensorflow.org/d/forum/tfjs) - Discussion
+    and peer support for TensorFlow.js.
+*   [tflite](https://groups.google.com/a/tensorflow.org/d/forum/tflite) -
+    Discussion and peer support for TensorFlow Lite.
+*   [tfprobability](https://groups.google.com/a/tensorflow.org/d/forum/tfprobability) -
+    Discussion and peer support for TensorFlow Probability.
+*   [tfx](https://groups.google.com/a/tensorflow.org/forum/#!forum/tfx) -
+    Discussion and collaboration around [TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx/).
+*   [tpu-users](https://groups.google.com/a/tensorflow.org/d/forum/tpu-users) -
+    Community discussion and support for TPU users.
+*   [xla-dev](https://groups.google.com/forum/#!forum/xla-dev) - Discussion for
+    developers contributing to the [XLA](https://www.tensorflow.org/xla)
+    compiler.
+
+## Special Interest Groups
+
+TensorFlow's
+[Special Interest Groups](https://github.com/tensorflow/community/tree/master/sigs) (SIGs)
+support community collaboration on particular project focuses. Members of these
+groups work together to build and support TensorFlow related projects. While their
+archives are public, different SIGs have their own membership policies.
+
+*   [addons](https://groups.google.com/a/tensorflow.org/d/forum/addons) -
+    Supporting SIG Addons, for extensions to TensorFlow that confirm to the
+    stable API.
+*   [build](https://groups.google.com/a/tensorflow.org/d/forum/build) -
+    Supporting SIG Build, for build, distribution and packaging of TensorFlow.
+*   [io](https://groups.google.com/a/tensorflow.org/d/forum/io) - Supporting SIG
+    IO, for file systems and formats not available in core TensorFlow.
+*   [jvm](https://groups.google.com/a/tensorflow.org/d/forum/jvm) - Supporting
+    SIG JVM, building Java and JVM support for TensorFlow.
+*   [keras](https://groups.google.com/forum/#!forum/keras-users) - Keras users
+    mailing list, for design reviews and discussions relating to SIG Keras.
+*   [micro](https://groups.google.com/a/tensorflow.org/d/forum/micro) -
+    Supporting SIG Micro, focusing on low power TF Lite deployment.
+*   [mlir](https://groups.google.com/a/tensorflow.org/d/forum/mlir) - Supporting
+    SIG MLIR, collaboration around MLIR, Multi-Level Intermediate
+    Representation.
+*   [networking](https://groups.google.com/a/tensorflow.org/d/forum/networking) -
+    Supporting SIG Networking, for adding network protocols other than gRPC.
+*   [rust](https://groups.google.com/a/tensorflow.org/d/forum/rust) - Supporting
+    SIG Rust, for the Rust language bindings.
+*   [swift](https://groups.google.com/a/tensorflow.org/d/forum/swift) -
+    Supporting SIG Swift, developing Swift for TensorFlow.
+*   [tensorboard](https://groups.google.com/a/tensorflow.org/d/forum/tensorboard) -
+    Supporting SIG TensorBoard, for plugin development and other contribution.
diff --git a/site/en/community/roadmap.md b/site/en/community/roadmap.md
deleted file mode 100644
index 1e66b97b25f..00000000000
--- a/site/en/community/roadmap.md
+++ /dev/null
@@ -1,141 +0,0 @@
-# Roadmap
-
-*Last updated: Jun 4, 2019*
-
-TensorFlow is a fast-moving, community supported project. This roadmap provides
-guidance about priorities and focus areas of the TensorFlow team and lists the
-functionality expected in upcoming releases of TensorFlow. Many of these areas
-are driven by community use cases, and we welcome further
-[contributions](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md)
-to TensorFlow.
-
-## TensorFlow 2.0 Beta is available
-
-[As announced previously](https://groups.google.com/a/tensorflow.org/forum/#!topic/discuss/bgug1G6a89A),
-we have been working on TensorFlow 2.0, which is a significant milestone and a
-major new release, with a focus on ease of use and simplification.
-
-The 2.0 Beta release is available now. Users can use this today and get started
-with all that TensorFlow 2.0 has to offer. This is an early version meant to
-share with users what the TensorFlow 2.0 API will be like, to gather feedback,
-and to identify and fix issues. Below are some of the key enhancements:
-
-*   Eager execution as a central feature of 2.0. It aligns users’ expectations
-    about the programming model better with TensorFlow practice and should make
-    TensorFlow easier to learn and apply.
-*   Keras tightly integrated with the TensorFlow ecosystem, and has support for
-    Eager execution, `tf.data` API, `tf.distribute.MirroredStrategy` for
-    multi-GPU training, TensorBoard visualization, and TF Lite and TF.js
-    conversion.
-*   Starter list of TF-Hub models loadable in TF 2.0.
-*   Autograph making it easier to write models with custom control flow ops and
-    getting graph performance with `tf.function`.
-*   Flexible API choices that let users build models quickly using high level
-    building blocks, or take full control by writing custom models and custom
-    training loops using lower level constructs.
-*   Support for TPUs using the TPUStrategy with `tf.estimator`.
-*   Support for multi worker synchronous training using `tf.distribute.Strategy`
-    with `tf.estimator`.
-*   New end-to-end ML framework for building ML pipelines with TFX.
-*   Simplified and cleaned-up API by removing deprecated APIs, reducing
-    redundancies, renaming symbols to more intuitive names, simplifying how
-    variables are treated.
-*   Removal of `tf.contrib` - These features have been either moved to
-    TensorFlow Core, or to tensorflow/addons, or are no longer part of the
-    TensorFlow build but are developed and maintained by their respective
-    owners.
-*   Updated and revised documentation, examples, and website, including
-    migration docs and TF 1.x to 2.0 converter guide.
-*   New releases of TensorFlow.js and Swift for TensorFlow packages.
-*   Many more add-ons and extensions (eg. TF Datasets, TF Federated, TF Privacy,
-    etc).
-
-## Roadmap
-
-After the TensorFlow 2.0 release, we will identify and fix issues, and test the
-TensorFlow 2.0 Beta with internal and external partners. The official 2.0
-release target is Q2 2019. This is a list of features on the short term roadmap
-and beyond:
-
-#### APIs
-
-*   Mixed precision training API in Keras.
-*   Premade estimators for boosted trees, random forest, approximate
-    nearest-neighbor search, k-means clustering, and more.
-*   `tf.distribute.Strategy` support for Keras subclass models, TPUs, and
-    multi-node training
-*   Improved support for model saving and loading `SavedModel`, and conversion
-    of existing 1.x TF-Hub modules
-
-#### Reference models
-
-*   Updated model repository with TF 2.0 best-practice reference models and
-    research model implementations. These will include ResNet, MobileNet,
-    DenseNet, Mask-RCNN, NMT, NCF, Transformer, and many other models
-*   Collection of [TF Hub modules](https://tfhub.dev/s?q=tf2-preview), loadable
-    in TensorFlow 2.0.
-*   More performance optimizations.
-
-#### TensorBoard
-
-*   General bug fixes and enhancements to make TensorBoard great with TensorFlow
-    2.0.
-*   Improvements to hyperparameter tuning capabilities and workflow.
-*   Enable hosted TensorBoard to make sharing dashboards easy and search/compare
-    experiments.
-*   Create a better plugin process for testing and adding TensorBoard plugins.
-*   Enable plugins to use different frontend technologies (like React).
-
-#### TensorFlow Lite
-
-*   Increase coverage of supported ops in TensorFlow Lite.
-*   Easier conversion of TensorFlow 2.0 models to use in TensorFlow Lite.
-*   Extended support for Edge TPUs, TPU AIY boards.
-*   More documentation and tutorials.
-
-#### TensorFlow.js
-
-*   Continued browser performance improvements.
-*   Implement prototype using compute shaders or WebGPU.
-*   Improve CPU performance, implement SIMD+ Web Assembly support (when
-    available).
-*   More off-the-shelf models for image, audio, text models, and more.
-*   Improve parity in features and performance on Node with core TensorFlow.
-*   Support for TensorBoard visualization in Node training with TensorFlow.js.
-*   Integration with more JavaScript platforms.
-*   More documentation and getting started content.
-
-#### TensorFlow with Swift
-
-*   Focus on researchers for the first half of 2019.
-*   Significant new feature development, such as support for transfer learning.
-*   Polish existing features, such as control flow support by the AutoDiff
-    system.
-*   Deeper integrations with the TensorFlow ecosystem, such as TensorBoard.
-*   End-to-end tutorials for getting started on Swift for TensorFlow in Colab
-    and additional technical documentation.
-*   Collaboration with fast.ai
-
-#### TensorFlow Extended (TFX)
-
-*   Integration of all TFX components with TensorFlow 2.0.
-*   Fully orchestrated end-to-end workflows with common configuration and
-    metadata tracking.
-*   Advanced training features, such as warm-starting.
-*   Notebook embedded visualizations and experiment tracking,
-
-#### Extensions and add-ons
-
-*   Migration and TensorFlow 2.0 support for all major extensions to TensorFlow,
-    including Probability, Agents, Tensor2Tensor, TensorRT, and more.
-*   More data sets in TensorFlow Datasets.
-*   Documentation and resources for extension libraries.
-
-#### Community and engagement
-
-*   New resources for community discussion and feedback.
-*   Launch new SIGs aligned with TensorFlow roadmap.
-*   Gather and highlight novel TensorFlow use cases and applications.
-
-Track the progress of these features and TensorFlow 2.0 development in the
-[GitHub project tracker](https://github.com/orgs/tensorflow/projects/4).
diff --git a/site/en/community/sig_playbook.md b/site/en/community/sig_playbook.md
index 75e277d3d96..6ec7a554a5b 100644
--- a/site/en/community/sig_playbook.md
+++ b/site/en/community/sig_playbook.md
@@ -55,7 +55,7 @@ must demonstrate:
     application area)
 *   Two or more contributors willing to act as group leads, existence of other
     contributors, and evidence of demand for the group
-*   Resources it will initially require (usually, mailing list and regular VC
+*   Resources it will initially require (usually, mailing list and regular video conference 
     call.) 
 
 Approval for the group will be given by a decision of the TF Community Team,
diff --git a/site/en/datasets/README.md b/site/en/datasets/README.md
deleted file mode 100644
index 422d0b5c9a3..00000000000
--- a/site/en/datasets/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Welcome to the warp zone!
-
-# TensorFlow Datasets
-
-These docs are available here: https://github.com/tensorflow/datasets/tree/master/docs
diff --git a/site/en/federated/README.md b/site/en/federated/README.md
deleted file mode 100644
index 945e5018dd4..00000000000
--- a/site/en/federated/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Welcome to the warp zone!
-
-# TensorFlow Federated
-
-These docs are available here: https://github.com/tensorflow/federated/tree/master/docs
diff --git a/site/en/graphics/README.md b/site/en/graphics/README.md
deleted file mode 100644
index 9182fae0c12..00000000000
--- a/site/en/graphics/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Welcome to the warp zone!
-
-# TensorFlow Graphics
-
-These docs are available here: https://github.com/tensorflow/graphics/tree/master/tensorflow_graphics/g3doc
diff --git a/site/en/guide/_graph_optimization.ipynb b/site/en/guide/_graph_optimization.ipynb
deleted file mode 100644
index f100b5453ce..00000000000
--- a/site/en/guide/_graph_optimization.ipynb
+++ /dev/null
@@ -1,463 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.6.9"
-    },
-    "colab": {
-      "name": "_graph_optimization.ipynb",
-      "provenance": [],
-      "private_outputs": true,
-      "toc_visible": true
-    }
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "FYLyuStTYesc",
-        "colab_type": "text"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "PVm-iEoxYesf",
-        "colab_type": "code",
-        "cellView": "form",
-        "colab": {}
-      },
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "3MPf91rVYesq",
-        "colab_type": "text"
-      },
-      "source": [
-        "# TensorFlow Graph Optimization"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "zmNCsZlgYesr",
-        "colab_type": "text"
-      },
-      "source": [
-        "\n",
-        "  \n",
-        "  \n",
-        "  \n",
-        "  \n",
-        "
\n",
-        "    View on TensorFlow.org\n",
-        "  \n",
-        "    Run in Google Colab\n",
-        "  \n",
-        "    View source on GitHub\n",
-        "  \n",
-        "    Download notebook\n",
-        "  
"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "l0qacLgyYess",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "TensorFlow optimizes the TensorFlow graph (built by `tf.function`) before executing operations.\n",
-        "This makes the TensorFlow graph more efficiently and less peak memory usage.\n",
-        "As most of optimizations are enabled automatically, you don't need to know what optimizations are perfomred under the TensorFlow.\n",
-        "\n",
-        "On the other hand, TensorFlow provides the way for the advanced TensorFlow users to enable/disable the optimization by using `tf.config.optimizer.set_experimental_options()`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WZAUsxyWYess",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "sQ0HvpqDYest",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x  #gpu\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "A-zkJgR5Yesw",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Graph Optimization performed in TensorFlow\n",
-        "\n",
-        "TensorFlow performs many optimizations to the TensorFlow Graph such as Constant Foling, Memory Optimizer, etc...  \n",
-        "All optimizations are listed in [tf.config.optimizer.set_experimental_options()](https://www.tensorflow.org/api_docs/python/tf/config/optimizer/set_experimental_options) API document page.\n",
-        "Below optimizations are typical optimizations performed in TensorFlow.\n",
-        "\n",
-        "* **Constant Folding** evaluates the operations whose input tensors are all constant.\n",
-        "* **Memory Optimizer** analyzes the TensorFlow graph to inspect the peak memory usage at each operations. To reduce a peak memory usage, Memory Optimizer inserts CPU-GPU memory copy operations for swapping GPU memory to CPU, or rewrites TensorFlow graph for the recomputation.\n",
-        "* **Layout Optimizer** inserts Transpose operation to change data-format in order to execute data-format depended operation more efficiently on GPU.\n",
-        "* **Arithmentic Optimizer** is an optimization to rewrite the TensorFlow graph using mathematical equivalence relation.\n",
-        "* **Debug Stripper** strips debug operations such as Assert.\n",
-        "\n",
-        "More detail information about the graph optimization can be found in [this slide](http://web.stanford.edu/class/cs245/slides/TFGraphOptimizationsStanford.pdf)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "Kx92OO9cYesy",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Checking Enabled Graph Optimization\n",
-        "\n",
-        "Enabled optimization configuration can be shown by calling `tf.config.optimizer.get_experimental_options()`.\n",
-        "\n",
-        "Below simple code shows the default optimization configuration."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "uRuhVoAlYesz",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.config.optimizer.get_experimental_options()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "qQQj5YA8Yes1",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Inspect the Optimized Graph\n",
-        "\n",
-        "The optimized graphs can be inspectable with using TensorBoard.\n",
-        "\n",
-        "Let us look at the case of \"Constant Folding\" optimization.\n",
-        "\"Constant Folding\" is enabled by default, so you will see the effects of the constant propagations without any procedures.\n",
-        "\n",
-        "To output TensorBoard summary data, you must call `tf.summary.trace_on()` before executing the graph.\n",
-        "All profile and graph data (includes optimized graphs) are stored in memory.\n",
-        "After the graph execution, TensorBoard summary data can be output by calling `tf.summary.trace_export()`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "GNy7cbz9Yes2",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "import tensorflow as tf\n",
-        "\n",
-        "@tf.function\n",
-        "def simple_func(arg):\n",
-        "    a = tf.constant(7.9)\n",
-        "    b = tf.constant(6.3)\n",
-        "    c = arg + a\n",
-        "    d = a * b\n",
-        "    ret = c + d\n",
-        "    \n",
-        "    return ret\n",
-        "\n",
-        "# Enable tracing data to inspect the optimized graph.\n",
-        "writer = tf.summary.create_file_writer(\"summary\")\n",
-        "tf.summary.trace_on(graph=True, profiler=True)\n",
-        "\n",
-        "arg = tf.constant(8.9)\n",
-        "print(simple_func(arg))\n",
-        "\n",
-        "# Output traced data as TensorBoard summary data.\n",
-        "with writer.as_default():\n",
-        "    tf.summary.trace_export(\"summary\", step=0, profiler_outdir=\"summary\")\n",
-        "\n",
-        "# Disable tracing data.\n",
-        "tf.summary.trace_off()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WD1-APylYes4",
-        "colab_type": "text"
-      },
-      "source": [
-        "Then, launch TensorBoard to inspect the optimized graph.\n",
-        "\n",
-        "The optimized graph is inspectable via Profile graph.\n",
-        "The first image shows the user defined graph, and the second image shows the optimized graph.\n",
-        "We can notice that mul node and it's input (Const) nodes are missing in the optimized graph.\n",
-        "This indicates that Constant Folding works well against the user defined graph.\n",
-        "\n",
-        "![User Defined Graph](https://github.com/nuka137/docs/blob/graph_optimization/site/en/guide/images/graph_optimization/user_defined_graph.png?raw=1)\n",
-        "\n",
-        "![Optimized Graph](https://github.com/nuka137/docs/blob/graph_optimization/site/en/guide/images/graph_optimization/optimized_graph.png?raw=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "RmzHjejOYes5",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Enable/Disable Graph Optimization\n",
-        "\n",
-        "Most of graph optimizations can be enabled/disabled by calling `tf.config.optimizer.set_experimental_options()`.\n",
-        "\n",
-        "Below simple code shows how to enable/disable the optimization."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "p-2EaOqmYes6",
-        "colab_type": "text"
-      },
-      "source": [
-        "### Disable \"Debug Stripper\"\n",
-        "\n",
-        "**Debug Stripper** which is disabled by default, strips the operations used for the debug purpose (Assert, CheckNumerics, Print).\n",
-        "Below code will raise an exception at `tf.Assert`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "NVpD71S1Yes8",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "import tensorflow as tf\n",
-        "\n",
-        "@tf.function\n",
-        "def assert_func():\n",
-        "    a = tf.constant(1.2)\n",
-        "    computation_graph = tf.Assert(tf.less_equal(a, 1.0), [a])   # Will raise an \"InvalidArgumentError\" exception.\n",
-        "    return a\n",
-        "\n",
-        "print(assert_func())"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "jTX6Z1MNYes-",
-        "colab_type": "text"
-      },
-      "source": [
-        "If you make Debug Stripper enabled, below code does not raise any exceptions."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "fxQjZu82Yes_",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "import tensorflow as tf\n",
-        "\n",
-        "# Enable \"Debug Stripper\".\n",
-        "tf.config.optimizer.set_experimental_options({'debug_stripper': True})\n",
-        "\n",
-        "@tf.function\n",
-        "def assert_func():\n",
-        "    a = tf.constant(1.2)\n",
-        "    computation_graph = tf.Assert(tf.less_equal(a, 1.0), [a])   # No exceptions are raised.\n",
-        "    return a\n",
-        "\n",
-        "print(assert_func())"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BY_L9lu-YetB",
-        "colab_type": "text"
-      },
-      "source": [
-        "### Disable All Graph Optimizations\n",
-        "\n",
-        "All Graph Optimization can be disabled when `'disable_meta_optimizer': False` is passed to `tf.config.optimizer.set_experimental_options()`.\n",
-        "\n",
-        "At first, let us check the optimized graph with default optimization configuration.\n",
-        "Below code builds the graph which has a seqence of Transpose operations."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "SePP0NEkYetC",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "import tensorflow as tf\n",
-        "import numpy as np\n",
-        "\n",
-        "@tf.function\n",
-        "def optimized(arg):\n",
-        "    a = arg * 2\n",
-        "\n",
-        "    # Will be simplified by Arithmetic Optimizer.\n",
-        "    b = tf.transpose(a, perm=[1, 0])\n",
-        "    ret = tf.transpose(b, perm=[1, 0])\n",
-        "\n",
-        "    return ret\n",
-        "\n",
-        "writer = tf.summary.create_file_writer(\"summary\")\n",
-        "tf.summary.trace_on(graph=True, profiler=True)\n",
-        "\n",
-        "arg = tf.constant(np.random.normal(size=(30, 40)))\n",
-        "optimized(arg)\n",
-        "\n",
-        "with writer.as_default():\n",
-        "    tf.summary.trace_export(\"summary\", step=0, profiler_outdir=\"summary\")\n",
-        "\n",
-        "tf.summary.trace_off()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4TGgrQ0vYetF",
-        "colab_type": "text"
-      },
-      "source": [
-        "As you can see in TensorBoard, Transpose operation was erased by Arithmetic Optimizer to simplify Transpose operation into NoOp.\n",
-        "\n",
-        "![Meta Optimizer Enabled](https://github.com/nuka137/docs/blob/graph_optimization/site/en/guide/images/graph_optimization/meta_optimizer_enabled.png?raw=1)\n",
-        "\n",
-        "Next, let us check the graph with `'disable_meta_optimizer'` on."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "P3pFWhdyYetG",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "import tensorflow as tf\n",
-        "import numpy as np\n",
-        "\n",
-        "# Disable all graph optimizations.\n",
-        "tf.config.optimizer.set_experimental_options({'disable_meta_optimizer': True})\n",
-        "\n",
-        "@tf.function\n",
-        "def not_optimized(arg):\n",
-        "    a = arg * 2\n",
-        "    b = tf.transpose(a, perm=[1, 0])\n",
-        "    ret = tf.transpose(b, perm=[1, 0])\n",
-        "\n",
-        "    return ret\n",
-        "\n",
-        "writer = tf.summary.create_file_writer(\"summary\")\n",
-        "tf.summary.trace_on(graph=True, profiler=True)\n",
-        "\n",
-        "arg = tf.constant(np.random.normal(size=(30, 40)))\n",
-        "not_optimized(arg)\n",
-        "\n",
-        "with writer.as_default():\n",
-        "    tf.summary.trace_export(\"summary\", step=0, profiler_outdir=\"summary\")\n",
-        "\n",
-        "tf.summary.trace_off()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "QKO5us9gYetJ",
-        "colab_type": "text"
-      },
-      "source": [
-        "In this graph, Transpose operations are remained in the graph, which indicates Arithmetic Optimizer was disabled.\n",
-        "\n",
-        "![Meta Optimizer Disabled](https://github.com/nuka137/docs/blob/graph_optimization/site/en/guide/images/graph_optimization/meta_optimizer_disabled.png?raw=1)"
-      ]
-    }
-  ]
-}
\ No newline at end of file
diff --git a/site/en/guide/_index.yaml b/site/en/guide/_index.yaml
index e14ad6d380c..e39dd37ead5 100644
--- a/site/en/guide/_index.yaml
+++ b/site/en/guide/_index.yaml
@@ -1,9 +1,14 @@
 book_path: /overview/_book.yaml
 project_path: /overview/_project.yaml
-description: 
+title: Guide
 landing_page:
   custom_css_path: /site-assets/css/style.css
   nav: left
+  meta_tags:
+  - name: description
+    content: >
+      Learn basic and advanced concepts of TensorFlow such as eager execution, Keras
+      high-level APIs and flexible model building.
   rows:
   - classname:
       devsite-landing-row-100
@@ -32,9 +37,9 @@ landing_page:
       path: /install
     - classname: tfo-landing-page-card
       description: >
-        
TensorFlow 2

-        TensorFlow 2 best practices and tools to migrate your code.
-      path: /guide/effective_tf2
+        
Migrate to TensorFlow 2

+        Learn how to migrate your TF1.x code to TF2.
+      path: /guide/migrate
     - classname: tfo-landing-page-card
       description: >
         
Keras

@@ -45,8 +50,9 @@ landing_page:
   - items:
     - classname: tfo-landing-page-card
       description: >
-        
Customization

-        TensorFlow for research and experimentation. Write custom layers and models, forward pass, and training loops.
+        
TensorFlow basics

+        Learn about the fundamental classes and features that make TensorFlow
+        work.
       path: /guide/eager
     - classname: tfo-landing-page-card
       description: >
@@ -56,10 +62,9 @@ landing_page:
       path: /guide/data
     - classname: tfo-landing-page-card
       description: >
-        
Estimators

-        A high-level API that represents a complete model, designed for scaling
-        and asynchronous training.
-      path: /guide/estimator
+        
TensorFlow 2 best practices

+        Learn about the best practices for effective development using TensorFlow 2.
+      path: /guide/effective_tf2
 
   - items:
     - classname: tfo-landing-page-card
@@ -72,6 +77,11 @@ landing_page:
         
Accelerators

         Distribute training across multiple GPUs, multiple machines or TPUs.
       path: /guide/distributed_training
+    - classname: tfo-landing-page-card
+      description: >
+        
Performance

+        Best practices and optimization techniques for optimal TensorFlow performance.
+      path: /guide/function
 
   - classname: devsite-landing-row-100
     items:
@@ -85,10 +95,11 @@ landing_page:
     items:
     - list:
       - description: >
-          
TensorBoard

-          A suite of visualization tools to understand, debug, and optimize
-          TensorFlow programs.
-        path: /tensorboard
+          
TensorFlow Decision Forests
+          

+          A library to train, run and interpret decision forest models (e.g., Random Forests,
+          Gradient Boosted Trees) in TensorFlow.
+        path: /decision_forests
         icon:
           icon_name: chevron_right
           foreground: theme
@@ -103,10 +114,10 @@ landing_page:
           foreground: theme
           background: grey
       - description: >
-          
Model Optimization

-          The TensorFlow Model Optimization Toolkit is a suite of tools for
-          optimizing ML models for deployment and execution.
-        path: /model_optimization
+          
Serving

+          A TFX serving system for ML models, designed for high-performance in
+          production environments.
+        path: /tfx/guide/serving
         icon:
           icon_name: chevron_right
           foreground: theme
@@ -137,7 +148,24 @@ landing_page:
           icon_name: chevron_right
           foreground: theme
           background: grey
+      - description: >
+          
SIG Addons

+          Extra functionality for TensorFlow, maintained by SIG Addons.
+        path: https://github.com/tensorflow/addons
+        icon:
+          icon_name: chevron_right
+          foreground: theme
+          background: grey
     - list:
+      - description: >
+          
TensorBoard

+          A suite of visualization tools to understand, debug, and optimize
+          TensorFlow programs.
+        path: /tensorboard
+        icon:
+          icon_name: chevron_right
+          foreground: theme
+          background: grey
       - description: >
           
Datasets

           A collection of datasets ready to use with TensorFlow.
@@ -147,10 +175,10 @@ landing_page:
           foreground: theme
           background: grey
       - description: >
-          
Serving

-          A TFX serving system for ML models, designed for high-performance in
-          production environments.
-        path: /tfx/guide/serving
+          
Model Optimization

+          The TensorFlow Model Optimization Toolkit is a suite of tools for
+          optimizing ML models for deployment and execution.
+        path: /model_optimization
         icon:
           icon_name: chevron_right
           foreground: theme
@@ -182,14 +210,6 @@ landing_page:
           icon_name: chevron_right
           foreground: theme
           background: grey
-      - description: >
-          
SIG Addons

-          Extra functionality for TensorFlow, maintained by SIG Addons.
-        path: https://github.com/tensorflow/addons
-        icon:
-          icon_name: chevron_right
-          foreground: theme
-          background: grey
       - description: >
           
SIG IO

           Dataset, streaming, and file system extensions, maintained by SIG IO.
diff --git a/site/en/guide/_toc.yaml b/site/en/guide/_toc.yaml
index dfae14918f5..92e5d6a80c3 100644
--- a/site/en/guide/_toc.yaml
+++ b/site/en/guide/_toc.yaml
@@ -2,76 +2,127 @@ toc:
 - title: "TensorFlow guide"
   path: /guide/
 
-- heading: "TensorFlow 2"
-- title: "Effective TensorFlow"
-  path: /guide/effective_tf2
-- title: "Migrate from TF1 to TF2"
-  path: /guide/migrate
-- title: "Convert with the upgrade script"
-  path: /guide/upgrade
-- title: "Performance with tf.function"
-  path: /guide/function
-- title: "Community testing FAQ"
-  path: https://github.com/tensorflow/community/blob/master/sigs/testing/faq.md
-  status: external
+- heading: "TensorFlow basics"
+- title: "Overview"
+  path: /guide/basics
+- title: "Tensors"
+  path: /guide/tensor
+- title: "Variables"
+  path: /guide/variable
+- title: "Automatic differentiation"
+  path: /guide/autodiff
+- title: "Graphs and functions"
+  path: /guide/intro_to_graphs
+- title: "Modules, layers, and models"
+  path: /guide/intro_to_modules
+- title: "Training loops"
+  path: /guide/basic_training_loops
 
 - heading: "Keras"
-- title: "Keras overview"
-  path: /guide/keras/overview
-- title: "Keras functional API"
-  path: /guide/keras/functional
-- title: "Train and evaluate"
-  path: /guide/keras/train_and_evaluate
-- title: "Write custom layers and models"
-  path: /guide/keras/custom_layers_and_models
-- title: "Save and serialize models"
-  path: /guide/keras/save_and_serialize
-- title: "Keras Recurrent Neural Networks"
-  path: /guide/keras/rnn
-- title: "Masking and padding"
-  path: /guide/keras/masking_and_padding
-- title: "Write custom callbacks"
-  path: /guide/keras/custom_callback
-- title: "Mixed precision"
-  path: /guide/keras/mixed_precision
-  status: experimental
+- title: "Overview"
+  path: /guide/keras
+- include: /guide/keras/_toc.yaml
 
-- heading: "Estimators"
-- title: "Estimator overview"
-  path: /guide/estimator
+- heading: "Build with Core"
+  status: new
+- title: "Overview"
+  path: /guide/core/index
+- title: "Quickstart for Core"
+  path: /guide/core/quickstart_core
+- title: "Logistic regression"
+  path: /guide/core/logistic_regression_core
+- title: "Multilayer perceptrons"
+  path: /guide/core/mlp_core
+- title: "Matrix approximation"
+  path: /guide/core/matrix_core
+- title: "Custom optimizers"
+  path: /guide/core/optimizers_core
+- title: "DTensor with Core APIs"
+  path: /guide/core/distribution
+  status: experimental
 
-- heading: "Customization"
-- title: "Eager execution"
-  path: /guide/eager
-- title: "Variable"
-  path: /guide/variable
-- title: "Tensor"
-  path: /guide/tensor
+- heading: "TensorFlow in depth"
+- title: "Tensor slicing"
+  path: /guide/tensor_slicing
+- title: "Advanced autodiff"
+  path: /guide/advanced_autodiff
 - title: "Ragged tensor"
   path: /guide/ragged_tensor
+- title: "Sparse tensor"
+  path: /guide/sparse_tensor
+- title: "Random number generation"
+  path: /guide/random_numbers
+- title: "NumPy API"
+  status: experimental
+  path: /guide/tf_numpy
+- title: "NumPy API Type Promotion"
+  status: nightly
+  path: /guide/tf_numpy_type_promotion
+- title: "DTensor concepts"
+  path: /guide/dtensor_overview
+  status: experimental
+- title: "Thinking in TensorFlow 2"
+  path: /guide/effective_tf2
+
+- heading: "Customization"
 - title: "Create an op"
   path: /guide/create_op
+- title: "Extension types"
+  path: /guide/extension_type
+  status: experimental
 
 - heading: "Data input pipelines"
 - title: "tf.data"
   path: /guide/data
 - title: "Optimize pipeline performance"
   path: /guide/data_performance
+- title: "Analyze pipeline performance"
+  path: /guide/data_performance_analysis
 
-- heading: "Save a model"
+- heading: "Import and export"
 - title: "Checkpoint"
   path: /guide/checkpoint
 - title: "SavedModel"
   path: /guide/saved_model
-- title: "Concrete functions"
-  path: /guide/concrete_function
-
+- title: "Import a JAX model using JAX2TF"
+  status: new
+  path: /guide/jax2tf
 
 - heading: "Accelerators"
 - title: "Distributed training"
   path: /guide/distributed_training
 - title: "GPU"
   path: /guide/gpu
+- title: "TPU"
+  path: /guide/tpu
+
+- heading: "Performance"
+- title: "Better performance with tf.function"
+  path: /guide/function
+- title: "Profile TensorFlow performance"
+  path: /guide/profiler
+- title: "Optimize GPU Performance"
+  path: /guide/gpu_performance_analysis
+- title: "Graph optimization"
+  path: /guide/graph_optimization
+- title: "Mixed precision"
+  path: /guide/mixed_precision
+
+- heading: "Model Garden"
+  status: new
+- title: "Overview"
+  path: /tfmodels
+- title: "Training with Orbit"
+  path: /tfmodels/orbit
+- title: "TFModels - NLP"
+  path: /tfmodels/nlp
+  status: external
+- include: /tfmodels/vision/_toc.yaml
+
+- heading: "Estimators"
+  status: deprecated
+- title: "Estimator overview"
+  path: /guide/estimator
 
 - heading: "Appendix"
 - title: "Version compatibility"
diff --git a/site/en/guide/advanced_autodiff.ipynb b/site/en/guide/advanced_autodiff.ipynb
new file mode 100644
index 00000000000..e04b9db4d77
--- /dev/null
+++ b/site/en/guide/advanced_autodiff.ipynb
@@ -0,0 +1,1133 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qFdPvlXBOdUN"
+      },
+      "source": [
+        "# Advanced automatic differentiation"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8a859404ce7e"
+      },
+      "source": [
+        "The [Introduction to gradients and automatic differentiation](autodiff.ipynb) guide includes everything required to calculate gradients in TensorFlow. This guide focuses on deeper, less common features of the `tf.GradientTape` API."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MUXex9ctTuDB"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IqR2PQG4ZaZ0"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "import matplotlib as mpl\n",
+        "import matplotlib.pyplot as plt\n",
+        "\n",
+        "mpl.rcParams['figure.figsize'] = (8, 6)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uGRJJRi8TCkJ"
+      },
+      "source": [
+        "## Controlling gradient recording\n",
+        "\n",
+        "In the [automatic differentiation guide](autodiff.ipynb) you saw how to control which variables and tensors are watched by the tape while building the gradient calculation.\n",
+        "\n",
+        "The tape also has methods to manipulate the recording."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gB_i0VnhQKt2"
+      },
+      "source": [
+        "### Stop recording\n",
+        "\n",
+        "If you wish to stop recording gradients, you can use `tf.GradientTape.stop_recording` to temporarily suspend recording.\n",
+        "\n",
+        "This may be useful to reduce overhead if you do not wish to differentiate a complicated operation in the middle of your model.  This could include calculating a metric or an intermediate result:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mhFSYf7uQWxR"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(2.0)\n",
+        "y = tf.Variable(3.0)\n",
+        "\n",
+        "with tf.GradientTape() as t:\n",
+        "  x_sq = x * x\n",
+        "  with t.stop_recording():\n",
+        "    y_sq = y * y\n",
+        "  z = x_sq + y_sq\n",
+        "\n",
+        "grad = t.gradient(z, {'x': x, 'y': y})\n",
+        "\n",
+        "print('dz/dx:', grad['x'])  # 2*x => 4\n",
+        "print('dz/dy:', grad['y'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DEHbEZ1h4p8A"
+      },
+      "source": [
+        "### Reset/start recording from scratch\n",
+        "\n",
+        "If you wish to start over entirely, use `tf.GradientTape.reset`. Simply exiting the gradient tape block and restarting is usually easier to read, but you can use the `reset` method when exiting the tape block is difficult or impossible."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lsMHsmrh4pqM"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(2.0)\n",
+        "y = tf.Variable(3.0)\n",
+        "reset = True\n",
+        "\n",
+        "with tf.GradientTape() as t:\n",
+        "  y_sq = y * y\n",
+        "  if reset:\n",
+        "    # Throw out all the tape recorded so far.\n",
+        "    t.reset()\n",
+        "  z = x * x + y_sq\n",
+        "\n",
+        "grad = t.gradient(z, {'x': x, 'y': y})\n",
+        "\n",
+        "print('dz/dx:', grad['x'])  # 2*x => 4\n",
+        "print('dz/dy:', grad['y'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6zS7cLmS6zMf"
+      },
+      "source": [
+        "## Stop gradient flow with precision\n",
+        "\n",
+        "In contrast to the global tape controls above, the `tf.stop_gradient` function is much more precise. It can be used to stop gradients from flowing along a particular path, without needing access to the tape itself:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "30qnZMe48BkB"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(2.0)\n",
+        "y = tf.Variable(3.0)\n",
+        "\n",
+        "with tf.GradientTape() as t:\n",
+        "  y_sq = y**2\n",
+        "  z = x**2 + tf.stop_gradient(y_sq)\n",
+        "\n",
+        "grad = t.gradient(z, {'x': x, 'y': y})\n",
+        "\n",
+        "print('dz/dx:', grad['x'])  # 2*x => 4\n",
+        "print('dz/dy:', grad['y'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mbb-9lnGVngH"
+      },
+      "source": [
+        "## Custom gradients\n",
+        "\n",
+        "In some cases, you may want to control exactly how gradients are calculated rather than using the default. These situations include:\n",
+        "\n",
+        "1. There is no defined gradient for a new op you are writing.\n",
+        "2. The default calculations are numerically unstable.\n",
+        "3. You wish to cache an expensive computation from the forward pass.\n",
+        "4. You want to modify  a value (for example, using `tf.clip_by_value` or `tf.math.round`) without modifying the gradient.\n",
+        "\n",
+        "For the first case, to write a new op you can use `tf.RegisterGradient` to set up your own (refer to the API docs for details). (Note that the gradient registry is global, so change it with caution.)\n",
+        "\n",
+        "For the latter three cases, you can use `tf.custom_gradient`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oHr31kc_irF_"
+      },
+      "source": [
+        "Here is an example that applies `tf.clip_by_norm` to the intermediate gradient:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Mjj01w4NYtwd"
+      },
+      "outputs": [],
+      "source": [
+        "# Establish an identity operation, but clip during the gradient pass.\n",
+        "@tf.custom_gradient\n",
+        "def clip_gradients(y):\n",
+        "  def backward(dy):\n",
+        "    return tf.clip_by_norm(dy, 0.5)\n",
+        "  return y, backward\n",
+        "\n",
+        "v = tf.Variable(2.0)\n",
+        "with tf.GradientTape() as t:\n",
+        "  output = clip_gradients(v * v)\n",
+        "print(t.gradient(output, v))  # calls \"backward\", which clips 4 to 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n4t7S0scYrD3"
+      },
+      "source": [
+        "Refer to the `tf.custom_gradient` decorator API docs for more details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v0ODp4Oi--I0"
+      },
+      "source": [
+        "### Custom gradients in SavedModel\n",
+        "\n",
+        "Note: This feature is available from TensorFlow 2.6.\n",
+        "\n",
+        "Custom gradients can be saved to SavedModel by using the option `tf.saved_model.SaveOptions(experimental_custom_gradients=True)`.\n",
+        "\n",
+        "To be saved into the SavedModel, the gradient function must be traceable (to learn more, check out the [Better performance with tf.function](function.ipynb) guide)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q5JBgIBYjN1I"
+      },
+      "outputs": [],
+      "source": [
+        "class MyModule(tf.Module):\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(None)])\n",
+        "  def call_custom_grad(self, x):\n",
+        "    return clip_gradients(x)\n",
+        "\n",
+        "model = MyModule()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xZTrgy2q-9pq"
+      },
+      "outputs": [],
+      "source": [
+        "tf.saved_model.save(\n",
+        "    model,\n",
+        "    'saved_model',\n",
+        "    options=tf.saved_model.SaveOptions(experimental_custom_gradients=True))\n",
+        "\n",
+        "# The loaded gradients will be the same as the above example.\n",
+        "v = tf.Variable(2.0)\n",
+        "loaded = tf.saved_model.load('saved_model')\n",
+        "with tf.GradientTape() as t:\n",
+        "  output = loaded.call_custom_grad(v * v)\n",
+        "print(t.gradient(output, v))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d-LfRs5FbJCk"
+      },
+      "source": [
+        "A note about the above example: If you try replacing the above code with `tf.saved_model.SaveOptions(experimental_custom_gradients=False)`, the gradient will still produce the same result on loading. The reason is that the gradient registry still contains the custom gradient used in the function `call_custom_op`. However, if you restart the runtime after saving without custom gradients, running the loaded model under the `tf.GradientTape` will throw the error: `LookupError: No gradient defined for operation 'IdentityN' (op type: IdentityN)`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8aENEt6Veryb"
+      },
+      "source": [
+        "## Multiple tapes\n",
+        "\n",
+        "Multiple tapes interact seamlessly.\n",
+        "\n",
+        "For example, here each tape watches a different set of tensors:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BJ0HdMvte0VZ"
+      },
+      "outputs": [],
+      "source": [
+        "x0 = tf.constant(0.0)\n",
+        "x1 = tf.constant(0.0)\n",
+        "\n",
+        "with tf.GradientTape() as tape0, tf.GradientTape() as tape1:\n",
+        "  tape0.watch(x0)\n",
+        "  tape1.watch(x1)\n",
+        "\n",
+        "  y0 = tf.math.sin(x0)\n",
+        "  y1 = tf.nn.sigmoid(x1)\n",
+        "\n",
+        "  y = y0 + y1\n",
+        "\n",
+        "  ys = tf.reduce_sum(y)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6ApAoMNFfNz6"
+      },
+      "outputs": [],
+      "source": [
+        "tape0.gradient(ys, x0).numpy()   # cos(x) => 1.0"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rF1jrAJsfYW_"
+      },
+      "outputs": [],
+      "source": [
+        "tape1.gradient(ys, x1).numpy()   # sigmoid(x1)*(1-sigmoid(x1)) => 0.25"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DK05KXrAAld3"
+      },
+      "source": [
+        "### Higher-order gradients\n",
+        "\n",
+        "Operations inside of the `tf.GradientTape` context manager are recorded for automatic differentiation. If gradients are computed in that context, then the gradient computation is recorded as well. As a result, the exact same API works for higher-order gradients as well.\n",
+        "\n",
+        "For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cPQgthZ7ugRJ"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(1.0)  # Create a Tensorflow variable initialized to 1.0\n",
+        "\n",
+        "with tf.GradientTape() as t2:\n",
+        "  with tf.GradientTape() as t1:\n",
+        "    y = x * x * x\n",
+        "\n",
+        "  # Compute the gradient inside the outer `t2` context manager\n",
+        "  # which means the gradient computation is differentiable as well.\n",
+        "  dy_dx = t1.gradient(y, x)\n",
+        "d2y_dx2 = t2.gradient(dy_dx, x)\n",
+        "\n",
+        "print('dy_dx:', dy_dx.numpy())  # 3 * x**2 => 3.0\n",
+        "print('d2y_dx2:', d2y_dx2.numpy())  # 6 * x => 6.0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k0HV-Ah4_76i"
+      },
+      "source": [
+        "While that does give you the second derivative of a _scalar_ function, this pattern does not generalize to produce a Hessian matrix, since `tf.GradientTape.gradient` only computes the gradient of a scalar. To construct a [Hessian matrix](https://en.wikipedia.org/wiki/Hessian_matrix), go to the [Hessian example](#hessian) under the [Jacobian section](#jacobians).\n",
+        "\n",
+        "\"Nested calls to `tf.GradientTape.gradient`\" is a good pattern when you are calculating a scalar from a gradient, and then the resulting scalar acts as a source for a second gradient calculation, as in the following example.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "t7LRlcpVKHv1"
+      },
+      "source": [
+        "#### Example: Input gradient regularization\n",
+        "\n",
+        "Many models are susceptible to \"adversarial examples\". This collection of techniques modifies the model's input to confuse the model's output. The simplest implementation—such as the [Adversarial example using the Fast Gradient Signed Method attack](https://www.tensorflow.org/tutorials/generative/adversarial_fgsm)—takes a single step along the gradient of the output with respect to the input; the \"input gradient\".\n",
+        "\n",
+        "One technique to increase robustness to adversarial examples is [input gradient regularization](https://arxiv.org/abs/1905.11468) (Finlay & Oberman, 2019), which attempts to minimize the magnitude of the input gradient. If the input gradient is small, then the change in the output should be small too.\n",
+        "\n",
+        "Below is a naive implementation of input gradient regularization. The implementation is:\n",
+        "\n",
+        "1. Calculate the gradient of the output with respect to the input using an inner tape.\n",
+        "2. Calculate the magnitude of that input gradient.\n",
+        "3. Calculate the gradient of that magnitude with respect to the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tH3ZFuUfDLrR"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.random.normal([7, 5])\n",
+        "\n",
+        "layer = tf.keras.layers.Dense(10, activation=tf.nn.relu)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E6yOFsjEDR9u"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.GradientTape() as t2:\n",
+        "  # The inner tape only takes the gradient with respect to the input,\n",
+        "  # not the variables.\n",
+        "  with tf.GradientTape(watch_accessed_variables=False) as t1:\n",
+        "    t1.watch(x)\n",
+        "    y = layer(x)\n",
+        "    out = tf.reduce_sum(layer(x)**2)\n",
+        "  # 1. Calculate the input gradient.\n",
+        "  g1 = t1.gradient(out, x)\n",
+        "  # 2. Calculate the magnitude of the input gradient.\n",
+        "  g1_mag = tf.norm(g1)\n",
+        "\n",
+        "# 3. Calculate the gradient of the magnitude with respect to the model.\n",
+        "dg1_mag = t2.gradient(g1_mag, layer.trainable_variables)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "123QMq6PqK_d"
+      },
+      "outputs": [],
+      "source": [
+        "[var.shape for var in dg1_mag]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "E4xiYigexMtQ"
+      },
+      "source": [
+        "## Jacobians\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4-hVHVIeExkI"
+      },
+      "source": [
+        "All the previous examples took the gradients of a scalar target with respect to some source tensor(s).\n",
+        "\n",
+        "The [Jacobian matrix](https://en.wikipedia.org/wiki/Jacobian_matrix_and_determinant) represents the gradients of a vector valued function. Each row contains the gradient of one of the vector's elements.\n",
+        "\n",
+        "The `tf.GradientTape.jacobian` method allows you to efficiently calculate a Jacobian matrix."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KzNyIM0QBYIH"
+      },
+      "source": [
+        "Note that:\n",
+        "\n",
+        "* Like `gradient`: The `sources` argument can be a tensor or a container of tensors.\n",
+        "* Unlike `gradient`: The `target` tensor must be a single tensor."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O74K3hlxBC8a"
+      },
+      "source": [
+        "### Scalar source"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "B08OKn1Orkuc"
+      },
+      "source": [
+        "As a first example, here is the Jacobian of a vector-target with respect to a scalar-source."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bAFeIE8EuVIq"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.linspace(-10.0, 10.0, 200+1)\n",
+        "delta = tf.Variable(0.0)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  y = tf.nn.sigmoid(x+delta)\n",
+        "\n",
+        "dy_dx = tape.jacobian(y, delta)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BgHbUk3zr-WU"
+      },
+      "source": [
+        "When you take the Jacobian with respect to a scalar the result has the shape of the **target**, and gives the gradient of the each element with respect to the source:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iZ6awnDzr_BA"
+      },
+      "outputs": [],
+      "source": [
+        "print(y.shape)\n",
+        "print(dy_dx.shape)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "siNZaklc0_-e"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(x.numpy(), y, label='y')\n",
+        "plt.plot(x.numpy(), dy_dx, label='dy/dx')\n",
+        "plt.legend()\n",
+        "_ = plt.xlabel('x')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DsOMSD_1BGkD"
+      },
+      "source": [
+        "### Tensor source"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g3iXKN7KF-st"
+      },
+      "source": [
+        "Whether the input is scalar or tensor, `tf.GradientTape.jacobian` efficiently calculates the gradient of each element of the source with respect to each element of the target(s).\n",
+        "\n",
+        "For example, the output of this layer has a shape of `(10, 7)`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "39YXItgLxMBk"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.random.normal([7, 5])\n",
+        "layer = tf.keras.layers.Dense(10, activation=tf.nn.relu)\n",
+        "\n",
+        "with tf.GradientTape(persistent=True) as tape:\n",
+        "  y = layer(x)\n",
+        "\n",
+        "y.shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tshNRtfKuVP_"
+      },
+      "source": [
+        "And the layer's kernel's shape is `(5, 10)`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CigTWyfPvPuv"
+      },
+      "outputs": [],
+      "source": [
+        "layer.kernel.shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mN96JRpnAjpx"
+      },
+      "source": [
+        "The shape of the Jacobian of the output with respect to the kernel is those two shapes concatenated together:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pRLzTTbvEimH"
+      },
+      "outputs": [],
+      "source": [
+        "j = tape.jacobian(y, layer.kernel)\n",
+        "j.shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2Lrv7miMvTll"
+      },
+      "source": [
+        "If you sum over the target's dimensions, you're left with the gradient of the sum that would have been calculated by `tf.GradientTape.gradient`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FJjZpYRnDjVa"
+      },
+      "outputs": [],
+      "source": [
+        "g = tape.gradient(y, layer.kernel)\n",
+        "print('g.shape:', g.shape)\n",
+        "\n",
+        "j_sum = tf.reduce_sum(j, axis=[0, 1])\n",
+        "delta = tf.reduce_max(abs(g - j_sum)).numpy()\n",
+        "assert delta < 1e-3\n",
+        "print('delta:', delta)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZKajuGlk_krs"
+      },
+      "source": [
+        " \n",
+        "\n",
+        "#### Example: Hessian"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NYcsXeo8TDLi"
+      },
+      "source": [
+        "While `tf.GradientTape` doesn't give an explicit method for constructing a [Hessian matrix](https://en.wikipedia.org/wiki/Hessian_matrix) it's possible to build one using the `tf.GradientTape.jacobian` method.\n",
+        "\n",
+        "Note: The Hessian matrix contains `N**2` parameters. For this and other reasons it is not practical for most models. This example is included more as a demonstration of how to use the `tf.GradientTape.jacobian` method, and is not an endorsement of direct Hessian-based optimization. A Hessian-vector product can be [calculated efficiently with nested tapes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/eager/benchmarks/resnet50/hvp_test.py), and is a much more efficient approach to second-order optimization."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ELGTaell_j81"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.random.normal([7, 5])\n",
+        "layer1 = tf.keras.layers.Dense(8, activation=tf.nn.relu)\n",
+        "layer2 = tf.keras.layers.Dense(6, activation=tf.nn.relu)\n",
+        "\n",
+        "with tf.GradientTape() as t2:\n",
+        "  with tf.GradientTape() as t1:\n",
+        "    x = layer1(x)\n",
+        "    x = layer2(x)\n",
+        "    loss = tf.reduce_mean(x**2)\n",
+        "\n",
+        "  g = t1.gradient(loss, layer1.kernel)\n",
+        "\n",
+        "h = t2.jacobian(g, layer1.kernel)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FVqQuZj4XGjm"
+      },
+      "outputs": [],
+      "source": [
+        "print(f'layer.kernel.shape: {layer1.kernel.shape}')\n",
+        "print(f'h.shape: {h.shape}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_M7XElgaiMeP"
+      },
+      "source": [
+        "To use this Hessian for a [Newton's method](https://en.wikipedia.org/wiki/Newton%27s_method_in_optimization) step, you would first flatten out its axes into a matrix, and flatten out the gradient into a vector:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6te7N6wVXwXX"
+      },
+      "outputs": [],
+      "source": [
+        "n_params = tf.reduce_prod(layer1.kernel.shape)\n",
+        "\n",
+        "g_vec = tf.reshape(g, [n_params, 1])\n",
+        "h_mat = tf.reshape(h, [n_params, n_params])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L9rO8b-0mgOH"
+      },
+      "source": [
+        "The Hessian matrix should be symmetric:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8TCHc7Vrf52S"
+      },
+      "outputs": [],
+      "source": [
+        "def imshow_zero_center(image, **kwargs):\n",
+        "  lim = tf.reduce_max(abs(image))\n",
+        "  plt.imshow(image, vmin=-lim, vmax=lim, cmap='seismic', **kwargs)\n",
+        "  plt.colorbar()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DExOxd7Ok2H0"
+      },
+      "outputs": [],
+      "source": [
+        "imshow_zero_center(h_mat)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "13fBswmtQes4"
+      },
+      "source": [
+        "The Newton's method update step is shown below:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3DdnbynBdSor"
+      },
+      "outputs": [],
+      "source": [
+        "eps = 1e-3\n",
+        "eye_eps = tf.eye(h_mat.shape[0])*eps"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-zPdtyoWeUeV"
+      },
+      "source": [
+        "Note: [Don't  actually invert the matrix](https://www.johndcook.com/blog/2010/01/19/dont-invert-that-matrix/)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "k1LYftgmswOO"
+      },
+      "outputs": [],
+      "source": [
+        "# X(k+1) = X(k) - (∇²f(X(k)))^-1 @ ∇f(X(k))\n",
+        "# h_mat = ∇²f(X(k))\n",
+        "# g_vec = ∇f(X(k))\n",
+        "update = tf.linalg.solve(h_mat + eye_eps, g_vec)\n",
+        "\n",
+        "# Reshape the update and apply it to the variable.\n",
+        "_ = layer1.kernel.assign_sub(tf.reshape(update, layer1.kernel.shape))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pF6qjlHKWxF4"
+      },
+      "source": [
+        "While this is relatively simple for a single `tf.Variable`, applying this to a non-trivial model would require careful concatenation and slicing to produce a full Hessian across multiple variables."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PQWM0uN-GO5t"
+      },
+      "source": [
+        "### Batch Jacobian"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hKtB3rY6EySJ"
+      },
+      "source": [
+        "In some cases, you want to take the Jacobian of each of a stack of targets with respect to a stack of sources, where the Jacobians for each target-source pair are independent.\n",
+        "\n",
+        "For example, here the input `x` is shaped `(batch, ins)` and the output `y` is shaped `(batch, outs)`:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tQMndhIUHMes"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.random.normal([7, 5])\n",
+        "\n",
+        "layer1 = tf.keras.layers.Dense(8, activation=tf.nn.elu)\n",
+        "layer2 = tf.keras.layers.Dense(6, activation=tf.nn.elu)\n",
+        "\n",
+        "with tf.GradientTape(persistent=True, watch_accessed_variables=False) as tape:\n",
+        "  tape.watch(x)\n",
+        "  y = layer1(x)\n",
+        "  y = layer2(y)\n",
+        "\n",
+        "y.shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ff2spRHEJXBU"
+      },
+      "source": [
+        "The full Jacobian of `y` with respect to `x` has a shape of `(batch, ins, batch, outs)`, even if you only want `(batch, ins, outs)`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1zSl2A5-HhMH"
+      },
+      "outputs": [],
+      "source": [
+        "j = tape.jacobian(y, x)\n",
+        "j.shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UibJijPLJrpQ"
+      },
+      "source": [
+        "If the gradients of each item in the stack are independent, then every `(batch, batch)` slice of this tensor is a diagonal matrix:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZFl9uj3ueVSH"
+      },
+      "outputs": [],
+      "source": [
+        "imshow_zero_center(j[:, 0, :, 0])\n",
+        "_ = plt.title('A (batch, batch) slice')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "g4ZoRJcJNmy5"
+      },
+      "outputs": [],
+      "source": [
+        "def plot_as_patches(j):\n",
+        "  # Reorder axes so the diagonals will each form a contiguous patch.\n",
+        "  j = tf.transpose(j, [1, 0, 3, 2])\n",
+        "  # Pad in between each patch.\n",
+        "  lim = tf.reduce_max(abs(j))\n",
+        "  j = tf.pad(j, [[0, 0], [1, 1], [0, 0], [1, 1]],\n",
+        "             constant_values=-lim)\n",
+        "  # Reshape to form a single image.\n",
+        "  s = j.shape\n",
+        "  j = tf.reshape(j, [s[0]*s[1], s[2]*s[3]])\n",
+        "  imshow_zero_center(j, extent=[-0.5, s[2]-0.5, s[0]-0.5, -0.5])\n",
+        "\n",
+        "plot_as_patches(j)\n",
+        "_ = plt.title('All (batch, batch) slices are diagonal')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OXpTBKyeK84z"
+      },
+      "source": [
+        "To get the desired result, you can sum over the duplicate `batch` dimension, or else select the diagonals using `tf.einsum`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "v65OAjEgLQwl"
+      },
+      "outputs": [],
+      "source": [
+        "j_sum = tf.reduce_sum(j, axis=2)\n",
+        "print(j_sum.shape)\n",
+        "j_select = tf.einsum('bxby->bxy', j)\n",
+        "print(j_select.shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zT_VfR6lcwxD"
+      },
+      "source": [
+        "It would be much more efficient to do the calculation without the extra dimension in the first place. The `tf.GradientTape.batch_jacobian` method does exactly that:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YJLIl9WpHqYq"
+      },
+      "outputs": [],
+      "source": [
+        "jb = tape.batch_jacobian(y, x)\n",
+        "jb.shape"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-5t_q5SfHw7T"
+      },
+      "outputs": [],
+      "source": [
+        "error = tf.reduce_max(abs(jb - j_sum))\n",
+        "assert error < 1e-3\n",
+        "print(error.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IUeY2ZCiL31I"
+      },
+      "source": [
+        "Caution: `tf.GradientTape.batch_jacobian` only verifies that the first dimension of the source and target match. It doesn't check that the gradients are actually independent. It's up to you to make sure you only use `batch_jacobian` where it makes sense. For example, adding a `tf.keras.layers.BatchNormalization` destroys the independence, since it normalizes across the `batch` dimension:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tnDugVc-L4fj"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.random.normal([7, 5])\n",
+        "\n",
+        "layer1 = tf.keras.layers.Dense(8, activation=tf.nn.elu)\n",
+        "bn = tf.keras.layers.BatchNormalization()\n",
+        "layer2 = tf.keras.layers.Dense(6, activation=tf.nn.elu)\n",
+        "\n",
+        "with tf.GradientTape(persistent=True, watch_accessed_variables=False) as tape:\n",
+        "  tape.watch(x)\n",
+        "  y = layer1(x)\n",
+        "  y = bn(y, training=True)\n",
+        "  y = layer2(y)\n",
+        "\n",
+        "j = tape.jacobian(y, x)\n",
+        "print(f'j.shape: {j.shape}')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SNyZ1WhJMVLm"
+      },
+      "outputs": [],
+      "source": [
+        "plot_as_patches(j)\n",
+        "\n",
+        "_ = plt.title('These slices are not diagonal')\n",
+        "_ = plt.xlabel(\"Don't use `batch_jacobian`\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "M_x7ih5sarvG"
+      },
+      "source": [
+        "In this case, `batch_jacobian` still runs and returns _something_ with the expected shape, but its contents have an unclear meaning:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "k8_mICHoasCi"
+      },
+      "outputs": [],
+      "source": [
+        "jb = tape.batch_jacobian(y, x)\n",
+        "print(f'jb.shape: {jb.shape}')"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "advanced_autodiff.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/autodiff.ipynb b/site/en/guide/autodiff.ipynb
new file mode 100644
index 00000000000..237a224569b
--- /dev/null
+++ b/site/en/guide/autodiff.ipynb
@@ -0,0 +1,1034 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qFdPvlXBOdUN"
+      },
+      "source": [
+        "# Introduction to gradients and automatic differentiation"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r6P32iYYV27b"
+      },
+      "source": [
+        "## Automatic Differentiation and Gradients\n",
+        "\n",
+        "[Automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation)\n",
+        "is useful for implementing machine learning algorithms such as\n",
+        "[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training\n",
+        "neural networks.\n",
+        "\n",
+        "In this guide, you will explore ways to compute gradients with TensorFlow, especially in eager execution."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MUXex9ctTuDB"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IqR2PQG4ZaZ0"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "import matplotlib.pyplot as plt\n",
+        "\n",
+        "import tensorflow as tf"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xHxb-dlhMIzW"
+      },
+      "source": [
+        "## Computing gradients\n",
+        "\n",
+        "To differentiate automatically, TensorFlow needs to remember what operations happen in what order during the *forward* pass.  Then, during the *backward pass*, TensorFlow traverses this list of operations in reverse order to compute gradients."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1CLWJl0QliB0"
+      },
+      "source": [
+        "## Gradient tapes\n",
+        "\n",
+        "TensorFlow provides the `tf.GradientTape` API for automatic differentiation; that is, computing the gradient of a computation with respect to some inputs, usually `tf.Variable`s.\n",
+        "TensorFlow \"records\" relevant operations executed inside the context of a `tf.GradientTape` onto a \"tape\". TensorFlow then uses that tape to compute the gradients of a \"recorded\" computation using [reverse mode differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation).\n",
+        "\n",
+        "Here is a simple example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Xq9GgTCP7a4A"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(3.0)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  y = x**2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CR9tFAP_7cra"
+      },
+      "source": [
+        "Once you've recorded some operations, use `GradientTape.gradient(target, sources)` to calculate the gradient of some target (often a loss) relative to some source (often the model's variables):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LsvrwF6bHroC"
+      },
+      "outputs": [],
+      "source": [
+        "# dy = 2x * dx\n",
+        "dy_dx = tape.gradient(y, x)\n",
+        "dy_dx.numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q2_aqsO25Vx1"
+      },
+      "source": [
+        "The above example uses scalars, but `tf.GradientTape` works as easily on any tensor:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vacZ3-Ws5VdV"
+      },
+      "outputs": [],
+      "source": [
+        "w = tf.Variable(tf.random.normal((3, 2)), name='w')\n",
+        "b = tf.Variable(tf.zeros(2, dtype=tf.float32), name='b')\n",
+        "x = [[1., 2., 3.]]\n",
+        "\n",
+        "with tf.GradientTape(persistent=True) as tape:\n",
+        "  y = x @ w + b\n",
+        "  loss = tf.reduce_mean(y**2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i4eXOkrQ-9Pb"
+      },
+      "source": [
+        "To get the gradient of `loss` with respect to both variables, you can pass both as sources to the `gradient` method. The tape is flexible about how sources are passed and will accept any nested combination of lists or dictionaries and return the gradient structured the same way (see `tf.nest`)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "luOtK1Da_BR0"
+      },
+      "outputs": [],
+      "source": [
+        "[dl_dw, dl_db] = tape.gradient(loss, [w, b])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ei4iVXi6qgM7"
+      },
+      "source": [
+        "The gradient with respect to each source has the shape of the source:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aYbWRFPZqk4U"
+      },
+      "outputs": [],
+      "source": [
+        "print(w.shape)\n",
+        "print(dl_dw.shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dI_SzxHsvao1"
+      },
+      "source": [
+        "Here is the gradient calculation again, this time passing a dictionary of variables:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d73cY6NOuaMd"
+      },
+      "outputs": [],
+      "source": [
+        "my_vars = {\n",
+        "    'w': w,\n",
+        "    'b': b\n",
+        "}\n",
+        "\n",
+        "grad = tape.gradient(loss, my_vars)\n",
+        "grad['b']"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HZ2LvHifEMgO"
+      },
+      "source": [
+        "## Gradients with respect to a model\n",
+        "\n",
+        "It's common to collect `tf.Variables` into a `tf.Module` or one of its subclasses (`layers.Layer`, `keras.Model`) for [checkpointing](checkpoint.ipynb) and [exporting](saved_model.ipynb).\n",
+        "\n",
+        "In most cases, you will want to calculate gradients with respect to a model's trainable variables.  Since all subclasses of `tf.Module` aggregate their variables in the `Module.trainable_variables` property, you can calculate these gradients in a few lines of code: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JvesHtbQESc-"
+      },
+      "outputs": [],
+      "source": [
+        "layer = tf.keras.layers.Dense(2, activation='relu')\n",
+        "x = tf.constant([[1., 2., 3.]])\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  # Forward pass\n",
+        "  y = layer(x)\n",
+        "  loss = tf.reduce_mean(y**2)\n",
+        "\n",
+        "# Calculate gradients with respect to every trainable variable\n",
+        "grad = tape.gradient(loss, layer.trainable_variables)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PR_ezr6UFrpI"
+      },
+      "outputs": [],
+      "source": [
+        "for var, g in zip(layer.trainable_variables, grad):\n",
+        "  print(f'{var.name}, shape: {g.shape}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f6Gx6LS714zR"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "## Controlling what the tape watches"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N4VlqKFzzGaC"
+      },
+      "source": [
+        "The default behavior is to record all operations after accessing a trainable `tf.Variable`. The reasons for this are:\n",
+        "\n",
+        "* The tape needs to know which operations to record in the forward pass to calculate the gradients in the backwards pass.\n",
+        "* The tape holds references to intermediate outputs, so you don't want to record unnecessary operations.\n",
+        "* The most common use case involves calculating the gradient of a loss with respect to all a model's trainable variables.\n",
+        "\n",
+        "For example, the following fails to calculate a gradient because the `tf.Tensor` is not \"watched\" by default, and the `tf.Variable` is not trainable:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kj9gPckdB37a"
+      },
+      "outputs": [],
+      "source": [
+        "# A trainable variable\n",
+        "x0 = tf.Variable(3.0, name='x0')\n",
+        "# Not trainable\n",
+        "x1 = tf.Variable(3.0, name='x1', trainable=False)\n",
+        "# Not a Variable: A variable + tensor returns a tensor.\n",
+        "x2 = tf.Variable(2.0, name='x2') + 1.0\n",
+        "# Not a variable\n",
+        "x3 = tf.constant(3.0, name='x3')\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  y = (x0**2) + (x1**2) + (x2**2)\n",
+        "\n",
+        "grad = tape.gradient(y, [x0, x1, x2, x3])\n",
+        "\n",
+        "for g in grad:\n",
+        "  print(g)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RkcpQnLgNxgi"
+      },
+      "source": [
+        "You can list the variables being watched by the tape using the `GradientTape.watched_variables` method:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hwNwjW1eAkib"
+      },
+      "outputs": [],
+      "source": [
+        "[var.name for var in tape.watched_variables()]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NB9I1uFvB4tf"
+      },
+      "source": [
+        "`tf.GradientTape` provides hooks that give the user control over what is or is not watched.\n",
+        "\n",
+        "To record gradients with respect to a `tf.Tensor`, you need to call `GradientTape.watch(x)`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tVN1QqFRDHBK"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant(3.0)\n",
+        "with tf.GradientTape() as tape:\n",
+        "  tape.watch(x)\n",
+        "  y = x**2\n",
+        "\n",
+        "# dy = 2x * dx\n",
+        "dy_dx = tape.gradient(y, x)\n",
+        "print(dy_dx.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qxsiYnf2DN8K"
+      },
+      "source": [
+        "Conversely, to disable the default behavior of watching all `tf.Variables`, set `watch_accessed_variables=False` when creating the gradient tape. This calculation uses two variables, but only connects the gradient for one of the variables:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7QPzwWvSEwIp"
+      },
+      "outputs": [],
+      "source": [
+        "x0 = tf.Variable(0.0)\n",
+        "x1 = tf.Variable(10.0)\n",
+        "\n",
+        "with tf.GradientTape(watch_accessed_variables=False) as tape:\n",
+        "  tape.watch(x1)\n",
+        "  y0 = tf.math.sin(x0)\n",
+        "  y1 = tf.nn.softplus(x1)\n",
+        "  y = y0 + y1\n",
+        "  ys = tf.reduce_sum(y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TRduLbE1H2IJ"
+      },
+      "source": [
+        "Since `GradientTape.watch` was not called on `x0`, no gradient is computed with respect to it:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e6GM-3evH1Sz"
+      },
+      "outputs": [],
+      "source": [
+        "# dys/dx1 = exp(x1) / (1 + exp(x1)) = sigmoid(x1)\n",
+        "grad = tape.gradient(ys, {'x0': x0, 'x1': x1})\n",
+        "\n",
+        "print('dy/dx0:', grad['x0'])\n",
+        "print('dy/dx1:', grad['x1'].numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2g1nKB6P-OnA"
+      },
+      "source": [
+        "## Intermediate results\n",
+        "\n",
+        "You can also request gradients of the output with respect to intermediate values computed inside the `tf.GradientTape` context."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7XaPRAwUyYms"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant(3.0)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  tape.watch(x)\n",
+        "  y = x * x\n",
+        "  z = y * y\n",
+        "\n",
+        "# Use the tape to compute the gradient of z with respect to the\n",
+        "# intermediate value y.\n",
+        "# dz_dy = 2 * y and y = x ** 2 = 9\n",
+        "print(tape.gradient(z, y).numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ISkXuY7YzIcS"
+      },
+      "source": [
+        "By default, the resources held by a `GradientTape` are released as soon as the `GradientTape.gradient` method is called. To compute multiple gradients over the same computation, create a gradient tape with `persistent=True`. This allows multiple calls to the `gradient` method as resources are released when the tape object is garbage collected. For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zZaCm3-9zVCi"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant([1, 3.0])\n",
+        "with tf.GradientTape(persistent=True) as tape:\n",
+        "  tape.watch(x)\n",
+        "  y = x * x\n",
+        "  z = y * y\n",
+        "\n",
+        "print(tape.gradient(z, x).numpy())  # [4.0, 108.0] (4 * x**3 at x = [1.0, 3.0])\n",
+        "print(tape.gradient(y, x).numpy())  # [2.0, 6.0] (2 * x at x = [1.0, 3.0])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "j8bv_jQFg6CN"
+      },
+      "outputs": [],
+      "source": [
+        "del tape   # Drop the reference to the tape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O_ZY-9BUB7vX"
+      },
+      "source": [
+        "## Notes on performance\n",
+        "\n",
+        "* There is a tiny overhead associated with doing operations inside a gradient tape context. For most eager execution this will not be a noticeable cost, but you should still use tape context around the areas only where it is required.\n",
+        "\n",
+        "* Gradient tapes use memory to store intermediate results, including inputs and outputs, for use during the backwards pass.\n",
+        "\n",
+        "  For efficiency, some ops (like `ReLU`) don't need to keep their intermediate results and they are pruned during the forward pass. However, if you use `persistent=True` on your tape, *nothing is discarded* and your peak memory usage will be higher."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9dLBpZsJebFq"
+      },
+      "source": [
+        "## Gradients of non-scalar targets"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7pldU9F5duP2"
+      },
+      "source": [
+        "A gradient is fundamentally an operation on a scalar."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qI0sDV_WeXBb"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(2.0)\n",
+        "with tf.GradientTape(persistent=True) as tape:\n",
+        "  y0 = x**2\n",
+        "  y1 = 1 / x\n",
+        "\n",
+        "print(tape.gradient(y0, x).numpy())\n",
+        "print(tape.gradient(y1, x).numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "COEyYp34fxj4"
+      },
+      "source": [
+        "Thus, if you ask for the gradient of multiple targets, the result for each source is:\n",
+        "\n",
+        "* The gradient of the sum of the targets, or equivalently\n",
+        "* The sum of the gradients of each target."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o4a6_YOcfWKS"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(2.0)\n",
+        "with tf.GradientTape() as tape:\n",
+        "  y0 = x**2\n",
+        "  y1 = 1 / x\n",
+        "\n",
+        "print(tape.gradient({'y0': y0, 'y1': y1}, x).numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uvP-mkBMgbym"
+      },
+      "source": [
+        "Similarly, if the target(s) are not scalar the gradient of the sum is calculated:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DArPWqsSh5un"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(2.)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  y = x * [3., 4.]\n",
+        "\n",
+        "print(tape.gradient(y, x).numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "flDbx68Zh5Lb"
+      },
+      "source": [
+        "This makes it simple to take the gradient of the sum of a collection of losses, or the gradient of the sum of an element-wise loss calculation.\n",
+        "\n",
+        "If you need a separate gradient for each item, refer to [Jacobians](advanced_autodiff.ipynb#jacobians)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iwFswok8RAly"
+      },
+      "source": [
+        "In some cases you can skip the Jacobian. For an element-wise calculation, the gradient of the sum gives the derivative of each element with respect to its input-element, since each element is independent:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JQvk_jnMmTDS"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.linspace(-10.0, 10.0, 200+1)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  tape.watch(x)\n",
+        "  y = tf.nn.sigmoid(x)\n",
+        "\n",
+        "dy_dx = tape.gradient(y, x)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e_f2QgDPmcPE"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(x, y, label='y')\n",
+        "plt.plot(x, dy_dx, label='dy/dx')\n",
+        "plt.legend()\n",
+        "_ = plt.xlabel('x')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6kADybtQzYj4"
+      },
+      "source": [
+        "## Control flow\n",
+        "\n",
+        "Because a gradient tape records operations as they are executed, Python control flow is naturally handled (for example, `if` and `while` statements).\n",
+        "\n",
+        "Here a different variable is used on each branch of an `if`. The gradient only connects to the variable that was used:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ciFLizhrrjy7"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant(1.0)\n",
+        "\n",
+        "v0 = tf.Variable(2.0)\n",
+        "v1 = tf.Variable(2.0)\n",
+        "\n",
+        "with tf.GradientTape(persistent=True) as tape:\n",
+        "  tape.watch(x)\n",
+        "  if x > 0.0:\n",
+        "    result = v0\n",
+        "  else:\n",
+        "    result = v1**2 \n",
+        "\n",
+        "dv0, dv1 = tape.gradient(result, [v0, v1])\n",
+        "\n",
+        "print(dv0)\n",
+        "print(dv1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HKnLaiapsjeP"
+      },
+      "source": [
+        "Just remember that the control statements themselves are not differentiable, so they are invisible to gradient-based optimizers.\n",
+        "\n",
+        "Depending on the value of `x` in the above example, the tape either records `result = v0` or `result = v1**2`. The gradient with respect to `x` is always `None`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8k05WmuAwPm7"
+      },
+      "outputs": [],
+      "source": [
+        "dx = tape.gradient(result, x)\n",
+        "\n",
+        "print(dx)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "egypBxISAHhx"
+      },
+      "source": [
+        "## Cases where `gradient` returns `None`\n",
+        "\n",
+        "When a target is not connected to a source, `gradient` will return `None`.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CU185WDM81Ut"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(2.)\n",
+        "y = tf.Variable(3.)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  z = y * y\n",
+        "print(tape.gradient(z, x))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sZbKpHfBRJym"
+      },
+      "source": [
+        "Here `z` is obviously not connected to `x`, but there are several less-obvious ways that a gradient can be disconnected."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eHDzDOiQ8xmw"
+      },
+      "source": [
+        "### 1. Replaced a variable with a tensor\n",
+        "\n",
+        "In the section on [\"controlling what the tape watches\"](#watches) you saw that the tape will automatically watch a `tf.Variable` but not a `tf.Tensor`.\n",
+        "\n",
+        "One common error is to inadvertently replace a `tf.Variable` with a `tf.Tensor`, instead of using `Variable.assign` to update the `tf.Variable`. Here is an example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QPKY4Tn9zX7_"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(2.0)\n",
+        "\n",
+        "for epoch in range(2):\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    y = x+1\n",
+        "\n",
+        "  print(type(x).__name__, \":\", tape.gradient(y, x))\n",
+        "  x = x + 1   # This should be `x.assign_add(1)`"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3gwZKxgA97an"
+      },
+      "source": [
+        "### 2. Did calculations outside of TensorFlow\n",
+        "\n",
+        "The tape can't record the gradient path if the calculation exits TensorFlow.\n",
+        "For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jmoLCDJb_yw1"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable([[1.0, 2.0],\n",
+        "                 [3.0, 4.0]], dtype=tf.float32)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  x2 = x**2\n",
+        "\n",
+        "  # This step is calculated with NumPy\n",
+        "  y = np.mean(x2, axis=0)\n",
+        "\n",
+        "  # Like most ops, reduce_mean will cast the NumPy array to a constant tensor\n",
+        "  # using `tf.convert_to_tensor`.\n",
+        "  y = tf.reduce_mean(y, axis=0)\n",
+        "\n",
+        "print(tape.gradient(y, x))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "p3YVfP3R-tp7"
+      },
+      "source": [
+        "### 3. Took gradients through an integer or string\n",
+        "\n",
+        "Integers and strings are not differentiable. If a calculation path uses these data types there will be no gradient.\n",
+        "\n",
+        "Nobody expects strings to be differentiable, but it's easy to accidentally create an `int` constant or variable if you don't specify the `dtype`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9jlHXHqfASU3"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant(10)\n",
+        "\n",
+        "with tf.GradientTape() as g:\n",
+        "  g.watch(x)\n",
+        "  y = x * x\n",
+        "\n",
+        "print(g.gradient(y, x))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RsdP_mTHX9L1"
+      },
+      "source": [
+        "TensorFlow doesn't automatically cast between types, so, in practice, you'll often get a type error instead of a missing gradient."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WyAZ7C8qCEs6"
+      },
+      "source": [
+        "### 4. Took gradients through a stateful object\n",
+        "\n",
+        "State stops gradients. When you read from a stateful object, the tape can only observe the current state, not the history that lead to it.\n",
+        "\n",
+        "A `tf.Tensor` is immutable. You can't change a tensor once it's created. It has a _value_, but no _state_. All the operations discussed so far are also stateless: the output of a `tf.matmul` only depends on its inputs.\n",
+        "\n",
+        "A `tf.Variable` has internal state—its value. When you use the variable, the state is read. It's normal to calculate a gradient with respect to a variable, but the variable's state blocks gradient calculations from going farther back. For example:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "C1tLeeRFE479"
+      },
+      "outputs": [],
+      "source": [
+        "x0 = tf.Variable(3.0)\n",
+        "x1 = tf.Variable(0.0)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  # Update x1 = x1 + x0.\n",
+        "  x1.assign_add(x0)\n",
+        "  # The tape starts recording from x1.\n",
+        "  y = x1**2   # y = (x1 + x0)**2\n",
+        "\n",
+        "# This doesn't work.\n",
+        "print(tape.gradient(y, x0))   #dy/dx0 = 2*(x1 + x0)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xKA92-dqF2r-"
+      },
+      "source": [
+        "Similarly, `tf.data.Dataset` iterators and `tf.queue`s are stateful, and will stop all gradients on tensors that pass through them."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HHvcDGIbOj2I"
+      },
+      "source": [
+        "## No gradient registered"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aoc-A6AxVqry"
+      },
+      "source": [
+        "Some `tf.Operation`s are **registered as being non-differentiable** and will return `None`. Others have **no gradient registered**.\n",
+        "\n",
+        "The `tf.raw_ops` page shows which low-level ops have gradients registered.\n",
+        "\n",
+        "If you attempt to take a gradient through a float op that has no gradient registered the tape will throw an error instead of silently returning `None`. This way you know something has gone wrong.\n",
+        "\n",
+        "For example, the `tf.image.adjust_contrast` function wraps `raw_ops.AdjustContrastv2`, which could have a gradient but the gradient is not implemented:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HSb20FXc_V0U"
+      },
+      "outputs": [],
+      "source": [
+        "image = tf.Variable([[[0.5, 0.0, 0.0]]])\n",
+        "delta = tf.Variable(0.1)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  new_image = tf.image.adjust_contrast(image, delta)\n",
+        "\n",
+        "try:\n",
+        "  print(tape.gradient(new_image, [image, delta]))\n",
+        "  assert False   # This should not happen.\n",
+        "except LookupError as e:\n",
+        "  print(f'{type(e).__name__}: {e}')\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pDoutjzATiEm"
+      },
+      "source": [
+        "If you need to differentiate through this op, you'll either need to implement the gradient and register it (using `tf.RegisterGradient`) or re-implement the function using other ops."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GCTwc_dQXp2W"
+      },
+      "source": [
+        "## Zeros instead of None"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TYDrVogA89eA"
+      },
+      "source": [
+        "In some cases it would be convenient to get 0 instead of `None` for unconnected gradients.  You can decide what to return when you have unconnected gradients using the `unconnected_gradients` argument:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "U6zxk1sf9Ixx"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable([2., 2.])\n",
+        "y = tf.Variable(3.)\n",
+        "\n",
+        "with tf.GradientTape() as tape:\n",
+        "  z = y**2\n",
+        "print(tape.gradient(z, x, unconnected_gradients=tf.UnconnectedGradients.ZERO))"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "Tce3stUlHN0L"
+      ],
+      "name": "autodiff.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/basic_training_loops.ipynb b/site/en/guide/basic_training_loops.ipynb
new file mode 100644
index 00000000000..a1558b1903e
--- /dev/null
+++ b/site/en/guide/basic_training_loops.ipynb
@@ -0,0 +1,598 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5rmpybwysXGV"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "m8y3rGtQsYP2"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hrXv0rU9sIma"
+      },
+      "source": [
+        "# Basic training loops"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7S0BwJ_8sLu7"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k2o3TTG4TFpt"
+      },
+      "source": [
+        "In the previous guides, you have learned about [tensors](./tensor.ipynb), [variables](./variable.ipynb), [gradient tape](autodiff.ipynb), and [modules](./intro_to_modules.ipynb).  In this guide, you will fit these all together to train models.\n",
+        "\n",
+        "TensorFlow also includes the [tf.Keras API](https://www.tensorflow.org/guide/keras/overview), a high-level neural network API that provides useful abstractions to reduce boilerplate.  However, in this guide, you will use basic classes."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3LXMVuV0VhDr"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NiolgWMPgpwI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "\n",
+        "colors = plt.rcParams['axes.prop_cycle'].by_key()['color']"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iKD__8kFCKNt"
+      },
+      "source": [
+        "## Solving machine learning problems\n",
+        "\n",
+        "Solving a machine learning problem usually consists of the following steps:\n",
+        "\n",
+        " - Obtain training data.\n",
+        " - Define the model.\n",
+        " - Define a loss function.\n",
+        " - Run through the training data, calculating loss from the ideal value\n",
+        " - Calculate gradients for that loss and use an *optimizer* to adjust the variables to fit the data.\n",
+        " - Evaluate your results.\n",
+        "\n",
+        "For illustration purposes, in this guide you'll develop a simple linear model, $f(x) = x * W + b$, which has two variables: $W$ (weights) and $b$ (bias).\n",
+        "\n",
+        "This is the most basic of machine learning problems:  Given $x$ and $y$, try to find the slope and offset of a line via  [simple linear regression](https://en.wikipedia.org/wiki/Linear_regression#Simple_and_multiple_linear_regression)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qutT_fkl_CBc"
+      },
+      "source": [
+        "## Data\n",
+        "\n",
+        "Supervised learning uses *inputs* (usually denoted as *x*) and *outputs* (denoted *y*, often called *labels*).  The goal is to learn from paired inputs and outputs so that you can predict the value of an output from an input.\n",
+        "\n",
+        "Each input of your data, in TensorFlow, is almost always represented by a tensor, and is often a vector. In supervised training, the output (or value you'd like to predict) is also a tensor.\n",
+        "\n",
+        "Here is some data synthesized by adding Gaussian (Normal) noise to points along a line."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NzivK2ATByOz"
+      },
+      "outputs": [],
+      "source": [
+        "# The actual line\n",
+        "TRUE_W = 3.0\n",
+        "TRUE_B = 2.0\n",
+        "\n",
+        "NUM_EXAMPLES = 201\n",
+        "\n",
+        "# A vector of random x values\n",
+        "x = tf.linspace(-2,2, NUM_EXAMPLES)\n",
+        "x = tf.cast(x, tf.float32)\n",
+        "\n",
+        "def f(x):\n",
+        "  return x * TRUE_W + TRUE_B\n",
+        "\n",
+        "# Generate some noise\n",
+        "noise = tf.random.normal(shape=[NUM_EXAMPLES])\n",
+        "\n",
+        "# Calculate y\n",
+        "y = f(x) + noise"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IlFd_HVBFGIF"
+      },
+      "outputs": [],
+      "source": [
+        "# Plot all the data\n",
+        "plt.plot(x, y, '.')\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UH95XUzhL99d"
+      },
+      "source": [
+        "Tensors are usually gathered together in *batches*, or groups of inputs and outputs stacked together.  Batching can confer some training benefits and works well with accelerators and vectorized computation.  Given how small this dataset is, you can treat the entire dataset as a single batch."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gFzH64Jn9PIm"
+      },
+      "source": [
+        "## Define the model\n",
+        "\n",
+        "Use `tf.Variable` to represent all weights in a model.  A `tf.Variable` stores a value and provides this in tensor form as needed.  See the [variable guide](./variable.ipynb) for more details.\n",
+        "\n",
+        "Use `tf.Module` to encapsulate the variables and the computation.  You could use any Python object, but this way it can be easily saved.\n",
+        "\n",
+        "Here, you define both *w* and *b* as variables."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_WRu7Pze7wk8"
+      },
+      "outputs": [],
+      "source": [
+        "class MyModel(tf.Module):\n",
+        "  def __init__(self, **kwargs):\n",
+        "    super().__init__(**kwargs)\n",
+        "    # Initialize the weights to `5.0` and the bias to `0.0`\n",
+        "    # In practice, these should be randomly initialized\n",
+        "    self.w = tf.Variable(5.0)\n",
+        "    self.b = tf.Variable(0.0)\n",
+        "\n",
+        "  def __call__(self, x):\n",
+        "    return self.w * x + self.b\n",
+        "\n",
+        "model = MyModel()\n",
+        "\n",
+        "# List the variables tf.modules's built-in variable aggregation.\n",
+        "print(\"Variables:\", model.variables)\n",
+        "\n",
+        "# Verify the model works\n",
+        "assert model(3.0).numpy() == 15.0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rdpN_3ssG9D5"
+      },
+      "source": [
+        "The initial variables are set here in a fixed way, but Keras comes with any of a number of [initializers](https://www.tensorflow.org/api_docs/python/tf/keras/initializers) you could use, with or without the rest of Keras."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xa6j_yXa-j79"
+      },
+      "source": [
+        "### Define a loss function\n",
+        "\n",
+        "A loss function measures how well the output of a model for a given input matches the target output. The goal is to minimize this difference during training. Define the standard L2 loss, also known as the \"mean squared\" error:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y0ysUFGY924U"
+      },
+      "outputs": [],
+      "source": [
+        "# This computes a single loss value for an entire batch\n",
+        "def loss(target_y, predicted_y):\n",
+        "  return tf.reduce_mean(tf.square(target_y - predicted_y))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-50nq-wPBsAW"
+      },
+      "source": [
+        "Before training the model, you can visualize the loss value by plotting the model's predictions in red and the training data in blue:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_eb83LtrB4nt"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(x, y, '.', label=\"Data\")\n",
+        "plt.plot(x, f(x), label=\"Ground truth\")\n",
+        "plt.plot(x, model(x), label=\"Predictions\")\n",
+        "plt.legend()\n",
+        "plt.show()\n",
+        "\n",
+        "print(\"Current loss: %1.6f\" % loss(y, model(x)).numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sSDP-yeq_4jE"
+      },
+      "source": [
+        "### Define a training loop\n",
+        "\n",
+        "The training loop consists of repeatedly doing three tasks in order:\n",
+        "\n",
+        "* Sending a batch of inputs through the model to generate outputs\n",
+        "* Calculating the loss by comparing the outputs to the output (or label)\n",
+        "* Using gradient tape to find the gradients\n",
+        "* Optimizing the variables with those gradients\n",
+        "\n",
+        "For this example, you can train the model using [gradient descent](https://en.wikipedia.org/wiki/Gradient_descent).\n",
+        "\n",
+        "There are many variants of the gradient descent scheme that are captured in `tf.keras.optimizers`. But in the spirit of building from first principles, here you will implement the basic math yourself with the help of `tf.GradientTape` for automatic differentiation and `tf.assign_sub` for decrementing a value (which combines `tf.assign` and `tf.sub`):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MBIACgdnA55X"
+      },
+      "outputs": [],
+      "source": [
+        "# Given a callable model, inputs, outputs, and a learning rate...\n",
+        "def train(model, x, y, learning_rate):\n",
+        "\n",
+        "  with tf.GradientTape() as t:\n",
+        "    # Trainable variables are automatically tracked by GradientTape\n",
+        "    current_loss = loss(y, model(x))\n",
+        "\n",
+        "  # Use GradientTape to calculate the gradients with respect to W and b\n",
+        "  dw, db = t.gradient(current_loss, [model.w, model.b])\n",
+        "\n",
+        "  # Subtract the gradient scaled by the learning rate\n",
+        "  model.w.assign_sub(learning_rate * dw)\n",
+        "  model.b.assign_sub(learning_rate * db)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RwWPaJryD2aN"
+      },
+      "source": [
+        "For a look at training, you can send the same batch of *x* and *y* through the training loop, and see how `W` and `b` evolve."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XdfkR223D9dW"
+      },
+      "outputs": [],
+      "source": [
+        "model = MyModel()\n",
+        "\n",
+        "# Collect the history of W-values and b-values to plot later\n",
+        "weights = []\n",
+        "biases = []\n",
+        "epochs = range(10)\n",
+        "\n",
+        "# Define a training loop\n",
+        "def report(model, loss):\n",
+        "  return f\"W = {model.w.numpy():1.2f}, b = {model.b.numpy():1.2f}, loss={loss:2.5f}\"\n",
+        "\n",
+        "\n",
+        "def training_loop(model, x, y):\n",
+        "\n",
+        "  for epoch in epochs:\n",
+        "    # Update the model with the single giant batch\n",
+        "    train(model, x, y, learning_rate=0.1)\n",
+        "\n",
+        "    # Track this before I update\n",
+        "    weights.append(model.w.numpy())\n",
+        "    biases.append(model.b.numpy())\n",
+        "    current_loss = loss(y, model(x))\n",
+        "\n",
+        "    print(f\"Epoch {epoch:2d}:\")\n",
+        "    print(\"    \", report(model, current_loss))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8dKKLU4KkQEq"
+      },
+      "source": [
+        "Do the training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iRuNUghs1lHY"
+      },
+      "outputs": [],
+      "source": [
+        "current_loss = loss(y, model(x))\n",
+        "\n",
+        "print(f\"Starting:\")\n",
+        "print(\"    \", report(model, current_loss))\n",
+        "\n",
+        "training_loop(model, x, y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JPJgimg8kSA4"
+      },
+      "source": [
+        "Plot the evolution of the weights over time:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ND1fQw8sbTNr"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(epochs, weights, label='Weights', color=colors[0])\n",
+        "plt.plot(epochs, [TRUE_W] * len(epochs), '--',\n",
+        "         label = \"True weight\", color=colors[0])\n",
+        "\n",
+        "plt.plot(epochs, biases, label='bias', color=colors[1])\n",
+        "plt.plot(epochs, [TRUE_B] * len(epochs), \"--\",\n",
+        "         label=\"True bias\", color=colors[1])\n",
+        "\n",
+        "plt.legend()\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zhlwj1ojkcUP"
+      },
+      "source": [
+        "Visualize how the trained model performs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tpTEjWWex568"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(x, y, '.', label=\"Data\")\n",
+        "plt.plot(x, f(x), label=\"Ground truth\")\n",
+        "plt.plot(x, model(x), label=\"Predictions\")\n",
+        "plt.legend()\n",
+        "plt.show()\n",
+        "\n",
+        "print(\"Current loss: %1.6f\" % loss(model(x), y).numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DODMMmfLIiOC"
+      },
+      "source": [
+        "## The same solution, but with Keras\n",
+        "\n",
+        "It's useful to contrast the code above with the equivalent in Keras.\n",
+        "\n",
+        "Defining the model looks exactly the same if you subclass `tf.keras.Model`.  Remember that Keras models inherit ultimately from module."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Z86hCI0x1YX3"
+      },
+      "outputs": [],
+      "source": [
+        "class MyModelKeras(tf.keras.Model):\n",
+        "  def __init__(self, **kwargs):\n",
+        "    super().__init__(**kwargs)\n",
+        "    # Initialize the weights to `5.0` and the bias to `0.0`\n",
+        "    # In practice, these should be randomly initialized\n",
+        "    self.w = tf.Variable(5.0)\n",
+        "    self.b = tf.Variable(0.0)\n",
+        "\n",
+        "  def call(self, x):\n",
+        "    return self.w * x + self.b\n",
+        "\n",
+        "keras_model = MyModelKeras()\n",
+        "\n",
+        "# Reuse the training loop with a Keras model\n",
+        "training_loop(keras_model, x, y)\n",
+        "\n",
+        "# You can also save a checkpoint using Keras's built-in support\n",
+        "keras_model.save_weights(\"my_checkpoint\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6kw5P4jt2Az8"
+      },
+      "source": [
+        "Rather than write new training loops each time you create a model, you can use the built-in features of Keras as a shortcut.  This can be useful when you do not want to write or debug Python training loops.\n",
+        "\n",
+        "If you do, you will need to use `model.compile()` to set the parameters, and `model.fit()` to train.  It can be less code to use Keras implementations of L2 loss and gradient descent, again as a shortcut.  Keras losses and optimizers can be used outside of these convenience functions, too, and the previous example could have used them."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-nbLLfPE2pEl"
+      },
+      "outputs": [],
+      "source": [
+        "keras_model = MyModelKeras()\n",
+        "\n",
+        "# compile sets the training parameters\n",
+        "keras_model.compile(\n",
+        "    # By default, fit() uses tf.function().  You can\n",
+        "    # turn that off for debugging, but it is on now.\n",
+        "    run_eagerly=False,\n",
+        "\n",
+        "    # Using a built-in optimizer, configuring as an object\n",
+        "    optimizer=tf.keras.optimizers.SGD(learning_rate=0.1),\n",
+        "\n",
+        "    # Keras comes with built-in MSE error\n",
+        "    # However, you could use the loss function\n",
+        "    # defined above\n",
+        "    loss=tf.keras.losses.mean_squared_error,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lrlHODiZccu2"
+      },
+      "source": [
+        "Keras `fit` expects batched data or a complete dataset as a NumPy array.  NumPy arrays are chopped into batches and default to a batch size of 32.\n",
+        "\n",
+        "In this case, to match the behavior of the hand-written loop, you should pass `x` in as a single batch of size 1000."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zfAYqtu136PO"
+      },
+      "outputs": [],
+      "source": [
+        "print(x.shape[0])\n",
+        "keras_model.fit(x, y, epochs=10, batch_size=1000)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8zKZIO9P5s1G"
+      },
+      "source": [
+        "Note that Keras prints out the loss after training, not before, so the first loss appears lower, but otherwise this shows essentially the same training performance."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vPnIVuaSJwWz"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "In this guide, you have seen how to use the core classes of tensors, variables, modules, and gradient tape to build and train a model, and further how those ideas map to Keras.\n",
+        "\n",
+        "This is, however, an extremely simple problem. For a more practical introduction, see [Custom training walkthrough](../tutorials/customization/custom_training_walkthrough.ipynb).\n",
+        "\n",
+        "For more on using built-in Keras training loops, see [this guide](https://www.tensorflow.org/guide/keras/train_and_evaluate).  For more on training loops and Keras, see [this guide](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch).  For writing custom distributed training loops, see [this guide](distributed_training.ipynb#using_tfdistributestrategy_with_basic_training_loops_loops)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "5rmpybwysXGV",
+        "iKD__8kFCKNt"
+      ],
+      "name": "basic_training_loops.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/basics.ipynb b/site/en/guide/basics.ipynb
new file mode 100644
index 00000000000..5457f162a0e
--- /dev/null
+++ b/site/en/guide/basics.ipynb
@@ -0,0 +1,968 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5rmpybwysXGV"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "m8y3rGtQsYP2"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hrXv0rU9sIma"
+      },
+      "source": [
+        "# TensorFlow basics"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7S0BwJ_8sLu7"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iJyZUDbzBTIG"
+      },
+      "source": [
+        "This guide provides a quick overview of _TensorFlow basics_. Each section of this doc is an overview of a larger topic—you can find links to full guides at the end of each section.\n",
+        "\n",
+        "TensorFlow is an end-to-end platform for machine learning. It supports the following:\n",
+        "\n",
+        "* Multidimensional-array based numeric computation (similar to NumPy.)\n",
+        "* GPU and distributed processing\n",
+        "* Automatic differentiation\n",
+        "* Model construction, training, and export\n",
+        "* And more"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gvLegMMvBZYg"
+      },
+      "source": [
+        "## Tensors\n",
+        "\n",
+        "TensorFlow operates on multidimensional arrays or _tensors_ represented as `tf.Tensor` objects. Here is a two-dimensional tensor:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6ZqX5RnbBS1f"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "x = tf.constant([[1., 2., 3.],\n",
+        "                 [4., 5., 6.]])\n",
+        "\n",
+        "print(x)\n",
+        "print(x.shape)\n",
+        "print(x.dtype)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k-AOMqevQGN4"
+      },
+      "source": [
+        "The most important attributes of a `tf.Tensor` are its `shape` and `dtype`:\n",
+        "\n",
+        "* `Tensor.shape`: tells you the size of the tensor along each of its axes.\n",
+        "* `Tensor.dtype`: tells you the type of all the elements in the tensor."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bUkKeNWZCIJO"
+      },
+      "source": [
+        "TensorFlow implements standard mathematical operations on tensors, as well as many operations specialized for machine learning.\n",
+        "\n",
+        "For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BM7xXNDsBfN5"
+      },
+      "outputs": [],
+      "source": [
+        "x + x"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZLGqscTxB61v"
+      },
+      "outputs": [],
+      "source": [
+        "5 * x"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ImJHd8VfnWq"
+      },
+      "outputs": [],
+      "source": [
+        "x @ tf.transpose(x)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "U9JZD6TYCZWu"
+      },
+      "outputs": [],
+      "source": [
+        "tf.concat([x, x, x], axis=0)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "seGBLeD9P_PI"
+      },
+      "outputs": [],
+      "source": [
+        "tf.nn.softmax(x, axis=-1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YZNZRv1ECjf8"
+      },
+      "outputs": [],
+      "source": [
+        "tf.reduce_sum(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TNHnIjOVLJfA"
+      },
+      "source": [
+        "Note: Typically, anywhere a TensorFlow function expects a `Tensor` as input, the function will also accept anything that can be converted to a `Tensor` using `tf.convert_to_tensor`. See below for an example."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "i_XKgjDsL4GE"
+      },
+      "outputs": [],
+      "source": [
+        "tf.convert_to_tensor([1,2,3])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wTBt-JUqLJDJ"
+      },
+      "outputs": [],
+      "source": [
+        "tf.reduce_sum([1,2,3])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8-mi5031DVxz"
+      },
+      "source": [
+        "Running large calculations on CPU can be slow. When properly configured, TensorFlow can use accelerator hardware like GPUs to execute operations very quickly."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m97Gv5H6Dz0G"
+      },
+      "outputs": [],
+      "source": [
+        "if tf.config.list_physical_devices('GPU'):\n",
+        "  print(\"TensorFlow **IS** using the GPU\")\n",
+        "else:\n",
+        "  print(\"TensorFlow **IS NOT** using the GPU\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ln2FkLOqMX92"
+      },
+      "source": [
+        "Refer to the [Tensor guide](tensor.ipynb) for details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oVbomvMyEIVF"
+      },
+      "source": [
+        "## Variables\n",
+        "\n",
+        "Normal `tf.Tensor` objects are immutable. To store model weights (or other mutable state) in TensorFlow use a `tf.Variable`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SO8_bP4UEzxS"
+      },
+      "outputs": [],
+      "source": [
+        "var = tf.Variable([0.0, 0.0, 0.0])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aDLYFvu5FAFa"
+      },
+      "outputs": [],
+      "source": [
+        "var.assign([1, 2, 3])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9EpiOmxXFDSS"
+      },
+      "outputs": [],
+      "source": [
+        "var.assign_add([1, 1, 1])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tlvTpi1CMedC"
+      },
+      "source": [
+        "Refer to the [Variables guide](variable.ipynb) for details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rG1Dhv2QFkV3"
+      },
+      "source": [
+        "## Automatic differentiation\n",
+        "\n",
+        "_Gradient descent_ and related algorithms are a cornerstone of modern machine learning.\n",
+        "\n",
+        "To enable this, TensorFlow implements automatic differentiation (autodiff), which uses calculus to compute gradients. Typically you'll use this to calculate the gradient of a model's _error_ or _loss_ with respect to its weights."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cYKOi-z4GY9Y"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.Variable(1.0)\n",
+        "\n",
+        "def f(x):\n",
+        "  y = x**2 + 2*x - 5\n",
+        "  return y"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IQz99cxMGoF_"
+      },
+      "outputs": [],
+      "source": [
+        "f(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ozLLop0cHeYl"
+      },
+      "source": [
+        "At `x = 1.0`, `y = f(x) = (1**2 + 2*1 - 5) = -2`.\n",
+        "\n",
+        "The derivative of `y` is `y' = f'(x) = (2*x + 2) = 4`. TensorFlow can calculate this automatically:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "N02NfWpHGvw8"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.GradientTape() as tape:\n",
+        "  y = f(x)\n",
+        "\n",
+        "g_x = tape.gradient(y, x)  # g(x) = dy/dx\n",
+        "\n",
+        "g_x"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s-DVYJfcIRPd"
+      },
+      "source": [
+        "This simplified example only takes the derivative with respect to a single scalar (`x`), but TensorFlow can compute the gradient with respect to any number of non-scalar tensors simultaneously."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ECK3I9bUMk_r"
+      },
+      "source": [
+        "Refer to the [Autodiff guide](autodiff.ipynb) for details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VglUM4M3KhNz"
+      },
+      "source": [
+        "## Graphs and tf.function\n",
+        "\n",
+        "While you can use TensorFlow interactively like any Python library, TensorFlow also provides tools for:\n",
+        "\n",
+        "* **Performance optimization**: to speed up training and inference.\n",
+        "* **Export**: so you can save your model when it's done training.\n",
+        "\n",
+        "These require that you use `tf.function` to separate your pure-TensorFlow code from Python."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VitACyZWKJD_"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def my_func(x):\n",
+        "  print('Tracing.\\n')\n",
+        "  return tf.reduce_sum(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fBYDh-huNUBZ"
+      },
+      "source": [
+        "The first time you run the `tf.function`, although it executes in Python, it captures a complete, optimized graph representing the TensorFlow computations done within the function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vkOFSEkoM1bd"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant([1, 2, 3])\n",
+        "my_func(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a3aWzt-rNsBa"
+      },
+      "source": [
+        "On subsequent calls TensorFlow only executes the optimized graph, skipping any non-TensorFlow steps. Below, note that `my_func` doesn't print _tracing_ since `print` is a Python function, not a TensorFlow function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "23dMHWwwNIoa"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant([10, 9, 8])\n",
+        "my_func(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nSeTti6zki0n"
+      },
+      "source": [
+        "A graph may not be reusable for inputs with a different _signature_ (`shape` and `dtype`), so a new graph is generated instead:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OWffqyhqlVPf"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant([10.0, 9.1, 8.2], dtype=tf.float32)\n",
+        "my_func(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UWknAA_zNTOa"
+      },
+      "source": [
+        "These captured graphs provide two benefits:\n",
+        "\n",
+        "* In many cases they provide a significant speedup in execution (though not this trivial example).\n",
+        "* You can export these graphs, using `tf.saved_model`, to run on other systems like a [server](https://www.tensorflow.org/tfx/serving/docker) or a [mobile device](https://www.tensorflow.org/lite/guide), no Python installation required."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hLUJ6f2eMsA8"
+      },
+      "source": [
+        "Refer to [Intro to graphs](intro_to_graphs.ipynb) for more details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "t_36xPDPPBqp"
+      },
+      "source": [
+        "## Modules, layers, and models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oDaT7kCpUgnJ"
+      },
+      "source": [
+        "`tf.Module` is a class for managing your `tf.Variable` objects, and the `tf.function` objects that operate on them. The `tf.Module` class is necessary to support two significant features:\n",
+        "\n",
+        "1. You can save and restore the values of your variables using `tf.train.Checkpoint`. This is useful during training as it is quick to save and restore a model's state.\n",
+        "2. You can import and export the `tf.Variable` values _and_ the `tf.function` graphs using `tf.saved_model`. This allows you to run your model independently of the Python program that created it.\n",
+        "\n",
+        "Here is a complete example exporting a simple `tf.Module` object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1MqEcZOqPBDV"
+      },
+      "outputs": [],
+      "source": [
+        "class MyModule(tf.Module):\n",
+        "  def __init__(self, value):\n",
+        "    self.weight = tf.Variable(value)\n",
+        "\n",
+        "  @tf.function\n",
+        "  def multiply(self, x):\n",
+        "    return x * self.weight"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "la2G82HfVfU0"
+      },
+      "outputs": [],
+      "source": [
+        "mod = MyModule(3)\n",
+        "mod.multiply(tf.constant([1, 2, 3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GaSJX7zQXCm4"
+      },
+      "source": [
+        "Save the `Module`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1MlfbEMjVzG4"
+      },
+      "outputs": [],
+      "source": [
+        "save_path = './saved'\n",
+        "tf.saved_model.save(mod, save_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LgfoftD4XGJW"
+      },
+      "source": [
+        "The resulting SavedModel is independent of the code that created it. You can load a SavedModel from Python, other language bindings, or [TensorFlow Serving](https://www.tensorflow.org/tfx/serving/docker). You can also convert it to run with [TensorFlow Lite](https://www.tensorflow.org/lite/guide) or [TensorFlow JS](https://www.tensorflow.org/js/guide)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pWuLOIKBWZYG"
+      },
+      "outputs": [],
+      "source": [
+        "reloaded = tf.saved_model.load(save_path)\n",
+        "reloaded.multiply(tf.constant([1, 2, 3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nxU6P1RGwHyC"
+      },
+      "source": [
+        "The `tf.keras.layers.Layer` and `tf.keras.Model` classes build on `tf.Module` providing additional functionality and convenience methods for building, training, and saving models. Some of these are demonstrated in the next section."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tQzt3yaWMzLf"
+      },
+      "source": [
+        "Refer to [Intro to modules](intro_to_modules.ipynb) for details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Rk1IEG5aav7X"
+      },
+      "source": [
+        "## Training loops\n",
+        "\n",
+        "Now put this all together to build a basic model and train it from scratch.\n",
+        "\n",
+        "First, create some example data. This generates a cloud of points that loosely follows a quadratic curve:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VcuFr7KPRPzn"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib\n",
+        "from matplotlib import pyplot as plt\n",
+        "\n",
+        "matplotlib.rcParams['figure.figsize'] = [9, 6]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sXN9E_xf-GiP"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.linspace(-2, 2, 201)\n",
+        "x = tf.cast(x, tf.float32)\n",
+        "\n",
+        "def f(x):\n",
+        "  y = x**2 + 2*x - 5\n",
+        "  return y\n",
+        "\n",
+        "y = f(x) + tf.random.normal(shape=[201])\n",
+        "\n",
+        "plt.plot(x.numpy(), y.numpy(), '.', label='Data')\n",
+        "plt.plot(x, f(x), label='Ground truth')\n",
+        "plt.legend();"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "De5LldboSWcW"
+      },
+      "source": [
+        "Create a quadratic model with randomly initialized weights and a bias:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Pypd0GB4SRhf"
+      },
+      "outputs": [],
+      "source": [
+        "class Model(tf.Module):\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    # Randomly generate weight and bias terms\n",
+        "    rand_init = tf.random.uniform(shape=[3], minval=0., maxval=5., seed=22)\n",
+        "    # Initialize model parameters\n",
+        "    self.w_q = tf.Variable(rand_init[0])\n",
+        "    self.w_l = tf.Variable(rand_init[1])\n",
+        "    self.b = tf.Variable(rand_init[2])\n",
+        "  \n",
+        "  @tf.function\n",
+        "  def __call__(self, x):\n",
+        "    # Quadratic Model : quadratic_weight * x^2 + linear_weight * x + bias\n",
+        "    return self.w_q * (x**2) + self.w_l * x + self.b"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "36o7VjaesScg"
+      },
+      "source": [
+        "First, observe your model's performance before training:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GkwToC5BWV1c"
+      },
+      "outputs": [],
+      "source": [
+        "quad_model = Model()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ReWhH40wTY5F"
+      },
+      "outputs": [],
+      "source": [
+        "def plot_preds(x, y, f, model, title):\n",
+        "  plt.figure()\n",
+        "  plt.plot(x, y, '.', label='Data')\n",
+        "  plt.plot(x, f(x), label='Ground truth')\n",
+        "  plt.plot(x, model(x), label='Predictions')\n",
+        "  plt.title(title)\n",
+        "  plt.legend()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y0JtXQat-nlk"
+      },
+      "outputs": [],
+      "source": [
+        "plot_preds(x, y, f, quad_model, 'Before training')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hLzwD0-ascGf"
+      },
+      "source": [
+        "Now, define a loss for your model:\n",
+        "\n",
+        "Given that this model is intended to predict continuous values, the mean squared error (MSE) is a good choice for the loss function. Given a vector of predictions, $\\hat{y}$, and a vector of true targets, $y$, the MSE is defined as the mean of the squared differences between the predicted values and the ground truth.\n",
+        "\n",
+        "$MSE = \\frac{1}{m}\\sum_{i=1}^{m}(\\hat{y}_i -y_i)^2$"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eCtJ1uuCseZd"
+      },
+      "outputs": [],
+      "source": [
+        "def mse_loss(y_pred, y):\n",
+        "  return tf.reduce_mean(tf.square(y_pred - y))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7EWyDu3zot2w"
+      },
+      "source": [
+        "Write a basic training loop for the model. The loop will make use of the MSE loss function and its gradients with respect to the input in order to iteratively update the model's parameters. Using mini-batches for training provides both memory efficiency and faster convergence. The `tf.data.Dataset` API has useful functions for batching and shuffling."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8kX_-zily2Ia"
+      },
+      "outputs": [],
+      "source": [
+        "batch_size = 32\n",
+        "dataset = tf.data.Dataset.from_tensor_slices((x, y))\n",
+        "dataset = dataset.shuffle(buffer_size=x.shape[0]).batch(batch_size)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nOaES5gyTDtG"
+      },
+      "outputs": [],
+      "source": [
+        "# Set training parameters\n",
+        "epochs = 100\n",
+        "learning_rate = 0.01\n",
+        "losses = []\n",
+        "\n",
+        "# Format training loop\n",
+        "for epoch in range(epochs):\n",
+        "  for x_batch, y_batch in dataset:\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      batch_loss = mse_loss(quad_model(x_batch), y_batch)\n",
+        "    # Update parameters with respect to the gradient calculations\n",
+        "    grads = tape.gradient(batch_loss, quad_model.variables)\n",
+        "    for g,v in zip(grads, quad_model.variables):\n",
+        "        v.assign_sub(learning_rate*g)\n",
+        "  # Keep track of model loss per epoch\n",
+        "  loss = mse_loss(quad_model(x), y)\n",
+        "  losses.append(loss)\n",
+        "  if epoch % 10 == 0:\n",
+        "    print(f'Mean squared error for step {epoch}: {loss.numpy():0.3f}')\n",
+        "\n",
+        "# Plot model results\n",
+        "print(\"\\n\")\n",
+        "plt.plot(range(epochs), losses)\n",
+        "plt.xlabel(\"Epoch\")\n",
+        "plt.ylabel(\"Mean Squared Error (MSE)\")\n",
+        "plt.title('MSE loss vs training iterations');"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dW5B2TTRsvxE"
+      },
+      "source": [
+        "Now, observe your model's performance after training:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Qcvzyg3eYLh8"
+      },
+      "outputs": [],
+      "source": [
+        "plot_preds(x, y, f, quad_model, 'After training')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hbtmFJIXb6qm"
+      },
+      "source": [
+        "That's working, but remember that implementations of common training utilities are available in the `tf.keras` module. So, consider using those before writing your own. To start with, the `Model.compile` and `Model.fit` methods implement a training loop for you:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cjx23MiztFmT"
+      },
+      "source": [
+        "Begin by creating a Sequential Model in Keras using `tf.keras.Sequential`. One of the simplest Keras layers is the dense layer, which can be instantiated with `tf.keras.layers.Dense`. The dense layer is able to learn multidimensional linear relationships of the form $\\mathrm{Y} = \\mathrm{W}\\mathrm{X} +  \\vec{b}$. In order to learn a nonlinear equation of the form, $w_1x^2 + w_2x + b$, the dense layer's input should be a data matrix with $x^2$ and $x$ as features. The lambda layer, `tf.keras.layers.Lambda`, can be used to perform this stacking transformation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5rt8HP2TZhEM"
+      },
+      "outputs": [],
+      "source": [
+        "new_model = tf.keras.Sequential([\n",
+        "    tf.keras.layers.Lambda(lambda x: tf.stack([x, x**2], axis=1)),\n",
+        "    tf.keras.layers.Dense(units=1, kernel_initializer=tf.random.normal)])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "73kCo1BtP3rQ"
+      },
+      "outputs": [],
+      "source": [
+        "new_model.compile(\n",
+        "    loss=tf.keras.losses.MSE,\n",
+        "    optimizer=tf.keras.optimizers.SGD(learning_rate=0.01))\n",
+        "\n",
+        "history = new_model.fit(x, y,\n",
+        "                        epochs=100,\n",
+        "                        batch_size=32,\n",
+        "                        verbose=0)\n",
+        "\n",
+        "new_model.save('./my_new_model.keras')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "u3q5d1SzvzTq"
+      },
+      "source": [
+        "Observe your Keras model's performance after training:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Mo7zRV7XZjv7"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(history.history['loss'])\n",
+        "plt.xlabel('Epoch')\n",
+        "plt.ylim([0, max(plt.ylim())])\n",
+        "plt.ylabel('Loss [Mean Squared Error]')\n",
+        "plt.title('Keras training progress');"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bB44a9YsvnfK"
+      },
+      "outputs": [],
+      "source": [
+        "plot_preds(x, y, f, new_model, 'After Training: Keras')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ng-BY_eGS0bn"
+      },
+      "source": [
+        "Refer to [Basic training loops](basic_training_loops.ipynb) and the [Keras guide](https://www.tensorflow.org/guide/keras) for more details."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "basics.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/checkpoint.ipynb b/site/en/guide/checkpoint.ipynb
index 92a84a6f29b..fb3b45437f7 100644
--- a/site/en/guide/checkpoint.ipynb
+++ b/site/en/guide/checkpoint.ipynb
@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Pnn4rDWGqDZL"
       },
       "source": [
@@ -12,11 +11,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
         "id": "l534d35Gp68G"
       },
       "outputs": [],
@@ -37,7 +34,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3TI3Q3XBesaS"
       },
       "source": [
@@ -47,34 +43,31 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yw_a0iGucY8z"
       },
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/checkpoint\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/checkpoint.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/checkpoint.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/checkpoint.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "LeDp7dovcbus"
       },
       "source": [
-        "\n",
         "The phrase \"Saving a TensorFlow model\" typically means one of two things:\n",
         "\n",
         "  1. Checkpoints, OR \n",
@@ -90,7 +83,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "U0nm8k-6xfh2"
       },
       "source": [
@@ -99,29 +91,19 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "VEvpMYAKsC4z"
       },
       "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
         "import tensorflow as tf"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "OEQCseyeC4Ev"
       },
       "outputs": [],
@@ -139,10 +121,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "utqeoDADC5ZR"
       },
       "outputs": [],
@@ -153,24 +133,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5vsq3-pffo1I"
       },
       "source": [
         "## Saving from `tf.keras` training APIs\n",
         "\n",
         "See the [`tf.keras` guide on saving and\n",
-        "restoring](./keras/overview.ipynb#save_and_restore).\n",
+        "restoring](https://www.tensorflow.org/guide/keras/save_and_serialize).\n",
         "\n",
         "`tf.keras.Model.save_weights` saves a TensorFlow checkpoint. "
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "SuhmrYPEl4D_"
       },
       "outputs": [],
@@ -181,18 +158,15 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "XseWX5jDg4lQ"
       },
       "source": [
-        "## Writing checkpoints\n",
-        "\n"
+        "## Writing checkpoints\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "1jpZPz76ZP3K"
       },
       "source": [
@@ -206,17 +180,15 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "x0vFBr_Im73_"
       },
       "source": [
-        "You can easily save a model-checkpoint with `Model.save_weights`"
+        "You can easily save a model-checkpoint with `Model.save_weights`."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "FHTJ1JzxCi8a"
       },
       "source": [
@@ -226,7 +198,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6cF9fqYOCrEO"
       },
       "source": [
@@ -236,19 +207,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "fNjf9KaLdIRP"
       },
       "source": [
-        "To help demonstrate all the features of `tf.train.Checkpoint` define a toy dataset and optimization step:"
+        "To help demonstrate all the features of `tf.train.Checkpoint`, define a toy dataset and optimization step:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "tSNyP4IJ9nkU"
       },
       "outputs": [],
@@ -257,15 +225,13 @@
         "  inputs = tf.range(10.)[:, None]\n",
         "  labels = inputs * 5. + tf.range(5.)[None, :]\n",
         "  return tf.data.Dataset.from_tensor_slices(\n",
-        "    dict(x=inputs, y=labels)).repeat(10).batch(2)"
+        "    dict(x=inputs, y=labels)).repeat().batch(2)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ICm1cufh_JH8"
       },
       "outputs": [],
@@ -284,36 +250,34 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vxzGpHRbOVO6"
       },
       "source": [
         "#### Create the checkpoint objects\n",
         "\n",
-        "To manually make a checkpoint you will need a `tf.train.Checkpoint` object. Where the objects you want to checkpoint are set as attributes on the object.\n",
+        "Use a `tf.train.Checkpoint` object to manually create a checkpoint, where the objects you want to checkpoint are set as attributes on the object.\n",
         "\n",
         "A `tf.train.CheckpointManager` can also be helpful for managing multiple checkpoints."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ou5qarOQOWYl"
       },
       "outputs": [],
       "source": [
         "opt = tf.keras.optimizers.Adam(0.1)\n",
-        "ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=net)\n",
+        "dataset = toy_dataset()\n",
+        "iterator = iter(dataset)\n",
+        "ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=net, iterator=iterator)\n",
         "manager = tf.train.CheckpointManager(ckpt, './tf_ckpts', max_to_keep=3)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8ZbYSD4uCy96"
       },
       "source": [
@@ -323,7 +287,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "NP9IySmCeCkn"
       },
       "source": [
@@ -332,10 +295,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "BbCS5A6K1VSH"
       },
       "outputs": [],
@@ -347,7 +308,8 @@
         "  else:\n",
         "    print(\"Initializing from scratch.\")\n",
         "\n",
-        "  for example in toy_dataset():\n",
+        "  for _ in range(50):\n",
+        "    example = next(iterator)\n",
         "    loss = train_step(net, example, opt)\n",
         "    ckpt.step.assign_add(1)\n",
         "    if int(ckpt.step) % 10 == 0:\n",
@@ -358,10 +320,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Ik3IBMTdPW41"
       },
       "outputs": [],
@@ -372,7 +332,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2wzcc1xYN-sH"
       },
       "source": [
@@ -382,26 +341,25 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "lw1QeyRBgsLE"
       },
       "source": [
-        "After the first you can pass a new model and manager, but pickup training exactly where you left off:"
+        "After the first training cycle you can pass a new model and manager, but pick up training exactly where you left off:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "UjilkTOV2PBK"
       },
       "outputs": [],
       "source": [
         "opt = tf.keras.optimizers.Adam(0.1)\n",
         "net = Net()\n",
-        "ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=net)\n",
+        "dataset = toy_dataset()\n",
+        "iterator = iter(dataset)\n",
+        "ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=net, iterator=iterator)\n",
         "manager = tf.train.CheckpointManager(ckpt, './tf_ckpts', max_to_keep=3)\n",
         "\n",
         "train_and_checkpoint(net, manager)"
@@ -410,7 +368,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "dxJT9vV-2PnZ"
       },
       "source": [
@@ -419,10 +376,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "3zmM0a-F5XqC"
       },
       "outputs": [],
@@ -433,7 +388,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qwlYDyjemY4P"
       },
       "source": [
@@ -442,10 +396,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "t1feej9JntV_"
       },
       "outputs": [],
@@ -456,11 +408,10 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "DR2wQc9x6b3X"
       },
       "source": [
-        "\u003ca id=\"loading_mechanics\"/\u003e\n",
+        "\n",
         "\n",
         "## Loading mechanics\n",
         "\n",
@@ -470,27 +421,24 @@
         "\n",
         "![Visualization of the dependency graph for the example training loop](https://tensorflow.org/images/guide/whole_checkpoint.svg)\n",
         "\n",
-        "With the optimizer in red, regular variables in blue, and optimizer slot variables in orange. The other nodes, for example representing the `tf.train.Checkpoint`, are black.\n",
+        "The optimizer is in red, regular variables are in blue, and the optimizer slot variables are in orange. The other nodes—for example, representing the `tf.train.Checkpoint`—are in black.\n",
         "\n",
-        "Slot variables are part of the optimizer's state, but are created for a specific variable. For example the `'m'` edges above correspond to momentum, which the Adam optimizer tracks for each variable. Slot variables are only saved in a checkpoint if the variable and the optimizer would both be saved, thus the dashed edges."
+        "Slot variables are part of the optimizer's state, but are created for a specific variable. For example, the `'m'` edges above correspond to momentum, which the Adam optimizer tracks for each variable. Slot variables are only saved in a checkpoint if the variable and the optimizer would both be saved, thus the dashed edges."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "VpY5IuanUEQ0"
       },
       "source": [
-        "Calling `restore()` on a `tf.train.Checkpoint` object queues the requested restorations, restoring variable values as soon as there's a matching path from the `Checkpoint` object. For example we can load just the kernel from the model we defined above by reconstructing one path to it through the network and the layer."
+        "Calling `restore` on a `tf.train.Checkpoint` object queues the requested restorations, restoring variable values as soon as there's a matching path from the `Checkpoint` object. For example, you can load just the bias from the model you defined above by reconstructing one path to it through the network and the layer."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "wmX2AuyH7TVt"
       },
       "outputs": [],
@@ -501,29 +449,26 @@
         "fake_net = tf.train.Checkpoint(l1=fake_layer)\n",
         "new_root = tf.train.Checkpoint(net=fake_net)\n",
         "status = new_root.restore(tf.train.latest_checkpoint('./tf_ckpts/'))\n",
-        "print(to_restore.numpy())  # We get the restored value now"
+        "print(to_restore.numpy())  # This gets the restored value."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GqEW-_pJDAnE"
       },
       "source": [
-        "The dependency graph for these new objects is a much smaller subgraph of the larger checkpoint we wrote above. It includes only the bias and a save counter that `tf.train.Checkpoint` uses to number checkpoints.\n",
+        "The dependency graph for these new objects is a much smaller subgraph of the larger checkpoint you wrote above. It includes only the bias and a save counter that `tf.train.Checkpoint` uses to number checkpoints.\n",
         "\n",
-        "![Visualization of a subgraph for the bias variable](http://tensorflow.org/images/guide/partial_checkpoint.svg)\n",
+        "![Visualization of a subgraph for the bias variable](https://tensorflow.org/images/guide/partial_checkpoint.svg)\n",
         "\n",
-        "`restore()` returns a status object, which has optional assertions. All of the objects we've created in our new `Checkpoint` have been restored, so `status.assert_existing_objects_matched()` passes."
+        "`restore` returns a status object, which has optional assertions. All of the objects created in the new `Checkpoint` have been restored, so `status.assert_existing_objects_matched` passes."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "P9TQXl81Dq5r"
       },
       "outputs": [],
@@ -534,234 +479,197 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GoMwf8CFDu9r"
       },
       "source": [
-        "There are many objects in the checkpoint which haven't matched, including the layer's kernel and the optimizer's variables. `status.assert_consumed()` only passes if the checkpoint and the program match exactly, and would throw an exception here."
+        "There are many objects in the checkpoint which haven't matched, including the layer's kernel and the optimizer's variables. `status.assert_consumed` only passes if the checkpoint and the program match exactly, and would throw an exception here."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "KCcmJ-2j9RUP"
       },
       "source": [
-        "### Delayed restorations\n",
+        "### Deferred restorations\n",
         "\n",
-        "`Layer` objects in TensorFlow may delay the creation of variables to their first call, when input shapes are available. For example the shape of a `Dense` layer's kernel depends on both the layer's input and output shapes, and so the output shape required as a constructor argument is not enough information to create the variable on its own. Since calling a `Layer` also reads the variable's value, a restore must happen between the variable's creation and its first use.\n",
+        "`Layer` objects in TensorFlow may defer the creation of variables to their first call, when input shapes are available. For example, the shape of a `Dense` layer's kernel depends on both the layer's input and output shapes, and so the output shape required as a constructor argument is not enough information to create the variable on its own. Since calling a `Layer` also reads the variable's value, a restore must happen between the variable's creation and its first use.\n",
         "\n",
-        "To support this idiom, `tf.train.Checkpoint` queues restores which don't yet have a matching variable."
+        "To support this idiom, `tf.train.Checkpoint` defers restores which don't yet have a matching variable."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "TXYUCO3v-I72"
       },
       "outputs": [],
       "source": [
-        "delayed_restore = tf.Variable(tf.zeros([1, 5]))\n",
-        "print(delayed_restore.numpy())  # Not restored; still zeros\n",
-        "fake_layer.kernel = delayed_restore\n",
-        "print(delayed_restore.numpy())  # Restored"
+        "deferred_restore = tf.Variable(tf.zeros([1, 5]))\n",
+        "print(deferred_restore.numpy())  # Not restored; still zeros\n",
+        "fake_layer.kernel = deferred_restore\n",
+        "print(deferred_restore.numpy())  # Restored"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-DWhJ3glyobN"
       },
       "source": [
         "### Manually inspecting checkpoints\n",
         "\n",
-        "`tf.train.list_variables` lists the checkpoint keys and shapes of variables in a checkpoint. Checkpoint keys are paths in the graph displayed above."
+        "`tf.train.load_checkpoint` returns a `CheckpointReader` that gives lower level access to the checkpoint contents. It contains mappings from each variable's key, to the shape and dtype for each variable in the checkpoint. A variable's key is its object path, like in the graphs displayed above.\n",
+        "\n",
+        "Note: There is no higher level structure to the checkpoint. It only know's the paths and values for the variables, and has no concept of `models`, `layers` or how they are connected."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "RlRsADTezoBD"
       },
       "outputs": [],
       "source": [
-        "tf.train.list_variables(tf.train.latest_checkpoint('./tf_ckpts/'))"
+        "reader = tf.train.load_checkpoint('./tf_ckpts/')\n",
+        "shape_from_key = reader.get_variable_to_shape_map()\n",
+        "dtype_from_key = reader.get_variable_to_dtype_map()\n",
+        "\n",
+        "sorted(shape_from_key.keys())"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "5fxk_BnZ4W1b"
+        "id": "AVrdvbNvgq5V"
       },
       "source": [
-        "### List and dictionary tracking\n",
-        "\n",
-        "As with direct attribute assignments like `self.l1 = tf.keras.layers.Dense(5)`, assigning lists and dictionaries to attributes will track their contents."
+        "So if you're interested in the value of `net.l1.kernel` you can get the value with the following code:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "rfaIbDtDHAr_"
+        "id": "lYhX_XWCgl92"
       },
       "outputs": [],
       "source": [
-        "save = tf.train.Checkpoint()\n",
-        "save.listed = [tf.Variable(1.)]\n",
-        "save.listed.append(tf.Variable(2.))\n",
-        "save.mapped = {'one': save.listed[0]}\n",
-        "save.mapped['two'] = save.listed[1]\n",
-        "save_path = save.save('./tf_list_example')\n",
+        "key = 'net/l1/kernel/.ATTRIBUTES/VARIABLE_VALUE'\n",
         "\n",
-        "restore = tf.train.Checkpoint()\n",
-        "v2 = tf.Variable(0.)\n",
-        "assert 0. == v2.numpy()  # Not restored yet\n",
-        "restore.mapped = {'two': v2}\n",
-        "restore.restore(save_path)\n",
-        "assert 2. == v2.numpy()"
+        "print(\"Shape:\", shape_from_key[key])\n",
+        "print(\"Dtype:\", dtype_from_key[key].name)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "UTKvbxHcI3T2"
+        "id": "2Zk92jM5gRDW"
       },
       "source": [
-        "You may notice wrapper objects for lists and dictionaries. These wrappers are checkpointable versions of the underlying data-structures. Just like the attribute based loading, these wrappers restore a variable's value as soon as it's added to the container."
+        "It also provides a `get_tensor` method allowing you to inspect the value of a variable:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "s0Uq1Hv5JCmm"
+        "id": "cDJO3cgmecvi"
       },
       "outputs": [],
       "source": [
-        "restore.listed = []\n",
-        "print(restore.listed)  # ListWrapper([])\n",
-        "v1 = tf.Variable(0.)\n",
-        "restore.listed.append(v1)  # Restores v1, from restore() in the previous cell\n",
-        "assert 1. == v1.numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "OxCIf2J6JyQ8"
-      },
-      "source": [
-        "The same tracking is automatically applied to subclasses of `tf.keras.Model`, and may be used for example to track lists of layers."
+        "reader.get_tensor(key)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "zGG1tOM0L6iM"
+        "id": "5fxk_BnZ4W1b"
       },
       "source": [
-        "## Saving object-based checkpoints with Estimator\n",
+        "### Object tracking\n",
         "\n",
-        "See the [Estimator guide](https://www.tensorflow.org/guide/estimator).\n",
+        "Checkpoints save and restore the values of `tf.Variable` objects by \"tracking\" any variable or trackable object set in one of its attributes.  When executing a save, variables are gathered recursively from all of the reachable tracked objects.\n",
         "\n",
-        "Estimators by default save checkpoints with variable names rather than the object graph described in the previous sections. `tf.train.Checkpoint` will accept name-based checkpoints, but variable names may change when moving parts of a model outside of the Estimator's `model_fn`. Saving object-based checkpoints makes it easier to train a model inside an Estimator and then use it outside of one."
+        "As with direct attribute assignments like `self.l1 = tf.keras.layers.Dense(5)`, assigning lists and dictionaries to attributes will track their contents."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-8AMJeueNyoM"
+        "id": "rfaIbDtDHAr_"
       },
       "outputs": [],
       "source": [
-        "import tensorflow.compat.v1 as tf_compat"
+        "save = tf.train.Checkpoint()\n",
+        "save.listed = [tf.Variable(1.)]\n",
+        "save.listed.append(tf.Variable(2.))\n",
+        "save.mapped = {'one': save.listed[0]}\n",
+        "save.mapped['two'] = save.listed[1]\n",
+        "save_path = save.save('./tf_list_example')\n",
+        "\n",
+        "restore = tf.train.Checkpoint()\n",
+        "v2 = tf.Variable(0.)\n",
+        "assert 0. == v2.numpy()  # Not restored yet\n",
+        "restore.mapped = {'two': v2}\n",
+        "restore.restore(save_path)\n",
+        "assert 2. == v2.numpy()"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "T6fQsBzJQN2y"
+        "id": "UTKvbxHcI3T2"
       },
-      "outputs": [],
       "source": [
-        "def model_fn(features, labels, mode):\n",
-        "  net = Net()\n",
-        "  opt = tf.keras.optimizers.Adam(0.1)\n",
-        "  ckpt = tf.train.Checkpoint(step=tf_compat.train.get_global_step(),\n",
-        "                             optimizer=opt, net=net)\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    output = net(features['x'])\n",
-        "    loss = tf.reduce_mean(tf.abs(output - features['y']))\n",
-        "  variables = net.trainable_variables\n",
-        "  gradients = tape.gradient(loss, variables)\n",
-        "  return tf.estimator.EstimatorSpec(\n",
-        "    mode,\n",
-        "    loss=loss,\n",
-        "    train_op=tf.group(opt.apply_gradients(zip(gradients, variables)),\n",
-        "                      ckpt.step.assign_add(1)),\n",
-        "    # Tell the Estimator to save \"ckpt\" in an object-based format.\n",
-        "    scaffold=tf_compat.train.Scaffold(saver=ckpt))\n",
-        "\n",
-        "tf.keras.backend.clear_session()\n",
-        "est = tf.estimator.Estimator(model_fn, './tf_estimator_example/')\n",
-        "est.train(toy_dataset, steps=10)"
+        "You may notice wrapper objects for lists and dictionaries. These wrappers are checkpointable versions of the underlying data-structures. Just like the attribute based loading, these wrappers restore a variable's value as soon as it's added to the container."
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "tObYHnrrb_mL"
+        "id": "s0Uq1Hv5JCmm"
       },
+      "outputs": [],
       "source": [
-        "`tf.train.Checkpoint` can then load the Estimator's checkpoints from its `model_dir`."
+        "restore.listed = []\n",
+        "print(restore.listed)  # ListWrapper([])\n",
+        "v1 = tf.Variable(0.)\n",
+        "restore.listed.append(v1)  # Restores v1, from restore() in the previous cell\n",
+        "assert 1. == v1.numpy()"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Q6IP3Y_wb-fs"
+        "id": "OxCIf2J6JyQ8"
       },
-      "outputs": [],
       "source": [
-        "opt = tf.keras.optimizers.Adam(0.1)\n",
-        "net = Net()\n",
-        "ckpt = tf.train.Checkpoint(\n",
-        "  step=tf.Variable(1, dtype=tf.int64), optimizer=opt, net=net)\n",
-        "ckpt.restore(tf.train.latest_checkpoint('./tf_estimator_example/'))\n",
-        "ckpt.step.numpy()  # From est.train(..., steps=10)"
+        "Trackable objects include `tf.train.Checkpoint`, `tf.Module` and its subclasses (e.g. `keras.layers.Layer` and `keras.Model`), and recognized Python containers:\n",
+        "\n",
+        " * `dict` (and `collections.OrderedDict`)\n",
+        " * `list`\n",
+        " * `tuple` (and `collections.namedtuple`, `typing.NamedTuple`)\n",
+        "\n",
+        "Other container types are **not supported**, including:\n",
+        "\n",
+        " * `collections.defaultdict`\n",
+        " * `set`\n",
+        "\n",
+        "All other Python objects are **ignored**, including:\n",
+        "\n",
+        " * `int`\n",
+        " * `string`\n",
+        " * `float`\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "knyUFMrJg8y4"
       },
       "source": [
@@ -775,8 +683,6 @@
     "colab": {
       "collapsed_sections": [],
       "name": "checkpoint.ipynb",
-      "private_outputs": true,
-      "provenance": [],
       "toc_visible": true
     },
     "kernelspec": {
diff --git a/site/en/guide/concrete_function.ipynb b/site/en/guide/concrete_function.ipynb
deleted file mode 100644
index f76c6e947af..00000000000
--- a/site/en/guide/concrete_function.ipynb
+++ /dev/null
@@ -1,2559 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "YYZqhjGhK8nJ"
-      },
-      "source": [
-        "##### Copyright 2018 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "V1-OvloqK4CX"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\"); { display-mode: \"form\" }\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "VKRNwMTxmZXx"
-      },
-      "source": [
-        "# Concrete functions\n",
-        "\n",
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/concrete_function\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n",
-        "    View on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/concrete_function.ipynb\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n",
-        "    Run in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/concrete_function.ipynb\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n",
-        "    View source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/concrete_function.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "VgQvKIvXockG"
-      },
-      "source": [
-        "In the guide to [AutoGraph and `tf.functions`](function.ipynb) you saw how to use `tf.function`. This guide dives into the details of: \n",
-        "\n",
-        "* `tf.function` Tracing\n",
-        "* `tf.function` Signatures\n",
-        "* The Concrete functions generated by tracing:\n",
-        "  * How to access them\n",
-        "  * How to use them\n",
-        "\n",
-        "These details only become important:\n",
-        "\n",
-        "* If you're experiencing performance issues due to undesired tracing\n",
-        "  of a `tf.funcion`.\n",
-        "* When you need precise control over the TensorFlow Graphs generated by\n",
-        "  `tf.function`. For example for exporting the model to\n",
-        "  [TFLite](https://tensorflow.org/lite/converter) using\n",
-        "  `tf.lite.Converter.from_concrete_functions`.\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LwLkhiXIrT5m"
-      },
-      "source": [
-        "## Background\n",
-        "\n",
-        "In TensorFlow 2, eager execution is on by default. TensorFlow's eager\n",
-        "execution is an imperative programming environment that evaluates operations\n",
-        "immediately, without building graphs. Operations return values instead\n",
-        "of constructing a computational graph to run later. Here is a [detailed guide on eager\n",
-        "execution](eager.ipynb).\n",
-        "\n",
-        "Running imperatively makes development and debugging\n",
-        "more interactive, but doesn't allow for easy exporting.\n",
-        "\n",
-        "The `tf.function` API makes it possible to save models as graphs."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "GgJHWoOPrWtK"
-      },
-      "source": [
-        "## Terminology\n",
-        "\n",
-        "The following terminology is used in this document:\n",
-        "\n",
-        "*   **Signature** - A description of the inputs and outputs for a set of operations.\n",
-        "* **Polymorphic function** - Python callable that encapsulates several\n",
-        "    concrete function graphs behind one API.  \n",
-        "* **Concrete function** - Graph with a single signature.\n",
-        "  \n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vq126xVZr2CL"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "jtRSqqP6o_pQ"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "3_jlx1_ir1hb"
-      },
-      "outputs": [],
-      "source": [
-        "import traceback\n",
-        "import textwrap\n",
-        "\n",
-        "try:\n",
-        "  !pip install tf-nightly\n",
-        "except Exception:\n",
-        "  pass"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "AB6fMBGSofmm"
-      },
-      "outputs": [],
-      "source": [
-        "import tensorflow as tf"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "TpkKbIBerbWi"
-      },
-      "source": [
-        "## Create a `tf.function`\n",
-        "\n",
-        "Annotating a function with `tf.function` generates a *polymorphic function*\n",
-        "containing those operations. All operations that are not annotated with\n",
-        "`tf.function` will be evaluated with eager execution. The examples below show a quick example of `tf.function` usage."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "fPxeVDQ2mZXy"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function\n",
-        "def square(x):\n",
-        "  return x*x"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "yDO0s_LWBc9S"
-      },
-      "outputs": [],
-      "source": [
-        "square(2).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "8AhtB2hAsByK"
-      },
-      "source": [
-        "Remember that the python decorator syntax just calls the decorator with the decorated object as input:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "p-KckIFcmZX2"
-      },
-      "outputs": [],
-      "source": [
-        "def pow(x,y):\n",
-        "  return x ** y\n",
-        "\n",
-        "pow = tf.function(pow)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1moLFTi2sToR"
-      },
-      "outputs": [],
-      "source": [
-        "pow(3,4).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "hMhMhl3lmZX6"
-      },
-      "source": [
-        "### Attach a `tf.function` method to a `tf.Module`\n",
-        "\n",
-        "The `tf.function` can be optionally stored as part of a `tf.Module` object. The `tf.Module` class provides features for tracking variables and saving [checkpoints](checkpoints.ipynb) and [models](saved_models).\n",
-        "\n",
-        "Classes like `keras.layers.Layer` and `keras.Model` are subclasses of Module."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ht2K2vpDmZX7"
-      },
-      "outputs": [],
-      "source": [
-        "class Pow(tf.Module):\n",
-        "  def __init__(self, exponent):\n",
-        "    self.exponent = tf.Variable(exponent, dtype = tf.float32, name='Pow/exponent')\n",
-        "\n",
-        "  @tf.function\n",
-        "  def __call__(self, x):\n",
-        "    return x ** self.exponent"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1GVBiQfx8klb"
-      },
-      "outputs": [],
-      "source": [
-        "pow = Pow(3)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "KqnKlZe58nPe"
-      },
-      "outputs": [],
-      "source": [
-        "pow.variables"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "RQBVvuxLtGz2"
-      },
-      "outputs": [],
-      "source": [
-        "pow(tf.constant(2.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "kKiJrZlS5eeT"
-      },
-      "outputs": [],
-      "source": [
-        "pow.exponent.assign(4)\n",
-        "pow(tf.constant(2.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "t_HlUmTe5reS"
-      },
-      "outputs": [],
-      "source": [
-        "tf.saved_model.save(pow, 'pow')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "abpBoMRR5ypu"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded_pow = tf.saved_model.load('pow')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "GA_a3CH956MR"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded_pow(tf.constant(3.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rggP9vCK60R4"
-      },
-      "source": [
-        "### Assign a `tf.function` as an attribute\n",
-        "If you assign a `tf.Module` or a `tf.function` as an attribute of a module it will be serialized as well:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "kR2-Mwyg7GyO"
-      },
-      "outputs": [],
-      "source": [
-        "mod = tf.Module()\n",
-        "mod.increment_by = tf.Variable(2.0)\n",
-        "\n",
-        "@tf.function\n",
-        "def increment(x):\n",
-        "  return x+mod.increment_by\n",
-        "\n",
-        "mod.inc = increment\n",
-        "mod.inc(tf.constant(1.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "rR5D4q7Z76Uo"
-      },
-      "outputs": [],
-      "source": [
-        "mod.cube = Pow(3)\n",
-        "mod.cube(tf.constant(2.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "vNIzKT8I8XzC"
-      },
-      "outputs": [],
-      "source": [
-        "mod.variables"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "OQqnUgjz7QOW"
-      },
-      "outputs": [],
-      "source": [
-        "tf.saved_model.save(mod, 'mod')\n",
-        "reloaded_mod = tf.saved_model.load('mod')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Uvikzz7x7jB1"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded_mod.inc(4.0).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "n7eVyFhT8Bq6"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded_mod.cube(4.0).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Sj7P9wBFmZYW"
-      },
-      "source": [
-        "### Interoperability with `tf.keras`\n",
-        "\n",
-        "Keras classes like `keras.Model` and `keras.layers.Layer` are fully compatible with `tf.function` and `tf.Module`.\n",
-        "\n",
-        "For example, build a simple model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "WYHD6NMWmZYX"
-      },
-      "outputs": [],
-      "source": [
-        "linear = tf.keras.Sequential([tf.keras.layers.Dense(units=1, input_shape=[1])])\n",
-        "linear.compile(optimizer='adam', loss='mean_squared_error')\n",
-        "linear.fit(x=[-1, 0, 1, 2, 3, 4], y=[-3, -1, 1, 3, 5, 7], epochs=50, verbose=0)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "5rumHqek5oMN"
-      },
-      "outputs": [],
-      "source": [
-        "linear(tf.constant([[1],[2]]))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "dx2HIyx_UI9N"
-      },
-      "source": [
-        "Inspect it's variables"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "vLpavpL-86kn"
-      },
-      "outputs": [],
-      "source": [
-        "linear.variables"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "JEGvkaGH5hIZ"
-      },
-      "source": [
-        "Now attach it to a `tf.Module`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "muWIMr6-5B8A"
-      },
-      "outputs": [],
-      "source": [
-        "module = tf.Module()\n",
-        "module.linear = linear"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "SF24uHF59kH4"
-      },
-      "source": [
-        "The `tf.Module` also tracks the `tf.Variable`s:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Z6ovPDgT5RYz"
-      },
-      "outputs": [],
-      "source": [
-        "module.variables"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-ZyWZdJVtDp1"
-      },
-      "source": [
-        "The `tf.Module` will export the contents of the `keras.Model` as well:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ZgNSWOoi6ioK"
-      },
-      "outputs": [],
-      "source": [
-        "tf.saved_model.save(module,'module')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "RS3j67MK9zVR"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded = tf.saved_model.load('module')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "pxDCTrZd965w"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded.linear([[1.0]])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ROhab4u7idvG"
-      },
-      "source": [
-        "## Tracing"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "zQaBkCHB-W0Q"
-      },
-      "source": [
-        "The objects returned from `tf.function` are polymorphic functions. They will accept python objects, or `tf.Tensors` with any shape or `tf.dtype` as input.\n",
-        "\n",
-        "In the background TensorFlow builds `tf.Graph`s representing the calculation. \n",
-        "This graph is wrapped in a python callable: a concrete function. Each concrete function can only handle a single input signature.\n",
-        "\n",
-        "`tf.function` traces the python function each time in needs to create a concrete function. The easiest way to see when a function is traced is to add a call to print: \n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "FgvSApDrmJGq"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function\n",
-        "def mul(a, b):\n",
-        "  print('Tracing:\\n    {a}\\n    {b}\\n'.format(a=a, b=b))\n",
-        "  return a*b"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "jxzhLeKV_chb"
-      },
-      "source": [
-        "### Dtypes and shapes"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "i_OnPSdOCI7J"
-      },
-      "source": [
-        "If you call the polymorphic function with two different types of input, it will trace once for each:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "A2uKXbRMmTGj"
-      },
-      "outputs": [],
-      "source": [
-        "# Trace with ints\n",
-        "mul(tf.constant(2), tf.constant(3)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "a_rcmkWxCVwE"
-      },
-      "outputs": [],
-      "source": [
-        "# Trace with floats\n",
-        "mul(tf.constant(2.0), tf.constant(3.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Jmqq9rrDCdI4"
-      },
-      "source": [
-        "When you call it again with the same input types, it dispatches to an existing function instead of tracing:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "KFtAlPZPqM7A"
-      },
-      "outputs": [],
-      "source": [
-        "# Call with ints again =\u003e no trace\n",
-        "mul(tf.constant(10), tf.constant(10))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "mH920uLTCwK1"
-      },
-      "source": [
-        "Changing the sizes of the inputs also triggers a trace (setting `tf.function(experimental_relax_shapes=True)` may reduce this): "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "eb_UByibqGny"
-      },
-      "outputs": [],
-      "source": [
-        "# Trace with vectors\n",
-        "mul(tf.constant([1.0,3.0]), tf.constant(3.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "eXm0fgVzqVxT"
-      },
-      "outputs": [],
-      "source": [
-        "# Trace with different-sized vectors\n",
-        "mul(tf.constant([1.0,2.0,3.0, 4.0]), tf.constant(3.0))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "D-LoC287Ax5k"
-      },
-      "source": [
-        "### Immutable python objects"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "RpkfdNeKFbiU"
-      },
-      "source": [
-        "If you pass an immutable python object, like a `int`, `str`, or `tuple` to a `tf.function`, it executes a trace for each *value* of those python objects.\n",
-        "\n",
-        "This is useful to control what gets included in the `tf.Graph` (See: [The Autograph Guide](function.ipynb) for more details).\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-G1N7qh4BAhe"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function\n",
-        "def mul(a, b):\n",
-        "  print('Tracing:\\n    {a}\\n    {b}\\n'.format(a=a, b=b))\n",
-        "  return a*b"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ftCJz7HfDGof"
-      },
-      "outputs": [],
-      "source": [
-        "# Trace for a=3.0\n",
-        "mul(3.0, tf.constant(3.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "C8hiJCP_Du0I"
-      },
-      "outputs": [],
-      "source": [
-        "# Don't trace for a=3.0 the second time:\n",
-        "mul(3.0, tf.constant(3.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Sm8Cq0uTt88j"
-      },
-      "source": [
-        "Caution: It is easy to cause many traces by passing unique python values. This can be a significant performance problem. Often passing a `tf.Tensor` value is the solution.\n",
-        "\n",
-        "This loop traces the function for each unique int:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "4LTTGqQryjsj"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function\n",
-        "def power(a,b):\n",
-        "  print('Tracing \"power\": a={}'.format(a))\n",
-        "  return a**b"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "yyGenHr_t8LK"
-      },
-      "outputs": [],
-      "source": [
-        "p = tf.constant(2)\n",
-        "for n in range(12):\n",
-        "  power(n,p)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "xYgJ9l7_yY20"
-      },
-      "source": [
-        "On the second run each int has been traced, so there's no tracing to do:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "duPVVOg3yUCl"
-      },
-      "outputs": [],
-      "source": [
-        "p = tf.constant(2)\n",
-        "for n in range(12):\n",
-        "  power(n,p)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "6bX9b1t0uTQX"
-      },
-      "source": [
-        "To avoid excess retracing be sure to pass a `tf.Tensor` instead of python numbers or strings:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "F78ik9psuPn7"
-      },
-      "outputs": [],
-      "source": [
-        "p = tf.constant(2)\n",
-        "for n in tf.range(12):\n",
-        "  power(n,p)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "N-wwjPtF1Q25"
-      },
-      "source": [
-        "To shut off tracing altogether, pass a signature to the `tf.function` decorator:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "9EcEZfZY1Ylg"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function(input_signature=(\n",
-        "    tf.TensorSpec(shape=[], dtype=tf.float32),\n",
-        "    tf.TensorSpec(shape=[], dtype=tf.float32),)\n",
-        ")\n",
-        "def power_with_sig(a,b):\n",
-        "  print('Tracing \"power_with_sig\"')\n",
-        "  return a**b"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Pff46f4U1-6R"
-      },
-      "outputs": [],
-      "source": [
-        "power_with_sig(3.0, 3.0).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "cBT2IFiZ1wLS"
-      },
-      "outputs": [],
-      "source": [
-        "try:\n",
-        "  power_with_sig(tf.constant([1.0,2.0,3.0]),tf.constant(3.0))\n",
-        "  assert False\n",
-        "except ValueError:\n",
-        "  traceback.print_exc(limit=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kjeEQ4J4tt81"
-      },
-      "source": [
-        "### Example: Dropout"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-QbejAfEDLNX"
-      },
-      "source": [
-        "Retracing for specific values gives you control over what code gets generated by the `tf.function`.\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "kIgq7vO1sXh7"
-      },
-      "outputs": [],
-      "source": [
-        "class Dropout(tf.Module):\n",
-        "  def __init__(self, rate, name=None):\n",
-        "    super(Dropout, self).__init__(name)\n",
-        "    self.rate = tf.Variable(rate, dtype = tf.float32, trainable=False)\n",
-        "\n",
-        "  @tf.function\n",
-        "  def __call__(self, x, training=True):\n",
-        "    print(textwrap.dedent(\"\"\"\n",
-        "                          Tracing \"Dropout\":\n",
-        "                              training = {}\n",
-        "                              x = {}\n",
-        "                              name = {:s}\n",
-        "                          \"\"\".format(training, x, self.name)))\n",
-        "    if training:\n",
-        "      print('    - Train branch\\n')\n",
-        "      mask = tf.random.uniform(x.shape) \u003e self.rate\n",
-        "      return x * tf.cast(mask, tf.float32)/self.rate\n",
-        "    else:\n",
-        "      print('    - Test branch\\n')\n",
-        "      return x"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "o7TFSeINqhtc"
-      },
-      "source": [
-        "Create an instance of this simple `Dropout` layer:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "5DHE1io1pN2t"
-      },
-      "outputs": [],
-      "source": [
-        "dropout = Dropout(0.5)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "CE4fuvUP0BWE"
-      },
-      "source": [
-        "The first time you call it with a python `training=True` as input, it traces the `training` branch:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "O5xIiGATqNQY"
-      },
-      "outputs": [],
-      "source": [
-        "dropout(tf.range(10, dtype=tf.float32), training=True).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Cup3pXHI0WWH"
-      },
-      "source": [
-        "The second time, it doesn't need to re-trace the branch:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "VaMDgyM8zpge"
-      },
-      "outputs": [],
-      "source": [
-        "dropout(tf.range(10, dtype=tf.float32), training=True).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "yhiyPC7s0dF9"
-      },
-      "source": [
-        "Passing `training=False` triggers a trace on the first run since this is a different python value:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mgaR5PFTzk9A"
-      },
-      "outputs": [],
-      "source": [
-        "dropout(tf.range(10, dtype=tf.float32), training=False).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "DVeAfMkbzn-V"
-      },
-      "outputs": [],
-      "source": [
-        "dropout(tf.range(10, dtype=tf.float32), training=False).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "G6b57DUG32b7"
-      },
-      "source": [
-        "If you pass a `bool` tensor, it uses TensorFlow autograph rewrite the `if` to a `tf.cond`m and traces both branches:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "s_FsCDEaqYqq"
-      },
-      "outputs": [],
-      "source": [
-        "dropout(tf.range(10, dtype=tf.float32), training=tf.constant(False)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "UbmPUjhI8Rgf"
-      },
-      "source": [
-        "This captures the control flow in a single concrete function."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "BViHlbCN0wss"
-      },
-      "outputs": [],
-      "source": [
-        " dropout(tf.range(10, dtype=tf.float32), training=tf.constant(True)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "p8eT9zKXp8F5"
-      },
-      "outputs": [],
-      "source": [
-        "dropout(tf.range(10, dtype=tf.float32), training=tf.constant(False)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "smyOmjl2GCdt"
-      },
-      "source": [
-        "### Other python objects\n",
-        "\n",
-        "Since the generated `tf.Graphs` cannot contain complex python objects, these are included by tracing and variable capture. \n",
-        "\n",
-        "The `tf.function` runs a separate trace for each **instance**. So each trace includes its own variables, and can set its behavior based on the instance.\n",
-        "\n",
-        "The most common usage is on methods of Module, Layer or Module:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "C6SBvOEdO5Qw"
-      },
-      "outputs": [],
-      "source": [
-        "dropout_a = Dropout(0.5, name='dropout_a')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "uOC8COMxQY0J"
-      },
-      "outputs": [],
-      "source": [
-        "print(dropout_a(tf.range(10, dtype=tf.float32), True).numpy())\n",
-        "print(dropout_a(tf.range(10, dtype=tf.float32), True).numpy())"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mISom9-ZTf5g"
-      },
-      "outputs": [],
-      "source": [
-        "dropout_b = Dropout(0.5, name='dropout_b')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "95OE0xxeQ4t0"
-      },
-      "outputs": [],
-      "source": [
-        "print(dropout_b(tf.range(10, dtype=tf.float32), True).numpy())\n",
-        "print(dropout_b(tf.range(10, dtype=tf.float32), True).numpy())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Ts8nQVMcQfsK"
-      },
-      "source": [
-        "But the behavior is the same on a stand-alone `tf.function`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0xVE3zoLs61l"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function\n",
-        "def run(callable, x):\n",
-        "  print('Tracing \"run\":\\n    callable = {}\\n    x = {}\\n'.format(callable, x))\n",
-        "  return callable(x)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "WAs-suN2tjH0"
-      },
-      "outputs": [],
-      "source": [
-        "def plus_1(x):\n",
-        "  return x+1\n",
-        "\n",
-        "print(run(plus_1, tf.constant(2.0)).numpy())\n",
-        "print(run(plus_1, tf.constant(5.0)).numpy())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "g_m9M64oXFeW"
-      },
-      "source": [
-        "The tracing one `tf.function` can trigger tracing in another:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "lnib6kNTu4bF"
-      },
-      "outputs": [],
-      "source": [
-        "print(run(dropout, tf.range(10.0)).numpy())\n",
-        "print(run(dropout, tf.range(10.0)).numpy())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "4DE76z8KiVTG"
-      },
-      "source": [
-        "### Weak references"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LKaSfW5likUT"
-      },
-      "source": [
-        "Caution: Each trace only keeps a [weak-reference](https://docs.python.org/3/library/weakref.html) to any `tf.Variable`. If the variable is not kept alive by another reference, the trace may become unusable."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "yGUY0lVD8nMT"
-      },
-      "source": [
-        "For example here's a `tf.function` that refers to `var` from the enclosing scope:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "zwwnJwXR8PEb"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function\n",
-        "def plus_var(x):\n",
-        "  print('Tracing \"plus_var\":\\n    x = {}\\n    var = {}\\n\\n'.format(x, var.name))\n",
-        "  return x + var"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "9qmxAmFl8v5p"
-      },
-      "source": [
-        "Trace the function with one variable:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Nebd4e5y8UYa"
-      },
-      "outputs": [],
-      "source": [
-        "var = tf.Variable(1, name=\"IntVar\")\n",
-        "plus_var(tf.constant([1,2])).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "DaqblviJ8y6v"
-      },
-      "source": [
-        "And with another variable:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "s3EwG-yt8Zgx"
-      },
-      "outputs": [],
-      "source": [
-        "var = tf.Variable(2.0, name=\"FloatVar\")\n",
-        "plus_var(tf.constant([2.0, 10.0])).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "cJL43oZN86o4"
-      },
-      "source": [
-        "That worked, but because you no longer have a reference to `\"IntVar\"`, that first trace is broken:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Yb4RkESA8gs2"
-      },
-      "outputs": [],
-      "source": [
-        "try:\n",
-        "  plus_var(tf.constant([1,2])).numpy()\n",
-        "  assert False\n",
-        "except tf.errors.FailedPreconditionError:\n",
-        "  traceback.print_exc(limit=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "235EbM2HmZYB"
-      },
-      "source": [
-        "## Accessing concrete function\n",
-        "\n",
-        "In the previous section you saw the conditions for triggering a new trace of a polymorphic `tf.function`. Each trace generates a new concrete function.\n",
-        "\n",
-        "When you save `tf.Module` as a `tf.saved_model` It's those concrete functions that define the `tf.Graph`s that are exported. You don't save a `tf.function` you save the concrete functions that are created by tracing. \n",
-        "\n",
-        "To get a concrete function from the polymorphic `tf.function` you need to define the signature. Either:\n",
-        "\n",
-        "*   Pass an `input_signature` to `tf.function`, and call the \n",
-        "    `get_concrete_function()` method.\n",
-        "*   Pass a list of `tf.TensorSpec`s to `get_concrete_function`: `tf.TensorSpec(shape=[1], dtype=tf.float32)`.\n",
-        "*   Pass an example tensor of the correct shape and type to\n",
-        "    `get_concrete_function`: `tf.constant(1., shape=[1])`.\n",
-        "\n",
-        "The following example shows how to define the `input_signature` parameter for\n",
-        "`tf.function`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "O-tE5Pn6h1DU"
-      },
-      "source": [
-        "#### Using `input_signature`"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "hsMR5JBdhykq"
-      },
-      "source": [
-        "Specify input tensors in the call to `tf.function` as shown below.\n",
-        "This `tf.function`can only execute on tensors that match the specified signatutre.\n",
-        "\n",
-        "A `None` in the `shape` acts a wildcard. So this these `tf.TensroSpec` say \"A float32 vector of any length\".\n",
-        "\n",
-        "This pattern can be very important if your `tf.function` is expected to handle sequences of different length, or images of different sizes for each batch (See [Transformer](../tutorials/text/transformer.ipynb) and [Deep Dream](../tutorials/generative/deepdream.ipynb) tutrorials for example)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "INJhW9c5htAK"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function(input_signature=(\n",
-        "    tf.TensorSpec(shape=[None], dtype=tf.float32),\n",
-        "    tf.TensorSpec(shape=[None], dtype=tf.float32),)\n",
-        ")\n",
-        "def power_with_sig(a,b):\n",
-        "  print('Tracing \"power_with_sig\"\\n')\n",
-        "  return a**b"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Q0U5ytANi4V7"
-      },
-      "source": [
-        "Calling `get_concrete_function` will execute the trace (if necessary), and return a concrete function."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "yR3tqv1pijvh"
-      },
-      "outputs": [],
-      "source": [
-        "p = power_with_sig.get_concrete_function()\n",
-        "type(p)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "XvGhi41EipeQ"
-      },
-      "outputs": [],
-      "source": [
-        "p(tf.constant([2.0,3.0,4.0]), tf.constant([5.0,4.0,3.0])).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "bBKfDEeMz43m"
-      },
-      "source": [
-        "### Using `get_concrete_function`"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "IGlmMUSK0c3H"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function\n",
-        "def power(a,b):\n",
-        "  print('Tracing \"power\"\\n')\n",
-        "  return a**b"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "NYbxGotFqoDB"
-      },
-      "outputs": [],
-      "source": [
-        "float_power = power.get_concrete_function(\n",
-        "  a = tf.TensorSpec(shape=[], dtype=tf.float32),\n",
-        "  b = tf.TensorSpec(shape=[], dtype=tf.float32))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1cXOt1g0qpCL"
-      },
-      "outputs": [],
-      "source": [
-        "float_power(tf.constant(3.0),tf.constant(3.0))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "0dhioJFVbyKJ"
-      },
-      "source": [
-        "Remember that you can also pass tensors to `get_concrete_function`, in that case it returns the concrete function that would run for those inputs:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "dzeSYHN1cGke"
-      },
-      "outputs": [],
-      "source": [
-        "row = tf.range(10)\n",
-        "col = tf.constant([[1],[2],[3]])\n",
-        "\n",
-        "concrete_power = power.get_concrete_function(a = row, b = col)\n",
-        "concrete_power(row, col).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "JcsoDz8g9nJT"
-      },
-      "source": [
-        "## Using a concrete function"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "glxZkvRly1Am"
-      },
-      "source": [
-        "A concrete function only accepts tensors as input:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1AIbwzSz0aCj"
-      },
-      "outputs": [],
-      "source": [
-        "float_power(tf.constant(2.0), tf.constant(3.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "7xq-82FokX_d"
-      },
-      "outputs": [],
-      "source": [
-        "try:\n",
-        "  float_power(2.0,3.0)\n",
-        "  assert False\n",
-        "except ValueError:\n",
-        "  traceback.print_exc(limit=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "V1eBDquxy5m5"
-      },
-      "source": [
-        "It also only accepts inputs of the correct dtype:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "nSdGa_IJkayV"
-      },
-      "outputs": [],
-      "source": [
-        "try:\n",
-        "  float_power(tf.constant(1),tf.constant(3))\n",
-        "  assert False\n",
-        "except tf.errors.InvalidArgumentError:\n",
-        "  traceback.print_exc(limit=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kRc4YAxzsSdM"
-      },
-      "source": [
-        "But it will try to execute even if the input tensors do not match the expected shape:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "MT-Z0cwrrrgZ"
-      },
-      "outputs": [],
-      "source": [
-        "float_power(tf.constant([1.,2.,3.,4.,5.]),tf.constant(3.)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "zWr690sEqwu0"
-      },
-      "outputs": [],
-      "source": [
-        "try:\n",
-        "  float_power(tf.constant([1.,2.,3.]),tf.constant([4., 5.])).numpy()\n",
-        "  assert False\n",
-        "except tf.errors.InvalidArgumentError:  \n",
-        "  traceback.print_exc(limit=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "PtHTfAtamxcu"
-      },
-      "source": [
-        "By inspecting the concrete function you can see its inputs and outputs:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "d-8tBcOHmw7x"
-      },
-      "outputs": [],
-      "source": [
-        "print(float_power.structured_input_signature)\n",
-        "print(float_power.structured_outputs)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "64z5eSmMzv8l"
-      },
-      "source": [
-        "## Python Objects in signatures"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "1RTj0n2R0jGH"
-      },
-      "source": [
-        "As you saw when tracing, each python object generates a new trace. Concrete functions represent a single `tf.Graph`, they don't do any retracing. When you call `get_concrete_function` with a python object as one of the arguments the object is **bound** to the function."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "HxQtkTRYz-8u"
-      },
-      "outputs": [],
-      "source": [
-        "cube = power.get_concrete_function(\n",
-        "    a = tf.TensorSpec([], dtype=tf.float32),\n",
-        "    b = 3.0)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Iu8dRqCQ0SGW"
-      },
-      "source": [
-        "This `cube` function no longer has a `b` argument:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "duRJIrDU0L-I"
-      },
-      "outputs": [],
-      "source": [
-        "print(cube.structured_input_signature)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "PnEQtxxA3OgX"
-      },
-      "outputs": [],
-      "source": [
-        "cube(tf.constant(10.0)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "hL76DCMVsdet"
-      },
-      "source": [
-        "This is very similar to the way that standard python classes bind methods, and applies equally when you run `get_concrete_function` from a method:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "dV2R0AOIsdJs"
-      },
-      "outputs": [],
-      "source": [
-        "class Greeter(object):\n",
-        "  def __init__(self, greeting):\n",
-        "    self.greeting = greeting\n",
-        "\n",
-        "  def greet(self, who):\n",
-        "    return \" \".join([self.greeting, who])\n",
-        "\n",
-        "p = Greeter(\"Hello\")\n",
-        "m = p.greet\n",
-        "print(m)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "j9sEWo-wtP_K"
-      },
-      "outputs": [],
-      "source": [
-        "print(m(\"TensorFlow!\"))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "uv4X41_Htqof"
-      },
-      "source": [
-        "When you have a `tf.function` decorating a method, similar rules apply:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "h131HNsj3nYm"
-      },
-      "outputs": [],
-      "source": [
-        "class MyModel(tf.Module):\n",
-        "  def __init__(self, ins, outs):\n",
-        "    initializer = tf.initializers.GlorotNormal()\n",
-        "    self.W = tf.Variable(initializer([ins, outs]))\n",
-        "    self.B = tf.Variable(tf.zeros([outs], dtype = tf.float32))\n",
-        "\n",
-        "  @tf.function\n",
-        "  def run(self, x):\n",
-        "    print('Tracing \"MyModule\":\\n    x={}\\n'.format(x))\n",
-        "    return tf.matmul(x, self.W)+self.B"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "nGtPiUpZ3y0j"
-      },
-      "outputs": [],
-      "source": [
-        "mod = MyModel(ins=5, outs=3)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "3OVnEqJY6BHU"
-      },
-      "outputs": [],
-      "source": [
-        "mod.run([[1.0,1.0,1.0, 1.0, 1.0]]).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ca4tB9QguPBg"
-      },
-      "source": [
-        "If you call the method's `.get_concrete_function`, the `self` is automatically bound as the first argument:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0buNZlUw6XNN"
-      },
-      "outputs": [],
-      "source": [
-        "concrete_run = mod.run.get_concrete_function(x = tf.TensorSpec([None, None]))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "D-v1YUN562wH"
-      },
-      "outputs": [],
-      "source": [
-        "concrete_run(tf.constant([[1.0,1.0,1.0, 1.0, 1.0],\n",
-        "                          [2.0,2.0,2.0, 2.0, 2.0]])).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "voRoyyNBuq5-"
-      },
-      "source": [
-        "See how `self` is no longer part of the input signature:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "t4V1ChR97uNS"
-      },
-      "outputs": [],
-      "source": [
-        "print(concrete_run.structured_input_signature)\n",
-        "print(concrete_run.structured_outputs)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "WlHhcO30mZYN"
-      },
-      "source": [
-        "## Accessing concrete functions from a SavedModel\n",
-        "\n",
-        "When you save a [SavedModel](saved_model.ipynb) you're really saving the `tf.function's` cache of concrete functions.\n",
-        "\n",
-        "Because concrete functions are generated by tracing the input you need to execute at least one trace to save a SavedModel.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "L5iunDSWxS3b"
-      },
-      "outputs": [],
-      "source": [
-        "dropout = Dropout(0.5)\n",
-        "\n",
-        "_ = dropout(tf.range(10, dtype=tf.float32), tf.constant(True))\n",
-        "_ = dropout(tf.random.normal([2, 3]), tf.constant(True))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "QMr2Bz0FQeLN"
-      },
-      "source": [
-        "Note: `tf.saved_model` retraces all `concrete_functions` when saving them. This is to ensure that the exported concrete functions capture changes in the environment on export (e.g. distribution strategy scope)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "OtoZNlykta1L"
-      },
-      "outputs": [],
-      "source": [
-        "export_dir = 'dropout'\n",
-        "tf.saved_model.save(dropout, export_dir)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "83oSBW8jxau1"
-      },
-      "source": [
-        "### Direct access\n",
-        "\n",
-        "When you load a `tf.saved_model` your methods are restored as polymorphic functions:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "o8fVspUQtgZs"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded_dropout = tf.saved_model.load(export_dir)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "4IB6iNTUxLpk"
-      },
-      "outputs": [],
-      "source": [
-        "print(reloaded_dropout(tf.range(10, dtype=tf.float32), tf.constant(False)).numpy())\n",
-        "print(reloaded_dropout(tf.random.normal([2,3]), tf.constant(True)).numpy())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "237FYOTTxKeI"
-      },
-      "source": [
-        "But since the `saved_model` only contains the cache of concrete functions (an d not the python source and data), it cannot handle signatures that don't match: "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "NKIE3TxLwzj8"
-      },
-      "outputs": [],
-      "source": [
-        "try:\n",
-        "  reloaded_dropout(tf.range(12, dtype=tf.float32), tf.constant(True))\n",
-        "  assert False\n",
-        "except ValueError:\n",
-        "  traceback.print_exc(limit=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Fy8Bryv5yHFs"
-      },
-      "source": [
-        "From the reloaded module you can select a specific concrete function instead of relying on the dispatch by, again, using the `get_concrete_function` method: "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mxr0uk9YyGQu"
-      },
-      "outputs": [],
-      "source": [
-        "cf = reloaded_dropout.__call__.get_concrete_function(\n",
-        "    x = tf.TensorSpec([10]), \n",
-        "    training = tf.TensorSpec([], tf.bool))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1jRI6tByyX_8"
-      },
-      "outputs": [],
-      "source": [
-        "result = cf(tf.range(10, dtype=tf.float32), tf.constant(True)).numpy()\n",
-        "print(result)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "BCM5i0pZ2Vku"
-      },
-      "source": [
-        "### Named signatures: Exporting for C++"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-akdpHabyEIy"
-      },
-      "source": [
-        "C++ consumers of SavedModels do not use the above \"Direct Access\" method, or it's dynamic dispatch, to get and run concrete functions from the SavedModel. \n",
-        "\n",
-        "They use a more explicit interface called \"exported signatures\", where you specify exactly which concrete functions to export. \n",
-        "\n",
-        "You specify the concrete functions to export by passing a `signatures` argument to `tf.saved_model.save`.\n",
-        "\n",
-        "It takes either:\n",
-        "\n",
-        "* A dictionary of functions. This allows you to name each function.\n",
-        "* A single function. When a single function is exported, it will be named \"serving_default\", using `saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY`.\n",
-        "\n",
-        "These signatures are required when using TensorFlow Serving.\n",
-        "\n",
-        "Note: These \"exported signatures\" are wraped to always return a dictionary of results."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LeBeEh8t3j9Z"
-      },
-      "source": [
-        "#### Simple example"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "8ji4CDMVzTQn"
-      },
-      "outputs": [],
-      "source": [
-        "dropout = Dropout(0.5)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Sfge8hwHyJOl"
-      },
-      "outputs": [],
-      "source": [
-        "cf = dropout.__call__.get_concrete_function(tf.zeros((2,3), dtype=tf.float32), tf.constant(False))\n",
-        "\n",
-        "import time\n",
-        "export_dir = \"./saved/\"+str(time.time())\n",
-        "\n",
-        "tf.saved_model.save(dropout, export_dir, signatures = cf)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "6pP_EgrY2Ces"
-      },
-      "source": [
-        "This `saved_model` only contains the one signature, and it can be recovered by name, from the `signatures` dictionary:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Pt6Z8ZUH2Dc8"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded = tf.saved_model.load(export_dir)\n",
-        "\n",
-        "print(reloaded.signatures)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ODMPWghD4IRC"
-      },
-      "source": [
-        "When using a \"exported signatures\" these concrete functions always return a dictionary of outputs:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "USuqYJY84F4F"
-      },
-      "outputs": [],
-      "source": [
-        "cf = reloaded.signatures[tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY]\n",
-        "result = cf(x=tf.random.normal([2,3]), training=tf.constant(True))\n",
-        "\n",
-        "print(result)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "tlnJz2AEyyHV"
-      },
-      "source": [
-        "In the example above, the output names auto-generated by the signature is fairly generic. You can check the output names using the `structured_outputs` method:\n",
-        "\n",
-        "You can check the expected output-tensor names using the `.structured_outputs` method: "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "bJKjvia-xS-_"
-      },
-      "outputs": [],
-      "source": [
-        "cf.structured_outputs"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "EAKTLuvUrg21"
-      },
-      "source": [
-        "Typically you wannt to set the output names yourself."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "g7y03GXg3pii"
-      },
-      "source": [
-        "#### Example: Setting the output names"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LEkFHgo81Iex"
-      },
-      "source": [
-        "To control the names of the outputs, modify your `tf.function` to return a dictionary that maps names to output tensors.:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "kz8QOgr21VZR"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.function\n",
-        "def named_result(x, training=True):\n",
-        "  return {'dropout': dropout(x, training)}\n",
-        "\n",
-        "dropout.named_result = named_result\n",
-        "\n",
-        "cf = dropout.named_result.get_concrete_function(tf.zeros((2,3), dtype=tf.float32),\n",
-        "                                                tf.constant(False))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Nhg_2t-Jr-Pn"
-      },
-      "source": [
-        "#### Example: Setting the signature names"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "szEQyfU83yQ0"
-      },
-      "source": [
-        "To set the name of the signature pass a dictionary of concrete functions."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "fUL6ntYu31TL"
-      },
-      "outputs": [],
-      "source": [
-        "export_dir = \"./saved/\"+str(time.time())\n",
-        "tf.saved_model.save(dropout, export_dir, signatures = {'simple':cf})"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "aFw4kDvU195u"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded = tf.saved_model.load(export_dir)\n",
-        "cf = reloaded.signatures['simple']\n",
-        "result = cf(x=tf.random.normal([2,3]), training=tf.constant(True))\n",
-        "\n",
-        "print({key:value.numpy() for key,value in result.items()})"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Ho7_3_G13jVF"
-      },
-      "source": [
-        "To specify multiple signatures pass a dictionary of `(name, concrete_function)` pairs to `saved_model.save`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Z0vgRTcE2pvi"
-      },
-      "outputs": [],
-      "source": [
-        "vector = dropout.__call__.get_concrete_function(tf.TensorSpec((2,3), dtype=tf.float32), tf.constant(False))\n",
-        "matrix = dropout.__call__.get_concrete_function(tf.TensorSpec((2,3), dtype=tf.float32), tf.constant(False))\n",
-        "cube = dropout.__call__.get_concrete_function(tf.TensorSpec((2,3), dtype=tf.float32), tf.constant(False))\n",
-        "\n",
-        "export_dir = \"./saved/\"+str(time.time())\n",
-        "\n",
-        "tf.saved_model.save(dropout, export_dir, \n",
-        "                    signatures = {\n",
-        "                        \"vector\": vector,\n",
-        "                        \"matrix\": matrix,\n",
-        "                        \"cube\": cube\n",
-        "                    })"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "L7jpWSND4_VY"
-      },
-      "source": [
-        "Now reload that model and inspect the signature listing:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "YTVaGx4W42EO"
-      },
-      "outputs": [],
-      "source": [
-        "reloaded = tf.saved_model.load(export_dir)\n",
-        "print('{}'.format(reloaded.signatures).replace(\"{\",\"{\\n    \").replace(\"\u003e, \", \"\u003e,\\n    \"))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "cbVqU7xOsLR3"
-      },
-      "outputs": [],
-      "source": [
-        ""
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "concrete_function.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/core/distribution.ipynb b/site/en/guide/core/distribution.ipynb
new file mode 100644
index 00000000000..c7f13b2f4db
--- /dev/null
+++ b/site/en/guide/core/distribution.ipynb
@@ -0,0 +1,700 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FhGuhbZ6M5tl"
+      },
+      "source": [
+        "##### Copyright 2022 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "AwOEIRJC6Une"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EIdT9iu_Z4Rb"
+      },
+      "source": [
+        "# Distributed training with Core APIs and DTensor"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bBIlTPscrIT9"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SjAxxRpBzVYg"
+      },
+      "source": [
+        "## Introduction\n",
+        "\n",
+        "This notebook uses the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) and [DTensor](https://www.tensorflow.org/guide/dtensor_overview) to demonstrate a data parallel distributed training example. Visit the [Core APIs overview](https://www.tensorflow.org/guide/core) to learn more about TensorFlow Core and its intended use cases. Refer to the [DTensor Overview](https://www.tensorflow.org/guide/dtensor_overview) guide and [Distributed Training with DTensors](https://www.tensorflow.org/tutorials/distribute/dtensor_ml_tutorial) tutorial to learn more about DTensor.\n",
+        "\n",
+        "This example uses the same model and optimizer shown in the [multilayer perceptrons](https://www.tensorflow.org/guide/core/mlp_core) tutorial. See this tutorial first to get comfortable with writing an end-to-end machine learning workflow with the Core APIs.\n",
+        "\n",
+        "Note: DTensor is still an experimental TensorFlow API which means that its features are available for testing, and it is intended for use in test environments only."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d_OFkG0dyWCp"
+      },
+      "source": [
+        "## Overview of data parallel training with DTensor\n",
+        "\n",
+        "Before building an MLP that supports distribution, take a moment to explore the fundamentals of DTensor for data parallel training.\n",
+        "\n",
+        "DTensor allows you to run distributed training across devices to improve efficiency, reliability and scalability. DTensor distributes the program and tensors according to the sharding directives through a procedure called Single program, multiple data (SPMD) expansion. A variable of a `DTensor` aware layer is created as `dtensor.DVariable`, and the constructors of `DTensor` aware layer objects take additional `Layout` inputs in addition to the usual layer parameters.\n",
+        "\n",
+        "The main ideas for data parallel training are as follows:\n",
+        " - Model variables are replicated on N devices each.\n",
+        " - A global batch is split into N per-replica batches.\n",
+        " - Each per-replica batch is trained on the replica device.\n",
+        " - The gradient is reduced before weight up data is collectively performed on all replicas.\n",
+        " - Data parallel training provides nearly linear speed with respect to the number of devices"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nchsZfwEVtVs"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "DTensor is part of TensorFlow 2.9.0 release."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "latuqlI_Yvoo"
+      },
+      "outputs": [],
+      "source": [
+        "#!pip install --quiet --upgrade --pre tensorflow"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1rRo8oNqZ-Rj"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib\n",
+        "from matplotlib import pyplot as plt\n",
+        "# Preset Matplotlib figure sizes.\n",
+        "matplotlib.rcParams['figure.figsize'] = [9, 6]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9xQKvCJ85kCQ"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_datasets as tfds\n",
+        "from tensorflow.experimental import dtensor\n",
+        "print(tf.__version__)\n",
+        "# Set random seed for reproducible results \n",
+        "tf.random.set_seed(22)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vDH9-sy4sfPf"
+      },
+      "source": [
+        "Configure 8 virtual CPUs for this experiment. DTensor can also be used with GPU or TPU devices. Given that this notebook uses virtual devices, the speedup gained from distributed training is not noticeable. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H2iM-6J4s2D6"
+      },
+      "outputs": [],
+      "source": [
+        "def configure_virtual_cpus(ncpu):\n",
+        "  phy_devices = tf.config.list_physical_devices('CPU')\n",
+        "  tf.config.set_logical_device_configuration(phy_devices[0], [\n",
+        "        tf.config.LogicalDeviceConfiguration(),\n",
+        "    ] * ncpu)\n",
+        "\n",
+        "configure_virtual_cpus(8)\n",
+        "\n",
+        "DEVICES = [f'CPU:{i}' for i in range(8)]\n",
+        "devices = tf.config.list_logical_devices('CPU')\n",
+        "device_names = [d.name for d in devices]\n",
+        "device_names"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "F_72b0LCNbjx"
+      },
+      "source": [
+        "## The MNIST Dataset\n",
+        "\n",
+        "The dataset is available from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/mnist). Split the data into training and testing sets. Only use 5000 examples for training and testing to save time."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8h4fV_JCfPIX"
+      },
+      "outputs": [],
+      "source": [
+        "train_data, test_data = tfds.load(\"mnist\", split=['train[:5000]', 'test[:5000]'], batch_size=128, as_supervised=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "twkJ35YB6tSi"
+      },
+      "source": [
+        "### Preprocessing the data\n",
+        "\n",
+        "Preprocess the data by reshaping it to be 2-dimensional and by rescaling it to fit into the unit interval, [0,1]."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6Cmjhg0xCqbz"
+      },
+      "outputs": [],
+      "source": [
+        "def preprocess(x, y):\n",
+        "  # Reshaping the data\n",
+        "  x = tf.reshape(x, shape=[-1, 784])\n",
+        "  # Rescaling the data\n",
+        "  x = x/255\n",
+        "  return x, y\n",
+        "\n",
+        "train_data, test_data = train_data.map(preprocess), test_data.map(preprocess)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6o3CrycBXA2s"
+      },
+      "source": [
+        "## Build the MLP \n",
+        "\n",
+        "Build an MLP model with DTensor aware layers."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OHW6Yvg2yS6H"
+      },
+      "source": [
+        "### The dense layer\n",
+        "\n",
+        "Start by creating a dense layer module that supports DTensor. The `dtensor.call_with_layout` function can be used to call a function that takes in a DTensor input and produces a DTensor output. This is useful for initializing a DTensor variable, `dtensor.DVariable`, with a TensorFlow supported function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IM0yJos25FG5"
+      },
+      "outputs": [],
+      "source": [
+        "class DenseLayer(tf.Module):\n",
+        "\n",
+        "  def __init__(self, in_dim, out_dim, weight_layout, activation=tf.identity):\n",
+        "    super().__init__()\n",
+        "    # Initialize dimensions and the activation function\n",
+        "    self.in_dim, self.out_dim = in_dim, out_dim\n",
+        "    self.activation = activation\n",
+        "\n",
+        "    # Initialize the DTensor weights using the Xavier scheme\n",
+        "    uniform_initializer = tf.function(tf.random.stateless_uniform)\n",
+        "    xavier_lim = tf.sqrt(6.)/tf.sqrt(tf.cast(self.in_dim + self.out_dim, tf.float32))\n",
+        "    self.w = dtensor.DVariable(\n",
+        "      dtensor.call_with_layout(\n",
+        "          uniform_initializer, weight_layout,\n",
+        "          shape=(self.in_dim, self.out_dim), seed=(22, 23),\n",
+        "          minval=-xavier_lim, maxval=xavier_lim))\n",
+        "        \n",
+        "    # Initialize the bias with the zeros\n",
+        "    bias_layout = weight_layout.delete([0])\n",
+        "    self.b = dtensor.DVariable(\n",
+        "      dtensor.call_with_layout(tf.zeros, bias_layout, shape=[out_dim]))\n",
+        "\n",
+        "  def __call__(self, x):\n",
+        "    # Compute the forward pass\n",
+        "    z = tf.add(tf.matmul(x, self.w), self.b)\n",
+        "    return self.activation(z)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "X-7MzpjgyHg6"
+      },
+      "source": [
+        "### The MLP sequential model\n",
+        "\n",
+        "Now create an MLP module that executes the dense layers sequentially."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6XisRWiCyHAb"
+      },
+      "outputs": [],
+      "source": [
+        "class MLP(tf.Module):\n",
+        "\n",
+        "  def __init__(self, layers):\n",
+        "    self.layers = layers\n",
+        "   \n",
+        "  def __call__(self, x, preds=False): \n",
+        "    # Execute the model's layers sequentially\n",
+        "    for layer in self.layers:\n",
+        "      x = layer(x)\n",
+        "    return x"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r5HZJ0kv-V3v"
+      },
+      "source": [
+        "Performing \"data-parallel\" training with DTensor is equivalent to `tf.distribute.MirroredStrategy`. To do this each device will run the same model on a shard of the data batch. So you'll need the following:\n",
+        "\n",
+        "* A `dtensor.Mesh` with a single `\"batch\"` dimension\n",
+        "* A `dtensor.Layout` for all the weights that replicates them across the mesh (using `dtensor.UNSHARDED` for each axis)\n",
+        "* A `dtensor.Layout` for the data that splits the batch dimension across the mesh\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "Create a DTensor mesh that consists of a single batch dimension, where each device becomes a replica that receives a shard from the global batch. Use this mesh to instantiate an MLP mode with the following architecture:\n",
+        "\n",
+        "Forward Pass: ReLU(784 x 700) x ReLU(700 x 500) x Softmax(500 x 10)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VmlACuki3oPi"
+      },
+      "outputs": [],
+      "source": [
+        "mesh = dtensor.create_mesh([(\"batch\", 8)], devices=DEVICES)\n",
+        "weight_layout = dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh)\n",
+        "\n",
+        "input_size = 784\n",
+        "hidden_layer_1_size = 700\n",
+        "hidden_layer_2_size = 500\n",
+        "hidden_layer_2_size = 10\n",
+        "\n",
+        "mlp_model = MLP([\n",
+        "    DenseLayer(in_dim=input_size, out_dim=hidden_layer_1_size, \n",
+        "               weight_layout=weight_layout,\n",
+        "               activation=tf.nn.relu),\n",
+        "    DenseLayer(in_dim=hidden_layer_1_size , out_dim=hidden_layer_2_size,\n",
+        "               weight_layout=weight_layout,\n",
+        "               activation=tf.nn.relu),\n",
+        "    DenseLayer(in_dim=hidden_layer_2_size, out_dim=hidden_layer_2_size, \n",
+        "               weight_layout=weight_layout)])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tyBATDoRmDkg"
+      },
+      "source": [
+        "### Training metrics\n",
+        "\n",
+        "Use the cross-entropy loss function and accuracy metric for training."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rskOYA7FVCwg"
+      },
+      "outputs": [],
+      "source": [
+        "def cross_entropy_loss(y_pred, y):\n",
+        "  # Compute cross entropy loss with a sparse operation\n",
+        "  sparse_ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=y_pred)\n",
+        "  return tf.reduce_mean(sparse_ce)\n",
+        "\n",
+        "def accuracy(y_pred, y):\n",
+        "  # Compute accuracy after extracting class predictions\n",
+        "  class_preds = tf.argmax(y_pred, axis=1)\n",
+        "  is_equal = tf.equal(y, class_preds)\n",
+        "  return tf.reduce_mean(tf.cast(is_equal, tf.float32))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JSiNRhTOnKZr"
+      },
+      "source": [
+        "### Optimizer\n",
+        "\n",
+        "Using an optimizer can result in significantly faster convergence compared to standard gradient descent. The Adam optimizer is implemented below and has been configured to be compatible with DTensor. In order to use Keras optimizers with DTensor, refer to the experimental`tf.keras.dtensor.experimental.optimizers` module."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-9kIAI_lfXDS"
+      },
+      "outputs": [],
+      "source": [
+        "class Adam(tf.Module):\n",
+        "\n",
+        "    def __init__(self, model_vars, learning_rate=1e-3, beta_1=0.9, beta_2=0.999, ep=1e-7):\n",
+        "      # Initialize optimizer parameters and variable slots\n",
+        "      self.model_vars = model_vars\n",
+        "      self.beta_1 = beta_1\n",
+        "      self.beta_2 = beta_2\n",
+        "      self.learning_rate = learning_rate\n",
+        "      self.ep = ep\n",
+        "      self.t = 1.\n",
+        "      self.v_dvar, self.s_dvar = [], []\n",
+        "      # Initialize optimizer variable slots\n",
+        "      for var in model_vars:\n",
+        "        v = dtensor.DVariable(dtensor.call_with_layout(tf.zeros, var.layout, shape=var.shape))\n",
+        "        s = dtensor.DVariable(dtensor.call_with_layout(tf.zeros, var.layout, shape=var.shape))\n",
+        "        self.v_dvar.append(v)\n",
+        "        self.s_dvar.append(s)\n",
+        "\n",
+        "    def apply_gradients(self, grads):\n",
+        "      # Update the model variables given their gradients\n",
+        "      for i, (d_var, var) in enumerate(zip(grads, self.model_vars)):\n",
+        "        self.v_dvar[i].assign(self.beta_1*self.v_dvar[i] + (1-self.beta_1)*d_var)\n",
+        "        self.s_dvar[i].assign(self.beta_2*self.s_dvar[i] + (1-self.beta_2)*tf.square(d_var))\n",
+        "        v_dvar_bc = self.v_dvar[i]/(1-(self.beta_1**self.t))\n",
+        "        s_dvar_bc = self.s_dvar[i]/(1-(self.beta_2**self.t))\n",
+        "        var.assign_sub(self.learning_rate*(v_dvar_bc/(tf.sqrt(s_dvar_bc) + self.ep)))\n",
+        "      self.t += 1.\n",
+        "      return "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "w54b7GtLfn1j"
+      },
+      "source": [
+        "### Data packing\n",
+        "\n",
+        "Start by writing a helper function for transferring data to the device. This function should use `dtensor.pack` to send (and only send) the shard of the global batch that is intended for a replica to the device backing the replica. For simplicity, assume a single-client application.\n",
+        "\n",
+        "Next, write a function that uses this helper function to pack the training data batches into DTensors sharded along the batch (first) axis. This ensures that DTensor evenly distributes the training data to the 'batch' mesh dimension. Note that in DTensor, the batch size always refers to the global batch size; therefore, the batch size should be chosen such that it can be divided evenly by the size of the batch mesh dimension. Additional DTensor APIs to simplify `tf.data` integration are planned, so please stay tuned."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3Rx82djZ6ITm"
+      },
+      "outputs": [],
+      "source": [
+        "def repack_local_tensor(x, layout):\n",
+        "  # Repacks a local Tensor-like to a DTensor with layout\n",
+        "  # This function assumes a single-client application\n",
+        "  x = tf.convert_to_tensor(x)\n",
+        "  sharded_dims = []\n",
+        "\n",
+        "  # For every sharded dimension, use tf.split to split the along the dimension.\n",
+        "  # The result is a nested list of split-tensors in queue[0].\n",
+        "  queue = [x]\n",
+        "  for axis, dim in enumerate(layout.sharding_specs):\n",
+        "    if dim == dtensor.UNSHARDED:\n",
+        "      continue\n",
+        "    num_splits = layout.shape[axis]\n",
+        "    queue = tf.nest.map_structure(lambda x: tf.split(x, num_splits, axis=axis), queue)\n",
+        "    sharded_dims.append(dim)\n",
+        "\n",
+        "  # Now you can build the list of component tensors by looking up the location in\n",
+        "  # the nested list of split-tensors created in queue[0].\n",
+        "  components = []\n",
+        "  for locations in layout.mesh.local_device_locations():\n",
+        "    t = queue[0]\n",
+        "    for dim in sharded_dims:\n",
+        "      split_index = locations[dim]  # Only valid on single-client mesh.\n",
+        "      t = t[split_index]\n",
+        "    components.append(t)\n",
+        "\n",
+        "  return dtensor.pack(components, layout)\n",
+        "\n",
+        "def repack_batch(x, y, mesh):\n",
+        "  # Pack training data batches into DTensors along the batch axis\n",
+        "  x = repack_local_tensor(x, layout=dtensor.Layout(['batch', dtensor.UNSHARDED], mesh))\n",
+        "  y = repack_local_tensor(y, layout=dtensor.Layout(['batch'], mesh))\n",
+        "  return x, y"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "osEK3rqpYfKd"
+      },
+      "source": [
+        "### Training\n",
+        "\n",
+        "Write a traceable function that executes a single training step given a batch of data. This function does not require any special DTensor annotations. Also write a function that executes a test step and returns the appropriate performance metrics."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZICEsDGuSbDD"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def train_step(model, x_batch, y_batch, loss, metric, optimizer):\n",
+        "  # Execute a single training step\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    y_pred = model(x_batch)\n",
+        "    batch_loss = loss(y_pred, y_batch)\n",
+        "  # Compute gradients and update the model's parameters\n",
+        "  grads = tape.gradient(batch_loss, model.trainable_variables)\n",
+        "  optimizer.apply_gradients(grads)\n",
+        "  # Return batch loss and accuracy\n",
+        "  batch_acc = metric(y_pred, y_batch)\n",
+        "  return batch_loss, batch_acc\n",
+        "\n",
+        "@tf.function\n",
+        "def test_step(model, x_batch, y_batch, loss, metric):\n",
+        "  # Execute a single testing step\n",
+        "  y_pred = model(x_batch)\n",
+        "  batch_loss = loss(y_pred, y_batch)\n",
+        "  batch_acc = metric(y_pred, y_batch)\n",
+        "  return batch_loss, batch_acc"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RjIDVTwwX-Mr"
+      },
+      "source": [
+        "Now, train the MLP model for 3 epochs with a batch size of 128."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oC85kuZgmh3q"
+      },
+      "outputs": [],
+      "source": [
+        "# Initialize the training loop parameters and structures\n",
+        "epochs = 3\n",
+        "batch_size = 128\n",
+        "train_losses, test_losses = [], []\n",
+        "train_accs, test_accs = [], []\n",
+        "optimizer = Adam(mlp_model.trainable_variables)\n",
+        "\n",
+        "# Format training loop\n",
+        "for epoch in range(epochs):\n",
+        "  batch_losses_train, batch_accs_train = [], []\n",
+        "  batch_losses_test, batch_accs_test = [], []\n",
+        "\n",
+        "  # Iterate through training data\n",
+        "  for x_batch, y_batch in train_data:\n",
+        "    x_batch, y_batch = repack_batch(x_batch, y_batch, mesh)\n",
+        "    batch_loss, batch_acc = train_step(mlp_model, x_batch, y_batch, cross_entropy_loss, accuracy, optimizer)\n",
+        "   # Keep track of batch-level training performance\n",
+        "    batch_losses_train.append(batch_loss)\n",
+        "    batch_accs_train.append(batch_acc)\n",
+        "\n",
+        "  # Iterate through testing data\n",
+        "  for x_batch, y_batch in test_data:\n",
+        "    x_batch, y_batch = repack_batch(x_batch, y_batch, mesh)\n",
+        "    batch_loss, batch_acc = test_step(mlp_model, x_batch, y_batch, cross_entropy_loss, accuracy)\n",
+        "    # Keep track of batch-level testing\n",
+        "    batch_losses_test.append(batch_loss)\n",
+        "    batch_accs_test.append(batch_acc)\n",
+        "\n",
+        "# Keep track of epoch-level model performance\n",
+        "  train_loss, train_acc = tf.reduce_mean(batch_losses_train), tf.reduce_mean(batch_accs_train)\n",
+        "  test_loss, test_acc = tf.reduce_mean(batch_losses_test), tf.reduce_mean(batch_accs_test)\n",
+        "  train_losses.append(train_loss)\n",
+        "  train_accs.append(train_acc)\n",
+        "  test_losses.append(test_loss)\n",
+        "  test_accs.append(test_acc)\n",
+        "  print(f\"Epoch: {epoch}\")\n",
+        "  print(f\"Training loss: {train_loss.numpy():.3f}, Training accuracy: {train_acc.numpy():.3f}\")\n",
+        "  print(f\"Testing loss: {test_loss.numpy():.3f}, Testing accuracy: {test_acc.numpy():.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "j_RVmt43G12R"
+      },
+      "source": [
+        "### Performance evaluation\n",
+        "\n",
+        "Start by writing a plotting function to visualize the model's loss and accuracy during training. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VXTCYVtNDjAM"
+      },
+      "outputs": [],
+      "source": [
+        "def plot_metrics(train_metric, test_metric, metric_type):\n",
+        "  # Visualize metrics vs training Epochs\n",
+        "  plt.figure()\n",
+        "  plt.plot(range(len(train_metric)), train_metric, label = f\"Training {metric_type}\")\n",
+        "  plt.plot(range(len(test_metric)), test_metric, label = f\"Testing {metric_type}\")\n",
+        "  plt.xlabel(\"Epochs\")\n",
+        "  plt.ylabel(metric_type)\n",
+        "  plt.legend()\n",
+        "  plt.title(f\"{metric_type} vs Training Epochs\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "407qok7q2JIO"
+      },
+      "outputs": [],
+      "source": [
+        "plot_metrics(train_losses, test_losses, \"Cross entropy loss\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8H_TgxV92NfX"
+      },
+      "outputs": [],
+      "source": [
+        "plot_metrics(train_accs, test_accs, \"Accuracy\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DHO_u-3w4YRF"
+      },
+      "source": [
+        "## Saving your model\n",
+        "\n",
+        "The integration of `tf.saved_model` and DTensor is still under development. As of TensorFlow 2.9.0, tf.saved_model only accepts DTensor models with fully replicated variables. As a workaround, you can convert a DTensor model to a fully replicated one by reloading a checkpoint. However, after a model is saved, all DTensor annotations are lost and the saved signatures can only be used with regular Tensors. This tutorial will be updated to showcase the integration once it is solidified.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VFLfEH4ManbW"
+      },
+      "source": [
+        "## Conclusion\n",
+        "\n",
+        "This notebook provided an overview of distributed training with DTensor and the TensorFlow Core APIs. Here are a few more tips that may help:\n",
+        "\n",
+        "- The [TensorFlow Core APIs](https://www.tensorflow.org/guide/core) can be used to build highly-configurable machine learning workflows with support for distributed training.\n",
+        "- The [DTensor concepts](https://www.tensorflow.org/guide/dtensor_overview) guide and [Distributed training with DTensors](https://www.tensorflow.org/tutorials/distribute/dtensor_ml_tutorial) tutorial contain the most up-to-date information about DTensor and its integrations.\n",
+        "\n",
+        "For more examples of using the TensorFlow Core APIs, check out the [guide](https://www.tensorflow.org/guide/core). If you want to learn more about loading and preparing data, see the tutorials on [image data loading](https://www.tensorflow.org/tutorials/load_data/images) or [CSV data loading](https://www.tensorflow.org/tutorials/load_data/csv)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "FhGuhbZ6M5tl"
+      ],
+      "name": "distribution.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/index.md b/site/en/guide/core/index.md
new file mode 100644
index 00000000000..7f44e11b018
--- /dev/null
+++ b/site/en/guide/core/index.md
@@ -0,0 +1,112 @@
+# TensorFlow Core APIs overview
+
+The TensorFlow Core APIs provide a set of comprehensive, composable, and
+extensible low-level APIs for high-performance (distributed and accelerated)
+computation, primarily aimed at building machine learning (ML) models as well as
+authoring ML workflow tools and frameworks within the TensorFlow platform. These
+APIs provide a foundation for creating highly configurable models with
+fine-grained control and new frameworks from the ground up.
+
+The Core APIs can be used as an alternative to high-level machine learning APIs
+like Keras. These high-level APIs are best suited for general machine learning
+needs. They offer a variety of modules that abstract away the complexities of ML
+while also offering functionalities for customization through subclassing. If
+you are looking for an overview of TensorFlow using Keras, see the Quickstarts
+and Keras sections in the [tutorials](https://www.tensorflow.org/tutorials).
+
+## Who should use the Core APIs
+
+The TensorFlow Core low-level APIs are designed with the following ML developers
+in mind:
+
+*   Researchers building complex models with high levels of configurability
+*   Developers interested in using TensorFlow as a high-performance scientific
+    computing platform
+*   Framework authors building tools on top of the TensorFlow platform
+*   High-level API users interested in:
+    *   Adding additional functionalities to their machine learning workflows
+        such as custom layers, losses, models, and optimizers
+    *   Learning more about the inner workings of their models
+
+## Core API applications
+
+The TensorFlow Core APIs provide access to low level functionality within the
+TensorFlow ecosystem. This API provides more flexibility and control for
+building ML models, applications, and tools, compared to high-level APIs, such
+as Keras.
+
+### Build models and workflows
+
+The Core APIs are most commonly used to build highly customizable and optimized
+machine learning models and workflows. Here are some of the ways that the
+TensorFlow Core APIs can improve your machine learning models and workflow
+development:
+
+TensorFlow
+
+*   Building non-traditional models or layers that do not fully fit the
+    structures supported by high-level APIs
+*   Building custom layers, losses, models, and optimizers within Keras
+*   Implementing new optimization techniques to expedite convergence during
+    training
+*   Creating custom metrics for performance evaluation
+*   Designing highly-configurable training loops with support for features like
+    batching, cross-validation, and distribution strategies
+
+### Build frameworks and tools
+
+The TensorFlow Core APIs can also serve as the building blocks for new
+high-level frameworks. Here are some examples of tools and frameworks that are
+created with the low-level APIs:
+TensorFlow
+
+*   [Keras](https://keras.io): deep learning for humans
+*   [TensorFlow Model Optimization Toolkit](https://www.tensorflow.org/model_optimization):
+    a suite of tools to optimize ML models for deployment and execution
+*   [TensorFlow Graphics](https://www.tensorflow.org/graphics): a library for
+    making useful graphics functions widely accessible
+
+### Build for scientific computing
+
+The TensorFlow Core APIs can also be applied outside the realm of machine
+learning. Here are a few general-purpose use cases of TensorFlow for scientific
+computing:
+TensorFlow
+
+*   Physics simulations for solid mechanics and
+    [fluid dynamics](https://arxiv.org/abs/2108.11076) problems
+*   Graphics rendering applications like
+    [ray tracing](https://github.com/BachiLi/redner)
+*   Solving
+    [constrained optimization problems](https://github.com/google-research/tensorflow_constrained_optimization/blob/master/README.md)
+
+## Core API components
+
+Here are some of the fundamental components that comprise TensorFlow Core’s low-
+level APIs. Note that this is not an all-encompassing list:
+
+TensorFlow
+
+*   Data structures : `tf.Tensor`, `tf.Variable`, `tf.TensorArray`
+*   Primitive APIs: `tf.shape`,
+    [slicing](https://www.tensorflow.org/guide/tensor_slicing), `tf.concat`,
+    `tf.bitwise`
+*   Numerical: `tf.math`, `tf.linalg`, `tf.random`
+*   Functional components: `tf.function`, `tf.GradientTape`
+*   Distribution: [DTensor](https://www.tensorflow.org/guide/dtensor_overview)
+*   Export: `tf.saved_model`
+
+## Next steps
+
+The *Build with Core* documentation provides tutorials of basic machine learning
+concepts from scratch. The tutorials in this section help you get comfortable
+with writing low-level code with Core APIs that you can then apply to more
+complex use cases of your own.
+
+Note: You should not use the Core APIs to simply re-implement high-level APIs,
+and it is possible to use high-level APIs, such as Keras, with the Core APIs.
+
+To get started using and learning more about the Core APIs, check out the
+[Quickstart for TensorFlow Core](https://www.tensorflow.org/guide/core/quickstart_core).
diff --git a/site/en/guide/core/logistic_regression_core.ipynb b/site/en/guide/core/logistic_regression_core.ipynb
new file mode 100644
index 00000000000..5a9af324ad5
--- /dev/null
+++ b/site/en/guide/core/logistic_regression_core.ipynb
@@ -0,0 +1,935 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FhGuhbZ6M5tl"
+      },
+      "source": [
+        "##### Copyright 2022 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "AwOEIRJC6Une"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EIdT9iu_Z4Rb"
+      },
+      "source": [
+        "# Logistic regression for binary classification with Core APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bBIlTPscrIT9"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DauaqJ7WhIhO"
+      },
+      "source": [
+        "This guide demonstrates how to use the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) to perform [binary classification](https://developers.google.com/machine-learning/glossary#binary_classification) with [logistic regression](https://developers.google.com/machine-learning/crash-course/logistic-regression/). It uses the [Wisconsin Breast Cancer Dataset](https://archive.ics.uci.edu/ml/datasets/breast+cancer+wisconsin+(original)) for tumor classification.\n",
+        "\n",
+        "[Logistic regression](https://developers.google.com/machine-learning/crash-course/logistic-regression/) is one of the most popular algorithms for binary classification. Given a set of examples with features, the goal of logistic regression is to output values between 0 and 1, which can be interpreted as the probabilities of each example belonging to a particular  class. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nchsZfwEVtVs"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "This tutorial uses [pandas](https://pandas.pydata.org) for reading a CSV file into a [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html), [seaborn](https://seaborn.pydata.org) for plotting a pairwise relationship in a dataset, [Scikit-learn](https://scikit-learn.org/) for computing a confusion matrix, and [matplotlib](https://matplotlib.org/) for creating visualizations."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5lZoUK6AVTos"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q seaborn"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1rRo8oNqZ-Rj"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import pandas as pd\n",
+        "import matplotlib\n",
+        "from matplotlib import pyplot as plt\n",
+        "import seaborn as sns\n",
+        "import sklearn.metrics as sk_metrics\n",
+        "import tempfile\n",
+        "import os\n",
+        "\n",
+        "# Preset matplotlib figure sizes.\n",
+        "matplotlib.rcParams['figure.figsize'] = [9, 6]\n",
+        "\n",
+        "print(tf.__version__)\n",
+        "# To make the results reproducible, set the random seed value.\n",
+        "tf.random.set_seed(22)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gFh9ne3FZ-On"
+      },
+      "source": [
+        "## Load the data\n",
+        "\n",
+        "Next, load the [Wisconsin Breast Cancer Dataset](https://archive.ics.uci.edu/ml/datasets/breast+cancer+wisconsin+(original)) from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/). This dataset contains various features such as a tumor's radius, texture, and concavity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CiX2FI4gZtTt"
+      },
+      "outputs": [],
+      "source": [
+        "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/wdbc.data'\n",
+        "\n",
+        "features = ['radius', 'texture', 'perimeter', 'area', 'smoothness', 'compactness',\n",
+        "            'concavity', 'concave_poinits', 'symmetry', 'fractal_dimension']\n",
+        "column_names = ['id', 'diagnosis']\n",
+        "\n",
+        "for attr in ['mean', 'ste', 'largest']:\n",
+        "  for feature in features:\n",
+        "    column_names.append(feature + \"_\" + attr)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A3VR1aTP92nV"
+      },
+      "source": [
+        "Read the dataset into a pandas [DataFrame]() using [`pandas.read_csv`](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uvR2Bzb691lJ"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = pd.read_csv(url, names=column_names)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YB9eq6Zq-IZ4"
+      },
+      "outputs": [],
+      "source": [
+        "dataset.info()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0_Z1V6Dg-La_"
+      },
+      "source": [
+        "Display the first five rows:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hWxktwbv-KPp"
+      },
+      "outputs": [],
+      "source": [
+        "dataset.head()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s4-Wn2jzVC1W"
+      },
+      "source": [
+        "Split the dataset into training and test sets using [`pandas.DataFrame.sample`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.sample.html), [`pandas.DataFrame.drop`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.drop.html) and [`pandas.DataFrame.iloc`](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.iloc.html). Make sure to split the features from the target labels. The test set is used to evaluate your model's generalizability to unseen data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m2O60B-IVG9Q"
+      },
+      "outputs": [],
+      "source": [
+        "train_dataset = dataset.sample(frac=0.75, random_state=1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "i06vHFv_QB24"
+      },
+      "outputs": [],
+      "source": [
+        "len(train_dataset)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "19JaochhaQ3m"
+      },
+      "outputs": [],
+      "source": [
+        "test_dataset = dataset.drop(train_dataset.index)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LmHRcbAfaSag"
+      },
+      "outputs": [],
+      "source": [
+        "len(test_dataset)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "w6JxBhBc_wwO"
+      },
+      "outputs": [],
+      "source": [
+        "# The `id` column can be dropped since each row is unique\n",
+        "x_train, y_train = train_dataset.iloc[:, 2:], train_dataset.iloc[:, 1]\n",
+        "x_test, y_test = test_dataset.iloc[:, 2:], test_dataset.iloc[:, 1]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3MWuJTKEDM-f"
+      },
+      "source": [
+        "## Preprocess the data\n",
+        "\n",
+        "This dataset contains the mean, standard error, and largest values for each of the 10 tumor measurements collected per example. The `\"diagnosis\"` target column is a categorical variable with `'M'` indicating a malignant tumor and `'B'` indicating a benign tumor diagnosis. This column needs to be converted into a numerical binary format for model training.\n",
+        "\n",
+        "The [`pandas.Series.map`](https://pandas.pydata.org/docs/reference/api/pandas.Series.map.html) function is useful for mapping binary values to the categories.\n",
+        "\n",
+        "The dataset should also be converted to a tensor with the `tf.convert_to_tensor` function after the preprocessing is complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JEJHhN65a2VV"
+      },
+      "outputs": [],
+      "source": [
+        "y_train, y_test = y_train.map({'B': 0, 'M': 1}), y_test.map({'B': 0, 'M': 1})\n",
+        "x_train, y_train = tf.convert_to_tensor(x_train, dtype=tf.float32), tf.convert_to_tensor(y_train, dtype=tf.float32)\n",
+        "x_test, y_test = tf.convert_to_tensor(x_test, dtype=tf.float32), tf.convert_to_tensor(y_test, dtype=tf.float32)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J4ubs136WLNp"
+      },
+      "source": [
+        "Use [`seaborn.pairplot`](https://seaborn.pydata.org/generated/seaborn.pairplot.html) to review the joint distribution of a few pairs of mean-based features from the training set and observe how they relate to the target:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oRKO_x8gWKv-"
+      },
+      "outputs": [],
+      "source": [
+        "sns.pairplot(train_dataset.iloc[:, 1:6], hue = 'diagnosis', diag_kind='kde');"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5YOG5iKYKW_3"
+      },
+      "source": [
+        "This pairplot demonstrates that certain features such as radius, perimeter and area are highly correlated. This is expected since the tumor radius is directly involved in the computation of both perimeter and area. Additionally, note that malignant diagnoses seem to be more right-skewed for many of the features.\n",
+        "\n",
+        "Make sure to also check the overall statistics. Note how each feature covers a vastly different range of values."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yi2FzC3T21jR"
+      },
+      "outputs": [],
+      "source": [
+        "train_dataset.describe().transpose()[:10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_8pDCIFjMla8"
+      },
+      "source": [
+        "Given the inconsistent ranges, it is beneficial to standardize the data such that each feature has a zero mean and unit variance. This process is called [normalization](https://developers.google.com/machine-learning/glossary#normalization)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FrzKNFNjLQDl"
+      },
+      "outputs": [],
+      "source": [
+        "class Normalize(tf.Module):\n",
+        "  def __init__(self, x):\n",
+        "    # Initialize the mean and standard deviation for normalization\n",
+        "    self.mean = tf.Variable(tf.math.reduce_mean(x, axis=0))\n",
+        "    self.std = tf.Variable(tf.math.reduce_std(x, axis=0))\n",
+        "\n",
+        "  def norm(self, x):\n",
+        "    # Normalize the input\n",
+        "    return (x - self.mean)/self.std\n",
+        "\n",
+        "  def unnorm(self, x):\n",
+        "    # Unnormalize the input\n",
+        "    return (x * self.std) + self.mean\n",
+        "\n",
+        "norm_x = Normalize(x_train)\n",
+        "x_train_norm, x_test_norm = norm_x.norm(x_train), norm_x.norm(x_test)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6o3CrycBXA2s"
+      },
+      "source": [
+        "## Logistic regression\n",
+        "\n",
+        "Before building a logistic regression model, it is crucial to understand the method's differences compared to traditional linear regression.\n",
+        "\n",
+        "### Logistic regression fundamentals\n",
+        "\n",
+        "Linear regression returns a linear combination of its inputs; this output is unbounded. The output of a [logistic regression](https://developers.google.com/machine-learning/glossary#logistic_regression) is in the `(0, 1)` range. For each example, it represents the probability that the example belongs to the _positive_ class.\n",
+        "\n",
+        "Logistic regression maps the continuous outputs of traditional linear regression, `(-∞, ∞)`, to probabilities, `(0, 1)`. This transformation is also symmetric so that flipping the sign of the linear output results in the inverse of the original probability.\n",
+        "\n",
+        "Let $Y$ denote the probability of being in class `1` (the tumor is malignant). The desired mapping can be achieved by interpreting the linear regression output as the [log odds](https://developers.google.com/machine-learning/glossary#log-odds) ratio of being in class `1` as opposed to class `0`:\n",
+        "\n",
+        "$$\\ln(\\frac{Y}{1-Y}) = wX + b$$\n",
+        "\n",
+        "By setting $wX + b = z$, this equation can then be solved for $Y$:\n",
+        "\n",
+        "$$Y = \\frac{e^{z}}{1 + e^{z}} = \\frac{1}{1 + e^{-z}}$$\n",
+        "\n",
+        "The expression $\\frac{1}{1 + e^{-z}}$ is known as the [sigmoid function](https://developers.google.com/machine-learning/glossary#sigmoid_function) $\\sigma(z)$. Hence, the equation for logistic regression can be written as $Y = \\sigma(wX + b)$.\n",
+        "\n",
+        "The dataset in this tutorial deals with a high-dimensional feature matrix. Therefore, the above equation must be rewritten in a matrix vector form as follows:\n",
+        "\n",
+        "$${\\mathrm{Y}} = \\sigma({\\mathrm{X}}w + b)$$\n",
+        "\n",
+        "where:\n",
+        "\n",
+        "* $\\underset{m\\times 1}{\\mathrm{Y}}$: a target vector\n",
+        "* $\\underset{m\\times n}{\\mathrm{X}}$: a feature matrix\n",
+        "* $\\underset{n\\times 1}w$: a weight vector\n",
+        "* $b$: a bias\n",
+        "* $\\sigma$: a sigmoid function applied to each element of the output vector\n",
+        "\n",
+        "Start by visualizing the sigmoid function, which transforms the linear output, `(-∞, ∞)`, to fall between `0` and `1`. The sigmoid function is available in `tf.math.sigmoid`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ThHaV_RmucZl"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.linspace(-10, 10, 500)\n",
+        "x = tf.cast(x, tf.float32)\n",
+        "f = lambda x : (1/20)*x + 0.6\n",
+        "plt.plot(x, tf.math.sigmoid(x))\n",
+        "plt.ylim((-0.1,1.1))\n",
+        "plt.title(\"Sigmoid function\");"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VMXEhrZuKECV"
+      },
+      "source": [
+        "### The log loss function\n",
+        "\n",
+        "The  [log loss](https://developers.google.com/machine-learning/glossary#Log_Loss), or binary cross-entropy loss, is the ideal loss function for a binary classification problem with logistic regression. For each example, the log loss quantifies the similarity between a predicted probability and the example's true value. It is determined by the following equation:\n",
+        "\n",
+        "$$L = -\\frac{1}{m}\\sum_{i=1}^{m}y_i\\cdot\\log(\\hat{y}_i) + (1- y_i)\\cdot\\log(1 - \\hat{y}_i)$$\n",
+        "\n",
+        "where:\n",
+        "\n",
+        "* $\\hat{y}$: a vector of predicted probabilities\n",
+        "* $y$: a vector of true targets\n",
+        "\n",
+        "You can use the `tf.nn.sigmoid_cross_entropy_with_logits` function to compute the log loss. This function automatically applies the sigmoid activation to the regression output:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JVBInnSqS36W"
+      },
+      "outputs": [],
+      "source": [
+        "def log_loss(y_pred, y):\n",
+        "  # Compute the log loss function\n",
+        "  ce = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=y_pred)\n",
+        "  return tf.reduce_mean(ce)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q_mutLj0KNUb"
+      },
+      "source": [
+        "### The gradient descent update rule\n",
+        "\n",
+        "The TensorFlow Core APIs support automatic differentiation with `tf.GradientTape`. If you are curious about the mathematics behind the logistic regression [gradient updates](https://developers.google.com/machine-learning/glossary#gradient_descent), here is a short explanation:\n",
+        "\n",
+        "In the above equation for the log loss, recall that each $\\hat{y}_i$ can be rewritten in terms of the inputs as $\\sigma({\\mathrm{X_i}}w + b)$.\n",
+        "\n",
+        "The goal is to find a $w^*$ and $b^*$ that minimize the log loss:\n",
+        "\n",
+        "$$L = -\\frac{1}{m}\\sum_{i=1}^{m}y_i\\cdot\\log(\\sigma({\\mathrm{X_i}}w + b)) + (1- y_i)\\cdot\\log(1 - \\sigma({\\mathrm{X_i}}w + b))$$\n",
+        "\n",
+        "By taking the gradient $L$ with respect to $w$, you get the following:\n",
+        "\n",
+        "$$\\frac{\\partial L}{\\partial w} = \\frac{1}{m}(\\sigma({\\mathrm{X}}w + b) - y)X$$\n",
+        "\n",
+        "By taking the gradient $L$ with respect to $b$, you get the following:\n",
+        "\n",
+        "$$\\frac{\\partial L}{\\partial b} = \\frac{1}{m}\\sum_{i=1}^{m}\\sigma({\\mathrm{X_i}}w + b) - y_i$$"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uTCndUecKZho"
+      },
+      "source": [
+        "Now, build the logistic regression model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c0sXM7qLlKfZ"
+      },
+      "outputs": [],
+      "source": [
+        "class LogisticRegression(tf.Module):\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    self.built = False\n",
+        "    \n",
+        "  def __call__(self, x, train=True):\n",
+        "    # Initialize the model parameters on the first call\n",
+        "    if not self.built:\n",
+        "      # Randomly generate the weights and the bias term\n",
+        "      rand_w = tf.random.uniform(shape=[x.shape[-1], 1], seed=22)\n",
+        "      rand_b = tf.random.uniform(shape=[], seed=22)\n",
+        "      self.w = tf.Variable(rand_w)\n",
+        "      self.b = tf.Variable(rand_b)\n",
+        "      self.built = True\n",
+        "    # Compute the model output\n",
+        "    z = tf.add(tf.matmul(x, self.w), self.b)\n",
+        "    z = tf.squeeze(z, axis=1)\n",
+        "    if train:\n",
+        "      return z\n",
+        "    return tf.sigmoid(z)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eObQu9fDnXGL"
+      },
+      "source": [
+        "To validate, make sure the untrained model outputs values in the range of `(0, 1)` for a small subset of the training data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5bIovC0Z4QHJ"
+      },
+      "outputs": [],
+      "source": [
+        "log_reg = LogisticRegression()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QJ2ievISyf0p"
+      },
+      "outputs": [],
+      "source": [
+        "y_pred = log_reg(x_train_norm[:5], train=False)\n",
+        "y_pred.numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PribnwDHUksC"
+      },
+      "source": [
+        "Next, write an accuracy function to calculate the proportion of correct classifications during training. In order to retrieve the classifications from the predicted probabilities, set a threshold for which all probabilities higher than the threshold belong to class `1`. This is a configurable hyperparameter that can be set to `0.5` as a default."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ssnVcKg7oMe6"
+      },
+      "outputs": [],
+      "source": [
+        "def predict_class(y_pred, thresh=0.5):\n",
+        "  # Return a tensor with  `1` if `y_pred` > `0.5`, and `0` otherwise\n",
+        "  return tf.cast(y_pred > thresh, tf.float32)\n",
+        "\n",
+        "def accuracy(y_pred, y):\n",
+        "  # Return the proportion of matches between `y_pred` and `y`\n",
+        "  y_pred = tf.math.sigmoid(y_pred)\n",
+        "  y_pred_class = predict_class(y_pred)\n",
+        "  check_equal = tf.cast(y_pred_class == y,tf.float32)\n",
+        "  acc_val = tf.reduce_mean(check_equal)\n",
+        "  return acc_val"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J_0KHQ25_2dF"
+      },
+      "source": [
+        "### Train the model\n",
+        "\n",
+        "Using mini-batches for training provides both memory efficiency and faster convergence. The `tf.data.Dataset` API has useful functions for batching and shuffling. The API enables you to build complex input pipelines from simple, reusable pieces. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vJD7-4U0etqa"
+      },
+      "outputs": [],
+      "source": [
+        "batch_size = 64\n",
+        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train_norm, y_train))\n",
+        "train_dataset = train_dataset.shuffle(buffer_size=x_train.shape[0]).batch(batch_size)\n",
+        "test_dataset = tf.data.Dataset.from_tensor_slices((x_test_norm, y_test))\n",
+        "test_dataset = test_dataset.shuffle(buffer_size=x_test.shape[0]).batch(batch_size)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sLiWZZPBSDip"
+      },
+      "source": [
+        "Now write a training loop for the logistic regression model. The loop utilizes the log loss function and its gradients with respect to the input in order to iteratively update the model's parameters."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jNC3D1DGsGgK"
+      },
+      "outputs": [],
+      "source": [
+        "# Set training parameters\n",
+        "epochs = 200\n",
+        "learning_rate = 0.01\n",
+        "train_losses, test_losses = [], []\n",
+        "train_accs, test_accs = [], []\n",
+        "\n",
+        "# Set up the training loop and begin training\n",
+        "for epoch in range(epochs):\n",
+        "  batch_losses_train, batch_accs_train = [], []\n",
+        "  batch_losses_test, batch_accs_test = [], []\n",
+        "\n",
+        "  # Iterate over the training data\n",
+        "  for x_batch, y_batch in train_dataset:\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      y_pred_batch = log_reg(x_batch)\n",
+        "      batch_loss = log_loss(y_pred_batch, y_batch)\n",
+        "    batch_acc = accuracy(y_pred_batch, y_batch)\n",
+        "    # Update the parameters with respect to the gradient calculations\n",
+        "    grads = tape.gradient(batch_loss, log_reg.variables)\n",
+        "    for g,v in zip(grads, log_reg.variables):\n",
+        "      v.assign_sub(learning_rate * g)\n",
+        "    # Keep track of batch-level training performance\n",
+        "    batch_losses_train.append(batch_loss)\n",
+        "    batch_accs_train.append(batch_acc)\n",
+        "\n",
+        "  # Iterate over the testing data\n",
+        "  for x_batch, y_batch in test_dataset:\n",
+        "    y_pred_batch = log_reg(x_batch)\n",
+        "    batch_loss = log_loss(y_pred_batch, y_batch)\n",
+        "    batch_acc = accuracy(y_pred_batch, y_batch)\n",
+        "    # Keep track of batch-level testing performance\n",
+        "    batch_losses_test.append(batch_loss)\n",
+        "    batch_accs_test.append(batch_acc)\n",
+        "\n",
+        "  # Keep track of epoch-level model performance\n",
+        "  train_loss, train_acc = tf.reduce_mean(batch_losses_train), tf.reduce_mean(batch_accs_train)\n",
+        "  test_loss, test_acc = tf.reduce_mean(batch_losses_test), tf.reduce_mean(batch_accs_test)\n",
+        "  train_losses.append(train_loss)\n",
+        "  train_accs.append(train_acc)\n",
+        "  test_losses.append(test_loss)\n",
+        "  test_accs.append(test_acc)\n",
+        "  if epoch % 20 == 0:\n",
+        "    print(f\"Epoch: {epoch}, Training log loss: {train_loss:.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NoLiAg7fYft7"
+      },
+      "source": [
+        "### Performance evaluation\n",
+        "\n",
+        "Observe the changes in your model's loss and accuracy over time. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mv3oCQPvWhr0"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(range(epochs), train_losses, label = \"Training loss\")\n",
+        "plt.plot(range(epochs), test_losses, label = \"Testing loss\")\n",
+        "plt.xlabel(\"Epoch\")\n",
+        "plt.ylabel(\"Log loss\")\n",
+        "plt.legend()\n",
+        "plt.title(\"Log loss vs training iterations\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D2HDVGLPODIE"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(range(epochs), train_accs, label = \"Training accuracy\")\n",
+        "plt.plot(range(epochs), test_accs, label = \"Testing accuracy\")\n",
+        "plt.xlabel(\"Epoch\")\n",
+        "plt.ylabel(\"Accuracy (%)\")\n",
+        "plt.legend()\n",
+        "plt.title(\"Accuracy vs training iterations\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jonKhUzuPyfa"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Final training log loss: {train_losses[-1]:.3f}\")\n",
+        "print(f\"Final testing log Loss: {test_losses[-1]:.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d3DF4qyrPyke"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Final training accuracy: {train_accs[-1]:.3f}\")\n",
+        "print(f\"Final testing accuracy: {test_accs[-1]:.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yrj1TbOJasjA"
+      },
+      "source": [
+        "The model demonstrates a high accuracy and a low loss when it comes to classifying tumors in the training dataset and also generalizes well to the unseen test data. To go one step further, you can explore error rates that give more insight beyond the overall accuracy score. The two most popular error rates for binary classification problems are the false positive rate (FPR) and the false negative rate (FNR).\n",
+        "\n",
+        "For this problem, the FPR is the proportion of malignant tumor predictions amongst tumors that are actually benign. Conversely, the FNR is the proportion of benign tumor predictions among tumors that are actually malignant.\n",
+        "\n",
+        "Compute a confusion matrix using [`sklearn.metrics.confusion_matrix`](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html#sklearn.metrics.confusion_matrix), which evaluates the accuracy of the classification, and use matplotlib to display the matrix:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OJO7YkA8ZDMU"
+      },
+      "outputs": [],
+      "source": [
+        "def show_confusion_matrix(y, y_classes, typ):\n",
+        "  # Compute the confusion matrix and normalize it\n",
+        "  plt.figure(figsize=(10,10))\n",
+        "  confusion = sk_metrics.confusion_matrix(y.numpy(), y_classes.numpy())\n",
+        "  confusion_normalized = confusion / confusion.sum(axis=1, keepdims=True)\n",
+        "  axis_labels = range(2)\n",
+        "  ax = sns.heatmap(\n",
+        "      confusion_normalized, xticklabels=axis_labels, yticklabels=axis_labels,\n",
+        "      cmap='Blues', annot=True, fmt='.4f', square=True)\n",
+        "  plt.title(f\"Confusion matrix: {typ}\")\n",
+        "  plt.ylabel(\"True label\")\n",
+        "  plt.xlabel(\"Predicted label\")\n",
+        "\n",
+        "y_pred_train, y_pred_test = log_reg(x_train_norm, train=False), log_reg(x_test_norm, train=False)\n",
+        "train_classes, test_classes = predict_class(y_pred_train), predict_class(y_pred_test)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OQ5DFcleiDFm"
+      },
+      "outputs": [],
+      "source": [
+        "show_confusion_matrix(y_train, train_classes, 'Training')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gtfcsAp_iCNR"
+      },
+      "outputs": [],
+      "source": [
+        "show_confusion_matrix(y_test, test_classes, 'Testing')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DlivxaDmTnGq"
+      },
+      "source": [
+        "Observe the error rate measurements and interpret their significance in the context of this example. In many medical testing studies such as cancer detection, having a high false positive rate to ensure a low false negative rate is perfectly acceptable and in fact encouraged since the risk of missing a malignant tumor diagnosis (false negative) is a lot worse than misclassifying a benign tumor as malignant (false positive).\n",
+        "\n",
+        "In order to control for the FPR and FNR, try changing the threshold hyperparameter before classifying the probability predictions. A lower threshold increases the model's overall chances of making a malignant tumor classification. This inevitably increases the number of false positives and the FPR but it also helps to decrease the number of false negatives and the FNR."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7ADEN2rb4Nhj"
+      },
+      "source": [
+        "## Save the model\n",
+        "\n",
+        "Start by making an export module that takes in raw data and performs the following operations:\n",
+        "- Normalization\n",
+        "- Probability prediction\n",
+        "- Class prediction\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6KPRHCzg4ZxH"
+      },
+      "outputs": [],
+      "source": [
+        "class ExportModule(tf.Module):\n",
+        "  def __init__(self, model, norm_x, class_pred):\n",
+        "    # Initialize pre- and post-processing functions\n",
+        "    self.model = model\n",
+        "    self.norm_x = norm_x\n",
+        "    self.class_pred = class_pred\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(shape=[None, None], dtype=tf.float32)])\n",
+        "  def __call__(self, x):\n",
+        "    # Run the `ExportModule` for new data points\n",
+        "    x = self.norm_x.norm(x)\n",
+        "    y = self.model(x, train=False)\n",
+        "    y = self.class_pred(y)\n",
+        "    return y "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2YzRclo5-yjO"
+      },
+      "outputs": [],
+      "source": [
+        "log_reg_export = ExportModule(model=log_reg,\n",
+        "                              norm_x=norm_x,\n",
+        "                              class_pred=predict_class)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gtofGIBN_qFd"
+      },
+      "source": [
+        "If you want to save the model at its current state, you can do so with the `tf.saved_model.save` function. To load a saved model and make predictions, use the `tf.saved_model.load` function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a4Qum1Ts_pmF"
+      },
+      "outputs": [],
+      "source": [
+        "models = tempfile.mkdtemp()\n",
+        "save_path = os.path.join(models, 'log_reg_export')\n",
+        "tf.saved_model.save(log_reg_export, save_path)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3KPILr1i_M_c"
+      },
+      "outputs": [],
+      "source": [
+        "log_reg_loaded = tf.saved_model.load(save_path)\n",
+        "test_preds = log_reg_loaded(x_test)\n",
+        "test_preds[:10].numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vgGQuV-yqYZH"
+      },
+      "source": [
+        "## Conclusion\n",
+        "\n",
+        "This notebook introduced a few techniques to handle a logistic regression problem. Here are a few more tips that may help:\n",
+        "\n",
+        "- The [TensorFlow Core APIs](https://www.tensorflow.org/guide/core) can be used to build machine learning workflows with high levels of configurability\n",
+        "- Analyzing error rates is a great way to gain more insight about a classification model's performance beyond its overall accuracy score.\n",
+        "- Overfitting is another common problem for logistic regression models, though it wasn't a problem for this tutorial. Visit the [Overfit and underfit](../../tutorials/keras/overfit_and_underfit.ipynb) tutorial for more help with this.\n",
+        "\n",
+        "For more examples of using the TensorFlow Core APIs, check out the [guide](https://www.tensorflow.org/guide/core). If you want to learn more about loading and preparing data, see the tutorials on [image data loading](../../tutorials/load_data/images.ipynb) or [CSV data loading](../../tutorials/load_data/csv.ipynb)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "logistic_regression_core.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/matrix_core.ipynb b/site/en/guide/core/matrix_core.ipynb
new file mode 100644
index 00000000000..1d7d35ed047
--- /dev/null
+++ b/site/en/guide/core/matrix_core.ipynb
@@ -0,0 +1,731 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FhGuhbZ6M5tl"
+      },
+      "source": [
+        "##### Copyright 2022 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "AwOEIRJC6Une"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EIdT9iu_Z4Rb"
+      },
+      "source": [
+        "# Matrix approximation with Core APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bBIlTPscrIT9"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qGw8TF2vtzru"
+      },
+      "source": [
+        "## Introduction \n",
+        "\n",
+        "This notebook uses the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) to showcase TensorFlow's capabilities as a high-performance scientific computing platform. Visit the [Core APIs overview](https://www.tensorflow.org/guide/core) to learn more about TensorFlow Core and its intended use cases.\n",
+        "\n",
+        "This tutorial explores the technique of [singular value decomposition](https://developers.google.com/machine-learning/recommendation/collaborative/matrix) (SVD) and its applications for low-rank approximation problems. The SVD is used to factorize real or complex matrices and has a variety of use cases in data science such as image compression. The images for this tutorial come from Google Brain's [Imagen](https://imagen.research.google/) project. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5_FdwaovEkCC"
+      },
+      "source": [
+        ">![svd_intro](http://tensorflow.org/images/core/svd_intro.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nchsZfwEVtVs"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1rRo8oNqZ-Rj"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib\n",
+        "from matplotlib.image import imread\n",
+        "from matplotlib import pyplot as plt\n",
+        "import requests\n",
+        "# Preset Matplotlib figure sizes.\n",
+        "matplotlib.rcParams['figure.figsize'] = [16, 9]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9xQKvCJ85kCQ"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "print(tf.__version__)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "so_ewq3gAoEI"
+      },
+      "source": [
+        "## SVD fundamentals\n",
+        "\n",
+        "The singular value decomposition of a matrix, ${\\mathrm{A}}$, is determined by the following factorization:\n",
+        "\n",
+        "$${\\mathrm{A}} = {\\mathrm{U}} \\Sigma {\\mathrm{V}}^T$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $\\underset{m \\times n}{\\mathrm{A}}$: input matrix where $m \\geq n$\n",
+        "* $\\underset{m \\times n}{\\mathrm{U}}$: orthogonal matrix, ${\\mathrm{U}}^T{\\mathrm{U}} = {\\mathrm{I}}$, with each column, $u_i$, denoting a left singular vector of ${\\mathrm{A}}$\n",
+        "* $\\underset{n \\times n}{\\Sigma}$: diagonal matrix with each diagonal entry, $\\sigma_i$, denoting a singular value of ${\\mathrm{A}}$\n",
+        "* $\\underset{n \\times n}{{\\mathrm{V}}^T}$: orthogonal matrix, ${\\mathrm{V}}^T{\\mathrm{V}} = {\\mathrm{I}}$, with each row, $v_i$, denoting a right singular vector of ${\\mathrm{A}}$\n",
+        "\n",
+        "When $m < n$, ${\\mathrm{U}}$ and $\\Sigma$ both have dimension $(m \\times m)$, and ${\\mathrm{V}}^T$ has dimension $(m \\times n)$."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "enGGGXCQKNv8"
+      },
+      "source": [
+        ">![svd_full](http://tensorflow.org/images/core/svd_full.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NlP-cBdSKLtc"
+      },
+      "source": [
+        "TensorFlow's linear algebra package has a function, `tf.linalg.svd`, which can be used to compute the singular value decomposition of one or more matrices. Start by defining a simple matrix and computing its SVD factorization.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "C3QAcgyoeIpv"
+      },
+      "outputs": [],
+      "source": [
+        "A = tf.random.uniform(shape=[40,30])\n",
+        "# Compute the SVD factorization\n",
+        "s, U, V = tf.linalg.svd(A)\n",
+        "# Define Sigma and V Transpose\n",
+        "S = tf.linalg.diag(s)\n",
+        "V_T = tf.transpose(V)\n",
+        "# Reconstruct the original matrix\n",
+        "A_svd = U@S@V_T\n",
+        "# Visualize \n",
+        "plt.bar(range(len(s)), s);\n",
+        "plt.xlabel(\"Singular value rank\")\n",
+        "plt.ylabel(\"Singular value\")\n",
+        "plt.title(\"Bar graph of singular values\");"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6H_C9WhFACm4"
+      },
+      "source": [
+        "The `tf.einsum` function can be used to directly compute the matrix reconstruction from the outputs of `tf.linalg.svd`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TPE6QeMtADUn"
+      },
+      "outputs": [],
+      "source": [
+        "A_svd = tf.einsum('s,us,vs -> uv',s,U,V)\n",
+        "print('\\nReconstructed Matrix, A_svd', A_svd)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x1m6JIsM9DLP"
+      },
+      "source": [
+        "## Low rank approximation with the SVD\n",
+        "\n",
+        "The rank of a  matrix, ${\\mathrm{A}}$, is determined by the dimension of the vector space spanned by its columns. \n",
+        "The SVD can be used to approximate a matrix with a lower rank, which ultimately decreases the dimensionality of data required to store the information represented by the matrix.\n",
+        "\n",
+        "The rank-r approximation of ${\\mathrm{A}}$ in terms of the SVD is defined by the formula:\n",
+        "\n",
+        "$${\\mathrm{A_r}} = {\\mathrm{U_r}} \\Sigma_r {\\mathrm{V_r}}^T$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $\\underset{m \\times r}{\\mathrm{U_r}}$: matrix consisting of the first $r$ columns of ${\\mathrm{U}}$\n",
+        "* $\\underset{r \\times r}{\\Sigma_r}$:  diagonal matrix consisting of the first  $r$ singular values in $\\Sigma$\n",
+        "* $\\underset{r \\times n}{\\mathrm{V_r}}^T$: matrix consisting of the first $r$ rows of ${\\mathrm{V}}^T$"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nJWMJu36QyUV"
+      },
+      "source": [
+        ">![svd_approx](http://tensorflow.org/images/core/svd_approx.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TkiVUxeaQybq"
+      },
+      "source": [
+        "Start by writing a function to compute the rank-r approximation of a given matrix. This low-rank approximation procedure is used for image compression; therefore, it is also helpful to compute the physical data sizes for each approximation. For simplicity, assume that data size for an rank-r approximated matrix is equal to the total number of elements required to compute the approximation. Next, write a function to visualize the original matrix, $\\mathrm{A}$ its rank-r approximation, $\\mathrm{A}_r$ and the error matrix, $|\\mathrm{A} - \\mathrm{A}_r|$."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2oY3pMPagJrO"
+      },
+      "outputs": [],
+      "source": [
+        "def rank_r_approx(s, U, V, r, verbose=False):\n",
+        "  # Compute the matrices necessary for a rank-r approximation\n",
+        "  s_r, U_r, V_r = s[..., :r], U[..., :, :r], V[..., :, :r] # ... implies any number of extra batch axes\n",
+        "  # Compute the low-rank approximation and its size\n",
+        "  A_r = tf.einsum('...s,...us,...vs->...uv',s_r,U_r,V_r)\n",
+        "  A_r_size = tf.size(U_r) + tf.size(s_r) + tf.size(V_r)\n",
+        "  if verbose:\n",
+        "    print(f\"Approximation Size: {A_r_size}\")\n",
+        "  return A_r, A_r_size\n",
+        "\n",
+        "def viz_approx(A, A_r):\n",
+        "  # Plot A, A_r, and A - A_r\n",
+        "  vmin, vmax = 0, tf.reduce_max(A)\n",
+        "  fig, ax = plt.subplots(1,3)\n",
+        "  mats = [A, A_r, abs(A - A_r)]\n",
+        "  titles = ['Original A', 'Approximated A_r', 'Error |A - A_r|']\n",
+        "  for i, (mat, title) in enumerate(zip(mats, titles)):\n",
+        "    ax[i].pcolormesh(mat, vmin=vmin, vmax=vmax)\n",
+        "    ax[i].set_title(title)\n",
+        "    ax[i].axis('off')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "O3ZRkYCkX2FQ"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"Original Size of A: {tf.size(A)}\")\n",
+        "s, U, V = tf.linalg.svd(A)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S1DR83VMX4cM"
+      },
+      "outputs": [],
+      "source": [
+        "# Rank-15 approximation\n",
+        "A_15, A_15_size = rank_r_approx(s, U, V, 15, verbose = True)\n",
+        "viz_approx(A, A_15)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KgFT70XFX57E"
+      },
+      "outputs": [],
+      "source": [
+        "# Rank-3 approximation\n",
+        "A_3, A_3_size = rank_r_approx(s, U, V, 3, verbose = True)\n",
+        "viz_approx(A, A_3)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DS4XoSlTJgX0"
+      },
+      "source": [
+        "As expected, using lower ranks results in less-accurate approximations. However, the quality of these low-rank approximations are often good enough in real world scenarios. Also note that the main goal of low-rank approximation with SVD \n",
+        "is to reduce the dimensionality of the data but not to reduce the disk space of the data itself. However, as the input matrices become higher-dimensional, many low-rank approximations also end up benefiting from reduced data size. This reduction benefit is why the process is applicable for image compression problems."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IhsaiOnnZs6M"
+      },
+      "source": [
+        "## Image loading\n",
+        "\n",
+        "The following image is available on the [Imagen](https://imagen.research.google/) home page. Imagen is a text-to-image diffusion model developed by Google Research's Brain team. An AI created this image based on the prompt: \"A photo of a Corgi dog riding a bike in Times Square. It is wearing sunglasses and a beach hat.\" How cool is that! You can also change the url below to any .jpg link to load in a custom image of choice. \n",
+        "\n",
+        "Start by reading in and visualizing the image. After reading a JPEG file, Matplotlib outputs a matrix, ${\\mathrm{I}}$, of shape $(m \\times n \\times 3)$ which represents a 2-dimensional image with 3 color channels for red, green and blue respectively."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OVsZOQUAZ2C7"
+      },
+      "outputs": [],
+      "source": [
+        "img_link = \"https://imagen.research.google/main_gallery_images/a-photo-of-a-corgi-dog-riding-a-bike-in-times-square.jpg\"\n",
+        "img_path = requests.get(img_link, stream=True).raw\n",
+        "I = imread(img_path, 0)\n",
+        "print(\"Input Image Shape:\", I.shape)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Qvs7uftcZ54x"
+      },
+      "outputs": [],
+      "source": [
+        "def show_img(I):\n",
+        "  # Display the image in matplotlib\n",
+        "  img = plt.imshow(I)\n",
+        "  plt.axis('off')\n",
+        "  return"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZbesXO3HZ6Qs"
+      },
+      "outputs": [],
+      "source": [
+        "show_img(I)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tdnUBVg_JoOa"
+      },
+      "source": [
+        "## The image compression algorithm\n",
+        "\n",
+        "Now, use the SVD to compute low-rank approximations of the sample image. Recall that the image is of shape $(1024 \\times 1024 \\times 3)$ and that the theory SVD only applies for 2-dimensional matrices. This means that the sample image has to be batched into 3 equal-size matrices that correspond to each of the 3 color channels. This can be done so by transposing the matrix to be of shape $(3 \\times 1024 \\times 1024)$. In order to clearly visualize the approximation error, rescale the RGB values of the image from $[0,255]$ to $[0,1]$. Remember to clip the approximated values to fall within this interval before visualizing them. The `tf.clip_by_value` function is useful for this."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "i7DDp0h7oSIk"
+      },
+      "outputs": [],
+      "source": [
+        "def compress_image(I, r, verbose=False):\n",
+        "  # Compress an image with the SVD given a rank \n",
+        "  I_size = tf.size(I)\n",
+        "  print(f\"Original size of image: {I_size}\")\n",
+        "  # Compute SVD of image\n",
+        "  I = tf.convert_to_tensor(I)/255\n",
+        "  I_batched = tf.transpose(I, [2, 0, 1]) # einops.rearrange(I, 'h w c -> c h w')\n",
+        "  s, U, V = tf.linalg.svd(I_batched)\n",
+        "  # Compute low-rank approximation of image across each RGB channel\n",
+        "  I_r, I_r_size = rank_r_approx(s, U, V, r)\n",
+        "  I_r = tf.transpose(I_r, [1, 2, 0]) # einops.rearrange(I_r, 'c h w -> h w c')\n",
+        "  I_r_prop = (I_r_size / I_size)\n",
+        "  if verbose:\n",
+        "    # Display compressed image and attributes\n",
+        "    print(f\"Number of singular values used in compression: {r}\")\n",
+        "    print(f\"Compressed image size: {I_r_size}\")\n",
+        "    print(f\"Proportion of original size: {I_r_prop:.3f}\")\n",
+        "    ax_1 = plt.subplot(1,2,1)\n",
+        "    show_img(tf.clip_by_value(I_r,0.,1.))\n",
+        "    ax_1.set_title(\"Approximated image\")\n",
+        "    ax_2 = plt.subplot(1,2,2)\n",
+        "    show_img(tf.clip_by_value(0.5+abs(I-I_r),0.,1.))\n",
+        "    ax_2.set_title(\"Error\")\n",
+        "  return I_r, I_r_prop"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RGQ_rTyKDX9F"
+      },
+      "source": [
+        "Now, compute rank-r approximations for the following ranks : 100, 50, 10"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7GlKkVLGDjre"
+      },
+      "outputs": [],
+      "source": [
+        "I_100, I_100_prop = compress_image(I, 100, verbose=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XdvUkF5_E75D"
+      },
+      "outputs": [],
+      "source": [
+        "I_50, I_50_prop = compress_image(I, 50, verbose=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MsCNZ8416Sbk"
+      },
+      "outputs": [],
+      "source": [
+        "I_10, I_10_prop = compress_image(I, 10, verbose=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RfYYBhcuNkvH"
+      },
+      "source": [
+        "## Evaluating approximations\n",
+        "\n",
+        "There are a variety of interesting methods to measure the effectiveness and have more control over matrix approximations."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "D2Lotde9Zg7v"
+      },
+      "source": [
+        "### Compression factor vs rank\n",
+        "\n",
+        "For each of the above approximations, observe how the data sizes change with the rank."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "O1ariNQe6Wbl"
+      },
+      "outputs": [],
+      "source": [
+        "plt.figure(figsize=(11,6))\n",
+        "plt.plot([100, 50, 10], [I_100_prop, I_50_prop, I_10_prop])\n",
+        "plt.xlabel(\"Rank\")\n",
+        "plt.ylabel(\"Proportion of original image size\")\n",
+        "plt.title(\"Compression factor vs rank\");"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dvHcLRj2QoDg"
+      },
+      "source": [
+        "Based on this plot, there is a linear relationship between an approximated image's compression factor and its rank. To explore this further, recall that the data size of an approximated matrix, ${\\mathrm{A}}_r$, is defined as the total number of elements required for its computation. The following equations can be used to find the relationship between compression factor and rank:\n",
+        "\n",
+        "$$x = (m \\times r) + r + (r \\times n) = r \\times (m + n + 1)$$\n",
+        "\n",
+        "$$c = \\large \\frac{x}{y} = \\frac{r \\times (m + n + 1)}{m \\times n}$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $x$: size of ${\\mathrm{A_r}}$\n",
+        "* $y$: size of ${\\mathrm{A}}$\n",
+        "* $c = \\frac{x}{y}$: compression factor\n",
+        "* $r$: rank of the approximation\n",
+        "* $m$ and $n$: row and column dimensions of ${\\mathrm{A}}$\n",
+        "\n",
+        "In order to find the rank, $r$, that is necessary to compress an image to a desired factor, $c$, the above equation can be rearranged to solve for $r$:\n",
+        "\n",
+        "$$r = ⌊{\\large\\frac{c \\times m \\times n}{m + n + 1}}⌋$$\n",
+        "\n",
+        "Note that this formula is independent of the color channel dimension since each of the RGB approximations do not affect each other. Now, write a function to compress an input image given a desired compression factor."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "viVO-I60QynI"
+      },
+      "outputs": [],
+      "source": [
+        "def compress_image_with_factor(I, compression_factor, verbose=False):\n",
+        "  # Returns a compressed image based on a desired compression factor\n",
+        "  m,n,o = I.shape\n",
+        "  r = int((compression_factor * m * n)/(m + n + 1))\n",
+        "  I_r, I_r_prop = compress_image(I, r, verbose=verbose)\n",
+        "  return I_r"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gWSv58J6LSRQ"
+      },
+      "source": [
+        "Compress an image to 15% of its original size."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HVeeloIwQ1b6"
+      },
+      "outputs": [],
+      "source": [
+        "compression_factor = 0.15\n",
+        "I_r_img = compress_image_with_factor(I, compression_factor, verbose=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LkeRyms7jZMd"
+      },
+      "source": [
+        "### Cumulative sum of singular values\n",
+        "\n",
+        "The cumulative sum of singular values can be a useful indicator for the amount of energy captured by a rank-r approximation. Visualize the RGB-averaged cumulative proportion of singular values in the sample image. The `tf.cumsum` function can be useful for this."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CteJ6VbKlndu"
+      },
+      "outputs": [],
+      "source": [
+        "def viz_energy(I):\n",
+        "  # Visualize the energy captured based on rank\n",
+        "  # Computing SVD\n",
+        "  I = tf.convert_to_tensor(I)/255\n",
+        "  I_batched = tf.transpose(I, [2, 0, 1]) \n",
+        "  s, U, V = tf.linalg.svd(I_batched)\n",
+        "  # Plotting average proportion across RGB channels \n",
+        "  props_rgb = tf.map_fn(lambda x: tf.cumsum(x)/tf.reduce_sum(x), s)\n",
+        "  props_rgb_mean = tf.reduce_mean(props_rgb, axis=0)\n",
+        "  plt.figure(figsize=(11,6))\n",
+        "  plt.plot(range(len(I)), props_rgb_mean, color='k')\n",
+        "  plt.xlabel(\"Rank / singular value number\")\n",
+        "  plt.ylabel(\"Cumulative proportion of singular values\")\n",
+        "  plt.title(\"RGB-averaged proportion of energy captured by the first 'r' singular values\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Vl9PKow-GgCp"
+      },
+      "outputs": [],
+      "source": [
+        "viz_energy(I)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vQtwimKuQP19"
+      },
+      "source": [
+        "It looks like over 90% of the energy in this image is captured within the first 100 singular values. Now, write a function to compress an input image given a desired energy retention factor."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fum5Cvm7R5vH"
+      },
+      "outputs": [],
+      "source": [
+        "def compress_image_with_energy(I, energy_factor, verbose=False):\n",
+        "  # Returns a compressed image based on a desired energy factor\n",
+        "  # Computing SVD\n",
+        "  I_rescaled = tf.convert_to_tensor(I)/255\n",
+        "  I_batched = tf.transpose(I_rescaled, [2, 0, 1]) \n",
+        "  s, U, V = tf.linalg.svd(I_batched)\n",
+        "  # Extracting singular values\n",
+        "  props_rgb = tf.map_fn(lambda x: tf.cumsum(x)/tf.reduce_sum(x), s)\n",
+        "  props_rgb_mean = tf.reduce_mean(props_rgb, axis=0)\n",
+        "  # Find closest r that corresponds to the energy factor\n",
+        "  r = tf.argmin(tf.abs(props_rgb_mean - energy_factor)) + 1\n",
+        "  actual_ef = props_rgb_mean[r]\n",
+        "  I_r, I_r_prop = compress_image(I, r, verbose=verbose)\n",
+        "  print(f\"Proportion of energy captured by the first {r} singular values: {actual_ef:.3f}\")\n",
+        "  return I_r"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Y_rChG0OLby1"
+      },
+      "source": [
+        "Compress an image to retain 75% of its energy."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xDXBaZQ4c5jF"
+      },
+      "outputs": [],
+      "source": [
+        "energy_factor = 0.75\n",
+        "I_r_img = compress_image_with_energy(I, energy_factor, verbose=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2tmqTW0CYX-v"
+      },
+      "source": [
+        "### Error and singular values\n",
+        "\n",
+        "There is also an interesting relationship between the approximation error and the singular values. It turns out that the squared Frobenius norm of the approximation is equal to the sum of the squares of its singular values that were left out:\n",
+        "\n",
+        "$${||A - A_r||}^2 = \\sum_{i=r+1}^{R}σ_i^2$$\n",
+        "\n",
+        "Test out this relationship with a rank-10 approximation of the example matrix in the beginning of this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hctOvN8BckiS"
+      },
+      "outputs": [],
+      "source": [
+        "s, U, V = tf.linalg.svd(A)\n",
+        "A_10, A_10_size = rank_r_approx(s, U, V, 10)\n",
+        "squared_norm = tf.norm(A - A_10)**2\n",
+        "s_squared_sum = tf.reduce_sum(s[10:]**2)\n",
+        "print(f\"Squared Frobenius norm: {squared_norm:.3f}\")\n",
+        "print(f\"Sum of squared singular values left out: {s_squared_sum:.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vgGQuV-yqYZH"
+      },
+      "source": [
+        "## Conclusion\n",
+        "\n",
+        "This notebook introduced the process of implementing the singular value decomposition with TensorFlow and applying it to write an image compression algorithm. Here are a few more tips that may help:\n",
+        "\n",
+        "*   The [TensorFlow Core APIs](https://www.tensorflow.org/guide/core) can be utilized for a variety of high-performance scientific computing use cases.\n",
+        "*   To learn more about TensorFlow's linear algebra functionalities, visit the docs for the [linalg module](https://www.tensorflow.org/api_docs/python/tf/linalg).\n",
+        "*   The SVD can also be applied to build [recommendation systems](https://developers.google.com/machine-learning/recommendation/labs/movie-rec-programming-exercise).\n",
+        "\n",
+        "\n",
+        "For more examples of using the TensorFlow Core APIs, check out the [guide](https://www.tensorflow.org/guide/core). If you want learn more about loading and preparing data, see the tutorials on [image data loading](https://www.tensorflow.org/tutorials/load_data/images) or [CSV data loading](https://www.tensorflow.org/tutorials/load_data/csv)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "matrix_core.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/mlp_core.ipynb b/site/en/guide/core/mlp_core.ipynb
new file mode 100644
index 00000000000..a5975c20c6e
--- /dev/null
+++ b/site/en/guide/core/mlp_core.ipynb
@@ -0,0 +1,964 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FhGuhbZ6M5tl"
+      },
+      "source": [
+        "##### Copyright 2022 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "AwOEIRJC6Une"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EIdT9iu_Z4Rb"
+      },
+      "source": [
+        "# Multilayer perceptrons for digit recognition with Core APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bBIlTPscrIT9"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SjAxxRpBzVYg"
+      },
+      "source": [
+        "This notebook uses the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) to build an end-to-end machine learning workflow for handwritten digit classification with [multilayer perceptrons](https://developers.google.com/machine-learning/crash-course/introduction-to-neural-networks/anatomy) and the [MNIST dataset](http://yann.lecun.com/exdb/mnist). Visit the [Core APIs overview](https://www.tensorflow.org/guide/core) to learn more about TensorFlow Core and its intended use cases."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GHVMVIFHSzl1"
+      },
+      "source": [
+        "## Multilayer perceptron (MLP) overview\n",
+        "\n",
+        "The Multilayer Perceptron (MLP) is a type of feedforward neural network used to approach [multiclass classification](https://developers.google.com/machine-learning/crash-course/multi-class-neural-networks/video-lecture) problems. Before building an MLP, it is crucial to understand the concepts of perceptrons, layers, and activation functions.\n",
+        "\n",
+        "Multilayer Perceptrons are made up of functional units called perceptrons. The equation of a  perceptron is as follows:\n",
+        "\n",
+        "$$Z = \\vec{w}⋅\\mathrm{X} + b$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $Z$: perceptron output\n",
+        "* $\\mathrm{X}$: feature matrix\n",
+        "* $\\vec{w}$: weight vector\n",
+        "* $b$: bias\n",
+        "\n",
+        "When these perceptrons are stacked, they form structures called dense layers which can then be connected to build a neural network. A dense layer's equation is similar to that of a perceptron's but uses a weight matrix and a bias vector instead: \n",
+        "\n",
+        "$$Z = \\mathrm{W}⋅\\mathrm{X} + \\vec{b}$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $Z$: dense layer output\n",
+        "* $\\mathrm{X}$: feature matrix\n",
+        "* $\\mathrm{W}$: weight matrix\n",
+        "* $\\vec{b}$: bias vector\n",
+        "\n",
+        "\n",
+        "In an MLP, multiple dense layers are connected in such a way that the outputs of one layer are fully connected to the inputs of the next layer. Adding non-linear activation functions to the outputs of dense layers can help the MLP classifier learn complex decision boundaries and generalize well to unseen data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nchsZfwEVtVs"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Import TensorFlow, [pandas](https://pandas.pydata.org), [Matplotlib](https://matplotlib.org) and [seaborn](https://seaborn.pydata.org) to get started."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mSfgqmwBagw_"
+      },
+      "outputs": [],
+      "source": [
+        "# Use seaborn for countplot.\n",
+        "!pip install -q seaborn"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1rRo8oNqZ-Rj"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd\n",
+        "import matplotlib\n",
+        "from matplotlib import pyplot as plt\n",
+        "import seaborn as sns\n",
+        "import tempfile\n",
+        "import os\n",
+        "# Preset Matplotlib figure sizes.\n",
+        "matplotlib.rcParams['figure.figsize'] = [9, 6]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9xQKvCJ85kCQ"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_datasets as tfds\n",
+        "print(tf.__version__)\n",
+        "# Set random seed for reproducible results \n",
+        "tf.random.set_seed(22)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "F_72b0LCNbjx"
+      },
+      "source": [
+        "## Load the data\n",
+        "\n",
+        "This tutorial uses the [MNIST dataset](http://yann.lecun.com/exdb/mnist), and demonstrates how to build an MLP model that can classify handwritten digits. The dataset is available from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/mnist).\n",
+        "\n",
+        "Split the MNIST dataset into training, validation, and testing sets. The validation set can be used to gauge the model's generalizability during training so that the test set can serve as a final unbiased estimator for the model's performance.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Uiuh0B098_3p"
+      },
+      "outputs": [],
+      "source": [
+        "train_data, val_data, test_data = tfds.load(\"mnist\", \n",
+        "                                            split=['train[10000:]', 'train[0:10000]', 'test'],\n",
+        "                                            batch_size=128, as_supervised=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "X9uN3Lf6ANtn"
+      },
+      "source": [
+        "The MNIST dataset consists of handwritten digits and their corresponding true labels. Visualize a couple of examples below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6V8hSqJ7AMjQ"
+      },
+      "outputs": [],
+      "source": [
+        "x_viz, y_viz = tfds.load(\"mnist\", split=['train[:1500]'], batch_size=-1, as_supervised=True)[0]\n",
+        "x_viz = tf.squeeze(x_viz, axis=3)\n",
+        "\n",
+        "for i in range(9):\n",
+        "    plt.subplot(3,3,1+i)\n",
+        "    plt.axis('off')\n",
+        "    plt.imshow(x_viz[i], cmap='gray')\n",
+        "    plt.title(f\"True Label: {y_viz[i]}\")\n",
+        "    plt.subplots_adjust(hspace=.5)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bRald9dSE4qS"
+      },
+      "source": [
+        "Also review the distribution of digits in the training data to verify that each class is well represented in the dataset.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Rj3K4XgQE7qR"
+      },
+      "outputs": [],
+      "source": [
+        "sns.countplot(x=y_viz.numpy());\n",
+        "plt.xlabel('Digits')\n",
+        "plt.title(\"MNIST Digit Distribution\");"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x_Wt4bDx_BRV"
+      },
+      "source": [
+        "## Preprocess the data\n",
+        "\n",
+        "First, reshape the feature matrices to be 2-dimensional by flattening the images. Next, rescale the data so that the pixel values of [0,255] fit into the range of [0,1]. This step ensures that the input pixels have similar distributions and helps with training convergence."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JSyCm2V2_AvI"
+      },
+      "outputs": [],
+      "source": [
+        "def preprocess(x, y):\n",
+        "  # Reshaping the data\n",
+        "  x = tf.reshape(x, shape=[-1, 784])\n",
+        "  # Rescaling the data\n",
+        "  x = x/255\n",
+        "  return x, y\n",
+        "\n",
+        "train_data, val_data = train_data.map(preprocess), val_data.map(preprocess)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6o3CrycBXA2s"
+      },
+      "source": [
+        "## Build the MLP \n",
+        "\n",
+        "Start by visualizing the [ReLU](https://developers.google.com/machine-learning/glossary#ReLU) and [Softmax](https://developers.google.com/machine-learning/glossary#softmax) activation functions. Both functions are available in `tf.nn.relu` and `tf.nn.softmax` respectively. The ReLU is a non-linear activation function that outputs the input if it is positive and 0 otherwise: \n",
+        "\n",
+        "$$\\text{ReLU}(X) = max(0, X)$$"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hYunzt3UyT9G"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.linspace(-2, 2, 201)\n",
+        "x = tf.cast(x, tf.float32)\n",
+        "plt.plot(x, tf.nn.relu(x));\n",
+        "plt.xlabel('x')\n",
+        "plt.ylabel('ReLU(x)')\n",
+        "plt.title('ReLU activation function');"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fuGrM9jMwsRM"
+      },
+      "source": [
+        "The softmax activation function is a normalized exponential function that converts $m$ real numbers into a probability distribution with $m$ outcomes/classes. This is useful for predicting class probabilities from a neural network's output:\n",
+        "\n",
+        "$$\\text{Softmax}(X) = \\frac{e^{X}}{\\sum_{i=1}^{m}e^{X_i}}$$"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fVM8pvhWwuwI"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.linspace(-4, 4, 201)\n",
+        "x = tf.cast(x, tf.float32)\n",
+        "plt.plot(x, tf.nn.softmax(x, axis=0));\n",
+        "plt.xlabel('x')\n",
+        "plt.ylabel('Softmax(x)')\n",
+        "plt.title('Softmax activation function');"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OHW6Yvg2yS6H"
+      },
+      "source": [
+        "### The dense layer\n",
+        "\n",
+        "Create a class for the dense layer. By definition, the outputs of one layer are fully connected to the inputs of the next layer in an MLP. Therefore, the input dimension for a dense layer can be inferred based on the output dimension of its previous layer and does not need to be specified upfront during its initialization. The weights should also be initialized properly to prevent activation outputs from becoming too large or small. One of the most popular weight initialization methods is the Xavier scheme, where each element of the weight matrix is sampled in the following manner:\n",
+        "\n",
+        "$$W_{ij} \\sim \\text{Uniform}(-\\frac{\\sqrt{6}}{\\sqrt{n + m}},\\frac{\\sqrt{6}}{\\sqrt{n + m}})$$\n",
+        "\n",
+        "The bias vector can be initialized to zeros."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "re1SSFyBdMrS"
+      },
+      "outputs": [],
+      "source": [
+        "def xavier_init(shape):\n",
+        "  # Computes the xavier initialization values for a weight matrix\n",
+        "  in_dim, out_dim = shape\n",
+        "  xavier_lim = tf.sqrt(6.)/tf.sqrt(tf.cast(in_dim + out_dim, tf.float32))\n",
+        "  weight_vals = tf.random.uniform(shape=(in_dim, out_dim), \n",
+        "                                  minval=-xavier_lim, maxval=xavier_lim, seed=22)\n",
+        "  return weight_vals"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "otDFX4u6e6ml"
+      },
+      "source": [
+        "The Xavier initialization method can also be implemented with `tf.keras.initializers.GlorotUniform`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IM0yJos25FG5"
+      },
+      "outputs": [],
+      "source": [
+        "class DenseLayer(tf.Module):\n",
+        "\n",
+        "  def __init__(self, out_dim, weight_init=xavier_init, activation=tf.identity):\n",
+        "    # Initialize the dimensions and activation functions\n",
+        "    self.out_dim = out_dim\n",
+        "    self.weight_init = weight_init\n",
+        "    self.activation = activation\n",
+        "    self.built = False\n",
+        "\n",
+        "  def __call__(self, x):\n",
+        "    if not self.built:\n",
+        "      # Infer the input dimension based on first call\n",
+        "      self.in_dim = x.shape[1]\n",
+        "      # Initialize the weights and biases\n",
+        "      self.w = tf.Variable(self.weight_init(shape=(self.in_dim, self.out_dim)))\n",
+        "      self.b = tf.Variable(tf.zeros(shape=(self.out_dim,)))\n",
+        "      self.built = True\n",
+        "    # Compute the forward pass\n",
+        "    z = tf.add(tf.matmul(x, self.w), self.b)\n",
+        "    return self.activation(z)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "X-7MzpjgyHg6"
+      },
+      "source": [
+        "Next, build a class for the MLP model that executes layers sequentially.\n",
+        "Remember that the model variables are only available after the first sequence of dense layer calls due to dimension inference."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6XisRWiCyHAb"
+      },
+      "outputs": [],
+      "source": [
+        "class MLP(tf.Module):\n",
+        "\n",
+        "  def __init__(self, layers):\n",
+        "    self.layers = layers\n",
+        "   \n",
+        "  @tf.function\n",
+        "  def __call__(self, x, preds=False): \n",
+        "    # Execute the model's layers sequentially\n",
+        "    for layer in self.layers:\n",
+        "      x = layer(x)\n",
+        "    return x"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "luXKup-43nd7"
+      },
+      "source": [
+        "Initialize a MLP model with the following architecture:\n",
+        "\n",
+        "- Forward Pass: ReLU(784 x 700) x ReLU(700 x 500) x Softmax(500 x 10)\n",
+        "\n",
+        "The softmax activation function does not need to be applied by the MLP. It is computed separately in the loss and prediction functions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VmlACuki3oPi"
+      },
+      "outputs": [],
+      "source": [
+        "hidden_layer_1_size = 700\n",
+        "hidden_layer_2_size = 500\n",
+        "output_size = 10\n",
+        "\n",
+        "mlp_model = MLP([\n",
+        "    DenseLayer(out_dim=hidden_layer_1_size, activation=tf.nn.relu),\n",
+        "    DenseLayer(out_dim=hidden_layer_2_size, activation=tf.nn.relu),\n",
+        "    DenseLayer(out_dim=output_size)])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tyBATDoRmDkg"
+      },
+      "source": [
+        "### Define the loss function\n",
+        "\n",
+        "The cross-entropy loss function is a great choice for multiclass classification problems since it measures the negative-log-likelihood of the data according to the model's probability predictions. The higher the probability assigned to the true class, the lower the loss. The equation for the cross-entropy loss is as follows:\n",
+        "\n",
+        "$$L = -\\frac{1}{n}\\sum_{i=1}^{n}\\sum_{i=j}^{n} {y_j}^{[i]}⋅\\log(\\hat{{y_j}}^{[i]})$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $\\underset{n\\times m}{\\hat{y}}$: a matrix of predicted class distributions\n",
+        "* $\\underset{n\\times m}{y}$: a one hot encoded matrix of true classes\n",
+        "\n",
+        "The `tf.nn.sparse_softmax_cross_entropy_with_logits` function can be used to compute the cross-entropy loss. This function does not require the model's last layer to apply the softmax activation function nor does it require the class labels to be one hot encoded"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rskOYA7FVCwg"
+      },
+      "outputs": [],
+      "source": [
+        "def cross_entropy_loss(y_pred, y):\n",
+        "  # Compute cross entropy loss with a sparse operation\n",
+        "  sparse_ce = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=y_pred)\n",
+        "  return tf.reduce_mean(sparse_ce)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BvWxED1km8jh"
+      },
+      "source": [
+        "Write a basic accuracy function that calculates the proportion of correct classifications during training. In order to generate class predictions from softmax outputs, return the index that corresponds to the largest class probability. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jPJMWx2UgiBm"
+      },
+      "outputs": [],
+      "source": [
+        "def accuracy(y_pred, y):\n",
+        "  # Compute accuracy after extracting class predictions\n",
+        "  class_preds = tf.argmax(tf.nn.softmax(y_pred), axis=1)\n",
+        "  is_equal = tf.equal(y, class_preds)\n",
+        "  return tf.reduce_mean(tf.cast(is_equal, tf.float32))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JSiNRhTOnKZr"
+      },
+      "source": [
+        "### Train the model\n",
+        "\n",
+        "Using an optimizer can result in significantly faster convergence compared to standard gradient descent. The Adam optimizer is implemented below. Visit the [Optimizers](https://www.tensorflow.org/guide/core/optimizers_core) guide to learn more about designing custom optimizers with TensorFlow Core."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iGIBDk3cAv6a"
+      },
+      "outputs": [],
+      "source": [
+        "class Adam:\n",
+        "\n",
+        "    def __init__(self, learning_rate=1e-3, beta_1=0.9, beta_2=0.999, ep=1e-7):\n",
+        "      # Initialize optimizer parameters and variable slots\n",
+        "      self.beta_1 = beta_1\n",
+        "      self.beta_2 = beta_2\n",
+        "      self.learning_rate = learning_rate\n",
+        "      self.ep = ep\n",
+        "      self.t = 1.\n",
+        "      self.v_dvar, self.s_dvar = [], []\n",
+        "      self.built = False\n",
+        "      \n",
+        "    def apply_gradients(self, grads, vars):\n",
+        "      # Initialize variables on the first call\n",
+        "      if not self.built:\n",
+        "        for var in vars:\n",
+        "          v = tf.Variable(tf.zeros(shape=var.shape))\n",
+        "          s = tf.Variable(tf.zeros(shape=var.shape))\n",
+        "          self.v_dvar.append(v)\n",
+        "          self.s_dvar.append(s)\n",
+        "        self.built = True\n",
+        "      # Update the model variables given their gradients\n",
+        "      for i, (d_var, var) in enumerate(zip(grads, vars)):\n",
+        "        self.v_dvar[i].assign(self.beta_1*self.v_dvar[i] + (1-self.beta_1)*d_var)\n",
+        "        self.s_dvar[i].assign(self.beta_2*self.s_dvar[i] + (1-self.beta_2)*tf.square(d_var))\n",
+        "        v_dvar_bc = self.v_dvar[i]/(1-(self.beta_1**self.t))\n",
+        "        s_dvar_bc = self.s_dvar[i]/(1-(self.beta_2**self.t))\n",
+        "        var.assign_sub(self.learning_rate*(v_dvar_bc/(tf.sqrt(s_dvar_bc) + self.ep)))\n",
+        "      self.t += 1.\n",
+        "      return "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "osEK3rqpYfKd"
+      },
+      "source": [
+        "Now, write a custom training loop that updates the MLP parameters with mini-batch gradient descent. Using mini-batches for training provides both memory efficiency and faster convergence."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CJLeY2ao1aw6"
+      },
+      "outputs": [],
+      "source": [
+        "def train_step(x_batch, y_batch, loss, acc, model, optimizer):\n",
+        "  # Update the model state given a batch of data\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    y_pred = model(x_batch)\n",
+        "    batch_loss = loss(y_pred, y_batch)\n",
+        "  batch_acc = acc(y_pred, y_batch)\n",
+        "  grads = tape.gradient(batch_loss, model.variables)\n",
+        "  optimizer.apply_gradients(grads, model.variables)\n",
+        "  return batch_loss, batch_acc\n",
+        "\n",
+        "def val_step(x_batch, y_batch, loss, acc, model):\n",
+        "  # Evaluate the model on given a batch of validation data\n",
+        "  y_pred = model(x_batch)\n",
+        "  batch_loss = loss(y_pred, y_batch)\n",
+        "  batch_acc = acc(y_pred, y_batch)\n",
+        "  return batch_loss, batch_acc"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oC85kuZgmh3q"
+      },
+      "outputs": [],
+      "source": [
+        "def train_model(mlp, train_data, val_data, loss, acc, optimizer, epochs):\n",
+        "  # Initialize data structures\n",
+        "  train_losses, train_accs = [], []\n",
+        "  val_losses, val_accs = [], []\n",
+        "\n",
+        "  # Format training loop and begin training\n",
+        "  for epoch in range(epochs):\n",
+        "    batch_losses_train, batch_accs_train = [], []\n",
+        "    batch_losses_val, batch_accs_val = [], []\n",
+        "\n",
+        "    # Iterate over the training data\n",
+        "    for x_batch, y_batch in train_data:\n",
+        "      # Compute gradients and update the model's parameters\n",
+        "      batch_loss, batch_acc = train_step(x_batch, y_batch, loss, acc, mlp, optimizer)\n",
+        "      # Keep track of batch-level training performance\n",
+        "      batch_losses_train.append(batch_loss)\n",
+        "      batch_accs_train.append(batch_acc)\n",
+        "\n",
+        "    # Iterate over the validation data\n",
+        "    for x_batch, y_batch in val_data:\n",
+        "      batch_loss, batch_acc = val_step(x_batch, y_batch, loss, acc, mlp)\n",
+        "      batch_losses_val.append(batch_loss)\n",
+        "      batch_accs_val.append(batch_acc)\n",
+        "\n",
+        "    # Keep track of epoch-level model performance\n",
+        "    train_loss, train_acc = tf.reduce_mean(batch_losses_train), tf.reduce_mean(batch_accs_train)\n",
+        "    val_loss, val_acc = tf.reduce_mean(batch_losses_val), tf.reduce_mean(batch_accs_val)\n",
+        "    train_losses.append(train_loss)\n",
+        "    train_accs.append(train_acc)\n",
+        "    val_losses.append(val_loss)\n",
+        "    val_accs.append(val_acc)\n",
+        "    print(f\"Epoch: {epoch}\")\n",
+        "    print(f\"Training loss: {train_loss:.3f}, Training accuracy: {train_acc:.3f}\")\n",
+        "    print(f\"Validation loss: {val_loss:.3f}, Validation accuracy: {val_acc:.3f}\")\n",
+        "  return train_losses, train_accs, val_losses, val_accs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvbfXlN5lwwB"
+      },
+      "source": [
+        "Train the MLP model for 10 epochs with batch size of 128. Hardware accelerators like GPUs or TPUs can also help speed up training time. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zPlT8QfxptYl"
+      },
+      "outputs": [],
+      "source": [
+        "train_losses, train_accs, val_losses, val_accs = train_model(mlp_model, train_data, val_data, \n",
+        "                                                             loss=cross_entropy_loss, acc=accuracy,\n",
+        "                                                             optimizer=Adam(), epochs=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "j_RVmt43G12R"
+      },
+      "source": [
+        "### Performance evaluation\n",
+        "\n",
+        "Start by writing a plotting function to visualize the model's loss and accuracy during training. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VXTCYVtNDjAM"
+      },
+      "outputs": [],
+      "source": [
+        "def plot_metrics(train_metric, val_metric, metric_type):\n",
+        "  # Visualize metrics vs training Epochs\n",
+        "  plt.figure()\n",
+        "  plt.plot(range(len(train_metric)), train_metric, label = f\"Training {metric_type}\")\n",
+        "  plt.plot(range(len(val_metric)), val_metric, label = f\"Validation {metric_type}\")\n",
+        "  plt.xlabel(\"Epochs\")\n",
+        "  plt.ylabel(metric_type)\n",
+        "  plt.legend()\n",
+        "  plt.title(f\"{metric_type} vs Training epochs\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DC-qIvZbHo0G"
+      },
+      "outputs": [],
+      "source": [
+        "plot_metrics(train_losses, val_losses, \"cross entropy loss\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "P-w2xk2PIDve"
+      },
+      "outputs": [],
+      "source": [
+        "plot_metrics(train_accs, val_accs, \"accuracy\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tbrJJaFrD_XR"
+      },
+      "source": [
+        "## Save and load the model\n",
+        "\n",
+        "Start by making an export module that takes in raw data and performs the following operations:\n",
+        "- Data preprocessing \n",
+        "- Probability prediction\n",
+        "- Class prediction"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1sszfWuJJZoo"
+      },
+      "outputs": [],
+      "source": [
+        "class ExportModule(tf.Module):\n",
+        "  def __init__(self, model, preprocess, class_pred):\n",
+        "    # Initialize pre and postprocessing functions\n",
+        "    self.model = model\n",
+        "    self.preprocess = preprocess\n",
+        "    self.class_pred = class_pred\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(shape=[None, None, None, None], dtype=tf.uint8)]) \n",
+        "  def __call__(self, x):\n",
+        "    # Run the ExportModule for new data points\n",
+        "    x = self.preprocess(x)\n",
+        "    y = self.model(x)\n",
+        "    y = self.class_pred(y)\n",
+        "    return y "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p8x6gjTDVi5d"
+      },
+      "outputs": [],
+      "source": [
+        "def preprocess_test(x):\n",
+        "  # The export module takes in unprocessed and unlabeled data\n",
+        "  x = tf.reshape(x, shape=[-1, 784])\n",
+        "  x = x/255\n",
+        "  return x\n",
+        "\n",
+        "def class_pred_test(y):\n",
+        "  # Generate class predictions from MLP output\n",
+        "  return tf.argmax(tf.nn.softmax(y), axis=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vu9H5STrJzdo"
+      },
+      "source": [
+        "This export module can now be saved with the `tf.saved_model.save` function. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fN9pPBQTKTe3"
+      },
+      "outputs": [],
+      "source": [
+        "mlp_model_export = ExportModule(model=mlp_model,\n",
+        "                                preprocess=preprocess_test,\n",
+        "                                class_pred=class_pred_test)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "idS7rQKbKwRS"
+      },
+      "outputs": [],
+      "source": [
+        "models = tempfile.mkdtemp()\n",
+        "save_path = os.path.join(models, 'mlp_model_export')\n",
+        "tf.saved_model.save(mlp_model_export, save_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_zZxO8iqBGZ-"
+      },
+      "source": [
+        "Load the saved model with `tf.saved_model.load` and examine its performance on the unseen test data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "W5cwBTUqxldW"
+      },
+      "outputs": [],
+      "source": [
+        "mlp_loaded = tf.saved_model.load(save_path)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bmv0u6j_b5OC"
+      },
+      "outputs": [],
+      "source": [
+        "def accuracy_score(y_pred, y):\n",
+        "  # Generic accuracy function\n",
+        "  is_equal = tf.equal(y_pred, y)\n",
+        "  return tf.reduce_mean(tf.cast(is_equal, tf.float32))\n",
+        "\n",
+        "x_test, y_test = tfds.load(\"mnist\", split=['test'], batch_size=-1, as_supervised=True)[0]\n",
+        "test_classes = mlp_loaded(x_test)\n",
+        "test_acc = accuracy_score(test_classes, y_test)\n",
+        "print(f\"Test Accuracy: {test_acc:.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "j5t9vgv_ciQ_"
+      },
+      "source": [
+        "The model does a great job of classifying handwritten digits in the training dataset and also generalizes well to unseen data. Now, examine the model's class-wise accuracy to ensure good performance for each digit. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UD8YiC1Vfeyp"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Accuracy breakdown by digit:\")\n",
+        "print(\"---------------------------\")\n",
+        "label_accs = {}\n",
+        "for label in range(10):\n",
+        "  label_ind = (y_test == label)\n",
+        "  # extract predictions for specific true label\n",
+        "  pred_label = test_classes[label_ind]\n",
+        "  labels = y_test[label_ind]\n",
+        "  # compute class-wise accuracy\n",
+        "  label_accs[accuracy_score(pred_label, labels).numpy()] = label\n",
+        "for key in sorted(label_accs):\n",
+        "  print(f\"Digit {label_accs[key]}: {key:.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rcykuJFhdGb0"
+      },
+      "source": [
+        "It looks like the model struggles with some digits a little more than others which is quite common in many multiclass classification problems. As a final exercise, plot a confusion matrix of the model's predictions and its corresponding true labels to gather more class-level insights. Sklearn and seaborn have functions for generating and visualizing confusion matrices. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JqCaqPwwh1tN"
+      },
+      "outputs": [],
+      "source": [
+        "import sklearn.metrics as sk_metrics\n",
+        "\n",
+        "def show_confusion_matrix(test_labels, test_classes):\n",
+        "  # Compute confusion matrix and normalize\n",
+        "  plt.figure(figsize=(10,10))\n",
+        "  confusion = sk_metrics.confusion_matrix(test_labels.numpy(), \n",
+        "                                          test_classes.numpy())\n",
+        "  confusion_normalized = confusion / confusion.sum(axis=1, keepdims=True)\n",
+        "  axis_labels = range(10)\n",
+        "  ax = sns.heatmap(\n",
+        "      confusion_normalized, xticklabels=axis_labels, yticklabels=axis_labels,\n",
+        "      cmap='Blues', annot=True, fmt='.4f', square=True)\n",
+        "  plt.title(\"Confusion matrix\")\n",
+        "  plt.ylabel(\"True label\")\n",
+        "  plt.xlabel(\"Predicted label\")\n",
+        "\n",
+        "show_confusion_matrix(y_test, test_classes)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JT-WA7GVda6d"
+      },
+      "source": [
+        "Class-level insights can help identify reasons for misclassifications and improve model performance in future training cycles."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VFLfEH4ManbW"
+      },
+      "source": [
+        "## Conclusion\n",
+        "\n",
+        "This notebook introduced a few techniques to handle a multiclass classification problem with an [MLP](https://developers.google.com/machine-learning/crash-course/multi-class-neural-networks/softmax). Here are a few more tips that may help:\n",
+        "\n",
+        "- The [TensorFlow Core APIs](https://www.tensorflow.org/guide/core) can be used to build machine learning workflows with high levels of configurability\n",
+        "- Initialization schemes can help prevent model parameters from vanishing or exploding during training.\n",
+        "- Overfitting is another common problem for neural networks, though it wasn't a problem for this tutorial. Visit the [Overfit and underfit](overfit_and_underfit.ipynb) tutorial for more help with this.\n",
+        "\n",
+        "For more examples of using the TensorFlow Core APIs, check out the [guide](https://www.tensorflow.org/guide/core). If you want to learn more about loading and preparing data, see the tutorials on [image data loading](https://www.tensorflow.org/tutorials/load_data/images) or [CSV data loading](https://www.tensorflow.org/tutorials/load_data/csv)."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "FhGuhbZ6M5tl"
+      ],
+      "name": "mlp_core.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/optimizers_core.ipynb b/site/en/guide/core/optimizers_core.ipynb
new file mode 100644
index 00000000000..e22f0327419
--- /dev/null
+++ b/site/en/guide/core/optimizers_core.ipynb
@@ -0,0 +1,612 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FhGuhbZ6M5tl"
+      },
+      "source": [
+        "##### Copyright 2022 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "AwOEIRJC6Une"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EIdT9iu_Z4Rb"
+      },
+      "source": [
+        "# Optimizers with Core APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bBIlTPscrIT9"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SjAxxRpBzVYg"
+      },
+      "source": [
+        "## Introduction\n",
+        "\n",
+        "This notebook introduces the process of creating custom optimizers with the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core). Visit the [Core APIs overview](https://www.tensorflow.org/guide/core) to learn more about TensorFlow Core and its intended use cases. \n",
+        "\n",
+        "The [Keras optimizers](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers) module is the recommended optimization toolkit for many general training purposes. It includes a variety of prebuilt optimiziers as well as subclassing functionality for customization. The Keras optimizers are also compatible with custom layers, models, and training loops built with the Core APIs. These prebuilt and customizable optimizers are suitable for most cases, but the Core APIs allow for complete control over the optimization process. For example, techniques such as Sharpness-Aware Minimization (SAM) require the model and optimizer to be coupled, which does not fit the traditional definition of ML optimizers. This guide walks through the process of building custom optimizers from scratch with the Core APIs, giving you the power to have full control over the structure, implementation, and behavior of your optimizers."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nBmqYyodNRd_"
+      },
+      "source": [
+        "## Optimizers overview\n",
+        "\n",
+        "An optimizer is an algorithm used to minimize a loss function with respect to a model's trainable parameters. The most straightforward optimization technique is gradient descent, which iteratively updates a model's parameters by taking a step in the direction of its loss function's steepest descent. Its step size is directly proportional to the size of the gradient, which can be problematic when the gradient is either too large or too small. There are many other gradient-based optimizers such as Adam, Adagrad, and RMSprop that leverage various mathematical properties of gradients for memory efficiency and fast convergence."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nchsZfwEVtVs"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d9idwpXCltUl"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib\n",
+        "from matplotlib import pyplot as plt\n",
+        "# Preset Matplotlib figure sizes.\n",
+        "matplotlib.rcParams['figure.figsize'] = [9, 6]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9xQKvCJ85kCQ"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "print(tf.__version__)\n",
+        "# set random seed for reproducible results \n",
+        "tf.random.set_seed(22)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0UmF5aU3MnwX"
+      },
+      "source": [
+        "## Gradient descent\n",
+        "\n",
+        "The basic optimizer class should have an initialization method and a function to update a list of variables given a list of gradients. Start by implementing the basic gradient descent optimizer which updates each variable by subtracting its gradient scaled by a learning rate."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MWjmUmeOQFFN"
+      },
+      "outputs": [],
+      "source": [
+        "class GradientDescent(tf.Module):\n",
+        "\n",
+        "  def __init__(self, learning_rate=1e-3):\n",
+        "    # Initialize parameters\n",
+        "    self.learning_rate = learning_rate\n",
+        "    self.title = f\"Gradient descent optimizer: learning rate={self.learning_rate}\"\n",
+        "\n",
+        "  def apply_gradients(self, grads, vars):\n",
+        "    # Update variables\n",
+        "    for grad, var in zip(grads, vars):\n",
+        "      var.assign_sub(self.learning_rate*grad)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZSekgBHDRzmp"
+      },
+      "source": [
+        "To test this optimizer, create a sample loss function to minimize with respect to a single variable, $x$. Compute its gradient function and solve for its minimizing parameter value:\n",
+        "\n",
+        "$$L = 2x^4 + 3x^3 + 2$$\n",
+        "\n",
+        "$$\\frac{dL}{dx} = 8x^3 + 9x^2$$\n",
+        "\n",
+        "$\\frac{dL}{dx}$ is 0 at $x = 0$, which is a saddle point and at $x = - \\frac{9}{8}$, which is the global minimum. Therefore, the loss function is optimized at $x^\\star = - \\frac{9}{8}$."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VCtJaUo6Ry8V"
+      },
+      "outputs": [],
+      "source": [
+        "x_vals = tf.linspace(-2, 2, 201)\n",
+        "x_vals = tf.cast(x_vals, tf.float32)\n",
+        "\n",
+        "def loss(x):\n",
+        "  return 2*(x**4) + 3*(x**3) + 2\n",
+        "\n",
+        "def grad(f, x):\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    tape.watch(x)\n",
+        "    result = f(x)\n",
+        "  return tape.gradient(result, x)\n",
+        "\n",
+        "plt.plot(x_vals, loss(x_vals), c='k', label = \"Loss function\")\n",
+        "plt.plot(x_vals, grad(loss, x_vals), c='tab:blue', label = \"Gradient function\")\n",
+        "plt.plot(0, loss(0),  marker=\"o\", c='g', label = \"Inflection point\")\n",
+        "plt.plot(-9/8, loss(-9/8),  marker=\"o\", c='r', label = \"Global minimum\")\n",
+        "plt.legend()\n",
+        "plt.ylim(0,5)\n",
+        "plt.xlabel(\"x\")\n",
+        "plt.ylabel(\"loss\")\n",
+        "plt.title(\"Sample loss function and gradient\");"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fLlIBJ9yuwhE"
+      },
+      "source": [
+        "Write a function to test the convergence of an optimizer with a single variable loss function. Assume that convergence has been achieved when the updated parameter's value at timestep $t$ is the same as its value held at timestep $t-1$. Terminate the test after a set number of iterations and also keep track of any exploding gradients during the process. In order to truly challenge the optimization algorithm, initialize the parameter poorly. In the above example, $x = 2$ is a good choice since it involves an steep gradient and also leads into an inflection point."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SLQTc41ouv0F"
+      },
+      "outputs": [],
+      "source": [
+        "def convergence_test(optimizer, loss_fn, grad_fn=grad, init_val=2., max_iters=2000):\n",
+        "  # Function for optimizer convergence test\n",
+        "  print(optimizer.title)\n",
+        "  print(\"-------------------------------\")\n",
+        "  # Initializing variables and structures\n",
+        "  x_star = tf.Variable(init_val)\n",
+        "  param_path = []\n",
+        "  converged = False\n",
+        "\n",
+        "  for iter in range(1, max_iters + 1):\n",
+        "    x_grad = grad_fn(loss_fn, x_star)\n",
+        "\n",
+        "    # Case for exploding gradient\n",
+        "    if tf.math.is_nan(x_grad):\n",
+        "      print(f\"Gradient exploded at iteration {iter}\\n\")\n",
+        "      return []\n",
+        "\n",
+        "    # Updating the variable and storing its old-version\n",
+        "    x_old = x_star.numpy()\n",
+        "    optimizer.apply_gradients([x_grad], [x_star])\n",
+        "    param_path.append(x_star.numpy())\n",
+        "\n",
+        "    # Checking for convergence\n",
+        "    if x_star == x_old:\n",
+        "      print(f\"Converged in {iter} iterations\\n\")\n",
+        "      converged = True\n",
+        "      break\n",
+        "      \n",
+        "  # Print early termination message\n",
+        "  if not converged:\n",
+        "    print(f\"Exceeded maximum of {max_iters} iterations. Test terminated.\\n\")\n",
+        "  return param_path"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vK-7_TsmyAgI"
+      },
+      "source": [
+        "Test the convergence of the gradient descent optimizer for the following learning rates: 1e-3, 1e-2, 1e-1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lWRn8c91mqB0"
+      },
+      "outputs": [],
+      "source": [
+        "param_map_gd = {}\n",
+        "learning_rates = [1e-3, 1e-2, 1e-1]\n",
+        "for learning_rate in learning_rates:\n",
+        "  param_map_gd[learning_rate] = (convergence_test(\n",
+        "      GradientDescent(learning_rate=learning_rate), loss_fn=loss))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TydrGHF5y6iI"
+      },
+      "source": [
+        "Visualize the path of the parameters over a contour plot of the loss function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "piffzGHI_u5G"
+      },
+      "outputs": [],
+      "source": [
+        "def viz_paths(param_map, x_vals, loss_fn, title, max_iters=2000):\n",
+        "  # Creating a controur plot of the loss function\n",
+        "  t_vals = tf.range(1., max_iters + 100.)\n",
+        "  t_grid, x_grid = tf.meshgrid(t_vals, x_vals)\n",
+        "  loss_grid = tf.math.log(loss_fn(x_grid))\n",
+        "  plt.pcolormesh(t_vals, x_vals, loss_grid, vmin=0, shading='nearest')\n",
+        "  colors = ['r', 'w', 'c']\n",
+        "  # Plotting the parameter paths over the contour plot\n",
+        "  for i, learning_rate in enumerate(param_map):\n",
+        "    param_path = param_map[learning_rate]\n",
+        "    if len(param_path) > 0:\n",
+        "      x_star = param_path[-1]\n",
+        "      plt.plot(t_vals[:len(param_path)], param_path, c=colors[i])\n",
+        "      plt.plot(len(param_path), x_star, marker='o', c=colors[i], \n",
+        "              label = f\"x*: learning rate={learning_rate}\")\n",
+        "  plt.xlabel(\"Iterations\")\n",
+        "  plt.ylabel(\"Parameter value\")\n",
+        "  plt.legend()\n",
+        "  plt.title(f\"{title} parameter paths\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ssyj2sO4BcNY"
+      },
+      "outputs": [],
+      "source": [
+        "viz_paths(param_map_gd, x_vals, loss, \"Gradient descent\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MmM-5eDLFnmC"
+      },
+      "source": [
+        "Gradient descent seems to get stuck at the inflection point when using smaller learning rates. Increasing the learning rate can encourage faster movement around the plateau region due to a larger step size; however, this comes at the risk of having exploding gradients in early iterations when the loss function is extremely steep."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "m5CDeXN8S1SF"
+      },
+      "source": [
+        "## Gradient descent with momentum\n",
+        "\n",
+        "Gradient descent with momentum not only uses the gradient to update a variable but also involves the change in position of a variable based on its previous update. The momentum parameter determines the level of influence the update at timestep $t-1$ has on the update at timestep $t$. Accumulating momentum helps to move variables past plataeu regions faster than basic gradient descent. The momentum update rule is as follows:\n",
+        "\n",
+        "$$\\Delta_x^{[t]} = lr \\cdot L^\\prime(x^{[t-1]}) + p \\cdot \\Delta_x^{[t-1]}$$\n",
+        "\n",
+        "$$x^{[t]} = x^{[t-1]} - \\Delta_x^{[t]}$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $x$: the variable being optimized\n",
+        "* $\\Delta_x$: change in $x$ \n",
+        "* $lr$: learning rate\n",
+        "* $L^\\prime(x)$: gradient of the loss function with respect to x\n",
+        "* $p$: momentum parameter"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rOBY8Tz4S0dX"
+      },
+      "outputs": [],
+      "source": [
+        "class Momentum(tf.Module):\n",
+        "\n",
+        "  def __init__(self, learning_rate=1e-3, momentum=0.7):\n",
+        "    # Initialize parameters\n",
+        "    self.learning_rate = learning_rate\n",
+        "    self.momentum = momentum\n",
+        "    self.change = 0.\n",
+        "    self.title = f\"Gradient descent optimizer: learning rate={self.learning_rate}\"\n",
+        "\n",
+        "  def apply_gradients(self, grads, vars):\n",
+        "    # Update variables \n",
+        "    for grad, var in zip(grads, vars):\n",
+        "      curr_change = self.learning_rate*grad + self.momentum*self.change\n",
+        "      var.assign_sub(curr_change)\n",
+        "      self.change = curr_change"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "t_nDu38gW6Fu"
+      },
+      "source": [
+        "Test the convergence of the momentum optimizer for the following learning rates: 1e-3, 1e-2, 1e-1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tA6oQL-sW2xg"
+      },
+      "outputs": [],
+      "source": [
+        "param_map_mtm = {}\n",
+        "learning_rates = [1e-3, 1e-2, 1e-1]\n",
+        "for learning_rate in learning_rates:\n",
+        "  param_map_mtm[learning_rate] = (convergence_test(\n",
+        "      Momentum(learning_rate=learning_rate),\n",
+        "      loss_fn=loss, grad_fn=grad))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wz_LV0EPYE6k"
+      },
+      "source": [
+        "Visualize the path of the parameters over a contour plot of the loss function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qbW1eEKaX3T9"
+      },
+      "outputs": [],
+      "source": [
+        "viz_paths(param_map_mtm, x_vals, loss, \"Momentum\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4bEFnhPRTBXh"
+      },
+      "source": [
+        "## Adaptive moment estimation (Adam)\n",
+        "\n",
+        "The Adaptive Moment Estimation (Adam) algorithm is an efficient and highly generalizable optimization technique that leverages two key gradient descent methedologies: momentum, and root mean square propogation (RMSP). Momentum helps accelerate gradient descent by using the first moment (sum of gradients) along with a decay parameter. RMSP is similar; however, it leverages the second moment (sum of gradients squared). \n",
+        "\n",
+        "The Adam algorithm combines both the first and second moment to provide a more generalizable update rule. The sign of a variable, $x$, can be determined by computing $\\frac{x}{\\sqrt{x^2}}$. The Adam optimizer uses this fact to calculate an update step which is effectively a smoothed sign. Instead of calculating $\\frac{x}{\\sqrt{x^2}}$, the optimizer calculates a smoothed version of $x$ (first moment) and $x^2$ (second moment) for each variable update. \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WjgyqRiZ7XhA"
+      },
+      "source": [
+        "**Adam algorithm**\n",
+        "\n",
+        "$\\beta_1 \\gets 0.9 \\; \\triangleright \\text{literature value}$\n",
+        "\n",
+        "$\\beta_2 \\gets 0.999 \\; \\triangleright \\text{literature value}$\n",
+        "\n",
+        "$lr \\gets \\text{1e-3} \\; \\triangleright \\text{configurable learning rate}$\n",
+        "\n",
+        "$\\epsilon \\gets \\text{1e-7} \\; \\triangleright \\text{prevents divide by 0 error}$\n",
+        "\n",
+        "$V_{dv} \\gets \\vec {\\underset{n\\times1}{0}} \\;\\triangleright \\text{stores momentum updates for each variable}$\n",
+        "\n",
+        "$S_{dv} \\gets \\vec {\\underset{n\\times1}{0}} \\; \\triangleright \\text{stores RMSP updates for each variable}$\n",
+        "\n",
+        "$t \\gets 1$\n",
+        "\n",
+        "$\\text{On iteration } t:$\n",
+        "\n",
+        "$\\;\\;\\;\\; \\text{For} (\\frac{dL}{dv}, v) \\text{ in gradient variable pairs}:$\n",
+        "\n",
+        "$\\;\\;\\;\\;\\;\\;\\;\\; V_{dv\\_i} = \\beta_1V_{dv\\_i} + (1 - \\beta_1)\\frac{dL}{dv} \\; \\triangleright \\text{momentum update}$\n",
+        "\n",
+        "$\\;\\;\\;\\;\\;\\;\\;\\; S_{dv\\_i} = \\beta_2V_{dv\\_i} + (1 - \\beta_2)(\\frac{dL}{dv})^2 \\; \\triangleright \\text{RMSP update}$\n",
+        "\n",
+        "$\\;\\;\\;\\;\\;\\;\\;\\; v_{dv}^{bc} = \\frac{V_{dv\\_i}}{(1-\\beta_1)^t} \\; \\triangleright \\text{momentum bias correction}$\n",
+        "\n",
+        "$\\;\\;\\;\\;\\;\\;\\;\\; s_{dv}^{bc} = \\frac{S_{dv\\_i}}{(1-\\beta_2)^t} \\; \\triangleright \\text{RMSP bias correction}$\n",
+        "\n",
+        "$\\;\\;\\;\\;\\;\\;\\;\\; v = v - lr\\frac{v_{dv}^{bc}}{\\sqrt{s_{dv}^{bc}} + \\epsilon} \\; \\triangleright \\text{parameter update}$\n",
+        "\n",
+        "$\\;\\;\\;\\;\\;\\;\\;\\; t = t + 1$\n",
+        "\n",
+        "**End of algorithm**\n",
+        "\n",
+        "Given that $V_{dv}$ and $S_{dv}$ are initialized to 0 and that $\\beta_1$ and $\\beta_2$ are close to 1, the momentum and RMSP updates are naturally biased towards 0; therefore, the variables can benefit from bias correction. Bias correction also helps to control the osccilation of weights as they approach the global minimum."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hm5vffRJRsEc"
+      },
+      "outputs": [],
+      "source": [
+        "class Adam(tf.Module):\n",
+        "  \n",
+        "    def __init__(self, learning_rate=1e-3, beta_1=0.9, beta_2=0.999, ep=1e-7):\n",
+        "      # Initialize the Adam parameters\n",
+        "      self.beta_1 = beta_1\n",
+        "      self.beta_2 = beta_2\n",
+        "      self.learning_rate = learning_rate\n",
+        "      self.ep = ep\n",
+        "      self.t = 1.\n",
+        "      self.v_dvar, self.s_dvar = [], []\n",
+        "      self.title = f\"Adam: learning rate={self.learning_rate}\"\n",
+        "      self.built = False\n",
+        "\n",
+        "    def apply_gradients(self, grads, vars):\n",
+        "      # Set up moment and RMSprop slots for each variable on the first call\n",
+        "      if not self.built:\n",
+        "        for var in vars:\n",
+        "          v = tf.Variable(tf.zeros(shape=var.shape))\n",
+        "          s = tf.Variable(tf.zeros(shape=var.shape))\n",
+        "          self.v_dvar.append(v)\n",
+        "          self.s_dvar.append(s)\n",
+        "        self.built = True\n",
+        "      # Perform Adam updates\n",
+        "      for i, (d_var, var) in enumerate(zip(grads, vars)):\n",
+        "        # Moment calculation\n",
+        "        self.v_dvar[i] = self.beta_1*self.v_dvar[i] + (1-self.beta_1)*d_var\n",
+        "        # RMSprop calculation\n",
+        "        self.s_dvar[i] = self.beta_2*self.s_dvar[i] + (1-self.beta_2)*tf.square(d_var)\n",
+        "        # Bias correction\n",
+        "        v_dvar_bc = self.v_dvar[i]/(1-(self.beta_1**self.t))\n",
+        "        s_dvar_bc = self.s_dvar[i]/(1-(self.beta_2**self.t))\n",
+        "        # Update model variables\n",
+        "        var.assign_sub(self.learning_rate*(v_dvar_bc/(tf.sqrt(s_dvar_bc) + self.ep)))\n",
+        "      # Increment the iteration counter\n",
+        "      self.t += 1."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UWN4Qus7flUO"
+      },
+      "source": [
+        "Test the performance of the Adam optimizer with the same learning rates used with the gradient descent examples. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GXHCxtemFBpR"
+      },
+      "outputs": [],
+      "source": [
+        "param_map_adam = {}\n",
+        "learning_rates = [1e-3, 1e-2, 1e-1]\n",
+        "for learning_rate in learning_rates:\n",
+        "  param_map_adam[learning_rate] = (convergence_test(\n",
+        "      Adam(learning_rate=learning_rate), loss_fn=loss))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jgpUcs_xXEjX"
+      },
+      "source": [
+        "Visualize the path of the parameters over a contour plot of the loss function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ctvOUmlzFK8s"
+      },
+      "outputs": [],
+      "source": [
+        "viz_paths(param_map_adam, x_vals, loss, \"Adam\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_oGScF8zJcY4"
+      },
+      "source": [
+        "In this particular example, the Adam optimizer has slower convergence compared to traditional gradient descent when using small learning rates. However, the algorithm successfully moves past the plataeu region and converges to the global minimum when a larger learning rate. Exploding gradients are no longer an issue due to Adam's dynamic scaling of learning rates when encountering large gradients."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VFLfEH4ManbW"
+      },
+      "source": [
+        "## Conclusion\n",
+        "\n",
+        "This notebook introduced the basics of writing and comparing optimizers with the [TensorFlow Core APIs](https://www.tensorflow.org/guide/core). Although prebuilt optimizers like Adam are generalizable, they may not always be the best choice for every model or dataset. Having fine-grained control over the optimization process can help streamline ML training workflows and improve overall performance. Refer to the following documentation for more examples of custom optimizers:\n",
+        "\n",
+        "*   This Adam optimizer is used in the [Multilayer perceptrons](https://www.tensorflow.org/guide/core/mlp_core) tutorial and the [Distributed training]()\n",
+        "*   [Model Garden](https://blog.tensorflow.org/2020/03/introducing-model-garden-for-tensorflow-2.html) has a variety of [custom optimizers](https://github.com/tensorflow/models/tree/master/official/modeling/optimization) written with the Core APIs.\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "optimizers_core.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/core/quickstart_core.ipynb b/site/en/guide/core/quickstart_core.ipynb
new file mode 100644
index 00000000000..70586fd3f0c
--- /dev/null
+++ b/site/en/guide/core/quickstart_core.ipynb
@@ -0,0 +1,591 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rX8mhOLljYeM"
+      },
+      "source": [
+        "##### Copyright 2022 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "BZSlp3DAjdYf"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3wF5wszaj97Y"
+      },
+      "source": [
+        "# Quickstart for the TensorFlow Core APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DUNzJc4jTj6G"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "04QgGZc9bF5D"
+      },
+      "source": [
+        "This quickstart tutorial demonstrates how you can use the [TensorFlow Core low-level APIs](https://www.tensorflow.org/guide/core) to build and train a multiple linear regression model that predicts fuel efficiency. It uses the [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) dataset which contains fuel efficiency data for late-1970s and early 1980s automobiles.\n",
+        "\n",
+        "You will follow the typical stages of a machine learning process:\n",
+        "\n",
+        "1. Load the dataset.\n",
+        "2. Build an [input pipeline](../data.ipynb).\n",
+        "3. Build a multiple [linear regression](https://developers.google.com/machine-learning/glossary#linear-regression) model.\n",
+        "4. Evaluate the performance of the model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nnrWf3PCEzXL"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Import TensorFlow and other necessary libraries to get started:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0trJmd6DjqBZ"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import pandas as pd\n",
+        "import matplotlib\n",
+        "from matplotlib import pyplot as plt\n",
+        "print(\"TensorFlow version:\", tf.__version__)\n",
+        "# Set a random seed for reproducible results \n",
+        "tf.random.set_seed(22)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7NAbSZiaoJ4z"
+      },
+      "source": [
+        "## Load and preprocess the dataset\n",
+        "\n",
+        "Next, you need to load and preprocess the [Auto MPG dataset](https://archive.ics.uci.edu/ml/datasets/auto+mpg) from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/). This dataset uses a variety of quantitative and categorical features such as cylinders, displacement, horsepower and weight to predict the fuel efficiencies of automobiles in the late-1970s and early 1980s.\n",
+        "\n",
+        "The dataset contains a few unknown values. Make sure to drop any missing values with `pandas.DataFrame.dropna`, and convert the dataset to a `tf.float32` tensor type with the `tf.convert_to_tensor` and `tf.cast` functions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HglhDsUfrJ98"
+      },
+      "outputs": [],
+      "source": [
+        "url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data'\n",
+        "column_names = ['MPG', 'Cylinders', 'Displacement', 'Horsepower', 'Weight',\n",
+        "                'Acceleration', 'Model Year', 'Origin']\n",
+        "\n",
+        "dataset = pd.read_csv(url, names=column_names, na_values='?', comment='\\t',\n",
+        "                          sep=' ', skipinitialspace=True)\n",
+        "\n",
+        "dataset = dataset.dropna()\n",
+        "dataset_tf = tf.convert_to_tensor(dataset, dtype=tf.float32)\n",
+        "dataset.tail()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0vgoDL3hYesB"
+      },
+      "source": [
+        "Next, split the dataset into training and test sets. Make sure to shuffle the dataset with `tf.random.shuffle` to avoid biased splits."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0mJU4kt6YiAp"
+      },
+      "outputs": [],
+      "source": [
+        "dataset_shuffled = tf.random.shuffle(dataset_tf, seed=22)\n",
+        "train_data, test_data = dataset_shuffled[100:], dataset_shuffled[:100]\n",
+        "x_train, y_train = train_data[:, 1:], train_data[:, 0]\n",
+        "x_test, y_test = test_data[:, 1:], test_data[:, 0]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Bscb2Vsbi3TE"
+      },
+      "source": [
+        "Perform basic feature engineering by one-hot-encoding the `\"Origin\"` feature. The `tf.one_hot` function is useful for transforming this categorical column into 3 separate binary columns."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_B8N9IV1i6IV"
+      },
+      "outputs": [],
+      "source": [
+        "def onehot_origin(x):\n",
+        "  origin = tf.cast(x[:, -1], tf.int32)\n",
+        "  # Use `origin - 1` to account for 1-indexed feature\n",
+        "  origin_oh = tf.one_hot(origin - 1, 3)\n",
+        "  x_ohe = tf.concat([x[:, :-1], origin_oh], axis = 1)\n",
+        "  return x_ohe\n",
+        "\n",
+        "x_train_ohe, x_test_ohe = onehot_origin(x_train), onehot_origin(x_test)\n",
+        "x_train_ohe.numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qnoCDzzedite"
+      },
+      "source": [
+        "This example shows a multiple regression problem with predictors or features on vastly different scales. Therefore, it is beneficial to standardize the data so that each feature has zero mean and unit variance. Use the `tf.reduce_mean` and `tf.math.reduce_std` functions for standardization. The regression model's prediction can then be unstandardized to obtain its value in terms of the original units."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dJJFdvqydhyp"
+      },
+      "outputs": [],
+      "source": [
+        "class Normalize(tf.Module):\n",
+        "  def __init__(self, x):\n",
+        "    # Initialize the mean and standard deviation for normalization\n",
+        "    self.mean = tf.math.reduce_mean(x, axis=0)\n",
+        "    self.std = tf.math.reduce_std(x, axis=0)\n",
+        "\n",
+        "  def norm(self, x):\n",
+        "    # Normalize the input\n",
+        "    return (x - self.mean)/self.std\n",
+        "\n",
+        "  def unnorm(self, x):\n",
+        "    # Unnormalize the input\n",
+        "    return (x * self.std) + self.mean"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5BONV6fYYwZb"
+      },
+      "outputs": [],
+      "source": [
+        "norm_x = Normalize(x_train_ohe)\n",
+        "norm_y = Normalize(y_train)\n",
+        "x_train_norm, y_train_norm = norm_x.norm(x_train_ohe), norm_y.norm(y_train)\n",
+        "x_test_norm, y_test_norm = norm_x.norm(x_test_ohe), norm_y.norm(y_test)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BPZ68wASog_I"
+      },
+      "source": [
+        "## Build a machine learning model\n",
+        "\n",
+        "Build a linear regression model with the TensorFlow Core APIs. The equation for multiple linear regression is as follows:\n",
+        "\n",
+        "$${\\mathrm{Y}} = {\\mathrm{X}}w + b$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $\\underset{m\\times 1}{\\mathrm{Y}}$: target vector\n",
+        "* $\\underset{m\\times n}{\\mathrm{X}}$: feature matrix\n",
+        "* $\\underset{n\\times 1}w$: weight vector\n",
+        "* $b$: bias\n",
+        "\n",
+        "By using the `@tf.function` decorator, the corresponding Python code is traced to generate a callable TensorFlow graph. This approach is beneficial for saving and loading the model after training. It can also provide a performance boost for models with many layers and complex operations. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "h3IKyzTCDNGo"
+      },
+      "outputs": [],
+      "source": [
+        "class LinearRegression(tf.Module):\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    self.built = False\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self, x):\n",
+        "    # Initialize the model parameters on the first call\n",
+        "    if not self.built:\n",
+        "      # Randomly generate the weight vector and bias term\n",
+        "      rand_w = tf.random.uniform(shape=[x.shape[-1], 1])\n",
+        "      rand_b = tf.random.uniform(shape=[])\n",
+        "      self.w = tf.Variable(rand_w)\n",
+        "      self.b = tf.Variable(rand_b)\n",
+        "      self.built = True\n",
+        "    y = tf.add(tf.matmul(x, self.w), self.b)\n",
+        "    return tf.squeeze(y, axis=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l2hiez2eIUz8"
+      },
+      "source": [
+        "For each example, the model returns a prediction for the input automobile's MPG by computing the weighted sum of its features plus a bias term. This prediction can then be unstandardized to obtain its value in terms of the original units."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OeOrNdnkEEcR"
+      },
+      "outputs": [],
+      "source": [
+        "lin_reg = LinearRegression()\n",
+        "prediction = lin_reg(x_train_norm[:1])\n",
+        "prediction_unnorm = norm_y.unnorm(prediction)\n",
+        "prediction_unnorm.numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FIHANxNSvWr9"
+      },
+      "source": [
+        "## Define a loss function\n",
+        "\n",
+        "Now, define a loss function to evaluate the model's performance during the training process.\n",
+        "\n",
+        "Since regression problems deal with continuous outputs, the mean squared error (MSE) is an ideal choice for the loss function. The MSE is defined by the following equation:\n",
+        "\n",
+        "$$MSE = \\frac{1}{m}\\sum_{i=1}^{m}(\\hat{y}_i -y_i)^2$$\n",
+        "\n",
+        "where\n",
+        "\n",
+        "* $\\hat{y}$: vector of predictions\n",
+        "* $y$: vector of true targets\n",
+        "\n",
+        "The goal of this regression problem is to find the optimal weight vector, $w$, and bias, $b$, that minimizes the MSE loss function. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8tYNVUkmw35s"
+      },
+      "outputs": [],
+      "source": [
+        "def mse_loss(y_pred, y):\n",
+        "  return tf.reduce_mean(tf.square(y_pred - y))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "htI-7aJPqclK"
+      },
+      "source": [
+        "## Train and evaluate your model\n",
+        "\n",
+        "Using mini-batches for training provides both memory efficiency and faster convergence. The `tf.data.Dataset` API has useful functions for batching and shuffling. The API enables you to build complex input pipelines from simple, reusable pieces. Learn more about building TensorFlow input pipelines in [this guide](https://www.tensorflow.org/guide/data)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kxST2w_Nq0C5"
+      },
+      "outputs": [],
+      "source": [
+        "batch_size = 64\n",
+        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train_norm, y_train_norm))\n",
+        "train_dataset = train_dataset.shuffle(buffer_size=x_train.shape[0]).batch(batch_size)\n",
+        "test_dataset = tf.data.Dataset.from_tensor_slices((x_test_norm, y_test_norm))\n",
+        "test_dataset = test_dataset.shuffle(buffer_size=x_test.shape[0]).batch(batch_size)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "C9haUW8Yq3xD"
+      },
+      "source": [
+        "Next, write a training loop to iteratively update your model's parameters by making use of the MSE loss function and its gradients with respect to the input parameters.\n",
+        "\n",
+        "This iterative method is referred to as [gradient descent](https://developers.google.com/machine-learning/glossary#gradient-descent). At each iteration, the model's parameters are updated by taking a step in the opposite direction of their computed gradients. The size of this step is determined by the learning rate, which is a configurable hyperparameter. Recall that the gradient of a function indicates the direction of its steepest ascent; therefore, taking a step in the opposite direction indicates the direction of steepest descent, which ultimately helps to minimize the MSE loss function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "y7suUbJXVLqP"
+      },
+      "outputs": [],
+      "source": [
+        "# Set training parameters\n",
+        "epochs = 100\n",
+        "learning_rate = 0.01\n",
+        "train_losses, test_losses = [], []\n",
+        "\n",
+        "# Format training loop\n",
+        "for epoch in range(epochs):\n",
+        "  batch_losses_train, batch_losses_test = [], []\n",
+        "\n",
+        "  # Iterate through the training data\n",
+        "  for x_batch, y_batch in train_dataset:\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      y_pred_batch = lin_reg(x_batch)\n",
+        "      batch_loss = mse_loss(y_pred_batch, y_batch)\n",
+        "    # Update parameters with respect to the gradient calculations\n",
+        "    grads = tape.gradient(batch_loss, lin_reg.variables)\n",
+        "    for g,v in zip(grads, lin_reg.variables):\n",
+        "      v.assign_sub(learning_rate * g)\n",
+        "    # Keep track of batch-level training performance \n",
+        "    batch_losses_train.append(batch_loss)\n",
+        "  \n",
+        "  # Iterate through the testing data\n",
+        "  for x_batch, y_batch in test_dataset:\n",
+        "    y_pred_batch = lin_reg(x_batch)\n",
+        "    batch_loss = mse_loss(y_pred_batch, y_batch)\n",
+        "    # Keep track of batch-level testing performance \n",
+        "    batch_losses_test.append(batch_loss)\n",
+        "\n",
+        "  # Keep track of epoch-level model performance\n",
+        "  train_loss = tf.reduce_mean(batch_losses_train)\n",
+        "  test_loss = tf.reduce_mean(batch_losses_test)\n",
+        "  train_losses.append(train_loss)\n",
+        "  test_losses.append(test_loss)\n",
+        "  if epoch % 10 == 0:\n",
+        "    print(f'Mean squared error for step {epoch}: {train_loss.numpy():0.3f}')\n",
+        "\n",
+        "# Output final losses\n",
+        "print(f\"\\nFinal train loss: {train_loss:0.3f}\")\n",
+        "print(f\"Final test loss: {test_loss:0.3f}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4mDAAPFqVVgn"
+      },
+      "source": [
+        "Plot the changes in MSE loss over time. Calculating performance metrics on a designated [validation set](https://developers.google.com/machine-learning/glossary#validation-set) or [test set](https://developers.google.com/machine-learning/glossary#test-set) ensures the model does not overfit to the training dataset and can generalize well to unseen data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "F7dTAzgHDUh7"
+      },
+      "outputs": [],
+      "source": [
+        "matplotlib.rcParams['figure.figsize'] = [9, 6]\n",
+        "\n",
+        "plt.plot(range(epochs), train_losses, label = \"Training loss\")\n",
+        "plt.plot(range(epochs), test_losses, label = \"Testing loss\")\n",
+        "plt.xlabel(\"Epoch\")\n",
+        "plt.ylabel(\"Mean squared error loss\")\n",
+        "plt.legend()\n",
+        "plt.title(\"MSE loss vs training iterations\");"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Aj8NrlzlJqDG"
+      },
+      "source": [
+        "It seems like the model does a good job of fitting the training data while also generalizing well to the unseen test data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AUNIPubuPYDR"
+      },
+      "source": [
+        "## Save and load the model\n",
+        "\n",
+        "Start by making an export module that takes in raw data and performs the following operations:\n",
+        "- Feature extraction \n",
+        "- Normalization \n",
+        "- Prediction\n",
+        "- Unnormalization"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "g-uOrGa9ZehG"
+      },
+      "outputs": [],
+      "source": [
+        "class ExportModule(tf.Module):\n",
+        "  def __init__(self, model, extract_features, norm_x, norm_y):\n",
+        "    # Initialize pre and postprocessing functions\n",
+        "    self.model = model\n",
+        "    self.extract_features = extract_features\n",
+        "    self.norm_x = norm_x\n",
+        "    self.norm_y = norm_y\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(shape=[None, None], dtype=tf.float32)]) \n",
+        "  def __call__(self, x):\n",
+        "    # Run the ExportModule for new data points\n",
+        "    x = self.extract_features(x)\n",
+        "    x = self.norm_x.norm(x)\n",
+        "    y = self.model(x)\n",
+        "    y = self.norm_y.unnorm(y)\n",
+        "    return y "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YPYYLQ8EZiU8"
+      },
+      "outputs": [],
+      "source": [
+        "lin_reg_export = ExportModule(model=lin_reg,\n",
+        "                              extract_features=onehot_origin,\n",
+        "                              norm_x=norm_x,\n",
+        "                              norm_y=norm_y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6v8xi06XZWiC"
+      },
+      "source": [
+        "If you want to save the model at its current state, use the `tf.saved_model.save` function. To load a saved model for making predictions, use the `tf.saved_model.load` function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "K1IvMoHbptht"
+      },
+      "outputs": [],
+      "source": [
+        "import tempfile\n",
+        "import os\n",
+        "\n",
+        "models = tempfile.mkdtemp()\n",
+        "save_path = os.path.join(models, 'lin_reg_export')\n",
+        "tf.saved_model.save(lin_reg_export, save_path)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rYb6DrEH0GMv"
+      },
+      "outputs": [],
+      "source": [
+        "lin_reg_loaded = tf.saved_model.load(save_path)\n",
+        "test_preds = lin_reg_loaded(x_test)\n",
+        "test_preds[:10].numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-47O6_GLdRuT"
+      },
+      "source": [
+        "## Conclusion\n",
+        "\n",
+        "Congratulations! You have trained a regression model using the TensorFlow Core low-level APIs.\n",
+        "\n",
+        "For more examples of using TensorFlow Core APIs, check out the following guides:\n",
+        "* [Logistic regression](./logistic_regression_core.ipynb) for binary classification\n",
+        "* [Multi-layer perceptrons](./mlp_core.ipynb) for hand-written digit recognition\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "rX8mhOLljYeM"
+      ],
+      "name": "quickstart_core.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/create_op.md b/site/en/guide/create_op.md
index 9b8de0fb6ef..fa4f573fa32 100644
--- a/site/en/guide/create_op.md
+++ b/site/en/guide/create_op.md
@@ -47,7 +47,7 @@ To incorporate your custom op you'll need to:
     test the op in C++. If you define gradients, you can verify them with the
     Python `tf.test.compute_gradient_error`.
     See
-    [`relu_op_test.py`](https://www.tensorflow.org/code/tensorflow/python/kernel_tests/relu_op_test.py) as
+    [`relu_op_test.py`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/kernel_tests/nn_ops/relu_op_test.py) as
     an example that tests the forward functions of Relu-like operators and
     their gradients.
 
@@ -55,8 +55,8 @@ To incorporate your custom op you'll need to:
 
 *   Some familiarity with C++.
 *   Must have installed the
-    [TensorFlow binary](../../install), or must have
-    [downloaded TensorFlow source](../../install/source.md),
+    [TensorFlow binary](https://www.tensorflow.org/install), or must have
+    [downloaded TensorFlow source](https://www.tensorflow.org/install/source),
     and be able to build it.
 
 ## Define the op interface
@@ -92,9 +92,8 @@ to ensure that the output tensor is the same shape as the input tensor. For
 example, if the input is a tensor of shape [10, 20], then this shape function
 specifies that the output shape is also [10, 20].
 
-
->   A note on naming: The op name must be in CamelCase and it must be unique
->   among all other ops that are registered in the binary.
+Note: The op name must be in CamelCase and it must be unique among all other ops
+that are registered in the binary.
 
 ## Implement the kernel for the op
 
@@ -153,17 +152,17 @@ REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp);
 >   Important: Instances of your OpKernel may be accessed concurrently.
 >   Your `Compute` method must be thread-safe. Guard any access to class
 >   members with a mutex. Or better yet, don't share state via class members!
->   Consider using a [`ResourceMgr`](https://www.tensorflow.org/code/tensorflow/core/framework/resource_mgr.h)
+>   Consider using a [`ResourceMgr`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/resource_mgr.h)
 >   to keep track of op state.
 
 ### Multi-threaded CPU kernels
 
 To write a multi-threaded CPU kernel, the Shard function in
-[`work_sharder.h`](https://www.tensorflow.org/code/tensorflow/core/util/work_sharder.h)
+[`work_sharder.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/work_sharder.h)
 can be used. This function shards a computation function across the
 threads configured to be used for intra-op threading (see
 intra_op_parallelism_threads in
-[`config.proto`](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto)).
+[`config.proto`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto)).
 
 ### GPU kernels
 
@@ -189,6 +188,8 @@ Here is an example implementation.
 #ifndef KERNEL_EXAMPLE_H_
 #define KERNEL_EXAMPLE_H_
 
+#include 
+
 template 
 struct ExampleFunctor {
   void operator()(const Device& d, int size, const T* in, T* out);
@@ -196,8 +197,8 @@ struct ExampleFunctor {
 
 #if GOOGLE_CUDA
 // Partially specialize functor for GpuDevice.
-template 
-struct ExampleFunctor {
+template 
+struct ExampleFunctor {
   void operator()(const Eigen::GpuDevice& d, int size, const T* in, T* out);
 };
 #endif
@@ -208,6 +209,9 @@ struct ExampleFunctor {
 ```c++
 // kernel_example.cc
 #include "kernel_example.h"
+
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
 #include "tensorflow/core/framework/op_kernel.h"
 
 using namespace tensorflow;
@@ -215,6 +219,15 @@ using namespace tensorflow;
 using CPUDevice = Eigen::ThreadPoolDevice;
 using GPUDevice = Eigen::GpuDevice;
 
+REGISTER_OP("Example")
+    .Attr("T: numbertype")
+    .Input("input: T")
+    .Output("input_times_two: T")
+    .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) {
+      c->set_output(0, c->input(0));
+      return Status::OK();
+    });
+
 // CPU specialization of actual computation.
 template 
 struct ExampleFunctor {
@@ -264,7 +277,7 @@ REGISTER_CPU(int32);
 #ifdef GOOGLE_CUDA
 #define REGISTER_GPU(T)                                          \
   /* Declare explicit instantiations in kernel_example.cu.cc. */ \
-  extern template ExampleFunctor;                  \
+  extern template class ExampleFunctor;            \
   REGISTER_KERNEL_BUILDER(                                       \
       Name("Example").Device(DEVICE_GPU).TypeConstraint("T"), \
       ExampleOp);
@@ -277,7 +290,7 @@ REGISTER_GPU(int32);
 // kernel_example.cu.cc
 #ifdef GOOGLE_CUDA
 #define EIGEN_USE_GPU
-#include "example.h"
+#include "kernel_example.h"
 #include "tensorflow/core/util/gpu_kernel_helper.h"
 
 using namespace tensorflow;
@@ -289,7 +302,7 @@ template 
 __global__ void ExampleCudaKernel(const int size, const T* in, T* out) {
   for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < size;
        i += blockDim.x * gridDim.x) {
-    out[i] = 2 * ldg(in + i);
+    out[i] = 2 * __ldg(in + i);
   }
 }
 
@@ -330,9 +343,9 @@ Here are the outputs of these functions on an Ubuntu machine.
 $ python
 >>> import tensorflow as tf
 >>> tf.sysconfig.get_include()
-'/usr/local/lib/python2.7/site-packages/tensorflow/include'
+'/usr/local/lib/python3.6/site-packages/tensorflow/include'
 >>> tf.sysconfig.get_lib()
-'/usr/local/lib/python2.7/site-packages/tensorflow'
+'/usr/local/lib/python3.6/site-packages/tensorflow'
 ```
 
 Assuming you have `g++` installed, here is the sequence of commands you can use
@@ -341,18 +354,19 @@ to compile your op into a dynamic library.
 ```bash
 TF_CFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )
 TF_LFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') )
-g++ -std=c++11 -shared zero_out.cc -o zero_out.so -fPIC ${TF_CFLAGS[@]} ${TF_LFLAGS[@]} -O2
+g++ -std=c++14 -shared zero_out.cc -o zero_out.so -fPIC ${TF_CFLAGS[@]} ${TF_LFLAGS[@]} -O2
 ```
 
-On Mac OS X, the additional flag "-undefined dynamic_lookup" is required when
+On macOS, the additional flag "-undefined dynamic_lookup" is required when
 building the `.so` file.
 
->   Note on `gcc` version `>=5`: gcc uses the new C++
->   [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`. The binary pip
->   packages available on the TensorFlow website are built with `gcc4` that uses
->   the older ABI. If you compile your op library with `gcc>=5`, add
->   `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to make the library
->   compatible with the older abi.
+> Note on `gcc` version `>=5`: gcc uses the new C++
+> [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`.
+> TensorFlow 2.8 and earlier were built with `gcc4` that uses the older ABI. If
+> you are using these versions of TensorFlow and are trying to compile your op
+> library with `gcc>=5`, add `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to
+> make the library compatible with the older ABI. TensorFlow 2.9+ packages are
+> compatible with the newer ABI by default.
 
 ### Compile the op using bazel (TensorFlow source installation)
 
@@ -374,8 +388,30 @@ Run the following command to build `zero_out.so`.
 ```bash
 $ bazel build --config opt //tensorflow/core/user_ops:zero_out.so
 ```
->   As explained above, if you are compiling with gcc>=5 add `--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"`
->   to the bazel command line.
+
+For compiling the `Example` operation, with the CUDA Kernel, you need to use the `gpu_srcs` parameter
+of `tf_custom_op_library`. Place a BUILD file with the following Bazel build rule in a new folder
+inside the [`tensorflow/core/user_ops`][user_ops] directory (e.g. "example_gpu").
+
+```python
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_library")
+
+tf_custom_op_library(
+    # kernel_example.cc  kernel_example.cu.cc  kernel_example.h
+    name = "kernel_example.so",
+    srcs = ["kernel_example.h", "kernel_example.cc"],
+    gpu_srcs = ["kernel_example.cu.cc", "kernel_example.h"],
+)
+```
+
+Run the following command to build `kernel_example.so`.
+
+```bash
+$ bazel build --config opt //tensorflow/core/user_ops/example_gpu:kernel_example.so
+```
+
+Note: As explained above, if you are compiling with gcc>=5 add
+`--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"` to the Bazel command line arguments.
 
 >   Note: Although you can create a shared library (a `.so` file) with the
 >   standard `cc_library` rule, we strongly recommend that you use the
@@ -395,8 +431,7 @@ do the following to run it from Python:
 ```python
 import tensorflow as tf
 zero_out_module = tf.load_op_library('./zero_out.so')
-with tf.Session(''):
-  zero_out_module.zero_out([[1, 2], [3, 4]]).eval()
+print(zero_out_module.zero_out([[1, 2], [3, 4]]).numpy())
 
 # Prints
 array([[1, 0], [0, 0]], dtype=int32)
@@ -484,16 +519,16 @@ This asserts that the input is a vector, and returns having set the
 
 *   The `context`, which can either be an `OpKernelContext` or
     `OpKernelConstruction` pointer (see
-    [`tensorflow/core/framework/op_kernel.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op_kernel.h)),
+    [`tensorflow/core/framework/op_kernel.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_kernel.h)),
     for its `SetStatus()` method.
 *   The condition.  For example, there are functions for validating the shape
     of a tensor in
-    [`tensorflow/core/framework/tensor_shape.h`](https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.h)
+    [`tensorflow/core/framework/tensor_shape.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.h)
 *   The error itself, which is represented by a `Status` object, see
-    [`tensorflow/core/lib/core/status.h`](https://www.tensorflow.org/code/tensorflow/core/lib/core/status.h). A
+    [`tensorflow/core/platform/status.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/status.h). A
     `Status` has both a type (frequently `InvalidArgument`, but see the list of
     types) and a message.  Functions for constructing an error may be found in
-    [`tensorflow/core/lib/core/errors.h`][validation-macros].
+    [`tensorflow/core/platform/errors.h`][validation-macros].
 
 Alternatively, if you want to test whether a `Status` object returned from some
 function is an error, and if so return it, use
@@ -528,6 +563,7 @@ form [described below](#attr-types).
 
 For example, if you'd like the `ZeroOut` op to preserve a user-specified index,
 instead of only the 0th element, you can register the op like so:
+
 ```c++
 REGISTER_OP("ZeroOut")
     .Attr("preserve_index: int")
@@ -540,6 +576,7 @@ REGISTER_OP("ZeroOut")
 
 Your kernel can then access this attr in its constructor via the `context`
 parameter:
+
 ```c++
 class ZeroOutOp : public OpKernel {
  public:
@@ -561,6 +598,7 @@ class ZeroOutOp : public OpKernel {
 ```
 
 which can then be used in the `Compute` method:
+
 ```c++
   void Compute(OpKernelContext* context) override {
     // ...
@@ -591,7 +629,6 @@ The following types are supported in an attr:
 * `bool`: True or false.
 * `type`: One of the (non-ref) values of [`DataType`][DataTypeString].
 * `shape`: A [`TensorShapeProto`][TensorShapeProto].
-* `tensor`: A [`TensorProto`][TensorProto].
 * `list()`: A list of ``, where `` is one of the above types.
   Note that `list(list())` is invalid.
 
@@ -602,88 +639,84 @@ See also: [`op_def_builder.cc:FinalizeAttr`][FinalizeAttr] for a definitive list
 Attrs may have default values, and some types of attrs can have constraints. To
 define an attr with constraints, you can use the following ``s:
 
-* `{'', ''}`: The value must be a string that has either the
-  value `` or ``.  The name of the type, `string`, is implied
-  when you use this syntax.  This emulates an enum:
+`{'', ''}`: The value must be a string that has either the
+value `` or ``. The name of the type, `string`, is implied
+when you use this syntax. This emulates an enum:
 
-  ```c++
-  REGISTER_OP("EnumExample")
-      .Attr("e: {'apple', 'orange'}");
-  ```
+```c++
+REGISTER_OP("EnumExample")
+    .Attr("e: {'apple', 'orange'}");
+```
 
-* `{, }`: The value is of type `type`, and must be one of
-  `` or ``, where `` and `` are supported
-  `tf.DType`.  You don't specify
-  that the type of the attr is `type`. This is implied when you have a list of
-  types in `{...}`.  For example, in this case the attr `t` is a type that must
-  be an `int32`, a `float`, or a `bool`:
+`{, }`: The value is of type `type`, and must be one of ``
+or ``, where `` and `` are supported `tf.DType`. You don't
+specify that the type of the attr is `type`. This is implied when you have a
+list of types in `{...}`. For example, in this case the attr `t` is a type that
+must be an `int32`, a `float`, or a `bool`:
 
-  ```c++
-  REGISTER_OP("RestrictedTypeExample")
-      .Attr("t: {int32, float, bool}");
-  ```
+```c++
+REGISTER_OP("RestrictedTypeExample")
+    .Attr("t: {int32, float, bool}");
+```
 
-* There are shortcuts for common type constraints:
-    * `numbertype`: Type `type` restricted to the numeric (non-string and
-      non-bool) types.
-    * `realnumbertype`: Like `numbertype` without complex types.
-    * `quantizedtype`: Like `numbertype` but just the quantized number types.
+There are shortcuts for common type constraints:
 
-    The specific lists of types allowed by these are defined by the functions
-    (like `NumberTypes()`) in
-    [`tensorflow/core/framework/types.h`](https://www.tensorflow.org/code/tensorflow/core/framework/types.h).
-    In this example the attr `t` must be one of the numeric types:
+*   `numbertype`: Type `type` restricted to the numeric (non-string and
+    non-bool) types.
+*   `realnumbertype`: Like `numbertype` without complex types.
+*   `quantizedtype`: Like `numbertype` but just the quantized number types.
 
-    ```c++
-    REGISTER_OP("NumberType")
-        .Attr("t: numbertype");
-    ```
+The specific lists of types allowed by these are defined by the functions (like
+`NumberTypes()`) in
+[`tensorflow/core/framework/types.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.h).
+In this example the attr `t` must be one of the numeric types:
 
-    For this op:
+```c++
+REGISTER_OP("NumberType")
+    .Attr("t: numbertype");
+```
 
-    ```python
-    tf.number_type(t=tf.int32)  # Valid
-    tf.number_type(t=tf.bool)   # Invalid
-    ```
+For this op:
 
-    Lists can be combined with other lists and single types.  The following
-    op allows attr `t` to be any of the numeric types, or the bool type:
+```python
+tf.number_type(t=tf.int32)  # Valid
+tf.number_type(t=tf.bool)   # Invalid
+```
 
-    ```c++
-    REGISTER_OP("NumberOrBooleanType")
-        .Attr("t: {numbertype, bool}");
-    ```
+Lists can be combined with other lists and single types. The following op allows
+attr `t` to be any of the numeric types, or the bool type:
 
-    For this op:
+```c++
+REGISTER_OP("NumberOrBooleanType")
+    .Attr("t: {numbertype, bool}");
+```
 
-    ```python
-    tf.number_or_boolean_type(t=tf.int32)  # Valid
-    tf.number_or_boolean_type(t=tf.bool)   # Valid
-    tf.number_or_boolean_type(t=tf.string) # Invalid
-    ```
+For this op:
 
-* `int >= `: The value must be an int whose value is greater than or equal to
-  ``, where `` is a natural number.
+```python
+tf.number_or_boolean_type(t=tf.int32)  # Valid
+tf.number_or_boolean_type(t=tf.bool)   # Valid
+tf.number_or_boolean_type(t=tf.string) # Invalid
+```
 
-  For example, the following op registration specifies that the attr `a` must
-  have a value that is at least `2`:
+`int >= `: The value must be an int whose value is greater than or equal to
+``, where `` is a natural number. For example, the following op
+registration specifies that the attr `a` must have a value that is at least `2`:
 
-  ```c++
-  REGISTER_OP("MinIntExample")
-      .Attr("a: int >= 2");
-  ```
-
-* `list() >= `: A list of type `` whose length is greater than
-  or equal to ``.
+```c++
+REGISTER_OP("MinIntExample")
+    .Attr("a: int >= 2");
+```
 
-  For example, the following op registration specifies that the attr `a` is a
-  list of types (either `int32` or `float`), and that there must be at least 3
-  of them:
+`list() >= `: A list of type `` whose length is greater than or
+equal to ``. For example, the following op registration specifies that the
+attr `a` is a list of types (either `int32` or `float`), and that there must be
+at least 3 of them:
 
-  ```c++
-  REGISTER_OP("TypeListExample")
-      .Attr("a: list({int32, float}) >= 3");
-  ```
+```c++
+REGISTER_OP("TypeListExample")
+    .Attr("a: list({int32, float}) >= 3");
+```
 
 To set a default value for an attr (making it optional in the generated code),
 add `= ` to the end, as in:
@@ -693,6 +726,13 @@ REGISTER_OP("AttrDefaultExample")
     .Attr("i: int = 0");
 ```
 
+Additionally, both a constraint and a default value can be specified:
+
+```c++
+REGISTER_OP("AttrConstraintAndDefaultExample")
+    .Attr("i: int >= 1 = 1");
+```
+
 The supported syntax of the default value is what would be used in the proto
 representation of the resulting GraphDef definition.
 
@@ -725,6 +765,7 @@ you would then register an `OpKernel` for each supported type.
 
 For instance, if you'd like the `ZeroOut` op to work on `float`s
 in addition to `int32`s, your op registration might look like:
+
 ```c++
 REGISTER_OP("ZeroOut")
     .Attr("T: {float, int32}")
@@ -735,60 +776,62 @@ REGISTER_OP("ZeroOut")
 Your op registration now specifies that the input's type must be `float`, or
 `int32`, and that its output will be the same type, since both have type `T`.
 
-> A note on naming: Inputs, outputs, and attrs generally should be
-> given snake\_case names.  The one exception is attrs that are used as the type
-> of an input or in the type of an output. Those attrs can be inferred when the
-> op is added to the graph and so don't appear in the op's function.  For
-> example, this last definition of ZeroOut will generate a Python function that
-> looks like:
->
-> ```python
-> def zero_out(to_zero, name=None):
->   """...
->   Args:
->     to_zero: A `Tensor`. Must be one of the following types:
->         `float32`, `int32`.
->     name: A name for the operation (optional).
->
->   Returns:
->     A `Tensor`. Has the same type as `to_zero`.
->   """
-> ```
->
-> If `to_zero` is passed an `int32` tensor, then `T` is automatically set to
-> `int32` (well, actually `DT_INT32`). Those inferred attrs are given
-> Capitalized or CamelCase names.
->
-> Compare this with an op that has a type attr that determines the output
-> type:
->
-> ```c++
-> REGISTER_OP("StringToNumber")
->     .Input("string_tensor: string")
->     .Output("output: out_type")
->     .Attr("out_type: {float, int32} = DT_FLOAT");
->     .Doc(R"doc(
-> Converts each string in the input Tensor to the specified numeric type.
-> )doc");
-> ```
->
-> In this case, the user has to specify the output type, as in the generated
-> Python:
->
-> ```python
-> def string_to_number(string_tensor, out_type=None, name=None):
->   """Converts each string in the input Tensor to the specified numeric type.
->
->   Args:
->     string_tensor: A `Tensor` of type `string`.
->     out_type: An optional `tf.DType` from: `tf.float32, tf.int32`.
->       Defaults to `tf.float32`.
->     name: A name for the operation (optional).
->
->   Returns:
->     A `Tensor` of type `out_type`.
->   """
-> ```
+###### Naming
+
+Inputs, outputs, and attrs generally should be given snake\_case names. The one
+exception is attrs that are used as the type of an input or in the type of an
+output. Those attrs can be inferred when the op is added to the graph and so
+don't appear in the op's function. For example, this last definition of ZeroOut
+will generate a Python function that looks like:
+
+```python
+def zero_out(to_zero, name=None):
+  """...
+  Args:
+    to_zero: A `Tensor`. Must be one of the following types:
+        `float32`, `int32`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `to_zero`.
+  """
+```
+
+If `to_zero` is passed an `int32` tensor, then `T` is automatically set to
+`int32` (well, actually `DT_INT32`). Those inferred attrs are given Capitalized
+or CamelCase names.
+
+Compare this with an op that has a type attr that determines the output type:
+
+```c++
+REGISTER_OP("StringToNumber")
+    .Input("string_tensor: string")
+    .Output("output: out_type")
+    .Attr("out_type: {float, int32} = DT_FLOAT");
+    .Doc(R"doc(
+Converts each string in the input Tensor to the specified numeric type.
+)doc");
+```
+
+In this case, the user has to specify the output type, as in the generated
+Python:
+
+```python
+def string_to_number(string_tensor, out_type=None, name=None):
+  """Converts each string in the input Tensor to the specified numeric type.
+
+  Args:
+    string_tensor: A `Tensor` of type `string`.
+    out_type: An optional `tf.DType` from: `tf.float32, tf.int32`.
+      Defaults to `tf.float32`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of type `out_type`.
+  """
+```
+
+###### Type polymorphism example
 
 ```c++
 #include "tensorflow/core/framework/op_kernel.h"
@@ -839,18 +882,19 @@ REGISTER_KERNEL_BUILDER(
     ZeroOutFloatOp);
 ```
 
-> To preserve [backwards compatibility](#backwards-compatibility), you should
-> specify a [default value](#default-values-constraints) when adding an attr to
-> an existing op:
->
-> ```c++
-> REGISTER_OP("ZeroOut")
->   .Attr("T: {float, int32} = DT_INT32")
->   .Input("to_zero: T")
->   .Output("zeroed: T")
-> ```
+To preserve [backwards compatibility](#backwards-compatibility), you should
+specify a [default value](#default-values-and-constraints) when adding an attr
+to an existing op:
+
+```c++
+REGISTER_OP("ZeroOut")
+  .Attr("T: {float, int32} = DT_INT32")
+  .Input("to_zero: T")
+  .Output("zeroed: T")
+```
 
 Let's say you wanted to add more types, say `double`:
+
 ```c++
 REGISTER_OP("ZeroOut")
     .Attr("T: {float, double, int32}")
@@ -861,29 +905,30 @@ REGISTER_OP("ZeroOut")
 Instead of writing another `OpKernel` with redundant code as above, often you
 will be able to use a C++ template instead.  You will still have one kernel
 registration (`REGISTER_KERNEL_BUILDER` call) per overload.
+
 ```c++
 template 
 class ZeroOutOp : public OpKernel {
  public:
   explicit ZeroOutOp(OpKernelConstruction* context) : OpKernel(context) {}
-  
+
   void Compute(OpKernelContext* context) override {
     // Grab the input tensor
     const Tensor& input_tensor = context->input(0);
     auto input = input_tensor.flat();
-    
+
     // Create an output tensor
     Tensor* output = NULL;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, input_tensor.shape(), &output));
     auto output_flat = output->template flat();
-    
+
     // Set all the elements of the output tensor to 0
     const int N = input.size();
     for (int i = 0; i < N; i++) {
       output_flat(i) = 0;
     }
-    
+
     // Preserve the first input value
     if (N > 0) output_flat(0) = input(0);
   }
@@ -1119,11 +1164,10 @@ expressions:
 * For a reference to a tensor: `Ref()`, where `` is one of the
   previous types.
 
-> A note on naming: Any attr used in the type of an input will be inferred.  By
-> convention those inferred attrs use capital names (like `T` or `N`).
-> Otherwise inputs, outputs, and attrs have names like function parameters
-> (e.g. `num_outputs`).  For more details, see the
-> [earlier note on naming](#naming).
+Any attr used in the type of an input will be inferred. By convention those
+inferred attrs use capital names (like `T` or `N`). Otherwise inputs, outputs,
+and attrs have names like function parameters (e.g. `num_outputs`). For more
+details, see the [earlier section on naming](#naming).
 
 For more details, see
 [`tensorflow/core/framework/op_def_builder.h`][op_def_builder].
@@ -1142,43 +1186,47 @@ The details of `GraphDef` compatibility are
 
 There are several ways to preserve backwards-compatibility.
 
-1. Any new attrs added to an operation must have default values defined, and
-   with that default value the op must have the original behavior. To change an
-   operation from not polymorphic to polymorphic, you *must* give a default
-   value to the new type attr to preserve the original signature by default. For
-   example, if your operation was:
+1.  Any new attrs added to an operation must have default values defined, and
+    with that default value the op must have the original behavior. To change an
+    operation from not polymorphic to polymorphic, you *must* give a default
+    value to the new type attr to preserve the original signature by default.
+    For example, if your operation was:
 
-       REGISTER_OP("MyGeneralUnaryOp")
-           .Input("in: float")
-           .Output("out: float");
+    ```c++
+    REGISTER_OP("MyGeneralUnaryOp")
+        .Input("in: float")
+        .Output("out: float");
+    ```
 
-   you can make it polymorphic in a backwards-compatible way using:
+    you can make it polymorphic in a backwards-compatible way using:
 
-       REGISTER_OP("MyGeneralUnaryOp")
-           .Input("in: T")
-           .Output("out: T")
-           .Attr("T: numerictype = DT_FLOAT");
+    ```c++
+    REGISTER_OP("MyGeneralUnaryOp")
+        .Input("in: T")
+        .Output("out: T")
+        .Attr("T: numerictype = DT_FLOAT");
+    ```
 
-2. You can safely make a constraint on an attr less restrictive.  For example,
-   you can change from `{int32, int64}` to `{int32, int64, float}` or `type`.
-   Or you may change from `{"apple", "orange"}` to `{"apple", "banana",
-   "orange"}` or `string`.
+2.  You can safely make a constraint on an attr less restrictive. For example,
+    you can change from `{int32, int64}` to `{int32, int64, float}` or `type`.
+    Or you may change from `{"apple", "orange"}` to `{"apple", "banana",
+    "orange"}` or `string`.
 
-3. You can change single inputs / outputs into list inputs / outputs, as long as
-   the default for the list type matches the old signature.
+3.  You can change single inputs / outputs into list inputs / outputs, as long
+    as the default for the list type matches the old signature.
 
-4. You can add a new list input / output, if it defaults to empty.
+4.  You can add a new list input / output, if it defaults to empty.
 
-5. Namespace any new ops you create, by prefixing the op names with something
-   unique to your project. This avoids having your op colliding with any ops
-   that might be included in future versions of TensorFlow.
+5.  Namespace any new ops you create, by prefixing the op names with something
+    unique to your project. This avoids having your op colliding with any ops
+    that might be included in future versions of TensorFlow.
 
-6. Plan ahead! Try to anticipate future uses for the op. Some signature changes
-   can't be done in a compatible way (for example, making a list of the same
-   type into a list of varying types).
+6.  Plan ahead! Try to anticipate future uses for the op. Some signature changes
+    can't be done in a compatible way (for example, making a list of the same
+    type into a list of varying types).
 
 The full list of safe and unsafe changes can be found in
-[`tensorflow/core/framework/op_compatibility_test.cc`](https://www.tensorflow.org/code/tensorflow/core/framework/op_compatibility_test.cc).
+[`tensorflow/core/framework/op_compatibility_test.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_compatibility_test.cc).
 If you cannot make your change to an operation backwards compatible, then create
 a new operation with a new name with the new semantics.
 
@@ -1187,7 +1235,7 @@ generated Python code may change in a way that isn't compatible with old
 callers.  The Python API may be kept compatible by careful changes in a
 hand-written Python wrapper, by keeping the old signature except possibly adding
 new optional arguments to the end.  Generally incompatible changes may only be
-made when TensorFlow's changes major versions, and must conform to the
+made when TensorFlow changes major versions, and must conform to the
 [`GraphDef` version semantics](./versions.md#compatibility_of_graphs_and_checkpoints).
 
 ### GPU support
@@ -1195,16 +1243,16 @@ made when TensorFlow's changes major versions, and must conform to the
 You can implement different OpKernels and register one for CPU and another for
 GPU, just like you can [register kernels for different types](#polymorphism).
 There are several examples of kernels with GPU support in
-[`tensorflow/core/kernels/`](https://www.tensorflow.org/code/tensorflow/core/kernels/).
+[`tensorflow/core/kernels/`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/).
 Notice some kernels have a CPU version in a `.cc` file, a GPU version in a file
 ending in `_gpu.cu.cc`, and some code shared in common in a `.h` file.
 
 For example, the `tf.pad` has
 everything but the GPU kernel in [`tensorflow/core/kernels/pad_op.cc`][pad_op].
 The GPU kernel is in
-[`tensorflow/core/kernels/pad_op_gpu.cu.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op_gpu.cu.cc),
+[`tensorflow/core/kernels/pad_op_gpu.cu.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/pad_op_gpu.cu.cc),
 and the shared code is a templated class defined in
-[`tensorflow/core/kernels/pad_op.h`](https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op.h).
+[`tensorflow/core/kernels/pad_op.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/pad_op.h).
 We organize the code this way for two reasons: it allows you to share common
 code among the CPU and GPU implementations, and it puts the GPU implementation
 into a separate file so that it can be compiled only by the GPU compiler.
@@ -1225,23 +1273,23 @@ kept on the CPU, add a `HostMemory()` call to the kernel registration, e.g.:
 #### Compiling the kernel for the GPU device
 
 Look at
-[cuda_op_kernel.cu.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
+[cuda_op_kernel.cu.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
 for an example that uses a CUDA kernel to implement an op. The
 `tf_custom_op_library` accepts a `gpu_srcs` argument in which the list of source
 files containing the CUDA kernels (`*.cu.cc` files) can be specified. For use
 with a binary installation of TensorFlow, the CUDA kernels have to be compiled
 with NVIDIA's `nvcc` compiler. Here is the sequence of commands you can use to
 compile the
-[cuda_op_kernel.cu.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
+[cuda_op_kernel.cu.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
 and
-[cuda_op_kernel.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cc)
+[cuda_op_kernel.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/adding_an_op/cuda_op_kernel.cc)
 into a single dynamically loadable library:
 
 ```bash
-nvcc -std=c++11 -c -o cuda_op_kernel.cu.o cuda_op_kernel.cu.cc \
+nvcc -std=c++14 -c -o cuda_op_kernel.cu.o cuda_op_kernel.cu.cc \
   ${TF_CFLAGS[@]} -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC
 
-g++ -std=c++11 -shared -o cuda_op_kernel.so cuda_op_kernel.cc \
+g++ -std=c++14 -shared -o cuda_op_kernel.so cuda_op_kernel.cc \
   cuda_op_kernel.cu.o ${TF_CFLAGS[@]} -fPIC -lcudart ${TF_LFLAGS[@]}
 ```
 
@@ -1253,7 +1301,9 @@ you'll need to specify the path explicitly in the second (g++) command above.
 For example, add `-L /usr/local/cuda-8.0/lib64/` if your CUDA is installed in
 `/usr/local/cuda-8.0`.
 
->   Note in some linux settings, additional options to `nvcc` compiling step are needed. Add `-D_MWAITXINTRIN_H_INCLUDED` to the `nvcc` command line to avoid errors from `mwaitxintrin.h`.
+Note: In some Linux settings, additional options to `nvcc` compiling step are
+needed. Add `-D_MWAITXINTRIN_H_INCLUDED` to the `nvcc` command line to avoid
+errors from `mwaitxintrin.h`.
 
 ### Implement the gradient in Python
 
@@ -1305,14 +1355,10 @@ def _zero_out_grad(op, grad):
 Details about registering gradient functions with
 `tf.RegisterGradient`:
 
-* For an op with one output, the gradient function will take an
-  `tf.Operation` `op` and a
-  `tf.Tensor` `grad` and build new ops
-  out of the tensors
-  [`op.inputs[i]`](../../api_docs/python/framework.md#Operation.inputs),
-  [`op.outputs[i]`](../../api_docs/python/framework.md#Operation.outputs), and `grad`.  Information
-  about any attrs can be found via
-  `tf.Operation.get_attr`.
+* For an op with one output, the gradient function will take an `tf.Operation`,
+  `op`, and a `tf.Tensor` `grad` and build new ops out of the tensors
+  `op.inputs[i]`, `op.outputs[i]`, and `grad`. Information about any attrs can
+  be found via `tf.Operation.get_attr`.
 
 * If the op has multiple outputs, the gradient function will take `op` and
   `grads`, where `grads` is a list of gradients with respect to each output.
@@ -1334,6 +1380,13 @@ Note that at the time the gradient function is called, only the data flow graph
 of ops is available, not the tensor data itself.  Thus, all computation must be
 performed using other tensorflow ops, to be run at graph execution time.
 
+Add type hints when registering the custom gradient for an op type to make the
+code more readable, debuggable, easier to maintain, and more robust through data
+validation. For example, when taking an `op` as a parameter in a function,
+specify that the gradient function will take an
+tf.Operation
+as its parameter type.
+
 ### Shape functions in C++
 
 The TensorFlow API has a feature called "shape inference" that provides
@@ -1359,7 +1412,7 @@ be set to the first input's shape. If the output is selected by its index as in
 
 There are a number of common shape functions
 that apply to many ops, such as `shape_inference::UnchangedShape` which can be
-found in [common_shape_fns.h](https://www.tensorflow.org/code/tensorflow/core/framework/common_shape_fns.h) and used as follows:
+found in [common_shape_fns.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/common_shape_fns.h) and used as follows:
 
 ```c++
 REGISTER_OP("ZeroOut")
@@ -1406,7 +1459,7 @@ provides access to the attributes of the op).
 
 Since shape inference is an optional feature, and the shapes of tensors may vary
 dynamically, shape functions must be robust to incomplete shape information for
-any of the inputs. The `Merge` method in [`InferenceContext`](https://www.tensorflow.org/code/tensorflow/core/framework/shape_inference.h)
+any of the inputs. The `Merge` method in [`InferenceContext`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/shape_inference.h)
 allows the caller to assert that two shapes are the same, even if either
 or both of them do not have complete information. Shape functions are defined
 for all of the core TensorFlow ops and provide many different usage examples.
@@ -1431,7 +1484,7 @@ If you have a complicated shape function, you should consider adding a test for
 validating that various input shape combinations produce the expected output
 shape combinations.  You can see examples of how to write these tests in some
 our
-[core ops tests](https://www.tensorflow.org/code/tensorflow/core/ops/array_ops_test.cc).
+[core ops tests](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/ops/array_ops_test.cc).
 (The syntax of `INFER_OK` and `INFER_ERROR` are a little cryptic, but try to be
 compact in representing input and output shape specifications in tests.  For
 now, see the surrounding comments in those tests to get a sense of the shape
@@ -1444,20 +1497,20 @@ To build a `pip` package for your op, see the
 guide shows how to build custom ops from the TensorFlow pip package instead
 of building TensorFlow from source.
 
-[core-array_ops]:https://www.tensorflow.org/code/tensorflow/core/ops/array_ops.cc
-[python-user_ops]:https://www.tensorflow.org/code/tensorflow/python/user_ops/user_ops.py
-[tf-kernels]:https://www.tensorflow.org/code/tensorflow/core/kernels/
-[user_ops]:https://www.tensorflow.org/code/tensorflow/core/user_ops/
-[pad_op]:https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op.cc
-[standard_ops-py]:https://www.tensorflow.org/code/tensorflow/python/ops/standard_ops.py
-[standard_ops-cc]:https://www.tensorflow.org/code/tensorflow/cc/ops/standard_ops.h
-[python-BUILD]:https://www.tensorflow.org/code/tensorflow/python/BUILD
-[validation-macros]:https://www.tensorflow.org/code/tensorflow/core/lib/core/errors.h
-[op_def_builder]:https://www.tensorflow.org/code/tensorflow/core/framework/op_def_builder.h
-[register_types]:https://www.tensorflow.org/code/tensorflow/core/framework/register_types.h
-[FinalizeAttr]:https://www.tensorflow.org/code/tensorflow/core/framework/op_def_builder.cc
-[DataTypeString]:https://www.tensorflow.org/code/tensorflow/core/framework/types.cc
-[python-BUILD]:https://www.tensorflow.org/code/tensorflow/python/BUILD
-[types-proto]:https://www.tensorflow.org/code/tensorflow/core/framework/types.proto
-[TensorShapeProto]:https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.proto
-[TensorProto]:https://www.tensorflow.org/code/tensorflow/core/framework/tensor.proto
+[core-array_ops]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/ops/array_ops.cc
+[python-user_ops]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/user_ops/user_ops.py
+[tf-kernels]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/
+[user_ops]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/user_ops/
+[pad_op]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/kernels/pad_op.cc
+[standard_ops-py]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/standard_ops.py
+[standard_ops-cc]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/ops/standard_ops.h
+[python-BUILD]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD
+[validation-macros]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/errors.h
+[op_def_builder]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def_builder.h
+[register_types]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/register_types.h
+[FinalizeAttr]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def_builder.cc
+[DataTypeString]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.cc
+[python-BUILD]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD
+[types-proto]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto
+[TensorShapeProto]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto
+[TensorProto]:https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto
diff --git a/site/en/guide/data.ipynb b/site/en/guide/data.ipynb
index 367de945514..739ef131005 100644
--- a/site/en/guide/data.ipynb
+++ b/site/en/guide/data.ipynb
@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Jxv6goXm7oGF"
       },
       "source": [
@@ -14,10 +13,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "llMNufAK7nfK"
       },
       "outputs": [],
@@ -38,7 +35,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8Byow2J6LaPl"
       },
       "source": [
@@ -48,35 +44,31 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "kGXS3UWBBNoc"
       },
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/data\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/data.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/data.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/data.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "9Qo3HgDjbDcI"
       },
       "source": [
-        "\n",
-        "\n",
         "The `tf.data` API enables you to build complex input pipelines from simple,\n",
         "reusable pieces. For example, the pipeline for an image model might aggregate\n",
         "data from files in a distributed file system, apply random perturbations to each\n",
@@ -98,51 +90,30 @@
         "    one or more files.\n",
         "\n",
         "*   A data **transformation** constructs a dataset from one or more\n",
-        "    `tf.data.Dataset` objects.\n",
-        "\n"
+        "    `tf.data.Dataset` objects.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "SwS0yboeseZs"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "UJIEjEIBdf-h"
       },
       "outputs": [],
       "source": [
-        "try:\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
         "import tensorflow as tf"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "7Y0JtWBNR9E5"
       },
       "outputs": [],
       "source": [
         "import pathlib\n",
+        "import os\n",
         "import matplotlib.pyplot as plt\n",
         "import pandas as pd\n",
         "import numpy as np\n",
@@ -153,12 +124,11 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "0l4a0ALxdaWF"
       },
       "source": [
         "## Basic mechanics\n",
-        "\u003ca id=\"basic-mechanics\"/\u003e\n",
+        "\n",
         "\n",
         "To create an input pipeline, you must start with a data *source*. For example,\n",
         "to construct a `Dataset` from data in memory, you can use\n",
@@ -168,8 +138,8 @@
         "\n",
         "Once you have a `Dataset` object, you can *transform* it into a new `Dataset` by\n",
         "chaining method calls on the `tf.data.Dataset` object. For example, you can\n",
-        "apply per-element transformations such as `Dataset.map()`, and multi-element\n",
-        "transformations such as `Dataset.batch()`. See the documentation for\n",
+        "apply per-element transformations such as `Dataset.map`, and multi-element\n",
+        "transformations such as `Dataset.batch`. Refer to the documentation for\n",
         "`tf.data.Dataset` for a complete list of transformations.\n",
         "\n",
         "The `Dataset` object is a Python iterable. This makes it possible to consume its\n",
@@ -178,10 +148,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "0F-FDnjB6t6J"
       },
       "outputs": [],
@@ -192,10 +160,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "pwJsRJ-FbDcJ"
       },
       "outputs": [],
@@ -207,7 +173,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "m0yy80MobDcM"
       },
       "source": [
@@ -217,10 +182,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "03w9oxFfbDcM"
       },
       "outputs": [],
@@ -233,7 +196,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Q4CgCL8qbDcO"
       },
       "source": [
@@ -245,10 +207,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "C2bHAeNxbDcO"
       },
       "outputs": [],
@@ -259,32 +219,44 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "B2Fzwt2nbDcR"
       },
       "source": [
-        "\u003c!-- TODO(jsimsa): Talk about `tf.function` support. --\u003e\n",
+        "\n",
         "\n",
-        "\u003ca id=\"dataset_structure\"\u003e\u003c/a\u003e\n",
+        "\n",
         "### Dataset structure\n",
         "\n",
-        "A dataset contains elements that each have the same (nested) structure and the\n",
-        "individual components of the structure can be of any type representable by\n",
-        "`tf.TypeSpec`, including `Tensor`, `SparseTensor`, `RaggedTensor`,\n",
-        "`TensorArray`, or `Dataset`.\n",
+        "A dataset produces a sequence of *elements*, where each element is\n",
+        "the same (nested) structure of *components*. Individual components\n",
+        "of the structure can be of any type representable by\n",
+        "`tf.TypeSpec`, including `tf.Tensor`, `tf.sparse.SparseTensor`,\n",
+        "`tf.RaggedTensor`, `tf.TensorArray`, or `tf.data.Dataset`.\n",
         "\n",
-        "The `Dataset.element_spec` property allows you to inspect the type of each\n",
-        "element component. The property returns a *nested structure* of `tf.TypeSpec`\n",
-        "objects, matching the structure of the element, which may be a single component,\n",
-        "a tuple of components, or a nested tuple of components. For example:"
+        "The Python constructs that can be used to express the (nested)\n",
+        "structure of elements include `tuple`, `dict`, `NamedTuple`, and\n",
+        "`OrderedDict`. In particular, `list` is not a valid construct for\n",
+        "expressing the structure of dataset elements. This is because\n",
+        "early `tf.data` users felt strongly about `list` inputs (for example, when passed\n",
+        "to `tf.data.Dataset.from_tensors`) being automatically packed as\n",
+        "tensors and `list` outputs (for example, return values of user-defined\n",
+        "functions) being coerced into a `tuple`. As a consequence, if you\n",
+        "would like a `list` input to be treated as a structure, you need\n",
+        "to convert it into `tuple` and if you would like a `list` output\n",
+        "to be a single component, then you need to explicitly pack it\n",
+        "using `tf.stack`.\n",
+        "\n",
+        "The `Dataset.element_spec` property allows you to inspect the type\n",
+        "of each element component. The property returns a *nested structure*\n",
+        "of `tf.TypeSpec` objects, matching the structure of the element,\n",
+        "which may be a single component, a tuple of components, or a nested\n",
+        "tuple of components. For example:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Mg0m1beIhXGn"
       },
       "outputs": [],
@@ -296,10 +268,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "cwyemaghhXaG"
       },
       "outputs": [],
@@ -313,10 +283,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "1CL7aB0ahXn_"
       },
       "outputs": [],
@@ -328,10 +296,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "m5bz7R1xhX1f"
       },
       "outputs": [],
@@ -344,10 +310,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "lVOPHur_hYQv"
       },
       "outputs": [],
@@ -359,22 +323,19 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "r5xNsFFvhUnr"
       },
       "source": [
         "The `Dataset` transformations support datasets of any structure. When using the\n",
-        "`Dataset.map()`, and `Dataset.filter()` transformations,\n",
+        "`Dataset.map`, and `Dataset.filter` transformations,\n",
         "which apply a function to each element, the element structure determines the\n",
         "arguments of the function:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "2myAr3Pxd-zF"
       },
       "outputs": [],
@@ -387,10 +348,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "woPXMP14gUTg"
       },
       "outputs": [],
@@ -401,10 +360,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "53PA4x6XgLar"
       },
       "outputs": [],
@@ -418,10 +375,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "2ju4sNSebDcR"
       },
       "outputs": [],
@@ -433,10 +388,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "BgxsfAS2g6gk"
       },
       "outputs": [],
@@ -448,7 +401,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "M1s2K0g-bDcT"
       },
       "source": [
@@ -458,25 +410,22 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "F3JG2f0h2683"
       },
       "source": [
         "### Consuming NumPy arrays\n",
         "\n",
-        "See [Loading NumPy arrays](../tutorials/load_data/numpy.ipynb) for more examples.\n",
+        "Refer to the [Loading NumPy arrays](../tutorials/load_data/numpy.ipynb) tutorial for more examples.\n",
         "\n",
         "If all of your input data fits in memory, the simplest way to create a `Dataset`\n",
         "from them is to convert them to `tf.Tensor` objects and use\n",
-        "`Dataset.from_tensor_slices()`."
+        "`Dataset.from_tensor_slices`."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "NmaE6PjjhQ47"
       },
       "outputs": [],
@@ -486,10 +435,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "J6cNiuDBbDcU"
       },
       "outputs": [],
@@ -504,7 +451,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "XkwrDHN5bDcW"
       },
       "source": [
@@ -518,7 +464,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "pO4ua2gEmIhR"
       },
       "source": [
@@ -526,32 +471,28 @@
         "\n",
         "Another common data source that can easily be ingested as a `tf.data.Dataset` is the python generator.\n",
         "\n",
-        "Caution: While this is a convienient approach it has limited portability and scalibility. It must run in the same python process that created the generator, and is still subject to the Python [GIL](https://en.wikipedia.org/wiki/Global_interpreter_lock)."
+        "Caution: While this is a convenient approach it has limited portability and scalability. It must run in the same python process that created the generator, and is still subject to the Python [GIL](https://en.wikipedia.org/wiki/Global_interpreter_lock)."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "9njpME-jmDza"
       },
       "outputs": [],
       "source": [
         "def count(stop):\n",
         "  i = 0\n",
-        "  while i\u003cstop:\n",
+        "  while i 5:\n",
         "    break"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "LmkynGilx0qf"
       },
       "source": [
         "The first output is an `int32` the second is a `float32`.\n",
         "\n",
-        "The first item is a scalar, shape `()`, and the second is a vector of unknown length, shape `(None,)` "
+        "The first item is a scalar, shape `()`, and the second is a vector of unknown length, shape `(None,)`"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "zDTfhEzhsliM"
       },
       "outputs": [],
       "source": [
         "ds_series = tf.data.Dataset.from_generator(\n",
-        "    gen_series, \n",
-        "    output_types=(tf.int32, tf.float32), \n",
+        "    gen_series,\n",
+        "    output_types=(tf.int32, tf.float32),\n",
         "    output_shapes=((), (None,)))\n",
         "\n",
         "ds_series"
@@ -682,7 +610,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "WWxvSyQiyN0o"
       },
       "source": [
@@ -691,15 +618,13 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "A7jEpj3As1lO"
       },
       "outputs": [],
       "source": [
-        "ds_series_batch = ds_series.shuffle(20).padded_batch(10, padded_shapes=([], [None]))\n",
+        "ds_series_batch = ds_series.shuffle(20).padded_batch(10)\n",
         "\n",
         "ids, sequence_batch = next(iter(ds_series_batch))\n",
         "print(ids.numpy())\n",
@@ -710,7 +635,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "_hcqOccJ1CxG"
       },
       "source": [
@@ -721,10 +645,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "g-_JCFRQ1CXM"
       },
       "outputs": [],
@@ -738,7 +660,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "UIjPhvQ87jUT"
       },
       "source": [
@@ -747,10 +668,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "vPCZeBQE5DfH"
       },
       "outputs": [],
@@ -760,10 +679,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "my4PxqfH26p6"
       },
       "outputs": [],
@@ -773,10 +690,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Hd96nH1w3eKH"
       },
       "outputs": [],
@@ -787,33 +702,43 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "KvRwvt5E2rTH"
       },
       "outputs": [],
       "source": [
         "ds = tf.data.Dataset.from_generator(\n",
-        "    img_gen.flow_from_directory, args=[flowers], \n",
-        "    output_types=(tf.float32, tf.float32), \n",
+        "    lambda: img_gen.flow_from_directory(flowers),\n",
+        "    output_types=(tf.float32, tf.float32),\n",
         "    output_shapes=([32,256,256,3], [32,5])\n",
         ")\n",
         "\n",
-        "ds"
+        "ds.element_spec"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LcaULBCXj_2_"
+      },
+      "outputs": [],
+      "source": [
+        "for images, labels in ds.take(1):\n",
+        "  print('images.shape: ', images.shape)\n",
+        "  print('labels.shape: ', labels.shape)\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ma4XoYzih2f4"
       },
       "source": [
         "### Consuming TFRecord data\n",
         "\n",
-        "See [Loading TFRecords](../tutorials/load_data/tf_records.ipynb) for an end-to-end example.\n",
+        "Refer to the [Loading TFRecords](../tutorials/load_data/tfrecord.ipynb) tutorial for an end-to-end example.\n",
         "\n",
         "The `tf.data` API supports a variety of file formats so that you can process\n",
         "large datasets that do not fit in memory. For example, the TFRecord file format\n",
@@ -826,7 +751,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "LiatWUloRJc4"
       },
       "source": [
@@ -835,10 +759,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "jZo_4fzdbDcW"
       },
       "outputs": [],
@@ -850,23 +772,19 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "seD5bOH3RhBP"
       },
       "source": [
         "The `filenames` argument to the `TFRecordDataset` initializer can either be a\n",
         "string, a list of strings, or a `tf.Tensor` of strings. Therefore if you have\n",
         "two sets of files for training and validation purposes, you can create a factory\n",
-        "method that produces the dataset, taking filenames as an input argument:\n",
-        "\n"
+        "method that produces the dataset, taking filenames as an input argument:\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "e2WV5d7DRUA-"
       },
       "outputs": [],
@@ -878,7 +796,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "62NC3vz9U8ww"
       },
       "source": [
@@ -887,10 +804,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "3tk29nlMl5P3"
       },
       "outputs": [],
@@ -904,13 +819,12 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qJAUib10bDcb"
       },
       "source": [
         "### Consuming text data\n",
         "\n",
-        "See [Loading Text](../tutorials/load_data/text.ipynb) for an end to end example.\n",
+        "Refer to the [Load text](../tutorials/load_data/text.ipynb) tutorial for an end-to-end example.\n",
         "\n",
         "Many datasets are distributed as one or more text files. The\n",
         "`tf.data.TextLineDataset` provides an easy way to extract lines from one or more\n",
@@ -920,10 +834,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "hQMoFu2TbDcc"
       },
       "outputs": [],
@@ -939,10 +851,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "il4cOjiVwj95"
       },
       "outputs": [],
@@ -953,7 +863,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MevIbDiwy4MC"
       },
       "source": [
@@ -962,10 +871,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "vpEHKyvHxu8A"
       },
       "outputs": [],
@@ -977,7 +884,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "lJyVw8ro7fey"
       },
       "source": [
@@ -986,10 +892,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "1UCveWOt7fDE"
       },
       "outputs": [],
@@ -1006,22 +910,19 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2F_pOIDubDce"
       },
       "source": [
         "By default, a `TextLineDataset` yields *every* line of each file, which may\n",
         "not be desirable, for example, if the file starts with a header line, or contains comments. These lines can be removed using the `Dataset.skip()` or\n",
-        "`Dataset.filter()` transformations. Here we skip the first line, then filter to\n",
+        "`Dataset.filter` transformations. Here, you skip the first line, then filter to\n",
         "find only survivors."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "X6b20Gua2jPO"
       },
       "outputs": [],
@@ -1032,10 +933,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "5M1pauNT68B2"
       },
       "outputs": [],
@@ -1046,10 +945,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "dEIP95cibDcf"
       },
       "outputs": [],
@@ -1062,10 +959,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "odQ4618h1XqD"
       },
       "outputs": [],
@@ -1077,7 +972,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "x5z5B11UjDTd"
       },
       "source": [
@@ -1087,11 +981,10 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ChDHNi3qbDch"
       },
       "source": [
-        "See [Loading CSV Files](../tutorials/load_data/csv.ipynb), and [Loading Pandas DataFrames](../tutorials/load_data/pandas.ipynb) for more examples. \n",
+        "Refer to the [Loading CSV Files](../tutorials/load_data/csv.ipynb) and [Loading Pandas DataFrames](../tutorials/load_data/pandas_dataframe.ipynb) tutorials for more examples.\n",
         "\n",
         "The CSV file format is a popular format for storing tabular data in plain text.\n",
         "\n",
@@ -1100,10 +993,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "kj28j5u49Bjm"
       },
       "outputs": [],
@@ -1113,22 +1004,19 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ghvtmW40LM0B"
       },
       "outputs": [],
       "source": [
-        "df = pd.read_csv(titanic_file, index_col=None)\n",
+        "df = pd.read_csv(titanic_file)\n",
         "df.head()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "J9uBqt5oGsR-"
       },
       "source": [
@@ -1137,10 +1025,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "JmAMCiPJA0qO"
       },
       "outputs": [],
@@ -1155,23 +1041,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "47yippqaHFk6"
       },
       "source": [
-        "A more scalable approach is to load from disk as necessary. \n",
+        "A more scalable approach is to load from disk as necessary.\n",
         "\n",
         "The `tf.data` module provides methods to extract records from one or more CSV files that comply with [RFC 4180](https://tools.ietf.org/html/rfc4180).\n",
         "\n",
-        "The `experimental.make_csv_dataset` function is the high level interface for reading sets of csv files. It supports column type inference and many other features, like batching and shuffling, to make usage simple."
+        "The `tf.data.experimental.make_csv_dataset` function is the high-level interface for reading sets of CSV files. It supports column type inference and many other features, like batching and shuffling, to make usage simple."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "zHUDrM_s_brq"
       },
       "outputs": [],
@@ -1183,10 +1066,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "TsZfhz79_Wlg"
       },
       "outputs": [],
@@ -1201,7 +1082,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "k_5N7CdNGYAa"
       },
       "source": [
@@ -1210,10 +1090,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "H9KNHyDwF2Sc"
       },
       "outputs": [],
@@ -1225,10 +1103,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "7C2uosFnGIT8"
       },
       "outputs": [],
@@ -1242,24 +1118,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "TSVgJJ1HJD6M"
       },
       "source": [
-        "There is also a lower-level `experimental.CsvDataset` class which provides finer grained control. It does not support column type inference. Instead you must specify the type of each column. "
+        "There is also a lower-level `experimental.CsvDataset` class which provides finer grained control. It does not support column type inference. Instead you must specify the type of each column."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "wP1Y_NXA8bYl"
       },
       "outputs": [],
       "source": [
-        "titanic_types  = [tf.int32, tf.string, tf.float32, tf.int32, tf.int32, tf.float32, tf.string, tf.string, tf.string, tf.string] \n",
+        "titanic_types  = [tf.int32, tf.string, tf.float32, tf.int32, tf.int32, tf.float32, tf.string, tf.string, tf.string, tf.string]\n",
         "dataset = tf.data.experimental.CsvDataset(titanic_file, titanic_types , header=True)\n",
         "\n",
         "for line in dataset.take(10):\n",
@@ -1269,7 +1142,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "oZSuLVsTbDcj"
       },
       "source": [
@@ -1278,10 +1150,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Qry-g90FMo2I"
       },
       "outputs": [],
@@ -1297,10 +1167,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "d5_hbiE9bDck"
       },
       "outputs": [],
@@ -1316,10 +1184,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "__jc7iD9M9FC"
       },
       "outputs": [],
@@ -1331,7 +1197,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "z_4g0cIvbDcl"
       },
       "source": [
@@ -1344,10 +1209,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "p2IF_K0obDcm"
       },
       "outputs": [],
@@ -1362,10 +1225,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "-5aLprDeRNb0"
       },
       "outputs": [],
@@ -1377,7 +1238,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-CJfhb03koVN"
       },
       "source": [
@@ -1387,7 +1247,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yAO7SZDSk57_"
       },
       "source": [
@@ -1396,10 +1255,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "1dZwN3CS-jV2"
       },
       "outputs": [],
@@ -1414,7 +1271,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "4099UU8n-jHP"
       },
       "source": [
@@ -1424,7 +1280,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "FCyTYpmDs_jE"
       },
       "source": [
@@ -1433,10 +1288,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "_2iCXsHu6jJH"
       },
       "outputs": [],
@@ -1448,7 +1301,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Ylj9fgkamgWZ"
       },
       "source": [
@@ -1457,10 +1309,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "lAkQp5uxoINu"
       },
       "outputs": [],
@@ -1474,25 +1324,22 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "91CPfUUJ_8SZ"
       },
       "source": [
-        "We can read the data using the `tf.io.read_file` function and extract the label from the path, returning `(image, label)` pairs:"
+        "Read the data using the `tf.io.read_file` function and extract the label from the path, returning `(image, label)` pairs:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "-xhBRgvNqRRe"
       },
       "outputs": [],
       "source": [
         "def process_path(file_path):\n",
-        "  label = tf.strings.split(file_path, '/')[-2]\n",
+        "  label = tf.strings.split(file_path, os.sep)[-2]\n",
         "  return tf.io.read_file(file_path), label\n",
         "\n",
         "labeled_ds = list_ds.map(process_path)"
@@ -1500,10 +1347,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "kxrl0lGdnpRz"
       },
       "outputs": [],
@@ -1517,24 +1362,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yEh46Ee0oSH5"
       },
       "source": [
-        "\u003c!--\n",
+        "\n",
         "\n",
-        "## Batching dataset elements\n",
-        "\n"
+        "## Batching dataset elements\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "gR-2xY-8oSH4"
       },
       "source": [
@@ -1543,16 +1385,14 @@
         "The simplest form of batching stacks `n` consecutive elements of a dataset into\n",
         "a single element. The `Dataset.batch()` transformation does exactly this, with\n",
         "the same constraints as the `tf.stack()` operator, applied to each component\n",
-        "of the elements: i.e. for each component *i*, all elements must have a tensor\n",
+        "of the elements: i.e., for each component *i*, all elements must have a tensor\n",
         "of the exact same shape."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "xB7KeceLoSH0"
       },
       "outputs": [],
@@ -1569,7 +1409,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "LlV1tpFdoSH0"
       },
       "source": [
@@ -1578,10 +1417,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "yN7hn7OBoSHx"
       },
       "outputs": [],
@@ -1592,7 +1429,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "It1fPA3NoSHw"
       },
       "source": [
@@ -1601,10 +1437,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "BycWC7WCoSHt"
       },
       "outputs": [],
@@ -1616,26 +1450,23 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mj9nRxFZoSHs"
       },
       "source": [
         "### Batching tensors with padding\n",
         "\n",
         "The above recipe works for tensors that all have the same size. However, many\n",
-        "models (e.g. sequence models) work with input data that can have varying size\n",
-        "(e.g. sequences of different lengths). To handle this case, the\n",
-        "`Dataset.padded_batch()` transformation enables you to batch tensors of\n",
-        "different shape by specifying one or more dimensions in which they may be\n",
+        "models (including sequence models) work with input data that can have varying size\n",
+        "(for example, sequences of different lengths). To handle this case, the\n",
+        "`Dataset.padded_batch` transformation enables you to batch tensors of\n",
+        "different shapes by specifying one or more dimensions in which they may be\n",
         "padded."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "kycwO0JooSHn"
       },
       "outputs": [],
@@ -1652,38 +1483,33 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wl3yhth1oSHm"
       },
       "source": [
-        "The `Dataset.padded_batch()` transformation allows you to set different padding\n",
+        "The `Dataset.padded_batch` transformation allows you to set different padding\n",
         "for each dimension of each component, and it may be variable-length (signified\n",
         "by `None` in the example above) or constant-length. It is also possible to\n",
         "override the padding value, which defaults to 0.\n",
         "\n",
-        "\u003c!--\n",
+        "\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "G8zbAxMwoSHl"
       },
       "source": [
-        "## Training workflows\n",
-        "\n"
+        "## Training workflows\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "UnlhzF_AoSHk"
       },
       "source": [
@@ -1693,15 +1519,13 @@
         "data.\n",
         "\n",
         "The simplest way to iterate over a dataset in multiple epochs is to use the\n",
-        "`Dataset.repeat()` transformation. First we create a dataset of titanic data:"
+        "`Dataset.repeat()` transformation. First, create a dataset of titanic data:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "0tODHZzRoSHg"
       },
       "outputs": [],
@@ -1712,10 +1536,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "LMO6mlXxoSHc"
       },
       "outputs": [],
@@ -1730,7 +1552,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "WfVzmqL7oSHa"
       },
       "source": [
@@ -1744,10 +1565,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "nZ0G1cztoSHX"
       },
       "outputs": [],
@@ -1759,7 +1578,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "moH-4gBEoSHW"
       },
       "source": [
@@ -1768,10 +1586,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "wmbmdK1qoSHS"
       },
       "outputs": [],
@@ -1784,19 +1600,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "DlEM5f9loSHR"
       },
       "source": [
-        "If you would like to perform a custom computation (e.g. to collect statistics) at the end of each epoch then it's simplest to restart the dataset iteration on each epoch:"
+        "If you would like to perform a custom computation (for example, to collect statistics) at the end of each epoch then it's simplest to restart the dataset iteration on each epoch:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "YyekyeY7oSHO"
       },
       "outputs": [],
@@ -1813,7 +1626,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "_Bci79WCoSHN"
       },
       "source": [
@@ -1828,7 +1640,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6YvXr-qeoSHL"
       },
       "source": [
@@ -1837,10 +1648,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Io4iJH1toSHI"
       },
       "outputs": [],
@@ -1857,7 +1666,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "T6tNYRcsoSHH"
       },
       "source": [
@@ -1866,10 +1674,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ayM3FFFAoSHC"
       },
       "outputs": [],
@@ -1881,21 +1687,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "PLrfIjTHoSHB"
       },
       "source": [
         "As with `Dataset.batch` the order relative to `Dataset.repeat` matters.\n",
         "\n",
-        "`Dataset.shuffle` doesn't signal the end of an epoch until the shuffle buffer is empty. So a shuffle placed before a repeat will show every element of one epoch before moving to the next: "
+        "`Dataset.shuffle` doesn't signal the end of an epoch until the shuffle buffer is empty. So a shuffle placed before a repeat will show every element of one epoch before moving to the next:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "YX3pe7zZoSG6"
       },
       "outputs": [],
@@ -1910,10 +1713,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "H9hlE-lGoSGz"
       },
       "outputs": [],
@@ -1927,7 +1728,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "UucIgCxWoSGx"
       },
       "source": [
@@ -1936,10 +1736,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Bhxb5YGZoSGm"
       },
       "outputs": [],
@@ -1954,10 +1752,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "VAM4cbpZoSGL"
       },
       "outputs": [],
@@ -1973,7 +1769,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ianlfbrxbDco"
       },
       "source": [
@@ -1988,20 +1783,18 @@
         "that will represent a single element in the new dataset. Its implementation uses\n",
         "standard TensorFlow operations to transform one element into another.\n",
         "\n",
-        "This section covers common examples of how to use `Dataset.map()`.\n",
-        "\n"
+        "This section covers common examples of how to use `Dataset.map()`.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "UXw1IZVdbDcq"
       },
       "source": [
         "### Decoding image data and resizing it\n",
         "\n",
-        "\u003c!-- TODO(markdaoust): link to image augmentation when it exists --\u003e\n",
+        "\n",
         "When training a neural network on real-world image data, it is often necessary\n",
         "to convert images of different sizes to a common size, so that they may be\n",
         "batched into a fixed size.\n",
@@ -2011,10 +1804,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "rMGlj8V-u-NH"
       },
       "outputs": [],
@@ -2025,7 +1816,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GyhZLB8N5jBm"
       },
       "source": [
@@ -2034,10 +1824,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "fZObC0debDcr"
       },
       "outputs": [],
@@ -2045,11 +1833,11 @@
         "# Reads an image from a file, decodes it into a dense tensor, and resizes it\n",
         "# to a fixed shape.\n",
         "def parse_image(filename):\n",
-        "  parts = tf.strings.split(filename, '/')\n",
+        "  parts = tf.strings.split(filename, os.sep)\n",
         "  label = parts[-2]\n",
         "\n",
         "  image = tf.io.read_file(filename)\n",
-        "  image = tf.image.decode_jpeg(image)\n",
+        "  image = tf.io.decode_jpeg(image)\n",
         "  image = tf.image.convert_image_dtype(image, tf.float32)\n",
         "  image = tf.image.resize(image, [128, 128])\n",
         "  return image, label"
@@ -2058,7 +1846,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "e0dVJlCA5qHA"
       },
       "source": [
@@ -2067,10 +1854,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "y8xuN_HBzGup"
       },
       "outputs": [],
@@ -2090,7 +1875,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "d3P8N-S55vDu"
       },
       "source": [
@@ -2099,10 +1883,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "SzO8LI_H5Sk_"
       },
       "outputs": [],
@@ -2116,25 +1898,23 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3Ff7IqB9bDcs"
       },
       "source": [
         "### Applying arbitrary Python logic\n",
         "\n",
-        "For performance reasons, we encourage you to use TensorFlow operations for\n",
+        "For performance reasons, use TensorFlow operations for\n",
         "preprocessing your data whenever possible. However, it is sometimes useful to\n",
-        "call external Python libraries when parsing your input data. You can use the `tf.py_function()` operation in a `Dataset.map()` transformation."
+        "call external Python libraries when parsing your input data. You can use the `tf.py_function` operation in a `Dataset.map` transformation."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "R2u7CeA67DU8"
       },
       "source": [
-        "For example, if you want to apply a random rotation, the `tf.image` module only has `tf.image.rot90`, which is not very useful for image augmentation. \n",
+        "For example, if you want to apply a random rotation, the `tf.image` module only has `tf.image.rot90`, which is not very useful for image augmentation.\n",
         "\n",
         "Note: `tensorflow_addons` has a TensorFlow compatible `rotate` in `tensorflow_addons.image.rotate`.\n",
         "\n",
@@ -2143,16 +1923,15 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "tBUmbERt7Czz"
       },
       "outputs": [],
       "source": [
         "import scipy.ndimage as ndimage\n",
         "\n",
+        "@tf.py_function(Tout=tf.float32)\n",
         "def random_rotate_image(image):\n",
         "  image = ndimage.rotate(image, np.random.uniform(-30, 30), reshape=False)\n",
         "  return image"
@@ -2160,10 +1939,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "_wEyL7bS9S6t"
       },
       "outputs": [],
@@ -2176,7 +1953,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "KxVx7z-ABNyq"
       },
       "source": [
@@ -2185,27 +1961,23 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Cn2nIu92BMp0"
       },
       "outputs": [],
       "source": [
         "def tf_random_rotate_image(image, label):\n",
         "  im_shape = image.shape\n",
-        "  [image,] = tf.py_function(random_rotate_image, [image], [tf.float32])\n",
+        "  image = random_rotate_image(image)\n",
         "  image.set_shape(im_shape)\n",
         "  return image, label"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "bWPqKbTnbDct"
       },
       "outputs": [],
@@ -2219,7 +1991,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ykx59-cMBwOT"
       },
       "source": [
@@ -2232,10 +2003,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "6wnE134b32KY"
       },
       "outputs": [],
@@ -2248,7 +2017,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "HGypdgYOlXZz"
       },
       "source": [
@@ -2257,10 +2025,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "4znsVNqnF73C"
       },
       "outputs": [],
@@ -2278,10 +2044,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "cwzqp8IGC_vQ"
       },
       "outputs": [],
@@ -2291,10 +2055,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "y2X1dQNfC8Lu"
       },
       "outputs": [],
@@ -2310,10 +2072,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "lGJhKDp_61A_"
       },
       "outputs": [],
@@ -2325,10 +2085,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "8vFIUFzD5qIC"
       },
       "outputs": [],
@@ -2339,10 +2097,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "vRYNYkEej7Ix"
       },
       "outputs": [],
@@ -2354,11 +2110,10 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ry1n0UBeczit"
       },
       "source": [
-        "\u003ca id=\"time_series_windowing\"\u003e\u003c/a\u003e\n",
+        "\n",
         "\n",
         "### Time series windowing"
       ]
@@ -2366,17 +2121,15 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "t0JMgvXEz9y1"
       },
       "source": [
-        "For an end to end time series example see: [Time series forecasting](../../tutorials/text/time_series.ipynb)."
+        "For an end-to-end time series example see: [Time series forecasting](../../tutorials/structured_data/time_series.ipynb)."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hzBABBkAkkVJ"
       },
       "source": [
@@ -2387,10 +2140,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "kTQgo49skjuY"
       },
       "outputs": [],
@@ -2401,11 +2152,10 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "o6GLGhxgpazJ"
       },
       "source": [
-        "Typically, models based on this sort of data will want a contiguous time slice. \n",
+        "Typically, models based on this sort of data will want a contiguous time slice.\n",
         "\n",
         "The simplest approach would be to batch the data:"
       ]
@@ -2413,7 +2163,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ETqB7QvTCNty"
       },
       "source": [
@@ -2422,10 +2171,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "pSs9XqwQpvIN"
       },
       "outputs": [],
@@ -2439,7 +2186,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mgb2qikEtk5W"
       },
       "source": [
@@ -2448,10 +2194,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "47XfwPhetkIN"
       },
       "outputs": [],
@@ -2463,13 +2207,12 @@
         "predict_dense_1_step = batches.map(dense_1_step)\n",
         "\n",
         "for features, label in predict_dense_1_step.take(3):\n",
-        "  print(features.numpy(), \" =\u003e \", label.numpy())"
+        "  print(features.numpy(), \" => \", label.numpy())"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "DjsXuINKqsS_"
       },
       "source": [
@@ -2478,10 +2221,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "FMmkQB1Gqo6x"
       },
       "outputs": [],
@@ -2489,19 +2230,18 @@
         "batches = range_ds.batch(15, drop_remainder=True)\n",
         "\n",
         "def label_next_5_steps(batch):\n",
-        "  return (batch[:-5],   # Take the first 5 steps\n",
-        "          batch[-5:])   # take the remainder\n",
+        "  return (batch[:-5],   # Inputs: All except the last 5 steps\n",
+        "          batch[-5:])   # Labels: The last 5 steps\n",
         "\n",
         "predict_5_steps = batches.map(label_next_5_steps)\n",
         "\n",
         "for features, label in predict_5_steps.take(3):\n",
-        "  print(features.numpy(), \" =\u003e \", label.numpy())"
+        "  print(features.numpy(), \" => \", label.numpy())"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5a611Qr3jlhl"
       },
       "source": [
@@ -2510,30 +2250,27 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "11dF3wyFjk2J"
       },
       "outputs": [],
       "source": [
         "feature_length = 10\n",
-        "label_length = 5\n",
+        "label_length = 3\n",
         "\n",
         "features = range_ds.batch(feature_length, drop_remainder=True)\n",
-        "labels = range_ds.batch(feature_length).skip(1).map(lambda labels: labels[:-5])\n",
+        "labels = range_ds.batch(feature_length).skip(1).map(lambda labels: labels[:label_length])\n",
         "\n",
-        "predict_5_steps = tf.data.Dataset.zip((features, labels))\n",
+        "predicted_steps = tf.data.Dataset.zip((features, labels))\n",
         "\n",
-        "for features, label in predict_5_steps.take(3):\n",
-        "  print(features.numpy(), \" =\u003e \", label.numpy())"
+        "for features, label in predicted_steps.take(5):\n",
+        "  print(features.numpy(), \" => \", label.numpy())"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "adew3o2mCURC"
       },
       "source": [
@@ -2543,19 +2280,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "fF6pEdlduq8E"
       },
       "source": [
-        "While using `Dataset.batch` works, there are situations where you may need finer control. The `Dataset.window` method gives you complete control, but requires some care: it returns a `Dataset` of `Datasets`. See [Dataset structure](#dataset_structure) for details."
+        "While using `Dataset.batch` works, there are situations where you may need finer control. The `Dataset.window` method gives you complete control, but requires some care: it returns a `Dataset` of `Datasets`. Go to the [Dataset structure](#dataset_structure) section for details."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ZEI2W_EBw2OX"
       },
       "outputs": [],
@@ -2570,7 +2304,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "r82hWdk4x-46"
       },
       "source": [
@@ -2579,10 +2312,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "SB8AI03mnF8u"
       },
       "outputs": [],
@@ -2594,19 +2325,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "sgLIwq9Anc34"
       },
       "source": [
-        "In nearly all cases, you will want to `.batch` the dataset first:"
+        "In nearly all cases, you will want to `Dataset.batch` the dataset first:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "5j_y84rmyVQa"
       },
       "outputs": [],
@@ -2621,7 +2349,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hVugrmND3Grp"
       },
       "source": [
@@ -2632,10 +2359,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "LdFRv_0D4FqW"
       },
       "outputs": [],
@@ -2652,10 +2377,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "-iVxcVfEdf5b"
       },
       "outputs": [],
@@ -2669,7 +2392,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "fMGMTPQ4w8pr"
       },
       "source": [
@@ -2678,10 +2400,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "F0fPfZkZw6j_"
       },
       "outputs": [],
@@ -2689,13 +2409,12 @@
         "dense_labels_ds = ds.map(dense_1_step)\n",
         "\n",
         "for inputs,labels in dense_labels_ds.take(3):\n",
-        "  print(inputs.numpy(), \"=\u003e\", labels.numpy())"
+        "  print(inputs.numpy(), \"=>\", labels.numpy())"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vyi_-ft0kvy4"
       },
       "source": [
@@ -2703,15 +2422,13 @@
         "\n",
         "When working with a dataset that is very class-imbalanced, you may want to resample the dataset. `tf.data` provides two methods to do this. The credit card fraud dataset is a good example of this sort of problem.\n",
         "\n",
-        "Note: See [Imbalanced Data](../tutorials/keras/imbalanced_data.ipynb) for a full tutorial.\n"
+        "Note: Go to [Classification on imbalanced data](../tutorials/structured_data/imbalanced_data.ipynb) for a full tutorial.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "U2e8dxVUlFHO"
       },
       "outputs": [],
@@ -2726,10 +2443,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "EhkkM4Wx75S_"
       },
       "outputs": [],
@@ -2743,7 +2458,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "A8O47EmHlxYX"
       },
       "source": [
@@ -2752,10 +2466,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "a8-Ss69XlzXD"
       },
       "outputs": [],
@@ -2776,10 +2488,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "O1a3t_B4l_f6"
       },
       "outputs": [],
@@ -2798,7 +2508,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "z1b8lFhSnDdv"
       },
       "source": [
@@ -2808,7 +2517,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "y8jQWsgMnjQG"
       },
       "source": [
@@ -2818,21 +2526,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ov14SRrQyQE3"
       },
       "source": [
-        "One approach to resampling a dataset is to use `sample_from_datasets`. This is more applicable when you have a separate `data.Dataset` for each class.\n",
+        "One approach to resampling a dataset is to use `sample_from_datasets`. This is more applicable when you have a separate `tf.data.Dataset` for each class.\n",
         "\n",
         "Here, just use filter to generate them from the credit card fraud data:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "6YKfCPa-nioA"
       },
       "outputs": [],
@@ -2851,10 +2556,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "8FNd3sQjzl9-"
       },
       "outputs": [],
@@ -2866,43 +2569,37 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GxLAr-7p0ATX"
       },
       "source": [
-        "To use `tf.data.experimental.sample_from_datasets` pass the datasets, and the weight for each:"
+        "To use `tf.data.Dataset.sample_from_datasets` pass the datasets, and the weight for each:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "vjdPVIFCngOb"
       },
       "outputs": [],
       "source": [
-        "balanced_ds = tf.data.experimental.sample_from_datasets(\n",
+        "balanced_ds = tf.data.Dataset.sample_from_datasets(\n",
         "    [negative_ds, positive_ds], [0.5, 0.5]).batch(10)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2K4ObOms082B"
       },
       "source": [
-        "Now the dataset produces examples of each class with 50/50 probability:"
+        "Now the dataset produces examples of each class with a 50/50 probability:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Myvkw21Rz-fH"
       },
       "outputs": [],
@@ -2914,7 +2611,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "OUTE3eb9nckY"
       },
       "source": [
@@ -2924,27 +2620,24 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "kZ9ezkK6irMD"
       },
       "source": [
-        "One problem with the above `experimental.sample_from_datasets` approach is that\n",
-        "is that it needs a separate `tf.data.Dataset` per class. Using `Dataset.filter`\n",
-        "works, but results in all the data being loaded twice.\n",
+        "One problem with the above `Dataset.sample_from_datasets` approach is that\n",
+        "it needs a separate `tf.data.Dataset` per class. You could use `Dataset.filter`\n",
+        "to create those two datasets, but that results in all the data being loaded twice.\n",
         "\n",
-        "The `data.experimental.rejection_resample` function can be applied to a dataset to rebalance it, while only loading it once. Elements will be dropped from the dataset to achieve balance.\n",
+        "The `tf.data.Dataset.rejection_resample` method can be applied to a dataset to rebalance it, while only loading it once. Elements will be dropped or repeated to achieve balance.\n",
         "\n",
-        "`data.experimental.rejection_resample` takes a `class_func` argument. This `class_func` is applied to each dataset element, and is used to determine which class an example belongs to for the purposes of balancing.\n",
+        "The `rejection_resample` method takes a `class_func` argument. This `class_func` is applied to each dataset element, and is used to determine which class an example belongs to for the purposes of balancing.\n",
         "\n",
-        "The elements of `creditcard_ds` are already `(features, label)` pairs. So the `class_func` just needs to return those labels:"
+        "The goal here is to balance the label distribution, and the elements of `creditcard_ds` are already `(features, label)` pairs. So the `class_func` just needs to return those labels:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "zC_Cuzw8lhI5"
       },
       "outputs": [],
@@ -2956,127 +2649,148 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "DdKmE8Jumlp0"
+        "id": "YxJrOZVToGuE"
       },
       "source": [
-        "The resampler also needs a target distribution, and optionally an initial distribution estimate:"
+        "The resampling method deals with individual examples, so in this case you must `unbatch` the dataset before applying that method.\n",
+        "\n",
+        "The method needs a target distribution, and optionally an initial distribution estimate as inputs."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "9tv0tWNxmkzM"
+        "id": "fY6VIhr3oGHG"
       },
       "outputs": [],
       "source": [
-        "resampler = tf.data.experimental.rejection_resample(\n",
-        "    class_func, target_dist=[0.5, 0.5], initial_dist=fractions)"
+        "resample_ds = (\n",
+        "    creditcard_ds\n",
+        "    .unbatch()\n",
+        "    .rejection_resample(class_func, target_dist=[0.5,0.5],\n",
+        "                        initial_dist=fractions)\n",
+        "    .batch(10))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "YxJrOZVToGuE"
+        "id": "L-HnC1s8idqV"
       },
       "source": [
-        "The resampler deals with individual examples, so you must `unbatch` the dataset before applying the resampler:"
+        "The `rejection_resample` method returns `(class, example)` pairs where the `class` is the output of the `class_func`. In this case, the `example` was already a `(feature, label)` pair, so use `map` to drop the extra copy of the labels:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "fY6VIhr3oGHG"
+        "id": "KpfCGU6BiaZq"
       },
       "outputs": [],
       "source": [
-        "resample_ds = creditcard_ds.unbatch().apply(resampler).batch(10)"
+        "balanced_ds = resample_ds.map(lambda extra_label, features_and_label: features_and_label)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "L-HnC1s8idqV"
+        "id": "j3d2jyEhx9kD"
       },
       "source": [
-        "The resampler returns creates `(class, example)` pairs from the output of the `class_func`. In this case, the `example` was already a `(feature, label)` pair, so use `map` to drop the extra copy of the labels:"
+        "Now the dataset produces examples of each class with a 50/50 probability:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "KpfCGU6BiaZq"
+        "id": "XGLYChBQwkDV"
       },
       "outputs": [],
       "source": [
-        "balanced_ds = resample_ds.map(lambda extra_label, features_and_label: features_and_label)"
+        "for features, labels in balanced_ds.take(10):\n",
+        "  print(labels.numpy())"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "j3d2jyEhx9kD"
+        "id": "vYFKQx3bUBeU"
       },
       "source": [
-        "Now the dataset produces examples of each class with 50/50 probability:"
+        "## Iterator Checkpointing"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SOGg1UFhUE4z"
+      },
+      "source": [
+        "Tensorflow supports [taking checkpoints](./checkpoint.ipynb) so that when your training process restarts it can restore the latest checkpoint to recover most of its progress. In addition to checkpointing the model variables, you can also checkpoint the progress of the dataset iterator. This could be useful if you have a large dataset and don't want to start the dataset from the beginning on each restart. Note however that iterator checkpoints may be large, since transformations such as `Dataset.shuffle` and `Dataset.prefetch` require buffering elements within the iterator.\n",
+        "\n",
+        "To include your iterator in a checkpoint, pass the iterator to the `tf.train.Checkpoint` constructor."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "XGLYChBQwkDV"
+        "id": "3Fsm9wvKUsNC"
       },
       "outputs": [],
       "source": [
-        "for features, labels in balanced_ds.take(10):\n",
-        "  print(labels.numpy())"
+        "range_ds = tf.data.Dataset.range(20)\n",
+        "\n",
+        "iterator = iter(range_ds)\n",
+        "ckpt = tf.train.Checkpoint(step=tf.Variable(0), iterator=iterator)\n",
+        "manager = tf.train.CheckpointManager(ckpt, '/tmp/my_ckpt', max_to_keep=3)\n",
+        "\n",
+        "print([next(iterator).numpy() for _ in range(5)])\n",
+        "\n",
+        "save_path = manager.save()\n",
+        "\n",
+        "print([next(iterator).numpy() for _ in range(5)])\n",
+        "\n",
+        "ckpt.restore(manager.latest_checkpoint)\n",
+        "\n",
+        "print([next(iterator).numpy() for _ in range(5)])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gxWglTwX9Fex"
+      },
+      "source": [
+        "Note: It is not possible to checkpoint an iterator which relies on an external state, such as a `tf.py_function`. Attempting to do so will raise an exception complaining about the external state."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "uLRdedPpbDdD"
       },
       "source": [
-        "## Using high-level APIs\n",
-        "\n"
+        "## Using `tf.data` with `tf.keras`"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "JTQe8daMcgFz"
       },
       "source": [
-        "### tf.keras\n",
-        "\n",
         "The `tf.keras` API simplifies many aspects of creating and executing machine\n",
-        "learning models. Its `.fit()` and `.evaluate()` and `.predict()` APIs support datasets as inputs. Here is a quick dataset and model setup:"
+        "learning models. Its `Model.fit` and `Model.evaluate` and `Model.predict` APIs support datasets as inputs. Here is a quick dataset and model setup:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "-bfjqm0hOfES"
       },
       "outputs": [],
@@ -3090,10 +2804,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "wDhF3rGnbDdD"
       },
       "outputs": [],
@@ -3103,18 +2815,17 @@
         "\n",
         "model = tf.keras.Sequential([\n",
         "  tf.keras.layers.Flatten(),\n",
-        "  tf.keras.layers.Dense(10, activation='softmax')\n",
+        "  tf.keras.layers.Dense(10)\n",
         "])\n",
         "\n",
         "model.compile(optimizer='adam',\n",
-        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(), \n",
+        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
         "              metrics=['accuracy'])"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Rdogg8CfHs-G"
       },
       "source": [
@@ -3123,10 +2834,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "9cu4kPzOHnlt"
       },
       "outputs": [],
@@ -3137,19 +2846,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "FzpAQfJMJF41"
       },
       "source": [
-        "If you pass an infinite dataset, for example by calling `Dataset.repeat()`, you just need to also pass the `steps_per_epoch` argument:"
+        "If you pass an infinite dataset, for example by calling `Dataset.repeat`, you just need to also pass the `steps_per_epoch` argument:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Bp1BpzlyJinb"
       },
       "outputs": [],
@@ -3160,7 +2866,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "iTLsw_nqJpTw"
       },
       "source": [
@@ -3169,10 +2874,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "TnlRHlaL-XUI"
       },
       "outputs": [],
@@ -3185,7 +2888,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "C8UBU3CJKEA4"
       },
       "source": [
@@ -3194,10 +2896,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "uVgamf9HKDon"
       },
       "outputs": [],
@@ -3210,19 +2910,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "aZYhJ_YSIl6w"
       },
       "source": [
-        "The labels are not required in when calling `Model.predict`. "
+        "The labels are not required when calling `Model.predict`."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "343lXJ-pIqWD"
       },
       "outputs": [],
@@ -3235,7 +2932,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "YfzZORwLI202"
       },
       "source": [
@@ -3244,10 +2940,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "mgQJTPrT-2WF"
       },
       "outputs": [],
@@ -3255,128 +2949,11 @@
         "result = model.predict(fmnist_train_ds, steps = 10)\n",
         "print(result.shape)"
       ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "azNurSOubDdF"
-      },
-      "source": [
-        "### tf.estimator\n",
-        "\n",
-        "To use a `Dataset` in the `input_fn` of a `tf.estimator.Estimator`, simply\n",
-        "return the `Dataset` from the `input_fn` and the framework will take care of consuming its elements\n",
-        "for you. For example:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "e3RTyqhLbDdG"
-      },
-      "outputs": [],
-      "source": [
-        "import tensorflow_datasets as tfds\n",
-        "\n",
-        "def train_input_fn():\n",
-        "  titanic = tf.data.experimental.make_csv_dataset(\n",
-        "      titanic_file, batch_size=32,\n",
-        "      label_name=\"survived\")\n",
-        "  titanic_batches = (\n",
-        "      titanic.cache().repeat().shuffle(500)\n",
-        "      .prefetch(tf.data.experimental.AUTOTUNE))\n",
-        "  return titanic_batches"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "qJGw6gntBHFU"
-      },
-      "outputs": [],
-      "source": [
-        "embark = tf.feature_column.categorical_column_with_hash_bucket('embark_town', 32)\n",
-        "cls = tf.feature_column.categorical_column_with_vocabulary_list('class', ['First', 'Second', 'Third']) \n",
-        "age = tf.feature_column.numeric_column('age')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "v18FPnaT1RtK"
-      },
-      "outputs": [],
-      "source": [
-        "import tempfile\n",
-        "model_dir = tempfile.mkdtemp()\n",
-        "model = tf.estimator.LinearClassifier(\n",
-        "    model_dir=model_dir,\n",
-        "    feature_columns=[embark, cls, age],\n",
-        "    n_classes=2\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "iGaJKkmVBgo2"
-      },
-      "outputs": [],
-      "source": [
-        "model = model.train(input_fn=train_input_fn, steps=100)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "CXkivCNq0vfH"
-      },
-      "outputs": [],
-      "source": [
-        "result = model.evaluate(train_input_fn, steps=10)\n",
-        "\n",
-        "for key, value in result.items():\n",
-        "  print(key, \":\", value)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "CPLD8n4CLVi_"
-      },
-      "outputs": [],
-      "source": [
-        "for pred in model.predict(train_input_fn):\n",
-        "  for key, value in pred.items():\n",
-        "    print(key, \":\", value)\n",
-        "  break"
-      ]
     }
   ],
   "metadata": {
     "colab": {
-      "collapsed_sections": [],
       "name": "data.ipynb",
-      "private_outputs": true,
       "provenance": [],
       "toc_visible": true
     },
diff --git a/site/en/guide/data_performance.ipynb b/site/en/guide/data_performance.ipynb
index 92b7adc0204..81d8b3fd5b3 100644
--- a/site/en/guide/data_performance.ipynb
+++ b/site/en/guide/data_performance.ipynb
@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Tce3stUlHN0L"
       },
       "source": [
@@ -12,11 +11,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
         "id": "tuOe1ymfHZPu"
       },
       "outputs": [],
@@ -37,7 +34,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qFdPvlXBOdUN"
       },
       "source": [
@@ -47,30 +43,28 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MfBg1C5NB3X0"
       },
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/data_performance\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/data_performance.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/data_performance.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/data_performance.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "xHxb-dlhMIzW"
       },
       "source": [
@@ -81,26 +75,25 @@
         "The `tf.data` API helps to build flexible and efficient input pipelines.\n",
         "This document demonstrates how to use the `tf.data` API to build highly performant TensorFlow input pipelines.\n",
         "\n",
-        "Before you continue, read the \"[Build TensorFlow input pipelines](./data.ipynb)\" guide, to learn how to use the `tf.data` API."
+        "Before you continue, check the [Build TensorFlow input pipelines](./data.ipynb) guide to learn how to use the `tf.data` API."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "UhNtHfuxCGVy"
       },
       "source": [
         "## Resources\n",
         "\n",
         "* [Build TensorFlow input pipelines](./data.ipynb)\n",
-        "* `tf.data.Dataset` API"
+        "* `tf.data.Dataset` API\n",
+        "* [Analyze `tf.data` performance with the TF Profiler](./data_performance_analysis.md)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MUXex9ctTuDB"
       },
       "source": [
@@ -109,21 +102,12 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "IqR2PQG4ZaZ0"
       },
       "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "\n",
         "import tensorflow as tf\n",
         "\n",
         "import time"
@@ -132,43 +116,39 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "QthTHCKF-jKD"
       },
       "source": [
         "Throughout this guide, you will iterate across a dataset and measure the performance.\n",
-        "Making reproducible performance benchmarks can be difficult, different factors impacting it:\n",
+        "Making reproducible performance benchmarks can be difficult. Different factors affecting reproducibility include:\n",
         "\n",
-        "- the current CPU load,\n",
-        "- the network traffic,\n",
-        "- complex mechanisms like cache, etc.\n",
+        "- The current CPU load\n",
+        "- The network traffic\n",
+        "- Complex mechanisms, such as cache\n",
         "\n",
-        "Hence, to provide a reproducible benchmark, build an artificial example."
+        "To get a reproducible benchmark, you will build an artificial example."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3bU5gsSI-jKF"
       },
       "source": [
         "### The dataset\n",
         "\n",
-        "Define a class inheriting from `tf.data.Dataset` called `ArtificialDataset`.\n",
+        "Start with defining a class inheriting from `tf.data.Dataset` called `ArtificialDataset`.\n",
         "This dataset:\n",
         "\n",
-        "- generates `num_samples` samples (default is 3)\n",
-        "- sleeps for some time before the first item to simulate opening a file\n",
-        "- sleeps for some time before producing each item to simulate reading data from a file"
+        "- Generates `num_samples` samples (default is 3)\n",
+        "- Sleeps for some time before the first item to simulate opening a file\n",
+        "- Sleeps for some time before producing each item to simulate reading data from a file"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "zUQv4kCd-jKH"
       },
       "outputs": [],
@@ -187,8 +167,7 @@
         "    def __new__(cls, num_samples=3):\n",
         "        return tf.data.Dataset.from_generator(\n",
         "            cls._generator,\n",
-        "            output_types=tf.dtypes.int64,\n",
-        "            output_shapes=(1,),\n",
+        "            output_signature = tf.TensorSpec(shape = (1,), dtype = tf.int64),\n",
         "            args=(num_samples,)\n",
         "        )"
       ]
@@ -196,32 +175,28 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "O9y1WjNv-jKL"
       },
       "source": [
-        "This dataset is similar to the `tf.data.Dataset.range` one, adding a fixed delay at the beginning and between each sample."
+        "This dataset is similar to the `tf.data.Dataset.range` one, adding a fixed delay at the beginning of and in-between each sample."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "FGK1Y4jn-jKM"
       },
       "source": [
         "### The training loop\n",
         "\n",
-        "Write a dummy training loop that measures how long it takes to iterate over a dataset.\n",
+        "Next, write a dummy training loop that measures how long it takes to iterate over a dataset.\n",
         "Training time is simulated."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "MIaM3u00-jKP"
       },
       "outputs": [],
@@ -232,13 +207,12 @@
         "        for sample in dataset:\n",
         "            # Performing a training step\n",
         "            time.sleep(0.01)\n",
-        "    tf.print(\"Execution time:\", time.perf_counter() - start_time)"
+        "    print(\"Execution time:\", time.perf_counter() - start_time)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "KK58SuXS-jKT"
       },
       "source": [
@@ -250,7 +224,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Xi8t26y7-jKV"
       },
       "source": [
@@ -261,10 +234,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "_gP7J1y4-jKY"
       },
       "outputs": [],
@@ -275,23 +246,22 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Lxeat5dH-jKf"
       },
       "source": [
         "Under the hood, this is how your execution time was spent:\n",
         "\n",
-        "![Naive](https://www.tensorflow.org/guide/images/data_performance/naive.svg)\n",
+        "![Data execution time plot - a naive method](https://www.tensorflow.org/guide/images/data_performance/naive.svg)\n",
         "\n",
-        "You can see that performing a training step involves:\n",
+        "The plot shows that performing a training step involves:\n",
         "\n",
-        "- opening a file if it hasn't been opened yet,\n",
-        "- fetching a data entry from the file,\n",
-        "- using the data for training.\n",
+        "- Opening a file if it hasn't been opened yet\n",
+        "- Fetching a data entry from the file\n",
+        "- Using the data for training\n",
         "\n",
         "However, in a naive synchronous implementation like here, while your pipeline is fetching the data, your model is sitting idle. \n",
         "Conversely, while your model is training, the input pipeline is sitting idle.\n",
-        "The training step time is thus the sum of all, opening, reading and training time.\n",
+        "The training step time is thus the sum of opening, reading and training times.\n",
         "\n",
         "The next sections build on this input pipeline, illustrating best practices for designing performant TensorFlow input pipelines."
       ]
@@ -299,12 +269,13 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mfukBGNz-jKh"
       },
       "source": [
         "### Prefetching\n",
         "\n",
+        "\n",
+        "\n",
         "Prefetching overlaps the preprocessing and model execution of a training step.\n",
         "While the model is executing training step `s`, the input pipeline is reading the data for step `s+1`.\n",
         "Doing so reduces the step time to the maximum (as opposed to the sum) of the training and the time it takes to extract the data.\n",
@@ -313,7 +284,7 @@
         "It can be used to decouple the time when data is produced from the time when data is consumed.\n",
         "In particular, the transformation uses a background thread and an internal buffer to prefetch elements from the input dataset ahead of the time they are requested.\n",
         "The number of elements to prefetch should be equal to (or possibly greater than) the number of batches consumed by a single training step.\n",
-        "You could either manually tune this value, or set it to `tf.data.experimental.AUTOTUNE` which will prompt the\n",
+        "You could either manually tune this value, or set it to `tf.data.AUTOTUNE`, which will prompt the\n",
         "`tf.data` runtime to tune the value dynamically at runtime.\n",
         "\n",
         "Note that the prefetch transformation provides benefits any time there is an opportunity to overlap the work of a \"producer\" with the work of a \"consumer.\""
@@ -321,59 +292,56 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "DHpUVqH1-jKi"
       },
       "outputs": [],
       "source": [
         "benchmark(\n",
         "    ArtificialDataset()\n",
-        "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
+        "    .prefetch(tf.data.AUTOTUNE)\n",
         ")"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "h7z_kzo--jKn"
       },
       "source": [
-        "![Prefetched](https://www.tensorflow.org/guide/images/data_performance/prefetched.svg)\n",
+        "![Data execution time plot - prefetching method](https://www.tensorflow.org/guide/images/data_performance/prefetched.svg)\n",
         "\n",
-        "This time you can see that while the training step is running for sample 0, the input pipeline is reading the data for the sample 1, and so on."
+        "Now, as the data execution time plot shows, while the training step is running for sample 0, the input pipeline is reading the data for the sample 1, and so on."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "52QMKfaY-jKq"
       },
       "source": [
         "### Parallelizing data extraction\n",
         "\n",
-        "In a real-world setting, the input data may be stored remotely (for example, GCS or HDFS).\n",
+        "\n",
+        "\n",
+        "In a real-world setting, the input data may be stored remotely (for example, on Google Cloud Storage or HDFS).\n",
         "A dataset pipeline that works well when reading data locally might become bottlenecked on I/O when reading data remotely because of the following differences between local and remote storage:\n",
         "\n",
-        "*   **Time-to-first-byte:** Reading the first byte of a file from remote storage can take orders of magnitude longer than from local storage.\n",
-        "*   **Read throughput:** While remote storage typically offers large aggregate bandwidth, reading a single file might only be able to utilize a small fraction of this bandwidth.\n",
+        "- **Time-to-first-byte**: Reading the first byte of a file from remote storage can take orders of magnitude longer than from local storage.\n",
+        "- **Read throughput**: While remote storage typically offers large aggregate bandwidth, reading a single file might only be able to utilize a small fraction of this bandwidth.\n",
         "\n",
         "In addition, once the raw bytes are loaded into memory, it may also be necessary to deserialize and/or decrypt the data (e.g. [protobuf](https://developers.google.com/protocol-buffers/)), which requires additional computation.\n",
         "This overhead is present irrespective of whether the data is stored locally or remotely, but can be worse in the remote case if data is not prefetched effectively.\n",
         "\n",
         "To mitigate the impact of the various data extraction overheads, the `tf.data.Dataset.interleave` transformation can be used to parallelize the data loading step, interleaving the contents of other datasets (such as data file\n",
         "readers).\n",
-        "The number of datasets to overlap can be specified by the `cycle_length` argument, while the level of parallelism can be specified by the `num_parallel_calls` argument. Similar to the `prefetch` transformation, the `interleave` transformation supports `tf.data.experimental.AUTOTUNE` which will delegate the decision about what level of parallelism to use to the `tf.data` runtime."
+        "The number of datasets to overlap can be specified by the `cycle_length` argument, while the level of parallelism can be specified by the `num_parallel_calls` argument. Similar to the `prefetch` transformation, the `interleave` transformation supports `tf.data.AUTOTUNE`, which will delegate the decision about what level of parallelism to use to the `tf.data` runtime."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "gs8O8Vbu-jKu"
       },
       "source": [
@@ -384,52 +352,46 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "fDH12GiK-jKw"
       },
       "outputs": [],
       "source": [
         "benchmark(\n",
         "    tf.data.Dataset.range(2)\n",
-        "    .interleave(ArtificialDataset)\n",
+        "    .interleave(lambda _: ArtificialDataset())\n",
         ")"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "78CsSOnf-jK0"
       },
       "source": [
-        "![Sequential interleave](https://www.tensorflow.org/guide/images/data_performance/sequential_interleave.svg)\n",
+        "![Data execution time plot - sequential interleave](https://www.tensorflow.org/guide/images/data_performance/sequential_interleave.svg)\n",
         "\n",
-        "This plot allows to exhibit the behavior of the `interleave` transformation, fetching samples alternatively from the two datasets available.\n",
+        "This data execution time plot allows to exhibit the behavior of the `interleave` transformation, fetching samples alternatively from the two datasets available.\n",
         "However, no performance improvement is involved here."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "j3cqqmYl-jK2"
       },
       "source": [
         "#### Parallel interleave\n",
         "\n",
-        "Now use the `num_parallel_calls` argument of the `interleave` transformation.\n",
+        "Now, use the `num_parallel_calls` argument of the `interleave` transformation.\n",
         "This loads multiple datasets in parallel, reducing the time waiting for the files to be opened."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "a3FQcTPY-jK4"
       },
       "outputs": [],
@@ -437,8 +399,8 @@
         "benchmark(\n",
         "    tf.data.Dataset.range(2)\n",
         "    .interleave(\n",
-        "        ArtificialDataset,\n",
-        "        num_parallel_calls=tf.data.experimental.AUTOTUNE\n",
+        "        lambda _: ArtificialDataset(),\n",
+        "        num_parallel_calls=tf.data.AUTOTUNE\n",
         "    )\n",
         ")"
       ]
@@ -446,24 +408,24 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "RxRLPB6C-jLA"
       },
       "source": [
-        "![Parallel interleave](https://www.tensorflow.org/guide/images/data_performance/parallel_interleave.svg)\n",
+        "![Data execution time plot - parallel interleave method](https://www.tensorflow.org/guide/images/data_performance/parallel_interleave.svg)\n",
         "\n",
-        "This time, the reading of the two datasets is parallelized, reducing the global data processing time."
+        "This time, as the data execution time plot shows, the reading of the two datasets is parallelized, reducing the global data processing time."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5ZCLFWyv-jLB"
       },
       "source": [
         "### Parallelizing data transformation\n",
         "\n",
+        "\n",
+        "\n",
         "When preparing data, input elements may need to be pre-processed.\n",
         "To this end, the `tf.data` API offers the `tf.data.Dataset.map` transformation, which applies a user-defined function to each element of the input dataset.\n",
         "Because input elements are independent of one another, the pre-processing can be parallelized across multiple CPU cores.\n",
@@ -471,15 +433,13 @@
         "\n",
         "Choosing the best value for the `num_parallel_calls` argument depends on your hardware, characteristics of your training data (such as its size and shape), the cost of your map function, and what other processing is happening on the CPU at the same time.\n",
         "A simple heuristic is to use the number of available CPU cores.\n",
-        "However, as for the `prefetch` and `interleave` transformation, the `map` transformation supports `tf.data.experimental.AUTOTUNE` which will delegate the decision about what level of parallelism to use to the `tf.data` runtime."
+        "However, as for the `prefetch` and `interleave` transformation, the `map` transformation supports `tf.data.AUTOTUNE` which will delegate the decision about what level of parallelism to use to the `tf.data` runtime."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "GSkKetpx-jLD"
       },
       "outputs": [],
@@ -493,7 +453,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wiU7W_QC-jLI"
       },
       "source": [
@@ -504,10 +463,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ZSBvDpJG-jLL"
       },
       "outputs": [],
@@ -521,19 +478,17 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ngwMTDb6-jLR"
       },
       "source": [
-        "![Sequential mapping](https://www.tensorflow.org/guide/images/data_performance/sequential_map.svg)\n",
+        "![Data execution time plot - sequential mapping method](https://www.tensorflow.org/guide/images/data_performance/sequential_map.svg)\n",
         "\n",
-        "As for the [naive approach](#The-naive-approach), here the times spent for opening, reading, pre-processing (mapping) and training steps sum together for a single iteration."
+        "As for the [naive approach](#The-naive-approach), here, as the plot shows, the times spent for opening, reading, pre-processing (mapping) and training steps sum together for a single iteration."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "U-10PE1D-jLU"
       },
       "source": [
@@ -544,10 +499,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "F8AYLZbg-jLV"
       },
       "outputs": [],
@@ -556,7 +509,7 @@
         "    ArtificialDataset()\n",
         "    .map(\n",
         "        mapped_function,\n",
-        "        num_parallel_calls=tf.data.experimental.AUTOTUNE\n",
+        "        num_parallel_calls=tf.data.AUTOTUNE\n",
         "    )\n",
         ")"
       ]
@@ -564,34 +517,32 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-MoJklzP-jLe"
       },
       "source": [
-        "![Parallel mapping](https://www.tensorflow.org/guide/images/data_performance/parallel_map.svg)\n",
+        "![Data execution time - parallel mapping](https://www.tensorflow.org/guide/images/data_performance/parallel_map.svg)\n",
         "\n",
-        "Now, you can see on the plot that the pre-processing steps overlap, reducing the overall time for a single iteration."
+        "As the data plot demonstrates, the pre-processing steps overlap, reducing the overall time for a single iteration."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ZY1Q9kJO-jLh"
       },
       "source": [
         "### Caching\n",
         "\n",
+        "\n",
+        "\n",
         "The `tf.data.Dataset.cache` transformation can cache a dataset, either in memory or on local storage.\n",
         "This will save some operations (like file opening and data reading) from being executed during each epoch."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "xieLApaI-jLi"
       },
       "outputs": [],
@@ -609,13 +560,12 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "KeMgW9XI-jLn"
       },
       "source": [
-        "![Cached dataset](https://www.tensorflow.org/guide/images/data_performance/cached_dataset.svg)\n",
+        "![Data execution time - cached dataset method](https://www.tensorflow.org/guide/images/data_performance/cached_dataset.svg)\n",
         "\n",
-        "When you cache a dataset, the transformations before the `cache` one (like the file opening and data reading) are executed only during the first epoch.\n",
+        "Here, the data execution time plot shows that when you cache a dataset, the transformations before the `cache` one (like the file opening and data reading) are executed only during the first epoch.\n",
         "The next epochs will reuse the data cached by the`cache` transformation.\n",
         "\n",
         "If the user-defined function passed into the `map` transformation is expensive, apply the `cache` transformation after the `map` transformation as long as the resulting dataset can still fit into memory or local storage.\n",
@@ -625,14 +575,15 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "i3NtGI3r-jLp"
       },
       "source": [
         "### Vectorizing mapping\n",
         "\n",
+        "\n",
+        "\n",
         "Invoking a user-defined function passed into the `map` transformation has overhead related to scheduling and executing the user-defined function.\n",
-        "We recommend vectorizing the user-defined function (that is, have it operate over a batch of inputs at once) and apply the `batch` transformation _before_ the `map` transformation.\n",
+        "Vectorize the user-defined function (that is, have it operate over a batch of inputs at once) and apply the `batch` transformation _before_ the `map` transformation.\n",
         "\n",
         "To illustrate this good practice, your artificial dataset is not suitable.\n",
         "The scheduling delay is around 10 microseconds (10e-6 seconds), far less than the tens of milliseconds used in the `ArtificialDataset`, and thus its impact is hard to see.\n",
@@ -642,10 +593,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "xqtiYPmb-jLt"
       },
       "outputs": [],
@@ -666,7 +615,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Fj2gmsMT-jL5"
       },
       "source": [
@@ -675,10 +623,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Imn3SslJ-jMA"
       },
       "outputs": [],
@@ -695,21 +641,19 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "BWUNbPqv-jMF"
       },
       "source": [
-        "![Scalar map](https://www.tensorflow.org/guide/images/data_performance/scalar_map.svg)\n",
+        "![Data execution time - scalar map method](https://www.tensorflow.org/guide/images/data_performance/scalar_map.svg)\n",
         "\n",
-        "The plot above illustrate what is going on (with less samples).\n",
-        "You can see that the mapped function is applied for each sample.\n",
+        "The plot above illustrates what is going on (with less samples) using the scalar mapping method.\n",
+        "It shows that the mapped function is applied for each sample.\n",
         "While this function is very fast, it has some overhead that impact the time performance."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "tDVSM0A--jMG"
       },
       "source": [
@@ -718,10 +662,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "nAw1mDLw-jMI"
       },
       "outputs": [],
@@ -738,33 +680,26 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "DbMteMY9-jMO",
-        "scrolled": false
+        "id": "DbMteMY9-jMO"
       },
       "source": [
-        "![Vectorized map](https://www.tensorflow.org/guide/images/data_performance/vectorized_map.svg)\n",
+        "![Data execution time - vectorized map method](https://www.tensorflow.org/guide/images/data_performance/vectorized_map.svg)\n",
         "\n",
         "This time, the mapped function is called once and applies to a batch of sample.\n",
-        "While the function could takes more time to execute, the overhead appear only once, improving the overall time performance."
+        "As the data execution time plot shows, while the function could takes more time to execute, the overhead appear only once, improving the overall time performance."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hfueG0Wj-jMR"
       },
       "source": [
         "### Reducing memory footprint\n",
         "\n",
-        "A number of transformations, including `interleave`, `prefetch`, and `shuffle`,\n",
-        "maintain an internal buffer of elements. If the user-defined function passed\n",
-        "into the `map` transformation changes the size of the elements, then the\n",
-        "ordering of the map transformation and the transformations that buffer elements\n",
-        "affects the memory usage. In general, we recommend choosing the order that\n",
-        "results in lower memory footprint, unless different ordering is desirable for\n",
-        "performance.\n",
+        "\n",
+        "\n",
+        "A number of transformations, including `interleave`, `prefetch`, and `shuffle`, maintain an internal buffer of elements. If the user-defined function passed into the `map` transformation changes the size of the elements, then the ordering of the map transformation and the transformations that buffer elements affects the memory usage. In general, choose the order that results in lower memory footprint, unless different ordering is desirable for performance.\n",
         "\n",
         "#### Caching partial computations\n",
         "\n",
@@ -782,7 +717,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MYOHG69M-jMT"
       },
       "source": [
@@ -791,40 +725,37 @@
         "Here is a summary of the best practices for designing performant TensorFlow\n",
         "input pipelines:\n",
         "\n",
-        "*   [Use the `prefetch` transformation](#Pipelining) to overlap the work of a producer and consumer.\n",
-        "*   [Parallelize the data reading transformation](#Parallelizing-data-extraction) using the `interleave` transformation.\n",
-        "*   [Parallelize the `map` transformation](#Parallelizing-data-transformation) by setting the `num_parallel_calls` argument.\n",
-        "*   [Use the `cache` transformation](#Caching) to cache data in memory during the first epoch\n",
-        "*   [Vectorize user-defined functions](#Map-and-batch) passed in to the `map` transformation\n",
-        "*   [Reduce memory usage](#Reducing-memory-footprint) when applying the `interleave`, `prefetch`, and `shuffle` transformations."
+        "*   [Use the `prefetch` transformation](#prefetching) to overlap the work of a producer and consumer\n",
+        "*   [Parallelize the data reading transformation](#parallelizing_data_extraction) using the `interleave` transformation\n",
+        "*   [Parallelize the `map` transformation](#parallelizing_data_transformation) by setting the `num_parallel_calls` argument\n",
+        "*   [Use the `cache` transformation](#caching) to cache data in memory during the first epoch\n",
+        "*   [Vectorize user-defined functions](#vectorizing_mapping) passed in to the `map` transformation\n",
+        "*   [Reduce memory usage](#reducing_memory_footprint) when applying the `interleave`, `prefetch`, and `shuffle` transformations"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mP_EMFsQ-jMU"
       },
       "source": [
         "## Reproducing the figures\n",
         "\n",
-        "Note: The rest of this notebook is about how to reproduce the above figures, feel free to play around with this code, but understanding it is not an essential part of this tutorial.\n",
+        "Note: The rest of this notebook is about how to reproduce the above figures. Feel free to play around with this code, but understanding it is not an essential part of this tutorial.\n",
         "\n",
         "To go deeper in the `tf.data.Dataset` API understanding, you can play with your own pipelines.\n",
         "Below is the code used to plot the images from this guide.\n",
         "It can be a good starting point, showing some workarounds for common difficulties such as:\n",
         "\n",
-        "- Execution time reproducibility;\n",
-        "- Mapped functions eager execution;\n",
-        "- `interleave` transformation callable."
+        "- Execution time reproducibility\n",
+        "- Mapped functions eager execution\n",
+        "- `interleave` transformation callable"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "7M_jFLer-jMV"
       },
       "outputs": [],
@@ -840,7 +771,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Z3pjnxtK-jMa"
       },
       "source": [
@@ -851,10 +781,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "OgGl4U7t-jMc"
       },
       "outputs": [],
@@ -901,7 +829,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "YQqDP4jk-jMj"
       },
       "source": [
@@ -928,7 +855,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IQK913bB-jMm"
       },
       "source": [
@@ -940,10 +866,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "zAy-K_Cq-jMn"
       },
       "outputs": [],
@@ -987,7 +911,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "jw_WSQC8-jMs"
       },
       "source": [
@@ -998,17 +921,15 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "1j73RxiP-jMw"
       },
       "outputs": [],
       "source": [
         "def draw_timeline(timeline, title, width=0.5, annotate=False, save=False):\n",
         "    # Remove invalid entries (negative times, or empty steps) from the timelines\n",
-        "    invalid_mask = np.logical_and(timeline['times'] \u003e 0, timeline['steps'] != b'')[:,0]\n",
+        "    invalid_mask = np.logical_and(timeline['times'] > 0, timeline['steps'] != b'')[:,0]\n",
         "    steps = timeline['steps'][invalid_mask].numpy()\n",
         "    times = timeline['times'][invalid_mask].numpy()\n",
         "    values = timeline['values'][invalid_mask].numpy()\n",
@@ -1057,7 +978,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "xto6GNdO-jM1"
       },
       "source": [
@@ -1068,10 +988,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "39v7JD4L-jM2"
       },
       "outputs": [],
@@ -1090,7 +1008,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "7eJRCinb-jM5"
       },
       "source": [
@@ -1099,10 +1016,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "YwX4ndHE-jM6"
       },
       "outputs": [],
@@ -1116,7 +1031,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "EwxJT2aR-jNA"
       },
       "source": [
@@ -1125,10 +1039,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "wLKgurx_-jNC"
       },
       "outputs": [],
@@ -1136,8 +1048,8 @@
         "@map_decorator\n",
         "def naive_map(steps, times, values):\n",
         "    map_enter = time.perf_counter()\n",
-        "    time.sleep(0.001)  # Time contumming step\n",
-        "    time.sleep(0.0001)  # Memory consumming step\n",
+        "    time.sleep(0.001)  # Time consuming step\n",
+        "    time.sleep(0.0001)  # Memory consuming step\n",
         "    map_elapsed = time.perf_counter() - map_enter\n",
         "\n",
         "    return (\n",
@@ -1159,7 +1071,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "EJqUMDsO-jNG"
       },
       "source": [
@@ -1168,18 +1079,16 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "HYHcwabr-jNH"
       },
       "outputs": [],
       "source": [
         "@map_decorator\n",
-        "def time_consumming_map(steps, times, values):\n",
+        "def time_consuming_map(steps, times, values):\n",
         "    map_enter = time.perf_counter()\n",
-        "    time.sleep(0.001 * values.shape[0])  # Time contumming step\n",
+        "    time.sleep(0.001 * values.shape[0])  # Time consuming step\n",
         "    map_elapsed = time.perf_counter() - map_enter\n",
         "\n",
         "    return (\n",
@@ -1190,9 +1099,9 @@
         "\n",
         "\n",
         "@map_decorator\n",
-        "def memory_consumming_map(steps, times, values):\n",
+        "def memory_consuming_map(steps, times, values):\n",
         "    map_enter = time.perf_counter()\n",
-        "    time.sleep(0.0001 * values.shape[0])  # Memory consumming step\n",
+        "    time.sleep(0.0001 * values.shape[0])  # Memory consuming step\n",
         "    map_elapsed = time.perf_counter() - map_enter\n",
         "\n",
         "    # Use tf.tile to handle batch dimension\n",
@@ -1207,22 +1116,22 @@
         "    tf.data.Dataset.range(2)\n",
         "    .interleave(  # Parallelize data reading\n",
         "        dataset_generator_fun,\n",
-        "        num_parallel_calls=tf.data.experimental.AUTOTUNE\n",
+        "        num_parallel_calls=tf.data.AUTOTUNE\n",
         "    )\n",
         "    .batch(  # Vectorize your mapped function\n",
         "        _batch_map_num_items,\n",
         "        drop_remainder=True)\n",
         "    .map(  # Parallelize map transformation\n",
-        "        time_consumming_map,\n",
-        "        num_parallel_calls=tf.data.experimental.AUTOTUNE\n",
+        "        time_consuming_map,\n",
+        "        num_parallel_calls=tf.data.AUTOTUNE\n",
         "    )\n",
         "    .cache()  # Cache data\n",
         "    .map(  # Reduce memory usage\n",
-        "        memory_consumming_map,\n",
-        "        num_parallel_calls=tf.data.experimental.AUTOTUNE\n",
+        "        memory_consuming_map,\n",
+        "        num_parallel_calls=tf.data.AUTOTUNE\n",
         "    )\n",
         "    .prefetch(  # Overlap producer and consumer works\n",
-        "        tf.data.experimental.AUTOTUNE\n",
+        "        tf.data.AUTOTUNE\n",
         "    )\n",
         "    .unbatch(),\n",
         "    5\n",
@@ -1231,10 +1140,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "b_CSUbxL-jNK"
       },
       "outputs": [],
@@ -1244,10 +1151,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "DoovY7qr-jNR"
       },
       "outputs": [],
@@ -1257,12 +1162,9 @@
     }
   ],
   "metadata": {
-    "celltoolbar": "Format de la Cellule Texte Brut",
     "colab": {
       "collapsed_sections": [],
       "name": "data_performance.ipynb",
-      "private_outputs": true,
-      "provenance": [],
       "toc_visible": true
     },
     "kernelspec": {
diff --git a/site/en/guide/data_performance_analysis.md b/site/en/guide/data_performance_analysis.md
new file mode 100644
index 00000000000..c99b7e4742b
--- /dev/null
+++ b/site/en/guide/data_performance_analysis.md
@@ -0,0 +1,311 @@
+# Analyze `tf.data` performance with the TF Profiler
+
+## Overview
+
+This guide assumes familiarity with the TensorFlow
+[Profiler](https://www.tensorflow.org/guide/profiler) and
+[`tf.data`](https://www.tensorflow.org/guide/data). It aims to provide step by
+step instructions with examples to help users diagnose and fix input pipeline
+performance issues.
+
+To begin, collect a profile of your TensorFlow job. Instructions on how to do so
+are available for
+[CPUs/GPUs](https://www.tensorflow.org/guide/profiler#collect_performance_data)
+and
+[Cloud TPUs](https://cloud.google.com/tpu/docs/cloud-tpu-tools#capture_profile).
+
+![TensorFlow Trace Viewer](images/data_performance_analysis/trace_viewer.png "The trace viewer page of the TensorFlow Profiler")
+
+The analysis workflow detailed below focuses on the trace viewer tool in the
+Profiler. This tool displays a timeline that shows the duration of ops executed
+by your TensorFlow program and allows you to identify which ops take the longest
+to execute. For more information on the trace viewer, check out
+[this section](https://www.tensorflow.org/guide/profiler#trace_viewer) of the TF
+Profiler guide. In general, `tf.data` events will appear on the host CPU
+timeline.
+
+## Analysis Workflow
+
+_Please follow the workflow below. If you have feedback to help us improve it,
+please
+[create a github issue](https://github.com/tensorflow/tensorflow/issues/new/choose)
+with the label “comp:data”._
+
+### 1. Is your `tf.data` pipeline producing data fast enough?
+
+Begin by ascertaining whether the input pipeline is the bottleneck for your
+TensorFlow program.
+
+To do so, look for `IteratorGetNext::DoCompute` ops in the trace viewer. In
+general, you expect to see these at the start of a step. These slices represent
+the time it takes for your input pipeline to yield a batch of elements when it
+is requested. If you’re using keras or iterating over your dataset in a
+`tf.function`, these should be found in `tf_data_iterator_get_next` threads.
+
+Note that if you’re using a
+[distribution strategy](https://www.tensorflow.org/guide/distributed_training),
+you may see `IteratorGetNextAsOptional::DoCompute` events instead of
+`IteratorGetNext::DoCompute`(as of TF 2.3).
+
+![image](images/data_performance_analysis/get_next_fast.png "If your IteratorGetNext::DoCompute calls return quickly, `tf.data` is not your bottleneck.")
+
+**If the calls return quickly (<= 50 us),** this means that your data is
+available when it is requested. The input pipeline is not your bottleneck; see
+the [Profiler guide](https://www.tensorflow.org/guide/profiler) for more generic
+performance analysis tips.
+
+![image](images/data_performance_analysis/get_next_slow.png "If your IteratorGetNext::DoCompute calls return slowly, `tf.data` is not producing data quickly enough.")
+
+**If the calls return slowly,** `tf.data` is unable to keep up with the
+consumer’s requests. Continue to the next section.
+
+### 2. Are you prefetching data?
+
+The best practice for input pipeline performance is to insert a
+`tf.data.Dataset.prefetch` transformation at the end of your `tf.data` pipeline.
+This transformation overlaps the input pipeline’s preprocessing computation with
+the next step of model computation and is required for optimal input pipeline
+performance when training your model. If you’re prefetching data, you should see
+a `Iterator::Prefetch` slice on the same thread as the
+`IteratorGetNext::DoCompute` op.
+
+![image](images/data_performance_analysis/prefetch.png "If you're prefetching data, you should see a `Iterator::Prefetch` slice in the same stack as the `IteratorGetNext::DoCompute` op.")
+
+**If you don’t have a `prefetch` at the end of your pipeline**, you should add
+one. For more information about `tf.data` performance recommendations, see the
+[tf.data performance guide](https://www.tensorflow.org/guide/data_performance#prefetching).
+
+**If you’re already prefetching data**, and the input pipeline is still your
+bottleneck, continue to the next section to further analyze performance.
+
+### 3. Are you reaching high CPU utilization?
+
+`tf.data` achieves high throughput by trying to make the best possible use of
+available resources. In general, even when running your model on an accelerator
+like a GPU or TPU, the `tf.data` pipelines are run on the CPU. You can check
+your utilization with tools like [sar](https://linux.die.net/man/1/sar) and
+[htop](https://en.wikipedia.org/wiki/Htop), or in the
+[cloud monitoring console](https://cloud.google.com/monitoring/docs/monitoring_in_console) if you’re running on GCP.
+
+**If your utilization is low,** this suggests that your input pipeline may not
+be taking full advantage of the host CPU. You should consult the
+[tf.data performance guide](https://www.tensorflow.org/guide/data_performance)
+for best practices. If you have applied the best practices and utilization and
+throughput remain low, continue to [Bottleneck analysis](#4_bottleneck_analysis)
+below.
+
+**If your utilization is approaching the resource limit**, in order to improve
+performance further, you need to either improve the efficiency of your input
+pipeline (for example, avoiding unnecessary computation) or offload computation.
+
+You can improve the efficiency of your input pipeline by avoiding unnecessary
+computation in `tf.data`. One way of doing this is inserting a
+[`tf.data.Dataset.cache`](https://www.tensorflow.org/guide/data_performance#caching)
+transformation after computation-intensive work if your data fits into memory;
+this reduces computation at the cost of increased memory usage. Additionally,
+disabling intra-op parallelism in `tf.data` has the potential to increase
+efficiency by > 10%, and can be done by setting the following option on your
+input pipeline:
+
+```python
+dataset = ...
+options = tf.data.Options()
+options.experimental_threading.max_intra_op_parallelism = 1
+dataset = dataset.with_options(options)
+```
+
+### 4. Bottleneck Analysis
+
+The following section walks through how to read `tf.data` events in the trace
+viewer to understand where the bottleneck is and possible mitigation strategies.
+
+#### Understanding `tf.data` events in the Profiler
+
+Each `tf.data` event in the Profiler has the name `Iterator::`, where
+`` is the name of the dataset source or transformation. Each event also
+has the long name `Iterator::::...::`, which you can see
+by clicking on the `tf.data` event. In the long name, `` matches
+`` from the (short) name, and the other datasets in the long name
+represent downstream transformations.
+
+![image](images/data_performance_analysis/map_long_name.png "tf.data.Dataset.range(10).map(lambda x: x).repeat(2).batch(5)")
+
+For example, the above screenshot was generated from the following code:
+
+```python
+dataset = tf.data.Dataset.range(10)
+dataset = dataset.map(lambda x: x)
+dataset = dataset.repeat(2)
+dataset = dataset.batch(5)
+```
+
+Here, the `Iterator::Map` event has the long name
+`Iterator::BatchV2::FiniteRepeat::Map`. Note that the datasets name may differ
+slightly from the python API (for example, FiniteRepeat instead of Repeat), but
+should be intuitive enough to parse.
+
+##### Synchronous and asynchronous transformations
+
+For synchronous `tf.data` transformations (such as `Batch` and `Map`), you will
+see events from upstream transformations on the same thread. In the above
+example, since all the transformations used are synchronous, all the events
+appear on the same thread.
+
+For asynchronous transformations (such as `Prefetch`, `ParallelMap`,
+`ParallelInterleave` and `MapAndBatch`) events from upstream transformations
+will be on a different thread. In such cases, the “long name” can help you
+identify which transformation in a pipeline an event corresponds to.
+
+![image](images/data_performance_analysis/async_long_name.png "tf.data.Dataset.range(10).map(lambda x: x).repeat(2).batch(5).prefetch(1)")
+
+For example, the above screenshot was generated from the following code:
+
+```python
+dataset = tf.data.Dataset.range(10)
+dataset = dataset.map(lambda x: x)
+dataset = dataset.repeat(2)
+dataset = dataset.batch(5)
+dataset = dataset.prefetch(1)
+```
+
+Here, the `Iterator::Prefetch` events are on the `tf_data_iterator_get_next`
+threads. Since `Prefetch` is asynchronous, its input events (`BatchV2`) will be
+on a different thread, and can be located by searching for the long name
+`Iterator::Prefetch::BatchV2`. In this case, they are on the
+`tf_data_iterator_resource` thread. From its long name, you can deduce that
+`BatchV2` is upstream of `Prefetch`. Furthermore, the `parent_id` of the
+`BatchV2` event will match the ID of the `Prefetch` event.
+
+#### Identifying the bottleneck
+
+In general, to identify the bottleneck in your input pipeline, walk the input
+pipeline from the outermost transformation all the way to the source. Starting
+from the final transformation in your pipeline, recurse into upstream
+transformations until you find a slow transformation or reach a source dataset,
+such as `TFRecord`. In the example above, you would start from `Prefetch`, then
+walk upstream to `BatchV2`, `FiniteRepeat`, `Map`, and finally `Range`.
+
+In general, a slow transformation corresponds to one whose events are long, but
+whose input events are short. Some examples follow below.
+
+Note that the final (outermost) transformation in most host input pipelines is
+the `Iterator::Model` event. The Model transformation is introduced
+automatically by the `tf.data` runtime and is used for instrumenting and
+autotuning the input pipeline performance.
+
+If your job is using a
+[distribution strategy](https://www.tensorflow.org/guide/distributed_training),
+the trace viewer will contain additional events that correspond to the device
+input pipeline. The outermost transformation of the device pipeline (nested
+under `IteratorGetNextOp::DoCompute` or
+`IteratorGetNextAsOptionalOp::DoCompute`) will be an `Iterator::Prefetch` event
+with an upstream `Iterator::Generator` event. You can find the corresponding
+host pipeline by searching for `Iterator::Model` events.
+
+##### Example 1
+
+![image](images/data_performance_analysis/example_1_cropped.png "Example 1")
+
+The above screenshot is generated from the following input pipeline:
+
+```python
+dataset = tf.data.TFRecordDataset(filename)
+dataset = dataset.map(parse_record)
+dataset = dataset.batch(32)
+dataset = dataset.repeat()
+```
+
+In the screenshot, observe that (1) `Iterator::Map` events are long, but (2) its
+input events (`Iterator::FlatMap`) return quickly. This suggests that the
+sequential Map transformation is the bottleneck.
+
+Note that in the screenshot, the `InstantiatedCapturedFunction::Run` event
+corresponds to the time it takes to execute the map function.
+
+##### Example 2
+
+![image](images/data_performance_analysis/example_2_cropped.png "Example 2")
+
+The above screenshot is generated from the following input pipeline:
+
+```python
+dataset = tf.data.TFRecordDataset(filename)
+dataset = dataset.map(parse_record, num_parallel_calls=2)
+dataset = dataset.batch(32)
+dataset = dataset.repeat()
+```
+
+This example is similar to the above, but uses ParallelMap instead of Map. We
+notice here that (1) `Iterator::ParallelMap` events are long, but (2) its input
+events `Iterator::FlatMap` (which are on a different thread, since ParallelMap
+is asynchronous) are short. This suggests that the ParallelMap transformation is
+the bottleneck.
+
+#### Addressing the bottleneck
+
+##### Source datasets
+
+If you’ve identified a dataset source as the bottleneck, such as reading from
+TFRecord files, you can improve performance by parallelizing data extraction. To
+do so, ensure that your data is sharded across multiple files and use
+`tf.data.Dataset.interleave` with the `num_parallel_calls` parameter set to
+`tf.data.AUTOTUNE`. If determinism is not important to your
+program, you can further improve performance by setting the
+`deterministic=False` flag on `tf.data.Dataset.interleave` as of TF 2.2. For
+example, if you’re reading from TFRecords, you can do the following:
+
+```python
+dataset = tf.data.Dataset.from_tensor_slices(filenames)
+dataset = dataset.interleave(tf.data.TFRecordDataset,
+  num_parallel_calls=tf.data.AUTOTUNE,
+  deterministic=False)
+```
+
+Note that sharded files should be reasonably large to amortize the overhead of
+opening a file. For more details on parallel data extraction, see
+[this section](https://www.tensorflow.org/guide/data_performance#parallelizing_data_extraction)
+of the `tf.data` performance guide.
+
+##### Transformation datasets
+
+If you’ve identified an intermediate `tf.data` transformation as the bottleneck,
+you can address it by parallelizing the transformation or
+[caching the computation](https://www.tensorflow.org/guide/data_performance#caching)
+if your data fits into memory and it is appropriate. Some transformations such
+as `Map` have parallel counterparts; the
+[`tf.data` performance guide demonstrates](https://www.tensorflow.org/guide/data_performance#parallelizing_data_transformation)
+how to parallelize these. Other transformations, such as `Filter`, `Unbatch`,
+and `Batch` are inherently sequential; you can parallelize them by introducing
+“outer parallelism”. For example, supposing your input pipeline initially looks
+like the following, with `Batch` as the bottleneck:
+
+```python
+filenames = tf.data.Dataset.list_files(file_path, shuffle=is_training)
+dataset = filenames_to_dataset(filenames)
+dataset = dataset.batch(batch_size)
+```
+
+You can introduce “outer parallelism” by running multiple copies of the input
+pipeline over sharded inputs and combining the results:
+
+```python
+filenames = tf.data.Dataset.list_files(file_path, shuffle=is_training)
+
+def make_dataset(shard_index):
+  filenames = filenames.shard(NUM_SHARDS, shard_index)
+  dataset = filenames_to_dataset(filenames)
+  Return dataset.batch(batch_size)
+
+indices = tf.data.Dataset.range(NUM_SHARDS)
+dataset = indices.interleave(make_dataset,
+                             num_parallel_calls=tf.data.AUTOTUNE)
+dataset = dataset.prefetch(tf.data.AUTOTUNE)
+```
+
+## Additional resources
+
+*   [tf.data performance guide](https://www.tensorflow.org/guide/data_performance)
+    on how to write performance `tf.data` input pipelines
+*   [Inside TensorFlow video: `tf.data` best practices ](https://www.youtube.com/watch?v=ZnukSLKEw34)
+*   [Profiler guide](https://www.tensorflow.org/guide/profiler)
+*   [Profiler tutorial with colab](https://www.tensorflow.org/tensorboard/tensorboard_profiling_keras)
diff --git a/site/en/guide/distributed_training.ipynb b/site/en/guide/distributed_training.ipynb
index c562e94cccf..04b7118b1f2 100644
--- a/site/en/guide/distributed_training.ipynb
+++ b/site/en/guide/distributed_training.ipynb
@@ -3,20 +3,17 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Tce3stUlHN0L"
       },
       "source": [
-        "##### Copyright 2018 The TensorFlow Authors.\n",
-        "\n"
+        "##### Copyright 2018 The TensorFlow Authors.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
+        "cellView": "form",
         "id": "tuOe1ymfHZPu"
       },
       "outputs": [],
@@ -37,7 +34,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MfBg1C5NB3X0"
       },
       "source": [
@@ -47,124 +43,117 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "r6P32iYYV27b"
       },
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/distributed_training\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/distributed_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/distributed_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/distributed_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "xHxb-dlhMIzW"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "`tf.distribute.Strategy` is a TensorFlow API to distribute training \n",
-        "across multiple GPUs, multiple machines or TPUs. Using this API, you can distribute your existing models and training code with minimal code changes.\n",
+        "`tf.distribute.Strategy` is a TensorFlow API to distribute training across multiple GPUs, multiple machines, or TPUs. Using this API, you can distribute your existing models and training code with minimal code changes.\n",
         "\n",
         "`tf.distribute.Strategy` has been designed with these key goals in mind:\n",
         "\n",
-        "* Easy to use and support multiple user segments, including researchers, ML engineers, etc.\n",
+        "* Easy to use and support multiple user segments, including researchers, machine learning engineers, etc.\n",
         "* Provide good performance out of the box.\n",
         "* Easy switching between strategies.\n",
         "\n",
-        "`tf.distribute.Strategy` can be used with a high-level API like [Keras](https://www.tensorflow.org/guide/keras), and can also be used to distribute custom training loops (and, in general, any computation using TensorFlow).\n",
+        "You can distribute training using `tf.distribute.Strategy` with a high-level API like Keras `Model.fit`, as well as [custom training loops](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) (and, in general, any computation using TensorFlow).\n",
         "\n",
-        "In TensorFlow 2.0, you can execute your programs eagerly, or in a graph using [`tf.function`](function.ipynb). `tf.distribute.Strategy` intends to support both these modes of execution. Although we discuss training most of the time in this guide, this API can also be used for distributing evaluation and prediction on different platforms.\n",
+        "In TensorFlow 2.x, you can execute your programs eagerly, or in a graph using [`tf.function`](function.ipynb). `tf.distribute.Strategy` intends to support both these modes of execution, but works best with `tf.function`. Eager mode is only recommended for debugging purposes and not supported for `tf.distribute.TPUStrategy`. Although training is the focus of this guide, this API can also be used for distributing evaluation and prediction on different platforms.\n",
         "\n",
-        "You can use `tf.distribute.Strategy` with very few changes to your code, because we have changed the underlying components of TensorFlow to become strategy-aware. This includes variables, layers, models, optimizers, metrics, summaries, and checkpoints.\n",
+        "You can use `tf.distribute.Strategy` with very few changes to your code, because the underlying components of TensorFlow have been changed to become strategy-aware. This includes variables, layers, models, optimizers, metrics, summaries, and checkpoints.\n",
         "\n",
-        "In this guide, we explain various types of strategies and how you can use them in different situations.\n",
+        "In this guide, you will learn about various types of strategies and how you can use them in different situations. To learn how to debug performance issues, check out the [Optimize TensorFlow GPU performance](gpu_performance_analysis.md) guide.\n",
         "\n",
-        "Note: For a deeper understanding of the concepts, please watch [this deep-dive presentation](https://youtu.be/jKV53r9-H14). This is especially recommended if you plan to write your own training loop.\n"
+        "Note: For a deeper understanding of the concepts, watch the deep-dive presentation—[Inside TensorFlow: `tf.distribute.Strategy`](https://youtu.be/jKV53r9-H14). This is especially recommended if you plan to write your own training loop.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b3600ee25c8e"
+      },
+      "source": [
+        "## Set up TensorFlow"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "EVOZFbNgXghB"
       },
       "outputs": [],
       "source": [
-        "# Import TensorFlow\n",
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
         "import tensorflow as tf"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "eQ1QESxxEbCh"
       },
       "source": [
         "## Types of strategies\n",
-        "`tf.distribute.Strategy` intends to cover a number of use cases along different axes. Some of these combinations are currently supported and others will be added in the future. Some of these axes are:\n",
         "\n",
-        "* *Synchronous vs asynchronous training:* These are two common ways of distributing training with data parallelism. In sync training, all workers train over different slices of input data in sync, and aggregating gradients at each step. In async training, all workers are independently training over the input data and updating variables asynchronously. Typically sync training is supported via all-reduce and async through parameter server architecture.\n",
-        "* *Hardware platform:* You may want to scale your training onto multiple GPUs on one machine, or multiple machines in a network (with 0 or more GPUs each), or on Cloud TPUs.\n",
+        "`tf.distribute.Strategy` intends to cover a number of use cases along different axes. Some of these combinations are currently supported and others will be added in the future. Some of these axes are:\n",
         "\n",
-        "In order to support these use cases, there are six strategies available. In the next section we explain which of these are supported in which scenarios in TF 2.0 at this time. Here is a quick overview:\n",
+        "- *Synchronous vs asynchronous training:* These are two common ways of distributing training with data parallelism. In sync training, all workers train over different slices of input data in sync, and aggregating gradients at each step. In async training, all workers are independently training over the input data and updating variables asynchronously. Typically sync training is supported via all-reduce and async through parameter server architecture.\n",
+        "- *Hardware platform:* You may want to scale your training onto multiple GPUs on one machine, or multiple machines in a network (with 0 or more GPUs each), or on Cloud TPUs.\n",
         "\n",
-        "| Training API          \t| MirroredStrategy  \t| TPUStrategy         \t| MultiWorkerMirroredStrategy     \t| CentralStorageStrategy          \t| ParameterServerStrategy  \t| OneDeviceStrategy |\n",
-        "|:-----------------------\t|:-------------------\t|:---------------------\t|:---------------------------------\t|:---------------------------------\t|:--------------------------\t| :-------------------------------------- |\n",
-        "| **Keras API**             \t| Supported         \t| Experimental support \t| Experimental support \t| Experimental support \t| Supported planned post 2.0 \t| Supported |\n",
-        "| **Custom training loop**  \t| Experimental support \t| Experimental support   \t| Support planned post 2.0             \t| Support planned post 2.0            \t| No support yet          \t| Supported |\n",
-        "| **Estimator API**         \t| Limited Support         \t| Not supported           \t| Limited Support                       \t| Limited Support                       \t| Limited Support                \t| Limited Support |\n",
+        "In order to support these use cases, TensorFlow has `MirroredStrategy`, `TPUStrategy`, `MultiWorkerMirroredStrategy`, `ParameterServerStrategy`, `CentralStorageStrategy`, as well as other strategies available. The next section explains which of these are supported in which scenarios in TensorFlow. Here is a quick overview:\n",
         "\n",
+        "| Training API             | `MirroredStrategy` | `TPUStrategy` | `MultiWorkerMirroredStrategy` | `CentralStorageStrategy` | `ParameterServerStrategy` |\n",
+        "| :----------------------- | :----------------- | :------------ | :---------------------------- | :----------------------- | :------------------------ |\n",
+        "| **Keras `Model.fit`**    | Supported          | Supported     | Supported                     | Experimental support     | Experimental support      |\n",
+        "| **Custom training loop** | Supported          | Supported     | Supported                     | Experimental support     | Experimental support      |\n",
+        "| **Estimator API**        | Limited Support    | Not supported | Limited Support               | Limited Support          | Limited Support           |\n",
         "\n",
+        "Note: [Experimental support](https://www.tensorflow.org/guide/versions#what_is_not_covered) means the APIs are not covered by any compatibility guarantees.\n",
         "\n",
-        "Note: Estimator support is limited. Basic training and evaluation are experimental, and advanced features—such as scaffold—are not implemented. We recommend using Keras or custom training loops if a use case is not covered."
+        "Warning: Estimator support is limited. Basic training and evaluation are experimental, and advanced features—such as scaffold—are not implemented. You should be using Keras or custom training loops if a use case is not covered. Estimators are not recommended for new code. Estimators run `v1.Session`-style code which is more difficult to write correctly, and can behave unexpectedly, especially when combined with TF 2 code. Estimators do fall under our [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities. Go to the [migration guide](https://tensorflow.org/guide/migrate) for details."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "DoQKKK8dtfg6"
       },
       "source": [
         "### MirroredStrategy\n",
+        "\n",
         "`tf.distribute.MirroredStrategy` supports synchronous distributed training on multiple GPUs on one machine. It creates one replica per GPU device. Each variable in the model is mirrored across all the replicas. Together, these variables form a single conceptual variable called `MirroredVariable`. These variables are kept in sync with each other by applying identical updates.\n",
         "\n",
-        "Efficient all-reduce algorithms are used to communicate the variable updates across the devices.\n",
-        "All-reduce aggregates tensors across all the devices by adding them up, and makes them available on each device.\n",
-        "It’s a fused algorithm that is very efficient and can reduce the overhead of synchronization significantly. There are many all-reduce algorithms and implementations available, depending on the type of communication available between devices. By default, it uses NVIDIA NCCL as the all-reduce implementation. You can choose from a few other options we provide, or write your own.\n",
+        "Efficient all-reduce algorithms are used to communicate the variable updates across the devices. All-reduce aggregates tensors across all the devices by adding them up, and makes them available on each device. It’s a fused algorithm that is very efficient and can reduce the overhead of synchronization significantly. There are many all-reduce algorithms and implementations available, depending on the type of communication available between devices. By default, it uses the NVIDIA Collective Communication Library ([NCCL](https://developer.nvidia.com/nccl)) as the all-reduce implementation. You can choose from a few other options or write your own.\n",
         "\n",
-        "Here is the simplest way of creating `MirroredStrategy`:\n"
+        "Here is the simplest way of creating `MirroredStrategy`:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "9Z4FMAY9ADxK"
       },
       "outputs": [],
@@ -175,21 +164,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wldY4aFCAH4r"
       },
       "source": [
-        "This will create a `MirroredStrategy` instance which will use all the GPUs that are visible to TensorFlow, and use NCCL as the cross device communication.\n",
+        "This will create a `MirroredStrategy` instance, which will use all the GPUs that are visible to TensorFlow, and NCCL—as the cross-device communication.\n",
         "\n",
         "If you wish to use only some of the GPUs on your machine, you can do so like this:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "nbGleskCACv_"
       },
       "outputs": [],
@@ -200,19 +186,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8-KDnrJLAhav"
       },
       "source": [
-        "If you wish to override the cross device communication, you can do so using the `cross_device_ops` argument by supplying an instance of `tf.distribute.CrossDeviceOps`. Currently,  `tf.distribute.HierarchicalCopyAllReduce` and `tf.distribute.ReductionToOneDevice` are two options other than `tf.distribute.NcclAllReduce` which is the default."
+        "If you wish to override the cross device communication, you can do so using the `cross_device_ops` argument by supplying an instance of `tf.distribute.CrossDeviceOps`. Currently, `tf.distribute.HierarchicalCopyAllReduce` and `tf.distribute.ReductionToOneDevice` are two options other than `tf.distribute.NcclAllReduce`, which is the default."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "6-xIOIpgBItn"
       },
       "outputs": [],
@@ -224,283 +207,344 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "45H0Wa8WKI8z"
+        "id": "kPEBCMzsGaO5"
       },
       "source": [
-        "### CentralStorageStrategy\n",
-        "`tf.distribute.experimental.CentralStorageStrategy` does synchronous training as well. Variables are not mirrored, instead they are placed on the CPU and operations are replicated across all local GPUs. If there is only one GPU, all variables and operations will be placed on that GPU.\n",
+        "### TPUStrategy\n",
         "\n",
-        "Create an instance of `CentralStorageStrategy` by:\n"
+        "`tf.distribute.TPUStrategy` lets you run your TensorFlow training on [Tensor Processing Units (TPUs)](tpu.ipynb). TPUs are Google's specialized ASICs designed to dramatically accelerate machine learning workloads. They are available on [Google Colab](https://colab.research.google.com/), the [TPU Research Cloud](https://sites.research.google/trc/), and [Cloud TPU](https://cloud.google.com/tpu).\n",
+        "\n",
+        "In terms of distributed training architecture, `TPUStrategy` is the same `MirroredStrategy`—it implements synchronous distributed training. TPUs provide their own implementation of efficient all-reduce and other collective operations across multiple TPU cores, which are used in `TPUStrategy`.\n",
+        "\n",
+        "Here is how you would instantiate `TPUStrategy`:\n",
+        "\n",
+        "Note: To run any TPU code in Colab, you should select TPU as the Colab runtime. Refer to the [Use TPUs](tpu.ipynb) guide for a complete example.\n",
+        "\n",
+        "```python\n",
+        "cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(\n",
+        "    tpu=tpu_address)\n",
+        "tf.config.experimental_connect_to_cluster(cluster_resolver)\n",
+        "tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n",
+        "tpu_strategy = tf.distribute.TPUStrategy(cluster_resolver)\n",
+        "```\n",
+        "\n",
+        "The `TPUClusterResolver` instance helps locate the TPUs. In Colab, you don't need to specify any arguments to it.\n",
+        "\n",
+        "If you want to use this for Cloud TPUs:\n",
+        "\n",
+        "- You must specify the name of your TPU resource in the `tpu` argument.\n",
+        "- You must initialize the TPU system explicitly at the *start* of the program. This is required before TPUs can be used for computation. Initializing the TPU system also wipes out the TPU memory, so it's important to complete this step first in order to avoid losing state."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8Xc3gyo0Bejd"
+      },
+      "source": [
+        "### MultiWorkerMirroredStrategy\n",
+        "\n",
+        "`tf.distribute.MultiWorkerMirroredStrategy` is very similar to `MirroredStrategy`. It implements synchronous distributed training across multiple workers, each with potentially multiple GPUs. Similar to `tf.distribute.MirroredStrategy`, it creates copies of all variables in the model on each device across all workers.\n",
+        "\n",
+        "Here is the simplest way of creating `MultiWorkerMirroredStrategy`:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "rtjZOyaoMWrP"
+        "id": "m3a_6ebbEjre"
       },
       "outputs": [],
       "source": [
-        "central_storage_strategy = tf.distribute.experimental.CentralStorageStrategy()"
+        "strategy = tf.distribute.MultiWorkerMirroredStrategy()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "KY1nJHNkMl7b"
+        "id": "bt94JBvhEr4s"
       },
       "source": [
-        "This will create a `CentralStorageStrategy` instance which will use all visible GPUs and CPU. Update to variables on replicas will be aggregated before being applied to variables."
+        "`MultiWorkerMirroredStrategy` has two implementations for cross-device communications. `CommunicationImplementation.RING` is [RPC](https://en.wikipedia.org/wiki/Remote_procedure_call)-based and supports both CPUs and GPUs. `CommunicationImplementation.NCCL` uses NCCL and provides state-of-art performance on GPUs but it doesn't support CPUs. `CollectiveCommunication.AUTO` defers the choice to Tensorflow. You can specify them in the following way:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QGX_QAEtFQSv"
+      },
+      "outputs": [],
+      "source": [
+        "communication_options = tf.distribute.experimental.CommunicationOptions(\n",
+        "    implementation=tf.distribute.experimental.CommunicationImplementation.NCCL)\n",
+        "strategy = tf.distribute.MultiWorkerMirroredStrategy(\n",
+        "    communication_options=communication_options)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "aAFycYUiNCUb"
+        "id": "0JiImlw3F77E"
       },
       "source": [
-        "Note: This strategy is [`experimental`](https://www.tensorflow.org/guide/versions#what_is_not_covered) as we are currently improving it and making it work for more scenarios. As part of this, please expect the APIs to change in the future."
+        "One of the key differences to get multi worker training going, as compared to multi-GPU training, is the multi-worker setup. The `'TF_CONFIG'` environment variable is the standard way in TensorFlow to specify the cluster configuration to each worker that is part of the cluster. Learn more in the [setting up TF_CONFIG section](#TF_CONFIG) of this document.\n",
+        "\n",
+        "For more details about `MultiWorkerMirroredStrategy`, consider the following tutorials:\n",
+        "\n",
+        "- [Multi-worker training with Keras Model.fit](../tutorials/distribute/multi_worker_with_keras.ipynb)\n",
+        "- [Multi-worker training with a custom training loop](../tutorials/distribute/multi_worker_with_ctl.ipynb)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "8Xc3gyo0Bejd"
+        "id": "3ZLBhaP9NUNr"
       },
       "source": [
-        "### MultiWorkerMirroredStrategy\n",
+        "### ParameterServerStrategy\n",
         "\n",
-        "`tf.distribute.experimental.MultiWorkerMirroredStrategy` is very similar to `MirroredStrategy`. It implements synchronous distributed training across multiple workers, each with potentially multiple GPUs. Similar to `MirroredStrategy`, it creates copies of all variables in the model on each device across all workers.\n",
+        "Parameter server training is a common data-parallel method to scale up model training on multiple machines. A parameter server training cluster consists of workers and parameter servers. Variables are created on parameter servers and they are read and updated by workers in each step. Check out the [Parameter server training](../tutorials/distribute/parameter_server_training.ipynb) tutorial for details.\n",
         "\n",
-        "It uses [CollectiveOps](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/collective_ops.py) as the multi-worker all-reduce communication method used to keep variables in sync. A collective op is a single op in the TensorFlow graph which can automatically choose an all-reduce algorithm in the TensorFlow runtime according to hardware, network topology and tensor sizes.\n",
+        "In TensorFlow 2, parameter server training uses a central coordinator-based architecture via the `tf.distribute.experimental.coordinator.ClusterCoordinator` class.\n",
         "\n",
-        "It also implements additional performance optimizations. For example, it includes a static optimization that converts multiple all-reductions on small tensors into fewer all-reductions on larger tensors. In addition, we are designing it to have a plugin architecture - so that in the future, you will be able to plugin algorithms that are better tuned for your hardware. Note that collective ops also implement other collective operations such as broadcast and all-gather.\n",
+        "In this implementation, the `worker` and `parameter server` tasks run `tf.distribute.Server`s that listen for tasks from the coordinator. The coordinator creates resources, dispatches training tasks, writes checkpoints, and deals with task failures.\n",
         "\n",
-        "Here is the simplest way of creating `MultiWorkerMirroredStrategy`:"
+        "In the programming running on the coordinator, you will use a `ParameterServerStrategy` object to define a training step and use a `ClusterCoordinator` to dispatch training steps to remote workers. Here is the simplest way to create them:\n",
+        "\n",
+        "```python\n",
+        "strategy = tf.distribute.experimental.ParameterServerStrategy(\n",
+        "    tf.distribute.cluster_resolver.TFConfigClusterResolver(),\n",
+        "    variable_partitioner=variable_partitioner)\n",
+        "coordinator = tf.distribute.experimental.coordinator.ClusterCoordinator(\n",
+        "    strategy)\n",
+        "```\n",
+        "\n",
+        "To learn more about `ParameterServerStrategy`, check out the [Parameter server training with Keras Model.fit and a custom training loop](../tutorials/distribute/parameter_server_training.ipynb) tutorial.\n",
+        "\n",
+        "Note: You will need to configure the `'TF_CONFIG'` environment variable if you use `TFConfigClusterResolver`. It is similar to [`'TF_CONFIG'`](#TF_CONFIG) in `MultiWorkerMirroredStrategy` but has additional caveats.\n",
+        "\n",
+        "In TensorFlow 1, `ParameterServerStrategy` is available only with an Estimator via `tf.compat.v1.distribute.experimental.ParameterServerStrategy` symbol."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "m3a_6ebbEjre"
+        "id": "E20tG21LFfv1"
       },
-      "outputs": [],
       "source": [
-        "multiworker_strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()"
+        "Note: This strategy is [`experimental`](https://www.tensorflow.org/guide/versions#what_is_not_covered) as it is currently under active development."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "bt94JBvhEr4s"
+        "id": "45H0Wa8WKI8z"
       },
       "source": [
-        "`MultiWorkerMirroredStrategy` currently allows you to choose between two different implementations of collective ops.  `CollectiveCommunication.RING` implements ring-based collectives using gRPC as the communication layer.  `CollectiveCommunication.NCCL` uses [Nvidia's NCCL](https://developer.nvidia.com/nccl) to implement collectives.  `CollectiveCommunication.AUTO` defers the choice to the runtime.  The best choice of collective implementation depends upon the number and kind of GPUs, and the network interconnect in the cluster. You can specify them in the following way:\n"
+        "### CentralStorageStrategy\n",
+        "`tf.distribute.experimental.CentralStorageStrategy` does synchronous training as well. Variables are not mirrored, instead they are placed on the CPU and operations are replicated across all local GPUs. If there is only one GPU, all variables and operations will be placed on that GPU.\n",
+        "\n",
+        "Create an instance of `CentralStorageStrategy` by:\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "QGX_QAEtFQSv"
+        "id": "rtjZOyaoMWrP"
       },
       "outputs": [],
       "source": [
-        "multiworker_strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy(\n",
-        "    tf.distribute.experimental.CollectiveCommunication.NCCL)"
+        "central_storage_strategy = tf.distribute.experimental.CentralStorageStrategy()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "0JiImlw3F77E"
+        "id": "KY1nJHNkMl7b"
       },
       "source": [
-        "\n",
-        "One of the key differences to get multi worker training going, as compared to multi-GPU training, is the multi-worker setup. The `TF_CONFIG` environment variable is the standard way in TensorFlow to specify the cluster configuration to each worker that is part of the cluster. Learn more about [setting up TF_CONFIG](#TF_CONFIG)."
+        "This will create a `CentralStorageStrategy` instance which will use all visible GPUs and CPU. Update to variables on replicas will be aggregated before being applied to variables."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "E20tG21LFfv1"
+        "id": "aAFycYUiNCUb"
       },
       "source": [
-        "Note: This strategy is [`experimental`](https://www.tensorflow.org/guide/versions#what_is_not_covered) as we are currently improving it and making it work for more scenarios. As part of this, please expect the APIs to change in the future."
+        "Note: This strategy is [`experimental`](https://www.tensorflow.org/guide/versions#what_is_not_covered), as it is currently a work in progress."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "kPEBCMzsGaO5"
+        "id": "t2XUdmIxKljq"
       },
       "source": [
-        "### TPUStrategy\n",
-        "`tf.distribute.experimental.TPUStrategy` lets you run your TensorFlow training on Tensor Processing Units (TPUs). TPUs are Google's specialized ASICs designed to dramatically accelerate machine learning workloads. They are available on Google Colab, the [TensorFlow Research Cloud](https://www.tensorflow.org/tfrc) and [Cloud TPU](https://cloud.google.com/tpu).\n",
+        "### Other strategies\n",
         "\n",
-        "In terms of distributed training architecture, `TPUStrategy` is the same `MirroredStrategy` - it implements synchronous distributed training. TPUs provide their own implementation of efficient all-reduce and other collective operations across multiple TPU cores, which are used in `TPUStrategy`.\n",
-        "\n",
-        "Here is how you would instantiate `TPUStrategy`:\n",
-        "\n",
-        "Note: To run this code in Colab, you should select TPU as the Colab runtime. We will have a tutorial soon that will demonstrate how you can use TPUStrategy.\n",
-        "\n",
-        "```\n",
-        "cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(\n",
-        "    tpu=tpu_address)\n",
-        "tf.config.experimental_connect_to_cluster(cluster_resolver)\n",
-        "tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n",
-        "tpu_strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver)\n",
-        "```\n"
+        "In addition to the above strategies, there are two other strategies which might be useful for prototyping and debugging when using `tf.distribute` APIs."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "oQ7EqjpmK6DU"
+        "id": "UD5I1beTpc7a"
       },
       "source": [
-        "The `TPUClusterResolver` instance helps locate the TPUs. In Colab, you don't need to specify any arguments to it.\n",
+        "#### Default Strategy\n",
         "\n",
-        "If you want to use this for Cloud TPUs:\n",
-        "- You must specify the name of your TPU resource in the `tpu` argument.\n",
-        "- You must initialize the tpu system explicitly at the *start* of the program. This is required before TPUs can be used for computation. Initializing the tpu system also wipes out the TPU memory, so it's important to complete this step first in order to avoid losing state."
+        "The Default Strategy is a distribution strategy which is present when no explicit distribution strategy is in scope. It implements the `tf.distribute.Strategy` interface but is a pass-through and provides no actual distribution. For instance, `Strategy.run(fn)` will simply call `fn`. Code written using this strategy should behave exactly as code written without any strategy. You can think of it as a \"no-op\" strategy.\n",
+        "\n",
+        "The Default Strategy is a singleton—and one cannot create more instances of it. It can be obtained using `tf.distribute.get_strategy` outside any explicit strategy's scope (the same API that can be used to get the current strategy inside an explicit strategy's scope)."
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "jARHpraJMbRa"
+        "id": "ibHleFOOmPn9"
       },
+      "outputs": [],
       "source": [
-        "Note: This strategy is [`experimental`](https://www.tensorflow.org/guide/versions#what_is_not_covered) as we are currently improving it and making it work for more scenarios. As part of this, please expect the APIs to change in the future."
+        "default_strategy = tf.distribute.get_strategy()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "3ZLBhaP9NUNr"
+        "id": "EkxPl_5ImLzc"
       },
       "source": [
-        "### ParameterServerStrategy\n",
-        "`tf.distribute.experimental.ParameterServerStrategy` supports parameter servers training on multiple machines. In this setup, some machines are designated as workers and some as parameter servers. Each variable of the model is placed on one parameter server. Computation is replicated across all GPUs of all the workers.\n",
+        "This strategy serves two main purposes:\n",
         "\n",
-        "In terms of code, it looks similar to other strategies:\n",
-        "```\n",
-        "ps_strategy = tf.distribute.experimental.ParameterServerStrategy()\n",
-        "```"
+        "* It allows writing distribution-aware library code unconditionally. For example, in `tf.keras.optimizers` you can use `tf.distribute.get_strategy` and use that strategy for reducing gradients—it will always return a strategy object on which you can call the `Strategy.reduce` API.\n"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "zr1wPHYvOH0N"
+        "id": "WECeRzUdT6bU"
       },
+      "outputs": [],
       "source": [
-        "For multi worker training, `TF_CONFIG` needs to specify the configuration of parameter servers and workers in your cluster, which you can read more about in [TF_CONFIG below](#TF_CONFIG) below."
+        "# In optimizer or other library code\n",
+        "# Get currently active strategy\n",
+        "strategy = tf.distribute.get_strategy()\n",
+        "strategy.reduce(\"SUM\", 1., axis=None)  # reduce some values"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "t2XUdmIxKljq"
+        "id": "JURbH-pUT51B"
       },
       "source": [
-        "### OneDeviceStrategy\n",
+        "* Similar to library code, it can be used to write end users' programs to work with and without distribution strategy, without requiring conditional logic. Here's a sample code snippet illustrating this:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "O4Vmae5jmSE6"
+      },
+      "outputs": [],
+      "source": [
+        "if tf.config.list_physical_devices('GPU'):\n",
+        "  strategy = tf.distribute.MirroredStrategy()\n",
+        "else:  # Use the Default Strategy\n",
+        "  strategy = tf.distribute.get_strategy()\n",
         "\n",
-        "`tf.distribute.OneDeviceStrategy` runs on a single device. This strategy will place any variables created in its scope on the specified device. Input distributed through this strategy will be prefetched to the specified device. Moreover, any functions called via `strategy.experimental_run_v2` will also be placed on the specified device.\n",
+        "with strategy.scope():\n",
+        "  # Do something interesting\n",
+        "  print(tf.Variable(1.))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kTzsqN4lmJ0d"
+      },
+      "source": [
+        "#### OneDeviceStrategy\n",
         "\n",
-        "You can use this strategy to test your code before switching to other strategies which actually distributes to multiple devices/machines.\n",
+        "`tf.distribute.OneDeviceStrategy` is a strategy to place all variables and computation on a single specified device.\n",
         "\n",
-        "```\n",
+        "```python\n",
         "strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n",
         "```\n",
-        "\n"
+        "\n",
+        "This strategy is distinct from the Default Strategy in a number of ways. In the Default Strategy, the variable placement logic remains unchanged when compared to running TensorFlow without any distribution strategy. But when using `OneDeviceStrategy`, all variables created in its scope are explicitly placed on the specified device. Moreover, any functions called via `OneDeviceStrategy.run` will also be placed on the specified device.\n",
+        "\n",
+        "Input distributed through this strategy will be prefetched to the specified device. In the Default Strategy, there is no input distribution.\n",
+        "\n",
+        "Similar to the Default Strategy, this strategy could also be used to test your code before switching to other strategies which actually distribute to multiple devices/machines. This will exercise the distribution strategy machinery somewhat more than the Default Strategy, but not to the full extent of using, for example, `MirroredStrategy` or `TPUStrategy`. If you want code that behaves as if there is no strategy, then use the Default Strategy."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hQv1lm9UPDFy"
       },
       "source": [
-        "\n",
-        "So far we've talked about what are the different strategies available and how you can instantiate them. In the next few sections, we will talk about the different ways in which you can use them to distribute your training. We will show short code snippets in this guide and link off to full tutorials which you can run end to end."
+        "So far you've learned about different strategies and how you can instantiate them. The next few sections show the different ways in which you can use them to distribute your training."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "_mcuy3UhPcen"
       },
       "source": [
-        "## Using `tf.distribute.Strategy` with Keras\n",
-        "We've integrated `tf.distribute.Strategy` into `tf.keras` which is TensorFlow's implementation of the\n",
-        "[Keras API specification](https://keras.io). `tf.keras`  is a high-level API to build and train models. By integrating into `tf.keras` backend, we've made it seamless for you to distribute your training written in the Keras training framework.\n",
+        "## Use tf.distribute.Strategy with Keras Model.fit\n",
+        "\n",
+        "`tf.distribute.Strategy` is integrated into `tf.keras`, which is TensorFlow's implementation of the [Keras API specification](https://keras.io/api/). `tf.keras` is a high-level API to build and train models. By integrating into the `tf.keras` backend, it's seamless for you to distribute your training written in the Keras training framework [using Model.fit](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit).\n",
         "\n",
         "Here's what you need to change in your code:\n",
         "\n",
-        "1. Create an instance of the appropriate `tf.distribute.Strategy`\n",
-        "2. Move the creation and compiling of Keras model inside `strategy.scope`.\n",
+        "1. Create an instance of the appropriate `tf.distribute.Strategy`.\n",
+        "2. Move the creation of Keras model, optimizer and metrics inside `strategy.scope`. Thus the code in the model's `call()`, `train_step()`, and `test_step()` methods will all be distributed and executed on the accelerator(s).\n",
         "\n",
-        "We support all types of Keras models - sequential, functional and subclassed.\n",
+        "TensorFlow distribution strategies support all types of Keras models—[Sequential](https://www.tensorflow.org/guide/keras/sequential_model), [Functional](https://www.tensorflow.org/guide/keras/functional), and [subclassed](https://www.tensorflow.org/guide/keras/custom_layers_and_models)\n",
         "\n",
-        "Here is a snippet of code to do this for a very simple Keras model with one dense layer:"
+        "Here is a snippet of code to do this for a very simple Keras model with one `Dense` layer:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "gbbcpzRnPZ6V"
       },
       "outputs": [],
       "source": [
         "mirrored_strategy = tf.distribute.MirroredStrategy()\n",
+        "\n",
         "with mirrored_strategy.scope():\n",
-        "  model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
+        "  model = tf.keras.Sequential([\n",
+        "      tf.keras.layers.Dense(1, input_shape=(1,),\n",
+        "                            kernel_regularizer=tf.keras.regularizers.L2(1e-4))])\n",
         "  model.compile(loss='mse', optimizer='sgd')"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "773EOxCRVlTg"
       },
       "source": [
-        "In this example we used `MirroredStrategy` so we can run this on a machine with multiple GPUs. `strategy.scope()` indicated which parts of the code to run distributed. Creating a model inside this scope allows us to create mirrored variables instead of regular variables. Compiling under the scope allows us to know that the user intends to train this model using this strategy. Once this is set up, you can fit your model like you would normally. `MirroredStrategy` takes care of replicating the model's training on the available GPUs, aggregating gradients, and more."
+        "This example uses `MirroredStrategy`, so you can run this on a machine with multiple GPUs. `strategy.scope()` indicates to Keras which strategy to use to distribute the training. Creating models/optimizers/metrics inside this scope allows you to create distributed variables instead of regular variables. Once this is set up, you can fit your model like you would normally. `MirroredStrategy` takes care of replicating the model's training on the available GPUs, aggregating gradients, and more."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ZMmxEFRTEjH5"
       },
       "outputs": [],
@@ -513,24 +557,22 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "nofTLwyXWHK8"
       },
       "source": [
-        "Here we used a `tf.data.Dataset` to provide the training and eval input. You can also use numpy arrays:"
+        "Here a `tf.data.Dataset` provides the training and eval input. You can also use NumPy arrays:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Lqgd9SdxW5OW"
       },
       "outputs": [],
       "source": [
         "import numpy as np\n",
+        "\n",
         "inputs, targets = np.ones((100, 1)), np.ones((100, 1))\n",
         "model.fit(inputs, targets, epochs=2, batch_size=10)"
       ]
@@ -538,120 +580,120 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IKqaj7QwX0Zb"
       },
       "source": [
-        "In both cases (dataset or numpy), each batch of the given input is divided equally among the multiple replicas. For instance, if using `MirroredStrategy` with 2 GPUs, each batch of size 10 will get divided among the 2 GPUs, with each receiving 5 input examples in each step. Each epoch will then train faster as you add more GPUs. Typically, you would want to increase your batch size as you add more accelerators so as to make effective use of the extra computing power. You will also need to re-tune your learning rate, depending on the model. You can use `strategy.num_replicas_in_sync` to get the number of replicas."
+        "In both cases—with `Dataset` or NumPy—each batch of the given input is divided equally among the multiple replicas. For instance, if you are using the `MirroredStrategy` with 2 GPUs, each batch of size 10 will be divided among the 2 GPUs, with each receiving 5 input examples in each step. Each epoch will then train faster as you add more GPUs. Typically, you would want to increase your batch size as you add more accelerators, so as to make effective use of the extra computing power. You will also need to re-tune your learning rate, depending on the model. You can use `strategy.num_replicas_in_sync` to get the number of replicas."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
+      "metadata": {
+        "id": "8ZmJqErtS4A1"
+      },
+      "outputs": [],
+      "source": [
+        "mirrored_strategy.num_replicas_in_sync"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "quNNTytWdGBf"
       },
       "outputs": [],
       "source": [
-        "# Compute global batch size using number of replicas.\n",
+        "# Compute a global batch size using a number of replicas.\n",
         "BATCH_SIZE_PER_REPLICA = 5\n",
         "global_batch_size = (BATCH_SIZE_PER_REPLICA *\n",
         "                     mirrored_strategy.num_replicas_in_sync)\n",
         "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100)\n",
         "dataset = dataset.batch(global_batch_size)\n",
         "\n",
-        "LEARNING_RATES_BY_BATCH_SIZE = {5: 0.1, 10: 0.15}\n",
+        "LEARNING_RATES_BY_BATCH_SIZE = {5: 0.1, 10: 0.15, 20:0.175}\n",
         "learning_rate = LEARNING_RATES_BY_BATCH_SIZE[global_batch_size]"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "z1Muy0gDZwO5"
       },
       "source": [
         "### What's supported now?\n",
         "\n",
-        "In TF 2.0 release, `MirroredStrategy`, `TPUStrategy`, `CentralStorageStrategy` and `MultiWorkerMirroredStrategy` are supported in Keras. Except `MirroredStrategy`, others are currently experimental and are subject to change.\n",
-        "Support for other strategies will be coming soon. The API and how to use will be exactly the same as above.\n",
-        "\n",
-        "| Training API \t| MirroredStrategy  \t| TPUStrategy         \t| MultiWorkerMirroredStrategy     \t| CentralStorageStrategy          \t| ParameterServerStrategy \t| OneDeviceStrategy |\n",
-        "|----------------\t|---------------------\t|-----------------------\t|-----------------------------------\t|-----------------------------------\t|---------------------------\t| --------------------------------------- |\n",
-        "| Keras APIs   \t| Supported \t| Experimental support \t| Experimental support \t| Experimental support \t| Support planned post 2.0   \t| Supported |\n",
+        "| Training API      | `MirroredStrategy` | `TPUStrategy` | `MultiWorkerMirroredStrategy` | `ParameterServerStrategy` | `CentralStorageStrategy` |\n",
+        "| ----------------- | ------------------ | ------------- | ----------------------------- | ------------------------- | ------------------------ |\n",
+        "| Keras `Model.fit` | Supported          | Supported     | Supported                     | Experimental support      | Experimental support     |\n",
         "\n",
-        "### Examples and Tutorials\n",
+        "### Examples and tutorials\n",
         "\n",
-        "Here is a list of tutorials and examples that illustrate the above integration end to end with Keras:\n",
+        "Here is a list of tutorials and examples that illustrate the above integration end-to-end with Keras `Model.fit`:\n",
         "\n",
-        "1. Tutorial to train [MNIST](../tutorials/distribute/keras.ipynb) with `MirroredStrategy`.\n",
-        "2. Official [ResNet50](https://github.com/tensorflow/models/blob/master/official/vision/image_classification/resnet_imagenet_main.py) training with ImageNet data using `MirroredStrategy`.\n",
-        "3. [ResNet50](https://github.com/tensorflow/tpu/blob/master/models/experimental/resnet50_keras/resnet50_tf2.py) trained with Imagenet data on Cloud TPUs with `TPUStrategy`.\n",
-        "4. [Tutorial](../tutorials/distribute/multi_worker_with_keras.ipynb) to train MNIST using `MultiWorkerMirroredStrategy`.\n",
-        "5. [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) trained using `MirroredStrategy`.\n",
-        "2. [Transformer]( https://github.com/tensorflow/models/blob/master/official/transformer/v2/transformer_main.py) trained using `MirroredStrategy`."
+        "1. [Tutorial](../tutorials/distribute/keras.ipynb): Training with `Model.fit` and `MirroredStrategy`.\n",
+        "2. [Tutorial](../tutorials/distribute/multi_worker_with_keras.ipynb): Training with `Model.fit` and `MultiWorkerMirroredStrategy`.\n",
+        "3. [Guide](tpu.ipynb): Contains an example of using `Model.fit` and `TPUStrategy`.\n",
+        "4. [Tutorial](../tutorials/distribute/parameter_server_training.ipynb): Parameter server training with `Model.fit` and `ParameterServerStrategy`.\n",
+        "5. [Tutorial](https://www.tensorflow.org/text/tutorials/bert_glue): Fine-tuning BERT for many tasks from the GLUE benchmark with `Model.fit` and `TPUStrategy`.\n",
+        "6. TensorFlow Model Garden [repository](https://github.com/tensorflow/models/tree/master/official) containing collections of state-of-the-art models implemented using various strategies."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IlYVC0goepdk"
       },
       "source": [
-        "## Using `tf.distribute.Strategy` with custom training loops\n",
-        "As you've seen, using `tf.distribute.Strategy` with high-level APIs (Estimator and Keras) requires changing only a couple lines of your code. With a little more effort, you can also use `tf.distribute.Strategy` with custom training loops.\n",
+        "## Use tf.distribute.Strategy with custom training loops\n",
+        "\n",
+        "As demonstrated above, using `tf.distribute.Strategy` with Keras `Model.fit` requires changing only a couple lines of your code. With a little more effort, you can also use `tf.distribute.Strategy` [with custom training loops](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch).\n",
         "\n",
         "If you need more flexibility and control over your training loops than is possible with Estimator or Keras, you can write custom training loops. For instance, when using a GAN, you may want to take a different number of generator or discriminator steps each round. Similarly, the high level frameworks are not very suitable for Reinforcement Learning training.\n",
         "\n",
-        "To support custom training loops, we provide a core set of methods through the `tf.distribute.Strategy` classes. Using these may require minor restructuring of the code initially, but once that is done, you should be able to switch between GPUs, TPUs, and multiple machines simply by changing the strategy instance.\n",
+        "The `tf.distribute.Strategy` classes provide a core set of methods to support custom training loops. Using these may require minor restructuring of the code initially, but once that is done, you should be able to switch between GPUs, TPUs, and multiple machines simply by changing the strategy instance.\n",
         "\n",
-        "Here we will show a brief snippet illustrating this use case for a simple training example using the same Keras model as before.\n"
+        "Below is a brief snippet illustrating this use case for a simple training example using the same Keras model as before.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "XNHvSY32nVBi"
       },
       "source": [
-        "First, we create the model and optimizer inside the strategy's scope. This ensures that any variables created with the model and optimizer are mirrored variables."
+        "First, create the model and optimizer inside the strategy's scope. This ensures that any variables created with the model and optimizer are mirrored variables."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "W-3Bn-CaiPKD"
       },
       "outputs": [],
       "source": [
         "with mirrored_strategy.scope():\n",
-        "  model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
+        "  model = tf.keras.Sequential([\n",
+        "      tf.keras.layers.Dense(1, input_shape=(1,),\n",
+        "                            kernel_regularizer=tf.keras.regularizers.L2(1e-4))])\n",
         "  optimizer = tf.keras.optimizers.SGD()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mYkAyPeYnlXk"
       },
       "source": [
-        "Next, we create the input dataset and call `tf.distribute.Strategy.experimental_distribute_dataset` to distribute the dataset based on the strategy."
+        "Next, create the input dataset and call `tf.distribute.Strategy.experimental_distribute_dataset` to distribute the dataset based on the strategy."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "94BkvkLInkKd"
       },
       "outputs": [],
@@ -664,304 +706,174 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "grzmTlSvn2j8"
       },
       "source": [
-        "Then, we define one step of the training. We will use `tf.GradientTape` to compute gradients and optimizer to apply those gradients to update our model's variables. To distribute this training step, we put in a function `step_fn` and pass it to `tf.distrbute.Strategy.experimental_run_v2` along with the dataset inputs that we get from `dist_dataset` created before:"
+        "Then, define one step of the training. Use `tf.GradientTape` to compute gradients and optimizer to apply those gradients to update your model's variables. To distribute this training step, put it in a function `train_step` and pass it to `tf.distribute.Strategy.run` along with the dataset inputs you got from the `dist_dataset` created before:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "NJxL5YrVniDe"
       },
       "outputs": [],
       "source": [
-        "@tf.function\n",
-        "def train_step(dist_inputs):\n",
-        "  def step_fn(inputs):\n",
-        "    features, labels = inputs\n",
+        "# Sets `reduction=NONE` to leave it to tf.nn.compute_average_loss() below.\n",
+        "loss_object = tf.keras.losses.BinaryCrossentropy(\n",
+        "  from_logits=True,\n",
+        "  reduction=tf.keras.losses.Reduction.NONE)\n",
         "\n",
-        "    with tf.GradientTape() as tape:\n",
-        "      logits = model(features)\n",
-        "      cross_entropy = tf.nn.softmax_cross_entropy_with_logits(\n",
-        "          logits=logits, labels=labels)\n",
-        "      loss = tf.reduce_sum(cross_entropy) * (1.0 / global_batch_size)\n",
+        "def train_step(inputs):\n",
+        "  features, labels = inputs\n",
         "\n",
-        "    grads = tape.gradient(loss, model.trainable_variables)\n",
-        "    optimizer.apply_gradients(list(zip(grads, model.trainable_variables)))\n",
-        "    return cross_entropy\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    predictions = model(features, training=True)\n",
+        "    per_example_loss = loss_object(labels, predictions)\n",
+        "    loss = tf.nn.compute_average_loss(per_example_loss)\n",
+        "    model_losses = model.losses\n",
+        "    if model_losses:\n",
+        "      loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n",
         "\n",
-        "  per_example_losses = mirrored_strategy.experimental_run_v2(\n",
-        "      step_fn, args=(dist_inputs,))\n",
-        "  mean_loss = mirrored_strategy.reduce(\n",
-        "      tf.distribute.ReduceOp.MEAN, per_example_losses, axis=0)\n",
-        "  return mean_loss"
+        "  gradients = tape.gradient(loss, model.trainable_variables)\n",
+        "  optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
+        "  return loss\n",
+        "\n",
+        "@tf.function\n",
+        "def distributed_train_step(dist_inputs):\n",
+        "  per_replica_losses = mirrored_strategy.run(train_step, args=(dist_inputs,))\n",
+        "  return mirrored_strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n",
+        "                         axis=None)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yRL5u_NLoTvq"
       },
       "source": [
         "A few other things to note in the code above:\n",
         "\n",
-        "1. We used `tf.nn.softmax_cross_entropy_with_logits` to compute the loss. And then we scaled the total loss by the global batch size. This is important because all the replicas are training in sync and number of examples in each step of training is the global batch. So the loss needs to be divided by the global batch size and not by the replica (local) batch size.\n",
-        "2. We used the `tf.distribute.Strategy.reduce` API to aggregate the results returned by `tf.distribute.Strategy.experimental_run_v2`. `tf.distribute.Strategy.experimental_run_v2` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can `reduce` them to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results` to get the list of values contained in the result, one per local replica.\n",
-        "3. When `apply_gradients` is called within a distribution strategy scope, its behavior is modified. Specifically, before applying gradients on each parallel instance during synchronous training, it performs a sum-over-all-replicas of the gradients.\n",
+        "  1. You used `tf.nn.compute_average_loss` to reduce the per-example prediction losses to a scalar. `tf.nn.compute_average_loss` sums the per example loss and divides the sum by the global batch size. This is important because later after the gradients are calculated on each replica, they are aggregated across the replicas by **summing** them.\n",
+        "\n",
+        "  By default, the global batch size is taken to be `tf.get_strategy().num_replicas_in_sync * tf.shape(per_example_loss)[0]`. It can also be specified explicitly as a keyword argument `global_batch_size=`. Without short batches, the default is equivalent to `tf.nn.compute_average_loss(..., global_batch_size=global_batch_size)` with the `global_batch_size` defined above. (For more on short batches and how to avoid or handle them, see the [Custom Training tutorial](../tutorials/distribute/custom_training.ipynb).)\n",
+        "\n",
+        "  2. You used `tf.nn.scale_regularization_loss` to scale regularization losses registered with the `Model` object, if any, by `1/num_replicas_in_sync` as well. For those regularization losses that are input-dependent, it falls on the modeling code, not the custom training loop, to perform the averaging over the per-replica(!) batch size; that way the modeling code can remain agnostic of replication while the training loop remains agnostic of how regularization losses are computed.\n",
+        "\n",
+        "  3. When you call `apply_gradients` within a distribution strategy scope, its behavior is modified. Specifically, before applying gradients on each parallel instance during synchronous training, it performs a sum-over-all-replicas of the gradients.\n",
+        "\n",
+        "  4. You also used the `tf.distribute.Strategy.reduce` API to aggregate the results returned by `tf.distribute.Strategy.run` for reporting. `tf.distribute.Strategy.run` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can `reduce` them to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results` to get the list of values contained in the result, one per local replica.\n",
         "\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "o9k_6-6vpQ-P"
       },
       "source": [
-        "Finally, once we have defined the training step, we can iterate over `dist_dataset` and run the training in a loop:"
+        "Finally, once you have defined the training step, you can iterate over `dist_dataset` and run the training in a loop:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Egq9eufToRf6"
       },
       "outputs": [],
       "source": [
-        "with mirrored_strategy.scope():\n",
-        "  for inputs in dist_dataset:\n",
-        "    print(train_step(inputs))"
+        "for dist_inputs in dist_dataset:\n",
+        "  print(distributed_train_step(dist_inputs))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "jK8eQXF_q1Zs"
       },
       "source": [
-        "In the example above, we iterated over the `dist_dataset` to provide input to your training. We also provide the  `tf.distribute.Strategy.make_experimental_numpy_dataset` to support numpy inputs. You can use this API to create a dataset before calling `tf.distribute.Strategy.experimental_distribute_dataset`.\n",
+        "In the example above, you iterated over the `dist_dataset` to provide input to your training. You are also provided with the `tf.distribute.Strategy.make_experimental_numpy_dataset` to support NumPy inputs. You can use this API to create a dataset before calling `tf.distribute.Strategy.experimental_distribute_dataset`.\n",
         "\n",
-        "Another way of iterating over your data is to explicitly use iterators. You may want to do this when you want to run for a given number of steps as opposed to iterating over the entire dataset.\n",
-        "The above iteration would now be modified to first create an iterator and then explicitly call `next` on it to get the input data.\n"
+        "Another way of iterating over your data is to explicitly use iterators. You may want to do this when you want to run for a given number of steps as opposed to iterating over the entire dataset. The above iteration would now be modified to first create an iterator and then explicitly call `next` on it to get the input data."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "e5BEvR0-LJAc"
       },
       "outputs": [],
       "source": [
-        "with mirrored_strategy.scope():\n",
-        "  iterator = iter(dist_dataset)\n",
-        "  for _ in range(10):\n",
-        "    print(train_step(next(iterator)))"
+        "iterator = iter(dist_dataset)\n",
+        "for _ in range(10):\n",
+        "  print(distributed_train_step(next(iterator)))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vDJO8mnypqBA"
       },
       "source": [
-        "This covers the simplest case of using `tf.distribute.Strategy` API to distribute custom training loops. We are in the process of improving these APIs. Since this use case requires more work to adapt your code, we will be publishing a separate detailed guide in the future."
+        "This covers the simplest case of using `tf.distribute.Strategy` API to distribute custom training loops."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "BZjNwCt1qBdw"
       },
       "source": [
         "### What's supported now?\n",
-        "In TF 2.0 release, training with custom training loops is supported using `MirroredStrategy` as shown above and `TPUStrategy`. \n",
-        "`MultiWorkerMirorredStrategy` support will be coming in the future.\n",
-        "\n",
-        "| Training API          \t| MirroredStrategy  \t| TPUStrategy       \t| MultiWorkerMirroredStrategy \t| CentralStorageStrategy \t| ParameterServerStrategy \t| OneDeviceStrategy |\n",
-        "|:-----------------------\t|:-------------------\t|:-------------------\t|:-----------------------------\t|:------------------------\t|:-------------------------\t| :-------------------------- |\n",
-        "| Custom Training Loop  \t| Experimental support \t| Experimental support \t| Support planned post 2.0         \t| Support planned post 2.0    \t| No support yet         \t| Supported |\n",
         "\n",
-        "### Examples and Tutorials\n",
-        "Here are some examples for using distribution strategy with custom training loops:\n",
-        "\n",
-        "1. [Tutorial](../tutorials/distribute/custom_training.ipynb) to train MNIST using `MirroredStrategy`.\n",
-        "2. [DenseNet](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/densenet/distributed_train.py) example using `MirroredStrategy`.\n",
-        "1. [BERT](https://github.com/tensorflow/models/blob/master/official/nlp/bert/run_classifier.py) example trained using `MirroredStrategy` and `TPUStrategy`.\n",
-        "This example is particularly helpful for understanding how to load from a checkpoint and generate periodic checkpoints during distributed training etc.\n",
-        "2. [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) example trained using `MirroredStrategy` and `TPUStrategy` that can be enabled using the `keras_use_ctl` flag.\n",
-        "3. [NMT](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/nmt_with_attention/distributed_train.py) example trained using `MirroredStrategy`.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "nO0hmFCRoIll"
-      },
-      "source": [
-        "## Using `tf.distribute.Strategy` with Estimator (Limited support)\n",
-        "`tf.estimator` is a distributed training TensorFlow API that originally supported the async parameter server approach. Like with Keras, we've integrated `tf.distribute.Strategy` into `tf.Estimator`. If you're using Estimator for your training, you can easily change to distributed training with very few changes to your code. With this, Estimator users can now do synchronous distributed training on multiple GPUs and multiple workers, as well as use TPUs. This support in Estimator is, however, limited. See [What's supported now](#estimator_support) section below for more details.\n",
+        "| Training API         | `MirroredStrategy` | `TPUStrategy` | `MultiWorkerMirroredStrategy` | `ParameterServerStrategy` | `CentralStorageStrategy` |\n",
+        "| :------------------- | :----------------- | :------------ | :---------------------------- | :------------------------ | :----------------------- |\n",
+        "| Custom training loop | Supported          | Supported     | Supported                     | Experimental support      | Experimental support     |\n",
         "\n",
-        "The usage of `tf.distribute.Strategy` with Estimator is slightly different than the Keras case. Instead of using `strategy.scope`, now we pass the strategy object into the [`RunConfig`](https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig) for the Estimator.\n",
+        "### Examples and tutorials\n",
         "\n",
-        "Here is a snippet of code that shows this with a premade Estimator `LinearRegressor` and `MirroredStrategy`:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "oGFY5nW_B3YU"
-      },
-      "outputs": [],
-      "source": [
-        "mirrored_strategy = tf.distribute.MirroredStrategy()\n",
-        "config = tf.estimator.RunConfig(\n",
-        "    train_distribute=mirrored_strategy, eval_distribute=mirrored_strategy)\n",
-        "regressor = tf.estimator.LinearRegressor(\n",
-        "    feature_columns=[tf.feature_column.numeric_column('feats')],\n",
-        "    optimizer='SGD',\n",
-        "    config=config)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "n6eSfLN5RGY8"
-      },
-      "source": [
-        "We use a premade Estimator here, but the same code works with a custom Estimator as well. `train_distribute` determines how training will be distributed, and `eval_distribute` determines how evaluation will be distributed. This is another difference from Keras where we use the same strategy for both training and eval.\n",
+        "Here are some examples for using distribution strategies with custom training loops:\n",
         "\n",
-        "Now we can train and evaluate this Estimator with an input function:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "2ky2ve2PB3YP"
-      },
-      "outputs": [],
-      "source": [
-        "def input_fn():\n",
-        "  dataset = tf.data.Dataset.from_tensors(({\"feats\":[1.]}, [1.]))\n",
-        "  return dataset.repeat(1000).batch(10)\n",
-        "regressor.train(input_fn=input_fn, steps=10)\n",
-        "regressor.evaluate(input_fn=input_fn, steps=10)"
+        "1. [Tutorial](../tutorials/distribute/custom_training.ipynb): Training with a custom training loop and `MirroredStrategy`.\n",
+        "2. [Tutorial](../tutorials/distribute/multi_worker_with_ctl.ipynb): Training with a custom training loop and `MultiWorkerMirroredStrategy`.\n",
+        "3. [Guide](tpu.ipynb): Contains an example of a custom training loop with `TPUStrategy`.\n",
+        "4. [Tutorial](../tutorials/distribute/parameter_server_training.ipynb): Parameter server training with a custom training loop and `ParameterServerStrategy`.\n",
+        "5. TensorFlow Model Garden [repository](https://github.com/tensorflow/models/tree/master/official) containing collections of state-of-the-art models implemented using various strategies.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "hgaU9xQSSk2x"
+        "id": "Xk0JdsTHyUnE"
       },
       "source": [
-        "Another difference to highlight here between Estimator and Keras is the input handling. In Keras, we mentioned that each batch of the dataset is split automatically across the multiple replicas. In Estimator, however, we do not do automatic splitting of batch, nor automatically shard the data across different workers. You have full control over how you want your data to be distributed across workers and devices, and you must provide an `input_fn` to specify how to distribute your data.\n",
-        "\n",
-        "Your `input_fn` is called once per worker, thus giving one dataset per worker. Then one batch from that dataset is fed to one replica on that worker, thereby consuming N batches for N replicas on 1 worker. In other words, the dataset returned by the `input_fn` should provide batches of size `PER_REPLICA_BATCH_SIZE`. And the global batch size for a step can be obtained as `PER_REPLICA_BATCH_SIZE * strategy.num_replicas_in_sync`.\n",
+        "## Other topics\n",
         "\n",
-        "When doing multi worker training, you should either split your data across the workers, or shuffle with a random seed on each. You can see an example of how to do this in the [Multi-worker Training with Estimator](../tutorials/distribute/multi_worker_with_estimator.ipynb)."
+        "This section covers some topics that are relevant to multiple use cases."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "_098zB3vVhuV"
-      },
-      "source": [
-        "We showed an example of using `MirroredStrategy` with Estimator. You can also use `TPUStrategy` with Estimator as well, in the exact same way:\n",
-        "```\n",
-        "config = tf.estimator.RunConfig(\n",
-        "    train_distribute=tpu_strategy, eval_distribute=tpu_strategy)\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "G3ieQKfWZhhL"
-      },
-      "source": [
-        "And similarly, you can use multi worker and parameter server strategies as well. The code remains the same, but you need to use `tf.estimator.train_and_evaluate`, and set `TF_CONFIG` environment variables for each binary running in your cluster."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "A_lvUsSLZzVg"
+        "id": "cP6BUIBtudRk"
       },
       "source": [
-        "\u003ca name=\"estimator_support\"\u003e\u003c/a\u003e\n",
-        "### What's supported now?\n",
+        "\n",
+        "### Setting up the TF\\_CONFIG environment variable\n",
         "\n",
-        "In TF 2.0 release, there is limited support for training with Estimator using all strategies except `TPUStrategy`. Basic training and evaluation should work, but a number of advanced features such as scaffold do not yet work. There may also be a number of bugs in this integration. At this time, we do not plan to actively improve this support, and instead are focused on Keras and custom training loop support. If at all possible, you should prefer to use `tf.distribute` with those APIs instead.\n",
+        "For multi-worker training, as mentioned before, you need to set up the `'TF_CONFIG'` environment variable for each binary running in your cluster. The `'TF_CONFIG'` environment variable is a JSON string which specifies what tasks constitute a cluster, their addresses and each task's role in the cluster. The [`tensorflow/ecosystem`](https://github.com/tensorflow/ecosystem) repo provides a Kubernetes template, which sets up `'TF_CONFIG'` for your training tasks.\n",
         "\n",
-        "| Training API  \t| MirroredStrategy \t| TPUStrategy \t| MultiWorkerMirroredStrategy \t| CentralStorageStrategy \t| ParameterServerStrategy \t| OneDeviceStrategy |\n",
-        "|:---------------\t|:------------------\t|:-------------\t|:-----------------------------\t|:------------------------\t|:-------------------------\t| :-------------------------- |\n",
-        "| Estimator API \t| Limited Support        \t| Not supported   \t| Limited Support                   \t| Limited Support              \t| Limited Support               \t| Limited Support |\n",
+        "There are two components of `'TF_CONFIG'`: a cluster and a task.\n",
         "\n",
-        "### Examples and Tutorials\n",
-        "Here are some examples that show end to end usage of various strategies with Estimator:\n",
+        "- A cluster provides information about the training cluster, which is a dict consisting of different types of jobs such as workers. In multi-worker training, there is usually one worker that takes on a little more responsibility like saving checkpoint and writing summary file for TensorBoard in addition to what a regular worker does. Such worker is referred to as the \"chief\" worker, and it is customary that the worker with index `0` is appointed as the chief worker (in fact this is how `tf.distribute.Strategy` is implemented).\n",
+        "- A task on the other hand provides information about the current task. The first component cluster is the same for all workers, and the second component task is different on each worker and specifies the type and index of that worker.\n",
         "\n",
-        "1. [Multi-worker Training with Estimator](../tutorials/distribute/multi_worker_with_estimator.ipynb) to train MNIST with multiple workers using `MultiWorkerMirroredStrategy`.\n",
-        "2. [End to end example](https://github.com/tensorflow/ecosystem/tree/master/distribution_strategy) for multi worker training in tensorflow/ecosystem using Kubernetes templates. This example starts with a Keras model and converts it to an Estimator using the `tf.keras.estimator.model_to_estimator` API.\n",
-        "3. Official [ResNet50](https://github.com/tensorflow/models/blob/master/official/vision/image_classification/resnet_imagenet_main.py) model, which can be trained using either `MirroredStrategy` or `MultiWorkerMirroredStrategy`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Xk0JdsTHyUnE"
-      },
-      "source": [
-        "## Other topics\n",
-        "In this section, we will cover some topics that are relevant to multiple use cases."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "cP6BUIBtudRk"
-      },
-      "source": [
-        "\u003ca name=\"TF_CONFIG\"\u003e\u003c/a\u003e\n",
-        "### Setting up TF\\_CONFIG environment variable\n",
-        "\n",
-        "For multi-worker training, as mentioned before, you need to set `TF_CONFIG` environment variable for each\n",
-        "binary running in your cluster. The `TF_CONFIG` environment variable is a JSON string which specifies what\n",
-        "tasks constitute a cluster, their addresses and each task's role in the cluster. We provide a Kubernetes template in the\n",
-        "[tensorflow/ecosystem](https://github.com/tensorflow/ecosystem) repo which sets\n",
-        "`TF_CONFIG` for your training tasks.\n",
+        "One example of `'TF_CONFIG'` is:\n",
         "\n",
-        "One example of `TF_CONFIG` is:\n",
-        "```\n",
+        "```python\n",
         "os.environ[\"TF_CONFIG\"] = json.dumps({\n",
         "    \"cluster\": {\n",
         "        \"worker\": [\"host1:port\", \"host2:port\", \"host3:port\"],\n",
@@ -975,26 +887,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "fezd3aF8wj9r"
       },
       "source": [
-        "This `TF_CONFIG` specifies that there are three workers and two ps tasks in the\n",
-        "cluster along with their hosts and ports. The \"task\" part specifies that the\n",
-        "role of the current task in the cluster, worker 1 (the second worker). Valid roles in a cluster is\n",
-        "\"chief\", \"worker\", \"ps\" and \"evaluator\". There should be no \"ps\" job except when using `tf.distribute.experimental.ParameterServerStrategy`."
+        "This `'TF_CONFIG'` specifies that there are three workers and two `\"ps\"` tasks in the `\"cluster\"` along with their hosts and ports. The `\"task\"` part specifies the role of the current task in the `\"cluster\"`—worker `1` (the second worker). Valid roles in a cluster are `\"chief\"`, `\"worker\"`, `\"ps\"`, and `\"evaluator\"`. There should be no `\"ps\"` job except when using `tf.distribute.experimental.ParameterServerStrategy`."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GXIbqSW-sFVg"
       },
       "source": [
         "## What's next?\n",
         "\n",
-        "`tf.distribute.Strategy` is actively under development. We welcome you to try it out and provide and your feedback using [GitHub issues](https://github.com/tensorflow/tensorflow/issues/new)."
+        "`tf.distribute.Strategy` is actively under development. Try it out and provide your feedback using [GitHub issues](https://github.com/tensorflow/tensorflow/issues/new)."
       ]
     }
   ],
@@ -1004,8 +911,6 @@
         "Tce3stUlHN0L"
       ],
       "name": "distributed_training.ipynb",
-      "private_outputs": true,
-      "provenance": [],
       "toc_visible": true
     },
     "kernelspec": {
diff --git a/site/en/guide/dtensor_overview.ipynb b/site/en/guide/dtensor_overview.ipynb
new file mode 100644
index 00000000000..1b55ee0283f
--- /dev/null
+++ b/site/en/guide/dtensor_overview.ipynb
@@ -0,0 +1,1082 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1ljvLya59ep5"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Authors.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VcQIa1uG86Wh"
+      },
+      "source": [
+        "# DTensor concepts"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6dWNQEum9AfY"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MGZuakHVlVQf"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This colab introduces DTensor, an extension to TensorFlow for synchronous distributed computing.\n",
+        "\n",
+        "DTensor provides a global programming model that allows developers to compose applications that operate on Tensors globally while managing the distribution across devices internally. DTensor distributes the program and tensors according to the sharding directives through a procedure called *[Single program, multiple data (SPMD)](https://en.wikipedia.org/wiki/SPMD) expansion*.\n",
+        "\n",
+        "By decoupling the application from sharding directives, DTensor enables running the same application on a single device, multiple devices, or even multiple clients, while preserving its global semantics.\n",
+        "\n",
+        "This guide introduces DTensor concepts for distributed computing, and how DTensor integrates with TensorFlow. For a demo of using DTensor in model training, refer to the [Distributed training with DTensor](../tutorials/distribute/dtensor_ml_tutorial.ipynb) tutorial."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h7ZTDq7KngwA"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "DTensor (`tf.experimental.dtensor`) has been part of TensorFlow since the 2.9.0 release.\n",
+        "\n",
+        "Begin by importing TensorFlow, `dtensor`, and configure TensorFlow to use 6 virtual CPUs. Even though this example uses virtual CPUs, DTensor works the same way on CPU, GPU or TPU devices."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q92lo0zjwej8"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "from tensorflow.experimental import dtensor\n",
+        "\n",
+        "print('TensorFlow version:', tf.__version__)\n",
+        "\n",
+        "def configure_virtual_cpus(ncpu):\n",
+        "  phy_devices = tf.config.list_physical_devices('CPU')\n",
+        "  tf.config.set_logical_device_configuration(phy_devices[0], [\n",
+        "        tf.config.LogicalDeviceConfiguration(),\n",
+        "    ] * ncpu)\n",
+        "\n",
+        "configure_virtual_cpus(6)\n",
+        "DEVICES = [f'CPU:{i}' for i in range(6)]\n",
+        "\n",
+        "tf.config.list_logical_devices('CPU')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O-lsrxUnlsCC"
+      },
+      "source": [
+        "## DTensor's model of distributed tensors\n",
+        "\n",
+        "DTensor introduces two concepts: `dtensor.Mesh` and `dtensor.Layout`. They are abstractions to model the sharding of tensors across topologically related devices.\n",
+        "\n",
+        "- `Mesh` defines the device list for computation.\n",
+        "- `Layout` defines how to shard the Tensor dimension on a `Mesh`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JjiHaH0ql9yo"
+      },
+      "source": [
+        "### Mesh\n",
+        "\n",
+        "`Mesh` represents a logical Cartisian topology of a set of devices. Each dimension of the Cartisian grid is called a **Mesh dimension**, and referred to with a name. Names of mesh dimension within the same `Mesh` must be unique.\n",
+        "\n",
+        "Names of mesh dimensions are referenced by `Layout` to describe the sharding behavior of a `tf.Tensor` along each of its axes. This is described in more detail later in the section on `Layout`.\n",
+        "\n",
+        "`Mesh` can be thought of as a multi-dimensional array of devices."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_J6cOieEbaUw"
+      },
+      "source": [
+        "In a 1 dimensional `Mesh`, all devices form a list in a single mesh dimension. The following example uses `dtensor.create_mesh` to create a mesh from 6 CPU devices along a mesh dimension `'x'` with a size of 6 devices:\n",
+        "\n",
+        "\"A\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QLH5fgdBmA58"
+      },
+      "outputs": [],
+      "source": [
+        "mesh_1d = dtensor.create_mesh([('x', 6)], devices=DEVICES)\n",
+        "print(mesh_1d)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hSZwaUwnEgXB"
+      },
+      "source": [
+        "A `Mesh` can be multi dimensional as well. In the following example, 6 CPU devices form a `3x2` mesh, where the `'x'` mesh dimension has a size of 3 devices, and the `'y'` mesh dimension has a size of 2 devices:\n",
+        "\n",
+        "\"A"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "op6TmKUQE-sZ"
+      },
+      "outputs": [],
+      "source": [
+        "mesh_2d = dtensor.create_mesh([('x', 3), ('y', 2)], devices=DEVICES)\n",
+        "print(mesh_2d)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "deAqdrDPFn2f"
+      },
+      "source": [
+        "### Layout\n",
+        "\n",
+        "**`Layout`** specifies how a tensor is distributed, or sharded, on a `Mesh`.\n",
+        "\n",
+        "Note: In order to avoid confusions between `Mesh` and `Layout`, the term *dimension* is always associated with `Mesh`, and the term *axis* with `Tensor` and `Layout` in this guide.\n",
+        "\n",
+        "The rank of `Layout` should be the same as the rank of the `Tensor` where the `Layout` is applied. For each of the `Tensor`'s axes the `Layout` may specify a mesh dimension to shard the tensor across, or specify the axis as \"unsharded\".\n",
+        "The tensor is replicated across any mesh dimensions that it is not sharded across.\n",
+        "\n",
+        "The rank of a `Layout` and the number of dimensions of a `Mesh` do not need to match. The `unsharded` axes of a `Layout` do not need to be associated to a mesh dimension, and `unsharded` mesh dimensions do not need to be associated with a `layout` axis.\n",
+        "\n",
+        "\"Diagram"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Px_bF1c-bQ7e"
+      },
+      "source": [
+        "Let's analyze a few examples of `Layout` for the `Mesh`'s created in the previous section."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fqzCNlWAbm-c"
+      },
+      "source": [
+        "On a 1-dimensional mesh such as `[(\"x\", 6)]` (`mesh_1d` in the previous section), `Layout([\"unsharded\", \"unsharded\"], mesh_1d)` is a layout for a rank-2 tensor replicated across 6 devices.\n",
+        "\"A"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-a3EnmZag6x1"
+      },
+      "outputs": [],
+      "source": [
+        "layout = dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh_1d)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ywRJwuLDt2Qq"
+      },
+      "source": [
+        "Using the same tensor and mesh the layout `Layout(['unsharded', 'x'])` would shard the second axis of the tensor across the 6 devices.\n",
+        "\n",
+        "\"A"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7BgqL0jUvV5a"
+      },
+      "outputs": [],
+      "source": [
+        "layout = dtensor.Layout([dtensor.UNSHARDED, 'x'], mesh_1d)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DgciDNmK76l9"
+      },
+      "source": [
+        "Given a 2-dimensional 3x2 mesh such as `[(\"x\", 3), (\"y\", 2)]`, (`mesh_2d` from the previous section), `Layout([\"y\", \"x\"], mesh_2d)` is a layout for a rank-2 `Tensor` whose first axis is sharded across mesh dimension `\"y\"`, and whose second axis is sharded across mesh dimension `\"x\"`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Eyp_qOSyvieo"
+      },
+      "source": [
+        "\"A\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p8OrehEuhPbS"
+      },
+      "outputs": [],
+      "source": [
+        "layout = dtensor.Layout(['y', 'x'], mesh_2d)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1Kyg0V3ehMNJ"
+      },
+      "source": [
+        "For the same `mesh_2d`, the layout `Layout([\"x\", dtensor.UNSHARDED], mesh_2d)` is a layout for a rank-2 `Tensor` that is replicated across `\"y\"`, and whose first axis is sharded on mesh dimension `x`.\n",
+        "\n",
+        "\"A\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IkWe6mVl7uRb"
+      },
+      "outputs": [],
+      "source": [
+        "layout = dtensor.Layout([\"x\", dtensor.UNSHARDED], mesh_2d)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TTalu6M-ISYb"
+      },
+      "source": [
+        "### Single-client and multi-client applications\n",
+        "\n",
+        "DTensor supports both single-client and multi-client applications. The colab Python kernel is an example of a single client DTensor application, where there is a single Python process.\n",
+        "\n",
+        "In a multi-client DTensor application, multiple Python processes collectively perform as a coherent application. The Cartisian grid of a `Mesh` in a multi-client DTensor application can span across devices regardless of whether they are attached locally to the current client or attached remotely to another client. The set of all devices used by a `Mesh` are called the *global device list*.\n",
+        "\n",
+        "The creation of a `Mesh` in a multi-client DTensor application is a collective operation where the *global device list* is identical for all of the participating clients, and the creation of the `Mesh` serves as a global barrier.\n",
+        "\n",
+        "During `Mesh` creation, each client provides its *local device list* together with the expected *global device list*. DTensor validates that both lists are consistent. Please refer to the API documentation for `dtensor.create_mesh` and `dtensor.create_distributed_mesh`\n",
+        " for more information on multi-client mesh creation and the *global device list*.\n",
+        "\n",
+        "Single-client can be thought of as a special case of multi-client, with 1 client. In a single-client application, the *global device list* is identical to the *local device list*.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "P_F7DWkXkB4w"
+      },
+      "source": [
+        "## DTensor as a sharded tensor\n",
+        "\n",
+        "Now, start coding with `DTensor`. The helper function, `dtensor_from_array`, demonstrates creating DTensors from something that looks like a `tf.Tensor`. The function performs two steps:\n",
+        "\n",
+        "  - Replicates the tensor to every device on the mesh.\n",
+        "  - Shards the copy according to the layout requested in its arguments."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "s6aws-b8dN9L"
+      },
+      "outputs": [],
+      "source": [
+        "def dtensor_from_array(arr, layout, shape=None, dtype=None):\n",
+        "  \"\"\"Convert a DTensor from something that looks like an array or Tensor.\n",
+        "\n",
+        "  This function is convenient for quick doodling DTensors from a known,\n",
+        "  unsharded data object in a single-client environment. This is not the\n",
+        "  most efficient way of creating a DTensor, but it will do for this\n",
+        "  tutorial.\n",
+        "  \"\"\"\n",
+        "  if shape is not None or dtype is not None:\n",
+        "    arr = tf.constant(arr, shape=shape, dtype=dtype)\n",
+        "\n",
+        "  # replicate the input to the mesh\n",
+        "  a = dtensor.copy_to_mesh(arr,\n",
+        "          layout=dtensor.Layout.replicated(layout.mesh, rank=layout.rank))\n",
+        "  # shard the copy to the desirable layout\n",
+        "  return dtensor.relayout(a, layout=layout)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r3o6IysrlGMu"
+      },
+      "source": [
+        "### Anatomy of a DTensor\n",
+        "\n",
+        "A DTensor is a `tf.Tensor` object, but augumented with the `Layout` annotation that defines its sharding behavior. A DTensor consists of the following:\n",
+        "\n",
+        "  - Global tensor meta-data, including the global shape and dtype of the tensor.\n",
+        "  - A `Layout`, which defines the `Mesh` the `Tensor` belongs to, and how the `Tensor` is sharded onto the `Mesh`.\n",
+        "  - A list of **component tensors**, one item per local device in the `Mesh`.\n",
+        "\n",
+        "With `dtensor_from_array`, you can create your first DTensor, `my_first_dtensor`, and examine its contents:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mQu_nScGUvYH"
+      },
+      "outputs": [],
+      "source": [
+        "mesh = dtensor.create_mesh([(\"x\", 6)], devices=DEVICES)\n",
+        "layout = dtensor.Layout([dtensor.UNSHARDED], mesh)\n",
+        "\n",
+        "my_first_dtensor = dtensor_from_array([0, 1], layout)\n",
+        "\n",
+        "# Examine the DTensor content\n",
+        "print(my_first_dtensor)\n",
+        "print(\"global shape:\", my_first_dtensor.shape)\n",
+        "print(\"dtype:\", my_first_dtensor.dtype)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r8LQy1nqmvFy"
+      },
+      "source": [
+        "#### Layout and `fetch_layout`\n",
+        "\n",
+        "The layout of a DTensor is not a regular attribute of `tf.Tensor`. Instead, DTensor provides a function, `dtensor.fetch_layout` to access the layout of a DTensor:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dCSFyaAjmzGu"
+      },
+      "outputs": [],
+      "source": [
+        "print(dtensor.fetch_layout(my_first_dtensor))\n",
+        "assert layout == dtensor.fetch_layout(my_first_dtensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ed7i3l2lmatm"
+      },
+      "source": [
+        "#### Component tensors, `pack` and `unpack`\n",
+        "\n",
+        "A DTensor consists of a list of **component tensors**. The component tensor for a device in the `Mesh` is the `Tensor` object representing the piece of the global DTensor that is stored on this device.\n",
+        "\n",
+        "A DTensor can be unpacked into component tensors through `dtensor.unpack`. You can make use of `dtensor.unpack` to inspect the components of the DTensor, and confirm they are on all devices of the `Mesh`.\n",
+        "\n",
+        "Note that the positions of component tensors in the global view may overlap each other. For example, in the case of a fully replicated layout, all components are identical replicas of the global tensor."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BGbjqVAOnXMk"
+      },
+      "outputs": [],
+      "source": [
+        "for component_tensor in dtensor.unpack(my_first_dtensor):\n",
+        "  print(\"Device:\", component_tensor.device, \",\", component_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-tqIQM52k788"
+      },
+      "source": [
+        "As shown, `my_first_dtensor` is a tensor of `[0, 1]` replicated to all 6 devices."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6By3k-CGn3yv"
+      },
+      "source": [
+        "The inverse operation of `dtensor.unpack` is `dtensor.pack`. Component tensors can be packed back into a DTensor.\n",
+        "\n",
+        "The components must have the same rank and dtype, which will be the rank and dtype of the returned DTensor. However, there is no strict requirement on the device placement of component tensors as inputs of `dtensor.unpack`: the function will automatically copy the component tensors to their respective corresponding devices.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9lT-6qQwxOgf"
+      },
+      "outputs": [],
+      "source": [
+        "packed_dtensor = dtensor.pack(\n",
+        "    [[0, 1], [0, 1], [0, 1],\n",
+        "     [0, 1], [0, 1], [0, 1]],\n",
+        "     layout=layout\n",
+        ")\n",
+        "print(packed_dtensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zvS3autrpK2U"
+      },
+      "source": [
+        "### Sharding a DTensor to a Mesh\n",
+        "\n",
+        "So far you've worked with the `my_first_dtensor`, which is a rank-1 DTensor fully replicated across a dim-1 `Mesh`.\n",
+        "\n",
+        "Next, create and inspect DTensors that are sharded across a dim-2 `Mesh`. The following example does this with a 3x2 `Mesh` on 6 CPU devices, where size of mesh dimension `'x'` is 3 devices, and size of mesh dimension`'y'` is 2 devices:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KWb9Ae0VJ-Rc"
+      },
+      "outputs": [],
+      "source": [
+        "mesh = dtensor.create_mesh([(\"x\", 3), (\"y\", 2)], devices=DEVICES)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ndSeQSFWKQk9"
+      },
+      "source": [
+        "#### Fully sharded rank-2 Tensor on a dim-2 Mesh\n",
+        "\n",
+        "Create a 3x2 rank-2 DTensor, sharding its first axis along the `'x'` mesh dimension, and its second axis along the `'y'` mesh dimension.\n",
+        "\n",
+        "- Because the tensor shape equals to the mesh dimension along all of the sharded axes, each device receives a single element of the DTensor.\n",
+        "- The rank of the component tensor is always the same as the rank of the global shape. DTensor adopts this convention as a simple way to preserve information for locating the relation between a component tensor and the global DTensor."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ax_ZHouJp1MX"
+      },
+      "outputs": [],
+      "source": [
+        "fully_sharded_dtensor = dtensor_from_array(\n",
+        "    tf.reshape(tf.range(6), (3, 2)),\n",
+        "    layout=dtensor.Layout([\"x\", \"y\"], mesh))\n",
+        "\n",
+        "for raw_component in dtensor.unpack(fully_sharded_dtensor):\n",
+        "  print(\"Device:\", raw_component.device, \",\", raw_component)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zhsLC-NgrC2p"
+      },
+      "source": [
+        "#### Fully replicated rank-2 Tensor on a dim-2 Mesh\n",
+        "\n",
+        "For comparison, create a 3x2 rank-2 DTensor, fully replicated to the same dim-2 Mesh.\n",
+        "\n",
+        " - Because the DTensor is fully replicated, each device receives a full replica of the 3x2 DTensor.\n",
+        " - The rank of the component tensors are the same as the rank of the global shape -- this fact is trivial, because in this case, the shape of the component tensors are the same as the global shape anyway."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xmyC6H6Ec90P"
+      },
+      "outputs": [],
+      "source": [
+        "fully_replicated_dtensor = dtensor_from_array(\n",
+        "    tf.reshape(tf.range(6), (3, 2)),\n",
+        "    layout=dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh))\n",
+        "# Or, layout=tensor.Layout.fully_replicated(mesh, rank=2)\n",
+        "\n",
+        "for component_tensor in dtensor.unpack(fully_replicated_dtensor):\n",
+        "  print(\"Device:\", component_tensor.device, \",\", component_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KWoyv_oHMzk1"
+      },
+      "source": [
+        "#### Hybrid rank-2 Tensor on a dim-2 Mesh\n",
+        "\n",
+        "What about somewhere between fully sharded and fully replicated?\n",
+        "\n",
+        "DTensor allows a `Layout` to be a hybrid, sharded along some axes, but replicated along others.\n",
+        "\n",
+        "For example, you can shard the same 3x2 rank-2 DTensor in the following way:\n",
+        "\n",
+        "  - 1st axis sharded along the `'x'` mesh dimension.\n",
+        "  - 2nd axis replicated along the `'y'` mesh dimension.\n",
+        "\n",
+        "To achieve this sharding scheme, you just need to replace the sharding spec of the 2nd axis from `'y'` to `dtensor.UNSHARDED`, to indicate your intention of replicating along the 2nd axis. The layout object will look like `Layout(['x', dtensor.UNSHARDED], mesh)`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DygnbkQ1Lu42"
+      },
+      "outputs": [],
+      "source": [
+        "hybrid_sharded_dtensor = dtensor_from_array(\n",
+        "    tf.reshape(tf.range(6), (3, 2)),\n",
+        "    layout=dtensor.Layout(['x', dtensor.UNSHARDED], mesh))\n",
+        "\n",
+        "for component_tensor in dtensor.unpack(hybrid_sharded_dtensor):\n",
+        "  print(\"Device:\", component_tensor.device, \",\", component_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "T7FtZ9kQRZgE"
+      },
+      "source": [
+        "You can inspect the component tensors of the created DTensor and verify they are indeed sharded according to your scheme. It may be helpful to illustrate the situation with a chart:\n",
+        "\n",
+        "  \"A\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "auAkA38XjL-q"
+      },
+      "source": [
+        "#### Tensor.numpy() and sharded DTensor\n",
+        "\n",
+        "Be aware that calling the `.numpy()` method on a sharded DTensor raises an error. The rationale for erroring is to protect against unintended gathering of data from multiple computing devices to the host CPU device backing the returned NumPy array:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hNdwmnL0jAXS"
+      },
+      "outputs": [],
+      "source": [
+        "print(fully_replicated_dtensor.numpy())\n",
+        "\n",
+        "try:\n",
+        "  fully_sharded_dtensor.numpy()\n",
+        "except tf.errors.UnimplementedError:\n",
+        "  print(\"got an error as expected for fully_sharded_dtensor\")\n",
+        "\n",
+        "try:\n",
+        "  hybrid_sharded_dtensor.numpy()\n",
+        "except tf.errors.UnimplementedError:\n",
+        "  print(\"got an error as expected for hybrid_sharded_dtensor\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8WcMkiagPF_6"
+      },
+      "source": [
+        "## TensorFlow API on DTensor\n",
+        "\n",
+        "DTensor strives to be a drop-in replacement for tensor in your program. The TensorFlow Python API that consume `tf.Tensor`, such as the Ops library functions, `tf.function`, `tf.GradientTape`, also work with DTensor.\n",
+        "\n",
+        "To accomplish this, for each [TensorFlow Graph](https://www.tensorflow.org/guide/intro_to_graphs), DTensor produces and executes an equivalent [SPMD](https://en.wikipedia.org/wiki/SPMD) graph in a procedure called *SPMD expansion*. A few critical steps in DTensor SPMD expansion are:\n",
+        "\n",
+        "  - Propagating the sharding `Layout` of DTensor in the TensorFlow graph\n",
+        "  - Rewriting TensorFlow Ops on the global DTensor with equivalent TensorFlow Ops on the component tensors, inserting collective and communication Ops when necessary\n",
+        "  - Lowering backend neutral TensorFlow Ops to backend specific TensorFlow Ops.\n",
+        "\n",
+        "The final result is that **DTensor is a drop-in replacement for Tensor**.\n",
+        "\n",
+        "Note: DTensor is still an experimental API which means you will be exploring and pushing the boundaries and limits of the DTensor programming model.\n",
+        "\n",
+        "There are 2 ways of triggering DTensor execution:\n",
+        "\n",
+        "  - DTensor as operands of a Python function, such as `tf.matmul(a, b)`, will run through DTensor if `a`, `b`, or both are DTensors.\n",
+        "  - Requesting the result of a Python function to be a DTensor, such as `dtensor.call_with_layout(tf.ones, layout, shape=(3, 2))`, will run through DTensor because we requested the output of `tf.ones` to be sharded according to a `layout`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "urKzmqAoPssT"
+      },
+      "source": [
+        "### DTensor as operands\n",
+        "\n",
+        "Many TensorFlow API functions take `tf.Tensor` as their operands, and returns `tf.Tensor` as their results. For these functions, you can express intention to run a function through DTensor by passing in DTensor as operands. This section uses `tf.matmul(a, b)` as an example."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7LO8ZT7iWVga"
+      },
+      "source": [
+        "#### Fully replicated input and output\n",
+        "\n",
+        "In this case, the DTensors are fully replicated. On each of the devices of the `Mesh`,\n",
+        "  - the component tensor for operand `a` is `[[1, 2, 3], [4, 5, 6]]` (2x3)\n",
+        "  - the component tensor for operand `b` is `[[6, 5], [4, 3], [2, 1]]` (3x2)\n",
+        "  - the computation consists of a single `MatMul` of `(2x3, 3x2) -> 2x2`,\n",
+        "  - the component tensor for result `c` is `[[20, 14], [56,41]]` (2x2)\n",
+        "\n",
+        "Total number of floating point mul operations is `6 device * 4 result * 3 mul = 72`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TiZf2J9JNd2D"
+      },
+      "outputs": [],
+      "source": [
+        "mesh = dtensor.create_mesh([(\"x\", 6)], devices=DEVICES)\n",
+        "layout = dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh)\n",
+        "a = dtensor_from_array([[1, 2, 3], [4, 5, 6]], layout=layout)\n",
+        "b = dtensor_from_array([[6, 5], [4, 3], [2, 1]], layout=layout)\n",
+        "\n",
+        "c = tf.matmul(a, b) # runs 6 identical matmuls in parallel on 6 devices\n",
+        "\n",
+        "# `c` is a DTensor replicated on all devices (same as `a` and `b`)\n",
+        "print('Sharding spec:', dtensor.fetch_layout(c).sharding_specs)\n",
+        "print(\"components:\")\n",
+        "for component_tensor in dtensor.unpack(c):\n",
+        "  print(component_tensor.device, component_tensor.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QXtR9qgKWgWV"
+      },
+      "source": [
+        "#### Sharding operands along the contracted axis\n",
+        "\n",
+        "You can reduce the amount of computation per device by sharding the operands `a` and `b`. A popular sharding scheme for `tf.matmul` is to shard the operands along the axis of the contraction, which means sharding `a` along the second axis, and `b` along the first axis.\n",
+        "\n",
+        "The global matrix product sharded under this scheme can be performed efficiently, by local matmuls that runs concurrently, followed by a collective reduction to aggregate the local results. This is also the [canonical way](https://github.com/open-mpi/ompi/blob/ee87ec391f48512d3718fc7c8b13596403a09056/docs/man-openmpi/man3/MPI_Reduce.3.rst?plain=1#L265) of implementing a distributed matrix dot product.\n",
+        "\n",
+        "Total number of floating point mul operations is `6 devices * 4 result * 1 = 24`, a factor of 3 reduction compared to the fully replicated case (72) above. The factor of 3 is due to the sharding along `x` mesh dimension with a size of `3` devices.\n",
+        "\n",
+        "The reduction of the number of operations run sequentially is the main mechansism with which synchronuous model parallelism accelerates training."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EyVAUvMePbms"
+      },
+      "outputs": [],
+      "source": [
+        "mesh = dtensor.create_mesh([(\"x\", 3), (\"y\", 2)], devices=DEVICES)\n",
+        "a_layout = dtensor.Layout([dtensor.UNSHARDED, 'x'], mesh)\n",
+        "a = dtensor_from_array([[1, 2, 3], [4, 5, 6]], layout=a_layout)\n",
+        "b_layout = dtensor.Layout(['x', dtensor.UNSHARDED], mesh)\n",
+        "b = dtensor_from_array([[6, 5], [4, 3], [2, 1]], layout=b_layout)\n",
+        "\n",
+        "c = tf.matmul(a, b)\n",
+        "# `c` is a DTensor replicated on all devices (same as `a` and `b`)\n",
+        "print('Sharding spec:', dtensor.fetch_layout(c).sharding_specs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IhD8yYgJiCEh"
+      },
+      "source": [
+        "#### Additional sharding\n",
+        "\n",
+        "You can perform additional sharding on the inputs, and they are appropriately carried over to the results. For example, you can apply additional sharding of operand `a` along its first axis to the `'y'` mesh dimension. The additional sharding will be carried over to the first axis of the result `c`.\n",
+        "\n",
+        "Total number of floating point mul operations is `6 devices * 2 result * 1 = 12`, an additional factor of 2 reduction compared to the case (24) above. The factor of 2 is due to the sharding along `y` mesh dimension with a size of `2` devices."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0PYqe0neiOpR"
+      },
+      "outputs": [],
+      "source": [
+        "mesh = dtensor.create_mesh([(\"x\", 3), (\"y\", 2)], devices=DEVICES)\n",
+        "\n",
+        "a_layout = dtensor.Layout(['y', 'x'], mesh)\n",
+        "a = dtensor_from_array([[1, 2, 3], [4, 5, 6]], layout=a_layout)\n",
+        "b_layout = dtensor.Layout(['x', dtensor.UNSHARDED], mesh)\n",
+        "b = dtensor_from_array([[6, 5], [4, 3], [2, 1]], layout=b_layout)\n",
+        "\n",
+        "c = tf.matmul(a, b)\n",
+        "# The sharding of `a` on the first axis is carried to `c'\n",
+        "print('Sharding spec:', dtensor.fetch_layout(c).sharding_specs)\n",
+        "print(\"components:\")\n",
+        "for component_tensor in dtensor.unpack(c):\n",
+        "  print(component_tensor.device, component_tensor.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c-1NazCVmLWZ"
+      },
+      "source": [
+        "### DTensor as output\n",
+        "\n",
+        "What about Python functions that do not take operands, but returns a Tensor result that can be sharded? Examples of such functions are:\n",
+        "\n",
+        "  - `tf.ones`, `tf.zeros`, `tf.random.stateless_normal`\n",
+        "\n",
+        "For these Python functions, DTensor provides `dtensor.call_with_layout` which eagerly executes a Python function with DTensor, and ensures that the returned Tensor is a DTensor with the requested `Layout`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "J0jo_8NPtJiO"
+      },
+      "outputs": [],
+      "source": [
+        "help(dtensor.call_with_layout)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "V-YdLvfytM7g"
+      },
+      "source": [
+        "The eagerly executed Python function usually only contain a single non-trivial TensorFlow Op.\n",
+        "\n",
+        "To use a Python function that emits multiple TensorFlow Ops with `dtensor.call_with_layout`, the function should be converted to a `tf.function`. Calling a `tf.function` is a single TensorFlow Op. When the `tf.function` is called, DTensor can perform layout propagation when it analyzes the computing graph of the `tf.function`, before any of the intermediate tensors are materialized."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DLrksgFjqRLS"
+      },
+      "source": [
+        "#### APIs that emit a single TensorFlow Op\n",
+        "\n",
+        "If a function  emits a single TensorFlow Op, you can directly apply `dtensor.call_with_layout` to the function:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "G1CuKYSFtFeM"
+      },
+      "outputs": [],
+      "source": [
+        "help(tf.ones)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2m_EAwy-ozOh"
+      },
+      "outputs": [],
+      "source": [
+        "mesh = dtensor.create_mesh([(\"x\", 3), (\"y\", 2)], devices=DEVICES)\n",
+        "ones = dtensor.call_with_layout(tf.ones, dtensor.Layout(['x', 'y'], mesh), shape=(6, 4))\n",
+        "print(ones)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bx-7Xo8Cpb8S"
+      },
+      "source": [
+        "#### APIs that emit multiple TensorFlow Ops\n",
+        "\n",
+        "If the API emits multiple TensorFlow Ops, convert the function into a single Op through `tf.function`. For example, `tf.random.stateleess_normal`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H8BQSTRFtCih"
+      },
+      "outputs": [],
+      "source": [
+        "help(tf.random.stateless_normal)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TvP81eYopSPm"
+      },
+      "outputs": [],
+      "source": [
+        "ones = dtensor.call_with_layout(\n",
+        "    tf.function(tf.random.stateless_normal),\n",
+        "    dtensor.Layout(['x', 'y'], mesh),\n",
+        "    shape=(6, 4),\n",
+        "    seed=(1, 1))\n",
+        "print(ones)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qKoojp9ZyWzW"
+      },
+      "source": [
+        "Wrapping a Python function that emits a single TensorFlow Op with `tf.function` is allowed. The only caveat is paying the associated cost and complexity of creating a `tf.function` from a Python function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LbAtKrSkpOaq"
+      },
+      "outputs": [],
+      "source": [
+        "ones = dtensor.call_with_layout(\n",
+        "    tf.function(tf.ones),\n",
+        "    dtensor.Layout(['x', 'y'], mesh),\n",
+        "    shape=(6, 4))\n",
+        "print(ones)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "D-m1816JP3CE"
+      },
+      "source": [
+        "### From `tf.Variable` to `dtensor.DVariable`\n",
+        "\n",
+        "In Tensorflow, `tf.Variable` is the holder for a mutable `Tensor` value.\n",
+        "With DTensor, the corresponding variable semantics is provided by `dtensor.DVariable`.\n",
+        "\n",
+        "The reason a new type `DVariable` was introduced for DTensor variable is because DVariables have an additional requirement that the layout cannot change from its initial value."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "awRPuR26P0Sc"
+      },
+      "outputs": [],
+      "source": [
+        "mesh = dtensor.create_mesh([(\"x\", 6)], devices=DEVICES)\n",
+        "layout = dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh)\n",
+        "\n",
+        "v = dtensor.DVariable(\n",
+        "    initial_value=dtensor.call_with_layout(\n",
+        "        tf.function(tf.random.stateless_normal),\n",
+        "        layout=layout,\n",
+        "        shape=tf.TensorShape([64, 32]),\n",
+        "        seed=[1, 1],\n",
+        "        dtype=tf.float32))\n",
+        "\n",
+        "print(v.handle)\n",
+        "assert layout == dtensor.fetch_layout(v)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pb9jn473prC_"
+      },
+      "source": [
+        "Other than the requirement on matching the `layout`, a `DVariable` behaves the same as a `tf.Variable`. For example, you can add a DVariable to a DTensor,\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "adxFw9wJpqQQ"
+      },
+      "outputs": [],
+      "source": [
+        "a = dtensor.call_with_layout(tf.ones, layout=layout, shape=(64, 32))\n",
+        "b = v + a # add DVariable and DTensor\n",
+        "print(b)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QxBdNHWSu-kV"
+      },
+      "source": [
+        "You can also assign a DTensor to a DVariable:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oYwfiyw5P94U"
+      },
+      "outputs": [],
+      "source": [
+        "v.assign(a) # assign a DTensor to a DVariable\n",
+        "print(a)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4fvSk_VUvGnj"
+      },
+      "source": [
+        "Attempting to mutate the layout of a `DVariable`, by assigning a DTensor with an incompatible layout produces an error:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3pckUugYP_r-"
+      },
+      "outputs": [],
+      "source": [
+        "# variable's layout is immutable.\n",
+        "another_mesh = dtensor.create_mesh([(\"x\", 3), (\"y\", 2)], devices=DEVICES)\n",
+        "b = dtensor.call_with_layout(tf.ones,\n",
+        "                     layout=dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], another_mesh),\n",
+        "                     shape=(64, 32))\n",
+        "try:\n",
+        "  v.assign(b)\n",
+        "except:\n",
+        "  print(\"exception raised\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3LadIcwRvR6f"
+      },
+      "source": [
+        "## What's next?\n",
+        "\n",
+        "In this colab, you learned about DTensor, an extension to TensorFlow for distributed computing. To try out these concepts in a tutorial, check out [Distributed training with DTensor](../tutorials/distribute/dtensor_ml_tutorial.ipynb)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "dtensor_overview.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/eager.ipynb b/site/en/guide/eager.ipynb
deleted file mode 100644
index 35fa0a2e1f9..00000000000
--- a/site/en/guide/eager.ipynb
+++ /dev/null
@@ -1,1376 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "CCQY7jpBfMur"
-      },
-      "source": [
-        "##### Copyright 2018 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
-        "id": "z6X9omPnfO_h"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2QQJJyDzqGRb"
-      },
-      "source": [
-        "# Eager execution\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "B1xdylywqUSX"
-      },
-      "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/eager\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/eager.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/eager.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/eager.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "EGjDcGxIqEfX"
-      },
-      "source": [
-        "\n",
-        "\n",
-        "TensorFlow's eager execution is an imperative programming environment that\n",
-        "evaluates operations immediately, without building graphs: operations return\n",
-        "concrete values instead of constructing a computational graph to run later. This\n",
-        "makes it easy to get started with TensorFlow and debug models, and it\n",
-        "reduces boilerplate as well. To follow along with this guide, run the code\n",
-        "samples below in an interactive `python` interpreter.\n",
-        "\n",
-        "Eager execution is a flexible machine learning platform for research and\n",
-        "experimentation, providing:\n",
-        "\n",
-        "* *An intuitive interface*—Structure your code naturally and use Python data\n",
-        "  structures. Quickly iterate on small models and small data.\n",
-        "* *Easier debugging*—Call ops directly to inspect running models and test\n",
-        "  changes. Use standard Python debugging tools for immediate error reporting.\n",
-        "* *Natural control flow*—Use Python control flow instead of graph control\n",
-        "  flow, simplifying the specification of dynamic models.\n",
-        "\n",
-        "Eager execution supports most TensorFlow operations and GPU acceleration.\n",
-        "\n",
-        "Note: Some models may experience increased overhead with eager execution\n",
-        "enabled. Performance improvements are ongoing, but please\n",
-        "[file a bug](https://github.com/tensorflow/tensorflow/issues) if you find a\n",
-        "problem and share your benchmarks."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "RBAeIwOMrYk8"
-      },
-      "source": [
-        "## Setup and basic usage"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ByNsp4VqqEfa"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "import os\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x  #gpu\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "import cProfile"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "48P3-8q4qEfe"
-      },
-      "source": [
-        "In Tensorflow 2.0, eager execution is enabled by default."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "7aFsD8csqEff"
-      },
-      "outputs": [],
-      "source": [
-        "tf.executing_eagerly()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "x_G1zZT5qEfh"
-      },
-      "source": [
-        "Now you can run TensorFlow operations and the results will return immediately:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "9gsI54pbqEfj"
-      },
-      "outputs": [],
-      "source": [
-        "x = [[2.]]\n",
-        "m = tf.matmul(x, x)\n",
-        "print(\"hello, {}\".format(m))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ajFn6qsdqEfl"
-      },
-      "source": [
-        "Enabling eager execution changes how TensorFlow operations behave—now they\n",
-        "immediately evaluate and return their values to Python. `tf.Tensor` objects\n",
-        "reference concrete values instead of symbolic handles to nodes in a computational\n",
-        "graph. Since there isn't a computational graph to build and run later in a\n",
-        "session, it's easy to inspect results using `print()` or a debugger. Evaluating,\n",
-        "printing, and checking tensor values does not break the flow for computing\n",
-        "gradients.\n",
-        "\n",
-        "Eager execution works nicely with [NumPy](http://www.numpy.org/). NumPy\n",
-        "operations accept `tf.Tensor` arguments. TensorFlow\n",
-        "[math operations](https://www.tensorflow.org/api_guides/python/math_ops) convert\n",
-        "Python objects and NumPy arrays to `tf.Tensor` objects. The\n",
-        "`tf.Tensor.numpy` method returns the object's value as a NumPy `ndarray`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "sTO0_5TYqz1n"
-      },
-      "outputs": [],
-      "source": [
-        "a = tf.constant([[1, 2],\n",
-        "                 [3, 4]])\n",
-        "print(a)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Dp14YT8Gq4r1"
-      },
-      "outputs": [],
-      "source": [
-        "# Broadcasting support\n",
-        "b = tf.add(a, 1)\n",
-        "print(b)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "69p3waMfq8cQ"
-      },
-      "outputs": [],
-      "source": [
-        "# Operator overloading is supported\n",
-        "print(a * b)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "Ui025t1qqEfm"
-      },
-      "outputs": [],
-      "source": [
-        "# Use NumPy values\n",
-        "import numpy as np\n",
-        "\n",
-        "c = np.multiply(a, b)\n",
-        "print(c)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Tq_aFRzWrCua"
-      },
-      "outputs": [],
-      "source": [
-        "# Obtain numpy value from a tensor:\n",
-        "print(a.numpy())\n",
-        "# =\u003e [[1 2]\n",
-        "#     [3 4]]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "H08f9ss9qEft"
-      },
-      "source": [
-        "## Dynamic control flow\n",
-        "\n",
-        "A major benefit of eager execution is that all the functionality of the host\n",
-        "language is available while your model is executing. So, for example,\n",
-        "it is easy to write [fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz):"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "0fudRMeUqEfu"
-      },
-      "outputs": [],
-      "source": [
-        "def fizzbuzz(max_num):\n",
-        "  counter = tf.constant(0)\n",
-        "  max_num = tf.convert_to_tensor(max_num)\n",
-        "  for num in range(1, max_num.numpy()+1):\n",
-        "    num = tf.constant(num)\n",
-        "    if int(num % 3) == 0 and int(num % 5) == 0:\n",
-        "      print('FizzBuzz')\n",
-        "    elif int(num % 3) == 0:\n",
-        "      print('Fizz')\n",
-        "    elif int(num % 5) == 0:\n",
-        "      print('Buzz')\n",
-        "    else:\n",
-        "      print(num.numpy())\n",
-        "    counter += 1"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "P2cKknQWrJLB"
-      },
-      "outputs": [],
-      "source": [
-        "fizzbuzz(15)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "7kA-aC3BqEfy"
-      },
-      "source": [
-        "This has conditionals that depend on tensor values and it prints these values\n",
-        "at runtime."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "8huKpuuAwICq"
-      },
-      "source": [
-        "## Eager training"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "mp2lCCZYrxHd"
-      },
-      "source": [
-        "### Computing gradients\n",
-        "\n",
-        "[Automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation)\n",
-        "is useful for implementing machine learning algorithms such as\n",
-        "[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training\n",
-        "neural networks. During eager execution, use `tf.GradientTape` to trace\n",
-        "operations for computing gradients later.\n",
-        "\n",
-        "You can use `tf.GradientTape` to train and/or compute gradients in eager. It is especially useful for complicated training loops.  \n",
-        "\n",
-        "Since different operations can occur during each call, all\n",
-        "forward-pass operations get recorded to a \"tape\". To compute the gradient, play\n",
-        "the tape backwards and then discard. A particular `tf.GradientTape` can only\n",
-        "compute one gradient; subsequent calls throw a runtime error."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "7g1yWiSXqEf-"
-      },
-      "outputs": [],
-      "source": [
-        "w = tf.Variable([[1.0]])\n",
-        "with tf.GradientTape() as tape:\n",
-        "  loss = w * w\n",
-        "\n",
-        "grad = tape.gradient(loss, w)\n",
-        "print(grad)  # =\u003e tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vkHs32GqweYS"
-      },
-      "source": [
-        "### Train a model\n",
-        "\n",
-        "The following example creates a multi-layer model that classifies the standard\n",
-        "MNIST handwritten digits. It demonstrates the optimizer and layer APIs to build\n",
-        "trainable graphs in an eager execution environment."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "38kymXZowhhz"
-      },
-      "outputs": [],
-      "source": [
-        "# Fetch and format the mnist data\n",
-        "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n",
-        "\n",
-        "dataset = tf.data.Dataset.from_tensor_slices(\n",
-        "  (tf.cast(mnist_images[...,tf.newaxis]/255, tf.float32),\n",
-        "   tf.cast(mnist_labels,tf.int64)))\n",
-        "dataset = dataset.shuffle(1000).batch(32)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "rl1K8rOowmwT"
-      },
-      "outputs": [],
-      "source": [
-        "# Build the model\n",
-        "mnist_model = tf.keras.Sequential([\n",
-        "  tf.keras.layers.Conv2D(16,[3,3], activation='relu',\n",
-        "                         input_shape=(None, None, 1)),\n",
-        "  tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n",
-        "  tf.keras.layers.GlobalAveragePooling2D(),\n",
-        "  tf.keras.layers.Dense(10)\n",
-        "])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "fvyk-HgGwxwl"
-      },
-      "source": [
-        "\n",
-        "Even without training, call the model and inspect the output in eager execution:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "BsxystjBwxLS"
-      },
-      "outputs": [],
-      "source": [
-        "for images,labels in dataset.take(1):\n",
-        "  print(\"Logits: \", mnist_model(images[0:1]).numpy())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Y3PGa8G7qEgB"
-      },
-      "source": [
-        "While keras models have a builtin training loop (using the `fit` method), sometimes you need more customization. Here's an example, of a training loop implemented with eager:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "bzRhM7JDnaEG"
-      },
-      "outputs": [],
-      "source": [
-        "optimizer = tf.keras.optimizers.Adam()\n",
-        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
-        "\n",
-        "loss_history = []"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "tXaupYXRI2YM"
-      },
-      "source": [
-        "Note: Use the assert functions in `tf.debugging` to check if a condition holds up. This works in eager and graph execution."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "DDHrigtiCIA4"
-      },
-      "outputs": [],
-      "source": [
-        "def train_step(images, labels):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    logits = mnist_model(images, training=True)\n",
-        "    \n",
-        "    # Add asserts to check the shape of the output.\n",
-        "    tf.debugging.assert_equal(logits.shape, (32, 10))\n",
-        "    \n",
-        "    loss_value = loss_object(labels, logits)\n",
-        "\n",
-        "  loss_history.append(loss_value.numpy().mean())\n",
-        "  grads = tape.gradient(loss_value, mnist_model.trainable_variables)\n",
-        "  optimizer.apply_gradients(zip(grads, mnist_model.trainable_variables))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "0m1xAXrmqEgJ"
-      },
-      "outputs": [],
-      "source": [
-        "def train(epochs):\n",
-        "  for epoch in range(epochs):\n",
-        "    for (batch, (images, labels)) in enumerate(dataset):\n",
-        "      train_step(images, labels)\n",
-        "    print ('Epoch {} finished'.format(epoch))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "C5dGz0p_nf4W"
-      },
-      "outputs": [],
-      "source": [
-        "train(epochs = 3)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "5vG5ql_2vYB5"
-      },
-      "outputs": [],
-      "source": [
-        "import matplotlib.pyplot as plt\n",
-        "\n",
-        "plt.plot(loss_history)\n",
-        "plt.xlabel('Batch #')\n",
-        "plt.ylabel('Loss [entropy]')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kKpOlHPLqEgl"
-      },
-      "source": [
-        "### Variables and optimizers\n",
-        "\n",
-        "`tf.Variable` objects store mutable `tf.Tensor`-like values accessed during\n",
-        "training to make automatic differentiation easier. \n",
-        "\n",
-        "The collections of variables can be encapsulated into layers or models, along with methods that operate on them. See [Custom Keras layers and models](../keras/custom_layers_and_models.ipynb) for details. The main difference between layers and models is that models add methods like  `Model.fit`, `Model.evaluate`, and `Model.save`.\n",
-        "\n",
-        "For example, the automatic differentiation example above\n",
-        "can be rewritten:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "2qXcPngYk8dN"
-      },
-      "outputs": [],
-      "source": [
-        "class Linear(tf.keras.Model):\n",
-        "  def __init__(self):\n",
-        "    super(Linear, self).__init__()\n",
-        "    self.W = tf.Variable(5., name='weight')\n",
-        "    self.B = tf.Variable(10., name='bias')\n",
-        "  def call(self, inputs):\n",
-        "    return inputs * self.W + self.B"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "nnQLBYmEqEgm"
-      },
-      "outputs": [],
-      "source": [
-        "# A toy dataset of points around 3 * x + 2\n",
-        "NUM_EXAMPLES = 2000\n",
-        "training_inputs = tf.random.normal([NUM_EXAMPLES])\n",
-        "noise = tf.random.normal([NUM_EXAMPLES])\n",
-        "training_outputs = training_inputs * 3 + 2 + noise\n",
-        "\n",
-        "# The loss function to be optimized\n",
-        "def loss(model, inputs, targets):\n",
-        "  error = model(inputs) - targets\n",
-        "  return tf.reduce_mean(tf.square(error))\n",
-        "\n",
-        "def grad(model, inputs, targets):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    loss_value = loss(model, inputs, targets)\n",
-        "  return tape.gradient(loss_value, [model.W, model.B])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Q7x1CDurl3IG"
-      },
-      "source": [
-        "Next:\n",
-        "\n",
-        "1. Create the model.\n",
-        "2. The Derivatives of a loss function with respect to model parameters.\n",
-        "3. A strategy for updating the variables based on the derivatives."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "SbXJk0f2lztg"
-      },
-      "outputs": [],
-      "source": [
-        "model = Linear()\n",
-        "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)\n",
-        "\n",
-        "print(\"Initial loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n",
-        "\n",
-        "steps = 300\n",
-        "for i in range(steps):\n",
-        "  grads = grad(model, training_inputs, training_outputs)\n",
-        "  optimizer.apply_gradients(zip(grads, [model.W, model.B]))\n",
-        "  if i % 20 == 0:\n",
-        "    print(\"Loss at step {:03d}: {:.3f}\".format(i, loss(model, training_inputs, training_outputs)))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "PV_dqer7pzSH"
-      },
-      "outputs": [],
-      "source": [
-        "print(\"Final loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "rvt_Wj3Tp0hm"
-      },
-      "outputs": [],
-      "source": [
-        "print(\"W = {}, B = {}\".format(model.W.numpy(), model.B.numpy()))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rPjb8nRWqEgr"
-      },
-      "source": [
-        "\n",
-        "Note: Variables persist until the last reference to the python object\n",
-        "is removed, and is the variable is deleted."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "scMjg6L6qEgv"
-      },
-      "source": [
-        "### Object-based saving\n",
-        "\n",
-        "\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Y-0ZcCcjwkux"
-      },
-      "source": [
-        "A `tf.keras.Model` includes a covienient `save_weights` method allowing you to easily create a checkpoint: "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "oJrMX94PwD9s"
-      },
-      "outputs": [],
-      "source": [
-        "model.save_weights('weights')\n",
-        "status = model.load_weights('weights')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2EfTjWV_wEng"
-      },
-      "source": [
-        "Using `tf.train.Checkpoint` you can take full control over this process.\n",
-        "\n",
-        "This section is an abbreviated version of the [guide to training checkpoints](./checkpoint.ipynb).\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "7z5xRfdHzZOQ"
-      },
-      "outputs": [],
-      "source": [
-        "x = tf.Variable(10.)\n",
-        "checkpoint = tf.train.Checkpoint(x=x)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "IffrUVG7zyVb"
-      },
-      "outputs": [],
-      "source": [
-        "x.assign(2.)   # Assign a new value to the variables and save.\n",
-        "checkpoint_path = './ckpt/'\n",
-        "checkpoint.save('./ckpt/')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "eMT9koCoqEgw"
-      },
-      "outputs": [],
-      "source": [
-        "x.assign(11.)  # Change the variable after saving.\n",
-        "\n",
-        "# Restore values from the checkpoint\n",
-        "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_path))\n",
-        "\n",
-        "print(x)  # =\u003e 2.0"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vbFnP-yLqEgx"
-      },
-      "source": [
-        "To save and load models, `tf.train.Checkpoint` stores the internal state of objects,\n",
-        "without requiring hidden variables. To record the state of a `model`,\n",
-        "an `optimizer`, and a global step, pass them to a `tf.train.Checkpoint`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "hWZHyAXMqEg0"
-      },
-      "outputs": [],
-      "source": [
-        "model = tf.keras.Sequential([\n",
-        "  tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n",
-        "  tf.keras.layers.GlobalAveragePooling2D(),\n",
-        "  tf.keras.layers.Dense(10)\n",
-        "])\n",
-        "optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
-        "checkpoint_dir = 'path/to/model_dir'\n",
-        "if not os.path.exists(checkpoint_dir):\n",
-        "  os.makedirs(checkpoint_dir)\n",
-        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
-        "root = tf.train.Checkpoint(optimizer=optimizer,\n",
-        "                           model=model)\n",
-        "\n",
-        "root.save(checkpoint_prefix)\n",
-        "root.restore(tf.train.latest_checkpoint(checkpoint_dir))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "R-ITwkBCF6GJ"
-      },
-      "source": [
-        "Note: In many training loops, variables are created after `tf.train.Checkpoint.restore` is called. These variables will be restored as soon as they are created, and assertions are available to ensure that a checkpoint has been fully loaded. See the [guide to training checkpoints](./checkpoint.ipynb) for details."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "3yoD0VJ7qEg3"
-      },
-      "source": [
-        "### Object-oriented metrics\n",
-        "\n",
-        "`tf.keras.metrics` are stored as objects. Update a metric by passing the new data to\n",
-        "the callable, and retrieve the result using the `tf.keras.metrics.result` method,\n",
-        "for example:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "9ccu0iAaqEg5"
-      },
-      "outputs": [],
-      "source": [
-        "m = tf.keras.metrics.Mean(\"loss\")\n",
-        "m(0)\n",
-        "m(5)\n",
-        "m.result()  # =\u003e 2.5\n",
-        "m([8, 9])\n",
-        "m.result()  # =\u003e 5.5"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "aB8qWtT955pI"
-      },
-      "source": [
-        "### Summaries and TensorBoard\n",
-        "\n",
-        "[TensorBoard](https://tensorflow.org/tensorboard) is a visualization tool for\n",
-        "understanding, debugging and optimizing the model training process. It uses\n",
-        "summary events that are written while executing the program.\n",
-        "\n",
-        "You can use `tf.summary` to record summaries of variable in eager execution.\n",
-        "For example, to record summaries of `loss` once every 100 training steps:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "z6VInqhA6RH4"
-      },
-      "outputs": [],
-      "source": [
-        "logdir = \"./tb/\"\n",
-        "writer = tf.summary.create_file_writer(logdir)\n",
-        "\n",
-        "steps = 1000\n",
-        "with writer.as_default():  # or call writer.set_as_default() before the loop.\n",
-        "  for i in range(steps):\n",
-        "    step = i + 1\n",
-        "    # Calculate loss with your real train function.\n",
-        "    loss = 1 - 0.001 * step\n",
-        "    if step % 100 == 0:\n",
-        "      tf.summary.scalar('loss', loss, step=step)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "08QQD2j36TaI"
-      },
-      "outputs": [],
-      "source": [
-        "!ls tb/"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "xEL4yJe5qEhD"
-      },
-      "source": [
-        "## Advanced automatic differentiation topics\n",
-        "\n",
-        "### Dynamic models\n",
-        "\n",
-        "`tf.GradientTape` can also be used in dynamic models. This example for a\n",
-        "[backtracking line search](https://wikipedia.org/wiki/Backtracking_line_search)\n",
-        "algorithm looks like normal NumPy code, except there are gradients and is\n",
-        "differentiable, despite the complex control flow:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "L518n5dkqEhE"
-      },
-      "outputs": [],
-      "source": [
-        "def line_search_step(fn, init_x, rate=1.0):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    # Variables are automatically tracked.\n",
-        "    # But to calculate a gradient from a tensor, you must `watch` it.\n",
-        "    tape.watch(init_x)\n",
-        "    value = fn(init_x)\n",
-        "  grad = tape.gradient(value, init_x)\n",
-        "  grad_norm = tf.reduce_sum(grad * grad)\n",
-        "  init_value = value\n",
-        "  while value \u003e init_value - rate * grad_norm:\n",
-        "    x = init_x - rate * grad\n",
-        "    value = fn(x)\n",
-        "    rate /= 2.0\n",
-        "  return x, value"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "gieGOf_DqEhK"
-      },
-      "source": [
-        "### Custom gradients\n",
-        "\n",
-        "Custom gradients are an easy way to override gradients. Within the forward function, define the gradient with respect to the\n",
-        "inputs, outputs, or intermediate results. For example, here's an easy way to clip\n",
-        "the norm of the gradients in the backward pass:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "-OwwsWUAqEhK"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.custom_gradient\n",
-        "def clip_gradient_by_norm(x, norm):\n",
-        "  y = tf.identity(x)\n",
-        "  def grad_fn(dresult):\n",
-        "    return [tf.clip_by_norm(dresult, norm), None]\n",
-        "  return y, grad_fn"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "JPLDHkF_qEhN"
-      },
-      "source": [
-        "Custom gradients are commonly used to provide a numerically stable gradient for a\n",
-        "sequence of operations:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "24WiLROnqEhO"
-      },
-      "outputs": [],
-      "source": [
-        "def log1pexp(x):\n",
-        "  return tf.math.log(1 + tf.exp(x))\n",
-        "\n",
-        "def grad_log1pexp(x):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    tape.watch(x)\n",
-        "    value = log1pexp(x)\n",
-        "  return tape.gradient(value, x)\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "n8fq69r9-B-c"
-      },
-      "outputs": [],
-      "source": [
-        "# The gradient computation works fine at x = 0.\n",
-        "grad_log1pexp(tf.constant(0.)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "_VFSU0mG-FSp"
-      },
-      "outputs": [],
-      "source": [
-        "# However, x = 100 fails because of numerical instability.\n",
-        "grad_log1pexp(tf.constant(100.)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-VcTR34rqEhQ"
-      },
-      "source": [
-        "Here, the `log1pexp` function can be analytically simplified with a custom\n",
-        "gradient. The implementation below reuses the value for `tf.exp(x)` that is\n",
-        "computed during the forward pass—making it more efficient by eliminating\n",
-        "redundant calculations:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "Q7nvfx_-qEhS"
-      },
-      "outputs": [],
-      "source": [
-        "@tf.custom_gradient\n",
-        "def log1pexp(x):\n",
-        "  e = tf.exp(x)\n",
-        "  def grad(dy):\n",
-        "    return dy * (1 - 1 / (1 + e))\n",
-        "  return tf.math.log(1 + e), grad\n",
-        "\n",
-        "def grad_log1pexp(x):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    tape.watch(x)\n",
-        "    value = log1pexp(x)\n",
-        "  return tape.gradient(value, x)\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "5gHPKMfl-Kge"
-      },
-      "outputs": [],
-      "source": [
-        "# As before, the gradient computation works fine at x = 0.\n",
-        "grad_log1pexp(tf.constant(0.)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "u38MOfz3-MDE"
-      },
-      "outputs": [],
-      "source": [
-        "# And the gradient computation also works at x = 100.\n",
-        "grad_log1pexp(tf.constant(100.)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rnZXjfQzqEhV"
-      },
-      "source": [
-        "## Performance\n",
-        "\n",
-        "Computation is automatically offloaded to GPUs during eager execution. If you\n",
-        "want control over where a computation runs you can enclose it in a\n",
-        "`tf.device('/gpu:0')` block (or the CPU equivalent):"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "Ac9Y64H-qEhX"
-      },
-      "outputs": [],
-      "source": [
-        "import time\n",
-        "\n",
-        "def measure(x, steps):\n",
-        "  # TensorFlow initializes a GPU the first time it's used, exclude from timing.\n",
-        "  tf.matmul(x, x)\n",
-        "  start = time.time()\n",
-        "  for i in range(steps):\n",
-        "    x = tf.matmul(x, x)\n",
-        "  # tf.matmul can return before completing the matrix multiplication\n",
-        "  # (e.g., can return after enqueing the operation on a CUDA stream).\n",
-        "  # The x.numpy() call below will ensure that all enqueued operations\n",
-        "  # have completed (and will also copy the result to host memory,\n",
-        "  # so we're including a little more than just the matmul operation\n",
-        "  # time).\n",
-        "  _ = x.numpy()\n",
-        "  end = time.time()\n",
-        "  return end - start\n",
-        "\n",
-        "shape = (1000, 1000)\n",
-        "steps = 200\n",
-        "print(\"Time to multiply a {} matrix by itself {} times:\".format(shape, steps))\n",
-        "\n",
-        "# Run on CPU:\n",
-        "with tf.device(\"/cpu:0\"):\n",
-        "  print(\"CPU: {} secs\".format(measure(tf.random.normal(shape), steps)))\n",
-        "\n",
-        "# Run on GPU, if available:\n",
-        "if tf.config.experimental.list_physical_devices(\"GPU\"):\n",
-        "  with tf.device(\"/gpu:0\"):\n",
-        "    print(\"GPU: {} secs\".format(measure(tf.random.normal(shape), steps)))\n",
-        "else:\n",
-        "  print(\"GPU: not found\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "RLw3IS7UqEhe"
-      },
-      "source": [
-        "A `tf.Tensor` object can be copied to a different device to execute its\n",
-        "operations:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
-        "id": "ny6LX2BVqEhf"
-      },
-      "outputs": [],
-      "source": [
-        "if tf.config.experimental.list_physical_devices(\"GPU\"):\n",
-        "  x = tf.random.normal([10, 10])\n",
-        "\n",
-        "  x_gpu0 = x.gpu()\n",
-        "  x_cpu = x.cpu()\n",
-        "\n",
-        "  _ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU\n",
-        "  _ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "oA_qaII3-p6c"
-      },
-      "source": [
-        "### Benchmarks\n",
-        "\n",
-        "For compute-heavy models, such as\n",
-        "[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)\n",
-        "training on a GPU, eager execution performance is comparable to `tf.function` execution.\n",
-        "But this gap grows larger for models with less computation and there is work to\n",
-        "be done for optimizing hot code paths for models with lots of small operations.\n",
-        "\n",
-        "## Work with functions\n",
-        "\n",
-        "While eager execution makes development and debugging more interactive,\n",
-        "TensorFlow 1.x style graph execution has advantages for distributed training, performance\n",
-        "optimizations, and production deployment. To bridge this gap, TensorFlow 2.0 introduces `function`s via the `tf.function` API. For more information, see the [tf.function](./function.ipynb) guide."
-      ]
-    }
-  ],
-  "metadata": {
-    "accelerator": "GPU",
-    "colab": {
-      "collapsed_sections": [],
-      "name": "eager.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/effective_tf2.ipynb b/site/en/guide/effective_tf2.ipynb
new file mode 100644
index 00000000000..f4204c0971f
--- /dev/null
+++ b/site/en/guide/effective_tf2.ipynb
@@ -0,0 +1,797 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qFdPvlXBOdUN"
+      },
+      "source": [
+        "# Effective Tensorflow 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xHxb-dlhMIzW"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This guide provides a list of best practices for writing code using TensorFlow 2 (TF2), it is written for users who have recently switched over from TensorFlow 1 (TF1).  Refer to the [migrate section of the guide](https://tensorflow.org/guide/migrate) for more info on migrating your TF1 code to TF2."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MUXex9ctTuDB"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Import TensorFlow and other dependencies for the examples in this guide."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IqR2PQG4ZaZ0"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_datasets as tfds"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ngds9zateIY8"
+      },
+      "source": [
+        "## Recommendations for idiomatic TensorFlow 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "B3RdHaroMAi4"
+      },
+      "source": [
+        "### Refactor your code into smaller modules\n",
+        "\n",
+        "A good practice is to refactor your code into smaller functions that are called as needed. For best performance, you should try to decorate the largest blocks of computation that you can in a `tf.function` (note that the nested python functions called by a `tf.function` do not require their own separate decorations, unless you want to use different `jit_compile` settings for the `tf.function`). Depending on your use case, this could be multiple training steps or even your whole training loop. For inference use cases, it might be a single model forward pass."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Rua1l8et3Evd"
+      },
+      "source": [
+        "### Adjust the default learning rate for some `tf.keras.optimizer`s\n",
+        "\n",
+        "\n",
+        "Some Keras optimizers have different learning rates in TF2. If you see a change in convergence behavior for your models, check the default learning rates.\n",
+        "\n",
+        "There are no changes for `optimizers.SGD`, `optimizers.Adam`, or `optimizers.RMSprop`.\n",
+        "\n",
+        "The following default learning rates have changed:\n",
+        "\n",
+        "- `optimizers.Adagrad` from `0.01` to `0.001`\n",
+        "- `optimizers.Adadelta` from `1.0` to `0.001`\n",
+        "- `optimizers.Adamax` from `0.002` to `0.001`\n",
+        "- `optimizers.Nadam` from `0.002` to `0.001`"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z6LfkpsEldEV"
+      },
+      "source": [
+        "### Use `tf.Module`s and Keras layers to manage variables\n",
+        "\n",
+        "`tf.Module`s and `tf.keras.layers.Layer`s offer the convenient `variables` and\n",
+        "`trainable_variables` properties, which recursively gather up all dependent\n",
+        "variables. This makes it easy to manage variables locally to where they are\n",
+        "being used."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WQ2U0rj1oBlc"
+      },
+      "source": [
+        "Keras layers/models inherit from `tf.train.Checkpointable` and are integrated\n",
+        "with `@tf.function`, which makes it possible to directly checkpoint or export\n",
+        "SavedModels from Keras objects. You do not necessarily have to use Keras'\n",
+        "`Model.fit` API to take advantage of these integrations.\n",
+        "\n",
+        "Read the section on [transfer learning and fine-tuning](https://www.tensorflow.org/guide/keras/transfer_learning#transfer_learning_fine-tuning_with_a_custom_training_loop) in the Keras guide to learn how to collect a subset of relevant variables using Keras."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "j34MsfxWodG6"
+      },
+      "source": [
+        "### Combine `tf.data.Dataset`s and `tf.function`\n",
+        "\n",
+        "The [TensorFlow Datasets](https://tensorflow.org/datasets) package (`tfds`) contains utilities for loading predefined datasets as `tf.data.Dataset` objects. For this example, you can load the MNIST dataset using `tfds`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BMgxaLH74_s-"
+      },
+      "outputs": [],
+      "source": [
+        "datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n",
+        "mnist_train, mnist_test = datasets['train'], datasets['test']"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hPJhEuvj5VfR"
+      },
+      "source": [
+        "Then prepare the data for training:\n",
+        "\n",
+        "  - Re-scale each image.\n",
+        "  - Shuffle the order of the examples.\n",
+        "  - Collect batches of images and labels.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "StBRHtJM2S7o"
+      },
+      "outputs": [],
+      "source": [
+        "BUFFER_SIZE = 10 # Use a much larger value for real code\n",
+        "BATCH_SIZE = 64\n",
+        "NUM_EPOCHS = 5\n",
+        "\n",
+        "\n",
+        "def scale(image, label):\n",
+        "  image = tf.cast(image, tf.float32)\n",
+        "  image /= 255\n",
+        "\n",
+        "  return image, label"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SKq14zKKFAdv"
+      },
+      "source": [
+        "To keep the example short, trim the dataset to only return 5 batches:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_J-o4YjG2mkM"
+      },
+      "outputs": [],
+      "source": [
+        "train_data = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
+        "test_data = mnist_test.map(scale).batch(BATCH_SIZE)\n",
+        "\n",
+        "STEPS_PER_EPOCH = 5\n",
+        "\n",
+        "train_data = train_data.take(STEPS_PER_EPOCH)\n",
+        "test_data = test_data.take(STEPS_PER_EPOCH)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XEqdkH54VM6c"
+      },
+      "outputs": [],
+      "source": [
+        "image_batch, label_batch = next(iter(train_data))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "loTPH2Pz4_Oj"
+      },
+      "source": [
+        "Use regular Python iteration to iterate over training data that fits in memory. Otherwise, `tf.data.Dataset` is the best way to stream training data from disk. Datasets are [iterables (not iterators)](https://docs.python.org/3/glossary.html#term-iterable), and work just like other Python iterables in eager execution. You can fully utilize dataset async prefetching/streaming features by wrapping your code in `tf.function`, which replaces Python iteration with the equivalent graph operations using AutoGraph.\n",
+        "\n",
+        "```python\n",
+        "@tf.function\n",
+        "def train(model, dataset, optimizer):\n",
+        "  for x, y in dataset:\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      # training=True is only needed if there are layers with different\n",
+        "      # behavior during training versus inference (e.g. Dropout).\n",
+        "      prediction = model(x, training=True)\n",
+        "      loss = loss_fn(prediction, y)\n",
+        "    gradients = tape.gradient(loss, model.trainable_variables)\n",
+        "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
+        "```\n",
+        "\n",
+        "If you use the Keras `Model.fit` API, you won't have to worry about dataset\n",
+        "iteration.\n",
+        "\n",
+        "```python\n",
+        "model.compile(optimizer=optimizer, loss=loss_fn)\n",
+        "model.fit(dataset)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mSev7vZC5GJB"
+      },
+      "source": [
+        "\n",
+        "### Use Keras training loops\n",
+        "\n",
+        "If you don't need low-level control of your training process, using Keras' built-in `fit`, `evaluate`, and `predict` methods is recommended. These methods provide a uniform interface to train the model regardless of the implementation (sequential,  functional, or sub-classed).\n",
+        "\n",
+        "The advantages of these methods include:\n",
+        "\n",
+        "- They accept Numpy arrays, Python generators and, `tf.data.Datasets`.\n",
+        "- They apply regularization, and activation losses automatically.\n",
+        "- They support `tf.distribute` where the training code remains the same [regardless of the hardware configuration](distributed_training.ipynb).\n",
+        "- They support arbitrary callables as losses and metrics.\n",
+        "- They support callbacks like `tf.keras.callbacks.TensorBoard`, and custom callbacks.\n",
+        "- They are performant, automatically using TensorFlow graphs.\n",
+        "\n",
+        "Here is an example of training a model using a `Dataset`. For details on how this works, check out the [tutorials](https://tensorflow.org/tutorials)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uzHFCzd45Rae"
+      },
+      "outputs": [],
+      "source": [
+        "model = tf.keras.Sequential([\n",
+        "    tf.keras.layers.Conv2D(32, 3, activation='relu',\n",
+        "                           kernel_regularizer=tf.keras.regularizers.l2(0.02),\n",
+        "                           input_shape=(28, 28, 1)),\n",
+        "    tf.keras.layers.MaxPooling2D(),\n",
+        "    tf.keras.layers.Flatten(),\n",
+        "    tf.keras.layers.Dropout(0.1),\n",
+        "    tf.keras.layers.Dense(64, activation='relu'),\n",
+        "    tf.keras.layers.BatchNormalization(),\n",
+        "    tf.keras.layers.Dense(10)\n",
+        "])\n",
+        "\n",
+        "# Model is the full model w/o custom layers\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+        "              metrics=['accuracy'])\n",
+        "\n",
+        "model.fit(train_data, epochs=NUM_EPOCHS)\n",
+        "loss, acc = model.evaluate(test_data)\n",
+        "\n",
+        "print(\"Loss {}, Accuracy {}\".format(loss, acc))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LQTaHTuK5S5A"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "### Customize training and write your own loop\n",
+        "\n",
+        "If Keras models work for you, but you need more flexibility and control of the training step or the outer training loops, you can implement your own training steps or even entire training loops. See the Keras guide on [customizing `fit`](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit) to learn more.\n",
+        "\n",
+        "You can also implement many things as a `tf.keras.callbacks.Callback`.\n",
+        "\n",
+        "This method has many of the advantages [mentioned previously](#keras_training_loops), but gives you control of the train step and even the outer loop.\n",
+        "\n",
+        "There are three steps to a standard training loop:\n",
+        "\n",
+        "1. Iterate over a Python generator or `tf.data.Dataset` to get batches of examples.\n",
+        "2. Use `tf.GradientTape` to collect gradients.\n",
+        "3. Use one of the `tf.keras.optimizers` to apply weight updates to the model's variables.\n",
+        "\n",
+        "Remember:\n",
+        "\n",
+        "- Always include a `training` argument on the `call` method of subclassed layers and models.\n",
+        "- Make sure to call the model with the `training` argument set correctly.\n",
+        "- Depending on usage, model variables may not exist until the model is run on a batch of data.\n",
+        "- You need to manually handle things like regularization losses for the model.\n",
+        "\n",
+        "There is no need to run variable initializers or to add manual control dependencies. `tf.function` handles automatic control dependencies and variable initialization on creation for you."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gQooejfYlQeF"
+      },
+      "outputs": [],
+      "source": [
+        "model = tf.keras.Sequential([\n",
+        "    tf.keras.layers.Conv2D(32, 3, activation='relu',\n",
+        "                           kernel_regularizer=tf.keras.regularizers.l2(0.02),\n",
+        "                           input_shape=(28, 28, 1)),\n",
+        "    tf.keras.layers.MaxPooling2D(),\n",
+        "    tf.keras.layers.Flatten(),\n",
+        "    tf.keras.layers.Dropout(0.1),\n",
+        "    tf.keras.layers.Dense(64, activation='relu'),\n",
+        "    tf.keras.layers.BatchNormalization(),\n",
+        "    tf.keras.layers.Dense(10)\n",
+        "])\n",
+        "\n",
+        "optimizer = tf.keras.optimizers.Adam(0.001)\n",
+        "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
+        "\n",
+        "@tf.function\n",
+        "def train_step(inputs, labels):\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    predictions = model(inputs, training=True)\n",
+        "    regularization_loss=tf.math.add_n(model.losses)\n",
+        "    pred_loss=loss_fn(labels, predictions)\n",
+        "    total_loss=pred_loss + regularization_loss\n",
+        "\n",
+        "  gradients = tape.gradient(total_loss, model.trainable_variables)\n",
+        "  optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
+        "\n",
+        "for epoch in range(NUM_EPOCHS):\n",
+        "  for inputs, labels in train_data:\n",
+        "    train_step(inputs, labels)\n",
+        "  print(\"Finished epoch\", epoch)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WikxMFGgo3oZ"
+      },
+      "source": [
+        "### Take advantage of `tf.function` with Python control flow\n",
+        "\n",
+        "`tf.function` provides a way to convert data-dependent control flow into graph-mode\n",
+        "equivalents like `tf.cond` and `tf.while_loop`.\n",
+        "\n",
+        "One common place where data-dependent control flow appears is in sequence\n",
+        "models. `tf.keras.layers.RNN` wraps an RNN cell, allowing you to either\n",
+        "statically or dynamically unroll the recurrence. As an example, you could reimplement dynamic unroll as follows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "n5UebfChRu4T"
+      },
+      "outputs": [],
+      "source": [
+        "class DynamicRNN(tf.keras.Model):\n",
+        "\n",
+        "  def __init__(self, rnn_cell):\n",
+        "    super(DynamicRNN, self).__init__(self)\n",
+        "    self.cell = rnn_cell\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(dtype=tf.float32, shape=[None, None, 3])])\n",
+        "  def call(self, input_data):\n",
+        "\n",
+        "    # [batch, time, features] -> [time, batch, features]\n",
+        "    input_data = tf.transpose(input_data, [1, 0, 2])\n",
+        "    timesteps =  tf.shape(input_data)[0]\n",
+        "    batch_size = tf.shape(input_data)[1]\n",
+        "    outputs = tf.TensorArray(tf.float32, timesteps)\n",
+        "    state = self.cell.get_initial_state(batch_size = batch_size, dtype=tf.float32)\n",
+        "    for i in tf.range(timesteps):\n",
+        "      output, state = self.cell(input_data[i], state)\n",
+        "      outputs = outputs.write(i, output)\n",
+        "    return tf.transpose(outputs.stack(), [1, 0, 2]), state"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NhBI_SGKQVIB"
+      },
+      "outputs": [],
+      "source": [
+        "lstm_cell = tf.keras.layers.LSTMCell(units = 13)\n",
+        "\n",
+        "my_rnn = DynamicRNN(lstm_cell)\n",
+        "outputs, state = my_rnn(tf.random.normal(shape=[10,20,3]))\n",
+        "print(outputs.shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "du7bn3NX7iIr"
+      },
+      "source": [
+        "Read the [`tf.function` guide](https://www.tensorflow.org/guide/function) for a more information."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SUAYhgL_NomT"
+      },
+      "source": [
+        "### New-style metrics and losses\n",
+        "\n",
+        "Metrics and losses are both objects that work eagerly and in `tf.function`s.\n",
+        "\n",
+        "A loss object is callable, and expects (`y_true`, `y_pred`) as arguments:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Pf5gcwMzNs8F"
+      },
+      "outputs": [],
+      "source": [
+        "cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
+        "cce([[1, 0]], [[-1.0,3.0]]).numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a89m-wRfxyfV"
+      },
+      "source": [
+        "#### Use metrics to collect and display data\n",
+        "\n",
+        "You can use `tf.metrics` to aggregate data and `tf.summary` to log summaries and redirect it to a writer using a context manager. The summaries are emitted directly to the writer which means that you must provide the `step` value at the callsite.\n",
+        "\n",
+        "```python\n",
+        "summary_writer = tf.summary.create_file_writer('/tmp/summaries')\n",
+        "with summary_writer.as_default():\n",
+        "  tf.summary.scalar('loss', 0.1, step=42)\n",
+        "```\n",
+        "\n",
+        "Use `tf.metrics` to aggregate data before logging them as summaries. Metrics are stateful; they accumulate values and return a cumulative result when you call the `result` method (such as `Mean.result`). Clear accumulated values with `Model.reset_states`.\n",
+        "\n",
+        "```python\n",
+        "def train(model, optimizer, dataset, log_freq=10):\n",
+        "  avg_loss = tf.keras.metrics.Mean(name='loss', dtype=tf.float32)\n",
+        "  for images, labels in dataset:\n",
+        "    loss = train_step(model, optimizer, images, labels)\n",
+        "    avg_loss.update_state(loss)\n",
+        "    if tf.equal(optimizer.iterations % log_freq, 0):\n",
+        "      tf.summary.scalar('loss', avg_loss.result(), step=optimizer.iterations)\n",
+        "      avg_loss.reset_states()\n",
+        "\n",
+        "def test(model, test_x, test_y, step_num):\n",
+        "  # training=False is only needed if there are layers with different\n",
+        "  # behavior during training versus inference (e.g. Dropout).\n",
+        "  loss = loss_fn(model(test_x, training=False), test_y)\n",
+        "  tf.summary.scalar('loss', loss, step=step_num)\n",
+        "\n",
+        "train_summary_writer = tf.summary.create_file_writer('/tmp/summaries/train')\n",
+        "test_summary_writer = tf.summary.create_file_writer('/tmp/summaries/test')\n",
+        "\n",
+        "with train_summary_writer.as_default():\n",
+        "  train(model, optimizer, dataset)\n",
+        "\n",
+        "with test_summary_writer.as_default():\n",
+        "  test(model, test_x, test_y, optimizer.iterations)\n",
+        "```\n",
+        "\n",
+        "Visualize the generated summaries by pointing TensorBoard to the summary log\n",
+        "directory:\n",
+        "\n",
+        "```shell\n",
+        "tensorboard --logdir /tmp/summaries\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0tx7FyM_RHwJ"
+      },
+      "source": [
+        "Use the `tf.summary` API to write summary data for visualization in TensorBoard. For more info, read the [`tf.summary` guide](https://www.tensorflow.org/tensorboard/migrate#in_tf_2x)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HAbA0fKW58CH"
+      },
+      "outputs": [],
+      "source": [
+        "# Create the metrics\n",
+        "loss_metric = tf.keras.metrics.Mean(name='train_loss')\n",
+        "accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n",
+        "\n",
+        "@tf.function\n",
+        "def train_step(inputs, labels):\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    predictions = model(inputs, training=True)\n",
+        "    regularization_loss=tf.math.add_n(model.losses)\n",
+        "    pred_loss=loss_fn(labels, predictions)\n",
+        "    total_loss=pred_loss + regularization_loss\n",
+        "\n",
+        "  gradients = tape.gradient(total_loss, model.trainable_variables)\n",
+        "  optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
+        "  # Update the metrics\n",
+        "  loss_metric.update_state(total_loss)\n",
+        "  accuracy_metric.update_state(labels, predictions)\n",
+        "\n",
+        "\n",
+        "for epoch in range(NUM_EPOCHS):\n",
+        "  # Reset the metrics\n",
+        "  loss_metric.reset_states()\n",
+        "  accuracy_metric.reset_states()\n",
+        "\n",
+        "  for inputs, labels in train_data:\n",
+        "    train_step(inputs, labels)\n",
+        "  # Get the metric results\n",
+        "  mean_loss=loss_metric.result()\n",
+        "  mean_accuracy = accuracy_metric.result()\n",
+        "\n",
+        "  print('Epoch: ', epoch)\n",
+        "  print('  loss:     {:.3f}'.format(mean_loss))\n",
+        "  print('  accuracy: {:.3f}'.format(mean_accuracy))\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bG9AaMzih3eh"
+      },
+      "source": [
+        "#### Keras metric names\n",
+        "\n",
+        "\n",
+        "Keras models are consistent about handling metric names. When you pass a string in the list of metrics, that _exact_ string is used as the metric's `name`. These names are visible in the history object returned by `model.fit`, and in the logs passed to `keras.callbacks`. is set to the string you passed in the metric list. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1iODIsGDgyYd"
+      },
+      "outputs": [],
+      "source": [
+        "model.compile(\n",
+        "    optimizer = tf.keras.optimizers.Adam(0.001),\n",
+        "    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+        "    metrics = ['acc', 'accuracy', tf.keras.metrics.SparseCategoricalAccuracy(name=\"my_accuracy\")])\n",
+        "history = model.fit(train_data)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8oGzs_TlisKJ"
+      },
+      "outputs": [],
+      "source": [
+        "history.history.keys()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JaB2z2XIyhcr"
+      },
+      "source": [
+        "### Debugging\n",
+        "\n",
+        "Use eager execution to run your code step-by-step to inspect shapes, data types and values. Certain APIs, like `tf.function`, `tf.keras`,\n",
+        "etc. are designed to use Graph execution, for performance and portability. When\n",
+        "debugging, use `tf.config.run_functions_eagerly(True)` to use eager execution\n",
+        "inside this code.\n",
+        "\n",
+        "For example:\n",
+        "\n",
+        "```python\n",
+        "@tf.function\n",
+        "def f(x):\n",
+        "  if x > 0:\n",
+        "    import pdb\n",
+        "    pdb.set_trace()\n",
+        "    x = x + 1\n",
+        "  return x\n",
+        "\n",
+        "tf.config.run_functions_eagerly(True)\n",
+        "f(tf.constant(1))\n",
+        "```\n",
+        "\n",
+        "```\n",
+        ">>> f()\n",
+        "-> x = x + 1\n",
+        "(Pdb) l\n",
+        "  6     @tf.function\n",
+        "  7     def f(x):\n",
+        "  8       if x > 0:\n",
+        "  9         import pdb\n",
+        " 10         pdb.set_trace()\n",
+        " 11  ->     x = x + 1\n",
+        " 12       return x\n",
+        " 13\n",
+        " 14     tf.config.run_functions_eagerly(True)\n",
+        " 15     f(tf.constant(1))\n",
+        "[EOF]\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gdvGF2FvbBXZ"
+      },
+      "source": [
+        "This also works inside Keras models and other APIs that support eager execution:\n",
+        "\n",
+        "```python\n",
+        "class CustomModel(tf.keras.models.Model):\n",
+        "\n",
+        "  @tf.function\n",
+        "  def call(self, input_data):\n",
+        "    if tf.reduce_mean(input_data) > 0:\n",
+        "      return input_data\n",
+        "    else:\n",
+        "      import pdb\n",
+        "      pdb.set_trace()\n",
+        "      return input_data // 2\n",
+        "\n",
+        "\n",
+        "tf.config.run_functions_eagerly(True)\n",
+        "model = CustomModel()\n",
+        "model(tf.constant([-2, -4]))\n",
+        "```\n",
+        "\n",
+        "```\n",
+        ">>> call()\n",
+        "-> return input_data // 2\n",
+        "(Pdb) l\n",
+        " 10         if tf.reduce_mean(input_data) > 0:\n",
+        " 11           return input_data\n",
+        " 12         else:\n",
+        " 13           import pdb\n",
+        " 14           pdb.set_trace()\n",
+        " 15  ->       return input_data // 2\n",
+        " 16\n",
+        " 17\n",
+        " 18     tf.config.run_functions_eagerly(True)\n",
+        " 19     model = CustomModel()\n",
+        " 20     model(tf.constant([-2, -4]))\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "S0-F-bvJXKD8"
+      },
+      "source": [
+        "Notes:\n",
+        "* `tf.keras.Model` methods such as `fit`, `evaluate`, and `predict` execute as [graphs](https://www.tensorflow.org/guide/intro_to_graphs) with `tf.function` under the hood.\n",
+        "\n",
+        "* When using `tf.keras.Model.compile`, set `run_eagerly = True` to disable the `Model` logic from being wrapped in a `tf.function`.\n",
+        "\n",
+        "* Use `tf.data.experimental.enable_debug_mode` to enable the debug mode for `tf.data`. Read the [API docs](https://www.tensorflow.org/api_docs/python/tf/data/experimental/enable_debug_mode) for more details.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wxa5yKK7bym0"
+      },
+      "source": [
+        "### Do not keep `tf.Tensors` in your objects\n",
+        "\n",
+        "These tensor objects might get created either in a `tf.function` or in the eager context, and these tensors behave differently. Always use `tf.Tensor`s only for intermediate values.\n",
+        "\n",
+        "To track state, use `tf.Variable`s as they are always usable from both contexts. Read the [`tf.Variable` guide](https://www.tensorflow.org/guide/variable) to learn more.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FdXLLYa2uAyx"
+      },
+      "source": [
+        "## Resources and further reading\n",
+        "\n",
+        "* Read the TF2 [guides](https://tensorflow.org/guide) and [tutorials](https://tensorflow.org/tutorials) to learn more about how to use TF2.\n",
+        "\n",
+        "* If you previously used TF1.x, it is highly recommended you migrate your code to TF2. Read the [migration guides](https://tensorflow.org/guide/migrate) to learn more."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "effective_tf2.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/effective_tf2.md b/site/en/guide/effective_tf2.md
deleted file mode 100644
index b8d2df9da12..00000000000
--- a/site/en/guide/effective_tf2.md
+++ /dev/null
@@ -1,371 +0,0 @@
-# Effective TensorFlow 2
-
-There are multiple changes in TensorFlow 2.0 to make TensorFlow users more
-productive. TensorFlow 2.0 removes
-[redundant APIs](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md),
-makes APIs more consistent
-([Unified RNNs](https://github.com/tensorflow/community/blob/master/rfcs/20180920-unify-rnn-interface.md),
-[Unified Optimizers](https://github.com/tensorflow/community/blob/master/rfcs/20181016-optimizer-unification.md)),
-and better integrates with the Python runtime with
-[Eager execution](https://www.tensorflow.org/guide/eager).
-
-Many
-[RFCs](https://github.com/tensorflow/community/pulls?utf8=%E2%9C%93&q=is%3Apr)
-have explained the changes that have gone into making TensorFlow 2.0. This
-guide presents a vision for what development in TensorFlow 2.0 should look like.
-It's assumed you have some familiarity with TensorFlow 1.x.
-
-## A brief summary of major changes
-
-### API Cleanup
-
-Many APIs are either
-[gone or moved](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md)
-in TF 2.0. Some of the major changes include removing `tf.app`, `tf.flags`, and
-`tf.logging` in favor of the now open-source
-[absl-py](https://github.com/abseil/abseil-py), rehoming projects that lived in
-`tf.contrib`, and cleaning up the main `tf.*` namespace by moving lesser used
-functions into subpackages like `tf.math`. Some APIs have been replaced with
-their 2.0 equivalents - `tf.summary`, `tf.keras.metrics`, and
-`tf.keras.optimizers`. The easiest way to automatically apply these renames
-is to use the [v2 upgrade script](upgrade.md).
-
-### Eager execution
-
-TensorFlow 1.X requires users to manually stitch together an
-[abstract syntax tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree) (the
-graph) by making `tf.*` API calls. It then requires users to manually compile
-the abstract syntax tree by passing a set of output tensors and input tensors to
-a `session.run()` call. TensorFlow 2.0 executes eagerly (like Python normally
-does) and in 2.0, graphs and sessions should feel like implementation details.
-
-One notable byproduct of eager execution is that `tf.control_dependencies()` is
-no longer required, as all lines of code execute in order (within a
-`tf.function`, code with side effects execute in the order written).
-
-### No more globals
-
-TensorFlow 1.X relied heavily on implicitly global namespaces. When you called
-`tf.Variable()`, it would be put into the default graph, and it would remain
-there, even if you lost track of the Python variable pointing to it. You could
-then recover that `tf.Variable`, but only if you knew the name that it had been
-created with. This was difficult to do if you were not in control of the
-variable's creation. As a result, all sorts of mechanisms proliferated to
-attempt to help users find their variables again, and for frameworks to find
-user-created variables: Variable scopes, global collections, helper methods like
-`tf.get_global_step()`, `tf.global_variables_initializer()`, optimizers
-implicitly computing gradients over all trainable variables, and so on.
-TensorFlow 2.0 eliminates all of these mechanisms
-([Variables 2.0 RFC](https://github.com/tensorflow/community/pull/11)) in favor
-of the default mechanism: Keep track of your variables! If you lose track of a
-`tf.Variable`, it gets garbage collected.
-
-The requirement to track variables creates some extra work for the user, but
-with Keras objects (see below), the burden is minimized.
-
-### Functions, not sessions
-
-A `session.run()` call is almost like a function call: You specify the inputs
-and the function to be called, and you get back a set of outputs. In TensorFlow
-2.0, you can decorate a Python function using `tf.function()` to mark it for JIT
-compilation so that TensorFlow runs it as a single graph
-([Functions 2.0 RFC](https://github.com/tensorflow/community/pull/20)). This
-mechanism allows TensorFlow 2.0 to gain all of the benefits of graph mode:
-
--   Performance: The function can be optimized (node pruning, kernel fusion,
-    etc.)
--   Portability: The function can be exported/reimported
-    ([SavedModel 2.0 RFC](https://github.com/tensorflow/community/pull/34)),
-    allowing users to reuse and share modular TensorFlow functions.
-
-```python
-# TensorFlow 1.X
-outputs = session.run(f(placeholder), feed_dict={placeholder: input})
-# TensorFlow 2.0
-outputs = f(input)
-```
-
-With the power to freely intersperse Python and TensorFlow code, users can take advantage of Python's expressiveness. But portable
-TensorFlow executes in contexts without a Python interpreter, such as mobile, C++, and
-JavaScript. To help users avoid having to rewrite their code when adding `@tf.function`,
-[AutoGraph](function.ipynb) converts a subset of
-Python constructs into their TensorFlow equivalents:
-
-*   `for`/`while` -> `tf.while_loop` (`break` and `continue` are supported)
-*   `if` -> `tf.cond`
-*   `for _ in dataset` -> `dataset.reduce`
-
-AutoGraph supports arbitrary nestings of control flow, which makes it possible
-to performantly and concisely implement many complex ML programs such as
-sequence models, reinforcement learning, custom training loops, and more.
-
-## Recommendations for idiomatic TensorFlow 2.0
-
-### Refactor your code into smaller functions
-
-A common usage pattern in TensorFlow 1.X was the "kitchen sink" strategy, where
-the union of all possible computations was preemptively laid out, and then
-selected tensors were evaluated via `session.run()`. In TensorFlow 2.0, users
-should refactor their code into smaller functions that are called as needed. In
-general, it's not necessary to decorate each of these smaller functions with
-`tf.function`; only use `tf.function` to decorate high-level computations - for
-example, one step of training or the forward pass of your model.
-
-### Use Keras layers and models to manage variables
-
-Keras models and layers offer the convenient `variables` and
-`trainable_variables` properties, which recursively gather up all dependent
-variables. This makes it easy to manage variables locally to where they are
-being used.
-
-Contrast:
-
-```python
-def dense(x, W, b):
-  return tf.nn.sigmoid(tf.matmul(x, W) + b)
-
-@tf.function
-def multilayer_perceptron(x, w0, b0, w1, b1, w2, b2 ...):
-  x = dense(x, w0, b0)
-  x = dense(x, w1, b1)
-  x = dense(x, w2, b2)
-  ...
-
-# You still have to manage w_i and b_i, and their shapes are defined far away from the code.
-```
-
-with the Keras version:
-
-```python
-# Each layer can be called, with a signature equivalent to linear(x)
-layers = [tf.keras.layers.Dense(hidden_size, activation=tf.nn.sigmoid) for _ in range(n)]
-perceptron = tf.keras.Sequential(layers)
-
-# layers[3].trainable_variables => returns [w3, b3]
-# perceptron.trainable_variables => returns [w0, b0, ...]
-```
-
-Keras layers/models inherit from `tf.train.Checkpointable` and are integrated
-with `@tf.function`, which makes it possible to directly checkpoint or export
-SavedModels from Keras objects. You do not necessarily have to use Keras's
-`.fit()` API to take advantage of these integrations.
-
-Here's a transfer learning example that demonstrates how Keras makes it easy to
-collect a subset of relevant variables. Let's say you're training a multi-headed
-model with a shared trunk:
-
-```python
-trunk = tf.keras.Sequential([...])
-head1 = tf.keras.Sequential([...])
-head2 = tf.keras.Sequential([...])
-
-path1 = tf.keras.Sequential([trunk, head1])
-path2 = tf.keras.Sequential([trunk, head2])
-
-# Train on primary dataset
-for x, y in main_dataset:
-  with tf.GradientTape() as tape:
-    prediction = path1(x)
-    loss = loss_fn_head1(prediction, y)
-  # Simultaneously optimize trunk and head1 weights.
-  gradients = tape.gradient(loss, path1.trainable_variables)
-  optimizer.apply_gradients(zip(gradients, path1.trainable_variables))
-
-# Fine-tune second head, reusing the trunk
-for x, y in small_dataset:
-  with tf.GradientTape() as tape:
-    prediction = path2(x)
-    loss = loss_fn_head2(prediction, y)
-  # Only optimize head2 weights, not trunk weights
-  gradients = tape.gradient(loss, head2.trainable_variables)
-  optimizer.apply_gradients(zip(gradients, head2.trainable_variables))
-
-# You can publish just the trunk computation for other people to reuse.
-tf.saved_model.save(trunk, output_path)
-```
-
-### Combine tf.data.Datasets and @tf.function
-
-When iterating over training data that fits in memory, feel free to use regular
-Python iteration. Otherwise, `tf.data.Dataset` is the best way to stream
-training data from disk. Datasets are
-[iterables (not iterators)](https://docs.python.org/3/glossary.html#term-iterable),
-and work just like other Python iterables in Eager mode. You can fully utilize
-dataset async prefetching/streaming features by wrapping your code in
-`tf.function()`, which replaces Python iteration with the equivalent graph
-operations using AutoGraph.
-
-```python
-@tf.function
-def train(model, dataset, optimizer):
-  for x, y in dataset:
-    with tf.GradientTape() as tape:
-      prediction = model(x)
-      loss = loss_fn(prediction, y)
-    gradients = tape.gradient(loss, model.trainable_variables)
-    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
-```
-
-If you use the Keras `.fit()` API, you won't have to worry about dataset
-iteration.
-
-```python
-model.compile(optimizer=optimizer, loss=loss_fn)
-model.fit(dataset)
-```
-
-### Take advantage of AutoGraph with Python control flow
-
-AutoGraph provides a way to convert data-dependent control flow into graph-mode
-equivalents like `tf.cond` and `tf.while_loop`.
-
-One common place where data-dependent control flow appears is in sequence
-models. `tf.keras.layers.RNN` wraps an RNN cell, allowing you to either
-statically or dynamically unroll the recurrence. For demonstration's sake, you
-could reimplement dynamic unroll as follows:
-
-```python
-class DynamicRNN(tf.keras.Model):
-
-  def __init__(self, rnn_cell):
-    super(DynamicRNN, self).__init__(self)
-    self.cell = rnn_cell
-
-  def call(self, input_data):
-    # [batch, time, features] -> [time, batch, features]
-    input_data = tf.transpose(input_data, [1, 0, 2])
-    outputs = tf.TensorArray(tf.float32, input_data.shape[0])
-    state = self.cell.zero_state(input_data.shape[1], dtype=tf.float32)
-    for i in tf.range(input_data.shape[0]):
-      output, state = self.cell(input_data[i], state)
-      outputs = outputs.write(i, output)
-    return tf.transpose(outputs.stack(), [1, 0, 2]), state
-```
-
-For a more detailed overview of AutoGraph's features, see
-[the guide](./function.ipynb).
-
-### tf.metrics aggregates data and tf.summary logs them
-
-To log summaries, use `tf.summary.(scalar|histogram|...)` and redirect it to a
-writer using a context manager. (If you omit the context manager, nothing
-happens.) Unlike TF 1.x, the summaries are emitted directly to the writer; there
-is no separate "merge" op and no separate `add_summary()` call, which means that
-the `step` value must be provided at the callsite.
-
-```python
-summary_writer = tf.summary.create_file_writer('/tmp/summaries')
-with summary_writer.as_default():
-  tf.summary.scalar('loss', 0.1, step=42)
-```
-
-To aggregate data before logging them as summaries, use `tf.metrics`. Metrics
-are stateful: They accumulate values and return a cumulative result when you
-call `.result()`. Clear accumulated values with `.reset_states()`.
-
-```python
-def train(model, optimizer, dataset, log_freq=10):
-  avg_loss = tf.keras.metrics.Mean(name='loss', dtype=tf.float32)
-  for images, labels in dataset:
-    loss = train_step(model, optimizer, images, labels)
-    avg_loss.update_state(loss)
-    if tf.equal(optimizer.iterations % log_freq, 0):
-      tf.summary.scalar('loss', avg_loss.result(), step=optimizer.iterations)
-      avg_loss.reset_states()
-
-def test(model, test_x, test_y, step_num):
-  loss = loss_fn(model(test_x), test_y)
-  tf.summary.scalar('loss', loss, step=step_num)
-
-train_summary_writer = tf.summary.create_file_writer('/tmp/summaries/train')
-test_summary_writer = tf.summary.create_file_writer('/tmp/summaries/test')
-
-with train_summary_writer.as_default():
-  train(model, optimizer, dataset)
-
-with test_summary_writer.as_default():
-  test(model, test_x, test_y, optimizer.iterations)
-```
-
-Visualize the generated summaries by pointing TensorBoard at the summary log
-directory:
-
-
-```
-tensorboard --logdir /tmp/summaries
-```
-
-### Use tf.config.experimental_run_functions_eagerly() when debugging
-
-In TensorFlow 2.0, Eager execution lets you run the code step-by-step to inspect
-shapes, data types and values. Certain APIs, like `tf.function`, `tf.keras`,
-etc. are designed to use Graph execution, for performance and portability.
-When debugging, use `tf.config.experimental_run_functions_eagerly(True)` to
-use Eager execution inside this code.
-
-For example:
-
-```python
-@tf.function
-def f(x):
-  if x > 0:
-    import pdb
-    pdb.set_trace()
-    x = x + 1
-  return x
-
-tf.config.experimental_run_functions_eagerly(True)
-f(tf.constant(1))
-```
-```
->>> f()
--> x = x + 1
-(Pdb) l
-  6  	@tf.function
-  7  	def f(x):
-  8  	  if x > 0:
-  9  	    import pdb
- 10  	    pdb.set_trace()
- 11  ->	    x = x + 1
- 12  	  return x
- 13
- 14  	tf.config.experimental_run_functions_eagerly(True)
- 15  	f(tf.constant(1))
-[EOF]
-```
-
-This also works inside Keras models and other APIs that support Eager execution:
-
-```
-class CustomModel(tf.keras.models.Model):
-
-  @tf.function
-  def call(self, input_data):
-    if tf.reduce_mean(input_data) > 0:
-      return input_data
-    else:
-      import pdb
-      pdb.set_trace()
-      return input_data // 2
-
-
-tf.config.experimental_run_functions_eagerly(True)
-model = CustomModel()
-model(tf.constant([-2, -4]))
-```
-```
->>> call()
--> return input_data // 2
-(Pdb) l
- 10  	    if tf.reduce_mean(input_data) > 0:
- 11  	      return input_data
- 12  	    else:
- 13  	      import pdb
- 14  	      pdb.set_trace()
- 15  ->	      return input_data // 2
- 16
- 17
- 18  	tf.config.experimental_run_functions_eagerly(True)
- 19  	model = CustomModel()
- 20  	model(tf.constant([-2, -4]))
-```
diff --git a/site/en/guide/estimator.ipynb b/site/en/guide/estimator.ipynb
index 223f6b548bd..05e8fb4012a 100644
--- a/site/en/guide/estimator.ipynb
+++ b/site/en/guide/estimator.ipynb
@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Tce3stUlHN0L"
       },
       "source": [
@@ -12,11 +11,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
         "id": "tuOe1ymfHZPu"
       },
       "outputs": [],
@@ -37,7 +34,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qFdPvlXBOdUN"
       },
       "source": [
@@ -47,52 +43,91 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MfBg1C5NB3X0"
       },
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/estimator\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rILQuAiiRlI7"
+      },
+      "source": [
+        "> Warning: TensorFlow 2.15 included the final release of the `tf-estimator` package. Estimators will not be available in TensorFlow 2.16 or after. See the [migration guide](https://www.tensorflow.org/guide/migrate/migrating_estimator) for more information about how to convert off of Estimators."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "oEinLJt2Uowq"
       },
       "source": [
         "This document introduces `tf.estimator`—a high-level TensorFlow\n",
         "API. Estimators encapsulate the following actions:\n",
         "\n",
-        "* training\n",
-        "* evaluation\n",
-        "* prediction\n",
-        "* export for serving\n",
+        "* Training\n",
+        "* Evaluation\n",
+        "* Prediction\n",
+        "* Export for serving\n",
         "\n",
-        "You may either use the pre-made Estimators we provide or write your\n",
-        "own custom Estimators.  All Estimators—whether pre-made or custom—are\n",
-        "classes based on the `tf.estimator.Estimator` class.\n",
+        "TensorFlow implements several pre-made Estimators. Custom estimators are still suported, but mainly as a backwards compatibility measure. **Custom estimators should not be used for new code**.  All Estimators—pre-made or custom ones—are classes based on the `tf.estimator.Estimator` class.\n",
         "\n",
-        "For a quick example try [Estimator tutorials](../../tutorials/estimator/linear.ipynb). For an overview of the API design, see the [white paper](https://arxiv.org/abs/1708.02637)."
+        "For a quick example, try [Estimator tutorials](../tutorials/estimator/linear.ipynb). For an overview of the API design, check the [white paper](https://arxiv.org/abs/1708.02637)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KLdnqg4G2bmz"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cXRQ6mRM5gk0"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -U tensorflow_datasets"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "J_-C9ty22dkD"
+      },
+      "outputs": [],
+      "source": [
+        "import tempfile\n",
+        "import os\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_datasets as tfds"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Wg5zbBliQvNL"
       },
       "source": [
@@ -101,13 +136,12 @@
         "Similar to a `tf.keras.Model`, an `estimator` is a model-level abstraction. The `tf.estimator` provides some capabilities currently still under development for `tf.keras`. These are:\n",
         "\n",
         "  * Parameter server based training\n",
-        "  * Full [TFX](http://tensorflow.org/tfx) integration."
+        "  * Full [TFX](http://tensorflow.org/tfx) integration"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yQ8fQYt_VD5E"
       },
       "source": [
@@ -116,204 +150,263 @@
         "\n",
         "* You can run Estimator-based models on a local host or on a distributed multi-server environment without changing your model. Furthermore, you can run Estimator-based models on CPUs, GPUs, or TPUs without recoding your model.\n",
         "* Estimators provide a safe distributed training loop that controls how and when to:    \n",
-        "    * load data\n",
-        "    * handle exceptions\n",
-        "    * create checkpoint files and recover from failures\n",
-        "    * save summaries for TensorBoard\n",
+        "    * Load data\n",
+        "    * Handle exceptions\n",
+        "    * Create checkpoint files and recover from failures\n",
+        "    * Save summaries for TensorBoard\n",
         "\n",
-        "When writing an application with Estimators, you must separate the data input\n",
-        "pipeline from the model.  This separation simplifies experiments with\n",
-        "different data sets."
+        "When writing an application with Estimators, you must separate the data input pipeline from the model. This separation simplifies experiments with different datasets."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "sXNBeY-oVxGQ"
+        "id": "jQ2PsufpgIpM"
       },
       "source": [
-        "## Pre-made Estimators\n",
+        "## Using pre-made Estimators\n",
         "\n",
         "Pre-made Estimators enable you to work at a much higher conceptual level than the base TensorFlow APIs. You no longer have to worry about creating the computational graph or sessions since Estimators handle all the \"plumbing\" for you. Furthermore, pre-made Estimators let you experiment with different model architectures by making only minimal code changes.  `tf.estimator.DNNClassifier`, for example, is a pre-made Estimator class that trains classification models based on dense, feed-forward neural networks.\n",
         "\n",
-        "### Structure of a pre-made Estimators program\n",
-        "\n",
-        "A TensorFlow program relying on a pre-made Estimator typically consists of the following four steps:\n",
-        "\n",
-        "#### 1. Write one or more dataset importing functions.\n",
-        "\n",
-        "For example, you might create one function to import the training set and another function to import the test set. Each dataset importing function must return two objects:\n",
-        "\n",
-        "* a dictionary in which the keys are feature names and the values are Tensors (or SparseTensors) containing the corresponding feature data\n",
-        "* a Tensor containing one or more labels\n",
-        "\n",
-        "For example, the following code illustrates the basic skeleton for an input function:\n",
-        "\n",
-        "```\n",
-        "def input_fn(dataset):\n",
-        "    ...  # manipulate dataset, extracting the feature dict and the label\n",
-        "    return feature_dict, label\n",
-        "```\n",
-        "\n",
-        "See [data guide](../../guide/data.md) for details.\n",
-        "\n",
-        "#### 2. Define the feature columns.\n",
-        "\n",
-        "Each `tf.feature_column` identifies a feature name, its type, and any input pre-processing. For example, the following snippet creates three feature columns that hold integer or floating-point data. The first two feature columns simply identify the feature's name and type. The third feature column also specifies a lambda the program will invoke to scale the raw data:\n",
-        "\n",
-        "```\n",
-        "# Define three numeric feature columns.\n",
-        "population = tf.feature_column.numeric_column('population')\n",
-        "crime_rate = tf.feature_column.numeric_column('crime_rate')\n",
-        "median_education = tf.feature_column.numeric_column(\n",
-        "  'median_education',\n",
-        "  normalizer_fn=lambda x: x - global_education_mean)\n",
-        "```\n",
-        "For further information, see the [feature columns tutorial](https://www.tensorflow.org/tutorials/keras/feature_columns).\n",
-        "\n",
-        "#### 3. Instantiate the relevant pre-made Estimator.\n",
-        "\n",
-        "For example, here's a sample instantiation of a pre-made Estimator named `LinearClassifier`:\n",
+        "A TensorFlow program relying on a pre-made Estimator typically consists of the following four steps:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mIJPPe26gQpF"
+      },
+      "source": [
+        "### 1. Write an input functions\n",
         "\n",
-        "```\n",
-        "# Instantiate an estimator, passing the feature columns.\n",
-        "estimator = tf.estimator.LinearClassifier(\n",
-        "  feature_columns=[population, crime_rate, median_education])\n",
-        "```\n",
-        "For further information, see the [linear classifier tutorial](https://www.tensorflow.org/tutorials/estimator/linear).\n",
+        "For example, you might create one function to import the training set and another function to import the test set. Estimators expect their inputs to be formatted as a pair of objects:\n",
         "\n",
-        "#### 4. Call a training, evaluation, or inference method.\n",
+        "* A dictionary in which the keys are feature names and the values are Tensors (or SparseTensors) containing the corresponding feature data\n",
+        "* A Tensor containing one or more labels\n",
         "\n",
-        "For example, all Estimators provide a `train` method, which trains a model.\n",
+        "The `input_fn` should return a `tf.data.Dataset` that yields pairs in that format. \n",
         "\n",
-        "```\n",
-        "# `input_fn` is the function created in Step 1\n",
-        "estimator.train(input_fn=my_training_set, steps=2000)\n",
-        "```\n",
-        "You can see an example of this below.\n",
+        "For example, the following code builds a `tf.data.Dataset` from the Titanic dataset's `train.csv` file:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7fl_C5d6hEl3"
+      },
+      "outputs": [],
+      "source": [
+        "def train_input_fn():\n",
+        "  titanic_file = tf.keras.utils.get_file(\"train.csv\", \"https://storage.googleapis.com/tf-datasets/titanic/train.csv\")\n",
+        "  titanic = tf.data.experimental.make_csv_dataset(\n",
+        "      titanic_file, batch_size=32,\n",
+        "      label_name=\"survived\")\n",
+        "  titanic_batches = (\n",
+        "      titanic.cache().repeat().shuffle(500)\n",
+        "      .prefetch(tf.data.AUTOTUNE))\n",
+        "  return titanic_batches"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CjyrQGb3mCcp"
+      },
+      "source": [
+        "The `input_fn` is executed in a `tf.Graph` and can also directly return a `(features_dics, labels)` pair containing graph tensors, but this is error prone outside of simple cases like returning constants."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yJYjWUMxgTnq"
+      },
+      "source": [
+        "### 2. Define the feature columns.\n",
         "\n",
-        "### Benefits of pre-made Estimators\n",
+        "Each `tf.feature_column` identifies a feature name, its type, and any input pre-processing. \n",
         "\n",
-        "Pre-made Estimators encode best practices, providing the following benefits:\n",
+        "For example, the following snippet creates three feature columns.\n",
         "\n",
-        "* Best practices for determining where different parts of the computational graph should run, implementing strategies on a single machine or on a\n",
-        "    cluster.\n",
-        "*   Best practices for event (summary) writing and universally useful\n",
-        "    summaries.\n",
+        "- The first uses the `age` feature directly as a floating-point input. \n",
+        "- The second uses the `class` feature as a categorical input.\n",
+        "- The third uses the `embark_town` as a categorical input, but uses the `hashing trick` to avoid the need to enumerate the options, and to set the number of options.\n",
         "\n",
-        "If you don't use pre-made Estimators, you must implement the preceding features yourself."
+        "For further information, check the [feature columns tutorial](https://www.tensorflow.org/tutorials/keras/feature_columns)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lFd8Dnrmhjhr"
+      },
+      "outputs": [],
+      "source": [
+        "age = tf.feature_column.numeric_column('age')\n",
+        "cls = tf.feature_column.categorical_column_with_vocabulary_list('class', ['First', 'Second', 'Third']) \n",
+        "embark = tf.feature_column.categorical_column_with_hash_bucket('embark_town', 32)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "oIaPjYgnZdn6"
+        "id": "UIjqAozjgXdr"
       },
       "source": [
-        "## Custom Estimators\n",
-        "\n",
-        "The heart of every Estimator—whether pre-made or custom—is its *model function*, which is a method that builds graphs for training, evaluation, and prediction. When you are using a pre-made Estimator, someone else has already implemented the model function. When relying on a custom Estimator, you must write the model function yourself.\n",
+        "### 3. Instantiate the relevant pre-made Estimator.\n",
         "\n",
-        "## Recommended workflow\n",
+        "For example, here's a sample instantiation of a pre-made Estimator named `LinearClassifier`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CDOx6lZVoVB8"
+      },
+      "outputs": [],
+      "source": [
+        "model_dir = tempfile.mkdtemp()\n",
+        "model = tf.estimator.LinearClassifier(\n",
+        "    model_dir=model_dir,\n",
+        "    feature_columns=[embark, cls, age],\n",
+        "    n_classes=2\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QGl9oYuFoYj6"
+      },
+      "source": [
+        "For more information, you can go the [linear classifier tutorial](https://www.tensorflow.org/tutorials/estimator/linear)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sXNBeY-oVxGQ"
+      },
+      "source": [
+        "### 4. Call a training, evaluation, or inference method.\n",
         "\n",
-        "1. Assuming a suitable pre-made Estimator exists, use it to build your first model and use its results to establish a baseline.\n",
-        "2. Build and test your overall pipeline, including the integrity and reliability of your data with this pre-made Estimator.\n",
-        "3. If suitable alternative pre-made Estimators are available, run experiments to determine which pre-made Estimator produces the best results.\n",
-        "4. Possibly, further improve your model by building your own custom Estimator."
+        "All Estimators provide `train`, `evaluate`, and `predict` methods.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "l0QKHuEJ4Kc_"
+        "id": "iGaJKkmVBgo2"
       },
       "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals"
+        "model = model.train(input_fn=train_input_fn, steps=100)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "kRr7DGZxFApM"
+        "id": "CXkivCNq0vfH"
       },
       "outputs": [],
       "source": [
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf"
+        "result = model.evaluate(train_input_fn, steps=10)\n",
+        "\n",
+        "for key, value in result.items():\n",
+        "  print(key, \":\", value)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "IqR2PQG4ZaZ0"
+        "id": "CPLD8n4CLVi_"
       },
       "outputs": [],
       "source": [
-        "import tensorflow_datasets as tfds\n",
-        "tfds.disable_progress_bar()"
+        "for pred in model.predict(train_input_fn):\n",
+        "  for key, value in pred.items():\n",
+        "    print(key, \":\", value)\n",
+        "  break"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cbmrm9pFg5vo"
+      },
+      "source": [
+        "### Benefits of pre-made Estimators\n",
+        "\n",
+        "Pre-made Estimators encode best practices, providing the following benefits:\n",
+        "\n",
+        "* Best practices for determining where different parts of the computational graph should run, implementing strategies on a single machine or on a\n",
+        "    cluster.\n",
+        "*   Best practices for event (summary) writing and universally useful\n",
+        "    summaries.\n",
+        "\n",
+        "If you don't use pre-made Estimators, you must implement the preceding features yourself."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oIaPjYgnZdn6"
+      },
+      "source": [
+        "## Custom Estimators\n",
+        "\n",
+        "The heart of every Estimator—whether pre-made or custom—is its *model function*, `model_fn`, which is a method that builds graphs for training, evaluation, and prediction. When you are using a pre-made Estimator, someone else has already implemented the model function. When relying on a custom Estimator, you must write the model function yourself.\n",
+        "\n",
+        "> Note: A custom `model_fn` will still run in 1.x-style graph mode. This means there is no eager execution and no automatic control dependencies. You should plan to migrate away from `tf.estimator` with custom `model_fn`. The alternative APIs are `tf.keras` and `tf.distribute`. If you still need an `Estimator` for some part of your training you can use the `tf.keras.estimator.model_to_estimator` converter to create an `Estimator` from a `keras.Model`."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "P7aPNnXUbN4j"
       },
       "source": [
         "## Create an Estimator from a Keras model\n",
         "\n",
-        "You can convert existing Keras models to Estimators with `tf.keras.estimator.model_to_estimator`. Doing so enables your Keras\n",
-        "model to access Estimator's strengths, such as distributed training.\n",
+        "You can convert existing Keras models to Estimators with `tf.keras.estimator.model_to_estimator`. This is helpful if you want to modernize your model code, but your training pipeline still requires Estimators. \n",
         "\n",
         "Instantiate a Keras MobileNet V2 model and compile the model with the optimizer, loss, and metrics to train with:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "XE6NMcuGeDOP"
       },
       "outputs": [],
       "source": [
         "keras_mobilenet_v2 = tf.keras.applications.MobileNetV2(\n",
         "    input_shape=(160, 160, 3), include_top=False)\n",
+        "keras_mobilenet_v2.trainable = False\n",
         "\n",
         "estimator_model = tf.keras.Sequential([\n",
         "    keras_mobilenet_v2,\n",
-        "    tf.keras.layers.Flatten(),\n",
-        "    tf.keras.layers.Dense(1, activation='softmax')\n",
+        "    tf.keras.layers.GlobalAveragePooling2D(),\n",
+        "    tf.keras.layers.Dense(1)\n",
         "])\n",
         "\n",
         "# Compile the model\n",
         "estimator_model.compile(\n",
         "    optimizer='adam',\n",
-        "    loss='binary_crossentropy',\n",
+        "    loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
         "    metrics=['accuracy'])"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "A3hcxzcEfYfX"
       },
       "source": [
@@ -322,10 +415,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "UCSSifirfyHk"
       },
       "outputs": [],
@@ -336,7 +427,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8jRNRVb_fzGT"
       },
       "source": [
@@ -345,10 +435,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Rv9xJk51e1fB"
       },
       "outputs": [],
@@ -364,10 +452,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Fw8OjwujVBkc"
       },
       "outputs": [],
@@ -382,7 +468,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "JMb0cuy0gbTi"
       },
       "source": [
@@ -391,21 +476,18 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "4JsvMp8Jge80"
       },
       "outputs": [],
       "source": [
-        "est_mobilenet_v2.train(input_fn=lambda: train_input_fn(32), steps=500)"
+        "est_mobilenet_v2.train(input_fn=lambda: train_input_fn(32), steps=50)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "jvr_rAzngY9v"
       },
       "source": [
@@ -414,10 +496,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "kVNPqysQgYR2"
       },
       "outputs": [],
@@ -428,25 +508,368 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5HeTOvCYbjZb"
       },
       "source": [
         "For more details, please refer to the documentation for `tf.keras.estimator.model_to_estimator`."
       ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zGG1tOM0L6iM"
+      },
+      "source": [
+        "## Saving object-based checkpoints with Estimator\n",
+        "\n",
+        "Estimators by default save checkpoints with variable names rather than the object graph described in the [Checkpoint guide](checkpoint.ipynb). `tf.train.Checkpoint` will read name-based checkpoints, but variable names may change when moving parts of a model outside of the Estimator's `model_fn`. For forwards compatibility saving object-based checkpoints makes it easier to train a model inside an Estimator and then use it outside of one."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-8AMJeueNyoM"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow.compat.v1 as tf_compat"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "W5JbCEUGY-Xo"
+      },
+      "outputs": [],
+      "source": [
+        "def toy_dataset():\n",
+        "  inputs = tf.range(10.)[:, None]\n",
+        "  labels = inputs * 5. + tf.range(5.)[None, :]\n",
+        "  return tf.data.Dataset.from_tensor_slices(\n",
+        "    dict(x=inputs, y=labels)).repeat().batch(2)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gTZbsIRCZnCU"
+      },
+      "outputs": [],
+      "source": [
+        "class Net(tf.keras.Model):\n",
+        "  \"\"\"A simple linear model.\"\"\"\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    super(Net, self).__init__()\n",
+        "    self.l1 = tf.keras.layers.Dense(5)\n",
+        "\n",
+        "  def call(self, x):\n",
+        "    return self.l1(x)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "T6fQsBzJQN2y"
+      },
+      "outputs": [],
+      "source": [
+        "def model_fn(features, labels, mode):\n",
+        "  net = Net()\n",
+        "  opt = tf.keras.optimizers.Adam(0.1)\n",
+        "  ckpt = tf.train.Checkpoint(step=tf_compat.train.get_global_step(),\n",
+        "                             optimizer=opt, net=net)\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    output = net(features['x'])\n",
+        "    loss = tf.reduce_mean(tf.abs(output - features['y']))\n",
+        "  variables = net.trainable_variables\n",
+        "  gradients = tape.gradient(loss, variables)\n",
+        "  return tf.estimator.EstimatorSpec(\n",
+        "    mode,\n",
+        "    loss=loss,\n",
+        "    train_op=tf.group(opt.apply_gradients(zip(gradients, variables)),\n",
+        "                      ckpt.step.assign_add(1)),\n",
+        "    # Tell the Estimator to save \"ckpt\" in an object-based format.\n",
+        "    scaffold=tf_compat.train.Scaffold(saver=ckpt))\n",
+        "\n",
+        "tf.keras.backend.clear_session()\n",
+        "est = tf.estimator.Estimator(model_fn, './tf_estimator_example/')\n",
+        "est.train(toy_dataset, steps=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tObYHnrrb_mL"
+      },
+      "source": [
+        "`tf.train.Checkpoint` can then load the Estimator's checkpoints from its `model_dir`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q6IP3Y_wb-fs"
+      },
+      "outputs": [],
+      "source": [
+        "opt = tf.keras.optimizers.Adam(0.1)\n",
+        "net = Net()\n",
+        "ckpt = tf.train.Checkpoint(\n",
+        "  step=tf.Variable(1, dtype=tf.int64), optimizer=opt, net=net)\n",
+        "ckpt.restore(tf.train.latest_checkpoint('./tf_estimator_example/'))\n",
+        "ckpt.step.numpy()  # From est.train(..., steps=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Dk5wWyuMpuHx"
+      },
+      "source": [
+        "## SavedModels from Estimators\n",
+        "\n",
+        "Estimators export SavedModels through `tf.Estimator.export_saved_model`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B9KQq5qzpzbK"
+      },
+      "outputs": [],
+      "source": [
+        "input_column = tf.feature_column.numeric_column(\"x\")\n",
+        "\n",
+        "estimator = tf.estimator.LinearClassifier(feature_columns=[input_column])\n",
+        "\n",
+        "def input_fn():\n",
+        "  return tf.data.Dataset.from_tensor_slices(\n",
+        "    ({\"x\": [1., 2., 3., 4.]}, [1, 1, 0, 0])).repeat(200).shuffle(64).batch(16)\n",
+        "estimator.train(input_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "y9qCa6J6FVS5"
+      },
+      "source": [
+        "To save an `Estimator` you need to create a `serving_input_receiver`. This function builds a part of a `tf.Graph` that parses the raw data received by the SavedModel. \n",
+        "\n",
+        "The `tf.estimator.export` module contains functions to help build these `receivers`.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XJ4PJ-Cl4060"
+      },
+      "source": [
+        "The following code builds a receiver, based on the `feature_columns`, that accepts serialized `tf.Example` protocol buffers, which are often used with [tf-serving](https://tensorflow.org/serving)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lnmsmGOQFPED"
+      },
+      "outputs": [],
+      "source": [
+        "tmpdir = tempfile.mkdtemp()\n",
+        "\n",
+        "serving_input_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(\n",
+        "  tf.feature_column.make_parse_example_spec([input_column]))\n",
+        "\n",
+        "estimator_base_path = os.path.join(tmpdir, 'from_estimator')\n",
+        "estimator_path = estimator.export_saved_model(estimator_base_path, serving_input_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q7XtbLMDaie2"
+      },
+      "source": [
+        "You can also load and run that model, from python:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c_BUBBNB1UH9"
+      },
+      "outputs": [],
+      "source": [
+        "imported = tf.saved_model.load(estimator_path)\n",
+        "\n",
+        "def predict(x):\n",
+        "  example = tf.train.Example()\n",
+        "  example.features.feature[\"x\"].float_list.value.extend([x])\n",
+        "  return imported.signatures[\"predict\"](\n",
+        "    examples=tf.constant([example.SerializeToString()]))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "C1ylWZCQ1ahG"
+      },
+      "outputs": [],
+      "source": [
+        "print(predict(1.5))\n",
+        "print(predict(3.5))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_IrCCm0-isqA"
+      },
+      "source": [
+        "`tf.estimator.export.build_raw_serving_input_receiver_fn` allows you to create input functions which take raw tensors rather than `tf.train.Example`s."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nO0hmFCRoIll"
+      },
+      "source": [
+        "## Using `tf.distribute.Strategy` with Estimator (Limited support)\n",
+        "\n",
+        "`tf.estimator` is a distributed training TensorFlow API that originally supported the async parameter server approach. `tf.estimator` now supports `tf.distribute.Strategy`. If you're using `tf.estimator`, you can change to distributed training with very few changes to your code. With this, Estimator users can now do synchronous distributed training on multiple GPUs and multiple workers, as well as use TPUs. This support in Estimator is, however, limited. Check out the [What's supported now](#estimator_support) section below for more details.\n",
+        "\n",
+        "Using `tf.distribute.Strategy` with Estimator is slightly different than in the Keras case. Instead of using `strategy.scope`, now you pass the strategy object into the `RunConfig` for the Estimator.\n",
+        "\n",
+        "You can refer to the [distributed training guide](distributed_training.ipynb) for more information.\n",
+        "\n",
+        "Here is a snippet of code that shows this with a premade Estimator `LinearRegressor` and `MirroredStrategy`:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oGFY5nW_B3YU"
+      },
+      "outputs": [],
+      "source": [
+        "mirrored_strategy = tf.distribute.MirroredStrategy()\n",
+        "config = tf.estimator.RunConfig(\n",
+        "    train_distribute=mirrored_strategy, eval_distribute=mirrored_strategy)\n",
+        "regressor = tf.estimator.LinearRegressor(\n",
+        "    feature_columns=[tf.feature_column.numeric_column('feats')],\n",
+        "    optimizer='SGD',\n",
+        "    config=config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n6eSfLN5RGY8"
+      },
+      "source": [
+        "Here, you use a premade Estimator, but the same code works with a custom Estimator as well. `train_distribute` determines how training will be distributed, and `eval_distribute` determines how evaluation will be distributed. This is another difference from Keras where you use the same strategy for both training and eval.\n",
+        "\n",
+        "Now you can train and evaluate this Estimator with an input function:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ky2ve2PB3YP"
+      },
+      "outputs": [],
+      "source": [
+        "def input_fn():\n",
+        "  dataset = tf.data.Dataset.from_tensors(({\"feats\":[1.]}, [1.]))\n",
+        "  return dataset.repeat(1000).batch(10)\n",
+        "regressor.train(input_fn=input_fn, steps=10)\n",
+        "regressor.evaluate(input_fn=input_fn, steps=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hgaU9xQSSk2x"
+      },
+      "source": [
+        "Another difference to highlight here between Estimator and Keras is the input handling. In Keras, each batch of the dataset is split automatically across the multiple replicas. In Estimator, however, you do not perform automatic batch splitting, nor automatically shard the data across different workers. You have full control over how you want your data to be distributed across workers and devices, and you must provide an `input_fn` to specify how to distribute your data.\n",
+        "\n",
+        "Your `input_fn` is called once per worker, thus giving one dataset per worker. Then one batch from that dataset is fed to one replica on that worker, thereby consuming N batches for N replicas on 1 worker. In other words, the dataset returned by the `input_fn` should provide batches of size `PER_REPLICA_BATCH_SIZE`. And the global batch size for a step can be obtained as `PER_REPLICA_BATCH_SIZE * strategy.num_replicas_in_sync`.\n",
+        "\n",
+        "When performing multi-worker training, you should either split your data across the workers, or shuffle with a random seed on each. You can check an example of how to do this in the [Multi-worker training with Estimator](../tutorials/distribute/multi_worker_with_estimator.ipynb) tutorial."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G3ieQKfWZhhL"
+      },
+      "source": [
+        "And similarly, you can use multi worker and parameter server strategies as well. The code remains the same, but you need to use `tf.estimator.train_and_evaluate`, and set `TF_CONFIG` environment variables for each binary running in your cluster."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A_lvUsSLZzVg"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "### What's supported now?\n",
+        "\n",
+        "There is limited support for training with Estimator using all strategies except `TPUStrategy`. Basic training and evaluation should work, but a number of advanced features such as `v1.train.Scaffold` do not. There may also be a number of bugs in this integration and there are no plans to actively improve this support (the focus is on Keras and custom training loop support). If at all possible, you should prefer to use `tf.distribute` with those APIs instead.\n",
+        "\n",
+        "| Training API  \t| MirroredStrategy \t| TPUStrategy \t| MultiWorkerMirroredStrategy \t| CentralStorageStrategy \t| ParameterServerStrategy \t|\n",
+        "|:---------------\t|:------------------\t|:-------------\t|:-----------------------------\t|:------------------------\t|:-------------------------\t|\n",
+        "| Estimator API \t| Limited support        \t| Not supported   \t| Limited support                   \t| Limited support              \t| Limited support               \t|\n",
+        "\n",
+        "### Examples and tutorials\n",
+        "\n",
+        "Here are some end-to-end examples that show how to use various strategies with Estimator:\n",
+        "\n",
+        "1. The [Multi-worker Training with Estimator tutorial](../tutorials/distribute/multi_worker_with_estimator.ipynb) shows how you can train with multiple workers using `MultiWorkerMirroredStrategy` on the MNIST dataset.\n",
+        "2. An end-to-end example of [running multi-worker training with distribution strategies](https://github.com/tensorflow/ecosystem/tree/master/distribution_strategy) in `tensorflow/ecosystem` using Kubernetes templates. It starts with a Keras model and converts it to an Estimator using the `tf.keras.estimator.model_to_estimator` API.\n",
+        "3. The official [ResNet50](https://github.com/tensorflow/models/blob/master/official/vision/image_classification/resnet_imagenet_main.py) model, which can be trained using either `MirroredStrategy` or `MultiWorkerMirroredStrategy`."
+      ]
     }
   ],
   "metadata": {
     "accelerator": "GPU",
     "colab": {
       "collapsed_sections": [
-        "Tce3stUlHN0L"
+        "Tce3stUlHN0L",
+        "KLdnqg4G2bmz",
+        "Wg5zbBliQvNL",
+        "yQ8fQYt_VD5E",
+        "jQ2PsufpgIpM",
+        "mIJPPe26gQpF",
+        "yJYjWUMxgTnq",
+        "UIjqAozjgXdr",
+        "sXNBeY-oVxGQ",
+        "cbmrm9pFg5vo",
+        "oIaPjYgnZdn6",
+        "P7aPNnXUbN4j",
+        "zGG1tOM0L6iM",
+        "Dk5wWyuMpuHx",
+        "nO0hmFCRoIll",
+        "A_lvUsSLZzVg"
       ],
       "name": "estimator.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true,
-      "version": "0.3.2"
+      "toc_visible": true
     },
     "kernelspec": {
       "display_name": "Python 3",
diff --git a/site/en/guide/extension_type.ipynb b/site/en/guide/extension_type.ipynb
new file mode 100644
index 00000000000..7e8edeea7c9
--- /dev/null
+++ b/site/en/guide/extension_type.ipynb
@@ -0,0 +1,2130 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WrcIOXsUQh8U"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tXAbWHtqs1Y2"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HTgMAvQq-PU_"
+      },
+      "source": [
+        "# Extension types\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jHcw9MtgBo7e"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0MsE_F0WBpmc"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q tf_nightly\n",
+        "import tensorflow as tf\n",
+        "import numpy as np\n",
+        "from typing import Tuple, List, Mapping, Union, Optional\n",
+        "import tempfile"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1BAk3bji_0wl"
+      },
+      "source": [
+        "## Extension types\n",
+        "\n",
+        "User-defined types can make projects more readable, modular, maintainable. However, most TensorFlow APIs have very limited support for user-defined Python types. This includes both high-level APIs (such as [Keras](https://www.tensorflow.org/guide/keras/overview), [tf.function](https://www.tensorflow.org/guide/function), [`tf.SavedModel`](https://www.tensorflow.org/guide/saved_model)) and lower-level APIs (such as `tf.while_loop` and `tf.concat`). TensorFlow **extension types** can be used to create user-defined object-oriented types that work seamlessly with TensorFlow's APIs. To create an extension type, simply define a Python class with `tf.experimental.ExtensionType` as its base, and use [type annotations](https://www.python.org/dev/peps/pep-0484/) to specify the type for each field."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7o5KY7L5_nxy"
+      },
+      "outputs": [],
+      "source": [
+        "class TensorGraph(tf.experimental.ExtensionType):\n",
+        "  \"\"\"A collection of labeled nodes connected by weighted edges.\"\"\"\n",
+        "  edge_weights: tf.Tensor               # shape=[num_nodes, num_nodes]\n",
+        "  node_labels: Mapping[str, tf.Tensor]  # shape=[num_nodes]; dtype=any\n",
+        "\n",
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  \"\"\"A tensor paired with a boolean mask, indicating which values are valid.\"\"\"\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor       # shape=values.shape; false for missing/invalid values.\n",
+        "\n",
+        "class CSRSparseMatrix(tf.experimental.ExtensionType):\n",
+        "  \"\"\"Compressed sparse row matrix (https://en.wikipedia.org/wiki/Sparse_matrix).\"\"\"\n",
+        "  values: tf.Tensor     # shape=[num_nonzero]; dtype=any\n",
+        "  col_index: tf.Tensor  # shape=[num_nonzero]; dtype=int64\n",
+        "  row_index: tf.Tensor  # shape=[num_rows+1]; dtype=int64"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FiaNXPa7pNK-"
+      },
+      "source": [
+        "The `tf.experimental.ExtensionType` base class works similarly to [`typing.NamedTuple`](https://docs.python.org/3/library/typing.html#typing.NamedTuple) and [`@dataclasses.dataclass`](https://docs.python.org/3/library/dataclasses.html#dataclasses.dataclass) from the standard Python library. In particular, it automatically adds a constructor and special methods (such as `__repr__` and `__eq__`) based on the field type annotations."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JsE7X6_uMyLo"
+      },
+      "source": [
+        "Typically, extension types tend to fall into one of two categories:\n",
+        "\n",
+        "* ***Data structures***, which group together a collection of related values, and can provide useful operations based on those values. Data structures may be fairly general (such as the `TensorGraph` example above); or they may be highly customized to a specific model.\n",
+        "\n",
+        "* ***Tensor-like types***, which specialize or extend the concept of \"Tensor.\" Types in this category have a `rank`, a `shape`, and usually a `dtype`; and it makes sense to use them with Tensor operations (such as `tf.stack`, `tf.add`, or `tf.matmul`). `MaskedTensor` and `CSRSparseMatrix` are examples of tensor-like types."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uxngcajlMqIY"
+      },
+      "source": [
+        "## Supported APIs\n",
+        "\n",
+        "Extension types are supported by the following TensorFlow APIs:\n",
+        "\n",
+        "* **Keras**: Extension types can be used as inputs and outputs for Keras `Models` and `Layers`.\n",
+        "* **`tf.data.Dataset`**: Extension types can be included in `Datasets`, and returned by dataset `Iterators`.\n",
+        "* **TensorFlow Hub**: Extension types can be used as inputs and outputs for `tf.hub` modules.\n",
+        "* **SavedModel**: Extension types can be used as inputs and outputs for `SavedModel` functions.\n",
+        "* **`tf.function`**: Extension types can be used as arguments and return values for functions wrapped with the `@tf.function` decorator.\n",
+        "* **While loops**: Extension types can be used as loop variables in `tf.while_loop`, and can be used as arguments and return values for the while-loop's body.\n",
+        "* **Conditionals**: Extension types can be conditionally selected using `tf.cond` and `tf.case`.\n",
+        "* **`tf.py_function`**: Extension types can be used as arguments and return values for the `func` argument to `tf.py_function`.\n",
+        "* **Tensor ops**: Extension types can be extended to support most TensorFlow ops that accept Tensor inputs (such as `tf.matmul`, `tf.gather`, and `tf.reduce_sum`). Go to the \"*Dispatch*\" section below for more information.\n",
+        "* **Distribution strategy**: Extension types can be used as per-replica values.\n",
+        "\n",
+        "For more details, see the section on \"TensorFlow APIs that support ExtensionTypes\" below.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VIpZwuPVpwOX"
+      },
+      "source": [
+        "## Requirements\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nNk_TQeJGVwV"
+      },
+      "source": [
+        "### Field types\n",
+        "\n",
+        "All fields—instance variables—must be declared, and a type annotation must be provided for each field. The following type annotations are supported:\n",
+        "\n",
+        "Type | Example\n",
+        "---- | -------\n",
+        "Python integers | `i: int`\n",
+        "Python floats | `f: float`\n",
+        "Python strings | `s: str`\n",
+        "Python booleans | `b: bool`\n",
+        "Python `None` | `n: None`\n",
+        "[Tensor shapes](https://www.tensorflow.org/api_docs/python/tf/TensorShape) | `shape: tf.TensorShape`\n",
+        "[Tensor `dtype`s](https://www.tensorflow.org/api_docs/python/tf/dtypes/DType) | `dtype: tf.DType`\n",
+        "[Tensors](https://www.tensorflow.org/api_docs/python/tf/Tensor) | `t: tf.Tensor`\n",
+        "[Extension types](https://www.tensorflow.org/api_docs/python/tf/experimental/ExtensionType) | `mt: MyMaskedTensor`\n",
+        "[Ragged tensors](https://www.tensorflow.org/api_docs/python/tf/RaggedTensor) | `rt: tf.RaggedTensor`\n",
+        "[Sparse tensors](https://www.tensorflow.org/api_docs/python/tf/sparse/SparseTensor) | `st: tf.SparseTensor`\n",
+        "[Indexed slices](https://www.tensorflow.org/api_docs/python/tf/IndexedSlices) | `s: tf.IndexedSlices`\n",
+        "[Optional tensors](https://www.tensorflow.org/api_docs/python/tf/experimental/Optional) | `o: tf.experimental.Optional`\n",
+        "[Type unions](https://docs.python.org/3/library/typing.html#typing.Union) | `int_or_float: typing.Union[int, float]`\n",
+        "[Tuples](https://docs.python.org/3/library/typing.html#typing.Tuple) | `params: typing.Tuple[int, float, tf.Tensor, int]`\n",
+        "[Var-length tuples](https://docs.python.org/3/library/typing.html#typing.Tuple) | `lengths: typing.Tuple[int, ...]`\n",
+        "[Mappings](https://docs.python.org/3/library/typing.html#typing.Mapping) | `tags: typing.Mapping[str, tf.Tensor]`\n",
+        "[Optional values](https://docs.python.org/3/library/typing.html#typing.Optional) | `weight: typing.Optional[tf.Tensor]`"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iFetYyZsIvf6"
+      },
+      "source": [
+        "### Mutability\n",
+        "\n",
+        "Extension types are required to be immutable. This ensures that they can be properly tracked by TensorFlow's graph-tracing mechanisms.\n",
+        "If you find yourself wanting to mutate an extension type value, consider instead defining methods that transform values. For example, rather than defining a `set_mask` method to mutate a `MaskedTensor`, you could define a `replace_mask` method that returns a new `MaskedTensor`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DThZLYH2IwFh"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "\n",
+        "  def replace_mask(self, new_mask):\n",
+        "      self.values.shape.assert_is_compatible_with(new_mask.shape)\n",
+        "      return MaskedTensor(self.values, new_mask)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x3JyivI_qAtt"
+      },
+      "source": [
+        "## Functionality added by `ExtensionType`\n",
+        "\n",
+        "The `ExtensionType` base class provides the following functionality:\n",
+        "\n",
+        "* A constructor (`__init__`).\n",
+        "* A printable representation method (`__repr__`).\n",
+        "* Equality and inequality operators (`__eq__`).\n",
+        "* A validation method (`__validate__`).\n",
+        "* Enforced immutability.\n",
+        "* A nested `TypeSpec`.\n",
+        "* Tensor API dispatch support.\n",
+        "\n",
+        "Go to the \"Customizing `ExtensionType`s\" section below for more information on customizing this functionality."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pfSYs6P26gKq"
+      },
+      "source": [
+        "### Constructor\n",
+        "The constructor added by `ExtensionType` takes each field as a named argument (in the order they were listed in the class definition). This constructor will type-check each parameter, and convert them where necessary. In particular, `Tensor` fields are converted using `tf.convert_to_tensor`; `Tuple` fields are converted to `tuple`s; and `Mapping` fields are converted to immutable dicts."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DiXwyZ5M5KFW"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "\n",
+        "# Constructor takes one parameter for each field.\n",
+        "mt = MaskedTensor(values=[[1, 2, 3], [4, 5, 6]],\n",
+        "                  mask=[[True, True, False], [True, False, True]])\n",
+        "\n",
+        "# Fields are type-checked and converted to the declared types.\n",
+        "# For example, `mt.values` is converted to a Tensor.\n",
+        "print(mt.values)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ezNDe1cYF0Qb"
+      },
+      "source": [
+        "The constructor raises an `TypeError` if a field value can not be converted to its declared type:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6HnrMaabF5VS"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  MaskedTensor([1, 2, 3], None)\n",
+        "except TypeError as e:\n",
+        "  print(f\"Got expected TypeError: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FwQUI3X02s20"
+      },
+      "source": [
+        "The default value for a field can be specified by setting its value at the class level:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GbzDT9fz20JA"
+      },
+      "outputs": [],
+      "source": [
+        "class Pencil(tf.experimental.ExtensionType):\n",
+        "  color: str = \"black\"\n",
+        "  has_erasor: bool = True\n",
+        "  length: tf.Tensor = 1.0\n",
+        "\n",
+        "Pencil()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nOW7lS9P4Foc"
+      },
+      "outputs": [],
+      "source": [
+        "Pencil(length=0.5, color=\"blue\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "S5Eivtg07Aau"
+      },
+      "source": [
+        "### Printable representation\n",
+        "\n",
+        "`ExtensionType` adds a default printable representation method (`__repr__`) that includes the class name and the value for each field:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5SyiKTe55krG"
+      },
+      "outputs": [],
+      "source": [
+        "print(MaskedTensor(values=[1, 2, 3], mask=[True, True, False]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "q4l_gnQh6nXR"
+      },
+      "source": [
+        "### Equality operators\n",
+        "\n",
+        "`ExtensionType` adds default equality operators (`__eq__` and `__ne__`) that consider two values equal if they have the same type and all their fields are equal. Tensor fields are considered equal if they have the same shape and are elementwise equal for all elements."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bHdLg13V52Xm"
+      },
+      "outputs": [],
+      "source": [
+        "a = MaskedTensor([1, 2], [True, False])\n",
+        "b = MaskedTensor([[3, 4], [5, 6]], [[False, True], [True, True]])\n",
+        "print(f\"a == a: {a==a}\")\n",
+        "print(f\"a == b: {a==b}\")\n",
+        "print(f\"a == a.values: {a==a.values}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O3HqsO3jZlQq"
+      },
+      "source": [
+        "**Note:** if any field contains a `Tensor`, then `__eq__` may return a scalar boolean `Tensor` (rather than a Python boolean value)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hCpBfkKqCuip"
+      },
+      "source": [
+        "### Validation method\n",
+        "\n",
+        "`ExtensionType` adds a `__validate__` method, which can be overridden to perform validation checks on fields. It is run after the constructor is called, and after fields have been type-checked and converted to their declared types, so it can assume that all fields have their declared types.\n",
+        "\n",
+        "The following example updates `MaskedTensor` to validate the `shape`s and `dtype`s of its fields:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dgZOJRINDn00"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  \"\"\"A tensor paired with a boolean mask, indicating which values are valid.\"\"\"\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "  def __validate__(self):\n",
+        "    self.values.shape.assert_is_compatible_with(self.mask.shape)\n",
+        "    assert self.mask.dtype.is_bool, 'mask.dtype must be bool'"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ajSgkGUUn9WL"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  MaskedTensor([1, 2, 3], [0, 1, 0])  # Wrong `dtype` for mask.\n",
+        "except AssertionError as e:\n",
+        "  print(f\"Got expected AssertionError: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Fhb96luJn9K7"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  MaskedTensor([1, 2, 3], [True, False])  # shapes don't match.\n",
+        "except ValueError as e:\n",
+        "  print(f\"Got expected ValueError: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pjIPAF1OCAdO"
+      },
+      "source": [
+        "### Enforced immutability\n",
+        "\n",
+        "`ExtensionType` overrides the `__setattr__` and `__delattr__` methods to prevent mutation, ensuring that extension type values are immutable."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NgmJ1C7ilN5C"
+      },
+      "outputs": [],
+      "source": [
+        "mt = MaskedTensor([1, 2, 3], [True, False, True])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cMYmJr3RoFKp"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  mt.mask = [True, True, True]\n",
+        "except AttributeError as e:\n",
+        "  print(f\"Got expected AttributeError: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZWwA-zWdzqlU"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  mt.mask[0] = False\n",
+        "except TypeError as e:\n",
+        "  print(f\"Got expected TypeError: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PN_txJVKoFoF"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  del mt.mask\n",
+        "except AttributeError as e:\n",
+        "  print(f\"Got expected AttributeError: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FBVFtCYn69Ou"
+      },
+      "source": [
+        "### Nested TypeSpec\n",
+        "\n",
+        "Each `ExtensionType` class has a corresponding `TypeSpec` class, which is created automatically and stored as `.Spec`.\n",
+        "\n",
+        "This class captures all the information from a value *except* for the values of any nested tensors. In particular, the `TypeSpec` for a value is created by replacing any nested Tensor, ExtensionType, or CompositeTensor with its `TypeSpec`.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GRjANkGYKGnV"
+      },
+      "outputs": [],
+      "source": [
+        "class Player(tf.experimental.ExtensionType):\n",
+        "  name: tf.Tensor\n",
+        "  attributes: Mapping[str, tf.Tensor]\n",
+        "\n",
+        "anne = Player(\"Anne\", {\"height\": 8.3, \"speed\": 28.1})\n",
+        "anne_spec = tf.type_spec_from_value(anne)\n",
+        "print(anne_spec.name)  # Records `dtype` and `shape`, but not the string value.\n",
+        "print(anne_spec.attributes)  # Records keys and TensorSpecs for values."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "I2fkgckxO564"
+      },
+      "source": [
+        "`TypeSpec` values can be constructed explicitly, or they can be built from an `ExtensionType` value using `tf.type_spec_from_value`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1ehAa7d9OGai"
+      },
+      "outputs": [],
+      "source": [
+        "spec1 = Player.Spec(name=tf.TensorSpec([], tf.float32), attributes={})\n",
+        "spec2 = tf.type_spec_from_value(anne)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "owcFG3cAMCwA"
+      },
+      "source": [
+        "`TypeSpec`s are used by TensorFlow to divide values into a **static component** and a **dynamic component**:\n",
+        "\n",
+        "* The **static component** (which is fixed at graph-construction time) is encoded with a `tf.TypeSpec`.\n",
+        "* The **dynamic component** (which can vary each time the graph is run) is encoded as a list of `tf.Tensor`s.\n",
+        "\n",
+        "For example, `tf.function` retraces its wrapped function whenever an argument has a previously unseen `TypeSpec`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pg-m5YLRM1Nd"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def anonymize_player(player):\n",
+        "  print(\"<>\")\n",
+        "  return Player(\"\", player.attributes)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0CCGm7cpeIq-"
+      },
+      "outputs": [],
+      "source": [
+        "# Function gets traced (first time the function has been called):\n",
+        "anonymize_player(Player(\"Anne\", {\"height\": 8.3, \"speed\": 28.1}))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WB7bt7s83mFE"
+      },
+      "outputs": [],
+      "source": [
+        "# Function does NOT get traced (same TypeSpec: just tensor values changed)\n",
+        "anonymize_player(Player(\"Bart\", {\"height\": 8.1, \"speed\": 25.3}))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dNm7vLpR3nMH"
+      },
+      "outputs": [],
+      "source": [
+        "# Function gets traced (new TypeSpec: keys for attributes changed):\n",
+        "anonymize_player(Player(\"Chuck\", {\"height\": 11.0, \"jump\": 5.3}))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "U5rN1HPq25xC"
+      },
+      "source": [
+        "For more information, see the [tf.function Guide](https://www.tensorflow.org/guide/function#rules_of_tracing)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gX613uRk0qLz"
+      },
+      "source": [
+        "## Customizing `ExtensionType`s\n",
+        "\n",
+        "In addition to simply declaring fields and their types, extension types may:\n",
+        "\n",
+        "* Override the default printable representation (`__repr__`).\n",
+        "* Define methods.\n",
+        "* Define `classmethod`s and `staticmethod`s.\n",
+        "* Define properties.\n",
+        "* Override the default constructor (`__init__`).\n",
+        "* Override the default equality operator (`__eq__`).\n",
+        "* Define operators (such as `__add__` and `__lt__`).\n",
+        "* Declare default values for fields.\n",
+        "* Define subclasses.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MK-ePVDj-ROE"
+      },
+      "source": [
+        "### Overriding the default printable representation\n",
+        "\n",
+        "You can override this default string conversion operator for extension types. The following example updates the `MaskedTensor` class to generate a more readable string representation when values are printed in Eager mode."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gdPhjYEr8IGO"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  \"\"\"A tensor paired with a boolean mask, indicating which values are valid.\"\"\"\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor       # shape=values.shape; false for invalid values.\n",
+        "\n",
+        "  def __repr__(self):\n",
+        "    return masked_tensor_str(self.values, self.mask)\n",
+        "\n",
+        "def masked_tensor_str(values, mask):\n",
+        "  if isinstance(values, tf.Tensor):\n",
+        "    if hasattr(values, 'numpy') and hasattr(mask, 'numpy'):\n",
+        "      return f''\n",
+        "    else:\n",
+        "      return f'MaskedTensor(values={values}, mask={mask})'\n",
+        "  if len(values.shape) == 1:\n",
+        "    items = [repr(v) if m else '_' for (v, m) in zip(values, mask)]\n",
+        "  else:\n",
+        "    items = [masked_tensor_str(v, m) for (v, m) in zip(values, mask)]\n",
+        "  return '[%s]' % ', '.join(items)\n",
+        "\n",
+        "mt = MaskedTensor(values=[[1, 2, 3], [4, 5, 6]],\n",
+        "                  mask=[[True, True, False], [True, False, True]])\n",
+        "print(mt)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_MLQU2_v8VjG"
+      },
+      "source": [
+        "### Defining methods\n",
+        "\n",
+        "Extension types may define methods, just like any normal Python class. For example, the `MaskedTensor` type could define a `with_default` method that returns a copy of `self` with masked values replaced by a given `default` value. Methods may optionally be annotated with the `@tf.function` decorator."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7RR-tqee8ZdP"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "\n",
+        "  def with_default(self, default):\n",
+        "    return tf.where(self.mask, self.values, default)\n",
+        "\n",
+        "MaskedTensor([1, 2, 3], [True, False, True]).with_default(0)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Qwd_gGKp9RP0"
+      },
+      "source": [
+        "### Defining `classmethod`s and `staticmethod`s\n",
+        "\n",
+        "Extension types may define methods using the `@classmethod` and `@staticmethod` decorators. For example, the `MaskedTensor` type could define a factory method that masks any element with a given value:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BacCEJYU9sBR"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "\n",
+        "  def __repr__(self):\n",
+        "    return masked_tensor_str(self.values, self.mask)\n",
+        "\n",
+        "  @staticmethod\n",
+        "  def from_tensor_and_value_to_mask(values, value_to_mask):\n",
+        "    return MaskedTensor(values, values != value_to_mask)\n",
+        "\n",
+        "x = tf.constant([[1, 0, 2], [3, 0, 0]])\n",
+        "MaskedTensor.from_tensor_and_value_to_mask(x, 0)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xIPf9PZX9AwL"
+      },
+      "source": [
+        "### Defining properties\n",
+        "Extension types may define properties using the `@property` decorator, just like any normal Python class. For example, the `MaskedTensor` type could define a `dtype` property that's a shorthand for the `dtype` of the values:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "16E68wZ-9KXp"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "\n",
+        "  @property\n",
+        "  def dtype(self):\n",
+        "    return self.values.dtype\n",
+        "\n",
+        "MaskedTensor([1, 2, 3], [True, False, True]).dtype"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Mm5gxoG57nf3"
+      },
+      "source": [
+        "### Overriding the default constructor\n",
+        "\n",
+        "You can override the default constructor for extension types. Custom constructors must set a value for every declared field; and after the custom constructor returns, all fields will be type-checked, and values will be converted as described above."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-8K3KeB08G1S"
+      },
+      "outputs": [],
+      "source": [
+        "class Toy(tf.experimental.ExtensionType):\n",
+        "  name: str\n",
+        "  price: tf.Tensor\n",
+        "  def __init__(self, name, price, discount=0):\n",
+        "    self.name = name\n",
+        "    self.price = price * (1 - discount)\n",
+        "\n",
+        "print(Toy(\"ball\", 5.0, discount=0.2))  # On sale -- 20% off!"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qyQxMlwLFQt7"
+      },
+      "source": [
+        "Alternatively, you might consider leaving the default constructor as-is, but adding one or more factory methods. For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jiApK4hzFY89"
+      },
+      "outputs": [],
+      "source": [
+        "class Toy(tf.experimental.ExtensionType):\n",
+        "  name: str\n",
+        "  price: tf.Tensor\n",
+        "\n",
+        "  @staticmethod\n",
+        "  def new_toy_with_discount(name, price, discount):\n",
+        "    return Toy(name, price * (1 - discount))\n",
+        "\n",
+        "print(Toy.new_toy_with_discount(\"ball\", 5.0, discount=0.2))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pdVcRBhG-Uee"
+      },
+      "source": [
+        "### Overriding the default equality operator (`__eq__`)\n",
+        "\n",
+        "You can override the default `__eq__` operator for extension types. The following example updates `MaskedTensor` to ignore masked elements when comparing for equality."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dA7DyjfB-Yz0"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "\n",
+        "  def __repr__(self):\n",
+        "    return masked_tensor_str(self.values, self.mask)\n",
+        "\n",
+        "  def __eq__(self, other):\n",
+        "    result = tf.math.equal(self.values, other.values)\n",
+        "    result = result | ~(self.mask & other.mask)\n",
+        "    return tf.reduce_all(result)\n",
+        "\n",
+        "x = MaskedTensor([1, 2, 3, 4], [True, True, False, True])\n",
+        "y = MaskedTensor([5, 2, 0, 4], [False, True, False, True])\n",
+        "print(x == y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n1mZ1Lkyi14B"
+      },
+      "source": [
+        "**Note:** You generally don't need to override `__ne__`, since its default implementation simply calls `__eq__` and negates the result."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A_Jib1SQD1-z"
+      },
+      "source": [
+        "### Using forward references\n",
+        "\n",
+        "If the type for a field has not been defined yet, you may use a string containing the name of the type instead. In the following example, the string `\"Node\"` is used to annotate the `children` field because the `Node` type hasn't been (fully) defined yet.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_Z029QKED0Ao"
+      },
+      "outputs": [],
+      "source": [
+        "class Node(tf.experimental.ExtensionType):\n",
+        "  value: tf.Tensor\n",
+        "  children: Tuple[\"Node\", ...] = ()\n",
+        "\n",
+        "Node(3, [Node(5), Node(2)])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "boaNg1zHgoVn"
+      },
+      "source": [
+        "### Defining subclasses\n",
+        "\n",
+        "Extension types may be subclassed using the standard Python syntax. Extension type subclasses may add new fields, methods, and properties; and may override the constructor, the printable representation, and the equality operator. The following example defines a basic `TensorGraph` class that uses three `Tensor` fields to encode a set of edges between nodes. It then defines a subclass that adds a `Tensor` field to record a \"feature value\" for each node. The subclass also defines a method to propagate the feature values along the edges."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "58r6qRiK-uZh"
+      },
+      "outputs": [],
+      "source": [
+        "class TensorGraph(tf.experimental.ExtensionType):\n",
+        "  num_nodes: tf.Tensor\n",
+        "  edge_src: tf.Tensor   # edge_src[e] = index of src node for edge e.\n",
+        "  edge_dst: tf.Tensor   # edge_dst[e] = index of dst node for edge e.\n",
+        "\n",
+        "class TensorGraphWithNodeFeature(TensorGraph):\n",
+        "  node_features: tf.Tensor  # node_features[n] = feature value for node n.\n",
+        "\n",
+        "  def propagate_features(self, weight=1.0) -> 'TensorGraphWithNodeFeature':\n",
+        "    updates = tf.gather(self.node_features, self.edge_src) * weight\n",
+        "    new_node_features = tf.tensor_scatter_nd_add(\n",
+        "        self.node_features, tf.expand_dims(self.edge_dst, 1), updates)\n",
+        "    return TensorGraphWithNodeFeature(\n",
+        "        self.num_nodes, self.edge_src, self.edge_dst, new_node_features)\n",
+        "\n",
+        "g = TensorGraphWithNodeFeature(  # Edges: 0->1, 4->3, 2->2, 2->1\n",
+        "    num_nodes=5, edge_src=[0, 4, 2, 2], edge_dst=[1, 3, 2, 1],\n",
+        "    node_features=[10.0, 0.0, 2.0, 5.0, -1.0, 0.0])\n",
+        "\n",
+        "print(\"Original features:\", g.node_features)\n",
+        "print(\"After propagating:\", g.propagate_features().node_features)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "U_oElT5HzqSG"
+      },
+      "source": [
+        "### Defining private fields\n",
+        "\n",
+        "An extension type's fields may be marked private by prefixing them with an underscore (following standard Python conventions). This does not impact the way that TensorFlow treats the fields in any way; but simply serves as a signal to any users of the extension type that those fields are private.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oMdH7ORqh8Pl"
+      },
+      "source": [
+        "### Customizing the `ExtensionType`'s `TypeSpec`\n",
+        "\n",
+        "Each `ExtensionType` class has a corresponding `TypeSpec` class, which is created automatically and stored as `.Spec`. For more information, see the section \"Nested TypeSpec\" above.\n",
+        "\n",
+        "To customize the `TypeSpec`, simply define your own nested class named `Spec`, and `ExtensionType` will use that as the basis for the automatically constructed `TypeSpec`. You can customize the `Spec` class by:\n",
+        "\n",
+        "* Overriding the default printable representation.\n",
+        "* Overriding the default constructor.\n",
+        "* Defining methods, `classmethod`s, `staticmethod`s, and properties.\n",
+        "\n",
+        "The following example customizes the `MaskedTensor.Spec` class to make it easier to use:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Gm4RaqbkLlNG"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.ExtensionType):\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "\n",
+        "  shape = property(lambda self: self.values.shape)\n",
+        "  dtype = property(lambda self: self.values.dtype)\n",
+        "\n",
+        "  def __repr__(self):\n",
+        "    return masked_tensor_str(self.values, self.mask)\n",
+        "\n",
+        "  def with_values(self, new_values):\n",
+        "    return MaskedTensor(new_values, self.mask)\n",
+        "\n",
+        "  class Spec:\n",
+        "    def __init__(self, shape, dtype=tf.float32):\n",
+        "      self.values = tf.TensorSpec(shape, dtype)\n",
+        "      self.mask = tf.TensorSpec(shape, tf.bool)\n",
+        "\n",
+        "    def __repr__(self):\n",
+        "      return f\"MaskedTensor.Spec(shape={self.shape}, dtype={self.dtype})\"\n",
+        "\n",
+        "    shape = property(lambda self: self.values.shape)\n",
+        "    dtype = property(lambda self: self.values.dtype)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s3zzUXPSNF72"
+      },
+      "source": [
+        "**Note**: The custom `Spec` class may not use any instance variables that were not declared in the original `ExtensionType`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rip4GCuYPL7o"
+      },
+      "source": [
+        "## Tensor API dispatch\n",
+        "\n",
+        "Extension types can be \"tensor-like\", in the sense that they specialize or extend the interface defined by the `tf.Tensor` type. Examples of tensor-like extension types include `RaggedTensor`, `SparseTensor`, and `MaskedTensor`. ***Dispatch decorators*** can be used to override the default behavior of TensorFlow operations when applied to tensor-like extension types. TensorFlow currently defines three dispatch decorators:\n",
+        "\n",
+        "* `@tf.experimental.dispatch_for_api(tf_api)`\n",
+        "* `@tf.experimental.dispatch_for_unary_elementwise_apis(x_type)`\n",
+        "* `@tf.experimental.dispatch_for_binary_elementwise_apis(x_type, y_type)`"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5BTQHcY4gHwZ"
+      },
+      "source": [
+        "### Dispatch for a single API\n",
+        "\n",
+        "The `tf.experimental.dispatch_for_api` decorator overrides the default behavior of a specified TensorFlow operation when it is called with the specified signature. For example, you can use this decorator to specify how `tf.stack` should process `MaskedTensor` values:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B4QgO_fUW2o2"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.experimental.dispatch_for_api(tf.stack)\n",
+        "def masked_stack(values: List[MaskedTensor], axis = 0):\n",
+        "  return MaskedTensor(tf.stack([v.values for v in values], axis),\n",
+        "                      tf.stack([v.mask for v in values], axis))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FxKcKWNUaLvm"
+      },
+      "source": [
+        "This overrides the default implementation for `tf.stack` whenever it is called with a list of `MaskedTensor` values (since the `values` argument is annotated with `typing.List[MaskedTensor]`):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RqpFjaAvaA19"
+      },
+      "outputs": [],
+      "source": [
+        "x = MaskedTensor([1, 2, 3], [True, True, False])\n",
+        "y = MaskedTensor([4, 5, 6], [False, True, True])\n",
+        "tf.stack([x, y])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "loGi8taCa265"
+      },
+      "source": [
+        "To allow `tf.stack` to handle lists of mixed `MaskedTensor` and `Tensor` values,  you can refine the type annotation for the `values` parameter and update the body of the function appropriately:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_xySkm0ganAI"
+      },
+      "outputs": [],
+      "source": [
+        "tf.experimental.unregister_dispatch_for(masked_stack)\n",
+        "\n",
+        "def convert_to_masked_tensor(x):\n",
+        "  if isinstance(x, MaskedTensor):\n",
+        "    return x\n",
+        "  else:\n",
+        "    return MaskedTensor(x, tf.ones_like(x, tf.bool))\n",
+        "\n",
+        "@tf.experimental.dispatch_for_api(tf.stack)\n",
+        "def masked_stack_v2(values: List[Union[MaskedTensor, tf.Tensor]], axis = 0):\n",
+        "  values = [convert_to_masked_tensor(v) for v in values]\n",
+        "  return MaskedTensor(tf.stack([v.values for v in values], axis),\n",
+        "                      tf.stack([v.mask for v in values], axis))\n",
+        "x = MaskedTensor([1, 2, 3], [True, True, False])\n",
+        "y = tf.constant([4, 5, 6])\n",
+        "tf.stack([x, y, x])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ITioFCyjQm8V"
+      },
+      "source": [
+        "For a list of APIs that can be overridden, see the API documentation for `tf.experimental.dispatch_for_api`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f91SaHSqc-jO"
+      },
+      "source": [
+        "### Dispatch for all unary elementwise APIs\n",
+        "\n",
+        "The `tf.experimental.dispatch_for_unary_elementwise_apis` decorator overrides the default behavior of ***all*** unary elementwise ops (such as `tf.math.cos`) whenever the value for the first argument (typically named `x`) matches the type annotation `x_type`. The decorated function should take two arguments:\n",
+        "\n",
+        "* `api_func`: A function that takes a single parameter and performs the elementwise operation (for example, `tf.abs`).\n",
+        "* `x`: The first argument to the elementwise operation.\n",
+        "\n",
+        "The following example updates all unary elementwise operations to handle the `MaskedTensor` type:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cv5fV4xxZI9q"
+      },
+      "outputs": [],
+      "source": [
+        " @tf.experimental.dispatch_for_unary_elementwise_apis(MaskedTensor)\n",
+        " def masked_tensor_unary_elementwise_api_handler(api_func, x):\n",
+        "   return MaskedTensor(api_func(x.values), x.mask)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qiK4n6vaeFwo"
+      },
+      "source": [
+        "This function will now be used whenever a unary elementwise operation is called on a `MaskedTensor`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SkH0xi5gd_41"
+      },
+      "outputs": [],
+      "source": [
+        " x = MaskedTensor([1, -2, -3], [True, False, True])\n",
+        " print(tf.abs(x))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2Ej5fxLBfaXW"
+      },
+      "outputs": [],
+      "source": [
+        "print(tf.ones_like(x, dtype=tf.float32))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z9OgLyfEejqc"
+      },
+      "source": [
+        "### Dispatch for binary all elementwise APIs\n",
+        "\n",
+        "Similarly, `tf.experimental.dispatch_for_binary_elementwise_apis` can be used to update all binary elementwise operations to handle the `MaskedTensor` type:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Z8Du-GPofpCW"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.experimental.dispatch_for_binary_elementwise_apis(MaskedTensor, MaskedTensor)\n",
+        "def masked_tensor_binary_elementwise_api_handler(api_func, x, y):\n",
+        "  return MaskedTensor(api_func(x.values, y.values), x.mask & y.mask)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gghVHDfSfyi2"
+      },
+      "outputs": [],
+      "source": [
+        "x = MaskedTensor([1, -2, -3], [True, False, True])\n",
+        "y = MaskedTensor([[4], [5]], [[True], [False]])\n",
+        "tf.math.add(x, y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "txTGg9pzG0Ux"
+      },
+      "source": [
+        "For a list of the elementwise APIs that are overridden, go to the API documentation for `tf.experimental.dispatch_for_unary_elementwise_apis` and `tf.experimental.dispatch_for_binary_elementwise_apis`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UseRtohYKiE5"
+      },
+      "source": [
+        "## Batchable `ExtensionType`s\n",
+        "\n",
+        "An `ExtensionType` is *batchable* if a single instance can be used to represent a batch of values. Typically, this is accomplished by adding batch dimensions to all nested `Tensor`s. The following TensorFlow APIs require that any extension type inputs be batchable:\n",
+        "\n",
+        "* `tf.data.Dataset` (`batch`, `unbatch`, `from_tensor_slices`)\n",
+        "* `tf.keras` (`fit`, `evaluate`, `predict`)\n",
+        "* `tf.map_fn`"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hWPauKGj_yRz"
+      },
+      "source": [
+        "By default, `BatchableExtensionType` creates batched values by batching any nested `Tensor`s, `CompositeTensor`s, and `ExtensionType`s. If this is not appropriate for your class, then you will need to use `tf.experimental.ExtensionTypeBatchEncoder` to override this default behavior. For example, it would not be appropriate to create a batch of `tf.SparseTensor` values by simply stacking individual sparse tensors' `values`, `indices`, and `dense_shape` fields -- in most cases, you can't stack these tensors, since they have incompatible shapes; and even if you could, the result would not be a valid `SparseTensor`.\n",
+        "\n",
+        "\n",
+        "**Note**: `BatchableExtensionType`s do *not* automatically define dispatchers for `tf.stack`, `tf.concat`, `tf.slice`, etc. If your class needs to be supported by these APIs, then use the dispatch decorators described above."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xkOJ8ke8GH7s"
+      },
+      "source": [
+        "### `BatchableExtensionType` example: `Network`\n",
+        "As an example, consider a simple `Network` class used for load balancing, which tracks how much work is left to do at each node, and how much bandwidth is available to move work between nodes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tOeEXwCcfrPd"
+      },
+      "outputs": [],
+      "source": [
+        "class Network(tf.experimental.ExtensionType):  # This version is not batchable.\n",
+        "  work: tf.Tensor       # work[n] = work left to do at node n\n",
+        "  bandwidth: tf.Tensor  # bandwidth[n1, n2] = bandwidth from n1->n2\n",
+        "\n",
+        "net1 = Network([5., 3, 8], [[0., 2, 0], [2, 0, 3], [0, 3, 0]])\n",
+        "net2 = Network([3., 4, 2], [[0., 2, 2], [2, 0, 2], [2, 2, 0]])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PaOzUev6g3wT"
+      },
+      "source": [
+        "To make this type batchable, change the base type to `BatchableExtensionType`, and adjust the shape of each field to include optional batch dimensions. The following example also adds a `shape` field to keep track of the batch shape. This `shape` field is not required by `tf.data.Dataset` or `tf.map_fn`, but it *is* required by `tf.keras`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "T03WWBSMg2XC"
+      },
+      "outputs": [],
+      "source": [
+        "class Network(tf.experimental.BatchableExtensionType):\n",
+        "  shape: tf.TensorShape  # batch shape. A single network has shape=[].\n",
+        "  work: tf.Tensor        # work[*shape, n] = work left to do at node n\n",
+        "  bandwidth: tf.Tensor   # bandwidth[*shape, n1, n2] = bandwidth from n1->n2\n",
+        "\n",
+        "  def __init__(self, work, bandwidth):\n",
+        "    self.work = tf.convert_to_tensor(work)\n",
+        "    self.bandwidth = tf.convert_to_tensor(bandwidth)\n",
+        "    work_batch_shape = self.work.shape[:-1]\n",
+        "    bandwidth_batch_shape = self.bandwidth.shape[:-2]\n",
+        "    self.shape = work_batch_shape.merge_with(bandwidth_batch_shape)\n",
+        "\n",
+        "  def __repr__(self):\n",
+        "    return network_repr(self)\n",
+        "\n",
+        "def network_repr(network):\n",
+        "  work = network.work\n",
+        "  bandwidth = network.bandwidth\n",
+        "  if hasattr(work, 'numpy'):\n",
+        "    work = ' '.join(str(work.numpy()).split())\n",
+        "  if hasattr(bandwidth, 'numpy'):\n",
+        "    bandwidth = ' '.join(str(bandwidth.numpy()).split())\n",
+        "  return (f\"\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NUUJe9HuIPel"
+      },
+      "outputs": [],
+      "source": [
+        "net1 = Network([5., 3, 8], [[0., 2, 0], [2, 0, 3], [0, 3, 0]])\n",
+        "net2 = Network([3., 4, 2], [[0., 2, 2], [2, 0, 2], [2, 2, 0]])\n",
+        "batch_of_networks = Network(\n",
+        "    work=tf.stack([net1.work, net2.work]),\n",
+        "    bandwidth=tf.stack([net1.bandwidth, net2.bandwidth]))\n",
+        "print(f\"net1={net1}\")\n",
+        "print(f\"net2={net2}\")\n",
+        "print(f\"batch={batch_of_networks}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r0qWur5JGc3d"
+      },
+      "source": [
+        "You can then use `tf.data.Dataset` to iterate through a batch of networks:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BN_kixAUFZtv"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices(batch_of_networks)\n",
+        "for i, network in enumerate(dataset):\n",
+        "  print(f\"Batch element {i}: {network}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aXENhTzIIjbM"
+      },
+      "source": [
+        "And you can also use `map_fn` to apply a function to each batch element:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "j1XEsSWj9a3D"
+      },
+      "outputs": [],
+      "source": [
+        "def balance_work_greedy(network):\n",
+        "  delta = (tf.expand_dims(network.work, -1) - tf.expand_dims(network.work, -2))\n",
+        "  delta /= 4\n",
+        "  delta = tf.maximum(tf.minimum(delta, network.bandwidth), -network.bandwidth)\n",
+        "  new_work = network.work + tf.reduce_sum(delta, -1)\n",
+        "  return Network(new_work, network.bandwidth)\n",
+        "\n",
+        "tf.map_fn(balance_work_greedy, batch_of_networks)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f_HLsTT02Xul"
+      },
+      "source": [
+        "## TensorFlow APIs that support `ExtensionType`s"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NNiQad2U2alT"
+      },
+      "source": [
+        "### @tf.function\n",
+        "\n",
+        "[`tf.function`](https://www.tensorflow.org/guide/function) is a decorator that precomputes TensorFlow graphs for Python functions, which can substantially improve the performance of your TensorFlow code. Extension type values can be used transparently with `@tf.function`-decorated functions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jQ_rAvrA6qEb"
+      },
+      "outputs": [],
+      "source": [
+        "class Pastry(tf.experimental.ExtensionType):\n",
+        "  sweetness: tf.Tensor  # 2d embedding that encodes sweetness\n",
+        "  chewiness: tf.Tensor  # 2d embedding that encodes chewiness\n",
+        "\n",
+        "@tf.function\n",
+        "def combine_pastry_features(x: Pastry):\n",
+        "  return (x.sweetness + x.chewiness) / 2\n",
+        "\n",
+        "cookie = Pastry(sweetness=[1.2, 0.4], chewiness=[0.8, 0.2])\n",
+        "combine_pastry_features(cookie)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "u1P-0Udg71Vx"
+      },
+      "source": [
+        "If you wish to explicitly specify the `input_signature` for  `tf.function`, then you can do so using the extension type's `TypeSpec`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0df90E4x78d7"
+      },
+      "outputs": [],
+      "source": [
+        "pastry_spec = Pastry.Spec(tf.TensorSpec([2]), tf.TensorSpec(2))\n",
+        "\n",
+        "@tf.function(input_signature=[pastry_spec])\n",
+        "def increase_sweetness(x: Pastry, delta=1.0):\n",
+        "  return Pastry(x.sweetness + delta, x.chewiness)\n",
+        "\n",
+        "increase_sweetness(cookie)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CdTfc5nD9JpD"
+      },
+      "source": [
+        "#### Concrete functions\n",
+        "Concrete functions encapsulate individual traced graphs that are built by `tf.function`. Extension types can be used transparently with concrete functions.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FyHBBQWk9xz2"
+      },
+      "outputs": [],
+      "source": [
+        "cf = combine_pastry_features.get_concrete_function(pastry_spec)\n",
+        "cf(cookie)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LYas8gtG5IMA"
+      },
+      "source": [
+        "### Control flow operations\n",
+        "\n",
+        "Extension types are supported by TensorFlow's control-flow operations:\n",
+        "\n",
+        "* `tf.cond`\n",
+        "* `tf.case`\n",
+        "* `tf.while_loop`\n",
+        "* `tf.identity`\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6G2XE9ZtJu8z"
+      },
+      "outputs": [],
+      "source": [
+        "# Example: using tf.cond to select between two MaskedTensors. Note that the\n",
+        "# two MaskedTensors don't need to have the same shape.\n",
+        "a = MaskedTensor([1., 2, 3], [True, False, True])\n",
+        "b = MaskedTensor([22., 33, 108, 55], [True, True, True, False])\n",
+        "condition = tf.constant(True)\n",
+        "print(tf.cond(condition, lambda: a, lambda: b))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2NwLOw1kKSek"
+      },
+      "outputs": [],
+      "source": [
+        "# Example: using tf.while_loop with MaskedTensor.\n",
+        "cond = lambda i, _: i < 10\n",
+        "def body(i, mt):\n",
+        "  return i + 1, mt.with_values(mt.values + 3 / 7)\n",
+        "print(tf.while_loop(cond, body, [0, b])[1])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zkN7IuWVMRzn"
+      },
+      "source": [
+        "### Autograph control flow\n",
+        "\n",
+        "Extension types are also supported by control flow statements in `tf.function` (using autograph). In the following example, the `if` statement and `for` statements are automatically converted to `tf.cond` and `tf.while_loop` operations, which support extension types."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4RFySEl8gZ8w"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def fn(x, b):\n",
+        "  if b:\n",
+        "    x = MaskedTensor(x, tf.less(x, 0))\n",
+        "  else:\n",
+        "    x = MaskedTensor(x, tf.greater(x, 0))\n",
+        "  for i in tf.range(5 if b else 7):\n",
+        "    x = x.with_values(x.values + 1 / 2)\n",
+        "  return x\n",
+        "\n",
+        "print(fn(tf.constant([1., -2, 3]), tf.constant(True)))\n",
+        "print(fn(tf.constant([1., -2, 3]), tf.constant(False)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-FjZt2ohfja4"
+      },
+      "source": [
+        "### Keras\n",
+        "\n",
+        "[tf.keras](https://www.tensorflow.org/guide/keras) is TensorFlow's high-level API for building and training deep learning models. Extension types may be passed as inputs to a Keras model, passed between Keras layers, and returned by Keras models. Keras currently puts two requirements on extension types:\n",
+        "\n",
+        "* They must be batchable (go to \"Batchable `ExtensionType`s\" above).\n",
+        "* They must have a field or property named `shape`. `shape[0]` is assumed to be the batch dimension.\n",
+        "\n",
+        "The following two subsections give examples showing how extension types can be used with Keras.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QH1TXQYiGv8u"
+      },
+      "source": [
+        "#### Keras example: `Network`\n",
+        "\n",
+        "For the first example, consider the `Network` class defined in the \"Batchable `ExtensionType`s\" section above, which can be used for load balancing work between nodes. Its definition is repeated here:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zHj1RIS2PK50"
+      },
+      "outputs": [],
+      "source": [
+        "class Network(tf.experimental.BatchableExtensionType):\n",
+        "  shape: tf.TensorShape  # batch shape. A single network has shape=[].\n",
+        "  work: tf.Tensor        # work[*shape, n] = work left to do at node n\n",
+        "  bandwidth: tf.Tensor   # bandwidth[*shape, n1, n2] = bandwidth from n1->n2\n",
+        "\n",
+        "  def __init__(self, work, bandwidth):\n",
+        "    self.work = tf.convert_to_tensor(work)\n",
+        "    self.bandwidth = tf.convert_to_tensor(bandwidth)\n",
+        "    work_batch_shape = self.work.shape[:-1]\n",
+        "    bandwidth_batch_shape = self.bandwidth.shape[:-2]\n",
+        "    self.shape = work_batch_shape.merge_with(bandwidth_batch_shape)\n",
+        "\n",
+        "  def __repr__(self):\n",
+        "    return network_repr(self)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "w9LPTEVJD0FD"
+      },
+      "outputs": [],
+      "source": [
+        "single_network = Network(  # A single network with 4 nodes.\n",
+        "    work=[8.0, 5, 12, 2],\n",
+        "    bandwidth=[[0.0, 1, 2, 2], [1, 0, 0, 2], [2, 0, 0, 1], [2, 2, 1, 0]])\n",
+        "\n",
+        "batch_of_networks = Network(  # Batch of 2 networks, each w/ 2 nodes.\n",
+        "    work=[[8.0, 5], [3, 2]],\n",
+        "    bandwidth=[[[0.0, 1], [1, 0]], [[0, 2], [2, 0]]])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IUfWi3SDD0dj"
+      },
+      "source": [
+        "You can define a new Keras layer that processes `Network`s."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2WSYt58r4SF1"
+      },
+      "outputs": [],
+      "source": [
+        "class BalanceNetworkLayer(tf.keras.layers.Layer):\n",
+        "  \"\"\"Layer that balances work between nodes in a network.\n",
+        "\n",
+        "  Shifts work from more busy nodes to less busy nodes, constrained by bandwidth.\n",
+        "  \"\"\"\n",
+        "  def call(self, inputs):\n",
+        "    # This function is defined above in the \"Batchable `ExtensionType`s\" section.\n",
+        "    return balance_work_greedy(inputs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VWwFJNb1E03q"
+      },
+      "source": [
+        "You can then use these layers to create a simple model. To feed an `ExtensionType` into a model, you can use a `tf.keras.layer.Input` layer with `type_spec` set to the extension type's `TypeSpec`. If the Keras model will be used to process batches, then the `type_spec` must include the batch dimension."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "plTyqISRExA4"
+      },
+      "outputs": [],
+      "source": [
+        "input_spec = Network.Spec(shape=None,\n",
+        "                          work=tf.TensorSpec(None, tf.float32),\n",
+        "                          bandwidth=tf.TensorSpec(None, tf.float32))\n",
+        "model = tf.keras.Sequential([\n",
+        "    tf.keras.layers.Input(type_spec=input_spec),\n",
+        "    BalanceNetworkLayer(),\n",
+        "    ])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hyeAbt1WFIiO"
+      },
+      "source": [
+        "Finally, you can apply the model to a single network and to a batch of networks."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hH1EtA5lFHdN"
+      },
+      "outputs": [],
+      "source": [
+        "model(single_network)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "V7eM67M7FYYM"
+      },
+      "outputs": [],
+      "source": [
+        "model(batch_of_networks)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tOxtt9Z1HDCv"
+      },
+      "source": [
+        "#### Keras example: MaskedTensor\n",
+        "\n",
+        "In this example, `MaskedTensor` is extended to support `Keras`. `shape` is defined as a property that is calculated from the `values` field. Keras requires that you add this property to both the extension type and its `TypeSpec`. `MaskedTensor` also defines a `__name__` variable, which will be required for `SavedModel` serialization (below)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1JBZ_t48Ht7e"
+      },
+      "outputs": [],
+      "source": [
+        "class MaskedTensor(tf.experimental.BatchableExtensionType):\n",
+        "  # __name__ is required for serialization in SavedModel; see below for details.\n",
+        "  __name__ = 'extension_type_colab.MaskedTensor'\n",
+        "\n",
+        "  values: tf.Tensor\n",
+        "  mask: tf.Tensor\n",
+        "\n",
+        "  shape = property(lambda self: self.values.shape)\n",
+        "  dtype = property(lambda self: self.values.dtype)\n",
+        "\n",
+        "  def with_default(self, default):\n",
+        "    return tf.where(self.mask, self.values, default)\n",
+        "\n",
+        "  def __repr__(self):\n",
+        "    return masked_tensor_str(self.values, self.mask)\n",
+        "\n",
+        "  class Spec:\n",
+        "    def __init__(self, shape, dtype=tf.float32):\n",
+        "      self.values = tf.TensorSpec(shape, dtype)\n",
+        "      self.mask = tf.TensorSpec(shape, tf.bool)\n",
+        "\n",
+        "    shape = property(lambda self: self.values.shape)\n",
+        "    dtype = property(lambda self: self.values.dtype)\n",
+        "\n",
+        "    def with_shape(self):\n",
+        "      return MaskedTensor.Spec(tf.TensorSpec(shape, self.values.dtype),\n",
+        "                               tf.TensorSpec(shape, self.mask.dtype))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oer8BVc8H7_V"
+      },
+      "source": [
+        "Next, the dispatch decorators are used to override the default behavior of several TensorFlow APIs. Since these APIs are used by standard Keras layers (such as the `Dense` layer), overriding these will allow us to use those layers with `MaskedTensor`. For the purposes of this example, `matmul` for masked tensors is defined to treat the masked values as zeros (that is, to not include them in the product)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xy0dhQ_b-ca_"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.experimental.dispatch_for_unary_elementwise_apis(MaskedTensor)\n",
+        "def unary_elementwise_op_handler(op, x):\n",
+        " return MaskedTensor(op(x.values), x.mask)\n",
+        "\n",
+        "@tf.experimental.dispatch_for_binary_elementwise_apis(\n",
+        "    Union[MaskedTensor, tf.Tensor],\n",
+        "    Union[MaskedTensor, tf.Tensor])\n",
+        "def binary_elementwise_op_handler(op, x, y):\n",
+        "  x = convert_to_masked_tensor(x)\n",
+        "  y = convert_to_masked_tensor(y)\n",
+        "  return MaskedTensor(op(x.values, y.values), x.mask & y.mask)\n",
+        "\n",
+        "@tf.experimental.dispatch_for_api(tf.matmul)\n",
+        "def masked_matmul(a: MaskedTensor, b,\n",
+        "                  transpose_a=False, transpose_b=False,\n",
+        "                  adjoint_a=False, adjoint_b=False,\n",
+        "                  a_is_sparse=False, b_is_sparse=False,\n",
+        "                  output_type=None,\n",
+        "                  grad_a=False, grad_b=False,\n",
+        "                  name=None,\n",
+        "                  ):\n",
+        "  if isinstance(a, MaskedTensor):\n",
+        "    a = a.with_default(0)\n",
+        "  if isinstance(b, MaskedTensor):\n",
+        "    b = b.with_default(0)\n",
+        "  return tf.matmul(a, b, transpose_a, transpose_b, adjoint_a,\n",
+        "                   adjoint_b, a_is_sparse, b_is_sparse,\n",
+        "                   output_type)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "osJ_L-fKJusI"
+      },
+      "source": [
+        "You can then construct a Keras model that accepts `MaskedTensor` inputs, using standard Keras layers:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IS6JCVbk1rd0"
+      },
+      "outputs": [],
+      "source": [
+        "input_spec = MaskedTensor.Spec([None, 2], tf.float32)\n",
+        "\n",
+        "masked_tensor_model = tf.keras.Sequential([\n",
+        "    tf.keras.layers.Input(type_spec=input_spec),\n",
+        "    tf.keras.layers.Dense(16, activation=\"relu\"),\n",
+        "    tf.keras.layers.Dense(1)])\n",
+        "masked_tensor_model.compile(loss='binary_crossentropy', optimizer='rmsprop')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SB1WUSzn1RPj"
+      },
+      "outputs": [],
+      "source": [
+        "a = MaskedTensor([[1., 2], [3, 4], [5, 6]],\n",
+        "                  [[True, False], [False, True], [True, True]])\n",
+        "masked_tensor_model.fit(a, tf.constant([[1], [0], [1]]), epochs=3)\n",
+        "print(masked_tensor_model(a))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "msmd9XcL2bqb"
+      },
+      "source": [
+        "### SavedModel\n",
+        "\n",
+        "A [SavedModel](https://www.tensorflow.org/guide/saved_model) is a serialized TensorFlow program, including both weights and computation. It can be built from a Keras model or from a custom model. In either case, extension types can be used transparently with the functions and methods defined by a SavedModel.\n",
+        "\n",
+        "SavedModel can save models, layers, and functions that process extension types, as long as the extension types have a `__name__` field. This name is used to register the extension type, so it can be located when the model is loaded."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PEtbFrz6-Vku"
+      },
+      "source": [
+        "#### Example: saving a Keras model\n",
+        "\n",
+        "Keras models that use extension types may be saved using `SavedModel`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ecxQMnybSzV6"
+      },
+      "outputs": [],
+      "source": [
+        "masked_tensor_model_path = tempfile.mkdtemp()\n",
+        "tf.saved_model.save(masked_tensor_model, masked_tensor_model_path)\n",
+        "imported_model = tf.saved_model.load(masked_tensor_model_path)\n",
+        "imported_model(a)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ne2nu3r6-XMr"
+      },
+      "source": [
+        "#### Example: saving a custom model\n",
+        "\n",
+        "SavedModel can also be used to save custom `tf.Module` subclasses with functions that process extension types."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2V6hV3yOT2vz"
+      },
+      "outputs": [],
+      "source": [
+        "class CustomModule(tf.Module):\n",
+        "  def __init__(self, variable_value):\n",
+        "    super().__init__()\n",
+        "    self.v = tf.Variable(variable_value)\n",
+        "\n",
+        "  @tf.function\n",
+        "  def grow(self, x: MaskedTensor):\n",
+        "    \"\"\"Increase values in `x` by multiplying them by `self.v`.\"\"\"\n",
+        "    return MaskedTensor(x.values * self.v, x.mask)\n",
+        "\n",
+        "module = CustomModule(100.0)\n",
+        "\n",
+        "module.grow.get_concrete_function(MaskedTensor.Spec(shape=None,\n",
+        "                                                    dtype=tf.float32))\n",
+        "custom_module_path = tempfile.mkdtemp()\n",
+        "tf.saved_model.save(module, custom_module_path)\n",
+        "imported_model = tf.saved_model.load(custom_module_path)\n",
+        "imported_model.grow(MaskedTensor([1., 2, 3], [False, True, False]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o6beljh576ee"
+      },
+      "source": [
+        "#### Loading a SavedModel when the `ExtensionType` is unavailable\n",
+        "\n",
+        "If you load a `SavedModel` that uses an `ExtensionType`, but that `ExtensionType` is not available (that is, it has not been imported), then you will get a warning and TensorFlow will fall back to using an \"anonymous extension type\" object. This object will have the same fields as the original type, but will lack any further customization you have added for the type, such as custom methods or properties."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ec9PcUkJ9bFK"
+      },
+      "source": [
+        "#### Using `ExtensionType`s with TensorFlow Serving\n",
+        "\n",
+        "Currently, [TensorFlow Serving](https://www.tensorflow.org/tfx/guide/serving) (and other consumers of the SavedModel \"signatures\" dictionary) require that all inputs and outputs be raw tensors. If you wish to use TensorFlow Serving with a model that uses extension types, then you can add wrapper methods that compose or decompose extension type values from tensors. For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4VnzAwVo9tTc"
+      },
+      "outputs": [],
+      "source": [
+        "class CustomModuleWrapper(tf.Module):\n",
+        "  def __init__(self, variable_value):\n",
+        "    super().__init__()\n",
+        "    self.v = tf.Variable(variable_value)\n",
+        "\n",
+        "  @tf.function\n",
+        "  def var_weighted_mean(self, x: MaskedTensor):\n",
+        "    \"\"\"Mean value of unmasked values in x, weighted by self.v.\"\"\"\n",
+        "    x = MaskedTensor(x.values * self.v, x.mask)\n",
+        "    return (tf.reduce_sum(x.with_default(0)) /\n",
+        "            tf.reduce_sum(tf.cast(x.mask, x.dtype)))\n",
+        "\n",
+        "  @tf.function()\n",
+        "  def var_weighted_mean_wrapper(self, x_values, x_mask):\n",
+        "    \"\"\"Raw tensor wrapper for var_weighted_mean.\"\"\"\n",
+        "    return self.var_weighted_mean(MaskedTensor(x_values, x_mask))\n",
+        "\n",
+        "module = CustomModuleWrapper([3., 2., 8., 5.])\n",
+        "\n",
+        "module.var_weighted_mean_wrapper.get_concrete_function(\n",
+        "    tf.TensorSpec(None, tf.float32), tf.TensorSpec(None, tf.bool))\n",
+        "custom_module_path = tempfile.mkdtemp()\n",
+        "tf.saved_model.save(module, custom_module_path)\n",
+        "imported_model = tf.saved_model.load(custom_module_path)\n",
+        "x = MaskedTensor([1., 2., 3., 4.], [False, True, False, True])\n",
+        "imported_model.var_weighted_mean_wrapper(x.values, x.mask)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4dwBadWQ5G9_"
+      },
+      "source": [
+        "### `Dataset`s\n",
+        "\n",
+        "[`tf.data`](https://www.tensorflow.org/guide/data) is an API that enables you to build complex input pipelines from simple, reusable pieces. Its core data structure is `tf.data.Dataset`, which represents a sequence of elements, in which each element consists of one or more components."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GcIR19FuwRJV"
+      },
+      "source": [
+        "#### Building `Dataset`s with extension types\n",
+        "\n",
+        "Datasets can be built from extension type values using `Dataset.from_tensors`, `Dataset.from_tensor_slices`, or `Dataset.from_generator`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Oe7fRCkzwdub"
+      },
+      "outputs": [],
+      "source": [
+        "ds = tf.data.Dataset.from_tensors(Pastry(5, 5))\n",
+        "iter(ds).next()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fk9CD2fZx6yT"
+      },
+      "outputs": [],
+      "source": [
+        "mt = MaskedTensor(tf.reshape(range(20), [5, 4]), tf.ones([5, 4]))\n",
+        "ds = tf.data.Dataset.from_tensor_slices(mt)\n",
+        "for value in ds:\n",
+        "  print(value)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DGw8y87awsOJ"
+      },
+      "outputs": [],
+      "source": [
+        "def value_gen():\n",
+        "  for i in range(2, 7):\n",
+        "    yield MaskedTensor(range(10), [j%i != 0 for j in range(10)])\n",
+        "\n",
+        "ds = tf.data.Dataset.from_generator(\n",
+        "    value_gen, output_signature=MaskedTensor.Spec(shape=[10], dtype=tf.int32))\n",
+        "for value in ds:\n",
+        "  print(value)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wfEm4NInyqtj"
+      },
+      "source": [
+        "#### Batching and unbatching `Dataset`s with extension types\n",
+        "\n",
+        "Datasets with extension types can be batchand and unbatched using `Dataset.batch` and `Dataset.unbatch`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "snoOUE1ay1rO"
+      },
+      "outputs": [],
+      "source": [
+        "batched_ds = ds.batch(2)\n",
+        "for value in batched_ds:\n",
+        "  print(value)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f8PTky6EzBVY"
+      },
+      "outputs": [],
+      "source": [
+        "unbatched_ds = batched_ds.unbatch()\n",
+        "for value in unbatched_ds:\n",
+        "  print(value)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "extension_type.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/function.ipynb b/site/en/guide/function.ipynb
index decf9ed845f..f4677f21eb8 100644
--- a/site/en/guide/function.ipynb
+++ b/site/en/guide/function.ipynb
@@ -3,26 +3,22 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "Jxv6goXm7oGF"
+        "id": "ISubpr_SSsiM"
       },
       "source": [
-        "##### Copyright 2018 The TensorFlow Authors.\n",
-        "\n",
-        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+        "##### Copyright 2020 The TensorFlow Authors.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "llMNufAK7nfK"
+        "cellView": "form",
+        "id": "3jTMb1dySr3V"
       },
       "outputs": [],
       "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\"); { display-mode: \"form\" }\n",
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
         "# You may obtain a copy of the License at\n",
         "#\n",
@@ -38,199 +34,188 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "8Byow2J6LaPl"
+        "id": "6DWfyNThSziV"
       },
       "source": [
-        "# Better performance with tf.function and AutoGraph"
+        "# Better performance with tf.function\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "kGXS3UWBBNoc"
+        "id": "J122XQYG7W6w"
       },
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/function\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/function.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/function.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/function.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "In TensorFlow 2, [eager execution](basics.ipynb) is turned on by default. The user interface is intuitive and flexible (running one-off operations is much easier and faster), but this can come at the expense of performance and deployability.\n",
+        "\n",
+        "You can use `tf.function` to make graphs out of your programs. It is a transformation tool that creates Python-independent dataflow graphs out of your Python code. This will help you create performant and portable models, and it is required to use `SavedModel`.\n",
+        "\n",
+        "This guide will help you conceptualize how `tf.function` works under the hood, so you can use it effectively.\n",
+        "\n",
+        "The main takeaways and recommendations are:\n",
+        "\n",
+        "- Debug in eager mode, then decorate with `@tf.function`.\n",
+        "- Don't rely on Python side effects like object mutation or list appends.\n",
+        "- `tf.function` works best with TensorFlow ops; NumPy and Python calls are converted to constants.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "CydFK2CL7ZHA"
+        "id": "SjvqpgepHJPd"
       },
       "source": [
-        "TF 2.0 brings together the ease of eager execution and the power of TF 1.0. At the center of this merger is `tf.function`, which allows you to transform a subset of Python syntax into portable, high-performance TensorFlow graphs.\n",
-        "\n",
-        "A cool new feature of `tf.function` is AutoGraph, which lets you write graph code using natural Python syntax. For a list of the Python features that you can use with AutoGraph, see [AutoGraph Capabilities and Limitations](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/limitations.md). For more details about `tf.function`, see the RFC [TF 2.0: Functions, not Sessions](https://github.com/tensorflow/community/blob/master/rfcs/20180918-functions-not-sessions-20.md). For more details about AutoGraph, see `tf.autograph`.\n",
-        "\n",
-        "This tutorial will walk you through the basic features of `tf.function` and AutoGraph."
+        "## Setup"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "n4EKOpw9mObL"
+        "id": "otIdN1TS8N7S"
       },
+      "outputs": [],
       "source": [
-        "## Setup\n",
-        "\n",
-        "Import TensorFlow 2.0:"
+        "import tensorflow as tf"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "V9oECvVSI1Kj"
+        "id": "I0xDjO4SHLUD"
       },
-      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "import numpy as np"
+        "Define a helper function to demonstrate the kinds of errors you might encounter:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mT7meGqrZTz9"
+        "id": "D25apou9IOXa"
       },
       "outputs": [],
       "source": [
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf"
+        "import traceback\n",
+        "import contextlib\n",
+        "\n",
+        "# Some helper code to demonstrate the kinds of errors you might encounter.\n",
+        "@contextlib.contextmanager\n",
+        "def assert_raises(error_class):\n",
+        "  try:\n",
+        "    yield\n",
+        "  except error_class as e:\n",
+        "    print('Caught expected exception \\n  {}:'.format(error_class))\n",
+        "    traceback.print_exc(limit=2)\n",
+        "  except Exception as e:\n",
+        "    raise e\n",
+        "  else:\n",
+        "    raise Exception('Expected {} to be raised but no error was raised!'.format(\n",
+        "        error_class))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "77AsVr1GGtBP"
+        "id": "WPSfepzTHThq"
       },
       "source": [
-        "## The `tf.function` decorator\n",
-        "\n",
-        "When you annotate a function with `tf.function`, you can still call it like any other function. But it will be compiled into a graph, which means you get the benefits of faster execution, running on GPU or TPU, or exporting to SavedModel."
+        "## Basics"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "FhIg7-z6HNWj"
+        "id": "CNwYTIJ8r56W"
       },
-      "outputs": [],
       "source": [
-        "@tf.function\n",
-        "def simple_nn_layer(x, y):\n",
-        "  return tf.nn.relu(tf.matmul(x, y))\n",
+        "### Usage\n",
         "\n",
-        "\n",
-        "x = tf.random.uniform((3, 3))\n",
-        "y = tf.random.uniform((3, 3))\n",
-        "\n",
-        "simple_nn_layer(x, y)"
+        "A `tf.function` that you define (for example by applying the `@tf.function` decorator) is just like a core TensorFlow operation: You can execute it eagerly; you can compute gradients; and so on."
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "U-LAE4pMNR9g"
+        "id": "SbtT1-Wm70F2"
       },
+      "outputs": [],
       "source": [
-        "If we examine the result of the annotation, we can see that it's a special callable that handles all interactions with the TensorFlow runtime."
+        "@tf.function  # The decorator converts `add` into a `PolymorphicFunction`.\n",
+        "def add(a, b):\n",
+        "  return a + b\n",
+        "\n",
+        "add(tf.ones([2, 2]), tf.ones([2, 2]))  #  [[2., 2.], [2., 2.]]"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "q4t2iuS7Nqc0"
+        "id": "uP-zUelB8DbX"
       },
       "outputs": [],
       "source": [
-        "simple_nn_layer"
+        "v = tf.Variable(1.0)\n",
+        "with tf.GradientTape() as tape:\n",
+        "  result = add(v, 1.0)\n",
+        "tape.gradient(result, v)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "DqeefLGNXjZQ"
+        "id": "ocWZvqrmHnmX"
       },
       "source": [
-        "If your code uses multiple functions, you don't need to annotate them all - any functions called from an annotated function will also run in graph mode."
+        "You can use `tf.function`s inside other `tf.function`s."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "3VGF7tlVXiZY"
+        "id": "l5qRjdbBVdU6"
       },
       "outputs": [],
       "source": [
-        "def linear_layer(x):\n",
-        "  return 2 * x + 1\n",
-        "\n",
-        "\n",
         "@tf.function\n",
-        "def deep_net(x):\n",
-        "  return tf.nn.relu(linear_layer(x))\n",
+        "def dense_layer(x, w, b):\n",
+        "  return add(tf.matmul(x, w), b)\n",
         "\n",
-        "\n",
-        "deep_net(tf.constant((1, 2, 3)))"
+        "dense_layer(tf.ones([3, 2]), tf.ones([2, 2]), tf.ones([2]))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "yQvg6ZSKWyqE"
+        "id": "piBhz7gYsHqU"
       },
       "source": [
-        "Functions can be faster than eager code, for graphs with many small ops. But for graphs with a few expensive ops (like convolutions), you may not see much speedup.\n"
+        "`tf.function`s can be faster than eager code, especially for graphs with many small ops. But for graphs with a few expensive ops (like convolutions), you may not see much speedup.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0EL6lVwEWuFo"
+        "id": "zuXt4wRysI03"
       },
       "outputs": [],
       "source": [
@@ -242,7 +227,7 @@
         "  return conv_layer(image)\n",
         "\n",
         "image = tf.zeros([1, 200, 200, 100])\n",
-        "# warm up\n",
+        "# Warm up\n",
         "conv_layer(image); conv_fn(image)\n",
         "print(\"Eager conv:\", timeit.timeit(lambda: conv_layer(image), number=10))\n",
         "print(\"Function conv:\", timeit.timeit(lambda: conv_fn(image), number=10))\n",
@@ -250,719 +235,1779 @@
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "L4zj-jpH0jKH"
+        "id": "uZ4Do2AV80cO"
       },
-      "outputs": [],
       "source": [
-        "lstm_cell = tf.keras.layers.LSTMCell(10)\n",
+        "### Tracing\n",
         "\n",
-        "@tf.function\n",
-        "def lstm_fn(input, state):\n",
-        "  return lstm_cell(input, state)\n",
-        "\n",
-        "input = tf.zeros([10, 10])\n",
-        "state = [tf.zeros([10, 10])] * 2\n",
-        "# warm up\n",
-        "lstm_cell(input, state); lstm_fn(input, state)\n",
-        "print(\"eager lstm:\", timeit.timeit(lambda: lstm_cell(input, state), number=10))\n",
-        "print(\"function lstm:\", timeit.timeit(lambda: lstm_fn(input, state), number=10))\n"
+        "This section exposes how `tf.function` works under the hood, including implementation details *which may change in the future*. However, once you understand why and when tracing happens, it's much easier to use `tf.function` effectively!"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "ohbSnA79mcJV"
+        "id": "nhpUtRqsXoyM"
       },
       "source": [
-        "## Use Python control flow\n",
+        "#### What is \"tracing\"?\n",
+        "\n",
+        "A `tf.function` runs your program in a [TensorFlow Graph](https://www.tensorflow.org/guide/intro_to_graphs#what_are_graphs). However, a `tf.Graph` cannot represent all the things that you'd write in an eager TensorFlow program. For instance, Python supports polymorphism, but `tf.Graph` requires its inputs to have a specified data type and dimension. Or you may perform side tasks like reading command-line arguments, raising an error, or working with a more complex Python object; none of these things can run in a `tf.Graph`.\n",
+        "\n",
+        "`tf.function` bridges this gap by separating your code in two stages:\n",
         "\n",
-        "When using data-dependent control flow inside `tf.function`, you can use Python control flow statements and AutoGraph will convert them into appropriate TensorFlow ops. For example, `if` statements will be converted into `tf.cond()` if they depend on a `Tensor`.\n",
+        "  1)  In the first stage, referred to as \"**tracing**\", `tf.function` creates a new `tf.Graph`. Python code runs normally, but all TensorFlow operations (like adding two Tensors) are *deferred*: they are captured by the `tf.Graph` and not run.\n",
         "\n",
-        "In the example below, `x` is a `Tensor` but the `if` statement works as expected:"
+        "  2) In the second stage, a `tf.Graph` which contains everything that was deferred in the first stage is run. This stage is much faster than the tracing stage.\n",
+        "\n",
+        "Depending on its inputs, `tf.function` will not always run the first stage when it is called.  See [\"Rules of tracing\"](#rules_of_tracing) below to get a better sense of how it makes that determination. Skipping the first stage and only executing the second stage is what gives you TensorFlow's high performance.\n",
+        "\n",
+        "When `tf.function` does decide to trace, the tracing stage is immediately followed by the second stage, so calling the `tf.function` both creates and runs the `tf.Graph`. Later you will see how you can run only the tracing stage with [`get_concrete_function`](#obtaining_concrete_functions)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "K7scSzLx662f"
+      },
+      "source": [
+        "When you pass arguments of different types into a `tf.function`, both stages are run:\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "aA3gOodCBkOw"
+        "id": "kojmJrgq8U9v"
       },
       "outputs": [],
       "source": [
         "@tf.function\n",
-        "def square_if_positive(x):\n",
-        "  if x \u003e 0:\n",
-        "    x = x * x\n",
-        "  else:\n",
-        "    x = 0\n",
-        "  return x\n",
-        "\n",
-        "\n",
-        "print('square_if_positive(2) = {}'.format(square_if_positive(tf.constant(2))))\n",
-        "print('square_if_positive(-2) = {}'.format(square_if_positive(tf.constant(-2))))"
+        "def double(a):\n",
+        "  print(\"Tracing with\", a)\n",
+        "  return a + a\n",
+        "\n",
+        "print(double(tf.constant(1)))\n",
+        "print()\n",
+        "print(double(tf.constant(1.1)))\n",
+        "print()\n",
+        "print(double(tf.constant(\"a\")))\n",
+        "print()\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "GMiCUkdyoq98"
+        "id": "QPfouGUQrcNb"
+      },
+      "source": [
+        "Note that if you repeatedly call a `tf.function` with the same argument type, TensorFlow will skip the tracing stage and reuse a previously traced graph, as the generated graph would be identical."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hFccbWFRrsBp"
       },
+      "outputs": [],
       "source": [
-        "Note: The previous example uses simple conditionals with scalar values. \u003ca href=\"#batching\"\u003eBatching\u003c/a\u003e is typically used in real-world code."
+        "# This doesn't print 'Tracing with ...'\n",
+        "print(double(tf.constant(\"b\")))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "m-jWmsCmByyw"
+        "id": "fgIO_XEzcB9o"
       },
       "source": [
-        "AutoGraph supports common Python statements like `while`, `for`, `if`, `break`, `continue` and `return`, with support for nesting. That means you can use `Tensor` expressions in the condition of `while` and `if` statements, or iterate over a `Tensor` in a `for` loop."
+        "You can use `pretty_printed_concrete_signatures()` to see all of the available traces:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "toxKBOXbB1ro"
+        "id": "IiQc4IKAb-NX"
       },
       "outputs": [],
       "source": [
-        "@tf.function\n",
-        "def sum_even(items):\n",
-        "  s = 0\n",
-        "  for c in items:\n",
-        "    if c % 2 \u003e 0:\n",
-        "      continue\n",
-        "    s += c\n",
-        "  return s\n",
+        "print(double.pretty_printed_concrete_signatures())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rKQ92VEWI7n8"
+      },
+      "source": [
+        "So far, you've seen that `tf.function` creates a cached, dynamic dispatch layer over TensorFlow's graph tracing logic. To be more specific about the terminology:\n",
+        "\n",
+        "- A `tf.Graph` is the raw, language-agnostic, portable representation of a TensorFlow computation.\n",
+        "- Tracing is the process through which new `tf.Graph`s are generated from Python code.\n",
+        "- An instance of `tf.Graph` is specialized to the specific input types it was traced with. Differing types require retracing.\n",
+        "- Each traced `tf.Graph` has a corresponding `ConcreteFunction`.\n",
+        "- A `tf.function` manages a cache of `ConcreteFunction`s and picks the right one for your inputs.\n",
+        "- `tf.function` wraps the Python function that will be traced, returning a `tf.types.experimental.PolymorphicFunction` object.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "129-iRsPS-gY"
+      },
+      "source": [
+        "#### Rules of tracing\n",
+        "\n",
+        "When called, a `tf.function` first evaluates the type of each input argument using the `tf.types.experimental.TraceType` of each argument. This is used to construct a `tf.types.experimental.FunctionType` describing the signature of the desired `ConcreteFunction`. We compare this `FunctionType` to the `FunctionType`s of existing `ConcreteFunction`s. If a matching `ConcreteFunction` is found, the call is dispatched to it. If no match is found, a new `ConcreteFunction` is traced for the desired `FunctionType`.\n",
+        "\n",
+        "If multiple matches are found, the most specific signature is chosen. Matching is done by [subtyping](https://en.wikipedia.org/wiki/Subtyping), much like normal function calls in C++ or Java, for instance. For example, `TensorShape([1, 2])` is a subtype of `TensorShape([None, None])` and so a call to the tf.function with `TensorShape([1, 2])` can be dispatched to the `ConcreteFunction` produced with `TensorShape([None, None])` but if a `ConcreteFunction` with `TensorShape([1, None])` also exists then it will be prioritized since it is more specific.\n",
+        "\n",
+        "The `TraceType` is determined from input arguments as follows:\n",
+        "* For `Tensor`, the type is parameterized by the `Tensor`'s `dtype` and `shape`; ranked shapes are a subtype of unranked shapes; fixed dimensions are a subtype of unknown dimensions\n",
+        "* For `Variable`, the type is similar to `Tensor`, but also includes a unique resource ID of the variable, necessary to correctly wire control dependencies\n",
+        "* For Python primitive values, the type corresponds to the **value** itself. For example, the `TraceType` of the value `3` is `LiteralTraceType<3>`, not `int`.\n",
+        "* For Python ordered containers such as `list` and `tuple`, etc., the type is parameterized by the types of their elements; for example, the type of `[1, 2]` is `ListTraceType, LiteralTraceType<2>>` and the type for   `[2, 1]` is `ListTraceType, LiteralTraceType<1>>` which is different.\n",
+        "* For Python mappings such as `dict`, the type is also a mapping from the same keys but to the types of values instead of the actual values. For example, the type of `{1: 2, 3: 4}`, is `MappingTraceType<>>, >>>`. However, unlike ordered containers, `{1: 2, 3: 4}` and `{3: 4, 1: 2}` have equivalent types.\n",
+        "* For Python objects which implement the `__tf_tracing_type__` method, the type is whatever that method returns.\n",
+        "* For any other Python objects, the type is a generic `TraceType`, and the matching precedure is:\n",
+        "  * First it checks if the object is the same object used in the previous trace (using Python `id()` or `is`). Note that this will still match if the object has changed, so if you use Python objects as `tf.function` arguments it's best to use *immutable* ones.\n",
+        "  * Next it checks if the object is equal to the object used in the previous trace (using Python `==`).\n",
+        "      \n",
+        "  Note that this procedure only keeps a [weakref](https://docs.python.org/3/library/weakref.html) to the object and hence only works as long as the object is in scope/not deleted.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GNNN4lgRzpIs"
+      },
+      "source": [
+        "Note: `TraceType` is based on the `tf.function` input parameters so changes to global and [free variables](https://docs.python.org/3/reference/executionmodel.html#binding-of-names) alone will not create a new trace. See [this section](#depending_on_python_global_and_free_variables) for recommended practices when dealing with Python global and free variables."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PEDwbumO32Wh"
+      },
+      "source": [
+        "### Controlling retracing\n",
         "\n",
+        "Retracing, which is when your `tf.function` creates more than one trace, helps ensure that TensorFlow generates correct graphs for each set of inputs. However, tracing is an expensive operation! If your `tf.function` retraces a new graph for every call, you'll find that your code executes more slowly than if you didn't use `tf.function`.\n",
         "\n",
-        "sum_even(tf.constant([10, 12, 15, 20]))"
+        "To control the tracing behavior, you can use the following techniques:"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "AtDaLrbySw4j"
+        "id": "EUtycWJa34TT"
       },
       "source": [
-        "AutoGraph also provides a low-level API for advanced users. For example we can use it to have a look at the generated code."
+        "#### Pass a fixed `input_signature` to `tf.function`\n",
+        "\n",
+        "This forces `tf.function` to constrain itself to only one `tf.types.experimental.FunctionType` composed of the types enumerated by the `input_signature`. Calls that cannot be dispatched to this `FunctionType` will throw an error."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "aRsde3x_SjTQ"
+        "id": "_BDMIRmu1RGB"
       },
       "outputs": [],
       "source": [
-        "print(tf.autograph.to_code(sum_even.python_function))"
+        "@tf.function(input_signature=(tf.TensorSpec(shape=[None], dtype=tf.int32),))\n",
+        "def next_collatz(x):\n",
+        "  print(\"Tracing with\", x)\n",
+        "  return tf.where(x % 2 == 0, x // 2, 3 * x + 1)\n",
+        "\n",
+        "print(next_collatz(tf.constant([1, 2])))\n",
+        "# You specified a 1-D tensor in the input signature, so this should fail.\n",
+        "with assert_raises(TypeError):\n",
+        "  next_collatz(tf.constant([[1, 2], [3, 4]]))\n",
+        "\n",
+        "# You specified an int32 dtype in the input signature, so this should fail.\n",
+        "with assert_raises(TypeError):\n",
+        "  next_collatz(tf.constant([1.0, 2.0]))\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "rvJXCfk8VkLf"
+        "id": "ocxX-HVk7P2o"
       },
       "source": [
-        "Here's an example of more complicated control flow:"
+        "#### Use unknown dimensions for flexibility\n",
+        "\n",
+        "  Since TensorFlow matches tensors based on their shape, using a `None` dimension as a wildcard will allow `tf.function`s to reuse traces for variably-sized input. Variably-sized input can occur if you have sequences of different length, or images of different sizes for each batch. You can check out the [Transformer](https://www.tensorflow.org/text/tutorials/transformer) and [Deep Dream](../tutorials/generative/deepdream.ipynb) tutorials for examples."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "h-Z87IJqVlKl"
+        "id": "4Viun7dh7PmF"
       },
       "outputs": [],
       "source": [
-        "@tf.function\n",
-        "def fizzbuzz(n):\n",
-        "  for i in tf.range(n):\n",
-        "    if i % 3 == 0:\n",
-        "      tf.print('Fizz')\n",
-        "    elif i % 5 == 0:\n",
-        "      tf.print('Buzz')\n",
-        "    else:\n",
-        "      tf.print(i)\n",
+        "@tf.function(input_signature=(tf.TensorSpec(shape=[None], dtype=tf.int32),))\n",
+        "def g(x):\n",
+        "  print('Tracing with', x)\n",
+        "  return x\n",
         "\n",
-        "fizzbuzz(tf.constant(15))"
+        "# No retrace!\n",
+        "print(g(tf.constant([1, 2, 3])))\n",
+        "print(g(tf.constant([1, 2, 3, 4, 5])))\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "h_Y4uC1R1B55"
+        "id": "37cc12f93cbd"
       },
       "source": [
-        "## Keras and AutoGraph\n",
+        "#### Use `reduce_retracing` for automatic flexibility\n",
         "\n",
-        "AutoGraph is available by default in non-dynamic Keras models. For more information, see `tf.keras`."
+        "When `reduce_retracing` is enabled, `tf.function` automatically identifies supertypes of the input types it is observing and chooses to trace more generalized graphs automatically. It is less efficient than setting the `input_signature` directly but useful when many types need to be supported."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "cR6mpLKP1HLe"
+        "id": "0403fae03a1f"
       },
       "outputs": [],
       "source": [
-        "class CustomModel(tf.keras.models.Model):\n",
-        "\n",
-        "  @tf.function\n",
-        "  def call(self, input_data):\n",
-        "    if tf.reduce_mean(input_data) \u003e 0:\n",
-        "      return input_data\n",
-        "    else:\n",
-        "      return input_data // 2\n",
+        "@tf.function(reduce_retracing=True)\n",
+        "def g(x):\n",
+        "  print('Tracing with', x)\n",
+        "  return x\n",
         "\n",
+        "# Traces once.\n",
+        "print(g(tf.constant([1, 2, 3])))\n",
         "\n",
-        "model = CustomModel()\n",
+        "# Traces again, but more generalized this time.\n",
+        "print(g(tf.constant([1, 2, 3, 4, 5])))\n",
         "\n",
-        "model(tf.constant([-2, -4]))"
+        "# No more tracing!\n",
+        "print(g(tf.constant([1, 2, 3, 4, 5, 6, 7])))\n",
+        "print(g(tf.constant([1, 2, 3, 4, 5, 6, 7, 8, 9])))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "NTEvpBK9f8kj"
+        "id": "AY5oiQN0XIyA"
       },
       "source": [
-        "## Side effects\n",
+        "#### Pass tensors instead of python literals\n",
         "\n",
-        "Just like in eager mode, you can use operations with side effects, like `tf.assign` or `tf.print` normally inside `tf.function`, and it will insert the necessary control dependencies to ensure they execute in order."
+        "  Often, Python arguments are used to control hyperparameters and graph constructions - for example, `num_layers=10` or `training=True` or `nonlinearity='relu'`. So, if the Python argument changes, it makes sense that you'd have to retrace the graph.\n",
+        "\n",
+        "  However, it's possible that a Python argument is not being used to control graph construction. In these cases, a change in the Python value can trigger needless retracing. Take, for example, this training loop, which AutoGraph will dynamically unroll. Despite the multiple traces, the generated graph is actually identical, so retracing is unnecessary."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-Wd6i8S9gcuC"
+        "id": "uydzR5JYUU8H"
       },
       "outputs": [],
       "source": [
-        "v = tf.Variable(5)\n",
+        "def train_one_step():\n",
+        "  pass\n",
         "\n",
         "@tf.function\n",
-        "def find_next_odd():\n",
-        "  v.assign(v + 1)\n",
-        "  if v % 2 == 0:\n",
-        "    v.assign(v + 1)\n",
-        "\n",
+        "def train(num_steps):\n",
+        "  print(\"Tracing with num_steps = \", num_steps)\n",
+        "  tf.print(\"Executing with num_steps = \", num_steps)\n",
+        "  for _ in tf.range(num_steps):\n",
+        "    train_one_step()\n",
+        "\n",
+        "print(\"Retracing occurs for different Python arguments.\")\n",
+        "train(num_steps=10)\n",
+        "train(num_steps=20)\n",
+        "\n",
+        "print()\n",
+        "print(\"Traces are reused for Tensor arguments.\")\n",
+        "train(num_steps=tf.constant(10))\n",
+        "train(num_steps=tf.constant(20))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4pJqkDR_Q2wz"
+      },
+      "source": [
+        "If you need to force retracing, create a new `tf.function`. Separate `tf.function` objects are guaranteed not to share traces."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uHp4ousu4DdN"
+      },
+      "outputs": [],
+      "source": [
+        "def f():\n",
+        "  print('Tracing!')\n",
+        "  tf.print('Executing')\n",
         "\n",
-        "find_next_odd()\n",
-        "v"
+        "tf.function(f)()\n",
+        "tf.function(f)()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "4LfnJjm0Bm0B"
+        "id": "-tZoWrA6INvc"
       },
       "source": [
-        "\u003ca id=\"debugging\"\u003e\u003c/a\u003e\n",
+        "#### Use the tracing protocol\n",
         "\n",
-        "## Debugging\n",
+        "Where possible, you should prefer converting the Python type into a `tf.experimental.ExtensionType` instead. Moreover, the `TraceType` of an `ExtensionType` is the `tf.TypeSpec` associated with it. Therefore, if needed, you can simply override the default `tf.TypeSpec` to take control of an `ExtensionType`'s `Tracing Protocol`. Refer to the _Customizing the ExtensionType's TypeSpec_ section in the [Extension types](extension_type.ipynb) guide for details.\n",
         "\n",
-        "`tf.function` and AutoGraph work by generating code and tracing it into TensorFlow graphs. This mechanism does not yet support step-by-step debuggers like `pdb`. However, you can call `tf.config.experimental_run_functions_eagerly(True)` to temporarily enable eager execution inside the `tf.function' and use your favorite debugger:"
+        "Otherwise, for direct control over when `tf.function` should retrace in regards to a particular Python type, you can implement the `Tracing Protocol` for it yourself."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Yci8ve6hmgpF"
+        "id": "gZkIh7UaIKc6"
       },
       "outputs": [],
       "source": [
         "@tf.function\n",
-        "def f(x):\n",
-        "  if x \u003e 0:\n",
-        "    # Try setting a breakpoint here!\n",
-        "    # Example:\n",
-        "    #   import pdb\n",
-        "    #   pdb.set_trace()\n",
-        "    x = x + 1\n",
-        "  return x\n",
+        "def get_mixed_flavor(fruit_a, fruit_b):\n",
+        "  return fruit_a.flavor + fruit_b.flavor\n",
+        "\n",
+        "class Fruit:\n",
+        "  flavor = tf.constant([0, 0])\n",
         "\n",
-        "tf.config.experimental_run_functions_eagerly(True)\n",
+        "class Apple(Fruit):\n",
+        "  flavor = tf.constant([1, 2])\n",
         "\n",
-        "# You can now set breakpoints and run the code in a debugger.\n",
-        "f(tf.constant(1))\n",
+        "class Mango(Fruit):\n",
+        "  flavor = tf.constant([3, 4])\n",
         "\n",
-        "tf.config.experimental_run_functions_eagerly(False)"
+        "# As described in the above rules, a generic TraceType for `Apple` and `Mango`\n",
+        "# is generated (and a corresponding ConcreteFunction is traced) but it fails to\n",
+        "# match the second function call since the first pair of Apple() and Mango()\n",
+        "# have gone out out of scope by then and deleted.\n",
+        "get_mixed_flavor(Apple(), Mango()) # Traces a new concrete function\n",
+        "get_mixed_flavor(Apple(), Mango()) # Traces a new concrete function again\n",
+        "\n",
+        "# However, each subclass of the `Fruit` class has a fixed flavor, and you\n",
+        "# can reuse an existing traced concrete function if it was the same\n",
+        "# subclass. Avoiding such unnecessary tracing of concrete functions\n",
+        "# can have significant performance benefits.\n",
+        "\n",
+        "class FruitTraceType(tf.types.experimental.TraceType):\n",
+        "  def __init__(self, fruit):\n",
+        "    self.fruit_type = type(fruit)\n",
+        "    self.fruit_value = fruit\n",
+        "\n",
+        "  def is_subtype_of(self, other):\n",
+        "      # True if self subtypes `other` and `other`'s type matches FruitTraceType.\n",
+        "      return (type(other) is FruitTraceType and\n",
+        "              self.fruit_type is other.fruit_type)\n",
+        "\n",
+        "  def most_specific_common_supertype(self, others):\n",
+        "      # `self` is the specific common supertype if all input types match it.\n",
+        "      return self if all(self == other for other in others) else None\n",
+        "\n",
+        "  def placeholder_value(self, placeholder_context=None):\n",
+        "      # Use the fruit itself instead of the type for correct tracing.\n",
+        "      return self.fruit_value\n",
+        "\n",
+        "  def __eq__(self, other):\n",
+        "    return type(other) is FruitTraceType and self.fruit_type == other.fruit_type\n",
+        "\n",
+        "  def __hash__(self):\n",
+        "    return hash(self.fruit_type)\n",
+        "\n",
+        "class FruitWithTraceType:\n",
+        "\n",
+        "  def __tf_tracing_type__(self, context):\n",
+        "    return FruitTraceType(self)\n",
+        "\n",
+        "class AppleWithTraceType(FruitWithTraceType):\n",
+        "  flavor = tf.constant([1, 2])\n",
+        "\n",
+        "class MangoWithTraceType(FruitWithTraceType):\n",
+        "  flavor = tf.constant([3, 4])\n",
+        "\n",
+        "# Now if you try calling it again:\n",
+        "get_mixed_flavor(AppleWithTraceType(), MangoWithTraceType()) # Traces a new concrete function\n",
+        "get_mixed_flavor(AppleWithTraceType(), MangoWithTraceType()) # Re-uses the traced concrete function"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "NfpIQUv28Ht4"
+        "id": "96IxS2WR37fF"
       },
       "source": [
-        "## Advanced example: An in-graph training loop\n",
+        "### Obtaining concrete functions\n",
         "\n",
-        "The previous section showed that AutoGraph can be used inside Keras layers and models. Keras models can also be used in AutoGraph code.\n",
-        "\n",
-        "This example shows how to train a simple Keras model on MNIST with the entire training process—loading batches, calculating gradients, updating parameters, calculating validation accuracy, and repeating until convergence—is performed in-graph."
+        "Every time a function is traced, a new concrete function is created. You can directly obtain a concrete function, by using `get_concrete_function`.\n"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "Em5dzSUOtLRP"
+        "id": "mHg2CGtPQ3Hz"
       },
+      "outputs": [],
       "source": [
-        "### Download data"
+        "print(\"Obtaining concrete trace\")\n",
+        "double_strings = double.get_concrete_function(tf.constant(\"a\"))\n",
+        "print(\"Executing traced function\")\n",
+        "print(double_strings(tf.constant(\"a\")))\n",
+        "print(double_strings(a=tf.constant(\"b\")))\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "xqoxumv0ssQW"
+        "id": "6IVZ-NVf9vsx"
       },
       "outputs": [],
       "source": [
-        "def prepare_mnist_features_and_labels(x, y):\n",
-        "  x = tf.cast(x, tf.float32) / 255.0\n",
-        "  y = tf.cast(y, tf.int64)\n",
-        "  return x, y\n",
-        "\n",
-        "def mnist_dataset():\n",
-        "  (x, y), _ = tf.keras.datasets.mnist.load_data()\n",
-        "  ds = tf.data.Dataset.from_tensor_slices((x, y))\n",
-        "  ds = ds.map(prepare_mnist_features_and_labels)\n",
-        "  ds = ds.take(20000).shuffle(20000).batch(100)\n",
-        "  return ds\n",
-        "\n",
-        "train_dataset = mnist_dataset()"
+        "# You can also call get_concrete_function on an InputSpec\n",
+        "double_strings_from_inputspec = double.get_concrete_function(tf.TensorSpec(shape=[], dtype=tf.string))\n",
+        "print(double_strings_from_inputspec(tf.constant(\"c\")))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "znmy4l8ntMvW"
+        "id": "iR4fVmG34xvF"
       },
       "source": [
-        "### Define the model"
+        "Printing a `ConcreteFunction` displays a summary of its input arguments (with types) and its output type."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ltxyJVWTqNAO"
+        "id": "o3-JbkIk41r8"
       },
       "outputs": [],
       "source": [
-        "model = tf.keras.Sequential((\n",
-        "    tf.keras.layers.Reshape(target_shape=(28 * 28,), input_shape=(28, 28)),\n",
-        "    tf.keras.layers.Dense(100, activation='relu'),\n",
-        "    tf.keras.layers.Dense(100, activation='relu'),\n",
-        "    tf.keras.layers.Dense(10)))\n",
-        "model.build()\n",
-        "optimizer = tf.keras.optimizers.Adam()"
+        "print(double_strings)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "oeYV6mKnJGMr"
+        "id": "QtqfvljZeuOV"
       },
       "source": [
-        "### Define the training loop"
+        "You can also directly retrieve a concrete function's signature."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "3xtg_MMhJETd"
+        "id": "nzbrqFABe0zG"
       },
       "outputs": [],
       "source": [
-        "compute_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
-        "\n",
-        "compute_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n",
-        "\n",
-        "\n",
-        "def train_one_step(model, optimizer, x, y):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    logits = model(x)\n",
-        "    loss = compute_loss(y, logits)\n",
-        "\n",
-        "  grads = tape.gradient(loss, model.trainable_variables)\n",
-        "  optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
-        "\n",
-        "  compute_accuracy(y, logits)\n",
-        "  return loss\n",
-        "\n",
-        "\n",
-        "@tf.function\n",
-        "def train(model, optimizer):\n",
-        "  train_ds = mnist_dataset()\n",
-        "  step = 0\n",
-        "  loss = 0.0\n",
-        "  accuracy = 0.0\n",
-        "  for x, y in train_ds:\n",
-        "    step += 1\n",
-        "    loss = train_one_step(model, optimizer, x, y)\n",
-        "    if step % 10 == 0:\n",
-        "      tf.print('Step', step, ': loss', loss, '; accuracy', compute_accuracy.result())\n",
-        "  return step, loss, accuracy\n",
-        "\n",
-        "step, loss, accuracy = train(model, optimizer)\n",
-        "print('Final step', step, ': loss', loss, '; accuracy', compute_accuracy.result())"
+        "print(double_strings.function_type)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "SnsumiP6eRYL"
+        "id": "lar5A_5m5IG1"
       },
       "source": [
-        "## Batching\n",
-        "\n",
-        "In real applications batching is essential for performance. The best code to convert to AutoGraph is code where the control flow is decided at the _batch_ level. If making decisions at the individual _example_ level, try to use batch APIs to maintain performance.\n",
-        "\n",
-        "For example, if you have the following code in Python:\n"
+        "Using a concrete trace with incompatible types will throw an error"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "t31QoERiNccJ"
+        "id": "G5eeTK-T5KYj"
       },
       "outputs": [],
       "source": [
-        "def square_if_positive(x):\n",
-        "  return [i ** 2 if i \u003e 0 else i for i in x]\n",
-        "\n",
-        "\n",
-        "square_if_positive(range(-5, 5))"
+        "with assert_raises(tf.errors.InvalidArgumentError):\n",
+        "  double_strings(tf.constant(1))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "kSeEJ76uNgwD"
+        "id": "st2L9VNQVtSG"
       },
       "source": [
-        "You may be tempted to write it in TensorFlow as such (and this would work!):\n"
+        "You may notice that Python arguments are given special treatment in a concrete function's input signature. Prior to TensorFlow 2.3, Python arguments were simply removed from the concrete function's signature. Starting with TensorFlow 2.3, Python arguments remain in the signature, but are constrained to take the value set during tracing."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "RqR8WzSzNf87"
+        "id": "U_QyPSGoaC35"
       },
       "outputs": [],
       "source": [
         "@tf.function\n",
-        "def square_if_positive_naive(x):\n",
-        "  result = tf.TensorArray(tf.int32, size=x.shape[0])\n",
-        "  for i in tf.range(x.shape[0]):\n",
-        "    if x[i] \u003e 0:\n",
-        "      result = result.write(i, x[i] ** 2)\n",
-        "    else:\n",
-        "      result = result.write(i, x[i])\n",
-        "  return result.stack()\n",
+        "def pow(a, b):\n",
+        "  return a ** b\n",
+        "\n",
+        "square = pow.get_concrete_function(a=tf.TensorSpec(None, tf.float32), b=2)\n",
+        "print(square)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E76vIDhQbXIb"
+      },
+      "outputs": [],
+      "source": [
+        "assert square(tf.constant(10.0)) == 100\n",
         "\n",
+        "with assert_raises(TypeError):\n",
+        "  square(tf.constant(10.0), b=3)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "41gJh_JGIfuA"
+      },
+      "source": [
+        "### Obtaining graphs\n",
         "\n",
-        "square_if_positive_naive(tf.range(-5, 5))"
+        "Although retrieving the actual `tf.Graph` object is not something you'll normally need to do, you can obtain it easily from any concrete function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5UENeGHfaX8g"
+      },
+      "outputs": [],
+      "source": [
+        "graph = double_strings.graph\n",
+        "for node in graph.as_graph_def().node:\n",
+        "  print(f'{node.input} -> {node.name}')\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "gTcyWXVGN3gS"
+        "id": "2d49c486ccd4"
       },
       "source": [
-        "But in this case, it turns out you can write the following:\n"
+        "In reality, `tf.Graph`s are not directly callable. We actually use an `tf.types.experimental.AtomicFunction` to perform the computations described by the `tf.Graph`. You can access the `AtomicFunction` describing the traced `tf.Graph` and call it directly instead of the `ConcreteFunction`:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "VO2f6x-lNfVj"
+        "id": "4c3879aa0be0"
       },
       "outputs": [],
       "source": [
-        "def square_if_positive_vectorized(x):\n",
-        "  return tf.where(x \u003e 0, x ** 2, x)\n",
+        "atomic_fn = double_strings.inference_fn\n",
+        "atomic_fn(tf.constant(\"a\"))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c3bd1036c18c"
+      },
+      "source": [
+        "This has the advantage of having lower Python overhead for high-performance scenarios. But it should only be used for forward inference (no gradient support), and captured tensor values (if any) would need to be explicitly supplied."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aIKkgr6qdtp4"
+      },
+      "source": [
+        "### Debugging\n",
         "\n",
+        "In general, debugging code is easier in eager mode than inside `tf.function`. You should ensure that your code executes error-free in eager mode before decorating with `tf.function`. To assist in the debugging process, you can call `tf.config.run_functions_eagerly(True)` to globally disable and reenable `tf.function`.\n",
         "\n",
-        "square_if_positive_vectorized(tf.range(-5, 5))"
+        "When tracking down issues that only appear within `tf.function`, here are some tips:\n",
+        "- Plain old Python `print` calls only execute during tracing, helping you track down when your function gets (re)traced.\n",
+        "- `tf.print` calls will execute every time, and can help you track down intermediate values during execution.\n",
+        "- `tf.debugging.enable_check_numerics` is an easy way to track down where NaNs and Inf are created.\n",
+        "- `pdb` (the [Python debugger](https://docs.python.org/3/library/pdb.html)) can help you understand what's going on during tracing. (Caveat: `pdb` will drop you into AutoGraph-transformed source code.)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "EXFVx-cJ57_F"
+        "id": "5f05Vr_YBUCz"
       },
       "source": [
-        "## Re-tracing\n",
-        "\n",
-        "Key points:\n",
+        "## AutoGraph transformations\n",
         "\n",
-        "* Exercise caution when calling functions with non-tensor arguments, or with arguments that change shapes.\n",
-        "* Decorate module-level functions, and methods of module-level classes, and avoid decorating local functions or methods.\n",
+        "AutoGraph is a library that is on by default in `tf.function`, and transforms a subset of Python eager code into graph-compatible TensorFlow ops. This includes control flow like `if`, `for`, `while`.\n",
         "\n",
-        "`tf.function` can give you significant speedup over eager execution, at the cost of a slower first-time execution. This is because when executed for the first time, the function is also *traced* into a TensorFlow graph. Constructing and optimizing a graph is usually much slower compared to actually executing it:"
+        "TensorFlow ops like `tf.cond` and `tf.while_loop` continue to work, but control flow is often easier to write and understand when written in Python."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "iahT-4wT6vlA"
+        "id": "yCQTtTPTW3WF"
       },
       "outputs": [],
       "source": [
-        "import timeit\n",
-        "\n",
+        "# A simple loop\n",
         "\n",
         "@tf.function\n",
-        "def f(x, y):\n",
-        "  return tf.matmul(x, y)\n",
-        "\n",
-        "print(\n",
-        "    \"First invocation:\",\n",
-        "    timeit.timeit(lambda: f(tf.ones((10, 10)), tf.ones((10, 10))), number=1))\n",
+        "def f(x):\n",
+        "  while tf.reduce_sum(x) > 1:\n",
+        "    tf.print(x)\n",
+        "    x = tf.tanh(x)\n",
+        "  return x\n",
         "\n",
-        "print(\n",
-        "    \"Second invocation:\",\n",
-        "    timeit.timeit(lambda: f(tf.ones((10, 10)), tf.ones((10, 10))), number=1))"
+        "f(tf.random.uniform([5]))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "q0Wojo2Z7hKg"
+        "id": "KxwJ8znPI0Cg"
       },
       "source": [
-        "You can easily tell when a function is traced by adding a `print` statement to the top of the function. Because any Python code is only executed at trace time, you will only see the output of `print` when the function is traced:"
+        "If you're curious you can inspect the code AutoGraph generates."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "2IHE7-jT7gZs"
+        "id": "jlQD1ffRXJhl"
       },
       "outputs": [],
       "source": [
-        "@tf.function\n",
-        "def f():\n",
-        "  print('Tracing!')\n",
-        "  tf.print('Executing')\n",
-        "\n",
-        "print('First invocation:')\n",
-        "f()\n",
-        "\n",
-        "print('Second invocation:')\n",
-        "f()"
+        "print(tf.autograph.to_code(f.python_function))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "J2VBoQC58PdU"
+        "id": "xgKmkrNTZSyz"
       },
       "source": [
-        "`tf.function` may also *re-trace* when called with different non-tensor arguments:"
+        "### Conditionals\n",
+        "\n",
+        "AutoGraph will convert some `if ` statements into the equivalent `tf.cond` calls. This substitution is made if `` is a Tensor. Otherwise, the `if` statement is executed as a Python conditional.\n",
+        "\n",
+        "A Python conditional executes during tracing, so exactly one branch of the conditional will be added to the graph. Without AutoGraph, this traced graph would be unable to take the alternate branch if there is data-dependent control flow.\n",
+        "\n",
+        "`tf.cond` traces and adds both branches of the conditional to the graph, dynamically selecting a branch at execution time. Tracing can have unintended side effects; check out [AutoGraph tracing effects](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/control_flow.md#effects-of-the-tracing-process) for more information."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-c6VUwrz808l"
+        "id": "BOQl8PMq2Sf3"
       },
       "outputs": [],
       "source": [
         "@tf.function\n",
-        "def f(n):\n",
-        "  print(n, 'Tracing!')\n",
-        "  tf.print(n, 'Executing')\n",
-        "\n",
-        "f(1)\n",
-        "f(1)\n",
+        "def fizzbuzz(n):\n",
+        "  for i in tf.range(1, n + 1):\n",
+        "    print('Tracing for loop')\n",
+        "    if i % 15 == 0:\n",
+        "      print('Tracing fizzbuzz branch')\n",
+        "      tf.print('fizzbuzz')\n",
+        "    elif i % 3 == 0:\n",
+        "      print('Tracing fizz branch')\n",
+        "      tf.print('fizz')\n",
+        "    elif i % 5 == 0:\n",
+        "      print('Tracing buzz branch')\n",
+        "      tf.print('buzz')\n",
+        "    else:\n",
+        "      print('Tracing default branch')\n",
+        "      tf.print(i)\n",
         "\n",
-        "f(2)\n",
-        "f(2)"
+        "fizzbuzz(tf.constant(5))\n",
+        "fizzbuzz(tf.constant(20))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "aKOrjBLCE8cy"
+        "id": "4rBO5AQ15HVC"
       },
       "source": [
-        "A *re-trace* can also happen when tensor arguments change shape, unless you specified an `input_signature`:"
+        "See the [reference documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/control_flow.md#if-statements) for additional restrictions on AutoGraph-converted if statements."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "txhtkn0rE8dH"
+        "id": "yho4J0a0ZkQS"
       },
-      "outputs": [],
       "source": [
-        "@tf.function\n",
-        "def f(x):\n",
-        "  print(x.shape, 'Tracing!')\n",
-        "  tf.print(x, 'Executing')\n",
+        "### Loops\n",
+        "\n",
+        "AutoGraph will convert some `for` and `while` statements into the equivalent TensorFlow looping ops, like `tf.while_loop`. If not converted, the `for` or `while` loop is executed as a Python loop.\n",
+        "\n",
+        "This substitution is made in the following situations:\n",
         "\n",
-        "f(tf.constant([1]))\n",
-        "f(tf.constant([2]))\n",
+        "- `for x in y`: if `y` is a Tensor, convert to `tf.while_loop`. In the special case where `y` is a `tf.data.Dataset`, a combination of `tf.data.Dataset` ops are generated.\n",
+        "- `while `: if `` is a Tensor, convert to `tf.while_loop`.\n",
         "\n",
-        "f(tf.constant([1, 2]))\n",
-        "f(tf.constant([3, 4]))"
+        "A Python loop executes during tracing, adding additional ops to the `tf.Graph` for every iteration of the loop.\n",
+        "\n",
+        "A TensorFlow loop traces the body of the loop, and dynamically selects how many iterations to run at execution time.  The loop body only appears once in the generated `tf.Graph`.\n",
+        "\n",
+        "See the [reference documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/control_flow.md#while-statements) for additional restrictions on AutoGraph-converted `for` and `while` statements."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "sdN40ZqT9XaG"
+        "id": "sp4rbIdfbM6s"
       },
       "source": [
-        "In addition, tf.function always creates a new graph function with its own set of traces whenever it is called:"
+        "#### Looping over Python data\n",
+        "\n",
+        "A common pitfall is to loop over Python/NumPy data within a `tf.function`. This loop will execute during the tracing process, adding a copy of your model to the `tf.Graph` for each iteration of the loop.\n",
+        "\n",
+        "If you want to wrap the entire training loop in `tf.function`, the safest way to do this is to wrap your data as a `tf.data.Dataset` so that AutoGraph will dynamically unroll the training loop."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "GT1iBa5i9enE"
+        "id": "WGZ19LspbZ27"
       },
       "outputs": [],
       "source": [
-        "def f():\n",
-        "  print('Tracing!')\n",
-        "  tf.print('Executing')\n",
+        "def measure_graph_size(f, *args):\n",
+        "  g = f.get_concrete_function(*args).graph\n",
+        "  print(\"{}({}) contains {} nodes in its graph\".format(\n",
+        "      f.__name__, ', '.join(map(str, args)), len(g.as_graph_def().node)))\n",
         "\n",
-        "tf.function(f)()\n",
-        "tf.function(f)()"
+        "@tf.function\n",
+        "def train(dataset):\n",
+        "  loss = tf.constant(0)\n",
+        "  for x, y in dataset:\n",
+        "    loss += tf.abs(y - x) # Some dummy computation.\n",
+        "  return loss\n",
+        "\n",
+        "small_data = [(1, 1)] * 3\n",
+        "big_data = [(1, 1)] * 10\n",
+        "measure_graph_size(train, small_data)\n",
+        "measure_graph_size(train, big_data)\n",
+        "\n",
+        "measure_graph_size(train, tf.data.Dataset.from_generator(\n",
+        "    lambda: small_data, (tf.int32, tf.int32)))\n",
+        "measure_graph_size(train, tf.data.Dataset.from_generator(\n",
+        "    lambda: big_data, (tf.int32, tf.int32)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JeD2U-yrbfVb"
+      },
+      "source": [
+        "When wrapping Python/NumPy data in a Dataset, be mindful of `tf.data.Dataset.from_generator` versus ` tf.data.Dataset.from_tensor_slices`. The former will keep the data in Python and fetch it via `tf.py_function` which can have performance implications, whereas the latter will bundle a copy of the data as one large `tf.constant()` node in the graph, which can have memory implications.\n",
+        "\n",
+        "Reading data from files via `TFRecordDataset`, `CsvDataset`, etc. is the most effective way to consume data, as then TensorFlow itself can manage the asynchronous loading and prefetching of data, without having to involve Python. To learn more, see the [`tf.data`: Build TensorFlow input pipelines](data.ipynb) guide."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "n7_JDzFK9nnC"
+        "id": "hyksHW9TCukR"
       },
       "source": [
-        "This can lead to surprising behavior when using the `@tf.function` decorator in a nested function:"
+        "#### Accumulating values in a loop\n",
+        "\n",
+        "A common pattern is to accumulate intermediate values from a loop. Normally, this is accomplished by appending to a Python list or adding entries to a Python dictionary. However, as these are Python side effects, they will not work as expected in a dynamically unrolled loop. Use `tf.TensorArray` to accumulate results from a dynamically unrolled loop."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "P3pBG7Uf9u4g"
+        "id": "HJ3Vb3dXfefN"
       },
       "outputs": [],
       "source": [
-        "def outer():\n",
-        "  @tf.function\n",
-        "  def f():\n",
-        "    print('Tracing!')\n",
-        "    tf.print('Executing')\n",
-        "  f()\n",
+        "batch_size = 2\n",
+        "seq_len = 3\n",
+        "feature_size = 4\n",
+        "\n",
+        "def rnn_step(inp, state):\n",
+        "  return inp + state\n",
+        "\n",
+        "@tf.function\n",
+        "def dynamic_rnn(rnn_step, input_data, initial_state):\n",
+        "  # [batch, time, features] -> [time, batch, features]\n",
+        "  input_data = tf.transpose(input_data, [1, 0, 2])\n",
+        "  max_seq_len = input_data.shape[0]\n",
+        "\n",
+        "  states = tf.TensorArray(tf.float32, size=max_seq_len)\n",
+        "  state = initial_state\n",
+        "  for i in tf.range(max_seq_len):\n",
+        "    state = rnn_step(input_data[i], state)\n",
+        "    states = states.write(i, state)\n",
+        "  return tf.transpose(states.stack(), [1, 0, 2])\n",
+        "\n",
+        "dynamic_rnn(rnn_step,\n",
+        "            tf.random.uniform([batch_size, seq_len, feature_size]),\n",
+        "            tf.zeros([batch_size, feature_size]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i2MVoIVaNApG"
+      },
+      "source": [
+        "## Limitations\n",
+        "\n",
+        "`tf.function` has a few limitations by design that you should be aware of when converting a Python function to a `tf.function`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EJqHGFSVLIKl"
+      },
+      "source": [
+        "### Executing Python side effects\n",
+        "\n",
+        "Side effects, like printing, appending to lists, and mutating globals, can behave unexpectedly inside a `tf.function`, sometimes executing twice or not all. They only happen the first time you call a `tf.function` with a set of inputs. Afterwards, the traced `tf.Graph` is reexecuted, without executing the Python code.\n",
+        "\n",
+        "The general rule of thumb is to avoid relying on Python side effects in your logic and only use them to debug your traces. Otherwise, TensorFlow APIs like `tf.data`, `tf.print`, `tf.summary`, `tf.Variable.assign`, and `tf.TensorArray` are the best way to ensure your code will be executed by the TensorFlow runtime with each call."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "w2sACuZ9TTRk"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def f(x):\n",
+        "  print(\"Traced with\", x)\n",
+        "  tf.print(\"Executed with\", x)\n",
+        "\n",
+        "f(1)\n",
+        "f(1)\n",
+        "f(2)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e1I0dPiqTV8H"
+      },
+      "source": [
+        "If you would like to execute Python code during each invocation of a `tf.function`, `tf. py_function` is an exit hatch. The drawbacks of `tf.py_function` are that it's not portable or particularly performant, cannot be saved with `SavedModel`, and does not work well in distributed (multi-GPU, TPU) setups. Also, since `tf.py_function` has to be wired into the graph, it casts all inputs/outputs to tensors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZbI7XA_e6yA2"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.py_function(Tout=tf.float32)\n",
+        "def py_plus(x, y):\n",
+        "  print('Executing eagerly.')\n",
+        "  return x + y\n",
+        "\n",
+        "@tf.function\n",
+        "def tf_wrapper(x, y):\n",
+        "  print('Tracing.')\n",
+        "  return py_plus(x, y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h5ttN_sI7TdQ"
+      },
+      "source": [
+        "The `tf.function` will trace the first time:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mAK4XINl7Ldy"
+      },
+      "outputs": [],
+      "source": [
+        "tf_wrapper(tf.constant(1.0), tf.constant(2.0)).numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Atxvrd_o7dSy"
+      },
+      "source": [
+        "But the `tf.py_function` inside executes eagerly every time:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vv7qTiTU7bjy"
+      },
+      "outputs": [],
+      "source": [
+        "tf_wrapper(tf.constant(1.0), tf.constant(2.0)).numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bOW1v9WVKGgH"
+      },
+      "source": [
+        "#### Changing Python global and free variables\n",
+        "\n",
+        "Changing Python global and [free variables](https://docs.python.org/3/reference/executionmodel.html#binding-of-names) counts as a Python side effect, so it only happens during tracing.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7aJD--9qTWmg"
+      },
+      "outputs": [],
+      "source": [
+        "external_list = []\n",
+        "\n",
+        "@tf.function\n",
+        "def side_effect(x):\n",
+        "  print('Python side effect')\n",
+        "  external_list.append(x)\n",
+        "\n",
+        "side_effect(1)\n",
+        "side_effect(1)\n",
+        "side_effect(1)\n",
+        "# The list append only happened once!\n",
+        "assert len(external_list) == 1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5eZTFRv_k_nR"
+      },
+      "source": [
+        "Sometimes unexpected behaviors are very hard to notice. In the example below, the `counter` is intended to safeguard the increment of a variable. However because it is a python integer and not a TensorFlow object, it's value is captured during the first trace. When the `tf.function` is used, the `assign_add` will be recorded unconditionally in the underlying graph. Therefore `v` will increase by 1, every time the `tf.function` is called. This issue is common among users that try to migrate their Graph-mode Tensorflow code to Tensorflow 2 using `tf.function` decorators, when python side-effects (the `counter` in the example) are used to determine what ops to run (`assign_add` in the example). Usually, users realize this only after seeing suspicious numerical results, or significantly lower performance than expected (e.g. if the guarded operation is very costly)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5r6p7-9jk_3L"
+      },
+      "outputs": [],
+      "source": [
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.v = tf.Variable(0)\n",
+        "    self.counter = 0\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    if self.counter == 0:\n",
+        "      # A python side-effect\n",
+        "      self.counter += 1\n",
+        "      self.v.assign_add(1)\n",
+        "\n",
+        "    return self.v\n",
+        "\n",
+        "m = Model()\n",
+        "for n in range(3):\n",
+        "  print(m().numpy()) # prints 1, 2, 3"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tXCTcHoVcxhX"
+      },
+      "source": [
+        "A workaround to achieve the expected behavior is using [`tf.init_scope`](https://www.tensorflow.org/api_docs/python/tf/init_scope) to lift the operations outside of the function graph. This ensures that the variable increment is only done once during tracing time. It should be noted `init_scope` has other side effects including cleared control flow and gradient tape. Sometimes the usage of `init_scope` can become too complex to manage realistically."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "An4MrIbrcvi8"
+      },
+      "outputs": [],
+      "source": [
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.v = tf.Variable(0)\n",
+        "    self.counter = 0\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    if self.counter == 0:\n",
+        "      # Lifts ops out of function-building graphs\n",
+        "      with tf.init_scope():\n",
+        "        self.counter += 1\n",
+        "        self.v.assign_add(1)\n",
+        "\n",
+        "    return self.v\n",
+        "\n",
+        "m = Model()\n",
+        "for n in range(3):\n",
+        "  print(m().numpy()) # prints 1, 1, 1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pbFG5CX4LwQA"
+      },
+      "source": [
+        "In summary, as a rule of thumb, you should avoid mutating python objects such as integers or containers like lists that live outside the `tf.function`. Instead, use arguments and TF objects. For example, the section [\"Accumulating values in a loop\"](#accumulating_values_in_a_loop) has one example of how list-like operations can be implemented.\n",
+        "\n",
+        "You can, in some cases, capture and manipulate state if it is a [`tf.Variable`](https://www.tensorflow.org/guide/variable). This is how the weights of Keras models are updated with repeated calls to the same `ConcreteFunction`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "X_oNNGrAqPJ1"
+      },
+      "source": [
+        "#### Using Python iterators and generators"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "msTmv-oyUNaf"
+      },
+      "source": [
+        "Many Python features, such as generators and iterators, rely on the Python runtime to keep track of state. In general, while these constructs work as expected in eager mode, they are examples of Python side effects and therefore only happen during tracing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FNPD4unZUedH"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def buggy_consume_next(iterator):\n",
+        "  tf.print(\"Value:\", next(iterator))\n",
+        "\n",
+        "iterator = iter([1, 2, 3])\n",
+        "buggy_consume_next(iterator)\n",
+        "# This reuses the first value from the iterator, rather than consuming the next value.\n",
+        "buggy_consume_next(iterator)\n",
+        "buggy_consume_next(iterator)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wcS3TAgCjTWR"
+      },
+      "source": [
+        "Just like how TensorFlow has a specialized `tf.TensorArray` for list constructs, it has a specialized `tf.data.Iterator` for iteration constructs. See the section on [AutoGraph transformations](#autograph_transformations) for an overview. Also, the [`tf.data`](https://www.tensorflow.org/guide/data) API can help implement generator patterns:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8D_iKetXW6VE"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def good_consume_next(iterator):\n",
+        "  # This is ok, iterator is a tf.data.Iterator\n",
+        "  tf.print(\"Value:\", next(iterator))\n",
+        "\n",
+        "ds = tf.data.Dataset.from_tensor_slices([1, 2, 3])\n",
+        "iterator = iter(ds)\n",
+        "good_consume_next(iterator)\n",
+        "good_consume_next(iterator)\n",
+        "good_consume_next(iterator)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i8YAMYb6KEh4"
+      },
+      "source": [
+        "### All outputs of a tf.function must be return values\n",
+        "\n",
+        "With the exception of `tf.Variable`s, a tf.function must return all its\n",
+        "outputs. Attempting to directly access any tensors from a function without\n",
+        "going through return values causes \"leaks\".\n",
+        "\n",
+        "For example, the function below \"leaks\" the tensor `a` through the Python\n",
+        "global `x`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zrdp4rjxg6jo"
+      },
+      "outputs": [],
+      "source": [
+        "x = None\n",
+        "\n",
+        "@tf.function\n",
+        "def leaky_function(a):\n",
+        "  global x\n",
+        "  x = a + 1  # Bad - leaks local tensor\n",
+        "  return a + 2\n",
+        "\n",
+        "correct_a = leaky_function(tf.constant(1))\n",
+        "\n",
+        "print(correct_a.numpy())  # Good - value obtained from function's returns\n",
+        "try:\n",
+        "  x.numpy()  # Bad - tensor leaked from inside the function, cannot be used here\n",
+        "except AttributeError as expected:\n",
+        "  print(expected)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-d4_J_DC5rxX"
+      },
+      "source": [
+        "This is true even if the leaked value is also returned:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PrcpPB8C5s9T"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def leaky_function(a):\n",
+        "  global x\n",
+        "  x = a + 1  # Bad - leaks local tensor\n",
+        "  return x  # Good - uses local tensor\n",
+        "\n",
+        "correct_a = leaky_function(tf.constant(1))\n",
+        "\n",
+        "print(correct_a.numpy())  # Good - value obtained from function's returns\n",
+        "try:\n",
+        "  x.numpy()  # Bad - tensor leaked from inside the function, cannot be used here\n",
+        "except AttributeError as expected:\n",
+        "  print(expected)\n",
+        "\n",
+        "@tf.function\n",
+        "def captures_leaked_tensor(b):\n",
+        "  b += x  # Bad - `x` is leaked from `leaky_function`\n",
+        "  return b\n",
+        "\n",
+        "with assert_raises(TypeError):\n",
+        "  captures_leaked_tensor(tf.constant(2))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Sm2ghjyy50D4"
+      },
+      "source": [
+        "Usually, leaks such as these occur when you use Python statements or data structures.\n",
+        "In addition to leaking inaccessible tensors, such statements are also likely wrong because they count as Python side effects, and are not guaranteed to execute at every function call.\n",
+        "\n",
+        "Common ways to leak local tensors also include mutating an external Python collection, or an object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D7bLe8y652wU"
+      },
+      "outputs": [],
+      "source": [
+        "class MyClass:\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    self.field = None\n",
+        "\n",
+        "external_list = []\n",
+        "external_object = MyClass()\n",
+        "\n",
+        "def leaky_function():\n",
+        "  a = tf.constant(1)\n",
+        "  external_list.append(a)  # Bad - leaks tensor\n",
+        "  external_object.field = a  # Bad - leaks tensor"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g-XVQcD-wf5K"
+      },
+      "source": [
+        "### Recursive tf.functions are not supported\n",
+        "\n",
+        "Recursive `tf.function`s are not supported and could cause infinite loops. For example,"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QSN-T1m5EFcR"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def recursive_fn(n):\n",
+        "  if n > 0:\n",
+        "    return recursive_fn(n - 1)\n",
+        "  else:\n",
+        "    return 1\n",
+        "\n",
+        "with assert_raises(Exception):\n",
+        "  recursive_fn(tf.constant(5))  # Bad - maximum recursion error."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LyRyooKGUxNV"
+      },
+      "source": [
+        "Even if a recursive `tf.function` seems to work, the Python function will be traced multiple times and could have performance implications. For example,"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7FlmTqfMUwmT"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def recursive_fn(n):\n",
+        "  if n > 0:\n",
+        "    print('tracing')\n",
+        "    return recursive_fn(n - 1)\n",
+        "  else:\n",
+        "    return 1\n",
+        "\n",
+        "recursive_fn(5)  # Warning - multiple tracings"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-D6nh3QirXAd"
+      },
+      "source": [
+        "## Known Issues\n",
+        "\n",
+        "If your `tf.function` is not evaluating correctly, the error may be explained by these known issues which are planned to be fixed in the future."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZoPg5w1Pjqna"
+      },
+      "source": [
+        "### Depending on Python global and free variables\n",
+        "\n",
+        "`tf.function` creates a new `ConcreteFunction` when called with a new value of a Python argument. However, it does not do that for the Python closure, globals, or nonlocals of that `tf.function`. If their value changes in between calls to the `tf.function`, the `tf.function` will still use the values they had when it was traced. This is different from how regular Python functions work.\n",
+        "\n",
+        "For that reason, you should follow a functional programming style that uses arguments instead of closing over outer names."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oeJMdXd3M0cM"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def buggy_add():\n",
+        "  return 1 + foo\n",
+        "\n",
+        "@tf.function\n",
+        "def recommended_add(foo):\n",
+        "  return 1 + foo\n",
+        "\n",
+        "foo = 1\n",
+        "print(\"Buggy:\", buggy_add())\n",
+        "print(\"Correct:\", recommended_add(foo))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "L3q7sUJWZOSU"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Updating the value of `foo` to 100!\")\n",
+        "foo = 100\n",
+        "print(\"Buggy:\", buggy_add())  # Did not change!\n",
+        "print(\"Correct:\", recommended_add(foo))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZoPg5w1Pjqnb"
+      },
+      "source": [
+        "Another way to update a global value is to make it a `tf.Variable` and use the `Variable.assign` method instead.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oeJMdXd3M0cc"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def variable_add():\n",
+        "  return 1 + foo\n",
+        "\n",
+        "foo = tf.Variable(1)\n",
+        "print(\"Variable:\", variable_add())\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "L3q7sUJWZOSd"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Updating the value of `foo` to 100!\")\n",
+        "foo.assign(100)\n",
+        "print(\"Variable:\", variable_add())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hvwe9gTIWfx6"
+      },
+      "source": [
+        "### Depending on Python objects"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BJkZS-SwPvOQ"
+      },
+      "source": [
+        "Passing custom Python objects as arguments to `tf.function` is supported but has certain limitations.\n",
+        "\n",
+        "For maximum feature coverage, consider transforming the objects into [Extension types](extension_type.ipynb) before passing them to `tf.function`. You can also use Python primitives and `tf.nest`-compatible structures.\n",
+        "\n",
+        "However, as covered in the [rules of tracing](#rules_of_tracing), when a custom `TraceType` is not provided by the custom Python class, `tf.function` is forced to use instance-based equality which means it will **not create a new trace** when you pass the **same object with modified attributes**."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ux8KJESVWDxX"
+      },
+      "outputs": [],
+      "source": [
+        "class SimpleModel(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    # These values are *not* tf.Variables.\n",
+        "    self.bias = 0.\n",
+        "    self.weight = 2.\n",
+        "\n",
+        "@tf.function\n",
+        "def evaluate(model, x):\n",
+        "  return model.weight * x + model.bias\n",
+        "\n",
+        "simple_model = SimpleModel()\n",
+        "x = tf.constant(10.)\n",
+        "print(evaluate(simple_model, x))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mUxRF4ghZZvX"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Adding bias!\")\n",
+        "simple_model.bias += 5.0\n",
+        "print(evaluate(simple_model, x))  # Didn't change :("
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ytcgg2qFWaBF"
+      },
+      "source": [
+        "Using the same `tf.function` to evaluate the modified instance of the model will be buggy since it still has the [same instance-based TraceType](#rules_of_tracing) as the original model.\n",
+        "\n",
+        "For that reason, you're recommended to write your `tf.function` to avoid depending on mutable object attributes or implement the [Tracing Protocol](#use_the_tracing_protocol) for the objects to inform `tf.function` about such attributes.\n",
+        "\n",
+        "If that is not possible, one workaround is to make new `tf.function`s each time you modify your object to force retracing:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pFvWmWAAQjrv"
+      },
+      "outputs": [],
+      "source": [
+        "def evaluate(model, x):\n",
+        "  return model.weight * x + model.bias\n",
+        "\n",
+        "new_model = SimpleModel()\n",
+        "evaluate_no_bias = tf.function(evaluate).get_concrete_function(new_model, x)\n",
+        "# Don't pass in `new_model`. `tf.function` already captured its state during tracing.\n",
+        "print(evaluate_no_bias(x))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bdU2-jF4ZH0B"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Adding bias!\")\n",
+        "new_model.bias += 5.0\n",
+        "# Create new `tf.function` and `ConcreteFunction` since you modified `new_model`.\n",
+        "evaluate_with_bias = tf.function(evaluate).get_concrete_function(new_model, x)\n",
+        "print(evaluate_with_bias(x)) # Don't pass in `new_model`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uFgEZClsZrEi"
+      },
+      "source": [
+        "As [retracing can be expensive](https://www.tensorflow.org/guide/intro_to_graphs#tracing_and_performance), you can use `tf.Variable`s as object attributes, which can be mutated (but not changed, careful!) for a similar effect without needing a retrace.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "daAP_lucwS6w"
+      },
+      "outputs": [],
+      "source": [
+        "class BetterModel:\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    self.bias = tf.Variable(0.)\n",
+        "    self.weight = tf.Variable(2.)\n",
+        "\n",
+        "@tf.function\n",
+        "def evaluate(model, x):\n",
+        "  return model.weight * x + model.bias\n",
+        "\n",
+        "better_model = BetterModel()\n",
+        "print(evaluate(better_model, x))\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ktqwMJBqwTFj"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Adding bias!\")\n",
+        "better_model.bias.assign_add(5.0)  # Note: instead of better_model.bias += 5\n",
+        "print(evaluate(better_model, x))  # This works!"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lPr_6mK_AQWL"
+      },
+      "source": [
+        "### Creating tf.Variables\n",
+        "\n",
+        "`tf.function` only supports singleton `tf.Variable`s created once on the first call, and reused across subsequent function calls. The code snippet below would create a new `tf.Variable` in every function call, which results in a `ValueError` exception.\n",
+        "\n",
+        "Example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Tx0Vvnb_9OB-"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def f(x):\n",
+        "  v = tf.Variable(1.0)\n",
+        "  return v\n",
+        "\n",
+        "with assert_raises(ValueError):\n",
+        "  f(1.0)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KYm6-5GCILXQ"
+      },
+      "source": [
+        "A common pattern used to work around this limitation is to start with a Python None value, then conditionally create the `tf.Variable` if the value is None:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HQrG5_kOiKl_"
+      },
+      "outputs": [],
+      "source": [
+        "class Count(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.count = None\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    if self.count is None:\n",
+        "      self.count = tf.Variable(0)\n",
+        "    return self.count.assign_add(1)\n",
+        "\n",
+        "c = Count()\n",
+        "print(c())\n",
+        "print(c())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7uD6qI7aJwbR"
+      },
+      "source": [
+        "#### Using with multiple Keras optimizers\n",
+        "You may encounter `ValueError: tf.function only supports singleton tf.Variables created on the first call.` when using more than one Keras optimizer with a `tf.function`. This error occurs because optimizers internally create `tf.Variable`s when they apply gradients for the first time."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yWQ3-r99Jvze"
+      },
+      "outputs": [],
+      "source": [
+        "opt1 = tf.keras.optimizers.Adam(learning_rate = 1e-2)\n",
+        "opt2 = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n",
+        "\n",
+        "@tf.function\n",
+        "def train_step(w, x, y, optimizer):\n",
+        "   with tf.GradientTape() as tape:\n",
+        "       L = tf.reduce_sum(tf.square(w*x - y))\n",
+        "   gradients = tape.gradient(L, [w])\n",
+        "   optimizer.apply_gradients(zip(gradients, [w]))\n",
+        "\n",
+        "w = tf.Variable(2.)\n",
+        "x = tf.constant([-1.])\n",
+        "y = tf.constant([2.])\n",
+        "\n",
+        "train_step(w, x, y, opt1)\n",
+        "print(\"Calling `train_step` with different optimizer...\")\n",
+        "with assert_raises(ValueError):\n",
+        "  train_step(w, x, y, opt2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7Q8BRPCThTjB"
+      },
+      "source": [
+        "If you need to change a stateful object between calls, it's simplest to define a `tf.Module` subclass, and create instances to hold those objects:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3P59ocmIslHz"
+      },
+      "outputs": [],
+      "source": [
+        "class TrainStep(tf.Module):\n",
+        "  def __init__(self, optimizer):\n",
+        "    self.optimizer = optimizer\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self, w, x, y):\n",
+        "    with tf.GradientTape() as tape:\n",
+        "       L = tf.reduce_sum(tf.square(w*x - y))\n",
+        "    gradients = tape.gradient(L, [w])\n",
+        "    self.optimizer.apply_gradients(zip(gradients, [w]))\n",
+        "\n",
+        "\n",
+        "opt1 = tf.keras.optimizers.Adam(learning_rate = 1e-2)\n",
+        "opt2 = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n",
+        "\n",
+        "train_o1 = TrainStep(opt1)\n",
+        "train_o2 = TrainStep(opt2)\n",
+        "\n",
+        "train_o1(w, x, y)\n",
+        "train_o2(w, x, y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dUHUi881smHF"
+      },
+      "source": [
+        "You could also do this manually by creating multiple instances of the `@tf.function` wrapper, one for each optimizer:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YV5F2Gy9hSI3"
+      },
+      "outputs": [],
+      "source": [
+        "opt1 = tf.keras.optimizers.Adam(learning_rate = 1e-2)\n",
+        "opt2 = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n",
+        "\n",
+        "# Not a tf.function.\n",
+        "def train_step(w, x, y, optimizer):\n",
+        "   with tf.GradientTape() as tape:\n",
+        "       L = tf.reduce_sum(tf.square(w*x - y))\n",
+        "   gradients = tape.gradient(L, [w])\n",
+        "   optimizer.apply_gradients(zip(gradients, [w]))\n",
+        "\n",
+        "w = tf.Variable(2.)\n",
+        "x = tf.constant([-1.])\n",
+        "y = tf.constant([2.])\n",
+        "\n",
+        "# Make a new tf.function and ConcreteFunction for each optimizer.\n",
+        "train_step_1 = tf.function(train_step)\n",
+        "train_step_2 = tf.function(train_step)\n",
+        "for i in range(10):\n",
+        "  if i % 2 == 0:\n",
+        "    train_step_1(w, x, y, opt1)\n",
+        "  else:\n",
+        "    train_step_2(w, x, y, opt2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Xjnz5CcuqQac"
+      },
+      "source": [
+        "#### Using with multiple Keras models\n",
+        "\n",
+        "You may also encounter `ValueError: tf.function only supports singleton tf.Variables created on the first call.` when passing different model instances to the same `tf.function`.\n",
+        "\n",
+        "This error occurs because Keras models (which [do not have their input shape defined](https://www.tensorflow.org/guide/keras/custom_layers_and_models#best_practice_deferring_weight_creation_until_the_shape_of_the_inputs_is_known)) and Keras layers create `tf.Variable`s when they are first called. You may be attempting to initialize those variables inside a `tf.function`, which has already been called. To avoid this error, try calling `model.build(input_shape)` to initialize all the weights before training the model.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IKyrEY5GVX3M"
+      },
+      "source": [
+        "## Further reading\n",
         "\n",
-        "outer()\n",
-        "outer()"
+        "To learn about how to export and load a `tf.function`, see the [SavedModel guide](../../guide/saved_model). To learn more about graph optimizations that are performed after tracing, see the [Grappler guide](../../guide/graph_optimization). To learn how to optimize your data pipeline and profile your model, see the [Profiler guide](../../guide/profiler.md)."
       ]
     }
   ],
   "metadata": {
     "colab": {
-      "collapsed_sections": [
-        "Jxv6goXm7oGF"
-      ],
       "name": "function.ipynb",
       "private_outputs": true,
-      "provenance": [
-        {
-          "file_id": "1Hl4PR32Y_nhq4AZI-ku9FlpZIgXjEU7d",
-          "timestamp": 1544141357531
-        },
-        {
-          "file_id": "https://github.com/tensorflow/docs/blob/master/site/en/guide/function.ipynb",
-          "timestamp": 1542126584672
-        }
-      ],
+      "provenance": [],
       "toc_visible": true
     },
     "kernelspec": {
diff --git a/site/en/guide/gpu.ipynb b/site/en/guide/gpu.ipynb
index 3c683dcc016..8f299f057b7 100644
--- a/site/en/guide/gpu.ipynb
+++ b/site/en/guide/gpu.ipynb
@@ -1,718 +1,515 @@
 {
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "Tce3stUlHN0L"
-   },
-   "source": [
-    "##### Copyright 2018 The TensorFlow Authors.\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {},
-    "colab_type": "code",
-    "id": "tuOe1ymfHZPu"
-   },
-   "outputs": [],
-   "source": [
-    "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-    "# you may not use this file except in compliance with the License.\n",
-    "# You may obtain a copy of the License at\n",
-    "#\n",
-    "# https://www.apache.org/licenses/LICENSE-2.0\n",
-    "#\n",
-    "# Unless required by applicable law or agreed to in writing, software\n",
-    "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-    "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-    "# See the License for the specific language governing permissions and\n",
-    "# limitations under the License."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "MfBg1C5NB3X0"
-   },
-   "source": [
-    "# Use a GPU\n",
-    "\n",
-    "\n",
-    "  \n",
-    "  \n",
-    "  \n",
-    "  \n",
-    "
\n",
-    "    View on TensorFlow.org\n",
-    "  \n",
-    "    Run in Google Colab\n",
-    "  \n",
-    "    View source on GitHub\n",
-    "  \n",
-    "    Download notebook\n",
-    "  
"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "SoYIwe40vEPI"
-   },
-   "source": [
-    "TensorFlow code, and `tf.keras` models will transparently run on a single GPU with no code changes required.\n",
-    "\n",
-    "Note: Use `tf.config.experimental.list_physical_devices('GPU')` to confirm that TensorFlow is using the GPU.\n",
-    "\n",
-    "The simplest way to run on multiple GPUs, on one or many machines, is using [Distribution Strategies](distributed_training.ipynb).\n",
-    "\n",
-    "This guide is for users who have tried these approaches and found that they need fine-grained control of how TensorFlow uses the GPU."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "MUXex9ctTuDB"
-   },
-   "source": [
-    "## Setup\n",
-    "\n",
-    "Ensure you have the latest TensorFlow gpu release installed."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 34
-    },
-    "colab_type": "code",
-    "id": "IqR2PQG4ZaZ0",
-    "outputId": "b80829b0-df1d-4d4e-a6d3-a6531cd211a0"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Num GPUs Available:  2\n"
-     ]
-    }
-   ],
-   "source": [
-    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-    "\n",
-    "try:\n",
-    "  # %tensorflow_version only exists in Colab.\n",
-    "  %tensorflow_version 2.x\n",
-    "except Exception:\n",
-    "  pass\n",
-    "import tensorflow as tf\n",
-    "print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "ZELutYNetv-v"
-   },
-   "source": [
-    "## Overview\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "xHxb-dlhMIzW"
-   },
-   "source": [
-    "TensorFlow supports running computations on a variety of types of devices, including CPU and GPU. They are represented with string identifiers for example:\n",
-    "\n",
-    "*   `\"/device:CPU:0\"`: The CPU of your machine.\n",
-    "*   `\"/GPU:0\"`: Short-hand notation for the first GPU of your machine that is visible to TensorFlow.\n",
-    "*   `\"/job:localhost/replica:0/task:0/device:GPU:1\"`: Fully qualified name of the second GPU of your machine that is visible to TensorFlow.\n",
-    "\n",
-    "If a TensorFlow operation has both CPU and GPU implementations, by default the GPU devices will be given priority when the operation is assigned to a device. For example, `tf.matmul` has both CPU and GPU kernels. On a system with devices `CPU:0` and `GPU:0`, the `GPU:0` device will be selected to run `tf.matmul` unless you explicitly request running it on another device."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "UhNtHfuxCGVy"
-   },
-   "source": [
-    "## Logging device placement\n",
-    "\n",
-    "To find out which devices your operations and tensors are assigned to, put\n",
-    "`tf.debugging.set_log_device_placement(True)` as the first statement of your\n",
-    "program. Enabling device placement logging causes any Tensor allocations or operations to be printed."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 85
-    },
-    "colab_type": "code",
-    "id": "2Dbw0tpEirCd",
-    "outputId": "f9823d90-64d3-4eef-a6fe-a3bb421e0351"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Executing op MatMul in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "tf.Tensor(\n",
-      "[[22. 28.]\n",
-      " [49. 64.]], shape=(2, 2), dtype=float32)\n"
-     ]
-    }
-   ],
-   "source": [
-    "tf.debugging.set_log_device_placement(True)\n",
-    "\n",
-    "# Create some tensors\n",
-    "a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
-    "b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
-    "c = tf.matmul(a, b)\n",
-    "\n",
-    "print(c)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "kKhmFeraTdEI"
-   },
-   "source": [
-    "The above code will print an indication the `MatMul` op was executed on `GPU:0`."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "U88FspwGjB7W"
-   },
-   "source": [
-    "## Manual device placement\n",
-    "\n",
-    "If you would like a particular operation to run on a device of your choice\n",
-    "instead of what's automatically selected for you, you can use `with tf.device`\n",
-    "to create a device context, and all the operations within that context will\n",
-    "run on the same designated device."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 85
-    },
-    "colab_type": "code",
-    "id": "8wqaQfEhjHit",
-    "outputId": "7bbbb814-13d6-4e00-b4a6-b81d87e1d6ac"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Executing op MatMul in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "tf.Tensor(\n",
-      "[[22. 28.]\n",
-      " [49. 64.]], shape=(2, 2), dtype=float32)\n"
-     ]
-    }
-   ],
-   "source": [
-    "tf.debugging.set_log_device_placement(True)\n",
-    "\n",
-    "# Place tensors on the CPU\n",
-    "with tf.device('/CPU:0'):\n",
-    "  a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
-    "  b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
-    "\n",
-    "c = tf.matmul(a, b)\n",
-    "print(c)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "8ixO89gRjJUu"
-   },
-   "source": [
-    "You will see that now `a` and `b` are assigned to `CPU:0`. Since a device was\n",
-    "not explicitly specified for the `MatMul` operation, the TensorFlow runtime will\n",
-    "choose one based on the operation and available devices (`GPU:0` in this\n",
-    "example) and automatically copy tensors between devices if required."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "ARrRhwqijPzN"
-   },
-   "source": [
-    "## Limiting GPU memory growth\n",
-    "\n",
-    "By default, TensorFlow maps nearly all of the GPU memory of all GPUs (subject to\n",
-    "[`CUDA_VISIBLE_DEVICES`](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars)) visible to the process. This is done to more efficiently use the relatively precious GPU memory resources on the devices by reducing memory fragmentation. To limit TensorFlow to a specific set of GPUs we use the `tf.config.experimental.set_visible_devices` method."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 34
-    },
-    "colab_type": "code",
-    "id": "hPI--n_jhZhv",
-    "outputId": "0a52d17f-743b-4657-93a8-ee3ece54271a"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "2 Physical GPUs, 1 Logical GPU\n"
-     ]
-    }
-   ],
-   "source": [
-    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
-    "if gpus:\n",
-    "  # Restrict TensorFlow to only use the first GPU\n",
-    "  try:\n",
-    "    tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n",
-    "    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
-    "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPU\")\n",
-    "  except RuntimeError as e:\n",
-    "    # Visible devices must be set before GPUs have been initialized\n",
-    "    print(e)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "N3x4M55DhYk9"
-   },
-   "source": [
-    "In some cases it is desirable for the process to only allocate a subset of the available memory, or to only grow the memory usage as is needed by the process. TensorFlow provides two methods to control this.\n",
-    "\n",
-    "The first option is to turn on memory growth by calling `tf.config.experimental.set_memory_growth`, which attempts to allocate only as much GPU memory as needed for the runtime allocations: it starts out allocating very little memory, and as the program gets run and more GPU memory is needed, we extend the GPU memory region allocated to the TensorFlow process. Note we do not release memory, since it can lead to memory fragmentation. To turn on memory growth for a specific GPU, use the following code prior to allocating any tensors or executing any ops."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 34
-    },
-    "colab_type": "code",
-    "id": "jr3Kf1boFnCO",
-    "outputId": "a91bc31c-1059-48c2-cf0d-7193c1133db2"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "2 Physical GPUs, 2 Logical GPUs\n"
-     ]
-    }
-   ],
-   "source": [
-    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
-    "if gpus:\n",
-    "  try:\n",
-    "    # Currently, memory growth needs to be the same across GPUs\n",
-    "    for gpu in gpus:\n",
-    "      tf.config.experimental.set_memory_growth(gpu, True)\n",
-    "    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
-    "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
-    "  except RuntimeError as e:\n",
-    "    # Memory growth must be set before GPUs have been initialized\n",
-    "    print(e)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "I1o8t51QFnmv"
-   },
-   "source": [
-    "Another way to enable this option is to set the environmental variable `TF_FORCE_GPU_ALLOW_GROWTH` to `true`. This configuration is platform specific.\n",
-    "\n",
-    "The second method is to configure a virtual GPU device with `tf.config.experimental.set_virtual_device_configuration` and set a hard limit on the total memory to allocate on the GPU."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 34
-    },
-    "colab_type": "code",
-    "id": "2qO2cS9QFn42",
-    "outputId": "97202153-54fd-41ef-edb4-4738a86d8458"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "2 Physical GPUs, 2 Logical GPUs\n"
-     ]
-    }
-   ],
-   "source": [
-    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
-    "if gpus:\n",
-    "  # Restrict TensorFlow to only allocate 1GB of memory on the first GPU\n",
-    "  try:\n",
-    "    tf.config.experimental.set_virtual_device_configuration(\n",
-    "        gpus[0],\n",
-    "        [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024)])\n",
-    "    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
-    "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
-    "  except RuntimeError as e:\n",
-    "    # Virtual devices must be set before GPUs have been initialized\n",
-    "    print(e)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "Bsg1iLuHFoLW"
-   },
-   "source": [
-    "This is useful if you want to truly bound the amount of GPU memory available to the TensorFlow process. This is common practice for local development when the GPU is shared with other applications such as a workstation GUI."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "B27_-1gyjf-t"
-   },
-   "source": [
-    "## Using a single GPU on a multi-GPU system\n",
-    "\n",
-    "If you have more than one GPU in your system, the GPU with the lowest ID will be\n",
-    "selected by default. If you would like to run on a different GPU, you will need\n",
-    "to specify the preference explicitly:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 34
-    },
-    "colab_type": "code",
-    "id": "wep4iteljjG1",
-    "outputId": "79329b7e-b50f-4252-95f2-ffbb0e952532"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "/job:localhost/replica:0/task:0/device:GPU:2 unknown device.\n"
-     ]
-    }
-   ],
-   "source": [
-    "tf.debugging.set_log_device_placement(True)\n",
-    "\n",
-    "try:\n",
-    "  # Specify an invalid GPU device\n",
-    "  with tf.device('/device:GPU:2'):\n",
-    "    a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
-    "    b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
-    "    c = tf.matmul(a, b)\n",
-    "except RuntimeError as e:\n",
-    "  print(e)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "jy-4cCO_jn4G"
-   },
-    "source": [
-      "If the device you have specified does not exist, you will get a `RuntimeError`: `.../device:GPU:2 unknown device`.\n",
-      "\n",
-      "If you would like TensorFlow to automatically choose an existing and supported device to run the operations in case the specified one doesn't exist, you can call `tf.config.set_soft_device_placement(True)`."
-    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 85
-    },
-    "colab_type": "code",
-    "id": "sut_UHlkjvWd",
-    "outputId": "11433de9-96ba-4a48-d77e-ee433f54123a"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Executing op MatMul in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "tf.Tensor(\n",
-      "[[22. 28.]\n",
-      " [49. 64.]], shape=(2, 2), dtype=float32)\n"
-     ]
-    }
-   ],
-   "source": [
-    "tf.config.set_soft_device_placement(True)\n",
-    "tf.debugging.set_log_device_placement(True)\n",
-    "\n",
-    "# Creates some tensors\n",
-    "a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
-    "b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
-    "c = tf.matmul(a, b)\n",
-    "\n",
-    "print(c)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "sYTYPrQZj2d9"
-   },
-   "source": [
-    "## Using multiple GPUs\n",
-    "\n",
-    "Developing for multiple GPUs will allow a model to scale with the additional resources. If developing on a system with a single GPU, we can simulate multiple GPUs with virtual devices. This enables easy testing of multi-GPU setups without requiring additional resources."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 34
-    },
-    "colab_type": "code",
-    "id": "8EMGuGKbNkc6",
-    "outputId": "36d2a3f4-1aa0-460f-bb2e-acaefa44f1fc"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "2 Physical GPU, 3 Logical GPUs\n"
-     ]
-    }
-   ],
-   "source": [
-    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
-    "if gpus:\n",
-    "  # Create 2 virtual GPUs with 1GB memory each\n",
-    "  try:\n",
-    "    tf.config.experimental.set_virtual_device_configuration(\n",
-    "        gpus[0],\n",
-    "        [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024),\n",
-    "         tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024)])\n",
-    "    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
-    "    print(len(gpus), \"Physical GPU,\", len(logical_gpus), \"Logical GPUs\")\n",
-    "  except RuntimeError as e:\n",
-    "    # Virtual devices must be set before GPUs have been initialized\n",
-    "    print(e)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "xmNzO0FxNkol"
-   },
-   "source": [
-    "Once we have multiple logical GPUs available to the runtime, we can utilize the multiple GPUs with `tf.distribute.Strategy` or with manual placement."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "IDZmEGq4j6kG"
-   },
-   "source": [
-    "#### With `tf.distribute.Strategy`\n",
-    "\n",
-    "The best practice for using multiple GPUs is to use `tf.distribute.Strategy`.\n",
-    "Here is a simple example:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
-    "colab": {
-     "height": 340
-    },
-    "colab_type": "code",
-    "id": "1KgzY8V2AvRv",
-    "outputId": "09341ecf-f439-499b-8282-919cbf470c6d"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Executing op RandomUniform in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op Sub in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op Mul in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op Add in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op VarIsInitializedOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op LogicalNot in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op Assert in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op AssignVariableOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op ReadVariableOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op Identity in device /job:localhost/replica:0/task:0/device:GPU:1\n",
-      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
-      "Executing op VarIsInitializedOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
-      "Executing op LogicalNot in device /job:localhost/replica:0/task:0/device:GPU:1\n",
-      "Executing op Assert in device /job:localhost/replica:0/task:0/device:GPU:1\n",
-      "Executing op AssignVariableOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
-      "Executing op Reshape in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n"
-     ]
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Authors.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "# Use a GPU\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SoYIwe40vEPI"
+      },
+      "source": [
+        "TensorFlow code, and `tf.keras` models will transparently run on a single GPU with no code changes required.\n",
+        "\n",
+        "Note: Use `tf.config.list_physical_devices('GPU')` to confirm that TensorFlow is using the GPU.\n",
+        "\n",
+        "The simplest way to run on multiple GPUs, on one or many machines, is using [Distribution Strategies](distributed_training.ipynb).\n",
+        "\n",
+        "This guide is for users who have tried these approaches and found that they need fine-grained control of how TensorFlow uses the GPU. To learn how to debug performance issues for single and multi-GPU scenarios, see the [Optimize TensorFlow GPU Performance](gpu_performance_analysis.md) guide."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MUXex9ctTuDB"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Ensure you have the latest TensorFlow gpu release installed."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IqR2PQG4ZaZ0"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZELutYNetv-v"
+      },
+      "source": [
+        "## Overview\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xHxb-dlhMIzW"
+      },
+      "source": [
+        "TensorFlow supports running computations on a variety of types of devices, including CPU and GPU. They are represented with string identifiers for example:\n",
+        "\n",
+        "*   `\"/device:CPU:0\"`: The CPU of your machine.\n",
+        "*   `\"/GPU:0\"`: Short-hand notation for the first GPU of your machine that is visible to TensorFlow.\n",
+        "*   `\"/job:localhost/replica:0/task:0/device:GPU:1\"`: Fully qualified name of the second GPU of your machine that is visible to TensorFlow.\n",
+        "\n",
+        "If a TensorFlow operation has both CPU and GPU implementations, by default, the GPU device is prioritized when the operation is assigned. For example, `tf.matmul` has both CPU and GPU kernels and on a system with devices `CPU:0` and `GPU:0`, the `GPU:0` device is selected to run `tf.matmul` unless you explicitly request to run it on another device.\n",
+        "\n",
+        "If a TensorFlow operation has no corresponding GPU implementation, then the operation falls back to the CPU device. For example, since `tf.cast` only has a CPU kernel, on a system with devices `CPU:0` and `GPU:0`, the `CPU:0` device is selected to run `tf.cast`, even if requested to run on the `GPU:0` device."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UhNtHfuxCGVy"
+      },
+      "source": [
+        "## Logging device placement\n",
+        "\n",
+        "To find out which devices your operations and tensors are assigned to, put\n",
+        "`tf.debugging.set_log_device_placement(True)` as the first statement of your\n",
+        "program. Enabling device placement logging causes any Tensor allocations or operations to be printed."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2Dbw0tpEirCd"
+      },
+      "outputs": [],
+      "source": [
+        "tf.debugging.set_log_device_placement(True)\n",
+        "\n",
+        "# Create some tensors\n",
+        "a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
+        "b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
+        "c = tf.matmul(a, b)\n",
+        "\n",
+        "print(c)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kKhmFeraTdEI"
+      },
+      "source": [
+        "The above code will print an indication the `MatMul` op was executed on `GPU:0`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "U88FspwGjB7W"
+      },
+      "source": [
+        "## Manual device placement\n",
+        "\n",
+        "If you would like a particular operation to run on a device of your choice\n",
+        "instead of what's automatically selected for you, you can use `with tf.device`\n",
+        "to create a device context, and all the operations within that context will\n",
+        "run on the same designated device."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8wqaQfEhjHit"
+      },
+      "outputs": [],
+      "source": [
+        "tf.debugging.set_log_device_placement(True)\n",
+        "\n",
+        "# Place tensors on the CPU\n",
+        "with tf.device('/CPU:0'):\n",
+        "  a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
+        "  b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
+        "\n",
+        "# Run on the GPU\n",
+        "c = tf.matmul(a, b)\n",
+        "print(c)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8ixO89gRjJUu"
+      },
+      "source": [
+        "You will see that now `a` and `b` are assigned to `CPU:0`. Since a device was\n",
+        "not explicitly specified for the `MatMul` operation, the TensorFlow runtime will\n",
+        "choose one based on the operation and available devices (`GPU:0` in this\n",
+        "example) and automatically copy tensors between devices if required."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ARrRhwqijPzN"
+      },
+      "source": [
+        "## Limiting GPU memory growth\n",
+        "\n",
+        "By default, TensorFlow maps nearly all of the GPU memory of all GPUs (subject to\n",
+        "[`CUDA_VISIBLE_DEVICES`](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars)) visible to the process. This is done to more efficiently use the relatively precious GPU memory resources on the devices by reducing memory fragmentation. To limit TensorFlow to a specific set of GPUs, use the `tf.config.set_visible_devices` method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hPI--n_jhZhv"
+      },
+      "outputs": [],
+      "source": [
+        "gpus = tf.config.list_physical_devices('GPU')\n",
+        "if gpus:\n",
+        "  # Restrict TensorFlow to only use the first GPU\n",
+        "  try:\n",
+        "    tf.config.set_visible_devices(gpus[0], 'GPU')\n",
+        "    logical_gpus = tf.config.list_logical_devices('GPU')\n",
+        "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPU\")\n",
+        "  except RuntimeError as e:\n",
+        "    # Visible devices must be set before GPUs have been initialized\n",
+        "    print(e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N3x4M55DhYk9"
+      },
+      "source": [
+        "In some cases it is desirable for the process to only allocate a subset of the available memory, or to only grow the memory usage as is needed by the process. TensorFlow provides two methods to control this.\n",
+        "\n",
+        "The first option is to turn on memory growth by calling `tf.config.experimental.set_memory_growth`, which attempts to allocate only as much GPU memory as needed for the runtime allocations: it starts out allocating very little memory, and as the program gets run and more GPU memory is needed, the GPU memory region is extended for the TensorFlow process. Memory is not released since it can lead to memory fragmentation. To turn on memory growth for a specific GPU, use the following code prior to allocating any tensors or executing any ops."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jr3Kf1boFnCO"
+      },
+      "outputs": [],
+      "source": [
+        "gpus = tf.config.list_physical_devices('GPU')\n",
+        "if gpus:\n",
+        "  try:\n",
+        "    # Currently, memory growth needs to be the same across GPUs\n",
+        "    for gpu in gpus:\n",
+        "      tf.config.experimental.set_memory_growth(gpu, True)\n",
+        "    logical_gpus = tf.config.list_logical_devices('GPU')\n",
+        "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
+        "  except RuntimeError as e:\n",
+        "    # Memory growth must be set before GPUs have been initialized\n",
+        "    print(e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "I1o8t51QFnmv"
+      },
+      "source": [
+        "Another way to enable this option is to set the environmental variable `TF_FORCE_GPU_ALLOW_GROWTH` to `true`. This configuration is platform specific.\n",
+        "\n",
+        "The second method is to configure a virtual GPU device with `tf.config.set_logical_device_configuration` and set a hard limit on the total memory to allocate on the GPU."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2qO2cS9QFn42"
+      },
+      "outputs": [],
+      "source": [
+        "gpus = tf.config.list_physical_devices('GPU')\n",
+        "if gpus:\n",
+        "  # Restrict TensorFlow to only allocate 1GB of memory on the first GPU\n",
+        "  try:\n",
+        "    tf.config.set_logical_device_configuration(\n",
+        "        gpus[0],\n",
+        "        [tf.config.LogicalDeviceConfiguration(memory_limit=1024)])\n",
+        "    logical_gpus = tf.config.list_logical_devices('GPU')\n",
+        "    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
+        "  except RuntimeError as e:\n",
+        "    # Virtual devices must be set before GPUs have been initialized\n",
+        "    print(e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Bsg1iLuHFoLW"
+      },
+      "source": [
+        "This is useful if you want to truly bound the amount of GPU memory available to the TensorFlow process. This is common practice for local development when the GPU is shared with other applications such as a workstation GUI."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "B27_-1gyjf-t"
+      },
+      "source": [
+        "## Using a single GPU on a multi-GPU system\n",
+        "\n",
+        "If you have more than one GPU in your system, the GPU with the lowest ID will be\n",
+        "selected by default. If you would like to run on a different GPU, you will need\n",
+        "to specify the preference explicitly:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wep4iteljjG1"
+      },
+      "outputs": [],
+      "source": [
+        "tf.debugging.set_log_device_placement(True)\n",
+        "\n",
+        "try:\n",
+        "  # Specify an invalid GPU device\n",
+        "  with tf.device('/device:GPU:2'):\n",
+        "    a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
+        "    b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
+        "    c = tf.matmul(a, b)\n",
+        "except RuntimeError as e:\n",
+        "  print(e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jy-4cCO_jn4G"
+      },
+      "source": [
+        "If the device you have specified does not exist, you will get a `RuntimeError`: `.../device:GPU:2 unknown device`.\n",
+        "\n",
+        "If you would like TensorFlow to automatically choose an existing and supported device to run the operations in case the specified one doesn't exist, you can call `tf.config.set_soft_device_placement(True)`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sut_UHlkjvWd"
+      },
+      "outputs": [],
+      "source": [
+        "tf.config.set_soft_device_placement(True)\n",
+        "tf.debugging.set_log_device_placement(True)\n",
+        "\n",
+        "# Creates some tensors\n",
+        "a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
+        "b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
+        "c = tf.matmul(a, b)\n",
+        "\n",
+        "print(c)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sYTYPrQZj2d9"
+      },
+      "source": [
+        "## Using multiple GPUs\n",
+        "\n",
+        "Developing for multiple GPUs will allow a model to scale with the additional resources. If developing on a system with a single GPU, you can simulate multiple GPUs with virtual devices. This enables easy testing of multi-GPU setups without requiring additional resources."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8EMGuGKbNkc6"
+      },
+      "outputs": [],
+      "source": [
+        "gpus = tf.config.list_physical_devices('GPU')\n",
+        "if gpus:\n",
+        "  # Create 2 virtual GPUs with 1GB memory each\n",
+        "  try:\n",
+        "    tf.config.set_logical_device_configuration(\n",
+        "        gpus[0],\n",
+        "        [tf.config.LogicalDeviceConfiguration(memory_limit=1024),\n",
+        "         tf.config.LogicalDeviceConfiguration(memory_limit=1024)])\n",
+        "    logical_gpus = tf.config.list_logical_devices('GPU')\n",
+        "    print(len(gpus), \"Physical GPU,\", len(logical_gpus), \"Logical GPUs\")\n",
+        "  except RuntimeError as e:\n",
+        "    # Virtual devices must be set before GPUs have been initialized\n",
+        "    print(e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xmNzO0FxNkol"
+      },
+      "source": [
+        "Once there are multiple logical GPUs available to the runtime, you can utilize the multiple GPUs with `tf.distribute.Strategy` or with manual placement."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IDZmEGq4j6kG"
+      },
+      "source": [
+        "#### With `tf.distribute.Strategy`\n",
+        "\n",
+        "The best practice for using multiple GPUs is to use `tf.distribute.Strategy`.\n",
+        "Here is a simple example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1KgzY8V2AvRv"
+      },
+      "outputs": [],
+      "source": [
+        "tf.debugging.set_log_device_placement(True)\n",
+        "gpus = tf.config.list_logical_devices('GPU')\n",
+        "strategy = tf.distribute.MirroredStrategy(gpus)\n",
+        "with strategy.scope():\n",
+        "  inputs = tf.keras.layers.Input(shape=(1,))\n",
+        "  predictions = tf.keras.layers.Dense(1)(inputs)\n",
+        "  model = tf.keras.models.Model(inputs=inputs, outputs=predictions)\n",
+        "  model.compile(loss='mse',\n",
+        "                optimizer=tf.keras.optimizers.SGD(learning_rate=0.2))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Dy7nxlKsAxkK"
+      },
+      "source": [
+        "This program will run a copy of your model on each GPU, splitting the input data\n",
+        "between them, also known as \"[data parallelism](https://en.wikipedia.org/wiki/Data_parallelism)\".\n",
+        "\n",
+        "For more information about distribution strategies, check out the guide [here](./distributed_training.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8phxM5TVkAY_"
+      },
+      "source": [
+        "#### Manual placement\n",
+        "\n",
+        "`tf.distribute.Strategy` works under the hood by replicating computation across devices. You can manually implement replication by constructing your model on each GPU. For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AqPo9ltUA_EY"
+      },
+      "outputs": [],
+      "source": [
+        "tf.debugging.set_log_device_placement(True)\n",
+        "\n",
+        "gpus = tf.config.list_logical_devices('GPU')\n",
+        "if gpus:\n",
+        "  # Replicate your computation on multiple GPUs\n",
+        "  c = []\n",
+        "  for gpu in gpus:\n",
+        "    with tf.device(gpu.name):\n",
+        "      a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
+        "      b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
+        "      c.append(tf.matmul(a, b))\n",
+        "\n",
+        "  with tf.device('/CPU:0'):\n",
+        "    matmul_sum = tf.add_n(c)\n",
+        "\n",
+        "  print(matmul_sum)"
+      ]
     }
-   ],
-   "source": [
-    "tf.debugging.set_log_device_placement(True)\n",
-    "\n",
-    "strategy = tf.distribute.MirroredStrategy()\n",
-    "with strategy.scope():\n",
-    "  inputs = tf.keras.layers.Input(shape=(1,))\n",
-    "  predictions = tf.keras.layers.Dense(1)(inputs)\n",
-    "  model = tf.keras.models.Model(inputs=inputs, outputs=predictions)\n",
-    "  model.compile(loss='mse',\n",
-    "                optimizer=tf.keras.optimizers.SGD(learning_rate=0.2))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "Dy7nxlKsAxkK"
-   },
-   "source": [
-    "This program will run a copy of your model on each GPU, splitting the input data\n",
-    "between them, also known as \"[data parallelism](https://en.wikipedia.org/wiki/Data_parallelism)\".\n",
-    "\n",
-    "For more information about distribution strategies, check out the guide [here](./distributed_training.ipynb)."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "colab_type": "text",
-    "id": "8phxM5TVkAY_"
-   },
-   "source": [
-    "#### Manual placement\n",
-    "\n",
-    "`tf.distribute.Strategy` works under the hood by replicating computation across devices. You can manually implement replication by constructing your model on each GPU. For example:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 0,
-   "metadata": {
+  ],
+  "metadata": {
+    "accelerator": "GPU",
     "colab": {
-     "height": 119
-    },
-    "colab_type": "code",
-    "id": "AqPo9ltUA_EY",
-    "outputId": "966a7a6e-57ed-4090-92b6-625dc122f60b"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Executing op MatMul in device /job:localhost/replica:0/task:0/device:GPU:0\n",
-      "Executing op MatMul in device /job:localhost/replica:0/task:0/device:GPU:1\n",
-      "Executing op AddN in device /job:localhost/replica:0/task:0/device:CPU:0\n",
-      "tf.Tensor(\n",
-      "[[ 44.  56.]\n",
-      " [ 98. 128.]], shape=(2, 2), dtype=float32)\n"
-     ]
+      "collapsed_sections": [],
+      "name": "gpu.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
     }
-   ],
-   "source": [
-    "tf.debugging.set_log_device_placement(True)\n",
-    "\n",
-    "gpus = tf.config.experimental.list_logical_devices('GPU')\n",
-    "if gpus:\n",
-    "  # Replicate your computation on multiple GPUs\n",
-    "  c = []\n",
-    "  for gpu in gpus:\n",
-    "    with tf.device(gpu.name):\n",
-    "      a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
-    "      b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
-    "      c.append(tf.matmul(a, b))\n",
-    "\n",
-    "  with tf.device('/CPU:0'):\n",
-    "    matmul_sum = tf.add_n(c)\n",
-    "\n",
-    "  print(matmul_sum)"
-   ]
-  }
- ],
- "metadata": {
-  "colab": {
-   "collapsed_sections": [],
-   "name": "gpu.ipynb",
-   "toc_visible": true,
-   "version": "0.3.2"
   },
-  "kernelspec": {
-   "display_name": "Python 3",
-   "name": "python3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 0
+  "nbformat": 4,
+  "nbformat_minor": 0
 }
diff --git a/site/en/guide/gpu_performance_analysis.md b/site/en/guide/gpu_performance_analysis.md
new file mode 100644
index 00000000000..2832686d8f1
--- /dev/null
+++ b/site/en/guide/gpu_performance_analysis.md
@@ -0,0 +1,506 @@
+# Optimize TensorFlow GPU performance with the TensorFlow Profiler
+
+## Overview
+
+This guide will show you how to use the TensorFlow Profiler with TensorBoard to
+gain insight into and get the maximum performance out of your GPUs, and debug
+when one or more of your GPUs are underutilized.
+
+If you are new to the Profiler:
+
+-   Get started with the
+    [TensorFlow Profiler: Profile model performance](https://www.tensorflow.org/tensorboard/tensorboard_profiling_keras)
+    notebook with a Keras example and
+    [TensorBoard](https://www.tensorflow.org/tensorboard).
+-   Learn about various profiling tools and methods available for optimizing
+    TensorFlow performance on the host (CPU) with the
+    [Optimize TensorFlow performance using the Profiler](https://www.tensorflow.org/guide/profiler#profiler_tools)
+    guide.
+
+Keep in mind that offloading computations to GPU may not always be beneficial,
+particularly for small models. There can be overhead due to:
+
+-   Data transfer between the host (CPU) and the device (GPU); and
+-   Due to the latency involved when the host launches GPU kernels.
+
+### Performance optimization workflow
+
+This guide outlines how to debug performance issues starting with a single GPU,
+then moving to a single host with multiple GPUs.
+
+It is recommended to debug performance issues in the following order:
+
+1.  Optimize and debug the performance on one GPU:
+    1.  Check if the input pipeline is a bottleneck.
+    2.  Debug the performance of one GPU.
+    3.  Enable mixed precision (with `fp16` (float16)) and optionally enable
+        [XLA](https://www.tensorflow.org/xla).
+2.  Optimize and debug the performance on the multi-GPU single host.
+
+For example, if you are using a TensorFlow
+[distribution strategy](https://www.tensorflow.org/guide/distributed_training)
+to train a model on a single host with multiple GPUs and notice suboptimal GPU
+utilization, you should first optimize and debug the performance for one GPU
+before debugging the multi-GPU system.
+
+As a baseline for getting performant code on GPUs, this guide assumes you are
+already using `tf.function`. The Keras `Model.compile` and `Model.fit` APIs will
+utilize `tf.function` automatically under the hood. When writing a custom
+training loop with `tf.GradientTape`, refer to the
+[Better performance with tf.function](https://www.tensorflow.org/guide/function)
+on how to enable `tf.function`s.
+
+The next sections discuss suggested approaches for each of the scenarios above
+to help identify and fix performance bottlenecks.
+
+## 1. Optimize the performance on one GPU
+
+In an ideal case, your program should have high GPU utilization, minimal CPU
+(the host) to GPU (the device) communication, and no overhead from the input
+pipeline.
+
+The first step in analyzing the performance is to get a profile for a model
+running with one GPU.
+
+TensorBoard's Profiler
+[overview page](https://www.tensorflow.org/guide/profiler#overview_page)—which
+shows a top level view of how your model performed during a profile run—can
+provide an idea of how far away your program is from the ideal scenario.
+
+![TensorFlow Profiler Overview Page](images/gpu_perf_analysis/overview_page.png "The overview page of the TensorFlow Profiler")
+
+The key numbers to pay attention to the overview page are:
+
+1.  How much of the step time is from actual device execution
+2.  The percentage of ops placed on device vs host
+3.  How many kernels use `fp16`
+
+Achieving optimal performance means maximizing these numbers in all three cases.
+To get an in-depth understanding of your program, you will need to be familiar
+with TensorBoard's Profiler
+[trace viewer](https://www.tensorflow.org/guide/profiler#trace_viewer). The
+sections below show some common trace viewer patterns that you should look for
+when diagnosing performance bottlenecks.
+
+Below is an image of a model trace view running on one GPU. From the _TensorFlow
+Name Scope_ and _TensorFlow Ops_ sections, you can identify different parts of
+the model, like the forward pass, the loss function, backward pass/gradient
+calculation, and the optimizer weight update. You can also have the ops running
+on the GPU next to each _Stream_, which refer to CUDA streams. Each stream is
+used for specific tasks. In this trace, _Stream#118_ is used to launch compute
+kernels and device-to-device copies. _Stream#119_ is used for host-to-device
+copy and *Stream#120* for device to host copy.
+
+The trace below shows common characteristics of a performant model.
+
+![image](images/gpu_perf_analysis/traceview_ideal.png "An example TensorFlow Profiler trace view")
+
+For example, the GPU compute timeline (_Stream#118_) looks "busy" with very few
+gaps. There are minimal copies from host to device (_Stream #119_) and from
+device to host (_Stream #120_), as well as minimal gaps between steps. When you
+run the Profiler for your program, you may not be able to identify these ideal
+characteristics in your trace view. The rest of this guide covers common
+scenarios and how to fix them.
+
+### 1. Debug the input pipeline
+
+The first step in GPU performance debugging is to determine if your program is
+input-bound. The easiest way to figure this out is to use the Profiler’s
+[Input-pipeline analyzer](https://www.tensorflow.org/guide/profiler#input_pipeline_analyzer),
+on TensorBoard, which provides an overview of time spent in the input pipeline.
+
+![image](images/gpu_perf_analysis/input_pipeline_analyzer.png "TensorFlow Profiler Input-Analyzer")
+
+You can take the following potential actions if your input-pipeline contributes
+significantly to step time:
+
+-   You can use the `tf.data`-specific
+    [guide](https://www.tensorflow.org/guide/data_performance_analysis) to learn
+    how to debug your input pipeline.
+-   Another quick way to check if the input pipeline is the bottleneck is to use
+    randomly generated input data that does not need any pre-processing.
+    [Here is an example](https://github.com/tensorflow/models/blob/4a5770827edf1c3974274ba3e4169d0e5ba7478a/official/vision/image_classification/resnet/resnet_runnable.py#L50-L57)
+    of using this technique for a ResNet model. If the input pipeline is
+    optimal, you should experience similar performance with real data and with
+    generated random/synthetic data. The only overhead in the synthetic data
+    case will be due to input data copy which again can be prefetched and
+    optimized.
+
+In addition, refer to the
+[best practices for optimizing the input data pipeline](https://www.tensorflow.org/guide/profiler#optimize_the_input_data_pipeline).
+
+### 2. Debug the performance of one GPU
+
+There are several factors that can contribute to low GPU utilization. Below are
+some scenarios commonly observed when looking at the
+[trace viewer](https://www.tensorflow.org/guide/profiler#trace_viewer) and
+potential solutions.
+
+#### 1. Analyze gaps between steps
+
+A common observation when your program is not running optimally is gaps between
+training steps. In the image of the trace view below, there is a large gap
+between steps 8 and 9, meaning that the GPU is idle during that time.
+
+![image](images/gpu_perf_analysis/traceview_step_gaps.png "TensorFlow Profile trace view showing gaps between steps")
+
+If your trace viewer shows large gaps between steps, this could be an indication
+that your program is input bound. In that case you should refer to the previous
+section on debugging your input pipeline if you have not already done so.
+
+However, even with an optimized input pipeline, you can still have gaps between
+the end of one step and the start of another due to CPU thread contention.
+`tf.data` makes use of background threads to parallelize pipeline processing.
+These threads may interfere with GPU host-side activity that happens at the
+beginning of each step, such as copying data or scheduling GPU operations.
+
+If you notice large gaps on the host side, which schedules these ops on the GPU,
+you can set the environment variable `TF_GPU_THREAD_MODE=gpu_private`. This
+ensures that GPU kernels are launched from their own dedicated threads, and
+don't get queued behind `tf.data` work.
+
+Gaps between steps can also be caused by metric calculations, Keras callbacks,
+or ops outside of `tf.function` that run on the host. These ops don’t have as
+good performance as the ops inside a TensorFlow graph. Additionally, some of
+these ops run on the CPU and copy tensors back and forth from the GPU.
+
+If after optimizing your input pipeline you still notice gaps between steps in
+the trace viewer, you should look at the model code between steps and check if
+disabling callbacks/metrics improves performance. Some details of these ops are
+also on the trace viewer (both device and host side).The recommendation in this
+scenario is to amortize the overhead of these ops by executing them after a
+fixed number of steps instead of every step. When using the `Model.compile` method in
+the `tf.keras` API, setting the `steps_per_execution` flag does
+this automatically. For custom training loops, use `tf.while_loop`.
+
+#### 2. Achieve higher device utilization
+
+##### 1. Small GPU kernels and host kernel launch delays
+
+The host enqueues kernels to be run on the GPU, but there is a latency (around
+20-40 μs) involved before kernels are actually executed on the GPU. In an ideal
+case, the host enqueues enough kernels on the GPU such that the GPU spends most
+of its time executing, rather than waiting on the host to enqueue more kernels.
+
+The Profiler's
+[overview page](https://www.tensorflow.org/guide/profiler#overview_page) on
+TensorBoard shows how much time the GPU was idle due to waiting on the host to
+launch kernels. In the image below, the GPU is idle for about 10% of the step
+time waiting on kernels to be launched.
+
+![image](images/gpu_perf_analysis/performance_summary.png "Summary of performance from TensorFlow Profile")
+
+The [trace viewer](https://www.tensorflow.org/guide/profiler#trace_viewer) for
+this same program shows small gaps between kernels where the host is busy
+launching kernels on the GPU.
+
+![image](images/gpu_perf_analysis/traceview_kernel_gaps.png "TensorFlow Profile trace view demonstrating gaps between kernels")
+
+By launching a lot of small ops on the GPU (like a scalar add, for example), the
+host might not keep up with the GPU. The
+[TensorFlow Stats](https://www.tensorflow.org/guide/profiler#tensorflow_stats)
+tool in TensorBoard for the same Profile shows 126,224 Mul operations taking
+2.77 seconds. Thus, each kernel is about 21.9 μs, which is very small (around
+the same time as launch latency) and can potentially result in host kernel
+launch delays.
+
+![image](images/gpu_perf_analysis/tensorflow_stats_page.png "TensorFlow Profile stats page")
+
+If your [trace viewer](https://www.tensorflow.org/guide/profiler#trace_viewer)
+shows many small gaps between ops on the GPU like in the image above, you can:
+
+-   Concatenate small tensors and use vectorized ops or use a larger batch size
+    to make each launched kernel do more work, which will keep the GPU busy for
+    longer.
+-   Make sure you are using `tf.function` to create TensorFlow graphs, so that
+    you are not running ops in a pure eager mode. If you are using `Model.fit`
+    (as oppose to a custom training loop with `tf.GradientTape`), then
+    `tf.keras.Model.compile` will automatically do this for you.
+-   Fuse kernels using XLA with `tf.function(jit_compile=True)` or
+    auto-clustering. For more details, go to the
+    [Enable mixed precision and XLA](#3._enable_mixed_precision_and_xla) section
+    below to learn how to enable XLA to get higher performance. This feature can
+    lead to high device utilization.
+
+##### 2. TensorFlow op placement
+
+The Profiler
+[overview page](https://www.tensorflow.org/guide/profiler#overview_page) shows
+you the percentage of ops placed on the host vs. the device (you can also verify
+the placement of specific ops by looking at the
+[trace viewer](https://www.tensorflow.org/guide/profiler#trace_viewer). Like in
+the image below, you want the percentage of ops on the host to be very small
+compared to the device.
+
+![image](images/gpu_perf_analysis/opp_placement.png "TF Op Placement")
+
+Ideally, most of the compute intensive ops should be placed on the GPU.
+
+To find out which devices the operations and tensors in your model are assigned
+to, set `tf.debugging.set_log_device_placement(True)` as the first statement of
+your program.
+
+Note that in some cases, even if you specify an op to be placed on a particular
+device, its implementation might override this condition (example:`tf.unique`).
+Even for single GPU training, specifying a distribution strategy, such as
+`tf.distribute.OneDeviceStrategy`, can result in more deterministic placement of
+ops on your device.
+
+One reason for having the majority of ops placed on the GPU is to prevent
+excessive memory copies between the host and the device (memory copies for model
+input/output data between host and device are expected). An example of excessive
+copying is demonstrated in the trace view below on GPU streams _#167_, _#168_,
+and _#169_.
+
+![image](images/gpu_perf_analysis/traceview_excessive_copy.png "TensorFlow Profile trace view demonstrating excessive H2D/D2H copies")
+
+These copies can sometimes hurt the performance if they block GPU kernels from
+executing. Memory copy operations in the
+[trace viewer](https://www.tensorflow.org/guide/profiler#trace_viewer) have more
+information about the ops that are the source of these copied tensors, but it
+might not always be easy to associate a memCopy with an op. In these cases, it
+is helpful to look at the ops nearby to check if the memory copy happens at the
+same location in every step.
+
+#### 3. More efficient kernels on GPUs
+
+Once your program's GPU utilization is acceptable, the next step is to look into
+increasing the efficiency of the GPU kernels by utilizing Tensor Cores or fusing
+ops.
+
+##### 1. Utilize Tensor Cores
+
+Modern NVIDIA® GPUs have specialized
+[Tensor Cores](https://www.nvidia.com/en-gb/data-center/tensor-cores/) that can
+significantly improve the performance of eligible kernels.
+
+You can use TensorBoard's
+[GPU kernel stats](https://www.tensorflow.org/guide/profiler#gpu_kernel_stats)
+to visualize which GPU kernels are Tensor Core-eligible, and which kernels are
+using Tensor Cores. Enabling `fp16` (see Enabling Mixed Precision section below)
+is one way to make your program’s General Matrix Multiply (GEMM) kernels (matmul
+ops) utilize the Tensor Core. GPU kernels use the Tensor Cores efficiently when
+the precision is fp16 and input/output tensor dimensions are divisible by 8 or
+16 (for `int8`).
+
+Note: With cuDNN v7.6.3 and later, convolution dimensions will automatically be
+padded where necessary to leverage Tensor Cores.
+
+For other detailed recommendations on how to make kernels efficient for GPUs,
+refer to the
+[NVIDIA® deep learning performance](https://docs.nvidia.com/deeplearning/performance/index.html#perf-guidelines)
+guide.
+
+##### 2. Fuse ops
+
+Use `tf.function(jit_compile=True)` to fuse smaller ops to form bigger kernels
+leading to significant performance gains. To learn more, refer to the
+[XLA](https://www.tensorflow.org/xla) guide.
+
+### 3. Enable mixed precision and XLA
+
+After following the above steps, enabling mixed precision and XLA are two
+optional steps you can take to improve performance further. The suggested
+approach is to enable them one by one and verify that the performance benefits
+are as expected.
+
+#### 1. Enable mixed precision
+
+The TensorFlow
+[Mixed precision](https://www.tensorflow.org/guide/keras/mixed_precision) guide
+shows how to enable `fp16` precision on GPUs. Enable
+[AMP](https://developer.nvidia.com/automatic-mixed-precision) on NVIDIA® GPUs to
+use Tensor Cores and realize up to 3x overall speedups when compared to using
+just `fp32` (float32) precision on Volta and newer GPU architectures.
+
+Make sure that matrix/tensor dimensions satisfy requirements for calling kernels
+that use Tensor Cores. GPU kernels use the Tensor Cores efficiently when the
+precision is fp16 and input/output dimensions are divisible by 8 or 16 (for
+int8).
+
+Note that with cuDNN v7.6.3 and later, convolution dimensions will automatically
+be padded where necessary to leverage Tensor Cores.
+
+Follow the best practices below to maximize the performance benefits of `fp16`
+precision.
+
+##### 1. Use optimal fp16 kernels
+
+With `fp16` enabled, your program’s matrix multiplications (GEMM) kernels,
+should use the corresponding `fp16` version that utilizes the Tensor Cores.
+However, in some cases, this does not happen and you do not experience the
+expected speedup from enabling `fp16`, as your program falls back to the
+inefficient implementation instead.
+
+![image](images/gpu_perf_analysis/gpu_kernels.png "TensorFlow Profile GPU Kernel Stats page")
+
+The [GPU kernel](https://www.tensorflow.org/guide/profiler#gpu_kernel_stats)
+stats page shows which ops are Tensor Core eligible and which kernels are
+actually using the efficient Tensor Core. The
+[NVIDIA® guide on deep learning performance](https://docs.nvidia.com/deeplearning/performance/mixed-precision-training/index.html#opt-tensor-cores)
+contains additional suggestions on how to leverage Tensor Cores. Additionally,
+the benefits of using `fp16` will also show in kernels that were previously
+memory bound, as now the ops will take half the time.
+
+##### 2. Dynamic vs. static loss scaling
+
+Loss scaling is necessary when using `fp16` to prevent underflow due to low
+precision. There are two types of loss scaling, dynamic and static, both of
+which are explained in greater detail in the
+[Mixed Precision guide](https://www.tensorflow.org/guide/keras/mixed_precision).
+You can use the `mixed_float16` policy to automatically enable loss scaling
+within the Keras optimizer.
+
+Note: The Keras mixed precision API defaults to evaluating standalone softmax
+ops (ops not part of a Keras loss function) as `fp16` which can lead to
+numerical issues and poor convergence. Cast such ops to `fp32` for optimal
+performance.
+
+When trying to optimize performance, it is important to remember that dynamic
+loss scaling can introduce additional conditional ops that run on the host, and
+lead to gaps that will be visible between steps in the trace viewer. On the
+other hand, static loss scaling does not have such overheads and can be a better
+option in terms of performance with the catch that you need to specify the
+correct static-loss scale value.
+
+#### 2. Enable XLA with tf.function(jit_compile=True) or auto-clustering
+
+As a final step in getting the best performance with a single GPU, you can
+experiment with enabling XLA, which will fuse ops and lead to better device
+utilization and a lower memory footprint. For details on how to enable XLA in
+your program with `tf.function(jit_compile=True)` or auto-clustering, refer to
+the [XLA](https://www.tensorflow.org/xla) guide.
+
+You can set the global JIT level to `-1` (off), `1`, or `2`. A higher level is
+more aggressive and may reduce parallelism and use more memory. Set the value to
+`1` if you have memory restrictions. Note that XLA does not perform well for
+models with variable input tensor shapes as the XLA compiler would have to keep
+compiling kernels whenever it encounters new shapes.
+
+## 2. Optimize the performance on the multi-GPU single host
+
+The `tf.distribute.MirroredStrategy` API can be used to scale model training
+from one GPU to multiple GPUs on a single host. (To learn more about how to do
+distributed training with TensorFlow, refer to the
+[Distributed training with TensorFlow](https://www.tensorflow.org/guide/distributed_training),
+[Use a GPU](https://www.tensorflow.org/guide/gpu), and
+[Use TPUs](https://www.tensorflow.org/guide/tpu) guides and the
+[Distributed training with Keras](https://www.tensorflow.org/tutorials/distribute/keras)
+tutorial.)
+
+Although the transition from one GPU to multiple GPUs should ideally be scalable
+out of the box, you can sometimes encounter performance issues.
+
+When going from training with a single GPU to multiple GPUs on the same host,
+ideally you should experience the performance scaling with only the additional
+overhead of gradient communication and increased host thread utilization.
+Because of this overhead, you will not have an exact 2x speedup if you move from
+1 to 2 GPUs, for example.
+
+The trace view below shows an example of the extra communication overhead when
+training on multiple GPUs. There is some overhead to concatenate the gradients,
+communicate them across replicas, and split them before doing the weight update.
+
+![image](images/gpu_perf_analysis/traceview_multi_gpu.png "TensorFlow Profile trace view for single host multi GPU scenario")
+
+The following checklist will help you achieve better performance when optimizing
+the performance in the multi-GPU scenario:
+
+1.  Try to maximize the batch size, which will lead to higher device utilization
+    and amortize the costs of communication across multiple GPUs. Using the
+    [memory profiler](https://www.tensorflow.org/guide/profiler#memory_profile_summary)
+    helps get a sense of how close your program is to peak memory utilization.
+    Note that while a higher batch size can affect convergence, this is usually
+    outweighed by the performance benefits.
+2.  When moving from a single GPU to multiple GPUs, the same host now has to
+    process much more input data. So, after (1), it is recommended to re-check
+    the input pipeline performance and make sure it is not a bottleneck.
+3.  Check the GPU timeline in your program's trace view for any unnecessary
+    AllReduce calls, as this results in a synchronization across all devices. In
+    the trace view shown above, the AllReduce is done via the
+    [NCCL](https://developer.nvidia.com/nccl) kernel, and there is only one NCCL
+    call on each GPU for the gradients on each step.
+4.  Check for unnecessary D2H, H2D and D2D copy operations that can be
+    minimized.
+5.  Check the step time to make sure each replica is doing the same work. For
+    example, it can happen that one GPU (typically, `GPU0`) is oversubscribed
+    because the host mistakenly ends up putting more work on it.
+6.  Lastly, check the training step across all GPUs in your trace view for any
+    ops that are executing sequentially. This usually happens when your program
+    includes control dependencies from one GPU to another. In the past,
+    debugging the performance in this situation has been solved on a
+    case-by-case basis. If you observe this behavior in your program,
+    [file a GitHub issue](https://github.com/tensorflow/tensorflow/issues/new/choose)
+    with images of your trace view.
+
+### 1. Optimize gradient AllReduce
+
+When training with a synchronous strategy, each device receives a portion of the
+input data.
+
+After computing the forward and backwards passes through the model, the
+gradients calculated on each device need to be aggregated and reduced. This
+_gradient AllReduce_ happens after the gradient calculation on each device, and
+before the optimizer updates the model weights.
+
+Each GPU first concatenates the gradients across the model layers, communicates
+them across GPUs using `tf.distribute.CrossDeviceOps`
+(`tf.distribute.NcclAllReduce` is the default), and then returns the gradients
+after reduction per layer.
+
+The optimizer will use these reduced gradients to update the weights of your
+model. Ideally, this process should happen at the same time on all GPUs to
+prevent any overheads.
+
+The time to AllReduce should be approximately the same as:
+
+```
+(number of parameters * 4bytes)/ (communication bandwidth)
+```
+
+This calculation is useful as a quick check to understand whether the
+performance you have when running a distributed training job is as expected, or
+if you need to do further performance debugging. You can get the number of
+parameters in your model from `Model.summary`.
+
+Note that each model parameter is 4 bytes in size since TensorFlow uses `fp32`
+(float32) to communicate gradients. Even when you have `fp16` enabled, NCCL
+AllReduce utilizes `fp32` parameters.
+
+To get the benefits of scaling, the step-time needs to be much higher compared
+to these overheads. One way to achieve this is to use a higher batch size as
+batch size affects step time, but does not impact the communication overhead.
+
+### 2. GPU host thread contention
+
+When running multiple GPUs, the CPU’s job is to keep all of the devices busy by
+efficiently launching GPU kernels across the devices.
+
+However, when there are a lot of independent operations that the CPU can
+schedule on one GPU, the CPU can decide to use a lot of its host threads to keep
+one GPU busy, and then launch kernels on another GPU in a non-deterministic
+order. This can cause a skew or negative scaling, which can negatively affect
+the performance.
+
+The [trace viewer](https://www.tensorflow.org/guide/profiler#trace_viewer) below
+shows the overhead when the CPU staggers GPU kernel launches inefficiently, as
+`GPU1` is idle and then starts running ops after `GPU2` has started.
+
+![image](images/gpu_perf_analysis/traceview_gpu_idle.png "TensorFlow Profile device trace view demonstrating inefficient kernel launch")
+
+The trace view for the host shows that the host is launching kernels on `GPU2`
+before launching them on `GPU1` (note that the below `tf_Compute*` ops are not
+indicative of CPU threads).
+
+![image](images/gpu_perf_analysis/traceview_host_contention.png "TensorFlow Profile host trace view demonstrating inefficient kernel launch")
+
+If you experience this kind of staggering of GPU kernels in your program’s trace
+view, the recommended action is to:
+
+-   Set the TensorFlow environment variable `TF_GPU_THREAD_MODE` to
+    `gpu_private`. This environment variable will tell the host to keep threads
+    for a GPU private.
+-   By default,`TF_GPU_THREAD_MODE=gpu_private` sets the number of threads to 2,
+    which is sufficient in most cases. However, that number can be changed by
+    setting the TensorFlow environment variable `TF_GPU_THREAD_COUNT` to the
+    desired number of threads.
diff --git a/site/en/guide/graph_optimization.ipynb b/site/en/guide/graph_optimization.ipynb
new file mode 100644
index 00000000000..063d8817489
--- /dev/null
+++ b/site/en/guide/graph_optimization.ipynb
@@ -0,0 +1,380 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FYLyuStTYesc"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "PVm-iEoxYesf"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3MPf91rVYesq"
+      },
+      "source": [
+        "# TensorFlow graph optimization with Grappler"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zmNCsZlgYesr"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l0qacLgyYess"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "TensorFlow uses both graph and eager executions to execute computations. A `tf.Graph` contains a set of `tf.Operation` objects (ops) which represent units of computation and `tf.Tensor` objects which represent the units of data that flow between ops.\n",
+        "\n",
+        "Grappler is the default graph optimization system in the TensorFlow runtime. Grappler applies optimizations in graph mode (within `tf.function`) to improve the performance of your TensorFlow computations through graph simplifications and other high-level optimizations such as inlining function bodies to enable inter-procedural optimizations. Optimizing the `tf.Graph` also reduces the device peak memory usage and improves hardware utilization by optimizing the mapping of graph nodes to compute resources. \n",
+        "\n",
+        "Use `tf.config.optimizer.set_experimental_options()` for finer control over your `tf.Graph` optimizations.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A-zkJgR5Yesw"
+      },
+      "source": [
+        "## Available graph optimizers\n",
+        "\n",
+        "Grappler performs graph optimizations through a top-level driver called the `MetaOptimizer`. The following graph optimizers are available with TensorFlow:  \n",
+        "\n",
+        "* *Constant folding optimizer -* Statically infers the value of tensors when possible by folding constant nodes in the graph and materializes the result using constants.\n",
+        "* *Arithmetic optimizer -* Simplifies arithmetic operations by eliminating common subexpressions and simplifying arithmetic statements.  \n",
+        "* *Layout optimizer -* Optimizes tensor layouts to execute data format dependent operations such as convolutions more efficiently.\n",
+        "* *Remapper optimizer -* Remaps subgraphs onto more efficient implementations by replacing commonly occurring subgraphs with optimized fused monolithic kernels.\n",
+        "* *Memory optimizer -* Analyzes the graph to inspect the peak memory usage for each operation and inserts CPU-GPU memory copy operations for swapping GPU memory to CPU to reduce the peak memory usage.\n",
+        "* *Dependency optimizer -* Removes or rearranges control dependencies to shorten the critical path for a model step or enables other\n",
+        "optimizations. Also removes nodes that are effectively no-ops such as Identity.\n",
+        "* *Pruning optimizer -* Prunes nodes that have no effect on the output from the graph. It is usually run first to reduce the size of the graph and speed up processing in other Grappler passes.\n",
+        "* *Function optimizer -* Optimizes the function library of a TensorFlow program and inlines function bodies to enable other inter-procedural optimizations.\n",
+        "* *Shape optimizer -* Optimizes subgraphs that operate on shape and shape related information.\n",
+        "* *Autoparallel optimizer -* Automatically parallelizes graphs by splitting along the batch dimension. This optimizer is turned OFF by default.\n",
+        "* *Loop optimizer -* Optimizes the graph control flow by hoisting loop-invariant subgraphs out of loops and by removing redundant stack operations in loops. Also optimizes loops with statically known trip counts and removes statically known dead branches in conditionals.\n",
+        "* *Scoped allocator optimizer -* Introduces scoped allocators to reduce data movement and to consolidate some operations.\n",
+        "* *Pin to host optimizer -* Swaps small operations onto the CPU. This optimizer is turned OFF by default. \n",
+        "* *Auto mixed precision optimizer -* Converts data types to float16 where applicable to improve performance. Currently applies to GPUs and the latest Intel Xeon CPUs.\n",
+        "* *Debug stripper -* Strips nodes related to debugging operations such as `tf.debugging.Assert`, `tf.debugging.check_numerics`, and `tf.print` from the graph. This optimizer is turned OFF by default."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WZAUsxyWYess"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6BRIDzO6ypoY"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "import timeit\n",
+        "import traceback\n",
+        "import contextlib\n",
+        "\n",
+        "\n",
+        "import tensorflow as tf"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1O-XL1nxJX0X"
+      },
+      "source": [
+        "Create a context manager to easily toggle optimizer states."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uRuhVoAlYesz"
+      },
+      "outputs": [],
+      "source": [
+        "@contextlib.contextmanager\n",
+        "def options(options):\n",
+        "  old_opts = tf.config.optimizer.get_experimental_options()\n",
+        "  tf.config.optimizer.set_experimental_options(options)\n",
+        "  try:\n",
+        "    yield\n",
+        "  finally:\n",
+        "    tf.config.optimizer.set_experimental_options(old_opts)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "E2o4kZtK0DoA"
+      },
+      "source": [
+        "## Compare execution performance with and without Grappler\n",
+        "\n",
+        "TensorFlow 2 and beyond executes eagerly by default. Use `tf.function` to switch the default execution to Graph mode. Grappler runs automatically in the background to apply the graph optimizations above and improve execution performance. \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3sh8RoLJ96IT"
+      },
+      "source": [
+        "### Constant folding optimizer\n",
+        "\n",
+        "As a preliminary example, consider a function which performs operations on constants and returns an output."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jOW_OSzMJEvN"
+      },
+      "outputs": [],
+      "source": [
+        "def test_function_1():\n",
+        "  @tf.function\n",
+        "  def simple_function(input_arg):\n",
+        "    print('Tracing!')\n",
+        "    a = tf.constant(np.random.randn(2000,2000), dtype = tf.float32)\n",
+        "    c = a\n",
+        "    for n in range(50):\n",
+        "      c = c@a\n",
+        "    return tf.reduce_mean(c+input_arg)\n",
+        "\n",
+        "  return simple_function"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tFVgUhhzLKIo"
+      },
+      "source": [
+        "Turn off the constant folding optimizer and execute the function:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KDMGsOHrJqKD"
+      },
+      "outputs": [],
+      "source": [
+        "with options({'constant_folding': False}):\n",
+        "  print(tf.config.optimizer.get_experimental_options())\n",
+        "  simple_function = test_function_1()\n",
+        "  # Trace once\n",
+        "  x = tf.constant(2.2)\n",
+        "  simple_function(x)\n",
+        "  print(\"Vanilla execution:\", timeit.timeit(lambda: simple_function(x), number = 1), \"s\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ykMXfo8qO41z"
+      },
+      "source": [
+        "Enable the constant folding optimizer and execute the function again to observe a speed-up in function execution."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "73pc0gfFKY8l"
+      },
+      "outputs": [],
+      "source": [
+        "with options({'constant_folding': True}):\n",
+        "  print(tf.config.optimizer.get_experimental_options())\n",
+        "  simple_function = test_function_1()\n",
+        "  # Trace once\n",
+        "  x = tf.constant(2.2)\n",
+        "  simple_function(x)\n",
+        "  print(\"Constant folded execution:\", timeit.timeit(lambda: simple_function(x), number = 1), \"s\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "83w8rfcRVhWb"
+      },
+      "source": [
+        "### Debug stripper optimizer\n",
+        "\n",
+        "Consider a simple function that checks the numeric value of its input argument and returns it. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "j2DvqEr8haut"
+      },
+      "outputs": [],
+      "source": [
+        "def test_function_2():\n",
+        "  @tf.function\n",
+        "  def simple_func(input_arg):\n",
+        "    output = input_arg\n",
+        "    tf.debugging.check_numerics(output, \"Bad!\")\n",
+        "    return output\n",
+        "  return simple_func"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ywKG3WRbpYB8"
+      },
+      "source": [
+        "First, execute the function with the debug stripper optimizer turned off. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LsE-y6iQWSwH"
+      },
+      "outputs": [],
+      "source": [
+        "test_func = test_function_2()\n",
+        "p1 = tf.constant(float('inf'))\n",
+        "try:\n",
+        "  test_func(p1)\n",
+        "except tf.errors.InvalidArgumentError as e:\n",
+        "  traceback.print_exc(limit=2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "URHpboM8xLN6"
+      },
+      "source": [
+        "`tf.debugging.check_numerics` raises an invalid argument error because of the `Inf` argument to `test_func`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CuPSha9YmJRo"
+      },
+      "source": [
+        "Enable the debug stripper optimizer and execute the function again. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UPJ7ygHnWP6B"
+      },
+      "outputs": [],
+      "source": [
+        "with options({'debug_stripper': True}):\n",
+        "  test_func2 = test_function_2()\n",
+        "  p1 = tf.constant(float('inf'))\n",
+        "  try:\n",
+        "    test_func2(p1)\n",
+        "  except tf.errors.InvalidArgumentError as e:\n",
+        "    traceback.print_exc(limit=2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nAsESNUB1QpI"
+      },
+      "source": [
+        "The debug stripper optimizer strips the `tf.debug.check_numerics` node from the graph and executes the function without raising any errors. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wHC6tR9GvFgW"
+      },
+      "source": [
+        "## Summary\n",
+        "\n",
+        "The TensorFlow runtime uses Grappler to optimize graphs automatically before execution. Use `tf.config.optimizer.set_experimental_options` to enable or disable the various graph optimizers. \n",
+        "\n",
+        "For more information on Grappler, see TensorFlow Graph Optimizations."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "graph_optimization.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/images/data_performance_analysis/async_long_name.png b/site/en/guide/images/data_performance_analysis/async_long_name.png
new file mode 100644
index 00000000000..92da9524770
Binary files /dev/null and b/site/en/guide/images/data_performance_analysis/async_long_name.png differ
diff --git a/site/en/guide/images/data_performance_analysis/example_1_cropped.png b/site/en/guide/images/data_performance_analysis/example_1_cropped.png
new file mode 100644
index 00000000000..bba67ef50e3
Binary files /dev/null and b/site/en/guide/images/data_performance_analysis/example_1_cropped.png differ
diff --git a/site/en/guide/images/data_performance_analysis/example_2_cropped.png b/site/en/guide/images/data_performance_analysis/example_2_cropped.png
new file mode 100644
index 00000000000..ff4d405c867
Binary files /dev/null and b/site/en/guide/images/data_performance_analysis/example_2_cropped.png differ
diff --git a/site/en/guide/images/data_performance_analysis/get_next_fast.png b/site/en/guide/images/data_performance_analysis/get_next_fast.png
new file mode 100644
index 00000000000..2e8de08e16a
Binary files /dev/null and b/site/en/guide/images/data_performance_analysis/get_next_fast.png differ
diff --git a/site/en/guide/images/data_performance_analysis/get_next_slow.png b/site/en/guide/images/data_performance_analysis/get_next_slow.png
new file mode 100644
index 00000000000..c441da49820
Binary files /dev/null and b/site/en/guide/images/data_performance_analysis/get_next_slow.png differ
diff --git a/site/en/guide/images/data_performance_analysis/map_long_name.png b/site/en/guide/images/data_performance_analysis/map_long_name.png
new file mode 100644
index 00000000000..b01474e8b8f
Binary files /dev/null and b/site/en/guide/images/data_performance_analysis/map_long_name.png differ
diff --git a/site/en/guide/images/data_performance_analysis/prefetch.png b/site/en/guide/images/data_performance_analysis/prefetch.png
new file mode 100644
index 00000000000..f51c568b541
Binary files /dev/null and b/site/en/guide/images/data_performance_analysis/prefetch.png differ
diff --git a/site/en/guide/images/data_performance_analysis/trace_viewer.png b/site/en/guide/images/data_performance_analysis/trace_viewer.png
new file mode 100644
index 00000000000..d600e58d46b
Binary files /dev/null and b/site/en/guide/images/data_performance_analysis/trace_viewer.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/gpu_kernels.png b/site/en/guide/images/gpu_perf_analysis/gpu_kernels.png
new file mode 100644
index 00000000000..a0646eaac21
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/gpu_kernels.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/input_pipeline_analyzer.png b/site/en/guide/images/gpu_perf_analysis/input_pipeline_analyzer.png
new file mode 100644
index 00000000000..a797a3db8e6
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/input_pipeline_analyzer.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/opp_placement.png b/site/en/guide/images/gpu_perf_analysis/opp_placement.png
new file mode 100644
index 00000000000..0c9fd96a4c3
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/opp_placement.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/overview_page.png b/site/en/guide/images/gpu_perf_analysis/overview_page.png
new file mode 100644
index 00000000000..d50c93fa856
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/overview_page.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/performance_summary.png b/site/en/guide/images/gpu_perf_analysis/performance_summary.png
new file mode 100644
index 00000000000..35e0b94a971
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/performance_summary.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/tensorflow_stats_page.png b/site/en/guide/images/gpu_perf_analysis/tensorflow_stats_page.png
new file mode 100644
index 00000000000..9642d6b2c69
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/tensorflow_stats_page.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/traceview_excessive_copy.png b/site/en/guide/images/gpu_perf_analysis/traceview_excessive_copy.png
new file mode 100644
index 00000000000..69326a7e31c
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/traceview_excessive_copy.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/traceview_gpu_idle.png b/site/en/guide/images/gpu_perf_analysis/traceview_gpu_idle.png
new file mode 100644
index 00000000000..37f005cf662
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/traceview_gpu_idle.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/traceview_host_contention.png b/site/en/guide/images/gpu_perf_analysis/traceview_host_contention.png
new file mode 100644
index 00000000000..ee6255a0ae0
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/traceview_host_contention.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/traceview_ideal.png b/site/en/guide/images/gpu_perf_analysis/traceview_ideal.png
new file mode 100644
index 00000000000..4f07d16ca27
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/traceview_ideal.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/traceview_kernel_gaps.png b/site/en/guide/images/gpu_perf_analysis/traceview_kernel_gaps.png
new file mode 100644
index 00000000000..712a004c179
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/traceview_kernel_gaps.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/traceview_multi_gpu.png b/site/en/guide/images/gpu_perf_analysis/traceview_multi_gpu.png
new file mode 100644
index 00000000000..8b9444860ad
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/traceview_multi_gpu.png differ
diff --git a/site/en/guide/images/gpu_perf_analysis/traceview_step_gaps.png b/site/en/guide/images/gpu_perf_analysis/traceview_step_gaps.png
new file mode 100644
index 00000000000..f27459f023b
Binary files /dev/null and b/site/en/guide/images/gpu_perf_analysis/traceview_step_gaps.png differ
diff --git a/site/en/guide/images/intro_to_graphs/two-layer-network.png b/site/en/guide/images/intro_to_graphs/two-layer-network.png
new file mode 100644
index 00000000000..5e50adf85f2
Binary files /dev/null and b/site/en/guide/images/intro_to_graphs/two-layer-network.png differ
diff --git a/site/en/guide/images/new_type_promotion/type_promotion_lattice.png b/site/en/guide/images/new_type_promotion/type_promotion_lattice.png
new file mode 100644
index 00000000000..501698965a2
Binary files /dev/null and b/site/en/guide/images/new_type_promotion/type_promotion_lattice.png differ
diff --git a/site/en/guide/images/new_type_promotion/type_promotion_table.png b/site/en/guide/images/new_type_promotion/type_promotion_table.png
new file mode 100644
index 00000000000..62bb465212a
Binary files /dev/null and b/site/en/guide/images/new_type_promotion/type_promotion_table.png differ
diff --git a/site/en/guide/images/sparse_tensor.png b/site/en/guide/images/sparse_tensor.png
new file mode 100644
index 00000000000..bce16ba85ca
Binary files /dev/null and b/site/en/guide/images/sparse_tensor.png differ
diff --git a/site/en/guide/images/tensor/3-axis_block.png b/site/en/guide/images/tensor/3-axis_block.png
new file mode 100644
index 00000000000..ed10248e19a
Binary files /dev/null and b/site/en/guide/images/tensor/3-axis_block.png differ
diff --git a/site/en/guide/images/tensor/3-axis_front.png b/site/en/guide/images/tensor/3-axis_front.png
new file mode 100644
index 00000000000..29411cf49e0
Binary files /dev/null and b/site/en/guide/images/tensor/3-axis_front.png differ
diff --git a/site/en/guide/images/tensor/3-axis_numpy.png b/site/en/guide/images/tensor/3-axis_numpy.png
new file mode 100644
index 00000000000..717c7aee995
Binary files /dev/null and b/site/en/guide/images/tensor/3-axis_numpy.png differ
diff --git a/site/en/guide/images/tensor/3-axis_stack.png b/site/en/guide/images/tensor/3-axis_stack.png
new file mode 100644
index 00000000000..62fe2cbb5e3
Binary files /dev/null and b/site/en/guide/images/tensor/3-axis_stack.png differ
diff --git a/site/en/guide/images/tensor/4-axis_block.png b/site/en/guide/images/tensor/4-axis_block.png
new file mode 100644
index 00000000000..0e48076c750
Binary files /dev/null and b/site/en/guide/images/tensor/4-axis_block.png differ
diff --git a/site/en/guide/images/tensor/broadcasting.png b/site/en/guide/images/tensor/broadcasting.png
new file mode 100644
index 00000000000..1e5af1468d9
Binary files /dev/null and b/site/en/guide/images/tensor/broadcasting.png differ
diff --git a/site/en/guide/images/tensor/index1.png b/site/en/guide/images/tensor/index1.png
new file mode 100644
index 00000000000..f4aadf2ef5c
Binary files /dev/null and b/site/en/guide/images/tensor/index1.png differ
diff --git a/site/en/guide/images/tensor/index2.png b/site/en/guide/images/tensor/index2.png
new file mode 100644
index 00000000000..daa1144e520
Binary files /dev/null and b/site/en/guide/images/tensor/index2.png differ
diff --git a/site/en/guide/images/tensor/matrix.png b/site/en/guide/images/tensor/matrix.png
new file mode 100644
index 00000000000..bd69a7a3091
Binary files /dev/null and b/site/en/guide/images/tensor/matrix.png differ
diff --git a/site/en/guide/images/tensor/ragged.png b/site/en/guide/images/tensor/ragged.png
new file mode 100644
index 00000000000..bed51e871c8
Binary files /dev/null and b/site/en/guide/images/tensor/ragged.png differ
diff --git a/site/en/guide/images/tensor/reshape-bad.png b/site/en/guide/images/tensor/reshape-bad.png
new file mode 100644
index 00000000000..eae337ef63f
Binary files /dev/null and b/site/en/guide/images/tensor/reshape-bad.png differ
diff --git a/site/en/guide/images/tensor/reshape-bad2.png b/site/en/guide/images/tensor/reshape-bad2.png
new file mode 100644
index 00000000000..5edbf307fdb
Binary files /dev/null and b/site/en/guide/images/tensor/reshape-bad2.png differ
diff --git a/site/en/guide/images/tensor/reshape-bad3.png b/site/en/guide/images/tensor/reshape-bad3.png
new file mode 100644
index 00000000000..e86c3d036bd
Binary files /dev/null and b/site/en/guide/images/tensor/reshape-bad3.png differ
diff --git a/site/en/guide/images/tensor/reshape-bad4.png b/site/en/guide/images/tensor/reshape-bad4.png
new file mode 100644
index 00000000000..1771047b122
Binary files /dev/null and b/site/en/guide/images/tensor/reshape-bad4.png differ
diff --git a/site/en/guide/images/tensor/reshape-before.png b/site/en/guide/images/tensor/reshape-before.png
new file mode 100644
index 00000000000..9a8359b6a4b
Binary files /dev/null and b/site/en/guide/images/tensor/reshape-before.png differ
diff --git a/site/en/guide/images/tensor/reshape-good1.png b/site/en/guide/images/tensor/reshape-good1.png
new file mode 100644
index 00000000000..0a96cc091dd
Binary files /dev/null and b/site/en/guide/images/tensor/reshape-good1.png differ
diff --git a/site/en/guide/images/tensor/reshape-good2.png b/site/en/guide/images/tensor/reshape-good2.png
new file mode 100644
index 00000000000..5780e8906c0
Binary files /dev/null and b/site/en/guide/images/tensor/reshape-good2.png differ
diff --git a/site/en/guide/images/tensor/scalar.png b/site/en/guide/images/tensor/scalar.png
new file mode 100644
index 00000000000..112e35d282d
Binary files /dev/null and b/site/en/guide/images/tensor/scalar.png differ
diff --git a/site/en/guide/images/tensor/shape.png b/site/en/guide/images/tensor/shape.png
new file mode 100644
index 00000000000..76dd56356fb
Binary files /dev/null and b/site/en/guide/images/tensor/shape.png differ
diff --git a/site/en/guide/images/tensor/shape2.png b/site/en/guide/images/tensor/shape2.png
new file mode 100644
index 00000000000..a316359c8fc
Binary files /dev/null and b/site/en/guide/images/tensor/shape2.png differ
diff --git a/site/en/guide/images/tensor/sparse.png b/site/en/guide/images/tensor/sparse.png
new file mode 100644
index 00000000000..3e0d0454519
Binary files /dev/null and b/site/en/guide/images/tensor/sparse.png differ
diff --git a/site/en/guide/images/tensor/string-split.png b/site/en/guide/images/tensor/string-split.png
new file mode 100644
index 00000000000..272e6b79244
Binary files /dev/null and b/site/en/guide/images/tensor/string-split.png differ
diff --git a/site/en/guide/images/tensor/strings.png b/site/en/guide/images/tensor/strings.png
new file mode 100644
index 00000000000..1e91bf26c63
Binary files /dev/null and b/site/en/guide/images/tensor/strings.png differ
diff --git a/site/en/guide/images/tensor/vector.png b/site/en/guide/images/tensor/vector.png
new file mode 100644
index 00000000000..740a642b6db
Binary files /dev/null and b/site/en/guide/images/tensor/vector.png differ
diff --git a/site/en/guide/images/tensorboard_graph.png b/site/en/guide/images/tensorboard_graph.png
new file mode 100644
index 00000000000..44584996cec
Binary files /dev/null and b/site/en/guide/images/tensorboard_graph.png differ
diff --git a/site/en/guide/images/tf_profiler/capture_profile.png b/site/en/guide/images/tf_profiler/capture_profile.png
new file mode 100644
index 00000000000..9e0b8c3415a
Binary files /dev/null and b/site/en/guide/images/tf_profiler/capture_profile.png differ
diff --git a/site/en/guide/images/tf_profiler/gpu_kernel_stats.png b/site/en/guide/images/tf_profiler/gpu_kernel_stats.png
new file mode 100644
index 00000000000..ad4f5428945
Binary files /dev/null and b/site/en/guide/images/tf_profiler/gpu_kernel_stats.png differ
diff --git a/site/en/guide/images/tf_profiler/input_op_stats.png b/site/en/guide/images/tf_profiler/input_op_stats.png
new file mode 100644
index 00000000000..86d7914ed56
Binary files /dev/null and b/site/en/guide/images/tf_profiler/input_op_stats.png differ
diff --git a/site/en/guide/images/tf_profiler/input_pipeline_analyzer.png b/site/en/guide/images/tf_profiler/input_pipeline_analyzer.png
new file mode 100644
index 00000000000..2ca02128f58
Binary files /dev/null and b/site/en/guide/images/tf_profiler/input_pipeline_analyzer.png differ
diff --git a/site/en/guide/images/tf_profiler/memory_breakdown_table.png b/site/en/guide/images/tf_profiler/memory_breakdown_table.png
new file mode 100644
index 00000000000..568d2bae9ef
Binary files /dev/null and b/site/en/guide/images/tf_profiler/memory_breakdown_table.png differ
diff --git a/site/en/guide/images/tf_profiler/memory_profile_summary.png b/site/en/guide/images/tf_profiler/memory_profile_summary.png
new file mode 100644
index 00000000000..a5ab25c0156
Binary files /dev/null and b/site/en/guide/images/tf_profiler/memory_profile_summary.png differ
diff --git a/site/en/guide/images/tf_profiler/memory_timeline_graph.png b/site/en/guide/images/tf_profiler/memory_timeline_graph.png
new file mode 100644
index 00000000000..5c8ad7d6eb6
Binary files /dev/null and b/site/en/guide/images/tf_profiler/memory_timeline_graph.png differ
diff --git a/site/en/guide/images/tf_profiler/memory_timeline_graph_popup.png b/site/en/guide/images/tf_profiler/memory_timeline_graph_popup.png
new file mode 100644
index 00000000000..96c84a80c55
Binary files /dev/null and b/site/en/guide/images/tf_profiler/memory_timeline_graph_popup.png differ
diff --git a/site/en/guide/images/tf_profiler/overview_page.png b/site/en/guide/images/tf_profiler/overview_page.png
new file mode 100644
index 00000000000..e7057ef1f59
Binary files /dev/null and b/site/en/guide/images/tf_profiler/overview_page.png differ
diff --git a/site/en/guide/images/tf_profiler/pod_viewer.png b/site/en/guide/images/tf_profiler/pod_viewer.png
new file mode 100644
index 00000000000..13f0e3ef8c6
Binary files /dev/null and b/site/en/guide/images/tf_profiler/pod_viewer.png differ
diff --git a/site/en/guide/images/tf_profiler/python_tracer.png b/site/en/guide/images/tf_profiler/python_tracer.png
new file mode 100644
index 00000000000..2da04e0611f
Binary files /dev/null and b/site/en/guide/images/tf_profiler/python_tracer.png differ
diff --git a/site/en/guide/images/tf_profiler/tf_data_all_hosts.png b/site/en/guide/images/tf_profiler/tf_data_all_hosts.png
new file mode 100644
index 00000000000..42287b32f04
Binary files /dev/null and b/site/en/guide/images/tf_profiler/tf_data_all_hosts.png differ
diff --git a/site/en/guide/images/tf_profiler/tf_data_graph.png b/site/en/guide/images/tf_profiler/tf_data_graph.png
new file mode 100644
index 00000000000..0b9963d54d9
Binary files /dev/null and b/site/en/guide/images/tf_profiler/tf_data_graph.png differ
diff --git a/site/en/guide/images/tf_profiler/tf_data_graph_selector.png b/site/en/guide/images/tf_profiler/tf_data_graph_selector.png
new file mode 100644
index 00000000000..9573467ff6a
Binary files /dev/null and b/site/en/guide/images/tf_profiler/tf_data_graph_selector.png differ
diff --git a/site/en/guide/images/tf_profiler/tf_data_summary.png b/site/en/guide/images/tf_profiler/tf_data_summary.png
new file mode 100644
index 00000000000..9cd9628beb1
Binary files /dev/null and b/site/en/guide/images/tf_profiler/tf_data_summary.png differ
diff --git a/site/en/guide/images/tf_profiler/tf_stats.png b/site/en/guide/images/tf_profiler/tf_stats.png
new file mode 100644
index 00000000000..3b6564ca33f
Binary files /dev/null and b/site/en/guide/images/tf_profiler/tf_stats.png differ
diff --git a/site/en/guide/images/tf_profiler/trace_viewer.png b/site/en/guide/images/tf_profiler/trace_viewer.png
new file mode 100644
index 00000000000..99a53d4513f
Binary files /dev/null and b/site/en/guide/images/tf_profiler/trace_viewer.png differ
diff --git a/site/en/guide/images/tf_slicing/gather_1.png b/site/en/guide/images/tf_slicing/gather_1.png
new file mode 100644
index 00000000000..895bf469788
Binary files /dev/null and b/site/en/guide/images/tf_slicing/gather_1.png differ
diff --git a/site/en/guide/images/tf_slicing/gather_2.png b/site/en/guide/images/tf_slicing/gather_2.png
new file mode 100644
index 00000000000..69411cfe791
Binary files /dev/null and b/site/en/guide/images/tf_slicing/gather_2.png differ
diff --git a/site/en/guide/images/tf_slicing/gather_nd_sparse.png b/site/en/guide/images/tf_slicing/gather_nd_sparse.png
new file mode 100644
index 00000000000..644ae601178
Binary files /dev/null and b/site/en/guide/images/tf_slicing/gather_nd_sparse.png differ
diff --git a/site/en/guide/images/tf_slicing/slice_1d_1.png b/site/en/guide/images/tf_slicing/slice_1d_1.png
new file mode 100644
index 00000000000..c7dcd7a5c4e
Binary files /dev/null and b/site/en/guide/images/tf_slicing/slice_1d_1.png differ
diff --git a/site/en/guide/images/tf_slicing/slice_1d_2.png b/site/en/guide/images/tf_slicing/slice_1d_2.png
new file mode 100644
index 00000000000..4170a1ec893
Binary files /dev/null and b/site/en/guide/images/tf_slicing/slice_1d_2.png differ
diff --git a/site/en/guide/images/tf_slicing/slice_1d_3.png b/site/en/guide/images/tf_slicing/slice_1d_3.png
new file mode 100644
index 00000000000..9323042d4b7
Binary files /dev/null and b/site/en/guide/images/tf_slicing/slice_1d_3.png differ
diff --git a/site/en/guide/images/tf_slicing/slice_2d_1.png b/site/en/guide/images/tf_slicing/slice_2d_1.png
new file mode 100644
index 00000000000..984a1b7783c
Binary files /dev/null and b/site/en/guide/images/tf_slicing/slice_2d_1.png differ
diff --git a/site/en/guide/intro_to_graphs.ipynb b/site/en/guide/intro_to_graphs.ipynb
new file mode 100644
index 00000000000..4fe442632ba
--- /dev/null
+++ b/site/en/guide/intro_to_graphs.ipynb
@@ -0,0 +1,874 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N7ITxKLUkX0v"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "yOYx6tzSnWQ3"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6xgB0Oz5eGSQ"
+      },
+      "source": [
+        "# Introduction to graphs and tf.function"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "w4zzZVZtQb1w"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RBKqnXI9GOax"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This guide goes beneath the surface of TensorFlow and Keras to demonstrate how TensorFlow works. If you instead want to immediately get started with Keras, check out the [collection of Keras guides](https://www.tensorflow.org/guide/keras/).\n",
+        "\n",
+        "In this guide, you'll learn how TensorFlow allows you to make simple changes to your code to get graphs, how graphs are stored and represented, and how you can use them to accelerate your models.\n",
+        "\n",
+        "Note: For those of you who are only familiar with TensorFlow 1.x, this guide demonstrates a very different view of graphs.\n",
+        "\n",
+        "**This is a big-picture overview that covers how `tf.function` allows you to switch from eager execution to graph execution.** For a more complete specification of `tf.function`, go to the [Better performance with `tf.function`](./function.ipynb) guide.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v0DdlfacAdTZ"
+      },
+      "source": [
+        "### What are graphs?\n",
+        "\n",
+        "In the previous three guides, you ran TensorFlow **eagerly**. This means TensorFlow operations are executed by Python, operation by operation, and return results back to Python.\n",
+        "\n",
+        "While eager execution has several unique advantages, graph execution enables portability outside Python and tends to offer better performance. **Graph execution** means that tensor computations are executed as a *TensorFlow graph*, sometimes referred to as a `tf.Graph` or simply a \"graph.\"\n",
+        "\n",
+        "**Graphs are data structures that contain a set of `tf.Operation` objects, which represent units of computation; and `tf.Tensor` objects, which represent the units of data that flow between operations.** They are defined in a `tf.Graph` context. Since these graphs are data structures, they can be saved, run, and restored all without the original Python code.\n",
+        "\n",
+        "This is what a TensorFlow graph representing a two-layer neural network looks like when visualized in TensorBoard:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FvQ5aBuRGT1o"
+      },
+      "source": [
+        "\"A"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DHpY3avXGITP"
+      },
+      "source": [
+        "### The benefits of graphs\n",
+        "\n",
+        "With a graph, you have a great deal of flexibility.  You can use your TensorFlow graph in environments that don't have a Python interpreter, like mobile applications, embedded devices, and backend servers. TensorFlow uses graphs as the format for [saved models](./saved_model.ipynb) when it exports them from Python.\n",
+        "\n",
+        "Graphs are also easily optimized, allowing the compiler to do transformations like:\n",
+        "\n",
+        "* Statically infer the value of tensors by folding constant nodes in your computation *(\"constant folding\")*.\n",
+        "* Separate sub-parts of a computation that are independent and split them between threads or devices.\n",
+        "* Simplify arithmetic operations by eliminating common subexpressions.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o1x1EOD9GjnB"
+      },
+      "source": [
+        "There is an entire optimization system, [Grappler](./graph_optimization.ipynb), to perform this and other speedups.\n",
+        "\n",
+        "In short, graphs are extremely useful and let your TensorFlow run **fast**, run **in parallel**, and run efficiently **on multiple devices**.\n",
+        "\n",
+        "However, you still want to define your machine learning models (or other computations) in Python for convenience, and then automatically construct graphs when you need them."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k-6Qi0thw2i9"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0d1689fa928f"
+      },
+      "source": [
+        "Import some necessary libraries:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "goZwOXp_xyQj"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import timeit\n",
+        "from datetime import datetime"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pSZebVuWxDXu"
+      },
+      "source": [
+        "## Taking advantage of graphs\n",
+        "\n",
+        "You create and run a graph in TensorFlow by using `tf.function`, either as a direct call or as a decorator. `tf.function` takes a regular function as input and returns a `tf.types.experimental.PolymorphicFunction`. **A `PolymorphicFunction` is a Python callable that builds TensorFlow graphs from the Python function. You use a `tf.function` in the same way as its Python equivalent.**\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HKbLeJ1y0Umi"
+      },
+      "outputs": [],
+      "source": [
+        "# Define a Python function.\n",
+        "def a_regular_function(x, y, b):\n",
+        "  x = tf.matmul(x, y)\n",
+        "  x = x + b\n",
+        "  return x\n",
+        "\n",
+        "# The Python type of `a_function_that_uses_a_graph` will now be a\n",
+        "# `PolymorphicFunction`.\n",
+        "a_function_that_uses_a_graph = tf.function(a_regular_function)\n",
+        "\n",
+        "# Make some tensors.\n",
+        "x1 = tf.constant([[1.0, 2.0]])\n",
+        "y1 = tf.constant([[2.0], [3.0]])\n",
+        "b1 = tf.constant(4.0)\n",
+        "\n",
+        "orig_value = a_regular_function(x1, y1, b1).numpy()\n",
+        "# Call a `tf.function` like a Python function.\n",
+        "tf_function_value = a_function_that_uses_a_graph(x1, y1, b1).numpy()\n",
+        "assert(orig_value == tf_function_value)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PNvuAYpdrTOf"
+      },
+      "source": [
+        "On the outside, a `tf.function` looks like a regular function you write using TensorFlow operations. [Underneath](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/eager/polymorphic_function/polymorphic_function.py), however, it is *very different*. The underlying `PolymorphicFunction` **encapsulates several `tf.Graph`s behind one API** (learn more in the _Polymorphism_ section). That is how a `tf.function` is able to give you the benefits of graph execution, like speed and deployability (refer to _The benefits of graphs_ above)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MT7U8ozok0gV"
+      },
+      "source": [
+        "`tf.function` applies to a function *and all other functions it calls*:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rpz08iLplm9F"
+      },
+      "outputs": [],
+      "source": [
+        "def inner_function(x, y, b):\n",
+        "  x = tf.matmul(x, y)\n",
+        "  x = x + b\n",
+        "  return x\n",
+        "\n",
+        "# Using the `tf.function` decorator makes `outer_function` into a\n",
+        "# `PolymorphicFunction`.\n",
+        "@tf.function\n",
+        "def outer_function(x):\n",
+        "  y = tf.constant([[2.0], [3.0]])\n",
+        "  b = tf.constant(4.0)\n",
+        "\n",
+        "  return inner_function(x, y, b)\n",
+        "\n",
+        "# Note that the callable will create a graph that\n",
+        "# includes `inner_function` as well as `outer_function`.\n",
+        "outer_function(tf.constant([[1.0, 2.0]])).numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "P88fOr88qgCj"
+      },
+      "source": [
+        "If you have used TensorFlow 1.x, you will notice that at no time did you need to define a `Placeholder` or `tf.Session`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wfeKf0Nr1OEK"
+      },
+      "source": [
+        "### Converting Python functions to graphs\n",
+        "\n",
+        "Any function you write with TensorFlow will contain a mixture of built-in TF operations and Python logic, such as `if-then` clauses, loops, `break`, `return`, `continue`, and more. While TensorFlow operations are easily captured by a `tf.Graph`, Python-specific logic needs to undergo an extra step in order to become part of the graph. `tf.function` uses a library called AutoGraph (`tf.autograph`) to convert Python code into graph-generating code.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PFObpff1BMEb"
+      },
+      "outputs": [],
+      "source": [
+        "def simple_relu(x):\n",
+        "  if tf.greater(x, 0):\n",
+        "    return x\n",
+        "  else:\n",
+        "    return 0\n",
+        "\n",
+        "# Using `tf.function` makes `tf_simple_relu` a `PolymorphicFunction` that wraps\n",
+        "# `simple_relu`.\n",
+        "tf_simple_relu = tf.function(simple_relu)\n",
+        "\n",
+        "print(\"First branch, with graph:\", tf_simple_relu(tf.constant(1)).numpy())\n",
+        "print(\"Second branch, with graph:\", tf_simple_relu(tf.constant(-1)).numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hO4DBUNZBMwQ"
+      },
+      "source": [
+        "Though it is unlikely that you will need to view graphs directly, you can inspect the outputs to check the exact results. These are not easy to read, so no need to look too carefully!"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lAKaat3w0gnn"
+      },
+      "outputs": [],
+      "source": [
+        "# This is the graph-generating output of AutoGraph.\n",
+        "print(tf.autograph.to_code(simple_relu))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8x6RAqza1UWf"
+      },
+      "outputs": [],
+      "source": [
+        "# This is the graph itself.\n",
+        "print(tf_simple_relu.get_concrete_function(tf.constant(1)).graph.as_graph_def())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GZ4Ieg6tBE6l"
+      },
+      "source": [
+        "Most of the time, `tf.function` will work without  special considerations. However, there are some caveats, and the [`tf.function` guide](./function.ipynb) can help here, as well as the [complete AutoGraph reference](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/index.md)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sIpc_jfjEZEg"
+      },
+      "source": [
+        "### Polymorphism: one `tf.function`, many graphs\n",
+        "\n",
+        "A `tf.Graph` is specialized to a specific type of inputs (for example, tensors with a specific [`dtype`](https://www.tensorflow.org/api_docs/python/tf/dtypes/DType) or objects with the same [`id()`](https://docs.python.org/3/library/functions.html#id)).\n",
+        "\n",
+        "Each time you invoke a `tf.function` with a set of arguments that can't be handled by any of its existing graphs (such as arguments with new `dtypes` or incompatible shapes), it creates a new `tf.Graph` specialized to those new arguments. The type specification of a `tf.Graph`'s inputs is represented by `tf.types.experimental.FunctionType`, also referred to as the **signature**. For more information regarding when a new `tf.Graph` is generated, how that can be controlled, and how `FunctionType` can be useful, go to the _Rules of tracing_ section of the [Better performance with `tf.function`](./function.ipynb) guide.\n",
+        "\n",
+        "The `tf.function` stores the `tf.Graph` corresponding to that signature in a `ConcreteFunction`. **A `ConcreteFunction` can be thought of as a wrapper around a `tf.Graph`.**\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LOASwhbvIv_T"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def my_relu(x):\n",
+        "  return tf.maximum(0., x)\n",
+        "\n",
+        "# `my_relu` creates new graphs as it observes different input types.\n",
+        "print(my_relu(tf.constant(5.5)))\n",
+        "print(my_relu([1, -1]))\n",
+        "print(my_relu(tf.constant([3., -3.])))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1qRtw7R4KL9X"
+      },
+      "source": [
+        "If the `tf.function` has already been called with the same input types, it does not create a new `tf.Graph`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TjjbnL5OKNDP"
+      },
+      "outputs": [],
+      "source": [
+        "# These two calls do *not* create new graphs.\n",
+        "print(my_relu(tf.constant(-2.5))) # Input type matches `tf.constant(5.5)`.\n",
+        "print(my_relu(tf.constant([-1., 1.]))) # Input type matches `tf.constant([3., -3.])`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UohRmexhIpvQ"
+      },
+      "source": [
+        "Because it's backed by multiple graphs, a `tf.function` is (as the name \"PolymorphicFunction\" suggests) **polymorphic**. That enables it to support more input types than a single `tf.Graph` could represent, and to optimize each `tf.Graph` for better performance."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dxzqebDYFmLy"
+      },
+      "outputs": [],
+      "source": [
+        "# There are three `ConcreteFunction`s (one for each graph) in `my_relu`.\n",
+        "# The `ConcreteFunction` also knows the return type and shape!\n",
+        "print(my_relu.pretty_printed_concrete_signatures())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "V11zkxU22XeD"
+      },
+      "source": [
+        "## Using `tf.function`\n",
+        "\n",
+        "So far, you've learned how to convert a Python function into a graph simply by using `tf.function` as a decorator or wrapper. But in practice, getting `tf.function` to work correctly can be tricky! In the following sections, you'll learn how you can make your code work as expected with `tf.function`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yp_n0B5-P0RU"
+      },
+      "source": [
+        "### Graph execution vs. eager execution\n",
+        "\n",
+        "The code in a `tf.function` can be executed both eagerly and as a graph. By default, `tf.function` executes its code as a graph:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_R0BOvBFxqVZ"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def get_MSE(y_true, y_pred):\n",
+        "  sq_diff = tf.pow(y_true - y_pred, 2)\n",
+        "  return tf.reduce_mean(sq_diff)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zikMVPGhmDET"
+      },
+      "outputs": [],
+      "source": [
+        "y_true = tf.random.uniform([5], maxval=10, dtype=tf.int32)\n",
+        "y_pred = tf.random.uniform([5], maxval=10, dtype=tf.int32)\n",
+        "print(y_true)\n",
+        "print(y_pred)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "07r08Dh158ft"
+      },
+      "outputs": [],
+      "source": [
+        "get_MSE(y_true, y_pred)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cyZNCRcQorGO"
+      },
+      "source": [
+        "To verify that your `tf.function`'s graph is doing the same computation as its equivalent Python function, you can make it execute eagerly with `tf.config.run_functions_eagerly(True)`. This is a switch that **turns off `tf.function`'s ability to create and run graphs**, instead of executing the code normally."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lKoF6NjPoI8w"
+      },
+      "outputs": [],
+      "source": [
+        "tf.config.run_functions_eagerly(True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9ZLqTyn0oKeM"
+      },
+      "outputs": [],
+      "source": [
+        "get_MSE(y_true, y_pred)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cV7daQW9odn-"
+      },
+      "outputs": [],
+      "source": [
+        "# Don't forget to set it back when you are done.\n",
+        "tf.config.run_functions_eagerly(False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DKT3YBsqy0x4"
+      },
+      "source": [
+        "However, `tf.function` can behave differently under graph and eager execution. The Python [`print`](https://docs.python.org/3/library/functions.html#print) function is one example of how these two modes differ. Let's check out what happens when you insert a `print` statement to your function and call it repeatedly."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BEJeVeBEoGjV"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def get_MSE(y_true, y_pred):\n",
+        "  print(\"Calculating MSE!\")\n",
+        "  sq_diff = tf.pow(y_true - y_pred, 2)\n",
+        "  return tf.reduce_mean(sq_diff)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3sWTGwX3BzP1"
+      },
+      "source": [
+        "Observe what is printed:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3rJIeBg72T9n"
+      },
+      "outputs": [],
+      "source": [
+        "error = get_MSE(y_true, y_pred)\n",
+        "error = get_MSE(y_true, y_pred)\n",
+        "error = get_MSE(y_true, y_pred)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WLMXk1uxKQ44"
+      },
+      "source": [
+        "Is the output surprising? **`get_MSE` only printed once even though it was called *three* times.**\n",
+        "\n",
+        "To explain, the `print` statement is executed when `tf.function` runs the original code in order to create the graph in a process known as \"tracing\" (refer to the _Tracing_ section of the [`tf.function` guide](./function.ipynb). **Tracing captures the TensorFlow operations into a graph, and `print` is not captured in the graph.**  That graph is then executed for all three calls **without ever running the Python code again**.\n",
+        "\n",
+        "As a sanity check, let's turn off graph execution to compare:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oFSxRtcptYpe"
+      },
+      "outputs": [],
+      "source": [
+        "# Now, globally set everything to run eagerly to force eager execution.\n",
+        "tf.config.run_functions_eagerly(True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qYxrAtvzNgHR"
+      },
+      "outputs": [],
+      "source": [
+        "# Observe what is printed below.\n",
+        "error = get_MSE(y_true, y_pred)\n",
+        "error = get_MSE(y_true, y_pred)\n",
+        "error = get_MSE(y_true, y_pred)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_Df6ynXcAaup"
+      },
+      "outputs": [],
+      "source": [
+        "tf.config.run_functions_eagerly(False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PUR7qC_bquCn"
+      },
+      "source": [
+        "`print` is a *Python side effect*, and there are other differences that you should be aware of when converting a function into a `tf.function`. Learn more in the _Limitations_ section of the [Better performance with `tf.function`](./function.ipynb) guide."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oTZJfV_tccVp"
+      },
+      "source": [
+        "Note: If you would like to print values in both eager and graph execution, use `tf.print` instead."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rMT_Xf5yKn9o"
+      },
+      "source": [
+        "### Non-strict execution\n",
+        "\n",
+        "\n",
+        "\n",
+        "Graph execution only executes the operations necessary to produce the observable effects, which include:\n",
+        "\n",
+        "- The return value of the function\n",
+        "- Documented well-known side-effects such as:\n",
+        "  - Input/output operations, like `tf.print`\n",
+        "  - Debugging operations, such as the assert functions in `tf.debugging`\n",
+        "  - Mutations of `tf.Variable`\n",
+        "\n",
+        "This behavior is usually known as \"Non-strict execution\", and differs from eager execution, which steps through all of the program operations, needed or not.\n",
+        "\n",
+        "In particular, runtime error checking does not count as an observable effect. If an operation is skipped because it is unnecessary, it cannot raise any runtime errors.\n",
+        "\n",
+        "In the following example, the \"unnecessary\" operation `tf.gather` is skipped during graph execution, so the runtime error `InvalidArgumentError` is not raised as it would be in eager execution. Do not rely on an error being raised while executing a graph."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OdN0nKlUwj7M"
+      },
+      "outputs": [],
+      "source": [
+        "def unused_return_eager(x):\n",
+        "  # Get index 1 will fail when `len(x) == 1`\n",
+        "  tf.gather(x, [1]) # unused \n",
+        "  return x\n",
+        "\n",
+        "try:\n",
+        "  print(unused_return_eager(tf.constant([0.0])))\n",
+        "except tf.errors.InvalidArgumentError as e:\n",
+        "  # All operations are run during eager execution so an error is raised.\n",
+        "  print(f'{type(e).__name__}: {e}')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d80Fob4MwhTs"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def unused_return_graph(x):\n",
+        "  tf.gather(x, [1]) # unused\n",
+        "  return x\n",
+        "\n",
+        "# Only needed operations are run during graph execution. The error is not raised.\n",
+        "print(unused_return_graph(tf.constant([0.0])))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "def6MupG9R0O"
+      },
+      "source": [
+        "### `tf.function` best practices\n",
+        "\n",
+        "It may take some time to get used to the behavior of `tf.function`.  To get started quickly, first-time users should play around with decorating toy functions with `@tf.function` to get experience with going from eager to graph execution.\n",
+        "\n",
+        "*Designing for `tf.function`* may be your best bet for writing graph-compatible TensorFlow programs. Here are some tips:\n",
+        "-  Toggle between eager and graph execution early and often with `tf.config.run_functions_eagerly` to pinpoint if/ when the two modes diverge.\n",
+        "- Create `tf.Variable`s\n",
+        "outside the Python function and modify them on the inside. The same goes for objects that use `tf.Variable`, like `tf.keras.layers`, `tf.keras.Model`s and `tf.keras.optimizers`.\n",
+        "- Avoid writing functions that depend on outer Python variables, excluding `tf.Variable`s and Keras objects. Learn more in _Depending on Python global and free variables_ of the [`tf.function` guide](./function.ipynb).\n",
+        "- Prefer to write functions which take tensors and other TensorFlow types as input. You can pass in other object types but be careful! Learn more in _Depending on Python objects_ of the [`tf.function` guide](./function.ipynb).\n",
+        "- Include as much computation as possible under a `tf.function` to maximize the performance gain. For example, decorate a whole training step or the entire training loop.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ViM3oBJVJrDx"
+      },
+      "source": [
+        "## Seeing the speed-up"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A6NHDp7vAKcJ"
+      },
+      "source": [
+        "`tf.function` usually improves the performance of your code, but the amount of speed-up depends on the kind of computation you run. Small computations can be dominated by the overhead of calling a graph. You can measure the difference in performance like so:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jr7p1BBjauPK"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.random.uniform(shape=[10, 10], minval=-1, maxval=2, dtype=tf.dtypes.int32)\n",
+        "\n",
+        "def power(x, y):\n",
+        "  result = tf.eye(10, dtype=tf.dtypes.int32)\n",
+        "  for _ in range(y):\n",
+        "    result = tf.matmul(x, result)\n",
+        "  return result"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ms2yJyAnUYxK"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Eager execution:\", timeit.timeit(lambda: power(x, 100), number=1000), \"seconds\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gUB2mTyRYRAe"
+      },
+      "outputs": [],
+      "source": [
+        "power_as_graph = tf.function(power)\n",
+        "print(\"Graph execution:\", timeit.timeit(lambda: power_as_graph(x, 100), number=1000), \"seconds\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q1Pfo5YwwILi"
+      },
+      "source": [
+        "`tf.function` is commonly used to speed up training loops, and you can learn more about it in the _Speeding-up your training step with `tf.function`_ section of the [Writing a training loop from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) with Keras guide.\n",
+        "\n",
+        "Note: You can also try `tf.function(jit_compile=True)` for a more significant performance boost, especially if your code is heavy on TensorFlow control flow and uses many small tensors. Learn more in the _Explicit compilation with `tf.function(jit_compile=True)`_ section of the [XLA overview](https://www.tensorflow.org/xla)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sm0bNFp8PX53"
+      },
+      "source": [
+        "### Performance and trade-offs\n",
+        "\n",
+        "Graphs can speed up your code, but the process of creating them has some overhead. For some functions, the creation of the graph takes more time than the execution of the graph. **This investment is usually quickly paid back with the performance boost of subsequent executions, but it's important to be aware that the first few  steps of any large model training can be slower due to tracing.**\n",
+        "\n",
+        "No matter how large your model, you want to avoid tracing frequently. In the _Controlling retracing_ section, the [`tf.function` guide](./function.ipynb) discusses how to set input specifications and use tensor arguments to avoid retracing. If you find you are getting unusually poor performance, it's a good idea to check if you are retracing accidentally."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "F4InDaTjwmBA"
+      },
+      "source": [
+        "## When is a `tf.function` tracing?\n",
+        "\n",
+        "To figure out when your `tf.function` is tracing, add a `print` statement to its code. As a rule of thumb, `tf.function` will execute the `print` statement every time it traces."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hXtwlbpofLgW"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def a_function_with_python_side_effect(x):\n",
+        "  print(\"Tracing!\") # An eager-only side effect.\n",
+        "  return x * x + tf.constant(2)\n",
+        "\n",
+        "# This is traced the first time.\n",
+        "print(a_function_with_python_side_effect(tf.constant(2)))\n",
+        "# The second time through, you won't see the side effect.\n",
+        "print(a_function_with_python_side_effect(tf.constant(3)))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "inzSg8yzfNjl"
+      },
+      "outputs": [],
+      "source": [
+        "# This retraces each time the Python argument changes,\n",
+        "# as a Python argument could be an epoch count or other\n",
+        "# hyperparameter.\n",
+        "print(a_function_with_python_side_effect(2))\n",
+        "print(a_function_with_python_side_effect(3))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rtN8NW6AfKye"
+      },
+      "source": [
+        "New Python arguments always trigger the creation of a new graph, hence the extra tracing.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "D1kbr5ocpS6R"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "You can learn more about `tf.function` on the API reference page and by following the [Better performance with `tf.function`](./function.ipynb) guide."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "intro_to_graphs.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/intro_to_modules.ipynb b/site/en/guide/intro_to_modules.ipynb
new file mode 100644
index 00000000000..79bbe89ca56
--- /dev/null
+++ b/site/en/guide/intro_to_modules.ipynb
@@ -0,0 +1,1139 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ISubpr_SSsiM"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "3jTMb1dySr3V"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6DWfyNThSziV"
+      },
+      "source": [
+        "# Introduction to modules, layers, and models\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v0DdlfacAdTZ"
+      },
+      "source": [
+        "To do machine learning in TensorFlow, you are likely to need to define, save, and restore a model.\n",
+        "\n",
+        "A model is, abstractly: \n",
+        "\n",
+        "* A function that computes something on tensors (a **forward pass**)\n",
+        "* Some variables that can be updated in response to training\n",
+        "\n",
+        "In this guide, you will go below the surface of Keras to see how TensorFlow models are defined. This looks at how TensorFlow collects variables and models, as well as how they are saved and restored.\n",
+        "\n",
+        "Note: If you instead want to immediately get started with Keras, please see [the collection of Keras guides](./keras/).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VSa6ayJmfZxZ"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "goZwOXp_xyQj"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import keras\n",
+        "from datetime import datetime\n",
+        "\n",
+        "%load_ext tensorboard"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yt5HEbsYAbw1"
+      },
+      "source": [
+        "## TensorFlow Modules\n",
+        "\n",
+        "Most models are made of layers. Layers are functions with a known mathematical structure that can be reused and have trainable variables.  In TensorFlow, most high-level implementations of layers and models, such as Keras or [Sonnet](https://github.com/deepmind/sonnet), are built on the same foundational class: `tf.Module`.\n",
+        "\n",
+        "### Building Modules\n",
+        "\n",
+        "Here's an example of a very simple `tf.Module` that operates on a scalar tensor:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "alhYPVEtAiSy"
+      },
+      "outputs": [],
+      "source": [
+        "class SimpleModule(tf.Module):\n",
+        "  def __init__(self, name=None):\n",
+        "    super().__init__(name=name)\n",
+        "    self.a_variable = tf.Variable(5.0, name=\"train_me\")\n",
+        "    self.non_trainable_variable = tf.Variable(5.0, trainable=False, name=\"do_not_train_me\")\n",
+        "  def __call__(self, x):\n",
+        "    return self.a_variable * x + self.non_trainable_variable\n",
+        "\n",
+        "simple_module = SimpleModule(name=\"simple\")\n",
+        "\n",
+        "simple_module(tf.constant(5.0))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JwMc_zu5Ant8"
+      },
+      "source": [
+        "Modules and, by extension, layers are deep-learning terminology for \"objects\": they have internal state, and methods that use that state.\n",
+        "\n",
+        "There is nothing special about `__call__` except to act like a [Python callable](https://stackoverflow.com/questions/111234/what-is-a-callable); you can invoke your models with whatever functions you wish.\n",
+        "\n",
+        "You can set the trainability of variables on and off for any reason, including freezing layers and variables during fine-tuning.\n",
+        "\n",
+        "Note: `tf.Module` is the base class for both `tf.keras.layers.Layer` and `tf.keras.Model`, so everything you come across here also applies in Keras.  For historical compatibility reasons Keras layers do not collect variables from modules, so your models should use only modules or only Keras layers.  However, the methods shown below for inspecting variables are the same in either case.\n",
+        "\n",
+        "By subclassing `tf.Module`, any `tf.Variable` or `tf.Module` instances assigned to this object's properties are automatically collected.  This allows you to save and load variables, and also create collections of `tf.Module`s."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CyzYy4A_CbVf"
+      },
+      "outputs": [],
+      "source": [
+        "# All trainable variables\n",
+        "print(\"trainable variables:\", simple_module.trainable_variables)\n",
+        "# Every variable\n",
+        "print(\"all variables:\", simple_module.variables)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nuSFrRUNCaaW"
+      },
+      "source": [
+        "This is an example of a two-layer linear layer model made out of modules.\n",
+        "\n",
+        "First a dense (linear) layer:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Efb2p2bzAn-V"
+      },
+      "outputs": [],
+      "source": [
+        "class Dense(tf.Module):\n",
+        "  def __init__(self, in_features, out_features, name=None):\n",
+        "    super().__init__(name=name)\n",
+        "    self.w = tf.Variable(\n",
+        "      tf.random.normal([in_features, out_features]), name='w')\n",
+        "    self.b = tf.Variable(tf.zeros([out_features]), name='b')\n",
+        "  def __call__(self, x):\n",
+        "    y = tf.matmul(x, self.w) + self.b\n",
+        "    return tf.nn.relu(y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bAhMuC-UpnhX"
+      },
+      "source": [
+        "And then the complete model, which makes two layer instances and applies them:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QQ7qQf-DFw74"
+      },
+      "outputs": [],
+      "source": [
+        "class SequentialModule(tf.Module):\n",
+        "  def __init__(self, name=None):\n",
+        "    super().__init__(name=name)\n",
+        "\n",
+        "    self.dense_1 = Dense(in_features=3, out_features=3)\n",
+        "    self.dense_2 = Dense(in_features=3, out_features=2)\n",
+        "\n",
+        "  def __call__(self, x):\n",
+        "    x = self.dense_1(x)\n",
+        "    return self.dense_2(x)\n",
+        "\n",
+        "# You have made a model!\n",
+        "my_model = SequentialModule(name=\"the_model\")\n",
+        "\n",
+        "# Call it, with random results\n",
+        "print(\"Model results:\", my_model(tf.constant([[2.0, 2.0, 2.0]])))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d1oUzasJHHXf"
+      },
+      "source": [
+        "`tf.Module` instances will automatically collect, recursively, any `tf.Variable` or `tf.Module` instances assigned to it. This allows you to manage collections of `tf.Module`s with a single model instance, and save and load whole models."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JLFA5_PEGb6C"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Submodules:\", my_model.submodules)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6lzoB8pcRN12"
+      },
+      "outputs": [],
+      "source": [
+        "for var in my_model.variables:\n",
+        "  print(var, \"\\n\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hoaxL3zzm0vK"
+      },
+      "source": [
+        "### Waiting to create variables\n",
+        "\n",
+        "You may have noticed here that you have to define both input and output sizes to the layer.  This is so the `w` variable has a known shape and can be allocated.\n",
+        "\n",
+        "By deferring variable creation to the first time the module is called with a specific input shape, you do not need specify the input size up front."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XsGCLFXlnPum"
+      },
+      "outputs": [],
+      "source": [
+        "class FlexibleDenseModule(tf.Module):\n",
+        "  # Note: No need for `in_features`\n",
+        "  def __init__(self, out_features, name=None):\n",
+        "    super().__init__(name=name)\n",
+        "    self.is_built = False\n",
+        "    self.out_features = out_features\n",
+        "\n",
+        "  def __call__(self, x):\n",
+        "    # Create variables on first call.\n",
+        "    if not self.is_built:\n",
+        "      self.w = tf.Variable(\n",
+        "        tf.random.normal([x.shape[-1], self.out_features]), name='w')\n",
+        "      self.b = tf.Variable(tf.zeros([self.out_features]), name='b')\n",
+        "      self.is_built = True\n",
+        "\n",
+        "    y = tf.matmul(x, self.w) + self.b\n",
+        "    return tf.nn.relu(y)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8bjOWax9LOkP"
+      },
+      "outputs": [],
+      "source": [
+        "# Used in a module\n",
+        "class MySequentialModule(tf.Module):\n",
+        "  def __init__(self, name=None):\n",
+        "    super().__init__(name=name)\n",
+        "\n",
+        "    self.dense_1 = FlexibleDenseModule(out_features=3)\n",
+        "    self.dense_2 = FlexibleDenseModule(out_features=2)\n",
+        "\n",
+        "  def __call__(self, x):\n",
+        "    x = self.dense_1(x)\n",
+        "    return self.dense_2(x)\n",
+        "\n",
+        "my_model = MySequentialModule(name=\"the_model\")\n",
+        "print(\"Model results:\", my_model(tf.constant([[2.0, 2.0, 2.0]])))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "49JfbhVrpOLH"
+      },
+      "source": [
+        "This flexibility is why TensorFlow layers often only need to specify the shape of their outputs, such as in `tf.keras.layers.Dense`, rather than both  the input and output size."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JOLVVBT8J_dl"
+      },
+      "source": [
+        "### Saving weights\n",
+        "\n",
+        "You can save a `tf.Module` as both a [checkpoint](./checkpoint.ipynb) and a [SavedModel](./saved_model.ipynb).\n",
+        "\n",
+        "Checkpoints are just the weights (that is, the values of the set of variables inside the module and its submodules):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pHXKRDk7OLHA"
+      },
+      "outputs": [],
+      "source": [
+        "chkp_path = \"my_checkpoint\"\n",
+        "checkpoint = tf.train.Checkpoint(model=my_model)\n",
+        "checkpoint.write(chkp_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WXOPMBR4T4ZR"
+      },
+      "source": [
+        "Checkpoints consist of two kinds of files: the data itself and an index file for metadata. The index file keeps track of what is actually saved and the numbering of checkpoints, while the checkpoint data contains the variable values and their attribute lookup paths."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jBV3fprlTWqJ"
+      },
+      "outputs": [],
+      "source": [
+        "!ls my_checkpoint*"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CowCuBTvXgUu"
+      },
+      "source": [
+        "You can look inside a checkpoint to be sure the whole collection of variables is saved, sorted by the Python object that contains them."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o2QAdfpvS8tB"
+      },
+      "outputs": [],
+      "source": [
+        "tf.train.list_variables(chkp_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4eGaNiQWcK4j"
+      },
+      "source": [
+        "During distributed (multi-machine) training they can be sharded,  which is why they are numbered (e.g., '00000-of-00001').  In this case, though, there is only one shard.\n",
+        "\n",
+        "When you load models back in, you overwrite the values in your Python object."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UV8rdDzcwVVg"
+      },
+      "outputs": [],
+      "source": [
+        "new_model = MySequentialModule()\n",
+        "new_checkpoint = tf.train.Checkpoint(model=new_model)\n",
+        "new_checkpoint.restore(\"my_checkpoint\")\n",
+        "\n",
+        "# Should be the same result as above\n",
+        "new_model(tf.constant([[2.0, 2.0, 2.0]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BnPwDRwamdfq"
+      },
+      "source": [
+        "Note: As checkpoints are at the heart of long training workflows `tf.checkpoint.CheckpointManager` is a helper class that makes checkpoint management much easier. Refer to the [Training checkpoints guide](./checkpoint.ipynb) for more details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pSZebVuWxDXu"
+      },
+      "source": [
+        "### Saving functions\n",
+        "\n",
+        "TensorFlow can run models without the original Python objects, as demonstrated by [TensorFlow Serving](https://tensorflow.org/tfx) and [TensorFlow Lite](https://tensorflow.org/lite), even when you download a trained model from [TensorFlow Hub](https://tensorflow.org/hub).\n",
+        "\n",
+        "TensorFlow needs to know how to do the computations described in Python, but **without the original code**. To do this, you can make a **graph**, which is described in the [Introduction to graphs and functions guide](./intro_to_graphs.ipynb).\n",
+        "\n",
+        "This graph contains operations, or *ops*, that implement the function.\n",
+        "\n",
+        "You can define a graph in the model above by adding the `@tf.function` decorator to indicate that this code should run as a graph."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WQTvkapUh7lk"
+      },
+      "outputs": [],
+      "source": [
+        "class MySequentialModule(tf.Module):\n",
+        "  def __init__(self, name=None):\n",
+        "    super().__init__(name=name)\n",
+        "\n",
+        "    self.dense_1 = Dense(in_features=3, out_features=3)\n",
+        "    self.dense_2 = Dense(in_features=3, out_features=2)\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self, x):\n",
+        "    x = self.dense_1(x)\n",
+        "    return self.dense_2(x)\n",
+        "\n",
+        "# You have made a model with a graph!\n",
+        "my_model = MySequentialModule(name=\"the_model\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hW66YXBziLo9"
+      },
+      "source": [
+        "The module you have made works exactly the same as before.  Each unique signature passed into the function creates a separate graph. Check the [Introduction to graphs and functions guide](./intro_to_graphs.ipynb) for details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H5zUfti3iR52"
+      },
+      "outputs": [],
+      "source": [
+        "print(my_model([[2.0, 2.0, 2.0]]))\n",
+        "print(my_model([[[2.0, 2.0, 2.0], [2.0, 2.0, 2.0]]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lbGlU1kgyDo7"
+      },
+      "source": [
+        "You can visualize the graph by tracing it within a TensorBoard summary."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zmy-T67zhp-S"
+      },
+      "outputs": [],
+      "source": [
+        "# Set up logging.\n",
+        "stamp = datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
+        "logdir = \"logs/func/%s\" % stamp\n",
+        "writer = tf.summary.create_file_writer(logdir)\n",
+        "\n",
+        "# Create a new model to get a fresh trace\n",
+        "# Otherwise the summary will not see the graph.\n",
+        "new_model = MySequentialModule()\n",
+        "\n",
+        "# Bracket the function call with\n",
+        "# tf.summary.trace_on() and tf.summary.trace_export().\n",
+        "tf.summary.trace_on(graph=True)\n",
+        "tf.profiler.experimental.start(logdir)\n",
+        "# Call only one tf.function when tracing.\n",
+        "z = print(new_model(tf.constant([[2.0, 2.0, 2.0]])))\n",
+        "with writer.as_default():\n",
+        "  tf.summary.trace_export(\n",
+        "      name=\"my_func_trace\",\n",
+        "      step=0,\n",
+        "      profiler_outdir=logdir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gz4lwNZ9hR79"
+      },
+      "source": [
+        "Launch TensorBoard to view the resulting trace:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "V4MXDbgBnkJu"
+      },
+      "outputs": [],
+      "source": [
+        "#docs_infra: no_execute\n",
+        "%tensorboard --logdir logs/func"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Gjattu0AhYUl"
+      },
+      "source": [
+        "![A screenshot of the graph in TensorBoard](images/tensorboard_graph.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SQu3TVZecmL7"
+      },
+      "source": [
+        "### Creating a `SavedModel`\n",
+        "\n",
+        "The recommended way of sharing completely trained models is to use `SavedModel`.  `SavedModel` contains both a collection of functions and a collection of weights. \n",
+        "\n",
+        "You can save the model you have just trained as follows:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Awv_Tw__WK7a"
+      },
+      "outputs": [],
+      "source": [
+        "tf.saved_model.save(my_model, \"the_saved_model\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SXv3mEKsefGj"
+      },
+      "outputs": [],
+      "source": [
+        "# Inspect the SavedModel in the directory\n",
+        "!ls -l the_saved_model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vQQ3hEvHYdoR"
+      },
+      "outputs": [],
+      "source": [
+        "# The variables/ directory contains a checkpoint of the variables \n",
+        "!ls -l the_saved_model/variables"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xBqPop7ZesBU"
+      },
+      "source": [
+        "The `saved_model.pb` file is a [protocol buffer](https://developers.google.com/protocol-buffers) describing the functional `tf.Graph`.\n",
+        "\n",
+        "Models and layers can be loaded from this representation without actually making an instance of the class that created it.  This is desired in situations where you do not have (or want) a Python interpreter, such as serving at scale or on an edge device, or in situations where the original Python code is not available or practical to use.\n",
+        "\n",
+        "You can load the model as new object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zRFcA5wIefv4"
+      },
+      "outputs": [],
+      "source": [
+        "new_model = tf.saved_model.load(\"the_saved_model\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-9EF3mT7i3qN"
+      },
+      "source": [
+        "`new_model`, created from loading a saved model, is an internal TensorFlow user object without any of the class knowledge. It is not of type `SequentialModule`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EC_eQj7yi54G"
+      },
+      "outputs": [],
+      "source": [
+        "isinstance(new_model, SequentialModule)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-OrOX1zxiyhR"
+      },
+      "source": [
+        "This new model works on the already-defined input signatures. You can't add more signatures to a model restored like this."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_23BYYBWfKnc"
+      },
+      "outputs": [],
+      "source": [
+        "print(my_model([[2.0, 2.0, 2.0]]))\n",
+        "print(my_model([[[2.0, 2.0, 2.0], [2.0, 2.0, 2.0]]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qSFhoMtTjSR6"
+      },
+      "source": [
+        "Thus, using `SavedModel`, you are able to save TensorFlow weights and graphs using `tf.Module`, and then load them again."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Rb9IdN7hlUZK"
+      },
+      "source": [
+        "## Keras models and layers\n",
+        "\n",
+        "Note that up until this point, there is no mention of Keras. You can build your own high-level API on top of `tf.Module`, and people have.  \n",
+        "\n",
+        "In this section, you will examine how Keras uses `tf.Module`.  A complete user guide to Keras models can be found in the [Keras guide](https://www.tensorflow.org/guide/keras/sequential_model).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ds08u3touwe4t"
+      },
+      "source": [
+        "Keras layers and models have a lot more extra features including:\n",
+        "\n",
+        "* Optional losses\n",
+        "* Support for [metrics](https://keras.io/api/layers/base_layer/#add_metric-method)\n",
+        "* Built-in support for an optional `training` argument to differentiate between training and inference use\n",
+        "* Saving and restoring python objects instead of just black-box functions\n",
+        "* `get_config` and `from_config` methods that allow you to accurately store configurations to allow model cloning in Python\n",
+        "\n",
+        "These features allow for far more complex models through subclassing, such as a custom GAN or a Variational AutoEncoder (VAE) model. Read about them in the [full guide](./keras/custom_layers_and_models.ipynb) to custom layers and models.\n",
+        "\n",
+        "Keras models also come with extra functionality that makes them easy to train, evaluate, load, save, and even train on multiple machines."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uigsVGPreE-D"
+      },
+      "source": [
+        "### Keras layers\n",
+        "\n",
+        "`tf.keras.layers.Layer` is the base class of all Keras layers, and it inherits from `tf.Module`.\n",
+        "\n",
+        "You can convert a module into a Keras layer just by swapping out the parent and then changing `__call__` to `call`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "88YOGquhnQRd"
+      },
+      "outputs": [],
+      "source": [
+        "class MyDense(tf.keras.layers.Layer):\n",
+        "  # Adding **kwargs to support base Keras layer arguments\n",
+        "  def __init__(self, in_features, out_features, **kwargs):\n",
+        "    super().__init__(**kwargs)\n",
+        "\n",
+        "    # This will soon move to the build step; see below\n",
+        "    self.w = tf.Variable(\n",
+        "      tf.random.normal([in_features, out_features]), name='w')\n",
+        "    self.b = tf.Variable(tf.zeros([out_features]), name='b')\n",
+        "  def call(self, x):\n",
+        "    y = tf.matmul(x, self.w) + self.b\n",
+        "    return tf.nn.relu(y)\n",
+        "\n",
+        "simple_layer = MyDense(name=\"simple\", in_features=3, out_features=3)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nYGmAsPrws--"
+      },
+      "source": [
+        "Keras layers have their own `__call__` that does some bookkeeping described in the next section and then calls `call()`. You should notice no change in functionality."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nIqE8wOznYKG"
+      },
+      "outputs": [],
+      "source": [
+        "simple_layer([[2.0, 2.0, 2.0]])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tmN5vb1K18U1"
+      },
+      "source": [
+        "### The `build` step\n",
+        "\n",
+        "As noted, it's convenient in many cases to wait to create variables until you are sure of the input shape.\n",
+        "\n",
+        "Keras layers come with an extra lifecycle step that allows you more flexibility in how you define your layers. This is defined in the `build` function.\n",
+        "\n",
+        "`build` is called exactly once, and it is called with the shape of the input. It's usually used to create variables (weights).\n",
+        "\n",
+        "You can rewrite `MyDense` layer above to be flexible to the size of its inputs:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4YTfrlgdsURp"
+      },
+      "outputs": [],
+      "source": [
+        "class FlexibleDense(tf.keras.layers.Layer):\n",
+        "  # Note the added `**kwargs`, as Keras supports many arguments\n",
+        "  def __init__(self, out_features, **kwargs):\n",
+        "    super().__init__(**kwargs)\n",
+        "    self.out_features = out_features\n",
+        "\n",
+        "  def build(self, input_shape):  # Create the state of the layer (weights)\n",
+        "    self.w = tf.Variable(\n",
+        "      tf.random.normal([input_shape[-1], self.out_features]), name='w')\n",
+        "    self.b = tf.Variable(tf.zeros([self.out_features]), name='b')\n",
+        "\n",
+        "  def call(self, inputs):  # Defines the computation from inputs to outputs\n",
+        "    return tf.matmul(inputs, self.w) + self.b\n",
+        "\n",
+        "# Create the instance of the layer\n",
+        "flexible_dense = FlexibleDense(out_features=3)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Koc_uSqt2PRh"
+      },
+      "source": [
+        "At this point, the model has not been built, so there are no variables:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DgyTyUD32Ln4"
+      },
+      "outputs": [],
+      "source": [
+        "flexible_dense.variables"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-KdamIVl2W8Y"
+      },
+      "source": [
+        "Calling the function allocates appropriately-sized variables:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IkLyEx7uAoTK"
+      },
+      "outputs": [],
+      "source": [
+        "# Call it, with predictably random results\n",
+        "print(\"Model results:\", flexible_dense(tf.constant([[2.0, 2.0, 2.0], [3.0, 3.0, 3.0]])))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Swofpkrd2YDd"
+      },
+      "outputs": [],
+      "source": [
+        "flexible_dense.variables"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7PuNUnf0OIpF"
+      },
+      "source": [
+        "Since `build` is only called once, inputs will be rejected if the input shape is not compatible with the layer's variables:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "caYWDrHSAy_j"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  print(\"Model results:\", flexible_dense(tf.constant([[2.0, 2.0, 2.0, 2.0]])))\n",
+        "except tf.errors.InvalidArgumentError as e:\n",
+        "  print(\"Failed:\", e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L2kds2IHw2KD"
+      },
+      "source": [
+        "### Keras models\n",
+        "\n",
+        "You can define your model as nested Keras layers.\n",
+        "\n",
+        "However, Keras also provides a full-featured model class called `tf.keras.Model`. It inherits from `tf.keras.layers.Layer`, so a Keras model can be used and nested in the same way as Keras layers. Keras models come with extra functionality that makes them easy to train, evaluate, load, save, and even train on multiple machines.\n",
+        "\n",
+        "You can define the `SequentialModule` from above with nearly identical code, again converting `__call__` to `call()` and changing the parent:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hqjo1DiyrHrn"
+      },
+      "outputs": [],
+      "source": [
+        "@keras.saving.register_keras_serializable()\n",
+        "class MySequentialModel(tf.keras.Model):\n",
+        "  def __init__(self, name=None, **kwargs):\n",
+        "    super().__init__(**kwargs)\n",
+        "\n",
+        "    self.dense_1 = FlexibleDense(out_features=3)\n",
+        "    self.dense_2 = FlexibleDense(out_features=2)\n",
+        "  def call(self, x):\n",
+        "    x = self.dense_1(x)\n",
+        "    return self.dense_2(x)\n",
+        "\n",
+        "# You have made a Keras model!\n",
+        "my_sequential_model = MySequentialModel(name=\"the_model\")\n",
+        "\n",
+        "# Call it on a tensor, with random results\n",
+        "print(\"Model results:\", my_sequential_model(tf.constant([[2.0, 2.0, 2.0]])))\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8i-CR_h2xw3z"
+      },
+      "source": [
+        "All the same features are available, including tracking variables and submodules.\n",
+        "\n",
+        "Note: A raw `tf.Module` nested inside a Keras layer or model will not get its variables collected for training or saving.  Instead, nest Keras layers inside of Keras layers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hdLQFNdMsOz1"
+      },
+      "outputs": [],
+      "source": [
+        "my_sequential_model.variables"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JjVAMrAJsQ7G"
+      },
+      "outputs": [],
+      "source": [
+        "my_sequential_model.submodules"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FhP8EItC4oac"
+      },
+      "source": [
+        "Overriding `tf.keras.Model` is a very Pythonic approach to building TensorFlow models.  If you are migrating models from other frameworks, this can be very straightforward.\n",
+        "\n",
+        "If you are constructing models that are simple assemblages of existing layers and inputs, you can save time and space by using the [functional API](./keras/functional.ipynb), which comes with additional features around model reconstruction and architecture.\n",
+        "\n",
+        "Here is the same model with the functional API:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jJiZZiJ0fyqQ"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = tf.keras.Input(shape=[3,])\n",
+        "\n",
+        "x = FlexibleDense(3)(inputs)\n",
+        "x = FlexibleDense(2)(x)\n",
+        "\n",
+        "my_functional_model = tf.keras.Model(inputs=inputs, outputs=x)\n",
+        "\n",
+        "my_functional_model.summary()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kg-xAZw5gaG6"
+      },
+      "outputs": [],
+      "source": [
+        "my_functional_model(tf.constant([[2.0, 2.0, 2.0]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s_BK9XH5q9cq"
+      },
+      "source": [
+        "The major difference here is that the input shape is specified up front as part of the functional construction process. The `input_shape` argument in this case does not have to be completely specified; you can leave some dimensions as `None`.\n",
+        "\n",
+        "Note: You do not need to specify `input_shape` or an `InputLayer` in a subclassed model; these arguments and layers will be ignored."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qI9aXLnaHEFF"
+      },
+      "source": [
+        "### Saving Keras models\n",
+        "\n",
+        "Keras models have their own specialized zip archive saving format, marked by the `.keras` extension. When calling `tf.keras.Model.save`, add a `.keras` extension to the filename. For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SAz-KVZlzAJu"
+      },
+      "outputs": [],
+      "source": [
+        "my_sequential_model.save(\"exname_of_file.keras\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "C2urAeR-omns"
+      },
+      "source": [
+        "Just as easily, they can be loaded back in:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Wj5DW-LCopry"
+      },
+      "outputs": [],
+      "source": [
+        "reconstructed_model = tf.keras.models.load_model(\"exname_of_file.keras\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EA7P_MNvpviZ"
+      },
+      "source": [
+        "Keras zip archives — `.keras` files — also save metric, loss, and optimizer states.\n",
+        "\n",
+        "This reconstructed model can be used and will produce the same result when called on the same data:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "P_wGfQo5pe6T"
+      },
+      "outputs": [],
+      "source": [
+        "reconstructed_model(tf.constant([[2.0, 2.0, 2.0]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "seLIUG2354s"
+      },
+      "source": [
+        "### Checkpointing Keras models\n",
+        "\n",
+        "Keras models can also be checkpointed, and that will look the same as `tf.Module`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xKyjlkceqjwD"
+      },
+      "source": [
+        "There is more to know about saving and serialization of Keras models, including providing configuration methods for custom layers for feature support. Check out the [guide to saving and serialization](https://www.tensorflow.org/guide/keras/save_and_serialize)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kcdMMPYv7Krz"
+      },
+      "source": [
+        "# What's next\n",
+        "\n",
+        "If you want to know more details about Keras, you can follow the existing Keras guides [here](./keras/).\n",
+        "\n",
+        "Another example of a high-level API built on `tf.module` is Sonnet from DeepMind, which is covered on [their site](https://github.com/deepmind/sonnet)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "ISubpr_SSsiM"
+      ],
+      "name": "intro_to_modules.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/jax2tf.ipynb b/site/en/guide/jax2tf.ipynb
new file mode 100644
index 00000000000..613c622658d
--- /dev/null
+++ b/site/en/guide/jax2tf.ipynb
@@ -0,0 +1,851 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ckM5wJMsNTYL"
+      },
+      "source": [
+        "##### Copyright 2023 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "NKvERjPVNWxu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bqePLdDjNhNk"
+      },
+      "source": [
+        "# Import a JAX model using JAX2TF"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gw3w46yhNiK_"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IyrsY3uTOmPY"
+      },
+      "source": [
+        "This notebook provides a complete, runnable example of creating a model using [JAX](https://jax.readthedocs.io/en/latest/) and bringing it into TensorFlow to continue training. This is made possible by [JAX2TF](https://github.com/google/jax/tree/main/jax/experimental/jax2tf), a lightweight API that provides a pathway from the JAX ecosystem to the TensorFlow ecosystem. \n",
+        "\n",
+        "JAX is a high-performance array computing library. To create the model, this notebook uses [Flax](https://flax.readthedocs.io/en/latest/), a neural network library for JAX. To train it, it uses [Optax](https://optax.readthedocs.io), an optimization library for JAX.\n",
+        "\n",
+        "If you're a researcher using JAX, JAX2TF gives you a path to production using TensorFlow's proven tools.\n",
+        "\n",
+        "There are many ways this can be useful, here are just a few:\n",
+        "\n",
+        "* Inference: Taking a model written for JAX and deploying it either on a server using TF Serving, on-device using TFLite, or on the web using TensorFlow.js. \n",
+        "\n",
+        "* Fine-tuning: Taking a model that was trained using JAX, you can bring its components to TF using JAX2TF, and continue training it in TensorFlow with your existing training data and setup.\n",
+        "\n",
+        "* Fusion: Combining parts of models that were trained using JAX with those trained using TensorFlow, for maximum flexibility.\n",
+        "\n",
+        "The key to enabling this kind of interoperation between JAX and TensorFlow is `jax2tf.convert`, which takes in model components created on top of JAX (your loss function, prediction function, etc) and creates equivalent representations of them as TensorFlow functions, which can then be exported as a TensorFlow SavedModel."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G6rtu96yOepm"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9yqxfHzr0LPF"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import numpy as np\n",
+        "import jax\n",
+        "import jax.numpy as jnp\n",
+        "import flax\n",
+        "import optax\n",
+        "import os\n",
+        "from matplotlib import pyplot as plt\n",
+        "from jax.experimental import jax2tf\n",
+        "from threading import Lock # Only used in the visualization utility.\n",
+        "from functools import partial"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SDnTaZO0r872"
+      },
+      "outputs": [],
+      "source": [
+        "# Needed for TensorFlow and JAX to coexist in GPU memory.\n",
+        "os.environ['XLA_PYTHON_CLIENT_PREALLOCATE'] = \"false\"\n",
+        "gpus = tf.config.list_physical_devices('GPU')\n",
+        "if gpus:\n",
+        "  try:\n",
+        "    for gpu in gpus:\n",
+        "      tf.config.experimental.set_memory_growth(gpu, True)\n",
+        "  except RuntimeError as e:\n",
+        "    # Memory growth must be set before GPUs have been initialized.\n",
+        "    print(e)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "BXOjCNJxDLil"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Visualization utilities\n",
+        "\n",
+        "plt.rcParams[\"figure.figsize\"] = (20,8)\n",
+        "\n",
+        "# The utility for displaying training and validation curves.\n",
+        "def display_train_curves(loss, avg_loss, eval_loss, eval_accuracy, epochs, steps_per_epochs, ignore_first_n=10):\n",
+        "\n",
+        "  ignore_first_n_epochs = int(ignore_first_n/steps_per_epochs)\n",
+        "\n",
+        "  # The losses.\n",
+        "  ax = plt.subplot(121)\n",
+        "  if loss is not None:\n",
+        "    x = np.arange(len(loss)) / steps_per_epochs #* epochs\n",
+        "    ax.plot(x, loss)\n",
+        "  ax.plot(range(1, epochs+1), avg_loss, \"-o\", linewidth=3)\n",
+        "  ax.plot(range(1, epochs+1), eval_loss, \"-o\", linewidth=3)\n",
+        "  ax.set_title('Loss')\n",
+        "  ax.set_ylabel('loss')\n",
+        "  ax.set_xlabel('epoch')\n",
+        "  if loss is not None:\n",
+        "    ax.set_ylim(0, np.max(loss[ignore_first_n:]))\n",
+        "    ax.legend(['train', 'avg train', 'eval'])\n",
+        "  else:\n",
+        "    ymin = np.min(avg_loss[ignore_first_n_epochs:])\n",
+        "    ymax = np.max(avg_loss[ignore_first_n_epochs:])\n",
+        "    ax.set_ylim(ymin-(ymax-ymin)/10, ymax+(ymax-ymin)/10)\n",
+        "    ax.legend(['avg train', 'eval'])\n",
+        "\n",
+        "  # The accuracy.\n",
+        "  ax = plt.subplot(122)\n",
+        "  ax.set_title('Eval Accuracy')\n",
+        "  ax.set_ylabel('accuracy')\n",
+        "  ax.set_xlabel('epoch')\n",
+        "  ymin = np.min(eval_accuracy[ignore_first_n_epochs:])\n",
+        "  ymax = np.max(eval_accuracy[ignore_first_n_epochs:])\n",
+        "  ax.set_ylim(ymin-(ymax-ymin)/10, ymax+(ymax-ymin)/10)\n",
+        "  ax.plot(range(1, epochs+1), eval_accuracy, \"-o\", linewidth=3)\n",
+        "\n",
+        "class Progress:\n",
+        "    \"\"\"Text mode progress bar.\n",
+        "    Usage:\n",
+        "            p = Progress(30)\n",
+        "            p.step()\n",
+        "            p.step()\n",
+        "            p.step(reset=True) # to restart form 0%\n",
+        "    The progress bar displays a new header at each restart.\"\"\"\n",
+        "    def __init__(self, maxi, size=100, msg=\"\"):\n",
+        "        \"\"\"\n",
+        "        :param maxi: the number of steps required to reach 100%\n",
+        "        :param size: the number of characters taken on the screen by the progress bar\n",
+        "        :param msg: the message displayed in the header of the progress bar\n",
+        "        \"\"\"\n",
+        "        self.maxi = maxi\n",
+        "        self.p = self.__start_progress(maxi)()  # `()`: to get the iterator from the generator.\n",
+        "        self.header_printed = False\n",
+        "        self.msg = msg\n",
+        "        self.size = size\n",
+        "        self.lock = Lock()\n",
+        "\n",
+        "    def step(self, reset=False):\n",
+        "        with self.lock:\n",
+        "            if reset:\n",
+        "                self.__init__(self.maxi, self.size, self.msg)\n",
+        "            if not self.header_printed:\n",
+        "                self.__print_header()\n",
+        "            next(self.p)\n",
+        "\n",
+        "    def __print_header(self):\n",
+        "        print()\n",
+        "        format_string = \"0%{: ^\" + str(self.size - 6) + \"}100%\"\n",
+        "        print(format_string.format(self.msg))\n",
+        "        self.header_printed = True\n",
+        "\n",
+        "    def __start_progress(self, maxi):\n",
+        "        def print_progress():\n",
+        "            # Bresenham's algorithm. Yields the number of dots printed.\n",
+        "            # This will always print 100 dots in max invocations.\n",
+        "            dx = maxi\n",
+        "            dy = self.size\n",
+        "            d = dy - dx\n",
+        "            for x in range(maxi):\n",
+        "                k = 0\n",
+        "                while d >= 0:\n",
+        "                    print('=', end=\"\", flush=True)\n",
+        "                    k += 1\n",
+        "                    d -= dx\n",
+        "                d += dy\n",
+        "                yield k\n",
+        "            # Keep yielding the last result if there are too many steps.\n",
+        "            while True:\n",
+        "              yield k\n",
+        "\n",
+        "        return print_progress"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6xgS_8nDDIu8"
+      },
+      "source": [
+        "## Download and prepare the MNIST dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nbN7rmuF0VFB"
+      },
+      "outputs": [],
+      "source": [
+        "(x_train, train_labels), (x_test, test_labels) = tf.keras.datasets.mnist.load_data()\n",
+        "\n",
+        "train_data = tf.data.Dataset.from_tensor_slices((x_train, train_labels))\n",
+        "train_data = train_data.map(lambda x,y: (tf.expand_dims(tf.cast(x, tf.float32)/255.0, axis=-1),\n",
+        "                                         tf.one_hot(y, depth=10)))\n",
+        "\n",
+        "BATCH_SIZE = 256\n",
+        "train_data = train_data.batch(BATCH_SIZE, drop_remainder=True)\n",
+        "train_data = train_data.cache()\n",
+        "train_data = train_data.shuffle(5000, reshuffle_each_iteration=True)\n",
+        "\n",
+        "test_data = tf.data.Dataset.from_tensor_slices((x_test, test_labels))\n",
+        "test_data = test_data.map(lambda x,y: (tf.expand_dims(tf.cast(x, tf.float32)/255.0, axis=-1),\n",
+        "                                         tf.one_hot(y, depth=10)))\n",
+        "test_data = test_data.batch(10000)\n",
+        "test_data = test_data.cache()\n",
+        "\n",
+        "(one_batch, one_batch_labels) = next(iter(train_data)) # just one batch\n",
+        "(all_test_data, all_test_labels) = next(iter(test_data)) # all in one batch since batch size is 10000"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LuZTo7SM3W_n"
+      },
+      "source": [
+        "## Configure training\n",
+        "This notebook will create and train a simple model for demonstration purposes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3vbKB4yZ3aTL"
+      },
+      "outputs": [],
+      "source": [
+        "# Training hyperparameters.\n",
+        "JAX_EPOCHS = 3\n",
+        "TF_EPOCHS = 7\n",
+        "STEPS_PER_EPOCH = len(train_labels)//BATCH_SIZE\n",
+        "LEARNING_RATE = 0.01\n",
+        "LEARNING_RATE_EXP_DECAY = 0.6\n",
+        "\n",
+        "# The learning rate schedule for JAX (with Optax).\n",
+        "jlr_decay = optax.exponential_decay(LEARNING_RATE, transition_steps=STEPS_PER_EPOCH, decay_rate=LEARNING_RATE_EXP_DECAY, staircase=True)\n",
+        "\n",
+        "# THe learning rate schedule for TensorFlow.\n",
+        "tflr_decay = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=LEARNING_RATE, decay_steps=STEPS_PER_EPOCH, decay_rate=LEARNING_RATE_EXP_DECAY, staircase=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Od3sMwQxtC34"
+      },
+      "source": [
+        "## Create the model using Flax"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-ybqQF2zd2QX"
+      },
+      "outputs": [],
+      "source": [
+        "class ConvModel(flax.linen.Module):\n",
+        "\n",
+        "  @flax.linen.compact\n",
+        "  def __call__(self, x, train):\n",
+        "    x = flax.linen.Conv(features=12, kernel_size=(3,3), padding=\"SAME\", use_bias=False)(x)\n",
+        "    x = flax.linen.BatchNorm(use_running_average=not train, use_scale=False, use_bias=True)(x)\n",
+        "    x = x.reshape((x.shape[0], -1))  # flatten\n",
+        "    x = flax.linen.Dense(features=200, use_bias=True)(x)\n",
+        "    x = flax.linen.BatchNorm(use_running_average=not train, use_scale=False, use_bias=True)(x)\n",
+        "    x = flax.linen.Dropout(rate=0.3, deterministic=not train)(x)\n",
+        "    x = flax.linen.relu(x)\n",
+        "    x = flax.linen.Dense(features=10)(x)\n",
+        "    #x = flax.linen.log_softmax(x)\n",
+        "    return x\n",
+        "\n",
+        "  # JAX differentiation requires a function `f(params, other_state, data, labels)` -> `loss` (as a single number).\n",
+        "  # `jax.grad` will differentiate it against the fist argument.\n",
+        "  # The user must split trainable and non-trainable variables into `params` and `other_state`.\n",
+        "  # Must pass a different RNG key each time for the dropout mask to be different.\n",
+        "  def loss(self, params, other_state, rng, data, labels, train):\n",
+        "    logits, batch_stats = self.apply({'params': params, **other_state},\n",
+        "                                     data,\n",
+        "                                     mutable=['batch_stats'],\n",
+        "                                     rngs={'dropout': rng},\n",
+        "                                     train=train)\n",
+        "    # The loss averaged across the batch dimension.\n",
+        "    loss = optax.softmax_cross_entropy(logits, labels).mean()\n",
+        "    return loss, batch_stats\n",
+        "\n",
+        "  def predict(self, state, data):\n",
+        "    logits = self.apply(state, data, train=False) # predict and accuracy disable dropout and use accumulated batch norm stats (train=False)\n",
+        "    probabilities = flax.linen.log_softmax(logits)\n",
+        "    return probabilities\n",
+        "\n",
+        "  def accuracy(self, state, data, labels):\n",
+        "    probabilities = self.predict(state, data)\n",
+        "    predictions = jnp.argmax(probabilities, axis=-1)\n",
+        "    dense_labels = jnp.argmax(labels, axis=-1)\n",
+        "    accuracy = jnp.equal(predictions, dense_labels).mean()\n",
+        "    return accuracy"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7Cr0FRNFtHN4"
+      },
+      "source": [
+        "## Write the training step function"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tmDwApcpgZzw"
+      },
+      "outputs": [],
+      "source": [
+        "# The training step.\n",
+        "@partial(jax.jit, static_argnums=[0]) # this forces jax.jit to recompile for every new model\n",
+        "def train_step(model, state, optimizer_state, rng, data, labels):\n",
+        "\n",
+        "  other_state, params = state.pop('params') # differentiate only against 'params' which represents trainable variables\n",
+        "  (loss, batch_stats), grads = jax.value_and_grad(model.loss, has_aux=True)(params, other_state, rng, data, labels, train=True)\n",
+        "\n",
+        "  updates, optimizer_state = optimizer.update(grads, optimizer_state)\n",
+        "  params = optax.apply_updates(params, updates)\n",
+        "  new_state = state.copy(add_or_replace={**batch_stats, 'params': params})\n",
+        "\n",
+        "  rng, _ = jax.random.split(rng)\n",
+        "\n",
+        "  return new_state, optimizer_state, rng, loss"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Zr16g6NzV4O9"
+      },
+      "source": [
+        "## Write the training loop"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zbl5w-KUV7Qw"
+      },
+      "outputs": [],
+      "source": [
+        "def train(model, state, optimizer_state, train_data, epochs, losses, avg_losses, eval_losses, eval_accuracies):\n",
+        "  p = Progress(STEPS_PER_EPOCH)\n",
+        "  rng = jax.random.PRNGKey(0)\n",
+        "  for epoch in range(epochs):\n",
+        "\n",
+        "    # This is where the learning rate schedule state is stored in the optimizer state.\n",
+        "    optimizer_step = optimizer_state[1].count\n",
+        "\n",
+        "    # Run an epoch of training.\n",
+        "    for step, (data, labels) in enumerate(train_data):\n",
+        "      p.step(reset=(step==0))\n",
+        "      state, optimizer_state, rng, loss = train_step(model, state, optimizer_state, rng, data.numpy(), labels.numpy())\n",
+        "      losses.append(loss)\n",
+        "    avg_loss = np.mean(losses[-step:])\n",
+        "    avg_losses.append(avg_loss)\n",
+        "\n",
+        "    # Run one epoch of evals (10,000 test images in a single batch).\n",
+        "    other_state, params = state.pop('params')\n",
+        "    # Gotcha: must discard modified batch_stats here\n",
+        "    eval_loss, _ = model.loss(params, other_state, rng, all_test_data.numpy(), all_test_labels.numpy(), train=False)\n",
+        "    eval_losses.append(eval_loss)\n",
+        "    eval_accuracy = model.accuracy(state, all_test_data.numpy(), all_test_labels.numpy())\n",
+        "    eval_accuracies.append(eval_accuracy)\n",
+        "\n",
+        "    print(\"\\nEpoch\", epoch, \"train loss:\", avg_loss, \"eval loss:\", eval_loss, \"eval accuracy\", eval_accuracy, \"lr:\", jlr_decay(optimizer_step))\n",
+        "\n",
+        "  return state, optimizer_state"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DGB3W5g0Wt1H"
+      },
+      "source": [
+        "## Create the model and the optimizer (with Optax)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mW5mkmCWtN8W"
+      },
+      "outputs": [],
+      "source": [
+        "# The model.\n",
+        "model = ConvModel()\n",
+        "state = model.init({'params':jax.random.PRNGKey(0), 'dropout':jax.random.PRNGKey(0)}, one_batch, train=True) # Flax allows a separate RNG for \"dropout\"\n",
+        "\n",
+        "# The optimizer.\n",
+        "optimizer = optax.adam(learning_rate=jlr_decay) # Gotcha: it does not seem to be possible to pass just a callable as LR, must be an Optax Schedule\n",
+        "optimizer_state = optimizer.init(state['params'])\n",
+        "\n",
+        "losses=[]\n",
+        "avg_losses=[]\n",
+        "eval_losses=[]\n",
+        "eval_accuracies=[]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FJdsKghBNF"
+      },
+      "source": [
+        "## Train the model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nmcofTTBZSIb"
+      },
+      "outputs": [],
+      "source": [
+        "new_state, new_optimizer_state = train(model, state, optimizer_state, train_data, JAX_EPOCHS+TF_EPOCHS, losses, avg_losses, eval_losses, eval_accuracies)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "n_20vgvDXB5r"
+      },
+      "outputs": [],
+      "source": [
+        "display_train_curves(losses, avg_losses, eval_losses, eval_accuracies, len(eval_losses), STEPS_PER_EPOCH, ignore_first_n=1*STEPS_PER_EPOCH)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0lT3cdENCBzL"
+      },
+      "source": [
+        "## Partially train the model\n",
+        "\n",
+        "You will continue training the model in TensorFlow shortly."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KT-xqj5N7C6L"
+      },
+      "outputs": [],
+      "source": [
+        "model = ConvModel()\n",
+        "state = model.init({'params':jax.random.PRNGKey(0), 'dropout':jax.random.PRNGKey(0)}, one_batch, train=True) # Flax allows a separate RNG for \"dropout\"\n",
+        "\n",
+        "# The optimizer.\n",
+        "optimizer = optax.adam(learning_rate=jlr_decay) # LR must be an Optax LR Schedule\n",
+        "optimizer_state = optimizer.init(state['params'])\n",
+        "\n",
+        "losses, avg_losses, eval_losses, eval_accuracies = [], [], [], []"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oa362HMDbzDE"
+      },
+      "outputs": [],
+      "source": [
+        "state, optimizer_state = train(model, state, optimizer_state, train_data, JAX_EPOCHS, losses, avg_losses, eval_losses, eval_accuracies)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0IyZtUPPCt0y"
+      },
+      "outputs": [],
+      "source": [
+        "display_train_curves(losses, avg_losses, eval_losses, eval_accuracies, len(eval_losses), STEPS_PER_EPOCH, ignore_first_n=1*STEPS_PER_EPOCH)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uNtlSaOCCumB"
+      },
+      "source": [
+        "## Save just enough for inference\n",
+        "\n",
+        "If your goal is to deploy your JAX model (so you can run inference using `model.predict()`), simply exporting it to [SavedModel](https://www.tensorflow.org/guide/saved_model) is sufficient. This section demonstrates how to accomplish that."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "O653B3-5H8FL"
+      },
+      "outputs": [],
+      "source": [
+        "# Test data with a different batch size to test polymorphic shapes.\n",
+        "x, y = next(iter(train_data.unbatch().batch(13)))\n",
+        "\n",
+        "m = tf.Module()\n",
+        "# Wrap the JAX state in `tf.Variable` (needed when calling the converted JAX function.\n",
+        "state_vars = tf.nest.map_structure(tf.Variable, state)\n",
+        "# Keep the wrapped state as flat list (needed in TensorFlow fine-tuning).\n",
+        "m.vars = tf.nest.flatten(state_vars)\n",
+        "# Convert the desired JAX function (`model.predict`).\n",
+        "predict_fn = jax2tf.convert(model.predict, polymorphic_shapes=[\"...\", \"(b, 28, 28, 1)\"])\n",
+        "# Wrap the converted function in `tf.function` with the correct `tf.TensorSpec` (necessary for dynamic shapes to work).\n",
+        "@tf.function(autograph=False, input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32)])\n",
+        "def predict(data):\n",
+        "    return predict_fn(state_vars, data)\n",
+        "m.predict = predict\n",
+        "tf.saved_model.save(m, \"./\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8HFx67zStgvo"
+      },
+      "outputs": [],
+      "source": [
+        "# Test the converted function.\n",
+        "print(\"Converted function predictions:\", np.argmax(m.predict(x).numpy(), axis=-1))\n",
+        "# Reload the model.\n",
+        "reloaded_model = tf.saved_model.load(\"./\")\n",
+        "# Test the reloaded converted function (the result should be the same).\n",
+        "print(\"Reloaded  function predictions:\", np.argmax(reloaded_model.predict(x).numpy(), axis=-1))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eEk8wv4HJu94"
+      },
+      "source": [
+        "## Save everything\n",
+        "If your goal is a comprehensive export (useful if you're planning on brining the model into TensorFlow for fine-tuning, fusion, etc), this section demonstrates how to save the model so you can access methods including:\n",
+        "\n",
+        " - model.predict\n",
+        " - model.accuracy\n",
+        " - model.loss (including train=True/False bool, RNG for dropout and BatchNorm state updates)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9mty52pmvDDp"
+      },
+      "outputs": [],
+      "source": [
+        "from collections import abc\n",
+        "\n",
+        "def _fix_frozen(d):\n",
+        "  \"\"\"Changes any mappings (e.g. frozendict) back to dict.\"\"\"\n",
+        "  if isinstance(d, list):\n",
+        "    return [_fix_frozen(v) for v in d]\n",
+        "  elif isinstance(d, tuple):\n",
+        "    return tuple(_fix_frozen(v) for v in d)\n",
+        "  elif not isinstance(d, abc.Mapping):\n",
+        "    return d\n",
+        "  d = dict(d)\n",
+        "  for k, v in d.items():\n",
+        "    d[k] = _fix_frozen(v)\n",
+        "  return d"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3HEsKNXbCwXw"
+      },
+      "outputs": [],
+      "source": [
+        "class TFModel(tf.Module):\n",
+        "  def __init__(self, state, model):\n",
+        "    super().__init__()\n",
+        "\n",
+        "    # Special care needed for the train=True/False parameter in the loss\n",
+        "    @jax.jit\n",
+        "    def loss_with_train_bool(state, rng, data, labels, train):\n",
+        "      other_state, params = state.pop('params')\n",
+        "      loss, batch_stats = jax.lax.cond(train,\n",
+        "                                       lambda state, data, labels: model.loss(params, other_state, rng, data, labels, train=True),\n",
+        "                                       lambda state, data, labels: model.loss(params, other_state, rng, data, labels, train=False),\n",
+        "                                       state, data, labels)\n",
+        "      # must use JAX to split the RNG, therefore, must do it in a @jax.jit function\n",
+        "      new_rng, _ = jax.random.split(rng)\n",
+        "      return loss, batch_stats, new_rng\n",
+        "\n",
+        "    self.state_vars = tf.nest.map_structure(tf.Variable, state)\n",
+        "    self.vars = tf.nest.flatten(self.state_vars)\n",
+        "    self.jax_rng = tf.Variable(jax.random.PRNGKey(0))\n",
+        "\n",
+        "    self.loss_fn = jax2tf.convert(loss_with_train_bool, polymorphic_shapes=[\"...\", \"...\", \"(b, 28, 28, 1)\", \"(b, 10)\", \"...\"])\n",
+        "    self.accuracy_fn = jax2tf.convert(model.accuracy, polymorphic_shapes=[\"...\", \"(b, 28, 28, 1)\", \"(b, 10)\"])\n",
+        "    self.predict_fn = jax2tf.convert(model.predict, polymorphic_shapes=[\"...\", \"(b, 28, 28, 1)\"])\n",
+        "\n",
+        "  # Must specify TensorSpec manually for variable batch size to work\n",
+        "  @tf.function(autograph=False, input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32)])\n",
+        "  def predict(self, data):\n",
+        "    # Make sure the TfModel.predict function implicitly use self.state_vars and not the JAX state directly\n",
+        "    # otherwise, all model weights would be embedded in the TF graph as constants.\n",
+        "    return self.predict_fn(self.state_vars, data)\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32),\n",
+        "                                tf.TensorSpec(shape=(None, 10), dtype=tf.float32)],\n",
+        "               autograph=False)\n",
+        "  def train_loss(self, data, labels):\n",
+        "      loss, batch_stats, new_rng = self.loss_fn(self.state_vars, self.jax_rng, data, labels, True)\n",
+        "      # update batch norm stats\n",
+        "      flat_vars = tf.nest.flatten(self.state_vars['batch_stats'])\n",
+        "      flat_values = tf.nest.flatten(batch_stats['batch_stats'])\n",
+        "      for var, val in zip(flat_vars, flat_values):\n",
+        "        var.assign(val)\n",
+        "      # update RNG\n",
+        "      self.jax_rng.assign(new_rng)\n",
+        "      return loss\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32),\n",
+        "                                tf.TensorSpec(shape=(None, 10), dtype=tf.float32)],\n",
+        "               autograph=False)\n",
+        "  def eval_loss(self, data, labels):\n",
+        "      loss, batch_stats, new_rng = self.loss_fn(self.state_vars, self.jax_rng, data, labels, False)\n",
+        "      return loss\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(shape=(None, 28, 28, 1), dtype=tf.float32),\n",
+        "                                tf.TensorSpec(shape=(None, 10), dtype=tf.float32)],\n",
+        "               autograph=False)\n",
+        "  def accuracy(self, data, labels):\n",
+        "    return self.accuracy_fn(self.state_vars, data, labels)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "znJrAVpcxO9u"
+      },
+      "outputs": [],
+      "source": [
+        "# Instantiate the model.\n",
+        "tf_model = TFModel(state, model)\n",
+        "\n",
+        "# Save the model.\n",
+        "tf.saved_model.save(tf_model, \"./\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Y02DHEwTjNzV"
+      },
+      "source": [
+        "## Reload the model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "i75yS3v2jPpM"
+      },
+      "outputs": [],
+      "source": [
+        "reloaded_model = tf.saved_model.load(\"./\")\n",
+        "\n",
+        "# Test if it works and that the batch size is indeed variable.\n",
+        "x,y = next(iter(train_data.unbatch().batch(13)))\n",
+        "print(np.argmax(reloaded_model.predict(x).numpy(), axis=-1))\n",
+        "x,y = next(iter(train_data.unbatch().batch(20)))\n",
+        "print(np.argmax(reloaded_model.predict(x).numpy(), axis=-1))\n",
+        "\n",
+        "print(reloaded_model.accuracy(one_batch, one_batch_labels))\n",
+        "print(reloaded_model.accuracy(all_test_data, all_test_labels))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DiwEAwQmlx1x"
+      },
+      "source": [
+        "## Continue training the converted JAX model in TensorFlow"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MubFcO_jl2vE"
+      },
+      "outputs": [],
+      "source": [
+        "optimizer = tf.keras.optimizers.Adam(learning_rate=tflr_decay)\n",
+        "\n",
+        "# Set the iteration step for the learning rate to resume from where it left off in JAX.\n",
+        "optimizer.iterations.assign(len(eval_losses)*STEPS_PER_EPOCH)\n",
+        "\n",
+        "p = Progress(STEPS_PER_EPOCH)\n",
+        "\n",
+        "for epoch in range(JAX_EPOCHS, JAX_EPOCHS+TF_EPOCHS):\n",
+        "\n",
+        "  # This is where the learning rate schedule state is stored in the optimizer state.\n",
+        "  optimizer_step = optimizer.iterations\n",
+        "\n",
+        "  for step, (data, labels) in enumerate(train_data):\n",
+        "    p.step(reset=(step==0))\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      #loss = reloaded_model.loss(data, labels, True)\n",
+        "      loss = reloaded_model.train_loss(data, labels)\n",
+        "      grads = tape.gradient(loss, reloaded_model.vars)\n",
+        "      optimizer.apply_gradients(zip(grads, reloaded_model.vars))\n",
+        "      losses.append(loss)\n",
+        "  avg_loss = np.mean(losses[-step:])\n",
+        "  avg_losses.append(avg_loss)\n",
+        "\n",
+        "  eval_loss = reloaded_model.eval_loss(all_test_data.numpy(), all_test_labels.numpy()).numpy()\n",
+        "  eval_losses.append(eval_loss)\n",
+        "  eval_accuracy = reloaded_model.accuracy(all_test_data.numpy(), all_test_labels.numpy()).numpy()\n",
+        "  eval_accuracies.append(eval_accuracy)\n",
+        "\n",
+        "  print(\"\\nEpoch\", epoch, \"train loss:\", avg_loss, \"eval loss:\", eval_loss, \"eval accuracy\", eval_accuracy, \"lr:\", tflr_decay(optimizer.iterations).numpy())"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "50V1FSmI6UTk"
+      },
+      "outputs": [],
+      "source": [
+        "display_train_curves(losses, avg_losses, eval_losses, eval_accuracies, len(eval_losses), STEPS_PER_EPOCH, ignore_first_n=2*STEPS_PER_EPOCH)\n",
+        "\n",
+        "# The loss takes a hit when the training restarts, but does not go back to random levels.\n",
+        "# This is likely caused by the optimizer momentum being reinitialized."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L7lSziW0K0ny"
+      },
+      "source": [
+        "## Next steps\n",
+        "You can learn more about [JAX](https://jax.readthedocs.io/en/latest/index.html) and [Flax](https://flax.readthedocs.io/en/latest) on their documentation websites which contain detailed guides and examples. If you're new to JAX, be sure to explore the [JAX 101 tutorials](https://jax.readthedocs.io/en/latest/jax-101/index.html), and check out the [Flax quickstart](https://flax.readthedocs.io/en/latest/getting_started.html). To learn more about converting JAX models to TensorFlow format, check out the [jax2tf](https://github.com/google/jax/tree/main/jax/experimental/jax2tf) utility on GitHub. If you're interested in converting JAX models to run in the browser with TensorFlow.js, visit [JAX on the Web with TensorFlow.js](https://blog.tensorflow.org/2022/08/jax-on-web-with-tensorflowjs.html). If you'd like to prepare JAX models to run in TensorFLow Lite, visit the [JAX Model Conversion For TFLite](https://www.tensorflow.org/lite/examples/jax_conversion/overview) guide."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "name": "jax2tf.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/keras.md b/site/en/guide/keras.md
new file mode 100644
index 00000000000..3dee7be3aa1
--- /dev/null
+++ b/site/en/guide/keras.md
@@ -0,0 +1,135 @@
+# Keras: The high-level API for TensorFlow
+
+Keras is the high-level API of the TensorFlow platform. It provides an
+approachable, highly-productive interface for solving machine learning (ML)
+problems, with a focus on modern deep learning. Keras covers every step of the
+machine learning workflow, from data processing to hyperparameter tuning to
+deployment. It was developed with a focus on enabling fast experimentation.
+
+With Keras, you have full access to the scalability and cross-platform
+capabilities of TensorFlow. You can run Keras on a TPU Pod or large clusters of
+GPUs, and you can export Keras models to run in the browser or on mobile
+devices. You can also serve Keras models via a web API.
+
+Keras is designed to reduce cognitive load by achieving the following goals:
+
+* Offer simple, consistent interfaces.
+* Minimize the number of actions required for common use cases.
+* Provide clear, actionable error messages.
+* Follow the principle of progressive disclosure of complexity: It's easy to get
+  started, and you can complete advanced workflows by learning as you go.
+* Help you write concise, readable code.
+
+## Who should use Keras
+
+The short answer is that every TensorFlow user should use the Keras APIs by
+default. Whether you're an engineer, a researcher, or an ML practitioner, you
+should start with Keras.
+
+There are a few use cases (for example, building tools on top of TensorFlow or
+developing your own high-performance platform) that require the low-level
+[TensorFlow Core APIs](https://www.tensorflow.org/guide/core). But if your use
+case doesn't fall into one
+of the
+[Core API applications](https://www.tensorflow.org/guide/core#core_api_applications),
+you should prefer Keras.
+
+## Keras API components
+
+The core data structures of Keras are [layers](https://keras.io/api/layers/) and
+[models](https://keras.io/api/models/). A layer is a simple input/output
+transformation, and a model is a directed acyclic graph (DAG) of layers.
+
+### Layers
+
+The `tf.keras.layers.Layer` class is the fundamental abstraction in Keras. A
+`Layer` encapsulates a state (weights) and some computation (defined in the
+`tf.keras.layers.Layer.call` method).
+
+Weights created by layers can be trainable or non-trainable. Layers are
+recursively composable: If you assign a layer instance as an attribute of
+another layer, the outer layer will start tracking the weights created by the
+inner layer.
+
+You can also use layers to handle data preprocessing tasks like normalization
+and text vectorization. Preprocessing layers can be included directly into a
+model, either during or after training, which makes the model portable.
+
+### Models
+
+A model is an object that groups layers together and that can be trained on
+data.
+
+The simplest type of model is the
+[`Sequential` model](https://www.tensorflow.org/guide/keras/sequential_model),
+which is a linear stack of layers. For more complex architectures, you can
+either use the
+[Keras functional API](https://www.tensorflow.org/guide/keras/functional_api),
+which lets you build arbitrary graphs of layers, or
+[use subclassing to write models from scratch](https://www.tensorflow.org/guide/keras/making_new_layers_and_models_via_subclassing).
+
+The `tf.keras.Model` class features built-in training and evaluation methods:
+
+* `tf.keras.Model.fit`: Trains the model for a fixed number of epochs.
+* `tf.keras.Model.predict`: Generates output predictions for the input samples.
+* `tf.keras.Model.evaluate`: Returns the loss and metrics values for the model;
+  configured via the `tf.keras.Model.compile` method.
+
+These methods give you access to the following built-in training features:
+
+* [Callbacks](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks).
+  You can leverage built-in callbacks for early stopping, model checkpointing,
+  and [TensorBoard](https://www.tensorflow.org/tensorboard) monitoring. You can
+  also
+  [implement custom callbacks](https://www.tensorflow.org/guide/keras/writing_your_own_callbacks).
+* [Distributed training](https://www.tensorflow.org/guide/keras/distributed_training).
+  You can easily scale up your training to multiple GPUs, TPUs, or devices.
+* Step fusing. With the `steps_per_execution` argument in
+  `tf.keras.Model.compile`, you can process multiple batches in a single
+  `tf.function` call, which greatly improves device utilization on TPUs.
+
+For a detailed overview of how to use `fit`, see the
+[training and evaluation guide](https://www.tensorflow.org/guide/keras/training_with_built_in_methods).
+To learn how to customize the built-in training and evaluation loops, see
+[Customizing what happens in `fit()`](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit).
+
+### Other APIs and tools
+
+Keras provides many other APIs and tools for deep learning, including:
+
+* [Optimizers](https://keras.io/api/optimizers/)
+* [Metrics](https://keras.io/api/metrics/)
+* [Losses](https://keras.io/api/losses/)
+* [Data loading utilities](https://keras.io/api/data_loading/)
+
+For a full list of available APIs, see the
+[Keras API reference](https://keras.io/api/). To learn more about other Keras
+projects and initiatives, see
+[The Keras ecosystem](https://keras.io/getting_started/ecosystem/).
+
+## Next steps
+
+To get started using Keras with TensorFlow, check out the following topics:
+
+* [The Sequential model](https://www.tensorflow.org/guide/keras/sequential_model)
+* [The Functional API](https://www.tensorflow.org/guide/keras/functional)
+* [Training & evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/training_with_built_in_methods)
+* [Making new layers and models via subclassing](https://www.tensorflow.org/guide/keras/custom_layers_and_models)
+* [Serialization and saving](https://www.tensorflow.org/guide/keras/save_and_serialize)
+* [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers)
+* [Customizing what happens in fit()](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit)
+* [Writing a training loop from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch)
+* [Working with RNNs](https://www.tensorflow.org/guide/keras/rnn)
+* [Understanding masking & padding](https://www.tensorflow.org/guide/keras/masking_and_padding)
+* [Writing your own callbacks](https://www.tensorflow.org/guide/keras/custom_callback)
+* [Transfer learning & fine-tuning](https://www.tensorflow.org/guide/keras/transfer_learning)
+* [Multi-GPU and distributed training](https://www.tensorflow.org/guide/keras/distributed_training)
+
+To learn more about Keras, see the following topics at
+[keras.io](http://keras.io):
+
+* [About Keras](https://keras.io/about/)
+* [Introduction to Keras for Engineers](https://keras.io/getting_started/intro_to_keras_for_engineers/)
+* [Introduction to Keras for Researchers](https://keras.io/getting_started/intro_to_keras_for_researchers/)
+* [Keras API reference](https://keras.io/api/)
+* [The Keras ecosystem](https://keras.io/getting_started/ecosystem/)
\ No newline at end of file
diff --git a/site/en/guide/keras/README.md b/site/en/guide/keras/README.md
new file mode 100644
index 00000000000..5d254c06297
--- /dev/null
+++ b/site/en/guide/keras/README.md
@@ -0,0 +1,5 @@
+Welcome to the warp zone!
+
+# Keras
+
+These docs are available here: https://github.com/keras-team/keras-io/tree/master/guides
diff --git a/site/en/guide/keras/custom_callback.ipynb b/site/en/guide/keras/custom_callback.ipynb
deleted file mode 100644
index 36cdce075e1..00000000000
--- a/site/en/guide/keras/custom_callback.ipynb
+++ /dev/null
@@ -1,559 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "597OjogAI3fy"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "5bSCD8SyJC2g"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "E_ceEiH7g0MY"
-      },
-      "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/keras/custom_callback\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/keras/custom_callback.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/keras/custom_callback.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/keras/custom_callback.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "1OykC-6lI4gv"
-      },
-      "source": [
-        "# Keras custom callbacks\n",
-        "A custom callback is a powerful tool to customize the behavior of a Keras model during training, evaluation, or inference, including reading/changing the Keras model. Examples include `tf.keras.callbacks.TensorBoard` where the training progress and results can be exported and visualized with TensorBoard, or `tf.keras.callbacks.ModelCheckpoint` where the model is automatically saved during training, and more. In this guide, you will learn what Keras callback is, when it will be called, what it can do, and how you can build your own. Towards the end of this guide, there will be demos of creating a couple of simple callback applications to get you started on your custom callback."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "d5zZ8rZD69VW"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "7BazS4qD6-2n"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "0c_TYhQOUe1j"
-      },
-      "source": [
-        "## Introduction to Keras callbacks\n",
-        "In Keras, `Callback` is a python class meant to be subclassed to provide specific functionality, with a set of methods called at various stages of training (including batch/epoch start and ends), testing, and predicting. Callbacks are useful to get a view on internal states and statistics of the model during training. You can pass a list of callbacks (as the keyword argument `callbacks`) to any of `tf.keras.Model.fit()`, `tf.keras.Model.evaluate()`, and `tf.keras.Model.predict()` methods. The methods of the callbacks will then be called at different stages of training/evaluating/inference.\n",
-        "\n",
-        "To get started, let's import tensorflow and define a simple Sequential Keras model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Ct0VCSI2dt3a"
-      },
-      "outputs": [],
-      "source": [
-        "# Define the Keras model to add callbacks to\n",
-        "def get_model():\n",
-        "  model = tf.keras.Sequential()\n",
-        "  model.add(tf.keras.layers.Dense(1, activation = 'linear', input_dim = 784))\n",
-        "  model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.1), loss='mean_squared_error', metrics=['mae'])\n",
-        "  return model"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ySzdG1IqNgah"
-      },
-      "source": [
-        "Then, load the MNIST data for training and testing from Keras datasets API:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "fwo9LMKGNPWr"
-      },
-      "outputs": [],
-      "source": [
-        "# Load example MNIST data and pre-process it\n",
-        "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
-        "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n",
-        "x_test = x_test.reshape(10000, 784).astype('float32') / 255"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kHVK7kceNqH2"
-      },
-      "source": [
-        "Now, define a simple custom callback to track the start and end of every batch of data. During those calls, it prints the index of the current batch."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-dfuGTMINKRR"
-      },
-      "outputs": [],
-      "source": [
-        "import datetime\n",
-        "\n",
-        "class MyCustomCallback(tf.keras.callbacks.Callback):\n",
-        "\n",
-        "  def on_train_batch_begin(self, batch, logs=None):\n",
-        "    print('Training: batch {} begins at {}'.format(batch, datetime.datetime.now().time()))\n",
-        "\n",
-        "  def on_train_batch_end(self, batch, logs=None):\n",
-        "    print('Training: batch {} ends at {}'.format(batch, datetime.datetime.now().time()))\n",
-        "\n",
-        "  def on_test_batch_begin(self, batch, logs=None):\n",
-        "    print('Evaluating: batch {} begins at {}'.format(batch, datetime.datetime.now().time()))\n",
-        "\n",
-        "  def on_test_batch_end(self, batch, logs=None):\n",
-        "    print('Evaluating: batch {} ends at {}'.format(batch, datetime.datetime.now().time()))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Z4FTUUIBN3WG"
-      },
-      "source": [
-        "Providing a callback to model methods such as `tf.keras.Model.fit()` ensures the methods are called at those stages:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "NJV6Tj3sNGzg"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_model()\n",
-        "_ = model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          epochs=1,\n",
-        "          steps_per_epoch=5,\n",
-        "          verbose=0,\n",
-        "          callbacks=[MyCustomCallback()])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "fIy5JKMlZNmh"
-      },
-      "source": [
-        "## Model methods that take callbacks\n",
-        "Users can supply a list of callbacks to the following `tf.keras.Model` methods:\n",
-        "#### [`fit()`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit), [`fit_generator()`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit_generator)\n",
-        "Trains the model for a fixed number of epochs (iterations over a dataset, or data yielded batch-by-batch by a Python generator).\n",
-        "#### [`evaluate()`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#evaluate), [`evaluate_generator()`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#evaluate_generator)\n",
-        "Evaluates the model for given data or data generator. Outputs the loss and metric values from the evaluation.\n",
-        "#### [`predict()`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#predict), [`predict_generator()`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#predict_generator)\n",
-        "Generates output predictions for the input data or data generator.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "J00bXBbqdnJe"
-      },
-      "outputs": [],
-      "source": [
-        "_ = model.evaluate(x_test, y_test, batch_size=128, verbose=0, steps=5,\n",
-        "          callbacks=[MyCustomCallback()])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "13n44LVkYQsV"
-      },
-      "source": [
-        "## An overview of callback methods\n",
-        "\n",
-        "\n",
-        "### Common methods for training/testing/predicting\n",
-        "For training, testing, and predicting, following methods are provided to be overridden.\n",
-        "#### `on_(train|test|predict)_begin(self, logs=None)`\n",
-        "Called at the beginning of `fit`/`evaluate`/`predict`.\n",
-        "#### `on_(train|test|predict)_end(self, logs=None)`\n",
-        "Called at the end of `fit`/`evaluate`/`predict`.\n",
-        "#### `on_(train|test|predict)_batch_begin(self, batch, logs=None)`\n",
-        "Called right before processing a batch during training/testing/predicting. Within this method, `logs` is a dict with `batch` and `size` available keys, representing the current batch number and the size of the batch.\n",
-        "#### `on_(train|test|predict)_batch_end(self, batch, logs=None)`\n",
-        "Called at the end of training/testing/predicting a batch. Within this method, `logs` is a dict containing the stateful metrics result.\n",
-        "\n",
-        "### Training specific methods\n",
-        "In addition, for training, following are provided.\n",
-        "#### on_epoch_begin(self, epoch, logs=None)\n",
-        "Called at the beginning of an epoch during training.\n",
-        "#### on_epoch_end(self, epoch, logs=None)\n",
-        "Called at the end of an epoch during training.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "SWf3mXYoceCz"
-      },
-      "source": [
-        "### Usage of `logs` dict\n",
-        "The `logs` dict contains the loss value, and all the metrics at the end of a batch or epoch. Example includes the loss and mean absolute error."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "u4wIdcF9BjJH"
-      },
-      "outputs": [],
-      "source": [
-        "class LossAndErrorPrintingCallback(tf.keras.callbacks.Callback):\n",
-        "\n",
-        "  def on_train_batch_end(self, batch, logs=None):\n",
-        "    print('For batch {}, loss is {:7.2f}.'.format(batch, logs['loss']))\n",
-        "\n",
-        "  def on_test_batch_end(self, batch, logs=None):\n",
-        "    print('For batch {}, loss is {:7.2f}.'.format(batch, logs['loss']))\n",
-        "\n",
-        "  def on_epoch_end(self, epoch, logs=None):\n",
-        "    print('The average loss for epoch {} is {:7.2f} and mean absolute error is {:7.2f}.'.format(epoch, logs['loss'], logs['mae']))\n",
-        "\n",
-        "model = get_model()\n",
-        "_ = model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          steps_per_epoch=5,\n",
-        "          epochs=3,\n",
-        "          verbose=0,\n",
-        "          callbacks=[LossAndErrorPrintingCallback()])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LbXqvC8FHqeu"
-      },
-      "source": [
-        "Similarly, one can provide callbacks in `evaluate()` calls."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "jzTKYPQHwcxF"
-      },
-      "outputs": [],
-      "source": [
-        "_ = model.evaluate(x_test, y_test, batch_size=128, verbose=0, steps=20,\n",
-        "          callbacks=[LossAndErrorPrintingCallback()])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "HnSljqtsXKfb"
-      },
-      "source": [
-        "## Examples of Keras callback applications\n",
-        "The following section will guide you through creating simple Callback applications."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kptNF0--Lznv"
-      },
-      "source": [
-        "### Early stopping at minimum loss\n",
-        "First example showcases the creation of a `Callback` that stops the Keras training when the minimum of loss has been reached by mutating the attribute `model.stop_training` (boolean). Optionally, the user can provide an argument `patience` to specify how many epochs the training should wait before it eventually stops.\n",
-        "\n",
-        "`tf.keras.callbacks.EarlyStopping` provides a more complete and general implementation."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "BM31gfAV4mks"
-      },
-      "outputs": [],
-      "source": [
-        "import numpy as np\n",
-        "\n",
-        "class EarlyStoppingAtMinLoss(tf.keras.callbacks.Callback):\n",
-        "  \"\"\"Stop training when the loss is at its min, i.e. the loss stops decreasing.\n",
-        "\n",
-        "  Arguments:\n",
-        "      patience: Number of epochs to wait after min has been hit. After this\n",
-        "      number of no improvement, training stops.\n",
-        "  \"\"\"\n",
-        "\n",
-        "  def __init__(self, patience=0):\n",
-        "    super(EarlyStoppingAtMinLoss, self).__init__()\n",
-        "\n",
-        "    self.patience = patience\n",
-        "\n",
-        "    # best_weights to store the weights at which the minimum loss occurs.\n",
-        "    self.best_weights = None\n",
-        "\n",
-        "  def on_train_begin(self, logs=None):\n",
-        "    # The number of epoch it has waited when loss is no longer minimum.\n",
-        "    self.wait = 0\n",
-        "    # The epoch the training stops at.\n",
-        "    self.stopped_epoch = 0\n",
-        "    # Initialize the best as infinity.\n",
-        "    self.best = np.Inf\n",
-        "\n",
-        "  def on_epoch_end(self, epoch, logs=None):\n",
-        "    current = logs.get('loss')\n",
-        "    if np.less(current, self.best):\n",
-        "      self.best = current\n",
-        "      self.wait = 0\n",
-        "      # Record the best weights if current results is better (less).\n",
-        "      self.best_weights = self.model.get_weights()\n",
-        "    else:\n",
-        "      self.wait += 1\n",
-        "      if self.wait \u003e= self.patience:\n",
-        "        self.stopped_epoch = epoch\n",
-        "        self.model.stop_training = True\n",
-        "        print('Restoring model weights from the end of the best epoch.')\n",
-        "        self.model.set_weights(self.best_weights)\n",
-        "\n",
-        "  def on_train_end(self, logs=None):\n",
-        "    if self.stopped_epoch \u003e 0:\n",
-        "      print('Epoch %05d: early stopping' % (self.stopped_epoch + 1))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "xS4fa-7PFzzc"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_model()\n",
-        "_ = model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          steps_per_epoch=5,\n",
-        "          epochs=30,\n",
-        "          verbose=0,\n",
-        "          callbacks=[LossAndErrorPrintingCallback(), EarlyStoppingAtMinLoss()])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "SpVDjs_Dkkdh"
-      },
-      "source": [
-        "### Learning rate scheduling\n",
-        "\n",
-        "One thing that is commonly done in model training is changing the learning rate as more epochs have passed. Keras backend exposes get_value API which can be used to set the variables. In this example, we're showing how a custom Callback can be used to dynamically change the learning rate.\n",
-        "\n",
-        "Note: this is just an example implementation see `callbacks.LearningRateScheduler` and `keras.optimizers.schedules` for more general implementations."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "PGowEUC8klSz"
-      },
-      "outputs": [],
-      "source": [
-        "class LearningRateScheduler(tf.keras.callbacks.Callback):\n",
-        "  \"\"\"Learning rate scheduler which sets the learning rate according to schedule.\n",
-        "\n",
-        "  Arguments:\n",
-        "      schedule: a function that takes an epoch index\n",
-        "          (integer, indexed from 0) and current learning rate\n",
-        "          as inputs and returns a new learning rate as output (float).\n",
-        "  \"\"\"\n",
-        "\n",
-        "  def __init__(self, schedule):\n",
-        "    super(LearningRateScheduler, self).__init__()\n",
-        "    self.schedule = schedule\n",
-        "\n",
-        "  def on_epoch_begin(self, epoch, logs=None):\n",
-        "    if not hasattr(self.model.optimizer, 'lr'):\n",
-        "      raise ValueError('Optimizer must have a \"lr\" attribute.')\n",
-        "    # Get the current learning rate from model's optimizer.\n",
-        "    lr = float(tf.keras.backend.get_value(self.model.optimizer.lr))\n",
-        "    # Call schedule function to get the scheduled learning rate.\n",
-        "    scheduled_lr = self.schedule(epoch, lr)\n",
-        "    # Set the value back to the optimizer before this epoch starts\n",
-        "    tf.keras.backend.set_value(self.model.optimizer.lr, scheduled_lr)\n",
-        "    print('\\nEpoch %05d: Learning rate is %6.4f.' % (epoch, scheduled_lr))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1jL3pI5Ep5F8"
-      },
-      "outputs": [],
-      "source": [
-        "LR_SCHEDULE = [\n",
-        "    # (epoch to start, learning rate) tuples\n",
-        "    (3, 0.05), (6, 0.01), (9, 0.005), (12, 0.001)\n",
-        "]\n",
-        "\n",
-        "def lr_schedule(epoch, lr):\n",
-        "  \"\"\"Helper function to retrieve the scheduled learning rate based on epoch.\"\"\"\n",
-        "  if epoch \u003c LR_SCHEDULE[0][0] or epoch \u003e LR_SCHEDULE[-1][0]:\n",
-        "    return lr\n",
-        "  for i in range(len(LR_SCHEDULE)):\n",
-        "    if epoch == LR_SCHEDULE[i][0]:\n",
-        "      return LR_SCHEDULE[i][1]\n",
-        "  return lr\n",
-        "\n",
-        "model = get_model()\n",
-        "_ = model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          steps_per_epoch=5,\n",
-        "          epochs=15,\n",
-        "          verbose=0,\n",
-        "          callbacks=[LossAndErrorPrintingCallback(), LearningRateScheduler(lr_schedule)])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "9xMkm699JzK8"
-      },
-      "source": [
-        "### Standard Keras callbacks\n",
-        "Be sure to check out the existing Keras callbacks by [visiting the API doc](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks). Applications include logging to CSV, saving the model, visualizing on TensorBoard and a lot more."
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [
-        "9xMkm699JzK8"
-      ],
-      "name": "Writing custom Keras callback",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true,
-      "version": "0.3.2"
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/keras/custom_layers_and_models.ipynb b/site/en/guide/keras/custom_layers_and_models.ipynb
deleted file mode 100644
index 7cf64ce84ec..00000000000
--- a/site/en/guide/keras/custom_layers_and_models.ipynb
+++ /dev/null
@@ -1,1074 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "cPKvKMkAkRMn"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
-        "id": "TWofNaR-kS1s"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "xWVObL142EBs"
-      },
-      "source": [
-        "# Writing custom layers and models with Keras"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "p7VqaVrAvw9j"
-      },
-      "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/keras/custom_layers_and_models\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/keras/custom_layers_and_models.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/keras/custom_layers_and_models.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/keras/custom_layers_and_models.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "NrUIvL8oxlhj"
-      },
-      "source": [
-        "### Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Szd0mNROxqJ7"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.keras.backend.clear_session()  # For easy reset of notebook state."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "zVsTiIb62IbJ"
-      },
-      "source": [
-        "## The Layer class\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "D0KRUQWG2k4v"
-      },
-      "source": [
-        "### Layers encapsulate a state (weights) and some computation\n",
-        "\n",
-        "The main data structure you'll work with is the `Layer`.\n",
-        "A layer encapsulates both a state (the layer's \"weights\")\n",
-        "and a transformation from inputs to outputs (a \"call\", the layer's\n",
-        "forward pass).\n",
-        "\n",
-        "Here's a densely-connected layer. It has a state: the variables `w` and `b`.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "LisHKABR2-Nj"
-      },
-      "outputs": [],
-      "source": [
-        "from tensorflow.keras import layers\n",
-        "\n",
-        "\n",
-        "class Linear(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, units=32, input_dim=32):\n",
-        "    super(Linear, self).__init__()\n",
-        "    w_init = tf.random_normal_initializer()\n",
-        "    self.w = tf.Variable(initial_value=w_init(shape=(input_dim, units),\n",
-        "                                              dtype='float32'),\n",
-        "                         trainable=True)\n",
-        "    b_init = tf.zeros_initializer()\n",
-        "    self.b = tf.Variable(initial_value=b_init(shape=(units,),\n",
-        "                                              dtype='float32'),\n",
-        "                         trainable=True)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return tf.matmul(inputs, self.w) + self.b\n",
-        "\n",
-        "x = tf.ones((2, 2))\n",
-        "linear_layer = Linear(4, 2)\n",
-        "y = linear_layer(x)\n",
-        "print(y)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Y8RsI6Hr2OOd"
-      },
-      "source": [
-        "Note that the weights `w` and `b` are automatically tracked by the layer upon\n",
-        "being set as layer attributes:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "D7x3Hl8m2XEJ"
-      },
-      "outputs": [],
-      "source": [
-        "assert linear_layer.weights == [linear_layer.w, linear_layer.b]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "IXQPwqEs2gCH"
-      },
-      "source": [
-        "Note you also have access to a quicker shortcut for adding weight to a layer: the `add_weight` method:\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "8LSx6HDg2iPz"
-      },
-      "outputs": [],
-      "source": [
-        "class Linear(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, units=32, input_dim=32):\n",
-        "    super(Linear, self).__init__()\n",
-        "    self.w = self.add_weight(shape=(input_dim, units),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "    self.b = self.add_weight(shape=(units,),\n",
-        "                             initializer='zeros',\n",
-        "                             trainable=True)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return tf.matmul(inputs, self.w) + self.b\n",
-        "\n",
-        "x = tf.ones((2, 2))\n",
-        "linear_layer = Linear(4, 2)\n",
-        "y = linear_layer(x)\n",
-        "print(y)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vXjjEthGgr4y"
-      },
-      "source": [
-        "#### Layers can have non-trainable weights\n",
-        "\n",
-        "Besides trainable weights, you can add non-trainable weights to a layer as well.\n",
-        "Such weights are meant not to be taken into account during backpropagation,\n",
-        "when you are training the layer.\n",
-        "\n",
-        "Here's how to add and use a non-trainable weight:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "OIIfmpDIgyUy"
-      },
-      "outputs": [],
-      "source": [
-        "class ComputeSum(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, input_dim):\n",
-        "    super(ComputeSum, self).__init__()\n",
-        "    self.total = tf.Variable(initial_value=tf.zeros((input_dim,)),\n",
-        "                             trainable=False)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    self.total.assign_add(tf.reduce_sum(inputs, axis=0))\n",
-        "    return self.total\n",
-        "\n",
-        "x = tf.ones((2, 2))\n",
-        "my_sum = ComputeSum(2)\n",
-        "y = my_sum(x)\n",
-        "print(y.numpy())\n",
-        "y = my_sum(x)\n",
-        "print(y.numpy())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "qLiWVq-3g0c0"
-      },
-      "source": [
-        "It's part of `layer.weights`, but it gets categorized as a non-trainable weight:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "X7RhEZNvg2dE"
-      },
-      "outputs": [],
-      "source": [
-        "print('weights:', len(my_sum.weights))\n",
-        "print('non-trainable weights:', len(my_sum.non_trainable_weights))\n",
-        "\n",
-        "# It's not included in the trainable weights:\n",
-        "print('trainable_weights:', my_sum.trainable_weights)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "DOwYZ-Ew329E"
-      },
-      "source": [
-        "### Best practice: deferring weight creation until the shape of the inputs is known\n",
-        "\n",
-        "In the logistic regression example above, our `Linear` layer took an `input_dim` argument\n",
-        "that was used to compute the shape of the weights `w` and `b` in `__init__`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "tzxUxoPc3Esh"
-      },
-      "outputs": [],
-      "source": [
-        "class Linear(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, units=32, input_dim=32):\n",
-        "      super(Linear, self).__init__()\n",
-        "      self.w = self.add_weight(shape=(input_dim, units),\n",
-        "                               initializer='random_normal',\n",
-        "                               trainable=True)\n",
-        "      self.b = self.add_weight(shape=(units,),\n",
-        "                               initializer='zeros',\n",
-        "                               trainable=True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ejSYZGaP4CD6"
-      },
-      "source": [
-        "In many cases, you may not know in advance the size of your inputs, and you would\n",
-        "like to lazily create weights when that value becomes known,\n",
-        "some time after instantiating the layer.\n",
-        "\n",
-        "In the Keras API, we recommend creating layer weights in the `build(inputs_shape)` method of your layer.\n",
-        "Like this:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "AGhRg7Nt4EB8"
-      },
-      "outputs": [],
-      "source": [
-        "class Linear(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, units=32):\n",
-        "    super(Linear, self).__init__()\n",
-        "    self.units = units\n",
-        "\n",
-        "  def build(self, input_shape):\n",
-        "    self.w = self.add_weight(shape=(input_shape[-1], self.units),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "    self.b = self.add_weight(shape=(self.units,),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return tf.matmul(inputs, self.w) + self.b"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "0SpZzAag4Mk_"
-      },
-      "source": [
-        "The `__call__` method of your layer will automatically run `build` the first time it is called.\n",
-        "You now have a layer that's lazy and easy to use:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "_cdMFCUp4KSQ"
-      },
-      "outputs": [],
-      "source": [
-        "linear_layer = Linear(32)  # At instantiation, we don't know on what inputs this is going to get called\n",
-        "y = linear_layer(x)  # The layer's weights are created dynamically the first time the layer is called"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-kaDooBSC_Oc"
-      },
-      "source": [
-        "\n",
-        "### Layers are recursively composable\n",
-        "\n",
-        "If you assign a Layer instance as attribute of another Layer,\n",
-        "the outer layer will start tracking the weights of the inner layer.\n",
-        "\n",
-        "We recommend creating such sublayers in the `__init__` method (since the sublayers will typically have a `build` method, they will be built when the outer layer gets built)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-YPI4vwN4Ozo"
-      },
-      "outputs": [],
-      "source": [
-        "# Let's assume we are reusing the Linear class\n",
-        "# with a `build` method that we defined above.\n",
-        "\n",
-        "class MLPBlock(layers.Layer):\n",
-        "\n",
-        "  def __init__(self):\n",
-        "    super(MLPBlock, self).__init__()\n",
-        "    self.linear_1 = Linear(32)\n",
-        "    self.linear_2 = Linear(32)\n",
-        "    self.linear_3 = Linear(1)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    x = self.linear_1(inputs)\n",
-        "    x = tf.nn.relu(x)\n",
-        "    x = self.linear_2(x)\n",
-        "    x = tf.nn.relu(x)\n",
-        "    return self.linear_3(x)\n",
-        "\n",
-        "\n",
-        "mlp = MLPBlock()\n",
-        "y = mlp(tf.ones(shape=(3, 64)))  # The first call to the `mlp` will create the weights\n",
-        "print('weights:', len(mlp.weights))\n",
-        "print('trainable weights:', len(mlp.trainable_weights))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "fq5_AsbEh-BQ"
-      },
-      "source": [
-        "### Layers recursively collect losses created during the forward pass\n",
-        "\n",
-        "When writing the `call` method of a layer, you can create loss tensors that you will want to use later, when writing your training loop. This is doable by calling `self.add_loss(value)`:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "W66HsCzajERu"
-      },
-      "outputs": [],
-      "source": [
-        "# A layer that creates an activity regularization loss\n",
-        "class ActivityRegularizationLayer(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, rate=1e-2):\n",
-        "    super(ActivityRegularizationLayer, self).__init__()\n",
-        "    self.rate = rate\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    self.add_loss(self.rate * tf.reduce_sum(inputs))\n",
-        "    return inputs"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "p_dMrJ-QjcZH"
-      },
-      "source": [
-        "These losses (including those created by any inner layer) can be retrieved via `layer.losses`.\n",
-        "This property is reset at the start of every `__call__` to the top-level layer, so that `layer.losses` always contains the loss values created during the last forward pass."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "C1vYiSnVjdCc"
-      },
-      "outputs": [],
-      "source": [
-        "class OuterLayer(layers.Layer):\n",
-        "\n",
-        "  def __init__(self):\n",
-        "    super(OuterLayer, self).__init__()\n",
-        "    self.activity_reg = ActivityRegularizationLayer(1e-2)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return self.activity_reg(inputs)\n",
-        "\n",
-        "\n",
-        "layer = OuterLayer()\n",
-        "assert len(layer.losses) == 0  # No losses yet since the layer has never been called\n",
-        "_ = layer(tf.zeros(1, 1))\n",
-        "assert len(layer.losses) == 1  # We created one loss value\n",
-        "\n",
-        "# `layer.losses` gets reset at the start of each __call__\n",
-        "_ = layer(tf.zeros(1, 1))\n",
-        "assert len(layer.losses) == 1  # This is the loss created during the call above"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "9Jv3LKNfk_LL"
-      },
-      "source": [
-        "In addition, the `loss` property also contains regularization losses created for the weights of any inner layer:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "iokhhZfUlJUU"
-      },
-      "outputs": [],
-      "source": [
-        "class OuterLayer(layers.Layer):\n",
-        "\n",
-        "  def __init__(self):\n",
-        "    super(OuterLayer, self).__init__()\n",
-        "    self.dense = layers.Dense(32, kernel_regularizer=tf.keras.regularizers.l2(1e-3))\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return self.dense(inputs)\n",
-        "\n",
-        "\n",
-        "layer = OuterLayer()\n",
-        "_ = layer(tf.zeros((1, 1)))\n",
-        "\n",
-        "# This is `1e-3 * sum(layer.dense.kernel ** 2)`,\n",
-        "# created by the `kernel_regularizer` above.\n",
-        "print(layer.losses)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "P2Xdp_dvlGLG"
-      },
-      "source": [
-        "These losses are meant to be taken into account when writing training loops, like this:\n",
-        "\n",
-        "\n",
-        "```python\n",
-        "# Instantiate an optimizer.\n",
-        "optimizer = tf.keras.optimizers.SGD(learning_rate=1e-3)\n",
-        "loss_fn = keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
-        "\n",
-        "# Iterate over the batches of a dataset.\n",
-        "for x_batch_train, y_batch_train in train_dataset:\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    logits = layer(x_batch_train)  # Logits for this minibatch\n",
-        "    # Loss value for this minibatch\n",
-        "    loss_value = loss_fn(y_batch_train, logits)\n",
-        "    # Add extra losses created during this forward pass:\n",
-        "    loss_value += sum(model.losses)\n",
-        "\n",
-        "  grads = tape.gradient(loss_value, model.trainable_weights)\n",
-        "  optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
-        "```\n",
-        "\n",
-        "For a detailed guide about writing training loops, see the second section of the [guide to training and evaluation](./train_and_evaluate.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ozo04iqHohNg"
-      },
-      "source": [
-        "### You can optionally enable serialization on your layers\n",
-        "\n",
-        "If you need your custom layers to be serializable as part of a [Functional model](./functional.ipynb), you can optionally implement a `get_config` method:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ckT5Zbo0oxrz"
-      },
-      "outputs": [],
-      "source": [
-        "class Linear(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, units=32):\n",
-        "    super(Linear, self).__init__()\n",
-        "    self.units = units\n",
-        "\n",
-        "  def build(self, input_shape):\n",
-        "    self.w = self.add_weight(shape=(input_shape[-1], self.units),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "    self.b = self.add_weight(shape=(self.units,),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return tf.matmul(inputs, self.w) + self.b\n",
-        "\n",
-        "  def get_config(self):\n",
-        "    return {'units': self.units}\n",
-        "\n",
-        "\n",
-        "# Now you can recreate the layer from its config:\n",
-        "layer = Linear(64)\n",
-        "config = layer.get_config()\n",
-        "print(config)\n",
-        "new_layer = Linear.from_config(config)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "9fKngh4UozyM"
-      },
-      "source": [
-        "Note that the `__init__` method of the base `Layer` class takes some keyword arguments, in particular a `name` and a `dtype`. It's good practice to pass these arguments to the parent class in `__init__` and to include them in the layer config:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "UCMoN42no0D5"
-      },
-      "outputs": [],
-      "source": [
-        "class Linear(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, units=32, **kwargs):\n",
-        "    super(Linear, self).__init__(**kwargs)\n",
-        "    self.units = units\n",
-        "\n",
-        "  def build(self, input_shape):\n",
-        "    self.w = self.add_weight(shape=(input_shape[-1], self.units),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "    self.b = self.add_weight(shape=(self.units,),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return tf.matmul(inputs, self.w) + self.b\n",
-        "\n",
-        "  def get_config(self):\n",
-        "    config = super(Linear, self).get_config()\n",
-        "    config.update({'units': self.units})\n",
-        "    return config\n",
-        "\n",
-        "\n",
-        "layer = Linear(64)\n",
-        "config = layer.get_config()\n",
-        "print(config)\n",
-        "new_layer = Linear.from_config(config)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "sNrHV0zAo0Tc"
-      },
-      "source": [
-        "If you need more flexibility when deserializing the layer from its config, you can also override the `from_config` class method. This is the base implementation of `from_config`:\n",
-        "\n",
-        "```python\n",
-        "def from_config(cls, config):\n",
-        "  return cls(**config)\n",
-        "```\n",
-        "\n",
-        "To learn more about serialization and saving, see the complete [Guide to Saving and Serializing Models](./save_and_serialize.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-TB8iViSo4p9"
-      },
-      "source": [
-        "### Privileged `training` argument in the `call` method\n",
-        "\n",
-        "\n",
-        "Some layers, in particular the `BatchNormalization` layer and the `Dropout` layer, have different behaviors during training and inference. For such layers, it is standard practice to expose a `training` (boolean) argument in the `call` method.\n",
-        "\n",
-        "By exposing this argument in `call`, you enable the built-in training and evaluation loops (e.g. `fit`) to correctly use the layer in training and inference.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "QyI_b4Rgo-EE"
-      },
-      "outputs": [],
-      "source": [
-        "class CustomDropout(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, rate, **kwargs):\n",
-        "    super(CustomDropout, self).__init__(**kwargs)\n",
-        "    self.rate = rate\n",
-        "\n",
-        "  def call(self, inputs, training=None):\n",
-        "    if training:\n",
-        "        return tf.nn.dropout(inputs, rate=self.rate)\n",
-        "    return inputs"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "3X6eQH_K2wf1"
-      },
-      "source": [
-        "## Building Models"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "XZen-bAOE9I5"
-      },
-      "source": [
-        "\n",
-        "\n",
-        "### The Model class\n",
-        "\n",
-        "In general, you will use the `Layer` class to define inner computation blocks,\n",
-        "and will use the `Model` class to define the outer model -- the object you will train.\n",
-        "\n",
-        "For instance, in a ResNet50 model, you would have several ResNet blocks subclassing `Layer`,\n",
-        "and a single `Model` encompassing the entire ResNet50 network.\n",
-        "\n",
-        "The `Model` class has the same API as `Layer`, with the following differences:\n",
-        "\n",
-        "- It exposes built-in training, evaluation, and prediction loops (`model.fit()`, `model.evaluate()`, `model.predict()`).\n",
-        "- It exposes the list of its inner layers, via the `model.layers` property.\n",
-        "- It exposes saving and serialization APIs.\n",
-        "\n",
-        "Effectively, the \"Layer\" class corresponds to what we refer to in the literature\n",
-        "as a \"layer\" (as in \"convolution layer\" or \"recurrent layer\") or as a \"block\" (as in \"ResNet block\" or \"Inception block\").\n",
-        "\n",
-        "Meanwhile, the \"Model\" class corresponds to what is referred to in the literature\n",
-        "as a \"model\" (as in \"deep learning model\") or as a \"network\" (as in \"deep neural network\").\n",
-        "\n",
-        "For instance, we could take our mini-resnet example above, and use it to build a `Model` that we could\n",
-        "train with `fit()`, and that we could save with `save_weights`:\n",
-        "\n",
-        "```python\n",
-        "class ResNet(tf.keras.Model):\n",
-        "\n",
-        "    def __init__(self):\n",
-        "        super(ResNet, self).__init__()\n",
-        "        self.block_1 = ResNetBlock()\n",
-        "        self.block_2 = ResNetBlock()\n",
-        "        self.global_pool = layers.GlobalAveragePooling2D()\n",
-        "        self.classifier = Dense(num_classes)\n",
-        "\n",
-        "    def call(self, inputs):\n",
-        "        x = self.block_1(inputs)\n",
-        "        x = self.block_2(x)\n",
-        "        x = self.global_pool(x)\n",
-        "        return self.classifier(x)\n",
-        "\n",
-        "\n",
-        "resnet = ResNet()\n",
-        "dataset = ...\n",
-        "resnet.fit(dataset, epochs=10)\n",
-        "resnet.save_weights(filepath)\n",
-        "```\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "roVCX-TJqYzx"
-      },
-      "source": [
-        "### Putting it all together: an end-to-end example\n",
-        "\n",
-        "Here's what you've learned so far:\n",
-        "\n",
-        "- A `Layer` encapsulate a state (created in `__init__` or `build`) and some computation (in `call`).\n",
-        "- Layers can be recursively nested to create new, bigger computation blocks.\n",
-        "- Layers can create and track losses (typically regularization losses).\n",
-        "- The outer container, the thing you want to train, is a `Model`. A `Model` is just like a `Layer`, but with added training and serialization utilities.\n",
-        "\n",
-        "Let's put all of these things together into an end-to-end example: we're going to implement a Variational AutoEncoder (VAE). We'll train it on MNIST digits.\n",
-        "\n",
-        "Our VAE will be a subclass of `Model`, built as a nested composition of layers that subclass `Layer`. It will feature a regularization loss (KL divergence)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1QxkfjtzE4X2"
-      },
-      "outputs": [],
-      "source": [
-        "class Sampling(layers.Layer):\n",
-        "  \"\"\"Uses (z_mean, z_log_var) to sample z, the vector encoding a digit.\"\"\"\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    z_mean, z_log_var = inputs\n",
-        "    batch = tf.shape(z_mean)[0]\n",
-        "    dim = tf.shape(z_mean)[1]\n",
-        "    epsilon = tf.keras.backend.random_normal(shape=(batch, dim))\n",
-        "    return z_mean + tf.exp(0.5 * z_log_var) * epsilon\n",
-        "\n",
-        "\n",
-        "class Encoder(layers.Layer):\n",
-        "  \"\"\"Maps MNIST digits to a triplet (z_mean, z_log_var, z).\"\"\"\n",
-        "\n",
-        "  def __init__(self,\n",
-        "               latent_dim=32,\n",
-        "               intermediate_dim=64,\n",
-        "               name='encoder',\n",
-        "               **kwargs):\n",
-        "    super(Encoder, self).__init__(name=name, **kwargs)\n",
-        "    self.dense_proj = layers.Dense(intermediate_dim, activation='relu')\n",
-        "    self.dense_mean = layers.Dense(latent_dim)\n",
-        "    self.dense_log_var = layers.Dense(latent_dim)\n",
-        "    self.sampling = Sampling()\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    x = self.dense_proj(inputs)\n",
-        "    z_mean = self.dense_mean(x)\n",
-        "    z_log_var = self.dense_log_var(x)\n",
-        "    z = self.sampling((z_mean, z_log_var))\n",
-        "    return z_mean, z_log_var, z\n",
-        "\n",
-        "\n",
-        "class Decoder(layers.Layer):\n",
-        "  \"\"\"Converts z, the encoded digit vector, back into a readable digit.\"\"\"\n",
-        "\n",
-        "  def __init__(self,\n",
-        "               original_dim,\n",
-        "               intermediate_dim=64,\n",
-        "               name='decoder',\n",
-        "               **kwargs):\n",
-        "    super(Decoder, self).__init__(name=name, **kwargs)\n",
-        "    self.dense_proj = layers.Dense(intermediate_dim, activation='relu')\n",
-        "    self.dense_output = layers.Dense(original_dim, activation='sigmoid')\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    x = self.dense_proj(inputs)\n",
-        "    return self.dense_output(x)\n",
-        "\n",
-        "\n",
-        "class VariationalAutoEncoder(tf.keras.Model):\n",
-        "  \"\"\"Combines the encoder and decoder into an end-to-end model for training.\"\"\"\n",
-        "\n",
-        "  def __init__(self,\n",
-        "               original_dim,\n",
-        "               intermediate_dim=64,\n",
-        "               latent_dim=32,\n",
-        "               name='autoencoder',\n",
-        "               **kwargs):\n",
-        "    super(VariationalAutoEncoder, self).__init__(name=name, **kwargs)\n",
-        "    self.original_dim = original_dim\n",
-        "    self.encoder = Encoder(latent_dim=latent_dim,\n",
-        "                           intermediate_dim=intermediate_dim)\n",
-        "    self.decoder = Decoder(original_dim, intermediate_dim=intermediate_dim)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    z_mean, z_log_var, z = self.encoder(inputs)\n",
-        "    reconstructed = self.decoder(z)\n",
-        "    # Add KL divergence regularization loss.\n",
-        "    kl_loss = - 0.5 * tf.reduce_mean(\n",
-        "        z_log_var - tf.square(z_mean) - tf.exp(z_log_var) + 1)\n",
-        "    self.add_loss(kl_loss)\n",
-        "    return reconstructed"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "oDVSVl4Iu8kC"
-      },
-      "outputs": [],
-      "source": [
-        "original_dim = 784\n",
-        "vae = VariationalAutoEncoder(original_dim, 64, 32)\n",
-        "\n",
-        "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)\n",
-        "mse_loss_fn = tf.keras.losses.MeanSquaredError()\n",
-        "\n",
-        "loss_metric = tf.keras.metrics.Mean()\n",
-        "\n",
-        "(x_train, _), _ = tf.keras.datasets.mnist.load_data()\n",
-        "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n",
-        "\n",
-        "train_dataset = tf.data.Dataset.from_tensor_slices(x_train)\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(64)\n",
-        "\n",
-        "epochs = 3\n",
-        "\n",
-        "# Iterate over epochs.\n",
-        "for epoch in range(epochs):\n",
-        "  print('Start of epoch %d' % (epoch,))\n",
-        "\n",
-        "  # Iterate over the batches of the dataset.\n",
-        "  for step, x_batch_train in enumerate(train_dataset):\n",
-        "    with tf.GradientTape() as tape:\n",
-        "      reconstructed = vae(x_batch_train)\n",
-        "      # Compute reconstruction loss\n",
-        "      loss = mse_loss_fn(x_batch_train, reconstructed)\n",
-        "      loss += sum(vae.losses)  # Add KLD regularization loss\n",
-        "\n",
-        "    grads = tape.gradient(loss, vae.trainable_weights)\n",
-        "    optimizer.apply_gradients(zip(grads, vae.trainable_weights))\n",
-        "\n",
-        "    loss_metric(loss)\n",
-        "\n",
-        "    if step % 100 == 0:\n",
-        "      print('step %s: mean loss = %s' % (step, loss_metric.result()))\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "5hgOl_y34NZD"
-      },
-      "source": [
-        "Note that since the VAE is subclassing `Model`, it features built-in training loops. So you could also have trained it like this:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Y153oEzk4Piz"
-      },
-      "outputs": [],
-      "source": [
-        "vae = VariationalAutoEncoder(784, 64, 32)\n",
-        "\n",
-        "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)\n",
-        "\n",
-        "vae.compile(optimizer, loss=tf.keras.losses.MeanSquaredError())\n",
-        "vae.fit(x_train, x_train, epochs=3, batch_size=64)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ZVkFme-U0IHb"
-      },
-      "source": [
-        "### Beyond object-oriented development: the Functional API\n",
-        "\n",
-        "Was this example too much object-oriented development for you? You can also build models using [the Functional API](./functional.ipynb). Importantly, choosing one style or another does not prevent you from leveraging components written in the other style: you can always mix-and-match.\n",
-        "\n",
-        "For instance, the Functional API example below reuses the same `Sampling` layer we defined in the example above."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1QzeXGAl3Uxn"
-      },
-      "outputs": [],
-      "source": [
-        "original_dim = 784\n",
-        "intermediate_dim = 64\n",
-        "latent_dim = 32\n",
-        "\n",
-        "# Define encoder model.\n",
-        "original_inputs = tf.keras.Input(shape=(original_dim,), name='encoder_input')\n",
-        "x = layers.Dense(intermediate_dim, activation='relu')(original_inputs)\n",
-        "z_mean = layers.Dense(latent_dim, name='z_mean')(x)\n",
-        "z_log_var = layers.Dense(latent_dim, name='z_log_var')(x)\n",
-        "z = Sampling()((z_mean, z_log_var))\n",
-        "encoder = tf.keras.Model(inputs=original_inputs, outputs=z, name='encoder')\n",
-        "\n",
-        "# Define decoder model.\n",
-        "latent_inputs = tf.keras.Input(shape=(latent_dim,), name='z_sampling')\n",
-        "x = layers.Dense(intermediate_dim, activation='relu')(latent_inputs)\n",
-        "outputs = layers.Dense(original_dim, activation='sigmoid')(x)\n",
-        "decoder = tf.keras.Model(inputs=latent_inputs, outputs=outputs, name='decoder')\n",
-        "\n",
-        "# Define VAE model.\n",
-        "outputs = decoder(z)\n",
-        "vae = tf.keras.Model(inputs=original_inputs, outputs=outputs, name='vae')\n",
-        "\n",
-        "# Add KL divergence regularization loss.\n",
-        "kl_loss = - 0.5 * tf.reduce_mean(\n",
-        "    z_log_var - tf.square(z_mean) - tf.exp(z_log_var) + 1)\n",
-        "vae.add_loss(kl_loss)\n",
-        "\n",
-        "# Train.\n",
-        "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)\n",
-        "vae.compile(optimizer, loss=tf.keras.losses.MeanSquaredError())\n",
-        "vae.fit(x_train, x_train, epochs=3, batch_size=64)"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "custom_layers_and_models.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "language": "python",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/keras/functional.ipynb b/site/en/guide/keras/functional.ipynb
deleted file mode 100644
index cd8e7cffab0..00000000000
--- a/site/en/guide/keras/functional.ipynb
+++ /dev/null
@@ -1,1506 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-zMKQx6DkKwt"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
-        "id": "J307vsiDkMMW"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vCMYwDIE9dTT"
-      },
-      "source": [
-        "# The Keras functional API in TensorFlow"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "lAJfkZ-K9flj"
-      },
-      "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/keras/functional\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/keras/functional.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/keras/functional.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/keras/functional.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ITh3wzORxgpw"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "HFbM9dcfxh4l"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.keras.backend.clear_session()  # For easy reset of notebook state."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ZI47-lpfkZ5c"
-      },
-      "source": [
-        "\n",
-        "## Introduction\n",
-        "\n",
-        "You're already familiar with the use of `keras.Sequential()` to create models.\n",
-        "The Functional API is a way to create models that is more flexible than `Sequential`:\n",
-        "it can handle models with non-linear topology, models with shared layers,\n",
-        "and models with multiple inputs or outputs.\n",
-        "\n",
-        "It's based on the idea that a deep learning model\n",
-        "is usually a directed acyclic graph (DAG) of layers.\n",
-        "The Functional API a set of tools for **building graphs of layers**.\n",
-        "\n",
-        "Consider the following model:\n",
-        "\n",
-        "```\n",
-        "(input: 784-dimensional vectors)\n",
-        "       ↧\n",
-        "[Dense (64 units, relu activation)]\n",
-        "       ↧\n",
-        "[Dense (64 units, relu activation)]\n",
-        "       ↧\n",
-        "[Dense (10 units, softmax activation)]\n",
-        "       ↧\n",
-        "(output: probability distribution over 10 classes)\n",
-        "```\n",
-        "\n",
-        "It's a simple graph of 3 layers.\n",
-        "\n",
-        "To build this model with the functional API,\n",
-        "you would start by creating an input node:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Yxi0LaSHkDT-"
-      },
-      "outputs": [],
-      "source": [
-        "from tensorflow import keras\n",
-        "\n",
-        "inputs = keras.Input(shape=(784,))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Mr3Z_Pxcnf-H"
-      },
-      "source": [
-        "Here we just specify the shape of our data: 784-dimensional vectors.\n",
-        "Note that the batch size is always omitted, we only specify the shape of each sample.\n",
-        "For an input meant for images of shape `(32, 32, 3)`, we would have used:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0-2Q2nJNneIO"
-      },
-      "outputs": [],
-      "source": [
-        "img_inputs = keras.Input(shape=(32, 32, 3))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "HoMFNu-pnkgF"
-      },
-      "source": [
-        "What gets returned, `inputs`, contains information about the shape and dtype of the\n",
-        "input data that you expect to feed to your model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ddIr9LPJnibj"
-      },
-      "outputs": [],
-      "source": [
-        "inputs.shape"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "lZkLJeQonmTe"
-      },
-      "outputs": [],
-      "source": [
-        "inputs.dtype"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kZnhhndTnrzC"
-      },
-      "source": [
-        "You create a new node in the graph of layers by calling a layer on this `inputs` object:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "sMyyMTqDnpYV"
-      },
-      "outputs": [],
-      "source": [
-        "from tensorflow.keras import layers\n",
-        "\n",
-        "dense = layers.Dense(64, activation='relu')\n",
-        "x = dense(inputs)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "besm-lgFnveV"
-      },
-      "source": [
-        "The \"layer call\" action is like drawing an arrow from \"inputs\" to this layer we created.\n",
-        "We're \"passing\" the inputs to the `dense` layer, and out we get `x`.\n",
-        "\n",
-        "Let's add a few more layers to our graph of layers:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "DbF-MIO2ntf7"
-      },
-      "outputs": [],
-      "source": [
-        "x = layers.Dense(64, activation='relu')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax')(x)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "B38UlEIlnz_8"
-      },
-      "source": [
-        "At this point, we can create a `Model` by specifying its inputs and outputs in the graph of layers:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "MrSfwvl-nx9s"
-      },
-      "outputs": [],
-      "source": [
-        "model = keras.Model(inputs=inputs, outputs=outputs)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "5EeeV1xJn3jW"
-      },
-      "source": [
-        "To recap, here is our full model definition process:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "xkz7oqj2n1-q"
-      },
-      "outputs": [],
-      "source": [
-        "inputs = keras.Input(shape=(784,), name='img')\n",
-        "x = layers.Dense(64, activation='relu')(inputs)\n",
-        "x = layers.Dense(64, activation='relu')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax')(x)\n",
-        "\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs, name='mnist_model')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "jJzocCbdn6qj"
-      },
-      "source": [
-        "Let's check out what the model summary looks like:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "GirC9odQn5Ep"
-      },
-      "outputs": [],
-      "source": [
-        "model.summary()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "mbNqYAlOn-vA"
-      },
-      "source": [
-        "We can also plot the model as a graph:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "JYh2wLain8Oi"
-      },
-      "outputs": [],
-      "source": [
-        "keras.utils.plot_model(model, 'my_first_model.png')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "QtgX2RoGoDZo"
-      },
-      "source": [
-        "And optionally display the input and output shapes of each layer in the plotted graph:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "7FGesSSuoAG5"
-      },
-      "outputs": [],
-      "source": [
-        "keras.utils.plot_model(model, 'my_first_model_with_shape_info.png', show_shapes=True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "PBZ9XE6LoWvi"
-      },
-      "source": [
-        "\n",
-        "This figure and the code we wrote are virtually identical. In the code version,\n",
-        "the connection arrows are simply replaced by the call operation.\n",
-        "\n",
-        "A \"graph of layers\" is a very intuitive mental image for a deep learning model,\n",
-        "and the functional API is a way to create models that closely mirrors this mental image."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "WUUHMaKLoZDn"
-      },
-      "source": [
-        "\n",
-        "\n",
-        "## Training, evaluation, and inference\n",
-        "\n",
-        "Training, evaluation, and inference work exactly in the same way for models built\n",
-        "using the Functional API as for Sequential models.\n",
-        "\n",
-        "Here is a quick demonstration.\n",
-        "\n",
-        "Here we load MNIST image data, reshape it into vectors,\n",
-        "fit the model on the data (while monitoring performance on a validation split),\n",
-        "and finally we evaluate our model on the test data:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "DnHvkD22oFEY"
-      },
-      "outputs": [],
-      "source": [
-        "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
-        "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n",
-        "x_test = x_test.reshape(10000, 784).astype('float32') / 255\n",
-        "\n",
-        "model.compile(loss='sparse_categorical_crossentropy',\n",
-        "              optimizer=keras.optimizers.RMSprop(),\n",
-        "              metrics=['accuracy'])\n",
-        "history = model.fit(x_train, y_train,\n",
-        "                    batch_size=64,\n",
-        "                    epochs=5,\n",
-        "                    validation_split=0.2)\n",
-        "test_scores = model.evaluate(x_test, y_test, verbose=2)\n",
-        "print('Test loss:', test_scores[0])\n",
-        "print('Test accuracy:', test_scores[1])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "c3nq2fjiLCkE"
-      },
-      "source": [
-        "For a complete guide about model training and evaluation, see [guide to training and evaluation](./train_and_evaluate.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "XOsL56zDorLh"
-      },
-      "source": [
-        "## Saving and serialization\n",
-        "\n",
-        "Saving and serialization work exactly in the same way for models built\n",
-        "using the Functional API as for Sequential models.\n",
-        "\n",
-        "To standard way to save a Functional model is to call `model.save()` to save the whole model into a single file.\n",
-        "You can later recreate the same model from this file, even if you no longer have access to the code\n",
-        "that created the model.\n",
-        "\n",
-        "This file includes:\n",
-        "- The model's architecture\n",
-        "- The model's weight values (which were learned during training)\n",
-        "- The model's training config (what you passed to `compile`), if any\n",
-        "- The optimizer and its state, if any (this enables you to restart training where you left off)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "kN-AO7xvobtr"
-      },
-      "outputs": [],
-      "source": [
-        "model.save('path_to_my_model.h5')\n",
-        "del model\n",
-        "# Recreate the exact same model purely from the file:\n",
-        "model = keras.models.load_model('path_to_my_model.h5')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "u0J0tFPHK4pb"
-      },
-      "source": [
-        "For a complete guide about model saving, see [Guide to Saving and Serializing Models](./save_and_serialize.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "lKz1WWr2LUzF"
-      },
-      "source": [
-        "## Using the same graph of layers to define multiple models\n",
-        "\n",
-        "\n",
-        "In the functional API, models are created by specifying their inputs\n",
-        "and outputs in a graph of layers. That means that a single graph of layers\n",
-        "can be used to generate multiple models.\n",
-        "\n",
-        "In the example below, we use the same stack of layers to instantiate two models:\n",
-        "an `encoder` model that turns image inputs into 16-dimensional vectors,\n",
-        "and an end-to-end `autoencoder` model for training.\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "WItZQr6LuVbF"
-      },
-      "outputs": [],
-      "source": [
-        "encoder_input = keras.Input(shape=(28, 28, 1), name='img')\n",
-        "x = layers.Conv2D(16, 3, activation='relu')(encoder_input)\n",
-        "x = layers.Conv2D(32, 3, activation='relu')(x)\n",
-        "x = layers.MaxPooling2D(3)(x)\n",
-        "x = layers.Conv2D(32, 3, activation='relu')(x)\n",
-        "x = layers.Conv2D(16, 3, activation='relu')(x)\n",
-        "encoder_output = layers.GlobalMaxPooling2D()(x)\n",
-        "\n",
-        "encoder = keras.Model(encoder_input, encoder_output, name='encoder')\n",
-        "encoder.summary()\n",
-        "\n",
-        "x = layers.Reshape((4, 4, 1))(encoder_output)\n",
-        "x = layers.Conv2DTranspose(16, 3, activation='relu')(x)\n",
-        "x = layers.Conv2DTranspose(32, 3, activation='relu')(x)\n",
-        "x = layers.UpSampling2D(3)(x)\n",
-        "x = layers.Conv2DTranspose(16, 3, activation='relu')(x)\n",
-        "decoder_output = layers.Conv2DTranspose(1, 3, activation='relu')(x)\n",
-        "\n",
-        "autoencoder = keras.Model(encoder_input, decoder_output, name='autoencoder')\n",
-        "autoencoder.summary()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "oNeg3WWFuYZK"
-      },
-      "source": [
-        "Note that we make the decoding architecture strictly symmetrical to the encoding architecture,\n",
-        "so that we get an output shape that is the same as the input shape `(28, 28, 1)`.\n",
-        "The reverse of a `Conv2D` layer is a `Conv2DTranspose` layer, and the reverse of a `MaxPooling2D`\n",
-        "layer is an `UpSampling2D` layer."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "h1FVW4j-uc6Y"
-      },
-      "source": [
-        "\n",
-        "## All models are callable, just like layers\n",
-        "\n",
-        "You can treat any model as if it were a layer, by calling it on an `Input` or on the output of another layer.\n",
-        "Note that by calling a model you aren't just reusing the architecture of the model, you're also reusing its weights.\n",
-        "\n",
-        "Let's see this in action. Here's a different take on the autoencoder example that creates an encoder model, a decoder model,\n",
-        "and chain them in two calls to obtain the autoencoder model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Ld7KdsQ_uZbr"
-      },
-      "outputs": [],
-      "source": [
-        "encoder_input = keras.Input(shape=(28, 28, 1), name='original_img')\n",
-        "x = layers.Conv2D(16, 3, activation='relu')(encoder_input)\n",
-        "x = layers.Conv2D(32, 3, activation='relu')(x)\n",
-        "x = layers.MaxPooling2D(3)(x)\n",
-        "x = layers.Conv2D(32, 3, activation='relu')(x)\n",
-        "x = layers.Conv2D(16, 3, activation='relu')(x)\n",
-        "encoder_output = layers.GlobalMaxPooling2D()(x)\n",
-        "\n",
-        "encoder = keras.Model(encoder_input, encoder_output, name='encoder')\n",
-        "encoder.summary()\n",
-        "\n",
-        "decoder_input = keras.Input(shape=(16,), name='encoded_img')\n",
-        "x = layers.Reshape((4, 4, 1))(decoder_input)\n",
-        "x = layers.Conv2DTranspose(16, 3, activation='relu')(x)\n",
-        "x = layers.Conv2DTranspose(32, 3, activation='relu')(x)\n",
-        "x = layers.UpSampling2D(3)(x)\n",
-        "x = layers.Conv2DTranspose(16, 3, activation='relu')(x)\n",
-        "decoder_output = layers.Conv2DTranspose(1, 3, activation='relu')(x)\n",
-        "\n",
-        "decoder = keras.Model(decoder_input, decoder_output, name='decoder')\n",
-        "decoder.summary()\n",
-        "\n",
-        "autoencoder_input = keras.Input(shape=(28, 28, 1), name='img')\n",
-        "encoded_img = encoder(autoencoder_input)\n",
-        "decoded_img = decoder(encoded_img)\n",
-        "autoencoder = keras.Model(autoencoder_input, decoded_img, name='autoencoder')\n",
-        "autoencoder.summary()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "icQFny_huiXC"
-      },
-      "source": [
-        "As you can see, model can be nested: a model can contain submodels (since a model is just like a layer).\n",
-        "\n",
-        "A common use case for model nesting is *ensembling*.\n",
-        "As an example, here's how to ensemble a set of models into a single model that averages their predictions:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ZBlZbRn5uk-9"
-      },
-      "outputs": [],
-      "source": [
-        "def get_model():\n",
-        "  inputs = keras.Input(shape=(128,))\n",
-        "  outputs = layers.Dense(1, activation='sigmoid')(inputs)\n",
-        "  return keras.Model(inputs, outputs)\n",
-        "\n",
-        "model1 = get_model()\n",
-        "model2 = get_model()\n",
-        "model3 = get_model()\n",
-        "\n",
-        "inputs = keras.Input(shape=(128,))\n",
-        "y1 = model1(inputs)\n",
-        "y2 = model2(inputs)\n",
-        "y3 = model3(inputs)\n",
-        "outputs = layers.average([y1, y2, y3])\n",
-        "ensemble_model = keras.Model(inputs=inputs, outputs=outputs)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "e1za1TZxuoId"
-      },
-      "source": [
-        "## Manipulating complex graph topologies\n",
-        "\n",
-        "\n",
-        "### Models with multiple inputs and outputs\n",
-        "\n",
-        "The functional API makes it easy to manipulate multiple inputs and outputs.\n",
-        "This cannot be handled with the Sequential API.\n",
-        "\n",
-        "Here's a simple example.\n",
-        "\n",
-        "Let's say you're building a system for ranking custom issue tickets by priority and routing them to the right department.\n",
-        "\n",
-        "You model will have 3 inputs:\n",
-        "\n",
-        "- Title of the ticket (text input)\n",
-        "- Text body of the ticket (text input)\n",
-        "- Any tags added by the user (categorical input)\n",
-        "\n",
-        "It will have two outputs:\n",
-        "\n",
-        "- Priority score between 0 and 1 (scalar sigmoid output)\n",
-        "- The department that should handle the ticket (softmax output over the set of departments)\n",
-        "\n",
-        "Let's built this model in a few lines with the Functional API."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Gt91OtzbutJy"
-      },
-      "outputs": [],
-      "source": [
-        "num_tags = 12  # Number of unique issue tags\n",
-        "num_words = 10000  # Size of vocabulary obtained when preprocessing text data\n",
-        "num_departments = 4  # Number of departments for predictions\n",
-        "\n",
-        "title_input = keras.Input(shape=(None,), name='title')  # Variable-length sequence of ints\n",
-        "body_input = keras.Input(shape=(None,), name='body')  # Variable-length sequence of ints\n",
-        "tags_input = keras.Input(shape=(num_tags,), name='tags')  # Binary vectors of size `num_tags`\n",
-        "\n",
-        "# Embed each word in the title into a 64-dimensional vector\n",
-        "title_features = layers.Embedding(num_words, 64)(title_input)\n",
-        "# Embed each word in the text into a 64-dimensional vector\n",
-        "body_features = layers.Embedding(num_words, 64)(body_input)\n",
-        "\n",
-        "# Reduce sequence of embedded words in the title into a single 128-dimensional vector\n",
-        "title_features = layers.LSTM(128)(title_features)\n",
-        "# Reduce sequence of embedded words in the body into a single 32-dimensional vector\n",
-        "body_features = layers.LSTM(32)(body_features)\n",
-        "\n",
-        "# Merge all available features into a single large vector via concatenation\n",
-        "x = layers.concatenate([title_features, body_features, tags_input])\n",
-        "\n",
-        "# Stick a logistic regression for priority prediction on top of the features\n",
-        "priority_pred = layers.Dense(1, activation='sigmoid', name='priority')(x)\n",
-        "# Stick a department classifier on top of the features\n",
-        "department_pred = layers.Dense(num_departments, activation='softmax', name='department')(x)\n",
-        "\n",
-        "# Instantiate an end-to-end model predicting both priority and department\n",
-        "model = keras.Model(inputs=[title_input, body_input, tags_input],\n",
-        "                    outputs=[priority_pred, department_pred])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "KIS7lqW0uwh-"
-      },
-      "source": [
-        "Let's plot the model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "IMij4gzhuzYV"
-      },
-      "outputs": [],
-      "source": [
-        "keras.utils.plot_model(model, 'multi_input_and_output_model.png', show_shapes=True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "oOyuig2Hu00p"
-      },
-      "source": [
-        "When compiling this model, we can assign different losses to each output.\n",
-        "You can even assign different weights to each loss, to modulate their\n",
-        "contribution to the total training loss."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Crtdpi5Uu2cX"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "              loss=['binary_crossentropy', 'categorical_crossentropy'],\n",
-        "              loss_weights=[1., 0.2])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "t42Jrn0Yu5jL"
-      },
-      "source": [
-        "Since we gave names to our output layers, we could also specify the loss like this:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "dPM0EwW_u6mV"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "              loss={'priority': 'binary_crossentropy',\n",
-        "                    'department': 'categorical_crossentropy'},\n",
-        "              loss_weights=[1., 0.2])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "bpTx2sXnu3-W"
-      },
-      "source": [
-        "We can train the model by passing lists of Numpy arrays of inputs and targets:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "nB-upOoGu_k4"
-      },
-      "outputs": [],
-      "source": [
-        "import numpy as np\n",
-        "\n",
-        "# Dummy input data\n",
-        "title_data = np.random.randint(num_words, size=(1280, 10))\n",
-        "body_data = np.random.randint(num_words, size=(1280, 100))\n",
-        "tags_data = np.random.randint(2, size=(1280, num_tags)).astype('float32')\n",
-        "# Dummy target data\n",
-        "priority_targets = np.random.random(size=(1280, 1))\n",
-        "dept_targets = np.random.randint(2, size=(1280, num_departments))\n",
-        "\n",
-        "model.fit({'title': title_data, 'body': body_data, 'tags': tags_data},\n",
-        "          {'priority': priority_targets, 'department': dept_targets},\n",
-        "          epochs=2,\n",
-        "          batch_size=32)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "qNguhBWuvCtz"
-      },
-      "source": [
-        "When calling fit with a `Dataset` object, it should yield either a\n",
-        "tuple of lists like `([title_data, body_data, tags_data], [priority_targets, dept_targets])`\n",
-        "or a tuple of dictionaries like\n",
-        "`({'title': title_data, 'body': body_data, 'tags': tags_data}, {'priority': priority_targets, 'department': dept_targets})`.\n",
-        "\n",
-        "For more detailed explanation, refer to the complete guide [guide to training and evaluation](./train_and_evaluate.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "tR0X5tTOvPyg"
-      },
-      "source": [
-        "### A toy resnet model\n",
-        "\n",
-        "In addition to models with multiple inputs and outputs,\n",
-        "the Functional API makes it easy to manipulate non-linear connectivity topologies,\n",
-        "that is to say, models where layers are not connected sequentially.\n",
-        "This also cannot be handled with the Sequential API (as the name indicates).\n",
-        "\n",
-        "A common use case for this is residual connections.\n",
-        "\n",
-        "Let's build a toy ResNet model for CIFAR10 to demonstrate this."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "VzMoYrMNvXrm"
-      },
-      "outputs": [],
-      "source": [
-        "inputs = keras.Input(shape=(32, 32, 3), name='img')\n",
-        "x = layers.Conv2D(32, 3, activation='relu')(inputs)\n",
-        "x = layers.Conv2D(64, 3, activation='relu')(x)\n",
-        "block_1_output = layers.MaxPooling2D(3)(x)\n",
-        "\n",
-        "x = layers.Conv2D(64, 3, activation='relu', padding='same')(block_1_output)\n",
-        "x = layers.Conv2D(64, 3, activation='relu', padding='same')(x)\n",
-        "block_2_output = layers.add([x, block_1_output])\n",
-        "\n",
-        "x = layers.Conv2D(64, 3, activation='relu', padding='same')(block_2_output)\n",
-        "x = layers.Conv2D(64, 3, activation='relu', padding='same')(x)\n",
-        "block_3_output = layers.add([x, block_2_output])\n",
-        "\n",
-        "x = layers.Conv2D(64, 3, activation='relu')(block_3_output)\n",
-        "x = layers.GlobalAveragePooling2D()(x)\n",
-        "x = layers.Dense(256, activation='relu')(x)\n",
-        "x = layers.Dropout(0.5)(x)\n",
-        "outputs = layers.Dense(10, activation='softmax')(x)\n",
-        "\n",
-        "model = keras.Model(inputs, outputs, name='toy_resnet')\n",
-        "model.summary()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ISQX32bgrkis"
-      },
-      "source": [
-        "Let's plot the model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "pNFVkAd3rlCM"
-      },
-      "outputs": [],
-      "source": [
-        "keras.utils.plot_model(model, 'mini_resnet.png', show_shapes=True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ECcG87yZrxp5"
-      },
-      "source": [
-        "Let's train it:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "_iXGz5XEryou"
-      },
-      "outputs": [],
-      "source": [
-        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar10.load_data()\n",
-        "x_train = x_train.astype('float32') / 255.\n",
-        "x_test = x_test.astype('float32') / 255.\n",
-        "y_train = keras.utils.to_categorical(y_train, 10)\n",
-        "y_test = keras.utils.to_categorical(y_test, 10)\n",
-        "\n",
-        "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "              loss='categorical_crossentropy',\n",
-        "              metrics=['acc'])\n",
-        "model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          epochs=1,\n",
-        "          validation_split=0.2)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "XQfg0JUkr7SH"
-      },
-      "source": [
-        "## Sharing layers\n",
-        "\n",
-        "Another good use for the functional API are models that use shared layers. Shared layers are layer instances that get reused multiple times in a same model: they learn features that correspond to multiple paths in the graph-of-layers.\n",
-        "\n",
-        "Shared layers are often used to encode inputs that come from similar spaces (say, two different pieces of text that feature similar vocabulary), since they enable sharing of information across these different inputs, and they make it possible to train such a model on less data. If a given word is seen in one of the inputs, that will benefit the processing of all inputs that go through the shared layer.\n",
-        "\n",
-        "To share a layer in the Functional API, just call the same layer instance multiple times. For instance, here's an `Embedding` layer shared across two different text inputs:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "R9pAPQCnKuMR"
-      },
-      "outputs": [],
-      "source": [
-        "# Embedding for 1000 unique words mapped to 128-dimensional vectors\n",
-        "shared_embedding = layers.Embedding(1000, 128)\n",
-        "\n",
-        "# Variable-length sequence of integers\n",
-        "text_input_a = keras.Input(shape=(None,), dtype='int32')\n",
-        "\n",
-        "# Variable-length sequence of integers\n",
-        "text_input_b = keras.Input(shape=(None,), dtype='int32')\n",
-        "\n",
-        "# We reuse the same layer to encode both inputs\n",
-        "encoded_input_a = shared_embedding(text_input_a)\n",
-        "encoded_input_b = shared_embedding(text_input_b)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "xNEKvfUpr-Kf"
-      },
-      "source": [
-        "## Extracting and reusing nodes in the graph of layers"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "JHVGI6bEr-ze"
-      },
-      "source": [
-        "Because the graph of layers you are manipulating in the Functional API is a static datastructure, it can be accessed and inspected. This is how we are able to plot Functional models as images, for instance.\n",
-        "\n",
-        "This also means that we can access the activations of intermediate layers (\"nodes\" in the graph) and reuse them elsewhere. This is extremely useful for feature extraction, for example!\n",
-        "\n",
-        "Let's look at an example. This is a VGG19 model with weights pre-trained on ImageNet:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "c-gl3xHBH-oX"
-      },
-      "outputs": [],
-      "source": [
-        "from tensorflow.keras.applications import VGG19\n",
-        "\n",
-        "vgg19 = VGG19()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "AKefin_xIGBP"
-      },
-      "source": [
-        "And these are the intermediate activations of the model, obtained by querying the graph datastructure:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1_Ap05fgIRgE"
-      },
-      "outputs": [],
-      "source": [
-        "features_list = [layer.output for layer in vgg19.layers]"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "H1zx5qM7IYu4"
-      },
-      "source": [
-        "We can use these features to create a new feature-extraction model, that returns the values of the intermediate layer activations -- and we can do all of this in 3 lines."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "NrU82Pa8Igwo"
-      },
-      "outputs": [],
-      "source": [
-        "feat_extraction_model = keras.Model(inputs=vgg19.input, outputs=features_list)\n",
-        "\n",
-        "img = np.random.random((1, 224, 224, 3)).astype('float32')\n",
-        "extracted_features = feat_extraction_model(img)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "G-e2-jNCLIqy"
-      },
-      "source": [
-        "This comes in handy when [implementing neural style transfer](https://medium.com/tensorflow/neural-style-transfer-creating-art-with-deep-learning-using-tf-keras-and-eager-execution-7d541ac31398), among other things."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "t9M2Uvi3sBy0"
-      },
-      "source": [
-        "## Extending the API by writing custom layers\n",
-        "\n",
-        "tf.keras has a wide range of built-in layers. Here are a few examples:\n",
-        "\n",
-        "- Convolutional layers: `Conv1D`, `Conv2D`, `Conv3D`, `Conv2DTranspose`, etc.\n",
-        "- Pooling layers: `MaxPooling1D`, `MaxPooling2D`, `MaxPooling3D`, `AveragePooling1D`, etc.\n",
-        "- RNN layers: `GRU`, `LSTM`, `ConvLSTM2D`, etc.\n",
-        "- `BatchNormalization`, `Dropout`, `Embedding`, etc.\n",
-        "\n",
-        "If you don't find what you need, it's easy to extend the API by creating your own layers.\n",
-        "\n",
-        "All layers subclass the `Layer` class and implement:\n",
-        "- A `call` method, that specifies the computation done by the layer.\n",
-        "- A `build` method, that creates the weights of the layer (note that this is just a style convention; you could create weights in `__init__` as well).\n",
-        "\n",
-        "To learn more about creating layers from scratch, check out the guide [Guide to writing layers and models from scratch](./custom_layers_and_models.ipynb).\n",
-        "\n",
-        "Here's a simple implementation of a `Dense` layer:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ztAmarbgNV6V"
-      },
-      "outputs": [],
-      "source": [
-        "class CustomDense(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, units=32):\n",
-        "    super(CustomDense, self).__init__()\n",
-        "    self.units = units\n",
-        "\n",
-        "  def build(self, input_shape):\n",
-        "    self.w = self.add_weight(shape=(input_shape[-1], self.units),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "    self.b = self.add_weight(shape=(self.units,),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return tf.matmul(inputs, self.w) + self.b\n",
-        "\n",
-        "inputs = keras.Input((4,))\n",
-        "outputs = CustomDense(10)(inputs)\n",
-        "\n",
-        "model = keras.Model(inputs, outputs)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "NXxp_32bNWTy"
-      },
-      "source": [
-        "If you want your custom layer to support serialization, you should also define a `get_config` method,\n",
-        "that returns the constructor arguments of the layer instance:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "K3OQ4XxzNfAZ"
-      },
-      "outputs": [],
-      "source": [
-        "class CustomDense(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, units=32):\n",
-        "    super(CustomDense, self).__init__()\n",
-        "    self.units = units\n",
-        "\n",
-        "  def build(self, input_shape):\n",
-        "    self.w = self.add_weight(shape=(input_shape[-1], self.units),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "    self.b = self.add_weight(shape=(self.units,),\n",
-        "                             initializer='random_normal',\n",
-        "                             trainable=True)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return tf.matmul(inputs, self.w) + self.b\n",
-        "\n",
-        "  def get_config(self):\n",
-        "    return {'units': self.units}\n",
-        "\n",
-        "\n",
-        "inputs = keras.Input((4,))\n",
-        "outputs = CustomDense(10)(inputs)\n",
-        "\n",
-        "model = keras.Model(inputs, outputs)\n",
-        "config = model.get_config()\n",
-        "\n",
-        "new_model = keras.Model.from_config(\n",
-        "    config, custom_objects={'CustomDense': CustomDense})"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kXg6hZN_NfN8"
-      },
-      "source": [
-        "Optionally, you could also implement the classmethod `from_config(cls, config)`, which is in charge of recreating a layer instance given its config dictionary. The default implementation of `from_config` is:\n",
-        "\n",
-        "```python\n",
-        "def from_config(cls, config):\n",
-        "  return cls(**config)\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ifOVqn84sCNU"
-      },
-      "source": [
-        "## When to use the Functional API\n",
-        "\n",
-        "How to decide whether to use the Functional API to create a new model, or just subclass the `Model` class directly?\n",
-        "\n",
-        "In general, the Functional API is higher-level, easier \u0026 safer to use, and has a number of features that subclassed Models do not support.\n",
-        "\n",
-        "However, Model subclassing gives you greater flexibility when creating models that are not easily expressible as directed acyclic graphs of layers (for instance, you could not implement a Tree-RNN with the Functional API, you would have to subclass `Model` directly).\n",
-        "\n",
-        "\n",
-        "### Here are the strengths of the Functional API:\n",
-        "\n",
-        "The properties listed below are all true for Sequential models as well (which are also data structures), but they aren't true for subclassed models (which are Python bytecode, not data structures).\n",
-        "\n",
-        "\n",
-        "#### It is less verbose.\n",
-        "\n",
-        "No `super(MyClass, self).__init__(...)`, no `def call(self, ...):`, etc.\n",
-        "\n",
-        "Compare:\n",
-        "\n",
-        "```python\n",
-        "inputs = keras.Input(shape=(32,))\n",
-        "x = layers.Dense(64, activation='relu')(inputs)\n",
-        "outputs = layers.Dense(10)(x)\n",
-        "mlp = keras.Model(inputs, outputs)\n",
-        "```\n",
-        "\n",
-        "With the subclassed version:\n",
-        "\n",
-        "```python\n",
-        "class MLP(keras.Model):\n",
-        "\n",
-        "  def __init__(self, **kwargs):\n",
-        "    super(MLP, self).__init__(**kwargs)\n",
-        "    self.dense_1 = layers.Dense(64, activation='relu')\n",
-        "    self.dense_2 = layers.Dense(10)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    x = self.dense_1(inputs)\n",
-        "    return self.dense_2(x)\n",
-        "\n",
-        "# Instantiate the model.\n",
-        "mlp = MLP()\n",
-        "# Necessary to create the model's state.\n",
-        "# The model doesn't have a state until it's called at least once.\n",
-        "_ = mlp(tf.zeros((1, 32)))\n",
-        "```\n",
-        "\n",
-        "\n",
-        "#### It validates your model while you're defining it.\n",
-        "\n",
-        "In the Functional API, your input specification (shape and dtype) is created in advance (via `Input`), and every time you call a layer, the layer checks that the specification passed to it matches its assumptions, and it will raise a helpful error message if not.\n",
-        "\n",
-        "This guarantees that any model you can build with the Functional API will run. All debugging (other than convergence-related debugging) will happen statically during the model construction, and not at execution time. This is similar to typechecking in a compiler.\n",
-        "\n",
-        "\n",
-        "#### Your Functional model is plottable and inspectable.\n",
-        "\n",
-        "You can plot the model as a graph, and you can easily access intermediate nodes in this graph -- for instance, to extract and reuse the activations of intermediate layers, as we saw in a previous example:\n",
-        "\n",
-        "```python\n",
-        "features_list = [layer.output for layer in vgg19.layers]\n",
-        "feat_extraction_model = keras.Model(inputs=vgg19.input, outputs=features_list)\n",
-        "```\n",
-        "\n",
-        "\n",
-        "#### Your Functional model can be serialized or cloned.\n",
-        "\n",
-        "Because a Functional model is a data structure rather than a piece of code, it is safely serializable and can be saved as a single file that allows you to recreate the exact same model without having access to any of the original code. See our [saving and serialization guide](./save_and_serialize.ipynb) for more details.\n",
-        "\n",
-        "\n",
-        "### Here are the weaknesses of the Functional API:\n",
-        "\n",
-        "\n",
-        "#### It does not support dynamic architectures.\n",
-        "\n",
-        "The Functional API treats models as DAGs of layers. This is true for most deep learning architectures, but not all: for instance, recursive networks or Tree RNNs do not follow this assumption and cannot be implemented in the Functional API.\n",
-        "\n",
-        "\n",
-        "#### Sometimes, you just need to write everything from scratch.\n",
-        "\n",
-        "When writing advanced architectures, you may want to do things that are outside the scope of \"defining a DAG of layers\": for instance, you may want to expose multiple custom training and inference methods on your model instance. This requires subclassing.\n",
-        "\n",
-        "\n",
-        "---\n",
-        "\n",
-        "\n",
-        "To dive more in-depth into the differences between the Functional API and Model subclassing, you can read [What are Symbolic and Imperative APIs in TensorFlow 2.0?](https://medium.com/tensorflow/what-are-symbolic-and-imperative-apis-in-tensorflow-2-0-dfccecb01021)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Ym1jrCqusGvj"
-      },
-      "source": [
-        "## Mix-and-matching different API styles\n",
-        "\n",
-        "Importantly, choosing between the Functional API or Model subclassing isn't a binary decision that restricts you to one category of models. All models in the tf.keras API can interact with each, whether they're Sequential models, Functional models, or subclassed Models/Layers written from scratch.\n",
-        "\n",
-        "You can always use a Functional model or Sequential model as part of a subclassed Model/Layer:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "9zF5YTLy_vGZ"
-      },
-      "outputs": [],
-      "source": [
-        "units = 32\n",
-        "timesteps = 10\n",
-        "input_dim = 5\n",
-        "\n",
-        "# Define a Functional model\n",
-        "inputs = keras.Input((None, units))\n",
-        "x = layers.GlobalAveragePooling1D()(inputs)\n",
-        "outputs = layers.Dense(1, activation='sigmoid')(x)\n",
-        "model = keras.Model(inputs, outputs)\n",
-        "\n",
-        "\n",
-        "class CustomRNN(layers.Layer):\n",
-        "\n",
-        "  def __init__(self):\n",
-        "    super(CustomRNN, self).__init__()\n",
-        "    self.units = units\n",
-        "    self.projection_1 = layers.Dense(units=units, activation='tanh')\n",
-        "    self.projection_2 = layers.Dense(units=units, activation='tanh')\n",
-        "    # Our previously-defined Functional model\n",
-        "    self.classifier = model\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    outputs = []\n",
-        "    state = tf.zeros(shape=(inputs.shape[0], self.units))\n",
-        "    for t in range(inputs.shape[1]):\n",
-        "      x = inputs[:, t, :]\n",
-        "      h = self.projection_1(x)\n",
-        "      y = h + self.projection_2(state)\n",
-        "      state = y\n",
-        "      outputs.append(y)\n",
-        "    features = tf.stack(outputs, axis=1)\n",
-        "    print(features.shape)\n",
-        "    return self.classifier(features)\n",
-        "\n",
-        "rnn_model = CustomRNN()\n",
-        "_ = rnn_model(tf.zeros((1, timesteps, input_dim)))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "oxW1d0a8_ufg"
-      },
-      "source": [
-        "Inversely, you can use any subclassed Layer or Model in the Functional API as long as it implements a `call` method that follows one of the following patterns:\n",
-        "\n",
-        "- `call(self, inputs, **kwargs)`  where `inputs` is a tensor or a nested structure of tensors (e.g. a list of tensors), and where `**kwargs` are non-tensor arguments (non-inputs).\n",
-        "- `call(self, inputs, training=None, **kwargs)` where `training` is a boolean indicating whether the layer should behave in training mode and inference mode.\n",
-        "- `call(self, inputs, mask=None, **kwargs)` where `mask` is a boolean mask tensor (useful for RNNs, for instance).\n",
-        "- `call(self, inputs, training=None, mask=None, **kwargs)` -- of course you can have both masking and training-specific behavior at the same time.\n",
-        "\n",
-        "In addition, if you implement the `get_config` method on your custom Layer or Model, the Functional models you create with it will still be serializable and clonable.\n",
-        "\n",
-        "Here's a quick example where we use a custom RNN written from scratch in a Functional model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "TmTEZ6F3ArJR"
-      },
-      "outputs": [],
-      "source": [
-        "units = 32\n",
-        "timesteps = 10\n",
-        "input_dim = 5\n",
-        "batch_size = 16\n",
-        "\n",
-        "\n",
-        "class CustomRNN(layers.Layer):\n",
-        "\n",
-        "  def __init__(self):\n",
-        "    super(CustomRNN, self).__init__()\n",
-        "    self.units = units\n",
-        "    self.projection_1 = layers.Dense(units=units, activation='tanh')\n",
-        "    self.projection_2 = layers.Dense(units=units, activation='tanh')\n",
-        "    self.classifier = layers.Dense(1, activation='sigmoid')\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    outputs = []\n",
-        "    state = tf.zeros(shape=(inputs.shape[0], self.units))\n",
-        "    for t in range(inputs.shape[1]):\n",
-        "      x = inputs[:, t, :]\n",
-        "      h = self.projection_1(x)\n",
-        "      y = h + self.projection_2(state)\n",
-        "      state = y\n",
-        "      outputs.append(y)\n",
-        "    features = tf.stack(outputs, axis=1)\n",
-        "    return self.classifier(features)\n",
-        "\n",
-        "# Note that we specify a static batch size for the inputs with the `batch_shape`\n",
-        "# arg, because the inner computation of `CustomRNN` requires a static batch size\n",
-        "# (when we create the `state` zeros tensor).\n",
-        "inputs = keras.Input(batch_shape=(batch_size, timesteps, input_dim))\n",
-        "x = layers.Conv1D(32, 3)(inputs)\n",
-        "outputs = CustomRNN()(x)\n",
-        "\n",
-        "model = keras.Model(inputs, outputs)\n",
-        "\n",
-        "rnn_model = CustomRNN()\n",
-        "_ = rnn_model(tf.zeros((1, 10, 5)))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "6VxcYb4qArlb"
-      },
-      "source": [
-        "This concludes our guide on the Functional API!\n",
-        "\n",
-        "Now you have at your fingertips a powerful set of tools for building deep learning models."
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "functional.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true,
-      "version": "0.3.2"
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/keras/index.md b/site/en/guide/keras/index.md
deleted file mode 100644
index 5d38ed90af2..00000000000
--- a/site/en/guide/keras/index.md
+++ /dev/null
@@ -1,35 +0,0 @@
-# Keras
-
-`tf.keras` is TensorFlow's high-level API for building and training deep
-learning models. It's used for fast prototyping, state-of-the-art research,
-and production, with three key advantages:
-
--   *User-friendly*
 Keras has a simple, consistent interface optimized for
-    common use cases. It provides clear and actionable feedback for user errors.
--   *Modular and composable*
 Keras models are made by connecting
-    configurable building blocks together, with few restrictions.
--   *Easy to extend*
 Write custom building blocks to express new ideas for
-    research. Create new layers, metrics, loss functions, and develop
-    state-of-the-art models.
-
-The guide [Keras: A Quick Overview](./overview.ipynb) will help you get started.
-
-For a beginner-friendly introduction to machine learning with `tf.keras`,
-see [this set of starter tutorials](https://www.tensorflow.org/tutorials/keras).
-
-To dive more into the API, see the following set of guides that cover what you
-need to know as a TensorFlow Keras power user:
-
--   [Guide to the Keras functional API](./functional.ipynb)
--   [Guide to training and evaluation](./train_and_evaluate.ipynb)
--   [Guide to writing layers and models from scratch with subclassing](./custom_layers_and_models.ipynb)
--   [Guide to the recurrent neural network API](./rnn.ipynb)
--   [Guide to masking and padding](./masking_and_padding.ipynb)
--   [Guide to saving and serializing models](./save_and_serialize.ipynb)
--   [Guide to writing custom callbacks](./custom_callback.ipynb)
-
-Watch [Inside TensorFlow](https://www.youtube.com/playlist?list=PLQY2H8rRoyvzIuB8rZXs7pfyjiSUs8Vza)
-on YouTube for a Keras internals deep dive:
-
-- [Part 1](https://youtu.be/UYRBHFAvLSs)
-- [Part 2](https://youtu.be/uhzGTijaw8A)
diff --git a/site/en/guide/keras/masking_and_padding.ipynb b/site/en/guide/keras/masking_and_padding.ipynb
deleted file mode 100644
index 446589c35cc..00000000000
--- a/site/en/guide/keras/masking_and_padding.ipynb
+++ /dev/null
@@ -1,554 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "masking_and_padding.ipynb",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.5.2"
-    }
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "eIrvnAbGZ1wP"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "cellView": "form",
-        "colab_type": "code",
-        "id": "_A4IPZ-WZ9H7",
-        "colab": {}
-      },
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "WpaDQG8VaYFO"
-      },
-      "source": [
-        "# Masking and padding with Keras\n",
-        "\n",
-        "\n",
-        "  \n",
-        "  \n",
-        "  \n",
-        "  \n",
-        "
\n",
-        "    \n",
-        "    \n",
-        "    View on TensorFlow.org\n",
-        "  \n",
-        "    \n",
-        "    \n",
-        "    Run in Google Colab\n",
-        "  \n",
-        "    \n",
-        "    \n",
-        "    View source on GitHub\n",
-        "  \n",
-        "    \n",
-        "    \n",
-        "    Download notebook\n",
-        "  
"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "QGJH5EKYoSHZ"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "wJEBe8hTlB6W",
-        "colab": {}
-      },
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "import numpy as np\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "from tensorflow.keras import layers"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "5ShanCQ_pSPO"
-      },
-      "source": [
-        "## Padding sequence data\n",
-        "\n",
-        "When processing sequence data, it is very common for individual samples to have different lengths. Consider the following example (text tokenized as words):\n",
-        "\n",
-        "```\n",
-        "[\n",
-        "  [\"The\", \"weather\", \"will\", \"be\", \"nice\", \"tomorrow\"],\n",
-        "  [\"How\", \"are\", \"you\", \"doing\", \"today\"],\n",
-        "  [\"Hello\", \"world\", \"!\"]\n",
-        "]\n",
-        "```\n",
-        "\n",
-        "After vocabulary lookup, the data might be vectorized as integers, e.g.:\n",
-        "\n",
-        "```\n",
-        "[\n",
-        "  [83, 91, 1, 645, 1253, 927],\n",
-        "  [73, 8, 3215, 55, 927],\n",
-        "  [71, 1331, 4231]\n",
-        "]\n",
-        "```\n",
-        "\n",
-        "The data is a 2D list where individual samples have length 6, 5, and 3 respectively. Since the input data for a deep learning model must be a single tensor (of shape e.g. `(batch_size, 6, vocab_size)` in this case), samples that are shorter than the longest item need to be padded with some placeholder value (alternatively, one might also truncate long samples before padding short samples).\n",
-        "\n",
-        "Keras provides an API to easily truncate and pad sequences to a common length: `tf.keras.preprocessing.sequence.pad_sequences`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "xI-lHnyxfa2T",
-        "colab": {}
-      },
-      "source": [
-        "raw_inputs = [\n",
-        "  [83, 91, 1, 645, 1253, 927],\n",
-        "  [73, 8, 3215, 55, 927],\n",
-        "  [711, 632, 71]\n",
-        "]\n",
-        "\n",
-        "# By default, this will pad using 0s; it is configurable via the\n",
-        "# \"value\" parameter.\n",
-        "# Note that you could \"pre\" padding (at the beginning) or\n",
-        "# \"post\" padding (at the end).\n",
-        "# We recommend using \"post\" padding when working with RNN layers\n",
-        "# (in order to be able to use the \n",
-        "# CuDNN implementation of the layers).\n",
-        "padded_inputs = tf.keras.preprocessing.sequence.pad_sequences(raw_inputs,\n",
-        "                                                              padding='post')\n",
-        "\n",
-        "print(padded_inputs)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "HyHf90yAqkMr"
-      },
-      "source": [
-        "## Masking"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "o16pUIBLgc_Q"
-      },
-      "source": [
-        "Now that all samples have a uniform length, the model must be informed that some part of the data is actually padding and should be ignored. That mechanism is masking.\n",
-        "\n",
-        "There are three ways to introduce input masks in Keras models:\n",
-        "\n",
-        "- Add a `keras.layers.Masking` layer.\n",
-        "- Configure a `keras.layers.Embedding` layer with `mask_zero=True`.\n",
-        "- Pass a `mask` argument manually when calling layers that support this argument (e.g. RNN layers)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f6QMNceyf1GD",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Mask-generating layers: `Embedding` and `Masking`\n",
-        "\n",
-        "Under the hood, these layers will create a mask tensor (2D tensor with shape `(batch, sequence_length)`), and attach it to the tensor output returned by the `Masking` or `Embedding` layer."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "rYXQ589PkC0P",
-        "colab": {}
-      },
-      "source": [
-        "embedding = layers.Embedding(input_dim=5000, output_dim=16, mask_zero=True)\n",
-        "masked_output = embedding(padded_inputs)\n",
-        "\n",
-        "print(masked_output._keras_mask)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "-0VVscXQm1D1",
-        "colab": {}
-      },
-      "source": [
-        "masking_layer = layers.Masking()\n",
-        "# Simulate the embedding lookup by expanding the 2D input to 3D,\n",
-        "# with embedding dimension of 10.\n",
-        "unmasked_embedding = tf.cast(\n",
-        "    tf.tile(tf.expand_dims(padded_inputs, axis=-1), [1, 1, 10]),\n",
-        "    tf.float32)\n",
-        "\n",
-        "masked_embedding = masking_layer(unmasked_embedding)\n",
-        "print(masked_embedding._keras_mask)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "RL2vsiCBmVck"
-      },
-      "source": [
-        "As you can see from the printed result, the mask is a 2D boolean tensor with shape `(batch_size, sequence_length)`, where each individual `False` entry indicates that the corresponding timestep should be ignored during processing."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "s4jsu6oTrl2f"
-      },
-      "source": [
-        "## Mask propagation in the Functional API and Sequential API"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "0KgNt7fvm0Jx"
-      },
-      "source": [
-        "When using the Functional API or the Sequential API, a mask generated by an `Embedding` or `Masking` layer will be propagated through the network for any layer that is capable of using them (for example, RNN layers). Keras will automatically fetch the mask corresponding to an input and pass it to any layer that knows how to use it.\n",
-        "\n",
-        "Note that in the `call` method of a subclassed model or layer, masks aren't automatically propagated, so you will need to manually pass a `mask` argument to any layer that needs one. See the section below for details.\n",
-        "\n",
-        "For instance, in the following Sequential model, the `LSTM` layer will automatically receive a mask, which means it will ignore padded values:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "zfkxyf7yVyxJ",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "model = tf.keras.Sequential([\n",
-        "  layers.Embedding(input_dim=5000, output_dim=16, mask_zero=True),\n",
-        "  layers.LSTM(32),\n",
-        "])"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "UqZeTeEhWHLE",
-        "colab_type": "text"
-      },
-      "source": [
-        "This is also the case for the following Functional API model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "SYaVl6WSWJal",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "inputs = tf.keras.Input(shape=(None,), dtype='int32')\n",
-        "x = layers.Embedding(input_dim=5000, output_dim=16, mask_zero=True)(inputs)\n",
-        "outputs = layers.LSTM(32)(x)\n",
-        "\n",
-        "model = tf.keras.Model(inputs, outputs)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "crxoxoRDWg8t",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Passing mask tensors directly to layers"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "UfvcEl20XRYA",
-        "colab_type": "text"
-      },
-      "source": [
-        "Layers that can handle masks (such as the `LSTM` layer) have a `mask` argument in their `__call__` method.\n",
-        "\n",
-        "Meanwhile, layers that produce a mask (e.g. `Embedding`) expose a `compute_mask(input, previous_mask)` method which you can call.\n",
-        "\n",
-        "Thus, you can do something like this:\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "coCV26fqXmya",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "class MyLayer(layers.Layer):\n",
-        "  \n",
-        "  def __init__(self, **kwargs):\n",
-        "    super(MyLayer, self).__init__(**kwargs)\n",
-        "    self.embedding = layers.Embedding(input_dim=5000, output_dim=16, mask_zero=True)\n",
-        "    self.lstm = layers.LSTM(32)\n",
-        "    \n",
-        "  def call(self, inputs):\n",
-        "    x = self.embedding(inputs)\n",
-        "    # Note that you could also prepare a `mask` tensor manually.\n",
-        "    # It only needs to be a boolean tensor\n",
-        "    # with the right shape, i.e. (batch_size, timesteps).\n",
-        "    mask = self.embedding.compute_mask(inputs)\n",
-        "    output = self.lstm(x, mask=mask)  # The layer will ignore the masked values\n",
-        "    return output\n",
-        "\n",
-        "layer = MyLayer()\n",
-        "x = np.random.random((32, 10)) * 100\n",
-        "x = x.astype('int32')\n",
-        "layer(x)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "uSZP15mtWs9Z",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Supporting masking in your custom layers"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "w2gg7O3kVjC4",
-        "colab_type": "text"
-      },
-      "source": [
-        "Sometimes you may need to write layers that generate a mask (like `Embedding`), or layers that need to modify the current mask.\n",
-        "\n",
-        "For instance, any layer that produces a tensor with a different time dimension than its input, such as a `Concatenate` layer that concatenates on the time dimension, will need to modify the current mask so that downstream layers will be able to properly take masked timesteps into account.\n",
-        "\n",
-        "To do this, your layer should implement the `layer.compute_mask()` method, which produces a new mask given the input and the current mask. \n",
-        "\n",
-        "Most layers don't modify the time dimension, so don't need to worry about masking. The default behavior of `compute_mask()` is just pass the current mask through in such cases.\n",
-        "\n",
-        "Here is an example of a `TemporalSplit` layer that needs to modify the current mask."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "gaS_7dXyr-Z0",
-        "colab": {}
-      },
-      "source": [
-        "class TemporalSplit(tf.keras.layers.Layer):\n",
-        "  \"\"\"Split the input tensor into 2 tensors along the time dimension.\"\"\"\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    # Expect the input to be 3D and mask to be 2D, split the input tensor into 2\n",
-        "    # subtensors along the time axis (axis 1).\n",
-        "    return tf.split(inputs, 2, axis=1)\n",
-        "    \n",
-        "  def compute_mask(self, inputs, mask=None):\n",
-        "    # Also split the mask into 2 if it presents.\n",
-        "    if mask is None:\n",
-        "      return None\n",
-        "    return tf.split(mask, 2, axis=1)\n",
-        "\n",
-        "first_half, second_half = TemporalSplit()(masked_embedding)\n",
-        "print(first_half._keras_mask)\n",
-        "print(second_half._keras_mask)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "5t73OUJgjLLW",
-        "colab_type": "text"
-      },
-      "source": [
-        "Here is another example of a `CustomEmbedding` layer that is capable of generating a mask from input values:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "id": "fLSpf1iojSO7",
-        "colab_type": "code",
-        "colab": {}
-      },
-      "source": [
-        "class CustomEmbedding(tf.keras.layers.Layer):\n",
-        "  \n",
-        "  def __init__(self, input_dim, output_dim, mask_zero=False, **kwargs):\n",
-        "    super(CustomEmbedding, self).__init__(**kwargs)\n",
-        "    self.input_dim = input_dim\n",
-        "    self.output_dim = output_dim\n",
-        "    self.mask_zero = mask_zero\n",
-        "    \n",
-        "  def build(self, input_shape):\n",
-        "    self.embeddings = self.add_weight(\n",
-        "      shape=(self.input_dim, self.output_dim),\n",
-        "      initializer='random_normal',\n",
-        "      dtype='float32')\n",
-        "    \n",
-        "  def call(self, inputs):\n",
-        "    return tf.nn.embedding_lookup(self.embeddings, inputs)\n",
-        "  \n",
-        "  def compute_mask(self, inputs, mask=None):\n",
-        "    if not self.mask_zero:\n",
-        "      return None\n",
-        "    return tf.not_equal(inputs, 0)\n",
-        "  \n",
-        "  \n",
-        "layer = CustomEmbedding(10, 32, mask_zero=True)\n",
-        "x = np.random.random((3, 10)) * 9\n",
-        "x = x.astype('int32')\n",
-        "\n",
-        "y = layer(x)\n",
-        "mask = layer.compute_mask(x)\n",
-        "\n",
-        "print(mask)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "fUopC-DelkG2",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Writing layers that need mask information\n",
-        "\n",
-        "Some layers are mask *consumers*: they accept a `mask` argument in `call` and use it to determine whether to skip certain time steps.\n",
-        "\n",
-        "To write such a layer, you can simply add a `mask=None` argument in your `call` signature. The mask associated with the inputs will be passed to your layer whenever it is available.\n",
-        "\n",
-        "```python\n",
-        "class MaskConsumer(tf.keras.layers.Layer):\n",
-        "  \n",
-        "  def call(self, inputs, mask=None):\n",
-        "    ...\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "_50qkZjIn8b2",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Recap\n",
-        "\n",
-        "That is all you need to know about masking in Keras. To recap:\n",
-        "\n",
-        "- \"Masking\" is how layers are able to know when to skip / ignore certain timesteps in sequence inputs.\n",
-        "- Some layers are mask-generators: `Embedding` can generate a mask from input values (if `mask_zero=True`), and so can the `Masking` layer.\n",
-        "- Some layers are mask-consumers: they expose a `mask` argument in their `__call__` method. This is the case for RNN layers.\n",
-        "- In the Functional API and Sequential API, mask information is propagated automatically.\n",
-        "- When writing subclassed models or when using layers in a standalone way, pass the `mask` arguments to layers manually.\n",
-        "- You can easily write layers that modify the current mask, that generate a new mask, or that consume the mask associated with the inputs.\n"
-      ]
-    }
-  ]
-}
\ No newline at end of file
diff --git a/site/en/guide/keras/overview.ipynb b/site/en/guide/keras/overview.ipynb
deleted file mode 100644
index dc17ff98029..00000000000
--- a/site/en/guide/keras/overview.ipynb
+++ /dev/null
@@ -1,1278 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "CCQY7jpBfMur"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
-        "id": "z6X9omPnfO_h"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "F1xIRPtY0E1w"
-      },
-      "source": [
-        "# Keras overview"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "VyOjQZHhZxaA"
-      },
-      "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/keras/overview\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/keras/overview.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/keras/overview.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/keras/overview.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "VUJTep_x5-R8"
-      },
-      "source": [
-        "This guide gives you the basics to get started with Keras. It's a 10-minute read."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "IsK5aF2xZ-40"
-      },
-      "source": [
-        "## Import tf.keras\n",
-        "\n",
-        "`tf.keras` is TensorFlow's implementation of the\n",
-        "[Keras API specification](https://keras.io). This is a high-level\n",
-        "API to build and train models that includes first-class support for\n",
-        "TensorFlow-specific functionality, such as [eager execution](../eager.ipynb),\n",
-        "`tf.data` pipelines, and [Estimators](../estimator.ipynb).\n",
-        "`tf.keras` makes TensorFlow easier to use without sacrificing flexibility and\n",
-        "performance.\n",
-        "\n",
-        "To get started, import `tf.keras` as part of your TensorFlow program setup:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "TgPcBFru0E1z"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "from tensorflow import keras"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "lj03RamP0E13"
-      },
-      "source": [
-        "`tf.keras` can run any Keras-compatible code, but keep in mind:\n",
-        "\n",
-        "* The `tf.keras` version in the latest TensorFlow release might not be the same\n",
-        "  as the latest `keras` version from PyPI. Check `tf.keras.__version__`.\n",
-        "* When [saving a model's weights](./save_and_serialize.ipynb), `tf.keras` defaults to the\n",
-        "  [checkpoint format](../checkpoint.ipynb). Pass `save_format='h5'` to\n",
-        "  use HDF5 (or pass a filename that ends in `.h5`)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "7e1LPcXx0gR6"
-      },
-      "source": [
-        "## Build a simple model\n",
-        "\n",
-        "### Sequential model\n",
-        "\n",
-        "In Keras, you assemble *layers* to build *models*. A model is (usually) a graph\n",
-        "of layers. The most common type of model is a stack of layers: the\n",
-        "`tf.keras.Sequential` model.\n",
-        "\n",
-        "To build a simple, fully-connected network (i.e. multi-layer perceptron):"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "WM-DUVQB0E14"
-      },
-      "outputs": [],
-      "source": [
-        "from tensorflow.keras import layers\n",
-        "\n",
-        "model = tf.keras.Sequential()\n",
-        "# Adds a densely-connected layer with 64 units to the model:\n",
-        "model.add(layers.Dense(64, activation='relu'))\n",
-        "# Add another:\n",
-        "model.add(layers.Dense(64, activation='relu'))\n",
-        "# Add a softmax layer with 10 output units:\n",
-        "model.add(layers.Dense(10, activation='softmax'))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "I2oH0-cxH7YA"
-      },
-      "source": [
-        "You can find a complete, short example of how to use Sequential models [here](https://www.tensorflow.org/tutorials/quickstart/beginner).\n",
-        "\n",
-        "To learn about building more advanced models than Sequential models, see:\n",
-        "- [Guide to the Keras Functional API](./functional.ipynb)\n",
-        "- [Guide to writing layers and models from scratch with subclassing](./custom_layers_and_models.ipynb)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-ztyTipu0E18"
-      },
-      "source": [
-        "### Configure the layers\n",
-        "\n",
-        "There are many `tf.keras.layers` available. Most of them share some common constructor\n",
-        "arguments:\n",
-        "\n",
-        "* `activation`: Set the activation function for the layer. This parameter is\n",
-        "  specified by the name of a built-in function or as a callable object. By\n",
-        "  default, no activation is applied.\n",
-        "* `kernel_initializer` and `bias_initializer`: The initialization schemes\n",
-        "  that create the layer's weights (kernel and bias). This parameter is a name or\n",
-        "  a callable object. This defaults to the `\"Glorot uniform\"` initializer.\n",
-        "* `kernel_regularizer` and `bias_regularizer`: The regularization schemes\n",
-        "  that apply the layer's weights (kernel and bias), such as L1 or L2\n",
-        "  regularization. By default, no regularization is applied.\n",
-        "\n",
-        "The following instantiates `tf.keras.layers.Dense` layers using constructor\n",
-        "arguments:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "MlL7PBtp0E19"
-      },
-      "outputs": [],
-      "source": [
-        "# Create a sigmoid layer:\n",
-        "layers.Dense(64, activation='sigmoid')\n",
-        "# Or:\n",
-        "layers.Dense(64, activation=tf.keras.activations.sigmoid)\n",
-        "\n",
-        "# A linear layer with L1 regularization of factor 0.01 applied to the kernel matrix:\n",
-        "layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l1(0.01))\n",
-        "\n",
-        "# A linear layer with L2 regularization of factor 0.01 applied to the bias vector:\n",
-        "layers.Dense(64, bias_regularizer=tf.keras.regularizers.l2(0.01))\n",
-        "\n",
-        "# A linear layer with a kernel initialized to a random orthogonal matrix:\n",
-        "layers.Dense(64, kernel_initializer='orthogonal')\n",
-        "\n",
-        "# A linear layer with a bias vector initialized to 2.0s:\n",
-        "layers.Dense(64, bias_initializer=tf.keras.initializers.Constant(2.0))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "9NR6reyk0E2A"
-      },
-      "source": [
-        "## Train and evaluate\n",
-        "\n",
-        "### Set up training\n",
-        "\n",
-        "After the model is constructed, configure its learning process by calling the\n",
-        "`compile` method:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "sJ4AOn090E2A"
-      },
-      "outputs": [],
-      "source": [
-        "model = tf.keras.Sequential([\n",
-        "# Adds a densely-connected layer with 64 units to the model:\n",
-        "layers.Dense(64, activation='relu', input_shape=(32,)),\n",
-        "# Add another:\n",
-        "layers.Dense(64, activation='relu'),\n",
-        "# Add a softmax layer with 10 output units:\n",
-        "layers.Dense(10, activation='softmax')])\n",
-        "\n",
-        "model.compile(optimizer=tf.keras.optimizers.Adam(0.01),\n",
-        "              loss='categorical_crossentropy',\n",
-        "              metrics=['accuracy'])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "HG-RAa9F0E2D"
-      },
-      "source": [
-        "`tf.keras.Model.compile` takes three important arguments:\n",
-        "\n",
-        "* `optimizer`: This object specifies the training procedure. Pass it optimizer\n",
-        "  instances from the `tf.keras.optimizers` module, such as\n",
-        "  `tf.keras.optimizers.Adam` or\n",
-        "  `tf.keras.optimizers.SGD`. If you just want to use the default parameters, you can also specify optimizers via strings, such as `'adam'` or `'sgd'`.\n",
-        "* `loss`: The function to minimize during optimization. Common choices include\n",
-        "  mean square error (`mse`), `categorical_crossentropy`, and\n",
-        "  `binary_crossentropy`. Loss functions are specified by name or by\n",
-        "  passing a callable object from the `tf.keras.losses` module.\n",
-        "* `metrics`: Used to monitor training. These are string names or callables from\n",
-        "  the `tf.keras.metrics` module.\n",
-        "* Additionally, to make sure the model trains and evaluates eagerly, you can make sure to pass `run_eagerly=True` as a parameter to compile.\n",
-        "\n",
-        "\n",
-        "The following shows a few examples of configuring a model for training:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "St4Mgdar0E2E"
-      },
-      "outputs": [],
-      "source": [
-        "# Configure a model for mean-squared error regression.\n",
-        "model.compile(optimizer=tf.keras.optimizers.Adam(0.01),\n",
-        "              loss='mse',       # mean squared error\n",
-        "              metrics=['mae'])  # mean absolute error\n",
-        "\n",
-        "# Configure a model for categorical classification.\n",
-        "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.01),\n",
-        "              loss=tf.keras.losses.CategoricalCrossentropy(),\n",
-        "              metrics=[tf.keras.metrics.CategoricalAccuracy()])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "yjI5rbi80E2G"
-      },
-      "source": [
-        "### Train from NumPy data\n",
-        "\n",
-        "For small datasets, use in-memory [NumPy](https://www.numpy.org/)\n",
-        "arrays to train and evaluate a model. The model is \"fit\" to the training data\n",
-        "using the `fit` method:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "3CvP6L-m0E2I"
-      },
-      "outputs": [],
-      "source": [
-        "import numpy as np\n",
-        "\n",
-        "data = np.random.random((1000, 32))\n",
-        "labels = np.random.random((1000, 10))\n",
-        "\n",
-        "model.fit(data, labels, epochs=10, batch_size=32)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "N-pnVaFe0E2N"
-      },
-      "source": [
-        "`tf.keras.Model.fit` takes three important arguments:\n",
-        "\n",
-        "* `epochs`: Training is structured into *epochs*. An epoch is one iteration over\n",
-        "  the entire input data (this is done in smaller batches).\n",
-        "* `batch_size`: When passed NumPy data, the model slices the data into smaller\n",
-        "  batches and iterates over these batches during training. This integer\n",
-        "  specifies the size of each batch. Be aware that the last batch may be smaller\n",
-        "  if the total number of samples is not divisible by the batch size.\n",
-        "* `validation_data`: When prototyping a model, you want to easily monitor its\n",
-        "  performance on some validation data. Passing this argument—a tuple of inputs\n",
-        "  and labels—allows the model to display the loss and metrics in inference mode\n",
-        "  for the passed data, at the end of each epoch.\n",
-        "\n",
-        "Here's an example using `validation_data`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "gFcXzVQa0E2N"
-      },
-      "outputs": [],
-      "source": [
-        "import numpy as np\n",
-        "\n",
-        "data = np.random.random((1000, 32))\n",
-        "labels = np.random.random((1000, 10))\n",
-        "\n",
-        "val_data = np.random.random((100, 32))\n",
-        "val_labels = np.random.random((100, 10))\n",
-        "\n",
-        "model.fit(data, labels, epochs=10, batch_size=32,\n",
-        "          validation_data=(val_data, val_labels))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-6ImyXzz0E2Q"
-      },
-      "source": [
-        "### Train from tf.data datasets\n",
-        "\n",
-        "Use the [Datasets API](../data.ipynb) to scale to large datasets\n",
-        "or multi-device training. Pass a `tf.data.Dataset` instance to the `fit`\n",
-        "method:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "OziqhpIj0E2R"
-      },
-      "outputs": [],
-      "source": [
-        "# Instantiates a toy dataset instance:\n",
-        "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n",
-        "dataset = dataset.batch(32)\n",
-        "\n",
-        "model.fit(dataset, epochs=10)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "I7BcMHkB0E2U"
-      },
-      "source": [
-        "Since the `Dataset` yields batches of data, this snippet does not require a `batch_size`.\n",
-        "\n",
-        "Datasets can also be used for validation:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "YPMb3A0N0E2V"
-      },
-      "outputs": [],
-      "source": [
-        "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n",
-        "dataset = dataset.batch(32)\n",
-        "\n",
-        "val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_labels))\n",
-        "val_dataset = val_dataset.batch(32)\n",
-        "\n",
-        "model.fit(dataset, epochs=10,\n",
-        "          validation_data=val_dataset)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "IgGdlXso0E2X"
-      },
-      "source": [
-        "### Evaluate and predict\n",
-        "\n",
-        "The `tf.keras.Model.evaluate` and `tf.keras.Model.predict` methods can use NumPy\n",
-        "data and a `tf.data.Dataset`.\n",
-        "\n",
-        "Here's how to *evaluate* the inference-mode loss and metrics for the data provided:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mhDbOHEK0E2Y"
-      },
-      "outputs": [],
-      "source": [
-        "# With Numpy arrays\n",
-        "data = np.random.random((1000, 32))\n",
-        "labels = np.random.random((1000, 10))\n",
-        "\n",
-        "model.evaluate(data, labels, batch_size=32)\n",
-        "\n",
-        "# With a Dataset\n",
-        "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n",
-        "dataset = dataset.batch(32)\n",
-        "\n",
-        "model.evaluate(dataset)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "UXUTmDfb0E2b"
-      },
-      "source": [
-        "And here's how to *predict* the output of the last layer in inference for the data provided,\n",
-        "as a NumPy array:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "9e3JsSoQ0E2c"
-      },
-      "outputs": [],
-      "source": [
-        "result = model.predict(data, batch_size=32)\n",
-        "print(result.shape)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "GuTb71gYILLG"
-      },
-      "source": [
-        "For a complete guide on training and evaluation, including how to write custom training loops from scratch, see the [guide to training and evaluation](./train_and_evaluate.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "fzEOW4Cn0E2h"
-      },
-      "source": [
-        "## Build complex models\n",
-        "\n",
-        "### The Functional API\n",
-        "\n",
-        " The `tf.keras.Sequential` model is a simple stack of layers that cannot\n",
-        "represent arbitrary models. Use the\n",
-        "[Keras functional API](./functional.ipynb)\n",
-        "to build complex model topologies such as:\n",
-        "\n",
-        "* Multi-input models,\n",
-        "* Multi-output models,\n",
-        "* Models with shared layers (the same layer called several times),\n",
-        "* Models with non-sequential data flows (e.g. residual connections).\n",
-        "\n",
-        "Building a model with the functional API works like this:\n",
-        "\n",
-        "1. A layer instance is callable and returns a tensor.\n",
-        "2. Input tensors and output tensors are used to define a `tf.keras.Model`\n",
-        "   instance.\n",
-        "3. This model is trained just like the `Sequential` model.\n",
-        "\n",
-        "The following example uses the functional API to build a simple, fully-connected\n",
-        "network:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mROj832r0E2i"
-      },
-      "outputs": [],
-      "source": [
-        "inputs = tf.keras.Input(shape=(32,))  # Returns an input placeholder\n",
-        "\n",
-        "# A layer instance is callable on a tensor, and returns a tensor.\n",
-        "x = layers.Dense(64, activation='relu')(inputs)\n",
-        "x = layers.Dense(64, activation='relu')(x)\n",
-        "predictions = layers.Dense(10, activation='softmax')(x)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "AFmspHeG1_W7"
-      },
-      "source": [
-        "Instantiate the model given inputs and outputs."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "5k5uzlyu16HM"
-      },
-      "outputs": [],
-      "source": [
-        "model = tf.keras.Model(inputs=inputs, outputs=predictions)\n",
-        "\n",
-        "# The compile step specifies the training configuration.\n",
-        "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n",
-        "              loss='categorical_crossentropy',\n",
-        "              metrics=['accuracy'])\n",
-        "\n",
-        "# Trains for 5 epochs\n",
-        "model.fit(data, labels, batch_size=32, epochs=5)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "EcKSLH3i0E2k"
-      },
-      "source": [
-        "### Model subclassing\n",
-        "\n",
-        "Build a fully-customizable model by subclassing `tf.keras.Model` and defining\n",
-        "your own forward pass. Create layers in the `__init__` method and set them as\n",
-        "attributes of the class instance. Define the forward pass in the `call` method.\n",
-        "\n",
-        "Model subclassing is particularly useful when\n",
-        "[eager execution](../eager.ipynb) is enabled, because it allows the forward pass\n",
-        "to be written imperatively.\n",
-        "\n",
-        "Note: if you need your model to *always* run imperatively, you can set `dynamic=True` when calling the `super` constructor.\n",
-        "\n",
-        "\u003e Key Point: Use the right API for the job. While model subclassing offers\n",
-        "flexibility, it comes at a cost of greater complexity and more opportunities for\n",
-        "user errors. If possible, prefer the functional API.\n",
-        "\n",
-        "The following example shows a subclassed `tf.keras.Model` using a custom forward\n",
-        "pass that does not have to be run imperatively:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "KLiHWzcn2Fzk"
-      },
-      "outputs": [],
-      "source": [
-        "class MyModel(tf.keras.Model):\n",
-        "\n",
-        "  def __init__(self, num_classes=10):\n",
-        "    super(MyModel, self).__init__(name='my_model')\n",
-        "    self.num_classes = num_classes\n",
-        "    # Define your layers here.\n",
-        "    self.dense_1 = layers.Dense(32, activation='relu')\n",
-        "    self.dense_2 = layers.Dense(num_classes, activation='sigmoid')\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    # Define your forward pass here,\n",
-        "    # using layers you previously defined (in `__init__`).\n",
-        "    x = self.dense_1(inputs)\n",
-        "    return self.dense_2(x)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ShDD4fv72KGc"
-      },
-      "source": [
-        "Instantiate the new model class:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "42C-qQHm0E2l"
-      },
-      "outputs": [],
-      "source": [
-        "model = MyModel(num_classes=10)\n",
-        "\n",
-        "# The compile step specifies the training configuration.\n",
-        "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n",
-        "              loss='categorical_crossentropy',\n",
-        "              metrics=['accuracy'])\n",
-        "\n",
-        "# Trains for 5 epochs.\n",
-        "model.fit(data, labels, batch_size=32, epochs=5)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "yqRQiKj20E2o"
-      },
-      "source": [
-        "### Custom layers\n",
-        "\n",
-        "Create a custom layer by subclassing `tf.keras.layers.Layer` and implementing\n",
-        "the following methods:\n",
-        "\n",
-        "* `__init__`: Optionally define sublayers to be used by this layer.\n",
-        "* `build`: Create the weights of the layer. Add weights with the `add_weight`\n",
-        "  method.\n",
-        "* `call`: Define the forward pass.\n",
-        "* Optionally, a layer can be serialized by implementing the `get_config` method\n",
-        "  and the `from_config` class method.\n",
-        "\n",
-        "Here's an example of a custom layer that implements a `matmul` of an input with\n",
-        "a kernel matrix:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "l7BFnIHr2WNc"
-      },
-      "outputs": [],
-      "source": [
-        "class MyLayer(layers.Layer):\n",
-        "\n",
-        "  def __init__(self, output_dim, **kwargs):\n",
-        "    self.output_dim = output_dim\n",
-        "    super(MyLayer, self).__init__(**kwargs)\n",
-        "\n",
-        "  def build(self, input_shape):\n",
-        "    # Create a trainable weight variable for this layer.\n",
-        "    self.kernel = self.add_weight(name='kernel',\n",
-        "                                  shape=(input_shape[1], self.output_dim),\n",
-        "                                  initializer='uniform',\n",
-        "                                  trainable=True)\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    return tf.matmul(inputs, self.kernel)\n",
-        "\n",
-        "  def get_config(self):\n",
-        "    base_config = super(MyLayer, self).get_config()\n",
-        "    base_config['output_dim'] = self.output_dim\n",
-        "    return base_config\n",
-        "\n",
-        "  @classmethod\n",
-        "  def from_config(cls, config):\n",
-        "    return cls(**config)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "8wXDRgXV2ZrF"
-      },
-      "source": [
-        "Create a model using your custom layer:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "uqH-cY0h0E2p"
-      },
-      "outputs": [],
-      "source": [
-        "model = tf.keras.Sequential([\n",
-        "    MyLayer(10),\n",
-        "    layers.Activation('softmax')])\n",
-        "\n",
-        "# The compile step specifies the training configuration\n",
-        "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n",
-        "              loss='categorical_crossentropy',\n",
-        "              metrics=['accuracy'])\n",
-        "\n",
-        "# Trains for 5 epochs.\n",
-        "model.fit(data, labels, batch_size=32, epochs=5)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "llipvR5wIl_t"
-      },
-      "source": [
-        "Learn more about creating new layers and models from scratch with subclassing in the [Guide to writing layers and models from scratch](./custom_layers_and_models.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Lu8cc3AJ0E2v"
-      },
-      "source": [
-        "## Callbacks\n",
-        "\n",
-        "A callback is an object passed to a model to customize and extend its behavior\n",
-        "during training. You can write your own custom callback, or use the built-in\n",
-        "`tf.keras.callbacks` that include:\n",
-        "\n",
-        "* `tf.keras.callbacks.ModelCheckpoint`: Save checkpoints of your model at\n",
-        "  regular intervals.\n",
-        "* `tf.keras.callbacks.LearningRateScheduler`: Dynamically change the learning\n",
-        "  rate.\n",
-        "* `tf.keras.callbacks.EarlyStopping`: Interrupt training when validation\n",
-        "  performance has stopped improving.\n",
-        "* `tf.keras.callbacks.TensorBoard`: Monitor the model's behavior using\n",
-        "  [TensorBoard](https://tensorflow.org/tensorboard).\n",
-        "\n",
-        "To use a `tf.keras.callbacks.Callback`, pass it to the model's `fit` method:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "rdYwzSYV0E2v"
-      },
-      "outputs": [],
-      "source": [
-        "callbacks = [\n",
-        "  # Interrupt training if `val_loss` stops improving for over 2 epochs\n",
-        "  tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'),\n",
-        "  # Write TensorBoard logs to `./logs` directory\n",
-        "  tf.keras.callbacks.TensorBoard(log_dir='./logs')\n",
-        "]\n",
-        "model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks,\n",
-        "          validation_data=(val_data, val_labels))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ghhaGfX62abv"
-      },
-      "source": [
-        "\n",
-        "## Save and restore"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "qnl7K-aI0E2z"
-      },
-      "source": [
-        "\n",
-        "### Save just the weights values\n",
-        "\n",
-        "Save and load the weights of a model using `tf.keras.Model.save_weights`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "uQIANjB94fLB"
-      },
-      "outputs": [],
-      "source": [
-        "model = tf.keras.Sequential([\n",
-        "layers.Dense(64, activation='relu', input_shape=(32,)),\n",
-        "layers.Dense(10, activation='softmax')])\n",
-        "\n",
-        "model.compile(optimizer=tf.keras.optimizers.Adam(0.001),\n",
-        "              loss='categorical_crossentropy',\n",
-        "              metrics=['accuracy'])"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "4eoHJ-ny0E21"
-      },
-      "outputs": [],
-      "source": [
-        "# Save weights to a TensorFlow Checkpoint file\n",
-        "model.save_weights('./weights/my_model')\n",
-        "\n",
-        "# Restore the model's state,\n",
-        "# this requires a model with the same architecture.\n",
-        "model.load_weights('./weights/my_model')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "u25Id3xe0E25"
-      },
-      "source": [
-        "By default, this saves the model's weights in the\n",
-        "[TensorFlow checkpoint](../checkpoint.ipynb) file format. Weights can\n",
-        "also be saved to the Keras HDF5 format (the default for the multi-backend\n",
-        "implementation of Keras):"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "JSAYoFEd0E26"
-      },
-      "outputs": [],
-      "source": [
-        "# Save weights to a HDF5 file\n",
-        "model.save_weights('my_model.h5', save_format='h5')\n",
-        "\n",
-        "# Restore the model's state\n",
-        "model.load_weights('my_model.h5')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "mje_yKL10E29"
-      },
-      "source": [
-        "### Save just the model configuration\n",
-        "\n",
-        "A model's configuration can be saved—this serializes the model architecture\n",
-        "without any weights. A saved configuration can recreate and initialize the same\n",
-        "model, even without the code that defined the original model. Keras supports\n",
-        "JSON and YAML serialization formats:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "EbET0oJTzGkq"
-      },
-      "outputs": [],
-      "source": [
-        "# Serialize a model to JSON format\n",
-        "json_string = model.to_json()\n",
-        "json_string"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "pX_badhH3yWV"
-      },
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "import pprint\n",
-        "pprint.pprint(json.loads(json_string))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Q7CIa05r4yTb"
-      },
-      "source": [
-        "Recreate the model (newly initialized) from the JSON:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "J9UFv9k00E2_"
-      },
-      "outputs": [],
-      "source": [
-        "fresh_model = tf.keras.models.model_from_json(json_string)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "t5NHtICh4uHK"
-      },
-      "source": [
-        "Serializing a model to YAML format requires that you install `pyyaml` *before you import TensorFlow*:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "aj24KB3Z36S4"
-      },
-      "outputs": [],
-      "source": [
-        "yaml_string = model.to_yaml()\n",
-        "print(yaml_string)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "O53Kerfl43v7"
-      },
-      "source": [
-        "Recreate the model from the YAML:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "77yRuwg03_MG"
-      },
-      "outputs": [],
-      "source": [
-        "fresh_model = tf.keras.models.model_from_yaml(yaml_string)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "xPvOSSzM0E3B"
-      },
-      "source": [
-        "Caution: Subclassed models are not serializable because their architecture is\n",
-        "defined by the Python code in the body of the `call` method."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "iu8qMwld4-71"
-      },
-      "source": [
-        "\n",
-        "### Save the entire model in one file\n",
-        "\n",
-        "The entire model can be saved to a file that contains the weight values, the\n",
-        "model's configuration, and even the optimizer's configuration. This allows you\n",
-        "to checkpoint a model and resume training later—from the exact same\n",
-        "state—without access to the original code."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "45oNY34Z0E3C"
-      },
-      "outputs": [],
-      "source": [
-        "# Create a simple model\n",
-        "model = tf.keras.Sequential([\n",
-        "  layers.Dense(10, activation='softmax', input_shape=(32,)),\n",
-        "  layers.Dense(10, activation='softmax')\n",
-        "])\n",
-        "model.compile(optimizer='rmsprop',\n",
-        "              loss='categorical_crossentropy',\n",
-        "              metrics=['accuracy'])\n",
-        "model.fit(data, labels, batch_size=32, epochs=5)\n",
-        "\n",
-        "\n",
-        "# Save entire model to a HDF5 file\n",
-        "model.save('my_model.h5')\n",
-        "\n",
-        "# Recreate the exact same model, including weights and optimizer.\n",
-        "model = tf.keras.models.load_model('my_model.h5')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "wGVBURDtI_I6"
-      },
-      "source": [
-        "Learn more about saving and serialization for Keras models in the guide to [save and serialize models](./save_and_serialize.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "PMOWhDOB0E3E"
-      },
-      "source": [
-        "\n",
-        "## Eager execution\n",
-        "\n",
-        "[Eager execution](../eager.ipynb) is an imperative programming\n",
-        "environment that evaluates operations immediately. This is not required for\n",
-        "Keras, but is supported by `tf.keras` and useful for inspecting your program and\n",
-        "debugging.\n",
-        "\n",
-        "All of the `tf.keras` model-building APIs are compatible with eager execution.\n",
-        "And while the `Sequential` and functional APIs can be used, eager execution\n",
-        "especially benefits *model subclassing* and building *custom layers*—the APIs\n",
-        "that require you to write the forward pass as code (instead of the APIs that\n",
-        "create models by assembling existing layers).\n",
-        "\n",
-        "See the [eager execution guide](../eager.ipynb) for\n",
-        "examples of using Keras models with custom training loops and `tf.GradientTape`.\n",
-        "You can also find a complete, short example [here](https://www.tensorflow.org/tutorials/quickstart/advanced)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2wG3NVco5B5V"
-      },
-      "source": [
-        "## Distribution\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "6PJZ6e9J5JHF"
-      },
-      "source": [
-        "### Multiple GPUs\n",
-        "\n",
-        "`tf.keras` models can run on multiple GPUs using\n",
-        "`tf.distribute.Strategy`. This API provides distributed\n",
-        "training on multiple GPUs with almost no changes to existing code.\n",
-        "\n",
-        "Currently, `tf.distribute.MirroredStrategy` is the only supported\n",
-        "distribution strategy. `MirroredStrategy` does in-graph replication with\n",
-        "synchronous training using all-reduce on a single machine. To use\n",
-        "`distribute.Strategy`s , nest the optimizer instantiation and model construction and compilation in a `Strategy`'s `.scope()`, then\n",
-        "train the model.\n",
-        "\n",
-        "The following example distributes a `tf.keras.Model` across multiple GPUs on a\n",
-        "single machine.\n",
-        "\n",
-        "First, define a model inside the distributed strategy scope:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "sbaRr7g-0E3I"
-      },
-      "outputs": [],
-      "source": [
-        "strategy = tf.distribute.MirroredStrategy()\n",
-        "\n",
-        "with strategy.scope():\n",
-        "  model = tf.keras.Sequential()\n",
-        "  model.add(layers.Dense(16, activation='relu', input_shape=(10,)))\n",
-        "  model.add(layers.Dense(1, activation='sigmoid'))\n",
-        "\n",
-        "  optimizer = tf.keras.optimizers.SGD(0.2)\n",
-        "\n",
-        "  model.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
-        "\n",
-        "model.summary()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rO9MiL6X0E3O"
-      },
-      "source": [
-        "Next, train the model on data as usual:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "BEwFq4PM0E3P"
-      },
-      "outputs": [],
-      "source": [
-        "x = np.random.random((1024, 10))\n",
-        "y = np.random.randint(2, size=(1024, 1))\n",
-        "x = tf.cast(x, tf.float32)\n",
-        "dataset = tf.data.Dataset.from_tensor_slices((x, y))\n",
-        "dataset = dataset.shuffle(buffer_size=1024).batch(32)\n",
-        "\n",
-        "model.fit(dataset, epochs=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "N6BXU5F90E3U"
-      },
-      "source": [
-        "For more information, see the [full guide on Distributed Training in TensorFlow](../distributed_training.ipynb)."
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "overview.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true,
-      "version": "0.3.2"
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.5.2"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/keras/rnn.ipynb b/site/en/guide/keras/rnn.ipynb
deleted file mode 100644
index 4cb91019164..00000000000
--- a/site/en/guide/keras/rnn.ipynb
+++ /dev/null
@@ -1,859 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "rnn.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    },
-    "language_info": {
-      "codemirror_mode": {
-        "name": "ipython",
-        "version": 3
-      },
-      "file_extension": ".py",
-      "mimetype": "text/x-python",
-      "name": "python",
-      "nbconvert_exporter": "python",
-      "pygments_lexer": "ipython3",
-      "version": "3.5.2"
-    }
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "eIrvnAbGZ1wP"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "cellView": "form",
-        "colab_type": "code",
-        "id": "_A4IPZ-WZ9H7",
-        "colab": {}
-      },
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "zfiHTzhkmNwd"
-      },
-      "source": [
-        "# Recurrent Neural Networks (RNN) with Keras\n",
-        "\n",
-        "\n",
-        "  \n",
-        "  \n",
-        "  \n",
-        "  \n",
-        "
\n",
-        "    \n",
-        "    \n",
-        "    View on TensorFlow.org\n",
-        "  \n",
-        "    \n",
-        "    \n",
-        "    Run in Google Colab\n",
-        "  \n",
-        "    \n",
-        "    \n",
-        "    View source on GitHub\n",
-        "  \n",
-        "    \n",
-        "    \n",
-        "    Download notebook\n",
-        "  
\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "jfOdaQLhXLDR"
-      },
-      "source": [
-        "Recurrent neural networks (RNN) are a class of neural networks that is powerful for modeling sequence data such as time series or natural language.\n",
-        "\n",
-        "Schematically, a RNN layer uses a `for` loop to iterate over the timesteps of a sequence, while maintaining an internal state that encodes information about the timesteps it has seen so far.\n",
-        "\n",
-        "The Keras RNN API is designed with a focus on:\n",
-        "\n",
-        "- **Ease of use**: the built-in `tf.keras.layers.RNN`, `tf.keras.layers.LSTM`, `tf.keras.layers.GRU` layers enable you to quickly build recurrent models without having to make difficult configuration choices.\n",
-        "  \n",
-        "- **Ease of customization**: You can also define your own RNN cell layer (the inner part of the `for` loop) with custom behavior, and use it with the generic `tf.keras.layers.RNN` layer (the `for` loop itself). This allows you to quickly prototype different research ideas in a flexible way with minimal code.\n",
-        "  "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "QGJH5EKYoSHZ"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "wJEBe8hTlB6W",
-        "colab": {}
-      },
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "import collections\n",
-        "import matplotlib.pyplot as plt\n",
-        "import numpy as np\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "from tensorflow.keras import layers"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "DznzjxWCilt4"
-      },
-      "source": [
-        "## Build a simple model\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "H5tPG7KJirBj"
-      },
-      "source": [
-        "There are three built-in RNN layers in Keras:\n",
-        "\n",
-        "1. `tf.keras.layers.SimpleRNN`, a fully-connected RNN where the output from previous timestep is to be fed to next timestep.\n",
-        "\n",
-        "2. `tf.keras.layers.GRU`, first proposed in [Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation](https://arxiv.org/abs/1406.1078).\n",
-        "\n",
-        "3. `tf.keras.layers.LSTM`, first proposed in [Long Short-Term Memory](https://www.bioinf.jku.at/publications/older/2604.pdf).\n",
-        "\n",
-        "In early 2015, Keras had the first reusable open-source Python implementations of LSTM and GRU.\n",
-        "\n",
-        "Here is a simple example of a `Sequential` model that processes sequences of integers, embeds each integer into a 64-dimensional vector, then processes the sequence of vectors using a `LSTM` layer."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "QHdAFEATnFpn",
-        "colab": {}
-      },
-      "source": [
-        "model = tf.keras.Sequential()\n",
-        "# Add an Embedding layer expecting input vocab of size 1000, and\n",
-        "# output embedding dimension of size 64.\n",
-        "model.add(layers.Embedding(input_dim=1000, output_dim=64))\n",
-        "\n",
-        "# Add a LSTM layer with 128 internal units.\n",
-        "model.add(layers.LSTM(128))\n",
-        "\n",
-        "# Add a Dense layer with 10 units and softmax activation.\n",
-        "model.add(layers.Dense(10, activation='softmax'))\n",
-        "\n",
-        "model.summary()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "sVT4R7O3qDXM"
-      },
-      "source": [
-        "## Outputs and states"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "IOQnPR9eqLwk"
-      },
-      "source": [
-        "By default, the output of a RNN layer contain a single vector per sample. This vector is the RNN cell output corresponding to the last timestep, containing information about the entire input sequence. The shape of this output is `(batch_size, units)` where `units` corresponds to the `units` argument passed to the layer's constructor. \n",
-        "\n",
-        "A RNN layer can also return the entire sequence of outputs for each sample (one vector per timestep per sample), if you set `return_sequences=True`. The shape of this output is `(batch_size, timesteps, units)`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "wNlkR8oXpNEx",
-        "colab": {}
-      },
-      "source": [
-        "model = tf.keras.Sequential()\n",
-        "model.add(layers.Embedding(input_dim=1000, output_dim=64))\n",
-        "\n",
-        "# The output of GRU will be a 3D tensor of shape (batch_size, timesteps, 256)\n",
-        "model.add(layers.GRU(256, return_sequences=True))\n",
-        "\n",
-        "# The output of SimpleRNN will be a 2D tensor of shape (batch_size, 128)\n",
-        "model.add(layers.SimpleRNN(128))\n",
-        "\n",
-        "model.add(layers.Dense(10, activation='softmax'))\n",
-        "\n",
-        "model.summary() "
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "1HagyjYos5rD"
-      },
-      "source": [
-        "In addition, a RNN layer can return its final internal state(s). The returned states can be used to resume the RNN execution later, or [to initialize another RNN](https://arxiv.org/abs/1409.3215). This setting is commonly used in the encoder-decoder sequence-to-sequence model, where the encoder final state is used as the initial state of the decoder.\n",
-        "\n",
-        "To configure a RNN layer to return its internal state, set the `return_state` parameter to `True` when creating the layer. Note that `LSTM` has 2 state  tensors, but `GRU` only has one.\n",
-        "\n",
-        "To configure the initial state of the layer, just call the layer with additional keyword argument `initial_state`.\n",
-        "Note that the shape of the state needs to match the unit size of the layer, like in the example below."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "2_HsGBrDvaea",
-        "colab": {}
-      },
-      "source": [
-        "encoder_vocab = 1000\n",
-        "decoder_vocab = 2000\n",
-        "\n",
-        "encoder_input = layers.Input(shape=(None, ))\n",
-        "encoder_embedded = layers.Embedding(input_dim=encoder_vocab, output_dim=64)(encoder_input)\n",
-        "\n",
-        "# Return states in addition to output\n",
-        "output, state_h, state_c = layers.LSTM(\n",
-        "    64, return_state=True, name='encoder')(encoder_embedded)\n",
-        "encoder_state = [state_h, state_c]\n",
-        "\n",
-        "decoder_input = layers.Input(shape=(None, ))\n",
-        "decoder_embedded = layers.Embedding(input_dim=decoder_vocab, output_dim=64)(decoder_input)\n",
-        "\n",
-        "# Pass the 2 states to a new LSTM layer, as initial state\n",
-        "decoder_output = layers.LSTM(\n",
-        "    64, name='decoder')(decoder_embedded, initial_state=encoder_state)\n",
-        "output = layers.Dense(10, activation='softmax')(decoder_output)\n",
-        "\n",
-        "model = tf.keras.Model([encoder_input, decoder_input], output)\n",
-        "model.summary()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kJDJSXjZ2VaY"
-      },
-      "source": [
-        "## RNN layers and RNN cells"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "hQRxLRSS2gDf"
-      },
-      "source": [
-        "In addition to the built-in RNN layers, the RNN API also provides cell-level APIs. Unlike RNN layers, which processes whole batches of input sequences, the RNN cell only processes a single timestep.\n",
-        "\n",
-        "The cell is the inside of the `for` loop of a RNN layer. Wrapping a cell inside a `tf.keras.layers.RNN` layer gives you a layer capable of processing batches of sequences, e.g. `RNN(LSTMCell(10))`.\n",
-        "\n",
-        "Mathematically, `RNN(LSTMCell(10))` produces the same result as `LSTM(10)`. In fact, the implementation of this layer in TF v1.x was just creating the corresponding RNN cell and wrapping it in a RNN layer.  However using the built-in `GRU` and `LSTM` layers enables the use of CuDNN and you may see better performance.\n",
-        "\n",
-        "There are three built-in RNN cells, each of them corresponding to the matching RNN layer.\n",
-        "\n",
-        "- `tf.keras.layers.SimpleRNNCell` corresponds to the `SimpleRNN` layer.\n",
-        "\n",
-        "- `tf.keras.layers.GRUCell` corresponds to the `GRU` layer.\n",
-        "\n",
-        "- `tf.keras.layers.LSTMCell` corresponds to the `LSTM` layer.\n",
-        "\n",
-        "The cell abstraction, together with the generic `tf.keras.layers.RNN` class, make it very easy to implement custom RNN architectures for your research.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "veiCKSUU-ina",
-        "colab_type": "text"
-      },
-      "source": [
-        "## Cross-batch statefulness"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "EvAaiMJbWR2A"
-      },
-      "source": [
-        "When processing very long sequences (possibly infinite), you may want to use the pattern of **cross-batch statefulness**.\n",
-        "\n",
-        "Normally, the internal state of a RNN layer is reset every time it sees a new batch (i.e. every sample seen by the layer is assume to be independent from the past). The layer will only maintain a state while processing a given sample.\n",
-        "\n",
-        "If you have very long sequences though, it is useful to break them into shorter sequences, and to feed these shorter sequences sequentially into a RNN layer without resetting the layer's state. That way, the layer can retain information about the entirety of the sequence, even though it's only seeing one sub-sequence at a time.\n",
-        "\n",
-        "You can do this by setting `stateful=True` in the constructor.\n",
-        "\n",
-        "If you have a sequence `s = [t0, t1, ... t1546, t1547]`, you would split it into e.g.\n",
-        "\n",
-        "```\n",
-        "s1 = [t0, t1, ... t100]\n",
-        "s2 = [t101, ... t201]\n",
-        "...\n",
-        "s16 = [t1501, ... t1547]\n",
-        "```\n",
-        "\n",
-        "Then you would process it via:\n",
-        "\n",
-        "```python\n",
-        "lstm_layer = layers.LSTM(64, stateful=True)\n",
-        "for s in sub_sequences:\n",
-        "  output = lstm_layer(s)\n",
-        "```\n",
-        "\n",
-        "When you want to clear the state, you  can use `layer.reset_states()`.\n",
-        "\n",
-        "\n",
-        "> Note: In this setup, sample `i` in a given batch is assumed to be the continuation of sample `i` in the previous batch. This means that all batches should contain the same number of samples (batch size). E.g. if a batch contains `[sequence_A_from_t0_to_t100,  sequence_B_from_t0_to_t100]`, the next batch should contain `[sequence_A_from_t101_to_t200,  sequence_B_from_t101_to_t200]`.\n",
-        "\n",
-        "\n",
-        "\n",
-        "\n",
-        "Here is a complete example:\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "E6TsLXJ0X3Xd",
-        "colab": {}
-      },
-      "source": [
-        "paragraph1 = np.random.random((20, 10, 50)).astype(np.float32)\n",
-        "paragraph2 = np.random.random((20, 10, 50)).astype(np.float32)\n",
-        "paragraph3 = np.random.random((20, 10, 50)).astype(np.float32)\n",
-        "\n",
-        "lstm_layer = layers.LSTM(64, stateful=True)\n",
-        "output = lstm_layer(paragraph1)\n",
-        "output = lstm_layer(paragraph2)\n",
-        "output = lstm_layer(paragraph3)\n",
-        "\n",
-        "# reset_states() will reset the cached state to the original initial_state.\n",
-        "# If no initial_state was provided, zero-states will be used by default.\n",
-        "lstm_layer.reset_states()\n"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "7AtPur5BDzb4"
-      },
-      "source": [
-        "##Bidirectional RNNs"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "OsdEIXXREL_N"
-      },
-      "source": [
-        "For sequences other than time series (e.g. text), it is often the case that a RNN model can perform better if it not only processes sequence from start to end, but also backwards. For example, to predict the next word in a sentence, it is often useful to have the context around the word, not only just the words that come before it.\n",
-        "\n",
-        "Keras provides an easy API for you to build such bidirectional RNNs: the `tf.keras.layers.Bidirectional` wrapper."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "MNhYIAXqYl3B",
-        "colab": {}
-      },
-      "source": [
-        "model = tf.keras.Sequential()\n",
-        "\n",
-        "model.add(layers.Bidirectional(layers.LSTM(64, return_sequences=True), \n",
-        "                               input_shape=(5, 10)))\n",
-        "model.add(layers.Bidirectional(layers.LSTM(32)))\n",
-        "model.add(layers.Dense(10, activation='softmax'))\n",
-        "\n",
-        "model.summary()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ThwlodTjZCU0"
-      },
-      "source": [
-        "Under the hood, `Bidirectional` will copy the RNN layer passed in, and flip the `go_backwards` field of the newly copied layer, so that it will process the inputs in reverse order.\n",
-        "\n",
-        "The output of the `Bidirectional` RNN will be, by default, the sum of the forward layer output and the backward layer output. If you need a different merging behavior, e.g. concatenation, change the `merge_mode` parameter in the `Bidirectional` wrapper constructor. For more details about `Bidirectional`, please check [the API docs](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/Bidirectional)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ANGN956w6FRs"
-      },
-      "source": [
-        "## Performance optimization and CuDNN kernels in TensorFlow 2.0"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "76xAs7epaX21"
-      },
-      "source": [
-        "In TensorFlow 2.0, the built-in LSTM and GRU layers have been updated to leverage CuDNN kernels by default when a GPU is available. With this change, the prior `keras.layers.CuDNNLSTM/CuDNNGRU` layers have been deprecated, and you can build your model without worrying about the hardware it will run on.\n",
-        "\n",
-        "Since the CuDNN kernel is built with certain assumptions, this means the layer **will not be able to use the CuDNN kernel if you change the defaults of the built-in LSTM or GRU layers**. E.g.:\n",
-        "\n",
-        "- Changing the `activation` function from `tanh` to something else.\n",
-        "- Changing the `recurrent_activation` function from `sigmoid` to something else.\n",
-        "- Using `recurrent_dropout` > 0.\n",
-        "- Setting `unroll` to True, which forces LSTM/GRU to decompose the inner `tf.while_loop` into an unrolled `for` loop.\n",
-        "- Setting `use_bias` to False.\n",
-        "- Using masking when the input data is not strictly right padded (if the mask corresponds to strictly right padded data, CuDNN can still be used. This is the most common case).\n",
-        "\n",
-        "For the detailed list of constraints, please see the documentation for the [LSTM](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/LSTM) and [GRU](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/GRU) layers."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ybd73JmvqLp4"
-      },
-      "source": [
-        "### Using CuDNN kernels when available\n",
-        "\n",
-        "Let's build a simple LSTM model to demonstrate the performance difference.\n",
-        "\n",
-        "We'll use as input sequences the sequence of rows of MNIST digits (treating each row of pixels as a timestep), and we'll predict the digit's label.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "m9kM9hwRsxMx",
-        "colab": {}
-      },
-      "source": [
-        "batch_size = 64\n",
-        "# Each MNIST image batch is a tensor of shape (batch_size, 28, 28).\n",
-        "# Each input sequence will be of size (28, 28) (height is treated like time).\n",
-        "input_dim = 28\n",
-        "\n",
-        "units = 64\n",
-        "output_size = 10  # labels are from 0 to 9\n",
-        "\n",
-        "# Build the RNN model\n",
-        "def build_model(allow_cudnn_kernel=True):\n",
-        "  # CuDNN is only available at the layer level, and not at the cell level.\n",
-        "  # This means `LSTM(units)` will use the CuDNN kernel,\n",
-        "  # while RNN(LSTMCell(units)) will run on non-CuDNN kernel.\n",
-        "  if allow_cudnn_kernel:\n",
-        "    # The LSTM layer with default options uses CuDNN.\n",
-        "    lstm_layer = tf.keras.layers.LSTM(units, input_shape=(None, input_dim))\n",
-        "  else:\n",
-        "    # Wrapping a LSTMCell in a RNN layer will not use CuDNN.\n",
-        "    lstm_layer = tf.keras.layers.RNN(\n",
-        "        tf.keras.layers.LSTMCell(units),\n",
-        "        input_shape=(None, input_dim))\n",
-        "  model = tf.keras.models.Sequential([\n",
-        "      lstm_layer,\n",
-        "      tf.keras.layers.BatchNormalization(),\n",
-        "      tf.keras.layers.Dense(output_size, activation='softmax')]\n",
-        "  )\n",
-        "  return model\n"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "uuztNezFh0BL"
-      },
-      "source": [
-        "### Load MNIST dataset"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "m_kZTLDobchi",
-        "colab": {}
-      },
-      "source": [
-        "mnist = tf.keras.datasets.mnist\n",
-        "\n",
-        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
-        "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
-        "sample, sample_label = x_train[0], y_train[0]"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "UXF8elCuib8k"
-      },
-      "source": [
-        "### Create a model instance and compile it\n",
-        "We choose `sparse_categorical_crossentropy` as the loss function for the model. The output of the model has shape of `[batch_size, 10]`. The target for the model is a integer vector, each of the integer is in the range of 0 to 9."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "klgv6dfK0KNb",
-        "colab": {}
-      },
-      "source": [
-        "model = build_model(allow_cudnn_kernel=True)\n",
-        "\n",
-        "model.compile(loss='sparse_categorical_crossentropy', \n",
-        "              optimizer='sgd',\n",
-        "              metrics=['accuracy'])\n"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "qzeeo65r25CU",
-        "colab": {}
-      },
-      "source": [
-        "model.fit(x_train, y_train,\n",
-        "          validation_data=(x_test, y_test),\n",
-        "          batch_size=batch_size,\n",
-        "          epochs=5)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kvCWAssZjsdW"
-      },
-      "source": [
-        "### Build a new model without CuDNN kernel"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "H2JfHDOhOFtx",
-        "colab": {}
-      },
-      "source": [
-        "slow_model = build_model(allow_cudnn_kernel=False)\n",
-        "slow_model.set_weights(model.get_weights())\n",
-        "slow_model.compile(loss='sparse_categorical_crossentropy', \n",
-        "                   optimizer='sgd', \n",
-        "                   metrics=['accuracy'])\n",
-        "slow_model.fit(x_train, y_train, \n",
-        "               validation_data=(x_test, y_test), \n",
-        "               batch_size=batch_size,\n",
-        "               epochs=1)  # We only train for one epoch because it's slower."
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Zx8QLf81dTVr"
-      },
-      "source": [
-        "As you can see, the model built with CuDNN is much faster to train compared to the model that use the regular TensorFlow kernel.\n",
-        "\n",
-        "The same CuDNN-enabled model can also be use to run inference in a CPU-only environment. The `tf.device` annotation below is just forcing the device placement. The model will run on CPU by default if no GPU is available.\n",
-        "\n",
-        "You simply don't have to worry about the hardware you're running on anymore. Isn't that pretty cool?"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "z_z1eRh1fMBL",
-        "colab": {}
-      },
-      "source": [
-        "with tf.device('CPU:0'):\n",
-        "  cpu_model = build_model(allow_cudnn_kernel=True)\n",
-        "  cpu_model.set_weights(model.get_weights())\n",
-        "  result = tf.argmax(cpu_model.predict_on_batch(tf.expand_dims(sample, 0)), axis=1)\n",
-        "  print('Predicted result is: %s, target result is: %s' % (result.numpy(), sample_label))\n",
-        "  plt.imshow(sample, cmap=plt.get_cmap('gray'))"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2mCetBoTiqcB"
-      },
-      "source": [
-        "## RNNs with list/dict inputs, or nested inputs\n",
-        "\n",
-        "Nested structures allow implementers to include more information within a single timestep. For example, a video frame could have audio and video input at the same time. The data shape in this case could be:\n",
-        "\n",
-        "`[batch, timestep, {\"video\": [height, width, channel], \"audio\": [frequency]}]`\n",
-        "\n",
-        "In another example, handwriting data could have both coordinates x and y for the current position of the pen, as well as pressure information. So the data representation could be:\n",
-        "\n",
-        "`[batch, timestep, {\"location\": [x, y], \"pressure\": [force]}]`\n",
-        "\n",
-        "The following code provides an example of how to build a custom RNN cell that accepts such structured inputs.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "A1IkIxWykSZQ"
-      },
-      "source": [
-        "### Define a custom cell that support nested input/output"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "6yOT8nSqzp4A",
-        "colab": {}
-      },
-      "source": [
-        "NestedInput = collections.namedtuple('NestedInput', ['feature1', 'feature2'])\n",
-        "NestedState = collections.namedtuple('NestedState', ['state1', 'state2'])\n",
-        "\n",
-        "class NestedCell(tf.keras.layers.Layer):\n",
-        "\n",
-        "  def __init__(self, unit_1, unit_2, unit_3, **kwargs):\n",
-        "    self.unit_1 = unit_1\n",
-        "    self.unit_2 = unit_2\n",
-        "    self.unit_3 = unit_3\n",
-        "    self.state_size = NestedState(state1=unit_1, \n",
-        "                                  state2=tf.TensorShape([unit_2, unit_3]))\n",
-        "    self.output_size = (unit_1, tf.TensorShape([unit_2, unit_3]))\n",
-        "    super(NestedCell, self).__init__(**kwargs)\n",
-        "\n",
-        "  def build(self, input_shapes):\n",
-        "    # expect input_shape to contain 2 items, [(batch, i1), (batch, i2, i3)]\n",
-        "    input_1 = input_shapes.feature1[1]\n",
-        "    input_2, input_3 = input_shapes.feature2[1:]\n",
-        "\n",
-        "    self.kernel_1 = self.add_weight(\n",
-        "        shape=(input_1, self.unit_1), initializer='uniform', name='kernel_1')\n",
-        "    self.kernel_2_3 = self.add_weight(\n",
-        "        shape=(input_2, input_3, self.unit_2, self.unit_3),\n",
-        "        initializer='uniform',\n",
-        "        name='kernel_2_3')\n",
-        "\n",
-        "  def call(self, inputs, states):\n",
-        "    # inputs should be in [(batch, input_1), (batch, input_2, input_3)]\n",
-        "    # state should be in shape [(batch, unit_1), (batch, unit_2, unit_3)]\n",
-        "    input_1, input_2 = tf.nest.flatten(inputs)\n",
-        "    s1, s2 = states\n",
-        "\n",
-        "    output_1 = tf.matmul(input_1, self.kernel_1)\n",
-        "    output_2_3 = tf.einsum('bij,ijkl->bkl', input_2, self.kernel_2_3)\n",
-        "    state_1 = s1 + output_1\n",
-        "    state_2_3 = s2 + output_2_3\n",
-        "\n",
-        "    output = [output_1, output_2_3]\n",
-        "    new_states = NestedState(state1=state_1, state2=state_2_3)\n",
-        "\n",
-        "    return output, new_states"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "BJHOrrybk6Zy"
-      },
-      "source": [
-        "### Build a RNN model with nested input/output\n",
-        "\n",
-        "Let's build a Keras model that uses a `tf.keras.layers.RNN` layer and the custom cell we just defined."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "itrDe0Y2qPjP",
-        "colab": {}
-      },
-      "source": [
-        "unit_1 = 10\n",
-        "unit_2 = 20\n",
-        "unit_3 = 30\n",
-        "\n",
-        "input_1 = 32\n",
-        "input_2 = 64\n",
-        "input_3 = 32\n",
-        "batch_size = 64\n",
-        "num_batch = 100\n",
-        "timestep = 50\n",
-        "\n",
-        "cell = NestedCell(unit_1, unit_2, unit_3)\n",
-        "rnn = tf.keras.layers.RNN(cell)\n",
-        "\n",
-        "inp_1 = tf.keras.Input((None, input_1))\n",
-        "inp_2 = tf.keras.Input((None, input_2, input_3))\n",
-        "\n",
-        "outputs = rnn(NestedInput(feature1=inp_1, feature2=inp_2))\n",
-        "\n",
-        "model = tf.keras.models.Model([inp_1, inp_2], outputs)\n",
-        "\n",
-        "model.compile(optimizer='adam', loss='mse', metrics=['accuracy'])"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2MaihTM2mDcp"
-      },
-      "source": [
-        "### Train the model with randomly generated data\n",
-        "\n",
-        "Since there isn't a good candidate dataset for this model, we use random Numpy data for demonstration."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "lN-imRqElz2S",
-        "colab": {}
-      },
-      "source": [
-        "input_1_data = np.random.random((batch_size * num_batch, timestep, input_1))\n",
-        "input_2_data = np.random.random((batch_size * num_batch, timestep, input_2, input_3))\n",
-        "target_1_data = np.random.random((batch_size * num_batch, unit_1))\n",
-        "target_2_data = np.random.random((batch_size * num_batch, unit_2, unit_3))\n",
-        "input_data = [input_1_data, input_2_data]\n",
-        "target_data = [target_1_data, target_2_data]\n",
-        "\n",
-        "model.fit(input_data, target_data, batch_size=batch_size)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "oDdrwgBWnjYp"
-      },
-      "source": [
-        "With the Keras `tf.keras.layers.RNN` layer, You are only expected to define the math logic for individual step within the sequence, and the `tf.keras.layers.RNN` layer will handle the sequence iteration for you. It's an incredibly powerful way to quickly prototype new kinds of RNNs (e.g. a LSTM variant).\n",
-        "\n",
-        "For more details, please visit the [API docs](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/RNN)."
-      ]
-    }
-  ]
-}
diff --git a/site/en/guide/keras/save_and_serialize.ipynb b/site/en/guide/keras/save_and_serialize.ipynb
deleted file mode 100644
index 49a9e5ea910..00000000000
--- a/site/en/guide/keras/save_and_serialize.ipynb
+++ /dev/null
@@ -1,750 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "mHDxn9VHjxKn"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
-        "id": "3x19oys5j89H"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "hFDUpbtvv_3u"
-      },
-      "source": [
-        "# Save and serialize models with Keras"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "V94_3U2k9rWV"
-      },
-      "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/keras/save_and_serialize\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/keras/save_and_serialize.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/keras/save_and_serialize.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/keras/save_and_serialize.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ZwiVWAQc9tk7"
-      },
-      "source": [
-        "The first part of this guide covers saving and serialization for Sequential models and models built using the Functional API and for Sequential models. The saving and serialization APIs are the exact same for both of these types of models.\n",
-        "\n",
-        "Saving for custom subclasses of `Model` is covered in the section \"Saving Subclassed Models\". The APIs in this case are slightly different than for Sequential or Functional models."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "uqSgPMHguAAs"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "bx5w4U5muDAo"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.keras.backend.clear_session()  # For easy reset of notebook state."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "wwCxkE6RyyPy"
-      },
-      "source": [
-        "## Part I: Saving Sequential models or Functional models\n",
-        "\n",
-        "Let's consider the following model:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ILmySACTvSA9"
-      },
-      "outputs": [],
-      "source": [
-        "from tensorflow import keras\n",
-        "from tensorflow.keras import layers\n",
-        "\n",
-        "inputs = keras.Input(shape=(784,), name='digits')\n",
-        "x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "x = layers.Dense(64, activation='relu', name='dense_2')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
-        "\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs, name='3_layer_mlp')\n",
-        "model.summary()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "xPRqbd0yw8hY"
-      },
-      "source": [
-        "Optionally, let's train this model, just so it has weight values to save, as well as an optimizer state.\n",
-        "Of course, you can save models you've never trained, too, but obviously that's less interesting."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "gCygTeGQw74g"
-      },
-      "outputs": [],
-      "source": [
-        "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
-        "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n",
-        "x_test = x_test.reshape(10000, 784).astype('float32') / 255\n",
-        "\n",
-        "model.compile(loss='sparse_categorical_crossentropy',\n",
-        "              optimizer=keras.optimizers.RMSprop())\n",
-        "history = model.fit(x_train, y_train,\n",
-        "                    batch_size=64,\n",
-        "                    epochs=1)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "htnmbhz-iOwh"
-      },
-      "outputs": [],
-      "source": [
-        "# Save predictions for future checks\n",
-        "predictions = model.predict(x_test)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "opP1KROHwWwd"
-      },
-      "source": [
-        "\n",
-        "### Whole-model saving\n",
-        "\n",
-        "You can save a model built with the Functional API into a single file. You can later recreate the same model from this file, even if you no longer have access to the code that created the model.\n",
-        "\n",
-        "This file includes:\n",
-        "\n",
-        "- The model's architecture\n",
-        "- The model's weight values (which were learned during training)\n",
-        "- The model's training config (what you passed to `compile`), if any\n",
-        "- The optimizer and its state, if any (this enables you to restart training where you left off)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "HqHvq6Igw3wx"
-      },
-      "outputs": [],
-      "source": [
-        "# Save the model\n",
-        "model.save('path_to_my_model.h5')\n",
-        "\n",
-        "# Recreate the exact same model purely from the file\n",
-        "new_model = keras.models.load_model('path_to_my_model.h5')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mmIcF6UOItJE"
-      },
-      "outputs": [],
-      "source": [
-        "import numpy as np\n",
-        "\n",
-        "# Check that the state is preserved\n",
-        "new_predictions = new_model.predict(x_test)\n",
-        "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n",
-        "\n",
-        "# Note that the optimizer state is preserved as well:\n",
-        "# you can resume training where you left off."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-WEPW3n8ICyz"
-      },
-      "source": [
-        "### Export to SavedModel\n",
-        "\n",
-        "You can also export a whole model to the TensorFlow `SavedModel` format. `SavedModel` is a standalone serialization format for TensorFlow objects, supported by TensorFlow serving as well as TensorFlow implementations other than Python."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "cKASRTKCU5nv"
-      },
-      "outputs": [],
-      "source": [
-        "# Export the model to a SavedModel\n",
-        "model.save('path_to_saved_model', save_format='tf')\n",
-        "\n",
-        "# Recreate the exact same model\n",
-        "new_model = keras.models.load_model('path_to_saved_model')\n",
-        "\n",
-        "# Check that the state is preserved\n",
-        "new_predictions = new_model.predict(x_test)\n",
-        "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n",
-        "\n",
-        "# Note that the optimizer state is preserved as well:\n",
-        "# you can resume training where you left off."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "4AWgwkKWIhfj"
-      },
-      "source": [
-        "The `SavedModel` files that were created contain:\n",
-        "\n",
-        "- A TensorFlow checkpoint containing the model weights.\n",
-        "- A `SavedModel` proto containing the underlying TensorFlow graph."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "GkY8XP_XxgMI"
-      },
-      "source": [
-        "### Architecture-only saving\n",
-        "\n",
-        "Sometimes, you are only interested in the architecture of the model, and you don't need to save the weight values or the optimizer. In this case, you can retrieve the \"config\" of the model via the `get_config()` method. The config is a Python dict that enables you to recreate the same model -- initialized from scratch, without any of the information learned previously during training."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "yQGGvo2Fw4o-"
-      },
-      "outputs": [],
-      "source": [
-        "config = model.get_config()\n",
-        "reinitialized_model = keras.Model.from_config(config)\n",
-        "\n",
-        "# Note that the model state is not preserved! We only saved the architecture.\n",
-        "new_predictions = reinitialized_model.predict(x_test)\n",
-        "assert abs(np.sum(predictions - new_predictions)) \u003e 0."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "WsNBBvDgxsTS"
-      },
-      "source": [
-        "You can alternatively use `to_json()` from `from_json()`, which uses a JSON string to store the config instead of a Python dict. This is useful to save the config to disk."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "5a0z7_6XxqWV"
-      },
-      "outputs": [],
-      "source": [
-        "json_config = model.to_json()\n",
-        "reinitialized_model = keras.models.model_from_json(json_config)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "SGC7R6IIxy0o"
-      },
-      "source": [
-        "### Weights-only saving\n",
-        "\n",
-        "Sometimes, you are only interested in the state of the model -- its weights values -- and not in the architecture. In this case, you can retrieve the weights values as a list of Numpy arrays via `get_weights()`, and set the state of the model via `set_weights`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "B8tHwEvkxw5E"
-      },
-      "outputs": [],
-      "source": [
-        "weights = model.get_weights()  # Retrieves the state of the model.\n",
-        "model.set_weights(weights)  # Sets the state of the model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Ydwtw-u2x7xC"
-      },
-      "source": [
-        "You can combine `get_config()`/`from_config()` and `get_weights()`/`set_weights()` to recreate your model in the same state. However, unlike `model.save()`, this will not include the training config and the optimizer. You would have to call `compile()` again before using the model for training."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "LWVtuxtrx5lb"
-      },
-      "outputs": [],
-      "source": [
-        "config = model.get_config()\n",
-        "weights = model.get_weights()\n",
-        "\n",
-        "new_model = keras.Model.from_config(config)\n",
-        "new_model.set_weights(weights)\n",
-        "\n",
-        "# Check that the state is preserved\n",
-        "new_predictions = new_model.predict(x_test)\n",
-        "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n",
-        "\n",
-        "# Note that the optimizer was not preserved,\n",
-        "# so the model should be compiled anew before training\n",
-        "# (and the optimizer will start from a blank state)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "prk0GzwCyIYy"
-      },
-      "source": [
-        "The save-to-disk alternative to `get_weights()` and `set_weights(weights)`\n",
-        "is `save_weights(fpath)` and `load_weights(fpath)`.\n",
-        "\n",
-        "Here's an example that saves to disk:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "2irLnOUbyHlI"
-      },
-      "outputs": [],
-      "source": [
-        "# Save JSON config to disk\n",
-        "json_config = model.to_json()\n",
-        "with open('model_config.json', 'w') as json_file:\n",
-        "    json_file.write(json_config)\n",
-        "# Save weights to disk\n",
-        "model.save_weights('path_to_my_weights.h5')\n",
-        "\n",
-        "# Reload the model from the 2 files we saved\n",
-        "with open('model_config.json') as json_file:\n",
-        "    json_config = json_file.read()\n",
-        "new_model = keras.models.model_from_json(json_config)\n",
-        "new_model.load_weights('path_to_my_weights.h5')\n",
-        "\n",
-        "# Check that the state is preserved\n",
-        "new_predictions = new_model.predict(x_test)\n",
-        "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n",
-        "\n",
-        "# Note that the optimizer was not preserved."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "KBxcFAPHyYi5"
-      },
-      "source": [
-        "But remember that the simplest, recommended way is just this:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "DE4b3ndNyQh3"
-      },
-      "outputs": [],
-      "source": [
-        "model.save('path_to_my_model.h5')\n",
-        "del model\n",
-        "model = keras.models.load_model('path_to_my_model.h5')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "yKikmbdC3O_i"
-      },
-      "source": [
-        "### Weights-only saving using TensorFlow checkpoints\n",
-        "\n",
-        "Note that `save_weights` can create files either in the Keras HDF5 format,\n",
-        "or in the [TensorFlow Checkpoint format](https://www.tensorflow.org/api_docs/python/tf/train/Checkpoint). The format is inferred from the file extension you provide: if it is \".h5\" or \".keras\", the framework uses the Keras HDF5 format. Anything else defaults to Checkpoint."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0pYKb6LV3h2l"
-      },
-      "outputs": [],
-      "source": [
-        "model.save_weights('path_to_my_tf_checkpoint')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ZFwKv6JC3kyu"
-      },
-      "source": [
-        "For total explicitness, the format can be explicitly passed via the `save_format` argument, which can take the value \"tf\" or \"h5\":"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "oN9vOaWU34lA"
-      },
-      "outputs": [],
-      "source": [
-        "model.save_weights('path_to_my_tf_checkpoint', save_format='tf')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "xXgtNRCSyuIW"
-      },
-      "source": [
-        "## Saving Subclassed Models"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "mJqOn0snzCRy"
-      },
-      "source": [
-        "Sequential models and Functional models are datastructures that represent a DAG of layers. As such,\n",
-        "they can be safely serialized and deserialized.\n",
-        "\n",
-        "A subclassed model differs in that it's not a datastructure, it's a piece of code. The architecture of the model\n",
-        "is defined via the body of the `call` method. This means that the architecture of the model cannot be safely serialized. To load a model, you'll need to have access to the code that created it (the code of the model subclass). Alternatively, you could be serializing this code as bytecode (e.g. via pickling), but that's unsafe and generally not portable.\n",
-        "\n",
-        "For more information about these differences, see the article [\"What are Symbolic and Imperative APIs in TensorFlow 2.0?\"](https://medium.com/tensorflow/what-are-symbolic-and-imperative-apis-in-tensorflow-2-0-dfccecb01021)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Pkwyu5dVz12P"
-      },
-      "source": [
-        "Let's consider the following subclassed model, which follows the same structure as the model from the first section:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "4Onp-8rGyeQG"
-      },
-      "outputs": [],
-      "source": [
-        "class ThreeLayerMLP(keras.Model):\n",
-        "\n",
-        "  def __init__(self, name=None):\n",
-        "    super(ThreeLayerMLP, self).__init__(name=name)\n",
-        "    self.dense_1 = layers.Dense(64, activation='relu', name='dense_1')\n",
-        "    self.dense_2 = layers.Dense(64, activation='relu', name='dense_2')\n",
-        "    self.pred_layer = layers.Dense(10, activation='softmax', name='predictions')\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    x = self.dense_1(inputs)\n",
-        "    x = self.dense_2(x)\n",
-        "    return self.pred_layer(x)\n",
-        "\n",
-        "def get_model():\n",
-        "  return ThreeLayerMLP(name='3_layer_mlp')\n",
-        "\n",
-        "model = get_model()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "wwT_YoKA0yQW"
-      },
-      "source": [
-        "First of all, *a subclassed model that has never been used cannot be saved*.\n",
-        "\n",
-        "That's because a subclassed model needs to be called on some data in order to create its weights.\n",
-        "\n",
-        "Until the model has been called, it does not know the shape and dtype of the input data it should be\n",
-        "expecting, and thus cannot create its weight variables. You may remember that in the Functional model from the first section, the shape and dtype of the inputs was specified in advance (via `keras.Input(...)`) -- that's why Functional models have a state as soon as they're instantiated.\n",
-        "\n",
-        "Let's train the model, so as to give it a state:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "xqP4kIFN0fTZ"
-      },
-      "outputs": [],
-      "source": [
-        "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
-        "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n",
-        "x_test = x_test.reshape(10000, 784).astype('float32') / 255\n",
-        "\n",
-        "model.compile(loss='sparse_categorical_crossentropy',\n",
-        "              optimizer=keras.optimizers.RMSprop())\n",
-        "history = model.fit(x_train, y_train,\n",
-        "                    batch_size=64,\n",
-        "                    epochs=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rvGCpyX72HOC"
-      },
-      "source": [
-        "The recommended way to save a subclassed model is to use `save_weights` to create a TensorFlow SavedModel checkpoint, which will contain the value of all variables associated with the model:\n",
-        "- The layers' weights\n",
-        "- The optimizer's state\n",
-        "- Any variables associated with stateful model metrics (if any)\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "gMg87Tz01cxQ"
-      },
-      "outputs": [],
-      "source": [
-        "model.save_weights('path_to_my_weights', save_format='tf')"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "KOKNBojtsl0F"
-      },
-      "outputs": [],
-      "source": [
-        "# Save predictions for future checks\n",
-        "predictions = model.predict(x_test)\n",
-        "# Also save the loss on the first batch\n",
-        "# to later assert that the optimizer state was preserved\n",
-        "first_batch_loss = model.train_on_batch(x_train[:64], y_train[:64])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "h2PM_PL1SzPo"
-      },
-      "source": [
-        "To restore your model, you will need access to the code that created the model object.\n",
-        "\n",
-        "Note that in order to restore the optimizer state and the state of any stateful  metric, you should\n",
-        "compile the model (with the exact same arguments as before) and call it on some data before calling `load_weights`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "OOSGiSkHTERy"
-      },
-      "outputs": [],
-      "source": [
-        "# Recreate the model\n",
-        "new_model = get_model()\n",
-        "new_model.compile(loss='sparse_categorical_crossentropy',\n",
-        "                  optimizer=keras.optimizers.RMSprop())\n",
-        "\n",
-        "# This initializes the variables used by the optimizers,\n",
-        "# as well as any stateful metric variables\n",
-        "new_model.train_on_batch(x_train[:1], y_train[:1])\n",
-        "\n",
-        "# Load the state of the old model\n",
-        "new_model.load_weights('path_to_my_weights')\n",
-        "\n",
-        "# Check that the model state has been preserved\n",
-        "new_predictions = new_model.predict(x_test)\n",
-        "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n",
-        "\n",
-        "# The optimizer state is preserved as well,\n",
-        "# so you can resume training where you left off\n",
-        "new_first_batch_loss = new_model.train_on_batch(x_train[:64], y_train[:64])\n",
-        "assert first_batch_loss == new_first_batch_loss"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2_XaE5Oqv7Rh"
-      },
-      "source": [
-        "You've reached the end of this guide! This covers everything you need to know about saving and serializing models with tf.keras in TensorFlow 2.0."
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "save_and_serialize.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/keras/train_and_evaluate.ipynb b/site/en/guide/keras/train_and_evaluate.ipynb
deleted file mode 100644
index 37754b274d1..00000000000
--- a/site/en/guide/keras/train_and_evaluate.ipynb
+++ /dev/null
@@ -1,1881 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "A8MVXQUFkX3n"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
-        "id": "IcfrhafzkZbH"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "zOSK2IBG6-9Z"
-      },
-      "source": [
-        "# Train and evaluate with Keras\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "7s-G0Fajsvjn"
-      },
-      "source": [
-        "\n",
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/keras/train_and_evaluate\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/keras/train_and_evaluate.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/keras/train_and_evaluate.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/keras/train_and_evaluate.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "tc92ghg-sw2w"
-      },
-      "source": [
-        "\n",
-        "This guide covers training, evaluation, and prediction (inference)  models in TensorFlow 2.0 in two broad situations:\n",
-        "\n",
-        "- When using built-in APIs for training \u0026 validation (such as `model.fit()`, `model.evaluate()`, `model.predict()`). This is covered in the section **\"Using build-in training \u0026 evaluation loops\"**.\n",
-        "- When writing custom loops from scratch using eager execution and the `GradientTape` object. This is covered in the section **\"Writing your own training \u0026 evaluation loops from scratch\"**.\n",
-        "\n",
-        "In general, whether you are using built-in loops or writing your own, model training \u0026 evaluation works strictly in the same way across every kind of Keras model -- Sequential models, models built with the Functional API, and models written from scratch via model subclassing.\n",
-        "\n",
-        "This guide doesn't cover distributed training."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "RNG8OH3yuWrb"
-      },
-      "source": [
-        "## Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Uh_b0kqEuYah"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.keras.backend.clear_session()  # For easy reset of notebook state."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "052DbsQ175lP"
-      },
-      "source": [
-        "## Part I: Using build-in training \u0026 evaluation loops\n",
-        "\n",
-        "When passing data to the built-in training loops of a model, you should either use **Numpy arrays** (if your data is small and fits in memory) or **tf.data Dataset** objects. In the next few paragraphs, we'll use the MNIST dataset as Numpy arrays, in order to demonstrate how to use optimizers, losses, and metrics."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rXDWPajQ8V8V"
-      },
-      "source": [
-        "### API overview: a first end-to-end example\n",
-        "\n",
-        "Let's consider the following model (here, we build in with the Functional API, but it could be a Sequential model or a subclassed model as well):\n",
-        "\n",
-        "\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "_eJxUppsNjMj"
-      },
-      "outputs": [],
-      "source": [
-        "from tensorflow import keras\n",
-        "from tensorflow.keras import layers\n",
-        "\n",
-        "inputs = keras.Input(shape=(784,), name='digits')\n",
-        "x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "x = layers.Dense(64, activation='relu', name='dense_2')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
-        "\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "C02iiuYZNklq"
-      },
-      "source": [
-        "Here's what the typical end-to-end workflow looks like, consisting of training, validation on a holdout set generated from the original training data, and finally evaluation on the test data:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "YTzpaClpNvyJ"
-      },
-      "outputs": [],
-      "source": [
-        "# Load a toy dataset for the sake of this example\n",
-        "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
-        "\n",
-        "# Preprocess the data (these are Numpy arrays)\n",
-        "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n",
-        "x_test = x_test.reshape(10000, 784).astype('float32') / 255\n",
-        "\n",
-        "y_train = y_train.astype('float32')\n",
-        "y_test = y_test.astype('float32')\n",
-        "\n",
-        "# Reserve 10,000 samples for validation\n",
-        "x_val = x_train[-10000:]\n",
-        "y_val = y_train[-10000:]\n",
-        "x_train = x_train[:-10000]\n",
-        "y_train = y_train[:-10000]\n",
-        "\n",
-        "# Specify the training configuration (optimizer, loss, metrics)\n",
-        "model.compile(optimizer=keras.optimizers.RMSprop(),  # Optimizer\n",
-        "              # Loss function to minimize\n",
-        "              loss=keras.losses.SparseCategoricalCrossentropy(),\n",
-        "              # List of metrics to monitor\n",
-        "              metrics=[keras.metrics.SparseCategoricalAccuracy()])\n",
-        "\n",
-        "# Train the model by slicing the data into \"batches\"\n",
-        "# of size \"batch_size\", and repeatedly iterating over\n",
-        "# the entire dataset for a given number of \"epochs\"\n",
-        "print('# Fit model on training data')\n",
-        "history = model.fit(x_train, y_train,\n",
-        "                    batch_size=64,\n",
-        "                    epochs=3,\n",
-        "                    # We pass some validation for\n",
-        "                    # monitoring validation loss and metrics\n",
-        "                    # at the end of each epoch\n",
-        "                    validation_data=(x_val, y_val))\n",
-        "\n",
-        "# The returned \"history\" object holds a record\n",
-        "# of the loss values and metric values during training\n",
-        "print('\\nhistory dict:', history.history)\n",
-        "\n",
-        "# Evaluate the model on the test data using `evaluate`\n",
-        "print('\\n# Evaluate on test data')\n",
-        "results = model.evaluate(x_test, y_test, batch_size=128)\n",
-        "print('test loss, test acc:', results)\n",
-        "\n",
-        "# Generate predictions (probabilities -- the output of the last layer)\n",
-        "# on new data using `predict`\n",
-        "print('\\n# Generate predictions for 3 samples')\n",
-        "predictions = model.predict(x_test[:3])\n",
-        "print('predictions shape:', predictions.shape)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "R43f-GWfDkQ5"
-      },
-      "source": [
-        "### Specifying a loss, metrics, and an optimizer\n",
-        "\n",
-        "To train a model with `fit`, you need to specify a loss function, an optimizer, and optionally, some metrics to monitor.\n",
-        "\n",
-        "You pass these to the model as arguments to the `compile()` method:\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "CiB85T_hM_vQ"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=1e-3),\n",
-        "              loss=keras.losses.SparseCategoricalCrossentropy(),\n",
-        "              metrics=[keras.metrics.SparseCategoricalAccuracy()])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "lq3wsghNM_7C"
-      },
-      "source": [
-        "The `metrics` argument should be a list -- you model can have any number of metrics.\n",
-        "\n",
-        "If your model has multiple outputs, you can specify  different losses and metrics for each output,\n",
-        "and you can modulate the contribution of each output to the total loss of the model. You will find more details about this in the section \"**Passing data to multi-input, multi-output models**\".\n",
-        "\n",
-        "Note that in many cases, the loss and metrics are specified via string identifiers, as a shortcut:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "pOQOcScsNiT3"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=1e-3),\n",
-        "              loss='sparse_categorical_crossentropy',\n",
-        "              metrics=['sparse_categorical_accuracy'])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "IVHOVTqFknZP"
-      },
-      "source": [
-        "For later reuse, let's put our model definition and compile step in functions; we will call them several times across different examples in this guide."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "G3Zrjsf_kt8k"
-      },
-      "outputs": [],
-      "source": [
-        "def get_uncompiled_model():\n",
-        "  inputs = keras.Input(shape=(784,), name='digits')\n",
-        "  x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "  x = layers.Dense(64, activation='relu', name='dense_2')(x)\n",
-        "  outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
-        "  model = keras.Model(inputs=inputs, outputs=outputs)\n",
-        "  return model\n",
-        "\n",
-        "def get_compiled_model():\n",
-        "  model = get_uncompiled_model()\n",
-        "  model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=1e-3),\n",
-        "              loss='sparse_categorical_crossentropy',\n",
-        "              metrics=['sparse_categorical_accuracy'])\n",
-        "  return model"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "_1rDyyQEtC8E"
-      },
-      "source": [
-        "#### Many built-in optimizers, losses, and metrics are available\n",
-        "\n",
-        "In general, you won't have to create from scratch your own losses, metrics, or optimizers, because what you need is likely already part of the Keras API:\n",
-        "\n",
-        "Optimizers:\n",
-        "- `SGD()` (with or without momentum)\n",
-        "- `RMSprop()`\n",
-        "- `Adam()`\n",
-        "- etc.\n",
-        "\n",
-        "Losses:\n",
-        "- `MeanSquaredError()`\n",
-        "- `KLDivergence()`\n",
-        "- `CosineSimilarity()`\n",
-        "- etc.\n",
-        "\n",
-        "Metrics:\n",
-        "- `AUC()`\n",
-        "- `Precision()`\n",
-        "- `Recall()`\n",
-        "- etc."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LH1QjaltNm2U"
-      },
-      "source": [
-        "#### Custom losses\n",
-        "\n",
-        "There are two ways to provide custom losses with Keras. The first example creates a function that accepts inputs `y_true` and `y_pred`. The following example shows a loss function that computes the average distance between the real data and the predictions:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "IIn3_iD0Nyll"
-      },
-      "outputs": [],
-      "source": [
-        "def basic_loss_function(y_true, y_pred):\n",
-        "    return tf.math.reduce_mean(y_true - y_pred)\n",
-        "\n",
-        "model.compile(optimizer=keras.optimizers.Adam(),\n",
-        "              loss=basic_loss_function)\n",
-        "\n",
-        "model.fit(x_train, y_train, batch_size=64, epochs=3)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "RZqpsybNN_Wh"
-      },
-      "source": [
-        "If you need a loss function that takes in parameters beside `y_true` and `y_pred`, you can subclass the `tf.keras.losses.Loss` class and implement the following two methods:\n",
-        "\n",
-        "* `__init__(self)` —Accept parameters to pass during the call of your loss function\n",
-        "* `call(self, y_true, y_pred)` —Use the targets (`y_true`) and the model predictions (`y_pred`) to compute the model's loss\n",
-        "\n",
-        "Parameters passed into `__init__()` can be used during `call()` when calculating loss.\n",
-        "\n",
-        "The following example shows how to implement a `WeightedCrossEntropy` loss function that calculates a `BinaryCrossEntropy` loss, where the loss of a certain class or the whole function can be modified by a scalar."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "gr996axuOCfX"
-      },
-      "outputs": [],
-      "source": [
-        "class WeightedBinaryCrossEntropy(keras.losses.Loss):\n",
-        "    \"\"\"\n",
-        "    Args:\n",
-        "      pos_weight: Scalar to affect the positive labels of the loss function.\n",
-        "      weight: Scalar to affect the entirety of the loss function.\n",
-        "      from_logits: Whether to compute loss form logits or the probability.\n",
-        "      reduction: Type of tf.keras.losses.Reduction to apply to loss.\n",
-        "      name: Name of the loss function.\n",
-        "    \"\"\"\n",
-        "    def __init__(self, pos_weight, weight, from_logits=False,\n",
-        "                 reduction=keras.losses.Reduction.AUTO,\n",
-        "                 name='weighted_binary_crossentropy'):\n",
-        "        super(WeightedBinaryCrossEntropy, self).__init__(reduction=reduction,\n",
-        "                                                         name=name)\n",
-        "        self.pos_weight = pos_weight\n",
-        "        self.weight = weight\n",
-        "        self.from_logits = from_logits\n",
-        "\n",
-        "    def call(self, y_true, y_pred):\n",
-        "        if not self.from_logits:\n",
-        "            # Manually calculate the weighted cross entropy.\n",
-        "            # Formula is qz * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))\n",
-        "            # where z are labels, x is logits, and q is the weight.\n",
-        "            # Since the values passed are from sigmoid (assuming in this case)\n",
-        "            # sigmoid(x) will be replaced by y_pred\n",
-        "\n",
-        "            # qz * -log(sigmoid(x)) 1e-6 is added as an epsilon to stop passing a zero into the log\n",
-        "            x_1 = y_true * self.pos_weight * -tf.math.log(y_pred + 1e-6)\n",
-        "\n",
-        "            # (1 - z) * -log(1 - sigmoid(x)). Epsilon is added to prevent passing a zero into the log\n",
-        "            x_2 = (1 - y_true) * -tf.math.log(1 - y_pred + 1e-6)\n",
-        "\n",
-        "            return tf.add(x_1, x_2) * self.weight \n",
-        "\n",
-        "        # Use built in function\n",
-        "        return tf.nn.weighted_cross_entropy_with_logits(y_true, y_pred, self.pos_weight) * self.weight\n",
-        "\n",
-        "\n",
-        "model.compile(optimizer=keras.optimizers.Adam(),\n",
-        "              loss=WeightedBinaryCrossEntropy(0.5, 2))\n",
-        "\n",
-        "model.fit(x_train, y_train, batch_size=64, epochs=3)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "YUkJi7ENDnCg"
-      },
-      "source": [
-        "#### Custom metrics\n",
-        "\n",
-        "If you need a metric that isn't part of the API, you can easily create custom metrics by subclassing the `Metric` class. You will need to implement 4 methods:\n",
-        "\n",
-        "- `__init__(self)`,  in which you will create state variables for your metric.\n",
-        "- `update_state(self, y_true, y_pred, sample_weight=None)`, which uses the targets `y_true` and the model predictions `y_pred` to update the state variables.\n",
-        "- `result(self)`, which uses the state variables to compute the final results.\n",
-        "- `reset_states(self)`, which reinitializes the state of the metric.\n",
-        "\n",
-        "State update and results computation are kept separate (in `update_state()` and `result()`, respectively) because in some cases, results computation might be very expensive, and would only be done periodically.\n",
-        "\n",
-        "Here's a simple example showing how to implement a `CategoricalTruePositives` metric, that counts how many samples where correctly classified as belonging to a given class:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "MbSNO6uePVsF"
-      },
-      "outputs": [],
-      "source": [
-        "class CategoricalTruePositives(keras.metrics.Metric):\n",
-        "\n",
-        "    def __init__(self, name='categorical_true_positives', **kwargs):\n",
-        "      super(CategoricalTruePositives, self).__init__(name=name, **kwargs)\n",
-        "      self.true_positives = self.add_weight(name='tp', initializer='zeros')\n",
-        "\n",
-        "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
-        "      y_pred = tf.reshape(tf.argmax(y_pred, axis=1), shape=(-1, 1))\n",
-        "      values = tf.cast(y_true, 'int32') == tf.cast(y_pred, 'int32')\n",
-        "      values = tf.cast(values, 'float32')\n",
-        "      if sample_weight is not None:\n",
-        "        sample_weight = tf.cast(sample_weight, 'float32')\n",
-        "        values = tf.multiply(values, sample_weight)\n",
-        "      self.true_positives.assign_add(tf.reduce_sum(values))\n",
-        "\n",
-        "    def result(self):\n",
-        "      return self.true_positives\n",
-        "\n",
-        "    def reset_states(self):\n",
-        "      # The state of the metric will be reset at the start of each epoch.\n",
-        "      self.true_positives.assign(0.)\n",
-        "\n",
-        "\n",
-        "model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=1e-3),\n",
-        "              loss=keras.losses.SparseCategoricalCrossentropy(),\n",
-        "              metrics=[CategoricalTruePositives()])\n",
-        "model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          epochs=3)\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "HipGsW7qDo0Q"
-      },
-      "source": [
-        "#### Handling losses and metrics that don't fit the standard signature\n",
-        "\n",
-        "The overwhelming majority of losses and metrics can be computed from `y_true` and `y_pred`, where `y_pred` is an output of your model. But not all of them. For instance, a regularization loss may only require the activation of a layer (there are no targets in this case), and this activation may not be a model output.\n",
-        "\n",
-        "In such cases, you can call `self.add_loss(loss_value)` from inside the `call` method of a custom layer. Here's a simple example that adds activity regularization (note that activity regularization is built-in in all Keras layers -- this layer is just for the sake of providing a concrete example):\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "CrYwrDR0PWab"
-      },
-      "outputs": [],
-      "source": [
-        "class ActivityRegularizationLayer(layers.Layer):\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    self.add_loss(tf.reduce_sum(inputs) * 0.1)\n",
-        "    return inputs  # Pass-through layer.\n",
-        "\n",
-        "inputs = keras.Input(shape=(784,), name='digits')\n",
-        "x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "\n",
-        "# Insert activity regularization as a layer\n",
-        "x = ActivityRegularizationLayer()(x)\n",
-        "\n",
-        "x = layers.Dense(64, activation='relu', name='dense_2')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
-        "\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs)\n",
-        "model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=1e-3),\n",
-        "              loss='sparse_categorical_crossentropy')\n",
-        "\n",
-        "# The displayed loss will be much higher than before\n",
-        "# due to the regularization component.\n",
-        "model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          epochs=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "fWnqsdwt06us"
-      },
-      "source": [
-        "You can do the same for logging metric values:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ZBYYhVgJ0973"
-      },
-      "outputs": [],
-      "source": [
-        "class MetricLoggingLayer(layers.Layer):\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    # The `aggregation` argument defines\n",
-        "    # how to aggregate the per-batch values\n",
-        "    # over each epoch:\n",
-        "    # in this case we simply average them.\n",
-        "    self.add_metric(keras.backend.std(inputs),\n",
-        "                    name='std_of_activation',\n",
-        "                    aggregation='mean')\n",
-        "    return inputs  # Pass-through layer.\n",
-        "\n",
-        "\n",
-        "inputs = keras.Input(shape=(784,), name='digits')\n",
-        "x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "\n",
-        "# Insert std logging as a layer.\n",
-        "x = MetricLoggingLayer()(x)\n",
-        "\n",
-        "x = layers.Dense(64, activation='relu', name='dense_2')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
-        "\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs)\n",
-        "model.compile(optimizer=keras.optimizers.RMSprop(learning_rate=1e-3),\n",
-        "              loss='sparse_categorical_crossentropy')\n",
-        "model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          epochs=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "f9_9XQvA0Jap"
-      },
-      "source": [
-        "In the [Functional API](functional.ipynb), you can also call `model.add_loss(loss_tensor)`, or `model.add_metric(metric_tensor, name, aggregation)`.\n",
-        "\n",
-        "Here's a simple example:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "WrA1yFql0gXg"
-      },
-      "outputs": [],
-      "source": [
-        "inputs = keras.Input(shape=(784,), name='digits')\n",
-        "x1 = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "x2 = layers.Dense(64, activation='relu', name='dense_2')(x1)\n",
-        "outputs = layers.Dense(10, activation='softmax', name='predictions')(x2)\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs)\n",
-        "\n",
-        "model.add_loss(tf.reduce_sum(x1) * 0.1)\n",
-        "\n",
-        "model.add_metric(keras.backend.std(x1),\n",
-        "                 name='std_of_activation',\n",
-        "                 aggregation='mean')\n",
-        "\n",
-        "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "              loss='sparse_categorical_crossentropy')\n",
-        "model.fit(x_train, y_train,\n",
-        "          batch_size=64,\n",
-        "          epochs=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "sJqpjGyMAa82"
-      },
-      "source": [
-        "\n",
-        "#### Automatically setting apart a validation holdout set\n",
-        "\n",
-        "In the first end-to-end example you saw, we used the `validation_data` argument to pass a tuple\n",
-        "of Numpy arrays `(x_val, y_val)` to the model for evaluating a validation loss and validation metrics at the end of each epoch.\n",
-        "\n",
-        "Here's another option: the argument `validation_split` allows you to automatically reserve part of your training data for validation. The argument value represents the fraction of the data to be reserved for validation, so it should be set to a number higher than 0 and lower than 1. For instance, `validation_split=0.2` means \"use 20% of the data for validation\", and `validation_split=0.6` means \"use 60% of the data for validation\".\n",
-        "\n",
-        "The way the validation is computed is by *taking the last x% samples of the arrays received by the `fit` call, before any shuffling*.\n",
-        "\n",
-        "You can only use `validation_split` when training with Numpy data."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Z-DQyeRePYS-"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_compiled_model()\n",
-        "model.fit(x_train, y_train, batch_size=64, validation_split=0.2, epochs=1, steps_per_epoch=1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Uq0TDb15DBbc"
-      },
-      "source": [
-        "### Training \u0026 evaluation from tf.data Datasets\n",
-        "\n",
-        "In the past few paragraphs, you've seen how to handle losses, metrics, and optimizers, and you've seen how to use the `validation_data` and `validation_split` arguments in `fit`, when your data is passed as Numpy arrays.\n",
-        "\n",
-        "Let's now take a look at the case where your data comes in the form of a tf.data Dataset.\n",
-        "\n",
-        "The tf.data API is a set of utilities in TensorFlow 2.0 for loading and preprocessing data in a way that's fast and scalable.\n",
-        "\n",
-        "For a complete guide about creating Datasets, see [the tf.data documentation](https://www.tensorflow.org/guide/data).\n",
-        "\n",
-        "You can pass a Dataset instance directly to the methods `fit()`, `evaluate()`, and `predict()`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "iY27EJsmENBj"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_compiled_model()\n",
-        "\n",
-        "# First, let's create a training Dataset instance.\n",
-        "# For the sake of our example, we'll use the same MNIST data as before.\n",
-        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
-        "# Shuffle and slice the dataset.\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(64)\n",
-        "\n",
-        "# Now we get a test dataset.\n",
-        "test_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
-        "test_dataset = test_dataset.batch(64)\n",
-        "\n",
-        "# Since the dataset already takes care of batching,\n",
-        "# we don't pass a `batch_size` argument.\n",
-        "model.fit(train_dataset, epochs=3)\n",
-        "\n",
-        "# You can also evaluate or predict on a dataset.\n",
-        "print('\\n# Evaluate')\n",
-        "model.evaluate(test_dataset)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "r2yh2h79ENUy"
-      },
-      "source": [
-        "Note that the Dataset is reset at the end of each epoch, so it can be reused of the next epoch.\n",
-        "\n",
-        "If you want to run training only on a specific number of batches from this Dataset, you can pass the `steps_per_epoch` argument, which specifies how many training steps the model should run using this Dataset before moving on to the next epoch.\n",
-        "\n",
-        "If you do this, the dataset is not reset at the end of each epoch, instead we just keep drawing the next batches. The dataset will eventually run out of data (unless it is an infinitely-looping dataset)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "7t0V6K_EEdB7"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_compiled_model()\n",
-        "\n",
-        "# Prepare the training dataset\n",
-        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(64)\n",
-        "\n",
-        "# Only use the 100 batches per epoch (that's 64 * 100 samples)\n",
-        "model.fit(train_dataset.take(100), epochs=3)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "_e3b3rSmC-fL"
-      },
-      "source": [
-        "#### Using a validation dataset\n",
-        "\n",
-        "You can pass a Dataset instance as the `validation_data` argument in `fit`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "pv_53VE2C-11"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_compiled_model()\n",
-        "\n",
-        "# Prepare the training dataset\n",
-        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(64)\n",
-        "\n",
-        "# Prepare the validation dataset\n",
-        "val_dataset = tf.data.Dataset.from_tensor_slices((x_val, y_val))\n",
-        "val_dataset = val_dataset.batch(64)\n",
-        "\n",
-        "model.fit(train_dataset, epochs=3, validation_data=val_dataset)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "J2Z7pgmSC7Zk"
-      },
-      "source": [
-        "At the end of each epoch, the model will iterate over the validation Dataset and compute the validation loss and validation metrics.\n",
-        "\n",
-        "If you want to run validation only on a specific number of batches from this Dataset, you can pass the `validation_steps` argument, which specifies how many validation steps the model should run with the validation Dataset before interrupting validation and moving on to the next epoch:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "eGa4DQA0C8Mu"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_compiled_model()\n",
-        "\n",
-        "# Prepare the training dataset\n",
-        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(64)\n",
-        "\n",
-        "# Prepare the validation dataset\n",
-        "val_dataset = tf.data.Dataset.from_tensor_slices((x_val, y_val))\n",
-        "val_dataset = val_dataset.batch(64)\n",
-        "\n",
-        "model.fit(train_dataset, epochs=3,\n",
-        "          # Only run validation using the first 10 batches of the dataset\n",
-        "          # using the `validation_steps` argument\n",
-        "          validation_data=val_dataset, validation_steps=10)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "6qo-fWETC4sD"
-      },
-      "source": [
-        "Note that the validation Dataset will be reset after each use (so that you will always be evaluating on the same samples from epoch to epoch).\n",
-        "\n",
-        "The argument `validation_split` (generating a holdout set from the training data) is not supported when training from Dataset objects, since this features requires the ability to index the samples of the datasets, which is not possible in general with the Dataset API."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "cIGt6NNqC2ZR"
-      },
-      "source": [
-        "\n",
-        "### Other input formats supported\n",
-        "\n",
-        "Besides Numpy arrays and TensorFlow Datasets, it's possible to train a Keras model using Pandas dataframes, or from Python generators that yield batches.\n",
-        "\n",
-        "In general, we recommend that you use Numpy input data if your data is small and fits in memory, and Datasets otherwise."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Bid1XnTl_zNT"
-      },
-      "source": [
-        "### Using sample weighting and class weighting\n",
-        "\n",
-        "Besides input data and target data, it is possible to pass sample weights or class weights to a model when using `fit`:\n",
-        "\n",
-        "- When training from Numpy data: via the `sample_weight` and `class_weight` arguments.\n",
-        "- When training from Datasets: by having the Dataset return a tuple `(input_batch, target_batch, sample_weight_batch)` .\n",
-        "\n",
-        "A \"sample weights\" array is an array of numbers that specify how much weight each sample in a batch should have in computing the total loss. It is commonly used in imbalanced classification problems (the idea being to give more weight to rarely-seen classes). When the weights used are ones and zeros, the array can be used as a *mask* for the loss function (entirely discarding the contribution of certain samples to the total loss).\n",
-        "\n",
-        "A \"class weights\" dict is a more specific instance of the same concept: it maps class indices to the sample weight that should be used for samples belonging to this class. For instance, if class \"0\" is twice less represented than class \"1\" in your data, you could use `class_weight={0: 1., 1: 0.5}`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rbH4jT5Hp_gg"
-      },
-      "source": [
-        "Here's a Numpy example where we use class weights or sample weights to give more importance to the correct classification of class #5 (which is the digit \"5\" in the MNIST dataset)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0Y7QBNUXWTva"
-      },
-      "outputs": [],
-      "source": [
-        "import numpy as np\n",
-        "\n",
-        "class_weight = {0: 1., 1: 1., 2: 1., 3: 1., 4: 1.,\n",
-        "                # Set weight \"2\" for class \"5\",\n",
-        "                # making this class 2x more important\n",
-        "                5: 2.,\n",
-        "                6: 1., 7: 1., 8: 1., 9: 1.}\n",
-        "print('Fit with class weight')\n",
-        "model.fit(x_train, y_train,\n",
-        "          class_weight=class_weight,\n",
-        "          batch_size=64,\n",
-        "          epochs=4)\n",
-        "\n",
-        "# Here's the same example using `sample_weight` instead:\n",
-        "sample_weight = np.ones(shape=(len(y_train),))\n",
-        "sample_weight[y_train == 5] = 2.\n",
-        "print('\\nFit with sample weight')\n",
-        "\n",
-        "model = get_compiled_model()\n",
-        "model.fit(x_train, y_train,\n",
-        "          sample_weight=sample_weight,\n",
-        "          batch_size=64,\n",
-        "          epochs=4)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Qf1KvM0pqBaV"
-      },
-      "source": [
-        "Here's a matching Dataset example:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "LHD60GshxSpf"
-      },
-      "outputs": [],
-      "source": [
-        "sample_weight = np.ones(shape=(len(y_train),))\n",
-        "sample_weight[y_train == 5] = 2.\n",
-        "\n",
-        "# Create a Dataset that includes sample weights\n",
-        "# (3rd element in the return tuple).\n",
-        "train_dataset = tf.data.Dataset.from_tensor_slices(\n",
-        "    (x_train, y_train, sample_weight))\n",
-        "\n",
-        "# Shuffle and slice the dataset.\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(64)\n",
-        "\n",
-        "model = get_compiled_model()\n",
-        "model.fit(train_dataset, epochs=3)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "jXt0xQvDxMeq"
-      },
-      "source": [
-        "### Passing data to multi-input, multi-output models\n",
-        "\n",
-        "In the previous examples, we were considering a model with a single input (a tensor of shape `(764,)`) and a single output (a prediction tensor of shape `(10,)`). But what about models that have multiple inputs or outputs?\n",
-        "\n",
-        "Consider the following model, which has an image input of shape `(32, 32, 3)` (that's `(height, width, channels)`) and a timeseries input of shape `(None, 10)` (that's `(timesteps, features)`). Our model will have two outputs computed from the combination of these inputs: a \"score\" (of shape `(1,)`) and a probability distribution over five classes (of shape `(5,)`).\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "QNUSGfKq1cZ-"
-      },
-      "outputs": [],
-      "source": [
-        "from tensorflow import keras\n",
-        "from tensorflow.keras import layers\n",
-        "\n",
-        "image_input = keras.Input(shape=(32, 32, 3), name='img_input')\n",
-        "timeseries_input = keras.Input(shape=(None, 10), name='ts_input')\n",
-        "\n",
-        "x1 = layers.Conv2D(3, 3)(image_input)\n",
-        "x1 = layers.GlobalMaxPooling2D()(x1)\n",
-        "\n",
-        "x2 = layers.Conv1D(3, 3)(timeseries_input)\n",
-        "x2 = layers.GlobalMaxPooling1D()(x2)\n",
-        "\n",
-        "x = layers.concatenate([x1, x2])\n",
-        "\n",
-        "score_output = layers.Dense(1, name='score_output')(x)\n",
-        "class_output = layers.Dense(5, activation='softmax', name='class_output')(x)\n",
-        "\n",
-        "model = keras.Model(inputs=[image_input, timeseries_input],\n",
-        "                    outputs=[score_output, class_output])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "cIWhMd7p3NGp"
-      },
-      "source": [
-        "Let's plot this model, so you can clearly see what we're doing here (note that the shapes shown in the plot are batch shapes, rather than per-sample shapes)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "gKVQ4Y573Q_c"
-      },
-      "outputs": [],
-      "source": [
-        "keras.utils.plot_model(model, 'multi_input_and_output_model.png', show_shapes=True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vAauLxiT278G"
-      },
-      "source": [
-        "At compilation time, we can specify different losses to different outputs, by passing the loss functions as a list:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "vDa7JSz93phE"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(\n",
-        "    optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "    loss=[keras.losses.MeanSquaredError(),\n",
-        "          keras.losses.CategoricalCrossentropy()])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "bQip1Fex4-D5"
-      },
-      "source": [
-        "If we only passed a single loss function to the model, the same loss function would be applied to every output, which is not appropriate here.\n",
-        "\n",
-        "Likewise for metrics:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Tkl5LMGi4_gK"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(\n",
-        "    optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "    loss=[keras.losses.MeanSquaredError(),\n",
-        "          keras.losses.CategoricalCrossentropy()],\n",
-        "    metrics=[[keras.metrics.MeanAbsolutePercentageError(),\n",
-        "              keras.metrics.MeanAbsoluteError()],\n",
-        "             [keras.metrics.CategoricalAccuracy()]])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "w_WWk9b1374P"
-      },
-      "source": [
-        "Since we gave names to our output layers, we could also specify per-output losses and metrics via a dict:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1MEctLBD4APc"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(\n",
-        "    optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "    loss={'score_output': keras.losses.MeanSquaredError(),\n",
-        "          'class_output': keras.losses.CategoricalCrossentropy()},\n",
-        "    metrics={'score_output': [keras.metrics.MeanAbsolutePercentageError(),\n",
-        "                              keras.metrics.MeanAbsoluteError()],\n",
-        "             'class_output': [keras.metrics.CategoricalAccuracy()]})"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "NVXRX0PX6lIn"
-      },
-      "source": [
-        "We recommend the use of explicit names and dicts if you have more than 2 outputs."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "RX3FTDWw4BOA"
-      },
-      "source": [
-        "It's possible to give different weights to different output-specific losses (for instance, one might wish to privilege the \"score\" loss in our example, by giving to 2x the importance of the class loss), using the `loss_weights` argument:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mgVK3xAv4JNv"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(\n",
-        "    optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "    loss={'score_output': keras.losses.MeanSquaredError(),\n",
-        "          'class_output': keras.losses.CategoricalCrossentropy()},\n",
-        "    metrics={'score_output': [keras.metrics.MeanAbsolutePercentageError(),\n",
-        "                              keras.metrics.MeanAbsoluteError()],\n",
-        "             'class_output': [keras.metrics.CategoricalAccuracy()]},\n",
-        "    loss_weights={'score_output': 2., 'class_output': 1.})"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vd8VzIu-3p0n"
-      },
-      "source": [
-        "You could also chose not to compute a loss for certain outputs, if these outputs meant for prediction but not for training:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "aUyohARI4Kqn"
-      },
-      "outputs": [],
-      "source": [
-        "# List loss version\n",
-        "model.compile(\n",
-        "    optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "    loss=[None, keras.losses.CategoricalCrossentropy()])\n",
-        "\n",
-        "# Or dict loss version\n",
-        "model.compile(\n",
-        "    optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "    loss={'class_output': keras.losses.CategoricalCrossentropy()})"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "n0MwZc184MG_"
-      },
-      "source": [
-        "Passing data to a multi-input or multi-output model in `fit` works in a similar way as specifying a loss function in `compile`:\n",
-        "you can pass *lists of Numpy arrays (with 1:1 mapping to the outputs that received a loss function)* or *dicts mapping output names to Numpy arrays of training data*."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "So4cYwSW4la8"
-      },
-      "outputs": [],
-      "source": [
-        "model.compile(\n",
-        "    optimizer=keras.optimizers.RMSprop(1e-3),\n",
-        "    loss=[keras.losses.MeanSquaredError(),\n",
-        "          keras.losses.CategoricalCrossentropy()])\n",
-        "\n",
-        "# Generate dummy Numpy data\n",
-        "img_data = np.random.random_sample(size=(100, 32, 32, 3))\n",
-        "ts_data = np.random.random_sample(size=(100, 20, 10))\n",
-        "score_targets = np.random.random_sample(size=(100, 1))\n",
-        "class_targets = np.random.random_sample(size=(100, 5))\n",
-        "\n",
-        "# Fit on lists\n",
-        "model.fit([img_data, ts_data], [score_targets, class_targets],\n",
-        "          batch_size=32,\n",
-        "          epochs=3)\n",
-        "\n",
-        "# Alternatively, fit on dicts\n",
-        "model.fit({'img_input': img_data, 'ts_input': ts_data},\n",
-        "          {'score_output': score_targets, 'class_output': class_targets},\n",
-        "          batch_size=32,\n",
-        "          epochs=3)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "m7Ah1QUf4ll6"
-      },
-      "source": [
-        "Here's the Dataset use case: similarly as what we did for Numpy arrays, the Dataset should return\n",
-        "a tuple of dicts."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "vx34-FDZ4tdJ"
-      },
-      "outputs": [],
-      "source": [
-        "train_dataset = tf.data.Dataset.from_tensor_slices(\n",
-        "    ({'img_input': img_data, 'ts_input': ts_data},\n",
-        "     {'score_output': score_targets, 'class_output': class_targets}))\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(64)\n",
-        "\n",
-        "model.fit(train_dataset, epochs=3)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Y_gqVFcHAAga"
-      },
-      "source": [
-        "### Using callbacks\n",
-        "\n",
-        "Callbacks in Keras are objects that are called at different point during training (at the start of an epoch, at the end of a batch, at the end of an epoch, etc.) and which can be used to implement behaviors such as:\n",
-        "\n",
-        "- Doing validation at different points during training (beyond the built-in per-epoch validation)\n",
-        "- Checkpointing the model at regular intervals or when it exceeds a certain accuracy threshold\n",
-        "- Changing the learning rate of the model when training seems to be plateauing\n",
-        "- Doing fine-tuning of the top layers when training seems to be plateauing\n",
-        "- Sending email or instant message notifications when training ends or where a certain performance threshold is exceeded\n",
-        "- Etc.\n",
-        "\n",
-        "Callbacks can be passed as a list to your call to `fit`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "pNEhXWcnSG9B"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_compiled_model()\n",
-        "\n",
-        "callbacks = [\n",
-        "    keras.callbacks.EarlyStopping(\n",
-        "        # Stop training when `val_loss` is no longer improving\n",
-        "        monitor='val_loss',\n",
-        "        # \"no longer improving\" being defined as \"no better than 1e-2 less\"\n",
-        "        min_delta=1e-2,\n",
-        "        # \"no longer improving\" being further defined as \"for at least 2 epochs\"\n",
-        "        patience=2,\n",
-        "        verbose=1)\n",
-        "]\n",
-        "model.fit(x_train, y_train,\n",
-        "          epochs=20,\n",
-        "          batch_size=64,\n",
-        "          callbacks=callbacks,\n",
-        "          validation_split=0.2)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "PE3VK3WpSHL_"
-      },
-      "source": [
-        "#### Many built-in callbacks are available\n",
-        "\n",
-        "- `ModelCheckpoint`: Periodically save the model.\n",
-        "- `EarlyStopping`: Stop training when training is no longer improving the validation metrics.\n",
-        "- `TensorBoard`: periodically write model logs that can be visualized in TensorBoard (more details in the section \"Visualization\").\n",
-        "- `CSVLogger`: streams loss and metrics data to a CSV file.\n",
-        "- etc.\n",
-        "\n",
-        "\n",
-        "\n",
-        "#### Writing your own callback\n",
-        "\n",
-        "You can create a custom callback by extending the base class keras.callbacks.Callback. A callback has access to its associated model through the class property `self.model`.\n",
-        "\n",
-        "Here's a simple example saving a list of per-batch loss values during training:\n",
-        "\n",
-        "```python\n",
-        "class LossHistory(keras.callbacks.Callback):\n",
-        "\n",
-        "    def on_train_begin(self, logs):\n",
-        "        self.losses = []\n",
-        "\n",
-        "    def on_batch_end(self, batch, logs):\n",
-        "        self.losses.append(logs.get('loss'))\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "mC08S-e-yOd7"
-      },
-      "source": [
-        "### Checkpointing models\n",
-        "\n",
-        "When you're training model on relatively large datasets, it's crucial to save checkpoints of your model at frequent intervals.\n",
-        "\n",
-        "The easiest way to achieve this is with the `ModelCheckpoint` callback:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "yEER-qc-xXNC"
-      },
-      "outputs": [],
-      "source": [
-        "model = get_compiled_model()\n",
-        "\n",
-        "callbacks = [\n",
-        "    keras.callbacks.ModelCheckpoint(\n",
-        "        filepath='mymodel_{epoch}.h5',\n",
-        "        # Path where to save the model\n",
-        "        # The two parameters below mean that we will overwrite\n",
-        "        # the current checkpoint if and only if\n",
-        "        # the `val_loss` score has improved.\n",
-        "        save_best_only=True,\n",
-        "        monitor='val_loss',\n",
-        "        verbose=1)\n",
-        "]\n",
-        "model.fit(x_train, y_train,\n",
-        "          epochs=3,\n",
-        "          batch_size=64,\n",
-        "          callbacks=callbacks,\n",
-        "          validation_split=0.2)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "E1AZEf3ixXm8"
-      },
-      "source": [
-        "You call also write your own callback for saving and restoring models.\n",
-        "\n",
-        "For a complete guide on serialization and saving, see [Guide to Saving and Serializing Models](./save_and_serialize.ipynb)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "j22FtN-z_-Wj"
-      },
-      "source": [
-        "### Using learning rate schedules\n",
-        "\n",
-        "A common pattern when training deep learning models is to gradually reduce the learning as training progresses. This is generally known as \"learning rate decay\".\n",
-        "\n",
-        "The learning decay schedule could be static (fixed in advance, as a function of the current epoch or the current batch index), or dynamic (responding to the current behavior of the model, in particular the validation loss).\n",
-        "\n",
-        "#### Passing a schedule to an optimizer\n",
-        "\n",
-        "You can easily use a static learning rate decay schedule by passing a schedule object as the `learning_rate` argument in your optimizer:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Cu-pnB_ctEav"
-      },
-      "outputs": [],
-      "source": [
-        "initial_learning_rate = 0.1\n",
-        "lr_schedule = keras.optimizers.schedules.ExponentialDecay(\n",
-        "    initial_learning_rate,\n",
-        "    decay_steps=100000,\n",
-        "    decay_rate=0.96,\n",
-        "    staircase=True)\n",
-        "\n",
-        "optimizer = keras.optimizers.RMSprop(learning_rate=lr_schedule)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "9Ltf0Xtrtw-2"
-      },
-      "source": [
-        "Several built-in schedules are available: `ExponentialDecay`, `PiecewiseConstantDecay`, `PolynomialDecay`, and `InverseTimeDecay`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "13CvYhBptEjh"
-      },
-      "source": [
-        "#### Using callbacks to implement a dynamic learning rate schedule\n",
-        "\n",
-        "A dynamic learning rate schedule (for instance, decreasing the learning rate when the validation loss is no longer improving) cannot be achieved with these schedule objects since the optimizer does not have access to validation metrics.\n",
-        "\n",
-        "However, callbacks do have access to all metrics, including validation metrics! You can thus achieve this pattern by using a callback that modifies the current learning rate on the optimizer. In fact, this is even built-in as the `ReduceLROnPlateau` callback."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "tnUhGslABdvx"
-      },
-      "source": [
-        "### Visualizing loss and metrics during training\n",
-        "\n",
-        "The best way to keep an eye on your model during training is to use [TensorBoard](https://www.tensorflow.org/tensorboard), a browser-based application that you can run locally that provides you with:\n",
-        "\n",
-        "- Live plots of the loss and metrics for training and evaluation\n",
-        "- (optionally) Visualizations of the histograms of your layer activations\n",
-        "- (optionally) 3D visualizations of the embedding spaces learned by your `Embedding` layers\n",
-        "\n",
-        "If you have installed TensorFlow with pip, you should be able to launch TensorBoard from the command line:\n",
-        "\n",
-        "```\n",
-        "tensorboard --logdir=/full_path_to_your_logs\n",
-        "```\n",
-        "\n",
-        "#### Using the TensorBoard callback\n",
-        "\n",
-        "The easiest way to use TensorBoard with a Keras model and the `fit` method is the `TensorBoard` callback.\n",
-        "\n",
-        "In the simplest case, just specify where you want the callback to write logs, and you're good to go:\n",
-        "\n",
-        "```python\n",
-        "tensorboard_cbk = keras.callbacks.TensorBoard(log_dir='/full_path_to_your_logs')\n",
-        "model.fit(dataset, epochs=10, callbacks=[tensorboard_cbk])\n",
-        "```\n",
-        "\n",
-        "The `TensorBoard` callback has many useful options, including whether to log embeddings, histograms, and how often to write logs:\n",
-        "\n",
-        "```python\n",
-        "keras.callbacks.TensorBoard(\n",
-        "  log_dir='/full_path_to_your_logs',\n",
-        "  histogram_freq=0,  # How often to log histogram visualizations\n",
-        "  embeddings_freq=0,  # How often to log embedding visualizations\n",
-        "  update_freq='epoch')  # How often to write logs (default: once per epoch)\n",
-        "```\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "5r5ZnFry7-B7"
-      },
-      "source": [
-        "## Part II: Writing your own training \u0026 evaluation loops from scratch\n",
-        "\n",
-        "If you want lower-level over your training \u0026 evaluation loops than what `fit()` and `evaluate()` provide, you should write your own. It's actually pretty simple! But you should be ready to have a lot more debugging to do on your own."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "HaRvt057_37D"
-      },
-      "source": [
-        "### Using the GradientTape: a first end-to-end example\n",
-        "\n",
-        "Calling a model inside a `GradientTape` scope enables you to retrieve the gradients of the trainable weights of the layer with respect to a loss value. Using an optimizer instance, you can use these gradients to update these variables (which you can retrieve using `model.trainable_weights`).\n",
-        "\n",
-        "Let's reuse our initial MNIST model from Part I, and let's train it using mini-batch gradient with a custom training loop."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "XQqxREEL66Kg"
-      },
-      "outputs": [],
-      "source": [
-        "# Get the model.\n",
-        "inputs = keras.Input(shape=(784,), name='digits')\n",
-        "x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "x = layers.Dense(64, activation='relu', name='dense_2')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs)\n",
-        "\n",
-        "# Instantiate an optimizer.\n",
-        "optimizer = keras.optimizers.SGD(learning_rate=1e-3)\n",
-        "# Instantiate a loss function.\n",
-        "loss_fn = keras.losses.SparseCategoricalCrossentropy()\n",
-        "\n",
-        "# Prepare the training dataset.\n",
-        "batch_size = 64\n",
-        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(batch_size)\n",
-        "\n",
-        "# Iterate over epochs.\n",
-        "epochs = 3\n",
-        "for epoch in range(epochs):\n",
-        "  print('Start of epoch %d' % (epoch,))\n",
-        "\n",
-        "  # Iterate over the batches of the dataset.\n",
-        "  for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):\n",
-        "\n",
-        "    # Open a GradientTape to record the operations run\n",
-        "    # during the forward pass, which enables autodifferentiation.\n",
-        "    with tf.GradientTape() as tape:\n",
-        "\n",
-        "      # Run the forward pass of the layer.\n",
-        "      # The operations that the layer applies\n",
-        "      # to its inputs are going to be recorded\n",
-        "      # on the GradientTape.\n",
-        "      logits = model(x_batch_train)  # Logits for this minibatch\n",
-        "\n",
-        "      # Compute the loss value for this minibatch.\n",
-        "      loss_value = loss_fn(y_batch_train, logits)\n",
-        "\n",
-        "    # Use the gradient tape to automatically retrieve\n",
-        "    # the gradients of the trainable variables with respect to the loss.\n",
-        "    grads = tape.gradient(loss_value, model.trainable_weights)\n",
-        "\n",
-        "    # Run one step of gradient descent by updating\n",
-        "    # the value of the variables to minimize the loss.\n",
-        "    optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
-        "\n",
-        "    # Log every 200 batches.\n",
-        "    if step % 200 == 0:\n",
-        "        print('Training loss (for one batch) at step %s: %s' % (step, float(loss_value)))\n",
-        "        print('Seen so far: %s samples' % ((step + 1) * 64))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "1lt9Yva8xgCs"
-      },
-      "source": [
-        "### Low-level handling of metrics\n",
-        "\n",
-        "Let's add metrics to the mix. You can readily reuse the built-in metrics (or custom ones you wrote) in such training loops written from scratch. Here's the flow:\n",
-        "\n",
-        "- Instantiate the metric at the start of the loop\n",
-        "- Call `metric.update_state()` after each batch\n",
-        "- Call `metric.result()` when you need to display the current value of the metric\n",
-        "- Call `metric.reset_states()` when you need to clear the state of the metric (typically at the end of an epoch)\n",
-        "\n",
-        "Let's use this knowledge to compute `SparseCategoricalAccuracy` on validation data at the end of each epoch:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "JDKgp70UxwdR"
-      },
-      "outputs": [],
-      "source": [
-        "# Get model\n",
-        "inputs = keras.Input(shape=(784,), name='digits')\n",
-        "x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "x = layers.Dense(64, activation='relu', name='dense_2')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs)\n",
-        "\n",
-        "# Instantiate an optimizer to train the model.\n",
-        "optimizer = keras.optimizers.SGD(learning_rate=1e-3)\n",
-        "# Instantiate a loss function.\n",
-        "loss_fn = keras.losses.SparseCategoricalCrossentropy()\n",
-        "\n",
-        "# Prepare the metrics.\n",
-        "train_acc_metric = keras.metrics.SparseCategoricalAccuracy()\n",
-        "val_acc_metric = keras.metrics.SparseCategoricalAccuracy()\n",
-        "\n",
-        "# Prepare the training dataset.\n",
-        "batch_size = 64\n",
-        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
-        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(batch_size)\n",
-        "\n",
-        "# Prepare the validation dataset.\n",
-        "val_dataset = tf.data.Dataset.from_tensor_slices((x_val, y_val))\n",
-        "val_dataset = val_dataset.batch(64)\n",
-        "\n",
-        "\n",
-        "# Iterate over epochs.\n",
-        "epochs = 3\n",
-        "for epoch in range(epochs):\n",
-        "  print('Start of epoch %d' % (epoch,))\n",
-        "\n",
-        "  # Iterate over the batches of the dataset.\n",
-        "  for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):\n",
-        "    with tf.GradientTape() as tape:\n",
-        "      logits = model(x_batch_train)\n",
-        "      loss_value = loss_fn(y_batch_train, logits)\n",
-        "    grads = tape.gradient(loss_value, model.trainable_weights)\n",
-        "    optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
-        "\n",
-        "    # Update training metric.\n",
-        "    train_acc_metric(y_batch_train, logits)\n",
-        "\n",
-        "    # Log every 200 batches.\n",
-        "    if step % 200 == 0:\n",
-        "        print('Training loss (for one batch) at step %s: %s' % (step, float(loss_value)))\n",
-        "        print('Seen so far: %s samples' % ((step + 1) * 64))\n",
-        "\n",
-        "  # Display metrics at the end of each epoch.\n",
-        "  train_acc = train_acc_metric.result()\n",
-        "  print('Training acc over epoch: %s' % (float(train_acc),))\n",
-        "  # Reset training metrics at the end of each epoch\n",
-        "  train_acc_metric.reset_states()\n",
-        "\n",
-        "  # Run a validation loop at the end of each epoch.\n",
-        "  for x_batch_val, y_batch_val in val_dataset:\n",
-        "    val_logits = model(x_batch_val)\n",
-        "    # Update val metrics\n",
-        "    val_acc_metric(y_batch_val, val_logits)\n",
-        "  val_acc = val_acc_metric.result()\n",
-        "  val_acc_metric.reset_states()\n",
-        "  print('Validation acc: %s' % (float(val_acc),))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rjraqJAonBJK"
-      },
-      "source": [
-        "### Low-level handling of extra losses\n",
-        "\n",
-        "You saw in the previous section that it is possible for regularization losses to be added by a layer by calling `self.add_loss(value)` in the `call` method.\n",
-        "\n",
-        "In the general case, you will want to take these losses into account in your custom training loops (unless you've written the model yourself and you already know that it creates no such losses).\n",
-        "\n",
-        "Recall this example from the previous section, featuring a layer that creates a regularization loss:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "fglnb_wzwtxh"
-      },
-      "outputs": [],
-      "source": [
-        "class ActivityRegularizationLayer(layers.Layer):\n",
-        "\n",
-        "  def call(self, inputs):\n",
-        "    self.add_loss(1e-2 * tf.reduce_sum(inputs))\n",
-        "    return inputs\n",
-        "\n",
-        "inputs = keras.Input(shape=(784,), name='digits')\n",
-        "x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n",
-        "# Insert activity regularization as a layer\n",
-        "x = ActivityRegularizationLayer()(x)\n",
-        "x = layers.Dense(64, activation='relu', name='dense_2')(x)\n",
-        "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n",
-        "\n",
-        "model = keras.Model(inputs=inputs, outputs=outputs)\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "44zwXYXdwwss"
-      },
-      "source": [
-        "When you call a model, like this:\n",
-        "\n",
-        "```python\n",
-        "logits = model(x_train)\n",
-        "```\n",
-        "\n",
-        "the losses it creates during the forward pass are added to the `model.losses` attribute:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "KOWuXgDlxIbS"
-      },
-      "outputs": [],
-      "source": [
-        "logits = model(x_train[:64])\n",
-        "print(model.losses)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "EvkOERFYymF_"
-      },
-      "source": [
-        "The tracked losses are first cleared at the start of the model `__call__`, so you will only see the losses created during this one forward pass. For instance, calling the model repeatedly and then querying `losses` only displays the latest losses, created during the last call:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "MZh9ek_NzEtQ"
-      },
-      "outputs": [],
-      "source": [
-        "logits = model(x_train[:64])\n",
-        "logits = model(x_train[64: 128])\n",
-        "logits = model(x_train[128: 192])\n",
-        "print(model.losses)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "MmftRMi6zWLN"
-      },
-      "source": [
-        "To take these losses into account during training, all you have to do is to modify your training loop to add `sum(model.losses)` to your total loss:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ueLhIG1mzdFp"
-      },
-      "outputs": [],
-      "source": [
-        "optimizer = keras.optimizers.SGD(learning_rate=1e-3)\n",
-        "\n",
-        "epochs = 3\n",
-        "for epoch in range(epochs):\n",
-        "  print('Start of epoch %d' % (epoch,))\n",
-        "\n",
-        "  for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):\n",
-        "    with tf.GradientTape() as tape:\n",
-        "      logits = model(x_batch_train)\n",
-        "      loss_value = loss_fn(y_batch_train, logits)\n",
-        "\n",
-        "      # Add extra losses created during this forward pass:\n",
-        "      loss_value += sum(model.losses)\n",
-        "\n",
-        "    grads = tape.gradient(loss_value, model.trainable_weights)\n",
-        "    optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
-        "\n",
-        "    # Log every 200 batches.\n",
-        "    if step % 200 == 0:\n",
-        "        print('Training loss (for one batch) at step %s: %s' % (step, float(loss_value)))\n",
-        "        print('Seen so far: %s samples' % ((step + 1) * 64))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "wCOe4e4dxsB0"
-      },
-      "source": [
-        "That was the last piece of the puzzle! You've reached the end of this guide.\n",
-        "\n",
-        "Now you know everything there is to know about using built-in training loops and writing your own from scratch.\n"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "train_and_evaluate.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/migrate.ipynb b/site/en/guide/migrate.ipynb
deleted file mode 100644
index f0949230496..00000000000
--- a/site/en/guide/migrate.ipynb
+++ /dev/null
@@ -1,2108 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "wJcYs_ERTnnI"
-      },
-      "source": [
-        "##### Copyright 2018 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
-        "id": "HMUDt0CiUJk9"
-      },
-      "outputs": [],
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "77z2OchJTk0l"
-      },
-      "source": [
-        "# Migrate your TensorFlow 1 code to TensorFlow 2\n",
-        "\n",
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/migrate\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n",
-        "    View on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/migrate.ipynb\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n",
-        "    Run in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/migrate.ipynb\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n",
-        "    View source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/migrate.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "meUTrR4I6m1C"
-      },
-      "source": [
-        "Important: This doc for users of low level TensorFlow APIs. If you are using the high level APIs (`tf.keras`) there may be little or no action you need to take to make your code fully TensorFlow 2.0 compatible. Check your [optimizer's default learning rate](#keras_optimizer_lr). "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "C0V10enS1_WU"
-      },
-      "source": [
-        "It is still possible to run 1.X code, unmodified ([except for contrib](https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md)), in TensorFlow 2.0:\n",
-        "\n",
-        "```\n",
-        "import tensorflow.compat.v1 as tf\n",
-        "tf.disable_v2_behavior()\n",
-        "```\n",
-        "\n",
-        "However, this does not let you take advantage of many of the improvements made in TensorFlow 2.0. This guide will help you upgrade your code, making it simpler, more performant, and easier to maintain."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "GUp_x0bOgaac"
-      },
-      "source": [
-        "## Automatic conversion script\n",
-        "\n",
-        "The first step, before attempting to implement the changes described in this doc,  is to try running the [upgrade script](./upgrade.md).\n",
-        "\n",
-        "This will do an initial pass at upgrading your code to TensorFlow 2.0. But it can't make your code idiomatic to  2.0. Your code may still make use of `tf.compat.v1` endpoints to access placeholders, sessions, collections, and other 1.x-style functionality."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "0w5LiSYsy1mh"
-      },
-      "source": [
-        "## Top-level behavioral changes\n",
-        "\n",
-        "If your code works in TensorFlow 2.0 using `tf.compat.v1.disable_v2_behavior()`, there are still global behavioral changes you may need to address. The major changes are:"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "1y-W0Mz_zB6Y"
-      },
-      "source": [
-        "* *Eager execution, `v1.enable_eager_execution()`* : Any code that implicitly uses a `tf.Graph` will fail. Be sure to wrap this code in a `with tf.Graph().as_default()` context. \n",
-        "    \n",
-        "* *Resource variables, `v1.enable_resource_variables()`*: Some code may depends on non-deterministic behaviors enabled by TF reference variables. \n",
-        "Resource variables are locked while being written to, and so provide more intuitive consistency guarantees.\n",
-        "\n",
-        "  * This may change behavior in edge cases.\n",
-        "  * This may create extra copies and can have higher memory usage.\n",
-        "  * This can be disabled by passing `use_resource=False` to the `tf.Variable` constructor.\n",
-        "\n",
-        "* *Tensor shapes, `v1.enable_v2_tensorshape()`*: TF 2.0 simplifies the behavior of tensor shapes. Instead of `t.shape[0].value` you can say `t.shape[0]`. These changes should be small, and it makes sense to fix them right away. See [TensorShape](#tensorshape) for examples.\n",
-        "\n",
-        "* *Control flow, `v1.enable_control_flow_v2()`*: The TF 2.0 control flow implementation has been simplified, and so produces different graph representations. Please [file bugs](https://github.com/tensorflow/tensorflow/issues) for any issues."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "_Ni9zLLvwcOR"
-      },
-      "source": [
-        "## Make the code 2.0-native\n",
-        "\n",
-        "\n",
-        "This guide will walk through several examples of converting TensorFlow 1.x code to TensorFlow 2.0. These changes will let your code take advantage of performance optimizations and simplified API calls.\n",
-        "\n",
-        "In each case, the pattern is:"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "uP0O8Pc45LNs"
-      },
-      "source": [
-        "### 1. Replace `v1.Session.run` calls\n",
-        "\n",
-        "Every `v1.Session.run` call should be replaced by a Python function.\n",
-        "\n",
-        "* The `feed_dict` and `v1.placeholder`s become function arguments.\n",
-        "* The `fetches` become the function's return value. \n",
-        "* During conversion eager execution allows easy debugging with standard Python tools like `pdb`.\n",
-        "\n",
-        "After that add a `tf.function` decorator to make it run efficiently in graph. See the [Autograph Guide](function.ipynb) for more on how this works.\n",
-        "\n",
-        "Note that:\n",
-        "\n",
-        "* Unlike `v1.Session.run` a `tf.function` has a fixed return signature, and always returns all outputs. If this causes performance problems, create two separate functions.\n",
-        "\n",
-        "* There is no need for a `tf.control_dependencies` or similar operations: A `tf.function` behaves as if it were run in the order written. `tf.Variable` assignments and `tf.assert`s, for example, are executed automatically.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "jlBOqROL5NmN"
-      },
-      "source": [
-        "### 2. Use Python objects to track variables and losses\n",
-        "\n",
-        "All name-based variable tracking is strongly discouraged in TF 2.0. Use Python objects to to track variables.\n",
-        "\n",
-        "Use `tf.Variable` instead of `v1.get_variable`.\n",
-        "\n",
-        "Every `v1.variable_scope` should be converted to a Python object. Typically this will be one of:\n",
-        "\n",
-        "* `tf.keras.layers.Layer`\n",
-        "* `tf.keras.Model`\n",
-        "* `tf.Module`\n",
-        "\n",
-        "If you need to aggregate lists of variables (like `tf.Graph.get_collection(tf.GraphKeys.VARIABLES)`), use the `.variables` and `.trainable_variables` attributes of the `Layer` and `Model` objects.\n",
-        "\n",
-        "These `Layer` and `Model` classes implement several other properties that remove the need for global collections. Their `.losses` property can be a replacement for using the `tf.GraphKeys.LOSSES` collection.\n",
-        "\n",
-        "See the [keras guides](keras.ipynb) for details.\n",
-        "\n",
-        "Warning: Many `tf.compat.v1` symbols  use the global collections implicitly.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "rGFhBzoF5FIq"
-      },
-      "source": [
-        "### 3. Upgrade your training loops\n",
-        "\n",
-        "Use the highest level API that works for your use case.  Prefer `tf.keras.Model.fit` over building your own training loops.\n",
-        "\n",
-        "These high level functions manage a lot of the low-level details that might be easy to miss if you write your own training loop. For example, they automatically collect the regularization losses, and set the `training=True` argument when calling the model.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "oaY37_6L4la5"
-      },
-      "source": [
-        "\n",
-        "### 4. Upgrade your data input pipelines\n",
-        "\n",
-        "Use `tf.data` datasets for data input. These objects are efficient, expressive, and integrate well with tensorflow.\n",
-        "\n",
-        "They can be passed directly to the `tf.keras.Model.fit` method.\n",
-        "\n",
-        "```\n",
-        "model.fit(dataset, epochs=5)\n",
-        "```\n",
-        "\n",
-        "They can be iterated over directly standard Python:\n",
-        "\n",
-        "```\n",
-        "for example_batch, label_batch in dataset:\n",
-        "    break\n",
-        "```\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "_Mwsd0SK4oIs"
-      },
-      "source": [
-        "#### 5. Migrate off `compat.v1` symbols \n",
-        "\n",
-        "The `tf.compat.v1` module contains the complete TensorFlow 1.x API, with its original semantics.\n",
-        "\n",
-        "The [TF2 upgrade script](upgrade.ipynb)  will convert symbols to their 2.0 equivalents if such a conversion is safe, i.e., if it can determine that the behavior of the 2.0 version is exactly equivalent (for instance, it will rename `v1.arg_max` to `tf.argmax`, since those are the same function). \n",
-        "\n",
-        "After the upgrade script is done with a piece of code, it is likely there are many mentions of `compat.v1`. It is worth going through the code and converting these manually to the 2.0 equivalent (it should be mentioned in the log if there is one)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "X_ilfTGJ4Yml"
-      },
-      "source": [
-        "## Converting models\n",
-        "\n",
-        "### Setup"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "bad2N-Z115W1"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "\n",
-        "import tensorflow_datasets as tfds"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "FB99sqHX2Q5m"
-      },
-      "source": [
-        "### Low-level variables \u0026 operator execution\n",
-        "\n",
-        "Examples of low-level API use include:\n",
-        "\n",
-        "* using variable scopes to control reuse\n",
-        "* creating variables with `v1.get_variable`.\n",
-        "* accessing collections explicitly\n",
-        "* accessing collections implicitly with methods like :\n",
-        "\n",
-        "  * `v1.global_variables`\n",
-        "  * `v1.losses.get_regularization_loss`\n",
-        "\n",
-        "* using `v1.placeholder` to set up graph inputs\n",
-        "* executing graphs with `Session.run`\n",
-        "* initializing variables manually\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "e582IjyF2eje"
-      },
-      "source": [
-        "#### Before converting\n",
-        "\n",
-        "Here is what these patterns may look like in code using TensorFlow 1.x.\n",
-        "\n",
-        "```python\n",
-        "in_a = tf.placeholder(dtype=tf.float32, shape=(2))\n",
-        "in_b = tf.placeholder(dtype=tf.float32, shape=(2))\n",
-        "\n",
-        "def forward(x):\n",
-        "  with tf.variable_scope(\"matmul\", reuse=tf.AUTO_REUSE):\n",
-        "    W = tf.get_variable(\"W\", initializer=tf.ones(shape=(2,2)),\n",
-        "                        regularizer=tf.contrib.layers.l2_regularizer(0.04))\n",
-        "    b = tf.get_variable(\"b\", initializer=tf.zeros(shape=(2)))\n",
-        "    return W * x + b\n",
-        "\n",
-        "out_a = forward(in_a)\n",
-        "out_b = forward(in_b)\n",
-        "\n",
-        "reg_loss = tf.losses.get_regularization_loss(scope=\"matmul\")\n",
-        "\n",
-        "with tf.Session() as sess:\n",
-        "  sess.run(tf.global_variables_initializer())\n",
-        "  outs = sess.run([out_a, out_b, reg_loss],\n",
-        "      \t        feed_dict={in_a: [1, 0], in_b: [0, 1]})\n",
-        "\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "QARwz4Xd2lc2"
-      },
-      "source": [
-        "#### After converting"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "x0AVzBFRBPcU"
-      },
-      "source": [
-        "In the converted code:\n",
-        "\n",
-        "* The variables are local Python objects.\n",
-        "* The `forward` function still defines the calculation.\n",
-        "* The `Session.run` call is replaced with a call to `forward`\n",
-        "* The optional `tf.function` decorator can be added for performance.\n",
-        "* The regularizations are calculated manually, without referring to any global collection.\n",
-        "* **No sessions or placeholders.**"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "lXEZoLMP2cWJ"
-      },
-      "outputs": [],
-      "source": [
-        "W = tf.Variable(tf.ones(shape=(2,2)), name=\"W\")\n",
-        "b = tf.Variable(tf.zeros(shape=(2)), name=\"b\")\n",
-        "\n",
-        "@tf.function\n",
-        "def forward(x):\n",
-        "  return W * x + b\n",
-        "\n",
-        "out_a = forward([1,0])\n",
-        "print(out_a)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "YmE96A_1jZTg"
-      },
-      "outputs": [],
-      "source": [
-        "out_b = forward([0,1])\n",
-        "\n",
-        "regularizer = tf.keras.regularizers.l2(0.04)\n",
-        "reg_loss = regularizer(W)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ycDxY9nL268-"
-      },
-      "source": [
-        "### Models based on `tf.layers`"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "K-bIk7wL48U7"
-      },
-      "source": [
-        "The `v1.layers` module is used to contain layer-functions that relied on `v1.variable_scope` to define and reuse variables."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "8I_qKpT73KyM"
-      },
-      "source": [
-        "#### Before converting\n",
-        "```python\n",
-        "def model(x, training, scope='model'):\n",
-        "  with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):\n",
-        "    x = tf.layers.conv2d(x, 32, 3, activation=tf.nn.relu,\n",
-        "          kernel_regularizer=tf.contrib.layers.l2_regularizer(0.04))\n",
-        "    x = tf.layers.max_pooling2d(x, (2, 2), 1)\n",
-        "    x = tf.layers.flatten(x)\n",
-        "    x = tf.layers.dropout(x, 0.1, training=training)\n",
-        "    x = tf.layers.dense(x, 64, activation=tf.nn.relu)\n",
-        "    x = tf.layers.batch_normalization(x, training=training)\n",
-        "    x = tf.layers.dense(x, 10, activation=tf.nn.softmax)\n",
-        "    return x\n",
-        "\n",
-        "train_out = model(train_data, training=True)\n",
-        "test_out = model(test_data, training=False)\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "b8_Ii7CQ3fK-"
-      },
-      "source": [
-        "#### After converting"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "BsAseSMfB9XN"
-      },
-      "source": [
-        "* The simple stack of layers fits neatly into `tf.keras.Sequential`. (For more complex models see [custom layers and models](keras/custom_layers_and_models.ipynb), and [the functional API](keras/functional.ipynb).)\n",
-        "* The model tracks the variables, and regularization losses.\n",
-        "* The conversion was one-to-one because there is a direct mapping from `v1.layers` to `tf.keras.layers`.\n",
-        "\n",
-        "Most arguments stayed the same. But notice the differences:\n",
-        "\n",
-        "* The `training` argument is passed to each layer by the model when it runs.\n",
-        "* The first argument to the original `model` function (the input `x`) is gone. This is because object layers separate building the model from calling the model.\n",
-        "\n",
-        "\n",
-        "Also note that:\n",
-        "\n",
-        "* If you were using regularizers of initializers from  `tf.contrib`, these have more argument changes than others.\n",
-        "* The code no longer writes to collections, so functions like `v1.losses.get_regularization_loss` will no longer return these values, potentially breaking your training loops."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "DLAPORrN3lct"
-      },
-      "outputs": [],
-      "source": [
-        "model = tf.keras.Sequential([\n",
-        "    tf.keras.layers.Conv2D(32, 3, activation='relu',\n",
-        "                           kernel_regularizer=tf.keras.regularizers.l2(0.04),\n",
-        "                           input_shape=(28, 28, 1)),\n",
-        "    tf.keras.layers.MaxPooling2D(),\n",
-        "    tf.keras.layers.Flatten(),\n",
-        "    tf.keras.layers.Dropout(0.1),\n",
-        "    tf.keras.layers.Dense(64, activation='relu'),\n",
-        "    tf.keras.layers.BatchNormalization(),\n",
-        "    tf.keras.layers.Dense(10, activation='softmax')\n",
-        "])\n",
-        "\n",
-        "train_data = tf.ones(shape=(1, 28, 28, 1))\n",
-        "test_data = tf.ones(shape=(1, 28, 28, 1))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "6nWh6IXvkMKv"
-      },
-      "outputs": [],
-      "source": [
-        "train_out = model(train_data, training=True)\n",
-        "print(train_out)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "YnAdIDLlj3go"
-      },
-      "outputs": [],
-      "source": [
-        "test_out = model(test_data, training=False)\n",
-        "print(test_out)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "sAgqwCJBMx_x"
-      },
-      "outputs": [],
-      "source": [
-        "# Here are all the trainable variables.\n",
-        "len(model.trainable_variables)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "uX6knaYMNM8p"
-      },
-      "outputs": [],
-      "source": [
-        "# Here is the regularization loss.\n",
-        "model.losses"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "9moqw5E_4Cwl"
-      },
-      "source": [
-        "### Mixed variables \u0026 `v1.layers`\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "80DEsImmq6VX"
-      },
-      "source": [
-        "Existing code often mixes lower-level TF 1.x variables and operations with higher-level `v1.layers`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "oZe9L6RR4OcP"
-      },
-      "source": [
-        "#### Before converting\n",
-        "```python\n",
-        "def model(x, training, scope='model'):\n",
-        "  with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):\n",
-        "    W = tf.get_variable(\n",
-        "      \"W\", dtype=tf.float32,\n",
-        "      initializer=tf.ones(shape=x.shape),\n",
-        "      regularizer=tf.contrib.layers.l2_regularizer(0.04),\n",
-        "      trainable=True)\n",
-        "    if training:\n",
-        "      x = x + W\n",
-        "    else:\n",
-        "      x = x + W * 0.5\n",
-        "    x = tf.layers.conv2d(x, 32, 3, activation=tf.nn.relu)\n",
-        "    x = tf.layers.max_pooling2d(x, (2, 2), 1)\n",
-        "    x = tf.layers.flatten(x)\n",
-        "    return x\n",
-        "\n",
-        "train_out = model(train_data, training=True)\n",
-        "test_out = model(test_data, training=False)\n",
-        "```"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "y6ORX7cD4TkD"
-      },
-      "source": [
-        "#### After converting"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2BaRwog5CBpz"
-      },
-      "source": [
-        "To convert this code, follow the pattern of mapping layers to layers as in the previous example.\n",
-        "\n",
-        "A `v1.variable_scope` is effectively a layer of its own. So rewrite it as a `tf.keras.layers.Layer`. See [the guide](keras/custom_layers_and_models.ipynb) for details.\n",
-        "\n",
-        "The general pattern is:\n",
-        "\n",
-        "* Collect layer parameters in `__init__`.\n",
-        "* Build the variables in `build`.\n",
-        "* Execute the calculations in `call`, and return the result.\n",
-        "\n",
-        "The `v1.variable_scope` is essentially a layer of its own. So rewrite it as a `tf.keras.layers.Layer`. See [the guide](keras/custom_layers_and_models.ipynb) for details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "YcCAjNuP4NVh"
-      },
-      "outputs": [],
-      "source": [
-        "# Create a custom layer for part of the model\n",
-        "class CustomLayer(tf.keras.layers.Layer):\n",
-        "  def __init__(self, *args, **kwargs):\n",
-        "    super(CustomLayer, self).__init__(*args, **kwargs)\n",
-        "\n",
-        "  def build(self, input_shape):\n",
-        "    self.w = self.add_weight(\n",
-        "        shape=input_shape[1:],\n",
-        "        dtype=tf.float32,\n",
-        "        initializer=tf.keras.initializers.ones(),\n",
-        "        regularizer=tf.keras.regularizers.l2(0.02),\n",
-        "        trainable=True)\n",
-        "\n",
-        "  # Call method will sometimes get used in graph mode,\n",
-        "  # training will get turned into a tensor\n",
-        "  @tf.function\n",
-        "  def call(self, inputs, training=None):\n",
-        "    if training:\n",
-        "      return inputs + self.w\n",
-        "    else:\n",
-        "      return inputs + self.w * 0.5"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "dR_QO6_wBgMm"
-      },
-      "outputs": [],
-      "source": [
-        "custom_layer = CustomLayer()\n",
-        "print(custom_layer([1]).numpy())\n",
-        "print(custom_layer([1], training=True).numpy())"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "VzqaIf4E42oY"
-      },
-      "outputs": [],
-      "source": [
-        "train_data = tf.ones(shape=(1, 28, 28, 1))\n",
-        "test_data = tf.ones(shape=(1, 28, 28, 1))\n",
-        "\n",
-        "# Build the model including the custom layer\n",
-        "model = tf.keras.Sequential([\n",
-        "    CustomLayer(input_shape=(28, 28, 1)),\n",
-        "    tf.keras.layers.Conv2D(32, 3, activation='relu'),\n",
-        "    tf.keras.layers.MaxPooling2D(),\n",
-        "    tf.keras.layers.Flatten(),\n",
-        "])\n",
-        "\n",
-        "train_out = model(train_data, training=True)\n",
-        "test_out = model(test_data, training=False)\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "dS5ed_jjOkvh"
-      },
-      "source": [
-        "Some things to note:\n",
-        "\n",
-        "* Subclassed Keras models \u0026 layers need to run in both v1 graphs (no automatic control dependencies) and in eager mode\n",
-        "  * Wrap the `call()` in a `tf.function()` to get autograph and automatic control dependencies\n",
-        "\n",
-        "* Don't forget to accept a `training` argument to `call`.\n",
-        "    * Sometimes it is a `tf.Tensor`\n",
-        "    * Sometimes it is a Python boolean.\n",
-        "\n",
-        "* Create model variables in constructor or `Model.build` using `self.add_weight()`.\n",
-        "  * In `Model.build` you have access to the input shape, so can create weights with matching shape.\n",
-        "  * Using `tf.keras.layers.Layer.add_weight` allows Keras to track variables and regularization losses.\n",
-        "\n",
-        "* Don't keep `tf.Tensors` in your objects.\n",
-        "  * They might get created either in a `tf.function` or in the eager context, and these tensors behave differently.\n",
-        "  * Use `tf.Variable`s for state, they are always usable from both contexts\n",
-        "  * `tf.Tensors` are only for intermediate values."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ulaB1ymO4pw5"
-      },
-      "source": [
-        "### A note on Slim \u0026 contrib.layers\n",
-        "\n",
-        "A large amount of older TensorFlow 1.x code uses the [Slim](https://ai.googleblog.com/2016/08/tf-slim-high-level-library-to-define.html) library, which was packaged with TensorFlow 1.x as `tf.contrib.layers`. As a `contrib` module, this is no longer available in TensorFlow 2.0, even in `tf.compat.v1`. Converting code using Slim to TF 2.0 is more involved than converting repositories that use `v1.layers`. In fact, it may make sense to convert your Slim code to `v1.layers` first, then convert to Keras.\n",
-        "\n",
-        "* Remove `arg_scopes`, all args need to be explicit\n",
-        "* If you use them, split `normalizer_fn` and `activation_fn` into their own layers\n",
-        "* Separable conv layers map to one or more different Keras layers (depthwise, pointwise, and separable Keras layers)\n",
-        "* Slim and `v1.layers` have different arg names \u0026 default values\n",
-        "* Some args have different scales\n",
-        "* If you use Slim pre-trained models, try out Keras's pre-traimed models from `tf.keras.applications` or [TF Hub](https://tfhub.dev/s?q=slim%20tf2)'s TF2 SavedModels exported from the original Slim code.\n",
-        "\n",
-        "Some `tf.contrib` layers might not have been moved to core TensorFlow but have instead been moved to the [TF add-ons package](https://github.com/tensorflow/addons).\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "1w72KrXm4yZR"
-      },
-      "source": [
-        "## Training"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "56PQxTgy2bpI"
-      },
-      "source": [
-        "There are many ways to feed data to a `tf.keras` model. They will accept Python generators and Numpy arrays as input.\n",
-        "\n",
-        "The recommended way to feed data to a model is to use the `tf.data` package, which contains a collection of high performance classes for manipulating data.\n",
-        "\n",
-        "If you are still using `tf.queue`, these are now only supported as data-structures, not as input pipelines."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "m6htasZ7iBB4"
-      },
-      "source": [
-        "### Using Datasets"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "loTPH2Pz4_Oj"
-      },
-      "source": [
-        "The [TensorFlow Datasets](https://tensorflow.org/datasets) package (`tfds`) contains utilities for loading predefined datasets as `tf.data.Dataset` objects.\n",
-        "\n",
-        "For this example, load the MNISTdataset, using `tfds`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "BMgxaLH74_s-"
-      },
-      "outputs": [],
-      "source": [
-        "datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n",
-        "mnist_train, mnist_test = datasets['train'], datasets['test']"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "hPJhEuvj5VfR"
-      },
-      "source": [
-        "Then prepare the data for training:\n",
-        "\n",
-        "  * Re-scale each image.\n",
-        "  * Shuffle the order of the examples.\n",
-        "  * Collect batches of images and labels.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "StBRHtJM2S7o"
-      },
-      "outputs": [],
-      "source": [
-        "BUFFER_SIZE = 10 # Use a much larger value for real code.\n",
-        "BATCH_SIZE = 64\n",
-        "NUM_EPOCHS = 5\n",
-        "\n",
-        "\n",
-        "def scale(image, label):\n",
-        "  image = tf.cast(image, tf.float32)\n",
-        "  image /= 255\n",
-        "\n",
-        "  return image, label"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "SKq14zKKFAdv"
-      },
-      "source": [
-        " To keep the example short, trim the dataset to only return 5 batches:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "_J-o4YjG2mkM"
-      },
-      "outputs": [],
-      "source": [
-        "train_data = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
-        "test_data = mnist_test.map(scale).batch(BATCH_SIZE)\n",
-        "\n",
-        "STEPS_PER_EPOCH = 5\n",
-        "\n",
-        "train_data = train_data.take(STEPS_PER_EPOCH)\n",
-        "test_data = test_data.take(STEPS_PER_EPOCH)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "XEqdkH54VM6c"
-      },
-      "outputs": [],
-      "source": [
-        "image_batch, label_batch = next(iter(train_data))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "mSev7vZC5GJB"
-      },
-      "source": [
-        "### Use Keras training loops\n",
-        "\n",
-        "If you don't need low level control of your training process, using Keras's built-in `fit`, `evaluate`, and `predict` methods is recommended. These methods provide a uniform interface to train the model regardless of the implementation (sequential,  functional, or sub-classed).\n",
-        "\n",
-        "The advantages of these methods include:\n",
-        "\n",
-        "* They accept Numpy arrays, Python generators and, `tf.data.Datasets`\n",
-        "* They apply regularization, and activation losses automatically.\n",
-        "* They support `tf.distribute` [for multi-device training](distributed_training.ipynb).\n",
-        "* They support arbitrary callables as losses and metrics.\n",
-        "* They support callbacks like `tf.keras.callbacks.TensorBoard`, and custom callbacks.\n",
-        "* They are performant, automatically using TensorFlow graphs.\n",
-        "\n",
-        "Here is an example of training a model using a `Dataset`. (For details on how this works see [tutorials](../tutorials).)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "uzHFCzd45Rae"
-      },
-      "outputs": [],
-      "source": [
-        "model = tf.keras.Sequential([\n",
-        "    tf.keras.layers.Conv2D(32, 3, activation='relu',\n",
-        "                           kernel_regularizer=tf.keras.regularizers.l2(0.02),\n",
-        "                           input_shape=(28, 28, 1)),\n",
-        "    tf.keras.layers.MaxPooling2D(),\n",
-        "    tf.keras.layers.Flatten(),\n",
-        "    tf.keras.layers.Dropout(0.1),\n",
-        "    tf.keras.layers.Dense(64, activation='relu'),\n",
-        "    tf.keras.layers.BatchNormalization(),\n",
-        "    tf.keras.layers.Dense(10, activation='softmax')\n",
-        "])\n",
-        "\n",
-        "# Model is the full model w/o custom layers\n",
-        "model.compile(optimizer='adam',\n",
-        "              loss='sparse_categorical_crossentropy',\n",
-        "              metrics=['accuracy'])\n",
-        "\n",
-        "model.fit(train_data, epochs=NUM_EPOCHS)\n",
-        "loss, acc = model.evaluate(test_data)\n",
-        "\n",
-        "print(\"Loss {}, Accuracy {}\".format(loss, acc))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "akpeOb09YBhq"
-      },
-      "source": [
-        "### Write your own loop\n",
-        "\n",
-        "If the Keras model's training step works for you, but you need more control outside that step, consider using the `tf.keras.Model.train_on_batch` method,  in your own data-iteration loop.\n",
-        "\n",
-        "Remember: Many things can be implemented as a `tf.keras.callbacks.Callback`.\n",
-        "\n",
-        "This method has many of the advantages of the methods mentioned in the previous section, but gives the user control of the outer loop.\n",
-        "\n",
-        "You can also use `tf.keras.Model.test_on_batch` or `tf.keras.Model.evaluate` to check performance during training.\n",
-        "\n",
-        "Note: `train_on_batch` and `test_on_batch`, by default return the loss and metrics for the single batch. If you pass `reset_metrics=False` they return accumulated metrics and you must remember to appropriately reset the metric accumulators. Also remember that some metrics like `AUC` require `reset_metrics=False` to be calculated correctly.\n",
-        "\n",
-        "To continue training the above model:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "eXr4CyJMtJJ6"
-      },
-      "outputs": [],
-      "source": [
-        "# Model is the full model w/o custom layers\n",
-        "model.compile(optimizer='adam',\n",
-        "              loss='sparse_categorical_crossentropy',\n",
-        "              metrics=['accuracy'])\n",
-        "\n",
-        "metrics_names = model.metrics_names\n",
-        "\n",
-        "for epoch in range(NUM_EPOCHS):\n",
-        "  #Reset the metric accumulators\n",
-        "  model.reset_metrics()\n",
-        "\n",
-        "  for image_batch, label_batch in train_data:\n",
-        "    result = model.train_on_batch(image_batch, label_batch)\n",
-        "    print(\"train: \",\n",
-        "          \"{}: {:.3f}\".format(metrics_names[0], result[0]),\n",
-        "          \"{}: {:.3f}\".format(metrics_names[1], result[1]))\n",
-        "  for image_batch, label_batch in test_data:\n",
-        "    result = model.test_on_batch(image_batch, label_batch,\n",
-        "                                 # return accumulated metrics\n",
-        "                                 reset_metrics=False)\n",
-        "  print(\"\\neval: \",\n",
-        "        \"{}: {:.3f}\".format(metrics_names[0], result[0]),\n",
-        "        \"{}: {:.3f}\".format(metrics_names[1], result[1]))\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LQTaHTuK5S5A"
-      },
-      "source": [
-        "\u003ca name=\"custom_loop\"\u003e\u003c/a\u003e\n",
-        "\n",
-        "### Customize the training step\n",
-        "\n",
-        "If you need more flexibility and control, you can have it by implementing your own training loop. There are three steps:\n",
-        "\n",
-        "1. Iterate over a Python generator or `tf.data.Dataset` to get batches of examples.\n",
-        "2. Use `tf.GradientTape` to collect gradients.\n",
-        "3. Use one of the `tf.keras.optimizers` to apply weight updates to the model's variables.\n",
-        "\n",
-        "Remember:\n",
-        "\n",
-        "* Always include a `training` argument on the `call` method of subclassed layers and models.\n",
-        "* Make sure to call the model with the `training` argument set correctly.\n",
-        "* Depending on usage, model variables may not exist until the model is run on a batch of data.\n",
-        "* You need to manually handle things like regularization losses for the model.\n",
-        "\n",
-        "Note the simplifications relative to v1:\n",
-        "\n",
-        "* There is no need to run variable initializers. Variables are initialized on creation.\n",
-        "* There is no need to add manual control dependencies. Even in `tf.function` operations act as in eager mode."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "gQooejfYlQeF"
-      },
-      "outputs": [],
-      "source": [
-        "model = tf.keras.Sequential([\n",
-        "    tf.keras.layers.Conv2D(32, 3, activation='relu',\n",
-        "                           kernel_regularizer=tf.keras.regularizers.l2(0.02),\n",
-        "                           input_shape=(28, 28, 1)),\n",
-        "    tf.keras.layers.MaxPooling2D(),\n",
-        "    tf.keras.layers.Flatten(),\n",
-        "    tf.keras.layers.Dropout(0.1),\n",
-        "    tf.keras.layers.Dense(64, activation='relu'),\n",
-        "    tf.keras.layers.BatchNormalization(),\n",
-        "    tf.keras.layers.Dense(10, activation='softmax')\n",
-        "])\n",
-        "\n",
-        "optimizer = tf.keras.optimizers.Adam(0.001)\n",
-        "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n",
-        "\n",
-        "@tf.function\n",
-        "def train_step(inputs, labels):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    predictions = model(inputs, training=True)\n",
-        "    regularization_loss = tf.math.add_n(model.losses)\n",
-        "    pred_loss = loss_fn(labels, predictions)\n",
-        "    total_loss = pred_loss + regularization_loss\n",
-        "\n",
-        "  gradients = tape.gradient(total_loss, model.trainable_variables)\n",
-        "  optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
-        "\n",
-        "for epoch in range(NUM_EPOCHS):\n",
-        "  for inputs, labels in train_data:\n",
-        "    train_step(inputs, labels)\n",
-        "  print(\"Finished epoch\", epoch)\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kS7WW5Z75ve3"
-      },
-      "source": [
-        "### New-style metrics and losses\n",
-        "\n",
-        "In TensorFlow 2.0, metrics and losses are objects. These work both eagerly and in `tf.function`s. \n",
-        "\n",
-        "A loss object is callable, and expects the (y_true, y_pred) as arguments:\n",
-        "\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "C5_TVrBlbBcy"
-      },
-      "outputs": [],
-      "source": [
-        "cce = tf.losses.CategoricalCrossentropy(from_logits=True)\n",
-        "cce([[1, 0]], [[-1.0,3.0]]).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "JFDc1v0XbAyB"
-      },
-      "source": [
-        "A metric object has the following methods:\n",
-        "\n",
-        "* `Metric.update_state()` — add new observations\n",
-        "* `Metric.result()` —get the current result of the metric, given the observed values\n",
-        "* `Metric.reset_states()` — clear all observations.\n",
-        "\n",
-        "The object itself is callable. Calling updates the state with new observations, as with `update_state`, and returns the new result of the metric.\n",
-        "\n",
-        "You don't have to manually initialize a metric's variables, and because TensorFlow 2.0 has automatic control dependencies, you don't need to worry about those either.\n",
-        "\n",
-        "The code below uses a metric to keep track of the mean loss observed within a custom training loop."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "HAbA0fKW58CH"
-      },
-      "outputs": [],
-      "source": [
-        "# Create the metrics\n",
-        "loss_metric = tf.keras.metrics.Mean(name='train_loss')\n",
-        "accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n",
-        "\n",
-        "@tf.function\n",
-        "def train_step(inputs, labels):\n",
-        "  with tf.GradientTape() as tape:\n",
-        "    predictions = model(inputs, training=True)\n",
-        "    regularization_loss = tf.math.add_n(model.losses)\n",
-        "    pred_loss = loss_fn(labels, predictions)\n",
-        "    total_loss = pred_loss + regularization_loss\n",
-        "\n",
-        "  gradients = tape.gradient(total_loss, model.trainable_variables)\n",
-        "  optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
-        "  # Update the metrics\n",
-        "  loss_metric.update_state(total_loss)\n",
-        "  accuracy_metric.update_state(labels, predictions)\n",
-        "\n",
-        "\n",
-        "for epoch in range(NUM_EPOCHS):\n",
-        "  # Reset the metrics\n",
-        "  loss_metric.reset_states()\n",
-        "  accuracy_metric.reset_states()\n",
-        "\n",
-        "  for inputs, labels in train_data:\n",
-        "    train_step(inputs, labels)\n",
-        "  # Get the metric results\n",
-        "  mean_loss = loss_metric.result()\n",
-        "  mean_accuracy = accuracy_metric.result()\n",
-        "\n",
-        "  print('Epoch: ', epoch)\n",
-        "  print('  loss:     {:.3f}'.format(mean_loss))\n",
-        "  print('  accuracy: {:.3f}'.format(mean_accuracy))\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Hf718XCgDAGJ"
-      },
-      "source": [
-        "### Keras optimizers"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "A6El-NxAQ8aF"
-      },
-      "source": [
-        "The optimizers in `v1.train`, like `v1.train.AdamOptimizer` and `v1.train.GradientDescentOptimizer`, have equivalents in `tf.keras.optimizers`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "qgP89WdSDQx-"
-      },
-      "source": [
-        "#### Convert `v1.train` to `keras.optimizers`\n",
-        "\n",
-        "Here are things to keep in mind when converting your optimizers:\n",
-        "\n",
-        "* Upgrading your optimizers [may make old checkpoints incompatible](#checkpoints).\n",
-        "* All epsilons now default to `1e-7` instead of `1e-8` (which is negligible in most use cases).\n",
-        "* `v1.train.GradientDescentOptimizer` can be directly replaced by `tf.keras.optimizers.SGD`. \n",
-        "* `v1.train.MomentumOptimizer` can be directly replaced by the `SGD` optimizer using the momentum argument: `tf.keras.optimizers.SGD(..., momentum=...)`.\n",
-        "* `v1.train.AdamOptimizer` can be converted to use `tf.keras.optimizers.Adam`. The `beta1` and `beta2` arguments have been renamed to `beta_1` and `beta_2`.\n",
-        "* `v1.train.RMSPropOptimizer` can be converted to `tf.keras.optimizers.RMSprop`. The `decay` argument has been renamed to `rho`.\n",
-        "* `v1.train.AdadeltaOptimizer` can be converted directly to `tf.keras.optimizers.Adadelta`.\n",
-        "* `tf.train.AdagradOptimizer` can be converted directly to `tf.keras.optimizers.Adagrad`.\n",
-        "* `tf.train.FtrlOptimizer` can be converted directly to `tf.keras.optimizers.Ftrl`. The `accum_name` and `linear_name` arguments have been removed.\n",
-        "* The `tf.contrib.AdamaxOptimizer` and `tf.contrib.NadamOptimizer`, can be converted directly to `tf.keras.optimizers.Adamax` and `tf.keras.optimizers.Nadam`. The `beta1`, and `beta2` arguments have been renamed to `beta_1` and `beta_2`.\n",
-        "\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Ef60-wJ2bR3l"
-      },
-      "source": [
-        "\n",
-        "#### New defaults for some `tf.keras.optimizers`\n",
-        "\u003ca id=\"keras_optimizer_lr\"\u003e\u003c/a\u003e\n",
-        "\n",
-        "Warning: If you see a change in convergence behavior for your models, check the default learning rates.\n",
-        "\n",
-        "There are no changes for `optimizers.SGD`, `optimizers.Adam`, or `optimizers.RMSprop`.\n",
-        "\n",
-        "The following default learning rates have changed:\n",
-        "\n",
-        "* `optimizers.Adagrad` from 0.01 to 0.001\n",
-        "* `optimizers.Adadelta` from 1.0 to 0.001\n",
-        "* `optimizers.Adamax` from 0.002 to 0.001\n",
-        "* `optimizers.Nadam` from 0.002 to 0.001"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "5Cf1ks48Q3uc"
-      },
-      "source": [
-        "### TensorBoard"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "0tx7FyM_RHwJ"
-      },
-      "source": [
-        "TensorFlow 2.0 includes significant changes to the `tf.summary` API used to write summary data for visualization in TensorBoard.  For a general introduction to the new tf.summary, there are [several tutorials available](https://www.tensorflow.org/tensorboard/r2/get_started) that use the TF 2.0 API. This includes a [TensorBoard TF2.0 Migration Guide](https://www.tensorflow.org/tensorboard/r2/migrate_tf2)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "JmMLBKs66DeA"
-      },
-      "source": [
-        "## Saving \u0026 Loading\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "5_QKn3Kl6TUu"
-      },
-      "source": [
-        "\u003ca id=\"checkpoints\"\u003e\u003c/a\u003e\n",
-        "### Checkpoint compatibility\n",
-        "\n",
-        "TensorFlow 2.0 uses [object-based checkpoints](checkpoint.ipynb).\n",
-        "\n",
-        "Old-style name-based checkpoints can still be loaded, if you're careful.\n",
-        "The code conversion process may result in variable name changes, but there are workarounds.\n",
-        "\n",
-        "The simplest approach it to line up the names of the new model with the names in the checkpoint:\n",
-        "\n",
-        "* Variables still all have a `name` argument you can set.\n",
-        "* Keras models also take a `name` argument as which they set as the prefix for their variables.\n",
-        "* The `v1.name_scope` function can be used to set variable name prefixes. This is very different from `tf.variable_scope`. It only affects names, and  doesn't track variables \u0026 reuse.\n",
-        "\n",
-        "If that does not work for your use-case, try the `v1.train.init_from_checkpoint` function. It takes an `assignment_map` argument, which specifies the mapping from old names to new names.\n",
-        "\n",
-        "Note: Unlike object based checkpoints, which can [defer loading](checkpoint.ipynb#loading_mechanics), name-based checkpoints require that all variables be built when the function is called. Some models defer building variables until you call `build` or run the model on a batch of data.\n",
-        "\n",
-        "The [TensorFlow Estimator repository](https://github.com/tensorflow/estimator/blob/master/tensorflow_estimator/python/estimator/tools/checkpoint_converter.py) includes a [conversion tool](#checkpoint_converter) to upgrade the checkpoints for premade estimators from TensorFlow 1.X to 2.0. It may serve as an example of how to build a tool fr a similar use-case."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "_ONjobDD6Uur"
-      },
-      "source": [
-        "### Saved models compatibility\n",
-        "\n",
-        "There are no significant compatibility concerns for saved models.\n",
-        "\n",
-        "* TensorFlow 1.x saved_models work in TensorFlow 2.0.\n",
-        "* TensorFlow 2.0 saved_models even load work in TensorFlow 1.x if all the ops are supported."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-KpXprsf1tyb"
-      },
-      "source": [
-        "### A Graph.pb or Graph.pbtxt "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Tz4eAAGY19MM"
-      },
-      "source": [
-        "There is no straightforward way to upgrade a raw `Graph.pb` file to TensorFlow 2.0. Your best bet is to upgrade the code that generated the file.\n",
-        "\n",
-        "But, if you have a \"Frozen graph\" (a `tf.Graph` where the variables have been turned into constants), then it is possible to convert this to a [`concrete_function`](https://tensorflow.org/guide/concrete_function) using `v1.wrap_function`:\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1xl_fFhs1ur6"
-      },
-      "outputs": [],
-      "source": [
-        "def wrap_frozen_graph(graph_def, inputs, outputs):\n",
-        "  def _imports_graph_def():\n",
-        "    tf.compat.v1.import_graph_def(graph_def, name=\"\")\n",
-        "  wrapped_import = tf.compat.v1.wrap_function(_imports_graph_def, [])\n",
-        "  import_graph = wrapped_import.graph\n",
-        "  return wrapped_import.prune(\n",
-        "      tf.nest.map_structure(import_graph.as_graph_element, inputs),\n",
-        "      tf.nest.map_structure(import_graph.as_graph_element, outputs))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "bQHGXyYC4GT-"
-      },
-      "source": [
-        "For example, here is a frozed graph for Inception v1, from 2016:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "obfto7Bp3hHj"
-      },
-      "outputs": [],
-      "source": [
-        "path = tf.keras.utils.get_file(\n",
-        "    'inception_v1_2016_08_28_frozen.pb',\n",
-        "    'http://storage.googleapis.com/download.tensorflow.org/models/inception_v1_2016_08_28_frozen.pb.tar.gz',\n",
-        "    untar=True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "FHE3Ot4gSCJg"
-      },
-      "source": [
-        "Load the `tf.GraphDef`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "HqL_OTwr4n20"
-      },
-      "outputs": [],
-      "source": [
-        "graph_def = tf.compat.v1.GraphDef()\n",
-        "loaded = graph_def.ParseFromString(open(path,'rb').read())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "T9ai6Kh6SsQ4"
-      },
-      "source": [
-        "Wrap it into a `concrete_function`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "pajiVteo6H0U"
-      },
-      "outputs": [],
-      "source": [
-        "inception_func = wrap_frozen_graph(\n",
-        "    graph_def, inputs='input:0',\n",
-        "    outputs='InceptionV1/InceptionV1/Mixed_3b/Branch_1/Conv2d_0a_1x1/Relu:0')"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "FvknOO-MSw4n"
-      },
-      "source": [
-        "Pass it a tensor as input:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "gsPSIjAE7nry"
-      },
-      "outputs": [],
-      "source": [
-        "input_img = tf.ones([1,224,224,3], dtype=tf.float32)\n",
-        "inception_func(input_img).shape"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "ewl9P3oZ6ZtR"
-      },
-      "source": [
-        "## Estimators"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "YprVP9g3l6eG"
-      },
-      "source": [
-        "### Training with Estimators\n",
-        "\n",
-        "Estimators are supported in TensorFlow 2.0.\n",
-        "\n",
-        "When you use estimators, you can use `input_fn()`, `tf.estimator.TrainSpec`, and `tf.estimator.EvalSpec` from TensorFlow 1.x.\n",
-        "\n",
-        "Here is an example using `input_fn` with train and evaluate specs."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "N5kZeJsF8lS2"
-      },
-      "source": [
-        "#### Creating the input_fn and train/eval specs"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "AOlXGO4J6jDh"
-      },
-      "outputs": [],
-      "source": [
-        "# Define the estimator's input_fn\n",
-        "def input_fn():\n",
-        "  datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n",
-        "  mnist_train, mnist_test = datasets['train'], datasets['test']\n",
-        "\n",
-        "  BUFFER_SIZE = 10000\n",
-        "  BATCH_SIZE = 64\n",
-        "\n",
-        "  def scale(image, label):\n",
-        "    image = tf.cast(image, tf.float32)\n",
-        "    image /= 255\n",
-        "\n",
-        "    return image, label[..., tf.newaxis]\n",
-        "\n",
-        "  train_data = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
-        "  return train_data.repeat()\n",
-        "\n",
-        "# Define train \u0026 eval specs\n",
-        "train_spec = tf.estimator.TrainSpec(input_fn=input_fn,\n",
-        "                                    max_steps=STEPS_PER_EPOCH * NUM_EPOCHS)\n",
-        "eval_spec = tf.estimator.EvalSpec(input_fn=input_fn,\n",
-        "                                  steps=STEPS_PER_EPOCH)\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "_o6J48Nj9H5c"
-      },
-      "source": [
-        "### Using a Keras model definition"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "IXCQdhGq9SbB"
-      },
-      "source": [
-        "There are some differences in how to construct your estimators in TensorFlow 2.0.\n",
-        "\n",
-        "We recommend that you define your model using Keras, then use the `tf.keras.estimator.model_to_estimator` utility to turn your model into an estimator. The code below shows how to use this utility when creating and training an estimator."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "aelsClm3Cq4I"
-      },
-      "outputs": [],
-      "source": [
-        "def make_model():\n",
-        "  return tf.keras.Sequential([\n",
-        "    tf.keras.layers.Conv2D(32, 3, activation='relu',\n",
-        "                           kernel_regularizer=tf.keras.regularizers.l2(0.02),\n",
-        "                           input_shape=(28, 28, 1)),\n",
-        "    tf.keras.layers.MaxPooling2D(),\n",
-        "    tf.keras.layers.Flatten(),\n",
-        "    tf.keras.layers.Dropout(0.1),\n",
-        "    tf.keras.layers.Dense(64, activation='relu'),\n",
-        "    tf.keras.layers.BatchNormalization(),\n",
-        "    tf.keras.layers.Dense(10, activation='softmax')\n",
-        "  ])"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "HJb6f8dtl6rr"
-      },
-      "outputs": [],
-      "source": [
-        "model = make_model()\n",
-        "\n",
-        "model.compile(optimizer='adam',\n",
-        "              loss='sparse_categorical_crossentropy',\n",
-        "              metrics=['accuracy'])\n",
-        "\n",
-        "estimator = tf.keras.estimator.model_to_estimator(\n",
-        "  keras_model = model\n",
-        ")\n",
-        "\n",
-        "tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-ptTxL1q6flL"
-      },
-      "source": [
-        "### Using a custom `model_fn`\n",
-        "\n",
-        "If you have an existing custom estimator `model_fn` that you need to maintain, you can convert your `model_fn` to use a Keras model.\n",
-        "\n",
-        "However, for compatibility reasons, a custom `model_fn` will still run in 1.x-style graph mode. This means there is no eager execution and no automatic control dependencies."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Avgqf8IKfd51"
-      },
-      "source": [
-        "\u003ca name=\"minimal_changes\"\u003e\u003c/a\u003e\n",
-        "\n",
-        "#### Custom model_fn with minimal changes\n",
-        "To make your custom `model_fn` work in TF 2.0, if you prefer minimal changes to the existing code, `tf.compat.v1` symbols such as `optimizers` and `metrics` can be used.\n",
-        "\n",
-        "Using a Keras models in a custom `model_fn` is similar to using it in a custom training loop:\n",
-        "\n",
-        "* Set the `training` phase appropriately, based on the `mode` argument.\n",
-        "* Explicitly pass the model's `trainable_variables` to the optimizer.\n",
-        "\n",
-        "But there are important differences, relative to a [custom loop](#custom_loop):\n",
-        "\n",
-        "* Instead of using `Model.losses`, extract the losses using `Model.get_losses_for`.\n",
-        "* Extract the model's updates using `Model.get_updates_for`.\n",
-        "\n",
-        "Note: \"Updates\" are changes that need to be applied to a model after each batch. For example, the moving averages of the mean and variance in a `layers.BatchNormalization` layer.\n",
-        "\n",
-        "The following code creates an estimator from a custom `model_fn`, illustrating all of these concerns."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "iY16eZKW606-"
-      },
-      "outputs": [],
-      "source": [
-        "def my_model_fn(features, labels, mode):\n",
-        "  model = make_model()\n",
-        "\n",
-        "  optimizer = tf.compat.v1.train.AdamOptimizer()\n",
-        "  loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n",
-        "\n",
-        "  training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
-        "  predictions = model(features, training=training)\n",
-        "\n",
-        "  if mode == tf.estimator.ModeKeys.PREDICT:\n",
-        "    return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)\n",
-        "\n",
-        "  reg_losses = model.get_losses_for(None) + model.get_losses_for(features)\n",
-        "  total_loss = loss_fn(labels, predictions) + tf.math.add_n(reg_losses)\n",
-        "\n",
-        "  accuracy = tf.compat.v1.metrics.accuracy(labels=labels,\n",
-        "                                           predictions=tf.math.argmax(predictions, axis=1),\n",
-        "                                           name='acc_op')\n",
-        "\n",
-        "  update_ops = model.get_updates_for(None) + model.get_updates_for(features)\n",
-        "  minimize_op = optimizer.minimize(\n",
-        "      total_loss,\n",
-        "      var_list=model.trainable_variables,\n",
-        "      global_step=tf.compat.v1.train.get_or_create_global_step())\n",
-        "  train_op = tf.group(minimize_op, update_ops)\n",
-        "\n",
-        "  return tf.estimator.EstimatorSpec(\n",
-        "    mode=mode,\n",
-        "    predictions=predictions,\n",
-        "    loss=total_loss,\n",
-        "    train_op=train_op, eval_metric_ops={'accuracy': accuracy})\n",
-        "\n",
-        "# Create the Estimator \u0026 Train\n",
-        "estimator = tf.estimator.Estimator(model_fn=my_model_fn)\n",
-        "tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "XVxHmU2ccfAG"
-      },
-      "source": [
-        "#### Custom `model_fn` with TF 2.0 symbols\n",
-        "If you want to get rid of all TF 1.x symbols and upgrade your custom `model_fn` to native TF 2.0, you need to update the optimizer and metrics to `tf.keras.optimizers` and `tf.keras.metrics`.\n",
-        "\n",
-        "In the custom `model_fn`, besides the above [changes](#minimal_changes), more upgrades need to be made:\n",
-        "\n",
-        "* Use [`tf.keras.optimizers`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/optimizers) instead of `v1.train.Optimizer`.\n",
-        "* Explicitly pass the model's `trainable_variables` to the `tf.keras.optimizers`.\n",
-        "* To compute the `train_op/minimize_op`,\n",
-        "  * Use `Optimizer.get_updates()` if the loss is scalar loss `Tensor`(not a callable). The first element in the returned list is the desired `train_op/minimize_op`. \n",
-        "  * If the loss is a callable (such as a function), use `Optimizer.minimize()` to get the `train_op/minimize_op`.\n",
-        "* Use [`tf.keras.metrics`](https://www.tensorflow.org/api_docs/python/tf/keras/metrics) instead of `tf.compat.v1.metrics` for evaluation.\n",
-        "\n",
-        "For the above example of `my_model_fn`, the migrated code with 2.0 symbols is shown as:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "uf8J3nloeze2"
-      },
-      "outputs": [],
-      "source": [
-        "def my_model_fn(features, labels, mode):\n",
-        "  model = make_model()\n",
-        "\n",
-        "  training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
-        "  loss_obj = tf.keras.losses.SparseCategoricalCrossentropy()\n",
-        "  predictions = model(features, training=training)\n",
-        "\n",
-        "  # Get both the unconditional losses (the None part)\n",
-        "  # and the input-conditional losses (the features part).\n",
-        "  reg_losses = model.get_losses_for(None) + model.get_losses_for(features)\n",
-        "  total_loss = loss_obj(labels, predictions) + tf.math.add_n(reg_losses)\n",
-        "\n",
-        "  # Upgrade to tf.keras.metrics.\n",
-        "  accuracy_obj = tf.keras.metrics.Accuracy(name='acc_obj')\n",
-        "  accuracy = accuracy_obj.update_state(\n",
-        "      y_true=labels, y_pred=tf.math.argmax(predictions, axis=1))\n",
-        "\n",
-        "  train_op = None\n",
-        "  if training:\n",
-        "    # Upgrade to tf.keras.optimizers.\n",
-        "    optimizer = tf.keras.optimizers.Adam()\n",
-        "    # Manually assign tf.compat.v1.global_step variable to optimizer.iterations\n",
-        "    # to make tf.compat.v1.train.global_step increased correctly.\n",
-        "    # This assignment is a must for any `tf.train.SessionRunHook` specified in\n",
-        "    # estimator, as SessionRunHooks rely on global step.\n",
-        "    optimizer.iterations = tf.compat.v1.train.get_or_create_global_step()\n",
-        "    # Get both the unconditional updates (the None part)\n",
-        "    # and the input-conditional updates (the features part).\n",
-        "    update_ops = model.get_updates_for(None) + model.get_updates_for(features)\n",
-        "    # Compute the minimize_op.\n",
-        "    minimize_op = optimizer.get_updates(\n",
-        "        total_loss,\n",
-        "        model.trainable_variables)[0]\n",
-        "    train_op = tf.group(minimize_op, *update_ops)\n",
-        "\n",
-        "  return tf.estimator.EstimatorSpec(\n",
-        "    mode=mode,\n",
-        "    predictions=predictions,\n",
-        "    loss=total_loss,\n",
-        "    train_op=train_op,\n",
-        "    eval_metric_ops={'Accuracy': accuracy_obj})\n",
-        "\n",
-        "# Create the Estimator \u0026 Train.\n",
-        "estimator = tf.estimator.Estimator(model_fn=my_model_fn)\n",
-        "tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "g1l6VnOTodfA"
-      },
-      "source": [
-        "### Premade Estimators\n",
-        "\n",
-        "[Premade Estimators](https://www.tensorflow.org/guide/premade_estimators) in the family of `tf.estimator.DNN*`, `tf.estimator.Linear*` and `tf.estimator.DNNLinearCombined*` are still supported in the TensorFlow 2.0 API, however, some arguments have changed:\n",
-        "\n",
-        "1. `input_layer_partitioner`: Removed in 2.0.\n",
-        "2. `loss_reduction`: Updated to `tf.keras.losses.Reduction` instead of `tf.compat.v1.losses.Reduction`. Its default value is also changed to `tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE` from `tf.compat.v1.losses.Reduction.SUM`.\n",
-        "3. `optimizer`, `dnn_optimizer` and `linear_optimizer`: this arg has been updated to `tf.keras.optimizers` instead of the `tf.compat.v1.train.Optimizer`. \n",
-        "\n",
-        "To migrate the above changes:\n",
-        "1. No migration is needed for `input_layer_partitioner` since [`Distribution Strategy`](https://www.tensorflow.org/guide/distributed_training) will handle it automatically in TF 2.0.\n",
-        "2. For `loss_reduction`, check [`tf.keras.losses.Reduction`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/losses/Reduction) for the supported options.\n",
-        "3. For `optimizer` args, if you do not pass in an `optimizer`, `dnn_optimizer` or `linear_optimizer` arg, or if you specify the `optimizer` arg as a `string` in your code, you don't need to change anything. `tf.keras.optimizers` is used by default. Otherwise, you need to update it from `tf.compat.v1.train.Optimizer` to its corresponding [`tf.keras.optimizers`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/optimizers)\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "v0Kljg-AHyqv"
-      },
-      "source": [
-        "#### Checkpoint Converter\n",
-        "\u003ca id=\"checkpoint_converter\"\u003e\u003c/a\u003e\n",
-        "\n",
-        "The migration to `keras.optimizers` will break checkpoints saved using TF 1.x, as `tf.keras.optimizers` generates a different set of variables to be saved in checkpoints. To make old checkpoint reusable after your migration to TF 2.0, try the [checkpoint converter tool](https://github.com/tensorflow/estimator/blob/master/tensorflow_estimator/python/estimator/tools/checkpoint_converter.py)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "h9FiYN9mIPli"
-      },
-      "outputs": [],
-      "source": [
-        "! curl -O https://raw.githubusercontent.com/tensorflow/estimator/master/tensorflow_estimator/python/estimator/tools/checkpoint_converter.py"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "DMc6zDJaJwNw"
-      },
-      "source": [
-        "The tool has builtin help:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "9JNZFX3rJLXv"
-      },
-      "outputs": [],
-      "source": [
-        "! python checkpoint_converter.py -h"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "dt8ct9XCFqls"
-      },
-      "source": [
-        "\u003ca id=\"tensorshape\"\u003e\u003c/a\u003e\n",
-        "\n",
-        "## TensorShape\n",
-        "\n",
-        "This class was simplified to hold `int`s, instead of `tf.compat.v1.Dimension` objects. So there is no need to call `.value()` to get an `int`.\n",
-        "\n",
-        "Individual `tf.compat.v1.Dimension` objects are still accessible from `tf.TensorShape.dims`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "x36cWcmM8Eu1"
-      },
-      "source": [
-        "\n",
-        "\n",
-        "The following demonstrate the differences between TensorFlow 1.x and TensorFlow 2.0."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "PbpD-kHOZR4A"
-      },
-      "outputs": [],
-      "source": [
-        "# Create a shape and choose an index\n",
-        "i = 0\n",
-        "shape = tf.TensorShape([16, None, 256])\n",
-        "shape"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "kDFck03neNy0"
-      },
-      "source": [
-        "If you had this in TF 1.x:\n",
-        "\n",
-        "```python\n",
-        "value = shape[i].value\n",
-        "```\n",
-        "\n",
-        "Then do this in TF 2.0:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "KuR73QGEeNdH"
-      },
-      "outputs": [],
-      "source": [
-        "value = shape[i]\n",
-        "value"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "bPWPNKRiZmkd"
-      },
-      "source": [
-        "If you had this in TF 1.x:\n",
-        "\n",
-        "```python\n",
-        "for dim in shape:\n",
-        "    value = dim.value\n",
-        "    print(value)\n",
-        "```\n",
-        "\n",
-        "Then do this in TF 2.0:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "y6s0vuuprJfc"
-      },
-      "outputs": [],
-      "source": [
-        "for value in shape:\n",
-        "  print(value)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "YpRgngu3Zw-A"
-      },
-      "source": [
-        "If you had this in TF 1.x (Or used any other dimension method):\n",
-        "\n",
-        "```python\n",
-        "dim = shape[i]\n",
-        "dim.assert_is_compatible_with(other_dim)\n",
-        "```\n",
-        "\n",
-        "Then do this in TF 2.0:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "LpViGEcUZDGX"
-      },
-      "outputs": [],
-      "source": [
-        "other_dim = 16\n",
-        "Dimension = tf.compat.v1.Dimension\n",
-        "\n",
-        "if shape.rank is None:\n",
-        "  dim = Dimension(None)\n",
-        "else:\n",
-        "  dim = shape.dims[i]\n",
-        "dim.is_compatible_with(other_dim) # or any other dimension method"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "GaiGe36dOdZ_"
-      },
-      "outputs": [],
-      "source": [
-        "shape = tf.TensorShape(None)\n",
-        "\n",
-        "if shape:\n",
-        "  dim = shape.dims[i]\n",
-        "  dim.is_compatible_with(other_dim) # or any other dimension method"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "3kLLY0I3PI-l"
-      },
-      "source": [
-        "The boolean value of a `tf.TensorShape` is `True` if the rank is known, `False` otherwise."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-Ow1ndKpOnJd"
-      },
-      "outputs": [],
-      "source": [
-        "print(bool(tf.TensorShape([])))      # Scalar\n",
-        "print(bool(tf.TensorShape([0])))     # 0-length vector\n",
-        "print(bool(tf.TensorShape([1])))     # 1-length vector\n",
-        "print(bool(tf.TensorShape([None])))  # Unknown-length vector\n",
-        "print(bool(tf.TensorShape([1, 10, 100])))       # 3D tensor\n",
-        "print(bool(tf.TensorShape([None, None, None]))) # 3D tensor with no known dimensions\n",
-        "print()\n",
-        "print(bool(tf.TensorShape(None)))  # A tensor with unknown rank."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "8u63n5S7Y9IX"
-      },
-      "source": [
-        "## Other Changes\n",
-        "\n",
-        "* Remove `tf.colocate_with`: TensorFlow's device placement algorithms have improved significantly. This should no longer be necessary. If removing it causes a performance degredation [please file a bug](https://github.com/tensorflow/tensorflow/issues).\n",
-        "\n",
-        "* Replace `v1.ConfigProto` usage with the equivalent functions from `tf.config`.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vKX6AdTAQhB-"
-      },
-      "source": [
-        "## Conclusions\n",
-        "\n",
-        "The overall process is:\n",
-        "\n",
-        "1. Run the upgrade script.\n",
-        "2. Remove contrib symbols.\n",
-        "3. Switch your models to an object oriented style (Keras).\n",
-        "4. Use `tf.keras` or `tf.estimator` training and evaluation loops where you can.\n",
-        "5. Otherwise, use custom loops, but be sure to avoid sessions \u0026 collections.\n",
-        "\n",
-        "\n",
-        "It takes a little work to convert code to idiomatic TensorFlow 2.0, but every change results in:\n",
-        "\n",
-        "* Fewer lines of code.\n",
-        "* Increased clarity and simplicity.\n",
-        "* Easier debugging.\n",
-        "\n"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "migrate.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/site/en/guide/migrate/_index.yaml b/site/en/guide/migrate/_index.yaml
new file mode 100644
index 00000000000..74cec5dc3d2
--- /dev/null
+++ b/site/en/guide/migrate/_index.yaml
@@ -0,0 +1,89 @@
+book_path: /overview/_book.yaml
+project_path: /overview/_project.yaml
+title: Migrate to TensorFlow 2
+landing_page:
+  custom_css_path: /site-assets/css/style.css
+  nav: left
+  meta_tags:
+  - name: description
+    content: >
+      Learn how to migrate your TensorFlow code from TensorFlow 1.x to
+      TensorFlow 2.
+  rows:
+  - heading: "Migrate from TensorFlow 1.x to TensorFlow 2"
+    classname: tfo-landing-page-heading
+  - classname:
+      devsite-landing-row-100
+      devsite-landing-row-large-headings
+    items:
+    - description: >
+        
Learn how to migrate your TensorFlow code from TensorFlow 1.x to
+        TensorFlow 2. It may take a little work to convert your code,
+        but every change results in access to new features and models,
+        increased clarity and simplicity, and easier debugging. Before
+        starting to migrate, read the behaviors guide.
+        Briefly, the migration process is:

+        
    
+        
  1. Run the automated script to convert your TF1.x API usage to
+        tf.compat.v1.
    2. 
+        
  2. Remove old tf.contrib.layers and replace them with TF Slim symbols. Also check TF Addons for other tf.contrib symbols.
    3. 
+        
  3. Rewrite your TF1.x model forward passes to run in TF2 with eager execution enabled.
    4. 
+        
  4. Validate the accuracy and numerical correctness of your migrated code.
    5. 
+        
  5. Upgrade your training, evaluation and model saving code to TF2 equivalents.
    6. 
+        
  6. (Optional) Migrate your TF2-compatible tf.compat.v1 APIs including TF Slim usage to idiomatic TF2
+        APIs.
    7. 
+        

+
+  - description: "
Featured migration guides
"
+    classname:
+      devsite-landing-row-100
+    items:
+    - classname: tfo-landing-page-card
+      description: >
+        
TensorFlow 1.x vs TensorFlow 2

+        Learn how the TF2 API and behaviors differ fundamentally from TF1.x.
+      path: /guide/migrate/tf1_vs_tf2
+    - classname: tfo-landing-page-card
+      description: >
+        
Map TF1.x models to TF2

+        Begin using TF1.x models in TF2 right away using modeling shims.
+      path: /guide/migrate/model_mapping
+    - classname: tfo-landing-page-card
+      description: >
+        
Rewrite TF1.x API symbols

+        Programmatically upgrade some parts of your TF1.x code to TF2.
+      path: /guide/migrate/upgrade
+
+  - items:
+    - classname: tfo-landing-page-card
+      description: >
+        
Validate migrated TF2 code

+        Verify the correctness of your migrated TF2 code.
+      path: /guide/migrate/validate_correctness
+    - classname: tfo-landing-page-card
+      description: >
+        
Moving off Estimators

+        Migrate from your Estimator training pipelines to TF2.
+      path: /guide/migrate/migrating_estimator
+    - classname: tfo-landing-page-card
+      description: >
+        
Moving off feature columns

+        Learn how to migrate to Keras preprocessing layers from tf.feature_columns.
+      path: /guide/migrate/migrating_feature_columns
+
+  - items:
+    - classname: tfo-landing-page-card
+      description: >
+        
Multi-worker CPU/GPU training workflows

+        Learn how to migrate multi-worker distributed Estimator to TF2.
+      path: /guide/migrate/multi_worker_cpu_gpu_training
+    - classname: tfo-landing-page-card
+      description: >
+        
TPU workflows

+        Learn how to migrate the TPUEstimator API to TF2.
+      path: /guide/migrate/tpu_estimator
+    - classname: tfo-landing-page-card
+      description: >
+        
TensorFlow Lite

+        Learn how to migrate TF Lite code created with TF1.x to TF2.
+      path: /guide/migrate/tflite
diff --git a/site/en/guide/migrate/_toc.yaml b/site/en/guide/migrate/_toc.yaml
new file mode 100644
index 00000000000..ceb7e5f57ae
--- /dev/null
+++ b/site/en/guide/migrate/_toc.yaml
@@ -0,0 +1,71 @@
+toc:
+- title: Migration guide
+  path: /guide/migrate/
+- heading: Migration overview
+- title: Migrate to TensorFlow 2
+  path: /guide/migrate/migrate_tf2
+- title: TF1 vs TF2 behavior comparison
+  path: /guide/migrate/tf1_vs_tf2
+- title: Convert with the upgrade script
+  path: /guide/migrate/upgrade
+
+- heading: Migrate your models
+- title: TF2 model mapping
+  path: /guide/migrate/model_mapping
+- title: TF Lite models in TF2
+  path: /guide/migrate/tflite
+- title: Migrating checkpoints
+  path: /guide/migrate/migrating_checkpoints
+- title: Canned and boosted tree estimators
+  path: /guide/migrate/canned_estimators
+
+- heading: Migrate training and evaluation pipelines
+- title: Estimators
+  path: /guide/migrate/migrating_estimator
+- title: Summary APIs
+  path: https://www.tensorflow.org/tensorboard/migrate
+  status: external
+- title: Estimator logging and stop hooks
+  path: /guide/migrate/logging_stop_hook
+- title: tf.estimator.SessionRunHook API
+  path: /guide/migrate/sessionrunhook_callback
+#  - title: Migrate custom training loops
+#    path: /guide/migrate/custom_training_loops
+- title: Early stopping
+  path: /guide/migrate/early_stopping
+- title: Feature columns to Keras preprocessing layers
+  path: /guide/migrate/migrating_feature_columns
+- title: Metrics and optimizers
+  path: /guide/migrate/metrics_optimizers
+- title: Incorporate fault tolerance in training
+  path: /guide/migrate/fault_tolerance
+- title: Evaluate training metrics with SidecarEvaluator
+  path: /guide/migrate/evaluator
+- title: TensorBoard metric displays
+  path: /guide/migrate/tensorboard
+
+- heading: Migrate saving and export workflows
+- title: Export models to SavedModel
+  path: /guide/migrate/saved_model
+- title: Save checkpoints
+  path: /guide/migrate/checkpoint_saver
+
+- heading: Migrate distributed training workflows
+- title: Multi-GPU training
+  path: /guide/migrate/mirrored_strategy
+- title: Multi-worker training on CPU and GPU
+  path: /guide/migrate/multi_worker_cpu_gpu_training
+- title: Multi-worker training on TPU
+  path: /guide/migrate/tpu_estimator
+- title: TPU embedding_columns to TPUEmbedding layer
+  path: /guide/migrate/tpu_embedding
+# - title: Efficient distributed input processing
+#   path: /guide/migrate/distributed_input_processing
+
+- heading: Validate model quality and performance
+- title: Validate correctness and numerical equivalence
+  path: /guide/migrate/validate_correctness
+- title: Debug TF2 Migrated Training Pipeline
+  path: /guide/migrate/migration_debugging
+#  - title: Testing speed and throughput
+#    path: /guide/migrate/throughput_testing
diff --git a/site/en/guide/migrate/canned_estimators.ipynb b/site/en/guide/migrate/canned_estimators.ipynb
new file mode 100644
index 00000000000..68859511a84
--- /dev/null
+++ b/site/en/guide/migrate/canned_estimators.ipynb
@@ -0,0 +1,754 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migration examples: Canned Estimators\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "Canned (or Premade) Estimators have traditionally been used in TensorFlow 1 as quick and easy ways to train models for a variety of typical use cases. TensorFlow 2 provides straightforward approximate substitutes for a number of them by way of Keras models. For those canned estimators that do not have built-in TensorFlow 2 substitutes, you can still build your own replacement fairly easily.\n",
+        "\n",
+        "This guide will walk you through a few examples of direct equivalents and custom substitutions to demonstrate how TensorFlow 1's `tf.estimator`-derived models can be migrated to TensorFlow 2 with Keras.\n",
+        "\n",
+        "Namely, this guide includes examples for migrating:\n",
+        "*    From `tf.estimator`'s `LinearEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to Keras `tf.compat.v1.keras.models.LinearModel` in TensorFlow 2\n",
+        "*    From `tf.estimator`'s `DNNEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to a custom Keras DNN ModelKeras in TensorFlow 2\n",
+        "*    From `tf.estimator`'s `DNNLinearCombinedEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to `tf.compat.v1.keras.models.WideDeepModel` in TensorFlow 2\n",
+        "*    From `tf.estimator`'s `BoostedTreesEstimator`, `Classifier` or `Regressor` in TensorFlow 1 to `tfdf.keras.GradientBoostedTreesModel` in TensorFlow 2\n",
+        "\n",
+        "A common precursor to the training of a model is feature preprocessing, which is done for TensorFlow 1 Estimator models with `tf.feature_column`. For more information on feature preprocessing in TensorFlow 2, see [this guide on migrating from feature columns to the Keras preprocessing layers API](migrating_feature_columns.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Start with a couple of necessary TensorFlow imports,"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qsgZp0f-nu9s"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install tensorflow_decision_forests"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd\n",
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1\n",
+        "import tensorflow_decision_forests as tfdf\n",
+        "from tensorflow import keras\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Jsm9Rxx7s1OZ"
+      },
+      "source": [
+        "prepare some simple data for demonstration from the standard Titanic dataset,"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wC6i_bEZPrPY"
+      },
+      "outputs": [],
+      "source": [
+        "x_train = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/train.csv')\n",
+        "x_eval = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/eval.csv')\n",
+        "x_train['sex'].replace(('male', 'female'), (0, 1), inplace=True)\n",
+        "x_eval['sex'].replace(('male', 'female'), (0, 1), inplace=True)\n",
+        "\n",
+        "x_train['alone'].replace(('n', 'y'), (0, 1), inplace=True)\n",
+        "x_eval['alone'].replace(('n', 'y'), (0, 1), inplace=True)\n",
+        "\n",
+        "x_train['class'].replace(('First', 'Second', 'Third'), (1, 2, 3), inplace=True)\n",
+        "x_eval['class'].replace(('First', 'Second', 'Third'), (1, 2, 3), inplace=True)\n",
+        "\n",
+        "x_train.drop(['embark_town', 'deck'], axis=1, inplace=True)\n",
+        "x_eval.drop(['embark_town', 'deck'], axis=1, inplace=True)\n",
+        "\n",
+        "y_train = x_train.pop('survived')\n",
+        "y_eval = x_eval.pop('survived')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "# Data setup for TensorFlow 1 with `tf.estimator`\n",
+        "def _input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices((dict(x_train), y_train)).batch(32)\n",
+        "\n",
+        "\n",
+        "def _eval_input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices((dict(x_eval), y_eval)).batch(32)\n",
+        "\n",
+        "\n",
+        "FEATURE_NAMES = [\n",
+        "    'age', 'fare', 'sex', 'n_siblings_spouses', 'parch', 'class', 'alone'\n",
+        "]\n",
+        "\n",
+        "feature_columns = []\n",
+        "for fn in FEATURE_NAMES:\n",
+        "  feat_col = tf1.feature_column.numeric_column(fn, dtype=tf.float32)\n",
+        "  feature_columns.append(feat_col)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bYSgoezeMrpI"
+      },
+      "source": [
+        "and create a method to instantiate a simplistic sample optimizer to use with various TensorFlow 1 Estimator and TensorFlow 2 Keras models."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YHB_nuzVLVLe"
+      },
+      "outputs": [],
+      "source": [
+        "def create_sample_optimizer(tf_version):\n",
+        "  if tf_version == 'tf1':\n",
+        "    optimizer = lambda: tf.keras.optimizers.legacy.Ftrl(\n",
+        "        l1_regularization_strength=0.001,\n",
+        "        learning_rate=tf1.train.exponential_decay(\n",
+        "            learning_rate=0.1,\n",
+        "            global_step=tf1.train.get_global_step(),\n",
+        "            decay_steps=10000,\n",
+        "            decay_rate=0.9))\n",
+        "  elif tf_version == 'tf2':\n",
+        "    optimizer = tf.keras.optimizers.legacy.Ftrl(\n",
+        "        l1_regularization_strength=0.001,\n",
+        "        learning_rate=tf.keras.optimizers.schedules.ExponentialDecay(\n",
+        "            initial_learning_rate=0.1, decay_steps=10000, decay_rate=0.9))\n",
+        "  return optimizer"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4uXff1BEssdE"
+      },
+      "source": [
+        "## Example 1: Migrating from LinearEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_O7fyhCnpvED"
+      },
+      "source": [
+        "### TensorFlow 1: Using LinearEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A9560BqEOTpb"
+      },
+      "source": [
+        "In TensorFlow 1, you can use `tf.estimator.LinearEstimator` to create a baseline linear model for regression and classification problems."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oWfh0QW4IXTn"
+      },
+      "outputs": [],
+      "source": [
+        "linear_estimator = tf.estimator.LinearEstimator(\n",
+        "    head=tf.estimator.BinaryClassHead(),\n",
+        "    feature_columns=feature_columns,\n",
+        "    optimizer=create_sample_optimizer('tf1'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hi77Sg4k-0TR"
+      },
+      "outputs": [],
+      "source": [
+        "linear_estimator.train(input_fn=_input_fn, steps=100)\n",
+        "linear_estimator.evaluate(input_fn=_eval_input_fn, steps=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "### TensorFlow 2: Using Keras LinearModel"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fkgkGf_AOaRR"
+      },
+      "source": [
+        "In TensorFlow 2, you can create an instance of the Keras `tf.compat.v1.keras.models.LinearModel` which is the substitute to the `tf.estimator.LinearEstimator`. The `tf.compat.v1.keras` path is used to signify that the pre-made model exists for compatibility."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kip65sYBlKiu"
+      },
+      "outputs": [],
+      "source": [
+        "linear_model = tf.compat.v1.keras.experimental.LinearModel()\n",
+        "linear_model.compile(loss='mse', optimizer=create_sample_optimizer('tf2'), metrics=['accuracy'])\n",
+        "linear_model.fit(x_train, y_train, epochs=10)\n",
+        "linear_model.evaluate(x_eval, y_eval, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RRrj78Lqplni"
+      },
+      "source": [
+        "## Example 2: Migrating from DNNEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YKl6XZ7Bp1t5"
+      },
+      "source": [
+        "### TensorFlow 1: Using DNNEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J7wJUmgypln8"
+      },
+      "source": [
+        "In TensorFlow 1, you can use `tf.estimator.DNNEstimator` to create a baseline deep neural network (DNN) model for regression and classification problems."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qHbgXCzfpln9"
+      },
+      "outputs": [],
+      "source": [
+        "dnn_estimator = tf.estimator.DNNEstimator(\n",
+        "    head=tf.estimator.BinaryClassHead(),\n",
+        "    feature_columns=feature_columns,\n",
+        "    hidden_units=[128],\n",
+        "    activation_fn=tf.nn.relu,\n",
+        "    optimizer=create_sample_optimizer('tf1'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6DTnXxU2pln-"
+      },
+      "outputs": [],
+      "source": [
+        "dnn_estimator.train(input_fn=_input_fn, steps=100)\n",
+        "dnn_estimator.evaluate(input_fn=_eval_input_fn, steps=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6xJz6px6pln-"
+      },
+      "source": [
+        "### TensorFlow 2: Using Keras to create a custom DNN model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7cgc72rzpln-"
+      },
+      "source": [
+        "In TensorFlow 2, you can create a custom DNN model to substitute for one generated by `tf.estimator.DNNEstimator`, with similar levels of user-specified customization (for instance, as in the previous example, the ability to customize a chosen model optimizer).\n",
+        "\n",
+        "A similar workflow can be used to replace `tf.estimator.experimental.RNNEstimator` with a Keras recurrent neural network (RNN) model. Keras provides a number of built-in, customizable choices by way of `tf.keras.layers.RNN`, `tf.keras.layers.LSTM`, and `tf.keras.layers.GRU`. To learn more, check out the _Built-in RNN layers: a simple example_ section of [RNN with Keras guide](https://www.tensorflow.org/guide/keras/rnn)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B5SdsjlL49RG"
+      },
+      "outputs": [],
+      "source": [
+        "dnn_model = tf.keras.models.Sequential(\n",
+        "    [tf.keras.layers.Dense(128, activation='relu'),\n",
+        "     tf.keras.layers.Dense(1)])\n",
+        "\n",
+        "dnn_model.compile(loss='mse', optimizer=create_sample_optimizer('tf2'), metrics=['accuracy'])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JQmRw9_Upln_"
+      },
+      "outputs": [],
+      "source": [
+        "dnn_model.fit(x_train, y_train, epochs=10)\n",
+        "dnn_model.evaluate(x_eval, y_eval, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UeBHZ0cd1Pl2"
+      },
+      "source": [
+        "## Example 3: Migrating from DNNLinearCombinedEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GfRaObf5g4TU"
+      },
+      "source": [
+        "### TensorFlow 1: Using DNNLinearCombinedEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2r13RMX-g4TV"
+      },
+      "source": [
+        "In TensorFlow 1, you can use `tf.estimator.DNNLinearCombinedEstimator` to create a baseline combined model for regression and classification problems with customization capacity for both its linear and DNN components."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OyyDCqc5j7rf"
+      },
+      "outputs": [],
+      "source": [
+        "optimizer = create_sample_optimizer('tf1')\n",
+        "\n",
+        "combined_estimator = tf.estimator.DNNLinearCombinedEstimator(\n",
+        "    head=tf.estimator.BinaryClassHead(),\n",
+        "    # Wide settings\n",
+        "    linear_feature_columns=feature_columns,\n",
+        "    linear_optimizer=optimizer,\n",
+        "    # Deep settings\n",
+        "    dnn_feature_columns=feature_columns,\n",
+        "    dnn_hidden_units=[128],\n",
+        "    dnn_optimizer=optimizer)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aXN-BxwzmRaf"
+      },
+      "outputs": [],
+      "source": [
+        "combined_estimator.train(input_fn=_input_fn, steps=100)\n",
+        "combined_estimator.evaluate(input_fn=_eval_input_fn, steps=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BeMikL5ug4TX"
+      },
+      "source": [
+        "### TensorFlow 2: Using Keras WideDeepModel"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CYByxxBhg4TX"
+      },
+      "source": [
+        "In TensorFlow 2, you can create an instance of the Keras `tf.compat.v1.keras.models.WideDeepModel` to substitute for one generated by `tf.estimator.DNNLinearCombinedEstimator`, with similar levels of user-specified customization (for instance, as in the previous example, the ability to customize a chosen model optimizer).\n",
+        "\n",
+        "This `WideDeepModel` is constructed on the basis of a constituent `LinearModel` and a custom DNN Model, both of which are discussed in the preceding two examples. A custom linear model can also be used in place of the built-in Keras `LinearModel` if desired.\n",
+        "\n",
+        "If you would like to build your own model instead of using a canned estimator, check out the [Keras Sequential model](https://www.tensorflow.org/guide/keras/sequential_model) guide. For more information on custom training and optimizers, check out the [Custom training: walkthrough](https://www.tensorflow.org/tutorials/customization/custom_training_walkthrough) guide."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mIFM3e-_RLSX"
+      },
+      "outputs": [],
+      "source": [
+        "# Create LinearModel and DNN Model as in Examples 1 and 2\n",
+        "optimizer = create_sample_optimizer('tf2')\n",
+        "\n",
+        "linear_model = tf.compat.v1.keras.experimental.LinearModel()\n",
+        "linear_model.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])\n",
+        "linear_model.fit(x_train, y_train, epochs=10, verbose=0)\n",
+        "\n",
+        "dnn_model = tf.keras.models.Sequential(\n",
+        "    [tf.keras.layers.Dense(128, activation='relu'),\n",
+        "     tf.keras.layers.Dense(1)])\n",
+        "dnn_model.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mFmQz9kjmMSx"
+      },
+      "outputs": [],
+      "source": [
+        "combined_model = tf.compat.v1.keras.experimental.WideDeepModel(linear_model,\n",
+        "                                                               dnn_model)\n",
+        "combined_model.compile(\n",
+        "    optimizer=[optimizer, optimizer], loss='mse', metrics=['accuracy'])\n",
+        "combined_model.fit([x_train, x_train], y_train, epochs=10)\n",
+        "combined_model.evaluate(x_eval, y_eval, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wP1DBRhpeOJn"
+      },
+      "source": [
+        "## Example 4: Migrating from BoostedTreesEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_3mCQVDSeOKD"
+      },
+      "source": [
+        "### TensorFlow 1: Using BoostedTreesEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oEWYHNt4eOKD"
+      },
+      "source": [
+        "In TensorFlow 1, you could use `tf.estimator.BoostedTreesEstimator` to create a baseline to create a baseline Gradient Boosting model using an ensemble of decision trees for regression and classification problems. This functionality is no longer included in TensorFlow 2."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wliVIER1jLnA"
+      },
+      "source": [
+        "```\n",
+        "bt_estimator = tf1.estimator.BoostedTreesEstimator(\n",
+        "    head=tf.estimator.BinaryClassHead(),\n",
+        "    n_batches_per_layer=1,\n",
+        "    max_depth=10,\n",
+        "    n_trees=1000,\n",
+        "    feature_columns=feature_columns)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-K87uBrZjR0u"
+      },
+      "source": [
+        "```\n",
+        "bt_estimator.train(input_fn=_input_fn, steps=1000)\n",
+        "bt_estimator.evaluate(input_fn=_eval_input_fn, steps=100)\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eNuLP6BeeOKF"
+      },
+      "source": [
+        "### TensorFlow 2: Using TensorFlow Decision Forests"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "m3EVq388eOKF"
+      },
+      "source": [
+        "In TensorFlow 2, `tf.estimator.BoostedTreesEstimator` is replaced by [tfdf.keras.GradientBoostedTreesModel](https://www.tensorflow.org/decision_forests/api_docs/python/tfdf/keras/GradientBoostedTreesModel#attributes) from the [TensorFlow Decision Forests](https://www.tensorflow.org/decision_forests) package.\n",
+        "\n",
+        "TensorFlow Decision Forests provides various advantages over the `tf.estimator.BoostedTreesEstimator`, notably regarding quality, speed, ease of use and flexibility. To learn about TensorFlow Decision Forests, start with the [beginner colab](https://www.tensorflow.org/decision_forests/tutorials/beginner_colab).\n",
+        "\n",
+        "The following example shows how to train a Gradient Boosted Trees model using TensorFlow 2:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UB90fXJdVWC5"
+      },
+      "source": [
+        "Install TensorFlow Decision Forests."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9097mTCIVVE9"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install tensorflow_decision_forests"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "B1qTdAS-VpXk"
+      },
+      "source": [
+        "Create a TensorFlow dataset. Note that Decision Forests natively support many types of features and do not need pre-processing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jkjFHmDTVswY"
+      },
+      "outputs": [],
+      "source": [
+        "train_dataframe = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/train.csv')\n",
+        "eval_dataframe = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/eval.csv')\n",
+        "\n",
+        "# Convert the Pandas Dataframes into TensorFlow datasets.\n",
+        "train_dataset = tfdf.keras.pd_dataframe_to_tf_dataset(train_dataframe, label=\"survived\")\n",
+        "eval_dataset = tfdf.keras.pd_dataframe_to_tf_dataset(eval_dataframe, label=\"survived\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7fPa-LfDWDzB"
+      },
+      "source": [
+        "Train the model on the `train_dataset` dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JO0yCH9hWPvJ"
+      },
+      "outputs": [],
+      "source": [
+        "# Use the default hyper-parameters of the model.\n",
+        "gbt_model = tfdf.keras.GradientBoostedTreesModel()\n",
+        "gbt_model.fit(train_dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2Y5xm29AWGxt"
+      },
+      "source": [
+        "Evaluate the quality of the model on the `eval_dataset` dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JLS_2vKKeOKF"
+      },
+      "outputs": [],
+      "source": [
+        "gbt_model.compile(metrics=['accuracy'])\n",
+        "gbt_evaluation = gbt_model.evaluate(eval_dataset, return_dict=True)\n",
+        "print(gbt_evaluation)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z22UJ5SUqToQ"
+      },
+      "source": [
+        "Gradient Boosted Trees is just one of the many decision forest algorithms available in TensorFlow Decision Forests. For example, Random Forests (available as [tfdf.keras.GradientBoostedTreesModel](https://www.tensorflow.org/decision_forests/api_docs/python/tfdf/keras/RandomForestModel) is very resistant to overfitting) while CART (available as [tfdf.keras.CartModel](https://www.tensorflow.org/decision_forests/api_docs/python/tfdf/keras/CartModel)) is great for model interpretation.\n",
+        "\n",
+        "In the next example, train and plot a Random Forest model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "W3slOhn4Zi9X"
+      },
+      "outputs": [],
+      "source": [
+        "# Train a Random Forest model\n",
+        "rf_model = tfdf.keras.RandomForestModel()\n",
+        "rf_model.fit(train_dataset)\n",
+        "\n",
+        "# Evaluate the Random Forest model\n",
+        "rf_model.compile(metrics=['accuracy'])\n",
+        "rf_evaluation = rf_model.evaluate(eval_dataset, return_dict=True)\n",
+        "print(rf_evaluation)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z0QYolhoZb_k"
+      },
+      "source": [
+        "In the final example, train and evaluate a CART model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "027bGnCork_W"
+      },
+      "outputs": [],
+      "source": [
+        "# Train a CART model\n",
+        "cart_model = tfdf.keras.CartModel()\n",
+        "cart_model.fit(train_dataset)\n",
+        "\n",
+        "# Plot the CART model\n",
+        "tfdf.model_plotter.plot_model_in_colab(cart_model, max_depth=2)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "canned_estimators.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/checkpoint_saver.ipynb b/site/en/guide/migrate/checkpoint_saver.ipynb
new file mode 100644
index 00000000000..7e7f35f4c4e
--- /dev/null
+++ b/site/en/guide/migrate/checkpoint_saver.ipynb
@@ -0,0 +1,297 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate checkpoint saving\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hIo_p2FWFIRx"
+      },
+      "source": [
+        "Continually saving the \"best\" model or model weights/parameters has many benefits. These include being able to track the training progress and load saved models from different saved states.\n",
+        "\n",
+        "In TensorFlow 1, to configure checkpoint saving during training/validation with the `tf.estimator.Estimator` APIs, you specify a schedule in `tf.estimator.RunConfig` or use `tf.estimator.CheckpointSaverHook`. This guide demonstrates how to migrate from this workflow to TensorFlow 2 Keras APIs.\n",
+        "\n",
+        "In TensorFlow 2, you can configure `tf.keras.callbacks.ModelCheckpoint` in a number of ways:\n",
+        "\n",
+        "- Save the \"best\" version according to a metric monitored using the `save_best_only=True` parameter, where `monitor` can be, for example, `'loss'`, `'val_loss'`, `'accuracy', or `'val_accuracy'`.\n",
+        "- Save continually at a certain frequency (using the `save_freq` argument).\n",
+        "- Save the weights/parameters only instead of the whole model by setting `save_weights_only` to `True`.\n",
+        "\n",
+        "For more details, refer to the `tf.keras.callbacks.ModelCheckpoint` API docs and the *Save checkpoints during training* section in the [Save and load models](../../tutorials/keras/save_and_load.ipynb) tutorial. Learn more about the Checkpoint format in the *TF Checkpoint format* section in the [Save and load Keras models](https://www.tensorflow.org/guide/keras/save_and_serialize) guide. In addition, to add fault tolerance, you can use `tf.keras.callbacks.BackupAndRestore` or `tf.train.Checkpoint` for manual checkpointing. Learn more in the [Fault tolerance migration guide](fault_tolerance.ipynb).\n",
+        "\n",
+        "Keras [callbacks](https://www.tensorflow.org/guide/keras/custom_callback) are objects that are called at different points during training/evaluation/prediction in the built-in Keras `Model.fit`/`Model.evaluate`/`Model.predict` APIs. Learn more in the _Next steps_ section at the end of the guide."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f55c103999de"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Start with imports and a simple dataset for demonstration purposes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "X74yjOb-e18w"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow.compat.v1 as tf1\n",
+        "import tensorflow as tf\n",
+        "import numpy as np\n",
+        "import tempfile"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2r8r4d8FfMny"
+      },
+      "outputs": [],
+      "source": [
+        "mnist = tf.keras.datasets.mnist\n",
+        "\n",
+        "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
+        "x_train, x_test = x_train / 255.0, x_test / 255.0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wrqBkG4RFLP_"
+      },
+      "source": [
+        "## TensorFlow 1: Save checkpoints with tf.estimator APIs\n",
+        "\n",
+        "This TensorFlow 1 example shows how to configure `tf.estimator.RunConfig` to save checkpoints at every step during training/evaluation with the `tf.estimator.Estimator` APIs:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "upA8nuf3FEq5"
+      },
+      "outputs": [],
+      "source": [
+        "feature_columns = [tf1.feature_column.numeric_column(\"x\", shape=[28, 28])]\n",
+        "\n",
+        "config = tf1.estimator.RunConfig(save_summary_steps=1,\n",
+        "                                 save_checkpoints_steps=1)\n",
+        "\n",
+        "path = tempfile.mkdtemp()\n",
+        "\n",
+        "classifier = tf1.estimator.DNNClassifier(\n",
+        "    feature_columns=feature_columns,\n",
+        "    hidden_units=[256, 32],\n",
+        "    optimizer=tf1.train.AdamOptimizer(0.001),\n",
+        "    n_classes=10,\n",
+        "    dropout=0.2,\n",
+        "    model_dir=path,\n",
+        "    config = config\n",
+        ")\n",
+        "\n",
+        "train_input_fn = tf1.estimator.inputs.numpy_input_fn(\n",
+        "    x={\"x\": x_train},\n",
+        "    y=y_train.astype(np.int32),\n",
+        "    num_epochs=10,\n",
+        "    batch_size=50,\n",
+        "    shuffle=True,\n",
+        ")\n",
+        "\n",
+        "test_input_fn = tf1.estimator.inputs.numpy_input_fn(\n",
+        "    x={\"x\": x_test},\n",
+        "    y=y_test.astype(np.int32),\n",
+        "    num_epochs=10,\n",
+        "    shuffle=False\n",
+        ")\n",
+        "\n",
+        "train_spec = tf1.estimator.TrainSpec(input_fn=train_input_fn, max_steps=10)\n",
+        "eval_spec = tf1.estimator.EvalSpec(input_fn=test_input_fn,\n",
+        "                                   steps=10,\n",
+        "                                   throttle_secs=0)\n",
+        "\n",
+        "tf1.estimator.train_and_evaluate(estimator=classifier,\n",
+        "                                train_spec=train_spec,\n",
+        "                                eval_spec=eval_spec)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3u96G4MtRVqU"
+      },
+      "outputs": [],
+      "source": [
+        "%ls {classifier.model_dir}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QvE_uxDJFUX-"
+      },
+      "source": [
+        "## TensorFlow 2: Save checkpoints with a Keras callback for Model.fit\n",
+        "\n",
+        "In TensorFlow 2, when you use the built-in Keras `Model.fit` (or `Model.evaluate`) for training/evaluation, you can configure `tf.keras.callbacks.ModelCheckpoint` and then pass it to the `callbacks` parameter of `Model.fit` (or `Model.evaluate`). (Learn more in the API docs and the *Using callbacks* section in the [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate) guide.)\n",
+        "\n",
+        "In the example below, you will use a `tf.keras.callbacks.ModelCheckpoint` callback to store checkpoints in a temporary directory:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9FLBhT2BFX2H"
+      },
+      "outputs": [],
+      "source": [
+        "def create_model():\n",
+        "  return tf.keras.models.Sequential([\n",
+        "    tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
+        "    tf.keras.layers.Dense(512, activation='relu'),\n",
+        "    tf.keras.layers.Dropout(0.2),\n",
+        "    tf.keras.layers.Dense(10, activation='softmax')\n",
+        "  ])\n",
+        "\n",
+        "model = create_model()\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss='sparse_categorical_crossentropy',\n",
+        "              metrics=['accuracy'],\n",
+        "              steps_per_execution=10)\n",
+        "\n",
+        "log_dir = tempfile.mkdtemp()\n",
+        "\n",
+        "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
+        "    filepath=log_dir)\n",
+        "\n",
+        "model.fit(x=x_train,\n",
+        "          y=y_train,\n",
+        "          epochs=10,\n",
+        "          validation_data=(x_test, y_test),\n",
+        "          callbacks=[model_checkpoint_callback])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SROSmhyyLBA-"
+      },
+      "outputs": [],
+      "source": [
+        "%ls {model_checkpoint_callback.filepath}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rQUS8nO9FZlH"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "Learn more about checkpointing in:\n",
+        "\n",
+        "- API docs: `tf.keras.callbacks.ModelCheckpoint`\n",
+        "- Tutorial: [Save and load models](../../tutorials/keras/save_and_load.ipynb) (the *Save checkpoints during training* section)\n",
+        "- Guide: [Save and load Keras models](https://www.tensorflow.org/guide/keras/save_and_serialize) (the *TF Checkpoint format* section)\n",
+        "\n",
+        "Learn more about callbacks in:\n",
+        "\n",
+        "- API docs: `tf.keras.callbacks.Callback`\n",
+        "- Guide: [Writing your own callbacks](https://www.tensorflow.org/guide/keras/guide/keras/custom_callback)\n",
+        "- Guide: [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate) (the *Using callbacks* section)\n",
+        "\n",
+        "You may also find the following migration-related resources useful:\n",
+        "\n",
+        "- The [Fault tolerance migration guide](fault_tolerance.ipynb): `tf.keras.callbacks.BackupAndRestore` for `Model.fit`, or `tf.train.Checkpoint` and `tf.train.CheckpointManager` APIs for a custom training loop\n",
+        "- The [Early stopping migration guide](early_stopping.ipynb): `tf.keras.callbacks.EarlyStopping` is a built-in early stopping callback\n",
+        "- The [TensorBoard migration guide](tensorboard.ipynb): TensorBoard enables tracking and displaying metrics\n",
+        "- The [LoggingTensorHook and StopAtStepHook to Keras callbacks migration guide](logging_stop_hook.ipynb)\n",
+        "- The [SessionRunHook to Keras callbacks guide](sessionrunhook_callback.ipynb)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "checkpoint_saver.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/early_stopping.ipynb b/site/en/guide/migrate/early_stopping.ipynb
new file mode 100644
index 00000000000..1c1712e975b
--- /dev/null
+++ b/site/en/guide/migrate/early_stopping.ipynb
@@ -0,0 +1,524 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate early stopping\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "This notebook demonstrates how you can set up model training with early stopping, first, in TensorFlow 1 with `tf.estimator.Estimator` and an early stopping hook, and then, in TensorFlow 2 with Keras APIs or a custom training loop. Early stopping is a regularization technique that stops training if, for example, the validation loss reaches a certain threshold.\n",
+        "\n",
+        "In TensorFlow 2, there are three ways to implement early stopping:\n",
+        "- Use a built-in Keras callback—`tf.keras.callbacks.EarlyStopping`—and pass it to `Model.fit`.\n",
+        "- Define a custom callback and pass it to Keras `Model.fit`.\n",
+        "- Write a custom early stopping rule in a [custom training loop](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) (with `tf.GradientTape`)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import time\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1\n",
+        "import tensorflow_datasets as tfds"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4uXff1BEssdE"
+      },
+      "source": [
+        "## TensorFlow 1: Early stopping with an early stopping hook and tf.estimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JaHhhhW5o8lL"
+      },
+      "source": [
+        "Start by defining functions for MNIST dataset loading and preprocessing, and model definition to be used with `tf.estimator.Estimator`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "def normalize_img(image, label):\n",
+        "  return tf.cast(image, tf.float32) / 255., label\n",
+        "\n",
+        "def _input_fn():\n",
+        "  ds_train = tfds.load(\n",
+        "    name='mnist',\n",
+        "    split='train',\n",
+        "    shuffle_files=True,\n",
+        "    as_supervised=True)\n",
+        "\n",
+        "  ds_train = ds_train.map(\n",
+        "      normalize_img, num_parallel_calls=tf.data.AUTOTUNE)\n",
+        "  ds_train = ds_train.batch(128)\n",
+        "  ds_train = ds_train.repeat(100)\n",
+        "  return ds_train\n",
+        "\n",
+        "def _eval_input_fn():\n",
+        "  ds_test = tfds.load(\n",
+        "    name='mnist',\n",
+        "    split='test',\n",
+        "    shuffle_files=True,\n",
+        "    as_supervised=True)\n",
+        "  ds_test = ds_test.map(\n",
+        "    normalize_img, num_parallel_calls=tf.data.AUTOTUNE)\n",
+        "  ds_test = ds_test.batch(128)\n",
+        "  return ds_test\n",
+        "\n",
+        "def _model_fn(features, labels, mode):\n",
+        "  flatten = tf1.layers.Flatten()(features)\n",
+        "  features = tf1.layers.Dense(128, 'relu')(flatten)\n",
+        "  logits = tf1.layers.Dense(10)(features)\n",
+        "\n",
+        "  loss = tf1.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.005)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "\n",
+        "  return tf1.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hC_AY7KwqD0p"
+      },
+      "source": [
+        "In TensorFlow 1, early stopping works by setting up an early stopping hook with `tf.estimator.experimental.make_early_stopping_hook`. You pass the hook to the `make_early_stopping_hook` method as a parameter for `should_stop_fn`, which can accept a function without any arguments. The training stops once `should_stop_fn` returns `True`.\n",
+        "\n",
+        "The following example demonstrates how to implement an early stopping technique that limits the training time to a maximum of 20 seconds:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HsOpjW5plH9Q"
+      },
+      "outputs": [],
+      "source": [
+        "estimator = tf1.estimator.Estimator(model_fn=_model_fn)\n",
+        "\n",
+        "start_time = time.time()\n",
+        "max_train_seconds = 20\n",
+        "\n",
+        "def should_stop_fn():\n",
+        "  return time.time() - start_time > max_train_seconds\n",
+        "\n",
+        "early_stopping_hook = tf1.estimator.experimental.make_early_stopping_hook(\n",
+        "    estimator=estimator,\n",
+        "    should_stop_fn=should_stop_fn,\n",
+        "    run_every_secs=1,\n",
+        "    run_every_steps=None)\n",
+        "\n",
+        "train_spec = tf1.estimator.TrainSpec(\n",
+        "    input_fn=_input_fn,\n",
+        "    hooks=[early_stopping_hook])\n",
+        "\n",
+        "eval_spec = tf1.estimator.EvalSpec(input_fn=_eval_input_fn)\n",
+        "\n",
+        "tf1.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "### TensorFlow 2: Early stopping with a built-in callback and Model.fit"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GKwxnkIksPFW"
+      },
+      "source": [
+        "Prepare the MNIST dataset and a simple Keras model:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "atVciNgPs0fw"
+      },
+      "outputs": [],
+      "source": [
+        "(ds_train, ds_test), ds_info = tfds.load(\n",
+        "    'mnist',\n",
+        "    split=['train', 'test'],\n",
+        "    shuffle_files=True,\n",
+        "    as_supervised=True,\n",
+        "    with_info=True,\n",
+        ")\n",
+        "\n",
+        "ds_train = ds_train.map(\n",
+        "    normalize_img, num_parallel_calls=tf.data.AUTOTUNE)\n",
+        "ds_train = ds_train.batch(128)\n",
+        "\n",
+        "ds_test = ds_test.map(\n",
+        "    normalize_img, num_parallel_calls=tf.data.AUTOTUNE)\n",
+        "ds_test = ds_test.batch(128)\n",
+        "\n",
+        "model = tf.keras.models.Sequential([\n",
+        "  tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
+        "  tf.keras.layers.Dense(128, activation='relu'),\n",
+        "  tf.keras.layers.Dense(10)\n",
+        "])\n",
+        "\n",
+        "model.compile(\n",
+        "    optimizer=tf.keras.optimizers.Adam(0.005),\n",
+        "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+        "    metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "559Goxp3tOMl"
+      },
+      "source": [
+        "In TensorFlow 2, when you use the built-in Keras `Model.fit` (or `Model.evaluate`), you can configure early stopping by passing a built-in callback—`tf.keras.callbacks.EarlyStopping`—to the `callbacks` parameter of `Model.fit`.\n",
+        "\n",
+        "The `EarlyStopping` callback monitors a user-specified metric and ends training when it stops improving. (Check the [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate#using_callbacks) or the [API docs](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping) for more information.)\n",
+        "\n",
+        "Below is an example of an early stopping callback that monitors the loss and stops training after the number of epochs that show no improvements is set to `3` (`patience`): "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kip65sYBlKiu"
+      },
+      "outputs": [],
+      "source": [
+        "callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\n",
+        "\n",
+        "# Only around 25 epochs are run during training, instead of 100.\n",
+        "history = model.fit(\n",
+        "    ds_train,\n",
+        "    epochs=100,\n",
+        "    validation_data=ds_test,\n",
+        "    callbacks=[callback]\n",
+        ")\n",
+        "\n",
+        "len(history.history['loss'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a92c6ebb1a1c"
+      },
+      "source": [
+        "### TensorFlow 2: Early stopping with a custom callback and Model.fit"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wCwZ4BA8jaHY"
+      },
+      "source": [
+        "You can also implement a [custom early stopping callback](https://www.tensorflow.org/guide/keras/custom_callback/#early_stopping_at_minimum_loss), which can also be passed to the `callbacks` parameter of `Model.fit` (or `Model.evaluate`).\n",
+        "\n",
+        "In this example, the training process is stopped once `self.model.stop_training` is set to be `True`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Hns1fmwtjCg2"
+      },
+      "outputs": [],
+      "source": [
+        "class LimitTrainingTime(tf.keras.callbacks.Callback):\n",
+        "  def __init__(self, max_time_s):\n",
+        "    super().__init__()\n",
+        "    self.max_time_s = max_time_s\n",
+        "    self.start_time = None\n",
+        "\n",
+        "  def on_train_begin(self, logs):\n",
+        "    self.start_time = time.time()\n",
+        "\n",
+        "  def on_train_batch_end(self, batch, logs):\n",
+        "    now = time.time()\n",
+        "    if now - self.start_time >  self.max_time_s:\n",
+        "      self.model.stop_training = True"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "s5mIzDOAkUKA"
+      },
+      "outputs": [],
+      "source": [
+        "# Limit the training time to 30 seconds.\n",
+        "callback = LimitTrainingTime(30)\n",
+        "history = model.fit(\n",
+        "    ds_train,\n",
+        "    epochs=100,\n",
+        "    validation_data=ds_test,\n",
+        "    callbacks=[callback]\n",
+        ")\n",
+        "len(history.history['loss'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kro_lKyEu60-"
+      },
+      "source": [
+        "## TensorFlow 2: Early stopping with a custom training loop"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g5LU0lebvuIk"
+      },
+      "source": [
+        "In TensorFlow 2, you can implement early stopping in a [custom training loop](https://www.tensorflow.org/tutorials/customization/custom_training_walkthrough#training_loop) if you're not training and evaluating with the [built-in Keras methods](https://www.tensorflow.org/guide/keras/train_and_evaluate).\n",
+        "\n",
+        "Start by using Keras APIs to define another simple model, an optimizer, a loss function, and metrics:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oTGxr0PwAiQ4"
+      },
+      "outputs": [],
+      "source": [
+        "model = tf.keras.models.Sequential([\n",
+        "  tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
+        "  tf.keras.layers.Dense(128, activation='relu'),\n",
+        "  tf.keras.layers.Dense(10)\n",
+        "])\n",
+        "\n",
+        "optimizer = tf.keras.optimizers.Adam(0.005)\n",
+        "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
+        "\n",
+        "train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()\n",
+        "train_loss_metric = tf.keras.metrics.SparseCategoricalCrossentropy()\n",
+        "val_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()\n",
+        "val_loss_metric = tf.keras.metrics.SparseCategoricalCrossentropy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zecsnqRxvy0Q"
+      },
+      "source": [
+        "Define the parameter update functions [with tf.GradientTape](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) and the `@tf.function` decorator [for a speedup](https://www.tensorflow.org/guide/function):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "s3w_55n0Ah7L"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def train_step(x, y):\n",
+        "  with tf.GradientTape() as tape:\n",
+        "      logits = model(x, training=True)\n",
+        "      loss_value = loss_fn(y, logits)\n",
+        "  grads = tape.gradient(loss_value, model.trainable_weights)\n",
+        "  optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
+        "  train_acc_metric.update_state(y, logits)\n",
+        "  train_loss_metric.update_state(y, logits)\n",
+        "  return loss_value\n",
+        "\n",
+        "@tf.function\n",
+        "def test_step(x, y):\n",
+        "  logits = model(x, training=False)\n",
+        "  val_acc_metric.update_state(y, logits)\n",
+        "  val_loss_metric.update_state(y, logits)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-ZKS9ePGwd9r"
+      },
+      "source": [
+        "Next, write a custom training loop, where you can implement your early stopping rule manually.\n",
+        "\n",
+        "The example below shows how to stop training when the validation loss doesn't improve over a certain number of epochs:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iZOzHqqSAkpK"
+      },
+      "outputs": [],
+      "source": [
+        "epochs = 100\n",
+        "patience = 5\n",
+        "wait = 0\n",
+        "best = float('inf')\n",
+        "\n",
+        "for epoch in range(epochs):\n",
+        "    print(\"\\nStart of epoch %d\" % (epoch,))\n",
+        "    start_time = time.time()\n",
+        "\n",
+        "    for step, (x_batch_train, y_batch_train) in enumerate(ds_train):\n",
+        "      loss_value = train_step(x_batch_train, y_batch_train)\n",
+        "      if step % 200 == 0:\n",
+        "        print(\"Training loss at step %d: %.4f\" % (step, loss_value.numpy()))\n",
+        "        print(\"Seen so far: %s samples\" % ((step + 1) * 128))        \n",
+        "    train_acc = train_acc_metric.result()\n",
+        "    train_loss = train_loss_metric.result()\n",
+        "    train_acc_metric.reset_states()\n",
+        "    train_loss_metric.reset_states()\n",
+        "    print(\"Training acc over epoch: %.4f\" % (train_acc.numpy()))\n",
+        "\n",
+        "    for x_batch_val, y_batch_val in ds_test:\n",
+        "      test_step(x_batch_val, y_batch_val)\n",
+        "    val_acc = val_acc_metric.result()\n",
+        "    val_loss = val_loss_metric.result()\n",
+        "    val_acc_metric.reset_states()\n",
+        "    val_loss_metric.reset_states()\n",
+        "    print(\"Validation acc: %.4f\" % (float(val_acc),))\n",
+        "    print(\"Time taken: %.2fs\" % (time.time() - start_time))\n",
+        "\n",
+        "    # The early stopping strategy: stop the training if `val_loss` does not\n",
+        "    # decrease over a certain number of epochs.\n",
+        "    wait += 1\n",
+        "    if val_loss < best:\n",
+        "      best = val_loss\n",
+        "      wait = 0\n",
+        "    if wait >= patience:\n",
+        "      break"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e85558980a4b"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "- Learn more about the Keras built-in early stopping callback API in the [API docs](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping).\n",
+        "- Learn to [write custom Keras callbacks](https://www.tensorflow.org/guide/keras/custom_callback), including [early stopping at a minimum loss](https://www.tensorflow.org/guide/keras/custom_callback/#early_stopping_at_minimum_loss).\n",
+        "- Learn about [Training and evaluation with the Keras built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate#using_callbacks).\n",
+        "- Explore common regularization techniques in the [Overfit and underfit](tensorflow.org/tutorials/keras/overfit_and_underfit) tutorial that uses the `EarlyStopping` callback."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "early_stopping.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/evaluator.ipynb b/site/en/guide/migrate/evaluator.ipynb
new file mode 100644
index 00000000000..c8f848e4406
--- /dev/null
+++ b/site/en/guide/migrate/evaluator.ipynb
@@ -0,0 +1,292 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate evaluation\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n4O6fPyYTxZv"
+      },
+      "source": [
+        "Evaluation is a critical part of measuring and benchmarking models.\n",
+        "\n",
+        "This guide demonstrates how to migrate evaluator tasks from TensorFlow 1 to  TensorFlow 2. In Tensorflow 1  this functionality is implemented by `tf.estimator.train_and_evaluate`, when the API is running distributedly. In Tensorflow 2, you can use the built-in `tf.keras.utils.SidecarEvaluator`, or a custom evaluation loop on the evaluator task.\n",
+        "\n",
+        "There are simple serial evaluation options in both TensorFlow 1 (`tf.estimator.Estimator.evaluate`) and TensorFlow 2 (`Model.fit(..., validation_data=(...))` or `Model.evaluate`). The evaluator task is preferable when you would like your workers not switching between training and evaluation, and built-in evaluation in `Model.fit` is preferable when you would like your evaluation to be distributed.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pHJfmkCFUhQf"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VXnPvQi8Ui1F"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow.compat.v1 as tf1\n",
+        "import tensorflow as tf\n",
+        "import numpy as np\n",
+        "import tempfile\n",
+        "import time\n",
+        "import os"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Tww-uIoiUlsT"
+      },
+      "outputs": [],
+      "source": [
+        "mnist = tf.keras.datasets.mnist\n",
+        "\n",
+        "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
+        "x_train, x_test = x_train / 255.0, x_test / 255.0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TtlucRG_Uro_"
+      },
+      "source": [
+        "## TensorFlow 1: Evaluating using tf.estimator.train_and_evaluate\n",
+        "\n",
+        "In TensorFlow 1, you can configure a `tf.estimator` to evaluate the estimator using `tf.estimator.train_and_evaluate`.\n",
+        "\n",
+        "In this example, start by defining the `tf.estimator.Estimator` and specifying training and evaluation specifications:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q8shCkV2jKcc"
+      },
+      "outputs": [],
+      "source": [
+        "feature_columns = [tf1.feature_column.numeric_column(\"x\", shape=[28, 28])]\n",
+        "\n",
+        "classifier = tf1.estimator.DNNClassifier(\n",
+        "    feature_columns=feature_columns,\n",
+        "    hidden_units=[256, 32],\n",
+        "    optimizer=tf1.train.AdamOptimizer(0.001),\n",
+        "    n_classes=10,\n",
+        "    dropout=0.2\n",
+        ")\n",
+        "\n",
+        "train_input_fn = tf1.estimator.inputs.numpy_input_fn(\n",
+        "    x={\"x\": x_train},\n",
+        "    y=y_train.astype(np.int32),\n",
+        "    num_epochs=10,\n",
+        "    batch_size=50,\n",
+        "    shuffle=True,\n",
+        ")\n",
+        "\n",
+        "test_input_fn = tf1.estimator.inputs.numpy_input_fn(\n",
+        "    x={\"x\": x_test},\n",
+        "    y=y_test.astype(np.int32),\n",
+        "    num_epochs=10,\n",
+        "    shuffle=False\n",
+        ")\n",
+        "\n",
+        "train_spec = tf1.estimator.TrainSpec(input_fn=train_input_fn, max_steps=10)\n",
+        "eval_spec = tf1.estimator.EvalSpec(input_fn=test_input_fn,\n",
+        "                                   steps=10,\n",
+        "                                   throttle_secs=0)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sGP7Nyenk1gr"
+      },
+      "source": [
+        "Then, train and evaluate the model. The evaluation runs synchronously between training because it's limited as a local run in this notebook and alternates between training and evaluation. However, if the estimator is used distributedly, the evaluator will run as a dedicated evaluator task. For more information, check the [migration guide on distributed training](https://www.tensorflow.org/guide/migrate/multi_worker_cpu_gpu_training)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xWKMsmt6jYSL"
+      },
+      "outputs": [],
+      "source": [
+        "tf1.estimator.train_and_evaluate(estimator=classifier,\n",
+        "                                train_spec=train_spec,\n",
+        "                                eval_spec=eval_spec)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "T5LtVtmvYx7J"
+      },
+      "source": [
+        "## TensorFlow 2: Evaluating a Keras model\n",
+        "\n",
+        "In TensorFlow 2, if you use the Keras `Model.fit` API for training, you can evaluate the model with `tf.keras.utils.SidecarEvaluator`. You can also visualize the evaluation metrics in TensorBoard which is not shown in this guide.\n",
+        "\n",
+        "To help demonstrate this, let's first start by defining and training the model:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ci3yB6A5lwJu"
+      },
+      "outputs": [],
+      "source": [
+        "def create_model():\n",
+        "  return tf.keras.models.Sequential([\n",
+        "    tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
+        "    tf.keras.layers.Dense(512, activation='relu'),\n",
+        "    tf.keras.layers.Dropout(0.2),\n",
+        "    tf.keras.layers.Dense(10)\n",
+        "  ])\n",
+        "\n",
+        "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
+        "\n",
+        "model = create_model()\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss=loss,\n",
+        "              metrics=['accuracy'],\n",
+        "              steps_per_execution=10,\n",
+        "              run_eagerly=True)\n",
+        "\n",
+        "log_dir = tempfile.mkdtemp()\n",
+        "model_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
+        "    filepath=os.path.join(log_dir, 'ckpt-{epoch}'),\n",
+        "    save_weights_only=True)\n",
+        "\n",
+        "model.fit(x=x_train,\n",
+        "          y=y_train,\n",
+        "          epochs=1,\n",
+        "          callbacks=[model_checkpoint])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AhU3VTYZoDh-"
+      },
+      "source": [
+        "Then, evaluate the model using `tf.keras.utils.SidecarEvaluator`. In real training, it's recommended to use a separate job to conduct the evaluation to free up worker resources for training."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1VOQLDNkl2bl"
+      },
+      "outputs": [],
+      "source": [
+        "data = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
+        "data = data.batch(64)\n",
+        "\n",
+        "tf.keras.utils.SidecarEvaluator(\n",
+        "    model=model,\n",
+        "    data=data,\n",
+        "    checkpoint_dir=log_dir,\n",
+        "    max_evaluations=1\n",
+        ").start()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rQUS8nO9FZlH"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "- To learn more about sidecar evaluation consider reading the `tf.keras.utils.SidecarEvaluator` API docs.\n",
+        "- To consider alternating training and evaluation in Keras consider reading about [other built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "evaluator.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/fault_tolerance.ipynb b/site/en/guide/migrate/fault_tolerance.ipynb
new file mode 100644
index 00000000000..fdbd0b972c3
--- /dev/null
+++ b/site/en/guide/migrate/fault_tolerance.ipynb
@@ -0,0 +1,600 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate the fault tolerance mechanism\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "n4O6fPyYTxZv"
+      },
+      "source": [
+        "Fault tolerance refers to a mechanism of periodically saving the states of trackable objects, such as parameters and models. This enables you to recover  them in the event of a program/machine failure during training.\n",
+        "\n",
+        "This guide first demonstrates how to add fault tolerance to training with `tf.estimator.Estimator` in TensorFlow 1 by specifying metric saving with `tf.estimator.RunConfig`. Then, you will learn how to implement fault tolerance for training in Tensorflow 2 in two ways:\n",
+        "\n",
+        "- If you use the Keras `Model.fit` API, you can pass the `tf.keras.callbacks.BackupAndRestore` callback to it.\n",
+        "- If you use a custom training loop (with `tf.GradientTape`), you can arbitrarily save checkpoints using the `tf.train.Checkpoint` and `tf.train.CheckpointManager` APIs.\n",
+        "\n",
+        "Both of these methods will back up and restore the training states in [checkpoint](../../guide/checkpoint.ipynb) files.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pHJfmkCFUhQf"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TOVQubuDzdmA"
+      },
+      "source": [
+        "Install `tf-nightly`, as the frequency of checkpoint saving at a particular step with the `save_freq` argument in `tf.keras.callbacks.BackupAndRestore` is introduced from TensorFlow 2.10:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pGW0XhXkxY_q"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install tf-nightly"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VXnPvQi8Ui1F"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow.compat.v1 as tf1\n",
+        "import tensorflow as tf\n",
+        "import numpy as np\n",
+        "import tempfile\n",
+        "import time"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Tww-uIoiUlsT"
+      },
+      "outputs": [],
+      "source": [
+        "mnist = tf.keras.datasets.mnist\n",
+        "\n",
+        "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
+        "x_train, x_test = x_train / 255.0, x_test / 255.0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TtlucRG_Uro_"
+      },
+      "source": [
+        "## TensorFlow 1: Save checkpoints with `tf.estimator.RunConfig`\n",
+        "\n",
+        "In TensorFlow 1, you can configure a `tf.estimator` to save checkpoints every step by configuring `tf.estimator.RunConfig`.\n",
+        "\n",
+        "In this example, start by writing a hook that artificially throws an error during the fifth checkpoint:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q8shCkV2jKcc"
+      },
+      "outputs": [],
+      "source": [
+        "class InterruptHook(tf1.train.SessionRunHook):\n",
+        "  # A hook for artificially interrupting training.\n",
+        "  def begin(self):\n",
+        "    self._step = -1\n",
+        "\n",
+        "  def before_run(self, run_context):\n",
+        "    self._step += 1\n",
+        "\n",
+        "  def after_run(self, run_context, run_values):\n",
+        "    if self._step == 5:\n",
+        "      raise RuntimeError('Interruption')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZXbQ6cFlkoIM"
+      },
+      "source": [
+        "Next, configure `tf.estimator.Estimator` to save every checkpoint and use the MNIST dataset:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1EKXzi4Qj2Eb"
+      },
+      "outputs": [],
+      "source": [
+        "feature_columns = [tf1.feature_column.numeric_column(\"x\", shape=[28, 28])]\n",
+        "config = tf1.estimator.RunConfig(save_summary_steps=1,\n",
+        "                                 save_checkpoints_steps=1)\n",
+        "\n",
+        "path = tempfile.mkdtemp()\n",
+        "\n",
+        "classifier = tf1.estimator.DNNClassifier(\n",
+        "    feature_columns=feature_columns,\n",
+        "    hidden_units=[256, 32],\n",
+        "    optimizer=tf1.train.AdamOptimizer(0.001),\n",
+        "    n_classes=10,\n",
+        "    dropout=0.2,\n",
+        "    model_dir=path,\n",
+        "    config = config\n",
+        ")\n",
+        "\n",
+        "train_input_fn = tf1.estimator.inputs.numpy_input_fn(\n",
+        "    x={\"x\": x_train},\n",
+        "    y=y_train.astype(np.int32),\n",
+        "    num_epochs=10,\n",
+        "    batch_size=50,\n",
+        "    shuffle=True,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sGP7Nyenk1gr"
+      },
+      "source": [
+        "Begin training the model. An artificial exception will be raised by the hook you defined earlier."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xWKMsmt6jYSL"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  classifier.train(input_fn=train_input_fn,\n",
+        "                   hooks=[InterruptHook()],\n",
+        "                   max_steps=10)\n",
+        "except Exception as e:\n",
+        "  print(f'{type(e).__name__}:{e}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DekxJkgWk-4N"
+      },
+      "source": [
+        "Rebuild the `tf.estimator.Estimator` using the last saved checkpoint and continue training:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vqMVTiJMjcH7"
+      },
+      "outputs": [],
+      "source": [
+        "classifier = tf1.estimator.DNNClassifier(\n",
+        "    feature_columns=feature_columns,\n",
+        "    hidden_units=[256, 32],\n",
+        "    optimizer=tf1.train.AdamOptimizer(0.001),\n",
+        "    n_classes=10,\n",
+        "    dropout=0.2,\n",
+        "    model_dir=path,\n",
+        "    config = config\n",
+        ")\n",
+        "classifier.train(input_fn=train_input_fn,\n",
+        "                   max_steps = 10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "T5LtVtmvYx7J"
+      },
+      "source": [
+        "## TensorFlow 2: Back up and restore with a callback and `Model.fit`\n",
+        "\n",
+        "In TensorFlow 2, if you use the Keras `Model.fit` API for training, you can provide the `tf.keras.callbacks.BackupAndRestore` callback to add the fault tolerance functionality.\n",
+        "\n",
+        "To help demonstrate this, first start by defining a Keras `Callback` class that artificially throws an error during the fourth epoch checkpoint:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ci3yB6A5lwJu"
+      },
+      "outputs": [],
+      "source": [
+        "class InterruptAtEpoch(tf.keras.callbacks.Callback):\n",
+        "  # A callback for artificially interrupting training.\n",
+        "  def __init__(self, interrupting_epoch=3):\n",
+        "    self.interrupting_epoch = interrupting_epoch\n",
+        "\n",
+        "  def on_epoch_end(self, epoch, log=None):\n",
+        "    if epoch == self.interrupting_epoch:\n",
+        "      raise RuntimeError('Interruption')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AhU3VTYZoDh-"
+      },
+      "source": [
+        "Then, define and instantiate a simple Keras model, define the loss function, call `Model.compile`, and set up a `tf.keras.callbacks.BackupAndRestore` callback that will save the checkpoints in a temporary directory at epoch boundaries:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1VOQLDNkl2bl"
+      },
+      "outputs": [],
+      "source": [
+        "def create_model():\n",
+        "  return tf.keras.models.Sequential([\n",
+        "    tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
+        "    tf.keras.layers.Dense(512, activation='relu'),\n",
+        "    tf.keras.layers.Dropout(0.2),\n",
+        "    tf.keras.layers.Dense(10)\n",
+        "  ])\n",
+        "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
+        "model = create_model()\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss=loss,\n",
+        "              metrics=['accuracy'])\n",
+        "log_dir = tempfile.mkdtemp()\n",
+        "backup_restore_callback = tf.keras.callbacks.BackupAndRestore(\n",
+        "    backup_dir = log_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LRRWmZqsvMrq"
+      },
+      "source": [
+        "Start training the model with `Model.fit`. During training, checkpoints will be saved thanks to `tf.keras.callbacks.BackupAndRestore` instantiated above, while the `InterruptAtEpoch` class will raise an artificial exception to simulate a failure after the fourth epoch."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8bVO79qWl4Uv"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  model.fit(x=x_train,\n",
+        "            y=y_train,\n",
+        "            epochs=10,\n",
+        "            steps_per_epoch=100,\n",
+        "            validation_data=(x_test, y_test),\n",
+        "            callbacks=[backup_restore_callback, InterruptAtEpoch()])\n",
+        "except Exception as e:\n",
+        "  print(f'{type(e).__name__}:{e}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EWidh234vcRf"
+      },
+      "source": [
+        "Next, instantiate the Keras model, call `Model.compile`, and continue training the model with `Model.fit` from a previously saved checkpoint:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3IWPH0Cmn2wi"
+      },
+      "outputs": [],
+      "source": [
+        "model = create_model()\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss=loss,\n",
+        "              metrics=['accuracy'],\n",
+        "              steps_per_execution=10)\n",
+        "model.fit(x=x_train,\n",
+        "            y=y_train,\n",
+        "            epochs=10,\n",
+        "            steps_per_epoch=100,\n",
+        "            validation_data=(x_test, y_test),\n",
+        "            callbacks=[backup_restore_callback])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nP2dnpMPxtYj"
+      },
+      "source": [
+        "Define another `Callback` class that artificially throws an error during the 140th step:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YardkAaBxr-c"
+      },
+      "outputs": [],
+      "source": [
+        "class InterruptAtStep(tf.keras.callbacks.Callback):\n",
+        "  # A callback for artificially interrupting training.\n",
+        "  def __init__(self, interrupting_step=140):\n",
+        "    self.total_step_count = 0\n",
+        "    self.interrupting_step = interrupting_step\n",
+        "\n",
+        "  def on_batch_begin(self, batch, logs=None):\n",
+        "    self.total_step_count += 1\n",
+        "\n",
+        "  def on_batch_end(self, batch, logs=None):\n",
+        "    if self.total_step_count == self.interrupting_step:\n",
+        "      print(\"\\nInterrupting at step count\", self.total_step_count)\n",
+        "      raise RuntimeError('Interruption')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Af3VpehxyTpb"
+      },
+      "source": [
+        "Note: This section uses features that are only available in `tf-nightly` until Tensorflow 2.10 is released.\n",
+        "\n",
+        "To make sure the checkpoints are saved every 30 steps, set the `save_freq` in the `BackupAndRestore` callback to `30`. The `InterruptAtStep` will raise an artificial exception to simulate a failure at epoch 1 and step 40 (total step count 140). The checkpoint would be last saved at epoch 1 and step 20."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dHHCENDPyUHS"
+      },
+      "outputs": [],
+      "source": [
+        "log_dir_2 = tempfile.mkdtemp()\n",
+        "\n",
+        "backup_restore_callback = tf.keras.callbacks.BackupAndRestore(\n",
+        "    backup_dir = log_dir_2, save_freq=30\n",
+        ")\n",
+        "model = create_model()\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss=loss,\n",
+        "              metrics=['accuracy'])\n",
+        "try:\n",
+        "  model.fit(x=x_train,\n",
+        "            y=y_train,\n",
+        "            epochs=10,\n",
+        "            steps_per_epoch=100,\n",
+        "            validation_data=(x_test, y_test),\n",
+        "            callbacks=[backup_restore_callback, InterruptAtStep()])\n",
+        "except Exception as e:\n",
+        "  print(f'{type(e).__name__}:{e}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2-ggMFEHynMR"
+      },
+      "source": [
+        "Next, instantiate the Keras model, call `Model.compile`, and continue training the model with `Model.fit` from a previously saved checkpoint. Notice that the training starts from epoch 2 and step 21."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vT7Kx30NEqly"
+      },
+      "outputs": [],
+      "source": [
+        "model = create_model()\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss=loss,\n",
+        "              metrics=['accuracy'],\n",
+        "              steps_per_execution=10)\n",
+        "model.fit(x=x_train,\n",
+        "            y=y_train,\n",
+        "            epochs=10,\n",
+        "            steps_per_epoch=100,\n",
+        "            validation_data=(x_test, y_test),\n",
+        "            callbacks=[backup_restore_callback])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OdWexHUUaEB6"
+      },
+      "source": [
+        "## TensorFlow 2: Write manual checkpoints with a custom training loop\n",
+        "\n",
+        "If you use a custom training loop in TensorFlow 2, you can implement a fault tolerance mechanism with the `tf.train.Checkpoint` and `tf.train.CheckpointManager` APIs.\n",
+        "\n",
+        "This example demonstrates how to:\n",
+        "\n",
+        "- Use a `tf.train.Checkpoint` object to manually create a checkpoint, where the trackable objects you want to save are set as attributes.\n",
+        "- Use a `tf.train.CheckpointManager` to manage multiple checkpoints.\n",
+        "\n",
+        "Start by defining and instantiating the Keras model, the optimizer, and the loss function. Then, create a `Checkpoint` that manages two objects with trackable states (the model and the optimizer), as well as a `CheckpointManager` for logging and keeping several checkpoints in a temporary directory."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hPnIRKC8aDwE"
+      },
+      "outputs": [],
+      "source": [
+        "model = create_model()\n",
+        "optimizer = tf.keras.optimizers.SGD(learning_rate=0.001)\n",
+        "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
+        "log_dir = tempfile.mkdtemp()\n",
+        "epochs = 5\n",
+        "steps_per_epoch = 5\n",
+        "\n",
+        "checkpoint = tf.train.Checkpoint(model=model, optimizer=optimizer)\n",
+        "checkpoint_manager = tf.train.CheckpointManager(\n",
+        "            checkpoint, log_dir, max_to_keep=2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L2tK4fm6xNkJ"
+      },
+      "source": [
+        "Now, implement a custom training loop where after the first epoch every time a new epoch starts the last checkpoint is loaded:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GhQphF5jxPWU"
+      },
+      "outputs": [],
+      "source": [
+        "for epoch in range(epochs):\n",
+        "  if epoch > 0:\n",
+        "      tf.train.load_checkpoint(save_path)\n",
+        "  print(f\"\\nStart of epoch {epoch}\")\n",
+        "\n",
+        "  for step in range(steps_per_epoch):\n",
+        "    with tf.GradientTape() as tape:\n",
+        "\n",
+        "      logits = model(x_train, training=True)\n",
+        "      loss_value = loss_fn(y_train, logits)\n",
+        "\n",
+        "      grads = tape.gradient(loss_value, model.trainable_weights)\n",
+        "      optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
+        "\n",
+        "    save_path = checkpoint_manager.save()\n",
+        "    print(f\"Checkpoint saved to {save_path}\")\n",
+        "    print(f\"Training loss at step {step}: {loss_value}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rQUS8nO9FZlH"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "To learn more about fault tolerance and checkpointing in TensorFlow 2, consider the following documentation:\n",
+        "\n",
+        "- The `tf.keras.callbacks.BackupAndRestore` callback API docs.\n",
+        "- The `tf.train.Checkpoint` and `tf.train.CheckpointManager` API docs.\n",
+        "- The [Training checkpoints](../../guide/checkpoint.ipynb) guide, including the _Writing checkpoints_ section.\n",
+        "\n",
+        "You may also find the following material related to [distributed training](../..guide/distributed_training.ipynb) useful:\n",
+        "\n",
+        "- The _Fault tolerance_ section in the [Multi-worker training with Keras](../../tutorials/distribute/multi_worker_with_keras.ipynb) tutorial.\n",
+        "- The _Handing task failure_ section in the [Parameter server training](../../tutorials/distribute/parameter_server_training.ipynb) tutorial."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "fault_tolerance.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/images/tensorboard_TF1.png b/site/en/guide/migrate/images/tensorboard_TF1.png
new file mode 100644
index 00000000000..294fbbcc5b5
Binary files /dev/null and b/site/en/guide/migrate/images/tensorboard_TF1.png differ
diff --git a/site/en/guide/migrate/images/tensorboard_TF2.png b/site/en/guide/migrate/images/tensorboard_TF2.png
new file mode 100644
index 00000000000..bbad8768210
Binary files /dev/null and b/site/en/guide/migrate/images/tensorboard_TF2.png differ
diff --git a/site/en/guide/migrate/logging_stop_hook.ipynb b/site/en/guide/migrate/logging_stop_hook.ipynb
new file mode 100644
index 00000000000..a1d60243c37
--- /dev/null
+++ b/site/en/guide/migrate/logging_stop_hook.ipynb
@@ -0,0 +1,303 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate LoggingTensorHook and StopAtStepHook to Keras callbacks\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "In TensorFlow 1, you use `tf.estimator.LoggingTensorHook` to monitor and log tensors, while `tf.estimator.StopAtStepHook` helps stop training at a specified step when training with `tf.estimator.Estimator`. This notebook demonstrates how to migrate from these APIs to their equivalents in TensorFlow 2 using custom Keras callbacks (`tf.keras.callbacks.Callback`) with `Model.fit`.\n",
+        "\n",
+        "Keras [callbacks](https://www.tensorflow.org/guide/keras/custom_callback) are objects that are called at different points during training/evaluation/prediction in the built-in Keras `Model.fit`/`Model.evaluate`/`Model.predict` APIs. You can learn more about callbacks in the `tf.keras.callbacks.Callback` API docs, as well as the [Writing your own callbacks](../..guide/keras/custom_callback.ipynb/) and [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate) (the *Using callbacks* section) guides. For migrating from `SessionRunHook` in TensorFlow 1 to Keras callbacks in TensorFlow 2, check out the [Migrate training with assisted logic](sessionrunhook_callback.ipynb) guide."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Start with imports and a simple dataset for demonstration purposes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m7rnGxsXtDkV"
+      },
+      "outputs": [],
+      "source": [
+        "features = [[1., 1.5], [2., 2.5], [3., 3.5]]\n",
+        "labels = [[0.3], [0.5], [0.7]]\n",
+        "\n",
+        "# Define an input function.\n",
+        "def _input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices((features, labels)).batch(1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4uXff1BEssdE"
+      },
+      "source": [
+        "## TensorFlow 1: Log tensors and stop training with tf.estimator APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zW-X5cmzmkuw"
+      },
+      "source": [
+        "In TensorFlow 1, you define various hooks to control the training behavior. Then, you pass these hooks to `tf.estimator.EstimatorSpec`.\n",
+        "\n",
+        "In the example below:\n",
+        "\n",
+        "- To monitor/log tensors—for example, model weights or losses—you use `tf.estimator.LoggingTensorHook` (`tf.train.LoggingTensorHook` is its alias).\n",
+        "- To stop training at a specific step, you use `tf.estimator.StopAtStepHook` (`tf.train.StopAtStepHook` is its alias)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "def _model_fn(features, labels, mode):\n",
+        "  dense = tf1.layers.Dense(1)\n",
+        "  logits = dense(features)\n",
+        "  loss = tf1.losses.mean_squared_error(labels=labels, predictions=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.05)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "\n",
+        "  # Define the stop hook.\n",
+        "  stop_hook = tf1.train.StopAtStepHook(num_steps=2)\n",
+        "\n",
+        "  # Access tensors to be logged by names.\n",
+        "  kernel_name = tf.identity(dense.weights[0])\n",
+        "  bias_name = tf.identity(dense.weights[1])\n",
+        "  logging_weight_hook = tf1.train.LoggingTensorHook(\n",
+        "      tensors=[kernel_name, bias_name],\n",
+        "      every_n_iter=1)\n",
+        "  # Log the training loss by the tensor object.\n",
+        "  logging_loss_hook = tf1.train.LoggingTensorHook(\n",
+        "      {'loss from LoggingTensorHook': loss},\n",
+        "      every_n_secs=3)\n",
+        "\n",
+        "  # Pass all hooks to `EstimatorSpec`.\n",
+        "  return tf1.estimator.EstimatorSpec(mode,\n",
+        "                                     loss=loss,\n",
+        "                                     train_op=train_op,\n",
+        "                                     training_hooks=[stop_hook,\n",
+        "                                                     logging_weight_hook,\n",
+        "                                                     logging_loss_hook])\n",
+        "\n",
+        "estimator = tf1.estimator.Estimator(model_fn=_model_fn)\n",
+        "\n",
+        "# Begin training.\n",
+        "# The training will stop after 2 steps, and the weights/loss will also be logged.\n",
+        "estimator.train(_input_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "## TensorFlow 2: Log tensors and stop training with custom callbacks and Model.fit"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "839R9i4xheI5"
+      },
+      "source": [
+        "In TensorFlow 2, when you use the built-in Keras `Model.fit` (or `Model.evaluate`) for training/evaluation, you can configure tensor monitoring and training stopping by defining custom Keras `tf.keras.callbacks.Callback`s. Then, you pass them to the `callbacks` parameter of `Model.fit` (or `Model.evaluate`). (Learn more in the [Writing your own callbacks](../..guide/keras/custom_callback.ipynb) guide.)\n",
+        "\n",
+        "In the example below:\n",
+        "\n",
+        "- To recreate the functionalities of `StopAtStepHook`, define a custom callback (named `StopAtStepCallback` below) where you override the `on_batch_end` method to stop training after a certain number of steps.\n",
+        "- To recreate the `LoggingTensorHook` behavior, define a custom callback (`LoggingTensorCallback`) where you record and output the logged tensors manually, since accessing to tensors by names is not supported. You can also implement the logging frequency inside the custom callback. The example below will print the weights every two steps. Other strategies like logging every N seconds are also possible."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "atVciNgPs0fw"
+      },
+      "outputs": [],
+      "source": [
+        "class StopAtStepCallback(tf.keras.callbacks.Callback):\n",
+        "  def __init__(self, stop_step=None):\n",
+        "    super().__init__()\n",
+        "    self._stop_step = stop_step\n",
+        "\n",
+        "  def on_batch_end(self, batch, logs=None):\n",
+        "    if self.model.optimizer.iterations >= self._stop_step:\n",
+        "      self.model.stop_training = True\n",
+        "      print('\\nstop training now')\n",
+        "\n",
+        "class LoggingTensorCallback(tf.keras.callbacks.Callback):\n",
+        "  def __init__(self, every_n_iter):\n",
+        "      super().__init__()\n",
+        "      self._every_n_iter = every_n_iter\n",
+        "      self._log_count = every_n_iter\n",
+        "\n",
+        "  def on_batch_end(self, batch, logs=None):\n",
+        "    if self._log_count > 0:\n",
+        "      self._log_count -= 1\n",
+        "      print(\"Logging Tensor Callback: dense/kernel:\",\n",
+        "            model.layers[0].weights[0])\n",
+        "      print(\"Logging Tensor Callback: dense/bias:\",\n",
+        "            model.layers[0].weights[1])\n",
+        "      print(\"Logging Tensor Callback loss:\", logs[\"loss\"])\n",
+        "    else:\n",
+        "      self._log_count -= self._every_n_iter"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "30a8b71263e0"
+      },
+      "source": [
+        "When finished, pass the new callbacks—`StopAtStepCallback` and `LoggingTensorCallback`—to the `callbacks` parameter of `Model.fit`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kip65sYBlKiu"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "model = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])\n",
+        "optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+        "model.compile(optimizer, \"mse\")\n",
+        "\n",
+        "# Begin training.\n",
+        "# The training will stop after 2 steps, and the weights/loss will also be logged.\n",
+        "model.fit(dataset, callbacks=[StopAtStepCallback(stop_step=2),\n",
+        "                              LoggingTensorCallback(every_n_iter=2)])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "19508f4720f5"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "Learn more about callbacks in:\n",
+        "\n",
+        "- API docs: `tf.keras.callbacks.Callback`\n",
+        "- Guide: [Writing your own callbacks](../..guide/keras/custom_callback.ipynb/)\n",
+        "- Guide: [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate) (the *Using callbacks* section)\n",
+        "\n",
+        "You may also find the following migration-related resources useful:\n",
+        "\n",
+        "- The [Early stopping migration guide](early_stopping.ipynb): `tf.keras.callbacks.EarlyStopping` is a built-in early stopping callback\n",
+        "- The [TensorBoard migration guide](tensorboard.ipynb): TensorBoard enables tracking and displaying metrics\n",
+        "- The [Training with assisted logic migration guide](sessionrunhook_callback.ipynb): From `SessionRunHook` in TensorFlow 1 to Keras callbacks in TensorFlow 2"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "logging_stop_hook.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/metrics_optimizers.ipynb b/site/en/guide/migrate/metrics_optimizers.ipynb
new file mode 100644
index 00000000000..61afb35aea6
--- /dev/null
+++ b/site/en/guide/migrate/metrics_optimizers.ipynb
@@ -0,0 +1,383 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate metrics and optimizers\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "In TF1, `tf.metrics` is the API namespace for all the metric functions. Each of the metrics is a function that takes `label` and `prediction` as input parameters and returns the corresponding metrics tensor as result. In TF2, `tf.keras.metrics` contains all the metric functions and objects. The `Metric` object can be used with `tf.keras.Model` and `tf.keras.layers.layer` to calculate metric values."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Let's start with a couple of necessary TensorFlow imports,"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Jsm9Rxx7s1OZ"
+      },
+      "source": [
+        "and prepare some simple data for demonstration:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m7rnGxsXtDkV"
+      },
+      "outputs": [],
+      "source": [
+        "features = [[1., 1.5], [2., 2.5], [3., 3.5]]\n",
+        "labels = [0, 0, 1]\n",
+        "eval_features = [[4., 4.5], [5., 5.5], [6., 6.5]]\n",
+        "eval_labels = [0, 1, 1]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xswk0d4xrFaQ"
+      },
+      "source": [
+        "## TF1: tf.compat.v1.metrics with Estimator\n",
+        "\n",
+        "In TF1, the metrics can be added to `EstimatorSpec` as the `eval_metric_ops`, and the op is generated via all the metrics functions defined in `tf.metrics`. You can follow the example to see how to use `tf.metrics.accuracy`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "def _input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "\n",
+        "def _eval_input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).batch(1)\n",
+        "\n",
+        "def _model_fn(features, labels, mode):\n",
+        "  logits = tf1.layers.Dense(2)(features)\n",
+        "  predictions = tf.math.argmax(input=logits, axis=1)\n",
+        "  loss = tf1.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.05)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "  accuracy = tf1.metrics.accuracy(labels=labels, predictions=predictions)\n",
+        "  return tf1.estimator.EstimatorSpec(mode, \n",
+        "                                     predictions=predictions,\n",
+        "                                     loss=loss, \n",
+        "                                     train_op=train_op,\n",
+        "                                     eval_metric_ops={'accuracy': accuracy})\n",
+        "\n",
+        "estimator = tf1.estimator.Estimator(model_fn=_model_fn)\n",
+        "estimator.train(_input_fn)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HsOpjW5plH9Q"
+      },
+      "outputs": [],
+      "source": [
+        "estimator.evaluate(_eval_input_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Wk4C6qA_OaQx"
+      },
+      "source": [
+        "Also, metrics could be added to estimator directly via `tf.estimator.add_metrics()`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B2lpLOh9Owma"
+      },
+      "outputs": [],
+      "source": [
+        "def mean_squared_error(labels, predictions):\n",
+        "  labels = tf.cast(labels, predictions.dtype)\n",
+        "  return {\"mean_squared_error\": \n",
+        "          tf1.metrics.mean_squared_error(labels=labels, predictions=predictions)}\n",
+        "\n",
+        "estimator = tf1.estimator.add_metrics(estimator, mean_squared_error)\n",
+        "estimator.evaluate(_eval_input_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "## TF2: Keras Metrics API with tf.keras.Model\n",
+        "\n",
+        "In TF2, `tf.keras.metrics` contains all the metrics classes and functions. They are designed in a OOP style and integrate closely with other `tf.keras` API. All the metrics can be found in `tf.keras.metrics` namespace, and there is usually a direct mapping between `tf.compat.v1.metrics` with `tf.keras.metrics`. \n",
+        "\n",
+        "In the following example, the metrics are added in `model.compile()` method. Users only need to create the metric instance, without specifying the label and prediction tensor. The Keras model will route the model output and label to the metrics object."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "atVciNgPs0fw"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "eval_dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).batch(1)\n",
+        "\n",
+        "inputs = tf.keras.Input((2,))\n",
+        "logits = tf.keras.layers.Dense(2)(inputs)\n",
+        "predictions = tf.math.argmax(input=logits, axis=1)\n",
+        "model = tf.keras.models.Model(inputs, predictions)\n",
+        "optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+        "\n",
+        "model.compile(optimizer, loss='mse', metrics=[tf.keras.metrics.Accuracy()])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kip65sYBlKiu"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(eval_dataset, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_mcGoCm_X1V0"
+      },
+      "source": [
+        "With eager execution enabled, `tf.keras.metrics.Metric` instances can be directly used to evaluate numpy data or eager tensors. `tf.keras.metrics.Metric` objects are stateful containers. The metric value can be updated via `metric.update_state(y_true, y_pred)`, and the result can be retrieved by `metrics.result()`.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TVGn5_IhYhtG"
+      },
+      "outputs": [],
+      "source": [
+        "accuracy = tf.keras.metrics.Accuracy()\n",
+        "\n",
+        "accuracy.update_state(y_true=[0, 0, 1, 1], y_pred=[0, 0, 0, 1])\n",
+        "accuracy.result().numpy()\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wQEV2hHtY_su"
+      },
+      "outputs": [],
+      "source": [
+        "accuracy.update_state(y_true=[0, 0, 1, 1], y_pred=[0, 0, 0, 0])\n",
+        "accuracy.update_state(y_true=[0, 0, 1, 1], y_pred=[1, 1, 0, 0])\n",
+        "accuracy.result().numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "E3F3ElcyadW-"
+      },
+      "source": [
+        "For more details about `tf.keras.metrics.Metric`, please take a look for the API documentation at `tf.keras.metrics.Metric`, as well as the [migration guide](https://www.tensorflow.org/guide/effective_tf2#new-style_metrics_and_losses)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eXKY9HEulxQC"
+      },
+      "source": [
+        "## Migrate TF1.x optimizers to Keras optimizers\n",
+        "\n",
+        "The optimizers in `tf.compat.v1.train`, such as the\n",
+        "[Adam optimizer](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/AdamOptimizer)\n",
+        "and the\n",
+        "[gradient descent optimizer](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/GradientDescentOptimizer),\n",
+        "have equivalents in `tf.keras.optimizers`.\n",
+        "\n",
+        "The table below summarizes how you can convert these legacy optimizers to their\n",
+        "Keras equivalents. You can directly replace the TF1.x version with the TF2\n",
+        "version unless additional steps (such as\n",
+        "[updating the default learning rate](../../guide/effective_tf2.ipynb#optimizer_defaults))\n",
+        "are required.\n",
+        "\n",
+        "Note that converting your optimizers\n",
+        "[may make old checkpoints incompatible](./migrating_checkpoints.ipynb).\n",
+        "\n",
+        "\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "
TF1.xTF2Additional steps
`tf.v1.train.GradientDescentOptimizer``tf.keras.optimizers.SGD`None
`tf.v1.train.MomentumOptimizer``tf.keras.optimizers.SGD`Include the `momentum` argument
`tf.v1.train.AdamOptimizer``tf.keras.optimizers.Adam`Rename `beta1` and `beta2` arguments to `beta_1` and `beta_2`
`tf.v1.train.RMSPropOptimizer``tf.keras.optimizers.RMSprop`Rename the `decay` argument to `rho`
`tf.v1.train.AdadeltaOptimizer``tf.keras.optimizers.Adadelta`None
`tf.v1.train.AdagradOptimizer``tf.keras.optimizers.Adagrad`None
`tf.v1.train.FtrlOptimizer``tf.keras.optimizers.Ftrl`Remove the `accum_name` and `linear_name` arguments
`tf.contrib.AdamaxOptimizer``tf.keras.optimizers.Adamax`Rename the `beta1`, and `beta2` arguments to `beta_1` and `beta_2`
`tf.contrib.Nadam``tf.keras.optimizers.Nadam`Rename the `beta1`, and `beta2` arguments to `beta_1` and `beta_2`
\n",
+        "\n",
+        "Note: In TF2, all epsilons (numerical stability constants) now default to `1e-7`\n",
+        "instead of `1e-8`. This difference is negligible in most use cases."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "metrics_optimizers.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/migrate_tf2.md b/site/en/guide/migrate/migrate_tf2.md
new file mode 100644
index 00000000000..91cef0a1c9f
--- /dev/null
+++ b/site/en/guide/migrate/migrate_tf2.md
@@ -0,0 +1,304 @@
+# TF1.x -> TF2 migration overview
+
+TensorFlow 2 is fundamentally different from TF1.x in several ways. You can
+still run unmodified TF1.x code
+([except for contrib](https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md))
+against TF2 binary installations like so:
+
+```python
+import tensorflow.compat.v1 as tf
+tf.disable_v2_behavior()
+```
+
+However, this is *not* running TF2 behaviors and APIs, and may not work as
+expected with code written for TF2. If you are not running with TF2 behaviors
+active, you are effectively running TF1.x on top of a TF2 installation. Read the
+[TF1 vs TF2 behaviors guide](./tf1_vs_tf2.ipynb) for more details on how TF2 is
+different from TF1.x.
+
+This guide provides an overview of the process to migrate your TF1.x code to
+TF2. This enables you to take advantage of new and future feature improvements
+and also make your code simpler, more performant, and easier to maintain.
+
+If you are using `tf.keras`'s high level APIs and training exclusively with
+`model.fit`, your code should more or less be fully compatible with TF2 except
+for the following caveats:
+
+-   TF2 has new
+    [default learning rates](../../guide/effective_tf2.ipynb#optimizer_defaults)
+    for Keras optimizers.
+-   TF2 [may have changed](../../guide/effective_tf2.ipynb#keras_metric_names)
+    the "name" that metrics are logged to.
+
+## TF2 migration process
+
+Before migrating, learn about the behavior and API differences between TF1.x and
+TF2 by reading the [guide](./tf1_vs_tf2.ipynb).
+
+1.  Run the automated script to convert some of your TF1.x API usage to
+    `tf.compat.v1`.
+2.  Remove old `tf.contrib` symbols (check
+    [TF Addons](https://github.com/tensorflow/addons) and
+    [TF-Slim](https://github.com/google-research/tf-slim)).
+3.  Make your TF1.x model forward passes run in TF2 with eager execution
+    enabled.
+4.  Upgrade your TF1.x code for training loops and saving/loading models to TF2
+    equivalents.
+5.  (Optional) Migrate your TF2-compatible `tf.compat.v1` APIs to idiomatic TF2
+    APIs.
+
+The following sections expand upon the steps outlined above.
+
+## Run the symbol conversion script
+
+This executes an initial pass at rewriting your code symbols to run against TF
+2.x binaries, but won't make your code idiomatic to TF 2.x nor will it
+automatically make your code compatible with TF2 behaviors.
+
+Your code will most likely still make use of `tf.compat.v1` endpoints to access
+placeholders, sessions, collections, and other TF1.x-style functionality.
+
+Read the [guide](./upgrade.ipynb) to find out more about the best practices for
+using the symbol conversion script.
+
+## Remove usage of `tf.contrib`
+
+The `tf.contrib` module has been sunsetted and several of its submodules have
+been integrated into the core TF2 API. The other submodules are now spun-off
+into other projects like [TF IO](https://github.com/tensorflow/io) and
+[TF Addons](https://www.tensorflow.org/addons/overview).
+
+A large amount of older TF1.x code uses the
+[Slim](https://ai.googleblog.com/2016/08/tf-slim-high-level-library-to-define.html)
+library, which was packaged with TF1.x as `tf.contrib.layers`. When migrating
+your Slim code to TF2, switch your Slim API usages to point to the
+[tf-slim pip package](https://pypi.org/project/tf-slim/). Then, read the
+[model mapping guide](https://tensorflow.org/guide/migrate/model_mapping#a_note_on_slim_and_contriblayers)
+to learn how to convert Slim code.
+
+Alternatively, if you use Slim pre-trained models you may consider trying out
+Keras's pre-traimed models from `tf.keras.applications` or
+[TF Hub](https://tfhub.dev/s?tf-version=tf2&q=slim)'s TF2 `SavedModel`s exported
+from the original Slim code.
+
+## Make TF1.x model forward passes run with TF2 behaviors enabled
+
+### Track variables and losses
+
+[TF2 does not support global collections.](./tf1_vs_tf2.ipynb#no_more_globals)
+
+Eager execution in TF2 does not support `tf.Graph` collection-based APIs. This
+affects how you construct and track variables.
+
+For new TF2 code you would use `tf.Variable` instead of `v1.get_variable` and
+use Python objects to collect and track variables instead of
+`tf.compat.v1.variable_scope`. Typically this would be one of:
+
+*   `tf.keras.layers.Layer`
+*   `tf.keras.Model`
+*   `tf.Module`
+
+Aggregate lists of variables (like
+`tf.Graph.get_collection(tf.GraphKeys.VARIABLES)`) with the `.variables` and
+`.trainable_variables` attributes of the `Layer`, `Module`, or `Model` objects.
+
+The `Layer` and `Model` classes implement several other properties that remove
+the need for global collections. Their `.losses` property can be a replacement
+for using the `tf.GraphKeys.LOSSES` collection.
+
+Read the [model mapping guide](./model_mapping.ipynb) to find out more about
+using the TF2 code modeling shims to embed your existing `get_variable` and
+`variable_scope` based code inside of `Layers`, `Models`, and `Modules`. This
+will let you the execute forward passes with eager execution enabled without
+major rewrites.
+
+### Adapting to other behavior changes
+
+If the [model mapping guide](./model_mapping.ipynb) on its own is insufficient
+to get your model forward pass running other behavior changes that may be more
+details, see the guide on [TF1.x vs TF2 behaviors](./tf1_vs_tf2.ipynb) to learn
+about the other behavior changes and how you can adapt to them. Also check out
+the
+[making new Layers and Models via subclassing guide](https://tensorflow.org/guide/keras/custom_layers_and_models.ipynb)
+for details.
+
+### Validating your results
+
+See the [model validation guide](./validate_correctness.ipynb) for easy tools
+and guidance around how you can (numerically) validate that your model is
+behaving correctly when eager execution is enabled. You may find this especially
+useful when paired with the [model mapping guide](./model_mapping.ipynb).
+
+## Upgrade training, evaluation, and import/export code
+
+TF1.x training loops built with `v1.Session`-style `tf.estimator.Estimator`s and
+other collections-based approaches are not compatible with the new behaviors of
+TF2. It is important you migrate all your TF1.x training code as combining it
+with TF2 code can cause unexpected behaviors.
+
+You can choose from among several strategies to do this.
+
+The highest-level approach is to use `tf.keras`. The high level functions in
+Keras manage a lot of the low-level details that might be easy to miss if you
+write your own training loop. For example, they automatically collect the
+regularization losses, and set the `training=True` argument when calling the
+model.
+
+Refer to the [Estimator migration guide](./migrating_estimator.ipynb) to learn
+how you can migrate `tf.estimator.Estimator`s code to use
+[vanilla](./migrating_estimator.ipynb#tf2_keras_training_api) and
+[custom](./migrating_estimator.ipynb#tf2_keras_training_api_with_custom_training_step)
+`tf.keras` training loops.
+
+Custom training loops give you finer control over your model such as tracking
+the weights of individual layers. Read the guide on
+[building training loops from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch)
+to learn how to use `tf.GradientTape` to retrieve model weights and use them to
+update the model.
+
+### Convert TF1.x optimizers to Keras optimizers
+
+The optimizers in `tf.compat.v1.train`, such as the
+[Adam optimizer](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/AdamOptimizer)
+and the
+[gradient descent optimizer](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/GradientDescentOptimizer),
+have equivalents in `tf.keras.optimizers`.
+
+The table below summarizes how you can convert these legacy optimizers to their
+Keras equivalents. You can directly replace the TF1.x version with the TF2
+version unless additional steps (such as
+[updating the default learning rate](../../guide/effective_tf2.ipynb#optimizer_defaults))
+are required.
+
+Note that converting your optimizers
+[may make old checkpoints incompatible](./migrating_checkpoints.ipynb).
+
+
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+
TF1.xTF2Additional steps
`tf.v1.train.GradientDescentOptimizer``tf.keras.optimizers.SGD`None
`tf.v1.train.MomentumOptimizer``tf.keras.optimizers.SGD`Include the `momentum` argument
`tf.v1.train.AdamOptimizer``tf.keras.optimizers.Adam`Rename `beta1` and `beta2` arguments to `beta_1` and `beta_2`
`tf.v1.train.RMSPropOptimizer``tf.keras.optimizers.RMSprop`Rename the `decay` argument to `rho`
`tf.v1.train.AdadeltaOptimizer``tf.keras.optimizers.Adadelta`None
`tf.v1.train.AdagradOptimizer``tf.keras.optimizers.Adagrad`None
`tf.v1.train.FtrlOptimizer``tf.keras.optimizers.Ftrl`Remove the `accum_name` and `linear_name` arguments
`tf.contrib.AdamaxOptimizer``tf.keras.optimizers.Adamax`Rename the `beta1`, and `beta2` arguments to `beta_1` and `beta_2`
`tf.contrib.Nadam``tf.keras.optimizers.Nadam`Rename the `beta1`, and `beta2` arguments to `beta_1` and `beta_2`

+
+Note: In TF2, all epsilons (numerical stability constants) now default to `1e-7`
+instead of `1e-8`. This difference is negligible in most use cases.
+
+### Upgrade data input pipelines
+
+There are many ways to feed data to a `tf.keras` model. They will accept Python
+generators and Numpy arrays as input.
+
+The recommended way to feed data to a model is to use the `tf.data` package,
+which contains a collection of high performance classes for manipulating data.
+The `dataset`s belonging to `tf.data` are efficient, expressive, and integrate
+well with TF2.
+
+They can be passed directly to the `tf.keras.Model.fit` method.
+
+```python
+model.fit(dataset, epochs=5)
+```
+
+They can be iterated over directly standard Python:
+
+```python
+for example_batch, label_batch in dataset:
+    break
+```
+
+If you are still using `tf.queue`, these are now only supported as
+data-structures, not as input pipelines.
+
+You should also migrate all feature preprocessing code that
+uses`tf.feature_columns`. Read the
+[migration guide](./migrating_feature_columns.ipynb) for more details.
+
+### Saving and loading models
+
+TF2 uses object-based checkpoints. Read the
+[checkpoint migration guide](./migrating_checkpoints.ipynb) to learn more about
+migrating off name-based TF1.x checkpoints. Also read the
+[checkpoints guide](https://www.tensorflow.org/guide/checkpoint) in the core
+TensorFlow docs.
+
+There are no significant compatibility concerns for saved models. Read the
+[`SavedModel` guide](./saved_model.ipynb) for more information about migrating
+`SavedModel`s in TF1.x to TF2. In general,
+
+-   TF1.x saved_models work in TF2.
+-   TF2 saved_models work in TF1.x if all the ops are supported.
+
+Also refer to the
+[`GraphDef` section](./saved_model.ipynb#graphdef_and_metagraphdef) in the
+`SavedModel` migration guide for more information on working with `Graph.pb` and
+`Graph.pbtxt` objects.
+
+## (Optional) Migrate off `tf.compat.v1` symbols
+
+The `tf.compat.v1` module contains the complete TF1.x API, with its original
+semantics.
+
+Even after following the steps above and ending up with code that is fully
+compatible with all TF2 behaviors, it is likely there may be many mentions of
+`compat.v1` apis that happen to be compatible with TF2. You should avoid using
+these legacy `compat.v1` apis for any new code that you write, though they will
+continue working for your already-written code.
+
+However, you may choose to migrate the existing usages to non-legacy TF2 APIs.
+The docstrings of individual `compat.v1` symbols will often explain how to
+migrate them to non-legacy TF2 APIs. Additionally, the
+[model mapping guide's section on incremental migration to idiomatic TF2 APIs](./model_mapping.ipynb#incremental_migration_to_native_tf2)
+may help with this as well.
+
+## Resources and further reading
+
+As mentioned previously, it is a good practice to migrate all your TF1.x code to
+TF2. Read the guides in the
+[Migrate to TF2 section](https://tensorflow.org/guide/migrate) of the TensorFlow
+guide to learn more.
diff --git a/site/en/guide/migrate/migrating_checkpoints.ipynb b/site/en/guide/migrate/migrating_checkpoints.ipynb
new file mode 100644
index 00000000000..a63789037ff
--- /dev/null
+++ b/site/en/guide/migrate/migrating_checkpoints.ipynb
@@ -0,0 +1,1191 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6bYaCABobL5q"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "FlUw7tSKbtg4"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "61dp4Hg5ksTC"
+      },
+      "source": [
+        "# Migrating model checkpoints\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "avuMwzscPnHh"
+      },
+      "source": [
+        "Note: Checkpoints saved with `tf.compat.v1.Saver` are often referred as *TF1 or name-based* checkpoints. Checkpoints saved with `tf.train.Checkpoint` are referred as *TF2 or object-based* checkpoints.\n",
+        "\n",
+        "\n",
+        "## Overview \n",
+        "This guide assumes that you have a model that saves and loads checkpoints with [`tf.compat.v1.Saver`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/Saver), and want to migrate the code use the TF2 [`tf.train.Checkpoint`](https://www.tensorflow.org/api_docs/python/tf/train/Checkpoint) API, or use pre-existing checkpoints in your TF2 model.\n",
+        "\n",
+        "Below are some common scenarios that you may encounter:\n",
+        "\n",
+        "**Scenario 1**\n",
+        "\n",
+        "There are existing TF1 checkpoints from previous training runs that need to be loaded or converted to TF2.\n",
+        "\n",
+        "* To load the TF1 checkpoint in TF2, see the snippet [*Load a TF1 checkpoint in TF2*](#load-tf1-in-tf2).\n",
+        "* To convert the checkpoint to TF2, see [*Checkpoint conversion*](#checkpoint-conversion).\n",
+        " \n",
+        "**Scenario 2**\n",
+        "\n",
+        "You are adjusting your model in a way that risks changing variable names and paths (such as when incrementally migrating away from `get_variable` to explicit `tf.Variable` creation), and would like to maintain saving/loading of existing checkpoints along the way.\n",
+        "\n",
+        "See the section on [*How to maintain checkpoint compatibility during model migration*](#maintain-checkpoint-compat)\n",
+        "\n",
+        "**Scenario 3**\n",
+        "\n",
+        "You are migrating your training code and checkpoints to TF2, but your inference pipeline continues to require TF1 checkpoints for now (for production stability).\n",
+        "\n",
+        "*Option 1*\n",
+        "\n",
+        "Save both TF1 and TF2 checkpoints when training. \n",
+        "\n",
+        "* see [*Save a TF1 checkpoint in TF2*](#save-tf1-in-tf2)\n",
+        "\n",
+        "*Option 2*\n",
+        "\n",
+        "Convert the TF2 checkpoint to TF1.\n",
+        "\n",
+        "* see [*Checkpoint conversion*](#checkpoint-conversion)\n",
+        "\n",
+        "\n",
+        "\n",
+        "---\n",
+        "\n",
+        "\n",
+        "The examples below show all the combinations of saving and loading checkpoints in TF1/TF2, so you have some flexibility in determining how to migrate your model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TaYgaekzOAHf"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kcvTd5QhZ78L"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1\n",
+        "\n",
+        "def print_checkpoint(save_path):\n",
+        "  reader = tf.train.load_checkpoint(save_path)\n",
+        "  shapes = reader.get_variable_to_shape_map()\n",
+        "  dtypes = reader.get_variable_to_dtype_map()\n",
+        "  print(f\"Checkpoint at '{save_path}':\")\n",
+        "  for key in shapes:\n",
+        "    print(f\"  (key='{key}', shape={shapes[key]}, dtype={dtypes[key].name}, \"\n",
+        "          f\"value={reader.get_tensor(key)})\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gO8Q6QkulJlj"
+      },
+      "source": [
+        "## Changes from TF1 to TF2\n",
+        "\n",
+        "This section is included if you are curious about what has changed between TF1 and TF2, and what we mean by \"name-based\" (TF1) vs \"object-based\" (TF2) checkpoints. \n",
+        "\n",
+        "The two types of checkpoints are actually saved in the same format, which is essentially a key-value table. The difference lies in how the keys are generated.\n",
+        "\n",
+        " The keys in named-based checkpoints are the **names of the variables**. The keys in object-based checkpoints refer to the **path from the root object to the variable** (the examples below will help to get a better sense of what this means).\n",
+        "\n",
+        " First, save some checkpoints:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8YXzbXvOWvdF"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.Graph().as_default() as g:\n",
+        "  a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.zeros_initializer())\n",
+        "  b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.zeros_initializer())\n",
+        "  c = tf1.get_variable('scoped/c', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.zeros_initializer())\n",
+        "  with tf1.Session() as sess:\n",
+        "    saver = tf1.train.Saver()\n",
+        "    sess.run(a.assign(1))\n",
+        "    sess.run(b.assign(2))\n",
+        "    sess.run(c.assign(3))\n",
+        "    saver.save(sess, 'tf1-ckpt')\n",
+        "\n",
+        "print_checkpoint('tf1-ckpt')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "raOych1UaJzl"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable(5.0, name='a')\n",
+        "b = tf.Variable(6.0, name='b')\n",
+        "with tf.name_scope('scoped'):\n",
+        "  c = tf.Variable(7.0, name='c')\n",
+        "\n",
+        "ckpt = tf.train.Checkpoint(variables=[a, b, c])\n",
+        "save_path_v2 = ckpt.save('tf2-ckpt')\n",
+        "print_checkpoint(save_path_v2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UYyLhTYszcpl"
+      },
+      "source": [
+        "If you look at the keys in `tf2-ckpt`, they all refer to the object paths of each variable. For example, variable `a` is the first element in the `variables` list, so its key becomes `variables/0/...` (feel free to ignore the .ATTRIBUTES/VARIABLE_VALUE constant).\n",
+        "\n",
+        "A closer inspection of the `Checkpoint` object below:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kLOxvoosg4Al"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable(0.)\n",
+        "b = tf.Variable(0.)\n",
+        "c = tf.Variable(0.)\n",
+        "root = ckpt = tf.train.Checkpoint(variables=[a, b, c])\n",
+        "print(\"root type =\", type(root).__name__)\n",
+        "print(\"root.variables =\", root.variables)\n",
+        "print(\"root.variables[0] =\", root.variables[0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1Qaed1yAm3Ar"
+      },
+      "source": [
+        "Try experimenting with the below snippet and see how the checkpoint keys change with the object structure:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EdHJXlZOyDnn"
+      },
+      "outputs": [],
+      "source": [
+        "module = tf.Module()\n",
+        "module.d = tf.Variable(0.)\n",
+        "test_ckpt = tf.train.Checkpoint(v={'a': a, 'b': b}, \n",
+        "                                c=c,\n",
+        "                                module=module)\n",
+        "test_ckpt_path = test_ckpt.save('root-tf2-ckpt')\n",
+        "print_checkpoint(test_ckpt_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8iWitEsayDWs"
+      },
+      "source": [
+        "*Why does TF2 use this mechanism?* \n",
+        "\n",
+        "Because there is no more global graph in TF2, variable names are unreliable and can be inconsistent between programs. TF2 encourages the object-oriented modelling approach where variables are owned by layers, and layers are owned by a model:\n",
+        "\n",
+        "```\n",
+        "variable = tf.Variable(...)\n",
+        "layer.variable_name = variable\n",
+        "model.layer_name = layer\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9kv9SmyVjGLA"
+      },
+      "source": [
+        "## How to maintain checkpoint compatibility during model migration\n",
+        "\n",
+        "\n",
+        "\n",
+        "One important step in the migration process is *ensuring that all variables are initialized to the correct values*, which in turn allows you to  validate that the ops/functions are doing the correct computations. To accomplish this, you must consider the **checkpoint compatibility** between models in the various stages of migration. Essentially, this section answers the question, *how do I keep using the same checkpoint while changing the model*.\n",
+        "\n",
+        "Below are three ways of maintaining checkpoint compatibility, in order of increasing flexibility:\n",
+        "\n",
+        "1. The model has the **same variable names** as before.\n",
+        "2. The model has different variable names, and maintains a **assignment map** that maps variable names in the checkpoint to the new names.\n",
+        "3. The model has different variable names, and maintains a **TF2 Checkpoint object** that stores all of the variables."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L5JhCyPZDx43"
+      },
+      "source": [
+        "### When the variable names match\n",
+        "Long title: How to re-use checkpoints when the variable names match.\n",
+        "\n",
+        "Short answer: You can directly load the pre-existing checkpoint with either `tf1.train.Saver` or `tf.train.Checkpoint`.\n",
+        "\n",
+        "---\n",
+        "\n",
+        "If you are using `tf.compat.v1.keras.utils.track_tf1_style_variables`, then it will ensure that your model variable names are the same as before. You can also manually ensure that variable names match.\n",
+        "\n",
+        "When the variable names match in the migrated models, you may directly use either `tf.train.Checkpoint` or `tf.compat.v1.train.Saver` to load the checkpoint. Both APIs are compatible with eager and graph mode, so you can use them at any stage of the migration.\n",
+        "\n",
+        "Note: You can use `tf.train.Checkpoint` to load TF1 checkpoints, but you cannot use `tf.compat.v1.Saver` to load TF2 checkpoints without complicated name matching.\n",
+        "\n",
+        "Below are examples of using the same checkpoint with different models. First, save a TF1 checkpoint with `tf1.train.Saver`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ijlHS96URsfR"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.Graph().as_default() as g:\n",
+        "  a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.zeros_initializer())\n",
+        "  b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.zeros_initializer())\n",
+        "  c = tf1.get_variable('scoped/c', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.zeros_initializer())\n",
+        "  with tf1.Session() as sess:\n",
+        "    saver = tf1.train.Saver()\n",
+        "    sess.run(a.assign(1))\n",
+        "    sess.run(b.assign(2))\n",
+        "    sess.run(c.assign(3))\n",
+        "    save_path = saver.save(sess, 'tf1-ckpt')\n",
+        "print_checkpoint(save_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zg7nWZphQD9u"
+      },
+      "source": [
+        "The example below uses `tf.compat.v1.Saver` to load the checkpoint while in eager mode:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y4K16m0PPncQ"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable(0.0, name='a')\n",
+        "b = tf.Variable(0.0, name='b')\n",
+        "with tf.name_scope('scoped'):\n",
+        "  c = tf.Variable(0.0, name='c')\n",
+        "\n",
+        "# With the removal of collections in TF2, you must pass in the list of variables\n",
+        "# to the Saver object:\n",
+        "saver = tf1.train.Saver(var_list=[a, b, c])\n",
+        "saver.restore(sess=None, save_path=save_path)\n",
+        "print(f\"loaded values of [a, b, c]:  [{a.numpy()}, {b.numpy()}, {c.numpy()}]\")\n",
+        "\n",
+        "# Saving also works in eager (sess must be None).\n",
+        "path = saver.save(sess=None, save_path='tf1-ckpt-saved-in-eager')\n",
+        "print_checkpoint(path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dWnq1f5yAPkq"
+      },
+      "source": [
+        "The next snippet loads the checkpoint using the TF2 API `tf.train.Checkpoint`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "StyrzwGvW1YZ"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable(0.0, name='a')\n",
+        "b = tf.Variable(0.0, name='b')\n",
+        "with tf.name_scope('scoped'):\n",
+        "  c = tf.Variable(0.0, name='c')\n",
+        "\n",
+        "# Without the name_scope, name=\"scoped/c\" works too:\n",
+        "c_2 = tf.Variable(0.0, name='scoped/c')\n",
+        "\n",
+        "print(\"Variable names: \")\n",
+        "print(f\"  a.name = {a.name}\")\n",
+        "print(f\"  b.name = {b.name}\")\n",
+        "print(f\"  c.name = {c.name}\")\n",
+        "print(f\"  c_2.name = {c_2.name}\")\n",
+        "\n",
+        "# Restore the values with tf.train.Checkpoint\n",
+        "ckpt = tf.train.Checkpoint(variables=[a, b, c, c_2])\n",
+        "ckpt.restore(save_path)\n",
+        "print(f\"loaded values of [a, b, c, c_2]:  [{a.numpy()}, {b.numpy()}, {c.numpy()}, {c_2.numpy()}]\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DYYgbj8F7Yb7"
+      },
+      "source": [
+        "#### Variable names in TF2\n",
+        "\n",
+        "\n",
+        "- Variables still all have a `name` argument you can set.\n",
+        "- Keras models also take a `name` argument as which they set as the prefix for their variables.\n",
+        "- The `v1.name_scope` function can be used to set variable name prefixes. This is very different from `tf.variable_scope`. It only affects names, and doesn't track variables and reuse.\n",
+        "\n",
+        "\n",
+        "The `tf.compat.v1.keras.utils.track_tf1_style_variables` decorator is a shim that helps you maintain variable names and TF1 checkpoint compatibility, by keeping the naming and reuse semantics of `tf.variable_scope` and `tf.compat.v1.get_variable` unchanged. See the [Model mapping guide](./model_mapping.ipynb) for more info. \n",
+        "\n",
+        "**Note 1: If you are using the shim, use TF2 APIs to load your checkpoints (even when using pre-trained TF1 checkpoints).**\n",
+        "\n",
+        "See the section *Checkpoint Keras*.\n",
+        "\n",
+        "**Note 2: When migrating to `tf.Variable` from `get_variable`:**\n",
+        "\n",
+        "If your shim-decorated layer or module consists of some variables (or Keras layers/models) that use `tf.Variable` instead of `tf.compat.v1.get_variable` and get attached as properties/tracked in an object oriented way, they may have different variable naming semantics in TF1.x graphs/sessions versus during eager execution.\n",
+        "\n",
+        "In short, *the names may not be what you expect them to be* when running in TF2.\n",
+        "\n",
+        "Warning: Variables may have duplicate names in eager execution, which may cause problems if multiple variables in the name-based checkpoint need to be mapped to the same name. You may be able to explicitly adjust the layer and variable names using `tf.name_scope` and layer constructor or `tf.Variable` `name` arguments to adjust variable names and ensure there are no duplicates."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NkUQJUUyjOJz"
+      },
+      "source": [
+        "### Maintaining assignment maps\n",
+        "\n",
+        "Assignment maps are commonly used to transfer weights between TF1 models, and can also be used during your model migration if the variable names change. \n",
+        "\n",
+        "You can use these maps with [`tf.compat.v1.train.init_from_checkpoint`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/init_from_checkpoint), [`tf.compat.v1.train.Saver`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/Saver), and [`tf.train.load_checkpoint`](https://www.tensorflow.org/api_docs/python/tf/train/load_checkpoint) to load weights into models in which the variable or scope names may have changed.\n",
+        "\n",
+        "The examples in this section will use a previously saved checkpoint:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PItyo7DdJ6Ek"
+      },
+      "outputs": [],
+      "source": [
+        "print_checkpoint('tf1-ckpt')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rPryV_WBJrI3"
+      },
+      "source": [
+        "#### Loading with `init_from_checkpoint`\n",
+        "\n",
+        "[`tf1.train.init_from_checkpoint`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/init_from_checkpoint) must be called while in a Graph/Session, because it places the values in the variable initializers instead of creating an assign op. \n",
+        "\n",
+        "You can use the `assignment_map` argument to configure how the variables are loaded. From the documentation:\n",
+        "> Assignment map supports following syntax:\n",
+        "  * `'checkpoint_scope_name/': 'scope_name/'` - will load all variables in\n",
+        "    current `scope_name` from `checkpoint_scope_name` with matching tensor\n",
+        "    names.\n",
+        "  * `'checkpoint_scope_name/some_other_variable': 'scope_name/variable_name'` -\n",
+        "    will initialize `scope_name/variable_name` variable\n",
+        "    from `checkpoint_scope_name/some_other_variable`.\n",
+        "  * `'scope_variable_name': variable` - will initialize given `tf.Variable`\n",
+        "    object with tensor 'scope_variable_name' from the checkpoint.\n",
+        "  * `'scope_variable_name': list(variable)` - will initialize list of\n",
+        "    partitioned variables with tensor 'scope_variable_name' from the checkpoint.\n",
+        "  * `'/': 'scope_name/'` - will load all variables in current `scope_name` from\n",
+        "    checkpoint's root (e.g. no scope).\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZM_7OzRpdH0A"
+      },
+      "outputs": [],
+      "source": [
+        "# Restoring with tf1.train.init_from_checkpoint:\n",
+        "\n",
+        "# A new model with a different scope for the variables.\n",
+        "with tf.Graph().as_default() as g:\n",
+        "  with tf1.variable_scope('new_scope'):\n",
+        "    a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.zeros_initializer())\n",
+        "    b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.zeros_initializer())\n",
+        "    c = tf1.get_variable('scoped/c', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.zeros_initializer())\n",
+        "  with tf1.Session() as sess:\n",
+        "    # The assignment map will remap all variables in the checkpoint to the\n",
+        "    # new scope:\n",
+        "    tf1.train.init_from_checkpoint(\n",
+        "        'tf1-ckpt',\n",
+        "        assignment_map={'/': 'new_scope/'})\n",
+        "    # `init_from_checkpoint` adds the initializers to these variables.\n",
+        "    # Use `sess.run` to run these initializers.\n",
+        "    sess.run(tf1.global_variables_initializer())\n",
+        "\n",
+        "    print(\"Restored [a, b, c]: \", sess.run([a, b, c]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Za_8xhFWKVlH"
+      },
+      "source": [
+        "#### Loading with `tf1.train.Saver`\n",
+        "\n",
+        "Unlike `init_from_checkpoint`, [`tf.compat.v1.train.Saver`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/Saver) runs in both graph and eager mode. The `var_list` argument optionally accepts a dictionary, except it must map variable names to the `tf.Variable` object.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IiKNmdGJgoX9"
+      },
+      "outputs": [],
+      "source": [
+        "# Restoring with tf1.train.Saver (works in both graph and eager):\n",
+        "\n",
+        "# A new model with a different scope for the variables.\n",
+        "with tf1.variable_scope('new_scope'):\n",
+        "  a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                      initializer=tf1.zeros_initializer())\n",
+        "  b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                      initializer=tf1.zeros_initializer())\n",
+        "  c = tf1.get_variable('scoped/c', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.zeros_initializer())\n",
+        "# Initialize the saver with a dictionary with the original variable names:\n",
+        "saver = tf1.train.Saver({'a': a, 'b': b, 'scoped/c': c})\n",
+        "saver.restore(sess=None, save_path='tf1-ckpt')\n",
+        "print(\"Restored [a, b, c]: \", [a.numpy(), b.numpy(), c.numpy()])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7JsgCXt3Ly-h"
+      },
+      "source": [
+        "#### Loading with `tf.train.load_checkpoint`\n",
+        "\n",
+        "This option is for you if you need precise control over the variable values. Again, this works in both graph and eager modes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Pc39Bh6JMso6"
+      },
+      "outputs": [],
+      "source": [
+        "# Restoring with tf.train.load_checkpoint (works in both graph and eager):\n",
+        "\n",
+        "# A new model with a different scope for the variables.\n",
+        "with tf.Graph().as_default() as g:\n",
+        "  with tf1.variable_scope('new_scope'):\n",
+        "    a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.zeros_initializer())\n",
+        "    b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.zeros_initializer())\n",
+        "    c = tf1.get_variable('scoped/c', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.zeros_initializer())\n",
+        "  with tf1.Session() as sess:\n",
+        "    # It may be easier writing a loop if your model has a lot of variables.\n",
+        "    reader = tf.train.load_checkpoint('tf1-ckpt')\n",
+        "    sess.run(a.assign(reader.get_tensor('a')))\n",
+        "    sess.run(b.assign(reader.get_tensor('b')))\n",
+        "    sess.run(c.assign(reader.get_tensor('scoped/c')))\n",
+        "    print(\"Restored [a, b, c]: \", sess.run([a, b, c]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nBSTJVCNDKed"
+      },
+      "source": [
+        "### Maintaining a TF2 Checkpoint object\n",
+        "If the variable and scope names may change a lot during the migration, then use `tf.train.Checkpoint` and TF2 checkpoints. TF2 uses the **object structure** instead of variable names (more details in *Changes from TF1 to TF2*).\n",
+        "\n",
+        "In short, when creating a `tf.train.Checkpoint` to save or restore checkpoints, make sure it uses the same **ordering** (for lists) and **keys** (for dictionaries and keyword arguments to the `Checkpoint` initializer). Some examples of checkpoint compatibility:\n",
+        "\n",
+        "```\n",
+        "ckpt = tf.train.Checkpoint(foo=[var_a, var_b])\n",
+        "\n",
+        "# compatible with ckpt\n",
+        "tf.train.Checkpoint(foo=[var_a, var_b])\n",
+        "\n",
+        "# not compatible with ckpt\n",
+        "tf.train.Checkpoint(foo=[var_b, var_a])\n",
+        "tf.train.Checkpoint(bar=[var_a, var_b])\n",
+        "```\n",
+        "\n",
+        "The code samples below show how to use the \"same\" `tf.train.Checkpoint` to load variables with different names. First, save a TF2 checkpoint:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tCSkz_-Tbct6"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.Graph().as_default() as g:\n",
+        "  a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(1))\n",
+        "  b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(2))\n",
+        "  with tf1.variable_scope('scoped'):\n",
+        "    c = tf1.get_variable('c', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.constant_initializer(3))\n",
+        "  with tf1.Session() as sess:\n",
+        "    sess.run(tf1.global_variables_initializer())\n",
+        "    print(\"[a, b, c]: \", sess.run([a, b, c]))\n",
+        "\n",
+        "    # Save a TF2 checkpoint\n",
+        "    ckpt = tf.train.Checkpoint(unscoped=[a, b], scoped=[c])\n",
+        "    tf2_ckpt_path = ckpt.save('tf2-ckpt')\n",
+        "    print_checkpoint(tf2_ckpt_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "62MWdZMxezeP"
+      },
+      "source": [
+        "You can keep using `tf.train.Checkpoint` even if the variable/scope names change:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Vh61SGeqb27b"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.Graph().as_default() as g:\n",
+        "  a = tf1.get_variable('a_different_name', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.zeros_initializer())\n",
+        "  b = tf1.get_variable('b_different_name', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.zeros_initializer())\n",
+        "  with tf1.variable_scope('different_scope'):\n",
+        "    c = tf1.get_variable('c', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.zeros_initializer())\n",
+        "  with tf1.Session() as sess:\n",
+        "    sess.run(tf1.global_variables_initializer())\n",
+        "    print(\"Initialized [a, b, c]: \", sess.run([a, b, c]))\n",
+        "\n",
+        "    ckpt = tf.train.Checkpoint(unscoped=[a, b], scoped=[c])\n",
+        "    # `assert_consumed` validates that all checkpoint objects are restored from\n",
+        "    # the checkpoint. `run_restore_ops` is required when running in a TF1\n",
+        "    # session.\n",
+        "    ckpt.restore(tf2_ckpt_path).assert_consumed().run_restore_ops()\n",
+        "\n",
+        "    # Removing `assert_consumed` is fine if you want to skip the validation.\n",
+        "    # ckpt.restore(tf2_ckpt_path).run_restore_ops()\n",
+        "\n",
+        "    print(\"Restored [a, b, c]: \", sess.run([a, b, c]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "unDPmL-kldr2"
+      },
+      "source": [
+        "And in eager mode:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "79S0zMAnfzx7"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable(0.)\n",
+        "b = tf.Variable(0.)\n",
+        "c = tf.Variable(0.)\n",
+        "print(\"Initialized [a, b, c]: \", [a.numpy(), b.numpy(), c.numpy()])\n",
+        "\n",
+        "# The keys \"scoped\" and \"unscoped\" are no longer relevant, but are used to\n",
+        "# maintain compatibility with the saved checkpoints.\n",
+        "ckpt = tf.train.Checkpoint(unscoped=[a, b], scoped=[c])\n",
+        "\n",
+        "ckpt.restore(tf2_ckpt_path).assert_consumed().run_restore_ops()\n",
+        "print(\"Restored [a, b, c]: \", [a.numpy(), b.numpy(), c.numpy()])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dKfNAr8l3aFg"
+      },
+      "source": [
+        "## TF2 checkpoints in Estimator\n",
+        "\n",
+        "The sections above describe how to maintain checkpoint compatiblity while migrating your model. These concepts also apply for Estimator models, although the way the checkpoint is saved/loaded is slightly different. As you migrate your Estimator model to use TF2 APIs, you may want to switch from TF1 to TF2 checkpoints *while the model is still using the estimator*. This sections shows how to do so.\n",
+        "\n",
+        "[`tf.estimator.Estimator`](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator) and [`MonitoredSession`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/MonitoredSession) have a saving mechanism called the `scaffold`, a [`tf.compat.v1.train.Scaffold`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/Scaffold) object. The `Scaffold` can contain a `tf1.train.Saver` or `tf.train.Checkpoint`, which enables `Estimator` and `MonitoredSession` to save TF1- or TF2-style checkpoints.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D8AT_oO-5TXU"
+      },
+      "outputs": [],
+      "source": [
+        "# A model_fn that saves a TF1 checkpoint\n",
+        "def model_fn_tf1_ckpt(features, labels, mode):\n",
+        "  # This model adds 2 to the variable `v` in every train step.\n",
+        "  train_step = tf1.train.get_or_create_global_step()\n",
+        "  v = tf1.get_variable('var', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(0))\n",
+        "  return tf.estimator.EstimatorSpec(\n",
+        "      mode,\n",
+        "      predictions=v,\n",
+        "      train_op=tf.group(v.assign_add(2), train_step.assign_add(1)),\n",
+        "      loss=tf.constant(1.),\n",
+        "      scaffold=None\n",
+        "  )\n",
+        "\n",
+        "!rm -rf est-tf1\n",
+        "est = tf.estimator.Estimator(model_fn_tf1_ckpt, 'est-tf1')\n",
+        "\n",
+        "def train_fn():\n",
+        "  return tf.data.Dataset.from_tensor_slices(([1,2,3], [4,5,6]))\n",
+        "est.train(train_fn, steps=1)\n",
+        "\n",
+        "latest_checkpoint = tf.train.latest_checkpoint('est-tf1')\n",
+        "print_checkpoint(latest_checkpoint)  "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ttH6cUrl7jK2"
+      },
+      "outputs": [],
+      "source": [
+        "# A model_fn that saves a TF2 checkpoint\n",
+        "def model_fn_tf2_ckpt(features, labels, mode):\n",
+        "  # This model adds 2 to the variable `v` in every train step.\n",
+        "  train_step = tf1.train.get_or_create_global_step()\n",
+        "  v = tf1.get_variable('var', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(0))\n",
+        "  ckpt = tf.train.Checkpoint(var_list={'var': v}, step=train_step)\n",
+        "  return tf.estimator.EstimatorSpec(\n",
+        "      mode,\n",
+        "      predictions=v,\n",
+        "      train_op=tf.group(v.assign_add(2), train_step.assign_add(1)),\n",
+        "      loss=tf.constant(1.),\n",
+        "      scaffold=tf1.train.Scaffold(saver=ckpt)\n",
+        "  )\n",
+        "\n",
+        "!rm -rf est-tf2\n",
+        "est = tf.estimator.Estimator(model_fn_tf2_ckpt, 'est-tf2',\n",
+        "                             warm_start_from='est-tf1')\n",
+        "\n",
+        "def train_fn():\n",
+        "  return tf.data.Dataset.from_tensor_slices(([1,2,3], [4,5,6]))\n",
+        "est.train(train_fn, steps=1)\n",
+        "\n",
+        "latest_checkpoint = tf.train.latest_checkpoint('est-tf2')\n",
+        "print_checkpoint(latest_checkpoint)  \n",
+        "\n",
+        "assert est.get_variable_value('var_list/var/.ATTRIBUTES/VARIABLE_VALUE') == 4"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hYVYgahE8daL"
+      },
+      "source": [
+        "The final value of `v` should be `16`, after being warm-started from `est-tf1`, then trained for an additional 5 steps. The train step value doesn't carry over from the `warm_start` checkpoint.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pq8EjblQUIA2"
+      },
+      "source": [
+        "## Checkpointing Keras\n",
+        "\n",
+        "Models built with Keras still use `tf1.train.Saver` and `tf.train.Checkpoint` to load pre-existing weights. When your model is fully migrated, switch to using `model.save_weights` and `model.load_weights`, especially if you are using the `ModelCheckpoint` callback when training.\n",
+        "\n",
+        "Some things you should know about checkpoints and Keras:\n",
+        "\n",
+        "**Initialization vs Building**\n",
+        "\n",
+        "Keras models and layers must go through **two steps** before being fully created. First is the *initialization* of the Python object: `layer = tf.keras.layers.Dense(x)`. Second is the *build* step, in which most of the weights are actually created: `layer.build(input_shape)`. You can also build a model by calling it or running a single `train`, `eval`, or `predict` step (the first time only).\n",
+        "\n",
+        "If you find that `model.load_weights(path).assert_consumed()` is raising an error, then it is likely that the model/layers have not been built. \n",
+        "\n",
+        "**Keras uses TF2 checkpoints**\n",
+        "\n",
+        "`tf.train.Checkpoint(model).write` is equivalent to `model.save_weights`. Same with `tf.train.Checkpoint(model).read` and `model.load_weights`. Note that `Checkpoint(model) != Checkpoint(model=model)`.\n",
+        "\n",
+        "**TF2 checkpoints work with Keras's `build()` step**\n",
+        "\n",
+        "`tf.train.Checkpoint.restore` has a mechanism called *deferred restoration* which\n",
+        "allows `tf.Module` and Keras objects to store variable values if the variable has not yet been created. This allows *initialized* models to load weights and  *build* after.\n",
+        "\n",
+        "```\n",
+        "m = YourKerasModel()\n",
+        "status = m.load_weights(path)\n",
+        "\n",
+        "# This call builds the model. The variables are created with the restored\n",
+        "# values.\n",
+        "m.predict(inputs)\n",
+        "\n",
+        "status.assert_consumed()\n",
+        "```\n",
+        "\n",
+        "Because of this mechanism, we highly recommend that you use TF2 checkpoint loading APIs with Keras models (even when restoring pre-existing TF1 checkpoints into the [model mapping shims](./model_mapping.ipynb)). See more in the [checkpoint guide](https://www.tensorflow.org/guide/checkpoint#delayed_restorations).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xO2NucRtqMm6"
+      },
+      "source": [
+        "## Code Snippets\n",
+        "\n",
+        "The snippets below show the TF1/TF2 version compatibility in the checkpoint saving APIs. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "C3SSc74olkX3"
+      },
+      "source": [
+        "### Save a TF1 checkpoint in TF2\n",
+        "\n",
+        ""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "t2ZPk8BPloE1"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable(1.0, name='a')\n",
+        "b = tf.Variable(2.0, name='b')\n",
+        "with tf.name_scope('scoped'):\n",
+        "  c = tf.Variable(3.0, name='c')\n",
+        "\n",
+        "saver = tf1.train.Saver(var_list=[a, b, c])\n",
+        "path = saver.save(sess=None, save_path='tf1-ckpt-saved-in-eager')\n",
+        "print_checkpoint(path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BxyN5khVjhmA"
+      },
+      "source": [
+        "### Load a TF1 checkpoint in TF2\n",
+        "\n",
+        ""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Z5kSXy3FmA79"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable(0., name='a')\n",
+        "b = tf.Variable(0., name='b')\n",
+        "with tf.name_scope('scoped'):\n",
+        "  c = tf.Variable(0., name='c')\n",
+        "print(\"Initialized [a, b, c]: \", [a.numpy(), b.numpy(), c.numpy()])\n",
+        "saver = tf1.train.Saver(var_list=[a, b, c])\n",
+        "saver.restore(sess=None, save_path='tf1-ckpt-saved-in-eager')\n",
+        "print(\"Restored [a, b, c]: \", [a.numpy(), b.numpy(), c.numpy()])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ul3V4pEwloeN"
+      },
+      "source": [
+        "### Save a TF2 checkpoint in TF1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UhuP_2EIlRm4"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.Graph().as_default() as g:\n",
+        "  a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(1))\n",
+        "  b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(2))\n",
+        "  with tf1.variable_scope('scoped'):\n",
+        "    c = tf1.get_variable('c', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.constant_initializer(3))\n",
+        "  with tf1.Session() as sess:\n",
+        "    sess.run(tf1.global_variables_initializer())\n",
+        "    ckpt = tf.train.Checkpoint(\n",
+        "        var_list={v.name.split(':')[0]: v for v in tf1.global_variables()})\n",
+        "    tf2_in_tf1_path = ckpt.save('tf2-ckpt-saved-in-session')\n",
+        "    print_checkpoint(tf2_in_tf1_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GiViCjCDgxhz"
+      },
+      "source": [
+        "### Load a TF2 checkpoint in TF1\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "j-4hIPZvmXlb"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.Graph().as_default() as g:\n",
+        "  a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(0))\n",
+        "  b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(0))\n",
+        "  with tf1.variable_scope('scoped'):\n",
+        "    c = tf1.get_variable('c', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.constant_initializer(0))\n",
+        "  with tf1.Session() as sess:\n",
+        "    sess.run(tf1.global_variables_initializer())\n",
+        "    print(\"Initialized [a, b, c]: \", sess.run([a, b, c]))\n",
+        "    ckpt = tf.train.Checkpoint(\n",
+        "        var_list={v.name.split(':')[0]: v for v in tf1.global_variables()})\n",
+        "    ckpt.restore('tf2-ckpt-saved-in-session-1').run_restore_ops()\n",
+        "    print(\"Restored [a, b, c]: \", sess.run([a, b, c]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oRrSE2X6sgAM"
+      },
+      "source": [
+        "## Checkpoint conversion\n",
+        "\n",
+        "\n",
+        "\n",
+        "You can convert checkpoints between TF1 and TF2 by loading and re-saving the checkpoints. An alternative is `tf.train.load_checkpoint`, shown in the code below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o9KByaLous4q"
+      },
+      "source": [
+        "### Convert TF1 checkpoint to TF2"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NG8grCv2smAb"
+      },
+      "outputs": [],
+      "source": [
+        "def convert_tf1_to_tf2(checkpoint_path, output_prefix):\n",
+        "  \"\"\"Converts a TF1 checkpoint to TF2.\n",
+        "\n",
+        "  To load the converted checkpoint, you must build a dictionary that maps\n",
+        "  variable names to variable objects.\n",
+        "  ```\n",
+        "  ckpt = tf.train.Checkpoint(vars={name: variable})  \n",
+        "  ckpt.restore(converted_ckpt_path)\n",
+        "  ```\n",
+        "\n",
+        "  Args:\n",
+        "    checkpoint_path: Path to the TF1 checkpoint.\n",
+        "    output_prefix: Path prefix to the converted checkpoint.\n",
+        "\n",
+        "  Returns:\n",
+        "    Path to the converted checkpoint.\n",
+        "  \"\"\"\n",
+        "  vars = {}\n",
+        "  reader = tf.train.load_checkpoint(checkpoint_path)\n",
+        "  dtypes = reader.get_variable_to_dtype_map()\n",
+        "  for key in dtypes.keys():\n",
+        "    vars[key] = tf.Variable(reader.get_tensor(key))\n",
+        "  return tf.train.Checkpoint(vars=vars).save(output_prefix)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TyvqK6Sb3dad"
+      },
+      "source": [
+        "Convert the checkpoint saved in the snippet `Save a TF1 checkpoint in TF2`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gcHLN4lPvYvw"
+      },
+      "outputs": [],
+      "source": [
+        "# Make sure to run the snippet in `Save a TF1 checkpoint in TF2`.\n",
+        "print_checkpoint('tf1-ckpt-saved-in-eager')\n",
+        "converted_path = convert_tf1_to_tf2('tf1-ckpt-saved-in-eager', \n",
+        "                                     'converted-tf1-to-tf2')\n",
+        "print(\"\\n[Converted]\")\n",
+        "print_checkpoint(converted_path)\n",
+        "\n",
+        "# Try loading the converted checkpoint.\n",
+        "a = tf.Variable(0.)\n",
+        "b = tf.Variable(0.)\n",
+        "c = tf.Variable(0.)\n",
+        "ckpt = tf.train.Checkpoint(vars={'a': a, 'b': b, 'scoped/c': c})\n",
+        "ckpt.restore(converted_path).assert_consumed()\n",
+        "print(\"\\nRestored [a, b, c]: \", [a.numpy(), b.numpy(), c.numpy()])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fokg6ybZvE20"
+      },
+      "source": [
+        "### Convert TF2 checkpoint to TF1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NPQsXQveuQiC"
+      },
+      "outputs": [],
+      "source": [
+        "def convert_tf2_to_tf1(checkpoint_path, output_prefix):\n",
+        "  \"\"\"Converts a TF2 checkpoint to TF1.\n",
+        "\n",
+        "  The checkpoint must be saved using a \n",
+        "  `tf.train.Checkpoint(var_list={name: variable})`\n",
+        "\n",
+        "  To load the converted checkpoint with `tf.compat.v1.Saver`:\n",
+        "  ```\n",
+        "  saver = tf.compat.v1.train.Saver(var_list={name: variable}) \n",
+        "\n",
+        "  # An alternative, if the variable names match the keys:\n",
+        "  saver = tf.compat.v1.train.Saver(var_list=[variables]) \n",
+        "  saver.restore(sess, output_path)\n",
+        "  ```\n",
+        "  \"\"\"\n",
+        "  vars = {}\n",
+        "  reader = tf.train.load_checkpoint(checkpoint_path)\n",
+        "  dtypes = reader.get_variable_to_dtype_map()\n",
+        "  for key in dtypes.keys():\n",
+        "    # Get the \"name\" from the \n",
+        "    if key.startswith('var_list/'):\n",
+        "      var_name = key.split('/')[1]\n",
+        "      # TF2 checkpoint keys use '/', so if they appear in the user-defined name,\n",
+        "      # they are escaped to '.S'.\n",
+        "      var_name = var_name.replace('.S', '/')\n",
+        "      vars[var_name] = tf.Variable(reader.get_tensor(key))\n",
+        "  \n",
+        "  return tf1.train.Saver(var_list=vars).save(sess=None, save_path=output_prefix)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VjZD_OSf1mKX"
+      },
+      "source": [
+        "Convert the checkpoint saved in the snippet `Save a TF2 checkpoint in TF1`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vc1MVeV6z2DB"
+      },
+      "outputs": [],
+      "source": [
+        "# Make sure to run the snippet in `Save a TF2 checkpoint in TF1`.\n",
+        "print_checkpoint('tf2-ckpt-saved-in-session-1')\n",
+        "converted_path = convert_tf2_to_tf1('tf2-ckpt-saved-in-session-1',\n",
+        "                                    'converted-tf2-to-tf1')\n",
+        "print(\"\\n[Converted]\")\n",
+        "print_checkpoint(converted_path)\n",
+        "\n",
+        "# Try loading the converted checkpoint.\n",
+        "with tf.Graph().as_default() as g:\n",
+        "  a = tf1.get_variable('a', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(0))\n",
+        "  b = tf1.get_variable('b', shape=[], dtype=tf.float32, \n",
+        "                       initializer=tf1.constant_initializer(0))\n",
+        "  with tf1.variable_scope('scoped'):\n",
+        "    c = tf1.get_variable('c', shape=[], dtype=tf.float32, \n",
+        "                        initializer=tf1.constant_initializer(0))\n",
+        "  with tf1.Session() as sess:\n",
+        "    saver = tf1.train.Saver([a, b, c])\n",
+        "    saver.restore(sess, converted_path)\n",
+        "    print(\"\\nRestored [a, b, c]: \", sess.run([a, b, c]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JBMfArLQ0jb-"
+      },
+      "source": [
+        "## Related Guides\n",
+        "\n",
+        "* [Validating numerical equivalence and correctness](./validate_correctness.ipynb)\n",
+        "* [Model mapping guide](./model_mapping.ipynb) and `tf.compat.v1.keras.utils.track_tf1_style_variables`\n",
+        "* [TF2 Checkpoint guide](https://www.tensorflow.org/guide/checkpoint)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "migrating_checkpoints.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/migrating_estimator.ipynb b/site/en/guide/migrate/migrating_estimator.ipynb
new file mode 100644
index 00000000000..4d3259babb8
--- /dev/null
+++ b/site/en/guide/migrate/migrating_estimator.ipynb
@@ -0,0 +1,423 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate from Estimator to Keras APIs\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "This guide demonstrates how to migrate from TensorFlow 1's `tf.estimator.Estimator` APIs to TensorFlow 2's `tf.keras` APIs. First, you will set up and run a basic model for training and evaluation with `tf.estimator.Estimator`. Then, you will perform the equivalent steps in TensorFlow 2 with the `tf.keras` APIs. You will also learn how to customize the training step by subclassing `tf.keras.Model` and using `tf.GradientTape`.\n",
+        "\n",
+        "- In TensorFlow 1, the high-level `tf.estimator.Estimator` APIs let you train and evaluate a model, as well as perform inference and save your model (for serving).\n",
+        "- In TensorFlow 2, use the Keras APIs to perform the aforementioned tasks, such as [model building](https://www.tensorflow.org/guide/keras/custom_layers_and_models), gradient application, [training](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit), evaluation, and prediction.\n",
+        "\n",
+        "(For migrating model/checkpoint saving workflows to TensorFlow 2, check out the [SavedModel](saved_model.ipynb) and [Checkpoint](checkpoint_saved.ipynb) migration guides.)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Start with imports and a simple dataset:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m7rnGxsXtDkV"
+      },
+      "outputs": [],
+      "source": [
+        "features = [[1., 1.5], [2., 2.5], [3., 3.5]]\n",
+        "labels = [[0.3], [0.5], [0.7]]\n",
+        "eval_features = [[4., 4.5], [5., 5.5], [6., 6.5]]\n",
+        "eval_labels = [[0.8], [0.9], [1.]]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4uXff1BEssdE"
+      },
+      "source": [
+        "## TensorFlow 1: Train and evaluate with tf.estimator.Estimator\n",
+        "\n",
+        "This example shows how to perform training and evaluation with `tf.estimator.Estimator` in TensorFlow 1.\n",
+        "\n",
+        "Start by defining a few functions: an input function for the training data, an evaluation input function for the evaluation data, and a model function that tells the `Estimator` how the training op is defined with the features and labels:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "def _input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "\n",
+        "def _eval_input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).batch(1)\n",
+        "\n",
+        "def _model_fn(features, labels, mode):\n",
+        "  logits = tf1.layers.Dense(1)(features)\n",
+        "  loss = tf1.losses.mean_squared_error(labels=labels, predictions=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.05)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "  return tf1.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "44bf417bf9c0"
+      },
+      "source": [
+        "Instantiate your `Estimator`, and train the model:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "922720812527"
+      },
+      "outputs": [],
+      "source": [
+        "estimator = tf1.estimator.Estimator(model_fn=_model_fn)\n",
+        "estimator.train(_input_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "17c9933c2d89"
+      },
+      "source": [
+        "Evaluate the program with the evaluation set:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HsOpjW5plH9Q"
+      },
+      "outputs": [],
+      "source": [
+        "estimator.evaluate(_eval_input_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "## TensorFlow 2: Train and evaluate with the built-in Keras methods\n",
+        "\n",
+        "This example demonstrates how to perform training and evaluation with Keras `Model.fit` and `Model.evaluate` in TensorFlow 2. (You can learn more in the [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate) guide.)\n",
+        "\n",
+        "- Start by preparing the dataset pipeline with the `tf.data.Dataset` APIs.\n",
+        "- Define a simple Keras [Sequential](https://www.tensorflow.org/guide/keras/sequential_model) model with one linear (`tf.keras.layers.Dense`) layer.\n",
+        "- Instantiate an Adagrad optimizer (`tf.keras.optimizers.Adagrad`).\n",
+        "- Configure the model for training by passing the `optimizer` variable and the mean-squared error (`\"mse\"`) loss to `Model.compile`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "atVciNgPs0fw"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "eval_dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).batch(1)\n",
+        "\n",
+        "model = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])\n",
+        "optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+        "\n",
+        "model.compile(optimizer=optimizer, loss=\"mse\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ed17a6291959"
+      },
+      "source": [
+        "With that, you are ready to train the model by calling `Model.fit`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a0b732534501"
+      },
+      "outputs": [],
+      "source": [
+        "model.fit(dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "74767288a2ea"
+      },
+      "source": [
+        "Finally, evaluate the model with `Model.evaluate`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kip65sYBlKiu"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(eval_dataset, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BuVYN0CHs5sD"
+      },
+      "source": [
+        "## TensorFlow 2: Train and evaluate with a custom training step and built-in Keras methods"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gHx_RUL8xcJ3"
+      },
+      "source": [
+        "In TensorFlow 2, you can also write your own custom training step function with `tf.GradientTape` to perform forward and backward passes, while still taking advantage of the built-in training support, such as `tf.keras.callbacks.Callback` and `tf.distribute.Strategy`. (Learn more in [Customizing what happens in Model.fit](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit) and [Writing custom training loops from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch).)\n",
+        "\n",
+        "In this example, start by creating a custom `tf.keras.Model` by subclassing `tf.keras.Sequential` that overrides `Model.train_step`. (Learn more about [subclassing tf.keras.Model](https://www.tensorflow.org/guide/keras/custom_layers_and_models)). Inside that class, define a custom `train_step` function that for each batch of data performs a forward pass and backward pass during one training step.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rSz_y0zOs8h2"
+      },
+      "outputs": [],
+      "source": [
+        "class CustomModel(tf.keras.Sequential):\n",
+        "  \"\"\"A custom sequential model that overrides `Model.train_step`.\"\"\"\n",
+        "\n",
+        "  def train_step(self, data):\n",
+        "    batch_data, labels = data\n",
+        "\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      predictions = self(batch_data, training=True)\n",
+        "      # Compute the loss value (the loss function is configured\n",
+        "      # in `Model.compile`).\n",
+        "      loss = self.compiled_loss(labels, predictions)\n",
+        "\n",
+        "    # Compute the gradients of the parameters with respect to the loss.\n",
+        "    gradients = tape.gradient(loss, self.trainable_variables)\n",
+        "    # Perform gradient descent by updating the weights/parameters.\n",
+        "    self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))\n",
+        "    # Update the metrics (includes the metric that tracks the loss).\n",
+        "    self.compiled_metrics.update_state(labels, predictions)\n",
+        "    # Return a dict mapping metric names to the current values.\n",
+        "    return {m.name: m.result() for m in self.metrics}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ee7c4f94d69b"
+      },
+      "source": [
+        "Next, as before:\n",
+        "- Prepare the dataset pipeline with `tf.data.Dataset`.\n",
+        "- Define a simple model with one `tf.keras.layers.Dense` layer.\n",
+        "- Instantiate Adagrad (`tf.keras.optimizers.Adagrad`)\n",
+        "- Configure the model for training with `Model.compile`, while using mean-squared error (`\"mse\"`) as the loss function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "01fcc2b1292c"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "eval_dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).batch(1)\n",
+        "\n",
+        "model = CustomModel([tf.keras.layers.Dense(1)])\n",
+        "optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+        "\n",
+        "model.compile(optimizer=optimizer, loss=\"mse\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "844543802ff5"
+      },
+      "source": [
+        "Call `Model.fit` to train the model:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "211be3620765"
+      },
+      "outputs": [],
+      "source": [
+        "model.fit(dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c93b9d6fc9d7"
+      },
+      "source": [
+        "And, finally, evaluate the program with `Model.evaluate`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nYO2wI1SlNCG"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(eval_dataset, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e9b5c9a4747b"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "Additional Keras resources you may find useful:\n",
+        "\n",
+        "- Guide: [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate)\n",
+        "- Guide: [Customize what happens in Model.fit](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit)\n",
+        "- Guide: [Writing a training loop from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch)\n",
+        "- Guide: [Making new Keras layers and models via subclassing](https://www.tensorflow.org/guide/keras/custom_layers_and_models)\n",
+        "\n",
+        "The following guides can assist with migrating distribution strategy workflows from `tf.estimator` APIs:\n",
+        "\n",
+        "- [Migrate from TPUEstimator to TPUStrategy](tpu_estimator.ipynb)\n",
+        "- [Migrate single-worker multiple-GPU training](mirrored_strategy.ipynb)\n",
+        "- [Migrate multi-worker CPU/GPU training](multi_worker_cpu_gpu_training.ipynb)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "migrating_estimator.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/migrating_feature_columns.ipynb b/site/en/guide/migrate/migrating_feature_columns.ipynb
new file mode 100644
index 00000000000..b2dbc5fe7c0
--- /dev/null
+++ b/site/en/guide/migrate/migrating_feature_columns.ipynb
@@ -0,0 +1,995 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l-23gBrt4x2B"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate `tf.feature_column`s to Keras preprocessing layers\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-5jGPDA2PDPI"
+      },
+      "source": [
+        "Training a model usually comes with some amount of feature preprocessing, particularly when dealing with structured data. When training a `tf.estimator.Estimator` in TensorFlow 1, you usually perform feature preprocessing with the `tf.feature_column` API. In TensorFlow 2, you can do this directly with Keras preprocessing layers.\n",
+        "\n",
+        "This migration guide demonstrates common feature transformations using both feature columns and preprocessing layers, followed by training a complete model with both APIs.\n",
+        "\n",
+        "First, start with a couple of necessary imports:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1\n",
+        "import math"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NVPYTQAWtDwH"
+      },
+      "source": [
+        "Now, add a utility function for calling a feature column for demonstration:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LAaifuuytJjM"
+      },
+      "outputs": [],
+      "source": [
+        "def call_feature_columns(feature_columns, inputs):\n",
+        "  # This is a convenient way to call a `feature_column` outside of an estimator\n",
+        "  # to display its output.\n",
+        "  feature_layer = tf1.keras.layers.DenseFeatures(feature_columns)\n",
+        "  return feature_layer(inputs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZJnw07hYDGYt"
+      },
+      "source": [
+        "## Input handling\n",
+        "\n",
+        "To use feature columns with an estimator, model inputs are always expected to be a dictionary of tensors:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "y0WUpQxsKEzf"
+      },
+      "outputs": [],
+      "source": [
+        "input_dict = {\n",
+        "  'foo': tf.constant([1]),\n",
+        "  'bar': tf.constant([0]),\n",
+        "  'baz': tf.constant([-1])\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xYsC6H_BJ8l3"
+      },
+      "source": [
+        "Each feature column needs to be created with a key to index into the source data. The output of all feature columns is concatenated and used by the estimator model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3fvIe3V8Ffjt"
+      },
+      "outputs": [],
+      "source": [
+        "columns = [\n",
+        "  tf1.feature_column.numeric_column('foo'),\n",
+        "  tf1.feature_column.numeric_column('bar'),\n",
+        "  tf1.feature_column.numeric_column('baz'),\n",
+        "]\n",
+        "call_feature_columns(columns, input_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hvPfCK2XGTyl"
+      },
+      "source": [
+        "In Keras, model input is much more flexible. A `tf.keras.Model` can handle a single tensor input, a list of tensor features, or a dictionary of tensor features. You can handle dictionary input by passing a dictionary of `tf.keras.Input` on model creation. Inputs will not be concatenated automatically, which allows them to be used in much more flexible ways. They can be concatenated with `tf.keras.layers.Concatenate`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5sYWENkgLWJ2"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = {\n",
+        "  'foo': tf.keras.Input(shape=()),\n",
+        "  'bar': tf.keras.Input(shape=()),\n",
+        "  'baz': tf.keras.Input(shape=()),\n",
+        "}\n",
+        "# Inputs are typically transformed by preprocessing layers before concatenation.\n",
+        "outputs = tf.keras.layers.Concatenate()(inputs.values())\n",
+        "model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
+        "model(input_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GXkmiuwXTS-B"
+      },
+      "source": [
+        "## One-hot encoding integer IDs\n",
+        "\n",
+        "A common feature transformation is one-hot encoding integer inputs of a known range. Here is an example using feature columns:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XasXzOgatgRF"
+      },
+      "outputs": [],
+      "source": [
+        "categorical_col = tf1.feature_column.categorical_column_with_identity(\n",
+        "    'type', num_buckets=3)\n",
+        "indicator_col = tf1.feature_column.indicator_column(categorical_col)\n",
+        "call_feature_columns(indicator_col, {'type': [0, 1, 2]})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iSCkJEQ6U-ru"
+      },
+      "source": [
+        "Using Keras preprocessing layers, these columns can be replaced by a single `tf.keras.layers.CategoryEncoding` layer with `output_mode` set to `'one_hot'`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "799lbMNNuAVz"
+      },
+      "outputs": [],
+      "source": [
+        "one_hot_layer = tf.keras.layers.CategoryEncoding(\n",
+        "    num_tokens=3, output_mode='one_hot')\n",
+        "one_hot_layer([0, 1, 2])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kNzRtESU7tga"
+      },
+      "source": [
+        "Note: For large one-hot encodings, it is much more efficient to use a sparse representation of the output. If you pass `sparse=True` to the `CategoryEncoding` layer, the output of the layer will be a `tf.sparse.SparseTensor`, which can be efficiently handled as input to a `tf.keras.layers.Dense` layer."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Zf7kjhTiAErK"
+      },
+      "source": [
+        "## Normalizing numeric features\n",
+        "\n",
+        "When handling continuous, floating-point features with feature columns, you need to use a `tf.feature_column.numeric_column`. In the case where the input is already normalized, converting this to Keras is trivial. You can simply use a `tf.keras.Input` directly into your model, as shown above.\n",
+        "\n",
+        "A `numeric_column` can also be used to normalize input:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HbTMGB9XctGx"
+      },
+      "outputs": [],
+      "source": [
+        "def normalize(x):\n",
+        "  mean, variance = (2.0, 1.0)\n",
+        "  return (x - mean) / math.sqrt(variance)\n",
+        "numeric_col = tf1.feature_column.numeric_column('col', normalizer_fn=normalize)\n",
+        "call_feature_columns(numeric_col, {'col': tf.constant([[0.], [1.], [2.]])})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "M9cyhPR_drOz"
+      },
+      "source": [
+        "In contrast, with Keras, this normalization can be done with `tf.keras.layers.Normalization`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8bcgG-yOdqUH"
+      },
+      "outputs": [],
+      "source": [
+        "normalization_layer = tf.keras.layers.Normalization(mean=2.0, variance=1.0)\n",
+        "normalization_layer(tf.constant([[0.], [1.], [2.]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d1InD_4QLKU-"
+      },
+      "source": [
+        "## Bucketizing and one-hot encoding numeric features"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k5e0b8iOLRzd"
+      },
+      "source": [
+        "Another common transformation of continuous, floating point inputs is to bucketize then to integers of a fixed range.\n",
+        "\n",
+        "In feature columns, this can be achieved with a `tf.feature_column.bucketized_column`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_rbx6qQ-LQx7"
+      },
+      "outputs": [],
+      "source": [
+        "numeric_col = tf1.feature_column.numeric_column('col')\n",
+        "bucketized_col = tf1.feature_column.bucketized_column(numeric_col, [1, 4, 5])\n",
+        "call_feature_columns(bucketized_col, {'col': tf.constant([1., 2., 3., 4., 5.])})\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PCYu-XtwXahx"
+      },
+      "source": [
+        "In Keras, this can be replaced by `tf.keras.layers.Discretization`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QK1WOG2uVVsL"
+      },
+      "outputs": [],
+      "source": [
+        "discretization_layer = tf.keras.layers.Discretization(bin_boundaries=[1, 4, 5])\n",
+        "one_hot_layer = tf.keras.layers.CategoryEncoding(\n",
+        "    num_tokens=4, output_mode='one_hot')\n",
+        "one_hot_layer(discretization_layer([1., 2., 3., 4., 5.]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5bm9tJZAgpt4"
+      },
+      "source": [
+        "## One-hot encoding string data with a vocabulary\n",
+        "\n",
+        "Handling string features often requires a vocabulary lookup to translate strings into indices. Here is an example using feature columns to lookup strings and then one-hot encode the indices:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3fG_igjhukCO"
+      },
+      "outputs": [],
+      "source": [
+        "vocab_col = tf1.feature_column.categorical_column_with_vocabulary_list(\n",
+        "    'sizes',\n",
+        "    vocabulary_list=['small', 'medium', 'large'],\n",
+        "    num_oov_buckets=0)\n",
+        "indicator_col = tf1.feature_column.indicator_column(vocab_col)\n",
+        "call_feature_columns(indicator_col, {'sizes': ['small', 'medium', 'large']})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8rBgllRtY738"
+      },
+      "source": [
+        "Using Keras preprocessing layers, use the `tf.keras.layers.StringLookup` layer with `output_mode` set to `'one_hot'`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "arnPlSrWvDMe"
+      },
+      "outputs": [],
+      "source": [
+        "string_lookup_layer = tf.keras.layers.StringLookup(\n",
+        "    vocabulary=['small', 'medium', 'large'],\n",
+        "    num_oov_indices=0,\n",
+        "    output_mode='one_hot')\n",
+        "string_lookup_layer(['small', 'medium', 'large'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f76MVVYO8LB5"
+      },
+      "source": [
+        "Note: For large one-hot encodings, it is much more efficient to use a sparse representation of the output. If you pass `sparse=True` to the `StringLookup` layer, the output of the layer will be a `tf.sparse.SparseTensor`, which can be efficiently handled as input to a `tf.keras.layers.Dense` layer."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c1CmfSXQZHE5"
+      },
+      "source": [
+        "## Embedding string data with a vocabulary\n",
+        "\n",
+        "For larger vocabularies, an embedding is often needed for good performance. Here is an example embedding a string feature using feature columns:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "C3RK4HFazxlU"
+      },
+      "outputs": [],
+      "source": [
+        "vocab_col = tf1.feature_column.categorical_column_with_vocabulary_list(\n",
+        "    'col',\n",
+        "    vocabulary_list=['small', 'medium', 'large'],\n",
+        "    num_oov_buckets=0)\n",
+        "embedding_col = tf1.feature_column.embedding_column(vocab_col, 4)\n",
+        "call_feature_columns(embedding_col, {'col': ['small', 'medium', 'large']})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3aTRVJ6qZZH0"
+      },
+      "source": [
+        "Using Keras preprocessing layers, this can be achieved by combining a `tf.keras.layers.StringLookup` layer and an `tf.keras.layers.Embedding` layer. The default output for the `StringLookup` will be integer indices which can be fed directly into an embedding.\n",
+        "\n",
+        "Note: The `Embedding` layer contains trainable parameters. While the `StringLookup` layer can be applied to data inside or outside of a model, the `Embedding` must always be part of a trainable Keras model to train correctly."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8resGZPo0Fho"
+      },
+      "outputs": [],
+      "source": [
+        "string_lookup_layer = tf.keras.layers.StringLookup(\n",
+        "    vocabulary=['small', 'medium', 'large'], num_oov_indices=0)\n",
+        "embedding = tf.keras.layers.Embedding(3, 4)\n",
+        "embedding(string_lookup_layer(['small', 'medium', 'large']))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UwqvADV6HRdC"
+      },
+      "source": [
+        "## Summing weighted categorical data\n",
+        "\n",
+        "In some cases, you need to deal with categorical data where each occurance of a category comes with an associated weight. In feature columns, this is handled with `tf.feature_column.weighted_categorical_column`. When paired with an `indicator_column`, this has the effect of summing weights per category."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "02HqjPLMRxWn"
+      },
+      "outputs": [],
+      "source": [
+        "ids = tf.constant([[5, 11, 5, 17, 17]])\n",
+        "weights = tf.constant([[0.5, 1.5, 0.7, 1.8, 0.2]])\n",
+        "\n",
+        "categorical_col = tf1.feature_column.categorical_column_with_identity(\n",
+        "    'ids', num_buckets=20)\n",
+        "weighted_categorical_col = tf1.feature_column.weighted_categorical_column(\n",
+        "    categorical_col, 'weights')\n",
+        "indicator_col = tf1.feature_column.indicator_column(weighted_categorical_col)\n",
+        "call_feature_columns(indicator_col, {'ids': ids, 'weights': weights})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "98jaq7Q3S9aG"
+      },
+      "source": [
+        "In Keras, this can be done by passing a `count_weights` input to `tf.keras.layers.CategoryEncoding` with `output_mode='count'`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JsoYUUgRS7hu"
+      },
+      "outputs": [],
+      "source": [
+        "ids = tf.constant([[5, 11, 5, 17, 17]])\n",
+        "weights = tf.constant([[0.5, 1.5, 0.7, 1.8, 0.2]])\n",
+        "\n",
+        "# Using sparse output is more efficient when `num_tokens` is large.\n",
+        "count_layer = tf.keras.layers.CategoryEncoding(\n",
+        "    num_tokens=20, output_mode='count', sparse=True)\n",
+        "tf.sparse.to_dense(count_layer(ids, count_weights=weights))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gBJxb6y2GasI"
+      },
+      "source": [
+        "## Embedding weighted categorical data\n",
+        "\n",
+        "You might alternately want to embed weighted categorical inputs. In feature columns, the `embedding_column` contains a `combiner` argument. If any sample\n",
+        "contains multiple entries for a category, they will be combined according to the argument setting (by default `'mean'`)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AjOt1wgmT5mM"
+      },
+      "outputs": [],
+      "source": [
+        "ids = tf.constant([[5, 11, 5, 17, 17]])\n",
+        "weights = tf.constant([[0.5, 1.5, 0.7, 1.8, 0.2]])\n",
+        "\n",
+        "categorical_col = tf1.feature_column.categorical_column_with_identity(\n",
+        "    'ids', num_buckets=20)\n",
+        "weighted_categorical_col = tf1.feature_column.weighted_categorical_column(\n",
+        "    categorical_col, 'weights')\n",
+        "embedding_col = tf1.feature_column.embedding_column(\n",
+        "    weighted_categorical_col, 4, combiner='mean')\n",
+        "call_feature_columns(embedding_col, {'ids': ids, 'weights': weights})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fd6eluARXndC"
+      },
+      "source": [
+        "In Keras, there is no `combiner` option to `tf.keras.layers.Embedding`, but you can achieve the same effect with `tf.keras.layers.Dense`. The `embedding_column` above is simply linearly combining embedding vectors according to category weight. Though not obvious at first, it is exactly equivalent to representing your categorical inputs as a sparse weight vector of size `(num_tokens)`, and multiplying them by a `Dense` kernel of shape `(embedding_size, num_tokens)`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y-vZvPyiYilE"
+      },
+      "outputs": [],
+      "source": [
+        "ids = tf.constant([[5, 11, 5, 17, 17]])\n",
+        "weights = tf.constant([[0.5, 1.5, 0.7, 1.8, 0.2]])\n",
+        "\n",
+        "# For `combiner='mean'`, normalize your weights to sum to 1. Removing this line\n",
+        "# would be equivalent to an `embedding_column` with `combiner='sum'`.\n",
+        "weights = weights / tf.reduce_sum(weights, axis=-1, keepdims=True)\n",
+        "\n",
+        "count_layer = tf.keras.layers.CategoryEncoding(\n",
+        "    num_tokens=20, output_mode='count', sparse=True)\n",
+        "embedding_layer = tf.keras.layers.Dense(4, use_bias=False)\n",
+        "embedding_layer(count_layer(ids, count_weights=weights))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3I5loEx80MVm"
+      },
+      "source": [
+        "## Complete training example\n",
+        "\n",
+        "To show a complete training workflow, first prepare some data with three features of different types:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D_7nyBee0ZBV"
+      },
+      "outputs": [],
+      "source": [
+        "features = {\n",
+        "    'type': [0, 1, 1],\n",
+        "    'size': ['small', 'small', 'medium'],\n",
+        "    'weight': [2.7, 1.8, 1.6],\n",
+        "}\n",
+        "labels = [1, 1, 0]\n",
+        "predict_features = {'type': [0], 'size': ['foo'], 'weight': [-0.7]}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e_4Xx2c37lqD"
+      },
+      "source": [
+        "Define some common constants for both TensorFlow 1 and TensorFlow 2 workflows:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3cyfQZ7z8jZh"
+      },
+      "outputs": [],
+      "source": [
+        "vocab = ['small', 'medium', 'large']\n",
+        "one_hot_dims = 3\n",
+        "embedding_dims = 4\n",
+        "weight_mean = 2.0\n",
+        "weight_variance = 1.0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ywCgU7CMIfTH"
+      },
+      "source": [
+        "### With feature columns\n",
+        "\n",
+        "Feature columns must be passed as a list to the estimator on creation, and will be called implicitly during training."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Wsdhlm-uipr1"
+      },
+      "outputs": [],
+      "source": [
+        "categorical_col = tf1.feature_column.categorical_column_with_identity(\n",
+        "    'type', num_buckets=one_hot_dims)\n",
+        "# Convert index to one-hot; e.g., [2] -> [0,0,1].\n",
+        "indicator_col = tf1.feature_column.indicator_column(categorical_col)\n",
+        "\n",
+        "# Convert strings to indices; e.g., ['small'] -> [1].\n",
+        "vocab_col = tf1.feature_column.categorical_column_with_vocabulary_list(\n",
+        "    'size', vocabulary_list=vocab, num_oov_buckets=1)\n",
+        "# Embed the indices.\n",
+        "embedding_col = tf1.feature_column.embedding_column(vocab_col, embedding_dims)\n",
+        "\n",
+        "normalizer_fn = lambda x: (x - weight_mean) / math.sqrt(weight_variance)\n",
+        "# Normalize the numeric inputs; e.g., [2.0] -> [0.0].\n",
+        "numeric_col = tf1.feature_column.numeric_column(\n",
+        "    'weight', normalizer_fn=normalizer_fn)\n",
+        "\n",
+        "estimator = tf1.estimator.DNNClassifier(\n",
+        "    feature_columns=[indicator_col, embedding_col, numeric_col],\n",
+        "    hidden_units=[1])\n",
+        "\n",
+        "def _input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "\n",
+        "estimator.train(_input_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qPIeG_YtfNV1"
+      },
+      "source": [
+        "The feature columns will also be used to transform input data when running inference on the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "K-AIIB8CfSqt"
+      },
+      "outputs": [],
+      "source": [
+        "def _predict_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices(predict_features).batch(1)\n",
+        "\n",
+        "next(estimator.predict(_predict_fn))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "baMA01cBIivo"
+      },
+      "source": [
+        "### With Keras preprocessing layers\n",
+        "\n",
+        "Keras preprocessing layers are more flexible in where they can be called. A layer can be applied directly to tensors, used inside a `tf.data` input pipeline, or built directly into a trainable Keras model.\n",
+        "\n",
+        "In this example, you will apply preprocessing layers inside a `tf.data` input pipeline. To do this, you can define a separate `tf.keras.Model` to preprocess your input features. This model is not trainable, but is a convenient way to group preprocessing layers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NMz8RfMQdCZf"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = {\n",
+        "  'type': tf.keras.Input(shape=(), dtype='int64'),\n",
+        "  'size': tf.keras.Input(shape=(), dtype='string'),\n",
+        "  'weight': tf.keras.Input(shape=(), dtype='float32'),\n",
+        "}\n",
+        "# Convert index to one-hot; e.g., [2] -> [0,0,1].\n",
+        "type_output = tf.keras.layers.CategoryEncoding(\n",
+        "      one_hot_dims, output_mode='one_hot')(inputs['type'])\n",
+        "# Convert size strings to indices; e.g., ['small'] -> [1].\n",
+        "size_output = tf.keras.layers.StringLookup(vocabulary=vocab)(inputs['size'])\n",
+        "# Normalize the numeric inputs; e.g., [2.0] -> [0.0].\n",
+        "weight_output = tf.keras.layers.Normalization(\n",
+        "      axis=None, mean=weight_mean, variance=weight_variance)(inputs['weight'])\n",
+        "outputs = {\n",
+        "  'type': type_output,\n",
+        "  'size': size_output,\n",
+        "  'weight': weight_output,\n",
+        "}\n",
+        "preprocessing_model = tf.keras.Model(inputs, outputs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NRfISnj3NGlW"
+      },
+      "source": [
+        "Note: As an alternative to supplying a vocabulary and normalization statistics on layer creation, many preprocessing layers provide an `adapt()` method for learning layer state directly from the input data. See the [preprocessing guide](https://www.tensorflow.org/guide/keras/preprocessing_layers#the_adapt_method) for more details.\n",
+        "\n",
+        "You can now apply this model inside a call to `tf.data.Dataset.map`. Please note that the function passed to `map` will automatically be converted into\n",
+        "a `tf.function`, and usual caveats for writing `tf.function` code apply (no side effects)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c_6xAUnbNREh"
+      },
+      "outputs": [],
+      "source": [
+        "# Apply the preprocessing in tf.data.Dataset.map.\n",
+        "dataset = tf.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "dataset = dataset.map(lambda x, y: (preprocessing_model(x), y),\n",
+        "                      num_parallel_calls=tf.data.AUTOTUNE)\n",
+        "# Display a preprocessed input sample.\n",
+        "next(dataset.take(1).as_numpy_iterator())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8_4u3J4NdJ8R"
+      },
+      "source": [
+        "Next, you can define a separate `Model` containing the trainable layers. Note how the inputs to this model now reflect the preprocessed feature types and shapes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kC9OZO5ldmP-"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = {\n",
+        "  'type': tf.keras.Input(shape=(one_hot_dims,), dtype='float32'),\n",
+        "  'size': tf.keras.Input(shape=(), dtype='int64'),\n",
+        "  'weight': tf.keras.Input(shape=(), dtype='float32'),\n",
+        "}\n",
+        "# Since the embedding is trainable, it needs to be part of the training model.\n",
+        "embedding = tf.keras.layers.Embedding(len(vocab), embedding_dims)\n",
+        "outputs = tf.keras.layers.Concatenate()([\n",
+        "  inputs['type'],\n",
+        "  embedding(inputs['size']),\n",
+        "  tf.expand_dims(inputs['weight'], -1),\n",
+        "])\n",
+        "outputs = tf.keras.layers.Dense(1)(outputs)\n",
+        "training_model = tf.keras.Model(inputs, outputs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ir-cn2H_d5R7"
+      },
+      "source": [
+        "You can now train the `training_model` with `tf.keras.Model.fit`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6TS3YJ2vnvlW"
+      },
+      "outputs": [],
+      "source": [
+        "# Train on the preprocessed data.\n",
+        "training_model.compile(\n",
+        "    loss=tf.keras.losses.BinaryCrossentropy(from_logits=True))\n",
+        "training_model.fit(dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pSaEbOE4ecsy"
+      },
+      "source": [
+        "Finally, at inference time, it can be useful to combine these separate stages into a single model that handles raw feature inputs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QHjbIZYneboO"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = preprocessing_model.input\n",
+        "outputs = training_model(preprocessing_model(inputs))\n",
+        "inference_model = tf.keras.Model(inputs, outputs)\n",
+        "\n",
+        "predict_dataset = tf.data.Dataset.from_tensor_slices(predict_features).batch(1)\n",
+        "inference_model.predict(predict_dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O01VQIxCWBxU"
+      },
+      "source": [
+        "This composed model can be saved as a `.keras` file for later use."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6tsyVZgh7Pve"
+      },
+      "outputs": [],
+      "source": [
+        "inference_model.save('model.keras')\n",
+        "restored_model = tf.keras.models.load_model('model.keras')\n",
+        "restored_model.predict(predict_dataset)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IXMBwzggwUjI"
+      },
+      "source": [
+        "Note: Preprocessing layers are not trainable, which allows you to apply them *asynchronously* using `tf.data`. This has performance benefits, as you can both prefetch preprocessed batches, and free up any accelerators to focus on the differentiable parts of a model (learn more in the _Prefetching_ section of the [Better performance with the `tf.data` API](../data_performance.ipynb) guide). As this guide shows, separating preprocessing during training and composing it during inference is a flexible way to leverage these performance gains. However, if your model is small or preprocessing time is negligible, it may be simpler to build preprocessing into a complete model from the start. To do this you can build a single model starting with `tf.keras.Input`, followed by preprocessing layers, followed by trainable layers."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2pjp7Z18gRCQ"
+      },
+      "source": [
+        "## Feature column equivalence table\n",
+        "\n",
+        "For reference, here is an approximate correspondence between feature columns and\n",
+        "Keras preprocessing layers:\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "  \n",
+        "
Feature columnKeras layer
`tf.feature_column.bucketized_column``tf.keras.layers.Discretization`
`tf.feature_column.categorical_column_with_hash_bucket``tf.keras.layers.Hashing`
`tf.feature_column.categorical_column_with_identity``tf.keras.layers.CategoryEncoding`
`tf.feature_column.categorical_column_with_vocabulary_file``tf.keras.layers.StringLookup` or `tf.keras.layers.IntegerLookup`
`tf.feature_column.categorical_column_with_vocabulary_list``tf.keras.layers.StringLookup` or `tf.keras.layers.IntegerLookup`
`tf.feature_column.crossed_column``tf.keras.layers.experimental.preprocessing.HashedCrossing`
`tf.feature_column.embedding_column``tf.keras.layers.Embedding`
`tf.feature_column.indicator_column``output_mode='one_hot'` or `output_mode='multi_hot'`*
`tf.feature_column.numeric_column``tf.keras.layers.Normalization`
`tf.feature_column.sequence_categorical_column_with_hash_bucket``tf.keras.layers.Hashing`
`tf.feature_column.sequence_categorical_column_with_identity``tf.keras.layers.CategoryEncoding`
`tf.feature_column.sequence_categorical_column_with_vocabulary_file``tf.keras.layers.StringLookup`, `tf.keras.layers.IntegerLookup`, or `tf.keras.layer.TextVectorization`†
`tf.feature_column.sequence_categorical_column_with_vocabulary_list``tf.keras.layers.StringLookup`, `tf.keras.layers.IntegerLookup`, or `tf.keras.layer.TextVectorization`†
`tf.feature_column.sequence_numeric_column``tf.keras.layers.Normalization`
`tf.feature_column.weighted_categorical_column``tf.keras.layers.CategoryEncoding`
\n",
+        "\n",
+        "\\* The `output_mode` can be passed to `tf.keras.layers.CategoryEncoding`, `tf.keras.layers.StringLookup`, `tf.keras.layers.IntegerLookup`, and `tf.keras.layers.TextVectorization`.\n",
+        "\n",
+        "† `tf.keras.layers.TextVectorization` can handle freeform text input directly (for example, entire sentences or paragraphs). This is not one-to-one replacement for categorical sequence handling in TensorFlow 1, but may offer a convenient replacement for ad-hoc text preprocessing.\n",
+        "\n",
+        "Note: Linear estimators, such as `tf.estimator.LinearClassifier`, can handle direct categorical input (integer indices) without an `embedding_column` or `indicator_column`. However, integer indices cannot be passed directly to `tf.keras.layers.Dense` or `tf.keras.experimental.LinearModel`. These inputs should be first encoded with `tf.layers.CategoryEncoding` with `output_mode='count'` (and `sparse=True` if the category sizes are large) before calling into `Dense` or `LinearModel`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AQCJ6lM3YDq_"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        " - For more information on Keras preprocessing layers, go to the [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) guide.\n",
+        " - For a more in-depth example of applying preprocessing layers to structured data, refer to the [Classify structured data using Keras preprocessing layers](../../tutorials/structured_data/preprocessing_layers.ipynb) tutorial."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "migrating_feature_columns.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/migration_debugging.ipynb b/site/en/guide/migrate/migration_debugging.ipynb
new file mode 100644
index 00000000000..25cb7f9065f
--- /dev/null
+++ b/site/en/guide/migrate/migration_debugging.ipynb
@@ -0,0 +1,799 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FEL3NlTTDlSX"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "FlUw7tSKbtg4"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Debug a TensorFlow 2 migrated training pipeline\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zTwPu-w6M5sz"
+      },
+      "source": [
+        "This notebook demonstrates how to debug a training pipeline when migrating to TensorFlow 2 (TF2). It consists of following components:\n",
+        "1. Suggested steps and code samples for debugging training pipeline\n",
+        "2. Tools for debugging\n",
+        "3. Other related resources\n",
+        "\n",
+        "One assumption is you have the TensorFlow 1 (TF1.x) code and trained models for comparison, and you want to build a TF2 model that achieves similar validation accuracy.\n",
+        "\n",
+        "This notebook does **NOT** cover debugging performance issues for training/inference speed or memory usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fKm9R4CtOAP3"
+      },
+      "source": [
+        "## Debugging workflow\n",
+        "\n",
+        "Below is a general workflow for debugging your TF2 training pipelines. Note that you do not need to follow these steps in order. You can also use a binary search approach where you test the model in an intermediate step and narrow down the debugging scope. \n",
+        "\n",
+        "1.  Fix compile and runtime errors\n",
+        "\n",
+        "2.  Single forward pass validation (in a separate\n",
+        "    [guide](./validate_correctness.ipynb))\n",
+        "\n",
+        "    a. On single CPU device\n",
+        "\n",
+        "    *   Verify variables are created only once\n",
+        "    *   Check variable counts, names, and shapes match\n",
+        "    *   Reset all variables, check numerical equivalence with all randomness\n",
+        "        disabled\n",
+        "    *   Align random number generation, check numerical equivalence in inference\n",
+        "    *   (Optional) Check checkpoints are loaded properly and TF1.x/TF2 models\n",
+        "        generate identical output\n",
+        "\n",
+        "    b. On single GPU/TPU device\n",
+        "\n",
+        "    c. With multi-device strategies\n",
+        "\n",
+        "3.  Model training numerical equivalence validation for a few steps (code\n",
+        "    samples available below)\n",
+        "\n",
+        "    a. Single training step validation using small and fixed data on single CPU\n",
+        "    device. Specifically, check numerical equivalence for the following\n",
+        "    components\n",
+        "\n",
+        "    *   losses computation\n",
+        "    *   metrics\n",
+        "    *   learning rate\n",
+        "    *   gradient computation and update\n",
+        "\n",
+        "    b. Check statistics after training 3 or more steps to verify optimizer behaviors like the momentum, still with fixed data on single CPU device\n",
+        "\n",
+        "    c. On single GPU/TPU device\n",
+        "\n",
+        "    d. With multi-device strategies (check the intro for [MultiProcessRunner](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/distribute/multi_process_runner.py#L108) at the bottom)\n",
+        "\n",
+        "4.  End-to-end convergence testing on real dataset\n",
+        "\n",
+        "    a. Check training behaviors with TensorBoard\n",
+        "\n",
+        "    *   use simple optimizers e.g., SGD and simple distribution strategies e.g.\n",
+        "        `tf.distribute.OneDeviceStrategy` first\n",
+        "    *   training metrics\n",
+        "    *   evaluation metrics\n",
+        "    *   figure out what the reasonable tolerance for inherent randomness is\n",
+        "\n",
+        "    b. Check equivalence with advanced optimizer/learning rate\n",
+        "    scheduler/distribution strategies\n",
+        "\n",
+        "    c. Check equivalence when using mixed precision\n",
+        "\n",
+        "5.  Additional product benchmarks"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XKakQBI9-FLb"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "i1ghHyXl-Oqd"
+      },
+      "outputs": [],
+      "source": [
+        "# The `DeterministicRandomTestTool` is only available from Tensorflow 2.8:\n",
+        "!pip install -q \"tensorflow==2.9.*\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "usyRSlIRl3r2"
+      },
+      "source": [
+        "### Single forward pass validation \n",
+        "\n",
+        "Single forward pass validation, including checkpoint loading, is covered in a different [colab](./validate_correctness.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HVBQbsZeVL_V"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "import unittest\n",
+        "import numpy as np\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as v1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4M104dt7m5cC"
+      },
+      "source": [
+        "### Model training numerical equivalence validation for a few steps"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v2Nz2Ni1EkMz"
+      },
+      "source": [
+        "Set up model configuration and prepare a fake dataset."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hUxXadzKU9rT"
+      },
+      "outputs": [],
+      "source": [
+        "params = {\n",
+        "    'input_size': 3,\n",
+        "    'num_classes': 3,\n",
+        "    'layer_1_size': 2,\n",
+        "    'layer_2_size': 2,\n",
+        "    'num_train_steps': 100,\n",
+        "    'init_lr': 1e-3,\n",
+        "    'end_lr': 0.0,\n",
+        "    'decay_steps': 1000,\n",
+        "    'lr_power': 1.0,\n",
+        "}\n",
+        "\n",
+        "# make a small fixed dataset\n",
+        "fake_x = np.ones((2, params['input_size']), dtype=np.float32)\n",
+        "fake_y = np.zeros((2, params['num_classes']), dtype=np.int32)\n",
+        "fake_y[0][0] = 1\n",
+        "fake_y[1][1] = 1\n",
+        "\n",
+        "step_num = 3"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lV_n3Ukmz4Un"
+      },
+      "source": [
+        "Define the TF1.x model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ATa5fzL8mAwl"
+      },
+      "outputs": [],
+      "source": [
+        "# Assume there is an existing TF1.x model using estimator API\n",
+        "# Wrap the model_fn to log necessary tensors for result comparison\n",
+        "class SimpleModelWrapper():\n",
+        "  def __init__(self):\n",
+        "    self.logged_ops = {}\n",
+        "    self.logs = {\n",
+        "        'step': [],\n",
+        "        'lr': [],\n",
+        "        'loss': [],\n",
+        "        'grads_and_vars': [],\n",
+        "        'layer_out': []}\n",
+        "     \n",
+        "  def model_fn(self, features, labels, mode, params):\n",
+        "      out_1 = tf.compat.v1.layers.dense(features, units=params['layer_1_size'])\n",
+        "      out_2 = tf.compat.v1.layers.dense(out_1, units=params['layer_2_size'])\n",
+        "      logits = tf.compat.v1.layers.dense(out_2, units=params['num_classes'])\n",
+        "      loss = tf.compat.v1.losses.softmax_cross_entropy(labels, logits)\n",
+        "\n",
+        "      # skip EstimatorSpec details for prediction and evaluation \n",
+        "      if mode == tf.estimator.ModeKeys.PREDICT:\n",
+        "          pass\n",
+        "      if mode == tf.estimator.ModeKeys.EVAL:\n",
+        "          pass\n",
+        "      assert mode == tf.estimator.ModeKeys.TRAIN\n",
+        "\n",
+        "      global_step = tf.compat.v1.train.get_or_create_global_step()\n",
+        "      lr = tf.compat.v1.train.polynomial_decay(\n",
+        "        learning_rate=params['init_lr'],\n",
+        "        global_step=global_step,\n",
+        "        decay_steps=params['decay_steps'],\n",
+        "        end_learning_rate=params['end_lr'],\n",
+        "        power=params['lr_power'])\n",
+        "      \n",
+        "      optmizer = tf.compat.v1.train.GradientDescentOptimizer(lr)\n",
+        "      grads_and_vars = optmizer.compute_gradients(\n",
+        "          loss=loss,\n",
+        "          var_list=graph.get_collection(\n",
+        "              tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES))\n",
+        "      train_op = optmizer.apply_gradients(\n",
+        "          grads_and_vars,\n",
+        "          global_step=global_step)\n",
+        "      \n",
+        "      # log tensors\n",
+        "      self.logged_ops['step'] = global_step\n",
+        "      self.logged_ops['lr'] = lr\n",
+        "      self.logged_ops['loss'] = loss\n",
+        "      self.logged_ops['grads_and_vars'] = grads_and_vars\n",
+        "      self.logged_ops['layer_out'] = {\n",
+        "          'layer_1': out_1,\n",
+        "          'layer_2': out_2,\n",
+        "          'logits': logits}\n",
+        "\n",
+        "      return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)\n",
+        "\n",
+        "  def update_logs(self, logs):\n",
+        "    for key in logs.keys():\n",
+        "      model_tf1.logs[key].append(logs[key])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kki9yILSKS7f"
+      },
+      "source": [
+        "The following [`v1.keras.utils.DeterministicRandomTestTool`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/keras/utils/DeterministicRandomTestTool) class provides a context manager `scope()` that can make stateful random operations use the same seed across both TF1 graphs/sessions and eager execution,\n",
+        "\n",
+        "The tool provides two testing modes: \n",
+        "1. `constant` which uses the same seed for every single operation no matter how many times it has been called and,\n",
+        "2. `num_random_ops` which uses the number of previously-observed stateful random operations as the operation seed.\n",
+        "\n",
+        "This applies both to the stateful random operations used for creating and initializing variables, and to the stateful random operations used in computation (such as for dropout layers)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "X6Y3RWMoKOl8"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mk5-ZzxcErX5"
+      },
+      "source": [
+        "Run the TF1.x model in graph mode. Collect statistics for first 3 training steps for numerical equivalence comparison."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r5zhJHvsWA24"
+      },
+      "outputs": [],
+      "source": [
+        "with random_tool.scope():\n",
+        "  graph = tf.Graph()\n",
+        "  with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "    model_tf1 = SimpleModelWrapper()\n",
+        "    # build the model\n",
+        "    inputs = tf.compat.v1.placeholder(tf.float32, shape=(None, params['input_size']))\n",
+        "    labels = tf.compat.v1.placeholder(tf.float32, shape=(None, params['num_classes']))\n",
+        "    spec = model_tf1.model_fn(inputs, labels, tf.estimator.ModeKeys.TRAIN, params)\n",
+        "    train_op = spec.train_op\n",
+        "\n",
+        "    sess.run(tf.compat.v1.global_variables_initializer())\n",
+        "    for step in range(step_num):\n",
+        "      # log everything and update the model for one step\n",
+        "      logs, _ = sess.run(\n",
+        "          [model_tf1.logged_ops, train_op],\n",
+        "          feed_dict={inputs: fake_x, labels: fake_y})\n",
+        "      model_tf1.update_logs(logs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eZxjI8Nxz9Ea"
+      },
+      "source": [
+        "Define the TF2 model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AA67rh2TkS1M"
+      },
+      "outputs": [],
+      "source": [
+        "class SimpleModel(tf.keras.Model):\n",
+        "  def __init__(self, params, *args, **kwargs):\n",
+        "    super(SimpleModel, self).__init__(*args, **kwargs)\n",
+        "    # define the model\n",
+        "    self.dense_1 = tf.keras.layers.Dense(params['layer_1_size'])\n",
+        "    self.dense_2 = tf.keras.layers.Dense(params['layer_2_size'])\n",
+        "    self.out = tf.keras.layers.Dense(params['num_classes'])\n",
+        "    learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay(\n",
+        "      initial_learning_rate=params['init_lr'],\n",
+        "      decay_steps=params['decay_steps'],\n",
+        "      end_learning_rate=params['end_lr'],\n",
+        "      power=params['lr_power'])  \n",
+        "    self.optimizer = tf.keras.optimizers.legacy.SGD(learning_rate_fn)\n",
+        "    self.compiled_loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
+        "    self.logs = {\n",
+        "        'lr': [],\n",
+        "        'loss': [],\n",
+        "        'grads': [],\n",
+        "        'weights': [],\n",
+        "        'layer_out': []}\n",
+        "\n",
+        "  def call(self, inputs):\n",
+        "    out_1 = self.dense_1(inputs)\n",
+        "    out_2 = self.dense_2(out_1)\n",
+        "    logits = self.out(out_2)\n",
+        "    # log output features for every layer for comparison\n",
+        "    layer_wise_out = {\n",
+        "        'layer_1': out_1,\n",
+        "        'layer_2': out_2,\n",
+        "        'logits': logits}\n",
+        "    self.logs['layer_out'].append(layer_wise_out)\n",
+        "    return logits\n",
+        "\n",
+        "  def train_step(self, data):\n",
+        "    x, y = data\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      logits = self(x)\n",
+        "      loss = self.compiled_loss(y, logits)\n",
+        "    grads = tape.gradient(loss, self.trainable_weights)\n",
+        "    # log training statistics\n",
+        "    step = self.optimizer.iterations.numpy()\n",
+        "    self.logs['lr'].append(self.optimizer.learning_rate(step).numpy())\n",
+        "    self.logs['loss'].append(loss.numpy())\n",
+        "    self.logs['grads'].append(grads)\n",
+        "    self.logs['weights'].append(self.trainable_weights)\n",
+        "    # update model\n",
+        "    self.optimizer.apply_gradients(zip(grads, self.trainable_weights))\n",
+        "    return"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "I5smAcaEE8nX"
+      },
+      "source": [
+        "Run the TF2 model in eager mode. Collect statistics for first 3 training steps for numerical equivalence comparison."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q0AbXF_eE8cS"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  model_tf2 = SimpleModel(params)\n",
+        "  for step in range(step_num):\n",
+        "    model_tf2.train_step([fake_x, fake_y])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cjJDjLcAz_gU"
+      },
+      "source": [
+        "Compare numerical equivalence for first few training steps.\n",
+        "\n",
+        "You can also check the [Validating correctness & numerical equivalence notebook](./validate_correctness.ipynb) for additional advice for numerical equivalence."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6CbCUbsCiabC"
+      },
+      "outputs": [],
+      "source": [
+        "np.testing.assert_allclose(model_tf1.logs['lr'], model_tf2.logs['lr'])\n",
+        "np.testing.assert_allclose(model_tf1.logs['loss'], model_tf2.logs['loss'])\n",
+        "for step in range(step_num):\n",
+        "  for name in model_tf1.logs['layer_out'][step]:\n",
+        "    np.testing.assert_allclose(\n",
+        "        model_tf1.logs['layer_out'][step][name],\n",
+        "        model_tf2.logs['layer_out'][step][name])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dhVuuciimLIY"
+      },
+      "source": [
+        "#### Unit tests"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sXZYFC6Hhqeb"
+      },
+      "source": [
+        "There are a few types of unit testing that can help debug your migration code.\n",
+        "1. Single forward pass validation\n",
+        "2. Model training numerical equivalence validation for a few steps\n",
+        "3. Benchmark inference performance\n",
+        "4. The trained model makes correct predictions on fixed and simple data points\n",
+        "\n",
+        "You can use  `@parameterized.parameters` to test models with different configurations. [Details with code sample](https://github.com/abseil/abseil-py/blob/master/absl/testing/parameterized.py).\n",
+        "\n",
+        "Note that it's possible to run session APIs and eager execution in the same test case. The code snippets below show how."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CdHqkgPPM2Bj"
+      },
+      "outputs": [],
+      "source": [
+        "import unittest\n",
+        "\n",
+        "class TestNumericalEquivalence(unittest.TestCase):\n",
+        "\n",
+        "  # copied from code samples above\n",
+        "  def setup(self):\n",
+        "    # record statistics for 100 training steps\n",
+        "    step_num = 100\n",
+        "\n",
+        "    # setup TF 1 model\n",
+        "    random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "    with random_tool.scope():\n",
+        "      # run TF1.x code in graph mode with context management\n",
+        "      graph = tf.Graph()\n",
+        "      with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "        self.model_tf1 = SimpleModelWrapper()\n",
+        "        # build the model\n",
+        "        inputs = tf.compat.v1.placeholder(tf.float32, shape=(None, params['input_size']))\n",
+        "        labels = tf.compat.v1.placeholder(tf.float32, shape=(None, params['num_classes']))\n",
+        "        spec = self.model_tf1.model_fn(inputs, labels, tf.estimator.ModeKeys.TRAIN, params)\n",
+        "        train_op = spec.train_op\n",
+        "\n",
+        "        sess.run(tf.compat.v1.global_variables_initializer())\n",
+        "        for step in range(step_num):\n",
+        "          # log everything and update the model for one step\n",
+        "          logs, _ = sess.run(\n",
+        "              [self.model_tf1.logged_ops, train_op],\n",
+        "              feed_dict={inputs: fake_x, labels: fake_y})\n",
+        "          self.model_tf1.update_logs(logs)\n",
+        "\n",
+        "    # setup TF2 model\n",
+        "    random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "    with random_tool.scope():\n",
+        "      self.model_tf2 = SimpleModel(params)\n",
+        "      for step in range(step_num):\n",
+        "        self.model_tf2.train_step([fake_x, fake_y])\n",
+        "  \n",
+        "  def test_learning_rate(self):\n",
+        "    np.testing.assert_allclose(\n",
+        "        self.model_tf1.logs['lr'],\n",
+        "        self.model_tf2.logs['lr'])\n",
+        "\n",
+        "  def test_training_loss(self):\n",
+        "    # adopt different tolerance strategies before and after 10 steps\n",
+        "    first_n_step = 10\n",
+        "\n",
+        "    # absolute difference is limited below 1e-5\n",
+        "    # set `equal_nan` to be False to detect potential NaN loss issues\n",
+        "    abosolute_tolerance = 1e-5\n",
+        "    np.testing.assert_allclose(\n",
+        "        actual=self.model_tf1.logs['loss'][:first_n_step],\n",
+        "        desired=self.model_tf2.logs['loss'][:first_n_step],\n",
+        "        atol=abosolute_tolerance,\n",
+        "        equal_nan=False)\n",
+        "    \n",
+        "    # relative difference is limited below 5%\n",
+        "    relative_tolerance = 0.05\n",
+        "    np.testing.assert_allclose(self.model_tf1.logs['loss'][first_n_step:],\n",
+        "                               self.model_tf2.logs['loss'][first_n_step:],\n",
+        "                               rtol=relative_tolerance,\n",
+        "                               equal_nan=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gshSQdKIddpZ"
+      },
+      "source": [
+        "## Debugging tools"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CkMfCaJRclKv"
+      },
+      "source": [
+        "### tf.print\n",
+        "\n",
+        "tf.print vs print/logging.info\n",
+        "\n",
+        "- With configurable arguments, `tf.print` can recursively display the first and last few elements of each dimension for printed tensors. Check the [API docs](https://www.tensorflow.org/api_docs/python/tf/print) for details.\n",
+        "- For eager execution, both `print` and `tf.print` print the value of the tensor. But `print` may involve device-to-host copy, which can potentially slow down your code. \n",
+        "- For graph mode including usage inside `tf.function`, you need to use `tf.print` to print the actual tensor value. `tf.print` is compiled into an op in the graph, whereas `print` and `logging.info` only log at tracing time, which is often not what you want. \n",
+        "- `tf.print` also supports printing composite tensors like `tf.RaggedTensor` and `tf.sparse.SparseTensor`.\n",
+        "- You can also use a callback to monitor metrics and variables. Please check how to use custom callbacks with [logs dict](https://www.tensorflow.org/guide/keras/custom_callback#usage_of_logs_dict) and [self.model attribute](https://www.tensorflow.org/guide/keras/custom_callback#usage_of_selfmodel_attribute)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "S-5h3cX8Dc50"
+      },
+      "source": [
+        "tf.print vs print inside tf.function"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gRED9FMyDKih"
+      },
+      "outputs": [],
+      "source": [
+        "# `print` prints info of tensor object\n",
+        "# `tf.print` prints the tensor value\n",
+        "@tf.function\n",
+        "def dummy_func(num):\n",
+        "  num += 1\n",
+        "  print(num)\n",
+        "  tf.print(num)\n",
+        "  return num\n",
+        "\n",
+        "_ = dummy_func(tf.constant([1.0]))\n",
+        "\n",
+        "# Output:\n",
+        "# Tensor(\"add:0\", shape=(1,), dtype=float32)\n",
+        "# [2]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3QroLA_zDK2w"
+      },
+      "source": [
+        "tf.distribute.Strategy\n",
+        "\n",
+        "-  If the `tf.function` containing `tf.print` is executed on the workers, for example when using `TPUStrategy` or `ParameterServerStrategy`, you need to check worker/parameter server logs to find the printed values.\n",
+        "- For `print` or `logging.info`, logs will be printed on the coordinator when using `ParameterServerStrategy`, and logs will be printed on the STDOUT on worker0 when using TPUs.\n",
+        "\n",
+        "tf.keras.Model\n",
+        "- When using Sequential and Functional API models, if you want to print values, e.g., model inputs or intermediate features after some layers, you have following options.\n",
+        "  1. [Write a custom layer](https://www.tensorflow.org/guide/keras/custom_layers_and_models) that `tf.print` the inputs. \n",
+        "  2. Include the intermediate outputs you want to inspect in the model outputs.\n",
+        "- `tf.keras.layers.Lambda` layers have (de)serialization limitations. To avoid checkpoint loading issues, write a custom subclassed layer instead. Check the [API docs](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Lambda) for more details. \n",
+        "- You can't `tf.print` intermediate outputs in a `tf.keras.callbacks.LambdaCallback` if you don't have access to the actual values, but instead only to the symbolic Keras tensor objects.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aKazGTr1ZUMG"
+      },
+      "source": [
+        "Option 1: write a custom layer"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8w4aY7wO0B4W"
+      },
+      "outputs": [],
+      "source": [
+        "class PrintLayer(tf.keras.layers.Layer):\n",
+        "  def call(self, inputs):\n",
+        "    tf.print(inputs)\n",
+        "    return inputs\n",
+        "\n",
+        "def get_model():\n",
+        "  inputs = tf.keras.layers.Input(shape=(1,))\n",
+        "  out_1 = tf.keras.layers.Dense(4)(inputs)\n",
+        "  out_2 = tf.keras.layers.Dense(1)(out_1)\n",
+        "  # use custom layer to tf.print intermediate features\n",
+        "  out_3 = PrintLayer()(out_2)\n",
+        "  model = tf.keras.Model(inputs=inputs, outputs=out_3)\n",
+        "  return model\n",
+        "\n",
+        "model = get_model()\n",
+        "model.compile(optimizer=\"adam\", loss=\"mse\")\n",
+        "model.fit([1, 2, 3], [0.0, 0.0, 1.0])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KNESOatq7iM9"
+      },
+      "source": [
+        "Option 2: include the intermediate outputs you want to inspect in the model outputs.\n",
+        "\n",
+        "Note that in such case, you may need some [customizations](https://www.tensorflow.org/guide/keras/customizing_what_happens_in_fit) to use `Model.fit`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MiifvdLk7g9J"
+      },
+      "outputs": [],
+      "source": [
+        "def get_model():\n",
+        "  inputs = tf.keras.layers.Input(shape=(1,))\n",
+        "  out_1 = tf.keras.layers.Dense(4)(inputs)\n",
+        "  out_2 = tf.keras.layers.Dense(1)(out_1)\n",
+        "  # include intermediate values in model outputs\n",
+        "  model = tf.keras.Model(\n",
+        "      inputs=inputs,\n",
+        "      outputs={\n",
+        "          'inputs': inputs,\n",
+        "          'out_1': out_1,\n",
+        "          'out_2': out_2})\n",
+        "  return model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MvIKDZpHSLmQ"
+      },
+      "source": [
+        "### pdb\n",
+        "You can use [pdb](https://docs.python.org/3/library/pdb.html) both in terminal and Colab to inspect intermediate values for debugging.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Qu0n4O2umyT7"
+      },
+      "source": [
+        "### Visualize graph with TensorBoard\n",
+        "\n",
+        "You can [examine the TensorFlow graph with TensorBoard](https://www.tensorflow.org/tensorboard/graphs). TensorBoard is also [supported on colab](https://www.tensorflow.org/tensorboard/tensorboard_in_notebooks). TensorBoard is a great tool to visualize summaries. You can use it to compare learning rate, model weights, gradient scale, training/validation metrics, or even model intermediate outputs between TF1.x model and migrated TF2 model through the training process and seeing if the values look as expected."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vBnxB6_xzlnT"
+      },
+      "source": [
+        "### TensorFlow Profiler\n",
+        "\n",
+        "[TensorFlow Profiler](https://www.tensorflow.org/guide/profiler) can help you visualize the execution timeline on GPUs/TPUs. You can check out this [Colab Demo](https://www.tensorflow.org/tensorboard/tensorboard_profiling_keras) for its basic usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9wNmCSHBpiGM"
+      },
+      "source": [
+        "### MultiProcessRunner\n",
+        "[MultiProcessRunner](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/distribute/multi_process_runner.py#L108) is a useful tool when debugging with MultiWorkerMirroredStrategy and ParameterServerStrategy. You can take a look at [this concrete example](https://github.com/keras-team/keras/blob/master/keras/integration_test/mwms_multi_process_runner_test.py) for its usage.\n",
+        "\n",
+        "Specifically for the cases of these two strategies, you are recommended to 1) not only have unit tests to cover their flow, 2) but also to attempt to reproduce failures using it in unit test to avoid launch real distributed job every time when they attempt a fix."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "migration_debugging.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/mirrored_strategy.ipynb b/site/en/guide/migrate/mirrored_strategy.ipynb
new file mode 100644
index 00000000000..dfadc5c0b12
--- /dev/null
+++ b/site/en/guide/migrate/mirrored_strategy.ipynb
@@ -0,0 +1,263 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate single-worker multiple-GPU training\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "This guide demonstrates how to migrate the single-worker multiple-GPU workflows from TensorFlow 1 to TensorFlow 2.\n",
+        "\n",
+        "To perform synchronous training across multiple GPUs on one machine:\n",
+        "\n",
+        "- In TensorFlow 1, you use the `tf.estimator.Estimator` APIs with `tf.distribute.MirroredStrategy`.\n",
+        "- In TensorFlow 2, you can use [Keras Model.fit](https://www.tensorflow.org/tutorials/distribute/keras) or [a custom training loop](https://www.tensorflow.org/tutorials/distribute/custom_training) with `tf.distribute.MirroredStrategy`. Learn more in the [Distributed training with TensorFlow](https://www.tensorflow.org/guide/distributed_training#mirroredstrategy) guide."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6d466b39d0db"
+      },
+      "source": [
+        "Start with imports and a simple dataset for demonstration purposes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m7rnGxsXtDkV"
+      },
+      "outputs": [],
+      "source": [
+        "features = [[1., 1.5], [2., 2.5], [3., 3.5]]\n",
+        "labels = [[0.3], [0.5], [0.7]]\n",
+        "eval_features = [[4., 4.5], [5., 5.5], [6., 6.5]]\n",
+        "eval_labels = [[0.8], [0.9], [1.]]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4uXff1BEssdE"
+      },
+      "source": [
+        "## TensorFlow 1: Single-worker distributed training with tf.estimator.Estimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A9560BqEOTpb"
+      },
+      "source": [
+        "This example demonstrates the TensorFlow 1 canonical workflow of single-worker multiple-GPU training. You need to set the distribution strategy (`tf.distribute.MirroredStrategy`) through the `config` parameter of the `tf.estimator.Estimator`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "def _input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "\n",
+        "def _eval_input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).batch(1)\n",
+        "\n",
+        "def _model_fn(features, labels, mode):\n",
+        "  logits = tf1.layers.Dense(1)(features)\n",
+        "  loss = tf1.losses.mean_squared_error(labels=labels, predictions=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.05)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "  return tf1.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)\n",
+        "\n",
+        "strategy = tf1.distribute.MirroredStrategy()\n",
+        "config = tf1.estimator.RunConfig(\n",
+        "    train_distribute=strategy, eval_distribute=strategy)\n",
+        "estimator = tf1.estimator.Estimator(model_fn=_model_fn, config=config)\n",
+        "\n",
+        "train_spec = tf1.estimator.TrainSpec(input_fn=_input_fn)\n",
+        "eval_spec = tf1.estimator.EvalSpec(input_fn=_eval_input_fn)\n",
+        "tf1.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "## TensorFlow 2: Single-worker training with Keras"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fkgkGf_AOaRR"
+      },
+      "source": [
+        "When migrating to TensorFlow 2, you can use the Keras APIs with `tf.distribute.MirroredStrategy`.\n",
+        "\n",
+        "If you use the `tf.keras` APIs for model building and Keras `Model.fit` for training, the main difference is instantiating the Keras model, an optimizer, and metrics in the context of `Strategy.scope`, instead of defining a `config` for `tf.estimator.Estimator`.\n",
+        "\n",
+        "If you need to use a custom training loop, check out the [Using tf.distribute.Strategy with custom training loops](https://www.tensorflow.org/guide/distributed_training#using_tfdistributestrategy_with_custom_training_loops) guide."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "atVciNgPs0fw"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "eval_dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).batch(1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kip65sYBlKiu"
+      },
+      "outputs": [],
+      "source": [
+        "strategy = tf.distribute.MirroredStrategy()\n",
+        "with strategy.scope():\n",
+        "  model = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])\n",
+        "  optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+        "\n",
+        "model.compile(optimizer=optimizer, loss='mse')\n",
+        "model.fit(dataset)\n",
+        "model.evaluate(eval_dataset, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0431f3935485"
+      },
+      "source": [
+        "## Next steps"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a68d2a99f79b"
+      },
+      "source": [
+        "To learn more about distributed training with `tf.distribute.MirroredStrategy` in TensorFlow 2, check out the following documentation:\n",
+        "\n",
+        "- The [Distributed training on one machine with Keras](../../tutorials/distribute/keras) tutorial\n",
+        "- The [Distributed training on one machine with a custom training loop](../../tutorials/distribute/custom_training) tutorial\n",
+        "- The [Distributed training with TensorFlow](../../guide/distributed_training) guide\n",
+        "- The [Using multiple GPUs](../../guide/gpu#using_multiple_gpus) guide\n",
+        "- The [Optimize the performance on the multi-GPU single host (with the TensorFlow Profiler)](../../guide/gpu_performance_analysis#2_optimize_the_performance_on_the_multi-gpu_single_host) guide"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "mirrored_strategy.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/model_mapping.ipynb b/site/en/guide/migrate/model_mapping.ipynb
new file mode 100644
index 00000000000..2d4582839c0
--- /dev/null
+++ b/site/en/guide/migrate/model_mapping.ipynb
@@ -0,0 +1,1530 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6bYaCABobL5q"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "FlUw7tSKbtg4"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_-fogOi3K7nR"
+      },
+      "source": [
+        "# Use TF1.x models in TF2 workflows\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7-GwECUqrkqT"
+      },
+      "source": [
+        "This guide provides an overview and examples of a [modeling code shim](https://en.wikipedia.org/wiki/Shim_(computing)) that you can employ to use your existing TF1.x models in TF2 workflows such as eager execution, `tf.function`, and distribution strategies with minimal changes to your modeling code."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k_ezCbogxaqt"
+      },
+      "source": [
+        "## Scope of usage\n",
+        "\n",
+        "The shim described in this guide is designed for TF1.x models that rely on:\n",
+        "1. `tf.compat.v1.get_variable` and `tf.compat.v1.variable_scope` to control variable creation and reuse, and\n",
+        "1. Graph-collection based APIs such as `tf.compat.v1.global_variables()`, `tf.compat.v1.trainable_variables`, `tf.compat.v1.losses.get_regularization_losses()`, and `tf.compat.v1.get_collection()` to keep track of weights and regularization losses\n",
+        "\n",
+        "This includes most models built on top of `tf.compat.v1.layer`, `tf.contrib.layers` APIs, and [TensorFlow-Slim](https://github.com/google-research/tf-slim).\n",
+        "\n",
+        "The shim is **NOT** necessary for the following TF1.x models:\n",
+        "\n",
+        "1. Stand-alone Keras models that already track all of their trainable weights and regularization losses via `model.trainable_weights` and `model.losses` respectively.\n",
+        "1. `tf.Module`s that already track all of their trainable weights via `module.trainable_variables`, and only create weights if they have not already been created.\n",
+        "\n",
+        "These models are likely to work in TF2 with eager execution and `tf.function`s out-of-the-box."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3OQNFp8zgV0C"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Import TensorFlow and other dependencies."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EG2n3-qlD5mA"
+      },
+      "outputs": [],
+      "source": [
+        "!pip uninstall -y -q tensorflow"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mVfR3MBvD9Sc"
+      },
+      "outputs": [],
+      "source": [
+        "# Install tf-nightly as the DeterministicRandomTestTool is available only in\n",
+        "# Tensorflow 2.8\n",
+        "\n",
+        "!pip install -q tf-nightly"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PzkV-2cna823"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as v1\n",
+        "import sys\n",
+        "import numpy as np\n",
+        "\n",
+        "from contextlib import contextmanager"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ox4kn0DK8H0f"
+      },
+      "source": [
+        "## The `track_tf1_style_variables` decorator\n",
+        "\n",
+        "The key shim described in this guide is `tf.compat.v1.keras.utils.track_tf1_style_variables`, a decorator that you can use within methods belonging to `tf.keras.layers.Layer` and `tf.Module` to track TF1.x-style weights and capture regularization losses.\n",
+        "\n",
+        "Decorating a `tf.keras.layers.Layer`'s  or `tf.Module`'s call methods with `tf.compat.v1.keras.utils.track_tf1_style_variables` allows variable creation and reuse via `tf.compat.v1.get_variable` (and by extension `tf.compat.v1.layers`) to work correctly inside of the decorated method rather than always creating a new variable on each call. It will also cause the layer or module to implicitly track any weights created or accessed via `get_variable` inside the decorated method.\n",
+        "\n",
+        "In addition to tracking the weights themselves under the standard\n",
+        "`layer.variable`/`module.variable`/etc. properties, if the method belongs\n",
+        "to a `tf.keras.layers.Layer`, then any regularization losses specified via the\n",
+        "`get_variable` or `tf.compat.v1.layers` regularizer arguments will get\n",
+        "tracked by the layer under the standard `layer.losses` property.\n",
+        "\n",
+        "This tracking mechanism enables using large classes of TF1.x-style model-forward-pass code inside of Keras layers or `tf.Module`s in TF2 even with TF2 behaviors enabled.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Sq6IqZILmGmO"
+      },
+      "source": [
+        "## Usage examples\n",
+        "\n",
+        "The usage examples below demonstrate the modeling shims used to decorate `tf.keras.layers.Layer` methods, but except where they are specifically interacting with Keras features they are applicable when decorating `tf.Module` methods as well."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YWGPh6KmkHq6"
+      },
+      "source": [
+        "### Layer built with tf.compat.v1.get_variable\n",
+        "\n",
+        "Imagine you have a layer implemented directly on top of `tf.compat.v1.get_variable` as follows:\n",
+        "\n",
+        "```python\n",
+        "def dense(self, inputs, units):\n",
+        "  out = inputs\n",
+        "  with tf.compat.v1.variable_scope(\"dense\"):\n",
+        "    # The weights are created with a `regularizer`,\n",
+        "    kernel = tf.compat.v1.get_variable(\n",
+        "        shape=[out.shape[-1], units],\n",
+        "        regularizer=tf.keras.regularizers.L2(),\n",
+        "        initializer=tf.compat.v1.initializers.glorot_normal,\n",
+        "        name=\"kernel\")\n",
+        "    bias = tf.compat.v1.get_variable(\n",
+        "        shape=[units,],\n",
+        "        initializer=tf.compat.v1.initializers.zeros,\n",
+        "        name=\"bias\")\n",
+        "    out = tf.linalg.matmul(out, kernel)\n",
+        "    out = tf.compat.v1.nn.bias_add(out, bias)\n",
+        "  return out\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6sZWU7JSok2n"
+      },
+      "source": [
+        "Use the shim to turn it into a layer and call it on inputs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q3eKkcKtS_N4"
+      },
+      "outputs": [],
+      "source": [
+        "class DenseLayer(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, units, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.units = units\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs):\n",
+        "    out = inputs\n",
+        "    with tf.compat.v1.variable_scope(\"dense\"):\n",
+        "      # The weights are created with a `regularizer`,\n",
+        "      # so the layer should track their regularization losses\n",
+        "      kernel = tf.compat.v1.get_variable(\n",
+        "          shape=[out.shape[-1], self.units],\n",
+        "          regularizer=tf.keras.regularizers.L2(),\n",
+        "          initializer=tf.compat.v1.initializers.glorot_normal,\n",
+        "          name=\"kernel\")\n",
+        "      bias = tf.compat.v1.get_variable(\n",
+        "          shape=[self.units,],\n",
+        "          initializer=tf.compat.v1.initializers.zeros,\n",
+        "          name=\"bias\")\n",
+        "      out = tf.linalg.matmul(out, kernel)\n",
+        "      out = tf.compat.v1.nn.bias_add(out, bias)\n",
+        "    return out\n",
+        "\n",
+        "layer = DenseLayer(10)\n",
+        "x = tf.random.normal(shape=(8, 20))\n",
+        "layer(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JqXAlWnYgwcq"
+      },
+      "source": [
+        "Access the tracked variables and the captured regularization losses like a standard Keras layer."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZNz5HmkXg0B5"
+      },
+      "outputs": [],
+      "source": [
+        "layer.trainable_variables\n",
+        "layer.losses"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "W0z9GmRlhM9X"
+      },
+      "source": [
+        "To see that the weights get reused each time you call the layer, set all the weights to zero and call the layer again."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZJ4vOu2Rf-I2"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Resetting variables to zero:\", [var.name for var in layer.trainable_variables])\n",
+        "\n",
+        "for var in layer.trainable_variables:\n",
+        "  var.assign(var * 0.0)\n",
+        "\n",
+        "# Note: layer.losses is not a live view and\n",
+        "# will get reset only at each layer call\n",
+        "print(\"layer.losses:\", layer.losses)\n",
+        "print(\"calling layer again.\")\n",
+        "out = layer(x)\n",
+        "print(\"layer.losses: \", layer.losses)\n",
+        "out"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WwEprtA-lOh6"
+      },
+      "source": [
+        "You can use the converted layer directly in Keras functional model construction as well."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7E7ZCINHlaHU"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = tf.keras.Input(shape=(20))\n",
+        "outputs = DenseLayer(10)(inputs)\n",
+        "model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
+        "\n",
+        "x = tf.random.normal(shape=(8, 20))\n",
+        "model(x)\n",
+        "\n",
+        "# Access the model variables and regularization losses\n",
+        "model.weights\n",
+        "model.losses"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ew5TTEyZkZGU"
+      },
+      "source": [
+        "### Model built with `tf.compat.v1.layers`\n",
+        "\n",
+        "Imagine you have a layer or model implemented directly on top of `tf.compat.v1.layers` as follows:\n",
+        "\n",
+        "```python\n",
+        "def model(self, inputs, units):\n",
+        "  with tf.compat.v1.variable_scope('model'):\n",
+        "    out = tf.compat.v1.layers.conv2d(\n",
+        "        inputs, 3, 3,\n",
+        "        kernel_regularizer=\"l2\")\n",
+        "    out = tf.compat.v1.layers.flatten(out)\n",
+        "    out = tf.compat.v1.layers.dense(\n",
+        "        out, units,\n",
+        "        kernel_regularizer=\"l2\")\n",
+        "    return out\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gZolXllfpVx6"
+      },
+      "source": [
+        "Use the shim to turn it into a layer and call it on inputs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cBpfSHWTTTCv"
+      },
+      "outputs": [],
+      "source": [
+        "class CompatV1LayerModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, units, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.units = units\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs):\n",
+        "    with tf.compat.v1.variable_scope('model'):\n",
+        "      out = tf.compat.v1.layers.conv2d(\n",
+        "          inputs, 3, 3,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      out = tf.compat.v1.layers.flatten(out)\n",
+        "      out = tf.compat.v1.layers.dense(\n",
+        "          out, self.units,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      return out\n",
+        "\n",
+        "layer = CompatV1LayerModel(10)\n",
+        "x = tf.random.normal(shape=(8, 5, 5, 5))\n",
+        "layer(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OkG9oLlblfK_"
+      },
+      "source": [
+        "Warning: For safety reasons, make sure to put all `tf.compat.v1.layers` inside of a non-empty-string `variable_scope`. This is because `tf.compat.v1.layers` with auto-generated names will always auto-increment the name outside of any variable scope. This means the requested variable names will mismatch each time you call the layer/module. So, rather than reusing the already-made weights it will create a new set of variables every call."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zAVN6dy3p7ik"
+      },
+      "source": [
+        "Access the tracked variables and captured regularization losses like a standard Keras layer."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HTRF99vJp7ik"
+      },
+      "outputs": [],
+      "source": [
+        "layer.trainable_variables\n",
+        "layer.losses"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kkNuEcyIp7ik"
+      },
+      "source": [
+        "To see that the weights get reused each time you call the layer, set all the weights to zero and call the layer again."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4dk4XScdp7il"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Resetting variables to zero:\", [var.name for var in layer.trainable_variables])\n",
+        "\n",
+        "for var in layer.trainable_variables:\n",
+        "  var.assign(var * 0.0)\n",
+        "\n",
+        "out = layer(x)\n",
+        "print(\"layer.losses: \", layer.losses)\n",
+        "out"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7zD3a8PKzU7S"
+      },
+      "source": [
+        "You can use the converted layer directly in Keras functional model construction as well."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q88BgBCup7il"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = tf.keras.Input(shape=(5, 5, 5))\n",
+        "outputs = CompatV1LayerModel(10)(inputs)\n",
+        "model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
+        "\n",
+        "x = tf.random.normal(shape=(8, 5, 5, 5))\n",
+        "model(x)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2cioB6Zap7il"
+      },
+      "outputs": [],
+      "source": [
+        "# Access the model variables and regularization losses\n",
+        "model.weights\n",
+        "model.losses"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NBNODOx9ly6r"
+      },
+      "source": [
+        "### Capture batch normalization updates and model `training` args\n",
+        "\n",
+        "In TF1.x, you perform batch normalization like this:\n",
+        "\n",
+        "```python\n",
+        "  x_norm = tf.compat.v1.layers.batch_normalization(x, training=training)\n",
+        "\n",
+        "  # ...\n",
+        "\n",
+        "  update_ops = tf.compat.v1.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
+        "  train_op = optimizer.minimize(loss)\n",
+        "  train_op = tf.group([train_op, update_ops])\n",
+        "```\n",
+        "Note that:\n",
+        "1. The batch normalization moving average updates are tracked by `get_collection` which was called separately from the layer\n",
+        "2. `tf.compat.v1.layers.batch_normalization` requires a `training` argument (generally called `is_training` when using TF-Slim batch normalization layers)\n",
+        "\n",
+        "In TF2, due to [eager execution](https://www.tensorflow.org/guide/eager) and automatic control dependencies, the batch normalization moving average updates will be executed right away. There is no need to separately collect them from the updates collection and add them as explicit control dependencies.\n",
+        "\n",
+        "Additionally, if you give your `tf.keras.layers.Layer`'s forward pass method a `training` argument, Keras will be able to pass the current training phase and any nested layers to it just like it does for any other layer. See the API docs for `tf.keras.Model` for more information on how Keras handles the `training` argument.\n",
+        "\n",
+        "If you are decorating `tf.Module` methods, you need to make sure to manually pass all `training` arguments as needed. However, the batch normalization moving average updates will still be applied automatically with no need for explicit control dependencies.\n",
+        "\n",
+        "The following code snippets demonstrate how to embed batch normalization layers in the shim and how using it in a Keras model works (applicable to `tf.keras.layers.Layer`)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CjZE-J7mkS9p"
+      },
+      "outputs": [],
+      "source": [
+        "class CompatV1BatchNorm(tf.keras.layers.Layer):\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs, training=None):\n",
+        "    print(\"Forward pass called with `training` =\", training)\n",
+        "    with v1.variable_scope('batch_norm_layer'):\n",
+        "      return v1.layers.batch_normalization(x, training=training)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NGuvvElmY-fu"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Constructing model\")\n",
+        "inputs = tf.keras.Input(shape=(5, 5, 5))\n",
+        "outputs = CompatV1BatchNorm()(inputs)\n",
+        "model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
+        "\n",
+        "print(\"Calling model in inference mode\")\n",
+        "x = tf.random.normal(shape=(8, 5, 5, 5))\n",
+        "model(x, training=False)\n",
+        "\n",
+        "print(\"Moving average variables before training: \",\n",
+        "      {var.name: var.read_value() for var in model.non_trainable_variables})\n",
+        "\n",
+        "# Notice that when running TF2 and eager execution, the batchnorm layer directly\n",
+        "# updates the moving averages while training without needing any extra control\n",
+        "# dependencies\n",
+        "print(\"calling model in training mode\")\n",
+        "model(x, training=True)\n",
+        "\n",
+        "print(\"Moving average variables after training: \",\n",
+        "      {var.name: var.read_value() for var in model.non_trainable_variables})\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Gai4ikpmeRqR"
+      },
+      "source": [
+        "### Variable-scope based variable reuse\n",
+        "Any variable creations in the forward pass based on `get_variable` will maintain the same variable naming and reuse semantics that variable scopes have in TF1.x. This is true as long as you have at least one non-empty outer scope for any `tf.compat.v1.layers` with auto-generated names, as mentioned above.\n",
+        "\n",
+        "Note: Naming and reuse will be scoped to within a single layer/module instance. Calls to `get_variable` inside one shim-decorated layer or module will not be able to refer to variables created inside of layers or modules. You can get around this by using Python references to other variables directly if need be, rather than accessing variables via `get_variable`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6PzYZdX2nMVt"
+      },
+      "source": [
+        "### Eager execution & `tf.function`\n",
+        "\n",
+        "As seen above, decorated methods for `tf.keras.layers.Layer` and `tf.Module` run inside of eager execution and are also compatible with `tf.function`. This means you can use [pdb](https://docs.python.org/3/library/pdb.html) and other interactive tools to step through your forward pass as it is running.\n",
+        "\n",
+        "Warning: Although it is perfectly safe to call your shim-decorated layer/module methods from *inside* of a `tf.function`, it is not safe to put `tf.function`s inside of your shim-decorated methods if those `tf.functions` contain `get_variable` calls. Entering a `tf.function` resets `variable_scope`s, which means the TF1.x-style variable-scope-based variable reuse that the shim mimics will break down in this setting."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aPytVgZWnShe"
+      },
+      "source": [
+        "### Distribution strategies\n",
+        "\n",
+        "Calls to `get_variable` inside of `@track_tf1_style_variables`-decorated layer or module methods use standard `tf.Variable` variable creations under the hood. This means you can use them with the various distribution strategies available with `tf.distribute` such as `MirroredStrategy` and `TPUStrategy`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_DcK24FOA8A2"
+      },
+      "source": [
+        "## Nesting `tf.Variable`s, `tf.Module`s, `tf.keras.layers` & `tf.keras.models` in decorated calls\n",
+        "\n",
+        "Decorating your layer call in `tf.compat.v1.keras.utils.track_tf1_style_variables` will only add automatic implicit tracking of variables created (and reused) via `tf.compat.v1.get_variable`. It will not capture weights directly created by `tf.Variable` calls, such as those used by typical Keras layers and most `tf.Module`s. This section describes how to handle these nested cases.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Azxza3bVOZlv"
+      },
+      "source": [
+        "###  (Pre-existing usages) `tf.keras.layers` and `tf.keras.models`\n",
+        "\n",
+        "For pre-existing usages of nested Keras layers and models, use `tf.compat.v1.keras.utils.get_or_create_layer`. This is only recommended for easing migration of existing TF1.x nested Keras usages; new code should use explicit attribute setting as described below for tf.Variables and tf.Modules.\n",
+        "\n",
+        "To use `tf.compat.v1.keras.utils.get_or_create_layer`, wrap the code that constructs your nested model into a method, and pass it in to the method. Example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LN15TcRgHKsq"
+      },
+      "outputs": [],
+      "source": [
+        "class NestedModel(tf.keras.Model):\n",
+        "\n",
+        "  def __init__(self, units, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.units = units\n",
+        "\n",
+        "  def build_model(self):\n",
+        "    inp = tf.keras.Input(shape=(5, 5))\n",
+        "    dense_layer = tf.keras.layers.Dense(\n",
+        "        10, name=\"dense\", kernel_regularizer=\"l2\",\n",
+        "        kernel_initializer=tf.compat.v1.ones_initializer())\n",
+        "    model = tf.keras.Model(inputs=inp, outputs=dense_layer(inp))\n",
+        "    return model\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs):\n",
+        "    # Get or create a nested model without assigning it as an explicit property\n",
+        "    model = tf.compat.v1.keras.utils.get_or_create_layer(\n",
+        "        \"dense_model\", self.build_model)\n",
+        "    return model(inputs)\n",
+        "\n",
+        "layer = NestedModel(10)\n",
+        "layer(tf.ones(shape=(5,5)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DgsKlltPHI8z"
+      },
+      "source": [
+        "This method ensures that these nested layers are correctly reused and tracked by tensorflow. Note that the `@track_tf1_style_variables` decorator is still required on the appropriate method. The model builder method passed into `get_or_create_layer` (in this case, `self.build_model`), should take no arguments.\n",
+        "\n",
+        "Weights are tracked:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3zO5A78MJsqO"
+      },
+      "outputs": [],
+      "source": [
+        "assert len(layer.weights) == 2\n",
+        "weights = {x.name: x for x in layer.variables}\n",
+        "\n",
+        "assert set(weights.keys()) == {\"dense/bias:0\", \"dense/kernel:0\"}\n",
+        "\n",
+        "layer.weights"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o3Xsi-JbKTuj"
+      },
+      "source": [
+        "And regularization loss as well:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mdK5RGm5KW5C"
+      },
+      "outputs": [],
+      "source": [
+        "tf.add_n(layer.losses)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J_VRycQYJrXu"
+      },
+      "source": [
+        "###  Incremental migration: `tf.Variables` and `tf.Modules`\n",
+        "\n",
+        "If you need to embed `tf.Variable` calls or `tf.Module`s in your decorated methods (for example, if you are following the incremental migration to non-legacy TF2 APIs described later in this guide), you still need to explicitly track these, with the following requirements:\n",
+        "* Explicitly make sure that the variable/module/layer is only created once\n",
+        "* Explicitly attach them as instance attributes just as you would when defining a [typical module or layer](https://www.tensorflow.org/guide/intro_to_modules#defining_models_and_layers_in_tensorflow)\n",
+        "* Explicitly reuse the already-created object in follow-on calls\n",
+        "\n",
+        "This ensures that weights are not created new each call and are correctly reused. Additionally, this also ensures that existing weights and regularization losses get tracked.\n",
+        "\n",
+        "Here is an example of how this could look:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mrRPPoJ5ap5U"
+      },
+      "outputs": [],
+      "source": [
+        "class NestedLayer(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, units, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.units = units\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def __call__(self, inputs):\n",
+        "    out = inputs\n",
+        "    with tf.compat.v1.variable_scope(\"inner_dense\"):\n",
+        "      # The weights are created with a `regularizer`,\n",
+        "      # so the layer should track their regularization losses\n",
+        "      kernel = tf.compat.v1.get_variable(\n",
+        "          shape=[out.shape[-1], self.units],\n",
+        "          regularizer=tf.keras.regularizers.L2(),\n",
+        "          initializer=tf.compat.v1.initializers.glorot_normal,\n",
+        "          name=\"kernel\")\n",
+        "      bias = tf.compat.v1.get_variable(\n",
+        "          shape=[self.units,],\n",
+        "          initializer=tf.compat.v1.initializers.zeros,\n",
+        "          name=\"bias\")\n",
+        "      out = tf.linalg.matmul(out, kernel)\n",
+        "      out = tf.compat.v1.nn.bias_add(out, bias)\n",
+        "    return out\n",
+        "\n",
+        "class WrappedDenseLayer(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, units, **kwargs):\n",
+        "    super().__init__(**kwargs)\n",
+        "    self.units = units\n",
+        "    # Only create the nested tf.variable/module/layer/model\n",
+        "    # once, and then reuse it each time!\n",
+        "    self._dense_layer = NestedLayer(self.units)\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs):\n",
+        "    with tf.compat.v1.variable_scope('outer'):\n",
+        "      outputs = tf.compat.v1.layers.dense(inputs, 3)\n",
+        "      outputs = tf.compat.v1.layers.dense(inputs, 4)\n",
+        "      return self._dense_layer(outputs)\n",
+        "\n",
+        "layer = WrappedDenseLayer(10)\n",
+        "\n",
+        "layer(tf.ones(shape=(5, 5)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Lo9h6wc6bmEF"
+      },
+      "source": [
+        "Note that explicit tracking of the nested module is needed even though it is decorated with the `track_tf1_style_variables` decorator. This is because each module/layer with decorated methods has its own variable store associated with it. \n",
+        "\n",
+        "The weights are correctly tracked:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Qt6USaTVbauM"
+      },
+      "outputs": [],
+      "source": [
+        "assert len(layer.weights) == 6\n",
+        "weights = {x.name: x for x in layer.variables}\n",
+        "\n",
+        "assert set(weights.keys()) == {\"outer/inner_dense/bias:0\",\n",
+        "                               \"outer/inner_dense/kernel:0\",\n",
+        "                               \"outer/dense/bias:0\",\n",
+        "                               \"outer/dense/kernel:0\",\n",
+        "                               \"outer/dense_1/bias:0\",\n",
+        "                               \"outer/dense_1/kernel:0\"}\n",
+        "\n",
+        "layer.trainable_weights"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dHn-bJoNJw7l"
+      },
+      "source": [
+        "As well as regularization loss:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pq5GFtXjJyut"
+      },
+      "outputs": [],
+      "source": [
+        "layer.losses"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "p7VKJj3JOCEk"
+      },
+      "source": [
+        "Note that if the `NestedLayer` were a non-Keras `tf.Module` instead, variables would still be tracked but regularization losses would not be automatically tracked, so you would have to explicitly track them separately."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FsTgnydkdezQ"
+      },
+      "source": [
+        "### Guidance on variable names\n",
+        "\n",
+        "Explicit `tf.Variable` calls and Keras layers use a different layer name / variable name autogeneration mechanism than you may be used to from the combination of `get_variable` and `variable_scopes`. Although the shim will make your variable names match for variables created by `get_variable` even when going from TF1.x graphs to TF2 eager execution & `tf.function`, it cannot guarantee the same for the variable names generated for `tf.Variable` calls and Keras layers that you embed within your method decorators. It is even possible for multiple variables to share the same name in TF2 eager execution and `tf.function`.\n",
+        "\n",
+        "You should take special care with this when following the sections on validating correctness and mapping TF1.x checkpoints later on in this guide."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CaP7fxoUWfMm"
+      },
+      "source": [
+        "### Using `tf.compat.v1.make_template` in the decorated method\n",
+        "\n",
+        "**It is highly recommended you directly use `tf.compat.v1.keras.utils.track_tf1_style_variables` instead of using `tf.compat.v1.make_template`, as it is a thinner layer on top of TF2**. \n",
+        "\n",
+        "Follow the guidance in this section for prior TF1.x code that was already relying on `tf.compat.v1.make_template`.\n",
+        "\n",
+        "Because `tf.compat.v1.make_template` wraps code that uses `get_variable`, the `track_tf1_style_variables`  decorator allows you to use these templates in layer calls and successfully track the weights and regularization losses.\n",
+        "\n",
+        "However, do make sure to call `make_template` only once and then reuse the same template in each layer call. Otherwise, a new template will be created each time you call the layer along with a new set of variables.\n",
+        "\n",
+        "For example,"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iHEQN8z44dbK"
+      },
+      "outputs": [],
+      "source": [
+        "class CompatV1TemplateScaleByY(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, **kwargs):\n",
+        "    super().__init__(**kwargs)\n",
+        "    def my_op(x, scalar_name):\n",
+        "      var1 = tf.compat.v1.get_variable(scalar_name,\n",
+        "                            shape=[],\n",
+        "                            regularizer=tf.compat.v1.keras.regularizers.L2(),\n",
+        "                            initializer=tf.compat.v1.constant_initializer(1.5))\n",
+        "      return x * var1\n",
+        "    self.scale_by_y = tf.compat.v1.make_template('scale_by_y', my_op, scalar_name='y')\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs):\n",
+        "    with tf.compat.v1.variable_scope('layer'):\n",
+        "      # Using a scope ensures the `scale_by_y` name will not be incremented\n",
+        "      # for each instantiation of the layer.\n",
+        "      return self.scale_by_y(inputs)\n",
+        "\n",
+        "layer = CompatV1TemplateScaleByY()\n",
+        "\n",
+        "out = layer(tf.ones(shape=(2, 3)))\n",
+        "print(\"weights:\", layer.weights)\n",
+        "print(\"regularization loss:\", layer.losses)\n",
+        "print(\"output:\", out)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3vKTJ7IsTEe8"
+      },
+      "source": [
+        "Warning: Avoid sharing the same `make_template`-created template across multiple layer instances as it may break the variable and regularization loss tracking mechanisms of the shim decorator. Additionally, if you plan to use the same `make_template` name inside of multiple layer instances then you should nest the created template's usage inside of a `variable_scope`. If not, the generated name for the template's `variable_scope` will increment with each new instance of the layer. This could alter the weight names in unexpected ways."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "P4E3-XPhWD2N"
+      },
+      "source": [
+        "## Incremental migration to Native TF2\n",
+        "\n",
+        "As mentioned earlier, `track_tf1_style_variables` allows you to mix TF2-style object-oriented `tf.Variable`/`tf.keras.layers.Layer`/`tf.Module` usage with legacy `tf.compat.v1.get_variable`/`tf.compat.v1.layers`-style usage inside of the same decorated module/layer.\n",
+        "\n",
+        "This means that after you have made your TF1.x model fully-TF2-compatible, you can write all new model components with native (non-`tf.compat.v1`) TF2 APIs and have them interoperate with your older code.\n",
+        "\n",
+        "However, if you continue to modify your older model components, you may also choose to incrementally switch your legacy-style `tf.compat.v1` usage over to the purely-native object-oriented APIs that are recommended for newly written TF2 code.\n",
+        "\n",
+        "`tf.compat.v1.get_variable` usage can be replaced with either `self.add_weight` calls if you are decorating a Keras layer/model, or with `tf.Variable` calls if you are decorating Keras objects or `tf.Module`s.\n",
+        "\n",
+        "Both functional-style and object-oriented `tf.compat.v1.layers` can generally be replaced with the equivalent `tf.keras.layers` layer with no argument changes required.\n",
+        "\n",
+        "You may also consider chunks parts of your model or common patterns into individual layers/modules during your incremental move to purely-native APIs, which may themselves use `track_tf1_style_variables`.\n",
+        "\n",
+        "### A note on Slim and contrib.layers\n",
+        "\n",
+        "A large amount of older TF 1.x code uses the [Slim](https://ai.googleblog.com/2016/08/tf-slim-high-level-library-to-define.html) library, which was packaged with TF 1.x as `tf.contrib.layers`. Converting code using Slim to native TF 2 is more involved than converting `v1.layers`. In fact, it may make sense to convert your Slim code to `v1.layers` first, then convert to Keras. Below is some general guidance for converting Slim code.\n",
+        "\n",
+        "- Ensure all arguments are explicit. Remove `arg_scopes` if possible. If you still need to use them, split `normalizer_fn` and `activation_fn` into their own layers.\n",
+        "- Separable conv layers map to one or more different Keras layers (depthwise, pointwise, and separable Keras layers).\n",
+        "- Slim and `v1.layers` have different argument names and default values.\n",
+        "- Note that some arguments have different scales."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RFoULo-gazit"
+      },
+      "source": [
+        "### Migration to Native TF2 ignoring checkpoint compatibility\n",
+        "\n",
+        "The following code sample demonstrates an incremental move of a model to purely-native APIs without considering checkpoint compatibility."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dPO9YJsb6r-D"
+      },
+      "outputs": [],
+      "source": [
+        "class CompatModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, units, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.units = units\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs, training=None):\n",
+        "    with tf.compat.v1.variable_scope('model'):\n",
+        "      out = tf.compat.v1.layers.conv2d(\n",
+        "          inputs, 3, 3,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      out = tf.compat.v1.layers.flatten(out)\n",
+        "      out = tf.compat.v1.layers.dropout(out, training=training)\n",
+        "      out = tf.compat.v1.layers.dense(\n",
+        "          out, self.units,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      return out\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fp16xK6Oa8k9"
+      },
+      "source": [
+        "Next, replace the `compat.v1` APIs with their native object-oriented equivalents in a piecewise manner. Start by switching the convolution layer to a Keras object created in the layer constructor."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LOj1Swe16so3"
+      },
+      "outputs": [],
+      "source": [
+        "class PartiallyMigratedModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, units, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.units = units\n",
+        "    self.conv_layer = tf.keras.layers.Conv2D(\n",
+        "      3, 3,\n",
+        "      kernel_regularizer=\"l2\")\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs, training=None):\n",
+        "    with tf.compat.v1.variable_scope('model'):\n",
+        "      out = self.conv_layer(inputs)\n",
+        "      out = tf.compat.v1.layers.flatten(out)\n",
+        "      out = tf.compat.v1.layers.dropout(out, training=training)\n",
+        "      out = tf.compat.v1.layers.dense(\n",
+        "          out, self.units,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      return out\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kzJF0H0sbce8"
+      },
+      "source": [
+        "Use the [`v1.keras.utils.DeterministicRandomTestTool`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/keras/utils/DeterministicRandomTestTool) class to verify that this incremental change leaves the model with the same behavior as before."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MTJq0qW9_Tz2"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  tf.keras.utils.set_random_seed(42)\n",
+        "  layer = CompatModel(10)\n",
+        "\n",
+        "  inputs = tf.random.normal(shape=(10, 5, 5, 5))\n",
+        "  original_output = layer(inputs)\n",
+        "\n",
+        "  # Grab the regularization loss as well\n",
+        "  original_regularization_loss = tf.math.add_n(layer.losses)\n",
+        "\n",
+        "print(original_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "X4Wq3wuaHjEV"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  tf.keras.utils.set_random_seed(42)\n",
+        "  layer = PartiallyMigratedModel(10)\n",
+        "\n",
+        "  inputs = tf.random.normal(shape=(10, 5, 5, 5))\n",
+        "  migrated_output = layer(inputs)\n",
+        "\n",
+        "  # Grab the regularization loss as well\n",
+        "  migrated_regularization_loss = tf.math.add_n(layer.losses)\n",
+        "\n",
+        "print(migrated_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mMMXS7EHjvCy"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that the regularization loss and output both match\n",
+        "np.testing.assert_allclose(original_regularization_loss.numpy(), migrated_regularization_loss.numpy())\n",
+        "np.testing.assert_allclose(original_output.numpy(), migrated_output.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RMxiMVFwbiQy"
+      },
+      "source": [
+        "You have now replaced all of the individual `compat.v1.layers` with native Keras layers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3dFCnyYc9DrX"
+      },
+      "outputs": [],
+      "source": [
+        "class NearlyFullyNativeModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, units, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.units = units\n",
+        "    self.conv_layer = tf.keras.layers.Conv2D(\n",
+        "      3, 3,\n",
+        "      kernel_regularizer=\"l2\")\n",
+        "    self.flatten_layer = tf.keras.layers.Flatten()\n",
+        "    self.dense_layer = tf.keras.layers.Dense(\n",
+        "      self.units,\n",
+        "      kernel_regularizer=\"l2\")\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs):\n",
+        "    with tf.compat.v1.variable_scope('model'):\n",
+        "      out = self.conv_layer(inputs)\n",
+        "      out = self.flatten_layer(out)\n",
+        "      out = self.dense_layer(out)\n",
+        "      return out\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QGPqEjkGHgar"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  tf.keras.utils.set_random_seed(42)\n",
+        "  layer = NearlyFullyNativeModel(10)\n",
+        "\n",
+        "  inputs = tf.random.normal(shape=(10, 5, 5, 5))\n",
+        "  migrated_output = layer(inputs)\n",
+        "\n",
+        "  # Grab the regularization loss as well\n",
+        "  migrated_regularization_loss = tf.math.add_n(layer.losses)\n",
+        "\n",
+        "print(migrated_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uAs60eCdj6x_"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that the regularization loss and output both match\n",
+        "np.testing.assert_allclose(original_regularization_loss.numpy(), migrated_regularization_loss.numpy())\n",
+        "np.testing.assert_allclose(original_output.numpy(), migrated_output.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oA6viSo3bo3y"
+      },
+      "source": [
+        "Finally, remove both any remaining (no-longer-needed) `variable_scope` usage and the `track_tf1_style_variables` decorator itself.\n",
+        "\n",
+        "You are now left with a version of the model that uses entirely native APIs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mIHpHWIRDunU"
+      },
+      "outputs": [],
+      "source": [
+        "class FullyNativeModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, units, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.units = units\n",
+        "    self.conv_layer = tf.keras.layers.Conv2D(\n",
+        "      3, 3,\n",
+        "      kernel_regularizer=\"l2\")\n",
+        "    self.flatten_layer = tf.keras.layers.Flatten()\n",
+        "    self.dense_layer = tf.keras.layers.Dense(\n",
+        "      self.units,\n",
+        "      kernel_regularizer=\"l2\")\n",
+        "\n",
+        "  def call(self, inputs):\n",
+        "    out = self.conv_layer(inputs)\n",
+        "    out = self.flatten_layer(out)\n",
+        "    out = self.dense_layer(out)\n",
+        "    return out\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ttAmiCvLHW54"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  tf.keras.utils.set_random_seed(42)\n",
+        "  layer = FullyNativeModel(10)\n",
+        "\n",
+        "  inputs = tf.random.normal(shape=(10, 5, 5, 5))\n",
+        "  migrated_output = layer(inputs)\n",
+        "\n",
+        "  # Grab the regularization loss as well\n",
+        "  migrated_regularization_loss = tf.math.add_n(layer.losses)\n",
+        "\n",
+        "print(migrated_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ym5DYtT4j7e3"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that the regularization loss and output both match\n",
+        "np.testing.assert_allclose(original_regularization_loss.numpy(), migrated_regularization_loss.numpy())\n",
+        "np.testing.assert_allclose(original_output.numpy(), migrated_output.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oX4pdrzycIsa"
+      },
+      "source": [
+        "### Maintaining checkpoint compatibility during migration to Native TF2\n",
+        "\n",
+        "The above migration process to native TF2 APIs changed both the variable names (as Keras APIs produce very different weight names), and the object-oriented paths that point to different weights in the model. The impact of these changes is that they will have broken both any existing TF1-style name-based checkpoints or TF2-style object-oriented checkpoints.\n",
+        "\n",
+        "However, in some cases, you might be able to take your original name-based checkpoint and find a mapping of the variables to their new names with approaches like the one detailed in the [Reusing TF1.x checkpoints guide](./migrating_checkpoints.ipynb).\n",
+        "\n",
+        "Some tips to making this feasible are as follows:\n",
+        "- Variables still all have a `name` argument you can set.\n",
+        "- Keras models also take a `name` argument as which they set as the prefix for their variables.\n",
+        "- The `v1.name_scope` function can be used to set variable name prefixes. This is very different from `tf.variable_scope`. It only affects names, and doesn't track variables and reuse.\n",
+        "\n",
+        "With the above pointers in mind, the following code samples demonstrate a workflow you can adapt to your code to incrementally update part of a model while simultaneously updating checkpoints.\n",
+        "\n",
+        "Note: Due to the complexity of variable naming with Keras layers, this is not guaranteed to work for all use cases."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EFmMY3dcx3mR"
+      },
+      "source": [
+        "1. Begin by switching functional-style `tf.compat.v1.layers` to their object-oriented versions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cRxCFmNjl2ta"
+      },
+      "outputs": [],
+      "source": [
+        "class FunctionalStyleCompatModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs, training=None):\n",
+        "    with tf.compat.v1.variable_scope('model'):\n",
+        "      out = tf.compat.v1.layers.conv2d(\n",
+        "          inputs, 3, 3,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      out = tf.compat.v1.layers.conv2d(\n",
+        "          out, 4, 4,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      out = tf.compat.v1.layers.conv2d(\n",
+        "          out, 5, 5,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      return out\n",
+        "\n",
+        "layer = FunctionalStyleCompatModel()\n",
+        "layer(tf.ones(shape=(10, 10, 10, 10)))\n",
+        "[v.name for v in layer.weights]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QvzUyXxjydAd"
+      },
+      "source": [
+        "2. Next, assign the compat.v1.layer objects and any variables created by `compat.v1.get_variable` as properties of the `tf.keras.layers.Layer`/`tf.Module` object whose method is decorated with `track_tf1_style_variables` (note that any object-oriented TF2 style checkpoints will now save out both a path by variable name and the new object-oriented path)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "02jMQkJFmFwl"
+      },
+      "outputs": [],
+      "source": [
+        "class OOStyleCompatModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.conv_1 = tf.compat.v1.layers.Conv2D(\n",
+        "          3, 3,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "    self.conv_2 = tf.compat.v1.layers.Conv2D(\n",
+        "          4, 4,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs, training=None):\n",
+        "    with tf.compat.v1.variable_scope('model'):\n",
+        "      out = self.conv_1(inputs)\n",
+        "      out = self.conv_2(out)\n",
+        "      out = tf.compat.v1.layers.conv2d(\n",
+        "          out, 5, 5,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      return out\n",
+        "\n",
+        "layer = OOStyleCompatModel()\n",
+        "layer(tf.ones(shape=(10, 10, 10, 10)))\n",
+        "[v.name for v in layer.weights]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8evFpd8Nq63v"
+      },
+      "source": [
+        "3. Resave a loaded checkpoint at this point to save out paths both by the variable name (for compat.v1.layers), or by the object-oriented object graph."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7neFr-9pqmJX"
+      },
+      "outputs": [],
+      "source": [
+        "weights = {v.name: v for v in layer.weights}\n",
+        "assert weights['model/conv2d/kernel:0'] is layer.conv_1.kernel\n",
+        "assert weights['model/conv2d_1/bias:0'] is layer.conv_2.bias"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pvsi743Xh9wn"
+      },
+      "source": [
+        "4. You can now swap out the object-oriented `compat.v1.layers` for native Keras layers while still being able to load the recently-saved checkpoint. Ensure that you preserve variable names for the remaining `compat.v1.layers` by still recording the auto-generated `variable_scopes` of the replaced layers. These switched layers/variables will now only use the object attribute path to the variables in the checkpoint instead of the variable name path.\n",
+        "\n",
+        "In general, you can replace usage of `compat.v1.get_variable` in variables attached to properties by:\n",
+        "\n",
+        "* Switching them to using `tf.Variable`, **OR**  \n",
+        "* Updating them by using [`tf.keras.layers.Layer.add_weight`](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer#add_weight). Note that if you are not switching all layers in one go this may change auto-generated layer/variable naming for the remaining `compat.v1.layers` that are missing a `name` argument. If that is the case, you must keep the variable names for remaining `compat.v1.layers` the same by manually opening and closing a `variable_scope` corresponding to the removed `compat.v1.layer`'s generated scope name. Otherwise the paths from existing checkpoints may conflict and checkpoint loading will behave incorrectly.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NbixtIW-maoH"
+      },
+      "outputs": [],
+      "source": [
+        "def record_scope(scope_name):\n",
+        "  \"\"\"Record a variable_scope to make sure future ones get incremented.\"\"\"\n",
+        "  with tf.compat.v1.variable_scope(scope_name):\n",
+        "    pass\n",
+        "\n",
+        "class PartiallyNativeKerasLayersModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.conv_1 = tf.keras.layers.Conv2D(\n",
+        "          3, 3,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "    self.conv_2 = tf.keras.layers.Conv2D(\n",
+        "          4, 4,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs, training=None):\n",
+        "    with tf.compat.v1.variable_scope('model'):\n",
+        "      out = self.conv_1(inputs)\n",
+        "      record_scope('conv2d') # Only needed if follow-on compat.v1.layers do not pass a `name` arg\n",
+        "      out = self.conv_2(out)\n",
+        "      record_scope('conv2d_1') # Only needed if follow-on compat.v1.layers do not pass a `name` arg\n",
+        "      out = tf.compat.v1.layers.conv2d(\n",
+        "          out, 5, 5,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "      return out\n",
+        "\n",
+        "layer = PartiallyNativeKerasLayersModel()\n",
+        "layer(tf.ones(shape=(10, 10, 10, 10)))\n",
+        "[v.name for v in layer.weights]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2eaPpevGs3dA"
+      },
+      "source": [
+        "Saving a checkpoint out at this step after constructing the variables will make it contain ***only*** the currently-available object paths. \n",
+        "\n",
+        "Ensure you record the scopes of the removed `compat.v1.layers` to preserve the auto-generated weight names for the remaining `compat.v1.layers`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EK7vtWBprObA"
+      },
+      "outputs": [],
+      "source": [
+        "weights = set(v.name for v in layer.weights)\n",
+        "assert 'model/conv2d_2/kernel:0' in weights\n",
+        "assert 'model/conv2d_2/bias:0' in weights"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DQ5-SfmWFTvY"
+      },
+      "source": [
+        "5. Repeat the above steps until you have replaced all the `compat.v1.layers` and `compat.v1.get_variable`s in your model with fully-native equivalents."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PA1d2POtnTQa"
+      },
+      "outputs": [],
+      "source": [
+        "class FullyNativeKerasLayersModel(tf.keras.layers.Layer):\n",
+        "\n",
+        "  def __init__(self, *args, **kwargs):\n",
+        "    super().__init__(*args, **kwargs)\n",
+        "    self.conv_1 = tf.keras.layers.Conv2D(\n",
+        "          3, 3,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "    self.conv_2 = tf.keras.layers.Conv2D(\n",
+        "          4, 4,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "    self.conv_3 = tf.keras.layers.Conv2D(\n",
+        "          5, 5,\n",
+        "          kernel_regularizer=\"l2\")\n",
+        "\n",
+        "\n",
+        "  def call(self, inputs, training=None):\n",
+        "    with tf.compat.v1.variable_scope('model'):\n",
+        "      out = self.conv_1(inputs)\n",
+        "      out = self.conv_2(out)\n",
+        "      out = self.conv_3(out)\n",
+        "      return out\n",
+        "\n",
+        "layer = FullyNativeKerasLayersModel()\n",
+        "layer(tf.ones(shape=(10, 10, 10, 10)))\n",
+        "[v.name for v in layer.weights]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vZejG7rTsTb6"
+      },
+      "source": [
+        "Remember to test to make sure the newly updated checkpoint still behaves as you expect. Apply the techniques described in the [validate numerical correctness guide](./validate_correctness.ipynb) at every incremental step of this process to ensure your migrated code runs correctly."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ewi_h-cs6n-I"
+      },
+      "source": [
+        "## Handling TF1.x to TF2 behavior changes not covered by the modeling shims\n",
+        "\n",
+        "The modeling shims described in this guide can make sure that variables, layers, and regularization losses created with `get_variable`, `tf.compat.v1.layers`, and `variable_scope` semantics continue to work as before when using eager execution and `tf.function`, without having to rely on collections.\n",
+        "\n",
+        "This does not cover ***all*** TF1.x-specific semantics that your model forward passes may be relying on. In some cases, the shims might be insufficient to get your model forward pass running in TF2 on their own. Read the [TF1.x vs TF2 behaviors guide](./tf1_vs_tf2) to learn more about the behavioral differences between TF1.x and TF2."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "model_mapping.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/multi_worker_cpu_gpu_training.ipynb b/site/en/guide/migrate/multi_worker_cpu_gpu_training.ipynb
new file mode 100644
index 00000000000..8a95cb903d6
--- /dev/null
+++ b/site/en/guide/migrate/multi_worker_cpu_gpu_training.ipynb
@@ -0,0 +1,476 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate multi-worker CPU/GPU training\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "This guide demonstrates how to migrate your multi-worker distributed training workflow from TensorFlow 1 to TensorFlow 2.\n",
+        "\n",
+        "To perform multi-worker training with CPUs/GPUs:\n",
+        "\n",
+        "- In TensorFlow 1, you traditionally use the `tf.estimator.train_and_evaluate` and `tf.estimator.Estimator` APIs.\n",
+        "- In TensorFlow 2, use the Keras APIs for writing the model, the loss function, the optimizer, and metrics. Then, distribute the training with Keras `Model.fit` API or a custom training loop (with `tf.GradientTape`) across multiple workers with `tf.distribute.experimental.ParameterServerStrategy` or `tf.distribute.MultiWorkerMirroredStrategy`. For more details, refer to the following tutorials:\n",
+        "  - [Distributed training with TensorFlow](../../guide/distributed_training.ipynb)\n",
+        "  - [Parameter server training with Keras Model.fit/a custom training loop](../../tutorials/distribute/parameter_server_training.ipynb)\n",
+        "  - [MultiWorkerMirroredStrategy with Keras Model.fit](../../tutorials/distribute/multi_worker_with_keras.ipynb)\n",
+        "  - [MultiWorkerMirroredStrategy with a custom training loop](../../tutorials/distribute/multi_worker_with_ctl.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "28f46832b54d"
+      },
+      "source": [
+        "Start with some necessary imports and a simple dataset for demonstration purposes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "# The notebook uses a dataset instance for `Model.fit` with\n",
+        "# `ParameterServerStrategy`, which depends on symbols in TF 2.7.\n",
+        "# Install a utility needed for this demonstration\n",
+        "!pip install portpicker\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m7rnGxsXtDkV"
+      },
+      "outputs": [],
+      "source": [
+        "features = [[1., 1.5], [2., 2.5], [3., 3.5]]\n",
+        "labels = [[0.3], [0.5], [0.7]]\n",
+        "eval_features = [[4., 4.5], [5., 5.5], [6., 6.5]]\n",
+        "eval_labels = [[0.8], [0.9], [1.]]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "T2uaw9QaDM_X"
+      },
+      "source": [
+        "You will need the `'TF_CONFIG'` configuration environment variable for training on multiple machines in TensorFlow. Use `'TF_CONFIG'` to specify the `'cluster'` and the `'task'`s' addresses. (Learn more in the [Distributed_training](../...guide/distributed_training.ipynb) guide.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4OUzwoQgXgkG"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import os\n",
+        "\n",
+        "tf_config = {\n",
+        "    'cluster': {\n",
+        "        'chief': ['localhost:11111'],\n",
+        "        'worker': ['localhost:12345', 'localhost:23456', 'localhost:21212'],\n",
+        "        'ps': ['localhost:12121', 'localhost:13131'],\n",
+        "    },\n",
+        "    'task': {'type': 'chief', 'index': 0}\n",
+        "}\n",
+        "\n",
+        "os.environ['TF_CONFIG'] = json.dumps(tf_config)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PbeoSbbmDdc0"
+      },
+      "source": [
+        "Note: Unfortunately, since multi-worker training with `tf.estimator` APIs in TensorFlow 1 requires multiple clients (which would be especially tricky to be done here in this Colab notebook), you will make the notebook runnable without a `'TF_CONFIG'` environment variable, so it falls back to local training. (Learn more in the *Setting up the `'TF_CONFIG'` environment variable* section in the [Distributed training with TensorFlow](../../guide/distributed_training.ipynb) guide.)\n",
+        "\n",
+        "Use the `del` statement to remove the variable (but in real-world multi-worker training in TensorFlow 1, you won't have to do this):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AHuynAR5D8sU"
+      },
+      "outputs": [],
+      "source": [
+        "del os.environ['TF_CONFIG']"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4uXff1BEssdE"
+      },
+      "source": [
+        "## TensorFlow 1: Multi-worker distributed training with tf.estimator APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MpyINdiLEN3c"
+      },
+      "source": [
+        "The following code snippet demonstrates the canonical workflow of multi-worker training in TF1: you will use a `tf.estimator.Estimator`, a `tf.estimator.TrainSpec`, a `tf.estimator.EvalSpec`, and the `tf.estimator.train_and_evaluate` API to distribute the training:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "def _input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices((features, labels)).batch(1)\n",
+        "\n",
+        "def _eval_input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).batch(1)\n",
+        "\n",
+        "def _model_fn(features, labels, mode):\n",
+        "  logits = tf1.layers.Dense(1)(features)\n",
+        "  loss = tf1.losses.mean_squared_error(labels=labels, predictions=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.05)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "  return tf1.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)\n",
+        "\n",
+        "estimator = tf1.estimator.Estimator(model_fn=_model_fn)\n",
+        "train_spec = tf1.estimator.TrainSpec(input_fn=_input_fn)\n",
+        "eval_spec = tf1.estimator.EvalSpec(input_fn=_eval_input_fn)\n",
+        "tf1.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "## TensorFlow 2: Multi-worker training with distribution strategies"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Syb66qsbEp1x"
+      },
+      "source": [
+        "In TensorFlow 2, distributed training across multiple workers with CPUs, GPUs, and TPUs is done via `tf.distribute.Strategy`s.\n",
+        "\n",
+        "The following example demonstrates how to use two such strategies: `tf.distribute.experimental.ParameterServerStrategy` and `tf.distribute.MultiWorkerMirroredStrategy`, both of which are designed for CPU/GPU training with multiple workers.\n",
+        "\n",
+        "`ParameterServerStrategy` employs a _coordinator_ (`'chief'`), which makes it more friendly with the environment in this Colab notebook. You will be using some utilities here to set up the supporting elements essential for a runnable experience here: you will create an _in-process cluster_, where threads are used to simulate the parameter servers (`'ps'`) and workers (`'worker'`). For more information about parameter server training, refer to the [Parameter server training with ParameterServerStrategy](../../tutorials/distribute/parameter_server_training.ipynb) tutorial.\n",
+        "\n",
+        "In this example, first define the `'TF_CONFIG'` environment variable with a `tf.distribute.cluster_resolver.TFConfigClusterResolver` to provide the cluster information. If you are using a cluster management system for your distributed training, check if it provides `'TF_CONFIG'` for you already, in which case you don't need to explicitly set this environment variable. (Learn more in the *Setting up the `'TF_CONFIG'` environment variable* section in the [Distributed training with TensorFlow](../../guide/distributed_training.ipynb) guide.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rp-gFY0H5rF-"
+      },
+      "outputs": [],
+      "source": [
+        "# Find ports that are available for the `'chief'` (the coordinator),\n",
+        "# `'worker'`s, and `'ps'` (parameter servers).\n",
+        "import portpicker\n",
+        "\n",
+        "chief_port = portpicker.pick_unused_port()\n",
+        "worker_ports = [portpicker.pick_unused_port() for _ in range(3)]\n",
+        "ps_ports = [portpicker.pick_unused_port() for _ in range(2)]\n",
+        "\n",
+        "# Dump the cluster information to `'TF_CONFIG'`.\n",
+        "tf_config = {\n",
+        "    'cluster': {\n",
+        "        'chief': [\"localhost:%s\" % chief_port],\n",
+        "        'worker': [\"localhost:%s\" % port for port in worker_ports],\n",
+        "        'ps':  [\"localhost:%s\" % port for port in ps_ports],\n",
+        "    },\n",
+        "    'task': {'type': 'chief', 'index': 0}\n",
+        "}\n",
+        "os.environ['TF_CONFIG'] = json.dumps(tf_config)\n",
+        "\n",
+        "# Use a cluster resolver to bridge the information to the strategy created below.\n",
+        "cluster_resolver = tf.distribute.cluster_resolver.TFConfigClusterResolver()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o_8uVvJb6dqq"
+      },
+      "source": [
+        "Then, create `tf.distribute.Server`s for the workers and parameter servers one-by-one:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZJopinmG6b2z"
+      },
+      "outputs": [],
+      "source": [
+        "# Workers need some inter_ops threads to work properly.\n",
+        "# This is only needed for this notebook to demo. Real servers\n",
+        "# should not need this.\n",
+        "worker_config = tf.compat.v1.ConfigProto()\n",
+        "worker_config.inter_op_parallelism_threads = 4\n",
+        "\n",
+        "for i in range(3):\n",
+        "  tf.distribute.Server(\n",
+        "      cluster_resolver.cluster_spec(),\n",
+        "      job_name=\"worker\",\n",
+        "      task_index=i,\n",
+        "      config=worker_config)\n",
+        "\n",
+        "for i in range(2):\n",
+        "  tf.distribute.Server(\n",
+        "      cluster_resolver.cluster_spec(),\n",
+        "      job_name=\"ps\",\n",
+        "      task_index=i)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IpfCcF0g6Ao8"
+      },
+      "source": [
+        "In real-world distributed training, instead of starting all the `tf.distribute.Server`s on the coordinator, you will be using multiple machines, and the ones that are designated as `\"worker\"`s and `\"ps\"` (parameter servers) will each run a `tf.distribute.Server`. Refer to *Clusters in the real world* section in the [Parameter server training](../../tutorials/distribute/parameter_server_training.ipynb) tutorial for more details.\n",
+        "\n",
+        "With everything ready, create the `ParameterServerStrategy` object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "t45iQeBT7Us_"
+      },
+      "outputs": [],
+      "source": [
+        "strategy = tf.distribute.experimental.ParameterServerStrategy(cluster_resolver)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "diNsps1MGRS6"
+      },
+      "source": [
+        "Once you have created a strategy object, define the model,  the optimizer, and other variables, and call the Keras `Model.compile` within the `Strategy.scope` API to distribute the training. (Refer to the `Strategy.scope` API docs for more information.)\n",
+        "\n",
+        "If you prefer to customize your training by, for instance, defining the forward and backward passes, refer to *Training with a custom training loop* section in [Parameter server training](../../tutorials/distribute/parameter_server_training.ipynb) tutorial for more details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "atVciNgPs0fw"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "      (features, labels)).shuffle(10).repeat().batch(64)\n",
+        "\n",
+        "eval_dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "      (eval_features, eval_labels)).repeat().batch(1)\n",
+        "\n",
+        "with strategy.scope():\n",
+        "  model = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])\n",
+        "  optimizer = tf.keras.optimizers.legacy.Adagrad(learning_rate=0.05)\n",
+        "  model.compile(optimizer, \"mse\")\n",
+        "\n",
+        "model.fit(dataset, epochs=5, steps_per_epoch=10)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "akZ0aaaS1vA9"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(eval_dataset, steps=10, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pXbS71XmMSoO"
+      },
+      "source": [
+        "> **Partitioners (`tf.distribute.experimental.partitioners`)**\n",
+        ">\n",
+        "> `ParameterServerStrategy` in TensorFlow 2 supports variable partitioning and offers same partitioners as TensorFlow 1, with less confusing names:\n",
+        "> - `tf.compat.v1.variable_axis_size_partitioner` -> `tf.distribute.experimental.partitioners.MaxSizePartitioner`: a partitioner that keeps shards under a maximum size).\n",
+        "> - `tf.compat.v1.min_max_variable_partitioner` -> `tf.distribute.experimental.partitioners.MinSizePartitioner`: a partitioner that allocates a minimum size per shard.\n",
+        "> - `tf.compat.v1.fixed_size_partitioner` -> `tf.distribute.experimental.partitioners.FixedShardsPartitioner`: a partitioner that allocates a fixed number of shards."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ig0-uCUbGprd"
+      },
+      "source": [
+        "Alternatively, you can use a `MultiWorkerMirroredStrategy` object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xHXP8bOBGtXL"
+      },
+      "outputs": [],
+      "source": [
+        "# To clean up the `TF_CONFIG` used for `ParameterServerStrategy`.\n",
+        "del os.environ['TF_CONFIG']\n",
+        "strategy = tf.distribute.MultiWorkerMirroredStrategy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tOsmqefTGwUf"
+      },
+      "source": [
+        "You can replace the strategy used above with a `MultiWorkerMirroredStrategy` object to perform training with this strategy.\n",
+        "\n",
+        "As with the `tf.estimator` APIs, since `MultiWorkerMirroredStrategy` is a multi-client strategy, there is no easy way to run distributed training in this Colab notebook. Therefore, replacing the code above with this strategy ends up running things locally. The Multi-worker training [with Keras Model.fit](../../tutorials/distribute/multi_worker_with_keras.ipynb)/[a custom training loop](../../tutorials/distribute/multi_worker_with_ctl.ipynb) tutorials demonstrate how to run multi-worker training with\n",
+        " the `'TF_CONFIG'` variable set up, with two workers on a localhost in Colab. In practice, you would create multiple workers on external IP addresses/ports, and use the `'TF_CONFIG'` variable to specify the cluster configuration for each worker."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "917ef6135660"
+      },
+      "source": [
+        "## Next steps"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e76fd9d5c98c"
+      },
+      "source": [
+        "To learn more about multi-worker distributed training with `tf.distribute.experimental.ParameterServerStrategy` and `tf.distribute.MultiWorkerMirroredStrategy` in TensorFlow 2, consider the following resources:\n",
+        "\n",
+        "- Tutorial: [Parameter server training with ParameterServerStrategy and Keras Model.fit/a custom training loop](../../tutorials/distribute/parameter_server_training.ipynb)\n",
+        "- Tutorial: [Multi-worker training with MultiWorkerMirroredStrategy and Keras Model.fit](../../tutorials/distribute/multi_worker_with_keras.ipynb)\n",
+        "- Tutorial: [Multi-worker training with MultiWorkerMirroredStrategy and a custom training loop](../../tutorials/distribute/multi_worker_with_ctl.ipynb)\n",
+        "- Guide: [Distributed training with TensorFlow](../../guide/distributed_training.ipynb)\n",
+        "- Guide: [Optimize TensorFlow GPU performance with the TensorFlow Profiler](../../guide/gpu_performance_analysis.ipynb)\n",
+        "- Guide: [Use a GPU](../../guide/gpu.ipynb) (the Using multiple GPUs section)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "multi_worker_cpu_gpu_training.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/saved_model.ipynb b/site/en/guide/migrate/saved_model.ipynb
new file mode 100644
index 00000000000..e7e8ce8daa1
--- /dev/null
+++ b/site/en/guide/migrate/saved_model.ipynb
@@ -0,0 +1,767 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "q9KfUf1BI6Kl"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "WvqLCVQ6I58i"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZoFRICPTNUca"
+      },
+      "source": [
+        "# Migrate the SavedModel workflow\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nGyIb7MYJfaM"
+      },
+      "source": [
+        "Once you have migrated your model from TensorFlow 1's graphs and sessions to TensorFlow 2 APIs, such as `tf.function`, `tf.Module`, and `tf.keras.Model`, you can migrate the model saving and loading code. This notebook provides examples of how you can save and load in the SavedModel format in TensorFlow 1 and TensorFlow 2. Here is a quick overview of the related API changes for migration from TensorFlow 1 to TensorFlow 2:\n",
+        "\n",
+        "|                                                                                         | TensorFlow 1                                                                                                                                                         | Migration to TensorFlow 2                                                                                                                                                                  |\n",
+        "| --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |\n",
+        "| **Saving**                                                                              | `tf.compat.v1.saved_model.Builder`
`tf.compat.v1.saved_model.simple_save`                                                                                         | `tf.saved_model.save`
Keras: `tf.keras.models.save_model`                                                                                                                               |\n",
+        "| **Loading**                                                                             | `tf.compat.v1.saved_model.load`                                                                                                                                      | `tf.saved_model.load`
Keras: `tf.keras.models.load_model`                                                                                                                               |\n",
+        "| **Signatures**: a set of input
and output tensors that
can be used to run the
 | Generated using the `*.signature_def` utils
(e.g. `tf.compat.v1.saved_model.predict_signature_def`)                                                               | Write a `tf.function` and export it using the `signatures` argument
in `tf.saved_model.save`.                                                                                           |\n",
+        "| **Classification
and regression**:
special types of signatures                    | Generated with
`tf.compat.v1.saved_model.classification_signature_def`,
`tf.compat.v1.saved_model.regression_signature_def`,
and certain Estimator exports. | These two signature types have been removed from TensorFlow 2.
If the serving library requires these method names,
`tf.compat.v1.saved_model.signature_def_utils.MethodNameUpdater`. |\n",
+        "\n",
+        "For a more in-depth explanation of the mapping, refer to the [Changes from TensorFlow 1 to TensorFlow 2](#changes_from_tf1_to_tf2) section below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r5mR2xsNAGsB"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "The examples below show how to export and load the same dummy TensorFlow model (defined as `add_two` below) to a SavedModel format using the TensorFlow 1 and TensorFlow 2 APIs. Start by setting up the imports and utility functions:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B94QZyy-kOGQ"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1\n",
+        "import shutil\n",
+        "\n",
+        "def remove_dir(path):\n",
+        "  try:\n",
+        "    shutil.rmtree(path)\n",
+        "  except:\n",
+        "    pass\n",
+        "\n",
+        "def add_two(input):\n",
+        "  return input + 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZNVpH5tOCgd9"
+      },
+      "source": [
+        "## TensorFlow 1: Save and export a SavedModel\n",
+        "\n",
+        "In TensorFlow 1, you use the `tf.compat.v1.saved_model.Builder`, `tf.compat.v1.saved_model.simple_save`, and `tf.estimator.Estimator.export_saved_model` APIs to build, save, and export the TensorFlow graph and session:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "THRLul5ijmTE"
+      },
+      "source": [
+        "### 1. Save the graph as a SavedModel with SavedModelBuilder"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dcZDQaI8jl3h"
+      },
+      "outputs": [],
+      "source": [
+        "remove_dir(\"saved-model-builder\")\n",
+        "\n",
+        "with tf.Graph().as_default() as g:\n",
+        "  with tf1.Session() as sess:\n",
+        "    input = tf1.placeholder(tf.float32, shape=[])\n",
+        "    output = add_two(input)\n",
+        "    print(\"add two output: \", sess.run(output, {input: 3.}))\n",
+        "\n",
+        "    # Save with SavedModelBuilder\n",
+        "    builder = tf1.saved_model.Builder('saved-model-builder')\n",
+        "    sig_def = tf1.saved_model.predict_signature_def(\n",
+        "        inputs={'input': input},\n",
+        "        outputs={'output': output})\n",
+        "    builder.add_meta_graph_and_variables(\n",
+        "        sess, tags=[\"serve\"], signature_def_map={\n",
+        "            tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY: sig_def\n",
+        "    })\n",
+        "    builder.save()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PwtC27VFlwCa"
+      },
+      "outputs": [],
+      "source": [
+        "!saved_model_cli run --dir saved-model-builder --tag_set serve \\\n",
+        " --signature_def serving_default --input_exprs input=10"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gnBDNTxKG_vR"
+      },
+      "source": [
+        "### 2. Build a SavedModel for serving"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jtMxe2rjHSq9"
+      },
+      "outputs": [],
+      "source": [
+        "remove_dir(\"simple-save\")\n",
+        "\n",
+        "with tf.Graph().as_default() as g:\n",
+        "  with tf1.Session() as sess:\n",
+        "    input = tf1.placeholder(tf.float32, shape=[])\n",
+        "    output = add_two(input)\n",
+        "    print(\"add_two output: \", sess.run(output, {input: 3.}))\n",
+        "\n",
+        "    tf1.saved_model.simple_save(\n",
+        "        sess, 'simple-save',\n",
+        "        inputs={'input': input},\n",
+        "        outputs={'output': output})"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AdnqemvIHb2P"
+      },
+      "outputs": [],
+      "source": [
+        "!saved_model_cli run --dir simple-save --tag_set serve \\\n",
+        " --signature_def serving_default --input_exprs input=10"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r0BNzzAHjnkp"
+      },
+      "source": [
+        "### 3. Export the Estimator inference graph as a SavedModel\n",
+        "\n",
+        "In the definition of the Estimator `model_fn` (defined below), you can define signatures in your model by returning `export_outputs` in the `tf.estimator.EstimatorSpec`. There are different types of outputs:\n",
+        "\n",
+        "- `tf.estimator.export.ClassificationOutput`\n",
+        "- `tf.estimator.export.RegressionOutput`\n",
+        "- `tf.estimator.export.PredictOutput`\n",
+        "\n",
+        "These will produce classification, regression, and prediction signature types, respectively.\n",
+        "\n",
+        "When the estimator is exported with `tf.estimator.Estimator.export_saved_model`, these signatures will be saved with the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3nQ5Stnxjhfs"
+      },
+      "outputs": [],
+      "source": [
+        "def model_fn(features, labels, mode):\n",
+        "  output = add_two(features['input'])\n",
+        "  step = tf1.train.get_global_step()\n",
+        "  return tf.estimator.EstimatorSpec(\n",
+        "      mode,\n",
+        "      predictions=output,\n",
+        "      train_op=step.assign_add(1),\n",
+        "      loss=tf.constant(0.),\n",
+        "      export_outputs={\n",
+        "          tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY: \\\n",
+        "          tf.estimator.export.PredictOutput({'output': output})})\n",
+        "est = tf.estimator.Estimator(model_fn, 'estimator-checkpoints')\n",
+        "\n",
+        "# Train for one step to create a checkpoint.\n",
+        "def train_fn():\n",
+        "  return tf.data.Dataset.from_tensors({'input': 3.})\n",
+        "est.train(train_fn, steps=1)\n",
+        "\n",
+        "# This utility function `build_raw_serving_input_receiver_fn` takes in raw\n",
+        "# tensor features and builds an \"input serving receiver function\", which\n",
+        "# creates placeholder inputs to the model.\n",
+        "serving_input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn(\n",
+        "    {'input': tf.constant(3.)})  # Pass in a dummy input batch.\n",
+        "estimator_path = est.export_saved_model('exported-estimator', serving_input_fn)\n",
+        "\n",
+        "# Estimator's export_saved_model creates a time stamped directory. Move this\n",
+        "# to a set path so it can be inspected with `saved_model_cli` in the cell below.\n",
+        "!rm -rf estimator-model\n",
+        "import shutil\n",
+        "shutil.move(estimator_path, 'estimator-model')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8_gD2gkE7CMu"
+      },
+      "outputs": [],
+      "source": [
+        "!saved_model_cli run --dir estimator-model --tag_set serve \\\n",
+        " --signature_def serving_default --input_exprs input=[10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DyBvrNQgIhIo"
+      },
+      "source": [
+        "## TensorFlow 2: Save and export a SavedModel"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BZmFH-eIjqjB"
+      },
+      "source": [
+        "### Save and export a SavedModel defined with tf.Module\n",
+        "\n",
+        "To export your model in TensorFlow 2, you must define a `tf.Module` or a `tf.keras.Model` to hold all of your model's variables and functions. Then, you can call `tf.saved_model.save` to create a SavedModel. Refer to the _Saving a custom model_ section in the [Using the SavedModel format](../saved_model.ipynb) guide to learn more."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_j-PwgP_jrgw"
+      },
+      "outputs": [],
+      "source": [
+        "class MyModel(tf.Module):\n",
+        "  @tf.function\n",
+        "  def __call__(self, input):\n",
+        "    return add_two(input)\n",
+        "\n",
+        "model = MyModel()\n",
+        "\n",
+        "@tf.function\n",
+        "def serving_default(input):\n",
+        "  return {'output': model(input)}\n",
+        "\n",
+        "signature_function = serving_default.get_concrete_function(\n",
+        "    tf.TensorSpec(shape=[], dtype=tf.float32))\n",
+        "tf.saved_model.save(\n",
+        "    model, 'tf2-save', signatures={\n",
+        "        tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature_function})"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "slvU4vZN756F"
+      },
+      "outputs": [],
+      "source": [
+        "!saved_model_cli run --dir tf2-save --tag_set serve \\\n",
+        " --signature_def serving_default --input_exprs input=10"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UYpSfbBJjr33"
+      },
+      "source": [
+        "### Save and export a SavedModel defined with Keras\n",
+        "\n",
+        "\n",
+        "Deprecated: For Keras objects, it's recommended to use the new high-level `.keras` format and `tf.keras.Model.export`, as demonstrated in the guide [here](https://www.tensorflow.org/guide/keras/save_and_serialize). The low-level SavedModel format continues to be supported for existing code.\n",
+        "\n",
+        "\n",
+        "The Keras APIs for saving and exporting—`Model.save` or `tf.keras.models.save_model`—can export a SavedModel from a `tf.keras.Model`. Check out the [Save and load Keras models](../..guide/keras/save_and_serialize) for more details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mMcjhzyRjvp6"
+      },
+      "outputs": [],
+      "source": [
+        "inp = tf.keras.Input(3)\n",
+        "out = add_two(inp)\n",
+        "model = tf.keras.Model(inputs=inp, outputs=out)\n",
+        "\n",
+        "@tf.function(input_signature=[tf.TensorSpec(shape=[], dtype=tf.float32)])\n",
+        "def serving_default(input):\n",
+        "  return {'output': model(input)}\n",
+        "\n",
+        "model.save('keras-model', save_format='tf', signatures={\n",
+        "        tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY: serving_default})"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4P93WP5R7-VT"
+      },
+      "outputs": [],
+      "source": [
+        "!saved_model_cli run --dir keras-model --tag_set serve \\\n",
+        " --signature_def serving_default --input_exprs input=10"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SEKe9rGgoGCw"
+      },
+      "source": [
+        "## Loading a SavedModel\n",
+        "\n",
+        "A SavedModel saved with any of the above APIs can be loaded using either TensorFlow 1 or TensorFlow 2 APIs.\n",
+        "\n",
+        "A TensorFlow 1 SavedModel can generally be used for inference when loaded into TensorFlow 2, but training (generating gradients) is only possible if the SavedModel contains *resource variables*. You can check the dtype of the variables—if the variable dtype contains \"_ref\", then it is a reference variable.\n",
+        "\n",
+        "A TensorFlow 2 SavedModel can be loaded and executed from TensorFlow 1 as long as the SavedModel is saved with signatures.\n",
+        "\n",
+        "The sections below contain code samples showing how to load the SavedModels saved in the previous sections, and call the exported signature."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hLztK_0YoTEP"
+      },
+      "source": [
+        "### TensorFlow 1: Load a SavedModel with tf.saved_model.load\n",
+        "\n",
+        "In TensorFlow 1, you can import a SavedModel directly into the current graph and session using `tf.saved_model.load`. You can call `Session.run` on the tensor input and output names:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IMO0laj-m0p9"
+      },
+      "outputs": [],
+      "source": [
+        "def load_tf1(path, input):\n",
+        "  print('Loading from', path)\n",
+        "  with tf.Graph().as_default() as g:\n",
+        "    with tf1.Session() as sess:\n",
+        "      meta_graph = tf1.saved_model.load(sess, [\"serve\"], path)\n",
+        "      sig_def = meta_graph.signature_def[tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY]\n",
+        "      input_name = sig_def.inputs['input'].name\n",
+        "      output_name = sig_def.outputs['output'].name\n",
+        "      print('  Output with input', input, ': ', \n",
+        "            sess.run(output_name, feed_dict={input_name: input}))\n",
+        "\n",
+        "load_tf1('saved-model-builder', 5.)\n",
+        "load_tf1('simple-save', 5.)\n",
+        "load_tf1('estimator-model', [5.])  # Estimator's input must be batched.\n",
+        "load_tf1('tf2-save', 5.)\n",
+        "load_tf1('keras-model', 5.)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FbR3sfvooVBN"
+      },
+      "source": [
+        "### TensorFlow 2: Load a model saved with tf.saved_model\n",
+        "\n",
+        "In TensorFlow 2, objects are loaded into a Python object that stores the variables and functions. This is compatible with models saved from TensorFlow 1.\n",
+        "\n",
+        "Check out the `tf.saved_model.load` API docs and [Loading and using a custom model](../../guide/saved_model#loading_and_using_a_custom_model) section from the [Using the SavedModel format](../..guide/saved_model) guide for details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OA52ezWV_KgL"
+      },
+      "outputs": [],
+      "source": [
+        "def load_tf2(path, input):\n",
+        "  print('Loading from', path)\n",
+        "  loaded = tf.saved_model.load(path)\n",
+        "  out = loaded.signatures[tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY](\n",
+        "      tf.constant(input))['output']\n",
+        "  print('  Output with input', input, ': ', out)\n",
+        "\n",
+        "load_tf2('saved-model-builder', 5.)\n",
+        "load_tf2('simple-save', 5.)\n",
+        "load_tf2('estimator-model', [5.])  # Estimator's input must be batched.\n",
+        "load_tf2('tf2-save', 5.)\n",
+        "load_tf2('keras-model', 5.)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Gz3VFn5aAfmK"
+      },
+      "source": [
+        "Models saved with the TensorFlow 2 API can also access `tf.function`s and variables that are attached to the model (instead of those exported as signatures). For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IfMTp-TGAfOs"
+      },
+      "outputs": [],
+      "source": [
+        "loaded = tf.saved_model.load('tf2-save')\n",
+        "print('restored __call__:', loaded.__call__)\n",
+        "print('output with input 5.', loaded(5))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VMoErNKHoXEg"
+      },
+      "source": [
+        "### TensorFlow 2: Load a model saved with Keras\n",
+        "\n",
+        "Deprecated: For Keras objects, it's recommended to use the new high-level `.keras` format and `tf.keras.Model.export`, as demonstrated in the guide [here](https://www.tensorflow.org/guide/keras/save_and_serialize). The low-level SavedModel format continues to be supported for existing code.\n",
+        "\n",
+        "\n",
+        "The Keras loading API—`tf.keras.models.load_model`—allows you to reload a saved model back into a Keras Model object. Note that this only allows you to load SavedModels saved with Keras (`Model.save` or `tf.keras.models.save_model`).\n",
+        "\n",
+        "Models saved with `tf.saved_model.save` should be loaded with `tf.saved_model.load`. You can load a Keras model saved with `Model.save` using `tf.saved_model.load` but you will only get the TensorFlow graph. Refer to the `tf.keras.models.load_model` API docs and [Save and load Keras models](https://www.tensorflow.org/guide/keras/save_and_serialize#savedmodel_format) guide for details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZFUAxK0YeIAe"
+      },
+      "outputs": [],
+      "source": [
+        "loaded_model = tf.keras.models.load_model('keras-model')\n",
+        "loaded_model.predict_on_batch(tf.constant([1, 3, 4]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tz4eAAGY19MM"
+      },
+      "source": [
+        "## GraphDef and MetaGraphDef\n",
+        "\n",
+        " \n",
+        "\n",
+        "There is no straightforward way to load a raw `GraphDef` or `MetaGraphDef` to TF2. However, you can convert the TF1 code that imports the graph into a TF2 [`concrete_function`](https://tensorflow.org/guide/concrete_function) using [`v1.wrap_function`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/wrap_function).\n",
+        "\n",
+        "First, save a MetaGraphDef:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "grKam9zGnNRZ"
+      },
+      "outputs": [],
+      "source": [
+        "# Save a simple multiplication computation:\n",
+        "with tf.Graph().as_default() as g:\n",
+        "  x = tf1.placeholder(tf.float32, shape=[], name='x')\n",
+        "  v = tf.Variable(3.0, name='v')\n",
+        "  y = tf.multiply(x, v, name='y')\n",
+        "  with tf1.Session() as sess:\n",
+        "    sess.run(v.initializer)\n",
+        "    print(sess.run(y, feed_dict={x: 5}))\n",
+        "    s = tf1.train.Saver()\n",
+        "    s.export_meta_graph('multiply.pb', as_text=True)\n",
+        "    s.save(sess, 'multiply_values.ckpt')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bJnCe7eYrXev"
+      },
+      "source": [
+        "Using TF1 APIs, you can use `tf1.train.import_meta_graph` to import the graph and restore the values:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3bbcGZ4CoVDL"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.Graph().as_default() as g:\n",
+        "  meta = tf1.train.import_meta_graph('multiply.pb')\n",
+        "  x = g.get_tensor_by_name('x:0')\n",
+        "  y = g.get_tensor_by_name('y:0')\n",
+        "  with tf1.Session() as sess:\n",
+        "    meta.restore(sess, 'multiply_values.ckpt')\n",
+        "    print(sess.run(y, feed_dict={x: 5}))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s9OnHOLDrnco"
+      },
+      "source": [
+        "There are no TF2 APIs for loading the graph, but you can still import it into a concrete function that can be executed in eager mode:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yAyGmDLlpVBX"
+      },
+      "outputs": [],
+      "source": [
+        "def import_multiply():\n",
+        "  # Any graph-building code is allowed here.\n",
+        "  tf1.train.import_meta_graph('multiply.pb')\n",
+        "\n",
+        "# Creates a tf.function with all the imported elements in the function graph.\n",
+        "wrapped_import = tf1.wrap_function(import_multiply, [])\n",
+        "import_graph = wrapped_import.graph\n",
+        "x = import_graph.get_tensor_by_name('x:0')\n",
+        "y = import_graph.get_tensor_by_name('y:0')\n",
+        "\n",
+        "# Restore the variable values.\n",
+        "tf1.train.Saver(wrapped_import.variables).restore(\n",
+        "    sess=None, save_path='multiply_values.ckpt')\n",
+        "\n",
+        "# Create a concrete function by pruning the wrap_function (similar to sess.run).\n",
+        "multiply_fn = wrapped_import.prune(feeds=x, fetches=y)\n",
+        "\n",
+        "# Run this function\n",
+        "multiply_fn(tf.constant(5.))  # inputs to concrete functions must be Tensors."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GZ5vGJ0IDorc"
+      },
+      "source": [
+        "## Changes from TensorFlow 1 to TensorFlow 2\n",
+        "\n",
+        " \n",
+        "\n",
+        "This section lists out key saving and loading terms from TensorFlow 1, their TensorFlow 2 equivalents, and what has changed.\n",
+        "\n",
+        "### SavedModel\n",
+        "\n",
+        "[SavedModel](../../guide/saved_model.ipynb) is a format that stores a complete TensorFlow program with parameters and computation. It contains signatures used by serving platforms to run the model.\n",
+        "\n",
+        "The file format itself has not changed significantly, so SavedModels can be loaded and served using either TensorFlow 1 or TensorFlow 2 APIs.\n",
+        "\n",
+        "**Differences between TensorFlow 1 and TensorFlow 2**\n",
+        "\n",
+        "The *serving* and *inference* use cases have not been updated in TensorFlow 2, aside from the API changes—the improvement was introduced in the ability to *reuse* and *compose models* loaded from SavedModel.\n",
+        "\n",
+        "In TensorFlow 2, the program is represented by objects like `tf.Variable`, `tf.Module`, or higher-level Keras models (`tf.keras.Model`) and layers (`tf.keras.layers`). There are no more global variables that have values stored in a session, and the graph now exists in different `tf.function`s. Consequently, during a model export, SavedModel saves each component and function graphs separately.\n",
+        "\n",
+        "When you write a TensorFlow program with the TensorFlow Python APIs, you must build an object to manage the variables, functions, and other resources. Generally, this is accomplished by using the Keras APIs, but you can also build the object by creating or subclassing `tf.Module`.\n",
+        "\n",
+        "Keras models (`tf.keras.Model`) and `tf.Module` automatically track variables and functions attached to them. SavedModel saves these connections between modules, variables, and functions, so that they can be restored when loading.\n",
+        "\n",
+        "### Signatures\n",
+        "\n",
+        "Signatures are the endpoints of a SavedModel—they tell the user how to run the model and what inputs are needed.\n",
+        "\n",
+        "In TensorFlow 1, signatures are created by listing the input and output tensors. In TensorFlow 2, signatures are generated by passing in *concrete functions*. (Read more about TensorFlow functions in the [Introduction to graphs and tf.function](../intro_to_graphs.ipynb) guide, particularly the _Polymorphism: one Function, many graphs_ section.) In short, a concrete function is generated from a `tf.function`:\n",
+        "\n",
+        "```python\n",
+        "# Option 1: Specify an input signature.\n",
+        "@tf.function(input_signature=[...])\n",
+        "def fn(...):\n",
+        "  ...\n",
+        "  return outputs\n",
+        "\n",
+        "tf.saved_model.save(model, path, signatures={\n",
+        "    'name': fn\n",
+        "})\n",
+        "```\n",
+        "\n",
+        "```python\n",
+        "# Option 2: Call `get_concrete_function`\n",
+        "@tf.function\n",
+        "def fn(...):\n",
+        "  ...\n",
+        "  return outputs\n",
+        "\n",
+        "tf.saved_model.save(model, path, signatures={\n",
+        "    'name': fn.get_concrete_function(...)\n",
+        "})\n",
+        "```\n",
+        "\n",
+        "### `Session.run`\n",
+        "\n",
+        "In TensorFlow 1, you could call `Session.run` with the imported graph as long as you already know the tensor names. This allows you to retrieve the restored variable values, or run parts of the model that were not exported in the signatures.\n",
+        "\n",
+        "In TensorFlow 2, you can directly access a variable, such as a weights matrix (`kernel`):\n",
+        "\n",
+        "```python\n",
+        "model = tf.Module()\n",
+        "model.dense_layer = tf.keras.layers.Dense(...)\n",
+        "tf.saved_model.save('my_saved_model')\n",
+        "loaded = tf.saved_model.load('my_saved_model')\n",
+        "loaded.dense_layer.kernel\n",
+        "```\n",
+        "\n",
+        "\n",
+        "or call `tf.function`s attached to the model object: for example, `loaded.__call__`.\n",
+        "\n",
+        "Unlike TF1, there is no way to extract parts of a function and access intermediate values. You *must* export all of the needed functionality in the saved object.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b6NG9JvUwJxn"
+      },
+      "source": [
+        "## TensorFlow Serving migration notes\n",
+        "\n",
+        "SavedModel was originally created to work with [TensorFlow Serving](https://www.tensorflow.org/tfx/guide/serving). This platform offers different types of prediction requests: classify, regress, and predict.\n",
+        "\n",
+        "The **TensorFlow 1** API allows you to create these types of signatures with the utils:\n",
+        "\n",
+        "- `tf.compat.v1.saved_model.classification_signature_def`\n",
+        "- `tf.compat.v1.saved_model.regression_signature_def`\n",
+        "- `tf.compat.v1.saved_model.predict_signature_def`\n",
+        "\n",
+        "[Classification](https://www.tensorflow.org/tfx/serving/signature_defs#classification_signaturedef) (`classification_signature_def`) and [regression](https://www.tensorflow.org/tfx/serving/signature_defs#regression_signaturedef) (`regression_signature_def`) restrict the inputs and outputs, so the inputs must be a `tf.Example`, and the outputs must be `classes`, `scores` or `prediction`. Meanwhile, [the predict signature](https://www.tensorflow.org/tfx/serving/signature_defs#predict_signaturedef) (`predict_signature_def`) has no restrictions.\n",
+        "\n",
+        "SavedModels exported with the **TensorFlow 2** API are compatible with TensorFlow Serving, but will only contain prediction signatures. The classification and regression signatures have been removed.\n",
+        "\n",
+        "If you require the use of the classification and regression signatures, you may modify the exported SavedModel using `tf.compat.v1.saved_model.signature_def_utils.MethodNameUpdater`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a3acd3b86215"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "To learn more about SavedModels in TensorFlow 2, check out the following guides:\n",
+        "\n",
+        "- [Using the SavedModel format](https://www.tensorflow.org/guide/saved_model)\n",
+        "- [Save and load Keras models](https://www.tensorflow.org/guide/keras/save_and_serialize)\n",
+        "\n",
+        "If you are using TensorFlow Hub, you may find these guides useful:\n",
+        "\n",
+        "- [TensorFlow Hub: Model compatibility for TensorFlow 1/TensorFlow 2](https://www.tensorflow.org/hub/model_compatibility)\n",
+        "- [Migrating from TensorFlow 1 to TensorFlow 2 with TensorFlow Hub](https://www.tensorflow.org/hub/migration_tf2)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "saved_model.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/sessionrunhook_callback.ipynb b/site/en/guide/migrate/sessionrunhook_callback.ipynb
new file mode 100644
index 00000000000..7e20a1bab05
--- /dev/null
+++ b/site/en/guide/migrate/sessionrunhook_callback.ipynb
@@ -0,0 +1,272 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate SessionRunHook to Keras callbacks\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KZHPY55aFyXT"
+      },
+      "source": [
+        "In TensorFlow 1, to customize the behavior of training, you use `tf.estimator.SessionRunHook` with `tf.estimator.Estimator`. This guide demonstrates how to migrate from `SessionRunHook` to TensorFlow 2's custom callbacks with the `tf.keras.callbacks.Callback` API, which works with Keras `Model.fit` for training (as well as `Model.evaluate` and `Model.predict`). You will learn how to do this by implementing a `SessionRunHook` and a `Callback` task that measures examples per second during training.\n",
+        "\n",
+        "Examples of callbacks are checkpoint saving (`tf.keras.callbacks.ModelCheckpoint`) and [TensorBoard](`tf.keras.callbacks.TensorBoard`) summary writing. Keras [callbacks](https://www.tensorflow.org/guide/keras/custom_callback) are objects that are called at different points during training/evaluation/prediction in the built-in Keras `Model.fit`/`Model.evaluate`/`Model.predict` APIs. You can learn more about callbacks in the `tf.keras.callbacks.Callback` API docs, as well as the [Writing your own callbacks](https://www.tensorflow.org/guide/keras/custom_callback.ipynb/) and [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate) (the *Using callbacks* section) guides."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "29da56bf859d"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Start with imports and a simple dataset for demonstration purposes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "296d8b0DoKpV"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1\n",
+        "\n",
+        "import time\n",
+        "from datetime import datetime\n",
+        "from absl import flags"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xVGYtUXyXNuE"
+      },
+      "outputs": [],
+      "source": [
+        "features = [[1., 1.5], [2., 2.5], [3., 3.5]]\n",
+        "labels = [[0.3], [0.5], [0.7]]\n",
+        "eval_features = [[4., 4.5], [5., 5.5], [6., 6.5]]\n",
+        "eval_labels = [[0.8], [0.9], [1.]]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ON4zQifT0Vec"
+      },
+      "source": [
+        "## TensorFlow 1: Create a custom SessionRunHook with tf.estimator APIs\n",
+        "\n",
+        "The following TensorFlow 1 examples show how to set up a custom `SessionRunHook` that measures examples per second during training. After creating the hook (`LoggerHook`), pass it to the `hooks` parameter of `tf.estimator.Estimator.train`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S-myEclbXUL7"
+      },
+      "outputs": [],
+      "source": [
+        "def _input_fn():\n",
+        "  return tf1.data.Dataset.from_tensor_slices(\n",
+        "      (features, labels)).batch(1).repeat(100)\n",
+        "\n",
+        "def _model_fn(features, labels, mode):\n",
+        "  logits = tf1.layers.Dense(1)(features)\n",
+        "  loss = tf1.losses.mean_squared_error(labels=labels, predictions=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.05)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "  return tf1.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Xd9sPTkO0ZTD"
+      },
+      "outputs": [],
+      "source": [
+        "class LoggerHook(tf1.train.SessionRunHook):\n",
+        "  \"\"\"Logs loss and runtime.\"\"\"\n",
+        "\n",
+        "  def begin(self):\n",
+        "    self._step = -1\n",
+        "    self._start_time = time.time()\n",
+        "    self.log_frequency = 10\n",
+        "\n",
+        "  def before_run(self, run_context):\n",
+        "    self._step += 1\n",
+        "\n",
+        "  def after_run(self, run_context, run_values):\n",
+        "    if self._step % self.log_frequency == 0:\n",
+        "      current_time = time.time()\n",
+        "      duration = current_time - self._start_time\n",
+        "      self._start_time = current_time\n",
+        "      examples_per_sec = self.log_frequency / duration\n",
+        "      print('Time:', datetime.now(), ', Step #:', self._step,\n",
+        "            ', Examples per second:', examples_per_sec)\n",
+        "\n",
+        "estimator = tf1.estimator.Estimator(model_fn=_model_fn)\n",
+        "\n",
+        "# Begin training.\n",
+        "estimator.train(_input_fn, hooks=[LoggerHook()])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3uZCDMrM2CEg"
+      },
+      "source": [
+        "## TensorFlow 2: Create a custom Keras callback for Model.fit\n",
+        "\n",
+        "In TensorFlow 2, when you use the built-in Keras `Model.fit` (or `Model.evaluate`) for training/evaluation, you can configure a custom `tf.keras.callbacks.Callback`, which you then pass to the `callbacks` parameter of `Model.fit` (or `Model.evaluate`). (Learn more in the [Writing your own callbacks](../..guide/keras/custom_callback.ipynb) guide.)\n",
+        "\n",
+        "In the example below, you will write a custom `tf.keras.callbacks.Callback` that logs various metrics—it will measure examples per second, which should be comparable to the metrics in the previous `SessionRunHook` example."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UbMPoiB92KRG"
+      },
+      "outputs": [],
+      "source": [
+        "class CustomCallback(tf.keras.callbacks.Callback):\n",
+        "\n",
+        "    def on_train_begin(self, logs = None):\n",
+        "      self._step = -1\n",
+        "      self._start_time = time.time()\n",
+        "      self.log_frequency = 10\n",
+        "\n",
+        "    def on_train_batch_begin(self, batch, logs = None):\n",
+        "      self._step += 1\n",
+        "\n",
+        "    def on_train_batch_end(self, batch, logs = None):\n",
+        "      if self._step % self.log_frequency == 0:\n",
+        "        current_time = time.time()\n",
+        "        duration = current_time - self._start_time\n",
+        "        self._start_time = current_time\n",
+        "        examples_per_sec = self.log_frequency / duration\n",
+        "        print('Time:', datetime.now(), ', Step #:', self._step,\n",
+        "              ', Examples per second:', examples_per_sec)\n",
+        "\n",
+        "callback = CustomCallback()\n",
+        "\n",
+        "dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "    (features, labels)).batch(1).repeat(100)\n",
+        "\n",
+        "model = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])\n",
+        "optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+        "\n",
+        "model.compile(optimizer, \"mse\")\n",
+        "\n",
+        "# Begin training.\n",
+        "result = model.fit(dataset, callbacks=[callback], verbose = 0)\n",
+        "# Provide the results of training metrics.\n",
+        "result.history"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EFqFi21Ftskq"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "Learn more about callbacks in:\n",
+        "\n",
+        "- API docs: `tf.keras.callbacks.Callback`\n",
+        "- Guide: [Writing your own callbacks](../..guide/keras/custom_callback.ipynb/)\n",
+        "- Guide: [Training and evaluation with the built-in methods](https://www.tensorflow.org/guide/keras/train_and_evaluate) (the *Using callbacks* section)\n",
+        "\n",
+        "You may also find the following migration-related resources useful:\n",
+        "\n",
+        "- The [Early stopping migration guide](early_stopping.ipynb): `tf.keras.callbacks.EarlyStopping` is a built-in early stopping callback\n",
+        "- The [TensorBoard migration guide](tensorboard.ipynb): TensorBoard enables tracking and displaying metrics\n",
+        "- The [LoggingTensorHook and StopAtStepHook to Keras callbacks migration guide](logging_stop_hook.ipynb)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "sessionrunhook_callback.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/tensorboard.ipynb b/site/en/guide/migrate/tensorboard.ipynb
new file mode 100644
index 00000000000..ea0cd72b47e
--- /dev/null
+++ b/site/en/guide/migrate/tensorboard.ipynb
@@ -0,0 +1,291 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate TensorBoard: TensorFlow's visualization toolkit\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hIo_p2FWFIRx"
+      },
+      "source": [
+        "[TensorBoard](https://www.tensorflow.org/tensorboard) is a built-in tool for providing measurements and visualizations in TensorFlow. Common machine learning experiment metrics, such as accuracy and loss, can be tracked and displayed in TensorBoard. TensorBoard is compatible with TensorFlow 1 and 2 code.\n",
+        "\n",
+        "In TensorFlow 1, `tf.estimator.Estimator` saves summaries for TensorBoard by default. In comparison, in TensorFlow 2, summaries can be saved using a `tf.keras.callbacks.TensorBoard` callback.\n",
+        "\n",
+        "This guide demonstrates how to use TensorBoard, first, in TensorFlow 1 with Estimators, and then, how to carry out the equivalent process in TensorFlow 2."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f55c103999de"
+      },
+      "source": [
+        "### Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "X74yjOb-e18w"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow.compat.v1 as tf1\n",
+        "import tensorflow as tf\n",
+        "import tempfile\n",
+        "import numpy as np\n",
+        "import datetime\n",
+        "%load_ext tensorboard"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2r8r4d8FfMny"
+      },
+      "outputs": [],
+      "source": [
+        "mnist = tf.keras.datasets.mnist # The MNIST dataset.\n",
+        "\n",
+        "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
+        "x_train, x_test = x_train / 255.0, x_test / 255.0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wrqBkG4RFLP_"
+      },
+      "source": [
+        "### TensorFlow 1: TensorBoard with tf.estimator\n",
+        "\n",
+        "In this TensorFlow 1 example, you instantiate a `tf.estimator.DNNClassifier`, train and evaluate it on the MNIST dataset, and use TensorBoard to display the metrics:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "upA8nuf3FEq5"
+      },
+      "outputs": [],
+      "source": [
+        "%reload_ext tensorboard\n",
+        "\n",
+        "feature_columns = [tf1.feature_column.numeric_column(\"x\", shape=[28, 28])]\n",
+        "\n",
+        "config = tf1.estimator.RunConfig(save_summary_steps=1,\n",
+        "                                 save_checkpoints_steps=1)\n",
+        "\n",
+        "path = tempfile.mkdtemp()\n",
+        "\n",
+        "classifier = tf1.estimator.DNNClassifier(\n",
+        "    feature_columns=feature_columns,\n",
+        "    hidden_units=[256, 32],\n",
+        "    optimizer=tf1.train.AdamOptimizer(0.001),\n",
+        "    n_classes=10,\n",
+        "    dropout=0.1,\n",
+        "    model_dir=path,\n",
+        "    config = config\n",
+        ")\n",
+        "\n",
+        "train_input_fn = tf1.estimator.inputs.numpy_input_fn(\n",
+        "    x={\"x\": x_train},\n",
+        "    y=y_train.astype(np.int32),\n",
+        "    num_epochs=10,\n",
+        "    batch_size=50,\n",
+        "    shuffle=True,\n",
+        ")\n",
+        "\n",
+        "test_input_fn = tf1.estimator.inputs.numpy_input_fn(\n",
+        "    x={\"x\": x_test},\n",
+        "    y=y_test.astype(np.int32),\n",
+        "    num_epochs=10,\n",
+        "    shuffle=False\n",
+        ")\n",
+        "\n",
+        "train_spec = tf1.estimator.TrainSpec(input_fn=train_input_fn, max_steps=10)\n",
+        "eval_spec = tf1.estimator.EvalSpec(input_fn=test_input_fn,\n",
+        "                                   steps=10,\n",
+        "                                   throttle_secs=0)\n",
+        "\n",
+        "tf1.estimator.train_and_evaluate(estimator=classifier,\n",
+        "                                train_spec=train_spec,\n",
+        "                                eval_spec=eval_spec)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EBqO7JbR8bh2"
+      },
+      "outputs": [],
+      "source": [
+        "%tensorboard --logdir {classifier.model_dir}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GK8TK1CU88ns"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QvE_uxDJFUX-"
+      },
+      "source": [
+        "### TensorFlow 2: TensorBoard with a Keras callback and Model.fit\n",
+        "\n",
+        "In this TensorFlow 2 example, you create and store logs with the `tf.keras.callbacks.TensorBoard` callback, and train the model. The callback tracks the accuracy and loss per epoch. It is passed to `Model.fit` in the `callbacks` list."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9FLBhT2BFX2H"
+      },
+      "outputs": [],
+      "source": [
+        "%reload_ext tensorboard\n",
+        "\n",
+        "def create_model():\n",
+        "  return tf.keras.models.Sequential([\n",
+        "    tf.keras.layers.Flatten(input_shape=(28, 28), name='layers_flatten'),\n",
+        "    tf.keras.layers.Dense(512, activation='relu', name='layers_dense'),\n",
+        "    tf.keras.layers.Dropout(0.2, name='layers_dropout'),\n",
+        "    tf.keras.layers.Dense(10, activation='softmax', name='layers_dense_2')\n",
+        "  ])\n",
+        "\n",
+        "model = create_model()\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss='sparse_categorical_crossentropy',\n",
+        "              metrics=['accuracy'],\n",
+        "              steps_per_execution=10)\n",
+        "\n",
+        "log_dir = tempfile.mkdtemp()\n",
+        "tensorboard_callback = tf.keras.callbacks.TensorBoard(\n",
+        "  log_dir=log_dir,\n",
+        "  histogram_freq=1) # Enable histogram computation with each epoch.\n",
+        "\n",
+        "model.fit(x=x_train,\n",
+        "          y=y_train,\n",
+        "          epochs=10,\n",
+        "          validation_data=(x_test, y_test),\n",
+        "          callbacks=[tensorboard_callback])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ovPoLrCJ8t-R"
+      },
+      "outputs": [],
+      "source": [
+        "%tensorboard --logdir {tensorboard_callback.log_dir}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ip-IMGt_8xx9"
+      },
+      "source": [
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rQUS8nO9FZlH"
+      },
+      "source": [
+        "# Next steps\n",
+        "\n",
+        "- Learn more about TensorBoard in the [Get started](https://www.tensorflow.org/tensorboard/get_started) guide.\n",
+        "- For lower level APIs, refer to the [tf.summary migration to TensorFlow 2](https://www.tensorflow.org/tensorboard/migrate) guide."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tensorboard.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/tf1_vs_tf2.ipynb b/site/en/guide/migrate/tf1_vs_tf2.ipynb
new file mode 100644
index 00000000000..60791f72680
--- /dev/null
+++ b/site/en/guide/migrate/tf1_vs_tf2.ipynb
@@ -0,0 +1,1131 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "D70XgUYdLwI6"
+      },
+      "source": [
+        "# TensorFlow 1.x vs TensorFlow 2 - Behaviors and APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "akxmN3SQsEcb"
+      },
+      "source": [
+        "Under the hood, TensorFlow 2 follows a fundamentally different programming paradigm from TF1.x.\n",
+        "\n",
+        "This guide describes the fundamental differences between TF1.x and TF2 in terms of behaviors and the APIs, and how these all relate to your migration journey."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Xzy2mT87mwth"
+      },
+      "source": [
+        "## High-level summary of major changes\n",
+        "\n",
+        "Fundamentally, TF1.x and TF2 use a different set of runtime behaviors around execution (eager in TF2), variables, control flow, tensor shapes, and tensor equality comparisons. To be TF2 compatible, your code must be compatible with the full set of TF2 behaviors. During migration, you can enable or disable most of these behaviors individually via the `tf.compat.v1.enable_*` or `tf.compat.v1.disable_*` APIs. The one exception is the removal of collections, which is a side effect of enabling/disabling eager execution.\n",
+        "\n",
+        "At a high level, TensorFlow 2:\n",
+        "\n",
+        "* Removes [redundant APIs](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md).\n",
+        "* Makes APIs more consistent - for example, \n",
+        "[Unified RNNs](https://github.com/tensorflow/community/blob/master/rfcs/20180920-unify-rnn-interface.md) and \n",
+        "[Unified Optimizers](https://github.com/tensorflow/community/blob/master/rfcs/20181016-optimizer-unification.md).\n",
+        "* Prefers [functions over sessions](https://github.com/tensorflow/community/blob/master/rfcs/20180918-functions-not-sessions-20.md) and integrates better with the Python runtime with\n",
+        "[Eager execution](https://www.tensorflow.org/guide/eager) enabled by default along with `tf.function` that provides automatic control dependencies for graphs and compilation.\n",
+        "* Deprecates global graph [collections](https://github.com/tensorflow/community/blob/master/rfcs/20180905-deprecate-collections.md).\n",
+        "* Alters Variable concurrency semantics by using [`ResourceVariables` over `ReferenceVariables`](https://github.com/tensorflow/community/blob/master/rfcs/20180817-variables-20.md).\n",
+        "* Supports [function-based](https://github.com/tensorflow/community/blob/master/rfcs/20180507-cond-v2.md) and differentiable [control flow](https://github.com/tensorflow/community/blob/master/rfcs/20180821-differentiable-functional-while.md) (Control Flow v2).\n",
+        "* Simplifies the TensorShape API to hold `int`s instead of `tf.compat.v1.Dimension` objects.\n",
+        "* Updates tensor equality mechanics. In TF1.x the `==` operator on tensors and variables checks for object reference equality. In TF2 it checks for value equality. Additionally, tensors/variables are no longer hashable, but you can get hashable object references to them via `var.ref()` if you need to use them in sets or as `dict` keys.\n",
+        "\n",
+        "The sections below provide some more context on the differences between TF1.x and TF2. To learn more about the design process behind TF2, read the\n",
+        "[RFCs](https://github.com/tensorflow/community/pulls?utf8=%E2%9C%93&q=is%3Apr) and the [design docs](https://github.com/tensorflow/community/tree/master/rfcs)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dlCiIgEE2OhY"
+      },
+      "source": [
+        "## API cleanup\n",
+        "\n",
+        "Many APIs are either [gone or moved](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md) in TF2. Some of the major changes include removing `tf.app`, `tf.flags`, and `tf.logging` in favor of the now open-source [absl-py](https://github.com/abseil/abseil-py), rehoming projects that lived in `tf.contrib`, and cleaning up the main `tf.*` namespace by moving lesser used functions into subpackages like `tf.math`. Some APIs have been replaced with their TF2 equivalents - `tf.summary`, `tf.keras.metrics`, and\n",
+        "`tf.keras.optimizers`.\n",
+        "\n",
+        "### `tf.compat.v1`: Legacy and Compatibility API Endpoints\n",
+        "\n",
+        "Symbols under the `tf.compat` and `tf.compat.v1` namespaces are not considered TF2 APIs. These namespaces expose a mix of compatibility symbols, as well as legacy API endpoints from TF 1.x. These are intended to aid migration from TF1.x to TF2. However, as none of these `compat.v1` APIs are idiomatic TF2 APIs, do not use them for writing brand-new TF2 code.\n",
+        "\n",
+        "Individual `tf.compat.v1` symbols may be TF2 compatible because they continue to work even with TF2 behaviors enabled (such as `tf.compat.v1.losses.mean_squared_error`), while others are incompatible with TF2 (such as `tf.compat.v1.metrics.accuracy`). Many `compat.v1` symbols (though not all) contain dedicated migration information in their documentation that explains their degree of compatibility with TF2 behaviors, as well as how to migrate them to TF2 APIs.\n",
+        "\n",
+        "The [TF2 upgrade script](https://www.tensorflow.org/guide/migrate/upgrade) can map many `compat.v1` API symbols to equivalent TF2 APIs in the case where they are aliases or have the same arguments but with a different ordering. You can also use the upgrade script to automatically rename TF1.x APIs.\n",
+        "\n",
+        "### False friend APIs\n",
+        "\n",
+        "There are a set of \"false-friend\" symbols found in the TF2 `tf` namespace (not under `compat.v1`) that actually ignore TF2 behaviors under-the-hood, and/or are not fully compatible with the full set of TF2 behaviors. As such, these APIs are likely to misbehave with TF2 code, potentially in silent ways.\n",
+        "\n",
+        "* `tf.estimator.*`: Estimators create and use graphs and sessions under the hood. As such, these should not be considered TF2-compatible. If your code is running estimators, it is not using TF2 behaviors.\n",
+        "* `keras.Model.model_to_estimator(...)`: This creates an Estimator under the hood, which as mentioned above is not TF2-compatible.\n",
+        "* `tf.Graph().as_default()`: This enters TF1.x graph behaviors and does not follow standard TF2-compatible `tf.function` behaviors. Code that enters graphs like this will generally run them via Sessions, and should not be considered TF2-compatible.\n",
+        "* `tf.feature_column.*` The feature column APIs generally rely on TF1-style `tf.compat.v1.get_variable` variable creation and assume that the created variables will be accessed via global collections. As TF2 does not support collections, APIs may not work correctly when running them with TF2 behaviors enabled.\n",
+        "\n",
+        "### Other API changes\n",
+        "\n",
+        "* TF2 features significant improvements to the device placement algorithms which renders the usage of `tf.colocate_with` unnecessary. If removing it causes a performance degradation, [please file a bug](https://github.com/tensorflow/tensorflow/issues).\n",
+        "\n",
+        "* Replace all usage of `tf.v1.ConfigProto` with equivalent functions from `tf.config`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RxEU79Rd83Yz"
+      },
+      "source": [
+        "## Eager execution\n",
+        "\n",
+        "TF1.x required you to manually stitch together an [abstract syntax tree](https://en.wikipedia.org/wiki/Abstract_syntax_tree) (the graph) by making `tf.*` API calls and then manually compile the abstract syntax tree by passing a set of output tensors and input tensors to a `session.run` call. TF2 executes eagerly (like Python normally does) and makes graphs and sessions feel like implementation details.\n",
+        "\n",
+        "One notable byproduct of eager execution is that `tf.control_dependencies` is no\n",
+        "longer required, as all lines of code execute in order (within a `tf.function`,\n",
+        "code with side effects executes in the order written)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LH3YizX-9S7g"
+      },
+      "source": [
+        "## No more globals\n",
+        "\n",
+        "TF1.x relied heavily on implicit global namespaces and collections. When you call `tf.Variable`, it would be put into a collection in the default graph, and it would remain there, even if you lost track of the Python variable pointing to it. You could then recover that `tf.Variable`, but only if you knew the name that it had been created with. This was difficult to do if you were not in control of the variable's creation. As a result, all sorts of mechanisms proliferated to\n",
+        "attempt to help you find your variables again, and for frameworks to find\n",
+        "user-created variables. Some of these include: variable scopes, global collections, helper methods like `tf.get_global_step` and `tf.global_variables_initializer`, optimizers implicitly\n",
+        "computing gradients over all trainable variables, and so on. TF2 eliminates all of these mechanisms ([Variables 2.0 RFC](https://github.com/tensorflow/community/pull/11)) in favor of the default mechanism - you keep track of your variables. If you lose track of a `tf.Variable`, it gets garbage collected.\n",
+        "\n",
+        "The requirement to track variables creates some extra work, but with tools like the [modeling shims](./model_mapping.ipynb) and behaviors like [implicit object-oriented variable collections in `tf.Module`s and `tf.keras.layers.Layer`s](https://www.tensorflow.org/guide/intro_to_modules), the burden is minimized."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NXwBgAjJ98J2"
+      },
+      "source": [
+        "## Functions, not sessions\n",
+        "\n",
+        "A `session.run` call is almost like a function call: you specify the inputs and\n",
+        "the function to be called, and you get back a set of outputs. In TF2, you can decorate a Python function using `tf.function` to mark it for JIT compilation so that TensorFlow runs it as a single graph ([Functions 2.0 RFC](https://github.com/tensorflow/community/pull/20)). This mechanism allows TF2 to gain all of the benefits of graph mode:\n",
+        "\n",
+        "-   Performance: The function can be optimized (node pruning, kernel fusion,\n",
+        "    etc.)\n",
+        "-   Portability: The function can be exported/reimported\n",
+        "    ([SavedModel 2.0 RFC](https://github.com/tensorflow/community/pull/34)),\n",
+        "    allowing you to reuse and share modular TensorFlow functions.\n",
+        "\n",
+        "```python\n",
+        "# TF1.x\n",
+        "outputs = session.run(f(placeholder), feed_dict={placeholder: input})\n",
+        "# TF2\n",
+        "outputs = f(input)\n",
+        "```\n",
+        "\n",
+        "With the power to freely intersperse Python and TensorFlow code, you can take\n",
+        "advantage of Python's expressiveness. However, portable TensorFlow executes in\n",
+        "contexts without a Python interpreter, such as mobile, C++, and JavaScript. To\n",
+        "help avoid rewriting your code when adding `tf.function`, use [AutoGraph](https://tensorflow.org/guide/function) to convert a subset of Python constructs\n",
+        "into their TensorFlow equivalents:\n",
+        "\n",
+        "*   `for`/`while` -> `tf.while_loop` (`break` and `continue` are supported)\n",
+        "*   `if` -> `tf.cond`\n",
+        "*   `for _ in dataset` -> `dataset.reduce`\n",
+        "\n",
+        "AutoGraph supports arbitrary nestings of control flow, which makes it possible\n",
+        "to performantly and concisely implement many complex ML programs such as\n",
+        "sequence models, reinforcement learning, custom training loops, and more."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Mj3gaj4tpi7O"
+      },
+      "source": [
+        "## Adapting to TF 2.x Behavior Changes\n",
+        "\n",
+        "Your migration to TF2 is only complete once you have migrated to the full set of TF2 behaviors. The full set of behaviors can be enabled or disabled via `tf.compat.v1.enable_v2_behaviors` and `tf.compat.v1.disable_v2_behaviors`. The sections below discuss each major behavior change in detail."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_M0zEtR9p0XD"
+      },
+      "source": [
+        "### Using `tf.function`s\n",
+        "\n",
+        "The largest changes to your programs during migration are likely to come from the fundamental programming model paradigm shift from graphs and sessions to eager execution and `tf.function`. Refer to the [TF2 migration guides](https://tensorflow.org/guide/migrate) to learn more about moving from APIs that are incompatible with eager execution and `tf.function` to APIs that are compatible with them.\n",
+        "\n",
+        "Note: During migration you may choose to directly enable and disable eager execution with `tf.compat.v1.enable_eager_execution` and `tf.compat.v1.disable_eager_execution`, but this may only be done once during the lifetime of your program.\n",
+        "\n",
+        "Below are some common program patterns not tied to any one API that may cause problems when switching from `tf.Graph`s and `tf.compat.v1.Session`s to eager execution with `tf.function`s."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UgwEtwwN2PWy"
+      },
+      "source": [
+        "#### Pattern 1: Python object manipulation and variable creation intended to be done only once get run multiple times\n",
+        "\n",
+        "\n",
+        "In TF1.x programs that rely on graphs and sessions, the expectation is usually that all Python logic in your program will only run once. However, with eager execution and `tf.function` it is fair to expect that your Python logic will be run at least once, but possibly more times (either multiple times eagerly, or multiple times across different `tf.function` traces). Sometimes, `tf.function` will even trace twice on the same input, causing unexpected behaviors  (see Example 1 and 2). Refer to the `tf.function` [guide](https://www.tensorflow.org/guide/function) for more details.\n",
+        "\n",
+        "Note: This pattern usually causes your code to silently misbehave when executing eagerly without `tf.function`s, but generally raises an `InaccessibleTensorError` or a `ValueError` when attempting to wrap the problematic code inside of a `tf.function`. To discover and debug this issue, it is recommended you wrap your code with `tf.function` early on, and use [pdb](https://docs.python.org/3/library/pdb.html) or interactive debugging to identify the source of the `InaccessibleTensorError`.\n",
+        "\n",
+        "**Example 1: Variable creation**\n",
+        "\n",
+        "Consider the example below, where the function creates a variable when called:\n",
+        "\n",
+        "```python\n",
+        "def f():\n",
+        "  v = tf.Variable(1.0)\n",
+        "  return v\n",
+        "\n",
+        "with tf.Graph().as_default():\n",
+        "  with tf.compat.v1.Session() as sess:\n",
+        "    res = f()\n",
+        "    sess.run(tf.compat.v1.global_variables_initializer())\n",
+        "    sess.run(res)\n",
+        "```\n",
+        "\n",
+        "However, naively wrapping the above function that contains variable creation with `tf.function` is not allowed. `tf.function` only supports [singleton variable creations on the first call](https://www.tensorflow.org/guide/function#creating_tfvariables). To enforce this, when tf.function detects variable creation in the first call, it will attempt to trace again and raise an error if there is variable creation in the second trace.\n",
+        "\n",
+        "```python\n",
+        "@tf.function\n",
+        "def f():\n",
+        "  print(\"trace\") # This will print twice because the python body is run twice\n",
+        "  v = tf.Variable(1.0)\n",
+        "  return v\n",
+        "\n",
+        "try:\n",
+        "  f()\n",
+        "except ValueError as e:\n",
+        "  print(e)\n",
+        "```\n",
+        "\n",
+        "A workaround is caching and reusing the variable after it is created in the first call.\n",
+        "\n",
+        "```python\n",
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.v = None\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    print(\"trace\") # This will print twice because the python body is run twice\n",
+        "    if self.v is None:\n",
+        "      self.v = tf.Variable(0)\n",
+        "    return self.v\n",
+        "\n",
+        "m = Model()\n",
+        "m()\n",
+        "```\n",
+        "\n",
+        "**Example 2: Out-of-scope Tensors due to `tf.function` retracing**\n",
+        "\n",
+        "As demonstrated in Example 1, `tf.function` will retrace when it detects Variable creation in the first call. This can cause extra confusion, because the two tracings will create two graphs. When the second graph from retracing attempts to access a Tensor from the graph generated during the first tracing, Tensorflow will raise an error complaining that the Tensor is out of scope. To demonstrate the scenario, the code below creates a dataset on the first `tf.function` call. This would run as expected.\n",
+        "\n",
+        "```python\n",
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.dataset = None\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    print(\"trace\") # This will print once: only traced once\n",
+        "    if self.dataset is None:\n",
+        "      self.dataset = tf.data.Dataset.from_tensors([1, 2, 3])\n",
+        "    it = iter(self.dataset)\n",
+        "    return next(it)\n",
+        "\n",
+        "m = Model()\n",
+        "m()\n",
+        "```\n",
+        "\n",
+        "However, if we also attempt to create a variable on the first `tf.function` call, the code will raise an error complaining that the dataset is out of scope. This is because the dataset is in the first graph, while the second graph is also attempting to access it.\n",
+        "\n",
+        "```python\n",
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.v = None\n",
+        "    self.dataset = None\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    print(\"trace\") # This will print twice because the python body is run twice\n",
+        "    if self.v is None:\n",
+        "      self.v = tf.Variable(0)\n",
+        "    if self.dataset is None:\n",
+        "      self.dataset = tf.data.Dataset.from_tensors([1, 2, 3])\n",
+        "    it = iter(self.dataset)\n",
+        "    return [self.v, next(it)]\n",
+        "\n",
+        "m = Model()\n",
+        "try:\n",
+        "  m()\n",
+        "except TypeError as e:\n",
+        "  print(e) #  is out of scope and cannot be used here.\n",
+        "```\n",
+        "\n",
+        "The most straightforward solution is ensuring that the variable creation and dataset creation are both outside of the `tf.function` call. For example:\n",
+        "\n",
+        "```python\n",
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.v = None\n",
+        "    self.dataset = None\n",
+        "\n",
+        "  def initialize(self):\n",
+        "    if self.dataset is None:\n",
+        "      self.dataset = tf.data.Dataset.from_tensors([1, 2, 3])\n",
+        "    if self.v is None:\n",
+        "      self.v = tf.Variable(0)\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    it = iter(self.dataset)\n",
+        "    return [self.v, next(it)]\n",
+        "\n",
+        "m = Model()\n",
+        "m.initialize()\n",
+        "m()\n",
+        "```\n",
+        "\n",
+        "However, sometimes it's not avoidable to create variables in `tf.function` (such as slot variables in some [TF keras optimizers](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer#slots)). Still, we can simply move the dataset creation outside of the `tf.function` call. The reason that we can rely on this is because `tf.function` will receive the dataset as an implicit input and both graphs can access it properly.\n",
+        "\n",
+        "```python\n",
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.v = None\n",
+        "    self.dataset = None\n",
+        "\n",
+        "  def initialize(self):\n",
+        "    if self.dataset is None:\n",
+        "      self.dataset = tf.data.Dataset.from_tensors([1, 2, 3])\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    if self.v is None:\n",
+        "      self.v = tf.Variable(0)\n",
+        "    it = iter(self.dataset)\n",
+        "    return [self.v, next(it)]\n",
+        "\n",
+        "m = Model()\n",
+        "m.initialize()\n",
+        "m()\n",
+        "```\n",
+        "\n",
+        "**Example 3: Unexpected Tensorflow object re-creations due to dict usage**\n",
+        "\n",
+        "`tf.function` has very poor support for python side effects such as appending to a list, or checking/adding to a dictionary. More details are in [\"Better performance with tf.function\"](https://www.tensorflow.org/guide/function#executing_python_side_effects). In the example below, the code uses dictionaries to cache datasets and iterators. For the same key, each call to the model will return the same iterator of the dataset.\n",
+        "\n",
+        "```python\n",
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.datasets = {}\n",
+        "    self.iterators = {}\n",
+        "\n",
+        "  def __call__(self, key):\n",
+        "    if key not in self.datasets:\n",
+        "      self.datasets[key] = tf.compat.v1.data.Dataset.from_tensor_slices([1, 2, 3])\n",
+        "      self.iterators[key] = self.datasets[key].make_initializable_iterator()\n",
+        "    return self.iterators[key]\n",
+        "\n",
+        "with tf.Graph().as_default():\n",
+        "  with tf.compat.v1.Session() as sess:\n",
+        "    m = Model()\n",
+        "    it = m('a')\n",
+        "    sess.run(it.initializer)\n",
+        "    for _ in range(3):\n",
+        "      print(sess.run(it.get_next())) # prints 1, 2, 3\n",
+        "```\n",
+        "\n",
+        "However, the pattern above will not work as expected in `tf.function`. During tracing, `tf.function` will ignore the python side effect of addition to the dictionaries. Instead, it only remembers the creation of a new dataset and iterator. As a result, each call to the model will always return a new iterator. This issue is hard to notice unless the numerical results or performance are significant enough. Hence, we recommend users to think about the code carefully before wrapping `tf.function` naively onto the python code.\n",
+        "\n",
+        "```python\n",
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.datasets = {}\n",
+        "    self.iterators = {}\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self, key):\n",
+        "    if key not in self.datasets:\n",
+        "      self.datasets[key] = tf.data.Dataset.from_tensor_slices([1, 2, 3])\n",
+        "      self.iterators[key] = iter(self.datasets[key])\n",
+        "    return self.iterators[key]\n",
+        "\n",
+        "m = Model()\n",
+        "for _ in range(3):\n",
+        "  print(next(m('a'))) # prints 1, 1, 1\n",
+        "```\n",
+        "\n",
+        "We can use [`tf.init_scope`](https://www.tensorflow.org/api_docs/python/tf/init_scope) to lift the dataset and iterator creation outside of the graph, to achieve the expected behavior:\n",
+        "\n",
+        "```python\n",
+        "class Model(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    self.datasets = {}\n",
+        "    self.iterators = {}\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self, key):\n",
+        "    if key not in self.datasets:\n",
+        "      # Lifts ops out of function-building graphs\n",
+        "      with tf.init_scope():\n",
+        "        self.datasets[key] = tf.data.Dataset.from_tensor_slices([1, 2, 3])\n",
+        "        self.iterators[key] = iter(self.datasets[key])\n",
+        "    return self.iterators[key]\n",
+        "\n",
+        "m = Model()\n",
+        "for _ in range(3):\n",
+        "  print(next(m('a'))) # prints 1, 2, 3\n",
+        "```\n",
+        "\n",
+        "The general rule of thumb is to avoid relying on Python side effects in your logic and only use them to debug your traces.\n",
+        "\n",
+        "**Example 4: Manipulating a global Python list**\n",
+        "\n",
+        "The following TF1.x code uses a global list of losses that it uses to only maintain the list of losses generated by the current training step. Note that the Python logic that appends losses to the list will only be called once regardless of how many training steps the session is run for.\n",
+        "\n",
+        "```python\n",
+        "all_losses = []\n",
+        "\n",
+        "class Model():\n",
+        "  def __call__(...):\n",
+        "    ...\n",
+        "    all_losses.append(regularization_loss)\n",
+        "    all_losses.append(label_loss_a)\n",
+        "    all_losses.append(label_loss_b)\n",
+        "    ...\n",
+        "\n",
+        "g = tf.Graph()\n",
+        "with g.as_default():\n",
+        "  ...\n",
+        "  # initialize all objects\n",
+        "  model = Model()\n",
+        "  optimizer = ...\n",
+        "  ...\n",
+        "  # train step\n",
+        "  model(...)\n",
+        "  total_loss = tf.reduce_sum(all_losses)\n",
+        "  optimizer.minimize(total_loss)\n",
+        "  ...\n",
+        "...\n",
+        "sess = tf.compat.v1.Session(graph=g)\n",
+        "sess.run(...)  \n",
+        "```\n",
+        "\n",
+        "However, if this Python logic is naively mapped to TF2 with eager execution, the global list of losses will have new values appended to it in each training step. This means the training step code which previously expected the list to only contain losses from the current training step now actually sees the list of losses from all training steps run so far. This is an unintended behavior change, and the list will either need to be cleared at the start of each step or made local to the training step.\n",
+        "\n",
+        "```python\n",
+        "all_losses = []\n",
+        "\n",
+        "class Model():\n",
+        "  def __call__(...):\n",
+        "    ...\n",
+        "    all_losses.append(regularization_loss)\n",
+        "    all_losses.append(label_loss_a)\n",
+        "    all_losses.append(label_loss_b)\n",
+        "    ...\n",
+        "\n",
+        "# initialize all objects\n",
+        "model = Model()\n",
+        "optimizer = ...\n",
+        "\n",
+        "def train_step(...)\n",
+        "  ...\n",
+        "  model(...)\n",
+        "  total_loss = tf.reduce_sum(all_losses) # global list is never cleared,\n",
+        "  # Accidentally accumulates sum loss across all training steps\n",
+        "  optimizer.minimize(total_loss)\n",
+        "  ...\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qaYnjPo-tmTI"
+      },
+      "source": [
+        "#### Pattern 2: A symbolic tensor meant to be recomputed every step in TF1.x is accidentally cached with the initial value when switching to eager.\n",
+        "\n",
+        "\n",
+        "This pattern usually causes your code to silently misbehave when executing eagerly outside of tf.functions, but raises an `InaccessibleTensorError` if the initial value caching occurs inside of a `tf.function`. However, be aware that in order to avoid [Pattern 1](#pattern-1) above you will often inadvertently structure your code in such a way that this initial value caching will happen *outside* of any `tf.function` that would be able to raise an error. So, take extra care if you know your program may be susceptible to this pattern.\n",
+        "\n",
+        "The general solution to this pattern is to restructure the code or use Python callables if necessary to make sure the value is recomputed each time instead of being accidentally cached.\n",
+        "\n",
+        "**Example 1: Learning rate/hyperparameter/etc. schedules that depend on global step**\n",
+        "\n",
+        "In the following code snippet, the expectation is that every time the session is run the most recent `global_step` value will be read and a new learning rate will be computed.\n",
+        "```python\n",
+        "g = tf.Graph()\n",
+        "with g.as_default():\n",
+        "  ...\n",
+        "  global_step = tf.Variable(0)\n",
+        "  learning_rate = 1.0 / global_step\n",
+        "  opt = tf.compat.v1.train.GradientDescentOptimizer(learning_rate)\n",
+        "  ...\n",
+        "  global_step.assign_add(1)\n",
+        "...\n",
+        "sess = tf.compat.v1.Session(graph=g)\n",
+        "sess.run(...)\n",
+        "```\n",
+        "\n",
+        "However, when trying to switch to eager, be wary of ending up with the learning rate only being computed once then reused, rather than following the intended schedule:\n",
+        "```python\n",
+        "global_step = tf.Variable(0)\n",
+        "learning_rate = 1.0 / global_step # Wrong! Only computed once!\n",
+        "opt = tf.keras.optimizers.SGD(learning_rate)\n",
+        "\n",
+        "def train_step(...):\n",
+        "  ...\n",
+        "  opt.apply_gradients(...)\n",
+        "  global_step.assign_add(1)\n",
+        "  ...\n",
+        "```\n",
+        "\n",
+        "Because this specific example is a common pattern and optimizers should only be initialized once rather than at each training step, TF2 optimizers support `tf.keras.optimizers.schedules.LearningRateSchedule` schedules or Python callables as arguments for the learning rate and other hyperparameters.\n",
+        "\n",
+        "**Example 2: Symbolic random number initializations assigned as object attributes then reused via pointer are accidentally cached when switching to eager**\n",
+        "\n",
+        "Consider the following `NoiseAdder` module:\n",
+        "\n",
+        "```python\n",
+        "class NoiseAdder(tf.Module):\n",
+        "  def __init__(shape, mean):\n",
+        "    self.noise_distribution = tf.random.normal(shape=shape, mean=mean)\n",
+        "    self.trainable_scale = tf.Variable(1.0, trainable=True)\n",
+        "  \n",
+        "  def add_noise(input):\n",
+        "    return (self.noise_distribution + input) * self.trainable_scale\n",
+        "```\n",
+        "\n",
+        "Using it as follows in TF1.x will compute a new random noise tensor every time the session is run:\n",
+        "```python\n",
+        "g = tf.Graph()\n",
+        "with g.as_default():\n",
+        "  ...\n",
+        "  # initialize all variable-containing objects\n",
+        "  noise_adder = NoiseAdder(shape, mean)\n",
+        "  ...\n",
+        "  # computation pass\n",
+        "  x_with_noise = noise_adder.add_noise(x)\n",
+        "  ...\n",
+        "...\n",
+        "sess = tf.compat.v1.Session(graph=g)\n",
+        "sess.run(...)\n",
+        "```\n",
+        "\n",
+        "However, in TF2 initializing the `noise_adder` at the beginning will cause the `noise_distribution` to be only computed once and get frozen for all training steps:\n",
+        "```python\n",
+        "...\n",
+        "# initialize all variable-containing objects\n",
+        "noise_adder = NoiseAdder(shape, mean) # Freezes `self.noise_distribution`!\n",
+        "...\n",
+        "# computation pass\n",
+        "x_with_noise = noise_adder.add_noise(x)\n",
+        "...\n",
+        "```\n",
+        "\n",
+        "To fix this, refactor `NoiseAdder` to call `tf.random.normal` every time a new random tensor is needed, instead of referring to the same tensor object each time.\n",
+        "\n",
+        "```python\n",
+        "class NoiseAdder(tf.Module):\n",
+        "  def __init__(shape, mean):\n",
+        "    self.noise_distribution = lambda: tf.random.normal(shape=shape, mean=mean)\n",
+        "    self.trainable_scale = tf.Variable(1.0, trainable=True)\n",
+        "  \n",
+        "  def add_noise(input):\n",
+        "    return (self.noise_distribution() + input) * self.trainable_scale\n",
+        "```"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "j2PXkSflCaCl"
+      },
+      "source": [
+        "#### Pattern 3: TF1.x code directly relies on and looks up tensors by name\n",
+        "\n",
+        "\n",
+        "It is common for TF1.x code tests to rely on checking what tensors or operations are present in a graph. In some rare cases, modeling code will also rely on these lookups by name.\n",
+        "\n",
+        "Tensor names are not generated when executing eagerly outside of `tf.function` at all, so all usages of `tf.Tensor.name` must happen inside of a `tf.function`. Keep in mind the actual generated names are very likely to differ between TF1.x and TF2 even within the same `tf.function`, and API guarantees do not ensure stability of the generated names across TF versions.\n",
+        "\n",
+        "Note: Variable names are still generated even outside of `tf.function`s, but their names also are not guaranteed to match between TF1.x and TF2 except when following the relevant sections of the [model mapping guide](./model_mapping.ipynb).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5NB3bycl5Lde"
+      },
+      "source": [
+        "#### Pattern 4: TF1.x session selectively runs only part of the generated graph\n",
+        "\n",
+        "\n",
+        "In TF1.x, you can construct a graph and then choose to only selectively run only a subset of it with a session by choosing a set of inputs and outputs that do not require running every op in the graph.\n",
+        "\n",
+        "For example, you may have both a generator and a discriminator inside of a single graph, and use separate `tf.compat.v1.Session.run` calls to alternate between only training the discriminator or only training the generator.\n",
+        "\n",
+        "In TF2, due to automatic control dependencies in `tf.function` and eager execution, there is no selective pruning of `tf.function` traces. A full graph containing all variable updates would get run even if, for example, only the output of the discriminator or the generator is output from the `tf.function`.\n",
+        "\n",
+        "So, you would need to either use multiple `tf.function`s containing different parts of the program, or a conditional argument to the `tf.function` that you branch on so as to execute only the things you actually want to have run."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CnNaUmROp5fV"
+      },
+      "source": [
+        "### Collections Removal\n",
+        "\n",
+        "When eager execution is enabled, graph collection-related `compat.v1` APIs (including those that read or write to collections under the hood such as `tf.compat.v1.trainable_variables`) are no longer available. Some may raise `ValueError`s, while others may silently return empty lists.\n",
+        "\n",
+        "The most standard usage of collections in TF1.x is to maintain initializers, the global step, weights, regularization losses, model output losses, and variable updates that need to be run such as from `BatchNormalization` layers.\n",
+        "\n",
+        "To handle each of these standard usages:\n",
+        "1. Initializers - Ignore. Manual variable initialization is not required with eager execution enabled.\n",
+        "2. Global step - See the documentation of `tf.compat.v1.train.get_or_create_global_step` for migration instructions.\n",
+        "3. Weights - Map your models to `tf.Module`s/`tf.keras.layers.Layer`s/`tf.keras.Model`s by following the guidance in the [model mapping guide](./model_mapping.ipynb) and then use their respective weight-tracking mechanisms such as `tf.module.trainable_variables`.\n",
+        "4. Regularization losses - Map your models to `tf.Module`s/`tf.keras.layers.Layer`s/`tf.keras.Model`s by following the guidance in the [model mapping guide](./model_mapping.ipynb) and then use `tf.keras.losses`. Alternatively, you can also manually track your regularization losses.\n",
+        "5. Model output losses - Use `tf.keras.Model` loss management mechanisms or separately track your losses without using collections.\n",
+        "6. Weight updates - Ignore this collection. Eager execution and `tf.function` (with autograph and auto-control-dependencies) means all variable updates will get run automatically. So, you will not have to explicitly run all weight updates at the end, but note that it means the weight updates may happen at a different time than they did in your TF1.x code, depending on how you were using control dependencies.\n",
+        "7. Summaries - Refer to the [migrating summary API guide](https://www.tensorflow.org/tensorboard/migrate).\n",
+        "\n",
+        "More complex collections usage (such as using custom collections) may require you to refactor your code to either maintain your own global stores, or to make it not rely on global stores at all."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8J_ckZstp8y1"
+      },
+      "source": [
+        "### `ResourceVariables` instead of `ReferenceVariables`\n",
+        "\n",
+        "`ResourceVariables` have stronger read-write consistency guarantees than `ReferenceVariables`. This leads to more predictable, easier-to-reason semantics about whether or not you will observe the result of a previous write when using your variables. This change is extremely unlikely to cause existing code to raise errors or to break silently.\n",
+        "\n",
+        "However, it is ***possible though unlikely*** that these stronger consistency guarantees may increase the memory usage of your specific program. Please file an [issue](https://github.com/tensorflow/tensorflow/issues) if you find this to be the case. Additionally, if you have unit tests relying on exact string comparisons against the operator names in a graph corresponding to variable reads, be aware that enabling resource variables may slightly change the name of these operators.\n",
+        "\n",
+        "To isolate the impact of this behavior change on your code, if eager execution is disabled you can use `tf.compat.v1.disable_resource_variables()` and `tf.compat.v1.enable_resource_variables()` to globally disable or enable this behavior change. `ResourceVariables` will always be used if eager execution is enabled.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FTU-4P1vux0e"
+      },
+      "source": [
+        "### Control flow v2\n",
+        "\n",
+        "In TF1.x, control flow ops such as `tf.cond` and `tf.while_loop` inline low-level ops such as `Switch`, `Merge` etc. TF2 provides improved functional control flow ops that are implemented with separate `tf.function` traces for every branch and support higher-order differentiation.\n",
+        "\n",
+        "To isolate the impact of this behavior change on your code, if eager execution is disabled you can use `tf.compat.v1.disable_control_flow_v2()` and `tf.compat.v1.enable_control_flow_v2()` to globally disable or enable this behavior change. However, you can only disable control flow v2 if eager execution is also disabled. If it is enabled, control flow v2 will always be used.\n",
+        "\n",
+        "This behavior change can dramatically change the structure of generated TF programs that use control flow, as they will contain several nested function traces rather than one flat graph. So, any code that is highly dependent on the exact semantics of produced traces may require some modification. This includes:\n",
+        "* Code relying on operator and tensor names\n",
+        "* Code referring to tensors created within a TensorFlow control flow branch from outside of that branch. This is likely to produce an `InaccessibleTensorError`\n",
+        "\n",
+        "This behavior change is intended to be performance neutral to positive, but if you run into an issue where control flow v2 performs worse for you than TF1.x control flow then please file an [issue](https://github.com/tensorflow/tensorflow/issues) with reproduction steps. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "W7VwgVCGqE9S"
+      },
+      "source": [
+        "## TensorShape API behavior changes\n",
+        "\n",
+        "The `TensorShape` class was simplified to hold `int`s, instead of `tf.compat.v1.Dimension` objects. So there is no need to call `.value` to get an `int`.\n",
+        "\n",
+        "Individual `tf.compat.v1.Dimension` objects are still accessible from `tf.TensorShape.dims`.\n",
+        "\n",
+        "To isolate the impact of this behavior change on your code, you can use `tf.compat.v1.disable_v2_tensorshape()` and `tf.compat.v1.enable_v2_tensorshape()` to globally disable or enable this behavior change."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x36cWcmM8Eu1"
+      },
+      "source": [
+        "The following demonstrate the differences between TF1.x and TF2."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QF4un9UpVTRA"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PbpD-kHOZR4A"
+      },
+      "outputs": [],
+      "source": [
+        "# Create a shape and choose an index\n",
+        "i = 0\n",
+        "shape = tf.TensorShape([16, None, 256])\n",
+        "shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kDFck03neNy0"
+      },
+      "source": [
+        "If you had this in TF1.x:\n",
+        "\n",
+        "```python\n",
+        "value = shape[i].value\n",
+        "```\n",
+        "\n",
+        "Then do this in TF2:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KuR73QGEeNdH"
+      },
+      "outputs": [],
+      "source": [
+        "value = shape[i]\n",
+        "value"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bPWPNKRiZmkd"
+      },
+      "source": [
+        "If you had this in TF1.x:\n",
+        "\n",
+        "```python\n",
+        "for dim in shape:\n",
+        "    value = dim.value\n",
+        "    print(value)\n",
+        "```\n",
+        "\n",
+        "Then, do this in TF2:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "y6s0vuuprJfc"
+      },
+      "outputs": [],
+      "source": [
+        "for value in shape:\n",
+        "  print(value)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YpRgngu3Zw-A"
+      },
+      "source": [
+        "If you had this in TF1.x (or used any other dimension method):\n",
+        "\n",
+        "```python\n",
+        "dim = shape[i]\n",
+        "dim.assert_is_compatible_with(other_dim)\n",
+        "```\n",
+        "\n",
+        "Then do this in TF2:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LpViGEcUZDGX"
+      },
+      "outputs": [],
+      "source": [
+        "other_dim = 16\n",
+        "Dimension = tf.compat.v1.Dimension\n",
+        "\n",
+        "if shape.rank is None:\n",
+        "  dim = Dimension(None)\n",
+        "else:\n",
+        "  dim = shape.dims[i]\n",
+        "dim.is_compatible_with(other_dim) # or any other dimension method"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GaiGe36dOdZ_"
+      },
+      "outputs": [],
+      "source": [
+        "shape = tf.TensorShape(None)\n",
+        "\n",
+        "if shape:\n",
+        "  dim = shape.dims[i]\n",
+        "  dim.is_compatible_with(other_dim) # or any other dimension method"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3kLLY0I3PI-l"
+      },
+      "source": [
+        "The boolean value of a `tf.TensorShape` is `True` if the rank is known, `False` otherwise."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-Ow1ndKpOnJd"
+      },
+      "outputs": [],
+      "source": [
+        "print(bool(tf.TensorShape([])))      # Scalar\n",
+        "print(bool(tf.TensorShape([0])))     # 0-length vector\n",
+        "print(bool(tf.TensorShape([1])))     # 1-length vector\n",
+        "print(bool(tf.TensorShape([None])))  # Unknown-length vector\n",
+        "print(bool(tf.TensorShape([1, 10, 100])))       # 3D tensor\n",
+        "print(bool(tf.TensorShape([None, None, None]))) # 3D tensor with no known dimensions\n",
+        "print()\n",
+        "print(bool(tf.TensorShape(None)))  # A tensor with unknown rank."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KvfEd-uSsWqN"
+      },
+      "source": [
+        "### Potential errors due to TensorShape changes\n",
+        "\n",
+        "The TensorShape behavior changes are unlikely to silently break your code. However, you may see shape-related code begin to raise `AttributeError`s as `int`s and `None`s do not have the same attributes that `tf.compat.v1.Dimension`s do.  Below are some examples of these `AttributeError`s:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r18f8JAGsQi6"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  # Create a shape and choose an index\n",
+        "  shape = tf.TensorShape([16, None, 256])\n",
+        "  value = shape[0].value\n",
+        "except AttributeError as e:\n",
+        "  # 'int' object has no attribute 'value'\n",
+        "  print(e)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "t9flHru1uIdT"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  # Create a shape and choose an index\n",
+        "  shape = tf.TensorShape([16, None, 256])\n",
+        "  dim = shape[1]\n",
+        "  other_dim = shape[2]\n",
+        "  dim.assert_is_compatible_with(other_dim)\n",
+        "except AttributeError as e:\n",
+        "  # 'NoneType' object has no attribute 'assert_is_compatible_with'\n",
+        "  print(e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Og7H_TwJqIOF"
+      },
+      "source": [
+        "## Tensor Equality by Value\n",
+        "\n",
+        "The binary `==` and `!=` operators on variables and tensors were changed to compare by value in TF2 rather than comparing by object reference like in TF1.x. Additionally, tensors and variables are no longer directly hashable or usable in sets or dict keys, because it may not be possible to hash them by value. Instead, they expose a `.ref()` method that you can use to get a hashable reference to the tensor or variable.\n",
+        "\n",
+        "To isolate the impact of this behavior change, you can use `tf.compat.v1.disable_tensor_equality()` and `tf.compat.v1.enable_tensor_equality()` to globally disable or enable this behavior change."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NGN4oL3lz0ki"
+      },
+      "source": [
+        "For example, in TF1.x, two variables with the same value will return false when you use the `==` operator:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dkGPGpEZ5DI-"
+      },
+      "outputs": [],
+      "source": [
+        "tf.compat.v1.disable_tensor_equality()\n",
+        "x = tf.Variable(0.0)\n",
+        "y = tf.Variable(0.0)\n",
+        "\n",
+        "x == y"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RqbewjIFz_oz"
+      },
+      "source": [
+        "While in TF2 with tensor equality checks enabled, `x == y` will return `True`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "V5P_Rwy-zxVE"
+      },
+      "outputs": [],
+      "source": [
+        "tf.compat.v1.enable_tensor_equality()\n",
+        "x = tf.Variable(0.0)\n",
+        "y = tf.Variable(0.0)\n",
+        "\n",
+        "x == y"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BqdUPLhHypfs"
+      },
+      "source": [
+        "So, in TF2, if you need to compare by object reference make sure to use `is` and `is not`"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iEjXVxlu4uxo"
+      },
+      "outputs": [],
+      "source": [
+        "tf.compat.v1.enable_tensor_equality()\n",
+        "x = tf.Variable(0.0)\n",
+        "y = tf.Variable(0.0)\n",
+        "\n",
+        "x is y"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r2ai1BGN01VI"
+      },
+      "source": [
+        "### Hashing tensors and variables\n",
+        "With TF1.x behaviors you used to be able to directly add variables and tensors to data structures that require hashing, such as `set` and `dict` keys.\n",
+        "```python\n",
+        "x = tf.Variable(0.0)\n",
+        "set([x, tf.constant(2.0)])\n",
+        "```\n",
+        "\n",
+        "However, in TF2 with tensor equality enabled, tensors and variables are made unhashable due to the `==` and `!=` operator semantics changing to value equality checks."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-TR1KfJu462w"
+      },
+      "outputs": [],
+      "source": [
+        "tf.compat.v1.enable_tensor_equality()\n",
+        "x = tf.Variable(0.0)\n",
+        "\n",
+        "try:\n",
+        "  set([x, tf.constant(2.0)])\n",
+        "except TypeError as e:\n",
+        "  # TypeError: Variable is unhashable. Instead, use tensor.ref() as the key.\n",
+        "  print(e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CQY7NvNAa7be"
+      },
+      "source": [
+        "So, in TF2 if you need to use tensor or variable objects as keys or `set` contents, you can use `tensor.ref()` to get a hashable reference that can be used as a key:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p-1kVPs01ZuU"
+      },
+      "outputs": [],
+      "source": [
+        "tf.compat.v1.enable_tensor_equality()\n",
+        "x = tf.Variable(0.0)\n",
+        "\n",
+        "tensor_set = set([x.ref(), tf.constant(2.0).ref()])\n",
+        "assert x.ref() in tensor_set\n",
+        "\n",
+        "tensor_set"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PqqRqfOYbaOX"
+      },
+      "source": [
+        "If needed, you can also get the tensor or variable from the reference by using `reference.deref()`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DwRZMYV06M7q"
+      },
+      "outputs": [],
+      "source": [
+        "referenced_var = x.ref().deref()\n",
+        "assert referenced_var is x\n",
+        "referenced_var"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5XSFQbJaReVC"
+      },
+      "source": [
+        "## Resources and further reading\n",
+        "\n",
+        "* Visit the [Migrate to TF2](https://tensorflow.org/guide/migrate) section to read more about migrating to TF2 from TF1.x.\n",
+        "* Read the [model mapping guide](./model_mapping.ipynb) to learn more mapping your TF1.x models to work in TF2 directly. "
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tf1_vs_tf2.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/tflite.ipynb b/site/en/guide/migrate/tflite.ipynb
new file mode 100644
index 00000000000..0426655ee1a
--- /dev/null
+++ b/site/en/guide/migrate/tflite.ipynb
@@ -0,0 +1,448 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrating your TFLite code to TF2\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "[TensorFlow Lite](https://www.tensorflow.org/lite/guide) (TFLite) is a set of tools that helps developers run ML inference on-device (mobile, embedded, and IoT devices). The [TFLite converter](https://www.tensorflow.org/lite/convert)  is one such tool that converts existing TF models into an optimized TFLite model format that can be efficiently run on-device.\n",
+        "\n",
+        "In this doc, you'll learn what changes you need to make to your TF to TFLite  conversion code, followed by a few examples that do the same.\n",
+        "\n",
+        "\n",
+        "## Changes to your TF to TFLite  conversion code\n",
+        "\n",
+        "* If you're using a legacy TF1 model format (such as Keras file, frozen GraphDef, checkpoints, tf.Session), update it to TF1/TF2 SavedModel and use the TF2 converter API `tf.lite.TFLiteConverter.from_saved_model(...)` to convert it to a TFLite model (refer to Table 1).\n",
+        "\n",
+        "* Update the converter API flags (refer to Table 2).\n",
+        "* Remove legacy APIs such as `tf.lite.constants`. (eg: Replace `tf.lite.constants.INT8` with `tf.int8`)\n",
+        "\n",
+        "// Table 1 // TFLite Python Converter API Update\n",
+        "\n",
+        "TF1 API | TF2 API |\n",
+        "--- | --- |\n",
+        "`tf.lite.TFLiteConverter.from_saved_model('saved_model/',..)` | *supported* |\n",
+        "`tf.lite.TFLiteConverter.from_keras_model_file('model.h5',..)` | *removed (update to SavedModel format)* |\n",
+        "`tf.lite.TFLiteConverter.from_frozen_graph('model.pb',..)` | *removed (update to SavedModel format)* |\n",
+        "`tf.lite.TFLiteConverter.from_session(sess,...)` | *removed (update to SavedModel format)* |"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Rf75rjeedigq"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XbVlZNizW1-Y"
+      },
+      "source": [
+        "// Table 2 // TFLite Python Converter API Flags Update\n",
+        "\n",
+        "TF1 API | TF2 API |\n",
+        "--- | --- |\n",
+        "`allow_custom_ops`
`optimizations`
 `representative_dataset`
`target_spec` 
`inference_input_type`
`inference_output_type`
`experimental_new_converter`
 `experimental_new_quantizer` | *supported* 







 |\n",
+        "`input_tensors`
`output_tensors`
`input_arrays_with_shape`
`output_arrays`
`experimental_debug_info_func`| *removed (unsupported converter API arguments)*




 |\n",
+        "`change_concat_input_ranges`
`default_ranges_stats`
`get_input_arrays()`
`inference_type`
`quantized_input_stats`
 `reorder_across_fake_quant` | *removed (unsupported quantization workflows)*





 |\n",
+        "`conversion_summary_dir`
`dump_graphviz_dir`
`dump_graphviz_video` | *removed (instead, visualize models using [Netron](https://lutzroeder.github.io/netron/) or [visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/visualize.py))*


 |\n",
+        "`output_format`
`drop_control_dependency` | *removed (unsupported features in TF2)*

 |"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "##  Examples\n",
+        "\n",
+        "You'll now walk through some examples to convert legacy TF1 models to TF1/TF2 SavedModels and then convert them to TF2 TFLite models.\n",
+        "\n",
+        "### Setup\n",
+        "\n",
+        "Start with the necessary TensorFlow imports."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1\n",
+        "import numpy as np\n",
+        "\n",
+        "import logging\n",
+        "logger = tf.get_logger()\n",
+        "logger.setLevel(logging.ERROR)\n",
+        "\n",
+        "import shutil\n",
+        "def remove_dir(path):\n",
+        "  try:\n",
+        "    shutil.rmtree(path)\n",
+        "  except:\n",
+        "    pass"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "89VllCprnFto"
+      },
+      "source": [
+        "Create all the necessary TF1 model formats."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Bwq8EFiwjzjx"
+      },
+      "outputs": [],
+      "source": [
+        "# Create a TF1 SavedModel\n",
+        "SAVED_MODEL_DIR = \"tf_saved_model/\"\n",
+        "remove_dir(SAVED_MODEL_DIR)\n",
+        "with tf1.Graph().as_default() as g:\n",
+        "  with tf1.Session() as sess:\n",
+        "    input = tf1.placeholder(tf.float32, shape=(3,), name='input')\n",
+        "    output = input + 2\n",
+        "    # print(\"result: \", sess.run(output, {input: [0., 2., 4.]}))\n",
+        "    tf1.saved_model.simple_save(\n",
+        "        sess, SAVED_MODEL_DIR,\n",
+        "        inputs={'input': input}, \n",
+        "        outputs={'output': output})\n",
+        "print(\"TF1 SavedModel path: \", SAVED_MODEL_DIR)\n",
+        "\n",
+        "# Create a TF1 Keras model\n",
+        "KERAS_MODEL_PATH = 'tf_keras_model.h5'\n",
+        "model = tf1.keras.models.Sequential([\n",
+        "    tf1.keras.layers.InputLayer(input_shape=(128, 128, 3,), name='input'),\n",
+        "    tf1.keras.layers.Dense(units=16, input_shape=(128, 128, 3,), activation='relu'),\n",
+        "    tf1.keras.layers.Dense(units=1, name='output')\n",
+        "])\n",
+        "model.save(KERAS_MODEL_PATH, save_format='h5')\n",
+        "print(\"TF1 Keras Model path: \", KERAS_MODEL_PATH)\n",
+        "\n",
+        "# Create a TF1 frozen GraphDef model\n",
+        "GRAPH_DEF_MODEL_PATH = tf.keras.utils.get_file(\n",
+        "    'mobilenet_v1_0.25_128',\n",
+        "    origin='https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.25_128_frozen.tgz',\n",
+        "    untar=True,\n",
+        ") + '/frozen_graph.pb'\n",
+        "\n",
+        "print(\"TF1 frozen GraphDef path: \", GRAPH_DEF_MODEL_PATH)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EzMBpG5rdt-7"
+      },
+      "source": [
+        "### 1. Convert a TF1 SavedModel to a TFLite model\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GFWIlVridt_F"
+      },
+      "source": [
+        "#### Before: Converting with TF1\n",
+        "This is typical code for TF1-style TFlite conversion.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dzXHHBQRdt_F"
+      },
+      "outputs": [],
+      "source": [
+        "converter = tf1.lite.TFLiteConverter.from_saved_model(\n",
+        "    saved_model_dir=SAVED_MODEL_DIR,\n",
+        "    input_arrays=['input'],\n",
+        "    input_shapes={'input' : [3]}\n",
+        ")\n",
+        "converter.optimizations = {tf.lite.Optimize.DEFAULT}\n",
+        "converter.change_concat_input_ranges = True\n",
+        "tflite_model = converter.convert()\n",
+        "# Ignore warning: \"Use '@tf.function' or '@defun' to decorate the function.\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NUptsxK_MUy2"
+      },
+      "source": [
+        "#### After: Converting with TF2\n",
+        "\n",
+        "Directly convert the TF1 SavedModel to a TFLite model, with a smaller v2 converter flags set."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0OyBjZ6Kdt_F"
+      },
+      "outputs": [],
+      "source": [
+        "# Convert TF1 SavedModel to a TFLite model.\n",
+        "converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir=SAVED_MODEL_DIR)\n",
+        "converter.optimizations = {tf.lite.Optimize.DEFAULT}\n",
+        "tflite_model = converter.convert()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yiwu3sso__fH"
+      },
+      "source": [
+        "### 2. Convert a TF1 Keras model file to a TFLite model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9WTPvPih__fR"
+      },
+      "source": [
+        "#### Before: Converting with TF1\n",
+        "This is typical code for TF1-style TFlite conversion."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9EXO0xYq__fR"
+      },
+      "outputs": [],
+      "source": [
+        "converter = tf1.lite.TFLiteConverter.from_keras_model_file(model_file=KERAS_MODEL_PATH)\n",
+        "converter.optimizations = {tf.lite.Optimize.DEFAULT}\n",
+        "converter.change_concat_input_ranges = True\n",
+        "tflite_model = converter.convert()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9l6ppTtTZ5Bz"
+      },
+      "source": [
+        "#### After: Converting with TF2\n",
+        "\n",
+        "First, convert the TF1 Keras model file to a TF2 SavedModel and then convert it to a TFLite model, with a smaller v2 converter flags set."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IGB5ZMGl__fR"
+      },
+      "outputs": [],
+      "source": [
+        "# Convert TF1 Keras model file to TF2 SavedModel.\n",
+        "model = tf.keras.models.load_model(KERAS_MODEL_PATH)\n",
+        "model.save(filepath='saved_model_2/')\n",
+        "\n",
+        "# Convert TF2 SavedModel to a TFLite model.\n",
+        "converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir='saved_model_2/')\n",
+        "tflite_model = converter.convert()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v5Zf6G4M-sZz"
+      },
+      "source": [
+        "### 3. Convert a TF1 frozen GraphDef to a TFLite model\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DzCJOV7AUlGZ"
+      },
+      "source": [
+        "#### Before: Converting with TF1\n",
+        "\n",
+        "This is typical code for TF1-style TFlite conversion."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r7RvcdRv6lll"
+      },
+      "outputs": [],
+      "source": [
+        "converter = tf1.lite.TFLiteConverter.from_frozen_graph(\n",
+        "    graph_def_file=GRAPH_DEF_MODEL_PATH,\n",
+        "    input_arrays=['input'],\n",
+        "    input_shapes={'input' : [1, 128, 128, 3]},\n",
+        "    output_arrays=['MobilenetV1/Predictions/Softmax'],\n",
+        ")\n",
+        "converter.optimizations = {tf.lite.Optimize.DEFAULT}\n",
+        "converter.change_concat_input_ranges = True\n",
+        "tflite_model = converter.convert()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZdIogJsKaMNH"
+      },
+      "source": [
+        "#### After: Converting with TF2\n",
+        "\n",
+        "First, convert the TF1 frozen GraphDef to a TF1 SavedModel and then convert it to a TFLite model, with a smaller v2 converter flags set.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Oigap0TZxjWG"
+      },
+      "outputs": [],
+      "source": [
+        "## Convert TF1 frozen Graph to TF1 SavedModel.\n",
+        "\n",
+        "# Load the graph as a v1.GraphDef\n",
+        "import pathlib\n",
+        "gdef = tf.compat.v1.GraphDef()\n",
+        "gdef.ParseFromString(pathlib.Path(GRAPH_DEF_MODEL_PATH).read_bytes())\n",
+        "\n",
+        "# Convert the GraphDef to a tf.Graph\n",
+        "with tf.Graph().as_default() as g:\n",
+        "  tf.graph_util.import_graph_def(gdef, name=\"\")\n",
+        "\n",
+        "# Look up the input and output tensors.\n",
+        "input_tensor = g.get_tensor_by_name('input:0') \n",
+        "output_tensor = g.get_tensor_by_name('MobilenetV1/Predictions/Softmax:0')\n",
+        "\n",
+        "# Save the graph as a TF1 Savedmodel\n",
+        "remove_dir('saved_model_3/')\n",
+        "with tf.compat.v1.Session(graph=g) as s:\n",
+        "  tf.compat.v1.saved_model.simple_save(\n",
+        "      session=s,\n",
+        "      export_dir='saved_model_3/',\n",
+        "      inputs={'input':input_tensor},\n",
+        "      outputs={'output':output_tensor})\n",
+        "\n",
+        "# Convert TF1 SavedModel to a TFLite model.\n",
+        "converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir='saved_model_3/')\n",
+        "converter.optimizations = {tf.lite.Optimize.DEFAULT}\n",
+        "tflite_model = converter.convert()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MFbsddkOw4Wl"
+      },
+      "source": [
+        "# Further reading\n",
+        "\n",
+        "*   Refer to the [TFLite Guide](https://www.tensorflow.org/lite/guide) to learn more about the workflows and latest features.\n",
+        "* If you're using  TF1 code or legacy TF1 model formats (Keras `.h5` files, frozen GraphDef `.pb`, etc), please update your code and migrate your models to the [TF2 SavedModel format](https://www.tensorflow.org/guide/saved_model). \n"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tflite.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/tpu_embedding.ipynb b/site/en/guide/migrate/tpu_embedding.ipynb
new file mode 100644
index 00000000000..44105ea984b
--- /dev/null
+++ b/site/en/guide/migrate/tpu_embedding.ipynb
@@ -0,0 +1,589 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate from TPU embedding_columns to TPUEmbedding layer\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "This guide demonstrates how to migrate embedding training on  on [TPUs](../../guide/tpu.ipynb) from TensorFlow 1's `embedding_column` API with `TPUEstimator` to TensorFlow 2's `TPUEmbedding` layer API with `TPUStrategy`.\n",
+        "\n",
+        "Embeddings are (large) matrices. They are lookup tables that map from a sparse feature space to dense vectors. Embeddings provide efficient and dense representations, capturing complex similarities and relationships between features.\n",
+        "\n",
+        "TensorFlow includes specialized support for training embeddings on TPUs. This TPU-specific embedding support allows you to train embeddings that are larger than the memory of a single TPU device, and to use sparse and ragged inputs on TPUs.\n",
+        "\n",
+        "- In TensorFlow 1, `tf.compat.v1.estimator.tpu.TPUEstimator` is a high level API that encapsulates training, evaluation, prediction, and exporting for serving with TPUs. It has special support for `tf.compat.v1.tpu.experimental.embedding_column`.\n",
+        "- To implement this in TensorFlow 2, use the TensorFlow Recommenders' `tfrs.layers.embedding.TPUEmbedding` layer. For training and evaluation, use a TPU distribution strategy—`tf.distribute.TPUStrategy`—which is compatible with the Keras APIs for, for example, model building (`tf.keras.Model`), optimizers (`tf.keras.optimizers.Optimizer`), and training with `Model.fit` or a custom training loop with `tf.function` and `tf.GradientTape`.\n",
+        "\n",
+        "For additional information, refer to the `tfrs.layers.embedding.TPUEmbedding` layer's API documentation, as well as the `tf.tpu.experimental.embedding.TableConfig` and `tf.tpu.experimental.embedding.FeatureConfig` docs for additional information. For an overview of `tf.distribute.TPUStrategy`, check out the [Distributed training](../../guide/distributed_training.ipynb) guide and the [Use TPUs](../../guide/tpu.ipynb) guide. If you're migrating from `TPUEstimator` to `TPUStrategy`, check out [the TPU migration guide](tpu_estimator.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Start by installing [TensorFlow Recommenders](https://www.tensorflow.org/recommenders) and importing some necessary packages:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tYE3RnRN2jNu"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install tensorflow-recommenders"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1\n",
+        "\n",
+        "# TPUEmbedding layer is not part of TensorFlow.\n",
+        "import tensorflow_recommenders as tfrs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Jsm9Rxx7s1OZ"
+      },
+      "source": [
+        "And prepare a simple dataset for demonstration purposes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m7rnGxsXtDkV"
+      },
+      "outputs": [],
+      "source": [
+        "features = [[1., 1.5]]\n",
+        "embedding_features_indices = [[0, 0], [0, 1]]\n",
+        "embedding_features_values = [0, 5]\n",
+        "labels = [[0.3]]\n",
+        "eval_features = [[4., 4.5]]\n",
+        "eval_embedding_features_indices = [[0, 0], [0, 1]]\n",
+        "eval_embedding_features_values = [4, 3]\n",
+        "eval_labels = [[0.8]]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4uXff1BEssdE"
+      },
+      "source": [
+        "## TensorFlow 1: Train embeddings on TPUs with TPUEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pc-WSeYG2oje"
+      },
+      "source": [
+        "In TensorFlow 1, you set up TPU embeddings using the `tf.compat.v1.tpu.experimental.embedding_column` API and train/evaluate the model on TPUs with `tf.compat.v1.estimator.tpu.TPUEstimator`.\n",
+        "\n",
+        "The inputs are integers ranging from zero to the vocabulary size for the TPU embedding table. Begin with encoding the inputs to categorical ID with `tf.feature_column.categorical_column_with_identity`. Use `\"sparse_feature\"` for the `key` parameter, since the input features are integer-valued, while `num_buckets` is the vocabulary size for the embedding table (`10`)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sO_y-IRT3dcM"
+      },
+      "outputs": [],
+      "source": [
+        "embedding_id_column = (\n",
+        "      tf1.feature_column.categorical_column_with_identity(\n",
+        "          key=\"sparse_feature\", num_buckets=10))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "57e2dec8ed4a"
+      },
+      "source": [
+        "Next, convert the sparse categorical inputs to a dense representation with `tpu.experimental.embedding_column`, where `dimension` is the width of the embedding table. It will store an embedding vector for each of the `num_buckets`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6d61c855011f"
+      },
+      "outputs": [],
+      "source": [
+        "embedding_column = tf1.tpu.experimental.embedding_column(\n",
+        "    embedding_id_column, dimension=5)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c6061452ee5a"
+      },
+      "source": [
+        "Now, define the TPU-specific embedding configuration via `tf.estimator.tpu.experimental.EmbeddingConfigSpec`. You will pass it later to `tf.estimator.tpu.TPUEstimator` as an `embedding_config_spec` parameter."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6abbf967fc82"
+      },
+      "outputs": [],
+      "source": [
+        "embedding_config_spec = tf1.estimator.tpu.experimental.EmbeddingConfigSpec(\n",
+        "    feature_columns=(embedding_column,),\n",
+        "    optimization_parameters=(\n",
+        "        tf1.tpu.experimental.AdagradParameters(0.05)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BVWHEQj5a7rN"
+      },
+      "source": [
+        "Next, to use a `TPUEstimator`, define: \n",
+        "- An input function for the training data\n",
+        "- An evaluation input function for the evaluation data\n",
+        "- A model function for instructing the `TPUEstimator` how the training op is defined with the features and labels"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "def _input_fn(params):\n",
+        "  dataset = tf1.data.Dataset.from_tensor_slices((\n",
+        "      {\"dense_feature\": features,\n",
+        "       \"sparse_feature\": tf1.SparseTensor(\n",
+        "           embedding_features_indices,\n",
+        "           embedding_features_values, [1, 2])},\n",
+        "           labels))\n",
+        "  dataset = dataset.repeat()\n",
+        "  return dataset.batch(params['batch_size'], drop_remainder=True)\n",
+        "\n",
+        "def _eval_input_fn(params):\n",
+        "  dataset = tf1.data.Dataset.from_tensor_slices((\n",
+        "      {\"dense_feature\": eval_features,\n",
+        "       \"sparse_feature\": tf1.SparseTensor(\n",
+        "           eval_embedding_features_indices,\n",
+        "           eval_embedding_features_values, [1, 2])},\n",
+        "           eval_labels))\n",
+        "  dataset = dataset.repeat()\n",
+        "  return dataset.batch(params['batch_size'], drop_remainder=True)\n",
+        "\n",
+        "def _model_fn(features, labels, mode, params):\n",
+        "  embedding_features = tf1.keras.layers.DenseFeatures(embedding_column)(features)\n",
+        "  concatenated_features = tf1.keras.layers.Concatenate(axis=1)(\n",
+        "      [embedding_features, features[\"dense_feature\"]])\n",
+        "  logits = tf1.layers.Dense(1)(concatenated_features)\n",
+        "  loss = tf1.losses.mean_squared_error(labels=labels, predictions=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.05)\n",
+        "  optimizer = tf1.tpu.CrossShardOptimizer(optimizer)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "  return tf1.estimator.tpu.TPUEstimatorSpec(mode, loss=loss, train_op=train_op)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QYnP3Dszc-2R"
+      },
+      "source": [
+        "With those functions defined, create a `tf.distribute.cluster_resolver.TPUClusterResolver` that provides the cluster information, and a `tf.compat.v1.estimator.tpu.RunConfig` object.\n",
+        "\n",
+        "Along with the model function you have defined, you can now create a `TPUEstimator`. Here, you will simplify the flow by skipping checkpoint savings. Then, you will specify the batch size for both training and evaluation for the `TPUEstimator`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WAqyqawemlcl"
+      },
+      "outputs": [],
+      "source": [
+        "cluster_resolver = tf1.distribute.cluster_resolver.TPUClusterResolver(tpu='')\n",
+        "print(\"All devices: \", tf1.config.list_logical_devices('TPU'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HsOpjW5plH9Q"
+      },
+      "outputs": [],
+      "source": [
+        "tpu_config = tf1.estimator.tpu.TPUConfig(\n",
+        "    iterations_per_loop=10,\n",
+        "    per_host_input_for_training=tf1.estimator.tpu.InputPipelineConfig\n",
+        "          .PER_HOST_V2)\n",
+        "config = tf1.estimator.tpu.RunConfig(\n",
+        "    cluster=cluster_resolver,\n",
+        "    save_checkpoints_steps=None,\n",
+        "    tpu_config=tpu_config)\n",
+        "estimator = tf1.estimator.tpu.TPUEstimator(\n",
+        "    model_fn=_model_fn, config=config, train_batch_size=8, eval_batch_size=8,\n",
+        "    embedding_config_spec=embedding_config_spec)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Uxw7tWrcepaZ"
+      },
+      "source": [
+        "Call `TPUEstimator.train` to begin training the model:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WZPKFOMAcyrP"
+      },
+      "outputs": [],
+      "source": [
+        "estimator.train(_input_fn, steps=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ev1vjIz9euIw"
+      },
+      "source": [
+        "Then, call `TPUEstimator.evaluate` to evaluate the model using the evaluation data:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bqiKRiwWc0cz"
+      },
+      "outputs": [],
+      "source": [
+        "estimator.evaluate(_eval_input_fn, steps=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "## TensorFlow 2: Train embeddings on TPUs with TPUStrategy"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UesuXNbShrbi"
+      },
+      "source": [
+        "In TensorFlow 2, to train on the TPU workers, use `tf.distribute.TPUStrategy` together with the Keras APIs for model definition and training/evaluation. (Refer to the [Use TPUs](https://render.githubusercontent.com/guide/tpu.ipynb) guide for more examples of training with Keras Model.fit and a custom training loop (with `tf.function` and `tf.GradientTape`).)\n",
+        "\n",
+        "Since you need to perform some initialization work to connect to the remote cluster and initialize the TPU workers, start by creating a `TPUClusterResolver` to provide the cluster information and connect to the cluster. (Learn more in the *TPU initialization* section of the [Use TPUs](../../guide/tpu.ipynb) guide.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_TgdPNgXoS63"
+      },
+      "outputs": [],
+      "source": [
+        "cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')\n",
+        "tf.config.experimental_connect_to_cluster(cluster_resolver)\n",
+        "tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n",
+        "print(\"All devices: \", tf.config.list_logical_devices('TPU'))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "94JBD0HxmdPI"
+      },
+      "source": [
+        "Next, prepare your data. This is similar to how you created a dataset in the TensorFlow 1 example, except the dataset function is now passed a `tf.distribute.InputContext` object rather than a `params` dict. You can use this object to determine the local batch size (and which host this pipeline is for, so you can properly partition your data).\n",
+        "\n",
+        "- When using the `tfrs.layers.embedding.TPUEmbedding` API, it is important to include the `drop_remainder=True` option when batching the dataset with `Dataset.batch`, since `TPUEmbedding` requires a fixed batch size.\n",
+        "- Additionally, the same batch size must be used for evaluation and training if they are taking place on the same set of devices.\n",
+        "- Finally, you should use `tf.keras.utils.experimental.DatasetCreator` along with the special input option—`experimental_fetch_to_device=False`—in `tf.distribute.InputOptions` (which holds strategy-specific configurations). This is demonstrated below:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9NTruOw6mcy9"
+      },
+      "outputs": [],
+      "source": [
+        "global_batch_size = 8\n",
+        "\n",
+        "def _input_dataset(context: tf.distribute.InputContext):\n",
+        "  dataset = tf.data.Dataset.from_tensor_slices((\n",
+        "      {\"dense_feature\": features,\n",
+        "       \"sparse_feature\": tf.SparseTensor(\n",
+        "           embedding_features_indices,\n",
+        "           embedding_features_values, [1, 2])},\n",
+        "           labels))\n",
+        "  dataset = dataset.shuffle(10).repeat()\n",
+        "  dataset = dataset.batch(\n",
+        "      context.get_per_replica_batch_size(global_batch_size),\n",
+        "      drop_remainder=True)\n",
+        "  return dataset.prefetch(2)\n",
+        "\n",
+        "def _eval_dataset(context: tf.distribute.InputContext):\n",
+        "  dataset = tf.data.Dataset.from_tensor_slices((\n",
+        "      {\"dense_feature\": eval_features,\n",
+        "       \"sparse_feature\": tf.SparseTensor(\n",
+        "           eval_embedding_features_indices,\n",
+        "           eval_embedding_features_values, [1, 2])},\n",
+        "           eval_labels))\n",
+        "  dataset = dataset.repeat()\n",
+        "  dataset = dataset.batch(\n",
+        "      context.get_per_replica_batch_size(global_batch_size),\n",
+        "      drop_remainder=True)\n",
+        "  return dataset.prefetch(2)\n",
+        "\n",
+        "input_options = tf.distribute.InputOptions(\n",
+        "    experimental_fetch_to_device=False)\n",
+        "\n",
+        "input_dataset = tf.keras.utils.experimental.DatasetCreator(\n",
+        "    _input_dataset, input_options=input_options)\n",
+        "\n",
+        "eval_dataset = tf.keras.utils.experimental.DatasetCreator(\n",
+        "    _eval_dataset, input_options=input_options)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "R4EHXhN3CVmo"
+      },
+      "source": [
+        "Next, once the data is prepared, you will create a `TPUStrategy`, and define a model, metrics, and an optimizer under the scope of this strategy (`Strategy.scope`).\n",
+        "\n",
+        "You should pick a number for `steps_per_execution` in `Model.compile` since it specifies the number of batches to run during each `tf.function` call, and is critical for performance. This argument is similar to `iterations_per_loop` used in `TPUEstimator`.\n",
+        "\n",
+        "The features and table configuration that were specified in TensorFlow 1 via the `tf.tpu.experimental.embedding_column` (and `tf.tpu.experimental.shared_embedding_column`) can be specified directly in TensorFlow 2 via a pair of configuration objects:\n",
+        "- `tf.tpu.experimental.embedding.FeatureConfig`\n",
+        "- `tf.tpu.experimental.embedding.TableConfig`\n",
+        "\n",
+        "(Refer to the associated API documentation for more details.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "atVciNgPs0fw"
+      },
+      "outputs": [],
+      "source": [
+        "strategy = tf.distribute.TPUStrategy(cluster_resolver)\n",
+        "with strategy.scope():\n",
+        "  if hasattr(tf.keras.optimizers, \"legacy\"):\n",
+        "    optimizer = tf.keras.optimizers.legacy.Adagrad(learning_rate=0.05)\n",
+        "  else:\n",
+        "    optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+        "  dense_input = tf.keras.Input(shape=(2,), dtype=tf.float32, batch_size=global_batch_size)\n",
+        "  sparse_input = tf.keras.Input(shape=(), dtype=tf.int32, batch_size=global_batch_size)\n",
+        "  embedded_input = tfrs.layers.embedding.TPUEmbedding(\n",
+        "      feature_config=tf.tpu.experimental.embedding.FeatureConfig(\n",
+        "          table=tf.tpu.experimental.embedding.TableConfig(\n",
+        "              vocabulary_size=10,\n",
+        "              dim=5,\n",
+        "              initializer=tf.initializers.TruncatedNormal(mean=0.0, stddev=1)),\n",
+        "          name=\"sparse_input\"),\n",
+        "      optimizer=optimizer)(sparse_input)\n",
+        "  input = tf.keras.layers.Concatenate(axis=1)([dense_input, embedded_input])\n",
+        "  result = tf.keras.layers.Dense(1)(input)\n",
+        "  model = tf.keras.Model(inputs={\"dense_feature\": dense_input, \"sparse_feature\": sparse_input}, outputs=result)\n",
+        "  model.compile(optimizer, \"mse\", steps_per_execution=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FkM2VZyni98F"
+      },
+      "source": [
+        "With that, you are ready to train the model with the training dataset:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kip65sYBlKiu"
+      },
+      "outputs": [],
+      "source": [
+        "model.fit(input_dataset, epochs=5, steps_per_epoch=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r0AEK8sNjLOj"
+      },
+      "source": [
+        "Finally, evaluate the model using the evaluation dataset:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6tMRkyfKhqSL"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(eval_dataset, steps=1, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a97b888c1911"
+      },
+      "source": [
+        "## Next steps"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gHx_RUL8xcJ3"
+      },
+      "source": [
+        "Learn more about setting up TPU-specific embeddings in the API docs:\n",
+        "\n",
+        "- `tfrs.layers.embedding.TPUEmbedding`: particularly about feature and table configuration, setting the optimizer, creating a model (using the Keras [functional](https://www.tensorflow.org/guide/keras/functional) API or via [subclassing](../..guide/keras/custom_layers_and_models.ipynb) `tf.keras.Model`), training/evaluation, and model serving with `tf.saved_model`\n",
+        "- `tf.tpu.experimental.embedding.TableConfig`\n",
+        "- `tf.tpu.experimental.embedding.FeatureConfig`\n",
+        "\n",
+        "For more information about `TPUStrategy` in TensorFlow 2, consider the following resources:\n",
+        "\n",
+        "- Guide: [Use TPUs](../../guide/tpu.ipynb) (covering training with Keras `Model.fit`/a custom training loop with `tf.distribute.TPUStrategy`, as well as tips on improving the performance with `tf.function`)\n",
+        "- Guide: [Distributed training with TensorFlow](../../guide/distributed_training.ipynb)\n",
+        "- Guide: [Migrate from TPUEstimator to TPUStrategy](tpu_estimator.ipynb).\n",
+        "\n",
+        "To learn more about customizing your training, refer to:\n",
+        "\n",
+        "- Guide: [Customize what happens in Model.fit](../..guide/keras/customizing_what_happens_in_fit.ipynb)\n",
+        "- Guide: [Writing a training loop from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch)\n",
+        "\n",
+        "TPUs—Google's specialized ASICs for machine learning—are available through [Google Colab](https://colab.research.google.com/), the [TPU Research Cloud](https://sites.research.google/trc/), and [Cloud TPU](https://cloud.google.com/tpu)."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "TPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tpu_embedding.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/migrate/tpu_estimator.ipynb b/site/en/guide/migrate/tpu_estimator.ipynb
new file mode 100644
index 00000000000..9cc35dc8bae
--- /dev/null
+++ b/site/en/guide/migrate/tpu_estimator.ipynb
@@ -0,0 +1,392 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wJcYs_ERTnnI"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HMUDt0CiUJk9"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77z2OchJTk0l"
+      },
+      "source": [
+        "# Migrate from TPUEstimator to TPUStrategy\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "meUTrR4I6m1C"
+      },
+      "source": [
+        "This guide demonstrates how to migrate your workflows running on [TPUs](../../guide/tpu.ipynb) from TensorFlow 1's `TPUEstimator` API to TensorFlow 2's `TPUStrategy` API.\n",
+        "\n",
+        "- In TensorFlow 1, the `tf.compat.v1.estimator.tpu.TPUEstimator` API lets you train and evaluate a model, as well as perform inference and save your model (for serving) on (Cloud) TPUs.\n",
+        "- In TensorFlow 2, to perform synchronous training on TPUs and TPU Pods (a collection of TPU devices connected by dedicated high-speed network interfaces), you need to use a TPU distribution strategy—`tf.distribute.TPUStrategy`. The strategy can work with the Keras APIs—including for model building (`tf.keras.Model`), optimizers (`tf.keras.optimizers.Optimizer`), and training (`Model.fit`)—as well as a custom training loop (with `tf.function` and `tf.GradientTape`).\n",
+        "\n",
+        "For end-to-end TensorFlow 2 examples, check out the [Use TPUs](../../guide/tpu.ipynb) guide—namely, the *Classification on TPUs* section—and the [Solve GLUE tasks using BERT on TPU](https://www.tensorflow.org/text/tutorials/bert_glue) tutorial. You may also find the [Distributed training](../../guide/distributed_training.ipynb) guide useful, which covers all TensorFlow distribution strategies, including `TPUStrategy`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YdZSoIXEbhg-"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "Start with imports and a simple dataset for demonstration purposes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iE0vSfMXumKI"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m7rnGxsXtDkV"
+      },
+      "outputs": [],
+      "source": [
+        "features = [[1., 1.5]]\n",
+        "labels = [[0.3]]\n",
+        "eval_features = [[4., 4.5]]\n",
+        "eval_labels = [[0.8]]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4uXff1BEssdE"
+      },
+      "source": [
+        "## TensorFlow 1: Drive a model on TPUs with TPUEstimator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BVWHEQj5a7rN"
+      },
+      "source": [
+        "This section of the guide demonstrates how to perform training and evaluation with `tf.compat.v1.estimator.tpu.TPUEstimator` in TensorFlow 1.\n",
+        "\n",
+        "To use a `TPUEstimator`, first define a few functions: an input function for the training data, an evaluation input function for the evaluation data, and a model function that tells the `TPUEstimator` how the training op is defined with the features and labels:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lqe9obf7suIj"
+      },
+      "outputs": [],
+      "source": [
+        "def _input_fn(params):\n",
+        "  dataset = tf1.data.Dataset.from_tensor_slices((features, labels))\n",
+        "  dataset = dataset.repeat()\n",
+        "  return dataset.batch(params['batch_size'], drop_remainder=True)\n",
+        "\n",
+        "def _eval_input_fn(params):\n",
+        "  dataset = tf1.data.Dataset.from_tensor_slices((eval_features, eval_labels))\n",
+        "  dataset = dataset.repeat()\n",
+        "  return dataset.batch(params['batch_size'], drop_remainder=True)\n",
+        "\n",
+        "def _model_fn(features, labels, mode, params):\n",
+        "  logits = tf1.layers.Dense(1)(features)\n",
+        "  loss = tf1.losses.mean_squared_error(labels=labels, predictions=logits)\n",
+        "  optimizer = tf1.train.AdagradOptimizer(0.05)\n",
+        "  train_op = optimizer.minimize(loss, global_step=tf1.train.get_global_step())\n",
+        "  return tf1.estimator.tpu.TPUEstimatorSpec(mode, loss=loss, train_op=train_op)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QYnP3Dszc-2R"
+      },
+      "source": [
+        "With those functions defined, create a `tf.distribute.cluster_resolver.TPUClusterResolver` that provides the cluster information, and a `tf.compat.v1.estimator.tpu.RunConfig` object. Along with the model function you have defined, you can now create a `TPUEstimator`. Here, you will simplify the flow by skipping checkpoint savings. Then, you will specify the batch size for both training and evaluation for the `TPUEstimator`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WAqyqawemlcl"
+      },
+      "outputs": [],
+      "source": [
+        "cluster_resolver = tf1.distribute.cluster_resolver.TPUClusterResolver(tpu='')\n",
+        "print(\"All devices: \", tf1.config.list_logical_devices('TPU'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HsOpjW5plH9Q"
+      },
+      "outputs": [],
+      "source": [
+        "tpu_config = tf1.estimator.tpu.TPUConfig(iterations_per_loop=10)\n",
+        "config = tf1.estimator.tpu.RunConfig(\n",
+        "    cluster=cluster_resolver,\n",
+        "    save_checkpoints_steps=None,\n",
+        "    tpu_config=tpu_config)\n",
+        "estimator = tf1.estimator.tpu.TPUEstimator(\n",
+        "    model_fn=_model_fn,\n",
+        "    config=config,\n",
+        "    train_batch_size=8,\n",
+        "    eval_batch_size=8)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Uxw7tWrcepaZ"
+      },
+      "source": [
+        "Call `TPUEstimator.train` to begin training the model:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WZPKFOMAcyrP"
+      },
+      "outputs": [],
+      "source": [
+        "estimator.train(_input_fn, steps=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ev1vjIz9euIw"
+      },
+      "source": [
+        "Then, call `TPUEstimator.evaluate` to evaluate the model using the evaluation data:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bqiKRiwWc0cz"
+      },
+      "outputs": [],
+      "source": [
+        "estimator.evaluate(_eval_input_fn, steps=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KEmzBjfnsxwT"
+      },
+      "source": [
+        "## TensorFlow 2: Drive a model on TPUs with Keras Model.fit and TPUStrategy"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UesuXNbShrbi"
+      },
+      "source": [
+        "In TensorFlow 2, to train on the TPU workers, use `tf.distribute.TPUStrategy` together with the Keras APIs for model definition and training/evaluation. (Refer to the [Use TPUs](../../guide/tpu.ipynb) guide for more examples of training with Keras `Model.fit` and a custom training loop (with `tf.function` and `tf.GradientTape`).)\n",
+        "\n",
+        "Since you need to perform some initialization work to connect to the remote cluster and initialize the TPU workers, start by creating a `TPUClusterResolver` to provide the cluster information and connect to the cluster. (Learn more in the *TPU initialization* section of the [Use TPUs](../../guide/tpu.ipynb) guide.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_TgdPNgXoS63"
+      },
+      "outputs": [],
+      "source": [
+        "cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')\n",
+        "tf.config.experimental_connect_to_cluster(cluster_resolver)\n",
+        "tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n",
+        "print(\"All devices: \", tf.config.list_logical_devices('TPU'))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "R4EHXhN3CVmo"
+      },
+      "source": [
+        "Next, once your data is prepared, you will create a `TPUStrategy`, define a model, metrics, and an optimizer under the scope of this strategy.\n",
+        "\n",
+        "To achieve comparable training speed with `TPUStrategy`, you should make sure to pick a number for `steps_per_execution` in `Model.compile` because it specifies the number of batches to run during each `tf.function` call, and is critical for performance. This argument is similar to `iterations_per_loop` used in a `TPUEstimator`. If you are using custom training loops, you should make sure multiple steps are run within the `tf.function`-ed training function. Go to the *Improving performance with multiple steps inside tf.function* section of the [Use TPUs](../../guide/tpu.ipynb) guide for more information.\n",
+        "\n",
+        "`tf.distribute.TPUStrategy` can support bounded dynamic shapes, which is the case that the upper bound of the dynamic shape computation can be inferred. But dynamic shapes may introduce some performance overhead compared to static shapes. So, it is generally recommended to make your input shapes static if possible, especially in training. One common op that returns a dynamic shape is `tf.data.Dataset.batch(batch_size)`, since the number of samples remaining in a stream might be less than the batch size. Therefore, when training on the TPU, you should use `tf.data.Dataset.batch(..., drop_remainder=True)` for best training performance."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "atVciNgPs0fw"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "    (features, labels)).shuffle(10).repeat().batch(\n",
+        "        8, drop_remainder=True).prefetch(2)\n",
+        "eval_dataset = tf.data.Dataset.from_tensor_slices(\n",
+        "    (eval_features, eval_labels)).batch(1, drop_remainder=True)\n",
+        "\n",
+        "strategy = tf.distribute.TPUStrategy(cluster_resolver)\n",
+        "with strategy.scope():\n",
+        "  model = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])\n",
+        "  optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.05)\n",
+        "  model.compile(optimizer, \"mse\", steps_per_execution=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FkM2VZyni98F"
+      },
+      "source": [
+        "With that, you are ready to train the model with the training dataset:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kip65sYBlKiu"
+      },
+      "outputs": [],
+      "source": [
+        "model.fit(dataset, epochs=5, steps_per_epoch=10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r0AEK8sNjLOj"
+      },
+      "source": [
+        "Finally, evaluate the model using the evaluation dataset:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6tMRkyfKhqSL"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(eval_dataset, return_dict=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "67ec4d3f35d6"
+      },
+      "source": [
+        "## Next steps"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gHx_RUL8xcJ3"
+      },
+      "source": [
+        "To learn more about `TPUStrategy` in TensorFlow 2, consider the following resources:\n",
+        "\n",
+        "- Guide: [Use TPUs](../../guide/tpu.ipynb) (covering training with Keras `Model.fit`/a custom training loop with `tf.distribute.TPUStrategy`, as well as tips on improving the performance with `tf.function`)\n",
+        "- Guide: [Distributed training with TensorFlow](../../guide/distributed_training.ipynb)\n",
+        "\n",
+        "To learn more about customizing your training, refer to:\n",
+        "- Guide: [Customize what happens in Model.fit](../..guide/keras/customizing_what_happens_in_fit.ipynb)\n",
+        "- Guide: [Writing a training loop from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch)\n",
+        "\n",
+        "TPUs—Google's specialized ASICs for machine learning—are available through [Google Colab](https://colab.research.google.com/), the [TPU Research Cloud](https://sites.research.google/trc/), and [Cloud TPU](https://cloud.google.com/tpu)."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "TPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tpu_estimator.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/upgrade.ipynb b/site/en/guide/migrate/upgrade.ipynb
similarity index 61%
rename from site/en/guide/upgrade.ipynb
rename to site/en/guide/migrate/upgrade.ipynb
index 8b1c531a12d..7223a8c8c81 100644
--- a/site/en/guide/upgrade.ipynb
+++ b/site/en/guide/migrate/upgrade.ipynb
@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6bYaCABobL5q"
       },
       "source": [
@@ -12,11 +11,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
         "id": "FlUw7tSKbtg4"
       },
       "outputs": [],
@@ -37,158 +34,130 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "08OTcmxgqkc2"
       },
       "source": [
-        "# Automatically upgrade code to TensorFlow 2\n",
-        "\n",
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/upgrade\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n",
-        "    View on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/upgrade.ipynb\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n",
-        "    Run in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/upgrade.ipynb\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n",
-        "    View source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/upgrade.ipynb\"\u003e\n",
-        "    \u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003e\n",
-        "    Download notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e\n",
-        "\n"
+        "# Automatically rewrite TF 1.x and compat.v1 API symbols\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    \n",
+        "    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    \n",
+        "    \n",
+        "    Download notebook\n",
+        "  
\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hZSaRPoybOp5"
       },
       "source": [
-        "TensorFlow 2.0 includes many API changes, such as reordering arguments, renaming symbols, and changing default values for parameters. Manually performing all of these modifications would be tedious and prone to error. To streamline the changes, and to make your transition to TF 2.0 as seamless as possible, the TensorFlow team has created the `tf_upgrade_v2` utility to help transition legacy code to the new API.\n",
+        "TensorFlow 2.x includes many API changes from TF 1.x and the `tf.compat.v1` APIs, such as reordering arguments, renaming symbols, and changing default values for parameters. Manually performing all of these modifications would be tedious and prone to error. To streamline the changes, and to make your transition to TF 2.x as seamless as possible, the TensorFlow team has created the `tf_upgrade_v2` utility to help transition legacy code to the new API.\n",
         "\n",
-        "Note: `tf_upgrade_v2` is installed automatically for TensorFlow 1.13 and later (including all TF 2.0 builds).\n",
+        "Note: `tf_upgrade_v2` is installed automatically for TensorFlow 1.13 and later (including all TF 2.x builds).\n",
         "\n",
         "Typical usage is like this:\n",
         "\n",
-        "\u003cpre class=\"devsite-terminal devsite-click-to-copy prettyprint lang-bsh\"\u003e\n",
+        "
\n",
         "tf_upgrade_v2 \\\n",
         "  --intree my_project/ \\\n",
         "  --outtree my_project_v2/ \\\n",
         "  --reportfile report.txt\n",
-        "\u003c/pre\u003e\n",
+        "
\n",
         "\n",
-        "It will accelerate your upgrade process by converting existing TensorFlow 1.x Python scripts to TensorFlow 2.0.\n",
+        "It will accelerate your upgrade process by converting existing TensorFlow 1.x Python scripts to TensorFlow 2.x.\n",
         "\n",
-        "The conversion script automates as much as possible, but there are still syntactical and stylistic changes that cannot be performed by the script."
+        "The conversion script automates many mechanical API transformations, though many APIs cannot be automatically migrated. It is also not able to fully make your code compatible with TF2 behaviors and APIs. So, it is only a part of your migration journey."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "gP9v2vgptdfi"
       },
       "source": [
         "## Compatibility modules\n",
         "\n",
-        "Certain API symbols can not be upgraded simply by using a string replacement. To ensure your code is still supported in TensorFlow 2.0, the upgrade script includes a `compat.v1` module. This module replaces TF 1.x symbols like `tf.foo` with the equivalent `tf.compat.v1.foo` reference. While the compatibility module is nice, we recommend that you manually proofread replacements and migrate them to new APIs in the `tf.*` namespace instead of `tf.compat.v1` namespace as quickly as possible.\n",
+        "Certain API symbols can not be upgraded simply by using a string replacement. Those that cannot be automatically upgraded will be mapped to their locations in the `compat.v1` module. This module replaces TF 1.x symbols like `tf.foo` with the equivalent `tf.compat.v1.foo` reference. If you are already using `compat.v1` APIs by importing TF via `import tensorflow.compat.v1 as tf`, the `tf_upgrade_v2` script will attempt to convert these usages to the non-compat APIs where possible. Note that while some `compat.v1` APIs are compatible with TF2.x behaviors, many are not. Therefore, it's recommended to manually proofread replacements and migrate them to new APIs in the `tf.*` namespace instead of `tf.compat.v1` namespace as quickly as possible.\n",
         "\n",
-        "Because of TensorFlow 2.x module deprecations (for example, `tf.flags` and `tf.contrib`), some changes can not be worked around by switching to `compat.v1`. Upgrading this code may require using an additional library (for example, [`absl.flags`](https://github.com/abseil/abseil-py)) or switching to a package in [tensorflow/addons](http://www.github.com/tensorflow/addons).\n",
-        "\n"
+        "Because of TensorFlow 2.x module deprecations (for example, `tf.flags` and `tf.contrib`), some changes can not be worked around by switching to `compat.v1`. Upgrading this code may require using an additional library (for example, [`absl.flags`](https://github.com/abseil/abseil-py)) or switching to a package in [tensorflow/addons](http://www.github.com/tensorflow/addons).\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "s78bbfjkXYb7"
       },
       "source": [
         "## Recommended upgrade process\n",
         "\n",
-        "The rest of this guide demonstrates how to use the upgrade script. While the upgrade script is easy to use, it is strongly recomended that you use the script as part of the following process: \n",
+        "The rest of this guide demonstrates how to use the symbol-rewriting script. While the script is easy to use, it is strongly recommended that you use the script as part of the following process: \n",
         "\n",
-        "1. **Unit Test**: Ensure that the code you’re upgrading has a unit test suite with reasonable coverage. This is Python code, so the language won’t protect you from many classes of mistakes. Also ensure that any dependency you have has already been upgraded to be compatible with TensorFlow 2.0.\n",
+        "1. **Unit Test**: Ensure that the code you’re upgrading has a unit test suite with reasonable coverage. This is Python code, so the language won’t protect you from many classes of mistakes. Also ensure that any dependency you have has already been upgraded to be compatible with TensorFlow 2.x.\n",
         "\n",
-        "1. **Install TensorFlow 1.14**: Upgrade your TensorFlow to the latest TensorFlow 1.x version, at least 1.14. This includes the final TensorFlow 2.0 API in `tf.compat.v2`.\n",
+        "1. **Install TensorFlow 1.15**: Upgrade your TensorFlow to the latest TensorFlow 1.x version, at least 1.15. This includes the final TensorFlow 2.0 API in `tf.compat.v2`.\n",
         "\n",
-        "1. **Test With 1.14**: Ensure your unit tests pass at this point. You’ll be running them repeatedly as you upgrade so starting from green is important.\n",
+        "1. **Test With 1.15**: Ensure your unit tests pass at this point. You’ll be running them repeatedly as you upgrade so starting from green is important.\n",
         "\n",
-        "1. **Run the upgrade script**: Run `tf_upgrade_v2` on your entire source tree, tests included. This will upgrade your code to a format where it only uses symbols available in TensorFlow 2.0. Deprecated symbols will be accessed with `tf.compat.v1`. These will will eventually require manual attention, but not immediately.\n",
+        "1. **Run the upgrade script**: Run `tf_upgrade_v2` on your entire source tree, tests included. This will upgrade your code to a format where it only uses symbols available in TensorFlow 2.0. Deprecated symbols will be accessed with `tf.compat.v1`. These will eventually require manual attention, but not immediately.\n",
         "\n",
-        "1. **Run the converted tests with TensorFlow 1.14**: Your code should still run fine in TensorFlow 1.14. Run your unit tests again. Any error in your tests here means there’s a bug in the upgrade script. [Please let us know](https://github.com/tensorflow/tensorflow/issues).\n",
+        "1. **Run the converted tests with TensorFlow 1.15**: Your code should still run fine in TensorFlow 1.15. Run your unit tests again. Any error in your tests here means there’s a bug in the upgrade script. [Please let us know](https://github.com/tensorflow/tensorflow/issues).\n",
         "\n",
         "1. **Check the upgrade report for warnings and errors**: The script writes a report file that explains any conversions you should double-check, or any manual action you need to take. For example: Any remaining instances of contrib will require manual action to remove. Please consult [the RFC for more instructions](https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md). \n",
         "\n",
-        "1. **Install TensorFlow 2.0**: At this point it should be safe to switch to TensorFlow 2.0\n",
+        "1. **Install TensorFlow 2.x**: At this point it should be safe to switch to TensorFlow 2.x binaries, even if you are running with legacy behaviors\n",
         "\n",
-        "1. **Test with `v1.disable_v2_behavior`**: Re-running your tests with al `v1.disable_v2_behavior()` in the tests main function should give the same results as running under 1.14.\n",
+        "1. **Test with `v1.disable_v2_behavior`**: Re-running your tests with a `v1.disable_v2_behavior()` in the tests' main function should give the same results as running under 1.15.\n",
         "\n",
-        "1. **Enable V2 Behavior**: Now that your tests work using the v2 API, you can start looking into turning on v2 behavior. Depending on how your code is written this may require some changes. See the [Migration guide](migrate.ipynb) for details."
+        "1. **Enable V2 Behavior**: Now that your tests work using the TF2 binaries, you can now begin migrating your code to avoiding `tf.estimator`s and only using supported TF2 behaviors (with no TF2 behavior disabling). See the [Migration guides](https://tensorflow.org/guide/migrate) for details."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6pwSAQEwvscP"
       },
       "source": [
-        "## Using the upgrade script\n"
+        "## Using the symbol-rewriting `tf_upgrade_v2` script\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "I9NCvDt5GwX4"
       },
       "source": [
         "### Setup\n",
         "\n",
-        "Before getting started ensure that TensorlFlow 2.0 is installed."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "YKPlcTxLtgdN"
-      },
-      "outputs": [],
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals"
+        "Before getting started ensure that TensorFlow 2.x is installed."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "DWVYbvi1WCeY"
       },
       "outputs": [],
       "source": [
-        "try:\n",
-        "  import tensorflow.compat.v2 as tf\n",
-        "except Exception:\n",
-        "  pass\n",
-        "\n",
-        "tf.enable_v2_behavior()\n",
+        "import tensorflow as tf\n",
         "\n",
         "print(tf.__version__)"
       ]
@@ -196,7 +165,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Ycy3B5PNGutU"
       },
       "source": [
@@ -205,10 +173,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "jyckoWyAZEhZ"
       },
       "outputs": [],
@@ -219,11 +185,9 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wfHOhbkgvrKr"
       },
       "source": [
-        "\n",
         "### Read the help\n",
         "\n",
         "The script should be installed with TensorFlow. Here is the builtin help:"
@@ -231,10 +195,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "m2GF-tlntqTQ"
       },
       "outputs": [],
@@ -245,7 +207,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "se9Leqjm1CZR"
       },
       "source": [
@@ -255,7 +216,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "whD5i36s1SuM"
       },
       "source": [
@@ -264,10 +224,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "mhGbYQ9HwbeU"
       },
       "outputs": [],
@@ -278,50 +236,38 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "UGO7xSyL89wX"
       },
       "source": [
-        "With TensorFlow 2.0 installed it does not run:"
+        "With TensorFlow 2.x installed it does not run:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "TD7fFphX8_qE"
       },
       "outputs": [],
       "source": [
-        "!(cd models/samples/cookbook/regression \u0026\u0026 python custom_regression.py)"
+        "!(cd models/samples/cookbook/regression && python custom_regression.py)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "iZZHu0H0wLRJ"
       },
       "source": [
         "### Single file\n",
         "\n",
-        "The upgrade script can be run on a single Python file:"
+        "The script can be run on a single Python file:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "sh"
-          ],
-          "id": ""
-        },
-        "colab": {},
-        "colab_type": "code",
+      "execution_count": null,
+      "metadata": {
         "id": "xIBZVEjkqkc5"
       },
       "outputs": [],
@@ -334,7 +280,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "L9X2lxzqqkc9"
       },
       "source": [
@@ -344,7 +289,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "r7zpuE1vWSlL"
       },
       "source": [
@@ -354,24 +298,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2q7Gtuu8SdIC"
       },
       "source": [
-        "Typical projects, including this simple example, will use much more than one file. Typically want to upgrade an entire package, so the script can also be run on a directory tree:"
+        "Typical projects, including this simple example, will use much more than one file. Typically want to update an entire package, so the script can also be run on a directory tree:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "XGqcdkAPqkc-"
       },
       "outputs": [],
       "source": [
-        "# upgrade the .py files and copy all the other files to the outtree\n",
+        "# update the .py files and copy all the other files to the outtree\n",
         "!tf_upgrade_v2 \\\n",
         "    --intree models/samples/cookbook/regression/ \\\n",
         "    --outtree regression_v2/ \\\n",
@@ -381,34 +322,30 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2S4j7sqbSowC"
       },
       "source": [
         "Note the one warning about the `dataset.make_one_shot_iterator` function.\n",
         "\n",
-        "Now the script works in with TensorFlow 2.0:\n",
+        "Now the script works in with TensorFlow 2.x:\n",
         "\n",
-        "Note that because the `tf.compat.v1` module, the converted script will also run in TensorFlow 1.14. "
+        "Note that because the `tf.compat.v1` module is included in TF 1.15, the converted script will also run in TensorFlow 1.15."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "vh0cmW3y1tX9"
       },
       "outputs": [],
       "source": [
-        "!(cd regression_v2 \u0026\u0026 python custom_regression.py 2\u003e\u00261) | tail"
+        "!(cd regression_v2 && python custom_regression.py 2>&1) | tail"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "4EgZGGkdqkdC"
       },
       "source": [
@@ -419,10 +356,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "CtHaZbVaNMGV"
       },
       "outputs": [],
@@ -433,7 +368,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "1-UIFXP3cFSa"
       },
       "source": [
@@ -443,7 +377,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "oxQeYS1TN-jv"
       },
       "source": [
@@ -452,10 +385,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "WQs9kEvVN9th"
       },
       "outputs": [],
@@ -469,10 +400,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "7uOkacZsO3XX"
       },
       "outputs": [],
@@ -480,15 +409,13 @@
         "!tf_upgrade_v2 \\\n",
         "  --infile dropout.py \\\n",
         "  --outfile dropout_v2.py \\\n",
-        "  --reportfile dropout_report.txt \u003e /dev/null"
+        "  --reportfile dropout_report.txt > /dev/null"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "m-J82-scPMGl"
       },
       "outputs": [],
@@ -499,7 +426,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "DOOLN21nTGSS"
       },
       "source": [
@@ -508,10 +434,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "SrYcJk9-TFlU"
       },
       "outputs": [],
@@ -522,7 +446,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wI_sVNp_b4C4"
       },
       "source": [
@@ -531,10 +454,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "uzuY-bOvYBS7"
       },
       "outputs": [],
@@ -542,13 +463,12 @@
         "!tf_upgrade_v2 \\\n",
         "    --intree models/research/deeplab \\\n",
         "    --outtree deeplab_v2 \\\n",
-        "    --reportfile deeplab_report.txt \u003e /dev/null"
+        "    --reportfile deeplab_report.txt > /dev/null"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "FLhw3fm8drae"
       },
       "source": [
@@ -557,10 +477,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "4YYLRxWJdSvQ"
       },
       "outputs": [],
@@ -571,7 +489,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qtTC-cAZdEBy"
       },
       "source": [
@@ -580,10 +497,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "UVTNOohlcyVZ"
       },
       "outputs": [],
@@ -594,7 +509,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "gGBeDaFVRJ5l"
       },
       "source": [
@@ -604,7 +518,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "BnfCxB7SVtTO"
       },
       "source": [
@@ -613,10 +526,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "XdaVXCPWQCC5"
       },
       "outputs": [],
@@ -626,23 +537,19 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "c0tvRJLGRYEb"
       },
       "outputs": [],
       "source": [
-        "!tf_upgrade_v2 --mode SAFETY --infile dropout.py --outfile dropout_v2_safe.py \u003e /dev/null"
+        "!tf_upgrade_v2 --mode SAFETY --infile dropout.py --outfile dropout_v2_safe.py > /dev/null"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "91suN2RaRfIV"
       },
       "outputs": [],
@@ -653,33 +560,30 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "EOzTF7xbZqqW"
       },
       "source": [
-        "As you can see this doesn't upgrade your code, but does allow TensorFlow 1 code to run in TensorFlow 2"
+        "As you can see this doesn't upgrade your code, but does allow TensorFlow 1 code to run against TensorFlow 2 binaries. Note that this does not mean your code is running supported TF 2.x behaviors!"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "jGfXVApkqkdG"
       },
       "source": [
-        "\n",
         "## Caveats\n",
         "\n",
-        "- Do not update parts of your code manually before running this script. In particular, functions that have had reordered arguments like `tf.argmax` or `tf.batch_to_space` cause the script to incorrectly add keyword arguments that mismap your existing code.\n",
+        "- Do not update parts of your code manually before running this script. In particular, functions that have had reordered arguments like `tf.math.argmax` or `tf.batch_to_space` cause the script to incorrectly add keyword arguments that mismap your existing code.\n",
         "\n",
-        "- The script assumes that `tensorflow` is imported using `import tensorflow as tf`.\n",
+        "- The script assumes that `tensorflow` is imported using `import tensorflow as tf`, or `import tensorflow.compat.v1 as tf`.\n",
         "\n",
         "- This script does not reorder arguments. Instead, the script adds keyword arguments to functions that have their arguments reordered.\n",
         "\n",
-        "- Check out [tf2up.ml](http://tf2up.ml) for a convenient tool to upgrade Jupyter\n",
+        "- Check out [tf2up.ml](https://github.com/lc0/tf2up) for a convenient tool to upgrade Jupyter\n",
         "  notebooks and Python files in a GitHub repository.\n",
         "\n",
-        "To report upgrade script bugs or make feature requests, please file an issue on [GitHub](https://github.com/tensorflow/tensorflow/issues). And if you’re testing TensorFlow 2.0, we want to hear about it! Join the [TF 2.0 Testing community](https://groups.google.com/a/tensorflow.org/forum/#!forum/testing) and send questions and discussion to [testing@tensorflow.org](mailto:testing@tensorflow.org)."
+        "To report upgrade script bugs or make feature requests, please file an issue on [GitHub](https://github.com/tensorflow/tensorflow/issues)."
       ]
     }
   ],
@@ -687,8 +591,6 @@
     "colab": {
       "collapsed_sections": [],
       "name": "upgrade.ipynb",
-      "private_outputs": true,
-      "provenance": [],
       "toc_visible": true
     },
     "kernelspec": {
diff --git a/site/en/guide/migrate/validate_correctness.ipynb b/site/en/guide/migrate/validate_correctness.ipynb
new file mode 100644
index 00000000000..a0555cdd55c
--- /dev/null
+++ b/site/en/guide/migrate/validate_correctness.ipynb
@@ -0,0 +1,1291 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6bYaCABobL5q"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "FlUw7tSKbtg4"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yAMJsAn7NDbc"
+      },
+      "source": [
+        "# Validating correctness & numerical equivalence"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vyddl2kckpdN"
+      },
+      "source": [
+        "When migrating your TensorFlow code from TF1.x to TF2, it is a good practice to ensure that your migrated code behaves the same way in TF2 as it did in TF1.x. \n",
+        "\n",
+        "This guide covers migration code examples with the `tf.compat.v1.keras.utils.track_tf1_style_variables` modeling shim applied to `tf.keras.layers.Layer` methods. Read the [model mapping guide](./model_mapping.ipynb) to find out more about the TF2 modeling shims.\n",
+        "\n",
+        "This guide details approaches you can use to: \n",
+        "* Validate the correctness of the results obtained from training models using the migrated code \n",
+        "* Validate the numerical equivalence of your code across TensorFlow versions"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TaYgaekzOAHf"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FkHX044DzVsd"
+      },
+      "outputs": [],
+      "source": [
+        "!pip uninstall -y -q tensorflow"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "M1ZgieHtyzKI"
+      },
+      "outputs": [],
+      "source": [
+        "# Install tf-nightly as the DeterministicRandomTestTool is available only in\n",
+        "# Tensorflow 2.8\n",
+        "!pip install -q tf-nightly"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ohYETq4NCX4J"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q tf_slim"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MFey2HxcktP6"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as v1\n",
+        "\n",
+        "import numpy as np\n",
+        "import tf_slim as slim\n",
+        "import sys\n",
+        "\n",
+        "\n",
+        "from contextlib import contextmanager"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OriidSSAmRtW"
+      },
+      "outputs": [],
+      "source": [
+        "!git clone --depth=1 https://github.com/tensorflow/models.git\n",
+        "import models.research.slim.nets.inception_resnet_v2 as inception"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TRacYNxnN-nk"
+      },
+      "source": [
+        "If you're putting a nontrivial chunk of forward pass code into the shim, you want to know that it is behaving the same way as it did in TF1.x. For example, consider trying to put an entire TF-Slim Inception-Resnet-v2 model into the shim as such:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IijQZtxeaErg"
+      },
+      "outputs": [],
+      "source": [
+        "# TF1 Inception resnet v2 forward pass based on slim layers\n",
+        "def inception_resnet_v2(inputs, num_classes, is_training):\n",
+        "  with slim.arg_scope(\n",
+        "    inception.inception_resnet_v2_arg_scope(batch_norm_scale=True)):\n",
+        "    return inception.inception_resnet_v2(inputs, num_classes, is_training=is_training)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Z_-Oxg9OlSd4"
+      },
+      "outputs": [],
+      "source": [
+        "class InceptionResnetV2(tf.keras.layers.Layer):\n",
+        "  \"\"\"Slim InceptionResnetV2 forward pass as a Keras layer\"\"\"\n",
+        "\n",
+        "  def __init__(self, num_classes, **kwargs):\n",
+        "    super().__init__(**kwargs)\n",
+        "    self.num_classes = num_classes\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs, training=None):\n",
+        "    is_training = training or False \n",
+        "    \n",
+        "    # Slim does not accept `None` as a value for is_training,\n",
+        "    # Keras will still pass `None` to layers to construct functional models\n",
+        "    # without forcing the layer to always be in training or in inference.\n",
+        "    # However, `None` is generally considered to run layers in inference.\n",
+        "    \n",
+        "    with slim.arg_scope(\n",
+        "        inception.inception_resnet_v2_arg_scope(batch_norm_scale=True)):\n",
+        "      return inception.inception_resnet_v2(\n",
+        "          inputs, self.num_classes, is_training=is_training)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EqFmpktjlvh9"
+      },
+      "source": [
+        "As it so happens, this layer actually works perfectly fine out of the box (complete with accurate regularization loss tracking). \n",
+        "\n",
+        "However, this is not something you want to take for granted. Follow the below steps to verify that it is actually behaving as it did in TF1.x, down to observing perfect numerical equivalence. These steps can also help you triangulate what part of the forward pass is causing a divergence from TF1.x (identify if the divergence arises in the model forward pass as opposed to a different part of the model)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mmgubd9vkevp"
+      },
+      "source": [
+        "## Step 1: Verify variables are only created once\n",
+        "\n",
+        "The very first thing you should verify is that you have correctly built the model in a way that reuses variables in each call rather than accidentally creating and using new variables each time. For example, if your model creates a new Keras layer or calls `tf.Variable` in each forward pass call then it is most likely failing to capture variables and creating new ones each time.\n",
+        "\n",
+        "Below are two context manager scopes you can use to detect when your model is creating new variables and debug which part of the model is doing it."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VMTfTXC0zW97"
+      },
+      "outputs": [],
+      "source": [
+        "@contextmanager\n",
+        "def assert_no_variable_creations():\n",
+        "  \"\"\"Assert no variables are created in this context manager scope.\"\"\"\n",
+        "  def invalid_variable_creator(next_creator, **kwargs):\n",
+        "    raise ValueError(\"Attempted to create a new variable instead of reusing an existing one. Args: {}\".format(kwargs))\n",
+        "\n",
+        "  with tf.variable_creator_scope(invalid_variable_creator):\n",
+        "    yield\n",
+        "\n",
+        "@contextmanager\n",
+        "def catch_and_raise_created_variables():\n",
+        "  \"\"\"Raise all variables created within this context manager scope (if any).\"\"\"\n",
+        "  created_vars = []\n",
+        "  def variable_catcher(next_creator, **kwargs):\n",
+        "    var = next_creator(**kwargs)\n",
+        "    created_vars.append(var)\n",
+        "    return var\n",
+        "\n",
+        "  with tf.variable_creator_scope(variable_catcher):\n",
+        "    yield\n",
+        "  if created_vars:\n",
+        "    raise ValueError(\"Created vars:\", created_vars)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WOKUtciktQqv"
+      },
+      "source": [
+        "The first scope (`assert_no_variable_creations()`) will raise an error immediately once you try creating a variable within the scope. This allows you to inspect the stacktrace (and use interactive debugging) to figure out exactly what lines of code created a variable instead of reusing an existing one.\n",
+        "\n",
+        "The second scope (`catch_and_raise_created_variables()`) will raise an exception at the end of the scope if any variables ended up being created. This exception will include the list of all variables created in the scope. This is useful for figuring out what the set of all weights your model is creating is in case you can spot general patterns. However, it is less useful for identifying the exact lines of code where those variables got created.\n",
+        "\n",
+        "Use both scopes below to verify that the shim-based InceptionResnetV2 layer does not create any new variables after the first call (presumably reusing them)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "O9FAGotiuLbK"
+      },
+      "outputs": [],
+      "source": [
+        "model = InceptionResnetV2(1000)\n",
+        "height, width = 299, 299\n",
+        "num_classes = 1000\n",
+        "\n",
+        "inputs = tf.ones( (1, height, width, 3))\n",
+        "# Create all weights on the first call\n",
+        "model(inputs)\n",
+        "\n",
+        "# Verify that no new weights are created in followup calls\n",
+        "with assert_no_variable_creations():\n",
+        "  model(inputs)\n",
+        "with catch_and_raise_created_variables():\n",
+        "  model(inputs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9ylT-EIhu1lK"
+      },
+      "source": [
+        "In the example below, observe how these decorators work on a layer that incorrectly creates new weights each time instead of reusing existing ones."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gXqhPQWWtMAw"
+      },
+      "outputs": [],
+      "source": [
+        "class BrokenScalingLayer(tf.keras.layers.Layer):\n",
+        "  \"\"\"Scaling layer that incorrectly creates new weights each time:\"\"\"\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs):\n",
+        "    var = tf.Variable(initial_value=2.0)\n",
+        "    bias = tf.Variable(initial_value=2.0, name='bias')\n",
+        "    return inputs * var + bias"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ztUKlMdGvHSq"
+      },
+      "outputs": [],
+      "source": [
+        "model = BrokenScalingLayer()\n",
+        "inputs = tf.ones( (1, height, width, 3))\n",
+        "model(inputs)\n",
+        "\n",
+        "try:\n",
+        "  with assert_no_variable_creations():\n",
+        "    model(inputs)\n",
+        "except ValueError as err:\n",
+        "  import traceback\n",
+        "  traceback.print_exc()\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6VyfMJ50vZqZ"
+      },
+      "outputs": [],
+      "source": [
+        "model = BrokenScalingLayer()\n",
+        "inputs = tf.ones( (1, height, width, 3))\n",
+        "model(inputs)\n",
+        "\n",
+        "try:\n",
+        "  with catch_and_raise_created_variables():\n",
+        "    model(inputs)\n",
+        "except ValueError as err:\n",
+        "  print(err)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JDaiTArcv49M"
+      },
+      "source": [
+        "You can fix the layer by making sure it only creates the weights once and then reuses them each time."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FN1Oa10iviv8"
+      },
+      "outputs": [],
+      "source": [
+        "class FixedScalingLayer(tf.keras.layers.Layer):\n",
+        "  \"\"\"Scaling layer that incorrectly creates new weights each time:\"\"\"\n",
+        "  def __init__(self):\n",
+        "    super().__init__()\n",
+        "    self.var = None\n",
+        "    self.bias = None\n",
+        "\n",
+        "  @tf.compat.v1.keras.utils.track_tf1_style_variables\n",
+        "  def call(self, inputs):\n",
+        "    if self.var is None:\n",
+        "      self.var = tf.Variable(initial_value=2.0)\n",
+        "      self.bias = tf.Variable(initial_value=2.0, name='bias')\n",
+        "    return inputs * self.var + self.bias\n",
+        "\n",
+        "model = FixedScalingLayer()\n",
+        "inputs = tf.ones( (1, height, width, 3))\n",
+        "model(inputs)\n",
+        "\n",
+        "with assert_no_variable_creations():\n",
+        "  model(inputs)\n",
+        "with catch_and_raise_created_variables():\n",
+        "  model(inputs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MuiZZ7ktwCcn"
+      },
+      "source": [
+        "### Troubleshooting\n",
+        "\n",
+        "Here are some common reasons why your model might accidentally be creating new weights instead of reusing existing ones:\n",
+        "\n",
+        "1. It uses an explicit `tf.Variable` call without reusing already-created `tf.Variables`. Fix this by first checking if it has not been created then reusing the existing ones.\n",
+        "2. It creates a Keras layer or model directly in the forward pass each time (as opposed to `tf.compat.v1.layers`). Fix this by first checking if it has not been created then reusing the existing ones.\n",
+        "3. It is built on top of `tf.compat.v1.layers` but fails to assign all `compat.v1.layers` an explicit name or to wrap your `compat.v1.layer` usage inside of a named `variable_scope`, causing the autogenerated layer names to increment in each model call. Fix this by putting a named `tf.compat.v1.variable_scope` inside your shim-decorated method that wraps all of your `tf.compat.v1.layers` usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "V4iZLV9BnwKM"
+      },
+      "source": [
+        "## Step 2: Check that variable counts, names, and shapes match\n",
+        "\n",
+        "The second step is to make sure your layer running in TF2 creates the same number of weights, with the same shapes, as the corresponding code does in TF1.x.\n",
+        "\n",
+        "You can do a mix of manually checking them to see that they match, and doing the checks programmatically in a unit test as shown below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m_aqag5fpun5"
+      },
+      "outputs": [],
+      "source": [
+        "# Build the forward pass inside a TF1.x graph, and \n",
+        "# get the counts, shapes, and names of the variables\n",
+        "graph = tf.Graph()\n",
+        "with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "  height, width = 299, 299\n",
+        "  num_classes = 1000\n",
+        "  inputs = tf.ones( (1, height, width, 3))\n",
+        "\n",
+        "  out, endpoints = inception_resnet_v2(inputs, num_classes, is_training=False)\n",
+        "\n",
+        "  tf1_variable_names_and_shapes = {\n",
+        "      var.name: (var.trainable, var.shape) for var in tf.compat.v1.global_variables()}\n",
+        "  num_tf1_variables = len(tf.compat.v1.global_variables())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WT1-cm99vfNU"
+      },
+      "source": [
+        "Next, do the same for the shim-wrapped layer in TF2.\n",
+        "Notice that the model is also called multiple times before grabbing the weights. This is done to effectively test for variable reuse."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S7ND-lBSqmnE"
+      },
+      "outputs": [],
+      "source": [
+        "height, width = 299, 299\n",
+        "num_classes = 1000\n",
+        "\n",
+        "model = InceptionResnetV2(num_classes)\n",
+        "# The weights will not be created until you call the model\n",
+        "\n",
+        "inputs = tf.ones( (1, height, width, 3))\n",
+        "# Call the model multiple times before checking the weights, to verify variables\n",
+        "# get reused rather than accidentally creating additional variables\n",
+        "out, endpoints = model(inputs, training=False)\n",
+        "out, endpoints = model(inputs, training=False)\n",
+        "\n",
+        "# Grab the name: shape mapping and the total number of variables separately,\n",
+        "# because in TF2 variables can be created with the same name\n",
+        "num_tf2_variables = len(model.variables)\n",
+        "tf2_variable_names_and_shapes = {\n",
+        "    var.name: (var.trainable, var.shape) for var in model.variables}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pY2P_4wqsOYw"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that the variable counts, names, and shapes all match:\n",
+        "assert num_tf1_variables == num_tf2_variables\n",
+        "assert tf1_variable_names_and_shapes == tf2_variable_names_and_shapes"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N4YKJzSVwWkc"
+      },
+      "source": [
+        "The shim-based InceptionResnetV2 layer passes this test. However, in the case where they don't match, you can run it through a diff (text or other) to see where the differences are.\n",
+        "\n",
+        "This can provide a clue as to what part of the model isn't behaving as expected. With eager execution you can use pdb, interactive debugging, and breakpoints to dig into the parts of the model that seem suspicious, and debug what is going wrong in more depth."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2gYrt-_0xpRM"
+      },
+      "source": [
+        "### Troubleshooting\n",
+        "\n",
+        "* Pay close attention to the names of any variables created directly by explicit `tf.Variable` calls and Keras layers/models as their variable name generation semantics may differ slightly between TF1.x graphs and TF2 functionality such as eager execution and `tf.function` even if everything else is working properly. If this is the case for you, adjust your test to account for any slightly different naming semantics.\n",
+        "\n",
+        "* You may sometimes find that the `tf.Variable`s, `tf.keras.layers.Layer`s, or `tf.keras.Model`s created in your training loop's forward pass are missing from your TF2 variables list even if they were captured by the variables collection in TF1.x. Fix this by assigning the variables/layers/models that your forward pass creates to instance attributes in your model. See [here](https://www.tensorflow.org/guide/keras/custom_layers_and_models) for more info."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fOQJ_hUGnzkq"
+      },
+      "source": [
+        "## Step 3: Reset all variables, check numerical equivalence with all randomness disabled\n",
+        "\n",
+        "The next step is to verify numerical equivalence for both the actual outputs and the regularization loss tracking when you fix the model such that there is no random number generation involved (such as during inference).\n",
+        "\n",
+        "The exact way to do this may depend on your specific model, but in most models (such as this one), you can do this by:\n",
+        "1. Initializing the weights to the same value with no randomness. This can be done by resetting them to a fixed value after they have been created.\n",
+        "2. Running the model in inference mode to avoid triggering any dropout layers which can be sources of randomness.\n",
+        "\n",
+        "The following code demonstrates how you can compare the TF1.x and TF2 results this way."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kL4PzD2Cxzmp"
+      },
+      "outputs": [],
+      "source": [
+        "graph = tf.Graph()\n",
+        "with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "  height, width = 299, 299\n",
+        "  num_classes = 1000\n",
+        "  inputs = tf.ones( (1, height, width, 3))\n",
+        "\n",
+        "  out, endpoints = inception_resnet_v2(inputs, num_classes, is_training=False)\n",
+        "\n",
+        "  # Rather than running the global variable initializers,\n",
+        "  # reset all variables to a constant value\n",
+        "  var_reset = tf.group([var.assign(tf.ones_like(var) * 0.001) for var in tf.compat.v1.global_variables()])\n",
+        "  sess.run(var_reset)\n",
+        "\n",
+        "  # Grab the outputs & regularization loss\n",
+        "  reg_losses = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.REGULARIZATION_LOSSES)\n",
+        "  tf1_regularization_loss = sess.run(tf.math.add_n(reg_losses))\n",
+        "  tf1_output = sess.run(out)\n",
+        "\n",
+        "print(\"Regularization loss:\", tf1_regularization_loss)\n",
+        "tf1_output[0][:5]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IKkoM_x72rUa"
+      },
+      "source": [
+        "Get the TF2 results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kb086gJwzsNo"
+      },
+      "outputs": [],
+      "source": [
+        "height, width = 299, 299\n",
+        "num_classes = 1000\n",
+        "\n",
+        "model = InceptionResnetV2(num_classes)\n",
+        "\n",
+        "inputs = tf.ones((1, height, width, 3))\n",
+        "# Call the model once to create the weights\n",
+        "out, endpoints = model(inputs, training=False)\n",
+        "\n",
+        "# Reset all variables to the same fixed value as above, with no randomness\n",
+        "for var in model.variables:\n",
+        "  var.assign(tf.ones_like(var) * 0.001)\n",
+        "tf2_output, endpoints = model(inputs, training=False)\n",
+        "\n",
+        "# Get the regularization loss\n",
+        "tf2_regularization_loss = tf.math.add_n(model.losses)\n",
+        "\n",
+        "print(\"Regularization loss:\", tf2_regularization_loss)\n",
+        "tf2_output[0][:5]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CUfWqlgIK6ej"
+      },
+      "outputs": [],
+      "source": [
+        "# Create a dict of tolerance values\n",
+        "tol_dict={'rtol':1e-06, 'atol':1e-05}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R-C07eTo0WTr"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that the regularization loss and output both match\n",
+        "# when we fix the weights and avoid randomness by running inference:\n",
+        "np.testing.assert_allclose(tf1_regularization_loss, tf2_regularization_loss.numpy(), **tol_dict)\n",
+        "np.testing.assert_allclose(tf1_output, tf2_output.numpy(), **tol_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5UUq_Fuc2zDO"
+      },
+      "source": [
+        "The numbers match between TF1.x and TF2 when you remove sources of randomness, and the TF2-compatible `InceptionResnetV2` layer passes the test.\n",
+        "\n",
+        "If you are observing the results diverging for your own models, you can use printing or pdb and interactive debugging to identify where and why the results start to diverge. Eager execution can make this significantly easier. You can also use an ablation approach to run only small portions of the model on fixed intermediate inputs and isolate where the divergence happens.\n",
+        "\n",
+        "Conveniently, many slim nets (and other models) also expose intermediate endpoints that you can probe."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "btRbak-0ou15"
+      },
+      "source": [
+        "## Step 4: Align random number generation, check numerical equivalence in both training and inference\n",
+        "\n",
+        "The final step is to verify that the TF2 model numerically matches the TF1.x model, even when accounting for random number generation in variable initialization and in the forward pass itself (such as dropout layers during the forward pass).\n",
+        "\n",
+        "You can do this by using the testing tool below to make random number generation semantics match between TF1.x graphs/sessions and eager execution."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jYq-JHiC39QC"
+      },
+      "source": [
+        "TF1 legacy graphs/sessions and TF2 eager execution use different stateful random number generation semantics.\n",
+        "\n",
+        "In `tf.compat.v1.Session`s, if no seeds are specified, the random number generation depends on how many operations are in the graph at the time when the random operation is added, and how many times the graph is run. In eager execution, stateful random number generation depends on the global seed, the operation random seed, and how many times the operation with the operation with the given random seed is run. See \n",
+        "`tf.random.set_seed` for more info."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BQbb8Hyk5YVi"
+      },
+      "source": [
+        "The following [`v1.keras.utils.DeterministicRandomTestTool`](https://www.tensorflow.org/api_docs/python/tf/compat/v1/keras/utils/DeterministicRandomTestTool) class provides a context manager `scope()` that can make stateful random operations use the same seed across both TF1 graphs/sessions and eager execution.\n",
+        "\n",
+        "The tool provides two testing modes: \n",
+        "1. `constant` which uses the same seed for every single operation no matter how many times it has been called and,\n",
+        "2. `num_random_ops` which uses the number of previously-observed stateful random operations as the operation seed.\n",
+        "\n",
+        "This applies both to the stateful random operations used for creating and initializing variables, and to the stateful random operations used in computation (such as for dropout layers)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MoyZenhGHDA-"
+      },
+      "source": [
+        "Generate three random tensors to show how to use this tool to make stateful random number generation match between sessions and eager execution."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DDFfjrbXEWED"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool()\n",
+        "with random_tool.scope():\n",
+        "  graph = tf.Graph()\n",
+        "  with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "    a = tf.random.uniform(shape=(3,1))\n",
+        "    a = a * 3\n",
+        "    b = tf.random.uniform(shape=(3,3))\n",
+        "    b = b * 3\n",
+        "    c = tf.random.uniform(shape=(3,3))\n",
+        "    c = c * 3\n",
+        "    graph_a, graph_b, graph_c = sess.run([a, b, c])\n",
+        "\n",
+        "graph_a, graph_b, graph_c"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o9bkdPuTFpYr"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool()\n",
+        "with random_tool.scope():\n",
+        "  a = tf.random.uniform(shape=(3,1))\n",
+        "  a = a * 3\n",
+        "  b = tf.random.uniform(shape=(3,3))\n",
+        "  b = b * 3\n",
+        "  c = tf.random.uniform(shape=(3,3))\n",
+        "  c = c * 3\n",
+        "\n",
+        "a, b, c"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qRJYFydsGIbF"
+      },
+      "outputs": [],
+      "source": [
+        "# Demonstrate that the generated random numbers match\n",
+        "np.testing.assert_allclose(graph_a, a.numpy(), **tol_dict)\n",
+        "np.testing.assert_allclose(graph_b, b.numpy(), **tol_dict)\n",
+        "np.testing.assert_allclose(graph_c, c.numpy(), **tol_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J8IWCnS-WFrB"
+      },
+      "source": [
+        "However, notice that in `constant` mode, because `b` and `c` were generated with the same seed and have the same shape, they will have exactly the same values."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IdxV89q2WPid"
+      },
+      "outputs": [],
+      "source": [
+        "np.testing.assert_allclose(b.numpy(), c.numpy(), **tol_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vQTm7joHHh57"
+      },
+      "source": [
+        "### Trace order\n",
+        "If you are worried about some random numbers matching in `constant` mode reducing your confidence in your numerical equivalence test (for example if several weights take on the same initializations), you can use the `num_random_ops` mode to avoid this. In the `num_random_ops` mode, the generated random numbers will depend on the ordering of random ops in the program."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "L-AeD148VygJ"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  graph = tf.Graph()\n",
+        "  with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "    a = tf.random.uniform(shape=(3,1))\n",
+        "    a = a * 3\n",
+        "    b = tf.random.uniform(shape=(3,3))\n",
+        "    b = b * 3\n",
+        "    c = tf.random.uniform(shape=(3,3))\n",
+        "    c = c * 3\n",
+        "    graph_a, graph_b, graph_c = sess.run([a, b, c])\n",
+        "\n",
+        "graph_a, graph_b, graph_c"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CedD41NuVygK"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  a = tf.random.uniform(shape=(3,1))\n",
+        "  a = a * 3\n",
+        "  b = tf.random.uniform(shape=(3,3))\n",
+        "  b = b * 3\n",
+        "  c = tf.random.uniform(shape=(3,3))\n",
+        "  c = c * 3\n",
+        "\n",
+        "a, b, c"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5We2xSnLVygL"
+      },
+      "outputs": [],
+      "source": [
+        "# Demonstrate that the generated random numbers match\n",
+        "np.testing.assert_allclose(graph_a, a.numpy(), **tol_dict)\n",
+        "np.testing.assert_allclose(graph_b, b.numpy(), **tol_dict )\n",
+        "np.testing.assert_allclose(graph_c, c.numpy(), **tol_dict)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BBFG1xehWneM"
+      },
+      "outputs": [],
+      "source": [
+        "# Demonstrate that with the 'num_random_ops' mode,\n",
+        "# b & c took on different values even though\n",
+        "# their generated shape was the same\n",
+        "assert not np.allclose(b.numpy(), c.numpy(), **tol_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OfX_VexcVqSA"
+      },
+      "source": [
+        "However, notice that in this mode random generation is sensitive to program order, and so the following generated random numbers do not match."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cZt__ElEIDl_"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  a = tf.random.uniform(shape=(3,1))\n",
+        "  a = a * 3\n",
+        "  b = tf.random.uniform(shape=(3,3))\n",
+        "  b = b * 3\n",
+        "\n",
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  b_prime = tf.random.uniform(shape=(3,3))\n",
+        "  b_prime = b_prime * 3\n",
+        "  a_prime = tf.random.uniform(shape=(3,1))\n",
+        "  a_prime = a_prime * 3\n",
+        "\n",
+        "assert not np.allclose(a.numpy(), a_prime.numpy())\n",
+        "assert not np.allclose(b.numpy(), b_prime.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nHhOLHyQIkAe"
+      },
+      "source": [
+        "To allow for debugging variations due to tracing order, `DeterministicRandomTestTool` in `num_random_ops` mode allows you to see how many random operations have been traced with the `operation_seed` property."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "33RCSICuJEyV"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  print(random_tool.operation_seed)\n",
+        "  a = tf.random.uniform(shape=(3,1))\n",
+        "  a = a * 3\n",
+        "  print(random_tool.operation_seed)\n",
+        "  b = tf.random.uniform(shape=(3,3))\n",
+        "  b = b * 3\n",
+        "  print(random_tool.operation_seed)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bkQD3NpOMxIv"
+      },
+      "source": [
+        "If you need to account for varying trace order in your tests, you can even set the auto-incrementing `operation_seed` explicitly. For example, you can use this to make random number generation match across two different program orders."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6W4sS_wOM8CH"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  print(random_tool.operation_seed)\n",
+        "  a = tf.random.uniform(shape=(3,1))\n",
+        "  a = a * 3\n",
+        "  print(random_tool.operation_seed)\n",
+        "  b = tf.random.uniform(shape=(3,3))\n",
+        "  b = b * 3\n",
+        "\n",
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  random_tool.operation_seed = 1\n",
+        "  b_prime = tf.random.uniform(shape=(3,3))\n",
+        "  b_prime = b_prime * 3\n",
+        "  random_tool.operation_seed = 0\n",
+        "  a_prime = tf.random.uniform(shape=(3,1))\n",
+        "  a_prime = a_prime * 3\n",
+        "\n",
+        "np.testing.assert_allclose(a.numpy(), a_prime.numpy(), **tol_dict)\n",
+        "np.testing.assert_allclose(b.numpy(), b_prime.numpy(), **tol_dict)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bP5Kx1OcNbvM"
+      },
+      "source": [
+        "However, `DeterministicRandomTestTool` disallows reusing already-used operation seeds, so make sure the auto-incremented sequences cannot overlap. This is because eager execution generates different numbers for follow-on usages of the same operation seed while TF1 graphs and sessions do not, so raising an error helps keep session and eager stateful random number generation in line."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GmBgg5hzNa5H"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  random_tool.operation_seed = 1\n",
+        "  b_prime = tf.random.uniform(shape=(3,3))\n",
+        "  b_prime = b_prime * 3\n",
+        "  random_tool.operation_seed = 0\n",
+        "  a_prime = tf.random.uniform(shape=(3,1))\n",
+        "  a_prime = a_prime * 3\n",
+        "  try:\n",
+        "    c = tf.random.uniform(shape=(3,1))\n",
+        "    raise RuntimeError(\"An exception should have been raised before this, \" +\n",
+        "                     \"because the auto-incremented operation seed will \" +\n",
+        "                     \"overlap an already-used value\")\n",
+        "  except ValueError as err:\n",
+        "    print(err)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "U-bLOeCmOn-4"
+      },
+      "source": [
+        "### Verifying Inference\n",
+        "\n",
+        "You can now use the `DeterministicRandomTestTool` to make sure the `InceptionResnetV2` model matches in inference, even when using the random weight initialization. For a stronger test condition due to matching program order, use the `num_random_ops` mode."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8TWOrflkPa7T"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  graph = tf.Graph()\n",
+        "  with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "    height, width = 299, 299\n",
+        "    num_classes = 1000\n",
+        "    inputs = tf.ones( (1, height, width, 3))\n",
+        "\n",
+        "    out, endpoints = inception_resnet_v2(inputs, num_classes, is_training=False)\n",
+        "\n",
+        "    # Initialize the variables\n",
+        "    sess.run(tf.compat.v1.global_variables_initializer())\n",
+        "\n",
+        "    # Grab the outputs & regularization loss\n",
+        "    reg_losses = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.REGULARIZATION_LOSSES)\n",
+        "    tf1_regularization_loss = sess.run(tf.math.add_n(reg_losses))\n",
+        "    tf1_output = sess.run(out)\n",
+        "\n",
+        "  print(\"Regularization loss:\", tf1_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Qcx6ur4KPMI1"
+      },
+      "outputs": [],
+      "source": [
+        "height, width = 299, 299\n",
+        "num_classes = 1000\n",
+        "\n",
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  model = InceptionResnetV2(num_classes)\n",
+        "\n",
+        "  inputs = tf.ones((1, height, width, 3))\n",
+        "  tf2_output, endpoints = model(inputs, training=False)\n",
+        "\n",
+        "  # Grab the regularization loss as well\n",
+        "  tf2_regularization_loss = tf.math.add_n(model.losses)\n",
+        "\n",
+        "print(\"Regularization loss:\", tf2_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m_SS2b6qPFl1"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that the regularization loss and output both match\n",
+        "# when using the DeterministicRandomTestTool:\n",
+        "np.testing.assert_allclose(tf1_regularization_loss, tf2_regularization_loss.numpy(), **tol_dict)\n",
+        "np.testing.assert_allclose(tf1_output, tf2_output.numpy(), **tol_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TKSktIRaP-5b"
+      },
+      "source": [
+        "### Verifying Training\n",
+        "\n",
+        "Because `DeterministicRandomTestTool` works for *all* stateful random operations (including both weight initialization and computation such as dropout layers), you can use it to verify the models match in training mode as well. You can again use the `num_random_ops` mode because the program order of the stateful random ops matches."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nMBFVa1kQTJH"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  graph = tf.Graph()\n",
+        "  with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "    height, width = 299, 299\n",
+        "    num_classes = 1000\n",
+        "    inputs = tf.ones( (1, height, width, 3))\n",
+        "\n",
+        "    out, endpoints = inception_resnet_v2(inputs, num_classes, is_training=True)\n",
+        "\n",
+        "    # Initialize the variables\n",
+        "    sess.run(tf.compat.v1.global_variables_initializer())\n",
+        "\n",
+        "    # Grab the outputs & regularization loss\n",
+        "    reg_losses = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.REGULARIZATION_LOSSES)\n",
+        "    tf1_regularization_loss = sess.run(tf.math.add_n(reg_losses))\n",
+        "    tf1_output = sess.run(out)\n",
+        "\n",
+        "  print(\"Regularization loss:\", tf1_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-jlBkwI5QTJI"
+      },
+      "outputs": [],
+      "source": [
+        "height, width = 299, 299\n",
+        "num_classes = 1000\n",
+        "\n",
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool(mode='num_random_ops')\n",
+        "with random_tool.scope():\n",
+        "  model = InceptionResnetV2(num_classes)\n",
+        "\n",
+        "  inputs = tf.ones((1, height, width, 3))\n",
+        "  tf2_output, endpoints = model(inputs, training=True)\n",
+        "\n",
+        "  # Grab the regularization loss as well\n",
+        "  tf2_regularization_loss = tf.math.add_n(model.losses)\n",
+        "\n",
+        "print(\"Regularization loss:\", tf2_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IL9mjTLnQTJJ"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that the regularization loss and output both match\n",
+        "# when using the DeterministicRandomTestTool\n",
+        "np.testing.assert_allclose(tf1_regularization_loss, tf2_regularization_loss.numpy(), **tol_dict)\n",
+        "np.testing.assert_allclose(tf1_output, tf2_output.numpy(), **tol_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uJTZvmfnQqZH"
+      },
+      "source": [
+        "You have now verified that the `InceptionResnetV2` model running eagerly with decorators around `tf.keras.layers.Layer` numerically matches the slim network running in TF1 graphs and sessions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xpOAei5vRAPa"
+      },
+      "source": [
+        "Note: When using the `DeterministicRandomTestTool` in `num_random_ops` mode, it is suggested you directly use and call the `tf.keras.layers.Layer` method decorator when testing for numerical equivalence. Embedding it within a Keras functional model or other Keras models can produce differences in stateful random operation tracing order that can be tricky to reason about or match exactly when comparing TF1.x graphs/sessions and eager execution. \n",
+        "\n",
+        "For example, calling the `InceptionResnetV2` layer directly with `training=True` interleaves variable initialization with the dropout order according to the network creation order.\n",
+        "\n",
+        "On the other hand, first putting the `tf.keras.layers.Layer` decorator in a Keras functional model and only then calling the model with `training=True` is equivalent to initializing all variables then using the dropout layer. This produces a different tracing order and a different set of random numbers.\n",
+        "\n",
+        "However, the default `mode='constant'` is not sensitive to these differences in tracing order and will pass without extra work even when embedding the layer in a Keras functional model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0dSR4ZNvYNYm"
+      },
+      "outputs": [],
+      "source": [
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool()\n",
+        "with random_tool.scope():\n",
+        "  graph = tf.Graph()\n",
+        "  with graph.as_default(), tf.compat.v1.Session(graph=graph) as sess:\n",
+        "    height, width = 299, 299\n",
+        "    num_classes = 1000\n",
+        "    inputs = tf.ones( (1, height, width, 3))\n",
+        "\n",
+        "    out, endpoints = inception_resnet_v2(inputs, num_classes, is_training=True)\n",
+        "\n",
+        "    # Initialize the variables\n",
+        "    sess.run(tf.compat.v1.global_variables_initializer())\n",
+        "\n",
+        "    # Get the outputs & regularization losses\n",
+        "    reg_losses = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.REGULARIZATION_LOSSES)\n",
+        "    tf1_regularization_loss = sess.run(tf.math.add_n(reg_losses))\n",
+        "    tf1_output = sess.run(out)\n",
+        "\n",
+        "  print(\"Regularization loss:\", tf1_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iMPMMnPtYUY7"
+      },
+      "outputs": [],
+      "source": [
+        "height, width = 299, 299\n",
+        "num_classes = 1000\n",
+        "\n",
+        "random_tool = v1.keras.utils.DeterministicRandomTestTool()\n",
+        "with random_tool.scope():\n",
+        "  keras_input = tf.keras.Input(shape=(height, width, 3))\n",
+        "  layer = InceptionResnetV2(num_classes)\n",
+        "  model = tf.keras.Model(inputs=keras_input, outputs=layer(keras_input))\n",
+        "\n",
+        "  inputs = tf.ones((1, height, width, 3))\n",
+        "  tf2_output, endpoints = model(inputs, training=True)\n",
+        "\n",
+        "  # Get the regularization loss\n",
+        "  tf2_regularization_loss = tf.math.add_n(model.losses)\n",
+        "\n",
+        "print(\"Regularization loss:\", tf2_regularization_loss)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jf46KUVyYUY8"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that the regularization loss and output both match\n",
+        "# when using the DeterministicRandomTestTool\n",
+        "np.testing.assert_allclose(tf1_regularization_loss, tf2_regularization_loss.numpy(), **tol_dict)\n",
+        "np.testing.assert_allclose(tf1_output, tf2_output.numpy(), **tol_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hWXHjtkiZ09V"
+      },
+      "source": [
+        "## Step 3b or 4b (optional): Testing with pre-existing checkpoints\n",
+        "\n",
+        "After step 3 or step 4 above, it can be useful to run your numerical equivalence tests when starting from pre-existing name-based checkpoints if you have some. This can test both that your legacy checkpoint loading is working correctly and that the model itself is working right. The [Reusing TF1.x checkpoints guide](./migrating_checkpoints.ipynb) covers how to reuse your pre-existing TF1.x checkpoints and transfer them over to TF2 checkpoints.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v6i3MFmGcxYx"
+      },
+      "source": [
+        "## Additional Testing & Troubleshooting\n",
+        "\n",
+        "As you add more numerical equivalence tests, you may also choose to add a test that verifies your gradient computation (or even your optimizer updates) match.\n",
+        "\n",
+        "Backpropagation and gradient computation are more prone to floating point numerical instabilities than model forward passes. This means that as your equivalence tests cover more non-isolated parts of your training, you may begin to see non-trivial numerics differences between running fully eagerly and your TF1 graphs. This may be caused by TensorFlow's graph optimizations that do things such as replace subexpressions in a graph with fewer mathematical operations.\n",
+        "\n",
+        "To isolate whether this is likely to be the case, you can compare your TF1 code to TF2 computation happening inside of a `tf.function` (which applies graph optimization passes like your TF1 graph) rather than to a purely eager computation. Alternatively, you can try using `tf.config.optimizer.set_experimental_options` to disable optimization passes such as `\"arithmetic_optimization\"` before your TF1 computation to see if the result ends up numerically closer to your TF2 computation results. In your actual training runs it is recommended you use `tf.function` with optimization passes enabled for performance reasons, but you may find it useful to disable them in your numerical equivalence unit tests.\n",
+        "\n",
+        "Similarly, you may also find that `tf.compat.v1.train` optimizers and TF2 optimizers have slightly different floating point numerics properties than TF2 optimizers, even if the mathematical formulas they are representing are the same. This is less likely to be an issue in your training runs, but it may require a higher numerical tolerance in equivalence unit tests."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "validate_correctness.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/keras/mixed_precision.ipynb b/site/en/guide/mixed_precision.ipynb
similarity index 59%
rename from site/en/guide/keras/mixed_precision.ipynb
rename to site/en/guide/mixed_precision.ipynb
index acdcb89e2a2..a19d6f254f3 100644
--- a/site/en/guide/keras/mixed_precision.ipynb
+++ b/site/en/guide/mixed_precision.ipynb
@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Tce3stUlHN0L"
       },
       "source": [
@@ -12,11 +11,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
         "id": "tuOe1ymfHZPu"
       },
       "outputs": [],
@@ -37,7 +34,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qFdPvlXBOdUN"
       },
       "source": [
@@ -47,128 +43,95 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MfBg1C5NB3X0"
       },
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/keras/mixed_precision\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/keras/mixed_precision.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/keras/mixed_precision.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/keras/mixed_precision.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "xHxb-dlhMIzW"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "Mixed precision is the use of both 16-bit and 32-bit floating-point types in a model during training to make it run faster and use less memory. By keeping certain parts of the model in the 32-bit types for numeric stability, the model will have a lower step time and train equally as well in terms of the evaluation metrics such as accuracy. This guide describes how to use the experimental Keras mixed precision API to speed up your models. Using this API can improve performance by more than 3 times on modern GPUs and 60% on TPUs."
+        "Mixed precision is the use of both 16-bit and 32-bit floating-point types in a model during training to make it run faster and use less memory. By keeping certain parts of the model in the 32-bit types for numeric stability, the model will have a lower step time and train equally as well in terms of the evaluation metrics such as accuracy. This guide describes how to use the Keras mixed precision API to speed up your models. Using this API can improve performance by more than 3 times on modern GPUs, 60% on TPUs and more than 2 times on latest Intel CPUs."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "AEDHaeuR7kLX"
-      },
-      "source": [
-        "Note: The Keras mixed precision API is currently experimental and may change."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
         "id": "3vsYi_bv7gS_"
       },
       "source": [
         "Today, most models use the float32 dtype, which takes 32 bits of memory. However, there are two lower-precision dtypes, float16 and bfloat16, each which take 16 bits of memory instead. Modern accelerators can run operations faster in the 16-bit dtypes, as they have specialized hardware to run 16-bit computations and 16-bit dtypes can be read from memory faster.\n",
         "\n",
-        "NVIDIA GPUs can run operations in float16 faster than in float32, and TPUs can run operations in bfloat16 faster than float32. Therefore, these lower-precision dtypes should be used whenever possible on those devices. However, variables and a few computations should still be in float32 for numeric reasons so that the model trains to the same quality. The Keras mixed precision API allows you to use a mix of either float16 or bfloat16 with float32, to get the performance benefits from float16/bfloat16 and the numeric stability benefits from float32.\n",
+        "NVIDIA GPUs can run operations in float16 faster than in float32, and TPUs and supporting Intel CPUs can run operations in bfloat16 faster than float32. Therefore, these lower-precision dtypes should be used whenever possible on those devices. However, variables and a few computations should still be in float32 for numeric reasons so that the model trains to the same quality. The Keras mixed precision API allows you to use a mix of either float16 or bfloat16 with float32, to get the performance benefits from float16/bfloat16 and the numeric stability benefits from float32.\n",
         "\n",
-        "Note: In this guide, the term \"numeric stability\" refers to how a model's quality is affected by the use of a lower-precision dtype instead of a higher precision dtype. We say an operation is \"numerically unstable\" in float16 or bfloat16 if running it in one of those dtypes causes the model to have worse evaluation accuracy or other metrics compared to running the operation in float32."
+        "Note: In this guide, the term \"numeric stability\" refers to how a model's quality is affected by the use of a lower-precision dtype instead of a higher precision dtype. An operation is \"numerically unstable\" in float16 or bfloat16 if running it in one of those dtypes causes the model to have worse evaluation accuracy or other metrics compared to running the operation in float32."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MUXex9ctTuDB"
       },
       "source": [
         "## Setup"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "1Eh-iCRVBm0p"
-      },
-      "source": [
-        "The Keras mixed precision API is available in TensorFlow 2.1."
-      ]
-    },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "IqR2PQG4ZaZ0"
       },
       "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  !pip install tf-nightly\n",
-        "except Exception:\n",
-        "  pass\n",
         "import tensorflow as tf\n",
         "\n",
         "from tensorflow import keras\n",
         "from tensorflow.keras import layers\n",
-        "from tensorflow.keras.mixed_precision import experimental as mixed_precision"
+        "from tensorflow.keras import mixed_precision"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "814VXqdh8Q0r"
       },
       "source": [
         "## Supported hardware\n",
         "\n",
-        "While mixed precision will run on most hardware, it will only speed up models on recent NVIDIA GPUs and Cloud TPUs. NVIDIA GPUs support using a mix of float16 and float32, while TPUs support a mix of bfloat16 and float32.\n",
+        "While mixed precision will run on most hardware, it will only speed up models on recent NVIDIA GPUs, Cloud TPUs and recent Intel CPUs. NVIDIA GPUs support using a mix of float16 and float32, while TPUs and Intel CPUs support a mix of bfloat16 and float32.\n",
         "\n",
-        "Among NVIDIA GPUs, those with compute capability 7.0 or higher will see the greatest performance benefit from mixed precision because they have special hardware units, called Tensor Cores, to accelerate float16 matrix multiplications and convolutions. Older GPUs offer no math performance benefit for using mixed precision, however memory and bandwidth savings can enable some speedups. You can look up the compute capability for your GPU at NVIDIA's [CUDA GPU web page](https://developer.nvidia.com/cuda-gpus). Examples of GPUs that will benefit most from mixed precision include RTX GPUs, the Titan V, and the V100."
+        "Among NVIDIA GPUs, those with compute capability 7.0 or higher will see the greatest performance benefit from mixed precision because they have special hardware units, called Tensor Cores, to accelerate float16 matrix multiplications and convolutions. Older GPUs offer no math performance benefit for using mixed precision, however memory and bandwidth savings can enable some speedups. You can look up the compute capability for your GPU at NVIDIA's [CUDA GPU web page](https://developer.nvidia.com/cuda-gpus). Examples of GPUs that will benefit most from mixed precision include RTX GPUs, the V100, and the A100.\n",
+       "\n",
+       "Among Intel CPUs, starting with the 4th Gen Intel Xeon Processors (code name Sapphire Rapids), will see the greatest performance benefit from mixed precision as they can accelerate bfloat16 computations using AMX instructions (requires Tensorflow 2.12 or later)."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-q2hisD60F0_"
       },
       "source": [
-        "Note: If running this guide in Google Colab, the GPU runtime typically has a P100 connected. The P100 has compute capability 6.0 and is not expected to show a significant speedup.\n",
+        "Note: If running this guide in Google Colab, the GPU runtime typically has a P100 connected. The P100 has compute capability 6.0 and is not expected to show a significant speedup. If running on CPU runtime, there may be a slow down as the runtime likely has a CPU without AMX.\n",
         "\n",
         "You can check your GPU type with the following. The command only exists if the\n",
         "NVIDIA drivers are installed, so the following will raise an error otherwise."
@@ -176,10 +139,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "j-Yzg_lfkoa_"
       },
       "outputs": [],
@@ -190,19 +151,17 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hu_pvZDN0El3"
       },
       "source": [
         "All Cloud TPUs support bfloat16.\n",
         "\n",
-        "Even on CPUs and older GPUs, where no speedup is expected, mixed precision APIs can still be used for unit testing, debugging, or just to try out the API."
+        "Even on older Intel CPUs, other x86 CPUs without AMX, and older GPUs, where no speedup is expected, mixed precision APIs can still be used for unit testing, debugging, or just to try out the API. However, mixed_bfloat16 on CPUs without AMX instructions and mixed_float16 on all x86 CPUs will run significantly slower."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "HNOmvumB-orT"
       },
       "source": [
@@ -212,31 +171,48 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "54ecYY2Hn16E"
       },
       "source": [
-        "To use mixed precision in Keras, you need to create a `tf.keras.mixed_precision.experimental.Policy`, typically referred to as a *dtype policy*. Dtype policies specify the dtypes layers will run in. In this guide, you will construct a policy from the string `'mixed_float16'` and set it as the global policy. This will will cause subsequently created layers to use mixed precision with a mix of float16 and float32."
+        "To use mixed precision in Keras, you need to create a `tf.keras.mixed_precision.Policy`, typically referred to as a *dtype policy*. Dtype policies specify the dtypes layers will run in. In this guide, you will construct a policy from the string `'mixed_float16'` and set it as the global policy. This will cause subsequently created layers to use mixed precision with a mix of float16 and float32."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "x3kElPVH-siO"
       },
       "outputs": [],
       "source": [
         "policy = mixed_precision.Policy('mixed_float16')\n",
-        "mixed_precision.set_policy(policy)"
+        "mixed_precision.set_global_policy(policy)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ids1rT_UM5q"
+      },
+      "source": [
+        "For short, you can directly pass a string to `set_global_policy`, which is typically done in practice."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6a8iNFoBUSqR"
+      },
+      "outputs": [],
+      "source": [
+        "# Equivalent to the two lines above\n",
+        "mixed_precision.set_global_policy('mixed_float16')"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "oGAMaa0Ho3yk"
       },
       "source": [
@@ -245,10 +221,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "GQRbYm4f8p-k"
       },
       "outputs": [],
@@ -260,17 +234,15 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MOFEcna28o4T"
       },
       "source": [
-        "As mentioned before, the `mixed_float16` policy will most significantly improve performance on NVIDIA GPUs with compute capability of at least 7.0. The policy will run on other GPUs and CPUs but may not improve performance. For TPUs, the `mixed_bfloat16` policy should be used instead."
+        "As mentioned before, the `mixed_float16` policy will most significantly improve performance on NVIDIA GPUs with compute capability of at least 7.0. The policy will run on other GPUs and CPUs but may not improve performance. For TPUs and CPUs, the `mixed_bfloat16` policy should be used instead."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "cAHpt128tVpK"
       },
       "source": [
@@ -280,7 +252,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "nB6ujaR8qMAy"
       },
       "source": [
@@ -289,10 +260,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "0DQM24hL_14Q"
       },
       "outputs": [],
@@ -314,7 +283,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2dezdcqnOXHk"
       },
       "source": [
@@ -323,14 +291,13 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "kC58MzP4PEcC"
       },
       "outputs": [],
       "source": [
+        "print(dense1.dtype_policy)\n",
         "print('x.dtype: %s' % x.dtype.name)\n",
         "# 'kernel' is dense1's variable\n",
         "print('dense1.kernel.dtype: %s' % dense1.kernel.dtype.name)"
@@ -339,7 +306,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "_WAZeqDyqZcb"
       },
       "source": [
@@ -348,10 +314,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ybBq1JDwNIbz"
       },
       "outputs": [],
@@ -364,21 +328,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "D0gSWxc9NN7q"
       },
       "source": [
-        "A softmax activation at the end of the model should be float32. Because the dtype policy is `mixed_float16`, the softmax activation would normally have a float16 compute dtype and output a float16 tensors.\n",
+        "A softmax activation at the end of the model should be float32. Because the dtype policy is `mixed_float16`, the softmax activation would normally have a float16 compute dtype and output float16 tensors.\n",
         "\n",
-        "This can be fixed by separating the Dense and softmax layers, and by passing `dtype='float32'` to the softmax layer"
+        "This can be fixed by separating the Dense and softmax layers, and by passing `dtype='float32'` to the softmax layer:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "IGqCGn4BsODw"
       },
       "outputs": [],
@@ -392,26 +353,23 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "tUdkY_DHsP8i"
       },
       "source": [
-        "Passing `dtype='float32'` to the softmax layer constructor overrides the layer's dtype policy to be the `float32` policy, which does computations and keeps variables in float32. Equivalently, we could have instead passed `dtype=mixed_precision.Policy('float32')`; layers always convert the dtype argument to a policy. Because the `Activation` layer has no variables, the policy's variable dtype is ignored, but the policy's compute dtype of float32 causes softmax and the model output to be float32. \n",
+        "Passing `dtype='float32'` to the softmax layer constructor overrides the layer's dtype policy to be the `float32` policy, which does computations and keeps variables in float32. Equivalently, you could have instead passed `dtype=mixed_precision.Policy('float32')`; layers always convert the dtype argument to a policy. Because the `Activation` layer has no variables, the policy's variable dtype is ignored, but the policy's compute dtype of float32 causes softmax and the model output to be float32. \n",
         "\n",
         "\n",
         "Adding a float16 softmax in the middle of a model is fine, but a softmax at the end of the model should be in float32. The reason is that if the intermediate tensor flowing from the softmax to the loss is float16 or bfloat16, numeric issues may occur.\n",
         "\n",
         "You can override the dtype of any layer to be float32 by passing `dtype='float32'` if you think it will not be numerically stable with float16 computations. But typically, this is only necessary on the last layer of the model, as most layers have sufficient precision with `mixed_float16` and `mixed_bfloat16`.\n",
         "\n",
-        "If the model does not end in a softmax, the outputs should still be float32. While unnecessary for this model, the model outputs can be cast to float32 with the following.:"
+        "Even if the model does not end in a softmax, the outputs should still be float32. While unnecessary for this specific model, the model outputs can be cast to float32 with the following:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "dzVAoLI56jR8"
       },
       "outputs": [],
@@ -425,19 +383,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "tpY4ZP7us5hA"
       },
       "source": [
-        "Next, finish and compile the model, and generate input data."
+        "Next, finish and compile the model, and generate input data:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "g4OT3Z6kqYAL"
       },
       "outputs": [],
@@ -455,21 +410,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "0Sm8FJHegVRN"
       },
       "source": [
-        "This example cast the input data from int8 to float32. We don't cast to float16 since the division by 255 is on the CPU, which runs float16 operations slower than float32 operations. In this case, the performance difference in negligible, but in general you should run input processing math in float32 if it runs on the CPU. The first layer of the model will cast the inputs to float16, as each layer casts floating-point inputs to its compute dtype.\n",
+        "This example casts the input data from int8 to float32. You don't cast to float16 since the division by 255 is on the CPU, which runs float16 operations slower than float32 operations. In this case, the performance difference is negligible, but in general you should run input processing math in float32 if it runs on the CPU. The first layer of the model will cast the inputs to float16, as each layer casts floating-point inputs to its compute dtype.\n",
         "\n",
         "The initial weights of the model are retrieved. This will allow training from scratch again by loading the weights."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "0UYs-u_DgiA5"
       },
       "outputs": [],
@@ -480,21 +432,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "zlqz6eVKs9aU"
       },
       "source": [
         "## Training the model with Model.fit\n",
         "\n",
-        "Next, train the model."
+        "Next, train the model:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "hxI7-0ewmC0A"
       },
       "outputs": [],
@@ -511,51 +460,50 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MPhJ9OPWt4x5"
       },
       "source": [
-        "Notice the model prints the time per sample in the logs: for example, \"4us/sample\". The first epoch may be slower as TensorFlow spends some time optimizing the model, but afterwards the time per sample should stabilize. \n",
+        "Notice the model prints the time per step in the logs: for example, \"25ms/step\". The first epoch may be slower as TensorFlow spends some time optimizing the model, but afterwards the time per step should stabilize. \n",
+        " \n",
+        "If you are running this guide in Colab, you can compare the performance of mixed precision with float32. To do so, change the policy from `mixed_float16` to `float32` in the \"Setting the dtype policy\" section, then rerun all the cells up to this point. On GPUs with compute capability 7.X, you should see the time per step significantly increase, indicating mixed precision sped up the model.  Make sure to change the policy back to `mixed_float16` and rerun the cells before continuing with the guide.\n",
         "\n",
-        "If you are running this guide in Colab, you can compare the performance of mixed precision with float32. To do so, change the policy from `mixed_float16` to `float32` in the \"Setting the dtype policy\" section, then rerun all the cells up to this point. On GPUs with at least compute capability 7.0, you should see the time per sample significantly increase, indicating mixed precision sped up the model. For example, with a Titan V GPU, the per-sample time increases from 4us to 12us. Make sure to change the policy back to `mixed_float16` and rerun the cells before continuing with the guide.\n",
+        "On GPUs with compute capability of at least 8.0 (Ampere GPUs and above), you likely will see no performance improvement in the toy model in this guide when using mixed precision compared to float32. This is due to the use of [TensorFloat-32](https://www.tensorflow.org/api_docs/python/tf/config/experimental/enable_tensor_float_32_execution), which automatically uses lower precision math in certain float32 ops such as `tf.linalg.matmul`. TensorFloat-32 gives some of the performance advantages of mixed precision when using float32. However, in real-world models, you will still typically experience significant performance improvements from mixed precision due to memory bandwidth savings and ops which TensorFloat-32 does not support.\n",
         "\n",
-        "For many real-world models, mixed precision also allows you to double the batch size without running out of memory, as float16 tensors take half the memory. This does not apply however to this toy model, as you can likely run the model in any dtype where each batch consists of the entire MNIST dataset of 60,000 images.\n",
+        "If running mixed precision on a TPU, you will not see as much of a performance gain compared to running mixed precision on GPUs, especially pre-Ampere GPUs. This is because TPUs do certain ops in bfloat16 under the hood even with the default dtype policy of float32. This is similar to how Ampere GPUs use TensorFloat-32 by default. Compared to Ampere GPUs, TPUs typically see less performance gains with mixed precision on real-world models.\n",
         "\n",
-        "If running mixed precision on a TPU, you will not see as much of a performance gain compared to running mixed precision on GPUs. This is because TPUs already do certain ops in bfloat16 under the hood even with the default dtype policy of `float32`. TPU hardware does not support float32 for certain ops which are numerically stable in bfloat16, such as matmul. For such ops the TPU backend will silently use bfloat16 internally instead. As a consequence, passing `dtype='float32'` to layers which use such ops may have no numerical effect, however it is unlikely running such layers with bfloat16 computations will be harmful."
+        "For many real-world models, mixed precision also allows you to double the batch size without running out of memory, as float16 tensors take half the memory. This does not apply however to this toy model, as you can likely run the model in any dtype where each batch consists of the entire MNIST dataset of 60,000 images."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mNKMXlCvHgHb"
       },
       "source": [
         "## Loss scaling\n",
         "\n",
-        "Loss scaling is a technique which `tf.keras.Model.fit` automatically performs with the `mixed_float16` policy to avoid numeric underflow. This section describes loss scaling and how to customize its behavior."
+        "Loss scaling is a technique which `tf.keras.Model.fit` automatically performs with the `mixed_float16` policy to avoid numeric underflow. This section describes what loss scaling is and the next section describes how to use it with a custom training loop.\n",
+        "\n",
+        "Note: When using `mixed_bfloat16` policy, there is no need to do loss scaling."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "1xQX62t2ow0g"
       },
       "source": [
         "### Underflow and Overflow\n",
         "\n",
-        "Float16 has has a narrow dynamic range compared to float32. This means values above $65504$ will overflow to infinity and values below $6.0 \\times 10^{-8}$ will underflow to zero. Float32 and bfloat16 have a much higher dynamic range so that overflow and underflow are not a problem.\n",
+        "The float16 data type has a narrow dynamic range compared to float32. This means values above $65504$ will overflow to infinity and values below $6.0 \\times 10^{-8}$ will underflow to zero. float32 and bfloat16 have a much higher dynamic range so that overflow and underflow are not a problem.\n",
         "\n",
         "For example:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "CHmXRb-yRWbE"
       },
       "outputs": [],
@@ -566,10 +514,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "5unZLhN0RfQM"
       },
       "outputs": [],
@@ -581,7 +527,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "pUIbhQypRVe_"
       },
       "source": [
@@ -591,13 +536,12 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "FAL5qij_oNqJ"
       },
       "source": [
-        "### Loss scaling background\n",
+        "### Loss scaling overview\n",
         "\n",
-        "The basic concept of loss scaling is simple: Simply multiply the loss by some large number, say 1024. We call this number the *loss scale*. This will cause the gradients to be scaled by 1024 as well, greatly reducing the chance of underflow. Once the final gradients are computed, divide them by 1024 to bring them back to their correct values.\n",
+        "The basic concept of loss scaling is simple: simply multiply the loss by some large number, say $1024$, and you get the *loss scale* value. This will cause the gradients to scale by $1024$ as well, greatly reducing the chance of underflow. Once the final gradients are computed, divide them by $1024$ to bring them back to their correct values.\n",
         "\n",
         "The pseudocode for this process is:\n",
         "\n",
@@ -605,112 +549,19 @@
         "loss_scale = 1024\n",
         "loss = model(inputs)\n",
         "loss *= loss_scale\n",
-        "# We assume `grads` are float32. We do not want to divide float16 gradients\n",
+        "# Assume `grads` are float32. You do not want to divide float16 gradients.\n",
         "grads = compute_gradient(loss, model.trainable_variables)\n",
         "grads /= loss_scale\n",
         "```\n",
         "\n",
         "Choosing a loss scale can be tricky. If the loss scale is too low, gradients may still underflow to zero. If too high, the opposite the problem occurs: the gradients may overflow to infinity.\n",
         "\n",
-        "To solve this, TensorFlow dynamically determines the loss scale so you do not have to choose one manually. If you use `tf.keras.Model.fit`, loss scaling is done for you so you do not have to do any extra work. This is explained further in the next section.\n"
+        "To solve this, TensorFlow dynamically determines the loss scale so you do not have to choose one manually. If you use `tf.keras.Model.fit`, loss scaling is done for you so you do not have to do any extra work. If you use a custom training loop, you must explicitly use the special optimizer wrapper `tf.keras.mixed_precision.LossScaleOptimizer` in order to use loss scaling. This is described in the next section.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "qlbxbjxShf0m"
-      },
-      "source": [
-        "### Choosing the loss scale\n",
-        "\n",
-        "Each dtype policy optionally has an associated `tf.mixed_precision.experimental.LossScale` object, which represents a fixed or dynamic loss scale. By default, the loss scale for the `mixed_float16` policy is a `tf.mixed_precision.experimental.DynamicLossScale`, which dynamically determine the loss scale value. Other policies do not have a loss scale by default, as it is only necessary when float16 is used. You can query the loss scale of the policy:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0iSbassjdldy"
-      },
-      "outputs": [],
-      "source": [
-        "loss_scale = policy.loss_scale\n",
-        "print('Loss scale: %s' % loss_scale)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "l8c0ULUxdu2b"
-      },
-      "source": [
-        "The loss scale prints a lot of internal state, but you can ignore it. The most important part is the `current_loss_scale` part, which shows the loss scale's current value."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Mr0cAMCHfhlj"
-      },
-      "source": [
-        "You can instead use a static loss scale by passing a number when constructing a dtype policy."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "4Sa4bGFif9GW"
-      },
-      "outputs": [],
-      "source": [
-        "new_policy = mixed_precision.Policy('mixed_float16', loss_scale=1024)\n",
-        "print(new_policy.loss_scale)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "alD_0frtgFKK"
-      },
-      "source": [
-        "The dtype policy constructor always converts the loss scale to a `LossScale` object. In this case, it's converted to a `tf.mixed_precision.experimental.FixedLossScale`, the only other `LossScale` subclass other than `DynamicLossScale`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Gi1DLrLF7bnr"
-      },
-      "source": [
-        "Note: *Using anything other than a dynamic loss scale is not recommended*. Choosing a fixed loss scale can be difficult, as making it too low will cause the model to not train as well, and making it too high will cause Infs or NaNs to appear in the gradients. A dynamic loss scale is typically near the optimal loss scale, so you do not have to do any work. Currently, dynamic loss scales are a bit slower than fixed loss scales, but the performance will be improved in the future."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "LJ_1W-Axh2bp"
-      },
-      "source": [
-        "Models, like layers, each have a dtype policy. If present, a model uses its policy's loss scale to apply loss scaling in the `tf.keras.Model.fit` method. This means if `Model.fit` is used, you do not have to worry about loss scaling at all: The `mixed_float16` policy will have a dynamic loss scale by default, and `Model.fit` will apply it.\n",
-        "\n",
-        "With custom training loops, the model will ignore the policy's loss scale, and you will have to apply it manually. This is explained in the next section."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
         "id": "yqzbn8Ks9Q98"
       },
       "source": [
@@ -720,21 +571,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "CRANRZZ69nA7"
       },
       "source": [
-        "So far, you trained a Keras model with mixed precision using `tf.keras.Model.fit`. Next, you will use mixed precision with a custom training loop. If you do not already know what a custom training loop is, please read [the Custom training guide](../../tutorials/customization/custom_training_walkthrough.ipynb) first."
+        "So far, you have trained a Keras model with mixed precision using `tf.keras.Model.fit`. Next, you will use mixed precision with a custom training loop. If you do not already know what a custom training loop is, please read the [Custom training guide](../tutorials/customization/custom_training_walkthrough.ipynb) first."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wXTaM8EEyEuo"
       },
       "source": [
         "Running a custom training loop with mixed precision requires two changes over running it in float32:\n",
+        "\n",
         "1. Build the model with mixed precision (you already did this)\n",
         "2. Explicitly use loss scaling if `mixed_float16` is used.\n"
       ]
@@ -742,53 +592,46 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "M2zpp7_65mTZ"
       },
       "source": [
-        "For step (2), you will use the `tf.keras.mixed_precision.experimental.LossScaleOptimizer` class, which wraps an optimizer and applies loss scaling. It takes two arguments: the optimizer and the loss scale. Construct one as follows to use a dynamic loss scale"
+        "For step (2), you will use the `tf.keras.mixed_precision.LossScaleOptimizer` class, which wraps an optimizer and applies loss scaling. By default, it dynamically determines the loss scale so you do not have to choose one. Construct a `LossScaleOptimizer` as follows."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ogZN3rIH0vpj"
       },
       "outputs": [],
       "source": [
         "optimizer = keras.optimizers.RMSprop()\n",
-        "optimizer = mixed_precision.LossScaleOptimizer(optimizer, loss_scale='dynamic')"
+        "optimizer = mixed_precision.LossScaleOptimizer(optimizer)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "8-_ZZVEC2MZQ"
+        "id": "FVy5gnBqTE9z"
       },
       "source": [
-        "Passing `'dynamic'` is equivalent to passing `tf.mixed_precision.experimental.DynamicLossScale()`."
+        "If you want, it is possible choose an explicit loss scale or otherwise customize the loss scaling behavior, but it is highly recommended to keep the default loss scaling behavior, as it has been found to work well on all known models. See the `tf.keras.mixed_precision.LossScaleOptimizer` documentation if you want to customize the loss scaling behavior."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "JZYEr5hA3MXZ"
       },
       "source": [
-        "Next, define the loss object and the `tf.data.Dataset`s"
+        "Next, define the loss object and the `tf.data.Dataset`s:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "9cE7Mm533hxe"
       },
       "outputs": [],
@@ -802,23 +645,21 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "4W0zxrxC3nww"
       },
       "source": [
-        "Next, define the training step function. Two new methods from the loss scale optimizer are used in order to scale the loss and unscale the gradients:\n",
-        "* `get_scaled_loss(loss)`: Multiplies the loss by the loss scale\n",
-        "* `get_unscaled_gradients(gradients)`: Takes in a list of scaled gradients as inputs, and divides each one by the loss scale to unscale them\n",
+        "Next, define the training step function. You will use two new methods from the loss scale optimizer to scale the loss and unscale the gradients:\n",
+        "\n",
+        "- `get_scaled_loss(loss)`: Multiplies the loss by the loss scale\n",
+        "- `get_unscaled_gradients(gradients)`: Takes in a list of scaled gradients as inputs, and divides each one by the loss scale to unscale them\n",
         "\n",
-        "These functions must be used in order to prevent underflow in the gradients. `LossScaleOptimizer.apply_gradients` will then apply gradients if none of them have Infs or NaNs. It will also update the loss scale, halving it if the gradients had Infs or NaNs and potentially increasing it otherwise."
+        "These functions must be used in order to prevent underflow in the gradients. `LossScaleOptimizer.apply_gradients` will then apply gradients if none of them have `Inf`s or `NaN`s. It will also update the loss scale, halving it if the gradients had `Inf`s or `NaN`s and potentially increasing it otherwise."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "V0vHlust4Rug"
       },
       "outputs": [],
@@ -838,7 +679,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "rcFxEjia6YPQ"
       },
       "source": [
@@ -848,20 +688,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IHIvKKhg4Y-G"
       },
       "source": [
-        "Now define the test step.\n",
-        "\n"
+        "Now, define the test step:\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "nyk_xiZf42Tt"
       },
       "outputs": [],
@@ -874,19 +710,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hBs98MZyhBOB"
       },
       "source": [
-        "Load the initial weights of the model, so you can retrain from scratch."
+        "Load the initial weights of the model, so you can retrain from scratch:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "jpzOe3WEhFUJ"
       },
       "outputs": [],
@@ -897,19 +730,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "s9Pi1ADM47Ud"
       },
       "source": [
-        "Finally, run the custom training loop."
+        "Finally, run the custom training loop:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "N274tJ3e4_6t"
       },
       "outputs": [],
@@ -930,7 +760,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "d7daQKGerOFE"
       },
       "source": [
@@ -939,60 +768,63 @@
         "Here are some performance tips when using mixed precision on GPUs.\n",
         "\n",
         "### Increasing your batch size\n",
+        "\n",
         "If it doesn't affect model quality, try running with double the batch size when using mixed precision. As float16 tensors use half the memory, this often allows you to double your batch size without running out of memory. Increasing batch size typically increases training throughput, i.e. the training elements per second your model can run on.\n",
         "\n",
         "### Ensuring GPU Tensor Cores are used\n",
         "\n",
         "As mentioned previously, modern NVIDIA GPUs use a special hardware unit called Tensor Cores that can multiply float16 matrices very quickly. However, Tensor Cores requires certain dimensions of tensors to be a multiple of 8. In the examples below, an argument is bold if and only if it needs to be a multiple of 8 for Tensor Cores to be used.\n",
         "\n",
-        "* tf.keras.layers.Dense(**units=64**)\n",
-        "* tf.keras.layers.Conv2d(**filters=48**, kernel_size=7, stride=3)\n",
-        "  * And similarly for other convolutional layers, such as tf.keras.layers.Conv3d\n",
-        "* tf.keras.layers.LSTM(**units=64**)\n",
-        "  * And similar for other RNNs, such as tf.keras.layers.GRU\n",
-        "* tf.keras.Model.fit(epochs=2, **batch_size=128**)\n",
+        "- tf.keras.layers.Dense(**units=64**)\n",
+        "- tf.keras.layers.Conv2d(**filters=48**, kernel_size=7, stride=3)\n",
+        "  - And similarly for other convolutional layers, such as tf.keras.layers.Conv3d\n",
+        "- tf.keras.layers.LSTM(**units=64**)\n",
+        "  - And similar for other RNNs, such as tf.keras.layers.GRU\n",
+        "- tf.keras.Model.fit(epochs=2, **batch_size=128**)\n",
         "\n",
-        "You should try to use Tensor Cores when possible. If you want to learn more [NVIDIA deep learning performance guide](https://docs.nvidia.com/deeplearning/sdk/dl-performance-guide/index.html) describes the exact requirements for using Tensor Cores as well as other Tensor Core-related performance information.\n",
+        "You should try to use Tensor Cores when possible. If you want to learn more, [NVIDIA deep learning performance guide](https://docs.nvidia.com/deeplearning/sdk/dl-performance-guide/index.html) describes the exact requirements for using Tensor Cores as well as other Tensor Core-related performance information.\n",
         "\n",
         "### XLA\n",
         "\n",
-        "XLA is a compiler that can further increase mixed precision performance, as well as float32 performance to a lesser extent. See the [XLA guide](https://www.tensorflow.org/xla) for details."
+        "XLA is a compiler that can further increase mixed precision performance, as well as float32 performance to a lesser extent. Refer to the [XLA guide](https://www.tensorflow.org/xla) for details."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2tFDX8fm6o_3"
       },
       "source": [
         "## Cloud TPU performance tips\n",
-        "As on GPUs, you should try doubling your batch size, as bfloat16 tensors use half the memory. Doubling batch size may increase training throughput.\n",
         "\n",
-        "TPUs do not require any other mixed precision-specific tuning to get optimal performance. TPUs already require the use of XLA. They benefit from having certain dimensions being multiples of 128, but this applies equally to float32 as it does for mixed precision. See the [Cloud TPU Performance Guide](https://cloud.google.com/tpu/docs/performance-guide) for general TPU performance tips, which apply to mixed precision as well as float32."
+        "As with GPUs, you should try doubling your batch size when using Cloud TPUs because bfloat16 tensors use half the memory. Doubling batch size may increase training throughput.\n",
+        "\n",
+        "TPUs do not require any other mixed precision-specific tuning to get optimal performance. They already require the use of XLA. TPUs benefit from having certain dimensions being multiples of $128$, but this applies equally to the float32 type as it does for mixed precision. Check the [Cloud TPU performance guide](https://cloud.google.com/tpu/docs/performance-guide) for general TPU performance tips, which apply to mixed precision as well as float32 tensors."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "--wSEU91wO9w"
       },
       "source": [
         "## Summary\n",
-        "* You should use mixed precision if you use TPUs or NVIDIA GPUs with at least compute capability 7.0, as it will improve performance by up to 3x.\n",
-        "* You can use mixed precision with the following lines:\n",
-        "  ```\n",
-        "  # On TPUs, use 'mixed_bfloat16' instead\n",
-        "  policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n",
-        "  mixed_precision.set_policy(policy)\n",
+        "\n",
+        "- You should use mixed precision if you use TPUs, NVIDIA GPUs with at least compute capability 7.0, or Intel CPUs with support for AMX instructions, as it will improve performance by up to 3x.\n",
+        "- You can use mixed precision with the following lines:\n",
+        "\n",
+        "  ```python\n",
+        "  # On TPUs and CPUs, use 'mixed_bfloat16' instead\n",
+        "  mixed_precision.set_global_policy('mixed_float16')\n",
         "  ```\n",
+        "\n",
         "* If your model ends in softmax, make sure it is float32. And regardless of what your model ends in, make sure the output is float32.\n",
-        "* If you use a custom training loop with `mixed_float16`, in addition to the above lines, you need to wrap your optimizer with a `tf.keras.mixed_precision.experimental.LossScaleOptimizer`. Then call `optimizer.get_scaled_loss` to scale the loss, and `optimizer.get_unscaled_gradients` to unscale the gradients.\n",
+        "* If you use a custom training loop with `mixed_float16`, in addition to the above lines, you need to wrap your optimizer with a `tf.keras.mixed_precision.LossScaleOptimizer`. Then call `optimizer.get_scaled_loss` to scale the loss, and `optimizer.get_unscaled_gradients` to unscale the gradients.\n",
+        "* If you use a custom training loop with `mixed_bfloat16`, setting the global_policy mentioned above is sufficient.\n",
         "* Double the training batch size if it does not reduce evaluation accuracy\n",
-        "* On GPUs, ensure most tensor dimensions are a multiple of 8 to maximize performance\n",
+        "* On GPUs, ensure most tensor dimensions are a multiple of $8$ to maximize performance\n",
         "\n",
-        "For more examples of mixed precision using the `tf.keras.mixed_precision` API, see the [official models repository](https://github.com/tensorflow/models/tree/master/official). Most official models, such as [ResNet](https://github.com/tensorflow/models/tree/master/official/vision/image_classification) and [Transformer](https://github.com/tensorflow/models/tree/master/official/transformer/v2) will run using mixed precision by passing `--dtype=fp16`. \n"
+        "For an example of mixed precision using the `tf.keras.mixed_precision` API, check [functions and classes related to training performance](https://github.com/tensorflow/models/blob/master/official/modeling/performance.py). Check out the official models, such as [Transformer](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/transformer_encoder_block.py), for details.\n"
       ]
     }
   ],
@@ -1001,8 +833,6 @@
     "colab": {
       "collapsed_sections": [],
       "name": "mixed_precision.ipynb",
-      "private_outputs": true,
-      "provenance": [],
       "toc_visible": true
     },
     "kernelspec": {
diff --git a/site/en/guide/profiler.md b/site/en/guide/profiler.md
new file mode 100644
index 00000000000..dee8a5a84af
--- /dev/null
+++ b/site/en/guide/profiler.md
@@ -0,0 +1,1040 @@
+# Optimize TensorFlow performance using the Profiler
+
+[TOC]
+
+This guide demonstrates how to use the tools available with the TensorFlow
+Profiler to track the performance of your TensorFlow models. You will learn how
+to understand how your model performs on the host (CPU), the device (GPU), or on
+a combination of both the host and device(s).
+
+Profiling helps understand the hardware resource consumption (time and memory)
+of the various TensorFlow operations (ops) in your model and resolve performance
+bottlenecks and, ultimately, make the model execute faster.
+
+This guide will walk you through how to install the Profiler, the various tools
+available, the different modes of how the Profiler collects performance data,
+and some recommended best practices to optimize model performance.
+
+If you want to profile your model performance on Cloud TPUs, refer to the
+[Cloud TPU guide](https://cloud.google.com/tpu/docs/cloud-tpu-tools#capture_profile).
+
+## Install the Profiler and GPU prerequisites
+
+Install the Profiler plugin for TensorBoard with pip. Note that the Profiler
+requires the latest versions of TensorFlow and TensorBoard (>=2.2).
+
+```shell
+pip install -U tensorboard_plugin_profile
+```
+
+To profile on the GPU, you must:
+
+1.  Meet the NVIDIA® GPU drivers and CUDA® Toolkit requirements listed on
+    [TensorFlow GPU support software requirements](https://www.tensorflow.org/install/gpu#linux_setup).
+2.  Make sure the
+    [NVIDIA® CUDA® Profiling Tools Interface](https://developer.nvidia.com/cupti)
+    (CUPTI) exists on the path:
+
+    ```shell
+    /sbin/ldconfig -N -v $(sed 's/:/ /g' <<< $LD_LIBRARY_PATH) | \
+    grep libcupti
+    ```
+
+If you don't have CUPTI on the path, prepend its installation directory to the
+`$LD_LIBRARY_PATH` environment variable by running:
+
+```shell
+export LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+```
+
+Then, run the `ldconfig` command above again to verify that the CUPTI library is
+found.
+
+### Resolve privilege issues
+
+When you run profiling with CUDA® Toolkit in a Docker environment or on Linux,
+you may encounter issues related to insufficient CUPTI privileges
+(`CUPTI_ERROR_INSUFFICIENT_PRIVILEGES`). Go to the
+[NVIDIA Developer Docs](https://developer.nvidia.com/nvidia-development-tools-solutions-ERR_NVGPUCTRPERM-permission-issue-performance-counters)
+to learn more about how you can resolve these issues on Linux.
+
+To resolve CUPTI privilege issues in a Docker environment, run
+
+```shell
+docker run option '--privileged=true'
+```
+
+
+
+## Profiler tools
+
+Access the Profiler from the **Profile** tab in TensorBoard, which appears only
+after you have captured some model data.
+
+Note: The Profiler requires internet access to load the
+[Google Chart libraries](https://developers.google.com/chart/interactive/docs/basic_load_libs#basic-library-loading).
+Some charts and tables may be missing if you run TensorBoard entirely offline on
+your local machine, behind a corporate firewall, or in a data center.
+
+The Profiler has a selection of tools to help with performance analysis:
+
+-   Overview Page
+-   Input Pipeline Analyzer
+-   TensorFlow Stats
+-   Trace Viewer
+-   GPU Kernel Stats
+-   Memory Profile Tool
+-   Pod Viewer
+
+
+
+### Overview page
+
+The overview page provides a top level view of how your model performed during a
+profile run. The page shows you an aggregated overview page for your host and
+all devices, and some recommendations to improve your model training
+performance. You can also select individual hosts in the Host dropdown.
+
+The overview page displays data as follows:
+
+![image](./images/tf_profiler/overview_page.png)
+
+*   **Performance Summary**: Displays a high-level summary of your model
+    performance. The performance summary has two parts:
+
+    1.  Step-time breakdown: Breaks down the average step time into multiple
+        categories of where time is spent:
+
+        *   Compilation: Time spent compiling kernels.
+        *   Input: Time spent reading input data.
+        *   Output: Time spent reading output data.
+        *   Kernel launch: Time spent by the host to launch kernels
+        *   Host compute time..
+        *   Device-to-device communication time.
+        *   On-device compute time.
+        *   All others, including Python overhead.
+
+    2.  Device compute precisions - Reports the percentage of device compute
+        time that uses 16 and 32-bit computations.
+
+*   **Step-time Graph**: Displays a graph of device step time (in milliseconds)
+    over all the steps sampled. Each step is broken into the multiple categories
+    (with different colors) of where time is spent. The red area corresponds to
+    the portion of the step time the devices were sitting idle waiting for input
+    data from the host. The green area shows how much of time the device was
+    actually working.
+
+*   **Top 10 TensorFlow operations on device (e.g. GPU)**: Displays the
+    on-device ops that ran the longest.
+
+    Each row displays an op's self time (as the percentage of time taken by all
+    ops), cumulative time, category, and name.
+
+*   **Run Environment**: Displays a high-level summary of the model run
+    environment including:
+
+    *   Number of hosts used.
+    *   Device type (GPU/TPU).
+    *   Number of device cores.
+
+*   **Recommendation for Next Step**: Reports when a model is input bound and
+    recommends tools you can use to locate and resolve model performance
+    bottlenecks.
+
+
+
+### Input pipeline analyzer
+
+When a TensorFlow program reads data from a file it begins at the top of the
+TensorFlow graph in a pipelined manner. The read process is divided into
+multiple data processing stages connected in series, where the output of one
+stage is the input to the next one. This system of reading data is called the
+*input pipeline*.
+
+A typical pipeline for reading records from files has the following stages:
+
+1.  File reading.
+1.  File preprocessing (optional).
+1.  File transfer from the host to the device.
+
+An inefficient input pipeline can severely slow down your application. An
+application is considered **input bound** when it spends a significant portion
+of time in the input pipeline. Use the insights obtained from the input pipeline
+analyzer to understand where the input pipeline is inefficient.
+
+The input pipeline analyzer tells you immediately whether your program is input
+bound and walks you through device- and host-side analysis to debug performance
+bottlenecks at any stage in the input pipeline.
+
+Check the guidance on input pipeline performance for recommended best practices
+to optimize your data input pipelines.
+
+#### Input pipeline dashboard
+
+To open the input pipeline analyzer, select **Profile**, then select
+**input_pipeline_analyzer** from the **Tools** dropdown.
+
+![image](./images/tf_profiler/input_pipeline_analyzer.png)
+
+The dashboard contains three sections:
+
+1.  **Summary**: Summarizes the overall input pipeline with information on
+    whether your application is input bound and, if so, by how much.
+1.  **Device-side analysis**: Displays detailed, device-side analysis results,
+    including the device step-time and the range of device time spent waiting
+    for input data across cores at each step.
+1.  **Host-side analysis**: Shows a detailed analysis on the host side,
+    including a breakdown of input processing time on the host.
+
+#### Input pipeline summary
+
+The **Summary** reports if your program is input bound by presenting the
+percentage of device time spent on waiting for input from the host. If you are
+using a standard input pipeline that has been instrumented, the tool reports
+where most of the input processing time is spent.
+
+#### Device-side analysis
+
+The device-side analysis provides insights on time spent on the device versus on
+the host and how much device time was spent waiting for input data from the
+host.
+
+1.  **Step time plotted against step number**: Displays a graph of device step
+    time (in milliseconds) over all the steps sampled. Each step is broken into
+    the multiple categories (with different colors) of where time is spent. The
+    red area corresponds to the portion of the step time the devices were
+    sitting idle waiting for input data from the host. The green area shows how
+    much of the time the device was actually working.
+1.  **Step time statistics**: Reports the average, standard deviation, and range
+    (\[minimum, maximum\]) of the device step time.
+
+#### Host-side analysis
+
+The host-side analysis reports a breakdown of the input processing time (the
+time spent on `tf.data` API ops) on the host into several categories:
+
+-   **Reading data from files on demand**: Time spent on reading data from files
+    without caching, prefetching, and interleaving.
+-   **Reading data from files in advance**: Time spent reading files, including
+    caching, prefetching, and interleaving.
+-   **Data preprocessing**: Time spent on preprocessing ops, such as image
+    decompression.
+-   **Enqueuing data to be transferred to device**: Time spent putting data into
+    an infeed queue before transferring the data to the device.
+
+Expand **Input Op Statistics** to inspect the statistics for individual input
+ops and their categories broken down by execution time.
+
+![image](./images/tf_profiler/input_op_stats.png)
+
+A source data table will appear with each entry containing the following
+information:
+
+1.  **Input Op**: Shows the TensorFlow op name of the input op.
+1.  **Count**: Shows the total number of instances of op execution during the
+    profiling period.
+1.  **Total Time (in ms)**: Shows the cumulative sum of time spent on each of
+    those instances.
+1.  **Total Time %**: Shows the total time spent on an op as a fraction of the
+    total time spent in input processing.
+1.  **Total Self Time (in ms)**: Shows the cumulative sum of the self time spent
+    on each of those instances. The self time here measures the time spent
+    inside the function body, excluding the time spent in the function it calls.
+1.  **Total Self Time %**. Shows the total self time as a fraction of the total
+    time spent on input processing.
+1.  **Category**. Shows the processing category of the input op.
+
+
+
+### TensorFlow stats
+
+The TensorFlow Stats tool displays the performance of every TensorFlow op (op)
+that is executed on the host or device during a profiling session.
+
+![image](./images/tf_profiler/tf_stats.png)
+
+The tool displays performance information in two panes:
+
+-   The upper pane displays up to four pie charts:
+
+    1.  The distribution of self-execution time of each op on the host.
+    1.  The distribution of self-execution time of each op type on the host.
+    1.  The distribution of self-execution time of each op on the device.
+    1.  The distribution of self-execution time of each op type on the device.
+
+-   The lower pane shows a table that reports data about TensorFlow ops with one
+    row for each op and one column for each type of data (sort columns by
+    clicking the heading of the column). Click the **Export as CSV button** on
+    the right side of the upper pane to export the data from this table as a CSV
+    file.
+
+    Note that:
+
+    *   If any ops have child ops:
+
+        *   The total "accumulated" time of an op includes the time spent inside
+            the child ops.
+        *   The total "self" time of an op does not include the time spent
+            inside the child ops.
+
+    *   If an op executes on the host:
+
+        *   The percentage of the total self-time on device incurred by the op
+            on will be 0.
+        *   The cumulative percentage of the total self-time on device up to and
+            including this op will be 0.
+
+    *   If an op executes on the device:
+
+        *   The percentage of the total self-time on host incurred by this op
+            will be 0.
+        *   The cumulative percentage of the total self-time on host up to and
+            including this op will be 0.
+
+You can choose to include or exclude Idle time in the pie charts and table.
+
+
+
+### Trace viewer
+
+The trace viewer displays a timeline that shows:
+
+-   Durations for the ops that were executed by your TensorFlow model
+-   Which part of the system (host or device) executed an op. Typically, the
+    host executes input operations, preprocesses training data and transfers it
+    to the device, while the device executes the actual model training
+
+The trace viewer allows you to identify performance problems in your model, then
+take steps to resolve them. For example, at a high level, you can identify
+whether input or model training is taking the majority of the time. Drilling
+down, you can identify which ops take the longest to execute. Note that the
+trace viewer is limited to 1 million events per device.
+
+#### Trace viewer interface
+
+When you open the trace viewer, it appears displaying your most recent run:
+
+![image](./images/tf_profiler/trace_viewer.png)
+
+This screen contains the following main elements:
+
+1.  **Timeline pane**: Shows ops that the device and the host executed over
+    time.
+1.  **Details pane**: Shows additional information for ops selected in the
+    Timeline pane.
+
+The Timeline pane contains the following elements:
+
+1.  **Top bar**: Contains various auxiliary controls.
+1.  **Time axis**: Shows time relative to the beginning of the trace.
+1.  **Section and track labels**: Each section contains multiple tracks and has
+    a triangle on the left that you can click to expand and collapse the
+    section. There is one section for every processing element in the system.
+1.  **Tool selector**: Contains various tools for interacting with the trace
+    viewer such as Zoom, Pan, Select, and Timing. Use the Timing tool to mark a
+    time interval.
+1.  **Events**: These show the time during which an op was executed or the
+    duration of meta-events, such as training steps.
+
+##### Sections and tracks
+
+The trace viewer contains the following sections:
+
+-   **One section for each device node**, labeled with the number of the device
+    chip and the device node within the chip (for example, `/device:GPU:0 (pid
+    0)`). Each device node section contains the following tracks:
+    -   **Step**: Shows the duration of the training steps that were running on
+        the device
+    -   **TensorFlow Ops**: Shows the ops executed on the device
+    -   **XLA Ops**: Shows [XLA](https://www.tensorflow.org/xla/) operations
+        (ops) that ran on the device if XLA is the compiler used (each
+        TensorFlow op is translated into one or several XLA ops. The XLA
+        compiler translates the XLA ops into code that runs on the device).
+-   **One section for threads running on the host machine's CPU,** labeled
+    **"Host Threads"**. The section contains one track for each CPU thread. Note
+    that you can ignore the information displayed alongside the section labels.
+
+##### Events
+
+Events within the timeline are displayed in different colors; the colors
+themselves have no specific meaning.
+
+The trace viewer can also display traces of Python function calls in your
+TensorFlow program. If you use the `tf.profiler.experimental.start` API, you can
+enable Python tracing by using the `ProfilerOptions` namedtuple when starting
+profiling. Alternatively, if you use the sampling mode for profiling, you can
+select the level of tracing by using the dropdown options in the **Capture
+Profile** dialog.
+
+![image](./images/tf_profiler/python_tracer.png)
+
+
+
+### GPU kernel stats
+
+This tool shows performance statistics and the originating op for every GPU
+accelerated kernel.
+
+![image](./images/tf_profiler/gpu_kernel_stats.png)
+
+The tool displays information in two panes:
+
+-   The upper pane displays a pie chart which shows the CUDA kernels that have
+    the highest total time elapsed.
+
+-   The lower pane displays a table with the following data for each unique
+    kernel-op pair:
+
+    *   A rank in descending order of total elapsed GPU duration grouped by
+        kernel-op pair.
+    *   The name of the launched kernel.
+    *   The number of GPU registers used by the kernel.
+    *   The total size of shared (static + dynamic shared) memory used in bytes.
+    *   The block dimension expressed as `blockDim.x, blockDim.y, blockDim.z`.
+    *   The grid dimensions expressed as `gridDim.x, gridDim.y, gridDim.z`.
+    *   Whether the op is eligible to use
+        [Tensor Cores](https://www.nvidia.com/en-gb/data-center/tensor-cores/).
+    *   Whether the kernel contains Tensor Core instructions.
+    *   The name of the op that launched this kernel.
+    *   The number of occurrences of this kernel-op pair.
+    *   The total elapsed GPU time in microseconds.
+    *   The average elapsed GPU time in microseconds.
+    *   The minimum elapsed GPU time in microseconds.
+    *   The maximum elapsed GPU time in microseconds.
+
+
+
+### Memory profile tool {: id = 'memory_profile_tool'}
+
+The **Memory Profile** tool monitors the memory usage of your device during the
+profiling interval. You can use this tool to:
+
+*   Debug out of memory (OOM) issues by pinpointing peak memory usage and the
+    corresponding memory allocation to TensorFlow ops. You can also debug OOM
+    issues that may arise when you run
+    [multi-tenancy](https://arxiv.org/pdf/1901.06887.pdf) inference.
+*   Debug memory fragmentation issues.
+
+The memory profile tool displays data in three sections:
+
+1.  **Memory Profile Summary**
+1.  **Memory Timeline Graph**
+1.  **Memory Breakdown Table**
+
+#### Memory profile summary
+
+This section displays a high-level summary of the memory profile of your
+TensorFlow program as shown below:
+
+
+
+The memory profile summary has six fields:
+
+1.  **Memory ID**: Dropdown which lists all available device memory systems.
+    Select the memory system you want to view from the dropdown.
+1.  **#Allocation**: The number of memory allocations made during the profiling
+    interval.
+1.  **#Deallocation**: The number of memory deallocations in the profiling
+    interval
+1.  **Memory Capacity**: The total capacity (in GiBs) of the memory system that
+    you select.
+1.  **Peak Heap Usage**: The peak memory usage (in GiBs) since the model started
+    running.
+1.  **Peak Memory Usage**: The peak memory usage (in GiBs) in the profiling
+    interval. This field contains the following sub-fields:
+    1.  **Timestamp**: The timestamp of when the peak memory usage occurred on
+        the Timeline Graph.
+    1.  **Stack Reservation**: Amount of memory reserved on the stack (in GiBs).
+    1.  **Heap Allocation**: Amount of memory allocated on the heap (in GiBs).
+    1.  **Free Memory**: Amount of free memory (in GiBs). The Memory Capacity is
+        the sum total of the Stack Reservation, Heap Allocation, and Free
+        Memory.
+    1.  **Fragmentation**: The percentage of fragmentation (lower is better). It
+        is calculated as a percentage of `(1 - Size of the largest chunk of free
+        memory / Total free memory)`.
+
+#### Memory timeline graph
+
+This section displays a plot of the memory usage (in GiBs) and the percentage of
+fragmentation versus time (in ms).
+
+![image](./images/tf_profiler/memory_timeline_graph.png)
+
+The X-axis represents the timeline (in ms) of the profiling interval. The Y-axis
+on the left represents the memory usage (in GiBs) and the Y-axis on the right
+represents the percentage of fragmentation. At each point in time on the X-axis,
+the total memory is broken down into three categories: stack (in red), heap (in
+orange), and free (in green). Hover over a specific timestamp to view the
+details about the memory allocation/deallocation events at that point like
+below:
+
+![image](./images/tf_profiler/memory_timeline_graph_popup.png)
+
+The pop-up window displays the following information:
+
+*   **timestamp(ms)**: The location of the selected event on the timeline.
+*   **event**: The type of event (allocation or deallocation).
+*   **requested_size(GiBs)**: The amount of memory requested. This will be a
+    negative number for deallocation events.
+*   **allocation_size(GiBs)**: The actual amount of memory allocated. This will
+    be a negative number for deallocation events.
+*   **tf_op**: The TensorFlow op that requests the allocation/deallocation.
+*   **step_id**: The training step in which this event occurred.
+*   **region_type**: The data entity type that this allocated memory is for.
+    Possible values are `temp` for temporaries, `output` for activations and
+    gradients, and `persist`/`dynamic` for weights and constants.
+*   **data_type**: The tensor element type (e.g., uint8 for 8-bit unsigned
+    integer).
+*   **tensor_shape**: The shape of the tensor being allocated/deallocated.
+*   **memory_in_use(GiBs)**: The total memory that is in use at this point of
+    time.
+
+#### Memory breakdown table
+
+This table shows the active memory allocations at the point of peak memory usage
+in the profiling interval.
+
+![image](./images/tf_profiler/memory_breakdown_table.png)
+
+There is one row for each TensorFlow Op and each row has the following columns:
+
+*   **Op Name**: The name of the TensorFlow op.
+*   **Allocation Size (GiBs)**: The total amount of memory allocated to this op.
+*   **Requested Size (GiBs)**: The total amount of memory requested for this op.
+*   **Occurrences**: The number of allocations for this op.
+*   **Region type**: The data entity type that this allocated memory is for.
+    Possible values are `temp` for temporaries, `output` for activations and
+    gradients, and `persist`/`dynamic` for weights and constants.
+*   **Data type**: The tensor element type.
+*   **Shape**: The shape of the allocated tensors.
+
+Note: You can sort any column in the table and also filter rows by op name.
+
+
+
+### Pod viewer
+
+The Pod Viewer tool shows the breakdown of a training step across all workers.
+
+![image](./images/tf_profiler/pod_viewer.png)
+
+-   The upper pane has a slider for selecting the step number.
+-   The lower pane displays a stacked column chart. This is a high level view of
+    broken down step-time categories placed atop one another. Each stacked
+    column represents a unique worker.
+-   When you hover over a stacked column, the card on the left-hand side shows
+    more details about the step breakdown.
+
+
+
+### tf.data bottleneck analysis
+
+Warning: This tool is experimental. Please open a
+[GitHub Issue](https://github.com/tensorflow/profiler/issues) if the analysis
+result seems incorrect.
+
+The `tf.data` bottleneck analysis tool automatically detects bottlenecks in
+`tf.data` input pipelines in your program and provides recommendations on how to
+fix them. It works with any program using `tf.data` regardless of the platform
+(CPU/GPU/TPU). Its analysis and recommendations are based on this
+[guide](https://www.tensorflow.org/guide/data_performance_analysis).
+
+It detects a bottleneck by following these steps:
+
+1.  Find the most input bound host.
+1.  Find the slowest execution of a `tf.data` input pipeline.
+1.  Reconstruct the input pipeline graph from the profiler trace.
+1.  Find the critical path in the input pipeline graph.
+1.  Identify the slowest transformation on the critical path as a bottleneck.
+
+The UI is divided into three sections: **Performance Analysis Summary**,
+**Summary of All Input Pipelines** and **Input Pipeline Graph**.
+
+#### Performance analysis summary
+
+![image](./images/tf_profiler/tf_data_summary.png)
+
+This section provides the summary of the analysis. It reports on slow `tf.data`
+input pipelines detected in the profile. This section also shows the most input
+bound host and its slowest input pipeline with the max latency. Most
+importantly, it identifies which part of the input pipeline is the bottleneck
+and how to fix it. The bottleneck information is provided with the iterator type
+and its long name.
+
+##### How to read tf.data iterator's long name
+
+A long name is formatted as `Iterator::::...::`. In the
+long name, `` matches the iterator type and the other datasets in the
+long name represent downstream transformations.
+
+For example, consider the following input pipeline dataset:
+
+```python
+dataset = tf.data.Dataset.range(10).map(lambda x: x).repeat(2).batch(5)
+```
+
+The long names for the iterators from the above dataset will be:
+
+Iterator Type | Long Name
+:------------ | :----------------------------------
+Range         | Iterator::Batch::Repeat::Map::Range
+Map           | Iterator::Batch::Repeat::Map
+Repeat        | Iterator::Batch::Repeat
+Batch         | Iterator::Batch
+
+#### Summary of all input pipelines
+
+![image](./images/tf_profiler/tf_data_all_hosts.png)
+
+This section provides the summary of all input pipelines across all hosts.
+Typically there is one input pipeline. When using the distribution strategy,
+there is one host input pipeline running the program's `tf.data` code and
+multiple device input pipelines retrieving data from the host input pipeline and
+transferring it to the devices.
+
+For each input pipeline, it shows the statistics of its execution time. A call
+is counted as slow if it takes longer than 50 μs.
+
+#### Input pipeline graph
+
+![image](./images/tf_profiler/tf_data_graph_selector.png)
+
+This section shows the input pipeline graph with the execution time information.
+You can use "Host" and "Input Pipeline" to choose which host and input pipeline
+to see. Executions of the input pipeline are sorted by the execution time in
+descending order which you can choose using the **Rank** dropdown.
+
+![image](./images/tf_profiler/tf_data_graph.png)
+
+The nodes on the critical path have bold outlines. The bottleneck node, which is
+the node with the longest self time on the critical path, has a red outline. The
+other non-critical nodes have gray dashed outlines.
+
+In each node,**Start Time** indicates the start time of the execution. The same
+node may be executed multiple times, for example, if there is a `Batch` op in
+the input pipeline. If it is executed multiple times, it is the start time of
+the first execution.
+
+**Total Duration** is the wall time of the execution. If it is executed multiple
+times, it is the sum of the wall times of all executions.
+
+**Self Time** is **Total Time** without the overlapped time with its immediate
+child nodes.
+
+"# Calls" is the number of times the input pipeline is executed.
+
+
+
+## Collect performance data
+
+The TensorFlow Profiler collects host activities and GPU traces of your
+TensorFlow model. You can configure the Profiler to collect performance data
+through either the programmatic mode or the sampling mode.
+
+### Profiling APIs
+
+You can use the following APIs to perform profiling.
+
+*   Programmatic mode using the TensorBoard Keras Callback
+    (`tf.keras.callbacks.TensorBoard`)
+
+    ```python
+    # Profile from batches 10 to 15
+    tb_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
+                                                 profile_batch='10, 15')
+
+    # Train the model and use the TensorBoard Keras callback to collect
+    # performance profiling data
+    model.fit(train_data,
+              steps_per_epoch=20,
+              epochs=5,
+              callbacks=[tb_callback])
+    ```
+
+*   Programmatic mode using the `tf.profiler` Function API
+
+    ```python
+    tf.profiler.experimental.start('logdir')
+    # Train the model here
+    tf.profiler.experimental.stop()
+    ```
+
+*   Programmatic mode using the context manager
+
+    ```python
+    with tf.profiler.experimental.Profile('logdir'):
+        # Train the model here
+        pass
+    ```
+
+Note: Running the Profiler for too long can cause it to run out of memory. It is
+recommended to profile no more than 10 steps at a time. Avoid profiling the
+first few batches to avoid inaccuracies due to initialization overhead.
+
+
+
+*   Sampling mode: Perform on-demand profiling by using
+    `tf.profiler.experimental.server.start` to start a gRPC server with your
+    TensorFlow model run. After starting the gRPC server and running your model,
+    you can capture a profile through the **Capture Profile** button in the
+    TensorBoard profile plugin. Use the script in the Install profiler section
+    above to launch a TensorBoard instance if it is not already running.
+
+    As an example,
+
+    ```python
+    # Start a profiler server before your model runs.
+    tf.profiler.experimental.server.start(6009)
+    # (Model code goes here).
+    #  Send a request to the profiler server to collect a trace of your model.
+    tf.profiler.experimental.client.trace('grpc://localhost:6009',
+                                          'gs://your_tb_logdir', 2000)
+    ```
+
+    An example for profiling multiple workers:
+
+    ```python
+    # E.g., your worker IP addresses are 10.0.0.2, 10.0.0.3, 10.0.0.4, and you
+    # would like to profile for a duration of 2 seconds.
+    tf.profiler.experimental.client.trace(
+        'grpc://10.0.0.2:8466,grpc://10.0.0.3:8466,grpc://10.0.0.4:8466',
+        'gs://your_tb_logdir',
+        2000)
+    ```
+
+
+
+
+
+Use the **Capture Profile** dialog to specify:
+
+*   A comma-delimited list of profile service URLs or TPU names.
+*   A profiling duration.
+*   The level of device, host, and Python function call tracing.
+*   How many times you want the Profiler to retry capturing profiles if
+    unsuccessful at first.
+
+### Profiling custom training loops
+
+To profile custom training loops in your TensorFlow code, instrument the
+training loop with the `tf.profiler.experimental.Trace` API to mark the step
+boundaries for the Profiler.
+
+The `name` argument is used as a prefix for the step names, the `step_num`
+keyword argument is appended in the step names, and the `_r` keyword argument
+makes this trace event get processed as a step event by the Profiler.
+
+As an example,
+
+```python
+for step in range(NUM_STEPS):
+    with tf.profiler.experimental.Trace('train', step_num=step, _r=1):
+        train_data = next(dataset)
+        train_step(train_data)
+```
+
+This will enable the Profiler's step-based performance analysis and cause the
+step events to show up in the trace viewer.
+
+Make sure that you include the dataset iterator within the
+`tf.profiler.experimental.Trace` context for accurate analysis of the input
+pipeline.
+
+The code snippet below is an anti-pattern:
+
+Warning: This will result in inaccurate analysis of the input pipeline.
+
+```python
+for step, train_data in enumerate(dataset):
+    with tf.profiler.experimental.Trace('train', step_num=step, _r=1):
+        train_step(train_data)
+```
+
+### Profiling use cases
+
+The profiler covers a number of use cases along four different axes. Some of the
+combinations are currently supported and others will be added in the future.
+Some of the use cases are:
+
+*   _Local vs. remote profiling_: These are two common ways of setting up your
+    profiling environment. In local profiling, the profiling API is called on
+    the same machine your model is executing, for example, a local workstation
+    with GPUs. In remote profiling, the profiling API is called on a different
+    machine from where your model is executing, for example, on a Cloud TPU.
+*   _Profiling multiple workers_: You can profile multiple machines when using
+    the distributed training capabilities of TensorFlow.
+*   _Hardware platform_: Profile CPUs, GPUs, and TPUs.
+
+The table below provides a quick overview of the TensorFlow-supported use cases
+mentioned above:
+
+
+
+| Profiling API                | Local     | Remote    | Multiple  | Hardware  |
+:                              :           :           : workers   : Platforms :
+| :--------------------------- | :-------- | :-------- | :-------- | :-------- |
+| **TensorBoard Keras          | Supported | Not       | Not       | CPU, GPU  |
+: Callback**                   :           : Supported : Supported :           :
+| **`tf.profiler.experimental` | Supported | Not       | Not       | CPU, GPU  |
+: start/stop [API][API_0]**    :           : Supported : Supported :           :
+| **`tf.profiler.experimental` | Supported | Supported | Supported | CPU, GPU, |
+: client.trace [API][API_1]**  :           :           :           : TPU       :
+| **Context manager API**      | Supported | Not       | Not       | CPU, GPU  |
+:                              :           : supported : Supported :           :
+
+[API_0]: https://www.tensorflow.org/api_docs/python/tf/profiler/experimental#functions_2
+[API_1]: https://www.tensorflow.org/api_docs/python/tf/profiler/experimental/client/trace
+
+
+
+## Best practices for optimal model performance
+
+Use the following recommendations as applicable for your TensorFlow models to
+achieve optimal performance.
+
+In general, perform all transformations on the device and ensure that you use
+the latest compatible version of libraries like cuDNN and Intel MKL for your
+platform.
+
+### Optimize the input data pipeline
+
+Use the data from the [#input_pipeline_analyzer] to optimize your data input
+pipeline. An efficient data input pipeline can drastically improve the speed of
+your model execution by reducing device idle time. Try to incorporate the best
+practices detailed in the
+[Better performance with the tf.data API](https://www.tensorflow.org/guide/data_performance)
+guide and below to make your data input pipeline more efficient.
+
+*   In general, parallelizing any ops that do not need to be executed
+    sequentially can significantly optimize the data input pipeline.
+
+*   In many cases, it helps to change the order of some calls or to tune the
+    arguments such that it works best for your model. While optimizing the input
+    data pipeline, benchmark only the data loader without the training and
+    backpropagation steps to quantify the effect of the optimizations
+    independently.
+
+*   Try running your model with synthetic data to check if the input pipeline is
+    a performance bottleneck.
+
+*   Use `tf.data.Dataset.shard` for multi-GPU training. Ensure you shard very
+    early on in the input loop to prevent reductions in throughput. When working
+    with TFRecords, ensure you shard the list of TFRecords and not the contents
+    of the TFRecords.
+
+*   Parallelize several ops by dynamically setting the value of
+    `num_parallel_calls` using `tf.data.AUTOTUNE`.
+
+*   Consider limiting the usage of `tf.data.Dataset.from_generator` as it is
+    slower compared to pure TensorFlow ops.
+
+*   Consider limiting the usage of `tf.py_function` as it cannot be serialized
+    and is not supported to run in distributed TensorFlow.
+
+*   Use `tf.data.Options` to control static optimizations to the input pipeline.
+
+Also read the `tf.data` performance analysis
+[guide](https://www.tensorflow.org/guide/data_performance_analysis) for more
+guidance on optimizing your input pipeline.
+
+#### Optimize data augmentation
+
+When working with image data, make your
+[data augmentation](https://www.tensorflow.org/tutorials/images/data_augmentation)
+more efficient by casting to different data types after applying
+spatial transformations, such as flipping, cropping, rotating, etc.
+
+Note: Some ops like `tf.image.resize` transparently change the `dtype` to
+`fp32`. Make sure you normalize your data to lie between `0` and `1` if it's not
+done automatically. Skipping this step could lead to `NaN` errors if you have
+enabled [AMP](https://developer.nvidia.com/automatic-mixed-precision).
+
+#### Use NVIDIA® DALI
+
+In some instances, such as when you have a system with a high GPU to CPU ratio,
+all of the above optimizations may not be enough to eliminate bottlenecks in the
+data loader caused due to limitations of CPU cycles.
+
+If you are using NVIDIA® GPUs for computer vision and audio deep learning
+applications, consider using the Data Loading Library
+([DALI](https://docs.nvidia.com/deeplearning/dali/user-guide/docs/examples/getting%20started.html))
+to accelerate the data pipeline.
+
+Check the
+[NVIDIA® DALI: Operations](https://docs.nvidia.com/deeplearning/dali/user-guide/docs/supported_ops.html)
+documentation for a list of supported DALI ops.
+
+### Use threading and parallel execution
+
+Run ops on multiple CPU threads with the `tf.config.threading` API to execute
+them faster.
+
+TensorFlow automatically sets the number of parallelism threads by default. The
+thread pool available for running TensorFlow ops depends on the number of CPU
+threads available.
+
+Control the maximum parallel speedup for a single op by using
+`tf.config.threading.set_intra_op_parallelism_threads`. Note that if you run
+multiple ops in parallel, they will all share the available thread pool.
+
+If you have independent non-blocking ops (ops with no directed path between them
+on the graph), use `tf.config.threading.set_inter_op_parallelism_threads` to run
+them concurrently using the available thread pool.
+
+### Miscellaneous
+
+When working with smaller models on NVIDIA® GPUs, you can set
+`tf.compat.v1.ConfigProto.force_gpu_compatible=True` to force all CPU tensors to
+be allocated with CUDA pinned memory to give a significant boost to model
+performance. However, exercise caution while using this option for unknown/very
+large models as this might negatively impact the host (CPU) performance.
+
+### Improve device performance
+
+Follow the best practices detailed here and in the
+[GPU performance optimization guide](https://www.tensorflow.org/guide/gpu_performance_analysis)
+to optimize on-device TensorFlow model performance.
+
+If you are using NVIDIA GPUs, log the GPU and memory utilization to a CSV file
+by running:
+
+```shell
+nvidia-smi
+--query-gpu=utilization.gpu,utilization.memory,memory.total,
+memory.free,memory.used --format=csv
+```
+
+#### Configure data layout
+
+When working with data that contains channel information (like images), optimize
+the data layout format to prefer channels last (NHWC over NCHW).
+
+Channel-last data formats improve
+[Tensor Core](https://www.nvidia.com/en-gb/data-center/tensor-cores/)
+utilization and provide significant performance improvements especially in
+convolutional models when coupled with AMP. NCHW data layouts can still be
+operated on by Tensor Cores, but introduce additional overhead due to automatic
+transpose ops.
+
+You can optimize the data layout to prefer NHWC layouts by setting
+`data_format="channels_last"` for layers such as `tf.keras.layers.Conv2D`,
+`tf.keras.layers.Conv3D`, and
+`tf.keras.layers.RandomRotation`.
+
+Use `tf.keras.backend.set_image_data_format` to set the default data layout
+format for the Keras backend API.
+
+#### Max out the L2 cache
+
+When working with NVIDIA® GPUs, execute the code snippet below before the
+training loop to max out the L2 fetch granularity to 128 bytes.
+
+```python
+import ctypes
+
+_libcudart = ctypes.CDLL('libcudart.so')
+# Set device limit on the current device
+# cudaLimitMaxL2FetchGranularity = 0x05
+pValue = ctypes.cast((ctypes.c_int*1)(), ctypes.POINTER(ctypes.c_int))
+_libcudart.cudaDeviceSetLimit(ctypes.c_int(0x05), ctypes.c_int(128))
+_libcudart.cudaDeviceGetLimit(pValue, ctypes.c_int(0x05))
+assert pValue.contents.value == 128
+```
+
+#### Configure GPU thread usage
+
+The GPU thread mode decides how GPU threads are used.
+
+Set the thread mode to `gpu_private` to make sure that preprocessing does not
+steal all the GPU threads. This will reduce the kernel launch delay during
+training. You can also set the number of threads per GPU. Set these values using
+environment variables.
+
+```python
+import os
+
+os.environ['TF_GPU_THREAD_MODE']='gpu_private'
+os.environ['TF_GPU_THREAD_COUNT']='1'
+```
+
+#### Configure GPU memory options
+
+In general, increase the batch size and scale the model to better utilize GPUs
+and get higher throughput. Note that increasing the batch size will change the
+model’s accuracy so the model needs to be scaled by tuning hyperparameters like
+the learning rate to meet the target accuracy.
+
+Also, use `tf.config.experimental.set_memory_growth` to allow GPU memory to grow
+to prevent all the available memory from being fully allocated to ops that
+require only a fraction of the memory. This allows other processes which consume
+GPU memory to run on the same device.
+
+To learn more, check out the
+[Limiting GPU memory growth](https://www.tensorflow.org/guide/gpu#limiting_gpu_memory_growth)
+guidance in the GPU guide to learn more.
+
+#### Miscellaneous
+
+*   Increase the training mini-batch size (number of training samples used per
+    device in one iteration of the training loop) to the maximum amount that
+    fits without an out of memory (OOM) error on the GPU. Increasing the batch
+    size impacts the model's accuracy—so make sure you scale the model by tuning
+    hyperparameters to meet the target accuracy.
+
+*   Disable reporting OOM errors during tensor allocation in production code.
+    Set `report_tensor_allocations_upon_oom=False` in `tf.compat.v1.RunOptions`.
+
+*   For models with convolution layers, remove bias addition if using batch
+    normalization. Batch normalization shifts values by their mean and this
+    removes the need to have a constant bias term.
+
+*   Use TF Stats to find out how efficiently on-device ops run.
+
+*   Use `tf.function` to perform computations and optionally, enable the
+    `jit_compile=True` flag (`tf.function(jit_compile=True`). To learn more, go
+    to
+    [Use XLA tf.function](https://www.tensorflow.org/xla/tutorials/jit_compile).
+
+*   Minimize host Python operations between steps and reduce callbacks.
+    Calculate metrics every few steps instead of at every step.
+
+*   Keep the device compute units busy.
+
+*   Send data to multiple devices in parallel.
+
+*   Consider
+    [using 16-bit numerical representations](https://www.tensorflow.org/guide/mixed_precision),
+    such as `fp16`—the half-precision floating point format specified by IEEE—or
+    the Brain floating-point
+    [bfloat16](https://cloud.google.com/tpu/docs/bfloat16) format.
+
+## Additional resources
+
+-   The
+    [TensorFlow Profiler: Profile model performance](https://www.tensorflow.org/tensorboard/tensorboard_profiling_keras)
+    tutorial with Keras and TensorBoard where you can apply the advice in this
+    guide.
+-   The
+    [Performance profiling in TensorFlow 2](https://www.youtube.com/watch?v=pXHAQIhhMhI)
+    talk from the TensorFlow Dev Summit 2020.
+-   The [TensorFlow Profiler demo](https://www.youtube.com/watch?v=e4_4D7uNvf8)
+    from the TensorFlow Dev Summit 2020.
+
+## Known limitations
+
+### Profiling multiple GPUs on TensorFlow 2.2 and TensorFlow 2.3
+
+TensorFlow 2.2 and 2.3 support multiple GPU profiling for single host systems
+only; multiple GPU profiling for multi-host systems is not supported. To profile
+multi-worker GPU configurations, each worker has to be profiled independently.
+From TensorFlow 2.4 multiple workers can be profiled using the
+`tf.profiler.experimental.client.trace` API.
+
+CUDA® Toolkit 10.2 or later is required to profile multiple GPUs. As TensorFlow
+2.2 and 2.3 support CUDA® Toolkit versions only up to 10.1, you need to create
+symbolic links to `libcudart.so.10.1` and `libcupti.so.10.1`:
+
+```shell
+sudo ln -s /usr/local/cuda/lib64/libcudart.so.10.2 /usr/local/cuda/lib64/libcudart.so.10.1
+sudo ln -s /usr/local/cuda/extras/CUPTI/lib64/libcupti.so.10.2 /usr/local/cuda/extras/CUPTI/lib64/libcupti.so.10.1
+```
diff --git a/site/en/guide/ragged_tensor.ipynb b/site/en/guide/ragged_tensor.ipynb
index 9be4ae05ddb..ba0be2928ce 100644
--- a/site/en/guide/ragged_tensor.ipynb
+++ b/site/en/guide/ragged_tensor.ipynb
@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "nibpbUnTsxTd"
       },
       "source": [
@@ -12,10 +11,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
+        "cellView": "form",
         "id": "tXAbWHtqs1Y2"
       },
       "outputs": [],
@@ -36,32 +34,39 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "HTgMAvQq-PU_"
       },
       "source": [
         "# Ragged tensors\n",
         "\n",
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/ragged_tensor\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/ragged_tensor.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/ragged_tensor.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/ragged_tensor.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5DP8XNP-6zlu"
+      },
+      "source": [
+        "**API Documentation:** [`tf.RaggedTensor`](https://www.tensorflow.org/api_docs/python/tf/RaggedTensor) [`tf.ragged`](https://www.tensorflow.org/api_docs/python/tf/ragged)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "cDIUjj07-rQg"
       },
       "source": [
@@ -70,59 +75,48 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "KKvdSorS-pDD"
       },
       "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
+        "!pip install --pre -U tensorflow\n",
         "import math\n",
-        "try:\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
         "import tensorflow as tf"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "pxi0m_yf-te5"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "Your data comes in many shapes; your tensors should too.\n",
-        "*Ragged tensors* are the TensorFlow equivalent of nested variable-length\n",
-        "lists. They make it easy to store and process data with non-uniform shapes,\n",
-        "including:\n",
-        "\n",
-        "*   Variable-length features, such as the set of actors in a movie.\n",
-        "*   Batches of variable-length sequential inputs, such as sentences or video\n",
-        "    clips.\n",
-        "*   Hierarchical inputs, such as text documents that are subdivided into\n",
-        "    sections, paragraphs, sentences, and words.\n",
-        "*   Individual fields in structured inputs, such as protocol buffers.\n",
+        "Your data comes in many shapes; your tensors should too. *Ragged tensors* are the TensorFlow equivalent of nested variable-length lists. They make it easy to store and process data with non-uniform shapes, including:\n",
         "\n",
+        "- Variable-length features, such as the set of actors in a movie.\n",
+        "- Batches of variable-length sequential inputs, such as sentences or video clips.\n",
+        "- Hierarchical inputs, such as text documents that are subdivided into sections, paragraphs, sentences, and words.\n",
+        "- Individual fields in structured inputs, such as protocol buffers.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1mhU_qY3_mla"
+      },
+      "source": [
         "### What you can do with a ragged tensor\n",
         "\n",
-        "Ragged tensors are supported by more than a hundred TensorFlow operations,\n",
-        "including math operations (such as `tf.add` and `tf.reduce_mean`), array operations\n",
-        "(such as `tf.concat` and `tf.tile`), string manipulation ops (such as\n",
-        "`tf.substr`), and many others:\n"
+        "Ragged tensors are supported by more than a hundred TensorFlow operations, including math operations (such as `tf.add` and `tf.reduce_mean`), array operations (such as `tf.concat` and `tf.tile`), string manipulation ops (such as `tf.strings.substr`), control flow operations (such as `tf.while_loop` and `tf.map_fn`), and many others:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "vGmJGSf_-PVB"
       },
       "outputs": [],
@@ -133,33 +127,45 @@
         "print(tf.reduce_mean(digits, axis=1))\n",
         "print(tf.concat([digits, [[5, 3]]], axis=0))\n",
         "print(tf.tile(digits, [1, 2]))\n",
-        "print(tf.strings.substr(words, 0, 2))"
+        "print(tf.strings.substr(words, 0, 2))\n",
+        "print(tf.map_fn(tf.math.square, digits))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Pt-5OIc8-PVG"
       },
       "source": [
         "There are also a number of methods and operations that are\n",
         "specific to ragged tensors, including factory methods, conversion methods,\n",
         "and value-mapping operations.\n",
-        "For a list of supported ops, see the `tf.ragged` package\n",
-        "documentation.\n",
-        "\n",
-        "As with normal tensors, you can use Python-style indexing to access specific\n",
-        "slices of a ragged tensor. For more information, see the section on\n",
-        "**Indexing** below."
+        "For a list of supported ops, see the **`tf.ragged` package\n",
+        "documentation**."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r8fjGgf3B_6z"
+      },
+      "source": [
+        "Ragged tensors are supported by many TensorFlow APIs, including [Keras](https://www.tensorflow.org/guide/keras), [Datasets](https://www.tensorflow.org/guide/data), [tf.function](https://www.tensorflow.org/guide/function), [SavedModels](https://www.tensorflow.org/guide/saved_model), and [tf.Example](https://www.tensorflow.org/tutorials/load_data/tfrecord).  For more information, check the section on **TensorFlow APIs** below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aTXLjQlcHP8a"
+      },
+      "source": [
+        "As with normal tensors, you can use Python-style indexing to access specific slices of a ragged tensor. For more information, refer to the section on **Indexing** below."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "n8YMKXpI-PVH"
       },
       "outputs": [],
@@ -169,10 +175,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Awi8i9q5_DuX"
       },
       "outputs": [],
@@ -182,10 +186,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "sXgQtTcgHHMR"
       },
       "outputs": [],
@@ -196,21 +198,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6FU5T_-8-PVK"
       },
       "source": [
-        "And just like normal tensors, you can use Python arithmetic and comparison\n",
-        "operators to perform elementwise operations. For more information, see the section on\n",
-        "**Overloaded Operators** below."
+        "And just like normal tensors, you can use Python arithmetic and comparison operators to perform elementwise operations. For more information, check the section on **Overloaded operators** below."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "2tdUEtb7-PVL"
       },
       "outputs": [],
@@ -220,10 +217,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "X-bxG0nc_Nmf"
       },
       "outputs": [],
@@ -234,7 +229,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2tsw8mN0ESIT"
       },
       "source": [
@@ -243,10 +237,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "pvt5URbdEt-D"
       },
       "outputs": [],
@@ -258,23 +250,49 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "HNxF6_QKAzkl"
+      },
+      "source": [
+        "Ragged tensors can be converted to nested Python `list`s and NumPy `array`s:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "A5NHb8ViA9dt"
+      },
+      "outputs": [],
+      "source": [
+        "digits.to_list()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2o1wogVyA6Yp"
+      },
+      "outputs": [],
+      "source": [
+        "digits.numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "7M5RHOgp-PVN"
       },
       "source": [
         "### Constructing a ragged tensor\n",
         "\n",
-        "The simplest way to construct a ragged tensor is using\n",
-        "`tf.ragged.constant`, which builds the\n",
-        "`RaggedTensor` corresponding to a given nested Python `list`:"
+        "The simplest way to construct a ragged tensor is using `tf.ragged.constant`, which builds the `RaggedTensor` corresponding to a given nested Python `list` or NumPy `array`:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "yhgKMozw-PVP"
       },
       "outputs": [],
@@ -287,10 +305,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "TW1g7eE2ee8M"
       },
       "outputs": [],
@@ -305,41 +321,34 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "SPLn5xHn-PVR"
       },
       "source": [
-        "Ragged tensors can also be constructed by pairing flat *values* tensors with\n",
-        "*row-partitioning* tensors indicating how those values should be divided into\n",
-        "rows, using factory classmethods such as `tf.RaggedTensor.from_value_rowids`,\n",
-        "`tf.RaggedTensor.from_row_lengths`, and\n",
-        "`tf.RaggedTensor.from_row_splits`.\n",
+        "Ragged tensors can also be constructed by pairing flat *values* tensors with *row-partitioning* tensors indicating how those values should be divided into rows, using factory classmethods such as `tf.RaggedTensor.from_value_rowids`, `tf.RaggedTensor.from_row_lengths`, and `tf.RaggedTensor.from_row_splits`.\n",
         "\n",
         "#### `tf.RaggedTensor.from_value_rowids`\n",
-        "If you know which row each value belongs in, then you can build a `RaggedTensor` using a `value_rowids` row-partitioning tensor:\n",
         "\n",
-        "![value_rowids](https://www.tensorflow.org/images/ragged_tensors/value_rowids.png)"
+        "If you know which row each value belongs to, then you can build a `RaggedTensor` using a `value_rowids` row-partitioning tensor:\n",
+        "\n",
+        "![value_rowids row-partitioning tensor](https://www.tensorflow.org/images/ragged_tensors/value_rowids.png)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "SEvcPUcl-PVS"
       },
       "outputs": [],
       "source": [
         "print(tf.RaggedTensor.from_value_rowids(\n",
-        "    values=[3, 1, 4, 1, 5, 9, 2, 6],\n",
-        "    value_rowids=[0, 0, 0, 0, 2, 2, 2, 3]))"
+        "    values=[3, 1, 4, 1, 5, 9, 2],\n",
+        "    value_rowids=[0, 0, 0, 0, 2, 2, 3]))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "RBQh8sYc-PVV"
       },
       "source": [
@@ -347,28 +356,25 @@
         "\n",
         "If you know how long each row is, then you can use a `row_lengths` row-partitioning tensor:\n",
         "\n",
-        "![row_lengths](https://www.tensorflow.org/images/ragged_tensors/row_lengths.png)"
+        "![row_lengths row-partitioning tensor](https://www.tensorflow.org/images/ragged_tensors/row_lengths.png)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "LBY81WXl-PVW"
       },
       "outputs": [],
       "source": [
         "print(tf.RaggedTensor.from_row_lengths(\n",
-        "    values=[3, 1, 4, 1, 5, 9, 2, 6],\n",
-        "    row_lengths=[4, 0, 3, 1]))"
+        "    values=[3, 1, 4, 1, 5, 9, 2],\n",
+        "    row_lengths=[4, 0, 2, 1]))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8p5V8_Iu-PVa"
       },
       "source": [
@@ -376,38 +382,36 @@
         "\n",
         "If you know the index where each row starts and ends, then you can use a `row_splits` row-partitioning tensor:\n",
         "\n",
-        "![row_splits](https://www.tensorflow.org/images/ragged_tensors/row_splits.png)"
+        "![row_splits row-partitioning tensor](https://www.tensorflow.org/images/ragged_tensors/row_splits.png)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "FwizuqZI-PVb"
       },
       "outputs": [],
       "source": [
         "print(tf.RaggedTensor.from_row_splits(\n",
-        "    values=[3, 1, 4, 1, 5, 9, 2, 6],\n",
-        "    row_splits=[0, 4, 4, 7, 8]))"
+        "    values=[3, 1, 4, 1, 5, 9, 2],\n",
+        "    row_splits=[0, 4, 4, 6, 7]))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "E-9imo8DhwuA"
       },
       "source": [
-        "See the `tf.RaggedTensor` class documentation for a full list of factory methods."
+        "See the `tf.RaggedTensor` class documentation for a full list of factory methods.\n",
+        "\n",
+        "Note: By default, these factory methods add assertions that the row partition tensor is well-formed and consistent with the number of values.  The `validate=False` parameter can be used to skip these checks if you can guarantee that the inputs are well-formed and consistent."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "YQAOsT1_-PVg"
       },
       "source": [
@@ -420,10 +424,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "SqbPBd_w-PVi"
       },
       "outputs": [],
@@ -433,10 +435,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "83ZCSJnQAWAf"
       },
       "outputs": [],
@@ -446,10 +446,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ewA3cISdDfmP"
       },
       "outputs": [],
@@ -462,10 +460,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "EOWIlVidDl-n"
       },
       "outputs": [],
@@ -479,24 +475,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "nhHMFhSp-PVq"
       },
       "source": [
-        "### Example use case\n",
+        "## Example use case\n",
         "\n",
-        "The following example demonstrates how `RaggedTensor`s can be used to construct\n",
-        "and combine unigram and bigram embeddings for a batch of variable-length\n",
-        "queries, using special markers for the beginning and end of each sentence.\n",
-        "For more details on the ops used in this example, see the `tf.ragged` package documentation."
+        "The following example demonstrates how `RaggedTensor`s can be used to construct and combine unigram and bigram embeddings for a batch of variable-length queries, using special markers for the beginning and end of each sentence. For more details on the ops used in this example, check the `tf.ragged` package documentation."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ZBs_V7e--PVr"
       },
       "outputs": [],
@@ -514,21 +504,17 @@
         "\n",
         "# Look up the embedding for each word.\n",
         "word_buckets = tf.strings.to_hash_bucket_fast(queries, num_buckets)\n",
-        "word_embeddings = tf.ragged.map_flat_values(\n",
-        "    tf.nn.embedding_lookup, embedding_table, word_buckets)                  # ①\n",
+        "word_embeddings = tf.nn.embedding_lookup(embedding_table, word_buckets)     # ①\n",
         "\n",
         "# Add markers to the beginning and end of each sentence.\n",
         "marker = tf.fill([queries.nrows(), 1], '#')\n",
         "padded = tf.concat([marker, queries, marker], axis=1)                       # ②\n",
         "\n",
-        "# Build word bigrams \u0026 look up embeddings.\n",
-        "bigrams = tf.strings.join([padded[:, :-1],\n",
-        "                               padded[:, 1:]],\n",
-        "                              separator='+')                                # ③\n",
+        "# Build word bigrams and look up embeddings.\n",
+        "bigrams = tf.strings.join([padded[:, :-1], padded[:, 1:]], separator='+')   # ③\n",
         "\n",
         "bigram_buckets = tf.strings.to_hash_bucket_fast(bigrams, num_buckets)\n",
-        "bigram_embeddings = tf.ragged.map_flat_values(\n",
-        "    tf.nn.embedding_lookup, embedding_table, bigram_buckets)                # ④\n",
+        "bigram_embeddings = tf.nn.embedding_lookup(embedding_table, bigram_buckets) # ④\n",
         "\n",
         "# Find the average embedding for each sentence\n",
         "all_embeddings = tf.concat([word_embeddings, bigram_embeddings], axis=1)    # ⑤\n",
@@ -539,84 +525,39 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Y_lE_LAVcWQH"
       },
       "source": [
-        "![ragged_example](https://www.tensorflow.org/images/ragged_tensors/ragged_example.png)"
+        "![Ragged tensor example](https://www.tensorflow.org/images/ragged_tensors/ragged_example.png)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "An_k0pX1-PVt"
       },
       "source": [
-        "## Ragged tensors: definitions\n",
-        "\n",
-        "### Ragged and uniform dimensions\n",
-        "\n",
-        "A *ragged tensor* is a tensor with one or more *ragged dimensions*,\n",
-        "which are dimensions whose slices may have different lengths. For example, the\n",
-        "inner (column) dimension of `rt=[[3, 1, 4, 1], [], [5, 9, 2], [6], []]` is\n",
-        "ragged, since the column slices (`rt[0, :]`, ..., `rt[4, :]`) have different\n",
-        "lengths. Dimensions whose slices all have the same length are called *uniform\n",
-        "dimensions*.\n",
-        "\n",
-        "The outermost dimension of a ragged tensor is always uniform, since it consists\n",
-        "of a single slice (and so there is no possibility for differing slice lengths).\n",
-        "In addition to the uniform outermost dimension, ragged tensors may also have\n",
-        "uniform inner dimensions. For example, we might store the word embeddings for\n",
-        "each word in a batch of sentences using a ragged tensor with shape\n",
-        "`[num_sentences, (num_words), embedding_size]`, where the parentheses around\n",
-        "`(num_words)` indicate that the dimension is ragged.\n",
-        "\n",
-        "![sent_word_embed](https://www.tensorflow.org/images/ragged_tensors/sent_word_embed.png)\n",
+        "## Ragged and uniform dimensions\n",
         "\n",
-        "Ragged tensors may have multiple ragged dimensions. For example, we could store\n",
-        "a batch of structured text documents using a tensor with shape `[num_documents,\n",
-        "(num_paragraphs), (num_sentences), (num_words)]` (where again parentheses are\n",
-        "used to indicate ragged dimensions).\n",
+        "A ***ragged dimension*** is a dimension whose slices may have different lengths. For example, the inner (column) dimension of `rt=[[3, 1, 4, 1], [], [5, 9, 2], [6], []]` is ragged, since the column slices (`rt[0, :]`, ..., `rt[4, :]`) have different lengths. Dimensions whose slices all have the same length are called *uniform dimensions*.\n",
         "\n",
-        "#### Ragged tensor shape restrictions\n",
+        "The outermost dimension of a ragged tensor is always uniform, since it consists of a single slice (and, therefore, there is no possibility for differing slice lengths). The remaining dimensions may be either ragged or uniform. For example, you may store the word embeddings for each word in a batch of sentences using a ragged tensor with shape `[num_sentences, (num_words), embedding_size]`, where the parentheses around `(num_words)` indicate that the dimension is ragged.\n",
         "\n",
-        "The shape of a ragged tensor is currently restricted to have the following form:\n",
+        "![Word embeddings using a ragged tensor](https://www.tensorflow.org/images/ragged_tensors/sent_word_embed.png)\n",
         "\n",
-        "*   A single uniform dimension\n",
-        "*   Followed by one or more ragged dimensions\n",
-        "*   Followed by zero or more uniform dimensions.\n",
+        "Ragged tensors may have multiple ragged dimensions. For example, you could store a batch of structured text documents using a tensor with shape `[num_documents, (num_paragraphs), (num_sentences), (num_words)]` (where again parentheses are used to indicate ragged dimensions).\n",
         "\n",
-        "Note: These restrictions are a consequence of the current implementation, and we\n",
-        "may relax them in the future.\n",
+        "As with `tf.Tensor`, the ***rank*** of a ragged tensor is its total number of dimensions (including both ragged and uniform dimensions). A ***potentially ragged tensor*** is a value that might be either a `tf.Tensor` or a `tf.RaggedTensor`.\n",
         "\n",
-        "### Rank and ragged rank\n",
+        "When describing the shape of a RaggedTensor, ragged dimensions are conventionally indicated by enclosing them in parentheses. For example, as you saw above, the shape of a 3D RaggedTensor that stores word embeddings for each word in a batch of sentences can be written as `[num_sentences, (num_words), embedding_size]`.\n",
         "\n",
-        "The total number of dimensions in a ragged tensor is called its ***rank***, and\n",
-        "the number of ragged dimensions in a ragged tensor is called its ***ragged\n",
-        "rank***. In graph execution mode (i.e., non-eager mode), a tensor's ragged rank\n",
-        "is fixed at creation time: it can't depend\n",
-        "on runtime values, and can't vary dynamically for different session runs.\n",
-        "A ***potentially ragged tensor*** is a value that might be\n",
-        "either a `tf.Tensor` or a `tf.RaggedTensor`. The\n",
-        "ragged rank of a `tf.Tensor` is defined to be zero.\n",
-        "\n",
-        "### RaggedTensor shapes\n",
-        "\n",
-        "When describing the shape of a RaggedTensor, ragged dimensions are indicated by\n",
-        "enclosing them in parentheses. For example, as we saw above, the shape of a 3-D\n",
-        "RaggedTensor that stores word embeddings for each word in a batch of sentences\n",
-        "can be written as `[num_sentences, (num_words), embedding_size]`.\n",
-        "The `RaggedTensor.shape` attribute returns a `tf.TensorShape` for a\n",
-        "ragged tensor, where ragged dimensions have size `None`:\n"
+        "The `RaggedTensor.shape` attribute returns a `tf.TensorShape` for a ragged tensor where ragged dimensions have size `None`:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "M2Wzx4JEIvmb"
       },
       "outputs": [],
@@ -627,7 +568,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "G9tfJOeFlijE"
       },
       "source": [
@@ -637,10 +577,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "5DHaqXHxlWi0"
       },
       "outputs": [],
@@ -651,28 +589,25 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "V8e7x95UcLS6"
       },
       "source": [
-        "## Ragged vs sparse tensors\n",
+        "## Ragged vs sparse\n",
         "\n",
-        "A ragged tensor should *not* be thought of as a type of sparse tensor, but\n",
-        "rather as a dense tensor with an irregular shape.\n",
+        "A ragged tensor should *not* be thought of as a type of sparse tensor. In particular, sparse tensors are *efficient encodings for `tf.Tensor`* that model the same data in a compact format; but ragged tensor is an *extension to `tf.Tensor`* that models an expanded class of data. This difference is crucial when defining operations:\n",
         "\n",
-        "As an illustrative example, consider how array operations such as `concat`,\n",
-        "`stack`, and `tile` are defined for ragged vs. sparse tensors. Concatenating\n",
-        "ragged tensors joins each row to form a single row with the combined length:\n",
+        "- Applying an op to a sparse or dense tensor should always give the same result.\n",
+        "- Applying an op to a ragged or sparse tensor may give different results.\n",
         "\n",
-        "![ragged_concat](https://www.tensorflow.org/images/ragged_tensors/ragged_concat.png)\n"
+        "As an illustrative example, consider how array operations such as `concat`, `stack`, and `tile` are defined for ragged vs. sparse tensors. Concatenating ragged tensors joins each row to form a single row with the combined length:\n",
+        "\n",
+        "![Concatenating ragged tensors](https://www.tensorflow.org/images/ragged_tensors/ragged_concat.png)\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ush7IGUWLXIn"
       },
       "outputs": [],
@@ -685,23 +620,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "pvQzZG8zMoWa"
       },
       "source": [
+        "However, concatenating sparse tensors is equivalent to concatenating the corresponding dense tensors, as illustrated by the following example (where Ø indicates missing values):\n",
         "\n",
-        "But concatenating sparse tensors is equivalent to concatenating the corresponding dense tensors,\n",
-        "as illustrated by the following example (where Ø indicates missing values):\n",
-        "\n",
-        "![sparse_concat](https://www.tensorflow.org/images/ragged_tensors/sparse_concat.png)\n"
+        "![Concatenating sparse tensors](https://www.tensorflow.org/images/ragged_tensors/sparse_concat.png)\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "eTIhGayQL0gI"
       },
       "outputs": [],
@@ -715,7 +645,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Vl8eQN8pMuYx"
       },
       "source": [
@@ -731,586 +660,1252 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "cRcHzS6pcHYC"
-      },
-      "source": [
-        "## Overloaded operators\n",
-        "\n",
-        "The `RaggedTensor` class overloads the standard Python arithmetic and comparison\n",
-        "operators, making it easy to perform basic elementwise math:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "skScd37P-PVu"
+        "id": "u4yjxcK7IPXc"
       },
-      "outputs": [],
       "source": [
-        "x = tf.ragged.constant([[1, 2], [3], [4, 5, 6]])\n",
-        "y = tf.ragged.constant([[1, 1], [2], [3, 3, 3]])\n",
-        "print(x + y)"
+        "## TensorFlow APIs"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "XEGgbZHV-PVw"
+        "id": "VoZGwFQjIYU5"
       },
       "source": [
-        "Since the overloaded operators perform elementwise computations, the inputs to\n",
-        "all binary operations must have the same shape, or be broadcastable to the same\n",
-        "shape. In the simplest broadcasting case, a single scalar is combined\n",
-        "elementwise with each value in a ragged tensor:"
+        "### Keras\n",
+        "\n",
+        "[tf.keras](https://www.tensorflow.org/guide/keras) is TensorFlow's high-level API for building and training deep learning models. It doesn't have ragged support. But it does support masked tensors. So the easiest way to use a ragged tensor in a Keras model  is to convert the ragged tensor to a dense tensor, using `.to_tensor()` and then using Keras's builtin masking:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "IYybEEWc-PVx"
+        "id": "ucYf2sSzTvQo"
       },
       "outputs": [],
       "source": [
-        "x = tf.ragged.constant([[1, 2], [3], [4, 5, 6]])\n",
-        "print(x + 3)"
+        "# Task: predict whether each sentence is a question or not.\n",
+        "sentences = tf.constant(\n",
+        "    ['What makes you think she is a witch?',\n",
+        "     'She turned me into a newt.',\n",
+        "     'A newt?',\n",
+        "     'Well, I got better.'])\n",
+        "is_question = tf.constant([True, False, True, False])"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "okGb9dIi-PVz"
+        "id": "MGYKmizJTw8B"
       },
+      "outputs": [],
       "source": [
-        "For a discussion of more advanced cases, see the section on\n",
-        "**Broadcasting**.\n",
-        "\n",
-        "Ragged tensors overload the same set of operators as normal `Tensor`s: the unary\n",
-        "operators `-`, `~`, and `abs()`; and the binary operators `+`, `-`, `*`, `/`,\n",
-        "`//`, `%`, `**`, `\u0026`, `|`, `^`, `==`, `\u003c`, `\u003c=`, `\u003e`, and `\u003e=`.\n"
+        "# Preprocess the input strings.\n",
+        "hash_buckets = 1000\n",
+        "words = tf.strings.split(sentences, ' ')\n",
+        "hashed_words = tf.strings.to_hash_bucket_fast(words, hash_buckets)\n",
+        "hashed_words.to_list()"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "f2anbs6ZnFtl"
+        "id": "7FTujwOlUT8J"
       },
+      "outputs": [],
       "source": [
-        "## Indexing\n",
-        "\n",
-        "Ragged tensors support Python-style indexing, including multidimensional\n",
-        "indexing and slicing. The following examples demonstrate ragged tensor indexing\n",
-        "with a 2-D and a 3-D ragged tensor.\n",
-        "\n",
-        "### Indexing a 2-D ragged tensor with 1 ragged dimension"
+        "hashed_words.to_tensor()"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "MbSRZRDz-PV1"
+        "id": "vzWudaESUBOZ"
       },
       "outputs": [],
       "source": [
-        "queries = tf.ragged.constant(\n",
-        "    [['Who', 'is', 'George', 'Washington'],\n",
-        "     ['What', 'is', 'the', 'weather', 'tomorrow'],\n",
-        "     ['Goodnight']])\n",
-        "print(queries[1])"
+        "tf.keras.Input?"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "EFfjZV7YA3UH"
+        "id": "pHls7hQVJlk5"
       },
       "outputs": [],
       "source": [
-        "print(queries[1, 2])                # A single word"
+        "# Build the Keras model.\n",
+        "keras_model = tf.keras.Sequential([\n",
+        "    tf.keras.layers.Embedding(hash_buckets, 16, mask_zero=True),\n",
+        "    tf.keras.layers.LSTM(32, return_sequences=True, use_bias=False),\n",
+        "    tf.keras.layers.GlobalAveragePooling1D(),\n",
+        "    tf.keras.layers.Dense(32),\n",
+        "    tf.keras.layers.Activation(tf.nn.relu),\n",
+        "    tf.keras.layers.Dense(1)\n",
+        "])\n",
+        "\n",
+        "keras_model.compile(loss='binary_crossentropy', optimizer='rmsprop')\n",
+        "keras_model.fit(hashed_words.to_tensor(), is_question, epochs=5)\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "VISRPQSdA3xn"
+        "id": "1IAjjmdTU9OU"
       },
       "outputs": [],
       "source": [
-        "print(queries[1:])                  # Everything but the first row"
+        "print(keras_model.predict(hashed_words.to_tensor()))"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "J1PpSyKQBMng"
+        "id": "8B_sdlt6Ij61"
       },
-      "outputs": [],
       "source": [
-        "print(queries[:, :3])               # The first 3 words of each query"
+        "### tf.Example\n",
+        "\n",
+        "[tf.Example](https://www.tensorflow.org/tutorials/load_data/tfrecord) is a standard [protobuf](https://developers.google.com/protocol-buffers/) encoding for TensorFlow data.  Data encoded with `tf.Example`s often includes variable-length features.  For example, the following code defines a batch of four `tf.Example` messages with different feature lengths:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ixrhHmJBeidy"
+        "id": "xsiglYM7TXGr"
       },
       "outputs": [],
       "source": [
-        "print(queries[:, -2:])              # The last 2 words of each query"
+        "import google.protobuf.text_format as pbtext\n",
+        "\n",
+        "def build_tf_example(s):\n",
+        "  return pbtext.Merge(s, tf.train.Example()).SerializeToString()\n",
+        "\n",
+        "example_batch = [\n",
+        "  build_tf_example(r'''\n",
+        "    features {\n",
+        "      feature {key: \"colors\" value {bytes_list {value: [\"red\", \"blue\"]} } }\n",
+        "      feature {key: \"lengths\" value {int64_list {value: [7]} } } }'''),\n",
+        "  build_tf_example(r'''\n",
+        "    features {\n",
+        "      feature {key: \"colors\" value {bytes_list {value: [\"orange\"]} } }\n",
+        "      feature {key: \"lengths\" value {int64_list {value: []} } } }'''),\n",
+        "  build_tf_example(r'''\n",
+        "    features {\n",
+        "      feature {key: \"colors\" value {bytes_list {value: [\"black\", \"yellow\"]} } }\n",
+        "      feature {key: \"lengths\" value {int64_list {value: [1, 3]} } } }'''),\n",
+        "  build_tf_example(r'''\n",
+        "    features {\n",
+        "      feature {key: \"colors\" value {bytes_list {value: [\"green\"]} } }\n",
+        "      feature {key: \"lengths\" value {int64_list {value: [3, 5, 2]} } } }''')]"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "cnOP6Vza-PV4"
+        "id": "szUuXFvtUL2o"
       },
       "source": [
-        "### Indexing a 3-D ragged tensor with 2 ragged dimensions"
+        "You can parse this encoded data using `tf.io.parse_example`, which takes a tensor of serialized strings and a feature specification dictionary, and returns a dictionary mapping feature names to tensors. To read the variable-length features into ragged tensors, you simply use `tf.io.RaggedFeature` in the feature specification dictionary:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "8VbqbKcE-PV6"
+        "id": "xcdaIbYVT4mo"
       },
       "outputs": [],
       "source": [
-        "rt = tf.ragged.constant([[[1, 2, 3], [4]],\n",
-        "                         [[5], [], [6]],\n",
-        "                         [[7]],\n",
-        "                         [[8, 9], [10]]])"
+        "feature_specification = {\n",
+        "    'colors': tf.io.RaggedFeature(tf.string),\n",
+        "    'lengths': tf.io.RaggedFeature(tf.int64),\n",
+        "}\n",
+        "feature_tensors = tf.io.parse_example(example_batch, feature_specification)\n",
+        "for name, value in feature_tensors.items():\n",
+        "  print(\"{}={}\".format(name, value))"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "f9WPVWf4grVp"
+        "id": "IK9X_8rXVr8h"
       },
-      "outputs": [],
       "source": [
-        "print(rt[1])                        # Second row (2-D RaggedTensor)"
+        "`tf.io.RaggedFeature` can also be used to read features with multiple ragged dimensions. For details, refer to the [API documentation](https://www.tensorflow.org/api_docs/python/tf/io/RaggedFeature)."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "ad8FGJoABjQH"
+        "id": "UJowRhlxIX0R"
       },
-      "outputs": [],
       "source": [
-        "print(rt[3, 0])                     # First element of fourth row (1-D Tensor)"
+        "### Datasets\n",
+        "\n",
+        "[tf.data](https://www.tensorflow.org/guide/data) is an API that enables you to build complex input pipelines from simple, reusable pieces.  Its core data structure is `tf.data.Dataset`, which represents a sequence of elements, in which each element consists of one or more components."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "MPPr-a-bBjFE"
+        "id": "fBml1m2G2vO9"
       },
       "outputs": [],
       "source": [
-        "print(rt[:, 1:3])                   # Items 1-3 of each row (3-D RaggedTensor)"
+        "# Helper function used to print datasets in the examples below.\n",
+        "def print_dictionary_dataset(dataset):\n",
+        "  for i, element in enumerate(dataset):\n",
+        "    print(\"Element {}:\".format(i))\n",
+        "    for (feature_name, feature_value) in element.items():\n",
+        "      print('{:>14} = {}'.format(feature_name, feature_value))"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "6SIDeoIUBi4z"
+        "id": "gEu_H1Sp2jz1"
       },
-      "outputs": [],
       "source": [
-        "print(rt[:, -1:])                   # Last item of each row (3-D RaggedTensor)"
+        "#### Building Datasets with ragged tensors\n",
+        "\n",
+        "Datasets can be built from ragged tensors using the same methods that are used to build them from `tf.Tensor`s or NumPy `array`s, such as `Dataset.from_tensor_slices`:"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "_d3nBh1GnWvU"
+        "id": "BuelF_y2mEq9"
       },
+      "outputs": [],
       "source": [
-        "`RaggedTensor`s supports multidimensional indexing and slicing, with one\n",
-        "restriction: indexing into a ragged dimension is not allowed. This case is\n",
-        "problematic because the indicated value may exist in some rows but not others.\n",
-        "In such cases, it's not obvious whether we should (1) raise an `IndexError`; (2)\n",
-        "use a default value; or (3) skip that value and return a tensor with fewer rows\n",
-        "than we started with. Following the\n",
-        "[guiding principles of Python](https://www.python.org/dev/peps/pep-0020/)\n",
-        "(\"In the face\n",
-        "of ambiguity, refuse the temptation to guess\" ), we currently disallow this\n",
-        "operation."
+        "dataset = tf.data.Dataset.from_tensor_slices(feature_tensors)\n",
+        "print_dictionary_dataset(dataset)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "IsWKETULAJbN"
+        "id": "mC-QNkJc56De"
       },
       "source": [
-        "## Tensor Type Conversion\n",
-        "\n",
-        "The `RaggedTensor` class defines methods that can be used to convert\n",
-        "between `RaggedTensor`s and `tf.Tensor`s or `tf.SparseTensors`:"
+        "Note: `Dataset.from_generator` does not support ragged tensors yet, but support will be added soon."
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "INnfmZGcBoU_"
+        "id": "K0UKvBLf1VMu"
       },
-      "outputs": [],
       "source": [
-        "ragged_sentences = tf.ragged.constant([\n",
-        "    ['Hi'], ['Welcome', 'to', 'the', 'fair'], ['Have', 'fun']])\n",
-        "print(ragged_sentences.to_tensor(default_value=''))"
+        "#### Batching and unbatching Datasets with ragged tensors\n",
+        "\n",
+        "Datasets with ragged tensors can be batched (which combines *n* consecutive elements into a single elements) using the `Dataset.batch` method."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "41WAZLXNnbwH"
+        "id": "lk62aRz63IZn"
       },
       "outputs": [],
       "source": [
-        "print(ragged_sentences.to_sparse())"
+        "batched_dataset = dataset.batch(2)\n",
+        "print_dictionary_dataset(batched_dataset)"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "-rfiyYqne8QN"
+        "id": "NLSGiYEQ5A8N"
       },
-      "outputs": [],
       "source": [
-        "x = [[1, 3, -1, -1], [2, -1, -1, -1], [4, 5, 8, 9]]\n",
-        "print(tf.RaggedTensor.from_tensor(x, padding=-1))"
+        "Conversely, a batched dataset can be transformed into a flat dataset using `Dataset.unbatch`."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "S8MkYo2hfVhj"
+        "id": "CxLlaPw_5Je4"
       },
       "outputs": [],
       "source": [
-        "st = tf.SparseTensor(indices=[[0, 0], [2, 0], [2, 1]],\n",
-        "                     values=['a', 'b', 'c'],\n",
-        "                     dense_shape=[3, 3])\n",
-        "print(tf.RaggedTensor.from_sparse(st))"
+        "unbatched_dataset = batched_dataset.unbatch()\n",
+        "print_dictionary_dataset(unbatched_dataset)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "qx025sNMkAHH"
+        "id": "YzpLQFh33q0N"
       },
       "source": [
-        "## Evaluating ragged tensors\n",
-        "\n",
-        "### Eager execution\n",
-        "\n",
-        "In eager execution mode, ragged tensors are evaluated immediately. To access the\n",
-        "values they contain, you can:\n",
+        "#### Batching Datasets with variable-length non-ragged tensors\n",
         "\n",
-        "*   Use the\n",
-        "    `tf.RaggedTensor.to_list()`\n",
-        "    method, which converts the ragged tensor to a Python `list`."
+        "If you have a Dataset that contains non-ragged tensors, and tensor lengths vary across elements, then you can batch those non-ragged tensors into ragged tensors by applying the `dense_to_ragged_batch` transformation:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "uMm1WMkc-PV_"
+        "id": "PYnhERwh3_mf"
       },
       "outputs": [],
       "source": [
-        "rt = tf.ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]])\n",
-        "print(rt.to_list())"
+        "non_ragged_dataset = tf.data.Dataset.from_tensor_slices([1, 5, 3, 2, 8])\n",
+        "non_ragged_dataset = non_ragged_dataset.map(tf.range)\n",
+        "batched_non_ragged_dataset = non_ragged_dataset.apply(\n",
+        "    tf.data.experimental.dense_to_ragged_batch(2))\n",
+        "for element in batched_non_ragged_dataset:\n",
+        "  print(element)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "SrizmqTc-PWC"
+        "id": "nXFPeE-CzJ-s"
       },
       "source": [
-        "*   Use Python indexing. If the tensor piece you select contains no ragged\n",
-        "    dimensions, then it will be returned as an `EagerTensor`. You can then use\n",
-        "    the `numpy()` method to access the value directly."
+        "#### Transforming Datasets with ragged tensors\n",
+        "\n",
+        "You can also create or transform ragged tensors in Datasets using `Dataset.map`:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "HpRHhfLe-PWD"
+        "id": "Ios1GuG-pf9U"
       },
       "outputs": [],
       "source": [
-        "print(rt[1].numpy())"
+        "def transform_lengths(features):\n",
+        "  return {\n",
+        "      'mean_length': tf.math.reduce_mean(features['lengths']),\n",
+        "      'length_ranges': tf.ragged.range(features['lengths'])}\n",
+        "transformed_dataset = dataset.map(transform_lengths)\n",
+        "print_dictionary_dataset(transformed_dataset)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "sNlpI2fR-PWF"
+        "id": "WD2lWw3fIXrg"
       },
       "source": [
-        "*   Decompose the ragged tensor into its components, using the\n",
-        "    `tf.RaggedTensor.values`\n",
-        "    and\n",
-        "    `tf.RaggedTensor.row_splits`\n",
-        "    properties, or row-paritioning methods such as `tf.RaggedTensor.row_lengths()`\n",
-        "    and `tf.RaggedTensor.value_rowids()`."
+        "### tf.function\n",
+        "\n",
+        "[tf.function](https://www.tensorflow.org/guide/function) is a decorator that precomputes TensorFlow graphs for Python functions, which can substantially improve the performance of your TensorFlow code.  Ragged tensors can be used transparently with `@tf.function`-decorated functions. For example, the following function works with both ragged and non-ragged tensors:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "yTckrLdB-PWG"
+        "id": "PfyxgVaj_8tl"
       },
       "outputs": [],
       "source": [
-        "print(rt.values)"
+        "@tf.function\n",
+        "def make_palindrome(x, axis):\n",
+        "  return tf.concat([x, tf.reverse(x, [axis])], axis)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "B8OnG9NzCEnv"
+        "id": "vcZdzvEnDEt0"
       },
       "outputs": [],
       "source": [
-        "print(rt.row_splits)"
+        "make_palindrome(tf.constant([[1, 2], [3, 4], [5, 6]]), axis=1)"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "EdljbNPq-PWS"
+        "id": "4WfCMIgdDMxj"
       },
+      "outputs": [],
       "source": [
-        "### Broadcasting\n",
-        "\n",
-        "Broadcasting is the process of making tensors with different shapes have\n",
-        "compatible shapes for elementwise operations. For more background on\n",
-        "broadcasting, see:\n",
-        "\n",
-        "*   [Numpy: Broadcasting](https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)\n",
-        "*   `tf.broadcast_dynamic_shape`\n",
-        "*   `tf.broadcast_to`\n",
-        "\n",
-        "The basic steps for broadcasting two inputs `x` and `y` to have compatible\n",
-        "shapes are:\n",
-        "\n",
-        "1.  If `x` and `y` do not have the same number of dimensions, then add outer\n",
-        "    dimensions (with size 1) until they do.\n",
-        "\n",
-        "2.  For each dimension where `x` and `y` have different sizes:\n",
-        "\n",
-        "    *   If `x` or `y` have size `1` in dimension `d`, then repeat its values\n",
-        "        across dimension `d` to match the other input's size.\n",
-        "\n",
-        "    *   Otherwise, raise an exception (`x` and `y` are not broadcast\n",
-        "        compatible)."
+        "make_palindrome(tf.ragged.constant([[1, 2], [3], [4, 5, 6]]), axis=1)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "-S2hOUWx-PWU"
+        "id": "X2p69YPOBUz8"
       },
       "source": [
-        "Where the size of a tensor in a uniform dimension is a single number (the size\n",
-        "of slices across that dimension); and the size of a tensor in a ragged dimension\n",
-        "is a list of slice lengths (for all slices across that dimension).\n",
-        "\n",
-        "#### Broadcasting examples"
+        "If you wish to explicitly specify the `input_signature` for the `tf.function`, then you can do so using `tf.RaggedTensorSpec`."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0n095XdR-PWU"
+        "id": "k6-hkhdDBk6G"
       },
       "outputs": [],
       "source": [
-        "# x       (2D ragged):  2 x (num_rows)\n",
-        "# y       (scalar)\n",
-        "# result  (2D ragged):  2 x (num_rows)\n",
-        "x = tf.ragged.constant([[1, 2], [3]])\n",
-        "y = 3\n",
-        "print(x + y)"
+        "@tf.function(\n",
+        "    input_signature=[tf.RaggedTensorSpec(shape=[None, None], dtype=tf.int32)])\n",
+        "def max_and_min(rt):\n",
+        "  return (tf.math.reduce_max(rt, axis=-1), tf.math.reduce_min(rt, axis=-1))\n",
+        "\n",
+        "max_and_min(tf.ragged.constant([[1, 2], [3], [4, 5, 6]]))"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0SVYk5AP-PWW"
+        "id": "fSs-7E0VD85q"
       },
-      "outputs": [],
       "source": [
-        "# x         (2d ragged):  3 x (num_rows)\n",
-        "# y         (2d tensor):  3 x          1\n",
-        "# Result    (2d ragged):  3 x (num_rows)\n",
-        "x = tf.ragged.constant(\n",
-        "   [[10, 87, 12],\n",
-        "    [19, 53],\n",
-        "    [12, 32]])\n",
-        "y = [[1000], [2000], [3000]]\n",
-        "print(x + y)"
+        "#### Concrete functions\n",
+        "\n",
+        "[Concrete functions](https://www.tensorflow.org/guide/function#obtaining_concrete_functions) encapsulate individual traced graphs that are built by `tf.function`. Ragged tensors can be used transparently with concrete functions.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "MsfBMD80s8Ux"
+        "id": "yyJeXJ4wFWox"
       },
       "outputs": [],
       "source": [
-        "# x      (3d ragged):  2 x (r1) x 2\n",
-        "# y      (2d ragged):         1 x 1\n",
-        "# Result (3d ragged):  2 x (r1) x 2\n",
-        "x = tf.ragged.constant(\n",
-        "    [[[1, 2], [3, 4], [5, 6]],\n",
-        "     [[7, 8]]],\n",
-        "    ragged_rank=1)\n",
-        "y = tf.constant([[10]])\n",
-        "print(x + y)"
+        "@tf.function\n",
+        "def increment(x):\n",
+        "  return x + 1\n",
+        "\n",
+        "rt = tf.ragged.constant([[1, 2], [3], [4, 5, 6]])\n",
+        "cf = increment.get_concrete_function(rt)\n",
+        "print(cf(rt))\n"
       ]
     },
     {
-      "cell_type": "code",
-      "execution_count": 0,
+      "cell_type": "markdown",
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "rEj5QVfnva0t"
+        "id": "iYLyPlatIXhh"
       },
-      "outputs": [],
       "source": [
-        "# x      (3d ragged):  2 x (r1) x (r2) x 1\n",
-        "# y      (1d tensor):                    3\n",
-        "# Result (3d ragged):  2 x (r1) x (r2) x 3\n",
-        "x = tf.ragged.constant(\n",
-        "    [\n",
-        "        [\n",
-        "            [[1], [2]],\n",
-        "            [],\n",
-        "            [[3]],\n",
-        "            [[4]],\n",
-        "        ],\n",
-        "        [\n",
-        "            [[5], [6]],\n",
-        "            [[7]]\n",
-        "        ]\n",
-        "    ],\n",
-        "    ragged_rank=2)\n",
-        "y = tf.constant([10, 20, 30])\n",
-        "print(x + y)"
+        "### SavedModels\n",
+        "\n",
+        "A [SavedModel](https://www.tensorflow.org/guide/saved_model) is a serialized TensorFlow program, including both weights and computation.  It can be built from a Keras model or from a custom model.  In either case, ragged tensors can be used transparently with the functions and methods defined by a SavedModel.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "uennZ64Aqftb"
+        "id": "98VpBSdOgWqL"
       },
       "source": [
-        "Here are some examples of shapes that do not broadcast:"
+        "#### Example: saving a Keras model"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "UpI0FlfL4Eim"
+        "id": "D-Dg9w7Je5pU"
       },
       "outputs": [],
       "source": [
-        "# x      (2d ragged): 3 x (r1)\n",
-        "# y      (2d tensor): 3 x    4  # trailing dimensions do not match\n",
-        "x = tf.ragged.constant([[1, 2], [3, 4, 5, 6], [7]])\n",
-        "y = tf.constant([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])\n",
+        "import tempfile\n",
+        "\n",
+        "keras_module_path = tempfile.mkdtemp()\n",
+        "keras_model.save(keras_module_path+\"/my_model.keras\")\n",
+        "\n",
+        "imported_model = tf.keras.models.load_model(keras_module_path+\"/my_model.keras\")\n",
+        "\n",
+        "imported_model(hashed_words.to_tensor())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9-7k-E92gaoR"
+      },
+      "source": [
+        "#### Example: saving a custom model\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Sfem1ESrdGzX"
+      },
+      "outputs": [],
+      "source": [
+        "class CustomModule(tf.Module):\n",
+        "  def __init__(self, variable_value):\n",
+        "    super(CustomModule, self).__init__()\n",
+        "    self.v = tf.Variable(variable_value)\n",
+        "\n",
+        "  @tf.function\n",
+        "  def grow(self, x):\n",
+        "    return x * self.v\n",
+        "\n",
+        "module = CustomModule(100.0)\n",
+        "\n",
+        "# Before saving a custom model, you must ensure that concrete functions are\n",
+        "# built for each input signature that you will need.\n",
+        "module.grow.get_concrete_function(tf.RaggedTensorSpec(shape=[None, None],\n",
+        "                                                      dtype=tf.float32))\n",
+        "\n",
+        "custom_module_path = tempfile.mkdtemp()\n",
+        "tf.saved_model.save(module, custom_module_path)\n",
+        "imported_model = tf.saved_model.load(custom_module_path)\n",
+        "imported_model.grow(tf.ragged.constant([[1.0, 4.0, 3.0], [2.0]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SAxis5KBhrBN"
+      },
+      "source": [
+        "Note: SavedModel [signatures](https://www.tensorflow.org/guide/saved_model#specifying_signatures_during_export) are concrete functions.  As discussed in the section on Concrete Functions above, ragged tensors are only handled correctly by concrete functions starting with TensorFlow 2.3. If you need to use SavedModel signatures in a previous version of TensorFlow, then it's recommended that you decompose the ragged tensor into its component tensors."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cRcHzS6pcHYC"
+      },
+      "source": [
+        "## Overloaded operators\n",
+        "\n",
+        "The `RaggedTensor` class overloads the standard Python arithmetic and comparison operators, making it easy to perform basic elementwise math:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "skScd37P-PVu"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.ragged.constant([[1, 2], [3], [4, 5, 6]])\n",
+        "y = tf.ragged.constant([[1, 1], [2], [3, 3, 3]])\n",
+        "print(x + y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XEGgbZHV-PVw"
+      },
+      "source": [
+        "Since the overloaded operators perform elementwise computations, the inputs to all binary operations must have the same shape or be broadcastable to the same shape. In the simplest broadcasting case, a single scalar is combined elementwise with each value in a ragged tensor:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IYybEEWc-PVx"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.ragged.constant([[1, 2], [3], [4, 5, 6]])\n",
+        "print(x + 3)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "okGb9dIi-PVz"
+      },
+      "source": [
+        "For a discussion of more advanced cases, check the section on **Broadcasting**.\n",
+        "\n",
+        "Ragged tensors overload the same set of operators as normal `Tensor`s: the unary operators `-`, `~`, and `abs()`; and the binary operators `+`, `-`, `*`, `/`, `//`, `%`, `**`, `&`, `|`, `^`, `==`, `<`, `<=`, `>`, and `>=`.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f2anbs6ZnFtl"
+      },
+      "source": [
+        "## Indexing\n",
+        "\n",
+        "Ragged tensors support Python-style indexing, including multidimensional indexing and slicing. The following examples demonstrate ragged tensor indexing with a 2D and a 3D ragged tensor."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XuEwmC3t_ITL"
+      },
+      "source": [
+        "### Indexing examples: 2D ragged tensor"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MbSRZRDz-PV1"
+      },
+      "outputs": [],
+      "source": [
+        "queries = tf.ragged.constant(\n",
+        "    [['Who', 'is', 'George', 'Washington'],\n",
+        "     ['What', 'is', 'the', 'weather', 'tomorrow'],\n",
+        "     ['Goodnight']])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2HRs2xhh-vZE"
+      },
+      "outputs": [],
+      "source": [
+        "print(queries[1])                   # A single query"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EFfjZV7YA3UH"
+      },
+      "outputs": [],
+      "source": [
+        "print(queries[1, 2])                # A single word"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VISRPQSdA3xn"
+      },
+      "outputs": [],
+      "source": [
+        "print(queries[1:])                  # Everything but the first row"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "J1PpSyKQBMng"
+      },
+      "outputs": [],
+      "source": [
+        "print(queries[:, :3])               # The first 3 words of each query"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ixrhHmJBeidy"
+      },
+      "outputs": [],
+      "source": [
+        "print(queries[:, -2:])              # The last 2 words of each query"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cnOP6Vza-PV4"
+      },
+      "source": [
+        "### Indexing examples: 3D ragged tensor"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8VbqbKcE-PV6"
+      },
+      "outputs": [],
+      "source": [
+        "rt = tf.ragged.constant([[[1, 2, 3], [4]],\n",
+        "                         [[5], [], [6]],\n",
+        "                         [[7]],\n",
+        "                         [[8, 9], [10]]])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f9WPVWf4grVp"
+      },
+      "outputs": [],
+      "source": [
+        "print(rt[1])                        # Second row (2D RaggedTensor)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ad8FGJoABjQH"
+      },
+      "outputs": [],
+      "source": [
+        "print(rt[3, 0])                     # First element of fourth row (1D Tensor)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MPPr-a-bBjFE"
+      },
+      "outputs": [],
+      "source": [
+        "print(rt[:, 1:3])                   # Items 1-3 of each row (3D RaggedTensor)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6SIDeoIUBi4z"
+      },
+      "outputs": [],
+      "source": [
+        "print(rt[:, -1:])                   # Last item of each row (3D RaggedTensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_d3nBh1GnWvU"
+      },
+      "source": [
+        "`RaggedTensor`s support multidimensional indexing and slicing with one restriction: indexing into a ragged dimension is not allowed. This case is problematic because the indicated value may exist in some rows but not others. In such cases, it's not obvious whether you should (1) raise an `IndexError`; (2) use a default value; or (3) skip that value and return a tensor with fewer rows than you started with. Following the [guiding principles of Python](https://www.python.org/dev/peps/pep-0020/) (\"In the face of ambiguity, refuse the temptation to guess\"), this operation is currently disallowed."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IsWKETULAJbN"
+      },
+      "source": [
+        "## Tensor type conversion\n",
+        "\n",
+        "The `RaggedTensor` class defines methods that can be used to convert\n",
+        "between `RaggedTensor`s and `tf.Tensor`s or `tf.SparseTensors`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "INnfmZGcBoU_"
+      },
+      "outputs": [],
+      "source": [
+        "ragged_sentences = tf.ragged.constant([\n",
+        "    ['Hi'], ['Welcome', 'to', 'the', 'fair'], ['Have', 'fun']])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "__iJ4iXtkGOx"
+      },
+      "outputs": [],
+      "source": [
+        "# RaggedTensor -> Tensor\n",
+        "print(ragged_sentences.to_tensor(default_value='', shape=[None, 10]))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-rfiyYqne8QN"
+      },
+      "outputs": [],
+      "source": [
+        "# Tensor -> RaggedTensor\n",
+        "x = [[1, 3, -1, -1], [2, -1, -1, -1], [4, 5, 8, 9]]\n",
+        "print(tf.RaggedTensor.from_tensor(x, padding=-1))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "41WAZLXNnbwH"
+      },
+      "outputs": [],
+      "source": [
+        "#RaggedTensor -> SparseTensor\n",
+        "print(ragged_sentences.to_sparse())"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S8MkYo2hfVhj"
+      },
+      "outputs": [],
+      "source": [
+        "# SparseTensor -> RaggedTensor\n",
+        "st = tf.SparseTensor(indices=[[0, 0], [2, 0], [2, 1]],\n",
+        "                     values=['a', 'b', 'c'],\n",
+        "                     dense_shape=[3, 3])\n",
+        "print(tf.RaggedTensor.from_sparse(st))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qx025sNMkAHH"
+      },
+      "source": [
+        "## Evaluating ragged tensors\n",
+        "\n",
+        "To access the values in a ragged tensor, you can:\n",
+        "\n",
+        "1.  Use `tf.RaggedTensor.to_list` to convert the ragged tensor to a nested Python list.\n",
+        "2.  Use `tf.RaggedTensor.numpy` to convert the ragged tensor to a NumPy array whose values are nested NumPy arrays.\n",
+        "3.  Decompose the ragged tensor into its components, using the `tf.RaggedTensor.values` and `tf.RaggedTensor.row_splits` properties, or row-partitioning methods such as `tf.RaggedTensor.row_lengths` and `tf.RaggedTensor.value_rowids`.\n",
+        "4.  Use Python indexing to select values from the ragged tensor.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uMm1WMkc-PV_"
+      },
+      "outputs": [],
+      "source": [
+        "rt = tf.ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]])\n",
+        "print(\"Python list:\", rt.to_list())\n",
+        "print(\"NumPy array:\", rt.numpy())\n",
+        "print(\"Values:\", rt.values.numpy())\n",
+        "print(\"Splits:\", rt.row_splits.numpy())\n",
+        "print(\"Indexed value:\", rt[1].numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J87jMZa0M_YW"
+      },
+      "source": [
+        "## Ragged Shapes\n",
+        "\n",
+        "The shape of a tensor specifies the size of each axis.  For example, the shape of `[[1, 2], [3, 4], [5, 6]]` is `[3, 2]`, since there are 3 rows and 2 columns.  TensorFlow has two separate but related ways to describe shapes:\n",
+        "\n",
+        "* ***static shape***: Information about axis sizes that is known statically (e.g., while tracing a `tf.function`).  May be partially specified.\n",
+        "\n",
+        "* ***dynamic shape***: Runtime information about the axis sizes."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IOETE_OLPLZo"
+      },
+      "source": [
+        "### Static shape\n",
+        "\n",
+        "A Tensor's static shape contains information about its axis sizes that is known at graph-construction time.  For both `tf.Tensor` and `tf.RaggedTensor`, it is available using the `.shape` property, and is encoded using `tf.TensorShape`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "btGDjT4uNgQy"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant([[1, 2], [3, 4], [5, 6]])\n",
+        "x.shape  # shape of a tf.tensor"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "__OgvmrGPEjq"
+      },
+      "outputs": [],
+      "source": [
+        "rt = tf.ragged.constant([[1], [2, 3], [], [4]])\n",
+        "rt.shape  # shape of a tf.RaggedTensor"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9EWnQd3qPWaw"
+      },
+      "source": [
+        "The static shape of a ragged dimension is always `None` (i.e., unspecified).  However, the inverse is not true -- if a `TensorShape` dimension is `None`, then that could indicate that the dimension is ragged, *or* it could indicate that the dimension is uniform but that its size is not statically known."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "75E9YXYMNfne"
+      },
+      "source": [
+        "### Dynamic shape\n",
+        "\n",
+        "A tensor's dynamic shape contains information about its axis sizes that is known when the graph is run.  It is constructed using the `tf.shape` operation.  For `tf.Tensor`, `tf.shape` returns the shape as a 1D integer `Tensor`, where `tf.shape(x)[i]` is the size of axis `i`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kWJ7Cn1EQTD_"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant([['a', 'b'], ['c', 'd'], ['e', 'f']])\n",
+        "tf.shape(x)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BeZEfxwmRcSv"
+      },
+      "source": [
+        "However, a 1D `Tensor` is not expressive enough to describe the shape of a `tf.RaggedTensor`.  Instead, the dynamic shape for ragged tensors is encoded using a dedicated type, `tf.experimental.DynamicRaggedShape`.  In the following example, the `DynamicRaggedShape` returned by `tf.shape(rt)` indicates that the ragged tensor has 4 rows, with lengths 1, 3, 0, and 2:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nZc2wqgQQUFU"
+      },
+      "outputs": [],
+      "source": [
+        "rt = tf.ragged.constant([[1], [2, 3, 4], [], [5, 6]])\n",
+        "rt_shape = tf.shape(rt)\n",
+        "print(rt_shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EphU60YvTf98"
+      },
+      "source": [
+        "#### Dynamic shape: operations\n",
+        "\n",
+        "`DynamicRaggedShape`s can be used with most TensorFlow ops that expect shapes, including `tf.reshape`, `tf.zeros`, `tf.ones`. `tf.fill`, `tf.broadcast_dynamic_shape`, and `tf.broadcast_to`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pclAODLXT6Gr"
+      },
+      "outputs": [],
+      "source": [
+        "print(f\"tf.reshape(x, rt_shape) = {tf.reshape(x, rt_shape)}\")\n",
+        "print(f\"tf.zeros(rt_shape) = {tf.zeros(rt_shape)}\")\n",
+        "print(f\"tf.ones(rt_shape) = {tf.ones(rt_shape)}\")\n",
+        "print(f\"tf.fill(rt_shape, 9) = {tf.fill(rt_shape, 'x')}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rNP_3_btRAHj"
+      },
+      "source": [
+        "#### Dynamic shape: indexing and slicing\n",
+        "\n",
+        "`DynamicRaggedShape` can be also be indexed to get the sizes of uniform dimensions.  For example, we can find the number of rows in a raggedtensor using `tf.shape(rt)[0]` (just as we would for a non-ragged tensor):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MzQvPhsxS6HN"
+      },
+      "outputs": [],
+      "source": [
+        "rt_shape[0]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wvr2iT6zS_e8"
+      },
+      "source": [
+        "However, it is an error to use indexing to try to retrieve the size of a ragged dimension, since it doesn't have a single size.  (Since `RaggedTensor` keeps track of which axes are ragged, this error is only thrown during eager execution or when tracing a `tf.function`; it will never be thrown when executing a concrete function.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HgGMk0LeTGik"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  rt_shape[1]\n",
+        "except ValueError as e:\n",
+        "  print(\"Got expected ValueError:\", e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5QUsdawGU0SM"
+      },
+      "source": [
+        "`DynamicRaggedShape`s can also be sliced, as long as the slice either begins with axis `0`, or contains only dense dimensions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "APT72EaBU70t"
+      },
+      "outputs": [],
+      "source": [
+        "rt_shape[:1]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a-Wl9IrQXcdY"
+      },
+      "source": [
+        "#### Dynamic shape: encoding\n",
+        "\n",
+        "`DynamicRaggedShape` is encoded using two fields:\n",
+        "\n",
+        "* `inner_shape`: An integer vector giving the shape of a dense `tf.Tensor`.\n",
+        "* `row_partitions`: A list of `tf.experimental.RowPartition` objects, describing how the outermost dimension of that inner shape should be partitioned to add ragged axes.\n",
+        "\n",
+        "For more information about row partitions, see the \"RaggedTensor encoding\" section below, and the API docs for `tf.experimental.RowPartition`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jfeY9tTcV_zL"
+      },
+      "source": [
+        "#### Dynamic shape: construction\n",
+        "\n",
+        "`DynamicRaggedShape` is most often constructed by applying `tf.shape` to a `RaggedTensor`, but it can also be constructed directly:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NSRgD667WwIZ"
+      },
+      "outputs": [],
+      "source": [
+        "tf.experimental.DynamicRaggedShape(\n",
+        "    row_partitions=[tf.experimental.RowPartition.from_row_lengths([5, 3, 2])],\n",
+        "    inner_shape=[10, 8])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EjzVjs9MXIIA"
+      },
+      "source": [
+        "If the lengths of all rows are known statically, `DynamicRaggedShape.from_lengths` can also be used to construct a dynamic ragged shape.  (This is mostly useful for testing and demonstration code, since it's rare for the lengths of ragged dimensions to be known statically).\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gMxCzADUYIjY"
+      },
+      "outputs": [],
+      "source": [
+        "tf.experimental.DynamicRaggedShape.from_lengths([4, (2, 1, 0, 8), 12])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EdljbNPq-PWS"
+      },
+      "source": [
+        "### Broadcasting\n",
+        "\n",
+        "Broadcasting is the process of making tensors with different shapes have compatible shapes for elementwise operations. For more background on broadcasting, refer to:\n",
+        "\n",
+        "- [NumPy: Broadcasting](https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)\n",
+        "- `tf.broadcast_dynamic_shape`\n",
+        "- `tf.broadcast_to`\n",
+        "\n",
+        "The basic steps for broadcasting two inputs `x` and `y` to have compatible shapes are:\n",
+        "\n",
+        "1.  If `x` and `y` do not have the same number of dimensions, then add outer dimensions (with size 1) until they do.\n",
+        "\n",
+        "2.  For each dimension where `x` and `y` have different sizes:\n",
+        "\n",
+        "- If `x` or `y` have size `1` in dimension `d`, then repeat its values across dimension `d` to match the other input's size.\n",
+        "- Otherwise, raise an exception (`x` and `y` are not broadcast compatible).\n",
+        "\n",
+        "Where the size of a tensor in a uniform dimension is a single number (the size of slices across that dimension); and the size of a tensor in a ragged dimension is a list of slice lengths (for all slices across that dimension)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-S2hOUWx-PWU"
+      },
+      "source": [
+        "#### Broadcasting examples"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0n095XdR-PWU"
+      },
+      "outputs": [],
+      "source": [
+        "# x       (2D ragged):  2 x (num_rows)\n",
+        "# y       (scalar)\n",
+        "# result  (2D ragged):  2 x (num_rows)\n",
+        "x = tf.ragged.constant([[1, 2], [3]])\n",
+        "y = 3\n",
+        "print(x + y)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0SVYk5AP-PWW"
+      },
+      "outputs": [],
+      "source": [
+        "# x         (2d ragged):  3 x (num_rows)\n",
+        "# y         (2d tensor):  3 x          1\n",
+        "# Result    (2d ragged):  3 x (num_rows)\n",
+        "x = tf.ragged.constant(\n",
+        "   [[10, 87, 12],\n",
+        "    [19, 53],\n",
+        "    [12, 32]])\n",
+        "y = [[1000], [2000], [3000]]\n",
+        "print(x + y)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MsfBMD80s8Ux"
+      },
+      "outputs": [],
+      "source": [
+        "# x      (3d ragged):  2 x (r1) x 2\n",
+        "# y      (2d ragged):         1 x 1\n",
+        "# Result (3d ragged):  2 x (r1) x 2\n",
+        "x = tf.ragged.constant(\n",
+        "    [[[1, 2], [3, 4], [5, 6]],\n",
+        "     [[7, 8]]],\n",
+        "    ragged_rank=1)\n",
+        "y = tf.constant([[10]])\n",
+        "print(x + y)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rEj5QVfnva0t"
+      },
+      "outputs": [],
+      "source": [
+        "# x      (3d ragged):  2 x (r1) x (r2) x 1\n",
+        "# y      (1d tensor):                    3\n",
+        "# Result (3d ragged):  2 x (r1) x (r2) x 3\n",
+        "x = tf.ragged.constant(\n",
+        "    [\n",
+        "        [\n",
+        "            [[1], [2]],\n",
+        "            [],\n",
+        "            [[3]],\n",
+        "            [[4]],\n",
+        "        ],\n",
+        "        [\n",
+        "            [[5], [6]],\n",
+        "            [[7]]\n",
+        "        ]\n",
+        "    ],\n",
+        "    ragged_rank=2)\n",
+        "y = tf.constant([10, 20, 30])\n",
+        "print(x + y)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uennZ64Aqftb"
+      },
+      "source": [
+        "Here are some examples of shapes that do not broadcast:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UpI0FlfL4Eim"
+      },
+      "outputs": [],
+      "source": [
+        "# x      (2d ragged): 3 x (r1)\n",
+        "# y      (2d tensor): 3 x    4  # trailing dimensions do not match\n",
+        "x = tf.ragged.constant([[1, 2], [3, 4, 5, 6], [7]])\n",
+        "y = tf.constant([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])\n",
         "try:\n",
         "  x + y\n",
         "except tf.errors.InvalidArgumentError as exception:\n",
@@ -1319,10 +1914,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "qGq1zOT4zMoc"
       },
       "outputs": [],
@@ -1339,10 +1932,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "CvLae5vMqeji"
       },
       "outputs": [],
@@ -1362,31 +1953,34 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "m0wQkLfV-PWa"
       },
       "source": [
         "## RaggedTensor encoding\n",
         "\n",
-        "Ragged tensors are encoded using the `RaggedTensor` class. Internally, each\n",
-        "`RaggedTensor` consists of:\n",
+        "Ragged tensors are encoded using the `RaggedTensor` class. Internally, each `RaggedTensor` consists of:\n",
+        "\n",
+        "- A `values` tensor, which concatenates the variable-length rows into a flattened list.\n",
+        "- A `row_partition`, which indicates how those flattened values are divided into rows.\n",
+        "\n",
+        "![RaggedTensor encoding](https://www.tensorflow.org/images/ragged_tensors/ragged_encoding_2.png)\n",
+        "\n",
+        "The `row_partition` can be stored using four different encodings:\n",
         "\n",
-        "*   A `values` tensor, which concatenates the variable-length rows into a\n",
-        "    flattened list.\n",
-        "*   A `row_splits` vector, which indicates how those flattened values are\n",
-        "    divided into rows. In particular, the values for row `rt[i]` are stored in\n",
-        "    the slice `rt.values[rt.row_splits[i]:rt.row_splits[i+1]]`.\n",
+        "- `row_splits` is an integer vector specifying the split points between rows.\n",
+        "- `value_rowids` is an integer vector specifying the row index for each value.\n",
+        "- `row_lengths` is an integer vector specifying the length of each row.\n",
+        "- `uniform_row_length` is an integer scalar specifying a single length for all rows.\n",
         "\n",
-        "![ragged_encoding](https://www.tensorflow.org/images/ragged_tensors/ragged_encoding.png)\n",
-        "\n"
+        "![row_partition encodings](https://www.tensorflow.org/images/ragged_tensors/partition_encodings.png)\n",
+        "\n",
+        "An integer scalar `nrows` can also be included in the `row_partition` encoding to account for empty trailing rows with `value_rowids` or empty rows with `uniform_row_length`.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "MrLgMu0gPuo-"
       },
       "outputs": [],
@@ -1400,25 +1994,35 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "wEfZOKwN1Ra_"
+      },
+      "source": [
+        "The choice of which encoding to use for row partitions is managed internally by ragged tensors to improve efficiency in some contexts. In particular, some of the advantages and disadvantages of the different row-partitioning schemes are:\n",
+        "\n",
+        "- **Efficient indexing**: The `row_splits` encoding enables constant-time indexing and slicing into ragged tensors.\n",
+        "- **Efficient concatenation**: The `row_lengths` encoding is more efficient when concatenating ragged tensors, since row lengths do not change when two tensors are concatenated together.\n",
+        "- **Small encoding size**: The `value_rowids` encoding is more efficient when storing ragged tensors that have a large number of empty rows, since the size of the tensor depends only on the total number of values. On the other hand, the `row_splits` and `row_lengths` encodings are more efficient when storing ragged tensors with longer rows, since they require only one scalar value for each row.\n",
+        "- **Compatibility**: The `value_rowids` scheme matches the [segmentation](https://www.tensorflow.org/api_docs/python/tf/math#about_segmentation) format used by operations, such as `tf.segment_sum`. The `row_limits` scheme matches the format used by ops such as `tf.sequence_mask`.\n",
+        "- **Uniform dimensions**: As discussed below, the `uniform_row_length` encoding is used to encode ragged tensors with uniform dimensions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "bpB7xKoUPtU6"
       },
       "source": [
         "### Multiple ragged dimensions\n",
         "\n",
-        "A ragged tensor with multiple ragged dimensions is encoded by using a nested\n",
-        "`RaggedTensor` for the `values` tensor. Each nested `RaggedTensor` adds a single\n",
-        "ragged dimension.\n",
+        "A ragged tensor with multiple ragged dimensions is encoded by using a nested `RaggedTensor` for the `values` tensor. Each nested `RaggedTensor` adds a single ragged dimension.\n",
         "\n",
-        "![ragged_rank_2](https://www.tensorflow.org/images/ragged_tensors/ragged_rank_2.png)"
+        "![Encoding of a ragged tensor with multiple ragged dimensions (rank 2)](https://www.tensorflow.org/images/ragged_tensors/ragged_rank_2.png)\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "yy3IGT2a-PWb"
       },
       "outputs": [],
@@ -1430,27 +2034,22 @@
         "    row_splits=[0, 1, 1, 5])\n",
         "print(rt)\n",
         "print(\"Shape: {}\".format(rt.shape))\n",
-        "print(\"Number of ragged dimensions: {}\".format(rt.ragged_rank))"
+        "print(\"Number of partitioned dimensions: {}\".format(rt.ragged_rank))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5HqEEDzk-PWc"
       },
       "source": [
-        "The factory function `tf.RaggedTensor.from_nested_row_splits` may be used to construct a\n",
-        "RaggedTensor with multiple ragged dimensions directly, by providing a list of\n",
-        "`row_splits` tensors:"
+        "The factory function `tf.RaggedTensor.from_nested_row_splits` may be used to construct a RaggedTensor with multiple ragged dimensions directly by providing a list of `row_splits` tensors:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "AKYhtFcT-PWd"
       },
       "outputs": [],
@@ -1464,197 +2063,123 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "uba2EnAY-PWf"
+        "id": "BqAfbkAC56m0"
       },
       "source": [
-        "### Uniform Inner Dimensions\n",
-        "\n",
-        "Ragged tensors with uniform inner dimensions are encoded by using a\n",
-        "multidimensional `tf.Tensor` for `values`.\n",
+        "### Ragged rank and flat values\n",
         "\n",
-        "![uniform_inner](https://www.tensorflow.org/images/ragged_tensors/uniform_inner.png)"
+        "A ragged tensor's ***ragged rank*** is the number of times that the underlying `values` tensor has been partitioned (i.e. the nesting depth of  `RaggedTensor` objects). The innermost `values` tensor is known as its ***flat_values***.  In the following example, `conversations` has ragged_rank=3, and its `flat_values` is a 1D `Tensor` with 24 strings:\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "z2sHwHdy-PWg"
+        "id": "BXp-Tt2bClem"
       },
       "outputs": [],
       "source": [
-        "rt = tf.RaggedTensor.from_row_splits(\n",
-        "    values=[[1, 3], [0, 0], [1, 3], [5, 3], [3, 3], [1, 2]],\n",
-        "    row_splits=[0, 3, 4, 6])\n",
-        "print(rt)\n",
-        "print(\"Shape: {}\".format(rt.shape))\n",
-        "print(\"Number of ragged dimensions: {}\".format(rt.ragged_rank))"
+        "# shape = [batch, (paragraph), (sentence), (word)]\n",
+        "conversations = tf.ragged.constant(\n",
+        "    [[[[\"I\", \"like\", \"ragged\", \"tensors.\"]],\n",
+        "      [[\"Oh\", \"yeah?\"], [\"What\", \"can\", \"you\", \"use\", \"them\", \"for?\"]],\n",
+        "      [[\"Processing\", \"variable\", \"length\", \"data!\"]]],\n",
+        "     [[[\"I\", \"like\", \"cheese.\"], [\"Do\", \"you?\"]],\n",
+        "      [[\"Yes.\"], [\"I\", \"do.\"]]]])\n",
+        "conversations.shape"
       ]
     },
     {
-      "cell_type": "markdown",
+      "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "8yYaNrgX-PWh"
+        "id": "DZUMrgxXFd5s"
       },
+      "outputs": [],
       "source": [
-        "### Alternative row-partitioning schemes\n",
-        "\n",
-        "The `RaggedTensor` class uses `row_splits` as the primary mechanism to store\n",
-        "information about how the values are partitioned into rows. However,\n",
-        "`RaggedTensor` also provides support for four alternative row-partitioning\n",
-        "schemes, which can be more convenient to use depending on how your data is\n",
-        "formatted. Internally, `RaggedTensor` uses these additional schemes to improve\n",
-        "efficiency in some contexts.\n",
-        "\n",
-        "\u003cdl\u003e\n",
-        "  \u003cdt\u003eRow lengths\u003c/dt\u003e\n",
-        "    \u003cdd\u003e`row_lengths` is a vector with shape `[nrows]`, which specifies the\n",
-        "    length of each row.\u003c/dd\u003e\n",
-        "\n",
-        "  \u003cdt\u003eRow starts\u003c/dt\u003e\n",
-        "    \u003cdd\u003e`row_starts` is a vector with shape `[nrows]`, which specifies the start\n",
-        "    offset of each row. Equivalent to `row_splits[:-1]`.\u003c/dd\u003e\n",
-        "\n",
-        "  \u003cdt\u003eRow limits\u003c/dt\u003e\n",
-        "    \u003cdd\u003e`row_limits` is a vector with shape `[nrows]`, which specifies the stop\n",
-        "    offset of each row. Equivalent to `row_splits[1:]`.\u003c/dd\u003e\n",
-        "\n",
-        "  \u003cdt\u003eRow indices and number of rows\u003c/dt\u003e\n",
-        "    \u003cdd\u003e`value_rowids` is a vector with shape `[nvals]`, corresponding\n",
-        "    one-to-one with values, which specifies each value's row index. In\n",
-        "    particular, the row `rt[row]` consists of the values `rt.values[j]` where\n",
-        "    `value_rowids[j]==row`. \\\n",
-        "    `nrows` is an integer that specifies the number of rows in the\n",
-        "    `RaggedTensor`. In particular, `nrows` is used to indicate trailing empty\n",
-        "    rows.\u003c/dd\u003e\n",
-        "\u003c/dl\u003e\n",
-        "\n",
-        "For example, the following ragged tensors are equivalent:"
+        "assert conversations.ragged_rank == len(conversations.nested_row_splits)\n",
+        "conversations.ragged_rank  # Number of partitioned dimensions."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "4TH6XoQ8-PWh"
+        "id": "xXLSNpS0Fdvp"
       },
       "outputs": [],
       "source": [
-        "values = [3, 1, 4, 1, 5, 9, 2, 6]\n",
-        "print(tf.RaggedTensor.from_row_splits(values, row_splits=[0, 4, 4, 7, 8, 8]))\n",
-        "print(tf.RaggedTensor.from_row_lengths(values, row_lengths=[4, 0, 3, 1, 0]))\n",
-        "print(tf.RaggedTensor.from_row_starts(values, row_starts=[0, 4, 4, 7, 8]))\n",
-        "print(tf.RaggedTensor.from_row_limits(values, row_limits=[4, 4, 7, 8, 8]))\n",
-        "print(tf.RaggedTensor.from_value_rowids(\n",
-        "    values, value_rowids=[0, 0, 0, 0, 2, 2, 2, 3], nrows=5))"
+        "conversations.flat_values.numpy()"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "ZGRrpwxjsOGr"
+        "id": "uba2EnAY-PWf"
       },
       "source": [
-        "The RaggedTensor class defines methods which can be used to construct\n",
-        "each of these row-partitioning tensors."
+        "### Uniform inner dimensions\n",
+        "\n",
+        "Ragged tensors with uniform inner dimensions are encoded by using a\n",
+        "multidimensional `tf.Tensor` for the flat_values (i.e., the innermost `values`).\n",
+        "\n",
+        "![Encoding of ragged tensors with uniform inner dimensions](https://www.tensorflow.org/images/ragged_tensors/uniform_inner.png)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "fIdn-hUBsoSj"
+        "id": "z2sHwHdy-PWg"
       },
       "outputs": [],
       "source": [
-        "rt = tf.ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])\n",
-        "print(\"      values: {}\".format(rt.values))\n",
-        "print(\"  row_splits: {}\".format(rt.row_splits))\n",
-        "print(\" row_lengths: {}\".format(rt.row_lengths()))\n",
-        "print(\"  row_starts: {}\".format(rt.row_starts()))\n",
-        "print(\"  row_limits: {}\".format(rt.row_limits()))\n",
-        "print(\"value_rowids: {}\".format(rt.value_rowids()))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2r9XUpLUsdOa"
-      },
-      "source": [
-        "(Note that `tf.RaggedTensor.values` and `tf.RaggedTensors.row_splits` are properties, while the remaining row-partitioning accessors are all methods.  This reflects the fact that the `row_splits` are the primary underlying representation, and the other row-partitioning tensors must be computed.)"
+        "rt = tf.RaggedTensor.from_row_splits(\n",
+        "    values=[[1, 3], [0, 0], [1, 3], [5, 3], [3, 3], [1, 2]],\n",
+        "    row_splits=[0, 3, 4, 6])\n",
+        "print(rt)\n",
+        "print(\"Shape: {}\".format(rt.shape))\n",
+        "print(\"Number of partitioned dimensions: {}\".format(rt.ragged_rank))\n",
+        "print(\"Flat values shape: {}\".format(rt.flat_values.shape))\n",
+        "print(\"Flat values:\\n{}\".format(rt.flat_values))"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "NBX15kEr-PWi"
+        "id": "WoGRKd50x_qz"
       },
       "source": [
-        "Some of the advantages and disadvantages of the different row-partitioning\n",
-        "schemes are:\n",
-        "\n",
-        "+ **Efficient indexing**:\n",
-        "    The `row_splits`, `row_starts`, and `row_limits` schemes all enable\n",
-        "    constant-time indexing into ragged tensors. The `value_rowids` and\n",
-        "     `row_lengths` schemes do not.\n",
+        "### Uniform non-inner dimensions\n",
         "\n",
-        "+ **Small encoding size**:\n",
-        "    The `value_rowids` scheme is more efficient when storing ragged tensors that\n",
-        "    have a large number of empty rows, since the size of the tensor depends only\n",
-        "    on the total number of values. On the other hand, the other four encodings\n",
-        "    are more efficient when storing ragged tensors with longer rows, since they\n",
-        "    require only one scalar value for each row.\n",
+        "Ragged tensors with uniform non-inner dimensions are encoded by partitioning rows with `uniform_row_length`.\n",
         "\n",
-        "+ **Efficient concatenation**:\n",
-        "   The `row_lengths` scheme is more efficient when concatenating ragged\n",
-        "    tensors, since row lengths do not change when two tensors are concatenated\n",
-        "   together (but row splits and row indices do).\n",
-        "\n",
-        "+ **Compatibility**:\n",
-        "    The `value_rowids` scheme matches the\n",
-        "    [segmentation](../api_guides/python/math_ops.md#Segmentation)\n",
-        "    format used by operations such as `tf.segment_sum`. The `row_limits` scheme\n",
-        "    matches the format used by ops such as `tf.sequence_mask`."
+        "![Encoding of ragged tensors with uniform non-inner dimensions](https://www.tensorflow.org/images/ragged_tensors/uniform_outer.png)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "R2lzYuYrNmw6"
+        "id": "70q1aCKwySgS"
       },
       "outputs": [],
       "source": [
-        ""
+        "rt = tf.RaggedTensor.from_uniform_row_length(\n",
+        "    values=tf.RaggedTensor.from_row_splits(\n",
+        "        values=[10, 11, 12, 13, 14, 15, 16, 17, 18, 19],\n",
+        "        row_splits=[0, 3, 5, 9, 10]),\n",
+        "    uniform_row_length=2)\n",
+        "print(rt)\n",
+        "print(\"Shape: {}\".format(rt.shape))\n",
+        "print(\"Number of partitioned dimensions: {}\".format(rt.ragged_rank))"
       ]
     }
   ],
   "metadata": {
     "colab": {
-      "collapsed_sections": [],
-      "last_runtime": {
-        "build_target": "",
-        "kind": "local"
-      },
       "name": "ragged_tensor.ipynb",
-      "private_outputs": true,
-      "provenance": [],
-      "toc_visible": true,
-      "version": "0.3.2"
+      "toc_visible": true
     },
     "kernelspec": {
       "display_name": "Python 3",
diff --git a/site/en/guide/random_numbers.ipynb b/site/en/guide/random_numbers.ipynb
new file mode 100644
index 00000000000..f8b824ad906
--- /dev/null
+++ b/site/en/guide/random_numbers.ipynb
@@ -0,0 +1,887 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qFdPvlXBOdUN"
+      },
+      "source": [
+        "# Random number generation"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BlGY1iiph_C2"
+      },
+      "source": [
+        "TensorFlow provides a set of pseudo-random number generators (RNG), in the `tf.random` module. This document describes how you can control the random number generators, and how these generators interact with other tensorflow sub-systems. \n",
+        "\n",
+        "Note: The random numbers are not guaranteed to be consistent across TensorFlow versions. See: [Version Compatibility](https://www.tensorflow.org/guide/versions#what_is_not_covered)\n",
+        "\n",
+        "TensorFlow provides two approaches for controlling the random number generation process:\n",
+        "\n",
+        "1. Through the explicit use of `tf.random.Generator` objects. Each such object maintains a state (in `tf.Variable`) that will be changed after each number generation.\n",
+        "\n",
+        "2. Through the purely-functional stateless random functions like `tf.random.stateless_uniform`. Calling these functions with the same arguments (which include the seed) and on the same device will always produce the same results.\n",
+        "\n",
+        "Warning: The old RNGs from TF 1.x such as `tf.random.uniform` and `tf.random.normal` are not yet deprecated but strongly discouraged."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zIGh9faCOp6x"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ECDrttf0s8Nu"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "# Creates some virtual devices (cpu:0, cpu:1, etc.) for using distribution strategy\n",
+        "physical_devices = tf.config.list_physical_devices(\"CPU\")\n",
+        "tf.config.experimental.set_virtual_device_configuration(\n",
+        "    physical_devices[0], [\n",
+        "        tf.config.experimental.VirtualDeviceConfiguration(),\n",
+        "        tf.config.experimental.VirtualDeviceConfiguration(),\n",
+        "        tf.config.experimental.VirtualDeviceConfiguration()\n",
+        "    ])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eqMlrUsVu2Ai"
+      },
+      "source": [
+        "## The `tf.random.Generator` class\n",
+        "\n",
+        "The `tf.random.Generator` class is used in cases where you want each RNG call to produce different results. It maintains an internal state (managed by a `tf.Variable` object) which will be updated every time random numbers are generated. Because the state is managed by `tf.Variable`, it enjoys all facilities provided by `tf.Variable` such as easy checkpointing, automatic control-dependency and thread safety.\n",
+        "\n",
+        "You can get a `tf.random.Generator` by manually creating an object of the class or call `tf.random.get_global_generator()` to get the default global generator:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7yU1E3JvxOQD"
+      },
+      "outputs": [],
+      "source": [
+        "g1 = tf.random.Generator.from_seed(1)\n",
+        "print(g1.normal(shape=[2, 3]))\n",
+        "g2 = tf.random.get_global_generator()\n",
+        "print(g2.normal(shape=[2, 3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QmRCeAvTxulW"
+      },
+      "source": [
+        "There are multiple ways to create a generator object. The easiest is `Generator.from_seed`, as shown above, that creates a generator from a seed. A seed is any non-negative integer. `from_seed` also takes an optional argument `alg` which is the RNG algorithm that will be used by this generator:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kISbOE4Xfjhv"
+      },
+      "outputs": [],
+      "source": [
+        "g1 = tf.random.Generator.from_seed(1, alg='philox')\n",
+        "print(g1.normal(shape=[2, 3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_mCRaN7dfd8j"
+      },
+      "source": [
+        "See the *Algorithms* section below for more information about it.\n",
+        "\n",
+        "Another way to create a generator is with `Generator.from_non_deterministic_state`. A generator created this way will start from a non-deterministic state, depending on e.g., time and OS."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gxPLCLsz00qY"
+      },
+      "outputs": [],
+      "source": [
+        "g = tf.random.Generator.from_non_deterministic_state()\n",
+        "print(g.normal(shape=[2, 3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zSAp2BMj1JZ6"
+      },
+      "source": [
+        "There are yet other ways to create generators, such as from explicit states, which are not covered by this guide.\n",
+        "\n",
+        "When using `tf.random.get_global_generator` to get the global generator, you need to be careful about device placement. The global generator is created (from a non-deterministic state) at the first time `tf.random.get_global_generator` is called, and placed on the default device at that call. So, for example, if the first site you call `tf.random.get_global_generator` is within a `tf.device(\"gpu\")` scope, the global generator will be placed on the GPU, and using the global generator later on from the CPU will incur a GPU-to-CPU copy.\n",
+        "\n",
+        "There is also a function `tf.random.set_global_generator` for replacing the global generator with another generator object. This function should be used with caution though, because the old global generator may have been captured by a `tf.function` (as a weak reference), and replacing it will cause it to be garbage collected, breaking the `tf.function`. A better way to reset the global generator is to use one of the \"reset\" functions such as `Generator.reset_from_seed`, which won't create new generator objects."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "324S5bpd9HRg"
+      },
+      "outputs": [],
+      "source": [
+        "g = tf.random.Generator.from_seed(1)\n",
+        "print(g.normal([]))\n",
+        "print(g.normal([]))\n",
+        "g.reset_from_seed(1)\n",
+        "print(g.normal([]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z9H0wuvp9VwH"
+      },
+      "source": [
+        "### Creating independent random-number streams\n",
+        "\n",
+        "In many applications one needs multiple independent random-number streams, independent in the sense that they won't overlap and won't have any statistically detectable correlations. This is achieved by using `Generator.split` to create multiple generators that are guaranteed to be independent of each other (i.e. generating independent streams)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Vg5_KN18OZjo"
+      },
+      "outputs": [],
+      "source": [
+        "g = tf.random.Generator.from_seed(1)\n",
+        "print(g.normal([]))\n",
+        "new_gs = g.split(3)\n",
+        "for new_g in new_gs:\n",
+        "  print(new_g.normal([]))\n",
+        "print(g.normal([]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dqOaGVzKOsRJ"
+      },
+      "source": [
+        "`split` will change the state of the generator on which it is called (`g` in the above example), similar to an RNG method such as `normal`. In addition to being independent of each other, the new generators (`new_gs`) are also guaranteed to be independent of the old one (`g`).\n",
+        "\n",
+        "Spawning new generators is also useful when you want to make sure the generator you use is on the same device as other computations, to avoid the overhead of cross-device copy. For example: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5jSnJBlUQzF3"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.device(\"cpu\"):  # change \"cpu\" to the device you want\n",
+        "  g = tf.random.get_global_generator().split(1)[0]  \n",
+        "  print(g.normal([]))  # use of g won't cause cross-device copy, unlike the global generator"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sCxbccYMRdd4"
+      },
+      "source": [
+        "Note: In theory, you can use constructors such as `from_seed` instead of `split` here to obtain a new generator, but by doing so you lose the guarantee that the new generator is independent of the global generator. You will also run the risk that you may accidentally create two generators with the same seed or with seeds that lead to overlapping random-number streams.\n",
+        "\n",
+        "You can do splitting recursively, calling `split` on split generators. There are no limits (barring integer overflow) on the depth of recursions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8JUgnQM_O0lg"
+      },
+      "source": [
+        "### Interaction with `tf.function`\n",
+        "\n",
+        "`tf.random.Generator` obeys the same rules as `tf.Variable` when used with `tf.function`. This includes three aspects."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jnSjhY6WM-J8"
+      },
+      "source": [
+        "#### Creating generators outside `tf.function` \n",
+        "\n",
+        "`tf.function` can use a generator created outside of it."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a5EEy0E2UHMw"
+      },
+      "outputs": [],
+      "source": [
+        "g = tf.random.Generator.from_seed(1)\n",
+        "@tf.function\n",
+        "def foo():\n",
+        "  return g.normal([])\n",
+        "print(foo())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L_8kC7kbO5uu"
+      },
+      "source": [
+        "The user needs to make sure that the generator object is still alive (not garbage-collected) when the function is called."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PwIrBv_zUYwI"
+      },
+      "source": [
+        "#### Creating generators inside `tf.function` \n",
+        "\n",
+        "Creation of generators inside a `tf.function` can only happened during the first run of the function. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3JzpUvqJU4MW"
+      },
+      "outputs": [],
+      "source": [
+        "g = None\n",
+        "@tf.function\n",
+        "def foo():\n",
+        "  global g\n",
+        "  if g is None:\n",
+        "    g = tf.random.Generator.from_seed(1)\n",
+        "  return g.normal([])\n",
+        "print(foo())\n",
+        "print(foo())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UaTVnOhHVM9a"
+      },
+      "source": [
+        "#### Passing generators as arguments to `tf.function`\n",
+        "\n",
+        "When used as an argument to a `tf.function`, different generator objects will cause retracing of the `tf.function`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DeR9kvt0V-ad"
+      },
+      "outputs": [],
+      "source": [
+        "num_traces = 0\n",
+        "@tf.function\n",
+        "def foo(g):\n",
+        "  global num_traces\n",
+        "  num_traces += 1\n",
+        "  return g.normal([])\n",
+        "foo(tf.random.Generator.from_seed(1))\n",
+        "foo(tf.random.Generator.from_seed(2))\n",
+        "print(num_traces)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "E0RxllJzkGfo"
+      },
+      "source": [
+        "Note that this retracing behavior is consistent with `tf.Variable`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oWD2f_qxkSe7"
+      },
+      "outputs": [],
+      "source": [
+        "num_traces = 0\n",
+        "@tf.function\n",
+        "def foo(v):\n",
+        "  global num_traces\n",
+        "  num_traces += 1\n",
+        "  return v.read_value()\n",
+        "foo(tf.Variable(1))\n",
+        "foo(tf.Variable(2))\n",
+        "print(num_traces)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fxcS6IY8WZuh"
+      },
+      "source": [
+        "### Interaction with distribution strategies\n",
+        "\n",
+        "There are two ways in which `Generator` interacts with distribution strategies."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GyZv9QJkZfkQ"
+      },
+      "source": [
+        "#### Creating generators outside distribution strategies\n",
+        "\n",
+        "If a generator is created outside strategy scopes, all replicas’ access to the generator will be serialized, and hence the replicas will get different random numbers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HX_beT9SZWMp"
+      },
+      "outputs": [],
+      "source": [
+        "g = tf.random.Generator.from_seed(1)\n",
+        "strat = tf.distribute.MirroredStrategy(devices=[\"cpu:0\", \"cpu:1\"])\n",
+        "with strat.scope():\n",
+        "  def f():\n",
+        "    print(g.normal([]))\n",
+        "  results = strat.run(f)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ydYQbUqLPAgH"
+      },
+      "source": [
+        "Note that this usage may have performance issues because the generator's device is different from the replicas."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Yal4LbBKbAeN"
+      },
+      "source": [
+        "#### Creating generators inside distribution strategies\n",
+        "\n",
+        "If a generator is created inside a strategy scope, each replica will get a different and independent stream of random numbers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5SeUu7IFmTyQ"
+      },
+      "outputs": [],
+      "source": [
+        "strat = tf.distribute.MirroredStrategy(devices=[\"cpu:0\", \"cpu:1\"])\n",
+        "with strat.scope():\n",
+        "  g = tf.random.Generator.from_seed(1)\n",
+        "  print(strat.run(lambda: g.normal([])))\n",
+        "  print(strat.run(lambda: g.normal([])))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PFBlrOudfu9u"
+      },
+      "source": [
+        "Note: Currently `tf.random.Generator` doesn't provide an option to let different replicas get identical (instead of different) streams (which is technically not hard). If you have a use case for this feature, please let the TensorFlow developers know.\n",
+        "\n",
+        "If the generator is seeded (e.g. created by `Generator.from_seed`), the random numbers are determined by the seed, even though different replicas get different and uncorrelated numbers. One can think of a random number generated on a replica as a hash of the replica ID and a \"primary\" random number that is common to all replicas. Hence, the whole system is still deterministic.\n",
+        "\n",
+        "`tf.random.Generator` can also be created inside `Strategy.run`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nlQXi5Msb1Wu"
+      },
+      "outputs": [],
+      "source": [
+        "strat = tf.distribute.MirroredStrategy(devices=[\"cpu:0\", \"cpu:1\"])\n",
+        "with strat.scope():\n",
+        "  def f():\n",
+        "    g = tf.random.Generator.from_seed(1)\n",
+        "    a = g.normal([])\n",
+        "    b = g.normal([])\n",
+        "    return tf.stack([a, b])\n",
+        "  print(strat.run(f))\n",
+        "  print(strat.run(f))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4Sv-aiaOmrOr"
+      },
+      "source": [
+        "We no longer recommend passing `tf.random.Generator` as arguments to `Strategy.run`, because `Strategy.run` generally expects the arguments to be tensors, not generators."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8RbM4vabtiWM"
+      },
+      "source": [
+        "### Saving generators\n",
+        "\n",
+        "Generally for saving or serializing you can handle a `tf.random.Generator` the same way you would handle a `tf.Variable` or a `tf.Module` (or its subclasses). In TF there are two mechanisms for serialization: [Checkpoint](https://www.tensorflow.org/guide/checkpoint) and [SavedModel](https://www.tensorflow.org/guide/saved_model)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PDtySQDotWQc"
+      },
+      "source": [
+        "#### Checkpoint\n",
+        "\n",
+        "Generators can be freely saved and restored using `tf.train.Checkpoint`. The random-number stream from the restoring point will be the same as that from the saving point. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uB_bDSbzpbne"
+      },
+      "outputs": [],
+      "source": [
+        "filename = \"./checkpoint\"\n",
+        "g = tf.random.Generator.from_seed(1)\n",
+        "cp = tf.train.Checkpoint(generator=g)\n",
+        "print(g.normal([]))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bKKtRWeIkIjX"
+      },
+      "outputs": [],
+      "source": [
+        "cp.write(filename)\n",
+        "print(\"RNG stream from saving point:\")\n",
+        "print(g.normal([]))\n",
+        "print(g.normal([]))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-cIHcHwRkQp3"
+      },
+      "outputs": [],
+      "source": [
+        "cp.restore(filename)\n",
+        "print(\"RNG stream from restoring point:\")\n",
+        "print(g.normal([]))\n",
+        "print(g.normal([]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A-OeUUQEJ37X"
+      },
+      "source": [
+        "You can also save and restore within a distribution strategy:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3aI6TQ2lq28w"
+      },
+      "outputs": [],
+      "source": [
+        "filename = \"./checkpoint\"\n",
+        "strat = tf.distribute.MirroredStrategy(devices=[\"cpu:0\", \"cpu:1\"])\n",
+        "with strat.scope():\n",
+        "  g = tf.random.Generator.from_seed(1)\n",
+        "  cp = tf.train.Checkpoint(my_generator=g)\n",
+        "  print(strat.run(lambda: g.normal([])))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kTZcdaMwkvJI"
+      },
+      "outputs": [],
+      "source": [
+        "with strat.scope():\n",
+        "  cp.write(filename)\n",
+        "  print(\"RNG stream from saving point:\")\n",
+        "  print(strat.run(lambda: g.normal([])))\n",
+        "  print(strat.run(lambda: g.normal([])))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nizFA5IrkzN1"
+      },
+      "outputs": [],
+      "source": [
+        "with strat.scope():\n",
+        "  cp.restore(filename)\n",
+        "  print(\"RNG stream from restoring point:\")\n",
+        "  print(strat.run(lambda: g.normal([])))\n",
+        "  print(strat.run(lambda: g.normal([])))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z2rsPfp9J6JA"
+      },
+      "source": [
+        "You should make sure that the replicas don't diverge in their RNG call history (e.g. one replica makes one RNG call while another makes two RNG calls) before saving. Otherwise, their internal RNG states will diverge and `tf.train.Checkpoint` (which only saves the first replica's state) won't properly restore all the replicas.\n",
+        "\n",
+        "You can also restore a saved checkpoint to a different distribution strategy with a different number of replicas. Because a `tf.random.Generator` object created in a strategy can only be used in the same strategy, to restore to a different strategy, you have to create a new `tf.random.Generator` in the target strategy and a new `tf.train.Checkpoint` for it, as shown in this example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zgoFRf59-IvW"
+      },
+      "outputs": [],
+      "source": [
+        "filename = \"./checkpoint\"\n",
+        "strat1 = tf.distribute.MirroredStrategy(devices=[\"cpu:0\", \"cpu:1\"])\n",
+        "with strat1.scope():\n",
+        "  g1 = tf.random.Generator.from_seed(1)\n",
+        "  cp1 = tf.train.Checkpoint(my_generator=g1)\n",
+        "  print(strat1.run(lambda: g1.normal([])))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Lu79ETxMlDpO"
+      },
+      "outputs": [],
+      "source": [
+        "with strat1.scope():\n",
+        "  cp1.write(filename)\n",
+        "  print(\"RNG stream from saving point:\")\n",
+        "  print(strat1.run(lambda: g1.normal([])))\n",
+        "  print(strat1.run(lambda: g1.normal([])))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VYoRFUjklKOk"
+      },
+      "outputs": [],
+      "source": [
+        "strat2 = tf.distribute.MirroredStrategy(devices=[\"cpu:0\", \"cpu:1\", \"cpu:2\"])\n",
+        "with strat2.scope():\n",
+        "  g2 = tf.random.Generator.from_seed(1)\n",
+        "  cp2 = tf.train.Checkpoint(my_generator=g2)\n",
+        "  cp2.restore(filename)\n",
+        "  print(\"RNG stream from restoring point:\")\n",
+        "  print(strat2.run(lambda: g2.normal([])))\n",
+        "  print(strat2.run(lambda: g2.normal([])))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kMltUKbANqgl"
+      },
+      "source": [
+        "Although `g1` and `cp1` are different objects from `g2` and `cp2`, they are linked via the common checkpoint file `filename` and object name `my_generator`. Overlapping replicas between strategies (e.g. `cpu:0` and `cpu:1` above) will have their RNG streams properly restored like in previous examples. This guarantee doesn't cover the case when a generator is saved in a strategy scope and restored outside of any strategy scope or vice versa, because a device outside strategies is treated as different from any replica in a strategy."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "w9dqrp1LnTaJ"
+      },
+      "source": [
+        "#### SavedModel\n",
+        "\n",
+        "`tf.random.Generator` can be saved to a SavedModel. The generator can be created within a strategy scope. The saving can also happen within a strategy scope. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0AKO5SnUtyqx"
+      },
+      "outputs": [],
+      "source": [
+        "filename = \"./saved_model\"\n",
+        "\n",
+        "class MyModule(tf.Module):\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    super(MyModule, self).__init__()\n",
+        "    self.g = tf.random.Generator.from_seed(0)\n",
+        "\n",
+        "  @tf.function\n",
+        "  def __call__(self):\n",
+        "    return self.g.normal([])\n",
+        "\n",
+        "  @tf.function\n",
+        "  def state(self):\n",
+        "    return self.g.state\n",
+        "\n",
+        "strat = tf.distribute.MirroredStrategy(devices=[\"cpu:0\", \"cpu:1\"])\n",
+        "with strat.scope():\n",
+        "  m = MyModule()\n",
+        "  print(strat.run(m))\n",
+        "  print(\"state:\", m.state())"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jg2148hulfLB"
+      },
+      "outputs": [],
+      "source": [
+        "with strat.scope():\n",
+        "  tf.saved_model.save(m, filename)\n",
+        "  print(\"RNG stream from saving point:\")\n",
+        "  print(strat.run(m))\n",
+        "  print(\"state:\", m.state())\n",
+        "  print(strat.run(m))\n",
+        "  print(\"state:\", m.state())"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "93AgVyzOllG7"
+      },
+      "outputs": [],
+      "source": [
+        "imported = tf.saved_model.load(filename)\n",
+        "print(\"RNG stream from loading point:\")\n",
+        "print(\"state:\", imported.state())\n",
+        "print(imported())\n",
+        "print(\"state:\", imported.state())\n",
+        "print(imported())\n",
+        "print(\"state:\", imported.state())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sbb23j3pZNNq"
+      },
+      "source": [
+        "Loading a SavedModel containing `tf.random.Generator` into a distribution strategy is not recommended because the replicas will all generate the same random-number stream (which is because replica ID is frozen in SavedModel's graph). \n",
+        "\n",
+        "Loading a distributed `tf.random.Generator` (a generator created within a distribution strategy) into a non-strategy environment, like the above example, also has a caveat. The RNG state will be properly restored, but the random numbers generated will be different from the original generator in its strategy (again because a device outside strategies is treated as different from any replica in a strategy)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "73an1POpsi6V"
+      },
+      "source": [
+        "## Stateless RNGs\n",
+        "\n",
+        "Usage of stateless RNGs is simple. Since they are just pure functions, there is no state or side effect involved."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0-aOOA3gasn_"
+      },
+      "outputs": [],
+      "source": [
+        "print(tf.random.stateless_normal(shape=[2, 3], seed=[1, 2]))\n",
+        "print(tf.random.stateless_normal(shape=[2, 3], seed=[1, 2]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2O_D-RAFNH2Q"
+      },
+      "source": [
+        "Every stateless RNG requires a `seed` argument, which needs to be an integer Tensor of shape `[2]`. The results of the op are fully determined by this seed.\n",
+        "\n",
+        "The RNG algorithm used by stateless RNGs is device-dependent, meaning the same op running on a different device may produce different outputs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4BvGkPnaOUPF"
+      },
+      "source": [
+        "## Algorithms"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "58-8kvR4pRwO"
+      },
+      "source": [
+        "### General\n",
+        "\n",
+        "Both the `tf.random.Generator` class and the `stateless` functions support the Philox algorithm (written as `\"philox\"` or `tf.random.Algorithm.PHILOX`) on all devices.\n",
+        "\n",
+        "Different devices will generate the same integer numbers, if using the same algorithm and starting from the same state. They will also generate \"almost the same\" float-point numbers, though there may be small numerical discrepancies caused by the different ways the devices carry out the float-point computation (e.g. reduction order)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WETA04F1OYPL"
+      },
+      "source": [
+        "### XLA devices\n",
+        "\n",
+        "On XLA-driven devices (such as TPU, and also CPU/GPU when XLA is enabled) the ThreeFry algorithm (written as `\"threefry\"` or `tf.random.Algorithm.THREEFRY`) is also supported. This algorithm is fast on TPU but slow on CPU/GPU compared to Philox. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c04JkebCPTPu"
+      },
+      "source": [
+        "See paper ['Parallel Random Numbers: As Easy as 1, 2, 3'](https://www.thesalmons.org/john/random123/papers/random123sc11.pdf) for more details about these algorithms."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "random_numbers.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/saved_model.ipynb b/site/en/guide/saved_model.ipynb
index 1d040550d85..2601e504669 100644
--- a/site/en/guide/saved_model.ipynb
+++ b/site/en/guide/saved_model.ipynb
@@ -3,7 +3,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6bYaCABobL5q"
       },
       "source": [
@@ -12,11 +11,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab": {},
-        "colab_type": "code",
         "id": "FlUw7tSKbtg4"
       },
       "outputs": [],
@@ -37,7 +34,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "xc1srSc51n_4"
       },
       "source": [
@@ -47,96 +43,105 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-nBUqG2rchGH"
       },
       "source": [
-        "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/saved_model\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/guide/saved_model.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/guide/saved_model.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "  \u003ctd\u003e\n",
-        "    \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/guide/saved_model.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n",
-        "  \u003c/td\u003e\n",
-        "\u003c/table\u003e"
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "CPE-fshLTsXU"
       },
       "source": [
-        "A SavedModel contains a complete TensorFlow program, including weights and computation. It does not require the original model building code to run, which makes it useful for sharing or deploying (with [TFLite](https://tensorflow.org/lite), [TensorFlow.js](https://js.tensorflow.org/), [TensorFlow Serving](https://www.tensorflow.org/tfx/serving/tutorials/Serving_REST_simple), or [TensorFlow Hub](https://tensorflow.org/hub)).\n",
+        "A SavedModel contains a complete TensorFlow program, including trained parameters (i.e, `tf.Variable`s) and computation. It does not require the original model building code to run, which makes it useful for sharing or deploying with [TFLite](https://tensorflow.org/lite), [TensorFlow.js](https://js.tensorflow.org/), [TensorFlow Serving](https://www.tensorflow.org/tfx/serving/tutorials/Serving_REST_simple), or [TensorFlow Hub](https://tensorflow.org/hub).\n",
         "\n",
-        "This document dives into some of the details of how to use the low-level `tf.saved_model` api:\n",
+        "You can save and load a model in the SavedModel format using the following APIs:\n",
+        "- Low-level `tf.saved_model` API. This document describes how to use this API in detail.\n",
+        " - Save: `tf.saved_model.save(model, path_to_dir)`\n",
+        " - Load: `model = tf.saved_model.load(path_to_dir)`\n",
+        "- High-level `tf.keras.Model` API. Refer to [the keras save and serialize guide](https://www.tensorflow.org/guide/keras/save_and_serialize).\n",
+        "- If you just want to save/load weights during training, refer to [the checkpoints guide](./checkpoint.ipynb).\n",
         "\n",
-        "- If you are using a `tf.keras.Model` the `keras.Model.save(output_path)` method may be all you need: See the [Keras save and serialize](keras/save_and_serialize.ipynb)\n",
-        "\n",
-        "- If you just want to save/load weights during training see the [guide to training checkpoints](./checkpoint.ipynb).\n"
+        "Caution: TensorFlow models are code and it is important to be careful with untrusted code. Learn more in [Using TensorFlow securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md).\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "9SuIC7FiI9g8"
       },
       "source": [
-        "## Creating a SavedModel from Keras\n",
-        "\n",
+        "## Creating a SavedModel from Keras"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AtSmftAvhJvE"
+      },
+      "source": [
+        "Deprecated: For Keras objects, it's recommended to use the new high-level `.keras` format and `tf.keras.Model.export`, as demonstrated in the guide [here](https://www.tensorflow.org/guide/keras/save_and_serialize). The low-level SavedModel format continues to be supported for existing code."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eLSOptpYhJvE"
+      },
+      "source": [
         "For a quick introduction, this section exports a pre-trained Keras model and serves image classification requests with it. The rest of the guide will fill in details and discuss other ways to create SavedModels."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Le5OB-fBHHW7"
       },
       "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "try:\n",
-        "  # %tensorflow_version only exists in Colab.\n",
-        "  %tensorflow_version 2.x\n",
-        "except Exception:\n",
-        "  pass\n",
-        "import tensorflow as tf\n",
+        "import os\n",
+        "import tempfile\n",
+        "\n",
         "from matplotlib import pyplot as plt\n",
-        "import numpy as np"
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "tmpdir = tempfile.mkdtemp()"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "wlho4HEWoHUT"
       },
       "outputs": [],
       "source": [
-        "physical_devices = tf.config.experimental.list_physical_devices('GPU')\n",
-        "if physical_devices:\n",
-        "  tf.config.experimental.set_memory_growth(physical_devices[0], True)"
+        "physical_devices = tf.config.list_physical_devices('GPU')\n",
+        "for device in physical_devices:\n",
+        "  tf.config.experimental.set_memory_growth(device, True)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "SofdPKo0G8Lb"
       },
       "outputs": [],
@@ -144,10 +149,10 @@
         "file = tf.keras.utils.get_file(\n",
         "    \"grace_hopper.jpg\",\n",
         "    \"https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg\")\n",
-        "img = tf.keras.preprocessing.image.load_img(file, target_size=[224, 224])\n",
+        "img = tf.keras.utils.load_img(file, target_size=[224, 224])\n",
         "plt.imshow(img)\n",
         "plt.axis('off')\n",
-        "x = tf.keras.preprocessing.image.img_to_array(img)\n",
+        "x = tf.keras.utils.img_to_array(img)\n",
         "x = tf.keras.applications.mobilenet.preprocess_input(\n",
         "    x[tf.newaxis,...])"
       ]
@@ -155,19 +160,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "sqVcFL10JkF0"
       },
       "source": [
-        "We'll use an image of Grace Hopper as a running example, and a Keras pre-trained image classification model since it's easy to use. Custom models work too, and are covered in detail later."
+        "You'll use an image of Grace Hopper as a running example, and a Keras pre-trained image classification model since it's easy to use. Custom models work too, and are covered in detail later."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "JhVecdzJTsKE"
       },
       "outputs": [],
@@ -180,10 +182,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "aEHSYjW6JZHV"
       },
       "outputs": [],
@@ -199,7 +199,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "r4KIsQDZJ5PS"
       },
       "source": [
@@ -208,82 +207,52 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "8nfznDmHCW6F"
       },
       "outputs": [],
       "source": [
-        "tf.saved_model.save(pretrained_model, \"/tmp/mobilenet/1/\")"
+        "mobilenet_save_path = os.path.join(tmpdir, \"mobilenet/1/\")\n",
+        "tf.saved_model.save(pretrained_model, mobilenet_save_path)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "pyX-ETE3wX63"
       },
       "source": [
         "The save-path follows a convention used by TensorFlow Serving where the last path component (`1/` here) is a version number for your model - it allows tools like Tensorflow Serving to reason about the relative freshness.\n",
         "\n",
-        "SavedModels have named functions called signatures. Keras models export their forward pass under the `serving_default` signature key. The [SavedModel command line interface](#saved_model_cli) is useful for inspecting SavedModels on disk:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "djmcTavtIZyT"
-      },
-      "outputs": [],
-      "source": [
-        "!saved_model_cli show --dir /tmp/mobilenet/1 --tag_set serve --signature_def serving_default"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "VCZZ8avqLF1g"
-      },
-      "source": [
-        "We can load the SavedModel back into Python with `tf.saved_model.load` and see how Admiral Hopper's image is classified."
+        "You can load the SavedModel back into Python with `tf.saved_model.load` and see how Admiral Hopper's image is classified."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "NP2UpVFRV7N_"
       },
       "outputs": [],
       "source": [
-        "loaded = tf.saved_model.load(\"/tmp/mobilenet/1/\")\n",
+        "loaded = tf.saved_model.load(mobilenet_save_path)\n",
         "print(list(loaded.signatures.keys()))  # [\"serving_default\"]"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "K5srGzowfWff"
       },
       "source": [
-        "Imported signatures always return dictionaries."
+        "Imported signatures always return dictionaries. To customize signature names and output dictionary keys, see [Specifying signatures during export](#specifying_signatures_during_export)."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "ChFLpegYfQGR"
       },
       "outputs": [],
@@ -295,7 +264,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "cJYyZnptfuru"
       },
       "source": [
@@ -304,10 +272,8 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "9WjGEaS3XfX7"
       },
       "outputs": [],
@@ -322,7 +288,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "SJEkdXjTWbtl"
       },
       "source": [
@@ -330,37 +295,12 @@
         "\n",
         "SavedModels are usable from Python (more on that below), but production environments typically use a dedicated service for inference without running Python code. This is easy to set up from a SavedModel using TensorFlow Serving.\n",
         "\n",
-        "See the [TensorFlow Serving REST tutorial](https://www.tensorflow.org/tfx/tutorials/serving/rest_simple) for more details about serving, including instructions for installing `tensorflow_model_server` in a notebook or on your local machine. As a quick sketch, to serve the `mobilenet` model exported above just point the model server at the SavedModel directory:\n",
-        "\n",
-        "```bash\n",
-        "nohup tensorflow_model_server \\\n",
-        "  --rest_api_port=8501 \\\n",
-        "  --model_name=mobilenet \\\n",
-        "  --model_base_path=\"/tmp/mobilenet\" \u003eserver.log 2\u003e\u00261\n",
-        "```\n",
-        "\n",
-        "  Then send a request.\n",
-        "\n",
-        "```python\n",
-        "!pip install requests\n",
-        "import json\n",
-        "import numpy\n",
-        "import requests\n",
-        "data = json.dumps({\"signature_name\": \"serving_default\",\n",
-        "                   \"instances\": x.tolist()})\n",
-        "headers = {\"content-type\": \"application/json\"}\n",
-        "json_response = requests.post('http://localhost:8501/v1/models/mobilenet:predict',\n",
-        "                              data=data, headers=headers)\n",
-        "predictions = numpy.array(json.loads(json_response.text)[\"predictions\"])\n",
-        "```\n",
-        "\n",
-        "The resulting `predictions` are identical to the results from Python."
+        "See the [TensorFlow Serving REST tutorial](https://www.tensorflow.org/tfx/tutorials/serving/rest_simple) for an end-to-end tensorflow-serving example."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Bi0ILzu1XdWw"
       },
       "source": [
@@ -371,21 +311,18 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "6u3YZuYZXyTO"
       },
       "outputs": [],
       "source": [
-        "!ls /tmp/mobilenet/1"
+        "!ls {mobilenet_save_path}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ple4X5utX8ue"
       },
       "source": [
@@ -396,21 +333,18 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "Pus0dOYTYXbI"
       },
       "outputs": [],
       "source": [
-        "!saved_model_cli show --dir /tmp/mobilenet/1 --tag_set serve"
+        "!saved_model_cli show --dir {mobilenet_save_path} --tag_set serve"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "eALHpGvRZOhk"
       },
       "source": [
@@ -419,47 +353,45 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "EDYqhDlNZAC2"
       },
       "outputs": [],
       "source": [
-        "!ls /tmp/mobilenet/1/variables"
+        "!ls {mobilenet_save_path}/variables"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "VKmaZQpHahGh"
       },
       "source": [
         "The `assets` directory contains files used by the TensorFlow graph, for example text files used to initialize vocabulary tables. It is unused in this example.\n",
         "\n",
-        "SavedModels may have an `assets.extra` directory for any files not used by the TensorFlow graph, for example information for consumers about what to do with the SavedModel. TensorFlow itself does not use this directory."
+        "SavedModels may have an `assets.extra` directory for any files not used by the TensorFlow graph, for example information for consumers about what to do with the SavedModel. TensorFlow itself does not use this directory.\n",
+        "\n",
+        "The `fingerprint.pb` file contains the [fingerprint](https://en.wikipedia.org/wiki/Fingerprint_(computing)) of the SavedModel, which is composed of several 64-bit hashes that uniquely identify the contents of the SavedModel. The fingerprinting API is currently experimental, but `tf.saved_model.experimental.read_fingerprint` can be used to read the SavedModel fingerprint into a `tf.saved_model.experimental.Fingerprint` object."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "zIceoF_CYmaF"
       },
       "source": [
-        "## Exporting custom models\n",
+        "## Saving a custom model\n",
+        "\n",
+        "`tf.saved_model.save` supports saving `tf.Module` objects and its subclasses, like `tf.keras.Layer` and `tf.keras.Model`.\n",
         "\n",
-        "In the first section, `tf.saved_model.save` automatically determined a signature for the `tf.keras.Model` object. This worked because Keras `Model` objects have an unambiguous method to export and known input shapes. `tf.saved_model.save` works just as well with low-level model building APIs, but you will need to indicate which function to use as a signature if you're planning to serve a model."
+        "Let's look at an example of saving and restoring a `tf.Module`.\n"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
         "id": "6EPvKiqXMm3d"
       },
       "outputs": [],
@@ -472,6 +404,7 @@
         "\n",
         "  @tf.function\n",
         "  def __call__(self, x):\n",
+        "    print('Tracing with', x)\n",
         "    return x * self.v\n",
         "\n",
         "  @tf.function(input_signature=[tf.TensorSpec([], tf.float32)])\n",
@@ -484,297 +417,101 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "fUrCTSK2HV2b"
+        "id": "J4FcP-Co3Fnw"
       },
       "source": [
-        "This module has two methods decorated with `tf.function`. While these functions will be included in the SavedModel and available if the SavedModel is reloaded via `tf.saved_model.load` into a Python program, without explicitly declaring the serving signature tools like Tensorflow Serving and `saved_model_cli` cannot access them.\n",
+        "When you save a `tf.Module`, any `tf.Variable` attributes, `tf.function`-decorated methods, and `tf.Module`s found via recursive traversal are saved. (See the [Checkpoint tutorial](./checkpoint.ipynb) for more about this recursive traversal.) However, any Python attributes, functions, and data are lost. This means that when a `tf.function` is saved, no Python code is saved.\n",
         "\n",
-        "The `mutate` method has a declared `input_signature`, and so there is enough information to save its computation graph in the SavedModel already. The `__call__` method has no declared signature, and so its signature(s) are inferred from how it has been used before saving: Calling the method one or more times will create computation graphs for each particular combination of the tensor shapes and dtypes seen in the arguments. (The [tf.function](../tutorials/customization/performance.ipynb#tracing) tutorial calls that *tracing* the function.)\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "85PUO9iWH7xn"
-      },
-      "outputs": [],
-      "source": [
-        "module(tf.constant(0.))\n",
-        "tf.saved_model.save(module, \"/tmp/module_no_signatures\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "eyWD4wr-Ng7m"
-      },
-      "source": [
-        "For functions without an `input_signature`, any input shapes used before saving will be available after loading. Since we called `__call__` with just a scalar, it will accept only scalar values."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "xy7oCex1Ibj1"
-      },
-      "outputs": [],
-      "source": [
-        "imported = tf.saved_model.load(\"/tmp/module_no_signatures\")\n",
-        "assert 3. == imported(tf.constant(3.)).numpy()\n",
-        "imported.mutate(tf.constant(2.))\n",
-        "assert 6. == imported(tf.constant(3.)).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "lbLNVfVJOTfb"
-      },
-      "source": [
-        "The function will not accept new shapes like vectors.\n",
+        "If no Python code is saved, how does SavedModel know how to restore the function?\n",
         "\n",
-        "```python\n",
-        "imported(tf.constant([3.]))\n",
-        "```\n",
+        "Briefly, `tf.function` works by tracing the Python code to generate a ConcreteFunction (a callable wrapper around `tf.Graph`). When saving a `tf.function`, you're really saving the `tf.function`'s cache of ConcreteFunctions.\n",
         "\n",
-        "\u003cpre\u003e\n",
-        "ValueError: Could not find matching function to call for canonicalized inputs ((\u003ctf.Tensor 'args_0:0' shape=(1,) dtype=float32\u003e,), {}). Only existing signatures are [((TensorSpec(shape=(), dtype=tf.float32, name=u'x'),), {})].\n",
-        "\u003c/pre\u003e"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "WrJqD-epPGnr"
-      },
-      "source": [
-        "`get_concrete_function` lets you add input shapes to a function without calling it. It takes `tf.TensorSpec` objects in place of `Tensor` arguments, indicating the shapes and dtypes of inputs. Shapes can either be `None`, indicating that any shape is acceptable, or a list of axis sizes. If an axis size is `None` then any size is acceptable for that axis. (This is often used for batch size.) `tf.TensorSpecs` can also have names, which default to the function's argument keywords (\"x\" here)."
+        "To learn more about the relationship between `tf.function` and ConcreteFunctions, refer to the [tf.function guide](function.ipynb)."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "1m9Okb75PFmb"
-      },
-      "outputs": [],
-      "source": [
-        "module.__call__.get_concrete_function(x=tf.TensorSpec([None], tf.float32))\n",
-        "tf.saved_model.save(module, \"/tmp/module_no_signatures\")\n",
-        "imported = tf.saved_model.load(\"/tmp/module_no_signatures\")\n",
-        "assert [3.] == imported(tf.constant([3.])).numpy()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "text",
-        "id": "gvy3GFl4IfSW"
-      },
-      "source": [
-        "We didn't identify any of the functions we exported as a signature, so it has none."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "uNTV6o_TIeRu"
+        "id": "85PUO9iWH7xn"
       },
       "outputs": [],
       "source": [
-        "!saved_model_cli show --dir /tmp/module_no_signatures --tag_set serve"
+        "module_no_signatures_path = os.path.join(tmpdir, 'module_no_signatures')\n",
+        "module(tf.constant(0.))\n",
+        "print('Saving model...')\n",
+        "tf.saved_model.save(module, module_no_signatures_path)"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "BiNtaMZSI8Tb"
-      },
-      "source": [
-        "### Identifying a signature to export\n",
-        "\n",
-        "To indicate that a function should be a signature, specify the `signatures` argument when saving."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "_pAdgIORR2yH"
+        "id": "2ujwmMQg7OUo"
       },
-      "outputs": [],
       "source": [
-        "call = module.__call__.get_concrete_function(tf.TensorSpec(None, tf.float32))\n",
-        "tf.saved_model.save(module, \"/tmp/module_with_signature\", signatures=call)"
+        "## Loading and using a custom model"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "lHiBm-kdKBmG"
-      },
-      "source": [
-        "Notice that we first converted the `tf.function` to a `ConcreteFunction` with `get_concrete_function`. This is necessary because the function was created without a fixed `input_signature`, and so did not have a definite set of `Tensor` inputs associated with it."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "nAzRHR0UT4hv"
+        "id": "QpxQy5Eb77qJ"
       },
-      "outputs": [],
       "source": [
-        "!saved_model_cli show --dir /tmp/module_with_signature --tag_set serve --signature_def serving_default"
+        "When you load a SavedModel in Python, all `tf.Variable` attributes, `tf.function`-decorated methods, and `tf.Module`s are restored in the same object structure as the original saved `tf.Module`."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "0B25WsscTZoC"
+        "id": "EMASjADPxPso"
       },
       "outputs": [],
       "source": [
-        "imported = tf.saved_model.load(\"/tmp/module_with_signature\")\n",
-        "signature = imported.signatures[\"serving_default\"]\n",
-        "assert [3.] == signature(x=tf.constant([3.]))[\"output_0\"].numpy()\n",
+        "imported = tf.saved_model.load(module_no_signatures_path)\n",
+        "assert imported(tf.constant(3.)).numpy() == 3\n",
         "imported.mutate(tf.constant(2.))\n",
-        "assert [6.] == signature(x=tf.constant([3.]))[\"output_0\"].numpy()\n",
-        "assert 2. == imported.v.numpy()"
+        "assert imported(tf.constant(3.)).numpy() == 6"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "_gH91j1IR4tq"
-      },
-      "source": [
-        "We exported a single signature, and its key defaulted to \"serving_default\". To export multiple signatures, pass a dictionary."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "6VYAiQmLUiox"
+        "id": "CDiauvb_99uk"
       },
-      "outputs": [],
       "source": [
-        "@tf.function(input_signature=[tf.TensorSpec([], tf.string)])\n",
-        "def parse_string(string_input):\n",
-        "  return imported(tf.strings.to_number(string_input))\n",
-        "\n",
-        "signatures = {\"serving_default\": parse_string,\n",
-        "              \"from_float\": imported.signatures[\"serving_default\"]}\n",
+        "Because no Python code is saved, calling a `tf.function` with a new input signature will fail:\n",
         "\n",
-        "tf.saved_model.save(imported, \"/tmp/module_with_multiple_signatures\", signatures)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "8IPx_0RWEx07"
-      },
-      "outputs": [],
-      "source": [
-        "!saved_model_cli show --dir /tmp/module_with_multiple_signatures --tag_set serve"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "sRdSPjKFfQpx"
-      },
-      "source": [
-        "`saved_model_cli` can also run SavedModels directly from the command line."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "hcHmgVytfODo"
-      },
-      "outputs": [],
-      "source": [
-        "!saved_model_cli run --dir /tmp/module_with_multiple_signatures --tag_set serve --signature_def serving_default --input_exprs=\"string_input='3.'\"\n",
-        "!saved_model_cli run --dir /tmp/module_with_multiple_signatures --tag_set serve --signature_def from_float --input_exprs=\"x=3.\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "WiNhHa_Ne82K"
-      },
-      "source": [
-        "## Reusing SavedModels in Python\n",
-        "\n",
-        "Let us look again at class `CustomModule` above, and how `module` objects of this type were saved as SavedModels and loaded back. Did you notice the call to `module(...)` without the use of signatures?\n",
-        "\n",
-        "Calling `tf.saved_model.save(obj, '/path')` on objects derived from `tf.keras.Model` or `tf.Module` saves `tf.Variable` attributes as described in the [training checkpoints](./checkpoint.ipynb) tutorial: `obj` and, recursively, the objects referenced by its attributes are traversed and the `tf.Variable` attributes found on those objects have their current values saved. Likewise, the `tf.function`-decorated methods found on those objects have their computation graphs saved. However, the original Python types, Python code of methods, and Python-valued data members are lost.\n",
-        "\n",
-        "Calling `obj = tf.saved_model.load('/path')` restores the saved objects (now reduced to a placeholder type), their `tf.Variable` attributes with their respective saved values, and their `tf.function`-decorated methods. Those methods can be called as before, as long as a computation graph has been saved that covers the combination of `Tensor` shapes and non-`Tensor` values used in the call. [Retracing the tf.function from Python code](../tutorials/customization/performance.ipynb#tracing) is no longer possible and will raise an exception.\n",
+        "```python\n",
+        "imported(tf.constant([3.]))\n",
+        "```\n",
         "\n",
-        "The restored `tf.function`s provide a richer, more Pythonic API to the restored model than the concrete functions in the `.signatures` dict. However, this API is inaccessible to the non-Python environments that rely on signatures."
+        "
\n",
+        "ValueError: Could not find matching function to call for canonicalized inputs ((,), {}). Only existing signatures are [((TensorSpec(shape=(), dtype=tf.float32, name=u'x'),), {})].\n",
+        "
"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "V5sfikXOmvDq"
+        "id": "4Vsva3UZ-2sf"
       },
       "source": [
         "### Basic fine-tuning\n",
         "\n",
-        "Variable objects are available, and we can backprop through imported functions. That is enough to fine-tune (i.e. retrain) a SavedModel in simple cases."
+        "Variable objects are available, and you can backprop through imported functions. That is enough to fine-tune (i.e. retrain) a SavedModel in simple cases."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "mSchcIB2e-n0"
+        "id": "PEkQNarJ-7nT"
       },
       "outputs": [],
       "source": [
-        "optimizer = tf.optimizers.SGD(0.05)\n",
+        "optimizer = tf.keras.optimizers.SGD(0.05)\n",
         "\n",
         "def train_step():\n",
         "  with tf.GradientTape() as tape:\n",
@@ -787,11 +524,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "Yx9bO2taJJxm"
+        "id": "p41NM6fF---3"
       },
       "outputs": [],
       "source": [
@@ -803,8 +538,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "pGNJvLeCaK6G"
+        "id": "XuXtkHSD_KSW"
       },
       "source": [
         "### General fine-tuning\n",
@@ -815,20 +549,18 @@
         "  * Next to the `__call__` attribute, there are `.variable` and `.trainable_variable` attributes with the corresponding lists of variables. A variable that was originally trainable but is meant to be frozen during fine-tuning is omitted from `.trainable_variables`.\n",
         "  * For the sake of frameworks like Keras that represent weight regularizers as attributes of layers or sub-models, there can also be a `.regularization_losses` attribute. It holds a list of zero-argument functions whose values are meant for addition to the total loss.\n",
         "\n",
-        "Going back to the initial MobileNet example, we can see some of those in action:"
+        "Going back to the initial MobileNet example, you can see some of those in action:"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "SF5H5O6ApUbH"
+        "id": "Y6EUFdY8_PRD"
       },
       "outputs": [],
       "source": [
-        "loaded = tf.saved_model.load(\"/tmp/mobilenet/1/\")\n",
+        "loaded = tf.saved_model.load(mobilenet_save_path)\n",
         "print(\"MobileNet has {} trainable variables: {}, ...\".format(\n",
         "          len(loaded.trainable_variables),\n",
         "          \", \".join([v.name for v in loaded.trainable_variables[:5]])))"
@@ -836,11 +568,9 @@
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "q24TRUD5pUbV"
+        "id": "B-mQJ8iP_R0h"
       },
       "outputs": [],
       "source": [
@@ -855,146 +585,185 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "qyL9tOPrg5Zw"
+        "id": "qGlHlbd3_eyO"
       },
       "source": [
-        "## Control flow in SavedModels\n",
+        "## Specifying signatures during export\n",
+        "\n",
+        "Tools like TensorFlow Serving and `saved_model_cli` can interact with SavedModels. To help these tools determine which ConcreteFunctions to use, you need to specify serving signatures. `tf.keras.Model`s automatically specify serving signatures, but you'll have to explicitly declare a serving signature for our custom modules.\n",
         "\n",
-        "Anything that can go in a `tf.function` can go in a SavedModel. With [AutoGraph](./function.ipynb) this includes conditional logic which depends on Tensors, specified with regular Python control flow."
+        "IMPORTANT: Unless you need to export your model to an environment other than TensorFlow 2.x with Python, you probably don't need to export signatures explicitly. If you're looking for a way of enforcing an input signature for a specific function, see the [`input_signature`](https://www.tensorflow.org/api_docs/python/tf/function#args_1) argument to `tf.function`.\n",
+        "\n",
+        "By default, no signatures are declared in a custom `tf.Module`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "h-IB5Xa0NxLa"
+      },
+      "outputs": [],
+      "source": [
+        "assert len(imported.signatures) == 0"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BiNtaMZSI8Tb"
+      },
+      "source": [
+        "To declare a serving signature, specify a ConcreteFunction using the `signatures` kwarg. When specifying a single signature, its signature key will be `'serving_default'`, which is saved as the constant `tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY`."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "tfbh3uGMgBpH"
+        "id": "_pAdgIORR2yH"
       },
       "outputs": [],
       "source": [
-        "@tf.function(input_signature=[tf.TensorSpec([], tf.int32)])\n",
-        "def control_flow(x):\n",
-        "  if x \u003c 0:\n",
-        "    tf.print(\"Invalid!\")\n",
-        "  else:\n",
-        "    tf.print(x % 3)\n",
-        "\n",
-        "to_export = tf.Module()\n",
-        "to_export.control_flow = control_flow\n",
-        "tf.saved_model.save(to_export, \"/tmp/control_flow\")"
+        "module_with_signature_path = os.path.join(tmpdir, 'module_with_signature')\n",
+        "call = module.__call__.get_concrete_function(tf.TensorSpec(None, tf.float32))\n",
+        "tf.saved_model.save(module, module_with_signature_path, signatures=call)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "bv4EXevIjHch"
+        "id": "nAzRHR0UT4hv"
       },
       "outputs": [],
       "source": [
-        "imported = tf.saved_model.load(\"/tmp/control_flow\")\n",
-        "imported.control_flow(tf.constant(-1))  # Invalid!\n",
-        "imported.control_flow(tf.constant(2))   # 2\n",
-        "imported.control_flow(tf.constant(3))   # 0"
+        "imported_with_signatures = tf.saved_model.load(module_with_signature_path)\n",
+        "list(imported_with_signatures.signatures.keys())  # [\"serving_default\"]"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "Dk5wWyuMpuHx"
+        "id": "_gH91j1IR4tq"
       },
       "source": [
-        "## SavedModels from Estimators\n",
+        "To export multiple signatures, pass a dictionary of signature keys to ConcreteFunctions. Each signature key corresponds to one ConcreteFunction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6VYAiQmLUiox"
+      },
+      "outputs": [],
+      "source": [
+        "module_multiple_signatures_path = os.path.join(tmpdir, 'module_with_multiple_signatures')\n",
+        "signatures = {\"serving_default\": call,\n",
+        "              \"array_input\": module.__call__.get_concrete_function(tf.TensorSpec([None], tf.float32))}\n",
         "\n",
-        "Estimators export SavedModels through [`tf.Estimator.export_saved_model`](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator#export_saved_model). See the [guide to Estimator](https://www.tensorflow.org/guide/estimator) for details."
+        "tf.saved_model.save(module, module_multiple_signatures_path, signatures=signatures)"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "B9KQq5qzpzbK"
+        "id": "8IPx_0RWEx07"
       },
       "outputs": [],
       "source": [
-        "input_column = tf.feature_column.numeric_column(\"x\")\n",
-        "estimator = tf.estimator.LinearClassifier(feature_columns=[input_column])\n",
-        "\n",
-        "def input_fn():\n",
-        "  return tf.data.Dataset.from_tensor_slices(\n",
-        "    ({\"x\": [1., 2., 3., 4.]}, [1, 1, 0, 0])).repeat(200).shuffle(64).batch(16)\n",
-        "estimator.train(input_fn)\n",
-        "\n",
-        "serving_input_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(\n",
-        "  tf.feature_column.make_parse_example_spec([input_column]))\n",
-        "export_path = estimator.export_saved_model(\n",
-        "  \"/tmp/from_estimator/\", serving_input_fn)"
+        "imported_with_multiple_signatures = tf.saved_model.load(\n",
+        "    module_multiple_signatures_path\n",
+        ")\n",
+        "list(\n",
+        "    imported_with_multiple_signatures.signatures.keys()\n",
+        ")  # [\"serving_default\", \"array_input\"]"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "XJ4PJ-Cl4060"
+        "id": "43_Qv2W_DJZZ"
       },
       "source": [
-        "This SavedModel accepts serialized `tf.Example` protocol buffers, which are useful for serving. But we can also load it with `tf.saved_model.load` and run it from Python."
+        "By default, the output tensor names are fairly generic, like `output_0`. To control the names of outputs, modify your `tf.function` to return a dictionary that maps output names to outputs. The names of inputs are derived from the Python function arg names."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "c_BUBBNB1UH9"
+        "id": "ACKPl1X8G1gw"
       },
       "outputs": [],
       "source": [
-        "imported = tf.saved_model.load(export_path)\n",
+        "class CustomModuleWithOutputName(tf.Module):\n",
+        "  def __init__(self):\n",
+        "    super(CustomModuleWithOutputName, self).__init__()\n",
+        "    self.v = tf.Variable(1.)\n",
+        "\n",
+        "  @tf.function(input_signature=[tf.TensorSpec(None, tf.float32)])\n",
+        "  def __call__(self, x):\n",
+        "    return {'custom_output_name': x * self.v}\n",
         "\n",
-        "def predict(x):\n",
-        "  example = tf.train.Example()\n",
-        "  example.features.feature[\"x\"].float_list.value.extend([x])\n",
-        "  return imported.signatures[\"predict\"](\n",
-        "    examples=tf.constant([example.SerializeToString()]))"
+        "module_output = CustomModuleWithOutputName()\n",
+        "call_output = module_output.__call__.get_concrete_function(tf.TensorSpec(None, tf.float32))\n",
+        "module_output_path = os.path.join(tmpdir, 'module_with_output_name')\n",
+        "tf.saved_model.save(module_output, module_output_path,\n",
+        "                    signatures={'serving_default': call_output})"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 0,
+      "execution_count": null,
       "metadata": {
-        "colab": {},
-        "colab_type": "code",
-        "id": "C1ylWZCQ1ahG"
+        "id": "1yGVy4MuH-V0"
       },
       "outputs": [],
       "source": [
-        "print(predict(1.5))\n",
-        "print(predict(3.5))"
+        "imported_with_output_name = tf.saved_model.load(module_output_path)\n",
+        "imported_with_output_name.signatures[\n",
+        "    'serving_default'\n",
+        "].structured_outputs  # {'custom_output_name': TensorSpec(shape=, dtype=tf.float32, name='custom_output_name')}"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "_IrCCm0-isqA"
+        "id": "Q4bCK55x1IBW"
       },
       "source": [
-        "`tf.estimator.export.build_raw_serving_input_receiver_fn` allows you to create input functions which take raw tensors rather than `tf.train.Example`s."
+        "## Proto-splitting\n",
+        "\n",
+        "Note: This feature will be part of the TensorFlow 2.15 release. It is currently available in the nightly build which you cqan install with `pip install tf-nightly`.\n",
+        "\n",
+        "Due to limits of the protobuf implementation, proto sizes cannot exceed 2GB. This can lead to the following errors when attempting to save very large models:\n",
+        "\n",
+        "```\n",
+        "ValueError: Message tensorflow.SavedModel exceeds maximum protobuf size of 2GB: ...\n",
+        "```\n",
+        "\n",
+        "```\n",
+        "google.protobuf.message.DecodeError: Error parsing message as the message exceeded the protobuf limit with type 'tensorflow.GraphDef'\n",
+        "```\n",
+        "\n",
+        "If you wish to save models that exceed the 2GB limit, then you'll need to save using the new proto-splitting option:\n",
+        "\n",
+        "```python\n",
+        "tf.saved_model.save(\n",
+        "  ...,\n",
+        "  options=tf.saved_model.SaveOptions(experimental_image_format=True)\n",
+        ")\n",
+        "```\n",
+        "\n",
+        "More information can be found in the [Proto Splitter / Merger Library guide](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/proto_splitter/g3doc/in-depth-guide.md)."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Co6fDbzw_UnD"
       },
       "source": [
@@ -1007,18 +776,17 @@
         "SavedModelBundle bundle;\n",
         "...\n",
         "LoadSavedModel(session_options, run_options, export_dir, {kSavedModelTagTrain},\n",
-        "               \u0026bundle);\n",
+        "               &bundle);\n",
         "```"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "b33KuyEuAO3Z"
       },
       "source": [
-        "\u003ca id=saved_model_cli/\u003e\n",
+        "\n",
         "\n",
         "## Details of the SavedModel command line interface\n",
         "\n",
@@ -1048,7 +816,7 @@
         "additional command to build `saved_model_cli`:\n",
         "\n",
         "```\n",
-        "$ bazel build tensorflow/python/tools:saved_model_cli\n",
+        "$ bazel build //tensorflow/python/tools:saved_model_cli\n",
         "```\n",
         "\n",
         "### Overview of commands\n",
@@ -1094,9 +862,9 @@
         "If there are *multiple* tags in the tag-set, you must specify\n",
         "all tags, each tag separated by a comma. For example:\n",
         "\n",
-        "\u003cpre\u003e\n",
+        "
\n",
         "$ saved_model_cli show --dir /tmp/saved_model_dir --tag_set serve,gpu\n",
-        "\u003c/pre\u003e\n",
+        "
\n",
         "\n",
         "To show all inputs and outputs TensorInfo for a specific `SignatureDef`, pass in\n",
         "the `SignatureDef` key to `signature_def` option. This is very useful when you\n",
@@ -1122,7 +890,7 @@
         "To show all available information in the SavedModel, use the `--all` option.\n",
         "For example:\n",
         "\n",
-        "\u003cpre\u003e\n",
+        "
\n",
         "$ saved_model_cli show --dir /tmp/saved_model_dir --all\n",
         "MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:\n",
         "\n",
@@ -1153,7 +921,7 @@
         "        shape: (-1, 1)\n",
         "        name: y:0\n",
         "  Method name is: tensorflow/serving/predict\n",
-        "\u003c/pre\u003e\n",
+        "
\n",
         "\n",
         "\n",
         "### `run` command\n",
@@ -1182,13 +950,13 @@
         "following general format:\n",
         "\n",
         "```bsh\n",
-        "--inputs \u003cINPUTS\u003e\n",
+        "--inputs \n",
         "```\n",
         "\n",
         "where *INPUTS* is either of the following formats:\n",
         "\n",
-        "*  `\u003cinput_key\u003e=\u003cfilename\u003e`\n",
-        "*  `\u003cinput_key\u003e=\u003cfilename\u003e[\u003cvariable_name\u003e]`\n",
+        "*  `=`\n",
+        "*  `=[]`\n",
         "\n",
         "You may pass multiple *INPUTS*. If you do pass multiple inputs, use a semicolon\n",
         "to separate each of the *INPUTS*.\n",
@@ -1227,14 +995,14 @@
         "For example:\n",
         "\n",
         "```bsh\n",
-        "`\u003cinput_key\u003e=[[1],[2],[3]]`\n",
+        "`=[[1],[2],[3]]`\n",
         "```\n",
         "\n",
         "In addition to Python expressions, you may also pass numpy functions. For\n",
         "example:\n",
         "\n",
         "```bsh\n",
-        "`\u003cinput_key\u003e=np.ones((32,32,3))`\n",
+        "`=np.ones((32,32,3))`\n",
         "```\n",
         "\n",
         "(Note that the `numpy` module is already available to you as `np`.)\n",
@@ -1249,7 +1017,7 @@
         "For example:\n",
         "\n",
         "```bsh\n",
-        "`\u003cinput_key\u003e=[{\"age\":[22,24],\"education\":[\"BS\",\"MS\"]}]`\n",
+        "`=[{\"age\":[22,24],\"education\":[\"BS\",\"MS\"]}]`\n",
         "```\n",
         "\n",
         "#### Save output\n",
@@ -1264,9 +1032,7 @@
   ],
   "metadata": {
     "colab": {
-      "collapsed_sections": [],
       "name": "saved_model.ipynb",
-      "private_outputs": true,
       "provenance": [],
       "toc_visible": true
     },
diff --git a/site/en/guide/sparse_tensor.ipynb b/site/en/guide/sparse_tensor.ipynb
new file mode 100644
index 00000000000..3d4daca7fad
--- /dev/null
+++ b/site/en/guide/sparse_tensor.ipynb
@@ -0,0 +1,693 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "drGgRRpWf2Qm"
+      },
+      "source": [
+        "# Working with sparse tensors"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UIiXFIS4fj1m"
+      },
+      "source": [
+        "When working with tensors that contain a lot of zero values, it is important to store them in a space- and time-efficient manner. Sparse tensors enable efficient storage and processing of tensors that contain a lot of zero values. Sparse tensors are used extensively in encoding schemes like [TF-IDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf) as part of data pre-processing in NLP applications and for pre-processing images with a lot of dark pixels in computer vision applications."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A8XXQW3ENU5m"
+      },
+      "source": [
+        "## Sparse tensors in TensorFlow\n",
+        "\n",
+        "TensorFlow represents sparse tensors through the `tf.sparse.SparseTensor` object. Currently, sparse tensors in TensorFlow are encoded using the coordinate list (COO) format. This encoding format is optimized for hyper-sparse matrices such as embeddings.\n",
+        "\n",
+        "The COO encoding for sparse tensors is comprised of:\n",
+        "\n",
+        "  * `values`: A 1D tensor with shape `[N]` containing all nonzero values.\n",
+        "  * `indices`: A 2D tensor with shape `[N, rank]`, containing the indices of the nonzero values.\n",
+        "  * `dense_shape`: A 1D tensor with shape `[rank]`, specifying the shape of the tensor.\n",
+        "\n",
+        "A ***nonzero*** value in the context of a `tf.sparse.SparseTensor` is a value that's not explicitly encoded. It is possible to explicitly include zero values in the `values` of a COO sparse matrix, but these \"explicit zeros\" are generally not included when referring to nonzero values in a sparse tensor.\n",
+        "\n",
+        "Note: `tf.sparse.SparseTensor` does not require that indices/values be in any particular order, but several ops assume that they're in row-major order. Use `tf.sparse.reorder` to create a copy of the sparse tensor that is sorted in the canonical row-major order. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6Aq7ruwlyz79"
+      },
+      "source": [
+        "## Creating a `tf.sparse.SparseTensor`\n",
+        "\n",
+        "Construct sparse tensors by directly specifying their `values`, `indices`, and `dense_shape`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SI2Mv3tihcmY"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vqQKGva4zSCs"
+      },
+      "outputs": [],
+      "source": [
+        "st1 = tf.sparse.SparseTensor(indices=[[0, 3], [2, 4]],\n",
+        "                      values=[10, 20],\n",
+        "                      dense_shape=[3, 10])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l9eJeh31fWyr"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-M3fMTFL0hXa"
+      },
+      "source": [
+        "When you use the `print()` function to print a sparse tensor, it shows the contents of the three component tensors:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3oHWtmsBMLAI"
+      },
+      "outputs": [],
+      "source": [
+        "print(st1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qqePKJG6MNWk"
+      },
+      "source": [
+        "It is easier to understand the contents of a sparse tensor if the nonzero `values` are aligned with their corresponding `indices`. Define a helper function to pretty-print sparse tensors such that each nonzero value is shown on its own line."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R_xFYuOo1ZE_"
+      },
+      "outputs": [],
+      "source": [
+        "def pprint_sparse_tensor(st):\n",
+        "  s = \"\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "be4Dyiqt0fEH"
+      },
+      "outputs": [],
+      "source": [
+        "print(pprint_sparse_tensor(st1))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3FBt8qk_zmz5"
+      },
+      "source": [
+        "You can also construct sparse tensors from dense tensors by using `tf.sparse.from_dense`, and convert them back to dense tensors by using `tf.sparse.to_dense`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cYwuCuNMf0Fu"
+      },
+      "outputs": [],
+      "source": [
+        "st2 = tf.sparse.from_dense([[1, 0, 0, 8], [0, 0, 0, 0], [0, 0, 3, 0]])\n",
+        "print(pprint_sparse_tensor(st2))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eFVPrwNPzyZw"
+      },
+      "outputs": [],
+      "source": [
+        "st3 = tf.sparse.to_dense(st2)\n",
+        "print(st3)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GeuvyL_Z0Mwh"
+      },
+      "source": [
+        "## Manipulating sparse tensors\n",
+        "\n",
+        "Use the utilities in the `tf.sparse` package to manipulate sparse tensors. Ops like `tf.math.add` that you can use for arithmetic manipulation of dense tensors do not work with sparse tensors."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LMYW4U4Qavvd"
+      },
+      "source": [
+        "Add sparse tensors of the same shape by using `tf.sparse.add`.  "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vJwuSQIjayiN"
+      },
+      "outputs": [],
+      "source": [
+        "st_a = tf.sparse.SparseTensor(indices=[[0, 2], [3, 4]],\n",
+        "                       values=[31, 2], \n",
+        "                       dense_shape=[4, 10])\n",
+        "\n",
+        "st_b = tf.sparse.SparseTensor(indices=[[0, 2], [3, 0]],\n",
+        "                       values=[56, 38],\n",
+        "                       dense_shape=[4, 10])\n",
+        "\n",
+        "st_sum = tf.sparse.add(st_a, st_b)\n",
+        "\n",
+        "print(pprint_sparse_tensor(st_sum))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ls8_aQvnqZMj"
+      },
+      "source": [
+        "Use `tf.sparse.sparse_dense_matmul` to multiply sparse tensors with dense matrices."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S0tWRLiE04uL"
+      },
+      "outputs": [],
+      "source": [
+        "st_c = tf.sparse.SparseTensor(indices=([0, 1], [1, 0], [1, 1]),\n",
+        "                       values=[13, 15, 17],\n",
+        "                       dense_shape=(2,2))\n",
+        "\n",
+        "mb = tf.constant([[4], [6]])\n",
+        "product = tf.sparse.sparse_dense_matmul(st_c, mb)\n",
+        "\n",
+        "print(product)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9hxClYvfceZA"
+      },
+      "source": [
+        "Put sparse tensors together by using `tf.sparse.concat` and take them apart by using `tf.sparse.slice`.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cp4NEW_5yLEY"
+      },
+      "outputs": [],
+      "source": [
+        "sparse_pattern_A = tf.sparse.SparseTensor(indices = [[2,4], [3,3], [3,4], [4,3], [4,4], [5,4]],\n",
+        "                         values = [1,1,1,1,1,1],\n",
+        "                         dense_shape = [8,5])\n",
+        "sparse_pattern_B = tf.sparse.SparseTensor(indices = [[0,2], [1,1], [1,3], [2,0], [2,4], [2,5], [3,5], \n",
+        "                                              [4,5], [5,0], [5,4], [5,5], [6,1], [6,3], [7,2]],\n",
+        "                         values = [1,1,1,1,1,1,1,1,1,1,1,1,1,1],\n",
+        "                         dense_shape = [8,6])\n",
+        "sparse_pattern_C = tf.sparse.SparseTensor(indices = [[3,0], [4,0]],\n",
+        "                         values = [1,1],\n",
+        "                         dense_shape = [8,6])\n",
+        "\n",
+        "sparse_patterns_list = [sparse_pattern_A, sparse_pattern_B, sparse_pattern_C]\n",
+        "sparse_pattern = tf.sparse.concat(axis=1, sp_inputs=sparse_patterns_list)\n",
+        "print(tf.sparse.to_dense(sparse_pattern))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XmE87XVPWPmc"
+      },
+      "outputs": [],
+      "source": [
+        "sparse_slice_A = tf.sparse.slice(sparse_pattern_A, start = [0,0], size = [8,5])\n",
+        "sparse_slice_B = tf.sparse.slice(sparse_pattern_B, start = [0,5], size = [8,6])\n",
+        "sparse_slice_C = tf.sparse.slice(sparse_pattern_C, start = [0,10], size = [8,6])\n",
+        "print(tf.sparse.to_dense(sparse_slice_A))\n",
+        "print(tf.sparse.to_dense(sparse_slice_B))\n",
+        "print(tf.sparse.to_dense(sparse_slice_C))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "37SOx7wB1eSX"
+      },
+      "source": [
+        "If you're using TensorFlow 2.4 or above, use `tf.sparse.map_values` for elementwise operations on nonzero values in sparse tensors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "daZaPkkA1d09"
+      },
+      "outputs": [],
+      "source": [
+        "st2_plus_5 = tf.sparse.map_values(tf.add, st2, 5)\n",
+        "print(tf.sparse.to_dense(st2_plus_5))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3zkRcxeo2Elw"
+      },
+      "source": [
+        "Note that only the nonzero values were modified – the zero values stay zero.\n",
+        "\n",
+        "Equivalently, you can follow the design pattern below for earlier versions of TensorFlow:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bFSNOOqC0ySb"
+      },
+      "outputs": [],
+      "source": [
+        "st2_plus_5 = tf.sparse.SparseTensor(\n",
+        "    st2.indices,\n",
+        "    st2.values + 5,\n",
+        "    st2.dense_shape)\n",
+        "print(tf.sparse.to_dense(st2_plus_5))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GFhO2ZZ53ga1"
+      },
+      "source": [
+        "## Using `tf.sparse.SparseTensor` with other TensorFlow APIs\n",
+        "\n",
+        "Sparse tensors work transparently with these TensorFlow APIs:\n",
+        "\n",
+        "* `tf.keras`\n",
+        "* `tf.data`\n",
+        "* `tf.Train.Example` protobuf\n",
+        "* `tf.function`\n",
+        "* `tf.while_loop`\n",
+        "* `tf.cond`\n",
+        "* `tf.identity`\n",
+        "* `tf.cast`\n",
+        "* `tf.print`\n",
+        "* `tf.saved_model`\n",
+        "* `tf.io.serialize_sparse`\n",
+        "* `tf.io.serialize_many_sparse`\n",
+        "* `tf.io.deserialize_many_sparse`\n",
+        "* `tf.math.abs`\n",
+        "* `tf.math.negative`\n",
+        "* `tf.math.sign`\n",
+        "* `tf.math.square`\n",
+        "* `tf.math.sqrt`\n",
+        "* `tf.math.erf`\n",
+        "* `tf.math.tanh`\n",
+        "* `tf.math.bessel_i0e`\n",
+        "* `tf.math.bessel_i1e`\n",
+        "\n",
+        "Examples are shown below for a few of the above APIs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6uNUl7EgSYGC"
+      },
+      "source": [
+        "### `tf.keras`\n",
+        "\n",
+        "A subset of the `tf.keras` API supports sparse tensors without expensive casting or conversion ops. The Keras API lets you pass sparse tensors as inputs to a Keras model. Set `sparse=True` when calling `tf.keras.Input` or `tf.keras.layers.InputLayer`. You can pass sparse tensors between Keras layers, and also have Keras models return them as outputs. If you use sparse tensors in `tf.keras.layers.Dense` layers in your model, they will output dense tensors.\n",
+        "\n",
+        "The example below shows you how to pass a sparse tensor as an input to a Keras model if you use only layers that support sparse inputs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E8za5DK8vfo7"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.keras.Input(shape=(4,), sparse=True)\n",
+        "y = tf.keras.layers.Dense(4)(x)\n",
+        "model = tf.keras.Model(x, y)\n",
+        "\n",
+        "sparse_data = tf.sparse.SparseTensor(\n",
+        "    indices = [(0,0),(0,1),(0,2),\n",
+        "               (4,3),(5,0),(5,1)],\n",
+        "    values = [1,1,1,1,1,1],\n",
+        "    dense_shape = (6,4)\n",
+        ")\n",
+        "\n",
+        "model(sparse_data)\n",
+        "\n",
+        "model.predict(sparse_data)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZtVYmr7dt0-x"
+      },
+      "source": [
+        "### `tf.data`\n",
+        "\n",
+        "The `tf.data` API enables you to build complex input pipelines from simple, reusable pieces. Its core data structure is `tf.data.Dataset`, which represents a sequence of elements in which each element consists of one or more components.\n",
+        "\n",
+        "#### Building datasets with sparse tensors\n",
+        "\n",
+        "Build datasets from sparse tensors using the same methods that are used to build them from `tf.Tensor`s or NumPy arrays, such as `tf.data.Dataset.from_tensor_slices`. This op preserves the sparsity (or sparse nature) of the data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3y9tiwuZ5oTD"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = tf.data.Dataset.from_tensor_slices(sparse_data)\n",
+        "for element in dataset: \n",
+        "  print(pprint_sparse_tensor(element))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hFaY5Org59qk"
+      },
+      "source": [
+        "#### Batching and unbatching datasets with sparse tensors\n",
+        "\n",
+        "You can batch (combine consecutive elements into a single element) and unbatch datasets with sparse tensors using the `Dataset.batch` and `Dataset.unbatch` methods respectively."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WkKE0VY66Ii2"
+      },
+      "outputs": [],
+      "source": [
+        "batched_dataset = dataset.batch(2)\n",
+        "for element in batched_dataset:\n",
+        "  print (pprint_sparse_tensor(element))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ikZzPxl56bx1"
+      },
+      "outputs": [],
+      "source": [
+        "unbatched_dataset = batched_dataset.unbatch()\n",
+        "for element in unbatched_dataset:\n",
+        "  print (pprint_sparse_tensor(element))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ywfpD_EIMd3"
+      },
+      "source": [
+        "You can also use `tf.data.experimental.dense_to_sparse_batch` to batch dataset elements of varying shapes into sparse tensors. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oB8QKh7p6ltl"
+      },
+      "source": [
+        "#### Transforming Datasets with sparse tensors\n",
+        "\n",
+        "Transform and create sparse tensors in Datasets using `Dataset.map`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E5lhicwef7Ah"
+      },
+      "outputs": [],
+      "source": [
+        "transform_dataset = dataset.map(lambda x: x*2)\n",
+        "for i in transform_dataset:\n",
+        "  print(pprint_sparse_tensor(i))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DBfQvIVutp65"
+      },
+      "source": [
+        "### tf.train.Example\n",
+        "\n",
+        "`tf.train.Example` is a standard protobuf encoding for TensorFlow data. When using sparse tensors with `tf.train.Example`, you can:\n",
+        "\n",
+        "* Read variable-length data into a `tf.sparse.SparseTensor` using `tf.io.VarLenFeature`. However, you should consider using `tf.io.RaggedFeature` instead.\n",
+        "\n",
+        "* Read arbitrary sparse data into a `tf.sparse.SparseTensor` using `tf.io.SparseFeature`, which uses three separate feature keys to store the `indices`, `values`, and `dense_shape`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pir2Xt3nSe-4"
+      },
+      "source": [
+        "### `tf.function`\n",
+        "\n",
+        "The `tf.function` decorator precomputes TensorFlow graphs for Python functions, which can substantially improve the performance of your TensorFlow code. Sparse tensors work transparently with both `tf.function` and [concrete functions](https://www.tensorflow.org/guide/function#obtaining_concrete_functions)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6jXDueTOSeYO"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def f(x,y):\n",
+        "  return tf.sparse.sparse_dense_matmul(x,y)\n",
+        "\n",
+        "a = tf.sparse.SparseTensor(indices=[[0, 3], [2, 4]],\n",
+        "                    values=[15, 25],\n",
+        "                    dense_shape=[3, 10])\n",
+        "\n",
+        "b = tf.sparse.to_dense(tf.sparse.transpose(a))\n",
+        "\n",
+        "c = f(a,b)\n",
+        "\n",
+        "print(c)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YPe5uC_X7XjZ"
+      },
+      "source": [
+        "## Distinguishing missing values from zero values\n",
+        "\n",
+        "Most ops on `tf.sparse.SparseTensor`s treat missing values and explicit zero values identically. This is by design — a `tf.sparse.SparseTensor` is supposed to act just like a dense tensor.\n",
+        "\n",
+        "However, there are a few cases where it can be useful to distinguish zero values from missing values. In particular, this allows for one way to encode missing/unknown data in your training data. For example, consider a use case where you have a tensor of scores (that can have any floating point value from -Inf to +Inf), with some missing scores. You can encode this tensor using a sparse tensor where the explicit zeros are known zero scores but the implicit zero values actually represent missing data and not zero. \n",
+        "\n",
+        "Note: This is generally not the intended usage of `tf.sparse.SparseTensor`s; and you might want to also consider other techniques for encoding this such as for example using a separate mask tensor that identifies the locations of known/unknown values. However, exercise caution while using this approach, since most sparse operations will treat explicit and implicit zero values identically."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tZ17F9e3ZJDS"
+      },
+      "source": [
+        "Note that some ops like `tf.sparse.reduce_max` do not treat missing values as if they were zero. For example, when you run the code block below, the expected output is `0`. However, because of this exception, the output is `-3`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kcNBVVtBZav_"
+      },
+      "outputs": [],
+      "source": [
+        "print(tf.sparse.reduce_max(tf.sparse.from_dense([-5, 0, -3])))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zhzWLW-bMfI5"
+      },
+      "source": [
+        "In contrast, when you apply `tf.math.reduce_max` to a dense tensor, the output is 0 as expected."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7Xy-g3VDNK9d"
+      },
+      "outputs": [],
+      "source": [
+        "print(tf.math.reduce_max([-5, 0, -3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uK3U8l0kNL37"
+      },
+      "source": [
+        "## Further reading and resources\n",
+        "\n",
+        "* Refer to the [tensor guide](https://www.tensorflow.org/guide/tensor) to learn about tensors.\n",
+        "* Read the [ragged tensor guide](https://www.tensorflow.org/guide/ragged_tensor) to learn how to work with ragged tensors, a type of tensor that lets you work with non-uniform data.\n",
+        "*  Check out this object detection model in the [TensorFlow Model Garden](https://github.com/tensorflow/models) that uses sparse tensors in a [`tf.Example` data decoder](https://github.com/tensorflow/models/blob/9139a7b90112562aec1d7e328593681bd410e1e7/research/object_detection/data_decoders/tf_example_decoder.py).\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "sparse_tensor.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/tensor.ipynb b/site/en/guide/tensor.ipynb
new file mode 100644
index 00000000000..2eb261aad75
--- /dev/null
+++ b/site/en/guide/tensor.ipynb
@@ -0,0 +1,1576 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qFdPvlXBOdUN"
+      },
+      "source": [
+        "# Introduction to Tensors"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AL2hzxorJiWy"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import numpy as np"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VQ3s2J8Vgowq"
+      },
+      "source": [
+        "Tensors are multi-dimensional arrays with a uniform type (called a `dtype`).  You can see all supported `dtypes` at `tf.dtypes`.\n",
+        "\n",
+        "If you're familiar with [NumPy](https://numpy.org/devdocs/user/quickstart.html), tensors are (kind of) like `np.arrays`.\n",
+        "\n",
+        "All tensors are immutable like Python numbers and strings: you can never update the contents of a tensor, only create a new one.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DRK5-9EpYbzG"
+      },
+      "source": [
+        "## Basics\n",
+        "\n",
+        "First, create some basic tensors."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uSHRFT6LJbxq"
+      },
+      "source": [
+        "Here is a \"scalar\" or \"rank-0\" tensor . A scalar contains a single value, and no \"axes\"."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d5JcgLFR6gHv"
+      },
+      "outputs": [],
+      "source": [
+        "# This will be an int32 tensor by default; see \"dtypes\" below.\n",
+        "rank_0_tensor = tf.constant(4)\n",
+        "print(rank_0_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tdmPAn9fWYs5"
+      },
+      "source": [
+        "A \"vector\" or \"rank-1\" tensor is like a list of values. A vector has one axis:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oZos8o_R6oE7"
+      },
+      "outputs": [],
+      "source": [
+        "# Let's make this a float tensor.\n",
+        "rank_1_tensor = tf.constant([2.0, 3.0, 4.0])\n",
+        "print(rank_1_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G3IJG-ug_H4u"
+      },
+      "source": [
+        "A \"matrix\" or \"rank-2\" tensor has two axes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cnOIA_xb6u0M"
+      },
+      "outputs": [],
+      "source": [
+        "# If you want to be specific, you can set the dtype (see below) at creation time\n",
+        "rank_2_tensor = tf.constant([[1, 2],\n",
+        "                             [3, 4],\n",
+        "                             [5, 6]], dtype=tf.float16)\n",
+        "print(rank_2_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "19m72qEPkfxi"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "  \n",
+        "  \n",
+        "\n",
+        "
A scalar, shape: []A vector, shape: [3]A matrix, shape: [3, 2]
\n",
+        "   \"A\n",
+        "  \n",
+        "   \"The\n",
+        "  \n",
+        "   \"A\n",
+        "  
\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fjFvzcn4_ehD"
+      },
+      "source": [
+        "Tensors may have more axes; here is a tensor with three axes:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sesW7gw6JkXy"
+      },
+      "outputs": [],
+      "source": [
+        "# There can be an arbitrary number of\n",
+        "# axes (sometimes called \"dimensions\")\n",
+        "rank_3_tensor = tf.constant([\n",
+        "  [[0, 1, 2, 3, 4],\n",
+        "   [5, 6, 7, 8, 9]],\n",
+        "  [[10, 11, 12, 13, 14],\n",
+        "   [15, 16, 17, 18, 19]],\n",
+        "  [[20, 21, 22, 23, 24],\n",
+        "   [25, 26, 27, 28, 29]],])\n",
+        "\n",
+        "print(rank_3_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rM2sTGIkoE3S"
+      },
+      "source": [
+        "There are many ways you might visualize a tensor with more than two axes."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NFiYfNMMhDgL"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "\n",
+        "  \n",
+        "\n",
+        "\n",
+        "
A 3-axis tensor, shape: [3, 2, 5]
\n",
+        "   \n",
+        "  \n",
+        "   \n",
+        "  \n",
+        "   \n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oWAc0U8OZwNb"
+      },
+      "source": [
+        "You can convert a tensor to a NumPy array either using `np.array` or the `tensor.numpy` method:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "J5u6_6ZYaS7B"
+      },
+      "outputs": [],
+      "source": [
+        "np.array(rank_2_tensor)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c6Taz2gIaZeo"
+      },
+      "outputs": [],
+      "source": [
+        "rank_2_tensor.numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hnz19F0ocEKD"
+      },
+      "source": [
+        "Tensors often contain floats and ints, but have many other types, including:\n",
+        "\n",
+        "* complex numbers\n",
+        "* strings\n",
+        "\n",
+        "The base `tf.Tensor` class requires tensors to be \"rectangular\"---that is, along each axis, every element is the same size.  However, there are specialized types of tensors that can handle different shapes:\n",
+        "\n",
+        "* Ragged tensors (see [RaggedTensor](#ragged_tensors) below)\n",
+        "* Sparse tensors (see [SparseTensor](#sparse_tensors) below)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SDC7OGeAIJr8"
+      },
+      "source": [
+        "You can do basic math on tensors, including addition, element-wise multiplication, and matrix multiplication."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-DTkjwDOIIDa"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.constant([[1, 2],\n",
+        "                 [3, 4]])\n",
+        "b = tf.constant([[1, 1],\n",
+        "                 [1, 1]]) # Could have also said `tf.ones([2,2], dtype=tf.int32)`\n",
+        "\n",
+        "print(tf.add(a, b), \"\\n\")\n",
+        "print(tf.multiply(a, b), \"\\n\")\n",
+        "print(tf.matmul(a, b), \"\\n\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2smoWeUz-N2q"
+      },
+      "outputs": [],
+      "source": [
+        "print(a + b, \"\\n\") # element-wise addition\n",
+        "print(a * b, \"\\n\") # element-wise multiplication\n",
+        "print(a @ b, \"\\n\") # matrix multiplication"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "S3_vIAl2JPVc"
+      },
+      "source": [
+        "Tensors are used in all kinds of operations (or \"Ops\")."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Gp4WUYzGIbnv"
+      },
+      "outputs": [],
+      "source": [
+        "c = tf.constant([[4.0, 5.0], [10.0, 1.0]])\n",
+        "\n",
+        "# Find the largest value\n",
+        "print(tf.reduce_max(c))\n",
+        "# Find the index of the largest value\n",
+        "print(tf.math.argmax(c))\n",
+        "# Compute the softmax\n",
+        "print(tf.nn.softmax(c))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0MNM-q7-MZLz"
+      },
+      "source": [
+        "Note: Typically, anywhere a TensorFlow function expects a `Tensor` as input, the function will also accept anything that can be converted to a `Tensor` using `tf.convert_to_tensor`. See below for an example."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_wch0N8xNEt-"
+      },
+      "outputs": [],
+      "source": [
+        "tf.convert_to_tensor([1,2,3])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ngqIeWYeNJVI"
+      },
+      "outputs": [],
+      "source": [
+        "tf.reduce_max([1,2,3])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ThVMxqbVNOq3"
+      },
+      "outputs": [],
+      "source": [
+        "tf.reduce_max(np.array([1,2,3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NvSAbowVVuRr"
+      },
+      "source": [
+        "## About shapes"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hkaBIqkTCcGY"
+      },
+      "source": [
+        "Tensors have shapes.  Some vocabulary:\n",
+        "\n",
+        "* **Shape**: The length (number of elements) of each of the axes of a tensor.\n",
+        "* **Rank**: Number of tensor axes.  A scalar has rank 0, a vector has rank 1, a matrix is rank 2.\n",
+        "* **Axis** or **Dimension**: A particular dimension of a tensor.\n",
+        "* **Size**: The total number of items in the tensor, the product of the shape vector's elements.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "E9L3-kCQq2f6"
+      },
+      "source": [
+        "Note: Although you may see reference to a \"tensor of two dimensions\", a rank-2 tensor does not usually describe a 2D space."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VFOyG2tn8LhW"
+      },
+      "source": [
+        "Tensors and `tf.TensorShape` objects have convenient properties for accessing these:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RyD3yewUKdnK"
+      },
+      "outputs": [],
+      "source": [
+        "rank_4_tensor = tf.zeros([3, 2, 4, 5])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oTZZW9ziq4og"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "
A rank-4 tensor, shape: [3, 2, 4, 5]
\n",
+        "\"A\n",
+        "    \n",
+        "\"A\n",
+        "  
\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MHm9vSqogsBk"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Type of every element:\", rank_4_tensor.dtype)\n",
+        "print(\"Number of axes:\", rank_4_tensor.ndim)\n",
+        "print(\"Shape of tensor:\", rank_4_tensor.shape)\n",
+        "print(\"Elements along axis 0 of tensor:\", rank_4_tensor.shape[0])\n",
+        "print(\"Elements along the last axis of tensor:\", rank_4_tensor.shape[-1])\n",
+        "print(\"Total number of elements (3*2*4*5): \", tf.size(rank_4_tensor).numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2ZGZp_JOOPOv"
+      },
+      "source": [
+        "But note that the `Tensor.ndim` and `Tensor.shape` attributes don't return `Tensor` objects. If you need a `Tensor` use the `tf.rank` or `tf.shape` function. This difference is subtle, but it can be important when building graphs (later)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ptq0-y6APCpD"
+      },
+      "outputs": [],
+      "source": [
+        "tf.rank(rank_4_tensor)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HslrDOEBPICN"
+      },
+      "outputs": [],
+      "source": [
+        "tf.shape(rank_4_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bQmE_Vx5JilS"
+      },
+      "source": [
+        "While axes are often referred to by their indices, you should always keep track of the meaning of each. Often axes are ordered from global to local: The batch axis first, followed by spatial dimensions, and features for each location last. This way feature vectors are contiguous regions of memory.\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "    \n",
+        "\n",
+        "
Typical axis order
\n",
+        "\"Keep\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FlPoVvJS75Bb"
+      },
+      "source": [
+        "## Indexing"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "apOkCKqCZIZu"
+      },
+      "source": [
+        "### Single-axis indexing\n",
+        "\n",
+        "TensorFlow follows standard Python indexing rules, similar to [indexing a list or a string in Python](https://docs.python.org/3/tutorial/introduction.html#strings), and the basic rules for NumPy indexing.\n",
+        "\n",
+        "* indexes start at `0`\n",
+        "* negative indices count backwards from the end\n",
+        "* colons, `:`, are used for slices: `start:stop:step`\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SQ-CrJxLXTIM"
+      },
+      "outputs": [],
+      "source": [
+        "rank_1_tensor = tf.constant([0, 1, 1, 2, 3, 5, 8, 13, 21, 34])\n",
+        "print(rank_1_tensor.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mQYYL56PXSak"
+      },
+      "source": [
+        "Indexing with a scalar removes the axis:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "n6tqHciOWMt5"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"First:\", rank_1_tensor[0].numpy())\n",
+        "print(\"Second:\", rank_1_tensor[1].numpy())\n",
+        "print(\"Last:\", rank_1_tensor[-1].numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qJLHU_a2XwpG"
+      },
+      "source": [
+        "Indexing with a `:` slice keeps the axis:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "giVPPcfQX-cu"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Everything:\", rank_1_tensor[:].numpy())\n",
+        "print(\"Before 4:\", rank_1_tensor[:4].numpy())\n",
+        "print(\"From 4 to the end:\", rank_1_tensor[4:].numpy())\n",
+        "print(\"From 2, before 7:\", rank_1_tensor[2:7].numpy())\n",
+        "print(\"Every other item:\", rank_1_tensor[::2].numpy())\n",
+        "print(\"Reversed:\", rank_1_tensor[::-1].numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "elDSxXi7X-Bh"
+      },
+      "source": [
+        "### Multi-axis indexing"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Cgk0uRUYZiai"
+      },
+      "source": [
+        "Higher rank tensors are indexed by passing multiple indices.\n",
+        "\n",
+        "The exact same rules as in the single-axis case apply to each axis independently."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Tc5X_WlsZXmd"
+      },
+      "outputs": [],
+      "source": [
+        "print(rank_2_tensor.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "w07U9vq5ipQk"
+      },
+      "source": [
+        "Passing an integer for each index, the result is a scalar."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PvILXc1PjqTM"
+      },
+      "outputs": [],
+      "source": [
+        "# Pull out a single value from a 2-rank tensor\n",
+        "print(rank_2_tensor[1, 1].numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3RLCzAOHjfEH"
+      },
+      "source": [
+        "You can index using any combination of integers and slices:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YTqNqsfJkJP_"
+      },
+      "outputs": [],
+      "source": [
+        "# Get row and column tensors\n",
+        "print(\"Second row:\", rank_2_tensor[1, :].numpy())\n",
+        "print(\"Second column:\", rank_2_tensor[:, 1].numpy())\n",
+        "print(\"Last row:\", rank_2_tensor[-1, :].numpy())\n",
+        "print(\"First item in last column:\", rank_2_tensor[0, -1].numpy())\n",
+        "print(\"Skip the first row:\")\n",
+        "print(rank_2_tensor[1:, :].numpy(), \"\\n\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "P45TwSUVSK6G"
+      },
+      "source": [
+        "Here is an example with a 3-axis tensor:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GuLoMoCVSLxK"
+      },
+      "outputs": [],
+      "source": [
+        "print(rank_3_tensor[:, :, 4])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9NgmHq27TJOE"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "\n",
+        "    \n",
+        "      \n",
+        "\n",
+        "
Selecting the last feature across all locations in each example in the batch 
\n",
+        "\"A\n",
+        "  \n",
+        "\"The\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "t9V83-thHn89"
+      },
+      "source": [
+        "Read the [tensor slicing guide](https://tensorflow.org/guide/tensor_slicing) to learn how you can apply indexing to manipulate individual elements in your tensors."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fpr7R0t4SVb0"
+      },
+      "source": [
+        "## Manipulating Shapes\n",
+        "\n",
+        "Reshaping a tensor is of great utility. \n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EMeTtga5Wq8j"
+      },
+      "outputs": [],
+      "source": [
+        "# Shape returns a `TensorShape` object that shows the size along each axis\n",
+        "x = tf.constant([[1], [2], [3]])\n",
+        "print(x.shape)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "38jc2RXziT3W"
+      },
+      "outputs": [],
+      "source": [
+        "# You can convert this object into a Python list, too\n",
+        "print(x.shape.as_list())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J_xRlHZMKYnF"
+      },
+      "source": [
+        "You can reshape a tensor into a new shape. The `tf.reshape` operation is fast and cheap as the underlying data does not need to be duplicated."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pa9JCgMLWy87"
+      },
+      "outputs": [],
+      "source": [
+        "# You can reshape a tensor to a new shape.\n",
+        "# Note that you're passing in a list\n",
+        "reshaped = tf.reshape(x, [1, 3])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Mcq7iXOkW3LK"
+      },
+      "outputs": [],
+      "source": [
+        "print(x.shape)\n",
+        "print(reshaped.shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gIB2tOkoVr6E"
+      },
+      "source": [
+        "The data maintains its layout in memory and a new tensor is created, with the requested shape, pointing to the same data. TensorFlow uses C-style \"row-major\" memory ordering, where incrementing the rightmost index corresponds to a single step in memory."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7kMfM0RpUgI8"
+      },
+      "outputs": [],
+      "source": [
+        "print(rank_3_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TcDtfQkJWzIx"
+      },
+      "source": [
+        "If you flatten a tensor you can see what order it is laid out in memory."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "COnHEPuaWDQp"
+      },
+      "outputs": [],
+      "source": [
+        "# A `-1` passed in the `shape` argument says \"Whatever fits\".\n",
+        "print(tf.reshape(rank_3_tensor, [-1]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jJZRira2W--c"
+      },
+      "source": [
+        "Typically the only reasonable use of `tf.reshape` is to combine or split adjacent axes (or add/remove `1`s).\n",
+        "\n",
+        "For this 3x2x5 tensor, reshaping to (3x2)x5 or 3x(2x5) are both reasonable things to do, as the slices do not mix:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zP2Iqc7zWu_J"
+      },
+      "outputs": [],
+      "source": [
+        "print(tf.reshape(rank_3_tensor, [3*2, 5]), \"\\n\")\n",
+        "print(tf.reshape(rank_3_tensor, [3, -1]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6ZsZRUhihlDB"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "\n",
+        "
\n",
+        "Some good reshapes.\n",
+        "
\n",
+        "\"A\n",
+        "  \n",
+        "  \"The\n",
+        "  \n",
+        "\"The\n",
+        "  
\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nOcRxDC3jNIU"
+      },
+      "source": [
+        "Reshaping will \"work\" for any new shape with the same total number of elements, but it will not do anything useful if you do not respect the order of the axes.\n",
+        "\n",
+        "Swapping axes in `tf.reshape` does not work; you need `tf.transpose` for that. \n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "I9qDL_8u7cBH"
+      },
+      "outputs": [],
+      "source": [
+        "# Bad examples: don't do this\n",
+        "\n",
+        "# You can't reorder axes with reshape.\n",
+        "print(tf.reshape(rank_3_tensor, [2, 3, 5]), \"\\n\") \n",
+        "\n",
+        "# This is a mess\n",
+        "print(tf.reshape(rank_3_tensor, [5, 6]), \"\\n\")\n",
+        "\n",
+        "# This doesn't work at all\n",
+        "try:\n",
+        "  tf.reshape(rank_3_tensor, [7, -1])\n",
+        "except Exception as e:\n",
+        "  print(f\"{type(e).__name__}: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qTM9-5eh68oo"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "\n",
+        "
\n",
+        "Some bad reshapes.\n",
+        "
\n",
+        "\"You\n",
+        "  \n",
+        "\"Anything\n",
+        "  \n",
+        "\"The\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N9r90BvHCbTt"
+      },
+      "source": [
+        "You may run across not-fully-specified shapes. Either the shape contains a `None` (an axis-length is unknown) or the whole shape is `None` (the rank of the tensor is unknown).\n",
+        "\n",
+        "Except for [tf.RaggedTensor](#ragged_tensors), such shapes will only occur in the context of TensorFlow's symbolic, graph-building  APIs:\n",
+        "\n",
+        "* [tf.function](function.ipynb) \n",
+        "* The [keras functional API](https://www.tensorflow.org/guide/keras/functional).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fDmFtFM7k0R2"
+      },
+      "source": [
+        "## More on `DTypes`\n",
+        "\n",
+        "To inspect a `tf.Tensor`'s data type use the `Tensor.dtype` property.\n",
+        "\n",
+        "When creating a `tf.Tensor` from a Python object you may optionally specify the datatype.\n",
+        "\n",
+        "If you don't, TensorFlow chooses a datatype that can represent your data. TensorFlow converts Python integers to `tf.int32` and Python floating point numbers to `tf.float32`. Otherwise TensorFlow uses the same rules NumPy uses when converting to arrays.\n",
+        "\n",
+        "You can cast from type to type."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5mSTDWbelUvu"
+      },
+      "outputs": [],
+      "source": [
+        "the_f64_tensor = tf.constant([2.2, 3.3, 4.4], dtype=tf.float64)\n",
+        "the_f16_tensor = tf.cast(the_f64_tensor, dtype=tf.float16)\n",
+        "# Now, cast to an uint8 and lose the decimal precision\n",
+        "the_u8_tensor = tf.cast(the_f16_tensor, dtype=tf.uint8)\n",
+        "print(the_u8_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s1yBlJsVlFSu"
+      },
+      "source": [
+        "## Broadcasting\n",
+        "\n",
+        "Broadcasting is a concept borrowed from the [equivalent feature in NumPy](https://numpy.org/doc/stable/user/basics.broadcasting.html).  In short, under certain conditions, smaller tensors are \"stretched\" automatically to fit larger tensors when running combined operations on them.\n",
+        "\n",
+        "The simplest and most common case is when you attempt to multiply or add a tensor to a scalar.  In that case, the scalar is broadcast to be the same shape as the other argument. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "P8sypqmagHQN"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant([1, 2, 3])\n",
+        "\n",
+        "y = tf.constant(2)\n",
+        "z = tf.constant([2, 2, 2])\n",
+        "# All of these are the same computation\n",
+        "print(tf.multiply(x, 2))\n",
+        "print(x * y)\n",
+        "print(x * z)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o0SBoR6voWcb"
+      },
+      "source": [
+        "Likewise, axes with length 1 can be stretched out to match the other arguments.  Both arguments can be stretched in the same computation.\n",
+        "\n",
+        "In this case a 3x1 matrix is element-wise multiplied by a 1x4 matrix to produce a 3x4 matrix. Note how the leading 1 is optional: The shape of y is `[4]`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6sGmkPg3XANr"
+      },
+      "outputs": [],
+      "source": [
+        "# These are the same computations\n",
+        "x = tf.reshape(x,[3,1])\n",
+        "y = tf.range(1, 5)\n",
+        "print(x, \"\\n\")\n",
+        "print(y, \"\\n\")\n",
+        "print(tf.multiply(x, y))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "t_7sh-EUYLrE"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "
A broadcasted add: a [3, 1] times a [1, 4] gives a [3,4] 
\n",
+        "\"Adding\n",
+        "  
\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9V3KgSJcKDRz"
+      },
+      "source": [
+        "Here is the same operation without broadcasting:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "elrF6v63igY8"
+      },
+      "outputs": [],
+      "source": [
+        "x_stretch = tf.constant([[1, 1, 1, 1],\n",
+        "                         [2, 2, 2, 2],\n",
+        "                         [3, 3, 3, 3]])\n",
+        "\n",
+        "y_stretch = tf.constant([[1, 2, 3, 4],\n",
+        "                         [1, 2, 3, 4],\n",
+        "                         [1, 2, 3, 4]])\n",
+        "\n",
+        "print(x_stretch * y_stretch)  # Again, operator overloading"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "14KobqYu85gi"
+      },
+      "source": [
+        "Most of the time, broadcasting is both time and space efficient, as the broadcast operation never materializes the expanded tensors in memory.  \n",
+        "\n",
+        "You see what broadcasting looks like using `tf.broadcast_to`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GW2Q59_r8hZ6"
+      },
+      "outputs": [],
+      "source": [
+        "print(tf.broadcast_to(tf.constant([1, 2, 3]), [3, 3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Z2bAMMQY-jpP"
+      },
+      "source": [
+        "Unlike a mathematical op, for example, `broadcast_to` does nothing special to save memory.  Here, you are materializing the tensor.\n",
+        "\n",
+        "It can get even more complicated.  [This section](https://jakevdp.github.io/PythonDataScienceHandbook/02.05-computation-on-arrays-broadcasting.html) of Jake VanderPlas's book _Python Data Science Handbook_ shows more broadcasting tricks (again in NumPy)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o4Rpz0xAsKSI"
+      },
+      "source": [
+        "## tf.convert_to_tensor\n",
+        "\n",
+        "Most ops, like `tf.matmul` and `tf.reshape` take arguments of class `tf.Tensor`.  However, you'll notice in the above case, Python objects shaped like tensors are accepted.\n",
+        "\n",
+        "Most, but not all, ops call `convert_to_tensor` on non-tensor arguments.  There is a registry of conversions, and most object classes like NumPy's `ndarray`, `TensorShape`, Python lists, and `tf.Variable` will all convert automatically.\n",
+        "\n",
+        "See `tf.register_tensor_conversion_function` for more details, and if you have your own type you'd like to automatically convert to a tensor."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "05bBVBVYV0y6"
+      },
+      "source": [
+        "## Ragged Tensors\n",
+        "\n",
+        "A tensor with variable numbers of elements along some axis is called \"ragged\". Use `tf.ragged.RaggedTensor` for ragged data.\n",
+        "\n",
+        "For example, This cannot be represented as a regular tensor:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VPc3jGoeJqB7"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "
A `tf.RaggedTensor`, shape: [4, None]
\n",
+        "\"A\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VsbTjoFfNVBF"
+      },
+      "outputs": [],
+      "source": [
+        "ragged_list = [\n",
+        "    [0, 1, 2, 3],\n",
+        "    [4, 5],\n",
+        "    [6, 7, 8],\n",
+        "    [9]]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p4xKTo57tutG"
+      },
+      "outputs": [],
+      "source": [
+        "try:\n",
+        "  tensor = tf.constant(ragged_list)\n",
+        "except Exception as e:\n",
+        "  print(f\"{type(e).__name__}: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0cm9KuEeMLGI"
+      },
+      "source": [
+        "Instead create a `tf.RaggedTensor` using `tf.ragged.constant`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XhF3QV3jiqTj"
+      },
+      "outputs": [],
+      "source": [
+        "ragged_tensor = tf.ragged.constant(ragged_list)\n",
+        "print(ragged_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sFgHduHVNoIE"
+      },
+      "source": [
+        "The shape of a `tf.RaggedTensor` will contain some axes with unknown lengths:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Eo_3wJUWNgqB"
+      },
+      "outputs": [],
+      "source": [
+        "print(ragged_tensor.shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "V9njclVkkN7G"
+      },
+      "source": [
+        "## String tensors\n",
+        "\n",
+        "`tf.string` is a `dtype`, which is to say you can represent data as strings (variable-length byte arrays) in tensors.\n",
+        "\n",
+        "The strings are atomic and cannot be indexed the way Python strings are. The length of the string is not one of the axes of the tensor. See `tf.strings` for functions to manipulate them."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5P_8spEGQ0wp"
+      },
+      "source": [
+        "Here is a scalar string tensor:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sBosmM8MkIh4"
+      },
+      "outputs": [],
+      "source": [
+        "# Tensors can be strings, too here is a scalar string.\n",
+        "scalar_string_tensor = tf.constant(\"Gray wolf\")\n",
+        "print(scalar_string_tensor)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CMFBSl1FQ3vE"
+      },
+      "source": [
+        "And a vector of strings:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IO-c3Tq3RC1L"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "
A vector of strings, shape: [3,]
\n",
+        "\"The\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "41Dv2kL9QrtO"
+      },
+      "outputs": [],
+      "source": [
+        "# If you have three string tensors of different lengths, this is OK.\n",
+        "tensor_of_strings = tf.constant([\"Gray wolf\",\n",
+        "                                 \"Quick brown fox\",\n",
+        "                                 \"Lazy dog\"])\n",
+        "# Note that the shape is (3,). The string length is not included.\n",
+        "print(tensor_of_strings)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "76gQ9qrgSMzS"
+      },
+      "source": [
+        "In the above printout the `b` prefix indicates that `tf.string` dtype is not a unicode string, but a byte-string. See the [Unicode Tutorial](https://www.tensorflow.org/tutorials/load_data/unicode) for more about working with unicode text in TensorFlow."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ClSBPK-lZBQp"
+      },
+      "source": [
+        "If you pass unicode characters they are utf-8 encoded."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GTgL53jxSMd9"
+      },
+      "outputs": [],
+      "source": [
+        "tf.constant(\"🥳👍\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ir9cY42MMAei"
+      },
+      "source": [
+        "Some basic functions with strings can be found in `tf.strings`, including `tf.strings.split`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8k2K0VTFyj8e"
+      },
+      "outputs": [],
+      "source": [
+        "# You can use split to split a string into a set of tensors\n",
+        "print(tf.strings.split(scalar_string_tensor, sep=\" \"))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zgGAn1dfR-04"
+      },
+      "outputs": [],
+      "source": [
+        "# ...but it turns into a `RaggedTensor` if you split up a tensor of strings,\n",
+        "# as each string might be split into a different number of parts.\n",
+        "print(tf.strings.split(tensor_of_strings))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HsAn1kPeO84m"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "
Three strings split, shape: [3, None]
\n",
+        "\"Splitting\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "st9OxrUxWSKY"
+      },
+      "source": [
+        "And `tf.strings.to_number`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3nRtx3X9WRfN"
+      },
+      "outputs": [],
+      "source": [
+        "text = tf.constant(\"1 10 100\")\n",
+        "print(tf.strings.to_number(tf.strings.split(text, \" \")))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "r2EZtBbJBns4"
+      },
+      "source": [
+        "Although you can't use `tf.cast` to turn a string tensor into numbers, you can convert it into bytes, and then into numbers."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fo8BjmH7gyTj"
+      },
+      "outputs": [],
+      "source": [
+        "byte_strings = tf.strings.bytes_split(tf.constant(\"Duck\"))\n",
+        "byte_ints = tf.io.decode_raw(tf.constant(\"Duck\"), tf.uint8)\n",
+        "print(\"Byte strings:\", byte_strings)\n",
+        "print(\"Bytes:\", byte_ints)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uSQnZ7d1jCSQ"
+      },
+      "outputs": [],
+      "source": [
+        "# Or split it up as unicode and then decode it\n",
+        "unicode_bytes = tf.constant(\"アヒル 🦆\")\n",
+        "unicode_char_bytes = tf.strings.unicode_split(unicode_bytes, \"UTF-8\")\n",
+        "unicode_values = tf.strings.unicode_decode(unicode_bytes, \"UTF-8\")\n",
+        "\n",
+        "print(\"\\nUnicode bytes:\", unicode_bytes)\n",
+        "print(\"\\nUnicode chars:\", unicode_char_bytes)\n",
+        "print(\"\\nUnicode values:\", unicode_values)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fE7nKJ2YW3aY"
+      },
+      "source": [
+        "The `tf.string` dtype is used for all raw bytes data in TensorFlow. The `tf.io` module contains functions for converting data to and from bytes, including decoding images and parsing csv."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ua8BnAzxkRKV"
+      },
+      "source": [
+        "## Sparse tensors\n",
+        "\n",
+        "Sometimes, your data is sparse, like a very wide embedding space.  TensorFlow supports `tf.sparse.SparseTensor` and related operations to store sparse data efficiently."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mS5zgqgUTPRb"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "\n",
+        "  \n",
+        "\n",
+        "
A `tf.SparseTensor`, shape: [3, 4]
\n",
+        "\"An\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B9nbO1E2kSUN"
+      },
+      "outputs": [],
+      "source": [
+        "# Sparse tensors store values by index in a memory-efficient manner\n",
+        "sparse_tensor = tf.sparse.SparseTensor(indices=[[0, 0], [1, 2]],\n",
+        "                                       values=[1, 2],\n",
+        "                                       dense_shape=[3, 4])\n",
+        "print(sparse_tensor, \"\\n\")\n",
+        "\n",
+        "# You can convert sparse tensors to dense\n",
+        "print(tf.sparse.to_dense(sparse_tensor))"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "Tce3stUlHN0L"
+      ],
+      "name": "tensor.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/tensor.md b/site/en/guide/tensor.md
deleted file mode 100644
index ff78a0ba90e..00000000000
--- a/site/en/guide/tensor.md
+++ /dev/null
@@ -1,296 +0,0 @@
-# TensorFlow tensors
-
-TensorFlow, as the name indicates, is a framework to define and run computations
-involving tensors. A *tensor* is a generalization of vectors and matrices to
-potentially higher dimensions. Internally, TensorFlow represents tensors as
-n-dimensional arrays of base datatypes.
-
-When writing a TensorFlow program, the main object you manipulate and pass
-around is the `tf.Tensor`. A `tf.Tensor` object represents a partially defined
-computation that will eventually produce a value. TensorFlow programs work by
-first building a graph of `tf.Tensor` objects, detailing how each tensor is
-computed based on the other available tensors and then by running parts of this
-graph to achieve the desired results.
-
-A `tf.Tensor` has the following properties:
-
- * a data type (`float32`, `int32`, or `string`, for example)
- * a shape
-
-
-Each element in the Tensor has the same data type, and the data type is always
-known. The shape (that is, the number of dimensions it has and the size of each
-dimension) might be only partially known. Most operations produce tensors of
-fully-known shapes if the shapes of their inputs are also fully known, but in
-some cases it's only possible to find the shape of a tensor at graph execution
-time.
-
-Some types of tensors are special, and these will be covered in other
-units of the TensorFlow guide. The main ones are:
-
-  * `tf.Variable`
-  * `tf.constant`
-  * `tf.placeholder`
-  * `tf.SparseTensor`
-
-With the exception of `tf.Variable`, the value of a tensor is immutable, which
-means that in the context of a single execution tensors only have a single
-value. However, evaluating the same tensor twice can return different values;
-for example that tensor can be the result of reading data from disk, or
-generating a random number.
-
-## Rank
-
-The **rank** of a `tf.Tensor` object is its number of dimensions. Synonyms for
-rank include **order** or **degree** or **n-dimension**.
-Note that rank in TensorFlow is not the same as matrix rank in mathematics.
-As the following table shows, each rank in TensorFlow corresponds to a
-different mathematical entity:
-
-Rank | Math entity
---- | ---
-0 | Scalar (magnitude only)
-1 | Vector (magnitude and direction)
-2 | Matrix (table of numbers)
-3 | 3-Tensor (cube of numbers)
-n | n-Tensor (you get the idea)
-
-
-### Rank 0
-
-The following snippet demonstrates creating a few rank 0 variables:
-
-```python
-mammal = tf.Variable("Elephant", tf.string)
-ignition = tf.Variable(451, tf.int16)
-floating = tf.Variable(3.14159265359, tf.float64)
-its_complicated = tf.Variable(12.3 - 4.85j, tf.complex64)
-```
-
-Note: A string is treated as a single object in TensorFlow, not as a sequence of
-characters. It is possible to have scalar strings, vectors of strings, etc.
-
-### Rank 1
-
-To create a rank 1 `tf.Tensor` object, you can pass a list of items as the
-initial value. For example:
-
-```python
-mystr = tf.Variable(["Hello"], tf.string)
-cool_numbers  = tf.Variable([3.14159, 2.71828], tf.float32)
-first_primes = tf.Variable([2, 3, 5, 7, 11], tf.int32)
-its_very_complicated = tf.Variable([12.3 - 4.85j, 7.5 - 6.23j], tf.complex64)
-```
-
-
-### Higher ranks
-
-A rank 2 `tf.Tensor` object consists of at least one row and at least
-one column:
-
-```python
-mymat = tf.Variable([[7],[11]], tf.int16)
-myxor = tf.Variable([[False, True],[True, False]], tf.bool)
-linear_squares = tf.Variable([[4], [9], [16], [25]], tf.int32)
-squarish_squares = tf.Variable([ [4, 9], [16, 25] ], tf.int32)
-rank_of_squares = tf.rank(squarish_squares)
-mymatC = tf.Variable([[7],[11]], tf.int32)
-```
-
-Higher-rank Tensors, similarly, consist of an n-dimensional array. For example,
-during image processing, many tensors of rank 4 are used, with dimensions
-corresponding to example-in-batch, image height, image width, and color channel.
-
-``` python
-my_image = tf.zeros([10, 299, 299, 3])  # batch x height x width x color
-```
-
-### Getting a `tf.Tensor` object's rank
-
-To determine the rank of a `tf.Tensor` object, call the `tf.rank` method.
-For example, the following method programmatically determines the rank
-of the `tf.Tensor` defined in the previous section:
-
-```python
-r = tf.rank(my_image)
-# After the graph runs, r will hold the value 4.
-```
-
-### Referring to `tf.Tensor` slices
-
-Since a `tf.Tensor` is an n-dimensional array of cells, to access a single cell
-in a `tf.Tensor` you need to specify n indices.
-
-For a rank 0 tensor (a scalar), no indices are necessary, since it is already a
-single number.
-
-For a rank 1 tensor (a vector), passing a single index allows you to access a
-number:
-
-```python
-my_scalar = my_vector[2]
-```
-
-Note that the index passed inside the `[]` can itself be a scalar `tf.Tensor`, if
-you want to dynamically choose an element from the vector.
-
-For tensors of rank 2 or higher, the situation is more interesting. For a
-`tf.Tensor` of rank 2, passing two numbers returns a scalar, as expected:
-
-
-```python
-my_scalar = my_matrix[1, 2]
-```
-
-
-Passing a single number, however, returns a subvector of a matrix, as follows:
-
-
-```python
-my_row_vector = my_matrix[2]
-my_column_vector = my_matrix[:, 3]
-```
-
-The `:` notation is python slicing syntax for "leave this dimension alone". This
-is useful in higher-rank Tensors, as it allows you to access its subvectors,
-submatrices, and even other subtensors.
-
-
-## Shape
-
-The **shape** of a tensor is the number of elements in each dimension.
-TensorFlow automatically infers shapes during graph construction. These inferred
-shapes might have known or unknown rank. If the rank is known, the sizes of each
-dimension might be known or unknown.
-
-The TensorFlow documentation uses three notational conventions to describe
-tensor dimensionality: rank, shape, and dimension number. The following table
-shows how these relate to one another:
-
-Rank | Shape | Dimension number | Example
---- | --- | --- | ---
-0 | [] | 0-D | A 0-D tensor.  A scalar.
-1 | [D0] | 1-D | A 1-D tensor with shape [5].
-2 | [D0, D1] | 2-D | A 2-D tensor with shape [3, 4].
-3 | [D0, D1, D2] | 3-D | A 3-D tensor with shape [1, 4, 3].
-n | [D0, D1, ... Dn-1] | n-D | A tensor with shape [D0, D1, ... Dn-1].
-
-Shapes can be represented via Python lists / tuples of ints, or with the
-`tf.TensorShape`.
-
-### Getting a `tf.Tensor` object's shape
-
-There are two ways of accessing the shape of a `tf.Tensor`. While building the
-graph, it is often useful to ask what is already known about a tensor's
-shape. This can be done by reading the `shape` property of a `tf.Tensor` object.
-This method returns a `TensorShape` object, which is a convenient way of
-representing partially-specified shapes (since, when building the graph, not all
-shapes will be fully known).
-
-It is also possible to get a `tf.Tensor` that will represent the fully-defined
-shape of another `tf.Tensor` at runtime. This is done by calling the `tf.shape`
-operation. This way, you can build a graph that manipulates the shapes of
-tensors by building other tensors that depend on the dynamic shape of the input
-`tf.Tensor`.
-
-For example, here is how to make a vector of zeros with the same size as the
-number of columns in a given matrix:
-
-``` python
-zeros = tf.zeros(my_matrix.shape[1])
-```
-
-### Changing the shape of a `tf.Tensor`
-
-The **number of elements** of a tensor is the product of the sizes of all its
-shapes. The number of elements of a scalar is always `1`. Since there are often
-many different shapes that have the same number of elements, it's often
-convenient to be able to change the shape of a `tf.Tensor`, keeping its elements
-fixed. This can be done with `tf.reshape`.
-
-The following examples demonstrate how to reshape tensors:
-
-```python
-rank_three_tensor = tf.ones([3, 4, 5])
-matrix = tf.reshape(rank_three_tensor, [6, 10])  # Reshape existing content into
-                                                 # a 6x10 matrix
-matrixB = tf.reshape(matrix, [3, -1])  #  Reshape existing content into a 3x20
-                                       # matrix. -1 tells reshape to calculate
-                                       # the size of this dimension.
-matrixAlt = tf.reshape(matrixB, [4, 3, -1])  # Reshape existing content into a
-                                             #4x3x5 tensor
-
-# Note that the number of elements of the reshaped Tensors has to match the
-# original number of elements. Therefore, the following example generates an
-# error because no possible value for the last dimension will match the number
-# of elements.
-yet_another = tf.reshape(matrixAlt, [13, 2, -1])  # ERROR!
-```
-
-## Data types
-
-In addition to dimensionality, Tensors have a data type. Refer to the
-`tf.DType` page for a complete list of the data types.
-
-It is not possible to have a `tf.Tensor` with more than one data type. It is
-possible, however, to serialize arbitrary data structures as `string`s and store
-those in `tf.Tensor`s.
-
-It is possible to cast `tf.Tensor`s from one datatype to another using
-`tf.cast`:
-
-``` python
-# Cast a constant integer tensor into floating point.
-float_tensor = tf.cast(tf.constant([1, 2, 3]), dtype=tf.float32)
-```
-
-To inspect a `tf.Tensor`'s data type use the `Tensor.dtype` property.
-
-When creating a `tf.Tensor` from a python object you may optionally specify the
-datatype. If you don't, TensorFlow chooses a datatype that can represent your
-data. TensorFlow converts Python integers to `tf.int32` and python floating
-point numbers to `tf.float32`. Otherwise TensorFlow uses the same rules numpy
-uses when converting to arrays.
-
-## Evaluate tensors
-
-Once the computation graph has been built, you can run the computation that
-produces a particular `tf.Tensor` and fetch the value assigned to it. This is
-often useful for debugging as well as being required for much of TensorFlow to
-work.
-
-The simplest way to evaluate a Tensor is using the `Tensor.eval` method. For
-example:
-
-```python
-constant = tf.constant([1, 2, 3])
-tensor = constant * constant
-print(tensor.eval())
-```
-
-The `eval` method only works when a default `tf.Session` is active (see
-[Graphs and Sessions](./graphs.md) for more information).
-
-`Tensor.eval` returns a numpy array with the same contents as the tensor.
-
-Sometimes it is not possible to evaluate a `tf.Tensor` with no context because
-its value might depend on dynamic information that is not available. For
-example, tensors that depend on `placeholder`s can't be evaluated without
-providing a value for the `placeholder`.
-
-``` python
-p = tf.placeholder(tf.float32)
-t = p + 1.0
-t.eval()  # This will fail, since the placeholder did not get a value.
-t.eval(feed_dict={p:2.0})  # This will succeed because we're feeding a value
-                           # to the placeholder.
-```
-
-Note that it is possible to feed any `tf.Tensor`, not just placeholders.
-
-Other model constructs might make evaluating a `tf.Tensor`
-complicated. TensorFlow can't directly evaluate `tf.Tensor`s defined inside
-functions or inside control flow constructs. If a `tf.Tensor` depends on a value
-from a queue, evaluating the `tf.Tensor` will only work once something has been
-enqueued; otherwise, evaluating it will hang. When working with queues, remember
-to call `tf.train.start_queue_runners` before evaluating any `tf.Tensor`s.
diff --git a/site/en/guide/tensor_slicing.ipynb b/site/en/guide/tensor_slicing.ipynb
new file mode 100644
index 00000000000..c5cb2d71356
--- /dev/null
+++ b/site/en/guide/tensor_slicing.ipynb
@@ -0,0 +1,647 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VZ-KA8k5kybx"
+      },
+      "source": [
+        "# Introduction to tensor slicing"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AixIdVeRk3CO"
+      },
+      "source": [
+        "When working on ML applications such as object detection and NLP, it is sometimes necessary to work with sub-sections (slices) of tensors. For example, if your model architecture includes routing, where one layer might control which training example gets routed to the next layer. In this case, you could use tensor slicing ops to split the tensors up and put them back together in the right order.\n",
+        "\n",
+        "In NLP applications, you can use tensor slicing to perform word masking while training. For example, you can generate training data from a list of sentences by choosing a word index to mask in each sentence, taking the word out as a label, and then replacing the chosen word with a mask token.  \n",
+        "\n",
+        "In this guide, you will learn how to use the TensorFlow APIs to:\n",
+        "\n",
+        "* Extract slices from a tensor\n",
+        "* Insert data at specific indices in a tensor\n",
+        "\n",
+        "This guide assumes familiarity with tensor indexing. Read the indexing sections of the [Tensor](https://www.tensorflow.org/guide/tensor#indexing) and [TensorFlow NumPy](https://www.tensorflow.org/guide/tf_numpy#indexing) guides before getting started with this guide."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FcWhWYn7eXkF"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "m6uvewqi0jso"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import numpy as np"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "K-muS4ej5zoN"
+      },
+      "source": [
+        "## Extract tensor slices\n",
+        "\n",
+        "Perform NumPy-like tensor slicing using `tf.slice`.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wZep0cjs0Oai"
+      },
+      "outputs": [],
+      "source": [
+        "t1 = tf.constant([0, 1, 2, 3, 4, 5, 6, 7])\n",
+        "\n",
+        "print(tf.slice(t1,\n",
+        "               begin=[1],\n",
+        "               size=[3]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Vh3xI3j0DRJ2"
+      },
+      "source": [
+        "Alternatively, you can use a more Pythonic syntax. Note that tensor slices are evenly spaced over a start-stop range."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "P1MtEyKuWuDD"
+      },
+      "outputs": [],
+      "source": [
+        "print(t1[1:4])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cjq1o8D2wKKs"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UunuLTIuwDA-"
+      },
+      "outputs": [],
+      "source": [
+        "print(t1[-3:])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EHvRB-XTwRTd"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SW1zFFTnUpCQ"
+      },
+      "source": [
+        "For 2-dimensional tensors,you can use something like:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kThZhmpAVAQw"
+      },
+      "outputs": [],
+      "source": [
+        "t2 = tf.constant([[0, 1, 2, 3, 4],\n",
+        "                  [5, 6, 7, 8, 9],\n",
+        "                  [10, 11, 12, 13, 14],\n",
+        "                  [15, 16, 17, 18, 19]])\n",
+        "\n",
+        "print(t2[:-1, 1:3])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xA5Xt4OdVUui"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iJPggqsH15fI"
+      },
+      "source": [
+        "You can use `tf.slice` on higher dimensional tensors as well."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Re5eX1OXnKOZ"
+      },
+      "outputs": [],
+      "source": [
+        "t3 = tf.constant([[[1, 3, 5, 7],\n",
+        "                   [9, 11, 13, 15]],\n",
+        "                  [[17, 19, 21, 23],\n",
+        "                   [25, 27, 29, 31]]\n",
+        "                  ])\n",
+        "\n",
+        "print(tf.slice(t3,\n",
+        "               begin=[1, 1, 0],\n",
+        "               size=[1, 1, 2]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x-O5FNV9qOJK"
+      },
+      "source": [
+        "You can also use `tf.strided_slice` to extract slices of tensors by 'striding' over the tensor dimensions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b9FhvrOnJsJb"
+      },
+      "source": [
+        "Use `tf.gather` to extract specific indices from a single axis of a tensor."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TwviZrrIj2h7"
+      },
+      "outputs": [],
+      "source": [
+        "print(tf.gather(t1,\n",
+        "                indices=[0, 3, 6]))\n",
+        "\n",
+        "# This is similar to doing\n",
+        "\n",
+        "t1[::3]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oKyjGi2zyzEC"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "obrjeKy1WfTN"
+      },
+      "source": [
+        "`tf.gather` does not require indices to be evenly spaced."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LjJcwcZ0druw"
+      },
+      "outputs": [],
+      "source": [
+        "alphabet = tf.constant(list('abcdefghijklmnopqrstuvwxyz'))\n",
+        "\n",
+        "print(tf.gather(alphabet,\n",
+        "                indices=[2, 0, 19, 18]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mSHmUXIyeaJG"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XsxMx49SOaVu"
+      },
+      "source": [
+        "To extract slices from multiple axes of a tensor, use `tf.gather_nd`. This is useful when you want to gather the elements of a matrix as opposed to just its rows or columns."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mT52NFWVdiTe"
+      },
+      "outputs": [],
+      "source": [
+        "t4 = tf.constant([[0, 5],\n",
+        "                  [1, 6],\n",
+        "                  [2, 7],\n",
+        "                  [3, 8],\n",
+        "                  [4, 9]])\n",
+        "\n",
+        "print(tf.gather_nd(t4,\n",
+        "                   indices=[[2], [3], [0]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "87NN7YQhh2-a"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_z6F2WcPJ9Rh"
+      },
+      "outputs": [],
+      "source": [
+        "t5 = np.reshape(np.arange(18), [2, 3, 3])\n",
+        "\n",
+        "print(tf.gather_nd(t5,\n",
+        "                   indices=[[0, 0, 0], [1, 2, 1]]))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gyIjhm7cV2N0"
+      },
+      "outputs": [],
+      "source": [
+        "# Return a list of two matrices\n",
+        "\n",
+        "print(tf.gather_nd(t5,\n",
+        "                   indices=[[[0, 0], [0, 2]], [[1, 0], [1, 2]]]))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "368D4ciDWB3r"
+      },
+      "outputs": [],
+      "source": [
+        "# Return one matrix\n",
+        "\n",
+        "print(tf.gather_nd(t5,\n",
+        "                   indices=[[0, 0], [0, 2], [1, 0], [1, 2]]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "od51VzS2SSPS"
+      },
+      "source": [
+        "## Insert data into tensors\n",
+        "\n",
+        "Use `tf.scatter_nd` to insert data at specific slices/indices of a tensor. Note that the tensor into which you insert values is zero-initialized."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jlALYLWm1KhN"
+      },
+      "outputs": [],
+      "source": [
+        "t6 = tf.constant([10])\n",
+        "indices = tf.constant([[1], [3], [5], [7], [9]])\n",
+        "data = tf.constant([2, 4, 6, 8, 10])\n",
+        "\n",
+        "print(tf.scatter_nd(indices=indices,\n",
+        "                    updates=data,\n",
+        "                    shape=t6))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CD5vd-kxksW7"
+      },
+      "source": [
+        "Methods like `tf.scatter_nd` which require zero-initialized tensors are similar to sparse tensor initializers. You can use `tf.gather_nd` and `tf.scatter_nd` to mimic the behavior of sparse tensor ops.\n",
+        "\n",
+        "Consider an example where you construct a sparse tensor using these two methods in conjunction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xyK69QgRmrlW"
+      },
+      "outputs": [],
+      "source": [
+        "# Gather values from one tensor by specifying indices\n",
+        "\n",
+        "new_indices = tf.constant([[0, 2], [2, 1], [3, 3]])\n",
+        "t7 = tf.gather_nd(t2, indices=new_indices)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_7V_Qfa4qkdn"
+      },
+      "source": [
+        ""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QWT1E1Weqjx2"
+      },
+      "outputs": [],
+      "source": [
+        "# Add these values into a new tensor\n",
+        "\n",
+        "t8 = tf.scatter_nd(indices=new_indices, updates=t7, shape=tf.constant([4, 5]))\n",
+        "\n",
+        "print(t8)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NUyYjnvCn_vu"
+      },
+      "source": [
+        "This is similar to:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LeqFwUgroE4j"
+      },
+      "outputs": [],
+      "source": [
+        "t9 = tf.SparseTensor(indices=[[0, 2], [2, 1], [3, 3]],\n",
+        "                     values=[2, 11, 18],\n",
+        "                     dense_shape=[4, 5])\n",
+        "\n",
+        "print(t9)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5MaF6RlJot33"
+      },
+      "outputs": [],
+      "source": [
+        "# Convert the sparse tensor into a dense tensor\n",
+        "\n",
+        "t10 = tf.sparse.to_dense(t9)\n",
+        "\n",
+        "print(t10)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4sf3F3Xk56Bt"
+      },
+      "source": [
+        "To insert data into a tensor with pre-existing values, use `tf.tensor_scatter_nd_add`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mte2ifOb6sQO"
+      },
+      "outputs": [],
+      "source": [
+        "t11 = tf.constant([[2, 7, 0],\n",
+        "                   [9, 0, 1],\n",
+        "                   [0, 3, 8]])\n",
+        "\n",
+        "# Convert the tensor into a magic square by inserting numbers at appropriate indices\n",
+        "\n",
+        "t12 = tf.tensor_scatter_nd_add(t11,\n",
+        "                               indices=[[0, 2], [1, 1], [2, 0]],\n",
+        "                               updates=[6, 5, 4])\n",
+        "\n",
+        "print(t12)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2dQYyROU09G6"
+      },
+      "source": [
+        "Similarly, use `tf.tensor_scatter_nd_sub` to subtract values from a tensor with pre-existing values."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ac6_i6uK1EI6"
+      },
+      "outputs": [],
+      "source": [
+        "# Convert the tensor into an identity matrix\n",
+        "\n",
+        "t13 = tf.tensor_scatter_nd_sub(t11,\n",
+        "                               indices=[[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [2, 1], [2, 2]],\n",
+        "                               updates=[1, 7, 9, -1, 1, 3, 7])\n",
+        "\n",
+        "print(t13)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "B_2DuzRRwVc8"
+      },
+      "source": [
+        "Use `tf.tensor_scatter_nd_min` to copy element-wise minimum values from one tensor to another."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "T_4FrHrHlkHK"
+      },
+      "outputs": [],
+      "source": [
+        "t14 = tf.constant([[-2, -7, 0],\n",
+        "                   [-9, 0, 1],\n",
+        "                   [0, -3, -8]])\n",
+        "\n",
+        "t15 = tf.tensor_scatter_nd_min(t14,\n",
+        "                               indices=[[0, 2], [1, 1], [2, 0]],\n",
+        "                               updates=[-6, -5, -4])\n",
+        "\n",
+        "print(t15)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PkaiKyrF0WtX"
+      },
+      "source": [
+        "Similarly, use `tf.tensor_scatter_nd_max` to copy element-wise maximum values from one tensor to another."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "izJu0nXi0GDq"
+      },
+      "outputs": [],
+      "source": [
+        "t16 = tf.tensor_scatter_nd_max(t14,\n",
+        "                               indices=[[0, 2], [1, 1], [2, 0]],\n",
+        "                               updates=[6, 5, 4])\n",
+        "\n",
+        "print(t16)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QAffUOa-85lF"
+      },
+      "source": [
+        "## Further reading and resources\n",
+        "\n",
+        "In this guide, you learned how to use the tensor slicing ops available with TensorFlow to exert finer control over the elements in your tensors.\n",
+        "\n",
+        "* Check out the slicing ops available with TensorFlow NumPy such as `tf.experimental.numpy.take_along_axis` and `tf.experimental.numpy.take`.\n",
+        "\n",
+        "* Also check out the [Tensor guide](https://www.tensorflow.org/guide/tensor) and the [Variable guide](https://www.tensorflow.org/guide/variable)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tensor_slicing.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/tf_numpy.ipynb b/site/en/guide/tf_numpy.ipynb
new file mode 100644
index 00000000000..3083acb147d
--- /dev/null
+++ b/site/en/guide/tf_numpy.ipynb
@@ -0,0 +1,963 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZjN_IJ8mhJ-4"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "sY3Ffd83hK3b"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03Pw58e6mTHI"
+      },
+      "source": [
+        "# NumPy API on TensorFlow"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7WpGysDJmZsg"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s2enCDi_FvCR"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "TensorFlow implements a subset of the [NumPy API](https://numpy.org/doc/stable/index.html), available as `tf.experimental.numpy`. This allows running NumPy code, accelerated by TensorFlow, while also allowing access to all of TensorFlow's APIs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ob1HNwUmYR5b"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AJR558zjAZQu"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "import tensorflow.experimental.numpy as tnp\n",
+        "import timeit\n",
+        "\n",
+        "print(\"Using TensorFlow version %s\" % tf.__version__)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "M6tacoy0DU6e"
+      },
+      "source": [
+        "### Enabling NumPy behavior\n",
+        "\n",
+        "In order to use `tnp` as NumPy, enable NumPy behavior for TensorFlow:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TfCyofpFDQxm"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.experimental_enable_numpy_behavior()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "et9D5wq0D1H2"
+      },
+      "source": [
+        "This call enables type promotion in TensorFlow and also changes type inference, when converting literals to tensors, to more strictly follow the NumPy standard.\n",
+        "\n",
+        "Note: This call will change the behavior of entire TensorFlow, not just the `tf.experimental.numpy` module."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yh2BwqUzH3C3"
+      },
+      "source": [
+        "## TensorFlow NumPy ND array\n",
+        "\n",
+        "An instance of `tf.experimental.numpy.ndarray`, called **ND Array**, represents a multidimensional dense array of a given `dtype` placed on a certain device. It is an alias to `tf.Tensor`. Check out the ND array class for useful methods like `ndarray.T`, `ndarray.reshape`, `ndarray.ravel` and others.\n",
+        "\n",
+        "First create an ND array object, and then invoke different methods."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-BHJjxigJ2H1"
+      },
+      "outputs": [],
+      "source": [
+        "# Create an ND array and check out different attributes.\n",
+        "ones = tnp.ones([5, 3], dtype=tnp.float32)\n",
+        "print(\"Created ND array with shape = %s, rank = %s, \"\n",
+        "      \"dtype = %s on device = %s\\n\" % (\n",
+        "          ones.shape, ones.ndim, ones.dtype, ones.device))\n",
+        "\n",
+        "# `ndarray` is just an alias to `tf.Tensor`.\n",
+        "print(\"Is `ones` an instance of tf.Tensor: %s\\n\" % isinstance(ones, tf.Tensor))\n",
+        "\n",
+        "# Try commonly used member functions.\n",
+        "print(\"ndarray.T has shape %s\" % str(ones.T.shape))\n",
+        "print(\"narray.reshape(-1) has shape %s\" % ones.reshape(-1).shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-BOY8CGRKEhE"
+      },
+      "source": [
+        "### Type promotion\n",
+        "\n",
+        "There are 4 options for type promotion in TensorFlow.\n",
+        "\n",
+        "- By default, TensorFlow raises errors instead of promoting types for mixed type operations.\n",
+        "- Running `tf.numpy.experimental_enable_numpy_behavior()` switches TensorFlow to use `NumPy` type promotion rules (described below).\n",
+        "- After TensorFlow 2.15, there are two new options (refer to [TF NumPy Type Promotion](tf_numpy_type_promotion.ipynb) for details):\n",
+        "  - `tf.numpy.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")` uses Jax type promotion rules.\n",
+        "  - `tf.numpy.experimental_enable_numpy_behavior(dtype_conversion_mode=\"safe\")` uses Jax type promotion rules, but disallows certain unsafe promotions."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SXskSHrX5J45"
+      },
+      "source": [
+        "#### NumPy Type Promotion\n",
+        "\n",
+        "TensorFlow NumPy APIs have well-defined semantics for converting literals to ND array, as well as for performing type promotion on ND array inputs. Please see [`np.result_type`](https://numpy.org/doc/1.16/reference/generated/numpy.result_type.html) for more details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vcRznNaMj27J"
+      },
+      "source": [
+        "TensorFlow APIs leave `tf.Tensor` inputs unchanged and do not perform type promotion on them, while TensorFlow NumPy APIs promote all inputs according to NumPy type promotion rules. In the next example, you will perform type promotion. First, run addition on ND array inputs of different types and note the output types. None of these type promotions would be allowed by TensorFlow APIs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uHmBi4KZI2t1"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Type promotion for operations\")\n",
+        "values = [tnp.asarray(1, dtype=d) for d in\n",
+        "          (tnp.int32, tnp.int64, tnp.float32, tnp.float64)]\n",
+        "for i, v1 in enumerate(values):\n",
+        "  for v2 in values[i + 1:]:\n",
+        "    print(\"%s + %s => %s\" %\n",
+        "          (v1.dtype.name, v2.dtype.name, (v1 + v2).dtype.name))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CrpIoOc7oqox"
+      },
+      "source": [
+        "Finally, convert literals to ND array using `ndarray.asarray` and note the resulting type."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1m1cp8_VooNk"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Type inference during array creation\")\n",
+        "print(\"tnp.asarray(1).dtype == tnp.%s\" % tnp.asarray(1).dtype.name)\n",
+        "print(\"tnp.asarray(1.).dtype == tnp.%s\\n\" % tnp.asarray(1.).dtype.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kd-_iccXoRL8"
+      },
+      "source": [
+        "When converting literals to ND array, NumPy prefers wide types like `tnp.int64` and `tnp.float64`. In contrast, `tf.convert_to_tensor` prefers `tf.int32` and `tf.float32` types for converting constants to `tf.Tensor`. TensorFlow NumPy APIs adhere to the NumPy behavior for integers. As for floats, the `prefer_float32` argument of `experimental_enable_numpy_behavior` lets you control whether to prefer `tf.float32` over `tf.float64` (default to `False`). For example:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4gKasnH0j84C"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.experimental_enable_numpy_behavior(prefer_float32=True)\n",
+        "print(\"When prefer_float32 is True:\")\n",
+        "print(\"tnp.asarray(1.).dtype == tnp.%s\" % tnp.asarray(1.).dtype.name)\n",
+        "print(\"tnp.add(1., 2.).dtype == tnp.%s\" % tnp.add(1., 2.).dtype.name)\n",
+        "\n",
+        "tnp.experimental_enable_numpy_behavior(prefer_float32=False)\n",
+        "print(\"When prefer_float32 is False:\")\n",
+        "print(\"tnp.asarray(1.).dtype == tnp.%s\" % tnp.asarray(1.).dtype.name)\n",
+        "print(\"tnp.add(1., 2.).dtype == tnp.%s\" % tnp.add(1., 2.).dtype.name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MwCCDxSZOfA1"
+      },
+      "source": [
+        "### Broadcasting\n",
+        "\n",
+        "Similar to TensorFlow, NumPy defines rich semantics for \"broadcasting\" values.\n",
+        "You can check out the [NumPy broadcasting guide](https://numpy.org/doc/1.16/user/basics.broadcasting.html) for more information and compare this with [TensorFlow broadcasting semantics](https://www.tensorflow.org/guide/tensor#broadcasting)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qlyOShxIO0s2"
+      },
+      "outputs": [],
+      "source": [
+        "x = tnp.ones([2, 3])\n",
+        "y = tnp.ones([3])\n",
+        "z = tnp.ones([1, 2, 1])\n",
+        "print(\"Broadcasting shapes %s, %s and %s gives shape %s\" % (\n",
+        "    x.shape, y.shape, z.shape, (x + y + z).shape))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LEVr4ctRPrqR"
+      },
+      "source": [
+        "### Indexing\n",
+        "\n",
+        "NumPy defines very sophisticated indexing rules. See the [NumPy Indexing guide](https://numpy.org/doc/1.16/reference/arrays.indexing.html). Note the use of ND arrays as indices below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lRsrtnd3YyMj"
+      },
+      "outputs": [],
+      "source": [
+        "x = tnp.arange(24).reshape(2, 3, 4)\n",
+        "\n",
+        "print(\"Basic indexing\")\n",
+        "print(x[1, tnp.newaxis, 1:3, ...], \"\\n\")\n",
+        "\n",
+        "print(\"Boolean indexing\")\n",
+        "print(x[:, (True, False, True)], \"\\n\")\n",
+        "\n",
+        "print(\"Advanced indexing\")\n",
+        "print(x[1, (0, 0, 1), tnp.asarray([0, 1, 1])])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yRAaiGhlaNw7"
+      },
+      "outputs": [],
+      "source": [
+        "# Mutation is currently not supported\n",
+        "try:\n",
+        "  tnp.arange(6)[1] = -1\n",
+        "except TypeError:\n",
+        "  print(\"Currently, TensorFlow NumPy does not support mutation.\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5XfJ602j-GVD"
+      },
+      "source": [
+        "### Example Model\n",
+        "\n",
+        "Next, you can see how to create a model and run inference on it. This simple model applies a relu layer followed by a linear projection. Later sections will show how to compute gradients for this model using TensorFlow's `GradientTape`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kR_KCh4kYEhm"
+      },
+      "outputs": [],
+      "source": [
+        "class Model(object):\n",
+        "  \"\"\"Model with a dense and a linear layer.\"\"\"\n",
+        "\n",
+        "  def __init__(self):\n",
+        "    self.weights = None\n",
+        "\n",
+        "  def predict(self, inputs):\n",
+        "    if self.weights is None:\n",
+        "      size = inputs.shape[1]\n",
+        "      # Note that type `tnp.float32` is used for performance.\n",
+        "      stddev = tnp.sqrt(size).astype(tnp.float32)\n",
+        "      w1 = tnp.random.randn(size, 64).astype(tnp.float32) / stddev\n",
+        "      bias = tnp.random.randn(64).astype(tnp.float32)\n",
+        "      w2 = tnp.random.randn(64, 2).astype(tnp.float32) / 8\n",
+        "      self.weights = (w1, bias, w2)\n",
+        "    else:\n",
+        "      w1, bias, w2 = self.weights\n",
+        "    y = tnp.matmul(inputs, w1) + bias\n",
+        "    y = tnp.maximum(y, 0)  # Relu\n",
+        "    return tnp.matmul(y, w2)  # Linear projection\n",
+        "\n",
+        "model = Model()\n",
+        "# Create input data and compute predictions.\n",
+        "print(model.predict(tnp.ones([2, 32], dtype=tnp.float32)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kSR7Ou5YcS38"
+      },
+      "source": [
+        "## TensorFlow NumPy and NumPy\n",
+        "\n",
+        "TensorFlow NumPy implements a subset of the full NumPy spec. While more symbols will be added over time, there are systematic features that will not be supported in the near future. These include NumPy C API support, Swig integration, Fortran storage order, views and `stride_tricks`, and some `dtype`s (like `np.recarray` and `np.object`). For more details, please see the [TensorFlow NumPy API Documentation](https://www.tensorflow.org/api_docs/python/tf/experimental/numpy).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Jb1KXak2YlNN"
+      },
+      "source": [
+        "### NumPy interoperability\n",
+        "\n",
+        "TensorFlow ND arrays can interoperate with NumPy functions. These objects implement the `__array__` interface. NumPy uses this interface to convert function arguments to `np.ndarray` values before processing them.\n",
+        "\n",
+        "Similarly, TensorFlow NumPy functions can accept inputs of different types including `np.ndarray`. These inputs are converted to an ND array by calling `ndarray.asarray` on them.\n",
+        "\n",
+        "Conversion of the ND array to and from `np.ndarray` may trigger actual data copies. Please see the section on [buffer copies](#buffer-copies) for more details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cMOCgzQmeXRU"
+      },
+      "outputs": [],
+      "source": [
+        "# ND array passed into NumPy function.\n",
+        "np_sum = np.sum(tnp.ones([2, 3]))\n",
+        "print(\"sum = %s. Class: %s\" % (float(np_sum), np_sum.__class__))\n",
+        "\n",
+        "# `np.ndarray` passed into TensorFlow NumPy function.\n",
+        "tnp_sum = tnp.sum(np.ones([2, 3]))\n",
+        "print(\"sum = %s. Class: %s\" % (float(tnp_sum), tnp_sum.__class__))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZaLPjzxft780"
+      },
+      "outputs": [],
+      "source": [
+        "# It is easy to plot ND arrays, given the __array__ interface.\n",
+        "labels = 15 + 2 * tnp.random.randn(1, 1000)\n",
+        "_ = plt.hist(labels)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kF-Xyw3XWKqJ"
+      },
+      "source": [
+        "### Buffer copies\n",
+        "\n",
+        "Intermixing TensorFlow NumPy with NumPy code may trigger data copies. This is because TensorFlow NumPy has stricter requirements on memory alignment than those of NumPy.\n",
+        "\n",
+        "When a `np.ndarray` is passed to TensorFlow NumPy, it will check for alignment requirements and trigger a copy if needed. When passing an ND array CPU buffer to NumPy, generally the buffer will satisfy alignment requirements and NumPy will not need to create a copy.\n",
+        "\n",
+        "ND arrays can refer to buffers placed on devices other than the local CPU memory. In such cases, invoking a NumPy function will trigger copies across the network or device as needed.\n",
+        "\n",
+        "Given this, intermixing with NumPy API calls should generally be done with caution and the user should watch out for overheads of copying data. Interleaving TensorFlow NumPy calls with TensorFlow calls is generally safe and avoids copying data. See the section on [TensorFlow interoperability](#tensorflow-interoperability) for more details."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RwljbqkBc7Ro"
+      },
+      "source": [
+        "### Operator precedence\n",
+        "\n",
+        "TensorFlow NumPy defines an `__array_priority__` higher than NumPy's. This means that for operators involving both ND array and `np.ndarray`, the former will take precedence, i.e., `np.ndarray` input will get converted to an ND array and the TensorFlow NumPy implementation of the operator will get invoked."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Cbw8a3G_WUO7"
+      },
+      "outputs": [],
+      "source": [
+        "x = tnp.ones([2]) + np.ones([2])\n",
+        "print(\"x = %s\\nclass = %s\" % (x, x.__class__))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DNEab_Ctky83"
+      },
+      "source": [
+        "## TF NumPy and TensorFlow\n",
+        "\n",
+        "TensorFlow NumPy is built on top of TensorFlow and hence interoperates seamlessly with TensorFlow."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fCcfgrlOnAhQ"
+      },
+      "source": [
+        "### `tf.Tensor` and ND array\n",
+        "\n",
+        "ND array is an alias to `tf.Tensor`, so obviously they can be intermixed without triggering actual data copies."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BkHVauKwnky_"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.constant([1, 2])\n",
+        "print(x)\n",
+        "\n",
+        "# `asarray` and `convert_to_tensor` here are no-ops.\n",
+        "tnp_x = tnp.asarray(x)\n",
+        "print(tnp_x)\n",
+        "print(tf.convert_to_tensor(tnp_x))\n",
+        "\n",
+        "# Note that tf.Tensor.numpy() will continue to return `np.ndarray`.\n",
+        "print(x.numpy(), x.numpy().__class__)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_151HQVBooxG"
+      },
+      "source": [
+        "### TensorFlow interoperability\n",
+        "\n",
+        "An ND array can be passed to TensorFlow APIs, since ND array is just an alias to `tf.Tensor`. As mentioned earlier, such interoperation does not do data copies, even for data placed on accelerators or remote devices.\n",
+        "\n",
+        "Conversely, `tf.Tensor` objects can be passed to `tf.experimental.numpy` APIs, without performing data copies."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-QvxNhrFoz09"
+      },
+      "outputs": [],
+      "source": [
+        "# ND array passed into TensorFlow function.\n",
+        "tf_sum = tf.reduce_sum(tnp.ones([2, 3], tnp.float32))\n",
+        "print(\"Output = %s\" % tf_sum)\n",
+        "\n",
+        "# `tf.Tensor` passed into TensorFlow NumPy function.\n",
+        "tnp_sum = tnp.sum(tf.ones([2, 3]))\n",
+        "print(\"Output = %s\" % tnp_sum)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1b4HeAkhprF_"
+      },
+      "source": [
+        "### Gradients and Jacobians: tf.GradientTape\n",
+        "\n",
+        "TensorFlow's GradientTape can be used for backpropagation through TensorFlow and TensorFlow NumPy code.\n",
+        "\n",
+        "Use the model created in [Example Model](#example-model) section, and compute gradients and jacobians."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "T47C9KS8pbsP"
+      },
+      "outputs": [],
+      "source": [
+        "def create_batch(batch_size=32):\n",
+        "  \"\"\"Creates a batch of input and labels.\"\"\"\n",
+        "  return (tnp.random.randn(batch_size, 32).astype(tnp.float32),\n",
+        "          tnp.random.randn(batch_size, 2).astype(tnp.float32))\n",
+        "\n",
+        "def compute_gradients(model, inputs, labels):\n",
+        "  \"\"\"Computes gradients of squared loss between model prediction and labels.\"\"\"\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    assert model.weights is not None\n",
+        "    # Note that `model.weights` need to be explicitly watched since they\n",
+        "    # are not tf.Variables.\n",
+        "    tape.watch(model.weights)\n",
+        "    # Compute prediction and loss\n",
+        "    prediction = model.predict(inputs)\n",
+        "    loss = tnp.sum(tnp.square(prediction - labels))\n",
+        "  # This call computes the gradient through the computation above.\n",
+        "  return tape.gradient(loss, model.weights)\n",
+        "\n",
+        "inputs, labels = create_batch()\n",
+        "gradients = compute_gradients(model, inputs, labels)\n",
+        "\n",
+        "# Inspect the shapes of returned gradients to verify they match the\n",
+        "# parameter shapes.\n",
+        "print(\"Parameter shapes:\", [w.shape for w in model.weights])\n",
+        "print(\"Gradient shapes:\", [g.shape for g in gradients])\n",
+        "# Verify that gradients are of type ND array.\n",
+        "assert isinstance(gradients[0], tnp.ndarray)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TujVPDFwrdqp"
+      },
+      "outputs": [],
+      "source": [
+        "# Computes a batch of jacobians. Each row is the jacobian of an element in the\n",
+        "# batch of outputs w.r.t. the corresponding input batch element.\n",
+        "def prediction_batch_jacobian(inputs):\n",
+        "  with tf.GradientTape() as tape:\n",
+        "    tape.watch(inputs)\n",
+        "    prediction = model.predict(inputs)\n",
+        "  return prediction, tape.batch_jacobian(prediction, inputs)\n",
+        "\n",
+        "inp_batch = tnp.ones([16, 32], tnp.float32)\n",
+        "output, batch_jacobian = prediction_batch_jacobian(inp_batch)\n",
+        "# Note how the batch jacobian shape relates to the input and output shapes.\n",
+        "print(\"Output shape: %s, input shape: %s\" % (output.shape, inp_batch.shape))\n",
+        "print(\"Batch jacobian shape:\", batch_jacobian.shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MYq9wxfc1Dv_"
+      },
+      "source": [
+        "### Trace compilation: tf.function\n",
+        "\n",
+        "TensorFlow's `tf.function` works by \"trace compiling\" the code and then optimizing these traces for much faster performance. See the [Introduction to Graphs and Functions](./intro_to_graphs.ipynb).\n",
+        "\n",
+        "`tf.function` can be used to optimize TensorFlow NumPy code as well. Here is a simple example to demonstrate the speedups. Note that the body of `tf.function` code includes calls to TensorFlow NumPy APIs.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "05SrUulm1OlL"
+      },
+      "outputs": [],
+      "source": [
+        "inputs, labels = create_batch(512)\n",
+        "print(\"Eager performance\")\n",
+        "compute_gradients(model, inputs, labels)\n",
+        "print(timeit.timeit(lambda: compute_gradients(model, inputs, labels),\n",
+        "                    number=10) * 100, \"ms\")\n",
+        "\n",
+        "print(\"\\ntf.function compiled performance\")\n",
+        "compiled_compute_gradients = tf.function(compute_gradients)\n",
+        "compiled_compute_gradients(model, inputs, labels)  # warmup\n",
+        "print(timeit.timeit(lambda: compiled_compute_gradients(model, inputs, labels),\n",
+        "                    number=10) * 100, \"ms\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5w8YxR6ELmo1"
+      },
+      "source": [
+        "### Vectorization: tf.vectorized_map\n",
+        "\n",
+        "TensorFlow has inbuilt support for vectorizing parallel loops, which allows speedups of one to two orders of magnitude. These speedups are accessible via the `tf.vectorized_map` API and apply to TensorFlow NumPy code as well.\n",
+        "\n",
+        "It is sometimes useful to compute the gradient of each output in a batch w.r.t. the corresponding input batch element. Such computation can be done efficiently using `tf.vectorized_map` as shown below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PemSIrs5L-VJ"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def vectorized_per_example_gradients(inputs, labels):\n",
+        "  def single_example_gradient(arg):\n",
+        "    inp, label = arg\n",
+        "    return compute_gradients(model,\n",
+        "                             tnp.expand_dims(inp, 0),\n",
+        "                             tnp.expand_dims(label, 0))\n",
+        "  # Note that a call to `tf.vectorized_map` semantically maps\n",
+        "  # `single_example_gradient` over each row of `inputs` and `labels`.\n",
+        "  # The interface is similar to `tf.map_fn`.\n",
+        "  # The underlying machinery vectorizes away this map loop which gives\n",
+        "  # nice speedups.\n",
+        "  return tf.vectorized_map(single_example_gradient, (inputs, labels))\n",
+        "\n",
+        "batch_size = 128\n",
+        "inputs, labels = create_batch(batch_size)\n",
+        "\n",
+        "per_example_gradients = vectorized_per_example_gradients(inputs, labels)\n",
+        "for w, p in zip(model.weights, per_example_gradients):\n",
+        "  print(\"Weight shape: %s, batch size: %s, per example gradient shape: %s \" % (\n",
+        "      w.shape, batch_size, p.shape))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_QZ5BjJmRAlG"
+      },
+      "outputs": [],
+      "source": [
+        "# Benchmark the vectorized computation above and compare with\n",
+        "# unvectorized sequential computation using `tf.map_fn`.\n",
+        "@tf.function\n",
+        "def unvectorized_per_example_gradients(inputs, labels):\n",
+        "  def single_example_gradient(arg):\n",
+        "    inp, label = arg\n",
+        "    return compute_gradients(model,\n",
+        "                             tnp.expand_dims(inp, 0),\n",
+        "                             tnp.expand_dims(label, 0))\n",
+        "\n",
+        "  return tf.map_fn(single_example_gradient, (inputs, labels),\n",
+        "                   fn_output_signature=(tf.float32, tf.float32, tf.float32))\n",
+        "\n",
+        "print(\"Running vectorized computation\")\n",
+        "print(timeit.timeit(lambda: vectorized_per_example_gradients(inputs, labels),\n",
+        "                    number=10) * 100, \"ms\")\n",
+        "\n",
+        "print(\"\\nRunning unvectorized computation\")\n",
+        "per_example_gradients = unvectorized_per_example_gradients(inputs, labels)\n",
+        "print(timeit.timeit(lambda: unvectorized_per_example_gradients(inputs, labels),\n",
+        "                    number=10) * 100, \"ms\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UOTh-nkzaJd9"
+      },
+      "source": [
+        "### Device placement\n",
+        "\n",
+        "TensorFlow NumPy can place operations on CPUs, GPUs, TPUs and remote devices. It uses standard TensorFlow mechanisms for device placement. Below a simple example shows how to list all devices and then place some computation on a particular device.\n",
+        "\n",
+        "TensorFlow also has APIs for replicating computation across devices and performing collective reductions which will not be covered here."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-0gHrwYYaTCE"
+      },
+      "source": [
+        "#### List devices\n",
+        "\n",
+        "`tf.config.list_logical_devices` and `tf.config.list_physical_devices` can be used to find what devices to use."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NDEAd9m9aemS"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"All logical devices:\", tf.config.list_logical_devices())\n",
+        "print(\"All physical devices:\", tf.config.list_physical_devices())\n",
+        "\n",
+        "# Try to get the GPU device. If unavailable, fallback to CPU.\n",
+        "try:\n",
+        "  device = tf.config.list_logical_devices(device_type=\"GPU\")[0]\n",
+        "except IndexError:\n",
+        "  device = \"/device:CPU:0\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fihgfF_tahVx"
+      },
+      "source": [
+        "#### Placing operations: **`tf.device`**\n",
+        "\n",
+        "Operations can be placed on a device by calling it in a `tf.device` scope.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c7ELvLmnazfV"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Using device: %s\" % str(device))\n",
+        "# Run operations in the `tf.device` scope.\n",
+        "# If a GPU is available, these operations execute on the GPU and outputs are\n",
+        "# placed on the GPU memory.\n",
+        "with tf.device(device):\n",
+        "  prediction = model.predict(create_batch(5)[0])\n",
+        "\n",
+        "print(\"prediction is placed on %s\" % prediction.device)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e-LK6wsHbBiM"
+      },
+      "source": [
+        "#### Copying ND arrays across devices: **`tnp.copy`**\n",
+        "\n",
+        "A call to `tnp.copy`, placed in a certain device scope, will copy the data to that device, unless the data is already on that device."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CCesyidaa-UT"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.device(\"/device:CPU:0\"):\n",
+        "  prediction_cpu = tnp.copy(prediction)\n",
+        "print(prediction.device)\n",
+        "print(prediction_cpu.device)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AiYzRDOtKzAH"
+      },
+      "source": [
+        "## Performance comparisons\n",
+        "\n",
+        "TensorFlow NumPy uses highly optimized TensorFlow kernels that can be dispatched on CPUs, GPUs and TPUs. TensorFlow also performs many compiler optimizations, like operation fusion, which translate to performance and memory improvements. See [TensorFlow graph optimization with Grappler](./graph_optimization.ipynb) to learn more.\n",
+        "\n",
+        "However TensorFlow has higher overheads for dispatching operations compared to NumPy. For workloads composed of small operations (less than about 10 microseconds), these overheads can dominate the runtime and NumPy could provide better performance. For other cases, TensorFlow should generally provide better performance.\n",
+        "\n",
+        "Run the benchmark below to compare NumPy and TensorFlow NumPy performance for different input sizes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "code",
+        "id": "RExwjI9_pJG0"
+      },
+      "outputs": [],
+      "source": [
+        "def benchmark(f, inputs, number=30, force_gpu_sync=False):\n",
+        "  \"\"\"Utility to benchmark `f` on each value in `inputs`.\"\"\"\n",
+        "  times = []\n",
+        "  for inp in inputs:\n",
+        "    def _g():\n",
+        "      if force_gpu_sync:\n",
+        "        one = tnp.asarray(1)\n",
+        "      f(inp)\n",
+        "      if force_gpu_sync:\n",
+        "        with tf.device(\"CPU:0\"):\n",
+        "          tnp.copy(one)  # Force a sync for GPU case\n",
+        "\n",
+        "    _g()  # warmup\n",
+        "    t = timeit.timeit(_g, number=number)\n",
+        "    times.append(t * 1000. / number)\n",
+        "  return times\n",
+        "\n",
+        "\n",
+        "def plot(np_times, tnp_times, compiled_tnp_times, has_gpu, tnp_times_gpu):\n",
+        "  \"\"\"Plot the different runtimes.\"\"\"\n",
+        "  plt.xlabel(\"size\")\n",
+        "  plt.ylabel(\"time (ms)\")\n",
+        "  plt.title(\"Sigmoid benchmark: TF NumPy vs NumPy\")\n",
+        "  plt.plot(sizes, np_times, label=\"NumPy\")\n",
+        "  plt.plot(sizes, tnp_times, label=\"TF NumPy (CPU)\")\n",
+        "  plt.plot(sizes, compiled_tnp_times, label=\"Compiled TF NumPy (CPU)\")\n",
+        "  if has_gpu:\n",
+        "    plt.plot(sizes, tnp_times_gpu, label=\"TF NumPy (GPU)\")\n",
+        "  plt.legend()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p-fs_H1lkLfV"
+      },
+      "outputs": [],
+      "source": [
+        "# Define a simple implementation of `sigmoid`, and benchmark it using\n",
+        "# NumPy and TensorFlow NumPy for different input sizes.\n",
+        "\n",
+        "def np_sigmoid(y):\n",
+        "  return 1. / (1. + np.exp(-y))\n",
+        "\n",
+        "def tnp_sigmoid(y):\n",
+        "  return 1. / (1. + tnp.exp(-y))\n",
+        "\n",
+        "@tf.function\n",
+        "def compiled_tnp_sigmoid(y):\n",
+        "  return tnp_sigmoid(y)\n",
+        "\n",
+        "sizes = (2 ** 0, 2 ** 5, 2 ** 10, 2 ** 15, 2 ** 20)\n",
+        "np_inputs = [np.random.randn(size).astype(np.float32) for size in sizes]\n",
+        "np_times = benchmark(np_sigmoid, np_inputs)\n",
+        "\n",
+        "with tf.device(\"/device:CPU:0\"):\n",
+        "  tnp_inputs = [tnp.random.randn(size).astype(np.float32) for size in sizes]\n",
+        "  tnp_times = benchmark(tnp_sigmoid, tnp_inputs)\n",
+        "  compiled_tnp_times = benchmark(compiled_tnp_sigmoid, tnp_inputs)\n",
+        "\n",
+        "has_gpu = len(tf.config.list_logical_devices(\"GPU\"))\n",
+        "if has_gpu:\n",
+        "  with tf.device(\"/device:GPU:0\"):\n",
+        "    tnp_inputs = [tnp.random.randn(size).astype(np.float32) for size in sizes]\n",
+        "    tnp_times_gpu = benchmark(compiled_tnp_sigmoid, tnp_inputs, 100, True)\n",
+        "else:\n",
+        "  tnp_times_gpu = None\n",
+        "plot(np_times, tnp_times, compiled_tnp_times, has_gpu, tnp_times_gpu)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ReK_9k5D8BZQ"
+      },
+      "source": [
+        "## Further reading\n",
+        "\n",
+        "- [TensorFlow NumPy: Distributed Image Classification Tutorial](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/numpy_ops/g3doc/TensorFlow_Numpy_Distributed_Image_Classification.ipynb)\n",
+        "- [TensorFlow NumPy: Keras and Distribution Strategy](\n",
+        "  https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/numpy_ops/g3doc/TensorFlow_NumPy_Keras_and_Distribution_Strategy.ipynb)\n",
+        "- [Sentiment Analysis with Trax and TensorFlow NumPy](\n",
+        "  https://github.com/google/trax/blob/master/trax/tf_numpy_and_keras.ipynb)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "name": "tf_numpy.ipynb",
+      "provenance": [],
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/tf_numpy_type_promotion.ipynb b/site/en/guide/tf_numpy_type_promotion.ipynb
new file mode 100644
index 00000000000..f984310822a
--- /dev/null
+++ b/site/en/guide/tf_numpy_type_promotion.ipynb
@@ -0,0 +1,1138 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZjN_IJ8mhJ-4"
+      },
+      "source": [
+        "##### Copyright 2023 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sY3Ffd83hK3b"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "03Pw58e6mTHI"
+      },
+      "source": [
+        "# TF-NumPy Type Promotion"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l9nPKvxK-_pM"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uma-W5v__DYh"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "There are 4 options for type promotion in TensorFlow.\n",
+        "\n",
+        "- By default, TensorFlow raises errors instead of promoting types for mixed type operations.\n",
+        "- Running `tf.numpy.experimental_enable_numpy_behavior()` switches TensorFlow to use [NumPy type promotion rules](https://www.tensorflow.org/guide/tf_numpy#type_promotion).\n",
+        "- **This doc** describes two new options that will be available in TensorFlow 2.15 (or currently in `tf-nightly`):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vMvEKDFOsau7"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q tf_nightly"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a6hOFBfPsd3y"
+      },
+      "source": [
+        " **Note**: `experimental_enable_numpy_behavior` changes the behavior of all of TensorFlow."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ob1HNwUmYR5b"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AJR558zjAZQu"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "import tensorflow.experimental.numpy as tnp\n",
+        "\n",
+        "print(\"Using TensorFlow version %s\" % tf.__version__)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "M6tacoy0DU6e"
+      },
+      "source": [
+        "### Enabling the new type promotion\n",
+        "\n",
+        "In order to use the [JAX-like type promotion](https://jax.readthedocs.io/en/latest/type_promotion.html) in TF-Numpy, specify either `'all'` or `'safe'` as the dtype conversion mode when enabling NumPy behavior for TensorFlow.\n",
+        "\n",
+        "This new system (with `dtype_conversion_mode=\"all\"`) is associative, commutative, and makes it easy to control what width of float you end up with (it doesn't automatically convert to wider floats). It does introduce some risks of overflows and precision loss, but `dtype_conversion_mode=\"safe\"` forces you to handle those cases explicitly. The two modes are explained more in detail in the [next section](#two_modes)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TfCyofpFDQxm"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sEMXK8-ZWMun"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "## Two Modes : ALL mode vs SAFE mode\n",
+        "\n",
+        "In the new type promotion system, we introduce two modes: `ALL` mode and `SAFE` mode. `SAFE` mode is used to mitigate the concerns of \"risky\" promotions that can result in precision loss or bit-widening."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-ULvTWj_KnHU"
+      },
+      "source": [
+        "### Dtypes\n",
+        "\n",
+        "We will be using the following abbreviations for brevity.\n",
+        "\n",
+        "*   `b` means `tf.bool`\n",
+        "*   `u8` means `tf.uint8`\n",
+        "*   `i16` means `tf.int16`\n",
+        "*   `i32` means `tf.int32`\n",
+        "*   `bf16` means `tf.bfloat16`\n",
+        "*   `f32` means `tf.float32`\n",
+        "*   `f64` means `tf.float64`\n",
+        "*   `i32*` means Python `int` or weakly-typed `i32`\n",
+        "*   `f32*` means Python `float` or weakly-typed `f32`\n",
+        "*   `c128*` means Python `complex` or weakly-typed `c128`\n",
+        "\n",
+        "The asterisk (*) denotes that the corresponding type is “weak” - such a dtype is temporarily inferred by the system, and could defer to other dtypes. This concept is explained more in detail [here](#weak_tensor)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hXZxLCkuzzq3"
+      },
+      "source": [
+        "### Example of precision losing operations\n",
+        "\n",
+        "In the following example, `i32` + `f32` is allowed in `ALL` mode but\n",
+        "not in `SAFE` mode due to the risk of precision loss."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y-yeIvstWStL"
+      },
+      "outputs": [],
+      "source": [
+        "# i32 + f32 returns a f32 result in ALL mode.\n",
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")\n",
+        "a = tf.constant(10, dtype = tf.int32)\n",
+        "b = tf.constant(5.0, dtype = tf.float32)\n",
+        "a + b  # "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JNNmZow2WY3G"
+      },
+      "outputs": [],
+      "source": [
+        "# This promotion is not allowed in SAFE mode.\n",
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"safe\")\n",
+        "a = tf.constant(10, dtype = tf.int32)\n",
+        "b = tf.constant(5.0, dtype = tf.float32)\n",
+        "try:\n",
+        "  a + b\n",
+        "except TypeError as e:\n",
+        "   print(f'{type(e)}: {e}')  # TypeError: explicitly specify the dtype or switch to ALL mode."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f0x4Qhff0AKS"
+      },
+      "source": [
+        "### Example of bit-widening operations\n",
+        "\n",
+        "In the following example, `i8` + `u32` is allowed in `ALL` mode but\n",
+        "not in `SAFE` mode due to bit-widening, which means using more bits than the number of bits in the inputs. Note that the new type promotion semantics only allows necessary bit-widening."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Etbv-WoWzUXf"
+      },
+      "outputs": [],
+      "source": [
+        "# i8 + u32 returns an i64 result in ALL mode.\n",
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")\n",
+        "a = tf.constant(10, dtype = tf.int8)\n",
+        "b = tf.constant(5, dtype = tf.uint32)\n",
+        "a + b"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yKRdvtvw0Lvt"
+      },
+      "outputs": [],
+      "source": [
+        "# This promotion is not allowed in SAFE mode.\n",
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"safe\")\n",
+        "a = tf.constant(10, dtype = tf.int8)\n",
+        "b = tf.constant(5, dtype = tf.uint32)\n",
+        "try:\n",
+        "  a + b\n",
+        "except TypeError as e:\n",
+        "   print(f'{type(e)}: {e}')  # TypeError: explicitly specify the dtype or switch to ALL mode."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yh2BwqUzH3C3"
+      },
+      "source": [
+        "## A System Based on a Lattice"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HHUnfTPiYVN5"
+      },
+      "source": [
+        "### Type Promotion Lattice\n",
+        "\n",
+        "The new type promotion behavior is determined via the following type promotion lattice:\n",
+        "\n",
+        "![Type Promotion Lattice](https://tensorflow.org/guide/images/new_type_promotion/type_promotion_lattice.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QykluwRyDDle"
+      },
+      "source": [
+        "More specifically, promotion between any two types is determined by finding the first common child of the two nodes (including the nodes themselves).\n",
+        "\n",
+        "For example, in the diagram above, the first common child of `i8` and `i32` is `i32` because the two nodes intersect for the first time at `i32` when following the direction of the arrows.\n",
+        "\n",
+        "Similarly as another example, the result promotion type between `u64` and `f16` would be `f16`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nthziRHaDAUY"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "### Type Promotion Table\n",
+        "\n",
+        "Following the lattice generates the binary promotion table below:\n",
+        "\n",
+        "**Note**: `SAFE` mode disallows the highlighted cells. `ALL` mode allows all cases.\n",
+        "\n",
+        "![Type Promotion Table](https://tensorflow.org/guide/images/new_type_promotion/type_promotion_table.png)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TPDt5QTkucSC"
+      },
+      "source": [
+        "## Advantages of The New Type Promotion\n",
+        "\n",
+        "We adopt a JAX-like lattice-based system for our new type promotion, which offers the following advantages:"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NUS_b13nue1p"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "#### Advantages of Lattice-Based System\n",
+        "\n",
+        "First, using a lattice-based system ensures three very important properties:\n",
+        "\n",
+        "*   Existence: There is a unique result promotion type for any combinations of types.\n",
+        "*   Commutativity: `a + b = b + a`\n",
+        "*   Associativity: `a + (b + c) = (a + b) = c`\n",
+        "\n",
+        "These three properties are critical for constructing a type promotion semantics that is consistent and predictable."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Sz88hRR6uhls"
+      },
+      "source": [
+        "#### Advantages of JAX-like Lattice System\n",
+        "\n",
+        "Another crucial advantage of the JAX-like lattice system is that outside unsigned ints, it avoids all wider-than-necessary promotions. This means you cannot get 64-bit results without 64-bit inputs. This is especially beneficial for working on accelerators as it avoids unnecessary 64-bit values, which was frequent in the old type promotion."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rlylb7ieOVbJ"
+      },
+      "source": [
+        "However, this comes with a trade-off: mixed float/integer promotion is very prone to precision loss. For instance, in the example below, `i64` + `f16` results in promoting `i64` to `f16`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "abqIkV02OXEF"
+      },
+      "outputs": [],
+      "source": [
+        "# The first input is promoted to f16 in ALL mode.\n",
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")\n",
+        "tf.constant(1, tf.int64) + tf.constant(3.2, tf.float16)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mYnh1gZdObfI"
+      },
+      "source": [
+        "To migitage this concern, we introduced a `SAFE` mode that will disallow these \"risky\" promotions.\n",
+        "\n",
+        "**Note**: To learn more about the design considerations in constructing the lattice system, please refer to the [Design of Type Promotion Semantics for JAX](https://jax.readthedocs.io/en/latest/jep/9407-type-promotion.html)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gAc7LFV0S2dP"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "## WeakTensor"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "olQ2gsFlS9BH"
+      },
+      "source": [
+        "### Overview\n",
+        "\n",
+        "*Weak tensors* are Tensors that are \"weakly typed\", similar to a [concept in JAX](https://jax.readthedocs.io/en/latest/type_promotion.html#weakly-typed-values-in-jax).\n",
+        "\n",
+        "`WeakTensor`'s dtype is temporarily inferred by the system, and could defer to other dtypes. This concept is introduced in the new type promotion to prevent unwanted type promotion within binary operations between TF values and values with no explicitly user-specified type, such as Python scalar literals."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MYmoFIqZTFtw"
+      },
+      "source": [
+        "For instance, in the example below, `tf.constant(1.2)` is considered \"weak\" because it doesn't have a specific dtype. Therefore, `tf.constant(1.2)` defers to the type of `tf.constant(3.1, tf.float16)`, resulting in a `f16` output."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eSBv_mzyTE97"
+      },
+      "outputs": [],
+      "source": [
+        "tf.constant(1.2) + tf.constant(3.1, tf.float16)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KxuqBIFuTm5Z"
+      },
+      "source": [
+        "### WeakTensor Construction\n",
+        "\n",
+        "WeakTensors are created if you create a tensor without specifying a dtype the result is a WeakTensor. You can check whether a Tensor is \"weak\" or not by checking the weak attribute at the end of the Tensor's string representation."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7UmunnJ8True3"
+      },
+      "source": [
+        "**First Case**: When `tf.constant` is called with an input with no user-specified dtype."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fLEtMluNTsI5"
+      },
+      "outputs": [],
+      "source": [
+        "tf.constant(5)  # "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZQX6MBWHTt__"
+      },
+      "outputs": [],
+      "source": [
+        "tf.constant([5.0, 10.0, 3])  # "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ftsKSC5BTweP"
+      },
+      "outputs": [],
+      "source": [
+        "# A normal Tensor is created when dtype arg is specified.\n",
+        "tf.constant(5, tf.int32)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RqhoRy5iTyag"
+      },
+      "source": [
+        "**Second Case**: When an input with no user-specified dtype is passed into a [WeakTensor-supporting API](#weak_tensor_apis)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DuwpgoQJTzE-"
+      },
+      "outputs": [],
+      "source": [
+        "tf.math.abs([100.0, 4.0])  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UTcoR1xvR39k"
+      },
+      "source": [
+        "##Effects of turning on the new type promotion\n",
+        "\n",
+        "Below is a non-exhaustive list of changes that result from turning on the new type promotion.\n",
+        "\n",
+        "*   More consistent and predictable promotion results.\n",
+        "*   Reduced risk of bit-widening.\n",
+        "*   `tf.Tensor` mathematical dunder methods use new type promotion.\n",
+        "*   `tf.constant` can return `WeakTensor`.\n",
+        "*   `tf.constant` allows implicit conversions when a Tensor input with a dtype different from the `dtype` arg is passed in.\n",
+        "*   `tf.Variable` in-place ops (`assign`, `assign-add`, `assign-sub`) allow implicit conversions.\n",
+        "*   `tnp.array(1)` and `tnp.array(1.0)` returns 32-bit WeakTensor.\n",
+        "*   `WeakTensor`s will be created and used for [WeakTensor-supporting unary and binary API](#weak_tensor_apis)'s.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KyvonwYcsFX2"
+      },
+      "source": [
+        "### More consistent and predictable promotion results\n",
+        "\n",
+        "Using a [lattice-based system](#lattice_system_design) allows the new type promotion to produce consistent and predictable type promotion results."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "q0Z1njfb7lRa"
+      },
+      "source": [
+        "#### Old Type Promotion\n",
+        "\n",
+        "Changing the order of operations produces inconsistent results using old type promotion."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "M1Ca9v4m7z8e"
+      },
+      "outputs": [],
+      "source": [
+        "# Setup\n",
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"legacy\")\n",
+        "a = np.array(1, dtype=np.int8)\n",
+        "b = tf.constant(1)\n",
+        "c = np.array(1, dtype=np.float16)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WwhTzJ-a4rTc"
+      },
+      "outputs": [],
+      "source": [
+        "# (a + b) + c throws an InvalidArgumentError.\n",
+        "try:\n",
+        "  tf.add(tf.add(a, b), c)\n",
+        "except tf.errors.InvalidArgumentError as e:\n",
+        "  print(f'{type(e)}: {e}')  # InvalidArgumentError"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d3qDgVYn7ezT"
+      },
+      "outputs": [],
+      "source": [
+        "# (b + a) + c returns an i32 result.\n",
+        "tf.add(tf.add(b, a), c)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YMH1skEs7oI5"
+      },
+      "source": [
+        "#### New Type Promotion\n",
+        "\n",
+        "New type promotion produces consistent results regardless of the order."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BOHyJJ8z8uCN"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")\n",
+        "a = np.array(1, dtype=np.int8)\n",
+        "b = tf.constant(1)\n",
+        "c = np.array(1, dtype=np.float16)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZUKU70jf7E1l"
+      },
+      "outputs": [],
+      "source": [
+        "# (a + b) + c returns a f16 result.\n",
+        "tf.add(tf.add(a, b), c)  # "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YOEycjFx7qDn"
+      },
+      "outputs": [],
+      "source": [
+        "# (b + a) + c also returns a f16 result.\n",
+        "tf.add(tf.add(b, a), c)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FpGMkm6aJsn6"
+      },
+      "source": [
+        "### Reduced risk of bit-widening"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JxV2AL-U9Grg"
+      },
+      "source": [
+        "#### Old Type Promotion\n",
+        "\n",
+        "Old type promotion often resulted in 64-bit results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7L1pxyvn9MlP"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"legacy\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zMJVFdWf4XHp"
+      },
+      "outputs": [],
+      "source": [
+        "np.array(3.2, np.float16) + tf.constant(1, tf.int8) + tf.constant(50)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fBhUH_wD9Is7"
+      },
+      "source": [
+        "#### New Type Promotion\n",
+        "\n",
+        "New type promotion returns results with minimal number of bits necessary."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aJsj2ZyI9T9Y"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jj0N_Plp4X9l"
+      },
+      "outputs": [],
+      "source": [
+        "np.array(3.2, np.float16) + tf.constant(1, tf.int8) + tf.constant(50)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yKUx7xe-KZ5O"
+      },
+      "source": [
+        "### tf.Tensor mathematical dunder methods\n",
+        "\n",
+        "All `tf.Tensor` mathematical dunder methods will follow the new type promotion."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2c3icBUX4wNl"
+      },
+      "outputs": [],
+      "source": [
+        "-tf.constant(5)  # "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ydJHQjid45s7"
+      },
+      "outputs": [],
+      "source": [
+        "tf.constant(5, tf.int16) - tf.constant(1, tf.float32)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pLbIjIvbKqcU"
+      },
+      "source": [
+        "### tf.Variable in-place ops\n",
+        "\n",
+        "Implicit conversions will be allowed in `tf.Variable` in-place ops.\n",
+        "\n",
+        "**Note**: Any promotion that results in a dtype that is different from the variable's original dtype will be not allowed. This is because `tf.Variable` cannot change its dtype."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QsXhyK1h-i5S"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")\n",
+        "a = tf.Variable(10, tf.int32)\n",
+        "a.assign_add(tf.constant(5, tf.int16))  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PiA4H-otLDit"
+      },
+      "source": [
+        "### tf.constant implicit conversions\n",
+        "\n",
+        "In the old type promotion, `tf.constant` required an input Tensor to have the same dtype as the dtype argument. However, in the new type promotion, we implicitly convert Tensor to the specified dtype."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ArrQ9Dj0_OR8"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.experimental_enable_numpy_behavior(dtype_conversion_mode=\"all\")\n",
+        "a = tf.constant(10, tf.int16)\n",
+        "tf.constant(a, tf.float32)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WAcK_-XnLWaP"
+      },
+      "source": [
+        "### TF-NumPy Array\n",
+        "\n",
+        "`tnp.array` defaults to `i32*` and `f32*` for python inputs using the new type promotion."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "K1pZnYNh_ahm"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.array(1)  # "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QoQl2PYP_fMT"
+      },
+      "outputs": [],
+      "source": [
+        "tnp.array(1.0)  # "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wK5DpQ3Pz3k5"
+      },
+      "source": [
+        "##Input Type Inference\n",
+        "\n",
+        "This is how different inputs' types are inferred in the new type promotion.\n",
+        "\n",
+        "\n",
+        "*   `tf.Tensor`: Since `tf.Tensor` has a dtype property, we don't do further inference.\n",
+        "*   NumPy types: This includes types like `np.array(1)`, `np.int16(1)`, and `np.float`. Since NumPy inputs also have a dtype property, we take the dtype property as the result inference type. Note that NumPy defaults to `i64` and `f64`.\n",
+        "*   Python scalars/Nested types: This includes types like `1`, `[1, 2, 3]`, and `(1.0, 2.0)`.\n",
+        "   *   Python `int` is inferred as `i32*`.\n",
+        "   *   Python `float` is inferred as `f32*`.\n",
+        "   *   Python `complex` is inferred as `c128*`.\n",
+        "*  If the input doesn't fall into any of the above categories but has a dtype property, we take the dtype property as the result inference type."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g_SPfalfSPgg"
+      },
+      "source": [
+        "# Further Reading\n",
+        "\n",
+        "The new type promotion closely resembles JAX-NumPy's type promotion. If you want to know more details about the new type promotion and the design choices, check out the resources below.\n",
+        "\n",
+        "*  [JAX Type Promotion Semantics](https://jax.readthedocs.io/en/latest/type_promotion.html)\n",
+        "*  [Design of Type Promotion Semantics for JAX](https://jax.readthedocs.io/en/latest/jep/9407-type-promotion.html)\n",
+        "*  [Old TF-NumPy Promotion Semantics](https://www.tensorflow.org/guide/tf_numpy#type_promotion)\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Qg5xBbImT31S"
+      },
+      "source": [
+        "# References"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gjB0CVhVXBfW"
+      },
+      "source": [
+        "\n",
+        "\n",
+        "## WeakTensor-supporting APIs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_GVbqlN9aBS2"
+      },
+      "source": [
+        "Below is a list of APIs that supports `WeakTensor`.\n",
+        "\n",
+        "For an unary op, this means that if an input with no user-specified type is passed in, it will return a `WeakTensor`.\n",
+        "\n",
+        "For a binary op, it will follow the promotion table [here](#promotion_table). It may or may not return a `WeakTensor` depending on the promotion result of the two inputs.\n",
+        "\n",
+        "**Note**: All mathematical operations (`+`, `-`, `*`, ...) are supported."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Gi-G68Z8WN2P"
+      },
+      "source": [
+        "* `tf.bitwise.invert`\n",
+        "* `tf.clip_by_value`\n",
+        "* `tf.debugging.check_numerics`\n",
+        "* `tf.expand_dims`\n",
+        "* `tf.identity`\n",
+        "* `tf.image.adjust_brightness`\n",
+        "* `tf.image.adjust_gamma`\n",
+        "* `tf.image.extract_patches`\n",
+        "* `tf.image.random_brightness`\n",
+        "* `tf.image.stateless_random_brightness`\n",
+        "* `tf.linalg.diag`\n",
+        "* `tf.linalg.diag_part`\n",
+        "* `tf.linalg.matmul`\n",
+        "* `tf.linalg.matrix_transpose`\n",
+        "* `tf.linalg.tensor_diag_part`\n",
+        "* `tf.linalg.trace`\n",
+        "* `tf.math.abs`\n",
+        "* `tf.math.acos`\n",
+        "* `tf.math.acosh`\n",
+        "* `tf.math.add`\n",
+        "* `tf.math.angle`\n",
+        "* `tf.math.asin`\n",
+        "* `tf.math.asinh`\n",
+        "* `tf.math.atan`\n",
+        "* `tf.math.atanh`\n",
+        "* `tf.math.ceil`\n",
+        "* `tf.math.conj`\n",
+        "* `tf.math.cos`\n",
+        "* `tf.math.cosh`\n",
+        "* `tf.math.digamma`\n",
+        "* `tf.math.divide_no_nan`\n",
+        "* `tf.math.divide`\n",
+        "* `tf.math.erf`\n",
+        "* `tf.math.erfc`\n",
+        "* `tf.math.erfcinv`\n",
+        "* `tf.math.erfinv`\n",
+        "* `tf.math.exp`\n",
+        "* `tf.math.expm1`\n",
+        "* `tf.math.floor`\n",
+        "* `tf.math.floordiv`\n",
+        "* `tf.math.floormod`\n",
+        "* `tf.math.imag`\n",
+        "* `tf.math.lgamma`\n",
+        "* `tf.math.log1p`\n",
+        "* `tf.math.log_sigmoid`\n",
+        "* `tf.math.log`\n",
+        "* `tf.math.multiply_no_nan`\n",
+        "* `tf.math.multiply`\n",
+        "* `tf.math.ndtri`\n",
+        "* `tf.math.negative`\n",
+        "* `tf.math.pow`\n",
+        "* `tf.math.real`\n",
+        "* `tf.math.real`\n",
+        "* `tf.math.reciprocal_no_nan`\n",
+        "* `tf.math.reciprocal`\n",
+        "* `tf.math.reduce_euclidean_norm`\n",
+        "* `tf.math.reduce_logsumexp`\n",
+        "* `tf.math.reduce_max`\n",
+        "* `tf.math.reduce_mean`\n",
+        "* `tf.math.reduce_min`\n",
+        "* `tf.math.reduce_prod`\n",
+        "* `tf.math.reduce_std`\n",
+        "* `tf.math.reduce_sum`\n",
+        "* `tf.math.reduce_variance`\n",
+        "* `tf.math.rint`\n",
+        "* `tf.math.round`\n",
+        "* `tf.math.rsqrt`\n",
+        "* `tf.math.scalar_mul`\n",
+        "* `tf.math.sigmoid`\n",
+        "* `tf.math.sign`\n",
+        "* `tf.math.sin`\n",
+        "* `tf.math.sinh`\n",
+        "* `tf.math.softplus`\n",
+        "* `tf.math.special.bessel_i0`\n",
+        "* `tf.math.special.bessel_i0e`\n",
+        "* `tf.math.special.bessel_i1`\n",
+        "* `tf.math.special.bessel_i1e`\n",
+        "* `tf.math.special.bessel_j0`\n",
+        "* `tf.math.special.bessel_j1`\n",
+        "* `tf.math.special.bessel_k0`\n",
+        "* `tf.math.special.bessel_k0e`\n",
+        "* `tf.math.special.bessel_k1`\n",
+        "* `tf.math.special.bessel_k1e`\n",
+        "* `tf.math.special.bessel_y0`\n",
+        "* `tf.math.special.bessel_y1`\n",
+        "* `tf.math.special.dawsn`\n",
+        "* `tf.math.special.expint`\n",
+        "* `tf.math.special.fresnel_cos`\n",
+        "* `tf.math.special.fresnel_sin`\n",
+        "* `tf.math.special.spence`\n",
+        "* `tf.math.sqrt`\n",
+        "* `tf.math.square`\n",
+        "* `tf.math.subtract`\n",
+        "* `tf.math.tan`\n",
+        "* `tf.math.tanh`\n",
+        "* `tf.nn.depth_to_space`\n",
+        "* `tf.nn.elu`\n",
+        "* `tf.nn.gelu`\n",
+        "* `tf.nn.leaky_relu`\n",
+        "* `tf.nn.log_softmax`\n",
+        "* `tf.nn.relu6`\n",
+        "* `tf.nn.relu`\n",
+        "* `tf.nn.selu`\n",
+        "* `tf.nn.softsign`\n",
+        "* `tf.nn.space_to_depth`\n",
+        "* `tf.nn.swish`\n",
+        "* `tf.ones_like`\n",
+        "* `tf.realdiv`\n",
+        "* `tf.reshape`\n",
+        "* `tf.squeeze`\n",
+        "* `tf.stop_gradient`\n",
+        "* `tf.transpose`\n",
+        "* `tf.truncatediv`\n",
+        "* `tf.truncatemod`\n",
+        "* `tf.zeros_like`\n",
+        "* `tf.experimental.numpy.abs`\n",
+        "* `tf.experimental.numpy.absolute`\n",
+        "* `tf.experimental.numpy.amax`\n",
+        "* `tf.experimental.numpy.amin`\n",
+        "* `tf.experimental.numpy.angle`\n",
+        "* `tf.experimental.numpy.arange`\n",
+        "* `tf.experimental.numpy.arccos`\n",
+        "* `tf.experimental.numpy.arccosh`\n",
+        "* `tf.experimental.numpy.arcsin`\n",
+        "* `tf.experimental.numpy.arcsinh`\n",
+        "* `tf.experimental.numpy.arctan`\n",
+        "* `tf.experimental.numpy.arctanh`\n",
+        "* `tf.experimental.numpy.around`\n",
+        "* `tf.experimental.numpy.array`\n",
+        "* `tf.experimental.numpy.asanyarray`\n",
+        "* `tf.experimental.numpy.asarray`\n",
+        "* `tf.experimental.numpy.ascontiguousarray`\n",
+        "* `tf.experimental.numpy.average`\n",
+        "* `tf.experimental.numpy.bitwise_not`\n",
+        "* `tf.experimental.numpy.cbrt`\n",
+        "* `tf.experimental.numpy.ceil`\n",
+        "* `tf.experimental.numpy.conj`\n",
+        "* `tf.experimental.numpy.conjugate`\n",
+        "* `tf.experimental.numpy.copy`\n",
+        "* `tf.experimental.numpy.cos`\n",
+        "* `tf.experimental.numpy.cosh`\n",
+        "* `tf.experimental.numpy.cumprod`\n",
+        "* `tf.experimental.numpy.cumsum`\n",
+        "* `tf.experimental.numpy.deg2rad`\n",
+        "* `tf.experimental.numpy.diag`\n",
+        "* `tf.experimental.numpy.diagflat`\n",
+        "* `tf.experimental.numpy.diagonal`\n",
+        "* `tf.experimental.numpy.diff`\n",
+        "* `tf.experimental.numpy.empty_like`\n",
+        "* `tf.experimental.numpy.exp2`\n",
+        "* `tf.experimental.numpy.exp`\n",
+        "* `tf.experimental.numpy.expand_dims`\n",
+        "* `tf.experimental.numpy.expm1`\n",
+        "* `tf.experimental.numpy.fabs`\n",
+        "* `tf.experimental.numpy.fix`\n",
+        "* `tf.experimental.numpy.flatten`\n",
+        "* `tf.experimental.numpy.flip`\n",
+        "* `tf.experimental.numpy.fliplr`\n",
+        "* `tf.experimental.numpy.flipud`\n",
+        "* `tf.experimental.numpy.floor`\n",
+        "* `tf.experimental.numpy.full_like`\n",
+        "* `tf.experimental.numpy.imag`\n",
+        "* `tf.experimental.numpy.log10`\n",
+        "* `tf.experimental.numpy.log1p`\n",
+        "* `tf.experimental.numpy.log2`\n",
+        "* `tf.experimental.numpy.log`\n",
+        "* `tf.experimental.numpy.max`\n",
+        "* `tf.experimental.numpy.mean`\n",
+        "* `tf.experimental.numpy.min`\n",
+        "* `tf.experimental.numpy.moveaxis`\n",
+        "* `tf.experimental.numpy.nanmean`\n",
+        "* `tf.experimental.numpy.negative`\n",
+        "* `tf.experimental.numpy.ones_like`\n",
+        "* `tf.experimental.numpy.positive`\n",
+        "* `tf.experimental.numpy.prod`\n",
+        "* `tf.experimental.numpy.rad2deg`\n",
+        "* `tf.experimental.numpy.ravel`\n",
+        "* `tf.experimental.numpy.real`\n",
+        "* `tf.experimental.numpy.reciprocal`\n",
+        "* `tf.experimental.numpy.repeat`\n",
+        "* `tf.experimental.numpy.reshape`\n",
+        "* `tf.experimental.numpy.rot90`\n",
+        "* `tf.experimental.numpy.round`\n",
+        "* `tf.experimental.numpy.signbit`\n",
+        "* `tf.experimental.numpy.sin`\n",
+        "* `tf.experimental.numpy.sinc`\n",
+        "* `tf.experimental.numpy.sinh`\n",
+        "* `tf.experimental.numpy.sort`\n",
+        "* `tf.experimental.numpy.sqrt`\n",
+        "* `tf.experimental.numpy.square`\n",
+        "* `tf.experimental.numpy.squeeze`\n",
+        "* `tf.experimental.numpy.std`\n",
+        "* `tf.experimental.numpy.sum`\n",
+        "* `tf.experimental.numpy.swapaxes`\n",
+        "* `tf.experimental.numpy.tan`\n",
+        "* `tf.experimental.numpy.tanh`\n",
+        "* `tf.experimental.numpy.trace`\n",
+        "* `tf.experimental.numpy.transpose`\n",
+        "* `tf.experimental.numpy.triu`\n",
+        "* `tf.experimental.numpy.vander`\n",
+        "* `tf.experimental.numpy.var`\n",
+        "* `tf.experimental.numpy.zeros_like`"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "name": "tf_numpy_type_promotion.ipynb",
+      "provenance": [],
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/tpu.ipynb b/site/en/guide/tpu.ipynb
new file mode 100644
index 00000000000..49eee544bec
--- /dev/null
+++ b/site/en/guide/tpu.ipynb
@@ -0,0 +1,603 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2024 The TensorFlow Authors.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "# Use TPUs\n",
+        "\n",
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ys81cOhXOWUP"
+      },
+      "source": [
+        "This guide demonstrates how to perform basic training on [Tensor Processing Units (TPUs)](https://cloud.google.com/tpu/) and TPU Pods, a collection of TPU devices connected by dedicated high-speed network interfaces, with `tf.keras` and custom training loops.\n",
+        "\n",
+        "TPUs are Google's custom-developed application-specific integrated circuits (ASICs) used to accelerate machine learning workloads. They are available through [Google Colab](https://colab.research.google.com/), the [TPU Research Cloud](https://sites.research.google/trc/), and [Cloud TPU](https://cloud.google.com/tpu)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ek5Hop74NVKm"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ebf7f8489bb7"
+      },
+      "source": [
+        "Before you run this Colab notebook, make sure that your hardware accelerator is a TPU by checking your notebook settings: **Runtime** > **Change runtime type** > **Hardware accelerator** > **TPU v2**.\n",
+        "\n",
+        "Import some necessary libraries, including TensorFlow Datasets:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Cw0WRaChRxTL"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "import os\n",
+        "import tensorflow_datasets as tfds"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yDWaRxSpwBN1"
+      },
+      "source": [
+        "## TPU initialization\n",
+        "\n",
+        "TPUs are typically [Cloud TPU](https://cloud.google.com/tpu/docs/) workers, which are different from the local process running the user's Python program. Thus, you need to do some initialization work to connect to the remote cluster and initialize the TPUs. Note that the `tpu` argument to `tf.distribute.cluster_resolver.TPUClusterResolver` is a special address just for Colab. If you are running your code on Google Compute Engine (GCE), you should instead pass in the name of your Cloud TPU."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dCqWMqvtwOLs"
+      },
+      "source": [
+        "Note: The TPU initialization code has to be at the beginning of your program."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dKPqF8d1wJCV"
+      },
+      "outputs": [],
+      "source": [
+        "resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='local')\n",
+        "tf.config.experimental_connect_to_cluster(resolver)\n",
+        "# This is the TPU initialization code that has to be at the beginning.\n",
+        "tf.tpu.experimental.initialize_tpu_system(resolver)\n",
+        "print(\"All devices: \", tf.config.list_logical_devices('TPU'))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Mv7kehTZ1Lq_"
+      },
+      "source": [
+        "## Manual device placement\n",
+        "\n",
+        "After the TPU is initialized, you can use manual device placement to place the computation on a single TPU device:\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XRZ4kMoxBNND"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
+        "b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
+        "\n",
+        "with tf.device('/TPU:0'):\n",
+        "  c = tf.matmul(a, b)\n",
+        "\n",
+        "print(\"c device: \", c.device)\n",
+        "print(c)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_NJm-kgFO0cC"
+      },
+      "source": [
+        "## Distribution strategies\n",
+        "\n",
+        "Usually, you run your model on multiple TPUs in a data-parallel way. To distribute your model on multiple TPUs (as well as multiple GPUs or multiple machines), TensorFlow offers the `tf.distribute.Strategy` API. You can replace your distribution strategy and the model will run on any given (TPU) device. Learn more in the [Distributed training with TensorFlow](./distributed_training.ipynb) guide."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DcDPMZs-9uLJ"
+      },
+      "source": [
+        "Using the `tf.distribute.TPUStrategy` option implements synchronous distributed training. TPUs provide their own implementation of efficient all-reduce and other collective operations across multiple TPU cores, which are used in `TPUStrategy`.\n",
+        "\n",
+        "To demonstrate this, create a `tf.distribute.TPUStrategy` object:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7SO23K8oRpjI"
+      },
+      "outputs": [],
+      "source": [
+        "strategy = tf.distribute.TPUStrategy(resolver)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JlaAmswWPsU6"
+      },
+      "source": [
+        "To replicate a computation so it can run in all TPU cores, you can pass it into the `Strategy.run` API. Below is an example that shows all cores receiving the same inputs `(a, b)` and performing matrix multiplication on each core independently. The outputs will be the values from all the replicas."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-90CL5uFPTOa"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def matmul_fn(x, y):\n",
+        "  z = tf.matmul(x, y)\n",
+        "  return z\n",
+        "\n",
+        "z = strategy.run(matmul_fn, args=(a, b))\n",
+        "print(z)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uxgYl6kGHJLc"
+      },
+      "source": [
+        "## Classification on TPUs\n",
+        "\n",
+        "Having covered the basic concepts, consider a more concrete example. This section demonstrates how to use the distribution strategy—`tf.distribute.TPUStrategy`—to train a Keras model on a Cloud TPU."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gKRALGgt_kCo"
+      },
+      "source": [
+        "### Define a Keras model\n",
+        "\n",
+        "Start with a definition of a [`Sequential` Keras model](https://www.tensorflow.org/guide/keras/sequential_model) for image classification on the MNIST dataset. It's no different than what you would use if you were training on CPUs or GPUs. Note that Keras model creation needs to be inside the `Strategy.scope`, so the variables can be created on each TPU device. Other parts of the code are not necessary to be inside the `Strategy` scope."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DiBiN-Z_R7P7"
+      },
+      "outputs": [],
+      "source": [
+        "def create_model():\n",
+        "  regularizer = tf.keras.regularizers.L2(1e-5)\n",
+        "  return tf.keras.Sequential(\n",
+        "      [tf.keras.layers.Conv2D(256, 3, input_shape=(28, 28, 1),\n",
+        "                              activation='relu',\n",
+        "                              kernel_regularizer=regularizer),\n",
+        "       tf.keras.layers.Conv2D(256, 3,\n",
+        "                              activation='relu',\n",
+        "                              kernel_regularizer=regularizer),\n",
+        "       tf.keras.layers.Flatten(),\n",
+        "       tf.keras.layers.Dense(256,\n",
+        "                             activation='relu',\n",
+        "                             kernel_regularizer=regularizer),\n",
+        "       tf.keras.layers.Dense(128,\n",
+        "                             activation='relu',\n",
+        "                             kernel_regularizer=regularizer),\n",
+        "       tf.keras.layers.Dense(10,\n",
+        "                             kernel_regularizer=regularizer)])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h-2qaXgfyONQ"
+      },
+      "source": [
+        "This model puts L2 regularization terms on the weights of each layer, so that the custom training loop below can show how you pick them up from `Model.losses`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qYOYjYTg_31l"
+      },
+      "source": [
+        "### Load the dataset\n",
+        "\n",
+        "Efficient use of the `tf.data.Dataset` API is critical when using a Cloud TPU. You can learn more about dataset performance in the [Input pipeline performance guide](./data_performance.ipynb).\n",
+        "\n",
+        "If you are using [TPU Nodes](https://cloud.google.com/tpu/docs/managing-tpus-tpu-vm), you need to store all data files read by the TensorFlow `Dataset` in [Google Cloud Storage (GCS) buckets](https://cloud.google.com/tpu/docs/storage-buckets). If you are using [TPU VMs](https://cloud.google.com/tpu/docs/users-guide-tpu-vm), you can store data wherever you like. For more information on TPU Nodes and TPU VMs, refer to the [TPU System Architecture](https://cloud.google.com/tpu/docs/system-architecture-tpu-vm) documentation.\n",
+        "\n",
+        "For most use cases, it is recommended to convert your data into the `TFRecord` format and use a `tf.data.TFRecordDataset` to read it. Check the [TFRecord and tf.Example tutorial](../tutorials/load_data/tfrecord.ipynb) for details on how to do this. It is not a hard requirement and you can use other dataset readers, such as `tf.data.FixedLengthRecordDataset` or `tf.data.TextLineDataset`.\n",
+        "\n",
+        "You can load entire small datasets into memory using `tf.data.Dataset.cache`.\n",
+        "\n",
+        "Regardless of the data format used, it is strongly recommended that you use large files on the order of 100MB. This is especially important in this networked setting, as the overhead of opening a file is significantly higher.\n",
+        "\n",
+        "As shown in the code below, you should use the Tensorflow Datasets `tfds.load` module to get a copy of the MNIST training and test data. Note that `try_gcs` is specified to use a copy that is available in a public GCS bucket. If you don't specify this, the TPU will not be able to access the downloaded data."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "noAd416KSCo7"
+      },
+      "outputs": [],
+      "source": [
+        "def get_dataset(batch_size, is_training=True):\n",
+        "  split = 'train' if is_training else 'test'\n",
+        "  dataset, info = tfds.load(name='mnist', split=split, with_info=True,\n",
+        "                            as_supervised=True, try_gcs=True)\n",
+        "\n",
+        "  # Normalize the input data.\n",
+        "  def scale(image, label):\n",
+        "    image = tf.cast(image, tf.float32)\n",
+        "    image /= 255.0\n",
+        "    return image, label\n",
+        "\n",
+        "  dataset = dataset.map(scale)\n",
+        "\n",
+        "  # Only shuffle and repeat the dataset in training. The advantage of having an\n",
+        "  # infinite dataset for training is to avoid the potential last partial batch\n",
+        "  # in each epoch, so that you don't need to think about scaling the gradients\n",
+        "  # based on the actual batch size.\n",
+        "  if is_training:\n",
+        "    dataset = dataset.shuffle(10000)\n",
+        "    dataset = dataset.repeat()\n",
+        "\n",
+        "  dataset = dataset.batch(batch_size)\n",
+        "\n",
+        "  return dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mgUC6A-zCMEr"
+      },
+      "source": [
+        "### Train the model using Keras high-level APIs\n",
+        "\n",
+        "You can train your model with Keras `Model.fit` and `Model.compile` APIs. There is nothing TPU-specific in this step—you write the code as if you were using multiple GPUs and a `MirroredStrategy` instead of the `TPUStrategy`. You can learn more in the [Distributed training with Keras](../tutorials/distribute/keras.ipynb) tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ubmDchPqSIx0"
+      },
+      "outputs": [],
+      "source": [
+        "with strategy.scope():\n",
+        "  model = create_model()\n",
+        "  model.compile(optimizer='adam',\n",
+        "                loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+        "                metrics=['sparse_categorical_accuracy'])\n",
+        "\n",
+        "batch_size = 200\n",
+        "steps_per_epoch = 60000 // batch_size\n",
+        "validation_steps = 10000 // batch_size\n",
+        "\n",
+        "train_dataset = get_dataset(batch_size, is_training=True)\n",
+        "test_dataset = get_dataset(batch_size, is_training=False)\n",
+        "\n",
+        "model.fit(train_dataset,\n",
+        "          epochs=5,\n",
+        "          steps_per_epoch=steps_per_epoch,\n",
+        "          validation_data=test_dataset,\n",
+        "          validation_steps=validation_steps)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8hSGBIYtUugJ"
+      },
+      "source": [
+        "To reduce Python overhead and maximize the performance of your TPU, pass in the `steps_per_execution` argument to Keras `Model.compile`. In this example, it increases throughput by about 50%:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "M6e3aVVLUorL"
+      },
+      "outputs": [],
+      "source": [
+        "with strategy.scope():\n",
+        "  model = create_model()\n",
+        "  model.compile(optimizer='adam',\n",
+        "                # Anything between 2 and `steps_per_epoch` could help here.\n",
+        "                steps_per_execution = 50,\n",
+        "                loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+        "                metrics=['sparse_categorical_accuracy'])\n",
+        "\n",
+        "model.fit(train_dataset,\n",
+        "          epochs=5,\n",
+        "          steps_per_epoch=steps_per_epoch,\n",
+        "          validation_data=test_dataset,\n",
+        "          validation_steps=validation_steps)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0rRALBZNCO4A"
+      },
+      "source": [
+        "### Train the model using a custom training loop\n",
+        "\n",
+        "You can also create and train your model using `tf.function` and `tf.distribute` APIs directly. You can use the `Strategy.distribute_datasets_from_function` API to distribute the `tf.data.Dataset` given a dataset function. Note that in the example below the batch size passed into the `Dataset` is the per-replica batch size instead of the global batch size. To learn more, check out the [Custom training with `tf.distribute.Strategy`](../tutorials/distribute/custom_training.ipynb) tutorial.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DxdgXPAL6iFE"
+      },
+      "source": [
+        "First, create the model, datasets and `tf.function`s:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9aHhqwao2Fxi"
+      },
+      "outputs": [],
+      "source": [
+        "# Create the model, optimizer and metrics inside the `tf.distribute.Strategy`\n",
+        "# scope, so that the variables can be mirrored on each device.\n",
+        "with strategy.scope():\n",
+        "  model = create_model()\n",
+        "  optimizer = tf.keras.optimizers.Adam()\n",
+        "  training_loss = tf.keras.metrics.Mean('training_loss', dtype=tf.float32)\n",
+        "  training_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n",
+        "      'training_accuracy', dtype=tf.float32)\n",
+        "\n",
+        "# Calculate per replica batch size, and distribute the `tf.data.Dataset`s\n",
+        "# on each TPU worker.\n",
+        "per_replica_batch_size = batch_size // strategy.num_replicas_in_sync\n",
+        "\n",
+        "train_dataset = strategy.distribute_datasets_from_function(\n",
+        "    lambda _: get_dataset(per_replica_batch_size, is_training=True))\n",
+        "\n",
+        "@tf.function\n",
+        "def train_step(iterator):\n",
+        "  \"\"\"The step function for one training step.\"\"\"\n",
+        "\n",
+        "  def step_fn(inputs):\n",
+        "    \"\"\"The computation to run on each TPU device.\"\"\"\n",
+        "    images, labels = inputs\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      logits = model(images, training=True)\n",
+        "      per_example_loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
+        "          labels, logits, from_logits=True)\n",
+        "      loss = tf.nn.compute_average_loss(per_example_loss)\n",
+        "      model_losses = model.losses\n",
+        "      if model_losses:\n",
+        "        loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n",
+        "\n",
+        "    grads = tape.gradient(loss, model.trainable_variables)\n",
+        "    optimizer.apply_gradients(list(zip(grads, model.trainable_variables)))\n",
+        "    training_loss.update_state(loss * strategy.num_replicas_in_sync)\n",
+        "    training_accuracy.update_state(labels, logits)\n",
+        "\n",
+        "  strategy.run(step_fn, args=(next(iterator),))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ibi7Z97V6xsQ"
+      },
+      "source": [
+        "Then, run the training loop:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1du5cXWt6Vtw"
+      },
+      "outputs": [],
+      "source": [
+        "steps_per_eval = 10000 // batch_size\n",
+        "\n",
+        "train_iterator = iter(train_dataset)\n",
+        "for epoch in range(5):\n",
+        "  print('Epoch: {}/5'.format(epoch))\n",
+        "\n",
+        "  for step in range(steps_per_epoch):\n",
+        "    train_step(train_iterator)\n",
+        "  print('Current step: {}, training loss: {}, training accuracy: {}%'.format(\n",
+        "      optimizer.iterations.numpy(),\n",
+        "      round(float(training_loss.result()), 4),\n",
+        "      round(float(training_accuracy.result()) * 100, 2)))\n",
+        "  training_loss.reset_states()\n",
+        "  training_accuracy.reset_states()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TnZJUM3qIjKu"
+      },
+      "source": [
+        "### Improving performance with multiple steps inside `tf.function`\n",
+        "\n",
+        "You can improve the performance by running multiple steps within a `tf.function`. This is achieved by wrapping the `Strategy.run` call with a `tf.range` inside `tf.function`, and AutoGraph will convert it to a `tf.while_loop` on the TPU worker. You can learn more about `tf.function`s in the [Better performance with `tf.function`](./function.ipynb) guide.\n",
+        "\n",
+        "Despite the improved performance, there are tradeoffs with this method compared to running a single step inside a `tf.function`. Running multiple steps in a `tf.function` is less flexible—you cannot run things eagerly or arbitrary Python code within the steps.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2grYvXLzJYkP"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function\n",
+        "def train_multiple_steps(iterator, steps):\n",
+        "  \"\"\"The step function for one training step.\"\"\"\n",
+        "\n",
+        "  def step_fn(inputs):\n",
+        "    \"\"\"The computation to run on each TPU device.\"\"\"\n",
+        "    images, labels = inputs\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      logits = model(images, training=True)\n",
+        "      per_example_loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
+        "          labels, logits, from_logits=True)\n",
+        "      loss = tf.nn.compute_average_loss(per_example_loss)\n",
+        "      model_losses = model.losses\n",
+        "      if model_losses:\n",
+        "        loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n",
+        "    grads = tape.gradient(loss, model.trainable_variables)\n",
+        "    optimizer.apply_gradients(list(zip(grads, model.trainable_variables)))\n",
+        "    training_loss.update_state(loss * strategy.num_replicas_in_sync)\n",
+        "    training_accuracy.update_state(labels, logits)\n",
+        "\n",
+        "  for _ in tf.range(steps):\n",
+        "    strategy.run(step_fn, args=(next(iterator),))\n",
+        "\n",
+        "# Convert `steps_per_epoch` to `tf.Tensor` so the `tf.function` won't get\n",
+        "# retraced if the value changes.\n",
+        "train_multiple_steps(train_iterator, tf.convert_to_tensor(steps_per_epoch))\n",
+        "\n",
+        "print('Current step: {}, training loss: {}, training accuracy: {}%'.format(\n",
+        "      optimizer.iterations.numpy(),\n",
+        "      round(float(training_loss.result()), 4),\n",
+        "      round(float(training_accuracy.result()) * 100, 2)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WBKVhMvWjibf"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "To learn more about Cloud TPUs and how to use them:\n",
+        "\n",
+        "- [Google Cloud TPU](https://cloud.google.com/tpu): The Google Cloud TPU homepage.\n",
+        "- [Google Cloud TPU documentation](https://cloud.google.com/tpu/docs/): Google Cloud TPU documentation, which includes:\n",
+        "  - [Introduction to Cloud TPU](https://cloud.google.com/tpu/docs/intro-to-tpu): An overview of working with Cloud TPUs.\n",
+        "  - [Cloud TPU quickstarts](https://cloud.google.com/tpu/docs/quick-starts): Quickstart introductions to working with Cloud TPU VMs using TensorFlow and other main machine learning frameworks.\n",
+        "- [Google Cloud TPU Colab notebooks](https://cloud.google.com/tpu/docs/colabs): End-to-end training examples.\n",
+        "- [Google Cloud TPU performance guide](https://cloud.google.com/tpu/docs/performance-guide): Enhance Cloud TPU performance further by adjusting Cloud TPU configuration parameters for your application\n",
+        "- [Distributed training with TensorFlow](./distributed_training.ipynb): How to use distribution strategies—including `tf.distribute.TPUStrategy`—with examples showing best practices.\n",
+        "- TPU embeddings: TensorFlow includes specialized support for training embeddings on TPUs via `tf.tpu.experimental.embedding`. In addition, [TensorFlow Recommenders](https://www.tensorflow.org/recommenders) has `tfrs.layers.embedding.TPUEmbedding`. Embeddings provide efficient and dense representations, capturing complex similarities and relationships between features. TensorFlow's TPU-specific embedding support allows you to train embeddings that are larger than the memory of a single TPU device, and to use sparse and ragged inputs on TPUs.\n",
+        "- [TPU Research Cloud (TRC)](https://sites.research.google/trc/about/): TRC enables researchers to apply for access to a cluster of more than 1,000 Cloud TPU devices.\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "TPU",
+    "colab": {
+      "name": "tpu.ipynb",
+      "toc_visible": true,
+      "machine_shape": "hm",
+      "gpuType": "V28"
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
\ No newline at end of file
diff --git a/site/en/guide/variable.ipynb b/site/en/guide/variable.ipynb
new file mode 100644
index 00000000000..868ee9119e2
--- /dev/null
+++ b/site/en/guide/variable.ipynb
@@ -0,0 +1,390 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Tce3stUlHN0L"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Authors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qFdPvlXBOdUN"
+      },
+      "source": [
+        "# Introduction to Variables"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View source on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AKhB9CMxndDs"
+      },
+      "source": [
+        "A TensorFlow **variable** is the recommended way to represent shared, persistent state your program manipulates. This guide covers how to create, update, and manage instances of `tf.Variable` in TensorFlow.\n",
+        "\n",
+        "Variables are created and tracked via the `tf.Variable` class. A `tf.Variable` represents a tensor whose value can be changed by running ops on it.  Specific ops allow you to read and modify the values of this tensor. Higher level libraries like `tf.keras` use `tf.Variable` to store model parameters. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xZoJJ4vdvTrD"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "This notebook discusses variable placement.  If you want to see on what device your variables are placed, uncomment this line."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7tUZJk7lDiGo"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "# Uncomment to see where your variables get placed (see below)\n",
+        "# tf.debugging.set_log_device_placement(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vORGXDarogWm"
+      },
+      "source": [
+        "## Create a variable\n",
+        "\n",
+        "To create a variable, provide an initial value.  The `tf.Variable` will have the same `dtype` as the initialization value."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dsYXSqleojj7"
+      },
+      "outputs": [],
+      "source": [
+        "my_tensor = tf.constant([[1.0, 2.0], [3.0, 4.0]])\n",
+        "my_variable = tf.Variable(my_tensor)\n",
+        "\n",
+        "# Variables can be all kinds of types, just like tensors\n",
+        "bool_variable = tf.Variable([False, False, False, True])\n",
+        "complex_variable = tf.Variable([5 + 4j, 6 + 1j])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VQHwJ_Itoujf"
+      },
+      "source": [
+        "A variable looks and acts like a tensor, and, in fact, is a data structure backed by a `tf.Tensor`.  Like tensors, they have a `dtype` and a shape, and can be exported to NumPy."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GhNfPwCYpvlq"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Shape: \", my_variable.shape)\n",
+        "print(\"DType: \", my_variable.dtype)\n",
+        "print(\"As NumPy: \", my_variable.numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eZmSBYViqDoU"
+      },
+      "source": [
+        "Most tensor operations work on variables as expected, although variables cannot be reshaped."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TrIaExVNp_LK"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"A variable:\", my_variable)\n",
+        "print(\"\\nViewed as a tensor:\", tf.convert_to_tensor(my_variable))\n",
+        "print(\"\\nIndex of highest value:\", tf.math.argmax(my_variable))\n",
+        "\n",
+        "# This creates a new tensor; it does not reshape the variable.\n",
+        "print(\"\\nCopying and reshaping: \", tf.reshape(my_variable, [1,4]))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qbLCcG6Pc29Y"
+      },
+      "source": [
+        "As noted above, variables are backed by tensors. You can reassign the tensor using `tf.Variable.assign`.  Calling `assign` does not (usually) allocate a new tensor; instead, the existing tensor's memory is reused."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yeEpO309QbB2"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable([2.0, 3.0])\n",
+        "# This will keep the same dtype, float32\n",
+        "a.assign([1, 2]) \n",
+        "# Not allowed as it resizes the variable: \n",
+        "try:\n",
+        "  a.assign([1.0, 2.0, 3.0])\n",
+        "except Exception as e:\n",
+        "  print(f\"{type(e).__name__}: {e}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "okeywjLdQ1tY"
+      },
+      "source": [
+        "If you use a variable like a tensor in operations, you will usually operate on the backing tensor.  \n",
+        "\n",
+        "Creating new variables from existing variables duplicates the backing tensors. Two variables will not share the same memory."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2CnfGc6ucbXc"
+      },
+      "outputs": [],
+      "source": [
+        "a = tf.Variable([2.0, 3.0])\n",
+        "# Create b based on the value of a\n",
+        "b = tf.Variable(a)\n",
+        "a.assign([5, 6])\n",
+        "\n",
+        "# a and b are different\n",
+        "print(a.numpy())\n",
+        "print(b.numpy())\n",
+        "\n",
+        "# There are other versions of assign\n",
+        "print(a.assign_add([2,3]).numpy())  # [7. 9.]\n",
+        "print(a.assign_sub([7,9]).numpy())  # [0. 0.]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZtzepotYUe7B"
+      },
+      "source": [
+        "## Lifecycles, naming, and watching\n",
+        "\n",
+        "In Python-based TensorFlow, `tf.Variable` instance have the same lifecycle as other Python objects. When there are no references to a variable it is automatically deallocated.\n",
+        "\n",
+        "Variables can also be named which can help you track and debug them.  You can give two variables the same name."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VBFbzKj8RaPf"
+      },
+      "outputs": [],
+      "source": [
+        "# Create a and b; they will have the same name but will be backed by\n",
+        "# different tensors.\n",
+        "a = tf.Variable(my_tensor, name=\"Mark\")\n",
+        "# A new variable with the same name, but different value\n",
+        "# Note that the scalar add is broadcast\n",
+        "b = tf.Variable(my_tensor + 1, name=\"Mark\")\n",
+        "\n",
+        "# These are elementwise-unequal, despite having the same name\n",
+        "print(a == b)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "789QikItVA_E"
+      },
+      "source": [
+        "Variable names are preserved when saving and loading models. By default, variables in models will acquire unique variable names automatically, so you don't need to assign them yourself unless you want to.\n",
+        "\n",
+        "Although variables are important for differentiation, some variables will not need to be differentiated.  You can turn off gradients for a variable by setting `trainable` to false at creation. An example of a variable that would not need gradients is a training step counter."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B5Sj1DqhbZvx"
+      },
+      "outputs": [],
+      "source": [
+        "step_counter = tf.Variable(1, trainable=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DD_xfDLDTDNU"
+      },
+      "source": [
+        "## Placing variables and tensors\n",
+        "\n",
+        "For better performance, TensorFlow will attempt to place tensors and variables on the fastest device compatible with its `dtype`.  This means most variables are placed on a GPU if one is available.\n",
+        "\n",
+        "However, you can override this.  In this snippet, place a float tensor and a variable on the CPU, even if a GPU is available.  By turning on device placement logging (see [Setup](#scrollTo=xZoJJ4vdvTrD)), you can see where the variable is placed. \n",
+        "\n",
+        "Note: Although manual placement works, using [distribution strategies](distributed_training.ipynb) can be a more convenient and scalable way to optimize your computation.\n",
+        "\n",
+        "If you run this notebook on different backends with and without a GPU you will see different logging.  *Note that logging device placement must be turned on at the start of the session.*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2SjpD7wVUSBJ"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.device('CPU:0'):\n",
+        "\n",
+        "  # Create some tensors\n",
+        "  a = tf.Variable([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
+        "  b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
+        "  c = tf.matmul(a, b)\n",
+        "\n",
+        "print(c)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PXbh-p2BXKcr"
+      },
+      "source": [
+        "It's possible to set the location of a variable or tensor on one device and do the computation on another device.  This will introduce delay, as data needs to be copied between the devices.\n",
+        "\n",
+        "You might do this, however, if you had multiple GPU workers but only want one copy of the variables."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dgWHN3QSfNiQ"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.device('CPU:0'):\n",
+        "  a = tf.Variable([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
+        "  b = tf.Variable([[1.0, 2.0, 3.0]])\n",
+        "\n",
+        "with tf.device('GPU:0'):\n",
+        "  # Element-wise multiply\n",
+        "  k = a * b\n",
+        "\n",
+        "print(k)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fksvRaqoYfay"
+      },
+      "source": [
+        "Note: Because `tf.config.set_soft_device_placement` is turned on by default, even if you run this code on a device without a GPU, it will still run.  The multiplication step will happen on the CPU.\n",
+        "\n",
+        "For more on distributed training, refer to the [guide](distributed_training.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SzCkWlF2S4yo"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "To understand how variables are typically used, see our guide on [automatic differentiation](autodiff.ipynb)."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "variable.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/guide/variable.md b/site/en/guide/variable.md
deleted file mode 100644
index a5a7f4a4f28..00000000000
--- a/site/en/guide/variable.md
+++ /dev/null
@@ -1,110 +0,0 @@
-# TensorFlow variables
-
-A TensorFlow **variable** is the best way to represent shared, persistent state
-manipulated by your program.
-
-Variables are manipulated via the `tf.Variable` class. A `tf.Variable`
-represents a tensor whose value can be changed by running ops on it.  Specific
-ops allow you to read and modify the values of this tensor. Higher level
-libraries like `tf.keras` use `tf.Variable` to store model parameters. This
-guide covers how to create, update, and manage `tf.Variable`s in TensorFlow.
-
-## Create a variable
-
-To create a variable, simply provide the initial value
-
-``` python
-my_variable = tf.Variable(tf.zeros([1., 2., 3.]))
-```
-
-This creates a variable which is a three-dimensional tensor with shape `[1, 2,
-3]` filled with zeros. This variable will, by default, have the `dtype`
-`tf.float32`. The dtype is, if not specified, inferred from the initial
-value.
-
-If there's a `tf.device` scope active, the variable will be placed on that
-device; otherwise the variable will be placed on the "fastest" device compatible
-with its dtype (this means most variables are automatically placed on a GPU if
-one is available). For example, the following snippet creates a variable named
-`v` and places it on the second GPU device:
-
-``` python
-with tf.device("/device:GPU:1"):
-  v = tf.Variable(tf.zeros([10, 10]))
-```
-
-Ideally though you should use the `tf.distribute` API, as that allows you to
-write your code once and have it work under many different distributed setups.
-
-## Use a variable
-
-To use the value of a `tf.Variable` in a TensorFlow graph, simply treat it like
-a normal `tf.Tensor`:
-
-``` python
-v = tf.Variable(0.0)
-w = v + 1  # w is a tf.Tensor which is computed based on the value of v.
-           # Any time a variable is used in an expression it gets automatically
-           # converted to a tf.Tensor representing its value.
-```
-
-To assign a value to a variable, use the methods `assign`, `assign_add`, and
-friends in the `tf.Variable` class. For example, here is how you can call these
-methods:
-
-``` python
-v = tf.Variable(0.0)
-v.assign_add(1)
-```
-
-Most TensorFlow optimizers have specialized ops that efficiently update the
-values of variables according to some gradient descent-like algorithm. See
-`tf.keras.optimizers.Optimizer` for an explanation of how to use optimizers.
-
-You can also explicitly read the current value of a variable, using
-`read_value`:
-
-```python
-v = tf.Variable(0.0)
-v.assign_add(1)
-v.read_value()  # 1.0
-```
-
-When the last reference to a `tf.Variable` goes out of scope its memory is
-freed.
-
-### Keep track of variables
-
-A Variable in TensorFlow is a Python object. As you build your layers, models,
-optimizers, and other related tools, you will likely want to get a list of all
-variables in a (say) model.
-
-A common use case is [implementing `Layer` subclasses](
-https://www.tensorflow.org/guide/keras/custom_layers_and_models#the_layer_class).
-The `Layer` class recursively tracks variables set as instance attributes:
-
-```python
-class MyLayer(tf.keras.layers.Layer):
-
-  def __init__(self):
-    super(MyLayer, self).__init__()
-    self.my_var = tf.Variable(1.0)
-    self.my_var_list = [tf.Variable(x) for x in range(10)]
-
-class MyOtherLayer(tf.keras.layers.Layer):
-
-  def __init__(self):
-    super(MyOtherLayer, self).__init__()
-    self.sublayer = MyLayer()
-    self.my_other_var = tf.Variable(10.0)
-
-m = MyOtherLayer()
-print(len(m.variables))  # 12 (11 from MyLayer, plus my_other_var)
-```
-
-If you aren't developing a new `Layer`, TensorFlow also features a more
-generic `tf.Module` base class which _only_ implements variable tracking.
-Instances of `tf.Module` have a `variables` and a `trainable_variables`
-property which return all (trainable) variables reachable from that model,
-potentially navigating through other modules (much like the tracking done by
-the `Layer` class).
diff --git a/site/en/guide/versions.md b/site/en/guide/versions.md
index 00ead902ec8..5b1206cc5f4 100644
--- a/site/en/guide/versions.md
+++ b/site/en/guide/versions.md
@@ -6,10 +6,11 @@ to modify TensorFlow while preserving compatibility.
 
 ## Semantic versioning 2.0
 
-TensorFlow follows Semantic Versioning 2.0 ([semver](http://semver.org)) for its
-public API. Each release version of TensorFlow has the form `MAJOR.MINOR.PATCH`.
-For example, TensorFlow version 1.2.3 has `MAJOR` version 1, `MINOR` version 2,
-and `PATCH` version 3. Changes to each number have the following meaning:
+TensorFlow mostly follows Semantic Versioning 2.0 ([semver](http://semver.org))
+for its public API. Each release version of TensorFlow has the form
+`MAJOR.MINOR.PATCH`. For example, TensorFlow version 1.2.3 has `MAJOR` version
+1, `MINOR` version 2, and `PATCH` version 3. Changes to each number have the
+following meaning:
 
 * **MAJOR**:  Potentially backwards incompatible changes.  Code and data that
   worked with a previous major release will not necessarily work with the new
@@ -22,6 +23,10 @@ and `PATCH` version 3. Changes to each number have the following meaning:
   data that worked with a previous minor release *and* which depends only on the
   non-experimental public API will continue to work unchanged.  For details on
   what is and is not the public API, see [What is covered](#what_is_covered).
+  Note that TensorFlow sometimes makes breaking changes in new minor releases,
+  where the impact is expected to be minor. For examples of these kinds of
+  changes, see the "Breaking Changes" sections for past minor releases at
+  https://github.com/tensorflow/tensorflow/releases.
 
 * **PATCH**: Backwards compatible bug fixes.
 
@@ -34,44 +39,153 @@ release 0.12.1.  However, release 1.1.1 was backwards *compatible* with release
 Only the public APIs of TensorFlow are backwards compatible across minor and
 patch versions.  The public APIs consist of
 
-* All the documented [Python](../api_docs/python) functions and classes in the
-  `tensorflow` module and its submodules, except for
+*   All the documented [Python](https://www.tensorflow.org/api_docs/python)
+    functions and classes in the `tensorflow` module and its submodules, except
+    for
+
+    *   Private symbols: any function, class, etc., whose name start with `_`
+    *   Experimental and `tf.contrib` symbols, see [below](#not_covered) for
+        details.
+
+    Note that the code in the `examples/` and `tools/` directories is not
+    reachable through the `tensorflow` Python module and is thus not covered by
+    the compatibility guarantee.
+
+    If a symbol is available through the `tensorflow` Python module or its
+    submodules, but is not documented, then it is **not** considered part of the
+    public API.
+
+*   The compatibility API (in Python, the `tf.compat` module). At major
+    versions, we may release utilities and additional endpoints to help users
+    with the transition to a new major version. These API symbols are deprecated
+    and not supported (i.e., we will not add any features, and we will not fix
+    bugs other than to fix vulnerabilities), but they do fall under our
+    compatibility guarantees.
+
+*   The TensorFlow C API:
+
+    *   [tensorflow/c/c_api.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/c_api.h)
+
+*   The following protocol buffer files:
+
+    *   [`attr_value`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto)
+    *   [`config`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto)
+    *   [`event`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/event.proto)
+    *   [`graph`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto)
+    *   [`op_def`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def.proto)
+    *   [`reader_base`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/reader_base.proto)
+    *   [`summary`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto)
+    *   [`tensor`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto)
+    *   [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto)
+    *   [`types`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto)
+
+
+
+## Separate version number for TensorFlow Lite
+
+Currently TensorFlow Lite is distributed as a part of TensorFlow. However, we
+reserve the right to in future release changes to the TensorFlow Lite APIs on a
+different schedule than for the other TensorFlow APIs, or even to move
+TensorFlow Lite into a separate source distribution and/or a separate source
+repository than TensorFlow.
+
+Because of this, we use a different version number for TensorFlow Lite
+(`TFLITE_VERSION_STRING` in `tensorflow/lite/version.h`, and `TfLiteVersion()`
+in `tensorflow/lite/c/c_api.h`) than for TensorFlow (`TF_VERSION_STRING` in
+`tensorflow/core/public/release_version.h`, and `TF_Version()` in
+`tensorflow/c/c_api.h`). Currently, these two version numbers happen to have the
+same value. But in future, they may diverge; for example, we may increment the
+major version number for TensorFlow Lite without incrementing the major version
+number for TensorFlow, or vice versa.
+
+The API surface that is covered by the TensorFlow Lite version number is
+comprised of the following public APIs:
+
+*   The TensorFlow Lite C API:
+
+    *   [tensorflow/lite/c/c_api.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/c_api.h)
+    *   [tensorflow/lite/c/c_api_types.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/c_api_types.h).
+
+*   The TensorFlow Lite Android (Java/Kotlin) API:
+
+    *   In `org.tensorflow.lite`:
+        *   [org.tensorflow.lite.TensorFlowLite](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/TensorFlowLite)
+        *   [org.tensorflow.lite.InterpreterApi](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/InterpreterApi)
+        *   [org.tensorflow.lite.Delegate](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Delegate)
+        *   [org.tensorflow.lite.DelegateFactory](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/DelegateFactory)
+        *   [org.tensorflow.lite.Tensor](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/Tensor)
+        *   [org.tensorflow.lite.DataType](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/DataType)
+        *   [org.tensorflow.lite.RuntimeFlavor](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/RuntimeFlavor)
+    *   In `org.tensorflow.lite.gpu`:
+        *   [org.tensorflow.lite.gpu.GpuDelegate](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/gpu/GpuDelegate)
+        *   [org.tensorflow.lite.gpu.GpuDelegateFactory](https://www.tensorflow.org/lite/api_docs/java/org/tensorflow/lite/gpu/GpuDelegateFactory)
+
+*   The TensorFlow Lite Objective-C APIs:
+
+    *   [tensorflow/lite/objc/apis/](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/objc/apis/)
+        *   TFLCoreMLDelegate.h
+        *   TFLDelegate.h
+        *   TFLInterpreter.h
+        *   TFLInterpreterOptions.h
+        *   TFLMetalDelegate.h
+        *   TFLQuantizationParameters.h
+        *   TFLSignatureRunner.h
+        *   TFLTensorFlowLite.h
+        *   TFLTensor.h
+
+*   The TensorFlow Lite Swift APIs:
+
+    *   [tensorflow/lite/swift/Sources/](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/swift/Sources/).
+        *   CoreMLDelegate.swift
+        *   Delegate.swift
+        *   InterpreterError.swift
+        *   Interpreter.swift
+        *   MetalDelegate.swift
+        *   Model.swift
+        *   QuantizationParameters.swift
+        *   SignatureRunnerError.swift
+        *   SignatureRunner.swift
+        *   TensorFlowLite.swift
+        *   Tensor.swift
+
+Experimental symbols are not covered; see [below](#not_covered) for details.
+
+## Separate version number for TensorFlow Lite Extension APIs
+
+TensorFlow Lite provides C APIs for extending the TensorFlow Lite interpreter
+with "custom ops", which provide user-defined operations in a graph, or
+"delegates", which allow delegating the computation for a graph (or for a subset
+of a graph) to a custom backend. These APIs, which we collectively call the
+"TensorFlow Lite Extension APIs", require more intimate dependencies on some of
+the details of the TensorFlow Lite implementation.
+
+We reserve the right to in future release changes to these APIs, potentially
+including non-backwards-compatible changes, on a different schedule than for the
+other TensorFlow Lite APIs. So we use a different version number for the
+TensorFlow Lite Extension APIs than the version numbers for TensorFlow Lite or
+TensorFlow (which were described in the previous section). We are introducing
+some new APIs in TensorFlow Lite version 2.15 to get the TensorFlow Lite
+Extension APIs version (`TFLITE_EXTENSION_APIS_VERSION_STRING` in
+`tensorflow/lite/version.h`, and TfLiteExtensionApisVersion() in
+`tensorflow/lite/c/c_api.h`). The version number for the TensorFlow Lite
+Extension APIs is currently the same as the version number for TensorFlow and
+TensorFlow Lite. But in future, they may diverge; for example, we may increment
+the major version number for the TensorFlow Lite Extension APIs without
+incrementing the major version number for TensorFlow Lite, or vice versa.
+
+The API surface that is covered by the TensorFlow Lite Extension APIs version
+number is comprised of the following public APIs:
+
+*   [tensorflow/lite/c/c_api_opaque.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/c_api_opaque.h)
+*   [tensorflow/lite/c/common.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/common.h)
+*   [tensorflow/lite/c/builtin_op_data.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/builtin_op_data.h)
+*   [tensorflow/lite/builtin_ops.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/builtin_ops.h)
+
+Again, experimental symbols are not covered; see [below](#not_covered) for
+details.
+
+
 
-    * Private symbols: any function, class, etc., whose name start with `_`
-    * Experimental and `tf.contrib` symbols, see [below](#not_covered) for
-      details.
-
-  Note that the code in the `examples/` and `tools/` directories is not
-  reachable through the `tensorflow` Python module and is thus not covered by
-  the compatibility guarantee.
-
-  If a symbol is available through the `tensorflow` Python module or its
-  submodules, but is not documented, then it is **not** considered part of the
-  public API.
-
-* The compatibility API (in Python, the `tf.compat` module). At major versions,
-  we may release utilities and additional endpoints to help users with the
-  transition to a new major version. These API symbols are deprecated and not
-  supported (i.e., we will not add any features, and we will not fix bugs
-  other than to fix vulnerabilities), but they do fall under our compatibility
-  guarantees.
-
-* The [C API](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/c_api.h).
-
-* The following protocol buffer files:
-
-    * [`attr_value`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto)
-    * [`config`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto)
-    * [`event`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/event.proto)
-    * [`graph`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto)
-    * [`op_def`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def.proto)
-    * [`reader_base`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/reader_base.proto)
-    * [`summary`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto)
-    * [`tensor`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto)
-    * [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto)
-    * [`types`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto)
-
-
 ## What is *not* covered
 
 Some parts of TensorFlow can change in backward incompatible ways at any point.
@@ -81,44 +195,48 @@ These include:
     clearly marked as experimental from the compatibility guarantees. In
     particular, the following are not covered by any compatibility guarantees:
 
-  - any symbol in the `tf.contrib` module or its submodules;
-  - any symbol (module, function, argument, property, class, or constant) whose
-    name contains `experimental` or `Experimental`; or
-  - any symbol whose fully qualified name includes a module or class which is
-    itself experimental. This includes fields and submessages of any protocol
-    buffer called `experimental`.
+    -   any symbol in the `tf.contrib` module or its submodules;
+    -   any symbol (module, function, argument, property, class, constant, type,
+        package, etc.) whose name contains `experimental` or `Experimental`; or
+    -   any symbol whose fully qualified name includes a module or class or
+        package which is itself experimental. This includes fields and
+        submessages of any protocol buffer called `experimental`.
 
 *   **Other languages**: TensorFlow APIs in languages other than Python and C,
     such as:
 
-  - [C++](../api_guides/cc/guide.md) (exposed through header files in
-    [`tensorflow/cc`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/cc)).
-  - [Java](../api_docs/java/reference/org/tensorflow/package-summary),
-  - [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go)
-  - [JavaScript](https://js.tensorflow.org)
+    -   [C++](../install/lang_c.ipynb) (exposed through header files in
+        [`tensorflow/cc/`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/cc)).
+    -   [Java](../install/lang_java_legacy.md),
+    -   [Go](https://github.com/tensorflow/build/blob/master/golang_install_guide/README.md)
+    -   [JavaScript](https://www.tensorflow.org/js)
+
+    and TensorFlow **Lite** APIs in languages other than Java/Kotlin, C,
+    Objective-C, and Swift, in particular
+
+    -   **C++** (exposed through header files in
+        [`tensorflow/lite/`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/))
 
 *   **Details of composite ops:** Many public functions in Python expand to
     several primitive ops in the graph, and these details will be part of any
-    graphs saved to disk as `GraphDef`s. These details may change for
-    minor releases. In particular, regressions tests that check for exact
-    matching between graphs are likely to break across minor releases, even
-    though the behavior of the graph should be unchanged and existing
-    checkpoints will still work.
+    graphs saved to disk as `GraphDef`s. These details may change for minor
+    releases. In particular, regression tests that check for exact matching
+    between graphs are likely to break across minor releases, even though the
+    behavior of the graph should be unchanged and existing checkpoints will
+    still work.
 
 *   **Floating point numerical details:** The specific floating point values
-    computed by ops may change at any time.  Users should rely only on
+    computed by ops may change at any time. Users should rely only on
     approximate accuracy and numerical stability, not on the specific bits
     computed. Changes to numerical formulas in minor and patch releases should
     result in comparable or improved accuracy, with the caveat that in machine
     learning improved accuracy of specific formulas may result in decreased
     accuracy for the overall system.
 
-*   **Random numbers:** The specific random numbers computed by the
-    [random ops](../api_guides/python/constant_op.md#Random_Tensors) may change
-    at any time. Users should rely only on approximately correct distributions
-    and statistical strength, not the specific bits computed. However, we will
-    make changes to random bits rarely (or perhaps never) for patch releases. We
-    will, of course, document all such changes.
+*   **Random numbers:** The specific random numbers computed may change at any
+    time. Users should rely only on approximately correct distributions and
+    statistical strength, not the specific bits computed. See the
+    [random number generation](random_numbers.ipynb) guide for details.
 
 *   **Version skew in distributed Tensorflow:** Running two different versions
     of TensorFlow in a single cluster is unsupported. There are no guarantees
@@ -127,18 +245,27 @@ These include:
 *   **Bugs:** We reserve the right to make backwards incompatible behavior
     (though not API) changes if the current implementation is clearly broken,
     that is, if it contradicts the documentation or if a well-known and
-    well-defined intended behavior is not properly implemented due to a bug.
-    For example, if an optimizer claims to implement a well-known optimization
+    well-defined intended behavior is not properly implemented due to a bug. For
+    example, if an optimizer claims to implement a well-known optimization
     algorithm but does not match that algorithm due to a bug, then we will fix
     the optimizer. Our fix may break code relying on the wrong behavior for
     convergence. We will note such changes in the release notes.
 
+*   **Unused API:** We reserve the right to make backwards incompatible changes
+    to APIs for which we find no documented uses (by performing audit of
+    TensorFlow usage through GitHub search). Before making any such changes, we
+    will announce our intention to make the change on the
+    [announce@ mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce),
+    providing instructions for how to address any breakages (if applicable), and
+    wait for two weeks to give our community a chance to share their feedback.
+
 *   **Error behavior:** We may replace errors with non-error behavior. For
-    instance, we may change a function to compute a result instead or raising
-    an error, even if that error is documented. We also reserve the right to
-    change the text of error messages. In addition, the type of an error may
-    change unless the exception type for a specific error condition is specified
-    in the documentation.
+    instance, we may change a function to compute a result instead of raising an
+    error, even if that error is documented. We also reserve the right to change
+    the text of error messages. In addition, the type of an error may change
+    unless the exception type for a specific error condition is specified in the
+    documentation.
+
 
 ## Compatibility of SavedModels, graphs and checkpoints
 
@@ -215,7 +342,8 @@ This section is relevant only when making incompatible changes to the `GraphDef`
 format, such as when adding ops, removing ops, or changing the functionality
 of existing ops.  The previous section should suffice for most users.
 
-
+
+
 
 ### Backward and partial forward compatibility
 
@@ -246,7 +374,9 @@ guidelines for evolving `GraphDef` versions.
 There are different data versions for graphs and checkpoints. The two data
 formats evolve at different rates from each other and also at different rates
 from TensorFlow. Both versioning systems are defined in
-[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h).
+[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h)
+and
+[`core/public/release_version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/release_version.h).
 Whenever a new version is added, a note is added to the header detailing what
 changed and the date.
 
@@ -346,7 +476,7 @@ existing producer scripts will not suddenly use the new functionality.
 1.  Add a new similar op named `SomethingV2` or similar and go through the
     process of adding it and switching existing Python wrappers to use it.
     To ensure forward compatibility use the checks suggested in
-    [compat.py](https://www.tensorflow.org/code/tensorflow/python/compat/compat.py)
+    [compat.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/compat/compat.py)
     when changing the Python wrappers.
 2.  Remove the old op (Can only take place with a major version change due to
     backward compatibility).
diff --git a/site/en/hub/README.md b/site/en/hub/README.md
deleted file mode 100644
index 3ca77792508..00000000000
--- a/site/en/hub/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Welcome to the warp zone!
-
-# TensorFlow Hub
-
-These docs are available here: https://github.com/tensorflow/hub/tree/master/docs
diff --git a/site/en/hub/_book.yaml b/site/en/hub/_book.yaml
new file mode 100644
index 00000000000..4a969d413bc
--- /dev/null
+++ b/site/en/hub/_book.yaml
@@ -0,0 +1,85 @@
+upper_tabs:
+# Tabs left of dropdown menu
+- include: /_upper_tabs_left.yaml
+- include: /api_docs/_upper_tabs_api.yaml
+# Dropdown menu
+- name: Resources
+  path: /resources
+  is_default: true
+  menu:
+  - include: /resources/_menu_toc.yaml
+  lower_tabs:
+    # Subsite tabs
+    other:
+    # [Guide]
+    - name: "Guide"
+      contents:
+      # TF Hub Platform overview.
+      - heading: Getting Started
+      - title: Overview
+        path: /hub/overview
+      - title: Installation
+        path: /hub/installation
+      - title: Community and support
+        path: /hub/community
+      # Python library usage information
+      - heading: Using the library
+      - title: Overview
+        path: /hub/lib_overview
+      - title: SavedModels for TensorFlow 2
+        path: /hub/tf2_saved_model
+      - title: Caching model downloads
+        path: /hub/caching
+      - title: Migration to TF2
+        path: /hub/migration_tf2
+      - title: Model compatibility for TF1/TF2
+        path: /hub/model_compatibility
+      - title: "Deprecated: TF1 Hub format"
+        path: /hub/tf1_hub_module
+        status: deprecated
+      # SavedModel APIs
+      - heading: Common SavedModel APIs
+      - title: Overview
+        path: /hub/common_saved_model_apis/index.md
+      - title: Reusable SavedModels (for all tasks)
+        path: /hub/reusable_saved_models
+      - title: Image tasks
+        path: /hub/common_saved_model_apis/images
+      - title: Text tasks
+        path: /hub/common_saved_model_apis/text
+      # Publishing models
+      - heading: Publishing models
+      - title: Publishing process
+        path: /hub/publish
+      - title: Data portability and deletion
+        path: /hub/portability_and_deletion
+      # Advanced developer info
+      - heading: Advanced developer info
+      - title: Model formats
+        path: /hub/model_formats
+      - title: Model hosting protocol
+        path: /hub/hosting
+      - title: Build from source
+        path: /hub/build_from_source
+      - title: Common issues
+        path: /hub/common_issues
+      - title: Contribute to TensorFlow Hub
+        path: /hub/contribute
+    # [Tutorials]
+    - name: Tutorials
+      path: /hub/tutorials
+      contents:
+      - include: /hub/tutorials/_toc.yaml
+    # [API]
+    - name: API
+      skip_translation: true
+      contents:
+      - include: /hub/api_docs/python/hub/_toc.yaml
+    # [Models]
+    - name: "Models ↗"
+      contents:
+      - title: Models
+        path: https://tfhub.dev
+        status: external
+
+- include: /_upper_tabs_right.yaml
diff --git a/site/en/hub/_index.yaml b/site/en/hub/_index.yaml
new file mode 100644
index 00000000000..00e67b15265
--- /dev/null
+++ b/site/en/hub/_index.yaml
@@ -0,0 +1,145 @@
+# This file is rendered on tensorflow.org/hub.
+# ../README.md is rendered on github.com/tensorflow/hub.
+# Both link to ./overview.md and ./*.md for detailed docs.
+book_path: /hub/_book.yaml
+project_path: /hub/_project.yaml
+description: >
+  TensorFlow Hub is a repository of trained machine learning models ready for fine-tuning and
+  deployable anywhere. Reuse trained models like BERT and Faster R-CNN with just a few lines of code.
+landing_page:
+  custom_css_path: /site-assets/css/style.css
+  rows:
+  - heading: TensorFlow Hub is a repository of trained machine learning models.
+    items:
+    - classname:
+        tfo-landing-row-item-code-block
+        devsite-landing-row-50
+      description: >
+        TensorFlow Hub is a repository of trained machine learning models ready for fine-tuning and
+        deployable anywhere. Reuse trained models like BERT and Faster R-CNN with just a few lines of code.
+      list:
+      - heading: See the guide
+        description: Learn about how to use TensorFlow Hub and how it works.
+        path: /hub/overview
+        icon:
+          path: /hub/images/guide_basics.png
+      - heading: See tutorials
+        description: Tutorials show you end-to-end examples using TensorFlow Hub.
+        path: /hub/tutorials
+        icon:
+          path: /site-assets/images/marketing/learn/lite-pick.svg
+      - heading: See models
+        description: Find trained TF, TFLite, and TF.js models for your use case.
+        path: https://tfhub.dev
+        icon:
+          path: /site-assets/images/marketing/learn/js-run.svg
+      code_block: |
+        
+          !pip install --upgrade tensorflow_hub
+
+          import tensorflow_hub as hub
+
+          model = hub.KerasLayer("https://tfhub.dev/google/nnlm-en-dim128/2")
+          embeddings = model(["The rain in Spain.", "falls",
+                              "mainly", "In the plain!"])
+
+          print(embeddings.shape)  #(4,128)
+        

+  - options:
+    - cards
+    - centered-header
+    heading: >
+      

Models
+    description: >
+      Find trained models from the TensorFlow community on TFHub.dev
+    items:
+    - heading: BERT
+      description: Check out BERT for NLP tasks including text classification and question answering.
+      path: https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3
+      image_path: /hub/images/bert.png
+      buttons:
+      - label: See the model
+        path: https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3
+    - heading: Object detection
+      description: Use the Faster R-CNN Inception ResNet V2 640x640 model for detecting objects in images.
+      path: https://tfhub.dev/tensorflow/faster_rcnn/inception_resnet_v2_640x640/1
+      image_path: /hub/images/object_detection.png
+      buttons:
+      - label: See the model
+        path: https://tfhub.dev/tensorflow/faster_rcnn/inception_resnet_v2_640x640/1
+    - heading: Style transfer
+      description: Transfer the style of one image to another using the image style transfer model.
+      path: https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2
+      image_path: /hub/images/style_transfer.png
+      buttons:
+      - label: See the model
+        path: https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2
+    - heading: On-device food classifier
+      description: Use this TFLite model to classify photos of food on a mobile device.
+      path: https://tfhub.dev/google/lite-model/aiy/vision/classifier/food_V1/1
+      image_path: /hub/images/food.png
+      buttons:
+      - label: See the model
+        path: https://tfhub.dev/google/lite-model/aiy/vision/classifier/food_V1/1
+  - options:
+    - cards
+    - centered-header
+    heading: >
+      

News & announcements
+    description: >
+      Check out our blog for more announcements and view the latest #TFHub updates on Twitter
+    items:
+    - heading: TensorFlow Hub for Real World Impact at Google I/O
+      youtube_id: BE5nkhFe3AE
+      description: >
+        Learn how you can use TensorFlow Hub to build ML solutions with real world impact.
+      buttons:
+      - label: Watch the video
+        path: https://www.youtube.com/watch?v=BE5nkhFe3AE
+    - heading: "On-device ML solutions"
+      description: >
+        To explore ML solutions for your mobile and web apps including TensorFlow Hub, visit the Google on-device machine learning page.
+      path: https://g.co/on-device-ml
+      image_path: /hub/images/odml.png
+      buttons:
+      - label: Visit the site
+        path: https://g.co/on-device-ml
+    - heading: "Making BERT Easier with Preprocessing Models From TensorFlow Hub"
+      description: >
+        TensorFlow Hub makes BERT simple to use with new preprocessing models.
+      path: https://blog.tensorflow.org/2020/12/making-bert-easier-with-preprocessing-models-from-tensorflow-hub.html
+      image_path: /hub/images/bert_preprocess_wide.png
+      buttons:
+      - label: Read the blog
+        path: https://blog.tensorflow.org/2020/12/making-bert-easier-with-preprocessing-models-from-tensorflow-hub.html
+    - heading: "From singing to musical scores: Estimating pitch with SPICE and Tensorflow Hub"
+      description: >
+        Learn how to use the SPICE model to automatically transcribe sheet music from live audio.
+      path: https://blog.tensorflow.org/2020/06/estimating-pitch-with-spice-and-tensorflow-hub.html
+      image_path: /hub/images/spice_blog.png
+      buttons:
+      - label: Read the blog
+        path: https://blog.tensorflow.org/2020/06/estimating-pitch-with-spice-and-tensorflow-hub.html
+  - options:
+    - cards
+    - centered-header
+    heading: >
+      

Community
+    description: Join the TensorFlow Hub community
+    items:
+    - heading: TensorFlow Hub on GitHub
+      icon:
+        path: /hub/images/github_icon.svg
+      path: https://github.com/tensorflow/hub
+    - heading: Contribute models
+      icon:
+        name: publish
+      path: /hub/publish
+  - options:
+    - cta
+    items:
+    - heading: Get started with TensorFlow Hub
+      buttons:
+      - label: Find trained models 
+        path: https://tfhub.dev
+        classname: button
diff --git a/site/en/hub/_redirects.yaml b/site/en/hub/_redirects.yaml
new file mode 100644
index 00000000000..bee1cbec873
--- /dev/null
+++ b/site/en/hub/_redirects.yaml
@@ -0,0 +1,7 @@
+redirects:
+- from: /hub/becoming_a_publisher
+  to: /hub/publish
+- from: /hub/writing_model_documentation
+  to: /hub/writing_documentation#model
+- from: /hub/creating_a_collection
+  to: /hub/writing_documentation#collection
diff --git a/site/en/hub/build_from_source.md b/site/en/hub/build_from_source.md
new file mode 100644
index 00000000000..42e19eb6208
--- /dev/null
+++ b/site/en/hub/build_from_source.md
@@ -0,0 +1,195 @@
+
+
+
+# Creating the TensorFlow Hub pip package using Linux
+
+Note: This document is for developers interested in modifying TensorFlow Hub
+itself. To _use_ TensorFlow Hub, see the [Install instructions](installation.md)
+
+If you make changes to TensorFlow Hub pip package, you will likely want to
+rebuild the pip package from source to try out your changes.
+
+This requires:
+
+*   Python
+*   TensorFlow
+*   Git
+*   [Bazel](https://docs.bazel.build/versions/master/install.html)
+
+Alternatively, if you install the protobuf compiler you can
+[try out your changes without using bazel](#develop).
+
+## Setup a virtualenv {:#setup}
+
+### Activate virtualenv
+
+Install virtualenv if it's not installed already:
+
+```shell
+~$ sudo apt-get install python-virtualenv
+```
+
+Create a virtual environment for the package creation:
+
+```shell
+~$ virtualenv --system-site-packages tensorflow_hub_env
+```
+
+And activate it:
+
+```shell
+~$ source ~/tensorflow_hub_env/bin/activate  # bash, sh, ksh, or zsh
+~$ source ~/tensorflow_hub_env/bin/activate.csh  # csh or tcsh
+```
+
+### Clone the TensorFlow Hub repository.
+
+```shell
+(tensorflow_hub_env)~/$ git clone https://github.com/tensorflow/hub
+(tensorflow_hub_env)~/$ cd hub
+```
+
+## Test your changes
+
+### Run TensorFlow Hub's tests
+
+```shell
+(tensorflow_hub_env)~/hub/$ bazel test tensorflow_hub:all
+```
+
+## Build and install the package
+
+### Build TensorFlow Hub pip packaging script
+
+To build a pip package for TensorFlow Hub:
+
+```shell
+(tensorflow_hub_env)~/hub/$ bazel build tensorflow_hub/pip_package:build_pip_package
+```
+
+### Create the TensorFlow Hub pip package
+
+```shell
+(tensorflow_hub_env)~/hub/$ bazel-bin/tensorflow_hub/pip_package/build_pip_package \
+/tmp/tensorflow_hub_pkg
+```
+
+### Install and test the pip package (optional)
+
+Run the following commands to install the pip package.
+
+```shell
+(tensorflow_hub_env)~/hub/$ pip install /tmp/tensorflow_hub_pkg/*.whl
+```
+
+Test import TensorFlow Hub:
+
+```shell
+(tensorflow_hub_env)~/hub/$ cd ..  # exit the directory to avoid confusion
+(tensorflow_hub_env)~/$ python -c "import tensorflow_hub as hub"
+```
+
+## "Developer" install (experimental)
+
+
+
+Warning: This approach to running TensorFlow is experimental, and not officially
+supported by the TensorFlow Hub team.
+
+Building the package with bazel is the only officially supported method. However
+if you are unfamiliar with bazel simpler to work with open source tools. For
+that you can do a "developer install" of the package.
+
+This installation method allows you to install the working directory into your
+python environment, so that ongoing changes are reflected when you import the
+package.
+
+### Setup the repository
+
+First setup the virtualenv and repository, as described [above](#setup).
+
+### Install `protoc`
+
+Because TensorFlow Hub uses protobufs you will need the protobuf compiler to
+create the necessary python `_pb2.py` files from the `.proto` files.
+
+#### On a Mac:
+
+```
+(tensorflow_hub_env)~/hub/$ brew install protobuf
+```
+
+#### On Linux
+
+```
+(tensorflow_hub_env)~/hub/$ sudo apt install protobuf-compiler
+```
+
+### Compile the `.proto` files
+
+Initially there are no `_pb2.py` files in the directory:
+
+```
+(tensorflow_hub_env)~/hub/$ ls -1 tensorflow_hub/*_pb2.py
+```
+
+Run `protoc` to create them:
+
+```
+(tensorflow_hub_env)~/hub/$ protoc -I=tensorflow_hub --python_out=tensorflow_hub tensorflow_hub/*.proto
+(tensorflow_hub_env)~/hub/$ ls -1 tensorflow_hub/*_pb2.py
+```
+
+
+tensorflow_hub/image_module_info_pb2.py
+tensorflow_hub/module_attachment_pb2.py
+tensorflow_hub/module_def_pb2.py
+

+
+Note: Don't forget to recompile the `_pb2.py` files if you make changes to the
+`.proto` definitions.
+
+### Import directly from the repository
+
+With the `_pb2.py` files in place, you can use try out your modifications
+directly from the TensorFlow Hub directory:
+
+```
+(tensorflow_hub_env)~/$ python -c "import tensorflow_hub as hub"
+```
+
+### Install in "developer" mode
+
+Or to use this from outside the repository root, you can use the `setup.py
+develop` installation:
+
+```
+(tensorflow_hub_env)~/hub/$ python tensorflow_hub/pip_package/setup.py develop
+```
+
+Now you can use your local changes in a regular python virtualenv, without the
+need to rebuild and install the pip package for each new change:
+
+```shell
+(tensorflow_hub_env)~/hub/$ cd ..  # exit the directory to avoid confusion
+(tensorflow_hub_env)~/$ python -c "import tensorflow_hub as hub"
+```
+
+## De-activate the virtualenv
+
+```shell
+(tensorflow_hub_env)~/hub/$ deactivate
+```
diff --git a/site/en/hub/caching.md b/site/en/hub/caching.md
new file mode 100644
index 00000000000..678b2c22af0
--- /dev/null
+++ b/site/en/hub/caching.md
@@ -0,0 +1,86 @@
+
+# Caching model downloads from TF Hub
+
+## Overview
+
+The `tensorflow_hub` library currently supports two modes for downloading
+models. By default, a model is downloaded as a compressed archive and cached on
+disk. Secondly, models can directly be read from remote storage into TensorFlow.
+Either way, the calls to `tensorflow_hub` functions in the actual Python code
+can and should continue to use the canonical tfhub.dev URLs of models, which are
+portable across systems and navigable for documentation. In the rare case that
+user code needs the actual filesystem location (after downloading  and
+decompressing, or after resolving a model handle into a filesystem path),
+it can be obtained by the function `hub.resolve(handle)`.
+
+### Caching of compressed downloads
+
+The `tensorflow_hub` library by default caches models on the filesystem when
+they have been downloaded from tfhub.dev (or other [hosting sites](hosting.md))
+and decompressed. This mode is recommended for most environments, except if disk
+space is scarce but network bandwidth and latency are superb.
+
+The download location defaults to a local temporary directory but can be
+customized by setting the environment variable `TFHUB_CACHE_DIR` (recommended)
+or by passing the command-line flag `--tfhub_cache_dir`. The default cache
+location `/tmp/tfhub_modules` (or whatever `os.path.join(tempfile.gettempdir(),
+"tfhub_modules")` is evaluated to) should work in most cases.
+
+Users who prefer persistent caching across system reboots can instead set
+`TFHUB_CACHE_DIR` to a location in their home directory. For example, a user of
+the bash shell on a Linux system can add a line like the following to
+`~/.bashrc`:
+
+```bash
+export TFHUB_CACHE_DIR=$HOME/.cache/tfhub_modules
+```
+
+...restart the shell, and then this location will be used. When using a
+persistent location, be aware that there is no automatic cleanup.
+
+### Reading from remote storage
+
+Users can instruct the `tensorflow_hub` library to directly read models from
+remote storage (GCS) instead of downloading the models locally with:
+
+```shell
+os.environ["TFHUB_MODEL_LOAD_FORMAT"] = "UNCOMPRESSED"
+```
+
+or by setting the command-line flag `--tfhub_model_load_format` to
+`UNCOMPRESSED`. This way, no caching directory is needed, which is especially
+helpful in environments that provide little disk space but a fast internet
+connection.
+
+### Running on TPU in Colab notebooks
+
+On [colab.research.google.com](https://colab.research.google.com), downloading
+compressed models will conflict with the TPU runtime since the computation
+workload is delegated to another machine that does not have access to the cache
+location by default. There are two workarounds for this situation:
+
+#### 1) Use a GCS bucket that the TPU worker can access
+
+The easiest solution is to instruct the `tensorflow_hub` library to read the
+models from TF Hub's GCS bucket as explained above. Users with their own GCS
+bucket can instead specify a directory in their bucket as the cache location
+with code like:
+
+```python
+import os
+os.environ["TFHUB_CACHE_DIR"] = "gs://my-bucket/tfhub-modules-cache"
+```
+
+...before calling the `tensorflow_hub` library.
+
+#### 2) Redirect all reads through the Colab host
+
+Another workaround is to redirect all reads (even of large variables) through
+the Colab host:
+
+```python
+load_options =
+tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')
+reloaded_model = hub.load("https://tfhub.dev/...", options=load_options)
+```
+**Note:** See more information regarding valid handles [here](tf2_saved_model.md#model_handles).
diff --git a/site/en/hub/common_issues.md b/site/en/hub/common_issues.md
new file mode 100644
index 00000000000..03ba4a62a8e
--- /dev/null
+++ b/site/en/hub/common_issues.md
@@ -0,0 +1,148 @@
+
+# Common issues
+
+If your issue is not listed here, please search the
+[github issues](https://github.com/tensorflow/hub/issues) before filling a new
+one.
+
+**Note:** This documentation uses TFhub.dev URL handles in examples. See more
+information regarding other valid handle types [here](tf2_saved_model.md#model_handles).
+
+## TypeError: 'AutoTrackable' object is not callable
+
+```python
+# BAD: Raises error
+embed = hub.load('https://tfhub.dev/google/nnlm-en-dim128/1')
+embed(['my text', 'batch'])
+```
+
+This error frequently arises when loading models in TF1 Hub format with the
+`hub.load()` API in TF2. Adding the correct signature should fix this problem.
+See the [TF-Hub migration guide for TF2](migration_tf2.md) for more details on
+moving to TF2 and the use of models in TF1 Hub format in TF2.
+
+```python
+
+embed = hub.load('https://tfhub.dev/google/nnlm-en-dim128/1')
+embed.signatures['default'](['my text', 'batch'])
+```
+
+## Cannot download a module
+
+In the process of using a module from an URL there are many errors that can show
+up due to the network stack. Often this is a problem specific to the machine
+running the code and not an issue with the library. Here is a list of the common
+ones:
+
+*   **"EOF occurred in violation of protocol"** - This issue is likely to be
+    generated if the installed python version does not support the TLS
+    requirements of the server hosting the module. Notably, python 2.7.5 is
+    known to fail resolving modules from tfhub.dev domain. **FIX**: Please
+    update to a newer python version.
+
+*   **"cannot verify tfhub.dev's certificate"** - This issue is likely to be
+    generated if something on the network is trying to act as the dev gTLD.
+    Before .dev was used as a gTLD, developers and frameworks would sometimes
+    use .dev names to help testing code. **FIX:** Identify and reconfigure the
+    software that intercepts name resolution in the ".dev" domain.
+
+*   Failures to write to the cache directory `/tmp/tfhub_modules` (or similar):
+    see [Caching](caching.md) for what that is and how to change its location.
+
+If the above errors and fixes do not work, one can try to manually download a
+module by simulating the protocol of attaching `?tf-hub-format=compressed` to
+the URL to download a tar compressed file that has to be manually decompressed
+into a local file. The path to the local file can then be used instead of the
+URL. Here is a quick example:
+
+```bash
+# Create a folder for the TF hub module.
+$ mkdir /tmp/moduleA
+# Download the module, and uncompress it to the destination folder. You might want to do this manually.
+$ curl -L "https://tfhub.dev/google/universal-sentence-encoder/2?tf-hub-format=compressed" | tar -zxvC /tmp/moduleA
+# Test to make sure it works.
+$ python
+> import tensorflow_hub as hub
+> hub.Module("/tmp/moduleA")
+```
+
+## Running inference on a pre-initialized module
+
+If you are writing a Python program that applies a module many times on input
+data, you can apply the following recipes. (Note: For serving requests in
+production services, consider
+[TensorFlow Serving](https://www.tensorflow.org/tfx/guide/serving) or other
+scalable, Python-free solutions.)
+
+Assuming your use-case model is **initialization** and subsequent **requests**
+(for example Django, Flask, custom HTTP server, etc.), you can set-up the
+serving as follows:
+
+### TF2 SavedModels
+
+*   In the initialization part:
+    *   Load the TF2.0 model.
+
+```python
+import tensorflow_hub as hub
+
+embedding_fn = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
+```
+
+*   In the request part:
+    *   Use the embedding function to run inference.
+
+```python
+embedding_fn(["Hello world"])
+```
+
+This call of a tf.function is optimized for performance, see
+[tf.function guide](https://www.tensorflow.org/guide/function).
+
+### TF1 Hub modules
+
+*   In the initialization part:
+    *   Build the graph with a **placeholder** - entry point into the graph.
+    *   Initialize the session.
+
+```python
+import tensorflow as tf
+import tensorflow_hub as hub
+
+# Create graph and finalize (finalizing optional but recommended).
+g = tf.Graph()
+with g.as_default():
+  # We will be feeding 1D tensors of text into the graph.
+  text_input = tf.placeholder(dtype=tf.string, shape=[None])
+  embed = hub.Module("https://tfhub.dev/google/universal-sentence-encoder/2")
+  embedded_text = embed(text_input)
+  init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])
+g.finalize()
+
+# Create session and initialize.
+session = tf.Session(graph=g)
+session.run(init_op)
+```
+
+*   In the request part:
+    *   Use the session to feed data into the graph through the placeholder.
+
+```python
+result = session.run(embedded_text, feed_dict={text_input: ["Hello world"]})
+```
+
+## Cannot change a model's dtype (e.g., float32 to bfloat16)
+
+TensorFlow's SavedModels (shared on TF Hub or otherwise) contain operations that
+work on fixed data types (often, float32 for the weights and intermediate
+activations of neural networks). These cannot be changed after the fact when
+loading the SavedModel (but model publishers can choose to publish different
+models with different data types).
+
+## Update a model version
+
+The documentation metadata of model versions can be updated. However, the
+version's assets (model files) are immutable. If you want to change the model
+assets, you can publish a newer version of the model. It's a good practice to
+extend the documentation with a change log that describes what changed between
+versions.
diff --git a/site/en/hub/common_saved_model_apis/images.md b/site/en/hub/common_saved_model_apis/images.md
new file mode 100644
index 00000000000..5413f0adc07
--- /dev/null
+++ b/site/en/hub/common_saved_model_apis/images.md
@@ -0,0 +1,155 @@
+
+# Common SavedModel APIs for Image Tasks
+
+This page describes how [TF2 SavedModels](../tf2_saved_model.md) for
+image-related tasks should implement the
+[Reusable SavedModel API](../reusable_saved_models.md). (This replaces the
+[Common Signatures for Images](../common_signatures/images.md) for the
+now-deprecated [TF1 Hub format](../tf1_hub_module).)
+
+
+
+## Image Feature Vector
+
+### Usage summary
+
+An **image feature vector** is a dense 1-D tensor that represents a whole image,
+typically for use by a simple feed-forward classifier in the consumer model. (In
+terms of classic CNNs, this is the bottleneck value after the spatial extent has
+been pooled or flattened away, but before classification is done; for that, see
+[image classification](#classification) below.)
+
+A Reusable SavedModel for image feature extraction has a `__call__` method on
+the root object that maps a batch of images to a batch of feature vectors. It
+can be used like so:
+
+```python
+obj = hub.load("path/to/model")  # That's tf.saved_model.load() after download.
+images = ...  # A batch of images with shape [batch_size, height, width, 3].
+features = obj(images)   # A batch with shape [batch_size, num_features].
+```
+
+In Keras, the equivalent is
+
+```python
+features = hub.KerasLayer("path/to/model")(images)
+```
+
+The input follows the general convention for [input of images](#input). The
+model documentation specifies the permissible range for `height` and `width` of
+the input.
+
+The output is a single tensor of dtype `float32` and shape `[batch_size,
+num_features]`. The `batch_size` is the same as in the input. `num_features` is
+a module-specific constant independent of input size.
+
+### API details
+
+The [Reusable SavedModel API](../reusable_saved_models.md) also provides a list
+of `obj.variables` (e.g., for initialization when not loading eagerly).
+
+A model that supports fine-tuning provides a list of `obj.trainable_variables`.
+It may require you to pass `training=True` to execute in training mode (e.g.,
+for dropout). Some models allow optional arguments to override hyperparameters
+(e.g., dropout rate; to be described in model documentation). The model may also
+provide a list of `obj.regularization_losses`. For details, see the
+[Reusable SavedModel API](../reusable_saved_models.md).
+
+In Keras, this is taken care of by `hub.KerasLayer`: initialize it with
+`trainable=True` to enable fine-tuning, and (in the rare case that hparam
+overrides apply) with `arguments=dict(some_hparam=some_value, ...))`.
+
+### Notes
+
+Applying dropout to the output features (or not) should be left to the model
+consumer. The SavedModel itself should not perform dropout on the actual outputs
+(even if it uses dropout internally in other places).
+
+### Examples
+
+Reusable SavedModels for image feature vectors are used in
+
+*   the Colab tutorial
+    [Retraining an Image Classifier](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_image_retraining.ipynb),
+
+
+
+## Image Classification
+
+### Usage summary
+
+**Image classification** maps the pixels of an image to linear scores (logits)
+for membership in the classes of a taxonomy _selected by the module publisher_.
+This allows model consumers to to draw conclusions from the particular
+classification learned by the publisher module. (For image classification with
+a new set of classes, it is common to reuse an
+[Image Feature Vector](#feature-vector) model with a new classifier instead.)
+
+A Reusable SavedModel for image classification has a `__call__` method on the
+root object that maps a batch of images to a batch of logits. It can be used
+like so:
+
+```python
+obj = hub.load("path/to/model")  # That's tf.saved_model.load() after download.
+images = ...  # A batch of images with shape [batch_size, height, width, 3].
+logits = obj(images)   # A batch with shape [batch_size, num_classes].
+```
+
+In Keras, the equivalent is
+
+```python
+logits = hub.KerasLayer("path/to/model")(images)
+```
+
+The input follows the general convention for [input of images](#input). The
+model documentation specifies the permissible range for `height` and `width` of
+the input.
+
+The output `logits` is a single tensor of dtype `float32` and shape
+`[batch_size, num_classes]`. The `batch_size` is the same as in the input.
+`num_classes` is the number of classes in the classification, which is a
+model-specific constant.
+
+The value `logits[i, c]` is a score predicting the membership of example `i` in
+the class with index `c`.
+
+It depends on the underlying classification whether these scores are meant to be
+used with softmax (for mutually exclusive classes), sigmoid (for orthogonal
+classes), or something else. The module documentation should describe this, and
+refer to a definition of the class indices.
+
+### API details
+
+The [Reusable SavedModel API](../reusable_saved_models.md) also provides a list
+of `obj.variables` (e.g., for initialization when not loading eagerly).
+
+A model that supports fine-tuning provides a list of `obj.trainable_variables`.
+It may require you to pass `training=True` to execute in training mode (e.g.,
+for dropout). Some models allow optional arguments to override hyperparameters
+(e.g., dropout rate; to be described in model documentation). The model may also
+provide a list of `obj.regularization_losses`. For details, see the
+[Reusable SavedModel API](../reusable_saved_models.md).
+
+In Keras, this is taken care of by `hub.KerasLayer`: initialize it with
+`trainable=True` to enable fine-tuning, and (in the rare case that hparam
+overrides apply) with `arguments=dict(some_hparam=some_value, ...))`.
+
+
+
+## Image input
+
+This is common to all types of image models.
+
+A model that takes a batch of images as input accepts them as a dense 4-D tensor
+of dtype `float32` and shape `[batch_size, height, width, 3]` whose elements are
+RGB color values of pixels normalized to the range [0, 1]. This is what you get
+from `tf.image.decode_*()` followed by `tf.image.convert_image_dtype(...,
+tf.float32)`.
+
+The model accepts any `batch_size`. The model documentation specifies the
+permissible range for `height` and `width`. The last dimension is fixed to 3 RGB
+channels.
+
+It is recommended that models use the `channels_last` (or `NHWC`) layout of
+Tensors throughout, and leave it to TensorFlow's graph optimizer to rewrite to
+`channels_first` (or `NCHW`) if needed.
diff --git a/site/en/hub/common_saved_model_apis/index.md b/site/en/hub/common_saved_model_apis/index.md
new file mode 100644
index 00000000000..356505f9952
--- /dev/null
+++ b/site/en/hub/common_saved_model_apis/index.md
@@ -0,0 +1,46 @@
+
+# Common SavedModel APIs for TF Hub
+
+## Introduction
+
+[TensorFlow Hub](https://tfhub.dev) hosts models for a variety of tasks. Models
+for the same task are encouraged to implement a common API so that model
+consumers can easily exchange them without modifying the code that uses them,
+even if they come from different publishers.
+
+The goal is to make exchanging different models for the same task as simple as
+switching a string-valued hyperparameter. With that, model consumers can easily
+find the best one for their problem.
+
+This directory collects specifications of common APIs for models in the
+[TF2 SavedModel format](../tf2_saved_model.md). (It replaces the
+[Common Signatures](../common_signatures/index.md) for the now-deprecated
+[TF1 Hub format](../tf1_hub_module.md).)
+
+## Reusable SavedModel: the common foundation
+
+The [Reusable SavedModel API](../reusable_saved_models.md) defines general
+conventions how to load a SavedModel back into a Python program and reuse it as
+part of a bigger TensorFlow model.
+
+Basic usage:
+
+```python
+obj = hub.load("path/to/model")  # That's tf.saved_model.load() after download.
+outputs = obj(inputs, training=False)  # Invokes the tf.function obj.__call__.
+```
+
+Key point: This uses the object-based interface to restored SavedModels that was
+added in TensorFlow 2, not the SavedModel signatures for serving.
+
+For Keras users, the `hub.KerasLayer` class relies on this API to wrap the
+Reusable SavedModel as a Keras Layer (shielding Keras users from its details),
+with inputs and outputs according to the task-specific APIs listed below.
+
+## Task-specific APIs
+
+These refine the [Reusable SavedModel API](../reusable_saved_models.md) with
+conventions for particular ML tasks and types of data.
+
+*   [Image tasks](images.md)
+*   [Text tasks](text.md)
diff --git a/site/en/hub/common_saved_model_apis/text.md b/site/en/hub/common_saved_model_apis/text.md
new file mode 100644
index 00000000000..209319f27a9
--- /dev/null
+++ b/site/en/hub/common_saved_model_apis/text.md
@@ -0,0 +1,361 @@
+
+# Common SavedModel APIs for Text Tasks
+
+This page describes how [TF2 SavedModels](../tf2_saved_model.md) for
+text-related tasks should implement the
+[Reusable SavedModel API](../reusable_saved_models.md). (This replaces and
+extends the [Common Signatures for Text](../common_signatures/text.md) for the
+now-deprecated [TF1 Hub format](../tf1_hub_module).)
+
+## Overview
+
+There are several APIs to compute **text embeddings** (also known as dense
+representations of text, or text feature vectors).
+
+*   The API for *text embeddings from text inputs* is implemented by a
+    SavedModel that maps a batch of strings to a batch of embedding vectors.
+    This is very easy to use, and many models on TF Hub have implemented it.
+    However, this does not allow fine-tuning the model on TPU.
+
+*   The API for *text embeddings with preprocessed inputs* solves the same task,
+    but is implemented by two separate SavedModels:
+
+    *   a *preprocessor* that can run inside a tf.data input pipeline and
+        converts strings and other variable-length data into numeric Tensors,
+    *   an *encoder* that accepts the results of the preprocessor and performs
+        the trainable part of the embedding computation.
+
+    This split allows inputs to be preprocessed asynchronously before being fed
+    into the training loop. In particular, it allows building encoders that can
+    be run and fine-tuned on [TPU](https://www.tensorflow.org/guide/tpu).
+
+*   The API for *text embeddings with Transformer encoders* extends the API for
+    text embeddings from preprocessed inputs to the particular case of BERT and
+    other Transformer encoders.
+
+    *   The *preprocessor* is extended to build encoder inputs from more than
+        one segment of input text.
+    *   The *Transformer encoder* exposes the context-aware embeddings of
+        individual tokens.
+
+In each case, the text inputs are UTF-8 encoded strings, typically of plain
+text, unless the model documentation provides otherwise.
+
+Regardless of API, different models have been pre-trained on text from different
+languages and domains, and with different tasks in mind. Therefore, not every
+text embedding model is suitable for every problem.
+
+
+
+
+## Text Embedding from Text Inputs
+
+A SavedModel for **text embeddings from text inputs** accepts a batch of inputs
+in a string Tensor of shape `[batch_size]` and maps them to a float32 Tensor of
+shape `[batch_size, dim]` with dense representations (feature vectors) of the
+inputs.
+
+### Usage synopsis
+
+```python
+obj = hub.load("path/to/model")
+text_input = ["A long sentence.",
+              "single-word",
+              "http://example.com"]
+embeddings = obj(text_input)
+```
+
+Recall from the [Reusable SavedModel API](../reusable_saved_models.md) that
+running the model in training mode (e.g., for dropout) may require a keyword
+argument `obj(..., training=True)`, and that `obj` provides attributes
+`.variables`, `.trainable_variables` and `.regularization_losses` as applicable.
+
+In Keras, all this is taken care of by
+
+```python
+embeddings = hub.KerasLayer("path/to/model", trainable=...)(text_input)
+```
+
+### Distributed training
+
+If the text embedding is used as part of a model that gets trained with a
+distribution strategy, the call to `hub.load("path/to/model")` or
+`hub.KerasLayer("path/to/model", ...)`, resp., must happen inside the
+DistributionStrategy scope in order to create the model's variables in the
+distributed way. For example
+
+```python
+  with strategy.scope():
+    ...
+    model = hub.load("path/to/model")
+    ...
+```
+
+### Examples
+
+*   Colab tutorial
+    [Text Classification with Movie Reviews](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_text_classification.ipynb).
+
+
+
+## Text Embeddings with Preprocessed Inputs
+
+A **text embedding with preprocessed inputs** is implemented by two separate
+SavedModels:
+
+*   a **preprocessor** that maps a string Tensor of shape `[batch_size]` to a
+    dict of numeric Tensors,
+*   an **encoder** that accepts a dict of Tensors as returned by the
+    preprocessor, performs the trainable part of the embedding computation, and
+    returns a dict of outputs. The output under key `"default"` is a float32
+    Tensor of shape `[batch_size, dim]`.
+
+This allows to run the preprocessor in an input pipeline but fine-tune the
+embeddings computed by the encoder as part of a larger model. In particular, it
+allows to build encoders that can be run and fine-tuned on
+[TPU](https://www.tensorflow.org/guide/tpu).
+
+It is an implementation detail which Tensors are contained in the preprocessor's
+output, and which (if any) additional Tensors besides `"default"` are contained
+in the encoder's output.
+
+The documentation of the encoder must specify which preprocessor to use with it.
+Typically, there is exactly one correct choice.
+
+### Usage synopsis
+
+```python
+text_input = tf.constant(["A long sentence.",
+                          "single-word",
+                          "http://example.com"])
+preprocessor = hub.load("path/to/preprocessor")  # Must match `encoder`.
+encoder_inputs = preprocessor(text_input)
+
+encoder = hub.load("path/to/encoder")
+encoder_outputs = encoder(encoder_inputs)
+embeddings = encoder_outputs["default"]
+```
+
+Recall from the [Reusable SavedModel API](../reusable_saved_models.md) that
+running the encoder in training mode (e.g., for dropout) may require a keyword
+argument `encoder(..., training=True)`, and that `encoder` provides attributes
+`.variables`, `.trainable_variables` and `.regularization_losses` as applicable.
+
+The `preprocessor` model may have `.variables` but is not meant to be trained
+further. Preprocessing is not mode-dependent: if `preprocessor()` has a
+`training=...` argument at all, it has no effect.
+
+In Keras, all this is taken care of by
+
+```python
+encoder_inputs = hub.KerasLayer("path/to/preprocessor")(text_input)
+encoder_outputs = hub.KerasLayer("path/to/encoder", trainable=True)(encoder_inputs)
+embeddings = encoder_outputs["default"]
+```
+
+### Distributed training
+
+If the encoder is used as part of a model that gets trained with a distribution
+strategy, the call to `hub.load("path/to/encoder")` or
+`hub.KerasLayer("path/to/encoder", ...)`, resp., must happen inside
+
+```python
+  with strategy.scope():
+    ...
+```
+
+in order to re-create the encoder variables in the distributed way.
+
+Likewise, if the preprocessor is part of the trained model (as in the simple
+example above), it also needs to be loaded under the distribution strategy
+scope. If, however, the preprocessor is used in an input pipeline (e.g., in a
+callable passed to `tf.data.Dataset.map()`), its loading must happen *outside*
+the distribution strategy scope, in order to place its variables (if any) on the
+host CPU.
+
+### Examples
+
+*   Colab tutorial
+    [Classify text with BERT](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/classify_text_with_bert.ipynb).
+
+
+
+## Text embeddings with Transformer Encoders
+
+Transformer encoders for text operate on a batch of input sequences, each
+sequence comprising *n* ≥ 1 segments of tokenized text, within some
+model-specific bound on *n*. For BERT and many of its extensions, that bound is
+2, so they accept single segments and segment pairs.
+
+The API for **text embeddings with Transformer encoders** extends the API for
+text embeddings with preprocessed inputs to this setting.
+
+### Preprocessor
+
+A preprocessor SavedModel for text embeddings with Transformer encoders
+implements the API of a preprocessor SavedModel for text embeddings with
+preprocessed inputs (see above), which provides a way to map single-segment text
+inputs directly to encoder inputs.
+
+In addition, the preprocessor SavedModel provides callable subobjects `tokenize`
+for tokenization (separately per segment) and `bert_pack_inputs` for packing *n*
+tokenized segments into one input sequence for the encoder. Each subobject
+follows the [Reusable SavedModel API](../reusable_saved_models.md).
+
+#### Usage synopsis
+
+As a concrete example for two segments of text, let us look at a sentence
+entailment task that asks whether a premise (first segment) does or does not
+imply a hypothesis (second segment).
+
+```python
+preprocessor = hub.load("path/to/preprocessor")
+
+# Tokenize batches of both text inputs.
+text_premises = tf.constant(["The quick brown fox jumped over the lazy dog.",
+                             "Good day."])
+tokenized_premises = preprocessor.tokenize(text_premises)
+text_hypotheses = tf.constant(["The dog was lazy.",  # Implied.
+                               "Axe handle!"])       # Not implied.
+tokenized_hypotheses = preprocessor.tokenize(text_hypotheses)
+
+# Pack input sequences for the Transformer encoder.
+seq_length = 128
+encoder_inputs = preprocessor.bert_pack_inputs(
+    [tokenized_premises, tokenized_hypotheses],
+    seq_length=seq_length)  # Optional argument.
+```
+
+In Keras, this computation can be expressed as
+
+```python
+tokenize = hub.KerasLayer(preprocessor.tokenize)
+tokenized_hypotheses = tokenize(text_hypotheses)
+tokenized_premises = tokenize(text_premises)
+
+bert_pack_inputs = hub.KerasLayer(
+    preprocessor.bert_pack_inputs,
+    arguments=dict(seq_length=seq_length))  # Optional argument.
+encoder_inputs = bert_pack_inputs([tokenized_premises, tokenized_hypotheses])
+```
+
+#### Details of `tokenize`
+
+A call to `preprocessor.tokenize()` accepts a string Tensor of shape
+`[batch_size]` and returns a
+[RaggedTensor](https://www.tensorflow.org/guide/ragged_tensor) of shape
+`[batch_size, ...]` whose values are int32 token ids representing the input
+strings. There can be *r* ≥ 1 ragged dimensions after `batch_size` but no other
+uniform dimension.
+
+*   If *r*=1, the shape is `[batch_size, (tokens)]`, and each input is simply
+    tokenized into a flat sequence of tokens.
+*   If *r*>1, there are *r*-1 additional levels of grouping. For example,
+    [tensorflow_text.BertTokenizer](https://github.com/tensorflow/text/blob/v2.3.0/tensorflow_text/python/ops/bert_tokenizer.py#L138)
+    uses *r*=2 to group tokens by words and yields shape `[batch_size, (words),
+    (tokens_per_word)]`. It is up to the model at hand how many of these extra
+    level(s) exist, if any, and what groupings they represent.
+
+The user can (but need not) modify tokenized inputs, e.g., to accommodate the
+seq_length limit that will be enforced in packing encoder inputs. Extra
+dimensions in the tokenizer output can help here (e.g., to respect word
+boundaries) but become meaningless in the next step.
+
+In terms of the [Reusable SavedModel API](../reusable_saved_models.md), the
+`preprocessor.tokenize` object may have `.variables` but is not meant to be
+trained further. Tokenization is not mode-dependent: if
+`preprocessor.tokenize()` has a `training=...` argument at all, it has no
+effect.
+
+#### Details of `bert_pack_inputs`
+
+A call to `preprocessor.bert_pack_inputs()` accepts a Python list of tokenized
+inputs (batched separately for each input segment) and returns a dict of Tensors
+representing a batch of fixed-length input sequences for the Transformer encoder
+model.
+
+Each tokenized input is an int32 RaggedTensor of shape `[batch_size, ...]`,
+where the number *r* of ragged dimensions after batch_size is either 1 or the
+same as in the output of `preprocessor.tokenize().` (The latter is for
+convenience only; the extra dimensions are flattened out before packing.)
+
+Packing adds special tokens around the input segments as expected by the
+encoder. The `bert_pack_inputs()` call implements exactly the packing scheme
+used by the original BERT models and many of their extensions: the packed
+sequence starts with one start-of-sequence token, followed by the tokenized
+segments, each terminated by one end-of-segment token. Remaining positions up to
+seq_length, if any, are filled up with padding tokens.
+
+If a packed sequence would exceed seq_length, `bert_pack_inputs()` truncates its
+segments to prefixes of approximately equal sizes so that the packed sequence
+fits exactly within seq_length.
+
+Packing is not mode-dependent: if `preprocessor.bert_pack_inputs()` has a
+`training=...` argument at all, it has no effect. Also,
+`preprocessor.bert_pack_inputs` is not expected to have variables, or support
+fine-tuning.
+
+### Encoder
+
+The encoder is called on the dict of `encoder_inputs` in the same way as in the
+API for text embeddings with preprocessed inputs (see above), including the
+provisions from the [Reusable SavedModel API](../reusable_saved_models.md).
+
+#### Usage synopsis
+
+```python
+encoder = hub.load("path/to/encoder")
+encoder_outputs = encoder(encoder_inputs)
+```
+
+or equivalently in Keras:
+
+```python
+encoder = hub.KerasLayer("path/to/encoder", trainable=True)
+encoder_outputs = encoder(encoder_inputs)
+```
+
+#### Details
+
+The `encoder_outputs` are a dict of Tensors with the following keys.
+
+
+*   `"sequence_output"`: a float32 Tensor of shape `[batch_size, seq_length,
+    dim]` with the context-aware embedding of each token of every packed input
+    sequence.
+*   `"pooled_output"`: a float32 Tensor of shape `[batch_size, dim]` with the
+    embedding of each input sequence as a whole, derived from sequence_output in
+    some trainable manner.
+*   `"default"`, as required by the API for text embeddings with preprocessed
+    inputs: a float32 Tensor of shape `[batch_size, dim]` with the embedding of
+    each input sequence. (This might be just an alias of pooled_output.)
+
+The contents of the `encoder_inputs` are not strictly required by this API
+definition. However, for encoders that use BERT-style inputs, it is recommended
+to use the following names (from the
+[NLP Modeling Toolkit of TensorFlow Model Garden](https://github.com/tensorflow/models/tree/master/official/nlp))
+to minimize friction in interchanging encoders and reusing preprocessor models:
+
+*   `"input_word_ids"`: an int32 Tensor of shape `[batch_size, seq_length]` with
+    the token ids of the packed input sequence (that is, including a
+    start-of-sequence token, end-of-segment tokens, and padding).
+*   `"input_mask"`: an int32 Tensor of shape `[batch_size, seq_length]` with
+    value 1 at the position of all input tokens present before padding and value
+    0 for the padding tokens.
+*   `"input_type_ids"`: an int32 Tensor of shape `[batch_size, seq_length]` with
+    the index of the input segment that gave rise to the input token at the
+    respective position. The first input segment (index 0) includes the
+    start-of-sequence token and its end-of-segment token. The second and later
+    segments (if present) include their respective end-of-segment token. Padding
+    tokens get index 0 again.
+
+### Distributed training
+
+For loading the preprocessor and encoder objects inside or outside a
+distribution strategy scope, the same rules apply as in the API for text
+embeddings with preprocessed inputs (see above).
+
+### Examples
+
+*   Colab tutorial
+    [Solve GLUE tasks using BERT on TPU](https://colab.research.google.com/github/tensorflow/text/blob/master/docs/tutorials/bert_glue.ipynb).
diff --git a/site/en/hub/common_signatures/images.md b/site/en/hub/common_signatures/images.md
new file mode 100644
index 00000000000..5e41c3e2960
--- /dev/null
+++ b/site/en/hub/common_signatures/images.md
@@ -0,0 +1,155 @@
+
+# Common Signatures for Images
+
+This page describes common signatures that should be implemented by modules in
+the [TF1 Hub format](../tf1_hub_module.md) for image-related tasks. (For the
+[TF2 SavedModel format](../tf2_saved_model.md), see the analogous
+[SavedModel API](../common_saved_model_apis/images.md).)
+
+Some modules can be used for more than one task (e.g., image classification
+modules tend to do some feature extraction on the way). Therefore, each module
+provides (1) named signatures for all the tasks anticipated by the publisher,
+and (2) a default signature `output = m(images)` for its designated primary
+task.
+
+
+## Image Feature Vector
+
+### Usage summary
+
+An **image feature vector** is a dense 1-D tensor that represents a whole image,
+typically for classification by the consumer model. (Unlike the intermediate
+activations of CNNs, it does not offer a spatial breakdown. Unlike [image
+classification](#classification), it discards the classification learned
+by the publisher model.)
+
+A module for image feature extraction has a default signature that maps a batch
+of images to a batch of feature vectors. It can be used like so:
+
+```python
+  module_spec = hub.load_module_spec("path/to/module")
+  height, width = hub.get_expected_image_size(module_spec)
+  images = ...  # A batch of images with shape [batch_size, height, width, 3].
+  module = hub.Module(module_spec)
+  features = module(images)   # A batch with shape [batch_size, num_features].
+```
+
+It also defines the corresponding named signature.
+
+### Signature specification
+
+The named signature for extracting image feature vectors is invoked as
+
+```python
+  outputs = module(dict(images=images), signature="image_feature_vector",
+                   as_dict=True)
+  features = outputs["default"]
+```
+
+The input follows the general convention for
+[input of images](#input).
+
+The outputs dictionary contains a `"default"` output of dtype `float32` and
+shape `[batch_size, num_features]`. The `batch_size` is the same as in the
+input, but not known at graph construction time. `num_features` is a known,
+module-specific constant independent of input size.
+
+These feature vectors are meant to be usable for classification with a simple
+feed-forward classifier (like the pooled features from the topmost convolutional
+layer in a typical CNN for image classification).
+
+Applying dropout to the output features (or not) should be left to the module
+consumer. The module itself should not perform dropout on the actual outputs
+(even if it uses dropout internally in other places).
+
+The outputs dictionary may provide further outputs, for example, the activations
+of hidden layers inside the module. Their keys and values are module-dependent.
+It is recommended to prefix architecture-dependent keys with an architecture
+name (e.g., to avoid confusing the intermediate layer `"InceptionV3/Mixed_5c"`
+with the topmost convolutional layer `"InceptionV2/Mixed_5c"`).
+
+
+## Image Classification
+
+### Usage summary
+
+**Image classification** maps the pixels of an image to linear scores (logits)
+for membership in the classes of a taxonomy _selected by the module publisher_.
+This allows consumers to draw conclusions from the particular classification
+learned by the publisher module, and not just its underlying features (cf.
+[Image Feature Vector](#feature-vector)).
+
+A module for image feature extraction has a default signature that maps a batch
+of images to a batch of logits. It can be used like so:
+
+```python
+  module_spec = hub.load_module_spec("path/to/module")
+  height, width = hub.get_expected_image_size(module_spec)
+  images = ...  # A batch of images with shape [batch_size, height, width, 3].
+  module = hub.Module(module_spec)
+  logits = module(images)   # A batch with shape [batch_size, num_classes].
+```
+
+It also defines the corresponding named signature.
+
+### Signature specification
+
+The named signature for extracting image feature vectors is invoked as
+
+```python
+  outputs = module(dict(images=images), signature="image_classification",
+                   as_dict=True)
+  logits = outputs["default"]
+```
+
+The input follows the general convention for
+[input of images](#input).
+
+The outputs dictionary contains a `"default"` output of dtype `float32` and
+shape `[batch_size, num_classes]`. The `batch_size` is the same as in the input,
+but not known at graph construction time. `num_classes` is the number of classes
+in the classification, which is a known constant independent of input size.
+
+Evaluating `outputs["default"][i, c]` yields a score predicting the membership
+of example `i` in the class with index `c`.
+
+It depends on the underlying classification whether these scores are meant to be
+used with softmax (for mutually exclusive classes), sigmoid (for orthogonal
+classes), or something else. The module documentation should describe this,
+and refer to a definition of the class indices.
+
+The outputs dictionary may provide further outputs, for example, the activations
+of hidden layers inside the module. Their keys and values are module-dependent.
+It is recommended to prefix architecture-dependent keys with an architecture
+name (e.g., to avoid confusing the intermediate layer `"InceptionV3/Mixed_5c"`
+with the topmost convolutional layer `"InceptionV2/Mixed_5c"`).
+
+
+## Image input
+
+This is common to all types of image modules and image signatures.
+
+A signature that takes a batch of images as input accepts them as a dense 4-D
+tensor of dtype `float32` and shape `[batch_size, height, width, 3]` whose
+elements are RGB color values of pixels normalized to the range [0, 1]. This is
+what you get from `tf.image.decode_*()` followed by
+`tf.image.convert_image_dtype(..., tf.float32)`.
+
+A module with exactly one (or one principal) input of images uses the name
+`"images"` for this input.
+
+The module accepts any `batch_size`, and correspondingly sets the first
+dimension of TensorInfo.tensor_shape to "unknown". The last dimension is fixed
+to the number `3` of RGB channels. The `height` and `width` dimensions are
+fixed to the expected size of input images. (Future work may remove that
+restriction for fully convolutional modules.)
+
+Consumers of the module should not inspect the shape directly, but obtain
+the size information by calling hub.get_expected_image_size()
+on the module or module spec, and are expected to resize input images
+accordingly (typically before/during batching).
+
+For simplicity, TF-Hub modules use the `channels_last`
+(or `NHWC`) layout of Tensors, and leave it to TensorFlow's graph optimizer
+to rewrite to `channels_first` (or `NCHW`) if needed. It has been doing that
+by default since TensorFlow version 1.7.
diff --git a/site/en/hub/common_signatures/index.md b/site/en/hub/common_signatures/index.md
new file mode 100644
index 00000000000..05eacc8b37f
--- /dev/null
+++ b/site/en/hub/common_signatures/index.md
@@ -0,0 +1,25 @@
+
+# Common Signatures for TF Hub Modules
+
+## Introduction
+
+[TensorFlow Hub](https://tfhub.dev) hosts models for a variety of tasks. Models
+for the same task are encouraged to implement a common API so that model
+consumers can easily exchange them without modifying the code that uses them,
+even if they come from different publishers.
+
+The goal is to make exchanging different models for the same task as simple as
+switching a string-valued hyperparameter. With that, model consumers can easily
+find the best one for their problem.
+
+This directory collects specifications of common signatures for modules in the
+[TF1 Hub format](../tf1_hub_module.md).
+
+Note that the TF1 Hub format has been **deprecated** in favor of the
+[TF2 SavedModel format](../tf2_saved_model.md) and its
+[Common SavedModel APIs](../common_saved_model_apis/index.md).
+
+## Signatures
+
+*   [Image Signatures](images.md)
+*   [Text Signatures](text.md)
diff --git a/site/en/hub/common_signatures/text.md b/site/en/hub/common_signatures/text.md
new file mode 100644
index 00000000000..3ea8f27c91d
--- /dev/null
+++ b/site/en/hub/common_signatures/text.md
@@ -0,0 +1,46 @@
+
+# Common Signatures for Text
+
+This page describes common signatures that should be implemented by modules in
+the [TF1 Hub format](../tf1_hub_module.md) for tasks that accept text inputs.
+(For the [TF2 SavedModel format](../tf2_saved_model.md), see the analogous
+[SavedModel API](../common_saved_model_apis/text.md).)
+
+## Text feature vector
+
+A **text feature vector** module creates a dense vector representation
+from text features.
+It accepts a batch of strings of shape `[batch_size]` and maps them to
+a `float32` tensor of shape `[batch_size, N]`. This is often called
+**text embedding** in dimension `N`.
+
+### Basic usage
+
+```python
+  embed = hub.Module("path/to/module")
+  representations = embed([
+      "A long sentence.",
+      "single-word",
+      "http://example.com"])
+```
+
+### Feature column usage
+
+```python
+    feature_columns = [
+      hub.text_embedding_column("comment", "path/to/module", trainable=False),
+    ]
+    input_fn = tf.estimator.inputs.numpy_input_fn(features, labels, shuffle=True)
+    estimator = tf.estimator.DNNClassifier(hidden_units, feature_columns)
+    estimator.train(input_fn, max_steps=100)
+```
+
+## Notes
+
+Modules have been pre-trained on different domains and/or tasks,
+and therefore not every text feature vector module would be suitable for
+your problem. E.g.: some modules could have been trained on a single language.
+
+This interface does not allow fine-tuning of the text representation on TPUs,
+because it requires the module to instantiate both string processing and the
+trainable variables at the same time.
diff --git a/site/en/hub/community.md b/site/en/hub/community.md
new file mode 100644
index 00000000000..a7a4c2bf0ec
--- /dev/null
+++ b/site/en/hub/community.md
@@ -0,0 +1,6 @@
+
+# Community and support
+
+*   The source code is available on [GitHub](https://github.com/tensorflow/hub).
+    We use [GitHub issues](https://github.com/tensorflow/hub/issues) for
+    tracking feature requests and bugs.
\ No newline at end of file
diff --git a/site/en/hub/contribute.md b/site/en/hub/contribute.md
new file mode 100644
index 00000000000..e537f79f766
--- /dev/null
+++ b/site/en/hub/contribute.md
@@ -0,0 +1,16 @@
+
+# Contribute
+
+To learn more about how to publish a model or model collection on
+[tfhub.dev](https://tfhub.dev/), see the [becoming_a_publisher](publish.md)
+guide.
+
+You can find more information of how to contribute to the
+[TensorFlow Hub library](https://github.com/tensorflow/hub) in our
+[GitHub contributing guide](https://github.com/tensorflow/hub/blob/master/CONTRIBUTING.md).
+
+Content published to tfhub.dev can be automatically mirrored to other model
+hubs, provided it follows a specified format and is permitted by our Terms
+(https://tfhub.dev/terms). See [our publishing documentation](publish.md) for
+more details, and [our contribution documentation](contribute_a_model.md) if
+you'd like to opt-out of mirroring.
diff --git a/site/en/hub/hosting.md b/site/en/hub/hosting.md
new file mode 100644
index 00000000000..ce2ce76b0a6
--- /dev/null
+++ b/site/en/hub/hosting.md
@@ -0,0 +1,175 @@
+
+# Model hosting protocol
+
+This document describes the URL conventions used when hosting all model types on
+[tfhub.dev](https://tfhub.dev) - TFJS, TF Lite and TensorFlow models. It also
+describes the HTTP(S)-based protocol implemented by the `tensorflow_hub` library
+in order to load TensorFlow models from [tfhub.dev](https://tfhub.dev) and
+compatible services into TensorFlow programs.
+
+Its key feature is to use the same URL in code to load a model and in a browser
+to view the model documentation.
+
+## General URL conventions
+
+[tfhub.dev](https://tfhub.dev) supports the following URL formats:
+
+*   TF Hub publishers follow `https://tfhub.dev/`
+*   TF Hub collections follow
+    `https://tfhub.dev//collection/`
+*   TF Hub models have versioned url
+    `https://tfhub.dev///` and unversioned url
+    `https://tfhub.dev//` that resolves to the latest
+    version of the model.
+
+TF Hub models can be downloaded as compressed assets by appending URL parameters
+to the [tfhub.dev](https://tfhub.dev) model URL. However, the URL parameters
+required to achieve that depend on the model type:
+
+*   TensorFlow models (both SavedModel and TF1 Hub formats): append
+    `?tf-hub-format=compressed` to the TensorFlow model url.
+*   TFJS models: append `?tfjs-format=compressed` to the TFJS model url to
+    download the compressed or `/model.json?tfjs-format=file` to read if from
+    remote storage.
+*   TF lite models: append `?lite-format=tflite` to the TF Lite model url.
+
+For example:
+
+
+  
+    +    +    +    +    +    
+    
+    
+    
+    
+  
+  
+    
+    
+    
+    
+    
+  
+  
+    
+    
+    
+    
+    
+  
+  
+    
+    
+    
+    
+    
+  
+






TypeModel URLDownload typeURL paramDownload URL
TensorFlow (SavedModel, TF1 Hub format)https://tfhub.dev/google/spice/2.tar.gz?tf-hub-format=compressed https://tfhub.dev/google/spice/2?tf-hub-format=compressed
TF Litehttps://tfhub.dev/google/lite-model/spice/1.tflite?lite-format=tflitehttps://tfhub.dev/google/lite-model/spice/1?lite-format=tflite
TF.jshttps://tfhub.dev/google/tfjs-model/spice/2/default/1.tar.gz?tfjs-format=compressedhttps://tfhub.dev/google/tfjs-model/spice/2/default/1?tfjs-format=compressed

+
+Additionally, some models also are hosted in a format that can be read directly
+from remote storage without being downloaded. This is especially useful if there
+is no local storage available, such as running a TF.js model in the browser or
+loading a SavedModel on [Colab](https://colab.research.google.com/). Be
+conscious that reading models that are hosted remotely without being downloaded
+locally may increase latency.
+
+
+  
+    +    +    +    +    +    
+    
+    
+    
+    
+  
+  
+    
+    
+    
+    
+    
+  
+  
+    
+    
+    
+    
+    
+  
+






TypeModel URLResponse typeURL paramRequest URL
TensorFlow (SavedModel, TF1 Hub format)https://tfhub.dev/google/spice/2String (Path to GCS folder where the uncompressed model is stored)?tf-hub-format=uncompressedhttps://tfhub.dev/google/spice/2?tf-hub-format=uncompressed
TF.jshttps://tfhub.dev/google/tfjs-model/spice/2/default/1.json?tfjs-format=filehttps://tfhub.dev/google/tfjs-model/spice/2/default/1/model.json?tfjs-format=file

+
+## tensorflow_hub library protocol
+
+This section describes how we host models on [tfhub.dev](https://tfhub.dev) for
+use with the tensorflow_hub library. If you want to host your own model
+repository to work with the tensorflow_hub library, your HTTP(s) distribution
+service should provide an implementation of this protocol.
+
+Note that this section does not address hosting TF Lite and TFJS models since
+they are not downloaded via the `tensorflow_hub` library. For more information
+on hosting these model types, please check [above](#general-url-conventions).
+
+### Compressed Hosting
+
+Models are stored on [tfhub.dev](https://tfhub.dev) as compressed tar.gz files.
+By default, the tensorflow_hub library automatically downloads the compressed
+model. They can also be manually downloaded by appending the
+`?tf-hub-format=compressed` to the model url, for example:
+
+```shell
+wget https://tfhub.dev/tensorflow/albert_en_xxlarge/1?tf-hub-format=compressed
+```
+
+The root of the archive is the root of the model directory and should contain a
+SavedModel, as in this example:
+
+```shell
+# Create a compressed model from a SavedModel directory.
+$ tar -cz -f model.tar.gz --owner=0 --group=0 -C /tmp/export-model/ .
+
+# Inspect files inside a compressed model
+$ tar -tf model.tar.gz
+./
+./variables/
+./variables/variables.data-00000-of-00001
+./variables/variables.index
+./assets/
+./saved_model.pb
+```
+
+Tarballs for use with the legacy
+[TF1 Hub format](https://www.tensorflow.org/hub/tf1_hub_module) will also
+contain a `./tfhub_module.pb` file.
+
+When one of `tensorflow_hub` library model loading APIs is invoked
+([hub.KerasLayer](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer),
+[hub.load](https://www.tensorflow.org/hub/api_docs/python/hub/load), etc) the
+library downloads the model, uncompresses the model and caches it locally. The
+`tensorflow_hub` library expects that model URLs are versioned and that the
+model content of a given version is immutable, so that it can be cached
+indefinitely. Learn more about [caching models](caching.md).
+
+![](https://raw.githubusercontent.com/tensorflow/hub/master/docs/images/library_download_cache.png)
+
+### Uncompressed Hosting
+
+When the environment variable `TFHUB_MODEL_LOAD_FORMAT` or the command-line flag
+`--tfhub_model_load_format` is set to `UNCOMPRESSED`, the model is read directly
+from remote storage (GCS) instead of being downloaded and uncompressed locally.
+When this behavior is enabled the library appends `?tf-hub-format=uncompressed`
+to the model URL. That request returns the path to the folder on GCS that
+contains the uncompressed model files. As an example, \
+`https://tfhub.dev/google/spice/2?tf-hub-format=uncompressed` \
+returns \
+`gs://kaggle-tfhub-models-uncompressed/tfhub-modules/google/spice/2/uncompressed`
+in the body of the 303 response. The library then reads the model from that GCS
+destination.
diff --git a/site/en/hub/images/action_recognition.gif b/site/en/hub/images/action_recognition.gif
new file mode 100644
index 00000000000..a58c22ac8c3
Binary files /dev/null and b/site/en/hub/images/action_recognition.gif differ
diff --git a/site/en/hub/images/bert.png b/site/en/hub/images/bert.png
new file mode 100644
index 00000000000..e36f69c9c7b
Binary files /dev/null and b/site/en/hub/images/bert.png differ
diff --git a/site/en/hub/images/bert_preprocess.png b/site/en/hub/images/bert_preprocess.png
new file mode 100644
index 00000000000..18b3b435d1b
Binary files /dev/null and b/site/en/hub/images/bert_preprocess.png differ
diff --git a/site/en/hub/images/bert_preprocess_wide.png b/site/en/hub/images/bert_preprocess_wide.png
new file mode 100644
index 00000000000..b414196724e
Binary files /dev/null and b/site/en/hub/images/bert_preprocess_wide.png differ
diff --git a/site/en/hub/images/bit_blog.jpg b/site/en/hub/images/bit_blog.jpg
new file mode 100644
index 00000000000..260415bf0b1
Binary files /dev/null and b/site/en/hub/images/bit_blog.jpg differ
diff --git a/site/en/hub/images/boundless.png b/site/en/hub/images/boundless.png
new file mode 100644
index 00000000000..ccc52d17f84
Binary files /dev/null and b/site/en/hub/images/boundless.png differ
diff --git a/site/en/hub/images/colab_logo.svg b/site/en/hub/images/colab_logo.svg
new file mode 100644
index 00000000000..d03f1106221
--- /dev/null
+++ b/site/en/hub/images/colab_logo.svg
@@ -0,0 +1 @@
+
diff --git a/site/en/hub/images/food.png b/site/en/hub/images/food.png
new file mode 100644
index 00000000000..41865be3984
Binary files /dev/null and b/site/en/hub/images/food.png differ
diff --git a/site/en/hub/images/gan_faces.gif b/site/en/hub/images/gan_faces.gif
new file mode 100644
index 00000000000..a34b8d517f4
Binary files /dev/null and b/site/en/hub/images/gan_faces.gif differ
diff --git a/site/en/hub/images/github_icon.svg b/site/en/hub/images/github_icon.svg
new file mode 100644
index 00000000000..0a607bb98b3
--- /dev/null
+++ b/site/en/hub/images/github_icon.svg
@@ -0,0 +1 @@
+
diff --git a/site/en/hub/images/guide_basics.png b/site/en/hub/images/guide_basics.png
new file mode 100644
index 00000000000..e6aee34f516
Binary files /dev/null and b/site/en/hub/images/guide_basics.png differ
diff --git a/site/en/hub/images/image_classification.png b/site/en/hub/images/image_classification.png
new file mode 100644
index 00000000000..a3840e3482c
Binary files /dev/null and b/site/en/hub/images/image_classification.png differ
diff --git a/site/en/hub/images/interpolation.png b/site/en/hub/images/interpolation.png
new file mode 100644
index 00000000000..d2f062da7c1
Binary files /dev/null and b/site/en/hub/images/interpolation.png differ
diff --git a/site/en/hub/images/library_download_cache.png b/site/en/hub/images/library_download_cache.png
new file mode 100644
index 00000000000..1b581a4a819
Binary files /dev/null and b/site/en/hub/images/library_download_cache.png differ
diff --git a/site/en/hub/images/object_detection.png b/site/en/hub/images/object_detection.png
new file mode 100644
index 00000000000..57b327099ae
Binary files /dev/null and b/site/en/hub/images/object_detection.png differ
diff --git a/site/en/hub/images/odml.png b/site/en/hub/images/odml.png
new file mode 100644
index 00000000000..29bf3bcc61b
Binary files /dev/null and b/site/en/hub/images/odml.png differ
diff --git a/site/en/hub/images/similarity.png b/site/en/hub/images/similarity.png
new file mode 100644
index 00000000000..3155e8706e1
Binary files /dev/null and b/site/en/hub/images/similarity.png differ
diff --git a/site/en/hub/images/spice_blog.png b/site/en/hub/images/spice_blog.png
new file mode 100644
index 00000000000..cf19769e6d8
Binary files /dev/null and b/site/en/hub/images/spice_blog.png differ
diff --git a/site/en/hub/images/spice_color.png b/site/en/hub/images/spice_color.png
new file mode 100644
index 00000000000..35b68d7c444
Binary files /dev/null and b/site/en/hub/images/spice_color.png differ
diff --git a/site/en/hub/images/stackoverflow_icon.svg b/site/en/hub/images/stackoverflow_icon.svg
new file mode 100644
index 00000000000..491a75e464d
--- /dev/null
+++ b/site/en/hub/images/stackoverflow_icon.svg
@@ -0,0 +1 @@
+
diff --git a/site/en/hub/images/style_transfer.png b/site/en/hub/images/style_transfer.png
new file mode 100644
index 00000000000..d0427408830
Binary files /dev/null and b/site/en/hub/images/style_transfer.png differ
diff --git a/site/en/hub/images/super_resolution.png b/site/en/hub/images/super_resolution.png
new file mode 100644
index 00000000000..7d3f3741077
Binary files /dev/null and b/site/en/hub/images/super_resolution.png differ
diff --git a/site/en/hub/images/text_video.gif b/site/en/hub/images/text_video.gif
new file mode 100644
index 00000000000..5fe639b1eea
Binary files /dev/null and b/site/en/hub/images/text_video.gif differ
diff --git a/site/en/hub/images/yamnet.png b/site/en/hub/images/yamnet.png
new file mode 100644
index 00000000000..416956ad6fb
Binary files /dev/null and b/site/en/hub/images/yamnet.png differ
diff --git a/site/en/hub/installation.md b/site/en/hub/installation.md
new file mode 100644
index 00000000000..2381fbea614
--- /dev/null
+++ b/site/en/hub/installation.md
@@ -0,0 +1,57 @@
+
+# Installation
+
+## Installing tensorflow_hub
+
+The `tensorflow_hub` library can be installed alongside TensorFlow 1 and
+TensorFlow 2. We recommend that new users start with TensorFlow 2 right away,
+and current users upgrade to it.
+
+### Use with TensorFlow 2
+
+Use [pip](https://pip.pypa.io/) to
+[install TensorFlow 2](https://www.tensorflow.org/install) as usual. (See there
+for extra instructions about GPU support.) Then install a current version of
+[`tensorflow-hub`](https://pypi.org/project/tensorflow-hub/) next to it (must be
+0.5.0 or newer).
+
+```bash
+$ pip install "tensorflow>=2.0.0"
+$ pip install --upgrade tensorflow-hub
+```
+
+The TF1-style API of TensorFlow Hub works with the v1 compatibility mode of
+TensorFlow 2.
+
+### Legacy use with TensorFlow 1
+
+TensorFlow 1.15 is the only version of TensorFlow 1.x still supported by the
+`tensorflow_hub` library (as of release 0.11.0). TensorFlow 1.15 defaults to
+TF1-compatible behavior but contains many TF2 features under the hood to allow
+some use of TensorFlow Hub's TF2-style APIs.
+
+```bash
+$ pip install "tensorflow>=1.15,<2.0"
+$ pip install --upgrade tensorflow-hub
+```
+
+### Use of pre-release versions
+
+The pip packages `tf-nightly` and `tf-hub-nightly` are built automatically from
+the source code on github, with no release testing. This lets developers try out
+the latest code without [building from source](build_from_source.md).
+
+```bash
+$ pip install tf-nightly
+$ pip install --upgrade tf-hub-nightly
+```
+
+## Next Steps
+
+-   [Library overview](lib_overview.md)
+-   Tutorials:
+    -   [Text classification](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_text_classification.ipynb)
+    -   [Image classification](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_image_retraining.ipynb)
+    -   Additional examples
+        [on GitHub](https://github.com/tensorflow/hub/blob/master/examples/README.md)
+-   Find models on [tfhub.dev](https://tfhub.dev).
\ No newline at end of file
diff --git a/site/en/hub/lib_overview.md b/site/en/hub/lib_overview.md
new file mode 100644
index 00000000000..c480ad2fbdf
--- /dev/null
+++ b/site/en/hub/lib_overview.md
@@ -0,0 +1,50 @@
+
+# TensorFlow Hub Library Overview
+
+The [`tensorflow_hub`](https://github.com/tensorflow/hub) library lets you
+download and reuse trained models in your TensorFlow program with a minimum
+amount of code. The main way to load a trained model is using the
+`hub.KerasLayer` API.
+
+```python
+import tensorflow_hub as hub
+
+embed = hub.KerasLayer("https://tfhub.dev/google/nnlm-en-dim128/2")
+embeddings = embed(["A long sentence.", "single-word", "http://example.com"])
+print(embeddings.shape, embeddings.dtype)
+```
+**Note:** This documentation uses TFhub.dev URL handles in examples. See more
+information regarding other valid handle types [here](tf2_saved_model.md#model_handles).
+
+## Setting the cache location for downloads.
+
+By default, `tensorflow_hub` uses a system-wide, temporary directory to cache
+downloaded and uncompressed models. See [Caching](caching.md) for options to use
+other, possibly more persistent locations.
+
+## API stability
+
+Although we hope to prevent breaking changes, this project is still under active
+development and is not yet guaranteed to have a stable API or model format.
+
+## Fairness
+
+As in all of machine learning, [fairness](http://ml-fairness.com) is an
+[important](https://research.googleblog.com/2016/10/equality-of-opportunity-in-machine.html)
+consideration. Many pre-trained models are trained on large datasets. When
+reusing any model, it’s important to be mindful of what data the model was
+trained on (and whether there are any existing biases there), and how these
+might impact your use of it.
+
+## Security
+
+Since they contain arbitrary TensorFlow graphs, models can be thought of as
+programs.
+[Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)
+describes the security implications of referencing a model from an untrusted
+source.
+
+## Next Steps
+
+-   [Use the library](tf2_saved_model.md)
+-   [Reusable SavedModels](reusable_saved_models.md)
diff --git a/site/en/hub/migration_tf2.md b/site/en/hub/migration_tf2.md
new file mode 100644
index 00000000000..c2cc4b50759
--- /dev/null
+++ b/site/en/hub/migration_tf2.md
@@ -0,0 +1,114 @@
+
+# Migrating from TF1 to TF2 with TensorFlow Hub
+
+This page explains how to keep using TensorFlow Hub while migrating your
+TensorFlow code from TensorFlow 1 to TensorFlow 2. It complements TensorFlow's
+general [migration guide](https://www.tensorflow.org/guide/migrate).
+
+For TF2, TF Hub has switched away from the legacy `hub.Module` API for building
+a `tf.compat.v1.Graph` like `tf.contrib.v1.layers` do. Instead, there is now a
+`hub.KerasLayer` for use alongside other Keras layers for building a
+`tf.keras.Model` (typically in TF2's new
+[eager execution environment](https://www.tensorflow.org/api_docs/python/tf/executing_eagerly))
+and its underlying `hub.load()` method for low-level TensorFlow code.
+
+The `hub.Module` API remains available in the `tensorflow_hub` library for use
+in TF1 and in the TF1 compatibility mode of TF2. It can only load models in the
+[TF1 Hub format](tf1_hub_module.md).
+
+The new API of `hub.load()` and `hub.KerasLayer` works for TensorFlow 1.15 (in
+eager and graph mode) and in TensorFlow 2. This new API can load the new
+[TF2 SavedModel](tf2_saved_model.md) assets, and, with the restrictions laid out
+in the [model compatibility guide](model_compatibility.md), the legacy models in
+TF1 Hub format.
+
+In general, it is recommended to use new API wherever possible.
+
+## Summary of the new API
+
+`hub.load()` is the new low-level function to load a SavedModel from TensorFlow
+Hub (or compatible services). It wraps TF2's `tf.saved_model.load()`;
+TensorFlow's [SavedModel Guide](https://www.tensorflow.org/guide/saved_model)
+describes what you can do with the result.
+
+```python
+m = hub.load(handle)
+outputs = m(inputs)
+```
+
+The `hub.KerasLayer` class calls `hub.load()` and adapts the result for use in
+Keras alongside other Keras layers. (It may even be a convenient wrapper for
+loaded SavedModels used in other ways.)
+
+```python
+model = tf.keras.Sequential([
+    hub.KerasLayer(handle),
+    ...])
+```
+
+Many tutorials show these APIs in action. Here are some examples:
+
+*   [Text classification example notebook](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_text_classification.ipynb)
+*   [Image classification example notebook](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_image_retraining.ipynb)
+
+### Using the new API in Estimator training
+
+If you use a TF2 SavedModel in an Estimator for training with parameter servers
+(or otherwise in a TF1 Session with variables placed on remote devices), you
+need to set `experimental.share_cluster_devices_in_session` in the tf.Session's
+ConfigProto, or else you will get an error like "Assigned device
+'/job:ps/replica:0/task:0/device:CPU:0' does not match any device."
+
+The necessary option can be set like
+
+```python
+session_config = tf.compat.v1.ConfigProto()
+session_config.experimental.share_cluster_devices_in_session = True
+run_config = tf.estimator.RunConfig(..., session_config=session_config)
+estimator = tf.estimator.Estimator(..., config=run_config)
+```
+
+Starting with TF2.2, this option is no longer experimental, and the
+`.experimental` piece can be dropped.
+
+## Loading legacy models in TF1 Hub format
+
+It can happen that a new TF2 SavedModel is not yet available for your use-case
+and you need to load an legacy model in TF1 Hub format. Starting in
+`tensorflow_hub` release 0.7, you can use legacy model in TF1 Hub format
+together with `hub.KerasLayer` as shown below:
+
+```python
+m = hub.KerasLayer(handle)
+tensor_out = m(tensor_in)
+```
+
+Additionally `KerasLayer` exposes the ability to specify `tags`, `signature`,
+`output_key` and `signature_outputs_as_dict` for more specific usages of legacy
+models in TF1 Hub format and legacy SavedModels.
+
+For more information on TF1 Hub format compatibility see the
+[model compatibility guide](model_compatibility.md).
+
+## Using lower level APIs
+
+Legacy TF1 Hub format models can be loaded via `tf.saved_model.load`. Instead of
+
+```python
+# DEPRECATED: TensorFlow 1
+m = hub.Module(handle, tags={"foo", "bar"})
+tensors_out_dict = m(dict(x1=..., x2=...), signature="sig", as_dict=True)
+```
+
+it is recommended to use:
+
+```python
+# TensorFlow 2
+m = hub.load(path, tags={"foo", "bar"})
+tensors_out_dict = m.signatures["sig"](x1=..., x2=...)
+```
+
+In these examples `m.signatures` is a dict of TensorFlow
+[concrete functions](https://www.tensorflow.org/tutorials/customization/performance#tracing)
+keyed by signature names. Calling such a function computes all its outputs, even
+if unused. (This is different from the lazy evaluation of TF1's graph mode.)
diff --git a/site/en/hub/model_compatibility.md b/site/en/hub/model_compatibility.md
new file mode 100644
index 00000000000..e37ed717c3b
--- /dev/null
+++ b/site/en/hub/model_compatibility.md
@@ -0,0 +1,144 @@
+
+# Model compatibility for TF1/TF2
+
+## TF Hub model formats
+
+TF Hub offers reusable model pieces that can be loaded back, built upon, and
+possibly be retrained in a TensorFlow program. These come in two different
+formats:
+
+*   The custom [TF1 Hub format](https://www.tensorflow.org/hub/tf1_hub_module) .
+    Its main intended use is in TF1 (or TF1 compatibility mode in TF2) via its
+    [hub.Module API](https://www.tensorflow.org/hub/api_docs/python/hub/Module).
+    Full compatibility details [below](#compatibility_of_hubmodule).
+*   The native [TF2 SavedModel](https://www.tensorflow.org/hub/tf2_saved_model)
+    format. Its main intended use is in TF2 via the
+    [hub.load](https://www.tensorflow.org/hub/api_docs/python/hub/load) and
+    [hub.KerasLayer](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer)
+    APIs. Full compatibility details [below](#compatibility_of_tf2_savedmodel).
+
+The model format can be found on the model page on
+[tfhub.dev](https://tfhub.dev). Model **loading/inference**, **fine-tuning** or
+**creation** might not be supported in TF1/2 based on the model formats.
+
+## Compatibility of the TF1 Hub format {:#compatibility_of_hubmodule}
+
+
+  
+    +    +    +    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+




OperationTF1/ TF1 compat mode in TF2 [1]TF2
Loading / Inference
+      Fully supported (complete TF1 Hub format loading guide)
+      
m = hub.Module(handle)
+outputs = m(inputs)

+    		 It's recommended to use either hub.load
+    
m = hub.load(handle)
+outputs = m.signatures["sig"](inputs)

+      or hub.KerasLayer
+      
m = hub.KerasLayer(handle, signature="sig")
+outputs = m(inputs)

+    
Fine-tuning
+      Fully supported (complete TF1 Hub format fine-tuning guide)
+    
m = hub.Module(handle,
+               trainable=True,
+               tags=["train"]*is_training)
+outputs = m(inputs)

+      

+      Note: modules that don't need a separate train graph don't have a train
+        tag.
+      

+    		
+      Not supported
+    
Creation Fully supported (see complete TF1 Hub format creation guide) 
 

+      Note: The TF1 Hub format is geared towards TF1 and is only partially supported in TF2. Consider creating a TF2 SavedModel.
+      
Not supported

+
+## Compatibility of TF2 SavedModel {:#compatibility_of_tf2_savedmodel}
+
+Not supported before TF1.15.
+
+  
+    +    +    +    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+    
+  
+  
+    
+    
+   
+  
+




OperationTF1.15/ TF1 compat mode in TF2 [1]TF2
Loading / Inference
+      Use either hub.load
+    
m = hub.load(handle)
+outputs = m(inputs)

+      or hub.KerasLayer
+      
m = hub.KerasLayer(handle)
+outputs = m(inputs)

+    		 Fully supported (complete TF2 SavedModel loading guide). Use either hub.load
+    
m = hub.load(handle)
+outputs = m(inputs)

+      or hub.KerasLayer
+      
m = hub.KerasLayer(handle)
+outputs = m(inputs)

+    
Fine-tuning
+      Supported for a hub.KerasLayer used in  tf.keras.Model when trained with
+      Model.fit() or trained in an Estimator whose model_fn wraps the Model per the custom model_fn guide.
+      

+        Note: hub.KerasLayer does not
+        fill in graph collections like the old tf.compat.v1.layers or hub.Module
+        APIs did.
+      

+    		
+      Fully supported (complete TF2 SavedModel fine-tuning guide).
+      Use either hub.load:
+      
m = hub.load(handle)
+outputs = m(inputs, training=is_training)

+      or hub.KerasLayer:
+      
m =  hub.KerasLayer(handle, trainable=True)
+outputs = m(inputs)

+    
Creation
+     The TF2 API 
+      tf.saved_model.save() can be called from within compat mode.
+   Fully supported (see complete TF2 SavedModel creation guide) 

+
+
[1] "TF1 compat mode in TF2" refers to the combined
+  effect of importing TF2 with
+  import tensorflow.compat.v1 as tf
+  and running
+  tf.disable_v2_behavior()
+ as described in the
+  TensorFlow Migration guide
+  .

diff --git a/site/en/hub/model_formats.md b/site/en/hub/model_formats.md
new file mode 100644
index 00000000000..73ae7c247a1
--- /dev/null
+++ b/site/en/hub/model_formats.md
@@ -0,0 +1,79 @@
+
+# Model formats
+
+[tfhub.dev](https://tfhub.dev) hosts the following model
+formats: TF2 SavedModel, TF1 Hub format, TF.js and TFLite. This page provides an
+overview of each model format.
+
+Content published to tfhub.dev can be automatically mirrored to other model
+hubs, provided it follows a specified format and is permitted by our Terms
+(https://tfhub.dev/terms). See [our publishing documentation](publish.md) for
+more details, and [our contribution documentation](contribute_a_model.md) if
+you'd like to opt-out of mirroring.
+
+## TensorFlow formats
+
+[tfhub.dev](https://tfhub.dev) hosts TensorFlow models in the TF2 SavedModel
+format and TF1 Hub format. We recommend using models in the standardized TF2
+SavedModel format instead of the deprecated TF1 Hub format when possible.
+
+### SavedModel
+
+TF2 SavedModel is the recommended format for sharing TensorFlow models. You can
+learn more about the SavedModel format in the
+[TensorFlow SavedModel](https://www.tensorflow.org/guide/saved_model) guide.
+
+You can browse SavedModels on tfhub.dev by using the TF2 version filter on the
+[tfhub.dev browse page](https://tfhub.dev/s?subtype=module,placeholder) or by
+following
+[this link](https://tfhub.dev/s?subtype=module,placeholder&tf-version=tf2).
+
+You can use SavedModels from tfhub.dev without depending on the `tensorflow_hub`
+library, since this format is a part of core TensorFlow.
+
+Learn more about SavedModels on TF Hub:
+
+*   [Using TF2 SavedModels](tf2_saved_model.md)
+*   [Exporting a TF2 SavedModel](exporting_tf2_saved_model.md)
+*   [TF1/TF2 compatibility of TF2 SavedModels](model_compatibility.md)
+
+### TF1 Hub format
+
+The TF1 Hub format is a custom serialization format used in by TF Hub library.
+The TF1 Hub format is similar to the SavedModel format of TensorFlow 1 on a
+syntactic level (same file names and protocol messages) but semantically
+different to allow for module reuse, composition and re-training (e.g.,
+different storage of resource initializers, different tagging conventions for
+metagraphs). The easiest way to tell them apart on disk is the presence or
+absence of the `tfhub_module.pb` file.
+
+You can browse models in the TF1 Hub format on tfhub.dev by using the TF1
+version filter on the
+[tfhub.dev browse page](https://tfhub.dev/s?subtype=module,placeholder) or by
+following
+[this link](https://tfhub.dev/s?subtype=module,placeholder&tf-version=tf1).
+
+Learn more about models in TF1 Hub format on TF Hub:
+
+*   [Using TF1 Hub format models](tf1_hub_module.md)
+*   [Exporting a model in the TF1 Hub format](exporting_hub_format.md)
+*   [TF1/TF2 compatibility of TF1 Hub format](model_compatibility.md)
+
+## TFLite format
+
+The TFLite format is used for on-device inference. You can learn more at the
+[TFLite documentation](https://www.tensorflow.org/lite).
+
+You can browse TF Lite models on tfhub.dev by using the TF Lite model format
+filter on the
+[tfhub.dev browse page](https://tfhub.dev/s?subtype=module,placeholder) or by
+following [this link](https://tfhub.dev/lite).
+
+## TFJS format
+
+The TF.js format is used for in-browser ML. You can learn more at the
+[TF.js documentation](https://www.tensorflow.org/js).
+
+You can browse TF.js models on tfhub.dev by using the TF.js model format filter
+on the [tfhub.dev browse page](https://tfhub.dev/s?subtype=module,placeholder)
+or by following [this link](https://tfhub.dev/js).
diff --git a/site/en/hub/overview.md b/site/en/hub/overview.md
new file mode 100644
index 00000000000..b6d814eba73
--- /dev/null
+++ b/site/en/hub/overview.md
@@ -0,0 +1,31 @@
+
+# TensorFlow Hub
+
+TensorFlow Hub is an open repository and library for reusable machine learning.
+The [tfhub.dev](https://tfhub.dev) repository provides many pre-trained models:
+text embeddings, image classification models, TF.js/TFLite models and much more.
+The repository is open to
+[community contributors](https://tfhub.dev/s?subtype=publisher).
+
+The [`tensorflow_hub`](https://github.com/tensorflow/hub) library lets you
+download and reuse them in your TensorFlow program with a minimum amount of
+code.
+
+```python
+import tensorflow_hub as hub
+
+model = hub.KerasLayer("https://tfhub.dev/google/nnlm-en-dim128/2")
+embeddings = model(["The rain in Spain.", "falls",
+                    "mainly", "In the plain!"])
+
+print(embeddings.shape)  #(4,128)
+```
+
+## Next Steps
+
+-   [Find models on tfhub.dev](https://tfhub.dev)
+-   [Publish models on tfhub.dev](publish.md)
+-   TensorFlow Hub library
+    -   [Install TensorFlow Hub](installation.md)
+    -   [Library overview](lib_overview.md)
+-   [Follow tutorials](tutorials)
diff --git a/site/en/hub/portability_and_deletion.md b/site/en/hub/portability_and_deletion.md
new file mode 100644
index 00000000000..67fa401d161
--- /dev/null
+++ b/site/en/hub/portability_and_deletion.md
@@ -0,0 +1,18 @@
+
+## I want to see what I’ve uploaded to TensorFlow Hub. Can I get a copy of my data?
+
+Yes. If you’d like the Kaggle Team to **send you a copy** of all of the
+data you have uploaded, please send us an email at [support@kaggle.com](mailto:support@kaggle.com)
+and we’ll respond as soon as possible.
+
+## How do I delete what I’ve uploaded to TensorFlow Hub?
+
+Similarly, if you’d like us to **delete or remove content**, please send us an
+email at [support@kaggle.com](mailto:support@kaggle.com) and we’ll delete
+all copies that we have and stop serving it on tfhub.dev. Please note:
+
+*   Because TensorFlow Hub is an open-source platform, copies of your assets may
+still be retained by members of the public.
+*   Deletion is permanent and cannot be undone.
+*   Deletion can cause downstream breakages if users are not caching your model
+locally and/or are not properly warned prior to deletion.
diff --git a/site/en/hub/publish.md b/site/en/hub/publish.md
new file mode 100644
index 00000000000..7fc5e7c1751
--- /dev/null
+++ b/site/en/hub/publish.md
@@ -0,0 +1,19 @@
+
+# Publishing Process
+
+Thank you for considering to publish your models!
+
+**Please join the Early Access Model Publishing (EAP) on
+[Kaggle Models](https://www.kaggle.com/models):**
+
+-   Email [kaggle-models@google.com](mailto:kaggle-models@google.com) and
+    provide the following to get access to EAP:
+    -   (1) Your Kaggle username
+    -   (2) Your desired organization slug
+    -   (3) A URL to a square-shaped profile image (which is needed for the
+        organization creation)
+-   Follow the
+    [documentation instructions](https://www.kaggle.com/model-publishing-instructions)
+    to create and publish your model
+-   Feel free to raise any questions and get support from
+    [Kaggle Discord channel](https://discord.gg/rKEyxj9WF)
diff --git a/site/en/hub/reusable_saved_models.md b/site/en/hub/reusable_saved_models.md
new file mode 100644
index 00000000000..b2114135d77
--- /dev/null
+++ b/site/en/hub/reusable_saved_models.md
@@ -0,0 +1,208 @@
+
+# Reusable SavedModels
+
+## Introduction
+
+TensorFlow Hub hosts SavedModels for TensorFlow 2, among other assets.
+They can be loaded back into a Python program with `obj = hub.load(url)`
+[[learn more](tf2_saved_model)]. The returned `obj` is the result
+of `tf.saved_model.load()` (see TensorFlow's
+[SavedModel guide](https://www.tensorflow.org/guide/saved_model)).
+This object can have arbitrary attributes that are tf.functions,
+tf.Variables (initialized from their pre-trained values), other resources
+and, recursively, more such objects.
+
+This page describes an interface to be implemented by the loaded `obj`
+in order to be *reused* in a TensorFlow Python program.
+SavedModels conforming to this interface are called *Reusable SavedModels*.
+
+Reusing means building a larger model around `obj`, including the ability
+to fine-tune it. Fine-tuning means further training of the weights in the loaded
+`obj` as part of the surrounding model. The loss function and the
+optimizer are determined by the surrounding model; `obj` only defines
+the mapping of input to output activations (the "forward pass"), possibly
+including techniques such as dropout or batch normalization.
+
+**The TensorFlow Hub team recommends implementing the Reusable SavedModel
+interface** in all SavedModels that are meant to be reused in the above sense.
+Many utilities from the `tensorflow_hub` library, notably `hub.KerasLayer`,
+require SavedModels to implement it.
+
+### Relation to SignatureDefs
+
+This interface in terms of tf.functions and other TF2 features
+is separate from the SavedModel's signatures, which have been
+available since TF1 and continue to be used in TF2 for inference
+(such as deploying SavedModels to TF Serving or TF Lite).
+Signatures for inference are not expressive enough to support fine-tuning,
+and [`tf.function`](https://www.tensorflow.org/api_docs/python/tf/function)
+provides a more natural and expressive
+[Python API](https://www.tensorflow.org/tutorials/customization/performance)
+for the reused model.
+
+### Relation to model-building libraries
+
+A Reusable SavedModel uses only TensorFlow 2 primitives, independent of any
+particular model-building library like Keras or Sonnet. This facilitates reuse
+across model-building libraries, free from dependencies on the original
+model-building code.
+
+Some amount of adaptation will be needed load Reusable SavedModels into or save
+them from any given model-building library. For Keras,
+[hub.KerasLayer](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer)
+provides the loading, and Keras's built-in saving in the SavedModel format has
+been redesigned for TF2 with the goal of providing a superset of this interface
+(see the
+[RFC](https://github.com/tensorflow/community/blob/master/rfcs/20190509-keras-saved-model.md)
+from May 2019).
+
+### Relation to task-specific "Common SavedModel APIs"
+
+The interface definition on this page allows for any number and type of inputs
+and outputs. The
+[Common SavedModel APIs for TF Hub](common_saved_model_apis/index.md) refine
+this general interface with usage conventions for specific tasks to make models
+easily interchangeable.
+
+## Interface definition
+
+### Attributes
+
+A Reusable SavedModel is a TensorFlow 2 SavedModel such that
+`obj = tf.saved_model.load(...)` returns an object that has the following
+attributes
+
+  * `__call__`. Required. A tf.function implementing the model's computation
+    (the "forward pass") subject to the specification below.
+
+  * `variables`: A list of tf.Variable objects, listing all the variables
+    used by any possible invocation of `__call__`, including both
+    trainable and non-trainable ones.
+
+    This list can be omitted if empty.
+
+    Note: Conveniently, this name coincides with the attribute synthesized by
+    `tf.saved_model.load(...)` when loading a TF1 SavedModel to represent
+    its `GLOBAL_VARIABLES` collection.
+
+  * `trainable_variables`: A list of tf.Variable objects such that
+    `v.trainable` is true for all elements.
+    These variables must be a subset of `variables`.
+    These are the variables to be trained when fine-tuning the object.
+    The SavedModel creator may choose to omit some variables here that were
+    originally trainable to indicate that these should not be modified during
+    fine-tuning.
+
+    This list can be omitted if empty, in particular, if the SavedModel does not
+    support fine-tuning.
+
+  * `regularization_losses`: A list of tf.functions, each taking zero inputs
+    and returning a single scalar float tensor. For fine-tuning, the
+    SavedModel user is advised to include these as additional regularization
+    terms into the loss (in the simplest case without further scaling).
+    Typically, these are used to represent weight regularizers.
+    (For lack of inputs, these tf.functions cannot express
+    activity regularizers.)
+
+    This list can be omitted if empty, in particular, if the SavedModel does not
+    support fine-tuning or does not wish to prescribe weight regularization.
+
+### The `__call__` function
+
+A Restored SavedModel `obj` has an `obj.__call__` attribute that is
+a restored tf.function and allows `obj` to be called as follows.
+
+Synopsis (pseudo-code):
+
+```python
+outputs = obj(inputs, trainable=..., **kwargs)
+```
+
+#### Arguments
+
+The arguments are as follows.
+
+  * There is one positional, required argument with a batch of input activations
+    of the SavedModel. Its type is one of
+
+      * a single Tensor for a single input,
+      * a list of Tensors for an ordered sequence of unnamed inputs,
+      * a dict of Tensors keyed by a particular set of input names.
+
+    (Future revisions of this interface may allow more general nests.)
+    The SavedModel creator chooses one of those and the tensor shapes
+    and dtypes. Where useful, some dimensions of the shape should be
+    undefined (notably batch size).
+
+  * There may be an optional keyword argument `training` that accepts a Python
+    boolean, `True` or `False`. The default is `False`.
+    If the model supports fine-tuning, and if its computation differs between
+    the two (e.g., as in dropout and batch normalization), that distinction
+    is implemented with this argument. Otherwise, this argument may be absent.
+
+    It is not required that `__call__` accept a Tensor-valued `training`
+    argument. It falls on the caller to use `tf.cond()` if necessary
+    to dispatch between them.
+
+  * The SavedModel creator may choose to accept more optional `kwargs`
+    of particular names.
+
+      * For Tensor-valued arguments, the SavedModel creator defines their
+        permissible dtypes and shapes. `tf.function` accepts a Python default
+        value on an argument that is traced with a tf.TensorSpec input.
+        Such arguments can be used to allow customization of numeric
+        hyperparameters involved in `__call__` (e.g., dropout rate).
+
+      * For Python-valued arguments, the SavedModel creator defines their
+        permissible values. Such arguments can be used as flags to make
+        discrete choices in the traced function (but mind the combinatorial
+        explosion of traces).
+
+The restored `__call__` function must provide traces for all permissible
+combinations of arguments. Flipping `training` between `True` and `False`
+must not change the permissibility of arguments.
+
+#### Result
+
+The `outputs` from calling `obj` can be
+
+  * a single Tensor for a single output,
+  * a list of Tensors for an ordered sequence of unnamed outputs,
+  * a dict of Tensors keyed by a particular set of output names.
+
+(Future revisions of this interface may allow more general nests.)
+The return type may vary depending on the Python-valued kwargs.
+This allows for flags producing extra outputs.
+The SavedModel creator defines the output dtypes and shapes and their
+dependency on inputs.
+
+
+### Named callables
+
+A Reusable SavedModel can provide multiple model pieces in the way
+described above by putting them into named subobjects, for example,
+`obj.foo`, `obj.bar` and so on.
+Each subobject provides a `__call__` method and supporting attributes
+about the variables etc. specific to that model piece.
+For the example above, there would be `obj.foo.__call__`,
+`obj.foo.variables` and so on.
+
+Note that this interface does *not* cover the approach of adding
+a bare tf.function directly as `tf.foo`.
+
+Users of Reusable SavedModels are only expected to handle one level of nesting
+(`obj.bar` but not `obj.bar.baz`). (Future revisions of this interface may allow
+deeper nesting, and may waive the requirement that the top-level object be
+callable itself.)
+
+## Closing remarks
+
+### Relation to in-process APIs
+
+This document describes an interface of a Python class which consists
+of primitives like tf.function and tf.Variable that survive a
+round-trip through serialization via `tf.saved_model.save()`
+and `tf.saved_model.load()`. However, the interface was already present
+on the original object that was passed to `tf.saved_model.save()`.
+Adaptation to that interface enables the exchange of model pieces
+across model-building APIs within a single TensorFlow program.
diff --git a/site/en/hub/tf1_hub_module.md b/site/en/hub/tf1_hub_module.md
new file mode 100644
index 00000000000..7601878dc1b
--- /dev/null
+++ b/site/en/hub/tf1_hub_module.md
@@ -0,0 +1,198 @@
+
+# TF1 Hub format
+
+At its launch in 2018, TensorFlow Hub offered a single type of asset: TF1 Hub
+format for import into TensorFlow 1 programs.
+
+This page explains how to use TF1 Hub format in TF1 (or the TF1 compatibility
+mode of TF2) with the `hub.Module` class and associated APIs. (The typical use
+is to build a `tf.Graph`, possibly inside a TF1 `Estimator`, by combining one or
+more models in TF1 Hub format with `tf.compat.layers` or `tf.layers`).
+
+Users of TensorFlow 2 (outside TF1 compatibility mode) must use
+[the new API with `hub.load()` or `hub.KerasLayer`](tf2_saved_model.md). The new
+API loads the new TF2 SavedModel asset type, but also has limited
+[support for loading TF1 Hub format into TF2](migration_tf2.md).
+
+## Using a model in TF1 Hub format
+
+### Instantiating a model in TF1 Hub format
+
+A model in TF1 Hub format is imported into a TensorFlow program by creating a
+`hub.Module` object from a string with its URL or filesystem path, such as:
+
+```python
+m = hub.Module("path/to/a/module_dir")
+```
+**Note:** See more information regarding other valid handle types [here](tf2_saved_model.md#model_handles).
+
+This adds the module's variables to the current TensorFlow graph.
+Running their initializers will read their pre-trained values from disk.
+Likewise, tables and other state is added to the graph.
+
+### Caching Modules
+
+When creating a module from a URL, the module content is downloaded and cached
+in the local system temporary directory. The location where modules are cached
+can be overridden using `TFHUB_CACHE_DIR` environment variable. For details, see
+[Caching](caching.md).
+
+### Applying a Module
+
+Once instantiated, a module `m` can be called zero or more times like a Python
+function from tensor inputs to tensor outputs:
+
+```python
+y = m(x)
+```
+
+Each such call adds operations to the current TensorFlow graph to compute
+`y` from `x`. If this involves variables with trained weights, these are
+shared between all applications.
+
+Modules can define multiple named *signatures* in order to allow being applied
+in more than one way (similar to how Python objects have *methods*).
+A module's documentation should describe the available
+signatures. The call above applies the signature named `"default"`. Any
+signature can be selected by passing its name to the optional `signature=`
+argument.
+
+If a signature has multiple inputs, they must be passed as a dict, with the keys
+defined by the signature. Likewise, if a signature has multiple outputs, these
+can be retrieved as a dict by passing `as_dict=True`, under the keys defined by
+the signature (the key `"default"` is for the single output returned if
+`as_dict=False`). So the most general form of applying a Module looks like:
+
+```python
+outputs = m(dict(apples=x1, oranges=x2), signature="fruit_to_pet", as_dict=True)
+y1 = outputs["cats"]
+y2 = outputs["dogs"]
+```
+
+A caller must supply all inputs defined by a signature, but there is no
+requirement to use all of a module's outputs.
+TensorFlow will run only those parts of the module that end up
+as dependencies of a target in `tf.Session.run()`. Indeed, module publishers may
+choose to provide various outputs for advanced uses (like activations of
+intermediate layers) along with the main outputs. Module consumers should
+handle additional outputs gracefully.
+
+### Trying out alternative modules
+
+Whenever there are multiple modules for the same task, TensorFlow Hub
+encourages to equip them with compatible signatures (interfaces)
+such that trying different ones is as easy as varying the module handle
+as a string-valued hyperparameter.
+
+To this end, we maintain a collection of recommended
+[Common Signatures](common_signatures/index.md) for popular tasks.
+
+
+## Creating a New Module
+
+### Compatibility note
+
+The TF1 Hub format is geared towards TensorFlow 1. It is only partially
+supported by TF Hub in TensorFlow 2. Please do consider publishing in the new
+[TF2 SavedModel](tf2_saved_model.md) format instead.
+
+The TF1 Hub format is similar to the SavedModel format of TensorFlow 1 on a
+syntactic level (same file names and protocol messages) but semantically
+different to allow for module reuse, composition and re-training (e.g.,
+different storage of resource initializers, different tagging conventions for
+metagraphs). The easiest way to tell them apart on disk is the presence or
+absence of the `tfhub_module.pb` file.
+
+### General approach
+
+To define a new module, a publisher calls `hub.create_module_spec()` with a
+function `module_fn`. This function constructs a graph representing the module's
+internal structure, using `tf.placeholder()` for inputs to be supplied by
+the caller. Then it defines signatures by calling
+`hub.add_signature(name, inputs, outputs)` one or more times.
+
+For example:
+
+```python
+def module_fn():
+  inputs = tf.placeholder(dtype=tf.float32, shape=[None, 50])
+  layer1 = tf.layers.dense(inputs, 200)
+  layer2 = tf.layers.dense(layer1, 100)
+  outputs = dict(default=layer2, hidden_activations=layer1)
+  # Add default signature.
+  hub.add_signature(inputs=inputs, outputs=outputs)
+
+...
+spec = hub.create_module_spec(module_fn)
+```
+
+The result of `hub.create_module_spec()` can be used, instead of a path,
+to instantiate a module object within a particular TensorFlow graph. In
+such case, there is no checkpoint, and the module instance will use the
+variable initializers instead.
+
+Any module instance can be serialized to disk via its `export(path, session)`
+method. Exporting a module serializes its definition together with the current
+state of its variables in `session` into the passed path. This can be used
+when exporting a module for the first time, as well as when exporting a fine
+tuned module.
+
+For compatibility with TensorFlow Estimators, `hub.LatestModuleExporter` exports
+modules from the latest checkpoint, just like `tf.estimator.LatestExporter`
+exports the entire model from the latest checkpoint.
+
+Module publishers should implement a [common
+signature](common_signatures/index.md) when possible, so that consumers can
+easily exchange modules and find the best one for their problem.
+
+### Real example
+
+Take a look at our [text embedding module exporter](https://github.com/tensorflow/hub/blob/master/examples/text_embeddings/export.py)
+for a real-world example of how to create a module from a common text embedding
+format.
+
+
+## Fine-Tuning
+
+Training the variables of an imported module together with those of the model
+around it is called *fine-tuning*. Fine-tuning can result in better quality, but
+adds new complications. We advise consumers to look into fine-tuning only after
+exploring simpler quality tweaks, and only if the module publisher recommends
+it.
+
+### For Consumers
+
+To enable fine-tuning, instantiate the module with
+`hub.Module(..., trainable=True)` to make its variables trainable and
+import TensorFlow's `REGULARIZATION_LOSSES`. If the module has multiple
+graph variants, make sure to pick the one appropriate for training.
+Usually, that's the one with tags `{"train"}`.
+
+Choose a training regime that does not ruin the pre-trained weights,
+for example, a lower learning rate than for training from scratch.
+
+### For Publishers
+
+To make fine-tuning easier for consumers, please be mindful of the following:
+
+*   Fine-tuning needs regularization. Your module is exported with the
+    `REGULARIZATION_LOSSES` collection, which is what puts your choice of
+    `tf.layers.dense(..., kernel_regularizer=...)` etc. into what the consumer
+    gets from `tf.losses.get_regularization_losses()`. Prefer this way of
+    defining L1/L2 regularization losses.
+
+*   In the publisher model, avoid defining L1/L2 regularization via the `l1_`
+    and `l2_regularization_strength` parameters of `tf.train.FtrlOptimizer`,
+    `tf.train.ProximalGradientDescentOptimizer`, and other proximal optimizers.
+    These are not exported alongside the module, and setting regularization
+    strengths globally may not be appropriate for the consumer. Except for L1
+    regularization in wide (i.e. sparse linear) or wide & deep models, it should
+    be possible to use individual regularization losses instead.
+
+*   If you use dropout, batch normalization, or similar training techniques, set
+    their hyperparameters to values that make sense across many expected uses.
+    The dropout rate may have to be adjusted to the target problem's propensity
+    to overfitting. In batch normalization, the momentum (a.k.a. decay
+    coefficient) should be small enough to enable fine-tuning with small
+    datasets and/or large batches. For advanced consumers, consider adding a
+    signature that exposes control over critical hyperparameters.
diff --git a/site/en/hub/tf2_saved_model.md b/site/en/hub/tf2_saved_model.md
new file mode 100644
index 00000000000..e41337b2548
--- /dev/null
+++ b/site/en/hub/tf2_saved_model.md
@@ -0,0 +1,289 @@
+
+# SavedModels from TF Hub in TensorFlow 2
+
+The
+[SavedModel format of TensorFlow 2](https://www.tensorflow.org/guide/saved_model)
+is the recommended way to share pre-trained models and model pieces on
+TensorFlow Hub. It replaces the older [TF1 Hub format](tf1_hub_module.md) and
+comes with a new set of APIs.
+
+This page explains how to reuse TF2 SavedModels in a TensorFlow 2 program with
+the low-level `hub.load()` API and its `hub.KerasLayer` wrapper. (Typically,
+`hub.KerasLayer` is combined with other `tf.keras.layers` to build a Keras model
+or the `model_fn` of a TF2 Estimator.) These APIs can also load the legacy
+models in TF1 Hub format, within limits, see the
+[compatibility guide](model_compatibility.md).
+
+Users of TensorFlow 1 can update to TF 1.15 and then use the same APIs.
+Older versions of TF1 do not work.
+
+## Using SavedModels from TF Hub
+
+### Using a SavedModel in Keras
+
+[Keras](https://www.tensorflow.org/guide/keras/) is TensorFlow's high-level API
+for building deep learning models by composing Keras Layer objects.
+The `tensorflow_hub` library provides the class `hub.KerasLayer` that gets
+initialized with the URL (or filesystem path) of a SavedModel and then
+provides the computation from the SavedModel, including its pre-trained
+weights.
+
+Here is an example of using a pre-trained text embedding:
+
+```python
+import tensorflow as tf
+import tensorflow_hub as hub
+
+hub_url = "https://tfhub.dev/google/nnlm-en-dim128/2"
+embed = hub.KerasLayer(hub_url)
+embeddings = embed(["A long sentence.", "single-word", "http://example.com"])
+print(embeddings.shape, embeddings.dtype)
+```
+
+From this, a text classifier can be built in the usual Keras way:
+
+```python
+model = tf.keras.Sequential([
+    embed,
+    tf.keras.layers.Dense(16, activation="relu"),
+    tf.keras.layers.Dense(1, activation="sigmoid"),
+])
+```
+
+The [Text classification
+colab](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_text_classification.ipynb)
+is a complete example how to train and evaluate such a classifier.
+
+The model weights in a `hub.KerasLayer` are set to non-trainable by default.
+See the section on fine-tuning below for how to change that. Weights are
+shared between all applications of the same layer object, as usual in Keras.
+
+
+### Using a SavedModel in an Estimator
+
+Users of TensorFlow's
+[Estimator](https://www.tensorflow.org/tutorials/distribute/multi_worker_with_estimator)
+API for distributed training can use SavedModels from TF Hub by
+writing their `model_fn` in terms of  `hub.KerasLayer` among other
+`tf.keras.layers`.
+
+
+### Behind the scenes: SavedModel downloading and caching
+
+Using a SavedModel from TensorFlow Hub (or other HTTPS servers that implement
+its [hosting](hosting.md) protocol) downloads and decompresses it to the local
+filesystem if not already present. The environment variable `TFHUB_CACHE_DIR`
+can be set to override the default temporary location for caching the downloaded
+and uncompressed SavedModels. For details, see [Caching](caching.md).
+
+### Using a SavedModel in low-level TensorFlow
+#### Model Handles
+
+SavedModels can be loaded from a specified `handle`, where the `handle` is a
+filesystem path, valid TFhub.dev model URL (e.g. "https://tfhub.dev/...").
+Kaggle Models URLs mirror TFhub.dev handles in accordance with our Terms and the
+license associated with the model assets, e.g., "https://www.kaggle.com/...".
+Handles from Kaggle Models are equivalent to their corresponding TFhub.dev
+handle.
+
+The function `hub.load(handle)` downloads and decompresses a SavedModel
+(unless `handle` is already a filesystem path) and then returns the result
+of loading it with TensorFlow's built-in function `tf.saved_model.load()`.
+Therefore, `hub.load()` can handle any valid SavedModel (unlike its
+predecessor `hub.Module` for TF1).
+
+#### Advanced topic: what to expect from the SavedModel after loading
+
+Depending on the contents of the SavedModel, the result of
+`obj = hub.load(...)` can be invoked in various ways (as explained in
+much greater detail in TensorFlow's [SavedModel
+Guide](https://www.tensorflow.org/guide/saved_model):
+
+  * The serving signatures of the SavedModel (if any) are represented as a
+    dictionary of concrete functions and can be called like
+    `tensors_out = obj.signatures["serving_default"](**tensors_in)`,
+    with dictionaries of tensors keyed by the respective input and output
+    names and subject to the signature's shape and dtype constraints.
+
+  * The
+    [`@tf.function`](https://www.tensorflow.org/api_docs/python/tf/function)-decorated
+    methods of the saved object (if any) are restored as tf.function objects
+    that can be called by all combinations of Tensor and non-Tensor arguments
+    for which the tf.function had been
+    [traced](https://www.tensorflow.org/tutorials/customization/performance#tracing)
+    prior to saving. In particular, if there is an `obj.__call__` method
+    with suitable traces, `obj` itself can be called like a Python function.
+    A simple example could look like
+    `output_tensor = obj(input_tensor, training=False)`.
+
+This leaves enormous liberty in the interfaces that SavedModels can
+implement. The [Reusable SavedModels interface](reusable_saved_models.md)
+for `obj` establishes conventions such that client code, including adapters
+like `hub.KerasLayer`, know how to use the SavedModel.
+
+Some SavedModels may not follow that convention, especially whole models
+not meant to be reused in larger models, and just provide serving signatures.
+
+The trainable variables in a SavedModel are reloaded as trainable,
+and `tf.GradientTape` will watch them by default. See the section on
+fine-tuning below for some caveats, and consider avoiding this for starters.
+Even if you want to fine-tune, you may want to see if `obj.trainable_variables`
+advises to re-train only a subset of the originally trainable variables.
+
+
+## Creating SavedModels for TF Hub
+
+### Overview
+
+SavedModel is TensorFlow's standard serialization format for trained models
+or model pieces.
+It stores the model's trained weights together with the exact TensorFlow
+operations to perform its computation. It can be used independently from
+the code that created it. In particular, it can be reused across different
+high-level model-building APIs like Keras, because TensorFlow operations
+are their common basic language.
+
+### Saving from Keras
+
+Starting with TensorFlow 2, `tf.keras.Model.save()` and
+`tf.keras.models.save_model()` default to the SavedModel format (not HDF5).
+The resulting SavedModels that can be used with `hub.load()`,
+`hub.KerasLayer` and similar adapters for other high-level APIs
+as they become available.
+
+To share a complete Keras Model, just save it with `include_optimizer=False`.
+
+To share a piece of a Keras Model, make the piece a Model in itself and then
+save that. You can either lay out the code like that from the start....
+
+```python
+piece_to_share = tf.keras.Model(...)
+full_model = tf.keras.Sequential([piece_to_share, ...])
+full_model.fit(...)
+piece_to_share.save(...)
+```
+
+...or cut out the piece to share after the fact (if it aligns with the
+layering of your full model):
+
+```python
+full_model = tf.keras.Model(...)
+sharing_input = full_model.get_layer(...).get_output_at(0)
+sharing_output = full_model.get_layer(...).get_output_at(0)
+piece_to_share = tf.keras.Model(sharing_input, sharing_output)
+piece_to_share.save(..., include_optimizer=False)
+```
+
+[TensorFlow Models](https://github.com/tensorflow/models) on GitHub uses the
+former approach for BERT (see
+[nlp/tools/export_tfhub_lib.py](https://github.com/tensorflow/models/blob/master/official/nlp/tools/export_tfhub_lib.py),
+note the split between `core_model` for export and the `pretrainer` for
+restoring the checkpoint) and the latter approach for ResNet (see
+[legacy/image_classification/tfhub_export.py](https://github.com/tensorflow/models/blob/master/official/legacy/image_classification/resnet/tfhub_export.py)).
+
+### Saving from low-level TensorFlow
+
+This requires good familiarity with TensorFlow's [SavedModel
+Guide](https://www.tensorflow.org/guide/saved_model).
+
+If you want to provide more than just a serving signature, you should
+implement the [Reusable SavedModel interface](reusable_saved_models.md).
+Conceptually, this looks like
+
+```python
+class MyMulModel(tf.train.Checkpoint):
+  def __init__(self, v_init):
+    super().__init__()
+    self.v = tf.Variable(v_init)
+    self.variables = [self.v]
+    self.trainable_variables = [self.v]
+    self.regularization_losses = [
+        tf.function(input_signature=[])(lambda: 0.001 * self.v**2),
+    ]
+
+  @tf.function(input_signature=[tf.TensorSpec(shape=None, dtype=tf.float32)])
+  def __call__(self, inputs):
+    return tf.multiply(inputs, self.v)
+
+tf.saved_model.save(MyMulModel(2.0), "/tmp/my_mul")
+
+layer = hub.KerasLayer("/tmp/my_mul")
+print(layer([10., 20.]))  # [20., 40.]
+layer.trainable = True
+print(layer.trainable_weights)  # [2.]
+print(layer.losses)  # 0.004
+```
+
+
+## Fine-Tuning
+
+Training the already-trained variables of an imported SavedModel together with
+those of the model around it is called *fine-tuning* the SavedModel.
+This can result in better quality, but often makes the training more
+demanding (may take more time, depend more on the optimizer and its
+hyperparameters, increase the risk of overfitting and require dataset
+augmentation, esp. for CNNs). We advise SavedModel consumers to look into
+fine-tuning only after having established a good training regime,
+and only if the SavedModel publisher recommends it.
+
+Fine-tuning changes the "continuous" model parameters that are trained.
+It does not change hard-coded transformations, such as tokenizing text
+input and mapping tokens to their corresponding entries in an embedding matrix.
+
+### For SavedModel consumers
+
+Creating a `hub.KerasLayer` like
+
+```python
+layer = hub.KerasLayer(..., trainable=True)
+```
+
+enables fine-tuning of the SavedModel loaded by the layer. It adds the
+trainable weights and weight regularizers declared in the SavedModel
+to the Keras model, and runs the SavedModel's computation in training
+mode (think of dropout etc.).
+
+The [image classification
+colab](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_image_retraining.ipynb)
+contains an end-to-end example with optional fine-tuning.
+
+#### Re-exporting the fine-tuning result
+
+Advanced users may want to save the results of fine-tuning back into
+a SavedModel that can be used instead of the originally loaded one.
+This can be done with code like
+
+```python
+loaded_obj = hub.load("https://tfhub.dev/...")
+hub_layer = hub.KerasLayer(loaded_obj, trainable=True, ...)
+
+model = keras.Sequential([..., hub_layer, ...])
+model.compile(...)
+model.fit(...)
+
+export_module_dir = os.path.join(os.getcwd(), "finetuned_model_export")
+tf.saved_model.save(loaded_obj, export_module_dir)
+```
+
+### For SavedModel creators
+
+When creating a SavedModel for sharing on TensorFlow Hub,
+think ahead if and how its consumers should fine-tune it,
+and provide guidance in the documentation.
+
+Saving from a Keras Model should make all the mechanics of fine-tuning work
+(saving weight regularization losses, declaring trainable variables, tracing
+`__call__` for both `training=True` and `training=False`, etc.)
+
+Choose a model interface that plays well with gradient flow,
+e.g., output logits instead of softmax probabilities or top-k predictions.
+
+If the model use dropout, batch normalization, or similar training techniques
+that involve hyperparameters, set them to values that make sense across many
+expected target problems and batch sizes. (As of this writing, saving from
+Keras does not make it easy to let consumers adjust them.)
+
+Weight regularizers on individual layers are saved (with their regularization
+strength coefficients), but weight regularization from within the optimizer
+(like `tf.keras.optimizers.Ftrl.l1_regularization_strength=...)`)
+is lost. Advise consumers of your SavedModel accordingly.
diff --git a/site/en/hub/tutorials/_index.yaml b/site/en/hub/tutorials/_index.yaml
new file mode 100644
index 00000000000..deb98108393
--- /dev/null
+++ b/site/en/hub/tutorials/_index.yaml
@@ -0,0 +1,174 @@
+book_path: /hub/_book.yaml
+project_path: /hub/_project.yaml
+title: Tutorials
+landing_page:
+  custom_css_path: /site-assets/css/style.css
+  nav: left
+  meta_tags:
+  - name: description
+    content: >
+      TensorFlow Hub tutorials to help you get started with using and adapting pre-trained
+      machine learning models to your needs.
+  rows:
+  # Getting started
+  - classname: devsite-landing-row-100
+    items:
+    - description: >
+        
Getting started

+        
TensorFlow Hub is a comprehensive repository of pre-trained
+        models ready for fine-tuning and deployable anywhere. Download the latest trained models
+        with a minimal amount of code with the tensorflow_hub library.

+        
The following tutorials should help you getting
+        started with using and applying models from TF Hub for your needs. Interactive tutorials let you
+        modify them and execute them with your changes. Click the Run in Google Colab
+        button at the top of an interactive tutorial to tinker with it.

+
+  # For beginners
+  - classname: devsite-landing-row-100
+    items:
+    - description: >
+        
For beginners

+        
If you are unfamiliar with machine learning and TensorFlow, you can start by getting
+        an overview of how to classify images and text, detecting objects in images, or by stylizing your own pictures like famous artwork:

+
+  - classname: devsite-landing-row-100
+    items:
+    - classname: tfo-landing-page-card
+      description: >
+        
Image Classification

+        Build a Keras model on top of a pre-trained image classifier to distinguish flowers.
+      path: /hub/tutorials/tf2_image_retraining
+      image_path: /hub/images/image_classification.png
+    - classname: tfo-landing-page-card
+      description: >
+        
Classify Text with BERT

+        Use BERT to build a Keras model to solve a text classificaton sentiment analysis task.
+      path: /tutorials/text/classify_text_with_bert
+      image_path: /hub/images/bert_preprocess.png
+    - classname: tfo-landing-page-card
+      description: >
+        
+          
Style Transfer

+        
+        Let a neural network redraw an image in the style of Picasso, van Gogh or like your own style image.
+      path: /hub/tutorials/tf2_arbitrary_image_stylization
+      image_path: /hub/images/style_transfer.png
+    - classname: tfo-landing-page-card
+      description: >
+        
Object Detection

+        Detect objects in images using models like FasterRCNN or SSD.
+      path: /hub/tutorials/tf2_object_detection
+      image_path: /hub/images/object_detection.png
+
+  # More advanced users
+  - classname: devsite-landing-row-100
+    items:
+    - description: >
+        
For experienced developers

+        
Check out more advanced tutorials for how to use NLP, images, audio, and video models from TensorFlow Hub.

+
+  # NLP tutorials
+  - classname: devsite-landing-row-100
+    items:
+    - description: >
+        
NLP tutorials

+        
Solve common NLP tasks with models from TensorFlow Hub. View all available NLP tutorials in the left nav.

+
+  - classname: devsite-landing-row-100
+    items:
+    - classname: tfo-landing-page-card
+      description: >
+        
Semantic Similarity

+        Classify and semantically compare sentences with the Universal Sentence Encoder.
+      path: /hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder
+      image_path: /hub/images/similarity.png
+    - classname: tfo-landing-page-card
+      description: >
+        
BERT on TPU

+        Use BERT to solve GLUE benchmark tasks running on TPU.
+      path: /tutorials/text/solve_glue_tasks_using_bert_on_tpu
+      image_path: /hub/images/bert.png
+    - classname: tfo-landing-page-card
+      description: >
+        
Multilingual Universal Sentence Encoder Q&A

+        Answer cross-lingual questions from the SQuAD dataset using the multilingual universal sentence encoder Q&A model.
+      path: /hub/tutorials/retrieval_with_tf_hub_universal_encoder_qa
+      image_path: /hub/images/colab_logo.svg
+
+  # Image tutorials
+  - classname: devsite-landing-row-100
+    items:
+    - description: >
+        
Image tutorials

+        
Explore how to use GANs, super resolution models and more. View all available image tutorials in the left nav.

+
+  - classname: devsite-landing-row-100
+    items:
+    - classname: tfo-landing-page-card
+      description: >
+        
GANS for Image Generation

+        Generate artificial faces and interpolate between them using GANs.
+      path: /hub/tutorials/tf_hub_generative_image_module
+      image_path: /hub/images/gan_faces.gif
+    - classname: tfo-landing-page-card
+      description: >
+        
Super Resolution

+        Enhance the resolution of downsampled images.
+      path: /hub/tutorials/image_enhancing
+      image_path: /hub/images/super_resolution.png
+    - classname: tfo-landing-page-card
+      description: >
+        
Image Extension

+        Fill the masked part of given images.
+      path: /hub/tutorials/boundless
+      image_path: /hub/images/boundless.png
+
+  # Audio tutorials
+  - classname: devsite-landing-row-100
+    items:
+    - description: >
+        
Audio tutorials

+        
Explore tutorials using trained models for audio data including pitch recognition and sound classification.

+
+  - classname: devsite-landing-row-100
+    items:
+    - classname: tfo-landing-page-card
+      description: >
+        
Pitch recognition

+        Record yourself singing and detect the pitch of your voice using the SPICE model.
+      path: /hub/tutorials/spice
+      image_path: /hub/images/spice_color.png
+    - classname: tfo-landing-page-card
+      description: >
+        
Sound classification

+        Use the YAMNet model to classify sounds as 521 audio event classes from the AudioSet-YouTube corpus.
+      path: /hub/tutorials/yamnet
+      image_path: /hub/images/yamnet.png
+
+  # Video tutorials
+  - classname: devsite-landing-row-100
+    items:
+    - description: >
+        
Video tutorials

+        
Try out trained ML models for video data for action recognition, video interpolation, and more.

+
+  - classname: devsite-landing-row-100
+    items:
+    - classname: tfo-landing-page-card
+      description: >
+        
Action Recognition

+        Detect one of 400 actions in a video using the Inflated 3D ConvNet model.
+      path: /hub/tutorials/action_recognition_with_tf_hub
+      image_path: /hub/images/action_recognition.gif
+    - classname: tfo-landing-page-card
+      description: >
+        
Video Interpolation

+        Interpolate between video frames using Inbetweening with 3D Convolutions.
+      path: /hub/tutorials/tweening_conv3d
+      image_path: /hub/images/interpolation.png
+    - classname: tfo-landing-page-card
+      description: >
+        
Text-to-Video Retrieval

+        Find videos that are the most related to a text query.
+      path: /hub/tutorials/text_to_video_retrieval_with_s3d_milnce
+      image_path: /hub/images/text_video.gif
diff --git a/site/en/hub/tutorials/_toc.yaml b/site/en/hub/tutorials/_toc.yaml
new file mode 100644
index 00000000000..04d95a267d7
--- /dev/null
+++ b/site/en/hub/tutorials/_toc.yaml
@@ -0,0 +1,118 @@
+toc:
+- heading: "Getting started"
+  style: divider
+- title: Overview
+  path: /hub/tutorials/_index.yaml
+
+- heading: "NLP Tutorials"
+  style: divider
+- title: Text classification
+  path: /hub/tutorials/tf2_text_classification
+- title: Classify text with BERT
+  path: /tutorials/text/classify_text_with_bert
+  status: external
+- title: BERT on TPU
+  path: /tutorials/text/solve_glue_tasks_using_bert_on_tpu
+  status: external
+- title: Real-time semantic search
+  path: /hub/tutorials/tf2_semantic_approximate_nearest_neighbors
+- title: Multilingual question answering
+  path: /hub/tutorials/retrieval_with_tf_hub_universal_encoder_qa
+- title: "Additional NLP tutorials"
+  style: accordion
+  section:
+  - title: BERT Experts
+    path: /hub/tutorials/bert_experts
+  - title: Semantic similarity
+    path: /hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder
+  - title: Text classification on Kaggle
+    path: /hub/tutorials/text_classification_with_tf_hub_on_kaggle
+  - title: Bangla article classifier
+    path: /hub/tutorials/bangla_article_classifier
+  - title: Explore CORD-19 text embeddings
+    path: /hub/tutorials/cord_19_embeddings_keras
+  - title: Multilingual universal sentence encoder
+    path: /hub/tutorials/cross_lingual_similarity_with_tf_hub_multilingual_universal_encoder
+  - title: Text cookbook
+    path: /hub/tutorials/text_cookbook
+  - title: SentEval for Universal Sentence Encoder CMLM model.
+    path: /hub/tutorials/senteval_for_universal_sentence_encoder_cmlm
+
+- heading: "Image Tutorials"
+  style: divider
+- title: Image classification
+  path: /hub/tutorials/image_classification
+- title: Transfer Learning for Image classification
+  path: /hub/tutorials/tf2_image_retraining
+- title: Style transfer
+  path: /hub/tutorials/tf2_arbitrary_image_stylization
+- title: Large-scale image retrieval with DELF
+  path: /hub/tutorials/tf_hub_delf_module
+- title: Object detection
+  path: /hub/tutorials/tf2_object_detection
+- title: GANs for image generation
+  path: /hub/tutorials/tf_hub_generative_image_module
+- title: Human Pose Estimation
+  path: /hub/tutorials/movenet
+- title: "Additional image tutorials"
+  style: accordion
+  section:
+  - title: "CropNet: Cassava Disease Detection"
+    path: /hub/tutorials/cropnet_cassava
+  - title: "CropNet: Fine tuning models for on-device inference"
+    path: /hub/tutorials/cropnet_on_device
+  - title: Boundless GAN
+    path: /hub/tutorials/boundless
+  - title: Super resolution
+    path: /hub/tutorials/image_enhancing
+  - title: HRNet model inference for semantic segmentation
+    path: /hub/tutorials/hrnet_semantic_segmentation
+    status: new
+
+- heading: "Audio Tutorials"
+  style: divider
+- title: Pitch recognition
+  path: /hub/tutorials/spice
+- title: Sound classification
+  path: /hub/tutorials/yamnet
+- title: Automatic speech recognition with Wav2Vec2
+  path: /hub/tutorials/wav2vec2_saved_model_finetuning
+
+- heading: "Video Tutorials"
+  style: divider
+- title: Frame interpolation with FILM
+  path: /hub/tutorials/tf_hub_film_example
+  status: new
+- title: Action recognition
+  path: /hub/tutorials/action_recognition_with_tf_hub
+- title: Streaming action recognition
+  path: /hub/tutorials/movinet
+- title: Video interpolation
+  path: /hub/tutorials/tweening_conv3d
+- title: Text-to-video retrieval
+  path: /hub/tutorials/text_to_video_retrieval_with_s3d_milnce
+
+- title: "Tutorials (TF1)"
+  style: accordion
+  status: deprecated
+  section:
+  - heading: "Image Tutorials"
+  - title: Image classification
+    path: /hub/tutorials/image_feature_vector
+  - title: Object detection
+    path: /hub/tutorials/object_detection
+  - title: BigGAN image generation
+    path: /hub/tutorials/biggan_generation_with_tf_hub
+  - title: BigBiGAN image generation
+    path: /hub/tutorials/bigbigan_with_tf_hub
+  - title: S3 GAN image generation
+    path: /hub/tutorials/s3gan_generation_with_tf_hub
+  - heading: "NLP Tutorials"
+  - title: Semantic similarity lite
+    path: /hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder_lite
+  - title: Nearest neighbor index for real-time semantic search
+    path: /hub/tutorials/semantic_approximate_nearest_neighbors
+  - title: Explore CORD-19 text embeddings
+    path: /hub/tutorials/cord_19_embeddings
+  - title: Wiki40B Language Models
+    path: /hub/tutorials/wiki40b_lm
diff --git a/site/en/hub/tutorials/action_recognition_with_tf_hub.ipynb b/site/en/hub/tutorials/action_recognition_with_tf_hub.ipynb
new file mode 100644
index 00000000000..3f586991ba9
--- /dev/null
+++ b/site/en/hub/tutorials/action_recognition_with_tf_hub.ipynb
@@ -0,0 +1,438 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x8Q7Un821X1A"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1W4rIAFt1Ui3"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cDq0CIKc1vO_"
+      },
+      "source": [
+        "# Action Recognition with an Inflated 3D CNN\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h6W3FhoP3TxC"
+      },
+      "source": [
+        "This Colab demonstrates recognizing actions in video data using the\n",
+        "[tfhub.dev/deepmind/i3d-kinetics-400/1](https://tfhub.dev/deepmind/i3d-kinetics-400/1) module. More models to detect actions in videos can be found [here](https://tfhub.dev/s?module-type=video-classification).\n",
+        "\n",
+        "The underlying model is described in the paper \"[Quo Vadis, Action Recognition? A New\n",
+        "Model and the Kinetics Dataset](https://arxiv.org/abs/1705.07750)\" by Joao\n",
+        "Carreira and Andrew Zisserman. The paper was posted on arXiv in May 2017, and\n",
+        "was published as a CVPR 2017 conference paper.\n",
+        "The source code is publicly available on\n",
+        "[github](https://github.com/deepmind/kinetics-i3d).\n",
+        "\n",
+        "\"Quo Vadis\" introduced a new architecture for video classification, the Inflated\n",
+        "3D Convnet or I3D. This architecture achieved state-of-the-art results on the UCF101\n",
+        "and HMDB51 datasets from fine-tuning these models. I3D models pre-trained on Kinetics\n",
+        "also placed first in the CVPR 2017 [Charades challenge](http://vuchallenge.org/charades.html).\n",
+        "\n",
+        "The original module was trained on the [kinetics-400 dateset](https://www.deepmind.com/open-source/kinetics)\n",
+        "and knows about 400 different actions.\n",
+        "Labels for these actions can be found in the\n",
+        "[label map file](https://github.com/deepmind/kinetics-i3d/blob/master/data/label_map.txt).\n",
+        "\n",
+        "In this Colab we will use it recognize activities in videos from a UCF101 dataset."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "R_0xc2jyNGRp"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mOHMWsFnITdi"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q imageio\n",
+        "!pip install -q opencv-python\n",
+        "!pip install -q git+https://github.com/tensorflow/docs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "USf0UvkYIlKo"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Import the necessary modules\n",
+        "# TensorFlow and TF-Hub modules.\n",
+        "from absl import logging\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "from tensorflow_docs.vis import embed\n",
+        "\n",
+        "logging.set_verbosity(logging.ERROR)\n",
+        "\n",
+        "# Some modules to help with reading the UCF101 dataset.\n",
+        "import random\n",
+        "import re\n",
+        "import os\n",
+        "import tempfile\n",
+        "import ssl\n",
+        "import cv2\n",
+        "import numpy as np\n",
+        "\n",
+        "# Some modules to display an animation using imageio.\n",
+        "import imageio\n",
+        "from IPython import display\n",
+        "\n",
+        "from urllib import request  # requires python3"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "IuMMS3TGdws7"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Helper functions for the UCF101 dataset\n",
+        "\n",
+        "# Utilities to fetch videos from UCF101 dataset\n",
+        "UCF_ROOT = \"https://www.crcv.ucf.edu/THUMOS14/UCF101/UCF101/\"\n",
+        "_VIDEO_LIST = None\n",
+        "_CACHE_DIR = tempfile.mkdtemp()\n",
+        "# As of July 2020, crcv.ucf.edu doesn't use a certificate accepted by the\n",
+        "# default Colab environment anymore.\n",
+        "unverified_context = ssl._create_unverified_context()\n",
+        "\n",
+        "def list_ucf_videos():\n",
+        "  \"\"\"Lists videos available in UCF101 dataset.\"\"\"\n",
+        "  global _VIDEO_LIST\n",
+        "  if not _VIDEO_LIST:\n",
+        "    index = request.urlopen(UCF_ROOT, context=unverified_context).read().decode(\"utf-8\")\n",
+        "    videos = re.findall(\"(v_[\\w_]+\\.avi)\", index)\n",
+        "    _VIDEO_LIST = sorted(set(videos))\n",
+        "  return list(_VIDEO_LIST)\n",
+        "\n",
+        "def fetch_ucf_video(video):\n",
+        "  \"\"\"Fetches a video and cache into local filesystem.\"\"\"\n",
+        "  cache_path = os.path.join(_CACHE_DIR, video)\n",
+        "  if not os.path.exists(cache_path):\n",
+        "    urlpath = request.urljoin(UCF_ROOT, video)\n",
+        "    print(\"Fetching %s => %s\" % (urlpath, cache_path))\n",
+        "    data = request.urlopen(urlpath, context=unverified_context).read()\n",
+        "    open(cache_path, \"wb\").write(data)\n",
+        "  return cache_path\n",
+        "\n",
+        "# Utilities to open video files using CV2\n",
+        "def crop_center_square(frame):\n",
+        "  y, x = frame.shape[0:2]\n",
+        "  min_dim = min(y, x)\n",
+        "  start_x = (x // 2) - (min_dim // 2)\n",
+        "  start_y = (y // 2) - (min_dim // 2)\n",
+        "  return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]\n",
+        "\n",
+        "def load_video(path, max_frames=0, resize=(224, 224)):\n",
+        "  cap = cv2.VideoCapture(path)\n",
+        "  frames = []\n",
+        "  try:\n",
+        "    while True:\n",
+        "      ret, frame = cap.read()\n",
+        "      if not ret:\n",
+        "        break\n",
+        "      frame = crop_center_square(frame)\n",
+        "      frame = cv2.resize(frame, resize)\n",
+        "      frame = frame[:, :, [2, 1, 0]]\n",
+        "      frames.append(frame)\n",
+        "      \n",
+        "      if len(frames) == max_frames:\n",
+        "        break\n",
+        "  finally:\n",
+        "    cap.release()\n",
+        "  return np.array(frames) / 255.0\n",
+        "\n",
+        "def to_gif(images):\n",
+        "  converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)\n",
+        "  imageio.mimsave('./animation.gif', converted_images, duration=40)\n",
+        "  return embed.embed_file('./animation.gif')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "pIKTs-KneUfz"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Get the kinetics-400 labels\n",
+        "# Get the kinetics-400 action labels from the GitHub repository.\n",
+        "KINETICS_URL = \"https://raw.githubusercontent.com/deepmind/kinetics-i3d/master/data/label_map.txt\"\n",
+        "with request.urlopen(KINETICS_URL) as obj:\n",
+        "  labels = [line.decode(\"utf-8\").strip() for line in obj.readlines()]\n",
+        "print(\"Found %d labels.\" % len(labels))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GBvmjVICIp3W"
+      },
+      "source": [
+        "# Using the UCF101 dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "V-QcxdhLIfi2"
+      },
+      "outputs": [],
+      "source": [
+        "# Get the list of videos in the dataset.\n",
+        "ucf_videos = list_ucf_videos()\n",
+        "  \n",
+        "categories = {}\n",
+        "for video in ucf_videos:\n",
+        "  category = video[2:-12]\n",
+        "  if category not in categories:\n",
+        "    categories[category] = []\n",
+        "  categories[category].append(video)\n",
+        "print(\"Found %d videos in %d categories.\" % (len(ucf_videos), len(categories)))\n",
+        "\n",
+        "for category, sequences in categories.items():\n",
+        "  summary = \", \".join(sequences[:2])\n",
+        "  print(\"%-20s %4d videos (%s, ...)\" % (category, len(sequences), summary))\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "c0ZvVDruN2nU"
+      },
+      "outputs": [],
+      "source": [
+        "# Get a sample cricket video.\n",
+        "video_path = fetch_ucf_video(\"v_CricketShot_g04_c02.avi\")\n",
+        "sample_video = load_video(video_path)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hASLA90YFPTO"
+      },
+      "outputs": [],
+      "source": [
+        "sample_video.shape"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "POf5XgffvXlD"
+      },
+      "outputs": [],
+      "source": [
+        "i3d = hub.load(\"https://tfhub.dev/deepmind/i3d-kinetics-400/1\").signatures['default']"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mDXgaOD1zhMP"
+      },
+      "source": [
+        "Run the id3 model and print the top-5 action predictions."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3mTbqA5JGYUx"
+      },
+      "outputs": [],
+      "source": [
+        "def predict(sample_video):\n",
+        "  # Add a batch axis to the sample video.\n",
+        "  model_input = tf.constant(sample_video, dtype=tf.float32)[tf.newaxis, ...]\n",
+        "\n",
+        "  logits = i3d(model_input)['default'][0]\n",
+        "  probabilities = tf.nn.softmax(logits)\n",
+        "\n",
+        "  print(\"Top 5 actions:\")\n",
+        "  for i in np.argsort(probabilities)[::-1][:5]:\n",
+        "    print(f\"  {labels[i]:22}: {probabilities[i] * 100:5.2f}%\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ykaXQcGRvK4E"
+      },
+      "outputs": [],
+      "source": [
+        "predict(sample_video)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PHsq0lHXCsD4"
+      },
+      "source": [
+        "Now try a new video, from: https://commons.wikimedia.org/wiki/Category:Videos_of_sports\n",
+        "\n",
+        "How about [this video](https://commons.wikimedia.org/wiki/File:End_of_a_jam.ogv) by Patrick Gillett: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p-mZ9fFPCoNq"
+      },
+      "outputs": [],
+      "source": [
+        "!curl -O https://upload.wikimedia.org/wikipedia/commons/8/86/End_of_a_jam.ogv"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lpLmE8rjEbAF"
+      },
+      "outputs": [],
+      "source": [
+        "video_path = \"End_of_a_jam.ogv\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CHZJ9qTLErhV"
+      },
+      "outputs": [],
+      "source": [
+        "sample_video = load_video(video_path)[:100]\n",
+        "sample_video.shape"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ZNLkEZ9Er-c"
+      },
+      "outputs": [],
+      "source": [
+        "to_gif(sample_video)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yskHIRbxEtjS"
+      },
+      "outputs": [],
+      "source": [
+        "predict(sample_video)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "x8Q7Un821X1A"
+      ],
+      "name": "action_recognition_with_tf_hub.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/bangla_article_classifier.ipynb b/site/en/hub/tutorials/bangla_article_classifier.ipynb
new file mode 100644
index 00000000000..988a68c4023
--- /dev/null
+++ b/site/en/hub/tutorials/bangla_article_classifier.ipynb
@@ -0,0 +1,646 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IDdZSPcLtKx4"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-g5By3P4tavy"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS, \n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vpaLrN0mteAS"
+      },
+      "source": [
+        "# Bangla Article Classification With TF-Hub"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GhN2WtIrBQ4y"
+      },
+      "source": [
+        "Caution: In addition to installing Python packages with pip, this notebook uses\n",
+        "`sudo apt install` to install system packages: `unzip`.\n",
+        "\n",
+        "This Colab is a demonstration of using [Tensorflow Hub](https://www.tensorflow.org/hub/) for text classification in non-English/local languages. Here we choose [Bangla](https://en.wikipedia.org/wiki/Bengali_language) as the local language and use pretrained word embeddings to solve a multiclass classification task where we classify Bangla news articles in 5 categories.  The pretrained embeddings for Bangla comes from [fastText](https://fasttext.cc/docs/en/crawl-vectors.html) which is a library by Facebook with released pretrained word vectors for 157 languages. \n",
+        "\n",
+        "We'll use TF-Hub's pretrained embedding exporter for converting the word embeddings to a text embedding module first and then use the module to train a classifier with [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras), Tensorflow's high level user friendly API to build deep learning models.  Even if we are using fastText embeddings here, it's possible to export any other embeddings pretrained from other tasks and quickly get results with Tensorflow hub. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q4DN769E2O_R"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9Vt-StAAZguA"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash\n",
+        "# https://github.com/pypa/setuptools/issues/1694#issuecomment-466010982\n",
+        "pip install gdown --no-use-pep517"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WcBA19FlDPZO"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash\n",
+        "sudo apt-get install -y unzip"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zSeyZMq-BYsu"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "import gdown\n",
+        "import numpy as np\n",
+        "from sklearn.metrics import classification_report\n",
+        "import matplotlib.pyplot as plt\n",
+        "import seaborn as sns"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9FB7gLU4F54l"
+      },
+      "source": [
+        "# Dataset\n",
+        "\n",
+        "We will use [BARD](https://www.researchgate.net/publication/328214545_BARD_Bangla_Article_Classification_Using_a_New_Comprehensive_Dataset) (Bangla Article Dataset) which has around 376,226 articles collected from different Bangla news portals and labelled with 5 categories: economy, state, international, sports, and entertainment. We download the file from Google Drive this ([bit.ly/BARD_DATASET](https://bit.ly/BARD_DATASET)) link is referring to from [this](https://github.com/tanvirfahim15/BARD-Bangla-Article-Classifier) GitHub repository.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zdQrL_rwa-1K"
+      },
+      "outputs": [],
+      "source": [
+        "gdown.download(\n",
+        "    url='https://drive.google.com/uc?id=1Ag0jd21oRwJhVFIBohmX_ogeojVtapLy',\n",
+        "    output='bard.zip',\n",
+        "    quiet=True\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "P2YW4GGa9Y5o"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash\n",
+        "unzip -qo bard.zip"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "js75OARBF_B8"
+      },
+      "source": [
+        "# Export pretrained word vectors to TF-Hub module"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-uAicYA6vLsf"
+      },
+      "source": [
+        "TF-Hub provides some useful scripts for converting word embeddings to TF-hub text embedding modules [here](https://github.com/tensorflow/hub/tree/master/examples/text_embeddings_v2). To make the module for Bangla or any other languages, we simply have to download the word embedding `.txt` or `.vec` file to the same directory as `export_v2.py` and run the script.\n",
+        "\n",
+        "\n",
+        "The exporter reads the embedding vectors and exports it to a Tensorflow [SavedModel](https://www.tensorflow.org/beta/guide/saved_model). A SavedModel contains a complete TensorFlow program including weights and graph. TF-Hub can load the SavedModel as a [module](https://www.tensorflow.org/hub/api_docs/python/hub/Module), which we will use to build the model for text classification. Since we are using `tf.keras` to build the model, we will use [hub.KerasLayer](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer), which provides a wrapper for a TF-Hub module to use as a Keras Layer.\n",
+        "\n",
+        "First we will get our word embeddings from fastText and embedding exporter from TF-Hub [repo](https://github.com/tensorflow/hub).\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5DY5Ze6pO1G5"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash\n",
+        "curl -O https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.bn.300.vec.gz\n",
+        "curl -O https://raw.githubusercontent.com/tensorflow/hub/master/examples/text_embeddings_v2/export_v2.py\n",
+        "gunzip -qf cc.bn.300.vec.gz --k"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PAzdNZaHmdl1"
+      },
+      "source": [
+        "Then, we will run the exporter script on our embedding file. Since fastText embeddings have a header line and are pretty large (around 3.3 GB for Bangla after converting to a module) we ignore the first line and export only the first 100, 000 tokens to the text embedding module."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Tkv5acr_Q9UU"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash\n",
+        "python export_v2.py --embedding_file=cc.bn.300.vec --export_path=text_module --num_lines_to_ignore=1 --num_lines_to_use=100000"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "k9WEpmedF_3_"
+      },
+      "outputs": [],
+      "source": [
+        "module_path = \"text_module\"\n",
+        "embedding_layer = hub.KerasLayer(module_path, trainable=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fQHbmS_D4YIo"
+      },
+      "source": [
+        "The text embedding module takes a batch of sentences in a 1D tensor of strings as input and outputs the embedding vectors of shape (batch_size, embedding_dim) corresponding to the sentences. It preprocesses the input by splitting on spaces. Word embeddings are combined to sentence embeddings with the `sqrtn` combiner(See [here](https://www.tensorflow.org/api_docs/python/tf/nn/embedding_lookup_sparse)). For demonstration we pass a list of Bangla words as input and get the corresponding embedding vectors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Z1MBnaBUihWn"
+      },
+      "outputs": [],
+      "source": [
+        "embedding_layer(['বাস', 'বসবাস', 'ট্রেন', 'যাত্রী', 'ট্রাক']) "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4KY8LiFOHmcd"
+      },
+      "source": [
+        "# Convert to Tensorflow Dataset \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pNguCDNe6bvz"
+      },
+      "source": [
+        "Since the dataset is really large instead of loading the entire dataset in memory we will use a generator to yield samples in run-time in batches using [Tensorflow Dataset](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) functions. The dataset is also very imbalanced, so, before using the generator, we will shuffle the dataset. \n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bYv6LqlEChO1"
+      },
+      "outputs": [],
+      "source": [
+        "dir_names = ['economy', 'sports', 'entertainment', 'state', 'international']\n",
+        "\n",
+        "file_paths = []\n",
+        "labels = []\n",
+        "for i, dir in enumerate(dir_names):\n",
+        "  file_names = [\"/\".join([dir, name]) for name in os.listdir(dir)]\n",
+        "  file_paths += file_names\n",
+        "  labels += [i] * len(os.listdir(dir))\n",
+        "  \n",
+        "np.random.seed(42)\n",
+        "permutation = np.random.permutation(len(file_paths))\n",
+        "\n",
+        "file_paths = np.array(file_paths)[permutation]\n",
+        "labels = np.array(labels)[permutation]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8b-UtAP5TL-W"
+      },
+      "source": [
+        "We can check the distribution of labels in the training and validation examples after shuffling."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mimhWVSzzAmS"
+      },
+      "outputs": [],
+      "source": [
+        "train_frac = 0.8\n",
+        "train_size = int(len(file_paths) * train_frac)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4BNXFrkotAYu"
+      },
+      "outputs": [],
+      "source": [
+        "# plot training vs validation distribution\n",
+        "plt.subplot(1, 2, 1)\n",
+        "plt.hist(labels[0:train_size])\n",
+        "plt.title(\"Train labels\")\n",
+        "plt.subplot(1, 2, 2)\n",
+        "plt.hist(labels[train_size:])\n",
+        "plt.title(\"Validation labels\")\n",
+        "plt.tight_layout()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RVbHb2I3TUNA"
+      },
+      "source": [
+        "To create a [Dataset](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) using a generator, we first write a generator function which reads each of the articles from `file_paths` and the labels from the label array, and yields one training example at each step. We pass this generator function to the [`tf.data.Dataset.from_generator`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_generator) method and specify the output types. Each training example is a tuple containing an article of `tf.string` data type and one-hot encoded label. We split the dataset with a train-validation split of 80-20 using [`tf.data.Dataset.skip`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#skip) and [`tf.data.Dataset.take`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#take) methods."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eZRGTzEhUi7Q"
+      },
+      "outputs": [],
+      "source": [
+        "def load_file(path, label):\n",
+        "    return tf.io.read_file(path), label"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2g4nRflB7fbF"
+      },
+      "outputs": [],
+      "source": [
+        "def make_datasets(train_size):\n",
+        "  batch_size = 256\n",
+        "\n",
+        "  train_files = file_paths[:train_size]\n",
+        "  train_labels = labels[:train_size]\n",
+        "  train_ds = tf.data.Dataset.from_tensor_slices((train_files, train_labels))\n",
+        "  train_ds = train_ds.map(load_file).shuffle(5000)\n",
+        "  train_ds = train_ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n",
+        "\n",
+        "  test_files = file_paths[train_size:]\n",
+        "  test_labels = labels[train_size:]\n",
+        "  test_ds = tf.data.Dataset.from_tensor_slices((test_files, test_labels))\n",
+        "  test_ds = test_ds.map(load_file)\n",
+        "  test_ds = test_ds.batch(batch_size).prefetch(tf.data.AUTOTUNE)\n",
+        "\n",
+        "\n",
+        "  return train_ds, test_ds"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8PuuN6el8tv9"
+      },
+      "outputs": [],
+      "source": [
+        "train_data, validation_data = make_datasets(train_size)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MrdZI6FqPJNP"
+      },
+      "source": [
+        "# Model Training and Evaluation"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jgr7YScGVS58"
+      },
+      "source": [
+        "Since we have already added a wrapper around our module to use it as any other layer in Keras, we can create a small [Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential) model which is a linear stack of layers. We can add our text embedding module with `model.add` just like any other layer. We compile the model by specifying the loss and optimizer and train it for 10 epochs. The `tf.keras` API can handle Tensorflow Datasets as input, so we can pass a Dataset instance to the fit method for model training. Since we are using the generator function, `tf.data` will handle generating the samples, batching them and feeding them to the model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WhCqbDK2uUV5"
+      },
+      "source": [
+        "## Model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nHUw807XPPM9"
+      },
+      "outputs": [],
+      "source": [
+        "def create_model():\n",
+        "  model = tf.keras.Sequential([\n",
+        "    tf.keras.layers.Input(shape=[], dtype=tf.string),\n",
+        "    embedding_layer,\n",
+        "    tf.keras.layers.Dense(64, activation=\"relu\"),\n",
+        "    tf.keras.layers.Dense(16, activation=\"relu\"),\n",
+        "    tf.keras.layers.Dense(5),\n",
+        "  ])\n",
+        "  model.compile(loss=tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+        "      optimizer=\"adam\", metrics=['accuracy'])\n",
+        "  return model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5J4EXJUmPVNG"
+      },
+      "outputs": [],
+      "source": [
+        "model = create_model()\n",
+        "# Create earlystopping callback\n",
+        "early_stopping_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZZ7XJLg2u2No"
+      },
+      "source": [
+        "## Training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OoBkN2tAaXWD"
+      },
+      "outputs": [],
+      "source": [
+        "history = model.fit(train_data, \n",
+        "                    validation_data=validation_data, \n",
+        "                    epochs=5, \n",
+        "                    callbacks=[early_stopping_callback])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoDk8otmMoT7"
+      },
+      "source": [
+        "## Evaluation"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G5ZRKGOsXEh4"
+      },
+      "source": [
+        "We can visualize the accuracy and loss curves for training and validation data using the `tf.keras.callbacks.History` object returned by the `tf.keras.Model.fit` method, which contains the loss and accuracy value for each epoch."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "V6tOnByIOeGn"
+      },
+      "outputs": [],
+      "source": [
+        "# Plot training & validation accuracy values\n",
+        "plt.plot(history.history['accuracy'])\n",
+        "plt.plot(history.history['val_accuracy'])\n",
+        "plt.title('Model accuracy')\n",
+        "plt.ylabel('Accuracy')\n",
+        "plt.xlabel('Epoch')\n",
+        "plt.legend(['Train', 'Test'], loc='upper left')\n",
+        "plt.show()\n",
+        "\n",
+        "# Plot training & validation loss values\n",
+        "plt.plot(history.history['loss'])\n",
+        "plt.plot(history.history['val_loss'])\n",
+        "plt.title('Model loss')\n",
+        "plt.ylabel('Loss')\n",
+        "plt.xlabel('Epoch')\n",
+        "plt.legend(['Train', 'Test'], loc='upper left')\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "D54IXLqcG8Cq"
+      },
+      "source": [
+        "## Prediction\n",
+        "\n",
+        "We can get the predictions for the validation data and check the confusion matrix to see the model's performance for each of the 5 classes. Because `tf.keras.Model.predict` method returns an n-d array for probabilities for each class, they can be converted to class labels using `np.argmax`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dptEywzZJk4l"
+      },
+      "outputs": [],
+      "source": [
+        "y_pred = model.predict(validation_data)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7Dzeml6Pk0ub"
+      },
+      "outputs": [],
+      "source": [
+        "y_pred = np.argmax(y_pred, axis=1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "T4M3Lzg8jHcB"
+      },
+      "outputs": [],
+      "source": [
+        "samples = file_paths[0:3]\n",
+        "for i, sample in enumerate(samples):\n",
+        "  f = open(sample)\n",
+        "  text = f.read()\n",
+        "  print(text[0:100])\n",
+        "  print(\"True Class: \", sample.split(\"/\")[0])\n",
+        "  print(\"Predicted Class: \", dir_names[y_pred[i]])\n",
+        "  f.close()\n",
+        "  "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PlDTIpMBu6h-"
+      },
+      "source": [
+        "## Compare Performance\n",
+        "\n",
+        "Now we can take the correct labels for the validation data from `labels` and compare them with our predictions to get a [classification_report](http://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html). "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mqrERUCS1Xn7"
+      },
+      "outputs": [],
+      "source": [
+        "y_true = np.array(labels[train_size:])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NX5w-NuTKuVP"
+      },
+      "outputs": [],
+      "source": [
+        "print(classification_report(y_true, y_pred, target_names=dir_names))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "p5e9m3bV6oXK"
+      },
+      "source": [
+        "We can also compare our model's performance with the published results obtained in the original [paper](https://www.researchgate.net/publication/328214545_BARD_Bangla_Article_Classification_Using_a_New_Comprehensive_Dataset), which had a 0.96 precision .The original authors described many preprocessing steps performed on the dataset, such as dropping punctuations and digits, removing top 25 most frequest stop words. As we can see in the `classification_report`, we also manage to obtain a 0.96 precision and accuracy after training for only 5 epochs without any preprocessing! \n",
+        "\n",
+        "In this example, when we created the Keras layer from our embedding module, we set the parameter`trainable=False`, which means the embedding weights will not be updated during training. Try setting it to `True` to reach around 97% accuracy using this dataset after only 2 epochs. "
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "IDdZSPcLtKx4"
+      ],
+      "name": "bangla_article_classifier.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/bert_experts.ipynb b/site/en/hub/tutorials/bert_experts.ipynb
new file mode 100644
index 00000000000..5440909f7cb
--- /dev/null
+++ b/site/en/hub/tutorials/bert_experts.ipynb
@@ -0,0 +1,286 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-1vOMEXIhMQt"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "pRfq9ZU5hQhg"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2020 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mTL0TERThT6z"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FkthMlVk8bHp"
+      },
+      "source": [
+        "# BERT Experts from TF-Hub\n",
+        "\n",
+        "This colab demonstrates how to:\n",
+        "* Load BERT models from [TensorFlow Hub](https://tfhub.dev) that have been trained on different tasks including MNLI, SQuAD, and PubMed\n",
+        "* Use a matching preprocessing model to tokenize raw text and convert it to ids\n",
+        "* Generate the pooled and sequence output from the token input ids using the loaded model\n",
+        "* Look at the semantic similarity of the pooled outputs of different sentences\n",
+        "\n",
+        "#### Note: This colab should be run with a GPU runtime"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jspO02jDPfPG"
+      },
+      "source": [
+        "## Set up and imports"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r-ed8zj-dbwm"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install --quiet \"tensorflow-text==2.11.*\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "czDmtrGKYw_5"
+      },
+      "outputs": [],
+      "source": [
+        "import seaborn as sns\n",
+        "from sklearn.metrics import pairwise\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import tensorflow_text as text  # Imports TF ops for preprocessing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "GSuDcPSaY5aB"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Configure the model { run: \"auto\" }\n",
+        "BERT_MODEL = \"https://tfhub.dev/google/experts/bert/wiki_books/2\" # @param {type: \"string\"} [\"https://tfhub.dev/google/experts/bert/wiki_books/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/mnli/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/qnli/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/qqp/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/squad2/2\", \"https://tfhub.dev/google/experts/bert/wiki_books/sst2/2\",  \"https://tfhub.dev/google/experts/bert/pubmed/2\", \"https://tfhub.dev/google/experts/bert/pubmed/squad2/2\"]\n",
+        "# Preprocessing must match the model, but all the above use the same.\n",
+        "PREPROCESS_MODEL = \"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pvaZiGVgwtqw"
+      },
+      "source": [
+        "## Sentences\n",
+        "\n",
+        "Let's take some sentences from Wikipedia to run through the model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tytu-rSpeDNG"
+      },
+      "outputs": [],
+      "source": [
+        "sentences = [\n",
+        "  \"Here We Go Then, You And I is a 1999 album by Norwegian pop artist Morten Abel. It was Abel's second CD as a solo artist.\",\n",
+        "  \"The album went straight to number one on the Norwegian album chart, and sold to double platinum.\",\n",
+        "  \"Among the singles released from the album were the songs \\\"Be My Lover\\\" and \\\"Hard To Stay Awake\\\".\",\n",
+        "  \"Riccardo Zegna is an Italian jazz musician.\",\n",
+        "  \"Rajko Maksimović is a composer, writer, and music pedagogue.\",\n",
+        "  \"One of the most significant Serbian composers of our time, Maksimović has been and remains active in creating works for different ensembles.\",\n",
+        "  \"Ceylon spinach is a common name for several plants and may refer to: Basella alba Talinum fruticosum\",\n",
+        "  \"A solar eclipse occurs when the Moon passes between Earth and the Sun, thereby totally or partly obscuring the image of the Sun for a viewer on Earth.\",\n",
+        "  \"A partial solar eclipse occurs in the polar regions of the Earth when the center of the Moon's shadow misses the Earth.\",\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zI39475kxCKh"
+      },
+      "source": [
+        "## Run the model\n",
+        "\n",
+        "We'll load the BERT model from TF-Hub, tokenize our sentences using the matching preprocessing model from TF-Hub, then feed in the tokenized sentences to the model. To keep this colab fast and simple, we recommend running on GPU.\n",
+        "\n",
+        "Go to **Runtime** → **Change runtime type** to make sure that **GPU** is selected"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "x4t6r22ErQg0"
+      },
+      "outputs": [],
+      "source": [
+        "preprocess = hub.load(PREPROCESS_MODEL)\n",
+        "bert = hub.load(BERT_MODEL)\n",
+        "inputs = preprocess(sentences)\n",
+        "outputs = bert(inputs)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gItjCg4315Cv"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Sentences:\")\n",
+        "print(sentences)\n",
+        "\n",
+        "print(\"\\nBERT inputs:\")\n",
+        "print(inputs)\n",
+        "\n",
+        "print(\"\\nPooled embeddings:\")\n",
+        "print(outputs[\"pooled_output\"])\n",
+        "\n",
+        "print(\"\\nPer token embeddings:\")\n",
+        "print(outputs[\"sequence_output\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ptiW2mgw6x-l"
+      },
+      "source": [
+        "## Semantic similarity\n",
+        "\n",
+        "Now let's take a look at the `pooled_output` embeddings of our sentences and compare how similar they are across sentences."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "GXrSO2Vc1Qtr"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Helper functions\n",
+        "\n",
+        "def plot_similarity(features, labels):\n",
+        "  \"\"\"Plot a similarity matrix of the embeddings.\"\"\"\n",
+        "  cos_sim = pairwise.cosine_similarity(features)\n",
+        "  sns.set(font_scale=1.2)\n",
+        "  cbar_kws=dict(use_gridspec=False, location=\"left\")\n",
+        "  g = sns.heatmap(\n",
+        "      cos_sim, xticklabels=labels, yticklabels=labels,\n",
+        "      vmin=0, vmax=1, cmap=\"Blues\", cbar_kws=cbar_kws)\n",
+        "  g.tick_params(labelright=True, labelleft=False)\n",
+        "  g.set_yticklabels(labels, rotation=0)\n",
+        "  g.set_title(\"Semantic Textual Similarity\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "td6jcT0pJMZ5"
+      },
+      "outputs": [],
+      "source": [
+        "plot_similarity(outputs[\"pooled_output\"], sentences)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tJ4QCyzhSL7B"
+      },
+      "source": [
+        "## Learn more\n",
+        "\n",
+        "* Find more BERT models on [TensorFlow Hub](https://tfhub.dev)\n",
+        "* This notebook demonstrates simple inference with BERT, you can find a more advanced tutorial about fine-tuning BERT at [tensorflow.org/official_models/fine_tuning_bert](https://www.tensorflow.org/official_models/fine_tuning_bert)\n",
+        "* We used just one GPU chip to run the model, you can learn more about how to load models using tf.distribute at [tensorflow.org/tutorials/distribute/save_and_load](https://www.tensorflow.org/tutorials/distribute/save_and_load)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "bert_experts.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/bigbigan_with_tf_hub.ipynb b/site/en/hub/tutorials/bigbigan_with_tf_hub.ipynb
new file mode 100644
index 00000000000..919abc7e354
--- /dev/null
+++ b/site/en/hub/tutorials/bigbigan_with_tf_hub.ipynb
@@ -0,0 +1,713 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pLOYL1PJAAtK"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3fJWQ8WSAFhh"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-1NTVIH6ABK-"
+      },
+      "source": [
+        "# Generating Images with BigBiGAN\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AVvOoEhswyZg"
+      },
+      "source": [
+        "This notebook is a demo for the *BigBiGAN* models available on [TF Hub](https://tfhub.dev/s?publisher=deepmind&q=bigbigan).\n",
+        "\n",
+        "BigBiGAN extends standard (Big)GANs by adding an *encoder* module which can be used for unsupervised representation learning. Roughly speaking, the encoder inverts the generator by predicting latents `z` given real data `x`. See the [BigBiGAN paper on arXiv](https://arxiv.org/abs/1907.02544) [1] for more information about these models.\n",
+        "\n",
+        "After connecting to a runtime, get started by following these instructions:\n",
+        "\n",
+        "1. (Optional) Update the selected **`module_path`** in the first code cell below to load a BigBiGAN generator for a different encoder architecture.\n",
+        "2. Click **Runtime > Run all** to run each cell in order. Afterwards, the outputs, including visualizations of BigBiGAN samples and reconstructions, should automatically appear below.\n",
+        "\n",
+        "Note: if you run into any issues, it can help to click **Runtime > Restart and run all...** to restart your runtime and rerun all cells from scratch.\n",
+        "\n",
+        "[1] Jeff Donahue and Karen Simonyan. [Large Scale Adversarial Representation Learning](https://arxiv.org/abs/1907.02544). *arxiv:1907.02544*, 2019."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DtGFwUKOA9jt"
+      },
+      "source": [
+        "First, set the module path.\n",
+        "By default, we load the BigBiGAN model with the smaller ResNet-50-based encoder from  **`https://tfhub.dev/deepmind/bigbigan-resnet50/1`**.\n",
+        "To load the larger RevNet-50-x4 based model used to achieve the best representation learning results, comment out the active **`module_path`** setting and uncomment the other."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xoY9pl0FBoUS"
+      },
+      "outputs": [],
+      "source": [
+        "module_path = 'https://tfhub.dev/deepmind/bigbigan-resnet50/1'  # ResNet-50\n",
+        "# module_path = 'https://tfhub.dev/deepmind/bigbigan-revnet50x4/1'  # RevNet-50 x4"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Lr01cszC_vcC"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TPdT-hYj1XXQ"
+      },
+      "outputs": [],
+      "source": [
+        "import io\n",
+        "import IPython.display\n",
+        "import PIL.Image\n",
+        "from pprint import pformat\n",
+        "\n",
+        "import numpy as np\n",
+        "\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "tf.disable_v2_behavior()\n",
+        "\n",
+        "import tensorflow_hub as hub"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ouePZy6-CFJl"
+      },
+      "source": [
+        "## Define some functions to display images"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MBQPtmrY2N91"
+      },
+      "outputs": [],
+      "source": [
+        "def imgrid(imarray, cols=4, pad=1, padval=255, row_major=True):\n",
+        "  \"\"\"Lays out a [N, H, W, C] image array as a single image grid.\"\"\"\n",
+        "  pad = int(pad)\n",
+        "  if pad < 0:\n",
+        "    raise ValueError('pad must be non-negative')\n",
+        "  cols = int(cols)\n",
+        "  assert cols >= 1\n",
+        "  N, H, W, C = imarray.shape\n",
+        "  rows = N // cols + int(N % cols != 0)\n",
+        "  batch_pad = rows * cols - N\n",
+        "  assert batch_pad >= 0\n",
+        "  post_pad = [batch_pad, pad, pad, 0]\n",
+        "  pad_arg = [[0, p] for p in post_pad]\n",
+        "  imarray = np.pad(imarray, pad_arg, 'constant', constant_values=padval)\n",
+        "  H += pad\n",
+        "  W += pad\n",
+        "  grid = (imarray\n",
+        "          .reshape(rows, cols, H, W, C)\n",
+        "          .transpose(0, 2, 1, 3, 4)\n",
+        "          .reshape(rows*H, cols*W, C))\n",
+        "  if pad:\n",
+        "    grid = grid[:-pad, :-pad]\n",
+        "  return grid\n",
+        "\n",
+        "def interleave(*args):\n",
+        "  \"\"\"Interleaves input arrays of the same shape along the batch axis.\"\"\"\n",
+        "  if not args:\n",
+        "    raise ValueError('At least one argument is required.')\n",
+        "  a0 = args[0]\n",
+        "  if any(a.shape != a0.shape for a in args):\n",
+        "    raise ValueError('All inputs must have the same shape.')\n",
+        "  if not a0.shape:\n",
+        "    raise ValueError('Inputs must have at least one axis.')\n",
+        "  out = np.transpose(args, [1, 0] + list(range(2, len(a0.shape) + 1)))\n",
+        "  out = out.reshape(-1, *a0.shape[1:])\n",
+        "  return out\n",
+        "\n",
+        "def imshow(a, format='png', jpeg_fallback=True):\n",
+        "  \"\"\"Displays an image in the given format.\"\"\"\n",
+        "  a = a.astype(np.uint8)\n",
+        "  data = io.BytesIO()\n",
+        "  PIL.Image.fromarray(a).save(data, format)\n",
+        "  im_data = data.getvalue()\n",
+        "  try:\n",
+        "    disp = IPython.display.display(IPython.display.Image(im_data))\n",
+        "  except IOError:\n",
+        "    if jpeg_fallback and format != 'jpeg':\n",
+        "      print ('Warning: image was too large to display in format \"{}\"; '\n",
+        "             'trying jpeg instead.').format(format)\n",
+        "      return imshow(a, format='jpeg')\n",
+        "    else:\n",
+        "      raise\n",
+        "  return disp\n",
+        "\n",
+        "def image_to_uint8(x):\n",
+        "  \"\"\"Converts [-1, 1] float array to [0, 255] uint8.\"\"\"\n",
+        "  x = np.asarray(x)\n",
+        "  x = (256. / 2.) * (x + 1.)\n",
+        "  x = np.clip(x, 0, 255)\n",
+        "  x = x.astype(np.uint8)\n",
+        "  return x"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8ASXPMb6CaXR"
+      },
+      "source": [
+        "## Load a BigBiGAN TF Hub module and display its available functionality"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IuG7G1ToCtaf"
+      },
+      "outputs": [],
+      "source": [
+        "# module = hub.Module(module_path, trainable=True, tags={'train'})  # training\n",
+        "module = hub.Module(module_path)  # inference\n",
+        "\n",
+        "for signature in module.get_signature_names():\n",
+        "  print('Signature:', signature)\n",
+        "  print('Inputs:', pformat(module.get_input_info_dict(signature)))\n",
+        "  print('Outputs:', pformat(module.get_output_info_dict(signature)))\n",
+        "  print()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sAY-AmcNCj9_"
+      },
+      "source": [
+        "## Define a wrapper class for convenient access to various functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aTKHkxfx1dAL"
+      },
+      "outputs": [],
+      "source": [
+        "class BigBiGAN(object):\n",
+        "\n",
+        "  def __init__(self, module):\n",
+        "    \"\"\"Initialize a BigBiGAN from the given TF Hub module.\"\"\"\n",
+        "    self._module = module\n",
+        "\n",
+        "  def generate(self, z, upsample=False):\n",
+        "    \"\"\"Run a batch of latents z through the generator to generate images.\n",
+        "\n",
+        "    Args:\n",
+        "      z: A batch of 120D Gaussian latents, shape [N, 120].\n",
+        "\n",
+        "    Returns: a batch of generated RGB images, shape [N, 128, 128, 3], range\n",
+        "      [-1, 1].\n",
+        "    \"\"\"\n",
+        "    outputs = self._module(z, signature='generate', as_dict=True)\n",
+        "    return outputs['upsampled' if upsample else 'default']\n",
+        "\n",
+        "  def make_generator_ph(self):\n",
+        "    \"\"\"Creates a tf.placeholder with the dtype & shape of generator inputs.\"\"\"\n",
+        "    info = self._module.get_input_info_dict('generate')['z']\n",
+        "    return tf.placeholder(dtype=info.dtype, shape=info.get_shape())\n",
+        "\n",
+        "  def gen_pairs_for_disc(self, z):\n",
+        "    \"\"\"Compute generator input pairs (G(z), z) for discriminator, given z.\n",
+        "\n",
+        "    Args:\n",
+        "      z: A batch of latents (120D standard Gaussians), shape [N, 120].\n",
+        "\n",
+        "    Returns: a tuple (G(z), z) of discriminator inputs.\n",
+        "    \"\"\"\n",
+        "    # Downsample 256x256 image x for 128x128 discriminator input.\n",
+        "    x = self.generate(z)\n",
+        "    return x, z\n",
+        "\n",
+        "  def encode(self, x, return_all_features=False):\n",
+        "    \"\"\"Run a batch of images x through the encoder.\n",
+        "\n",
+        "    Args:\n",
+        "      x: A batch of data (256x256 RGB images), shape [N, 256, 256, 3], range\n",
+        "        [-1, 1].\n",
+        "      return_all_features: If True, return all features computed by the encoder.\n",
+        "        Otherwise (default) just return a sample z_hat.\n",
+        "\n",
+        "    Returns: the sample z_hat of shape [N, 120] (or a dict of all features if\n",
+        "      return_all_features).\n",
+        "    \"\"\"\n",
+        "    outputs = self._module(x, signature='encode', as_dict=True)\n",
+        "    return outputs if return_all_features else outputs['z_sample']\n",
+        "\n",
+        "  def make_encoder_ph(self):\n",
+        "    \"\"\"Creates a tf.placeholder with the dtype & shape of encoder inputs.\"\"\"\n",
+        "    info = self._module.get_input_info_dict('encode')['x']\n",
+        "    return tf.placeholder(dtype=info.dtype, shape=info.get_shape())\n",
+        "\n",
+        "  def enc_pairs_for_disc(self, x):\n",
+        "    \"\"\"Compute encoder input pairs (x, E(x)) for discriminator, given x.\n",
+        "\n",
+        "    Args:\n",
+        "      x: A batch of data (256x256 RGB images), shape [N, 256, 256, 3], range\n",
+        "        [-1, 1].\n",
+        "\n",
+        "    Returns: a tuple (downsample(x), E(x)) of discriminator inputs.\n",
+        "    \"\"\"\n",
+        "    # Downsample 256x256 image x for 128x128 discriminator input.\n",
+        "    x_down = tf.nn.avg_pool(x, ksize=2, strides=2, padding='SAME')\n",
+        "    z = self.encode(x)\n",
+        "    return x_down, z\n",
+        "\n",
+        "  def discriminate(self, x, z):\n",
+        "    \"\"\"Compute the discriminator scores for pairs of data (x, z).\n",
+        "\n",
+        "    (x, z) must be batches with the same leading batch dimension, and joint\n",
+        "      scores are computed on corresponding pairs x[i] and z[i].\n",
+        "\n",
+        "    Args:\n",
+        "      x: A batch of data (128x128 RGB images), shape [N, 128, 128, 3], range\n",
+        "        [-1, 1].\n",
+        "      z: A batch of latents (120D standard Gaussians), shape [N, 120].\n",
+        "\n",
+        "    Returns:\n",
+        "      A dict of scores:\n",
+        "        score_xz: the joint scores for the (x, z) pairs.\n",
+        "        score_x: the unary scores for x only.\n",
+        "        score_z: the unary scores for z only.\n",
+        "    \"\"\"\n",
+        "    inputs = dict(x=x, z=z)\n",
+        "    return self._module(inputs, signature='discriminate', as_dict=True)\n",
+        "\n",
+        "  def reconstruct_x(self, x, use_sample=True, upsample=False):\n",
+        "    \"\"\"Compute BigBiGAN reconstructions of images x via G(E(x)).\n",
+        "\n",
+        "    Args:\n",
+        "      x: A batch of data (256x256 RGB images), shape [N, 256, 256, 3], range\n",
+        "        [-1, 1].\n",
+        "      use_sample: takes a sample z_hat ~ E(x). Otherwise, deterministically\n",
+        "        use the mean. (Though a sample z_hat may be far from the mean z,\n",
+        "        typically the resulting recons G(z_hat) and G(z) are very\n",
+        "        similar.\n",
+        "      upsample: if set, upsample the reconstruction to the input resolution\n",
+        "        (256x256). Otherwise return the raw lower resolution generator output\n",
+        "        (128x128).\n",
+        "\n",
+        "    Returns: a batch of recons G(E(x)), shape [N, 256, 256, 3] if\n",
+        "      `upsample`, otherwise [N, 128, 128, 3].\n",
+        "    \"\"\"\n",
+        "    if use_sample:\n",
+        "      z = self.encode(x)\n",
+        "    else:\n",
+        "      z = self.encode(x, return_all_features=True)['z_mean']\n",
+        "    recons = self.generate(z, upsample=upsample)\n",
+        "    return recons\n",
+        "\n",
+        "  def losses(self, x, z):\n",
+        "    \"\"\"Compute per-module BigBiGAN losses given data & latent sample batches.\n",
+        "\n",
+        "    Args:\n",
+        "      x: A batch of data (256x256 RGB images), shape [N, 256, 256, 3], range\n",
+        "        [-1, 1].\n",
+        "      z: A batch of latents (120D standard Gaussians), shape [M, 120].\n",
+        "\n",
+        "    For the original BigBiGAN losses, pass batches of size N=M=2048, with z's\n",
+        "    sampled from a 120D standard Gaussian (e.g., np.random.randn(2048, 120)),\n",
+        "    and x's sampled from the ImageNet (ILSVRC2012) training set with the\n",
+        "    \"ResNet-style\" preprocessing from:\n",
+        "\n",
+        "        https://github.com/tensorflow/tpu/blob/master/models/official/resnet/resnet_preprocessing.py\n",
+        "\n",
+        "    Returns:\n",
+        "      A dict of per-module losses:\n",
+        "        disc: loss for the discriminator.\n",
+        "        enc: loss for the encoder.\n",
+        "        gen: loss for the generator.\n",
+        "    \"\"\"\n",
+        "    # Compute discriminator scores on (x, E(x)) pairs.\n",
+        "    # Downsample 256x256 image x for 128x128 discriminator input.\n",
+        "    scores_enc_x_dict = self.discriminate(*self.enc_pairs_for_disc(x))\n",
+        "    scores_enc_x = tf.concat([scores_enc_x_dict['score_xz'],\n",
+        "                              scores_enc_x_dict['score_x'],\n",
+        "                              scores_enc_x_dict['score_z']], axis=0)\n",
+        "\n",
+        "    # Compute discriminator scores on (G(z), z) pairs.\n",
+        "    scores_gen_z_dict = self.discriminate(*self.gen_pairs_for_disc(z))\n",
+        "    scores_gen_z = tf.concat([scores_gen_z_dict['score_xz'],\n",
+        "                              scores_gen_z_dict['score_x'],\n",
+        "                              scores_gen_z_dict['score_z']], axis=0)\n",
+        "\n",
+        "    disc_loss_enc_x = tf.reduce_mean(tf.nn.relu(1. - scores_enc_x))\n",
+        "    disc_loss_gen_z = tf.reduce_mean(tf.nn.relu(1. + scores_gen_z))\n",
+        "    disc_loss = disc_loss_enc_x + disc_loss_gen_z\n",
+        "\n",
+        "    enc_loss = tf.reduce_mean(scores_enc_x)\n",
+        "    gen_loss = tf.reduce_mean(-scores_gen_z)\n",
+        "\n",
+        "    return dict(disc=disc_loss, enc=enc_loss, gen=gen_loss)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5L5SFfH4C9gu"
+      },
+      "source": [
+        "## Create tensors to be used later for computing samples, reconstructions, discriminator scores, and losses"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "goxtzcb-19NA"
+      },
+      "outputs": [],
+      "source": [
+        "bigbigan = BigBiGAN(module)\n",
+        "\n",
+        "# Make input placeholders for x (`enc_ph`) and z (`gen_ph`).\n",
+        "enc_ph = bigbigan.make_encoder_ph()\n",
+        "gen_ph = bigbigan.make_generator_ph()\n",
+        "\n",
+        "# Compute samples G(z) from encoder input z (`gen_ph`).\n",
+        "gen_samples = bigbigan.generate(gen_ph)\n",
+        "\n",
+        "# Compute reconstructions G(E(x)) of encoder input x (`enc_ph`).\n",
+        "recon_x = bigbigan.reconstruct_x(enc_ph, upsample=True)\n",
+        "\n",
+        "# Compute encoder features used for representation learning evaluations given\n",
+        "# encoder input x (`enc_ph`).\n",
+        "enc_features = bigbigan.encode(enc_ph, return_all_features=True)\n",
+        "\n",
+        "# Compute discriminator scores for encoder pairs (x, E(x)) given x (`enc_ph`)\n",
+        "# and generator pairs (G(z), z) given z (`gen_ph`).\n",
+        "disc_scores_enc = bigbigan.discriminate(*bigbigan.enc_pairs_for_disc(enc_ph))\n",
+        "disc_scores_gen = bigbigan.discriminate(*bigbigan.gen_pairs_for_disc(gen_ph))\n",
+        "\n",
+        "# Compute losses.\n",
+        "losses = bigbigan.losses(enc_ph, gen_ph)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ly7LWnSUDQ_P"
+      },
+      "source": [
+        "## Create a TensorFlow session and initialize variables"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CPnzCHDWFJwx"
+      },
+      "outputs": [],
+      "source": [
+        "init = tf.global_variables_initializer()\n",
+        "sess = tf.Session()\n",
+        "sess.run(init)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gcEVS26D-ues"
+      },
+      "source": [
+        "# Generator samples"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LYSA8Zvb-w7S"
+      },
+      "source": [
+        "First, we'll visualize samples from the pretrained BigBiGAN generator by sampling generator inputs `z` from a standard Gaussian (via `np.random.randn`) and displaying the images it produces. So far we're not going beyond the capabilites of a standard GAN -- we're just using the generator (and ignoring the encoder) for now."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9zfpvw8fGNMr"
+      },
+      "outputs": [],
+      "source": [
+        "feed_dict = {gen_ph: np.random.randn(32, 120)}\n",
+        "_out_samples = sess.run(gen_samples, feed_dict=feed_dict)\n",
+        "print('samples shape:', _out_samples.shape)\n",
+        "imshow(imgrid(image_to_uint8(_out_samples), cols=4))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9v58CTfl8jTc"
+      },
+      "source": [
+        "# Load `test_images` from the TF-Flowers dataset"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o0kmzQ4EqKJt"
+      },
+      "source": [
+        "BigBiGAN is trained on ImageNet, but as it's too large to work with in this demo, we use the smaller TF-Flowers [1] dataset as our inputs for visualizing reconstructions and computing encoder features.\n",
+        "\n",
+        "In this cell we load TF-Flowers (downloading the dataset if needed) and store a fixed batch of 256x256 RGB image samples in a NumPy array `test_images`.\n",
+        "\n",
+        "[1] https://www.tensorflow.org/datasets/catalog/tf_flowers"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OBgpkMdkUjL-"
+      },
+      "outputs": [],
+      "source": [
+        "def get_flowers_data():\n",
+        "  \"\"\"Returns a [32, 256, 256, 3] np.array of preprocessed TF-Flowers samples.\"\"\"\n",
+        "  import tensorflow_datasets as tfds\n",
+        "  ds, info = tfds.load('tf_flowers', split='train', with_info=True)\n",
+        "\n",
+        "  # Just get the images themselves as we don't need labels for this demo.\n",
+        "  ds = ds.map(lambda x: x['image'])\n",
+        "\n",
+        "  # Filter out small images (with minor edge length <256).\n",
+        "  ds = ds.filter(lambda x: tf.reduce_min(tf.shape(x)[:2]) >= 256)\n",
+        "\n",
+        "  # Take the center square crop of the image and resize to 256x256.\n",
+        "  def crop_and_resize(image):\n",
+        "    imsize = tf.shape(image)[:2]\n",
+        "    minor_edge = tf.reduce_min(imsize)\n",
+        "    start = (imsize - minor_edge) // 2\n",
+        "    stop = start + minor_edge\n",
+        "    cropped_image = image[start[0] : stop[0], start[1] : stop[1]]\n",
+        "    resized_image = tf.image.resize_bicubic([cropped_image], [256, 256])[0]\n",
+        "    return resized_image\n",
+        "  ds = ds.map(crop_and_resize)\n",
+        "\n",
+        "  # Convert images from [0, 255] uint8 to [-1, 1] float32.\n",
+        "  ds = ds.map(lambda image: tf.cast(image, tf.float32) / (255. / 2.) - 1)\n",
+        "\n",
+        "  # Take the first 32 samples.\n",
+        "  ds = ds.take(32)\n",
+        "\n",
+        "  return np.array(list(tfds.as_numpy(ds)))\n",
+        "\n",
+        "test_images = get_flowers_data()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QAFJQU597n2A"
+      },
+      "source": [
+        "# Reconstructions"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EmCQ9N9b7ptM"
+      },
+      "source": [
+        "Now we visualize BigBiGAN reconstructions by passing real images through the encoder and back through the generator, computing `G(E(x))` given images `x`.\n",
+        "Below, input images `x` are shown in the left column, and corresponding reconstructions are shown on the right.\n",
+        "\n",
+        "Note that reconstructions are not pixel-perfect matches to the input images; rather, they tend to capture the higher level semantic content of the input while \"forgetting\" most of the low-level detail. This suggests the BigBiGAN encoder may learn to capture the types of high level semantic information about images that we'd like to see in a representation learning approach.\n",
+        "\n",
+        "Also note that the raw reconstructions of the 256x256 input images are at the lower resolution produced by our generator -- 128x128. We upsample them for visualization purposes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R2F3eq8aFRle"
+      },
+      "outputs": [],
+      "source": [
+        "test_images_batch = test_images[:16]\n",
+        "_out_recons = sess.run(recon_x, feed_dict={enc_ph: test_images_batch})\n",
+        "print('reconstructions shape:', _out_recons.shape)\n",
+        "\n",
+        "inputs_and_recons = interleave(test_images_batch, _out_recons)\n",
+        "print('inputs_and_recons shape:', inputs_and_recons.shape)\n",
+        "imshow(imgrid(image_to_uint8(inputs_and_recons), cols=2))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zPpW3qdbEpXL"
+      },
+      "source": [
+        "# Encoder features"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2gAW76YxEsZa"
+      },
+      "source": [
+        "We now demonstrate how to compute features from the encoder used for standard representation learning evaluations.\n",
+        "\n",
+        "These features could be used in a linear or nearest neighbors-based classifier. We include the standard feature taken after the global average pooling (key `avepool_feat`) as well as the larger \"BN+CReLU\" feature (key `bn_crelu_feat`) used to achieve the best results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hpZYe5S_FQEw"
+      },
+      "outputs": [],
+      "source": [
+        "_out_features = sess.run(enc_features, feed_dict={enc_ph: test_images_batch})\n",
+        "print('AvePool features shape:', _out_features['avepool_feat'].shape)\n",
+        "print('BN+CReLU features shape:', _out_features['bn_crelu_feat'].shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TGzahsms2w9a"
+      },
+      "source": [
+        "# Discriminator scores and losses"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "B2_5BIBN21Hr"
+      },
+      "source": [
+        "Finally, we'll compute the discriminator scores and losses on batches of encoder and generator pairs. These losses could be passed into an optimizer to train BigBiGAN.\n",
+        "\n",
+        "We use our batch of images above as the encoder inputs `x`, computing the encoder score as `D(x, E(x))`. For the generator inputs we sample `z` from a 120D standard Gaussian via `np.random.randn`, computing the generator score as `D(G(z), z)`.\n",
+        "\n",
+        "The discriminator predicts a joint score `score_xz` for the `(x, z)` pairs as well as unary scores `score_x` and `score_z` for `x` and `z` alone, respectively. It's trained to give high (positive) scores to encoder pairs and low (negative) scores to generator pairs. This mostly holds below, though the unary `score_z` is negative in both cases, indicating that the encoder outputs `E(x)` resemble actual samples from a Gaussian."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8JJ8Go0dr22-"
+      },
+      "outputs": [],
+      "source": [
+        "feed_dict = {enc_ph: test_images, gen_ph: np.random.randn(32, 120)}\n",
+        "_out_scores_enc, _out_scores_gen, _out_losses = sess.run(\n",
+        "    [disc_scores_enc, disc_scores_gen, losses], feed_dict=feed_dict)\n",
+        "print('Encoder scores:', {k: v.mean() for k, v in _out_scores_enc.items()})\n",
+        "print('Generator scores:', {k: v.mean() for k, v in _out_scores_gen.items()})\n",
+        "print('Losses:', _out_losses)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "9v58CTfl8jTc"
+      ],
+      "name": "bigbigan_with_tf_hub.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/biggan_generation_with_tf_hub.ipynb b/site/en/hub/tutorials/biggan_generation_with_tf_hub.ipynb
new file mode 100644
index 00000000000..e388f91fbcc
--- /dev/null
+++ b/site/en/hub/tutorials/biggan_generation_with_tf_hub.ipynb
@@ -0,0 +1,421 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pLOYL1PJAAtK"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3fJWQ8WSAFhh"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Cd1dhL4Ykbm7"
+      },
+      "source": [
+        "# Generating Images with BigGAN\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-1NTVIH6ABK-"
+      },
+      "source": [
+        "This notebook is a demo for the *BigGAN* image generators available on [TF Hub](https://tfhub.dev/s?publisher=deepmind&q=biggan).\n",
+        "\n",
+        "See the [BigGAN paper on arXiv](https://arxiv.org/abs/1809.11096) [1] for more information about these models.\n",
+        "\n",
+        "After connecting to a runtime, get started by following these instructions:\n",
+        "\n",
+        "1. (Optional) Update the selected **`module_path`** in the first code cell below to load a BigGAN generator for a different image resolution.\n",
+        "2. Click **Runtime > Run all** to run each cell in order.\n",
+        "  * Afterwards, the interactive visualizations should update automatically when you modify the settings using the sliders and dropdown menus.\n",
+        "  * If not, press the **Play** button by the cell to re-render outputs manually.\n",
+        "\n",
+        "Note: if you run into any issues, it can help to click **Runtime > Restart and run all...** to restart your runtime and rerun all cells from scratch.\n",
+        "\n",
+        "[1] Andrew Brock, Jeff Donahue, and Karen Simonyan. [Large Scale GAN Training for High Fidelity Natural Image Synthesis](https://arxiv.org/abs/1809.11096). *arxiv:1809.11096*, 2018."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XS1_N6hKj8cz"
+      },
+      "source": [
+        "First, set the module path.\n",
+        "By default, we load the BigGAN-deep generator for 256x256 images from **`https://tfhub.dev/deepmind/biggan-deep-256/1`**.\n",
+        "To generate 128x128 or 512x512 images or to use the original BigGAN generators, comment out the active **`module_path`** setting and uncomment one of the others."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OJCIhQPClKJ1"
+      },
+      "outputs": [],
+      "source": [
+        "# BigGAN-deep models\n",
+        "# module_path = 'https://tfhub.dev/deepmind/biggan-deep-128/1'  # 128x128 BigGAN-deep\n",
+        "module_path = 'https://tfhub.dev/deepmind/biggan-deep-256/1'  # 256x256 BigGAN-deep\n",
+        "# module_path = 'https://tfhub.dev/deepmind/biggan-deep-512/1'  # 512x512 BigGAN-deep\n",
+        "\n",
+        "# BigGAN (original) models\n",
+        "# module_path = 'https://tfhub.dev/deepmind/biggan-128/2'  # 128x128 BigGAN\n",
+        "# module_path = 'https://tfhub.dev/deepmind/biggan-256/2'  # 256x256 BigGAN\n",
+        "# module_path = 'https://tfhub.dev/deepmind/biggan-512/2'  # 512x512 BigGAN"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JJrTM6hAi0CJ"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lOZnst2jeWDL"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow.compat.v1 as tf\n",
+        "tf.disable_v2_behavior()\n",
+        "\n",
+        "import os\n",
+        "import io\n",
+        "import IPython.display\n",
+        "import numpy as np\n",
+        "import PIL.Image\n",
+        "from scipy.stats import truncnorm\n",
+        "import tensorflow_hub as hub\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "stWb21nlcyCm"
+      },
+      "source": [
+        "## Load a BigGAN generator module from TF Hub"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tVgwgJiCH3PV"
+      },
+      "outputs": [],
+      "source": [
+        "tf.reset_default_graph()\n",
+        "print('Loading BigGAN module from:', module_path)\n",
+        "module = hub.Module(module_path)\n",
+        "inputs = {k: tf.placeholder(v.dtype, v.get_shape().as_list(), k)\n",
+        "          for k, v in module.get_input_info_dict().items()}\n",
+        "output = module(inputs)\n",
+        "\n",
+        "print()\n",
+        "print('Inputs:\\n', '\\n'.join(\n",
+        "    '  {}: {}'.format(*kv) for kv in inputs.items()))\n",
+        "print()\n",
+        "print('Output:', output)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ry62-8SWfuds"
+      },
+      "source": [
+        "## Define some functions for sampling and displaying BigGAN images"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "46M8prJPDEsV"
+      },
+      "outputs": [],
+      "source": [
+        "input_z = inputs['z']\n",
+        "input_y = inputs['y']\n",
+        "input_trunc = inputs['truncation']\n",
+        "\n",
+        "dim_z = input_z.shape.as_list()[1]\n",
+        "vocab_size = input_y.shape.as_list()[1]\n",
+        "\n",
+        "def truncated_z_sample(batch_size, truncation=1., seed=None):\n",
+        "  state = None if seed is None else np.random.RandomState(seed)\n",
+        "  values = truncnorm.rvs(-2, 2, size=(batch_size, dim_z), random_state=state)\n",
+        "  return truncation * values\n",
+        "\n",
+        "def one_hot(index, vocab_size=vocab_size):\n",
+        "  index = np.asarray(index)\n",
+        "  if len(index.shape) == 0:\n",
+        "    index = np.asarray([index])\n",
+        "  assert len(index.shape) == 1\n",
+        "  num = index.shape[0]\n",
+        "  output = np.zeros((num, vocab_size), dtype=np.float32)\n",
+        "  output[np.arange(num), index] = 1\n",
+        "  return output\n",
+        "\n",
+        "def one_hot_if_needed(label, vocab_size=vocab_size):\n",
+        "  label = np.asarray(label)\n",
+        "  if len(label.shape) <= 1:\n",
+        "    label = one_hot(label, vocab_size)\n",
+        "  assert len(label.shape) == 2\n",
+        "  return label\n",
+        "\n",
+        "def sample(sess, noise, label, truncation=1., batch_size=8,\n",
+        "           vocab_size=vocab_size):\n",
+        "  noise = np.asarray(noise)\n",
+        "  label = np.asarray(label)\n",
+        "  num = noise.shape[0]\n",
+        "  if len(label.shape) == 0:\n",
+        "    label = np.asarray([label] * num)\n",
+        "  if label.shape[0] != num:\n",
+        "    raise ValueError('Got # noise samples ({}) != # label samples ({})'\n",
+        "                     .format(noise.shape[0], label.shape[0]))\n",
+        "  label = one_hot_if_needed(label, vocab_size)\n",
+        "  ims = []\n",
+        "  for batch_start in range(0, num, batch_size):\n",
+        "    s = slice(batch_start, min(num, batch_start + batch_size))\n",
+        "    feed_dict = {input_z: noise[s], input_y: label[s], input_trunc: truncation}\n",
+        "    ims.append(sess.run(output, feed_dict=feed_dict))\n",
+        "  ims = np.concatenate(ims, axis=0)\n",
+        "  assert ims.shape[0] == num\n",
+        "  ims = np.clip(((ims + 1) / 2.0) * 256, 0, 255)\n",
+        "  ims = np.uint8(ims)\n",
+        "  return ims\n",
+        "\n",
+        "def interpolate(A, B, num_interps):\n",
+        "  if A.shape != B.shape:\n",
+        "    raise ValueError('A and B must have the same shape to interpolate.')\n",
+        "  alphas = np.linspace(0, 1, num_interps)\n",
+        "  return np.array([(1-a)*A + a*B for a in alphas])\n",
+        "\n",
+        "def imgrid(imarray, cols=5, pad=1):\n",
+        "  if imarray.dtype != np.uint8:\n",
+        "    raise ValueError('imgrid input imarray must be uint8')\n",
+        "  pad = int(pad)\n",
+        "  assert pad >= 0\n",
+        "  cols = int(cols)\n",
+        "  assert cols >= 1\n",
+        "  N, H, W, C = imarray.shape\n",
+        "  rows = N // cols + int(N % cols != 0)\n",
+        "  batch_pad = rows * cols - N\n",
+        "  assert batch_pad >= 0\n",
+        "  post_pad = [batch_pad, pad, pad, 0]\n",
+        "  pad_arg = [[0, p] for p in post_pad]\n",
+        "  imarray = np.pad(imarray, pad_arg, 'constant', constant_values=255)\n",
+        "  H += pad\n",
+        "  W += pad\n",
+        "  grid = (imarray\n",
+        "          .reshape(rows, cols, H, W, C)\n",
+        "          .transpose(0, 2, 1, 3, 4)\n",
+        "          .reshape(rows*H, cols*W, C))\n",
+        "  if pad:\n",
+        "    grid = grid[:-pad, :-pad]\n",
+        "  return grid\n",
+        "\n",
+        "def imshow(a, format='png', jpeg_fallback=True):\n",
+        "  a = np.asarray(a, dtype=np.uint8)\n",
+        "  data = io.BytesIO()\n",
+        "  PIL.Image.fromarray(a).save(data, format)\n",
+        "  im_data = data.getvalue()\n",
+        "  try:\n",
+        "    disp = IPython.display.display(IPython.display.Image(im_data))\n",
+        "  except IOError:\n",
+        "    if jpeg_fallback and format != 'jpeg':\n",
+        "      print(('Warning: image was too large to display in format \"{}\"; '\n",
+        "             'trying jpeg instead.').format(format))\n",
+        "      return imshow(a, format='jpeg')\n",
+        "    else:\n",
+        "      raise\n",
+        "  return disp"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uCeCg3Sdf8Nv"
+      },
+      "source": [
+        "## Create a TensorFlow session and initialize variables"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rYJor5bOaVn1"
+      },
+      "outputs": [],
+      "source": [
+        "initializer = tf.global_variables_initializer()\n",
+        "sess = tf.Session()\n",
+        "sess.run(initializer)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SeZ7u3rWd9jz"
+      },
+      "source": [
+        "# Explore BigGAN samples of a particular category\n",
+        "\n",
+        "Try varying the **`truncation`** value.\n",
+        "\n",
+        "(Double-click on the cell to view code.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HuCO9tv3IKT2"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Category-conditional sampling { display-mode: \"form\", run: \"auto\" }\n",
+        "\n",
+        "num_samples = 10 #@param {type:\"slider\", min:1, max:20, step:1}\n",
+        "truncation = 0.4 #@param {type:\"slider\", min:0.02, max:1, step:0.02}\n",
+        "noise_seed = 0 #@param {type:\"slider\", min:0, max:100, step:1}\n",
+        "category = \"933) cheeseburger\"\n", 
+        "\n",
+        "z = truncated_z_sample(num_samples, truncation, noise_seed)\n",
+        "y = int(category.split(')')[0])\n",
+        "\n",
+        "ims = sample(sess, z, y, truncation=truncation)\n",
+        "imshow(imgrid(ims, cols=min(num_samples, 5)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hHNXtvuQgKwa"
+      },
+      "source": [
+        "# Interpolate between BigGAN samples\n",
+        "\n",
+        "Try setting different **`category`**s  with the same **`noise_seed`**s, or the same **`category`**s with different **`noise_seed`**s. Or go wild and set both any way you like!\n",
+        "\n",
+        "(Double-click on the cell to view code.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dSAyfDfnVugs"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Interpolation { display-mode: \"form\", run: \"auto\" }\n",
+        "\n",
+        "num_samples = 2 #@param {type:\"slider\", min:1, max:5, step:1}\n",
+        "num_interps = 5 #@param {type:\"slider\", min:2, max:10, step:1}\n",
+        "truncation = 0.2 #@param {type:\"slider\", min:0.02, max:1, step:0.02}\n",
+        "noise_seed_A = 0 #@param {type:\"slider\", min:0, max:100, step:1}\n",
+        "category_A = \"207) golden retriever\"\n",
+        "noise_seed_B = 0 #@param {type:\"slider\", min:0, max:100, step:1}\n",
+        "category_B = \"8) hen\"\n",
+        "\n",
+        "def interpolate_and_shape(A, B, num_interps):\n",
+        "  interps = interpolate(A, B, num_interps)\n",
+        "  return (interps.transpose(1, 0, *range(2, len(interps.shape)))\n",
+        "                 .reshape(num_samples * num_interps, *interps.shape[2:]))\n",
+        "\n",
+        "z_A, z_B = [truncated_z_sample(num_samples, truncation, noise_seed)\n",
+        "            for noise_seed in [noise_seed_A, noise_seed_B]]\n",
+        "y_A, y_B = [one_hot([int(category.split(')')[0])] * num_samples)\n",
+        "            for category in [category_A, category_B]]\n",
+        "\n",
+        "z_interp = interpolate_and_shape(z_A, z_B, num_interps)\n",
+        "y_interp = interpolate_and_shape(y_A, y_B, num_interps)\n",
+        "\n",
+        "ims = sample(sess, z_interp, y_interp, truncation=truncation)\n",
+        "imshow(imgrid(ims, cols=num_interps))"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "pLOYL1PJAAtK"
+      ],
+      "name": "biggan_generation_with_tf_hub.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/bird_vocalization_classifier.ipynb b/site/en/hub/tutorials/bird_vocalization_classifier.ipynb
new file mode 100644
index 00000000000..563be9b425a
--- /dev/null
+++ b/site/en/hub/tutorials/bird_vocalization_classifier.ipynb
@@ -0,0 +1,375 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QD3FvutQsaqc"
+      },
+      "source": [
+        "##### Copyright 2023 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-5fm9kVRsfuG"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2023 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QNDQZiSGtXMu"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1JAO_rv_QEBr"
+      },
+      "source": [
+        "# Using Google Bird Vocalization model\n",
+        "\n",
+        "The Google Bird Vocalization is a global bird embedding and classification model.\n",
+        "\n",
+        "This model expects as input a 5-second audio segment sampled at 32kHz\n",
+        "\n",
+        "The model outputs both the logits and the embeddigs for each input window of audio.\n",
+        "\n",
+        "On this notebook you'll learn how to feed the audio properly to the model and how to use the logits for inference.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bytIYq0MjEKT"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q \"tensorflow_io==0.28.*\"\n",
+        "!pip install -q librosa"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aXXTdq-eq6lk"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import tensorflow_io as tfio\n",
+        "\n",
+        "import numpy as np\n",
+        "import librosa\n",
+        "\n",
+        "import csv\n",
+        "import io\n",
+        "\n",
+        "from IPython.display import Audio"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "B6mFpgMWQjgk"
+      },
+      "source": [
+        "Loading the Model from TFHub"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CQ1P3IkpQiya"
+      },
+      "outputs": [],
+      "source": [
+        "model_handle = \"https://tfhub.dev/google/bird-vocalization-classifier/1\"\n",
+        "model = hub.load(model_handle)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3OOw23B3fZT6"
+      },
+      "source": [
+        "Lets load the labels that the model was trained on.\n",
+        "\n",
+        "The labels file is in the assets forlder under label.csv. Each line is an ebird id."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f5i-R4k9ZhwN"
+      },
+      "outputs": [],
+      "source": [
+        "# Find the name of the class with the top score when mean-aggregated across frames.\n",
+        "def class_names_from_csv(class_map_csv_text):\n",
+        "  \"\"\"Returns list of class names corresponding to score vector.\"\"\"\n",
+        "  with open(labels_path) as csv_file:\n",
+        "    csv_reader = csv.reader(csv_file, delimiter=',')\n",
+        "    class_names = [mid for mid, desc in csv_reader]\n",
+        "    return class_names[1:]\n",
+        "\n",
+        "labels_path = hub.resolve(model_handle) + \"/assets/label.csv\"\n",
+        "classes = class_names_from_csv(labels_path)\n",
+        "print(classes)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b2JYPafeRRi_"
+      },
+      "source": [
+        "The ```frame_audio``` function is based on the [Chirp lib](https://github.com/google-research/chirp/blob/10c5faa325a3c3468fa6f18a736fc1aeb9bf8129/chirp/inference/interface.py#L128) version but using tf.signal instead of librosa.\n",
+        "\n",
+        "The `ensure_sample_rate` is a function to make sure that any audio used with the model has the expected sample rate of 32kHz"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "t65gi_DTrRaa"
+      },
+      "outputs": [],
+      "source": [
+        "def frame_audio(\n",
+        "      audio_array: np.ndarray,\n",
+        "      window_size_s: float = 5.0,\n",
+        "      hop_size_s: float = 5.0,\n",
+        "      sample_rate = 32000,\n",
+        "  ) -> np.ndarray:\n",
+        "    \"\"\"Helper function for framing audio for inference.\"\"\"\n",
+        "    if window_size_s is None or window_size_s < 0:\n",
+        "      return audio_array[np.newaxis, :]\n",
+        "    frame_length = int(window_size_s * sample_rate)\n",
+        "    hop_length = int(hop_size_s * sample_rate)\n",
+        "    framed_audio = tf.signal.frame(audio_array, frame_length, hop_length, pad_end=True)\n",
+        "    return framed_audio\n",
+        "\n",
+        "def ensure_sample_rate(waveform, original_sample_rate,\n",
+        "                       desired_sample_rate=32000):\n",
+        "  \"\"\"Resample waveform if required.\"\"\"\n",
+        "  if original_sample_rate != desired_sample_rate:\n",
+        "    waveform = tfio.audio.resample(waveform, original_sample_rate, desired_sample_rate)\n",
+        "  return desired_sample_rate, waveform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G7uAuI4f6ehb"
+      },
+      "source": [
+        "Lets load a file from Wikipedia.\n",
+        "\n",
+        "To be more precise, the audio of a [Common Blackbird](https://es.wikipedia.org/wiki/Turdus_merula)\n",
+        "\n",
+        "|
\"Common|\n",
+        "|:--:|\n",
+        "| *By Andreas Trepte - Own work, CC BY-SA 2.5, Link*
 |\n",
+        "\n",
+        "\n",
+        "The audio was contributed by Oona Räisänen (Mysid) under the public domain license."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "whkmGeJ9lmyd"
+      },
+      "outputs": [],
+      "source": [
+        "!curl -O  \"https://upload.wikimedia.org/wikipedia/commons/7/7c/Turdus_merula_2.ogg\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ff6nOV2EurAO"
+      },
+      "outputs": [],
+      "source": [
+        "turdus_merula = \"Turdus_merula_2.ogg\"\n",
+        "\n",
+        "audio, sample_rate = librosa.load(turdus_merula)\n",
+        "\n",
+        "sample_rate, wav_data_turdus = ensure_sample_rate(audio, sample_rate)\n",
+        "Audio(wav_data_turdus, rate=sample_rate)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sjpKLk9K7TTV"
+      },
+      "source": [
+        "The audio has 24 seconds and the model expects chunks of 5 seconds.\n",
+        "\n",
+        "The `frame_audio` function can fix that and split the audio in proper frames"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VzgK0xWw9g8X"
+      },
+      "outputs": [],
+      "source": [
+        "fixed_tm = frame_audio(wav_data_turdus)\n",
+        "fixed_tm.shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rU5-UqaCAVZ7"
+      },
+      "source": [
+        "Let's apply the model only on the first frame:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0zveWSOU9QBC"
+      },
+      "outputs": [],
+      "source": [
+        "logits, embeddings = model.infer_tf(fixed_tm[:1])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "osmRNWciEEuG"
+      },
+      "source": [
+        "The label.csv file contains ebirds ids.\n",
+        "The ebird id for Turdus Merula is eurbla"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E-UehjA6Acn_"
+      },
+      "outputs": [],
+      "source": [
+        "probabilities = tf.nn.softmax(logits)\n",
+        "argmax = np.argmax(probabilities)\n",
+        "print(f\"The audio is from the class {classes[argmax]} (element:{argmax} in the label.csv file), with probability of {probabilities[0][argmax]}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bGK84egXBg2f"
+      },
+      "source": [
+        "Lets apply the model on all the frames now:\n",
+        "\n",
+        "*note*: this code is also based on the [Chirp library](https://github.com/google-research/chirp/blob/d6ff5e7cee3865940f31697bf4b70176c1072572/chirp/inference/models.py#L174)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UT_Im9i50EGy"
+      },
+      "outputs": [],
+      "source": [
+        "all_logits, all_embeddings = model.infer_tf(fixed_tm[:1])\n",
+        "for window in fixed_tm[1:]:\n",
+        "  logits, embeddings = model.infer_tf(window[np.newaxis, :])\n",
+        "  all_logits = np.concatenate([all_logits, logits], axis=0)\n",
+        "\n",
+        "all_logits.shape"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kKuJWq4SxyR1"
+      },
+      "outputs": [],
+      "source": [
+        "frame = 0\n",
+        "for frame_logits in all_logits:\n",
+        "  probabilities = tf.nn.softmax(frame_logits)\n",
+        "  argmax = np.argmax(probabilities)\n",
+        "  print(f\"For frame {frame}, the audio is from the class {classes[argmax]} (element:{argmax} in the label.csv file), with probability of {probabilities[argmax]}\")\n",
+        "  frame += 1"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "name": "bird_vocalization_classifier.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/boundless.ipynb b/site/en/hub/tutorials/boundless.ipynb
new file mode 100644
index 00000000000..f53fc5bb004
--- /dev/null
+++ b/site/en/hub/tutorials/boundless.ipynb
@@ -0,0 +1,306 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9veUEV0CfmHX"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "BlCInyRifxHS"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2020 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_LRMeRxCfzC4"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QOjczJJ4gWHS"
+      },
+      "source": [
+        "# Boundless Colab\n",
+        "\n",
+        "Welcome to the Boundless model Colab! This notebook will take you through the steps of running the model on images and visualize the results.\n",
+        "\n",
+        "## Overview\n",
+        "\n",
+        "Boundless is a model for image extrapolation. This model takes an image, internally masks a portion of it ([1/2](https://tfhub.dev/google/boundless/half/1), [1/4](https://tfhub.dev/google/boundless/quarter/1), [3/4](https://tfhub.dev/google/boundless/three_quarter/1)) and completes the masked part. For more details refer to [Boundless: Generative Adversarial Networks for Image Extension](https://arxiv.org/pdf/1908.07007.pdf) or the model documentation on TensorFlow Hub."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hDKbpAEZf8Lt"
+      },
+      "source": [
+        "## Imports and setup\n",
+        "\n",
+        "Start with the base imports:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xJMFtTqPr7lf"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "from io import BytesIO\n",
+        "from PIL import Image as PilImage\n",
+        "import numpy as np\n",
+        "from matplotlib import pyplot as plt\n",
+        "from six.moves.urllib.request import urlopen"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pigUDIXtciQO"
+      },
+      "source": [
+        "## Create a function for reading an image\n",
+        "\n",
+        "Create a utility function to help load an image and format it for the model (257x257x3). This method will also crop the image to a square to avoid distortion and you can use it with local images or from the internet."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KTEVPgXH6rtV"
+      },
+      "outputs": [],
+      "source": [
+        "  def read_image(filename):\n",
+        "    fd = None\n",
+        "    if(filename.startswith('http')):\n",
+        "      fd = urlopen(filename)\n",
+        "    else:\n",
+        "      fd = tf.io.gfile.GFile(filename, 'rb')\n",
+        "\n",
+        "    pil_image = PilImage.open(fd)\n",
+        "    width, height = pil_image.size\n",
+        "    # crop to make the image square\n",
+        "    pil_image = pil_image.crop((0, 0, height, height))\n",
+        "    pil_image = pil_image.resize((257,257),PilImage.LANCZOS)\n",
+        "    image_unscaled = np.array(pil_image)\n",
+        "    image_np = np.expand_dims(\n",
+        "        image_unscaled.astype(np.float32) / 255., axis=0)\n",
+        "    return image_np"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lonrLxuKcsL0"
+      },
+      "source": [
+        "## Create a visualization function\n",
+        "\n",
+        "Create a visualization function to show the original image side-by-side with the masked version and the \"filled\" version, both generated by the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "j7AkoMFG7r-O"
+      },
+      "outputs": [],
+      "source": [
+        "def visualize_output_comparison(img_original, img_masked, img_filled):\n",
+        "  plt.figure(figsize=(24,12))\n",
+        "  plt.subplot(131)\n",
+        "  plt.imshow((np.squeeze(img_original)))\n",
+        "  plt.title(\"Original\", fontsize=24)\n",
+        "  plt.axis('off')\n",
+        "  plt.subplot(132)\n",
+        "  plt.imshow((np.squeeze(img_masked)))\n",
+        "  plt.title(\"Masked\", fontsize=24)\n",
+        "  plt.axis('off')\n",
+        "  plt.subplot(133)\n",
+        "  plt.imshow((np.squeeze(img_filled)))\n",
+        "  plt.title(\"Generated\", fontsize=24)\n",
+        "  plt.axis('off')\n",
+        "  plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8rwaCWmxdJGH"
+      },
+      "source": [
+        "## Load an image\n",
+        "\n",
+        "Now you can load a sample image. Feel free to use your own image by uploading it to the Colab notebook. Remember that the model may have some limitations regarding human images."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "92w-Jfbm60XA"
+      },
+      "outputs": [],
+      "source": [
+        "wikimedia = \"https://upload.wikimedia.org/wikipedia/commons/thumb/3/31/Nusfjord_road%2C_2010_09.jpg/800px-Nusfjord_road%2C_2010_09.jpg\"\n",
+        "# wikimedia = \"https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/Beech_forest_M%C3%A1tra_in_winter.jpg/640px-Beech_forest_M%C3%A1tra_in_winter.jpg\"\n",
+        "# wikimedia = \"https://upload.wikimedia.org/wikipedia/commons/thumb/b/b2/Marmolada_Sunset.jpg/640px-Marmolada_Sunset.jpg\"\n",
+        "# wikimedia = \"https://upload.wikimedia.org/wikipedia/commons/thumb/9/9d/Aegina_sunset.jpg/640px-Aegina_sunset.jpg\"\n",
+        "\n",
+        "input_img = read_image(wikimedia)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4lIkmZL_dtyX"
+      },
+      "source": [
+        "## Select a model from TensorFlow Hub\n",
+        "\n",
+        "On TensorFlow Hub there are three versions of the Boundless model: Half, Quarter and Three Quarters.\n",
+        "In the following cell you can choose any of the models and apply them on your image. If you want to pick another model, select it below and then run the following cells."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B3myNctEQ5GE"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Model Selection { display-mode: \"form\" }\n",
+        "model_name = 'Boundless Quarter' # @param ['Boundless Half', 'Boundless Quarter', 'Boundless Three Quarters']\n",
+        "model_handle_map = {\n",
+        "    'Boundless Half' : 'https://tfhub.dev/google/boundless/half/1',\n",
+        "    'Boundless Quarter' : 'https://tfhub.dev/google/boundless/quarter/1', \n",
+        "    'Boundless Three Quarters' : 'https://tfhub.dev/google/boundless/three_quarter/1'\n",
+        "}\n",
+        "\n",
+        "model_handle = model_handle_map[model_name]\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aSJFeNNSeOn8"
+      },
+      "source": [
+        "After choosing your model, you can load it from TensorFlow Hub.\n",
+        "\n",
+        "**Note**: You can point to a model handle to read the model's documentation."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0IDKMNyYSWsj"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Loading model {} ({})\".format(model_name, model_handle))\n",
+        "model = hub.load(model_handle)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L4G7CPOaeuQb"
+      },
+      "source": [
+        "## Perform inference\n",
+        "\n",
+        "The boundless model has two outputs:\n",
+        "\n",
+        "*   The input image with a mask applied\n",
+        "*   The masked image with the extrapolation to complete it\n",
+        "\n",
+        "You can compare these two images with a visualization as follows:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "W7uCAuKxSd-M"
+      },
+      "outputs": [],
+      "source": [
+        "result = model.signatures['default'](tf.constant(input_img))\n",
+        "generated_image =  result['default']\n",
+        "masked_image = result['masked_image']\n",
+        "\n",
+        "visualize_output_comparison(input_img, masked_image, generated_image)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "boundless.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/cord_19_embeddings.ipynb b/site/en/hub/tutorials/cord_19_embeddings.ipynb
new file mode 100644
index 00000000000..01f43e5f9a9
--- /dev/null
+++ b/site/en/hub/tutorials/cord_19_embeddings.ipynb
@@ -0,0 +1,537 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5wFF5JFyD2Ki"
+      },
+      "source": [
+        "#### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Uf6NouXxDqGk"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ORy-KvWXGXBo"
+      },
+      "source": [
+        "# Exploring the TF-Hub CORD-19 Swivel Embeddings\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9VusdTAH0isl"
+      },
+      "source": [
+        "The CORD-19 Swivel text embedding module from TF-Hub (https://tfhub.dev/tensorflow/cord-19/swivel-128d/1)\n",
+        " was built to support researchers analyzing natural languages text related to COVID-19.\n",
+        "These embeddings were trained on the titles, authors, abstracts, body texts, and\n",
+        "reference titles of articles in the [CORD-19 dataset](https://api.semanticscholar.org/CorpusID:216056360).\n",
+        "\n",
+        "In this colab we will:\n",
+        "- Analyze semantically similar words in the embedding space\n",
+        "- Train a classifier on the SciCite dataset using the CORD-19 embeddings\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L69VQv2Z0isl"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ym2nXOPuPV__"
+      },
+      "outputs": [],
+      "source": [
+        "import functools\n",
+        "import itertools\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import seaborn as sns\n",
+        "import pandas as pd\n",
+        "\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "tf.disable_eager_execution()\n",
+        "tf.logging.set_verbosity('ERROR')\n",
+        "\n",
+        "import tensorflow_datasets as tfds\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "try:\n",
+        "  from google.colab import data_table\n",
+        "  def display_df(df):\n",
+        "    return data_table.DataTable(df, include_index=False)\n",
+        "except ModuleNotFoundError:\n",
+        "  # If google-colab is not available, just display the raw DataFrame\n",
+        "  def display_df(df):\n",
+        "    return df"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_VgRRf2I7tER"
+      },
+      "source": [
+        "# Analyze the embeddings\n",
+        "\n",
+        "Let's start off by analyzing the embedding by calculating and plotting a correlation matrix between different terms. If the embedding learned to successfully capture the meaning of different words, the embedding vectors of semantically similar words should be close together. Let's take a look at some COVID-19 related terms."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HNN_9bBKSLHU"
+      },
+      "outputs": [],
+      "source": [
+        "# Use the inner product between two embedding vectors as the similarity measure\n",
+        "def plot_correlation(labels, features):\n",
+        "  corr = np.inner(features, features)\n",
+        "  corr /= np.max(corr)\n",
+        "  sns.heatmap(corr, xticklabels=labels, yticklabels=labels)\n",
+        "\n",
+        "\n",
+        "with tf.Graph().as_default():\n",
+        "  # Load the module\n",
+        "  query_input = tf.placeholder(tf.string)\n",
+        "  module = hub.Module('https://tfhub.dev/tensorflow/cord-19/swivel-128d/1')\n",
+        "  embeddings = module(query_input)\n",
+        "\n",
+        "  with tf.train.MonitoredTrainingSession() as sess:\n",
+        "\n",
+        "    # Generate embeddings for some terms\n",
+        "    queries = [\n",
+        "        # Related viruses\n",
+        "        \"coronavirus\", \"SARS\", \"MERS\",\n",
+        "        # Regions\n",
+        "        \"Italy\", \"Spain\", \"Europe\",\n",
+        "        # Symptoms\n",
+        "        \"cough\", \"fever\", \"throat\"\n",
+        "    ]\n",
+        "\n",
+        "    features = sess.run(embeddings, feed_dict={query_input: queries})\n",
+        "    plot_correlation(queries, features)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Bg-PGqtm8B7K"
+      },
+      "source": [
+        "We can see that the embedding successfully captured the meaning of the different terms. Each word is similar to the other words of its cluster (i.e. \"coronavirus\" highly correlates with \"SARS\" and \"MERS\"), while they are different from terms of other clusters (i.e. the similarity between \"SARS\" and \"Spain\" is close to 0).\n",
+        "\n",
+        "Now let's see how we can use these embeddings to solve a specific task."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "idJ1jFmH7xMa"
+      },
+      "source": [
+        "## SciCite: Citation Intent Classification\n",
+        "\n",
+        "This section shows how one can use the embedding for downstream tasks such as text classification. We'll use the [SciCite dataset](https://www.tensorflow.org/datasets/catalog/scicite) from TensorFlow Datasets to classify citation intents in academic papers. Given a sentence with a citation from an academic paper, classify whether the main intent of the citation is as background information, use of methods, or comparing results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "-FB19HLfVp2V"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Set up the dataset from TFDS\n",
+        "\n",
+        "class Dataset:\n",
+        "  \"\"\"Build a dataset from a TFDS dataset.\"\"\"\n",
+        "  def __init__(self, tfds_name, feature_name, label_name):\n",
+        "    self.dataset_builder = tfds.builder(tfds_name)\n",
+        "    self.dataset_builder.download_and_prepare()\n",
+        "    self.feature_name = feature_name\n",
+        "    self.label_name = label_name\n",
+        "  \n",
+        "  def get_data(self, for_eval):\n",
+        "    splits = THE_DATASET.dataset_builder.info.splits\n",
+        "    if tfds.Split.TEST in splits:\n",
+        "      split = tfds.Split.TEST if for_eval else tfds.Split.TRAIN\n",
+        "    else:\n",
+        "      SPLIT_PERCENT = 80\n",
+        "      split = \"train[{}%:]\".format(SPLIT_PERCENT) if for_eval else \"train[:{}%]\".format(SPLIT_PERCENT)\n",
+        "    return self.dataset_builder.as_dataset(split=split)\n",
+        "\n",
+        "  def num_classes(self):\n",
+        "    return self.dataset_builder.info.features[self.label_name].num_classes\n",
+        "\n",
+        "  def class_names(self):\n",
+        "    return self.dataset_builder.info.features[self.label_name].names\n",
+        "\n",
+        "  def preprocess_fn(self, data):\n",
+        "    return data[self.feature_name], data[self.label_name]\n",
+        "\n",
+        "  def example_fn(self, data):\n",
+        "    feature, label = self.preprocess_fn(data)\n",
+        "    return {'feature': feature, 'label': label}, label\n",
+        "\n",
+        "\n",
+        "def get_example_data(dataset, num_examples, **data_kw):\n",
+        "  \"\"\"Show example data\"\"\"\n",
+        "  with tf.Session() as sess:\n",
+        "    batched_ds = dataset.get_data(**data_kw).take(num_examples).map(dataset.preprocess_fn).batch(num_examples)\n",
+        "    it = tf.data.make_one_shot_iterator(batched_ds).get_next()\n",
+        "    data = sess.run(it)\n",
+        "  return data\n",
+        "\n",
+        "\n",
+        "TFDS_NAME = 'scicite' #@param {type: \"string\"}\n",
+        "TEXT_FEATURE_NAME = 'string' #@param {type: \"string\"}\n",
+        "LABEL_NAME = 'label' #@param {type: \"string\"}\n",
+        "THE_DATASET = Dataset(TFDS_NAME, TEXT_FEATURE_NAME, LABEL_NAME)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "CVjyBD0ZPh4Z"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Let's take a look at a few labeled examples from the training set\n",
+        "NUM_EXAMPLES = 20  #@param {type:\"integer\"}\n",
+        "data = get_example_data(THE_DATASET, NUM_EXAMPLES, for_eval=False)\n",
+        "display_df(\n",
+        "    pd.DataFrame({\n",
+        "        TEXT_FEATURE_NAME: [ex.decode('utf8') for ex in data[0]],\n",
+        "        LABEL_NAME: [THE_DATASET.class_names()[x] for x in data[1]]\n",
+        "    }))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "65s9UpYJ_1ct"
+      },
+      "source": [
+        "## Training a citaton intent classifier\n",
+        "\n",
+        "We'll train a classifier on the [SciCite dataset](https://www.tensorflow.org/datasets/catalog/scicite) using an Estimator. Let's set up the input_fns to read the dataset into the model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "OldapWmKSGsW"
+      },
+      "outputs": [],
+      "source": [
+        "def preprocessed_input_fn(for_eval):\n",
+        "  data = THE_DATASET.get_data(for_eval=for_eval)\n",
+        "  data = data.map(THE_DATASET.example_fn, num_parallel_calls=1)\n",
+        "  return data\n",
+        "\n",
+        "\n",
+        "def input_fn_train(params):\n",
+        "  data = preprocessed_input_fn(for_eval=False)\n",
+        "  data = data.repeat(None)\n",
+        "  data = data.shuffle(1024)\n",
+        "  data = data.batch(batch_size=params['batch_size'])\n",
+        "  return data\n",
+        "\n",
+        "\n",
+        "def input_fn_eval(params):\n",
+        "  data = preprocessed_input_fn(for_eval=True)\n",
+        "  data = data.repeat(1)\n",
+        "  data = data.batch(batch_size=params['batch_size'])\n",
+        "  return data\n",
+        "\n",
+        "\n",
+        "def input_fn_predict(params):\n",
+        "  data = preprocessed_input_fn(for_eval=True)\n",
+        "  data = data.batch(batch_size=params['batch_size'])\n",
+        "  return data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KcrmWUkVKg2u"
+      },
+      "source": [
+        "Let's build a model which use the CORD-19 embeddings with a classification layer on top."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ff0uKqJCA9zh"
+      },
+      "outputs": [],
+      "source": [
+        "def model_fn(features, labels, mode, params):\n",
+        "  # Embed the text\n",
+        "  embed = hub.Module(params['module_name'], trainable=params['trainable_module'])\n",
+        "  embeddings = embed(features['feature'])\n",
+        "\n",
+        "  # Add a linear layer on top\n",
+        "  logits = tf.layers.dense(\n",
+        "      embeddings, units=THE_DATASET.num_classes(), activation=None)\n",
+        "  predictions = tf.argmax(input=logits, axis=1)\n",
+        "\n",
+        "  if mode == tf.estimator.ModeKeys.PREDICT:\n",
+        "    return tf.estimator.EstimatorSpec(\n",
+        "        mode=mode,\n",
+        "        predictions={\n",
+        "            'logits': logits,\n",
+        "            'predictions': predictions,\n",
+        "            'features': features['feature'],\n",
+        "            'labels': features['label']\n",
+        "        })\n",
+        "  \n",
+        "  # Set up a multi-class classification head\n",
+        "  loss = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
+        "      labels=labels, logits=logits)\n",
+        "  loss = tf.reduce_mean(loss)\n",
+        "\n",
+        "  if mode == tf.estimator.ModeKeys.TRAIN:\n",
+        "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=params['learning_rate'])\n",
+        "    train_op = optimizer.minimize(loss, global_step=tf.train.get_or_create_global_step())\n",
+        "    return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n",
+        "\n",
+        "  elif mode == tf.estimator.ModeKeys.EVAL:\n",
+        "    accuracy = tf.metrics.accuracy(labels=labels, predictions=predictions)\n",
+        "    precision = tf.metrics.precision(labels=labels, predictions=predictions)\n",
+        "    recall = tf.metrics.recall(labels=labels, predictions=predictions)\n",
+        "\n",
+        "    return tf.estimator.EstimatorSpec(\n",
+        "        mode=mode,\n",
+        "        loss=loss,\n",
+        "        eval_metric_ops={\n",
+        "            'accuracy': accuracy,\n",
+        "            'precision': precision,\n",
+        "            'recall': recall,\n",
+        "        })\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "yZUclu8xBYlj"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Hyperparmeters { run: \"auto\" }\n",
+        "\n",
+        "EMBEDDING = 'https://tfhub.dev/tensorflow/cord-19/swivel-128d/1'  #@param {type: \"string\"}\n",
+        "TRAINABLE_MODULE = False  #@param {type: \"boolean\"}\n",
+        "STEPS =   8000#@param {type: \"integer\"}\n",
+        "EVAL_EVERY = 200  #@param {type: \"integer\"}\n",
+        "BATCH_SIZE = 10  #@param {type: \"integer\"}\n",
+        "LEARNING_RATE = 0.01  #@param {type: \"number\"}\n",
+        "\n",
+        "params = {\n",
+        "    'batch_size': BATCH_SIZE,\n",
+        "    'learning_rate': LEARNING_RATE,\n",
+        "    'module_name': EMBEDDING,\n",
+        "    'trainable_module': TRAINABLE_MODULE\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "weZKWK-pLBll"
+      },
+      "source": [
+        "## Train and evaluate the model\n",
+        "\n",
+        "Let's train and evaluate the model to see the performance on the SciCite task"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cO1FWkZW2WS9"
+      },
+      "outputs": [],
+      "source": [
+        "estimator = tf.estimator.Estimator(functools.partial(model_fn, params=params))\n",
+        "metrics = []\n",
+        "\n",
+        "for step in range(0, STEPS, EVAL_EVERY):\n",
+        "  estimator.train(input_fn=functools.partial(input_fn_train, params=params), steps=EVAL_EVERY)\n",
+        "  step_metrics = estimator.evaluate(input_fn=functools.partial(input_fn_eval, params=params))\n",
+        "  print('Global step {}: loss {:.3f}, accuracy {:.3f}'.format(step, step_metrics['loss'], step_metrics['accuracy']))\n",
+        "  metrics.append(step_metrics)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RUNGAeyf1ygC"
+      },
+      "outputs": [],
+      "source": [
+        "global_steps = [x['global_step'] for x in metrics]\n",
+        "fig, axes = plt.subplots(ncols=2, figsize=(20,8))\n",
+        "\n",
+        "for axes_index, metric_names in enumerate([['accuracy', 'precision', 'recall'],\n",
+        "                                            ['loss']]):\n",
+        "  for metric_name in metric_names:\n",
+        "    axes[axes_index].plot(global_steps, [x[metric_name] for x in metrics], label=metric_name)\n",
+        "  axes[axes_index].legend()\n",
+        "  axes[axes_index].set_xlabel(\"Global Step\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1biWylvB6ayg"
+      },
+      "source": [
+        "We can see that the loss quickly decreases while especially the accuracy rapidly increases. Let's plot some examples to check how the prediction relates to the true labels:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zK_NJXtoyG2o"
+      },
+      "outputs": [],
+      "source": [
+        "predictions = estimator.predict(functools.partial(input_fn_predict, params))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nlxFER_Oriam"
+      },
+      "outputs": [],
+      "source": [
+        "first_10_predictions = list(itertools.islice(predictions, 10))\n",
+        "\n",
+        "display_df(\n",
+        "  pd.DataFrame({\n",
+        "      TEXT_FEATURE_NAME: [pred['features'].decode('utf8') for pred in first_10_predictions],\n",
+        "      LABEL_NAME: [THE_DATASET.class_names()[pred['labels']] for pred in first_10_predictions],\n",
+        "      'prediction': [THE_DATASET.class_names()[pred['predictions']] for pred in first_10_predictions]\n",
+        "  }))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OSGcrkE069_Q"
+      },
+      "source": [
+        "We can see that for this random sample, the model predicts the correct label most of the times, indicating that it can embed scientific sentences pretty well."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oLE0kCfO5CIA"
+      },
+      "source": [
+        "# What's next?\n",
+        "\n",
+        "Now that you've gotten to know a bit more about the CORD-19 Swivel embeddings from TF-Hub, we encourage you to participate in the CORD-19 Kaggle competition to contribute to gaining scientific insights from COVID-19 related academic texts.\n",
+        "\n",
+        "* Participate in the [CORD-19 Kaggle Challenge](https://www.kaggle.com/allen-institute-for-ai/CORD-19-research-challenge)\n",
+        "* Learn more about the [COVID-19 Open Research Dataset (CORD-19)](https://api.semanticscholar.org/CorpusID:216056360)\n",
+        "* See documentation and more about the TF-Hub embeddings at https://tfhub.dev/tensorflow/cord-19/swivel-128d/1\n",
+        "* Explore the CORD-19 embedding space with the [TensorFlow Embedding Projector](http://projector.tensorflow.org/?config=https://storage.googleapis.com/tfhub-examples/tensorflow/cord-19/swivel-128d/1/tensorboard/full_projector_config.json)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "5wFF5JFyD2Ki"
+      ],
+      "name": "cord_19_embeddings.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/cord_19_embeddings_keras.ipynb b/site/en/hub/tutorials/cord_19_embeddings_keras.ipynb
new file mode 100644
index 00000000000..388de741e34
--- /dev/null
+++ b/site/en/hub/tutorials/cord_19_embeddings_keras.ipynb
@@ -0,0 +1,421 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5wFF5JFyD2Ki"
+      },
+      "source": [
+        "#### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Uf6NouXxDqGk"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ORy-KvWXGXBo"
+      },
+      "source": [
+        "# Exploring the TF-Hub CORD-19 Swivel Embeddings\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yI6Mh3-P0_Pk"
+      },
+      "source": [
+        "The CORD-19 Swivel text embedding module from TF-Hub (https://tfhub.dev/tensorflow/cord-19/swivel-128d/3)\n",
+        " was built to support researchers analyzing natural languages text related to COVID-19.\n",
+        "These embeddings were trained on the titles, authors, abstracts, body texts, and\n",
+        "reference titles of articles in the [CORD-19 dataset](https://api.semanticscholar.org/CorpusID:216056360).\n",
+        "\n",
+        "In this colab we will:\n",
+        "- Analyze semantically similar words in the embedding space\n",
+        "- Train a classifier on the SciCite dataset using the CORD-19 embeddings\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gVWOrccw0_Pl"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ym2nXOPuPV__"
+      },
+      "outputs": [],
+      "source": [
+        "import functools\n",
+        "import itertools\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import seaborn as sns\n",
+        "import pandas as pd\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "import tensorflow_datasets as tfds\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "from tqdm import trange"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_VgRRf2I7tER"
+      },
+      "source": [
+        "# Analyze the embeddings\n",
+        "\n",
+        "Let's start off by analyzing the embedding by calculating and plotting a correlation matrix between different terms. If the embedding learned to successfully capture the meaning of different words, the embedding vectors of semantically similar words should be close together. Let's take a look at some COVID-19 related terms."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HNN_9bBKSLHU"
+      },
+      "outputs": [],
+      "source": [
+        "# Use the inner product between two embedding vectors as the similarity measure\n",
+        "def plot_correlation(labels, features):\n",
+        "  corr = np.inner(features, features)\n",
+        "  corr /= np.max(corr)\n",
+        "  sns.heatmap(corr, xticklabels=labels, yticklabels=labels)\n",
+        "\n",
+        "# Generate embeddings for some terms\n",
+        "queries = [\n",
+        "  # Related viruses\n",
+        "  'coronavirus', 'SARS', 'MERS',\n",
+        "  # Regions\n",
+        "  'Italy', 'Spain', 'Europe',\n",
+        "  # Symptoms\n",
+        "  'cough', 'fever', 'throat'\n",
+        "]\n",
+        "\n",
+        "module = hub.load('https://tfhub.dev/tensorflow/cord-19/swivel-128d/3')\n",
+        "embeddings = module(queries)\n",
+        "\n",
+        "plot_correlation(queries, embeddings)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Bg-PGqtm8B7K"
+      },
+      "source": [
+        "We can see that the embedding successfully captured the meaning of the different terms. Each word is similar to the other words of its cluster (i.e. \"coronavirus\" highly correlates with \"SARS\" and \"MERS\"), while they are different from terms of other clusters (i.e. the similarity between \"SARS\" and \"Spain\" is close to 0).\n",
+        "\n",
+        "Now let's see how we can use these embeddings to solve a specific task."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "idJ1jFmH7xMa"
+      },
+      "source": [
+        "## SciCite: Citation Intent Classification\n",
+        "\n",
+        "This section shows how one can use the embedding for downstream tasks such as text classification. We'll use the [SciCite dataset](https://www.tensorflow.org/datasets/catalog/scicite) from TensorFlow Datasets to classify citation intents in academic papers. Given a sentence with a citation from an academic paper, classify whether the main intent of the citation is as background information, use of methods, or comparing results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ghc-CzT8DDaZ"
+      },
+      "outputs": [],
+      "source": [
+        "builder = tfds.builder(name='scicite')\n",
+        "builder.download_and_prepare()\n",
+        "train_data, validation_data, test_data = builder.as_dataset(\n",
+        "    split=('train', 'validation', 'test'),\n",
+        "    as_supervised=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CVjyBD0ZPh4Z"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Let's take a look at a few labeled examples from the training set\n",
+        "NUM_EXAMPLES =   10#@param {type:\"integer\"}\n",
+        "\n",
+        "TEXT_FEATURE_NAME = builder.info.supervised_keys[0]\n",
+        "LABEL_NAME = builder.info.supervised_keys[1]\n",
+        "\n",
+        "def label2str(numeric_label):\n",
+        "  m = builder.info.features[LABEL_NAME].names\n",
+        "  return m[numeric_label]\n",
+        "\n",
+        "data = next(iter(train_data.batch(NUM_EXAMPLES)))\n",
+        "\n",
+        "\n",
+        "pd.DataFrame({\n",
+        "    TEXT_FEATURE_NAME: [ex.numpy().decode('utf8') for ex in data[0]],\n",
+        "    LABEL_NAME: [label2str(x) for x in data[1]]\n",
+        "})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "65s9UpYJ_1ct"
+      },
+      "source": [
+        "## Training a citaton intent classifier\n",
+        "\n",
+        "We'll train a classifier on the [SciCite dataset](https://www.tensorflow.org/datasets/catalog/scicite) using Keras.  Let's build a model which use the CORD-19 embeddings with a classification layer on top."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yZUclu8xBYlj"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Hyperparameters { run: \"auto\" }\n",
+        "\n",
+        "EMBEDDING = 'https://tfhub.dev/tensorflow/cord-19/swivel-128d/3'  #@param {type: \"string\"}\n",
+        "TRAINABLE_MODULE = False  #@param {type: \"boolean\"}\n",
+        "\n",
+        "hub_layer = hub.KerasLayer(EMBEDDING, input_shape=[], \n",
+        "                           dtype=tf.string, trainable=TRAINABLE_MODULE)\n",
+        "\n",
+        "model = tf.keras.Sequential()\n",
+        "model.add(hub_layer)\n",
+        "model.add(tf.keras.layers.Dense(3))\n",
+        "model.summary()\n",
+        "model.compile(optimizer='adam',\n",
+        "              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+        "              metrics=['accuracy'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "weZKWK-pLBll"
+      },
+      "source": [
+        "## Train and evaluate the model\n",
+        "\n",
+        "Let's train and evaluate the model to see the performance on the SciCite task"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cO1FWkZW2WS9"
+      },
+      "outputs": [],
+      "source": [
+        "EPOCHS = 35#@param {type: \"integer\"}\n",
+        "BATCH_SIZE = 32#@param {type: \"integer\"}\n",
+        "\n",
+        "history = model.fit(train_data.shuffle(10000).batch(BATCH_SIZE),\n",
+        "                    epochs=EPOCHS,\n",
+        "                    validation_data=validation_data.batch(BATCH_SIZE),\n",
+        "                    verbose=1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2sKE7kEyLJQZ"
+      },
+      "outputs": [],
+      "source": [
+        "from matplotlib import pyplot as plt\n",
+        "def display_training_curves(training, validation, title, subplot):\n",
+        "  if subplot%10==1: # set up the subplots on the first call\n",
+        "    plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n",
+        "    plt.tight_layout()\n",
+        "  ax = plt.subplot(subplot)\n",
+        "  ax.set_facecolor('#F8F8F8')\n",
+        "  ax.plot(training)\n",
+        "  ax.plot(validation)\n",
+        "  ax.set_title('model '+ title)\n",
+        "  ax.set_ylabel(title)\n",
+        "  ax.set_xlabel('epoch')\n",
+        "  ax.legend(['train', 'valid.'])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nnQfxevhLKld"
+      },
+      "outputs": [],
+      "source": [
+        "display_training_curves(history.history['accuracy'], history.history['val_accuracy'], 'accuracy', 211)\n",
+        "display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 212)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BjvtOw72Lpyw"
+      },
+      "source": [
+        "## Evaluate the model\n",
+        "\n",
+        "And let's see how the model performs. Two values will be returned. Loss (a number which represents our error, lower values are better), and accuracy."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "y0ExC8D0LX8m"
+      },
+      "outputs": [],
+      "source": [
+        "results = model.evaluate(test_data.batch(512), verbose=2)\n",
+        "\n",
+        "for name, value in zip(model.metrics_names, results):\n",
+        "  print('%s: %.3f' % (name, value))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dWp5OWeTL2EW"
+      },
+      "source": [
+        "We can see that the loss quickly decreases while especially the accuracy rapidly increases. Let's plot some examples to check how the prediction relates to the true labels:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VzHzAOaaOVC0"
+      },
+      "outputs": [],
+      "source": [
+        "prediction_dataset = next(iter(test_data.batch(20)))\n",
+        "\n",
+        "prediction_texts = [ex.numpy().decode('utf8') for ex in prediction_dataset[0]]\n",
+        "prediction_labels = [label2str(x) for x in prediction_dataset[1]]\n",
+        "\n",
+        "predictions = [\n",
+        "    label2str(x) for x in np.argmax(model.predict(prediction_texts), axis=-1)]\n",
+        "\n",
+        "\n",
+        "pd.DataFrame({\n",
+        "    TEXT_FEATURE_NAME: prediction_texts,\n",
+        "    LABEL_NAME: prediction_labels,\n",
+        "    'prediction': predictions\n",
+        "})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OSGcrkE069_Q"
+      },
+      "source": [
+        "We can see that for this random sample, the model predicts the correct label most of the times, indicating that it can embed scientific sentences pretty well."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oLE0kCfO5CIA"
+      },
+      "source": [
+        "# What's next?\n",
+        "\n",
+        "Now that you've gotten to know a bit more about the CORD-19 Swivel embeddings from TF-Hub, we encourage you to participate in the CORD-19 Kaggle competition to contribute to gaining scientific insights from COVID-19 related academic texts.\n",
+        "\n",
+        "* Participate in the [CORD-19 Kaggle Challenge](https://www.kaggle.com/allen-institute-for-ai/CORD-19-research-challenge)\n",
+        "* Learn more about the [COVID-19 Open Research Dataset (CORD-19)](https://api.semanticscholar.org/CorpusID:216056360)\n",
+        "* See documentation and more about the TF-Hub embeddings at https://tfhub.dev/tensorflow/cord-19/swivel-128d/3\n",
+        "* Explore the CORD-19 embedding space with the [TensorFlow Embedding Projector](http://projector.tensorflow.org/?config=https://storage.googleapis.com/tfhub-examples/tensorflow/cord-19/swivel-128d/3/tensorboard/projector_config.json)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "cord_19_embeddings_keras.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/cropnet_cassava.ipynb b/site/en/hub/tutorials/cropnet_cassava.ipynb
new file mode 100644
index 00000000000..926b5395e41
--- /dev/null
+++ b/site/en/hub/tutorials/cropnet_cassava.ipynb
@@ -0,0 +1,413 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vtNtfcHHoHNP"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jZwnHZ70oUIM"
+      },
+      "source": [
+        "# CropNet: Cassava Disease Detection"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6sg9wHP9oR3q"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "grEgSWu2iTxm"
+      },
+      "source": [
+        "This notebook shows how to use the CropNet [cassava disease classifier](https://tfhub.dev/google/cropnet/classifier/cassava_disease_V1/2) model from **TensorFlow Hub**. The model classifies images of cassava leaves into one of 6 classes: *bacterial blight, brown streak disease, green mite, mosaic disease, healthy, or unknown*.\n",
+        "\n",
+        "This colab demonstrates how to:\n",
+        " * Load the https://tfhub.dev/google/cropnet/classifier/cassava_disease_V1/2 model from **TensorFlow Hub**\n",
+        " * Load the [cassava](https://www.tensorflow.org/datasets/catalog/cassava) dataset from **TensorFlow Datasets (TFDS)**\n",
+        " * Classify images of cassava leaves into 4 distinct cassava disease categories or as healthy or unknown.\n",
+        " * Evaluate the *accuracy* of the classifier and look at how *robust* the model is  when applied to out of domain images."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bKn4Fiq2OD7u"
+      },
+      "source": [
+        "## Imports and setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LMgjpSoYqJIz"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install matplotlib==3.2.2"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "FIP4rkjp45MG"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "import matplotlib.pyplot as plt\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_datasets as tfds\n",
+        "import tensorflow_hub as hub"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "mIqmq_qmWw78"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Helper function for displaying examples\n",
+        "def plot(examples, predictions=None):\n",
+        "  # Get the images, labels, and optionally predictions\n",
+        "  images = examples['image']\n",
+        "  labels = examples['label']\n",
+        "  batch_size = len(images)\n",
+        "  if predictions is None:\n",
+        "    predictions = batch_size * [None]\n",
+        "\n",
+        "  # Configure the layout of the grid\n",
+        "  x = np.ceil(np.sqrt(batch_size))\n",
+        "  y = np.ceil(batch_size / x)\n",
+        "  fig = plt.figure(figsize=(x * 6, y * 7))\n",
+        "\n",
+        "  for i, (image, label, prediction) in enumerate(zip(images, labels, predictions)):\n",
+        "    # Render the image\n",
+        "    ax = fig.add_subplot(x, y, i+1)\n",
+        "    ax.imshow(image, aspect='auto')\n",
+        "    ax.grid(False)\n",
+        "    ax.set_xticks([])\n",
+        "    ax.set_yticks([])\n",
+        "\n",
+        "    # Display the label and optionally prediction\n",
+        "    x_label = 'Label: ' + name_map[class_names[label]]\n",
+        "    if prediction is not None:\n",
+        "      x_label = 'Prediction: ' + name_map[class_names[prediction]] + '\\n' + x_label\n",
+        "      ax.xaxis.label.set_color('green' if label == prediction else 'red')\n",
+        "    ax.set_xlabel(x_label)\n",
+        "\n",
+        "  plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kwrg9yIlaUSb"
+      },
+      "source": [
+        "## Dataset\n",
+        "\n",
+        "Let's load the *cassava* dataset from TFDS"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0rTcnxoSkp31"
+      },
+      "outputs": [],
+      "source": [
+        "dataset, info = tfds.load('cassava', with_info=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GpC71TFDhJFO"
+      },
+      "source": [
+        "Let's take a look at the dataset info to learn more about it, like the description and citation and information about how many examples are available"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "btJBMovmbYtR"
+      },
+      "outputs": [],
+      "source": [
+        "info"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QT3XWAtR6BRy"
+      },
+      "source": [
+        "The *cassava* dataset has images of cassava leaves with 4 distinct diseases as well as healthy cassava leaves. The model can predict all of these classes as well as sixth class for \"unknown\" when the model is not confident in its prediction."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9NT9q8yyXZfX"
+      },
+      "outputs": [],
+      "source": [
+        "# Extend the cassava dataset classes with 'unknown'\n",
+        "class_names = info.features['label'].names + ['unknown']\n",
+        "\n",
+        "# Map the class names to human readable names\n",
+        "name_map = dict(\n",
+        "    cmd='Mosaic Disease',\n",
+        "    cbb='Bacterial Blight',\n",
+        "    cgm='Green Mite',\n",
+        "    cbsd='Brown Streak Disease',\n",
+        "    healthy='Healthy',\n",
+        "    unknown='Unknown')\n",
+        "\n",
+        "print(len(class_names), 'classes:')\n",
+        "print(class_names)\n",
+        "print([name_map[name] for name in class_names])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "I6y_MGxgiW09"
+      },
+      "source": [
+        "Before we can feed the data to the model, we need to do a bit of preprocessing. The model expects 224 x 224 images with RGB channel values in [0, 1]. Let's normalize and resize the images."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UxtxvqRjh7Nm"
+      },
+      "outputs": [],
+      "source": [
+        "def preprocess_fn(data):\n",
+        "  image = data['image']\n",
+        "\n",
+        "  # Normalize [0, 255] to [0, 1]\n",
+        "  image = tf.cast(image, tf.float32)\n",
+        "  image = image / 255.\n",
+        "\n",
+        "  # Resize the images to 224 x 224\n",
+        "  image = tf.image.resize(image, (224, 224))\n",
+        "\n",
+        "  data['image'] = image\n",
+        "  return data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qz27YrZahdvn"
+      },
+      "source": [
+        "Let's take a look at a few examples from the dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "j6LkAxv3f-aJ"
+      },
+      "outputs": [],
+      "source": [
+        "batch = dataset['validation'].map(preprocess_fn).batch(25).as_numpy_iterator()\n",
+        "examples = next(batch)\n",
+        "plot(examples)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eHlEAhL3hq2R"
+      },
+      "source": [
+        "## Model\n",
+        "\n",
+        "Let's load the classifier from TF Hub and get some predictions and see the predictions of the model is on a few examples"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b6eIWkTjIQhS"
+      },
+      "outputs": [],
+      "source": [
+        "classifier = hub.KerasLayer('https://tfhub.dev/google/cropnet/classifier/cassava_disease_V1/2')\n",
+        "probabilities = classifier(examples['image'])\n",
+        "predictions = tf.argmax(probabilities, axis=-1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MTQA1YAltfRZ"
+      },
+      "outputs": [],
+      "source": [
+        "plot(examples, predictions)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MuFE8A5aZv9z"
+      },
+      "source": [
+        "## Evaluation & robustness\n",
+        "\n",
+        "Let's measure the *accuracy* of our classifier on a split of the dataset. We can also look at the *robustness* of the model by evaluating its performance on a non-cassava dataset. For image of other plant datasets like iNaturalist or beans, the model should almost always return *unknown*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "0ERcNxs0kHd3"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Parameters {run: \"auto\"}\n",
+        "\n",
+        "DATASET = 'cassava'  #@param {type:\"string\"} ['cassava', 'beans', 'i_naturalist2017']\n",
+        "DATASET_SPLIT = 'test' #@param {type:\"string\"} ['train', 'test', 'validation']\n",
+        "BATCH_SIZE =  32 #@param {type:\"integer\"}\n",
+        "MAX_EXAMPLES = 1000 #@param {type:\"integer\"}\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Mt0-IVmZplbb"
+      },
+      "outputs": [],
+      "source": [
+        "def label_to_unknown_fn(data):\n",
+        "  data['label'] = 5  # Override label to unknown.\n",
+        "  return data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cQYvY3IvY2Nx"
+      },
+      "outputs": [],
+      "source": [
+        "# Preprocess the examples and map the image label to unknown for non-cassava datasets.\n",
+        "ds = tfds.load(DATASET, split=DATASET_SPLIT).map(preprocess_fn).take(MAX_EXAMPLES)\n",
+        "dataset_description = DATASET\n",
+        "if DATASET != 'cassava':\n",
+        "  ds = ds.map(label_to_unknown_fn)\n",
+        "  dataset_description += ' (labels mapped to unknown)'\n",
+        "ds = ds.batch(BATCH_SIZE)\n",
+        "\n",
+        "# Calculate the accuracy of the model\n",
+        "metric = tf.keras.metrics.Accuracy()\n",
+        "for examples in ds:\n",
+        "  probabilities = classifier(examples['image'])\n",
+        "  predictions = tf.math.argmax(probabilities, axis=-1)\n",
+        "  labels = examples['label']\n",
+        "  metric.update_state(labels, predictions)\n",
+        "\n",
+        "print('Accuracy on %s: %.2f' % (dataset_description, metric.result().numpy()))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rvS18sBExpdL"
+      },
+      "source": [
+        "## Learn more\n",
+        "\n",
+        "* Learn more about the model on TensorFlow Hub: https://tfhub.dev/google/cropnet/classifier/cassava_disease_V1/2\n",
+        "* Learn how to build a custom image classifier running on a mobile phone with [ML Kit](https://developers.google.com/ml-kit/custom-models#tfhub) with the [TensorFlow Lite version of this model](https://tfhub.dev/google/lite-model/cropnet/classifier/cassava_disease_V1/1)."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "cropnet_cassava.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/cropnet_on_device.ipynb b/site/en/hub/tutorials/cropnet_on_device.ipynb
new file mode 100644
index 00000000000..0e1cb1e0b0d
--- /dev/null
+++ b/site/en/hub/tutorials/cropnet_on_device.ipynb
@@ -0,0 +1,724 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3XX46cTrh6iD"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Hub Authors. \n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sKrlWr6Kh-mF"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DMVmlJ0fAMkH"
+      },
+      "source": [
+        "# Fine tuning models for plant disease detection\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hk5u_9KN1m-t"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OEHq-hV5sWYO"
+      },
+      "source": [
+        "This notebook shows you how to **fine-tune CropNet models from TensorFlow Hub** on a dataset from TFDS or your own crop disease detection dataset.\n",
+        "\n",
+        "You will:\n",
+        "- Load the TFDS cassava dataset or your own data\n",
+        "- Enrich the data with unknown (negative) examples to get a more robust model\n",
+        "- Apply image augmentations to the data\n",
+        "- Load and fine tune a [CropNet model](https://tfhub.dev/s?module-type=image-feature-vector&q=cropnet) from TF Hub\n",
+        "- Export a TFLite model, ready to be deployed on your app with [Task Library](https://www.tensorflow.org/lite/inference_with_metadata/task_library/image_classifier), [MLKit](https://developers.google.com/ml-kit/vision/image-labeling/custom-models/android) or [TFLite](https://www.tensorflow.org/lite/guide/inference) directly"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dQvS4p807mZf"
+      },
+      "source": [
+        "## Imports and Dependencies\n",
+        "\n",
+        "Before starting, you'll need to install some of the dependencies that will be needed like [Model Maker](https://www.tensorflow.org/lite/guide/model_maker#installation) and the latest version of TensorFlow Datasets."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5BDTEMtexXE3"
+      },
+      "outputs": [],
+      "source": [
+        "!sudo apt install -q libportaudio2\n",
+        "## image_classifier library requires numpy <= 1.23.5\n",
+        "!pip install \"numpy<=1.23.5\"\n",
+        "!pip install --use-deprecated=legacy-resolver tflite-model-maker-nightly\n",
+        "!pip install -U tensorflow-datasets\n",
+        "## scann library requires tensorflow < 2.9.0\n",
+        "!pip install \"tensorflow<2.9.0\"\n",
+        "!pip install \"tensorflow-datasets~=4.8.0\"  # protobuf>=3.12.2\n",
+        "!pip install tensorflow-metadata~=1.10.0  # protobuf>=3.13\n",
+        "## tensorflowjs requires packaging < 20.10\n",
+        "!pip install \"packaging<20.10\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nekG9Iwgxbx0"
+      },
+      "outputs": [],
+      "source": [
+        "import matplotlib.pyplot as plt\n",
+        "import os\n",
+        "import seaborn as sns\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_datasets as tfds\n",
+        "\n",
+        "from tensorflow_examples.lite.model_maker.core.export_format import ExportFormat\n",
+        "from tensorflow_examples.lite.model_maker.core.task import image_preprocessing\n",
+        "\n",
+        "from tflite_model_maker import image_classifier\n",
+        "from tflite_model_maker import ImageClassifierDataLoader\n",
+        "from tflite_model_maker.image_classifier import ModelSpec"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fV0k2Q4x4N_4"
+      },
+      "source": [
+        "## Load a TFDS dataset to fine-tune on\n",
+        "\n",
+        "Lets use the publicly available [Cassava Leaf Disease dataset](https://www.tensorflow.org/datasets/catalog/cassava) from TFDS."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TTaD5W_1xjUz"
+      },
+      "outputs": [],
+      "source": [
+        "tfds_name = 'cassava'\n",
+        "(ds_train, ds_validation, ds_test), ds_info = tfds.load(\n",
+        "    name=tfds_name,\n",
+        "    split=['train', 'validation', 'test'],\n",
+        "    with_info=True,\n",
+        "    as_supervised=True)\n",
+        "TFLITE_NAME_PREFIX = tfds_name"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xDuDGUAxyHtA"
+      },
+      "source": [
+        "## Or alternatively load your own data to fine-tune on\n",
+        "\n",
+        "Instead of using a TFDS dataset, you can also train on your own data. This code snippet shows how to load your own custom dataset. See [this](https://www.tensorflow.org/datasets/api_docs/python/tfds/folder_dataset/ImageFolder) link for the supported structure of the data. An example is provided here using the publicly available [Cassava Leaf Disease dataset](https://www.tensorflow.org/datasets/catalog/cassava)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "k003tLvflHpC"
+      },
+      "outputs": [],
+      "source": [
+        "# data_root_dir = tf.keras.utils.get_file(\n",
+        "#     'cassavaleafdata.zip',\n",
+        "#     'https://storage.googleapis.com/emcassavadata/cassavaleafdata.zip',\n",
+        "#     extract=True)\n",
+        "# data_root_dir = os.path.splitext(data_root_dir)[0]  # Remove the .zip extension\n",
+        "\n",
+        "# builder = tfds.ImageFolder(data_root_dir)\n",
+        "\n",
+        "# ds_info = builder.info\n",
+        "# ds_train = builder.as_dataset(split='train', as_supervised=True)\n",
+        "# ds_validation = builder.as_dataset(split='validation', as_supervised=True)\n",
+        "# ds_test = builder.as_dataset(split='test', as_supervised=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hs3XCVLo4Fa1"
+      },
+      "source": [
+        "## Visualize samples from train split\n",
+        "\n",
+        "Let's take a look at some examples from the dataset including the class id and the class name for the image samples and their labels."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "89GkD60Eyfe0"
+      },
+      "outputs": [],
+      "source": [
+        "_ = tfds.show_examples(ds_train, ds_info)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-KW-n0lV4AZ-"
+      },
+      "source": [
+        "## Add images to be used as Unknown examples from TFDS datasets\n",
+        "\n",
+        "Add additional unknown (negative) examples to the training dataset and assign a new unknown class label number to them. The goal is to have a model that, when used in practice (e.g. in the field), has the option of predicting \"Unknown\" when it sees something unexpected.\n",
+        "\n",
+        "Below you can see a list of datasets that will be used to sample the additional unknown imagery. It includes 3 completely different datasets to increase diversity. One of them is a beans leaf disease dataset, so that the model has exposure to diseased plants other than cassava.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SYDMjRhDkDnd"
+      },
+      "outputs": [],
+      "source": [
+        "UNKNOWN_TFDS_DATASETS = [{\n",
+        "    'tfds_name': 'imagenet_v2/matched-frequency',\n",
+        "    'train_split': 'test[:80%]',\n",
+        "    'test_split': 'test[80%:]',\n",
+        "    'num_examples_ratio_to_normal': 1.0,\n",
+        "}, {\n",
+        "    'tfds_name': 'oxford_flowers102',\n",
+        "    'train_split': 'train',\n",
+        "    'test_split': 'test',\n",
+        "    'num_examples_ratio_to_normal': 1.0,\n",
+        "}, {\n",
+        "    'tfds_name': 'beans',\n",
+        "    'train_split': 'train',\n",
+        "    'test_split': 'test',\n",
+        "    'num_examples_ratio_to_normal': 1.0,\n",
+        "}]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XUM_d0evktGi"
+      },
+      "source": [
+        "The UNKNOWN datasets are also loaded from TFDS."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5DdWgBTe8uKR"
+      },
+      "outputs": [],
+      "source": [
+        "# Load unknown datasets.\n",
+        "weights = [\n",
+        "    spec['num_examples_ratio_to_normal'] for spec in UNKNOWN_TFDS_DATASETS\n",
+        "]\n",
+        "num_unknown_train_examples = sum(\n",
+        "    int(w * ds_train.cardinality().numpy()) for w in weights)\n",
+        "ds_unknown_train = tf.data.Dataset.sample_from_datasets([\n",
+        "    tfds.load(\n",
+        "        name=spec['tfds_name'], split=spec['train_split'],\n",
+        "        as_supervised=True).repeat(-1) for spec in UNKNOWN_TFDS_DATASETS\n",
+        "], weights).take(num_unknown_train_examples)\n",
+        "ds_unknown_train = ds_unknown_train.apply(\n",
+        "    tf.data.experimental.assert_cardinality(num_unknown_train_examples))\n",
+        "ds_unknown_tests = [\n",
+        "    tfds.load(\n",
+        "        name=spec['tfds_name'], split=spec['test_split'], as_supervised=True)\n",
+        "    for spec in UNKNOWN_TFDS_DATASETS\n",
+        "]\n",
+        "ds_unknown_test = ds_unknown_tests[0]\n",
+        "for ds in ds_unknown_tests[1:]:\n",
+        "  ds_unknown_test = ds_unknown_test.concatenate(ds)\n",
+        "\n",
+        "# All examples from the unknown datasets will get a new class label number.\n",
+        "num_normal_classes = len(ds_info.features['label'].names)\n",
+        "unknown_label_value = tf.convert_to_tensor(num_normal_classes, tf.int64)\n",
+        "ds_unknown_train = ds_unknown_train.map(lambda image, _:\n",
+        "                                        (image, unknown_label_value))\n",
+        "ds_unknown_test = ds_unknown_test.map(lambda image, _:\n",
+        "                                      (image, unknown_label_value))\n",
+        "\n",
+        "# Merge the normal train dataset with the unknown train dataset.\n",
+        "weights = [\n",
+        "    ds_train.cardinality().numpy(),\n",
+        "    ds_unknown_train.cardinality().numpy()\n",
+        "]\n",
+        "ds_train_with_unknown = tf.data.Dataset.sample_from_datasets(\n",
+        "    [ds_train, ds_unknown_train], [float(w) for w in weights])\n",
+        "ds_train_with_unknown = ds_train_with_unknown.apply(\n",
+        "    tf.data.experimental.assert_cardinality(sum(weights)))\n",
+        "\n",
+        "print((f\"Added {ds_unknown_train.cardinality().numpy()} negative examples.\"\n",
+        "       f\"Training dataset has now {ds_train_with_unknown.cardinality().numpy()}\"\n",
+        "       ' examples in total.'))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "am6eKbzt7raH"
+      },
+      "source": [
+        "## Apply augmentations"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sxIUP0Flk35V"
+      },
+      "source": [
+        "For all the images, to make them more diverse, you'll apply some augmentation, like changes in:\n",
+        "- Brightness\n",
+        "- Contrast\n",
+        "- Saturation\n",
+        "- Hue\n",
+        "- Crop\n",
+        "\n",
+        "These types of augmentations help make the model more robust to variations in image inputs.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "q_BiOkXjqRju"
+      },
+      "outputs": [],
+      "source": [
+        "def random_crop_and_random_augmentations_fn(image):\n",
+        "  # preprocess_for_train does random crop and resize internally.\n",
+        "  image = image_preprocessing.preprocess_for_train(image)\n",
+        "  image = tf.image.random_brightness(image, 0.2)\n",
+        "  image = tf.image.random_contrast(image, 0.5, 2.0)\n",
+        "  image = tf.image.random_saturation(image, 0.75, 1.25)\n",
+        "  image = tf.image.random_hue(image, 0.1)\n",
+        "  return image\n",
+        "\n",
+        "\n",
+        "def random_crop_fn(image):\n",
+        "  # preprocess_for_train does random crop and resize internally.\n",
+        "  image = image_preprocessing.preprocess_for_train(image)\n",
+        "  return image\n",
+        "\n",
+        "\n",
+        "def resize_and_center_crop_fn(image):\n",
+        "  image = tf.image.resize(image, (256, 256))\n",
+        "  image = image[16:240, 16:240]\n",
+        "  return image\n",
+        "\n",
+        "\n",
+        "no_augment_fn = lambda image: image\n",
+        "\n",
+        "train_augment_fn = lambda image, label: (\n",
+        "    random_crop_and_random_augmentations_fn(image), label)\n",
+        "eval_augment_fn = lambda image, label: (resize_and_center_crop_fn(image), label)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RUfqE1c3l6my"
+      },
+      "source": [
+        "To apply the augmentation, it uses the `map` method from the Dataset class."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Uq-NCtaH_h8j"
+      },
+      "outputs": [],
+      "source": [
+        "ds_train_with_unknown = ds_train_with_unknown.map(train_augment_fn)\n",
+        "ds_validation = ds_validation.map(eval_augment_fn)\n",
+        "ds_test = ds_test.map(eval_augment_fn)\n",
+        "ds_unknown_test = ds_unknown_test.map(eval_augment_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DvnwolLiCqYX"
+      },
+      "source": [
+        "## Wrap the data into Model Maker friendly format\n",
+        "\n",
+        "To use these dataset with Model Maker, they need to be in a ImageClassifierDataLoader class."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OXPWEDFDRlVu"
+      },
+      "outputs": [],
+      "source": [
+        "label_names = ds_info.features['label'].names + ['UNKNOWN']\n",
+        "\n",
+        "train_data = ImageClassifierDataLoader(ds_train_with_unknown,\n",
+        "                                       ds_train_with_unknown.cardinality(),\n",
+        "                                       label_names)\n",
+        "validation_data = ImageClassifierDataLoader(ds_validation,\n",
+        "                                            ds_validation.cardinality(),\n",
+        "                                            label_names)\n",
+        "test_data = ImageClassifierDataLoader(ds_test, ds_test.cardinality(),\n",
+        "                                      label_names)\n",
+        "unknown_test_data = ImageClassifierDataLoader(ds_unknown_test,\n",
+        "                                              ds_unknown_test.cardinality(),\n",
+        "                                              label_names)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "j2iDwq2Njpb_"
+      },
+      "source": [
+        "## Run training\n",
+        "\n",
+        "[TensorFlow Hub](https://tfhub.dev) has multiple models available for Transfer Learning.\n",
+        "\n",
+        "Here you can choose one and you can also keep experimenting with other ones to try to get better results.\n",
+        "\n",
+        "If you want even more models to try, you can add them from this [collection](https://tfhub.dev/google/collections/image/1).\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "5UhNpR0Ex_5-"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Choose a base model\n",
+        "\n",
+        "model_name = 'mobilenet_v3_large_100_224'  #@param ['cropnet_cassava', 'cropnet_concat', 'cropnet_imagenet', 'mobilenet_v3_large_100_224']\n",
+        "\n",
+        "map_model_name = {\n",
+        "    'cropnet_cassava':\n",
+        "        'https://tfhub.dev/google/cropnet/feature_vector/cassava_disease_V1/1',\n",
+        "    'cropnet_concat':\n",
+        "        'https://tfhub.dev/google/cropnet/feature_vector/concat/1',\n",
+        "    'cropnet_imagenet':\n",
+        "        'https://tfhub.dev/google/cropnet/feature_vector/imagenet/1',\n",
+        "    'mobilenet_v3_large_100_224':\n",
+        "        'https://tfhub.dev/google/imagenet/mobilenet_v3_large_100_224/feature_vector/5',\n",
+        "}\n",
+        "\n",
+        "model_handle = map_model_name[model_name]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Y1ecXlQgR5Uk"
+      },
+      "source": [
+        "To fine tune the model, you will use Model Maker. This  makes the overall solution easier since after the training of the model, it'll also convert it to TFLite.\n",
+        "\n",
+        "Model Maker makes this conversion be the best one possible and with all the necessary information to easily deploy the model on-device later.\n",
+        "\n",
+        "The model spec is how you tell Model Maker which base model you'd like to use."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "L8P-VTqJ8GaF"
+      },
+      "outputs": [],
+      "source": [
+        "image_model_spec = ModelSpec(uri=model_handle)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AnWN3kk6jCHf"
+      },
+      "source": [
+        "One important detail here is setting `train_whole_model` which will make the base model fine tuned during training. This makes the process slower but the final model has a higher accuracy. Setting `shuffle` will make sure the model sees the data in a random shuffled order which is a best practice for model learning."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KRbSDbnA6Xap"
+      },
+      "outputs": [],
+      "source": [
+        "model = image_classifier.create(\n",
+        "    train_data,\n",
+        "    model_spec=image_model_spec,\n",
+        "    batch_size=128,\n",
+        "    learning_rate=0.03,\n",
+        "    epochs=5,\n",
+        "    shuffle=True,\n",
+        "    train_whole_model=True,\n",
+        "    validation_data=validation_data)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "buFDW0izBqIQ"
+      },
+      "source": [
+        "## Evaluate model on test split"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OYIZ1rlV7lxm"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(test_data)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YJaReZ_OVU71"
+      },
+      "source": [
+        "To have an even better understanding of the fine tuned model, it's good to analyse the confusion matrix. This will show how often one class is predicted as another."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o9_vs1nNKOLF"
+      },
+      "outputs": [],
+      "source": [
+        "def predict_class_label_number(dataset):\n",
+        "  \"\"\"Runs inference and returns predictions as class label numbers.\"\"\"\n",
+        "  rev_label_names = {l: i for i, l in enumerate(label_names)}\n",
+        "  return [\n",
+        "      rev_label_names[o[0][0]]\n",
+        "      for o in model.predict_top_k(dataset, batch_size=128)\n",
+        "  ]\n",
+        "\n",
+        "def show_confusion_matrix(cm, labels):\n",
+        "  plt.figure(figsize=(10, 8))\n",
+        "  sns.heatmap(cm, xticklabels=labels, yticklabels=labels, \n",
+        "              annot=True, fmt='g')\n",
+        "  plt.xlabel('Prediction')\n",
+        "  plt.ylabel('Label')\n",
+        "  plt.show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7BWZCKerCNF_"
+      },
+      "outputs": [],
+      "source": [
+        "confusion_mtx = tf.math.confusion_matrix(\n",
+        "    list(ds_test.map(lambda x, y: y)),\n",
+        "    predict_class_label_number(test_data),\n",
+        "    num_classes=len(label_names))\n",
+        "\n",
+        "show_confusion_matrix(confusion_mtx, label_names)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ksu9BFULBvmj"
+      },
+      "source": [
+        "## Evaluate model on unknown test data\n",
+        "\n",
+        "In this evaluation we expect the model to have accuracy of almost 1. All images the model is tested on are not related to the normal dataset and hence we expect the model to predict the \"Unknown\" class label."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f5wvZwliZcJP"
+      },
+      "outputs": [],
+      "source": [
+        "model.evaluate(unknown_test_data)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jm47Odo5Vaiq"
+      },
+      "source": [
+        "Print the confusion matrix."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E_gEX3oWH1YT"
+      },
+      "outputs": [],
+      "source": [
+        "unknown_confusion_mtx = tf.math.confusion_matrix(\n",
+        "    list(ds_unknown_test.map(lambda x, y: y)),\n",
+        "    predict_class_label_number(unknown_test_data),\n",
+        "    num_classes=len(label_names))\n",
+        "\n",
+        "show_confusion_matrix(unknown_confusion_mtx, label_names)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o2agDx2fCHyd"
+      },
+      "source": [
+        "## Export the model as TFLite and SavedModel\n",
+        "\n",
+        "Now we can export the trained models in TFLite and SavedModel formats for deploying on-device and using for inference in TensorFlow."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bAFvBmMr7owW"
+      },
+      "outputs": [],
+      "source": [
+        "tflite_filename = f'{TFLITE_NAME_PREFIX}_model_{model_name}.tflite'\n",
+        "model.export(export_dir='.', tflite_filename=tflite_filename)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Pz0-6To2C4yM"
+      },
+      "outputs": [],
+      "source": [
+        "# Export saved model version.\n",
+        "model.export(export_dir='.', export_format=ExportFormat.SAVED_MODEL)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4V4GdQqxjEU7"
+      },
+      "source": [
+        "## Next steps\n",
+        "\n",
+        "The model that you've just trained can be used on mobile devices and even deployed in the field!\n",
+        "\n",
+        "**To download the model, click the folder icon for the Files menu on the left side of the colab, and choose the download option.**\n",
+        "\n",
+        "The same technique used here could be applied to other plant diseases tasks that might be more suitable for your use case or any other type of image classification task. If you want to follow up and deploy on an Android app, you can continue on this [Android quickstart guide](https://www.tensorflow.org/lite/android/quickstart)."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "3XX46cTrh6iD",
+        "xDuDGUAxyHtA"
+      ],
+      "name": "cropnet_on_device.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/cross_lingual_similarity_with_tf_hub_multilingual_universal_encoder.ipynb b/site/en/hub/tutorials/cross_lingual_similarity_with_tf_hub_multilingual_universal_encoder.ipynb
new file mode 100644
index 00000000000..920d197811e
--- /dev/null
+++ b/site/en/hub/tutorials/cross_lingual_similarity_with_tf_hub_multilingual_universal_encoder.ipynb
@@ -0,0 +1,4463 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RUymE2l9GZfO"
+      },
+      "source": [
+        "**Copyright 2019 The TensorFlow Hub Authors.**\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "code",
+        "id": "JMyTNwSJGGWg"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "co7MV6sX7Xto"
+      },
+      "source": [
+        "# Cross-Lingual Similarity and Semantic Search Engine with Multilingual Universal Sentence Encoder\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eAVQGidpL8v5"
+      },
+      "source": [
+        "This notebook illustrates how to access the Multilingual Universal Sentence Encoder module and use it for sentence similarity across multiple languages. This module is an extension of the [original Universal Encoder module](https://tfhub.dev/google/universal-sentence-encoder/2).\n",
+        "\n",
+        "The notebook is divided as follows:\n",
+        "\n",
+        "*   The first section shows a visualization of sentences between pair of languages. This is a more academic exercise. \n",
+        "*   In the second section, we show how to build a semantic search engine from a sample of a Wikipedia corpus in multiple languages."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UvNRbHGarYeR"
+      },
+      "source": [
+        "## Citation\n",
+        "\n",
+        "*Research papers that make use of the models explored in this colab should cite:*\n",
+        "\n",
+        "### [Multilingual universal sentence encoder for semantic retrieval](https://arxiv.org/abs/1907.04307)\n",
+        "Yinfei Yang, Daniel Cer, Amin Ahmad, Mandy Guo, Jax Law, Noah Constant, Gustavo Hernandez Abrego, Steve Yuan, Chris Tar, Yun-Hsuan Sung, Brian Strope, and Ray Kurzweil. 2019.\n",
+        " arXiv preprint arXiv:1907.04307"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pOTzp8O36CyQ"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "This section sets up the environment for access to the Multilingual Universal Sentence Encoder Module and also prepares a set of English sentences and their translations. In the following sections, the multilingual module will be used to compute similarity *across languages*."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "lVjNK8shFKOC"
+      },
+      "outputs": [],
+      "source": [
+        "%%capture\n",
+        "#@title Setup Environment\n",
+        "# Install the latest Tensorflow version.\n",
+        "!pip install \"tensorflow-text==2.11.*\"\n",
+        "!pip install bokeh\n",
+        "!pip install simpleneighbors[annoy]\n",
+        "!pip install tqdm"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "MSeY-MUQo2Ha"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Setup common imports and functions\n",
+        "import bokeh\n",
+        "import bokeh.models\n",
+        "import bokeh.plotting\n",
+        "import numpy as np\n",
+        "import os\n",
+        "import pandas as pd\n",
+        "import tensorflow.compat.v2 as tf\n",
+        "import tensorflow_hub as hub\n",
+        "from tensorflow_text import SentencepieceTokenizer\n",
+        "import sklearn.metrics.pairwise\n",
+        "\n",
+        "from simpleneighbors import SimpleNeighbors\n",
+        "from tqdm import tqdm\n",
+        "from tqdm import trange\n",
+        "\n",
+        "def visualize_similarity(embeddings_1, embeddings_2, labels_1, labels_2,\n",
+        "                         plot_title,\n",
+        "                         plot_width=1200, plot_height=600,\n",
+        "                         xaxis_font_size='12pt', yaxis_font_size='12pt'):\n",
+        "\n",
+        "  assert len(embeddings_1) == len(labels_1)\n",
+        "  assert len(embeddings_2) == len(labels_2)\n",
+        "\n",
+        "  # arccos based text similarity (Yang et al. 2019; Cer et al. 2019)\n",
+        "  sim = 1 - np.arccos(\n",
+        "      sklearn.metrics.pairwise.cosine_similarity(embeddings_1,\n",
+        "                                                 embeddings_2))/np.pi\n",
+        "\n",
+        "  embeddings_1_col, embeddings_2_col, sim_col = [], [], []\n",
+        "  for i in range(len(embeddings_1)):\n",
+        "    for j in range(len(embeddings_2)):\n",
+        "      embeddings_1_col.append(labels_1[i])\n",
+        "      embeddings_2_col.append(labels_2[j])\n",
+        "      sim_col.append(sim[i][j])\n",
+        "  df = pd.DataFrame(zip(embeddings_1_col, embeddings_2_col, sim_col),\n",
+        "                    columns=['embeddings_1', 'embeddings_2', 'sim'])\n",
+        "\n",
+        "  mapper = bokeh.models.LinearColorMapper(\n",
+        "      palette=[*reversed(bokeh.palettes.YlOrRd[9])], low=df.sim.min(),\n",
+        "      high=df.sim.max())\n",
+        "\n",
+        "  p = bokeh.plotting.figure(title=plot_title, x_range=labels_1,\n",
+        "                            x_axis_location=\"above\",\n",
+        "                            y_range=[*reversed(labels_2)],\n",
+        "                            plot_width=plot_width, plot_height=plot_height,\n",
+        "                            tools=\"save\",toolbar_location='below', tooltips=[\n",
+        "                                ('pair', '@embeddings_1 ||| @embeddings_2'),\n",
+        "                                ('sim', '@sim')])\n",
+        "  p.rect(x=\"embeddings_1\", y=\"embeddings_2\", width=1, height=1, source=df,\n",
+        "         fill_color={'field': 'sim', 'transform': mapper}, line_color=None)\n",
+        "\n",
+        "  p.title.text_font_size = '12pt'\n",
+        "  p.axis.axis_line_color = None\n",
+        "  p.axis.major_tick_line_color = None\n",
+        "  p.axis.major_label_standoff = 16\n",
+        "  p.xaxis.major_label_text_font_size = xaxis_font_size\n",
+        "  p.xaxis.major_label_orientation = 0.25 * np.pi\n",
+        "  p.yaxis.major_label_text_font_size = yaxis_font_size\n",
+        "  p.min_border_right = 300\n",
+        "\n",
+        "  bokeh.io.output_notebook()\n",
+        "  bokeh.io.show(p)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gk2IRjZFGDsK"
+      },
+      "source": [
+        "This is additional boilerplate code where we import the pre-trained ML model we will use to encode text throughout this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mkmF3w8WGLcM"
+      },
+      "outputs": [],
+      "source": [
+        "# The 16-language multilingual module is the default but feel free\n",
+        "# to pick others from the list and compare the results.\n",
+        "module_url = 'https://tfhub.dev/google/universal-sentence-encoder-multilingual/3' #@param ['https://tfhub.dev/google/universal-sentence-encoder-multilingual/3', 'https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3']\n",
+        "\n",
+        "model = hub.load(module_url)\n",
+        "\n",
+        "def embed_text(input):\n",
+        "  return model(input)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jhLPq6AROyFk"
+      },
+      "source": [
+        "# Visualize Text Similarity Between Languages\n",
+        "With the sentence embeddings now in hand, we can visualize semantic similarity across different languages."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8xdAogbxJDTD"
+      },
+      "source": [
+        "## Computing Text Embeddings\n",
+        "\n",
+        "We first define a set of sentences translated to various languages in parallel. Then, we precompute the embeddings for all of our sentences."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q8F4LNGFqOiq"
+      },
+      "outputs": [],
+      "source": [
+        "# Some texts of different lengths in different languages.\n",
+        "arabic_sentences = ['كلب', 'الجراء لطيفة.', 'أستمتع بالمشي لمسافات طويلة على طول الشاطئ مع كلبي.']\n",
+        "chinese_sentences = ['狗', '小狗很好。', '我喜欢和我的狗一起沿着海滩散步。']\n",
+        "english_sentences = ['dog', 'Puppies are nice.', 'I enjoy taking long walks along the beach with my dog.']\n",
+        "french_sentences = ['chien', 'Les chiots sont gentils.', 'J\\'aime faire de longues promenades sur la plage avec mon chien.']\n",
+        "german_sentences = ['Hund', 'Welpen sind nett.', 'Ich genieße lange Spaziergänge am Strand entlang mit meinem Hund.']\n",
+        "italian_sentences = ['cane', 'I cuccioli sono carini.', 'Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.']\n",
+        "japanese_sentences = ['犬', '子犬はいいです', '私は犬と一緒にビーチを散歩するのが好きです']\n",
+        "korean_sentences = ['개', '강아지가 좋다.', '나는 나의 개와 해변을 따라 길게 산책하는 것을 즐긴다.']\n",
+        "russian_sentences = ['собака', 'Милые щенки.', 'Мне нравится подолгу гулять по пляжу со своей собакой.']\n",
+        "spanish_sentences = ['perro', 'Los cachorros son agradables.', 'Disfruto de dar largos paseos por la playa con mi perro.']\n",
+        "\n",
+        "# Multilingual example\n",
+        "multilingual_example = [\"Willkommen zu einfachen, aber\", \"verrassend krachtige\", \"multilingüe\", \"compréhension du language naturel\", \"модели.\", \"大家是什么意思\" , \"보다 중요한\", \".اللغة التي يتحدثونها\"]\n",
+        "multilingual_example_in_en =  [\"Welcome to simple yet\", \"surprisingly powerful\", \"multilingual\", \"natural language understanding\", \"models.\", \"What people mean\", \"matters more than\", \"the language they speak.\"]\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "weXZqLtTJY9b"
+      },
+      "outputs": [],
+      "source": [
+        "# Compute embeddings.\n",
+        "ar_result = embed_text(arabic_sentences)\n",
+        "en_result = embed_text(english_sentences)\n",
+        "es_result = embed_text(spanish_sentences)\n",
+        "de_result = embed_text(german_sentences)\n",
+        "fr_result = embed_text(french_sentences)\n",
+        "it_result = embed_text(italian_sentences)\n",
+        "ja_result = embed_text(japanese_sentences)\n",
+        "ko_result = embed_text(korean_sentences)\n",
+        "ru_result = embed_text(russian_sentences)\n",
+        "zh_result = embed_text(chinese_sentences)\n",
+        "\n",
+        "multilingual_result = embed_text(multilingual_example)\n",
+        "multilingual_in_en_result = embed_text(multilingual_example_in_en)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_3zGWuF-GhUm"
+      },
+      "source": [
+        "## Visualizing Similarity\n",
+        "\n",
+        "With text embeddings in hand, we can take their dot-product to visualize how similar sentences are between languages. A darker color indicates the embeddings are semantically similar."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WOEIJA0mh70g"
+      },
+      "source": [
+        "### Multilingual Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R2hbCMhmiDWR"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"b33996e2-7bdd-4097-888b-8bf79a526bff\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1013\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1016\",\"type\":\"Grid\"},{\"id\":\"1020\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1017\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1030\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1003\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1023\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1005\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1009\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1007\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1011\",\"type\":\"CategoricalScale\"}},\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"Welcome to simple yet\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"surprisingly powerful\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"multilingual\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"natural language understanding\",\"models.\",\"models.\",\"models.\",\"models.\",\"models.\",\"models.\",\"models.\",\"models.\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"What people mean\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"matters more than\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\",\"the language they speak.\"],\"embeddings_2\":[\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"Willkommen zu einfachen, aber\",\"verrassend krachtige\",\"multiling\\u00fce\",\"compr\\u00e9hension du langage naturel\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\"],\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],\"sim\":{\"__ndarray__\":\"AAAAwK4B6j8AAACgVl7iPwAAAMD5l94/AAAAQOVr4T8AAABA8/TfPwAAACDzG+A/AAAAAIa+4T8AAABAMivgPwAAAKBdieE/AAAAgBBH6T8AAAAA5griPwAAAIAUkeE/AAAAAJhp4D8AAACAkSTgPwAAAICvreI/AAAAgIcw4j8AAAAAELXfPwAAAKD19uE/AAAAwH/P6T8AAADAZbjjPwAAAABaX+I/AAAAgG6I4T8AAACAx/viPwAAACBrc+Y/AAAAQApe4T8AAACA32XiPwAAAAD0j+M/AAAAADsK7D8AAADgBnHhPwAAAAAIZeE/AAAAAHZm4T8AAACgXprmPwAAAAB5Jt8/AAAAQMci4D8AAACgPI7hPwAAAKBmSOE/AAAAQArv7D8AAABA7O7fPwAAAID9d+E/AAAAgIRv4T8AAACAN4PgPwAAAMD/yOA/AAAAgEIH4j8AAADAAFbiPwAAAMDwOuE/AAAAQE1v5j8AAABg2bLiPwAAAMDSyOI/AAAAwLXb4D8AAACAztLhPwAAAOB1HeI/AAAAgCd/4T8AAAAg1xHhPwAAAAAPQeE/AAAAAELU6T8AAAAgZjLiPwAAAACyYd8/AAAA4FKw4D8AAACgbUHkPwAAAEDcVeU/AAAAIHV34j8AAAAARznhPwAAAOCXLOE/AAAAQP8H6T8=\",\"dtype\":\"float64\",\"shape\":[64]}},\"selected\":{\"id\":\"1037\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1038\",\"type\":\"UnionRenderers\"}},\"id\":\"1026\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"text\":\"Multilingual Universal Sentence Encoder for Semantic Retrieval (Yang et al., 2019)\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1003\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1033\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"callback\":null,\"factors\":[\".\\u0627\\u0644\\u0644\\u063a\\u0629 \\u0627\\u0644\\u062a\\u064a \\u064a\\u062a\\u062d\\u062f\\u062b\\u0648\\u0646\\u0647\\u0627\",\"\\ubcf4\\ub2e4 \\uc911\\uc694\\ud55c\",\"\\u5927\\u5bb6\\u662f\\u4ec0\\u4e48\\u610f\\u601d\",\"\\u043c\\u043e\\u0434\\u0435\\u043b\\u0438.\",\"compr\\u00e9hension du langage naturel\",\"multiling\\u00fce\",\"verrassend krachtige\",\"Willkommen zu einfachen, aber\"]},\"id\":\"1007\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Welcome to simple yet\",\"surprisingly powerful\",\"multilingual\",\"natural language understanding\",\"models.\",\"What people mean\",\"matters more than\",\"the language they speak.\"]},\"id\":\"1005\",\"type\":\"FactorRange\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1021\",\"type\":\"SaveTool\"},{\"id\":\"1022\",\"type\":\"HoverTool\"}]},\"id\":\"1023\",\"type\":\"Toolbar\"},{\"attributes\":{\"data_source\":{\"id\":\"1026\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1028\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1029\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1031\",\"type\":\"CDSView\"}},\"id\":\"1030\",\"type\":\"GlyphRenderer\"},{\"attributes\":{},\"id\":\"1009\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"source\":{\"id\":\"1026\",\"type\":\"ColumnDataSource\"}},\"id\":\"1031\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1011\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1035\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1014\",\"type\":\"CategoricalTicker\"}},\"id\":\"1013\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"ticker\":{\"id\":\"1014\",\"type\":\"CategoricalTicker\"}},\"id\":\"1016\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1014\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1037\",\"type\":\"Selection\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1033\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1018\",\"type\":\"CategoricalTicker\"}},\"id\":\"1017\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1038\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1018\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1035\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1018\",\"type\":\"CategoricalTicker\"}},\"id\":\"1020\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1021\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1022\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1001\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1028\",\"type\":\"Rect\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1029\",\"type\":\"Rect\"},{\"attributes\":{\"high\":0.9041796922683716,\"low\":0.4780258536338806,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1001\",\"type\":\"LinearColorMapper\"}],\"root_ids\":[\"1002\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"b33996e2-7bdd-4097-888b-8bf79a526bff\",\"roots\":{\"1002\":\"c23bc7cf-8f04-4fa0-a38c-20c29e5098b9\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1002"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(multilingual_in_en_result, multilingual_result,\n",
+        "                     multilingual_example_in_en, multilingual_example,  \"Multilingual Universal Sentence Encoder for Semantic Retrieval (Yang et al., 2019)\")\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h3TEhllsq3ax"
+      },
+      "source": [
+        "### English-Arabic Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q9UDpStmq7Ii"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"a8ebb685-fc25-4a41-b1f1-80df179ccab5\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1093\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1096\",\"type\":\"Grid\"},{\"id\":\"1100\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1097\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1110\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1083\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1103\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1085\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1089\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1087\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1091\",\"type\":\"CategoricalScale\"}},\"id\":\"1082\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"data_source\":{\"id\":\"1106\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1108\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1109\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1111\",\"type\":\"CDSView\"}},\"id\":\"1110\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1106\",\"type\":\"ColumnDataSource\"}},\"id\":\"1111\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1120\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1109\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1102\",\"type\":\"HoverTool\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1101\",\"type\":\"SaveTool\"},{\"id\":\"1102\",\"type\":\"HoverTool\"}]},\"id\":\"1103\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1094\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1098\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1098\",\"type\":\"CategoricalTicker\"}},\"id\":\"1100\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1122\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1124\",\"type\":\"Selection\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1120\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1098\",\"type\":\"CategoricalTicker\"}},\"id\":\"1097\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1125\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"ticker\":{\"id\":\"1094\",\"type\":\"CategoricalTicker\"}},\"id\":\"1096\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1091\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1122\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1094\",\"type\":\"CategoricalTicker\"}},\"id\":\"1093\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1089\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\u0623\\u0633\\u062a\\u0645\\u062a\\u0639 \\u0628\\u0627\\u0644\\u0645\\u0634\\u064a \\u0644\\u0645\\u0633\\u0627\\u0641\\u0627\\u062a \\u0637\\u0648\\u064a\\u0644\\u0629 \\u0639\\u0644\\u0649 \\u0637\\u0648\\u0644 \\u0627\\u0644\\u0634\\u0627\\u0637\\u0626 \\u0645\\u0639 \\u0643\\u0644\\u0628\\u064a.\",\"\\u0627\\u0644\\u062c\\u0631\\u0627\\u0621 \\u0644\\u0637\\u064a\\u0641\\u0629.\",\"\\u0643\\u0644\\u0628\"]},\"id\":\"1087\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1085\",\"type\":\"FactorRange\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1081\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1108\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"\\u0643\\u0644\\u0628\",\"\\u0627\\u0644\\u062c\\u0631\\u0627\\u0621 \\u0644\\u0637\\u064a\\u0641\\u0629.\",\"\\u0623\\u0633\\u062a\\u0645\\u062a\\u0639 \\u0628\\u0627\\u0644\\u0645\\u0634\\u064a \\u0644\\u0645\\u0633\\u0627\\u0641\\u0627\\u062a \\u0637\\u0648\\u064a\\u0644\\u0629 \\u0639\\u0644\\u0649 \\u0637\\u0648\\u0644 \\u0627\\u0644\\u0634\\u0627\\u0637\\u0626 \\u0645\\u0639 \\u0643\\u0644\\u0628\\u064a.\",\"\\u0643\\u0644\\u0628\",\"\\u0627\\u0644\\u062c\\u0631\\u0627\\u0621 \\u0644\\u0637\\u064a\\u0641\\u0629.\",\"\\u0623\\u0633\\u062a\\u0645\\u062a\\u0639 \\u0628\\u0627\\u0644\\u0645\\u0634\\u064a \\u0644\\u0645\\u0633\\u0627\\u0641\\u0627\\u062a \\u0637\\u0648\\u064a\\u0644\\u0629 \\u0639\\u0644\\u0649 \\u0637\\u0648\\u0644 \\u0627\\u0644\\u0634\\u0627\\u0637\\u0626 \\u0645\\u0639 \\u0643\\u0644\\u0628\\u064a.\",\"\\u0643\\u0644\\u0628\",\"\\u0627\\u0644\\u062c\\u0631\\u0627\\u0621 \\u0644\\u0637\\u064a\\u0641\\u0629.\",\"\\u0623\\u0633\\u062a\\u0645\\u062a\\u0639 \\u0628\\u0627\\u0644\\u0645\\u0634\\u064a \\u0644\\u0645\\u0633\\u0627\\u0641\\u0627\\u062a \\u0637\\u0648\\u064a\\u0644\\u0629 \\u0639\\u0644\\u0649 \\u0637\\u0648\\u0644 \\u0627\\u0644\\u0634\\u0627\\u0637\\u0626 \\u0645\\u0639 \\u0643\\u0644\\u0628\\u064a.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAQLvE6j8AAABAlJHkPwAAAAAOj+A/AAAAwOqU5D8AAACA9aTnPwAAAMACcuA/AAAAAFtl4j8AAACAFvjiPwAAAIB6QOg/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1124\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1125\",\"type\":\"UnionRenderers\"}},\"id\":\"1106\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1101\",\"type\":\"SaveTool\"},{\"attributes\":{\"text\":\"English-Arabic Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1083\",\"type\":\"Title\"},{\"attributes\":{\"high\":0.8365150690078735,\"low\":0.5139173269271851,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1081\",\"type\":\"LinearColorMapper\"}],\"root_ids\":[\"1082\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"a8ebb685-fc25-4a41-b1f1-80df179ccab5\",\"roots\":{\"1082\":\"d20b027b-6bf0-444d-8763-df8d299e1642\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1082"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(en_result, ar_result, english_sentences, arabic_sentences, 'English-Arabic Similarity')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QF9z48HMp4WL"
+      },
+      "source": [
+        "### Engish-Russian Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QE68UejYp86z"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"c09fc8f6-d5fa-4ba4-ae4a-0ce276d613ba\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1180\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1183\",\"type\":\"Grid\"},{\"id\":\"1187\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1184\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1197\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1170\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1190\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1172\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1176\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1174\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1178\",\"type\":\"CategoricalScale\"}},\"id\":\"1169\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1189\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1218\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1216\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1219\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"ticker\":{\"id\":\"1181\",\"type\":\"CategoricalTicker\"}},\"id\":\"1183\",\"type\":\"Grid\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1216\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1181\",\"type\":\"CategoricalTicker\"}},\"id\":\"1180\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1178\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1176\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\u041c\\u043d\\u0435 \\u043d\\u0440\\u0430\\u0432\\u0438\\u0442\\u0441\\u044f \\u043f\\u043e\\u0434\\u043e\\u043b\\u0433\\u0443 \\u0433\\u0443\\u043b\\u044f\\u0442\\u044c \\u043f\\u043e \\u043f\\u043b\\u044f\\u0436\\u0443 \\u0441\\u043e \\u0441\\u0432\\u043e\\u0435\\u0439 \\u0441\\u043e\\u0431\\u0430\\u043a\\u043e\\u0439.\",\"\\u041c\\u0438\\u043b\\u044b\\u0435 \\u0449\\u0435\\u043d\\u043a\\u0438.\",\"\\u0441\\u043e\\u0431\\u0430\\u043a\\u0430\"]},\"id\":\"1174\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1185\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1214\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1185\",\"type\":\"CategoricalTicker\"}},\"id\":\"1184\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1168\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1195\",\"type\":\"Rect\"},{\"attributes\":{\"high\":0.8845847249031067,\"low\":0.5800867676734924,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1168\",\"type\":\"LinearColorMapper\"},{\"attributes\":{},\"id\":\"1181\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"data_source\":{\"id\":\"1193\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1195\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1196\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1198\",\"type\":\"CDSView\"}},\"id\":\"1197\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1188\",\"type\":\"SaveTool\"},{\"id\":\"1189\",\"type\":\"HoverTool\"}]},\"id\":\"1190\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1172\",\"type\":\"FactorRange\"},{\"attributes\":{\"text\":\"English-Russian Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1170\",\"type\":\"Title\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1185\",\"type\":\"CategoricalTicker\"}},\"id\":\"1187\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"\\u0441\\u043e\\u0431\\u0430\\u043a\\u0430\",\"\\u041c\\u0438\\u043b\\u044b\\u0435 \\u0449\\u0435\\u043d\\u043a\\u0438.\",\"\\u041c\\u043d\\u0435 \\u043d\\u0440\\u0430\\u0432\\u0438\\u0442\\u0441\\u044f \\u043f\\u043e\\u0434\\u043e\\u043b\\u0433\\u0443 \\u0433\\u0443\\u043b\\u044f\\u0442\\u044c \\u043f\\u043e \\u043f\\u043b\\u044f\\u0436\\u0443 \\u0441\\u043e \\u0441\\u0432\\u043e\\u0435\\u0439 \\u0441\\u043e\\u0431\\u0430\\u043a\\u043e\\u0439.\",\"\\u0441\\u043e\\u0431\\u0430\\u043a\\u0430\",\"\\u041c\\u0438\\u043b\\u044b\\u0435 \\u0449\\u0435\\u043d\\u043a\\u0438.\",\"\\u041c\\u043d\\u0435 \\u043d\\u0440\\u0430\\u0432\\u0438\\u0442\\u0441\\u044f \\u043f\\u043e\\u0434\\u043e\\u043b\\u0433\\u0443 \\u0433\\u0443\\u043b\\u044f\\u0442\\u044c \\u043f\\u043e \\u043f\\u043b\\u044f\\u0436\\u0443 \\u0441\\u043e \\u0441\\u0432\\u043e\\u0435\\u0439 \\u0441\\u043e\\u0431\\u0430\\u043a\\u043e\\u0439.\",\"\\u0441\\u043e\\u0431\\u0430\\u043a\\u0430\",\"\\u041c\\u0438\\u043b\\u044b\\u0435 \\u0449\\u0435\\u043d\\u043a\\u0438.\",\"\\u041c\\u043d\\u0435 \\u043d\\u0440\\u0430\\u0432\\u0438\\u0442\\u0441\\u044f \\u043f\\u043e\\u0434\\u043e\\u043b\\u0433\\u0443 \\u0433\\u0443\\u043b\\u044f\\u0442\\u044c \\u043f\\u043e \\u043f\\u043b\\u044f\\u0436\\u0443 \\u0441\\u043e \\u0441\\u0432\\u043e\\u0435\\u0439 \\u0441\\u043e\\u0431\\u0430\\u043a\\u043e\\u0439.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAoIRO7D8AAABgiPvkPwAAAMCko+M/AAAAgDBm5D8AAADAi1DoPwAAAKCr8eI/AAAAIBKQ4j8AAABARq3iPwAAAMAlV+g/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1218\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1219\",\"type\":\"UnionRenderers\"}},\"id\":\"1193\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1214\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1196\",\"type\":\"Rect\"},{\"attributes\":{\"source\":{\"id\":\"1193\",\"type\":\"ColumnDataSource\"}},\"id\":\"1198\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1188\",\"type\":\"SaveTool\"}],\"root_ids\":[\"1169\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"c09fc8f6-d5fa-4ba4-ae4a-0ce276d613ba\",\"roots\":{\"1169\":\"02b6f17f-c980-4d2c-b9d7-58e2d001b1bf\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1169"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(en_result, ru_result, english_sentences, russian_sentences, 'English-Russian Similarity')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BJkL6Az0QXNN"
+      },
+      "source": [
+        "### English-Spanish Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CH_BXVGhQ0GL"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"bb848e12-e360-4876-aa19-21896caab34d\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1274\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1277\",\"type\":\"Grid\"},{\"id\":\"1281\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1278\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1291\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1264\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1284\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1266\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1270\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1268\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1272\",\"type\":\"CategoricalScale\"}},\"id\":\"1263\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1272\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1270\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Disfruto de dar largos paseos por la playa con mi perro.\",\"Los cachorros son agradables.\",\"perro\"]},\"id\":\"1268\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1266\",\"type\":\"FactorRange\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1262\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1289\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\",\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\",\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAA4AIT7T8AAAAAcHfkPwAAAKAngeI/AAAAQNoA5D8AAADAvfvoPwAAAGCGFeI/AAAAgMNr4j8AAAAArbPjPwAAAGCJ1eo/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1319\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1320\",\"type\":\"UnionRenderers\"}},\"id\":\"1287\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"high\":0.9085707068443298,\"low\":0.5651275515556335,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1262\",\"type\":\"LinearColorMapper\"},{\"attributes\":{\"source\":{\"id\":\"1287\",\"type\":\"ColumnDataSource\"}},\"id\":\"1292\",\"type\":\"CDSView\"},{\"attributes\":{\"text\":\"English-Spanish Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1264\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1283\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1290\",\"type\":\"Rect\"},{\"attributes\":{\"ticker\":{\"id\":\"1275\",\"type\":\"CategoricalTicker\"}},\"id\":\"1277\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1282\",\"type\":\"SaveTool\"},{\"attributes\":{\"data_source\":{\"id\":\"1287\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1289\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1290\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1292\",\"type\":\"CDSView\"}},\"id\":\"1291\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1279\",\"type\":\"CategoricalTicker\"}},\"id\":\"1281\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1315\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1282\",\"type\":\"SaveTool\"},{\"id\":\"1283\",\"type\":\"HoverTool\"}]},\"id\":\"1284\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1320\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1279\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1315\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1279\",\"type\":\"CategoricalTicker\"}},\"id\":\"1278\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1319\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1317\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1317\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1275\",\"type\":\"CategoricalTicker\"}},\"id\":\"1274\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1275\",\"type\":\"CategoricalTicker\"}],\"root_ids\":[\"1263\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"bb848e12-e360-4876-aa19-21896caab34d\",\"roots\":{\"1263\":\"81e993c9-fc6b-4169-8c6b-a0101097b959\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1263"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(en_result, es_result, english_sentences, spanish_sentences, 'English-Spanish Similarity')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "imn28LCiQO7d"
+      },
+      "source": [
+        "### English-Italian Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "X9uD3DirPIGd"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"85d1d6b6-c6cd-4e71-b29b-2cc49ada74c5\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1375\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1378\",\"type\":\"Grid\"},{\"id\":\"1382\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1379\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1392\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1365\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1385\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1367\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1371\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1369\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1373\",\"type\":\"CategoricalScale\"}},\"id\":\"1364\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1371\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1427\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1373\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1423\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1425\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1376\",\"type\":\"CategoricalTicker\"}},\"id\":\"1375\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"I cuccioli sono carini.\",\"cane\"]},\"id\":\"1369\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1376\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1425\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"ticker\":{\"id\":\"1376\",\"type\":\"CategoricalTicker\"}},\"id\":\"1378\",\"type\":\"Grid\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1423\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1380\",\"type\":\"CategoricalTicker\"}},\"id\":\"1379\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1380\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"cane\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"cane\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"cane\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAoHcI7T8AAADAU2/jPwAAAIBLIeM/AAAAAO8N5D8AAAAA5GToPwAAAIDhjeI/AAAAQLlt4j8AAAAAEj3iPwAAAGCPHuw/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1427\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1428\",\"type\":\"UnionRenderers\"}},\"id\":\"1388\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1380\",\"type\":\"CategoricalTicker\"}},\"id\":\"1382\",\"type\":\"Grid\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1383\",\"type\":\"SaveTool\"},{\"id\":\"1384\",\"type\":\"HoverTool\"}]},\"id\":\"1385\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1383\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1384\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1391\",\"type\":\"Rect\"},{\"attributes\":{\"text\":\"English-Italian Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1365\",\"type\":\"Title\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1363\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1390\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1367\",\"type\":\"FactorRange\"},{\"attributes\":{\"data_source\":{\"id\":\"1388\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1390\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1391\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1393\",\"type\":\"CDSView\"}},\"id\":\"1392\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1388\",\"type\":\"ColumnDataSource\"}},\"id\":\"1393\",\"type\":\"CDSView\"},{\"attributes\":{\"high\":0.90728360414505,\"low\":0.5699548721313477,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1363\",\"type\":\"LinearColorMapper\"},{\"attributes\":{},\"id\":\"1428\",\"type\":\"UnionRenderers\"}],\"root_ids\":[\"1364\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"85d1d6b6-c6cd-4e71-b29b-2cc49ada74c5\",\"roots\":{\"1364\":\"5e20475c-62a7-4a19-87ed-a605dc444c96\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1364"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(en_result, it_result, english_sentences, italian_sentences, 'English-Italian Similarity')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "m6ySvEGbQaTM"
+      },
+      "source": [
+        "### Italian-Spanish Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "irfwIeitQ7V6"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"9eb33289-2019-49f5-b3a2-c34c93bf2800\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1483\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1486\",\"type\":\"Grid\"},{\"id\":\"1490\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1487\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1500\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1473\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1493\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1475\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1479\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1477\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1481\",\"type\":\"CategoricalScale\"}},\"id\":\"1472\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1491\",\"type\":\"SaveTool\"},{\"id\":\"1492\",\"type\":\"HoverTool\"}]},\"id\":\"1493\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1492\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1499\",\"type\":\"Rect\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1471\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1498\",\"type\":\"Rect\"},{\"attributes\":{\"data_source\":{\"id\":\"1496\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1498\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1499\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1501\",\"type\":\"CDSView\"}},\"id\":\"1500\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"ticker\":{\"id\":\"1484\",\"type\":\"CategoricalTicker\"}},\"id\":\"1486\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"1496\",\"type\":\"ColumnDataSource\"}},\"id\":\"1501\",\"type\":\"CDSView\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Disfruto de dar largos paseos por la playa con mi perro.\",\"Los cachorros son agradables.\",\"perro\"]},\"id\":\"1477\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1491\",\"type\":\"SaveTool\"},{\"attributes\":{\"high\":0.9059451818466187,\"low\":0.564821720123291,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1471\",\"type\":\"LinearColorMapper\"},{\"attributes\":{},\"id\":\"1488\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1540\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1484\",\"type\":\"CategoricalTicker\"}},\"id\":\"1483\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1484\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1538\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1488\",\"type\":\"CategoricalTicker\"}},\"id\":\"1487\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1481\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1543\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1479\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1542\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1540\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"text\":\"Italian-Spanish Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1473\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"factors\":[\"cane\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\"]},\"id\":\"1475\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1538\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"cane\",\"cane\",\"cane\",\"I cuccioli sono carini.\",\"I cuccioli sono carini.\",\"I cuccioli sono carini.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\",\"Mi piace fare lunghe passeggiate lungo la spiaggia con il mio cane.\"],\"embeddings_2\":[\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\",\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\",\"perro\",\"Los cachorros son agradables.\",\"Disfruto de dar largos paseos por la playa con mi perro.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAwID97D8AAACAiUjkPwAAAICyQuI/AAAAQJl+4z8AAABAqebnPwAAAAAFE+I/AAAAgDrw4j8AAAAg++LjPwAAAEDo1eo/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1542\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1543\",\"type\":\"UnionRenderers\"}},\"id\":\"1496\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1488\",\"type\":\"CategoricalTicker\"}},\"id\":\"1490\",\"type\":\"Grid\"}],\"root_ids\":[\"1472\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"9eb33289-2019-49f5-b3a2-c34c93bf2800\",\"roots\":{\"1472\":\"6f559e42-2dec-4a29-a3d9-62c969a8c08a\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1472"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(it_result, es_result, italian_sentences, spanish_sentences, 'Italian-Spanish Similarity')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ueoRO8balwwr"
+      },
+      "source": [
+        "### English-Chinese Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xA7anofVlxL7"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"e65d5833-780d-410f-a7be-915f30e03b59\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1598\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1601\",\"type\":\"Grid\"},{\"id\":\"1605\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1602\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1615\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1588\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1608\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1590\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1594\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1592\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1596\",\"type\":\"CategoricalScale\"}},\"id\":\"1587\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1664\",\"type\":\"Selection\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1603\",\"type\":\"CategoricalTicker\"}},\"id\":\"1605\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1665\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1603\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1660\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1603\",\"type\":\"CategoricalTicker\"}},\"id\":\"1602\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1662\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1599\",\"type\":\"CategoricalTicker\"}},\"id\":\"1598\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1607\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1599\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1596\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1594\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u72d7\"]},\"id\":\"1592\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1590\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"\\u72d7\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\",\"\\u72d7\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\",\"\\u72d7\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAAF0H7D8AAAAgK6rlPwAAACCmLeM/AAAAgLC95D8AAABAJcfnPwAAAMDQZOM/AAAAAFSj4j8AAACg4TfjPwAAAIBszug/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1664\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1665\",\"type\":\"UnionRenderers\"}},\"id\":\"1611\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1606\",\"type\":\"SaveTool\"},{\"id\":\"1607\",\"type\":\"HoverTool\"}]},\"id\":\"1608\",\"type\":\"Toolbar\"},{\"attributes\":{\"text\":\"English-Chinese Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1588\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1662\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"high\":0.8758988380432129,\"low\":0.5824375152587891,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1586\",\"type\":\"LinearColorMapper\"},{\"attributes\":{\"source\":{\"id\":\"1611\",\"type\":\"ColumnDataSource\"}},\"id\":\"1616\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1660\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"data_source\":{\"id\":\"1611\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1613\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1614\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1616\",\"type\":\"CDSView\"}},\"id\":\"1615\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1586\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1613\",\"type\":\"Rect\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1614\",\"type\":\"Rect\"},{\"attributes\":{},\"id\":\"1606\",\"type\":\"SaveTool\"},{\"attributes\":{\"ticker\":{\"id\":\"1599\",\"type\":\"CategoricalTicker\"}},\"id\":\"1601\",\"type\":\"Grid\"}],\"root_ids\":[\"1587\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"e65d5833-780d-410f-a7be-915f30e03b59\",\"roots\":{\"1587\":\"5b8e7d08-b7e7-4a05-a22d-c27aa1873e6d\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1587"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(en_result, zh_result, english_sentences, chinese_sentences, 'English-Chinese Similarity')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8zV1BJc3mL3W"
+      },
+      "source": [
+        "### English-Korean Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iqWy1e1UmQeX"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"1bbe715c-608d-49a6-8927-e818fa752480\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1720\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1723\",\"type\":\"Grid\"},{\"id\":\"1727\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1724\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1737\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1710\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1730\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1712\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1716\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1714\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1718\",\"type\":\"CategoricalScale\"}},\"id\":\"1709\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1794\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1725\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1789\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1725\",\"type\":\"CategoricalTicker\"}},\"id\":\"1724\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1725\",\"type\":\"CategoricalTicker\"}},\"id\":\"1727\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1721\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1728\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"factors\":[\"dog\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\"]},\"id\":\"1712\",\"type\":\"FactorRange\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1728\",\"type\":\"SaveTool\"},{\"id\":\"1729\",\"type\":\"HoverTool\"}]},\"id\":\"1730\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1718\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1793\",\"type\":\"Selection\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1791\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1721\",\"type\":\"CategoricalTicker\"}},\"id\":\"1720\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1729\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1716\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1708\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1735\",\"type\":\"Rect\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\uac1c\"]},\"id\":\"1714\",\"type\":\"FactorRange\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1736\",\"type\":\"Rect\"},{\"attributes\":{\"source\":{\"id\":\"1733\",\"type\":\"ColumnDataSource\"}},\"id\":\"1738\",\"type\":\"CDSView\"},{\"attributes\":{\"data_source\":{\"id\":\"1733\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1735\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1736\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1738\",\"type\":\"CDSView\"}},\"id\":\"1737\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"text\":\"English-Korean Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1710\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"dog\",\"dog\",\"dog\",\"Puppies are nice.\",\"Puppies are nice.\",\"Puppies are nice.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\",\"I enjoy taking long walks along the beach with my dog.\"],\"embeddings_2\":[\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAgICf6z8AAAAAKEjlPwAAAKBNk+A/AAAAwFZl5D8AAAAAUpnnPwAAAID69uA/AAAAQJua4j8AAADgCQ3jPwAAAIC9gOg/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1793\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1794\",\"type\":\"UnionRenderers\"}},\"id\":\"1733\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1789\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1791\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"ticker\":{\"id\":\"1721\",\"type\":\"CategoricalTicker\"}},\"id\":\"1723\",\"type\":\"Grid\"},{\"attributes\":{\"high\":0.8632204532623291,\"low\":0.5179813504219055,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1708\",\"type\":\"LinearColorMapper\"}],\"root_ids\":[\"1709\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"1bbe715c-608d-49a6-8927-e818fa752480\",\"roots\":{\"1709\":\"7b449243-0dbd-46b6-8b02-a89fdf92645e\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1709"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(en_result, ko_result, english_sentences, korean_sentences, 'English-Korean Similarity')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dfTj-JaunFTv"
+      },
+      "source": [
+        "### Chinese-Korean Similarity"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MndSgKGPnJuF"
+      },
+      "outputs": [
+        {
+          "data": {
+            "application/javascript": [
+              "\n",
+              "(function(root) {\n",
+              "  function now() {\n",
+              "    return new Date();\n",
+              "  }\n",
+              "\n",
+              "  var force = true;\n",
+              "\n",
+              "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
+              "    root._bokeh_onload_callbacks = [];\n",
+              "    root._bokeh_is_loading = undefined;\n",
+              "  }\n",
+              "\n",
+              "  var JS_MIME_TYPE = 'application/javascript';\n",
+              "  var HTML_MIME_TYPE = 'text/html';\n",
+              "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
+              "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
+              "\n",
+              "  /**\n",
+              "   * Render data to the DOM node\n",
+              "   */\n",
+              "  function render(props, node) {\n",
+              "    var script = document.createElement(\"script\");\n",
+              "    node.appendChild(script);\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when an output is cleared or removed\n",
+              "   */\n",
+              "  function handleClearOutput(event, handle) {\n",
+              "    var cell = handle.cell;\n",
+              "\n",
+              "    var id = cell.output_area._bokeh_element_id;\n",
+              "    var server_id = cell.output_area._bokeh_server_id;\n",
+              "    // Clean up Bokeh references\n",
+              "    if (id != null && id in Bokeh.index) {\n",
+              "      Bokeh.index[id].model.document.clear();\n",
+              "      delete Bokeh.index[id];\n",
+              "    }\n",
+              "\n",
+              "    if (server_id !== undefined) {\n",
+              "      // Clean up Bokeh references\n",
+              "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
+              "      cell.notebook.kernel.execute(cmd, {\n",
+              "        iopub: {\n",
+              "          output: function(msg) {\n",
+              "            var id = msg.content.text.trim();\n",
+              "            if (id in Bokeh.index) {\n",
+              "              Bokeh.index[id].model.document.clear();\n",
+              "              delete Bokeh.index[id];\n",
+              "            }\n",
+              "          }\n",
+              "        }\n",
+              "      });\n",
+              "      // Destroy server and session\n",
+              "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
+              "      cell.notebook.kernel.execute(cmd);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  /**\n",
+              "   * Handle when a new output is added\n",
+              "   */\n",
+              "  function handleAddOutput(event, handle) {\n",
+              "    var output_area = handle.output_area;\n",
+              "    var output = handle.output;\n",
+              "\n",
+              "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
+              "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
+              "      return\n",
+              "    }\n",
+              "\n",
+              "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
+              "\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
+              "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
+              "      // store reference to embed id on output_area\n",
+              "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
+              "    }\n",
+              "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
+              "      var bk_div = document.createElement(\"div\");\n",
+              "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
+              "      var script_attrs = bk_div.children[0].attributes;\n",
+              "      for (var i = 0; i < script_attrs.length; i++) {\n",
+              "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
+              "      }\n",
+              "      // store reference to server id on output_area\n",
+              "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  function register_renderer(events, OutputArea) {\n",
+              "\n",
+              "    function append_mime(data, metadata, element) {\n",
+              "      // create a DOM node to render to\n",
+              "      var toinsert = this.create_output_subarea(\n",
+              "        metadata,\n",
+              "        CLASS_NAME,\n",
+              "        EXEC_MIME_TYPE\n",
+              "      );\n",
+              "      this.keyboard_manager.register_events(toinsert);\n",
+              "      // Render to node\n",
+              "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
+              "      render(props, toinsert[toinsert.length - 1]);\n",
+              "      element.append(toinsert);\n",
+              "      return toinsert\n",
+              "    }\n",
+              "\n",
+              "    /* Handle when an output is cleared or removed */\n",
+              "    events.on('clear_output.CodeCell', handleClearOutput);\n",
+              "    events.on('delete.Cell', handleClearOutput);\n",
+              "\n",
+              "    /* Handle when a new output is added */\n",
+              "    events.on('output_added.OutputArea', handleAddOutput);\n",
+              "\n",
+              "    /**\n",
+              "     * Register the mime type and append_mime function with output_area\n",
+              "     */\n",
+              "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
+              "      /* Is output safe? */\n",
+              "      safe: true,\n",
+              "      /* Index of renderer in `output_area.display_order` */\n",
+              "      index: 0\n",
+              "    });\n",
+              "  }\n",
+              "\n",
+              "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
+              "  if (root.Jupyter !== undefined) {\n",
+              "    var events = require('base/js/events');\n",
+              "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
+              "\n",
+              "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
+              "      register_renderer(events, OutputArea);\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
+              "    root._bokeh_timeout = Date.now() + 5000;\n",
+              "    root._bokeh_failed_load = false;\n",
+              "  }\n",
+              "\n",
+              "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
+              "     \"
\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
+              "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"
    \\n\"+\n",
+              "     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n",
+              "     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n",
+              "     \"
\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"from bokeh.resources import INLINE\\n\"+\n",
+              "     \"output_notebook(resources=INLINE)\\n\"+\n",
+              "     \"\\n\"+\n",
+              "     \"
\"}};\n",
+              "\n",
+              "  function display_loaded() {\n",
+              "    var el = document.getElementById(null);\n",
+              "    if (el != null) {\n",
+              "      el.textContent = \"BokehJS is loading...\";\n",
+              "    }\n",
+              "    if (root.Bokeh !== undefined) {\n",
+              "      if (el != null) {\n",
+              "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
+              "      }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(display_loaded, 100)\n",
+              "    }\n",
+              "  }\n",
+              "\n",
+              "\n",
+              "  function run_callbacks() {\n",
+              "    try {\n",
+              "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
+              "        if (callback != null)\n",
+              "          callback();\n",
+              "      });\n",
+              "    } finally {\n",
+              "      delete root._bokeh_onload_callbacks\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: all callbacks have finished\");\n",
+              "  }\n",
+              "\n",
+              "  function load_libs(css_urls, js_urls, callback) {\n",
+              "    if (css_urls == null) css_urls = [];\n",
+              "    if (js_urls == null) js_urls = [];\n",
+              "\n",
+              "    root._bokeh_onload_callbacks.push(callback);\n",
+              "    if (root._bokeh_is_loading > 0) {\n",
+              "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
+              "      return null;\n",
+              "    }\n",
+              "    if (js_urls == null || js_urls.length === 0) {\n",
+              "      run_callbacks();\n",
+              "      return null;\n",
+              "    }\n",
+              "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
+              "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
+              "\n",
+              "    function on_load() {\n",
+              "      root._bokeh_is_loading--;\n",
+              "      if (root._bokeh_is_loading === 0) {\n",
+              "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
+              "        run_callbacks()\n",
+              "      }\n",
+              "    }\n",
+              "\n",
+              "    function on_error() {\n",
+              "      console.error(\"failed to load \" + url);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < css_urls.length; i++) {\n",
+              "      var url = css_urls[i];\n",
+              "      const element = document.createElement(\"link\");\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.rel = \"stylesheet\";\n",
+              "      element.type = \"text/css\";\n",
+              "      element.href = url;\n",
+              "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
+              "      document.body.appendChild(element);\n",
+              "    }\n",
+              "\n",
+              "    for (var i = 0; i < js_urls.length; i++) {\n",
+              "      var url = js_urls[i];\n",
+              "      var element = document.createElement('script');\n",
+              "      element.onload = on_load;\n",
+              "      element.onerror = on_error;\n",
+              "      element.async = false;\n",
+              "      element.src = url;\n",
+              "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
+              "      document.head.appendChild(element);\n",
+              "    }\n",
+              "  };\n",
+              "\n",
+              "  function inject_raw_css(css) {\n",
+              "    const element = document.createElement(\"style\");\n",
+              "    element.appendChild(document.createTextNode(css));\n",
+              "    document.body.appendChild(element);\n",
+              "  }\n",
+              "\n",
+              "  \n",
+              "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n",
+              "  var css_urls = [];\n",
+              "  \n",
+              "\n",
+              "  var inline_js = [\n",
+              "    function(Bokeh) {\n",
+              "      Bokeh.set_log_level(\"info\");\n",
+              "    },\n",
+              "    function(Bokeh) {\n",
+              "    \n",
+              "    \n",
+              "    }\n",
+              "  ];\n",
+              "\n",
+              "  function run_inline_js() {\n",
+              "    \n",
+              "    if (root.Bokeh !== undefined || force === true) {\n",
+              "      \n",
+              "    for (var i = 0; i < inline_js.length; i++) {\n",
+              "      inline_js[i].call(root, root.Bokeh);\n",
+              "    }\n",
+              "    } else if (Date.now() < root._bokeh_timeout) {\n",
+              "      setTimeout(run_inline_js, 100);\n",
+              "    } else if (!root._bokeh_failed_load) {\n",
+              "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
+              "      root._bokeh_failed_load = true;\n",
+              "    } else if (force !== true) {\n",
+              "      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n",
+              "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
+              "    }\n",
+              "\n",
+              "  }\n",
+              "\n",
+              "  if (root._bokeh_is_loading === 0) {\n",
+              "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
+              "    run_inline_js();\n",
+              "  } else {\n",
+              "    load_libs(css_urls, js_urls, function() {\n",
+              "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
+              "      run_inline_js();\n",
+              "    });\n",
+              "  }\n",
+              "}(window));"
+            ],
+            "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"
\\n\"+\n     \"
\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"
\\n\"+\n     \"
    \\n\"+\n     \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n     \"
  • use INLINE resources instead, as so:
    • \\n\"+\n     \"
\\n\"+\n     \"\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"\\n\"+\n     \"
\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(null);\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  \n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.4.0.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.4.0.min.js\"];\n  var css_urls = [];\n  \n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    function(Bokeh) {\n    \n    \n    }\n  ];\n\n  function run_inline_js() {\n    \n    if (root.Bokeh !== undefined || force === true) {\n      \n    for (var i = 0; i < inline_js.length; i++) {\n      inline_js[i].call(root, root.Bokeh);\n    }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(null)).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "text/html": [
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "\n",
+              "  
\n"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        },
+        {
+          "data": {
+            "application/javascript": [
+              "(function(root) {\n",
+              "  function embed_document(root) {\n",
+              "    \n",
+              "  var docs_json = {\"3d4a0aff-b8c3-43fb-a5af-6dfe1b9d0e1f\":{\"roots\":{\"references\":[{\"attributes\":{\"above\":[{\"id\":\"1849\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1852\",\"type\":\"Grid\"},{\"id\":\"1856\",\"type\":\"Grid\"}],\"left\":[{\"id\":\"1853\",\"type\":\"CategoricalAxis\"}],\"min_border_right\":300,\"plot_width\":1200,\"renderers\":[{\"id\":\"1866\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1839\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1859\",\"type\":\"Toolbar\"},\"toolbar_location\":\"below\",\"x_range\":{\"id\":\"1841\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1845\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1843\",\"type\":\"FactorRange\"},\"y_scale\":{\"id\":\"1847\",\"type\":\"CategoricalScale\"}},\"id\":\"1838\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1925\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1927\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.7853981633974483,\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1850\",\"type\":\"CategoricalTicker\"}},\"id\":\"1849\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\u72d7\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\"]},\"id\":\"1841\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1929\",\"type\":\"Selection\"},{\"attributes\":{},\"id\":\"1850\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1927\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"ticker\":{\"id\":\"1850\",\"type\":\"CategoricalTicker\"}},\"id\":\"1852\",\"type\":\"Grid\"},{\"attributes\":{\"axis_line_color\":{\"value\":null},\"formatter\":{\"id\":\"1925\",\"type\":\"CategoricalTickFormatter\"},\"major_label_standoff\":16,\"major_label_text_font_size\":{\"value\":\"12pt\"},\"major_tick_line_color\":{\"value\":null},\"ticker\":{\"id\":\"1854\",\"type\":\"CategoricalTicker\"}},\"id\":\"1853\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"1854\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1854\",\"type\":\"CategoricalTicker\"}},\"id\":\"1856\",\"type\":\"Grid\"},{\"attributes\":{\"text\":\"Chinese-Korean Similarity\",\"text_font_size\":{\"value\":\"12pt\"}},\"id\":\"1839\",\"type\":\"Title\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1857\",\"type\":\"SaveTool\"},{\"id\":\"1858\",\"type\":\"HoverTool\"}]},\"id\":\"1859\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1857\",\"type\":\"SaveTool\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"pair\",\"@embeddings_1 ||| @embeddings_2\"],[\"sim\",\"@sim\"]]},\"id\":\"1858\",\"type\":\"HoverTool\"},{\"attributes\":{\"callback\":null,\"data\":{\"embeddings_1\":[\"\\u72d7\",\"\\u72d7\",\"\\u72d7\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u5c0f\\u72d7\\u5f88\\u597d\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\",\"\\u6211\\u559c\\u6b22\\u548c\\u6211\\u7684\\u72d7\\u4e00\\u8d77\\u6cbf\\u7740\\u6d77\\u6ee9\\u6563\\u6b65\\u3002\"],\"embeddings_2\":[\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac1c\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\"],\"index\":[0,1,2,3,4,5,6,7,8],\"sim\":{\"__ndarray__\":\"AAAAwIKP6z8AAACAHL7lPwAAAKDsSuA/AAAAoN0A5j8AAACgWsboPwAAAIANGeE/AAAAQMFJ4z8AAADA8D7jPwAAAABna+c/\",\"dtype\":\"float64\",\"shape\":[9]}},\"selected\":{\"id\":\"1929\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1930\",\"type\":\"UnionRenderers\"}},\"id\":\"1862\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"height\":{\"units\":\"data\",\"value\":1},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1865\",\"type\":\"Rect\"},{\"attributes\":{\"fill_color\":{\"field\":\"sim\",\"transform\":{\"id\":\"1837\",\"type\":\"LinearColorMapper\"}},\"height\":{\"units\":\"data\",\"value\":1},\"line_color\":{\"value\":null},\"width\":{\"units\":\"data\",\"value\":1},\"x\":{\"field\":\"embeddings_1\"},\"y\":{\"field\":\"embeddings_2\"}},\"id\":\"1864\",\"type\":\"Rect\"},{\"attributes\":{},\"id\":\"1847\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"data_source\":{\"id\":\"1862\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1864\",\"type\":\"Rect\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1865\",\"type\":\"Rect\"},\"selection_glyph\":null,\"view\":{\"id\":\"1867\",\"type\":\"CDSView\"}},\"id\":\"1866\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1862\",\"type\":\"ColumnDataSource\"}},\"id\":\"1867\",\"type\":\"CDSView\"},{\"attributes\":{\"high\":0.8612684011459351,\"low\":0.5091460347175598,\"palette\":[\"#ffffcc\",\"#ffeda0\",\"#fed976\",\"#feb24c\",\"#fd8d3c\",\"#fc4e2a\",\"#e31a1c\",\"#bd0026\",\"#800026\"]},\"id\":\"1837\",\"type\":\"LinearColorMapper\"},{\"attributes\":{\"callback\":null,\"factors\":[\"\\ub098\\ub294 \\ub098\\uc758 \\uc0b0\\ucc45\\uc744 \\ud574\\ubcc0\\uc744 \\ub530\\ub77c \\uae38\\uac8c \\uc0b0\\ucc45\\ud558\\ub294 \\uac83\\uc744 \\uc990\\uae34\\ub2e4.\",\"\\uac15\\uc544\\uc9c0\\uac00 \\uc88b\\ub2e4.\",\"\\uac1c\"]},\"id\":\"1843\",\"type\":\"FactorRange\"},{\"attributes\":{},\"id\":\"1845\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1930\",\"type\":\"UnionRenderers\"}],\"root_ids\":[\"1838\"]},\"title\":\"Bokeh Application\",\"version\":\"1.4.0\"}};\n",
+              "  var render_items = [{\"docid\":\"3d4a0aff-b8c3-43fb-a5af-6dfe1b9d0e1f\",\"roots\":{\"1838\":\"63952aa4-d54a-4445-ad10-ef5bef98f1ef\"}}];\n",
+              "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
+              "\n",
+              "  }\n",
+              "  if (root.Bokeh !== undefined) {\n",
+              "    embed_document(root);\n",
+              "  } else {\n",
+              "    var attempts = 0;\n",
+              "    var timer = setInterval(function(root) {\n",
+              "      if (root.Bokeh !== undefined) {\n",
+              "        clearInterval(timer);\n",
+              "        embed_document(root);\n",
+              "      } else {\n",
+              "        attempts++;\n",
+              "        if (attempts > 100) {\n",
+              "          clearInterval(timer);\n",
+              "          console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
+              "        }\n",
+              "      }\n",
+              "    }, 10, root)\n",
+              "  }\n",
+              "})(window);"
+            ],
+            "application/vnd.bokehjs_exec.v0+json": ""
+          },
+          "metadata": {
+            "application/vnd.bokehjs_exec.v0+json": {
+              "id": "1838"
+            },
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "visualize_similarity(zh_result, ko_result, chinese_sentences, korean_sentences, 'Chinese-Korean Similarity')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rRabHHQYQfLr"
+      },
+      "source": [
+        "### And more...\n",
+        "\n",
+        "The above examples can be extended to any language pair from **English, Arabic, Chinese, Dutch, French, German, Italian, Japanese, Korean, Polish, Portuguese, Russian, Spanish, Thai and Turkish**. Happy coding!"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mxAFAJI9xsAU"
+      },
+      "source": [
+        "# Creating a Multilingual Semantic-Similarity Search Engine\n",
+        "\n",
+        "Whereas in the previous example we visualized a handful of sentences, in this section we will build a semantic-search index of about 200,000 sentences from a Wikipedia Corpus. About half will be in English and the other half in Spanish to demonstrate the multilingual capabilities of the Universal Sentence Encoder.\n",
+        "\n",
+        "## Download Data to Index\n",
+        "First, we will download news sentences in multiples languages from the [News Commentary Corpus](http://opus.nlpl.eu/News-Commentary-v11.php) [1].  Without loss of generality, this approach should also work for indexing the rest of the supported languages.\n",
+        "\n",
+        "To speed up the demo, we limit to 1000 sentences per language."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "587I9ye6yXEU"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Downloading data from http://opus.nlpl.eu/download.php?f=News-Commentary/v11/moses/ar-en.txt.zip\n",
+            "24715264/24714354 [==============================] - 2s 0us/step\n",
+            "1,000 Arabic sentences\n",
+            "Downloading data from http://opus.nlpl.eu/download.php?f=News-Commentary/v11/moses/en-zh.txt.zip\n",
+            "18104320/18101984 [==============================] - 2s 0us/step\n",
+            "1,000 Chinese sentences\n",
+            "Downloading data from http://opus.nlpl.eu/download.php?f=News-Commentary/v11/moses/en-es.txt.zip\n",
+            "28106752/28106064 [==============================] - 2s 0us/step\n",
+            "1,000 English sentences\n",
+            "Downloading data from http://opus.nlpl.eu/download.php?f=News-Commentary/v11/moses/en-ru.txt.zip\n",
+            "24854528/24849511 [==============================] - 2s 0us/step\n",
+            "1,000 Russian sentences\n",
+            "1,000 Spanish sentences\n"
+          ]
+        }
+      ],
+      "source": [
+        "corpus_metadata = [\n",
+        "    ('ar', 'ar-en.txt.zip', 'News-Commentary.ar-en.ar', 'Arabic'),\n",
+        "    ('zh', 'en-zh.txt.zip', 'News-Commentary.en-zh.zh', 'Chinese'),\n",
+        "    ('en', 'en-es.txt.zip', 'News-Commentary.en-es.en', 'English'),\n",
+        "    ('ru', 'en-ru.txt.zip', 'News-Commentary.en-ru.ru', 'Russian'),\n",
+        "    ('es', 'en-es.txt.zip', 'News-Commentary.en-es.es', 'Spanish'),\n",
+        "]\n",
+        "\n",
+        "language_to_sentences = {}\n",
+        "language_to_news_path = {}\n",
+        "for language_code, zip_file, news_file, language_name in corpus_metadata:\n",
+        "  zip_path = tf.keras.utils.get_file(\n",
+        "      fname=zip_file,\n",
+        "      origin='http://opus.nlpl.eu/download.php?f=News-Commentary/v11/moses/' + zip_file,\n",
+        "      extract=True)\n",
+        "  news_path = os.path.join(os.path.dirname(zip_path), news_file)\n",
+        "  language_to_sentences[language_code] = pd.read_csv(news_path, sep='\\t', header=None)[0][:1000]\n",
+        "  language_to_news_path[language_code] = news_path\n",
+        "\n",
+        "  print('{:,} {} sentences'.format(len(language_to_sentences[language_code]), language_name))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "m3DIT9uT7Z34"
+      },
+      "source": [
+        "## Using a pre-trained model to transform sentences into vectors\n",
+        "\n",
+        "We compute embeddings in _batches_ so that they fit in the GPU's RAM."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yRoRT5qCEIYy"
+      },
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "\r  0%|          | 0/1000 [00:00\n",
+        "  \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  \n",
+        ""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "U5POcTVNB_dv"
+      },
+      "source": [
+        "# HRNet based model for semantic segmentation\n",
+        "\n",
+        "In this notebook, you will:\n",
+        "\n",
+        "- Choose and load one of the 17 pre-trained HRNet models on different semantic segmentation datasets\n",
+        "- Run inference to extract features from the model backbone and predictions from the model head"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_XgTpm9ZxoN9"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import matplotlib.pyplot as plt\n",
+        "from PIL import Image\n",
+        "import numpy as np"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UVtEyxDFpKE1"
+      },
+      "source": [
+        "## Loading models from TensorFlow Hub\n",
+        "\n",
+        "Here you can choose the pre-trained HRNet model to load, different models means a different training dataset used. All models have the same architecture, except for the model head, which has a different dimension based on the number of classes contained in the training dataset (dataset_output_classes). For more information about the different datasets we refer to the links above and the [factors of influence dataset collection](https://github.com/google-research/google-research/tree/master/factors_of_influence)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "y8_ctG55-uTX"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Choose a pre-trained HRNet model to load.\n",
+        "\n",
+        "hrnet_model_name = 'ade20k-hrnetv2-w48/1'  #@param [\"ade20k-hrnetv2-w48/1\", \"isprs-hrnetv2-w48/1\", \"vkitti2-hrnetv2-w48/1\", \"vgallery-hrnetv2-w48/1\", \"sunrgbd-hrnetv2-w48/1\", \"suim-hrnetv2-w48/1\", \"scannet-hrnetv2-w48/1\", \"pvoc-hrnetv2-w48/1\", \"msegpcontext-hrnetv2-w48/1\", \"mapillary-hrnetv2-w48/1\", \"kitti-hrnetv2-w48/1\", \"isaid-hrnetv2-w48/1\", \"idd-hrnetv2-w48/1\", \"coco-hrnetv2-w48/1\", \"city-hrnetv2-w48/1\", \"camvid-hrnetv2-w48/1\", \"bdd-hrnetv2-w48/1\"]\n",
+        "\n",
+        "tfhub_model_name = 'https://tfhub.dev/google/HRNet/' + hrnet_model_name\n",
+        "\n",
+        "print('HRNet model selected           :', tfhub_model_name)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "T-yHJ5X55kWN"
+      },
+      "outputs": [],
+      "source": [
+        "hrnet_model = hub.load(tfhub_model_name)\n",
+        "\n",
+        "print('HRNet model loaded           :', tfhub_model_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pMP_7v9x6kol"
+      },
+      "source": [
+        "## Loading an image and running inference\n",
+        "\n",
+        "This is a demonstration on how to run inference for extracting features and predictions from an image. The image was taken from the scene150 dataset.\n",
+        "\n",
+        "To perform inference on the datasets that were used during training we refer to the [factors of influence dataset collection](https://github.com/google-research/google-research/tree/master/factors_of_influence)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GNzjieS66td_"
+      },
+      "outputs": [],
+      "source": [
+        "img_file = tf.keras.utils.get_file(origin=\"https://tensorflow.org/images/bedroom_hrnet_tutorial.jpg\")\n",
+        "img = np.array(Image.open(img_file))/255.0"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Lp54vD_FZuHw"
+      },
+      "outputs": [],
+      "source": [
+        "plt.imshow(img)\n",
+        "plt.show()\n",
+        "\n",
+        "# Predictions will have shape (batch_size, h, w, dataset_output_classes)\n",
+        "predictions = hrnet_model.predict([img])\n",
+        "plt.imshow(predictions[0,:,:,1])\n",
+        "plt.title('Predictions for class #1')\n",
+        "plt.show() \n",
+        "# Features will have shape (batch_size, h/4, w/4, 720)\n",
+        "features = hrnet_model.get_features([img])\n",
+        "plt.imshow(features[0,:,:,1])\n",
+        "plt.title('Feature #1 out of 720')\n",
+        "plt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "hrnet_semantic_segmentation.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/image_classification.ipynb b/site/en/hub/tutorials/image_classification.ipynb
new file mode 100644
index 00000000000..91aadab727e
--- /dev/null
+++ b/site/en/hub/tutorials/image_classification.ipynb
@@ -0,0 +1,463 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ScitaPqhKtuW"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jvztxQ6VsK2k"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7t7KGfIwHaXz"
+      },
+      "source": [
+        "# Image Classification with TensorFlow Hub\n",
+        "\n",
+        "In this colab, you'll try multiple image classification models from TensorFlow Hub and decide which one is best for your use case.\n",
+        "\n",
+        "Because TF Hub encourages a [consistent input convention](https://www.tensorflow.org/hub/common_saved_model_apis/images#image_input) for models that operate on images, it's easy to experiment with different architectures to find the one that best fits your needs."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "N8H5ufxkc2mk"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "import requests\n",
+        "from PIL import Image\n",
+        "from io import BytesIO\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "oKvj6lY6kZx8"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Helper functions for loading image (hidden)\n",
+        "\n",
+        "original_image_cache = {}\n",
+        "\n",
+        "def preprocess_image(image):\n",
+        "  image = np.array(image)\n",
+        "  # reshape into shape [batch_size, height, width, num_channels]\n",
+        "  img_reshaped = tf.reshape(image, [1, image.shape[0], image.shape[1], image.shape[2]])\n",
+        "  # Use `convert_image_dtype` to convert to floats in the [0,1] range.\n",
+        "  image = tf.image.convert_image_dtype(img_reshaped, tf.float32)\n",
+        "  return image\n",
+        "\n",
+        "def load_image_from_url(img_url):\n",
+        "  \"\"\"Returns an image with shape [1, height, width, num_channels].\"\"\"\n",
+        "  user_agent = {'User-agent': 'Colab Sample (https://tensorflow.org)'}\n",
+        "  response = requests.get(img_url, headers=user_agent)\n",
+        "  image = Image.open(BytesIO(response.content))\n",
+        "  image = preprocess_image(image)\n",
+        "  return image\n",
+        "\n",
+        "def load_image(image_url, image_size=256, dynamic_size=False, max_dynamic_size=512):\n",
+        "  \"\"\"Loads and preprocesses images.\"\"\"\n",
+        "  # Cache image file locally.\n",
+        "  if image_url in original_image_cache:\n",
+        "    img = original_image_cache[image_url]\n",
+        "  elif image_url.startswith('https://'):\n",
+        "    img = load_image_from_url(image_url)\n",
+        "  else:\n",
+        "    fd = tf.io.gfile.GFile(image_url, 'rb')\n",
+        "    img = preprocess_image(Image.open(fd))\n",
+        "  original_image_cache[image_url] = img\n",
+        "  # Load and convert to float32 numpy array, add batch dimension, and normalize to range [0, 1].\n",
+        "  img_raw = img\n",
+        "  if tf.reduce_max(img) > 1.0:\n",
+        "    img = img / 255.\n",
+        "  if len(img.shape) == 3:\n",
+        "    img = tf.stack([img, img, img], axis=-1)\n",
+        "  if not dynamic_size:\n",
+        "    img = tf.image.resize_with_pad(img, image_size, image_size)\n",
+        "  elif img.shape[1] > max_dynamic_size or img.shape[2] > max_dynamic_size:\n",
+        "    img = tf.image.resize_with_pad(img, max_dynamic_size, max_dynamic_size)\n",
+        "  return img, img_raw\n",
+        "\n",
+        "def show_image(image, title=''):\n",
+        "  image_size = image.shape[1]\n",
+        "  w = (image_size * 6) // 320\n",
+        "  plt.figure(figsize=(w, w))\n",
+        "  plt.imshow(image[0], aspect='equal')\n",
+        "  plt.axis('off')\n",
+        "  plt.title(title)\n",
+        "  plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ws1AMDT_CDPq"
+      },
+      "source": [
+        "Select an Image Classification Model. After that, some internal variables are set and the labels file is downloaded and prepared for use.\n",
+        "\n",
+        "There are some technical differences between the models, like different input size, model size, accuracy, and inference time. Here you can change the model you are using until you find the one most suitable for your use case.\n",
+        "\n",
+        "The handle (url) of the model is printed for your convenience. More documentation about each model is available there.\n",
+        "\n",
+        "Note: All these models were trained on the ImageNet dataset"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iQ3aamrBfs-c"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Select an Image Classification model\n",
+        "\n",
+        "image_size = 224\n",
+        "dynamic_size = False\n",
+        "\n",
+        "model_name = \"efficientnetv2-s\" # @param ['efficientnetv2-s', 'efficientnetv2-m', 'efficientnetv2-l', 'efficientnetv2-s-21k', 'efficientnetv2-m-21k', 'efficientnetv2-l-21k', 'efficientnetv2-xl-21k', 'efficientnetv2-b0-21k', 'efficientnetv2-b1-21k', 'efficientnetv2-b2-21k', 'efficientnetv2-b3-21k', 'efficientnetv2-s-21k-ft1k', 'efficientnetv2-m-21k-ft1k', 'efficientnetv2-l-21k-ft1k', 'efficientnetv2-xl-21k-ft1k', 'efficientnetv2-b0-21k-ft1k', 'efficientnetv2-b1-21k-ft1k', 'efficientnetv2-b2-21k-ft1k', 'efficientnetv2-b3-21k-ft1k', 'efficientnetv2-b0', 'efficientnetv2-b1', 'efficientnetv2-b2', 'efficientnetv2-b3', 'efficientnet_b0', 'efficientnet_b1', 'efficientnet_b2', 'efficientnet_b3', 'efficientnet_b4', 'efficientnet_b5', 'efficientnet_b6', 'efficientnet_b7', 'bit_s-r50x1', 'inception_v3', 'inception_resnet_v2', 'resnet_v1_50', 'resnet_v1_101', 'resnet_v1_152', 'resnet_v2_50', 'resnet_v2_101', 'resnet_v2_152', 'nasnet_large', 'nasnet_mobile', 'pnasnet_large', 'mobilenet_v2_100_224', 'mobilenet_v2_130_224', 'mobilenet_v2_140_224', 'mobilenet_v3_small_100_224', 'mobilenet_v3_small_075_224', 'mobilenet_v3_large_100_224', 'mobilenet_v3_large_075_224']\n",
+        "\n",
+        "model_handle_map = {\n",
+        "  \"efficientnetv2-s\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_s/classification/2\",\n",
+        "  \"efficientnetv2-m\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_m/classification/2\",\n",
+        "  \"efficientnetv2-l\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_l/classification/2\",\n",
+        "  \"efficientnetv2-s-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_s/classification/2\",\n",
+        "  \"efficientnetv2-m-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_m/classification/2\",\n",
+        "  \"efficientnetv2-l-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_l/classification/2\",\n",
+        "  \"efficientnetv2-xl-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_xl/classification/2\",\n",
+        "  \"efficientnetv2-b0-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_b0/classification/2\",\n",
+        "  \"efficientnetv2-b1-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_b1/classification/2\",\n",
+        "  \"efficientnetv2-b2-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_b2/classification/2\",\n",
+        "  \"efficientnetv2-b3-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_b3/classification/2\",\n",
+        "  \"efficientnetv2-s-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_s/classification/2\",\n",
+        "  \"efficientnetv2-m-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_m/classification/2\",\n",
+        "  \"efficientnetv2-l-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_l/classification/2\",\n",
+        "  \"efficientnetv2-xl-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_xl/classification/2\",\n",
+        "  \"efficientnetv2-b0-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_b0/classification/2\",\n",
+        "  \"efficientnetv2-b1-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_b1/classification/2\",\n",
+        "  \"efficientnetv2-b2-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_b2/classification/2\",\n",
+        "  \"efficientnetv2-b3-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_b3/classification/2\",\n",
+        "  \"efficientnetv2-b0\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b0/classification/2\",\n",
+        "  \"efficientnetv2-b1\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b1/classification/2\",\n",
+        "  \"efficientnetv2-b2\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b2/classification/2\",\n",
+        "  \"efficientnetv2-b3\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b3/classification/2\",\n",
+        "  \"efficientnet_b0\": \"https://tfhub.dev/tensorflow/efficientnet/b0/classification/1\",\n",
+        "  \"efficientnet_b1\": \"https://tfhub.dev/tensorflow/efficientnet/b1/classification/1\",\n",
+        "  \"efficientnet_b2\": \"https://tfhub.dev/tensorflow/efficientnet/b2/classification/1\",\n",
+        "  \"efficientnet_b3\": \"https://tfhub.dev/tensorflow/efficientnet/b3/classification/1\",\n",
+        "  \"efficientnet_b4\": \"https://tfhub.dev/tensorflow/efficientnet/b4/classification/1\",\n",
+        "  \"efficientnet_b5\": \"https://tfhub.dev/tensorflow/efficientnet/b5/classification/1\",\n",
+        "  \"efficientnet_b6\": \"https://tfhub.dev/tensorflow/efficientnet/b6/classification/1\",\n",
+        "  \"efficientnet_b7\": \"https://tfhub.dev/tensorflow/efficientnet/b7/classification/1\",\n",
+        "  \"bit_s-r50x1\": \"https://tfhub.dev/google/bit/s-r50x1/ilsvrc2012_classification/1\",\n",
+        "  \"inception_v3\": \"https://tfhub.dev/google/imagenet/inception_v3/classification/4\",\n",
+        "  \"inception_resnet_v2\": \"https://tfhub.dev/google/imagenet/inception_resnet_v2/classification/4\",\n",
+        "  \"resnet_v1_50\": \"https://tfhub.dev/google/imagenet/resnet_v1_50/classification/4\",\n",
+        "  \"resnet_v1_101\": \"https://tfhub.dev/google/imagenet/resnet_v1_101/classification/4\",\n",
+        "  \"resnet_v1_152\": \"https://tfhub.dev/google/imagenet/resnet_v1_152/classification/4\",\n",
+        "  \"resnet_v2_50\": \"https://tfhub.dev/google/imagenet/resnet_v2_50/classification/4\",\n",
+        "  \"resnet_v2_101\": \"https://tfhub.dev/google/imagenet/resnet_v2_101/classification/4\",\n",
+        "  \"resnet_v2_152\": \"https://tfhub.dev/google/imagenet/resnet_v2_152/classification/4\",\n",
+        "  \"nasnet_large\": \"https://tfhub.dev/google/imagenet/nasnet_large/classification/4\",\n",
+        "  \"nasnet_mobile\": \"https://tfhub.dev/google/imagenet/nasnet_mobile/classification/4\",\n",
+        "  \"pnasnet_large\": \"https://tfhub.dev/google/imagenet/pnasnet_large/classification/4\",\n",
+        "  \"mobilenet_v2_100_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/classification/4\",\n",
+        "  \"mobilenet_v2_130_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v2_130_224/classification/4\",\n",
+        "  \"mobilenet_v2_140_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v2_140_224/classification/4\",\n",
+        "  \"mobilenet_v3_small_100_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v3_small_100_224/classification/5\",\n",
+        "  \"mobilenet_v3_small_075_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v3_small_075_224/classification/5\",\n",
+        "  \"mobilenet_v3_large_100_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v3_large_100_224/classification/5\",\n",
+        "  \"mobilenet_v3_large_075_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v3_large_075_224/classification/5\",\n",
+        "}\n",
+        "\n",
+        "model_image_size_map = {\n",
+        "  \"efficientnetv2-s\": 384,\n",
+        "  \"efficientnetv2-m\": 480,\n",
+        "  \"efficientnetv2-l\": 480,\n",
+        "  \"efficientnetv2-b0\": 224,\n",
+        "  \"efficientnetv2-b1\": 240,\n",
+        "  \"efficientnetv2-b2\": 260,\n",
+        "  \"efficientnetv2-b3\": 300,\n",
+        "  \"efficientnetv2-s-21k\": 384,\n",
+        "  \"efficientnetv2-m-21k\": 480,\n",
+        "  \"efficientnetv2-l-21k\": 480,\n",
+        "  \"efficientnetv2-xl-21k\": 512,\n",
+        "  \"efficientnetv2-b0-21k\": 224,\n",
+        "  \"efficientnetv2-b1-21k\": 240,\n",
+        "  \"efficientnetv2-b2-21k\": 260,\n",
+        "  \"efficientnetv2-b3-21k\": 300,\n",
+        "  \"efficientnetv2-s-21k-ft1k\": 384,\n",
+        "  \"efficientnetv2-m-21k-ft1k\": 480,\n",
+        "  \"efficientnetv2-l-21k-ft1k\": 480,\n",
+        "  \"efficientnetv2-xl-21k-ft1k\": 512,\n",
+        "  \"efficientnetv2-b0-21k-ft1k\": 224,\n",
+        "  \"efficientnetv2-b1-21k-ft1k\": 240,\n",
+        "  \"efficientnetv2-b2-21k-ft1k\": 260,\n",
+        "  \"efficientnetv2-b3-21k-ft1k\": 300, \n",
+        "  \"efficientnet_b0\": 224,\n",
+        "  \"efficientnet_b1\": 240,\n",
+        "  \"efficientnet_b2\": 260,\n",
+        "  \"efficientnet_b3\": 300,\n",
+        "  \"efficientnet_b4\": 380,\n",
+        "  \"efficientnet_b5\": 456,\n",
+        "  \"efficientnet_b6\": 528,\n",
+        "  \"efficientnet_b7\": 600,\n",
+        "  \"inception_v3\": 299,\n",
+        "  \"inception_resnet_v2\": 299,\n",
+        "  \"mobilenet_v2_100_224\": 224,\n",
+        "  \"mobilenet_v2_130_224\": 224,\n",
+        "  \"mobilenet_v2_140_224\": 224,\n",
+        "  \"nasnet_large\": 331,\n",
+        "  \"nasnet_mobile\": 224,\n",
+        "  \"pnasnet_large\": 331,\n",
+        "  \"resnet_v1_50\": 224,\n",
+        "  \"resnet_v1_101\": 224,\n",
+        "  \"resnet_v1_152\": 224,\n",
+        "  \"resnet_v2_50\": 224,\n",
+        "  \"resnet_v2_101\": 224,\n",
+        "  \"resnet_v2_152\": 224,\n",
+        "  \"mobilenet_v3_small_100_224\": 224,\n",
+        "  \"mobilenet_v3_small_075_224\": 224,\n",
+        "  \"mobilenet_v3_large_100_224\": 224,\n",
+        "  \"mobilenet_v3_large_075_224\": 224,\n",
+        "}\n",
+        "\n",
+        "model_handle = model_handle_map[model_name]\n",
+        "\n",
+        "print(f\"Selected model: {model_name} : {model_handle}\")\n",
+        "\n",
+        "\n",
+        "max_dynamic_size = 512\n",
+        "if model_name in model_image_size_map:\n",
+        "  image_size = model_image_size_map[model_name]\n",
+        "  dynamic_size = False\n",
+        "  print(f\"Images will be converted to {image_size}x{image_size}\")\n",
+        "else:\n",
+        "  dynamic_size = True\n",
+        "  print(f\"Images will be capped to a max size of {max_dynamic_size}x{max_dynamic_size}\")\n",
+        "\n",
+        "labels_file = \"https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt\"\n",
+        "\n",
+        "#download labels and creates a maps\n",
+        "downloaded_file = tf.keras.utils.get_file(\"labels.txt\", origin=labels_file)\n",
+        "\n",
+        "classes = []\n",
+        "\n",
+        "with open(downloaded_file) as f:\n",
+        "  labels = f.readlines()\n",
+        "  classes = [l.strip() for l in labels]\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vxcASidjBAE8"
+      },
+      "source": [
+        "You can select one of the images below, or use your own image. Just remember that the input size for the models vary and some of them use a dynamic input size (enabling inference on the unscaled image). Given that, the method `load_image` will already rescale the image to the expected format."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "o2rMsr4CgET2"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Select an Input Image\n",
+        "\n",
+        "image_name = \"turtle\" # @param ['tiger', 'bus', 'car', 'cat', 'dog', 'apple', 'banana', 'turtle', 'flamingo', 'piano', 'honeycomb', 'teapot']\n",
+        "\n",
+        "images_for_test_map = {\n",
+        "    \"tiger\": \"https://upload.wikimedia.org/wikipedia/commons/b/b0/Bengal_tiger_%28Panthera_tigris_tigris%29_female_3_crop.jpg\",\n",
+        "    #by Charles James Sharp, CC BY-SA 4.0 , via Wikimedia Commons\n",
+        "    \"bus\": \"https://upload.wikimedia.org/wikipedia/commons/6/63/LT_471_%28LTZ_1471%29_Arriva_London_New_Routemaster_%2819522859218%29.jpg\",\n",
+        "    #by Martin49 from London, England, CC BY 2.0 , via Wikimedia Commons\n",
+        "    \"car\": \"https://upload.wikimedia.org/wikipedia/commons/4/49/2013-2016_Toyota_Corolla_%28ZRE172R%29_SX_sedan_%282018-09-17%29_01.jpg\",\n",
+        "    #by EurovisionNim, CC BY-SA 4.0 , via Wikimedia Commons\n",
+        "    \"cat\": \"https://upload.wikimedia.org/wikipedia/commons/4/4d/Cat_November_2010-1a.jpg\",\n",
+        "    #by Alvesgaspar, CC BY-SA 3.0 , via Wikimedia Commons\n",
+        "    \"dog\": \"https://upload.wikimedia.org/wikipedia/commons/archive/a/a9/20090914031557%21Saluki_dog_breed.jpg\",\n",
+        "    #by Craig Pemberton, CC BY-SA 3.0 , via Wikimedia Commons\n",
+        "    \"apple\": \"https://upload.wikimedia.org/wikipedia/commons/1/15/Red_Apple.jpg\",\n",
+        "    #by Abhijit Tembhekar from Mumbai, India, CC BY 2.0 , via Wikimedia Commons\n",
+        "    \"banana\": \"https://upload.wikimedia.org/wikipedia/commons/1/1c/Bananas_white_background.jpg\",\n",
+        "    #by fir0002  flagstaffotos [at] gmail.com\t\tCanon 20D + Tamron 28-75mm f/2.8, GFDL 1.2 , via Wikimedia Commons\n",
+        "    \"turtle\": \"https://upload.wikimedia.org/wikipedia/commons/8/80/Turtle_golfina_escobilla_oaxaca_mexico_claudio_giovenzana_2010.jpg\",\n",
+        "    #by Claudio Giovenzana, CC BY-SA 3.0 , via Wikimedia Commons\n",
+        "    \"flamingo\": \"https://upload.wikimedia.org/wikipedia/commons/b/b8/James_Flamingos_MC.jpg\",\n",
+        "    #by Christian Mehlführer, User:Chmehl, CC BY 3.0 , via Wikimedia Commons\n",
+        "    \"piano\": \"https://upload.wikimedia.org/wikipedia/commons/d/da/Steinway_%26_Sons_upright_piano%2C_model_K-132%2C_manufactured_at_Steinway%27s_factory_in_Hamburg%2C_Germany.png\",\n",
+        "    #by \"Photo: © Copyright Steinway & Sons\", CC BY-SA 3.0 , via Wikimedia Commons\n",
+        "    \"honeycomb\": \"https://upload.wikimedia.org/wikipedia/commons/f/f7/Honey_comb.jpg\",\n",
+        "    #by Merdal, CC BY-SA 3.0 , via Wikimedia Commons\n",
+        "    \"teapot\": \"https://upload.wikimedia.org/wikipedia/commons/4/44/Black_tea_pot_cropped.jpg\",\n",
+        "    #by Mendhak, CC BY-SA 2.0 , via Wikimedia Commons\n",
+        "}\n",
+        "\n",
+        "img_url = images_for_test_map[image_name]\n",
+        "image, original_image = load_image(img_url, image_size, dynamic_size, max_dynamic_size)\n",
+        "show_image(image, 'Scaled image')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CMwWx8_8Aw3X"
+      },
+      "source": [
+        "Now that the model was chosen, loading it with TensorFlow Hub is simple.\n",
+        "\n",
+        "This also calls the model with a random input as a \"warmup\" run. Subsequent calls are often much faster, and you can compare this with the latency below.\n",
+        "\n",
+        "*Note:* models that use a dynamic size might need a fresh \"warmup\" run for each image size."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LRAccT3UhRga"
+      },
+      "outputs": [],
+      "source": [
+        "classifier = hub.load(model_handle)\n",
+        "\n",
+        "input_shape = image.shape\n",
+        "warmup_input = tf.random.uniform(input_shape, 0, 1.0)\n",
+        "%time warmup_logits = classifier(warmup_input).numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e7vkdUqpBkfE"
+      },
+      "source": [
+        "Everything is ready for inference. Here you can see the top 5 results from the model for the selected image."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "I0QNHg3bk-G1"
+      },
+      "outputs": [],
+      "source": [
+        "# Run model on image\n",
+        "%time probabilities = tf.nn.softmax(classifier(image)).numpy()\n",
+        "\n",
+        "top_5 = tf.argsort(probabilities, axis=-1, direction=\"DESCENDING\")[0][:5].numpy()\n",
+        "np_classes = np.array(classes)\n",
+        "\n",
+        "# Some models include an additional 'background' class in the predictions, so\n",
+        "# we must account for this when reading the class labels.\n",
+        "includes_background_class = probabilities.shape[1] == 1001\n",
+        "\n",
+        "for i, item in enumerate(top_5):\n",
+        "  class_index = item if includes_background_class else item + 1\n",
+        "  line = f'({i+1}) {class_index:4} - {classes[class_index]}: {probabilities[0][top_5][i]}'\n",
+        "  print(line)\n",
+        "\n",
+        "show_image(image, '')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e4IJrq5eZDWl"
+      },
+      "source": [
+        "## Learn More\n",
+        "\n",
+        "If you want to learn more and try how to do Transfer Learning with these models you can try this tutorial: [Transfer Learning for Image classification](https://www.tensorflow.org/hub/tutorials/tf2_image_retraining) \n",
+        "\n",
+        "If you want to check on more image models you can check them out on [tfhub.dev](https://tfhub.dev/s?module-type=image-augmentation,image-classification,image-classification-logits,image-classifier,image-feature-vector,image-generator,image-object-detection,image-others,image-pose-detection,image-segmentation,image-style-transfer,image-super-resolution,image-rnn-agent)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "image_classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/image_enhancing.ipynb b/site/en/hub/tutorials/image_enhancing.ipynb
new file mode 100644
index 00000000000..3710ebd6d66
--- /dev/null
+++ b/site/en/hub/tutorials/image_enhancing.ipynb
@@ -0,0 +1,455 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GeerbrLA0uju"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "\n",
+        "Created by @[Adrish Dey](https://github.com/captain-pool) for [Google Summer of Code](https://summerofcode.withgoogle.com/) 2019"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yntM0JbY0uj5"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS, \n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UJeo2a5C0uj2"
+      },
+      "source": [
+        "# Image Super Resolution using ESRGAN"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ps4toA1d_tkc"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LkW9jAmt_zjB"
+      },
+      "source": [
+        "This colab demonstrates use of TensorFlow Hub Module for Enhanced Super Resolution Generative Adversarial Network (*by Xintao Wang et.al.*) [[Paper](https://arxiv.org/pdf/1809.00219.pdf)] [[Code](https://github.com/captain-pool/GSOC/)]\n",
+        "\n",
+        "for image enhancing. *(Preferrably bicubically downsampled images).*\n",
+        "\n",
+        "Model trained on DIV2K Dataset (on bicubically downsampled images) on image patches of size 128 x 128."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LBGty4O_0ukJ"
+      },
+      "source": [
+        "**Preparing Environment**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lnyLTyUt0ukN"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import time\n",
+        "from PIL import Image\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import matplotlib.pyplot as plt\n",
+        "os.environ[\"TFHUB_DOWNLOAD_PROGRESS\"] = \"True\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dremsFdh0ukX"
+      },
+      "outputs": [],
+      "source": [
+        "!wget \"https://user-images.githubusercontent.com/12981474/40157448-eff91f06-5953-11e8-9a37-f6b5693fa03f.png\" -O original.png"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DXot2kru0ukh"
+      },
+      "outputs": [],
+      "source": [
+        "# Declaring Constants\n",
+        "IMAGE_PATH = \"original.png\"\n",
+        "SAVED_MODEL_PATH = \"https://tfhub.dev/captain-pool/esrgan-tf2/1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KF_tHde-p3rn"
+      },
+      "source": [
+        "**Defining Helper Functions**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IslbQmTj0ukz"
+      },
+      "outputs": [],
+      "source": [
+        "def preprocess_image(image_path):\n",
+        "  \"\"\" Loads image from path and preprocesses to make it model ready\n",
+        "      Args:\n",
+        "        image_path: Path to the image file\n",
+        "  \"\"\"\n",
+        "  hr_image = tf.image.decode_image(tf.io.read_file(image_path))\n",
+        "  # If PNG, remove the alpha channel. The model only supports\n",
+        "  # images with 3 color channels.\n",
+        "  if hr_image.shape[-1] == 4:\n",
+        "    hr_image = hr_image[...,:-1]\n",
+        "  hr_size = (tf.convert_to_tensor(hr_image.shape[:-1]) // 4) * 4\n",
+        "  hr_image = tf.image.crop_to_bounding_box(hr_image, 0, 0, hr_size[0], hr_size[1])\n",
+        "  hr_image = tf.cast(hr_image, tf.float32)\n",
+        "  return tf.expand_dims(hr_image, 0)\n",
+        "\n",
+        "def save_image(image, filename):\n",
+        "  \"\"\"\n",
+        "    Saves unscaled Tensor Images.\n",
+        "    Args:\n",
+        "      image: 3D image tensor. [height, width, channels]\n",
+        "      filename: Name of the file to save.\n",
+        "  \"\"\"\n",
+        "  if not isinstance(image, Image.Image):\n",
+        "    image = tf.clip_by_value(image, 0, 255)\n",
+        "    image = Image.fromarray(tf.cast(image, tf.uint8).numpy())\n",
+        "  image.save(\"%s.jpg\" % filename)\n",
+        "  print(\"Saved as %s.jpg\" % filename)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uh1E2rBpnWxV"
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline\n",
+        "def plot_image(image, title=\"\"):\n",
+        "  \"\"\"\n",
+        "    Plots images from image tensors.\n",
+        "    Args:\n",
+        "      image: 3D image tensor. [height, width, channels].\n",
+        "      title: Title to display in the plot.\n",
+        "  \"\"\"\n",
+        "  image = np.asarray(image)\n",
+        "  image = tf.clip_by_value(image, 0, 255)\n",
+        "  image = Image.fromarray(tf.cast(image, tf.uint8).numpy())\n",
+        "  plt.imshow(image)\n",
+        "  plt.axis(\"off\")\n",
+        "  plt.title(title)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ycrCTvmlqBMD"
+      },
+      "source": [
+        "#### Performing Super Resolution of images loaded from path"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "L7XpMk8Y0uk7"
+      },
+      "outputs": [],
+      "source": [
+        "hr_image = preprocess_image(IMAGE_PATH)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hWgCbUa_0ulG"
+      },
+      "outputs": [],
+      "source": [
+        "# Plotting Original Resolution image\n",
+        "plot_image(tf.squeeze(hr_image), title=\"Original Image\")\n",
+        "save_image(tf.squeeze(hr_image), filename=\"Original Image\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ouwEyKLA0ulO"
+      },
+      "outputs": [],
+      "source": [
+        "model = hub.load(SAVED_MODEL_PATH)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dz79ncnT0ulX"
+      },
+      "outputs": [],
+      "source": [
+        "start = time.time()\n",
+        "fake_image = model(hr_image)\n",
+        "fake_image = tf.squeeze(fake_image)\n",
+        "print(\"Time Taken: %f\" % (time.time() - start))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ABjkkJHC2jNL"
+      },
+      "outputs": [],
+      "source": [
+        "# Plotting Super Resolution Image\n",
+        "plot_image(tf.squeeze(fake_image), title=\"Super Resolution\")\n",
+        "save_image(tf.squeeze(fake_image), filename=\"Super Resolution\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tuKu18UYptkx"
+      },
+      "source": [
+        "### Evaluating Performance of the Model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Qdz55sxMgiwO"
+      },
+      "outputs": [],
+      "source": [
+        "!wget \"https://lh4.googleusercontent.com/-Anmw5df4gj0/AAAAAAAAAAI/AAAAAAAAAAc/6HxU8XFLnQE/photo.jpg64\" -O test.jpg\n",
+        "IMAGE_PATH = \"test.jpg\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "F6tMNtqy0ukq"
+      },
+      "outputs": [],
+      "source": [
+        "# Defining helper functions\n",
+        "def downscale_image(image):\n",
+        "  \"\"\"\n",
+        "      Scales down images using bicubic downsampling.\n",
+        "      Args:\n",
+        "          image: 3D or 4D tensor of preprocessed image\n",
+        "  \"\"\"\n",
+        "  image_size = []\n",
+        "  if len(image.shape) == 3:\n",
+        "    image_size = [image.shape[1], image.shape[0]]\n",
+        "  else:\n",
+        "    raise ValueError(\"Dimension mismatch. Can work only on single image.\")\n",
+        "\n",
+        "  image = tf.squeeze(\n",
+        "      tf.cast(\n",
+        "          tf.clip_by_value(image, 0, 255), tf.uint8))\n",
+        "\n",
+        "  lr_image = np.asarray(\n",
+        "    Image.fromarray(image.numpy())\n",
+        "    .resize([image_size[0] // 4, image_size[1] // 4],\n",
+        "              Image.BICUBIC))\n",
+        "\n",
+        "  lr_image = tf.expand_dims(lr_image, 0)\n",
+        "  lr_image = tf.cast(lr_image, tf.float32)\n",
+        "  return lr_image"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r2ANR1XDy77I"
+      },
+      "outputs": [],
+      "source": [
+        "hr_image = preprocess_image(IMAGE_PATH)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r_defaultO6qbTV"
+      },
+      "outputs": [],
+      "source": [
+        "lr_image = downscale_image(tf.squeeze(hr_image))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jRw1x6xY0ulj"
+      },
+      "outputs": [],
+      "source": [
+        "# Plotting Low Resolution Image\n",
+        "plot_image(tf.squeeze(lr_image), title=\"Low Resolution\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "g--yyHg7qXCw"
+      },
+      "outputs": [],
+      "source": [
+        "model = hub.load(SAVED_MODEL_PATH)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZX-deZlhqaYz"
+      },
+      "outputs": [],
+      "source": [
+        "start = time.time()\n",
+        "fake_image = model(lr_image)\n",
+        "fake_image = tf.squeeze(fake_image)\n",
+        "print(\"Time Taken: %f\" % (time.time() - start))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AmSga6MSq1PB"
+      },
+      "outputs": [],
+      "source": [
+        "plot_image(tf.squeeze(fake_image), title=\"Super Resolution\")\n",
+        "# Calculating PSNR wrt Original Image\n",
+        "psnr = tf.image.psnr(\n",
+        "    tf.clip_by_value(fake_image, 0, 255),\n",
+        "    tf.clip_by_value(hr_image, 0, 255), max_val=255)\n",
+        "print(\"PSNR Achieved: %f\" % psnr)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5YTBKCXPq9UZ"
+      },
+      "source": [
+        "**Comparing Outputs size by side.**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ubdupldDypCy"
+      },
+      "outputs": [],
+      "source": [
+        "plt.rcParams['figure.figsize'] = [15, 10]\n",
+        "fig, axes = plt.subplots(1, 3)\n",
+        "fig.tight_layout()\n",
+        "plt.subplot(131)\n",
+        "plot_image(tf.squeeze(hr_image), title=\"Original\")\n",
+        "plt.subplot(132)\n",
+        "fig.tight_layout()\n",
+        "plot_image(tf.squeeze(lr_image), \"x4 Bicubic\")\n",
+        "plt.subplot(133)\n",
+        "fig.tight_layout()\n",
+        "plot_image(tf.squeeze(fake_image), \"Super Resolution\")\n",
+        "plt.savefig(\"ESRGAN_DIV2K.jpg\", bbox_inches=\"tight\")\n",
+        "print(\"PSNR: %f\" % psnr)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "image_enhancing.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/image_feature_vector.ipynb b/site/en/hub/tutorials/image_feature_vector.ipynb
new file mode 100644
index 00000000000..b5283c45b3d
--- /dev/null
+++ b/site/en/hub/tutorials/image_feature_vector.ipynb
@@ -0,0 +1,533 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ScitaPqhKtuW"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bNnChGfZK2_w"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9Z_ZvMk5JPFV"
+      },
+      "source": [
+        "# Classify Flowers with Transfer Learning\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gh-LWtlqLtgH"
+      },
+      "source": [
+        "Have you ever seen a beautiful flower and wondered what kind of flower it is? Well, you're not the first, so let's build a way to identify the type of flower from a photo!\n",
+        "\n",
+        "For classifying images, a particular type of *deep neural network*, called a *convolutional neural network* has proved to be particularly powerful. However, modern convolutional neural networks have millions of parameters. Training them from scratch requires a lot of labeled training data and a lot of computing power (hundreds of GPU-hours or more). We only have about three thousand labeled photos and want to spend much less time, so we need to be more clever.\n",
+        "\n",
+        "We will use a technique called *transfer learning* where we take a pre-trained network (trained on about a million general images), use it to extract features, and train a new layer on top for our own task of classifying images of flowers.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ORy-KvWXGXBo"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NTrs9zBKJK1c"
+      },
+      "outputs": [],
+      "source": [
+        "import collections\n",
+        "import io\n",
+        "import math\n",
+        "import os\n",
+        "import random\n",
+        "from six.moves import urllib\n",
+        "\n",
+        "from IPython.display import clear_output, Image, display, HTML\n",
+        "\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "tf.disable_v2_behavior()\n",
+        "\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "import numpy as np\n",
+        "import matplotlib.pyplot as plt\n",
+        "import seaborn as sns\n",
+        "import sklearn.metrics as sk_metrics\n",
+        "import time"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Do-T63G7NCSB"
+      },
+      "source": [
+        "## The flowers dataset\n",
+        "\n",
+        "The flowers dataset consists of images of flowers with 5 possible class labels.\n",
+        "\n",
+        "When training a machine learning model, we split our data into training and test datasets. We will train the model on our training data and then evaluate how well the model performs on data it has never seen - the test set.\n",
+        "\n",
+        "Let's download our training and test examples (it may take a while) and split them into train and test sets.\n",
+        "\n",
+        "Run the following two cells:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "HYQr1SILIxSK"
+      },
+      "outputs": [],
+      "source": [
+        "FLOWERS_DIR = './flower_photos'\n",
+        "TRAIN_FRACTION = 0.8\n",
+        "RANDOM_SEED = 2018\n",
+        "\n",
+        "\n",
+        "def download_images():\n",
+        "  \"\"\"If the images aren't already downloaded, save them to FLOWERS_DIR.\"\"\"\n",
+        "  if not os.path.exists(FLOWERS_DIR):\n",
+        "    DOWNLOAD_URL = 'http://download.tensorflow.org/example_images/flower_photos.tgz'\n",
+        "    print('Downloading flower images from %s...' % DOWNLOAD_URL)\n",
+        "    urllib.request.urlretrieve(DOWNLOAD_URL, 'flower_photos.tgz')\n",
+        "    !tar xfz flower_photos.tgz\n",
+        "  print('Flower photos are located in %s' % FLOWERS_DIR)\n",
+        "\n",
+        "\n",
+        "def make_train_and_test_sets():\n",
+        "  \"\"\"Split the data into train and test sets and get the label classes.\"\"\"\n",
+        "  train_examples, test_examples = [], []\n",
+        "  shuffler = random.Random(RANDOM_SEED)\n",
+        "  is_root = True\n",
+        "  for (dirname, subdirs, filenames) in tf.gfile.Walk(FLOWERS_DIR):\n",
+        "    # The root directory gives us the classes\n",
+        "    if is_root:\n",
+        "      subdirs = sorted(subdirs)\n",
+        "      classes = collections.OrderedDict(enumerate(subdirs))\n",
+        "      label_to_class = dict([(x, i) for i, x in enumerate(subdirs)])\n",
+        "      is_root = False\n",
+        "    # The sub directories give us the image files for training.\n",
+        "    else:\n",
+        "      filenames.sort()\n",
+        "      shuffler.shuffle(filenames)\n",
+        "      full_filenames = [os.path.join(dirname, f) for f in filenames]\n",
+        "      label = dirname.split('/')[-1]\n",
+        "      label_class = label_to_class[label]\n",
+        "      # An example is the image file and it's label class.\n",
+        "      examples = list(zip(full_filenames, [label_class] * len(filenames)))\n",
+        "      num_train = int(len(filenames) * TRAIN_FRACTION)\n",
+        "      train_examples.extend(examples[:num_train])\n",
+        "      test_examples.extend(examples[num_train:])\n",
+        "\n",
+        "  shuffler.shuffle(train_examples)\n",
+        "  shuffler.shuffle(test_examples)\n",
+        "  return train_examples, test_examples, classes\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_9NklpcANhtB"
+      },
+      "outputs": [],
+      "source": [
+        "# Download the images and split the images into train and test sets.\n",
+        "download_images()\n",
+        "TRAIN_EXAMPLES, TEST_EXAMPLES, CLASSES = make_train_and_test_sets()\n",
+        "NUM_CLASSES = len(CLASSES)\n",
+        "\n",
+        "print('\\nThe dataset has %d label classes: %s' % (NUM_CLASSES, CLASSES.values()))\n",
+        "print('There are %d training images' % len(TRAIN_EXAMPLES))\n",
+        "print('there are %d test images' % len(TEST_EXAMPLES))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tHF7bHTfnD6S"
+      },
+      "source": [
+        "## Explore the data\n",
+        "\n",
+        "The flowers dataset consists of examples which are labeled images of flowers. Each example contains a JPEG flower image and the class label: what type of flower it is. Let's display a few images together with their labels."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "1friUvN6kPYM"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Show some labeled images\n",
+        "def get_label(example):\n",
+        "  \"\"\"Get the label (number) for given example.\"\"\"\n",
+        "  return example[1]\n",
+        "\n",
+        "def get_class(example):\n",
+        "  \"\"\"Get the class (string) of given example.\"\"\"\n",
+        "  return CLASSES[get_label(example)]\n",
+        "\n",
+        "def get_encoded_image(example):\n",
+        "  \"\"\"Get the image data (encoded jpg) of given example.\"\"\"\n",
+        "  image_path = example[0]\n",
+        "  return tf.gfile.GFile(image_path, 'rb').read()\n",
+        "\n",
+        "def get_image(example):\n",
+        "  \"\"\"Get image as np.array of pixels for given example.\"\"\"\n",
+        "  return plt.imread(io.BytesIO(get_encoded_image(example)), format='jpg')\n",
+        "\n",
+        "def display_images(images_and_classes, cols=5):\n",
+        "  \"\"\"Display given images and their labels in a grid.\"\"\"\n",
+        "  rows = int(math.ceil(len(images_and_classes) / cols))\n",
+        "  fig = plt.figure()\n",
+        "  fig.set_size_inches(cols * 3, rows * 3)\n",
+        "  for i, (image, flower_class) in enumerate(images_and_classes):\n",
+        "    plt.subplot(rows, cols, i + 1)\n",
+        "    plt.axis('off')\n",
+        "    plt.imshow(image)\n",
+        "    plt.title(flower_class)\n",
+        "\n",
+        "NUM_IMAGES = 15 #@param {type: 'integer'}\n",
+        "display_images([(get_image(example), get_class(example))\n",
+        "               for example in TRAIN_EXAMPLES[:NUM_IMAGES]])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Hyjr6PuboTAg"
+      },
+      "source": [
+        "## Build the model\n",
+        "\n",
+        "We will load a [TF-Hub](https://tensorflow.org/hub) image feature vector module, stack a linear classifier on it, and add training and evaluation ops. The following cell builds a TF graph describing the model and its training, but it doesn't run the training (that will be the next step)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LbkSRaK_oW5Y"
+      },
+      "outputs": [],
+      "source": [
+        "LEARNING_RATE = 0.01\n",
+        "\n",
+        "tf.reset_default_graph()\n",
+        "\n",
+        "# Load a pre-trained TF-Hub module for extracting features from images. We've\n",
+        "# chosen this particular module for speed, but many other choices are available.\n",
+        "image_module = hub.Module('https://tfhub.dev/google/imagenet/mobilenet_v2_035_128/feature_vector/2')\n",
+        "\n",
+        "# Preprocessing images into tensors with size expected by the image module.\n",
+        "encoded_images = tf.placeholder(tf.string, shape=[None])\n",
+        "image_size = hub.get_expected_image_size(image_module)\n",
+        "\n",
+        "\n",
+        "def decode_and_resize_image(encoded):\n",
+        "  decoded = tf.image.decode_jpeg(encoded, channels=3)\n",
+        "  decoded = tf.image.convert_image_dtype(decoded, tf.float32)\n",
+        "  return tf.image.resize_images(decoded, image_size)\n",
+        "\n",
+        "\n",
+        "batch_images = tf.map_fn(decode_and_resize_image, encoded_images, dtype=tf.float32)\n",
+        "\n",
+        "# The image module can be applied as a function to extract feature vectors for a\n",
+        "# batch of images.\n",
+        "features = image_module(batch_images)\n",
+        "\n",
+        "\n",
+        "def create_model(features):\n",
+        "  \"\"\"Build a model for classification from extracted features.\"\"\"\n",
+        "  # Currently, the model is just a single linear layer. You can try to add\n",
+        "  # another layer, but be careful... two linear layers (when activation=None)\n",
+        "  # are equivalent to a single linear layer. You can create a nonlinear layer\n",
+        "  # like this:\n",
+        "  # layer = tf.layers.dense(inputs=..., units=..., activation=tf.nn.relu)\n",
+        "  layer = tf.layers.dense(inputs=features, units=NUM_CLASSES, activation=None)\n",
+        "  return layer\n",
+        "\n",
+        "\n",
+        "# For each class (kind of flower), the model outputs some real number as a score\n",
+        "# how much the input resembles this class. This vector of numbers is often\n",
+        "# called the \"logits\".\n",
+        "logits = create_model(features)\n",
+        "labels = tf.placeholder(tf.float32, [None, NUM_CLASSES])\n",
+        "\n",
+        "# Mathematically, a good way to measure how much the predicted probabilities\n",
+        "# diverge from the truth is the \"cross-entropy\" between the two probability\n",
+        "# distributions. For numerical stability, this is best done directly from the\n",
+        "# logits, not the probabilities extracted from them.\n",
+        "cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels)\n",
+        "cross_entropy_mean = tf.reduce_mean(cross_entropy)\n",
+        "\n",
+        "# Let's add an optimizer so we can train the network.\n",
+        "optimizer = tf.train.GradientDescentOptimizer(learning_rate=LEARNING_RATE)\n",
+        "train_op = optimizer.minimize(loss=cross_entropy_mean)\n",
+        "\n",
+        "# The \"softmax\" function transforms the logits vector into a vector of\n",
+        "# probabilities: non-negative numbers that sum up to one, and the i-th number\n",
+        "# says how likely the input comes from class i.\n",
+        "probabilities = tf.nn.softmax(logits)\n",
+        "\n",
+        "# We choose the highest one as the predicted class.\n",
+        "prediction = tf.argmax(probabilities, 1)\n",
+        "correct_prediction = tf.equal(prediction, tf.argmax(labels, 1))\n",
+        "\n",
+        "# The accuracy will allow us to eval on our test set. \n",
+        "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0vvhYQ7-3AG_"
+      },
+      "source": [
+        "## Train the network\n",
+        "\n",
+        "Now that our model is built, let's train it and see how it performs on our test set."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1YnBg7-OS3Dz"
+      },
+      "outputs": [],
+      "source": [
+        "# How long will we train the network (number of batches).\n",
+        "NUM_TRAIN_STEPS = 100 #@param {type: 'integer'}\n",
+        "# How many training examples we use in each step.\n",
+        "TRAIN_BATCH_SIZE = 10 #@param {type: 'integer'}\n",
+        "# How often to evaluate the model performance.\n",
+        "EVAL_EVERY = 10 #@param {type: 'integer'}\n",
+        "\n",
+        "def get_batch(batch_size=None, test=False):\n",
+        "  \"\"\"Get a random batch of examples.\"\"\"\n",
+        "  examples = TEST_EXAMPLES if test else TRAIN_EXAMPLES\n",
+        "  batch_examples = random.sample(examples, batch_size) if batch_size else examples\n",
+        "  return batch_examples\n",
+        "\n",
+        "def get_images_and_labels(batch_examples):\n",
+        "  images = [get_encoded_image(e) for e in batch_examples]\n",
+        "  one_hot_labels = [get_label_one_hot(e) for e in batch_examples]\n",
+        "  return images, one_hot_labels\n",
+        "\n",
+        "def get_label_one_hot(example):\n",
+        "  \"\"\"Get the one hot encoding vector for the example.\"\"\"\n",
+        "  one_hot_vector = np.zeros(NUM_CLASSES)\n",
+        "  np.put(one_hot_vector, get_label(example), 1)\n",
+        "  return one_hot_vector\n",
+        "\n",
+        "with tf.Session() as sess:\n",
+        "  sess.run(tf.global_variables_initializer())\n",
+        "  for i in range(NUM_TRAIN_STEPS):\n",
+        "    # Get a random batch of training examples.\n",
+        "    train_batch = get_batch(batch_size=TRAIN_BATCH_SIZE)\n",
+        "    batch_images, batch_labels = get_images_and_labels(train_batch)\n",
+        "    # Run the train_op to train the model.\n",
+        "    train_loss, _, train_accuracy = sess.run(\n",
+        "        [cross_entropy_mean, train_op, accuracy],\n",
+        "        feed_dict={encoded_images: batch_images, labels: batch_labels})\n",
+        "    is_final_step = (i == (NUM_TRAIN_STEPS - 1))\n",
+        "    if i % EVAL_EVERY == 0 or is_final_step:\n",
+        "      # Get a batch of test examples.\n",
+        "      test_batch = get_batch(batch_size=None, test=True)\n",
+        "      batch_images, batch_labels = get_images_and_labels(test_batch)\n",
+        "      # Evaluate how well our model performs on the test set.\n",
+        "      test_loss, test_accuracy, test_prediction, correct_predicate = sess.run(\n",
+        "        [cross_entropy_mean, accuracy, prediction, correct_prediction],\n",
+        "        feed_dict={encoded_images: batch_images, labels: batch_labels})\n",
+        "      print('Test accuracy at step %s: %.2f%%' % (i, (test_accuracy * 100)))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZFUNJxuH2t0V"
+      },
+      "outputs": [],
+      "source": [
+        "def show_confusion_matrix(test_labels, predictions):\n",
+        "  \"\"\"Compute confusion matrix and normalize.\"\"\"\n",
+        "  confusion = sk_metrics.confusion_matrix(\n",
+        "    np.argmax(test_labels, axis=1), predictions)\n",
+        "  confusion_normalized = confusion.astype(\"float\") / confusion.sum(axis=1)\n",
+        "  axis_labels = list(CLASSES.values())\n",
+        "  ax = sns.heatmap(\n",
+        "      confusion_normalized, xticklabels=axis_labels, yticklabels=axis_labels,\n",
+        "      cmap='Blues', annot=True, fmt='.2f', square=True)\n",
+        "  plt.title(\"Confusion matrix\")\n",
+        "  plt.ylabel(\"True label\")\n",
+        "  plt.xlabel(\"Predicted label\")\n",
+        "\n",
+        "show_confusion_matrix(batch_labels, test_prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Uu3vo8DK8BdL"
+      },
+      "source": [
+        "## Incorrect predictions\n",
+        "\n",
+        "Let's a take a closer look at the test examples that our model got wrong.\n",
+        "\n",
+        "- Are there any mislabeled examples in our test set?\n",
+        "- Is there any bad data in the test set - images that aren't actually pictures of flowers?\n",
+        "- Are there images where you can understand why the model made a mistake?"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hqa0V3WN8C9M"
+      },
+      "outputs": [],
+      "source": [
+        "incorrect = [\n",
+        "    (example, CLASSES[prediction])\n",
+        "    for example, prediction, is_correct in zip(test_batch, test_prediction, correct_predicate)\n",
+        "    if not is_correct\n",
+        "]\n",
+        "display_images(\n",
+        "  [(get_image(example), \"prediction: {0}\\nlabel:{1}\".format(incorrect_prediction, get_class(example)))\n",
+        "   for (example, incorrect_prediction) in incorrect[:20]])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YN_s04Il8TvK"
+      },
+      "source": [
+        "## Exercises: Improve the model!\n",
+        "\n",
+        "We've trained a baseline model, now let's try to improve it to achieve better accuracy. (Remember that you'll need to re-run the cells when you make a change.)\n",
+        "\n",
+        "### Exercise 1:  Try a different image model.\n",
+        "With TF-Hub, trying a few different image models is simple. Just replace the `\"https://tfhub.dev/google/imagenet/mobilenet_v2_050_128/feature_vector/2\"` handle in the `hub.Module()` call with a handle of different module and rerun all the code. You can see all available image modules at [tfhub.dev](https://tfhub.dev/s?module-type=image-feature-vector). \n",
+        "\n",
+        "A good choice might be one of the other [MobileNet V2 modules](https://tfhub.dev/s?module-type=image-feature-vector&network-architecture=mobilenet-v2). Many of the modules -- including the MobileNet modules -- were trained on the [ImageNet dataset](https://www.tensorflow.org/datasets/catalog/imagenet2012) which contains over 1 million images and 1000 classes. Choosing a network architecture provides a tradeoff between speed and classification accuracy: models like MobileNet or NASNet Mobile are fast and small, more traditional architectures like Inception and ResNet were designed for accuracy.\n",
+        "\n",
+        "For the larger Inception V3 architecture, you can also explore the benefits of pre-training on a domain closer to your own task: it is also available as a [module trained on the iNaturalist dataset](https://tfhub.dev/google/inaturalist/inception_v3/feature_vector/1) of plants and animals.\n",
+        "\n",
+        "### Exercise 2: Add a hidden layer.\n",
+        "Stack a hidden layer between extracted image features and the linear classifier (in function `create_model()` above). To create a non-linear hidden layer with e.g. 100 nodes, use  [tf.layers.dense](https://www.tensorflow.org/api_docs/python/tf/compat/v1/layers/dense) with units set to 100 and activation set to `tf.nn.relu`. Does changing the size of the hidden layer affect the test accuracy? Does adding second hidden layer improve the accuracy?\n",
+        "\n",
+        "### Exercise 3: Change hyperparameters.\n",
+        "Does increasing  *number of training steps*  improves final accuracy? Can you *change the learning rate* to make your model converge more quickly? Does the training *batch size* affect your model's performance?\n",
+        "\n",
+        "### Exercise 4: Try a different optimizer.\n",
+        "\n",
+        "Replace the basic GradientDescentOptimizer with a more sophisticate optimizer, e.g. [AdagradOptimizer](https://www.tensorflow.org/api_docs/python/tf/compat/v1/train/AdagradOptimizer). Does it make a difference to your model training? If you want to learn more about the benefits of different optimization algorithms, check out [this post](http://ruder.io/optimizing-gradient-descent/)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kdwVXO1eJS5-"
+      },
+      "source": [
+        "## Want to learn more?\n",
+        "\n",
+        "If you are interested in a more advanced version of this tutorial, check out the [TensorFlow image retraining tutorial](https://www.tensorflow.org/hub/tutorials/image_retraining) which walks you through visualizing the training using TensorBoard, advanced techniques like dataset augmentation by distorting images, and replacing the flowers dataset to learn an image classifier on your own dataset.\n",
+        "\n",
+        "You can learn more about TensorFlow at [tensorflow.org](http://tensorflow.org) and see the TF-Hub API documentation is available at [tensorflow.org/hub](https://www.tensorflow.org/hub/). Find available TensorFlow Hub modules at [tfhub.dev](http://tfhub.dev) including more image feature vector modules and text embedding modules.\n",
+        "\n",
+        "Also check out the [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/) which is Google's fast-paced, practical introduction to machine learning."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "ScitaPqhKtuW"
+      ],
+      "name": "image_feature_vector.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/movenet.ipynb b/site/en/hub/tutorials/movenet.ipynb
new file mode 100644
index 00000000000..f7955a5253b
--- /dev/null
+++ b/site/en/hub/tutorials/movenet.ipynb
@@ -0,0 +1,816 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "toCy3v03Dwx7"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QKe-ubNcDvgv"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KqtQzBCpIJ7Y"
+      },
+      "source": [
+        "# MoveNet: Ultra fast and accurate pose detection model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MCmFOosnSkCd"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6x99e0aEY_d6"
+      },
+      "source": [
+        "**[MoveNet](https://t.co/QpfnVL0YYI?amp=1)** is an ultra fast and accurate model that detects 17 keypoints of a body. The model is offered on [TF Hub](https://tfhub.dev/s?q=movenet) with two variants, known as Lightning and Thunder. Lightning is intended for latency-critical applications, while Thunder is intended for applications that require high accuracy. Both models run faster than real time (30+ FPS) on most modern desktops, laptops, and phones, which proves crucial for live fitness, health, and wellness applications.\n",
+        "\n",
+        "\n",
+        "\"drawing\"/\n",
+        "\n",
+        "*Images downloaded from Pexels (https://www.pexels.com/)\n",
+        "\n",
+        "This Colab walks you through the details of how to load MoveNet, and run inference on the input image and video below.\n",
+        "\n",
+        "Note: check out the [live demo](https://storage.googleapis.com/tfjs-models/demos/pose-detection/index.html?model=movenet) for how the model works!"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "10_zkgbZBkIE"
+      },
+      "source": [
+        "# Human Pose Estimation with MoveNet"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9u_VGR6_BmbZ"
+      },
+      "source": [
+        "## Visualization libraries & Imports"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TtcwSIcgbIVN"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q imageio\n",
+        "!pip install -q opencv-python\n",
+        "!pip install -q git+https://github.com/tensorflow/docs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9BLeJv-pCCld"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "from tensorflow_docs.vis import embed\n",
+        "import numpy as np\n",
+        "import cv2\n",
+        "\n",
+        "# Import matplotlib libraries\n",
+        "from matplotlib import pyplot as plt\n",
+        "from matplotlib.collections import LineCollection\n",
+        "import matplotlib.patches as patches\n",
+        "\n",
+        "# Some modules to display an animation using imageio.\n",
+        "import imageio\n",
+        "from IPython.display import HTML, display"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "bEJBMeRb3YUy"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Helper functions for visualization\n",
+        "\n",
+        "# Dictionary that maps from joint names to keypoint indices.\n",
+        "KEYPOINT_DICT = {\n",
+        "    'nose': 0,\n",
+        "    'left_eye': 1,\n",
+        "    'right_eye': 2,\n",
+        "    'left_ear': 3,\n",
+        "    'right_ear': 4,\n",
+        "    'left_shoulder': 5,\n",
+        "    'right_shoulder': 6,\n",
+        "    'left_elbow': 7,\n",
+        "    'right_elbow': 8,\n",
+        "    'left_wrist': 9,\n",
+        "    'right_wrist': 10,\n",
+        "    'left_hip': 11,\n",
+        "    'right_hip': 12,\n",
+        "    'left_knee': 13,\n",
+        "    'right_knee': 14,\n",
+        "    'left_ankle': 15,\n",
+        "    'right_ankle': 16\n",
+        "}\n",
+        "\n",
+        "# Maps bones to a matplotlib color name.\n",
+        "KEYPOINT_EDGE_INDS_TO_COLOR = {\n",
+        "    (0, 1): 'm',\n",
+        "    (0, 2): 'c',\n",
+        "    (1, 3): 'm',\n",
+        "    (2, 4): 'c',\n",
+        "    (0, 5): 'm',\n",
+        "    (0, 6): 'c',\n",
+        "    (5, 7): 'm',\n",
+        "    (7, 9): 'm',\n",
+        "    (6, 8): 'c',\n",
+        "    (8, 10): 'c',\n",
+        "    (5, 6): 'y',\n",
+        "    (5, 11): 'm',\n",
+        "    (6, 12): 'c',\n",
+        "    (11, 12): 'y',\n",
+        "    (11, 13): 'm',\n",
+        "    (13, 15): 'm',\n",
+        "    (12, 14): 'c',\n",
+        "    (14, 16): 'c'\n",
+        "}\n",
+        "\n",
+        "def _keypoints_and_edges_for_display(keypoints_with_scores,\n",
+        "                                     height,\n",
+        "                                     width,\n",
+        "                                     keypoint_threshold=0.11):\n",
+        "  \"\"\"Returns high confidence keypoints and edges for visualization.\n",
+        "\n",
+        "  Args:\n",
+        "    keypoints_with_scores: A numpy array with shape [1, 1, 17, 3] representing\n",
+        "      the keypoint coordinates and scores returned from the MoveNet model.\n",
+        "    height: height of the image in pixels.\n",
+        "    width: width of the image in pixels.\n",
+        "    keypoint_threshold: minimum confidence score for a keypoint to be\n",
+        "      visualized.\n",
+        "\n",
+        "  Returns:\n",
+        "    A (keypoints_xy, edges_xy, edge_colors) containing:\n",
+        "      * the coordinates of all keypoints of all detected entities;\n",
+        "      * the coordinates of all skeleton edges of all detected entities;\n",
+        "      * the colors in which the edges should be plotted.\n",
+        "  \"\"\"\n",
+        "  keypoints_all = []\n",
+        "  keypoint_edges_all = []\n",
+        "  edge_colors = []\n",
+        "  num_instances, _, _, _ = keypoints_with_scores.shape\n",
+        "  for idx in range(num_instances):\n",
+        "    kpts_x = keypoints_with_scores[0, idx, :, 1]\n",
+        "    kpts_y = keypoints_with_scores[0, idx, :, 0]\n",
+        "    kpts_scores = keypoints_with_scores[0, idx, :, 2]\n",
+        "    kpts_absolute_xy = np.stack(\n",
+        "        [width * np.array(kpts_x), height * np.array(kpts_y)], axis=-1)\n",
+        "    kpts_above_thresh_absolute = kpts_absolute_xy[\n",
+        "        kpts_scores > keypoint_threshold, :]\n",
+        "    keypoints_all.append(kpts_above_thresh_absolute)\n",
+        "\n",
+        "    for edge_pair, color in KEYPOINT_EDGE_INDS_TO_COLOR.items():\n",
+        "      if (kpts_scores[edge_pair[0]] > keypoint_threshold and\n",
+        "          kpts_scores[edge_pair[1]] > keypoint_threshold):\n",
+        "        x_start = kpts_absolute_xy[edge_pair[0], 0]\n",
+        "        y_start = kpts_absolute_xy[edge_pair[0], 1]\n",
+        "        x_end = kpts_absolute_xy[edge_pair[1], 0]\n",
+        "        y_end = kpts_absolute_xy[edge_pair[1], 1]\n",
+        "        line_seg = np.array([[x_start, y_start], [x_end, y_end]])\n",
+        "        keypoint_edges_all.append(line_seg)\n",
+        "        edge_colors.append(color)\n",
+        "  if keypoints_all:\n",
+        "    keypoints_xy = np.concatenate(keypoints_all, axis=0)\n",
+        "  else:\n",
+        "    keypoints_xy = np.zeros((0, 17, 2))\n",
+        "\n",
+        "  if keypoint_edges_all:\n",
+        "    edges_xy = np.stack(keypoint_edges_all, axis=0)\n",
+        "  else:\n",
+        "    edges_xy = np.zeros((0, 2, 2))\n",
+        "  return keypoints_xy, edges_xy, edge_colors\n",
+        "\n",
+        "\n",
+        "def draw_prediction_on_image(\n",
+        "    image, keypoints_with_scores, crop_region=None, close_figure=False,\n",
+        "    output_image_height=None):\n",
+        "  \"\"\"Draws the keypoint predictions on image.\n",
+        "\n",
+        "  Args:\n",
+        "    image: A numpy array with shape [height, width, channel] representing the\n",
+        "      pixel values of the input image.\n",
+        "    keypoints_with_scores: A numpy array with shape [1, 1, 17, 3] representing\n",
+        "      the keypoint coordinates and scores returned from the MoveNet model.\n",
+        "    crop_region: A dictionary that defines the coordinates of the bounding box\n",
+        "      of the crop region in normalized coordinates (see the init_crop_region\n",
+        "      function below for more detail). If provided, this function will also\n",
+        "      draw the bounding box on the image.\n",
+        "    output_image_height: An integer indicating the height of the output image.\n",
+        "      Note that the image aspect ratio will be the same as the input image.\n",
+        "\n",
+        "  Returns:\n",
+        "    A numpy array with shape [out_height, out_width, channel] representing the\n",
+        "    image overlaid with keypoint predictions.\n",
+        "  \"\"\"\n",
+        "  height, width, channel = image.shape\n",
+        "  aspect_ratio = float(width) / height\n",
+        "  fig, ax = plt.subplots(figsize=(12 * aspect_ratio, 12))\n",
+        "  # To remove the huge white borders\n",
+        "  fig.tight_layout(pad=0)\n",
+        "  ax.margins(0)\n",
+        "  ax.set_yticklabels([])\n",
+        "  ax.set_xticklabels([])\n",
+        "  plt.axis('off')\n",
+        "\n",
+        "  im = ax.imshow(image)\n",
+        "  line_segments = LineCollection([], linewidths=(4), linestyle='solid')\n",
+        "  ax.add_collection(line_segments)\n",
+        "  # Turn off tick labels\n",
+        "  scat = ax.scatter([], [], s=60, color='#FF1493', zorder=3)\n",
+        "\n",
+        "  (keypoint_locs, keypoint_edges,\n",
+        "   edge_colors) = _keypoints_and_edges_for_display(\n",
+        "       keypoints_with_scores, height, width)\n",
+        "\n",
+        "  line_segments.set_segments(keypoint_edges)\n",
+        "  line_segments.set_color(edge_colors)\n",
+        "  if keypoint_edges.shape[0]:\n",
+        "    line_segments.set_segments(keypoint_edges)\n",
+        "    line_segments.set_color(edge_colors)\n",
+        "  if keypoint_locs.shape[0]:\n",
+        "    scat.set_offsets(keypoint_locs)\n",
+        "\n",
+        "  if crop_region is not None:\n",
+        "    xmin = max(crop_region['x_min'] * width, 0.0)\n",
+        "    ymin = max(crop_region['y_min'] * height, 0.0)\n",
+        "    rec_width = min(crop_region['x_max'], 0.99) * width - xmin\n",
+        "    rec_height = min(crop_region['y_max'], 0.99) * height - ymin\n",
+        "    rect = patches.Rectangle(\n",
+        "        (xmin,ymin),rec_width,rec_height,\n",
+        "        linewidth=1,edgecolor='b',facecolor='none')\n",
+        "    ax.add_patch(rect)\n",
+        "\n",
+        "  fig.canvas.draw()\n",
+        "  image_from_plot = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)\n",
+        "  image_from_plot = image_from_plot.reshape(\n",
+        "      fig.canvas.get_width_height()[::-1] + (3,))\n",
+        "  plt.close(fig)\n",
+        "  if output_image_height is not None:\n",
+        "    output_image_width = int(output_image_height / height * width)\n",
+        "    image_from_plot = cv2.resize(\n",
+        "        image_from_plot, dsize=(output_image_width, output_image_height),\n",
+        "         interpolation=cv2.INTER_CUBIC)\n",
+        "  return image_from_plot\n",
+        "\n",
+        "def to_gif(images, duration):\n",
+        "  \"\"\"Converts image sequence (4D numpy array) to gif.\"\"\"\n",
+        "  imageio.mimsave('./animation.gif', images, duration=duration)\n",
+        "  return embed.embed_file('./animation.gif')\n",
+        "\n",
+        "def progress(value, max=100):\n",
+        "  return HTML(\"\"\"\n",
+        "      \n",
+        "          {value}\n",
+        "      \n",
+        "  \"\"\".format(value=value, max=max))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UvrN0iQiOxhR"
+      },
+      "source": [
+        "## Load Model from TF hub"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zeGHgANcT7a1"
+      },
+      "outputs": [],
+      "source": [
+        "model_name = \"movenet_lightning\" #@param [\"movenet_lightning\", \"movenet_thunder\", \"movenet_lightning_f16.tflite\", \"movenet_thunder_f16.tflite\", \"movenet_lightning_int8.tflite\", \"movenet_thunder_int8.tflite\"]\n",
+        "\n",
+        "if \"tflite\" in model_name:\n",
+        "  if \"movenet_lightning_f16\" in model_name:\n",
+        "    !wget -q -O model.tflite https://tfhub.dev/google/lite-model/movenet/singlepose/lightning/tflite/float16/4?lite-format=tflite\n",
+        "    input_size = 192\n",
+        "  elif \"movenet_thunder_f16\" in model_name:\n",
+        "    !wget -q -O model.tflite https://tfhub.dev/google/lite-model/movenet/singlepose/thunder/tflite/float16/4?lite-format=tflite\n",
+        "    input_size = 256\n",
+        "  elif \"movenet_lightning_int8\" in model_name:\n",
+        "    !wget -q -O model.tflite https://tfhub.dev/google/lite-model/movenet/singlepose/lightning/tflite/int8/4?lite-format=tflite\n",
+        "    input_size = 192\n",
+        "  elif \"movenet_thunder_int8\" in model_name:\n",
+        "    !wget -q -O model.tflite https://tfhub.dev/google/lite-model/movenet/singlepose/thunder/tflite/int8/4?lite-format=tflite\n",
+        "    input_size = 256\n",
+        "  else:\n",
+        "    raise ValueError(\"Unsupported model name: %s\" % model_name)\n",
+        "\n",
+        "  # Initialize the TFLite interpreter\n",
+        "  interpreter = tf.lite.Interpreter(model_path=\"model.tflite\")\n",
+        "  interpreter.allocate_tensors()\n",
+        "\n",
+        "  def movenet(input_image):\n",
+        "    \"\"\"Runs detection on an input image.\n",
+        "\n",
+        "    Args:\n",
+        "      input_image: A [1, height, width, 3] tensor represents the input image\n",
+        "        pixels. Note that the height/width should already be resized and match the\n",
+        "        expected input resolution of the model before passing into this function.\n",
+        "\n",
+        "    Returns:\n",
+        "      A [1, 1, 17, 3] float numpy array representing the predicted keypoint\n",
+        "      coordinates and scores.\n",
+        "    \"\"\"\n",
+        "    # TF Lite format expects tensor type of uint8.\n",
+        "    input_image = tf.cast(input_image, dtype=tf.uint8)\n",
+        "    input_details = interpreter.get_input_details()\n",
+        "    output_details = interpreter.get_output_details()\n",
+        "    interpreter.set_tensor(input_details[0]['index'], input_image.numpy())\n",
+        "    # Invoke inference.\n",
+        "    interpreter.invoke()\n",
+        "    # Get the model prediction.\n",
+        "    keypoints_with_scores = interpreter.get_tensor(output_details[0]['index'])\n",
+        "    return keypoints_with_scores\n",
+        "\n",
+        "else:\n",
+        "  if \"movenet_lightning\" in model_name:\n",
+        "    module = hub.load(\"https://tfhub.dev/google/movenet/singlepose/lightning/4\")\n",
+        "    input_size = 192\n",
+        "  elif \"movenet_thunder\" in model_name:\n",
+        "    module = hub.load(\"https://tfhub.dev/google/movenet/singlepose/thunder/4\")\n",
+        "    input_size = 256\n",
+        "  else:\n",
+        "    raise ValueError(\"Unsupported model name: %s\" % model_name)\n",
+        "\n",
+        "  def movenet(input_image):\n",
+        "    \"\"\"Runs detection on an input image.\n",
+        "\n",
+        "    Args:\n",
+        "      input_image: A [1, height, width, 3] tensor represents the input image\n",
+        "        pixels. Note that the height/width should already be resized and match the\n",
+        "        expected input resolution of the model before passing into this function.\n",
+        "\n",
+        "    Returns:\n",
+        "      A [1, 1, 17, 3] float numpy array representing the predicted keypoint\n",
+        "      coordinates and scores.\n",
+        "    \"\"\"\n",
+        "    model = module.signatures['serving_default']\n",
+        "\n",
+        "    # SavedModel format expects tensor type of int32.\n",
+        "    input_image = tf.cast(input_image, dtype=tf.int32)\n",
+        "    # Run model inference.\n",
+        "    outputs = model(input_image)\n",
+        "    # Output is a [1, 1, 17, 3] tensor.\n",
+        "    keypoints_with_scores = outputs['output_0'].numpy()\n",
+        "    return keypoints_with_scores"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-h1qHYaqD9ap"
+      },
+      "source": [
+        "## Single Image Example"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ymTVR2I9x22I"
+      },
+      "source": [
+        "This session demonstrates the minimum working example of running the model on a **single image** to predict the 17 human keypoints."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5I3xBq80E3N_"
+      },
+      "source": [
+        "### Load Input Image"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GMO4B-wx5psP"
+      },
+      "outputs": [],
+      "source": [
+        "!curl -o input_image.jpeg https://images.pexels.com/photos/4384679/pexels-photo-4384679.jpeg --silent"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lJZYQ8KYFQ6x"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the input image.\n",
+        "image_path = 'input_image.jpeg'\n",
+        "image = tf.io.read_file(image_path)\n",
+        "image = tf.image.decode_jpeg(image)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "S_UWRdQxE6WN"
+      },
+      "source": [
+        "### Run Inference"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VHmTwACwFW-v"
+      },
+      "outputs": [],
+      "source": [
+        "# Resize and pad the image to keep the aspect ratio and fit the expected size.\n",
+        "input_image = tf.expand_dims(image, axis=0)\n",
+        "input_image = tf.image.resize_with_pad(input_image, input_size, input_size)\n",
+        "\n",
+        "# Run model inference.\n",
+        "keypoints_with_scores = movenet(input_image)\n",
+        "\n",
+        "# Visualize the predictions with image.\n",
+        "display_image = tf.expand_dims(image, axis=0)\n",
+        "display_image = tf.cast(tf.image.resize_with_pad(\n",
+        "    display_image, 1280, 1280), dtype=tf.int32)\n",
+        "output_overlay = draw_prediction_on_image(\n",
+        "    np.squeeze(display_image.numpy(), axis=0), keypoints_with_scores)\n",
+        "\n",
+        "plt.figure(figsize=(5, 5))\n",
+        "plt.imshow(output_overlay)\n",
+        "_ = plt.axis('off')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rKm-B0eMYeg8"
+      },
+      "source": [
+        "## Video (Image Sequence) Example"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gdPFXabLyiKv"
+      },
+      "source": [
+        "This section demonstrates how to apply intelligent cropping based on detections from the previous frame when the input is a sequence of frames. This allows the model to devote its attention and resources to the main subject, resulting in much better prediction quality without sacrificing the speed.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "SYFdK-JHYhrv"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Cropping Algorithm\n",
+        "\n",
+        "# Confidence score to determine whether a keypoint prediction is reliable.\n",
+        "MIN_CROP_KEYPOINT_SCORE = 0.2\n",
+        "\n",
+        "def init_crop_region(image_height, image_width):\n",
+        "  \"\"\"Defines the default crop region.\n",
+        "\n",
+        "  The function provides the initial crop region (pads the full image from both\n",
+        "  sides to make it a square image) when the algorithm cannot reliably determine\n",
+        "  the crop region from the previous frame.\n",
+        "  \"\"\"\n",
+        "  if image_width > image_height:\n",
+        "    box_height = image_width / image_height\n",
+        "    box_width = 1.0\n",
+        "    y_min = (image_height / 2 - image_width / 2) / image_height\n",
+        "    x_min = 0.0\n",
+        "  else:\n",
+        "    box_height = 1.0\n",
+        "    box_width = image_height / image_width\n",
+        "    y_min = 0.0\n",
+        "    x_min = (image_width / 2 - image_height / 2) / image_width\n",
+        "\n",
+        "  return {\n",
+        "    'y_min': y_min,\n",
+        "    'x_min': x_min,\n",
+        "    'y_max': y_min + box_height,\n",
+        "    'x_max': x_min + box_width,\n",
+        "    'height': box_height,\n",
+        "    'width': box_width\n",
+        "  }\n",
+        "\n",
+        "def torso_visible(keypoints):\n",
+        "  \"\"\"Checks whether there are enough torso keypoints.\n",
+        "\n",
+        "  This function checks whether the model is confident at predicting one of the\n",
+        "  shoulders/hips which is required to determine a good crop region.\n",
+        "  \"\"\"\n",
+        "  return ((keypoints[0, 0, KEYPOINT_DICT['left_hip'], 2] >\n",
+        "           MIN_CROP_KEYPOINT_SCORE or\n",
+        "          keypoints[0, 0, KEYPOINT_DICT['right_hip'], 2] >\n",
+        "           MIN_CROP_KEYPOINT_SCORE) and\n",
+        "          (keypoints[0, 0, KEYPOINT_DICT['left_shoulder'], 2] >\n",
+        "           MIN_CROP_KEYPOINT_SCORE or\n",
+        "          keypoints[0, 0, KEYPOINT_DICT['right_shoulder'], 2] >\n",
+        "           MIN_CROP_KEYPOINT_SCORE))\n",
+        "\n",
+        "def determine_torso_and_body_range(\n",
+        "    keypoints, target_keypoints, center_y, center_x):\n",
+        "  \"\"\"Calculates the maximum distance from each keypoints to the center location.\n",
+        "\n",
+        "  The function returns the maximum distances from the two sets of keypoints:\n",
+        "  full 17 keypoints and 4 torso keypoints. The returned information will be\n",
+        "  used to determine the crop size. See determineCropRegion for more detail.\n",
+        "  \"\"\"\n",
+        "  torso_joints = ['left_shoulder', 'right_shoulder', 'left_hip', 'right_hip']\n",
+        "  max_torso_yrange = 0.0\n",
+        "  max_torso_xrange = 0.0\n",
+        "  for joint in torso_joints:\n",
+        "    dist_y = abs(center_y - target_keypoints[joint][0])\n",
+        "    dist_x = abs(center_x - target_keypoints[joint][1])\n",
+        "    if dist_y > max_torso_yrange:\n",
+        "      max_torso_yrange = dist_y\n",
+        "    if dist_x > max_torso_xrange:\n",
+        "      max_torso_xrange = dist_x\n",
+        "\n",
+        "  max_body_yrange = 0.0\n",
+        "  max_body_xrange = 0.0\n",
+        "  for joint in KEYPOINT_DICT.keys():\n",
+        "    if keypoints[0, 0, KEYPOINT_DICT[joint], 2] < MIN_CROP_KEYPOINT_SCORE:\n",
+        "      continue\n",
+        "    dist_y = abs(center_y - target_keypoints[joint][0]);\n",
+        "    dist_x = abs(center_x - target_keypoints[joint][1]);\n",
+        "    if dist_y > max_body_yrange:\n",
+        "      max_body_yrange = dist_y\n",
+        "\n",
+        "    if dist_x > max_body_xrange:\n",
+        "      max_body_xrange = dist_x\n",
+        "\n",
+        "  return [max_torso_yrange, max_torso_xrange, max_body_yrange, max_body_xrange]\n",
+        "\n",
+        "def determine_crop_region(\n",
+        "      keypoints, image_height,\n",
+        "      image_width):\n",
+        "  \"\"\"Determines the region to crop the image for the model to run inference on.\n",
+        "\n",
+        "  The algorithm uses the detected joints from the previous frame to estimate\n",
+        "  the square region that encloses the full body of the target person and\n",
+        "  centers at the midpoint of two hip joints. The crop size is determined by\n",
+        "  the distances between each joints and the center point.\n",
+        "  When the model is not confident with the four torso joint predictions, the\n",
+        "  function returns a default crop which is the full image padded to square.\n",
+        "  \"\"\"\n",
+        "  target_keypoints = {}\n",
+        "  for joint in KEYPOINT_DICT.keys():\n",
+        "    target_keypoints[joint] = [\n",
+        "      keypoints[0, 0, KEYPOINT_DICT[joint], 0] * image_height,\n",
+        "      keypoints[0, 0, KEYPOINT_DICT[joint], 1] * image_width\n",
+        "    ]\n",
+        "\n",
+        "  if torso_visible(keypoints):\n",
+        "    center_y = (target_keypoints['left_hip'][0] +\n",
+        "                target_keypoints['right_hip'][0]) / 2;\n",
+        "    center_x = (target_keypoints['left_hip'][1] +\n",
+        "                target_keypoints['right_hip'][1]) / 2;\n",
+        "\n",
+        "    (max_torso_yrange, max_torso_xrange,\n",
+        "      max_body_yrange, max_body_xrange) = determine_torso_and_body_range(\n",
+        "          keypoints, target_keypoints, center_y, center_x)\n",
+        "\n",
+        "    crop_length_half = np.amax(\n",
+        "        [max_torso_xrange * 1.9, max_torso_yrange * 1.9,\n",
+        "          max_body_yrange * 1.2, max_body_xrange * 1.2])\n",
+        "\n",
+        "    tmp = np.array(\n",
+        "        [center_x, image_width - center_x, center_y, image_height - center_y])\n",
+        "    crop_length_half = np.amin(\n",
+        "        [crop_length_half, np.amax(tmp)]);\n",
+        "\n",
+        "    crop_corner = [center_y - crop_length_half, center_x - crop_length_half];\n",
+        "\n",
+        "    if crop_length_half > max(image_width, image_height) / 2:\n",
+        "      return init_crop_region(image_height, image_width)\n",
+        "    else:\n",
+        "      crop_length = crop_length_half * 2;\n",
+        "      return {\n",
+        "        'y_min': crop_corner[0] / image_height,\n",
+        "        'x_min': crop_corner[1] / image_width,\n",
+        "        'y_max': (crop_corner[0] + crop_length) / image_height,\n",
+        "        'x_max': (crop_corner[1] + crop_length) / image_width,\n",
+        "        'height': (crop_corner[0] + crop_length) / image_height -\n",
+        "            crop_corner[0] / image_height,\n",
+        "        'width': (crop_corner[1] + crop_length) / image_width -\n",
+        "            crop_corner[1] / image_width\n",
+        "      }\n",
+        "  else:\n",
+        "    return init_crop_region(image_height, image_width)\n",
+        "\n",
+        "def crop_and_resize(image, crop_region, crop_size):\n",
+        "  \"\"\"Crops and resize the image to prepare for the model input.\"\"\"\n",
+        "  boxes=[[crop_region['y_min'], crop_region['x_min'],\n",
+        "          crop_region['y_max'], crop_region['x_max']]]\n",
+        "  output_image = tf.image.crop_and_resize(\n",
+        "      image, box_indices=[0], boxes=boxes, crop_size=crop_size)\n",
+        "  return output_image\n",
+        "\n",
+        "def run_inference(movenet, image, crop_region, crop_size):\n",
+        "  \"\"\"Runs model inference on the cropped region.\n",
+        "\n",
+        "  The function runs the model inference on the cropped region and updates the\n",
+        "  model output to the original image coordinate system.\n",
+        "  \"\"\"\n",
+        "  image_height, image_width, _ = image.shape\n",
+        "  input_image = crop_and_resize(\n",
+        "    tf.expand_dims(image, axis=0), crop_region, crop_size=crop_size)\n",
+        "  # Run model inference.\n",
+        "  keypoints_with_scores = movenet(input_image)\n",
+        "  # Update the coordinates.\n",
+        "  for idx in range(17):\n",
+        "    keypoints_with_scores[0, 0, idx, 0] = (\n",
+        "        crop_region['y_min'] * image_height +\n",
+        "        crop_region['height'] * image_height *\n",
+        "        keypoints_with_scores[0, 0, idx, 0]) / image_height\n",
+        "    keypoints_with_scores[0, 0, idx, 1] = (\n",
+        "        crop_region['x_min'] * image_width +\n",
+        "        crop_region['width'] * image_width *\n",
+        "        keypoints_with_scores[0, 0, idx, 1]) / image_width\n",
+        "  return keypoints_with_scores"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L2JmA1xAEntQ"
+      },
+      "source": [
+        "### Load Input Image Sequence"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CzJxbxDckWl2"
+      },
+      "outputs": [],
+      "source": [
+        "!wget -q -O dance.gif https://github.com/tensorflow/tfjs-models/raw/master/pose-detection/assets/dance_input.gif"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "IxbMFZJUkd6W"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the input image.\n",
+        "image_path = 'dance.gif'\n",
+        "image = tf.io.read_file(image_path)\n",
+        "image = tf.image.decode_gif(image)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CJKeQ4siEtU9"
+      },
+      "source": [
+        "### Run Inference with Cropping Algorithm"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9B57XS0NZPIy"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the input image.\n",
+        "num_frames, image_height, image_width, _ = image.shape\n",
+        "crop_region = init_crop_region(image_height, image_width)\n",
+        "\n",
+        "output_images = []\n",
+        "bar = display(progress(0, num_frames-1), display_id=True)\n",
+        "for frame_idx in range(num_frames):\n",
+        "  keypoints_with_scores = run_inference(\n",
+        "      movenet, image[frame_idx, :, :, :], crop_region,\n",
+        "      crop_size=[input_size, input_size])\n",
+        "  output_images.append(draw_prediction_on_image(\n",
+        "      image[frame_idx, :, :, :].numpy().astype(np.int32),\n",
+        "      keypoints_with_scores, crop_region=None,\n",
+        "      close_figure=True, output_image_height=300))\n",
+        "  crop_region = determine_crop_region(\n",
+        "      keypoints_with_scores, image_height, image_width)\n",
+        "  bar.update(progress(frame_idx, num_frames-1))\n",
+        "\n",
+        "# Prepare gif visualization.\n",
+        "output = np.stack(output_images, axis=0)\n",
+        "to_gif(output, duration=100)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "9u_VGR6_BmbZ",
+        "5I3xBq80E3N_",
+        "L2JmA1xAEntQ"
+      ],
+      "name": "movenet.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/movinet.ipynb b/site/en/hub/tutorials/movinet.ipynb
new file mode 100644
index 00000000000..24600256cf9
--- /dev/null
+++ b/site/en/hub/tutorials/movinet.ipynb
@@ -0,0 +1,1047 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "toCy3v03Dwx7"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QKe-ubNcDvgv"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qFdPvlXBOdUN"
+      },
+      "source": [
+        "# MoViNet for streaming action recognition "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "-vxk2Kbc_KSP"
+      },
+      "source": [
+        "This tutorial demonstrates how to use a pretrained video classification model to classify an activity (such as dancing, swimming, biking etc) in the given video.   \n",
+        "\n",
+        "The model architecture used in this tutorial is called [MoViNet](https://arxiv.org/pdf/2103.11511.pdf) (Mobile Video Networks). MoVieNets are a family of efficient video classification models trained on huge dataset ([Kinetics 600](https://deepmind.com/research/open-source/kinetics)).\n",
+        "\n",
+        "In contrast to the [i3d models](https://tfhub.dev/s?q=i3d-kinetics) available on TF Hub, MoViNets also support frame-by-frame inference on streaming video. \n",
+        "\n",
+        "The pretrained models are available from [TF Hub](https://tfhub.dev/google/collections/movinet/1). The TF Hub collection also includes quantized models optimized for [TFLite](https://tensorflow.org/lite).\n",
+        "\n",
+        "The source for these models is available in the [TensorFlow Model Garden](https://github.com/tensorflow/models/tree/master/official/projects/movinet). This includes a [longer version of this tutorial](https://colab.sandbox.google.com/github/tensorflow/models/blob/master/official/projects/movinet/movinet_tutorial.ipynb) that also covers building and fine-tuning a MoViNet model. \n",
+        "\n",
+        "This MoViNet tutorial is part of a series of TensorFlow video tutorials. Here are the other three tutorials:\n",
+        "\n",
+        "- [Load video data](https://www.tensorflow.org/tutorials/load_data/video): This tutorial explains how to load and preprocess video data into a TensorFlow dataset pipeline from scratch.\n",
+        "- [Build a 3D CNN model for video classification](https://www.tensorflow.org/tutorials/video/video_classification). Note that this tutorial uses a (2+1)D CNN that decomposes the spatial and temporal aspects of 3D data; if you are using volumetric data such as an MRI scan, consider using a 3D CNN instead of a (2+1)D CNN.\n",
+        "- [Transfer learning for video classification with MoViNet](https://www.tensorflow.org/tutorials/video/transfer_learning_with_movinet): This tutorial explains how to use a pre-trained video classification model trained on a different dataset with the UCF-101 dataset.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3E96e1UKQ8uR"
+      },
+      "source": [
+        "![jumping jacks plot](https://storage.googleapis.com/tf_model_garden/vision/movinet/artifacts/jumpingjacks_plot.gif)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8_oLnvJy7kz5"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "For inference on smaller models (A0-A2), CPU is sufficient for this Colab."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GUgUMGmY1yq-"
+      },
+      "outputs": [],
+      "source": [
+        "!sudo apt install -y ffmpeg\n",
+        "!pip install -q mediapy"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "s3khsunT7kWa"
+      },
+      "outputs": [],
+      "source": [
+        "!pip uninstall -q -y opencv-python-headless\n",
+        "!pip install -q \"opencv-python-headless<4.3\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dI_1csl6Q-gH"
+      },
+      "outputs": [],
+      "source": [
+        "# Import libraries\n",
+        "import pathlib\n",
+        "\n",
+        "import matplotlib as mpl\n",
+        "import matplotlib.pyplot as plt\n",
+        "import mediapy as media\n",
+        "import numpy as np\n",
+        "import PIL\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import tqdm\n",
+        "\n",
+        "mpl.rcParams.update({\n",
+        "    'font.size': 10,\n",
+        "})"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pn8K9oWbmREi"
+      },
+      "source": [
+        "Get the kinetics 600 label list, and print the first few labels:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2VJUAcjhkfb3"
+      },
+      "outputs": [],
+      "source": [
+        "labels_path = tf.keras.utils.get_file(\n",
+        "    fname='labels.txt',\n",
+        "    origin='https://raw.githubusercontent.com/tensorflow/models/f8af2291cced43fc9f1d9b41ddbf772ae7b0d7d2/official/projects/movinet/files/kinetics_600_labels.txt'\n",
+        ")\n",
+        "labels_path = pathlib.Path(labels_path)\n",
+        "\n",
+        "lines = labels_path.read_text().splitlines()\n",
+        "KINETICS_600_LABELS = np.array([line.strip() for line in lines])\n",
+        "KINETICS_600_LABELS[:20]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G9BU5XsOmaq3"
+      },
+      "source": [
+        "To provide a simple example video for classification, we can load a short gif of jumping jacks being performed.\n",
+        "\n",
+        "![jumping jacks](https://github.com/tensorflow/models/raw/f8af2291cced43fc9f1d9b41ddbf772ae7b0d7d2/official/projects/movinet/files/jumpingjack.gif)\n",
+        "\n",
+        "Attribution: Footage shared by [Coach Bobby Bluford](https://www.youtube.com/watch?v=-AxHpj-EuPg) on YouTube under the CC-BY license."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8aFKMbr4mfSg"
+      },
+      "source": [
+        "Download the gif."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "w62jqXhaSb15"
+      },
+      "outputs": [],
+      "source": [
+        "jumpingjack_url = 'https://github.com/tensorflow/models/raw/f8af2291cced43fc9f1d9b41ddbf772ae7b0d7d2/official/projects/movinet/files/jumpingjack.gif'\n",
+        "jumpingjack_path = tf.keras.utils.get_file(\n",
+        "    fname='jumpingjack.gif',\n",
+        "    origin=jumpingjack_url,\n",
+        "    cache_dir='.', cache_subdir='.',\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hdRS_22PebfB"
+      },
+      "source": [
+        "Define a function to read a gif file into a `tf.Tensor`:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mPhmCu6oSi5f"
+      },
+      "outputs": [],
+      "source": [
+        "#@title\n",
+        "# Read and process a video\n",
+        "def load_gif(file_path, image_size=(224, 224)):\n",
+        "  \"\"\"Loads a gif file into a TF tensor.\n",
+        "\n",
+        "  Use images resized to match what's expected by your model.\n",
+        "  The model pages say the \"A2\" models expect 224 x 224 images at 5 fps\n",
+        "\n",
+        "  Args:\n",
+        "    file_path: path to the location of a gif file.\n",
+        "    image_size: a tuple of target size.\n",
+        "\n",
+        "  Returns:\n",
+        "    a video of the gif file\n",
+        "  \"\"\"\n",
+        "  # Load a gif file, convert it to a TF tensor\n",
+        "  raw = tf.io.read_file(file_path)\n",
+        "  video = tf.io.decode_gif(raw)\n",
+        "  # Resize the video\n",
+        "  video = tf.image.resize(video, image_size)\n",
+        "  # change dtype to a float32\n",
+        "  # Hub models always want images normalized to [0,1]\n",
+        "  # ref: https://www.tensorflow.org/hub/common_signatures/images#input\n",
+        "  video = tf.cast(video, tf.float32) / 255.\n",
+        "  return video"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Xx7cZm8vpDJm"
+      },
+      "source": [
+        "The video's shape is `(frames, height, width, colors)`"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E7k_PmbFSkHv"
+      },
+      "outputs": [],
+      "source": [
+        "jumpingjack=load_gif(jumpingjack_path)\n",
+        "jumpingjack.shape"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LcKFy3oedBvF"
+      },
+      "source": [
+        "## How to use the model\n",
+        "\n",
+        "This section contains a walkthrough showing how to use the [models from TensorFlow Hub](https://tfhub.dev/google/collections/movinet/1). If you just want to see the models in action, skip to the next section.\n",
+        "\n",
+        "There are two versions of each model: `base` and `streaming`.\n",
+        "\n",
+        "* The `base` version takes a video as input, and returns the probabilities averaged over the frames.\n",
+        "* The `streaming` version takes a video frame and an RNN state as input, and returns the predictions for that frame, and the new RNN state. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WQO6Zb8Hm-9q"
+      },
+      "source": [
+        "### The base model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RfnYU20JnPqp"
+      },
+      "source": [
+        "Download the [pretrained model from TensorFlow Hub](https://tfhub.dev/tensorflow/movinet/a2/base/kinetics-600/classification/3). "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FnpPo6HSR7qv"
+      },
+      "outputs": [],
+      "source": [
+        "%%time\n",
+        "id = 'a2'\n",
+        "mode = 'base'\n",
+        "version = '3'\n",
+        "hub_url = f'https://tfhub.dev/tensorflow/movinet/{id}/{mode}/kinetics-600/classification/{version}'\n",
+        "model = hub.load(hub_url)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jvaFwKhxndmb"
+      },
+      "source": [
+        "This version of the model has one `signature`. It takes an `image` argument which is a `tf.float32` with shape `(batch, frames, height, width, colors)`. It returns a dictionary containing one output: A `tf.float32` tensor of logits with shape `(batch, classes)`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7GzZ4Y03T_gH"
+      },
+      "outputs": [],
+      "source": [
+        "sig = model.signatures['serving_default']\n",
+        "print(sig.pretty_printed_signature())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "M4Xny1ANomi4"
+      },
+      "source": [
+        "To run this signature on the video you need to add the outer `batch` dimension to the video first."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LBOFEDG1XvZE"
+      },
+      "outputs": [],
+      "source": [
+        "#warmup\n",
+        "sig(image = jumpingjack[tf.newaxis, :1]);"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jCeW3KycVbGn"
+      },
+      "outputs": [],
+      "source": [
+        "%%time\n",
+        "logits = sig(image = jumpingjack[tf.newaxis, ...])\n",
+        "logits = logits['classifier_head'][0]\n",
+        "\n",
+        "print(logits.shape)\n",
+        "print()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AE8doqkPpxED"
+      },
+      "source": [
+        "Define a `get_top_k` function that packages the above output processing for later."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OozPNO6LvZ00"
+      },
+      "outputs": [],
+      "source": [
+        "#@title\n",
+        "# Get top_k labels and probabilities\n",
+        "def get_top_k(probs, k=5, label_map=KINETICS_600_LABELS):\n",
+        "  \"\"\"Outputs the top k model labels and probabilities on the given video.\n",
+        "\n",
+        "  Args:\n",
+        "    probs: probability tensor of shape (num_frames, num_classes) that represents\n",
+        "      the probability of each class on each frame.\n",
+        "    k: the number of top predictions to select.\n",
+        "    label_map: a list of labels to map logit indices to label strings.\n",
+        "\n",
+        "  Returns:\n",
+        "    a tuple of the top-k labels and probabilities.\n",
+        "  \"\"\"\n",
+        "  # Sort predictions to find top_k\n",
+        "  top_predictions = tf.argsort(probs, axis=-1, direction='DESCENDING')[:k]\n",
+        "  # collect the labels of top_k predictions\n",
+        "  top_labels = tf.gather(label_map, top_predictions, axis=-1)\n",
+        "  # decode lablels\n",
+        "  top_labels = [label.decode('utf8') for label in top_labels.numpy()]\n",
+        "  # top_k probabilities of the predictions\n",
+        "  top_probs = tf.gather(probs, top_predictions, axis=-1).numpy()\n",
+        "  return tuple(zip(top_labels, top_probs))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kTfKMT29pP_Z"
+      },
+      "source": [
+        "Convert the `logits` to probabilities, and look up the top 5 classes for the video. The model confirms that the video is probably of `jumping jacks`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Z-SrNGsGV5Mt"
+      },
+      "outputs": [],
+      "source": [
+        "probs = tf.nn.softmax(logits, axis=-1)\n",
+        "for label, p in get_top_k(probs):\n",
+        "  print(f'{label:20s}: {p:.3f}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ltdijoQpqjxZ"
+      },
+      "source": [
+        "### The streaming model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9dqdUPQXq45b"
+      },
+      "source": [
+        "The previous section used a model that runs over a whole video. Often when processing a video you don't want a single prediction at the end, you want to update predictions frame by frame. The `stream` versions of the model allow you to do this.\n",
+        "\n",
+        "Load the `stream` version of the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mxt0hRXFZkAM"
+      },
+      "outputs": [],
+      "source": [
+        "%%time\n",
+        "id = 'a2'\n",
+        "mode = 'stream'\n",
+        "version = '3'\n",
+        "hub_url = f'https://tfhub.dev/tensorflow/movinet/{id}/{mode}/kinetics-600/classification/{version}'\n",
+        "model = hub.load(hub_url)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pDswtsGgsYGS"
+      },
+      "source": [
+        "Using this model is slightly more complex than the `base` model. You have to keep track of the internal state of the model's RNNs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0fM_Vb1VsbDm"
+      },
+      "outputs": [],
+      "source": [
+        "list(model.signatures.keys())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ojr1_iYCtPvp"
+      },
+      "source": [
+        "The `init_states` signature takes the video's **shape** `(batch, frames, height, width, colors)` as input, and returns a large dictionary of tensors containing the initial RNN states: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "67loYFGpo_RP"
+      },
+      "outputs": [],
+      "source": [
+        "lines = model.signatures['init_states'].pretty_printed_signature().splitlines()\n",
+        "lines = lines[:10]\n",
+        "lines.append('      ...')\n",
+        "print('.\\n'.join(lines))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "v5lG3vejn5df"
+      },
+      "outputs": [],
+      "source": [
+        "initial_state = model.init_states(jumpingjack[tf.newaxis, ...].shape)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "J3DwmyHnuhH_"
+      },
+      "outputs": [],
+      "source": [
+        "type(initial_state)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "K8SyiEU6tB-e"
+      },
+      "outputs": [],
+      "source": [
+        "list(sorted(initial_state.keys()))[:5]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xeMCzJMBvwRF"
+      },
+      "source": [
+        "Once you have the initial state for the RNNs, you can pass the state and a video frame as input (keeping the `(batch, frames, height, width, colors)` shape for the video frame). The model returns a `(logits, state)` pair. \n",
+        "\n",
+        "After just seeing the first frame, the model is not convinced that the video is of \"jumping jacks\":"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "McSLdIgtsI3d"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = initial_state.copy()\n",
+        "\n",
+        "# Add the batch axis, take the first frme, but keep the frame-axis.\n",
+        "inputs['image'] = jumpingjack[tf.newaxis, 0:1, ...] "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WlH7PqLPX664"
+      },
+      "outputs": [],
+      "source": [
+        "# warmup\n",
+        "model(inputs);"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7uzNXtu7X5sr"
+      },
+      "outputs": [],
+      "source": [
+        "logits, new_state = model(inputs)\n",
+        "logits = logits[0]\n",
+        "probs = tf.nn.softmax(logits, axis=-1)\n",
+        "\n",
+        "for label, p in get_top_k(probs):\n",
+        "  print(f'{label:20s}: {p:.3f}')\n",
+        "\n",
+        "print()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oLU644FQwXSb"
+      },
+      "source": [
+        "If you run the model in a loop, passing the updated state with each frame, the model quickly converges to the correct result:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Fzm7T4ImmIEg"
+      },
+      "outputs": [],
+      "source": [
+        "%%time\n",
+        "state = initial_state.copy()\n",
+        "all_logits = []\n",
+        "\n",
+        "for n in range(len(jumpingjack)):\n",
+        "  inputs = state\n",
+        "  inputs['image'] = jumpingjack[tf.newaxis, n:n+1, ...]\n",
+        "  result, state = model(inputs)\n",
+        "  all_logits.append(logits)\n",
+        "\n",
+        "probabilities = tf.nn.softmax(all_logits, axis=-1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "B7UtHoSWcOT2"
+      },
+      "outputs": [],
+      "source": [
+        "for label, p in get_top_k(probabilities[-1]):\n",
+        "  print(f'{label:20s}: {p:.3f}')"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6ffV3NhZcsrv"
+      },
+      "outputs": [],
+      "source": [
+        "id = tf.argmax(probabilities[-1])\n",
+        "plt.plot(probabilities[:, id])\n",
+        "plt.xlabel('Frame #')\n",
+        "plt.ylabel(f\"p('{KINETICS_600_LABELS[id]}')\");"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d7MZ_AfRW845"
+      },
+      "source": [
+        "You may notice that the final probability is much more certain than in the previous section where you ran the `base` model. The `base` model returns an average of the predictions over the frames."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0Wij4tsyW8dR"
+      },
+      "outputs": [],
+      "source": [
+        "for label, p in get_top_k(tf.reduce_mean(probabilities, axis=0)):\n",
+        "  print(f'{label:20s}: {p:.3f}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qLUoC9ejggGo"
+      },
+      "source": [
+        "## Animate the predictions over time\n",
+        "\n",
+        "The previous section went into some details about how to use these models. This section builds on top of that to produce some nice inference animations. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "OnFqOXazoWgy"
+      },
+      "source": [
+        "The hidden cell below to defines helper functions used in this section."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "dx55NK3ZoZeh"
+      },
+      "outputs": [],
+      "source": [
+        "#@title\n",
+        "# Get top_k labels and probabilities predicted using MoViNets streaming model\n",
+        "def get_top_k_streaming_labels(probs, k=5, label_map=KINETICS_600_LABELS):\n",
+        "  \"\"\"Returns the top-k labels over an entire video sequence.\n",
+        "\n",
+        "  Args:\n",
+        "    probs: probability tensor of shape (num_frames, num_classes) that represents\n",
+        "      the probability of each class on each frame.\n",
+        "    k: the number of top predictions to select.\n",
+        "    label_map: a list of labels to map logit indices to label strings.\n",
+        "\n",
+        "  Returns:\n",
+        "    a tuple of the top-k probabilities, labels, and logit indices\n",
+        "  \"\"\"\n",
+        "  top_categories_last = tf.argsort(probs, -1, 'DESCENDING')[-1, :1]\n",
+        "  # Sort predictions to find top_k\n",
+        "  categories = tf.argsort(probs, -1, 'DESCENDING')[:, :k]\n",
+        "  categories = tf.reshape(categories, [-1])\n",
+        "\n",
+        "  counts = sorted([\n",
+        "      (i.numpy(), tf.reduce_sum(tf.cast(categories == i, tf.int32)).numpy())\n",
+        "      for i in tf.unique(categories)[0]\n",
+        "  ], key=lambda x: x[1], reverse=True)\n",
+        "\n",
+        "  top_probs_idx = tf.constant([i for i, _ in counts[:k]])\n",
+        "  top_probs_idx = tf.concat([top_categories_last, top_probs_idx], 0)\n",
+        "  # find unique indices of categories\n",
+        "  top_probs_idx = tf.unique(top_probs_idx)[0][:k+1]\n",
+        "  # top_k probabilities of the predictions\n",
+        "  top_probs = tf.gather(probs, top_probs_idx, axis=-1)\n",
+        "  top_probs = tf.transpose(top_probs, perm=(1, 0))\n",
+        "  # collect the labels of top_k predictions\n",
+        "  top_labels = tf.gather(label_map, top_probs_idx, axis=0)\n",
+        "  # decode the top_k labels\n",
+        "  top_labels = [label.decode('utf8') for label in top_labels.numpy()]\n",
+        "\n",
+        "  return top_probs, top_labels, top_probs_idx\n",
+        "\n",
+        "# Plot top_k predictions at a given time step\n",
+        "def plot_streaming_top_preds_at_step(\n",
+        "    top_probs,\n",
+        "    top_labels,\n",
+        "    step=None,\n",
+        "    image=None,\n",
+        "    legend_loc='lower left',\n",
+        "    duration_seconds=10,\n",
+        "    figure_height=500,\n",
+        "    playhead_scale=0.8,\n",
+        "    grid_alpha=0.3):\n",
+        "  \"\"\"Generates a plot of the top video model predictions at a given time step.\n",
+        "\n",
+        "  Args:\n",
+        "    top_probs: a tensor of shape (k, num_frames) representing the top-k\n",
+        "      probabilities over all frames.\n",
+        "    top_labels: a list of length k that represents the top-k label strings.\n",
+        "    step: the current time step in the range [0, num_frames].\n",
+        "    image: the image frame to display at the current time step.\n",
+        "    legend_loc: the placement location of the legend.\n",
+        "    duration_seconds: the total duration of the video.\n",
+        "    figure_height: the output figure height.\n",
+        "    playhead_scale: scale value for the playhead.\n",
+        "    grid_alpha: alpha value for the gridlines.\n",
+        "\n",
+        "  Returns:\n",
+        "    A tuple of the output numpy image, figure, and axes.\n",
+        "  \"\"\"\n",
+        "  # find number of top_k labels and frames in the video\n",
+        "  num_labels, num_frames = top_probs.shape\n",
+        "  if step is None:\n",
+        "    step = num_frames\n",
+        "  # Visualize frames and top_k probabilities of streaming video\n",
+        "  fig = plt.figure(figsize=(6.5, 7), dpi=300)\n",
+        "  gs = mpl.gridspec.GridSpec(8, 1)\n",
+        "  ax2 = plt.subplot(gs[:-3, :])\n",
+        "  ax = plt.subplot(gs[-3:, :])\n",
+        "  # display the frame\n",
+        "  if image is not None:\n",
+        "    ax2.imshow(image, interpolation='nearest')\n",
+        "    ax2.axis('off')\n",
+        "  # x-axis (frame number)\n",
+        "  preview_line_x = tf.linspace(0., duration_seconds, num_frames)\n",
+        "  # y-axis (top_k probabilities)\n",
+        "  preview_line_y = top_probs\n",
+        "\n",
+        "  line_x = preview_line_x[:step+1]\n",
+        "  line_y = preview_line_y[:, :step+1]\n",
+        "\n",
+        "  for i in range(num_labels):\n",
+        "    ax.plot(preview_line_x, preview_line_y[i], label=None, linewidth='1.5',\n",
+        "            linestyle=':', color='gray')\n",
+        "    ax.plot(line_x, line_y[i], label=top_labels[i], linewidth='2.0')\n",
+        "\n",
+        "\n",
+        "  ax.grid(which='major', linestyle=':', linewidth='1.0', alpha=grid_alpha)\n",
+        "  ax.grid(which='minor', linestyle=':', linewidth='0.5', alpha=grid_alpha)\n",
+        "\n",
+        "  min_height = tf.reduce_min(top_probs) * playhead_scale\n",
+        "  max_height = tf.reduce_max(top_probs)\n",
+        "  ax.vlines(preview_line_x[step], min_height, max_height, colors='red')\n",
+        "  ax.scatter(preview_line_x[step], max_height, color='red')\n",
+        "\n",
+        "  ax.legend(loc=legend_loc)\n",
+        "\n",
+        "  plt.xlim(0, duration_seconds)\n",
+        "  plt.ylabel('Probability')\n",
+        "  plt.xlabel('Time (s)')\n",
+        "  plt.yscale('log')\n",
+        "\n",
+        "  fig.tight_layout()\n",
+        "  fig.canvas.draw()\n",
+        "\n",
+        "  data = np.frombuffer(fig.canvas.tostring_rgb(), dtype=np.uint8)\n",
+        "  data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))\n",
+        "  plt.close()\n",
+        "\n",
+        "  figure_width = int(figure_height * data.shape[1] / data.shape[0])\n",
+        "  image = PIL.Image.fromarray(data).resize([figure_width, figure_height])\n",
+        "  image = np.array(image)\n",
+        "\n",
+        "  return image\n",
+        "\n",
+        "# Plotting top_k predictions from MoViNets streaming model\n",
+        "def plot_streaming_top_preds(\n",
+        "    probs,\n",
+        "    video,\n",
+        "    top_k=5,\n",
+        "    video_fps=25.,\n",
+        "    figure_height=500,\n",
+        "    use_progbar=True):\n",
+        "  \"\"\"Generates a video plot of the top video model predictions.\n",
+        "\n",
+        "  Args:\n",
+        "    probs: probability tensor of shape (num_frames, num_classes) that represents\n",
+        "      the probability of each class on each frame.\n",
+        "    video: the video to display in the plot.\n",
+        "    top_k: the number of top predictions to select.\n",
+        "    video_fps: the input video fps.\n",
+        "    figure_fps: the output video fps.\n",
+        "    figure_height: the height of the output video.\n",
+        "    use_progbar: display a progress bar.\n",
+        "\n",
+        "  Returns:\n",
+        "    A numpy array representing the output video.\n",
+        "  \"\"\"\n",
+        "  # select number of frames per second\n",
+        "  video_fps = 8.\n",
+        "  # select height of the image\n",
+        "  figure_height = 500\n",
+        "  # number of time steps of the given video\n",
+        "  steps = video.shape[0]\n",
+        "  # estimate duration of the video (in seconds)\n",
+        "  duration = steps / video_fps\n",
+        "  # estimate top_k probabilities and corresponding labels\n",
+        "  top_probs, top_labels, _ = get_top_k_streaming_labels(probs, k=top_k)\n",
+        "\n",
+        "  images = []\n",
+        "  step_generator = tqdm.trange(steps) if use_progbar else range(steps)\n",
+        "  for i in step_generator:\n",
+        "    image = plot_streaming_top_preds_at_step(\n",
+        "        top_probs=top_probs,\n",
+        "        top_labels=top_labels,\n",
+        "        step=i,\n",
+        "        image=video[i],\n",
+        "        duration_seconds=duration,\n",
+        "        figure_height=figure_height,\n",
+        "    )\n",
+        "    images.append(image)\n",
+        "\n",
+        "  return np.array(images)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eLgFBslcZOQO"
+      },
+      "source": [
+        "Start by running the streaming model across the frames of the video, and collecting the logits:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tXWR13wthnK5"
+      },
+      "outputs": [],
+      "source": [
+        "init_states = model.init_states(jumpingjack[tf.newaxis].shape)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YqSkt7l8ltwt"
+      },
+      "outputs": [],
+      "source": [
+        "# Insert your video clip here\n",
+        "video = jumpingjack\n",
+        "images = tf.split(video[tf.newaxis], video.shape[0], axis=1)\n",
+        "\n",
+        "all_logits = []\n",
+        "\n",
+        "# To run on a video, pass in one frame at a time\n",
+        "states = init_states\n",
+        "for image in tqdm.tqdm(images):\n",
+        "  # predictions for each frame\n",
+        "  logits, states = model({**states, 'image': image})\n",
+        "  all_logits.append(logits)\n",
+        "\n",
+        "# concatenating all the logits\n",
+        "logits = tf.concat(all_logits, 0)\n",
+        "# estimating probabilities\n",
+        "probs = tf.nn.softmax(logits, axis=-1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OOGcCMMJyuPl"
+      },
+      "outputs": [],
+      "source": [
+        "final_probs = probs[-1]\n",
+        "print('Top_k predictions and their probablities\\n')\n",
+        "for label, p in get_top_k(final_probs):\n",
+        "  print(f'{label:20s}: {p:.3f}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GaybT0rbZct-"
+      },
+      "source": [
+        "Convert the sequence of probabilities into a video:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Xdox556CtMRb"
+      },
+      "outputs": [],
+      "source": [
+        "# Generate a plot and output to a video tensor\n",
+        "plot_video = plot_streaming_top_preds(probs, video, video_fps=8.)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NSStKE9klCs3"
+      },
+      "outputs": [],
+      "source": [
+        "# For gif format, set codec='gif'\n",
+        "media.show_video(plot_video, fps=3)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LCImgZ3OdJw7"
+      },
+      "source": [
+        "## Resources\n",
+        "\n",
+        "The pretrained models are available from [TF Hub](https://tfhub.dev/google/collections/movinet/1). The TF Hub collection also includes quantized models optimized for [TFLite](https://tensorflow.org/lite).\n",
+        "\n",
+        "The source for these models is available in the [TensorFlow Model Garden](https://github.com/tensorflow/models/tree/master/official/projects/movinet). This includes a [longer version of this tutorial](https://colab.sandbox.google.com/github/tensorflow/models/blob/master/official/projects/movinet/movinet_tutorial.ipynb) that also covers building and fine-tuning a MoViNet model. "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gh5lLAo-HpVF"
+      },
+      "source": [
+        "## Next Steps\n",
+        "\n",
+        "To learn more about working with video data in TensorFlow, check out the following tutorials:\n",
+        "\n",
+        "* [Load video data](https://www.tensorflow.org/tutorials/load_data/video)\n",
+        "* [Build a 3D CNN model for video classification](https://www.tensorflow.org/tutorials/video/video_classification)\n",
+        "* [Transfer learning for video classification with MoViNet](https://www.tensorflow.org/tutorials/video/transfer_learning_with_movinet)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "movinet.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/object_detection.ipynb b/site/en/hub/tutorials/object_detection.ipynb
new file mode 100644
index 00000000000..e1262f3084c
--- /dev/null
+++ b/site/en/hub/tutorials/object_detection.ipynb
@@ -0,0 +1,442 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N6ZDpd9XzFeN"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "KUu4vOt5zI9d"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CxmDMK4yupqg"
+      },
+      "source": [
+        "# Object Detection\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Sy553YSVmYiK"
+      },
+      "source": [
+        "This Colab demonstrates use of a TF-Hub module trained to perform object detection."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v4XGxDrCkeip"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "6cPY9Ou4sWs_"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Imports and function definitions\n",
+        "\n",
+        "# For running inference on the TF-Hub module.\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "# For downloading the image.\n",
+        "import matplotlib.pyplot as plt\n",
+        "import tempfile\n",
+        "from six.moves.urllib.request import urlopen\n",
+        "from six import BytesIO\n",
+        "\n",
+        "# For drawing onto the image.\n",
+        "import numpy as np\n",
+        "from PIL import Image\n",
+        "from PIL import ImageColor\n",
+        "from PIL import ImageDraw\n",
+        "from PIL import ImageFont\n",
+        "from PIL import ImageOps\n",
+        "\n",
+        "# For measuring the inference time.\n",
+        "import time\n",
+        "\n",
+        "# Print Tensorflow version\n",
+        "print(tf.__version__)\n",
+        "\n",
+        "# Check available GPU devices.\n",
+        "print(\"The following GPU devices are available: %s\" % tf.test.gpu_device_name())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZGkrXGy62409"
+      },
+      "source": [
+        "## Example use"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vlA3CftFpRiW"
+      },
+      "source": [
+        "### Helper functions for downloading images and for visualization.\n",
+        "\n",
+        "Visualization code adapted from [TF object detection API](https://github.com/tensorflow/models/blob/master/research/object_detection/utils/visualization_utils.py) for the simplest required functionality."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D9IwDpOtpIHW"
+      },
+      "outputs": [],
+      "source": [
+        "def display_image(image):\n",
+        "  fig = plt.figure(figsize=(20, 15))\n",
+        "  plt.grid(False)\n",
+        "  plt.imshow(image)\n",
+        "\n",
+        "\n",
+        "def download_and_resize_image(url, new_width=256, new_height=256,\n",
+        "                              display=False):\n",
+        "  _, filename = tempfile.mkstemp(suffix=\".jpg\")\n",
+        "  response = urlopen(url)\n",
+        "  image_data = response.read()\n",
+        "  image_data = BytesIO(image_data)\n",
+        "  pil_image = Image.open(image_data)\n",
+        "  pil_image = ImageOps.fit(pil_image, (new_width, new_height), Image.LANCZOS)\n",
+        "  pil_image_rgb = pil_image.convert(\"RGB\")\n",
+        "  pil_image_rgb.save(filename, format=\"JPEG\", quality=90)\n",
+        "  print(\"Image downloaded to %s.\" % filename)\n",
+        "  if display:\n",
+        "    display_image(pil_image)\n",
+        "  return filename\n",
+        "\n",
+        "\n",
+        "def draw_bounding_box_on_image(image,\n",
+        "                               ymin,\n",
+        "                               xmin,\n",
+        "                               ymax,\n",
+        "                               xmax,\n",
+        "                               color,\n",
+        "                               font,\n",
+        "                               thickness=4,\n",
+        "                               display_str_list=()):\n",
+        "  \"\"\"Adds a bounding box to an image.\"\"\"\n",
+        "  draw = ImageDraw.Draw(image)\n",
+        "  im_width, im_height = image.size\n",
+        "  (left, right, top, bottom) = (xmin * im_width, xmax * im_width,\n",
+        "                                ymin * im_height, ymax * im_height)\n",
+        "  draw.line([(left, top), (left, bottom), (right, bottom), (right, top),\n",
+        "             (left, top)],\n",
+        "            width=thickness,\n",
+        "            fill=color)\n",
+        "\n",
+        "  # If the total height of the display strings added to the top of the bounding\n",
+        "  # box exceeds the top of the image, stack the strings below the bounding box\n",
+        "  # instead of above.\n",
+        "  display_str_heights = [font.getbbox(ds)[3] for ds in display_str_list]\n",
+        "  # Each display_str has a top and bottom margin of 0.05x.\n",
+        "  total_display_str_height = (1 + 2 * 0.05) * sum(display_str_heights)\n",
+        "\n",
+        "  if top > total_display_str_height:\n",
+        "    text_bottom = top\n",
+        "  else:\n",
+        "    text_bottom = top + total_display_str_height\n",
+        "  # Reverse list and print from bottom to top.\n",
+        "  for display_str in display_str_list[::-1]:\n",
+        "    bbox = font.getbbox(display_str)\n",
+        "    text_width, text_height = bbox[2], bbox[3]\n",
+        "    margin = np.ceil(0.05 * text_height)\n",
+        "    draw.rectangle([(left, text_bottom - text_height - 2 * margin),\n",
+        "                    (left + text_width, text_bottom)],\n",
+        "                   fill=color)\n",
+        "    draw.text((left + margin, text_bottom - text_height - margin),\n",
+        "              display_str,\n",
+        "              fill=\"black\",\n",
+        "              font=font)\n",
+        "    text_bottom -= text_height - 2 * margin\n",
+        "\n",
+        "\n",
+        "def draw_boxes(image, boxes, class_names, scores, max_boxes=10, min_score=0.1):\n",
+        "  \"\"\"Overlay labeled boxes on an image with formatted scores and label names.\"\"\"\n",
+        "  colors = list(ImageColor.colormap.values())\n",
+        "\n",
+        "  try:\n",
+        "    font = ImageFont.truetype(\"/usr/share/fonts/truetype/liberation/LiberationSansNarrow-Regular.ttf\",\n",
+        "                              25)\n",
+        "  except IOError:\n",
+        "    print(\"Font not found, using default font.\")\n",
+        "    font = ImageFont.load_default()\n",
+        "\n",
+        "  for i in range(min(boxes.shape[0], max_boxes)):\n",
+        "    if scores[i] >= min_score:\n",
+        "      ymin, xmin, ymax, xmax = tuple(boxes[i])\n",
+        "      display_str = \"{}: {}%\".format(class_names[i].decode(\"ascii\"),\n",
+        "                                     int(100 * scores[i]))\n",
+        "      color = colors[hash(class_names[i]) % len(colors)]\n",
+        "      image_pil = Image.fromarray(np.uint8(image)).convert(\"RGB\")\n",
+        "      draw_bounding_box_on_image(\n",
+        "          image_pil,\n",
+        "          ymin,\n",
+        "          xmin,\n",
+        "          ymax,\n",
+        "          xmax,\n",
+        "          color,\n",
+        "          font,\n",
+        "          display_str_list=[display_str])\n",
+        "      np.copyto(image, np.array(image_pil))\n",
+        "  return image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "D19UCu9Q2-_8"
+      },
+      "source": [
+        "## Apply module\n",
+        "\n",
+        "Load a public image from Open Images v4, save locally, and display."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "YLWNhjUY1mhg"
+      },
+      "outputs": [],
+      "source": [
+        "# By Heiko Gorski, Source: https://commons.wikimedia.org/wiki/File:Naxos_Taverna.jpg\n",
+        "image_url = \"https://upload.wikimedia.org/wikipedia/commons/6/60/Naxos_Taverna.jpg\"  #@param\n",
+        "downloaded_image_path = download_and_resize_image(image_url, 1280, 856, True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "t-VdfLbC1w51"
+      },
+      "source": [
+        "Pick an object detection module and apply on the downloaded image. Modules:\n",
+        "* **FasterRCNN+InceptionResNet V2**: high accuracy,\n",
+        "* **ssd+mobilenet V2**: small and fast."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "uazJ5ASc2_QE"
+      },
+      "outputs": [],
+      "source": [
+        "module_handle = \"https://tfhub.dev/google/faster_rcnn/openimages_v4/inception_resnet_v2/1\" #@param [\"https://tfhub.dev/google/openimages_v4/ssd/mobilenet_v2/1\", \"https://tfhub.dev/google/faster_rcnn/openimages_v4/inception_resnet_v2/1\"]\n",
+        "\n",
+        "detector = hub.load(module_handle).signatures['default']"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "znW8Fq1EC0x7"
+      },
+      "outputs": [],
+      "source": [
+        "def load_img(path):\n",
+        "  img = tf.io.read_file(path)\n",
+        "  img = tf.image.decode_jpeg(img, channels=3)\n",
+        "  return img"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kwGJV96WWBLH"
+      },
+      "outputs": [],
+      "source": [
+        "def run_detector(detector, path):\n",
+        "  img = load_img(path)\n",
+        "\n",
+        "  converted_img  = tf.image.convert_image_dtype(img, tf.float32)[tf.newaxis, ...]\n",
+        "  start_time = time.time()\n",
+        "  result = detector(converted_img)\n",
+        "  end_time = time.time()\n",
+        "\n",
+        "  result = {key:value.numpy() for key,value in result.items()}\n",
+        "\n",
+        "  print(\"Found %d objects.\" % len(result[\"detection_scores\"]))\n",
+        "  print(\"Inference time: \", end_time-start_time)\n",
+        "\n",
+        "  image_with_boxes = draw_boxes(\n",
+        "      img.numpy(), result[\"detection_boxes\"],\n",
+        "      result[\"detection_class_entities\"], result[\"detection_scores\"])\n",
+        "\n",
+        "  display_image(image_with_boxes)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vchaUW1XDodD"
+      },
+      "outputs": [],
+      "source": [
+        "run_detector(detector, downloaded_image_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WUUY3nfRX7VF"
+      },
+      "source": [
+        "### More images\n",
+        "Perform inference on some additional images with time tracking.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rubdr2JXfsa1"
+      },
+      "outputs": [],
+      "source": [
+        "image_urls = [\n",
+        "  # Source: https://commons.wikimedia.org/wiki/File:The_Coleoptera_of_the_British_islands_(Plate_125)_(8592917784).jpg\n",
+        "  \"https://upload.wikimedia.org/wikipedia/commons/1/1b/The_Coleoptera_of_the_British_islands_%28Plate_125%29_%288592917784%29.jpg\",\n",
+        "  # By Américo Toledano, Source: https://commons.wikimedia.org/wiki/File:Biblioteca_Maim%C3%B3nides,_Campus_Universitario_de_Rabanales_007.jpg\n",
+        "  \"https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/Biblioteca_Maim%C3%B3nides%2C_Campus_Universitario_de_Rabanales_007.jpg/1024px-Biblioteca_Maim%C3%B3nides%2C_Campus_Universitario_de_Rabanales_007.jpg\",\n",
+        "  # Source: https://commons.wikimedia.org/wiki/File:The_smaller_British_birds_(8053836633).jpg\n",
+        "  \"https://upload.wikimedia.org/wikipedia/commons/0/09/The_smaller_British_birds_%288053836633%29.jpg\",\n",
+        "  ]\n",
+        "\n",
+        "def detect_img(image_url):\n",
+        "  start_time = time.time()\n",
+        "  image_path = download_and_resize_image(image_url, 640, 480)\n",
+        "  run_detector(detector, image_path)\n",
+        "  end_time = time.time()\n",
+        "  print(\"Inference time:\",end_time-start_time)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "otPnrxMKIrj5"
+      },
+      "outputs": [],
+      "source": [
+        "detect_img(image_urls[0])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "H5F7DkD5NtOx"
+      },
+      "outputs": [],
+      "source": [
+        "detect_img(image_urls[1])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "DZ18R7dWNyoU"
+      },
+      "outputs": [],
+      "source": [
+        "detect_img(image_urls[2])"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "object_detection.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/retrieval_with_tf_hub_universal_encoder_qa.ipynb b/site/en/hub/tutorials/retrieval_with_tf_hub_universal_encoder_qa.ipynb
new file mode 100644
index 00000000000..0166a7408d5
--- /dev/null
+++ b/site/en/hub/tutorials/retrieval_with_tf_hub_universal_encoder_qa.ipynb
@@ -0,0 +1,361 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "VFMCdVJIIraw"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "code",
+        "id": "ZxMYj8OpIrCp"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0fO2R2BBKx3l"
+      },
+      "source": [
+        "# Multilingual Universal Sentence Encoder Q&A Retrieval\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
\n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zsDm_WgMNlJQ"
+      },
+      "source": [
+        "This is a demo for using [Universal Encoder Multilingual Q&A model](https://tfhub.dev/google/universal-sentence-encoder-multilingual-qa/3) for question-answer retrieval of text, illustrating the use of **question_encoder** and **response_encoder** of the model. We use sentences from [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) paragraphs as the demo dataset, each sentence and its context (the text surrounding the sentence) is encoded into high dimension embeddings with the **response_encoder**. These embeddings are stored in an index built using the [simpleneighbors](https://pypi.org/project/simpleneighbors/) library for question-answer retrieval.\n",
+        "\n",
+        "On retrieval a random question is selected from the [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) dataset and encoded into high dimension embedding with the **question_encoder** and query the  simpleneighbors index returning a list of approximate nearest neighbors in semantic space."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "U0eOW2LTWiLg"
+      },
+      "source": [
+        "### More models\n",
+        "You can find all currently hosted text embedding models [here](https://tfhub.dev/s?module-type=text-embedding) and all models that have been trained on SQuAD as well [here](https://tfhub.dev/s?dataset=squad)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ORy-KvWXGXBo"
+      },
+      "source": [
+        "## Setup\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "x00t_uJCEbeb"
+      },
+      "outputs": [],
+      "source": [
+        "%%capture\n",
+        "#@title Setup Environment\n",
+        "# Install the latest Tensorflow version.\n",
+        "!pip install -q \"tensorflow-text==2.11.*\"\n",
+        "!pip install -q simpleneighbors[annoy]\n",
+        "!pip install -q nltk\n",
+        "!pip install -q tqdm"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "DmeFAuVsyWxg"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Setup common imports and functions\n",
+        "import json\n",
+        "import nltk\n",
+        "import os\n",
+        "import pprint\n",
+        "import random\n",
+        "import simpleneighbors\n",
+        "import urllib\n",
+        "from IPython.display import HTML, display\n",
+        "from tqdm.notebook import tqdm\n",
+        "\n",
+        "import tensorflow.compat.v2 as tf\n",
+        "import tensorflow_hub as hub\n",
+        "from tensorflow_text import SentencepieceTokenizer\n",
+        "\n",
+        "nltk.download('punkt')\n",
+        "\n",
+        "\n",
+        "def download_squad(url):\n",
+        "  return json.load(urllib.request.urlopen(url))\n",
+        "\n",
+        "def extract_sentences_from_squad_json(squad):\n",
+        "  all_sentences = []\n",
+        "  for data in squad['data']:\n",
+        "    for paragraph in data['paragraphs']:\n",
+        "      sentences = nltk.tokenize.sent_tokenize(paragraph['context'])\n",
+        "      all_sentences.extend(zip(sentences, [paragraph['context']] * len(sentences)))\n",
+        "  return list(set(all_sentences)) # remove duplicates\n",
+        "\n",
+        "def extract_questions_from_squad_json(squad):\n",
+        "  questions = []\n",
+        "  for data in squad['data']:\n",
+        "    for paragraph in data['paragraphs']:\n",
+        "      for qas in paragraph['qas']:\n",
+        "        if qas['answers']:\n",
+        "          questions.append((qas['question'], qas['answers'][0]['text']))\n",
+        "  return list(set(questions))\n",
+        "\n",
+        "def output_with_highlight(text, highlight):\n",
+        "  output = \"
  •  \"\n",
+        "  i = text.find(highlight)\n",
+        "  while True:\n",
+        "    if i == -1:\n",
+        "      output += text\n",
+        "      break\n",
+        "    output += text[0:i]\n",
+        "    output += ''+text[i:i+len(highlight)]+''\n",
+        "    text = text[i+len(highlight):]\n",
+        "    i = text.find(highlight)\n",
+        "  return output + \"
    • \\n\"\n",
+        "\n",
+        "def display_nearest_neighbors(query_text, answer_text=None):\n",
+        "  query_embedding = model.signatures['question_encoder'](tf.constant([query_text]))['outputs'][0]\n",
+        "  search_results = index.nearest(query_embedding, n=num_results)\n",
+        "\n",
+        "  if answer_text:\n",
+        "    result_md = '''\n",
+        "    
    Random Question from SQuAD:
    \n",
+        "    
      %s
    \n",
+        "    
    Answer:
    \n",
+        "    
      %s
    \n",
+        "    ''' % (query_text , answer_text)\n",
+        "  else:\n",
+        "    result_md = '''\n",
+        "    
    Question:
    \n",
+        "    
      %s
    \n",
+        "    ''' % query_text\n",
+        "\n",
+        "  result_md += '''\n",
+        "    
    Retrieved sentences :\n",
+        "    
      \n",
+        "  '''\n",
+        "\n",
+        "  if answer_text:\n",
+        "    for s in search_results:\n",
+        "      result_md += output_with_highlight(s, answer_text)\n",
+        "  else:\n",
+        "    for s in search_results:\n",
+        "      result_md += '
    1. ' + s + '
      2. \\n'\n",
+        "\n",
+        "  result_md += \"
    \"\n",
+        "  display(HTML(result_md))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1kbkT8i3FL_C"
+      },
+      "source": [
+        "Run the following code block to download and extract the SQuAD dataset into:\n",
+        "\n",
+        "* **sentences** is a list of (text, context) tuples - each paragraph from the SQuAD dataset are split into sentences using nltk library and the sentence and paragraph text forms the (text, context) tuple.\n",
+        "* **questions** is a list of (question, answer) tuples.\n",
+        "\n",
+        "Note: You can use this demo to index the SQuAD train dataset or the smaller dev dataset (1.1 or 2.0) by selecting the **squad_url** below.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "iYqV2GAty_Eh"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Download and extract SQuAD data\n",
+        "squad_url = 'https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json' #@param [\"https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json\", \"https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json\", \"https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json\", \"https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json\"]\n",
+        "\n",
+        "squad_json = download_squad(squad_url)\n",
+        "sentences = extract_sentences_from_squad_json(squad_json)\n",
+        "questions = extract_questions_from_squad_json(squad_json)\n",
+        "print(\"%s sentences, %s questions extracted from SQuAD %s\" % (len(sentences), len(questions), squad_url))\n",
+        "\n",
+        "print(\"\\nExample sentence and context:\\n\")\n",
+        "sentence = random.choice(sentences)\n",
+        "print(\"sentence:\\n\")\n",
+        "pprint.pprint(sentence[0])\n",
+        "print(\"\\ncontext:\\n\")\n",
+        "pprint.pprint(sentence[1])\n",
+        "print()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9x3u-2uSGbDf"
+      },
+      "source": [
+        "The following code block setup the tensorflow graph **g** and **session** with the [Universal Encoder Multilingual Q&A model](https://tfhub.dev/google/universal-sentence-encoder-multilingual-qa/3)'s **question_encoder** and **response_encoder** signatures."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "44I0uCRQRiFO"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Load model from tensorflow hub\n",
+        "module_url = \"https://tfhub.dev/google/universal-sentence-encoder-multilingual-qa/3\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder-multilingual-qa/3\", \"https://tfhub.dev/google/universal-sentence-encoder-qa/3\"]\n",
+        "model = hub.load(module_url)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SCQpDmTZG0O6"
+      },
+      "source": [
+        "The following code block compute the embeddings for all the text, context tuples and store them in a [simpleneighbors](https://pypi.org/project/simpleneighbors/) index using the **response_encoder**.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FwDUryIfSLp2"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Compute embeddings and build simpleneighbors index\n",
+        "batch_size = 100\n",
+        "\n",
+        "encodings = model.signatures['response_encoder'](\n",
+        "  input=tf.constant([sentences[0][0]]),\n",
+        "  context=tf.constant([sentences[0][1]]))\n",
+        "index = simpleneighbors.SimpleNeighbors(\n",
+        "    len(encodings['outputs'][0]), metric='angular')\n",
+        "\n",
+        "print('Computing embeddings for %s sentences' % len(sentences))\n",
+        "slices = zip(*(iter(sentences),) * batch_size)\n",
+        "num_batches = int(len(sentences) / batch_size)\n",
+        "for s in tqdm(slices, total=num_batches):\n",
+        "  response_batch = list([r for r, c in s])\n",
+        "  context_batch = list([c for r, c in s])\n",
+        "  encodings = model.signatures['response_encoder'](\n",
+        "    input=tf.constant(response_batch),\n",
+        "    context=tf.constant(context_batch)\n",
+        "  )\n",
+        "  for batch_index, batch in enumerate(response_batch):\n",
+        "    index.add_one(batch, encodings['outputs'][batch_index])\n",
+        "\n",
+        "index.build()\n",
+        "print('simpleneighbors index for %s sentences built.' % len(sentences))\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZkNcjoPzHJpP"
+      },
+      "source": [
+        "On retrieval, the question is encoded using the **question_encoder** and the question embedding is used to query the simpleneighbors index."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "J0xTw2w3UViK"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Retrieve nearest neighbors for a random question from SQuAD\n",
+        "num_results = 25 #@param {type:\"slider\", min:5, max:40, step:1}\n",
+        "\n",
+        "query = random.choice(questions)\n",
+        "display_nearest_neighbors(query[0], query[1])"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "VFMCdVJIIraw"
+      ],
+      "name": "retrieval_with_tf_hub_universal_encoder_qa.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/s3gan_generation_with_tf_hub.ipynb b/site/en/hub/tutorials/s3gan_generation_with_tf_hub.ipynb
new file mode 100644
index 00000000000..bd73cffebdf
--- /dev/null
+++ b/site/en/hub/tutorials/s3gan_generation_with_tf_hub.ipynb
@@ -0,0 +1,429 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BhN1AplL0Hpv"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LMgeG2swVVi6"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AqBuuwrIxlGs"
+      },
+      "source": [
+        "# Generating Images with Little Data Using S3GAN\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "p5AWAusyySDA"
+      },
+      "source": [
+        "This notebook is a demo of Generative Adversarial Networks trained on ImageNet with as little as 2.5% labeled data using self- and semi-supervised learning techniques. Both generator and discriminator models are available on [TF Hub](https://tfhub.dev/s?publisher=google&q=compare_gan).\n",
+        "\n",
+        "For more information about the models and the training procedure see our [blogpost](https://ai.googleblog.com/2019/03/reducing-need-for-labeled-data-in.html) and the [paper](https://arxiv.org/abs/1903.02271) [1].\n",
+        "The code for training these models is available on [GitHub](https://github.com/google/compare_gan).\n",
+        "\n",
+        "To get started, connect to a runtime and follow these steps:\n",
+        "\n",
+        "1. (Optional) Select a model in the second code cell below.\n",
+        "2. Click **Runtime > Run all** to run each cell in order.\n",
+        "  * Afterwards, the interactive visualizations should update automatically when you modify the settings using the sliders and dropdown menus.\n",
+        "\n",
+        "Note: if you run into any issues, you can try restarting the runtime and rerunning all cells from scratch by clicking **Runtime > Restart and run all...**.\n",
+        "\n",
+        "[1] Mario Lucic\\*, Michael Tschannen\\*, Marvin Ritter\\*, Xiaohua Zhai, Olivier\n",
+        "    Bachem, Sylvain Gelly, [High-Fidelity Image Generation With Fewer Labels](https://arxiv.org/abs/1903.02271), ICML 2019."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_m5jsOM9kXWP"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NhlMa_tHs0_W"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Imports and utility functions\n",
+        "import os\n",
+        "\n",
+        "import IPython\n",
+        "from IPython.display import display\n",
+        "import numpy as np\n",
+        "import PIL.Image\n",
+        "import pandas as pd\n",
+        "import six\n",
+        "\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "tf.disable_v2_behavior()\n",
+        "\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "def imgrid(imarray, cols=8, pad=1):\n",
+        "  pad = int(pad)\n",
+        "  assert pad >= 0\n",
+        "  cols = int(cols)\n",
+        "  assert cols >= 1\n",
+        "  N, H, W, C = imarray.shape\n",
+        "  rows = int(np.ceil(N / float(cols)))\n",
+        "  batch_pad = rows * cols - N\n",
+        "  assert batch_pad >= 0\n",
+        "  post_pad = [batch_pad, pad, pad, 0]\n",
+        "  pad_arg = [[0, p] for p in post_pad]\n",
+        "  imarray = np.pad(imarray, pad_arg, 'constant')\n",
+        "  H += pad\n",
+        "  W += pad\n",
+        "  grid = (imarray\n",
+        "          .reshape(rows, cols, H, W, C)\n",
+        "          .transpose(0, 2, 1, 3, 4)\n",
+        "          .reshape(rows*H, cols*W, C))\n",
+        "  return grid[:-pad, :-pad]\n",
+        "\n",
+        "\n",
+        "def imshow(a, format='png', jpeg_fallback=True):\n",
+        "  a = np.asarray(a, dtype=np.uint8)\n",
+        "  if six.PY3:\n",
+        "    str_file = six.BytesIO()\n",
+        "  else:\n",
+        "    str_file = six.StringIO()\n",
+        "  PIL.Image.fromarray(a).save(str_file, format)\n",
+        "  png_data = str_file.getvalue()\n",
+        "  try:\n",
+        "    disp = display(IPython.display.Image(png_data))\n",
+        "  except IOError:\n",
+        "    if jpeg_fallback and format != 'jpeg':\n",
+        "      print ('Warning: image was too large to display in format \"{}\"; '\n",
+        "             'trying jpeg instead.').format(format)\n",
+        "      return imshow(a, format='jpeg')\n",
+        "    else:\n",
+        "      raise\n",
+        "  return disp\n",
+        "\n",
+        "\n",
+        "class Generator(object):\n",
+        "\n",
+        "  def __init__(self, module_spec):\n",
+        "    self._module_spec = module_spec\n",
+        "    self._sess = None\n",
+        "    self._graph = tf.Graph()\n",
+        "    self._load_model()\n",
+        "\n",
+        "  @property\n",
+        "  def z_dim(self):\n",
+        "    return self._z.shape[-1].value\n",
+        "\n",
+        "  @property\n",
+        "  def conditional(self):\n",
+        "    return self._labels is not None\n",
+        "\n",
+        "  def _load_model(self):\n",
+        "    with self._graph.as_default():\n",
+        "      self._generator = hub.Module(self._module_spec, name=\"gen_module\",\n",
+        "                                   tags={\"gen\", \"bsNone\"})\n",
+        "      input_info = self._generator.get_input_info_dict()\n",
+        "      inputs = {k: tf.placeholder(v.dtype, v.get_shape().as_list(), k)\n",
+        "                for k, v in self._generator.get_input_info_dict().items()}\n",
+        "      self._samples = self._generator(inputs=inputs, as_dict=True)[\"generated\"]\n",
+        "      print(\"Inputs:\", inputs)\n",
+        "      print(\"Outputs:\", self._samples)\n",
+        "      self._z = inputs[\"z\"]\n",
+        "      self._labels = inputs.get(\"labels\", None)\n",
+        "\n",
+        "  def _init_session(self):\n",
+        "    if self._sess is None:\n",
+        "      self._sess = tf.Session(graph=self._graph)\n",
+        "      self._sess.run(tf.global_variables_initializer())\n",
+        "\n",
+        "  def get_noise(self, num_samples, seed=None):\n",
+        "    if np.isscalar(seed):\n",
+        "      np.random.seed(seed)\n",
+        "      return np.random.normal(size=[num_samples, self.z_dim])\n",
+        "    z = np.empty(shape=(len(seed), self.z_dim), dtype=np.float32)\n",
+        "    for i, s in enumerate(seed):\n",
+        "      np.random.seed(s)\n",
+        "      z[i] = np.random.normal(size=[self.z_dim])\n",
+        "    return z\n",
+        "\n",
+        "  def get_samples(self, z, labels=None):\n",
+        "    with self._graph.as_default():\n",
+        "      self._init_session()\n",
+        "      feed_dict = {self._z: z}\n",
+        "      if self.conditional:\n",
+        "        assert labels is not None\n",
+        "        assert labels.shape[0] == z.shape[0]\n",
+        "        feed_dict[self._labels] = labels\n",
+        "      samples = self._sess.run(self._samples, feed_dict=feed_dict)\n",
+        "      return np.uint8(np.clip(256 * samples, 0, 255))\n",
+        "\n",
+        "\n",
+        "class Discriminator(object):\n",
+        "\n",
+        "  def __init__(self, module_spec):\n",
+        "    self._module_spec = module_spec\n",
+        "    self._sess = None\n",
+        "    self._graph = tf.Graph()\n",
+        "    self._load_model()\n",
+        "\n",
+        "  @property\n",
+        "  def conditional(self):\n",
+        "    return \"labels\" in self._inputs\n",
+        "\n",
+        "  @property\n",
+        "  def image_shape(self):\n",
+        "    return self._inputs[\"images\"].shape.as_list()[1:]\n",
+        "\n",
+        "  def _load_model(self):\n",
+        "    with self._graph.as_default():\n",
+        "      self._discriminator = hub.Module(self._module_spec, name=\"disc_module\",\n",
+        "                                       tags={\"disc\", \"bsNone\"})\n",
+        "      input_info = self._discriminator.get_input_info_dict()\n",
+        "      self._inputs = {k: tf.placeholder(v.dtype, v.get_shape().as_list(), k)\n",
+        "                      for k, v in input_info.items()}\n",
+        "      self._outputs = self._discriminator(inputs=self._inputs, as_dict=True)\n",
+        "      print(\"Inputs:\", self._inputs)\n",
+        "      print(\"Outputs:\", self._outputs)\n",
+        "\n",
+        "  def _init_session(self):\n",
+        "    if self._sess is None:\n",
+        "      self._sess = tf.Session(graph=self._graph)\n",
+        "      self._sess.run(tf.global_variables_initializer())\n",
+        "\n",
+        "  def predict(self, images, labels=None):\n",
+        "    with self._graph.as_default():\n",
+        "      self._init_session()\n",
+        "      feed_dict = {self._inputs[\"images\"]: images}\n",
+        "      if \"labels\" in self._inputs:\n",
+        "        assert labels is not None\n",
+        "        assert labels.shape[0] == images.shape[0]\n",
+        "        feed_dict[self._inputs[\"labels\"]] = labels\n",
+        "      return self._sess.run(self._outputs, feed_dict=feed_dict)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "msTFS1UPkugr"
+      },
+      "source": [
+        "## Select a model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-hBEi9IFdoI-"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Select a model { run: \"auto\" }\n",
+        "\n",
+        "model_name = \"S3GAN 128x128 20% labels (FID 6.9, IS 98.1)\"  # @param [\"S3GAN 256x256 10% labels (FID 8.8, IS 130.7)\", \"S3GAN 128x128 2.5% labels (FID 12.6, IS 48.7)\", \"S3GAN 128x128 5% labels (FID 8.4, IS 74.0)\", \"S3GAN 128x128 10% labels (FID 7.6, IS 90.3)\", \"S3GAN 128x128 20% labels (FID 6.9, IS 98.1)\"]\n",
+        "models = {\n",
+        "    \"S3GAN 256x256 10% labels\": \"https://tfhub.dev/google/compare_gan/s3gan_10_256x256/1\",\n",
+        "    \"S3GAN 128x128 2.5% labels\": \"https://tfhub.dev/google/compare_gan/s3gan_2_5_128x128/1\",\n",
+        "    \"S3GAN 128x128 5% labels\": \"https://tfhub.dev/google/compare_gan/s3gan_5_128x128/1\",\n",
+        "    \"S3GAN 128x128 10% labels\": \"https://tfhub.dev/google/compare_gan/s3gan_10_128x128/1\",\n",
+        "    \"S3GAN 128x128 20% labels\": \"https://tfhub.dev/google/compare_gan/s3gan_20_128x128/1\",\n",
+        "}\n",
+        "\n",
+        "module_spec = models[model_name.split(\" (\")[0]]\n",
+        "print(\"Module spec:\", module_spec)\n",
+        "\n",
+        "tf.reset_default_graph()\n",
+        "print(\"Loading model...\")\n",
+        "sampler = Generator(module_spec)\n",
+        "print(\"Model loaded.\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ePQuAme_kxLj"
+      },
+      "source": [
+        "## Sample"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "kGgTXtFYq_FV"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Sampling { run: \"auto\" }\n",
+        "\n",
+        "num_rows = 2  # @param {type: \"slider\", min:1, max:16}\n",
+        "num_cols = 3  # @param {type: \"slider\", min:1, max:16}\n",
+        "noise_seed = 23  # @param {type:\"slider\", min:0, max:100, step:1}\n",
+        "label_str = \"980) volcano\"  # @param [\"-1) Random\", \"0) tench, Tinca tinca\", \"1) goldfish, Carassius auratus\", \"2) great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias\", \"3) tiger shark, Galeocerdo cuvieri\", \"4) hammerhead, hammerhead shark\", \"5) electric ray, crampfish, numbfish, torpedo\", \"6) stingray\", \"7) cock\", \"8) hen\", \"9) ostrich, Struthio camelus\", \"10) brambling, Fringilla montifringilla\", \"11) goldfinch, Carduelis carduelis\", \"12) house finch, linnet, Carpodacus mexicanus\", \"13) junco, snowbird\", \"14) indigo bunting, indigo finch, indigo bird, Passerina cyanea\", \"15) robin, American robin, Turdus migratorius\", \"16) bulbul\", \"17) jay\", \"18) magpie\", \"19) chickadee\", \"20) water ouzel, dipper\", \"21) kite\", \"22) bald eagle, American eagle, Haliaeetus leucocephalus\", \"23) vulture\", \"24) great grey owl, great gray owl, Strix nebulosa\", \"25) European fire salamander, Salamandra salamandra\", \"980) volcano\"]\n",
+        "\n",
+        "num_samples = num_rows * num_cols\n",
+        "z = sampler.get_noise(num_samples, seed=noise_seed)\n",
+        "\n",
+        "label = int(label_str.split(')')[0])\n",
+        "if label == -1:\n",
+        "  labels = np.random.randint(0, num_classes, size=(num_samples))\n",
+        "else:\n",
+        "  labels = np.asarray([label] * num_samples)\n",
+        "\n",
+        "samples = sampler.get_samples(z, labels)\n",
+        "imshow(imgrid(samples, cols=num_cols))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "vCffdVZvTtxL"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Interpolation { run: \"auto\" }\n",
+        "\n",
+        "num_samples = 1  # @param {type: \"slider\", min: 1, max: 6, step: 1}\n",
+        "num_interps = 6  # @param {type: \"slider\", min: 2, max: 10, step: 1}\n",
+        "noise_seed_A = 11  # @param {type: \"slider\", min: 0, max: 100, step: 1}\n",
+        "noise_seed_B = 0  # @param {type: \"slider\", min: 0, max: 100, step: 1}\n",
+        "label_str = \"1) goldfish, Carassius auratus\"  # @param [\"0) tench, Tinca tinca\", \"1) goldfish, Carassius auratus\", \"2) great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias\", \"3) tiger shark, Galeocerdo cuvieri\", \"4) hammerhead, hammerhead shark\", \"5) electric ray, crampfish, numbfish, torpedo\", \"6) stingray\", \"7) cock\", \"8) hen\", \"9) ostrich, Struthio camelus\", \"10) brambling, Fringilla montifringilla\", \"11) goldfinch, Carduelis carduelis\", \"12) house finch, linnet, Carpodacus mexicanus\", \"13) junco, snowbird\", \"14) indigo bunting, indigo finch, indigo bird, Passerina cyanea\", \"15) robin, American robin, Turdus migratorius\", \"16) bulbul\", \"17) jay\", \"18) magpie\", \"19) chickadee\", \"20) water ouzel, dipper\", \"21) kite\", \"22) bald eagle, American eagle, Haliaeetus leucocephalus\", \"23) vulture\", \"24) great grey owl, great gray owl, Strix nebulosa\", \"25) European fire salamander, Salamandra salamandra\"]\n",
+        "\n",
+        "\n",
+        "def interpolate(A, B, num_interps):\n",
+        "  alphas = np.linspace(0, 1, num_interps)\n",
+        "  if A.shape != B.shape:\n",
+        "    raise ValueError('A and B must have the same shape to interpolate.')\n",
+        "  return np.array([((1-a)*A + a*B)/np.sqrt(a**2 + (1-a)**2) for a in alphas])\n",
+        "\n",
+        "\n",
+        "def interpolate_and_shape(A, B, num_interps):\n",
+        "  interps = interpolate(A, B, num_interps)\n",
+        "  return (interps.transpose(1, 0, *range(2, len(interps.shape)))\n",
+        "                 .reshape(num_samples * num_interps, -1))\n",
+        "\n",
+        "label = int(label_str.split(')')[0])\n",
+        "labels = np.asarray([label] * num_samples * num_interps)\n",
+        "\n",
+        "\n",
+        "z_A = sampler.get_noise(num_samples, seed=noise_seed_A)\n",
+        "z_B = sampler.get_noise(num_samples, seed=noise_seed_B)\n",
+        "z = interpolate_and_shape(z_A, z_B, num_interps)\n",
+        "\n",
+        "samples = sampler.get_samples(z, labels)\n",
+        "imshow(imgrid(samples, cols=num_interps))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "esW0Up95Ob6U"
+      },
+      "source": [
+        "## Discriminator"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ButxPSq0OzgL"
+      },
+      "outputs": [],
+      "source": [
+        "disc = Discriminator(module_spec)\n",
+        "\n",
+        "batch_size = 4\n",
+        "num_classes = 1000\n",
+        "images = np.random.random(size=[batch_size] + disc.image_shape)\n",
+        "labels = np.random.randint(0, num_classes, size=(batch_size))\n",
+        "\n",
+        "disc.predict(images, labels=labels)"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "BhN1AplL0Hpv"
+      ],
+      "name": "s3gan_generation_with_tf_hub.ipynb",
+      "provenance": [],
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/semantic_approximate_nearest_neighbors.ipynb b/site/en/hub/tutorials/semantic_approximate_nearest_neighbors.ipynb
new file mode 100644
index 00000000000..55bcebcc447
--- /dev/null
+++ b/site/en/hub/tutorials/semantic_approximate_nearest_neighbors.ipynb
@@ -0,0 +1,882 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ACbjNjyO4f_8"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MCM50vaM4jiK"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9qOVy-_vmuUP"
+      },
+      "source": [
+        "# Semantic Search with Approximate Nearest Neighbors and Text Embeddings\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7Hks9F5qq6m2"
+      },
+      "source": [
+        "This tutorial illustrates how to generate embeddings from a [TensorFlow Hub](https://tfhub.dev) (TF-Hub) module given input data, and build an approximate nearest neighbours (ANN) index using the extracted embeddings. The index can then be used for real-time similarity matching and retrieval. \n",
+        "\n",
+        "When dealing with a large corpus of data, it's not efficient to perform exact matching by scanning the whole repository to find the most similar items to a given query in real-time. Thus, we use an approximate similarity matching algorithm which allows us to trade off a little bit of accuracy in finding exact nearest neighbor matches for a significant boost in speed. \n",
+        "\n",
+        "In this tutorial, we show an example of real-time text search over a corpus of news headlines to find the headlines that are most similar to a query. Unlike keyword search, this captures the semantic similarity encoded in the text embedding.\n",
+        "\n",
+        "The steps of this tutorial are:\n",
+        "1. Download sample data.\n",
+        "2. Generate embeddings for the data using a TF-Hub module\n",
+        "3. Build an ANN index for the embeddings\n",
+        "4. Use the index for similarity matching\n",
+        "\n",
+        "We use [Apache Beam](https://beam.apache.org/documentation/programming-guide/) with [TensorFlow Transform](https://www.tensorflow.org/tfx/tutorials/transform/simple) (TF-Transform) to generate the embeddings from the TF-Hub module. We also use Spotify's [ANNOY](https://github.com/spotify/annoy) library to build the approximate nearest neighbours index. You can find benchmarking of ANN framework in this [Github repository](https://github.com/erikbern/ann-benchmarks).\n",
+        "\n",
+        "This tutorial uses TensorFlow 1.0 and works only with TF1 [Hub modules](https://www.tensorflow.org/hub/tf1_hub_module) from TF-Hub. See the updated [TF2 version of this tutorial](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/tf2_semantic_approximate_nearest_neighbors.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q0jr0QK9qO5P"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "whMRj9qeqed4"
+      },
+      "source": [
+        "Install the required libraries."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qmXkLPoaqS--"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q apache_beam\n",
+        "!pip install -q 'scikit_learn~=0.23.0'  # For gaussian_random_matrix.\n",
+        "!pip install -q annoy"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A-vBZiCCqld0"
+      },
+      "source": [
+        "Import the required libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6NTYbdWcseuK"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "import pathlib\n",
+        "import pickle\n",
+        "from collections import namedtuple\n",
+        "from datetime import datetime\n",
+        "\n",
+        "import numpy as np\n",
+        "import apache_beam as beam\n",
+        "import annoy\n",
+        "from sklearn.random_projection import gaussian_random_matrix\n",
+        "\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "import tensorflow_hub as hub"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_GF0GnLqGdPQ"
+      },
+      "outputs": [],
+      "source": [
+        "# TFT needs to be installed afterwards\n",
+        "!pip install -q tensorflow_transform==0.24\n",
+        "import tensorflow_transform as tft\n",
+        "import tensorflow_transform.beam as tft_beam"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tx0SZa6-7b-f"
+      },
+      "outputs": [],
+      "source": [
+        "print('TF version: {}'.format(tf.__version__))\n",
+        "print('TF-Hub version: {}'.format(hub.__version__))\n",
+        "print('TF-Transform version: {}'.format(tft.__version__))\n",
+        "print('Apache Beam version: {}'.format(beam.__version__))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "P6Imq876rLWx"
+      },
+      "source": [
+        "## 1. Download Sample Data\n",
+        "\n",
+        "[A Million News Headlines](https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/SYBGZL#) dataset contains news headlines published over a period of 15 years sourced from the reputable Australian Broadcasting Corp. (ABC). This news dataset has a summarised historical record of noteworthy events in the globe from early-2003 to end-2017 with a more granular focus on Australia. \n",
+        "\n",
+        "**Format**: Tab-separated two-column data: 1) publication date and 2) headline text. We are only interested in the headline text.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OpF57n8e5C9D"
+      },
+      "outputs": [],
+      "source": [
+        "!wget 'https://dataverse.harvard.edu/api/access/datafile/3450625?format=tab&gbrecs=true' -O raw.tsv\n",
+        "!wc -l raw.tsv\n",
+        "!head raw.tsv"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Reeoc9z0zTxJ"
+      },
+      "source": [
+        "For simplicity, we only keep the headline text and remove the publication date"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "INPWa4upv_yJ"
+      },
+      "outputs": [],
+      "source": [
+        "!rm -r corpus\n",
+        "!mkdir corpus\n",
+        "\n",
+        "with open('corpus/text.txt', 'w') as out_file:\n",
+        "  with open('raw.tsv', 'r') as in_file:\n",
+        "    for line in in_file:\n",
+        "      headline = line.split('\\t')[1].strip().strip('\"')\n",
+        "      out_file.write(headline+\"\\n\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5-oedX40z6o2"
+      },
+      "outputs": [],
+      "source": [
+        "!tail corpus/text.txt"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ls0Zh7kYz3PM"
+      },
+      "source": [
+        "## Helper function to load a TF-Hub module"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vSt_jmyKz3Xp"
+      },
+      "outputs": [],
+      "source": [
+        "def load_module(module_url):\n",
+        "  embed_module = hub.Module(module_url)\n",
+        "  placeholder = tf.placeholder(dtype=tf.string)\n",
+        "  embed = embed_module(placeholder)\n",
+        "  session = tf.Session()\n",
+        "  session.run([tf.global_variables_initializer(), tf.tables_initializer()])\n",
+        "  print('TF-Hub module is loaded.')\n",
+        "\n",
+        "  def _embeddings_fn(sentences):\n",
+        "    computed_embeddings = session.run(\n",
+        "        embed, feed_dict={placeholder: sentences})\n",
+        "    return computed_embeddings\n",
+        "\n",
+        "  return _embeddings_fn"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2AngMtH50jNb"
+      },
+      "source": [
+        "## 2. Generate Embeddings for the Data.\n",
+        "\n",
+        "In this tutorial, we use the [Universal Sentence Encoder](https://tfhub.dev/google/universal-sentence-encoder/2) to generate embeddings for the headline data. The sentence embeddings can then be easily used to compute sentence level meaning similarity. We run the embedding generation process using Apache Beam and TF-Transform."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "F_DvXnDB1pEX"
+      },
+      "source": [
+        "### Embedding extraction method"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yL7OEY1E0A35"
+      },
+      "outputs": [],
+      "source": [
+        "encoder = None\n",
+        "\n",
+        "def embed_text(text, module_url, random_projection_matrix):\n",
+        "  # Beam will run this function in different processes that need to\n",
+        "  # import hub and load embed_fn (if not previously loaded)\n",
+        "  global encoder\n",
+        "  if not encoder:\n",
+        "    encoder = hub.Module(module_url)\n",
+        "  embedding = encoder(text)\n",
+        "  if random_projection_matrix is not None:\n",
+        "    # Perform random projection for the embedding\n",
+        "    embedding = tf.matmul(\n",
+        "        embedding, tf.cast(random_projection_matrix, embedding.dtype))\n",
+        "  return embedding\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_don5gXy9D59"
+      },
+      "source": [
+        "### Make TFT preprocess_fn method"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fwYlrzzK9ECE"
+      },
+      "outputs": [],
+      "source": [
+        "def make_preprocess_fn(module_url, random_projection_matrix=None):\n",
+        "  '''Makes a tft preprocess_fn'''\n",
+        "\n",
+        "  def _preprocess_fn(input_features):\n",
+        "    '''tft preprocess_fn'''\n",
+        "    text = input_features['text']\n",
+        "    # Generate the embedding for the input text\n",
+        "    embedding = embed_text(text, module_url, random_projection_matrix)\n",
+        "    \n",
+        "    output_features = {\n",
+        "        'text': text, \n",
+        "        'embedding': embedding\n",
+        "        }\n",
+        "        \n",
+        "    return output_features\n",
+        "  \n",
+        "  return _preprocess_fn"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SQ492LN7A-NZ"
+      },
+      "source": [
+        "### Create dataset metadata"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d2D4332VA-2V"
+      },
+      "outputs": [],
+      "source": [
+        "def create_metadata():\n",
+        "  '''Creates metadata for the raw data'''\n",
+        "  from tensorflow_transform.tf_metadata import dataset_metadata\n",
+        "  from tensorflow_transform.tf_metadata import schema_utils\n",
+        "  feature_spec = {'text': tf.FixedLenFeature([], dtype=tf.string)}\n",
+        "  schema = schema_utils.schema_from_feature_spec(feature_spec)\n",
+        "  metadata = dataset_metadata.DatasetMetadata(schema)\n",
+        "  return metadata"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5zlSLPzRBm6H"
+      },
+      "source": [
+        "### Beam pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jCGUIB172m2G"
+      },
+      "outputs": [],
+      "source": [
+        "def run_hub2emb(args):\n",
+        "  '''Runs the embedding generation pipeline'''\n",
+        "\n",
+        "  options = beam.options.pipeline_options.PipelineOptions(**args)\n",
+        "  args = namedtuple(\"options\", args.keys())(*args.values())\n",
+        "\n",
+        "  raw_metadata = create_metadata()\n",
+        "  converter = tft.coders.CsvCoder(\n",
+        "      column_names=['text'], schema=raw_metadata.schema)\n",
+        "\n",
+        "  with beam.Pipeline(args.runner, options=options) as pipeline:\n",
+        "    with tft_beam.Context(args.temporary_dir):\n",
+        "      # Read the sentences from the input file\n",
+        "      sentences = ( \n",
+        "          pipeline\n",
+        "          | 'Read sentences from files' >> beam.io.ReadFromText(\n",
+        "              file_pattern=args.data_dir)\n",
+        "          | 'Convert to dictionary' >> beam.Map(converter.decode)\n",
+        "      )\n",
+        "\n",
+        "      sentences_dataset = (sentences, raw_metadata)\n",
+        "      preprocess_fn = make_preprocess_fn(args.module_url, args.random_projection_matrix)\n",
+        "      # Generate the embeddings for the sentence using the TF-Hub module\n",
+        "      embeddings_dataset, _ = (\n",
+        "          sentences_dataset\n",
+        "          | 'Extract embeddings' >> tft_beam.AnalyzeAndTransformDataset(preprocess_fn)\n",
+        "      )\n",
+        "\n",
+        "      embeddings, transformed_metadata = embeddings_dataset\n",
+        "      # Write the embeddings to TFRecords files\n",
+        "      embeddings | 'Write embeddings to TFRecords' >> beam.io.tfrecordio.WriteToTFRecord(\n",
+        "          file_path_prefix='{}/emb'.format(args.output_dir),\n",
+        "          file_name_suffix='.tfrecords',\n",
+        "          coder=tft.coders.ExampleProtoCoder(transformed_metadata.schema))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uHbq4t2gCDAG"
+      },
+      "source": [
+        "### Generaring Random Projection Weight Matrix\n",
+        "\n",
+        "[Random projection](https://en.wikipedia.org/wiki/Random_projection) is a simple, yet powerfull technique used to reduce the dimensionality of a set of points which lie in Euclidean space. For a theoretical background, see the [Johnson-Lindenstrauss lemma](https://en.wikipedia.org/wiki/Johnson%E2%80%93Lindenstrauss_lemma).\n",
+        "\n",
+        "Reducing the dimensionality of the embeddings with random projection means less time needed to build and query the ANN index.\n",
+        "\n",
+        "In this tutorial we use [Gaussian Random Projection](https://en.wikipedia.org/wiki/Random_projection#Gaussian_random_projection) from the [Scikit-learn](https://scikit-learn.org/stable/modules/random_projection.html#gaussian-random-projection) library."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "T1aYPeOUCDIP"
+      },
+      "outputs": [],
+      "source": [
+        "def generate_random_projection_weights(original_dim, projected_dim):\n",
+        "  random_projection_matrix = None\n",
+        "  if projected_dim and original_dim > projected_dim:\n",
+        "    random_projection_matrix = gaussian_random_matrix(\n",
+        "        n_components=projected_dim, n_features=original_dim).T\n",
+        "    print(\"A Gaussian random weight matrix was creates with shape of {}\".format(random_projection_matrix.shape))\n",
+        "    print('Storing random projection matrix to disk...')\n",
+        "    with open('random_projection_matrix', 'wb') as handle:\n",
+        "      pickle.dump(random_projection_matrix, \n",
+        "                  handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
+        "        \n",
+        "  return random_projection_matrix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CHxZX2Z3Nk64"
+      },
+      "source": [
+        "### Set parameters\n",
+        "If you want to build an index using the original embedding space without random projection, set the `projected_dim` parameter to `None`. Note that this will slow down the indexing step for high-dimensional embeddings."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "feMVXFL0NlIM"
+      },
+      "outputs": [],
+      "source": [
+        "module_url = 'https://tfhub.dev/google/universal-sentence-encoder/2' #@param {type:\"string\"}\n",
+        "projected_dim = 64  #@param {type:\"number\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "On-MbzD922kb"
+      },
+      "source": [
+        "### Run pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y3I1Wv4i21yY"
+      },
+      "outputs": [],
+      "source": [
+        "import tempfile\n",
+        "\n",
+        "output_dir = pathlib.Path(tempfile.mkdtemp())\n",
+        "temporary_dir = pathlib.Path(tempfile.mkdtemp())\n",
+        "\n",
+        "g = tf.Graph()\n",
+        "with g.as_default():\n",
+        "  original_dim = load_module(module_url)(['']).shape[1]\n",
+        "  random_projection_matrix = None\n",
+        "\n",
+        "  if projected_dim:\n",
+        "    random_projection_matrix = generate_random_projection_weights(\n",
+        "        original_dim, projected_dim)\n",
+        "\n",
+        "args = {\n",
+        "    'job_name': 'hub2emb-{}'.format(datetime.utcnow().strftime('%y%m%d-%H%M%S')),\n",
+        "    'runner': 'DirectRunner',\n",
+        "    'batch_size': 1024,\n",
+        "    'data_dir': 'corpus/*.txt',\n",
+        "    'output_dir': output_dir,\n",
+        "    'temporary_dir': temporary_dir,\n",
+        "    'module_url': module_url,\n",
+        "    'random_projection_matrix': random_projection_matrix,\n",
+        "}\n",
+        "\n",
+        "print(\"Pipeline args are set.\")\n",
+        "args"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iS9obmeP4ZOA"
+      },
+      "outputs": [],
+      "source": [
+        "!rm -r {output_dir}\n",
+        "!rm -r {temporary_dir}\n",
+        "\n",
+        "print(\"Running pipeline...\")\n",
+        "%time run_hub2emb(args)\n",
+        "print(\"Pipeline is done.\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JAwOo7gQWvVd"
+      },
+      "outputs": [],
+      "source": [
+        "!ls {output_dir}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HVnee4e6U90u"
+      },
+      "source": [
+        "Read some of the generated embeddings..."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-K7pGXlXOj1N"
+      },
+      "outputs": [],
+      "source": [
+        "import itertools\n",
+        "\n",
+        "embed_file = os.path.join(output_dir, 'emb-00000-of-00001.tfrecords')\n",
+        "sample = 5\n",
+        "record_iterator =  tf.io.tf_record_iterator(path=embed_file)\n",
+        "for string_record in itertools.islice(record_iterator, sample):\n",
+        "  example = tf.train.Example()\n",
+        "  example.ParseFromString(string_record)\n",
+        "  text = example.features.feature['text'].bytes_list.value\n",
+        "  embedding = np.array(example.features.feature['embedding'].float_list.value)\n",
+        "  print(\"Embedding dimensions: {}\".format(embedding.shape[0]))\n",
+        "  print(\"{}: {}\".format(text, embedding[:10]))\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "agGoaMSgY8wN"
+      },
+      "source": [
+        "## 3. Build the ANN Index for the Embeddings\n",
+        "\n",
+        "[ANNOY](https://github.com/spotify/annoy) (Approximate Nearest Neighbors Oh Yeah) is a C++ library with Python bindings to search for points in space that are close to a given query point. It also creates large read-only file-based data structures that are mmapped into memory. It is built and used by [Spotify](https://www.spotify.com) for music recommendations."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UcPDspU3WjgH"
+      },
+      "outputs": [],
+      "source": [
+        "def build_index(embedding_files_pattern, index_filename, vector_length, \n",
+        "    metric='angular', num_trees=100):\n",
+        "  '''Builds an ANNOY index'''\n",
+        "\n",
+        "  annoy_index = annoy.AnnoyIndex(vector_length, metric=metric)\n",
+        "  # Mapping between the item and its identifier in the index\n",
+        "  mapping = {}\n",
+        "\n",
+        "  embed_files = tf.gfile.Glob(embedding_files_pattern)\n",
+        "  print('Found {} embedding file(s).'.format(len(embed_files)))\n",
+        "\n",
+        "  item_counter = 0\n",
+        "  for f, embed_file in enumerate(embed_files):\n",
+        "    print('Loading embeddings in file {} of {}...'.format(\n",
+        "      f+1, len(embed_files)))\n",
+        "    record_iterator = tf.io.tf_record_iterator(\n",
+        "      path=embed_file)\n",
+        "\n",
+        "    for string_record in record_iterator:\n",
+        "      example = tf.train.Example()\n",
+        "      example.ParseFromString(string_record)\n",
+        "      text = example.features.feature['text'].bytes_list.value[0].decode(\"utf-8\")\n",
+        "      mapping[item_counter] = text\n",
+        "      embedding = np.array(\n",
+        "        example.features.feature['embedding'].float_list.value)\n",
+        "      annoy_index.add_item(item_counter, embedding)\n",
+        "      item_counter += 1\n",
+        "      if item_counter % 100000 == 0:\n",
+        "        print('{} items loaded to the index'.format(item_counter))\n",
+        "\n",
+        "  print('A total of {} items added to the index'.format(item_counter))\n",
+        "\n",
+        "  print('Building the index with {} trees...'.format(num_trees))\n",
+        "  annoy_index.build(n_trees=num_trees)\n",
+        "  print('Index is successfully built.')\n",
+        "  \n",
+        "  print('Saving index to disk...')\n",
+        "  annoy_index.save(index_filename)\n",
+        "  print('Index is saved to disk.')\n",
+        "  print(\"Index file size: {} GB\".format(\n",
+        "    round(os.path.getsize(index_filename) / float(1024 ** 3), 2)))\n",
+        "  annoy_index.unload()\n",
+        "\n",
+        "  print('Saving mapping to disk...')\n",
+        "  with open(index_filename + '.mapping', 'wb') as handle:\n",
+        "    pickle.dump(mapping, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
+        "  print('Mapping is saved to disk.')\n",
+        "  print(\"Mapping file size: {} MB\".format(\n",
+        "    round(os.path.getsize(index_filename + '.mapping') / float(1024 ** 2), 2)))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AgyOQhUq6FNE"
+      },
+      "outputs": [],
+      "source": [
+        "embedding_files = \"{}/emb-*.tfrecords\".format(output_dir)\n",
+        "embedding_dimension = projected_dim\n",
+        "index_filename = \"index\"\n",
+        "\n",
+        "!rm {index_filename}\n",
+        "!rm {index_filename}.mapping\n",
+        "\n",
+        "%time build_index(embedding_files, index_filename, embedding_dimension)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ic31Tm5cgAd5"
+      },
+      "outputs": [],
+      "source": [
+        "!ls"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "maGxDl8ufP-p"
+      },
+      "source": [
+        "## 4. Use the Index for Similarity Matching\n",
+        "Now we can use the ANN index to find news headlines that are semantically close to an input query."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_dIs8W78fYPp"
+      },
+      "source": [
+        "### Load the index and the mapping files"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jlTTrbQHayvb"
+      },
+      "outputs": [],
+      "source": [
+        "index = annoy.AnnoyIndex(embedding_dimension)\n",
+        "index.load(index_filename, prefault=True)\n",
+        "print('Annoy index is loaded.')\n",
+        "with open(index_filename + '.mapping', 'rb') as handle:\n",
+        "  mapping = pickle.load(handle)\n",
+        "print('Mapping file is loaded.')\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "y6liFMSUh08J"
+      },
+      "source": [
+        "### Similarity matching method"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mUxjTag8hc16"
+      },
+      "outputs": [],
+      "source": [
+        "def find_similar_items(embedding, num_matches=5):\n",
+        "  '''Finds similar items to a given embedding in the ANN index'''\n",
+        "  ids = index.get_nns_by_vector(\n",
+        "  embedding, num_matches, search_k=-1, include_distances=False)\n",
+        "  items = [mapping[i] for i in ids]\n",
+        "  return items"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hjerNpmZja0A"
+      },
+      "source": [
+        "### Extract embedding from a given query"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a0IIXzfBjZ19"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the TF-Hub module\n",
+        "print(\"Loading the TF-Hub module...\")\n",
+        "g = tf.Graph()\n",
+        "with g.as_default():\n",
+        "  embed_fn = load_module(module_url)\n",
+        "print(\"TF-Hub module is loaded.\")\n",
+        "\n",
+        "random_projection_matrix = None\n",
+        "if os.path.exists('random_projection_matrix'):\n",
+        "  print(\"Loading random projection matrix...\")\n",
+        "  with open('random_projection_matrix', 'rb') as handle:\n",
+        "    random_projection_matrix = pickle.load(handle)\n",
+        "  print('random projection matrix is loaded.')\n",
+        "\n",
+        "def extract_embeddings(query):\n",
+        "  '''Generates the embedding for the query'''\n",
+        "  query_embedding =  embed_fn([query])[0]\n",
+        "  if random_projection_matrix is not None:\n",
+        "    query_embedding = query_embedding.dot(random_projection_matrix)\n",
+        "  return query_embedding"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kCoCNROujEIO"
+      },
+      "outputs": [],
+      "source": [
+        "extract_embeddings(\"Hello Machine Learning!\")[:10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nE_Q60nCk_ZB"
+      },
+      "source": [
+        "### Enter a query to find the most similar items"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "wC0uLjvfk5nB"
+      },
+      "outputs": [],
+      "source": [
+        "#@title { run: \"auto\" }\n",
+        "query = \"confronting global challenges\" #@param {type:\"string\"}\n",
+        "print(\"Generating embedding for the query...\")\n",
+        "%time query_embedding = extract_embeddings(query)\n",
+        "\n",
+        "print(\"\")\n",
+        "print(\"Finding relevant items in the index...\")\n",
+        "%time items = find_similar_items(query_embedding, 10)\n",
+        "\n",
+        "print(\"\")\n",
+        "print(\"Results:\")\n",
+        "print(\"=========\")\n",
+        "for item in items:\n",
+        "  print(item)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wwtMtyOeDKwt"
+      },
+      "source": [
+        "## Want to learn more?\n",
+        "\n",
+        "You can learn more about TensorFlow at [tensorflow.org](https://www.tensorflow.org/) and see the TF-Hub API documentation at [tensorflow.org/hub](https://www.tensorflow.org/hub/). Find available TensorFlow Hub modules at [tfhub.dev](https://tfhub.dev/) including more text embedding modules and image feature vector modules.\n",
+        "\n",
+        "Also check out the [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/) which is Google's fast-paced, practical introduction to machine learning."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "ls0Zh7kYz3PM",
+        "_don5gXy9D59",
+        "SQ492LN7A-NZ"
+      ],
+      "name": "semantic_approximate_nearest_neighbors.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder.ipynb b/site/en/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder.ipynb
new file mode 100644
index 00000000000..0c2874bc030
--- /dev/null
+++ b/site/en/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder.ipynb
@@ -0,0 +1,363 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RUymE2l9GZfO"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "code",
+        "id": "JMyTNwSJGGWg"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "co7MV6sX7Xto"
+      },
+      "source": [
+        "# Universal Sentence Encoder\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eAVQGidpL8v5"
+      },
+      "source": [
+        "This notebook illustrates how to access the Universal Sentence Encoder and use it for sentence similarity and sentence classification tasks.\n",
+        "\n",
+        "The Universal Sentence Encoder makes getting sentence level embeddings as easy as it has historically been to lookup the embeddings for individual words. The sentence embeddings can then be trivially used to compute sentence level meaning similarity as well as to enable better performance on downstream classification tasks using less supervised training data.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pOTzp8O36CyQ"
+      },
+      "source": [
+        "## Setup\n",
+        "\n",
+        "This section sets up the environment for access to the Universal Sentence Encoder on TF Hub and provides examples of applying the encoder to words, sentences, and paragraphs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lVjNK8shFKOC"
+      },
+      "outputs": [],
+      "source": [
+        "%%capture\n",
+        "!pip3 install seaborn"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "63Pd3nJnTl-i"
+      },
+      "source": [
+        "More detailed information about installing Tensorflow can be found at [https://www.tensorflow.org/install/](https://www.tensorflow.org/install/)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "zwty8Z6mAkdV"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Load the Universal Sentence Encoder's TF Hub module\n",
+        "from absl import logging\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "import tensorflow_hub as hub\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import os\n",
+        "import pandas as pd\n",
+        "import re\n",
+        "import seaborn as sns\n",
+        "\n",
+        "module_url = \"https://tfhub.dev/google/universal-sentence-encoder/4\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/4\", \"https://tfhub.dev/google/universal-sentence-encoder-large/5\"]\n",
+        "model = hub.load(module_url)\n",
+        "print (\"module %s loaded\" % module_url)\n",
+        "def embed(input):\n",
+        "  return model(input)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q8F4LNGFqOiq"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Compute a representation for each message, showing various lengths supported.\n",
+        "word = \"Elephant\"\n",
+        "sentence = \"I am a sentence for which I would like to get its embedding.\"\n",
+        "paragraph = (\n",
+        "    \"Universal Sentence Encoder embeddings also support short paragraphs. \"\n",
+        "    \"There is no hard limit on how long the paragraph is. Roughly, the longer \"\n",
+        "    \"the more 'diluted' the embedding will be.\")\n",
+        "messages = [word, sentence, paragraph]\n",
+        "\n",
+        "# Reduce logging output.\n",
+        "logging.set_verbosity(logging.ERROR)\n",
+        "\n",
+        "message_embeddings = embed(messages)\n",
+        "\n",
+        "for i, message_embedding in enumerate(np.array(message_embeddings).tolist()):\n",
+        "  print(\"Message: {}\".format(messages[i]))\n",
+        "  print(\"Embedding size: {}\".format(len(message_embedding)))\n",
+        "  message_embedding_snippet = \", \".join(\n",
+        "      (str(x) for x in message_embedding[:3]))\n",
+        "  print(\"Embedding: [{}, ...]\\n\".format(message_embedding_snippet))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BnvjATdy64eR"
+      },
+      "source": [
+        "# Semantic Textual Similarity Task Example\n",
+        "\n",
+        "The embeddings produced by the Universal Sentence Encoder are approximately normalized. The semantic similarity of two sentences can be trivially computed as the inner product of the encodings."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "h1FFCTKm7ba4"
+      },
+      "outputs": [],
+      "source": [
+        "def plot_similarity(labels, features, rotation):\n",
+        "  corr = np.inner(features, features)\n",
+        "  sns.set(font_scale=1.2)\n",
+        "  g = sns.heatmap(\n",
+        "      corr,\n",
+        "      xticklabels=labels,\n",
+        "      yticklabels=labels,\n",
+        "      vmin=0,\n",
+        "      vmax=1,\n",
+        "      cmap=\"YlOrRd\")\n",
+        "  g.set_xticklabels(labels, rotation=rotation)\n",
+        "  g.set_title(\"Semantic Textual Similarity\")\n",
+        "\n",
+        "def run_and_plot(messages_):\n",
+        "  message_embeddings_ = embed(messages_)\n",
+        "  plot_similarity(messages_, message_embeddings_, 90)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "339tuJ5Pwqqv"
+      },
+      "source": [
+        "## Similarity Visualized\n",
+        "Here we show the similarity in a heat map. The final graph is a 9x9 matrix where each entry `[i, j]` is colored based on the inner product of the encodings for sentence `i` and `j`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cPMCaxrZwp7t"
+      },
+      "outputs": [],
+      "source": [
+        "messages = [\n",
+        "    # Smartphones\n",
+        "    \"I like my phone\",\n",
+        "    \"My phone is not good.\",\n",
+        "    \"Your cellphone looks great.\",\n",
+        "\n",
+        "    # Weather\n",
+        "    \"Will it snow tomorrow?\",\n",
+        "    \"Recently a lot of hurricanes have hit the US\",\n",
+        "    \"Global warming is real\",\n",
+        "\n",
+        "    # Food and health\n",
+        "    \"An apple a day, keeps the doctors away\",\n",
+        "    \"Eating strawberries is healthy\",\n",
+        "    \"Is paleo better than keto?\",\n",
+        "\n",
+        "    # Asking about age\n",
+        "    \"How old are you?\",\n",
+        "    \"what is your age?\",\n",
+        "]\n",
+        "\n",
+        "run_and_plot(messages)\n",
+        "               "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6FjdeCqPJeg-"
+      },
+      "source": [
+        "## Evaluation: STS (Semantic Textual Similarity) Benchmark\n",
+        "\n",
+        "The [**STS Benchmark**](https://ixa2.si.ehu.eus/stswiki/stswiki.html#STS_benchmark) provides an intrinsic evaluation of the degree to which similarity scores computed using sentence embeddings align with human judgements. The benchmark requires systems to return similarity scores for a diverse selection of sentence pairs. [Pearson correlation](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient) is then used to evaluate the quality of the machine similarity scores against human judgements."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "q5nuBbI1iFQR"
+      },
+      "source": [
+        "### Download data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VOs8ZfOnJeBF"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas\n",
+        "import scipy\n",
+        "import math\n",
+        "import csv\n",
+        "\n",
+        "sts_dataset = tf.keras.utils.get_file(\n",
+        "    fname=\"Stsbenchmark.tar.gz\",\n",
+        "    origin=\"http://ixa2.si.ehu.es/stswiki/images/4/48/Stsbenchmark.tar.gz\",\n",
+        "    extract=True)\n",
+        "sts_dev = pandas.read_table(\n",
+        "    os.path.join(os.path.dirname(sts_dataset), \"stsbenchmark\", \"sts-dev.csv\"),\n",
+        "    skip_blank_lines=True,\n",
+        "    usecols=[4, 5, 6],\n",
+        "    names=[\"sim\", \"sent_1\", \"sent_2\"])\n",
+        "sts_test = pandas.read_table(\n",
+        "    os.path.join(\n",
+        "        os.path.dirname(sts_dataset), \"stsbenchmark\", \"sts-test.csv\"),\n",
+        "    quoting=csv.QUOTE_NONE,\n",
+        "    skip_blank_lines=True,\n",
+        "    usecols=[4, 5, 6],\n",
+        "    names=[\"sim\", \"sent_1\", \"sent_2\"])\n",
+        "# cleanup some NaN values in sts_dev\n",
+        "sts_dev = sts_dev[[isinstance(s, str) for s in sts_dev['sent_2']]]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8OKy8WhnKRe_"
+      },
+      "source": [
+        "### Evaluate Sentence Embeddings"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "W-q2r7jyZGb7"
+      },
+      "outputs": [],
+      "source": [
+        "sts_data = sts_dev #@param [\"sts_dev\", \"sts_test\"] {type:\"raw\"}\n",
+        "\n",
+        "def run_sts_benchmark(batch):\n",
+        "  sts_encode1 = tf.nn.l2_normalize(embed(tf.constant(batch['sent_1'].tolist())), axis=1)\n",
+        "  sts_encode2 = tf.nn.l2_normalize(embed(tf.constant(batch['sent_2'].tolist())), axis=1)\n",
+        "  cosine_similarities = tf.reduce_sum(tf.multiply(sts_encode1, sts_encode2), axis=1)\n",
+        "  clip_cosine_similarities = tf.clip_by_value(cosine_similarities, -1.0, 1.0)\n",
+        "  scores = 1.0 - tf.acos(clip_cosine_similarities) / math.pi\n",
+        "  \"\"\"Returns the similarity scores\"\"\"\n",
+        "  return scores\n",
+        "\n",
+        "dev_scores = sts_data['sim'].tolist()\n",
+        "scores = []\n",
+        "for batch in np.array_split(sts_data, 10):\n",
+        "  scores.extend(run_sts_benchmark(batch))\n",
+        "\n",
+        "pearson_correlation = scipy.stats.pearsonr(scores, dev_scores)\n",
+        "print('Pearson correlation coefficient = {0}\\np-value = {1}'.format(\n",
+        "    pearson_correlation[0], pearson_correlation[1]))"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "RUymE2l9GZfO"
+      ],
+      "name": "semantic_similarity_with_tf_hub_universal_encoder.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder_lite.ipynb b/site/en/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder_lite.ipynb
new file mode 100644
index 00000000000..78d4eebadb0
--- /dev/null
+++ b/site/en/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder_lite.ipynb
@@ -0,0 +1,537 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IJhWonqQN7u0"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MegtYH2UN8tT"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MlHqSdgSEwPE"
+      },
+      "source": [
+        "# Universal Sentence Encoder-Lite demo\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "j0HuiScHQ3OK"
+      },
+      "source": [
+        "This Colab illustrates how to use the Universal Sentence Encoder-Lite for sentence similarity task. This module is very similar to [Universal Sentence Encoder](https://www.tensorflow.org/hub/modules/google/universal-sentence-encoder/2) with the only difference that you need to run [SentencePiece](https://github.com/google/sentencepiece) processing on your input sentences.\n",
+        "\n",
+        "The Universal Sentence Encoder makes getting sentence level embeddings as easy as it has historically been to lookup the embeddings for individual words. The sentence embeddings can then be trivially used to compute sentence level meaning similarity as well as to enable better performance on downstream classification tasks using less supervised training data."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wqCB2pyK-WSU"
+      },
+      "source": [
+        "# Getting started"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rWeEjoO5M0Cx"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f5_potQBMzcU"
+      },
+      "outputs": [],
+      "source": [
+        "# Install seaborn for pretty visualizations\n",
+        "!pip3 install --quiet seaborn\n",
+        "# Install SentencePiece package\n",
+        "# SentencePiece package is needed for Universal Sentence Encoder Lite. We'll\n",
+        "# use it for all the text processing and sentence feature ID lookup.\n",
+        "!pip3 install --quiet sentencepiece"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dMTa6V4a-cmf"
+      },
+      "outputs": [],
+      "source": [
+        "from absl import logging\n",
+        "\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "tf.disable_v2_behavior()\n",
+        "\n",
+        "import tensorflow_hub as hub\n",
+        "import sentencepiece as spm\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import os\n",
+        "import pandas as pd\n",
+        "import re\n",
+        "import seaborn as sns"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WPXYQDBiFJHd"
+      },
+      "source": [
+        "## Load the module from TF-Hub"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HEWUT-lmAkxM"
+      },
+      "outputs": [],
+      "source": [
+        "module = hub.Module(\"https://tfhub.dev/google/universal-sentence-encoder-lite/2\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5277Z-9qARYF"
+      },
+      "outputs": [],
+      "source": [
+        "input_placeholder = tf.sparse_placeholder(tf.int64, shape=[None, None])\n",
+        "encodings = module(\n",
+        "    inputs=dict(\n",
+        "        values=input_placeholder.values,\n",
+        "        indices=input_placeholder.indices,\n",
+        "        dense_shape=input_placeholder.dense_shape))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Yydbhuba_nek"
+      },
+      "source": [
+        "## Load SentencePiece model from the TF-Hub Module\n",
+        "The SentencePiece model is conveniently stored inside the module's assets. It has to be loaded in order to initialize the processor."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2CyUjKzE_tcJ"
+      },
+      "outputs": [],
+      "source": [
+        "with tf.Session() as sess:\n",
+        "  spm_path = sess.run(module(signature=\"spm_path\"))\n",
+        "\n",
+        "sp = spm.SentencePieceProcessor()\n",
+        "with tf.io.gfile.GFile(spm_path, mode=\"rb\") as f:\n",
+        "  sp.LoadFromSerializedProto(f.read())\n",
+        "print(\"SentencePiece model loaded at {}.\".format(spm_path))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6y5kkN-l-5QV"
+      },
+      "outputs": [],
+      "source": [
+        "def process_to_IDs_in_sparse_format(sp, sentences):\n",
+        "  # An utility method that processes sentences with the sentence piece processor\n",
+        "  # 'sp' and returns the results in tf.SparseTensor-similar format:\n",
+        "  # (values, indices, dense_shape)\n",
+        "  ids = [sp.EncodeAsIds(x) for x in sentences]\n",
+        "  max_len = max(len(x) for x in ids)\n",
+        "  dense_shape=(len(ids), max_len)\n",
+        "  values=[item for sublist in ids for item in sublist]\n",
+        "  indices=[[row,col] for row in range(len(ids)) for col in range(len(ids[row]))]\n",
+        "  return (values, indices, dense_shape)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PVpHEWrPAdxR"
+      },
+      "source": [
+        "### Test the module with a few examples"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pSkjuGYoCBfU"
+      },
+      "outputs": [],
+      "source": [
+        "# Compute a representation for each message, showing various lengths supported.\n",
+        "word = \"Elephant\"\n",
+        "sentence = \"I am a sentence for which I would like to get its embedding.\"\n",
+        "paragraph = (\n",
+        "    \"Universal Sentence Encoder embeddings also support short paragraphs. \"\n",
+        "    \"There is no hard limit on how long the paragraph is. Roughly, the longer \"\n",
+        "    \"the more 'diluted' the embedding will be.\")\n",
+        "messages = [word, sentence, paragraph]\n",
+        "\n",
+        "values, indices, dense_shape = process_to_IDs_in_sparse_format(sp, messages)\n",
+        "\n",
+        "# Reduce logging output.\n",
+        "logging.set_verbosity(logging.ERROR)\n",
+        "\n",
+        "with tf.Session() as session:\n",
+        "  session.run([tf.global_variables_initializer(), tf.tables_initializer()])\n",
+        "  message_embeddings = session.run(\n",
+        "      encodings,\n",
+        "      feed_dict={input_placeholder.values: values,\n",
+        "                input_placeholder.indices: indices,\n",
+        "                input_placeholder.dense_shape: dense_shape})\n",
+        "\n",
+        "  for i, message_embedding in enumerate(np.array(message_embeddings).tolist()):\n",
+        "    print(\"Message: {}\".format(messages[i]))\n",
+        "    print(\"Embedding size: {}\".format(len(message_embedding)))\n",
+        "    message_embedding_snippet = \", \".join(\n",
+        "        (str(x) for x in message_embedding[:3]))\n",
+        "    print(\"Embedding: [{}, ...]\\n\".format(message_embedding_snippet))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "46jrIgHyFDz9"
+      },
+      "source": [
+        "# Semantic Textual Similarity (STS) task example\n",
+        "\n",
+        "The embeddings produced by the Universal Sentence Encoder are approximately normalized. The semantic similarity of two sentences can be trivially computed as the inner product of the encodings."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OIQudHgWBGSk"
+      },
+      "outputs": [],
+      "source": [
+        "def plot_similarity(labels, features, rotation):\n",
+        "  corr = np.inner(features, features)\n",
+        "  sns.set(font_scale=1.2)\n",
+        "  g = sns.heatmap(\n",
+        "      corr,\n",
+        "      xticklabels=labels,\n",
+        "      yticklabels=labels,\n",
+        "      vmin=0,\n",
+        "      vmax=1,\n",
+        "      cmap=\"YlOrRd\")\n",
+        "  g.set_xticklabels(labels, rotation=rotation)\n",
+        "  g.set_title(\"Semantic Textual Similarity\")\n",
+        "\n",
+        "\n",
+        "def run_and_plot(session, input_placeholder, messages):\n",
+        "  values, indices, dense_shape = process_to_IDs_in_sparse_format(sp,messages)\n",
+        "\n",
+        "  message_embeddings = session.run(\n",
+        "      encodings,\n",
+        "      feed_dict={input_placeholder.values: values,\n",
+        "                input_placeholder.indices: indices,\n",
+        "                input_placeholder.dense_shape: dense_shape})\n",
+        "  \n",
+        "  plot_similarity(messages, message_embeddings, 90)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wlDqttNcE0Bx"
+      },
+      "source": [
+        "## Similarity visualized\n",
+        "Here we show the similarity in a heat map. The final graph is a 9x9 matrix where each entry `[i, j]` is colored based on the inner product of the encodings for sentence `i` and `j`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_GSCW5QIBKVe"
+      },
+      "outputs": [],
+      "source": [
+        "messages = [\n",
+        "    # Smartphones\n",
+        "    \"I like my phone\",\n",
+        "    \"My phone is not good.\",\n",
+        "    \"Your cellphone looks great.\",\n",
+        "\n",
+        "    # Weather\n",
+        "    \"Will it snow tomorrow?\",\n",
+        "    \"Recently a lot of hurricanes have hit the US\",\n",
+        "    \"Global warming is real\",\n",
+        "\n",
+        "    # Food and health\n",
+        "    \"An apple a day, keeps the doctors away\",\n",
+        "    \"Eating strawberries is healthy\",\n",
+        "    \"Is paleo better than keto?\",\n",
+        "\n",
+        "    # Asking about age\n",
+        "    \"How old are you?\",\n",
+        "    \"what is your age?\",\n",
+        "]\n",
+        "\n",
+        "\n",
+        "with tf.Session() as session:\n",
+        "  session.run(tf.global_variables_initializer())\n",
+        "  session.run(tf.tables_initializer())\n",
+        "  run_and_plot(session, input_placeholder, messages)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QkZ4sRBYBnL8"
+      },
+      "source": [
+        "## Evaluation: STS (Semantic Textual Similarity) Benchmark\n",
+        "\n",
+        "The [**STS Benchmark**](https://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) provides an intristic evaluation of the degree to which similarity scores computed using sentence embeddings align with human judgements. The benchmark requires systems to return similarity scores for a diverse selection of sentence pairs. [Pearson correlation](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient) is then used to evaluate the quality of the machine similarity scores against human judgements."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kNMVfSelBsHW"
+      },
+      "source": [
+        "### Download data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8zAWVzBMBptq"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas\n",
+        "import scipy\n",
+        "import math\n",
+        "\n",
+        "\n",
+        "def load_sts_dataset(filename):\n",
+        "  # Loads a subset of the STS dataset into a DataFrame. In particular both\n",
+        "  # sentences and their human rated similarity score.\n",
+        "  sent_pairs = []\n",
+        "  with tf.gfile.GFile(filename, \"r\") as f:\n",
+        "    for line in f:\n",
+        "      ts = line.strip().split(\"\\t\")\n",
+        "      # (sent_1, sent_2, similarity_score)\n",
+        "      sent_pairs.append((ts[5], ts[6], float(ts[4])))\n",
+        "  return pandas.DataFrame(sent_pairs, columns=[\"sent_1\", \"sent_2\", \"sim\"])\n",
+        "\n",
+        "\n",
+        "def download_and_load_sts_data():\n",
+        "  sts_dataset = tf.keras.utils.get_file(\n",
+        "      fname=\"Stsbenchmark.tar.gz\",\n",
+        "      origin=\"http://ixa2.si.ehu.es/stswiki/images/4/48/Stsbenchmark.tar.gz\",\n",
+        "      extract=True)\n",
+        "\n",
+        "  sts_dev = load_sts_dataset(\n",
+        "      os.path.join(os.path.dirname(sts_dataset), \"stsbenchmark\", \"sts-dev.csv\"))\n",
+        "  sts_test = load_sts_dataset(\n",
+        "      os.path.join(\n",
+        "          os.path.dirname(sts_dataset), \"stsbenchmark\", \"sts-test.csv\"))\n",
+        "\n",
+        "  return sts_dev, sts_test\n",
+        "\n",
+        "\n",
+        "sts_dev, sts_test = download_and_load_sts_data()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l8lEawD6B4Fr"
+      },
+      "source": [
+        "### Build evaluation graph"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "etiZUkP-B6bR"
+      },
+      "outputs": [],
+      "source": [
+        "sts_input1 = tf.sparse_placeholder(tf.int64, shape=(None, None))\n",
+        "sts_input2 = tf.sparse_placeholder(tf.int64, shape=(None, None))\n",
+        "\n",
+        "# For evaluation we use exactly normalized rather than\n",
+        "# approximately normalized.\n",
+        "sts_encode1 = tf.nn.l2_normalize(\n",
+        "    module(\n",
+        "        inputs=dict(values=sts_input1.values,\n",
+        "                    indices=sts_input1.indices,\n",
+        "                    dense_shape=sts_input1.dense_shape)),\n",
+        "    axis=1)\n",
+        "sts_encode2 = tf.nn.l2_normalize(\n",
+        "    module(\n",
+        "        inputs=dict(values=sts_input2.values,\n",
+        "                    indices=sts_input2.indices,\n",
+        "                    dense_shape=sts_input2.dense_shape)),\n",
+        "    axis=1)\n",
+        "\n",
+        "sim_scores = -tf.acos(tf.reduce_sum(tf.multiply(sts_encode1, sts_encode2), axis=1))\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e4Q34ssLB-rw"
+      },
+      "source": [
+        "### Evaluate sentence embeddings"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "-vRFEFPJPyeF"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Choose dataset for benchmark\n",
+        "dataset = sts_dev #@param [\"sts_dev\", \"sts_test\"] {type:\"raw\"}\n",
+        "\n",
+        "values1, indices1, dense_shape1 = process_to_IDs_in_sparse_format(sp, dataset['sent_1'].tolist())\n",
+        "values2, indices2, dense_shape2 = process_to_IDs_in_sparse_format(sp, dataset['sent_2'].tolist())\n",
+        "similarity_scores = dataset['sim'].tolist()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_QJ2DI85CBDh"
+      },
+      "outputs": [],
+      "source": [
+        "def run_sts_benchmark(session):\n",
+        "  \"\"\"Returns the similarity scores\"\"\"\n",
+        "  scores = session.run(\n",
+        "      sim_scores,\n",
+        "      feed_dict={\n",
+        "          sts_input1.values: values1,\n",
+        "          sts_input1.indices:  indices1,\n",
+        "          sts_input1.dense_shape:  dense_shape1,\n",
+        "          sts_input2.values:  values2,\n",
+        "          sts_input2.indices:  indices2,\n",
+        "          sts_input2.dense_shape:  dense_shape2,\n",
+        "      })\n",
+        "  return scores\n",
+        "\n",
+        "\n",
+        "with tf.Session() as session:\n",
+        "  session.run(tf.global_variables_initializer())\n",
+        "  session.run(tf.tables_initializer())\n",
+        "  scores = run_sts_benchmark(session)\n",
+        "\n",
+        "pearson_correlation = scipy.stats.pearsonr(scores, similarity_scores)\n",
+        "print('Pearson correlation coefficient = {0}\\np-value = {1}'.format(\n",
+        "    pearson_correlation[0], pearson_correlation[1]))"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "IJhWonqQN7u0"
+      ],
+      "name": "semantic_similarity_with_tf_hub_universal_encoder_lite.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/senteval_for_universal_sentence_encoder_cmlm.ipynb b/site/en/hub/tutorials/senteval_for_universal_sentence_encoder_cmlm.ipynb
new file mode 100644
index 00000000000..c33dce64c92
--- /dev/null
+++ b/site/en/hub/tutorials/senteval_for_universal_sentence_encoder_cmlm.ipynb
@@ -0,0 +1,248 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CGyzr0tfeUTQ"
+      },
+      "source": [
+        "**Copyright 2021 The TensorFlow Hub Authors.**\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zV1OQAGReaGQ"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L5bsDhkRfTpq"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "owWqOcw1e-RZ"
+      },
+      "source": [
+        "# Universal Sentence Encoder SentEval demo\n",
+        "This colab demostrates the [Universal Sentence Encoder CMLM model](https://tfhub.dev/google/universal-sentence-encoder-cmlm/en-base/1) using the [SentEval](https://github.com/facebookresearch/SentEval) toolkit, which is a library for measuring the quality of sentence embeddings. The SentEval toolkit includes a diverse set of downstream tasks that are able to evaluate the generalization power of an embedding model and to evaluate the linguistic properties encoded.\n",
+        "\n",
+        "Run the first two code blocks to setup the environment, in the third code block you can pick a SentEval task to evaluate the model. A GPU runtime is recommended to run this Colab.\n",
+        "\n",
+        "To learn more about the Universal Sentence Encoder CMLM model, see https://openreview.net/forum?id=WDVD4lUCTzU."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-CerULCLsjzV"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Install dependencies\n",
+        "!pip install --quiet \"tensorflow-text==2.11.*\"\n",
+        "!pip install --quiet torch==1.8.1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LjqkqD6aiZGU"
+      },
+      "source": [
+        "## Download SentEval and task data\n",
+        "This step download SentEval from github and execute the data script to download the task data. It may take up to 5 minutes to complete."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3UwhHQiKJmSc"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Install SentEval and download task data\n",
+        "!rm -rf ./SentEval\n",
+        "!git clone https://github.com/facebookresearch/SentEval.git\n",
+        "!cd $PWD/SentEval/data/downstream && bash get_transfer_data.bash > /dev/null 2>&1"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a2ohPn8vMe2"
+      },
+      "source": [
+        "#Execute a SentEval evaluation task\n",
+        "The following code block executes a SentEval task and output the results, choose one of the following tasks to evaluate the USE CMLM model:\n",
+        "\n",
+        "```\n",
+        "MR\tCR\tSUBJ\tMPQA\tSST\tTREC\tMRPC\tSICK-E\n",
+        "```\n",
+        "\n",
+        "Select a model, params and task to run. The rapid prototyping params can be used for reducing computation time for faster result.\n",
+        "\n",
+        "It typically takes 5-15 mins to complete a task with the **'rapid prototyping'** params and up to an hour with the **'slower, best performance'** params.\n",
+        "\n",
+        "```\n",
+        "params = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 5}\n",
+        "params['classifier'] = {'nhid': 0, 'optim': 'rmsprop', 'batch_size': 128,\n",
+        "                                 'tenacity': 3, 'epoch_size': 2}\n",
+        "```\n",
+        "\n",
+        "For better result, use the slower **'slower, best performance'** params, computation may take up to 1 hour:\n",
+        "\n",
+        "```\n",
+        "params = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 10}\n",
+        "params['classifier'] = {'nhid': 0, 'optim': 'adam', 'batch_size': 16,\n",
+        "                                 'tenacity': 5, 'epoch_size': 6}\n",
+        "```\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nenCcawjwowt"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
+        "\n",
+        "import sys\n",
+        "sys.path.append(f'{os.getcwd()}/SentEval')\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "# Prevent TF from claiming all GPU memory so there is some left for pytorch.\n",
+        "gpus = tf.config.list_physical_devices('GPU')\n",
+        "if gpus:\n",
+        "  # Memory growth needs to be the same across GPUs.\n",
+        "  for gpu in gpus:\n",
+        "    tf.config.experimental.set_memory_growth(gpu, True)\n",
+        "\n",
+        "import tensorflow_hub as hub\n",
+        "import tensorflow_text\n",
+        "import senteval\n",
+        "import time\n",
+        "\n",
+        "PATH_TO_DATA = f'{os.getcwd()}/SentEval/data'\n",
+        "MODEL = 'https://tfhub.dev/google/universal-sentence-encoder-cmlm/en-base/1' #@param ['https://tfhub.dev/google/universal-sentence-encoder-cmlm/en-base/1', 'https://tfhub.dev/google/universal-sentence-encoder-cmlm/en-large/1']\n",
+        "PARAMS = 'rapid prototyping' #@param ['slower, best performance', 'rapid prototyping']\n",
+        "TASK = 'CR' #@param ['CR','MR', 'MPQA', 'MRPC', 'SICKEntailment', 'SNLI', 'SST2', 'SUBJ', 'TREC']\n",
+        "\n",
+        "params_prototyping = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 5}\n",
+        "params_prototyping['classifier'] = {'nhid': 0, 'optim': 'rmsprop', 'batch_size': 128,\n",
+        "                                 'tenacity': 3, 'epoch_size': 2}\n",
+        "\n",
+        "params_best = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 10}\n",
+        "params_best['classifier'] = {'nhid': 0, 'optim': 'adam', 'batch_size': 16,\n",
+        "                                 'tenacity': 5, 'epoch_size': 6}\n",
+        "\n",
+        "params = params_best if PARAMS == 'slower, best performance' else params_prototyping\n",
+        "\n",
+        "preprocessor = hub.KerasLayer(\n",
+        "    \"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\")\n",
+        "encoder = hub.KerasLayer(\n",
+        "    \"https://tfhub.dev/google/universal-sentence-encoder-cmlm/en-base/1\")\n",
+        "\n",
+        "inputs = tf.keras.Input(shape=tf.shape(''), dtype=tf.string)\n",
+        "outputs = encoder(preprocessor(inputs))\n",
+        "\n",
+        "model = tf.keras.Model(inputs=inputs, outputs=outputs)\n",
+        "\n",
+        "def prepare(params, samples):\n",
+        "    return\n",
+        "\n",
+        "def batcher(_, batch):\n",
+        "    batch = [' '.join(sent) if sent else '.' for sent in batch]\n",
+        "    return model.predict(tf.constant(batch))[\"default\"]\n",
+        "\n",
+        "\n",
+        "se = senteval.engine.SE(params, batcher, prepare)\n",
+        "print(\"Evaluating task %s with %s parameters\" % (TASK, PARAMS))\n",
+        "start = time.time()\n",
+        "results = se.eval(TASK)\n",
+        "end = time.time()\n",
+        "print('Time took on task %s : %.1f. seconds' % (TASK, end - start))\n",
+        "print(results)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SNvsY6Hsvs0_"
+      },
+      "source": [
+        "#Learn More\n",
+        "\n",
+        "*   Find more text embedding models on [TensorFlow Hub](https://tfhub.dev)\n",
+        "*   See also the [Multilingual Universal Sentence Encoder CMLM model](https://tfhub.dev/google/universal-sentence-encoder-cmlm/multilingual-base-br/1)\n",
+        "*   Check out other [Universal Sentence Encoder models](https://tfhub.dev/google/collections/universal-sentence-encoder/1)\n",
+        "\n",
+        "## Reference\n",
+        "\n",
+        "*   Ziyi Yang, Yinfei Yang, Daniel Cer, Jax Law, Eric Darve. [Universal Sentence Representations Learning with Conditional Masked Language Model. November 2020](https://openreview.net/forum?id=WDVD4lUCTzU)\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "senteval_for_universal_sentence_encoder_cmlm.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/spice.ipynb b/site/en/hub/tutorials/spice.ipynb
new file mode 100644
index 00000000000..9ff6cd3bd62
--- /dev/null
+++ b/site/en/hub/tutorials/spice.ipynb
@@ -0,0 +1,937 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aXehiGc3Kr2I"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "-6LKjmi8Ktoh"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2020 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sPQKw4x4bL8w"
+      },
+      "source": [
+        "# Pitch Detection with SPICE\n",
+        "\n",
+        "This colab will show you how to use the SPICE model downloaded from TensorFlow Hub."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rfKwZlPnPwD1"
+      },
+      "outputs": [],
+      "source": [
+        "!sudo apt-get install -q -y timidity libsndfile1"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dYrIdOS8SW3b"
+      },
+      "outputs": [],
+      "source": [
+        "# All the imports to deal with sound data\n",
+        "!pip install pydub librosa music21"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "p09o78LGYdnz"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "import numpy as np\n",
+        "import matplotlib.pyplot as plt\n",
+        "import librosa\n",
+        "from librosa import display as librosadisplay\n",
+        "\n",
+        "import logging\n",
+        "import math\n",
+        "import statistics\n",
+        "import sys\n",
+        "\n",
+        "from IPython.display import Audio, Javascript\n",
+        "from scipy.io import wavfile\n",
+        "\n",
+        "from base64 import b64decode\n",
+        "\n",
+        "import music21\n",
+        "from pydub import AudioSegment\n",
+        "\n",
+        "logger = logging.getLogger()\n",
+        "logger.setLevel(logging.ERROR)\n",
+        "\n",
+        "print(\"tensorflow: %s\" % tf.__version__)\n",
+        "#print(\"librosa: %s\" % librosa.__version__)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wHxox8hXc3w1"
+      },
+      "source": [
+        "# The audio input file\n",
+        "Now the hardest part: Record your singing! :)\n",
+        "\n",
+        "We provide four methods to obtain an audio file:\n",
+        "\n",
+        "1.   Record audio directly in colab\n",
+        "2.   Upload from your computer\n",
+        "3.   Use a file saved on Google Drive\n",
+        "4.   Download the file from the web\n",
+        "\n",
+        "Choose one of the four methods below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "HaCAHOqiVu5B"
+      },
+      "outputs": [],
+      "source": [
+        "#@title [Run this] Definition of the JS code to record audio straight from the browser\n",
+        "\n",
+        "RECORD = \"\"\"\n",
+        "const sleep  = time => new Promise(resolve => setTimeout(resolve, time))\n",
+        "const b2text = blob => new Promise(resolve => {\n",
+        "  const reader = new FileReader()\n",
+        "  reader.onloadend = e => resolve(e.srcElement.result)\n",
+        "  reader.readAsDataURL(blob)\n",
+        "})\n",
+        "var record = time => new Promise(async resolve => {\n",
+        "  stream = await navigator.mediaDevices.getUserMedia({ audio: true })\n",
+        "  recorder = new MediaRecorder(stream)\n",
+        "  chunks = []\n",
+        "  recorder.ondataavailable = e => chunks.push(e.data)\n",
+        "  recorder.start()\n",
+        "  await sleep(time)\n",
+        "  recorder.onstop = async ()=>{\n",
+        "    blob = new Blob(chunks)\n",
+        "    text = await b2text(blob)\n",
+        "    resolve(text)\n",
+        "  }\n",
+        "  recorder.stop()\n",
+        "})\n",
+        "\"\"\"\n",
+        "\n",
+        "def record(sec=5):\n",
+        "  try:\n",
+        "    from google.colab import output\n",
+        "  except ImportError:\n",
+        "    print('No possible to import output from google.colab')\n",
+        "    return ''\n",
+        "  else:\n",
+        "    print('Recording')\n",
+        "    display(Javascript(RECORD))\n",
+        "    s = output.eval_js('record(%d)' % (sec*1000))\n",
+        "    fname = 'recorded_audio.wav'\n",
+        "    print('Saving to', fname)\n",
+        "    b = b64decode(s.split(',')[1])\n",
+        "    with open(fname, 'wb') as f:\n",
+        "      f.write(b)\n",
+        "    return fname"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "sBpWWkTzfUYR"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Select how to input your audio  { run: \"auto\" }\n",
+        "INPUT_SOURCE = 'https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav' #@param [\"https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav\", \"RECORD\", \"UPLOAD\", \"./drive/My Drive/YOUR_MUSIC_FILE.wav\"] {allow-input: true}\n",
+        "\n",
+        "print('You selected', INPUT_SOURCE)\n",
+        "\n",
+        "if INPUT_SOURCE == 'RECORD':\n",
+        "  uploaded_file_name = record(5)\n",
+        "elif INPUT_SOURCE == 'UPLOAD':\n",
+        "  try:\n",
+        "    from google.colab import files\n",
+        "  except ImportError:\n",
+        "    print(\"ImportError: files from google.colab seems to not be available\")\n",
+        "  else:\n",
+        "    uploaded = files.upload()\n",
+        "    for fn in uploaded.keys():\n",
+        "      print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
+        "          name=fn, length=len(uploaded[fn])))\n",
+        "    uploaded_file_name = next(iter(uploaded))\n",
+        "    print('Uploaded file: ' + uploaded_file_name)\n",
+        "elif INPUT_SOURCE.startswith('./drive/'):\n",
+        "  try:\n",
+        "    from google.colab import drive\n",
+        "  except ImportError:\n",
+        "    print(\"ImportError: files from google.colab seems to not be available\")\n",
+        "  else:\n",
+        "    drive.mount('/content/drive')\n",
+        "    # don't forget to change the name of the file you\n",
+        "    # will you here!\n",
+        "    gdrive_audio_file = 'YOUR_MUSIC_FILE.wav'\n",
+        "    uploaded_file_name = INPUT_SOURCE\n",
+        "elif INPUT_SOURCE.startswith('http'):\n",
+        "  !wget --no-check-certificate 'https://storage.googleapis.com/download.tensorflow.org/data/c-scale-metronome.wav' -O c-scale.wav\n",
+        "  uploaded_file_name = 'c-scale.wav'\n",
+        "else:\n",
+        "  print('Unrecognized input format!')\n",
+        "  print('Please select \"RECORD\", \"UPLOAD\", or specify a file hosted on Google Drive or a file from the web to download file to download')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "4S2BvIoDf9nf"
+      },
+      "source": [
+        "# Preparing the audio data\n",
+        "\n",
+        "Now we have the audio, let's convert it to the expected format and then listen to it!\n",
+        "\n",
+        "The SPICE model needs as input an audio file at a sampling rate of 16kHz and with only one channel (mono). \n",
+        "\n",
+        "To help you with this part, we created a function (`convert_audio_for_model`) to convert any wav file you have to the model's expected format:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bQ1362i-JoFI"
+      },
+      "outputs": [],
+      "source": [
+        "# Function that converts the user-created audio to the format that the model \n",
+        "# expects: bitrate 16kHz and only one channel (mono).\n",
+        "\n",
+        "EXPECTED_SAMPLE_RATE = 16000\n",
+        "\n",
+        "def convert_audio_for_model(user_file, output_file='converted_audio_file.wav'):\n",
+        "  audio = AudioSegment.from_file(user_file)\n",
+        "  audio = audio.set_frame_rate(EXPECTED_SAMPLE_RATE).set_channels(1)\n",
+        "  audio.export(output_file, format=\"wav\")\n",
+        "  return output_file"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oL9pftZ2nPm9"
+      },
+      "outputs": [],
+      "source": [
+        "# Converting to the expected format for the model\n",
+        "# in all the input 4 input method before, the uploaded file name is at\n",
+        "# the variable uploaded_file_name\n",
+        "converted_audio_file = convert_audio_for_model(uploaded_file_name)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TslkX2AOZN0p"
+      },
+      "outputs": [],
+      "source": [
+        "# Loading audio samples from the wav file:\n",
+        "sample_rate, audio_samples = wavfile.read(converted_audio_file, 'rb')\n",
+        "\n",
+        "# Show some basic information about the audio.\n",
+        "duration = len(audio_samples)/sample_rate\n",
+        "print(f'Sample rate: {sample_rate} Hz')\n",
+        "print(f'Total duration: {duration:.2f}s')\n",
+        "print(f'Size of the input: {len(audio_samples)}')\n",
+        "\n",
+        "# Let's listen to the wav file.\n",
+        "Audio(audio_samples, rate=sample_rate)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iBicZu5AgcpR"
+      },
+      "source": [
+        "First thing, let's take a look at the waveform of our singing."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aAa2M3CLZcWW"
+      },
+      "outputs": [],
+      "source": [
+        "# We can visualize the audio as a waveform.\n",
+        "_ = plt.plot(audio_samples)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J1eI0b8qgn08"
+      },
+      "source": [
+        "A more informative visualization is the [spectrogram](https://en.wikipedia.org/wiki/Spectrogram), which shows frequencies present over time.\n",
+        "\n",
+        "Here, we use a logarithmic frequency scale, to make the singing more clearly visible.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fGR4UZtpZvWI"
+      },
+      "outputs": [],
+      "source": [
+        "MAX_ABS_INT16 = 32768.0\n",
+        "\n",
+        "def plot_stft(x, sample_rate, show_black_and_white=False):\n",
+        "  x_stft = np.abs(librosa.stft(x, n_fft=2048))\n",
+        "  fig, ax = plt.subplots()\n",
+        "  fig.set_size_inches(20, 10)\n",
+        "  x_stft_db = librosa.amplitude_to_db(x_stft, ref=np.max)\n",
+        "  if(show_black_and_white):\n",
+        "    librosadisplay.specshow(data=x_stft_db, y_axis='log', \n",
+        "                             sr=sample_rate, cmap='gray_r')\n",
+        "  else:\n",
+        "    librosadisplay.specshow(data=x_stft_db, y_axis='log', sr=sample_rate)\n",
+        "\n",
+        "  plt.colorbar(format='%+2.0f dB')\n",
+        "\n",
+        "plot_stft(audio_samples / MAX_ABS_INT16 , sample_rate=EXPECTED_SAMPLE_RATE)\n",
+        "plt.show()\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MGCzo_cjjH-7"
+      },
+      "source": [
+        "We need one last conversion here. The audio samples are in int16 format. They need to be normalized to floats between -1 and 1."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dv4H4O1Xb8T8"
+      },
+      "outputs": [],
+      "source": [
+        "audio_samples = audio_samples / float(MAX_ABS_INT16)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yTdo_TwljVUV"
+      },
+      "source": [
+        "# Executing the Model\n",
+        "Now is the easy part, let's load the model with **TensorFlow Hub**, and feed the audio to it.\n",
+        "SPICE will give us two outputs: pitch and uncertainty\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xUptYSTAbc3I"
+      },
+      "source": [
+        "**TensorFlow Hub** is a library for the publication, discovery, and consumption of reusable parts of machine learning models. It makes easy to use machine learning to solve your challenges.\n",
+        "\n",
+        "To load the model you just need the Hub module and the URL pointing to the model:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ri0A0DSXY_Yd"
+      },
+      "outputs": [],
+      "source": [
+        "# Loading the SPICE model is easy:\n",
+        "model = hub.load(\"https://tfhub.dev/google/spice/2\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kQV5H6J4suMT"
+      },
+      "source": [
+        "**Note:** An interesting detail here is that all the model urls from Hub can be used for download and also to read the documentation, so if you point your browser to that link you can read documentation on how to use the model and learn more about how it was trained."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GUVICjIps9hI"
+      },
+      "source": [
+        "With the model loaded, data prepared, we need 3 lines to get the result: "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tP55fXBYcBhb"
+      },
+      "outputs": [],
+      "source": [
+        "# We now feed the audio to the SPICE tf.hub model to obtain pitch and uncertainty outputs as tensors.\n",
+        "model_output = model.signatures[\"serving_default\"](tf.constant(audio_samples, tf.float32))\n",
+        "\n",
+        "pitch_outputs = model_output[\"pitch\"]\n",
+        "uncertainty_outputs = model_output[\"uncertainty\"]\n",
+        "\n",
+        "# 'Uncertainty' basically means the inverse of confidence.\n",
+        "confidence_outputs = 1.0 - uncertainty_outputs\n",
+        "\n",
+        "fig, ax = plt.subplots()\n",
+        "fig.set_size_inches(20, 10)\n",
+        "plt.plot(pitch_outputs, label='pitch')\n",
+        "plt.plot(confidence_outputs, label='confidence')\n",
+        "plt.legend(loc=\"lower right\")\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "blJwFWR4kMul"
+      },
+      "source": [
+        "Let's make the results easier to understand by removing all pitch estimates with low confidence (confidence < 0.9) and plot the remaining ones.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d1MRmcm2cEkM"
+      },
+      "outputs": [],
+      "source": [
+        "confidence_outputs = list(confidence_outputs)\n",
+        "pitch_outputs = [ float(x) for x in pitch_outputs]\n",
+        "\n",
+        "indices = range(len (pitch_outputs))\n",
+        "confident_pitch_outputs = [ (i,p)  \n",
+        "  for i, p, c in zip(indices, pitch_outputs, confidence_outputs) if  c >= 0.9  ]\n",
+        "confident_pitch_outputs_x, confident_pitch_outputs_y = zip(*confident_pitch_outputs)\n",
+        " \n",
+        "fig, ax = plt.subplots()\n",
+        "fig.set_size_inches(20, 10)\n",
+        "ax.set_ylim([0, 1])\n",
+        "plt.scatter(confident_pitch_outputs_x, confident_pitch_outputs_y, )\n",
+        "plt.scatter(confident_pitch_outputs_x, confident_pitch_outputs_y, c=\"r\")\n",
+        "\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vNBZ7ZblkxOm"
+      },
+      "source": [
+        "The pitch values returned by SPICE are in the range from 0 to 1. Let's convert them to absolute pitch values in Hz."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "n-CnpKzmcQi9"
+      },
+      "outputs": [],
+      "source": [
+        "def output2hz(pitch_output):\n",
+        "  # Constants taken from https://tfhub.dev/google/spice/2\n",
+        "  PT_OFFSET = 25.58\n",
+        "  PT_SLOPE = 63.07\n",
+        "  FMIN = 10.0;\n",
+        "  BINS_PER_OCTAVE = 12.0;\n",
+        "  cqt_bin = pitch_output * PT_SLOPE + PT_OFFSET;\n",
+        "  return FMIN * 2.0 ** (1.0 * cqt_bin / BINS_PER_OCTAVE)\n",
+        "    \n",
+        "confident_pitch_values_hz = [ output2hz(p) for p in confident_pitch_outputs_y ]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "24yK0a6HjCSZ"
+      },
+      "source": [
+        "Now, let's see how good the prediction is: We will overlay the predicted pitches over the original spectrogram. To make the pitch predictions more visible, we changed the spectrogram to black and white."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "L1kaAcX9rrDo"
+      },
+      "outputs": [],
+      "source": [
+        "plot_stft(audio_samples / MAX_ABS_INT16 , \n",
+        "          sample_rate=EXPECTED_SAMPLE_RATE, show_black_and_white=True)\n",
+        "# Note: conveniently, since the plot is in log scale, the pitch outputs \n",
+        "# also get converted to the log scale automatically by matplotlib.\n",
+        "plt.scatter(confident_pitch_outputs_x, confident_pitch_values_hz, c=\"r\")\n",
+        "\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NskqpiHLxq6V"
+      },
+      "source": [
+        "# Converting to musical notes\n",
+        "\n",
+        "Now that we have the pitch values, let's convert them to notes!\n",
+        "This is part is challenging by itself. We have to take into account two things:\n",
+        "1. the rests (when there's no singing) \n",
+        "2. the size of each note (offsets) "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "KDOlm9PLTTjt"
+      },
+      "source": [
+        "### 1: Adding zeros to the output to indicate when there's no singing"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9uSQ3bJmTZmo"
+      },
+      "outputs": [],
+      "source": [
+        "pitch_outputs_and_rests = [\n",
+        "    output2hz(p) if c >= 0.9 else 0\n",
+        "    for i, p, c in zip(indices, pitch_outputs, confidence_outputs)\n",
+        "]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9fM0UwlsTt4w"
+      },
+      "source": [
+        "### 2: Adding note offsets\n",
+        "\n",
+        "When a person sings freely, the melody may have an offset to the absolute pitch values that notes can represent.\n",
+        "Hence, to convert predictions to notes, one needs to correct for this possible offset.\n",
+        "This is what the following code computes."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fsJu-P5ksdFW"
+      },
+      "outputs": [],
+      "source": [
+        "A4 = 440\n",
+        "C0 = A4 * pow(2, -4.75)\n",
+        "note_names = [\"C\", \"C#\", \"D\", \"D#\", \"E\", \"F\", \"F#\", \"G\", \"G#\", \"A\", \"A#\", \"B\"]\n",
+        "\n",
+        "def hz2offset(freq):\n",
+        "  # This measures the quantization error for a single note.\n",
+        "  if freq == 0:  # Rests always have zero error.\n",
+        "    return None\n",
+        "  # Quantized note.\n",
+        "  h = round(12 * math.log2(freq / C0))\n",
+        "  return 12 * math.log2(freq / C0) - h\n",
+        "\n",
+        "\n",
+        "# The ideal offset is the mean quantization error for all the notes\n",
+        "# (excluding rests):\n",
+        "offsets = [hz2offset(p) for p in pitch_outputs_and_rests if p != 0]\n",
+        "print(\"offsets: \", offsets)\n",
+        "\n",
+        "ideal_offset = statistics.mean(offsets)\n",
+        "print(\"ideal offset: \", ideal_offset)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "K17It_qT2DtE"
+      },
+      "source": [
+        "We can now use some heuristics to try and estimate the most likely sequence of notes that were sung.\n",
+        "The ideal offset computed above is one ingredient - but we also need to know the speed (how many predictions make, say, an eighth?), and the time offset to start quantizing.  To keep it simple, we'll just try different speeds and time offsets and measure the quantization error, using in the end the values that minimize this error."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eMULTI4L52ZHA"
+      },
+      "outputs": [],
+      "source": [
+        "def quantize_predictions(group, ideal_offset):\n",
+        "  # Group values are either 0, or a pitch in Hz.\n",
+        "  non_zero_values = [v for v in group if v != 0]\n",
+        "  zero_values_count = len(group) - len(non_zero_values)\n",
+        "\n",
+        "  # Create a rest if 80% is silent, otherwise create a note.\n",
+        "  if zero_values_count > 0.8 * len(group):\n",
+        "    # Interpret as a rest. Count each dropped note as an error, weighted a bit\n",
+        "    # worse than a badly sung note (which would 'cost' 0.5).\n",
+        "    return 0.51 * len(non_zero_values), \"Rest\"\n",
+        "  else:\n",
+        "    # Interpret as note, estimating as mean of non-rest predictions.\n",
+        "    h = round(\n",
+        "        statistics.mean([\n",
+        "            12 * math.log2(freq / C0) - ideal_offset for freq in non_zero_values\n",
+        "        ]))\n",
+        "    octave = h // 12\n",
+        "    n = h % 12\n",
+        "    note = note_names[n] + str(octave)\n",
+        "    # Quantization error is the total difference from the quantized note.\n",
+        "    error = sum([\n",
+        "        abs(12 * math.log2(freq / C0) - ideal_offset - h)\n",
+        "        for freq in non_zero_values\n",
+        "    ])\n",
+        "    return error, note\n",
+        "\n",
+        "\n",
+        "def get_quantization_and_error(pitch_outputs_and_rests, predictions_per_eighth,\n",
+        "                               prediction_start_offset, ideal_offset):\n",
+        "  # Apply the start offset - we can just add the offset as rests.\n",
+        "  pitch_outputs_and_rests = [0] * prediction_start_offset + \\\n",
+        "                            pitch_outputs_and_rests\n",
+        "  # Collect the predictions for each note (or rest).\n",
+        "  groups = [\n",
+        "      pitch_outputs_and_rests[i:i + predictions_per_eighth]\n",
+        "      for i in range(0, len(pitch_outputs_and_rests), predictions_per_eighth)\n",
+        "  ]\n",
+        "\n",
+        "  quantization_error = 0\n",
+        "\n",
+        "  notes_and_rests = []\n",
+        "  for group in groups:\n",
+        "    error, note_or_rest = quantize_predictions(group, ideal_offset)\n",
+        "    quantization_error += error\n",
+        "    notes_and_rests.append(note_or_rest)\n",
+        "\n",
+        "  return quantization_error, notes_and_rests\n",
+        "\n",
+        "\n",
+        "best_error = float(\"inf\")\n",
+        "best_notes_and_rests = None\n",
+        "best_predictions_per_note = None\n",
+        "\n",
+        "for predictions_per_note in range(20, 65, 1):\n",
+        "  for prediction_start_offset in range(predictions_per_note):\n",
+        "\n",
+        "    error, notes_and_rests = get_quantization_and_error(\n",
+        "        pitch_outputs_and_rests, predictions_per_note,\n",
+        "        prediction_start_offset, ideal_offset)\n",
+        "\n",
+        "    if error < best_error:      \n",
+        "      best_error = error\n",
+        "      best_notes_and_rests = notes_and_rests\n",
+        "      best_predictions_per_note = predictions_per_note\n",
+        "\n",
+        "# At this point, best_notes_and_rests contains the best quantization.\n",
+        "# Since we don't need to have rests at the beginning, let's remove these:\n",
+        "while best_notes_and_rests[0] == 'Rest':\n",
+        "  best_notes_and_rests = best_notes_and_rests[1:]\n",
+        "# Also remove silence at the end.\n",
+        "while best_notes_and_rests[-1] == 'Rest':\n",
+        "  best_notes_and_rests = best_notes_and_rests[:-1]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vMZbWA3aVqee"
+      },
+      "source": [
+        "Now let's write the quantized notes as sheet music score!\n",
+        "\n",
+        "To do it we will use two libraries: [music21](http://web.mit.edu/music21/) and [Open Sheet Music Display](https://github.com/opensheetmusicdisplay/opensheetmusicdisplay)\n",
+        "\n",
+        "**Note:** for simplicity, we assume here that all notes have the same duration (a half note)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yVrk_IOIzpQR"
+      },
+      "outputs": [],
+      "source": [
+        "# Creating the sheet music score.\n",
+        "sc = music21.stream.Score()\n",
+        "# Adjust the speed to match the actual singing.\n",
+        "bpm = 60 * 60 / best_predictions_per_note\n",
+        "print ('bpm: ', bpm)\n",
+        "a = music21.tempo.MetronomeMark(number=bpm)\n",
+        "sc.insert(0,a)\n",
+        "\n",
+        "for snote in best_notes_and_rests:   \n",
+        "    d = 'half'\n",
+        "    if snote == 'Rest':      \n",
+        "      sc.append(music21.note.Rest(type=d))\n",
+        "    else:\n",
+        "      sc.append(music21.note.Note(snote, type=d))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "CEleCWHtG2s4"
+      },
+      "outputs": [],
+      "source": [
+        "#@title [Run this] Helper function to use Open Sheet Music Display (JS code) to show a music score\n",
+        "\n",
+        "from IPython.core.display import display, HTML, Javascript\n",
+        "import json, random\n",
+        "\n",
+        "def showScore(score):\n",
+        "    xml = open(score.write('musicxml')).read()\n",
+        "    showMusicXML(xml)\n",
+        "    \n",
+        "def showMusicXML(xml):\n",
+        "    DIV_ID = \"OSMD_div\"\n",
+        "    display(HTML('
    loading OpenSheetMusicDisplay
    '))\n",
+        "    script = \"\"\"\n",
+        "    var div_id = %%DIV_ID%%;\n",
+        "    function loadOSMD() { \n",
+        "        return new Promise(function(resolve, reject){\n",
+        "            if (window.opensheetmusicdisplay) {\n",
+        "                return resolve(window.opensheetmusicdisplay)\n",
+        "            }\n",
+        "            // OSMD script has a 'define' call which conflicts with requirejs\n",
+        "            var _define = window.define // save the define object \n",
+        "            window.define = undefined // now the loaded script will ignore requirejs\n",
+        "            var s = document.createElement( 'script' );\n",
+        "            s.setAttribute( 'src', \"https://cdn.jsdelivr.net/npm/opensheetmusicdisplay@0.7.6/build/opensheetmusicdisplay.min.js\" );\n",
+        "            //s.setAttribute( 'src', \"/custom/opensheetmusicdisplay.js\" );\n",
+        "            s.onload=function(){\n",
+        "                window.define = _define\n",
+        "                resolve(opensheetmusicdisplay);\n",
+        "            };\n",
+        "            document.body.appendChild( s ); // browser will try to load the new script tag\n",
+        "        }) \n",
+        "    }\n",
+        "    loadOSMD().then((OSMD)=>{\n",
+        "        window.openSheetMusicDisplay = new OSMD.OpenSheetMusicDisplay(div_id, {\n",
+        "          drawingParameters: \"compacttight\"\n",
+        "        });\n",
+        "        openSheetMusicDisplay\n",
+        "            .load(%%data%%)\n",
+        "            .then(\n",
+        "              function() {\n",
+        "                openSheetMusicDisplay.render();\n",
+        "              }\n",
+        "            );\n",
+        "    })\n",
+        "    \"\"\".replace('%%DIV_ID%%',DIV_ID).replace('%%data%%',json.dumps(xml))\n",
+        "    display(Javascript(script))\n",
+        "    return"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WTu4phq4WeAI"
+      },
+      "outputs": [],
+      "source": [
+        "# rendering the music score\n",
+        "showScore(sc)\n",
+        "print(best_notes_and_rests)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fGPXm6Z83U2g"
+      },
+      "source": [
+        "Let's convert the music notes to a MIDI file and listen to it.\n",
+        "\n",
+        "To create this file, we can use the stream we created before."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "klYoWjgmPaod"
+      },
+      "outputs": [],
+      "source": [
+        "# Saving the recognized musical notes as a MIDI file\n",
+        "converted_audio_file_as_midi = converted_audio_file[:-4] + '.mid'\n",
+        "fp = sc.write('midi', fp=converted_audio_file_as_midi)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tz7Mj3Qx1lpR"
+      },
+      "outputs": [],
+      "source": [
+        "wav_from_created_midi = converted_audio_file_as_midi.replace(' ', '_') + \"_midioutput.wav\"\n",
+        "print(wav_from_created_midi)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ahss5EOiWDDp"
+      },
+      "source": [
+        "To listen to it on colab, we need to convert it back to wav. An easy way of doing that is using Timidity."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XmeJ-UITV2nq"
+      },
+      "outputs": [],
+      "source": [
+        "!timidity $converted_audio_file_as_midi -Ow -o $wav_from_created_midi"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bnvwmyNj7kCC"
+      },
+      "source": [
+        "And finally, listen the audio, created from notes, created via MIDI from the predicted pitches, inferred by the model!\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qNLBB0zJV6vN"
+      },
+      "outputs": [],
+      "source": [
+        "Audio(wav_from_created_midi)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "spice.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/text_classification_with_tf_hub_on_kaggle.ipynb b/site/en/hub/tutorials/text_classification_with_tf_hub_on_kaggle.ipynb
new file mode 100644
index 00000000000..e2985bda51e
--- /dev/null
+++ b/site/en/hub/tutorials/text_classification_with_tf_hub_on_kaggle.ipynb
@@ -0,0 +1,477 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N6ZDpd9XzFeN"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "KUu4vOt5zI9d"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ok9PfyoQ2rH_"
+      },
+      "source": [
+        "# How to solve a problem on Kaggle with TF-Hub\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "556YQZLUO4Ih"
+      },
+      "source": [
+        "TF-Hub is a platform to share machine learning expertise packaged in reusable resources, notably pre-trained **modules**. In this tutorial, we will use a TF-Hub text embedding module to train a simple sentiment classifier with a reasonable baseline accuracy. We will then submit the predictions to Kaggle.\n",
+        "\n",
+        "For more detailed tutorial on text classification with TF-Hub and further steps for improving the accuracy, take a look at [Text classification with TF-Hub](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/hub/tutorials/text_classification_with_tf_hub.ipynb)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q4DN769E2O_R"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9KyLct9rq0lo"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q kaggle"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "v7hy0bhngTUp"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "import seaborn as sns\n",
+        "import zipfile\n",
+        "\n",
+        "from sklearn import model_selection"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JvgBdeMsuu_3"
+      },
+      "source": [
+        "Since this tutorial will be using a dataset from Kaggle, it requires [creating an API Token](https://github.com/Kaggle/kaggle-api) for your Kaggle account, and uploading it to the Colab environment."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nI7C-Zc4urOH"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import pathlib\n",
+        "\n",
+        "# Upload the API token.\n",
+        "def get_kaggle():\n",
+        "  try:\n",
+        "    import kaggle\n",
+        "    return kaggle\n",
+        "  except OSError:\n",
+        "    pass\n",
+        "\n",
+        "  token_file = pathlib.Path(\"~/.kaggle/kaggle.json\").expanduser()\n",
+        "  token_file.parent.mkdir(exist_ok=True, parents=True)\n",
+        "\n",
+        "  try:\n",
+        "    from google.colab import files\n",
+        "  except ImportError:\n",
+        "    raise ValueError(\"Could not find kaggle token.\")\n",
+        "\n",
+        "  uploaded = files.upload()\n",
+        "  token_content = uploaded.get('kaggle.json', None)\n",
+        "  if token_content:\n",
+        "    token_file.write_bytes(token_content)\n",
+        "    token_file.chmod(0o600)\n",
+        "  else:\n",
+        "    raise ValueError('Need a file named \"kaggle.json\"')\n",
+        "  \n",
+        "  import kaggle\n",
+        "  return kaggle\n",
+        "\n",
+        "\n",
+        "kaggle = get_kaggle()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6OPyVxHuiTEE"
+      },
+      "source": [
+        "# Getting started\n",
+        "\n",
+        "## Data\n",
+        "We will try to solve the [Sentiment Analysis on Movie Reviews](https://www.kaggle.com/c/sentiment-analysis-on-movie-reviews/data) task from Kaggle. The dataset consists of syntactic subphrases of the Rotten Tomatoes movie reviews. The task is to label the phrases as **negative** or **positive** on the scale from 1 to 5.\n",
+        "\n",
+        "You must [accept the competition rules](https://www.kaggle.com/c/sentiment-analysis-on-movie-reviews/data) before you can use the API to download the data.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "rKzc-fOGV72G"
+      },
+      "outputs": [],
+      "source": [
+        "SENTIMENT_LABELS = [\n",
+        "    \"negative\", \"somewhat negative\", \"neutral\", \"somewhat positive\", \"positive\"\n",
+        "]\n",
+        "\n",
+        "# Add a column with readable values representing the sentiment.\n",
+        "def add_readable_labels_column(df, sentiment_value_column):\n",
+        "  df[\"SentimentLabel\"] = df[sentiment_value_column].replace(\n",
+        "      range(5), SENTIMENT_LABELS)\n",
+        "    \n",
+        "# Download data from Kaggle and create a DataFrame.\n",
+        "def load_data_from_zip(path):\n",
+        "  with zipfile.ZipFile(path, \"r\") as zip_ref:\n",
+        "    name = zip_ref.namelist()[0]\n",
+        "    with zip_ref.open(name) as zf:\n",
+        "      return pd.read_csv(zf, sep=\"\\t\", index_col=0)\n",
+        "\n",
+        "\n",
+        "# The data does not come with a validation set so we'll create one from the\n",
+        "# training set.\n",
+        "def get_data(competition, train_file, test_file, validation_set_ratio=0.1):\n",
+        "  data_path = pathlib.Path(\"data\")\n",
+        "  kaggle.api.competition_download_files(competition, data_path)\n",
+        "  competition_path = (data_path/competition)\n",
+        "  competition_path.mkdir(exist_ok=True, parents=True)\n",
+        "  competition_zip_path = competition_path.with_suffix(\".zip\")\n",
+        "\n",
+        "  with zipfile.ZipFile(competition_zip_path, \"r\") as zip_ref:\n",
+        "    zip_ref.extractall(competition_path)\n",
+        "  \n",
+        "  train_df = load_data_from_zip(competition_path/train_file)\n",
+        "  test_df = load_data_from_zip(competition_path/test_file)\n",
+        "\n",
+        "  # Add a human readable label.\n",
+        "  add_readable_labels_column(train_df, \"Sentiment\")\n",
+        "\n",
+        "  # We split by sentence ids, because we don't want to have phrases belonging\n",
+        "  # to the same sentence in both training and validation set.\n",
+        "  train_indices, validation_indices = model_selection.train_test_split(\n",
+        "      np.unique(train_df[\"SentenceId\"]),\n",
+        "      test_size=validation_set_ratio,\n",
+        "      random_state=0)\n",
+        "\n",
+        "  validation_df = train_df[train_df[\"SentenceId\"].isin(validation_indices)]\n",
+        "  train_df = train_df[train_df[\"SentenceId\"].isin(train_indices)]\n",
+        "  print(\"Split the training data into %d training and %d validation examples.\" %\n",
+        "        (len(train_df), len(validation_df)))\n",
+        "\n",
+        "  return train_df, validation_df, test_df\n",
+        "\n",
+        "\n",
+        "train_df, validation_df, test_df = get_data(\n",
+        "    \"sentiment-analysis-on-movie-reviews\",\n",
+        "    \"train.tsv.zip\", \"test.tsv.zip\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DFq_EyS1BEyK"
+      },
+      "source": [
+        "Note: In this competition the task is not to rate entire reviews, but individual phrases from within the reviews. This is a much harder task."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "42hgsiWNq5y9"
+      },
+      "outputs": [],
+      "source": [
+        "train_df.head(20)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YPuHgx3BWBOg"
+      },
+      "source": [
+        "## Training an Model\n",
+        "\n",
+        "*Note: We could model this task also as a regression, see [Text classification with TF-Hub](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/hub/tutorials/text_classification_with_tf_hub.ipynb).*"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "23U30yEkVq4w"
+      },
+      "outputs": [],
+      "source": [
+        "class MyModel(tf.keras.Model):\n",
+        "  def __init__(self, hub_url):\n",
+        "    super().__init__()\n",
+        "    self.hub_url = hub_url\n",
+        "    self.embed = hub.load(self.hub_url).signatures['default']\n",
+        "    self.sequential = tf.keras.Sequential([\n",
+        "      tf.keras.layers.Dense(500),\n",
+        "      tf.keras.layers.Dense(100),\n",
+        "      tf.keras.layers.Dense(5),\n",
+        "    ])\n",
+        "\n",
+        "  def call(self, inputs):\n",
+        "    phrases = inputs['Phrase'][:,0]\n",
+        "    embedding = 5*self.embed(phrases)['default']\n",
+        "    return self.sequential(embedding)\n",
+        "\n",
+        "  def get_config(self):\n",
+        "    return {\"hub_url\":self.hub_url}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JE--GDMM2tSp"
+      },
+      "outputs": [],
+      "source": [
+        "model = MyModel(\"https://tfhub.dev/google/nnlm-en-dim128/1\")\n",
+        "model.compile(\n",
+        "    loss = tf.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
+        "    optimizer=tf.optimizers.Adam(), \n",
+        "    metrics = [tf.keras.metrics.SparseCategoricalAccuracy(name=\"accuracy\")])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SRr-lvhstiNw"
+      },
+      "outputs": [],
+      "source": [
+        "history = model.fit(x=dict(train_df), y=train_df['Sentiment'],\n",
+        "          validation_data=(dict(validation_df), validation_df['Sentiment']),\n",
+        "          epochs = 25)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "s8j7YTRSe7Pj"
+      },
+      "source": [
+        "# Prediction\n",
+        "\n",
+        "Run predictions for the validation set and training set."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iGqVNSl87bgN"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(history.history['accuracy'])\n",
+        "plt.plot(history.history['val_accuracy'])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zbLg5LzGwAfC"
+      },
+      "outputs": [],
+      "source": [
+        "train_eval_result = model.evaluate(dict(train_df), train_df['Sentiment'])\n",
+        "validation_eval_result = model.evaluate(dict(validation_df), validation_df['Sentiment'])\n",
+        "\n",
+        "print(f\"Training set accuracy: {train_eval_result[1]}\")\n",
+        "print(f\"Validation set accuracy: {validation_eval_result[1]}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DR2IsTF5vuAX"
+      },
+      "source": [
+        "## Confusion matrix\n",
+        "\n",
+        "Another very interesting statistic, especially for multiclass problems, is the [confusion matrix](https://en.wikipedia.org/wiki/Confusion_matrix). The confusion matrix allows visualization of the proportion of correctly and incorrectly labelled examples. We can easily see how much our classifier is biased and whether the distribution of labels makes sense. Ideally the largest fraction of predictions should be distributed along the diagonal."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yKUnJFYY8bO_"
+      },
+      "outputs": [],
+      "source": [
+        "predictions = model.predict(dict(validation_df))\n",
+        "predictions = tf.argmax(predictions, axis=-1)\n",
+        "predictions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fjAs8W_Z9BvP"
+      },
+      "outputs": [],
+      "source": [
+        "cm = tf.math.confusion_matrix(validation_df['Sentiment'], predictions)\n",
+        "cm = cm/cm.numpy().sum(axis=1)[:, tf.newaxis]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nT71CtArpsKz"
+      },
+      "outputs": [],
+      "source": [
+        "sns.heatmap(\n",
+        "    cm, annot=True,\n",
+        "    xticklabels=SENTIMENT_LABELS,\n",
+        "    yticklabels=SENTIMENT_LABELS)\n",
+        "plt.xlabel(\"Predicted\")\n",
+        "plt.ylabel(\"True\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Pic7o2m04weY"
+      },
+      "source": [
+        "We can easily submit the predictions back to Kaggle by pasting the following code to a code cell and executing it:\n",
+        "\n",
+        "``` python\n",
+        "test_predictions = model.predict(dict(test_df))\n",
+        "test_predictions = np.argmax(test_predictions, axis=-1)\n",
+        "\n",
+        "result_df = test_df.copy()\n",
+        "\n",
+        "result_df[\"Predictions\"] = test_predictions\n",
+        "\n",
+        "result_df.to_csv(\n",
+        "    \"predictions.csv\",\n",
+        "    columns=[\"Predictions\"],\n",
+        "    header=[\"Sentiment\"])\n",
+        "kaggle.api.competition_submit(\"predictions.csv\", \"Submitted from Colab\",\n",
+        "                              \"sentiment-analysis-on-movie-reviews\")\n",
+        "```\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "50BLu-JX_dlm"
+      },
+      "source": [
+        "After submitting, [check the leaderboard](https://www.kaggle.com/c/sentiment-analysis-on-movie-reviews/leaderboard) to see how you did."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "text_classification_with_tf_hub_on_kaggle.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/text_cookbook.md b/site/en/hub/tutorials/text_cookbook.md
new file mode 100644
index 00000000000..dee9c1cf466
--- /dev/null
+++ b/site/en/hub/tutorials/text_cookbook.md
@@ -0,0 +1,101 @@
+# Text Cookbook
+
+This page lists a set of known guides and tools solving problems in the text
+domain with TensorFlow Hub. It is a starting place for anybody who wants to
+solve typical ML problems using pre-trained ML components rather than starting
+from scratch.
+
+## Classification
+
+When we want to predict a class for a given example, for example **sentiment**,
+**toxicity**, **article category**, or any other characteristic.
+
+![Text Classification Graphic](https://www.gstatic.com/aihub/tfhub/universal-sentence-encoder/example-classification.png)
+
+The tutorials below are solving the same task from different perspectives and
+using different tools.
+
+### Keras
+
+[Text classification with Keras](https://www.tensorflow.org/tutorials/keras/text_classification_with_hub) -
+example for building an IMDB sentiment classifier with Keras and TensorFlow
+Datasets.
+
+### Estimator
+
+[Text classification](https://github.com/tensorflow/docs/blob/master/g3doc/en/hub/tutorials/text_classification_with_tf_hub.ipynb) -
+example for building an IMDB sentiment classifier with Estimator. Contains
+multiple tips for improvement and a module comparison section.
+
+### BERT
+[Predicting Movie Review Sentiment with BERT on TF Hub](https://github.com/google-research/bert/blob/master/predicting_movie_reviews_with_bert_on_tf_hub.ipynb) -
+shows how to use a BERT module for classification. Includes use of `bert`
+library for tokenization and preprocessing.
+
+### Kaggle
+
+[IMDB classification on Kaggle](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/text_classification_with_tf_hub_on_kaggle.ipynb) -
+shows how to easily interact with a Kaggle competition from a Colab, including
+downloading the data and submitting the results.
+
+                                                                                                                                                                                         | Estimator                                                                                         | Keras                                                                                             | TF2                                                                                               | TF Datasets                                                                                       | BERT                                                                                              | Kaggle APIs
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -----------
+[Text classification](https://www.tensorflow.org/hub/tutorials/text_classification_with_tf_hub)                                                                                          | ![done](https://www.gstatic.com/images/icons/material/system_gm/1x/bigtop_done_googblue_18dp.png) |                                                                                                   |                                                                                                   |                                                                                                   |                                                                                                   |
+[Text classification with Keras](https://www.tensorflow.org/tutorials/keras/text_classification_with_hub)                                                                                |                                                                                                   | ![done](https://www.gstatic.com/images/icons/material/system_gm/1x/bigtop_done_googblue_18dp.png) | ![done](https://www.gstatic.com/images/icons/material/system_gm/1x/bigtop_done_googblue_18dp.png) | ![done](https://www.gstatic.com/images/icons/material/system_gm/1x/bigtop_done_googblue_18dp.png) |                                                                                                   |
+[Predicting Movie Review Sentiment with BERT on TF Hub](https://github.com/google-research/bert/blob/master/predicting_movie_reviews_with_bert_on_tf_hub.ipynb)                          | ![done](https://www.gstatic.com/images/icons/material/system_gm/1x/bigtop_done_googblue_18dp.png) |                                                                                                   |                                                                                                   |                                                                                                   | ![done](https://www.gstatic.com/images/icons/material/system_gm/1x/bigtop_done_googblue_18dp.png) |
+[IMDB classification on Kaggle](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/text_classification_with_tf_hub_on_kaggle.ipynb) | ![done](https://www.gstatic.com/images/icons/material/system_gm/1x/bigtop_done_googblue_18dp.png) |                                                                                                   |                                                                                                   |                                                                                                   |                                                                                                   | ![done](https://www.gstatic.com/images/icons/material/system_gm/1x/bigtop_done_googblue_18dp.png)
+
+### Bangla task with FastText embeddings
+TensorFlow Hub does not currently offer a module in every language. The
+following tutorial shows how to leverage TensorFlow Hub for fast experimentation
+and modular ML development.
+
+[Bangla Article Classifier](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/bangla_article_classifier.ipynb) -
+demonstrates how to create a reusable TensorFlow Hub text embedding, and use it
+to train a Keras classifier for
+[BARD Bangla Article dataset](https://github.com/tanvirfahim15/BARD-Bangla-Article-Classifier).
+
+## Semantic similarity
+
+When we want to find out which sentences correlate with each other in zero-shot
+setup (no training examples).
+
+![Semantic Similarity Graphic](https://www.gstatic.com/aihub/tfhub/universal-sentence-encoder/example-similarity.png)
+
+### Basic
+
+[Semantic similarity](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder.ipynb) -
+shows how to use the sentence encoder module to compute sentence similarity.
+
+### Cross-lingual
+
+[Cross-lingual semantic similarity](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/cross_lingual_similarity_with_tf_hub_multilingual_universal_encoder.ipynb) -
+shows how to use one of the cross-lingual sentence encoders to compute sentence
+similarity across languages.
+
+### Semantic retrieval
+
+[Semantic retrieval](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/retrieval_with_tf_hub_universal_encoder_qa.ipynb) -
+shows how to use Q/A sentence encoder to index a collection of documents for
+retrieval based on semantic similarity.
+
+### SentencePiece input
+
+[Semantic similarity with universal encoder lite](https://github.com/tensorflow/docs/blob/master/site/en/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder_lite.ipynb) -
+shows how to use sentence encoder modules that accept
+[SentencePiece](https://github.com/google/sentencepiece) ids on input instead of
+text.
+
+## Module creation
+Instead of using only modules on [tfhub.dev](https://tfhub.dev), there are ways
+to create own modules. This can be a useful tool for better ML codebase
+modularity and for further sharing.
+
+### Wrapping existing pre-trained embeddings
+[Text embedding module exporter](https://github.com/tensorflow/hub/blob/master/examples/text_embeddings/export.py) -
+a tool to wrap an existing pre-trained embedding into a module. Shows how to
+include text pre-processing ops into the module. This allows to create a
+sentence embedding module from token embeddings.
+
+[Text embedding module exporter v2](https://github.com/tensorflow/hub/blob/master/examples/text_embeddings_v2/export_v2.py) -
+same as above, but compatible with TensorFlow 2 and eager execution.
diff --git a/site/en/hub/tutorials/text_to_video_retrieval_with_s3d_milnce.ipynb b/site/en/hub/tutorials/text_to_video_retrieval_with_s3d_milnce.ipynb
new file mode 100644
index 00000000000..52fb3c0e4ab
--- /dev/null
+++ b/site/en/hub/tutorials/text_to_video_retrieval_with_s3d_milnce.ipynb
@@ -0,0 +1,275 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8JSGdaDHc_f4"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z2_BHI6XdJ30"
+      },
+      "source": [
+        "# Text-to-Video retrieval with S3D MIL-NCE"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Rm0K9ZTgfISB"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bC_xJPpQd-LO"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q opencv-python\n",
+        "\n",
+        "import os\n",
+        "\n",
+        "import tensorflow.compat.v2 as tf\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "import numpy as np\n",
+        "import cv2\n",
+        "from IPython import display\n",
+        "import math"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZxwaK-jf7qkW"
+      },
+      "source": [
+        "## Import TF-Hub model\n",
+        "\n",
+        "This tutorial demonstrates how to use the [S3D MIL-NCE model](https://tfhub.dev/deepmind/mil-nce/s3d/1) from TensorFlow Hub to do **text-to-video retrieval** to find the most similar videos for a given text query.\n",
+        "\n",
+        "The model has 2 signatures, one for generating *video embeddings* and one for generating *text embeddings*. We will use these embedding to find the nearest neighbors in the embedding space."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nwv4ZQ4qmak5"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the model once from TF-Hub.\n",
+        "hub_handle = 'https://tfhub.dev/deepmind/mil-nce/s3d/1'\n",
+        "hub_model = hub.load(hub_handle)\n",
+        "\n",
+        "def generate_embeddings(model, input_frames, input_words):\n",
+        "  \"\"\"Generate embeddings from the model from video frames and input words.\"\"\"\n",
+        "  # Input_frames must be normalized in [0, 1] and of the shape Batch x T x H x W x 3\n",
+        "  vision_output = model.signatures['video'](tf.constant(tf.cast(input_frames, dtype=tf.float32)))\n",
+        "  text_output = model.signatures['text'](tf.constant(input_words))\n",
+        "  return vision_output['video_embedding'], text_output['text_embedding']"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EOZzu9ddekEj"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Define video loading and visualization functions  { display-mode: \"form\" }\n",
+        "\n",
+        "# Utilities to open video files using CV2\n",
+        "def crop_center_square(frame):\n",
+        "  y, x = frame.shape[0:2]\n",
+        "  min_dim = min(y, x)\n",
+        "  start_x = (x // 2) - (min_dim // 2)\n",
+        "  start_y = (y // 2) - (min_dim // 2)\n",
+        "  return frame[start_y:start_y+min_dim,start_x:start_x+min_dim]\n",
+        "\n",
+        "\n",
+        "def load_video(video_url, max_frames=32, resize=(224, 224)):\n",
+        "  path = tf.keras.utils.get_file(os.path.basename(video_url)[-128:], video_url)\n",
+        "  cap = cv2.VideoCapture(path)\n",
+        "  frames = []\n",
+        "  try:\n",
+        "    while True:\n",
+        "      ret, frame = cap.read()\n",
+        "      if not ret:\n",
+        "        break\n",
+        "      frame = crop_center_square(frame)\n",
+        "      frame = cv2.resize(frame, resize)\n",
+        "      frame = frame[:, :, [2, 1, 0]]\n",
+        "      frames.append(frame)\n",
+        "\n",
+        "      if len(frames) == max_frames:\n",
+        "        break\n",
+        "  finally:\n",
+        "    cap.release()\n",
+        "  frames = np.array(frames)\n",
+        "  if len(frames) < max_frames:\n",
+        "    n_repeat = int(math.ceil(max_frames / float(len(frames))))\n",
+        "    frames = frames.repeat(n_repeat, axis=0)\n",
+        "  frames = frames[:max_frames]\n",
+        "  return frames / 255.0\n",
+        "\n",
+        "def display_video(urls):\n",
+        "    html = ''\n",
+        "    html += ''\n",
+        "    for url in urls:\n",
+        "        html += ''\n",
+        "    html += '
    Video 1Video 2Video 3
    '\n",
+        "        html += ''.format(url)\n",
+        "        html += '
    '\n",
+        "    return display.HTML(html)\n",
+        "\n",
+        "def display_query_and_results_video(query, urls, scores):\n",
+        "  \"\"\"Display a text query and the top result videos and scores.\"\"\"\n",
+        "  sorted_ix = np.argsort(-scores)\n",
+        "  html = ''\n",
+        "  html += '
    Input query: {} 
    '.format(query)\n",
+        "  html += 'Results: 
    '\n",
+        "  html += ''\n",
+        "  html += ''.format(scores[sorted_ix[0]])\n",
+        "  html += ''.format(scores[sorted_ix[1]])\n",
+        "  html += ''.format(scores[sorted_ix[2]])\n",
+        "  for i, idx in enumerate(sorted_ix):\n",
+        "    url = urls[sorted_ix[i]];\n",
+        "    html += ''\n",
+        "  html += '
    Rank #1, Score:{:.2f}Rank #2, Score:{:.2f}Rank #3, Score:{:.2f}
    '\n",
+        "    html += ''.format(url)\n",
+        "    html += '
    '\n",
+        "  return html\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ime5V4kDewh8"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Load example videos and define text queries  { display-mode: \"form\" }\n",
+        "\n",
+        "video_1_url = 'https://upload.wikimedia.org/wikipedia/commons/b/b0/YosriAirTerjun.gif' # @param {type:\"string\"}\n",
+        "video_2_url = 'https://upload.wikimedia.org/wikipedia/commons/e/e6/Guitar_solo_gif.gif' # @param {type:\"string\"}\n",
+        "video_3_url = 'https://upload.wikimedia.org/wikipedia/commons/3/30/2009-08-16-autodrift-by-RalfR-gif-by-wau.gif' # @param {type:\"string\"}\n",
+        "\n",
+        "video_1 = load_video(video_1_url)\n",
+        "video_2 = load_video(video_2_url)\n",
+        "video_3 = load_video(video_3_url)\n",
+        "all_videos = [video_1, video_2, video_3]\n",
+        "\n",
+        "query_1_video = 'waterfall' # @param {type:\"string\"}\n",
+        "query_2_video = 'playing guitar' # @param {type:\"string\"}\n",
+        "query_3_video = 'car drifting' # @param {type:\"string\"}\n",
+        "all_queries_video = [query_1_video, query_2_video, query_3_video]\n",
+        "all_videos_urls = [video_1_url, video_2_url, video_3_url]\n",
+        "display_video(all_videos_urls)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NCLKv_L_8Anc"
+      },
+      "source": [
+        "## Demonstrate text to video retrieval\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9oX8ItFUjybi"
+      },
+      "outputs": [],
+      "source": [
+        "# Prepare video inputs.\n",
+        "videos_np = np.stack(all_videos, axis=0)\n",
+        "\n",
+        "# Prepare text input.\n",
+        "words_np = np.array(all_queries_video)\n",
+        "\n",
+        "# Generate the video and text embeddings.\n",
+        "video_embd, text_embd = generate_embeddings(hub_model, videos_np, words_np)\n",
+        "\n",
+        "# Scores between video and text is computed by dot products.\n",
+        "all_scores = np.dot(text_embd, tf.transpose(video_embd))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d4AwYmODmE9Y"
+      },
+      "outputs": [],
+      "source": [
+        "# Display results.\n",
+        "html = ''\n",
+        "for i, words in enumerate(words_np):\n",
+        "  html += display_query_and_results_video(words, all_videos_urls, all_scores[i, :])\n",
+        "  html += '
    '\n",
+        "display.HTML(html)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "text_to_video_retrieval_with_s3d_milnce.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf2_arbitrary_image_stylization.ipynb b/site/en/hub/tutorials/tf2_arbitrary_image_stylization.ipynb
new file mode 100644
index 00000000000..3a0cb09113e
--- /dev/null
+++ b/site/en/hub/tutorials/tf2_arbitrary_image_stylization.ipynb
@@ -0,0 +1,375 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ScitaPqhKtuW"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jvztxQ6VsK2k"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oXlcl8lqBgAD"
+      },
+      "source": [
+        "# Fast Style Transfer for Arbitrary Styles\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YeeuYzbZcJzs"
+      },
+      "source": [
+        "Based on the model code in [magenta](https://github.com/tensorflow/magenta/tree/master/magenta/models/arbitrary_image_stylization) and the publication:\n",
+        "\n",
+        "[Exploring the structure of a real-time, arbitrary neural artistic stylization\n",
+        "network](https://arxiv.org/abs/1705.06830).\n",
+        "*Golnaz Ghiasi, Honglak Lee,\n",
+        "Manjunath Kudlur, Vincent Dumoulin, Jonathon Shlens*,\n",
+        "Proceedings of the British Machine Vision Conference (BMVC), 2017.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TaM8BVxrCA2E"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "J65jog2ncJzt"
+      },
+      "source": [
+        "Let's start with importing TF2 and all relevant dependencies."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "v-KXRY5XBu2u"
+      },
+      "outputs": [],
+      "source": [
+        "import functools\n",
+        "import os\n",
+        "\n",
+        "from matplotlib import gridspec\n",
+        "import matplotlib.pylab as plt\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "print(\"TF Version: \", tf.__version__)\n",
+        "print(\"TF Hub version: \", hub.__version__)\n",
+        "print(\"Eager mode enabled: \", tf.executing_eagerly())\n",
+        "print(\"GPU available: \", tf.config.list_physical_devices('GPU'))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tsoDv_9geoZn"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Define image loading and visualization functions  { display-mode: \"form\" }\n",
+        "\n",
+        "def crop_center(image):\n",
+        "  \"\"\"Returns a cropped square image.\"\"\"\n",
+        "  shape = image.shape\n",
+        "  new_shape = min(shape[1], shape[2])\n",
+        "  offset_y = max(shape[1] - shape[2], 0) // 2\n",
+        "  offset_x = max(shape[2] - shape[1], 0) // 2\n",
+        "  image = tf.image.crop_to_bounding_box(\n",
+        "      image, offset_y, offset_x, new_shape, new_shape)\n",
+        "  return image\n",
+        "\n",
+        "@functools.lru_cache(maxsize=None)\n",
+        "def load_image(image_url, image_size=(256, 256), preserve_aspect_ratio=True):\n",
+        "  \"\"\"Loads and preprocesses images.\"\"\"\n",
+        "  # Cache image file locally.\n",
+        "  image_path = tf.keras.utils.get_file(os.path.basename(image_url)[-128:], image_url)\n",
+        "  # Load and convert to float32 numpy array, add batch dimension, and normalize to range [0, 1].\n",
+        "  img = tf.io.decode_image(\n",
+        "      tf.io.read_file(image_path),\n",
+        "      channels=3, dtype=tf.float32)[tf.newaxis, ...]\n",
+        "  img = crop_center(img)\n",
+        "  img = tf.image.resize(img, image_size, preserve_aspect_ratio=True)\n",
+        "  return img\n",
+        "\n",
+        "def show_n(images, titles=('',)):\n",
+        "  n = len(images)\n",
+        "  image_sizes = [image.shape[1] for image in images]\n",
+        "  w = (image_sizes[0] * 6) // 320\n",
+        "  plt.figure(figsize=(w * n, w))\n",
+        "  gs = gridspec.GridSpec(1, n, width_ratios=image_sizes)\n",
+        "  for i in range(n):\n",
+        "    plt.subplot(gs[i])\n",
+        "    plt.imshow(images[i][0], aspect='equal')\n",
+        "    plt.axis('off')\n",
+        "    plt.title(titles[i] if len(titles) > i else '')\n",
+        "  plt.show()\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8etHh05-CJHc"
+      },
+      "source": [
+        "Let's get as well some images to play with."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dRc0vat3Alzo"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Load example images  { display-mode: \"form\" }\n",
+        "\n",
+        "content_image_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/f/fd/Golden_Gate_Bridge_from_Battery_Spencer.jpg/640px-Golden_Gate_Bridge_from_Battery_Spencer.jpg'  # @param {type:\"string\"}\n",
+        "style_image_url = 'https://upload.wikimedia.org/wikipedia/commons/0/0a/The_Great_Wave_off_Kanagawa.jpg'  # @param {type:\"string\"}\n",
+        "output_image_size = 384  # @param {type:\"integer\"}\n",
+        "\n",
+        "# The content image size can be arbitrary.\n",
+        "content_img_size = (output_image_size, output_image_size)\n",
+        "# The style prediction model was trained with image size 256 and it's the \n",
+        "# recommended image size for the style image (though, other sizes work as \n",
+        "# well but will lead to different results).\n",
+        "style_img_size = (256, 256)  # Recommended to keep it at 256.\n",
+        "\n",
+        "content_image = load_image(content_image_url, content_img_size)\n",
+        "style_image = load_image(style_image_url, style_img_size)\n",
+        "style_image = tf.nn.avg_pool(style_image, ksize=[3,3], strides=[1,1], padding='SAME')\n",
+        "show_n([content_image, style_image], ['Content image', 'Style image'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yL2Bn5ThR1nY"
+      },
+      "source": [
+        "## Import TF Hub module"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "467AVDSuzBPc"
+      },
+      "outputs": [],
+      "source": [
+        "# Load TF Hub module.\n",
+        "\n",
+        "hub_handle = 'https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2'\n",
+        "hub_module = hub.load(hub_handle)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "uAR70_3wLEDB"
+      },
+      "source": [
+        "The signature of this hub module for image stylization is:\n",
+        "```\n",
+        "outputs = hub_module(content_image, style_image)\n",
+        "stylized_image = outputs[0]\n",
+        "```\n",
+        "Where `content_image`, `style_image`, and `stylized_image` are expected to be 4-D Tensors with shapes `[batch_size, image_height, image_width, 3]`.\n",
+        "\n",
+        "In the current example we provide only single images and therefore the batch dimension is 1, but one can use the same module to process more images at the same time.\n",
+        "\n",
+        "The input and output values of the images should be in the range [0, 1].\n",
+        "\n",
+        "The shapes of content and style image don't have to match. Output image shape\n",
+        "is the same as the content image shape."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qEhYJno1R7rP"
+      },
+      "source": [
+        "## Demonstrate image stylization"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lnAv-F3O9fLV"
+      },
+      "outputs": [],
+      "source": [
+        "# Stylize content image with given style image.\n",
+        "# This is pretty fast within a few milliseconds on a GPU.\n",
+        "\n",
+        "outputs = hub_module(tf.constant(content_image), tf.constant(style_image))\n",
+        "stylized_image = outputs[0]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OEAPEdq698gs"
+      },
+      "outputs": [],
+      "source": [
+        "# Visualize input images and the generated stylized image.\n",
+        "\n",
+        "show_n([content_image, style_image, stylized_image], titles=['Original content image', 'Style image', 'Stylized image'])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v-gYvjTWK-lx"
+      },
+      "source": [
+        "## Let's try it on more images"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WSMaY0YBNfkK"
+      },
+      "outputs": [],
+      "source": [
+        "# @title To Run: Load more images { display-mode: \"form\" }\n",
+        "\n",
+        "content_urls = dict(\n",
+        "  sea_turtle='https://upload.wikimedia.org/wikipedia/commons/d/d7/Green_Sea_Turtle_grazing_seagrass.jpg',\n",
+        "  tuebingen='https://upload.wikimedia.org/wikipedia/commons/0/00/Tuebingen_Neckarfront.jpg',\n",
+        "  grace_hopper='https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg',\n",
+        "  )\n",
+        "style_urls = dict(\n",
+        "  kanagawa_great_wave='https://upload.wikimedia.org/wikipedia/commons/0/0a/The_Great_Wave_off_Kanagawa.jpg',\n",
+        "  kandinsky_composition_7='https://upload.wikimedia.org/wikipedia/commons/b/b4/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg',\n",
+        "  hubble_pillars_of_creation='https://upload.wikimedia.org/wikipedia/commons/6/68/Pillars_of_creation_2014_HST_WFC3-UVIS_full-res_denoised.jpg',\n",
+        "  van_gogh_starry_night='https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1024px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg',\n",
+        "  turner_nantes='https://upload.wikimedia.org/wikipedia/commons/b/b7/JMW_Turner_-_Nantes_from_the_Ile_Feydeau.jpg',\n",
+        "  munch_scream='https://upload.wikimedia.org/wikipedia/commons/c/c5/Edvard_Munch%2C_1893%2C_The_Scream%2C_oil%2C_tempera_and_pastel_on_cardboard%2C_91_x_73_cm%2C_National_Gallery_of_Norway.jpg',\n",
+        "  picasso_demoiselles_avignon='https://upload.wikimedia.org/wikipedia/en/4/4c/Les_Demoiselles_d%27Avignon.jpg',\n",
+        "  picasso_violin='https://upload.wikimedia.org/wikipedia/en/3/3c/Pablo_Picasso%2C_1911-12%2C_Violon_%28Violin%29%2C_oil_on_canvas%2C_Kr%C3%B6ller-M%C3%BCller_Museum%2C_Otterlo%2C_Netherlands.jpg',\n",
+        "  picasso_bottle_of_rum='https://upload.wikimedia.org/wikipedia/en/7/7f/Pablo_Picasso%2C_1911%2C_Still_Life_with_a_Bottle_of_Rum%2C_oil_on_canvas%2C_61.3_x_50.5_cm%2C_Metropolitan_Museum_of_Art%2C_New_York.jpg',\n",
+        "  fire='https://upload.wikimedia.org/wikipedia/commons/3/36/Large_bonfire.jpg',\n",
+        "  derkovits_woman_head='https://upload.wikimedia.org/wikipedia/commons/0/0d/Derkovits_Gyula_Woman_head_1922.jpg',\n",
+        "  amadeo_style_life='https://upload.wikimedia.org/wikipedia/commons/8/8e/Untitled_%28Still_life%29_%281913%29_-_Amadeo_Souza-Cardoso_%281887-1918%29_%2817385824283%29.jpg',\n",
+        "  derkovtis_talig='https://upload.wikimedia.org/wikipedia/commons/3/37/Derkovits_Gyula_Talig%C3%A1s_1920.jpg',\n",
+        "  amadeo_cardoso='https://upload.wikimedia.org/wikipedia/commons/7/7d/Amadeo_de_Souza-Cardoso%2C_1915_-_Landscape_with_black_figure.jpg'\n",
+        ")\n",
+        "\n",
+        "content_image_size = 384\n",
+        "style_image_size = 256\n",
+        "content_images = {k: load_image(v, (content_image_size, content_image_size)) for k, v in content_urls.items()}\n",
+        "style_images = {k: load_image(v, (style_image_size, style_image_size)) for k, v in style_urls.items()}\n",
+        "style_images = {k: tf.nn.avg_pool(style_image, ksize=[3,3], strides=[1,1], padding='SAME') for k, style_image in style_images.items()}\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dqB6aNTLNVkK"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Specify the main content image and the style you want to use.  { display-mode: \"form\" }\n",
+        "\n",
+        "content_name = 'sea_turtle'  # @param ['sea_turtle', 'tuebingen', 'grace_hopper']\n",
+        "style_name = 'munch_scream'  # @param ['kanagawa_great_wave', 'kandinsky_composition_7', 'hubble_pillars_of_creation', 'van_gogh_starry_night', 'turner_nantes', 'munch_scream', 'picasso_demoiselles_avignon', 'picasso_violin', 'picasso_bottle_of_rum', 'fire', 'derkovits_woman_head', 'amadeo_style_life', 'derkovtis_talig', 'amadeo_cardoso']\n",
+        "\n",
+        "stylized_image = hub_module(tf.constant(content_images[content_name]),\n",
+        "                            tf.constant(style_images[style_name]))[0]\n",
+        "\n",
+        "show_n([content_images[content_name], style_images[style_name], stylized_image],\n",
+        "       titles=['Original content image', 'Style image', 'Stylized image'])"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tf2_arbitrary_image_stylization.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf2_image_retraining.ipynb b/site/en/hub/tutorials/tf2_image_retraining.ipynb
new file mode 100644
index 00000000000..0266f4683c1
--- /dev/null
+++ b/site/en/hub/tutorials/tf2_image_retraining.ipynb
@@ -0,0 +1,605 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ScitaPqhKtuW"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jvztxQ6VsK2k"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oYM61xrTsP5d"
+      },
+      "source": [
+        "# Retraining an Image Classifier\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L1otmJgmbahf"
+      },
+      "source": [
+        "## Introduction\n",
+        "\n",
+        "Image classification models have millions of parameters. Training them from\n",
+        "scratch requires a lot of labeled training data and a lot of computing power. Transfer learning is a technique that shortcuts much of this by taking a piece of a model that has already been trained on a related task and reusing it in a new model.\n",
+        "\n",
+        "This Colab demonstrates how to build a Keras model for classifying five species of flowers by using a pre-trained TF2 SavedModel from TensorFlow Hub for image feature extraction, trained on the much larger and more general ImageNet dataset. Optionally, the feature extractor can be trained (\"fine-tuned\") alongside the newly added classifier.\n",
+        "\n",
+        "### Looking for a tool instead?\n",
+        "\n",
+        "This is a TensorFlow coding tutorial. If you want a tool that just builds the TensorFlow or TFLite model for, take a look at the [make_image_classifier](https://github.com/tensorflow/hub/tree/master/tensorflow_hub/tools/make_image_classifier) command-line tool that gets [installed](https://www.tensorflow.org/hub/installation) by the PIP package `tensorflow-hub[make_image_classifier]`, or at [this](https://colab.sandbox.google.com/github/tensorflow/examples/blob/master/tensorflow_examples/lite/model_maker/demo/image_classification.ipynb) TFLite colab.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "bL54LWCHt5q5"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "dlauq-4FWGZM"
+      },
+      "outputs": [],
+      "source": [
+        "import itertools\n",
+        "import os\n",
+        "\n",
+        "import matplotlib.pylab as plt\n",
+        "import numpy as np\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "print(\"TF version:\", tf.__version__)\n",
+        "print(\"Hub version:\", hub.__version__)\n",
+        "print(\"GPU is\", \"available\" if tf.config.list_physical_devices('GPU') else \"NOT AVAILABLE\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mmaHHH7Pvmth"
+      },
+      "source": [
+        "## Select the TF2 SavedModel module to use\n",
+        "\n",
+        "For starters, use [https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/feature_vector/4](https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/feature_vector/4). The same URL can be used in code to identify the SavedModel and in your browser to show its documentation. (Note that models in TF1 Hub format won't work here.)\n",
+        "\n",
+        "You can find more TF2 models that generate image feature vectors [here](https://tfhub.dev/s?module-type=image-feature-vector&tf-version=tf2).\n",
+        "\n",
+        "There are multiple possible models to try. All you need to do is select a different one on the cell below and follow up with the notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FlsEcKVeuCnf"
+      },
+      "outputs": [],
+      "source": [
+        "#@title\n",
+        "\n",
+        "model_name = \"efficientnetv2-xl-21k\" # @param ['efficientnetv2-s', 'efficientnetv2-m', 'efficientnetv2-l', 'efficientnetv2-s-21k', 'efficientnetv2-m-21k', 'efficientnetv2-l-21k', 'efficientnetv2-xl-21k', 'efficientnetv2-b0-21k', 'efficientnetv2-b1-21k', 'efficientnetv2-b2-21k', 'efficientnetv2-b3-21k', 'efficientnetv2-s-21k-ft1k', 'efficientnetv2-m-21k-ft1k', 'efficientnetv2-l-21k-ft1k', 'efficientnetv2-xl-21k-ft1k', 'efficientnetv2-b0-21k-ft1k', 'efficientnetv2-b1-21k-ft1k', 'efficientnetv2-b2-21k-ft1k', 'efficientnetv2-b3-21k-ft1k', 'efficientnetv2-b0', 'efficientnetv2-b1', 'efficientnetv2-b2', 'efficientnetv2-b3', 'efficientnet_b0', 'efficientnet_b1', 'efficientnet_b2', 'efficientnet_b3', 'efficientnet_b4', 'efficientnet_b5', 'efficientnet_b6', 'efficientnet_b7', 'bit_s-r50x1', 'inception_v3', 'inception_resnet_v2', 'resnet_v1_50', 'resnet_v1_101', 'resnet_v1_152', 'resnet_v2_50', 'resnet_v2_101', 'resnet_v2_152', 'nasnet_large', 'nasnet_mobile', 'pnasnet_large', 'mobilenet_v2_100_224', 'mobilenet_v2_130_224', 'mobilenet_v2_140_224', 'mobilenet_v3_small_100_224', 'mobilenet_v3_small_075_224', 'mobilenet_v3_large_100_224', 'mobilenet_v3_large_075_224']\n",
+        "\n",
+        "model_handle_map = {\n",
+        "  \"efficientnetv2-s\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_s/feature_vector/2\",\n",
+        "  \"efficientnetv2-m\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_m/feature_vector/2\",\n",
+        "  \"efficientnetv2-l\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_l/feature_vector/2\",\n",
+        "  \"efficientnetv2-s-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_s/feature_vector/2\",\n",
+        "  \"efficientnetv2-m-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_m/feature_vector/2\",\n",
+        "  \"efficientnetv2-l-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_l/feature_vector/2\",\n",
+        "  \"efficientnetv2-xl-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_xl/feature_vector/2\",\n",
+        "  \"efficientnetv2-b0-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_b0/feature_vector/2\",\n",
+        "  \"efficientnetv2-b1-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_b1/feature_vector/2\",\n",
+        "  \"efficientnetv2-b2-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_b2/feature_vector/2\",\n",
+        "  \"efficientnetv2-b3-21k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_b3/feature_vector/2\",\n",
+        "  \"efficientnetv2-s-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_s/feature_vector/2\",\n",
+        "  \"efficientnetv2-m-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_m/feature_vector/2\",\n",
+        "  \"efficientnetv2-l-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_l/feature_vector/2\",\n",
+        "  \"efficientnetv2-xl-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_xl/feature_vector/2\",\n",
+        "  \"efficientnetv2-b0-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_b0/feature_vector/2\",\n",
+        "  \"efficientnetv2-b1-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_b1/feature_vector/2\",\n",
+        "  \"efficientnetv2-b2-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_b2/feature_vector/2\",\n",
+        "  \"efficientnetv2-b3-21k-ft1k\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet21k_ft1k_b3/feature_vector/2\",\n",
+        "  \"efficientnetv2-b0\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b0/feature_vector/2\",\n",
+        "  \"efficientnetv2-b1\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b1/feature_vector/2\",\n",
+        "  \"efficientnetv2-b2\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b2/feature_vector/2\",\n",
+        "  \"efficientnetv2-b3\": \"https://tfhub.dev/google/imagenet/efficientnet_v2_imagenet1k_b3/feature_vector/2\",\n",
+        "  \"efficientnet_b0\": \"https://tfhub.dev/tensorflow/efficientnet/b0/feature-vector/1\",\n",
+        "  \"efficientnet_b1\": \"https://tfhub.dev/tensorflow/efficientnet/b1/feature-vector/1\",\n",
+        "  \"efficientnet_b2\": \"https://tfhub.dev/tensorflow/efficientnet/b2/feature-vector/1\",\n",
+        "  \"efficientnet_b3\": \"https://tfhub.dev/tensorflow/efficientnet/b3/feature-vector/1\",\n",
+        "  \"efficientnet_b4\": \"https://tfhub.dev/tensorflow/efficientnet/b4/feature-vector/1\",\n",
+        "  \"efficientnet_b5\": \"https://tfhub.dev/tensorflow/efficientnet/b5/feature-vector/1\",\n",
+        "  \"efficientnet_b6\": \"https://tfhub.dev/tensorflow/efficientnet/b6/feature-vector/1\",\n",
+        "  \"efficientnet_b7\": \"https://tfhub.dev/tensorflow/efficientnet/b7/feature-vector/1\",\n",
+        "  \"bit_s-r50x1\": \"https://tfhub.dev/google/bit/s-r50x1/1\",\n",
+        "  \"inception_v3\": \"https://tfhub.dev/google/imagenet/inception_v3/feature-vector/4\",\n",
+        "  \"inception_resnet_v2\": \"https://tfhub.dev/google/imagenet/inception_resnet_v2/feature-vector/4\",\n",
+        "  \"resnet_v1_50\": \"https://tfhub.dev/google/imagenet/resnet_v1_50/feature-vector/4\",\n",
+        "  \"resnet_v1_101\": \"https://tfhub.dev/google/imagenet/resnet_v1_101/feature-vector/4\",\n",
+        "  \"resnet_v1_152\": \"https://tfhub.dev/google/imagenet/resnet_v1_152/feature-vector/4\",\n",
+        "  \"resnet_v2_50\": \"https://tfhub.dev/google/imagenet/resnet_v2_50/feature-vector/4\",\n",
+        "  \"resnet_v2_101\": \"https://tfhub.dev/google/imagenet/resnet_v2_101/feature-vector/4\",\n",
+        "  \"resnet_v2_152\": \"https://tfhub.dev/google/imagenet/resnet_v2_152/feature-vector/4\",\n",
+        "  \"nasnet_large\": \"https://tfhub.dev/google/imagenet/nasnet_large/feature_vector/4\",\n",
+        "  \"nasnet_mobile\": \"https://tfhub.dev/google/imagenet/nasnet_mobile/feature_vector/4\",\n",
+        "  \"pnasnet_large\": \"https://tfhub.dev/google/imagenet/pnasnet_large/feature_vector/4\",\n",
+        "  \"mobilenet_v2_100_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v2_100_224/feature_vector/4\",\n",
+        "  \"mobilenet_v2_130_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v2_130_224/feature_vector/4\",\n",
+        "  \"mobilenet_v2_140_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v2_140_224/feature_vector/4\",\n",
+        "  \"mobilenet_v3_small_100_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v3_small_100_224/feature_vector/5\",\n",
+        "  \"mobilenet_v3_small_075_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v3_small_075_224/feature_vector/5\",\n",
+        "  \"mobilenet_v3_large_100_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v3_large_100_224/feature_vector/5\",\n",
+        "  \"mobilenet_v3_large_075_224\": \"https://tfhub.dev/google/imagenet/mobilenet_v3_large_075_224/feature_vector/5\",\n",
+        "}\n",
+        "\n",
+        "model_image_size_map = {\n",
+        "  \"efficientnetv2-s\": 384,\n",
+        "  \"efficientnetv2-m\": 480,\n",
+        "  \"efficientnetv2-l\": 480,\n",
+        "  \"efficientnetv2-b0\": 224,\n",
+        "  \"efficientnetv2-b1\": 240,\n",
+        "  \"efficientnetv2-b2\": 260,\n",
+        "  \"efficientnetv2-b3\": 300,\n",
+        "  \"efficientnetv2-s-21k\": 384,\n",
+        "  \"efficientnetv2-m-21k\": 480,\n",
+        "  \"efficientnetv2-l-21k\": 480,\n",
+        "  \"efficientnetv2-xl-21k\": 512,\n",
+        "  \"efficientnetv2-b0-21k\": 224,\n",
+        "  \"efficientnetv2-b1-21k\": 240,\n",
+        "  \"efficientnetv2-b2-21k\": 260,\n",
+        "  \"efficientnetv2-b3-21k\": 300,\n",
+        "  \"efficientnetv2-s-21k-ft1k\": 384,\n",
+        "  \"efficientnetv2-m-21k-ft1k\": 480,\n",
+        "  \"efficientnetv2-l-21k-ft1k\": 480,\n",
+        "  \"efficientnetv2-xl-21k-ft1k\": 512,\n",
+        "  \"efficientnetv2-b0-21k-ft1k\": 224,\n",
+        "  \"efficientnetv2-b1-21k-ft1k\": 240,\n",
+        "  \"efficientnetv2-b2-21k-ft1k\": 260,\n",
+        "  \"efficientnetv2-b3-21k-ft1k\": 300, \n",
+        "  \"efficientnet_b0\": 224,\n",
+        "  \"efficientnet_b1\": 240,\n",
+        "  \"efficientnet_b2\": 260,\n",
+        "  \"efficientnet_b3\": 300,\n",
+        "  \"efficientnet_b4\": 380,\n",
+        "  \"efficientnet_b5\": 456,\n",
+        "  \"efficientnet_b6\": 528,\n",
+        "  \"efficientnet_b7\": 600,\n",
+        "  \"inception_v3\": 299,\n",
+        "  \"inception_resnet_v2\": 299,\n",
+        "  \"nasnet_large\": 331,\n",
+        "  \"pnasnet_large\": 331,\n",
+        "}\n",
+        "\n",
+        "model_handle = model_handle_map.get(model_name)\n",
+        "pixels = model_image_size_map.get(model_name, 224)\n",
+        "\n",
+        "print(f\"Selected model: {model_name} : {model_handle}\")\n",
+        "\n",
+        "IMAGE_SIZE = (pixels, pixels)\n",
+        "print(f\"Input size {IMAGE_SIZE}\")\n",
+        "\n",
+        "BATCH_SIZE = 16#@param {type:\"integer\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yTY8qzyYv3vl"
+      },
+      "source": [
+        "## Set up the Flowers dataset\n",
+        "\n",
+        "Inputs are suitably resized for the selected module. Dataset augmentation (i.e., random distortions of an image each time it is read) improves training, esp. when fine-tuning."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WBtFK1hO8KsO"
+      },
+      "outputs": [],
+      "source": [
+        "data_dir = tf.keras.utils.get_file(\n",
+        "    'flower_photos',\n",
+        "    'https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n",
+        "    untar=True)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "umB5tswsfTEQ"
+      },
+      "outputs": [],
+      "source": [
+        "def build_dataset(subset):\n",
+        "  return tf.keras.preprocessing.image_dataset_from_directory(\n",
+        "      data_dir,\n",
+        "      validation_split=.20,\n",
+        "      subset=subset,\n",
+        "      label_mode=\"categorical\",\n",
+        "      # Seed needs to provided when using validation_split and shuffle = True.\n",
+        "      # A fixed seed is used so that the validation set is stable across runs.\n",
+        "      seed=123,\n",
+        "      image_size=IMAGE_SIZE,\n",
+        "      batch_size=1)\n",
+        "\n",
+        "train_ds = build_dataset(\"training\")\n",
+        "class_names = tuple(train_ds.class_names)\n",
+        "train_size = train_ds.cardinality().numpy()\n",
+        "train_ds = train_ds.unbatch().batch(BATCH_SIZE)\n",
+        "train_ds = train_ds.repeat()\n",
+        "\n",
+        "normalization_layer = tf.keras.layers.Rescaling(1. / 255)\n",
+        "preprocessing_model = tf.keras.Sequential([normalization_layer])\n",
+        "do_data_augmentation = False #@param {type:\"boolean\"}\n",
+        "if do_data_augmentation:\n",
+        "  preprocessing_model.add(\n",
+        "      tf.keras.layers.RandomRotation(40))\n",
+        "  preprocessing_model.add(\n",
+        "      tf.keras.layers.RandomTranslation(0, 0.2))\n",
+        "  preprocessing_model.add(\n",
+        "      tf.keras.layers.RandomTranslation(0.2, 0))\n",
+        "  # Like the old tf.keras.preprocessing.image.ImageDataGenerator(),\n",
+        "  # image sizes are fixed when reading, and then a random zoom is applied.\n",
+        "  # If all training inputs are larger than image_size, one could also use\n",
+        "  # RandomCrop with a batch size of 1 and rebatch later.\n",
+        "  preprocessing_model.add(\n",
+        "      tf.keras.layers.RandomZoom(0.2, 0.2))\n",
+        "  preprocessing_model.add(\n",
+        "      tf.keras.layers.RandomFlip(mode=\"horizontal\"))\n",
+        "train_ds = train_ds.map(lambda images, labels:\n",
+        "                        (preprocessing_model(images), labels))\n",
+        "\n",
+        "val_ds = build_dataset(\"validation\")\n",
+        "valid_size = val_ds.cardinality().numpy()\n",
+        "val_ds = val_ds.unbatch().batch(BATCH_SIZE)\n",
+        "val_ds = val_ds.map(lambda images, labels:\n",
+        "                    (normalization_layer(images), labels))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FS_gVStowW3G"
+      },
+      "source": [
+        "## Defining the model\n",
+        "\n",
+        "All it takes is to put a linear classifier on top of the `feature_extractor_layer` with the Hub module.\n",
+        "\n",
+        "For speed, we start out with a non-trainable `feature_extractor_layer`, but you can also enable fine-tuning for greater accuracy."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RaJW3XrPyFiF"
+      },
+      "outputs": [],
+      "source": [
+        "do_fine_tuning = False #@param {type:\"boolean\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "50FYNIb1dmJH"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Building model with\", model_handle)\n",
+        "model = tf.keras.Sequential([\n",
+        "    # Explicitly define the input shape so the model can be properly\n",
+        "    # loaded by the TFLiteConverter\n",
+        "    tf.keras.layers.InputLayer(input_shape=IMAGE_SIZE + (3,)),\n",
+        "    hub.KerasLayer(model_handle, trainable=do_fine_tuning),\n",
+        "    tf.keras.layers.Dropout(rate=0.2),\n",
+        "    tf.keras.layers.Dense(len(class_names),\n",
+        "                          kernel_regularizer=tf.keras.regularizers.l2(0.0001))\n",
+        "])\n",
+        "model.build((None,)+IMAGE_SIZE+(3,))\n",
+        "model.summary()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "u2e5WupIw2N2"
+      },
+      "source": [
+        "## Training the model"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "9f3yBUvkd_VJ"
+      },
+      "outputs": [],
+      "source": [
+        "model.compile(\n",
+        "  optimizer=tf.keras.optimizers.SGD(learning_rate=0.005, momentum=0.9), \n",
+        "  loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1),\n",
+        "  metrics=['accuracy'])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "w_YKX2Qnfg6x"
+      },
+      "outputs": [],
+      "source": [
+        "steps_per_epoch = train_size // BATCH_SIZE\n",
+        "validation_steps = valid_size // BATCH_SIZE\n",
+        "hist = model.fit(\n",
+        "    train_ds,\n",
+        "    epochs=5, steps_per_epoch=steps_per_epoch,\n",
+        "    validation_data=val_ds,\n",
+        "    validation_steps=validation_steps).history"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "CYOw0fTO1W4x"
+      },
+      "outputs": [],
+      "source": [
+        "plt.figure()\n",
+        "plt.ylabel(\"Loss (training and validation)\")\n",
+        "plt.xlabel(\"Training Steps\")\n",
+        "plt.ylim([0,2])\n",
+        "plt.plot(hist[\"loss\"])\n",
+        "plt.plot(hist[\"val_loss\"])\n",
+        "\n",
+        "plt.figure()\n",
+        "plt.ylabel(\"Accuracy (training and validation)\")\n",
+        "plt.xlabel(\"Training Steps\")\n",
+        "plt.ylim([0,1])\n",
+        "plt.plot(hist[\"accuracy\"])\n",
+        "plt.plot(hist[\"val_accuracy\"])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jZ8DKKgeKv4-"
+      },
+      "source": [
+        "Try out the model on an image from the validation data:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oi1iCNB9K1Ai"
+      },
+      "outputs": [],
+      "source": [
+        "x, y = next(iter(val_ds))\n",
+        "image = x[0, :, :, :]\n",
+        "true_index = np.argmax(y[0])\n",
+        "plt.imshow(image)\n",
+        "plt.axis('off')\n",
+        "plt.show()\n",
+        "\n",
+        "# Expand the validation image to (1, 224, 224, 3) before predicting the label\n",
+        "prediction_scores = model.predict(np.expand_dims(image, axis=0))\n",
+        "predicted_index = np.argmax(prediction_scores)\n",
+        "print(\"True label: \" + class_names[true_index])\n",
+        "print(\"Predicted label: \" + class_names[predicted_index])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YCsAsQM1IRvA"
+      },
+      "source": [
+        "Finally, the trained model can be saved for deployment to TF Serving or TFLite (on mobile) as follows."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LGvTi69oIc2d"
+      },
+      "outputs": [],
+      "source": [
+        "saved_model_path = f\"/tmp/saved_flowers_model_{model_name}\"\n",
+        "tf.saved_model.save(model, saved_model_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QzW4oNRjILaq"
+      },
+      "source": [
+        "## Optional: Deployment to TensorFlow Lite\n",
+        "\n",
+        "[TensorFlow Lite](https://www.tensorflow.org/lite) lets you deploy TensorFlow models to mobile and IoT devices. The code below shows how to convert the trained model to TFLite and apply post-training tools from the [TensorFlow Model Optimization Toolkit](https://www.tensorflow.org/model_optimization). Finally, it runs it in the TFLite Interpreter to examine the resulting quality\n",
+        "\n",
+        "  * Converting without optimization provides the same results as before (up to roundoff error).\n",
+        "  * Converting with optimization without any data quantizes the model weights to 8 bits, but inference still uses floating-point computation for the neural network activations. This reduces model size almost by a factor of 4 and improves CPU latency on mobile devices.\n",
+        "  * On top, computation of the neural network activations can be quantized to 8-bit integers as well if a small reference dataset is provided to calibrate the quantization range. On a mobile device, this accelerates inference further and makes it possible to run on accelerators like Edge TPU."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Va1Vo92fSyV6"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Optimization settings\n",
+        "optimize_lite_model = False  #@param {type:\"boolean\"}\n",
+        "#@markdown Setting a value greater than zero enables quantization of neural network activations. A few dozen is already a useful amount.\n",
+        "num_calibration_examples = 60  #@param {type:\"slider\", min:0, max:1000, step:1}\n",
+        "representative_dataset = None\n",
+        "if optimize_lite_model and num_calibration_examples:\n",
+        "  # Use a bounded number of training examples without labels for calibration.\n",
+        "  # TFLiteConverter expects a list of input tensors, each with batch size 1.\n",
+        "  representative_dataset = lambda: itertools.islice(\n",
+        "      ([image[None, ...]] for batch, _ in train_ds for image in batch),\n",
+        "      num_calibration_examples)\n",
+        "\n",
+        "converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_path)\n",
+        "if optimize_lite_model:\n",
+        "  converter.optimizations = [tf.lite.Optimize.DEFAULT]\n",
+        "  if representative_dataset:  # This is optional, see above.\n",
+        "    converter.representative_dataset = representative_dataset\n",
+        "lite_model_content = converter.convert()\n",
+        "\n",
+        "with open(f\"/tmp/lite_flowers_model_{model_name}.tflite\", \"wb\") as f:\n",
+        "  f.write(lite_model_content)\n",
+        "print(\"Wrote %sTFLite model of %d bytes.\" %\n",
+        "      (\"optimized \" if optimize_lite_model else \"\", len(lite_model_content)))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_wqEmD0xIqeG"
+      },
+      "outputs": [],
+      "source": [
+        "interpreter = tf.lite.Interpreter(model_content=lite_model_content)\n",
+        "# This little helper wraps the TFLite Interpreter as a numpy-to-numpy function.\n",
+        "def lite_model(images):\n",
+        "  interpreter.allocate_tensors()\n",
+        "  interpreter.set_tensor(interpreter.get_input_details()[0]['index'], images)\n",
+        "  interpreter.invoke()\n",
+        "  return interpreter.get_tensor(interpreter.get_output_details()[0]['index'])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JMMK-fZrKrk8"
+      },
+      "outputs": [],
+      "source": [
+        "#@markdown For rapid experimentation, start with a moderate number of examples.\n",
+        "num_eval_examples = 50  #@param {type:\"slider\", min:0, max:700}\n",
+        "eval_dataset = ((image, label)  # TFLite expects batch size 1.\n",
+        "                for batch in train_ds\n",
+        "                for (image, label) in zip(*batch))\n",
+        "count = 0\n",
+        "count_lite_tf_agree = 0\n",
+        "count_lite_correct = 0\n",
+        "for image, label in eval_dataset:\n",
+        "  probs_lite = lite_model(image[None, ...])[0]\n",
+        "  probs_tf = model(image[None, ...]).numpy()[0]\n",
+        "  y_lite = np.argmax(probs_lite)\n",
+        "  y_tf = np.argmax(probs_tf)\n",
+        "  y_true = np.argmax(label)\n",
+        "  count +=1\n",
+        "  if y_lite == y_tf: count_lite_tf_agree += 1\n",
+        "  if y_lite == y_true: count_lite_correct += 1\n",
+        "  if count >= num_eval_examples: break\n",
+        "print(\"TFLite model agrees with original model on %d of %d examples (%g%%).\" %\n",
+        "      (count_lite_tf_agree, count, 100.0 * count_lite_tf_agree / count))\n",
+        "print(\"TFLite model is accurate on %d of %d examples (%g%%).\" %\n",
+        "      (count_lite_correct, count, 100.0 * count_lite_correct / count))"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "ScitaPqhKtuW"
+      ],
+      "name": "tf2_image_retraining.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf2_object_detection.ipynb b/site/en/hub/tutorials/tf2_object_detection.ipynb
new file mode 100644
index 00000000000..d06ad401824
--- /dev/null
+++ b/site/en/hub/tutorials/tf2_object_detection.ipynb
@@ -0,0 +1,616 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "98rds-2OU-Rd"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "1c95xMGcU5_Z"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2020 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "V1UUX8SUUiMO"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rOvvWAVTkMR7"
+      },
+      "source": [
+        "# TensorFlow Hub Object Detection Colab\n",
+        "\n",
+        "Welcome to the TensorFlow Hub Object Detection Colab! This notebook will take you through the steps of running an \"out-of-the-box\" object detection model on images."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IRImnk_7WOq1"
+      },
+      "source": [
+        "### More models\n",
+        "[This](https://tfhub.dev/tensorflow/collections/object_detection/1) collection contains TF2 object detection models that have been trained on the COCO 2017 dataset. [Here](https://tfhub.dev/s?module-type=image-object-detection) you can find all object detection models that are currently hosted on [tfhub.dev](https://tfhub.dev/)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vPs64QA1Zdov"
+      },
+      "source": [
+        "## Imports and Setup\n",
+        "\n",
+        "Let's start with the base imports."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Xk4FU-jx9kc3"
+      },
+      "outputs": [],
+      "source": [
+        "# This Colab requires a recent numpy version.\n",
+        "!pip install numpy==1.24.3\n",
+        "!pip install protobuf==3.20.3\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yn5_uV1HLvaz"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import pathlib\n",
+        "\n",
+        "import matplotlib\n",
+        "import matplotlib.pyplot as plt\n",
+        "\n",
+        "import io\n",
+        "import scipy.misc\n",
+        "import numpy as np\n",
+        "from six import BytesIO\n",
+        "from PIL import Image, ImageDraw, ImageFont\n",
+        "from six.moves.urllib.request import urlopen\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "tf.get_logger().setLevel('ERROR')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IogyryF2lFBL"
+      },
+      "source": [
+        "## Utilities\n",
+        "\n",
+        "Run the following cell to create some utils that will be needed later:\n",
+        "\n",
+        "- Helper method to load an image\n",
+        "- Map of Model Name to TF Hub handle\n",
+        "- List of tuples with Human Keypoints for the COCO 2017 dataset. This is needed for models with keypoints."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "-y9R0Xllefec"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Run this!!\n",
+        "\n",
+        "def load_image_into_numpy_array(path):\n",
+        "  \"\"\"Load an image from file into a numpy array.\n",
+        "\n",
+        "  Puts image into numpy array to feed into tensorflow graph.\n",
+        "  Note that by convention we put it into a numpy array with shape\n",
+        "  (height, width, channels), where channels=3 for RGB.\n",
+        "\n",
+        "  Args:\n",
+        "    path: the file path to the image\n",
+        "\n",
+        "  Returns:\n",
+        "    uint8 numpy array with shape (img_height, img_width, 3)\n",
+        "  \"\"\"\n",
+        "  image = None\n",
+        "  if(path.startswith('http')):\n",
+        "    response = urlopen(path)\n",
+        "    image_data = response.read()\n",
+        "    image_data = BytesIO(image_data)\n",
+        "    image = Image.open(image_data)\n",
+        "  else:\n",
+        "    image_data = tf.io.gfile.GFile(path, 'rb').read()\n",
+        "    image = Image.open(BytesIO(image_data))\n",
+        "\n",
+        "  (im_width, im_height) = image.size\n",
+        "  return np.array(image.getdata()).reshape(\n",
+        "      (1, im_height, im_width, 3)).astype(np.uint8)\n",
+        "\n",
+        "\n",
+        "ALL_MODELS = {\n",
+        "'CenterNet HourGlass104 512x512' : 'https://tfhub.dev/tensorflow/centernet/hourglass_512x512/1',\n",
+        "'CenterNet HourGlass104 Keypoints 512x512' : 'https://tfhub.dev/tensorflow/centernet/hourglass_512x512_kpts/1',\n",
+        "'CenterNet HourGlass104 1024x1024' : 'https://tfhub.dev/tensorflow/centernet/hourglass_1024x1024/1',\n",
+        "'CenterNet HourGlass104 Keypoints 1024x1024' : 'https://tfhub.dev/tensorflow/centernet/hourglass_1024x1024_kpts/1',\n",
+        "'CenterNet Resnet50 V1 FPN 512x512' : 'https://tfhub.dev/tensorflow/centernet/resnet50v1_fpn_512x512/1',\n",
+        "'CenterNet Resnet50 V1 FPN Keypoints 512x512' : 'https://tfhub.dev/tensorflow/centernet/resnet50v1_fpn_512x512_kpts/1',\n",
+        "'CenterNet Resnet101 V1 FPN 512x512' : 'https://tfhub.dev/tensorflow/centernet/resnet101v1_fpn_512x512/1',\n",
+        "'CenterNet Resnet50 V2 512x512' : 'https://tfhub.dev/tensorflow/centernet/resnet50v2_512x512/1',\n",
+        "'CenterNet Resnet50 V2 Keypoints 512x512' : 'https://tfhub.dev/tensorflow/centernet/resnet50v2_512x512_kpts/1',\n",
+        "'EfficientDet D0 512x512' : 'https://tfhub.dev/tensorflow/efficientdet/d0/1',\n",
+        "'EfficientDet D1 640x640' : 'https://tfhub.dev/tensorflow/efficientdet/d1/1',\n",
+        "'EfficientDet D2 768x768' : 'https://tfhub.dev/tensorflow/efficientdet/d2/1',\n",
+        "'EfficientDet D3 896x896' : 'https://tfhub.dev/tensorflow/efficientdet/d3/1',\n",
+        "'EfficientDet D4 1024x1024' : 'https://tfhub.dev/tensorflow/efficientdet/d4/1',\n",
+        "'EfficientDet D5 1280x1280' : 'https://tfhub.dev/tensorflow/efficientdet/d5/1',\n",
+        "'EfficientDet D6 1280x1280' : 'https://tfhub.dev/tensorflow/efficientdet/d6/1',\n",
+        "'EfficientDet D7 1536x1536' : 'https://tfhub.dev/tensorflow/efficientdet/d7/1',\n",
+        "'SSD MobileNet v2 320x320' : 'https://tfhub.dev/tensorflow/ssd_mobilenet_v2/2',\n",
+        "'SSD MobileNet V1 FPN 640x640' : 'https://tfhub.dev/tensorflow/ssd_mobilenet_v1/fpn_640x640/1',\n",
+        "'SSD MobileNet V2 FPNLite 320x320' : 'https://tfhub.dev/tensorflow/ssd_mobilenet_v2/fpnlite_320x320/1',\n",
+        "'SSD MobileNet V2 FPNLite 640x640' : 'https://tfhub.dev/tensorflow/ssd_mobilenet_v2/fpnlite_640x640/1',\n",
+        "'SSD ResNet50 V1 FPN 640x640 (RetinaNet50)' : 'https://tfhub.dev/tensorflow/retinanet/resnet50_v1_fpn_640x640/1',\n",
+        "'SSD ResNet50 V1 FPN 1024x1024 (RetinaNet50)' : 'https://tfhub.dev/tensorflow/retinanet/resnet50_v1_fpn_1024x1024/1',\n",
+        "'SSD ResNet101 V1 FPN 640x640 (RetinaNet101)' : 'https://tfhub.dev/tensorflow/retinanet/resnet101_v1_fpn_640x640/1',\n",
+        "'SSD ResNet101 V1 FPN 1024x1024 (RetinaNet101)' : 'https://tfhub.dev/tensorflow/retinanet/resnet101_v1_fpn_1024x1024/1',\n",
+        "'SSD ResNet152 V1 FPN 640x640 (RetinaNet152)' : 'https://tfhub.dev/tensorflow/retinanet/resnet152_v1_fpn_640x640/1',\n",
+        "'SSD ResNet152 V1 FPN 1024x1024 (RetinaNet152)' : 'https://tfhub.dev/tensorflow/retinanet/resnet152_v1_fpn_1024x1024/1',\n",
+        "'Faster R-CNN ResNet50 V1 640x640' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_640x640/1',\n",
+        "'Faster R-CNN ResNet50 V1 1024x1024' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_1024x1024/1',\n",
+        "'Faster R-CNN ResNet50 V1 800x1333' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet50_v1_800x1333/1',\n",
+        "'Faster R-CNN ResNet101 V1 640x640' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet101_v1_640x640/1',\n",
+        "'Faster R-CNN ResNet101 V1 1024x1024' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet101_v1_1024x1024/1',\n",
+        "'Faster R-CNN ResNet101 V1 800x1333' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet101_v1_800x1333/1',\n",
+        "'Faster R-CNN ResNet152 V1 640x640' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet152_v1_640x640/1',\n",
+        "'Faster R-CNN ResNet152 V1 1024x1024' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet152_v1_1024x1024/1',\n",
+        "'Faster R-CNN ResNet152 V1 800x1333' : 'https://tfhub.dev/tensorflow/faster_rcnn/resnet152_v1_800x1333/1',\n",
+        "'Faster R-CNN Inception ResNet V2 640x640' : 'https://tfhub.dev/tensorflow/faster_rcnn/inception_resnet_v2_640x640/1',\n",
+        "'Faster R-CNN Inception ResNet V2 1024x1024' : 'https://tfhub.dev/tensorflow/faster_rcnn/inception_resnet_v2_1024x1024/1',\n",
+        "'Mask R-CNN Inception ResNet V2 1024x1024' : 'https://tfhub.dev/tensorflow/mask_rcnn/inception_resnet_v2_1024x1024/1'\n",
+        "}\n",
+        "\n",
+        "IMAGES_FOR_TEST = {\n",
+        "  'Beach' : 'models/research/object_detection/test_images/image2.jpg',\n",
+        "  'Dogs' : 'models/research/object_detection/test_images/image1.jpg',\n",
+        "  # By Heiko Gorski, Source: https://commons.wikimedia.org/wiki/File:Naxos_Taverna.jpg\n",
+        "  'Naxos Taverna' : 'https://upload.wikimedia.org/wikipedia/commons/6/60/Naxos_Taverna.jpg',\n",
+        "  # Source: https://commons.wikimedia.org/wiki/File:The_Coleoptera_of_the_British_islands_(Plate_125)_(8592917784).jpg\n",
+        "  'Beatles' : 'https://upload.wikimedia.org/wikipedia/commons/1/1b/The_Coleoptera_of_the_British_islands_%28Plate_125%29_%288592917784%29.jpg',\n",
+        "  # By Américo Toledano, Source: https://commons.wikimedia.org/wiki/File:Biblioteca_Maim%C3%B3nides,_Campus_Universitario_de_Rabanales_007.jpg\n",
+        "  'Phones' : 'https://upload.wikimedia.org/wikipedia/commons/thumb/0/0d/Biblioteca_Maim%C3%B3nides%2C_Campus_Universitario_de_Rabanales_007.jpg/1024px-Biblioteca_Maim%C3%B3nides%2C_Campus_Universitario_de_Rabanales_007.jpg',\n",
+        "  # Source: https://commons.wikimedia.org/wiki/File:The_smaller_British_birds_(8053836633).jpg\n",
+        "  'Birds' : 'https://upload.wikimedia.org/wikipedia/commons/0/09/The_smaller_British_birds_%288053836633%29.jpg',\n",
+        "}\n",
+        "\n",
+        "COCO17_HUMAN_POSE_KEYPOINTS = [(0, 1),\n",
+        " (0, 2),\n",
+        " (1, 3),\n",
+        " (2, 4),\n",
+        " (0, 5),\n",
+        " (0, 6),\n",
+        " (5, 7),\n",
+        " (7, 9),\n",
+        " (6, 8),\n",
+        " (8, 10),\n",
+        " (5, 6),\n",
+        " (5, 11),\n",
+        " (6, 12),\n",
+        " (11, 12),\n",
+        " (11, 13),\n",
+        " (13, 15),\n",
+        " (12, 14),\n",
+        " (14, 16)]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "14bNk1gzh0TN"
+      },
+      "source": [
+        "## Visualization tools\n",
+        "\n",
+        "To visualize the images with the proper detected boxes, keypoints and segmentation, we will use the TensorFlow Object Detection API. To install it we will clone the repo."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oi28cqGGFWnY"
+      },
+      "outputs": [],
+      "source": [
+        "# Clone the tensorflow models repository\n",
+        "!git clone --depth 1 https://github.com/tensorflow/models"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yX3pb_pXDjYA"
+      },
+      "source": [
+        "Installing the Object Detection API"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NwdsBdGhFanc"
+      },
+      "outputs": [],
+      "source": [
+        "%%bash\n",
+        "sudo apt install -y protobuf-compiler\n",
+        "cd models/research/\n",
+        "protoc object_detection/protos/*.proto --python_out=.\n",
+        "cp object_detection/packages/tf2/setup.py .\n",
+        "python -m pip install .\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3yDNgIx-kV7X"
+      },
+      "source": [
+        "Now we can import the dependencies we will need later"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2JCeQU3fkayh"
+      },
+      "outputs": [],
+      "source": [
+        "from object_detection.utils import label_map_util\n",
+        "from object_detection.utils import visualization_utils as viz_utils\n",
+        "from object_detection.utils import ops as utils_ops\n",
+        "\n",
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NKtD0IeclbL5"
+      },
+      "source": [
+        "### Load label map data (for plotting).\n",
+        "\n",
+        "Label maps correspond index numbers to category names, so that when our convolution network predicts `5`, we know that this corresponds to `airplane`.  Here we use internal utility functions, but anything that returns a dictionary mapping integers to appropriate string labels would be fine.\n",
+        "\n",
+        "We are going, for simplicity, to load from the repository that we loaded the Object Detection API code"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5mucYUS6exUJ"
+      },
+      "outputs": [],
+      "source": [
+        "PATH_TO_LABELS = './models/research/object_detection/data/mscoco_label_map.pbtxt'\n",
+        "category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS, use_display_name=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6917xnUSlp9x"
+      },
+      "source": [
+        "## Build a detection model and load pre-trained model weights\n",
+        "\n",
+        "Here we will choose which Object Detection model we will use.\n",
+        "Select the architecture and it will be loaded automatically.\n",
+        "If you want to change the model to try other architectures later, just change the next cell and execute following ones.\n",
+        "\n",
+        "**Tip:** if you want to read more details about the selected model, you can follow the link (model handle) and read additional documentation on TF Hub. After you select a model, we will print the handle to make it easier."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HtwrSqvakTNn"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Model Selection { display-mode: \"form\", run: \"auto\" }\n",
+        "model_display_name = 'CenterNet HourGlass104 Keypoints 512x512' # @param ['CenterNet HourGlass104 512x512','CenterNet HourGlass104 Keypoints 512x512','CenterNet HourGlass104 1024x1024','CenterNet HourGlass104 Keypoints 1024x1024','CenterNet Resnet50 V1 FPN 512x512','CenterNet Resnet50 V1 FPN Keypoints 512x512','CenterNet Resnet101 V1 FPN 512x512','CenterNet Resnet50 V2 512x512','CenterNet Resnet50 V2 Keypoints 512x512','EfficientDet D0 512x512','EfficientDet D1 640x640','EfficientDet D2 768x768','EfficientDet D3 896x896','EfficientDet D4 1024x1024','EfficientDet D5 1280x1280','EfficientDet D6 1280x1280','EfficientDet D7 1536x1536','SSD MobileNet v2 320x320','SSD MobileNet V1 FPN 640x640','SSD MobileNet V2 FPNLite 320x320','SSD MobileNet V2 FPNLite 640x640','SSD ResNet50 V1 FPN 640x640 (RetinaNet50)','SSD ResNet50 V1 FPN 1024x1024 (RetinaNet50)','SSD ResNet101 V1 FPN 640x640 (RetinaNet101)','SSD ResNet101 V1 FPN 1024x1024 (RetinaNet101)','SSD ResNet152 V1 FPN 640x640 (RetinaNet152)','SSD ResNet152 V1 FPN 1024x1024 (RetinaNet152)','Faster R-CNN ResNet50 V1 640x640','Faster R-CNN ResNet50 V1 1024x1024','Faster R-CNN ResNet50 V1 800x1333','Faster R-CNN ResNet101 V1 640x640','Faster R-CNN ResNet101 V1 1024x1024','Faster R-CNN ResNet101 V1 800x1333','Faster R-CNN ResNet152 V1 640x640','Faster R-CNN ResNet152 V1 1024x1024','Faster R-CNN ResNet152 V1 800x1333','Faster R-CNN Inception ResNet V2 640x640','Faster R-CNN Inception ResNet V2 1024x1024','Mask R-CNN Inception ResNet V2 1024x1024']\n",
+        "model_handle = ALL_MODELS[model_display_name]\n",
+        "\n",
+        "print('Selected model:'+ model_display_name)\n",
+        "print('Model Handle at TensorFlow Hub: {}'.format(model_handle))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "muhUt-wWL582"
+      },
+      "source": [
+        "## Loading the selected model from TensorFlow Hub\n",
+        "\n",
+        "Here we just need the model handle that was selected and use the Tensorflow Hub library to load it to memory.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rBuD07fLlcEO"
+      },
+      "outputs": [],
+      "source": [
+        "print('loading model...')\n",
+        "hub_model = hub.load(model_handle)\n",
+        "print('model loaded!')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GIawRDKPPnd4"
+      },
+      "source": [
+        "## Loading an image\n",
+        "\n",
+        "Let's try the model on a simple image. To help with this, we provide a list of test images.\n",
+        "\n",
+        "Here are some simple things to try out if you are curious:\n",
+        "* Try running inference on your own images, just upload them to colab and load the same way it's done in the cell below.\n",
+        "* Modify some of the input images and see if detection still works.  Some simple things to try out here include flipping the image horizontally, or converting to grayscale (note that we still expect the input image to have 3 channels).\n",
+        "\n",
+        "**Be careful:** when using images with an alpha channel, the model expect 3 channels images and the alpha will count as a 4th.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hX-AWUQ1wIEr"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Image Selection (don't forget to execute the cell!) { display-mode: \"form\"}\n",
+        "selected_image = 'Beach' # @param ['Beach', 'Dogs', 'Naxos Taverna', 'Beatles', 'Phones', 'Birds']\n",
+        "flip_image_horizontally = False #@param {type:\"boolean\"}\n",
+        "convert_image_to_grayscale = False #@param {type:\"boolean\"}\n",
+        "\n",
+        "image_path = IMAGES_FOR_TEST[selected_image]\n",
+        "image_np = load_image_into_numpy_array(image_path)\n",
+        "\n",
+        "# Flip horizontally\n",
+        "if(flip_image_horizontally):\n",
+        "  image_np[0] = np.fliplr(image_np[0]).copy()\n",
+        "\n",
+        "# Convert image to grayscale\n",
+        "if(convert_image_to_grayscale):\n",
+        "  image_np[0] = np.tile(\n",
+        "    np.mean(image_np[0], 2, keepdims=True), (1, 1, 3)).astype(np.uint8)\n",
+        "\n",
+        "plt.figure(figsize=(24,32))\n",
+        "plt.imshow(image_np[0])\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "FTHsFjR6HNwb"
+      },
+      "source": [
+        "## Doing the inference\n",
+        "\n",
+        "To do the inference we just need to call our TF Hub loaded model.\n",
+        "\n",
+        "Things you can try:\n",
+        "* Print out `result['detection_boxes']` and try to match the box locations to the boxes in the image.  Notice that coordinates are given in normalized form (i.e., in the interval [0, 1]).\n",
+        "* inspect other output keys present in the result. A full documentation can be seen on the models documentation page (pointing your browser to the model handle printed earlier)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Gb_siXKcnnGC"
+      },
+      "outputs": [],
+      "source": [
+        "# running inference\n",
+        "results = hub_model(image_np)\n",
+        "\n",
+        "# different object detection models have additional results\n",
+        "# all of them are explained in the documentation\n",
+        "result = {key:value.numpy() for key,value in results.items()}\n",
+        "print(result.keys())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IZ5VYaBoeeFM"
+      },
+      "source": [
+        "## Visualizing the results\n",
+        "\n",
+        "Here is where we will need the TensorFlow Object Detection API to show the squares from the inference step (and the keypoints when available).\n",
+        "\n",
+        "the full documentation of this method can be seen [here](https://github.com/tensorflow/models/blob/master/research/object_detection/utils/visualization_utils.py)\n",
+        "\n",
+        "Here you can, for example, set `min_score_thresh` to other values (between 0 and 1) to allow more detections in or to filter out more detections."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2O7rV8g9s8Bz"
+      },
+      "outputs": [],
+      "source": [
+        "label_id_offset = 0\n",
+        "image_np_with_detections = image_np.copy()\n",
+        "\n",
+        "# Use keypoints if available in detections\n",
+        "keypoints, keypoint_scores = None, None\n",
+        "if 'detection_keypoints' in result:\n",
+        "  keypoints = result['detection_keypoints'][0]\n",
+        "  keypoint_scores = result['detection_keypoint_scores'][0]\n",
+        "\n",
+        "viz_utils.visualize_boxes_and_labels_on_image_array(\n",
+        "      image_np_with_detections[0],\n",
+        "      result['detection_boxes'][0],\n",
+        "      (result['detection_classes'][0] + label_id_offset).astype(int),\n",
+        "      result['detection_scores'][0],\n",
+        "      category_index,\n",
+        "      use_normalized_coordinates=True,\n",
+        "      max_boxes_to_draw=200,\n",
+        "      min_score_thresh=.30,\n",
+        "      agnostic_mode=False,\n",
+        "      keypoints=keypoints,\n",
+        "      keypoint_scores=keypoint_scores,\n",
+        "      keypoint_edges=COCO17_HUMAN_POSE_KEYPOINTS)\n",
+        "\n",
+        "plt.figure(figsize=(24,32))\n",
+        "plt.imshow(image_np_with_detections[0])\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Qaw6Xi08NpEP"
+      },
+      "source": [
+        "## [Optional]\n",
+        "\n",
+        "Among the available object detection models there's Mask R-CNN and the output of this model allows instance segmentation.\n",
+        "\n",
+        "To visualize it we will use the same method we did before but adding an additional parameter: `instance_masks=output_dict.get('detection_masks_reframed', None)`\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zl3qdtR1OvM_"
+      },
+      "outputs": [],
+      "source": [
+        "# Handle models with masks:\n",
+        "image_np_with_mask = image_np.copy()\n",
+        "\n",
+        "if 'detection_masks' in result:\n",
+        "  # we need to convert np.arrays to tensors\n",
+        "  detection_masks = tf.convert_to_tensor(result['detection_masks'][0])\n",
+        "  detection_boxes = tf.convert_to_tensor(result['detection_boxes'][0])\n",
+        "\n",
+        "  # Reframe the bbox mask to the image size.\n",
+        "  detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(\n",
+        "            detection_masks, detection_boxes,\n",
+        "              image_np.shape[1], image_np.shape[2])\n",
+        "  detection_masks_reframed = tf.cast(detection_masks_reframed > 0.5,\n",
+        "                                      tf.uint8)\n",
+        "  result['detection_masks_reframed'] = detection_masks_reframed.numpy()\n",
+        "\n",
+        "viz_utils.visualize_boxes_and_labels_on_image_array(\n",
+        "      image_np_with_mask[0],\n",
+        "      result['detection_boxes'][0],\n",
+        "      (result['detection_classes'][0] + label_id_offset).astype(int),\n",
+        "      result['detection_scores'][0],\n",
+        "      category_index,\n",
+        "      use_normalized_coordinates=True,\n",
+        "      max_boxes_to_draw=200,\n",
+        "      min_score_thresh=.30,\n",
+        "      agnostic_mode=False,\n",
+        "      instance_masks=result.get('detection_masks_reframed', None),\n",
+        "      line_thickness=8)\n",
+        "\n",
+        "plt.figure(figsize=(24,32))\n",
+        "plt.imshow(image_np_with_mask[0])\n",
+        "plt.show()"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tf2_object_detection.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf2_semantic_approximate_nearest_neighbors.ipynb b/site/en/hub/tutorials/tf2_semantic_approximate_nearest_neighbors.ipynb
new file mode 100644
index 00000000000..786065ff5a5
--- /dev/null
+++ b/site/en/hub/tutorials/tf2_semantic_approximate_nearest_neighbors.ipynb
@@ -0,0 +1,790 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ACbjNjyO4f_8"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MCM50vaM4jiK"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9qOVy-_vmuUP"
+      },
+      "source": [
+        "# Semantic Search with Approximate Nearest Neighbors and Text Embeddings\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "3T4d77AJaKte"
+      },
+      "source": [
+        "This tutorial illustrates how to generate embeddings from a [TensorFlow Hub](https://tfhub.dev) (TF-Hub) model given input data, and build an approximate nearest neighbours (ANN) index using the extracted embeddings. The index can then be used for real-time similarity matching and retrieval.\n",
+        "\n",
+        "When dealing with a large corpus of data, it's not efficient to perform exact matching by scanning the whole repository to find the most similar items to a given query in real-time. Thus, we use an approximate similarity matching algorithm which allows us to trade off a little bit of accuracy in finding exact nearest neighbor matches for a significant boost in speed.\n",
+        "\n",
+        "In this tutorial, we show an example of real-time text search over a corpus of news headlines to find the headlines that are most similar to a query. Unlike keyword search, this captures the semantic similarity encoded in the text embedding.\n",
+        "\n",
+        "The steps of this tutorial are:\n",
+        "1. Download sample data.\n",
+        "2. Generate embeddings for the data using a TF-Hub model\n",
+        "3. Build an ANN index for the embeddings\n",
+        "4. Use the index for similarity matching\n",
+        "\n",
+        "We use [Apache Beam](https://beam.apache.org/documentation/programming-guide/) to generate the embeddings from the TF-Hub model. We also use Spotify's [ANNOY](https://github.com/spotify/annoy) library to build the approximate nearest neighbor index."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nM17v_mEVSnd"
+      },
+      "source": [
+        "### More models\n",
+        "For models that have the same architecture but were trained on a different language, refer to [this](https://tfhub.dev/google/collections/nnlm/1) collection. [Here](https://tfhub.dev/s?module-type=text-embedding) you can find all text embeddings that are currently hosted on [tfhub.dev](https://tfhub.dev/). "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q0jr0QK9qO5P"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "whMRj9qeqed4"
+      },
+      "source": [
+        "Install the required libraries."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qmXkLPoaqS--"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install apache_beam\n",
+        "!pip install 'scikit_learn~=0.23.0'  # For gaussian_random_matrix.\n",
+        "!pip install annoy"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A-vBZiCCqld0"
+      },
+      "source": [
+        "Import the required libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6NTYbdWcseuK"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "import pickle\n",
+        "from collections import namedtuple\n",
+        "from datetime import datetime\n",
+        "import numpy as np\n",
+        "import apache_beam as beam\n",
+        "from apache_beam.transforms import util\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import annoy\n",
+        "from sklearn.random_projection import gaussian_random_matrix"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tx0SZa6-7b-f"
+      },
+      "outputs": [],
+      "source": [
+        "print('TF version: {}'.format(tf.__version__))\n",
+        "print('TF-Hub version: {}'.format(hub.__version__))\n",
+        "print('Apache Beam version: {}'.format(beam.__version__))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "P6Imq876rLWx"
+      },
+      "source": [
+        "## 1. Download Sample Data\n",
+        "\n",
+        "[A Million News Headlines](https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/SYBGZL#) dataset contains news headlines published over a period of 15 years sourced from the reputable Australian Broadcasting Corp. (ABC). This news dataset has a summarised historical record of noteworthy events in the globe from early-2003 to end-2017 with a more granular focus on Australia. \n",
+        "\n",
+        "**Format**: Tab-separated two-column data: 1) publication date and 2) headline text. We are only interested in the headline text.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OpF57n8e5C9D"
+      },
+      "outputs": [],
+      "source": [
+        "!wget 'https://dataverse.harvard.edu/api/access/datafile/3450625?format=tab&gbrecs=true' -O raw.tsv\n",
+        "!wc -l raw.tsv\n",
+        "!head raw.tsv"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Reeoc9z0zTxJ"
+      },
+      "source": [
+        "For simplicity, we only keep the headline text and remove the publication date"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "INPWa4upv_yJ"
+      },
+      "outputs": [],
+      "source": [
+        "!rm -r corpus\n",
+        "!mkdir corpus\n",
+        "\n",
+        "with open('corpus/text.txt', 'w') as out_file:\n",
+        "  with open('raw.tsv', 'r') as in_file:\n",
+        "    for line in in_file:\n",
+        "      headline = line.split('\\t')[1].strip().strip('\"')\n",
+        "      out_file.write(headline+\"\\n\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5-oedX40z6o2"
+      },
+      "outputs": [],
+      "source": [
+        "!tail corpus/text.txt"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2AngMtH50jNb"
+      },
+      "source": [
+        "## 2. Generate Embeddings for the Data.\n",
+        "\n",
+        "In this tutorial, we use the [Neural Network Language Model (NNLM)](https://tfhub.dev/google/nnlm-en-dim128/2) to generate embeddings for the headline data. The sentence embeddings can then be easily used to compute sentence level meaning similarity. We run the embedding generation process using Apache Beam."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "F_DvXnDB1pEX"
+      },
+      "source": [
+        "### Embedding extraction method"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yL7OEY1E0A35"
+      },
+      "outputs": [],
+      "source": [
+        "embed_fn = None\n",
+        "\n",
+        "def generate_embeddings(text, model_url, random_projection_matrix=None):\n",
+        "  # Beam will run this function in different processes that need to\n",
+        "  # import hub and load embed_fn (if not previously loaded)\n",
+        "  global embed_fn\n",
+        "  if embed_fn is None:\n",
+        "    embed_fn = hub.load(model_url)\n",
+        "  embedding = embed_fn(text).numpy()\n",
+        "  if random_projection_matrix is not None:\n",
+        "    embedding = embedding.dot(random_projection_matrix)\n",
+        "  return text, embedding\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "g6pXBVxsVUbm"
+      },
+      "source": [
+        "### Convert to tf.Example method"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JMjqjWZNVVzd"
+      },
+      "outputs": [],
+      "source": [
+        "def to_tf_example(entries):\n",
+        "  examples = []\n",
+        "\n",
+        "  text_list, embedding_list = entries\n",
+        "  for i in range(len(text_list)):\n",
+        "    text = text_list[i]\n",
+        "    embedding = embedding_list[i]\n",
+        "\n",
+        "    features = {\n",
+        "        'text': tf.train.Feature(\n",
+        "            bytes_list=tf.train.BytesList(value=[text.encode('utf-8')])),\n",
+        "        'embedding': tf.train.Feature(\n",
+        "            float_list=tf.train.FloatList(value=embedding.tolist()))\n",
+        "    }\n",
+        "  \n",
+        "    example = tf.train.Example(\n",
+        "        features=tf.train.Features(\n",
+        "            feature=features)).SerializeToString(deterministic=True)\n",
+        "  \n",
+        "    examples.append(example)\n",
+        "  \n",
+        "  return examples"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gDiV4uQCVYGH"
+      },
+      "source": [
+        "### Beam pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jCGUIB172m2G"
+      },
+      "outputs": [],
+      "source": [
+        "def run_hub2emb(args):\n",
+        "  '''Runs the embedding generation pipeline'''\n",
+        "\n",
+        "  options = beam.options.pipeline_options.PipelineOptions(**args)\n",
+        "  args = namedtuple(\"options\", args.keys())(*args.values())\n",
+        "\n",
+        "  with beam.Pipeline(args.runner, options=options) as pipeline:\n",
+        "    (\n",
+        "        pipeline\n",
+        "        | 'Read sentences from files' >> beam.io.ReadFromText(\n",
+        "            file_pattern=args.data_dir)\n",
+        "        | 'Batch elements' >> util.BatchElements(\n",
+        "            min_batch_size=args.batch_size, max_batch_size=args.batch_size)\n",
+        "        | 'Generate embeddings' >> beam.Map(\n",
+        "            generate_embeddings, args.model_url, args.random_projection_matrix)\n",
+        "        | 'Encode to tf example' >> beam.FlatMap(to_tf_example)\n",
+        "        | 'Write to TFRecords files' >> beam.io.WriteToTFRecord(\n",
+        "            file_path_prefix='{}/emb'.format(args.output_dir),\n",
+        "            file_name_suffix='.tfrecords')\n",
+        "    )"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nlbQdiYNVvne"
+      },
+      "source": [
+        "### Generating Random Projection Weight Matrix\n",
+        "\n",
+        "[Random projection](https://en.wikipedia.org/wiki/Random_projection) is a simple, yet powerful technique used to reduce the dimensionality of a set of points which lie in Euclidean space. For a theoretical background, see the [Johnson-Lindenstrauss lemma](https://en.wikipedia.org/wiki/Johnson%E2%80%93Lindenstrauss_lemma).\n",
+        "\n",
+        "Reducing the dimensionality of the embeddings with random projection means less time needed to build and query the ANN index.\n",
+        "\n",
+        "In this tutorial we use [Gaussian Random Projection](https://en.wikipedia.org/wiki/Random_projection#Gaussian_random_projection) from the [Scikit-learn](https://scikit-learn.org/stable/modules/random_projection.html#gaussian-random-projection) library."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1yw1xgtNVv52"
+      },
+      "outputs": [],
+      "source": [
+        "def generate_random_projection_weights(original_dim, projected_dim):\n",
+        "  random_projection_matrix = None\n",
+        "  random_projection_matrix = gaussian_random_matrix(\n",
+        "      n_components=projected_dim, n_features=original_dim).T\n",
+        "  print(\"A Gaussian random weight matrix was creates with shape of {}\".format(random_projection_matrix.shape))\n",
+        "  print('Storing random projection matrix to disk...')\n",
+        "  with open('random_projection_matrix', 'wb') as handle:\n",
+        "    pickle.dump(random_projection_matrix, \n",
+        "                handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
+        "        \n",
+        "  return random_projection_matrix"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aJZUfT3NE7kj"
+      },
+      "source": [
+        "### Set parameters\n",
+        "If you want to build an index using the original embedding space without random projection, set the `projected_dim` parameter to `None`. Note that this will slow down the indexing step for high-dimensional embeddings."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "77-Cow7uE74T"
+      },
+      "outputs": [],
+      "source": [
+        "model_url = 'https://tfhub.dev/google/nnlm-en-dim128/2' #@param {type:\"string\"}\n",
+        "projected_dim = 64  #@param {type:\"number\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "On-MbzD922kb"
+      },
+      "source": [
+        "### Run pipeline"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y3I1Wv4i21yY"
+      },
+      "outputs": [],
+      "source": [
+        "import tempfile\n",
+        "\n",
+        "output_dir = tempfile.mkdtemp()\n",
+        "original_dim = hub.load(model_url)(['']).shape[1]\n",
+        "random_projection_matrix = None\n",
+        "\n",
+        "if projected_dim:\n",
+        "  random_projection_matrix = generate_random_projection_weights(\n",
+        "      original_dim, projected_dim)\n",
+        "\n",
+        "args = {\n",
+        "    'job_name': 'hub2emb-{}'.format(datetime.utcnow().strftime('%y%m%d-%H%M%S')),\n",
+        "    'runner': 'DirectRunner',\n",
+        "    'batch_size': 1024,\n",
+        "    'data_dir': 'corpus/*.txt',\n",
+        "    'output_dir': output_dir,\n",
+        "    'model_url': model_url,\n",
+        "    'random_projection_matrix': random_projection_matrix,\n",
+        "}\n",
+        "\n",
+        "print(\"Pipeline args are set.\")\n",
+        "args"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iS9obmeP4ZOA"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Running pipeline...\")\n",
+        "%time run_hub2emb(args)\n",
+        "print(\"Pipeline is done.\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "JAwOo7gQWvVd"
+      },
+      "outputs": [],
+      "source": [
+        "!ls {output_dir}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HVnee4e6U90u"
+      },
+      "source": [
+        "Read some of the generated embeddings..."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-K7pGXlXOj1N"
+      },
+      "outputs": [],
+      "source": [
+        "embed_file = os.path.join(output_dir, 'emb-00000-of-00001.tfrecords')\n",
+        "sample = 5\n",
+        "\n",
+        "# Create a description of the features.\n",
+        "feature_description = {\n",
+        "    'text': tf.io.FixedLenFeature([], tf.string),\n",
+        "    'embedding': tf.io.FixedLenFeature([projected_dim], tf.float32)\n",
+        "}\n",
+        "\n",
+        "def _parse_example(example):\n",
+        "  # Parse the input `tf.Example` proto using the dictionary above.\n",
+        "  return tf.io.parse_single_example(example, feature_description)\n",
+        "\n",
+        "dataset = tf.data.TFRecordDataset(embed_file)\n",
+        "for record in dataset.take(sample).map(_parse_example):\n",
+        "  print(\"{}: {}\".format(record['text'].numpy().decode('utf-8'), record['embedding'].numpy()[:10]))\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "agGoaMSgY8wN"
+      },
+      "source": [
+        "## 3. Build the ANN Index for the Embeddings\n",
+        "\n",
+        "[ANNOY](https://github.com/spotify/annoy) (Approximate Nearest Neighbors Oh Yeah) is a C++ library with Python bindings to search for points in space that are close to a given query point. It also creates large read-only file-based data structures that are mapped into memory. It is built and used by [Spotify](https://www.spotify.com) for music recommendations. If you are interested you can play along with other alternatives to ANNOY such as [NGT](https://github.com/yahoojapan/NGT), [FAISS](https://github.com/facebookresearch/faiss), etc. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "UcPDspU3WjgH"
+      },
+      "outputs": [],
+      "source": [
+        "def build_index(embedding_files_pattern, index_filename, vector_length, \n",
+        "    metric='angular', num_trees=100):\n",
+        "  '''Builds an ANNOY index'''\n",
+        "\n",
+        "  annoy_index = annoy.AnnoyIndex(vector_length, metric=metric)\n",
+        "  # Mapping between the item and its identifier in the index\n",
+        "  mapping = {}\n",
+        "\n",
+        "  embed_files = tf.io.gfile.glob(embedding_files_pattern)\n",
+        "  num_files = len(embed_files)\n",
+        "  print('Found {} embedding file(s).'.format(num_files))\n",
+        "\n",
+        "  item_counter = 0\n",
+        "  for i, embed_file in enumerate(embed_files):\n",
+        "    print('Loading embeddings in file {} of {}...'.format(i+1, num_files))\n",
+        "    dataset = tf.data.TFRecordDataset(embed_file)\n",
+        "    for record in dataset.map(_parse_example):\n",
+        "      text = record['text'].numpy().decode(\"utf-8\")\n",
+        "      embedding = record['embedding'].numpy()\n",
+        "      mapping[item_counter] = text\n",
+        "      annoy_index.add_item(item_counter, embedding)\n",
+        "      item_counter += 1\n",
+        "      if item_counter % 100000 == 0:\n",
+        "        print('{} items loaded to the index'.format(item_counter))\n",
+        "\n",
+        "  print('A total of {} items added to the index'.format(item_counter))\n",
+        "\n",
+        "  print('Building the index with {} trees...'.format(num_trees))\n",
+        "  annoy_index.build(n_trees=num_trees)\n",
+        "  print('Index is successfully built.')\n",
+        "  \n",
+        "  print('Saving index to disk...')\n",
+        "  annoy_index.save(index_filename)\n",
+        "  print('Index is saved to disk.')\n",
+        "  print(\"Index file size: {} GB\".format(\n",
+        "    round(os.path.getsize(index_filename) / float(1024 ** 3), 2)))\n",
+        "  annoy_index.unload()\n",
+        "\n",
+        "  print('Saving mapping to disk...')\n",
+        "  with open(index_filename + '.mapping', 'wb') as handle:\n",
+        "    pickle.dump(mapping, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
+        "  print('Mapping is saved to disk.')\n",
+        "  print(\"Mapping file size: {} MB\".format(\n",
+        "    round(os.path.getsize(index_filename + '.mapping') / float(1024 ** 2), 2)))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "AgyOQhUq6FNE"
+      },
+      "outputs": [],
+      "source": [
+        "embedding_files = \"{}/emb-*.tfrecords\".format(output_dir)\n",
+        "embedding_dimension = projected_dim\n",
+        "index_filename = \"index\"\n",
+        "\n",
+        "!rm {index_filename}\n",
+        "!rm {index_filename}.mapping\n",
+        "\n",
+        "%time build_index(embedding_files, index_filename, embedding_dimension)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Ic31Tm5cgAd5"
+      },
+      "outputs": [],
+      "source": [
+        "!ls"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "maGxDl8ufP-p"
+      },
+      "source": [
+        "## 4. Use the Index for Similarity Matching\n",
+        "Now we can use the ANN index to find news headlines that are semantically close to an input query."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_dIs8W78fYPp"
+      },
+      "source": [
+        "### Load the index and the mapping files"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jlTTrbQHayvb"
+      },
+      "outputs": [],
+      "source": [
+        "index = annoy.AnnoyIndex(embedding_dimension)\n",
+        "index.load(index_filename, prefault=True)\n",
+        "print('Annoy index is loaded.')\n",
+        "with open(index_filename + '.mapping', 'rb') as handle:\n",
+        "  mapping = pickle.load(handle)\n",
+        "print('Mapping file is loaded.')\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "y6liFMSUh08J"
+      },
+      "source": [
+        "### Similarity matching method"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "mUxjTag8hc16"
+      },
+      "outputs": [],
+      "source": [
+        "def find_similar_items(embedding, num_matches=5):\n",
+        "  '''Finds similar items to a given embedding in the ANN index'''\n",
+        "  ids = index.get_nns_by_vector(\n",
+        "  embedding, num_matches, search_k=-1, include_distances=False)\n",
+        "  items = [mapping[i] for i in ids]\n",
+        "  return items"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hjerNpmZja0A"
+      },
+      "source": [
+        "### Extract embedding from a given query"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a0IIXzfBjZ19"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the TF-Hub model\n",
+        "print(\"Loading the TF-Hub model...\")\n",
+        "%time embed_fn = hub.load(model_url)\n",
+        "print(\"TF-Hub model is loaded.\")\n",
+        "\n",
+        "random_projection_matrix = None\n",
+        "if os.path.exists('random_projection_matrix'):\n",
+        "  print(\"Loading random projection matrix...\")\n",
+        "  with open('random_projection_matrix', 'rb') as handle:\n",
+        "    random_projection_matrix = pickle.load(handle)\n",
+        "  print('random projection matrix is loaded.')\n",
+        "\n",
+        "def extract_embeddings(query):\n",
+        "  '''Generates the embedding for the query'''\n",
+        "  query_embedding =  embed_fn([query])[0].numpy()\n",
+        "  if random_projection_matrix is not None:\n",
+        "    query_embedding = query_embedding.dot(random_projection_matrix)\n",
+        "  return query_embedding\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kCoCNROujEIO"
+      },
+      "outputs": [],
+      "source": [
+        "extract_embeddings(\"Hello Machine Learning!\")[:10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "koINo8Du--8C"
+      },
+      "source": [
+        "### Enter a query to find the most similar items"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "wC0uLjvfk5nB"
+      },
+      "outputs": [],
+      "source": [
+        "#@title { run: \"auto\" }\n",
+        "query = \"confronting global challenges\" #@param {type:\"string\"}\n",
+        "\n",
+        "print(\"Generating embedding for the query...\")\n",
+        "%time query_embedding = extract_embeddings(query)\n",
+        "\n",
+        "print(\"\")\n",
+        "print(\"Finding relevant items in the index...\")\n",
+        "%time items = find_similar_items(query_embedding, 10)\n",
+        "\n",
+        "print(\"\")\n",
+        "print(\"Results:\")\n",
+        "print(\"=========\")\n",
+        "for item in items:\n",
+        "  print(item)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TkRSqs77tDuX"
+      },
+      "source": [
+        "## Want to learn more?\n",
+        "\n",
+        "You can learn more about TensorFlow at [tensorflow.org](https://www.tensorflow.org/) and see the TF-Hub API documentation at [tensorflow.org/hub](https://www.tensorflow.org/hub/). Find available TensorFlow Hub models at [tfhub.dev](https://tfhub.dev/) including more text embedding models and image feature vector models.\n",
+        "\n",
+        "Also check out the [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/) which is Google's fast-paced, practical introduction to machine learning."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "ACbjNjyO4f_8",
+        "g6pXBVxsVUbm"
+      ],
+      "name": "tf2_semantic_approximate_nearest_neighbors.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf2_text_classification.ipynb b/site/en/hub/tutorials/tf2_text_classification.ipynb
new file mode 100644
index 00000000000..e2dae15bde0
--- /dev/null
+++ b/site/en/hub/tutorials/tf2_text_classification.ipynb
@@ -0,0 +1,571 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ic4_occAAiAT"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "ioaprt5q5US7"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "yCl0eTNH5RS3"
+      },
+      "outputs": [],
+      "source": [
+        "#@title MIT License\n",
+        "#\n",
+        "# Copyright (c) 2017 François Chollet                                  # IGNORE_COPYRIGHT: cleared by OSS licensing\n",
+        "#\n",
+        "# Permission is hereby granted, free of charge, to any person obtaining a\n",
+        "# copy of this software and associated documentation files (the \"Software\"),\n",
+        "# to deal in the Software without restriction, including without limitation\n",
+        "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n",
+        "# and/or sell copies of the Software, and to permit persons to whom the\n",
+        "# Software is furnished to do so, subject to the following conditions:\n",
+        "#\n",
+        "# The above copyright notice and this permission notice shall be included in\n",
+        "# all copies or substantial portions of the Software.\n",
+        "#\n",
+        "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
+        "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
+        "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n",
+        "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
+        "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n",
+        "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n",
+        "# DEALINGS IN THE SOFTWARE."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ItXfxkxvosLH"
+      },
+      "source": [
+        "# Text Classification with Movie Reviews"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Eg62Pmz3o83v"
+      },
+      "source": [
+        "This notebook classifies movie reviews as *positive* or *negative* using the text of the review. This is an example of *binary*—or two-class—classification, an important and widely applicable kind of machine learning problem. \n",
+        "\n",
+        "We'll use the [IMDB dataset](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) that contains the text of 50,000 movie reviews from the [Internet Movie Database](https://www.imdb.com/). These are split into 25,000 reviews for training and 25,000 reviews for testing. The training and testing sets are *balanced*, meaning they contain an equal number of positive and negative reviews. \n",
+        "\n",
+        "This notebook uses [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras), a high-level API to build and train models in TensorFlow, and [TensorFlow Hub](https://www.tensorflow.org/hub), a library and platform for transfer learning. For a more advanced text classification tutorial using `tf.keras`, see the [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qrk8NjzhSBh-"
+      },
+      "source": [
+        "### More models\n",
+        "[Here](https://tfhub.dev/s?module-type=text-embedding) you can find more expressive or performant models that you could use to generate the text embedding."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q4DN769E2O_R"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2ew7HTbPpCJH"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import tensorflow_datasets as tfds\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "\n",
+        "print(\"Version: \", tf.__version__)\n",
+        "print(\"Eager mode: \", tf.executing_eagerly())\n",
+        "print(\"Hub version: \", hub.__version__)\n",
+        "print(\"GPU is\", \"available\" if tf.config.list_physical_devices('GPU') else \"NOT AVAILABLE\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iAsKG535pHep"
+      },
+      "source": [
+        "## Download the IMDB dataset\n",
+        "\n",
+        "The IMDB dataset is available on [TensorFlow datasets](https://github.com/tensorflow/datasets). The following code downloads the IMDB dataset to your machine (or the colab runtime):"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zXXx5Oc3pOmN"
+      },
+      "outputs": [],
+      "source": [
+        "train_data, test_data = tfds.load(name=\"imdb_reviews\", split=[\"train\", \"test\"], \n",
+        "                                  batch_size=-1, as_supervised=True)\n",
+        "\n",
+        "train_examples, train_labels = tfds.as_numpy(train_data)\n",
+        "test_examples, test_labels = tfds.as_numpy(test_data)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l50X3GfjpU4r"
+      },
+      "source": [
+        "## Explore the data \n",
+        "\n",
+        "Let's take a moment to understand the format of the data. Each example is a sentence representing the movie review and a corresponding label. The sentence is not preprocessed in any way. The label is an integer value of either 0 or 1, where 0 is a negative review, and 1 is a positive review."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "y8qCnve_-lkO"
+      },
+      "outputs": [],
+      "source": [
+        "print(\"Training entries: {}, test entries: {}\".format(len(train_examples), len(test_examples)))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RnKvHWW4-lkW"
+      },
+      "source": [
+        "Let's print first 10 examples."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QtTS4kpEpjbi"
+      },
+      "outputs": [],
+      "source": [
+        "train_examples[:10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "IFtaCHTdc-GY"
+      },
+      "source": [
+        "Let's also print the first 10 labels."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tvAjVXOWc6Mj"
+      },
+      "outputs": [],
+      "source": [
+        "train_labels[:10]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LLC02j2g-llC"
+      },
+      "source": [
+        "## Build the model\n",
+        "\n",
+        "The neural network is created by stacking layers—this requires three main architectural decisions:\n",
+        "\n",
+        "* How to represent the text?\n",
+        "* How many layers to use in the model?\n",
+        "* How many *hidden units* to use for each layer?\n",
+        "\n",
+        "In this example, the input data consists of sentences. The labels to predict are either 0 or 1.\n",
+        "\n",
+        "One way to represent the text is to convert sentences into embeddings vectors. We can use a pre-trained text embedding as the first layer, which will have two advantages:\n",
+        "*   we don't have to worry about text preprocessing,\n",
+        "*   we can benefit from transfer learning.\n",
+        "\n",
+        "For this example we will use a model from [TensorFlow Hub](https://www.tensorflow.org/hub) called [google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2).\n",
+        "\n",
+        "There are two other models to test for the sake of this tutorial:\n",
+        "* [google/nnlm-en-dim50-with-normalization/2](https://tfhub.dev/google/nnlm-en-dim50-with-normalization/2) - same as [google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2), but with additional text normalization to remove punctuation. This can help to get better coverage of in-vocabulary embeddings for tokens on your input text.\n",
+        "* [google/nnlm-en-dim128-with-normalization/2](https://tfhub.dev/google/nnlm-en-dim128-with-normalization/2) - A larger model with an embedding dimension of 128 instead of the smaller 50."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "In2nDpTLkgKa"
+      },
+      "source": [
+        "Let's first create a Keras layer that uses a TensorFlow Hub model to embed the sentences, and try it out on a couple of input examples. Note that the output shape of the produced embeddings is a expected: `(num_examples, embedding_dimension)`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_NUbzVeYkgcO"
+      },
+      "outputs": [],
+      "source": [
+        "model = \"https://tfhub.dev/google/nnlm-en-dim50/2\"\n",
+        "hub_layer = hub.KerasLayer(model, input_shape=[], dtype=tf.string, trainable=True)\n",
+        "hub_layer(train_examples[:3])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dfSbV6igl1EH"
+      },
+      "source": [
+        "Let's now build the full model:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "xpKOoWgu-llD"
+      },
+      "outputs": [],
+      "source": [
+        "model = tf.keras.Sequential()\n",
+        "model.add(hub_layer)\n",
+        "model.add(tf.keras.layers.Dense(16, activation='relu'))\n",
+        "model.add(tf.keras.layers.Dense(1))\n",
+        "\n",
+        "model.summary()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "6PbKQ6mucuKL"
+      },
+      "source": [
+        "The layers are stacked sequentially to build the classifier:\n",
+        "\n",
+        "1. The first layer is a TensorFlow Hub layer. This layer uses a pre-trained Saved Model to map a sentence into its embedding vector. The model that we are using ([google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2)) splits the sentence into tokens, embeds each token and then combines the embedding. The resulting dimensions are: `(num_examples, embedding_dimension)`.\n",
+        "2. This fixed-length output vector is piped through a fully-connected (`Dense`) layer with 16 hidden units.\n",
+        "3. The last layer is densely connected with a single output node. This outputs logits: the log-odds of the true class, according to the model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0XMwnDOp-llH"
+      },
+      "source": [
+        "### Hidden units\n",
+        "\n",
+        "The above model has two intermediate or \"hidden\" layers, between the input and output. The number of outputs (units, nodes, or neurons) is the dimension of the representational space for the layer. In other words, the amount of freedom the network is allowed when learning an internal representation.\n",
+        "\n",
+        "If a model has more hidden units (a higher-dimensional representation space), and/or more layers, then the network can learn more complex representations. However, it makes the network more computationally expensive and may lead to learning unwanted patterns—patterns that improve performance on training data but not on the test data. This is called *overfitting*, and we'll explore it later."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L4EqVWg4-llM"
+      },
+      "source": [
+        "### Loss function and optimizer\n",
+        "\n",
+        "A model needs a loss function and an optimizer for training. Since this is a binary classification problem and the model outputs a probability (a single-unit layer with a sigmoid activation), we'll use the `binary_crossentropy` loss function. \n",
+        "\n",
+        "This isn't the only choice for a loss function, you could, for instance, choose `mean_squared_error`. But, generally, `binary_crossentropy` is better for dealing with probabilities—it measures the \"distance\" between probability distributions, or in our case, between the ground-truth distribution and the predictions.\n",
+        "\n",
+        "Later, when we are exploring regression problems (say, to predict the price of a house), we will see how to use another loss function called mean squared error.\n",
+        "\n",
+        "Now, configure the model to use an optimizer and a loss function:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Mr0GP-cQ-llN"
+      },
+      "outputs": [],
+      "source": [
+        "model.compile(optimizer='adam',\n",
+        "              loss=tf.losses.BinaryCrossentropy(from_logits=True),\n",
+        "              metrics=[tf.metrics.BinaryAccuracy(threshold=0.0, name='accuracy')])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hCWYwkug-llQ"
+      },
+      "source": [
+        "## Create a validation set\n",
+        "\n",
+        "When training, we want to check the accuracy of the model on data it hasn't seen before. Create a *validation set* by setting apart 10,000 examples from the original training data. (Why not use the testing set now? Our goal is to develop and tune our model using only the training data, then use the test data just once to evaluate our accuracy)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "-NpcXY9--llS"
+      },
+      "outputs": [],
+      "source": [
+        "x_val = train_examples[:10000]\n",
+        "partial_x_train = train_examples[10000:]\n",
+        "\n",
+        "y_val = train_labels[:10000]\n",
+        "partial_y_train = train_labels[10000:]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "35jv_fzP-llU"
+      },
+      "source": [
+        "## Train the model\n",
+        "\n",
+        "Train the model for 40 epochs in mini-batches of 512 samples. This is 40 iterations over all samples in the `x_train` and `y_train` tensors. While training, monitor the model's loss and accuracy on the 10,000 samples from the validation set:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tXSGrjWZ-llW"
+      },
+      "outputs": [],
+      "source": [
+        "history = model.fit(partial_x_train,\n",
+        "                    partial_y_train,\n",
+        "                    epochs=40,\n",
+        "                    batch_size=512,\n",
+        "                    validation_data=(x_val, y_val),\n",
+        "                    verbose=1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9EEGuDVuzb5r"
+      },
+      "source": [
+        "## Evaluate the model\n",
+        "\n",
+        "And let's see how the model performs. Two values will be returned. Loss (a number which represents our error, lower values are better), and accuracy."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zOMKywn4zReN"
+      },
+      "outputs": [],
+      "source": [
+        "results = model.evaluate(test_examples, test_labels)\n",
+        "\n",
+        "print(results)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "z1iEXVTR0Z2t"
+      },
+      "source": [
+        "This fairly naive approach achieves an accuracy of about 87%. With more advanced approaches, the model should get closer to 95%."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5KggXVeL-llZ"
+      },
+      "source": [
+        "## Create a graph of accuracy and loss over time\n",
+        "\n",
+        "`model.fit()` returns a `History` object that contains a dictionary with everything that happened during training:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VcvSXvhp-llb"
+      },
+      "outputs": [],
+      "source": [
+        "history_dict = history.history\n",
+        "history_dict.keys()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nRKsqL40-lle"
+      },
+      "source": [
+        "There are four entries: one for each monitored metric during training and validation. We can use these to plot the training and validation loss for comparison, as well as the training and validation accuracy:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "nGoYf2Js-lle"
+      },
+      "outputs": [],
+      "source": [
+        "acc = history_dict['accuracy']\n",
+        "val_acc = history_dict['val_accuracy']\n",
+        "loss = history_dict['loss']\n",
+        "val_loss = history_dict['val_loss']\n",
+        "\n",
+        "epochs = range(1, len(acc) + 1)\n",
+        "\n",
+        "# \"bo\" is for \"blue dot\"\n",
+        "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
+        "# b is for \"solid blue line\"\n",
+        "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
+        "plt.title('Training and validation loss')\n",
+        "plt.xlabel('Epochs')\n",
+        "plt.ylabel('Loss')\n",
+        "plt.legend()\n",
+        "\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "6hXx-xOv-llh"
+      },
+      "outputs": [],
+      "source": [
+        "plt.clf()   # clear figure\n",
+        "\n",
+        "plt.plot(epochs, acc, 'bo', label='Training acc')\n",
+        "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
+        "plt.title('Training and validation accuracy')\n",
+        "plt.xlabel('Epochs')\n",
+        "plt.ylabel('Accuracy')\n",
+        "plt.legend()\n",
+        "\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oFEmZ5zq-llk"
+      },
+      "source": [
+        "In this plot, the dots represent the training loss and accuracy, and the solid lines are the validation loss and accuracy.\n",
+        "\n",
+        "Notice the training loss *decreases* with each epoch and the training accuracy *increases* with each epoch. This is expected when using a gradient descent optimization—it should minimize the desired quantity on every iteration.\n",
+        "\n",
+        "This isn't the case for the validation loss and accuracy—they seem to peak after about twenty epochs. This is an example of overfitting: the model performs better on the training data than it does on data it has never seen before. After this point, the model over-optimizes and learns representations *specific* to the training data that do not *generalize* to test data.\n",
+        "\n",
+        "For this particular case, we could prevent overfitting by simply stopping the training after twenty or so epochs. Later, you'll see how to do this automatically with a callback."
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tf2_text_classification.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf_hub_delf_module.ipynb b/site/en/hub/tutorials/tf_hub_delf_module.ipynb
new file mode 100644
index 00000000000..b6dec2eae00
--- /dev/null
+++ b/site/en/hub/tutorials/tf_hub_delf_module.ipynb
@@ -0,0 +1,372 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "RUymE2l9GZfO"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "code",
+        "id": "JMyTNwSJGGWg"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0DmDwGPOGfaQ"
+      },
+      "source": [
+        "# How to match images using DELF and TensorFlow Hub\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f3nk38tIKytQ"
+      },
+      "source": [
+        "TensorFlow Hub (TF-Hub) is a platform to share machine learning expertise packaged in reusable resources, notably pre-trained **modules**.\n",
+        "\n",
+        "In this colab, we will use a module that packages the [DELF](https://github.com/tensorflow/models/tree/master/research/delf) neural network and logic for processing images to identify keypoints and their descriptors. The weights of the neural network were trained on images of landmarks as described in [this paper](https://arxiv.org/abs/1612.06321)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q4DN769E2O_R"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lrKaWOB_cuS3"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install scikit-image"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SI7eVflHHxvi"
+      },
+      "outputs": [],
+      "source": [
+        "from absl import logging\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "from PIL import Image, ImageOps\n",
+        "from scipy.spatial import cKDTree\n",
+        "from skimage.feature import plot_matched_features\n",
+        "from skimage.measure import ransac\n",
+        "from skimage.transform import AffineTransform\n",
+        "from six import BytesIO\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "import tensorflow_hub as hub\n",
+        "from six.moves.urllib.request import urlopen"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qquo2HiONiDK"
+      },
+      "source": [
+        "## The data\n",
+        "\n",
+        "In the next cell, we specify the URLs of two images we would like to process with DELF in order to match and compare them."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "l93ye4WFIqIV"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Choose images\n",
+        "images = \"Bridge of Sighs\" #@param [\"Bridge of Sighs\", \"Golden Gate\", \"Acropolis\", \"Eiffel tower\"]\n",
+        "if images == \"Bridge of Sighs\":\n",
+        "  # from: https://commons.wikimedia.org/wiki/File:Bridge_of_Sighs,_Oxford.jpg\n",
+        "  # by: N.H. Fischer\n",
+        "  IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/2/28/Bridge_of_Sighs%2C_Oxford.jpg'\n",
+        "  # from https://commons.wikimedia.org/wiki/File:The_Bridge_of_Sighs_and_Sheldonian_Theatre,_Oxford.jpg\n",
+        "  # by: Matthew Hoser\n",
+        "  IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/c/c3/The_Bridge_of_Sighs_and_Sheldonian_Theatre%2C_Oxford.jpg'\n",
+        "elif images == \"Golden Gate\":\n",
+        "  IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/1/1e/Golden_gate2.jpg'\n",
+        "  IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/3/3e/GoldenGateBridge.jpg'\n",
+        "elif images == \"Acropolis\":\n",
+        "  IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/c/ce/2006_01_21_Ath%C3%A8nes_Parth%C3%A9non.JPG'\n",
+        "  IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/5/5c/ACROPOLIS_1969_-_panoramio_-_jean_melis.jpg'\n",
+        "else:\n",
+        "  IMAGE_1_URL = 'https://upload.wikimedia.org/wikipedia/commons/d/d8/Eiffel_Tower%2C_November_15%2C_2011.jpg'\n",
+        "  IMAGE_2_URL = 'https://upload.wikimedia.org/wikipedia/commons/a/a8/Eiffel_Tower_from_immediately_beside_it%2C_Paris_May_2008.jpg'"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ttlHtcmiN6QF"
+      },
+      "source": [
+        "Download, resize, save and display the images."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E6RMomGJSfeb"
+      },
+      "outputs": [],
+      "source": [
+        "def download_and_resize(name, url, new_width=256, new_height=256):\n",
+        "  path = tf.keras.utils.get_file(url.split('/')[-1], url)\n",
+        "  image = Image.open(path)\n",
+        "  image = ImageOps.fit(image, (new_width, new_height), Image.LANCZOS)\n",
+        "  return image"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "reajtO7XSj7Y"
+      },
+      "outputs": [],
+      "source": [
+        "image1 = download_and_resize('image_1.jpg', IMAGE_1_URL)\n",
+        "image2 = download_and_resize('image_2.jpg', IMAGE_2_URL)\n",
+        "\n",
+        "plt.subplot(1,2,1)\n",
+        "plt.imshow(image1)\n",
+        "plt.subplot(1,2,2)\n",
+        "plt.imshow(image2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "leKqkoT9OP7r"
+      },
+      "source": [
+        "## Apply the DELF module to the data"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A3WoT1-SPoTI"
+      },
+      "source": [
+        "The DELF module takes an image as input and will describe noteworthy points with vectors. The following cell contains the core of this colab's logic."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pXr2tUhvp1Ue"
+      },
+      "outputs": [],
+      "source": [
+        "delf = hub.load('https://tfhub.dev/google/delf/1').signatures['default']"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pvAU_gUHoYcY"
+      },
+      "outputs": [],
+      "source": [
+        "def run_delf(image):\n",
+        "  np_image = np.array(image)\n",
+        "  float_image = tf.image.convert_image_dtype(np_image, tf.float32)\n",
+        "\n",
+        "  return delf(\n",
+        "      image=float_image,\n",
+        "      score_threshold=tf.constant(100.0),\n",
+        "      image_scales=tf.constant([0.25, 0.3536, 0.5, 0.7071, 1.0, 1.4142, 2.0]),\n",
+        "      max_feature_num=tf.constant(1000))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FEzgHAT0UDNP"
+      },
+      "outputs": [],
+      "source": [
+        "result1 = run_delf(image1)\n",
+        "result2 = run_delf(image2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NByyBA5yOL2b"
+      },
+      "source": [
+        "## Use the locations and description vectors to match the images"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "mVaKXT3cMSib"
+      },
+      "outputs": [],
+      "source": [
+        "#@title TensorFlow is not needed for this post-processing and visualization\n",
+        "def match_images(image1, image2, result1, result2):\n",
+        "  distance_threshold = 0.8\n",
+        "\n",
+        "  # Read features.\n",
+        "  num_features_1 = result1['locations'].shape[0]\n",
+        "  print(\"Loaded image 1's %d features\" % num_features_1)\n",
+        "  \n",
+        "  num_features_2 = result2['locations'].shape[0]\n",
+        "  print(\"Loaded image 2's %d features\" % num_features_2)\n",
+        "\n",
+        "  # Find nearest-neighbor matches using a KD tree.\n",
+        "  d1_tree = cKDTree(result1['descriptors'])\n",
+        "  _, indices = d1_tree.query(\n",
+        "      result2['descriptors'],\n",
+        "      distance_upper_bound=distance_threshold)\n",
+        "\n",
+        "  # Select feature locations for putative matches.\n",
+        "  locations_2_to_use = np.array([\n",
+        "      result2['locations'][i,]\n",
+        "      for i in range(num_features_2)\n",
+        "      if indices[i] != num_features_1\n",
+        "  ])\n",
+        "  locations_1_to_use = np.array([\n",
+        "      result1['locations'][indices[i],]\n",
+        "      for i in range(num_features_2)\n",
+        "      if indices[i] != num_features_1\n",
+        "  ])\n",
+        "\n",
+        "  # Perform geometric verification using RANSAC.\n",
+        "  _, inliers = ransac(\n",
+        "      (locations_1_to_use, locations_2_to_use),\n",
+        "      AffineTransform,\n",
+        "      min_samples=3,\n",
+        "      residual_threshold=20,\n",
+        "      max_trials=1000)\n",
+        "\n",
+        "  print('Found %d inliers' % sum(inliers))\n",
+        "\n",
+        "  # Visualize correspondences.\n",
+        "  _, ax = plt.subplots()\n",
+        "  inlier_idxs = np.nonzero(inliers)[0]\n",
+        "  plot_matched_features(\n",
+        "      image1,\n",
+        "      image2,\n",
+        "      keypoints0=locations_1_to_use,\n",
+        "      keypoints1=locations_2_to_use,\n",
+        "      matches=np.column_stack((inlier_idxs, inlier_idxs)),\n",
+        "      ax=ax,\n",
+        "  )\n",
+        "\n",
+        "  ax.axis('off')\n",
+        "  ax.set_title('DELF correspondences')\n",
+        "\n",
+        "  for line in ax.lines:\n",
+        "    line.set_color('b')\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tpEgqOvCYlPY"
+      },
+      "outputs": [],
+      "source": [
+        "match_images(image1, image2, result1, result2)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "RUymE2l9GZfO"
+      ],
+      "name": "tf_hub_delf_module.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf_hub_film_example.ipynb b/site/en/hub/tutorials/tf_hub_film_example.ipynb
new file mode 100644
index 00000000000..83bcd4bd12c
--- /dev/null
+++ b/site/en/hub/tutorials/tf_hub_film_example.ipynb
@@ -0,0 +1,576 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qNLUPuRpkFv_"
+      },
+      "source": [
+        "##### Copyright 2022 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "DQcWZm0FkPk-"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2022 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Exbxve1rHlrF"
+      },
+      "source": [
+        "# Frame interpolation using the FILM model\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "jMWFVTlbrQ8m"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "61H28S7ArUAZ"
+      },
+      "source": [
+        "Frame interpolation is the task of synthesizing many in-between images from a given set of images. The technique is often used for frame rate upsampling or creating slow-motion video effects.\n",
+        "\n",
+        "In this colab, you will use the FILM model to do frame interpolation. The colab also provides code snippets to create videos from the interpolated in-between images.\n",
+        "\n",
+        "For more information on FILM research, you can read more here:\n",
+        "- Google AI Blog: [Large Motion Frame Interpolation](https://ai.googleblog.com/2022/10/large-motion-frame-interpolation.html)\n",
+        "- Project Page: FILM: [Frame Interpolation for Large Motion](https://film-net.github.io/)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dVX7s6zMulsu"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oi5t2OEJsGBW"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install mediapy\n",
+        "!sudo apt-get install -y ffmpeg"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BA1tq39MjOiF"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "\n",
+        "import requests\n",
+        "import numpy as np\n",
+        "\n",
+        "from typing import Generator, Iterable, List, Optional\n",
+        "import mediapy as media"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GTgXmeYGnT7q"
+      },
+      "source": [
+        "## Load the model from TFHub\n",
+        "\n",
+        "To load a model from TensorFlow Hub you need the tfhub library and the model handle which is its documentation url."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GojhvyAtjUt0"
+      },
+      "outputs": [],
+      "source": [
+        "model = hub.load(\"https://tfhub.dev/google/film/1\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "DOQJPsu2CwPk"
+      },
+      "source": [
+        "## Util function to load images from a url or locally\n",
+        "\n",
+        "This function loads an image and make it ready to be used by the model later."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BPnh5uhQvFln"
+      },
+      "outputs": [],
+      "source": [
+        "_UINT8_MAX_F = float(np.iinfo(np.uint8).max)\n",
+        "\n",
+        "def load_image(img_url: str):\n",
+        "  \"\"\"Returns an image with shape [height, width, num_channels], with pixels in [0..1] range, and type np.float32.\"\"\"\n",
+        "\n",
+        "  if (img_url.startswith(\"https\")):\n",
+        "    user_agent = {'User-agent': 'Colab Sample (https://tensorflow.org)'}\n",
+        "    response = requests.get(img_url, headers=user_agent)\n",
+        "    image_data = response.content\n",
+        "  else:\n",
+        "    image_data = tf.io.read_file(img_url)\n",
+        "\n",
+        "  image = tf.io.decode_image(image_data, channels=3)\n",
+        "  image_numpy = tf.cast(image, dtype=tf.float32).numpy()\n",
+        "  return image_numpy / _UINT8_MAX_F\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yjDFns1zp5y6"
+      },
+      "source": [
+        "FILM's model input is a dictionary with the keys `time`, `x0`, `x1`:\n",
+        "\n",
+        "- `time`: position of the interpolated frame. Midway is `0.5`.\n",
+        "- `x0`: is the initial frame.\n",
+        "- `x1`: is the final frame.\n",
+        "\n",
+        "Both frames need to be normalized (done in the function `load_image` above) where each pixel is in the range of `[0..1]`.\n",
+        "\n",
+        "`time` is a value between `[0..1]` and it says where the generated image should be. 0.5 is midway between the input images.\n",
+        "\n",
+        "All three values need to have a batch dimension too."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VEQNQlHGsWSM"
+      },
+      "outputs": [],
+      "source": [
+        "# using images from the FILM repository (https://github.com/google-research/frame-interpolation/)\n",
+        "\n",
+        "image_1_url = \"https://github.com/google-research/frame-interpolation/blob/main/photos/one.png?raw=true\"\n",
+        "image_2_url = \"https://github.com/google-research/frame-interpolation/blob/main/photos/two.png?raw=true\"\n",
+        "\n",
+        "time = np.array([0.5], dtype=np.float32)\n",
+        "\n",
+        "image1 = load_image(image_1_url)\n",
+        "image2 = load_image(image_2_url)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "r6_MQE9EuF_K"
+      },
+      "outputs": [],
+      "source": [
+        "input = {\n",
+        "    'time': np.expand_dims(time, axis=0), # adding the batch dimension to the time\n",
+        "     'x0': np.expand_dims(image1, axis=0), # adding the batch dimension to the image\n",
+        "     'x1': np.expand_dims(image2, axis=0)  # adding the batch dimension to the image\n",
+        "}\n",
+        "mid_frame = model(input)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nZkzYE2bptfD"
+      },
+      "source": [
+        "The model outputs a couple of results but what you'll use here is the `image` key, whose value is the interpolated frame."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eClVbNFhA5Py"
+      },
+      "outputs": [],
+      "source": [
+        "print(mid_frame.keys())"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rE2csH3u8ePe"
+      },
+      "outputs": [],
+      "source": [
+        "frames = [image1, mid_frame['image'][0].numpy(), image2]\n",
+        "\n",
+        "media.show_images(frames, titles=['input image one', 'generated image', 'input image two'], height=250)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fS1AT8kn-f_l"
+      },
+      "source": [
+        "Let's create a video from the generated frames"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oFc53B3p37SH"
+      },
+      "outputs": [],
+      "source": [
+        "media.show_video(frames, fps=3, title='FILM interpolated video')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x5AOFNkj-lfO"
+      },
+      "source": [
+        "## Define a Frame Interpolator Library\n",
+        "\n",
+        "As you can see, the transition is not too smooth. \n",
+        "\n",
+        "To improve that you'll need many more interpolated frames.\n",
+        "\n",
+        "You could just keep running the model many times with intermediary images but there is a better solution.\n",
+        "\n",
+        "To generate many interpolated images and have a  smoother video you'll create an interpolator library."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tsoDv_9geoZn"
+      },
+      "outputs": [],
+      "source": [
+        "\"\"\"A wrapper class for running a frame interpolation based on the FILM model on TFHub\n",
+        "\n",
+        "Usage:\n",
+        "  interpolator = Interpolator()\n",
+        "  result_batch = interpolator(image_batch_0, image_batch_1, batch_dt)\n",
+        "  Where image_batch_1 and image_batch_2 are numpy tensors with TF standard\n",
+        "  (B,H,W,C) layout, batch_dt is the sub-frame time in range [0..1], (B,) layout.\n",
+        "\"\"\"\n",
+        "\n",
+        "\n",
+        "def _pad_to_align(x, align):\n",
+        "  \"\"\"Pads image batch x so width and height divide by align.\n",
+        "\n",
+        "  Args:\n",
+        "    x: Image batch to align.\n",
+        "    align: Number to align to.\n",
+        "\n",
+        "  Returns:\n",
+        "    1) An image padded so width % align == 0 and height % align == 0.\n",
+        "    2) A bounding box that can be fed readily to tf.image.crop_to_bounding_box\n",
+        "      to undo the padding.\n",
+        "  \"\"\"\n",
+        "  # Input checking.\n",
+        "  assert np.ndim(x) == 4\n",
+        "  assert align > 0, 'align must be a positive number.'\n",
+        "\n",
+        "  height, width = x.shape[-3:-1]\n",
+        "  height_to_pad = (align - height % align) if height % align != 0 else 0\n",
+        "  width_to_pad = (align - width % align) if width % align != 0 else 0\n",
+        "\n",
+        "  bbox_to_pad = {\n",
+        "      'offset_height': height_to_pad // 2,\n",
+        "      'offset_width': width_to_pad // 2,\n",
+        "      'target_height': height + height_to_pad,\n",
+        "      'target_width': width + width_to_pad\n",
+        "  }\n",
+        "  padded_x = tf.image.pad_to_bounding_box(x, **bbox_to_pad)\n",
+        "  bbox_to_crop = {\n",
+        "      'offset_height': height_to_pad // 2,\n",
+        "      'offset_width': width_to_pad // 2,\n",
+        "      'target_height': height,\n",
+        "      'target_width': width\n",
+        "  }\n",
+        "  return padded_x, bbox_to_crop\n",
+        "\n",
+        "\n",
+        "class Interpolator:\n",
+        "  \"\"\"A class for generating interpolated frames between two input frames.\n",
+        "\n",
+        "  Uses the Film model from TFHub\n",
+        "  \"\"\"\n",
+        "\n",
+        "  def __init__(self, align: int = 64) -> None:\n",
+        "    \"\"\"Loads a saved model.\n",
+        "\n",
+        "    Args:\n",
+        "      align: 'If >1, pad the input size so it divides with this before\n",
+        "        inference.'\n",
+        "    \"\"\"\n",
+        "    self._model = hub.load(\"https://tfhub.dev/google/film/1\")\n",
+        "    self._align = align\n",
+        "\n",
+        "  def __call__(self, x0: np.ndarray, x1: np.ndarray,\n",
+        "               dt: np.ndarray) -> np.ndarray:\n",
+        "    \"\"\"Generates an interpolated frame between given two batches of frames.\n",
+        "\n",
+        "    All inputs should be np.float32 datatype.\n",
+        "\n",
+        "    Args:\n",
+        "      x0: First image batch. Dimensions: (batch_size, height, width, channels)\n",
+        "      x1: Second image batch. Dimensions: (batch_size, height, width, channels)\n",
+        "      dt: Sub-frame time. Range [0,1]. Dimensions: (batch_size,)\n",
+        "\n",
+        "    Returns:\n",
+        "      The result with dimensions (batch_size, height, width, channels).\n",
+        "    \"\"\"\n",
+        "    if self._align is not None:\n",
+        "      x0, bbox_to_crop = _pad_to_align(x0, self._align)\n",
+        "      x1, _ = _pad_to_align(x1, self._align)\n",
+        "\n",
+        "    inputs = {'x0': x0, 'x1': x1, 'time': dt[..., np.newaxis]}\n",
+        "    result = self._model(inputs, training=False)\n",
+        "    image = result['image']\n",
+        "\n",
+        "    if self._align is not None:\n",
+        "      image = tf.image.crop_to_bounding_box(image, **bbox_to_crop)\n",
+        "    return image.numpy()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ZeGYaNBd_7a5"
+      },
+      "source": [
+        "## Frame and Video Generation Utility Functions"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gOJxup6s_1DP"
+      },
+      "outputs": [],
+      "source": [
+        "def _recursive_generator(\n",
+        "    frame1: np.ndarray, frame2: np.ndarray, num_recursions: int,\n",
+        "    interpolator: Interpolator) -> Generator[np.ndarray, None, None]:\n",
+        "  \"\"\"Splits halfway to repeatedly generate more frames.\n",
+        "\n",
+        "  Args:\n",
+        "    frame1: Input image 1.\n",
+        "    frame2: Input image 2.\n",
+        "    num_recursions: How many times to interpolate the consecutive image pairs.\n",
+        "    interpolator: The frame interpolator instance.\n",
+        "\n",
+        "  Yields:\n",
+        "    The interpolated frames, including the first frame (frame1), but excluding\n",
+        "    the final frame2.\n",
+        "  \"\"\"\n",
+        "  if num_recursions == 0:\n",
+        "    yield frame1\n",
+        "  else:\n",
+        "    # Adds the batch dimension to all inputs before calling the interpolator,\n",
+        "    # and remove it afterwards.\n",
+        "    time = np.full(shape=(1,), fill_value=0.5, dtype=np.float32)\n",
+        "    mid_frame = interpolator(\n",
+        "        np.expand_dims(frame1, axis=0), np.expand_dims(frame2, axis=0), time)[0]\n",
+        "    yield from _recursive_generator(frame1, mid_frame, num_recursions - 1,\n",
+        "                                    interpolator)\n",
+        "    yield from _recursive_generator(mid_frame, frame2, num_recursions - 1,\n",
+        "                                    interpolator)\n",
+        "\n",
+        "\n",
+        "def interpolate_recursively(\n",
+        "    frames: List[np.ndarray], num_recursions: int,\n",
+        "    interpolator: Interpolator) -> Iterable[np.ndarray]:\n",
+        "  \"\"\"Generates interpolated frames by repeatedly interpolating the midpoint.\n",
+        "\n",
+        "  Args:\n",
+        "    frames: List of input frames. Expected shape (H, W, 3). The colors should be\n",
+        "      in the range[0, 1] and in gamma space.\n",
+        "    num_recursions: Number of times to do recursive midpoint\n",
+        "      interpolation.\n",
+        "    interpolator: The frame interpolation model to use.\n",
+        "\n",
+        "  Yields:\n",
+        "    The interpolated frames (including the inputs).\n",
+        "  \"\"\"\n",
+        "  n = len(frames)\n",
+        "  for i in range(1, n):\n",
+        "    yield from _recursive_generator(frames[i - 1], frames[i],\n",
+        "                                    times_to_interpolate, interpolator)\n",
+        "  # Separately yield the final frame.\n",
+        "  yield frames[-1]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "X1R2KjhEAHu0"
+      },
+      "outputs": [],
+      "source": [
+        "times_to_interpolate = 6\n",
+        "interpolator = Interpolator()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AZUo8tg1AYvZ"
+      },
+      "source": [
+        "## Running the Interpolator"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "QMMNjs7sAWTG"
+      },
+      "outputs": [],
+      "source": [
+        "input_frames = [image1, image2]\n",
+        "frames = list(\n",
+        "    interpolate_recursively(input_frames, times_to_interpolate,\n",
+        "                                        interpolator))\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "s9mHHyCAAhrM"
+      },
+      "outputs": [],
+      "source": [
+        "print(f'video with {len(frames)} frames')\n",
+        "media.show_video(frames, fps=30, title='FILM interpolated video')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "_0AZKeMVFwAc"
+      },
+      "source": [
+        "For more information, you can visit [FILM's model repository](https://github.com/google-research/frame-interpolation).\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8764ry3SGDks"
+      },
+      "source": [
+        "## Citation\n",
+        "\n",
+        "If you find this model and code useful in your works, please acknowledge it appropriately by citing:\n",
+        "\n",
+        "```\n",
+        "@inproceedings{reda2022film,\n",
+        " title = {FILM: Frame Interpolation for Large Motion},\n",
+        " author = {Fitsum Reda and Janne Kontkanen and Eric Tabellion and Deqing Sun and Caroline Pantofaru and Brian Curless},\n",
+        " booktitle = {The European Conference on Computer Vision (ECCV)},\n",
+        " year = {2022}\n",
+        "}\n",
+        "```\n",
+        "\n",
+        "```\n",
+        "@misc{film-tf,\n",
+        "  title = {Tensorflow 2 Implementation of \"FILM: Frame Interpolation for Large Motion\"},\n",
+        "  author = {Fitsum Reda and Janne Kontkanen and Eric Tabellion and Deqing Sun and Caroline Pantofaru and Brian Curless},\n",
+        "  year = {2022},\n",
+        "  publisher = {GitHub},\n",
+        "  journal = {GitHub repository},\n",
+        "  howpublished = {\\url{https://github.com/google-research/frame-interpolation}}\n",
+        "}\n",
+        "```"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "name": "tf_hub_film_example.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tf_hub_generative_image_module.ipynb b/site/en/hub/tutorials/tf_hub_generative_image_module.ipynb
new file mode 100644
index 00000000000..4937bc2eb22
--- /dev/null
+++ b/site/en/hub/tutorials/tf_hub_generative_image_module.ipynb
@@ -0,0 +1,447 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N6ZDpd9XzFeN"
+      },
+      "source": [
+        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "KUu4vOt5zI9d"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CxmDMK4yupqg"
+      },
+      "source": [
+        "# Generate Artificial Faces with CelebA Progressive GAN Model\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Sy553YSVmYiK"
+      },
+      "source": [
+        "This Colab demonstrates use of a TF Hub module based on a generative adversarial network (GAN). The module maps from N-dimensional vectors, called latent space, to RGB images.\n",
+        "\n",
+        "Two examples are provided:\n",
+        "* **Mapping** from latent space to images, and\n",
+        "* Given a target image, **using gradient descent to find** a latent vector that generates an image similar to the target image."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "v4XGxDrCkeip"
+      },
+      "source": [
+        "## Optional prerequisites\n",
+        "\n",
+        "* Familiarity with [low level Tensorflow concepts](https://www.tensorflow.org/guide/eager).\n",
+        "* [Generative Adversarial Network](https://en.wikipedia.org/wiki/Generative_adversarial_network) on Wikipedia.\n",
+        "* Paper on Progressive GANs: [Progressive Growing of GANs for Improved Quality, Stability, and Variation](https://arxiv.org/abs/1710.10196)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "HK3Q2vIaVw56"
+      },
+      "source": [
+        "### More models\n",
+        "[Here](https://tfhub.dev/s?module-type=image-generator) you can find all models currently hosted on [tfhub.dev](https://tfhub.dev/) that can generate images."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q4DN769E2O_R"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KNM3kA0arrUu"
+      },
+      "outputs": [],
+      "source": [
+        "# Install imageio for creating animations.  \n",
+        "!pip -q install imageio\n",
+        "!pip -q install scikit-image\n",
+        "!pip install git+https://github.com/tensorflow/docs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "6cPY9Ou4sWs_"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Imports and function definitions\n",
+        "from absl import logging\n",
+        "\n",
+        "import imageio\n",
+        "import PIL.Image\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "tf.random.set_seed(0)\n",
+        "\n",
+        "import tensorflow_hub as hub\n",
+        "from tensorflow_docs.vis import embed\n",
+        "import time\n",
+        "\n",
+        "try:\n",
+        "  from google.colab import files\n",
+        "except ImportError:\n",
+        "  pass\n",
+        "\n",
+        "from IPython import display\n",
+        "from skimage import transform\n",
+        "\n",
+        "# We could retrieve this value from module.get_input_shapes() if we didn't know\n",
+        "# beforehand which module we will be using.\n",
+        "latent_dim = 512\n",
+        "\n",
+        "\n",
+        "# Interpolates between two vectors that are non-zero and don't both lie on a\n",
+        "# line going through origin. First normalizes v2 to have the same norm as v1. \n",
+        "# Then interpolates between the two vectors on the hypersphere.\n",
+        "def interpolate_hypersphere(v1, v2, num_steps):\n",
+        "  v1_norm = tf.norm(v1)\n",
+        "  v2_norm = tf.norm(v2)\n",
+        "  v2_normalized = v2 * (v1_norm / v2_norm)\n",
+        "\n",
+        "  vectors = []\n",
+        "  for step in range(num_steps):\n",
+        "    interpolated = v1 + (v2_normalized - v1) * step / (num_steps - 1)\n",
+        "    interpolated_norm = tf.norm(interpolated)\n",
+        "    interpolated_normalized = interpolated * (v1_norm / interpolated_norm)\n",
+        "    vectors.append(interpolated_normalized)\n",
+        "  return tf.stack(vectors)\n",
+        "\n",
+        "# Simple way to display an image.\n",
+        "def display_image(image):\n",
+        "  image = tf.constant(image)\n",
+        "  image = tf.image.convert_image_dtype(image, tf.uint8)\n",
+        "  return PIL.Image.fromarray(image.numpy())\n",
+        "\n",
+        "# Given a set of images, show an animation.\n",
+        "def animate(images):\n",
+        "  images = np.array(images)\n",
+        "  converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)\n",
+        "  imageio.mimsave('./animation.gif', converted_images)\n",
+        "  return embed.embed_file('./animation.gif')\n",
+        "\n",
+        "logging.set_verbosity(logging.ERROR)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f5EESfBvukYI"
+      },
+      "source": [
+        "## Latent space interpolation"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "nJb9gFmRvynZ"
+      },
+      "source": [
+        "### Random vectors\n",
+        "\n",
+        "Latent space interpolation between two randomly initialized vectors. We will use a TF Hub module [progan-128](https://tfhub.dev/google/progan-128/1) that contains a pre-trained Progressive GAN."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "8StEe9x9wGma"
+      },
+      "outputs": [],
+      "source": [
+        "progan = hub.load(\"https://tfhub.dev/google/progan-128/1\").signatures['default']"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "fZ0O5_5Jhwio"
+      },
+      "outputs": [],
+      "source": [
+        "def interpolate_between_vectors():\n",
+        "  v1 = tf.random.normal([latent_dim])\n",
+        "  v2 = tf.random.normal([latent_dim])\n",
+        "    \n",
+        "  # Creates a tensor with 25 steps of interpolation between v1 and v2.\n",
+        "  vectors = interpolate_hypersphere(v1, v2, 50)\n",
+        "\n",
+        "  # Uses module to generate images from the latent space.\n",
+        "  interpolated_images = progan(vectors)['default']\n",
+        "\n",
+        "  return interpolated_images\n",
+        "\n",
+        "interpolated_images = interpolate_between_vectors()\n",
+        "animate(interpolated_images)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L9-uXoTHuXQC"
+      },
+      "source": [
+        "## Finding closest vector in latent space\n",
+        "Fix a target image. As an example use an image generated from the module or upload your own."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "both",
+        "id": "phT4W66pMmko"
+      },
+      "outputs": [],
+      "source": [
+        "image_from_module_space = True  # @param { isTemplate:true, type:\"boolean\" }\n",
+        "\n",
+        "def get_module_space_image():\n",
+        "  vector = tf.random.normal([1, latent_dim])\n",
+        "  images = progan(vector)['default'][0]\n",
+        "  return images\n",
+        "\n",
+        "def upload_image():\n",
+        "  uploaded = files.upload()\n",
+        "  image = imageio.imread(uploaded[list(uploaded.keys())[0]])\n",
+        "  return transform.resize(image, [128, 128])\n",
+        "\n",
+        "if image_from_module_space:\n",
+        "  target_image = get_module_space_image()\n",
+        "else:\n",
+        "  target_image = upload_image()\n",
+        "\n",
+        "display_image(target_image)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rBIt3Q4qvhuq"
+      },
+      "source": [
+        "After defining a loss function between the target image and the image generated by a latent space variable, we can use gradient descent to find variable values that minimize the loss."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cUGakLdbML2Q"
+      },
+      "outputs": [],
+      "source": [
+        "tf.random.set_seed(42)\n",
+        "initial_vector = tf.random.normal([1, latent_dim])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "u7MGzDE5MU20"
+      },
+      "outputs": [],
+      "source": [
+        "display_image(progan(initial_vector)['default'][0])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "q_4Z7tnyg-ZY"
+      },
+      "outputs": [],
+      "source": [
+        "def find_closest_latent_vector(initial_vector, num_optimization_steps,\n",
+        "                               steps_per_image):\n",
+        "  images = []\n",
+        "  losses = []\n",
+        "\n",
+        "  vector = tf.Variable(initial_vector)  \n",
+        "  optimizer = tf.optimizers.Adam(learning_rate=0.01)\n",
+        "  loss_fn = tf.losses.MeanAbsoluteError(reduction=\"sum\")\n",
+        "\n",
+        "  for step in range(num_optimization_steps):\n",
+        "    if (step % 100)==0:\n",
+        "      print()\n",
+        "    print('.', end='')\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      image = progan(vector.read_value())['default'][0]\n",
+        "      if (step % steps_per_image) == 0:\n",
+        "        images.append(image.numpy())\n",
+        "      target_image_difference = loss_fn(image, target_image[:,:,:3])\n",
+        "      # The latent vectors were sampled from a normal distribution. We can get\n",
+        "      # more realistic images if we regularize the length of the latent vector to \n",
+        "      # the average length of vector from this distribution.\n",
+        "      regularizer = tf.abs(tf.norm(vector) - np.sqrt(latent_dim))\n",
+        "      \n",
+        "      loss = target_image_difference + regularizer\n",
+        "      losses.append(loss.numpy())\n",
+        "    grads = tape.gradient(loss, [vector])\n",
+        "    optimizer.apply_gradients(zip(grads, [vector]))\n",
+        "    \n",
+        "  return images, losses\n",
+        "\n",
+        "\n",
+        "num_optimization_steps=200\n",
+        "steps_per_image=5\n",
+        "images, loss = find_closest_latent_vector(initial_vector, num_optimization_steps, steps_per_image)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pRbeF2oSAcOB"
+      },
+      "outputs": [],
+      "source": [
+        "plt.plot(loss)\n",
+        "plt.ylim([0,max(plt.ylim())])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KnZkDy2FEsTt"
+      },
+      "outputs": [],
+      "source": [
+        "animate(np.stack(images))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GGKfuCdfPQKH"
+      },
+      "source": [
+        "Compare the result to the target:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TK1P5z3bNuIl"
+      },
+      "outputs": [],
+      "source": [
+        "display_image(np.concatenate([images[-1], target_image], axis=1))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tDt15dLsJwMy"
+      },
+      "source": [
+        "### Playing with the above example\n",
+        "If image is from the module space, the descent is quick and converges to a reasonable sample. Try out descending to an image that is **not from the module space**. The descent will only converge if the image is reasonably close to the space of training images.\n",
+        "\n",
+        "How to make it descend faster and to a more realistic image? One can try:\n",
+        "* using different loss on the image difference, e.g., quadratic,\n",
+        "* using different regularizer on the latent vector,\n",
+        "* initializing from a random vector in multiple runs,\n",
+        "* etc.\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "N6ZDpd9XzFeN"
+      ],
+      "name": "tf_hub_generative_image_module.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/tweening_conv3d.ipynb b/site/en/hub/tutorials/tweening_conv3d.ipynb
new file mode 100644
index 00000000000..8c53929021f
--- /dev/null
+++ b/site/en/hub/tutorials/tweening_conv3d.ipynb
@@ -0,0 +1,297 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wC0PtNm3Sa_T"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "hgOqPjRKSa-7"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oKAkxAYuONU6"
+      },
+      "source": [
+        "# Video Inbetweening using 3D Convolutions\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MfBg1C5NB3X0"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "cvMgkVIBpT-Y"
+      },
+      "source": [
+        "Yunpeng Li, Dominik Roblek, and Marco Tagliasacchi. From Here to There: Video Inbetweening Using Direct 3D Convolutions, 2019.\n",
+        "\n",
+        "https://arxiv.org/abs/1905.10240\n",
+        "\n",
+        "\n",
+        "Current Hub characteristics:\n",
+        "- has models for BAIR Robot pushing videos and KTH action video dataset (though this colab uses only BAIR)\n",
+        "- BAIR dataset already available in Hub. However, KTH videos need to be supplied by the users themselves.\n",
+        "- only evaluation (video generation) for now\n",
+        "- batch size and frame size are hard-coded\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Q4DN769E2O_R"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EsQFWvxrYrHg"
+      },
+      "source": [
+        "Since `tfds.load('bair_robot_pushing_small', split='test')` would download a 30GB archive that also contains the training data, we download a separated archive that only contains the 190MB test data. The used dataset has been published by [this paper](https://arxiv.org/abs/1710.05268) and is licensed as Creative Commons BY 4.0."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GhIKakhc7JYL"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import seaborn as sns\n",
+        "import tensorflow_hub as hub\n",
+        "import tensorflow_datasets as tfds\n",
+        "\n",
+        "from tensorflow_datasets.core import SplitGenerator\n",
+        "from tensorflow_datasets.video.bair_robot_pushing import BairRobotPushingSmall\n",
+        "\n",
+        "import tempfile\n",
+        "import pathlib\n",
+        "\n",
+        "TEST_DIR = pathlib.Path(tempfile.mkdtemp()) / \"bair_robot_pushing_small/softmotion30_44k/test/\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zBMz14GmYkwz"
+      },
+      "outputs": [],
+      "source": [
+        "# Download the test split to $TEST_DIR\n",
+        "!mkdir -p $TEST_DIR\n",
+        "!wget -nv https://storage.googleapis.com/download.tensorflow.org/data/bair_test_traj_0_to_255.tfrecords -O $TEST_DIR/traj_0_to_255.tfrecords"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "irRJ2Q0iYoW0"
+      },
+      "outputs": [],
+      "source": [
+        "# Since the dataset builder expects the train and test split to be downloaded,\n",
+        "# patch it so it only expects the test data to be available\n",
+        "builder = BairRobotPushingSmall()\n",
+        "test_generator = SplitGenerator(name='test', gen_kwargs={\"filedir\": str(TEST_DIR)})\n",
+        "builder._split_generators = lambda _: [test_generator]\n",
+        "builder.download_and_prepare()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "iaGU8hhBPi_6"
+      },
+      "source": [
+        "## BAIR: Demo based on numpy array inputs"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "IgWmW8YzEiDo"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Load some example data (BAIR).\n",
+        "batch_size = 16\n",
+        "\n",
+        "# If unable to download the dataset automatically due to \"not enough disk space\", please download manually to Google Drive and\n",
+        "# load using tf.data.TFRecordDataset.\n",
+        "ds = builder.as_dataset(split=\"test\")\n",
+        "test_videos = ds.batch(batch_size)\n",
+        "first_batch = next(iter(test_videos))\n",
+        "input_frames = first_batch['image_aux1'][:, ::15]\n",
+        "input_frames = tf.cast(input_frames, tf.float32)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "96Jd5XefGHRr"
+      },
+      "outputs": [],
+      "source": [
+        "# @title Visualize loaded videos start and end frames.\n",
+        "\n",
+        "print('Test videos shape [batch_size, start/end frame, height, width, num_channels]: ', input_frames.shape)\n",
+        "sns.set_style('white')\n",
+        "plt.figure(figsize=(4, 2*batch_size))\n",
+        "\n",
+        "for i in range(batch_size)[:4]:\n",
+        "  plt.subplot(batch_size, 2, 1 + 2*i)\n",
+        "  plt.imshow(input_frames[i, 0] / 255.0)\n",
+        "  plt.title('Video {}: First frame'.format(i))\n",
+        "  plt.axis('off')\n",
+        "  plt.subplot(batch_size, 2, 2 + 2*i)\n",
+        "  plt.imshow(input_frames[i, 1] / 255.0)\n",
+        "  plt.title('Video {}: Last frame'.format(i))\n",
+        "  plt.axis('off')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "w0FFhkikQABy"
+      },
+      "source": [
+        "### Load Hub Module"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cLAUiWfEQAB5"
+      },
+      "outputs": [],
+      "source": [
+        "hub_handle = 'https://tfhub.dev/google/tweening_conv3d_bair/1'\n",
+        "module = hub.load(hub_handle).signatures['default']"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PVHTdXnhbGsK"
+      },
+      "source": [
+        "### Generate and show the videos"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FHAwBW-zyegP"
+      },
+      "outputs": [],
+      "source": [
+        "filled_frames = module(input_frames)['default'] / 255.0"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tVesWHTnSW1Z"
+      },
+      "outputs": [],
+      "source": [
+        "# Show sequences of generated video frames.\n",
+        "\n",
+        "# Concatenate start/end frames and the generated filled frames for the new videos.\n",
+        "generated_videos = np.concatenate([input_frames[:, :1] / 255.0, filled_frames, input_frames[:, 1:] / 255.0], axis=1)\n",
+        "\n",
+        "for video_id in range(4):\n",
+        "  fig = plt.figure(figsize=(10 * 2, 2))\n",
+        "  for frame_id in range(1, 16):\n",
+        "    ax = fig.add_axes([frame_id * 1 / 16., 0, (frame_id + 1) * 1 / 16., 1],\n",
+        "                      xmargin=0, ymargin=0)\n",
+        "    ax.imshow(generated_videos[video_id, frame_id])\n",
+        "    ax.axis('off')"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "Q4DN769E2O_R"
+      ],
+      "name": "tweening_conv3d.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/wav2vec2_saved_model_finetuning.ipynb b/site/en/hub/tutorials/wav2vec2_saved_model_finetuning.ipynb
new file mode 100644
index 00000000000..879bdbd0edb
--- /dev/null
+++ b/site/en/hub/tutorials/wav2vec2_saved_model_finetuning.ipynb
@@ -0,0 +1,984 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yCs7P9JTMlzV"
+      },
+      "source": [
+        "##### Copyright 2021 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Jqn-HYw-Mkea"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2021 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "stRetE8gMlmZ"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ndG8MjmJeicp"
+      },
+      "source": [
+        "# Fine-tuning Wav2Vec2 with an LM head\n",
+        "\n",
+        "In this notebook, we will load the pre-trained wav2vec2 model from [TFHub](https://tfhub.dev) and will fine-tune it on [LibriSpeech dataset](https://huggingface.co/datasets/librispeech_asr) by appending Language Modeling head (LM) over the top of our pre-trained model. The underlying task is to build a model for **Automatic Speech Recognition** i.e. given some speech, the model should be able to transcribe it into text."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rWk8nL6Ui-_0"
+      },
+      "source": [
+        "## Setting Up\n",
+        "\n",
+        "Before running this notebook, please ensure that you are on GPU runtime (`Runtime` > `Change runtime type` > `GPU`). The following cell will install [`gsoc-wav2vec2`](https://github.com/vasudevgupta7/gsoc-wav2vec2) package & its dependencies."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "seqTlMyeZvM4"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install -q git+https://github.com/vasudevgupta7/gsoc-wav2vec2@main\n",
+        "!sudo apt-get install -y libsndfile1-dev\n",
+        "!pip3 install -q SoundFile"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wvuJL8-f0zn5"
+      },
+      "source": [
+        "## Model setup using `TFHub`\n",
+        "\n",
+        "We will start by importing some libraries/modules."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "M3_fgx4eZvM7"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "from wav2vec2 import Wav2Vec2Config\n",
+        "\n",
+        "config = Wav2Vec2Config()\n",
+        "\n",
+        "print(\"TF version:\", tf.__version__)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "y0rVUxyWsS5f"
+      },
+      "source": [
+        "First, we will download our model from TFHub & will wrap our model signature with [`hub.KerasLayer`](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer) to be able to use this model like any other Keras layer. Fortunately, `hub.KerasLayer` can do both in just 1 line.\n",
+        "\n",
+        "**Note:** When loading model with `hub.KerasLayer`, model becomes a bit opaque but sometimes we need finer controls over the model, then we can load the model with `tf.keras.models.load_model(...)`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NO6QRC7KZvM9"
+      },
+      "outputs": [],
+      "source": [
+        "pretrained_layer = hub.KerasLayer(\"https://tfhub.dev/vasudevgupta7/wav2vec2/1\", trainable=True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "pCputyVBv2e9"
+      },
+      "source": [
+        "You can refer to this [script](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/export2hub.py) in case you are interested in the model exporting script. Object `pretrained_layer` is the freezed version of [`Wav2Vec2Model`](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/wav2vec2/modeling.py). These pre-trained weights were converted from HuggingFace PyTorch [pre-trained weights](https://huggingface.co/facebook/wav2vec2-base) using [this script](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/convert_torch_to_tf.py).\n",
+        "\n",
+        "Originally, wav2vec2 was pre-trained with a masked language modelling approach with the objective to identify the true quantized latent speech representation for a masked time step. You can read more about the training objective in the paper- [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SseDnCr7hyhC"
+      },
+      "source": [
+        "Now, we will be defining a few constants and hyper-parameters which will be useful in the next few cells. `AUDIO_MAXLEN` is intentionally set to `246000` as the model signature only accepts static sequence length of `246000`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "eiILuMBERxlO"
+      },
+      "outputs": [],
+      "source": [
+        "AUDIO_MAXLEN = 246000\n",
+        "LABEL_MAXLEN = 256\n",
+        "BATCH_SIZE = 2"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1V4gTgGLgXvO"
+      },
+      "source": [
+        "In the following cell, we will wrap `pretrained_layer` & a dense layer (LM head) with the [Keras's Functional API](https://www.tensorflow.org/guide/keras/functional)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a3CUN1KEB10Q"
+      },
+      "outputs": [],
+      "source": [
+        "inputs = tf.keras.Input(shape=(AUDIO_MAXLEN,))\n",
+        "hidden_states = pretrained_layer(inputs)\n",
+        "outputs = tf.keras.layers.Dense(config.vocab_size)(hidden_states)\n",
+        "\n",
+        "model = tf.keras.Model(inputs=inputs, outputs=outputs)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5zDXuoMXhDMo"
+      },
+      "source": [
+        "The dense layer (defined above) is having an output dimension of `vocab_size` as we want to predict probabilities of each token in the vocabulary at each time step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "oPp18ZHRtnq-"
+      },
+      "source": [
+        "## Setting up training state"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ATQy1ZK3vFr7"
+      },
+      "source": [
+        "In TensorFlow, model weights are built only when `model.call` or `model.build` is called for the first time, so the following cell will build the model weights for us. Further, we will be running `model.summary()` to check the total number of trainable parameters."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZgL5wyaXZvM-"
+      },
+      "outputs": [],
+      "source": [
+        "model(tf.random.uniform(shape=(BATCH_SIZE, AUDIO_MAXLEN)))\n",
+        "model.summary()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EQxxA4Fevp7m"
+      },
+      "source": [
+        "Now, we need to define the `loss_fn` and optimizer to be able to train the model. The following cell will do that for us. We will be using the `Adam` optimizer for simplicity. `CTCLoss` is a common loss type that is used for tasks (like `ASR`) where input sub-parts can't be easily aligned with output sub-parts. You can read more about CTC-loss from this amazing [blog post](https://distill.pub/2017/ctc/).\n",
+        "\n",
+        "\n",
+        "`CTCLoss` (from [`gsoc-wav2vec2`](https://github.com/vasudevgupta7/gsoc-wav2vec2) package) accepts 3 arguments: `config`, `model_input_shape` & `division_factor`. If `division_factor=1`, then loss will simply get summed, so pass `division_factor` accordingly to get mean over batch."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "glDepVEHZvM_"
+      },
+      "outputs": [],
+      "source": [
+        "from wav2vec2 import CTCLoss\n",
+        "\n",
+        "LEARNING_RATE = 5e-5\n",
+        "\n",
+        "loss_fn = CTCLoss(config, (BATCH_SIZE, AUDIO_MAXLEN), division_factor=BATCH_SIZE)\n",
+        "optimizer = tf.keras.optimizers.Adam(LEARNING_RATE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "1mvTuOXpwsQe"
+      },
+      "source": [
+        "## Loading & Pre-processing data\n",
+        "\n",
+        "Let's now download the LibriSpeech dataset from the [official website](http://www.openslr.org/12) and set it up."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "I4kIEC77cBCM"
+      },
+      "outputs": [],
+      "source": [
+        "!wget https://www.openslr.org/resources/12/dev-clean.tar.gz -P ./data/train/\n",
+        "!tar -xf ./data/train/dev-clean.tar.gz -C ./data/train/"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "LsQpmpn6jrMI"
+      },
+      "source": [
+        "**Note:** We are using `dev-clean` configuration as this notebook is just for demonstration purposes, so we need a small amount of data. Complete training data can be easily downloaded from [LibriSpeech website](http://www.openslr.org/12)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ynxAjtGHGFpM"
+      },
+      "outputs": [],
+      "source": [
+        "ls ./data/train/"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yBMiORo0xJD0"
+      },
+      "source": [
+        "Our dataset lies in the LibriSpeech directory. Let's explore these files."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "jkIu_Wt4ZvNA"
+      },
+      "outputs": [],
+      "source": [
+        "data_dir = \"./data/train/LibriSpeech/dev-clean/2428/83705/\"\n",
+        "all_files = os.listdir(data_dir)\n",
+        "\n",
+        "flac_files = [f for f in all_files if f.endswith(\".flac\")]\n",
+        "txt_files = [f for f in all_files if f.endswith(\".txt\")]\n",
+        "\n",
+        "print(\"Transcription files:\", txt_files, \"\\nSound files:\", flac_files)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XEObi_Apk3ZD"
+      },
+      "source": [
+        "Alright, so each sub-directory has many `.flac` files and a `.txt` file. The `.txt` file contains text transcriptions for all the speech samples (i.e. `.flac` files) present in that sub-directory."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WYW6WKJflO2e"
+      },
+      "source": [
+        "We can load this text data as follows:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cEBKxQblHPwq"
+      },
+      "outputs": [],
+      "source": [
+        "def read_txt_file(f):\n",
+        "  with open(f, \"r\") as f:\n",
+        "    samples = f.read().split(\"\\n\")\n",
+        "    samples = {s.split()[0]: \" \".join(s.split()[1:]) for s in samples if len(s.split()) > 2}\n",
+        "  return samples"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Ldkf_ceb0_YW"
+      },
+      "source": [
+        "Similarly, we will define a function for loading a speech sample from a `.flac` file.\n",
+        "\n",
+        "`REQUIRED_SAMPLE_RATE` is set to `16000` as wav2vec2 was pre-trained with `16K` frequency and it's recommended to fine-tune it without any major change in data distribution due to frequency."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YOJ3OzPsTyXv"
+      },
+      "outputs": [],
+      "source": [
+        "import soundfile as sf\n",
+        "\n",
+        "REQUIRED_SAMPLE_RATE = 16000\n",
+        "\n",
+        "def read_flac_file(file_path):\n",
+        "  with open(file_path, \"rb\") as f:\n",
+        "      audio, sample_rate = sf.read(f)\n",
+        "  if sample_rate != REQUIRED_SAMPLE_RATE:\n",
+        "      raise ValueError(\n",
+        "          f\"sample rate (={sample_rate}) of your files must be {REQUIRED_SAMPLE_RATE}\"\n",
+        "      )\n",
+        "  file_id = os.path.split(file_path)[-1][:-len(\".flac\")]\n",
+        "  return {file_id: audio}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2sxDN8P4nWkW"
+      },
+      "source": [
+        "Now, we will pick some random samples & will try to visualize them."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HI5J-2Dfm_wT"
+      },
+      "outputs": [],
+      "source": [
+        "from IPython.display import Audio\n",
+        "import random\n",
+        "\n",
+        "file_id = random.choice([f[:-len(\".flac\")] for f in flac_files])\n",
+        "flac_file_path, txt_file_path = os.path.join(data_dir, f\"{file_id}.flac\"), os.path.join(data_dir, \"2428-83705.trans.txt\")\n",
+        "\n",
+        "print(\"Text Transcription:\", read_txt_file(txt_file_path)[file_id], \"\\nAudio:\")\n",
+        "Audio(filename=flac_file_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "M8jJ7Ed81p_A"
+      },
+      "source": [
+        "Now, we will combine all the speech & text samples and will define the function (in next cell) for that purpose."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MI-5YCzaTsei"
+      },
+      "outputs": [],
+      "source": [
+        "def fetch_sound_text_mapping(data_dir):\n",
+        "  all_files = os.listdir(data_dir)\n",
+        "\n",
+        "  flac_files = [os.path.join(data_dir, f) for f in all_files if f.endswith(\".flac\")]\n",
+        "  txt_files = [os.path.join(data_dir, f) for f in all_files if f.endswith(\".txt\")]\n",
+        "\n",
+        "  txt_samples = {}\n",
+        "  for f in txt_files:\n",
+        "    txt_samples.update(read_txt_file(f))\n",
+        "\n",
+        "  speech_samples = {}\n",
+        "  for f in flac_files:\n",
+        "    speech_samples.update(read_flac_file(f))\n",
+        "\n",
+        "  assert len(txt_samples) == len(speech_samples)\n",
+        "\n",
+        "  samples = [(speech_samples[file_id], txt_samples[file_id]) for file_id in speech_samples.keys() if len(speech_samples[file_id]) < AUDIO_MAXLEN]\n",
+        "  return samples"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mx95Lxvu0nT4"
+      },
+      "source": [
+        "It's time to have a look at a few samples ..."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_Ls7X_jqIz4R"
+      },
+      "outputs": [],
+      "source": [
+        "samples = fetch_sound_text_mapping(data_dir)\n",
+        "samples[:5]"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TUjhSWfsnlCL"
+      },
+      "source": [
+        "Note: We are loading this data into memory as we working with a small amount of dataset in this notebook. But for training on the complete dataset (~300 GBs), you will have to load data lazily. You can refer to [this script](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/data_utils.py) to know more on that."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xg8Zia1kzw0J"
+      },
+      "source": [
+        "Let's pre-process the data now !!!\n",
+        "\n",
+        "We will first define the tokenizer & processor using `gsoc-wav2vec2` package. Then, we will do very simple pre-processing. `processor` will normalize raw speech w.r.to frames axis and `tokenizer` will convert our model outputs into the string (using the defined vocabulary) & will take care of the removal of special tokens (depending on your tokenizer configuration)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gaat_hMLNVHF"
+      },
+      "outputs": [],
+      "source": [
+        "from wav2vec2 import Wav2Vec2Processor\n",
+        "tokenizer = Wav2Vec2Processor(is_tokenizer=True)\n",
+        "processor = Wav2Vec2Processor(is_tokenizer=False)\n",
+        "\n",
+        "def preprocess_text(text):\n",
+        "  label = tokenizer(text)\n",
+        "  return tf.constant(label, dtype=tf.int32)\n",
+        "\n",
+        "def preprocess_speech(audio):\n",
+        "  audio = tf.constant(audio, dtype=tf.float32)\n",
+        "  return processor(tf.transpose(audio))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "GyKl8QP-zRFC"
+      },
+      "source": [
+        "Now, we will define the python generator to call the preprocessing functions we defined in above cells."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PoQrRalwMpQ6"
+      },
+      "outputs": [],
+      "source": [
+        "def inputs_generator():\n",
+        "  for speech, text in samples:\n",
+        "    yield preprocess_speech(speech), preprocess_text(text)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7Vlm3ySFULsG"
+      },
+      "source": [
+        "## Setting up `tf.data.Dataset`\n",
+        "\n",
+        "Following cell will setup `tf.data.Dataset` object using its `.from_generator(...)` method. We will be using the `generator` object, we defined in the above cell.\n",
+        "\n",
+        "**Note:** For distributed training (especially on TPUs), `.from_generator(...)` doesn't work currently and it is recommended to train on data stored in `.tfrecord` format (Note: The TFRecords should ideally be stored inside a GCS Bucket in order for the TPUs to work to the fullest extent).\n",
+        "\n",
+        "You can refer to [this script](https://github.com/vasudevgupta7/gsoc-wav2vec2/blob/main/src/make_tfrecords.py) for more details on how to convert LibriSpeech data into tfrecords."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LbQ_dMwGO62h"
+      },
+      "outputs": [],
+      "source": [
+        "output_signature = (\n",
+        "    tf.TensorSpec(shape=(None),  dtype=tf.float32),\n",
+        "    tf.TensorSpec(shape=(None), dtype=tf.int32),\n",
+        ")\n",
+        "\n",
+        "dataset = tf.data.Dataset.from_generator(inputs_generator, output_signature=output_signature)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HXBbNsRyPyw3"
+      },
+      "outputs": [],
+      "source": [
+        "BUFFER_SIZE = len(flac_files)\n",
+        "SEED = 42\n",
+        "\n",
+        "dataset = dataset.shuffle(BUFFER_SIZE, seed=SEED)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9DAUmns3pXfr"
+      },
+      "source": [
+        "We will pass the dataset into multiple batches, so let's prepare batches in the following cell. Now, all the sequences in a batch should be padded to a constant length. We will use the`.padded_batch(...)` method for that purpose."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Okhko1IWRida"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = dataset.padded_batch(BATCH_SIZE, padded_shapes=(AUDIO_MAXLEN, LABEL_MAXLEN), padding_values=(0.0, 0))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "A45CjQG5qSbV"
+      },
+      "source": [
+        "Accelerators (like GPUs/TPUs) are very fast and often data-loading (& pre-processing) becomes the bottleneck during training as the data-loading part happens on CPUs. This can increase the training time significantly especially when there is a lot of online pre-processing involved or data is streamed online from GCS buckets. To handle those issues, `tf.data.Dataset` offers the `.prefetch(...)` method. This method helps in preparing the next few batches in parallel (on CPUs) while the model is making predictions (on GPUs/TPUs) on the current batch."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f-bKu2YjRior"
+      },
+      "outputs": [],
+      "source": [
+        "dataset = dataset.prefetch(tf.data.AUTOTUNE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Lqk2cs6LxVIh"
+      },
+      "source": [
+        "Since this notebook is made for demonstration purposes, we will be taking first `num_train_batches` and will perform training over only that. You are encouraged to train on the whole dataset though. Similarly, we will evaluate only `num_val_batches`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "z6GO5oYUxXtz"
+      },
+      "outputs": [],
+      "source": [
+        "num_train_batches = 10\n",
+        "num_val_batches = 4\n",
+        "\n",
+        "train_dataset = dataset.take(num_train_batches)\n",
+        "val_dataset = dataset.skip(num_train_batches).take(num_val_batches)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CzAOI78tky08"
+      },
+      "source": [
+        "## Model training\n",
+        "\n",
+        "For training our model, we will be directly calling `.fit(...)` method after compiling our model with `.compile(...)`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vuBY2sZElgwg"
+      },
+      "outputs": [],
+      "source": [
+        "model.compile(optimizer, loss=loss_fn)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qswxafSl0HjO"
+      },
+      "source": [
+        "The above cell will set up our training state. Now we can initiate training with the `.fit(...)` method."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vtuSfnj1l-I_"
+      },
+      "outputs": [],
+      "source": [
+        "history = model.fit(train_dataset, validation_data=val_dataset, epochs=3)\n",
+        "history.history"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ySvp8r2E1q_V"
+      },
+      "source": [
+        "Let's save our model with `.save(...)` method to be able to perform inference later. You can also export this SavedModel to TFHub by following [TFHub documentation](https://www.tensorflow.org/hub/publish)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "C0KEYcwydwjF"
+      },
+      "outputs": [],
+      "source": [
+        "save_dir = \"finetuned-wav2vec2\"\n",
+        "model.save(save_dir, include_optimizer=False)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "MkOpp9rZ211t"
+      },
+      "source": [
+        "Note: We are setting `include_optimizer=False` as we want to use this model for inference only."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SJfPlTgezD0i"
+      },
+      "source": [
+        "## Evaluation\n",
+        "\n",
+        "Now we will be computing Word Error Rate over the validation dataset\n",
+        "\n",
+        "**Word error rate** (WER) is a common metric for measuring the performance of an automatic speech recognition system. The WER is derived from the Levenshtein distance, working at the word level. Word error rate can then be computed as: WER = (S + D + I) / N = (S + D + I) / (S + D + C) where S is the number of substitutions, D is the number of deletions, I is the number of insertions, C is the number of correct words, N is the number of words in the reference (N=S+D+C). This value indicates the percentage of words that were incorrectly predicted. \n",
+        "\n",
+        "You can refer to [this paper](https://www.isca-speech.org/archive_v0/interspeech_2004/i04_2765.html) to learn more about WER."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Io_91Y7-r3xu"
+      },
+      "source": [
+        "We will use `load_metric(...)` function from [HuggingFace datasets](https://huggingface.co/docs/datasets/) library. Let's first install the `datasets` library using `pip` and then define the `metric` object."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GW9F_oVDU1TZ"
+      },
+      "outputs": [],
+      "source": [
+        "!pip3 install -q datasets\n",
+        "\n",
+        "from datasets import load_metric\n",
+        "metric = load_metric(\"wer\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ssWXWc7CZvNB"
+      },
+      "outputs": [],
+      "source": [
+        "@tf.function(jit_compile=True)\n",
+        "def eval_fwd(batch):\n",
+        "  logits = model(batch, training=False)\n",
+        "  return tf.argmax(logits, axis=-1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "NFh1myg1x4ua"
+      },
+      "source": [
+        "It's time to run the evaluation on validation data now."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EQTFVjZghckJ"
+      },
+      "outputs": [],
+      "source": [
+        "from tqdm.auto import tqdm\n",
+        "\n",
+        "for speech, labels in tqdm(val_dataset, total=num_val_batches):\n",
+        "    predictions  = eval_fwd(speech)\n",
+        "    predictions = [tokenizer.decode(pred) for pred in predictions.numpy().tolist()]\n",
+        "    references = [tokenizer.decode(label, group_tokens=False) for label in labels.numpy().tolist()]\n",
+        "    metric.add_batch(references=references, predictions=predictions)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WWCc8qBesv3e"
+      },
+      "source": [
+        "We are using the `tokenizer.decode(...)` method for decoding our predictions and labels back into the text and will add them to the metric for `WER` computation later."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XI_URj8Wtb2g"
+      },
+      "source": [
+        "Now, let's calculate the metric value in following cell:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "a83wekLgWMod"
+      },
+      "outputs": [],
+      "source": [
+        "metric.compute()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "c_cD1OgVEjl4"
+      },
+      "source": [
+        "**Note:** Here metric value doesn't make any sense as the model is trained on very small data and ASR-like tasks often require a large amount of data to learn a mapping from speech to text. You should probably train on large data to get some good results. This notebook gives you a template to fine-tune a pre-trained speech model."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G14o706kdTE1"
+      },
+      "source": [
+        "## Inference\n",
+        "\n",
+        "Now that we are satisfied with the training process & have saved the model in `save_dir`, we will see how this model can be used for inference.\n",
+        "\n",
+        "First, we will load our model using `tf.keras.models.load_model(...)`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wrTrExiUdaED"
+      },
+      "outputs": [],
+      "source": [
+        "finetuned_model = tf.keras.models.load_model(save_dir)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "luodSroz20SR"
+      },
+      "source": [
+        "Let's download some speech samples for performing inference. You can replace the following sample with your speech sample also."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "HUE0shded6Ej"
+      },
+      "outputs": [],
+      "source": [
+        "!wget https://github.com/vasudevgupta7/gsoc-wav2vec2/raw/main/data/SA2.wav"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ycBjU_U53FjL"
+      },
+      "source": [
+        "Now, we will read the speech sample using `soundfile.read(...)` and pad it to `AUDIO_MAXLEN` to satisfy the model signature. Then we will normalize that speech sample using the `Wav2Vec2Processor` instance & will feed it into the model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "z7CARje4d5_H"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "\n",
+        "speech, _ = sf.read(\"SA2.wav\")\n",
+        "speech = np.pad(speech, (0, AUDIO_MAXLEN - len(speech)))\n",
+        "speech = tf.expand_dims(processor(tf.constant(speech)), 0)\n",
+        "\n",
+        "outputs = finetuned_model(speech)\n",
+        "outputs"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lUSttSPa30qP"
+      },
+      "source": [
+        "Let's decode numbers back into text sequence using the `Wav2Vec2tokenizer` instance, we defined above."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RYdJqxQ4llgI"
+      },
+      "outputs": [],
+      "source": [
+        "predictions = tf.argmax(outputs, axis=-1)\n",
+        "predictions = [tokenizer.decode(pred) for pred in predictions.numpy().tolist()]\n",
+        "predictions"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7DXC757bztJc"
+      },
+      "source": [
+        "This prediction is quite random as the model was never trained on large data in this notebook (as this notebook is not meant for doing complete training). You will get good predictions if you train this model on complete LibriSpeech dataset.\n",
+        "\n",
+        "Finally, we have reached an end to this notebook. But it's not an end of learning TensorFlow for speech-related tasks, this [repository](https://github.com/tulasiram58827/TTS_TFLite) contains some more amazing tutorials. In case you encountered any bug in this notebook, please create an issue [here](https://github.com/vasudevgupta7/gsoc-wav2vec2/issues)."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [
+        "rWk8nL6Ui-_0",
+        "wvuJL8-f0zn5",
+        "oPp18ZHRtnq-",
+        "1mvTuOXpwsQe",
+        "7Vlm3ySFULsG",
+        "CzAOI78tky08",
+        "SJfPlTgezD0i",
+        "G14o706kdTE1"
+      ],
+      "name": "wav2vec2_saved_model_finetuning.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/wiki40b_lm.ipynb b/site/en/hub/tutorials/wiki40b_lm.ipynb
new file mode 100644
index 00000000000..ad94ce0aab8
--- /dev/null
+++ b/site/en/hub/tutorials/wiki40b_lm.ipynb
@@ -0,0 +1,451 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Oxb_tjw13y4G"
+      },
+      "source": [
+        "##### Copyright 2019 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EAkh2aBJLg6q"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2019 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "owAopeOtirc9"
+      },
+      "source": [
+        "# Wiki40B Language Models\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "T-nCyGRri-KO"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub models\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "8eY9jkGpjf3d"
+      },
+      "source": [
+        "Generate Wikipedia-like text using the **Wiki40B language models** from [TensorFlow Hub](https://tfhub.dev)!\n",
+        "\n",
+        "This notebook illustrates how to:\n",
+        "* Load the 41 monolingual and 2 multilingual language models that are part of the [Wiki40b-LM collection](https://tfhub.dev/google/collections/wiki40b-lm/1) on TF-Hub\n",
+        "* Use the models to obtain perplexity, per layer activations, and word embeddings for a given piece of text\n",
+        "* Generate text token-by-token from a piece of seed text\n",
+        "\n",
+        "The language models are trained on the newly published, cleaned-up [Wiki40B dataset](https://www.tensorflow.org/datasets/catalog/wiki40b) available on TensorFlow Datasets. The training setup is based on the paper [“Wiki-40B: Multilingual Language Model Dataset”](https://research.google/pubs/pub49029/)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "wK2YnrEhLjDf"
+      },
+      "source": [
+        "## Setup"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "sv2CmI7BdaML"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Installing Dependencies\n",
+        "!pip install --quiet \"tensorflow-text==2.11.*\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "8uSkaQ-Vdon2"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Imports\n",
+        "import numpy as np\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import tensorflow_text as tf_text\n",
+        "\n",
+        "tf.disable_eager_execution()\n",
+        "tf.logging.set_verbosity(tf.logging.WARN)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d2MvP-cyL-BN"
+      },
+      "source": [
+        "## Choose Language\n",
+        "\n",
+        "Let's choose **which language model** to load from TF-Hub and the **length of text** to be generated. \n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "33zYlSXwMA_o"
+      },
+      "outputs": [],
+      "source": [
+        "#@title { run: \"auto\" }\n",
+        "language = \"en\" #@param [\"en\", \"ar\", \"zh-cn\", \"zh-tw\", \"nl\", \"fr\", \"de\", \"it\", \"ja\", \"ko\", \"pl\", \"pt\", \"ru\", \"es\", \"th\", \"tr\", \"bg\", \"ca\", \"cs\", \"da\", \"el\", \"et\", \"fa\", \"fi\", \"he\", \"hi\", \"hr\", \"hu\", \"id\", \"lt\", \"lv\", \"ms\", \"no\", \"ro\", \"sk\", \"sl\", \"sr\", \"sv\", \"tl\", \"uk\", \"vi\", \"multilingual-64k\", \"multilingual-128k\"]\n",
+        "hub_module = \"https://tfhub.dev/google/wiki40b-lm-{}/1\".format(language)\n",
+        "max_gen_len = 20 #@param\n",
+        "\n",
+        "print(\"Using the {} model to generate sequences of max length {}.\".format(hub_module, max_gen_len))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dgw2qW4xZbMj"
+      },
+      "source": [
+        "## Build the Model\n",
+        "\n",
+        "Okay, now that we've configured which pre-trained model to use, let's configure it to generate text up to `max_gen_len`. We will need to load the language model from TF-Hub, feed in a piece of starter text, and then iteratively feed in tokens as they are generated."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "pUypKuc3Mlpa"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Load the language model pieces\n",
+        "g = tf.Graph()\n",
+        "n_layer = 12\n",
+        "model_dim = 768\n",
+        "\n",
+        "with g.as_default():\n",
+        "  text = tf.placeholder(dtype=tf.string, shape=(1,))\n",
+        "\n",
+        "  # Load the pretrained model from TF-Hub\n",
+        "  module = hub.Module(hub_module)\n",
+        "\n",
+        "  # Get the word embeddings, activations at each layer, negative log likelihood\n",
+        "  # of the text, and calculate the perplexity.\n",
+        "  embeddings = module(dict(text=text), signature=\"word_embeddings\", as_dict=True)[\"word_embeddings\"]\n",
+        "  activations = module(dict(text=text), signature=\"activations\", as_dict=True)[\"activations\"]\n",
+        "  neg_log_likelihood = module(dict(text=text), signature=\"neg_log_likelihood\", as_dict=True)[\"neg_log_likelihood\"]\n",
+        "  ppl = tf.exp(tf.reduce_mean(neg_log_likelihood, axis=1))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "ZOS2Z2n0MsuC"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Construct the per-token generation graph\n",
+        "def feedforward_step(module, inputs, mems):\n",
+        "  \"\"\"Generate one step.\"\"\"\n",
+        "  # Set up the input dict for one step of generation\n",
+        "  inputs = tf.dtypes.cast(inputs, tf.int64)\n",
+        "  generation_input_dict = dict(input_tokens=inputs)\n",
+        "  mems_dict = {\"mem_{}\".format(i): mems[i] for i in range(n_layer)}\n",
+        "  generation_input_dict.update(mems_dict)\n",
+        "\n",
+        "  # Generate the tokens from the language model\n",
+        "  generation_outputs = module(generation_input_dict, signature=\"prediction\", as_dict=True)\n",
+        "\n",
+        "  # Get the probabilities and the inputs for the next steps\n",
+        "  probs = generation_outputs[\"probs\"]\n",
+        "  new_mems = [generation_outputs[\"new_mem_{}\".format(i)] for i in range(n_layer)]\n",
+        "\n",
+        "  return probs, new_mems"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "S9ss6amQMyVY"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Build the statically unrolled graph for `max_gen_len` tokens\n",
+        "with g.as_default():\n",
+        "  # Tokenization with the sentencepiece model.\n",
+        "  token_ids = module(dict(text=text), signature=\"tokenization\", as_dict=True)[\"token_ids\"]\n",
+        "  inputs_np = token_ids\n",
+        "  # Generate text by statically unrolling the computational graph\n",
+        "  mems_np = [np.zeros([1, 0, model_dim], dtype=np.float32) for _ in range(n_layer)]\n",
+        "\n",
+        "  # Generate up to `max_gen_len` tokens\n",
+        "  sampled_ids = []\n",
+        "  for step in range(max_gen_len):\n",
+        "    probs, mems_np = feedforward_step(module, inputs_np, mems_np)\n",
+        "    sampled_id = tf.random.categorical(tf.math.log(probs[0]), num_samples=1, dtype=tf.int32)\n",
+        "    sampled_id = tf.squeeze(sampled_id)\n",
+        "    sampled_ids.append(sampled_id)\n",
+        "    inputs_np = tf.reshape(sampled_id, [1, 1])\n",
+        "\n",
+        "  # Transform the ids into text\n",
+        "  sampled_ids = tf.expand_dims(sampled_ids, axis=0)\n",
+        "  generated_text = module(dict(token_ids=sampled_ids), signature=\"detokenization\", as_dict=True)[\"text\"]\n",
+        "\n",
+        "  init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "K5SYcRrxM7vS"
+      },
+      "source": [
+        "## Generate some text\n",
+        "\n",
+        "Let's generate some text! We'll set a text `seed` to prompt the language model.\n",
+        "\n",
+        "You can use one of the **predefined** seeds or _optionally_ **enter your own**. This text will be used as seed for the language model to help prompt the language model for what to generate next.\n",
+        "\n",
+        "You can use the following special tokens precede special parts of the generated article. Use **`_START_ARTICLE_`** to indicate the beginning of the article, **`_START_SECTION_`** to indicate the beginning of a section, and **`_START_PARAGRAPH_`** to generate text in the article\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "GmZxv7bzMIcL"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Predefined Seeds\n",
+        "lang_to_seed = {\"en\": \"\\n_START_ARTICLE_\\n1882 Prince Edward Island general election\\n_START_PARAGRAPH_\\nThe 1882 Prince Edward Island election was held on May 8, 1882 to elect members of the House of Assembly of the province of Prince Edward Island, Canada.\",\n",
+        "                \"ar\": \"\\n_START_ARTICLE_\\nأوليفيا كوك\\n_START_SECTION_\\nنشأتها والتعلي \\n_START_PARAGRAPH_\\nولدت أوليفيا كوك في أولدهام في مانشستر الكبرى لأسرة تتكون من أب يعمل كظابط شرطة، وأمها تعمل كممثلة مبيعات. عندما كانت صغيرة بدأت تأخذ دروساً في الباليه الجمباز. وفي المدرسة شاركت في المسرحيات المدرسية، إضافةً إلى عملها في مسرح سندريلا . وفي سن الرابعة عشر عاماً، حصلت على وكيلة لها في مانشستر وهي وقعت عقداً مع وكالة الفنانين المبدعين في مانشستر،\",\n",
+        "                \"zh-cn\": \"\\n_START_ARTICLE_\\n上尾事件\\n_START_SECTION_\\n日本国铁劳资关系恶化\\n_START_PARAGRAPH_\\n由于日本国铁财政恶化，管理层开始重整人手安排，令工会及员工感到受威胁。但日本国铁作为公营企业，其雇员均受公营企业等劳资关系法规管——该法第17条规定公营企业员工不得发动任何罢工行为。为了规避该法例\",\n",
+        "                \"zh-tw\": \"\\n_START_ARTICLE_\\n乌森\\n_START_PARAGRAPH_\\n烏森（法語：Houssen，發音：[usən]；德語：Hausen；阿爾薩斯語：Hüse）是法國上萊茵省的一個市鎮，位於該省北部，屬於科爾馬-里博維萊區（Colmar-Ribeauvillé）第二科爾馬縣（Colmar-2）。該市鎮總面積6.7平方公里，2009年時的人口為\",\n",
+        "                \"nl\": \"\\n_START_ARTICLE_\\n1001 vrouwen uit de Nederlandse geschiedenis\\n_START_SECTION_\\nSelectie van vrouwen\\n_START_PARAGRAPH_\\nDe 'oudste' biografie in het boek is gewijd aan de beschermheilige\",\n",
+        "                \"fr\": \"\\n_START_ARTICLE_\\nꝹ\\n_START_SECTION_\\nUtilisation\\n_START_PARAGRAPH_\\nLe d insulaire est utilisé comme lettre additionnelle dans l’édition de 1941 du recueil de chroniques galloises Brut y Tywysogion\",\n",
+        "                \"de\": \"\\n_START_ARTICLE_\\nÜnal Demirkıran\\n_START_SECTION_\\nLaufbahn\\n_START_PARAGRAPH_\\nDemirkıran debütierte als junges Talent am 25. September 1999 im Auswärtsspiel des SSV Ulm 1846 bei Werder Bremen (2:2) in der Bundesliga, als er kurz\",\n",
+        "                \"it\": \"\\n_START_ARTICLE_\\n28th Street (linea IRT Lexington Avenue)\\n_START_SECTION_\\nStoria\\n_START_PARAGRAPH_\\nLa stazione, i cui lavori di costruzione ebbero inizio nel 1900, venne aperta il 27 ottobre 1904, come\",\n",
+        "                \"ja\": \"\\n_START_ARTICLE_\\nしのぶ・まさみshow'05 恋してラララ\\n_START_SECTION_\\n概要\\n_START_PARAGRAPH_\\n『上海ルーキーSHOW』の打ち切り後に放送された年末特番で、同番組MCの大竹しのぶと久本雅美が恋愛にまつわるテーマでトークや音楽企画を展開していた。基本は女\",\n",
+        "                \"ko\": \"\\n_START_ARTICLE_\\n녹턴, Op. 9 (쇼팽)\\n_START_SECTION_\\n녹턴 3번 나장조\\n_START_PARAGRAPH_\\n쇼팽의 녹턴 3번은 세도막 형식인 (A-B-A)형식을 취하고 있다. 첫 부분은 알레그레토(Allegretto)의 빠르기가 지시되어 있으며 물 흐르듯이 부드럽게 전개되나\",\n",
+        "                \"pl\": \"\\n_START_ARTICLE_\\nAK-176\\n_START_SECTION_\\nHistoria\\n_START_PARAGRAPH_\\nPod koniec lat 60 XX w. w ZSRR dostrzeżono potrzebę posiadania lekkiej armaty uniwersalnej średniego kalibru o stosunkowo dużej mocy ogniowej, która\",\n",
+        "                \"pt\": \"\\n_START_ARTICLE_\\nÁcido ribonucleico\\n_START_SECTION_\\nIntermediário da transferência de informação\\n_START_PARAGRAPH_\\nEm 1957 Elliot Volkin e Lawrence Astrachan fizeram uma observação significativa. Eles descobriram que uma das mais marcantes mudanças\",\n",
+        "                \"ru\": \"\\n_START_ARTICLE_\\nАрнольд, Ремо\\n_START_SECTION_\\nКлубная карьера\\n_START_PARAGRAPH_\\nАрнольд перешёл в академию «Люцерна» в 12 лет. С 2014 года выступал за вторую команду, где провёл пятнадцать встреч. С сезона 2015/2016 находится в составе основной команды. 27 сентября 2015 года дебютировал\",\n",
+        "                \"es\": \"\\n_START_ARTICLE_\\n(200012) 2007 LK20\\n_START_SECTION_\\nDesignación y nombre\\n_START_PARAGRAPH_\\nDesignado provisionalmente como 2007 LK20.\\n_START_SECTION_\\nCaracterísticas orbitales\\n_START_PARAGRAPH_\\n2007 LK20\",\n",
+        "                \"th\": \"\\n_START_ARTICLE_\\nการนัดหยุดเรียนเพื่อภูมิอากาศ\\n_START_SECTION_\\nเกรียตา ทืนแบร์ย\\n_START_PARAGRAPH_\\nวันที่ 20 สิงหาคม 2561 เกรียตา ทืนแบร์ย นักกิจกรรมภูมิอากาศชาวสวีเดน ซึ่งขณะนั้นศึกษาอยู่ในชั้นเกรด 9 (เทียบเท่ามัธยมศึกษาปีที่ 3) ตัดสินใจไม่เข้าเรียนจนกระทั่งการเลือกตั้งทั่วไปในประเทศสวีเดนปี\",\n",
+        "                \"tr\": \"\\n_START_ARTICLE_\\nİsrail'in Muhafazakar Dostları\\n_START_SECTION_\\nFaaliyetleri\\n_START_PARAGRAPH_\\nGrubun 2005 stratejisi ile aşağıdaki faaliyet alanları tespit edilmiştir:_NEWLINE_İsrail'i destekleme\",\n",
+        "                \"bg\": \"\\n_START_ARTICLE_\\nАвтомобил с повишена проходимост\\n_START_SECTION_\\nОсобености на конструкцията\\n_START_PARAGRAPH_\\nВ исторически план леки автомобили с висока проходимост се произвеждат и имат военно\",\n",
+        "                \"ca\": \"\\n_START_ARTICLE_\\nAuchy-la-Montagne\\n_START_SECTION_\\nPoblació\\n_START_PARAGRAPH_\\nEl 2007 la població de fet d'Auchy-la-Montagne era de 469 persones. Hi havia 160 famílies de les quals 28\",\n",
+        "                \"cs\": \"\\n_START_ARTICLE_\\nŘemeslo\\n_START_PARAGRAPH_\\nŘemeslo je určitý druh manuální dovednosti, provozovaný za účelem obživy, resp. vytváření zisku. Pro řemeslné práce je charakteristický vysoký podíl ruční práce, spojený s používáním specializovaných nástrojů a pomůcek. Řemeslné práce\",\n",
+        "                \"da\": \"\\n_START_ARTICLE_\\nÖrenäs slot\\n_START_PARAGRAPH_\\nÖrenäs slot (svensk: Örenäs slott) er et slot nær Glumslöv i Landskrona stad tæt på Øresunds-kysten i Skåne i Sverige._NEWLINE_Örenäs ligger\",\n",
+        "                \"el\": \"\\n_START_ARTICLE_\\nΆλβαρο Ρεκόμπα\\n_START_SECTION_\\nΒιογραφικά στοιχεία\\n_START_PARAGRAPH_\\nΟ Άλβαρο Ρεκόμπα γεννήθηκε στις 17 Μαρτίου 1976 στο Μοντεβίδεο της Ουρουγουάης από\",\n",
+        "                \"et\": \"\\n_START_ARTICLE_\\nAus deutscher Geistesarbeit\\n_START_PARAGRAPH_\\nAus deutscher Geistesarbeit (alapealkiri Wochenblatt für wissenschaftliche und kulturelle Fragen der Gegenwart) oli ajakiri, mis 1924–1934 ilmus Tallinnas. Ajakirja andis 1932–1934\",\n",
+        "                \"fa\": \"\\n_START_ARTICLE_\\nتفسیر بغوی\\n_START_PARAGRAPH_\\nایرانی حسین بن مسعود بغوی است. این کتاب خلاصه ای از تفسیر الکشف و البیان عن تفسیر القرآن ابواسحاق احمد ثعلبی می‌باشد. این کتاب در ۴ جلد موجود می‌باش\",\n",
+        "                \"fi\": \"\\n_START_ARTICLE_\\nBovesin verilöyly\\n_START_SECTION_\\nVerilöyly\\n_START_PARAGRAPH_\\n19. syyskuuta 1943 partisaaniryhmä saapui Bovesiin tarkoituksenaan ostaa leipää kylästä. Kylässä sattui olemaan kaksi SS-miestä, jotka\",\n",
+        "                \"he\": \"\\n_START_ARTICLE_\\nאוגדה 85\\n_START_SECTION_\\nהיסטוריה\\n_START_PARAGRAPH_\\nהאוגדה הוקמה בהתחלה כמשלט העמקים בשנות השבעים. בשנות השמונים הפכה להיות אוגדה מרחבית עם שתי\",\n",
+        "                \"hi\": \"\\n_START_ARTICLE_\\nऑडी\\n_START_SECTION_\\nऑडी इंडिया\\n_START_PARAGRAPH_\\nऑडी इंडिया की स्थापना मार्च 2007 में फोक्सवैगन ग्रुप सेल्स इंडिया के एक विभाजन के रूप में की गई थी। दुनिया भर में 110\",\n",
+        "                \"hr\": \"\\n_START_ARTICLE_\\nČimariko (jezična porodica)\\n_START_PARAGRAPH_\\nChimarikan.-porodica sjevernoameričkih indijanskih jezika koja prema Powersu obuhvaća jezike Indijanaca Chimariko (Chemaŕeko) sa rijeke Trinity i Chimalakwe\",\n",
+        "                \"hu\": \"\\n_START_ARTICLE_\\nÁllami Politikai Igazgatóság\\n_START_PARAGRAPH_\\nAz Állami Politikai Igazgatóság (rövidítve: GPU, oroszul: Государственное политическое управление), majd később Egyesített Állami Politikai Igazgatóság Szovjet-Oroszország\",\n",
+        "                \"id\": \"\\n_START_ARTICLE_\\n(257195) 2008 QY41\\n_START_SECTION_\\nPembentukan\\n_START_PARAGRAPH_\\nSeperti asteroid secara keseluruhan, asteroid ini terbentuk dari nebula matahari primordial sebagai pecahan planetisimal, sesuatu di\",\n",
+        "                \"lt\": \"\\n_START_ARTICLE_\\nŠavijos–Uardigo regionas\\n_START_SECTION_\\nGeografija\\n_START_PARAGRAPH_\\nŠavijos-Uardigo regionas yra Atlanto vandenynu pakrantės lygumoje\",\n",
+        "                \"lv\": \"\\n_START_ARTICLE_\\nApatīts\\n_START_SECTION_\\nĪpašības\\n_START_PARAGRAPH_\\nApatīta kopējā ķīmiskā formula ir Ca₁₀(PO₄)₆(OH,F,Cl)₂, ir trīs atšķirīgi apatīta veidi: apatīts: Ca₁₀(PO₄)₆(OH)₂, fluorapatīts Ca₁₀(PO₄)₆(F)₂ un hlorapatīts: Ca₁₀(PO₄)₆(Cl)₂. Pēc sastāva\",\n",
+        "                \"ms\": \"\\n_START_ARTICLE_\\nEdward C. Prescott\\n_START_PARAGRAPH_\\nEdward Christian Prescott (lahir 26 Disember 1940) ialah seorang ahli ekonomi Amerika. Beliau menerima Hadiah Peringatan Nobel dalam Sains Ekonomi pada tahun 2004, berkongsi\",\n",
+        "                \"no\": \"\\n_START_ARTICLE_\\nAl-Minya\\n_START_SECTION_\\nEtymologi\\n_START_PARAGRAPH_\\nDet er sprikende forklaringer på bynavnet. Det kan komme fra gammelegyptisk Men'at Khufu, i betydning byen hvor Khufu ble ammet, noe som knytter byen til farao Khufu (Keops), som\",\n",
+        "                \"ro\": \"\\n_START_ARTICLE_\\nDealurile Cernăuțiului\\n_START_PARAGRAPH_\\nDealurile Cernăuțiului sunt un lanț deluros striat, care se întinde în partea centrală a interfluviului dintre Prut și Siret, în cadrul regiunii Cernăuți din\",\n",
+        "                \"sk\": \"\\n_START_ARTICLE_\\n10. peruť RAAF\\n_START_PARAGRAPH_\\n10. peruť RAAF je námorná hliadkovacia peruť kráľovských austrálskych vzdušných síl (Royal Australian Air Force – RAAF) založená na základni Edinburgh v Južnej Austrálii ako súčasť 92\",\n",
+        "                \"sl\": \"\\n_START_ARTICLE_\\n105 Artemida\\n_START_SECTION_\\nOdkritje\\n_START_PARAGRAPH_\\nAsteroid je 16. septembra 1868 odkril James Craig Watson (1838 – 1880). Poimenovan je po Artemidi, boginji Lune iz grške\",\n",
+        "                \"sr\": \"\\n_START_ARTICLE_\\nЉанос Морелос 1. Сексион (Истапангахоја)\\n_START_SECTION_\\nСтановништво\\n_START_PARAGRAPH_\\nПрема подацима из 2010. године у насељу је живело 212\",\n",
+        "                \"sv\": \"\\n_START_ARTICLE_\\nÖstra Torps landskommun\\n_START_SECTION_\\nAdministrativ historik\\n_START_PARAGRAPH_\\nKommunen bildades i Östra Torps socken i Vemmenhögs härad i Skåne när 1862 års kommunalförordningar trädde i kraft. _NEWLINE_Vid kommunreformen\",\n",
+        "                \"tl\": \"\\n_START_ARTICLE_\\nBésame Mucho\\n_START_PARAGRAPH_\\nAng Bésame Mucho ay isang awit na nasa Kastila. Isinulat ito ng Mehikanang si Consuelo Velázquez noong 1940, bago sumapit ang kanyang ika-16 na\",\n",
+        "                \"uk\": \"\\n_START_ARTICLE_\\nІслам та інші релігії\\n_START_PARAGRAPH_\\nПротягом багатовікової ісламської історії мусульманські правителі, ісламські вчені і звичайні мусульмани вступали у різні відносини з представниками інших релігій. Стиль цих\",\n",
+        "                \"vi\": \"\\n_START_ARTICLE_\\nĐường tỉnh 316\\n_START_PARAGRAPH_\\nĐường tỉnh 316 hay tỉnh lộ 316, viết tắt ĐT316 hay TL316, là đường tỉnh ở các huyện Thanh Sơn, Thanh Thủy, Tam Nông tỉnh Phú Thọ ._NEWLINE_ĐT316 bắt đầu từ xã Tinh Nhuệ\",\n",
+        "                \"multilingual-64k\": \"\\n_START_ARTICLE_\\n1882 Prince Edward Island general election\\n_START_PARAGRAPH_\\nThe 1882 Prince Edward Island election was held on May 8, 1882 to elect members of the House of Assembly of the province of Prince Edward Island, Canada.\",\n",
+        "                \"multilingual-128k\": \"\\n_START_ARTICLE_\\n1882 Prince Edward Island general election\\n_START_PARAGRAPH_\\nThe 1882 Prince Edward Island election was held on May 8, 1882 to elect members of the House of Assembly of the province of Prince Edward Island, Canada.\"}\n",
+        "\n",
+        "seed = lang_to_seed[language]"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "mZDGsSyUM_Mg"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Enter your own seed (Optional).\n",
+        "user_seed = \"\" #@param { type: \"string\" }\n",
+        "if user_seed.strip():\n",
+        "  seed = user_seed.strip()\n",
+        "\n",
+        "# The seed must start with \"_START_ARTICLE_\" or the generated text will be gibberish\n",
+        "START_ARTICLE = \"_START_ARTICLE_\"\n",
+        "if START_ARTICLE not in seed:\n",
+        "  seed = \"\\n{}\\n{}\".format(START_ARTICLE, seed)\n",
+        "\n",
+        "print(\"Generating text from seed:\\n{}\".format(seed))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "5dMuShi3XuLd"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Initialize session.\n",
+        "with tf.Session(graph=g).as_default() as session:\n",
+        "  session.run(init_op)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "aS53xjmbbw0Z"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Generate text\n",
+        "\n",
+        "with session.as_default():\n",
+        "  results = session.run([embeddings, neg_log_likelihood, ppl, activations, token_ids, generated_text], feed_dict={text: [seed]})\n",
+        "  embeddings_result, neg_log_likelihood_result, ppl_result, activations_result, token_ids_result, generated_text_result = results\n",
+        "  generated_text_output = generated_text_result[0].decode('utf-8')\n",
+        "\n",
+        "print(generated_text_output)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tjQf3N1wdND0"
+      },
+      "source": [
+        "We can also look at the other outputs of the model - the perplexity, the token ids, the intermediate activations, and the embeddings"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pGfw3CQWNC_n"
+      },
+      "outputs": [],
+      "source": [
+        "ppl_result"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "FLlgJObFNEmj"
+      },
+      "outputs": [],
+      "source": [
+        "token_ids_result"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5SaH36M-NGXc"
+      },
+      "outputs": [],
+      "source": [
+        "activations_result.shape"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "k9Eb_DPfQdUu"
+      },
+      "outputs": [],
+      "source": [
+        "embeddings_result"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "wiki40b_lm.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/hub/tutorials/yamnet.ipynb b/site/en/hub/tutorials/yamnet.ipynb
new file mode 100644
index 00000000000..e6c9fbca5a1
--- /dev/null
+++ b/site/en/hub/tutorials/yamnet.ipynb
@@ -0,0 +1,359 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "laa9tRjJ59bl"
+      },
+      "source": [
+        "##### Copyright 2020 The TensorFlow Hub Authors.\n",
+        "\n",
+        "Licensed under the Apache License, Version 2.0 (the \"License\");"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "T4ZHtBpK6Dom"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Copyright 2020 The TensorFlow Hub Authors. All Rights Reserved.\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License.\n",
+        "# =============================================================================="
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hk5u_9KN1m-t"
+      },
+      "source": [
+        "\n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "  \n",
+        "
    \n",
+        "    View on TensorFlow.org\n",
+        "  \n",
+        "    Run in Google Colab\n",
+        "  \n",
+        "    View on GitHub\n",
+        "  \n",
+        "    Download notebook\n",
+        "  \n",
+        "    See TF Hub model\n",
+        "  
    "
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "x2ep-q7k_5R-"
+      },
+      "source": [
+        "# Sound classification with YAMNet\n",
+        "\n",
+        "YAMNet is a deep net that predicts 521 audio event [classes](https://github.com/tensorflow/models/blob/master/research/audioset/yamnet/yamnet_class_map.csv) from the [AudioSet-YouTube corpus](http://g.co/audioset) it was trained on. It employs the\n",
+        "[Mobilenet_v1](https://arxiv.org/pdf/1704.04861.pdf) depthwise-separable\n",
+        "convolution architecture."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Bteu7pfkpt_f"
+      },
+      "outputs": [],
+      "source": [
+        "import tensorflow as tf\n",
+        "import tensorflow_hub as hub\n",
+        "import numpy as np\n",
+        "import csv\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "from IPython.display import Audio\n",
+        "from scipy.io import wavfile"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YSVs3zRrrYmY"
+      },
+      "source": [
+        "Load the Model from TensorFlow Hub.\n",
+        "\n",
+        "Note: to read the documentation just follow the model's [url](https://tfhub.dev/google/yamnet/1)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "VX8Vzs6EpwMo"
+      },
+      "outputs": [],
+      "source": [
+        "# Load the model.\n",
+        "model = hub.load('https://tfhub.dev/google/yamnet/1')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lxWx6tOdtdBP"
+      },
+      "source": [
+        "The labels file will be loaded from the models assets and is present at `model.class_map_path()`.\n",
+        "You will load it on the `class_names` variable."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EHSToAW--o4U"
+      },
+      "outputs": [],
+      "source": [
+        "# Find the name of the class with the top score when mean-aggregated across frames.\n",
+        "def class_names_from_csv(class_map_csv_text):\n",
+        "  \"\"\"Returns list of class names corresponding to score vector.\"\"\"\n",
+        "  class_names = []\n",
+        "  with tf.io.gfile.GFile(class_map_csv_text) as csvfile:\n",
+        "    reader = csv.DictReader(csvfile)\n",
+        "    for row in reader:\n",
+        "      class_names.append(row['display_name'])\n",
+        "\n",
+        "  return class_names\n",
+        "\n",
+        "class_map_path = model.class_map_path().numpy()\n",
+        "class_names = class_names_from_csv(class_map_path)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "mSFjRwkZ59lU"
+      },
+      "source": [
+        "Add a method to verify and convert a loaded audio is on the proper sample_rate (16K), otherwise it would affect the model's results."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LizGwWjc5w6A"
+      },
+      "outputs": [],
+      "source": [
+        "def ensure_sample_rate(original_sample_rate, waveform,\n",
+        "                       desired_sample_rate=16000):\n",
+        "  \"\"\"Resample waveform if required.\"\"\"\n",
+        "  if original_sample_rate != desired_sample_rate:\n",
+        "    desired_length = int(round(float(len(waveform)) /\n",
+        "                               original_sample_rate * desired_sample_rate))\n",
+        "    waveform = scipy.signal.resample(waveform, desired_length)\n",
+        "  return desired_sample_rate, waveform"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AZEgCobA9bWl"
+      },
+      "source": [
+        "## Downloading and preparing the sound file\n",
+        "\n",
+        "Here you will download a wav file and listen to it.\n",
+        "If you have a file already available, just upload it to colab and use it instead.\n",
+        "\n",
+        "Note: The expected audio file should be a mono wav file at 16kHz sample rate."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WzZHvyTtsJrc"
+      },
+      "outputs": [],
+      "source": [
+        "!curl -O https://storage.googleapis.com/audioset/speech_whistling2.wav"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "D8LKmqvGzZzr"
+      },
+      "outputs": [],
+      "source": [
+        "!curl -O https://storage.googleapis.com/audioset/miaow_16k.wav"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Wo9KJb-5zuz1"
+      },
+      "outputs": [],
+      "source": [
+        "# wav_file_name = 'speech_whistling2.wav'\n",
+        "wav_file_name = 'miaow_16k.wav'\n",
+        "sample_rate, wav_data = wavfile.read(wav_file_name, 'rb')\n",
+        "sample_rate, wav_data = ensure_sample_rate(sample_rate, wav_data)\n",
+        "\n",
+        "# Show some basic information about the audio.\n",
+        "duration = len(wav_data)/sample_rate\n",
+        "print(f'Sample rate: {sample_rate} Hz')\n",
+        "print(f'Total duration: {duration:.2f}s')\n",
+        "print(f'Size of the input: {len(wav_data)}')\n",
+        "\n",
+        "# Listening to the wav file.\n",
+        "Audio(wav_data, rate=sample_rate)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "P9I290COsMBm"
+      },
+      "source": [
+        "The `wav_data` needs to be normalized to values in `[-1.0, 1.0]` (as stated in the model's [documentation](https://tfhub.dev/google/yamnet/1))."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bKr78aCBsQo3"
+      },
+      "outputs": [],
+      "source": [
+        "waveform = wav_data / tf.int16.max"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e_Xwd4GPuMsB"
+      },
+      "source": [
+        "## Executing the Model\n",
+        "\n",
+        "Now the easy part: using the data already prepared, you just call the model and get the: scores, embedding and the spectrogram.\n",
+        "\n",
+        "The score is the main result you will use.\n",
+        "The spectrogram you will use to do some visualizations later."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BJGP6r-At_Jc"
+      },
+      "outputs": [],
+      "source": [
+        "# Run the model, check the output.\n",
+        "scores, embeddings, spectrogram = model(waveform)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Vmo7griQprDk"
+      },
+      "outputs": [],
+      "source": [
+        "scores_np = scores.numpy()\n",
+        "spectrogram_np = spectrogram.numpy()\n",
+        "infered_class = class_names[scores_np.mean(axis=0).argmax()]\n",
+        "print(f'The main sound is: {infered_class}')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Uj2xLf-P_ndS"
+      },
+      "source": [
+        "## Visualization\n",
+        "\n",
+        "YAMNet also returns some additional information that we can use for visualization.\n",
+        "Let's take a look on the Waveform, spectrogram and the top classes inferred."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_QSTkmv7wr2M"
+      },
+      "outputs": [],
+      "source": [
+        "plt.figure(figsize=(10, 6))\n",
+        "\n",
+        "# Plot the waveform.\n",
+        "plt.subplot(3, 1, 1)\n",
+        "plt.plot(waveform)\n",
+        "plt.xlim([0, len(waveform)])\n",
+        "\n",
+        "# Plot the log-mel spectrogram (returned by the model).\n",
+        "plt.subplot(3, 1, 2)\n",
+        "plt.imshow(spectrogram_np.T, aspect='auto', interpolation='nearest', origin='lower')\n",
+        "\n",
+        "# Plot and label the model output scores for the top-scoring classes.\n",
+        "mean_scores = np.mean(scores, axis=0)\n",
+        "top_n = 10\n",
+        "top_class_indices = np.argsort(mean_scores)[::-1][:top_n]\n",
+        "plt.subplot(3, 1, 3)\n",
+        "plt.imshow(scores_np[:, top_class_indices].T, aspect='auto', interpolation='nearest', cmap='gray_r')\n",
+        "\n",
+        "# patch_padding = (PATCH_WINDOW_SECONDS / 2) / PATCH_HOP_SECONDS\n",
+        "# values from the model documentation\n",
+        "patch_padding = (0.025 / 2) / 0.01\n",
+        "plt.xlim([-patch_padding-0.5, scores.shape[0] + patch_padding-0.5])\n",
+        "# Label the top_N classes.\n",
+        "yticks = range(0, top_n, 1)\n",
+        "plt.yticks(yticks, [class_names[top_class_indices[x]] for x in yticks])\n",
+        "_ = plt.ylim(-0.5 + np.array([top_n, 0]))"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "yamnet.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/install/_index.yaml b/site/en/install/_index.yaml
index 57eff0ea1d8..71bc660f81d 100644
--- a/site/en/install/_index.yaml
+++ b/site/en/install/_index.yaml
@@ -1,9 +1,14 @@
 project_path: /_project.yaml
 book_path: /_book.yaml
-description: 
+title: Install TensorFlow 2
 landing_page:
   custom_css_path: /site-assets/css/style.css
   nav: both
+  meta_tags:
+  - name: description
+    content: >
+      Learn how to install TensorFlow on your system. Download a pip package, run in a Docker
+      container, or build from source. Enable the GPU on supported cards.
   rows:
   - classname:
       devsite-landing-row-large-headings
@@ -15,13 +20,14 @@ landing_page:
         
    
           
    
             
      
+              
    • Python 3.9–3.12
      • 
               
    • Ubuntu 16.04 or later
      • 
-              
    • Windows 7 or later
      • 
+              
    • Windows 7 or later (with C++ redistributable)
      • 
             
    
               		
             
      
               
    • macOS 10.12.6 (Sierra) or later (no GPU support)
      • 
-              
    • Raspbian 9.0 or later
      • 
+              
    • WSL2 via Windows 10 19044 or higher including GPUs (Experimental)
      • 
             
    
           
    
         
    
@@ -33,9 +39,8 @@ landing_page:
     - heading: Download a package
       description: >
         
    Install TensorFlow with Python's pip package manager.
    
-        
-        
    Official packages available for Ubuntu, Windows, macOS, and the Raspberry Pi.
    
-        
    See the GPU guide for CUDA®-enabled cards.
    
+        
+        
    Official packages available for Ubuntu, Windows, and macOS.
    
 
       buttons:
       - label: Read the pip install guide
@@ -46,9 +51,11 @@ landing_page:
         
             # Requires the latest pip
         pip install --upgrade pip
    
-        # Current stable release for CPU-only
+        # Current stable release for CPU
         pip install tensorflow
    
-        # Or preview build for CPU/GPU (unstable)
+        # Current stable release for GPU (Linux / WSL2)
+        pip install tensorflow[and-cuda]
    
+        # Or try the preview build (unstable)
         pip install tf-nightly
    
 
   - classname:
@@ -61,12 +68,11 @@ landing_page:
           The TensorFlow
           Docker images are already configured to run TensorFlow. A
           Docker container runs in a
-          virtual environment and is the easiest way to set up GPU
-          support.
+          virtual environment and is the easiest way to set up GPU support.
         
    
         
    -        docker pull tensorflow/tensorflow                  # Download latest image
    
-        docker run -it -p 8888:8888 tensorflow/tensorflow  # Start a Jupyter notebook server
+        docker pull tensorflow/tensorflow:latest  # Download latest stable image
    
+        docker run -it -p 8888:8888 tensorflow/tensorflow:latest-jupyter  # Start Jupyter server
         
    
       buttons:
       - label: Read the Docker install guide
@@ -97,7 +103,7 @@ landing_page:
       path: https://js.tensorflow.org
       description: >
         TensorFlow.js is a WebGL accelerated, JavaScript library to train and
-        deploy ML models in the browser and for Node.js.
+        deploy ML models in the browser, Node.js, mobile, and more.
     - classname: tfo-landing-row-item-inset-white
       heading: Mobile developers
       path: /lite/
diff --git a/site/en/install/_toc.yaml b/site/en/install/_toc.yaml
index 1d07f4a4092..26cdb270bb8 100644
--- a/site/en/install/_toc.yaml
+++ b/site/en/install/_toc.yaml
@@ -7,8 +7,8 @@ toc:
 - title: Docker
   path: /install/docker
 - heading: Additional setup
-- title: GPU support
-  path: /install/gpu
+- title: GPU device plugins
+  path: /install/gpu_plugins
 - title: Problems
   path: /install/errors
 - heading: Build from source
@@ -16,12 +16,18 @@ toc:
   path: /install/source
 - title: Windows
   path: /install/source_windows
-- title: Raspberry Pi
-  path: /install/source_rpi
+- title: SIG Build
+  path: https://github.com/tensorflow/build
+  status: external
 - heading: Language bindings
 - title: Java
-  path: /install/lang_java
+  path: /jvm/install
+  status: external
+- title: Java (legacy)
+  status: deprecated
+  path: /install/lang_java_legacy
 - title: C
   path: /install/lang_c
 - title: Go
-  path: /install/lang_go
+  path: https://github.com/tensorflow/build/tree/master/golang_install_guide
+  status: external
diff --git a/site/en/install/docker.md b/site/en/install/docker.md
index c2d816eb20d..376ca0820a7 100644
--- a/site/en/install/docker.md
+++ b/site/en/install/docker.md
@@ -1,51 +1,49 @@
 # Docker
 
-[Docker](https://docs.docker.com/install/){:.external} uses *containers* to
+[Docker](https://docs.docker.com/install/) uses *containers* to
 create virtual environments that isolate a TensorFlow installation from the rest
 of the system. TensorFlow programs are run *within* this virtual environment that
 can share resources with its host machine (access directories, use the GPU,
 connect to the Internet, etc.). The
-[TensorFlow Docker images](https://hub.docker.com/r/tensorflow/tensorflow/){:.external}
+[TensorFlow Docker images](https://hub.docker.com/r/tensorflow/tensorflow/)
 are tested for each release.
 
-Docker is the easiest way to enable TensorFlow [GPU support](./gpu.md) on Linux since only the
-[NVIDIA® GPU driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver){:.external}
+Docker is the easiest way to enable TensorFlow [GPU support](./pip.md) on Linux since only the
+[NVIDIA® GPU driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver)
 is required on the *host* machine (the *NVIDIA® CUDA® Toolkit* does not need to
 be installed).
 
 
 ## TensorFlow Docker requirements
 
-1. [Install Docker](https://docs.docker.com/install/){:.external} on
+1. [Install Docker](https://docs.docker.com/install/) on
    your local *host* machine.
-2. For GPU support on Linux, [install NVIDIA Docker support](https://github.com/NVIDIA/nvidia-docker){:.external}.
+2. For GPU support on Linux, [install NVIDIA Docker support](https://github.com/NVIDIA/nvidia-container-toolkit).
    * Take note of your Docker version with `docker -v`. Versions __earlier than__ 19.03 require nvidia-docker2 and the `--runtime=nvidia` flag. On versions __including and after__ 19.03, you will use the `nvidia-container-toolkit` package and the `--gpus all` flag. Both options are documented on the page linked above.
 
 Note: To run the `docker` command without `sudo`, create the `docker` group and
 add your user. For details, see the
-[post-installation steps for Linux](https://docs.docker.com/install/linux/linux-postinstall/){:.external}.
+[post-installation steps for Linux](https://docs.docker.com/install/linux/linux-postinstall/).
 
 
 ## Download a TensorFlow Docker image
 
 The official TensorFlow Docker images are located in the 
-[tensorflow/tensorflow](https://hub.docker.com/r/tensorflow/tensorflow/){:.external}
-Docker Hub repository. Image releases [are tagged](https://hub.docker.com/r/tensorflow/tensorflow/tags/){:.external}
+[tensorflow/tensorflow](https://hub.docker.com/r/tensorflow/tensorflow/)
+Docker Hub repository. Image releases [are tagged](https://hub.docker.com/r/tensorflow/tensorflow/tags/)
 using the following format:
 
-| Tag         | Description                                                                                       |
-| ---         | ---                                                                                               |
-| `latest`    | The latest release of TensorFlow CPU binary image. Default.                                       |
-| `nightly`   | Nightly builds of the TensorFlow image. (unstable)                                                |
-| *`version`* | Specify the *version* of the TensorFlow binary image, for example: *2.0.0*                       |
-| `devel`     | Nightly builds of a TensorFlow `master` development environment. Includes TensorFlow source code. |
+| Tag         | Description                                                                                                          |
+|-------------|----------------------------------------------------------------------------------------------------------------------|
+| `latest`    | The latest release of TensorFlow CPU binary image. Default.                                                          |
+| `nightly`   | Nightly builds of the TensorFlow image. (Unstable.)                                                                  |
+| *`version`* | Specify the *version* of the TensorFlow binary image, for example\: *2.8.3*                                          |
 
 Each base *tag* has variants that add or change functionality:
 
 | Tag Variants      | Description                                                                       |
 | ---               | ---                                                                               |
 | *`tag`*`-gpu`     | The specified *tag* release with GPU support. ([See below](#gpu_support))         |
-| *`tag`*`-py3`     | The specified *tag* release with Python 3 support.                                |
 | *`tag`*`-jupyter` | The specified *tag* release with Jupyter (includes TensorFlow tutorial notebooks) |
 
 You can use multiple variants at once. For example, the following downloads
@@ -66,7 +64,7 @@ To start a TensorFlow-configured container, use the following command form:
 docker run [-it] [--rm] [-p hostPort:containerPort] tensorflow/tensorflow[:tag] [command]
 
    
 
-For details, see the [docker run reference](https://docs.docker.com/engine/reference/run/){:.external}.
+For details, see the [docker run reference](https://docs.docker.com/engine/reference/run/).
 
 ### Examples using CPU-only images
 
@@ -100,11 +98,11 @@ docker run -it --rm -v $PWD:/tmp -w /tmp tensorflow/tensorflow python ./script.p
 Permission issues can arise when files created within a container are exposed to
 the host. It's usually best to edit files on the host system.
 
-Start a [Jupyter Notebook](https://jupyter.org/){:.external} server using
-TensorFlow's nightly build with Python 3 support:
+Start a [Jupyter Notebook](https://jupyter.org/) server using
+TensorFlow's nightly build:
 
 
    -docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-py3-jupyter
+docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-jupyter
 
    
 
 Follow the instructions and open the URL in your host web browser:
@@ -114,13 +112,13 @@ Follow the instructions and open the URL in your host web browser:
 ## GPU support
 
 Docker is the easiest way to run TensorFlow on a GPU since the *host* machine
-only requires the [NVIDIA® driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver){:.external}
+only requires the [NVIDIA® driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver)
 (the *NVIDIA® CUDA® Toolkit* is not required).
 
-Install the [Nvidia Container Toolkit](https://github.com/NVIDIA/nvidia-docker/blob/master/README.md#quickstart){:.external} 
+Install the [Nvidia Container Toolkit](https://github.com/NVIDIA/nvidia-docker/blob/master/README.md#quickstart) 
 to add NVIDIA® GPU support to Docker. `nvidia-container-runtime` is only
 available for Linux. See the `nvidia-container-runtime` 
-[platform support FAQ](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#platform-support){:.external}
+[platform support FAQ](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#platform-support)
 for details.
 
 Check if a GPU is available:
@@ -132,7 +130,7 @@ lspci | grep -i nvidia
 Verify your `nvidia-docker` installation:
 
 
    -docker run --gpus all --rm nvidia/cuda nvidia-smi
+docker run --rm --runtime=nvidia --gpus all ubuntu nvidia-smi
 
    
 
 Note: `nvidia-docker` v2 uses `--runtime=nvidia` instead of `--gpus all`. `nvidia-docker` v1 uses the `nvidia-docker` alias, 
@@ -147,7 +145,7 @@ docker run --gpus all -it --rm tensorflow/tensorflow:latest-gpu \
    python -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
    -It can take a while to set up the GPU-enabled image. If repeatably running +It can take a while to set up the GPU-enabled image. If repeatedly running GPU-based scripts, you can use `docker exec` to reuse a container. Use the latest TensorFlow GPU image to start a `bash` shell session in the container: @@ -156,4 +154,5 @@ Use the latest TensorFlow GPU image to start a `bash` shell session in the conta docker run --gpus all -it tensorflow/tensorflow:latest-gpu bash -Success: TensorFlow is now installed. Read the [tutorials](../tutorials) to get started. +Success: TensorFlow is now installed. Read the [tutorials](../tutorials) to get +started. diff --git a/site/en/install/errors.md b/site/en/install/errors.md index 4ce900ff521..938ba8b454f 100644 --- a/site/en/install/errors.md +++ b/site/en/install/errors.md @@ -1,8 +1,9 @@ # Build and install error messages -TensorFlow uses [GitHub issues](https://github.com/tensorflow/tensorflow/issues){:.external} -and [Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow){:.external} -to track and document build and installation problems. +TensorFlow uses [GitHub issues](https://github.com/tensorflow/tensorflow/issues), +[Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow) and +[TensorFlow Forum](https://discuss.tensorflow.org/c/general-discussion/6) +to track, document, and discuss build and installation problems. The following list links error messages to a solution or discussion. If you find an installation or build problem that is not listed, please search the GitHub @@ -11,6 +12,14 @@ question on Stack Overflow with the `tensorflow` tag. + + + - + + +CodedInputStream::SetTotalBytesLimit() in google/protobuf/io/coded_stream.h. @@ -52,7 +61,7 @@ unzip: cannot find zipfile directory in one of ./bazel-bin/tensorflow/tools/pip_ @@ -103,7 +112,7 @@ ImportError: cannot import name 'descriptor' @@ -218,7 +227,7 @@ ImportError: cannot import name 'descriptor' diff --git a/site/en/install/gpu.md b/site/en/install/gpu.md deleted file mode 100644 index 8558234ae45..00000000000 --- a/site/en/install/gpu.md +++ /dev/null @@ -1,202 +0,0 @@ -# GPU support - -Note: GPU support is available for Ubuntu and Windows with CUDA®-enabled cards. - -TensorFlow GPU support requires an assortment of drivers and libraries. To -simplify installation and avoid library conflicts, we recommend using a -[TensorFlow Docker image with GPU support](./docker.md) (Linux only). This setup -only requires the [NVIDIA® GPU drivers](https://www.nvidia.com/drivers){:.external}. - -These install instructions are for the latest release of TensorFlow. See the -[tested build configurations](./source.md#linux) for CUDA and cuDNN versions to use -with older TensorFlow releases. - -## Pip package - -See the [pip install guide](./pip) for available packages, systems -requirements, and instructions. To `pip` install a TensorFlow package with -GPU support, choose a stable or development package: - -
    -pip install tensorflow-gpu  # stable
-
-pip install tf-nightly      # preview
-
    - -### Older versions of TensorFlow - -For the 1.15 release, CPU and GPU support are included in a single package: - -
    -pip install --pre "tensorflow==1.15.*"
-
    - -For releases 1.14 and older, CPU and GPU packages are separate: - -
    -pip install tensorflow==1.14      # CPU
-pip install tensorflow-gpu==1.14  # GPU
-
    - -## Hardware requirements - -The following GPU-enabled devices are supported: - -* NVIDIA® GPU card with CUDA® Compute Capability 3.5 or higher. See the list of - [CUDA-enabled GPU cards](https://developer.nvidia.com/cuda-gpus){:.external}. - - -## Software requirements - -The following NVIDIA® software must be installed on your system: - -* [NVIDIA® GPU drivers](https://www.nvidia.com/drivers){:.external} —CUDA 10.0 requires 410.x or higher. -* [CUDA® Toolkit](https://developer.nvidia.com/cuda-toolkit-archive){:.external} - —TensorFlow supports CUDA 10.0 (TensorFlow >= 1.13.0) -* [CUPTI](http://docs.nvidia.com/cuda/cupti/){:.external} ships with the CUDA Toolkit. -* [cuDNN SDK](https://developer.nvidia.com/cudnn){:.external} (>= 7.4.1) -* *(Optional)* [TensorRT 5.0](https://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html){:.external} - to improve latency and throughput for inference on some models. - - -## Linux setup - -The `apt` instructions below are the easiest way to install the required NVIDIA -software on Ubuntu. However, if [building TensorFlow from source](./source.md), -manually install the software requirements listed above, and consider using a -`-devel` [TensorFlow Docker image](./docker.md) as a base. - -Install [CUPTI](http://docs.nvidia.com/cuda/cupti/){:.external} which ships with -the CUDA® Toolkit. Append its installation directory to the `$LD_LIBRARY_PATH` -environmental variable: - -
    -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64
-
    - -For a GPU with CUDA Compute Capability 3.0, or different versions of the -NVIDIA libraries, see the [Linux build from source](./source.md) guide. - -### Install CUDA with apt - -This section shows how to install CUDA 10 (TensorFlow >= 1.13.0) and CUDA 9 -for Ubuntu 16.04 and 18.04. These instructions may work for other Debian-based -distros. - -Caution: [Secure Boot](https://wiki.ubuntu.com/UEFI/SecureBoot){:.external} -complicates installation of the NVIDIA driver and is beyond the scope of these instructions. - - -#### Ubuntu 18.04 (CUDA 10) - -
    -# Add NVIDIA package repositories
-wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb
-sudo dpkg -i cuda-repo-ubuntu1804_10.0.130-1_amd64.deb
-sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
-sudo apt-get update
-wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
-sudo apt install ./nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb
-sudo apt-get update
-
-# Install NVIDIA driver
-sudo apt-get install --no-install-recommends nvidia-driver-418
-# Reboot. Check that GPUs are visible using the command: nvidia-smi
-
-# Install development and runtime libraries (~4GB)
-sudo apt-get install --no-install-recommends \
-    cuda-10-0 \
-    libcudnn7=7.6.2.24-1+cuda10.0  \
-    libcudnn7-dev=7.6.2.24-1+cuda10.0
-
-
-# Install TensorRT. Requires that libcudnn7 is installed above.
-sudo apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.0 \
-    libnvinfer-dev=5.1.5-1+cuda10.0
-
-
    - - -#### Ubuntu 16.04 (CUDA 10) - -
    -# Add NVIDIA package repositories
-# Add HTTPS support for apt-key
-sudo apt-get install gnupg-curl
-wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_10.0.130-1_amd64.deb
-sudo dpkg -i cuda-repo-ubuntu1604_10.0.130-1_amd64.deb
-sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub
-sudo apt-get update
-wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb
-sudo apt install ./nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb
-sudo apt-get update
-
-# Install NVIDIA driver
-# Issue with driver install requires creating /usr/lib/nvidia
-sudo mkdir /usr/lib/nvidia
-sudo apt-get install --no-install-recommends nvidia-418
-# Reboot. Check that GPUs are visible using the command: nvidia-smi
-
-# Install development and runtime libraries (~4GB)
-sudo apt-get install --no-install-recommends \
-    cuda-10-0 \
-    libcudnn7=7.6.2.24-1+cuda10.0  \
-    libcudnn7-dev=7.6.2.24-1+cuda10.0
-
-
-# Install TensorRT. Requires that libcudnn7 is installed above.
-sudo apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.0 \
-    libnvinfer-dev=5.1.5-1+cuda10.0
-
-
    - - -#### Ubuntu 16.04 (CUDA 9.0 for TensorFlow < 1.13.0) - -
    -# Add NVIDIA package repository
-sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub
-wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_9.1.85-1_amd64.deb
-sudo apt install ./cuda-repo-ubuntu1604_9.1.85-1_amd64.deb
-wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb
-sudo apt install ./nvidia-machine-learning-repo-ubuntu1604_1.0.0-1_amd64.deb
-sudo apt update
-
-# Install the NVIDIA driver
-# Issue with driver install requires creating /usr/lib/nvidia
-sudo mkdir /usr/lib/nvidia
-sudo apt-get install --no-install-recommends nvidia-410
-# Reboot. Check that GPUs are visible using the command: nvidia-smi
-
-# Install CUDA and tools. Include optional NCCL 2.x
-sudo apt install cuda9.0 cuda-cublas-9-0 cuda-cufft-9-0 cuda-curand-9-0 \
-    cuda-cusolver-9-0 cuda-cusparse-9-0 libcudnn7=7.2.1.38-1+cuda9.0 \
-    libnccl2=2.2.13-1+cuda9.0 cuda-command-line-tools-9-0
-
-# Optional: Install the TensorRT runtime (must be after CUDA install)
-sudo apt update
-sudo apt install libnvinfer4=4.1.2-1+cuda9.0
-
    - - -## Windows setup - -See the [hardware requirements](#hardware_requirements) and -[software requirements](#software_requirements) listed above. Read the -[CUDA® install guide for Windows](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/){:.external}. - -Make sure the installed NVIDIA software packages match the versions listed above. In -particular, TensorFlow will not load without the `cuDNN64_7.dll` file. To use a -different version, see the [Windows build from source](./source_windows.md) guide. - -Add the CUDA, CUPTI, and cuDNN installation directories to the `%PATH%` -environmental variable. For example, if the CUDA Toolkit is installed to -`C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0` and cuDNN to -`C:\tools\cuda`, update your `%PATH%` to match: - -
    -SET PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0\bin;%PATH%
-SET PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0\extras\CUPTI\libx64;%PATH%
-SET PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0\include;%PATH%
-SET PATH=C:\tools\cuda\bin;%PATH%
-
    diff --git a/site/en/install/gpu_plugins.md b/site/en/install/gpu_plugins.md new file mode 100644 index 00000000000..39e3cf09b29 --- /dev/null +++ b/site/en/install/gpu_plugins.md @@ -0,0 +1,80 @@ +# GPU device plugins + +Note: This page is for non-NVIDIA® GPU devices. For NVIDIA® GPU support, go to +the [Install TensorFlow with pip](./pip.md) guide. + +TensorFlow's +[pluggable device](https://github.com/tensorflow/community/blob/master/rfcs/20200624-pluggable-device-for-tensorflow.md) +architecture adds new device support as separate plug-in packages that are +installed alongside the official TensorFlow package. + +The mechanism requires no device-specific changes in the TensorFlow code. It +relies on C APIs to communicate with the TensorFlow binary in a stable manner. +Plug-in developers maintain separate code repositories and distribution packages +for their plugins and are responsible for testing their devices. + +## Use device plugins + +To use a particular device, like one would a native device in TensorFlow, users +only have to install the device plug-in package for that device. The following +code snippet shows how the plugin for a new demonstration device, *Awesome +Processing Unit (APU)*, is installed and used. For simplicity, this sample APU +plug-in only has one custom kernel for ReLU: + +```sh +# Install the APU example plug-in package +$ pip install tensorflow-apu-0.0.1-cp36-cp36m-linux_x86_64.whl +... +Successfully installed tensorflow-apu-0.0.1 +``` + +With the plug-in installed, test that the device is visible and run an operation +on the new APU device: + +```python +import tensorflow as tf # TensorFlow registers PluggableDevices here. +tf.config.list_physical_devices() # APU device is visible to TensorFlow. +[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:APU:0', device_type='APU')] + +a = tf.random.normal(shape=[5], dtype=tf.float32) # Runs on CPU. +b = tf.nn.relu(a) # Runs on APU. + +with tf.device("/APU:0"): # Users can also use 'with tf.device' syntax. + c = tf.nn.relu(a) # Runs on APU. + +with tf.device("/CPU:0"): + c = tf.nn.relu(a) # Runs on CPU. + +@tf.function # Defining a tf.function +def run(): + d = tf.random.uniform(shape=[100], dtype=tf.float32) # Runs on CPU. + e = tf.nn.relu(d) # Runs on APU. + +run() # PluggableDevices also work with tf.function and graph mode. +``` + +## Available devices + +Metal `PluggableDevice` for macOS GPUs: + +* Works with TF 2.5 or later. +* [Getting started guide](https://developer.apple.com/metal/tensorflow-plugin/). +* For questions and feedback, please visit the + [Apple Developer Forum](https://developer.apple.com/forums/tags/tensorflow-metal). + +DirectML `PluggableDevice` for Windows and WSL (preview): + +* Works with `tensorflow-cpu` package, version 2.10 or later. +* [PyPI wheel](https://pypi.org/project/tensorflow-directml-plugin/). +* [GitHub repo](https://github.com/microsoft/tensorflow-directml-plugin). +* For questions, feedback or to raise issues, please visit the + [Issues page of `tensorflow-directml-plugin` on GitHub](https://github.com/microsoft/tensorflow-directml-plugin/issues). + +Intel® Extension for TensorFlow `PluggableDevice` for Linux and WSL: + +* Works with TF 2.10 or later. +* [Getting started guide](https://intel.github.io/intel-extension-for-tensorflow/latest/get_started.html) +* [PyPI wheel](https://pypi.org/project/intel-extension-for-tensorflow/). +* [GitHub repo](https://github.com/intel/intel-extension-for-tensorflow). +* For questions, feedback, or to raise issues, please visit the + [Issues page of `intel-extension-for-tensorflow` on GitHub](https://github.com/intel/intel-extension-for-tensorflow/issues). diff --git a/site/en/install/lang_c.ipynb b/site/en/install/lang_c.ipynb new file mode 100644 index 00000000000..788a5e6c891 --- /dev/null +++ b/site/en/install/lang_c.ipynb @@ -0,0 +1,383 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Tce3stUlHN0L" + }, + "source": [ + "##### Copyright 2018 The TensorFlow Authors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "tuOe1ymfHZPu" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s7Bo2MipUnXX" + }, + "source": [ + "# Install TensorFlow for C" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Birwb-khUOIq" + }, + "source": [ + "
    GitHub issue or Stack Overflow Error Message
    38896424 + 31058"No matching distribution found for tensorflow": + Pip can't find a TensorFlow package compatible with your system. Check the + system requirements and + Python version +
    22390 
    Unzipping simple_console_for_windows.zip to create runfiles tree...
@@ -35,12 +44,12 @@ unzip: cannot find zipfile directory in one of ./bazel-bin/tensorflow/tools/pip_
   No such file or directory
    36371137 and - here36371137 
    libprotobuf ERROR google/protobuf/src/google/protobuf/io/coded_stream.cc:207] A
   protocol message was rejected because it was too big (more than 67108864 bytes).
   To increase the limit (or to disable these warnings), see
-  CodedInputStream::SetTotalBytesLimit() in google/protobuf/io/coded_stream.h.
    35252888
    33623453 
    IOError: [Errno 2] No such file or directory:
-  '/tmp/pip-o6Tpui-build/setup.py'
    + '/tmp/pip-o6Tpui-build/setup.py'
    42006320
    33623453 
    IOError: [Errno 2] No such file or directory:
-  '/tmp/pip-o6Tpui-build/setup.py'
    + '/tmp/pip-o6Tpui-build/setup.py'
    35190574
    33623453 
    IOError: [Errno 2] No such file or directory:
-  '/tmp/pip-o6Tpui-build/setup.py'
    + '/tmp/pip-o6Tpui-build/setup.py'
    35190574
    \n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kFmEkitOFJSw" + }, + "source": [ + "TensorFlow provides a C API that can be used to build\n", + "[bindings for other languages](https://github.com/tensorflow/docs/tree/master/site/en/r1/guide/extend/bindings.md).\n", + "The API is defined in\n", + "c_api.h\n", + "and designed for simplicity and uniformity rather than convenience.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Vk--31hqIwSV" + }, + "source": [ + "## Nightly libtensorflow C packages\n", + "\n", + "libtensorflow packages are built nightly and uploaded to GCS for all supported\n", + "platforms. They are uploaded to the\n", + "[libtensorflow-nightly GCS bucket](https://storage.googleapis.com/libtensorflow-nightly)\n", + "and are indexed by operating system and date built. For MacOS and Linux shared\n", + "objects, there is a\n", + "[script](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/ci_build/builds/libtensorflow_nightly_symlink.sh)\n", + "that renames the `.so` files versioned to the current date copied into the\n", + "directory with the artifacts." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qowtdsijFMYZ" + }, + "source": [ + "## Supported Platforms\n", + "\n", + "TensorFlow for C is supported on the following systems:\n", + "\n", + "* Linux, 64-bit, x86\n", + "* macOS, Version 10.12.6 (Sierra) or higher\n", + "* Windows, 64-bit x86" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hnhAk8y-FSBN" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "y50y01XUFVb2" + }, + "source": [ + "### Download and extract\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    TensorFlow C libraryURL
    Linux\n", + " \n", + "
    Linux CPU onlyhttps://storage.googleapis.com/tensorflow/versions/2.18.0/libtensorflow-cpu-linux-x86_64.tar.gz
    Linux GPU supporthttps://storage.googleapis.com/tensorflow/versions/2.18.0/libtensorflow-gpu-linux-x86_64.tar.gz
    macOS\n", + " \n", + "
    macOS CPU onlyhttps://storage.googleapis.com/tensorflow/versions/2.16.2/libtensorflow-cpu-darwin-x86_64.tar.gz
    macOS ARM64 CPU onlyhttps://storage.googleapis.com/tensorflow/versions/2.18.0/libtensorflow-cpu-darwin-arm64.tar.gz
    Windows\n", + " \n", + "
    Windows CPU onlyhttps://storage.googleapis.com/tensorflow/versions/2.18.1/libtensorflow-cpu-windows-x86_64.zip
    Windows GPU onlyhttps://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-windows-x86_64-2.10.0.zip

    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "b4kWu6k0FaT9" + }, + "source": [ + "Extract the downloaded archive, which contains the header files to include in\n", + "your C program and the shared libraries to link against.\n", + "\n", + "On Linux and macOS, you may want to extract to `/usr/local/lib`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DrjVyjVJFcon" + }, + "outputs": [], + "source": [ + "%%bash\n", + "FILENAME=libtensorflow-cpu-linux-x86_64.tar.gz\n", + "wget -q --no-check-certificate https://storage.googleapis.com/tensorflow/versions/2.18.1/${FILENAME}\n", + "sudo tar -C /usr/local -xzf ${FILENAME}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fcBJDdojJDyk" + }, + "source": [ + "### Linker\n", + "\n", + "On Linux/macOS, if you extract the TensorFlow C library to a system directory,\n", + "such as `/usr/local`, configure the linker with `ldconfig`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "h0STAG82JDZs" + }, + "outputs": [], + "source": [ + "%%bash\n", + "sudo ldconfig /usr/local/lib" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ix4HdnNGH6aF" + }, + "source": [ + "If you extract the TensorFlow C library to a non-system directory, such as\n", + "`~/mydir`, then configure the linker environmental variables:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "E6E99eJzIJQs" + }, + "source": [ + "
    \n", + "
    \n", + "

    Linux

    \n", + "
    \n",
+        "export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib\n",
+        "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/mydir/lib\n",
+        "
    \n", + "
    \n", + "
    \n", + "

    macOS

    \n", + "
    \n",
+        "export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib\n",
+        "export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib\n",
+        "
    \n", + "
    \n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qYVWjxqaJVPs" + }, + "source": [ + "## Build" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UoMUuMJrJXp8" + }, + "source": [ + "### Example program\n", + "\n", + "With the TensorFlow C library installed, create an example program with the\n", + "following source code (`hello_tf.c`):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "b5851f1b" + }, + "outputs": [], + "source": [ + "%%writefile hello_tf.c\n", + "#include \n", + "#include \n", + "\n", + "int main() {\n", + " printf(\"Hello from TensorFlow C library version %s\\n\", TF_Version());\n", + " return 0;\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H1GFidbrIWzU" + }, + "source": [ + "### Compile\n", + "\n", + "Compile the example program to create an executable, then run:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Jph67SAjIX0M" + }, + "outputs": [], + "source": [ + "%%bash\n", + "gcc hello_tf.c -ltensorflow -o hello_tf\n", + "\n", + "./hello_tf" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0qtHXROoJwoz" + }, + "source": [ + "Success: The TensorFlow C library is configured.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YbqbjqOSJ0IL" + }, + "source": [ + "If the program doesn't build, make sure that `gcc` can access the TensorFlow C\n", + "library. If extracted to `/usr/local`, explicitly pass the library location to\n", + "the compiler:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CdPmM35VJ_77" + }, + "outputs": [], + "source": [ + "%%bash\n", + "gcc -I/usr/local/include -L/usr/local/lib hello_tf.c -ltensorflow -o hello_tf\n", + "\n", + "./hello_tf" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ea5fd208" + }, + "source": [ + "## Build from source\n", + "\n", + "TensorFlow is open source. Read\n", + "[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md)\n", + "to build TensorFlow's C library from source code." + ] + } + ], + "metadata": { + "colab": { + "name": "lang_c.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/install/lang_c.md b/site/en/install/lang_c.md deleted file mode 100644 index 3348021a658..00000000000 --- a/site/en/install/lang_c.md +++ /dev/null @@ -1,135 +0,0 @@ -# Install TensorFlow for C - -TensorFlow provides a C API that can be used to build -[bindings for other languages](../extend/language_bindings.md). The API is defined in -c_api.h -and designed for simplicity and uniformity rather than convenience. - -Note: There is no `libtensorflow` support for TensorFlow 2 yet. It is expected -in a future release. - -## Supported Platforms - -TensorFlow for C is supported on the following systems: - -* Linux, 64-bit, x86 -* macOS X, Version 10.12.6 (Sierra) or higher -* Windows, 64-bit x86 - -## Setup - -### Download - - - - - - - - - - - - - - - - - - - - - - - - - - -
    TensorFlow C libraryURL
    Linux
    Linux CPU onlyhttps://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.15.0.tar.gz
    Linux GPU supporthttps://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-1.15.0.tar.gz
    macOS
    macOS CPU onlyhttps://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-1.15.0.tar.gz
    Windows
    Windows CPU onlyhttps://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-windows-x86_64-1.15.0.zip
    Windows GPU onlyhttps://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-windows-x86_64-1.15.0.zip
    - -### Extract - -Extract the downloaded archive, which contains the header files to include in -your C program and the shared libraries to link against. - -On Linux and macOS, you may want to extract to `/usr/local/lib`: - -
    -sudo tar -C /usr/local -xzf (downloaded file)
-
    - -### Linker - -On Linux/macOS, if you extract the TensorFlow C library to a system directory, -such as `/usr/local`, configure the linker with `ldconfig`: - -
    -sudo ldconfig
-
    - -If you extract the TensorFlow C library to a non-system directory, such as -`~/mydir`, then configure the linker environmental variables: - -
    -
    -

    Linux

    -
    -export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/mydir/lib
-
    -
    -
    -

    mac OS

    -
    -export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib
-export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib
-
    -
    -
    - - -## Build - -### Example program - -With the TensorFlow C library installed, create an example program with the -following source code (`hello_tf.c`): - -```c -#include -#include - -int main() { - printf("Hello from TensorFlow C library version %s\n", TF_Version()); - return 0; -} -``` - -### Compile - -Compile the example program to create an executable, then run: - -
    -gcc hello_tf.c -ltensorflow -o hello_tf
-
-./hello_tf
-
    - -The command outputs: Hello from TensorFlow C library version number - -Success: The TensorFlow C library is configured. - -If the program doesn't build, make sure that `gcc` can access the TensorFlow C -library. If extracted to `/usr/local`, explicitly pass the library location to -the compiler: - -
    -gcc -I/usr/local/include -L/usr/local/lib hello_tf.c -ltensorflow -o hello_tf
-
    - - -## Build from source - -TensorFlow is open source. Read -[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/tools/lib_package/README.md){:.external} -to build TensorFlow's C library from source code. diff --git a/site/en/install/lang_go.md b/site/en/install/lang_go.md deleted file mode 100644 index fc8667c91d2..00000000000 --- a/site/en/install/lang_go.md +++ /dev/null @@ -1,104 +0,0 @@ -# Install TensorFlow for Go - -TensorFlow provides a -[Go API](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go){:.external}— -particularly useful for loading models created with Python and running them -within a Go application. - -Caution: The TensorFlow Go API is *not* covered by the TensorFlow -[API stability guarantees](../guide/versions.md). - - -## Supported Platforms - -TensorFlow for Go is supported on the following systems: - -* Linux, 64-bit, x86 -* macOS X, Version 10.12.6 (Sierra) or higher - - -## Setup - -### TensorFlow C library - -Install the [TensorFlow C library](./lang_c.md) which is required for the -TensorFlow Go package. - -### Download - -Download and install the TensorFlow Go package and its dependencies: - -
    -go get github.com/tensorflow/tensorflow/tensorflow/go
-
    - -And validate your installation: - -
    -go test github.com/tensorflow/tensorflow/tensorflow/go
-
    - - -## Build - -### Example program - -With the TensorFlow Go package installed, create an example program with the -following source code (`hello_tf.go`): - -```go -package main - -import ( - tf "github.com/tensorflow/tensorflow/tensorflow/go" - "github.com/tensorflow/tensorflow/tensorflow/go/op" - "fmt" -) - -func main() { - // Construct a graph with an operation that produces a string constant. - s := op.NewScope() - c := op.Const(s, "Hello from TensorFlow version " + tf.Version()) - graph, err := s.Finalize() - if err != nil { - panic(err) - } - - // Execute the graph in a session. - sess, err := tf.NewSession(graph, nil) - if err != nil { - panic(err) - } - output, err := sess.Run(nil, []tf.Output{c}, nil) - if err != nil { - panic(err) - } - fmt.Println(output[0].Value()) -} -``` - -### Run - -Run the example program: - -
    -go run hello_tf.go
-
    - -The command outputs: Hello from TensorFlow version number - -Success: The TensorFlow for Go is configured. - -The program may generate the following warning messages, which you can ignore: - -
    -W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library
-wasn't compiled to use *Type* instructions, but these are available on your
-machine and could speed up CPU computations.
-
    - -## Build from source - -TensorFlow is open source. Read -[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/go/README.md){:.external} -to build TensorFlow for Go from source code. diff --git a/site/en/install/lang_java.md b/site/en/install/lang_java_legacy.md similarity index 72% rename from site/en/install/lang_java.md rename to site/en/install/lang_java_legacy.md index 2669db8a5f1..37341c36659 100644 --- a/site/en/install/lang_java.md +++ b/site/en/install/lang_java_legacy.md @@ -1,16 +1,19 @@ # Install TensorFlow for Java +Warning: TensorFlow for Java is deprecated and will be removed in a future +version of TensorFlow once [the replacement](https://www.tensorflow.org/jvm) is stable. + TensorFlow provides a [Java API](https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/package-summary)— -particularly useful for loading models created with Python and running them -within a Java application. - -Note: There is no `libtensorflow` support for TensorFlow 2 yet. It is expected -in a future release. +useful for loading models created with Python and running them within a Java +application. -Caution: The TensorFlow Java API is *not* covered by the TensorFlow -[API stability guarantees](../guide/versions.md). +## Nightly Libtensorflow Java packages +Libtensorflow JNI packages are built nightly and uploaded to GCS for all +supported platforms. They are uploaded to the +[libtensorflow-nightly GCS bucket](https://storage.googleapis.com/libtensorflow-nightly) +and are indexed by operating system and date built. ## Supported Platforms @@ -20,39 +23,36 @@ TensorFlow for Java is supported on the following systems: * macOS 10.12.6 (Sierra) or higher * Windows 7 or higher; 64-bit, x86 -To install TensorFlow on Android, see -[Android TensorFlow support](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/android/inference_interface){:.external} -and the -[TensorFlow Android Camera Demo](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android){:.external}. +To use TensorFlow on Android see [TensorFlow Lite](https://tensorflow.org/lite) ## TensorFlow with Apache Maven -To use TensorFlow with [Apache Maven](https://maven.apache.org){:.external}, +To use TensorFlow with [Apache Maven](https://maven.apache.org), add the dependency to the project's `pom.xml` file: ```xml org.tensorflow tensorflow - 1.14.0 + 2.4.0 ``` ### GPU support -If your system has [GPU support](./gpu.md), add the following TensorFlow +If your system has [GPU support](./pip.md), add the following TensorFlow dependencies to the project's `pom.xml` file: ```xml org.tensorflow libtensorflow - 1.14.0 + 2.4.0 org.tensorflow libtensorflow_jni_gpu - 1.14.0 + 2.4.0 ``` @@ -142,41 +142,41 @@ system and processor support: Linux Linux CPU only - https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-linux-x86_64-1.14.0.tar.gz + https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-linux-x86_64-2.4.0.tar.gz Linux GPU support - https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-gpu-linux-x86_64-1.14.0.tar.gz + https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-gpu-linux-x86_64-2.4.0.tar.gz macOS macOS CPU only - https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-darwin-x86_64-1.14.0.tar.gz + https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-darwin-x86_64-2.4.0.tar.gz Windows Windows CPU only - https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.14.0.zip + https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-2.4.0.zip Windows GPU support - https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-gpu-windows-x86_64-1.14.0.zip + https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-gpu-windows-x86_64-2.4.0.zip Note: On Windows, the native library (`tensorflow_jni.dll`) requires -`msvcp140.dll` at runtime. See the [Windows build from source](./source_windows.md) -guide to install the [Visual C++ 2015 Redistributable](https://www.microsoft.com/en-us/download/details.aspx?id=48145){:.external}. - +`msvcp140.dll` at runtime. See the +[Windows build from source](./source_windows.md) guide to install the +[Visual C++ 2019 Redistributable](https://visualstudio.microsoft.com/vs/). ### Compile -Using the `HelloTensorFlow.java` file from the [previous example](#example), +Using the `HelloTensorFlow.java` file from the [previous example](#example-program), compile a program that uses TensorFlow. Make sure the `libtensorflow.jar` is accessible to your `classpath`: 
    -javac -cp libtensorflow-1.14.0.jar HelloTensorFlow.java
+javac -cp libtensorflow-2.4.0.jar HelloTensorFlow.java
    ### Run @@ -186,12 +186,12 @@ the extracted JNI library.
    -

    Linux / mac OS

    -
    java -cp libtensorflow-1.14.0.jar:. -Djava.library.path=./jni HelloTensorFlow
    +

    Linux / macOS

    +
    java -cp libtensorflow-2.4.0.jar:. -Djava.library.path=./jni HelloTensorFlow

    Windows

    -
    java -cp libtensorflow-1.14.0.jar;. -Djava.library.path=jni HelloTensorFlow
    +
    java -cp libtensorflow-2.4.0.jar;. -Djava.library.path=jni HelloTensorFlow
    @@ -203,5 +203,5 @@ Success: TensorFlow for Java is configured. ## Build from source TensorFlow is open source. Read -[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/java/README.md){:.external} +[the instructions](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/java/README.md) to build TensorFlow's Java and native libraries from source code. diff --git a/site/en/install/pip.html b/site/en/install/pip.html deleted file mode 100644 index 1f279e9876d..00000000000 --- a/site/en/install/pip.html +++ /dev/null @@ -1,437 +0,0 @@ - - - Install TensorFlow with pip - - - - - - -

    TensorFlow 2 packages are available

    -
      -
    • tensorflow —Latest stable release for CPU-only
      • -
    • tensorflow-gpu —Latest stable release with GPU support (Ubuntu and Windows)
      • -
    • tf-nightly —Preview build (unstable). Ubuntu and Windows include GPU support.
      • -
    - - -

    Older versions of TensorFlow

    -

    For the 1.15 release, CPU and GPU support are included in a single package:

    -
      -
    • tensorflow==1.15 —The final 1.x release. Ubuntu and Windows include GPU support
      • -
    - -

    For releases 1.14 and older, CPU and GPU packages are separate:

    - -
      -
    • tensorflow==1.14 —Release for CPU-only
      • -
    • tensorflow-gpu==1.14 —Release with GPU support (Ubuntu and Windows)
      • -
    - - -

    System requirements

    -
      -
    • pip 19.0 or later (requires manylinux2010 support)
      • -
    • Ubuntu 16.04 or later (64-bit)
      • -
    • macOS 10.12.6 (Sierra) or later (64-bit) (no GPU support)
      • -
    • Windows 7 or later (64-bit) (Python 3 only)
      • -
    • Raspbian 9.0 or later
      • -
    - - - -

    Hardware requirements

    -
      -
    • Starting with TensorFlow 1.6, binaries use AVX instructions which may not run on older CPUs.
      • -
    • Read the GPU support guide to set up a CUDA®-enabled GPU card on Ubuntu or Windows.
      • -
    - - -

    1. Install the Python development environment on your system

    - - - - - - -

    - Check if your Python environment is already configured: -

    - -{% dynamic if request.query_string.lang == "python2" %} - - -
    -python --version
-pip --version
-virtualenv --version
-
    -{% dynamic else %} - - -
    -python3 --version
-pip3 --version
-virtualenv --version
-
    -{% dynamic endif %} - -

    - If these packages are already installed, skip to the next step.
    - Otherwise, install Python, the - pip package manager, - and Virtualenv: -

    - -{% dynamic if request.query_string.lang == "python2" %} - -
    -
    -

    Ubuntu

    -
    -sudo apt update
-sudo apt install python-dev python-pip
-sudo pip install -U virtualenv  # system-wide install
-
    -
    - -
    -

    mac OS

    -

    Install using the Homebrew package manager:

    -
    -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-export PATH="/usr/local/bin:/usr/local/sbin:$PATH"
-brew update
-brew install python@2  # Python 2
-sudo pip install -U virtualenv  # system-wide install
-
    -
    - -
    -

    Raspberry Pi

    -

    Requirements for the Raspbian operating system:

    -
    -sudo apt update
-sudo apt install python-dev python-pip
-sudo apt install libatlas-base-dev     # required for numpy
-sudo pip install -U virtualenv         # system-wide install
-
    -
    - -
    -

    Other

    -
    -curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-python get-pip.py
-sudo pip install -U virtualenv  # system-wide install
-
    -
    -
    - -{% dynamic else %} - -
    -
    -

    Ubuntu

    -
    -sudo apt update
-sudo apt install python3-dev python3-pip
-sudo pip3 install -U virtualenv  # system-wide install
-
    -
    - -
    -

    mac OS

    -

    Install using the Homebrew package manager:

    -
    -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-export PATH="/usr/local/bin:/usr/local/sbin:$PATH"
-brew update
-brew install python  # Python 3
-sudo pip3 install -U virtualenv  # system-wide install
-
    -
    - -
    -

    Windows

    -

    - Install the Microsoft Visual C++ 2015 Redistributable Update 3. This - comes with Visual Studio 2015 but can be installed separately: -

    -
      -
    1. Go to the Visual Studio downloads,
      2. -
    2. Select Redistributables and Build Tools,
      3. -
    3. Download and install the Microsoft Visual C++ 2015 Redistributable Update 3.
      4. -
    -

    Make sure long paths are enabled on Windows.

    -

    Install the 64-bit Python 3 release for Windows (select pip as an optional feature).

    -
    pip3 install -U pip virtualenv
    -
    - -
    -

    Raspberry Pi

    -

    Requirements for the Raspbian operating system:

    -
    -sudo apt update
-sudo apt install python3-dev python3-pip
-sudo apt install libatlas-base-dev        # required for numpy
-sudo pip3 install -U virtualenv           # system-wide install
-
    -
    - -
    -

    Other

    -
    -curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-python get-pip.py
-sudo pip3 install -U virtualenv  # system-wide install
-
    -
    -
    - -{% dynamic endif %} - - - - -

    2. Create a virtual environment (recommended)

    - -

    - Python virtual environments are used to isolate package installation from the system. -

    - -
    -
    -

    Ubuntu / mac OS

    -

    - Create a new virtual environment by choosing a Python interpreter and making a - ./venv directory to hold it: -

    -{% dynamic if request.query_string.lang == "python2" %} -
    virtualenv --system-site-packages -p python2.7 ./venv
    -{% dynamic else %} -
    virtualenv --system-site-packages -p python3 ./venv
    -{% dynamic endif %} -

    - Activate the virtual environment using a shell-specific command: -

    -
    source ./venv/bin/activate  # sh, bash, ksh, or zsh
    -

    - When virtualenv is active, your shell prompt is prefixed with (venv). -

    -

    - Install packages within a virtual environment without affecting the host system - setup. Start by upgrading pip: -

    -
    -pip install --upgrade pip
-
-pip list  # show packages installed within the virtual environment
-
    -

    - And to exit virtualenv later: -

    -
    deactivate  # don't exit until you're done using TensorFlow
    -
    - - -
    -

    Windows

    -

    - Create a new virtual environment by choosing a Python interpreter and making a - .\venv directory to hold it: -

    -{% dynamic if request.query_string.lang == "python2" %} - -{% dynamic else %} -
    virtualenv --system-site-packages -p python3 ./venv
    -{% dynamic endif %} -

    - Activate the virtual environment: -

    -
    .\venv\Scripts\activate
    -

    - Install packages within a virtual environment without affecting the host system - setup. Start by upgrading pip: -

    -
    -pip install --upgrade pip
-
-pip list  # show packages installed within the virtual environment
-
    -

    - And to exit virtualenv later: -

    -
    deactivate  # don't exit until you're done using TensorFlow
    -
    - - -
    -

    Conda

    - -

    - Create a new virtual environment by choosing a Python interpreter and making a - ./venv directory to hold it: -

    -{% dynamic if request.query_string.lang == "python2" %} -
    conda create -n venv pip python=2.7
    -{% dynamic else %} -
    conda create -n venv pip python=3.7  # select python version
    -{% dynamic endif %} -

    Activate the virtual environment:

    -
    source activate venv
    -

    - Within the virtual environment, install the TensorFlow pip package using its complete URL: -

    -
    pip install --ignore-installed --upgrade packageURL
    -

    - And to exit virtualenv later: -

    -
    source deactivate
    -
    -
    - - -

    3. Install the TensorFlow pip package

    - -

    - Choose one of the following TensorFlow packages to install from PyPI: -

    - -
      -
    • tensorflow —Latest stable release (2.x) for CPU-only (recommended for beginners).
      • -
    • tensorflow-gpu —Latest stable release with GPU support (Ubuntu and Windows).
      • -
    • tf-nightly —Preview build (unstable). Ubuntu and Windows include GPU support.
      • -
    • tensorflow==1.15 —The final version of TensorFlow 1.x.
      • -
    - - - -
    -
    -

    Virtualenv install

    -
    pip install --upgrade tensorflow
    -

    Verify the install:

    -
    python -c "import tensorflow as tf;print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
    -
    - -
    -

    System install

    -{% dynamic if request.query_string.lang == "python2" %} -
    pip install --user --upgrade tensorflow  # install in $HOME
    -

    Verify the install:

    -
    python -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
    -{% dynamic else %} -
    pip3 install --user --upgrade tensorflow  # install in $HOME
    -

    Verify the install:

    -
    python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))"
    -{% dynamic endif %} -
    -
    - - - -

    Package location

    - -

    - A few installation mechanisms require the URL of the TensorFlow Python package. - The value you specify depends on your Python version. -

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    VersionURL
    Linux
    Python 2.7 CPU-onlyhttps://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-2.0.0-cp27-cp27mu-manylinux2010_x86_64.whl
    Python 2.7 GPU supporthttps://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.0.0-cp27-cp27mu-manylinux2010_x86_64.whl
    Python 3.5 CPU-onlyhttps://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-2.0.0-cp35-cp35m-manylinux2010_x86_64.whl
    Python 3.5 GPU supporthttps://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.0.0-cp35-cp35m-manylinux2010_x86_64.whl
    Python 3.6 CPU-onlyhttps://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl
    Python 3.6 GPU supporthttps://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.0.0-cp36-cp36m-manylinux2010_x86_64.whl
    Python 3.7 CPU-onlyhttps://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl
    Python 3.7 GPU supporthttps://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-2.0.0-cp37-cp37m-manylinux2010_x86_64.whl
    macOS (CPU-only)
    Python 2.7https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-2.0.0-py2-none-any.whl
    Windows
    Python 3.5 CPU-onlyhttps://storage.googleapis.com/tensorflow/windows/cpu/tensorflow-2.0.0-cp35-cp35m-win_amd64.whl
    Python 3.5 GPU supporthttps://storage.googleapis.com/tensorflow/windows/gpu/tensorflow_gpu-2.0.0-cp35-cp35m-win_amd64.whl
    Python 3.6 CPU-onlyhttps://storage.googleapis.com/tensorflow/windows/cpu/tensorflow-2.0.0-cp36-cp36m-win_amd64.whl
    Python 3.6 GPU supporthttps://storage.googleapis.com/tensorflow/windows/gpu/tensorflow_gpu-2.0.0-cp36-cp36m-win_amd64.whl
    Raspberry PI (CPU-only)
    There is no TensorFlow 2 support for RPi yet, it is expected in a future release
    Python 2.7, Pi0 or Pi1https://storage.googleapis.com/tensorflow/raspberrypi/tensorflow-1.14.0-cp27-none-linux_armv6l.whl
    Python 2.7, Pi2 or Pi3https://storage.googleapis.com/tensorflow/raspberrypi/tensorflow-1.14.0-cp27-none-linux_armv7l.whl
    Python 3, Pi0 or Pi1https://storage.googleapis.com/tensorflow/raspberrypi/tensorflow-1.14.0-cp34-none-linux_armv6l.whl
    Python 3, Pi2 or Pi3https://storage.googleapis.com/tensorflow/raspberrypi/tensorflow-1.14.0-cp34-none-linux_armv7l.whl
    - - - diff --git a/site/en/install/pip.md b/site/en/install/pip.md new file mode 100644 index 00000000000..a9e4bf4bf74 --- /dev/null +++ b/site/en/install/pip.md @@ -0,0 +1,658 @@ + +# Install TensorFlow with pip + + +This guide is for the latest stable version of TensorFlow. For the +preview build *(nightly)*, use the pip package named +`tf-nightly`. Refer to [these tables](./source#tested_build_configurations) for +older TensorFlow version requirements. For the CPU-only build, use the pip +package named `tensorflow-cpu`. + +Here are the quick versions of the install commands. Scroll down for the +step-by-step instructions. + +* {Linux} + + Note: Starting with TensorFlow `2.10`, Linux CPU-builds for Aarch64/ARM64 + processors are built, maintained, tested and released by a third party: + [AWS](https://aws.amazon.com/). + Installing the [`tensorflow`](https://pypi.org/project/tensorflow/) + package on an ARM machine installs AWS's + [`tensorflow-cpu-aws`](https://pypi.org/project/tensorflow-cpu-aws/) package. + They are provided as-is. Tensorflow will use reasonable efforts to maintain + the availability and integrity of this pip package. There may be delays if + the third party fails to release the pip package. See + [this blog post](https://blog.tensorflow.org/2022/09/announcing-tensorflow-official-build-collaborators.html) + for more information about this collaboration. + + ```bash + python3 -m pip install 'tensorflow[and-cuda]' + # Verify the installation: + python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))" + ``` + +* {MacOS} + + ```bash + # There is currently no official GPU support for MacOS. + python3 -m pip install tensorflow + # Verify the installation: + python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))" + ``` + +* {Windows Native} + + Caution: TensorFlow `2.10` was the **last** TensorFlow release that + supported GPU on native-Windows. + Starting with TensorFlow `2.11`, you will need to install + [TensorFlow in WSL2](https://tensorflow.org/install/pip#windows-wsl2), + or install `tensorflow` or `tensorflow-cpu` and, optionally, try the + [TensorFlow-DirectML-Plugin](https://github.com/microsoft/tensorflow-directml-plugin#tensorflow-directml-plugin-) + + ```bash + conda install -c conda-forge cudatoolkit=11.2 cudnn=8.1.0 + # Anything above 2.10 is not supported on the GPU on Windows Native + python -m pip install "tensorflow<2.11" + # Verify the installation: + python -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))" + ``` + +* {Windows WSL2} + + Note: TensorFlow with GPU access is supported for WSL2 on Windows 10 19044 or + higher. This corresponds to Windows 10 version 21H2, the November 2021 + update. You can get the latest update from here: + [Download Windows 10](https://www.microsoft.com/software-download/windows10). + For instructions, see + [Install WSL2](https://docs.microsoft.com/windows/wsl/install) + and + [NVIDIA’s setup docs](https://docs.nvidia.com/cuda/wsl-user-guide/index.html) + for CUDA in WSL. + + ```bash + python3 -m pip install tensorflow[and-cuda] + # Verify the installation: + python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))" + ``` + +* {CPU} + + Note: Starting with TensorFlow `2.10`, Windows CPU-builds for x86/x64 + processors are built, maintained, tested and released by a third party: + [Intel](https://www.intel.com/). + Installing the Windows-native [`tensorflow`](https://pypi.org/project/tensorflow/) + or [`tensorflow-cpu`](https://pypi.org/project/tensorflow-cpu/) + package installs Intel's + [`tensorflow-intel`](https://pypi.org/project/tensorflow-intel/) + package. These packages are provided as-is. Tensorflow will use reasonable + efforts to maintain the availability and integrity of this pip package. + There may be delays if the third party fails to release the pip package. See + [this blog post](https://blog.tensorflow.org/2022/09/announcing-tensorflow-official-build-collaborators.html) + for more information about this + collaboration. + + ```bash + python3 -m pip install tensorflow + # Verify the installation: + python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))" + ``` + +* {Nightly} + + ```bash + python3 -m pip install tf-nightly + # Verify the installation: + python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))" + ``` + +## Hardware requirements + +Note: TensorFlow binaries use +[AVX instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions#CPUs_with_AVX) +which may not run on older CPUs. + +The following GPU-enabled devices are supported: + +* NVIDIA® GPU card with CUDA® architectures 3.5, 5.0, 6.0, 7.0, 7.5, 8.0 and + higher. See the list of + [CUDA®-enabled GPU cards](https://developer.nvidia.com/cuda-gpus). +* For GPUs with unsupported CUDA® architectures, or to avoid JIT compilation + from PTX, or to use different versions of the NVIDIA® libraries, see the + [Linux build from source](./source.md) guide. +* Packages do not contain PTX code except for the latest supported CUDA® + architecture; therefore, TensorFlow fails to load on older GPUs when + `CUDA_FORCE_PTX_JIT=1` is set. (See + [Application Compatibility](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#application-compatibility) + for details.) + +Note: The error message "Status: device kernel image is invalid" indicates that +the TensorFlow package does not contain PTX for your architecture. You can +enable compute capabilities by [building TensorFlow from source](./source.md). + +## System requirements + +* Ubuntu 16.04 or higher (64-bit) +* macOS 12.0 (Monterey) or higher (64-bit) *(no GPU support)* +* Windows Native - Windows 7 or higher (64-bit) *(no GPU support after TF 2.10)* +* Windows WSL2 - Windows 10 19044 or higher (64-bit) + +Note: GPU support is available for Ubuntu and Windows with CUDA®-enabled cards. + +## Software requirements + +* Python 3.9–3.12 +* pip version 19.0 or higher for Linux (requires `manylinux2014` support) and + Windows. pip version 20.3 or higher for macOS. +* Windows Native Requires + [Microsoft Visual C++ Redistributable for Visual Studio 2015, 2017 and 2019](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist) + + +The following NVIDIA® software are only required for GPU support. + +* [NVIDIA® GPU drivers](https://www.nvidia.com/drivers) + * >= 525.60.13 for Linux + * >= 528.33 for WSL on Windows +* [CUDA® Toolkit 12.3](https://developer.nvidia.com/cuda-toolkit-archive). +* [cuDNN SDK 8.9.7](https://developer.nvidia.com/cudnn). +* *(Optional)* + [TensorRT](https://docs.nvidia.com/deeplearning/tensorrt/archives/index.html#trt_7) + to improve latency and throughput for inference. + +## Step-by-step instructions + +* {Linux} + + ### 1. System requirements + + * Ubuntu 16.04 or higher (64-bit) + + TensorFlow only officially supports Ubuntu. However, the following + instructions may also work for other Linux distros. + + Note: Starting with TensorFlow `2.10`, Linux CPU-builds for Aarch64/ARM64 + processors are built, maintained, tested and released by a third party: + [AWS](https://aws.amazon.com/). + Installing the [`tensorflow`](https://pypi.org/project/tensorflow/) + package on an ARM machine installs AWS's + [`tensorflow-cpu-aws`](https://pypi.org/project/tensorflow-cpu-aws/) package. + They are provided as-is. Tensorflow will use reasonable efforts to maintain + the availability and integrity of this pip package. There may be delays if + the third party fails to release the pip package. See + [this blog post](https://blog.tensorflow.org/2022/09/announcing-tensorflow-official-build-collaborators.html) + for more information about this collaboration. + + ### 2. GPU setup + + You can skip this section if you only run TensorFlow on the CPU. + + Install the + [NVIDIA GPU driver](https://www.nvidia.com/Download/index.aspx) + if you have not. You can use the following command to verify it is + installed. + + ```bash + nvidia-smi + ``` + + ### 3. Create a virtual environment with [venv](https://docs.python.org/3/library/venv.html){:.external} + + The venv module is part of Python’s standard library and is the officially recommended way to create virtual environments. + + Navigate to your desired virtual environments directory and create a new venv environment named `tf` with the following command. + + ```bash + python3 -m venv tf + ``` + + You can activate it with the following command. + + ```bash + source tf/bin/activate + ``` + + Make sure that the virtual environment is activated for the rest of the installation. + + ### 4. Install TensorFlow + + TensorFlow requires a recent version of pip, so upgrade your pip + installation to be sure you're running the latest version. + + ```bash + pip install --upgrade pip + ``` + + Then, install TensorFlow with pip. + + ```bash + # For GPU users + pip install tensorflow[and-cuda] + # For CPU users + pip install tensorflow + ``` + + **Note:** Do not install TensorFlow with `conda`. It may not have the latest stable version. `pip` is recommended since TensorFlow is only officially released to PyPI. + + ### 6. Verify the installation + + Verify the CPU setup: + + ```bash + python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))" + ``` + + If a tensor is returned, you've installed TensorFlow successfully. + + Verify the GPU setup: + + ```bash + python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))" + ``` + + If a list of GPU devices is returned, you've installed TensorFlow + successfully. **If not continue to the next step**. + + ### 6. [GPU only] Virtual environment configuration + + If the GPU test in the last section was unsuccessful, the most likely cause is that components aren't being detected, + and/or conflict with the existing system CUDA installation. So you need to add some symbolic links to fix this. + + * Create symbolic links to NVIDIA shared libraries: + + ```bash + pushd $(dirname $(python -c 'print(__import__("tensorflow").__file__)')) + ln -svf ../nvidia/*/lib/*.so* . + popd + ``` + + * Create a symbolic link to ptxas: + + ```bash + ln -sf $(find $(dirname $(dirname $(python -c "import nvidia.cuda_nvcc; + print(nvidia.cuda_nvcc.__file__)"))/*/bin/) -name ptxas -print -quit) $VIRTUAL_ENV/bin/ptxas + ``` + + Verify the GPU setup: + + ```bash + python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))" + ``` + + + + +* {MacOS} + + ### 1. System requirements + + * macOS 10.12.6 (Sierra) or higher (64-bit) + + Note: While TensorFlow supports Apple Silicon (M1), packages that include + custom C++ extensions for TensorFlow also need to be compiled for Apple M1. + Some packages, like + [tensorflow_decision_forests](https://www.tensorflow.org/decision_forests) + publish M1-compatible versions, but many packages don't. To use those + libraries, you will have to use TensorFlow with x86 emulation and Rosetta. + + Currently there is no official GPU support for running TensorFlow on + MacOS. The following instructions are for running on CPU. + + ### 2. Check Python version + + Check if your Python environment is already configured: + + Note: Requires Python 3.9–3.11, and pip >= 20.3 for MacOS. + + ```bash + python3 --version + python3 -m pip --version + ``` + + ### 3. Install TensorFlow + + TensorFlow requires a recent version of pip, so upgrade your pip + installation to be sure you're running the latest version. + + ```bash + pip install --upgrade pip + ``` + + Then, install TensorFlow with pip. + + ```bash + pip install tensorflow + ``` + + ### 4. Verify the installation + + ```bash + python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))" + ``` + + If a tensor is returned, you've installed TensorFlow successfully. + +* {Windows Native} + + Caution: TensorFlow `2.10` was the **last** TensorFlow release that + supported GPU on native-Windows. + Starting with TensorFlow `2.11`, you will need to install + [TensorFlow in WSL2](https://tensorflow.org/install/pip#windows-[wsl2]), + or install `tensorflow-cpu` and, optionally, try the + [TensorFlow-DirectML-Plugin](https://github.com/microsoft/tensorflow-directml-plugin#tensorflow-directml-plugin-) + + ## 1. System requirements + + * Windows 7 or higher (64-bit) + + Note: Starting with TensorFlow `2.10`, Windows CPU-builds for x86/x64 + processors are built, maintained, tested and released by a third party: + [Intel](https://www.intel.com/). + Installing the windows-native [`tensorflow`](https://pypi.org/project/tensorflow/) + or [`tensorflow-cpu`](https://pypi.org/project/tensorflow-cpu/) + package installs Intel's + [`tensorflow-intel`](https://pypi.org/project/tensorflow-intel/) + package. These packages are provided as-is. Tensorflow will use reasonable + efforts to maintain the availability and integrity of this pip package. + There may be delays if the third party fails to release the pip package. See + [this blog post](https://blog.tensorflow.org/2022/09/announcing-tensorflow-official-build-collaborators.html) + for more information about this + collaboration. + + ### 2. Install Microsoft Visual C++ Redistributable + + Install the *Microsoft Visual C++ Redistributable for Visual Studio 2015, + 2017, and 2019*. Starting with the TensorFlow 2.1.0 version, the + `msvcp140_1.dll` file is required from this package (which may not be + provided from older redistributable packages). The redistributable comes + with *Visual Studio 2019* but can be installed separately: + + 1. Go to the + [Microsoft Visual C++ downloads](https://support.microsoft.com/help/2977003/the-latest-supported-visual-c-downloads). + 2. Scroll down the page to the *Visual Studio 2015, 2017 and 2019* section. + 3. Download and install the *Microsoft Visual C++ Redistributable for + Visual Studio 2015, 2017 and 2019* for your platform. + + Make sure + [long paths are enabled](https://superuser.com/questions/1119883/windows-10-enable-ntfs-long-paths-policy-option-missing) + on Windows. + + ### 3. Install Miniconda + + [Miniconda](https://docs.conda.io/en/latest/miniconda.html) + is the recommended approach for installing TensorFlow with GPU support. + It creates a separate environment to avoid changing any installed + software in your system. This is also the easiest way to install the + required software especially for the GPU setup. + + Download the + [Miniconda Windows Installer](https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe). + Double-click the downloaded file and follow the instructions on the screen. + + ### 4. Create a conda environment + + Create a new conda environment named `tf` with the following command. + + ```bash + conda create --name tf python=3.9 + ``` + + You can deactivate and activate it with the following commands. + + ```bash + conda deactivate + conda activate tf + ``` + + Make sure it is activated for the rest of the installation. + + ### 5. GPU setup + + You can skip this section if you only run TensorFlow on CPU. + + First install + [NVIDIA GPU driver](https://www.nvidia.com/Download/index.aspx) + if you have not. + + Then install the CUDA, cuDNN with conda. + + ```bash + conda install -c conda-forge cudatoolkit=11.2 cudnn=8.1.0 + ``` + + ### 6. Install TensorFlow + + TensorFlow requires a recent version of pip, so upgrade your pip + installation to be sure you're running the latest version. + + ```bash + pip install --upgrade pip + ``` + + Then, install TensorFlow with pip. + + Note: Do not install TensorFlow with conda. It may not have the latest stable + version. pip is recommended since TensorFlow is only officially released to + PyPI. + + ```bash + # Anything above 2.10 is not supported on the GPU on Windows Native + pip install "tensorflow<2.11" + ``` + + ### 7. Verify the installation + + Verify the CPU setup: + + ```bash + python -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))" + ``` + + If a tensor is returned, you've installed TensorFlow successfully. + + Verify the GPU setup: + + ```bash + python -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))" + ``` + + If a list of GPU devices is returned, you've installed TensorFlow + successfully. + +* {Windows WSL2} + + ### 1. System requirements + + * Windows 10 19044 or higher (64-bit). This corresponds to Windows 10 + version 21H2, the November 2021 update. + + See the following documents to: + + * [Download the latest Windows 10 update](https://www.microsoft.com/software-download/windows10). + * [Install WSL2](https://docs.microsoft.com/windows/wsl/install) + * [Setup NVIDIA® GPU support in WSL2](https://docs.nvidia.com/cuda/wsl-user-guide/index.html) + + ### 2. GPU setup + + You can skip this section if you only run TensorFlow on the CPU. + + Install the + [NVIDIA GPU driver](https://www.nvidia.com/Download/index.aspx) + if you have not. You can use the following command to verify it is + installed. + + ```bash + nvidia-smi + ``` + + ### 3. Install TensorFlow + + TensorFlow requires a recent version of pip, so upgrade your pip + installation to be sure you're running the latest version. + + ```bash + pip install --upgrade pip + ``` + + Then, install TensorFlow with pip. + + ```bash + # For GPU users + pip install tensorflow[and-cuda] + # For CPU users + pip install tensorflow + ``` + + ### 4. Verify the installation + + Verify the CPU setup: + + ```bash + python3 -c "import tensorflow as tf; print(tf.reduce_sum(tf.random.normal([1000, 1000])))" + ``` + + If a tensor is returned, you've installed TensorFlow successfully. + + Verify the GPU setup: + + ```bash + python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))" + ``` + + If a list of GPU devices is returned, you've installed TensorFlow + successfully. + + +## Package location + +A few installation mechanisms require the URL of the TensorFlow Python package. +The value you specify depends on your Python version. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    VersionURL
    Linux x86
    Python 3.9 GPU supporthttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.9 CPU-onlyhttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.10 GPU supporthttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.10 CPU-onlyhttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.11 GPU supporthttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.11 CPU-onlyhttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.12 GPU supporthttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.12 CPU-onlyhttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.13 GPU supporthttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Python 3.13 CPU-onlyhttps://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
    Linux Arm64 (CPU-only)
    Python 3.9https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
    Python 3.10https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
    Python 3.11https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
    Python 3.12https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
    Python 3.13https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
    macOS x86 (CPU-only)
    Caution: TensorFlow 2.16 was the last TensorFlow release that supported macOS x86
    Python 3.9https://storage.googleapis.com/tensorflow/versions/2.16.2/tensorflow-2.16.2-cp39-cp39-macosx_10_15_x86_64.whl
    Python 3.10https://storage.googleapis.com/tensorflow/versions/2.16.2/tensorflow-2.16.2-cp310-cp310-macosx_10_15_x86_64.whl
    Python 3.11https://storage.googleapis.com/tensorflow/versions/2.16.2/tensorflow-2.16.2-cp311-cp311-macosx_10_15_x86_64.whl
    Python 3.12https://storage.googleapis.com/tensorflow/versions/2.16.2/tensorflow-2.16.2-cp312-cp312-macosx_10_15_x86_64.whl
    macOS Arm64 (CPU-only)
    Python 3.9https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp39-cp39-macosx_12_0_arm64.whl
    Python 3.10https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp310-cp310-macosx_12_0_arm64.whl
    Python 3.11https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp311-cp311-macosx_12_0_arm64.whl
    Python 3.12https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp312-cp312-macosx_12_0_arm64.whl
    Python 3.13https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow-2.20.0-cp313-cp313-macosx_12_0_arm64.whl
    Windows (CPU-only)
    Python 3.9https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp39-cp39-win_amd64.whl
    Python 3.10https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp310-cp310-win_amd64.whl
    Python 3.11https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp311-cp311-win_amd64.whl
    Python 3.12https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp312-cp312-win_amd64.whl
    Python 3.13https://storage.googleapis.com/tensorflow/versions/2.20.0/tensorflow_cpu-2.20.0-cp313-cp313-win_amd64.whl
    diff --git a/site/en/install/source.md b/site/en/install/source.md index 0394012f91d..dc847f017e9 100644 --- a/site/en/install/source.md +++ b/site/en/install/source.md @@ -1,12 +1,11 @@ # Build from source Build a TensorFlow *pip* package from source and install it on Ubuntu Linux and -macOS. While the instructions might work for other systems, it is only tested and -supported for Ubuntu and macOS. - -Note: We already provide well-tested, pre-built [TensorFlow packages](./pip.md) -for Linux and macOS systems. +macOS. While the instructions might work for other systems, it is only tested +and supported for Ubuntu and macOS. +Note: Well-tested, pre-built [TensorFlow packages](./pip.md) for Linux and macOS +systems are already provided. ## Setup for Linux and macOS @@ -18,33 +17,29 @@ Install the following build tools to configure your development environment.

    Ubuntu

    -sudo apt install python-dev python-pip  # or python3-dev python3-pip
+sudo apt install python3-dev python3-pip
    -

    mac OS

    +

    macOS

    Requires Xcode 9.2 or later.

    Install using the Homebrew package manager:

    -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-export PATH="/usr/local/bin:/usr/local/sbin:$PATH"
-brew install python@2  # or python (Python 3)
+brew install python
    -Install the TensorFlow *pip* package dependencies (if using a virtual environment, -omit the `--user` argument): +Install the TensorFlow *pip* package dependencies (if using a virtual +environment, omit the `--user` argument): 
    -pip install -U --user pip six numpy wheel setuptools mock 'future>=0.17.1'
-pip install -U --user keras_applications --no-deps
-pip install -U --user keras_preprocessing --no-deps
+pip install -U --user pip
    -Note: A `pip` version >19.0 is required to install the TensorFlow 2.0 `.whl` +Note: A `pip` version >19.0 is required to install the TensorFlow 2 `.whl` package. Additional required dependencies are listed in the -setup.py +setup.py.tpl file under `REQUIRED_PACKAGES`. ### Install Bazel @@ -55,132 +50,160 @@ Bazel and automatically downloads the correct Bazel version for TensorFlow. For ease of use, add Bazelisk as the `bazel` executable in your `PATH`. If Bazelisk is not available, you can manually -[install Bazel](https://docs.bazel.build/versions/master/install.html). Make -sure to install a supported Bazel version: any version between -`_TF_MIN_BAZEL_VERSION` and `_TF_MAX_BAZEL_VERSION` as specified in -`tensorflow/configure.py`. +[install Bazel](https://bazel.build/install). Make +sure to install the correct Bazel version from TensorFlow's +[.bazelversion](https://github.com/tensorflow/tensorflow/blob/master/.bazelversion) +file. + +### Install Clang (recommended, Linux only) + +Clang is a C/C++/Objective-C compiler that is compiled in C++ based on LLVM. It +is the default compiler to build TensorFlow starting with TensorFlow 2.13. The +current supported version is LLVM/Clang 17. + +[LLVM Debian/Ubuntu nightly packages](https://apt.llvm.org) provide an automatic +installation script and packages for manual installation on Linux. Make sure you +run the following command if you manually add llvm apt repository to your +package sources: + +
    +sudo apt-get update && sudo apt-get install -y llvm-17 clang-17
+
    + +Now that `/usr/lib/llvm-17/bin/clang` is the actual path to clang in this case. + +Alternatively, you can download and unpack the pre-built +[Clang + LLVM 17](https://github.com/llvm/llvm-project/releases/tag/llvmorg-17.0.2). + +Below is an example of steps you can take to set up the downloaded Clang + LLVM +17 binaries on Debian/Ubuntu operating systems: + +1. Change to the desired destination directory: `cd ` + +1. Load and extract an archive file...(suitable to your architecture): + 
    +    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.2/clang+llvm-17.0.2-x86_64-linux-gnu-ubuntu-22.04.tar.xz
+    
+    tar -xvf clang+llvm-17.0.2-x86_64-linux-gnu-ubuntu-22.04.tar.xz
+    
+
    + +1. Copy the extracted contents (directories and files) to `/usr` (you may need + sudo permissions, and the correct directory may vary by distribution). This + effectively installs Clang and LLVM, and adds it to the path. You should not + have to replace anything, unless you have a previous installation, in which + case you should replace the files: + 
    +    cp -r clang+llvm-17.0.2-x86_64-linux-gnu-ubuntu-22.04/* /usr
+
    + +1. Check the obtained Clang + LLVM 17 binaries version: + 
    +    clang --version
+
    + +1. Now that `/usr/bin/clang` is the actual path to your new clang. You can run + the `./configure` script or manually set environment variables `CC` and + `BAZEL_COMPILER` to this path. ### Install GPU support (optional, Linux only) There is *no* GPU support for macOS. -Read the [GPU support](./gpu.md) guide to install the drivers and additional +Read the [GPU support](./pip.md) guide to install the drivers and additional software required to run TensorFlow on a GPU. Note: It is easier to set up one of TensorFlow's GPU-enabled [Docker images](#docker_linux_builds). ### Download the TensorFlow source code -Use [Git](https://git-scm.com/){:.external} to clone the -[TensorFlow repository](https://github.com/tensorflow/tensorflow){:.external}: +Use [Git](https://git-scm.com/) to clone the +[TensorFlow repository](https://github.com/tensorflow/tensorflow): 
     git clone https://github.com/tensorflow/tensorflow.git
 cd tensorflow
    -The repo defaults to the `master` development branch. You can also checkout a -[release branch](https://github.com/tensorflow/tensorflow/releases){:.external} +The repo defaults to the `master` development branch. You can also check out a +[release branch](https://github.com/tensorflow/tensorflow/releases) to build: 
    -git checkout branch_name  # r1.9, r1.10, etc.
+git checkout branch_name  # r2.2, r2.3, etc.
    ## Configure the build -Configure your system build by running the `./configure` at the root of your -TensorFlow source tree. This script prompts you for the location of TensorFlow -dependencies and asks for additional build configuration options (compiler -flags, for example). +TensorFlow builds are configured by the `.bazelrc` file in the repository's +root directory. The `./configure` or `./configure.py` scripts can be used to +adjust common settings. + +Please run the `./configure` script from the repository's root directory. This +script will prompt you for the location of TensorFlow dependencies and asks for +additional build configuration options (compiler flags, for example). Refer to +the _Sample session_ section for details. 
     ./configure
    +There is also a python version of this script, `./configure.py`. If using a +virtual environment, `python configure.py` prioritizes paths +within the environment, whereas `./configure` prioritizes paths outside +the environment. In both cases you can change the default. + ### Sample session -The `./configure` script The following shows a sample run of `./configure` (your +The following shows a sample run of `./configure` script (your session may differ): @@ -188,7 +211,14 @@ Configuration finished #### GPU support -For [GPU support](./gpu.md), set `cuda=Y` during configuration and specify the +##### from v.2.18.0 +For [GPU support](./pip.md), set `cuda=Y` during configuration and specify the +versions of CUDA and cuDNN if required. Bazel will download CUDA and CUDNN +packages automatically or point to CUDA/CUDNN/NCCL redistributions on local file +system if required. + +##### before v.2.18.0 +For [GPU support](./pip.md), set `cuda=Y` during configuration and specify the versions of CUDA and cuDNN. If your system has multiple versions of CUDA or cuDNN installed, explicitly set the version instead of relying on the default. `./configure` creates symbolic links to your system's CUDA libraries—so if you @@ -198,9 +228,9 @@ building. #### Optimizations For compilation optimization flags, the default (`-march=native`) optimizes the -generated code for your machine's CPU type. However, if building TensorFlow for a -different CPU type, consider a more specific optimization flag. See the -[GCC manual](https://gcc.gnu.org/onlinedocs/gcc-4.5.3/gcc/i386-and-x86_002d64-Options.html){:.external} +generated code for your machine's CPU type. However, if building TensorFlow for +a different CPU type, consider a more specific optimization flag. Check the +[GCC manual](https://gcc.gnu.org/onlinedocs/gcc-4.5.3/gcc/i386-and-x86_002d64-Options.html) for examples. #### Preconfigured configurations @@ -208,97 +238,67 @@ for examples. There are some preconfigured build configs available that can be added to the `bazel build` command, for example: -* `--config=mkl` —Support for the [Intel® MKL-DNN](https://github.com/intel/mkl-dnn){:.external}. -* `--config=monolithic` —Configuration for a mostly static, monolithic build. -* `--config=v1` —Build TensorFlow 1.x instead of 2.x. +* `--config=dbg` —Build with debug info. See + [CONTRIBUTING.md](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md) + for details. +* `--config=mkl` —Support for the + [Intel® MKL-DNN](https://github.com/intel/mkl-dnn). +* `--config=monolithic` —Configuration for a mostly static, monolithic build. -Note: Starting with TensorFlow 1.6, binaries use AVX instructions which may not -run on older CPUs. +## Build and install the pip package -## Build the pip package - -### TensorFlow 2.x +#### Bazel build options -tensorflow:master repo has been updated to build 2.x by default. -[Install Bazel](https://docs.bazel.build/versions/master/install.html) and use -`bazel build ` to create the TensorFlow package. +Refer to the Bazel +[command-line reference](https://bazel.build/reference/command-line-reference) +for +[build options](https://bazel.build/reference/command-line-reference#build-options). -
    -bazel build //tensorflow/tools/pip_package:build_pip_package
-
    +Building TensorFlow from source can use a lot of RAM. If your system is +memory-constrained, limit Bazel's RAM usage with: `--local_ram_resources=2048`. -Note: For GPU support, enable CUDA with `cuda=Y` during the `./configure` stage. +The [official TensorFlow packages](./pip.md) are built with a Clang toolchain +that complies with the manylinux2014 package standard. -### TensorFlow 1.x +### Build the package -To build the 1.x version of TensorFlow from master, use -`bazel build --config=v1` to create a TensorFlow 1.x package. +To build pip package, you need to specify `--repo_env=WHEEL_NAME` flag. +depending on the provided name, package will be created, e.g: +To build tensorflow CPU package: 
    -bazel build --config=v1 //tensorflow/tools/pip_package:build_pip_package
+bazel build //tensorflow/tools/pip_package:wheel --repo_env=USE_PYWRAP_RULES=1 --repo_env=WHEEL_NAME=tensorflow_cpu
    -#### CPU-only - -Use `bazel` to make the TensorFlow package builder with CPU-only support: - +To build tensorflow GPU package: 
    -bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
+bazel build //tensorflow/tools/pip_package:wheel --repo_env=USE_PYWRAP_RULES=1 --repo_env=WHEEL_NAME=tensorflow --config=cuda --config=cuda_wheel
    -#### GPU support - -To make the TensorFlow package builder with GPU support: - +To build tensorflow TPU package: 
    -bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package
+bazel build //tensorflow/tools/pip_package:wheel --repo_env=USE_PYWRAP_RULES=1 --repo_env=WHEEL_NAME=tensorflow_tpu --config=tpu
    -#### Bazel build options - -See the Bazel [command-line reference](https://docs.bazel.build/versions/master/command-line-reference.html) -for -[build options](https://docs.bazel.build/versions/master/command-line-reference.html#build-options). - -Building TensorFlow from source can use a lot of RAM. If your system is -memory-constrained, limit Bazel's RAM usage with: `--local_ram_resources=2048`. - -The [official TensorFlow packages](./pip.md) are built with GCC 4 and use the -older ABI. For GCC 5 and later, make your build compatible with the older ABI -using: `--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"`. ABI compatibility ensures that -custom ops built against the official TensorFlow package continue to work with -the GCC 5 built package. - -### Build the package - -The `bazel build` command creates an executable named `build_pip_package`—this -is the program that builds the `pip` package. Run the executable as shown -below to build a `.whl` package in the `/tmp/tensorflow_pkg` directory. - -To build from a release branch: - +To build nightly package, set `tf_nightly` instead of `tensorflow`, e.g. +to build CPU nightly package: 
    -./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg
+bazel build //tensorflow/tools/pip_package:wheel --repo_env=USE_PYWRAP_RULES=1 --repo_env=WHEEL_NAME=tf_nightly_cpu
    -To build from master, use `--nightly_flag` to get the right dependencies: - +As a result, generated wheel will be located in 
    -./bazel-bin/tensorflow/tools/pip_package/build_pip_package --nightly_flag /tmp/tensorflow_pkg
+bazel-bin/tensorflow/tools/pip_package/wheel_house/
    -Although it is possible to build both CUDA and non-CUDA configurations under the -same source tree, it's recommended to run `bazel clean` when switching between -these two configurations in the same source tree. - ### Install the package The filename of the generated `.whl` file depends on the TensorFlow version and your platform. Use `pip install` to install the package, for example: 
    -pip install /tmp/tensorflow_pkg/tensorflow-version-tags.whl
+pip install bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-version-tags.whl
    Success: TensorFlow is now installed. @@ -308,31 +308,32 @@ Success: TensorFlow is now installed. TensorFlow's Docker development images are an easy way to set up an environment to build Linux packages from source. These images already contain the source -code and dependencies required to build TensorFlow. See the TensorFlow -[Docker guide](./docker.md) for installation and the -[list of available image tags](https://hub.docker.com/r/tensorflow/tensorflow/tags/){:.external}. +code and dependencies required to build TensorFlow. Go to the TensorFlow +[Docker guide](./docker.md) for installation instructions and the +[list of available image tags](https://hub.docker.com/r/tensorflow/tensorflow/tags/). ### CPU-only -The following example uses the `:devel` image to build a CPU-only -Python 2 package from the latest TensorFlow source code. See the -[Docker guide](./docker.md) for available TensorFlow `-devel` tags. +The following example uses the `:devel` image to build a CPU-only package from +the latest TensorFlow source code. Check the [Docker guide](./docker.md) for +available TensorFlow `-devel` tags. -Download the latest development image and start a Docker container that we'll +Download the latest development image and start a Docker container that you'll use to build the *pip* package: 
     docker pull tensorflow/tensorflow:devel
-docker run -it -w /tensorflow -v $PWD:/mnt -e HOST_PERMS="$(id -u):$(id -g)" \
+docker run -it -w /tensorflow_src -v $PWD:/mnt -e HOST_PERMS="$(id -u):$(id -g)" \
     tensorflow/tensorflow:devel bash
 
 git pull  # within the container, download the latest source code
    -The above `docker run` command starts a shell in the `/tensorflow` directory—the -root of the source tree. It mounts the host's current directory in the container's -`/mnt` directory, and passes the host user's information to the container through -an environmental variable (used to set permissions—Docker can make this tricky). +The above `docker run` command starts a shell in the `/tensorflow_src` +directory—the root of the source tree. It mounts the host's current directory in +the container's `/mnt` directory, and passes the host user's information to the +container through an environmental variable (used to set permissions—Docker can +make this tricky). Alternatively, to build a host copy of TensorFlow within a container, mount the host source tree at the container's `/tensorflow` directory: @@ -345,19 +346,20 @@ docker run -it -w /tensorflow -v /path/to/tensorflow:/tensorflow -v $ With the source tree set up, build the TensorFlow package within the container's virtual environment: -1. Configure the build—this prompts the user to answer build configuration questions. -2. Build the tool used to create the *pip* package. -3. Run the tool to create the *pip* package. -4. Adjust the ownership permissions of the file for outside the container. +1. Optional: Configure the build—this prompts the user to answer build + configuration questions. +2. Build the *pip* package. +3. Adjust the ownership permissions of the file for outside the container. 
    -./configure  # answer prompts or use defaults
-
-bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
-
-./bazel-bin/tensorflow/tools/pip_package/build_pip_package /mnt  # create package
-
-chown $HOST_PERMS /mnt/tensorflow-version-tags.whl
+./configure  # if necessary
+
+
+bazel build //tensorflow/tools/pip_package:wheel \
+--repo_env=USE_PYWRAP_RULES=1 --repo_env=WHEEL_NAME=tensorflow_cpu --config=opt
+
+`
+chown $HOST_PERMS bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-version-tags.whl
    Install and verify the package within the container: @@ -365,7 +367,7 @@ Install and verify the package within the container: 
     pip uninstall tensorflow  # remove current version
 
-pip install /mnt/tensorflow-version-tags.whl
+pip install bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-version-tags.whl
 cd /tmp  # don't import from source directory
 python -c "import tensorflow as tf; print(tf.__version__)"
    @@ -373,38 +375,46 @@ Install and verify the package within the container: Success: TensorFlow is now installed. On your host machine, the TensorFlow *pip* package is in the current directory -(with host user permissions): ./tensorflow-version-tags.whl +(with host user permissions): +./tensorflow-version-tags.whl ### GPU support +Note: Starting from Tensorflow v.2.18.0 the wheels can be built from +source on a machine without GPUs and without NVIDIA driver installed. + Docker is the easiest way to build GPU support for TensorFlow since the *host* machine only requires the -[NVIDIA® driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver){:.external} -(the *NVIDIA® CUDA® Toolkit* doesn't have to be installed). See the -[GPU support guide](./gpu.md) and the TensorFlow [Docker guide](./docker.md) -to set up [nvidia-docker](https://github.com/NVIDIA/nvidia-docker){:.external} (Linux only). +[NVIDIA® driver](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions#how-do-i-install-the-nvidia-driver) +(the *NVIDIA® CUDA® Toolkit* doesn't have to be installed). Refer to the +[GPU support guide](./pip.md) and the TensorFlow [Docker guide](./docker.md) to +set up [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) +(Linux only). -The following example downloads the TensorFlow `:devel-gpu-py3` image -and uses `nvidia-docker` to run the GPU-enabled container. This development image -is configured to build a Python 3 *pip* package with GPU support: +The following example downloads the TensorFlow `:devel-gpu` image and uses +`nvidia-docker` to run the GPU-enabled container. This development image is +configured to build a *pip* package with GPU support: 
    -docker pull tensorflow/tensorflow:devel-gpu-py3
-docker run --runtime=nvidia -it -w /tensorflow -v $PWD:/mnt -e HOST_PERMS="$(id -u):$(id -g)" \
-    tensorflow/tensorflow:devel-gpu-py3 bash
+docker pull tensorflow/tensorflow:devel-gpu
+docker run --gpus all -it -w /tensorflow -v $PWD:/mnt -e HOST_PERMS="$(id -u):$(id -g)" \
+    tensorflow/tensorflow:devel-gpu bash
+git pull  # within the container, download the latest source code
    Then, within the container's virtual environment, build the TensorFlow package with GPU support: 
    -./configure  # answer prompts or use defaults
-
-bazel build --config=opt --config=cuda //tensorflow/tools/pip_package:build_pip_package
+./configure  # if necessary
 
-./bazel-bin/tensorflow/tools/pip_package/build_pip_package /mnt  # create package
+
+bazel build //tensorflow/tools/pip_package:wheel \
+--repo_env=USE_PYWRAP_RULES=1 --repo_env=WHEEL_NAME=tensorflow --config=cuda \
+--config=cuda_wheel --config=opt
+
 
-chown $HOST_PERMS /mnt/tensorflow-version-tags.whl
+chown $HOST_PERMS bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-version-tags.whl
    Install and verify the package within the container and check for a GPU: @@ -412,9 +422,9 @@ Install and verify the package within the container and check for a GPU: 
     pip uninstall tensorflow  # remove current version
 
-pip install /mnt/tensorflow-version-tags.whl
+pip install bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-version-tags.whl
 cd /tmp  # don't import from source directory
-python -c "import tensorflow as tf; print(tf.contrib.eager.num_gpus())"
+python -c "import tensorflow as tf; print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))"
    Success: TensorFlow is now installed. @@ -429,7 +439,28 @@ Success: TensorFlow is now installed. + + + + + + + + + + + + + + + + + + + + + @@ -451,7 +482,28 @@ Success: TensorFlow is now installed.
    VersionPython versionCompilerBuild tools
    tensorflow-2.20.03.9-3.13Clang 18.1.8Bazel 7.4.1
    tensorflow-2.19.03.9-3.12Clang 18.1.8Bazel 6.5.0
    tensorflow-2.18.03.9-3.12Clang 17.0.6Bazel 6.5.0
    tensorflow-2.17.03.9-3.12Clang 17.0.6Bazel 6.5.0
    tensorflow-2.16.13.9-3.12Clang 17.0.6Bazel 6.5.0
    tensorflow-2.15.03.9-3.11Clang 16.0.0Bazel 6.1.0
    tensorflow-2.14.03.9-3.11Clang 16.0.0Bazel 6.1.0
    tensorflow-2.13.03.8-3.11Clang 16.0.0Bazel 5.3.0
    tensorflow-2.12.03.8-3.11GCC 9.3.1Bazel 5.3.0
    tensorflow-2.11.03.7-3.10GCC 9.3.1Bazel 5.3.0
    tensorflow-2.10.03.7-3.10GCC 9.3.1Bazel 5.1.1
    tensorflow-2.9.03.7-3.10GCC 9.3.1Bazel 5.0.0
    tensorflow-2.8.03.7-3.10GCC 7.3.1Bazel 4.2.1
    tensorflow-2.7.03.7-3.9GCC 7.3.1Bazel 3.7.2
    tensorflow-2.6.03.6-3.9GCC 7.3.1Bazel 3.7.2
    tensorflow-2.5.03.6-3.9GCC 7.3.1Bazel 3.7.2
    tensorflow-2.4.03.6-3.8GCC 7.3.1Bazel 3.1.0
    tensorflow-2.3.03.5-3.8GCC 7.3.1Bazel 3.1.0
    tensorflow-2.2.03.5-3.8GCC 7.3.1Bazel 2.0.0
    tensorflow-2.1.02.7, 3.5-3.7GCC 7.3.1Bazel 0.27.1
    tensorflow-2.0.02.7, 3.3-3.7GCC 7.3.1Bazel 0.26.1
    tensorflow-1.15.02.7, 3.3-3.7GCC 7.3.1Bazel 0.26.1
    tensorflow-1.14.02.7, 3.3-3.7GCC 4.8Bazel 0.24.1
    tensorflow-1.13.12.7, 3.3-3.7GCC 4.8Bazel 0.19.2
    tensorflow-1.12.02.7, 3.3-3.6GCC 4.8Bazel 0.15.0
    + + + + + + + + + + + + + + + + + + + + + @@ -475,7 +527,25 @@ Success: TensorFlow is now installed.
    VersionPython versionCompilerBuild toolscuDNNCUDA
    tensorflow-2.20.03.9-3.13Clang 18.1.8Bazel 7.4.19.312.5
    tensorflow-2.19.03.9-3.12Clang 18.1.8Bazel 6.5.09.312.5
    tensorflow-2.18.03.9-3.12Clang 17.0.6Bazel 6.5.09.312.5
    tensorflow-2.17.03.9-3.12Clang 17.0.6Bazel 6.5.08.912.3
    tensorflow-2.16.13.9-3.12Clang 17.0.6Bazel 6.5.08.912.3
    tensorflow-2.15.03.9-3.11Clang 16.0.0Bazel 6.1.08.912.2
    tensorflow-2.14.03.9-3.11Clang 16.0.0Bazel 6.1.08.711.8
    tensorflow-2.13.03.8-3.11Clang 16.0.0Bazel 5.3.08.611.8
    tensorflow-2.12.03.8-3.11GCC 9.3.1Bazel 5.3.08.611.8
    tensorflow-2.11.03.7-3.10GCC 9.3.1Bazel 5.3.08.111.2
    tensorflow-2.10.03.7-3.10GCC 9.3.1Bazel 5.1.18.111.2
    tensorflow-2.9.03.7-3.10GCC 9.3.1Bazel 5.0.08.111.2
    tensorflow-2.8.03.7-3.10GCC 7.3.1Bazel 4.2.18.111.2
    tensorflow-2.7.03.7-3.9GCC 7.3.1Bazel 3.7.28.111.2
    tensorflow-2.6.03.6-3.9GCC 7.3.1Bazel 3.7.28.111.2
    tensorflow-2.5.03.6-3.9GCC 7.3.1Bazel 3.7.28.111.2
    tensorflow-2.4.03.6-3.8GCC 7.3.1Bazel 3.1.08.011.0
    tensorflow-2.3.03.5-3.8GCC 7.3.1Bazel 3.1.07.610.1
    tensorflow-2.2.03.5-3.8GCC 7.3.1Bazel 2.0.07.610.1
    tensorflow-2.1.02.7, 3.5-3.7GCC 7.3.1Bazel 0.27.17.610.1
    tensorflow-2.0.02.7, 3.3-3.7GCC 7.3.1Bazel 0.26.17.410.0
    tensorflow_gpu-1.15.02.7, 3.3-3.7GCC 7.3.1Bazel 0.26.17.410.0
    tensorflow_gpu-1.14.02.7, 3.3-3.7GCC 4.8Bazel 0.24.17.410.0
    tensorflow_gpu-1.13.12.7, 3.3-3.7GCC 4.8Bazel 0.19.27.410.0
    tensorflow_gpu-1.12.02.7, 3.3-3.6GCC 4.8Bazel 0.15.079
    + + + + + + + + + + + + + + + + + + diff --git a/site/en/install/source_rpi.md b/site/en/install/source_rpi.md deleted file mode 100644 index 5982bc4fd01..00000000000 --- a/site/en/install/source_rpi.md +++ /dev/null @@ -1,92 +0,0 @@ -# Build from source for the Raspberry Pi - -This guide builds a TensorFlow package for a -[Raspberry Pi](https://www.raspberrypi.org/){:.external} device running -[Raspbian 9.0](https://www.raspberrypi.org/downloads/raspbian/){:.external}. -While the instructions might work for other Raspberry Pi variants, it is only -tested and supported for this configuration. - -We recommend *cross-compiling* the TensorFlow Raspbian package. Cross-compilation -is using a different platform to build the package than deploy to. Instead of -using the Raspberry Pi's limited RAM and comparatively slow processor, it's -easier to build TensorFlow on a more powerful host machine running Linux, macOS, -or Windows. - -Note: We already provide well-tested, pre-built [TensorFlow packages](./pip.md) -for Raspbian systems. - - -## Setup for host - -### Install Docker - -To simplify dependency management, the build script uses -[Docker](https://docs.docker.com/install/){:.external} to create a virtual Linux -development environment for compilation. Verify your Docker install by executing: -`docker run --rm hello-world` - -### Download the TensorFlow source code - -Use [Git](https://git-scm.com/){:.external} to clone the -[TensorFlow repository](https://github.com/tensorflow/tensorflow){:.external}: - -
    -git clone https://github.com/tensorflow/tensorflow.git
-cd tensorflow
-
    - -The repo defaults to the `master` development branch. You can also checkout a -[release branch](https://github.com/tensorflow/tensorflow/releases){:.external} -to build: - -
    -git checkout branch_name  # r1.9, r1.10, etc.
-
    - -Key Point: If you're having build problems on the latest development branch, try -a release branch that is known to work. - - -## Build from source - -Cross-compile the TensorFlow source code to build a Python *pip* package with -ARMv7 [NEON instructions](https://developer.arm.com/technologies/neon){:.external} -that works on Raspberry Pi 2 and 3 devices. The build script launches a Docker -container for compilation. Choose between Python 3 and Python 2.7 for the target -package: - -
    -
    -

    Python 3

    -
    -CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3 -e CROSSTOOL_PYTHON_INCLUDE_PATH=/usr/include/python3.4" \\
-    tensorflow/tools/ci_build/ci_build.sh PI-PYTHON3 \\
-    tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
-
    -
    -
    -

    Python 2.7

    -
    -tensorflow/tools/ci_build/ci_build.sh PI \\
-    tensorflow/tools/ci_build/pi/build_raspberry_pi.sh
-
    -
    -
    - -To build a package that supports all Raspberry Pi devices—including the Pi 1 and -Zero—pass the `PI_ONE` argument, for example: - -
    -tensorflow/tools/ci_build/ci_build.sh PI \
-    tensorflow/tools/ci_build/pi/build_raspberry_pi.sh PI_ONE
-
    - -When the build finishes (~30 minutes), a `.whl` package file is created in the -output-artifacts directory of the host's source tree. Copy the wheel file to the -Raspberry Pi and install with `pip`: - -
    -pip install tensorflow-version-cp34-none-linux_armv7l.whl
-
    - -Success: TensorFlow is now installed on Raspbian. diff --git a/site/en/install/source_windows.md b/site/en/install/source_windows.md index 86ba195a15e..efc0f7a9286 100644 --- a/site/en/install/source_windows.md +++ b/site/en/install/source_windows.md @@ -1,9 +1,9 @@ # Build from source on Windows -Build a TensorFlow *pip* package from source and install it on Windows. +Build a TensorFlow *pip* package from the source and install it on Windows. -Note: We already provide well-tested, pre-built [TensorFlow packages](./pip.md) -for Windows systems. +Note: We already provide well-tested, pre-built +[TensorFlow packages](./pip.md) for Windows systems. ## Setup for Windows @@ -13,15 +13,16 @@ environment. ### Install Python and the TensorFlow package dependencies Install a -[Python 3.5.x or Python 3.6.x 64-bit release for Windows](https://www.python.org/downloads/windows/){:.external}. -Select *pip* as an optional feature and add it to your `%PATH%` environmental variable. +[Python 3.9+ 64-bit release for Windows](https://www.python.org/downloads/windows/). +Select *pip* as an optional feature and add it to your `%PATH%` environmental +variable. Install the TensorFlow *pip* package dependencies: 
    -pip3 install six numpy wheel
-pip3 install keras_applications==1.0.6 --no-deps
-pip3 install keras_preprocessing==1.0.5 --no-deps
+pip3 install -U pip
+pip3 install -U six numpy wheel packaging
+pip3 install -U keras_preprocessing --no-deps
    The dependencies are listed in the @@ -30,49 +31,64 @@ file under `REQUIRED_PACKAGES`. ### Install Bazel -[Install Bazel 0.24.1](https://docs.bazel.build/versions/master/install-windows.html){:.external}, -the build tool used to compile TensorFlow. Set up Bazel to -[build C++](https://docs.bazel.build/versions/master/windows.html#build-c){:.external}. +[Install Bazel](./source.md#install-bazel), the build tool used to compile +TensorFlow. For Bazel version, see the +[tested build configurations](#tested-build-configurations) for Windows. +Configure Bazel to +build +C++. Add the location of the Bazel executable to your `%PATH%` environment variable. -Ensure that your version of Bazel uses a [tested build configuration](#tested_build_configurations) of TensorFlow. - ### Install MSYS2 -[Install MSYS2](https://www.msys2.org/){:.external} for the bin tools needed to +[Install MSYS2](https://www.msys2.org/) for the bin tools needed to build TensorFlow. If MSYS2 is installed to `C:\msys64`, add `C:\msys64\usr\bin` to your `%PATH%` environment variable. Then, using `cmd.exe`, run: 
    +pacman -Syu (requires a console restart)
 pacman -S git patch unzip
+pacman -S git patch unzip rsync
    -### Install Visual C++ Build Tools 2017 +Note: Clang will be the preferred compiler to build TensorFlow CPU wheels on the Windows Platform starting with TF 2.16.1 The currently supported version is LLVM/clang 17.0.6. + +Note: To build with Clang on Windows, it is required to install both LLVM and Visual C++ Build tools as although Windows uses clang-cl.exe as the compiler, Visual C++ Build tools are needed to link to Visual C++ libraries + +### Install Visual C++ Build Tools 2022 -Install the *Visual C++ build tools 2017*. This comes with *Visual Studio 2017* +Install the *Visual C++ build tools 2022*. This comes with *Visual Studio Community 2022* but can be installed separately: 1. Go to the - [Visual Studio downloads](https://visualstudio.microsoft.com/vs/older-downloads/){:.external}, -2. Select *Redistributables and Build Tools*, + [Visual Studio downloads](https://visualstudio.microsoft.com/downloads/), +2. Select *Tools for Visual Studio or Other Tools, Framework and Redistributables*, 3. Download and install: - - *Microsoft Visual C++ 2017 Redistributable* - - *Microsoft Build Tools 2017* + - *Build Tools for Visual Studio 2022* + - *Microsoft Visual C++ Redistributables for Visual Studio 2022* + +Note: TensorFlow is tested against the *Visual Studio Community 2022*. + +### Install LLVM + +1. Go to the + [LLVM downloads](https://github.com/llvm/llvm-project/releases/), +2. Download and install Windows-compatible LLVM in C:/Program Files/LLVM e.g., LLVM-17.0.6-win64.exe -Note: TensorFlow is tested against the *Visual Studio 2017*. ### Install GPU support (optional) -See the Windows [GPU support](./gpu.md) guide to install the drivers and additional -software required to run TensorFlow on a GPU. +See the Windows [GPU support](./gpu.md) guide to install the drivers and +additional software required to run TensorFlow on a GPU. +Note: GPU support on native-Windows is only available for 2.10 or earlier versions, starting in TF 2.11, CUDA build is not supported for Windows. For using TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2 or use tensorflow-cpu with TensorFlow-DirectML-Plugin ### Download the TensorFlow source code -Use [Git](https://git-scm.com/){:.external} to clone the -[TensorFlow repository](https://github.com/tensorflow/tensorflow){:.external} +Use [Git](https://git-scm.com/) to clone the +[TensorFlow repository](https://github.com/tensorflow/tensorflow) (`git` is installed with MSYS2): 
    @@ -80,8 +96,8 @@ Use [Git](https://git-scm.com/){:.external} to clone the
 cd tensorflow
    -The repo defaults to the `master` development branch. You can also checkout a -[release branch](https://github.com/tensorflow/tensorflow/releases){:.external} +The repo defaults to the `master` development branch. You can also check out a +[release branch](https://github.com/tensorflow/tensorflow/releases) to build: 
    @@ -91,11 +107,38 @@ git checkout branch_name  # r1.9, r1.10, etc.
 Key Point: If you're having build problems on the latest development branch, try
 a release branch that is known to work.
 
+## Optional: Environmental Variable Set Up
+Run the following commands before running the build command to avoid issues with package creation:
+(If the below commands were set up while installing the packages, please ignore them). Run `set` to check if all the paths were set correctly, run `echo %Environmental Variable%` e.g., `echo %BAZEL_VC%` to check the path set up for a specific Environmental Variable
+
+ Python path set up issue [tensorflow:issue#59943](https://github.com/tensorflow/tensorflow/issues/59943),[tensorflow:issue#9436](https://github.com/tensorflow/tensorflow/issues/9436),[tensorflow:issue#60083](https://github.com/tensorflow/tensorflow/issues/60083)
 
-## Configure the build
+
    +set PATH=path/to/python;%PATH% # [e.g. (C:/Python311)]
+set PATH=path/to/python/Scripts;%PATH% # [e.g. (C:/Python311/Scripts)] 
+set PYTHON_BIN_PATH=path/to/python_virtualenv/Scripts/python.exe 
+set PYTHON_LIB_PATH=path/to/python virtualenv/lib/site-packages 
+set PYTHON_DIRECTORY=path/to/python_virtualenv/Scripts 
+
    
 
-Configure your system build by running the following at the root of your
-TensorFlow source tree:
+Bazel/MSVC/CLANG path set up issue [tensorflow:issue#54578](https://github.com/tensorflow/tensorflow/issues/54578)
+
+
    +set BAZEL_SH=C:/msys64/usr/bin/bash.exe 
+set BAZEL_VS=C:/Program Files/Microsoft Visual Studio/2022/BuildTools 
+set BAZEL_VC=C:/Program Files/Microsoft Visual Studio/2022/BuildTools/VC 
+set Bazel_LLVM=C:/Program Files/LLVM (explicitly tell Bazel where LLVM is installed by BAZEL_LLVM, needed while using CLANG)
+set PATH=C:/Program Files/LLVM/bin;%PATH% (Optional, needed while using CLANG as Compiler)
+
    
+
+## Optional: Configure the build
+
+TensorFlow builds are configured by the `.bazelrc` file in the repository's
+root directory. The `./configure` or `./configure.py` scripts can be used to
+adjust common settings.
+
+If you need to change the configuration, run the `./configure` script from
+the repository's root directory.
 
 
     python ./configure.py
@@ -103,98 +146,106 @@ python ./configure.py
 
 This script prompts you for the location of TensorFlow dependencies and asks for
 additional build configuration options (compiler flags, for example). The
-following shows a sample run of `python ./configure.py` (your session may differ):
+following shows a sample run of `python ./configure.py` (your session may
+differ):
 
 
 
-### Configuration options
-
-For [GPU support](./gpu.md), specify the versions of CUDA and cuDNN. If your
-system has multiple versions of CUDA or cuDNN installed, explicitly set the
-version instead of relying on the default. `./configure.py` creates symbolic links
-to your system's CUDA libraries—so if you update your CUDA library paths, this
-configuration step must be run again before building.
-
-Note: Starting with TensorFlow 1.6, binaries use AVX instructions which may not
-run on older CPUs.
+## Build and install the pip package
 
+The pip package is built in two steps. A `bazel build` command creates a
+"package-builder" program. You then run the package-builder to create the
+package.
 
-## Build the pip package
-
-### TensorFlow 2.x
+### Build the package-builder
 
 tensorflow:master repo has been updated to build 2.x by default.
 [Install Bazel](https://docs.bazel.build/versions/master/install.html) and use
-`bazel build ` to create the TensorFlow package.
+`bazel build ` to create the TensorFlow package-builder.
 
 
    -bazel build //tensorflow/tools/pip_package:build_pip_package
+bazel build //tensorflow/tools/pip_package:wheel
 
    
 
+#### CPU-only
 
-### TensorFlow 1.x
-
-To build the 1.x version of TensorFlow from master, use
-`bazel build --config=v1` to create a TensorFlow 1.x package.
+Use `bazel` to make the TensorFlow package builder with CPU-only support:
 
+##### Build with MSVC 
 
    -bazel build --config=v1 //tensorflow/tools/pip_package:build_pip_package
+bazel build --config=opt --repo_env=TF_PYTHON_VERSION=3.11 //tensorflow/tools/pip_package:wheel --repo_env=WHEEL_NAME=tensorflow_cpu
 
    
 
-#### CPU-only
-
-Use `bazel` to make the TensorFlow package builder with CPU-only support:
+##### Build with CLANG 
+Use --config=`win_clang` to build TenorFlow with the CLANG Compiler:
 
 
    -bazel build --config=opt //tensorflow/tools/pip_package:build_pip_package
+bazel build --config=win_clang --repo_env=TF_PYTHON_VERSION=3.11 //tensorflow/tools/pip_package:wheel --repo_env=WHEEL_NAME=tensorflow_cpu
 
    
 
 #### GPU support
 
+Note: GPU support on native-Windows is only available for 2.10 or earlier versions, starting in TF 2.11, CUDA build is not supported for Windows. For using TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2 or use tensorflow-cpu with TensorFlow-DirectML-Plugin
+
 To make the TensorFlow package builder with GPU support:
 
 
     bazel build --config=opt --config=cuda --define=no_tensorflow_py_deps=true //tensorflow/tools/pip_package:build_pip_package
 
    
 
+Commands to clean the bazel cache to resolve errors due to invalid or outdated cached data, bazel clean with --expunge flag removes files permanently
+
+
    +bazel clean 
+bazel clean --expunge  
+
    
+
 #### Bazel build options
 
-Use this option when building to avoid issue with package creation:
+Use this option when building to avoid issues with package creation:
 [tensorflow:issue#22390](https://github.com/tensorflow/tensorflow/issues/22390)
 
 
    @@ -213,30 +264,37 @@ to suppress nvcc warning messages.
 
 ### Build the package
 
-The `bazel build` command creates an executable named `build_pip_package`—this
-is the program that builds the `pip` package. For example, the following builds a
-`.whl` package in the `C:/tmp/tensorflow_pkg` directory:
+To build a pip package, you need to specify the --repo_env=WHEEL_NAME flag. 
+Depending on the provided name, the package will be created. For example:
 
-
    -bazel-bin\tensorflow\tools\pip_package\build_pip_package C:/tmp/tensorflow_pkg
+To build tensorflow CPU package:
+
    +bazel build //tensorflow/tools/pip_package:wheel --repo_env=WHEEL_NAME=tensorflow_cpu
+
    
+
+To build nightly package, set `tf_nightly` instead of `tensorflow`, e.g.
+to build CPU nightly package:
+
    +bazel build //tensorflow/tools/pip_package:wheel --repo_env=WHEEL_NAME=tf_nightly_cpu
+
    
+
+As a result, generated wheel will be located in
+
    +bazel-bin/tensorflow/tools/pip_package/wheel_house/
 
    
 
-Although it is possible to build both CUDA and non-CUDA configs under the
-same source tree, we recommend running `bazel clean` when switching between
-these two configurations in the same source tree.
 
 ### Install the package
 
 The filename of the generated `.whl` file depends on the TensorFlow version and
-your platform. Use `pip3 install` to install the package, for example:
+your platform. Use `pip install` to install the package, for example:
 
-
    -pip3 install C:/tmp/tensorflow_pkg/tensorflow-version-cp36-cp36m-win_amd64.whl
+
    +pip install bazel-bin/tensorflow/tools/pip_package/wheel_house/tensorflow-version-tags.whl
 
    
 
 Success: TensorFlow is now installed.
 
-
 ## Build using the MSYS shell
 
 TensorFlow can also be built using the MSYS shell. Make the changes listed
@@ -245,9 +303,9 @@ below, then follow the previous instructions for the Windows native command line
 
 ### Disable MSYS path conversion {:.hide-from-toc}
 
-MSYS automatically converts arguments that look like Unix paths to Windows paths,
-and this doesn't work with `bazel`. (The label `//foo/bar:bin` is considered a
-Unix absolute path since it starts with a slash.)
+MSYS automatically converts arguments that look like Unix paths to Windows
+paths, and this doesn't work with `bazel`. (The label `//path/to:bin` is
+considered a Unix absolute path since it starts with a slash.)
 
 
     export MSYS_NO_PATHCONV=1
@@ -258,22 +316,23 @@ Unix absolute path since it starts with a slash.)
 
 Add the Bazel and Python installation directories to your `$PATH` environmental
 variable. If Bazel is installed to `C:\tools\bazel.exe`, and Python to
-`C:\Python36\python.exe`, set your `PATH` with:
+`C:\Python\python.exe`, set your `PATH` with:
 
 
     # Use Unix-style with ':' as separator
 export PATH="/c/tools:$PATH"
-export PATH="/c/Python36:$PATH"
+export PATH="/c/path/to/Python:$PATH"
 
    
 
 For GPU support, add the CUDA and cuDNN bin directories to your `$PATH`:
 
 
    -export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.0/bin:$PATH"
-export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.0/extras/CUPTI/libx64:$PATH"
+export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.0/bin:$PATH"
+export PATH="/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.0/extras/CUPTI/libx64:$PATH"
 export PATH="/c/tools/cuda/bin:$PATH"
 
    
 
+Note: Starting in TF 2.11, CUDA build is not supported for Windows. For using TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2 or use tensorflow-cpu with TensorFlow-DirectML-Plugin
 
 
 ## Tested build configurations
@@ -282,7 +341,27 @@ For GPU support, add the CUDA and cuDNN bin directories to your `$PATH`:
    VersionPython versionCompilerBuild tools
    tensorflow-2.16.13.9-3.12Clang from Xcode 13.6Bazel 6.5.0
    tensorflow-2.15.03.9-3.11Clang from xcode 10.15Bazel 6.1.0
    tensorflow-2.14.03.9-3.11Clang from xcode 10.15Bazel 6.1.0
    tensorflow-2.13.03.8-3.11Clang from xcode 10.15Bazel 5.3.0
    tensorflow-2.12.03.8-3.11Clang from xcode 10.15Bazel 5.3.0
    tensorflow-2.11.03.7-3.10Clang from xcode 10.14Bazel 5.3.0
    tensorflow-2.10.03.7-3.10Clang from xcode 10.14Bazel 5.1.1
    tensorflow-2.9.03.7-3.10Clang from xcode 10.14Bazel 5.0.0
    tensorflow-2.8.03.7-3.10Clang from xcode 10.14Bazel 4.2.1
    tensorflow-2.7.03.7-3.9Clang from xcode 10.11Bazel 3.7.2
    tensorflow-2.6.03.6-3.9Clang from xcode 10.11Bazel 3.7.2
    tensorflow-2.5.03.6-3.9Clang from xcode 10.11Bazel 3.7.2
    tensorflow-2.4.03.6-3.8Clang from xcode 10.3Bazel 3.1.0
    tensorflow-2.3.03.5-3.8Clang from xcode 10.1Bazel 3.1.0
    tensorflow-2.2.03.5-3.8Clang from xcode 10.1Bazel 2.0.0
    tensorflow-2.1.02.7, 3.5-3.7Clang from xcode 10.1Bazel 0.27.1
    tensorflow-2.0.02.7, 3.5-3.7Clang from xcode 10.1Bazel 0.27.1
    tensorflow-2.0.02.7, 3.3-3.7Clang from xcode 10.1Bazel 0.26.1
    tensorflow-1.15.02.7, 3.3-3.7Clang from xcode 10.1Bazel 0.26.1
    tensorflow-1.14.02.7, 3.3-3.7Clang from xcodeBazel 0.24.1
    tensorflow-1.13.12.7, 3.3-3.7Clang from xcodeBazel 0.19.2
    tensorflow-1.12.02.7, 3.3-3.6Clang from xcodeBazel 0.15.0
    + + + + + + + + + + + + + + + + + + + + @@ -301,13 +380,25 @@ For GPU support, add the CUDA and cuDNN bin directories to your `$PATH`:
    VersionPython versionCompilerBuild tools
    tensorflow-2.20.03.9-3.13CLANG 18.1.4Bazel 7.4.1
    tensorflow-2.19.03.9-3.12CLANG 18.1.4Bazel 6.5.0
    tensorflow-2.18.03.9-3.12CLANG 17.0.6Bazel 6.5.0
    tensorflow-2.17.03.9-3.12CLANG 17.0.6Bazel 6.5.0
    tensorflow-2.16.13.9-3.12CLANG 17.0.6Bazel 6.5.0
    tensorflow-2.15.03.9-3.11MSVC 2019Bazel 6.1.0
    tensorflow-2.14.03.9-3.11MSVC 2019Bazel 6.1.0
    tensorflow-2.12.03.8-3.11MSVC 2019Bazel 5.3.0
    tensorflow-2.11.03.7-3.10MSVC 2019Bazel 5.3.0
    tensorflow-2.10.03.7-3.10MSVC 2019Bazel 5.1.1
    tensorflow-2.9.03.7-3.10MSVC 2019Bazel 5.0.0
    tensorflow-2.8.03.7-3.10MSVC 2019Bazel 4.2.1
    tensorflow-2.7.03.7-3.9MSVC 2019Bazel 3.7.2
    tensorflow-2.6.03.6-3.9MSVC 2019Bazel 3.7.2
    tensorflow-2.5.03.6-3.9MSVC 2019Bazel 3.7.2
    tensorflow-2.4.03.6-3.8MSVC 2019Bazel 3.1.0
    tensorflow-2.3.03.5-3.8MSVC 2019Bazel 3.1.0
    tensorflow-2.2.03.5-3.8MSVC 2019Bazel 2.0.0
    tensorflow-2.1.03.5-3.7MSVC 2019Bazel 0.27.1-0.29.1
    tensorflow-2.0.03.5-3.7MSVC 2017Bazel 0.26.1
    tensorflow-1.15.03.5-3.7MSVC 2017Bazel 0.26.1
    tensorflow-1.14.03.5-3.7MSVC 2017Bazel 0.24.1-0.25.2
    tensorflow-1.13.03.5-3.7MSVC 2015 update 3Bazel 0.19.0-0.21.0
    tensorflow-1.12.03.5-3.6MSVC 2015 update 3Bazel 0.15.0
    ### GPU +Note: GPU support on native-Windows is only available for 2.10 or earlier versions, starting in TF 2.11, CUDA build is not supported for Windows. For using TensorFlow GPU on Windows, you will need to build/install TensorFlow in WSL2 or use tensorflow-cpu with TensorFlow-DirectML-Plugin + + + + + + + + + + + - + diff --git a/site/en/js/README.md b/site/en/js/README.md deleted file mode 100644 index 5a3a34677b4..00000000000 --- a/site/en/js/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Welcome to the warp zone! - -# TensorFlow.js - -These docs are available here: https://github.com/tensorflow/tfjs-website/tree/master/docs diff --git a/site/en/lite/README.md b/site/en/lite/README.md deleted file mode 100644 index 43c3249dc7b..00000000000 --- a/site/en/lite/README.md +++ /dev/null @@ -1,6 +0,0 @@ -Welcome to the warp zone! - -# TensorFlow Lite - -These docs are available here: -https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/g3doc diff --git a/site/en/mlir/README.md b/site/en/mlir/README.md deleted file mode 100644 index 614f9f693c8..00000000000 --- a/site/en/mlir/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Welcome to the warp zone! - -# TensorFlow MLIR - -These docs are available here: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/mlir/g3doc diff --git a/site/en/neural_structured_learning/README.md b/site/en/neural_structured_learning/README.md deleted file mode 100644 index 85c905af170..00000000000 --- a/site/en/neural_structured_learning/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Welcome to the warp zone! - -# Neural Structured Learning - -These docs are available here: https://github.com/tensorflow/neural-structured-learning/tree/master/g3doc diff --git a/site/en/probability/README.md b/site/en/probability/README.md deleted file mode 100644 index c17e5ba447b..00000000000 --- a/site/en/probability/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Welcome to the warp zone! - -# TensorFlow Probability - -These docs are available here: https://github.com/tensorflow/probability/tree/master/tensorflow_probability/g3doc diff --git a/site/en/r1/guide/autograph.ipynb b/site/en/r1/guide/autograph.ipynb index 7cdbfd706d6..64d631a52b3 100644 --- a/site/en/r1/guide/autograph.ipynb +++ b/site/en/r1/guide/autograph.ipynb @@ -1,25 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "autograph.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Jxv6goXm7oGF" }, "source": [ @@ -30,11 +13,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "llMNufAK7nfK", - "colab": {} + "cellView": "form", + "id": "llMNufAK7nfK" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\"); { display-mode: \"form\" }\n", "# you may not use this file except in compliance with the License.\n", @@ -47,14 +31,11 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8Byow2J6LaPl" }, "source": [ @@ -64,7 +45,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kGXS3UWBBNoc" }, "source": [ @@ -81,17 +61,29 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "kGXS3UWBBNoc" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "CydFK2CL7ZHA" }, "source": [ - "[AutoGraph](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/) helps you write complicated graph code using normal Python. Behind the scenes, AutoGraph automatically transforms your code into the equivalent [TensorFlow graph code](https://www.tensorflow.org/r1/guide/graphs). AutoGraph already supports much of the Python language, and that coverage continues to grow. For a list of supported Python language features, see the [Autograph capabilities and limitations](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/limitations.md)." + "[AutoGraph](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/autograph/) helps you write complicated graph code using normal Python. Behind the scenes, AutoGraph automatically transforms your code into the equivalent [TensorFlow graph code](https://www.tensorflow.org/r1/guide/graphs). AutoGraph already supports much of the Python language, and that coverage continues to grow. For a list of supported Python language features, see the [Autograph capabilities and limitations](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/autograph/g3doc/reference/limitations.md)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "n4EKOpw9mObL" }, "source": [ @@ -102,50 +94,41 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "mT7meGqrZTz9", - "colab": {} + "id": "794l9aK_BjFq" }, + "outputs": [], + "source": [ + "import tensorflow.compat.v1 as tf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mT7meGqrZTz9" + }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", "layers = tf.keras.layers\n", "\n", "import numpy as np\n", "import matplotlib.pyplot as plt" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Hh1PajmUJMNp" }, "source": [ "We'll enable [eager execution](https://www.tensorflow.org/r1/guide/eager) for demonstration purposes, but AutoGraph works in both eager and [graph execution](https://www.tensorflow.org/r1/guide/graphs) environments:" ] }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ks_hiqcSJNvg", - "colab": {} - }, - "source": [ - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WR4lG3hsuWQT" }, "source": [ @@ -155,7 +138,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ohbSnA79mcJV" }, "source": [ @@ -170,11 +152,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "aA3gOodCBkOw", - "colab": {} + "id": "aA3gOodCBkOw" }, + "outputs": [], "source": [ "def square_if_positive(x):\n", " if x > 0:\n", @@ -182,14 +164,11 @@ " else:\n", " x = 0.0\n", " return x" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LICw4XQFZrhH" }, "source": [ @@ -198,21 +177,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "_EMhGUjRZoKQ", - "colab": {} + "id": "_EMhGUjRZoKQ" }, + "outputs": [], "source": [ "print(tf.autograph.to_code(square_if_positive))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xpK0m4TCvkJq" }, "source": [ @@ -221,22 +197,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "I1RtBvoKBxq5", - "colab": {} + "id": "I1RtBvoKBxq5" }, + "outputs": [], "source": [ "print('Eager results: %2.2f, %2.2f' % (square_if_positive(tf.constant(9.0)),\n", " square_if_positive(tf.constant(-9.0))))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Fpk3MxVVv5gn" }, "source": [ @@ -245,11 +218,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "SGjSq0WQvwGs", - "colab": {} + "id": "SGjSq0WQvwGs" }, + "outputs": [], "source": [ "tf_square_if_positive = tf.autograph.to_graph(square_if_positive)\n", "\n", @@ -260,27 +233,24 @@ " g_out2 = tf_square_if_positive(tf.constant(-9.0))\n", " with tf.Session() as sess:\n", " print('Graph results: %2.2f, %2.2f\\n' % (sess.run(g_out1), sess.run(g_out2)))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "m-jWmsCmByyw" }, "source": [ - "AutoGraph supports common Python statements like `while`, `for`, `if`, `break`, and `return`, with support for nesting. Compare this function with the complicated graph verson displayed in the following code blocks:" + "AutoGraph supports common Python statements like `while`, `for`, `if`, `break`, and `return`, with support for nesting. Compare this function with the complicated graph version displayed in the following code blocks:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "toxKBOXbB1ro", - "colab": {} + "id": "toxKBOXbB1ro" }, + "outputs": [], "source": [ "# Continue in a loop\n", "def sum_even(items):\n", @@ -297,27 +267,22 @@ "\n", "with tf.Graph().as_default(), tf.Session() as sess:\n", " print('Graph result: %d\\n\\n' % sess.run(tf_sum_even(tf.constant([10,12,15,20]))))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jlyQgxYsYSXr", - "colab": {} + "id": "jlyQgxYsYSXr" }, + "outputs": [], "source": [ "print(tf.autograph.to_code(sum_even))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FUJJ-WTdCGeq" }, "source": [ @@ -328,11 +293,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "BKhFNXDic4Mw", - "colab": {} + "id": "BKhFNXDic4Mw" }, + "outputs": [], "source": [ "@tf.function(\n", " experimental_autograph_options=tf.autograph.experimental.Feature.EQUALITY_OPERATORS)\n", @@ -354,16 +319,12 @@ " # The result works like a regular op: takes tensors in, returns tensors.\n", " # You can inspect the graph using tf.get_default_graph().as_graph_def()\n", " with tf.Session() as sess:\n", - " sess.run(final_i)\n", - "\n" - ], - "execution_count": 0, - "outputs": [] + " sess.run(final_i)\n" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-pkEH6OecW7h" }, "source": [ @@ -375,7 +336,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "axoRAkWi0CQG" }, "source": [ @@ -386,11 +346,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "IAOgh62zCPZ4", - "colab": {} + "id": "IAOgh62zCPZ4" }, + "outputs": [], "source": [ "@tf.function(\n", " experimental_autograph_options=(\n", @@ -405,14 +365,11 @@ " print(sess.run(inverse(tf.constant(0.0))))\n", " except tf.errors.InvalidArgumentError as e:\n", " print('Got error message:\\n %s' % e.message)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1UYgrNtCV9p7" }, "source": [ @@ -423,11 +380,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ehBac9rUR6nh", - "colab": {} + "id": "ehBac9rUR6nh" }, + "outputs": [], "source": [ "@tf.function(\n", " experimental_autograph_options=tf.autograph.experimental.Feature.BUILTIN_FUNCTIONS)\n", @@ -440,14 +397,11 @@ "\n", "with tf.Graph().as_default(), tf.Session() as sess:\n", " sess.run(count(tf.constant(5)))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mtpegD_YR6HK" }, "source": [ @@ -458,11 +412,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ABX070KwCczR", - "colab": {} + "id": "ABX070KwCczR" }, + "outputs": [], "source": [ "@tf.function(\n", " experimental_autograph_options=tf.autograph.experimental.Feature.LISTS)\n", @@ -477,14 +431,11 @@ "\n", "with tf.Graph().as_default(), tf.Session() as sess:\n", " print(sess.run(arange(tf.constant(10))))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qj7am2I_xvTJ" }, "source": [ @@ -493,11 +444,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "4yyNOf-Twr6s", - "colab": {} + "id": "4yyNOf-Twr6s" }, + "outputs": [], "source": [ "@tf.function(\n", " experimental_autograph_options=tf.autograph.experimental.Feature.EQUALITY_OPERATORS)\n", @@ -513,14 +464,11 @@ " print(sess.run(nearest_odd_square(tf.constant(4))))\n", " print(sess.run(nearest_odd_square(tf.constant(5))))\n", " print(sess.run(nearest_odd_square(tf.constant(6))))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jXAxjeBr1qWK" }, "source": [ @@ -529,11 +477,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ucmZyQVL03bF", - "colab": {} + "id": "ucmZyQVL03bF" }, + "outputs": [], "source": [ "@tf.function\n", "def square_until_stop(x, y):\n", @@ -544,14 +492,11 @@ "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", " print(sess.run(square_until_stop(tf.constant(4), tf.constant(100))))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3N1mz7sNY87N" }, "source": [ @@ -560,11 +505,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "CFk2fszrY8af", - "colab": {} + "id": "CFk2fszrY8af" }, + "outputs": [], "source": [ "@tf.function(\n", " experimental_autograph_options=tf.autograph.experimental.Feature.LISTS)\n", @@ -580,14 +525,11 @@ "with tf.Graph().as_default():\n", " with tf.Session() as sess:\n", " print(sess.run(squares(tf.constant(np.arange(10)))))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FXB0Zbwl13PY" }, "source": [ @@ -596,11 +538,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "1sjaFcL717Ig", - "colab": {} + "id": "1sjaFcL717Ig" }, + "outputs": [], "source": [ "@tf.function\n", "def argwhere_cumsum(x, threshold):\n", @@ -618,14 +560,11 @@ " with tf.Session() as sess:\n", " idx = argwhere_cumsum(tf.ones(N), tf.constant(float(N/2)))\n", " print(sess.run(idx))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LW3nMVABOGaN" }, "source": [ @@ -637,7 +576,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LHpr4VlWPEiS" }, "source": [ @@ -648,11 +586,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "c1O12C8FOE5U", - "colab": {} + "id": "c1O12C8FOE5U" }, + "outputs": [], "source": [ "import numpy as np\n", "\n", @@ -681,14 +619,11 @@ "\n", " result = model.predict(np.array([6171]))\n", " print(result)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "neKBM7AfOEv8" }, "source": [ @@ -703,11 +638,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "E2_tocmEOEWF", - "colab": {} + "id": "E2_tocmEOEWF" }, + "outputs": [], "source": [ "# `K` is used to check if we're in train or test mode.\n", "K = tf.keras.backend\n", @@ -743,14 +678,11 @@ " return x, count\n", "\n", "StochasticNetworkDepth.call = tf.autograph.to_graph(StochasticNetworkDepth.call)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ZbCzHrKlOEOR" }, "source": [ @@ -759,21 +691,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "rxahwlVfOELA", - "colab": {} + "id": "rxahwlVfOELA" }, + "outputs": [], "source": [ "train_batch = np.random.randn(64, 28, 28, 1).astype(np.float32)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "y3_hw-qPOEH9" }, "source": [ @@ -782,11 +711,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "2NiXFOO6OEE1", - "colab": {} + "id": "2NiXFOO6OEE1" }, + "outputs": [], "source": [ "with tf.Graph().as_default() as g:\n", " model = StochasticNetworkDepth(\n", @@ -801,14 +730,11 @@ " model.build(tf.TensorShape((None, None, None, 1)))\n", "\n", " init = tf.global_variables_initializer()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uM3g_v7mvrkg" }, "source": [ @@ -817,11 +743,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7tdmuh5Zvm3D", - "colab": {} + "id": "7tdmuh5Zvm3D" }, + "outputs": [], "source": [ "# Use an explicit session here so we can set the train/test switch, and\n", "# inspect the layer count returned by `call`\n", @@ -840,14 +766,11 @@ " print(\" layers 1st call: \", count1)\n", " print(\" layers 2nd call: \", count2)\n", " print()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4LfnJjm0Bm0B" }, "source": [ @@ -863,7 +786,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Em5dzSUOtLRP" }, "source": [ @@ -872,21 +794,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "xqoxumv0ssQW", - "colab": {} + "id": "xqoxumv0ssQW" }, + "outputs": [], "source": [ "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "znmy4l8ntMvW" }, "source": [ @@ -895,11 +814,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Pe-erWQdBoC5", - "colab": {} + "id": "Pe-erWQdBoC5" }, + "outputs": [], "source": [ "def mlp_model(input_shape):\n", " model = tf.keras.Sequential((\n", @@ -940,14 +859,11 @@ " ds = ds.repeat()\n", " ds = ds.batch(batch_size)\n", " return ds\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "oeYV6mKnJGMr" }, "source": [ @@ -956,11 +872,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3xtg_MMhJETd", - "colab": {} + "id": "3xtg_MMhJETd" }, + "outputs": [], "source": [ "def train(train_ds, test_ds, learning_rate, max_steps):\n", " m = mlp_model((28 * 28,))\n", @@ -1003,14 +919,11 @@ " tf.autograph.experimental.Feature.BUILTIN_FUNCTIONS,\n", " tf.autograph.experimental.Feature.EQUALITY_OPERATORS,\n", " tf.autograph.experimental.Feature.AUTO_CONTROL_DEPS))\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IsHLDZniauLV" }, "source": [ @@ -1019,11 +932,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "HYh6MSZyJOag", - "colab": {} + "id": "HYh6MSZyJOag" }, + "outputs": [], "source": [ "with tf.Graph().as_default() as g:\n", " learning_rate = 0.005\n", @@ -1041,17 +954,15 @@ " (train_losses, test_losses, train_accuracies,\n", " test_accuracies) = sess.run([train_losses, test_losses, train_accuracies,\n", " test_accuracies])\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "JzifaV9PGnH6", - "colab": {} + "id": "JzifaV9PGnH6" }, + "outputs": [], "source": [ "plt.title('MNIST train/test losses')\n", "plt.plot(train_losses, label='train loss')\n", @@ -1067,9 +978,20 @@ "plt.xlabel('Training step')\n", "plt.ylabel('Accuracy')\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "autograph.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/guide/checkpoints.md b/site/en/r1/guide/checkpoints.md index a6316e66264..41544f52b25 100644 --- a/site/en/r1/guide/checkpoints.md +++ b/site/en/r1/guide/checkpoints.md @@ -15,7 +15,7 @@ This document focuses on checkpoints. For details on `SavedModel`, see the ## Sample code This document relies on the same -[Iris classification example](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py) detailed in [Getting Started with TensorFlow](../guide/premade_estimators.md). +[Iris classification example](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/premade_estimator.py) detailed in [Getting Started with TensorFlow](../guide/premade_estimators.md). To download and access the example, invoke the following two commands: ```shell @@ -56,8 +56,8 @@ Suppose you call the Estimator's `train` method. For example: ```python classifier.train( - input_fn=lambda:train_input_fn(train_x, train_y, batch_size=100), - steps=200) + input_fn=lambda: train_input_fn(train_x, train_y, batch_size=100), + steps=200) ``` As suggested by the following diagrams, the first call to `train` diff --git a/site/en/r1/guide/custom_estimators.md b/site/en/r1/guide/custom_estimators.md index c71eb90395a..7bbf3573909 100644 --- a/site/en/r1/guide/custom_estimators.md +++ b/site/en/r1/guide/custom_estimators.md @@ -16,7 +16,7 @@ cd models/samples/core/get_started ``` In this document we will be looking at -[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py). +[`custom_estimator.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/custom_estimator.py). You can run it with the following command: ```bsh @@ -24,9 +24,9 @@ python custom_estimator.py ``` If you are feeling impatient, feel free to compare and contrast -[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py) +[`custom_estimator.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/custom_estimator.py) with -[`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py). +[`premade_estimator.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/premade_estimator.py). (which is in the same directory). @@ -85,7 +85,7 @@ and a logits output layer. Our custom Estimator implementation uses the same input function as our [pre-made Estimator implementation](../guide/premade_estimators.md), from -[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py). +[`iris_data.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/iris_data.py). Namely: ```python @@ -142,7 +142,7 @@ requesting training, predicting, or evaluation. The caller may pass `params` to an Estimator's constructor. Any `params` passed to the constructor are in turn passed on to the `model_fn`. In -[`custom_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/custom_estimator.py) +[`custom_estimator.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/custom_estimator.py) the following lines create the estimator and set the params to configure the model. This configuration step is similar to how we configured the `tf.estimator.DNNClassifier` in [Premade Estimators](../guide/premade_estimators.md). @@ -592,10 +592,10 @@ function for custom Estimators; everything else is the same. For more details, be sure to check out: * The - [official TensorFlow implementation of MNIST](https://github.com/tensorflow/models/tree/master/official/r1/mnist), + [official TensorFlow implementation of MNIST](https://github.com/tensorflow/models/tree/r1.15/official/r1/mnist), which uses a custom estimator. * The TensorFlow - [official models repository](https://github.com/tensorflow/models/tree/master/official), + [official models repository](https://github.com/tensorflow/models/tree/r1.15/official), which contains more curated examples using custom estimators. * This [TensorBoard video](https://youtu.be/eBbEDRsCmv4), which introduces TensorBoard. diff --git a/site/en/r1/guide/datasets.md b/site/en/r1/guide/datasets.md index b1ed1b6e113..d7c38bf2f92 100644 --- a/site/en/r1/guide/datasets.md +++ b/site/en/r1/guide/datasets.md @@ -437,7 +437,7 @@ dataset = dataset.batch(32) iterator = dataset.make_initializable_iterator() # You can feed the initializer with the appropriate filenames for the current -# phase of execution, e.g. training vs. validation. +# phase of execution, e.g., training vs. validation. # Initialize `iterator` with training data. training_filenames = ["/var/data/file1.tfrecord", "/var/data/file2.tfrecord"] @@ -639,7 +639,7 @@ TODO(mrry): Add this section. The simplest form of batching stacks `n` consecutive elements of a dataset into a single element. The `Dataset.batch()` transformation does exactly this, with the same constraints as the `tf.stack()` operator, applied to each component -of the elements: i.e. for each component *i*, all elements must have a tensor +of the elements: i.e., for each component *i*, all elements must have a tensor of the exact same shape. ```python diff --git a/site/en/r1/guide/datasets_for_estimators.md b/site/en/r1/guide/datasets_for_estimators.md index bc12a81cff9..3767763efd6 100644 --- a/site/en/r1/guide/datasets_for_estimators.md +++ b/site/en/r1/guide/datasets_for_estimators.md @@ -16,7 +16,7 @@ Taking slices from an array is the simplest way to get started with `tf.data`. The [Premade Estimators](../guide/premade_estimators.md) chapter describes the following `train_input_fn`, from -[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py), +[`iris_data.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/iris_data.py), to pipe the data into the Estimator: ``` python @@ -50,7 +50,7 @@ which, as we will see, has special meaning for `Datasets`. for each example. * `batch_size` : An integer indicating the desired batch size. -In [`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py) +In [`premade_estimator.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/premade_estimator.py) we retrieved the Iris data using the `iris_data.load_data()` function. You can run it, and unpack the results as follows: @@ -249,7 +249,7 @@ import iris_data train_path, test_path = iris_data.maybe_download() ``` -The [`iris_data.csv_input_fn`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py) +The [`iris_data.csv_input_fn`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/iris_data.py) function contains an alternative implementation that parses the csv files using a `Dataset`. diff --git a/site/en/r1/guide/debugger.md b/site/en/r1/guide/debugger.md index 1b961b23919..963765b97db 100644 --- a/site/en/r1/guide/debugger.md +++ b/site/en/r1/guide/debugger.md @@ -10,11 +10,11 @@ due to TensorFlow's computation-graph paradigm. This guide focuses on the command-line interface (CLI) of `tfdbg`. For guide on how to use the graphical user interface (GUI) of tfdbg, i.e., the **TensorBoard Debugger Plugin**, please visit -[its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md). +[its README](https://github.com/tensorflow/tensorboard/blob/r1.15/tensorboard/plugins/debugger/README.md). Note: The TensorFlow debugger uses a [curses](https://en.wikipedia.org/wiki/Curses_\(programming_library\))-based text -user interface. On Mac OS X, the `ncurses` library is required and can be +user interface. On macOS, the `ncurses` library is required and can be installed with `brew install ncurses`. On Windows, curses isn't as well supported, so a [readline](https://en.wikipedia.org/wiki/GNU_Readline)-based interface can be used with tfdbg by installing `pyreadline` with `pip`. If you @@ -35,7 +35,7 @@ TensorFlow. Later sections of this document describe how to use **tfdbg** with higher-level APIs of TensorFlow, including `tf.estimator`, `tf.keras` / `keras` and `tf.contrib.slim`. To *observe* such an issue, run the following command without the debugger (the source code can be found -[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/v1/debug_mnist.py)): +[here](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/debug/examples/v1/debug_mnist.py)): 
     python -m tensorflow.python.debug.examples.v1.debug_mnist
@@ -64,7 +64,7 @@ numeric problem first surfaced.
 To add support for tfdbg in our example, all that is needed is to add the
 following lines of code and wrap the Session object with a debugger wrapper.
 This code is already added in
-[debug_mnist.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/debug/examples/v1/debug_mnist.py),
+[debug_mnist.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/debug/examples/v1/debug_mnist.py),
 so you can activate tfdbg CLI with the `--debug` flag at the command line.
 
 ```python
@@ -201,7 +201,7 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at
 | | `--op_type_filter ` | Execute the next `Session.run`, watching only nodes with op types matching the given regular-expression pattern. | `run --op_type_filter Variable.*` |
 | | `--tensor_dtype_filter ` | Execute the next `Session.run`, dumping only Tensors with data types (`dtype`s) matching the given regular-expression pattern. | `run --tensor_dtype_filter int.*` |
 | | `-p` | Execute the next `Session.run` call in profiling mode. | `run -p` |
-| **`ri`** | | **Display information about the run the current run, including fetches and feeds.** | `ri` |
+| **`ri`** | | **Display information about the current run, including fetches and feeds.** | `ri` |
 | **`config`** | | **Set or show persistent TFDBG UI configuration.** | |
 | | `set` | Set the value of a config item: {`graph_recursion_depth`, `mouse_mode`}. | `config set graph_recursion_depth 3` |
 | | `show` | Show current persistent UI configuration. | `config show` |
@@ -370,7 +370,7 @@ traceback of the node's construction.
 
 From the traceback, you can see that the op is constructed at the following
 line:
-[`debug_mnist.py`](https://www.tensorflow.org/code/tensorflow/python/debug/examples/v1/debug_mnist.py):
+[`debug_mnist.py`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/debug/examples/v1/debug_mnist.py):
 
 ```python
 diff = y_ * tf.log(y)
@@ -457,7 +457,7 @@ accuracy_score = classifier.evaluate(eval_input_fn,
 predict_results = classifier.predict(predict_input_fn, hooks=hooks)
 ```
 
-[debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/v1/debug_tflearn_iris.py),
+[debug_tflearn_iris.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/debug/examples/v1/debug_tflearn_iris.py),
 contains a full example of how to use the tfdbg with `Estimator`s. To run this
 example, do:
 
@@ -501,7 +501,7 @@ TensorFlow backend. You just need to replace `tf.keras.backend` with
 ## Debugging tf-slim with TFDBG
 
 TFDBG supports debugging of training and evaluation with
-[tf-slim](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/slim).
+[tf-slim](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/contrib/slim).
 As detailed below, training and evaluation require slightly different debugging
 workflows.
 
@@ -605,7 +605,7 @@ The `watch_fn` argument accepts a `Callable` that allows you to configure what
 If your model code is written in C++ or other languages, you can also
 modify the `debug_options` field of `RunOptions` to generate debug dumps that
 can be inspected offline. See
-[the proto definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/debug.proto)
+[the proto definition](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/debug.proto)
 for more details.
 
 ### Debugging Remotely-Running Estimators
@@ -648,7 +648,7 @@ python -m tensorflow.python.debug.cli.offline_analyzer \
        model, check out
 
    1. The profiling mode of tfdbg: `tfdbg> run -p`.
-   2. [tfprof](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/profiler)
+   2. [tfprof](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/core/profiler)
       and other profiling tools for TensorFlow.
 
 **Q**: _How do I link tfdbg against my `Session` in Bazel? Why do I see an
@@ -808,4 +808,4 @@ tensor dumps.
        and conditional breakpoints, and tying tensors to their
        graph-construction source code, all in the browser environment.
        To get started, please visit
-       [its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md).
+       [its README](https://github.com/tensorflow/tensorboard/blob/r1.15/tensorboard/plugins/debugger/README.md).
diff --git a/site/en/r1/guide/distribute_strategy.ipynb b/site/en/r1/guide/distribute_strategy.ipynb
index ee76f84ed66..4dd502d331b 100644
--- a/site/en/r1/guide/distribute_strategy.ipynb
+++ b/site/en/r1/guide/distribute_strategy.ipynb
@@ -1,41 +1,22 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "distribute_strategy.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [
-        "Tce3stUlHN0L"
-      ],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Tce3stUlHN0L"
       },
       "source": [
-        "##### Copyright 2018 The TensorFlow Authors.\n",
-        "\n"
+        "##### Copyright 2018 The TensorFlow Authors.\n"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "tuOe1ymfHZPu",
-        "colab": {}
+        "cellView": "form",
+        "id": "tuOe1ymfHZPu"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -48,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MfBg1C5NB3X0"
       },
       "source": [
@@ -65,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "r6P32iYYV27b"
       },
       "source": [
@@ -82,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "r6P32iYYV27b"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "xHxb-dlhMIzW"
       },
       "source": [
@@ -109,31 +99,26 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "EVOZFbNgXghB",
-        "colab": {}
+        "id": "DKyxxRP2BvIi"
       },
+      "outputs": [],
       "source": [
-        "# Import TensorFlow\n",
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "import tensorflow as tf"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "import tensorflow.compat.v1 as tf\n",
+        "tf.disable_v2_behavior()"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "eQ1QESxxEbCh"
       },
       "source": [
         "## Types of strategies\n",
         "`tf.distribute.Strategy` intends to cover a number of use cases along different axes. Some of these combinations are currently supported and others will be added in the future. Some of these axes are:\n",
         "\n",
-        "* Syncronous vs asynchronous training: These are two common ways of distributing training with data parallelism. In sync training, all workers train over different slices of input data in sync, and aggregating gradients at each step. In async training, all workers are independently training over the input data and updating variables asynchronously. Typically sync training is supported via all-reduce and async through parameter server architecture.\n",
+        "* Synchronous vs asynchronous training: These are two common ways of distributing training with data parallelism. In sync training, all workers train over different slices of input data in sync, and aggregating gradients at each step. In async training, all workers are independently training over the input data and updating variables asynchronously. Typically sync training is supported via all-reduce and async through parameter server architecture.\n",
         "* Hardware platform: Users may want to scale their training onto multiple GPUs on one machine, or multiple machines in a network (with 0 or more GPUs each), or on Cloud TPUs.\n",
         "\n",
         "In order to support these use cases, we have 4 strategies available. In the next section we will talk about which of these are supported in which scenarios in TF."
@@ -142,7 +127,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "DoQKKK8dtfg6"
       },
       "source": [
@@ -158,21 +142,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "9Z4FMAY9ADxK",
-        "colab": {}
+        "id": "9Z4FMAY9ADxK"
       },
+      "outputs": [],
       "source": [
         "mirrored_strategy = tf.distribute.MirroredStrategy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wldY4aFCAH4r"
       },
       "source": [
@@ -183,21 +164,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "nbGleskCACv_",
-        "colab": {}
+        "id": "nbGleskCACv_"
       },
+      "outputs": [],
       "source": [
         "mirrored_strategy = tf.distribute.MirroredStrategy(devices=[\"/gpu:0\", \"/gpu:1\"])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8-KDnrJLAhav"
       },
       "source": [
@@ -206,22 +184,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "6-xIOIpgBItn",
-        "colab": {}
+        "id": "6-xIOIpgBItn"
       },
+      "outputs": [],
       "source": [
         "mirrored_strategy = tf.distribute.MirroredStrategy(\n",
         "    cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "45H0Wa8WKI8z"
       },
       "source": [
@@ -233,31 +208,27 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "rtjZOyaoMWrP",
-        "colab": {}
+        "id": "rtjZOyaoMWrP"
       },
+      "outputs": [],
       "source": [
         "central_storage_strategy = tf.distribute.experimental.CentralStorageStrategy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "KY1nJHNkMl7b"
       },
       "source": [
-        "This will create a `CentralStorageStrategy` instance which will use all visible GPUs and CPU. Update to variables on replicas will be aggragated before being applied to variables."
+        "This will create a `CentralStorageStrategy` instance which will use all visible GPUs and CPU. Update to variables on replicas will be aggregated before being applied to variables."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "aAFycYUiNCUb"
       },
       "source": [
@@ -267,7 +238,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8Xc3gyo0Bejd"
       },
       "source": [
@@ -275,7 +245,7 @@
         "\n",
         "`tf.distribute.experimental.MultiWorkerMirroredStrategy` is very similar to `MirroredStrategy`. It implements synchronous distributed training across multiple workers, each with potentially multiple GPUs. Similar to `MirroredStrategy`, it creates copies of all variables in the model on each device across all workers.\n",
         "\n",
-        "It uses [CollectiveOps](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/collective_ops.py) as the multi-worker all-reduce communication method used to keep variables in sync. A collective op is a single op in the TensorFlow graph which can automatically choose an all-reduce algorithm in the TensorFlow runtime according to hardware, network topology and tensor sizes.\n",
+        "It uses [CollectiveOps](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/ops/collective_ops.py) as the multi-worker all-reduce communication method used to keep variables in sync. A collective op is a single op in the TensorFlow graph which can automatically choose an all-reduce algorithm in the TensorFlow runtime according to hardware, network topology and tensor sizes.\n",
         "\n",
         "It also implements additional performance optimizations. For example, it includes a static optimization that converts multiple all-reductions on small tensors into fewer all-reductions on larger tensors. In addition, we are designing it to have a plugin architecture - so that in the future, users will be able to plugin algorithms that are better tuned for their hardware. Note that collective ops also implement other collective operations such as broadcast and all-gather.\n",
         "\n",
@@ -284,21 +254,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "m3a_6ebbEjre",
-        "colab": {}
+        "id": "m3a_6ebbEjre"
       },
+      "outputs": [],
       "source": [
         "multiworker_strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "bt94JBvhEr4s"
       },
       "source": [
@@ -307,34 +274,28 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "QGX_QAEtFQSv",
-        "colab": {}
+        "id": "QGX_QAEtFQSv"
       },
+      "outputs": [],
       "source": [
         "multiworker_strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy(\n",
         "    tf.distribute.experimental.CollectiveCommunication.NCCL)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "0JiImlw3F77E"
       },
       "source": [
-        "\n",
-        "One of the key differences to get multi worker training going, as compared to multi-GPU training, is the multi-worker setup. \"TF_CONFIG\" environment variable is the standard way in TensorFlow to specify the cluster configuration to each worker that is part of the cluster. See section on [\"TF_CONFIG\" below](#TF_CONFIG) for more details on how this can be done.\n",
-        "\n"
+        "One of the key differences to get multi worker training going, as compared to multi-GPU training, is the multi-worker setup. \"TF_CONFIG\" environment variable is the standard way in TensorFlow to specify the cluster configuration to each worker that is part of the cluster. See section on [\"TF_CONFIG\" below](#TF_CONFIG) for more details on how this can be done.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "E20tG21LFfv1"
       },
       "source": [
@@ -344,7 +305,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "kPEBCMzsGaO5"
       },
       "source": [
@@ -366,7 +326,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "oQ7EqjpmK6DU"
       },
       "source": [
@@ -376,7 +335,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "jARHpraJMbRa"
       },
       "source": [
@@ -386,7 +344,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3ZLBhaP9NUNr"
       },
       "source": [
@@ -402,7 +359,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "zr1wPHYvOH0N"
       },
       "source": [
@@ -412,18 +368,15 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hQv1lm9UPDFy"
       },
       "source": [
-        "\n",
-        "So far we've talked about what are the different stategies available and how you can instantiate them. In the next few sections, we will talk about the different ways in which you can use them to distribute your training. We will show short code snippets in this guide and link off to full tutorials which you can run end to end."
+        "So far we've talked about what are the different strategies available and how you can instantiate them. In the next few sections, we will talk about the different ways in which you can use them to distribute your training. We will show short code snippets in this guide and link off to full tutorials which you can run end to end."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "_mcuy3UhPcen"
       },
       "source": [
@@ -436,24 +389,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "gbbcpzRnPZ6V",
-        "colab": {}
+        "id": "gbbcpzRnPZ6V"
       },
+      "outputs": [],
       "source": [
         "mirrored_strategy = tf.distribute.MirroredStrategy()\n",
         "with mirrored_strategy.scope():\n",
         "  model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
         "  model.compile(loss='mse', optimizer='sgd')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "773EOxCRVlTg"
       },
       "source": [
@@ -462,23 +412,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ZMmxEFRTEjH5",
-        "colab": {}
+        "id": "ZMmxEFRTEjH5"
       },
+      "outputs": [],
       "source": [
         "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100).batch(10)\n",
         "model.fit(dataset, epochs=2)\n",
         "model.evaluate(dataset)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "nofTLwyXWHK8"
       },
       "source": [
@@ -487,23 +434,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Lqgd9SdxW5OW",
-        "colab": {}
+        "id": "Lqgd9SdxW5OW"
       },
+      "outputs": [],
       "source": [
         "import numpy as np\n",
         "inputs, targets = np.ones((100, 1)), np.ones((100, 1))\n",
         "model.fit(inputs, targets, epochs=2, batch_size=10)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IKqaj7QwX0Zb"
       },
       "source": [
@@ -512,11 +456,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "quNNTytWdGBf",
-        "colab": {}
+        "id": "quNNTytWdGBf"
       },
+      "outputs": [],
       "source": [
         "# Compute global batch size using number of replicas.\n",
         "BATCH_SIZE_PER_REPLICA = 5\n",
@@ -527,14 +471,11 @@
         "\n",
         "LEARNING_RATES_BY_BATCH_SIZE = {5: 0.1, 10: 0.15}\n",
         "learning_rate = LEARNING_RATES_BY_BATCH_SIZE[global_batch_size]"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "z1Muy0gDZwO5"
       },
       "source": [
@@ -549,19 +490,18 @@
         "Here is a list of tutorials and examples that illustrate the above integration end to end with Keras:\n",
         "\n",
         "1. [Tutorial](../tutorials/distribute/keras.ipynb) to train MNIST with `MirroredStrategy`.\n",
-        "2. Official [ResNet50](https://github.com/tensorflow/models/blob/master/official/vision/image_classification/resnet_imagenet_main.py) training with ImageNet data using `MirroredStrategy`.\n",
-        "3. [ResNet50](https://github.com/tensorflow/tpu/blob/master/models/experimental/resnet50_keras/resnet50.py) trained with Imagenet data on Cloud TPus with `TPUStrategy`."
+        "2. Official [ResNet50](https://github.com/tensorflow/models/blob/r1.15/official/vision/image_classification/resnet_imagenet_main.py) training with ImageNet data using `MirroredStrategy`.\n",
+        "3. [ResNet50](https://github.com/tensorflow/tpu/blob/1.15/models/experimental/resnet50_keras/resnet50.py) trained with Imagenet data on Cloud TPus with `TPUStrategy`."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "nO0hmFCRoIll"
       },
       "source": [
         "## Using `tf.distribute.Strategy` with Estimator\n",
-        "`tf.estimator` is a distributed training TensorFlow API that originally supported the async parameter server approach. Like with Keras, we've integrated `tf.distribute.Strategy` into `tf.Estimator` so that a user who is using Estimator for their training can easily change their training is distributed with very few changes to your their code. With this, estimator users can now do synchronous distributed training on multiple GPUs and multiple workers, as well as use TPUs.\n",
+        "`tf.estimator` is a distributed training TensorFlow API that originally supported the async parameter server approach. Like with Keras, we've integrated `tf.distribute.Strategy` into `tf.Estimator` so that a user who is using Estimator for their training can easily change how their training is distributed with very few changes to their code. With this, estimator users can now do synchronous distributed training on multiple GPUs and multiple workers, as well as use TPUs.\n",
         "\n",
         "The usage of `tf.distribute.Strategy` with Estimator is slightly different than the Keras case. Instead of using `strategy.scope`, now we pass the strategy object into the [`RunConfig`](https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig) for the Estimator.\n",
         "\n",
@@ -570,11 +510,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "oGFY5nW_B3YU",
-        "colab": {}
+        "id": "oGFY5nW_B3YU"
       },
+      "outputs": [],
       "source": [
         "mirrored_strategy = tf.distribute.MirroredStrategy()\n",
         "config = tf.estimator.RunConfig(\n",
@@ -583,14 +523,11 @@
         "    feature_columns=[tf.feature_column.numeric_column('feats')],\n",
         "    optimizer='SGD',\n",
         "    config=config)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "n6eSfLN5RGY8"
       },
       "source": [
@@ -601,25 +538,22 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "2ky2ve2PB3YP",
-        "colab": {}
+        "id": "2ky2ve2PB3YP"
       },
+      "outputs": [],
       "source": [
         "def input_fn():\n",
         "  dataset = tf.data.Dataset.from_tensors(({\"feats\":[1.]}, [1.]))\n",
         "  return dataset.repeat(1000).batch(10)\n",
         "regressor.train(input_fn=input_fn, steps=10)\n",
         "regressor.evaluate(input_fn=input_fn, steps=10)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hgaU9xQSSk2x"
       },
       "source": [
@@ -629,7 +563,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "_098zB3vVhuV"
       },
       "source": [
@@ -643,7 +576,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "G3ieQKfWZhhL"
       },
       "source": [
@@ -653,7 +585,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "A_lvUsSLZzVg"
       },
       "source": [
@@ -664,20 +595,19 @@
         "### Examples and Tutorials\n",
         "Here are some examples that show end to end usage of various strategies with Estimator:\n",
         "\n",
-        "1. [End to end example](https://github.com/tensorflow/ecosystem/tree/master/distribution_strategy) for multi worker training in tensorflow/ecosystem using Kuberentes templates. This example starts with a Keras model and converts it to an Estimator using the `tf.keras.estimator.model_to_estimator` API.\n",
-        "2. Official [ResNet50](https://github.com/tensorflow/models/blob/master/official/r1/resnet/imagenet_main.py) model, which can be trained using either `MirroredStrategy` or `MultiWorkerMirroredStrategy`.\n",
-        "3. [ResNet50](https://github.com/tensorflow/tpu/blob/master/models/experimental/distribution_strategy/resnet_estimator.py) example with TPUStrategy."
+        "1. [End to end example](https://github.com/tensorflow/ecosystem/tree/r1.15/distribution_strategy) for multi worker training in tensorflow/ecosystem using Kuberentes templates. This example starts with a Keras model and converts it to an Estimator using the `tf.keras.estimator.model_to_estimator` API.\n",
+        "2. Official [ResNet50](https://github.com/tensorflow/models/blob/r1.15/official/r1/resnet/imagenet_main.py) model, which can be trained using either `MirroredStrategy` or `MultiWorkerMirroredStrategy`.\n",
+        "3. [ResNet50](https://github.com/tensorflow/tpu/blob/1.15/models/experimental/distribution_strategy/resnet_estimator.py) example with TPUStrategy."
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IlYVC0goepdk"
       },
       "source": [
         "## Using `tf.distribute.Strategy` with custom training loops\n",
-        "As you've seen, using `tf.distrbute.Strategy` with high level APIs is only a couple lines of code change. With a little more effort, `tf.distrbute.Strategy` can also be used by other users who are not using these frameworks.\n",
+        "As you've seen, using `tf.distribute.Strategy` with high level APIs is only a couple lines of code change. With a little more effort, `tf.distribute.Strategy` can also be used by other users who are not using these frameworks.\n",
         "\n",
         "TensorFlow is used for a wide variety of use cases and some users (such as researchers) require more flexibility and control over their training loops. This makes it hard for them to use the high level frameworks such as Estimator or Keras. For instance, someone using a GAN may want to take a different number of generator or discriminator steps each round. Similarly, the high level frameworks are not very suitable for Reinforcement Learning training. So these users will usually write their own training loops.\n",
         "\n",
@@ -690,7 +620,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "XNHvSY32nVBi"
       },
       "source": [
@@ -699,23 +628,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "W-3Bn-CaiPKD",
-        "colab": {}
+        "id": "W-3Bn-CaiPKD"
       },
+      "outputs": [],
       "source": [
         "with mirrored_strategy.scope():\n",
         "  model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n",
         "  optimizer = tf.train.GradientDescentOptimizer(0.1)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mYkAyPeYnlXk"
       },
       "source": [
@@ -724,36 +650,33 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "94BkvkLInkKd",
-        "colab": {}
+        "id": "94BkvkLInkKd"
       },
+      "outputs": [],
       "source": [
         "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(1000).batch(\n",
         "    global_batch_size)\n",
         "dist_dataset = mirrored_strategy.experimental_distribute_dataset(dataset)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "grzmTlSvn2j8"
       },
       "source": [
-        "Then, we define one step of the training. We will use `tf.GradientTape` to compute gradients and optimizer to apply those gradients to update our model's variables. To distribute this training step, we put it in a function `step_fn` and pass it to `tf.distribute.Strategy.experimental_run_v2` along with the inputs from the iterator:"
+        "Then, we define one step of the training. We will use `tf.GradientTape` to compute gradients and optimizer to apply those gradients to update our model's variables. To distribute this training step, we put it in a function `step_fn` and pass it to `tf.distribute.Strategy.run` along with the inputs from the iterator:"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "NJxL5YrVniDe",
-        "colab": {}
+        "id": "NJxL5YrVniDe"
       },
+      "outputs": [],
       "source": [
         "def train_step(dist_inputs):\n",
         "  def step_fn(inputs):\n",
@@ -766,32 +689,28 @@
         "    with tf.control_dependencies([train_op]):\n",
         "      return tf.identity(loss)\n",
         "\n",
-        "  per_replica_losses = mirrored_strategy.experimental_run_v2(\n",
+        "  per_replica_losses = mirrored_strategy.run(\n",
         "      step_fn, args=(dist_inputs,))\n",
         "  mean_loss = mirrored_strategy.reduce(\n",
         "      tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n",
         "  return mean_loss"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yRL5u_NLoTvq"
       },
       "source": [
         "A few other things to note in the code above:\n",
         "\n",
         "1. We used `tf.nn.softmax_cross_entropy_with_logits` to compute the loss. And then we scaled the total loss by the global batch size. This is important because all the replicas are training in sync and number of examples in each step of training is the global batch. So the loss needs to be divided by the global batch size and not by the replica (local) batch size.\n",
-        "2. We used the `strategy.reduce` API to aggregate the results returned by `tf.distribute.Strategy.experimental_run_v2`. `tf.distribute.Strategy.experimental_run_v2` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can `reduce` them to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results(results)`to get the list of values contained in the result, one per local replica.\n"
+        "2. We used the `strategy.reduce` API to aggregate the results returned by `tf.distribute.Strategy.run`. `tf.distribute.Strategy.run` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can `reduce` them to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results(results)`to get the list of values contained in the result, one per local replica.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "o9k_6-6vpQ-P"
       },
       "source": [
@@ -800,29 +719,26 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Egq9eufToRf6",
-        "colab": {}
+        "id": "Egq9eufToRf6"
       },
+      "outputs": [],
       "source": [
         "with mirrored_strategy.scope():\n",
         "  input_iterator = dist_dataset.make_initializable_iterator()\n",
-        "  iterator_init = input_iterator.initialize()\n",
+        "  iterator_init = input_iterator.initializer\n",
         "  var_init = tf.global_variables_initializer()\n",
         "  loss = train_step(input_iterator.get_next())\n",
         "  with tf.Session() as sess:\n",
         "    sess.run([var_init, iterator_init])\n",
         "    for _ in range(10):\n",
         "      print(sess.run(loss))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "jK8eQXF_q1Zs"
       },
       "source": [
@@ -832,7 +748,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vDJO8mnypqBA"
       },
       "source": [
@@ -842,7 +757,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "BZjNwCt1qBdw"
       },
       "source": [
@@ -858,7 +772,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Xk0JdsTHyUnE"
       },
       "source": [
@@ -869,7 +782,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "cP6BUIBtudRk"
       },
       "source": [
@@ -897,7 +809,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "fezd3aF8wj9r"
       },
       "source": [
@@ -910,7 +821,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GXIbqSW-sFVg"
       },
       "source": [
@@ -919,5 +829,20 @@
         "`tf.distribute.Strategy` is actively under development. We welcome you to try it out and provide your feedback via [issues on GitHub](https://github.com/tensorflow/tensorflow/issues/new)."
       ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "Tce3stUlHN0L"
+      ],
+      "name": "distribute_strategy.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/guide/eager.ipynb b/site/en/r1/guide/eager.ipynb
index 69d1e703d0c..f76acb4b702 100644
--- a/site/en/r1/guide/eager.ipynb
+++ b/site/en/r1/guide/eager.ipynb
@@ -1,25 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "eager.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "CCQY7jpBfMur"
       },
       "source": [
@@ -28,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "z6X9omPnfO_h",
-        "colab": {}
+        "id": "z6X9omPnfO_h"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -46,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2QQJJyDzqGRb"
       },
       "source": [
@@ -63,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "B1xdylywqUSX"
       },
       "source": [
@@ -80,12 +59,23 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "B1xdylywqUSX"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "EGjDcGxIqEfX"
       },
       "source": [
-        "\n",
-        "\n",
         "TensorFlow's eager execution is an imperative programming environment that\n",
         "evaluates operations immediately, without building graphs: operations return\n",
         "concrete values instead of constructing a computational graph to run later. This\n",
@@ -105,7 +95,7 @@
         "\n",
         "Eager execution supports most TensorFlow operations and GPU acceleration. For a\n",
         "collection of examples running in eager execution, see:\n",
-        "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).\n",
+        "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/contrib/eager/python/examples).\n",
         "\n",
         "Note: Some models may experience increased overhead with eager execution\n",
         "enabled. Performance improvements are ongoing, but please\n",
@@ -116,7 +106,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "RBAeIwOMrYk8"
       },
       "source": [
@@ -126,42 +115,28 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "48P3-8q4qEfe"
       },
       "source": [
-        "To start eager execution, add `tf.enable_eager_execution()` to the beginning of\n",
+        "To start eager execution, add `` to the beginning of\n",
         "the program or console session. Do not add this operation to other modules that\n",
         "the program calls."
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "7aFsD8csqEff",
-        "colab": {}
+        "id": "7aFsD8csqEff"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.enable_eager_execution()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "import tensorflow.compat.v1 as tf"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "x_G1zZT5qEfh"
       },
       "source": [
@@ -170,42 +145,31 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "5hien2IEqwLQ",
-        "colab": {}
+        "id": "5hien2IEqwLQ"
       },
+      "outputs": [],
       "source": [
         "tf.executing_eagerly()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "9gsI54pbqEfj",
-        "colab": {}
+        "id": "9gsI54pbqEfj"
       },
+      "outputs": [],
       "source": [
         "x = [[2.]]\n",
         "m = tf.matmul(x, x)\n",
         "print(\"hello, {}\".format(m))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ajFn6qsdqEfl"
       },
       "source": [
@@ -226,121 +190,74 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "sTO0_5TYqz1n",
-        "colab": {}
+        "id": "sTO0_5TYqz1n"
       },
+      "outputs": [],
       "source": [
         "a = tf.constant([[1, 2],\n",
         "                 [3, 4]])\n",
         "print(a)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Dp14YT8Gq4r1",
-        "colab": {}
+        "id": "Dp14YT8Gq4r1"
       },
+      "outputs": [],
       "source": [
         "# Broadcasting support\n",
         "b = tf.add(a, 1)\n",
         "print(b)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "69p3waMfq8cQ",
-        "colab": {}
+        "id": "69p3waMfq8cQ"
       },
+      "outputs": [],
       "source": [
         "# Operator overloading is supported\n",
         "print(a * b)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "Ui025t1qqEfm",
-        "colab": {}
+        "id": "Ui025t1qqEfm"
       },
+      "outputs": [],
       "source": [
         "# Use NumPy values\n",
         "import numpy as np\n",
         "\n",
         "c = np.multiply(a, b)\n",
         "print(c)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Tq_aFRzWrCua",
-        "colab": {}
+        "id": "Tq_aFRzWrCua"
       },
+      "outputs": [],
       "source": [
         "# Obtain numpy value from a tensor:\n",
         "print(a.numpy())\n",
         "# => [[1 2]\n",
         "#     [3 4]]"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "Jdg3nZFwqEfp"
-      },
-      "source": [
-        "The `tf.contrib.eager` module contains symbols available to both eager and graph execution\n",
-        "environments and is useful for writing code to [work with graphs](#work_with_graphs):"
       ]
     },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "N-2lSGiHqEfq",
-        "colab": {}
-      },
-      "source": [
-        "tfe = tf.contrib.eager"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "H08f9ss9qEft"
       },
       "source": [
@@ -353,17 +270,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "0fudRMeUqEfu",
-        "colab": {}
+        "id": "0fudRMeUqEfu"
       },
+      "outputs": [],
       "source": [
         "def fizzbuzz(max_num):\n",
         "  counter = tf.constant(0)\n",
@@ -379,27 +290,22 @@
         "    else:\n",
         "      print(num.numpy())\n",
         "    counter += 1"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "P2cKknQWrJLB",
-        "colab": {}
+        "id": "P2cKknQWrJLB"
       },
+      "outputs": [],
       "source": [
         "fizzbuzz(15)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "7kA-aC3BqEfy"
       },
       "source": [
@@ -410,7 +316,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "zn3mp-j6rjnk"
       },
       "source": [
@@ -427,17 +332,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "kvXiJsmyqEfz",
-        "colab": {}
+        "id": "kvXiJsmyqEfz"
       },
+      "outputs": [],
       "source": [
         "class MySimpleLayer(tf.keras.layers.Layer):\n",
         "  def __init__(self, output_units):\n",
@@ -456,14 +355,11 @@
         "  def call(self, input):\n",
         "    # Override call() instead of __call__ so we can perform some bookkeeping.\n",
         "    return tf.matmul(input, self.kernel)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "eo3qgrVCqEf2"
       },
       "source": [
@@ -476,30 +372,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "VrfLnhNPqEf3",
-        "colab": {}
+        "id": "VrfLnhNPqEf3"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Sequential([\n",
         "  tf.keras.layers.Dense(10, input_shape=(784,)),  # must declare input shape\n",
         "  tf.keras.layers.Dense(10)\n",
         "])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Dms3mduTqEf6"
       },
       "source": [
@@ -510,17 +397,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "MwWxQmNOqEf7",
-        "colab": {}
+        "id": "MwWxQmNOqEf7"
       },
+      "outputs": [],
       "source": [
         "class MNISTModel(tf.keras.Model):\n",
         "  def __init__(self):\n",
@@ -536,14 +417,11 @@
         "    return result\n",
         "\n",
         "model = MNISTModel()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "a639YaF4qEf-"
       },
       "source": [
@@ -558,7 +436,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8huKpuuAwICq"
       },
       "source": [
@@ -568,7 +445,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mp2lCCZYrxHd"
       },
       "source": [
@@ -589,17 +465,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "7g1yWiSXqEf-",
-        "colab": {}
+        "id": "7g1yWiSXqEf-"
       },
+      "outputs": [],
       "source": [
         "w = tf.Variable([[1.0]])\n",
         "with tf.GradientTape() as tape:\n",
@@ -607,14 +477,11 @@
         "\n",
         "grad = tape.gradient(loss, w)\n",
         "print(grad)  # => tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vkHs32GqweYS"
       },
       "source": [
@@ -627,11 +494,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "38kymXZowhhz",
-        "colab": {}
+        "id": "38kymXZowhhz"
       },
+      "outputs": [],
       "source": [
         "# Fetch and format the mnist data\n",
         "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n",
@@ -640,17 +507,15 @@
         "  (tf.cast(mnist_images[...,tf.newaxis]/255, tf.float32),\n",
         "   tf.cast(mnist_labels,tf.int64)))\n",
         "dataset = dataset.shuffle(1000).batch(32)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "rl1K8rOowmwT",
-        "colab": {}
+        "id": "rl1K8rOowmwT"
       },
+      "outputs": [],
       "source": [
         "# Build the model\n",
         "mnist_model = tf.keras.Sequential([\n",
@@ -659,39 +524,32 @@
         "  tf.keras.layers.GlobalAveragePooling2D(),\n",
         "  tf.keras.layers.Dense(10)\n",
         "])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "fvyk-HgGwxwl"
       },
       "source": [
-        "\n",
         "Even without training, call the model and inspect the output in eager execution:"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "BsxystjBwxLS",
-        "colab": {}
+        "id": "BsxystjBwxLS"
       },
+      "outputs": [],
       "source": [
         "for images,labels in dataset.take(1):\n",
         "  print(\"Logits: \", mnist_model(images[0:1]).numpy())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Y3PGa8G7qEgB"
       },
       "source": [
@@ -700,32 +558,24 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "bzRhM7JDnaEG",
-        "colab": {}
+        "id": "bzRhM7JDnaEG"
       },
+      "outputs": [],
       "source": [
         "optimizer = tf.train.AdamOptimizer()\n",
         "\n",
         "loss_history = []"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "0m1xAXrmqEgJ",
-        "colab": {}
+        "id": "0m1xAXrmqEgJ"
       },
+      "outputs": [],
       "source": [
         "for (batch, (images, labels)) in enumerate(dataset.take(400)):\n",
         "  if batch % 10 == 0:\n",
@@ -738,31 +588,26 @@
         "  grads = tape.gradient(loss_value, mnist_model.trainable_variables)\n",
         "  optimizer.apply_gradients(zip(grads, mnist_model.trainable_variables),\n",
         "                            global_step=tf.train.get_or_create_global_step())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "5vG5ql_2vYB5",
-        "colab": {}
+        "id": "5vG5ql_2vYB5"
       },
+      "outputs": [],
       "source": [
         "import matplotlib.pyplot as plt\n",
         "\n",
         "plt.plot(loss_history)\n",
         "plt.xlabel('Batch #')\n",
         "plt.ylabel('Loss [entropy]')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "kKpOlHPLqEgl"
       },
       "source": [
@@ -779,17 +624,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "nnQLBYmEqEgm",
-        "colab": {}
+        "id": "nnQLBYmEqEgm"
       },
+      "outputs": [],
       "source": [
         "class Model(tf.keras.Model):\n",
         "  def __init__(self):\n",
@@ -834,14 +673,11 @@
         "\n",
         "print(\"Final loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n",
         "print(\"W = {}, B = {}\".format(model.W.numpy(), model.B.numpy()))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "rPjb8nRWqEgr"
       },
       "source": [
@@ -860,30 +696,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "A2boS674qEgs",
-        "colab": {}
+        "id": "A2boS674qEgs"
       },
+      "outputs": [],
       "source": [
-        "if tf.test.is_gpu_available():\n",
+        "if tf.config.list_physical_devices('GPU'):\n",
         "  with tf.device(\"gpu:0\"):\n",
         "    v = tf.Variable(tf.random_normal([1000, 1000]))\n",
         "    v = None  # v no longer takes up GPU memory"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "scMjg6L6qEgv"
       },
       "source": [
@@ -895,46 +722,36 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "7z5xRfdHzZOQ",
-        "colab": {}
+        "id": "7z5xRfdHzZOQ"
       },
+      "outputs": [],
       "source": [
         "x = tf.Variable(10.)\n",
         "checkpoint = tf.train.Checkpoint(x=x)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "IffrUVG7zyVb",
-        "colab": {}
+        "id": "IffrUVG7zyVb"
       },
+      "outputs": [],
       "source": [
         "x.assign(2.)   # Assign a new value to the variables and save.\n",
         "checkpoint_path = './ckpt/'\n",
         "checkpoint.save('./ckpt/')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "eMT9koCoqEgw",
-        "colab": {}
+        "id": "eMT9koCoqEgw"
       },
+      "outputs": [],
       "source": [
         "x.assign(11.)  # Change the variable after saving.\n",
         "\n",
@@ -942,14 +759,11 @@
         "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_path))\n",
         "\n",
         "print(x)  # => 2.0"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vbFnP-yLqEgx"
       },
       "source": [
@@ -960,17 +774,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "hWZHyAXMqEg0",
-        "colab": {}
+        "id": "hWZHyAXMqEg0"
       },
+      "outputs": [],
       "source": [
         "import os\n",
         "import tempfile\n",
@@ -989,52 +797,40 @@
         "\n",
         "root.save(checkpoint_prefix)\n",
         "root.restore(tf.train.latest_checkpoint(checkpoint_dir))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3yoD0VJ7qEg3"
       },
       "source": [
         "### Object-oriented metrics\n",
         "\n",
-        "`tfe.metrics` are stored as objects. Update a metric by passing the new data to\n",
-        "the callable, and retrieve the result using the `tfe.metrics.result` method,\n",
+        "`tf.metrics` are stored as objects. Update a metric by passing the new data to\n",
+        "the callable, and retrieve the result using the `tf.metrics.result` method,\n",
         "for example:"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "9ccu0iAaqEg5",
-        "colab": {}
+        "id": "9ccu0iAaqEg5"
       },
+      "outputs": [],
       "source": [
-        "m = tfe.metrics.Mean(\"loss\")\n",
+        "m = tf.keras.metrics.Mean(\"loss\")\n",
         "m(0)\n",
         "m(5)\n",
         "m.result()  # => 2.5\n",
         "m([8, 9])\n",
         "m.result()  # => 5.5"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "BE8cXErYqEg8"
       },
       "source": [
@@ -1044,59 +840,45 @@
         "understanding, debugging and optimizing the model training process. It uses\n",
         "summary events that are written while executing the program.\n",
         "\n",
-        "`tf.contrib.summary` is compatible with both eager and graph execution\n",
-        "environments. Summary operations, such as `tf.contrib.summary.scalar`, are\n",
-        "inserted during model construction. For example, to record summaries once every\n",
-        "100 global steps:"
+        "TensorFlow 1 summaries only work in eager mode, but can be run with the `compat.v2` module:"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "3PXAJB1GqEg9",
-        "colab": {}
+        "id": "3PXAJB1GqEg9"
       },
+      "outputs": [],
       "source": [
+        "from tensorflow.compat.v2 import summary\n",
+        "\n",
         "global_step = tf.train.get_or_create_global_step()\n",
         "\n",
         "logdir = \"./tb/\"\n",
-        "writer = tf.contrib.summary.create_file_writer(logdir)\n",
+        "writer = summary.create_file_writer(logdir)\n",
         "writer.set_as_default()\n",
         "\n",
         "for _ in range(10):\n",
         "  global_step.assign_add(1)\n",
-        "  # Must include a record_summaries method\n",
-        "  with tf.contrib.summary.record_summaries_every_n_global_steps(100):\n",
-        "    # your model code goes here\n",
-        "    tf.contrib.summary.scalar('global_step', global_step)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "  # your model code goes here\n",
+        "  summary.scalar('global_step', global_step, step=global_step)"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "6TJSs_wG8Spg",
-        "colab": {}
+        "id": "6TJSs_wG8Spg"
       },
+      "outputs": [],
       "source": [
         "!ls tb/"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "xEL4yJe5qEhD"
       },
       "source": [
@@ -1112,17 +894,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "L518n5dkqEhE",
-        "colab": {}
+        "id": "L518n5dkqEhE"
       },
+      "outputs": [],
       "source": [
         "def line_search_step(fn, init_x, rate=1.0):\n",
         "  with tf.GradientTape() as tape:\n",
@@ -1137,167 +913,11 @@
         "    value = fn(x)\n",
         "    rate /= 2.0\n",
         "  return x, value"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "03owpCFrqEhH"
-      },
-      "source": [
-        "### Additional functions to compute gradients\n",
-        "\n",
-        "`tf.GradientTape` is a powerful interface for computing gradients, but there\n",
-        "is another [Autograd](https://github.com/HIPS/autograd)-style API available for\n",
-        "automatic differentiation. These functions are useful if writing math code with\n",
-        "only tensors and gradient functions, and without `tf.variables`:\n",
-        "\n",
-        "* `tfe.gradients_function` —Returns a function that computes the derivatives\n",
-        "  of its input function parameter with respect to its arguments. The input\n",
-        "  function parameter must return a scalar value. When the returned function is\n",
-        "  invoked, it returns a list of `tf.Tensor` objects: one element for each\n",
-        "  argument of the input function. Since anything of interest must be passed as a\n",
-        "  function parameter, this becomes unwieldy if there's a dependency on many\n",
-        "  trainable parameters.\n",
-        "* `tfe.value_and_gradients_function` —Similar to\n",
-        "  `tfe.gradients_function`, but when the returned function is invoked, it\n",
-        "  returns the value from the input function in addition to the list of\n",
-        "  derivatives of the input function with respect to its arguments.\n",
-        "\n",
-        "In the following example, `tfe.gradients_function` takes the `square`\n",
-        "function as an argument and returns a function that computes the partial\n",
-        "derivatives of `square` with respect to its inputs. To calculate the derivative\n",
-        "of `square` at `3`, `grad(3.0)` returns `6`."
       ]
     },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "QDPFUG-68i-B",
-        "colab": {}
-      },
-      "source": [
-        "def square(x):\n",
-        "  return tf.multiply(x, x)\n",
-        "\n",
-        "grad = tfe.gradients_function(square)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "_Ow9LHre8k_3",
-        "colab": {}
-      },
-      "source": [
-        "square(3.).numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "Rg5ea0kC8nEQ",
-        "colab": {}
-      },
-      "source": [
-        "grad(3.)[0].numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "41D1LzcG87p8",
-        "colab": {}
-      },
-      "source": [
-        "# The second-order derivative of square:\n",
-        "gradgrad = tfe.gradients_function(lambda x: grad(x)[0])\n",
-        "gradgrad(3.)[0].numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "DotEGcx5qEhH",
-        "colab": {}
-      },
-      "source": [
-        "# The third-order derivative is None:\n",
-        "gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0])\n",
-        "gradgradgrad(3.)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "9DRAews99F4-",
-        "colab": {}
-      },
-      "source": [
-        "# With flow control:\n",
-        "def abs(x):\n",
-        "  return x if x > 0. else -x\n",
-        "\n",
-        "grad = tfe.gradients_function(abs)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "hTA1jmu_9gyB",
-        "colab": {}
-      },
-      "source": [
-        "grad(3.)[0].numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "7zs8JpKw9kir",
-        "colab": {}
-      },
-      "source": [
-        "grad(-3.)[0].numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "gieGOf_DqEhK"
       },
       "source": [
@@ -1311,17 +931,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "-OwwsWUAqEhK",
-        "colab": {}
+        "id": "-OwwsWUAqEhK"
       },
+      "outputs": [],
       "source": [
         "@tf.custom_gradient\n",
         "def clip_gradient_by_norm(x, norm):\n",
@@ -1329,14 +943,11 @@
         "  def grad_fn(dresult):\n",
         "    return [tf.clip_by_norm(dresult, norm), None]\n",
         "  return y, grad_fn"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "JPLDHkF_qEhN"
       },
       "source": [
@@ -1346,57 +957,66 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "24WiLROnqEhO",
-        "colab": {}
+        "id": "24WiLROnqEhO"
       },
+      "outputs": [],
       "source": [
         "def log1pexp(x):\n",
         "  return tf.log(1 + tf.exp(x))\n",
-        "grad_log1pexp = tfe.gradients_function(log1pexp)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "\n",
+        "class Grad(object):\n",
+        "  def __init__(self, f):\n",
+        "    self.f = f\n",
+        "\n",
+        "  def __call__(self, x):\n",
+        "    x = tf.convert_to_tensor(x)\n",
+        "    with tf.GradientTape() as tape:\n",
+        "      tape.watch(x)\n",
+        "      r = self.f(x)\n",
+        "    g = tape.gradient(r, x)\n",
+        "    return g"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "I98MFBraTCYx"
+      },
+      "outputs": [],
+      "source": [
+        "grad_log1pexp = Grad(log1pexp)"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "n8fq69r9-B-c",
-        "colab": {}
+        "id": "n8fq69r9-B-c"
       },
+      "outputs": [],
       "source": [
         "# The gradient computation works fine at x = 0.\n",
-        "grad_log1pexp(0.)[0].numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "grad_log1pexp(0.).numpy()"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "_VFSU0mG-FSp",
-        "colab": {}
+        "id": "_VFSU0mG-FSp"
       },
+      "outputs": [],
       "source": [
         "# However, x = 100 fails because of numerical instability.\n",
-        "grad_log1pexp(100.)[0].numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "grad_log1pexp(100.).numpy()"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-VcTR34rqEhQ"
       },
       "source": [
@@ -1408,17 +1028,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "Q7nvfx_-qEhS",
-        "colab": {}
+        "id": "Q7nvfx_-qEhS"
       },
+      "outputs": [],
       "source": [
         "@tf.custom_gradient\n",
         "def log1pexp(x):\n",
@@ -1427,43 +1041,36 @@
         "    return dy * (1 - 1 / (1 + e))\n",
         "  return tf.log(1 + e), grad\n",
         "\n",
-        "grad_log1pexp = tfe.gradients_function(log1pexp)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "grad_log1pexp = Grad(log1pexp)"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "5gHPKMfl-Kge",
-        "colab": {}
+        "id": "5gHPKMfl-Kge"
       },
+      "outputs": [],
       "source": [
         "# As before, the gradient computation works fine at x = 0.\n",
-        "grad_log1pexp(0.)[0].numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "grad_log1pexp(0.).numpy()"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "u38MOfz3-MDE",
-        "colab": {}
+        "id": "u38MOfz3-MDE"
       },
+      "outputs": [],
       "source": [
         "# And the gradient computation also works at x = 100.\n",
-        "grad_log1pexp(100.)[0].numpy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "grad_log1pexp(100.).numpy()"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "rnZXjfQzqEhV"
       },
       "source": [
@@ -1476,17 +1083,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "Ac9Y64H-qEhX",
-        "colab": {}
+        "id": "Ac9Y64H-qEhX"
       },
+      "outputs": [],
       "source": [
         "import time\n",
         "\n",
@@ -1515,19 +1116,16 @@
         "  print(\"CPU: {} secs\".format(measure(tf.random_normal(shape), steps)))\n",
         "\n",
         "# Run on GPU, if available:\n",
-        "if tfe.num_gpus() > 0:\n",
+        "if tf.config.list_physical_devices('GPU'):\n",
         "  with tf.device(\"/gpu:0\"):\n",
         "    print(\"GPU: {} secs\".format(measure(tf.random_normal(shape), steps)))\n",
         "else:\n",
         "  print(\"GPU: not found\")"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "RLw3IS7UqEhe"
       },
       "source": [
@@ -1537,45 +1135,32 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "ny6LX2BVqEhf",
-        "colab": {}
+        "id": "ny6LX2BVqEhf"
       },
+      "outputs": [],
       "source": [
-        "if tf.test.is_gpu_available():\n",
+        "if tf.config.list_physical_devices('GPU'):\n",
         "  x = tf.random_normal([10, 10])\n",
         "\n",
         "  x_gpu0 = x.gpu()\n",
         "  x_cpu = x.cpu()\n",
         "\n",
         "  _ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU\n",
-        "  _ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0\n",
-        "\n",
-        "  if tfe.num_gpus() > 1:\n",
-        "    x_gpu1 = x.gpu(1)\n",
-        "    _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "  _ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "oA_qaII3-p6c"
       },
       "source": [
         "### Benchmarks\n",
         "\n",
         "For compute-heavy models, such as\n",
-        "[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)\n",
+        "[ResNet50](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/contrib/eager/python/examples/resnet50)\n",
         "training on a GPU, eager execution performance is comparable to graph execution.\n",
         "But this gap grows larger for models with less computation and there is work to\n",
         "be done for optimizing hot code paths for models with lots of small operations."
@@ -1584,7 +1169,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "TjMTyFIp-sSE"
       },
       "source": [
@@ -1612,7 +1196,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hll3ZbE5qEhh"
       },
       "source": [
@@ -1642,37 +1225,29 @@
         "production deployment. Use `tf.train.Checkpoint` to save and restore model\n",
         "variables, this allows movement between eager and graph execution environments.\n",
         "See the examples in:\n",
-        "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).\n",
-        "\n"
+        "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/contrib/eager/python/examples).\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "sPoSUqmL-ts5"
       },
       "source": [
         "### Use eager execution in a graph environment\n",
         "\n",
         "Selectively enable eager execution in a TensorFlow graph environment using\n",
-        "`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not*\n",
+        "`tfe.py_func`. This is used when `` has *not*\n",
         "been called."
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "attributes": {
-          "classes": [
-            "py"
-          ],
-          "id": ""
-        },
-        "colab_type": "code",
-        "id": "Lks-3LB0qEhi",
-        "colab": {}
+        "id": "Lks-3LB0qEhi"
       },
+      "outputs": [],
       "source": [
         "def my_py_func(x):\n",
         "  x = tf.matmul(x, x)  # You can use tf ops\n",
@@ -1682,12 +1257,23 @@
         "with tf.Session() as sess:\n",
         "  x = tf.placeholder(dtype=tf.float32)\n",
         "  # Call eager function in graph!\n",
-        "  pf = tfe.py_func(my_py_func, [x], tf.float32)\n",
+        "  pf = tf.py_func(my_py_func, [x], tf.float32)\n",
         "\n",
         "  sess.run(pf, feed_dict={x: [[2.0]]})  # [[4.0]]"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "eager.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/guide/extend/architecture.md b/site/en/r1/guide/extend/architecture.md
index 91219ac273b..0753824e15e 100644
--- a/site/en/r1/guide/extend/architecture.md
+++ b/site/en/r1/guide/extend/architecture.md
@@ -8,7 +8,7 @@ This document describes the system architecture that makes this
 combination of scale and flexibility possible. It assumes that you have basic familiarity
 with TensorFlow programming concepts such as the computation graph, operations,
 and sessions. See [this document](../low_level_intro.md) for an introduction to
-these topics. Some familiarity with [distributed TensorFlow](../../deploy/distributed.md)
+these topics. Some familiarity with [distributed TensorFlow](../distribute_strategy.ipynb)
 will also be helpful.
 
 This document is for developers who want to extend TensorFlow in some way not
@@ -24,7 +24,7 @@ The TensorFlow runtime is a cross-platform library. Figure 1 illustrates its
 general architecture. A C API separates user level code in different languages
 from the core runtime.
 
-![TensorFlow Layers](https://www.tensorflow.org/images/layers.png){: width="300"}
+TensorFlow Layers
 
 **Figure 1**
 
@@ -34,7 +34,7 @@ This document focuses on the following layers:
 *  **Client**:
    *  Defines the computation as a dataflow graph.
    *  Initiates graph execution using a [**session**](
-      https://www.tensorflow.org/code/tensorflow/python/client/session.py).
+      https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/client/session.py).
 *  **Distributed Master**
    *  Prunes a specific subgraph from the graph, as defined by the arguments
       to Session.run().
@@ -56,7 +56,7 @@ Other tasks send updates to these parameters as they work on optimizing the
 parameters. This particular division of labor between tasks is not required, but
  is common for distributed training.
 
-![TensorFlow Architecture Diagram](https://www.tensorflow.org/images/diag1.svg){: width="500"}
+TensorFlow Architecture Diagram
 
 **Figure 2**
 
@@ -90,7 +90,7 @@ In Figure 3, the client has built a graph that applies weights (w) to a
 feature vector (x), adds a bias term (b) and saves the result in a variable
 (s).
 
-![TensorFlow Architecture Diagram: Client](https://www.tensorflow.org/images/graph_client.svg){: width="700"}
+TensorFlow Architecture Diagram: Client
 
 **Figure 3**
 
@@ -113,7 +113,7 @@ a step, it applies standard optimizations such as common subexpression
 elimination and constant folding. It then coordinates execution of the
 optimized subgraphs across a set of tasks.
 
-![TensorFlow Architecture Diagram: Master](https://www.tensorflow.org/images/graph_master_cln.svg){: width="700"}
+TensorFlow Architecture Diagram: Master
 
 **Figure 4**
 
@@ -122,7 +122,7 @@ Figure 5 shows a possible partition of our example graph. The distributed
 master has grouped the model parameters in order to place them together on the
 parameter server.
 
-![Partitioned Graph](https://www.tensorflow.org/images/graph_split1.svg){: width="700"}
+Partitioned Graph
 
 **Figure 5**
 
@@ -131,21 +131,21 @@ Where graph edges are cut by the partition, the distributed master inserts
 send and receive nodes to pass information between the distributed tasks
 (Figure 6).
 
-![Partitioned Graph](https://www.tensorflow.org/images/graph_split2.svg){: width="700"}
+Partitioned Graph
 
 **Figure 6**
 
 
 The distributed master then ships the graph pieces to the distributed tasks.
 
-![Partitioned Graph](https://www.tensorflow.org/images/graph_workers_cln.svg){: width="700"}
+Partitioned Graph
 
 **Figure 7**
 
 ### Code
 
-*  [MasterService API definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/master_service.proto)
-*  [Master interface](https://www.tensorflow.org/code/tensorflow/core/distributed_runtime/master_interface.h)
+*  [MasterService API definition](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/master_service.proto)
+*  [Master interface](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/distributed_runtime/master_interface.h)
 
 ## Worker Service
 
@@ -178,17 +178,17 @@ For transfers between tasks, TensorFlow uses multiple protocols, including:
 
 We also have preliminary support for NVIDIA's NCCL library for multi-GPU
 communication, see:
-[`tf.contrib.nccl`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/nccl_ops.py).
+[`tf.contrib.nccl`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/ops/nccl_ops.py).
 
-![Partitioned Graph](https://www.tensorflow.org/images/graph_send_recv.svg){: width="700"}
+Partitioned Graph
 
 **Figure 8**
 
 ### Code
 
-*   [WorkerService API definition](https://www.tensorflow.org/code/tensorflow/core/protobuf/worker_service.proto)
-*   [Worker interface](https://www.tensorflow.org/code/tensorflow/core/distributed_runtime/worker_interface.h)
-*   [Remote rendezvous (for Send and Recv implementations)](https://www.tensorflow.org/code/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h)
+*   [WorkerService API definition](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/worker_service.proto)
+*   [Worker interface](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/distributed_runtime/worker_interface.h)
+*   [Remote rendezvous (for Send and Recv implementations)](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h)
 
 ## Kernel Implementations
 
@@ -199,7 +199,7 @@ Many of the operation kernels are implemented using Eigen::Tensor, which uses
 C++ templates to generate efficient parallel code for multicore CPUs and GPUs;
 however, we liberally use libraries like cuDNN where a more efficient kernel
 implementation is possible. We have also implemented
-[quantization](../../lite/performance/post_training_quantization.md), which enables
+[quantization](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/lite/g3doc/performance/post_training_quantization.md), which enables
 faster inference in environments such as mobile devices and high-throughput
 datacenter applications, and use the
 [gemmlowp](https://github.com/google/gemmlowp) low-precision matrix library to
@@ -209,10 +209,10 @@ If it is difficult or inefficient to represent a subcomputation as a composition
 of operations, users can register additional kernels that provide an efficient
 implementation written in C++. For example, we recommend registering your own
 fused kernels for some performance critical operations, such as the ReLU and
-Sigmoid activation functions and their corresponding gradients. The [XLA Compiler](../../xla/) has an
+Sigmoid activation functions and their corresponding gradients. The [XLA Compiler](../../../xla/README.md) has an
 experimental implementation of automatic kernel fusion.
 
 
 ### Code
 
-*   [`OpKernel` interface](https://www.tensorflow.org/code/tensorflow/core/framework/op_kernel.h)
+*   [`OpKernel` interface](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_kernel.h)
diff --git a/site/en/r1/guide/extend/bindings.md b/site/en/r1/guide/extend/bindings.md
index 9c10e90840f..7daa2212106 100644
--- a/site/en/r1/guide/extend/bindings.md
+++ b/site/en/r1/guide/extend/bindings.md
@@ -112,11 +112,11 @@ There are a few ways to get a list of the `OpDef`s for the registered ops:
     to interpret the `OpDef` messages.
 -   The C++ function `OpRegistry::Global()->GetRegisteredOps()` returns the same
     list of all registered `OpDef`s (defined in
-    [`tensorflow/core/framework/op.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op.h)). This can be used to write the generator
+    [`tensorflow/core/framework/op.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op.h)). This can be used to write the generator
     in C++ (particularly useful for languages that do not have protocol buffer
     support).
 -   The ASCII-serialized version of that list is periodically checked in to
-    [`tensorflow/core/ops/ops.pbtxt`](https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt) by an automated process.
+    [`tensorflow/core/ops/ops.pbtxt`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/ops.pbtxt) by an automated process.
 
 The `OpDef` specifies the following:
 
@@ -159,7 +159,7 @@ between the generated code and the `OpDef`s checked into the repository, but is
 useful for languages where code is expected to be generated ahead of time like
 `go get` for Go and `cargo ops` for Rust. At the other end of the spectrum, for
 some languages the code could be generated dynamically from
-[`tensorflow/core/ops/ops.pbtxt`](https://www.tensorflow.org/code/tensorflow/core/ops/ops.pbtxt).
+[`tensorflow/core/ops/ops.pbtxt`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/ops.pbtxt).
 
 #### Handling Constants
 
@@ -228,4 +228,4 @@ At this time, support for gradients, functions and control flow operations ("if"
 and "while") is not available in languages other than Python. This will be
 updated when the [C API] provides necessary support.
 
-[C API]: https://www.tensorflow.org/code/tensorflow/c/c_api.h
+[C API]: https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/c/c_api.h
diff --git a/site/en/r1/guide/extend/filesystem.md b/site/en/r1/guide/extend/filesystem.md
index 4d34c07102e..2d6ea0c4645 100644
--- a/site/en/r1/guide/extend/filesystem.md
+++ b/site/en/r1/guide/extend/filesystem.md
@@ -54,7 +54,7 @@ To implement a custom filesystem plugin, you must do the following:
 ### The FileSystem interface
 
 The `FileSystem` interface is an abstract C++ interface defined in
-[file_system.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/file_system.h).
+[file_system.h](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/file_system.h).
 An implementation of the `FileSystem` interface should implement all relevant
 the methods defined by the interface. Implementing the interface requires
 defining operations such as creating `RandomAccessFile`, `WritableFile`, and
@@ -70,26 +70,26 @@ involves calling `stat()` on the file and then returns the filesize as reported
 by the return of the stat object. Similarly, for the `HDFSFileSystem`
 implementation, these calls simply delegate to the `libHDFS` implementation of
 similar functionality, such as `hdfsDelete` for
-[DeleteFile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/hadoop/hadoop_file_system.cc#L386).
+[DeleteFile](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/hadoop/hadoop_file_system.cc#L386).
 
 We suggest looking through these code examples to get an idea of how different
 filesystem implementations call their existing libraries. Examples include:
 
 *   [POSIX
-    plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/posix/posix_file_system.h)
+    plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/posix/posix_file_system.h)
 *   [HDFS
-    plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/hadoop/hadoop_file_system.h)
+    plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/hadoop/hadoop_file_system.h)
 *   [GCS
-    plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/cloud/gcs_file_system.h)
+    plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/cloud/gcs_file_system.h)
 *   [S3
-    plugin](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/s3/s3_file_system.h)
+    plugin](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/s3/s3_file_system.h)
 
 #### The File interfaces
 
 Beyond operations that allow you to query and manipulate files and directories
 in a filesystem, the `FileSystem` interface requires you to implement factories
 that return implementations of abstract objects such as the
-[RandomAccessFile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/platform/file_system.h#L223),
+[RandomAccessFile](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/platform/file_system.h#L223),
 the `WritableFile`, so that TensorFlow code and read and write to files in that
 `FileSystem` implementation.
 
@@ -224,7 +224,7 @@ it will use the `FooBarFileSystem` implementation.
 
 Next, you must build a shared object containing this implementation. An example
 of doing so using bazel's `cc_binary` rule can be found
-[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD#L244),
+[here](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/BUILD#L244),
 but you may use any build system to do so. See the section on [building the op library](../extend/op.md#build_the_op_library) for similar
 instructions.
 
@@ -236,7 +236,7 @@ passing the path to the shared object. Calling this in your client program loads
 the shared object in the process, thus registering your implementation as
 available for any file operations going through the `FileSystem` interface. You
 can see
-[test_file_system.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/framework/file_system_test.py)
+[test_file_system.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/framework/file_system_test.py)
 for an example.
 
 ## What goes through this interface?
diff --git a/site/en/r1/guide/extend/formats.md b/site/en/r1/guide/extend/formats.md
index 3b7b4aafbd6..bdebee5487d 100644
--- a/site/en/r1/guide/extend/formats.md
+++ b/site/en/r1/guide/extend/formats.md
@@ -28,11 +28,11 @@ individual records in a file. There are several examples of "reader" datasets
 that are already built into TensorFlow:
 
 *   `tf.data.TFRecordDataset`
-    ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+    ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
 *   `tf.data.FixedLengthRecordDataset`
-    ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+    ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
 *   `tf.data.TextLineDataset`
-    ([source in `kernels/data/reader_dataset_ops.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/data/reader_dataset_ops.cc))
+    ([source in `kernels/data/reader_dataset_ops.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/data/reader_dataset_ops.cc))
 
 Each of these implementations comprises three related classes:
 
@@ -279,7 +279,7 @@ if __name__ == "__main__":
 ```
 
 You can see some examples of `Dataset` wrapper classes in
-[`tensorflow/python/data/ops/dataset_ops.py`](https://www.tensorflow.org/code/tensorflow/python/data/ops/dataset_ops.py).
+[`tensorflow/python/data/ops/dataset_ops.py`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/data/ops/dataset_ops.py).
 
 ## Writing an Op for a record format
 
@@ -297,7 +297,7 @@ Examples of Ops useful for decoding records:
 
 Note that it can be useful to use multiple Ops to decode a particular record
 format.  For example, you may have an image saved as a string in
-[a `tf.train.Example` protocol buffer](https://www.tensorflow.org/code/tensorflow/core/example/example.proto).
+[a `tf.train.Example` protocol buffer](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto).
 Depending on the format of that image, you might take the corresponding output
 from a `tf.parse_single_example` op and call `tf.image.decode_jpeg`,
 `tf.image.decode_png`, or `tf.decode_raw`.  It is common to take the output
diff --git a/site/en/r1/guide/extend/model_files.md b/site/en/r1/guide/extend/model_files.md
index 30e73a5169e..e590fcf1f27 100644
--- a/site/en/r1/guide/extend/model_files.md
+++ b/site/en/r1/guide/extend/model_files.md
@@ -28,7 +28,7 @@ by calling `as_graph_def()`, which returns a `GraphDef` object.
 
 The GraphDef class is an object created by the ProtoBuf library from the
 definition in
-[tensorflow/core/framework/graph.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto). The protobuf tools parse
+[tensorflow/core/framework/graph.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/graph.proto). The protobuf tools parse
 this text file, and generate the code to load, store, and manipulate graph
 definitions. If you see a standalone TensorFlow file representing a model, it's
 likely to contain a serialized version of one of these `GraphDef` objects
@@ -87,7 +87,7 @@ for node in graph_def.node
 ```
 
 Each node is a `NodeDef` object, defined in
-[tensorflow/core/framework/node_def.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/node_def.proto). These
+[tensorflow/core/framework/node_def.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/node_def.proto). These
 are the fundamental building blocks of TensorFlow graphs, with each one defining
 a single operation along with its input connections. Here are the members of a
 `NodeDef`, and what they mean.
@@ -107,7 +107,7 @@ This defines what operation to run, for example `"Add"`, `"MatMul"`, or
 `"Conv2D"`. When a graph is run, this op name is looked up in a registry to
 find an implementation. The registry is populated by calls to the
 `REGISTER_OP()` macro, like those in
-[tensorflow/core/ops/nn_ops.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/ops/nn_ops.cc).
+[tensorflow/core/ops/nn_ops.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/nn_ops.cc).
 
 ### `input`
 
@@ -133,7 +133,7 @@ size of filters for convolutions, or the values of constant ops. Because there
 can be so many different types of attribute values, from strings, to ints, to
 arrays of tensor values, there's a separate protobuf file defining the data
 structure that holds them, in
-[tensorflow/core/framework/attr_value.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto).
+[tensorflow/core/framework/attr_value.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/attr_value.proto).
 
 Each attribute has a unique name string, and the expected attributes are listed
 when the operation is defined. If an attribute isn't present in a node, but it
@@ -151,7 +151,7 @@ the file format during training. Instead, they're held in separate checkpoint
 files, and there are `Variable` ops in the graph that load the latest values
 when they're initialized. It's often not very convenient to have separate files
 when you're deploying to production, so there's the
-[freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py) script that takes a graph definition and a set
+[freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/tools/freeze_graph.py) script that takes a graph definition and a set
 of checkpoints and freezes them together into a single file.
 
 What this does is load the `GraphDef`, pull in the values for all the variables
@@ -167,7 +167,7 @@ the most common problems is extracting and interpreting the weight values. A
 common way to store them, for example in graphs created by the freeze_graph
 script, is as `Const` ops containing the weights as `Tensors`. These are
 defined in
-[tensorflow/core/framework/tensor.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto), and contain information
+[tensorflow/core/framework/tensor.proto](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor.proto), and contain information
 about the size and type of the data, as well as the values themselves. In
 Python, you get a `TensorProto` object from a `NodeDef` representing a `Const`
 op by calling something like `some_node_def.attr['value'].tensor`.
diff --git a/site/en/r1/guide/extend/op.md b/site/en/r1/guide/extend/op.md
index 97cba10f7bb..186d9c28c04 100644
--- a/site/en/r1/guide/extend/op.md
+++ b/site/en/r1/guide/extend/op.md
@@ -47,7 +47,7 @@ To incorporate your custom op you'll need to:
     test the op in C++. If you define gradients, you can verify them with the
     Python `tf.test.compute_gradient_error`.
     See
-    [`relu_op_test.py`](https://www.tensorflow.org/code/tensorflow/python/kernel_tests/relu_op_test.py) as
+    [`relu_op_test.py`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/kernel_tests/relu_op_test.py) as
     an example that tests the forward functions of Relu-like operators and
     their gradients.
 
@@ -155,17 +155,17 @@ REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp);
 >   Important: Instances of your OpKernel may be accessed concurrently.
 >   Your `Compute` method must be thread-safe. Guard any access to class
 >   members with a mutex. Or better yet, don't share state via class members!
->   Consider using a [`ResourceMgr`](https://www.tensorflow.org/code/tensorflow/core/framework/resource_mgr.h)
+>   Consider using a [`ResourceMgr`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/resource_mgr.h)
 >   to keep track of op state.
 
 ### Multi-threaded CPU kernels
 
 To write a multi-threaded CPU kernel, the Shard function in
-[`work_sharder.h`](https://www.tensorflow.org/code/tensorflow/core/util/work_sharder.h)
+[`work_sharder.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/util/work_sharder.h)
 can be used. This function shards a computation function across the
 threads configured to be used for intra-op threading (see
 intra_op_parallelism_threads in
-[`config.proto`](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto)).
+[`config.proto`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/config.proto)).
 
 ### GPU kernels
 
@@ -331,9 +331,9 @@ Here are the outputs of these functions on an Ubuntu machine.
 $ python
 >>> import tensorflow as tf
 >>> tf.sysconfig.get_include()
-'/usr/local/lib/python2.7/site-packages/tensorflow/include'
+'/usr/local/lib/python3.6/site-packages/tensorflow/include'
 >>> tf.sysconfig.get_lib()
-'/usr/local/lib/python2.7/site-packages/tensorflow'
+'/usr/local/lib/python3.6/site-packages/tensorflow'
 ```
 
 Assuming you have `g++` installed, here is the sequence of commands you can use
@@ -345,15 +345,16 @@ TF_LFLAGS=( $(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.ge
 g++ -std=c++11 -shared zero_out.cc -o zero_out.so -fPIC ${TF_CFLAGS[@]} ${TF_LFLAGS[@]} -O2
 ```
 
-On Mac OS X, the additional flag "-undefined dynamic_lookup" is required when
+On macOS, the additional flag "-undefined dynamic_lookup" is required when
 building the `.so` file.
 
->   Note on `gcc` version `>=5`: gcc uses the new C++
->   [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`. The binary pip
->   packages available on the TensorFlow website are built with `gcc4` that uses
->   the older ABI. If you compile your op library with `gcc>=5`, add
->   `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to make the library
->   compatible with the older abi.
+> Note on `gcc` version `>=5`: gcc uses the new C++
+> [ABI](https://gcc.gnu.org/gcc-5/changes.html#libstdcxx) since version `5`.
+> TensorFlow 2.8 and earlier were built with `gcc4` that uses the older ABI. If
+> you are using these versions of TensorFlow and are trying to compile your op
+> library with `gcc>=5`, add `-D_GLIBCXX_USE_CXX11_ABI=0` to the command line to
+> make the library compatible with the older ABI. TensorFlow 2.9+ packages are
+> compatible with the newer ABI by default.
 
 ### Compile the op using bazel (TensorFlow source installation)
 
@@ -485,13 +486,13 @@ This asserts that the input is a vector, and returns having set the
 
 *   The `context`, which can either be an `OpKernelContext` or
     `OpKernelConstruction` pointer (see
-    [`tensorflow/core/framework/op_kernel.h`](https://www.tensorflow.org/code/tensorflow/core/framework/op_kernel.h)),
+    [`tensorflow/core/framework/op_kernel.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_kernel.h)),
     for its `SetStatus()` method.
 *   The condition.  For example, there are functions for validating the shape
     of a tensor in
-    [`tensorflow/core/framework/tensor_shape.h`](https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.h)
+    [`tensorflow/core/framework/tensor_shape.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor_shape.h)
 *   The error itself, which is represented by a `Status` object, see
-    [`tensorflow/core/lib/core/status.h`](https://www.tensorflow.org/code/tensorflow/core/lib/core/status.h). A
+    [`tensorflow/core/lib/core/status.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/lib/core/status.h). A
     `Status` has both a type (frequently `InvalidArgument`, but see the list of
     types) and a message.  Functions for constructing an error may be found in
     [`tensorflow/core/lib/core/errors.h`][validation-macros].
@@ -632,7 +633,7 @@ define an attr with constraints, you can use the following ``s:
 
     The specific lists of types allowed by these are defined by the functions
     (like `NumberTypes()`) in
-    [`tensorflow/core/framework/types.h`](https://www.tensorflow.org/code/tensorflow/core/framework/types.h).
+    [`tensorflow/core/framework/types.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/types.h).
     In this example the attr `t` must be one of the numeric types:
 
     ```c++
@@ -1179,7 +1180,7 @@ There are several ways to preserve backwards-compatibility.
    type into a list of varying types).
 
 The full list of safe and unsafe changes can be found in
-[`tensorflow/core/framework/op_compatibility_test.cc`](https://www.tensorflow.org/code/tensorflow/core/framework/op_compatibility_test.cc).
+[`tensorflow/core/framework/op_compatibility_test.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_compatibility_test.cc).
 If you cannot make your change to an operation backwards compatible, then create
 a new operation with a new name with the new semantics.
 
@@ -1188,24 +1189,24 @@ generated Python code may change in a way that isn't compatible with old
 callers.  The Python API may be kept compatible by careful changes in a
 hand-written Python wrapper, by keeping the old signature except possibly adding
 new optional arguments to the end.  Generally incompatible changes may only be
-made when TensorFlow's changes major versions, and must conform to the
-[`GraphDef` version semantics](../guide/version_compat.md#compatibility_of_graphs_and_checkpoints).
+made when TensorFlow changes major versions, and must conform to the
+[`GraphDef` version semantics](../version_compat.md).
 
 ### GPU Support
 
 You can implement different OpKernels and register one for CPU and another for
 GPU, just like you can [register kernels for different types](#polymorphism).
 There are several examples of kernels with GPU support in
-[`tensorflow/core/kernels/`](https://www.tensorflow.org/code/tensorflow/core/kernels/).
+[`tensorflow/core/kernels/`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/).
 Notice some kernels have a CPU version in a `.cc` file, a GPU version in a file
 ending in `_gpu.cu.cc`, and some code shared in common in a `.h` file.
 
 For example, the `tf.pad` has
 everything but the GPU kernel in [`tensorflow/core/kernels/pad_op.cc`][pad_op].
 The GPU kernel is in
-[`tensorflow/core/kernels/pad_op_gpu.cu.cc`](https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op_gpu.cu.cc),
+[`tensorflow/core/kernels/pad_op_gpu.cu.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/pad_op_gpu.cu.cc),
 and the shared code is a templated class defined in
-[`tensorflow/core/kernels/pad_op.h`](https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op.h).
+[`tensorflow/core/kernels/pad_op.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/pad_op.h).
 We organize the code this way for two reasons: it allows you to share common
 code among the CPU and GPU implementations, and it puts the GPU implementation
 into a separate file so that it can be compiled only by the GPU compiler.
@@ -1226,16 +1227,16 @@ kept on the CPU, add a `HostMemory()` call to the kernel registration, e.g.:
 #### Compiling the kernel for the GPU device
 
 Look at
-[cuda_op_kernel.cu.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
+[cuda_op_kernel.cu.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
 for an example that uses a CUDA kernel to implement an op. The
 `tf_custom_op_library` accepts a `gpu_srcs` argument in which the list of source
 files containing the CUDA kernels (`*.cu.cc` files) can be specified. For use
 with a binary installation of TensorFlow, the CUDA kernels have to be compiled
 with NVIDIA's `nvcc` compiler. Here is the sequence of commands you can use to
 compile the
-[cuda_op_kernel.cu.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
+[cuda_op_kernel.cu.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc)
 and
-[cuda_op_kernel.cc](https://www.tensorflow.org/code/tensorflow/examples/adding_an_op/cuda_op_kernel.cc)
+[cuda_op_kernel.cc](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/adding_an_op/cuda_op_kernel.cc)
 into a single dynamically loadable library:
 
 ```bash
@@ -1360,7 +1361,7 @@ be set to the first input's shape. If the output is selected by its index as in
 
 There are a number of common shape functions
 that apply to many ops, such as `shape_inference::UnchangedShape` which can be
-found in [common_shape_fns.h](https://www.tensorflow.org/code/tensorflow/core/framework/common_shape_fns.h) and used as follows:
+found in [common_shape_fns.h](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/common_shape_fns.h) and used as follows:
 
 ```c++
 REGISTER_OP("ZeroOut")
@@ -1407,7 +1408,7 @@ provides access to the attributes of the op).
 
 Since shape inference is an optional feature, and the shapes of tensors may vary
 dynamically, shape functions must be robust to incomplete shape information for
-any of the inputs. The `Merge` method in [`InferenceContext`](https://www.tensorflow.org/code/tensorflow/core/framework/shape_inference.h)
+any of the inputs. The `Merge` method in [`InferenceContext`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/shape_inference.h)
 allows the caller to assert that two shapes are the same, even if either
 or both of them do not have complete information. Shape functions are defined
 for all of the core TensorFlow ops and provide many different usage examples.
@@ -1432,7 +1433,7 @@ If you have a complicated shape function, you should consider adding a test for
 validating that various input shape combinations produce the expected output
 shape combinations.  You can see examples of how to write these tests in some
 our
-[core ops tests](https://www.tensorflow.org/code/tensorflow/core/ops/array_ops_test.cc).
+[core ops tests](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/array_ops_test.cc).
 (The syntax of `INFER_OK` and `INFER_ERROR` are a little cryptic, but try to be
 compact in representing input and output shape specifications in tests.  For
 now, see the surrounding comments in those tests to get a sense of the shape
@@ -1445,20 +1446,20 @@ To build a `pip` package for your op, see the
 guide shows how to build custom ops from the TensorFlow pip package instead
 of building TensorFlow from source.
 
-[core-array_ops]:https://www.tensorflow.org/code/tensorflow/core/ops/array_ops.cc
-[python-user_ops]:https://www.tensorflow.org/code/tensorflow/python/user_ops/user_ops.py
-[tf-kernels]:https://www.tensorflow.org/code/tensorflow/core/kernels/
-[user_ops]:https://www.tensorflow.org/code/tensorflow/core/user_ops/
-[pad_op]:https://www.tensorflow.org/code/tensorflow/core/kernels/pad_op.cc
-[standard_ops-py]:https://www.tensorflow.org/code/tensorflow/python/ops/standard_ops.py
-[standard_ops-cc]:https://www.tensorflow.org/code/tensorflow/cc/ops/standard_ops.h
-[python-BUILD]:https://www.tensorflow.org/code/tensorflow/python/BUILD
-[validation-macros]:https://www.tensorflow.org/code/tensorflow/core/lib/core/errors.h
-[op_def_builder]:https://www.tensorflow.org/code/tensorflow/core/framework/op_def_builder.h
-[register_types]:https://www.tensorflow.org/code/tensorflow/core/framework/register_types.h
-[FinalizeAttr]:https://www.tensorflow.org/code/tensorflow/core/framework/op_def_builder.cc
-[DataTypeString]:https://www.tensorflow.org/code/tensorflow/core/framework/types.cc
-[python-BUILD]:https://www.tensorflow.org/code/tensorflow/python/BUILD
-[types-proto]:https://www.tensorflow.org/code/tensorflow/core/framework/types.proto
-[TensorShapeProto]:https://www.tensorflow.org/code/tensorflow/core/framework/tensor_shape.proto
-[TensorProto]:https://www.tensorflow.org/code/tensorflow/core/framework/tensor.proto
+[core-array_ops]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/ops/array_ops.cc
+[python-user_ops]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/user_ops/user_ops.py
+[tf-kernels]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/
+[user_ops]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/user_ops/
+[pad_op]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/kernels/pad_op.cc
+[standard_ops-py]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/ops/standard_ops.py
+[standard_ops-cc]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/cc/ops/standard_ops.h
+[python-BUILD]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/BUILD
+[validation-macros]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/lib/core/errors.h
+[op_def_builder]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_def_builder.h
+[register_types]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/register_types.h
+[FinalizeAttr]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_def_builder.cc
+[DataTypeString]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/types.cc
+[python-BUILD]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/BUILD
+[types-proto]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/types.proto
+[TensorShapeProto]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor_shape.proto
+[TensorProto]:https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor.proto
diff --git a/site/en/r1/guide/feature_columns.md b/site/en/r1/guide/feature_columns.md
index 5a4dfbbf46d..e4259f85e9f 100644
--- a/site/en/r1/guide/feature_columns.md
+++ b/site/en/r1/guide/feature_columns.md
@@ -562,7 +562,7 @@ For more examples on feature columns, view the following:
 
 * The [Low Level Introduction](../guide/low_level_intro.md#feature_columns) demonstrates how
   experiment directly with `feature_columns` using TensorFlow's low level APIs.
-* The [Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep)
+* The [Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep)
   solves a binary classification problem using `feature_columns` on a variety of
   input data types.
 
diff --git a/site/en/r1/guide/graph_viz.md b/site/en/r1/guide/graph_viz.md
index 5d65c16602b..1e3780e7928 100644
--- a/site/en/r1/guide/graph_viz.md
+++ b/site/en/r1/guide/graph_viz.md
@@ -251,9 +251,9 @@ is a snippet from the train and test section of a modification of the
 [Estimators MNIST tutorial](../tutorials/estimators/cnn.md), in which we have
 recorded summaries and
 runtime statistics. See the
-[Tensorboard](https://tensorflow.org/tensorboard)
+[TensorBoard](https://tensorflow.org/tensorboard)
 for details on how to record summaries.
-Full source is [here](https://www.tensorflow.org/code/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py).
+Full source is [here](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py).
 
 ```python
   # Train the model, and also write summaries.
diff --git a/site/en/r1/guide/keras.ipynb b/site/en/r1/guide/keras.ipynb
index eb6f3787fe5..3a0cd8e55c5 100644
--- a/site/en/r1/guide/keras.ipynb
+++ b/site/en/r1/guide/keras.ipynb
@@ -1,25 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "keras.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "CCQY7jpBfMur"
       },
       "source": [
@@ -28,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "z6X9omPnfO_h",
-        "colab": {}
+        "id": "z6X9omPnfO_h"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -46,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "F1xIRPtY0E1w"
       },
       "source": [
@@ -63,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "VyOjQZHhZxaA"
       },
       "source": [
@@ -80,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "BPe-206xGPfe"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "ViAXWoKlZ8s6"
       },
       "source": [
@@ -101,7 +93,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IsK5aF2xZ-40"
       },
       "source": [
@@ -120,40 +111,34 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Qoc055N3wiUG",
-        "colab": {}
+        "id": "Qoc055N3wiUG"
       },
+      "outputs": [],
       "source": [
         "!pip install pyyaml  # Required to save models in YAML format"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "TgPcBFru0E1z",
-        "colab": {}
+        "id": "TgPcBFru0E1z"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
+        "import tensorflow.compat.v1 as tf\n",
         "\n",
-        "import tensorflow as tf\n",
         "from tensorflow.keras import layers\n",
         "\n",
         "print(tf.version.VERSION)\n",
         "print(tf.keras.__version__)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "lj03RamP0E13"
       },
       "source": [
@@ -169,7 +154,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "7e1LPcXx0gR6"
       },
       "source": [
@@ -186,11 +170,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "WM-DUVQB0E14",
-        "colab": {}
+        "id": "WM-DUVQB0E14"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Sequential()\n",
         "# Adds a densely-connected layer with 64 units to the model:\n",
@@ -199,14 +183,11 @@
         "model.add(layers.Dense(64, activation='relu'))\n",
         "# Add a softmax layer with 10 output units:\n",
         "model.add(layers.Dense(10, activation='softmax'))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-ztyTipu0E18"
       },
       "source": [
@@ -220,7 +201,8 @@
         "  default, no activation is applied.\n",
         "* `kernel_initializer` and `bias_initializer`: The initialization schemes\n",
         "  that create the layer's weights (kernel and bias). This parameter is a name or\n",
-        "  a callable object. This defaults to the `\"Glorot uniform\"` initializer.\n",
+        "  a callable object. The kernel defaults to the `\"Glorot uniform\"` initializer, \n",
+        "  and the bias defaults to zeros.\n",
         "* `kernel_regularizer` and `bias_regularizer`: The regularization schemes\n",
         "  that apply the layer's weights (kernel and bias), such as L1 or L2\n",
         "  regularization. By default, no regularization is applied.\n",
@@ -231,11 +213,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "MlL7PBtp0E19",
-        "colab": {}
+        "id": "MlL7PBtp0E19"
       },
+      "outputs": [],
       "source": [
         "# Create a sigmoid layer:\n",
         "layers.Dense(64, activation='sigmoid')\n",
@@ -253,14 +235,11 @@
         "\n",
         "# A linear layer with a bias vector initialized to 2.0s:\n",
         "layers.Dense(64, bias_initializer=tf.keras.initializers.constant(2.0))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "9NR6reyk0E2A"
       },
       "source": [
@@ -274,11 +253,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "sJ4AOn090E2A",
-        "colab": {}
+        "id": "sJ4AOn090E2A"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Sequential([\n",
         "# Adds a densely-connected layer with 64 units to the model:\n",
@@ -291,14 +270,11 @@
         "model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n",
         "              loss='categorical_crossentropy',\n",
         "              metrics=['accuracy'])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "HG-RAa9F0E2D"
       },
       "source": [
@@ -320,11 +296,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "St4Mgdar0E2E",
-        "colab": {}
+        "id": "St4Mgdar0E2E"
       },
+      "outputs": [],
       "source": [
         "# Configure a model for mean-squared error regression.\n",
         "model.compile(optimizer=tf.train.AdamOptimizer(0.01),\n",
@@ -335,14 +311,11 @@
         "model.compile(optimizer=tf.train.RMSPropOptimizer(0.01),\n",
         "              loss=tf.keras.losses.categorical_crossentropy,\n",
         "              metrics=[tf.keras.metrics.categorical_accuracy])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yjI5rbi80E2G"
       },
       "source": [
@@ -355,11 +328,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "3CvP6L-m0E2I",
-        "colab": {}
+        "id": "3CvP6L-m0E2I"
       },
+      "outputs": [],
       "source": [
         "import numpy as np\n",
         "\n",
@@ -374,14 +347,11 @@
         "labels = random_one_hot_labels((1000, 10))\n",
         "\n",
         "model.fit(data, labels, epochs=10, batch_size=32)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "N-pnVaFe0E2N"
       },
       "source": [
@@ -403,11 +373,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "gFcXzVQa0E2N",
-        "colab": {}
+        "id": "gFcXzVQa0E2N"
       },
+      "outputs": [],
       "source": [
         "import numpy as np\n",
         "\n",
@@ -419,14 +389,11 @@
         "\n",
         "model.fit(data, labels, epochs=10, batch_size=32,\n",
         "          validation_data=(val_data, val_labels))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-6ImyXzz0E2Q"
       },
       "source": [
@@ -439,11 +406,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "OziqhpIj0E2R",
-        "colab": {}
+        "id": "OziqhpIj0E2R"
       },
+      "outputs": [],
       "source": [
         "# Instantiates a toy dataset instance:\n",
         "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n",
@@ -452,14 +419,11 @@
         "\n",
         "# Don't forget to specify `steps_per_epoch` when calling `fit` on a dataset.\n",
         "model.fit(dataset, epochs=10, steps_per_epoch=30)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "I7BcMHkB0E2U"
       },
       "source": [
@@ -472,11 +436,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "YPMb3A0N0E2V",
-        "colab": {}
+        "id": "YPMb3A0N0E2V"
       },
+      "outputs": [],
       "source": [
         "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n",
         "dataset = dataset.batch(32).repeat()\n",
@@ -487,14 +451,11 @@
         "model.fit(dataset, epochs=10, steps_per_epoch=30,\n",
         "          validation_data=val_dataset,\n",
         "          validation_steps=3)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IgGdlXso0E2X"
       },
       "source": [
@@ -508,11 +469,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "mhDbOHEK0E2Y",
-        "colab": {}
+        "id": "mhDbOHEK0E2Y"
       },
+      "outputs": [],
       "source": [
         "data = np.random.random((1000, 32))\n",
         "labels = random_one_hot_labels((1000, 10))\n",
@@ -520,14 +481,11 @@
         "model.evaluate(data, labels, batch_size=32)\n",
         "\n",
         "model.evaluate(dataset, steps=30)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "UXUTmDfb0E2b"
       },
       "source": [
@@ -537,22 +495,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "9e3JsSoQ0E2c",
-        "colab": {}
+        "id": "9e3JsSoQ0E2c"
       },
+      "outputs": [],
       "source": [
         "result = model.predict(data, batch_size=32)\n",
         "print(result.shape)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "fzEOW4Cn0E2h"
       },
       "source": [
@@ -583,11 +538,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "mROj832r0E2i",
-        "colab": {}
+        "id": "mROj832r0E2i"
       },
+      "outputs": [],
       "source": [
         "inputs = tf.keras.Input(shape=(32,))  # Returns a placeholder tensor\n",
         "\n",
@@ -595,14 +550,11 @@
         "x = layers.Dense(64, activation='relu')(inputs)\n",
         "x = layers.Dense(64, activation='relu')(x)\n",
         "predictions = layers.Dense(10, activation='softmax')(x)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "AFmspHeG1_W7"
       },
       "source": [
@@ -611,11 +563,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "5k5uzlyu16HM",
-        "colab": {}
+        "id": "5k5uzlyu16HM"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Model(inputs=inputs, outputs=predictions)\n",
         "\n",
@@ -626,14 +578,11 @@
         "\n",
         "# Trains for 5 epochs\n",
         "model.fit(data, labels, batch_size=32, epochs=5)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "EcKSLH3i0E2k"
       },
       "source": [
@@ -657,11 +606,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "KLiHWzcn2Fzk",
-        "colab": {}
+        "id": "KLiHWzcn2Fzk"
       },
+      "outputs": [],
       "source": [
         "class MyModel(tf.keras.Model):\n",
         "\n",
@@ -685,14 +634,11 @@
         "    shape = tf.TensorShape(input_shape).as_list()\n",
         "    shape[-1] = self.num_classes\n",
         "    return tf.TensorShape(shape)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ShDD4fv72KGc"
       },
       "source": [
@@ -701,11 +647,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "42C-qQHm0E2l",
-        "colab": {}
+        "id": "42C-qQHm0E2l"
       },
+      "outputs": [],
       "source": [
         "model = MyModel(num_classes=10)\n",
         "\n",
@@ -716,14 +662,11 @@
         "\n",
         "# Trains for 5 epochs.\n",
         "model.fit(data, labels, batch_size=32, epochs=5)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yqRQiKj20E2o"
       },
       "source": [
@@ -746,11 +689,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "l7BFnIHr2WNc",
-        "colab": {}
+        "id": "l7BFnIHr2WNc"
       },
+      "outputs": [],
       "source": [
         "class MyLayer(layers.Layer):\n",
         "\n",
@@ -784,14 +727,11 @@
         "  @classmethod\n",
         "  def from_config(cls, config):\n",
         "    return cls(**config)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8wXDRgXV2ZrF"
       },
       "source": [
@@ -800,11 +740,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "uqH-cY0h0E2p",
-        "colab": {}
+        "id": "uqH-cY0h0E2p"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Sequential([\n",
         "    MyLayer(10),\n",
@@ -817,14 +757,11 @@
         "\n",
         "# Trains for 5 epochs.\n",
         "model.fit(data, labels, batch_size=32, epochs=5)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Lu8cc3AJ0E2v"
       },
       "source": [
@@ -848,11 +785,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "rdYwzSYV0E2v",
-        "colab": {}
+        "id": "rdYwzSYV0E2v"
       },
+      "outputs": [],
       "source": [
         "callbacks = [\n",
         "  # Interrupt training if `val_loss` stops improving for over 2 epochs\n",
@@ -862,14 +799,11 @@
         "]\n",
         "model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks,\n",
         "          validation_data=(val_data, val_labels))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ghhaGfX62abv"
       },
       "source": [
@@ -880,7 +814,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qnl7K-aI0E2z"
       },
       "source": [
@@ -892,11 +825,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "uQIANjB94fLB",
-        "colab": {}
+        "id": "uQIANjB94fLB"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Sequential([\n",
         "layers.Dense(64, activation='relu', input_shape=(32,)),\n",
@@ -905,17 +838,15 @@
         "model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n",
         "              loss='categorical_crossentropy',\n",
         "              metrics=['accuracy'])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "4eoHJ-ny0E21",
-        "colab": {}
+        "id": "4eoHJ-ny0E21"
       },
+      "outputs": [],
       "source": [
         "# Save weights to a TensorFlow Checkpoint file\n",
         "model.save_weights('./weights/my_model')\n",
@@ -923,14 +854,11 @@
         "# Restore the model's state,\n",
         "# this requires a model with the same architecture.\n",
         "model.load_weights('./weights/my_model')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "u25Id3xe0E25"
       },
       "source": [
@@ -942,25 +870,22 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "JSAYoFEd0E26",
-        "colab": {}
+        "id": "JSAYoFEd0E26"
       },
+      "outputs": [],
       "source": [
         "# Save weights to a HDF5 file\n",
         "model.save_weights('my_model.h5', save_format='h5')\n",
         "\n",
         "# Restore the model's state\n",
         "model.load_weights('my_model.h5')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "mje_yKL10E29"
       },
       "source": [
@@ -968,44 +893,39 @@
         "\n",
         "A model's configuration can be saved—this serializes the model architecture\n",
         "without any weights. A saved configuration can recreate and initialize the same\n",
-        "model, even without the code that defined the original model. Keras supports\n",
-        "JSON and YAML serialization formats:"
+        "model, even without the code that defined the original model. Keras supports the\n",
+        "JSON serialization format:"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "EbET0oJTzGkq",
-        "colab": {}
+        "id": "EbET0oJTzGkq"
       },
+      "outputs": [],
       "source": [
         "# Serialize a model to JSON format\n",
         "json_string = model.to_json()\n",
         "json_string"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "pX_badhH3yWV",
-        "colab": {}
+        "id": "pX_badhH3yWV"
       },
+      "outputs": [],
       "source": [
         "import json\n",
         "import pprint\n",
         "pprint.pprint(json.loads(json_string))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Q7CIa05r4yTb"
       },
       "source": [
@@ -1014,68 +934,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "J9UFv9k00E2_",
-        "colab": {}
+        "id": "J9UFv9k00E2_"
       },
+      "outputs": [],
       "source": [
         "fresh_model = tf.keras.models.model_from_json(json_string)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "t5NHtICh4uHK"
-      },
-      "source": [
-        "Serializing a model to YAML format requires that you install `pyyaml` *before you import TensorFlow*:"
       ]
     },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "aj24KB3Z36S4",
-        "colab": {}
-      },
-      "source": [
-        "yaml_string = model.to_yaml()\n",
-        "print(yaml_string)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
-        "id": "O53Kerfl43v7"
-      },
-      "source": [
-        "Recreate the model from the YAML:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "77yRuwg03_MG",
-        "colab": {}
-      },
-      "source": [
-        "fresh_model = tf.keras.models.model_from_yaml(yaml_string)"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
         "id": "xPvOSSzM0E3B"
       },
       "source": [
@@ -1086,11 +956,9 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "iu8qMwld4-71"
       },
       "source": [
-        "\n",
         "### Entire model\n",
         "\n",
         "The entire model can be saved to a file that contains the weight values, the\n",
@@ -1101,11 +969,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "45oNY34Z0E3C",
-        "colab": {}
+        "id": "45oNY34Z0E3C"
       },
+      "outputs": [],
       "source": [
         "# Create a trivial model\n",
         "model = tf.keras.Sequential([\n",
@@ -1123,14 +991,11 @@
         "\n",
         "# Recreate the exact same model, including weights and optimizer.\n",
         "model = tf.keras.models.load_model('my_model.h5')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "PMOWhDOB0E3E"
       },
       "source": [
@@ -1155,7 +1020,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2wG3NVco5B5V"
       },
       "source": [
@@ -1175,11 +1039,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "cVg0vfTO0E3E",
-        "colab": {}
+        "id": "cVg0vfTO0E3E"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Sequential([layers.Dense(64, activation='relu', input_shape=(32,)),\n",
         "                          layers.Dense(10,activation='softmax')])\n",
@@ -1189,14 +1053,11 @@
         "              metrics=['accuracy'])\n",
         "\n",
         "estimator = tf.keras.estimator.model_to_estimator(model)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "S7FKvikO0E3H"
       },
       "source": [
@@ -1208,17 +1069,16 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6PJZ6e9J5JHF"
       },
       "source": [
         "### Multiple GPUs\n",
         "\n",
         "`tf.keras` models can run on multiple GPUs using\n",
-        "`tf.contrib.distribute.DistributionStrategy`. This API provides distributed\n",
+        "`tf.distribute.DistributionStrategy`. This API provides distributed\n",
         "training on multiple GPUs with almost no changes to existing code.\n",
         "\n",
-        "Currently, `tf.contrib.distribute.MirroredStrategy` is the only supported\n",
+        "Currently, `tf.distribute.MirroredStrategy` is the only supported\n",
         "distribution strategy. `MirroredStrategy` does in-graph replication with\n",
         "synchronous training using all-reduce on a single machine. To use\n",
         "`DistributionStrategy` with Keras, convert the `tf.keras.Model` to a\n",
@@ -1233,11 +1093,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "sbaRr7g-0E3I",
-        "colab": {}
+        "id": "sbaRr7g-0E3I"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Sequential()\n",
         "model.add(layers.Dense(16, activation='relu', input_shape=(10,)))\n",
@@ -1247,14 +1107,11 @@
         "\n",
         "model.compile(loss='binary_crossentropy', optimizer=optimizer)\n",
         "model.summary()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yw4hSJme0E3L"
       },
       "source": [
@@ -1265,11 +1122,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "CxJW1QMH0E3L",
-        "colab": {}
+        "id": "CxJW1QMH0E3L"
       },
+      "outputs": [],
       "source": [
         "def input_fn():\n",
         "  x = np.random.random((1024, 10))\n",
@@ -1279,41 +1136,35 @@
         "  dataset = dataset.repeat(10)\n",
         "  dataset = dataset.batch(32)\n",
         "  return dataset"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "rO9MiL6X0E3O"
       },
       "source": [
         "Next, create a `tf.estimator.RunConfig` and set the `train_distribute` argument\n",
-        "to the `tf.contrib.distribute.MirroredStrategy` instance. When creating\n",
+        "to the `tf.distribute.MirroredStrategy` instance. When creating\n",
         "`MirroredStrategy`, you can specify a list of devices or set the `num_gpus`\n",
         "argument. The default uses all available GPUs, like the following:"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "BEwFq4PM0E3P",
-        "colab": {}
+        "id": "BEwFq4PM0E3P"
       },
+      "outputs": [],
       "source": [
-        "strategy = tf.contrib.distribute.MirroredStrategy()\n",
+        "strategy = tf.distribute.MirroredStrategy()\n",
         "config = tf.estimator.RunConfig(train_distribute=strategy)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "TcnwYVun0E3R"
       },
       "source": [
@@ -1322,24 +1173,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "VSvbuIID0E3S",
-        "colab": {}
+        "id": "VSvbuIID0E3S"
       },
+      "outputs": [],
       "source": [
         "keras_estimator = tf.keras.estimator.model_to_estimator(\n",
         "  keras_model=model,\n",
         "  config=config,\n",
         "  model_dir='/tmp/model_dir')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "N6BXU5F90E3U"
       },
       "source": [
@@ -1349,16 +1197,27 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "XKoJ2wUH0E3U",
-        "colab": {}
+        "id": "XKoJ2wUH0E3U"
       },
+      "outputs": [],
       "source": [
         "keras_estimator.train(input_fn=input_fn, steps=10)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "keras.ipynb",
+            "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/guide/performance/benchmarks.md b/site/en/r1/guide/performance/benchmarks.md
index 8998c0723db..a56959ea416 100644
--- a/site/en/r1/guide/performance/benchmarks.md
+++ b/site/en/r1/guide/performance/benchmarks.md
@@ -401,7 +401,7 @@ GPUs | InceptionV3 (batch size 32) | ResNet-50 (batch size 32)
 ## Methodology
 
 This
-[script](https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks)
+[script](https://github.com/tensorflow/benchmarks/tree/r1.15/scripts/tf_cnn_benchmarks)
 was run on the various platforms to generate the above results.
 
 In order to create results that are as repeatable as possible, each test was run
diff --git a/site/en/r1/guide/performance/overview.md b/site/en/r1/guide/performance/overview.md
index 117ffe1d1d6..be7217f4b99 100644
--- a/site/en/r1/guide/performance/overview.md
+++ b/site/en/r1/guide/performance/overview.md
@@ -19,9 +19,9 @@ Reading large numbers of small files significantly impacts I/O performance.
 One approach to get maximum I/O throughput is to preprocess input data into
 larger (~100MB) `TFRecord` files. For smaller data sets (200MB-1GB), the best
 approach is often to load the entire data set into memory. The document
-[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/research/slim#downloading-and-converting-to-tfrecord-format)
+[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/r1.15/research/slim#downloading-and-converting-to-tfrecord-format)
 includes information and scripts for creating `TFRecord`s, and this
-[script](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator/generate_cifar10_tfrecords.py)
+[script](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10_estimator/generate_cifar10_tfrecords.py)
 converts the CIFAR-10 dataset into `TFRecord`s.
 
 While feeding data using a `feed_dict` offers a high level of flexibility, in
@@ -122,7 +122,7 @@ tf.Session(config=config)
 Intel® has added optimizations to TensorFlow for Intel® Xeon® and Intel® Xeon
 Phi™ through the use of the Intel® Math Kernel Library for Deep Neural Networks
 (Intel® MKL-DNN) optimized primitives. The optimizations also provide speedups
-for the consumer line of processors, e.g. i5 and i7 Intel processors. The Intel
+for the consumer line of processors, e.g., i5 and i7 Intel processors. The Intel
 published paper
 [TensorFlow* Optimizations on Modern Intel® Architecture](https://software.intel.com/en-us/articles/tensorflow-optimizations-on-modern-intel-architecture)
 contains additional details on the implementation.
@@ -136,7 +136,7 @@ and AVX2. The result is a single binary that is optimized and compatible with
 most modern (post-2011) processors.
 
 TensorFlow can be compiled with the MKL optimizations using the following
-commands that depending on the version of the TensorFlow source used.
+commands that depend on the version of the TensorFlow source used.
 
 For TensorFlow source versions after 1.3.0:
 
@@ -255,10 +255,10 @@ bazel build -c opt --copt=-march="broadwell" --config=cuda //tensorflow/tools/pi
   a docker container, the data is not cached and the penalty is paid each time
   TensorFlow starts. The best practice is to include the
   [compute capabilities](http://developer.nvidia.com/cuda-gpus)
-  of the GPUs that will be used, e.g. P100: 6.0, Titan X (Pascal): 6.1,
+  of the GPUs that will be used, e.g., P100: 6.0, Titan X (Pascal): 6.1,
   Titan X (Maxwell): 5.2, and K80: 3.7.
 * Use a version of `gcc` that supports all of the optimizations of the target
-  CPU. The recommended minimum gcc version is 4.8.3. On OS X, upgrade to the
+  CPU. The recommended minimum gcc version is 4.8.3. On macOS, upgrade to the
   latest Xcode version and use the version of clang that comes with Xcode.
 * Install the latest stable CUDA platform and cuDNN libraries supported by
   TensorFlow.
diff --git a/site/en/r1/guide/premade_estimators.md b/site/en/r1/guide/premade_estimators.md
index dd09d6be087..b17ad623437 100644
--- a/site/en/r1/guide/premade_estimators.md
+++ b/site/en/r1/guide/premade_estimators.md
@@ -9,7 +9,7 @@ Prior to using the sample code in this document, you'll need to do the
 following:
 
 * [Install TensorFlow](../install).
-* If you installed TensorFlow with virtualenv or Anaconda, activate your
+* If you installed TensorFlow with venv or Anaconda, activate your
   TensorFlow environment.
 * Install or upgrade pandas by issuing the following command:
 
@@ -30,9 +30,9 @@ Take the following steps to get the sample code we'll be going through:
         cd models/samples/core/get_started/
 
 The program described in this document is
-[`premade_estimator.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/premade_estimator.py).
+[`premade_estimator.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/premade_estimator.py).
 This program uses
-[`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py)
+[`iris_data.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/iris_data.py)
 to fetch its training data.
 
 ### Running the program
@@ -253,7 +253,7 @@ To keep things simple in this example we are going to load the data with
 in-memory data.
 
 Here is the input function used for training in this program, which is available
-in [`iris_data.py`](https://github.com/tensorflow/models/blob/master/samples/core/get_started/iris_data.py):
+in [`iris_data.py`](https://github.com/tensorflow/models/blob/r1.13.0/samples/core/get_started/iris_data.py):
 
 ``` python
 def train_input_fn(features, labels, batch_size):
diff --git a/site/en/r1/guide/ragged_tensors.ipynb b/site/en/r1/guide/ragged_tensors.ipynb
index 48d42a6f2b1..289d29ce82e 100644
--- a/site/en/r1/guide/ragged_tensors.ipynb
+++ b/site/en/r1/guide/ragged_tensors.ipynb
@@ -1,25 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "ragged_tensors.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "nibpbUnTsxTd"
       },
       "source": [
@@ -28,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "tXAbWHtqs1Y2",
-        "colab": {},
-        "cellView": "form"
+        "cellView": "form",
+        "id": "tXAbWHtqs1Y2"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -46,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "HTgMAvQq-PU_"
       },
       "source": [
@@ -72,7 +52,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "HTgMAvQq-PU_"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "cDIUjj07-rQg"
       },
       "source": [
@@ -81,25 +74,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "KKvdSorS-pDD",
-        "colab": {}
+        "id": "KKvdSorS-pDD"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
         "import math\n",
-        "import tensorflow as tf\n",
-        "tf.enable_eager_execution()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "import tensorflow.compat.v1 as tf"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "pxi0m_yf-te5"
       },
       "source": [
@@ -127,11 +114,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "vGmJGSf_-PVB",
-        "colab": {}
+        "id": "vGmJGSf_-PVB"
       },
+      "outputs": [],
       "source": [
         "digits = tf.ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])\n",
         "words = tf.ragged.constant([[\"So\", \"long\"], [\"thanks\", \"for\", \"all\", \"the\", \"fish\"]])\n",
@@ -140,14 +127,11 @@
         "print(tf.concat([digits, [[5, 3]]], axis=0))\n",
         "print(tf.tile(digits, [1, 2]))\n",
         "print(tf.strings.substr(words, 0, 2))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Pt-5OIc8-PVG"
       },
       "source": [
@@ -164,47 +148,40 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "n8YMKXpI-PVH",
-        "colab": {}
+        "id": "n8YMKXpI-PVH"
       },
+      "outputs": [],
       "source": [
         "print(digits[0])       # First row"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Awi8i9q5_DuX",
-        "colab": {}
+        "id": "Awi8i9q5_DuX"
       },
+      "outputs": [],
       "source": [
         "print(digits[:, :2])   # First two values in each row."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "sXgQtTcgHHMR",
-        "colab": {}
+        "id": "sXgQtTcgHHMR"
       },
+      "outputs": [],
       "source": [
         "print(digits[:, -2:])  # Last two values in each row."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6FU5T_-8-PVK"
       },
       "source": [
@@ -215,34 +192,29 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "2tdUEtb7-PVL",
-        "colab": {}
+        "id": "2tdUEtb7-PVL"
       },
+      "outputs": [],
       "source": [
         "print(digits + 3)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "X-bxG0nc_Nmf",
-        "colab": {}
+        "id": "X-bxG0nc_Nmf"
       },
+      "outputs": [],
       "source": [
         "print(digits + tf.ragged.constant([[1, 2, 3, 4], [], [5, 6, 7], [8], []]))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2tsw8mN0ESIT"
       },
       "source": [
@@ -251,22 +223,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "pvt5URbdEt-D",
-        "colab": {}
+        "id": "pvt5URbdEt-D"
       },
+      "outputs": [],
       "source": [
         "times_two_plus_one = lambda x: x * 2 + 1\n",
         "print(tf.ragged.map_flat_values(times_two_plus_one, digits))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "7M5RHOgp-PVN"
       },
       "source": [
@@ -279,41 +248,36 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "yhgKMozw-PVP",
-        "colab": {}
+        "id": "yhgKMozw-PVP"
       },
+      "outputs": [],
       "source": [
         "sentences = tf.ragged.constant([\n",
         "    [\"Let's\", \"build\", \"some\", \"ragged\", \"tensors\", \"!\"],\n",
         "    [\"We\", \"can\", \"use\", \"tf.ragged.constant\", \".\"]])\n",
         "print(sentences)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "TW1g7eE2ee8M",
-        "colab": {}
+        "id": "TW1g7eE2ee8M"
       },
+      "outputs": [],
       "source": [
         "paragraphs = tf.ragged.constant([\n",
         "    [['I', 'have', 'a', 'cat'], ['His', 'name', 'is', 'Mat']],\n",
         "    [['Do', 'you', 'want', 'to', 'come', 'visit'], [\"I'm\", 'free', 'tomorrow']],\n",
         "])\n",
         "print(paragraphs)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "SPLn5xHn-PVR"
       },
       "source": [
@@ -331,23 +295,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "SEvcPUcl-PVS",
-        "colab": {}
+        "id": "SEvcPUcl-PVS"
       },
+      "outputs": [],
       "source": [
         "print(tf.RaggedTensor.from_value_rowids(\n",
         "    values=[3, 1, 4, 1, 5, 9, 2, 6],\n",
         "    value_rowids=[0, 0, 0, 0, 2, 2, 2, 3]))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "RBQh8sYc-PVV"
       },
       "source": [
@@ -360,23 +321,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "LBY81WXl-PVW",
-        "colab": {}
+        "id": "LBY81WXl-PVW"
       },
+      "outputs": [],
       "source": [
         "print(tf.RaggedTensor.from_row_lengths(\n",
         "    values=[3, 1, 4, 1, 5, 9, 2, 6],\n",
         "    row_lengths=[4, 0, 3, 1]))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8p5V8_Iu-PVa"
       },
       "source": [
@@ -389,23 +347,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "FwizuqZI-PVb",
-        "colab": {}
+        "id": "FwizuqZI-PVb"
       },
+      "outputs": [],
       "source": [
         "print(tf.RaggedTensor.from_row_splits(\n",
         "    values=[3, 1, 4, 1, 5, 9, 2, 6],\n",
         "    row_splits=[0, 4, 4, 7, 8]))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "E-9imo8DhwuA"
       },
       "source": [
@@ -415,7 +370,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "YQAOsT1_-PVg"
       },
       "source": [
@@ -428,66 +382,57 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "SqbPBd_w-PVi",
-        "colab": {}
+        "id": "SqbPBd_w-PVi"
       },
+      "outputs": [],
       "source": [
         "print(tf.ragged.constant([[\"Hi\"], [\"How\", \"are\", \"you\"]]))  # ok: type=string, rank=2"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "83ZCSJnQAWAf",
-        "colab": {}
+        "id": "83ZCSJnQAWAf"
       },
+      "outputs": [],
       "source": [
         "print(tf.ragged.constant([[[1, 2], [3]], [[4, 5]]]))        # ok: type=int32, rank=3"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ewA3cISdDfmP",
-        "colab": {}
+        "id": "ewA3cISdDfmP"
       },
+      "outputs": [],
       "source": [
         "try:\n",
         "  tf.ragged.constant([[\"one\", \"two\"], [3, 4]])              # bad: multiple types\n",
         "except ValueError as exception:\n",
         "  print(exception)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "EOWIlVidDl-n",
-        "colab": {}
+        "id": "EOWIlVidDl-n"
       },
+      "outputs": [],
       "source": [
         "try:\n",
         "  tf.ragged.constant([\"A\", [\"B\", \"C\"]])                     # bad: multiple nesting depths\n",
         "except ValueError as exception:\n",
         "  print(exception)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "nhHMFhSp-PVq"
       },
       "source": [
@@ -501,11 +446,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ZBs_V7e--PVr",
-        "colab": {}
+        "id": "ZBs_V7e--PVr"
       },
+      "outputs": [],
       "source": [
         "queries = tf.ragged.constant([['Who', 'is', 'Dan', 'Smith'],\n",
         "                              ['Pause'],\n",
@@ -538,14 +483,11 @@
         "all_embeddings = tf.concat([word_embeddings, bigram_embeddings], axis=1)    # ⑤\n",
         "avg_embedding = tf.reduce_mean(all_embeddings, axis=1)                      # ⑥\n",
         "print(avg_embedding)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Y_lE_LAVcWQH"
       },
       "source": [
@@ -555,7 +497,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "An_k0pX1-PVt"
       },
       "source": [
@@ -619,21 +560,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "M2Wzx4JEIvmb",
-        "colab": {}
+        "id": "M2Wzx4JEIvmb"
       },
+      "outputs": [],
       "source": [
         "tf.ragged.constant([[\"Hi\"], [\"How\", \"are\", \"you\"]]).shape"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "G9tfJOeFlijE"
       },
       "source": [
@@ -643,21 +581,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "5DHaqXHxlWi0",
-        "colab": {}
+        "id": "5DHaqXHxlWi0"
       },
+      "outputs": [],
       "source": [
         "print(tf.ragged.constant([[\"Hi\"], [\"How\", \"are\", \"you\"]]).bounding_shape())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "V8e7x95UcLS6"
       },
       "source": [
@@ -675,27 +610,23 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ush7IGUWLXIn",
-        "colab": {}
+        "id": "ush7IGUWLXIn"
       },
+      "outputs": [],
       "source": [
         "ragged_x = tf.ragged.constant([[\"John\"], [\"a\", \"big\", \"dog\"], [\"my\", \"cat\"]])\n",
         "ragged_y = tf.ragged.constant([[\"fell\", \"asleep\"], [\"barked\"], [\"is\", \"fuzzy\"]])\n",
         "print(tf.concat([ragged_x, ragged_y], axis=1))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "pvQzZG8zMoWa"
       },
       "source": [
-        "\n",
         "But concatenating sparse tensors is equivalent to concatenating the corresponding dense tensors,\n",
         "as illustrated by the following example (where Ø indicates missing values):\n",
         "\n",
@@ -704,24 +635,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "eTIhGayQL0gI",
-        "colab": {}
+        "id": "eTIhGayQL0gI"
       },
+      "outputs": [],
       "source": [
         "sparse_x = ragged_x.to_sparse()\n",
         "sparse_y = ragged_y.to_sparse()\n",
         "sparse_result = tf.sparse.concat(sp_inputs=[sparse_x, sparse_y], axis=1)\n",
         "print(tf.sparse.to_dense(sparse_result, ''))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Vl8eQN8pMuYx"
       },
       "source": [
@@ -737,7 +665,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "cRcHzS6pcHYC"
       },
       "source": [
@@ -749,23 +676,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "skScd37P-PVu",
-        "colab": {}
+        "id": "skScd37P-PVu"
       },
+      "outputs": [],
       "source": [
         "x = tf.ragged.constant([[1, 2], [3], [4, 5, 6]])\n",
         "y = tf.ragged.constant([[1, 1], [2], [3, 3, 3]])\n",
         "print(x + y)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "XEGgbZHV-PVw"
       },
       "source": [
@@ -777,22 +701,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "IYybEEWc-PVx",
-        "colab": {}
+        "id": "IYybEEWc-PVx"
       },
+      "outputs": [],
       "source": [
         "x = tf.ragged.constant([[1, 2], [3], [4, 5, 6]])\n",
         "print(x + 3)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "okGb9dIi-PVz"
       },
       "source": [
@@ -809,7 +730,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "f2anbs6ZnFtl"
       },
       "source": [
@@ -824,77 +744,66 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "MbSRZRDz-PV1",
-        "colab": {}
+        "id": "MbSRZRDz-PV1"
       },
+      "outputs": [],
       "source": [
         "queries = tf.ragged.constant(\n",
         "    [['Who', 'is', 'George', 'Washington'],\n",
         "     ['What', 'is', 'the', 'weather', 'tomorrow'],\n",
         "     ['Goodnight']])\n",
         "print(queries[1])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "EFfjZV7YA3UH",
-        "colab": {}
+        "id": "EFfjZV7YA3UH"
       },
+      "outputs": [],
       "source": [
         "print(queries[1, 2])                # A single word"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "VISRPQSdA3xn",
-        "colab": {}
+        "id": "VISRPQSdA3xn"
       },
+      "outputs": [],
       "source": [
         "print(queries[1:])                  # Everything but the first row"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "J1PpSyKQBMng",
-        "colab": {}
+        "id": "J1PpSyKQBMng"
       },
+      "outputs": [],
       "source": [
         "print(queries[:, :3])               # The first 3 words of each query"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ixrhHmJBeidy",
-        "colab": {}
+        "id": "ixrhHmJBeidy"
       },
+      "outputs": [],
       "source": [
         "print(queries[:, -2:])              # The last 2 words of each query"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "cnOP6Vza-PV4"
       },
       "source": [
@@ -903,76 +812,65 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "8VbqbKcE-PV6",
-        "colab": {}
+        "id": "8VbqbKcE-PV6"
       },
+      "outputs": [],
       "source": [
         "rt = tf.ragged.constant([[[1, 2, 3], [4]],\n",
         "                         [[5], [], [6]],\n",
         "                         [[7]],\n",
         "                         [[8, 9], [10]]])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "f9WPVWf4grVp",
-        "colab": {}
+        "id": "f9WPVWf4grVp"
       },
+      "outputs": [],
       "source": [
         "print(rt[1])                        # Second row (2-D RaggedTensor)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ad8FGJoABjQH",
-        "colab": {}
+        "id": "ad8FGJoABjQH"
       },
+      "outputs": [],
       "source": [
         "print(rt[3, 0])                     # First element of fourth row (1-D Tensor)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "MPPr-a-bBjFE",
-        "colab": {}
+        "id": "MPPr-a-bBjFE"
       },
+      "outputs": [],
       "source": [
         "print(rt[:, 1:3])                   # Items 1-3 of each row (3-D RaggedTensor)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "6SIDeoIUBi4z",
-        "colab": {}
+        "id": "6SIDeoIUBi4z"
       },
+      "outputs": [],
       "source": [
         "print(rt[:, -1:])                   # Last item of each row (3-D RaggedTensor)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "_d3nBh1GnWvU"
       },
       "source": [
@@ -991,7 +889,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IsWKETULAJbN"
       },
       "source": [
@@ -1003,66 +900,57 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "INnfmZGcBoU_",
-        "colab": {}
+        "id": "INnfmZGcBoU_"
       },
+      "outputs": [],
       "source": [
         "ragged_sentences = tf.ragged.constant([\n",
         "    ['Hi'], ['Welcome', 'to', 'the', 'fair'], ['Have', 'fun']])\n",
         "print(ragged_sentences.to_tensor(default_value=''))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "41WAZLXNnbwH",
-        "colab": {}
+        "id": "41WAZLXNnbwH"
       },
+      "outputs": [],
       "source": [
         "print(ragged_sentences.to_sparse())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "-rfiyYqne8QN",
-        "colab": {}
+        "id": "-rfiyYqne8QN"
       },
+      "outputs": [],
       "source": [
         "x = [[1, 3, -1, -1], [2, -1, -1, -1], [4, 5, 8, 9]]\n",
         "print(tf.RaggedTensor.from_tensor(x, padding=-1))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "S8MkYo2hfVhj",
-        "colab": {}
+        "id": "S8MkYo2hfVhj"
       },
+      "outputs": [],
       "source": [
         "st = tf.SparseTensor(indices=[[0, 0], [2, 0], [2, 1]],\n",
         "                     values=['a', 'b', 'c'],\n",
         "                     dense_shape=[3, 3])\n",
         "print(tf.RaggedTensor.from_sparse(st))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qx025sNMkAHH"
       },
       "source": [
@@ -1080,22 +968,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "uMm1WMkc-PV_",
-        "colab": {}
+        "id": "uMm1WMkc-PV_"
       },
+      "outputs": [],
       "source": [
         "rt = tf.ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]])\n",
         "print(rt.to_list())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "SrizmqTc-PWC"
       },
       "source": [
@@ -1106,21 +991,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "HpRHhfLe-PWD",
-        "colab": {}
+        "id": "HpRHhfLe-PWD"
       },
+      "outputs": [],
       "source": [
         "print(rt[1].numpy())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "sNlpI2fR-PWF"
       },
       "source": [
@@ -1128,40 +1010,35 @@
         "    `tf.RaggedTensor.values`\n",
         "    and\n",
         "    `tf.RaggedTensor.row_splits`\n",
-        "    properties, or row-paritioning methods such as `tf.RaggedTensor.row_lengths()`\n",
+        "    properties, or row-partitioning methods such as `tf.RaggedTensor.row_lengths()`\n",
         "    and `tf.RaggedTensor.value_rowids()`."
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "yTckrLdB-PWG",
-        "colab": {}
+        "id": "yTckrLdB-PWG"
       },
+      "outputs": [],
       "source": [
         "print(rt.values)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "B8OnG9NzCEnv",
-        "colab": {}
+        "id": "B8OnG9NzCEnv"
       },
+      "outputs": [],
       "source": [
         "print(rt.row_splits)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6tG3kBAo-PWI"
       },
       "source": [
@@ -1173,23 +1050,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "aDhVIrIs-PWJ",
-        "colab": {}
+        "id": "aDhVIrIs-PWJ"
       },
+      "outputs": [],
       "source": [
         "with tf.Session() as session:\n",
         "  rt = tf.ragged.constant([[1, 2], [3, 4, 5], [6], [], [7]])\n",
         "  rt_value = session.run(rt)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "0-K5pqwJ-PWL"
       },
       "source": [
@@ -1204,21 +1078,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "R2U3WZf8-PWM",
-        "colab": {}
+        "id": "R2U3WZf8-PWM"
       },
+      "outputs": [],
       "source": [
         "print(rt_value.to_list())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "4x4b7DpY-PWO"
       },
       "source": [
@@ -1231,47 +1102,40 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "RtREVSPB-PWO",
-        "colab": {}
+        "id": "RtREVSPB-PWO"
       },
+      "outputs": [],
       "source": [
         "print(rt_value.values)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "9BIpKNBnCmjV",
-        "colab": {}
+        "id": "9BIpKNBnCmjV"
       },
+      "outputs": [],
       "source": [
         "print(rt_value.row_splits)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "qEmnOr01Cdl3",
-        "colab": {}
+        "id": "qEmnOr01Cdl3"
       },
+      "outputs": [],
       "source": [
         "tf.enable_eager_execution()  # Resume eager execution mode."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "EdljbNPq-PWS"
       },
       "source": [
@@ -1303,7 +1167,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-S2hOUWx-PWU"
       },
       "source": [
@@ -1316,11 +1179,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "0n095XdR-PWU",
-        "colab": {}
+        "id": "0n095XdR-PWU"
       },
+      "outputs": [],
       "source": [
         "# x       (2D ragged):  2 x (num_rows)\n",
         "# y       (scalar)\n",
@@ -1328,17 +1191,15 @@
         "x = tf.ragged.constant([[1, 2], [3]])\n",
         "y = 3\n",
         "print(x + y)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "0SVYk5AP-PWW",
-        "colab": {}
+        "id": "0SVYk5AP-PWW"
       },
+      "outputs": [],
       "source": [
         "# x         (2d ragged):  3 x (num_rows)\n",
         "# y         (2d tensor):  3 x          1\n",
@@ -1349,17 +1210,15 @@
         "    [12, 32]])\n",
         "y = [[1000], [2000], [3000]]\n",
         "print(x + y)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "MsfBMD80s8Ux",
-        "colab": {}
+        "id": "MsfBMD80s8Ux"
       },
+      "outputs": [],
       "source": [
         "# x      (3d ragged):  2 x (r1) x 2\n",
         "# y      (2d ragged):         1 x 1\n",
@@ -1370,17 +1229,15 @@
         "    ragged_rank=1)\n",
         "y = tf.constant([[10]])\n",
         "print(x + y)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "rEj5QVfnva0t",
-        "colab": {}
+        "id": "rEj5QVfnva0t"
       },
+      "outputs": [],
       "source": [
         "# x      (3d ragged):  2 x (r1) x (r2) x 1\n",
         "# y      (1d tensor):                    3\n",
@@ -1401,14 +1258,11 @@
         "    ragged_rank=2)\n",
         "y = tf.constant([10, 20, 30])\n",
         "print(x + y)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "uennZ64Aqftb"
       },
       "source": [
@@ -1417,11 +1271,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "UpI0FlfL4Eim",
-        "colab": {}
+        "id": "UpI0FlfL4Eim"
       },
+      "outputs": [],
       "source": [
         "# x      (2d ragged): 3 x (r1)\n",
         "# y      (2d tensor): 3 x    4  # trailing dimensions do not match\n",
@@ -1431,17 +1285,15 @@
         "  x + y\n",
         "except tf.errors.InvalidArgumentError as exception:\n",
         "  print(exception)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "qGq1zOT4zMoc",
-        "colab": {}
+        "id": "qGq1zOT4zMoc"
       },
+      "outputs": [],
       "source": [
         "# x      (2d ragged): 3 x (r1)\n",
         "# y      (2d ragged): 3 x (r2)  # ragged dimensions do not match.\n",
@@ -1451,17 +1303,15 @@
         "  x + y\n",
         "except tf.errors.InvalidArgumentError as exception:\n",
         "  print(exception)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "CvLae5vMqeji",
-        "colab": {}
+        "id": "CvLae5vMqeji"
       },
+      "outputs": [],
       "source": [
         "# x      (3d ragged): 3 x (r1) x 2\n",
         "# y      (3d ragged): 3 x (r1) x 3  # trailing dimensions do not match\n",
@@ -1473,14 +1323,11 @@
         "  x + y\n",
         "except tf.errors.InvalidArgumentError as exception:\n",
         "  print(exception)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "m0wQkLfV-PWa"
       },
       "source": [
@@ -1495,30 +1342,26 @@
         "    divided into rows. In particular, the values for row `rt[i]` are stored in\n",
         "    the slice `rt.values[rt.row_splits[i]:rt.row_splits[i+1]]`.\n",
         "\n",
-        "![ragged_encoding](https://www.tensorflow.org/images/ragged_tensors/ragged_encoding.png)\n",
-        "\n"
+        "![ragged_encoding](https://www.tensorflow.org/images/ragged_tensors/ragged_encoding.png)\n"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "MrLgMu0gPuo-",
-        "colab": {}
+        "id": "MrLgMu0gPuo-"
       },
+      "outputs": [],
       "source": [
         "rt = tf.RaggedTensor.from_row_splits(\n",
         "    values=[3, 1, 4, 1, 5, 9, 2],\n",
         "    row_splits=[0, 4, 4, 6, 7])\n",
         "print(rt)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "bpB7xKoUPtU6"
       },
       "source": [
@@ -1533,11 +1376,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "yy3IGT2a-PWb",
-        "colab": {}
+        "id": "yy3IGT2a-PWb"
       },
+      "outputs": [],
       "source": [
         "rt = tf.RaggedTensor.from_row_splits(\n",
         "    values=tf.RaggedTensor.from_row_splits(\n",
@@ -1547,14 +1390,11 @@
         "print(rt)\n",
         "print(\"Shape: {}\".format(rt.shape))\n",
         "print(\"Number of ragged dimensions: {}\".format(rt.ragged_rank))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5HqEEDzk-PWc"
       },
       "source": [
@@ -1565,24 +1405,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "AKYhtFcT-PWd",
-        "colab": {}
+        "id": "AKYhtFcT-PWd"
       },
+      "outputs": [],
       "source": [
         "rt = tf.RaggedTensor.from_nested_row_splits(\n",
         "    flat_values=[10, 11, 12, 13, 14, 15, 16, 17, 18, 19],\n",
         "    nested_row_splits=([0, 1, 1, 5], [0, 3, 3, 5, 9, 10]))\n",
         "print(rt)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "uba2EnAY-PWf"
       },
       "source": [
@@ -1596,11 +1433,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "z2sHwHdy-PWg",
-        "colab": {}
+        "id": "z2sHwHdy-PWg"
       },
+      "outputs": [],
       "source": [
         "rt = tf.RaggedTensor.from_row_splits(\n",
         "    values=[[1, 3], [0, 0], [1, 3], [5, 3], [3, 3], [1, 2]],\n",
@@ -1608,14 +1445,11 @@
         "print(rt)\n",
         "print(\"Shape: {}\".format(rt.shape))\n",
         "print(\"Number of ragged dimensions: {}\".format(rt.ragged_rank))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8yYaNrgX-PWh"
       },
       "source": [
@@ -1656,11 +1490,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "4TH6XoQ8-PWh",
-        "colab": {}
+        "id": "4TH6XoQ8-PWh"
       },
+      "outputs": [],
       "source": [
         "values = [3, 1, 4, 1, 5, 9, 2, 6]\n",
         "print(tf.RaggedTensor.from_row_splits(values, row_splits=[0, 4, 4, 7, 8, 8]))\n",
@@ -1669,14 +1503,11 @@
         "print(tf.RaggedTensor.from_row_limits(values, row_limits=[4, 4, 7, 8, 8]))\n",
         "print(tf.RaggedTensor.from_value_rowids(\n",
         "    values, value_rowids=[0, 0, 0, 0, 2, 2, 2, 3], nrows=5))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ZGRrpwxjsOGr"
       },
       "source": [
@@ -1686,11 +1517,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "fIdn-hUBsoSj",
-        "colab": {}
+        "id": "fIdn-hUBsoSj"
       },
+      "outputs": [],
       "source": [
         "rt = tf.ragged.constant([[3, 1, 4, 1], [], [5, 9, 2], [6], []])\n",
         "print(\"      values: {}\".format(rt.values))\n",
@@ -1699,14 +1530,11 @@
         "print(\"  row_starts: {}\".format(rt.row_starts()))\n",
         "print(\"  row_limits: {}\".format(rt.row_limits()))\n",
         "print(\"value_rowids: {}\".format(rt.value_rowids()))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2r9XUpLUsdOa"
       },
       "source": [
@@ -1716,7 +1544,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "NBX15kEr-PWi"
       },
       "source": [
@@ -1747,5 +1574,18 @@
         "    matches the format used by ops such as `tf.sequence_mask`."
       ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "ragged_tensors.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/guide/saved_model.md b/site/en/r1/guide/saved_model.md
index 60b51d9569a..34447ffe861 100644
--- a/site/en/r1/guide/saved_model.md
+++ b/site/en/r1/guide/saved_model.md
@@ -23,7 +23,7 @@ TensorFlow saves variables in binary *checkpoint files* that map variable
 names to tensor values.
 
 Caution: TensorFlow model files are code. Be careful with untrusted code.
-See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)
+See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/r1.15/SECURITY.md)
 for details.
 
 ### Save variables
@@ -148,7 +148,7 @@ Notes:
    `tf.variables_initializer` for more information.
 
 *  To inspect the variables in a checkpoint, you can use the
-   [`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py)
+   [`inspect_checkpoint`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/tools/inspect_checkpoint.py)
    library, particularly the `print_tensors_in_checkpoint_file` function.
 
 *  By default, `Saver` uses the value of the `tf.Variable.name` property
@@ -159,7 +159,7 @@ Notes:
 ### Inspect variables in a checkpoint
 
 We can quickly inspect variables in a checkpoint with the
-[`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) library.
+[`inspect_checkpoint`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/tools/inspect_checkpoint.py) library.
 
 Continuing from the save/restore examples shown earlier:
 
@@ -216,7 +216,7 @@ simple_save(session,
 
 This configures the `SavedModel` so it can be loaded by
 [TensorFlow serving](https://www.tensorflow.org/tfx/tutorials/serving/rest_simple) and supports the
-[Predict API](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/predict.proto).
+[Predict API](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis/predict.proto).
 To access the classify, regress, or multi-inference APIs, use the manual
 `SavedModel` builder APIs or an `tf.estimator.Estimator`.
 
@@ -328,7 +328,7 @@ with tf.Session(graph=tf.Graph()) as sess:
 ### Load a SavedModel in C++
 
 The C++ version of the SavedModel
-[loader](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/loader.h)
+[loader](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/cc/saved_model/loader.h)
 provides an API to load a SavedModel from a path, while allowing
 `SessionOptions` and `RunOptions`.
 You have to specify the tags associated with the graph to be loaded.
@@ -383,20 +383,20 @@ reuse and share across tools consistently.
 You may use sets of tags to uniquely identify a `MetaGraphDef` saved in a
 SavedModel. A subset of commonly used tags is specified in:
 
-* [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py)
-* [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/tag_constants.h)
+* [Python](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/saved_model/tag_constants.py)
+* [C++](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/cc/saved_model/tag_constants.h)
 
 
 #### Standard SignatureDef constants
 
-A [**SignatureDef**](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto)
+A [**SignatureDef**](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/meta_graph.proto)
 is a protocol buffer that defines the signature of a computation
 supported by a graph.
 Commonly used input keys, output keys, and method names are
 defined in:
 
-* [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py)
-* [C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/signature_constants.h)
+* [Python](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/saved_model/signature_constants.py)
+* [C++](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/cc/saved_model/signature_constants.h)
 
 ## Using SavedModel with Estimators
 
@@ -408,7 +408,7 @@ To prepare a trained Estimator for serving, you must export it in the standard
 SavedModel format. This section explains how to:
 
 * Specify the output nodes and the corresponding
-  [APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto)
+  [APIs](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis/prediction_service.proto)
   that can be served (Classify, Regress, or Predict).
 * Export your model to the SavedModel format.
 * Serve the model from a local server and request predictions.
@@ -506,7 +506,7 @@ Each `output` value must be an `ExportOutput` object  such as
 `tf.estimator.export.PredictOutput`.
 
 These output types map straightforwardly to the
-[TensorFlow Serving APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto),
+[TensorFlow Serving APIs](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis/prediction_service.proto),
 and so determine which request types will be honored.
 
 Note: In the multi-headed case, a `SignatureDef` will be generated for each
@@ -515,7 +515,7 @@ the same keys.  These `SignatureDef`s differ only in their outputs, as
 provided by the corresponding `ExportOutput` entry.  The inputs are always
 those provided by the `serving_input_receiver_fn`.
 An inference request may specify the head by name.  One head must be named
-using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://www.tensorflow.org/code/tensorflow/python/saved_model/signature_constants.py)
+using [`signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/saved_model/signature_constants.py)
 indicating which `SignatureDef` will be served when an inference request
 does not specify one.
 
@@ -566,9 +566,9 @@ Now you have a server listening for inference requests via gRPC on port 9000!
 ### Request predictions from a local server
 
 The server responds to gRPC requests according to the
-[PredictionService](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto#L15)
+[PredictionService](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis/prediction_service.proto#L15)
 gRPC API service definition.  (The nested protocol buffers are defined in
-various [neighboring files](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis)).
+various [neighboring files](https://github.com/tensorflow/serving/blob/r1.15/tensorflow_serving/apis)).
 
 From the API service definition, the gRPC framework generates client libraries
 in various languages providing remote access to the API.  In a project using the
@@ -620,7 +620,7 @@ The returned result in this example is a `ClassificationResponse` protocol
 buffer.
 
 This is a skeletal example; please see the [Tensorflow Serving](../deploy/index.md)
-documentation and [examples](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example)
+documentation and [examples](https://github.com/tensorflow/serving/tree/r1.15/tensorflow_serving/example)
 for more details.
 
 > Note: `ClassificationRequest` and `RegressionRequest` contain a
@@ -672,7 +672,7 @@ If you built TensorFlow from source code, you must run the following
 additional command to build `saved_model_cli`:
 
 ```
-$ bazel build tensorflow/python/tools:saved_model_cli
+$ bazel build third_party/tensorflow/python/tools:saved_model_cli
 ```
 
 ### Overview of commands
diff --git a/site/en/r1/guide/using_tpu.md b/site/en/r1/guide/using_tpu.md
index 74169092189..e3e338adf49 100644
--- a/site/en/r1/guide/using_tpu.md
+++ b/site/en/r1/guide/using_tpu.md
@@ -7,8 +7,8 @@ changing the *hardware accelerator* in your notebook settings:
 TPU-enabled Colab notebooks are available to test:
 
   1. [A quick test, just to measure FLOPS](https://colab.research.google.com/notebooks/tpu.ipynb).
-  2. [A CNN image classifier with `tf.keras`](https://colab.research.google.com/github/tensorflow/tpu/blob/master/tools/colab/fashion_mnist.ipynb).
-  3. [An LSTM markov chain text generator with `tf.keras`](https://colab.research.google.com/github/tensorflow/tpu/blob/master/tools/colab/shakespeare_with_tpu_and_keras.ipynb)
+  2. [A CNN image classifier with `tf.keras`](https://colab.research.google.com/github/tensorflow/tpu/blob/r1.15/tools/colab/fashion_mnist.ipynb).
+  3. [An LSTM markov chain text generator with `tf.keras`](https://colab.research.google.com/github/tensorflow/tpu/blob/r1.15/tools/colab/shakespeare_with_tpu_and_keras.ipynb)
 
 ## TPUEstimator
 
@@ -25,7 +25,7 @@ Cloud TPU is to define the model's inference phase (from inputs to predictions)
 outside of the `model_fn`. Then maintain separate implementations of the
 `Estimator` setup and `model_fn`, both wrapping this inference step. For an
 example of this pattern compare the `mnist.py` and `mnist_tpu.py` implementation in
-[tensorflow/models](https://github.com/tensorflow/models/tree/master/official/r1/mnist).
+[tensorflow/models](https://github.com/tensorflow/models/tree/r1.15/official/r1/mnist).
 
 ### Run a TPUEstimator locally
 
@@ -350,10 +350,10 @@ in bytes. A minimum of a few MB (`buffer_size=8*1024*1024`) is recommended so
 that data is available when needed.
 
 The TPU-demos repo includes
-[a script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py)
+[a script](https://github.com/tensorflow/tpu/blob/1.15/tools/datasets/imagenet_to_gcs.py)
 for downloading the imagenet dataset and converting it to an appropriate format.
 This together with the imagenet
-[models](https://github.com/tensorflow/tpu/tree/master/models)
+[models](https://github.com/tensorflow/tpu/tree/r1.15/models)
 included in the repo demonstrate all of these best-practices.
 
 ## Next steps
diff --git a/site/en/r1/guide/version_compat.md b/site/en/r1/guide/version_compat.md
index 1f26aaebd62..a765620518d 100644
--- a/site/en/r1/guide/version_compat.md
+++ b/site/en/r1/guide/version_compat.md
@@ -49,19 +49,19 @@ patch versions.  The public APIs consist of
   submodules, but is not documented, then it is **not** considered part of the
   public API.
 
-* The [C API](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/c_api.h).
+* The [C API](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/c/c_api.h).
 
 * The following protocol buffer files:
-    * [`attr_value`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto)
-    * [`config`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto)
-    * [`event`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/event.proto)
-    * [`graph`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto)
-    * [`op_def`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def.proto)
-    * [`reader_base`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/reader_base.proto)
-    * [`summary`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto)
-    * [`tensor`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto)
-    * [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto)
-    * [`types`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto)
+    * [`attr_value`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/attr_value.proto)
+    * [`config`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/protobuf/config.proto)
+    * [`event`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/util/event.proto)
+    * [`graph`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/graph.proto)
+    * [`op_def`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/op_def.proto)
+    * [`reader_base`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/reader_base.proto)
+    * [`summary`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/summary.proto)
+    * [`tensor`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor.proto)
+    * [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/tensor_shape.proto)
+    * [`types`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/types.proto)
 
 
 ## What is *not* covered
@@ -79,9 +79,9 @@ backward incompatible ways between minor releases. These include:
     such as:
 
   - [C++](./extend/cc.md) (exposed through header files in
-    [`tensorflow/cc`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/cc)).
+    [`tensorflow/cc`](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/cc)).
   - [Java](../api_docs/java/reference/org/tensorflow/package-summary),
-  - [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go)
+  - [Go](https://pkg.go.dev/github.com/tensorflow/tensorflow/tensorflow/go)
   - [JavaScript](https://js.tensorflow.org)
 
 *   **Details of composite ops:** Many public functions in Python expand to
@@ -209,7 +209,7 @@ guidelines for evolving `GraphDef` versions.
 There are different data versions for graphs and checkpoints. The two data
 formats evolve at different rates from each other and also at different rates
 from TensorFlow. Both versioning systems are defined in
-[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h).
+[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/public/version.h).
 Whenever a new version is added, a note is added to the header detailing what
 changed and the date.
 
@@ -224,7 +224,7 @@ We distinguish between the following kinds of data version information:
   (`min_producer`).
 
 Each piece of versioned data has a [`VersionDef
-versions`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/versions.proto)
+versions`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/framework/versions.proto)
 field which records the `producer` that made the data, the `min_consumer`
 that it is compatible with, and a list of `bad_consumers` versions that are
 disallowed.
@@ -239,7 +239,7 @@ accept a piece of data if the following are all true:
 *   `consumer` not in data's `bad_consumers`
 
 Since both producers and consumers come from the same TensorFlow code base,
-[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h)
+[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/public/version.h)
 contains a main data version which is treated as either `producer` or
 `consumer` depending on context and both `min_consumer` and `min_producer`
 (needed by producers and consumers, respectively). Specifically,
@@ -309,7 +309,7 @@ existing producer scripts will not suddenly use the new functionality.
 1.  Add a new similar op named `SomethingV2` or similar and go through the
     process of adding it and switching existing Python wrappers to use it.
     To ensure forward compatibility use the checks suggested in
-    [compat.py](https://www.tensorflow.org/code/tensorflow/python/compat/compat.py)
+    [compat.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/compat/compat.py)
     when changing the Python wrappers.
 2.  Remove the old op (Can only take place with a major version change due to
     backward compatibility).
diff --git a/site/en/r1/tutorials/README.md b/site/en/r1/tutorials/README.md
index b6d932041bd..9ff164ad77c 100644
--- a/site/en/r1/tutorials/README.md
+++ b/site/en/r1/tutorials/README.md
@@ -10,7 +10,7 @@ desktop, mobile, web, and cloud. See the sections below to get started.
 
 The high-level Keras API provides building blocks to create and
 train deep learning models. Start with these beginner-friendly
-notebook examples, then read the [TensorFlow Keras guide](../guide/keras.ipynb).
+notebook examples, then read the [TensorFlow Keras guide](https://www.tensorflow.org/guide/keras).
 
 * [Basic classification](./keras/basic_classification.ipynb)
 * [Text classification](./keras/basic_text_classification.ipynb)
@@ -68,4 +68,4 @@ implement common ML algorithms. See the
 * [Boosted trees](./estimators/boosted_trees.ipynb)
 * [Gradient Boosted Trees: Model understanding](./estimators/boosted_trees_model_understanding.ipynb)
 * [Build a Convolutional Neural Network using Estimators](./estimators/cnn.ipynb)
-* [Wide and deep learning with Estimators](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep)
+* [Wide and deep learning with Estimators](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep)
diff --git a/site/en/r1/tutorials/_index.ipynb b/site/en/r1/tutorials/_index.ipynb
index ee198bf674f..eca1450964f 100644
--- a/site/en/r1/tutorials/_index.ipynb
+++ b/site/en/r1/tutorials/_index.ipynb
@@ -1,25 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "_index.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "rX8mhOLljYeM"
       },
       "source": [
@@ -28,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "BZSlp3DAjdYf",
-        "colab": {}
+        "id": "BZSlp3DAjdYf"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -46,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3wF5wszaj97Y"
       },
       "source": [
@@ -63,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "DUNzJc4jTj6G"
       },
       "source": [
@@ -80,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "DUNzJc4jTj6G"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "hiH7AC-NTniF"
       },
       "source": [
@@ -96,22 +88,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "0trJmd6DjqBZ",
-        "colab": {}
+        "id": "0trJmd6DjqBZ"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "import tensorflow as tf"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "import tensorflow.compat.v1 as tf"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "7NAbSZiaoJ4z"
       },
       "source": [
@@ -120,24 +108,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "7FP5258xjs-v",
-        "colab": {}
+        "id": "7FP5258xjs-v"
       },
+      "outputs": [],
       "source": [
         "mnist = tf.keras.datasets.mnist\n",
         "\n",
         "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
         "x_train, x_test = x_train / 255.0, x_test / 255.0"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "BPZ68wASog_I"
       },
       "source": [
@@ -146,11 +131,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "h3IKyzTCDNGo",
-        "colab": {}
+        "id": "h3IKyzTCDNGo"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.models.Sequential([\n",
         "  tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
@@ -162,14 +147,11 @@
         "model.compile(optimizer='adam',\n",
         "              loss='sparse_categorical_crossentropy',\n",
         "              metrics=['accuracy'])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ix4mEL65on-w"
       },
       "source": [
@@ -178,28 +160,38 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "F7dTAzgHDUh7",
-        "colab": {}
+        "id": "F7dTAzgHDUh7"
       },
+      "outputs": [],
       "source": [
         "model.fit(x_train, y_train, epochs=5)\n",
         "\n",
         "model.evaluate(x_test,  y_test, verbose=2)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "T4JfEh7kvx6m"
       },
       "source": [
         "You’ve now trained an image classifier with ~98% accuracy on this dataset. See [Get Started with TensorFlow](https://www.tensorflow.org/get_started/) to learn more."
       ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "_index.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/distribute/keras.ipynb b/site/en/r1/tutorials/distribute/keras.ipynb
index e3a8c53891c..14e8bf739a9 100644
--- a/site/en/r1/tutorials/distribute/keras.ipynb
+++ b/site/en/r1/tutorials/distribute/keras.ipynb
@@ -1,40 +1,22 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "keras.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Tce3stUlHN0L"
       },
       "source": [
-        "##### Copyright 2019 The TensorFlow Authors.\n",
-        "\n"
+        "##### Copyright 2019 The TensorFlow Authors.\n"
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "tuOe1ymfHZPu",
-        "colab": {}
+        "id": "tuOe1ymfHZPu"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -47,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MfBg1C5NB3X0"
       },
       "source": [
@@ -64,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "r6P32iYYV27b"
       },
       "source": [
@@ -81,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "r6P32iYYV27b"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "xHxb-dlhMIzW"
       },
       "source": [
@@ -95,14 +86,12 @@
         "Essentially, it copies all of the model's variables to each processor.\n",
         "Then, it uses [all-reduce](http://mpitutorial.com/tutorials/mpi-reduce-and-allreduce/) to combine the gradients from all processors and applies the combined value to all copies of the model.\n",
         "\n",
-        "`MirroredStategy` is one of several distribution strategy available in TensorFlow core. You can read about more strategies at [distribution strategy guide](../../guide/distribute_strategy.ipynb).\n",
-        "\n"
+        "`MirroredStrategy` is one of several distribution strategy available in TensorFlow core. You can read about more strategies at [distribution strategy guide](../../guide/distribute_strategy.ipynb).\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MUXex9ctTuDB"
       },
       "source": [
@@ -114,7 +103,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Dney9v7BsJij"
       },
       "source": [
@@ -123,38 +111,23 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "r8S3ublR7Ay8",
-        "colab": {}
-      },
-      "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals"
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "colab_type": "code",
-        "id": "74rHkS_DB3X2",
-        "colab": {}
+        "id": "74rHkS_DB3X2"
       },
+      "outputs": [],
       "source": [
         "# Import TensorFlow\n",
-        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "\n",
         "import tensorflow_datasets as tfds\n",
         "\n",
         "import os"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hXhefksNKk2I"
       },
       "source": [
@@ -164,7 +137,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "OtnnUwvmB3X5"
       },
       "source": [
@@ -174,7 +146,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "lHAPqG8MtS8M"
       },
       "source": [
@@ -184,22 +155,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "iXMJ3G9NB3X6",
-        "colab": {}
+        "id": "iXMJ3G9NB3X6"
       },
+      "outputs": [],
       "source": [
         "datasets, ds_info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n",
         "mnist_train, mnist_test = datasets['train'], datasets['test']"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GrjVhv-eKuHD"
       },
       "source": [
@@ -209,7 +177,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "TlH8vx6BB3X9"
       },
       "source": [
@@ -218,34 +185,29 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "4j0tdf4YB3X9",
-        "colab": {}
+        "id": "4j0tdf4YB3X9"
       },
+      "outputs": [],
       "source": [
         "strategy = tf.distribute.MirroredStrategy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "cY3KA_h2iVfN",
-        "colab": {}
+        "id": "cY3KA_h2iVfN"
       },
+      "outputs": [],
       "source": [
         "print ('Number of devices: {}'.format(strategy.num_replicas_in_sync))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "lNbPv0yAleW8"
       },
       "source": [
@@ -255,7 +217,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "psozqcuptXhK"
       },
       "source": [
@@ -264,11 +225,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "p1xWxKcnhar9",
-        "colab": {}
+        "id": "p1xWxKcnhar9"
       },
+      "outputs": [],
       "source": [
         "# You can also do ds_info.splits.total_num_examples to get the total\n",
         "# number of examples in the dataset.\n",
@@ -280,14 +241,11 @@
         "\n",
         "BATCH_SIZE_PER_REPLICA = 64\n",
         "BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "0Wm5rsL2KoDF"
       },
       "source": [
@@ -296,25 +254,22 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Eo9a46ZeJCkm",
-        "colab": {}
+        "id": "Eo9a46ZeJCkm"
       },
+      "outputs": [],
       "source": [
         "def scale(image, label):\n",
         "  image = tf.cast(image, tf.float32)\n",
         "  image /= 255\n",
         "\n",
         "  return image, label"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "WZCa5RLc5A91"
       },
       "source": [
@@ -323,22 +278,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "gRZu2maChwdT",
-        "colab": {}
+        "id": "gRZu2maChwdT"
       },
+      "outputs": [],
       "source": [
         "train_dataset = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
         "eval_dataset = mnist_test.map(scale).batch(BATCH_SIZE)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "4xsComp8Kz5H"
       },
       "source": [
@@ -348,7 +300,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "1BnQYQTpB3YA"
       },
       "source": [
@@ -357,11 +308,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "IexhL_vIB3YA",
-        "colab": {}
+        "id": "IexhL_vIB3YA"
       },
+      "outputs": [],
       "source": [
         "with strategy.scope():\n",
         "  model = tf.keras.Sequential([\n",
@@ -375,31 +326,26 @@
         "  model.compile(loss='sparse_categorical_crossentropy',\n",
         "                optimizer=tf.keras.optimizers.Adam(),\n",
         "                metrics=['accuracy'])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8i6OU5W9Vy2u"
       },
       "source": [
-        "## Define the callbacks.\n",
-        "\n"
+        "## Define the callbacks.\n"
       ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "YOXO5nvvK3US"
       },
       "source": [
         "The callbacks used here are:\n",
         "\n",
-        "*   *Tensorboard*: This callback writes a log for Tensorboard which allows you to visualize the graphs.\n",
+        "*   *TensorBoard*: This callback writes a log for TensorBoard which allows you to visualize the graphs.\n",
         "*   *Model Checkpoint*: This callback saves the model after every epoch.\n",
         "*   *Learning Rate Scheduler*: Using this callback, you can schedule the learning rate to change after every epoch/batch.\n",
         "\n",
@@ -408,28 +354,26 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "A9bwLCcXzSgy",
-        "colab": {}
+        "id": "A9bwLCcXzSgy"
       },
+      "outputs": [],
       "source": [
         "# Define the checkpoint directory to store the checkpoints\n",
         "\n",
         "checkpoint_dir = './training_checkpoints'\n",
         "# Name of the checkpoint files\n",
         "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "wpU-BEdzJDbK",
-        "colab": {}
+        "id": "wpU-BEdzJDbK"
       },
+      "outputs": [],
       "source": [
         "# Function for decaying the learning rate.\n",
         "# You can define any decay function you need.\n",
@@ -440,34 +384,30 @@
         "    return 1e-4\n",
         "  else:\n",
         "    return 1e-5"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "jKhiMgXtKq2w",
-        "colab": {}
+        "id": "jKhiMgXtKq2w"
       },
+      "outputs": [],
       "source": [
         "# Callback for printing the LR at the end of each epoch.\n",
         "class PrintLR(tf.keras.callbacks.Callback):\n",
         "  def on_epoch_end(self, epoch, logs=None):\n",
         "    print ('\\nLearning rate for epoch {} is {}'.format(\n",
         "        epoch + 1, tf.keras.backend.get_value(model.optimizer.lr)))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "YVqAbR6YyNQh",
-        "colab": {}
+        "id": "YVqAbR6YyNQh"
       },
+      "outputs": [],
       "source": [
         "callbacks = [\n",
         "    tf.keras.callbacks.TensorBoard(log_dir='./logs'),\n",
@@ -476,14 +416,11 @@
         "    tf.keras.callbacks.LearningRateScheduler(decay),\n",
         "    PrintLR()\n",
         "]"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "70HXgDQmK46q"
       },
       "source": [
@@ -493,7 +430,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6EophnOAB3YD"
       },
       "source": [
@@ -502,21 +438,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "7MVw_6CqB3YD",
-        "colab": {}
+        "id": "7MVw_6CqB3YD"
       },
+      "outputs": [],
       "source": [
         "model.fit(train_dataset, epochs=10, callbacks=callbacks)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "NUcWAUUupIvG"
       },
       "source": [
@@ -525,22 +458,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "JQ4zeSTxKEhB",
-        "colab": {}
+        "id": "JQ4zeSTxKEhB"
       },
+      "outputs": [],
       "source": [
         "# check the checkpoint directory\n",
         "!ls {checkpoint_dir}"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qor53h7FpMke"
       },
       "source": [
@@ -551,24 +481,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "JtEwxiTgpQoP",
-        "colab": {}
+        "id": "JtEwxiTgpQoP"
       },
+      "outputs": [],
       "source": [
         "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n",
         "\n",
         "eval_loss, eval_acc = model.evaluate(eval_dataset)\n",
         "print ('Eval loss: {}, Eval Accuracy: {}'.format(eval_loss, eval_acc))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IIeF2RWfYu4N"
       },
       "source": [
@@ -581,21 +508,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "LnyscOkvKKBR",
-        "colab": {}
+        "id": "LnyscOkvKKBR"
       },
+      "outputs": [],
       "source": [
         "!ls -sh ./logs"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "kBLlogrDvMgg"
       },
       "source": [
@@ -605,7 +529,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Xa87y_A0vRma"
       },
       "source": [
@@ -614,34 +537,29 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "h8Q4MKOLwG7K",
-        "colab": {}
+        "id": "h8Q4MKOLwG7K"
       },
+      "outputs": [],
       "source": [
         "path = 'saved_model/'"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "4HvcDmVsvQoa",
-        "colab": {}
+        "id": "4HvcDmVsvQoa"
       },
+      "outputs": [],
       "source": [
-        "tf.keras.experimental.export_saved_model(model, path)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "model.save(path)"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vKJT4w5JwVPI"
       },
       "source": [
@@ -650,13 +568,13 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "T_gT0RbRvQ3o",
-        "colab": {}
+        "id": "T_gT0RbRvQ3o"
       },
+      "outputs": [],
       "source": [
-        "unreplicated_model = tf.keras.experimental.load_from_saved_model(path)\n",
+        "unreplicated_model = tf.keras.models.load_model(path)\n",
         "\n",
         "unreplicated_model.compile(\n",
         "    loss='sparse_categorical_crossentropy',\n",
@@ -665,14 +583,11 @@
         "\n",
         "eval_loss, eval_acc = unreplicated_model.evaluate(eval_dataset)\n",
         "print ('Eval loss: {}, Eval Accuracy: {}'.format(eval_loss, eval_acc))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "8uNqWRdDMl5S"
       },
       "source": [
@@ -683,5 +598,18 @@
         "Note: `tf.distribute.Strategy` is actively under development and we will be adding more examples and tutorials in the near future. Please give it a try. We welcome your feedback via [issues on GitHub](https://github.com/tensorflow/tensorflow/issues/new)."
       ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "keras.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/distribute/tpu_custom_training.ipynb b/site/en/r1/tutorials/distribute/tpu_custom_training.ipynb
index 3d975dd0ecb..c61f893ca4c 100644
--- a/site/en/r1/tutorials/distribute/tpu_custom_training.ipynb
+++ b/site/en/r1/tutorials/distribute/tpu_custom_training.ipynb
@@ -1,26 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "accelerator": "TPU",
-    "colab": {
-      "name": "tpu_custom_training.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MhoQ0WE77laV"
       },
       "source": [
@@ -29,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "_ckMIh7O7s6D",
-        "colab": {}
+        "id": "_ckMIh7O7s6D"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -47,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "jYysdyb-CaWM"
       },
       "source": [
@@ -64,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "S5Uhzt6vVIB2"
       },
       "source": [
@@ -81,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "S5Uhzt6vVIB2"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "FbVhjPpzn6BM"
       },
       "source": [
@@ -92,16 +83,15 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "dzLKpmZICaWN",
-        "colab": {}
+        "id": "dzLKpmZICaWN"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
         "# Import TensorFlow\n",
-        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "tf.compat.v1.disable_eager_execution()\n",
         "\n",
         "# Helper libraries\n",
         "import numpy as np\n",
@@ -110,27 +100,22 @@
         "assert os.environ['COLAB_TPU_ADDR'], 'Make sure to select TPU from Edit > Notebook settings > Hardware accelerator'\n",
         "\n",
         "assert float('.'.join(tf.__version__.split('.')[:2])) >= 1.14, 'Make sure that Tensorflow version is at least 1.14'"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "5TPCyNk1LjAC",
-        "colab": {}
+        "id": "7nSiTgSt-RrV"
       },
+      "outputs": [],
       "source": [
         "TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MM6W__qraV55"
       },
       "source": [
@@ -140,7 +125,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "58ff7ew6MK9d"
       },
       "source": [
@@ -149,11 +133,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "7MqDQO0KCaWS",
-        "colab": {}
+        "id": "7MqDQO0KCaWS"
       },
+      "outputs": [],
       "source": [
         "def create_model(input_shape):\n",
         "  \"\"\"Creates a simple convolutional neural network model using the Keras API\"\"\"\n",
@@ -165,14 +149,11 @@
         "      tf.keras.layers.Dropout(0.2),\n",
         "      tf.keras.layers.Dense(10, activation=tf.nn.softmax),\n",
         "  ])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "4AXoHhrsbdF3"
       },
       "source": [
@@ -182,7 +163,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5mVuLZhbem8d"
       },
       "source": [
@@ -191,11 +171,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "F2VeZUWUj5S4",
-        "colab": {}
+        "id": "F2VeZUWUj5S4"
       },
+      "outputs": [],
       "source": [
         "def loss(model, x, y):\n",
         "  \"\"\"Calculates the loss given an example (x, y)\"\"\"\n",
@@ -206,14 +186,11 @@
         "  \"\"\"Calculates the loss and the gradients given an example (x, y)\"\"\"\n",
         "  logits, loss_value = loss(model, x, y)\n",
         "  return logits, loss_value, tf.gradients(loss_value, model.trainable_variables)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "k53F5I_IiGyI"
       },
       "source": [
@@ -223,7 +200,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "0Qb6nDgxiN_n"
       },
       "source": [
@@ -232,18 +208,17 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "jwJtsCQhHK-E",
-        "colab": {}
+        "id": "jwJtsCQhHK-E"
       },
+      "outputs": [],
       "source": [
         "tf.keras.backend.clear_session()\n",
         "\n",
-        "resolver = tf.contrib.cluster_resolver.TPUClusterResolver(tpu=TPU_WORKER)\n",
-        "tf.contrib.distribute.initialize_tpu_system(resolver)\n",
-        "strategy = tf.contrib.distribute.TPUStrategy(resolver)\n",
-        "\n",
+        "resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=TPU_WORKER)\n",
+        "tf.tpu.experimental.initialize_tpu_system(resolver)\n",
+        "strategy = tf.distribute.experimental.TPUStrategy(resolver)\n",
         "# Load MNIST training and test data\n",
         "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
         "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
@@ -268,14 +243,11 @@
         "\n",
         "train_steps_per_epoch = len(x_train) // BATCH_SIZE\n",
         "test_steps_per_epoch = len(x_test) // BATCH_SIZE"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GPrDC8IfOgCT"
       },
       "source": [
@@ -288,11 +260,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "s_suB7CZNw5W",
-        "colab": {}
+        "id": "s_suB7CZNw5W"
       },
+      "outputs": [],
       "source": [
         "with strategy.scope():\n",
         "  model = create_model(input_shape)\n",
@@ -305,14 +277,11 @@
         "  test_loss = tf.keras.metrics.Mean('test_loss', dtype=tf.float32)\n",
         "  test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n",
         "      'test_accuracy', dtype=tf.float32)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "d3iLK5ZtO1_R"
       },
       "source": [
@@ -321,11 +290,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "XAF6xfU0N5ID",
-        "colab": {}
+        "id": "XAF6xfU0N5ID"
       },
+      "outputs": [],
       "source": [
         "with strategy.scope():\n",
         "  def train_step(inputs):\n",
@@ -360,14 +329,11 @@
         "\n",
         "    with tf.control_dependencies([update_loss, update_accuracy]):\n",
         "      return tf.identity(loss_value)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "AhrK1-yEO7Nf"
       },
       "source": [
@@ -377,11 +343,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "or5osuheouVU",
-        "colab": {}
+        "id": "or5osuheouVU"
       },
+      "outputs": [],
       "source": [
         "def run_train():\n",
         "  # Train\n",
@@ -410,17 +376,15 @@
         "      session.run(test_accuracy_result) * 100))\n",
         "  test_loss.reset_states()\n",
         "  test_accuracy.reset_states()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "u5LvzAwjN95j",
-        "colab": {}
+        "id": "u5LvzAwjN95j"
       },
+      "outputs": [],
       "source": [
         "with strategy.scope():\n",
         "  training_loss_result = training_loss.result()\n",
@@ -447,10 +411,10 @@
         "    train_iterator = strategy.make_dataset_iterator(train_dataset)\n",
         "\n",
         "    test_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(BATCH_SIZE, drop_remainder=True)\n",
-        "    test_iterator = strategy.make_dataset_iterator(train_dataset)\n",
+        "    test_iterator = strategy.make_dataset_iterator(test_dataset)\n",
         "    \n",
-        "    train_iterator_init = train_iterator.initialize()\n",
-        "    test_iterator_init = test_iterator.initialize()\n",
+        "    train_iterator_init = train_iterator.initializer\n",
+        "    test_iterator_init = test_iterator.initializer\n",
         "\n",
         "    session.run([v.initializer for v in all_variables])\n",
         "    \n",
@@ -464,9 +428,21 @@
         "      run_train()\n",
         "\n",
         "      run_test()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "accelerator": "TPU",
+    "colab": {
+      "collapsed_sections": [],
+      "name": "tpu_custom_training.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/distribute/training_loops.ipynb b/site/en/r1/tutorials/distribute/training_loops.ipynb
index d272095eb23..8eb72c13030 100644
--- a/site/en/r1/tutorials/distribute/training_loops.ipynb
+++ b/site/en/r1/tutorials/distribute/training_loops.ipynb
@@ -1,25 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "training_loops.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MhoQ0WE77laV"
       },
       "source": [
@@ -28,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "_ckMIh7O7s6D",
-        "colab": {}
+        "id": "_ckMIh7O7s6D"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -46,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "jYysdyb-CaWM"
       },
       "source": [
@@ -63,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "S5Uhzt6vVIB2"
       },
       "source": [
@@ -80,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "S5Uhzt6vVIB2"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "FbVhjPpzn6BM"
       },
       "source": [
@@ -91,30 +83,26 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "dzLKpmZICaWN",
-        "colab": {}
+        "id": "dzLKpmZICaWN"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
         "# Import TensorFlow\n",
-        "import tensorflow as tf\n",
+        "import tensorflow.compat.v1 as tf\n",
+        "\n",
         "\n",
         "# Helper libraries\n",
         "import numpy as np\n",
         "import os\n",
         "\n",
         "print(tf.__version__)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "MM6W__qraV55"
       },
       "source": [
@@ -123,11 +111,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "7MqDQO0KCaWS",
-        "colab": {}
+        "id": "7MqDQO0KCaWS"
       },
+      "outputs": [],
       "source": [
         "fashion_mnist = tf.keras.datasets.fashion_mnist\n",
         "\n",
@@ -146,14 +134,11 @@
         "\n",
         "train_labels = train_labels.astype('int64')\n",
         "test_labels = test_labels.astype('int64')"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "4AXoHhrsbdF3"
       },
       "source": [
@@ -163,7 +148,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5mVuLZhbem8d"
       },
       "source": [
@@ -180,36 +164,31 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "F2VeZUWUj5S4",
-        "colab": {}
+        "id": "F2VeZUWUj5S4"
       },
+      "outputs": [],
       "source": [
         "# If the list of devices is not specified in the\n",
         "# `tf.distribute.MirroredStrategy` constructor, it will be auto-detected.\n",
         "strategy = tf.distribute.MirroredStrategy()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ZngeM_2o0_JO",
-        "colab": {}
+        "id": "ZngeM_2o0_JO"
       },
+      "outputs": [],
       "source": [
         "print ('Number of devices: {}'.format(strategy.num_replicas_in_sync))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "k53F5I_IiGyI"
       },
       "source": [
@@ -219,7 +198,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "0Qb6nDgxiN_n"
       },
       "source": [
@@ -228,11 +206,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "jwJtsCQhHK-E",
-        "colab": {}
+        "id": "jwJtsCQhHK-E"
       },
+      "outputs": [],
       "source": [
         "BUFFER_SIZE = len(train_images)\n",
         "\n",
@@ -240,14 +218,11 @@
         "BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n",
         "\n",
         "EPOCHS = 10"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "J7fj3GskHC8g"
       },
       "source": [
@@ -256,11 +231,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "WYrMNNDhAvVl",
-        "colab": {}
+        "id": "WYrMNNDhAvVl"
       },
+      "outputs": [],
       "source": [
         "train_dataset = tf.data.Dataset.from_tensor_slices(\n",
         "(train_images, train_labels)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
@@ -269,14 +244,11 @@
         "test_dataset = tf.data.Dataset.from_tensor_slices(\n",
         "    (test_images, test_labels)).batch(BATCH_SIZE)\n",
         "test_ds = strategy.experimental_distribute_dataset(test_dataset)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "bAXAo_wWbWSb"
       },
       "source": [
@@ -287,11 +259,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "9ODch-OFCaW4",
-        "colab": {}
+        "id": "9ODch-OFCaW4"
       },
+      "outputs": [],
       "source": [
         "with strategy.scope():\n",
         "  model = tf.keras.Sequential([\n",
@@ -305,14 +277,11 @@
         "      tf.keras.layers.Dense(10, activation='softmax')\n",
         "    ])\n",
         "  optimizer = tf.train.GradientDescentOptimizer(0.001)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "e-wlFFZbP33n"
       },
       "source": [
@@ -349,7 +318,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "iuKuNXPORfqJ"
       },
       "source": [
@@ -358,11 +326,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "47BLVkRkVQDO",
-        "colab": {}
+        "id": "47BLVkRkVQDO"
       },
+      "outputs": [],
       "source": [
         "with strategy.scope():\n",
         "  def train_step(dist_inputs):\n",
@@ -376,26 +344,24 @@
         "      with tf.control_dependencies([train_op]):\n",
         "        return tf.identity(loss)\n",
         "\n",
-        "    per_replica_losses = strategy.experimental_run_v2(\n",
+        "    per_replica_losses = strategy.run(\n",
         "        step_fn, args=(dist_inputs,))\n",
         "    mean_loss = strategy.reduce(\n",
         "        tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n",
         "    return mean_loss"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "7x7s5iYAYSGD",
-        "colab": {}
+        "id": "7x7s5iYAYSGD"
       },
+      "outputs": [],
       "source": [
         "with strategy.scope():\n",
         "  train_iterator = train_ds.make_initializable_iterator()\n",
-        "  iterator_init = train_iterator.initialize()\n",
+        "  iterator_init = train_iterator.initializer\n",
         "  var_init = tf.global_variables_initializer()\n",
         "  loss = train_step(next(train_iterator))\n",
         "  with tf.Session() as sess:\n",
@@ -407,14 +373,11 @@
         "            print('Epoch {} Step {} Loss {:.4f}'.format(epoch+1,\n",
         "                                                        step,\n",
         "                                                        sess.run(loss)))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6hEJNsokjOKs"
       },
       "source": [
@@ -423,5 +386,18 @@
         "Try out the new `tf.distribute.Strategy` API on your models."
       ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "training_loops.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/eager/automatic_differentiation.ipynb b/site/en/r1/tutorials/eager/automatic_differentiation.ipynb
index 8ad0b8b8316..df843bac3b8 100644
--- a/site/en/r1/tutorials/eager/automatic_differentiation.ipynb
+++ b/site/en/r1/tutorials/eager/automatic_differentiation.ipynb
@@ -1,26 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "automatic_differentiation.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "language": "python",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "t09eeeR5prIJ"
       },
       "source": [
@@ -29,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "GCCk8_dHpuNf",
-        "colab": {}
+        "id": "GCCk8_dHpuNf"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -47,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "xh8WkEwWpnm7"
       },
       "source": [
@@ -64,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "idv0bPeCp325"
       },
       "source": [
@@ -81,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "idv0bPeCp325"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "vDJ4XzMqodTy"
       },
       "source": [
@@ -91,7 +82,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GQJysDM__Qb0"
       },
       "source": [
@@ -100,25 +90,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "OiMPZStlibBv",
-        "colab": {}
+        "id": "OiMPZStlibBv"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.enable_eager_execution()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "import tensorflow.compat.v1 as tf"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "1CLWJl0QliB0"
       },
       "source": [
@@ -131,11 +114,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "bAFeIE8EuVIq",
-        "colab": {}
+        "id": "bAFeIE8EuVIq"
       },
+      "outputs": [],
       "source": [
         "x = tf.ones((2, 2))\n",
         "\n",
@@ -149,14 +132,11 @@
         "for i in [0, 1]:\n",
         "  for j in [0, 1]:\n",
         "    assert dz_dx[i][j].numpy() == 8.0"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "N4VlqKFzzGaC"
       },
       "source": [
@@ -165,11 +145,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "7XaPRAwUyYms",
-        "colab": {}
+        "id": "7XaPRAwUyYms"
       },
+      "outputs": [],
       "source": [
         "x = tf.ones((2, 2))\n",
         "\n",
@@ -182,14 +162,11 @@
         "# intermediate value y.\n",
         "dz_dy = t.gradient(z, y)\n",
         "assert dz_dy.numpy() == 8.0"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ISkXuY7YzIcS"
       },
       "source": [
@@ -198,11 +175,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "zZaCm3-9zVCi",
-        "colab": {}
+        "id": "zZaCm3-9zVCi"
       },
+      "outputs": [],
       "source": [
         "x = tf.constant(3.0)\n",
         "with tf.GradientTape(persistent=True) as t:\n",
@@ -212,14 +189,11 @@
         "dz_dx = t.gradient(z, x)  # 108.0 (4*x^3 at x = 3)\n",
         "dy_dx = t.gradient(y, x)  # 6.0\n",
         "del t  # Drop the reference to the tape"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "6kADybtQzYj4"
       },
       "source": [
@@ -230,11 +204,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "9FViq92UX7P8",
-        "colab": {}
+        "id": "9FViq92UX7P8"
       },
+      "outputs": [],
       "source": [
         "def f(x, y):\n",
         "  output = 1.0\n",
@@ -254,14 +228,11 @@
         "assert grad(x, 6).numpy() == 12.0\n",
         "assert grad(x, 5).numpy() == 12.0\n",
         "assert grad(x, 4).numpy() == 4.0\n"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "DK05KXrAAld3"
       },
       "source": [
@@ -272,11 +243,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "cPQgthZ7ugRJ",
-        "colab": {}
+        "id": "cPQgthZ7ugRJ"
       },
+      "outputs": [],
       "source": [
         "x = tf.Variable(1.0)  # Create a Tensorflow variable initialized to 1.0\n",
         "\n",
@@ -290,14 +261,11 @@
         "\n",
         "assert dy_dx.numpy() == 3.0\n",
         "assert d2y_dx2.numpy() == 6.0"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "4U1KKzUpNl58"
       },
       "source": [
@@ -306,5 +274,18 @@
         "In this tutorial we covered gradient computation in TensorFlow. With that we have enough of the primitives required to build and train neural networks."
       ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "automatic_differentiation.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/eager/custom_layers.ipynb b/site/en/r1/tutorials/eager/custom_layers.ipynb
index 803fa4072f4..48b55ed943e 100644
--- a/site/en/r1/tutorials/eager/custom_layers.ipynb
+++ b/site/en/r1/tutorials/eager/custom_layers.ipynb
@@ -1,26 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "custom_layers.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "language": "python",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "tDnwEv8FtJm7"
       },
       "source": [
@@ -29,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "JlknJBWQtKkI",
-        "colab": {}
+        "id": "JlknJBWQtKkI"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -47,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "60RdWsg1tETW"
       },
       "source": [
@@ -64,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "BcJg7Enms86w"
       },
       "source": [
@@ -81,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "BcJg7Enms86w"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "UEu3q4jmpKVT"
       },
       "source": [
@@ -90,25 +81,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "pwX7Fii1rwsJ",
-        "colab": {}
+        "id": "pwX7Fii1rwsJ"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.enable_eager_execution()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "import tensorflow.compat.v1 as tf"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "zSFfVVjkrrsI"
       },
       "source": [
@@ -123,11 +107,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "8PyXlPl-4TzQ",
-        "colab": {}
+        "id": "8PyXlPl-4TzQ"
       },
+      "outputs": [],
       "source": [
         "# In the tf.keras.layers package, layers are objects. To construct a layer,\n",
         "# simply construct the object. Most layers take as a first argument the number\n",
@@ -137,14 +121,11 @@
         "# the first time the layer is used, but it can be provided if you want to\n",
         "# specify it manually, which is useful in some complex models.\n",
         "layer = tf.keras.layers.Dense(10, input_shape=(None, 5))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Fn69xxPO5Psr"
       },
       "source": [
@@ -154,53 +135,46 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "E3XKNknP5Mhb",
-        "colab": {}
+        "id": "E3XKNknP5Mhb"
       },
+      "outputs": [],
       "source": [
         "# To use a layer, simply call it.\n",
         "layer(tf.zeros([10, 5]))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Wt_Nsv-L5t2s",
-        "colab": {}
+        "id": "Wt_Nsv-L5t2s"
       },
+      "outputs": [],
       "source": [
         "# Layers have many useful methods. For example, you can inspect all variables\n",
         "# in a layer using `layer.variables` and trainable variables using\n",
         "# `layer.trainable_variables`. In this case a fully-connected layer\n",
         "# will have variables for weights and biases.\n",
         "layer.variables"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "6ilvKjz8_4MQ",
-        "colab": {}
+        "id": "6ilvKjz8_4MQ"
       },
+      "outputs": [],
       "source": [
         "# The variables are also accessible through nice accessors\n",
         "layer.kernel, layer.bias"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "O0kDbE54-5VS"
       },
       "source": [
@@ -215,11 +189,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "5Byl3n1k5kIy",
-        "colab": {}
+        "id": "5Byl3n1k5kIy"
       },
+      "outputs": [],
       "source": [
         "class MyDenseLayer(tf.keras.layers.Layer):\n",
         "  def __init__(self, num_outputs):\n",
@@ -237,14 +211,11 @@
         "layer = MyDenseLayer(10)\n",
         "print(layer(tf.zeros([10, 5])))\n",
         "print(layer.trainable_variables)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "tk8E2vY0-z4Z"
       },
       "source": [
@@ -254,24 +225,23 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Qhg4KlbKrs3G"
       },
       "source": [
         "## Models: composing layers\n",
         "\n",
-        "Many interesting layer-like things in machine learning models are implemented by composing existing layers. For example, each residual block in a resnet is a composition of convolutions, batch normalizations, and a shortcut.\n",
+        "Many interesting layer-like things in machine learning models are implemented by composing existing layers. For example, each residual block in a ResNet is a composition of convolutions, batch normalizations, and a shortcut.\n",
         "\n",
         "The main class used when creating a layer-like thing which contains other layers is tf.keras.Model. Implementing one is done by inheriting from tf.keras.Model."
       ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "N30DTXiRASlb",
-        "colab": {}
+        "id": "N30DTXiRASlb"
       },
+      "outputs": [],
       "source": [
         "class ResnetIdentityBlock(tf.keras.Model):\n",
         "  def __init__(self, kernel_size, filters):\n",
@@ -306,14 +276,11 @@
         "block = ResnetIdentityBlock(1, [1, 2, 3])\n",
         "print(block(tf.zeros([1, 2, 3, 3])))\n",
         "print([x.name for x in block.trainable_variables])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wYfucVw65PMj"
       },
       "source": [
@@ -322,11 +289,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "L9frk7Ur4uvJ",
-        "colab": {}
+        "id": "L9frk7Ur4uvJ"
       },
+      "outputs": [],
       "source": [
         " my_seq = tf.keras.Sequential([tf.keras.layers.Conv2D(1, (1, 1)),\n",
         "                               tf.keras.layers.BatchNormalization(),\n",
@@ -336,14 +303,11 @@
         "                               tf.keras.layers.Conv2D(3, (1, 1)),\n",
         "                               tf.keras.layers.BatchNormalization()])\n",
         "my_seq(tf.zeros([1, 2, 3, 3]))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "c5YwYcnuK-wc"
       },
       "source": [
@@ -352,5 +316,18 @@
         "Now you can go back to the previous notebook and adapt the linear regression example to use layers and models to be better structured."
       ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "custom_layers.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/eager/custom_training.ipynb b/site/en/r1/tutorials/eager/custom_training.ipynb
index c90d2d563ce..f0f7faffa7f 100644
--- a/site/en/r1/tutorials/eager/custom_training.ipynb
+++ b/site/en/r1/tutorials/eager/custom_training.ipynb
@@ -1,25 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "custom_training.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "5rmpybwysXGV"
       },
       "source": [
@@ -28,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "m8y3rGtQsYP2",
-        "colab": {}
+        "id": "m8y3rGtQsYP2"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -46,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "hrXv0rU9sIma"
       },
       "source": [
@@ -63,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "7S0BwJ_8sLu7"
       },
       "source": [
@@ -80,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "7S0BwJ_8sLu7"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "k2o3TTG4TFpt"
       },
       "source": [
@@ -93,7 +85,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3LXMVuV0VhDr"
       },
       "source": [
@@ -102,25 +93,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "PJ64L90aVir3",
-        "colab": {}
+        "id": "PJ64L90aVir3"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.enable_eager_execution()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "import tensorflow.compat.v1 as tf"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "eMAWbDJFVmMk"
       },
       "source": [
@@ -131,25 +115,22 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "VkJwtLS_Jbn8",
-        "colab": {}
+        "id": "VkJwtLS_Jbn8"
       },
+      "outputs": [],
       "source": [
         "# Using python state\n",
         "x = tf.zeros([10, 10])\n",
         "x += 2  # This is equivalent to x = x + 2, which does not mutate the original\n",
         "        # value of x\n",
         "print(x)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wfneTXy7JcUz"
       },
       "source": [
@@ -160,11 +141,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "itxmrMil6DQi",
-        "colab": {}
+        "id": "itxmrMil6DQi"
       },
+      "outputs": [],
       "source": [
         "v = tf.Variable(1.0)\n",
         "assert v.numpy() == 1.0\n",
@@ -176,14 +157,11 @@
         "# Use `v` in a TensorFlow operation like tf.square() and reassign\n",
         "v.assign(tf.square(v))\n",
         "assert v.numpy() == 9.0"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-paSaeq1JzwC"
       },
       "source": [
@@ -195,7 +173,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "BMiFcDzE7Qu3"
       },
       "source": [
@@ -214,7 +191,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "gFzH64Jn9PIm"
       },
       "source": [
@@ -225,11 +201,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "_WRu7Pze7wk8",
-        "colab": {}
+        "id": "_WRu7Pze7wk8"
       },
+      "outputs": [],
       "source": [
         "class Model(object):\n",
         "  def __init__(self):\n",
@@ -244,14 +220,11 @@
         "model = Model()\n",
         "\n",
         "assert model(3.0).numpy() == 15.0"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "xa6j_yXa-j79"
       },
       "source": [
@@ -262,22 +235,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Y0ysUFGY924U",
-        "colab": {}
+        "id": "Y0ysUFGY924U"
       },
+      "outputs": [],
       "source": [
         "def loss(predicted_y, desired_y):\n",
         "  return tf.reduce_mean(tf.square(predicted_y - desired_y))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qutT_fkl_CBc"
       },
       "source": [
@@ -288,11 +258,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "gxPTb-kt_N5m",
-        "colab": {}
+        "id": "gxPTb-kt_N5m"
       },
+      "outputs": [],
       "source": [
         "TRUE_W = 3.0\n",
         "TRUE_b = 2.0\n",
@@ -301,14 +271,11 @@
         "inputs  = tf.random_normal(shape=[NUM_EXAMPLES])\n",
         "noise   = tf.random_normal(shape=[NUM_EXAMPLES])\n",
         "outputs = inputs * TRUE_W + TRUE_b + noise"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "-50nq-wPBsAW"
       },
       "source": [
@@ -317,11 +284,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "_eb83LtrB4nt",
-        "colab": {}
+        "id": "_eb83LtrB4nt"
       },
+      "outputs": [],
       "source": [
         "import matplotlib.pyplot as plt\n",
         "\n",
@@ -331,14 +298,11 @@
         "\n",
         "print('Current loss: '),\n",
         "print(loss(model(inputs), outputs).numpy())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "sSDP-yeq_4jE"
       },
       "source": [
@@ -349,11 +313,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "MBIACgdnA55X",
-        "colab": {}
+        "id": "MBIACgdnA55X"
       },
+      "outputs": [],
       "source": [
         "def train(model, inputs, outputs, learning_rate):\n",
         "  with tf.GradientTape() as t:\n",
@@ -361,14 +325,11 @@
         "  dW, db = t.gradient(current_loss, [model.W, model.b])\n",
         "  model.W.assign_sub(learning_rate * dW)\n",
         "  model.b.assign_sub(learning_rate * db)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "RwWPaJryD2aN"
       },
       "source": [
@@ -377,11 +338,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "XdfkR223D9dW",
-        "colab": {}
+        "id": "XdfkR223D9dW"
       },
+      "outputs": [],
       "source": [
         "model = Model()\n",
         "\n",
@@ -404,14 +365,11 @@
         "         [TRUE_b] * len(epochs), 'b--')\n",
         "plt.legend(['W', 'b', 'true W', 'true_b'])\n",
         "plt.show()\n"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vPnIVuaSJwWz"
       },
       "source": [
@@ -423,5 +381,18 @@
         "In practice, particularly for neural networks, the higher level APIs like `tf.keras` will be much more convenient since it provides higher level building blocks (called \"layers\"), utilities to save and restore state, a suite of loss functions, a suite of optimization strategies etc.\n"
       ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "custom_training.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/eager/custom_training_walkthrough.ipynb b/site/en/r1/tutorials/eager/custom_training_walkthrough.ipynb
index dac5d8d2fa9..3989f3e44bc 100644
--- a/site/en/r1/tutorials/eager/custom_training_walkthrough.ipynb
+++ b/site/en/r1/tutorials/eager/custom_training_walkthrough.ipynb
@@ -1,25 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "custom_training_walkthrough",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "rwxGnsA92emp"
       },
       "source": [
@@ -28,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "CPII1rGR2rF9",
-        "colab": {}
+        "id": "CPII1rGR2rF9"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -46,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "JtEZ1pCPn--z"
       },
       "source": [
@@ -63,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "GV1F7tVTN3Dn"
       },
       "source": [
@@ -80,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "GV1F7tVTN3Dn"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "LDrzLFXE8T1l"
       },
       "source": [
@@ -109,7 +101,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "yNr7H-AIoLOR"
       },
       "source": [
@@ -119,7 +110,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "1J3AuPBT9gyR"
       },
       "source": [
@@ -132,31 +122,24 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "g4Wzg69bnwK2",
-        "colab": {}
+        "id": "g4Wzg69bnwK2"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
         "import os\n",
         "import matplotlib.pyplot as plt\n",
         "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.enable_eager_execution()\n",
+        "import tensorflow.compat.v1 as tf\n",
         "\n",
         "print(\"TensorFlow version: {}\".format(tf.__version__))\n",
         "print(\"Eager execution: {}\".format(tf.executing_eagerly()))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Zx7wc0LuuxaJ"
       },
       "source": [
@@ -186,7 +169,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3Px6KAg0Jowz"
       },
       "source": [
@@ -201,11 +183,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "J6c7uEU9rjRM",
-        "colab": {}
+        "id": "J6c7uEU9rjRM"
       },
+      "outputs": [],
       "source": [
         "train_dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv\"\n",
         "\n",
@@ -213,14 +195,11 @@
         "                                           origin=train_dataset_url)\n",
         "\n",
         "print(\"Local copy of the dataset file: {}\".format(train_dataset_fp))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "qnX1-aLors4S"
       },
       "source": [
@@ -231,21 +210,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "FQvb_JYdrpPm",
-        "colab": {}
+        "id": "FQvb_JYdrpPm"
       },
+      "outputs": [],
       "source": [
         "!head -n5 {train_dataset_fp}"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "kQhzD6P-uBoq"
       },
       "source": [
@@ -262,11 +238,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "9Edhevw7exl6",
-        "colab": {}
+        "id": "9Edhevw7exl6"
       },
+      "outputs": [],
       "source": [
         "# column order in CSV file\n",
         "column_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']\n",
@@ -276,14 +252,11 @@
         "\n",
         "print(\"Features: {}\".format(feature_names))\n",
         "print(\"Label: {}\".format(label_name))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "CCtwLoJhhDNc"
       },
       "source": [
@@ -298,21 +271,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "sVNlJlUOhkoX",
-        "colab": {}
+        "id": "sVNlJlUOhkoX"
       },
+      "outputs": [],
       "source": [
         "class_names = ['Iris setosa', 'Iris versicolor', 'Iris virginica']"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "dqPkQExM2Pwt"
       },
       "source": [
@@ -326,28 +296,25 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "WsxHnz1ebJ2S",
-        "colab": {}
+        "id": "WsxHnz1ebJ2S"
       },
+      "outputs": [],
       "source": [
         "batch_size = 32\n",
         "\n",
-        "train_dataset = tf.contrib.data.make_csv_dataset(\n",
+        "train_dataset = tf.data.experimental.make_csv_dataset(\n",
         "    train_dataset_fp,\n",
         "    batch_size,\n",
         "    column_names=column_names,\n",
         "    label_name=label_name,\n",
         "    num_epochs=1)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "gB_RSn62c-3G"
       },
       "source": [
@@ -358,23 +325,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "iDuG94H-C122",
-        "colab": {}
+        "id": "iDuG94H-C122"
       },
+      "outputs": [],
       "source": [
         "features, labels = next(iter(train_dataset))\n",
         "\n",
         "features"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "E63mArnQaAGz"
       },
       "source": [
@@ -385,11 +349,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "me5Wn-9FcyyO",
-        "colab": {}
+        "id": "me5Wn-9FcyyO"
       },
+      "outputs": [],
       "source": [
         "plt.scatter(features['petal_length'].numpy(),\n",
         "            features['sepal_length'].numpy(),\n",
@@ -399,14 +363,11 @@
         "plt.xlabel(\"Petal length\")\n",
         "plt.ylabel(\"Sepal length\")\n",
         "plt.show()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "YlxpSyHlhT6M"
       },
       "source": [
@@ -417,24 +378,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "jm932WINcaGU",
-        "colab": {}
+        "id": "jm932WINcaGU"
       },
+      "outputs": [],
       "source": [
         "def pack_features_vector(features, labels):\n",
         "  \"\"\"Pack the features into a single array.\"\"\"\n",
         "  features = tf.stack(list(features.values()), axis=1)\n",
         "  return features, labels"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "V1Vuph_eDl8x"
       },
       "source": [
@@ -443,21 +401,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ZbDkzGZIkpXf",
-        "colab": {}
+        "id": "ZbDkzGZIkpXf"
       },
+      "outputs": [],
       "source": [
         "train_dataset = train_dataset.map(pack_features_vector)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "NLy0Q1xCldVO"
       },
       "source": [
@@ -466,23 +421,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "kex9ibEek6Tr",
-        "colab": {}
+        "id": "kex9ibEek6Tr"
       },
+      "outputs": [],
       "source": [
         "features, labels = next(iter(train_dataset))\n",
         "\n",
         "print(features[:5])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "LsaVrtNM3Tx5"
       },
       "source": [
@@ -514,7 +466,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "W23DIMVPQEBt"
       },
       "source": [
@@ -527,25 +478,22 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "2fZ6oL2ig3ZK",
-        "colab": {}
+        "id": "2fZ6oL2ig3ZK"
       },
+      "outputs": [],
       "source": [
         "model = tf.keras.Sequential([\n",
         "  tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(4,)),  # input shape required\n",
         "  tf.keras.layers.Dense(10, activation=tf.nn.relu),\n",
         "  tf.keras.layers.Dense(3)\n",
         "])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "FHcbEzMpxbHL"
       },
       "source": [
@@ -557,7 +505,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2wFKnhWCpDSS"
       },
       "source": [
@@ -568,22 +515,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "xe6SQ5NrpB-I",
-        "colab": {}
+        "id": "xe6SQ5NrpB-I"
       },
+      "outputs": [],
       "source": [
         "predictions = model(features)\n",
         "predictions[:5]"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "wxyXOhwVr5S3"
       },
       "source": [
@@ -594,21 +538,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "_tRwHZmTNTX2",
-        "colab": {}
+        "id": "_tRwHZmTNTX2"
       },
+      "outputs": [],
       "source": [
         "tf.nn.softmax(predictions[:5])"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "uRZmchElo481"
       },
       "source": [
@@ -617,22 +558,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "-Jzm_GoErz8B",
-        "colab": {}
+        "id": "-Jzm_GoErz8B"
       },
+      "outputs": [],
       "source": [
         "print(\"Prediction: {}\".format(tf.argmax(predictions, axis=1)))\n",
         "print(\"    Labels: {}\".format(labels))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Vzq2E5J2QMtw"
       },
       "source": [
@@ -646,7 +584,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "RaKp8aEjKX6B"
       },
       "source": [
@@ -659,11 +596,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "tMAT4DcMPwI-",
-        "colab": {}
+        "id": "tMAT4DcMPwI-"
       },
+      "outputs": [],
       "source": [
         "def loss(model, x, y):\n",
         "  y_ = model(x)\n",
@@ -672,14 +609,11 @@
         "\n",
         "l = loss(model, features, labels)\n",
         "print(\"Loss test: {}\".format(l))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "3IcPqA24QM6B"
       },
       "source": [
@@ -688,24 +622,21 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "x57HcKWhKkei",
-        "colab": {}
+        "id": "x57HcKWhKkei"
       },
+      "outputs": [],
       "source": [
         "def grad(model, inputs, targets):\n",
         "  with tf.GradientTape() as tape:\n",
         "    loss_value = loss(model, inputs, targets)\n",
         "  return loss_value, tape.gradient(loss_value, model.trainable_variables)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "lOxFimtlKruu"
       },
       "source": [
@@ -729,7 +660,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "XkUd6UiZa_dF"
       },
       "source": [
@@ -738,23 +668,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "8xxi2NNGKwG_",
-        "colab": {}
+        "id": "8xxi2NNGKwG_"
       },
+      "outputs": [],
       "source": [
         "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n",
         "\n",
         "global_step = tf.Variable(0)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "pJVRZ0hP52ZB"
       },
       "source": [
@@ -763,11 +690,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "rxRNTFVe56RG",
-        "colab": {}
+        "id": "rxRNTFVe56RG"
       },
+      "outputs": [],
       "source": [
         "loss_value, grads = grad(model, features, labels)\n",
         "\n",
@@ -778,14 +705,11 @@
         "\n",
         "print(\"Step: {},         Loss: {}\".format(global_step.numpy(),\n",
         "                                          loss(model, features, labels).numpy()))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "7Y2VSELvwAvW"
       },
       "source": [
@@ -805,17 +729,14 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "AIgulGRUhpto",
-        "colab": {}
+        "id": "AIgulGRUhpto"
       },
+      "outputs": [],
       "source": [
         "## Note: Rerunning this cell uses the same model variables\n",
         "\n",
-        "from tensorflow import contrib\n",
-        "tfe = contrib.eager\n",
-        "\n",
         "# keep results for plotting\n",
         "train_loss_results = []\n",
         "train_accuracy_results = []\n",
@@ -823,8 +744,8 @@
         "num_epochs = 201\n",
         "\n",
         "for epoch in range(num_epochs):\n",
-        "  epoch_loss_avg = tfe.metrics.Mean()\n",
-        "  epoch_accuracy = tfe.metrics.Accuracy()\n",
+        "  epoch_loss_avg = tf.keras.metrics.Mean()\n",
+        "  epoch_accuracy = tf.keras.metrics.Accuracy()\n",
         "\n",
         "  # Training loop - using batches of 32\n",
         "  for x, y in train_dataset:\n",
@@ -846,14 +767,11 @@
         "    print(\"Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}\".format(epoch,\n",
         "                                                                epoch_loss_avg.result(),\n",
         "                                                                epoch_accuracy.result()))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "2FQHVUnm_rjw"
       },
       "source": [
@@ -863,7 +781,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "j3wdbmtLVTyr"
       },
       "source": [
@@ -874,11 +791,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "agjvNd2iUGFn",
-        "colab": {}
+        "id": "agjvNd2iUGFn"
       },
+      "outputs": [],
       "source": [
         "fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))\n",
         "fig.suptitle('Training Metrics')\n",
@@ -890,14 +807,11 @@
         "axes[1].set_xlabel(\"Epoch\", fontsize=14)\n",
         "axes[1].plot(train_accuracy_results)\n",
         "plt.show()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Zg8GoMZhLpGH"
       },
       "source": [
@@ -942,7 +856,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "z-EvK7hGL0d8"
       },
       "source": [
@@ -955,29 +868,27 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Ps3_9dJ3Lodk",
-        "colab": {}
+        "id": "Ps3_9dJ3Lodk"
       },
+      "outputs": [],
       "source": [
         "test_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv\"\n",
         "\n",
         "test_fp = tf.keras.utils.get_file(fname=os.path.basename(test_url),\n",
         "                                  origin=test_url)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "SRMWCu30bnxH",
-        "colab": {}
+        "id": "SRMWCu30bnxH"
       },
+      "outputs": [],
       "source": [
-        "test_dataset = tf.contrib.data.make_csv_dataset(\n",
+        "test_dataset = tf.data.experimental.make_csv_dataset(\n",
         "    test_fp,\n",
         "    batch_size,\n",
         "    column_names=column_names,\n",
@@ -986,14 +897,11 @@
         "    shuffle=False)\n",
         "\n",
         "test_dataset = test_dataset.map(pack_features_vector)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "HFuOKXJdMAdm"
       },
       "source": [
@@ -1004,13 +912,13 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "Tw03-MK1cYId",
-        "colab": {}
+        "id": "Tw03-MK1cYId"
       },
+      "outputs": [],
       "source": [
-        "test_accuracy = tfe.metrics.Accuracy()\n",
+        "test_accuracy = tf.keras.metrics.Accuracy()\n",
         "\n",
         "for (x, y) in test_dataset:\n",
         "  logits = model(x)\n",
@@ -1018,14 +926,11 @@
         "  test_accuracy(prediction, y)\n",
         "\n",
         "print(\"Test set accuracy: {:.3%}\".format(test_accuracy.result()))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "HcKEZMtCOeK-"
       },
       "source": [
@@ -1034,21 +939,18 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "uNwt2eMeOane",
-        "colab": {}
+        "id": "uNwt2eMeOane"
       },
+      "outputs": [],
       "source": [
         "tf.stack([y,prediction],axis=1)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "7Li2r1tYvW7S"
       },
       "source": [
@@ -1065,11 +967,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "kesTS5Lzv-M2",
-        "colab": {}
+        "id": "kesTS5Lzv-M2"
       },
+      "outputs": [],
       "source": [
         "predict_dataset = tf.convert_to_tensor([\n",
         "    [5.1, 3.3, 1.7, 0.5,],\n",
@@ -1084,9 +986,20 @@
         "  p = tf.nn.softmax(logits)[class_idx]\n",
         "  name = class_names[class_idx]\n",
         "  print(\"Example {} prediction: {} ({:4.1f}%)\".format(i, name, 100*p))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "custom_training_walkthrough.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/eager/eager_basics.ipynb b/site/en/r1/tutorials/eager/eager_basics.ipynb
index ead43f21611..acd00ec2e20 100644
--- a/site/en/r1/tutorials/eager/eager_basics.ipynb
+++ b/site/en/r1/tutorials/eager/eager_basics.ipynb
@@ -1,25 +1,8 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "eager_basics.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "iPpI7RaYoZuE"
       },
       "source": [
@@ -28,12 +11,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "form",
-        "colab_type": "code",
-        "id": "hro2InpHobKk",
-        "colab": {}
+        "id": "hro2InpHobKk"
       },
+      "outputs": [],
       "source": [
         "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -46,14 +29,11 @@
         "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
         "# See the License for the specific language governing permissions and\n",
         "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "U9i2Dsh-ziXr"
       },
       "source": [
@@ -63,7 +43,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Hndw-YcxoOJK"
       },
       "source": [
@@ -80,7 +59,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
+        "id": "Hndw-YcxoOJK"
+      },
+      "source": [
+        "> Note: This is an archived TF1 notebook. These are configured\n",
+        "to run in TF2's \n",
+        "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n",
+        "but will run in TF1 as well. To use TF1 in Colab, use the\n",
+        "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n",
+        "magic."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
         "id": "6sILUVbHoSgH"
       },
       "source": [
@@ -95,7 +87,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "z1JcS5iBXMRO"
       },
       "source": [
@@ -107,26 +98,19 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "code",
-        "colab_type": "code",
-        "id": "RlIWhyeLoYnG",
-        "colab": {}
+        "id": "RlIWhyeLoYnG"
       },
+      "outputs": [],
       "source": [
-        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
-        "\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "tf.enable_eager_execution()"
-      ],
-      "execution_count": 0,
-      "outputs": []
+        "import tensorflow.compat.v1 as tf\n"
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "H9UySOPLXdaw"
       },
       "source": [
@@ -137,12 +121,12 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "code",
-        "colab_type": "code",
-        "id": "ngUe237Wt48W",
-        "colab": {}
+        "id": "ngUe237Wt48W"
       },
+      "outputs": [],
       "source": [
         "print(tf.add(1, 2))\n",
         "print(tf.add([1, 2], [3, 4]))\n",
@@ -152,14 +136,11 @@
         "\n",
         "# Operator overloading is also supported\n",
         "print(tf.square(2) + tf.square(3))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "IDY4WsYRhP81"
       },
       "source": [
@@ -168,23 +149,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "srYWH1MdJNG7",
-        "colab": {}
+        "id": "srYWH1MdJNG7"
       },
+      "outputs": [],
       "source": [
         "x = tf.matmul([[1]], [[2, 3]])\n",
         "print(x.shape)\n",
         "print(x.dtype)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "eBPw8e8vrsom"
       },
       "source": [
@@ -197,7 +175,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "Dwi1tdW3JBw6"
       },
       "source": [
@@ -214,11 +191,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "lCUWzso6mbqR",
-        "colab": {}
+        "id": "lCUWzso6mbqR"
       },
+      "outputs": [],
       "source": [
         "import numpy as np\n",
         "\n",
@@ -234,14 +211,11 @@
         "\n",
         "print(\"The .numpy() method explicitly converts a Tensor to a numpy array\")\n",
         "print(tensor.numpy())"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "PBNP8yTRfu_X"
       },
       "source": [
@@ -252,28 +226,25 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
         "cellView": "code",
-        "colab_type": "code",
-        "id": "3Twf_Rw-gQFM",
-        "colab": {}
+        "id": "3Twf_Rw-gQFM"
       },
+      "outputs": [],
       "source": [
         "x = tf.random.uniform([3, 3])\n",
         "\n",
         "print(\"Is there a GPU available: \"),\n",
-        "print(tf.test.is_gpu_available())\n",
+        "print(tf.config.list_physical_devices('GPU'))\n",
         "\n",
         "print(\"Is the Tensor on GPU #0:  \"),\n",
         "print(x.device.endswith('GPU:0'))"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vpgYzgVXW2Ud"
       },
       "source": [
@@ -285,12 +256,9 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "ZWZQCimzuqyP"
       },
       "source": [
-        "\n",
-        "\n",
         "### Explicit Device Placement\n",
         "\n",
         "The term \"placement\" in TensorFlow refers to how individual operations are assigned (placed on) a device for execution. As mentioned above, when there is no explicit guidance provided, TensorFlow automatically decides which device to execute an operation, and copies Tensors to that device if needed. However, TensorFlow operations can be explicitly placed on specific devices using the `tf.device` context manager. For example:"
@@ -298,11 +266,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "RjkNZTuauy-Q",
-        "colab": {}
+        "id": "RjkNZTuauy-Q"
       },
+      "outputs": [],
       "source": [
         "import time\n",
         "\n",
@@ -324,19 +292,16 @@
         "  time_matmul(x)\n",
         "\n",
         "# Force execution on GPU #0 if available\n",
-        "if tf.test.is_gpu_available():\n",
+        "if tf.config.list_physical_devices('GPU'):\n",
         "  with tf.device(\"GPU:0\"): # Or GPU:1 for the 2nd GPU, GPU:2 for the 3rd etc.\n",
         "    x = tf.random_uniform([1000, 1000])\n",
         "    assert x.device.endswith(\"GPU:0\")\n",
         "    time_matmul(x)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "o1K4dlhhHtQj"
       },
       "source": [
@@ -357,7 +322,6 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "zI0fmOynH-Ne"
       },
       "source": [
@@ -368,11 +332,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "F04fVOHQIBiG",
-        "colab": {}
+        "id": "F04fVOHQIBiG"
       },
+      "outputs": [],
       "source": [
         "ds_tensors = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n",
         "\n",
@@ -387,14 +351,11 @@
         "  \"\"\")\n",
         "\n",
         "ds_file = tf.data.TextLineDataset(filename)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "vbxIhC-5IPdf"
       },
       "source": [
@@ -405,23 +366,20 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "uXSDZWE-ISsd",
-        "colab": {}
+        "id": "uXSDZWE-ISsd"
       },
+      "outputs": [],
       "source": [
         "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n",
         "\n",
         "ds_file = ds_file.batch(2)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
       "metadata": {
-        "colab_type": "text",
         "id": "A8X1GNfoIZKJ"
       },
       "source": [
@@ -433,11 +391,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
       "metadata": {
-        "colab_type": "code",
-        "id": "ws-WKRk5Ic6-",
-        "colab": {}
+        "id": "ws-WKRk5Ic6-"
       },
+      "outputs": [],
       "source": [
         "print('Elements of ds_tensors:')\n",
         "for x in ds_tensors:\n",
@@ -446,9 +404,20 @@
         "print('\\nElements in ds_file:')\n",
         "for x in ds_file:\n",
         "  print(x)"
-      ],
-      "execution_count": 0,
-      "outputs": []
+      ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "eager_basics.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/site/en/r1/tutorials/estimators/boosted_trees.ipynb b/site/en/r1/tutorials/estimators/boosted_trees.ipynb
deleted file mode 100644
index 83dda1103f9..00000000000
--- a/site/en/r1/tutorials/estimators/boosted_trees.ipynb
+++ /dev/null
@@ -1,696 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "name": "boosted_trees.ipynb",
-      "version": "0.3.2",
-      "provenance": [],
-      "private_outputs": true,
-      "collapsed_sections": [],
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "7765UFHoyGx6"
-      },
-      "source": [
-        "##### Copyright 2019 The TensorFlow Authors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "metadata": {
-        "cellView": "form",
-        "colab_type": "code",
-        "id": "KVtTDrUNyL7x",
-        "colab": {}
-      },
-      "source": [
-        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ],
-      "execution_count": 0,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "xPYxZMrWyA0N"
-      },
-      "source": [
-        "#How to train Boosted Trees models in TensorFlow"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "p_vOREjRx-Y0"
-      },
-      "source": [
-        "
    VersionPython versionCompilerBuild toolscuDNNCUDA
    tensorflow_gpu-2.10.03.7-3.10MSVC 2019Bazel 5.1.18.111.2
    tensorflow_gpu-2.9.03.7-3.10MSVC 2019Bazel 5.0.08.111.2
    tensorflow_gpu-2.8.03.7-3.10MSVC 2019Bazel 4.2.18.111.2
    tensorflow_gpu-2.7.03.7-3.9MSVC 2019Bazel 3.7.28.111.2
    tensorflow_gpu-2.6.03.6-3.9MSVC 2019Bazel 3.7.28.111.2
    tensorflow_gpu-2.5.03.6-3.9MSVC 2019Bazel 3.7.28.111.2
    tensorflow_gpu-2.4.03.6-3.8MSVC 2019Bazel 3.1.08.011.0
    tensorflow_gpu-2.3.03.5-3.8MSVC 2019Bazel 3.1.07.610.1
    tensorflow_gpu-2.2.03.5-3.8MSVC 2019Bazel 2.0.07.610.1
    tensorflow_gpu-2.1.03.5-3.7MSVC 2019Bazel 0.27.1-0.29.17.610.1
    tensorflow_gpu-2.0.03.5-3.7MSVC 2017Bazel 0.26.17.410
    tensorflow_gpu-1.15.03.5-3.7MSVC 2017Bazel 0.26.17.410
    tensorflow_gpu-1.14.03.5-3.7MSVC 2017Bazel 0.24.1-0.25.27.410
    tensorflow_gpu-1.13.03.5-3.7MSVC 2015 update 3Bazel 0.19.0-0.21.07.410
    tensorflow_gpu-1.12.03.5-3.6MSVC 2015 update 3Bazel 0.15.079
    tensorflow_gpu-1.12.03.5-3.6MSVC 2015 update 3Bazel 0.15.07.29.0
    tensorflow_gpu-1.11.03.5-3.6MSVC 2015 update 3Bazel 0.15.079
    tensorflow_gpu-1.10.03.5-3.6MSVC 2015 update 3Cmake v3.6.379
    tensorflow_gpu-1.9.03.5-3.6MSVC 2015 update 3Cmake v3.6.379
    \n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dW3r7qVxzqN5" - }, - "source": [ - "This tutorial is an end-to-end walkthrough of training a Gradient Boosting model using decision trees with the `tf.estimator` API. Boosted Trees models are among the most popular and effective machine learning approaches for both regression and classification. It is an ensemble technique that combines the predictions from several (think 10s, 100s or even 1000s) tree models.\n", - "\n", - "Boosted Trees models are popular with many machine learning practioners as they can achieve impressive performance with minimal hyperparameter tuning." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eylrTPAN3rJV" - }, - "source": [ - "## Load the titanic dataset\n", - "You will be using the titanic dataset, where the (rather morbid) goal is to predict passenger survival, given characteristics such as gender, age, class, etc." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KuhAiPfZ3rJW", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "from matplotlib import pyplot as plt\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()\n", - "\n", - "tf.logging.set_verbosity(tf.logging.ERROR)\n", - "tf.set_random_seed(123)\n", - "\n", - "# Load dataset.\n", - "dftrain = pd.read_csv('https://storage.googleapis.com/tfbt/titanic_train.csv')\n", - "dfeval = pd.read_csv('https://storage.googleapis.com/tfbt/titanic_eval.csv')\n", - "y_train = dftrain.pop('survived')\n", - "y_eval = dfeval.pop('survived')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3ioodHdVJVdA" - }, - "source": [ - "The dataset consists of a training set and an evaluation set:\n", - "\n", - "* `dftrain` and `y_train` are the *training set*—the data the model uses to learn.\n", - "* The model is tested against the *eval set*, `dfeval`, and `y_eval`.\n", - "\n", - "For training you will use the following features:\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Feature NameDescription
    sexGender of passenger
    ageAge of passenger
    n_siblings_spouses# siblings and partners aboard
    parch# of parents and children aboard
    fareFare passenger paid.
    classPassenger's class on ship
    deckWhich deck passenger was on
    embark_townWhich town passenger embarked from
    aloneIf passenger was alone
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AoPiWsJALr-k" - }, - "source": [ - "## Explore the data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "slcat1yzmzw5" - }, - "source": [ - "Let's first preview some of the data and create summary statistics on the training set." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "15PLelXBlxEW", - "colab": {} - }, - "source": [ - "dftrain.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "j2hiM4ETmqP0", - "colab": {} - }, - "source": [ - "dftrain.describe()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-IR0e8V-LyJ4" - }, - "source": [ - "There are 627 and 264 examples in the training and evaluation sets, respectively." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_1NwYqGwDjFf", - "colab": {} - }, - "source": [ - "dftrain.shape[0], dfeval.shape[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "28UFJ4KSMK3V" - }, - "source": [ - "The majority of passengers are in their 20's and 30's." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CaVDmZtuDfux", - "colab": {} - }, - "source": [ - "dftrain.age.hist(bins=20)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1pifWiCoMbR5" - }, - "source": [ - "There are approximately twice as male passengers as female passengers aboard." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-WazAq30MO5J", - "colab": {} - }, - "source": [ - "dftrain.sex.value_counts().plot(kind='barh')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7_XkxrpmmVU_" - }, - "source": [ - "The majority of passengers were in the \"third\" class." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zZ3PvVy4l4gI", - "colab": {} - }, - "source": [ - "(dftrain['class']\n", - " .value_counts()\n", - " .plot(kind='barh'))\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HM5SlwlxmZMT" - }, - "source": [ - "Most passengers embarked from Southampton." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RVTSrdr4mZaC", - "colab": {} - }, - "source": [ - "(dftrain['embark_town']\n", - " .value_counts()\n", - " .plot(kind='barh'))\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aTn1niLPob3x" - }, - "source": [ - "Females have a much higher chance of surviving vs. males. This will clearly be a predictive feature for the model." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Eh3KW5oYkaNS", - "colab": {} - }, - "source": [ - "ax = (pd.concat([dftrain, y_train], axis=1)\\\n", - " .groupby('sex')\n", - " .survived\n", - " .mean()\n", - " .plot(kind='barh'))\n", - "ax.set_xlabel('% survive')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "krkRHuMp3rJn" - }, - "source": [ - "## Create feature columns and input functions\n", - "The Gradient Boosting estimator can utilize both numeric and categorical features. Feature columns work with all TensorFlow estimators and their purpose is to define the features used for modeling. Additionally they provide some feature engineering capabilities like one-hot-encoding, normalization, and bucketization. In this tutorial, the fields in `CATEGORICAL_COLUMNS` are transformed from categorical columns to one-hot-encoded columns ([indicator column](https://www.tensorflow.org/api_docs/python/tf/feature_column/indicator_column)):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "upaNWxcF3rJn", - "colab": {} - }, - "source": [ - "fc = tf.feature_column\n", - "CATEGORICAL_COLUMNS = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',\n", - " 'embark_town', 'alone']\n", - "NUMERIC_COLUMNS = ['age', 'fare']\n", - "\n", - "def one_hot_cat_column(feature_name, vocab):\n", - " return fc.indicator_column(\n", - " fc.categorical_column_with_vocabulary_list(feature_name,\n", - " vocab))\n", - "feature_columns = []\n", - "for feature_name in CATEGORICAL_COLUMNS:\n", - " # Need to one-hot encode categorical features.\n", - " vocabulary = dftrain[feature_name].unique()\n", - " feature_columns.append(one_hot_cat_column(feature_name, vocabulary))\n", - "\n", - "for feature_name in NUMERIC_COLUMNS:\n", - " feature_columns.append(fc.numeric_column(feature_name,\n", - " dtype=tf.float32))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "74GNtFpStSAz" - }, - "source": [ - "You can view the transformation that a feature column produces. For example, here is the output when using the `indicator_column` on a single example:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Eaq79D9FtmF8", - "colab": {} - }, - "source": [ - "example = dftrain.head(1)\n", - "class_fc = one_hot_cat_column('class', ('First', 'Second', 'Third'))\n", - "print('Feature value: \"{}\"'.format(example['class'].iloc[0]))\n", - "print('One-hot encoded: ', fc.input_layer(dict(example), [class_fc]).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YbCUn3nCusC3" - }, - "source": [ - "Additionally, you can view all of the feature column transformations together:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "omIYcsVws3g0", - "colab": {} - }, - "source": [ - "fc.input_layer(dict(example), feature_columns).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-UOlROp33rJo" - }, - "source": [ - "Next you need to create the input functions. These will specify how data will be read into our model for both training and inference. You will use the `from_tensor_slices` method in the [`tf.data`](https://www.tensorflow.org/api_docs/python/tf/data) API to read in data directly from Pandas. This is suitable for smaller, in-memory datasets. For larger datasets, the tf.data API supports a variety of file formats (including [csv](https://www.tensorflow.org/api_docs/python/tf/data/experimental/make_csv_dataset)) so that you can process datasets that do not fit in memory." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9dquwCQB3rJp", - "colab": {} - }, - "source": [ - "# Use entire batch since this is such a small dataset.\n", - "NUM_EXAMPLES = len(y_train)\n", - "\n", - "def make_input_fn(X, y, n_epochs=None, shuffle=True):\n", - " y = np.expand_dims(y, axis=1)\n", - " def input_fn():\n", - " dataset = tf.data.Dataset.from_tensor_slices((dict(X), y))\n", - " if shuffle:\n", - " dataset = dataset.shuffle(NUM_EXAMPLES)\n", - " # For training, cycle thru dataset as many times as need (n_epochs=None).\n", - " dataset = dataset.repeat(n_epochs)\n", - " # In memory training doesn't use batching.\n", - " dataset = dataset.batch(NUM_EXAMPLES)\n", - " return dataset\n", - " return input_fn\n", - "\n", - "# Training and evaluation input functions.\n", - "train_input_fn = make_input_fn(dftrain, y_train)\n", - "eval_input_fn = make_input_fn(dfeval, y_eval, shuffle=False, n_epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HttfNNlN3rJr" - }, - "source": [ - "## Train and evaluate the model\n", - "\n", - "Below you will do the following steps:\n", - "\n", - "1. Initialize the model, specifying the features and hyperparameters.\n", - "2. Feed the training data to the model using the `train_input_fn` and train the model using the `train` function.\n", - "3. You will assess model performance using the evaluation set—in this example, the `dfeval` DataFrame. You will verify that the predictions match the labels from the `y_eval` array.\n", - "\n", - "Before training a Boosted Trees model, let's first train a linear classifier (logistic regression model). It is best practice to start with simpler model to establish a benchmark." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JPOGpmmq3rJr", - "colab": {} - }, - "source": [ - "linear_est = tf.estimator.LinearClassifier(feature_columns)\n", - "\n", - "# Train model.\n", - "linear_est.train(train_input_fn, max_steps=100)\n", - "\n", - "# Evaluation.\n", - "results = linear_est.evaluate(eval_input_fn)\n", - "print('Accuracy : ', results['accuracy'])\n", - "print('Dummy model: ', results['accuracy_baseline'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BarkNXwA3rJu" - }, - "source": [ - "Next let's train a Boosted Trees model. For boosted trees, regression (`BoostedTreesRegressor`) and classification (`BoostedTreesClassifier`) are supported, along with using any twice differentiable custom loss (`BoostedTreesEstimator`). Since the goal is to predict a class - survive or not survive, you will use the `BoostedTreesClassifier`.\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tgEzMtlw3rJu", - "colab": {} - }, - "source": [ - "# Since data fits into memory, use entire dataset per layer. It will be faster.\n", - "# Above one batch is defined as the entire dataset.\n", - "n_batches = 1\n", - "est = tf.estimator.BoostedTreesClassifier(feature_columns,\n", - " n_batches_per_layer=n_batches)\n", - "\n", - "# The model will stop training once the specified number of trees is built, not\n", - "# based on the number of steps.\n", - "est.train(train_input_fn, max_steps=100)\n", - "\n", - "# Eval.\n", - "results = est.evaluate(eval_input_fn)\n", - "print('Accuracy : ', results['accuracy'])\n", - "print('Dummy model: ', results['accuracy_baseline'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cUrakbu6sqKe" - }, - "source": [ - "For performance reasons, when your data fits in memory, it is recommended to use the `boosted_trees_classifier_train_in_memory` function. However if training time is not of a concern or if you have a very large dataset and want to do distributed training, use the `tf.estimator.BoostedTrees` API shown above.\n", - "\n", - "\n", - "When using this method, you should not batch your input data, as the method operates on the entire dataset.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-4_xz3b_D0W5", - "colab": {} - }, - "source": [ - "def make_inmemory_train_input_fn(X, y):\n", - " y = np.expand_dims(y, axis=1)\n", - " def input_fn():\n", - " return dict(X), y\n", - " return input_fn\n", - "\n", - "\n", - "train_input_fn = make_inmemory_train_input_fn(dftrain, y_train)\n", - "eval_input_fn = make_input_fn(dfeval, y_eval, shuffle=False, n_epochs=1)\n", - "est = tf.contrib.estimator.boosted_trees_classifier_train_in_memory(\n", - " train_input_fn,\n", - " feature_columns)\n", - "print(est.evaluate(eval_input_fn)['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hEflwznXvuMP" - }, - "source": [ - "Now you can use the train model to make predictions on a passenger from the evaluation set. TensorFlow models are optimized to make predictions on a batch, or collection, of examples at once. Earlier, the `eval_input_fn` is defined using the entire evaluation set." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6zmIjTr73rJ4", - "colab": {} - }, - "source": [ - "pred_dicts = list(est.predict(eval_input_fn))\n", - "probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])\n", - "\n", - "probs.plot(kind='hist', bins=20, title='predicted probabilities')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBUaNN1BzJHG" - }, - "source": [ - "Finally you can also look at the receiver operating characteristic (ROC) of the results, which will give us a better idea of the tradeoff between the true positive rate and false positive rate." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NzxghvVz3rJ6", - "colab": {} - }, - "source": [ - "from sklearn.metrics import roc_curve\n", - "\n", - "fpr, tpr, _ = roc_curve(y_eval, probs)\n", - "plt.plot(fpr, tpr)\n", - "plt.title('ROC curve')\n", - "plt.xlabel('false positive rate')\n", - "plt.ylabel('true positive rate')\n", - "plt.xlim(0,)\n", - "plt.ylim(0,)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/en/r1/tutorials/estimators/boosted_trees_model_understanding.ipynb b/site/en/r1/tutorials/estimators/boosted_trees_model_understanding.ipynb deleted file mode 100644 index 3ed953d5ae5..00000000000 --- a/site/en/r1/tutorials/estimators/boosted_trees_model_understanding.ipynb +++ /dev/null @@ -1,1114 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "boosted_trees_model_understanding.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7765UFHoyGx6" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KVtTDrUNyL7x", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r0_fqL3ayLHX" - }, - "source": [ - "# Gradient Boosted Trees: Model understanding" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PS6_yKSoyLAl" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dW3r7qVxzqN5" - }, - "source": [ - "\n", - "For an end-to-end walkthrough of training a Gradient Boosting model check out the [boosted trees tutorial](https://www.tensorflow.org/r1/tutorials/estimators/boosted_trees). In this tutorial you will:\n", - "\n", - "* Learn how to interpret a Boosted Trees model both *locally* and *globally*\n", - "* Gain intution for how a Boosted Trees model fits a dataset\n", - "\n", - "## How to interpret Boosted Trees models both locally and globally\n", - "\n", - "Local interpretability refers to an understanding of a model’s predictions at the individual example level, while global interpretability refers to an understanding of the model as a whole. Such techniques can help machine learning (ML) practitioners detect bias and bugs during the model development stage\n", - "\n", - "For local interpretability, you will learn how to create and visualize per-instance contributions. To distinguish this from feature importances, we refer to these values as directional feature contributions (DFCs).\n", - "\n", - "For global interpretability you will retrieve and visualize gain-based feature importances, [permutation feature importances](https://www.stat.berkeley.edu/~breiman/randomforest2001.pdf) and also show aggregated DFCs." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eylrTPAN3rJV" - }, - "source": [ - "## Load the titanic dataset\n", - "You will be using the titanic dataset, where the (rather morbid) goal is to predict passenger survival, given characteristics such as gender, age, class, etc." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KuhAiPfZ3rJW", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import tensorflow as tf\n", - "\n", - "tf.logging.set_verbosity(tf.logging.ERROR)\n", - "tf.set_random_seed(123)\n", - "\n", - "# Load dataset.\n", - "dftrain = pd.read_csv('https://storage.googleapis.com/tfbt/titanic_train.csv')\n", - "dfeval = pd.read_csv('https://storage.googleapis.com/tfbt/titanic_eval.csv')\n", - "y_train = dftrain.pop('survived')\n", - "y_eval = dfeval.pop('survived')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3ioodHdVJVdA" - }, - "source": [ - "For a description of the features, please review the prior tutorial." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "krkRHuMp3rJn" - }, - "source": [ - "## Create feature columns, input_fn, and the train the estimator" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JiJ6K3hr1lXW" - }, - "source": [ - "### Preprocess the data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "udMytRJC05oW" - }, - "source": [ - "Create the feature columns, using the original numeric columns as is and one-hot-encoding categorical variables." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "upaNWxcF3rJn", - "colab": {} - }, - "source": [ - "fc = tf.feature_column\n", - "CATEGORICAL_COLUMNS = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',\n", - " 'embark_town', 'alone']\n", - "NUMERIC_COLUMNS = ['age', 'fare']\n", - "\n", - "def one_hot_cat_column(feature_name, vocab):\n", - " return fc.indicator_column(\n", - " fc.categorical_column_with_vocabulary_list(feature_name,\n", - " vocab))\n", - "feature_columns = []\n", - "for feature_name in CATEGORICAL_COLUMNS:\n", - " # Need to one-hot encode categorical features.\n", - " vocabulary = dftrain[feature_name].unique()\n", - " feature_columns.append(one_hot_cat_column(feature_name, vocabulary))\n", - "\n", - "for feature_name in NUMERIC_COLUMNS:\n", - " feature_columns.append(fc.numeric_column(feature_name,\n", - " dtype=tf.float32))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9rTefnXe1n0v" - }, - "source": [ - "### Build the input pipeline" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-UOlROp33rJo" - }, - "source": [ - "Create the input functions using the `from_tensor_slices` method in the [`tf.data`](https://www.tensorflow.org/api_docs/python/tf/data) API to read in data directly from Pandas." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9dquwCQB3rJp", - "colab": {} - }, - "source": [ - "# Use entire batch since this is such a small dataset.\n", - "NUM_EXAMPLES = len(y_train)\n", - "\n", - "def make_input_fn(X, y, n_epochs=None, shuffle=True):\n", - " y = np.expand_dims(y, axis=1)\n", - " def input_fn():\n", - " dataset = tf.data.Dataset.from_tensor_slices((X.to_dict(orient='list'), y))\n", - " if shuffle:\n", - " dataset = dataset.shuffle(NUM_EXAMPLES)\n", - " # For training, cycle thru dataset as many times as need (n_epochs=None).\n", - " dataset = (dataset\n", - " .repeat(n_epochs)\n", - " .batch(NUM_EXAMPLES))\n", - " return dataset\n", - " return input_fn\n", - "\n", - "# Training and evaluation input functions.\n", - "train_input_fn = make_input_fn(dftrain, y_train)\n", - "eval_input_fn = make_input_fn(dfeval, y_eval, shuffle=False, n_epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HttfNNlN3rJr" - }, - "source": [ - "### Train the model" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tgEzMtlw3rJu", - "colab": {} - }, - "source": [ - "params = {\n", - " 'n_trees': 50,\n", - " 'max_depth': 3,\n", - " 'n_batches_per_layer': 1,\n", - " # You must enable center_bias = True to get DFCs. This will force the model to\n", - " # make an initial prediction before using any features (e.g. use the mean of\n", - " # the training labels for regression or log odds for classification when\n", - " # using cross entropy loss).\n", - " 'center_bias': True\n", - "}\n", - "\n", - "est = tf.estimator.BoostedTreesClassifier(feature_columns, **params)\n", - "est.train(train_input_fn, max_steps=100)\n", - "results = est.evaluate(eval_input_fn)\n", - "pd.Series(results).to_frame()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cUrakbu6sqKe" - }, - "source": [ - "For performance reasons, when your data fits in memory, we recommend use the `boosted_trees_classifier_train_in_memory` function. However if training time is not of a concern or if you have a very large dataset and want to do distributed training, use the `tf.estimator.BoostedTrees` API shown above.\n", - "\n", - "\n", - "When using this method, you should not batch your input data, as the method operates on the entire dataset.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-4_xz3b_D0W5", - "colab": {} - }, - "source": [ - "in_memory_params = dict(params)\n", - "del in_memory_params['n_batches_per_layer']\n", - "# In-memory input_fn does not use batching.\n", - "def make_inmemory_train_input_fn(X, y):\n", - " y = np.expand_dims(y, axis=1)\n", - " def input_fn():\n", - " return dict(X), y\n", - " return input_fn\n", - "train_input_fn = make_inmemory_train_input_fn(dftrain, y_train)\n", - "\n", - "# Train the model.\n", - "est = tf.contrib.estimator.boosted_trees_classifier_train_in_memory(\n", - " train_input_fn,\n", - " feature_columns,\n", - " **in_memory_params)\n", - "print(est.evaluate(eval_input_fn))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TSZYqNcRuczV" - }, - "source": [ - "## Model interpretation and plotting" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BjcfLiI3uczW", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "sns_colors = sns.color_palette('colorblind')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ywTtbBvBuczY" - }, - "source": [ - "## Local interpretability\n", - "Next you will output the directional feature contributions (DFCs) to explain individual predictions using the approach outlined in [Palczewska et al](https://arxiv.org/pdf/1312.1121.pdf) and by Saabas in [Interpreting Random Forests](http://blog.datadive.net/interpreting-random-forests/) (this method is also available in scikit-learn for Random Forests in the [`treeinterpreter`](https://github.com/andosa/treeinterpreter) package). The DFCs are generated with:\n", - "\n", - "`pred_dicts = list(est.experimental_predict_with_explanations(pred_input_fn))`\n", - "\n", - "(Note: The method is named experimental as we may modify the API before dropping the experimental prefix.)" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TIL93B4sDRqE", - "colab": {} - }, - "source": [ - "pred_dicts = list(est.experimental_predict_with_explanations(eval_input_fn))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tDPoRx_ZaY1E", - "colab": {} - }, - "source": [ - "# Create DFC Pandas dataframe.\n", - "labels = y_eval.values\n", - "probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])\n", - "df_dfc = pd.DataFrame([pred['dfc'] for pred in pred_dicts])\n", - "df_dfc.describe().T" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EUKSaVoraY1C" - }, - "source": [ - "A nice property of DFCs is that the sum of the contributions + the bias is equal to the prediction for a given example." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Hd9VuizRaY1H", - "colab": {} - }, - "source": [ - "# Sum of DFCs + bias == probabality.\n", - "bias = pred_dicts[0]['bias']\n", - "dfc_prob = df_dfc.sum(axis=1) + bias\n", - "np.testing.assert_almost_equal(dfc_prob.values,\n", - " probs.values)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uIC7qm1gaY1L" - }, - "source": [ - "Plot DFCs for an individual passenger." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "P3u971LsuczZ", - "colab": {} - }, - "source": [ - "# Plot results.\n", - "ID = 182\n", - "example = df_dfc.iloc[ID] # Choose ith example from evaluation set.\n", - "TOP_N = 8 # View top 8 features.\n", - "sorted_ix = example.abs().sort_values()[-TOP_N:].index\n", - "ax = example[sorted_ix].plot(kind='barh', color=sns_colors[3])\n", - "ax.grid(False, axis='y')\n", - "\n", - "ax.set_title('Feature contributions for example {}\\n pred: {:1.2f}; label: {}'.format(ID, probs[ID], labels[ID]))\n", - "ax.set_xlabel('Contribution to predicted probability')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4i4mjK66FYg" - }, - "source": [ - "The larger magnitude contributions have a larger impact on the model's prediction. Negative contributions indicate the feature value for this given example reduced the model's prediction, while positive values contribute an increase in the prediction." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tx5p4vEhuczg" - }, - "source": [ - "### Improved plotting\n", - "Let's make the plot nice by color coding based on the contributions' directionality and add the feature values on figure." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6z_Tq1Pquczj", - "colab": {} - }, - "source": [ - "# Boilerplate code for plotting :)\n", - "def _get_color(value):\n", - " \"\"\"To make positive DFCs plot green, negative DFCs plot red.\"\"\"\n", - " green, red = sns.color_palette()[2:4]\n", - " if value >= 0: return green\n", - " return red\n", - "\n", - "def _add_feature_values(feature_values, ax):\n", - " \"\"\"Display feature's values on left of plot.\"\"\"\n", - " x_coord = ax.get_xlim()[0]\n", - " OFFSET = 0.15\n", - " for y_coord, (feat_name, feat_val) in enumerate(feature_values.items()):\n", - " t = plt.text(x_coord, y_coord - OFFSET, '{}'.format(feat_val), size=12)\n", - " t.set_bbox(dict(facecolor='white', alpha=0.5))\n", - " from matplotlib.font_manager import FontProperties\n", - " font = FontProperties()\n", - " font.set_weight('bold')\n", - " t = plt.text(x_coord, y_coord + 1 - OFFSET, 'feature\\nvalue',\n", - " fontproperties=font, size=12)\n", - "\n", - "def plot_example(example):\n", - " TOP_N = 8 # View top 8 features.\n", - " sorted_ix = example.abs().sort_values()[-TOP_N:].index # Sort by magnitude.\n", - " example = example[sorted_ix]\n", - " colors = example.map(_get_color).tolist()\n", - " ax = example.to_frame().plot(kind='barh',\n", - " color=[colors],\n", - " legend=None,\n", - " alpha=0.75,\n", - " figsize=(10,6))\n", - " ax.grid(False, axis='y')\n", - " ax.set_yticklabels(ax.get_yticklabels(), size=14)\n", - "\n", - " # Add feature values.\n", - " _add_feature_values(dfeval.iloc[ID][sorted_ix], ax)\n", - " return ax" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FlrsuOu8-Yds" - }, - "source": [ - "Plot example." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ht1P2-1euczk", - "colab": {} - }, - "source": [ - "example = df_dfc.iloc[ID] # Choose IDth example from evaluation set.\n", - "ax = plot_example(example)\n", - "ax.set_title('Feature contributions for example {}\\n pred: {:1.2f}; label: {}'.format(ID, probs[ID], labels[ID]))\n", - "ax.set_xlabel('Contribution to predicted probability', size=14)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0swvlkZFaY1Z" - }, - "source": [ - "You can also plot the example's DFCs compare with the entire distribution using a voilin plot." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zo7rNd1v_5e2", - "colab": {} - }, - "source": [ - "# Boilerplate plotting code.\n", - "def dist_violin_plot(df_dfc, ID):\n", - " # Initialize plot.\n", - " fig, ax = plt.subplots(1, 1, figsize=(10, 6))\n", - "\n", - " # Create example dataframe.\n", - " TOP_N = 8 # View top 8 features.\n", - " example = df_dfc.iloc[ID]\n", - " ix = example.abs().sort_values()[-TOP_N:].index\n", - " example = example[ix]\n", - " example_df = example.to_frame(name='dfc')\n", - "\n", - " # Add contributions of entire distribution.\n", - " parts=ax.violinplot([df_dfc[w] for w in ix],\n", - " vert=False,\n", - " showextrema=False,\n", - " widths=0.7,\n", - " positions=np.arange(len(ix)))\n", - " face_color = sns_colors[0]\n", - " alpha = 0.15\n", - " for pc in parts['bodies']:\n", - " pc.set_facecolor(face_color)\n", - " pc.set_alpha(alpha)\n", - "\n", - " # Add feature values.\n", - " _add_feature_values(dfeval.iloc[ID][sorted_ix], ax)\n", - "\n", - " # Add local contributions.\n", - " ax.scatter(example,\n", - " np.arange(example.shape[0]),\n", - " color=sns.color_palette()[2],\n", - " s=100,\n", - " marker=\"s\",\n", - " label='contributions for example')\n", - "\n", - " # Legend\n", - " # Proxy plot, to show violinplot dist on legend.\n", - " ax.plot([0,0], [1,1], label='eval set contributions\\ndistributions',\n", - " color=face_color, alpha=alpha, linewidth=10)\n", - " legend = ax.legend(loc='lower right', shadow=True, fontsize='x-large',\n", - " frameon=True)\n", - " legend.get_frame().set_facecolor('white')\n", - "\n", - " # Format plot.\n", - " ax.set_yticks(np.arange(example.shape[0]))\n", - " ax.set_yticklabels(example.index)\n", - " ax.grid(False, axis='y')\n", - " ax.set_xlabel('Contribution to predicted probability', size=14)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PiLw2tlm_9aK" - }, - "source": [ - "Plot this example." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VkCqraA2uczm", - "colab": {} - }, - "source": [ - "dist_violin_plot(df_dfc, ID)\n", - "plt.title('Feature contributions for example {}\\n pred: {:1.2f}; label: {}'.format(ID, probs[ID], labels[ID]))\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TVJFM85SAWVq" - }, - "source": [ - "Finally, third-party tools, such as [LIME](https://github.com/marcotcr/lime) and [shap](https://github.com/slundberg/shap), can also help understand individual predictions for a model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PnNXH6mZuczr" - }, - "source": [ - "## Global feature importances\n", - "\n", - "Additionally, you might want to understand the model as a whole, rather than studying individual predictions. Below, you will compute and use:\n", - "\n", - "* Gain-based feature importances using `est.experimental_feature_importances`\n", - "* Permutation importances\n", - "* Aggregate DFCs using `est.experimental_predict_with_explanations`\n", - "\n", - "Gain-based feature importances measure the loss change when splitting on a particular feature, while permutation feature importances are computed by evaluating model performance on the evaluation set by shuffling each feature one-by-one and attributing the change in model performance to the shuffled feature.\n", - "\n", - "In general, permutation feature importance are preferred to gain-based feature importance, though both methods can be unreliable in situations where potential predictor variables vary in their scale of measurement or their number of categories and when features are correlated ([source](https://bmcbioinformatics.biomedcentral.com/articles/10.1186/1471-2105-9-307)). Check out [this article](http://explained.ai/rf-importance/index.html) for an in-depth overview and great discussion on different feature importance types." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3ocBcMatuczs" - }, - "source": [ - "### Gain-based feature importances" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gMaxCgPbBJ-j" - }, - "source": [ - "Gain-based feature importances are built into the TensorFlow Boosted Trees estimators using `est.experimental_feature_importances`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pPTxbAaeuczt", - "colab": {} - }, - "source": [ - "importances = est.experimental_feature_importances(normalize=True)\n", - "df_imp = pd.Series(importances)\n", - "\n", - "# Visualize importances.\n", - "N = 8\n", - "ax = (df_imp.iloc[0:N][::-1]\n", - " .plot(kind='barh',\n", - " color=sns_colors[0],\n", - " title='Gain feature importances',\n", - " figsize=(10, 6)))\n", - "ax.grid(False, axis='y')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GvfAcBeGuczw" - }, - "source": [ - "### Average absolute DFCs\n", - "You can also average the absolute values of DFCs to understand impact at a global level." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JkvAWLWLuczx", - "colab": {} - }, - "source": [ - "# Plot.\n", - "dfc_mean = df_dfc.abs().mean()\n", - "N = 8\n", - "sorted_ix = dfc_mean.abs().sort_values()[-N:].index # Average and sort by absolute.\n", - "ax = dfc_mean[sorted_ix].plot(kind='barh',\n", - " color=sns_colors[1],\n", - " title='Mean |directional feature contributions|',\n", - " figsize=(10, 6))\n", - "ax.grid(False, axis='y')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z0k_DvPLaY1o" - }, - "source": [ - "You can also see how DFCs vary as a feature value varies." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZcIfN1IpaY1o", - "colab": {} - }, - "source": [ - "FEATURE = 'fare'\n", - "feature = pd.Series(df_dfc[FEATURE].values, index=dfeval[FEATURE].values).sort_index()\n", - "ax = sns.regplot(feature.index.values, feature.values, lowess=True)\n", - "ax.set_ylabel('contribution')\n", - "ax.set_xlabel(FEATURE)\n", - "ax.set_xlim(0, 100)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lbpG72ULucz0" - }, - "source": [ - "### Permutation feature importance" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6esOw1VOucz0", - "colab": {} - }, - "source": [ - "def permutation_importances(est, X_eval, y_eval, metric, features):\n", - " \"\"\"Column by column, shuffle values and observe effect on eval set.\n", - "\n", - " source: http://explained.ai/rf-importance/index.html\n", - " A similar approach can be done during training. See \"Drop-column importance\"\n", - " in the above article.\"\"\"\n", - " baseline = metric(est, X_eval, y_eval)\n", - " imp = []\n", - " for col in features:\n", - " save = X_eval[col].copy()\n", - " X_eval[col] = np.random.permutation(X_eval[col])\n", - " m = metric(est, X_eval, y_eval)\n", - " X_eval[col] = save\n", - " imp.append(baseline - m)\n", - " return np.array(imp)\n", - "\n", - "def accuracy_metric(est, X, y):\n", - " \"\"\"TensorFlow estimator accuracy.\"\"\"\n", - " eval_input_fn = make_input_fn(X,\n", - " y=y,\n", - " shuffle=False,\n", - " n_epochs=1)\n", - " return est.evaluate(input_fn=eval_input_fn)['accuracy']\n", - "features = CATEGORICAL_COLUMNS + NUMERIC_COLUMNS\n", - "importances = permutation_importances(est, dfeval, y_eval, accuracy_metric,\n", - " features)\n", - "df_imp = pd.Series(importances, index=features)\n", - "\n", - "sorted_ix = df_imp.abs().sort_values().index\n", - "ax = df_imp[sorted_ix][-5:].plot(kind='barh', color=sns_colors[2], figsize=(10, 6))\n", - "ax.grid(False, axis='y')\n", - "ax.set_title('Permutation feature importance')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E236y3pVEzHg" - }, - "source": [ - "## Visualizing model fitting" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TrcQ-839EzZ6" - }, - "source": [ - "Lets first simulate/create training data using the following formula:\n", - "\n", - "\n", - "$$z=x* e^{-x^2 - y^2}$$\n", - "\n", - "\n", - "Where \\\\(z\\\\) is the dependent variable you are trying to predict and \\\\(x\\\\) and \\\\(y\\\\) are the features." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "e8woaj81GGE9", - "colab": {} - }, - "source": [ - "from numpy.random import uniform, seed\n", - "from matplotlib.mlab import griddata\n", - "\n", - "# Create fake data\n", - "seed(0)\n", - "npts = 5000\n", - "x = uniform(-2, 2, npts)\n", - "y = uniform(-2, 2, npts)\n", - "z = x*np.exp(-x**2 - y**2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GRI3KHfLZsGP", - "colab": {} - }, - "source": [ - "# Prep data for training.\n", - "df = pd.DataFrame({'x': x, 'y': y, 'z': z})\n", - "\n", - "xi = np.linspace(-2.0, 2.0, 200),\n", - "yi = np.linspace(-2.1, 2.1, 210),\n", - "xi,yi = np.meshgrid(xi, yi)\n", - "\n", - "df_predict = pd.DataFrame({\n", - " 'x' : xi.flatten(),\n", - " 'y' : yi.flatten(),\n", - "})\n", - "predict_shape = xi.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "w0JnH4IhZuAb", - "colab": {} - }, - "source": [ - "def plot_contour(x, y, z, **kwargs):\n", - " # Grid the data.\n", - " plt.figure(figsize=(10, 8))\n", - " # Contour the gridded data, plotting dots at the nonuniform data points.\n", - " CS = plt.contour(x, y, z, 15, linewidths=0.5, colors='k')\n", - " CS = plt.contourf(x, y, z, 15,\n", - " vmax=abs(zi).max(), vmin=-abs(zi).max(), cmap='RdBu_r')\n", - " plt.colorbar() # Draw colorbar.\n", - " # Plot data points.\n", - " plt.xlim(-2, 2)\n", - " plt.ylim(-2, 2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KF7WsIcYGF_E" - }, - "source": [ - "You can visualize the function. Redder colors correspond to larger function values." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WrxuqaaXGFOK", - "colab": {} - }, - "source": [ - "zi = griddata(x, y, z, xi, yi, interp='linear')\n", - "plot_contour(xi, yi, zi)\n", - "plt.scatter(df.x, df.y, marker='.')\n", - "plt.title('Contour on training data')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hoANr0f2GFrM", - "colab": {} - }, - "source": [ - "fc = [tf.feature_column.numeric_column('x'),\n", - " tf.feature_column.numeric_column('y')]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xVRWyoY3ayTK", - "colab": {} - }, - "source": [ - "def predict(est):\n", - " \"\"\"Predictions from a given estimator.\"\"\"\n", - " predict_input_fn = lambda: tf.data.Dataset.from_tensors(dict(df_predict))\n", - " preds = np.array([p['predictions'][0] for p in est.predict(predict_input_fn)])\n", - " return preds.reshape(predict_shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uyPu5618GU7K" - }, - "source": [ - "First let's try to fit a linear model to the data." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zUIV2IVgGVSk", - "colab": {} - }, - "source": [ - "train_input_fn = make_input_fn(df, df.z)\n", - "est = tf.estimator.LinearRegressor(fc)\n", - "est.train(train_input_fn, max_steps=500);" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_u4WAcCqfbco", - "colab": {} - }, - "source": [ - "plot_contour(xi, yi, predict(est))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XD_fMAUtSCSa" - }, - "source": [ - "It's not a very good fit. Next let's try to fit a GBDT model to it and try to understand how the model fits the function." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ka1GgvqmSCK7", - "colab": {} - }, - "source": [ - "def create_bt_est(n_trees):\n", - " return tf.estimator.BoostedTreesRegressor(fc,\n", - " n_batches_per_layer=1,\n", - " n_trees=n_trees)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "w0s86Kq1R_Fc", - "colab": {} - }, - "source": [ - "N_TREES = [1,2,3,4,10,20,50,100]\n", - "for n in N_TREES:\n", - " est = create_bt_est(n)\n", - " est.train(train_input_fn, max_steps=500)\n", - " plot_contour(xi, yi, predict(est))\n", - " plt.text(-1.8, 2.1, '# trees: {}'.format(n), color='w', backgroundcolor='black', size=20)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5WcZ9fubh1wT" - }, - "source": [ - "As you increase the number of trees, the model's predictions better approximates the underlying function." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SMKoEZnCdrsp" - }, - "source": [ - "## Conclusion" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZSZUSrjXdw9g" - }, - "source": [ - "In this tutorial you learned how to interpret Boosted Trees models using directional feature contributions and feature importance techniques. These techniques provide insight into how the features impact a model's predictions. Finally, you also gained intution for how a Boosted Tree model fits a complex function by viewing the decision surface for several models." - ] - } - ] -} \ No newline at end of file diff --git a/site/en/r1/tutorials/estimators/cnn.ipynb b/site/en/r1/tutorials/estimators/cnn.ipynb deleted file mode 100644 index 704b1b1b6ca..00000000000 --- a/site/en/r1/tutorials/estimators/cnn.ipynb +++ /dev/null @@ -1,1011 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tuOe1ymfHZPu" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# Build a Convolutional Neural Network using Estimators\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/r1/tutorials/estimators/cnn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/r1/tutorials/estimators/cnn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "The `tf.layers` module provides a high-level API that makes\n", - "it easy to construct a neural network. It provides methods that facilitate the\n", - "creation of dense (fully connected) layers and convolutional layers, adding\n", - "activation functions, and applying dropout regularization. In this tutorial,\n", - "you'll learn how to use `layers` to build a convolutional neural network model\n", - "to recognize the handwritten digits in the MNIST data set.\n", - "\n", - "![handwritten digits 0–9 from the MNIST data set](https://www.tensorflow.org/images/mnist_0-9.png)\n", - "\n", - "The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) comprises 60,000\n", - "training examples and 10,000 test examples of the handwritten digits 0–9,\n", - "formatted as 28x28-pixel monochrome images." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wTe-6uXpP2Ts" - }, - "source": [ - "## Get Started\n", - "\n", - "Let's set up the imports for our TensorFlow program:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6-tpguHLP6Rm" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "import numpy as np\n", - "\n", - "tf.logging.set_verbosity(tf.logging.INFO)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4j5yyyDFQgSB" - }, - "source": [ - "## Intro to Convolutional Neural Networks\n", - "\n", - "Convolutional neural networks (CNNs) are the current state-of-the-art model\n", - "architecture for image classification tasks. CNNs apply a series of filters to\n", - "the raw pixel data of an image to extract and learn higher-level features, which\n", - "the model can then use for classification. CNNs contains three components:\n", - "\n", - "* **Convolutional layers**, which apply a specified number of convolution\n", - " filters to the image. For each subregion, the layer performs a set of\n", - " mathematical operations to produce a single value in the output feature map.\n", - " Convolutional layers then typically apply a\n", - " [ReLU activation function](https://en.wikipedia.org/wiki/Rectifier_\\(neural_networks\\)) to\n", - " the output to introduce nonlinearities into the model.\n", - "\n", - "* **Pooling layers**, which\n", - " [downsample the image data](https://en.wikipedia.org/wiki/Convolutional_neural_network#Pooling_layer)\n", - " extracted by the convolutional layers to reduce the dimensionality of the\n", - " feature map in order to decrease processing time. A commonly used pooling\n", - " algorithm is max pooling, which extracts subregions of the feature map\n", - " (e.g., 2x2-pixel tiles), keeps their maximum value, and discards all other\n", - " values.\n", - "\n", - "* **Dense (fully connected) layers**, which perform classification on the\n", - " features extracted by the convolutional layers and downsampled by the\n", - " pooling layers. In a dense layer, every node in the layer is connected to\n", - " every node in the preceding layer.\n", - "\n", - "Typically, a CNN is composed of a stack of convolutional modules that perform\n", - "feature extraction. Each module consists of a convolutional layer followed by a\n", - "pooling layer. The last convolutional module is followed by one or more dense\n", - "layers that perform classification. The final dense layer in a CNN contains a\n", - "single node for each target class in the model (all the possible classes the\n", - "model may predict), with a\n", - "[softmax](https://en.wikipedia.org/wiki/Softmax_function) activation function to\n", - "generate a value between 0–1 for each node (the sum of all these softmax values\n", - "is equal to 1). We can interpret the softmax values for a given image as\n", - "relative measurements of how likely it is that the image falls into each target\n", - "class.\n", - "\n", - "Note: For a more comprehensive walkthrough of CNN architecture, see Stanford University's [Convolutional Neural Networks for Visual Recognition course material](https://cs231n.github.io/convolutional-networks/)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j23E_Z0FQvZB" - }, - "source": [ - "## Building the CNN MNIST Classifier\n", - "\n", - "Let's build a model to classify the images in the MNIST dataset using the\n", - "following CNN architecture:\n", - "\n", - "1. **Convolutional Layer #1**: Applies 32 5x5 filters (extracting 5x5-pixel\n", - " subregions), with ReLU activation function\n", - "2. **Pooling Layer #1**: Performs max pooling with a 2x2 filter and stride of 2\n", - " (which specifies that pooled regions do not overlap)\n", - "3. **Convolutional Layer #2**: Applies 64 5x5 filters, with ReLU activation\n", - " function\n", - "4. **Pooling Layer #2**: Again, performs max pooling with a 2x2 filter and\n", - " stride of 2\n", - "5. **Dense Layer #1**: 1,024 neurons, with dropout regularization rate of 0.4\n", - " (probability of 0.4 that any given element will be dropped during training)\n", - "6. **Dense Layer #2 (Logits Layer)**: 10 neurons, one for each digit target\n", - " class (0–9).\n", - "\n", - "The `tf.layers` module contains methods to create each of the three layer types\n", - "above:\n", - "\n", - "* `conv2d()`. Constructs a two-dimensional convolutional layer. Takes number\n", - " of filters, filter kernel size, padding, and activation function as\n", - " arguments.\n", - "* `max_pooling2d()`. Constructs a two-dimensional pooling layer using the\n", - " max-pooling algorithm. Takes pooling filter size and stride as arguments.\n", - "* `dense()`. Constructs a dense layer. Takes number of neurons and activation\n", - " function as arguments.\n", - "\n", - "Each of these methods accepts a tensor as input and returns a transformed tensor\n", - "as output. This makes it easy to connect one layer to another: just take the\n", - "output from one layer-creation method and supply it as input to another.\n", - "\n", - "Add the following `cnn_model_fn` function, which\n", - "conforms to the interface expected by TensorFlow's Estimator API (more on this\n", - "later in [Create the Estimator](#create-the-estimator)). This function takes\n", - "MNIST feature data, labels, and mode (from\n", - "`tf.estimator.ModeKeys`: `TRAIN`, `EVAL`, `PREDICT`) as arguments;\n", - "configures the CNN; and returns predictions, loss, and a training operation:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gMR-_3rkRKPa" - }, - "outputs": [], - "source": [ - "def cnn_model_fn(features, labels, mode):\n", - " \"\"\"Model function for CNN.\"\"\"\n", - " # Input Layer\n", - " input_layer = tf.reshape(features[\"x\"], [-1, 28, 28, 1])\n", - "\n", - " # Convolutional Layer #1\n", - " conv1 = tf.layers.conv2d(\n", - " inputs=input_layer,\n", - " filters=32,\n", - " kernel_size=[5, 5],\n", - " padding=\"same\",\n", - " activation=tf.nn.relu)\n", - "\n", - " # Pooling Layer #1\n", - " pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)\n", - "\n", - " # Convolutional Layer #2 and Pooling Layer #2\n", - " conv2 = tf.layers.conv2d(\n", - " inputs=pool1,\n", - " filters=64,\n", - " kernel_size=[5, 5],\n", - " padding=\"same\",\n", - " activation=tf.nn.relu)\n", - " pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)\n", - "\n", - " # Dense Layer\n", - " pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])\n", - " dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)\n", - " dropout = tf.layers.dropout(\n", - " inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)\n", - "\n", - " # Logits Layer\n", - " logits = tf.layers.dense(inputs=dropout, units=10)\n", - "\n", - " predictions = {\n", - " # Generate predictions (for PREDICT and EVAL mode)\n", - " \"classes\": tf.argmax(input=logits, axis=1),\n", - " # Add `softmax_tensor` to the graph. It is used for PREDICT and by the\n", - " # `logging_hook`.\n", - " \"probabilities\": tf.nn.softmax(logits, name=\"softmax_tensor\")\n", - " }\n", - "\n", - " if mode == tf.estimator.ModeKeys.PREDICT:\n", - " return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)\n", - "\n", - " # Calculate Loss (for both TRAIN and EVAL modes)\n", - " loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n", - "\n", - " # Configure the Training Op (for TRAIN mode)\n", - " if mode == tf.estimator.ModeKeys.TRAIN:\n", - " optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n", - " train_op = optimizer.minimize(\n", - " loss=loss,\n", - " global_step=tf.train.get_global_step())\n", - " return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n", - "\n", - " # Add evaluation metrics (for EVAL mode)\n", - " eval_metric_ops = {\n", - " \"accuracy\": tf.metrics.accuracy(\n", - " labels=labels, predictions=predictions[\"classes\"])\n", - " }\n", - " return tf.estimator.EstimatorSpec(\n", - " mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b7z8qC9FRSLB" - }, - "source": [ - "The following sections (with headings corresponding to each code block above)\n", - "dive deeper into the `tf.layers` code used to create each layer, as well as how\n", - "to calculate loss, configure the training op, and generate predictions. If\n", - "you're already experienced with CNNs and [TensorFlow `Estimator`s](../../guide/custom_estimators.md),\n", - "and find the above code intuitive, you may want to skim these sections or just\n", - "skip ahead to [\"Training and Evaluating the CNN MNIST Classifier\"](#train_eval_mnist)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sFBXEYRlRUWu" - }, - "source": [ - "### Input Layer\n", - "\n", - "The methods in the `layers` module for creating convolutional and pooling layers\n", - "for two-dimensional image data expect input tensors to have a shape of\n", - "\u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, \u003cem\u003eimage_height\u003c/em\u003e, \u003cem\u003eimage_width\u003c/em\u003e,\n", - "\u003cem\u003echannels\u003c/em\u003e]\u003c/code\u003e by default. This behavior can be changed using the\n", - "\u003ccode\u003e\u003cem\u003edata_format\u003c/em\u003e\u003c/code\u003e parameter; defined as follows:\n", - "\n", - "* `batch_size` —Size of the subset of examples to use when performing\n", - " gradient descent during training.\n", - "* `image_height` —Height of the example images.\n", - "* `image_width` —Width of the example images.\n", - "* `channels` —Number of color channels in the example images. For color\n", - " images, the number of channels is 3 (red, green, blue). For monochrome\n", - " images, there is just 1 channel (black).\n", - "* `data_format` —A string, one of `channels_last` (default) or `channels_first`.\n", - " `channels_last` corresponds to inputs with shape\n", - " `(batch, ..., channels)` while `channels_first` corresponds to\n", - " inputs with shape `(batch, channels, ...)`.\n", - "\n", - "Here, our MNIST dataset is composed of monochrome 28x28 pixel images, so the\n", - "desired shape for our input layer is \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 28, 28,\n", - "1]\u003c/code\u003e.\n", - "\n", - "To convert our input feature map (`features`) to this shape, we can perform the\n", - "following `reshape` operation:\n", - "\n", - "```\n", - "input_layer = tf.reshape(features[\"x\"], [-1, 28, 28, 1])\n", - "```\n", - "\n", - "Note that we've indicated `-1` for batch size, which specifies that this\n", - "dimension should be dynamically computed based on the number of input values in\n", - "`features[\"x\"]`, holding the size of all other dimensions constant. This allows\n", - "us to treat `batch_size` as a hyperparameter that we can tune. For example, if\n", - "we feed examples into our model in batches of 5, `features[\"x\"]` will contain\n", - "3,920 values (one value for each pixel in each image), and `input_layer` will\n", - "have a shape of `[5, 28, 28, 1]`. Similarly, if we feed examples in batches of\n", - "100, `features[\"x\"]` will contain 78,400 values, and `input_layer` will have a\n", - "shape of `[100, 28, 28, 1]`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iU8Jr1_JRiKA" - }, - "source": [ - "### Convolutional Layer #1\n", - "\n", - "In our first convolutional layer, we want to apply 32 5x5 filters to the input\n", - "layer, with a ReLU activation function. We can use the `conv2d()` method in the\n", - "`layers` module to create this layer as follows:\n", - "\n", - "```\n", - "conv1 = tf.layers.conv2d(\n", - " inputs=input_layer,\n", - " filters=32,\n", - " kernel_size=[5, 5],\n", - " padding=\"same\",\n", - " activation=tf.nn.relu)\n", - "```\n", - "\n", - "The `inputs` argument specifies our input tensor, which must have the shape\n", - "\u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, \u003cem\u003eimage_height\u003c/em\u003e, \u003cem\u003eimage_width\u003c/em\u003e,\n", - "\u003cem\u003echannels\u003c/em\u003e]\u003c/code\u003e. Here, we're connecting our first convolutional layer\n", - "to `input_layer`, which has the shape \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 28, 28,\n", - "1]\u003c/code\u003e.\n", - "\n", - "Note: `conv2d()` will instead accept a shape of `[\u003cem\u003ebatch_size\u003c/em\u003e, \u003cem\u003echannels\u003c/em\u003e, \u003cem\u003eimage_height\u003c/em\u003e, \u003cem\u003eimage_width\u003c/em\u003e]` when passed the argument `data_format=channels_first`.\n", - "\n", - "The `filters` argument specifies the number of filters to apply (here, 32), and\n", - "`kernel_size` specifies the dimensions of the filters as `[\u003cem\u003eheight\u003c/em\u003e,\n", - "\u003cem\u003ewidth\u003c/em\u003e]\u003c/code\u003e (here, \u003ccode\u003e[5, 5]`).\n", - "\n", - "\u003cp class=\"tip\"\u003e\u003cb\u003eTIP:\u003c/b\u003e If filter height and width have the same value, you can instead specify a\n", - "single integer for \u003ccode\u003ekernel_size\u003c/code\u003e—e.g., \u003ccode\u003ekernel_size=5\u003c/code\u003e.\u003c/p\u003e\n", - "\n", - "The `padding` argument specifies one of two enumerated values\n", - "(case-insensitive): `valid` (default value) or `same`. To specify that the\n", - "output tensor should have the same height and width values as the input tensor,\n", - "we set `padding=same` here, which instructs TensorFlow to add 0 values to the\n", - "edges of the input tensor to preserve height and width of 28. (Without padding,\n", - "a 5x5 convolution over a 28x28 tensor will produce a 24x24 tensor, as there are\n", - "24x24 locations to extract a 5x5 tile from a 28x28 grid.)\n", - "\n", - "The `activation` argument specifies the activation function to apply to the\n", - "output of the convolution. Here, we specify ReLU activation with\n", - "`tf.nn.relu`.\n", - "\n", - "Our output tensor produced by `conv2d()` has a shape of\n", - "\u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 28, 28, 32]\u003c/code\u003e: the same height and width\n", - "dimensions as the input, but now with 32 channels holding the output from each\n", - "of the filters." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8qzx1ZMFRqt_" - }, - "source": [ - "### Pooling Layer #1\n", - "\n", - "Next, we connect our first pooling layer to the convolutional layer we just\n", - "created. We can use the `max_pooling2d()` method in `layers` to construct a\n", - "layer that performs max pooling with a 2x2 filter and stride of 2:\n", - "\n", - "```\n", - "pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)\n", - "```\n", - "\n", - "Again, `inputs` specifies the input tensor, with a shape of\n", - "\u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, \u003cem\u003eimage_height\u003c/em\u003e, \u003cem\u003eimage_width\u003c/em\u003e,\n", - "\u003cem\u003echannels\u003c/em\u003e]\u003c/code\u003e. Here, our input tensor is `conv1`, the output from\n", - "the first convolutional layer, which has a shape of \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e,\n", - "28, 28, 32]\u003c/code\u003e.\n", - "\n", - "Note: As with \u003ccode\u003econv2d()\u003c/code\u003e, \u003ccode\u003emax_pooling2d()\u003c/code\u003e will instead\n", - "accept a shape of \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, \u003cem\u003echannels\u003c/em\u003e,\n", - "\u003cem\u003eimage_height\u003c/em\u003e, \u003cem\u003eimage_width\u003c/em\u003e]\u003c/code\u003e when passed the argument\n", - "\u003ccode\u003edata_format=channels_first\u003c/code\u003e.\n", - "\n", - "The `pool_size` argument specifies the size of the max pooling filter as\n", - "\u003ccode\u003e[\u003cem\u003eheight\u003c/em\u003e, \u003cem\u003ewidth\u003c/em\u003e]\u003c/code\u003e (here, `[2, 2]`). If both\n", - "dimensions have the same value, you can instead specify a single integer (e.g.,\n", - "`pool_size=2`).\n", - "\n", - "The `strides` argument specifies the size of the stride. Here, we set a stride\n", - "of 2, which indicates that the subregions extracted by the filter should be\n", - "separated by 2 pixels in both the height and width dimensions (for a 2x2 filter,\n", - "this means that none of the regions extracted will overlap). If you want to set\n", - "different stride values for height and width, you can instead specify a tuple or\n", - "list (e.g., `stride=[3, 6]`).\n", - "\n", - "Our output tensor produced by `max_pooling2d()` (`pool1`) has a shape of\n", - "\u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 14, 14, 32]\u003c/code\u003e: the 2x2 filter reduces height and width by 50% each." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xXej53NlRzFh" - }, - "source": [ - "### Convolutional Layer #2 and Pooling Layer #2\n", - "\n", - "We can connect a second convolutional and pooling layer to our CNN using\n", - "`conv2d()` and `max_pooling2d()` as before. For convolutional layer #2, we\n", - "configure 64 5x5 filters with ReLU activation, and for pooling layer #2, we use\n", - "the same specs as pooling layer #1 (a 2x2 max pooling filter with stride of 2):\n", - "\n", - "```\n", - "conv2 = tf.layers.conv2d(\n", - " inputs=pool1,\n", - " filters=64,\n", - " kernel_size=[5, 5],\n", - " padding=\"same\",\n", - " activation=tf.nn.relu)\n", - "\n", - "pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)\n", - "```\n", - "\n", - "Note that convolutional layer #2 takes the output tensor of our first pooling\n", - "layer (`pool1`) as input, and produces the tensor `conv2` as output. `conv2`\n", - "has a shape of \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 14, 14, 64]\u003c/code\u003e, the same height and width as `pool1` (due to `padding=\"same\"`), and 64 channels for the 64\n", - "filters applied.\n", - "\n", - "Pooling layer #2 takes `conv2` as input, producing `pool2` as output. `pool2`\n", - "has shape \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 7, 7, 64]\u003c/code\u003e (50% reduction of height and width from `conv2`)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jjmLqVP7R7z6" - }, - "source": [ - "### Dense Layer\n", - "\n", - "Next, we want to add a dense layer (with 1,024 neurons and ReLU activation) to\n", - "our CNN to perform classification on the features extracted by the\n", - "convolution/pooling layers. Before we connect the layer, however, we'll flatten\n", - "our feature map (`pool2`) to shape \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e,\n", - "\u003cem\u003efeatures\u003c/em\u003e]\u003c/code\u003e, so that our tensor has only two dimensions:\n", - "\n", - "```\n", - "pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])\n", - "```\n", - "\n", - "In the `reshape()` operation above, the `-1` signifies that the *`batch_size`*\n", - "dimension will be dynamically calculated based on the number of examples in our\n", - "input data. Each example has 7 (`pool2` height) * 7 (`pool2` width) * 64\n", - "(`pool2` channels) features, so we want the `features` dimension to have a value\n", - "of 7 * 7 * 64 (3136 in total). The output tensor, `pool2_flat`, has shape\n", - "\u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 3136]\u003c/code\u003e.\n", - "\n", - "Now, we can use the `dense()` method in `layers` to connect our dense layer as\n", - "follows:\n", - "\n", - "```\n", - "dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)\n", - "```\n", - "\n", - "The `inputs` argument specifies the input tensor: our flattened feature map,\n", - "`pool2_flat`. The `units` argument specifies the number of neurons in the dense\n", - "layer (1,024). The `activation` argument takes the activation function; again,\n", - "we'll use `tf.nn.relu` to add ReLU activation.\n", - "\n", - "To help improve the results of our model, we also apply dropout regularization\n", - "to our dense layer, using the `dropout` method in `layers`:\n", - "\n", - "```\n", - "dropout = tf.layers.dropout(\n", - " inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)\n", - "```\n", - "\n", - "Again, `inputs` specifies the input tensor, which is the output tensor from our\n", - "dense layer (`dense`).\n", - "\n", - "The `rate` argument specifies the dropout rate; here, we use `0.4`, which means\n", - "40% of the elements will be randomly dropped out during training.\n", - "\n", - "The `training` argument takes a boolean specifying whether or not the model is\n", - "currently being run in training mode; dropout will only be performed if\n", - "`training` is `True`. Here, we check if the `mode` passed to our model function\n", - "`cnn_model_fn` is `TRAIN` mode.\n", - "\n", - "Our output tensor `dropout` has shape \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 1024]\u003c/code\u003e." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rzUcwkCZSTF7" - }, - "source": [ - "### Logits Layer\n", - "\n", - "The final layer in our neural network is the logits layer, which will return the\n", - "raw values for our predictions. We create a dense layer with 10 neurons (one for\n", - "each target class 0–9), with linear activation (the default):\n", - "\n", - "```\n", - "logits = tf.layers.dense(inputs=dropout, units=10)\n", - "```\n", - "\n", - "Our final output tensor of the CNN, `logits`, has shape `[batch_size, 10]`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y3uJ0V1KSakc" - }, - "source": [ - "### Generate Predictions {#generate_predictions}\n", - "\n", - "The logits layer of our model returns our predictions as raw values in a\n", - "\u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e, 10]\u003c/code\u003e-dimensional tensor. Let's convert these\n", - "raw values into two different formats that our model function can return:\n", - "\n", - "* The **predicted class** for each example: a digit from 0–9.\n", - "* The **probabilities** for each possible target class for each example: the\n", - " probability that the example is a 0, is a 1, is a 2, etc.\n", - "\n", - "For a given example, our predicted class is the element in the corresponding row\n", - "of the logits tensor with the highest raw value. We can find the index of this\n", - "element using the `tf.argmax`\n", - "function:\n", - "\n", - "```\n", - "tf.argmax(input=logits, axis=1)\n", - "```\n", - "\n", - "The `input` argument specifies the tensor from which to extract maximum\n", - "values—here `logits`. The `axis` argument specifies the axis of the `input`\n", - "tensor along which to find the greatest value. Here, we want to find the largest\n", - "value along the dimension with index of 1, which corresponds to our predictions\n", - "(recall that our logits tensor has shape \u003ccode\u003e[\u003cem\u003ebatch_size\u003c/em\u003e,\n", - "10]\u003c/code\u003e).\n", - "\n", - "We can derive probabilities from our logits layer by applying softmax activation\n", - "using `tf.nn.softmax`:\n", - "\n", - "```\n", - "tf.nn.softmax(logits, name=\"softmax_tensor\")\n", - "```\n", - "\n", - "Note: We use the `name` argument to explicitly name this operation `softmax_tensor`, so we can reference it later. (We'll set up logging for the softmax values in [\"Set Up a Logging Hook\"](#set-up-a-logging-hook)).\n", - "\n", - "We compile our predictions in a dict, and return an `EstimatorSpec` object:\n", - "\n", - "```\n", - "predictions = {\n", - " \"classes\": tf.argmax(input=logits, axis=1),\n", - " \"probabilities\": tf.nn.softmax(logits, name=\"softmax_tensor\")\n", - "}\n", - "if mode == tf.estimator.ModeKeys.PREDICT:\n", - " return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f2ks_tqSSucg" - }, - "source": [ - "### Calculate Loss {#calculating-loss}\n", - "\n", - "For both training and evaluation, we need to define a\n", - "[loss function](https://en.wikipedia.org/wiki/Loss_function)\n", - "that measures how closely the model's predictions match the target classes. For\n", - "multiclass classification problems like MNIST,\n", - "[cross entropy](https://en.wikipedia.org/wiki/Cross_entropy) is typically used\n", - "as the loss metric. The following code calculates cross entropy when the model\n", - "runs in either `TRAIN` or `EVAL` mode:\n", - "\n", - "```\n", - "loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)\n", - "```\n", - "\n", - "Let's take a closer look at what's happening above.\n", - "\n", - "Our `labels` tensor contains a list of prediction indices for our examples, e.g. `[1,\n", - "9, ...]`. `logits` contains the linear outputs of our last layer.\n", - "\n", - "`tf.losses.sparse_softmax_cross_entropy`, calculates the softmax crossentropy\n", - "(aka: categorical crossentropy, negative log-likelihood) from these two inputs\n", - "in an efficient, numerically stable way." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YgE7Ll3pS2FG" - }, - "source": [ - "### Configure the Training Op\n", - "\n", - "In the previous section, we defined loss for our CNN as the softmax\n", - "cross-entropy of the logits layer and our labels. Let's configure our model to\n", - "optimize this loss value during training. We'll use a learning rate of 0.001 and\n", - "[stochastic gradient descent](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)\n", - "as the optimization algorithm:\n", - "\n", - "```\n", - "if mode == tf.estimator.ModeKeys.TRAIN:\n", - " optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)\n", - " train_op = optimizer.minimize(\n", - " loss=loss,\n", - " global_step=tf.train.get_global_step())\n", - " return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rEJPnXAzS6m9" - }, - "source": [ - "Note: For a more in-depth look at configuring training ops for Estimator model functions, see [\"Defining the training op for the model\"](../../guide/custom_estimators.md#defining-the-training-op-for-the-model) in the [\"Creating Estimations in tf.estimator\"](../../guide/custom_estimators.md) tutorial." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QQuGDWvHTAib" - }, - "source": [ - "### Add evaluation metrics\n", - "\n", - "To add accuracy metric in our model, we define `eval_metric_ops` dict in EVAL\n", - "mode as follows:\n", - "\n", - "```\n", - "eval_metric_ops = {\n", - " \"accuracy\": tf.metrics.accuracy(\n", - " labels=labels, predictions=predictions[\"classes\"])\n", - "}\n", - "return tf.estimator.EstimatorSpec(\n", - " mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y2Bwe-AdTRzX" - }, - "source": [ - "\u003ca id=\"train_eval_mnist\"\u003e\u003c/a\u003e\n", - "## Training and Evaluating the CNN MNIST Classifier\n", - "\n", - "We've coded our MNIST CNN model function; now we're ready to train and evaluate\n", - "it." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6EC9aOY2TTLU" - }, - "source": [ - "### Load Training and Test Data\n", - "\n", - "First, let's load our training and test data with the following code:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ccobb0qETV-S" - }, - "outputs": [], - "source": [ - "# Load training and eval data\n", - "((train_data, train_labels),\n", - " (eval_data, eval_labels)) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_data = train_data/np.float32(255)\n", - "train_labels = train_labels.astype(np.int32) # not required\n", - "\n", - "eval_data = eval_data/np.float32(255)\n", - "eval_labels = eval_labels.astype(np.int32) # not required" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8l84-IxSTZnO" - }, - "source": [ - "We store the training feature data (the raw pixel values for 55,000 images of\n", - "hand-drawn digits) and training labels (the corresponding value from 0–9 for\n", - "each image) as [numpy\n", - "arrays](https://docs.scipy.org/doc/numpy/reference/generated/numpy.array.html)\n", - "in `train_data` and `train_labels`, respectively. Similarly, we store the\n", - "evaluation feature data (10,000 images) and evaluation labels in `eval_data`\n", - "and `eval_labels`, respectively." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S2_Isc7kTa45" - }, - "source": [ - "### Create the Estimator {#create-the-estimator}\n", - "\n", - "Next, let's create an `Estimator` (a TensorFlow class for performing high-level\n", - "model training, evaluation, and inference) for our model. Add the following code\n", - "to `main()`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yjC6HdwZTdg4" - }, - "outputs": [], - "source": [ - "# Create the Estimator\n", - "mnist_classifier = tf.estimator.Estimator(\n", - " model_fn=cnn_model_fn, model_dir=\"/tmp/mnist_convnet_model\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f78EBcg7TfTU" - }, - "source": [ - "The `model_fn` argument specifies the model function to use for training,\n", - "evaluation, and prediction; we pass it the `cnn_model_fn` we created in\n", - "[\"Building the CNN MNIST Classifier.\"](#building-the-cnn-mnist-classifier) The\n", - "`model_dir` argument specifies the directory where model data (checkpoints) will\n", - "be saved (here, we specify the temp directory `/tmp/mnist_convnet_model`, but\n", - "feel free to change to another directory of your choice).\n", - "\n", - "Note: For an in-depth walkthrough of the TensorFlow `Estimator` API, see the tutorial [Creating Estimators in tf.estimator](../../guide/custom_estimators.md)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_6ow7hVYTm3f" - }, - "source": [ - "### Set Up a Logging Hook {#set_up_a_logging_hook}\n", - "\n", - "Since CNNs can take a while to train, let's set up some logging so we can track\n", - "progress during training. We can use TensorFlow's `tf.train.SessionRunHook` to create a\n", - "`tf.train.LoggingTensorHook`\n", - "that will log the probability values from the softmax layer of our CNN. Add the\n", - "following to `main()`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "S6T10kssTpdz" - }, - "outputs": [], - "source": [ - "# Set up logging for predictions\n", - "tensors_to_log = {\"probabilities\": \"softmax_tensor\"}\n", - "\n", - "logging_hook = tf.train.LoggingTensorHook(\n", - " tensors=tensors_to_log, every_n_iter=50)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RZdtZ6JQTsmg" - }, - "source": [ - "We store a dict of the tensors we want to log in `tensors_to_log`. Each key is a\n", - "label of our choice that will be printed in the log output, and the\n", - "corresponding label is the name of a `Tensor` in the TensorFlow graph. Here, our\n", - "`probabilities` can be found in `softmax_tensor`, the name we gave our softmax\n", - "operation earlier when we generated the probabilities in `cnn_model_fn`.\n", - "\n", - "Note: If you don't explicitly assign a name to an operation via the `name` argument, TensorFlow will assign a default name. A couple easy ways to discover the names applied to operations are to visualize your graph on [TensorBoard](../../guide/graph_viz.md)) or to enable the [TensorFlow Debugger (tfdbg)](../../guide/debugger.md).\n", - "\n", - "Next, we create the `LoggingTensorHook`, passing `tensors_to_log` to the\n", - "`tensors` argument. We set `every_n_iter=50`, which specifies that probabilities\n", - "should be logged after every 50 steps of training." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "brVs1dRMT0NM" - }, - "source": [ - "### Train the Model\n", - "\n", - "Now we're ready to train our model, which we can do by creating `train_input_fn`\n", - "and calling `train()` on `mnist_classifier`. In the `numpy_input_fn` call, we pass the training feature data and labels to\n", - "`x` (as a dict) and `y`, respectively. We set a `batch_size` of `100` (which\n", - "means that the model will train on minibatches of 100 examples at each step).\n", - "`num_epochs=None` means that the model will train until the specified number of\n", - "steps is reached. We also set `shuffle=True` to shuffle the training data. Then train the model a single step and log the output:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "h-dewpleT2sk" - }, - "outputs": [], - "source": [ - "# Train the model\n", - "train_input_fn = tf.estimator.inputs.numpy_input_fn(\n", - " x={\"x\": train_data},\n", - " y=train_labels,\n", - " batch_size=100,\n", - " num_epochs=None,\n", - " shuffle=True)\n", - "\n", - "# train one step and display the probabilties\n", - "mnist_classifier.train(\n", - " input_fn=train_input_fn,\n", - " steps=1,\n", - " hooks=[logging_hook])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gyNSE3e-14Lq" - }, - "source": [ - "Now—without logging each step—set `steps=1000` to train the model longer, but in a reasonable time to run this example. Training CNNs is computationally intensive. To increase the accuracy of your model, increase the number of `steps` passed to `train()`, like 20,000 steps." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cri6zqcf2IXY" - }, - "outputs": [], - "source": [ - "mnist_classifier.train(input_fn=train_input_fn, steps=1000)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4bQdkLMeUE5U" - }, - "source": [ - "### Evaluate the Model\n", - "\n", - "Once training is complete, we want to evaluate our model to determine its\n", - "accuracy on the MNIST test set. We call the `evaluate` method, which evaluates\n", - "the metrics we specified in `eval_metric_ops` argument in the `model_fn`.\n", - "Add the following to `main()`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "I0RGiqd0UF0N" - }, - "outputs": [], - "source": [ - "eval_input_fn = tf.estimator.inputs.numpy_input_fn(\n", - " x={\"x\": eval_data},\n", - " y=eval_labels,\n", - " num_epochs=1,\n", - " shuffle=False)\n", - "\n", - "eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)\n", - "print(eval_results)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JIBVID6dUIXT" - }, - "source": [ - "To create `eval_input_fn`, we set `num_epochs=1`, so that the model evaluates\n", - "the metrics over one epoch of data and returns the result. We also set\n", - "`shuffle=False` to iterate through the data sequentially." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "htmLZ-zEUZZk" - }, - "source": [ - "## Additional Resources\n", - "\n", - "To learn more about TensorFlow Estimators and CNNs in TensorFlow, see the\n", - "following resources:\n", - "\n", - "* [Creating Estimators in tf.estimator](../../guide/custom_estimators.md)\n", - " provides an introduction to the TensorFlow Estimator API. It walks through\n", - " configuring an Estimator, writing a model function, calculating loss, and\n", - " defining a training op.\n", - "* [Advanced Convolutional Neural Networks](../../tutorials/images/deep_cnn.md) walks through how to build a MNIST CNN classification model\n", - " *without estimators* using lower-level TensorFlow operations." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [ - "Tce3stUlHN0L" - ], - "name": "cnn.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/r1/tutorials/estimators/linear.ipynb b/site/en/r1/tutorials/estimators/linear.ipynb deleted file mode 100644 index 3d77e3d23d9..00000000000 --- a/site/en/r1/tutorials/estimators/linear.ipynb +++ /dev/null @@ -1,1398 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "linear.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "MWW1TyjaecRh" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "MWW1TyjaecRh", - "colab_type": "text" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "mOtR1FzCef-u", - "colab_type": "code", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Zr7KpBhMcYvE", - "colab_type": "text" - }, - "source": [ - "# Build a linear model with Estimators" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "uJl4gaPFzxQz", - "colab_type": "text" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "77aETSYDcdoK", - "colab_type": "text" - }, - "source": [ - "This tutorial uses the `tf.estimator` API in TensorFlow to solve a benchmark binary classification problem. Estimators are TensorFlow's most scalable and production-oriented model type. For more information see the [Estimator guide](https://www.tensorflow.org/r1/guide/estimators).\n", - "\n", - "## Overview\n", - "\n", - "Using census data which contains data about a person's age, education, marital status, and occupation (the *features*), we will try to predict whether or not the person earns more than 50,000 dollars a year (the target *label*). We will train a *logistic regression* model that, given an individual's information, outputs a number between 0 and 1—this can be interpreted as the probability that the individual has an annual income of over 50,000 dollars.\n", - "\n", - "Key Point: As a modeler and developer, think about how this data is used and the potential benefits and harm a model's predictions can cause. A model like this could reinforce societal biases and disparities. Is each feature relevant to the problem you want to solve or will it introduce bias? For more information, read about [ML fairness](https://developers.google.com/machine-learning/fairness-overview/).\n", - "\n", - "## Setup\n", - "\n", - "Import TensorFlow, feature column support, and supporting modules:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "NQgONe5ecYvE", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "import tensorflow.feature_column as fc\n", - "\n", - "import os\n", - "import sys\n", - "\n", - "import matplotlib.pyplot as plt\n", - "from IPython.display import clear_output" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Rpb1JSMj1nqk", - "colab_type": "text" - }, - "source": [ - "And let's enable [eager execution](https://www.tensorflow.org/r1/guide/eager) to inspect this program as we run it:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "tQzxON782Eby", - "colab_type": "code", - "colab": {} - }, - "source": [ - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-MPr95UccYvL", - "colab_type": "text" - }, - "source": [ - "## Download the official implementation\n", - "\n", - "We'll use the [wide and deep model](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep/) available in TensorFlow's [model repository](https://github.com/tensorflow/models/). Download the code, add the root directory to your Python path, and jump to the `wide_deep` directory:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "tTwQzWcn8aBu", - "colab_type": "code", - "colab": {} - }, - "source": [ - "! pip install requests\n", - "! git clone --depth 1 https://github.com/tensorflow/models" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sRpuysc73Eb-", - "colab_type": "text" - }, - "source": [ - "Add the root directory of the repository to your Python path:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "yVvFyhnkcYvL", - "colab_type": "code", - "colab": {} - }, - "source": [ - "models_path = os.path.join(os.getcwd(), 'models')\n", - "\n", - "sys.path.append(models_path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "15Ethw-wcYvP", - "colab_type": "text" - }, - "source": [ - "Download the dataset:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "6QilS4-0cYvQ", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from official.r1.wide_deep import census_dataset\n", - "from official.r1.wide_deep import census_main\n", - "\n", - "census_dataset.download(\"/tmp/census_data/\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cD5e3ibAcYvS", - "colab_type": "text" - }, - "source": [ - "### Command line usage\n", - "\n", - "The repo includes a complete program for experimenting with this type of model.\n", - "\n", - "To execute the tutorial code from the command line first add the path to tensorflow/models to your `PYTHONPATH`." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "DYOkY8boUptJ", - "colab_type": "code", - "colab": {} - }, - "source": [ - "#export PYTHONPATH=${PYTHONPATH}:\"$(pwd)/models\"\n", - "#running from python you need to set the `os.environ` or the subprocess will not see the directory.\n", - "\n", - "if \"PYTHONPATH\" in os.environ:\n", - " os.environ['PYTHONPATH'] += os.pathsep + models_path\n", - "else:\n", - " os.environ['PYTHONPATH'] = models_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5r0V9YUMUyoh", - "colab_type": "text" - }, - "source": [ - "Use `--help` to see what command line options are available:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "1_3tBaLW4YM4", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!python -m official.r1.wide_deep.census_main --help" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RrMLazEN6DMj", - "colab_type": "text" - }, - "source": [ - "Now run the model:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "py7MarZl5Yh6", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!python -m official.r1.wide_deep.census_main --model_type=wide --train_epochs=2" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AmZ4CpaOcYvV", - "colab_type": "text" - }, - "source": [ - "## Read the U.S. Census data\n", - "\n", - "This example uses the [U.S Census Income Dataset](https://archive.ics.uci.edu/ml/datasets/Census+Income) from 1994 and 1995. We have provided the [census_dataset.py](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep/census_dataset.py) script to download the data and perform a little cleanup.\n", - "\n", - "Since the task is a *binary classification problem*, we'll construct a label column named \"label\" whose value is 1 if the income is over 50K, and 0 otherwise. For reference, see the `input_fn` in [census_main.py](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep/census_main.py).\n", - "\n", - "Let's look at the data to see which columns we can use to predict the target label:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "N6Tgye8bcYvX", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!ls /tmp/census_data/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "6y3mj9zKcYva", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_file = \"/tmp/census_data/adult.data\"\n", - "test_file = \"/tmp/census_data/adult.test\"" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EO_McKgE5il2", - "colab_type": "text" - }, - "source": [ - "[pandas](https://pandas.pydata.org/) provides some convenient utilities for data analysis. Here's a list of columns available in the Census Income dataset:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "vkn1FNmpcYvb", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import pandas\n", - "\n", - "train_df = pandas.read_csv(train_file, header = None, names = census_dataset._CSV_COLUMNS)\n", - "test_df = pandas.read_csv(test_file, header = None, names = census_dataset._CSV_COLUMNS)\n", - "\n", - "train_df.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "QZZtXes4cYvf", - "colab_type": "text" - }, - "source": [ - "The columns are grouped into two types: *categorical* and *continuous* columns:\n", - "\n", - "* A column is called *categorical* if its value can only be one of the categories in a finite set. For example, the relationship status of a person (wife, husband, unmarried, etc.) or the education level (high school, college, etc.) are categorical columns.\n", - "* A column is called *continuous* if its value can be any numerical value in a continuous range. For example, the capital gain of a person (e.g. $14,084) is a continuous column.\n", - "\n", - "## Converting Data into Tensors\n", - "\n", - "When building a `tf.estimator` model, the input data is specified by using an *input function* (or `input_fn`). This builder function returns a `tf.data.Dataset` of batches of `(features-dict, label)` pairs. It is not called until it is passed to `tf.estimator.Estimator` methods such as `train` and `evaluate`.\n", - "\n", - "The input builder function returns the following pair:\n", - "\n", - "1. `features`: A dict from feature names to `Tensors` or `SparseTensors` containing batches of features.\n", - "2. `labels`: A `Tensor` containing batches of labels.\n", - "\n", - "The keys of the `features` are used to configure the model's input layer.\n", - "\n", - "Note: The input function is called while constructing the TensorFlow graph, *not* while running the graph. It is returning a representation of the input data as a sequence of TensorFlow graph operations.\n", - "\n", - "For small problems like this, it's easy to make a `tf.data.Dataset` by slicing the `pandas.DataFrame`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "N7zNJflKcYvg", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def easy_input_function(df, label_key, num_epochs, shuffle, batch_size):\n", - " label = df[label_key]\n", - " ds = tf.data.Dataset.from_tensor_slices((dict(df),label))\n", - "\n", - " if shuffle:\n", - " ds = ds.shuffle(10000)\n", - "\n", - " ds = ds.batch(batch_size).repeat(num_epochs)\n", - "\n", - " return ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WeEgNR9AcYvh", - "colab_type": "text" - }, - "source": [ - "Since we have eager execution enabled, it's easy to inspect the resulting dataset:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "ygaKuikecYvi", - "colab_type": "code", - "colab": {} - }, - "source": [ - "ds = easy_input_function(train_df, label_key='income_bracket', num_epochs=5, shuffle=True, batch_size=10)\n", - "\n", - "for feature_batch, label_batch in ds.take(1):\n", - " print('Some feature keys:', list(feature_batch.keys())[:5])\n", - " print()\n", - " print('A batch of Ages :', feature_batch['age'])\n", - " print()\n", - " print('A batch of Labels:', label_batch )" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "O_KZxQUucYvm", - "colab_type": "text" - }, - "source": [ - "But this approach has severly-limited scalability. Larger datasets should be streamed from disk. The `census_dataset.input_fn` provides an example of how to do this using `tf.decode_csv` and `tf.data.TextLineDataset`:\n", - "\n", - "" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "vUTeXaEUcYvn", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import inspect\n", - "print(inspect.getsource(census_dataset.input_fn))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yyGcv_e-cYvq", - "colab_type": "text" - }, - "source": [ - "This `input_fn` returns equivalent output:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Mv3as_CEcYvu", - "colab_type": "code", - "colab": {} - }, - "source": [ - "ds = census_dataset.input_fn(train_file, num_epochs=5, shuffle=True, batch_size=10)\n", - "\n", - "for feature_batch, label_batch in ds.take(1):\n", - " print('Feature keys:', list(feature_batch.keys())[:5])\n", - " print()\n", - " print('Age batch :', feature_batch['age'])\n", - " print()\n", - " print('Label batch :', label_batch )" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "810fnfY5cYvz", - "colab_type": "text" - }, - "source": [ - "Because `Estimators` expect an `input_fn` that takes no arguments, we typically wrap configurable input function into an object with the expected signature. For this notebook configure the `train_inpf` to iterate over the data twice:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "wnQdpEcVcYv0", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import functools\n", - "\n", - "train_inpf = functools.partial(census_dataset.input_fn, train_file, num_epochs=2, shuffle=True, batch_size=64)\n", - "test_inpf = functools.partial(census_dataset.input_fn, test_file, num_epochs=1, shuffle=False, batch_size=64)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pboNpNWhcYv4", - "colab_type": "text" - }, - "source": [ - "## Selecting and Engineering Features for the Model\n", - "\n", - "Estimators use a system called [feature columns](https://www.tensorflow.org/r1/guide/feature_columns) to describe how the model should interpret each of the raw input features. An Estimator expects a vector of numeric inputs, and feature columns describe how the model should convert each feature.\n", - "\n", - "Selecting and crafting the right set of feature columns is key to learning an effective model. A *feature column* can be either one of the raw inputs in the original features `dict` (a *base feature column*), or any new columns created using transformations defined over one or multiple base columns (a *derived feature columns*).\n", - "\n", - "A feature column is an abstract concept of any raw or derived variable that can be used to predict the target label." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_hh-cWdU__Lq", - "colab_type": "text" - }, - "source": [ - "### Base Feature Columns" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BKz6LA8_ACI7", - "colab_type": "text" - }, - "source": [ - "#### Numeric columns\n", - "\n", - "The simplest `feature_column` is `numeric_column`. This indicates that a feature is a numeric value that should be input to the model directly. For example:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "ZX0r2T5OcYv6", - "colab_type": "code", - "colab": {} - }, - "source": [ - "age = fc.numeric_column('age')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tnLUiaHxcYv-", - "colab_type": "text" - }, - "source": [ - "The model will use the `feature_column` definitions to build the model input. You can inspect the resulting output using the `input_layer` function:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "kREtIPfwcYv_", - "colab_type": "code", - "colab": {} - }, - "source": [ - "fc.input_layer(feature_batch, [age]).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "OPuLduCucYwD", - "colab_type": "text" - }, - "source": [ - "The following will train and evaluate a model using only the `age` feature:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "9R5eSJ1pcYwE", - "colab_type": "code", - "colab": {} - }, - "source": [ - "classifier = tf.estimator.LinearClassifier(feature_columns=[age])\n", - "classifier.train(train_inpf)\n", - "result = classifier.evaluate(test_inpf)\n", - "\n", - "clear_output() # used for display in notebook\n", - "print(result)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YDZGcdTdcYwI", - "colab_type": "text" - }, - "source": [ - "Similarly, we can define a `NumericColumn` for each continuous feature column\n", - "that we want to use in the model:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "uqPbUqlxcYwJ", - "colab_type": "code", - "colab": {} - }, - "source": [ - "education_num = tf.feature_column.numeric_column('education_num')\n", - "capital_gain = tf.feature_column.numeric_column('capital_gain')\n", - "capital_loss = tf.feature_column.numeric_column('capital_loss')\n", - "hours_per_week = tf.feature_column.numeric_column('hours_per_week')\n", - "\n", - "my_numeric_columns = [age,education_num, capital_gain, capital_loss, hours_per_week]\n", - "\n", - "fc.input_layer(feature_batch, my_numeric_columns).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cBGDN97IcYwQ", - "colab_type": "text" - }, - "source": [ - "You could retrain a model on these features by changing the `feature_columns` argument to the constructor:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "XN8k5S95cYwR", - "colab_type": "code", - "colab": {} - }, - "source": [ - "classifier = tf.estimator.LinearClassifier(feature_columns=my_numeric_columns)\n", - "classifier.train(train_inpf)\n", - "\n", - "result = classifier.evaluate(test_inpf)\n", - "\n", - "clear_output()\n", - "\n", - "for key,value in sorted(result.items()):\n", - " print('%s: %s' % (key, value))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jBRq9_AzcYwU", - "colab_type": "text" - }, - "source": [ - "#### Categorical columns\n", - "\n", - "To define a feature column for a categorical feature, create a `CategoricalColumn` using one of the `tf.feature_column.categorical_column*` functions.\n", - "\n", - "If you know the set of all possible feature values of a column—and there are only a few of them—use `categorical_column_with_vocabulary_list`. Each key in the list is assigned an auto-incremented ID starting from 0. For example, for the `relationship` column we can assign the feature string `Husband` to an integer ID of 0 and \"Not-in-family\" to 1, etc." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "0IjqSi9tcYwV", - "colab_type": "code", - "colab": {} - }, - "source": [ - "relationship = fc.categorical_column_with_vocabulary_list(\n", - " 'relationship',\n", - " ['Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried', 'Other-relative'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-RjoWv-7cYwW", - "colab_type": "text" - }, - "source": [ - "This creates a sparse one-hot vector from the raw input feature.\n", - "\n", - "The `input_layer` function we're using is designed for DNN models and expects dense inputs. To demonstrate the categorical column we must wrap it in a `tf.feature_column.indicator_column` to create the dense one-hot output (Linear `Estimators` can often skip this dense-step).\n", - "\n", - "Note: the other sparse-to-dense option is `tf.feature_column.embedding_column`.\n", - "\n", - "Run the input layer, configured with both the `age` and `relationship` columns:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "kI43CYlncYwY", - "colab_type": "code", - "colab": {} - }, - "source": [ - "fc.input_layer(feature_batch, [age, fc.indicator_column(relationship)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tTudP7WHcYwb", - "colab_type": "text" - }, - "source": [ - "If we don't know the set of possible values in advance, use the `categorical_column_with_hash_bucket` instead:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "8pSBaliCcYwb", - "colab_type": "code", - "colab": {} - }, - "source": [ - "occupation = tf.feature_column.categorical_column_with_hash_bucket(\n", - " 'occupation', hash_bucket_size=1000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fSAPrqQkcYwd", - "colab_type": "text" - }, - "source": [ - "Here, each possible value in the feature column `occupation` is hashed to an integer ID as we encounter them in training. The example batch has a few different occupations:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "dCvQNv36cYwe", - "colab_type": "code", - "colab": {} - }, - "source": [ - "for item in feature_batch['occupation'].numpy():\n", - " print(item.decode())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KP5hN2rAcYwh", - "colab_type": "text" - }, - "source": [ - "If we run `input_layer` with the hashed column, we see that the output shape is `(batch_size, hash_bucket_size)`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "0Y16peWacYwh", - "colab_type": "code", - "colab": {} - }, - "source": [ - "occupation_result = fc.input_layer(feature_batch, [fc.indicator_column(occupation)])\n", - "\n", - "occupation_result.numpy().shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "HMW2MzWAcYwk", - "colab_type": "text" - }, - "source": [ - "It's easier to see the actual results if we take the `tf.argmax` over the `hash_bucket_size` dimension. Notice how any duplicate occupations are mapped to the same pseudo-random index:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "q_ryRglmcYwk", - "colab_type": "code", - "colab": {} - }, - "source": [ - "tf.argmax(occupation_result, axis=1).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "j1e5NfyKcYwn", - "colab_type": "text" - }, - "source": [ - "Note: Hash collisions are unavoidable, but often have minimal impact on model quality. The effect may be noticable if the hash buckets are being used to compress the input space. See [this notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/outreach/blogs/housing_prices.ipynb) for a more visual example of the effect of these hash collisions.\n", - "\n", - "No matter how we choose to define a `SparseColumn`, each feature string is mapped into an integer ID by looking up a fixed mapping or by hashing. Under the hood, the `LinearModel` class is responsible for managing the mapping and creating `tf.Variable` to store the model parameters (model *weights*) for each feature ID. The model parameters are learned through the model training process described later.\n", - "\n", - "Let's do the similar trick to define the other categorical features:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "0Z5eUrd_cYwo", - "colab_type": "code", - "colab": {} - }, - "source": [ - "education = tf.feature_column.categorical_column_with_vocabulary_list(\n", - " 'education', [\n", - " 'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college',\n", - " 'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school',\n", - " '5th-6th', '10th', '1st-4th', 'Preschool', '12th'])\n", - "\n", - "marital_status = tf.feature_column.categorical_column_with_vocabulary_list(\n", - " 'marital_status', [\n", - " 'Married-civ-spouse', 'Divorced', 'Married-spouse-absent',\n", - " 'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed'])\n", - "\n", - "workclass = tf.feature_column.categorical_column_with_vocabulary_list(\n", - " 'workclass', [\n", - " 'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov',\n", - " 'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked'])\n", - "\n", - "\n", - "my_categorical_columns = [relationship, occupation, education, marital_status, workclass]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ASQJM1pEcYwr", - "colab_type": "text" - }, - "source": [ - "It's easy to use both sets of columns to configure a model that uses all these features:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "_i_MLoo9cYws", - "colab_type": "code", - "colab": {} - }, - "source": [ - "classifier = tf.estimator.LinearClassifier(feature_columns=my_numeric_columns+my_categorical_columns)\n", - "classifier.train(train_inpf)\n", - "result = classifier.evaluate(test_inpf)\n", - "\n", - "clear_output()\n", - "\n", - "for key,value in sorted(result.items()):\n", - " print('%s: %s' % (key, value))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zdKEqF6xcYwv", - "colab_type": "text" - }, - "source": [ - "### Derived feature columns" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RgYaf_48FSU2", - "colab_type": "text" - }, - "source": [ - "#### Make Continuous Features Categorical through Bucketization\n", - "\n", - "Sometimes the relationship between a continuous feature and the label is not linear. For example, *age* and *income*—a person's income may grow in the early stage of their career, then the growth may slow at some point, and finally, the income decreases after retirement. In this scenario, using the raw `age` as a real-valued feature column might not be a good choice because the model can only learn one of the three cases:\n", - "\n", - "1. Income always increases at some rate as age grows (positive correlation),\n", - "2. Income always decreases at some rate as age grows (negative correlation), or\n", - "3. Income stays the same no matter at what age (no correlation).\n", - "\n", - "If we want to learn the fine-grained correlation between income and each age group separately, we can leverage *bucketization*. Bucketization is a process of dividing the entire range of a continuous feature into a set of consecutive buckets, and then converting the original numerical feature into a bucket ID (as a categorical feature) depending on which bucket that value falls into. So, we can define a `bucketized_column` over `age` as:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "KT4pjD9AcYww", - "colab_type": "code", - "colab": {} - }, - "source": [ - "age_buckets = tf.feature_column.bucketized_column(\n", - " age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "S-XOscrEcYwx", - "colab_type": "text" - }, - "source": [ - "`boundaries` is a list of bucket boundaries. In this case, there are 10 boundaries, resulting in 11 age group buckets (from age 17 and below, 18-24, 25-29, ..., to 65 and over).\n", - "\n", - "With bucketing, the model sees each bucket as a one-hot feature:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Lr40vm3qcYwy", - "colab_type": "code", - "colab": {} - }, - "source": [ - "fc.input_layer(feature_batch, [age, age_buckets]).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Z_tQI9j8cYw1", - "colab_type": "text" - }, - "source": [ - "#### Learn complex relationships with crossed column\n", - "\n", - "Using each base feature column separately may not be enough to explain the data. For example, the correlation between education and the label (earning > 50,000 dollars) may be different for different occupations. Therefore, if we only learn a single model weight for `education=\"Bachelors\"` and `education=\"Masters\"`, we won't capture every education-occupation combination (e.g. distinguishing between `education=\"Bachelors\"` AND `occupation=\"Exec-managerial\"` AND `education=\"Bachelors\" AND occupation=\"Craft-repair\"`).\n", - "\n", - "To learn the differences between different feature combinations, we can add *crossed feature columns* to the model:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "IAPhPzXscYw1", - "colab_type": "code", - "colab": {} - }, - "source": [ - "education_x_occupation = tf.feature_column.crossed_column(\n", - " ['education', 'occupation'], hash_bucket_size=1000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UeTxMunbcYw5", - "colab_type": "text" - }, - "source": [ - "We can also create a `crossed_column` over more than two columns. Each constituent column can be either a base feature column that is categorical (`SparseColumn`), a bucketized real-valued feature column, or even another `CrossColumn`. For example:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "y8UaBld9cYw7", - "colab_type": "code", - "colab": {} - }, - "source": [ - "age_buckets_x_education_x_occupation = tf.feature_column.crossed_column(\n", - " [age_buckets, 'education', 'occupation'], hash_bucket_size=1000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "HvKmW6U5cYw8", - "colab_type": "text" - }, - "source": [ - "These crossed columns always use hash buckets to avoid the exponential explosion in the number of categories, and put the control over number of model weights in the hands of the user.\n", - "\n", - "For a visual example the effect of hash-buckets with crossed columns see [this notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/outreach/blogs/housing_prices.ipynb)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "HtjpheB6cYw9", - "colab_type": "text" - }, - "source": [ - "## Define the logistic regression model\n", - "\n", - "After processing the input data and defining all the feature columns, we can put them together and build a *logistic regression* model. The previous section showed several types of base and derived feature columns, including:\n", - "\n", - "* `CategoricalColumn`\n", - "* `NumericColumn`\n", - "* `BucketizedColumn`\n", - "* `CrossedColumn`\n", - "\n", - "All of these are subclasses of the abstract `FeatureColumn` class and can be added to the `feature_columns` field of a model:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Klmf3OxpcYw-", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import tempfile\n", - "\n", - "base_columns = [\n", - " education, marital_status, relationship, workclass, occupation,\n", - " age_buckets,\n", - "]\n", - "\n", - "crossed_columns = [\n", - " tf.feature_column.crossed_column(\n", - " ['education', 'occupation'], hash_bucket_size=1000),\n", - " tf.feature_column.crossed_column(\n", - " [age_buckets, 'education', 'occupation'], hash_bucket_size=1000),\n", - "]\n", - "\n", - "model = tf.estimator.LinearClassifier(\n", - " model_dir=tempfile.mkdtemp(),\n", - " feature_columns=base_columns + crossed_columns,\n", - " optimizer=tf.train.FtrlOptimizer(learning_rate=0.1))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jRhnPxUucYxC", - "colab_type": "text" - }, - "source": [ - "The model automatically learns a bias term, which controls the prediction made without observing any features. The learned model files are stored in `model_dir`.\n", - "\n", - "## Train and evaluate the model\n", - "\n", - "After adding all the features to the model, let's train the model. Training a model is just a single command using the `tf.estimator` API:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "ZlrIBuoecYxD", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_inpf = functools.partial(census_dataset.input_fn, train_file,\n", - " num_epochs=40, shuffle=True, batch_size=64)\n", - "\n", - "model.train(train_inpf)\n", - "\n", - "clear_output() # used for notebook display" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IvY3a9pzcYxH", - "colab_type": "text" - }, - "source": [ - "After the model is trained, evaluate the accuracy of the model by predicting the labels of the holdout data:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "L9nVJEO8cYxI", - "colab_type": "code", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_inpf)\n", - "\n", - "clear_output()\n", - "\n", - "for key,value in sorted(results.items()):\n", - " print('%s: %0.2f' % (key, value))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "E0fAibNDcYxL", - "colab_type": "text" - }, - "source": [ - "The first line of the output should display something like: `accuracy: 0.84`, which means the accuracy is 84%. You can try using more features and transformations to see if you can do better!\n", - "\n", - "After the model is evaluated, we can use it to predict whether an individual has an annual income of over 50,000 dollars given an individual's information input.\n", - "\n", - "Let's look in more detail how the model performed:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "8R5bz5CxcYxL", - "colab_type": "code", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "predict_df = test_df[:20].copy()\n", - "\n", - "pred_iter = model.predict(\n", - " lambda:easy_input_function(predict_df, label_key='income_bracket',\n", - " num_epochs=1, shuffle=False, batch_size=10))\n", - "\n", - "classes = np.array(['<=50K', '>50K'])\n", - "pred_class_id = []\n", - "\n", - "for pred_dict in pred_iter:\n", - " pred_class_id.append(pred_dict['class_ids'])\n", - "\n", - "predict_df['predicted_class'] = classes[np.array(pred_class_id)]\n", - "predict_df['correct'] = predict_df['predicted_class'] == predict_df['income_bracket']\n", - "\n", - "clear_output()\n", - "\n", - "predict_df[['income_bracket','predicted_class', 'correct']]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "N_uCpFTicYxN", - "colab_type": "text" - }, - "source": [ - "For a working end-to-end example, download our [example code](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep/census_main.py) and set the `model_type` flag to `wide`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oyKy1lM_3gkL", - "colab_type": "text" - }, - "source": [ - "## Adding Regularization to Prevent Overfitting\n", - "\n", - "Regularization is a technique used to avoid overfitting. Overfitting happens when a model performs well on the data it is trained on, but worse on test data that the model has not seen before. Overfitting can occur when a model is excessively complex, such as having too many parameters relative to the number of observed training data. Regularization allows you to control the model's complexity and make the model more generalizable to unseen data.\n", - "\n", - "You can add L1 and L2 regularizations to the model with the following code:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "lzMUSBQ03hHx", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model_l1 = tf.estimator.LinearClassifier(\n", - " feature_columns=base_columns + crossed_columns,\n", - " optimizer=tf.train.FtrlOptimizer(\n", - " learning_rate=0.1,\n", - " l1_regularization_strength=10.0,\n", - " l2_regularization_strength=0.0))\n", - "\n", - "model_l1.train(train_inpf)\n", - "\n", - "results = model_l1.evaluate(test_inpf)\n", - "clear_output()\n", - "for key in sorted(results):\n", - " print('%s: %0.2f' % (key, results[key]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "ofmPL212JIy2", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model_l2 = tf.estimator.LinearClassifier(\n", - " feature_columns=base_columns + crossed_columns,\n", - " optimizer=tf.train.FtrlOptimizer(\n", - " learning_rate=0.1,\n", - " l1_regularization_strength=0.0,\n", - " l2_regularization_strength=10.0))\n", - "\n", - "model_l2.train(train_inpf)\n", - "\n", - "results = model_l2.evaluate(test_inpf)\n", - "clear_output()\n", - "for key in sorted(results):\n", - " print('%s: %0.2f' % (key, results[key]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Lp1Rfy_k4e7w", - "colab_type": "text" - }, - "source": [ - "These regularized models don't perform much better than the base model. Let's look at the model's weight distributions to better see the effect of the regularization:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Wb6093N04XlS", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def get_flat_weights(model):\n", - " weight_names = [\n", - " name for name in model.get_variable_names()\n", - " if \"linear_model\" in name and \"Ftrl\" not in name]\n", - "\n", - " weight_values = [model.get_variable_value(name) for name in weight_names]\n", - "\n", - " weights_flat = np.concatenate([item.flatten() for item in weight_values], axis=0)\n", - "\n", - " return weights_flat\n", - "\n", - "weights_flat = get_flat_weights(model)\n", - "weights_flat_l1 = get_flat_weights(model_l1)\n", - "weights_flat_l2 = get_flat_weights(model_l2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GskJmtfmL0p-", - "colab_type": "text" - }, - "source": [ - "The models have many zero-valued weights caused by unused hash bins (there are many more hash bins than categories in some columns). We can mask these weights when viewing the weight distributions:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "rM3agZe3MT3D", - "colab_type": "code", - "colab": {} - }, - "source": [ - "weight_mask = weights_flat != 0\n", - "\n", - "weights_base = weights_flat[weight_mask]\n", - "weights_l1 = weights_flat_l1[weight_mask]\n", - "weights_l2 = weights_flat_l2[weight_mask]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NqBpxLLQNEBE", - "colab_type": "text" - }, - "source": [ - "Now plot the distributions:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "IdFK7wWa5_0K", - "colab_type": "code", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "_ = plt.hist(weights_base, bins=np.linspace(-3,3,30))\n", - "plt.title('Base Model')\n", - "plt.ylim([0,500])\n", - "\n", - "plt.figure()\n", - "_ = plt.hist(weights_l1, bins=np.linspace(-3,3,30))\n", - "plt.title('L1 - Regularization')\n", - "plt.ylim([0,500])\n", - "\n", - "plt.figure()\n", - "_ = plt.hist(weights_l2, bins=np.linspace(-3,3,30))\n", - "plt.title('L2 - Regularization')\n", - "_=plt.ylim([0,500])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Mv6knhFa5-iJ", - "colab_type": "text" - }, - "source": [ - "Both types of regularization squeeze the distribution of weights towards zero. L2 regularization has a greater effect in the tails of the distribution eliminating extreme weights. L1 regularization produces more exactly-zero values, in this case it sets ~200 to zero." - ] - } - ] -} \ No newline at end of file diff --git a/site/en/r1/tutorials/images/deep_cnn.md b/site/en/r1/tutorials/images/deep_cnn.md index 9511b4f7ecf..885f3907aa7 100644 --- a/site/en/r1/tutorials/images/deep_cnn.md +++ b/site/en/r1/tutorials/images/deep_cnn.md @@ -80,15 +80,15 @@ for details. It consists of 1,068,298 learnable parameters and requires about ## Code Organization The code for this tutorial resides in -[`models/tutorials/image/cifar10/`](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10/). +[`models/tutorials/image/cifar10/`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/). File | Purpose --- | --- -[`cifar10_input.py`](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10/cifar10_input.py) | Loads CIFAR-10 dataset using [tensorflow-datasets library](https://github.com/tensorflow/datasets). -[`cifar10.py`](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10/cifar10.py) | Builds the CIFAR-10 model. -[`cifar10_train.py`](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10/cifar10_train.py) | Trains a CIFAR-10 model on a CPU or GPU. -[`cifar10_multi_gpu_train.py`](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10/cifar10_multi_gpu_train.py) | Trains a CIFAR-10 model on multiple GPUs. -[`cifar10_eval.py`](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10/cifar10_eval.py) | Evaluates the predictive performance of a CIFAR-10 model. +[`cifar10_input.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10_input.py) | Loads CIFAR-10 dataset using [tensorflow-datasets library](https://github.com/tensorflow/datasets). +[`cifar10.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10.py) | Builds the CIFAR-10 model. +[`cifar10_train.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10_train.py) | Trains a CIFAR-10 model on a CPU or GPU. +[`cifar10_multi_gpu_train.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10_multi_gpu_train.py) | Trains a CIFAR-10 model on multiple GPUs. +[`cifar10_eval.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10_eval.py) | Evaluates the predictive performance of a CIFAR-10 model. To run this tutorial, you will need to: @@ -99,7 +99,7 @@ pip install tensorflow-datasets ## CIFAR-10 Model The CIFAR-10 network is largely contained in -[`cifar10.py`](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10/cifar10.py). +[`cifar10.py`](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/image/cifar10/cifar10.py). The complete training graph contains roughly 765 operations. We find that we can make the code most reusable by constructing the graph with the following modules: @@ -108,7 +108,7 @@ reusable by constructing the graph with the following modules: operations that read and preprocess CIFAR images for evaluation and training, respectively. 1. [**Model prediction:**](#model-prediction) `inference()` -adds operations that perform inference, i.e. classification, on supplied images. +adds operations that perform inference, i.e., classification, on supplied images. 1. [**Model training:**](#model-training) `loss()` and `train()` add operations that compute the loss, gradients, variable updates and visualization summaries. @@ -405,7 +405,7 @@ a "tower". We must set two attributes for each tower: * A unique name for all operations within a tower. `tf.name_scope` provides this unique name by prepending a scope. For instance, all operations in -the first tower are prepended with `tower_0`, e.g. `tower_0/conv1/Conv2D`. +the first tower are prepended with `tower_0`, e.g., `tower_0/conv1/Conv2D`. * A preferred hardware device to run the operation within a tower. `tf.device` specifies this. For diff --git a/site/en/r1/tutorials/images/hub_with_keras.ipynb b/site/en/r1/tutorials/images/hub_with_keras.ipynb index 5879ce0e65d..f4e683e8936 100644 --- a/site/en/r1/tutorials/images/hub_with_keras.ipynb +++ b/site/en/r1/tutorials/images/hub_with_keras.ipynb @@ -1,28 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "hub_with_keras.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "W_tvPdyfA-BL" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "W_tvPdyfA-BL" }, "source": [ @@ -31,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "0O_LFhwSBCjm", - "colab": {} + "id": "0O_LFhwSBCjm" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -49,14 +29,11 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PWUmcKKjtwXL" }, "source": [ @@ -69,13 +46,29 @@ " \n", " View source on GitHub\n", " \n", + " \n", + " See TF Hub model\n", + " \n", "" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "PWUmcKKjtwXL" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "crU-iluJIEzw" }, "source": [ @@ -91,7 +84,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CKFUvuEho9Th" }, "source": [ @@ -101,7 +93,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7RVsYZLEpEWs" }, "source": [ @@ -110,54 +101,47 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "nSiOCtv_Rwi_", - "colab": {} + "id": "nSiOCtv_Rwi_" }, + "outputs": [], "source": [ "!pip install -U tensorflow_hub" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "OGNpmn43C0O6", - "colab": {} + "id": "OGNpmn43C0O6" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "import matplotlib.pylab as plt\n", "\n", - "import tensorflow as tf\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] + "import tensorflow.compat.v1 as tf\n" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "-V4l8oN8Lw2q", - "colab": {} + "id": "-V4l8oN8Lw2q" }, + "outputs": [], "source": [ + "import os\n", "import tensorflow_hub as hub\n", + "from tensorflow.keras import layers\n", "\n", - "from tensorflow.keras import layers" - ], - "execution_count": 0, - "outputs": [] + "# Load compressed models from tensorflow_hub\n", + "os.environ[\"TFHUB_MODEL_LOAD_FORMAT\"] = \"COMPRESSED\"" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "s4YuF5HvpM1W" }, "source": [ @@ -167,7 +151,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xEY_Ow5loN6q" }, "source": [ @@ -180,39 +163,34 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "both", - "colab_type": "code", - "id": "feiXojVXAbI9", - "colab": {} + "id": "feiXojVXAbI9" }, + "outputs": [], "source": [ "classifier_url =\"https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/2\" #@param {type:\"string\"}" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "y_6bGjoPtzau", - "colab": {} + "id": "y_6bGjoPtzau" }, + "outputs": [], "source": [ "IMAGE_SHAPE = (224, 224)\n", "\n", "classifier = tf.keras.Sequential([\n", " hub.KerasLayer(classifier_url, input_shape=IMAGE_SHAPE+(3,))\n", "])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pwZXaoV0uXp2" }, "source": [ @@ -222,7 +200,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TQItP1i55-di" }, "source": [ @@ -231,11 +208,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "w5wDjXNjuXGD", - "colab": {} + "id": "w5wDjXNjuXGD" }, + "outputs": [], "source": [ "import numpy as np\n", "import PIL.Image as Image\n", @@ -243,28 +220,23 @@ "grace_hopper = tf.keras.utils.get_file('image.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg')\n", "grace_hopper = Image.open(grace_hopper).resize(IMAGE_SHAPE)\n", "grace_hopper" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "BEmmBnGbLxPp", - "colab": {} + "id": "BEmmBnGbLxPp" }, + "outputs": [], "source": [ "grace_hopper = np.array(grace_hopper)/255.0\n", "grace_hopper.shape" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0Ic8OEEo2b73" }, "source": [ @@ -273,22 +245,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "EMquyn29v8q3", - "colab": {} + "id": "EMquyn29v8q3" }, + "outputs": [], "source": [ "result = classifier.predict(grace_hopper[np.newaxis, ...])\n", "result.shape" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NKzjqENF6jDF" }, "source": [ @@ -299,22 +268,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "rgXb44vt6goJ", - "colab": {} + "id": "rgXb44vt6goJ" }, + "outputs": [], "source": [ "predicted_class = np.argmax(result[0], axis=-1)\n", "predicted_class" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YrxLMajMoxkf" }, "source": [ @@ -326,38 +292,33 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ij6SrDxcxzry", - "colab": {} + "id": "ij6SrDxcxzry" }, + "outputs": [], "source": [ "labels_path = tf.keras.utils.get_file('ImageNetLabels.txt','https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt')\n", "imagenet_labels = np.array(open(labels_path).read().splitlines())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "uzziRK3Z2VQo", - "colab": {} + "id": "uzziRK3Z2VQo" }, + "outputs": [], "source": [ "plt.imshow(grace_hopper)\n", "plt.axis('off')\n", "predicted_class_name = imagenet_labels[predicted_class]\n", "_ = plt.title(\"Prediction: \" + predicted_class_name.title())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "amfzqn1Oo7Om" }, "source": [ @@ -367,7 +328,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "K-nIpVJ94xrw" }, "source": [ @@ -377,7 +337,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Z93vvAdGxDMD" }, "source": [ @@ -388,23 +347,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "DrIUV3V0xDL_", - "colab": {} + "id": "DrIUV3V0xDL_" }, + "outputs": [], "source": [ "data_root = tf.keras.utils.get_file(\n", " 'flower_photos','https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", " untar=True)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jFHdp18ccah7" }, "source": [ @@ -417,22 +373,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "2PwQ_wYDcii9", - "colab": {} + "id": "2PwQ_wYDcii9" }, + "outputs": [], "source": [ "image_generator = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1/255)\n", "image_data = image_generator.flow_from_directory(str(data_root), target_size=IMAGE_SHAPE)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0p7iDOhIcqY2" }, "source": [ @@ -441,24 +394,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "W4lDPkn2cpWZ", - "colab": {} + "id": "W4lDPkn2cpWZ" }, + "outputs": [], "source": [ "for image_batch, label_batch in image_data:\n", " print(\"Image batch shape: \", image_batch.shape)\n", " print(\"Label batch shape: \", label_batch.shape)\n", " break" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0gTN7M_GxDLx" }, "source": [ @@ -468,7 +418,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "O3fvrZR8xDLv" }, "source": [ @@ -477,36 +426,31 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "nbyg6tcyxDLh", - "colab": {} + "id": "nbyg6tcyxDLh" }, + "outputs": [], "source": [ "result_batch = classifier.predict(image_batch)\n", "result_batch.shape" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Kv7ZwuR4xDLc", - "colab": {} + "id": "Kv7ZwuR4xDLc" }, + "outputs": [], "source": [ "predicted_class_names = imagenet_labels[np.argmax(result_batch, axis=-1)]\n", "predicted_class_names" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QmvSWg9nxDLa" }, "source": [ @@ -515,11 +459,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "IXTB22SpxDLP", - "colab": {} + "id": "IXTB22SpxDLP" }, + "outputs": [], "source": [ "plt.figure(figsize=(10,9))\n", "plt.subplots_adjust(hspace=0.5)\n", @@ -529,14 +473,11 @@ " plt.title(predicted_class_names[n])\n", " plt.axis('off')\n", "_ = plt.suptitle(\"ImageNet predictions\")" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FUa3YkvhxDLM" }, "source": [ @@ -548,7 +489,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JzV457OXreQP" }, "source": [ @@ -561,22 +501,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "both", - "colab_type": "code", - "id": "4bw8Jf94DSnP", - "colab": {} + "id": "4bw8Jf94DSnP" }, + "outputs": [], "source": [ "feature_extractor_url = \"https://tfhub.dev/google/tf2-preview/mobilenet_v2/feature_vector/2\" #@param {type:\"string\"}" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "sgwmHugQF-PD" }, "source": [ @@ -585,22 +522,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "5wB030nezBwI", - "colab": {} + "id": "5wB030nezBwI" }, + "outputs": [], "source": [ "feature_extractor_layer = hub.KerasLayer(feature_extractor_url,\n", " input_shape=(224,224,3))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GUY-5Eyzuzlu" }, "source": [ @@ -609,22 +543,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Of7i-35F09ls", - "colab": {} + "id": "Of7i-35F09ls" }, + "outputs": [], "source": [ "feature_batch = feature_extractor_layer(image_batch)\n", "print(feature_batch.shape)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CtFmF7A5E4tk" }, "source": [ @@ -633,21 +564,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Jg5ar6rcE4H-", - "colab": {} + "id": "Jg5ar6rcE4H-" }, + "outputs": [], "source": [ "feature_extractor_layer.trainable = False" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RPVeouTksO9q" }, "source": [ @@ -658,11 +586,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "mGcY27fY1q3Q", - "colab": {} + "id": "mGcY27fY1q3Q" }, + "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " feature_extractor_layer,\n", @@ -670,40 +598,33 @@ "])\n", "\n", "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "G9VkAz00HOJx", - "colab": {} + "id": "G9VkAz00HOJx" }, + "outputs": [], "source": [ "predictions = model(image_batch)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "sB7sVGJ23vrY", - "colab": {} + "id": "sB7sVGJ23vrY" }, + "outputs": [], "source": [ "predictions.shape" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OHbXQqIquFxQ" }, "source": [ @@ -714,24 +635,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3n0Wb9ylKd8R", - "colab": {} + "id": "3n0Wb9ylKd8R" }, + "outputs": [], "source": [ "model.compile(\n", " optimizer=tf.keras.optimizers.Adam(),\n", " loss='categorical_crossentropy',\n", " metrics=['acc'])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "58-BLV7dupJA" }, "source": [ @@ -742,11 +660,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jZ54Gubac4Lu", - "colab": {} + "id": "jZ54Gubac4Lu" }, + "outputs": [], "source": [ "class CollectBatchStats(tf.keras.callbacks.Callback):\n", " def __init__(self):\n", @@ -757,17 +675,15 @@ " self.batch_losses.append(logs['loss'])\n", " self.batch_acc.append(logs['acc'])\n", " self.model.reset_metrics()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "EyMDJxt2HdHr", - "colab": {} + "id": "EyMDJxt2HdHr" }, + "outputs": [], "source": [ "steps_per_epoch = np.ceil(image_data.samples/image_data.batch_size)\n", "\n", @@ -776,14 +692,11 @@ "history = model.fit(image_data, epochs=2,\n", " steps_per_epoch=steps_per_epoch,\n", " callbacks = [batch_stats_callback])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Kd0N272B9Q0b" }, "source": [ @@ -792,42 +705,37 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "A5RfS1QIIP-P", - "colab": {} + "id": "A5RfS1QIIP-P" }, + "outputs": [], "source": [ "plt.figure()\n", "plt.ylabel(\"Loss\")\n", "plt.xlabel(\"Training Steps\")\n", "plt.ylim([0,2])\n", "plt.plot(batch_stats_callback.batch_losses)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3uvX11avTiDg", - "colab": {} + "id": "3uvX11avTiDg" }, + "outputs": [], "source": [ "plt.figure()\n", "plt.ylabel(\"Accuracy\")\n", "plt.xlabel(\"Training Steps\")\n", "plt.ylim([0,1])\n", "plt.plot(batch_stats_callback.batch_acc)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kb__ZN8uFn-D" }, "source": [ @@ -838,23 +746,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "JGbEf5l1I4jz", - "colab": {} + "id": "JGbEf5l1I4jz" }, + "outputs": [], "source": [ "class_names = sorted(image_data.class_indices.items(), key=lambda pair:pair[1])\n", "class_names = np.array([key.title() for key, value in class_names])\n", "class_names" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4Olg6MsNGJTL" }, "source": [ @@ -863,23 +768,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "fCLVCpEjJ_VP", - "colab": {} + "id": "fCLVCpEjJ_VP" }, + "outputs": [], "source": [ "predicted_batch = model.predict(image_batch)\n", "predicted_id = np.argmax(predicted_batch, axis=-1)\n", "predicted_label_batch = class_names[predicted_id]" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CkGbZxl9GZs-" }, "source": [ @@ -888,24 +790,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "rpFQR1MPMtT1", - "colab": {} + "id": "rpFQR1MPMtT1" }, + "outputs": [], "source": [ "label_id = np.argmax(label_batch, axis=-1)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "wC_AYRJU9NQe", - "colab": {} + "id": "wC_AYRJU9NQe" }, + "outputs": [], "source": [ "plt.figure(figsize=(10,9))\n", "plt.subplots_adjust(hspace=0.5)\n", @@ -916,14 +816,11 @@ " plt.title(predicted_label_batch[n].title(), color=color)\n", " plt.axis('off')\n", "_ = plt.suptitle(\"Model predictions (green: correct, red: incorrect)\")" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uRcJnAABr22x" }, "source": [ @@ -934,27 +831,24 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "PLcqg-RmsLno", - "colab": {} + "id": "PLcqg-RmsLno" }, + "outputs": [], "source": [ "import time\n", "t = time.time()\n", "\n", "export_path = \"/tmp/saved_models/{}\".format(int(t))\n", - "tf.keras.experimental.export_saved_model(model, export_path)\n", + "model.save(export_path)\n", "\n", "export_path" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AhQ9liIUsPsi" }, "source": [ @@ -963,54 +857,62 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7nI5fvkAQvbS", - "colab": {} + "id": "7nI5fvkAQvbS" }, + "outputs": [], "source": [ - "reloaded = tf.keras.experimental.load_from_saved_model(export_path, custom_objects={'KerasLayer':hub.KerasLayer})" - ], - "execution_count": 0, - "outputs": [] + "reloaded = tf.keras.models.load_model(export_path, custom_objects={'KerasLayer':hub.KerasLayer})" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jor83-LqI8xW", - "colab": {} + "id": "jor83-LqI8xW" }, + "outputs": [], "source": [ "result_batch = model.predict(image_batch)\n", "reloaded_result_batch = reloaded.predict(image_batch)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "dnZO14taYPH6", - "colab": {} + "id": "dnZO14taYPH6" }, + "outputs": [], "source": [ "abs(reloaded_result_batch - result_batch).max()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TYZd4MNiV3Rc" }, "source": [ - "This saved model can loaded for inference later, or converted to [TFLite](https://www.tensorflow.org/lite/convert/) or [TFjs](https://github.com/tensorflow/tfjs-converter).\n", - "\n" + "This saved model can be loaded for inference later, or converted to [TFLite](https://www.tensorflow.org/lite/convert/) or [TFjs](https://github.com/tensorflow/tfjs-converter).\n" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [ + "W_tvPdyfA-BL" + ], + "name": "hub_with_keras.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/images/image_recognition.md b/site/en/r1/tutorials/images/image_recognition.md index 0be884de403..cb66e594629 100644 --- a/site/en/r1/tutorials/images/image_recognition.md +++ b/site/en/r1/tutorials/images/image_recognition.md @@ -140,13 +140,13 @@ score of 0.8. -Next, try it out on your own images by supplying the --image= argument, e.g. +Next, try it out on your own images by supplying the --image= argument, e.g., ```bash bazel-bin/tensorflow/examples/label_image/label_image --image=my_image.png ``` -If you look inside the [`tensorflow/examples/label_image/main.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc) +If you look inside the [`tensorflow/examples/label_image/main.cc`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc) file, you can find out how it works. We hope this code will help you integrate TensorFlow into your own applications, so we will walk step by step through the main functions: @@ -164,7 +164,7 @@ training. If you have a graph that you've trained yourself, you'll just need to adjust the values to match whatever you used during your training process. You can see how they're applied to an image in the -[`ReadTensorFromImageFile()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L88) +[`ReadTensorFromImageFile()`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc#L88) function. ```C++ @@ -334,7 +334,7 @@ The `PrintTopLabels()` function takes those sorted results, and prints them out friendly way. The `CheckTopLabel()` function is very similar, but just makes sure that the top label is the one we expect, for debugging purposes. -At the end, [`main()`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/label_image/main.cc#L252) +At the end, [`main()`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/label_image/main.cc#L252) ties together all of these calls. ```C++ diff --git a/site/en/r1/tutorials/images/transfer_learning.ipynb b/site/en/r1/tutorials/images/transfer_learning.ipynb index 68aadb816db..25779babd17 100644 --- a/site/en/r1/tutorials/images/transfer_learning.ipynb +++ b/site/en/r1/tutorials/images/transfer_learning.ipynb @@ -1,26 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "transfer_learning.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "77gENRVX40S7" }, "source": [ @@ -29,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "d8jyt37T42Vf", - "colab": {} + "id": "d8jyt37T42Vf" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -47,14 +29,11 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hRTa3Ee15WsJ" }, "source": [ @@ -64,7 +43,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "dQHMcypT3vDT" }, "source": [ @@ -81,7 +59,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "dQHMcypT3vDT" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "2X4KyhORdSeO" }, "source": [ @@ -111,17 +102,16 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "iBMcobPHdD8O", - "colab": {} + "id": "iBMcobPHdD8O" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "import os\n", "\n", - "import tensorflow as tf\n", + "import tensorflow.compat.v1 as tf\n", + "\n", "from tensorflow import keras\n", "print(\"TensorFlow version is \", tf.__version__)\n", "\n", @@ -129,14 +119,11 @@ "\n", "import matplotlib.pyplot as plt\n", "import matplotlib.image as mpimg" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "v77rlkCKW0IJ" }, "source": [ @@ -146,7 +133,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "aXzwKdouXf1h" }, "source": [ @@ -156,23 +142,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "nRnO59Kr6enO", - "colab": {} + "id": "nRnO59Kr6enO" }, + "outputs": [], "source": [ "zip_file = tf.keras.utils.get_file(origin=\"https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip\",\n", " fname=\"cats_and_dogs_filtered.zip\", extract=True)\n", "base_dir, _ = os.path.splitext(zip_file)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9_6h-c5EXN91" }, "source": [ @@ -182,11 +165,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "RWcldM4TXLen", - "colab": {} + "id": "RWcldM4TXLen" }, + "outputs": [], "source": [ "train_dir = os.path.join(base_dir, 'train')\n", "validation_dir = os.path.join(base_dir, 'validation')\n", @@ -206,14 +189,11 @@ "# Directory with our validation dog pictures\n", "validation_dogs_dir = os.path.join(validation_dir, 'dogs')\n", "print ('Total validation dog images:', len(os.listdir(validation_dogs_dir)))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wvidPx6jeFzf" }, "source": [ @@ -228,11 +208,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "y3PM6GVHcC31", - "colab": {} + "id": "y3PM6GVHcC31" }, + "outputs": [], "source": [ "image_size = 160 # All images will be resized to 160x160\n", "batch_size = 32\n", @@ -257,14 +237,11 @@ " target_size=(image_size, image_size),\n", " batch_size=batch_size,\n", " class_mode='binary')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OkH-kazQecHB" }, "source": [ @@ -278,11 +255,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "19IQ2gqneqmS", - "colab": {} + "id": "19IQ2gqneqmS" }, + "outputs": [], "source": [ "IMG_SHAPE = (image_size, image_size, 3)\n", "\n", @@ -290,14 +267,11 @@ "base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,\n", " include_top=False,\n", " weights='imagenet')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rlx56nQtfe8Y" }, "source": [ @@ -308,7 +282,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CnMLieHBCwil" }, "source": [ @@ -318,35 +291,30 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "OTCJH4bphOeo", - "colab": {} + "id": "OTCJH4bphOeo" }, + "outputs": [], "source": [ "base_model.trainable = False" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "KpbzSmPkDa-N", - "colab": {} + "id": "KpbzSmPkDa-N" }, + "outputs": [], "source": [ "# Let's take a look at the base model architecture\n", "base_model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wdMRM8YModbk" }, "source": [ @@ -356,7 +324,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0iqnBeZrfoIc" }, "source": [ @@ -365,25 +332,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "eApvroIyn1K0", - "colab": {} + "id": "eApvroIyn1K0" }, + "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " base_model,\n", " keras.layers.GlobalAveragePooling2D(),\n", " keras.layers.Dense(1, activation='sigmoid')\n", "])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "g0ylJXE_kRLi" }, "source": [ @@ -394,36 +358,31 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "RpR8HdyMhukJ", - "colab": {} + "id": "RpR8HdyMhukJ" }, + "outputs": [], "source": [ - "model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.0001),\n", + "model.compile(optimizer=tf.keras.optimizers.RMSprop(learning_rate=0.0001),\n", " loss='binary_crossentropy',\n", " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "I8ARiyMFsgbH", - "colab": {} + "id": "I8ARiyMFsgbH" }, + "outputs": [], "source": [ "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lxOcmVr0ydFZ" }, "source": [ @@ -432,21 +391,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "krvBumovycVA", - "colab": {} + "id": "krvBumovycVA" }, + "outputs": [], "source": [ "len(model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RxvgOYTDSWTx" }, "source": [ @@ -459,11 +415,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Om4O3EESkab1", - "colab": {} + "id": "Om4O3EESkab1" }, + "outputs": [], "source": [ "epochs = 10\n", "steps_per_epoch = train_generator.n // batch_size\n", @@ -475,14 +431,11 @@ " workers=4,\n", " validation_data=validation_generator,\n", " validation_steps=validation_steps)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Hd94CKImf8vi" }, "source": [ @@ -494,7 +447,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "l7HOsQTPNgO9" }, "source": [ @@ -505,14 +457,14 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "53OTCh3jnbwV", - "colab": {} + "id": "53OTCh3jnbwV" }, + "outputs": [], "source": [ - "acc = history.history['acc']\n", - "val_acc = history.history['val_acc']\n", + "acc = history.history['accuracy']\n", + "val_acc = history.history['val_accuracy']\n", "\n", "loss = history.history['loss']\n", "val_loss = history.history['val_loss']\n", @@ -534,14 +486,11 @@ "plt.ylim([0,max(plt.ylim())])\n", "plt.title('Training and Validation Loss')\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CqwV-CRdS6Nv" }, "source": [ @@ -556,7 +505,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CPXnzUK0QonF" }, "source": [ @@ -566,7 +514,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rfxv_ifotQak" }, "source": [ @@ -575,24 +522,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "4nzcagVitLQm", - "colab": {} + "id": "4nzcagVitLQm" }, + "outputs": [], "source": [ "base_model.trainable = True" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "-4HgVAacRs5v", - "colab": {} + "id": "-4HgVAacRs5v" }, + "outputs": [], "source": [ "# Let's take a look to see how many layers are in the base model\n", "print(\"Number of layers in the base model: \", len(base_model.layers))\n", @@ -602,15 +547,12 @@ "\n", "# Freeze all the layers before the `fine_tune_at` layer\n", "for layer in base_model.layers[:fine_tune_at]:\n", - " layer.trainable = False" - ], - "execution_count": 0, - "outputs": [] + " layer.trainable = False" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4Uk1dgsxT0IS" }, "source": [ @@ -621,50 +563,42 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "NtUnaz0WUDva", - "colab": {} + "id": "NtUnaz0WUDva" }, + "outputs": [], "source": [ - "\n", - "model.compile(optimizer = tf.keras.optimizers.RMSprop(lr=2e-5),\n", + "model.compile(optimizer = tf.keras.optimizers.RMSprop(learning_rate=2e-5),\n", " loss='binary_crossentropy',\n", " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "WwBWy7J2kZvA", - "colab": {} + "id": "WwBWy7J2kZvA" }, + "outputs": [], "source": [ "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "bNXelbMQtonr", - "colab": {} + "id": "bNXelbMQtonr" }, + "outputs": [], "source": [ "len(model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4G5O4jd6TuAG" }, "source": [ @@ -674,7 +608,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0foWUN-yDLo_" }, "source": [ @@ -683,11 +616,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ECQLkAsFTlun", - "colab": {} + "id": "ECQLkAsFTlun" }, + "outputs": [], "source": [ "history_fine = model.fit_generator(train_generator,\n", " steps_per_epoch = steps_per_epoch,\n", @@ -695,14 +628,11 @@ " workers=4,\n", " validation_data=validation_generator,\n", " validation_steps=validation_steps)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TfXEmsxQf6eP" }, "source": [ @@ -716,7 +646,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DNtfNZKlInGT" }, "source": [ @@ -727,28 +656,26 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "PpA8PlpQKygw", - "colab": {} + "id": "PpA8PlpQKygw" }, + "outputs": [], "source": [ - "acc += history_fine.history['acc']\n", - "val_acc += history_fine.history['val_acc']\n", + "acc += history_fine.history['accuracy']\n", + "val_acc += history_fine.history['val_accuracy']\n", "\n", "loss += history_fine.history['loss']\n", "val_loss += history_fine.history['val_loss']" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "chW103JUItdk", - "colab": {} + "id": "chW103JUItdk" }, + "outputs": [], "source": [ "plt.figure(figsize=(8, 8))\n", "plt.subplot(2, 1, 1)\n", @@ -767,14 +694,33 @@ "plt.legend(loc='upper right')\n", "plt.title('Training and Validation Loss')\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tTrNmiH5vF4A" + }, + "outputs": [], + "source": [ + "acc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZzvAvWZevLWd" + }, + "outputs": [], + "source": [ + "loss" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_TZTwG7nhm0C" }, "source": [ @@ -786,5 +732,19 @@ "In this case, we tune our weights such that we learn highly specified and high level features specific to our dataset. This only make sense when the training dataset is large and very similar to the original dataset that the pre-trained model was trained on.\n" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "transfer_learning.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/keras/README.md b/site/en/r1/tutorials/keras/README.md index 4da2f72dca9..47aca7e0052 100644 --- a/site/en/r1/tutorials/keras/README.md +++ b/site/en/r1/tutorials/keras/README.md @@ -4,7 +4,7 @@ This notebook collection is inspired by the book *[Deep Learning with Python](https://books.google.com/books?id=Yo3CAQAACAAJ)*. These tutorials use `tf.keras`, TensorFlow's high-level Python API for building and training deep learning models. To learn more about using Keras with -TensorFlow, see the [TensorFlow Keras Guide](../../guide/keras.ipynb). +TensorFlow, see the [TensorFlow Keras Guide](https://www.tensorflow.org/guide/keras). Publisher's note: *Deep Learning with Python* introduces the field of deep learning using the Python language and the powerful Keras library. Written by diff --git a/site/en/r1/tutorials/keras/basic_classification.ipynb b/site/en/r1/tutorials/keras/basic_classification.ipynb index 58994768bf5..14950538ce4 100644 --- a/site/en/r1/tutorials/keras/basic_classification.ipynb +++ b/site/en/r1/tutorials/keras/basic_classification.ipynb @@ -1,26 +1,9 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "id": "MhoQ0WE77laV", - "colab_type": "text" + "id": "MhoQ0WE77laV" }, "source": [ "##### Copyright 2018 The TensorFlow Authors." @@ -28,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "_ckMIh7O7s6D", - "colab_type": "code", "cellView": "form", - "colab": {} + "id": "_ckMIh7O7s6D" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -46,18 +29,16 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "vasWnqRgy1H4", - "colab_type": "code", "cellView": "form", - "colab": {} + "id": "vasWnqRgy1H4" }, + "outputs": [], "source": [ "#@title MIT License\n", "#\n", @@ -80,15 +61,12 @@ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "jYysdyb-CaWM", - "colab_type": "text" + "id": "jYysdyb-CaWM" }, "source": [ "# Train your first neural network: basic classification" @@ -97,8 +75,7 @@ { "cell_type": "markdown", "metadata": { - "id": "S5Uhzt6vVIB2", - "colab_type": "text" + "id": "S5Uhzt6vVIB2" }, "source": [ "\n", @@ -114,8 +91,21 @@ { "cell_type": "markdown", "metadata": { - "id": "FbVhjPpzn6BM", - "colab_type": "text" + "id": "S5Uhzt6vVIB2" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FbVhjPpzn6BM" }, "source": [ "This guide trains a neural network model to classify images of clothing, like sneakers and shirts. It's okay if you don't understand all the details, this is a fast-paced overview of a complete TensorFlow program with the details explained as we go.\n", @@ -125,16 +115,15 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "dzLKpmZICaWN", - "colab_type": "code", - "colab": {} + "id": "dzLKpmZICaWN" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "# TensorFlow and tf.keras\n", - "import tensorflow as tf\n", + "import tensorflow.compat.v1 as tf\n", + "\n", "from tensorflow import keras\n", "\n", "# Helper libraries\n", @@ -142,15 +131,12 @@ "import matplotlib.pyplot as plt\n", "\n", "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "yR0EdgrLCaWR", - "colab_type": "text" + "id": "yR0EdgrLCaWR" }, "source": [ "## Import the Fashion MNIST dataset" @@ -159,8 +145,7 @@ { "cell_type": "markdown", "metadata": { - "id": "DLdCchMdCaWQ", - "colab_type": "text" + "id": "DLdCchMdCaWQ" }, "source": [ "This guide uses the [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset which contains 70,000 grayscale images in 10 categories. The images show individual articles of clothing at low resolution (28 by 28 pixels), as seen here:\n", @@ -184,24 +169,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "7MqDQO0KCaWS", - "colab_type": "code", - "colab": {} + "id": "7MqDQO0KCaWS" }, + "outputs": [], "source": [ "fashion_mnist = keras.datasets.fashion_mnist\n", "\n", "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "t9FDsUlxCaWW", - "colab_type": "text" + "id": "t9FDsUlxCaWW" }, "source": [ "Loading the dataset returns four NumPy arrays:\n", @@ -263,23 +245,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "IjnLH5S2CaWx", - "colab_type": "code", - "colab": {} + "id": "IjnLH5S2CaWx" }, + "outputs": [], "source": [ "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "Brm0b_KACaWX", - "colab_type": "text" + "id": "Brm0b_KACaWX" }, "source": [ "## Explore the data\n", @@ -289,22 +268,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "zW5k_xz1CaWX", - "colab_type": "code", - "colab": {} + "id": "zW5k_xz1CaWX" }, + "outputs": [], "source": [ "train_images.shape" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "cIAcvQqMCaWf", - "colab_type": "text" + "id": "cIAcvQqMCaWf" }, "source": [ "Likewise, there are 60,000 labels in the training set:" @@ -312,22 +288,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "TRFYHB2mCaWb", - "colab_type": "code", - "colab": {} + "id": "TRFYHB2mCaWb" }, + "outputs": [], "source": [ "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "YSlYxFuRCaWk", - "colab_type": "text" + "id": "YSlYxFuRCaWk" }, "source": [ "Each label is an integer between 0 and 9:" @@ -335,22 +308,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "XKnCTHz4CaWg", - "colab_type": "code", - "colab": {} + "id": "XKnCTHz4CaWg" }, + "outputs": [], "source": [ "train_labels" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "TMPI88iZpO2T", - "colab_type": "text" + "id": "TMPI88iZpO2T" }, "source": [ "There are 10,000 images in the test set. Again, each image is represented as 28 x 28 pixels:" @@ -358,22 +328,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "2KFnYlcwCaWl", - "colab_type": "code", - "colab": {} + "id": "2KFnYlcwCaWl" }, + "outputs": [], "source": [ "test_images.shape" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "rd0A0Iu0CaWq", - "colab_type": "text" + "id": "rd0A0Iu0CaWq" }, "source": [ "And the test set contains 10,000 images labels:" @@ -381,22 +348,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "iJmPr5-ACaWn", - "colab_type": "code", - "colab": {} + "id": "iJmPr5-ACaWn" }, + "outputs": [], "source": [ "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "ES6uQoLKCaWr", - "colab_type": "text" + "id": "ES6uQoLKCaWr" }, "source": [ "## Preprocess the data\n", @@ -406,26 +370,23 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "m4VEw8Ud9Quh", - "colab_type": "code", - "colab": {} + "id": "m4VEw8Ud9Quh" }, + "outputs": [], "source": [ "plt.figure()\n", "plt.imshow(train_images[0])\n", "plt.colorbar()\n", "plt.grid(False)\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "Wz7l27Lz9S1P", - "colab_type": "text" + "id": "Wz7l27Lz9S1P" }, "source": [ "We scale these values to a range of 0 to 1 before feeding to the neural network model. For this, we divide the values by 255. It's important that the *training set* and the *testing set* are preprocessed in the same way:" @@ -433,24 +394,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "bW5WzIPlCaWv", - "colab_type": "code", - "colab": {} + "id": "bW5WzIPlCaWv" }, + "outputs": [], "source": [ "train_images = train_images / 255.0\n", "\n", "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "Ee638AlnCaWz", - "colab_type": "text" + "id": "Ee638AlnCaWz" }, "source": [ "Display the first 25 images from the *training set* and display the class name below each image. Verify that the data is in the correct format and we're ready to build and train the network." @@ -458,11 +416,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "oZTImqg_CaW1", - "colab_type": "code", - "colab": {} + "id": "oZTImqg_CaW1" }, + "outputs": [], "source": [ "plt.figure(figsize=(10,10))\n", "for i in range(25):\n", @@ -473,15 +431,12 @@ " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", " plt.xlabel(class_names[train_labels[i]])\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "59veuiEZCaW4", - "colab_type": "text" + "id": "59veuiEZCaW4" }, "source": [ "## Build the model\n", @@ -492,8 +447,7 @@ { "cell_type": "markdown", "metadata": { - "id": "Gxg1XGm0eOBy", - "colab_type": "text" + "id": "Gxg1XGm0eOBy" }, "source": [ "### Setup the layers\n", @@ -505,26 +459,23 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "9ODch-OFCaW4", - "colab_type": "code", - "colab": {} + "id": "9ODch-OFCaW4" }, + "outputs": [], "source": [ "model = keras.Sequential([\n", " keras.layers.Flatten(input_shape=(28, 28)),\n", " keras.layers.Dense(128, activation=tf.nn.relu),\n", " keras.layers.Dense(10, activation=tf.nn.softmax)\n", "])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "gut8A_7rCaW6", - "colab_type": "text" + "id": "gut8A_7rCaW6" }, "source": [ "The first layer in this network, `tf.keras.layers.Flatten`, transforms the format of the images from a 2d-array (of 28 by 28 pixels), to a 1d-array of 28 * 28 = 784 pixels. Think of this layer as unstacking rows of pixels in the image and lining them up. This layer has no parameters to learn; it only reformats the data.\n", @@ -542,24 +493,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "Lhan11blCaW7", - "colab_type": "code", - "colab": {} + "id": "Lhan11blCaW7" }, + "outputs": [], "source": [ "model.compile(optimizer='adam',\n", " loss='sparse_categorical_crossentropy',\n", " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "qKF6uW-BCaW-", - "colab_type": "text" + "id": "qKF6uW-BCaW-" }, "source": [ "## Train the model\n", @@ -575,22 +523,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "xvwvpA64CaW_", - "colab_type": "code", - "colab": {} + "id": "xvwvpA64CaW_" }, + "outputs": [], "source": [ "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "W3ZVOhugCaXA", - "colab_type": "text" + "id": "W3ZVOhugCaXA" }, "source": [ "As the model trains, the loss and accuracy metrics are displayed. This model reaches an accuracy of about 0.88 (or 88%) on the training data." @@ -599,8 +544,7 @@ { "cell_type": "markdown", "metadata": { - "id": "oEw4bZgGCaXB", - "colab_type": "text" + "id": "oEw4bZgGCaXB" }, "source": [ "## Evaluate accuracy\n", @@ -610,24 +554,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "VflXLEeECaXC", - "colab_type": "code", - "colab": {} + "id": "VflXLEeECaXC" }, + "outputs": [], "source": [ "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", "\n", "print('Test accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "yWfgsmVXCaXG", - "colab_type": "text" + "id": "yWfgsmVXCaXG" }, "source": [ "It turns out, the accuracy on the test dataset is a little less than the accuracy on the training dataset. This gap between training accuracy and test accuracy is an example of *overfitting*. Overfitting is when a machine learning model performs worse on new data than on their training data." @@ -636,8 +577,7 @@ { "cell_type": "markdown", "metadata": { - "id": "xsoS7CPDCaXH", - "colab_type": "text" + "id": "xsoS7CPDCaXH" }, "source": [ "## Make predictions\n", @@ -647,22 +587,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "Gl91RPhdCaXI", - "colab_type": "code", - "colab": {} + "id": "Gl91RPhdCaXI" }, + "outputs": [], "source": [ "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "x9Kk1voUCaXJ", - "colab_type": "text" + "id": "x9Kk1voUCaXJ" }, "source": [ "Here, the model has predicted the label for each image in the testing set. Let's take a look at the first prediction:" @@ -670,22 +607,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "3DmJEUinCaXK", - "colab_type": "code", - "colab": {} + "id": "3DmJEUinCaXK" }, + "outputs": [], "source": [ "predictions[0]" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "-hw1hgeSCaXN", - "colab_type": "text" + "id": "-hw1hgeSCaXN" }, "source": [ "A prediction is an array of 10 numbers. These describe the \"confidence\" of the model that the image corresponds to each of the 10 different articles of clothing. We can see which label has the highest confidence value:" @@ -693,22 +627,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "qsqenuPnCaXO", - "colab_type": "code", - "colab": {} + "id": "qsqenuPnCaXO" }, + "outputs": [], "source": [ "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "E51yS7iCCaXO", - "colab_type": "text" + "id": "E51yS7iCCaXO" }, "source": [ "So the model is most confident that this image is an ankle boot, or `class_names[9]`. And we can check the test label to see this is correct:" @@ -716,22 +647,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "Sd7Pgsu6CaXP", - "colab_type": "code", - "colab": {} + "id": "Sd7Pgsu6CaXP" }, + "outputs": [], "source": [ "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "ygh2yYC972ne", - "colab_type": "text" + "id": "ygh2yYC972ne" }, "source": [ "We can graph this to look at the full set of 10 class predictions" @@ -739,11 +667,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "DvYmmrpIy6Y1", - "colab_type": "code", - "colab": {} + "id": "DvYmmrpIy6Y1" }, + "outputs": [], "source": [ "def plot_image(i, predictions_array, true_label, img):\n", " predictions_array, true_label, img = predictions_array, true_label[i], img[i]\n", @@ -775,15 +703,12 @@ " \n", " thisplot[predicted_label].set_color('red')\n", " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "d4Ov9OFDMmOD", - "colab_type": "text" + "id": "d4Ov9OFDMmOD" }, "source": [ "Let's look at the 0th image, predictions, and prediction array." @@ -791,11 +716,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "HV5jw-5HwSmO", - "colab_type": "code", - "colab": {} + "id": "HV5jw-5HwSmO" }, + "outputs": [], "source": [ "i = 0\n", "plt.figure(figsize=(6,3))\n", @@ -804,17 +729,15 @@ "plt.subplot(1,2,2)\n", "plot_value_array(i, predictions[i], test_labels)\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "Ko-uzOufSCSe", - "colab_type": "code", - "colab": {} + "id": "Ko-uzOufSCSe" }, + "outputs": [], "source": [ "i = 12\n", "plt.figure(figsize=(6,3))\n", @@ -823,15 +746,12 @@ "plt.subplot(1,2,2)\n", "plot_value_array(i, predictions[i], test_labels)\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "kgdvGD52CaXR", - "colab_type": "text" + "id": "kgdvGD52CaXR" }, "source": [ "Let's plot several images with their predictions. Correct prediction labels are blue and incorrect prediction labels are red. The number gives the percent (out of 100) for the predicted label. Note that it can be wrong even when very confident." @@ -839,11 +759,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "hQlnbqaw2Qu_", - "colab_type": "code", - "colab": {} + "id": "hQlnbqaw2Qu_" }, + "outputs": [], "source": [ "# Plot the first X test images, their predicted label, and the true label\n", "# Color correct predictions in blue, incorrect predictions in red\n", @@ -857,15 +777,12 @@ " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", " plot_value_array(i, predictions[i], test_labels)\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "R32zteKHCaXT", - "colab_type": "text" + "id": "R32zteKHCaXT" }, "source": [ "Finally, use the trained model to make a prediction about a single image." @@ -873,25 +790,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "yRJ7JU7JCaXT", - "colab_type": "code", - "colab": {} + "id": "yRJ7JU7JCaXT" }, + "outputs": [], "source": [ "# Grab an image from the test dataset\n", "img = test_images[1]\n", "\n", "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "vz3bVp21CaXV", - "colab_type": "text" + "id": "vz3bVp21CaXV" }, "source": [ "`tf.keras` models are optimized to make predictions on a *batch*, or collection, of examples at once. So even though we're using a single image, we need to add it to a list:" @@ -899,25 +813,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "lDFh5yF_CaXW", - "colab_type": "code", - "colab": {} + "id": "lDFh5yF_CaXW" }, + "outputs": [], "source": [ "# Add the image to a batch where it's the only member.\n", "img = (np.expand_dims(img,0))\n", "\n", "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "EQ5wLTkcCaXY", - "colab_type": "text" + "id": "EQ5wLTkcCaXY" }, "source": [ "Now predict the image:" @@ -925,39 +836,34 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "o_rzNSdrCaXY", - "colab_type": "code", - "colab": {} + "id": "o_rzNSdrCaXY" }, + "outputs": [], "source": [ "predictions_single = model.predict(img)\n", "\n", "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "6Ai-cpLjO-3A", - "colab_type": "code", - "colab": {} + "id": "6Ai-cpLjO-3A" }, + "outputs": [], "source": [ "plot_value_array(1, predictions_single[0], test_labels)\n", "plt.xticks(range(10), class_names, rotation=45)\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "cU1Y2OAMCaXb", - "colab_type": "text" + "id": "cU1Y2OAMCaXb" }, "source": [ "`model.predict` returns a list of lists, one for each image in the batch of data. Grab the predictions for our (only) image in the batch:" @@ -965,27 +871,37 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "2tRmdq_8CaXb", - "colab_type": "code", - "colab": {} + "id": "2tRmdq_8CaXb" }, + "outputs": [], "source": [ "prediction_result = np.argmax(predictions_single[0])\n", "print(prediction_result)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "YFc2HbEVCaXd", - "colab_type": "text" + "id": "YFc2HbEVCaXd" }, "source": [ "And the model predicts a label of 2." ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "basic_classification.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/keras/basic_regression.ipynb b/site/en/r1/tutorials/keras/basic_regression.ipynb index 704b64ae2cb..4bffd62f982 100644 --- a/site/en/r1/tutorials/keras/basic_regression.ipynb +++ b/site/en/r1/tutorials/keras/basic_regression.ipynb @@ -1,25 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FhGuhbZ6M5tl" }, "source": [ @@ -28,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} + "id": "AwOEIRJC6Une" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -46,18 +29,16 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} + "id": "KyPEtTqk6VdG" }, + "outputs": [], "source": [ "#@title MIT License\n", "#\n", @@ -80,14 +61,11 @@ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "EIdT9iu_Z4Rb" }, "source": [ @@ -97,7 +75,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bBIlTPscrIT9" }, "source": [ @@ -114,7 +91,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "bBIlTPscrIT9" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "AHp3M9ZmrIxj" }, "source": [ @@ -127,60 +117,52 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} + "id": "moB4tpEHxKB3" }, + "outputs": [], "source": [ "# Use seaborn for pairplot\n", "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} + "id": "1rRo8oNqZ-Rj" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "import pathlib\n", "\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "import seaborn as sns\n", "\n", - "import tensorflow as tf\n", + "import tensorflow.compat.v1 as tf\n", + "\n", "from tensorflow import keras\n", "from tensorflow.keras import layers\n", "\n", "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "F_72b0LCNbjx" }, "source": [ "## The Auto MPG dataset\n", "\n", - "The dataset is available from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/).\n", - "\n" + "The dataset is available from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/).\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gFh9ne3FZ-On" }, "source": [ @@ -190,22 +172,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} + "id": "p9kxxgzvzlyz" }, + "outputs": [], "source": [ "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", "dataset_path" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "nslsRLh7Zss4" }, "source": [ @@ -214,11 +193,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} + "id": "CiX2FI4gZtTt" }, + "outputs": [], "source": [ "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", " 'Acceleration', 'Model Year', 'Origin']\n", @@ -228,14 +207,11 @@ "\n", "dataset = raw_dataset.copy()\n", "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3MWuJTKEDM-f" }, "source": [ @@ -246,21 +222,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} + "id": "JEJHhN65a2VV" }, + "outputs": [], "source": [ "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9UPN0KBHa_WI" }, "source": [ @@ -269,21 +242,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} + "id": "4ZUDosChC1UN" }, + "outputs": [], "source": [ "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8XKitwaH4v8h" }, "source": [ @@ -292,37 +262,32 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} + "id": "gWNTD2QjBWFJ" }, + "outputs": [], "source": [ "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} + "id": "ulXz4J7PAUzk" }, + "outputs": [], "source": [ "dataset['USA'] = (origin == 1)*1.0\n", "dataset['Europe'] = (origin == 2)*1.0\n", "dataset['Japan'] = (origin == 3)*1.0\n", "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Cuym4yvk76vU" }, "source": [ @@ -335,22 +300,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} + "id": "qn-IGhUE7_1H" }, + "outputs": [], "source": [ "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "J4ubs136WLNp" }, "source": [ @@ -361,22 +323,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} + "id": "oRKO_x8gWKv-" }, + "outputs": [], "source": [ "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gavKO_6DWRMP" }, "source": [ @@ -385,24 +344,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} + "id": "yi2FzC3T21jR" }, + "outputs": [], "source": [ "train_stats = train_dataset.describe()\n", "train_stats.pop(\"MPG\")\n", "train_stats = train_stats.transpose()\n", "train_stats" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Db7Auq1yXUvh" }, "source": [ @@ -413,22 +369,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} + "id": "t2sluJdCW7jN" }, + "outputs": [], "source": [ "train_labels = train_dataset.pop('MPG')\n", "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mRklxK5s388r" }, "source": [ @@ -440,7 +393,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-ywmerQ6dSox" }, "source": [ @@ -451,24 +403,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} + "id": "JlC5ooJrgjQF" }, + "outputs": [], "source": [ "def norm(x):\n", " return (x - train_stats['mean']) / train_stats['std']\n", "normed_train_data = norm(train_dataset)\n", "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "BuiClDk45eS4" }, "source": [ @@ -480,7 +429,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SmjdzxKzEu1-" }, "source": [ @@ -490,7 +438,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6SWtkIjhrZwa" }, "source": [ @@ -501,11 +448,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} + "id": "c26juK7ZG8j-" }, + "outputs": [], "source": [ "def build_model():\n", " model = keras.Sequential([\n", @@ -520,27 +467,22 @@ " optimizer=optimizer,\n", " metrics=['mean_absolute_error', 'mean_squared_error'])\n", " return model" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} + "id": "cGbPb-PHGbhs" }, + "outputs": [], "source": [ "model = build_model()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Sj49Og4YGULr" }, "source": [ @@ -551,47 +493,40 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} + "id": "ReAD0n6MsFK-" }, + "outputs": [], "source": [ "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Vt6W50qGsJAL" }, "source": [ - "\n", "Now try out the model. Take a batch of `10` examples from the training data and call `model.predict` on it." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} + "id": "-d-gBaVtGTSC" }, + "outputs": [], "source": [ "example_batch = normed_train_data[:10]\n", "example_result = model.predict(example_batch)\n", "example_result" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QlM8KrSOsaYo" }, "source": [ @@ -601,7 +536,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0-qWCsh6DlyH" }, "source": [ @@ -612,11 +546,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} + "id": "sD7qHCmNIOY0" }, + "outputs": [], "source": [ "# Display training progress by printing a single dot for each completed epoch\n", "class PrintDot(keras.callbacks.Callback):\n", @@ -630,14 +564,11 @@ " normed_train_data, train_labels,\n", " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tQm3pc0FYPQB" }, "source": [ @@ -646,26 +577,24 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} + "id": "4Xj91b-dymEy" }, + "outputs": [], "source": [ "hist = pd.DataFrame(history.history)\n", "hist['epoch'] = history.epoch\n", "hist.tail()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} + "id": "B6XriGbVPh2t" }, + "outputs": [], "source": [ "def plot_history(history):\n", " hist = pd.DataFrame(history.history)\n", @@ -694,14 +623,11 @@ "\n", "\n", "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AqsuANc11FYv" }, "source": [ @@ -712,11 +638,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} + "id": "fdMZuhUgzMZ4" }, + "outputs": [], "source": [ "model = build_model()\n", "\n", @@ -727,14 +653,11 @@ " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", "\n", "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3St8-DmrX8P4" }, "source": [ @@ -745,23 +668,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} + "id": "jl_yNr5n1kms" }, + "outputs": [], "source": [ "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", "\n", "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ft603OzXuEZC" }, "source": [ @@ -772,11 +692,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} + "id": "Xe7RXH3N3CWU" }, + "outputs": [], "source": [ "test_predictions = model.predict(normed_test_data).flatten()\n", "\n", @@ -789,14 +709,11 @@ "plt.ylim([0,plt.ylim()[1]])\n", "_ = plt.plot([-100, 100], [-100, 100])\n", "plt.show()\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OrkHGKZcusUo" }, "source": [ @@ -805,25 +722,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} + "id": "f-OHX4DiXd8x" }, + "outputs": [], "source": [ "error = test_predictions - test_labels\n", "plt.hist(error, bins = 25)\n", "plt.xlabel(\"Prediction Error [MPG]\")\n", "_ = plt.ylabel(\"Count\")\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "r9_kI6MHu1UU" }, "source": [ @@ -833,7 +747,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vgGQuV-yqYZH" }, "source": [ @@ -848,5 +761,18 @@ "* Early stopping is a useful technique to prevent overfitting." ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "basic_regression.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/keras/basic_text_classification.ipynb b/site/en/r1/tutorials/keras/basic_text_classification.ipynb index c2ffaf3273a..5424185bcbd 100644 --- a/site/en/r1/tutorials/keras/basic_text_classification.ipynb +++ b/site/en/r1/tutorials/keras/basic_text_classification.ipynb @@ -1,25 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ic4_occAAiAT" }, "source": [ @@ -28,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} + "id": "ioaprt5q5US7" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -46,18 +29,16 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} + "id": "yCl0eTNH5RS3" }, + "outputs": [], "source": [ "#@title MIT License\n", "#\n", @@ -80,14 +61,11 @@ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ItXfxkxvosLH" }, "source": [ @@ -97,7 +75,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hKY4XMc9o8iB" }, "source": [ @@ -114,11 +91,23 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "3c5H4DsfCCmZ" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "Eg62Pmz3o83v" }, "source": [ - "\n", "This notebook classifies movie reviews as *positive* or *negative* using the text of the review. This is an example of *binary*—or two-class—classification, an important and widely applicable kind of machine learning problem.\n", "\n", "We'll use the [IMDB dataset](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) that contains the text of 50,000 movie reviews from the [Internet Movie Database](https://www.imdb.com/). These are split into 25,000 reviews for training and 25,000 reviews for testing. The training and testing sets are *balanced*, meaning they contain an equal number of positive and negative reviews.\n", @@ -128,42 +117,24 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "JL-LtD0CBSZR", - "colab": {} + "id": "2ew7HTbPpCJH" }, + "outputs": [], "source": [ - "# keras.datasets.imdb is broken in 1.13 and 1.14, by np 1.16.3\n", - "!pip install tf_nightly" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", + "import tensorflow.compat.v1 as tf\n", "\n", - "import tensorflow as tf\n", "from tensorflow import keras\n", "\n", "import numpy as np\n", "\n", "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iAsKG535pHep" }, "source": [ @@ -176,23 +147,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} + "id": "zXXx5Oc3pOmN" }, + "outputs": [], "source": [ "imdb = keras.datasets.imdb\n", "\n", "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "odr-KlzO-lkL" }, "source": [ @@ -202,7 +170,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "l50X3GfjpU4r" }, "source": [ @@ -213,21 +180,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} + "id": "y8qCnve_-lkO" }, + "outputs": [], "source": [ "print(\"Training entries: {}, labels: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RnKvHWW4-lkW" }, "source": [ @@ -236,21 +200,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} + "id": "QtTS4kpEpjbi" }, + "outputs": [], "source": [ "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hIE4l_72x7DP" }, "source": [ @@ -259,21 +220,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} + "id": "X-6Ii9Pfx6Nr" }, + "outputs": [], "source": [ "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4wJg2FiYpuoX" }, "source": [ @@ -284,11 +242,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} + "id": "tr5s_1alpzop" }, + "outputs": [], "source": [ "# A dictionary mapping words to an integer index\n", "word_index = imdb.get_word_index()\n", @@ -304,14 +262,11 @@ "\n", "def decode_review(text):\n", " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "U3CNRvEZVppl" }, "source": [ @@ -320,21 +275,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} + "id": "s_OqxmH6-lkn" }, + "outputs": [], "source": [ "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lFP_XKVRp4_S" }, "source": [ @@ -348,16 +300,18 @@ "\n", "In this tutorial, we will use the second approach.\n", "\n", + "If you are unfamiliar with embeddings, see the [embedding guide](../../guide/embedding).\n", + "\n", "Since the movie reviews must be the same length, we will use the [pad_sequences](https://keras.io/preprocessing/sequence/#pad_sequences) function to standardize the lengths:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} + "id": "2jQv-omsHurp" }, + "outputs": [], "source": [ "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", " value=word_index[\"\"],\n", @@ -368,14 +322,11 @@ " value=word_index[\"\"],\n", " padding='post',\n", " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VO5MBpyQdipD" }, "source": [ @@ -384,21 +335,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} + "id": "USSSBnkE-lky" }, + "outputs": [], "source": [ "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QJoxZGyfjT5V" }, "source": [ @@ -407,21 +355,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} + "id": "TG8X9cqi-lk9" }, + "outputs": [], "source": [ "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LLC02j2g-llC" }, "source": [ @@ -437,11 +382,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} + "id": "xpKOoWgu-llD" }, + "outputs": [], "source": [ "# input shape is the vocabulary count used for the movie reviews (10,000 words)\n", "vocab_size = 10000\n", @@ -453,14 +398,11 @@ "model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))\n", "\n", "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6PbKQ6mucuKL" }, "source": [ @@ -475,7 +417,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0XMwnDOp-llH" }, "source": [ @@ -489,7 +430,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "L4EqVWg4-llM" }, "source": [ @@ -506,23 +446,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} + "id": "Mr0GP-cQ-llN" }, + "outputs": [], "source": [ "model.compile(optimizer='adam',\n", " loss='binary_crossentropy',\n", " metrics=['acc'])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hCWYwkug-llQ" }, "source": [ @@ -533,25 +470,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} + "id": "-NpcXY9--llS" }, + "outputs": [], "source": [ "x_val = train_data[:10000]\n", "partial_x_train = train_data[10000:]\n", "\n", "y_val = train_labels[:10000]\n", "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "35jv_fzP-llU" }, "source": [ @@ -562,11 +496,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} + "id": "tXSGrjWZ-llW" }, + "outputs": [], "source": [ "history = model.fit(partial_x_train,\n", " partial_y_train,\n", @@ -574,14 +508,11 @@ " batch_size=512,\n", " validation_data=(x_val, y_val),\n", " verbose=1)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9EEGuDVuzb5r" }, "source": [ @@ -592,23 +523,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} + "id": "zOMKywn4zReN" }, + "outputs": [], "source": [ "results = model.evaluate(test_data, test_labels, verbose=2)\n", "\n", "print(results)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "z1iEXVTR0Z2t" }, "source": [ @@ -618,7 +546,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5KggXVeL-llZ" }, "source": [ @@ -629,22 +556,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} + "id": "VcvSXvhp-llb" }, + "outputs": [], "source": [ "history_dict = history.history\n", "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "nRKsqL40-lle" }, "source": [ @@ -653,11 +577,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} + "id": "nGoYf2Js-lle" }, + "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", @@ -678,17 +602,15 @@ "plt.legend()\n", "\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} + "id": "6hXx-xOv-llh" }, + "outputs": [], "source": [ "plt.clf() # clear figure\n", "\n", @@ -700,18 +622,14 @@ "plt.legend()\n", "\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "oFEmZ5zq-llk" }, "source": [ - "\n", "In this plot, the dots represent the training loss and accuracy, and the solid lines are the validation loss and accuracy.\n", "\n", "Notice the training loss *decreases* with each epoch and the training accuracy *increases* with each epoch. This is expected when using a gradient descent optimization—it should minimize the desired quantity on every iteration.\n", @@ -721,5 +639,18 @@ "For this particular case, we could prevent overfitting by simply stopping the training after twenty or so epochs. Later, you'll see how to do this automatically with a callback." ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "basic_text_classification.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/keras/overfit_and_underfit.ipynb b/site/en/r1/tutorials/keras/overfit_and_underfit.ipynb index 039d2eb0825..8e35b06e556 100644 --- a/site/en/r1/tutorials/keras/overfit_and_underfit.ipynb +++ b/site/en/r1/tutorials/keras/overfit_and_underfit.ipynb @@ -1,28 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "overfit_and_underfit.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "fTFj8ft5dlbS" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "accelerator": "GPU" - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "fTFj8ft5dlbS" }, "source": [ @@ -31,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "lzyBOpYMdp3F", - "colab": {} + "id": "lzyBOpYMdp3F" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -49,18 +29,16 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "m_x4KfSJ7Vt7", - "colab": {} + "id": "m_x4KfSJ7Vt7" }, + "outputs": [], "source": [ "#@title MIT License\n", "#\n", @@ -83,14 +61,11 @@ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "C9HmC2T4ld5B" }, "source": [ @@ -100,7 +75,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kRTxFhXAlnl1" }, "source": [ @@ -117,7 +91,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "kRTxFhXAlnl1" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "19rPukKZsPG6" }, "source": [ @@ -138,29 +125,25 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "5pZ8A2liqvgk", - "colab": {} + "id": "5pZ8A2liqvgk" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", + "import tensorflow.compat.v1 as tf\n", "\n", - "import tensorflow as tf\n", "from tensorflow import keras\n", "\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1cweoTiruj8O" }, "source": [ @@ -173,11 +156,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "QpzE4iqZtJly", - "colab": {} + "id": "QpzE4iqZtJly" }, + "outputs": [], "source": [ "NUM_WORDS = 10000\n", "\n", @@ -193,14 +176,11 @@ "\n", "train_data = multi_hot_sequences(train_data, dimension=NUM_WORDS)\n", "test_data = multi_hot_sequences(test_data, dimension=NUM_WORDS)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MzWVeXe3NBTn" }, "source": [ @@ -209,21 +189,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "71kr5rG4LkGM", - "colab": {} + "id": "71kr5rG4LkGM" }, + "outputs": [], "source": [ "plt.plot(train_data[0])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lglk41MwvU5o" }, "source": [ @@ -245,7 +222,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_ReKHdC2EgVu" }, "source": [ @@ -254,11 +230,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "QKgdXPx9usBa", - "colab": {} + "id": "QKgdXPx9usBa" }, + "outputs": [], "source": [ "baseline_model = keras.Sequential([\n", " # `input_shape` is only required here so that `.summary` works.\n", @@ -272,17 +248,15 @@ " metrics=['accuracy', 'binary_crossentropy'])\n", "\n", "baseline_model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "LqG3MXF5xSjR", - "colab": {} + "id": "LqG3MXF5xSjR" }, + "outputs": [], "source": [ "baseline_history = baseline_model.fit(train_data,\n", " train_labels,\n", @@ -290,14 +264,11 @@ " batch_size=512,\n", " validation_data=(test_data, test_labels),\n", " verbose=2)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "L-DGRBbGxI6G" }, "source": [ @@ -307,7 +278,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SrfoVQheYSO5" }, "source": [ @@ -316,11 +286,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jksi-XtaxDAh", - "colab": {} + "id": "jksi-XtaxDAh" }, + "outputs": [], "source": [ "smaller_model = keras.Sequential([\n", " keras.layers.Dense(4, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", @@ -333,14 +303,11 @@ " metrics=['accuracy', 'binary_crossentropy'])\n", "\n", "smaller_model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jbngCZliYdma" }, "source": [ @@ -349,11 +316,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Ofn1AwDhx-Fe", - "colab": {} + "id": "Ofn1AwDhx-Fe" }, + "outputs": [], "source": [ "smaller_history = smaller_model.fit(train_data,\n", " train_labels,\n", @@ -361,14 +328,11 @@ " batch_size=512,\n", " validation_data=(test_data, test_labels),\n", " verbose=2)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vIPuf23FFaVn" }, "source": [ @@ -379,11 +343,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ghQwwqwqvQM9", - "colab": {} + "id": "ghQwwqwqvQM9" }, + "outputs": [], "source": [ "bigger_model = keras.models.Sequential([\n", " keras.layers.Dense(512, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", @@ -396,14 +360,11 @@ " metrics=['accuracy','binary_crossentropy'])\n", "\n", "bigger_model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "D-d-i5DaYmr7" }, "source": [ @@ -412,25 +373,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "U1A99dhqvepf", - "colab": {} + "id": "U1A99dhqvepf" }, + "outputs": [], "source": [ "bigger_history = bigger_model.fit(train_data, train_labels,\n", " epochs=20,\n", " batch_size=512,\n", " validation_data=(test_data, test_labels),\n", " verbose=2)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Fy3CMUZpzH3d" }, "source": [ @@ -442,7 +400,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HSlo1F4xHuuM" }, "source": [ @@ -451,11 +408,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "0XmKDtOWzOpk", - "colab": {} + "id": "0XmKDtOWzOpk" }, + "outputs": [], "source": [ "def plot_history(histories, key='binary_crossentropy'):\n", " plt.figure(figsize=(16,10))\n", @@ -476,14 +433,11 @@ "plot_history([('baseline', baseline_history),\n", " ('smaller', smaller_history),\n", " ('bigger', bigger_history)])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Bi6hBhdnSfjA" }, "source": [ @@ -493,7 +447,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ASdv7nsgEFhx" }, "source": [ @@ -503,18 +456,15 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4rHoVWcswFLa" }, "source": [ - "### Add weight regularization\n", - "\n" + "### Add weight regularization\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kRxWepNawbBK" }, "source": [ @@ -531,11 +481,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "HFGmcwduwVyQ", - "colab": {} + "id": "HFGmcwduwVyQ" }, + "outputs": [], "source": [ "l2_model = keras.models.Sequential([\n", " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", @@ -554,14 +504,11 @@ " batch_size=512,\n", " validation_data=(test_data, test_labels),\n", " verbose=2)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bUUHoXb7w-_C" }, "source": [ @@ -572,22 +519,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7wkfLyxBZdh_", - "colab": {} + "id": "7wkfLyxBZdh_" }, + "outputs": [], "source": [ "plot_history([('baseline', baseline_history),\n", " ('l2', l2_model_history)])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Kx1YHMsVxWjP" }, "source": [ @@ -597,7 +541,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HmnBNOOVxiG8" }, "source": [ @@ -613,11 +556,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "OFEYvtrHxSWS", - "colab": {} + "id": "OFEYvtrHxSWS" }, + "outputs": [], "source": [ "dpt_model = keras.models.Sequential([\n", " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", @@ -636,28 +579,23 @@ " batch_size=512,\n", " validation_data=(test_data, test_labels),\n", " verbose=2)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "SPZqwVchx5xp", - "colab": {} + "id": "SPZqwVchx5xp" }, + "outputs": [], "source": [ "plot_history([('baseline', baseline_history),\n", " ('dropout', dpt_model_history)])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gjfnkEeQyAFG" }, "source": [ @@ -674,5 +612,21 @@ "And two important approaches not covered in this guide are data-augmentation and batch normalization." ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [ + "fTFj8ft5dlbS" + ], + "name": "overfit_and_underfit.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/keras/save_and_restore_models.ipynb b/site/en/r1/tutorials/keras/save_and_restore_models.ipynb index 4c0b34616e1..04cc94417a9 100644 --- a/site/en/r1/tutorials/keras/save_and_restore_models.ipynb +++ b/site/en/r1/tutorials/keras/save_and_restore_models.ipynb @@ -1,26 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "save_and_restore_models.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "g_nWetWWd_ns" }, "source": [ @@ -29,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} + "id": "2pHVBk_seED1" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -47,18 +29,16 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} + "id": "N_fMsQ-N8I7j" }, + "outputs": [], "source": [ "#@title MIT License\n", "#\n", @@ -81,14 +61,11 @@ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pZJ3uY9O17VN" }, "source": [ @@ -98,7 +75,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "M4Ata7_wMul1" }, "source": [ @@ -115,7 +91,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "M4Ata7_wMul1" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "mBdde4YJeJKF" }, "source": [ @@ -126,7 +115,7 @@ "\n", "Sharing this data helps others understand how the model works and try it themselves with new data.\n", "\n", - "Caution: Be careful with untrusted code—TensorFlow models are code. See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) for details.\n", + "Caution: Be careful with untrusted code—TensorFlow models are code. See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/r1.15/SECURITY.md) for details.\n", "\n", "### Options\n", "\n", @@ -136,7 +125,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xCUREq7WXgvg" }, "source": [ @@ -148,7 +136,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7l0MiTOrXtNv" }, "source": [ @@ -157,21 +144,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} + "id": "RzIOVSdnMYyO" }, + "outputs": [], "source": [ "!pip install h5py pyyaml\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SbGsznErXWt6" }, "source": [ @@ -182,31 +166,28 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} + "id": "7Nm7Tyb-gRt-" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "import os\n", "\n", - "import tensorflow as tf\n", + "import tensorflow.compat.v1 as tf\n", + "\n", "from tensorflow import keras\n", "\n", "tf.__version__" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} + "id": "9rGfFwE9XVwz" }, + "outputs": [], "source": [ "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", "\n", @@ -215,14 +196,11 @@ "\n", "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "anG3iVoXyZGI" }, "source": [ @@ -232,7 +210,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wynsOBfby0Pa" }, "source": [ @@ -241,11 +218,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} + "id": "0HZbJIjxyX1S" }, + "outputs": [], "source": [ "# Returns a short sequential model\n", "def create_model():\n", @@ -265,14 +242,11 @@ "# Create a basic model instance\n", "model = create_model()\n", "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "soDE0W_KH8rG" }, "source": [ @@ -282,7 +256,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mRyd5qQQIXZm" }, "source": [ @@ -297,11 +270,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} + "id": "IFPuhwntH8VH" }, + "outputs": [], "source": [ "checkpoint_path = \"training_1/cp.ckpt\"\n", "checkpoint_dir = os.path.dirname(checkpoint_path)\n", @@ -320,14 +293,11 @@ "# This may generate warnings related to saving the state of the optimizer.\n", "# These warnings (and similar warnings throughout this notebook)\n", "# are in place to discourage outdated usage, and can be ignored." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rlM-sgyJO084" }, "source": [ @@ -336,21 +306,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} + "id": "gXG5FVKFOVQ3" }, + "outputs": [], "source": [ "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wlRN_f56Pqa9" }, "source": [ @@ -361,24 +328,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} + "id": "Fp5gbuiaPqCT" }, + "outputs": [], "source": [ "model = create_model()\n", "\n", "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", "print(\"Untrained model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1DTKpZssRSo3" }, "source": [ @@ -387,23 +351,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} + "id": "2IZxbwiRRSD2" }, + "outputs": [], "source": [ "model.load_weights(checkpoint_path)\n", "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bpAbKkAyVPV8" }, "source": [ @@ -416,11 +377,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} + "id": "mQF_dlgIVOvq" }, + "outputs": [], "source": [ "# include the epoch in the file name. (uses `str.format`)\n", "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", @@ -437,14 +398,11 @@ " epochs = 50, callbacks = [cp_callback],\n", " validation_data = (test_images,test_labels),\n", " verbose=0)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1zFrKTjjavWI" }, "source": [ @@ -453,35 +411,30 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} + "id": "p64q3-V4sXt0" }, + "outputs": [], "source": [ "! ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} + "id": "1AN_fnuyR41H" }, + "outputs": [], "source": [ "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", "latest" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Zk2ciGbKg561" }, "source": [ @@ -492,24 +445,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} + "id": "3M04jyK-H3QK" }, + "outputs": [], "source": [ "model = create_model()\n", "model.load_weights(latest)\n", "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "c2OxsJOTHxia" }, "source": [ @@ -519,7 +469,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JtdYhvWnH2ib" }, "source": [ @@ -533,7 +482,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "S_FA-ZvxuXQV" }, "source": [ @@ -546,11 +494,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} + "id": "R7W5plyZ-u9X" }, + "outputs": [], "source": [ "# Save the weights\n", "model.save_weights('./checkpoints/my_checkpoint')\n", @@ -561,14 +509,11 @@ "\n", "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kOGlxPRBEvV1" }, "source": [ @@ -582,7 +527,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SkGwf-50zLNn" }, "source": [ @@ -593,11 +537,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} + "id": "m2dkmJVCGUia" }, + "outputs": [], "source": [ "model = create_model()\n", "\n", @@ -605,14 +549,11 @@ "\n", "# Save entire model to a HDF5 file\n", "model.save('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GWmttMOqS68S" }, "source": [ @@ -621,23 +562,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} + "id": "5NDMO_7kS6Do" }, + "outputs": [], "source": [ "# Recreate the exact same model, including weights and optimizer.\n", "new_model = keras.models.load_model('my_model.h5')\n", "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JXQpbTicTBwt" }, "source": [ @@ -646,22 +584,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} + "id": "jwEaj9DnTCVA" }, + "outputs": [], "source": [ "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "dGXqd4wWJl8O" }, "source": [ @@ -677,7 +612,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kPyhgcoVzqUB" }, "source": [ @@ -687,7 +621,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LtcN4VIb7JkK" }, "source": [ @@ -697,7 +630,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DSWiSB0Q8c46" }, "source": [ @@ -706,23 +638,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} + "id": "sI1YvCDFzpl3" }, + "outputs": [], "source": [ "model = create_model()\n", "\n", "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iUvT_3qE8hV5" }, "source": [ @@ -731,24 +660,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "colab": {} + "id": "sq8fPglI1RWA" }, + "outputs": [], "source": [ "import time\n", "\n", "saved_model_path = \"./saved_models/\"+str(int(time.time()))\n", - "tf.contrib.saved_model.save_keras_model(model, saved_model_path)" - ], - "execution_count": 0, - "outputs": [] + "model.save(saved_model_path, save_format='tf')\n" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MjpmyPfh8-1n" }, "source": [ @@ -757,45 +683,39 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ZtOvxA7V0iTv", - "colab": {} + "id": "ZtOvxA7V0iTv" }, + "outputs": [], "source": [ "!ls {saved_model_path}" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "B7qfpvpY9HCe" }, "source": [ - "Reload a fresh keras model from the saved model." + "Load the saved model." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} + "id": "0YofwHdN0pxa" }, + "outputs": [], "source": [ - "new_model = tf.contrib.saved_model.load_keras_model(saved_model_path)\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] + "new_model = tf.keras.models.load_model(saved_model_path)\n", + "new_model" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uWwgNaz19TH2" }, "source": [ @@ -804,11 +724,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Pc9e6G6w1AWG", - "colab": {} + "id": "Pc9e6G6w1AWG" }, + "outputs": [], "source": [ "# The model has to be compiled before evaluating.\n", "# This step is not required if the saved model is only being deployed.\n", @@ -820,14 +740,11 @@ "# Evaluate the restored model.\n", "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "eUYTzSz5VxL2" }, "source": [ @@ -842,5 +759,19 @@ "* The [Save and Restore](https://www.tensorflow.org/r1/guide/saved_model) guide has low-level details about TensorFlow saving." ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "save_and_restore_models.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/load_data/images.ipynb b/site/en/r1/tutorials/load_data/images.ipynb index 37c2d965678..923b95130d1 100644 --- a/site/en/r1/tutorials/load_data/images.ipynb +++ b/site/en/r1/tutorials/load_data/images.ipynb @@ -1,25 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "images.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mt9dL5dIir8X" }, "source": [ @@ -28,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "ufPx7EiCiqgR", - "colab": {} + "id": "ufPx7EiCiqgR" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -46,16 +29,11 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License.\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "deletable": true, - "editable": true, "id": "ucMoYase6URl" }, "source": [ @@ -65,7 +43,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_Wwu5SXZmEkB" }, "source": [ @@ -82,7 +59,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "_Wwu5SXZmEkB" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "Oxw4WahM7DU9" }, "source": [ @@ -94,9 +84,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "deletable": true, - "editable": true, "id": "hoQQiZDB6URn" }, "source": [ @@ -105,38 +92,31 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "DHz3JONNEHlj", - "colab": {} + "id": "DHz3JONNEHlj" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", + "import tensorflow.compat.v1 as tf\n", "\n", - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", "tf.__version__" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "KT6CcaqgQewg", - "colab": {} + "id": "KT6CcaqgQewg" }, + "outputs": [], "source": [ - "AUTOTUNE = tf.data.experimental.AUTOTUNE" - ], - "execution_count": 0, - "outputs": [] + "AUTOTUNE = tf.data.AUTOTUNE" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rxndJHNC8YPM" }, "source": [ @@ -146,9 +126,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "deletable": true, - "editable": true, "id": "wO0InzL66URu" }, "source": [ @@ -159,11 +136,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "rN-Pc6Zd6awg", - "colab": {} + "id": "rN-Pc6Zd6awg" }, + "outputs": [], "source": [ "import pathlib\n", "data_root_orig = tf.keras.utils.get_file('flower_photos',\n", @@ -171,14 +148,11 @@ " untar=True)\n", "data_root = pathlib.Path(data_root_orig)\n", "print(data_root)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rFkFK74oO--g" }, "source": [ @@ -187,25 +161,23 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7onR_lWE7Njj", - "colab": {} + "id": "7onR_lWE7Njj" }, + "outputs": [], "source": [ "for item in data_root.iterdir():\n", " print(item)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "4yYX3ZRqGOuq", - "colab": {} + "id": "4yYX3ZRqGOuq" }, + "outputs": [], "source": [ "import random\n", "all_image_paths = list(data_root.glob('*/*'))\n", @@ -214,27 +186,22 @@ "\n", "image_count = len(all_image_paths)\n", "image_count" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "t_BbYnLjbltQ", - "colab": {} + "id": "t_BbYnLjbltQ" }, + "outputs": [], "source": [ "all_image_paths[:10]" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vkM-IpB-6URx" }, "source": [ @@ -244,58 +211,51 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "wNGateQJ6UR1", - "colab": {} + "id": "wNGateQJ6UR1" }, + "outputs": [], "source": [ "import os\n", "attributions = (data_root/\"LICENSE.txt\").open(encoding='utf-8').readlines()[4:]\n", "attributions = [line.split(' CC-BY') for line in attributions]\n", "attributions = dict(attributions)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jgowG2xu88Io", - "colab": {} + "id": "jgowG2xu88Io" }, + "outputs": [], "source": [ "import IPython.display as display\n", "\n", "def caption_image(image_path):\n", " image_rel = pathlib.Path(image_path).relative_to(data_root)\n", " return \"Image (CC BY 2.0) \" + ' - '.join(attributions[str(image_rel)].split(' - ')[:-1])\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "YIjLi-nX0txI", - "colab": {} + "id": "YIjLi-nX0txI" }, + "outputs": [], "source": [ "for n in range(3):\n", " image_path = random.choice(all_image_paths)\n", " display.display(display.Image(image_path))\n", " print(caption_image(image_path))\n", " print()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OaNOr-co3WKk" }, "source": [ @@ -305,7 +265,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-weOQpDw2Jnu" }, "source": [ @@ -314,22 +273,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ssUZ7Qh96UR3", - "colab": {} + "id": "ssUZ7Qh96UR3" }, + "outputs": [], "source": [ "label_names = sorted(item.name for item in data_root.glob('*/') if item.is_dir())\n", "label_names" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9l_JEBql2OzS" }, "source": [ @@ -338,22 +294,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Y8pCV46CzPlp", - "colab": {} + "id": "Y8pCV46CzPlp" }, + "outputs": [], "source": [ "label_to_index = dict((name, index) for index,name in enumerate(label_names))\n", "label_to_index" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VkXsHg162T9F" }, "source": [ @@ -362,24 +315,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "q62i1RBP4Q02", - "colab": {} + "id": "q62i1RBP4Q02" }, + "outputs": [], "source": [ "all_image_labels = [label_to_index[pathlib.Path(path).parent.name]\n", " for path in all_image_paths]\n", "\n", "print(\"First 10 labels indices: \", all_image_labels[:10])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "i5L09icm9iph" }, "source": [ @@ -389,7 +339,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SbqqRUS79ooq" }, "source": [ @@ -398,22 +347,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jQZdySHvksOu", - "colab": {} + "id": "jQZdySHvksOu" }, + "outputs": [], "source": [ "img_path = all_image_paths[0]\n", "img_path" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2t2h2XCcmK1Y" }, "source": [ @@ -422,22 +368,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "LJfkyC_Qkt7A", - "colab": {} + "id": "LJfkyC_Qkt7A" }, + "outputs": [], "source": [ "img_raw = tf.io.read_file(img_path)\n", "print(repr(img_raw)[:100]+\"...\")" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "opN8AVc8mSbz" }, "source": [ @@ -446,24 +389,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Tm0tdrlfk0Bb", - "colab": {} + "id": "Tm0tdrlfk0Bb" }, + "outputs": [], "source": [ "img_tensor = tf.image.decode_image(img_raw)\n", "\n", "print(img_tensor.shape)\n", "print(img_tensor.dtype)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3k-Of2Tfmbeq" }, "source": [ @@ -472,25 +412,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "XFpz-3_vlJgp", - "colab": {} + "id": "XFpz-3_vlJgp" }, + "outputs": [], "source": [ "img_final = tf.image.resize(img_tensor, [192, 192])\n", "img_final = img_final/255.0\n", "print(img_final.shape)\n", "print(img_final.numpy().min())\n", "print(img_final.numpy().max())\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "aCsAa4Psl4AQ" }, "source": [ @@ -499,11 +436,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "HmUiZJNU73vA", - "colab": {} + "id": "HmUiZJNU73vA" }, + "outputs": [], "source": [ "def preprocess_image(image):\n", " image = tf.image.decode_jpeg(image, channels=3)\n", @@ -511,32 +448,28 @@ " image /= 255.0 # normalize to [0,1] range\n", "\n", " return image" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "einETrJnO-em", - "colab": {} + "id": "einETrJnO-em" }, + "outputs": [], "source": [ "def load_and_preprocess_image(path):\n", " image = tf.read_file(path)\n", " return preprocess_image(image)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3brWQcdtz78y", - "colab": {} + "id": "3brWQcdtz78y" }, + "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", @@ -548,14 +481,11 @@ "plt.xlabel(caption_image(img_path).encode('utf-8'))\n", "plt.title(label_names[label].title())\n", "print()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "n2TCr1TQ8pA3" }, "source": [ @@ -565,7 +495,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6H9Z5Mq63nSH" }, "source": [ @@ -575,7 +504,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GN-s04s-6Luq" }, "source": [ @@ -586,21 +514,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "6oRPG3Jz3ie_", - "colab": {} + "id": "6oRPG3Jz3ie_" }, + "outputs": [], "source": [ "path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uML4JeMmIAvO" }, "source": [ @@ -609,24 +534,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "mIsNflFbIK34", - "colab": {} + "id": "mIsNflFbIK34" }, + "outputs": [], "source": [ "print('shape: ', repr(path_ds.output_shapes))\n", "print('type: ', path_ds.output_types)\n", "print()\n", "print(path_ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ZjyGcM8OwBJ2" }, "source": [ @@ -635,24 +557,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "D1iba6f4khu-", - "colab": {} + "id": "D1iba6f4khu-" }, + "outputs": [], "source": [ "image_ds = path_ds.map(load_and_preprocess_image, num_parallel_calls=AUTOTUNE)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "JLUPs2a-lEEJ", - "colab": {} + "id": "JLUPs2a-lEEJ" }, + "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "\n", @@ -665,14 +585,11 @@ " plt.yticks([])\n", " plt.xlabel(caption_image(all_image_paths[n]))\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "P6FNqPbxkbdx" }, "source": [ @@ -682,7 +599,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YgvrWLKG67-x" }, "source": [ @@ -691,35 +607,30 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "AgBsAiV06udj", - "colab": {} + "id": "AgBsAiV06udj" }, + "outputs": [], "source": [ "label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(all_image_labels, tf.int64))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "HEsk5nN0vyeX", - "colab": {} + "id": "HEsk5nN0vyeX" }, + "outputs": [], "source": [ "for label in label_ds.take(10):\n", " print(label_names[label.numpy()])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jHjgrEeTxyYz" }, "source": [ @@ -728,21 +639,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "AOEWNMdQwsbN", - "colab": {} + "id": "AOEWNMdQwsbN" }, + "outputs": [], "source": [ "image_label_ds = tf.data.Dataset.zip((image_ds, label_ds))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "yA2F09SJLMuM" }, "source": [ @@ -751,21 +659,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "DuVYNinrLL-N", - "colab": {} + "id": "DuVYNinrLL-N" }, + "outputs": [], "source": [ "print(image_label_ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2WYMikoPWOQX" }, "source": [ @@ -774,11 +679,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "HOFwZI-2WhzV", - "colab": {} + "id": "HOFwZI-2WhzV" }, + "outputs": [], "source": [ "ds = tf.data.Dataset.from_tensor_slices((all_image_paths, all_image_labels))\n", "\n", @@ -788,14 +693,11 @@ "\n", "image_label_ds = ds.map(load_and_preprocess_from_path_label)\n", "image_label_ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vYGCgJuR_9Qp" }, "source": [ @@ -805,7 +707,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wwZavzgsIytz" }, "source": [ @@ -821,11 +722,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "uZmZJx8ePw_5", - "colab": {} + "id": "uZmZJx8ePw_5" }, + "outputs": [], "source": [ "BATCH_SIZE = 32\n", "\n", @@ -837,14 +738,11 @@ "# `prefetch` lets the dataset fetch batches, in the background while the model is training.\n", "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6JsM-xHiFCuW" }, "source": [ @@ -866,25 +764,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Ocr6PybXNDoO", - "colab": {} + "id": "Ocr6PybXNDoO" }, + "outputs": [], "source": [ "ds = image_label_ds.apply(\n", " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", "ds = ds.batch(BATCH_SIZE)\n", "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GBBZMSuAmQVL" }, "source": [ @@ -901,22 +796,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "KbJrXn9omO_g", - "colab": {} + "id": "KbJrXn9omO_g" }, + "outputs": [], "source": [ "mobile_net = tf.keras.applications.MobileNetV2(input_shape=(192, 192, 3), include_top=False)\n", "mobile_net.trainable=False" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Y7NVWiLF3Vbf" }, "source": [ @@ -937,7 +829,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CboYya6LmdQI" }, "source": [ @@ -946,24 +837,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "SNOkHUGv3FYq", - "colab": {} + "id": "SNOkHUGv3FYq" }, + "outputs": [], "source": [ "def change_range(image,label):\n", " return 2*image-1, label\n", "\n", "keras_ds = ds.map(change_range)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QDzZ3Nye5Rpv" }, "source": [ @@ -974,36 +862,31 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "OzAhGkEK6WuE", - "colab": {} + "id": "OzAhGkEK6WuE" }, + "outputs": [], "source": [ "# The dataset may take a few seconds to start, as it fills its shuffle buffer.\n", "image_batch, label_batch = next(iter(keras_ds))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "LcFdiWpO5WbV", - "colab": {} + "id": "LcFdiWpO5WbV" }, + "outputs": [], "source": [ "feature_map_batch = mobile_net(image_batch)\n", "print(feature_map_batch.shape)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vrbjEvaC5XmU" }, "source": [ @@ -1012,24 +895,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "X0ooIU9fNjPJ", - "colab": {} + "id": "X0ooIU9fNjPJ" }, + "outputs": [], "source": [ "model = tf.keras.Sequential([\n", " mobile_net,\n", " tf.keras.layers.GlobalAveragePooling2D(),\n", " tf.keras.layers.Dense(len(label_names), activation = 'softmax')])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "foQYUJs97V4V" }, "source": [ @@ -1038,11 +918,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "1nwYxvpj7ZEf", - "colab": {} + "id": "1nwYxvpj7ZEf" }, + "outputs": [], "source": [ "logit_batch = model(image_batch).numpy()\n", "\n", @@ -1051,14 +931,11 @@ "print()\n", "\n", "print(\"Shape:\", logit_batch.shape)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pFc4I_J2nNOJ" }, "source": [ @@ -1067,23 +944,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ZWGqLEWYRNvv", - "colab": {} + "id": "ZWGqLEWYRNvv" }, + "outputs": [], "source": [ "model.compile(optimizer=tf.train.AdamOptimizer(),\n", " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", " metrics=[\"accuracy\"])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tF1mO6haBOSd" }, "source": [ @@ -1092,34 +966,29 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "pPQ5yqyKBJMm", - "colab": {} + "id": "pPQ5yqyKBJMm" }, + "outputs": [], "source": [ "len(model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "kug5Wg66UJjl", - "colab": {} + "id": "kug5Wg66UJjl" }, + "outputs": [], "source": [ "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "f_glpYZ-nYC_" }, "source": [ @@ -1130,35 +999,30 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "AnXPRNWoTypI", - "colab": {} + "id": "AnXPRNWoTypI" }, + "outputs": [], "source": [ "steps_per_epoch=tf.ceil(len(all_image_paths)/BATCH_SIZE).numpy()\n", "steps_per_epoch" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "q_8sabaaSGAp", - "colab": {} + "id": "q_8sabaaSGAp" }, + "outputs": [], "source": [ "model.fit(ds, epochs=1, steps_per_epoch=3)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UMVnoBcG_NlQ" }, "source": [ @@ -1172,7 +1036,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "oNmQqgGhLWie" }, "source": [ @@ -1181,11 +1044,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "_gFVe1rp_MYr", - "colab": {} + "id": "_gFVe1rp_MYr" }, + "outputs": [], "source": [ "import time\n", "\n", @@ -1207,14 +1070,11 @@ " print(\"{} batches: {} s\".format(batches, duration))\n", " print(\"{:0.5f} Images/s\".format(BATCH_SIZE*batches/duration))\n", " print(\"Total time: {}s\".format(end-overall_start))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TYiOr4vdLcNX" }, "source": [ @@ -1223,37 +1083,32 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ZDxLwMJOReVe", - "colab": {} + "id": "ZDxLwMJOReVe" }, + "outputs": [], "source": [ "ds = image_label_ds.apply(\n", " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "IjouTJadRxyp", - "colab": {} + "id": "IjouTJadRxyp" }, + "outputs": [], "source": [ "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HsLlXMO7EWBR" }, "source": [ @@ -1263,7 +1118,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lV1NOn2zE2lR" }, "source": [ @@ -1274,38 +1128,33 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "qj_U09xpDvOg", - "colab": {} + "id": "qj_U09xpDvOg" }, + "outputs": [], "source": [ "ds = image_label_ds.cache()\n", "ds = ds.apply(\n", " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "rdxpvQ7VEo3y", - "colab": {} + "id": "rdxpvQ7VEo3y" }, + "outputs": [], "source": [ "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "usIv7MqqZQps" }, "source": [ @@ -1314,21 +1163,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "eKX6ergKb_xd", - "colab": {} + "id": "eKX6ergKb_xd" }, + "outputs": [], "source": [ "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jUzpG4lYNkN-" }, "source": [ @@ -1337,38 +1183,33 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "vIvF8K4GMq0g", - "colab": {} + "id": "vIvF8K4GMq0g" }, + "outputs": [], "source": [ "ds = image_label_ds.cache(filename='./cache.tf-data')\n", "ds = ds.apply(\n", " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", "ds = ds.batch(BATCH_SIZE).prefetch(1)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "eTIj6IOmM4yA", - "colab": {} + "id": "eTIj6IOmM4yA" }, + "outputs": [], "source": [ "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qqo3dyB0Z4t2" }, "source": [ @@ -1377,21 +1218,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "hZhVdR8MbaUj", - "colab": {} + "id": "hZhVdR8MbaUj" }, + "outputs": [], "source": [ "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WqOVlf8tFrDU" }, "source": [ @@ -1401,7 +1239,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "y1llOTwWFzmR" }, "source": [ @@ -1414,23 +1251,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "EqtARqKuHQLu", - "colab": {} + "id": "EqtARqKuHQLu" }, + "outputs": [], "source": [ "image_ds = tf.data.Dataset.from_tensor_slices(all_image_paths).map(tf.read_file)\n", "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", "tfrec.write(image_ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "flR2GXWFKcO1" }, "source": [ @@ -1439,21 +1273,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "j9PVUL2SFufn", - "colab": {} + "id": "j9PVUL2SFufn" }, + "outputs": [], "source": [ "image_ds = tf.data.TFRecordDataset('images.tfrec').map(preprocess_image)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "cRp1eZDRKzyN" }, "source": [ @@ -1462,38 +1293,33 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7XI_nDU2KuhS", - "colab": {} + "id": "7XI_nDU2KuhS" }, + "outputs": [], "source": [ "ds = tf.data.Dataset.zip((image_ds, label_ds))\n", "ds = ds.apply(\n", " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3ReSapoPK22E", - "colab": {} + "id": "3ReSapoPK22E" }, + "outputs": [], "source": [ "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wb7VyoKNOMms" }, "source": [ @@ -1503,7 +1329,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NF9W-CTKkM-f" }, "source": [ @@ -1513,7 +1338,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "J9HzljSPkxt0" }, "source": [ @@ -1522,23 +1346,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "OzS0Azukkjyw", - "colab": {} + "id": "OzS0Azukkjyw" }, + "outputs": [], "source": [ "paths_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)\n", "image_ds = paths_ds.map(load_and_preprocess_image)\n", "image_ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "onWOwLpYlzJQ" }, "source": [ @@ -1549,36 +1370,31 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "xxZSwnRllyf0", - "colab": {} + "id": "xxZSwnRllyf0" }, + "outputs": [], "source": [ "ds = image_ds.map(tf.serialize_tensor)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "w9N6hJWAkKPC", - "colab": {} + "id": "w9N6hJWAkKPC" }, + "outputs": [], "source": [ "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", "tfrec.write(ds)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OlFc9dJSmcx0" }, "source": [ @@ -1587,11 +1403,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "BsqFyTBFmSCZ", - "colab": {} + "id": "BsqFyTBFmSCZ" }, + "outputs": [], "source": [ "ds = tf.data.TFRecordDataset('images.tfrec')\n", "\n", @@ -1602,14 +1418,11 @@ "\n", "ds = ds.map(parse, num_parallel_calls=AUTOTUNE)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OPs_sLV9pQg5" }, "source": [ @@ -1618,33 +1431,42 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "XYxBwaLYnGop", - "colab": {} + "id": "XYxBwaLYnGop" }, + "outputs": [], "source": [ "ds = tf.data.Dataset.zip((ds, label_ds))\n", "ds = ds.apply(\n", " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", "ds" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "W8X6RmGan1-P", - "colab": {} + "id": "W8X6RmGan1-P" }, + "outputs": [], "source": [ "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] + ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "images.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/load_data/tf_records.ipynb b/site/en/r1/tutorials/load_data/tf_records.ipynb index 0db6b14f4a3..45635034c69 100644 --- a/site/en/r1/tutorials/load_data/tf_records.ipynb +++ b/site/en/r1/tutorials/load_data/tf_records.ipynb @@ -1,27 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "tf_records.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "pL--_KGdYoBz" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pL--_KGdYoBz" }, "source": [ @@ -30,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "cellView": "both", - "colab_type": "code", - "id": "uBDvXpYzYnGj", - "colab": {} + "cellView": "form", + "id": "uBDvXpYzYnGj" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -48,14 +29,11 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HQzaEQuJiW_d" }, "source": [ @@ -74,7 +52,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "HQzaEQuJiW_d" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "3pkUd_9IZCFO" }, "source": [ @@ -92,11 +83,9 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ac83J0QxjhFt" }, "source": [ - "\n", "This notebook will demonstrate how to create, parse, and use the `tf.Example` message, and then serialize, write, and read `tf.Example` messages to and from `.tfrecord` files.\n", "\n", "Note: While useful, these structures are optional. There is no need to convert existing code to use TFRecords, unless you are using [`tf.data`](https://www.tensorflow.org/r1/guide/datasets) and reading data is still the bottleneck to training. See [Data Input Pipeline Performance](https://www.tensorflow.org/r1/guide/performance/datasets) for dataset performance tips." @@ -105,7 +94,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WkRreBf1eDVc" }, "source": [ @@ -114,27 +102,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Ja7sezsmnXph", - "colab": {} + "id": "Ja7sezsmnXph" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", + "import tensorflow.compat.v1 as tf\n", "\n", - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", "\n", "import numpy as np\n", "import IPython.display as display" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "e5Kq88ccUWQV" }, "source": [ @@ -144,7 +127,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VrdQHgvNijTi" }, "source": [ @@ -154,13 +136,12 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lZw57Qrn4CTE" }, "source": [ "Fundamentally a `tf.Example` is a `{\"string\": tf.train.Feature}` mapping.\n", "\n", - "The `tf.train.Feature` message type can accept one of the following three types (See the [`.proto` file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto) for reference). Most other generic types can be coerced into one of these.\n", + "The `tf.train.Feature` message type can accept one of the following three types (See the [`.proto` file](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/feature.proto) for reference). Most other generic types can be coerced into one of these.\n", "\n", "1. `tf.train.BytesList` (the following types can be coerced)\n", "\n", @@ -185,7 +166,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_e3g9ExathXP" }, "source": [ @@ -196,11 +176,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "mbsPOUpVtYxA", - "colab": {} + "id": "mbsPOUpVtYxA" }, + "outputs": [], "source": [ "# The following functions can be used to convert a value to a type compatible\n", "# with tf.Example.\n", @@ -216,14 +196,11 @@ "def _int64_feature(value):\n", " \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n", " return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Wst0v9O8hgzy" }, "source": [ @@ -233,7 +210,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vsMbkkC8xxtB" }, "source": [ @@ -242,11 +218,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "hZzyLGr0u73y", - "colab": {} + "id": "hZzyLGr0u73y" }, + "outputs": [], "source": [ "print(_bytes_feature(b'test_string'))\n", "print(_bytes_feature(u'test_bytes'.encode('utf-8')))\n", @@ -255,14 +231,11 @@ "\n", "print(_int64_feature(True))\n", "print(_int64_feature(1))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "nj1qpfQU5qmi" }, "source": [ @@ -271,23 +244,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "5afZkORT5pjm", - "colab": {} + "id": "5afZkORT5pjm" }, + "outputs": [], "source": [ "feature = _float_feature(np.exp(1))\n", "\n", "feature.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "laKnw9F3hL-W" }, "source": [ @@ -297,7 +267,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "b_MEnhxchQPC" }, "source": [ @@ -307,13 +276,12 @@ "\n", "1. We create a map (dictionary) from the feature name string to the encoded feature value produced in #1.\n", "\n", - "1. The map produced in #2 is converted to a [`Features` message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto#L85)." + "1. The map produced in #2 is converted to a [`Features` message](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/feature.proto#L85)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4EgFQ2uHtchc" }, "source": [ @@ -330,11 +298,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "CnrguFAy3YQv", - "colab": {} + "id": "CnrguFAy3YQv" }, + "outputs": [], "source": [ "# the number of observations in the dataset\n", "n_observations = int(1e4)\n", @@ -351,14 +319,11 @@ "\n", "# float feature, from a standard normal distribution\n", "feature3 = np.random.randn(n_observations)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "aGrscehJr7Jd" }, "source": [ @@ -367,11 +332,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "RTCS49Ij_kUw", - "colab": {} + "id": "RTCS49Ij_kUw" }, + "outputs": [], "source": [ "def serialize_example(feature0, feature1, feature2, feature3):\n", " \"\"\"\n", @@ -392,27 +357,24 @@ "\n", " example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n", " return example_proto.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XftzX9CN_uGT" }, "source": [ - "For example, suppose we have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. We can create and print the `tf.Example` message for this observation using `create_message()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.Example` [message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message." + "For example, suppose we have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. We can create and print the `tf.Example` message for this observation using `create_message()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.Example` [message](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "N8BtSx2RjYcb", - "colab": {} + "id": "N8BtSx2RjYcb" }, + "outputs": [], "source": [ "# This is an example observation from the dataset.\n", "\n", @@ -420,14 +382,11 @@ "\n", "serialized_example = serialize_example(False, 4, b'goat', 0.9876)\n", "serialized_example" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_pbGATlG6u-4" }, "source": [ @@ -436,22 +395,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "dGim-mEm6vit", - "colab": {} + "id": "dGim-mEm6vit" }, + "outputs": [], "source": [ "example_proto = tf.train.Example.FromString(serialized_example)\n", "example_proto" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "o5UCsGIQ9_tV" }, "source": [ @@ -481,7 +437,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "y-Hjmee-fbLH" }, "source": [ @@ -491,7 +446,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GmehkCCT81Ez" }, "source": [ @@ -501,7 +455,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1FISEuz8ubu3" }, "source": [ @@ -514,21 +467,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "mXeaukvwu5_-", - "colab": {} + "id": "mXeaukvwu5_-" }, + "outputs": [], "source": [ "tf.data.Dataset.from_tensor_slices(feature1)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "f-q0VKyZvcad" }, "source": [ @@ -537,25 +487,23 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "H5sWyu1kxnvg", - "colab": {} + "id": "H5sWyu1kxnvg" }, + "outputs": [], "source": [ "features_dataset = tf.data.Dataset.from_tensor_slices((feature0, feature1, feature2, feature3))\n", "features_dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "m1C-t71Nywze", - "colab": {} + "id": "m1C-t71Nywze" }, + "outputs": [], "source": [ "# Use `take(1)` to only pull one example from the dataset.\n", "for f0,f1,f2,f3 in features_dataset.take(1):\n", @@ -563,14 +511,11 @@ " print(f1)\n", " print(f2)\n", " print(f3)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mhIe63awyZYd" }, "source": [ @@ -584,11 +529,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "1OzYTkOBAehm", - "colab": {} + "id": "1OzYTkOBAehm" }, + "outputs": [], "source": [ "def serialize_example_pyfunction(feature0, feature1, feature2, feature3):\n", " \"\"\"\n", @@ -609,17 +554,15 @@ "\n", " example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n", " return example_proto.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "apB5KYrJzjPI", - "colab": {} + "id": "apB5KYrJzjPI" }, + "outputs": [], "source": [ "def tf_serialize_example(f0,f1,f2,f3):\n", " tf_string = tf.py_function(\n", @@ -627,14 +570,11 @@ " (f0,f1,f2,f3), # pass these args to the above function.\n", " tf.string) # the return type is `tf.string`.\n", " return tf.reshape(tf_string, ()) # The result is a scalar" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CrFZ9avE3HUF" }, "source": [ @@ -643,22 +583,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "VDeqYVbW3ww9", - "colab": {} + "id": "VDeqYVbW3ww9" }, + "outputs": [], "source": [ "serialized_features_dataset = features_dataset.map(tf_serialize_example)\n", "serialized_features_dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "p6lw5VYpjZZC" }, "source": [ @@ -667,23 +604,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "vP1VgTO44UIE", - "colab": {} + "id": "vP1VgTO44UIE" }, + "outputs": [], "source": [ "filename = 'test.tfrecord'\n", "writer = tf.data.experimental.TFRecordWriter(filename)\n", "writer.write(serialized_features_dataset)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6aV0GQhV8tmp" }, "source": [ @@ -693,36 +627,32 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "o3J5D4gcSy8N" }, "source": [ "We can also read the TFRecord file using the `tf.data.TFRecordDataset` class.\n", "\n", - "More information on consuming TFRecord files using `tf.data` can be found [here](https://www.tensorflow.org/r1/guide/datasets#consuming_tfrecord_data).\n", + "More information on consuming TFRecord files using `tf.data` can be found [here](https://www.tensorflow.org/guide/data#consuming_tfrecord_data).\n", "\n", "Using `TFRecordDataset`s can be useful for standardizing input data and optimizing performance." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "6OjX6UZl-bHC", - "colab": {} + "id": "6OjX6UZl-bHC" }, + "outputs": [], "source": [ "filenames = [filename]\n", "raw_dataset = tf.data.TFRecordDataset(filenames)\n", "raw_dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6_EQ9i2E_-Fz" }, "source": [ @@ -735,22 +665,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "hxVXpLz_AJlm", - "colab": {} + "id": "hxVXpLz_AJlm" }, + "outputs": [], "source": [ "for raw_record in raw_dataset.take(10):\n", " print(repr(raw_record))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "W-6oNzM4luFQ" }, "source": [ @@ -761,11 +688,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zQjbIR1nleiy", - "colab": {} + "id": "zQjbIR1nleiy" }, + "outputs": [], "source": [ "# Create a description of the features.\n", "feature_description = {\n", @@ -778,14 +705,11 @@ "def _parse_function(example_proto):\n", " # Parse the input tf.Example proto using the dictionary above.\n", " return tf.parse_single_example(example_proto, feature_description)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gWETjUqhEQZf" }, "source": [ @@ -795,7 +719,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AH73hav6Bnmg" }, "source": [ @@ -804,22 +727,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "6Ob7D-zmBm1w", - "colab": {} + "id": "6Ob7D-zmBm1w" }, + "outputs": [], "source": [ "parsed_dataset = raw_dataset.map(_parse_function)\n", "parsed_dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "sNV-XclGnOvn" }, "source": [ @@ -828,22 +748,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "x2LT2JCqhoD_", - "colab": {} + "id": "x2LT2JCqhoD_" }, + "outputs": [], "source": [ "for parsed_record in parsed_dataset.take(10):\n", " print(repr(parsed_record))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Cig9EodTlDmg" }, "source": [ @@ -853,7 +770,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jyg1g3gU7DNn" }, "source": [ @@ -863,7 +779,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3FXG3miA7Kf1" }, "source": [ @@ -873,7 +788,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CKn5uql2lAaN" }, "source": [ @@ -883,7 +797,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LNW_FA-GQWXs" }, "source": [ @@ -892,38 +805,33 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "MKPHzoGv7q44", - "colab": {} + "id": "MKPHzoGv7q44" }, + "outputs": [], "source": [ "# Write the `tf.Example` observations to the file.\n", "with tf.python_io.TFRecordWriter(filename) as writer:\n", " for i in range(n_observations):\n", " example = serialize_example(feature0[i], feature1[i], feature2[i], feature3[i])\n", " writer.write(example)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "EjdFHHJMpUUo", - "colab": {} + "id": "EjdFHHJMpUUo" }, + "outputs": [], "source": [ "!ls" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wtQ7k0YWQ1cz" }, "source": [ @@ -933,7 +841,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "utkozytkQ-2K" }, "source": [ @@ -946,11 +853,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "36ltP9B8OezA", - "colab": {} + "id": "36ltP9B8OezA" }, + "outputs": [], "source": [ "record_iterator = tf.python_io.tf_record_iterator(path=filename)\n", "\n", @@ -962,14 +869,11 @@ "\n", " # Exit after 1 iteration as this is purely demonstrative.\n", " break" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "i3uquiiGTZTK" }, "source": [ @@ -978,21 +882,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "-UNzS7vsUBs0", - "colab": {} + "id": "-UNzS7vsUBs0" }, + "outputs": [], "source": [ "print(dict(example.features.feature))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "u1M-WrbqUUVW" }, "source": [ @@ -1001,21 +902,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "2yCBu70IUb2H", - "colab": {} + "id": "2yCBu70IUb2H" }, + "outputs": [], "source": [ "print(example.features.feature['feature3'])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4dw6_OI9UiNZ" }, "source": [ @@ -1024,21 +922,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "BdDYjDnDUlFe", - "colab": {} + "id": "BdDYjDnDUlFe" }, + "outputs": [], "source": [ "print(example.features.feature['feature3'].float_list.value)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "S0tFDrwdoj3q" }, "source": [ @@ -1048,7 +943,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rjN2LFxFpcR9" }, "source": [ @@ -1062,7 +956,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5Lk2qrKvN0yu" }, "source": [ @@ -1071,50 +964,43 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3a0fmwg8lHdF", - "colab": {} + "id": "3a0fmwg8lHdF" }, + "outputs": [], "source": [ "cat_in_snow = tf.keras.utils.get_file('320px-Felis_catus-cat_on_snow.jpg', 'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg')\n", "williamsburg_bridge = tf.keras.utils.get_file('194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7aJJh7vENeE4", - "colab": {} + "id": "7aJJh7vENeE4" }, + "outputs": [], "source": [ "display.display(display.Image(filename=cat_in_snow))\n", "display.display(display.HTML('Image cc-by: Von.grzanka'))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "KkW0uuhcXZqA", - "colab": {} + "id": "KkW0uuhcXZqA" }, + "outputs": [], "source": [ "display.display(display.Image(filename=williamsburg_bridge))\n", "display.display(display.HTML('source'))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VSOgJSwoN5TQ" }, "source": [ @@ -1124,7 +1010,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Azx83ryQEU6T" }, "source": [ @@ -1133,27 +1018,25 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "kC4TS1ZEONHr", - "colab": {} + "id": "kC4TS1ZEONHr" }, + "outputs": [], "source": [ "image_labels = {\n", " cat_in_snow : 0,\n", " williamsburg_bridge : 1,\n", "}" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "c5njMSYNEhNZ", - "colab": {} + "id": "c5njMSYNEhNZ" }, + "outputs": [], "source": [ "# This is an example, just using the cat image.\n", "image_string = open(cat_in_snow, 'rb').read()\n", @@ -1177,14 +1060,11 @@ "for line in str(image_example(image_string, label)).split('\\n')[:15]:\n", " print(line)\n", "print('...')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2G_o3O9MN0Qx" }, "source": [ @@ -1193,11 +1073,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "qcw06lQCOCZU", - "colab": {} + "id": "qcw06lQCOCZU" }, + "outputs": [], "source": [ "# Write the raw image files to images.tfrecords.\n", "# First, process the two images into tf.Example messages.\n", @@ -1208,27 +1088,22 @@ " image_string = open(filename, 'rb').read()\n", " tf_example = image_example(image_string, label)\n", " writer.write(tf_example.SerializeToString())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "yJrTe6tHPCfs", - "colab": {} + "id": "yJrTe6tHPCfs" }, + "outputs": [], "source": [ "!ls" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jJSsCkZLPH6K" }, "source": [ @@ -1239,11 +1114,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "M6Cnfd3cTKHN", - "colab": {} + "id": "M6Cnfd3cTKHN" }, + "outputs": [], "source": [ "raw_image_dataset = tf.data.TFRecordDataset('images.tfrecords')\n", "\n", @@ -1262,14 +1137,11 @@ "\n", "parsed_image_dataset = raw_image_dataset.map(_parse_image_function)\n", "parsed_image_dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0PEEFPk4NEg1" }, "source": [ @@ -1278,18 +1150,31 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "yZf8jOyEIjSF", - "colab": {} + "id": "yZf8jOyEIjSF" }, + "outputs": [], "source": [ "for image_features in parsed_image_dataset:\n", " image_raw = image_features['image_raw'].numpy()\n", " display.display(display.Image(data=image_raw))" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "pL--_KGdYoBz" ], - "execution_count": 0, - "outputs": [] + "name": "tf_records.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/non-ml/mandelbrot.ipynb b/site/en/r1/tutorials/non-ml/mandelbrot.ipynb index 0064830469f..bca8a142be4 100644 --- a/site/en/r1/tutorials/non-ml/mandelbrot.ipynb +++ b/site/en/r1/tutorials/non-ml/mandelbrot.ipynb @@ -1,25 +1,9 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "mandelbrot.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "id": "HhR5048dZ3e1", - "colab_type": "text" + "id": "HhR5048dZ3e1" }, "source": [ "##### Copyright 2019 The TensorFlow Authors." @@ -27,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "f0A2utIXbPc5", - "colab": {} + "id": "f0A2utIXbPc5" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -45,15 +29,12 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "1qF0JETfbfIR", - "colab_type": "text" + "id": "1qF0JETfbfIR" }, "source": [ "# Mandelbrot set" @@ -62,8 +43,7 @@ { "cell_type": "markdown", "metadata": { - "id": "p8Z8Pb5nbtZ3", - "colab_type": "text" + "id": "p8Z8Pb5nbtZ3" }, "source": [ "
    \n", @@ -79,8 +59,21 @@ { "cell_type": "markdown", "metadata": { - "id": "lqPLlJWqcSFZ", - "colab_type": "text" + "id": "p8Z8Pb5nbtZ3" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lqPLlJWqcSFZ" }, "source": [ "Visualizing the [Mandelbrot set](https://en.wikipedia.org/wiki/Mandelbrot_set) doesn't have anything to do with machine learning, but it makes for a fun example of how one can use TensorFlow for general mathematics. This is actually a pretty naive implementation of the visualization, but it makes the point. (We may end up providing a more elaborate implementation down the line to produce more truly beautiful images.)" @@ -89,8 +82,7 @@ { "cell_type": "markdown", "metadata": { - "id": "80RrFh7EcnLT", - "colab_type": "text" + "id": "80RrFh7EcnLT" }, "source": [ "## Basic setup\n", @@ -100,44 +92,39 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "xc-QSV_SdEG4", - "colab_type": "code", - "colab": {} + "id": "xc-QSV_SdEG4" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "# Import libraries for simulation\n", - "import tensorflow as tf\n", + "import tensorflow.compat.v1 as tf\n", + "\n", "import numpy as np\n", "\n", "# Imports for visualization\n", "import PIL.Image\n", "from io import BytesIO\n", "from IPython.display import clear_output, Image, display\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "mP5YEOuTieH0", - "colab_type": "text" + "id": "mP5YEOuTieH0" }, "source": [ - "\n", "Now you'll define a function to actually display the image once you have iteration counts." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "_q_HC5cGhX4h", - "colab_type": "code", - "colab": {} + "id": "_q_HC5cGhX4h" }, + "outputs": [], "source": [ "def DisplayFractal(a, fmt='jpeg'):\n", " \"\"\"Display an array of iteration counts as a\n", @@ -152,15 +139,12 @@ " f = BytesIO()\n", " PIL.Image.fromarray(a).save(f, fmt)\n", " display(Image(data=f.getvalue()))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "xEptO88QikEM", - "colab_type": "text" + "id": "xEptO88QikEM" }, "source": [ "# Session and variable initialization\n", @@ -170,99 +154,84 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "8_yDY6Uih7bD", - "colab_type": "code", - "colab": {} + "id": "8_yDY6Uih7bD" }, + "outputs": [], "source": [ "sess = tf.InteractiveSession()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "_NFwmNL5iqBd", - "colab_type": "text" + "id": "_NFwmNL5iqBd" }, "source": [ - "\n", "It's handy that you can freely mix NumPy and TensorFlow." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "fHu_sT7chbg_", - "colab_type": "code", - "colab": {} + "id": "fHu_sT7chbg_" }, + "outputs": [], "source": [ "# Use NumPy to create a 2D array of complex numbers\n", "\n", "Y, X = np.mgrid[-1.3:1.3:0.005, -2:1:0.005]\n", "Z = X+1j*Y" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "u7SsqtHqivVW", - "colab_type": "text" + "id": "u7SsqtHqivVW" }, "source": [ - "\n", "Now you define and initialize TensorFlow tensors." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "UpGYdAWQhhCN", - "colab_type": "code", - "colab": {} + "id": "UpGYdAWQhhCN" }, + "outputs": [], "source": [ "xs = tf.constant(Z.astype(np.complex64))\n", "zs = tf.Variable(xs)\n", "ns = tf.Variable(tf.zeros_like(xs, tf.float32))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "gqvhBLXbi4al", - "colab_type": "text" + "id": "gqvhBLXbi4al" }, "source": [ - "\n", "TensorFlow requires that you explicitly initialize variables before using them." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "RmjN39LHhob2", - "colab_type": "code", - "colab": {} + "id": "RmjN39LHhob2" }, + "outputs": [], "source": [ "tf.global_variables_initializer().run()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "ao_esnw4jAJp", - "colab_type": "text" + "id": "ao_esnw4jAJp" }, "source": [ "# Defining and running the computation\n", @@ -272,11 +241,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "ZMup0FHjiGEx", - "colab_type": "code", - "colab": {} + "id": "ZMup0FHjiGEx" }, + "outputs": [], "source": [ "# Compute the new values of z: z^2 + x\n", "zs_ = zs*zs + xs\n", @@ -294,67 +263,68 @@ " zs.assign(zs_),\n", " ns.assign_add(tf.cast(not_diverged, tf.float32))\n", " )" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "9qqqbNu7jCrj", - "colab_type": "text" + "id": "9qqqbNu7jCrj" }, "source": [ - "\n", "... and run it for a couple hundred steps" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "twC_FiUSiN8s", - "colab_type": "code", - "colab": {} + "id": "twC_FiUSiN8s" }, + "outputs": [], "source": [ "for i in range(200): step.run()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "vfoDAWtijLKd", - "colab_type": "text" + "id": "vfoDAWtijLKd" }, "source": [ - "\n", "Let's see what you've got." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "8qqfdbuOiV90", - "colab_type": "code", - "colab": {} + "id": "8qqfdbuOiV90" }, + "outputs": [], "source": [ "DisplayFractal(ns.eval())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "vB-3S5cFjVYQ", - "colab_type": "text" + "id": "vB-3S5cFjVYQ" }, "source": [ "Not bad!" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "name": "mandelbrot.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/non-ml/pdes.ipynb b/site/en/r1/tutorials/non-ml/pdes.ipynb index 8be44c48d71..832fa450523 100644 --- a/site/en/r1/tutorials/non-ml/pdes.ipynb +++ b/site/en/r1/tutorials/non-ml/pdes.ipynb @@ -1,25 +1,9 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "pdes.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "id": "_omgylxzm5i9", - "colab_type": "text" + "id": "_omgylxzm5i9" }, "source": [ "##### Copyright 2019 The TensorFlow Authors." @@ -27,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "f0A2utIXbPc5", - "colab": {} + "id": "f0A2utIXbPc5" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -45,15 +29,12 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "eriCSHTznS4U", - "colab_type": "text" + "id": "eriCSHTznS4U" }, "source": [ "# Partial Differential Equations" @@ -62,8 +43,7 @@ { "cell_type": "markdown", "metadata": { - "id": "uYCNQT4snWr6", - "colab_type": "text" + "id": "uYCNQT4snWr6" }, "source": [ "
    \n", @@ -79,8 +59,21 @@ { "cell_type": "markdown", "metadata": { - "id": "zxQvbi5gnyMm", - "colab_type": "text" + "id": "uYCNQT4snWr6" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zxQvbi5gnyMm" }, "source": [ "TensorFlow isn't just for machine learning. Here you will use TensorFlow to simulate the behavior of a [partial differential equation](https://en.wikipedia.org/wiki/Partial_differential_equation). You'll simulate the surface of a square pond as a few raindrops land on it.\n", @@ -92,31 +85,27 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "FG6DLet6ol3j", - "colab_type": "code", - "colab": {} + "id": "FG6DLet6ol3j" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "#Import libraries for simulation\n", - "import tensorflow as tf\n", + "import tensorflow.compat.v1 as tf\n", + "\n", "import numpy as np\n", "\n", "#Imports for visualization\n", "import PIL.Image\n", "from io import BytesIO\n", "from IPython.display import clear_output, Image, display\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "7vd7rHS0oqEF", - "colab_type": "text" + "id": "7vd7rHS0oqEF" }, "source": [ "A function for displaying the state of the pond's surface as an image." @@ -124,11 +113,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "fJ8SpYYUoq6G", - "colab_type": "code", - "colab": {} + "id": "fJ8SpYYUoq6G" }, + "outputs": [], "source": [ "def DisplayArray(a, fmt='jpeg', rng=[0,1]):\n", " \"\"\"Display an array as a picture.\"\"\"\n", @@ -138,39 +127,32 @@ " PIL.Image.fromarray(a).save(f, fmt)\n", " clear_output(wait = True)\n", " display(Image(data=f.getvalue()))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "NjiZ2_6Mou13", - "colab_type": "text" + "id": "NjiZ2_6Mou13" }, "source": [ - "\n", "Here you start an interactive TensorFlow session for convenience in playing around. A regular session would work as well if you were doing this in an executable .py file." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "cH82JlsPozdV", - "colab_type": "code", - "colab": {} + "id": "cH82JlsPozdV" }, + "outputs": [], "source": [ "sess = tf.InteractiveSession()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "Hbk97yero5a9", - "colab_type": "text" + "id": "Hbk97yero5a9" }, "source": [ "## Computational convenience functions" @@ -178,11 +160,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "XVomNV1OpBbX", - "colab_type": "code", - "colab": {} + "id": "XVomNV1OpBbX" }, + "outputs": [], "source": [ "def make_kernel(a):\n", " \"\"\"Transform a 2D array into a convolution kernel\"\"\"\n", @@ -202,15 +184,12 @@ " [1.0, -6., 1.0],\n", " [0.5, 1.0, 0.5]])\n", " return simple_conv(x, laplace_k)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "f9gBib2lpINO", - "colab_type": "text" + "id": "f9gBib2lpINO" }, "source": [ "## Define the PDE\n", @@ -220,35 +199,31 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "7faiwBQhpK1Z", - "colab_type": "code", - "colab": {} + "id": "7faiwBQhpK1Z" }, + "outputs": [], "source": [ "N = 500" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "U_DscmhfpPs0", - "colab_type": "text" + "id": "U_DscmhfpPs0" }, "source": [ - "\n", "Here you create a pond and hit it with some rain drops." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "Mtk8t0IOpSrb", - "colab_type": "code", - "colab": {} + "id": "Mtk8t0IOpSrb" }, + "outputs": [], "source": [ "# Initial Conditions -- some rain drops hit a pond\n", "\n", @@ -262,15 +237,12 @@ " u_init[a,b] = np.random.uniform()\n", "\n", "DisplayArray(u_init, rng=[-0.1, 0.1])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "5vzdx9rHpXsl", - "colab_type": "text" + "id": "5vzdx9rHpXsl" }, "source": [ "Now you specify the details of the differential equation." @@ -278,11 +250,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "c6uj8LFDpaZO", - "colab_type": "code", - "colab": {} + "id": "c6uj8LFDpaZO" }, + "outputs": [], "source": [ "# Parameters:\n", "# eps -- time resolution\n", @@ -302,15 +274,12 @@ "step = tf.group(\n", " U.assign(U_),\n", " Ut.assign(Ut_))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "eAjwNRjTppN-", - "colab_type": "text" + "id": "eAjwNRjTppN-" }, "source": [ "## Run the simulation\n", @@ -320,11 +289,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "id": "jJLvEydzprsy", - "colab_type": "code", - "colab": {} + "id": "jJLvEydzprsy" }, + "outputs": [], "source": [ "# Initialize state to initial conditions\n", "tf.global_variables_initializer().run()\n", @@ -336,19 +305,28 @@ "\n", "# Show final image\n", "DisplayArray(U.eval(), rng=[-0.1, 0.1])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "id": "8AcEDQfbpyDT", - "colab_type": "text" + "id": "8AcEDQfbpyDT" }, "source": [ "Look! Ripples!" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "name": "pdes.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/representation/kernel_methods.md b/site/en/r1/tutorials/representation/kernel_methods.md index 67adc4951c6..227fe81d515 100644 --- a/site/en/r1/tutorials/representation/kernel_methods.md +++ b/site/en/r1/tutorials/representation/kernel_methods.md @@ -24,7 +24,7 @@ following sources for an introduction: Currently, TensorFlow supports explicit kernel mappings for dense features only; TensorFlow will provide support for sparse features at a later release. -This tutorial uses [tf.contrib.learn](https://www.tensorflow.org/code/tensorflow/contrib/learn/python/learn) +This tutorial uses [tf.contrib.learn](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/contrib/learn/python/learn) (TensorFlow's high-level Machine Learning API) Estimators for our ML models. If you are not familiar with this API, The [Estimator guide](../../guide/estimators.md) is a good place to start. We will use the MNIST dataset. The tutorial consists @@ -131,7 +131,7 @@ In addition to experimenting with the (training) batch size and the number of training steps, there are a couple other parameters that can be tuned as well. For instance, you can change the optimization method used to minimize the loss by explicitly selecting another optimizer from the collection of -[available optimizers](https://www.tensorflow.org/code/tensorflow/python/training). +[available optimizers](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/python/training). As an example, the following code constructs a LinearClassifier estimator that uses the Follow-The-Regularized-Leader (FTRL) optimization strategy with a specific learning rate and L2-regularization. diff --git a/site/en/r1/tutorials/representation/linear.md b/site/en/r1/tutorials/representation/linear.md index 5516672b34a..d996a13bc1f 100644 --- a/site/en/r1/tutorials/representation/linear.md +++ b/site/en/r1/tutorials/representation/linear.md @@ -12,7 +12,7 @@ those tools. It explains: Read this overview to decide whether the Estimator's linear model tools might be useful to you. Then work through the -[Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep) +[Estimator wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep) to give it a try. This overview uses code samples from the tutorial, but the tutorial walks through the code in greater detail. @@ -177,7 +177,7 @@ the name of a `FeatureColumn`. Each key's value is a tensor containing the values of that feature for all data instances. See [Premade Estimators](../../guide/premade_estimators.md#input_fn) for a more comprehensive look at input functions, and `input_fn` in the -[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep) +[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep) for an example implementation of an input function. The input function is passed to the `train()` and `evaluate()` calls that @@ -236,4 +236,4 @@ e = tf.estimator.DNNLinearCombinedClassifier( dnn_hidden_units=[100, 50]) ``` For more information, see the -[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/master/official/r1/wide_deep). +[wide and deep learning tutorial](https://github.com/tensorflow/models/tree/r1.15/official/r1/wide_deep). diff --git a/site/en/r1/tutorials/representation/unicode.ipynb b/site/en/r1/tutorials/representation/unicode.ipynb index 9e04b101f15..f76977c3c92 100644 --- a/site/en/r1/tutorials/representation/unicode.ipynb +++ b/site/en/r1/tutorials/representation/unicode.ipynb @@ -1,28 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "unicode.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "oL9KopJirB2g" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "oL9KopJirB2g" }, "source": [ @@ -31,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "SKaX3Hd3ra6C", - "colab": {} + "id": "SKaX3Hd3ra6C" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -49,14 +29,11 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AXH1bmUctMld" }, "source": [ @@ -75,7 +52,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "AXH1bmUctMld" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "LrHJrKYis06U" }, "source": [ @@ -88,24 +78,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "OIKHl5Lvn4gh", - "colab": {} + "id": "OIKHl5Lvn4gh" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] + "import tensorflow.compat.v1 as tf\n" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "n-LkcI-vtWNj" }, "source": [ @@ -117,21 +101,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3yo-Qv6ntaFr", - "colab": {} + "id": "3yo-Qv6ntaFr" }, + "outputs": [], "source": [ "tf.constant(u\"Thanks 😊\")" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2kA1ziG2tyCT" }, "source": [ @@ -140,31 +121,27 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "eyINCmTztyyS", - "colab": {} + "id": "eyINCmTztyyS" }, + "outputs": [], "source": [ "tf.constant([u\"You're\", u\"welcome!\"]).shape" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jsMPnjb6UDJ1" }, "source": [ - "Note: When using python to construct strings, the handling of unicode differs betweeen v2 and v3. In v2, unicode strings are indicated by the \"u\" prefix, as above. In v3, strings are unicode-encoded by default." + "Note: When using python to construct strings, the handling of unicode differs between v2 and v3. In v2, unicode strings are indicated by the \"u\" prefix, as above. In v3, strings are unicode-encoded by default." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hUFZ7B1Lk-uj" }, "source": [ @@ -180,53 +157,46 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "cjQIkfJWvC_u", - "colab": {} + "id": "cjQIkfJWvC_u" }, + "outputs": [], "source": [ "# Unicode string, represented as a UTF-8 encoded string scalar.\n", "text_utf8 = tf.constant(u\"语言处理\")\n", "text_utf8" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "yQqcUECcvF2r", - "colab": {} + "id": "yQqcUECcvF2r" }, + "outputs": [], "source": [ "# Unicode string, represented as a UTF-16-BE encoded string scalar.\n", "text_utf16be = tf.constant(u\"语言处理\".encode(\"UTF-16-BE\"))\n", "text_utf16be" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ExdBr1t7vMuS", - "colab": {} + "id": "ExdBr1t7vMuS" }, + "outputs": [], "source": [ "# Unicode string, represented as a vector of Unicode code points.\n", "text_chars = tf.constant([ord(char) for char in u\"语言处理\"])\n", "text_chars" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "B8czv4JNpBnZ" }, "source": [ @@ -241,51 +211,44 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "qb-UQ_oLpAJg", - "colab": {} + "id": "qb-UQ_oLpAJg" }, + "outputs": [], "source": [ "tf.strings.unicode_decode(text_utf8,\n", " input_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "kEBUcunnp-9n", - "colab": {} + "id": "kEBUcunnp-9n" }, + "outputs": [], "source": [ "tf.strings.unicode_encode(text_chars,\n", " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "0MLhWcLZrph-", - "colab": {} + "id": "0MLhWcLZrph-" }, + "outputs": [], "source": [ "tf.strings.unicode_transcode(text_utf8,\n", " input_encoding='UTF8',\n", " output_encoding='UTF-16-BE')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QVeLeVohqN7I" }, "source": [ @@ -296,11 +259,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "N2jVzPymr_Mm", - "colab": {} + "id": "N2jVzPymr_Mm" }, + "outputs": [], "source": [ "# A batch of Unicode strings, each represented as a UTF8-encoded string.\n", "batch_utf8 = [s.encode('UTF-8') for s in\n", @@ -309,14 +272,11 @@ " input_encoding='UTF-8')\n", "for sentence_chars in batch_chars_ragged.to_list():\n", " print(sentence_chars)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iRh3n1hPsJ9v" }, "source": [ @@ -325,35 +285,30 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "yz17yeSMsUid", - "colab": {} + "id": "yz17yeSMsUid" }, + "outputs": [], "source": [ "batch_chars_padded = batch_chars_ragged.to_tensor(default_value=-1)\n", "print(batch_chars_padded.numpy())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "kBjsPQp3rhfm", - "colab": {} + "id": "kBjsPQp3rhfm" }, + "outputs": [], "source": [ "batch_chars_sparse = batch_chars_ragged.to_sparse()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GCCkZh-nwlbL" }, "source": [ @@ -362,22 +317,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "_lP62YUAwjK9", - "colab": {} + "id": "_lP62YUAwjK9" }, + "outputs": [], "source": [ "tf.strings.unicode_encode([[99, 97, 116], [100, 111, 103], [ 99, 111, 119]],\n", " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "w58CMRg9tamW" }, "source": [ @@ -386,21 +338,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "d7GtOtrltaMl", - "colab": {} + "id": "d7GtOtrltaMl" }, + "outputs": [], "source": [ "tf.strings.unicode_encode(batch_chars_ragged, output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "T2Nh5Aj9xob3" }, "source": [ @@ -409,38 +358,33 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "R2bYCYl0u-Ue", - "colab": {} + "id": "R2bYCYl0u-Ue" }, + "outputs": [], "source": [ " tf.strings.unicode_encode(\n", " tf.RaggedTensor.from_sparse(batch_chars_sparse),\n", " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "UlV2znh_u_zm", - "colab": {} + "id": "UlV2znh_u_zm" }, + "outputs": [], "source": [ " tf.strings.unicode_encode(\n", " tf.RaggedTensor.from_tensor(batch_chars_padded, padding=-1),\n", " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hQOOGkscvDpc" }, "source": [ @@ -450,7 +394,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NkmtsA_yvMB0" }, "source": [ @@ -461,66 +404,58 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "1ZzMe59mvLHr", - "colab": {} + "id": "1ZzMe59mvLHr" }, + "outputs": [], "source": [ "# Note that the final character takes up 4 bytes in UTF8.\n", "thanks = u'Thanks 😊'.encode('UTF-8')\n", "num_bytes = tf.strings.length(thanks).numpy()\n", "num_chars = tf.strings.length(thanks, unit='UTF8_CHAR').numpy()\n", "print('{} bytes; {} UTF-8 characters'.format(num_bytes, num_chars))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "fHG85gxlvVU0" }, "source": [ "### Character substrings\n", "\n", - "Similarly, the `tf.strings.substr` operation accepts the \"`unit`\" parameter, and uses it to determine what kind of offsets the \"`pos`\" and \"`len`\" paremeters contain." + "Similarly, the `tf.strings.substr` operation accepts the \"`unit`\" parameter, and uses it to determine what kind of offsets the \"`pos`\" and \"`len`\" parameters contain." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "WlWRLV-4xWYq", - "colab": {} + "id": "WlWRLV-4xWYq" }, + "outputs": [], "source": [ "# default: unit='BYTE'. With len=1, we return a single byte.\n", "tf.strings.substr(thanks, pos=7, len=1).numpy()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "JfNUVDPwxkCS", - "colab": {} + "id": "JfNUVDPwxkCS" }, + "outputs": [], "source": [ "# Specifying unit='UTF8_CHAR', we return a single character, which in this case\n", "# is 4 bytes.\n", "print(tf.strings.substr(thanks, pos=7, len=1, unit='UTF8_CHAR').numpy())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "zJUEsVSyeIa3" }, "source": [ @@ -531,21 +466,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "dDjkh5G1ejMt", - "colab": {} + "id": "dDjkh5G1ejMt" }, + "outputs": [], "source": [ "tf.strings.unicode_split(thanks, 'UTF-8').numpy()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HQqEEZEbdG9O" }, "source": [ @@ -556,24 +488,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Cug7cmwYdowd", - "colab": {} + "id": "Cug7cmwYdowd" }, + "outputs": [], "source": [ "codepoints, offsets = tf.strings.unicode_decode_with_offsets(u\"🎈🎉🎊\", 'UTF-8')\n", "\n", "for (codepoint, offset) in zip(codepoints.numpy(), offsets.numpy()):\n", " print(\"At byte offset {}: codepoint {}\".format(offset, codepoint))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2ZnCNxOvx66T" }, "source": [ @@ -583,7 +512,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "nRRHqkqNyGZ6" }, "source": [ @@ -595,23 +523,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "K7DeYHrRyFPy", - "colab": {} + "id": "K7DeYHrRyFPy" }, + "outputs": [], "source": [ "uscript = tf.strings.unicode_script([33464, 1041]) # ['芸', 'Б']\n", "\n", "print(uscript.numpy()) # [17, 8] == [USCRIPT_HAN, USCRIPT_CYRILLIC]" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2fW992a1lIY6" }, "source": [ @@ -620,21 +545,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "uR7b8meLlFnp", - "colab": {} + "id": "uR7b8meLlFnp" }, + "outputs": [], "source": [ "print(tf.strings.unicode_script(batch_chars_ragged))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mx7HEFpBzEsB" }, "source": [ @@ -647,37 +569,34 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "grsvFiC4BoPb", - "colab": {} + "id": "grsvFiC4BoPb" }, + "outputs": [], "source": [ "# dtype: string; shape: [num_sentences]\n", "#\n", "# The sentences to process. Edit this line to try out different inputs!\n", "sentence_texts = [u'Hello, world.', u'世界こんにちは']" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CapnbShuGU8i" }, "source": [ - "First, we decode the sentences into character codepoints, and find the script identifeir for each character." + "First, we decode the sentences into character codepoints, and find the script identifier for each character." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ReQVcDQh1MB8", - "colab": {} + "id": "ReQVcDQh1MB8" }, + "outputs": [], "source": [ "# dtype: int32; shape: [num_sentences, (num_chars_per_sentence)]\n", "#\n", @@ -692,14 +611,11 @@ "# the i'th sentence.\n", "sentence_char_script = tf.strings.unicode_script(sentence_char_codepoint)\n", "print(sentence_char_script)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "O2fapF5UGcUc" }, "source": [ @@ -708,11 +624,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7v5W6MOr1Rlc", - "colab": {} + "id": "7v5W6MOr1Rlc" }, + "outputs": [], "source": [ "# dtype: bool; shape: [num_sentences, (num_chars_per_sentence)]\n", "#\n", @@ -729,14 +645,11 @@ "# the flattened list of characters from all sentences).\n", "word_starts = tf.squeeze(tf.where(sentence_char_starts_word.values), axis=1)\n", "print(word_starts)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LAwh-1QkGuC9" }, "source": [ @@ -745,11 +658,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "bNiA1O_eBBCL", - "colab": {} + "id": "bNiA1O_eBBCL" }, + "outputs": [], "source": [ "# dtype: int32; shape: [num_words, (num_chars_per_word)]\n", "#\n", @@ -759,14 +672,11 @@ " values=sentence_char_codepoint.values,\n", " row_starts=word_starts)\n", "print(word_char_codepoint)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "66a2ZnYmG2ao" }, "source": [ @@ -775,11 +685,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "NCfwcqLSEjZb", - "colab": {} + "id": "NCfwcqLSEjZb" }, + "outputs": [], "source": [ "# dtype: int64; shape: [num_sentences]\n", "#\n", @@ -796,14 +706,11 @@ " values=word_char_codepoint,\n", " row_lengths=sentence_num_words)\n", "print(sentence_word_char_codepoint)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xWaX8WcbHyqY" }, "source": [ @@ -812,16 +719,29 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "HSivquOgFr3C", - "colab": {} + "id": "HSivquOgFr3C" }, + "outputs": [], "source": [ "tf.strings.unicode_encode(sentence_word_char_codepoint, 'UTF-8').to_list()" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "oL9KopJirB2g" ], - "execution_count": 0, - "outputs": [] + "name": "unicode.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/r1/tutorials/representation/word2vec.md b/site/en/r1/tutorials/representation/word2vec.md index bb8102eb742..517a5dbc5c5 100644 --- a/site/en/r1/tutorials/representation/word2vec.md +++ b/site/en/r1/tutorials/representation/word2vec.md @@ -19,11 +19,11 @@ represent words as vectors. We walk through the code later during the tutorial, but if you'd prefer to dive straight in, feel free to look at the minimalistic implementation in -[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py) +[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/examples/tutorials/word2vec/word2vec_basic.py) This basic example contains the code needed to download some data, train on it a bit and visualize the result. Once you get comfortable with reading and running the basic version, you can graduate to -[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/master/tutorials/embedding/word2vec.py) +[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/r1.13.0/tutorials/embedding/word2vec.py) which is a more serious implementation that showcases some more advanced TensorFlow principles about how to efficiently use threads to move data into a text model, how to checkpoint during training, etc. @@ -36,7 +36,7 @@ like to get your hands dirty with the details. Image and audio processing systems work with rich, high-dimensional datasets encoded as vectors of the individual raw pixel-intensities for image data, or -e.g. power spectral density coefficients for audio data. For tasks like object +e.g., power spectral density coefficients for audio data. For tasks like object or speech recognition we know that all the information required to successfully perform the task is encoded in the data (because humans can perform these tasks from the raw data). However, natural language processing systems traditionally @@ -109,7 +109,7 @@ $$ where \\(\text{score}(w_t, h)\\) computes the compatibility of word \\(w_t\\) with the context \\(h\\) (a dot product is commonly used). We train this model by maximizing its [log-likelihood](https://en.wikipedia.org/wiki/Likelihood_function) -on the training set, i.e. by maximizing +on the training set, i.e., by maximizing $$ \begin{align} @@ -176,7 +176,7 @@ As an example, let's consider the dataset We first form a dataset of words and the contexts in which they appear. We could define 'context' in any way that makes sense, and in fact people have looked at syntactic contexts (i.e. the syntactic dependents of the current -target word, see e.g. +target word, see e.g., [Levy et al.](https://levyomer.files.wordpress.com/2014/04/dependency-based-word-embeddings-acl-2014.pdf)), words-to-the-left of the target, words-to-the-right of the target, etc. For now, let's stick to the vanilla definition and define 'context' as the window @@ -204,7 +204,7 @@ where the goal is to predict `the` from `quick`. We select `num_noise` number of noisy (contrastive) examples by drawing from some noise distribution, typically the unigram distribution, \\(P(w)\\). For simplicity let's say `num_noise=1` and we select `sheep` as a noisy example. Next we compute the -loss for this pair of observed and noisy examples, i.e. the objective at time +loss for this pair of observed and noisy examples, i.e., the objective at time step \\(t\\) becomes $$J^{(t)}_\text{NEG} = \log Q_\theta(D=1 | \text{the, quick}) + @@ -212,7 +212,7 @@ $$J^{(t)}_\text{NEG} = \log Q_\theta(D=1 | \text{the, quick}) + The goal is to make an update to the embedding parameters \\(\theta\\) to improve (in this case, maximize) this objective function. We do this by deriving the -gradient of the loss with respect to the embedding parameters \\(\theta\\), i.e. +gradient of the loss with respect to the embedding parameters \\(\theta\\), i.e., \\(\frac{\partial}{\partial \theta} J_\text{NEG}\\) (luckily TensorFlow provides easy helper functions for doing this!). We then perform an update to the embeddings by taking a small step in the direction of the gradient. When this @@ -227,7 +227,7 @@ When we inspect these visualizations it becomes apparent that the vectors capture some general, and in fact quite useful, semantic information about words and their relationships to one another. It was very interesting when we first discovered that certain directions in the induced vector space specialize -towards certain semantic relationships, e.g. *male-female*, *verb tense* and +towards certain semantic relationships, e.g., *male-female*, *verb tense* and even *country-capital* relationships between words, as illustrated in the figure below (see also for example [Mikolov et al., 2013](https://www.aclweb.org/anthology/N13-1090)). @@ -271,7 +271,7 @@ nce_biases = tf.Variable(tf.zeros([vocabulary_size])) Now that we have the parameters in place, we can define our skip-gram model graph. For simplicity, let's suppose we've already integerized our text corpus with a vocabulary so that each word is represented as an integer (see -[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py) +[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://github.com/tensorflow/tensorflow/tree/r1.15/tensorflow/examples/tutorials/word2vec/word2vec_basic.py) for the details). The skip-gram model takes two inputs. One is a batch full of integers representing the source context words, the other is for the target words. Let's create placeholder nodes for these inputs, so that we can feed in @@ -327,7 +327,7 @@ for inputs, labels in generate_batch(...): ``` See the full example code in -[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://www.tensorflow.org/code/tensorflow/examples/tutorials/word2vec/word2vec_basic.py). +[tensorflow/examples/tutorials/word2vec/word2vec_basic.py](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/examples/tutorials/word2vec/word2vec_basic.py). ## Visualizing the learned embeddings @@ -341,7 +341,7 @@ t-SNE. Et voila! As expected, words that are similar end up clustering nearby each other. For a more heavyweight implementation of word2vec that showcases more of the advanced features of TensorFlow, see the implementation in -[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/master/tutorials/embedding/word2vec.py). +[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/embedding/word2vec.py). ## Evaluating embeddings: analogical reasoning @@ -357,7 +357,7 @@ Download the dataset for this task from To see how we do this evaluation, have a look at the `build_eval_graph()` and `eval()` functions in -[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/master/tutorials/embedding/word2vec.py). +[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/r1.15/research/tutorials/embedding/word2vec.py). The choice of hyperparameters can strongly influence the accuracy on this task. To achieve state-of-the-art performance on this task requires training over a @@ -378,20 +378,20 @@ out several different ideas and iterating quickly. Once you have a model structure you're satisfied with, it may be worth optimizing your implementation to run more efficiently (and cover more data in -less time). For example, the naive code we used in this tutorial would suffer +less time). For example, the naive code we used in this tutorial would suffer compromised speed because we use Python for reading and feeding data items -- -each of which require very little work on the TensorFlow back-end. If you find +each of which require very little work on the TensorFlow back-end. If you find your model is seriously bottlenecked on input data, you may want to implement a custom data reader for your problem, as described in -[New Data Formats](../../extend/new_data_formats.md). For the case of Skip-Gram +[New Data Formats](../../guide/extend/formats.md). For the case of Skip-Gram modeling, we've actually already done this for you as an example in -[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/master/tutorials/embedding/word2vec.py). +[models/tutorials/embedding/word2vec.py](https://github.com/tensorflow/models/tree/r1.13.0/tutorials/embedding/word2vec.py). If your model is no longer I/O bound but you want still more performance, you can take things further by writing your own TensorFlow Ops, as described in -[Adding a New Op](../../guide/extend/op.md). Again we've provided an -example of this for the Skip-Gram case -[models/tutorials/embedding/word2vec_optimized.py](https://github.com/tensorflow/models/tree/master/tutorials/embedding/word2vec_optimized.py). +[Adding a New Op](../../guide/extend/op.md). Again we've provided an example of +this for the Skip-Gram case +[models/tutorials/embedding/word2vec_optimized.py](https://github.com/tensorflow/models/tree/r1.13.0/tutorials/embedding/word2vec_optimized.py). Feel free to benchmark these against each other to measure performance improvements at each stage. diff --git a/site/en/r1/tutorials/sequences/audio_recognition.md b/site/en/r1/tutorials/sequences/audio_recognition.md index 65ca65136fc..0388514ec92 100644 --- a/site/en/r1/tutorials/sequences/audio_recognition.md +++ b/site/en/r1/tutorials/sequences/audio_recognition.md @@ -159,9 +159,9 @@ accuracy. If the training accuracy increases but the validation doesn't, that's a sign that overfitting is occurring, and your model is only learning things about the training clips, not broader patterns that generalize. -## Tensorboard +## TensorBoard -A good way to visualize how the training is progressing is using Tensorboard. By +A good way to visualize how the training is progressing is using TensorBoard. By default, the script saves out events to /tmp/retrain_logs, and you can load these by running: @@ -402,7 +402,7 @@ change to customize the results for your own requirements. ### Custom Training Data By default the script will download the [Speech Commands -dataset](https://download.tensorflow.org/data/speech_commands_v0.01.tar.gz), but +dataset](https://storage.googleapis.com/download.tensorflow.org/data/speech_commands_v0.02.tar.gz), but you can also supply your own training data. To train on your own data, you should make sure that you have at least several hundred recordings of each sound you would like to recognize, and arrange them into folders by class. For diff --git a/site/en/r1/tutorials/sequences/recurrent.md b/site/en/r1/tutorials/sequences/recurrent.md index 6654795d944..e7c1f8c0b16 100644 --- a/site/en/r1/tutorials/sequences/recurrent.md +++ b/site/en/r1/tutorials/sequences/recurrent.md @@ -2,7 +2,7 @@ ## Introduction -See [Understanding LSTM Networks](https://colah.github.io/posts/2015-08-Understanding-LSTMs/){:.external} +See [Understanding LSTM Networks](https://colah.github.io/posts/2015-08-Understanding-LSTMs/) for an introduction to recurrent neural networks and LSTMs. ## Language Modeling diff --git a/site/en/r1/tutorials/sequences/recurrent_quickdraw.md b/site/en/r1/tutorials/sequences/recurrent_quickdraw.md index fb102204de9..d6a85377d17 100644 --- a/site/en/r1/tutorials/sequences/recurrent_quickdraw.md +++ b/site/en/r1/tutorials/sequences/recurrent_quickdraw.md @@ -33,12 +33,14 @@ drawings in 345 categories. To try the code for this tutorial: 1. [Install TensorFlow](../../install) if you haven't already. -1. Download the [tutorial code](https://github.com/tensorflow/models/tree/master/tutorials/rnn/quickdraw/train_model.py). +1. Download the + [tutorial code](https://raw.githubusercontent.com/tensorflow/models/r1.13.0/tutorials/rnn/quickdraw/train_model.py). 1. [Download the data](#download-the-data) in `TFRecord` format from - [here](http://download.tensorflow.org/data/quickdraw_tutorial_dataset_v1.tar.gz) and unzip it. More details about [how to - obtain the original Quick, Draw! - data](#optional_download_the_full_quick_draw_data) and [how to convert that - to `TFRecord` files](#optional_converting_the_data) is available below. + [here](http://download.tensorflow.org/data/quickdraw_tutorial_dataset_v1.tar.gz) + and unzip it. More details about + [how to obtain the original Quick, Draw! data](#optional_download_the_full_quick_draw_data) + and [how to convert that to `TFRecord` files](#optional_converting_the_data) + is available below. 1. Execute the tutorial code with the following command to train the RNN-based model described in this tutorial. Make sure to adjust the paths to point to @@ -107,7 +109,7 @@ This download will take a while and download a bit more than 23GB of data. To convert the `ndjson` files to [TFRecord](../../api_guides/python/python_io.md#TFRecords_Format_Details) files containing -[`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto) +[`tf.train.Example`](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto) protos run the following command. ```shell @@ -211,7 +213,7 @@ screen coordinates and normalize the size such that the drawing has unit height. Finally, we compute the differences between consecutive points and store these as a `VarLenFeature` in a -[tensorflow.Example](https://www.tensorflow.org/code/tensorflow/core/example/example.proto) +[tensorflow.Example](https://github.com/tensorflow/tensorflow/blob/r1.15/tensorflow/core/example/example.proto) under the key `ink`. In addition we store the `class_index` as a single entry `FixedLengthFeature` and the `shape` of the `ink` as a `FixedLengthFeature` of length 2. diff --git a/site/en/r1/tutorials/sequences/text_generation.ipynb b/site/en/r1/tutorials/sequences/text_generation.ipynb index 8d808dd5639..84d942c8bd0 100644 --- a/site/en/r1/tutorials/sequences/text_generation.ipynb +++ b/site/en/r1/tutorials/sequences/text_generation.ipynb @@ -1,26 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_generation.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "t09eeeR5prIJ" }, "source": [ @@ -29,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "GCCk8_dHpuNf", - "colab": {} + "id": "GCCk8_dHpuNf" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -47,14 +29,11 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hcD2nPQvPOFM" }, "source": [ @@ -64,8 +43,7 @@ { "cell_type": "markdown", "metadata": { - "id": "joH3UQjMohtX", - "colab_type": "text" + "id": "joH3UQjMohtX" }, "source": [ "
    \n", @@ -82,13 +60,26 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "joH3UQjMohtX" + }, + "source": [ + "> Note: This is an archived TF1 notebook. These are configured\n", + "to run in TF2's \n", + "[compatibility mode](https://www.tensorflow.org/guide/migrate)\n", + "but will run in TF1 as well. To use TF1 in Colab, use the\n", + "[%tensorflow_version 1.x](https://colab.research.google.com/notebooks/tensorflow_version.ipynb)\n", + "magic." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "BwpJ5IffzRG6" }, "source": [ - "This tutorial demonstrates how to generate text using a character-based RNN. We will work with a dataset of Shakespeare's writing from Andrej Karpathy's [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). Given a sequence of characters from this data (\"Shakespear\"), train a model to predict the next character in the sequence (\"e\"). Longer sequences of text can be generated by calling the model repeatedly.\n", + "This tutorial demonstrates how to generate text using a character-based RNN. You will work with a dataset of Shakespeare's writing from Andrej Karpathy's [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). Given a sequence of characters from this data (\"Shakespear\"), train a model to predict the next character in the sequence (\"e\"). Longer sequences of text can be generated by calling the model repeatedly.\n", "\n", - "Note: Enable GPU acceleration to execute this notebook faster. In Colab: *Runtime > Change runtime type > Hardware acclerator > GPU*. If running locally make sure TensorFlow version >= 1.11.\n", + "Note: Enable GPU acceleration to execute this notebook faster. In Colab: *Runtime > Change runtime type > Hardware accelerator > GPU*. If running locally make sure TensorFlow version >= 1.11.\n", "\n", "This tutorial includes runnable code implemented using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). The following is sample output when the model in this tutorial trained for 30 epochs, and started with the string \"Q\":\n", "\n", @@ -107,7 +98,7 @@ "To watch the next way with his father with his face?\n", "\n", "ESCALUS:\n", - "The cause why then we are all resolved more sons.\n", + "The cause why then us all resolved more sons.\n", "\n", "VOLUMNIA:\n", "O, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, it is no sin it should be dead,\n", @@ -137,7 +128,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "srXC6pLGLwS6" }, "source": [ @@ -147,7 +137,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WGyKZj3bzf9p" }, "source": [ @@ -156,28 +145,23 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "yG_n40gFzf9s", - "colab": {} + "id": "yG_n40gFzf9s" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", + "import tensorflow.compat.v1 as tf\n", "\n", - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", "\n", "import numpy as np\n", "import os\n", "import time" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "EHDoRoc5PKWz" }, "source": [ @@ -188,21 +172,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "pD_55cOxLkAb", - "colab": {} + "id": "pD_55cOxLkAb" }, + "outputs": [], "source": [ "path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UHjdCjDuSvX_" }, "source": [ @@ -213,53 +194,46 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "aavnuByVymwK", - "colab": {} + "id": "aavnuByVymwK" }, + "outputs": [], "source": [ "# Read, then decode for py2 compat.\n", "text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n", "# length of text is the number of characters in it\n", "print ('Length of text: {} characters'.format(len(text)))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Duhg9NrUymwO", - "colab": {} + "id": "Duhg9NrUymwO" }, + "outputs": [], "source": [ "# Take a look at the first 250 characters in text\n", "print(text[:250])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "IlCgQBRVymwR", - "colab": {} + "id": "IlCgQBRVymwR" }, + "outputs": [], "source": [ "# The unique characters in the file\n", "vocab = sorted(set(text))\n", "print ('{} unique characters'.format(len(vocab)))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rNnrKn_lL-IJ" }, "source": [ @@ -269,76 +243,67 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LFjSVAlWzf-N" }, "source": [ "### Vectorize the text\n", "\n", - "Before training, we need to map strings to a numerical representation. Create two lookup tables: one mapping characters to numbers, and another for numbers to characters." + "Before training, you need to map strings to a numerical representation. Create two lookup tables: one mapping characters to numbers, and another for numbers to characters." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "IalZLbvOzf-F", - "colab": {} + "id": "IalZLbvOzf-F" }, + "outputs": [], "source": [ "# Creating a mapping from unique characters to indices\n", "char2idx = {u:i for i, u in enumerate(vocab)}\n", "idx2char = np.array(vocab)\n", "\n", "text_as_int = np.array([char2idx[c] for c in text])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tZfqhkYCymwX" }, "source": [ - "Now we have an integer representation for each character. Notice that we mapped the character as indexes from 0 to `len(unique)`." + "Now you have an integer representation for each character. Notice that you mapped the character as indexes from 0 to `len(unique)`." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "FYyNlCNXymwY", - "colab": {} + "id": "FYyNlCNXymwY" }, + "outputs": [], "source": [ "print('{')\n", "for char,_ in zip(char2idx, range(20)):\n", " print(' {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n", "print(' ...\\n}')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "l1VKcQHcymwb", - "colab": {} + "id": "l1VKcQHcymwb" }, + "outputs": [], "source": [ "# Show how the first 13 characters from the text are mapped to integers\n", "print ('{} ---- characters mapped to int ---- > {}'.format(repr(text[:13]), text_as_int[:13]))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bbmsf23Bymwe" }, "source": [ @@ -348,11 +313,10 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wssHQ1oGymwe" }, "source": [ - "Given a character, or a sequence of characters, what is the most probable next character? This is the task we're training the model to perform. The input to the model will be a sequence of characters, and we train the model to predict the output—the following character at each time step.\n", + "Given a character, or a sequence of characters, what is the most probable next character? This is the task you are training the model to perform. The input to the model will be a sequence of characters, and you train the model to predict the output—the following character at each time step.\n", "\n", "Since RNNs maintain an internal state that depends on the previously seen elements, given all the characters computed until this moment, what is the next character?\n" ] @@ -360,7 +324,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hgsVvVxnymwf" }, "source": [ @@ -377,13 +340,13 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "0UHJDA39zf-O", - "colab": {} + "id": "0UHJDA39zf-O" }, + "outputs": [], "source": [ - "# The maximum length sentence we want for a single input in characters\n", + "# The maximum length sentence you want for a single input in characters\n", "seq_length = 100\n", "examples_per_epoch = len(text)//seq_length\n", "\n", @@ -392,14 +355,11 @@ "\n", "for i in char_dataset.take(5):\n", " print(idx2char[i.numpy()])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-ZSYAcQV8OGP" }, "source": [ @@ -408,24 +368,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "l4hkDU3i7ozi", - "colab": {} + "id": "l4hkDU3i7ozi" }, + "outputs": [], "source": [ "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n", "\n", "for item in sequences.take(5):\n", " print(repr(''.join(idx2char[item.numpy()])))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UbLcIPBj_mWZ" }, "source": [ @@ -434,11 +391,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "9NGu-FkO_kYU", - "colab": {} + "id": "9NGu-FkO_kYU" }, + "outputs": [], "source": [ "def split_input_target(chunk):\n", " input_text = chunk[:-1]\n", @@ -446,14 +403,11 @@ " return input_text, target_text\n", "\n", "dataset = sequences.map(split_input_target)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hiCopyGZymwi" }, "source": [ @@ -462,23 +416,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "GNbw-iR0ymwj", - "colab": {} + "id": "GNbw-iR0ymwj" }, + "outputs": [], "source": [ "for input_example, target_example in dataset.take(1):\n", " print ('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n", " print ('Target data:', repr(''.join(idx2char[target_example.numpy()])))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_33OHL3b84i0" }, "source": [ @@ -487,39 +438,36 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "0eBu9WZG84i0", - "colab": {} + "id": "0eBu9WZG84i0" }, + "outputs": [], "source": [ "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n", " print(\"Step {:4d}\".format(i))\n", " print(\" input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n", " print(\" expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MJdfPmdqzf-R" }, "source": [ "### Create training batches\n", "\n", - "We used `tf.data` to split the text into manageable sequences. But before feeding this data into the model, we need to shuffle the data and pack it into batches." + "You used `tf.data` to split the text into manageable sequences. But before feeding this data into the model, you need to shuffle the data and pack it into batches." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "p2pGotuNzf-S", - "colab": {} + "id": "p2pGotuNzf-S" }, + "outputs": [], "source": [ "# Batch size\n", "BATCH_SIZE = 64\n", @@ -534,14 +482,11 @@ "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n", "\n", "dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "r6oUuElIMgVx" }, "source": [ @@ -551,7 +496,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "m8gPwEjRzf-Z" }, "source": [ @@ -564,11 +508,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zHT8cLh7EAsg", - "colab": {} + "id": "zHT8cLh7EAsg" }, + "outputs": [], "source": [ "# Length of the vocabulary in chars\n", "vocab_size = len(vocab)\n", @@ -578,14 +522,11 @@ "\n", "# Number of RNN units\n", "rnn_units = 1024" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NNVB-jmMEOzP" }, "source": [ @@ -596,29 +537,27 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "wjKZrC39ELy0", - "colab": {} + "id": "wjKZrC39ELy0" }, + "outputs": [], "source": [ - "if tf.test.is_gpu_available():\n", + "if tf.config.list_physical_devices('GPU'):\n", " rnn = tf.keras.layers.CuDNNGRU\n", "else:\n", " import functools\n", " rnn = functools.partial(\n", " tf.keras.layers.GRU, recurrent_activation='sigmoid')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "MtCrdfzEI2N0", - "colab": {} + "id": "MtCrdfzEI2N0" }, + "outputs": [], "source": [ "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n", " model = tf.keras.Sequential([\n", @@ -631,31 +570,26 @@ " tf.keras.layers.Dense(vocab_size)\n", " ])\n", " return model" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "wwsrpOik5zhv", - "colab": {} + "id": "wwsrpOik5zhv" }, + "outputs": [], "source": [ "model = build_model(\n", " vocab_size = len(vocab),\n", " embedding_dim=embedding_dim,\n", " rnn_units=rnn_units,\n", " batch_size=BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RkA5upJIJ7W7" }, "source": [ @@ -667,7 +601,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-ubPo0_9Prjb" }, "source": [ @@ -680,23 +613,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "C-_70kKAPrPU", - "colab": {} + "id": "C-_70kKAPrPU" }, + "outputs": [], "source": [ "for input_example_batch, target_example_batch in dataset.take(1):\n", " example_batch_predictions = model(input_example_batch)\n", " print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Q6NzLBi4VM4o" }, "source": [ @@ -705,25 +635,22 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "vPGmAAXmVLGC", - "colab": {} + "id": "vPGmAAXmVLGC" }, + "outputs": [], "source": [ "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uwv0gEkURfx1" }, "source": [ - "To get actual predictions from the model we need to sample from the output distribution, to get actual character indices. This distribution is defined by the logits over the character vocabulary.\n", + "To get actual predictions from the model you need to sample from the output distribution, to get actual character indices. This distribution is defined by the logits over the character vocabulary.\n", "\n", "Note: It is important to _sample_ from this distribution as taking the _argmax_ of the distribution can easily get the model stuck in a loop.\n", "\n", @@ -732,22 +659,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "4V4MfFg0RQJg", - "colab": {} + "id": "4V4MfFg0RQJg" }, + "outputs": [], "source": [ "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n", "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QM1Vbxs_URw5" }, "source": [ @@ -756,21 +680,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "YqFMUQc_UFgM", - "colab": {} + "id": "YqFMUQc_UFgM" }, + "outputs": [], "source": [ "sampled_indices" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LfLtsP3mUhCG" }, "source": [ @@ -779,23 +700,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "xWcFwPwLSo05", - "colab": {} + "id": "xWcFwPwLSo05" }, + "outputs": [], "source": [ "print(\"Input: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n", "print()\n", "print(\"Next Char Predictions: \\n\", repr(\"\".join(idx2char[sampled_indices ])))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LJL0Q0YPY6Ee" }, "source": [ @@ -805,7 +723,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YCbHQHiaa4Ic" }, "source": [ @@ -815,7 +732,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "trpqTWyvk0nr" }, "source": [ @@ -825,22 +741,21 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UAjbjY03eiQ4" }, "source": [ "The standard `tf.keras.losses.sparse_categorical_crossentropy` loss function works in this case because it is applied across the last dimension of the predictions.\n", "\n", - "Because our model returns logits, we need to set the `from_logits` flag.\n" + "Because our model returns logits, you need to set the `from_logits` flag.\n" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "4HrXTACTdzY-", - "colab": {} + "id": "4HrXTACTdzY-" }, + "outputs": [], "source": [ "def loss(labels, logits):\n", " return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n", @@ -848,39 +763,33 @@ "example_batch_loss = loss(target_example_batch, example_batch_predictions)\n", "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n", "print(\"scalar_loss: \", example_batch_loss.numpy().mean())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jeOXriLcymww" }, "source": [ - "Configure the training procedure using the `tf.keras.Model.compile` method. We'll use `tf.train.AdamOptimizer` with default arguments and the loss function." + "Configure the training procedure using the `tf.keras.Model.compile` method. You'll use `tf.train.AdamOptimizer` with default arguments and the loss function." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "DDl1_Een6rL0", - "colab": {} + "id": "DDl1_Een6rL0" }, + "outputs": [], "source": [ "model.compile(\n", " optimizer = tf.train.AdamOptimizer(),\n", " loss = loss)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ieSJdchZggUj" }, "source": [ @@ -890,7 +799,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "C6XBUUavgF56" }, "source": [ @@ -899,11 +807,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "W6fWTriUZP-n", - "colab": {} + "id": "W6fWTriUZP-n" }, + "outputs": [], "source": [ "# Directory where the checkpoints will be saved\n", "checkpoint_dir = './training_checkpoints'\n", @@ -913,14 +821,11 @@ "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n", " filepath=checkpoint_prefix,\n", " save_weights_only=True)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3Ky3F_BhgkTW" }, "source": [ @@ -930,7 +835,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IxdOA-rgyGvs" }, "source": [ @@ -939,34 +843,29 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7yGBE2zxMMHs", - "colab": {} + "id": "7yGBE2zxMMHs" }, + "outputs": [], "source": [ "EPOCHS=3" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "UK-hmKjYVoll", - "colab": {} + "id": "UK-hmKjYVoll" }, + "outputs": [], "source": [ "history = model.fit(dataset.repeat(), epochs=EPOCHS, steps_per_epoch=steps_per_epoch, callbacks=[checkpoint_callback])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kKkD5M6eoSiN" }, "source": [ @@ -976,7 +875,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JIPcXllKjkdr" }, "source": [ @@ -986,7 +884,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LyeYRiuVjodY" }, "source": [ @@ -994,56 +891,49 @@ "\n", "Because of the way the RNN state is passed from timestep to timestep, the model only accepts a fixed batch size once built.\n", "\n", - "To run the model with a different `batch_size`, we need to rebuild the model and restore the weights from the checkpoint.\n" + "To run the model with a different `batch_size`, you need to rebuild the model and restore the weights from the checkpoint.\n" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zk2WJ2-XjkGz", - "colab": {} + "id": "zk2WJ2-XjkGz" }, + "outputs": [], "source": [ "tf.train.latest_checkpoint(checkpoint_dir)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "LycQ-ot_jjyu", - "colab": {} + "id": "LycQ-ot_jjyu" }, + "outputs": [], "source": [ "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n", "\n", "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", "\n", "model.build(tf.TensorShape([1, None]))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "71xa6jnYVrAN", - "colab": {} + "id": "71xa6jnYVrAN" }, + "outputs": [], "source": [ "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DjGz1tDkzf-u" }, "source": [ @@ -1067,11 +957,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "WvuwZBX5Ogfd", - "colab": {} + "id": "WvuwZBX5Ogfd" }, + "outputs": [], "source": [ "def generate_text(model, start_string):\n", " # Evaluation step (generating text using the learned model)\n", @@ -1102,34 +992,29 @@ " predictions = predictions / temperature\n", " predicted_id = tf.multinomial(predictions, num_samples=1)[-1,0].numpy()\n", "\n", - " # We pass the predicted word as the next input to the model\n", + " # You pass the predicted word as the next input to the model\n", " # along with the previous hidden state\n", " input_eval = tf.expand_dims([predicted_id], 0)\n", "\n", " text_generated.append(idx2char[predicted_id])\n", "\n", " return (start_string + ''.join(text_generated))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ktovv0RFhrkn", - "colab": {} + "id": "ktovv0RFhrkn" }, + "outputs": [], "source": [ "print(generate_text(model, start_string=u\"ROMEO: \"))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AM2Uma_-yVIq" }, "source": [ @@ -1141,7 +1026,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Y4QwTjAM6A2O" }, "source": [ @@ -1151,11 +1035,11 @@ "\n", "So now that you've seen how to run the model manually let's unpack the training loop, and implement it ourselves. This gives a starting point, for example, to implement _curriculum learning_ to help stabilize the model's open-loop output.\n", "\n", - "We will use `tf.GradientTape` to track the gradients. You can learn more about this approach by reading the [eager execution guide](https://www.tensorflow.org/r1/guide/eager).\n", + "You will use `tf.GradientTape` to track the gradients. You can learn more about this approach by reading the [eager execution guide](https://www.tensorflow.org/r1/guide/eager).\n", "\n", "The procedure works as follows:\n", "\n", - "* First, initialize the RNN state. We do this by calling the `tf.keras.Model.reset_states` method.\n", + "* First, initialize the RNN state. You do this by calling the `tf.keras.Model.reset_states` method.\n", "\n", "* Next, iterate over the dataset (batch by batch) and calculate the *predictions* associated with each.\n", "\n", @@ -1163,47 +1047,42 @@ "\n", "* Calculate the gradients of the loss with respect to the model variables using the `tf.GradientTape.grads` method.\n", "\n", - "* Finally, take a step downwards by using the optimizer's `tf.train.Optimizer.apply_gradients` method.\n", - "\n" + "* Finally, take a step downwards by using the optimizer's `tf.train.Optimizer.apply_gradients` method.\n" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "_XAm7eCoKULT", - "colab": {} + "id": "_XAm7eCoKULT" }, + "outputs": [], "source": [ "model = build_model(\n", " vocab_size = len(vocab),\n", " embedding_dim=embedding_dim,\n", " rnn_units=rnn_units,\n", " batch_size=BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "qUKhnZtMVpoJ", - "colab": {} + "id": "qUKhnZtMVpoJ" }, + "outputs": [], "source": [ "optimizer = tf.train.AdamOptimizer()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "d4tSNwymzf-q", - "colab": {} + "id": "d4tSNwymzf-q" }, + "outputs": [], "source": [ "# Training step\n", "EPOCHS = 1\n", @@ -1237,9 +1116,21 @@ " print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n", "\n", "model.save_weights(checkpoint_prefix.format(epoch=epoch))" - ], - "execution_count": 0, - "outputs": [] + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "text_generation.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/swift/README.md b/site/en/swift/README.md deleted file mode 100644 index 8289bb57338..00000000000 --- a/site/en/swift/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Welcome to the warp zone! - -# Swift for TensorFlow - -These docs are available here: https://github.com/tensorflow/swift/tree/master/docs/site diff --git a/site/en/tensorboard/README.md b/site/en/tensorboard/README.md deleted file mode 100644 index ed123a5647b..00000000000 --- a/site/en/tensorboard/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Welcome to the warp zone! - -# TensorFlow Probability - -These docs are available here: https://github.com/tensorflow/tensorboard/tree/master/docs diff --git a/site/en/tfx/README.md b/site/en/tfx/README.md deleted file mode 100644 index c56ad2dbf01..00000000000 --- a/site/en/tfx/README.md +++ /dev/null @@ -1,10 +0,0 @@ -Welcome to the warp zone! - -# TensorFlow Extended (TFX) - -These docs are available here: - -* Data Validation: https://github.com/tensorflow/data-validation/tree/master/g3doc -* Model Analysis: https://github.com/tensorflow/model-analysis/tree/master/g3doc -* Transform: https://github.com/tensorflow/transform/tree/master/docs -* Serving: https://github.com/tensorflow/serving/tree/master/tensorflow_serving/g3doc diff --git a/site/en/tutorials/_index.yaml b/site/en/tutorials/_index.yaml index cec12512eae..0d09f04c5c7 100644 --- a/site/en/tutorials/_index.yaml +++ b/site/en/tutorials/_index.yaml @@ -1,9 +1,14 @@ book_path: /overview/_book.yaml project_path: /overview/_project.yaml -description: +title: Tutorials landing_page: custom_css_path: /site-assets/css/style.css nav: left + meta_tags: + - name: description + content: > + Complete, end-to-end examples to learn how to use TensorFlow for ML beginners and experts. Try + tutorials in Google Colab - no setup required. rows: - classname: devsite-landing-row-100 @@ -11,8 +16,9 @@ landing_page: - description: >

    The TensorFlow tutorials are written as Jupyter notebooks and run - directly in Google Colab—a hosted notebook environment that requires - no setup. Click the Run in Google Colab button. + directly in Google Colab—a hosted notebook environment that requires + no setup. At the top of each tutorial, you'll see a Run in Google Colab button. Click + the button to open the notebook and run the code yourself.

    - classname: devsite-landing-row-100 @@ -79,38 +85,16 @@ landing_page: - classname: devsite-landing-row-100 items: - description: > -

    Video and blog updates

    - Subscribe to the - TensorFlow blog, - YouTube channel, - and Twitter - for the latest updates. +

    Video tutorials

    + Check out these videos for an introduction to machine learning with TensorFlow: - items: - - heading: "Intro to Machine Learning" - path: "https://www.youtube.com/watch?v=KNAWp2S3w94" + - heading: "TensorFlow ML Zero to Hero" + path: "https://www.youtube.com/watch?v=KNAWp2S3w94&list=PLQY2H8rRoyvwWuPiWnuTDBHe7I0fMSsfO" youtube_id: "KNAWp2S3w94?rel=0&show_info=0" - - heading: "TensorFlow 2.0 and Keras" - path: "https://www.youtube.com/watch?v=wGI_VtE9CJM" - youtube_id: "wGI_VtE9CJM?rel=0&show_info=0" - - - classname: devsite-landing-row-cards - items: - - heading: "TensorFlow 2.0 is now available" - path: http://blog.tensorflow.org/tensorflow-2-0-is-now-available-57d706c2a9ab - buttons: - - label: "Read on the TensorFlow blog" - path: http://blog.tensorflow.org/tensorflow-2-0-is-now-available-57d706c2a9ab - - heading: "Standardizing on Keras: Guidance on High-level APIs in TensorFlow 2.0" - path: https://medium.com/tensorflow/standardizing-on-keras-guidance-on-high-level-apis-in-tensorflow-2-0-bad2b04c819a - buttons: - - label: "Read on the TensorFlow blog" - path: https://medium.com/tensorflow/standardizing-on-keras-guidance-on-high-level-apis-in-tensorflow-2-0-bad2b04c819a - - heading: "Contribute to TensorFlow: SIGs, RFCs, Testing, Docs" - path: https://medium.com/tensorflow/contributing-to-tensorflow-sigs-rfcs-testing-and-docs-1c0f8240166c - buttons: - - label: "Read on the TensorFlow blog" - path: https://medium.com/tensorflow/contributing-to-tensorflow-sigs-rfcs-testing-and-docs-1c0f8240166c + - heading: "Basic Computer Vision with ML" + path: "https://www.youtube.com/watch?v=bemDFpNooA8&list=PLQY2H8rRoyvwWuPiWnuTDBHe7I0fMSsfO" + youtube_id: "bemDFpNooA8?rel=0&show_info=0" - classname: devsite-landing-row-100 items: @@ -127,8 +111,8 @@ landing_page: - description: >

    TensorBoard

     path: /tensorboard icon: @@ -138,8 +122,8 @@ landing_page: - description: >

    TensorFlow Hub

     path: /hub icon: @@ -191,12 +175,12 @@ landing_page: foreground: theme background: grey - description: > -

    SIG Addons

     +

    SIG Addons

     - path: https://github.com/tensorflow/addons + path: https://www.tensorflow.org/addons icon: icon_name: chevron_right foreground: theme @@ -226,8 +210,8 @@ landing_page: - description: >

    Probability

     path: /probability icon: @@ -238,18 +222,30 @@ landing_page:

    XLA

     path: /xla icon: icon_name: chevron_right foreground: theme background: grey + - description: > +

    Decision Forests

     + + path: https://github.com/tensorflow/agents + icon: + icon_name: chevron_right + foreground: theme + background: grey + - description: >

    TensorFlow Agents

     path: https://github.com/tensorflow/agents icon: @@ -259,7 +255,7 @@ landing_page: - description: >

    TensorFlow Ranking

     path: https://github.com/tensorflow/ranking @@ -278,3 +274,13 @@ landing_page: icon_name: chevron_right foreground: theme background: grey + + - classname: devsite-landing-row-100 + items: + - description: > +

    TensorFlow updates

    + Subscribe to the + TensorFlow blog, + YouTube channel, + and Twitter + for the latest updates. diff --git a/site/en/tutorials/_toc.yaml b/site/en/tutorials/_toc.yaml index d1dde3b656c..a3907ffe9a4 100644 --- a/site/en/tutorials/_toc.yaml +++ b/site/en/tutorials/_toc.yaml @@ -14,53 +14,50 @@ toc: section: - title: "Basic image classification" path: /tutorials/keras/classification + - title: "Basic text classification" + path: /tutorials/keras/text_classification - title: "Text classification with TF Hub" path: /tutorials/keras/text_classification_with_hub - - title: "Text classification with preprocessed text" - path: /tutorials/keras/text_classification - title: "Regression" path: /tutorials/keras/regression - title: "Overfit and underfit" path: /tutorials/keras/overfit_and_underfit - title: "Save and load" path: /tutorials/keras/save_and_load + - title: "Tune hyperparameters with the Keras Tuner" + path: /tutorials/keras/keras_tuner + - title: "More examples on keras.io" + path: https://keras.io/examples/ + status: external - title: "Load and preprocess data" style: accordion section: + - title: "Images" + path: /tutorials/load_data/images + - title: "Video" + path: /tutorials/load_data/video + status: new - title: "CSV" path: /tutorials/load_data/csv - title: "NumPy" path: /tutorials/load_data/numpy - title: "pandas.DataFrame" path: /tutorials/load_data/pandas_dataframe - - title: "Images" - path: /tutorials/load_data/images - - title: "Text" - path: /tutorials/load_data/text - - title: "Unicode" - path: /tutorials/load_data/unicode - - title: "TF.Text" - path: /tutorials/tensorflow_text/intro - title: "TFRecord and tf.Example" path: /tutorials/load_data/tfrecord - title: "Additional formats with tf.io" path: https://github.com/tensorflow/io#tensorflow-io status: external - -- title: "Estimator" - style: accordion - section: - - title: "Premade estimator" - path: /tutorials/estimator/premade - - title: "Linear model" - path: /tutorials/estimator/linear - - title: "Boosted trees" - path: /tutorials/estimator/boosted_trees - - title: "Boosted trees model understanding" - path: /tutorials/estimator/boosted_trees_model_understanding - - title: "Keras model to Estimator" - path: /tutorials/estimator/keras_model_to_estimator + - title: "Text" + path: /tutorials/load_data/text + - heading: More text loading + - title: "Unicode" + path: /text/guide/unicode + status: external + - title: "Subword Tokenization" + path: /text/guide/subwords_tokenizer + status: external - heading: "Advanced" style: divider @@ -72,78 +69,108 @@ toc: path: /tutorials/customization/basics - title: "Custom layers" path: /tutorials/customization/custom_layers - - title: "Automatic differentiation" - path: /tutorials/customization/autodiff - - title: "Custom training" - path: /tutorials/customization/custom_training - title: "Custom training: walkthrough" path: /tutorials/customization/custom_training_walkthrough - - title: "Performance with tf.function" - path: /tutorials/customization/performance - title: "Distributed training" style: accordion section: - title: "Distributed training with Keras" path: /tutorials/distribute/keras + - title: "Distributed training with DTensors" + path: /tutorials/distribute/dtensor_ml_tutorial + status: experimental + - title: "Using DTensors with Keras" + path: /tutorials/distribute/dtensor_keras_tutorial + status: experimental - title: "Custom training loops" path: /tutorials/distribute/custom_training - title: "Multi-worker training with Keras" path: /tutorials/distribute/multi_worker_with_keras - - title: "Multi-worker training with Estimator" - path: /tutorials/distribute/multi_worker_with_estimator + - title: "Multi-worker training with CTL" + path: /tutorials/distribute/multi_worker_with_ctl + - title: "Parameter Server Training" + path: /tutorials/distribute/parameter_server_training + status: experimental - title: "Save and load" path: /tutorials/distribute/save_and_load + - title: "Distributed input" + path: /tutorials/distribute/input -- title: "Images" +- title: "Vision" style: accordion section: + - title: "Computer vision" + path: /tutorials/images + - title: "KerasCV" + path: https://keras.io/keras_cv/ + status: external - title: "Convolutional Neural Network" path: /tutorials/images/cnn - title: "Image classification" path: /tutorials/images/classification + - title: "Transfer learning and fine-tuning" + path: /tutorials/images/transfer_learning - title: "Transfer learning with TF Hub" path: /tutorials/images/transfer_learning_with_hub - - title: "Transfer learning with pretrained CNN" - path: /tutorials/images/transfer_learning + - title: "Data Augmentation" + path: /tutorials/images/data_augmentation - title: "Image segmentation" path: /tutorials/images/segmentation - title: "Object detection with TF Hub" - path: https://github.com/tensorflow/hub/blob/master/examples/colab/object_detection.ipynb - status: external - - title: "Object detection API" - path: https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb + path: /hub/tutorials/tf2_object_detection status: external + - title: "Video classification" + status: new + path: /tutorials/video/video_classification + - title: "Transfer learning with MoViNet" + status: new + path: /tutorials/video/transfer_learning_with_movinet - title: "Text" style: accordion section: - - title: "Word embeddings" - path: /tutorials/text/word_embeddings - - title: "Text classification with an RNN" - path: /tutorials/text/text_classification_rnn - - title: "Text generation with an RNN" - path: /tutorials/text/text_generation - - title: "Neural machine translation with attention" - path: /tutorials/text/nmt_with_attention - - title: "Image captioning" - path: /tutorials/text/image_captioning - - title: "Transformer model for language understanding" - path: /tutorials/text/transformer + - title: "Text and natural language processing" + path: /tutorials/text/index + - title: "Get started with KerasNLP" + path: https://keras.io/guides/keras_nlp/getting_started/ + status: external + - title: "Text and NLP guide" + path: /text + status: external + +- title: "Audio" + style: accordion + section: + - title: "Simple audio recognition" + path: /tutorials/audio/simple_audio + - title: "Transfer learning for audio recognition" + path: /tutorials/audio/transfer_learning_audio + - title: "Generate music with an RNN" + path: /tutorials/audio/music_generation - title: "Structured data" style: accordion section: - - title: "Classify structured data with feature columns" - path: /tutorials/structured_data/feature_columns + - title: "Classify structured data with preprocessing layers" + path: /tutorials/structured_data/preprocessing_layers - title: "Classification on imbalanced data" path: /tutorials/structured_data/imbalanced_data - title: "Time series forecasting" path: /tutorials/structured_data/time_series + - title: "Decision forest models" + status: external + path: /decision_forests/tutorials/beginner_colab + - title: "Recommenders" + status: external + path: /recommenders/examples/quickstart - title: "Generative" style: accordion section: + - title: "Stable Diffusion" + status: new + path: /tutorials/generative/generate_images_with_stable_diffusion - title: "Neural style transfer" path: /tutorials/generative/style_transfer - title: "DeepDream" @@ -156,5 +183,53 @@ toc: path: /tutorials/generative/cyclegan - title: "Adversarial FGSM" path: /tutorials/generative/adversarial_fgsm + - title: "Intro to Autoencoders" + path: /tutorials/generative/autoencoder - title: "Variational Autoencoder" path: /tutorials/generative/cvae + - title: "Lossy data compression" + path: /tutorials/generative/data_compression + +- title: "Model optimization" + style: accordion + section: + - title: "Scalable model compression with EPR" + path: /tutorials/optimization/compression + - title: "TensorFlow model optimization" + status: external + path: /model_optimization + +- title: "Model Understanding" + style: accordion + section: + - title: "Integrated gradients" + path: /tutorials/interpretability/integrated_gradients + - title: "Uncertainty quantification with SNGP" + path: /tutorials/understanding/sngp + - title: "Probabilistic regression" + path: /probability/examples/Probabilistic_Layers_Regression + status: external + +- title: "Reinforcement learning" + style: accordion + section: + - title: "Actor-Critic method" + path: /tutorials/reinforcement_learning/actor_critic + - title: "TensorFlow agents" + status: external + path: /agents + +- title: "tf.Estimator" + style: accordion + status: deprecated + section: + - title: "Premade estimator" + path: /tutorials/estimator/premade + - title: "Linear model" + path: /tutorials/estimator/linear + - title: "Keras model to Estimator" + path: /tutorials/estimator/keras_model_to_estimator + - title: "Multi-worker training with Estimator" + path: /tutorials/distribute/multi_worker_with_estimator + - title: "Feature columns" + path: /tutorials/structured_data/feature_columns diff --git a/site/en/tutorials/audio/music_generation.ipynb b/site/en/tutorials/audio/music_generation.ipynb new file mode 100644 index 00000000000..e1423ef7cf2 --- /dev/null +++ b/site/en/tutorials/audio/music_generation.ipynb @@ -0,0 +1,1264 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "0DH9bjZD_Cfi" + }, + "source": [ + "##### Copyright 2021 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "JO1GUwC1_T2x" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M4xOsFiu-1-c" + }, + "source": [ + "# Generate music with an RNN" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OyzAxV7Vu_9Y" + }, + "source": [ + "
    \n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hr78EkAY-FFg" + }, + "source": [ + "This tutorial shows you how to generate musical notes using a simple recurrent neural network (RNN). You will train a model using a collection of piano MIDI files from the [MAESTRO dataset](https://magenta.tensorflow.org/datasets/maestro). Given a sequence of notes, your model will learn to predict the next note in the sequence. You can generate longer sequences of notes by calling the model repeatedly.\n", + "\n", + "This tutorial contains complete code to parse and create MIDI files. You can learn more about how RNNs work by visiting the [Text generation with an RNN](https://www.tensorflow.org/text/tutorials/text_generation) tutorial." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4ZniYb7Y_0Ey" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3ks8__E_WUGt" + }, + "source": [ + "This tutorial uses the [`pretty_midi`](https://github.com/craffel/pretty-midi) library to create and parse MIDI files, and [`pyfluidsynth`](https://github.com/nwhitehead/pyfluidsynth) for generating audio playback in Colab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kahm6Z8v_TqC" + }, + "outputs": [], + "source": [ + "!sudo apt install -y fluidsynth" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "M0lAReB7_Vqb" + }, + "outputs": [], + "source": [ + "!pip install --upgrade pyfluidsynth" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "G46kKoQZmIa8" + }, + "outputs": [], + "source": [ + "!pip install pretty_midi" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GsLFq7nsiqcq" + }, + "outputs": [], + "source": [ + "import collections\n", + "import datetime\n", + "import fluidsynth\n", + "import glob\n", + "import numpy as np\n", + "import pathlib\n", + "import pandas as pd\n", + "import pretty_midi\n", + "import seaborn as sns\n", + "import tensorflow as tf\n", + "\n", + "from IPython import display\n", + "from matplotlib import pyplot as plt\n", + "from typing import Optional" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Efja_OtJNzAM" + }, + "outputs": [], + "source": [ + "seed = 42\n", + "tf.random.set_seed(seed)\n", + "np.random.seed(seed)\n", + "\n", + "# Sampling rate for audio playback\n", + "_SAMPLING_RATE = 16000" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FzIbfb-Ikgg7" + }, + "source": [ + "## Download the Maestro dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mwja4SWmibrL" + }, + "outputs": [], + "source": [ + "data_dir = pathlib.Path('data/maestro-v2.0.0')\n", + "if not data_dir.exists():\n", + " tf.keras.utils.get_file(\n", + " 'maestro-v2.0.0-midi.zip',\n", + " origin='https://storage.googleapis.com/magentadata/datasets/maestro/v2.0.0/maestro-v2.0.0-midi.zip',\n", + " extract=True,\n", + " cache_dir='.', cache_subdir='data',\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "k7UYBSxcINqJ" + }, + "source": [ + "The dataset contains about 1,200 MIDI files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "72iFI1bPB9o1" + }, + "outputs": [], + "source": [ + "filenames = glob.glob(str(data_dir/'**/*.mid*'))\n", + "print('Number of files:', len(filenames))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8BlRafYDIRgA" + }, + "source": [ + "## Process a MIDI file" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oFsmG87gXSbh" + }, + "source": [ + "First, use ```pretty_midi``` to parse a single MIDI file and inspect the format of the notes. If you would like to download the MIDI file below to play on your computer, you can do so in colab by writing ```files.download(sample_file)```.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6oSCbHvJNbci" + }, + "outputs": [], + "source": [ + "sample_file = filenames[1]\n", + "print(sample_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A48VdGEpXnLp" + }, + "source": [ + "Generate a `PrettyMIDI` object for the sample MIDI file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1YSQ5DjRI2md" + }, + "outputs": [], + "source": [ + "pm = pretty_midi.PrettyMIDI(sample_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FZNVsZuA_lef" + }, + "source": [ + "Play the sample file. The playback widget may take several seconds to load." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vzoHAaVY_kyY" + }, + "outputs": [], + "source": [ + "def display_audio(pm: pretty_midi.PrettyMIDI, seconds=30):\n", + " waveform = pm.fluidsynth(fs=_SAMPLING_RATE)\n", + " # Take a sample of the generated waveform to mitigate kernel resets\n", + " waveform_short = waveform[:seconds*_SAMPLING_RATE]\n", + " return display.Audio(waveform_short, rate=_SAMPLING_RATE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GOe-3AAi_sRw" + }, + "outputs": [], + "source": [ + "display_audio(pm)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7Lqe7nOsIyh1" + }, + "source": [ + "Do some inspection on the MIDI file. What kinds of instruments are used?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SIGHYQPZQnRo" + }, + "outputs": [], + "source": [ + "print('Number of instruments:', len(pm.instruments))\n", + "instrument = pm.instruments[0]\n", + "instrument_name = pretty_midi.program_to_instrument_name(instrument.program)\n", + "print('Instrument name:', instrument_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KVQfV2hVKB28" + }, + "source": [ + "## Extract notes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nYZm_VehYOTZ" + }, + "outputs": [], + "source": [ + "for i, note in enumerate(instrument.notes[:10]):\n", + " note_name = pretty_midi.note_number_to_name(note.pitch)\n", + " duration = note.end - note.start\n", + " print(f'{i}: pitch={note.pitch}, note_name={note_name},'\n", + " f' duration={duration:.4f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jutzynyqX_GC" + }, + "source": [ + "You will use three variables to represent a note when training the model: `pitch`, `step` and `duration`. The pitch is the perceptual quality of the sound as a MIDI note number. \n", + "The `step` is the time elapsed from the previous note or start of the track.\n", + "The `duration` is how long the note will be playing in seconds and is the difference between the note end and note start times. \n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KGn7Juv_PTi6" + }, + "source": [ + "Extract the notes from the sample MIDI file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Wyp_wdcEPWby" + }, + "outputs": [], + "source": [ + "def midi_to_notes(midi_file: str) -> pd.DataFrame:\n", + " pm = pretty_midi.PrettyMIDI(midi_file)\n", + " instrument = pm.instruments[0]\n", + " notes = collections.defaultdict(list)\n", + "\n", + " # Sort the notes by start time\n", + " sorted_notes = sorted(instrument.notes, key=lambda note: note.start)\n", + " prev_start = sorted_notes[0].start\n", + "\n", + " for note in sorted_notes:\n", + " start = note.start\n", + " end = note.end\n", + " notes['pitch'].append(note.pitch)\n", + " notes['start'].append(start)\n", + " notes['end'].append(end)\n", + " notes['step'].append(start - prev_start)\n", + " notes['duration'].append(end - start)\n", + " prev_start = start\n", + "\n", + " return pd.DataFrame({name: np.array(value) for name, value in notes.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "X0kPjLBlcnY6" + }, + "outputs": [], + "source": [ + "raw_notes = midi_to_notes(sample_file)\n", + "raw_notes.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-71LPvjubOSO" + }, + "source": [ + "It may be easier to interpret the note names rather than the pitches, so you can use the function below to convert from the numeric pitch values to note names. \n", + "The note name shows the type of note, accidental and octave number\n", + "(e.g. C#4). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WE9YXrGZbY2X" + }, + "outputs": [], + "source": [ + "get_note_names = np.vectorize(pretty_midi.note_number_to_name)\n", + "sample_note_names = get_note_names(raw_notes['pitch'])\n", + "sample_note_names[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Q7sjqbp1e_f-" + }, + "source": [ + "To visualize the musical piece, plot the note pitch, start and end across the length of the track (i.e. piano roll). Start with the first 100 notes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "liD2N7x_WOTp" + }, + "outputs": [], + "source": [ + "def plot_piano_roll(notes: pd.DataFrame, count: Optional[int] = None):\n", + " if count:\n", + " title = f'First {count} notes'\n", + " else:\n", + " title = f'Whole track'\n", + " count = len(notes['pitch'])\n", + " plt.figure(figsize=(20, 4))\n", + " plot_pitch = np.stack([notes['pitch'], notes['pitch']], axis=0)\n", + " plot_start_stop = np.stack([notes['start'], notes['end']], axis=0)\n", + " plt.plot(\n", + " plot_start_stop[:, :count], plot_pitch[:, :count], color=\"b\", marker=\".\")\n", + " plt.xlabel('Time [s]')\n", + " plt.ylabel('Pitch')\n", + " _ = plt.title(title)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vWeUbqmAXjOs" + }, + "outputs": [], + "source": [ + "plot_piano_roll(raw_notes, count=100)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gcUyCXYhXeVA" + }, + "source": [ + "Plot the notes for the entire track." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "G7l76hEDZX8Z" + }, + "outputs": [], + "source": [ + "plot_piano_roll(raw_notes)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5GM1bi3aX8rd" + }, + "source": [ + "Check the distribution of each note variable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Pq9C9XBBaK7W" + }, + "outputs": [], + "source": [ + "def plot_distributions(notes: pd.DataFrame, drop_percentile=2.5):\n", + " plt.figure(figsize=[15, 5])\n", + " plt.subplot(1, 3, 1)\n", + " sns.histplot(notes, x=\"pitch\", bins=20)\n", + "\n", + " plt.subplot(1, 3, 2)\n", + " max_step = np.percentile(notes['step'], 100 - drop_percentile)\n", + " sns.histplot(notes, x=\"step\", bins=np.linspace(0, max_step, 21))\n", + " \n", + " plt.subplot(1, 3, 3)\n", + " max_duration = np.percentile(notes['duration'], 100 - drop_percentile)\n", + " sns.histplot(notes, x=\"duration\", bins=np.linspace(0, max_duration, 21))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-Nu2Pw24acFD" + }, + "outputs": [], + "source": [ + "plot_distributions(raw_notes)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "poIivompcfS4" + }, + "source": [ + "## Create a MIDI file\n", + "\n", + "You can generate your own MIDI file from a list of notes using the function below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BD5rsMRARYoV" + }, + "outputs": [], + "source": [ + "def notes_to_midi(\n", + " notes: pd.DataFrame,\n", + " out_file: str, \n", + " instrument_name: str,\n", + " velocity: int = 100, # note loudness\n", + ") -> pretty_midi.PrettyMIDI:\n", + "\n", + " pm = pretty_midi.PrettyMIDI()\n", + " instrument = pretty_midi.Instrument(\n", + " program=pretty_midi.instrument_name_to_program(\n", + " instrument_name))\n", + "\n", + " prev_start = 0\n", + " for i, note in notes.iterrows():\n", + " start = float(prev_start + note['step'])\n", + " end = float(start + note['duration'])\n", + " note = pretty_midi.Note(\n", + " velocity=velocity,\n", + " pitch=int(note['pitch']),\n", + " start=start,\n", + " end=end,\n", + " )\n", + " instrument.notes.append(note)\n", + " prev_start = start\n", + "\n", + " pm.instruments.append(instrument)\n", + " pm.write(out_file)\n", + " return pm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wTazLbuWPIPF" + }, + "outputs": [], + "source": [ + "example_file = 'example.midi'\n", + "example_pm = notes_to_midi(\n", + " raw_notes, out_file=example_file, instrument_name=instrument_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XG0N9zZV_4Gp" + }, + "source": [ + "Play the generated MIDI file and see if there is any difference." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fGRLs-eR_4uK" + }, + "outputs": [], + "source": [ + "display_audio(example_pm)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CLrUscjhBzYc" + }, + "source": [ + "As before, you can write ```files.download(example_file)``` to download and play this file." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pfRNk9tEScuf" + }, + "source": [ + "## Create the training dataset\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "b77zHR1udDrK" + }, + "source": [ + "Create the training dataset by extracting notes from the MIDI files. You can start by using a small number of files, and experiment later with more. This may take a couple minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GiaQiTnXSW-T" + }, + "outputs": [], + "source": [ + "num_files = 5\n", + "all_notes = []\n", + "for f in filenames[:num_files]:\n", + " notes = midi_to_notes(f)\n", + " all_notes.append(notes)\n", + "\n", + "all_notes = pd.concat(all_notes)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "F4bMDeRvgWqx" + }, + "outputs": [], + "source": [ + "n_notes = len(all_notes)\n", + "print('Number of notes parsed:', n_notes)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xIBLvj-cODWS" + }, + "source": [ + "Next, create a `tf.data.Dataset` from the parsed notes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mvNHCHZdXG2P" + }, + "outputs": [], + "source": [ + "key_order = ['pitch', 'step', 'duration']\n", + "train_notes = np.stack([all_notes[key] for key in key_order], axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PLC_19tshyFk" + }, + "outputs": [], + "source": [ + "notes_ds = tf.data.Dataset.from_tensor_slices(train_notes)\n", + "notes_ds.element_spec" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Sj9SXRCjt3I7" + }, + "source": [ + "You will train the model on batches of sequences of notes. Each example will consist of a sequence of notes as the input features, and the next note as the label. In this way, the model will be trained to predict the next note in a sequence. You can find a diagram describing this process (and more details) in [Text classification with an RNN](https://www.tensorflow.org/text/tutorials/text_generation).\n", + "\n", + "You can use the handy [window](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#window) function with size `seq_length` to create the features and labels in this format." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZkEC-5s6wJJV" + }, + "outputs": [], + "source": [ + "def create_sequences(\n", + " dataset: tf.data.Dataset, \n", + " seq_length: int,\n", + " vocab_size = 128,\n", + ") -> tf.data.Dataset:\n", + " \"\"\"Returns TF Dataset of sequence and label examples.\"\"\"\n", + " seq_length = seq_length+1\n", + "\n", + " # Take 1 extra for the labels\n", + " windows = dataset.window(seq_length, shift=1, stride=1,\n", + " drop_remainder=True)\n", + "\n", + " # `flat_map` flattens the\" dataset of datasets\" into a dataset of tensors\n", + " flatten = lambda x: x.batch(seq_length, drop_remainder=True)\n", + " sequences = windows.flat_map(flatten)\n", + " \n", + " # Normalize note pitch\n", + " def scale_pitch(x):\n", + " x = x/[vocab_size,1.0,1.0]\n", + " return x\n", + "\n", + " # Split the labels\n", + " def split_labels(sequences):\n", + " inputs = sequences[:-1]\n", + " labels_dense = sequences[-1]\n", + " labels = {key:labels_dense[i] for i,key in enumerate(key_order)}\n", + "\n", + " return scale_pitch(inputs), labels\n", + "\n", + " return sequences.map(split_labels, num_parallel_calls=tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2xDX5pVkegrv" + }, + "source": [ + "Set the sequence length for each example. Experiment with different lengths (e.g. 50, 100, 150) to see which one works best for the data, or use [hyperparameter tuning](https://www.tensorflow.org/tutorials/keras/keras_tuner). The size of the vocabulary (`vocab_size`) is set to 128 representing all the pitches supported by `pretty_midi`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fGA3VxcFXZ4T" + }, + "outputs": [], + "source": [ + "seq_length = 25\n", + "vocab_size = 128\n", + "seq_ds = create_sequences(notes_ds, seq_length, vocab_size)\n", + "seq_ds.element_spec" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AX9nKmSYetGo" + }, + "source": [ + "The shape of the dataset is ```(100,1)```, meaning that the model will take 100 notes as input, and learn to predict the following note as output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ESK9cL7__TF3" + }, + "outputs": [], + "source": [ + "for seq, target in seq_ds.take(1):\n", + " print('sequence shape:', seq.shape)\n", + " print('sequence elements (first 10):', seq[0: 10])\n", + " print()\n", + " print('target:', target)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kR3TVZZGk5Qq" + }, + "source": [ + "Batch the examples, and configure the dataset for performance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fTpFoiM_AV_Y" + }, + "outputs": [], + "source": [ + "batch_size = 64\n", + "buffer_size = n_notes - seq_length # the number of items in the dataset\n", + "train_ds = (seq_ds\n", + " .shuffle(buffer_size)\n", + " .batch(batch_size, drop_remainder=True)\n", + " .cache()\n", + " .prefetch(tf.data.experimental.AUTOTUNE))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LySbjV0GzXQu" + }, + "outputs": [], + "source": [ + "train_ds.element_spec" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cWZmfkshqP8G" + }, + "source": [ + "## Create and train the model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iGQn32q-hdK2" + }, + "source": [ + "The model will have three outputs, one for each note variable. For `step` and `duration`, you will use a custom loss function based on mean squared error that encourages the model to output non-negative values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "erxLOif08e8v" + }, + "outputs": [], + "source": [ + "def mse_with_positive_pressure(y_true: tf.Tensor, y_pred: tf.Tensor):\n", + " mse = (y_true - y_pred) ** 2\n", + " positive_pressure = 10 * tf.maximum(-y_pred, 0.0)\n", + " return tf.reduce_mean(mse + positive_pressure)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kNaVWcCzAm5V" + }, + "outputs": [], + "source": [ + "input_shape = (seq_length, 3)\n", + "learning_rate = 0.005\n", + "\n", + "inputs = tf.keras.Input(input_shape)\n", + "x = tf.keras.layers.LSTM(128)(inputs)\n", + "\n", + "outputs = {\n", + " 'pitch': tf.keras.layers.Dense(128, name='pitch')(x),\n", + " 'step': tf.keras.layers.Dense(1, name='step')(x),\n", + " 'duration': tf.keras.layers.Dense(1, name='duration')(x),\n", + "}\n", + "\n", + "model = tf.keras.Model(inputs, outputs)\n", + "\n", + "loss = {\n", + " 'pitch': tf.keras.losses.SparseCategoricalCrossentropy(\n", + " from_logits=True),\n", + " 'step': mse_with_positive_pressure,\n", + " 'duration': mse_with_positive_pressure,\n", + "}\n", + "\n", + "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n", + "\n", + "model.compile(loss=loss, optimizer=optimizer)\n", + "\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VDL0Jypt3eU5" + }, + "source": [ + "Testing the `model.evaluate` function, you can see that the `pitch` loss is significantly greater than the `step` and `duration` losses. \n", + "Note that `loss` is the total loss computed by summing all the other losses and is currently dominated by the `pitch` loss." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BlATt7Rl0XJl" + }, + "outputs": [], + "source": [ + "losses = model.evaluate(train_ds, return_dict=True)\n", + "losses" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KLvNLvtR3W59" + }, + "source": [ + "One way balance this is to use the `loss_weights` argument to compile:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9fQB5SiN3ufX" + }, + "outputs": [], + "source": [ + "model.compile(\n", + " loss=loss,\n", + " loss_weights={\n", + " 'pitch': 0.05,\n", + " 'step': 1.0,\n", + " 'duration':1.0,\n", + " },\n", + " optimizer=optimizer,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nPMUnIMelHgR" + }, + "source": [ + "The `loss` then becomes the weighted sum of the individual losses." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "T7CzWmFR38ut" + }, + "outputs": [], + "source": [ + "model.evaluate(train_ds, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SJbn7HZgfosr" + }, + "source": [ + "Train the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uQA_rwKEgPjp" + }, + "outputs": [], + "source": [ + "callbacks = [\n", + " tf.keras.callbacks.ModelCheckpoint(\n", + " filepath='./training_checkpoints/ckpt_{epoch}',\n", + " save_weights_only=True),\n", + " tf.keras.callbacks.EarlyStopping(\n", + " monitor='loss',\n", + " patience=5,\n", + " verbose=1,\n", + " restore_best_weights=True),\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aLoYY8-XaPFN" + }, + "outputs": [], + "source": [ + "%%time\n", + "epochs = 50\n", + "\n", + "history = model.fit(\n", + " train_ds,\n", + " epochs=epochs,\n", + " callbacks=callbacks,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PYBSjgDWiUfT" + }, + "outputs": [], + "source": [ + "plt.plot(history.epoch, history.history['loss'], label='total loss')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aPWI94lQ8uQA" + }, + "source": [ + "## Generate notes" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wbaoiy4Hf-n5" + }, + "source": [ + "To use the model to generate notes, you will first need to provide a starting sequence of notes. The function below generates one note from a sequence of notes. \n", + "\n", + "For note pitch, it draws a sample from the softmax distribution of notes produced by the model, and does not simply pick the note with the highest probability.\n", + "Always picking the note with the highest probability would lead to repetitive sequences of notes being generated.\n", + "\n", + "The `temperature` parameter can be used to control the randomness of notes generated. You can find more details on temperature in [Text generation with an RNN](https://www.tensorflow.org/text/tutorials/text_generation)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1mil8ZyJNe1w" + }, + "outputs": [], + "source": [ + "def predict_next_note(\n", + " notes: np.ndarray, \n", + " model: tf.keras.Model, \n", + " temperature: float = 1.0) -> tuple[int, float, float]:\n", + " \"\"\"Generates a note as a tuple of (pitch, step, duration), using a trained sequence model.\"\"\"\n", + "\n", + " assert temperature > 0\n", + "\n", + " # Add batch dimension\n", + " inputs = tf.expand_dims(notes, 0)\n", + "\n", + " predictions = model.predict(inputs)\n", + " pitch_logits = predictions['pitch']\n", + " step = predictions['step']\n", + " duration = predictions['duration']\n", + " \n", + " pitch_logits /= temperature\n", + " pitch = tf.random.categorical(pitch_logits, num_samples=1)\n", + " pitch = tf.squeeze(pitch, axis=-1)\n", + " duration = tf.squeeze(duration, axis=-1)\n", + " step = tf.squeeze(step, axis=-1)\n", + "\n", + " # `step` and `duration` values should be non-negative\n", + " step = tf.maximum(0, step)\n", + " duration = tf.maximum(0, duration)\n", + "\n", + " return int(pitch), float(step), float(duration)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W64K-EX3hxU_" + }, + "source": [ + "Now generate some notes. You can play around with temperature and the starting sequence in `next_notes` and see what happens." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "87fPl4auPdR3" + }, + "outputs": [], + "source": [ + "temperature = 2.0\n", + "num_predictions = 120\n", + "\n", + "sample_notes = np.stack([raw_notes[key] for key in key_order], axis=1)\n", + "\n", + "# The initial sequence of notes; pitch is normalized similar to training\n", + "# sequences\n", + "input_notes = (\n", + " sample_notes[:seq_length] / np.array([vocab_size, 1, 1]))\n", + "\n", + "generated_notes = []\n", + "prev_start = 0\n", + "for _ in range(num_predictions):\n", + " pitch, step, duration = predict_next_note(input_notes, model, temperature)\n", + " start = prev_start + step\n", + " end = start + duration\n", + " input_note = (pitch, step, duration)\n", + " generated_notes.append((*input_note, start, end))\n", + " input_notes = np.delete(input_notes, 0, axis=0)\n", + " input_notes = np.append(input_notes, np.expand_dims(input_note, 0), axis=0)\n", + " prev_start = start\n", + "\n", + "generated_notes = pd.DataFrame(\n", + " generated_notes, columns=(*key_order, 'start', 'end'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0MK7HmqLuqka" + }, + "outputs": [], + "source": [ + "generated_notes.head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e9K9KHPaTNnK" + }, + "outputs": [], + "source": [ + "out_file = 'output.mid'\n", + "out_pm = notes_to_midi(\n", + " generated_notes, out_file=out_file, instrument_name=instrument_name)\n", + "display_audio(out_pm)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u4N9_Y03Kw-3" + }, + "source": [ + "You can also download the audio file by adding the two lines below:\n", + "\n", + "```\n", + "from google.colab import files\n", + "files.download(out_file)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "trp82gTqskPR" + }, + "source": [ + "Visualize the generated notes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NlNsxcnhvbcK" + }, + "outputs": [], + "source": [ + "plot_piano_roll(generated_notes)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p5_yA9lvvitC" + }, + "source": [ + "Check the distributions of `pitch`, `step` and `duration`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "j5bco2WVRkAa" + }, + "outputs": [], + "source": [ + "plot_distributions(generated_notes)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iAyxR7Itw3Wh" + }, + "source": [ + "In the above plots, you will notice the change in distribution of the note variables.\n", + "Since there is a feedback loop between the model's outputs and inputs, the model tends to generate similar sequences of outputs to reduce the loss. \n", + "This is particularly relevant for `step` and `duration`, which uses the MSE loss.\n", + "For `pitch`, you can increase the randomness by increasing the `temperature` in `predict_next_note`.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bkfe3GYZEu4l" + }, + "source": [ + "## Next steps\n", + "\n", + "This tutorial demonstrated the mechanics of using an RNN to generate sequences of notes from a dataset of MIDI files. To learn more, you can visit the closely related [Text generation with an RNN](https://www.tensorflow.org/text/tutorials/text_generation) tutorial, which contains additional diagrams and explanations. \n", + "\n", + "One of the alternatives to using RNNs for music generation is using GANs. Rather than generating audio, a GAN-based approach can generate an entire sequence in parallel. The Magenta team has done impressive work on this approach with [GANSynth](https://magenta.tensorflow.org/gansynth). You can also find many wonderful music and art projects and open-source code on [Magenta project website](https://magenta.tensorflow.org/)." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "music_generation.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/audio/simple_audio.ipynb b/site/en/tutorials/audio/simple_audio.ipynb new file mode 100644 index 00000000000..9d79742fbb7 --- /dev/null +++ b/site/en/tutorials/audio/simple_audio.ipynb @@ -0,0 +1,979 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "fluF3_oOgkWF" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "AJs7HHFmg1M9" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jYysdyb-CaWM" + }, + "source": [ + "# Simple audio recognition: Recognizing keywords" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CNbqmZy0gbyE" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SPfDNFlb66XF" + }, + "source": [ + "This tutorial demonstrates how to preprocess audio files in the WAV format and build and train a basic [automatic speech recognition](https://en.wikipedia.org/wiki/Speech_recognition) (ASR) model for recognizing ten different words. You will use a portion of the [Speech Commands dataset](https://www.tensorflow.org/datasets/catalog/speech_commands) ([Warden, 2018](https://arxiv.org/abs/1804.03209)), which contains short (one-second or less) audio clips of commands, such as \"down\", \"go\", \"left\", \"no\", \"right\", \"stop\", \"up\" and \"yes\".\n", + "\n", + "Real-world speech and audio recognition [systems](https://ai.googleblog.com/search/label/Speech%20Recognition) are complex. But, like [image classification with the MNIST dataset](../quickstart/beginner.ipynb), this tutorial should give you a basic understanding of the techniques involved." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Go9C3uLL8Izc" + }, + "source": [ + "## Setup\n", + "\n", + "Import necessary modules and dependencies. You'll be using `tf.keras.utils.audio_dataset_from_directory` (introduced in TensorFlow 2.10), which helps generate audio classification datasets from directories of `.wav` files. You'll also need [seaborn](https://seaborn.pydata.org) for visualization in this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hhNW45sjDEDe" + }, + "outputs": [], + "source": [ + "!pip install -U -q tensorflow tensorflow_datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dzLKpmZICaWN" + }, + "outputs": [], + "source": [ + "import os\n", + "import pathlib\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "import tensorflow as tf\n", + "\n", + "from tensorflow.keras import layers\n", + "from tensorflow.keras import models\n", + "from IPython import display\n", + "\n", + "# Set the seed value for experiment reproducibility.\n", + "seed = 42\n", + "tf.random.set_seed(seed)\n", + "np.random.seed(seed)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yR0EdgrLCaWR" + }, + "source": [ + "## Import the mini Speech Commands dataset\n", + "\n", + "To save time with data loading, you will be working with a smaller version of the Speech Commands dataset. The [original dataset](https://www.tensorflow.org/datasets/catalog/speech_commands) consists of over 105,000 audio files in the [WAV (Waveform) audio file format](https://www.aelius.com/njh/wavemetatools/doc/riffmci.pdf) of people saying 35 different words. This data was collected by Google and released under a CC BY license.\n", + "\n", + "Download and extract the `mini_speech_commands.zip` file containing the smaller Speech Commands datasets with `tf.keras.utils.get_file`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2-rayb7-3Y0I" + }, + "outputs": [], + "source": [ + "DATASET_PATH = 'data/mini_speech_commands'\n", + "\n", + "data_dir = pathlib.Path(DATASET_PATH)\n", + "if not data_dir.exists():\n", + " tf.keras.utils.get_file(\n", + " 'mini_speech_commands.zip',\n", + " origin=\"http://storage.googleapis.com/download.tensorflow.org/data/mini_speech_commands.zip\",\n", + " extract=True,\n", + " cache_dir='.', cache_subdir='data')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BgvFq3uYiS5G" + }, + "source": [ + "The dataset's audio clips are stored in eight folders corresponding to each speech command: `no`, `yes`, `down`, `go`, `left`, `up`, `right`, and `stop`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "70IBxSKxA1N9" + }, + "outputs": [], + "source": [ + "commands = np.array(tf.io.gfile.listdir(str(data_dir)))\n", + "commands = commands[(commands != 'README.md') & (commands != '.DS_Store')]\n", + "print('Commands:', commands)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TZ7GJjDvHqtt" + }, + "source": [ + "Divided into directories this way, you can easily load the data using `keras.utils.audio_dataset_from_directory`. \n", + "\n", + "The audio clips are 1 second or less at 16kHz. The `output_sequence_length=16000` pads the short ones to exactly 1 second (and would trim longer ones) so that they can be easily batched." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mFM4c3aMC8Qv" + }, + "outputs": [], + "source": [ + "train_ds, val_ds = tf.keras.utils.audio_dataset_from_directory(\n", + " directory=data_dir,\n", + " batch_size=64,\n", + " validation_split=0.2,\n", + " seed=0,\n", + " output_sequence_length=16000,\n", + " subset='both')\n", + "\n", + "label_names = np.array(train_ds.class_names)\n", + "print()\n", + "print(\"label names:\", label_names)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cestp83qFnU5" + }, + "source": [ + "The dataset now contains batches of audio clips and integer labels. The audio clips have a shape of `(batch, samples, channels)`. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3yU6SQGIFb3H" + }, + "outputs": [], + "source": [ + "train_ds.element_spec" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ppG9Dgq2Ex8R" + }, + "source": [ + "This dataset only contains single channel audio, so use the `tf.squeeze` function to drop the extra axis:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Xl-tnniUIBlM" + }, + "outputs": [], + "source": [ + "def squeeze(audio, labels):\n", + " audio = tf.squeeze(audio, axis=-1)\n", + " return audio, labels\n", + "\n", + "train_ds = train_ds.map(squeeze, tf.data.AUTOTUNE)\n", + "val_ds = val_ds.map(squeeze, tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DtsCSWZN5ILv" + }, + "source": [ + "The `utils.audio_dataset_from_directory` function only returns up to two splits. It's a good idea to keep a test set separate from your validation set.\n", + "Ideally you'd keep it in a separate directory, but in this case you can use `Dataset.shard` to split the validation set into two halves. Note that iterating over **any** shard will load **all** the data, and only keep its fraction. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "u5UEGsqM5Gss" + }, + "outputs": [], + "source": [ + "test_ds = val_ds.shard(num_shards=2, index=0)\n", + "val_ds = val_ds.shard(num_shards=2, index=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xIeoJcwJH5h9" + }, + "outputs": [], + "source": [ + "for example_audio, example_labels in train_ds.take(1): \n", + " print(example_audio.shape)\n", + " print(example_labels.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "voxGEwvuh2L7" + }, + "source": [ + "Let's plot a few audio waveforms:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dYtGq2zYNHuT" + }, + "outputs": [], + "source": [ + "label_names[[1,1,3,0]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8yuX6Nqzf6wT" + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(16, 10))\n", + "rows = 3\n", + "cols = 3\n", + "n = rows * cols\n", + "for i in range(n):\n", + " plt.subplot(rows, cols, i+1)\n", + " audio_signal = example_audio[i]\n", + " plt.plot(audio_signal)\n", + " plt.title(label_names[example_labels[i]])\n", + " plt.yticks(np.arange(-1.2, 1.2, 0.2))\n", + " plt.ylim([-1.1, 1.1])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EWXPphxm0B4m" + }, + "source": [ + "## Convert waveforms to spectrograms\n", + "\n", + "The waveforms in the dataset are represented in the time domain. Next, you'll transform the waveforms from the time-domain signals into the time-frequency-domain signals by computing the [short-time Fourier transform (STFT)](https://en.wikipedia.org/wiki/Short-time_Fourier_transform) to convert the waveforms to as [spectrograms](https://en.wikipedia.org/wiki/Spectrogram), which show frequency changes over time and can be represented as 2D images. You will feed the spectrogram images into your neural network to train the model.\n", + "\n", + "A Fourier transform (`tf.signal.fft`) converts a signal to its component frequencies, but loses all time information. In comparison, STFT (`tf.signal.stft`) splits the signal into windows of time and runs a Fourier transform on each window, preserving some time information, and returning a 2D tensor that you can run standard convolutions on.\n", + "\n", + "Create a utility function for converting waveforms to spectrograms:\n", + "\n", + "- The waveforms need to be of the same length, so that when you convert them to spectrograms, the results have similar dimensions. This can be done by simply zero-padding the audio clips that are shorter than one second (using `tf.zeros`).\n", + "- When calling `tf.signal.stft`, choose the `frame_length` and `frame_step` parameters such that the generated spectrogram \"image\" is almost square. For more information on the STFT parameters choice, refer to [this Coursera video](https://www.coursera.org/lecture/audio-signal-processing/stft-2-tjEQe) on audio signal processing and STFT.\n", + "- The STFT produces an array of complex numbers representing magnitude and phase. However, in this tutorial you'll only use the magnitude, which you can derive by applying `tf.abs` on the output of `tf.signal.stft`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_4CK75DHz_OR" + }, + "outputs": [], + "source": [ + "def get_spectrogram(waveform):\n", + " # Convert the waveform to a spectrogram via a STFT.\n", + " spectrogram = tf.signal.stft(\n", + " waveform, frame_length=255, frame_step=128)\n", + " # Obtain the magnitude of the STFT.\n", + " spectrogram = tf.abs(spectrogram)\n", + " # Add a `channels` dimension, so that the spectrogram can be used\n", + " # as image-like input data with convolution layers (which expect\n", + " # shape (`batch_size`, `height`, `width`, `channels`).\n", + " spectrogram = spectrogram[..., tf.newaxis]\n", + " return spectrogram" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5rdPiPYJphs2" + }, + "source": [ + "Next, start exploring the data. Print the shapes of one example's tensorized waveform and the corresponding spectrogram, and play the original audio:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4Mu6Y7Yz3C-V" + }, + "outputs": [], + "source": [ + "for i in range(3):\n", + " label = label_names[example_labels[i]]\n", + " waveform = example_audio[i]\n", + " spectrogram = get_spectrogram(waveform)\n", + "\n", + " print('Label:', label)\n", + " print('Waveform shape:', waveform.shape)\n", + " print('Spectrogram shape:', spectrogram.shape)\n", + " print('Audio playback')\n", + " display.display(display.Audio(waveform, rate=16000))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xnSuqyxJ1isF" + }, + "source": [ + "Now, define a function for displaying a spectrogram:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e62jzb36-Jog" + }, + "outputs": [], + "source": [ + "def plot_spectrogram(spectrogram, ax):\n", + " if len(spectrogram.shape) > 2:\n", + " assert len(spectrogram.shape) == 3\n", + " spectrogram = np.squeeze(spectrogram, axis=-1)\n", + " # Convert the frequencies to log scale and transpose, so that the time is\n", + " # represented on the x-axis (columns).\n", + " # Add an epsilon to avoid taking a log of zero.\n", + " log_spec = np.log(spectrogram.T + np.finfo(float).eps)\n", + " height = log_spec.shape[0]\n", + " width = log_spec.shape[1]\n", + " X = np.linspace(0, np.size(spectrogram), num=width, dtype=int)\n", + " Y = range(height)\n", + " ax.pcolormesh(X, Y, log_spec)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "baa5c91e8603" + }, + "source": [ + "Plot the example's waveform over time and the corresponding spectrogram (frequencies over time):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "d2_CikgY1tjv" + }, + "outputs": [], + "source": [ + "fig, axes = plt.subplots(2, figsize=(12, 8))\n", + "timescale = np.arange(waveform.shape[0])\n", + "axes[0].plot(timescale, waveform.numpy())\n", + "axes[0].set_title('Waveform')\n", + "axes[0].set_xlim([0, 16000])\n", + "\n", + "plot_spectrogram(spectrogram.numpy(), axes[1])\n", + "axes[1].set_title('Spectrogram')\n", + "plt.suptitle(label.title())\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GyYXjW07jCHA" + }, + "source": [ + "Now, create spectrogram datasets from the audio datasets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mAD0LpkgqtQo" + }, + "outputs": [], + "source": [ + "def make_spec_ds(ds):\n", + " return ds.map(\n", + " map_func=lambda audio,label: (get_spectrogram(audio), label),\n", + " num_parallel_calls=tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yEVb_oK0oBLQ" + }, + "outputs": [], + "source": [ + "train_spectrogram_ds = make_spec_ds(train_ds)\n", + "val_spectrogram_ds = make_spec_ds(val_ds)\n", + "test_spectrogram_ds = make_spec_ds(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6gQpAAgMnyDi" + }, + "source": [ + "Examine the spectrograms for different examples of the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EaM2q5aGis-d" + }, + "outputs": [], + "source": [ + "for example_spectrograms, example_spect_labels in train_spectrogram_ds.take(1):\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QUbHfTuon4iF" + }, + "outputs": [], + "source": [ + "rows = 3\n", + "cols = 3\n", + "n = rows*cols\n", + "fig, axes = plt.subplots(rows, cols, figsize=(16, 9))\n", + "\n", + "for i in range(n):\n", + " r = i // cols\n", + " c = i % cols\n", + " ax = axes[r][c]\n", + " plot_spectrogram(example_spectrograms[i].numpy(), ax)\n", + " ax.set_title(label_names[example_spect_labels[i].numpy()])\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "z5KdY8IF8rkt" + }, + "source": [ + "## Build and train the model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GS1uIh6F_TN9" + }, + "source": [ + "Add `Dataset.cache` and `Dataset.prefetch` operations to reduce read latency while training the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fdZ6M-F5_QzY" + }, + "outputs": [], + "source": [ + "train_spectrogram_ds = train_spectrogram_ds.cache().shuffle(10000).prefetch(tf.data.AUTOTUNE)\n", + "val_spectrogram_ds = val_spectrogram_ds.cache().prefetch(tf.data.AUTOTUNE)\n", + "test_spectrogram_ds = test_spectrogram_ds.cache().prefetch(tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rwHkKCQQb5oW" + }, + "source": [ + "For the model, you'll use a simple convolutional neural network (CNN), since you have transformed the audio files into spectrogram images.\n", + "\n", + "Your `tf.keras.Sequential` model will use the following Keras preprocessing layers:\n", + "\n", + "- `tf.keras.layers.Resizing`: to downsample the input to enable the model to train faster.\n", + "- `tf.keras.layers.Normalization`: to normalize each pixel in the image based on its mean and standard deviation.\n", + "\n", + "For the `Normalization` layer, its `adapt` method would first need to be called on the training data in order to compute aggregate statistics (that is, the mean and the standard deviation)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ALYz7PFCHblP" + }, + "outputs": [], + "source": [ + "input_shape = example_spectrograms.shape[1:]\n", + "print('Input shape:', input_shape)\n", + "num_labels = len(label_names)\n", + "\n", + "# Instantiate the `tf.keras.layers.Normalization` layer.\n", + "norm_layer = layers.Normalization()\n", + "# Fit the state of the layer to the spectrograms\n", + "# with `Normalization.adapt`.\n", + "norm_layer.adapt(data=train_spectrogram_ds.map(map_func=lambda spec, label: spec))\n", + "\n", + "model = models.Sequential([\n", + " layers.Input(shape=input_shape),\n", + " # Downsample the input.\n", + " layers.Resizing(32, 32),\n", + " # Normalize.\n", + " norm_layer,\n", + " layers.Conv2D(32, 3, activation='relu'),\n", + " layers.Conv2D(64, 3, activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Dropout(0.25),\n", + " layers.Flatten(),\n", + " layers.Dense(128, activation='relu'),\n", + " layers.Dropout(0.5),\n", + " layers.Dense(num_labels),\n", + "])\n", + "\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "de52e5afa2f3" + }, + "source": [ + "Configure the Keras model with the Adam optimizer and the cross-entropy loss:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wFjj7-EmsTD-" + }, + "outputs": [], + "source": [ + "model.compile(\n", + " optimizer=tf.keras.optimizers.Adam(),\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['accuracy'],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f42b9e3a4705" + }, + "source": [ + "Train the model over 10 epochs for demonstration purposes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ttioPJVMcGtq" + }, + "outputs": [], + "source": [ + "EPOCHS = 10\n", + "history = model.fit(\n", + " train_spectrogram_ds,\n", + " validation_data=val_spectrogram_ds,\n", + " epochs=EPOCHS,\n", + " callbacks=tf.keras.callbacks.EarlyStopping(verbose=1, patience=2),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gjpCDeQ4mUfS" + }, + "source": [ + "Let's plot the training and validation loss curves to check how your model has improved during training:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nzhipg3Gu2AY" + }, + "outputs": [], + "source": [ + "metrics = history.history\n", + "plt.figure(figsize=(16,6))\n", + "plt.subplot(1,2,1)\n", + "plt.plot(history.epoch, metrics['loss'], metrics['val_loss'])\n", + "plt.legend(['loss', 'val_loss'])\n", + "plt.ylim([0, max(plt.ylim())])\n", + "plt.xlabel('Epoch')\n", + "plt.ylabel('Loss [CrossEntropy]')\n", + "\n", + "plt.subplot(1,2,2)\n", + "plt.plot(history.epoch, 100*np.array(metrics['accuracy']), 100*np.array(metrics['val_accuracy']))\n", + "plt.legend(['accuracy', 'val_accuracy'])\n", + "plt.ylim([0, 100])\n", + "plt.xlabel('Epoch')\n", + "plt.ylabel('Accuracy [%]')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5ZTt3kO3mfm4" + }, + "source": [ + "## Evaluate the model performance\n", + "\n", + "Run the model on the test set and check the model's performance:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FapuRT_SsWGQ" + }, + "outputs": [], + "source": [ + "model.evaluate(test_spectrogram_ds, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "en9Znt1NOabH" + }, + "source": [ + "### Display a confusion matrix\n", + "\n", + "Use a [confusion matrix](https://developers.google.com/machine-learning/glossary#confusion-matrix) to check how well the model did classifying each of the commands in the test set:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5Y6vmWWQuuT1" + }, + "outputs": [], + "source": [ + "y_pred = model.predict(test_spectrogram_ds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "d6F0il82u7lW" + }, + "outputs": [], + "source": [ + "y_pred = tf.argmax(y_pred, axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vHSNoBYLvX81" + }, + "outputs": [], + "source": [ + "y_true = tf.concat(list(test_spectrogram_ds.map(lambda s,lab: lab)), axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LvoSAOiXU3lL" + }, + "outputs": [], + "source": [ + "confusion_mtx = tf.math.confusion_matrix(y_true, y_pred)\n", + "plt.figure(figsize=(10, 8))\n", + "sns.heatmap(confusion_mtx,\n", + " xticklabels=label_names,\n", + " yticklabels=label_names,\n", + " annot=True, fmt='g')\n", + "plt.xlabel('Prediction')\n", + "plt.ylabel('Label')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mQGi_mzPcLvl" + }, + "source": [ + "## Run inference on an audio file\n", + "\n", + "Finally, verify the model's prediction output using an input audio file of someone saying \"no\". How well does your model perform?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zRxauKMdhofU" + }, + "outputs": [], + "source": [ + "x = data_dir/'no/01bb6a2a_nohash_0.wav'\n", + "x = tf.io.read_file(str(x))\n", + "x, sample_rate = tf.audio.decode_wav(x, desired_channels=1, desired_samples=16000,)\n", + "x = tf.squeeze(x, axis=-1)\n", + "waveform = x\n", + "x = get_spectrogram(x)\n", + "x = x[tf.newaxis,...]\n", + "\n", + "prediction = model(x)\n", + "x_labels = ['no', 'yes', 'down', 'go', 'left', 'up', 'right', 'stop']\n", + "plt.bar(x_labels, tf.nn.softmax(prediction[0]))\n", + "plt.title('No')\n", + "plt.show()\n", + "\n", + "display.display(display.Audio(waveform, rate=16000))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VgWICqdqQNaQ" + }, + "source": [ + "As the output suggests, your model should have recognized the audio command as \"no\"." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h1icqlM3ISW0" + }, + "source": [ + "## Export the model with preprocessing" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r7HX-MjgIbji" + }, + "source": [ + "The model's not very easy to use if you have to apply those preprocessing steps before passing data to the model for inference. So build an end-to-end version:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2lIeXdWjIbDE" + }, + "outputs": [], + "source": [ + "class ExportModel(tf.Module):\n", + " def __init__(self, model):\n", + " self.model = model\n", + "\n", + " # Accept either a string-filename or a batch of waveforms.\n", + " # You could add additional signatures for a single wave, or a ragged-batch. \n", + " self.__call__.get_concrete_function(\n", + " x=tf.TensorSpec(shape=(), dtype=tf.string))\n", + " self.__call__.get_concrete_function(\n", + " x=tf.TensorSpec(shape=[None, 16000], dtype=tf.float32))\n", + "\n", + "\n", + " @tf.function\n", + " def __call__(self, x):\n", + " # If they pass a string, load the file and decode it. \n", + " if x.dtype == tf.string:\n", + " x = tf.io.read_file(x)\n", + " x, _ = tf.audio.decode_wav(x, desired_channels=1, desired_samples=16000,)\n", + " x = tf.squeeze(x, axis=-1)\n", + " x = x[tf.newaxis, :]\n", + " \n", + " x = get_spectrogram(x) \n", + " result = self.model(x, training=False)\n", + " \n", + " class_ids = tf.argmax(result, axis=-1)\n", + " class_names = tf.gather(label_names, class_ids)\n", + " return {'predictions':result,\n", + " 'class_ids': class_ids,\n", + " 'class_names': class_names}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gtZBmUiB9HGY" + }, + "source": [ + "Test run the \"export\" model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Z1_8TYaCIRue" + }, + "outputs": [], + "source": [ + "export = ExportModel(model)\n", + "export(tf.constant(str(data_dir/'no/01bb6a2a_nohash_0.wav')))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1J6Iuz829Cxo" + }, + "source": [ + "Save and reload the model, the reloaded model gives identical output:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wTAg4vsn3oEb" + }, + "outputs": [], + "source": [ + "tf.saved_model.save(export, \"saved\")\n", + "imported = tf.saved_model.load(\"saved\")\n", + "imported(waveform[tf.newaxis, :])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J3jF933m9z1J" + }, + "source": [ + "## Next steps\n", + "\n", + "This tutorial demonstrated how to carry out simple audio classification/automatic speech recognition using a convolutional neural network with TensorFlow and Python. To learn more, consider the following resources:\n", + "\n", + "- The [Sound classification with YAMNet](https://www.tensorflow.org/hub/tutorials/yamnet) tutorial shows how to use transfer learning for audio classification.\n", + "- The notebooks from [Kaggle's TensorFlow speech recognition challenge](https://www.kaggle.com/c/tensorflow-speech-recognition-challenge/overview).\n", + "- The \n", + "[TensorFlow.js - Audio recognition using transfer learning codelab](https://codelabs.developers.google.com/codelabs/tensorflowjs-audio-codelab/index.html#0) teaches how to build your own interactive web app for audio classification.\n", + "- [A tutorial on deep learning for music information retrieval](https://arxiv.org/abs/1709.04396) (Choi et al., 2017) on arXiv.\n", + "- TensorFlow also has additional support for [audio data preparation and augmentation](https://www.tensorflow.org/io/tutorials/audio) to help with your own audio-based projects.\n", + "- Consider using the [librosa](https://librosa.org/) library for music and audio analysis." + ] + } + ], + "metadata": { + "accelerator": "CPU", + "colab": { + "collapsed_sections": [], + "name": "simple_audio.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/audio/transfer_learning_audio.ipynb b/site/en/tutorials/audio/transfer_learning_audio.ipynb new file mode 100644 index 00000000000..160aeeb7103 --- /dev/null +++ b/site/en/tutorials/audio/transfer_learning_audio.ipynb @@ -0,0 +1,851 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Cb4espuLKJiA" + }, + "source": [ + "##### Copyright 2021 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "DjZQV2njKJ3U" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mTL0TERThT6z" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View on GitHub\n", + " \n", + " Download notebook\n", + " \n", + " See TF Hub model\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K2madPFAGHb3" + }, + "source": [ + "# Transfer learning with YAMNet for environmental sound classification\n", + "\n", + "[YAMNet](https://tfhub.dev/google/yamnet/1) is a pre-trained deep neural network that can predict audio events from [521 classes](https://github.com/tensorflow/models/blob/master/research/audioset/yamnet/yamnet_class_map.csv), such as laughter, barking, or a siren. \n", + "\n", + " In this tutorial you will learn how to:\n", + "\n", + "- Load and use the YAMNet model for inference.\n", + "- Build a new model using the YAMNet embeddings to classify cat and dog sounds.\n", + "- Evaluate and export your model.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5Mdp2TpBh96Y" + }, + "source": [ + "## Import TensorFlow and other libraries\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zCcKYqu_hvKe" + }, + "source": [ + "Start by installing [TensorFlow I/O](https://www.tensorflow.org/io), which will make it easier for you to load audio files off disk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "urBpRWDHTHHU" + }, + "outputs": [], + "source": [ + "!pip install -q \"tensorflow==2.11.*\"\n", + "# tensorflow_io 0.28 is compatible with TensorFlow 2.11\n", + "!pip install -q \"tensorflow_io==0.28.*\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7l3nqdWVF-kC" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from IPython import display\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "import tensorflow as tf\n", + "import tensorflow_hub as hub\n", + "import tensorflow_io as tfio" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v9ZhybCnt_bM" + }, + "source": [ + "## About YAMNet\n", + "\n", + "[YAMNet](https://github.com/tensorflow/models/tree/master/research/audioset/yamnet) is a pre-trained neural network that employs the [MobileNetV1](https://arxiv.org/abs/1704.04861) depthwise-separable convolution architecture. It can use an audio waveform as input and make independent predictions for each of the 521 audio events from the [AudioSet](http://g.co/audioset) corpus.\n", + "\n", + "Internally, the model extracts \"frames\" from the audio signal and processes batches of these frames. This version of the model uses frames that are 0.96 second long and extracts one frame every 0.48 seconds .\n", + "\n", + "The model accepts a 1-D float32 Tensor or NumPy array containing a waveform of arbitrary length, represented as single-channel (mono) 16 kHz samples in the range `[-1.0, +1.0]`. This tutorial contains code to help you convert WAV files into the supported format.\n", + "\n", + "The model returns 3 outputs, including the class scores, embeddings (which you will use for transfer learning), and the log mel [spectrogram](https://www.tensorflow.org/tutorials/audio/simple_audio#spectrogram). You can find more details [here](https://tfhub.dev/google/yamnet/1).\n", + "\n", + "One specific use of YAMNet is as a high-level feature extractor - the 1,024-dimensional embedding output. You will use the base (YAMNet) model's input features and feed them into your shallower model consisting of one hidden `tf.keras.layers.Dense` layer. Then, you will train the network on a small amount of data for audio classification _without_ requiring a lot of labeled data and training end-to-end. (This is similar to [transfer learning for image classification with TensorFlow Hub](https://www.tensorflow.org/tutorials/images/transfer_learning_with_hub) for more information.)\n", + "\n", + "First, you will test the model and see the results of classifying audio. You will then construct the data pre-processing pipeline.\n", + "\n", + "### Loading YAMNet from TensorFlow Hub\n", + "\n", + "You are going to use a pre-trained YAMNet from [Tensorflow Hub](https://tfhub.dev/) to extract the embeddings from the sound files.\n", + "\n", + "Loading a model from TensorFlow Hub is straightforward: choose the model, copy its URL, and use the `load` function.\n", + "\n", + "Note: to read the documentation of the model, use the model URL in your browser." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "06CWkBV5v3gr" + }, + "outputs": [], + "source": [ + "yamnet_model_handle = 'https://tfhub.dev/google/yamnet/1'\n", + "yamnet_model = hub.load(yamnet_model_handle)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GmrPJ0GHw9rr" + }, + "source": [ + "With the model loaded, you can follow the [YAMNet basic usage tutorial](https://www.tensorflow.org/hub/tutorials/yamnet) and download a sample WAV file to run the inference.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "C5i6xktEq00P" + }, + "outputs": [], + "source": [ + "testing_wav_file_name = tf.keras.utils.get_file('miaow_16k.wav',\n", + " 'https://storage.googleapis.com/audioset/miaow_16k.wav',\n", + " cache_dir='./',\n", + " cache_subdir='test_data')\n", + "\n", + "print(testing_wav_file_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mBm9y9iV2U_-" + }, + "source": [ + "You will need a function to load audio files, which will also be used later when working with the training data. (Learn more about reading audio files and their labels in [Simple audio recognition](https://www.tensorflow.org/tutorials/audio/simple_audio#reading_audio_files_and_their_labels).\n", + "\n", + "Note: The returned `wav_data` from `load_wav_16k_mono` is already normalized to values in the `[-1.0, 1.0]` range (for more information, go to [YAMNet's documentation on TF Hub](https://tfhub.dev/google/yamnet/1))." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Xwc9Wrdg2EtY" + }, + "outputs": [], + "source": [ + "# Utility functions for loading audio files and making sure the sample rate is correct.\n", + "\n", + "@tf.function\n", + "def load_wav_16k_mono(filename):\n", + " \"\"\" Load a WAV file, convert it to a float tensor, resample to 16 kHz single-channel audio. \"\"\"\n", + " file_contents = tf.io.read_file(filename)\n", + " wav, sample_rate = tf.audio.decode_wav(\n", + " file_contents,\n", + " desired_channels=1)\n", + " wav = tf.squeeze(wav, axis=-1)\n", + " sample_rate = tf.cast(sample_rate, dtype=tf.int64)\n", + " wav = tfio.audio.resample(wav, rate_in=sample_rate, rate_out=16000)\n", + " return wav" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FRqpjkwB0Jjw" + }, + "outputs": [], + "source": [ + "testing_wav_data = load_wav_16k_mono(testing_wav_file_name)\n", + "\n", + "_ = plt.plot(testing_wav_data)\n", + "\n", + "# Play the audio file.\n", + "display.Audio(testing_wav_data, rate=16000)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6z6rqlEz20YB" + }, + "source": [ + "### Load the class mapping\n", + "\n", + "It's important to load the class names that YAMNet is able to recognize. The mapping file is present at `yamnet_model.class_map_path()` in the CSV format." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6Gyj23e_3Mgr" + }, + "outputs": [], + "source": [ + "class_map_path = yamnet_model.class_map_path().numpy().decode('utf-8')\n", + "class_names =list(pd.read_csv(class_map_path)['display_name'])\n", + "\n", + "for name in class_names[:20]:\n", + " print(name)\n", + "print('...')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5xbycDnT40u0" + }, + "source": [ + "### Run inference\n", + "\n", + "YAMNet provides frame-level class-scores (i.e., 521 scores for every frame). In order to determine clip-level predictions, the scores can be aggregated per-class across frames (e.g., using mean or max aggregation). This is done below by `scores_np.mean(axis=0)`. Finally, to find the top-scored class at the clip-level, you take the maximum of the 521 aggregated scores.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NT0otp-A4Y3u" + }, + "outputs": [], + "source": [ + "scores, embeddings, spectrogram = yamnet_model(testing_wav_data)\n", + "class_scores = tf.reduce_mean(scores, axis=0)\n", + "top_class = tf.math.argmax(class_scores)\n", + "inferred_class = class_names[top_class]\n", + "\n", + "print(f'The main sound is: {inferred_class}')\n", + "print(f'The embeddings shape: {embeddings.shape}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YBaLNg5H5IWa" + }, + "source": [ + "Note: The model correctly inferred an animal sound. Your goal in this tutorial is to increase the model's accuracy for specific classes. Also, notice that the model generated 13 embeddings, 1 per frame." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fmthELBg1A2-" + }, + "source": [ + "## ESC-50 dataset\n", + "\n", + "The [ESC-50 dataset](https://github.com/karolpiczak/ESC-50#repository-content) ([Piczak, 2015](https://www.karolpiczak.com/papers/Piczak2015-ESC-Dataset.pdf)) is a labeled collection of 2,000 five-second long environmental audio recordings. The dataset consists of 50 classes, with 40 examples per class.\n", + "\n", + "Download the dataset and extract it. \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MWobqK8JmZOU" + }, + "outputs": [], + "source": [ + "_ = tf.keras.utils.get_file('esc-50.zip',\n", + " 'https://github.com/karoldvl/ESC-50/archive/master.zip',\n", + " cache_dir='./',\n", + " cache_subdir='datasets',\n", + " extract=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qcruxiuX1cO5" + }, + "source": [ + "### Explore the data\n", + "\n", + "The metadata for each file is specified in the csv file at `./datasets/ESC-50-master/meta/esc50.csv`\n", + "\n", + "and all the audio files are in `./datasets/ESC-50-master/audio/`\n", + "\n", + "You will create a pandas `DataFrame` with the mapping and use that to have a clearer view of the data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jwmLygPrMAbH" + }, + "outputs": [], + "source": [ + "esc50_csv = './datasets/ESC-50-master/meta/esc50.csv'\n", + "base_data_path = './datasets/ESC-50-master/audio/'\n", + "\n", + "pd_data = pd.read_csv(esc50_csv)\n", + "pd_data.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7d4rHBEQ2QAU" + }, + "source": [ + "### Filter the data\n", + "\n", + "Now that the data is stored in the `DataFrame`, apply some transformations:\n", + "\n", + "- Filter out rows and use only the selected classes - `dog` and `cat`. If you want to use any other classes, this is where you can choose them.\n", + "- Amend the filename to have the full path. This will make loading easier later.\n", + "- Change targets to be within a specific range. In this example, `dog` will remain at `0`, but `cat` will become `1` instead of its original value of `5`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tFnEoQjgs14I" + }, + "outputs": [], + "source": [ + "my_classes = ['dog', 'cat']\n", + "map_class_to_id = {'dog':0, 'cat':1}\n", + "\n", + "filtered_pd = pd_data[pd_data.category.isin(my_classes)]\n", + "\n", + "class_id = filtered_pd['category'].apply(lambda name: map_class_to_id[name])\n", + "filtered_pd = filtered_pd.assign(target=class_id)\n", + "\n", + "full_path = filtered_pd['filename'].apply(lambda row: os.path.join(base_data_path, row))\n", + "filtered_pd = filtered_pd.assign(filename=full_path)\n", + "\n", + "filtered_pd.head(10)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BkDcBS-aJdCz" + }, + "source": [ + "### Load the audio files and retrieve embeddings\n", + "\n", + "Here you'll apply the `load_wav_16k_mono` and prepare the WAV data for the model.\n", + "\n", + "When extracting embeddings from the WAV data, you get an array of shape `(N, 1024)` where `N` is the number of frames that YAMNet found (one for every 0.48 seconds of audio)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AKDT5RomaDKO" + }, + "source": [ + "Your model will use each frame as one input. Therefore, you need to create a new column that has one frame per row. You also need to expand the labels and the `fold` column to proper reflect these new rows.\n", + "\n", + "The expanded `fold` column keeps the original values. You cannot mix frames because, when performing the splits, you might end up having parts of the same audio on different splits, which would make your validation and test steps less effective." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "u5Rq3_PyKLtU" + }, + "outputs": [], + "source": [ + "filenames = filtered_pd['filename']\n", + "targets = filtered_pd['target']\n", + "folds = filtered_pd['fold']\n", + "\n", + "main_ds = tf.data.Dataset.from_tensor_slices((filenames, targets, folds))\n", + "main_ds.element_spec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rsEfovDVAHGY" + }, + "outputs": [], + "source": [ + "def load_wav_for_map(filename, label, fold):\n", + " return load_wav_16k_mono(filename), label, fold\n", + "\n", + "main_ds = main_ds.map(load_wav_for_map)\n", + "main_ds.element_spec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "k0tG8DBNAHcE" + }, + "outputs": [], + "source": [ + "# applies the embedding extraction model to a wav data\n", + "def extract_embedding(wav_data, label, fold):\n", + " ''' run YAMNet to extract embedding from the wav data '''\n", + " scores, embeddings, spectrogram = yamnet_model(wav_data)\n", + " num_embeddings = tf.shape(embeddings)[0]\n", + " return (embeddings,\n", + " tf.repeat(label, num_embeddings),\n", + " tf.repeat(fold, num_embeddings))\n", + "\n", + "# extract embedding\n", + "main_ds = main_ds.map(extract_embedding).unbatch()\n", + "main_ds.element_spec" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZdfPIeD0Qedk" + }, + "source": [ + "### Split the data\n", + "\n", + "You will use the `fold` column to split the dataset into train, validation and test sets.\n", + "\n", + "ESC-50 is arranged into five uniformly-sized cross-validation `fold`s, such that clips from the same original source are always in the same `fold` - find out more in the [ESC: Dataset for Environmental Sound Classification](https://www.karolpiczak.com/papers/Piczak2015-ESC-Dataset.pdf) paper.\n", + "\n", + "The last step is to remove the `fold` column from the dataset since you're not going to use it during training.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1ZYvlFiVsffC" + }, + "outputs": [], + "source": [ + "cached_ds = main_ds.cache()\n", + "train_ds = cached_ds.filter(lambda embedding, label, fold: fold < 4)\n", + "val_ds = cached_ds.filter(lambda embedding, label, fold: fold == 4)\n", + "test_ds = cached_ds.filter(lambda embedding, label, fold: fold == 5)\n", + "\n", + "# remove the folds column now that it's not needed anymore\n", + "remove_fold_column = lambda embedding, label, fold: (embedding, label)\n", + "\n", + "train_ds = train_ds.map(remove_fold_column)\n", + "val_ds = val_ds.map(remove_fold_column)\n", + "test_ds = test_ds.map(remove_fold_column)\n", + "\n", + "train_ds = train_ds.cache().shuffle(1000).batch(32).prefetch(tf.data.AUTOTUNE)\n", + "val_ds = val_ds.cache().batch(32).prefetch(tf.data.AUTOTUNE)\n", + "test_ds = test_ds.cache().batch(32).prefetch(tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v5PaMwvtcAIe" + }, + "source": [ + "## Create your model\n", + "\n", + "You did most of the work!\n", + "Next, define a very simple [Sequential](https://www.tensorflow.org/guide/keras/sequential_model) model with one hidden layer and two outputs to recognize cats and dogs from sounds.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JYCE0Fr1GpN3" + }, + "outputs": [], + "source": [ + "my_model = tf.keras.Sequential([\n", + " tf.keras.layers.Input(shape=(1024), dtype=tf.float32,\n", + " name='input_embedding'),\n", + " tf.keras.layers.Dense(512, activation='relu'),\n", + " tf.keras.layers.Dense(len(my_classes))\n", + "], name='my_model')\n", + "\n", + "my_model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "l1qgH35HY0SE" + }, + "outputs": [], + "source": [ + "my_model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " optimizer=\"adam\",\n", + " metrics=['accuracy'])\n", + "\n", + "callback = tf.keras.callbacks.EarlyStopping(monitor='loss',\n", + " patience=3,\n", + " restore_best_weights=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "T3sj84eOZ3pk" + }, + "outputs": [], + "source": [ + "history = my_model.fit(train_ds,\n", + " epochs=20,\n", + " validation_data=val_ds,\n", + " callbacks=callback)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OAbraYKYpdoE" + }, + "source": [ + "Let's run the `evaluate` method on the test data just to be sure there's no overfitting." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "H4Nh5nec3Sky" + }, + "outputs": [], + "source": [ + "loss, accuracy = my_model.evaluate(test_ds)\n", + "\n", + "print(\"Loss: \", loss)\n", + "print(\"Accuracy: \", accuracy)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cid-qIrIpqHS" + }, + "source": [ + "You did it!" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nCKZonrJcXab" + }, + "source": [ + "## Test your model\n", + "\n", + "Next, try your model on the embedding from the previous test using YAMNet only.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "79AFpA3_ctCF" + }, + "outputs": [], + "source": [ + "scores, embeddings, spectrogram = yamnet_model(testing_wav_data)\n", + "result = my_model(embeddings).numpy()\n", + "\n", + "inferred_class = my_classes[result.mean(axis=0).argmax()]\n", + "print(f'The main sound is: {inferred_class}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "k2yleeev645r" + }, + "source": [ + "## Save a model that can directly take a WAV file as input\n", + "\n", + "Your model works when you give it the embeddings as input.\n", + "\n", + "In a real-world scenario, you'll want to use audio data as a direct input.\n", + "\n", + "To do that, you will combine YAMNet with your model into a single model that you can export for other applications.\n", + "\n", + "To make it easier to use the model's result, the final layer will be a `reduce_mean` operation. When using this model for serving (which you will learn about later in the tutorial), you will need the name of the final layer. If you don't define one, TensorFlow will auto-define an incremental one that makes it hard to test, as it will keep changing every time you train the model. When using a raw TensorFlow operation, you can't assign a name to it. To address this issue, you'll create a custom layer that applies `reduce_mean` and call it `'classifier'`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QUVCI2Suunpw" + }, + "outputs": [], + "source": [ + "class ReduceMeanLayer(tf.keras.layers.Layer):\n", + " def __init__(self, axis=0, **kwargs):\n", + " super(ReduceMeanLayer, self).__init__(**kwargs)\n", + " self.axis = axis\n", + "\n", + " def call(self, input):\n", + " return tf.math.reduce_mean(input, axis=self.axis)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zE_Npm0nzlwc" + }, + "outputs": [], + "source": [ + "saved_model_path = './dogs_and_cats_yamnet'\n", + "\n", + "input_segment = tf.keras.layers.Input(shape=(), dtype=tf.float32, name='audio')\n", + "embedding_extraction_layer = hub.KerasLayer(yamnet_model_handle,\n", + " trainable=False, name='yamnet')\n", + "_, embeddings_output, _ = embedding_extraction_layer(input_segment)\n", + "serving_outputs = my_model(embeddings_output)\n", + "serving_outputs = ReduceMeanLayer(axis=0, name='classifier')(serving_outputs)\n", + "serving_model = tf.keras.Model(input_segment, serving_outputs)\n", + "serving_model.save(saved_model_path, include_optimizer=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "y-0bY5FMme1C" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(serving_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "btHQDN9mqxM_" + }, + "source": [ + "Load your saved model to verify that it works as expected." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KkYVpJS72WWB" + }, + "outputs": [], + "source": [ + "reloaded_model = tf.saved_model.load(saved_model_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4BkmvvNzq49l" + }, + "source": [ + "And for the final test: given some sound data, does your model return the correct result?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xeXtD5HO28y-" + }, + "outputs": [], + "source": [ + "reloaded_results = reloaded_model(testing_wav_data)\n", + "cat_or_dog = my_classes[tf.math.argmax(reloaded_results)]\n", + "print(f'The main sound is: {cat_or_dog}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZRrOcBYTUgwn" + }, + "source": [ + "If you want to try your new model on a serving setup, you can use the 'serving_default' signature." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ycC8zzDSUG2s" + }, + "outputs": [], + "source": [ + "serving_results = reloaded_model.signatures['serving_default'](testing_wav_data)\n", + "cat_or_dog = my_classes[tf.math.argmax(serving_results['classifier'])]\n", + "print(f'The main sound is: {cat_or_dog}')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "da7blblCHs8c" + }, + "source": [ + "## (Optional) Some more testing\n", + "\n", + "The model is ready.\n", + "\n", + "Let's compare it to YAMNet on the test dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vDf5MASIIN1z" + }, + "outputs": [], + "source": [ + "test_pd = filtered_pd.loc[filtered_pd['fold'] == 5]\n", + "row = test_pd.sample(1)\n", + "filename = row['filename'].item()\n", + "print(filename)\n", + "waveform = load_wav_16k_mono(filename)\n", + "print(f'Waveform values: {waveform}')\n", + "_ = plt.plot(waveform)\n", + "\n", + "display.Audio(waveform, rate=16000)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eYUzFxYJIcE1" + }, + "outputs": [], + "source": [ + "# Run the model, check the output.\n", + "scores, embeddings, spectrogram = yamnet_model(waveform)\n", + "class_scores = tf.reduce_mean(scores, axis=0)\n", + "top_class = tf.math.argmax(class_scores)\n", + "inferred_class = class_names[top_class]\n", + "top_score = class_scores[top_class]\n", + "print(f'[YAMNet] The main sound is: {inferred_class} ({top_score})')\n", + "\n", + "reloaded_results = reloaded_model(waveform)\n", + "your_top_class = tf.math.argmax(reloaded_results)\n", + "your_inferred_class = my_classes[your_top_class]\n", + "class_probabilities = tf.nn.softmax(reloaded_results, axis=-1)\n", + "your_top_score = class_probabilities[your_top_class]\n", + "print(f'[Your model] The main sound is: {your_inferred_class} ({your_top_score})')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "g8Tsym8Rq-0V" + }, + "source": [ + "## Next steps\n", + "\n", + "You have created a model that can classify sounds from dogs or cats. With the same idea and a different dataset you can try, for example, building an [acoustic identifier of birds](https://www.kaggle.com/c/birdclef-2021/) based on their singing.\n", + "\n", + "Share your project with the TensorFlow team on social media!\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "transfer_learning_audio.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/customization/autodiff.ipynb b/site/en/tutorials/customization/autodiff.ipynb deleted file mode 100644 index 1ad3fcbee14..00000000000 --- a/site/en/tutorials/customization/autodiff.ipynb +++ /dev/null @@ -1,332 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "GCCk8_dHpuNf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xh8WkEwWpnm7" - }, - "source": [ - "# Automatic differentiation and gradient tape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idv0bPeCp325" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/autodiff\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/customization/autodiff.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/customization/autodiff.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/customization/autodiff.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJ4XzMqodTy" - }, - "source": [ - "In the previous tutorial we introduced `Tensor`s and operations on them. In this tutorial we will cover [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation), a key technique for optimizing machine learning models." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GQJysDM__Qb0" - }, - "source": [ - "## Setup\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cxzaxo6ff2y3" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1CLWJl0QliB0" - }, - "source": [ - "## Gradient tapes\n", - "\n", - "TensorFlow provides the [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) API for automatic differentiation - computing the gradient of a computation with respect to its input variables. Tensorflow \"records\" all operations executed inside the context of a `tf.GradientTape` onto a \"tape\". Tensorflow then uses that tape and the gradients associated with each recorded operation to compute the gradients of a \"recorded\" computation using [reverse mode differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation).\n", - "\n", - "For example:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bAFeIE8EuVIq" - }, - "outputs": [], - "source": [ - "x = tf.ones((2, 2))\n", - "\n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# Derivative of z with respect to the original input tensor x\n", - "dz_dx = t.gradient(z, x)\n", - "for i in [0, 1]:\n", - " for j in [0, 1]:\n", - " assert dz_dx[i][j].numpy() == 8.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N4VlqKFzzGaC" - }, - "source": [ - "You can also request gradients of the output with respect to intermediate values computed during a \"recorded\" `tf.GradientTape` context." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7XaPRAwUyYms" - }, - "outputs": [], - "source": [ - "x = tf.ones((2, 2))\n", - "\n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# Use the tape to compute the derivative of z with respect to the\n", - "# intermediate value y.\n", - "dz_dy = t.gradient(z, y)\n", - "assert dz_dy.numpy() == 8.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISkXuY7YzIcS" - }, - "source": [ - "By default, the resources held by a GradientTape are released as soon as GradientTape.gradient() method is called. To compute multiple gradients over the same computation, create a `persistent` gradient tape. This allows multiple calls to the `gradient()` method as resources are released when the tape object is garbage collected. For example:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zZaCm3-9zVCi" - }, - "outputs": [], - "source": [ - "x = tf.constant(3.0)\n", - "with tf.GradientTape(persistent=True) as t:\n", - " t.watch(x)\n", - " y = x * x\n", - " z = y * y\n", - "dz_dx = t.gradient(z, x) # 108.0 (4*x^3 at x = 3)\n", - "dy_dx = t.gradient(y, x) # 6.0\n", - "del t # Drop the reference to the tape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6kADybtQzYj4" - }, - "source": [ - "### Recording control flow\n", - "\n", - "Because tapes record operations as they are executed, Python control flow (using `if`s and `while`s for example) is naturally handled:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9FViq92UX7P8" - }, - "outputs": [], - "source": [ - "def f(x, y):\n", - " output = 1.0\n", - " for i in range(y):\n", - " if i \u003e 1 and i \u003c 5:\n", - " output = tf.multiply(output, x)\n", - " return output\n", - "\n", - "def grad(x, y):\n", - " with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " out = f(x, y)\n", - " return t.gradient(out, x)\n", - "\n", - "x = tf.convert_to_tensor(2.0)\n", - "\n", - "assert grad(x, 6).numpy() == 12.0\n", - "assert grad(x, 5).numpy() == 12.0\n", - "assert grad(x, 4).numpy() == 4.0\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DK05KXrAAld3" - }, - "source": [ - "### Higher-order gradients\n", - "\n", - "Operations inside of the `GradientTape` context manager are recorded for automatic differentiation. If gradients are computed in that context, then the gradient computation is recorded as well. As a result, the exact same API works for higher-order gradients as well. For example:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cPQgthZ7ugRJ" - }, - "outputs": [], - "source": [ - "x = tf.Variable(1.0) # Create a Tensorflow variable initialized to 1.0\n", - "\n", - "with tf.GradientTape() as t:\n", - " with tf.GradientTape() as t2:\n", - " y = x * x * x\n", - " # Compute the gradient inside the 't' context manager\n", - " # which means the gradient computation is differentiable as well.\n", - " dy_dx = t2.gradient(y, x)\n", - "d2y_dx2 = t.gradient(dy_dx, x)\n", - "\n", - "assert dy_dx.numpy() == 3.0\n", - "assert d2y_dx2.numpy() == 6.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4U1KKzUpNl58" - }, - "source": [ - "## Next Steps\n", - "\n", - "In this tutorial we covered gradient computation in TensorFlow. With that we have enough of the primitives required to build and train neural networks." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "j8BT7T8yf6Rv" - }, - "outputs": [], - "source": [ - "" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "autodiff.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/customization/basics.ipynb b/site/en/tutorials/customization/basics.ipynb index d5ccd1811aa..2df0840ad5e 100644 --- a/site/en/tutorials/customization/basics.ipynb +++ b/site/en/tutorials/customization/basics.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iPpI7RaYoZuE" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "hro2InpHobKk" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "U9i2Dsh-ziXr" }, "source": [ @@ -47,78 +43,54 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Hndw-YcxoOJK" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/basics\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/customization/basics.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/customization/basics.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/customization/basics.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6sILUVbHoSgH" }, "source": [ - "This is an introductory TensorFlow tutorial shows how to:\n", + "This is an introductory TensorFlow tutorial that shows how to:\n", "\n", - "* Import the required package\n", - "* Create and use tensors\n", - "* Use GPU acceleration\n", - "* Demonstrate `tf.data.Dataset`" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "miTaGiqV9RjO" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" + "* Import the required package.\n", + "* Create and use tensors.\n", + "* Use GPU acceleration.\n", + "* Build a data pipeline with `tf.data.Dataset`." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "z1JcS5iBXMRO" }, "source": [ "## Import TensorFlow\n", "\n", - "To get started, import the `tensorflow` module. As of TensorFlow 2.0, eager execution is turned on by default. This enables a more interactive frontend to TensorFlow, the details of which we will discuss much later." + "To get started, import the `tensorflow` module. As of TensorFlow 2, eager execution is turned on by default. Eager execution enables a more interactive frontend to TensorFlow, which you will later explore in more detail." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "vjBPmYjLdFmk" }, "outputs": [], @@ -129,39 +101,35 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "H9UySOPLXdaw" }, "source": [ "## Tensors\n", "\n", - "A Tensor is a multi-dimensional array. Similar to NumPy `ndarray` objects, `tf.Tensor` objects have a data type and a shape. Additionally, `tf.Tensor`s can reside in accelerator memory (like a GPU). TensorFlow offers a rich library of operations ([tf.add](https://www.tensorflow.org/api_docs/python/tf/add), [tf.matmul](https://www.tensorflow.org/api_docs/python/tf/matmul), [tf.linalg.inv](https://www.tensorflow.org/api_docs/python/tf/linalg/inv) etc.) that consume and produce `tf.Tensor`s. These operations automatically convert native Python types, for example:\n" + "A Tensor is a multi-dimensional array. Similar to NumPy `ndarray` objects, `tf.Tensor` objects have a data type and a shape. Additionally, `tf.Tensor`s can reside in accelerator memory (like a GPU). TensorFlow offers a rich library of operations (for example, `tf.math.add`, `tf.linalg.matmul`, and `tf.linalg.inv`) that consume and produce `tf.Tensor`s. These operations automatically convert built-in Python types. For example:\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "code", - "colab": {}, - "colab_type": "code", "id": "ngUe237Wt48W" }, "outputs": [], "source": [ - "print(tf.add(1, 2))\n", - "print(tf.add([1, 2], [3, 4]))\n", - "print(tf.square(5))\n", - "print(tf.reduce_sum([1, 2, 3]))\n", + "print(tf.math.add(1, 2))\n", + "print(tf.math.add([1, 2], [3, 4]))\n", + "print(tf.math.square(5))\n", + "print(tf.math.reduce_sum([1, 2, 3]))\n", "\n", "# Operator overloading is also supported\n", - "print(tf.square(2) + tf.square(3))" + "print(tf.math.square(2) + tf.math.square(3))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IDY4WsYRhP81" }, "source": [ @@ -170,15 +138,13 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "srYWH1MdJNG7" }, "outputs": [], "source": [ - "x = tf.matmul([[1]], [[2, 3]])\n", + "x = tf.linalg.matmul([[1]], [[2, 3]])\n", "print(x)\n", "print(x.shape)\n", "print(x.dtype)" @@ -187,7 +153,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "eBPw8e8vrsom" }, "source": [ @@ -200,13 +165,12 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Dwi1tdW3JBw6" }, "source": [ - "### NumPy Compatibility\n", + "### NumPy compatibility\n", "\n", - "Converting between a TensorFlow `tf.Tensor`s and a NumPy `ndarray` is easy:\n", + "Converting between a TensorFlow `tf.Tensor` and a NumPy `ndarray` is easy:\n", "\n", "* TensorFlow operations automatically convert NumPy ndarrays to Tensors.\n", "* NumPy operations automatically convert Tensors to NumPy ndarrays.\n", @@ -216,10 +180,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "lCUWzso6mbqR" }, "outputs": [], @@ -229,11 +191,11 @@ "ndarray = np.ones([3, 3])\n", "\n", "print(\"TensorFlow operations convert numpy arrays to Tensors automatically\")\n", - "tensor = tf.multiply(ndarray, 42)\n", + "tensor = tf.math.multiply(ndarray, 42)\n", "print(tensor)\n", "\n", "\n", - "print(\"And NumPy operations convert Tensors to numpy arrays automatically\")\n", + "print(\"And NumPy operations convert Tensors to NumPy arrays automatically\")\n", "print(np.add(tensor, 1))\n", "\n", "print(\"The .numpy() method explicitly converts a Tensor to a numpy array\")\n", @@ -243,22 +205,19 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PBNP8yTRfu_X" }, "source": [ "## GPU acceleration\n", "\n", - "Many TensorFlow operations are accelerated using the GPU for computation. Without any annotations, TensorFlow automatically decides whether to use the GPU or CPU for an operation—copying the tensor between CPU and GPU memory, if necessary. Tensors produced by an operation are typically backed by the memory of the device on which the operation executed, for example:" + "Many TensorFlow operations are accelerated using the GPU for computation. Without any annotations, TensorFlow automatically decides whether to use the GPU or CPU for an operation—copying the tensor between CPU and GPU memory, if necessary. Tensors produced by an operation are typically backed by the memory of the device on which the operation executed. For example:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "code", - "colab": {}, - "colab_type": "code", "id": "3Twf_Rw-gQFM" }, "outputs": [], @@ -266,7 +225,7 @@ "x = tf.random.uniform([3, 3])\n", "\n", "print(\"Is there a GPU available: \"),\n", - "print(tf.config.experimental.list_physical_devices(\"GPU\"))\n", + "print(tf.config.list_physical_devices(\"GPU\"))\n", "\n", "print(\"Is the Tensor on GPU #0: \"),\n", "print(x.device.endswith('GPU:0'))" @@ -275,35 +234,31 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vpgYzgVXW2Ud" }, "source": [ - "### Device Names\n", + "### Device names\n", "\n", - "The `Tensor.device` property provides a fully qualified string name of the device hosting the contents of the tensor. This name encodes many details, such as an identifier of the network address of the host on which this program is executing and the device within that host. This is required for distributed execution of a TensorFlow program. The string ends with `GPU:\u003cN\u003e` if the tensor is placed on the `N`-th GPU on the host." + "The `Tensor.device` property provides a fully qualified string name of the device hosting the contents of the tensor. This name encodes many details, such as an identifier of the network address of the host on which this program is executing and the device within that host. This is required for distributed execution of a TensorFlow program. The string ends with `GPU:` if the tensor is placed on the `N`-th GPU on the host." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ZWZQCimzuqyP" }, "source": [ + "### Explicit device placement\n", "\n", + "In TensorFlow, *placement* refers to how individual operations are assigned (placed on) a device for execution. As mentioned, when there is no explicit guidance provided, TensorFlow automatically decides which device to execute an operation and copies tensors to that device, if needed.\n", "\n", - "### Explicit Device Placement\n", - "\n", - "In TensorFlow, *placement* refers to how individual operations are assigned (placed on) a device for execution. As mentioned, when there is no explicit guidance provided, TensorFlow automatically decides which device to execute an operation and copies tensors to that device, if needed. However, TensorFlow operations can be explicitly placed on specific devices using the `tf.device` context manager, for example:" + "However, TensorFlow operations can be explicitly placed on specific devices using the `tf.device` context manager. For example:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "RjkNZTuauy-Q" }, "outputs": [], @@ -313,7 +268,7 @@ "def time_matmul(x):\n", " start = time.time()\n", " for loop in range(10):\n", - " tf.matmul(x, x)\n", + " tf.linalg.matmul(x, x)\n", "\n", " result = time.time()-start\n", "\n", @@ -327,7 +282,7 @@ " time_matmul(x)\n", "\n", "# Force execution on GPU #0 if available\n", - "if tf.config.experimental.list_physical_devices(\"GPU\"):\n", + "if tf.config.list_physical_devices(\"GPU\"):\n", " print(\"On GPU:\")\n", " with tf.device(\"GPU:0\"): # Or GPU:1 for the 2nd GPU, GPU:2 for the 3rd etc.\n", " x = tf.random.uniform([1000, 1000])\n", @@ -338,33 +293,29 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "o1K4dlhhHtQj" }, "source": [ "## Datasets\n", "\n", - "This section uses the [`tf.data.Dataset` API](https://www.tensorflow.org/guide/datasets) to build a pipeline for feeding data to your model. The `tf.data.Dataset` API is used to build performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops." + "This section uses the `tf.data.Dataset` API to build a pipeline for feeding data to your model. `tf.data.Dataset` is used to build performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops. (Refer to the [tf.data: Build TensorFlow input pipelines](../../guide/data.ipynb) guide to learn more.)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "zI0fmOynH-Ne" }, "source": [ "### Create a source `Dataset`\n", "\n", - "Create a *source* dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices), or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset). See the [TensorFlow Dataset guide](https://www.tensorflow.org/guide/datasets#reading_input_data) for more information." + "Create a *source* dataset using one of the factory functions like `tf.data.Dataset.from_tensors`, `tf.data.Dataset.from_tensor_slices`, or using objects that read from files like `tf.data.TextLineDataset` or `tf.data.TFRecordDataset`. Refer to the _Reading input data_ section of the [tf.data: Build TensorFlow input pipelines](../../guide/data.ipynb) guide for more information." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "F04fVOHQIBiG" }, "outputs": [], @@ -387,26 +338,23 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vbxIhC-5IPdf" }, "source": [ "### Apply transformations\n", "\n", - "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch), and [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle) to apply transformations to dataset records." + "Use the transformations functions like `tf.data.Dataset.map`, `tf.data.Dataset.batch`, and `tf.data.Dataset.shuffle` to apply transformations to dataset records." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "uXSDZWE-ISsd" }, "outputs": [], "source": [ - "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", + "ds_tensors = ds_tensors.map(tf.math.square).shuffle(2).batch(2)\n", "\n", "ds_file = ds_file.batch(2)" ] @@ -414,7 +362,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "A8X1GNfoIZKJ" }, "source": [ @@ -425,10 +372,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ws-WKRk5Ic6-" }, "outputs": [], @@ -447,15 +392,8 @@ "accelerator": "GPU", "colab": { "collapsed_sections": [], - "last_runtime": { - "build_target": "", - "kind": "local" - }, "name": "basics.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/customization/custom_layers.ipynb b/site/en/tutorials/customization/custom_layers.ipynb index 3b7f6734bb4..8bfe0a01b09 100644 --- a/site/en/tutorials/customization/custom_layers.ipynb +++ b/site/en/tutorials/customization/custom_layers.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tDnwEv8FtJm7" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "JlknJBWQtKkI" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "60RdWsg1tETW" }, "source": [ @@ -47,30 +43,28 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "BcJg7Enms86w" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/custom_layers\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/customization/custom_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/customization/custom_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/customization/custom_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UEu3q4jmpKVT" }, "source": [ @@ -79,52 +73,29 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-sXDg19Q691F" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", "id": "Py0m-N6VgQFJ" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " !pip install tf-nightly\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "TluWFcB_2nP5" }, "outputs": [], "source": [ - "print(tf.test.is_gpu_available())" + "print(tf.config.list_physical_devices('GPU'))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "zSFfVVjkrrsI" }, "source": [ @@ -132,17 +103,15 @@ "\n", "Most of the time when writing code for machine learning models you want to operate at a higher level of abstraction than individual operations and manipulation of individual variables.\n", "\n", - "Many machine learning models are expressible as the composition and stacking of relatively simple layers, and TensorFlow provides both a set of many common layers as a well as easy ways for you to write your own application-specific layers either from scratch or as the composition of existing layers.\n", + "Many machine learning models are expressible as the composition and stacking of relatively simple layers, and TensorFlow provides both a set of many common layers as well as easy ways for you to write your own application-specific layers either from scratch or as the composition of existing layers.\n", "\n", "TensorFlow includes the full [Keras](https://keras.io) API in the tf.keras package, and the Keras layers are very useful when building your own models.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8PyXlPl-4TzQ" }, "outputs": [], @@ -160,7 +129,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Fn69xxPO5Psr" }, "source": [ @@ -170,10 +138,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "E3XKNknP5Mhb" }, "outputs": [], @@ -184,10 +150,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Wt_Nsv-L5t2s" }, "outputs": [], @@ -201,10 +165,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "6ilvKjz8_4MQ" }, "outputs": [], @@ -216,25 +178,23 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "O0kDbE54-5VS" }, "source": [ "## Implementing custom layers\n", "The best way to implement your own layer is extending the tf.keras.Layer class and implementing:\n", - " * `__init__` , where you can do all input-independent initialization\n", - " * `build`, where you know the shapes of the input tensors and can do the rest of the initialization\n", - " * `call`, where you do the forward computation\n", + "\n", + "1. `__init__` , where you can do all input-independent initialization\n", + "2. `build`, where you know the shapes of the input tensors and can do the rest of the initialization\n", + "3. `call`, where you do the forward computation\n", "\n", "Note that you don't have to wait until `build` is called to create your variables, you can also create them in `__init__`. However, the advantage of creating them in `build` is that it enables late variable creation based on the shape of the inputs the layer will operate on. On the other hand, creating variables in `__init__` would mean that shapes required to create the variables will need to be explicitly specified." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "5Byl3n1k5kIy" }, "outputs": [], @@ -245,22 +205,20 @@ " self.num_outputs = num_outputs\n", "\n", " def build(self, input_shape):\n", - " self.kernel = self.add_variable(\"kernel\",\n", - " shape=[int(input_shape[-1]),\n", - " self.num_outputs])\n", + " self.kernel = self.add_weight(\"kernel\",\n", + " shape=[int(input_shape[-1]),\n", + " self.num_outputs])\n", "\n", - " def call(self, input):\n", - " return tf.matmul(input, self.kernel)\n", + " def call(self, inputs):\n", + " return tf.matmul(inputs, self.kernel)\n", "\n", "layer = MyDenseLayer(10)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "vrmBsYGOnuGO" }, "outputs": [], @@ -270,10 +228,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "1bsLjiPfnvat" }, "outputs": [], @@ -284,7 +240,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tk8E2vY0-z4Z" }, "source": [ @@ -294,7 +249,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Qhg4KlbKrs3G" }, "source": [ @@ -302,7 +256,7 @@ "\n", "Many interesting layer-like things in machine learning models are implemented by composing existing layers. For example, each residual block in a resnet is a composition of convolutions, batch normalizations, and a shortcut. Layers can be nested inside other layers.\n", "\n", - "Typically you inherit from `keras.Model` when you need the model methods like: `Model.fit`,`Model.evaluate`, and `Model.save` (see [Custom Keras layers and models](../../guide/keras/custom_layers_and_models.ipynb) for details).\n", + "Typically you inherit from `keras.Model` when you need the model methods like: `Model.fit`,`Model.evaluate`, and `Model.save` (see [Custom Keras layers and models](https://www.tensorflow.org/guide/keras/custom_layers_and_models) for details).\n", "\n", "One other feature provided by `keras.Model` (instead of `keras.layers.Layer`) is that in addition to tracking variables, a `keras.Model` also tracks its internal layers, making them easier to inspect.\n", "\n", @@ -311,10 +265,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "N30DTXiRASlb" }, "outputs": [], @@ -354,10 +306,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7D8ZR5mqtokj" }, "outputs": [], @@ -367,10 +317,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "MJ8rzFpdoE_m" }, "outputs": [], @@ -380,10 +328,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dewldLuDvQRM" }, "outputs": [], @@ -393,10 +339,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "FrqIXeSetaYi" }, "outputs": [], @@ -407,7 +351,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wYfucVw65PMj" }, "source": [ @@ -416,10 +359,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "L9frk7Ur4uvJ" }, "outputs": [], @@ -438,10 +379,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "tVAsbFITuScB" }, "outputs": [], @@ -452,7 +391,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "c5YwYcnuK-wc" }, "source": [ @@ -466,8 +404,6 @@ "colab": { "collapsed_sections": [], "name": "custom_layers.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/customization/custom_training.ipynb b/site/en/tutorials/customization/custom_training.ipynb deleted file mode 100644 index 80fbf2ef382..00000000000 --- a/site/en/tutorials/customization/custom_training.ipynb +++ /dev/null @@ -1,435 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rmpybwysXGV" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m8y3rGtQsYP2", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hrXv0rU9sIma" - }, - "source": [ - "# Custom training: basics" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7S0BwJ_8sLu7" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k2o3TTG4TFpt" - }, - "source": [ - "In the previous tutorial, you covered the TensorFlow APIs for automatic differentiation—a basic building block for machine learning.\n", - "In this tutorial, you will use the TensorFlow primitives introduced in the prior tutorials to do some simple machine learning.\n", - "\n", - "TensorFlow also includes `tf.keras`—a high-level neural network API that provides useful abstractions to reduce boilerplate and makes TensorFlow easier to use without sacrificing flexibility and performance. We strongly recommend the [tf.Keras API](../../guide/keras/overview.ipynb) for development. However, in this short tutorial you will learn how to train a neural network from first principles to establish a strong foundation." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3LXMVuV0VhDr" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NiolgWMPgpwI", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eMAWbDJFVmMk" - }, - "source": [ - "## Variables\n", - "\n", - "Tensors in TensorFlow are immutable stateless objects. Machine learning models, however, must have changing state: as your model trains, the same code to compute predictions should behave differently over time (hopefully with a lower loss!). To represent this state, which needs to change over the course of your computation, you can choose to rely on the fact that Python is a stateful programming language:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VkJwtLS_Jbn8", - "colab": {} - }, - "source": [ - "# Using Python state\n", - "x = tf.zeros([10, 10])\n", - "x += 2 # This is equivalent to x = x + 2, which does not mutate the original\n", - " # value of x\n", - "print(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfneTXy7JcUz" - }, - "source": [ - "TensorFlow has stateful operations built-in, and these are often easier than using low-level Python representations for your state. Use `tf.Variable` to represent weights in a model.\n", - "\n", - "A `tf.Variable` object stores a value and implicitly reads from this stored value. There are operations (`tf.assign_sub`, `tf.scatter_update`, etc.) that manipulate the value stored in a TensorFlow variable." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "itxmrMil6DQi", - "colab": {} - }, - "source": [ - "v = tf.Variable(1.0)\n", - "# Use Python's `assert` as a debugging statement to test the condition\n", - "assert v.numpy() == 1.0\n", - "\n", - "# Reassign the value `v`\n", - "v.assign(3.0)\n", - "assert v.numpy() == 3.0\n", - "\n", - "# Use `v` in a TensorFlow `tf.square()` operation and reassign\n", - "v.assign(tf.square(v))\n", - "assert v.numpy() == 9.0\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-paSaeq1JzwC" - }, - "source": [ - "Computations using `tf.Variable` are automatically traced when computing gradients. For variables that represent embeddings, TensorFlow will do sparse updates by default, which are more computation and memory efficient.\n", - "\n", - "A `tf.Variable` is also a way to show a reader of your code that a piece of state is mutable." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BMiFcDzE7Qu3" - }, - "source": [ - "## Fit a linear model\n", - "\n", - "Let's use the concepts you have learned so far—`Tensor`, `Variable`, and `GradientTape`—to build and train a simple model. This typically involves a few steps:\n", - "\n", - "1. Define the model.\n", - "2. Define a loss function.\n", - "3. Obtain training data.\n", - "4. Run through the training data and use an \"optimizer\" to adjust the variables to fit the data.\n", - "\n", - "Here, you'll create a simple linear model, `f(x) = x * W + b`, which has two variables: `W` (weights) and `b` (bias). You'll synthesize data such that a well trained model would have `W = 3.0` and `b = 2.0`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFzH64Jn9PIm" - }, - "source": [ - "### Define the model\n", - "\n", - "Let's define a simple class to encapsulate the variables and the computation:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_WRu7Pze7wk8", - "colab": {} - }, - "source": [ - "class Model(object):\n", - " def __init__(self):\n", - " # Initialize the weights to `5.0` and the bias to `0.0`\n", - " # In practice, these should be initialized to random values (for example, with `tf.random.normal`)\n", - " self.W = tf.Variable(5.0)\n", - " self.b = tf.Variable(0.0)\n", - "\n", - " def __call__(self, x):\n", - " return self.W * x + self.b\n", - "\n", - "model = Model()\n", - "\n", - "assert model(3.0).numpy() == 15.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xa6j_yXa-j79" - }, - "source": [ - "### Define a loss function\n", - "\n", - "A loss function measures how well the output of a model for a given input matches the target output. The goal is to minimize this difference during training. Let's use the standard L2 loss, also known as the least square errors:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y0ysUFGY924U", - "colab": {} - }, - "source": [ - "def loss(predicted_y, target_y):\n", - " return tf.reduce_mean(tf.square(predicted_y - target_y))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qutT_fkl_CBc" - }, - "source": [ - "### Obtain training data\n", - "\n", - "First, synthesize the training data by adding random Gaussian (Normal) noise to the inputs:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gxPTb-kt_N5m", - "colab": {} - }, - "source": [ - "TRUE_W = 3.0\n", - "TRUE_b = 2.0\n", - "NUM_EXAMPLES = 1000\n", - "\n", - "inputs = tf.random.normal(shape=[NUM_EXAMPLES])\n", - "noise = tf.random.normal(shape=[NUM_EXAMPLES])\n", - "outputs = inputs * TRUE_W + TRUE_b + noise" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-50nq-wPBsAW" - }, - "source": [ - "Before training the model, visualize the loss value by plotting the model's predictions in red and the training data in blue:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_eb83LtrB4nt", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(inputs, outputs, c='b')\n", - "plt.scatter(inputs, model(inputs), c='r')\n", - "plt.show()\n", - "\n", - "print('Current loss: %1.6f' % loss(model(inputs), outputs).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sSDP-yeq_4jE" - }, - "source": [ - "### Define a training loop\n", - "\n", - "With the network and training data, train the model using [gradient descent](https://en.wikipedia.org/wiki/Gradient_descent) to update the weights variable (`W`) and the bias variable (`b`) to reduce the loss. There are many variants of the gradient descent scheme that are captured in `tf.train.Optimizer`—our recommended implementation. But in the spirit of building from first principles, here you will implement the basic math yourself with the help of `tf.GradientTape` for automatic differentiation and `tf.assign_sub` for decrementing a value (which combines `tf.assign` and `tf.sub`):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MBIACgdnA55X", - "colab": {} - }, - "source": [ - "def train(model, inputs, outputs, learning_rate):\n", - " with tf.GradientTape() as t:\n", - " current_loss = loss(model(inputs), outputs)\n", - " dW, db = t.gradient(current_loss, [model.W, model.b])\n", - " model.W.assign_sub(learning_rate * dW)\n", - " model.b.assign_sub(learning_rate * db)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RwWPaJryD2aN" - }, - "source": [ - "Finally, let's repeatedly run through the training data and see how `W` and `b` evolve." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XdfkR223D9dW", - "colab": {} - }, - "source": [ - "model = Model()\n", - "\n", - "# Collect the history of W-values and b-values to plot later\n", - "Ws, bs = [], []\n", - "epochs = range(10)\n", - "for epoch in epochs:\n", - " Ws.append(model.W.numpy())\n", - " bs.append(model.b.numpy())\n", - " current_loss = loss(model(inputs), outputs)\n", - "\n", - " train(model, inputs, outputs, learning_rate=0.1)\n", - " print('Epoch %2d: W=%1.2f b=%1.2f, loss=%2.5f' %\n", - " (epoch, Ws[-1], bs[-1], current_loss))\n", - "\n", - "# Let's plot it all\n", - "plt.plot(epochs, Ws, 'r',\n", - " epochs, bs, 'b')\n", - "plt.plot([TRUE_W] * len(epochs), 'r--',\n", - " [TRUE_b] * len(epochs), 'b--')\n", - "plt.legend(['W', 'b', 'True W', 'True b'])\n", - "plt.show()\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vPnIVuaSJwWz" - }, - "source": [ - "## Next steps\n", - "\n", - "This tutorial used `tf.Variable` to build and train a simple linear model.\n", - "\n", - "In practice, the high-level APIs—such as `tf.keras`—are much more convenient to build neural networks. `tf.keras` provides higher level building blocks (called \"layers\"), utilities to save and restore state, a suite of loss functions, a suite of optimization strategies, and more. Read the [TensorFlow Keras guide](../../guide/keras/overview.ipynb) to learn more.\n" - ] - } - ] -} diff --git a/site/en/tutorials/customization/custom_training_walkthrough.ipynb b/site/en/tutorials/customization/custom_training_walkthrough.ipynb index e2a4c9c0c5f..9a05d864815 100644 --- a/site/en/tutorials/customization/custom_training_walkthrough.ipynb +++ b/site/en/tutorials/customization/custom_training_walkthrough.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rwxGnsA92emp" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "CPII1rGR2rF9" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JtEZ1pCPn--z" }, "source": [ @@ -47,503 +43,354 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GV1F7tVTN3Dn" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/custom_training_walkthrough\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/customization/custom_training_walkthrough.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/customization/custom_training_walkthrough.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/customization/custom_training_walkthrough.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LDrzLFXE8T1l" }, "source": [ - "This guide uses machine learning to *categorize* Iris flowers by species. It uses TensorFlow to:\n", - "1. Build a model,\n", - "2. Train this model on example data, and\n", - "3. Use the model to make predictions about unknown data.\n", + "This tutorial shows you how to train a machine learning model with a custom training loop to *categorize* penguins by species. In this notebook, you use TensorFlow to accomplish the following:\n", "\n", - "## TensorFlow programming\n", - "\n", - "This guide uses these high-level TensorFlow concepts:\n", + "1. Import a dataset\n", + "2. Build a simple linear model\n", + "3. Train the model\n", + "4. Evaluate the model's effectiveness\n", + "5. Use the trained model to make predictions\n", "\n", - "* Use TensorFlow's default [eager execution](https://www.tensorflow.org/guide/eager) development environment,\n", - "* Import data with the [Datasets API](https://www.tensorflow.org/guide/datasets),\n", - "* Build models and layers with TensorFlow's [Keras API](https://keras.io/getting-started/sequential-model-guide/).\n", + "## TensorFlow programming\n", "\n", - "This tutorial is structured like many TensorFlow programs:\n", + "This tutorial demonstrates the following TensorFlow programming tasks:\n", "\n", - "1. Import and parse the dataset.\n", - "2. Select the type of model.\n", - "3. Train the model.\n", - "4. Evaluate the model's effectiveness.\n", - "5. Use the trained model to make predictions." + "* Importing data with the [TensorFlow Datasets API](https://www.tensorflow.org/datasets/overview#load_a_dataset)\n", + "* Building models and layers with the [Keras API](https://www.tensorflow.org/guide/keras/)\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "yNr7H-AIoLOR" + "id": "Zx7wc0LuuxaJ" }, "source": [ - "## Setup program" + "## Penguin classification problem \n", + "\n", + "Imagine you are an ornithologist seeking an automated way to categorize each penguin you find. Machine learning provides many algorithms to classify penguins statistically. For instance, a sophisticated machine learning program could classify penguins based on photographs. The model you build in this tutorial is a little simpler. It classifies penguins based on their body weight, flipper length, and beaks, specifically the length and width measurements of their [culmen](https://en.wikipedia.org/wiki/Beak#Culmen).\n", + "\n", + "There are 18 species of penguins, but in this tutorial you will only attempt to classify the following three:\n", + "\n", + "* Chinstrap penguins\n", + "* Gentoo penguins\n", + "* Adélie penguins\n", + "\n", + "\n", + " \n", + " \n", + "
    \n", + " \"Illustration\n", + "
    \n", + " Figure 1. Chinstratp, Gentoo, and Adélie penguins (Artwork by @allison_horst, CC BY-SA 2.0).
     \n", + "
    \n", + "\n", + "Fortunately, a research team has already created and shared a [dataset of 334 penguins](https://allisonhorst.github.io/palmerpenguins/) with body weight, flipper length, beak measurements, and other data. This dataset is also conveniently available as the [penguins](https://www.tensorflow.org/datasets/catalog/penguins) TensorFlow Dataset. " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1J3AuPBT9gyR" }, "source": [ - "### Configure imports\n", + "## Setup\n", "\n", - "Import TensorFlow and the other required Python modules. By default, TensorFlow uses [eager execution](https://www.tensorflow.org/guide/eager) to evaluate operations immediately, returning concrete values instead of creating a [computational graph](https://www.tensorflow.org/guide/graphs) that is executed later. If you are used to a REPL or the `python` interactive console, this feels familiar." + "Install the `tfds-nightly` package for the penguins dataset. The `tfds-nightly` package is the nightly released version of the TensorFlow Datasets (TFDS). For more information on TFDS, see [TensorFlow Datasets overview](https://www.tensorflow.org/datasets/overview)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jElLULrDhQZR" + "id": "4XXWn1eDZmET" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "import matplotlib.pyplot as plt" + "!pip install -q tfds-nightly" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bfV2Dai0Ow2o" + "id": "DtGeMicKRGzU" }, - "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" + "Then select **Runtime > Restart Runtime** from the Colab menu to restart the Colab runtime.\n", + "\n", + "Do not proceed with the rest of this tutorial without first restarting the runtime." ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "g4Wzg69bnwK2" + "id": "G9onjGZWZbA-" }, - "outputs": [], "source": [ - "print(\"TensorFlow version: {}\".format(tf.__version__))\n", - "print(\"Eager execution: {}\".format(tf.executing_eagerly()))" + "Import TensorFlow and the other required Python modules. " ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "Zx7wc0LuuxaJ" + "id": "jElLULrDhQZR" }, + "outputs": [], "source": [ - "## The Iris classification problem\n", - "\n", - "Imagine you are a botanist seeking an automated way to categorize each Iris flower you find. Machine learning provides many algorithms to classify flowers statistically. For instance, a sophisticated machine learning program could classify flowers based on photographs. Our ambitions are more modest—we're going to classify Iris flowers based on the length and width measurements of their [sepals](https://en.wikipedia.org/wiki/Sepal) and [petals](https://en.wikipedia.org/wiki/Petal).\n", - "\n", - "The Iris genus entails about 300 species, but our program will only classify the following three:\n", - "\n", - "* Iris setosa\n", - "* Iris virginica\n", - "* Iris versicolor\n", - "\n", - "\u003ctable\u003e\n", - " \u003ctr\u003e\u003ctd\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/iris_three_species.jpg\"\n", - " alt=\"Petal geometry compared for three iris species: Iris setosa, Iris virginica, and Iris versicolor\"\u003e\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\"\u003e\n", - " \u003cb\u003eFigure 1.\u003c/b\u003e \u003ca href=\"https://commons.wikimedia.org/w/index.php?curid=170298\"\u003eIris setosa\u003c/a\u003e (by \u003ca href=\"https://commons.wikimedia.org/wiki/User:Radomil\"\u003eRadomil\u003c/a\u003e, CC BY-SA 3.0), \u003ca href=\"https://commons.wikimedia.org/w/index.php?curid=248095\"\u003eIris versicolor\u003c/a\u003e, (by \u003ca href=\"https://commons.wikimedia.org/wiki/User:Dlanglois\"\u003eDlanglois\u003c/a\u003e, CC BY-SA 3.0), and \u003ca href=\"https://www.flickr.com/photos/33397993@N05/3352169862\"\u003eIris virginica\u003c/a\u003e (by \u003ca href=\"https://www.flickr.com/photos/33397993@N05\"\u003eFrank Mayfield\u003c/a\u003e, CC BY-SA 2.0).\u003cbr/\u003e\u0026nbsp;\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e\n", + "import os\n", + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "import matplotlib.pyplot as plt\n", "\n", - "Fortunately, someone has already created a [dataset of 120 Iris flowers](https://en.wikipedia.org/wiki/Iris_flower_data_set) with the sepal and petal measurements. This is a classic dataset that is popular for beginner machine learning classification problems." + "print(\"TensorFlow version: {}\".format(tf.__version__))\n", + "print(\"TensorFlow Datasets version: \",tfds.__version__)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3Px6KAg0Jowz" }, "source": [ - "## Import and parse the training dataset\n", - "\n", - "Download the dataset file and convert it into a structure that can be used by this Python program.\n", - "\n", - "### Download the dataset\n", - "\n", - "Download the training dataset file using the [tf.keras.utils.get_file](https://www.tensorflow.org/api_docs/python/tf/keras/utils/get_file) function. This returns the file path of the downloaded file:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "J6c7uEU9rjRM" - }, - "outputs": [], - "source": [ - "train_dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv\"\n", - "\n", - "train_dataset_fp = tf.keras.utils.get_file(fname=os.path.basename(train_dataset_url),\n", - " origin=train_dataset_url)\n", + "## Import the dataset\n", "\n", - "print(\"Local copy of the dataset file: {}\".format(train_dataset_fp))" + "The default [penguins/processed](https://www.tensorflow.org/datasets/catalog/penguins) TensorFlow Dataset is already cleaned, normalized, and ready for building a model. Before you download the processed data, preview a simplified version to get familiar with the original penguin survey data.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qnX1-aLors4S" }, "source": [ - "### Inspect the data\n", + "### Preview the data\n", "\n", - "This dataset, `iris_training.csv`, is a plain text file that stores tabular data formatted as comma-separated values (CSV). Use the `head -n5` command to take a peek at the first five entries:" + "Download the simplified version of the penguins dataset (`penguins/simple`) using the TensorFlow Datasets [`tfds.load`](https://www.tensorflow.org/datasets/api_docs/python/tfds/load) method. There are 344 data records in this dataset. Extract the first five records into a [`DataFrame`](https://www.tensorflow.org/datasets/api_docs/python/tfds/as_dataframe) object to inspect a sample of the values in this dataset:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "FQvb_JYdrpPm" }, "outputs": [], "source": [ - "!head -n5 {train_dataset_fp}" + "ds_preview, info = tfds.load('penguins/simple', split='train', with_info=True)\n", + "df = tfds.as_dataframe(ds_preview.take(5), info)\n", + "print(df)\n", + "print(info.features)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kQhzD6P-uBoq" }, "source": [ - "From this view of the dataset, notice the following:\n", - "\n", - "1. The first line is a header containing information about the dataset:\n", - " * There are 120 total examples. Each example has four features and one of three possible label names.\n", - "2. Subsequent rows are data records, one *[example](https://developers.google.com/machine-learning/glossary/#example)* per line, where:\n", - " * The first four fields are *[features](https://developers.google.com/machine-learning/glossary/#feature)*: these are the characteristics of an example. Here, the fields hold float numbers representing flower measurements.\n", - " * The last column is the *[label](https://developers.google.com/machine-learning/glossary/#label)*: this is the value we want to predict. For this dataset, it's an integer value of 0, 1, or 2 that corresponds to a flower name.\n", - "\n", - "Let's write that out in code:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9Edhevw7exl6" - }, - "outputs": [], - "source": [ - "# column order in CSV file\n", - "column_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']\n", - "\n", - "feature_names = column_names[:-1]\n", - "label_name = column_names[-1]\n", - "\n", - "print(\"Features: {}\".format(feature_names))\n", - "print(\"Label: {}\".format(label_name))" + "The numbered rows are data records, one _[example](https://developers.google.com/machine-learning/glossary/#example)_ per line, where:\n", + " * The first six fields are _[features](https://developers.google.com/machine-learning/glossary/#feature)_: these are the characteristics of an example. Here, the fields hold numbers representing penguin measurements.\n", + " * The last column is the _[label](https://developers.google.com/machine-learning/glossary/#label)_: this is the value you want to predict. For this dataset, it's an integer value of 0, 1, or 2 that corresponds to a penguin species name." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CCtwLoJhhDNc" }, "source": [ - "Each label is associated with string name (for example, \"setosa\"), but machine learning typically relies on numeric values. The label numbers are mapped to a named representation, such as:\n", + "In the dataset, the label for the penguin species is represented as a number to make it easier to work with in the model you are building. These numbers correspond to the following penguin species:\n", "\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica\n", + "* `0`: Adélie penguin\n", + "* `1`: Chinstrap penguin\n", + "* `2`: Gentoo penguin\n", "\n", - "For more information about features and labels, see the [ML Terminology section of the Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/framing/ml-terminology)." + "Create a list containing the penguin species names in this order. You will use this list to interpret the output of the classification model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "sVNlJlUOhkoX" }, "outputs": [], "source": [ - "class_names = ['Iris setosa', 'Iris versicolor', 'Iris virginica']" + "class_names = ['Adélie', 'Chinstrap', 'Gentoo']" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "dqPkQExM2Pwt" - }, - "source": [ - "### Create a `tf.data.Dataset`\n", - "\n", - "TensorFlow's [Dataset API](https://www.tensorflow.org/guide/datasets) handles many common cases for loading data into a model. This is a high-level API for reading data and transforming it into a form used for training. See the [Datasets Quick Start guide](https://www.tensorflow.org/get_started/datasets_quickstart) for more information.\n", - "\n", - "\n", - "Since the dataset is a CSV-formatted text file, use the [make_csv_dataset](https://www.tensorflow.org/api_docs/python/tf/data/experimental/make_csv_dataset) function to parse the data into a suitable format. Since this function generates data for training models, the default behavior is to shuffle the data (`shuffle=True, shuffle_buffer_size=10000`), and repeat the dataset forever (`num_epochs=None`). We also set the [batch_size](https://developers.google.com/machine-learning/glossary/#batch_size) parameter:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WsxHnz1ebJ2S" + "id": "iav9kEgxpY0s" }, - "outputs": [], "source": [ - "batch_size = 32\n", - "\n", - "train_dataset = tf.data.experimental.make_csv_dataset(\n", - " train_dataset_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name=label_name,\n", - " num_epochs=1)" + "For more information about features and labels, refer to the [ML Terminology section of the Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/framing/ml-terminology)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "gB_RSn62c-3G" + "id": "PD33PxSmCrtL" }, "source": [ - "The `make_csv_dataset` function returns a `tf.data.Dataset` of `(features, label)` pairs, where `features` is a dictionary: `{'feature_name': value}`\n", + "### Download the preprocessed dataset\n", "\n", - "These `Dataset` objects are iterable. Let's look at a batch of features:" + "Now, download the preprocessed penguins dataset (`penguins/processed`) with the `tfds.load` method, which returns a list of `tf.data.Dataset` objects. Note that the `penguins/processed` dataset doesn't come with its own test set, so use an 80:20 split to [slice the full dataset](https://www.tensorflow.org/datasets/splits) into the training and test sets. You will use the test dataset later to verify your model." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iDuG94H-C122" + "id": "EVV96zIYYAi8" }, "outputs": [], "source": [ - "features, labels = next(iter(train_dataset))\n", + "ds_split, info = tfds.load(\"penguins/processed\", split=['train[:20%]', 'train[20%:]'], as_supervised=True, with_info=True)\n", "\n", - "print(features)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E63mArnQaAGz" - }, - "source": [ - "Notice that like-features are grouped together, or *batched*. Each example row's fields are appended to the corresponding feature array. Change the `batch_size` to set the number of examples stored in these feature arrays.\n", + "ds_test = ds_split[0]\n", + "ds_train = ds_split[1]\n", + "assert isinstance(ds_test, tf.data.Dataset)\n", "\n", - "You can start to see some clusters by plotting a few features from the batch:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "me5Wn-9FcyyO" - }, - "outputs": [], - "source": [ - "plt.scatter(features['petal_length'],\n", - " features['sepal_length'],\n", - " c=labels,\n", - " cmap='viridis')\n", + "print(info.features)\n", + "df_test = tfds.as_dataframe(ds_test.take(5), info)\n", + "print(\"Test dataset sample: \")\n", + "print(df_test)\n", "\n", - "plt.xlabel(\"Petal length\")\n", - "plt.ylabel(\"Sepal length\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YlxpSyHlhT6M" - }, - "source": [ - "To simplify the model building step, create a function to repackage the features dictionary into a single array with shape: `(batch_size, num_features)`.\n", + "df_train = tfds.as_dataframe(ds_train.take(5), info)\n", + "print(\"Train dataset sample: \")\n", + "print(df_train)\n", "\n", - "This function uses the [tf.stack](https://www.tensorflow.org/api_docs/python/tf/stack) method which takes values from a list of tensors and creates a combined tensor at the specified dimension:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jm932WINcaGU" - }, - "outputs": [], - "source": [ - "def pack_features_vector(features, labels):\n", - " \"\"\"Pack the features into a single array.\"\"\"\n", - " features = tf.stack(list(features.values()), axis=1)\n", - " return features, labels" + "ds_train_batch = ds_train.batch(32)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "V1Vuph_eDl8x" + "id": "xX2NfLyQOK1y" }, "source": [ - "Then use the [tf.data.Dataset.map](https://www.tensorflow.org/api_docs/python/tf/data/dataset/map) method to pack the `features` of each `(features,label)` pair into the training dataset:" + "Notice that this version of the dataset has been processed by reducing the data down to four normalized features and a species label. In this format, the data can be quickly used to train a model without further processing." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ZbDkzGZIkpXf" + "id": "iDuG94H-C122" }, "outputs": [], "source": [ - "train_dataset = train_dataset.map(pack_features_vector)" + "features, labels = next(iter(ds_train_batch))\n", + "\n", + "print(features)\n", + "print(labels)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "NLy0Q1xCldVO" + "id": "E63mArnQaAGz" }, "source": [ - "The features element of the `Dataset` are now arrays with shape `(batch_size, num_features)`. Let's look at the first few examples:" + "You can visualize some clusters by plotting a few features from the batch:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kex9ibEek6Tr" + "id": "me5Wn-9FcyyO" }, "outputs": [], "source": [ - "features, labels = next(iter(train_dataset))\n", + "plt.scatter(features[:,0],\n", + " features[:,2],\n", + " c=labels,\n", + " cmap='viridis')\n", "\n", - "print(features[:5])" + "plt.xlabel(\"Body Mass\")\n", + "plt.ylabel(\"Culmen Length\")\n", + "plt.show()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LsaVrtNM3Tx5" }, "source": [ - "## Select the type of model\n", + "## Build a simple linear model\n", "\n", "### Why model?\n", "\n", - "A *[model](https://developers.google.com/machine-learning/crash-course/glossary#model)* is a relationship between features and the label. For the Iris classification problem, the model defines the relationship between the sepal and petal measurements and the predicted Iris species. Some simple models can be described with a few lines of algebra, but complex machine learning models have a large number of parameters that are difficult to summarize.\n", + "A *[model](https://developers.google.com/machine-learning/crash-course/glossary#model)* is a relationship between features and the label. For the penguin classification problem, the model defines the relationship between the body mass, flipper and culmen measurements and the predicted penguin species. Some simple models can be described with a few lines of algebra, but complex machine learning models have a large number of parameters that are difficult to summarize.\n", "\n", - "Could you determine the relationship between the four features and the Iris species *without* using machine learning? That is, could you use traditional programming techniques (for example, a lot of conditional statements) to create a model? Perhaps—if you analyzed the dataset long enough to determine the relationships between petal and sepal measurements to a particular species. And this becomes difficult—maybe impossible—on more complicated datasets. A good machine learning approach *determines the model for you*. If you feed enough representative examples into the right machine learning model type, the program will figure out the relationships for you.\n", + "Could you determine the relationship between the four features and the penguin species *without* using machine learning? That is, could you use traditional programming techniques (for example, a lot of conditional statements) to create a model? Perhaps—if you analyzed the dataset long enough to determine the relationships between body mass and culmen measurements to a particular species. And this becomes difficult—maybe impossible—on more complicated datasets. A good machine learning approach *determines the model for you*. If you feed enough representative examples into the right machine learning model type, the program figures out the relationships for you.\n", "\n", "### Select the model\n", "\n", - "We need to select the kind of model to train. There are many types of models and picking a good one takes experience. This tutorial uses a neural network to solve the Iris classification problem. *[Neural networks](https://developers.google.com/machine-learning/glossary/#neural_network)* can find complex relationships between features and the label. It is a highly-structured graph, organized into one or more *[hidden layers](https://developers.google.com/machine-learning/glossary/#hidden_layer)*. Each hidden layer consists of one or more *[neurons](https://developers.google.com/machine-learning/glossary/#neuron)*. There are several categories of neural networks and this program uses a dense, or *[fully-connected neural network](https://developers.google.com/machine-learning/glossary/#fully_connected_layer)*: the neurons in one layer receive input connections from *every* neuron in the previous layer. For example, Figure 2 illustrates a dense neural network consisting of an input layer, two hidden layers, and an output layer:\n", + "Next you need to select the kind of model to train. There are many types of models and picking a good one takes experience. This tutorial uses a neural network to solve the penguin classification problem. [*Neural networks*](https://developers.google.com/machine-learning/glossary/#neural_network) can find complex relationships between features and the label. It is a highly-structured graph, organized into one or more [*hidden layers*](https://developers.google.com/machine-learning/glossary/#hidden_layer). Each hidden layer consists of one or more [*neurons*](https://developers.google.com/machine-learning/glossary/#neuron). There are several categories of neural networks and this program uses a dense, or [*fully-connected neural network*](https://developers.google.com/machine-learning/glossary/#fully_connected_layer): the neurons in one layer receive input connections from *every* neuron in the previous layer. For example, Figure 2 illustrates a dense neural network consisting of an input layer, two hidden layers, and an output layer:\n", + "\n", "\n", - "\u003ctable\u003e\n", - " \u003ctr\u003e\u003ctd\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/custom_estimators/full_network.png\"\n", - " alt=\"A diagram of the network architecture: Inputs, 2 hidden layers, and outputs\"\u003e\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\"\u003e\n", - " \u003cb\u003eFigure 2.\u003c/b\u003e A neural network with features, hidden layers, and predictions.\u003cbr/\u003e\u0026nbsp;\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e\n", "\n", - "When the model from Figure 2 is trained and fed an unlabeled example, it yields three predictions: the likelihood that this flower is the given Iris species. This prediction is called *[inference](https://developers.google.com/machine-learning/crash-course/glossary#inference)*. For this example, the sum of the output predictions is 1.0. In Figure 2, this prediction breaks down as: `0.02` for *Iris setosa*, `0.95` for *Iris versicolor*, and `0.03` for *Iris virginica*. This means that the model predicts—with 95% probability—that an unlabeled example flower is an *Iris versicolor*." + "\n", + " \n", + " \n", + "
    \n", + " \n", + "
    \n", + " Figure 2. A neural network with features, hidden layers, and predictions.
     \n", + "
    \n", + "\n", + "When you train the model from Figure 2 and feed it an unlabeled example, it yields three predictions: the likelihood that this penguin is the given penguin species. This prediction is called [*inference*](https://developers.google.com/machine-learning/crash-course/glossary#inference). For this example, the sum of the output predictions is 1.0. In Figure 2, this prediction breaks down as: `0.02` for *Adelie*, `0.95` for *Chinstrap*, and `0.03` for *Gentoo* species. This means that the model predicts—with 95% probability—that an unlabeled example penguin is a *Chinstrap* penguin." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "W23DIMVPQEBt" }, "source": [ "### Create a model using Keras\n", "\n", - "The TensorFlow [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras) API is the preferred way to create models and layers. This makes it easy to build models and experiment while Keras handles the complexity of connecting everything together.\n", + "The TensorFlow `tf.keras` API is the preferred way to create models and layers. This makes it easy to build models and experiment while Keras handles the complexity of connecting everything together.\n", "\n", - "The [tf.keras.Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential) model is a linear stack of layers. Its constructor takes a list of layer instances, in this case, two [Dense](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense) layers with 10 nodes each, and an output layer with 3 nodes representing our label predictions. The first layer's `input_shape` parameter corresponds to the number of features from the dataset, and is required:" + "The `tf.keras.Sequential` model is a linear stack of layers. Its constructor takes a list of layer instances, in this case, two `tf.keras.layers.Dense` layers with 10 nodes each, and an output layer with 3 nodes representing your label predictions. The first layer's `input_shape` parameter corresponds to the number of features from the dataset, and is required:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "2fZ6oL2ig3ZK" }, "outputs": [], @@ -558,11 +405,10 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FHcbEzMpxbHL" }, "source": [ - "The *[activation function](https://developers.google.com/machine-learning/crash-course/glossary#activation_function)* determines the output shape of each node in the layer. These non-linearities are important—without them the model would be equivalent to a single layer. There are many [available activations](https://www.tensorflow.org/api_docs/python/tf/keras/activations), but [ReLU](https://developers.google.com/machine-learning/crash-course/glossary#ReLU) is common for hidden layers.\n", + "The [*activation function*](https://developers.google.com/machine-learning/crash-course/glossary#activation_function) determines the output shape of each node in the layer. These non-linearities are important—without them the model would be equivalent to a single layer. There are many `tf.keras.activations`, but [ReLU](https://developers.google.com/machine-learning/crash-course/glossary#ReLU) is common for hidden layers.\n", "\n", "The ideal number of hidden layers and neurons depends on the problem and the dataset. Like many aspects of machine learning, picking the best shape of the neural network requires a mixture of knowledge and experimentation. As a rule of thumb, increasing the number of hidden layers and neurons typically creates a more powerful model, which requires more data to train effectively." ] @@ -570,21 +416,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2wFKnhWCpDSS" }, "source": [ - "### Using the model\n", + "### Use the model\n", "\n", "Let's have a quick look at what this model does to a batch of features:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "xe6SQ5NrpB-I" }, "outputs": [], @@ -596,7 +439,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wxyXOhwVr5S3" }, "source": [ @@ -607,10 +449,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "_tRwHZmTNTX2" }, "outputs": [], @@ -621,61 +461,54 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uRZmchElo481" }, "source": [ - "Taking the `tf.argmax` across classes gives us the predicted class index. But, the model hasn't been trained yet, so these aren't good predictions:" + "Taking the `tf.math.argmax` across classes gives us the predicted class index. But, the model hasn't been trained yet, so these aren't good predictions:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "-Jzm_GoErz8B" }, "outputs": [], "source": [ - "print(\"Prediction: {}\".format(tf.argmax(predictions, axis=1)))\n", + "print(\"Prediction: {}\".format(tf.math.argmax(predictions, axis=1)))\n", "print(\" Labels: {}\".format(labels))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Vzq2E5J2QMtw" }, "source": [ "## Train the model\n", "\n", - "*[Training](https://developers.google.com/machine-learning/crash-course/glossary#training)* is the stage of machine learning when the model is gradually optimized, or the model *learns* the dataset. The goal is to learn enough about the structure of the training dataset to make predictions about unseen data. If you learn *too much* about the training dataset, then the predictions only work for the data it has seen and will not be generalizable. This problem is called *[overfitting](https://developers.google.com/machine-learning/crash-course/glossary#overfitting)*—it's like memorizing the answers instead of understanding how to solve a problem.\n", + "[*Training*](https://developers.google.com/machine-learning/crash-course/glossary#training) is the stage of machine learning when the model is gradually optimized, or the model *learns* the dataset. The goal is to learn enough about the structure of the training dataset to make predictions about unseen data. If you learn *too much* about the training dataset, then the predictions only work for the data it has seen and will not be generalizable. This problem is called [*overfitting*](https://developers.google.com/machine-learning/crash-course/glossary#overfitting)—it's like memorizing the answers instead of understanding how to solve a problem.\n", "\n", - "The Iris classification problem is an example of *[supervised machine learning](https://developers.google.com/machine-learning/glossary/#supervised_machine_learning)*: the model is trained from examples that contain labels. In *[unsupervised machine learning](https://developers.google.com/machine-learning/glossary/#unsupervised_machine_learning)*, the examples don't contain labels. Instead, the model typically finds patterns among the features." + "The penguin classification problem is an example of [*supervised machine learning*](https://developers.google.com/machine-learning/glossary/#supervised_machine_learning): the model is trained from examples that contain labels. In [*unsupervised machine learning*](https://developers.google.com/machine-learning/glossary/#unsupervised_machine_learning), the examples don't contain labels. Instead, the model typically finds patterns among the features." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RaKp8aEjKX6B" }, "source": [ - "### Define the loss and gradient function\n", + "### Define the loss and gradients function\n", "\n", - "Both training and evaluation stages need to calculate the model's *[loss](https://developers.google.com/machine-learning/crash-course/glossary#loss)*. This measures how off a model's predictions are from the desired label, in other words, how bad the model is performing. We want to minimize, or optimize, this value.\n", + "Both training and evaluation stages need to calculate the model's [*loss*](https://developers.google.com/machine-learning/crash-course/glossary#loss). This measures how off a model's predictions are from the desired label, in other words, how bad the model is performing. You want to minimize, or optimize, this value.\n", "\n", - "Our model will calculate its loss using the `tf.keras.losses.SparseCategoricalCrossentropy` function which takes the model's class probability predictions and the desired label, and returns the average loss across the examples." + "Your model will calculate its loss using the `tf.keras.losses.SparseCategoricalCrossentropy` function which takes the model's class probability predictions and the desired label, and returns the average loss across the examples." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "QOsi6b-1CXIn" }, "outputs": [], @@ -685,90 +518,82 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "tMAT4DcMPwI-" }, "outputs": [], "source": [ - "def loss(model, x, y):\n", - " y_ = model(x)\n", + "def loss(model, x, y, training):\n", + " # training=training is needed only if there are layers with different\n", + " # behavior during training versus inference (e.g. Dropout).\n", + " y_ = model(x, training=training)\n", "\n", " return loss_object(y_true=y, y_pred=y_)\n", "\n", - "\n", - "l = loss(model, features, labels)\n", + "l = loss(model, features, labels, training=False)\n", "print(\"Loss test: {}\".format(l))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3IcPqA24QM6B" }, "source": [ - "Use the [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) context to calculate the *[gradients](https://developers.google.com/machine-learning/crash-course/glossary#gradient)* used to optimize your model:" + "Use the `tf.GradientTape` context to calculate the [*gradients*](https://developers.google.com/machine-learning/crash-course/glossary#gradient) used to optimize your model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "x57HcKWhKkei" }, "outputs": [], "source": [ "def grad(model, inputs, targets):\n", " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", + " loss_value = loss(model, inputs, targets, training=True)\n", " return loss_value, tape.gradient(loss_value, model.trainable_variables)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lOxFimtlKruu" }, "source": [ "### Create an optimizer\n", "\n", - "An *[optimizer](https://developers.google.com/machine-learning/crash-course/glossary#optimizer)* applies the computed gradients to the model's variables to minimize the `loss` function. You can think of the loss function as a curved surface (see Figure 3) and we want to find its lowest point by walking around. The gradients point in the direction of steepest ascent—so we'll travel the opposite way and move down the hill. By iteratively calculating the loss and gradient for each batch, we'll adjust the model during training. Gradually, the model will find the best combination of weights and bias to minimize loss. And the lower the loss, the better the model's predictions.\n", + "An [*optimizer*](https://developers.google.com/machine-learning/crash-course/glossary#optimizer) applies the computed gradients to the model's parameters to minimize the `loss` function. You can think of the loss function as a curved surface (refer to Figure 3) and you want to find its lowest point by walking around. The gradients point in the direction of steepest ascent—so you'll travel the opposite way and move down the hill. By iteratively calculating the loss and gradient for each batch, you'll adjust the model during training. Gradually, the model will find the best combination of weights and bias to minimize the loss. And the lower the loss, the better the model's predictions.\n", "\n", - "\u003ctable\u003e\n", - " \u003ctr\u003e\u003ctd\u003e\n", - " \u003cimg src=\"https://cs231n.github.io/assets/nn3/opt1.gif\" width=\"70%\"\n", - " alt=\"Optimization algorithms visualized over time in 3D space.\"\u003e\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\"\u003e\n", - " \u003cb\u003eFigure 3.\u003c/b\u003e Optimization algorithms visualized over time in 3D space.\u003cbr/\u003e(Source: \u003ca href=\"http://cs231n.github.io/neural-networks-3/\"\u003eStanford class CS231n\u003c/a\u003e, MIT License, Image credit: \u003ca href=\"https://twitter.com/alecrad\"\u003eAlec Radford\u003c/a\u003e)\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e\n", + "\n", + " \n", + " \n", + "
    \n", + " \"Optimization\n", + "
    \n", + " Figure 3. Optimization algorithms visualized over time in 3D space.
    (Source: Stanford class CS231n, MIT License, Image credit: Alec Radford)\n", + "
    \n", "\n", - "TensorFlow has many [optimization algorithms](https://www.tensorflow.org/api_docs/python/tf/optimizers) available for training. This model uses the [tf.keras.optimizers.SGD](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/SGD) that implements the *[stochastic gradient descent](https://developers.google.com/machine-learning/crash-course/glossary#gradient_descent)* (SGD) algorithm. The `learning_rate` sets the step size to take for each iteration down the hill. This is a *hyperparameter* that you'll commonly adjust to achieve better results." + "TensorFlow has many optimization algorithms available for training. In this tutorial, you will use the `tf.keras.optimizers.SGD` that implements the [*stochastic gradient descent*](https://developers.google.com/machine-learning/crash-course/glossary#gradient_descent) (SGD) algorithm. The `learning_rate` parameter sets the step size to take for each iteration down the hill. This rate is a [*hyperparameter*](https://developers.google.com/machine-learning/glossary/#hyperparameter) that you'll commonly adjust to achieve better results." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XkUd6UiZa_dF" }, "source": [ - "Let's setup the optimizer:" + "Instantiate the optimizer with a [*learning rate*](https://developers.google.com/machine-learning/glossary#learning-rate) of `0.01`, a scalar value that is multiplied by the gradient at each iteration of the training:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8xxi2NNGKwG_" }, "outputs": [], @@ -779,19 +604,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pJVRZ0hP52ZB" }, "source": [ - "We'll use this to calculate a single optimization step:" + "Then use this object to calculate a single optimization step:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "rxRNTFVe56RG" }, "outputs": [], @@ -804,13 +626,12 @@ "optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", "\n", "print(\"Step: {}, Loss: {}\".format(optimizer.iterations.numpy(),\n", - " loss(model, features, labels).numpy()))" + " loss(model, features, labels, training=True).numpy()))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7Y2VSELvwAvW" }, "source": [ @@ -821,24 +642,22 @@ "1. Iterate each *epoch*. An epoch is one pass through the dataset.\n", "2. Within an epoch, iterate over each example in the training `Dataset` grabbing its *features* (`x`) and *label* (`y`).\n", "3. Using the example's features, make a prediction and compare it with the label. Measure the inaccuracy of the prediction and use that to calculate the model's loss and gradients.\n", - "4. Use an `optimizer` to update the model's variables.\n", + "4. Use an `optimizer` to update the model's parameters.\n", "5. Keep track of some stats for visualization.\n", "6. Repeat for each epoch.\n", "\n", - "The `num_epochs` variable is the number of times to loop over the dataset collection. Counter-intuitively, training a model longer does not guarantee a better model. `num_epochs` is a *[hyperparameter](https://developers.google.com/machine-learning/glossary/#hyperparameter)* that you can tune. Choosing the right number usually requires both experience and experimentation:" + "The `num_epochs` variable is the number of times to loop over the dataset collection. In the code below, `num_epochs` is set to 201 which means this training loop will run 201 times. Counter-intuitively, training a model longer does not guarantee a better model. `num_epochs` is a [*hyperparameter*](https://developers.google.com/machine-learning/glossary/#hyperparameter) that you can tune. Choosing the right number usually requires both experience and experimentation:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "AIgulGRUhpto" }, "outputs": [], "source": [ - "## Note: Rerunning this cell uses the same model variables\n", + "## Note: Rerunning this cell uses the same model parameters\n", "\n", "# Keep results for plotting\n", "train_loss_results = []\n", @@ -851,15 +670,17 @@ " epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n", "\n", " # Training loop - using batches of 32\n", - " for x, y in train_dataset:\n", + " for x, y in ds_train_batch:\n", " # Optimize the model\n", " loss_value, grads = grad(model, x, y)\n", " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", "\n", " # Track progress\n", - " epoch_loss_avg(loss_value) # Add current batch loss\n", + " epoch_loss_avg.update_state(loss_value) # Add current batch loss\n", " # Compare predicted label to actual label\n", - " epoch_accuracy(y, model(x))\n", + " # training=True is needed only if there are layers with different\n", + " # behavior during training versus inference (e.g. Dropout).\n", + " epoch_accuracy.update_state(y, model(x, training=True))\n", "\n", " # End epoch\n", " train_loss_results.append(epoch_loss_avg.result())\n", @@ -874,7 +695,15 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "Diep-ROEuKyl" + }, + "source": [ + "Alternatively, you could use the built-in Keras [`Model.fit(ds_train_batch)`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit) method to train your model. " + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "2FQHVUnm_rjw" }, "source": [ @@ -884,21 +713,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "j3wdbmtLVTyr" }, "source": [ - "While it's helpful to print out the model's training progress, it's often *more* helpful to see this progress. [TensorBoard](https://tensorflow.org/tensorboard) is a nice visualization tool that is packaged with TensorFlow, but we can create basic charts using the `matplotlib` module.\n", + "While it's helpful to print out the model's training progress, you can visualize the progress with [TensorBoard](https://www.tensorflow.org/tensorboard) - a visualization and metrics tool that is packaged with TensorFlow. For this simple example, you will create basic charts using the `matplotlib` module.\n", "\n", - "Interpreting these charts takes some experience, but you really want to see the *loss* go down and the *accuracy* go up:" + "Interpreting these charts takes some experience, but in general you want to see the *loss* decrease and the *accuracy* increase:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "agjvNd2iUGFn" }, "outputs": [], @@ -918,126 +744,87 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Zg8GoMZhLpGH" }, "source": [ "## Evaluate the model's effectiveness\n", "\n", - "Now that the model is trained, we can get some statistics on its performance.\n", - "\n", - "*Evaluating* means determining how effectively the model makes predictions. To determine the model's effectiveness at Iris classification, pass some sepal and petal measurements to the model and ask the model to predict what Iris species they represent. Then compare the model's predictions against the actual label. For example, a model that picked the correct species on half the input examples has an *[accuracy](https://developers.google.com/machine-learning/glossary/#accuracy)* of `0.5`. Figure 4 shows a slightly more effective model, getting 4 out of 5 predictions correct at 80% accuracy:\n", - "\n", - "\u003ctable cellpadding=\"8\" border=\"0\"\u003e\n", - " \u003ccolgroup\u003e\n", - " \u003ccol span=\"4\" \u003e\n", - " \u003ccol span=\"1\" bgcolor=\"lightblue\"\u003e\n", - " \u003ccol span=\"1\" bgcolor=\"lightgreen\"\u003e\n", - " \u003c/colgroup\u003e\n", - " \u003ctr bgcolor=\"lightgray\"\u003e\n", - " \u003cth colspan=\"4\"\u003eExample features\u003c/th\u003e\n", - " \u003cth colspan=\"1\"\u003eLabel\u003c/th\u003e\n", - " \u003cth colspan=\"1\" \u003eModel prediction\u003c/th\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e5.9\u003c/td\u003e\u003ctd\u003e3.0\u003c/td\u003e\u003ctd\u003e4.3\u003c/td\u003e\u003ctd\u003e1.5\u003c/td\u003e\u003ctd align=\"center\"\u003e1\u003c/td\u003e\u003ctd align=\"center\"\u003e1\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e6.9\u003c/td\u003e\u003ctd\u003e3.1\u003c/td\u003e\u003ctd\u003e5.4\u003c/td\u003e\u003ctd\u003e2.1\u003c/td\u003e\u003ctd align=\"center\"\u003e2\u003c/td\u003e\u003ctd align=\"center\"\u003e2\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e5.1\u003c/td\u003e\u003ctd\u003e3.3\u003c/td\u003e\u003ctd\u003e1.7\u003c/td\u003e\u003ctd\u003e0.5\u003c/td\u003e\u003ctd align=\"center\"\u003e0\u003c/td\u003e\u003ctd align=\"center\"\u003e0\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e6.0\u003c/td\u003e \u003ctd\u003e3.4\u003c/td\u003e \u003ctd\u003e4.5\u003c/td\u003e \u003ctd\u003e1.6\u003c/td\u003e \u003ctd align=\"center\"\u003e1\u003c/td\u003e\u003ctd align=\"center\" bgcolor=\"red\"\u003e2\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e5.5\u003c/td\u003e\u003ctd\u003e2.5\u003c/td\u003e\u003ctd\u003e4.0\u003c/td\u003e\u003ctd\u003e1.3\u003c/td\u003e\u003ctd align=\"center\"\u003e1\u003c/td\u003e\u003ctd align=\"center\"\u003e1\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\" colspan=\"6\"\u003e\n", - " \u003cb\u003eFigure 4.\u003c/b\u003e An Iris classifier that is 80% accurate.\u003cbr/\u003e\u0026nbsp;\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e" + "Now that the model is trained, you can get some statistics on its performance.\n", + "\n", + "*Evaluating* means determining how effectively the model makes predictions. To determine the model's effectiveness at penguin classification, pass some measurements to the model and ask the model to predict what penguin species they represent. Then compare the model's predictions against the actual label. For example, a model that picked the correct species on half the input examples has an [*accuracy*](https://developers.google.com/machine-learning/glossary/#accuracy) of `0.5`. Figure 4 shows a slightly more effective model, getting 4 out of 5 predictions correct at 80% accuracy:\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    Example featuresLabelModel prediction
    5.93.04.31.511
    6.93.15.42.122
    5.13.31.70.500
    6.0 3.4 4.5 1.6 12
    5.52.54.01.311
    \n", + " Figure 4. A penguin classifier that is 80% accurate.
     \n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "z-EvK7hGL0d8" }, "source": [ - "### Setup the test dataset\n", + "### Set up the test set\n", "\n", "Evaluating the model is similar to training the model. The biggest difference is the examples come from a separate *[test set](https://developers.google.com/machine-learning/crash-course/glossary#test_set)* rather than the training set. To fairly assess a model's effectiveness, the examples used to evaluate a model must be different from the examples used to train the model.\n", "\n", - "The setup for the test `Dataset` is similar to the setup for training `Dataset`. Download the CSV text file and parse that values, then give it a little shuffle:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Ps3_9dJ3Lodk" - }, - "outputs": [], - "source": [ - "test_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv\"\n", - "\n", - "test_fp = tf.keras.utils.get_file(fname=os.path.basename(test_url),\n", - " origin=test_url)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SRMWCu30bnxH" - }, - "outputs": [], - "source": [ - "test_dataset = tf.data.experimental.make_csv_dataset(\n", - " test_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name='species',\n", - " num_epochs=1,\n", - " shuffle=False)\n", - "\n", - "test_dataset = test_dataset.map(pack_features_vector)" + "The penguin dataset doesn't have a separate test dataset so in the previous Download the dataset section, you split the original dataset into test and train datasets. Use the `ds_test_batch` dataset for the evaluation." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HFuOKXJdMAdm" }, "source": [ "### Evaluate the model on the test dataset\n", "\n", - "Unlike the training stage, the model only evaluates a single [epoch](https://developers.google.com/machine-learning/glossary/#epoch) of the test data. In the following code cell, we iterate over each example in the test set and compare the model's prediction against the actual label. This is used to measure the model's accuracy across the entire test set:" + "Unlike the training stage, the model only evaluates a single [epoch](https://developers.google.com/machine-learning/glossary/#epoch) of the test data. The following code iterates over each example in the test set and compare the model's prediction against the actual label. This comparison is used to measure the model's accuracy across the entire test set:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Tw03-MK1cYId" }, "outputs": [], "source": [ "test_accuracy = tf.keras.metrics.Accuracy()\n", + "ds_test_batch = ds_test.batch(10)\n", "\n", - "for (x, y) in test_dataset:\n", - " logits = model(x)\n", - " prediction = tf.argmax(logits, axis=1, output_type=tf.int32)\n", + "for (x, y) in ds_test_batch:\n", + " # training=False is needed only if there are layers with different\n", + " # behavior during training versus inference (e.g. Dropout).\n", + " logits = model(x, training=False)\n", + " prediction = tf.math.argmax(logits, axis=1, output_type=tf.int64)\n", " test_accuracy(prediction, y)\n", "\n", "print(\"Test set accuracy: {:.3%}\".format(test_accuracy.result()))" @@ -1046,19 +833,25 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "Fel8ql2qzGlK" + }, + "source": [ + "You can also use the `model.evaluate(ds_test, return_dict=True)` keras function to get accuracy information on your test dataset. " + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "HcKEZMtCOeK-" }, "source": [ - "We can see on the last batch, for example, the model is usually correct:" + "By inspecting the last batch, for example, you can observe that the model predictions are usually correct.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "uNwt2eMeOane" }, "outputs": [], @@ -1069,68 +862,51 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7Li2r1tYvW7S" }, "source": [ "## Use the trained model to make predictions\n", "\n", - "We've trained a model and \"proven\" that it's good—but not perfect—at classifying Iris species. Now let's use the trained model to make some predictions on [unlabeled examples](https://developers.google.com/machine-learning/glossary/#unlabeled_example); that is, on examples that contain features but not a label.\n", + "You've trained a model and \"proven\" that it's good—but not perfect—at classifying penguin species. Now let's use the trained model to make some predictions on [*unlabeled examples*](https://developers.google.com/machine-learning/glossary/#unlabeled_example); that is, on examples that contain features but not labels.\n", "\n", - "In real-life, the unlabeled examples could come from lots of different sources including apps, CSV files, and data feeds. For now, we're going to manually provide three unlabeled examples to predict their labels. Recall, the label numbers are mapped to a named representation as:\n", + "In real-life, the unlabeled examples could come from lots of different sources including apps, CSV files, and data feeds. For this tutorial, manually provide three unlabeled examples to predict their labels. Recall, the label numbers are mapped to a named representation as:\n", "\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica" + "* `0`: Adélie penguin\n", + "* `1`: Chinstrap penguin\n", + "* `2`: Gentoo penguin" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "kesTS5Lzv-M2" }, "outputs": [], "source": [ "predict_dataset = tf.convert_to_tensor([\n", - " [5.1, 3.3, 1.7, 0.5,],\n", - " [5.9, 3.0, 4.2, 1.5,],\n", - " [6.9, 3.1, 5.4, 2.1]\n", + " [0.3, 0.8, 0.4, 0.5,],\n", + " [0.4, 0.1, 0.8, 0.5,],\n", + " [0.7, 0.9, 0.8, 0.4]\n", "])\n", "\n", - "predictions = model(predict_dataset)\n", + "# training=False is needed only if there are layers with different\n", + "# behavior during training versus inference (e.g. Dropout).\n", + "predictions = model(predict_dataset, training=False)\n", "\n", "for i, logits in enumerate(predictions):\n", - " class_idx = tf.argmax(logits).numpy()\n", + " class_idx = tf.math.argmax(logits).numpy()\n", " p = tf.nn.softmax(logits)[class_idx]\n", " name = class_names[class_idx]\n", " print(\"Example {} prediction: {} ({:4.1f}%)\".format(i, name, 100*p))" ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JliO3dfQRcbg" - }, - "outputs": [], - "source": [ - "" - ] } ], "metadata": { "colab": { "collapsed_sections": [], - "name": "custom_training_walkthrough", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "name": "custom_training_walkthrough.ipynb", + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/customization/images/full_network_penguin.png b/site/en/tutorials/customization/images/full_network_penguin.png new file mode 100644 index 00000000000..3fb940bd8bf Binary files /dev/null and b/site/en/tutorials/customization/images/full_network_penguin.png differ diff --git a/site/en/tutorials/customization/images/penguins_ds_species.png b/site/en/tutorials/customization/images/penguins_ds_species.png new file mode 100644 index 00000000000..736ae89b686 Binary files /dev/null and b/site/en/tutorials/customization/images/penguins_ds_species.png differ diff --git a/site/en/tutorials/customization/performance.ipynb b/site/en/tutorials/customization/performance.ipynb deleted file mode 100644 index 7052438d8ad..00000000000 --- a/site/en/tutorials/customization/performance.ipynb +++ /dev/null @@ -1,1600 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISubpr_SSsiM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "3jTMb1dySr3V" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6DWfyNThSziV" - }, - "source": [ - "# Better performance with tf.function\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/performance\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/customization/performance.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/customization/performance.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/customization/performance.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J122XQYG7W6w" - }, - "source": [ - "\n", - "In TensorFlow 2.0, eager execution is turned on by default. The user interface is intuitive and flexible (running one-off operations is much easier\n", - "and faster), but this can come at the expense of performance and deployability.\n", - "\n", - "To get peak performance and to make your model deployable anywhere, use\n", - "`tf.function` to make graphs out of your programs.\n", - "Thanks to AutoGraph, a surprising amount of Python code just works with\n", - "tf.function, but there are still pitfalls to be wary of.\n", - "\n", - "The main takeaways and recommendations are:\n", - "\n", - "- Don't rely on Python side effects like object mutation or list appends.\n", - "- tf.function works best with TensorFlow ops, rather than NumPy ops or Python primitives.\n", - "- When in doubt, use the `for x in y` idiom." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SjvqpgepHJPd" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "otIdN1TS8N7S" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "I0xDjO4SHLUD" - }, - "source": [ - "Define a helper function to demonstrate the kinds of errors you might encounter:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "D25apou9IOXa" - }, - "outputs": [], - "source": [ - "import traceback\n", - "import contextlib\n", - "\n", - "# Some helper code to demonstrate the kinds of errors you might encounter.\n", - "@contextlib.contextmanager\n", - "def assert_raises(error_class):\n", - " try:\n", - " yield\n", - " except error_class as e:\n", - " print('Caught expected exception \\n {}:'.format(error_class))\n", - " traceback.print_exc(limit=2)\n", - " except Exception as e:\n", - " raise e\n", - " else:\n", - " raise Exception('Expected {} to be raised but no error was raised!'.format(\n", - " error_class))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WPSfepzTHThq" - }, - "source": [ - "## Basics" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rfayNj-ZIkIB" - }, - "source": [ - "A `tf.function` you define is just like a core TensorFlow operation: You can execute it eagerly; you can use it in a graph; it has gradients; and so on." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SbtT1-Wm70F2" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def add(a, b):\n", - " return a + b\n", - "\n", - "add(tf.ones([2, 2]), tf.ones([2, 2])) # [[2., 2.], [2., 2.]]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uP-zUelB8DbX" - }, - "outputs": [], - "source": [ - "v = tf.Variable(1.0)\n", - "with tf.GradientTape() as tape:\n", - " result = add(v, 1.0)\n", - "tape.gradient(result, v)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ocWZvqrmHnmX" - }, - "source": [ - "You can use functions inside functions" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l5qRjdbBVdU6" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def dense_layer(x, w, b):\n", - " return add(tf.matmul(x, w), b)\n", - "\n", - "dense_layer(tf.ones([3, 2]), tf.ones([2, 2]), tf.ones([2]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uZ4Do2AV80cO" - }, - "source": [ - "\u003ca id=\"tracing\"\u003e\u003c/a\u003e\n", - "\n", - "## Tracing and polymorphism\n", - "\n", - "Python's dynamic typing means that you can call functions with a variety of argument types, and Python will do something different in each scenario.\n", - "\n", - "On the other hand, TensorFlow graphs require static dtypes and shape dimensions. `tf.function` bridges this gap by retracing the function when necessary to generate the correct graphs. Most of the subtlety of `tf.function` usage stems from this retracing behavior.\n", - "\n", - "You can call a function with arguments of different types to see what is happening." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kojmJrgq8U9v" - }, - "outputs": [], - "source": [ - "# Functions are polymorphic\n", - "\n", - "@tf.function\n", - "def double(a):\n", - " print(\"Tracing with\", a)\n", - " return a + a\n", - "\n", - "print(double(tf.constant(1)))\n", - "print()\n", - "print(double(tf.constant(1.1)))\n", - "print()\n", - "print(double(tf.constant(\"a\")))\n", - "print()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4pJqkDR_Q2wz" - }, - "source": [ - "To control the tracing behavior, use the following techniques:\n", - "\n", - "- Create a new `tf.function`. Separate `tf.function` objects are guaranteed not to share traces. \n", - "- Use the `get_concrete_function` method to get a specific trace\n", - "- Specify `input_signature` when calling `tf.function` to trace only once per calling graph." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mHg2CGtPQ3Hz" - }, - "outputs": [], - "source": [ - "print(\"Obtaining concrete trace\")\n", - "double_strings = double.get_concrete_function(tf.TensorSpec(shape=None, dtype=tf.string))\n", - "print(\"Executing traced function\")\n", - "print(double_strings(tf.constant(\"a\")))\n", - "print(double_strings(a=tf.constant(\"b\")))\n", - "print(\"Using a concrete trace with incompatible types will throw an error\")\n", - "with assert_raises(tf.errors.InvalidArgumentError):\n", - " double_strings(tf.constant(1))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_BDMIRmu1RGB" - }, - "outputs": [], - "source": [ - "@tf.function(input_signature=(tf.TensorSpec(shape=[None], dtype=tf.int32),))\n", - "def next_collatz(x):\n", - " print(\"Tracing with\", x)\n", - " return tf.where(x % 2 == 0, x // 2, 3 * x + 1)\n", - "\n", - "print(next_collatz(tf.constant([1, 2])))\n", - "# We specified a 1-D tensor in the input signature, so this should fail.\n", - "with assert_raises(ValueError):\n", - " next_collatz(tf.constant([[1, 2], [3, 4]]))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Es0WZkLIUSdu" - }, - "source": [ - "## When to retrace?\n", - "\n", - "A polymorphic `tf.function` keeps a cache of concrete functions generated by tracing. The cache keys are effectively tuples of keys generated from the function args and kwargs. The key generated for a `tf.Tensor` argument is its number of dimensions and type. The key generated for a Python primitive is its value. For all other Python types, the keys are based on the object `id()` so that methods are traced independently for each instance of a class. In the future, TensorFlow may add more sophisticated cachi\n", - "ng for Python objects that can be safely converted to tensors.\n", - "\n", - "See [Concrete functions](../../guide/concrete_funciton.ipynb)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AY5oiQN0XIyA" - }, - "source": [ - "## Python or Tensor args?\n", - "\n", - "Often, Python arguments are used to control hyperparameters and graph constructions - for example, `num_layers=10` or `training=True` or `nonlinearity='relu'`. So if the Python argument changes, it makes sense that you'd have to retrace the graph.\n", - "\n", - "However, it's possible that a Python argument is not being used to control graph construction. In these cases, a change in the Python value can trigger needless retracing. Take, for example, this training loop, which AutoGraph will dynamically unroll. Despite the multiple traces, the generated graph is actually identical, so this is a bit inefficient." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uydzR5JYUU8H" - }, - "outputs": [], - "source": [ - "def train_one_step():\n", - " pass\n", - "\n", - "@tf.function\n", - "def train(num_steps):\n", - " print(\"Tracing with num_steps = {}\".format(num_steps))\n", - " for _ in tf.range(num_steps):\n", - " train_one_step()\n", - "\n", - "train(num_steps=10)\n", - "train(num_steps=20)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f6pjnylLUW8P" - }, - "source": [ - "The simple workaround here is to cast your arguments to Tensors if they do not affect the shape of the generated graph." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TmL8T-w3UYes" - }, - "outputs": [], - "source": [ - "train(num_steps=tf.constant(10))\n", - "train(num_steps=tf.constant(20))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "129-iRsPS-gY" - }, - "source": [ - "## Side effects in `tf.function`\n", - "\n", - "In general, Python side effects (like printing or mutating objects) only happen during tracing. So how can you reliably trigger side effects from `tf.function`?\n", - "\n", - "The general rule of thumb is to only use Python side effects to debug your traces. Otherwise, TensorFlow ops like `tf.Variable.assign`, `tf.print`, and `tf.summary` are the best way to ensure your code will be traced and executed by the TensorFlow runtime with each call. In general using a functional style will yield the best results. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "w2sACuZ9TTRk" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def f(x):\n", - " print(\"Traced with\", x)\n", - " tf.print(\"Executed with\", x)\n", - "\n", - "f(1)\n", - "f(1)\n", - "f(2)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e1I0dPiqTV8H" - }, - "source": [ - "If you would like to execute Python code during each invocation of a `tf.function`, `tf.py_function` is an exit hatch. The drawback of `tf.py_function` is that it's not portable or particularly performant, nor does it work well in distributed (multi-GPU, TPU) setups. Also, since `tf.py_function` has to be wired into the graph for differentiability, it casts all inputs/outputs to tensors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7aJD--9qTWmg" - }, - "outputs": [], - "source": [ - "external_list = []\n", - "\n", - "def side_effect(x):\n", - " print('Python side effect')\n", - " external_list.append(x)\n", - "\n", - "@tf.function\n", - "def f(x):\n", - " tf.py_function(side_effect, inp=[x], Tout=[])\n", - "\n", - "f(1)\n", - "f(1)\n", - "f(1)\n", - "assert len(external_list) == 3\n", - "# .numpy() call required because py_function casts 1 to tf.constant(1)\n", - "assert external_list[0].numpy() == 1\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "msTmv-oyUNaf" - }, - "source": [ - "## Beware of Python state\n", - "\n", - "Many Python features, such as generators and iterators, rely on the Python runtime to keep track of state. In general, while these constructs work as expected in Eager mode, many unexpected things can happen inside a `tf.function` due to tracing behavior.\n", - "\n", - "To give one example, advancing iterator state is a Python side effect and therefore only happens during tracing." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FNPD4unZUedH" - }, - "outputs": [], - "source": [ - "external_var = tf.Variable(0)\n", - "@tf.function\n", - "def buggy_consume_next(iterator):\n", - " external_var.assign_add(next(iterator))\n", - " tf.print(\"Value of external_var:\", external_var)\n", - "\n", - "iterator = iter([0, 1, 2, 3])\n", - "buggy_consume_next(iterator)\n", - "# This reuses the first value from the iterator, rather than consuming the next value.\n", - "buggy_consume_next(iterator)\n", - "buggy_consume_next(iterator)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5XMGXMu-Ufjm" - }, - "source": [ - "If an iterator is generated and consumed entirely within the tf.function, then it should work correctly. However, the entire iterator is probably being traced, which can lead to a giant graph. This may be what you want. But if you're training on an large in-memory dataset represented as a Python list, then this can generate a very large graph, and `tf.function` is unlikely to yield a speedup.\n", - "\n", - "If you want to iterate over Python data, the safest way is to wrap it in a tf.data.Dataset and use the `for x in y` idiom. AutoGraph has special support for safely converting `for` loops when `y` is a tensor or tf.data.Dataset.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ms7f1o_QUiHE" - }, - "outputs": [], - "source": [ - "def measure_graph_size(f, *args):\n", - " g = f.get_concrete_function(*args).graph\n", - " print(\"{}({}) contains {} nodes in its graph\".format(\n", - " f.__name__, ', '.join(map(str, args)), len(g.as_graph_def().node)))\n", - "\n", - "@tf.function\n", - "def train(dataset):\n", - " loss = tf.constant(0)\n", - " for x, y in dataset:\n", - " loss += tf.abs(y - x) # Some dummy computation.\n", - " return loss\n", - "\n", - "small_data = [(1, 1)] * 2\n", - "big_data = [(1, 1)] * 10\n", - "measure_graph_size(train, small_data)\n", - "measure_graph_size(train, big_data)\n", - "\n", - "measure_graph_size(train, tf.data.Dataset.from_generator(\n", - " lambda: small_data, (tf.int32, tf.int32)))\n", - "measure_graph_size(train, tf.data.Dataset.from_generator(\n", - " lambda: big_data, (tf.int32, tf.int32)))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGDstsFpWHEI" - }, - "source": [ - "\n", - "When wrapping Python/Numpy data in a Dataset, be mindful of `tf.data.Dataset.from_generator` versus ` tf.data.Dataset.from_tensors`. The former will keep the data in Python and fetch it via `tf.py_function` which can have performance implications, whereas the latter will bundle a copy of the data as one large `tf.constant()` node in the graph, which can have memory implications.\n", - "\n", - "Reading data from files via TFRecordDataset/CsvDataset/etc. is the most effective way to consume data, as then TensorFlow itself can manage the asynchronous loading and prefetching of data, without having to involve Python." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tRdlnCfV_UTn" - }, - "source": [ - "## Automatic Control Dependencies\n", - "\n", - "A very appealing property of functions as the programming model, over a general dataflow graph, is that functions can give the runtime more information about what was the intended behavior of the code.\n", - "\n", - "For example, when writing code which has multiple reads and writes to the same variables, a dataflow graph might not naturally encode the originally intended order of operations. In `tf.function`, we resolve ambiguities in execution order by referring to the execution order of statements in the original Python code. This way, ordering of stateful operations in a `tf.function` replicates the semantics of Eager mode.\n", - "\n", - "This means there's no need to add manual control dependencies; `tf.function` is smart enough to add the minimal set of necessary and sufficient control dependencies for your code to run correctly." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SASm0ss8erVX" - }, - "outputs": [], - "source": [ - "# Automatic control dependencies\n", - "\n", - "a = tf.Variable(1.0)\n", - "b = tf.Variable(2.0)\n", - "\n", - "@tf.function\n", - "def f(x, y):\n", - " a.assign(y * b)\n", - " b.assign_add(x * a)\n", - " return a + b\n", - "\n", - "f(1.0, 2.0) # 10.0\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lPr_6mK_AQWL" - }, - "source": [ - "## Variables\n", - "\n", - "We can use the same idea of leveraging the intended execution order of the code to make variable creation and utilization very easy in `tf.function`. There is one very important caveat, though, which is that with variables it's possible to write code which behaves differently in eager mode and graph mode.\n", - "\n", - "Specifically, this will happen when you create a new Variable with each call. Due to tracing semantics, `tf.function` will reuse the same variable each call, but eager mode will create a new variable with each call. To guard against this mistake, `tf.function` will raise an error if it detects dangerous variable creation behavior." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Tx0Vvnb_9OB-" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def f(x):\n", - " v = tf.Variable(1.0)\n", - " v.assign_add(x)\n", - " return v\n", - "\n", - "with assert_raises(ValueError):\n", - " f(1.0)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ot8o7PeFIINo" - }, - "source": [ - "Non-ambiguous code is ok though" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DKzNjVg8h4ao" - }, - "outputs": [], - "source": [ - "v = tf.Variable(1.0)\n", - "\n", - "@tf.function\n", - "def f(x):\n", - " return v.assign_add(x)\n", - "\n", - "print(f(1.0)) # 2.0\n", - "print(f(2.0)) # 4.0\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KYm6-5GCILXQ" - }, - "source": [ - "You can also create variables inside a tf.function as long as we can prove\n", - "that those variables are created only the first time the function is executed." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HQrG5_kOiKl_" - }, - "outputs": [], - "source": [ - "class C:\n", - " pass\n", - "\n", - "obj = C()\n", - "obj.v = None\n", - "\n", - "@tf.function\n", - "def g(x):\n", - " if obj.v is None:\n", - " obj.v = tf.Variable(1.0)\n", - " return obj.v.assign_add(x)\n", - "\n", - "print(g(1.0)) # 2.0\n", - "print(g(2.0)) # 4.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SDT4oKJaIim8" - }, - "source": [ - "Variable initializers can depend on function arguments and on values of other\n", - "variables. We can figure out the right initialization order using the same\n", - "method we use to generate control dependencies." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_IOVc1eujMH2" - }, - "outputs": [], - "source": [ - "state = []\n", - "@tf.function\n", - "def fn(x):\n", - " if not state:\n", - " state.append(tf.Variable(2.0 * x))\n", - " state.append(tf.Variable(state[0] * 3.0))\n", - " return state[0] * x * state[1]\n", - "\n", - "print(fn(tf.constant(1.0)))\n", - "print(fn(tf.constant(3.0)))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5f05Vr_YBUCz" - }, - "source": [ - "## Using AutoGraph\n", - "\n", - "The [autograph](https://www.tensorflow.org/guide/function) library is fully integrated with `tf.function`, and it will rewrite conditionals and loops which depend on Tensors to run dynamically in the graph.\n", - "\n", - "`tf.cond` and `tf.while_loop` continue to work with `tf.function`, but code with control flow is often easier to write and understand when written in imperative style." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yCQTtTPTW3WF" - }, - "outputs": [], - "source": [ - "# Simple loop\n", - "\n", - "@tf.function\n", - "def f(x):\n", - " while tf.reduce_sum(x) \u003e 1:\n", - " tf.print(x)\n", - " x = tf.tanh(x)\n", - " return x\n", - "\n", - "f(tf.random.uniform([5]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KxwJ8znPI0Cg" - }, - "source": [ - "If you're curious you can inspect the code autograph generates.\n", - "It feels like reading assembly language, though." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jlQD1ffRXJhl" - }, - "outputs": [], - "source": [ - "def f(x):\n", - " while tf.reduce_sum(x) \u003e 1:\n", - " tf.print(x)\n", - " x = tf.tanh(x)\n", - " return x\n", - "\n", - "print(tf.autograph.to_code(f))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xgKmkrNTZSyz" - }, - "source": [ - "## AutoGraph: Conditionals\n", - "\n", - "AutoGraph will convert `if` statements into the equivalent `tf.cond` calls.\n", - "\n", - "This substitution is made if the condition is a Tensor. Otherwise, the conditional is executed during tracing." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "20WlM9T2I9EV" - }, - "source": [ - "Here is a function that checks if the resulting graph uses `tf.cond`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "E-7KllizZYsy" - }, - "outputs": [], - "source": [ - "def test_tf_cond(f, *args):\n", - " g = f.get_concrete_function(*args).graph\n", - " if any(node.name == 'cond' for node in g.as_graph_def().node):\n", - " print(\"{}({}) uses tf.cond.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n", - " else:\n", - " print(\"{}({}) executes normally.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n", - "\n", - " print(\" result: \",f(*args).numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DlqiutEEJHOe" - }, - "source": [ - "This substitution is made if the condition is a Tensor. Otherwise, the conditional is executed during tracing.\n", - "\n", - "Passing a python `True` executes the conditional normally:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fCMywOXwJLIQ" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def dropout(x, training=True):\n", - " if training:\n", - " x = tf.nn.dropout(x, rate=0.5)\n", - " return x" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "68D2RZ17JM8u" - }, - "outputs": [], - "source": [ - "test_tf_cond(dropout, tf.ones([10], dtype=tf.float32), True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WEz0QYucJPBa" - }, - "source": [ - "But passing a tensor replaces the python `if` with a `tf.cond`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "o86paGR-Zadi" - }, - "outputs": [], - "source": [ - "test_tf_cond(dropout, tf.ones([10], dtype=tf.float32), tf.constant(True))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5xFLfdApZh8q" - }, - "source": [ - "`tf.cond` has a number of subtleties.\n", - "\n", - "it works by tracing both sides of the conditional, and then choosing the appropriate branch at runtime, depending on the condition. Tracing both sides can result in unexpected execution of Python code." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VTMoZEVaZiwk" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def f(x):\n", - " if x \u003e 0:\n", - " x = x + 1.\n", - " print(\"Tracing `then` branch\")\n", - " else:\n", - " x = x - 1.\n", - " print(\"Tracing `else` branch\")\n", - " return x" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HqBVIZWb0Qzn" - }, - "outputs": [], - "source": [ - "f(-1.0).numpy()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BIMfbXlW0QdP" - }, - "outputs": [], - "source": [ - "f(1.0).numpy()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2nBnJ42v0Pvq" - }, - "outputs": [], - "source": [ - "f(tf.constant(1.0)).numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zyzzvtN5Jfpb" - }, - "source": [ - "It requires that if one branch creates a tensor used downstream, the other branch must also create that tensor." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "k_dxWHeFZlaQ" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def f():\n", - " if tf.constant(True):\n", - " x = tf.ones([3, 3])\n", - " return x\n", - "\n", - "# Throws an error because both branches need to define `x`.\n", - "with assert_raises(ValueError):\n", - " f()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wP-LZP6cztnu" - }, - "source": [ - "If you want to be sure that a particular section of control flow is never converted by autograph, then explicitly convert the object to a python type so an error is raised instead: " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iG_VDavjzrzV" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def f(x, y):\n", - " if bool(x):\n", - " y = y + 1.\n", - " print(\"Tracing `then` branch\")\n", - " else:\n", - " y = y - 1.\n", - " print(\"Tracing `else` branch\")\n", - " return y" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kQ4CRP9T0rH2" - }, - "outputs": [], - "source": [ - "f(True, 0).numpy()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ww9tCzHy0rkv" - }, - "outputs": [], - "source": [ - "f(False, 0).numpy()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ppuV7iug0r7i" - }, - "outputs": [], - "source": [ - "with assert_raises(TypeError):\n", - " f(tf.constant(True), 0.0)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yho4J0a0ZkQS" - }, - "source": [ - "## AutoGraph and loops\n", - "\n", - "AutoGraph has a few simple rules for converting loops.\n", - "\n", - "- `for`: Convert if the iterable is a tensor\n", - "- `while`: Convert if the while condition depends on a tensor\n", - "\n", - "If a loop is converted, it will be dynamically unrolled with `tf.while_loop`, or in the special case of a `for x in tf.data.Dataset`, transformed into `tf.data.Dataset.reduce`.\n", - "\n", - "If a loop is _not_ converted, it will be statically unrolled " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OyzGNQAuZsky" - }, - "outputs": [], - "source": [ - "def test_dynamically_unrolled(f, *args):\n", - " g = f.get_concrete_function(*args).graph\n", - " if any(node.name == 'while' for node in g.as_graph_def().node):\n", - " print(\"{}({}) uses tf.while_loop.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n", - " elif any(node.name == 'ReduceDataset' for node in g.as_graph_def().node):\n", - " print(\"{}({}) uses tf.data.Dataset.reduce.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n", - " else:\n", - " print(\"{}({}) gets unrolled.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KFO1BSN9JkRP" - }, - "source": [ - "### For loops\n", - "\n", - "Here is a `tf.function` that demonstrates static unrolling:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "frecgTco_00V" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def for_in_range():\n", - " x = 0\n", - " for i in range(5):\n", - " x += i\n", - " return x\n", - "\n", - "test_dynamically_unrolled(for_in_range)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "PMdl0azc_5d4" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def for_in_tfrange():\n", - " x = tf.constant(0, dtype=tf.int32)\n", - " for i in tf.range(5):\n", - " x += i\n", - " return x\n", - "\n", - "test_dynamically_unrolled(for_in_tfrange)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Q7tmncQTZt6_" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def for_in_tfdataset():\n", - " x = tf.constant(0, dtype=tf.int64)\n", - " for i in tf.data.Dataset.range(5):\n", - " x += i\n", - " return x\n", - "\n", - "test_dynamically_unrolled(for_in_tfdataset)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eyPzDYiJAC8f" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def while_py_cond():\n", - " x = 5\n", - " while x \u003e 0:\n", - " x -= 1\n", - " return x\n", - "\n", - "test_dynamically_unrolled(while_py_cond)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l6s7aU-padY5" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def while_tf_cond():\n", - " x = tf.constant(5)\n", - " while x \u003e 0:\n", - " x -= 1\n", - " return x\n", - "\n", - "test_dynamically_unrolled(while_tf_cond)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dSr64Xn6ap-S" - }, - "source": [ - " If you have a `break` or early `return` clause that depends on a tensor, the top-level condition or iterable should also be a tensor.\n", - "\n", - "Compare the following examples:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hG2Fe_OEAwpY" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def while_py_true_py_break(x):\n", - " while True: # py true\n", - " if x == 0: # py break\n", - " break\n", - " x -= 1\n", - " return x\n", - "\n", - "test_dynamically_unrolled(while_py_true_py_break, 5)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Sr2cn5bY_E_9" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def buggy_while_py_true_tf_break(x):\n", - " while True: # py true\n", - " if tf.equal(x, 0): # tf break\n", - " break\n", - " x -= 1\n", - " return x\n", - "\n", - "with assert_raises(TypeError):\n", - " test_dynamically_unrolled(buggy_while_py_true_tf_break, 5)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Q-VirD-5avdZ" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def while_tf_true_tf_break(x):\n", - " while tf.constant(True): # tf true\n", - " if x == 0: # py break\n", - " break\n", - " x -= 1\n", - " return x\n", - "\n", - "test_dynamically_unrolled(while_tf_true_tf_break, 5)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Upx5J0j8_Ldu" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def buggy_py_for_tf_break():\n", - " x = 0\n", - " for i in range(5): # py for\n", - " if tf.equal(i, 3): # tf break\n", - " break\n", - " x += i\n", - " return x\n", - "\n", - "with assert_raises(TypeError):\n", - " test_dynamically_unrolled(buggy_py_for_tf_break)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GQHbodav_QMt" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def tf_for_py_break():\n", - " x = 0\n", - " for i in tf.range(5): # tf for\n", - " if i == 3: # py break\n", - " break\n", - " x += i\n", - " return x\n", - "\n", - "test_dynamically_unrolled(tf_for_py_break)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hyksHW9TCukR" - }, - "source": [ - "In order to accumulate results from a dynamically unrolled loop, you'll want to use `tf.TensorArray`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HJ3Vb3dXfefN" - }, - "outputs": [], - "source": [ - "batch_size = 2\n", - "seq_len = 3\n", - "feature_size = 4\n", - "\n", - "def rnn_step(inp, state):\n", - " return inp + state\n", - "\n", - "@tf.function\n", - "def dynamic_rnn(rnn_step, input_data, initial_state):\n", - " # [batch, time, features] -\u003e [time, batch, features]\n", - " input_data = tf.transpose(input_data, [1, 0, 2])\n", - " max_seq_len = input_data.shape[0]\n", - "\n", - " states = tf.TensorArray(tf.float32, size=max_seq_len)\n", - " state = initial_state\n", - " for i in tf.range(max_seq_len):\n", - " state = rnn_step(input_data[i], state)\n", - " states = states.write(i, state)\n", - " return tf.transpose(states.stack(), [1, 0, 2])\n", - " \n", - "dynamic_rnn(rnn_step,\n", - " tf.random.uniform([batch_size, seq_len, feature_size]),\n", - " tf.zeros([batch_size, feature_size]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9gmLpHY-bkly" - }, - "source": [ - "### Gotcha's\n", - "\n", - "As with `tf.cond`, `tf.while_loop` also comes with a number of subtleties.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FJdfznhhKO7D" - }, - "source": [ - "#### Zero iterations\n", - "\n", - "Since a loop can execute 0 times, all tensors used downstream of the while_loop must be initialized above the loop.\n", - "\n", - "Here is an example of incorrect code:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "CocT5RHwblrQ" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def buggy_loop_var_uninitialized():\n", - " for i in tf.range(3):\n", - " x = i\n", - " return x\n", - "\n", - "with assert_raises(ValueError):\n", - " buggy_loop_var_uninitialized()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ncr7tRZ1KWh9" - }, - "source": [ - "And the correct version:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Wm7wIKXcCDGf" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def f():\n", - " x = tf.constant(0)\n", - " for i in tf.range(3):\n", - " x = i\n", - " return x\n", - "\n", - "f()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CM7qXVY0KZHB" - }, - "source": [ - "#### Consistent shapes and types\n", - "\n", - "The shape/dtypes of all loop variables must stay consistent with each iteration.\n", - "\n", - "Here is an incorrect example that attempts to change a tensor's type:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FSftc9cCbpAo" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def buggy_loop_type_changes():\n", - " x = tf.constant(0, dtype=tf.float32)\n", - " for i in tf.range(3): # Yields tensors of type tf.int32...\n", - " x = i\n", - " return x\n", - "\n", - "with assert_raises(TypeError):\n", - " buggy_loop_type_changes()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M5l90NAHKsUM" - }, - "source": [ - "Here is an incorrect example that attempts to change a Tensor's shape while iterating:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kWF189prbuK0" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def buggy_concat():\n", - " x = tf.ones([0, 10])\n", - " for i in tf.range(5):\n", - " x = tf.concat([x, tf.ones([1, 10])], axis=0)\n", - " return x\n", - "\n", - "with assert_raises(ValueError):\n", - " buggy_concat()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "miYnYcznCHeV" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def concat_with_padding():\n", - " x = tf.zeros([5, 10])\n", - " for i in tf.range(5):\n", - " x = tf.concat([x[:i], tf.ones([1, 10]), tf.zeros([4-i, 10])], axis=0)\n", - " x.set_shape([5, 10])\n", - " return x\n", - "\n", - "concat_with_padding()\n" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [ - "ISubpr_SSsiM" - ], - "name": "performace.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/distribute/custom_training.ipynb b/site/en/tutorials/distribute/custom_training.ipynb index 3322b80d73f..d14b0ac003c 100644 --- a/site/en/tutorials/distribute/custom_training.ipynb +++ b/site/en/tutorials/distribute/custom_training.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MhoQ0WE77laV" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "_ckMIh7O7s6D" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jYysdyb-CaWM" }, "source": [ @@ -47,56 +43,45 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "S5Uhzt6vVIB2" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/distribute/custom_training\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/distribute/custom_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/distribute/custom_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/distribute/custom_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FbVhjPpzn6BM" }, "source": [ - "This tutorial demonstrates how to use [`tf.distribute.Strategy`](https://www.tensorflow.org/guide/distributed_training) with custom training loops. We will train a simple CNN model on the fashion MNIST dataset. The fashion MNIST dataset contains 60000 train images of size 28 x 28 and 10000 test images of size 28 x 28.\n", + "This tutorial demonstrates how to use `tf.distribute.Strategy`—a TensorFlow API that provides an abstraction for [distributing your training](../../guide/distributed_training.ipynb) across multiple processing units (GPUs, multiple machines, or TPUs)—with custom training loops. In this example, you will train a simple convolutional neural network on the [Fashion MNIST dataset](https://github.com/zalandoresearch/fashion-mnist) containing 70,000 images of size 28 x 28.\n", "\n", - "We are using custom training loops to train our model because they give us flexibility and a greater control on training. Moreover, it is easier to debug the model and the training loop." + "[Custom training loops](../customization/custom_training_walkthrough.ipynb) provide flexibility and a greater control on training. They also make it easier to debug the model and the training loop." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dzLKpmZICaWN" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "# Import TensorFlow\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", "\n", "# Helper libraries\n", @@ -109,19 +94,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MM6W__qraV55" }, "source": [ - "## Download the fashion MNIST dataset" + "## Download the Fashion MNIST dataset" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7MqDQO0KCaWS" }, "outputs": [], @@ -130,14 +112,14 @@ "\n", "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\n", "\n", - "# Adding a dimension to the array -\u003e new shape == (28, 28, 1)\n", - "# We are doing this because the first layer in our model is a convolutional\n", + "# Add a dimension to the array -> new shape == (28, 28, 1)\n", + "# This is done because the first layer in our model is a convolutional\n", "# layer and it requires a 4D input (batch_size, height, width, channels).\n", "# batch_size dimension will be added later on.\n", "train_images = train_images[..., None]\n", "test_images = test_images[..., None]\n", "\n", - "# Getting the images in [0, 1] range.\n", + "# Scale the images to the [0, 1] range.\n", "train_images = train_images / np.float32(255)\n", "test_images = test_images / np.float32(255)" ] @@ -145,7 +127,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4AXoHhrsbdF3" }, "source": [ @@ -155,75 +136,57 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5mVuLZhbem8d" }, "source": [ "How does `tf.distribute.MirroredStrategy` strategy work?\n", "\n", - "* All the variables and the model graph is replicated on the replicas.\n", + "* All the variables and the model graph are replicated across the replicas.\n", "* Input is evenly distributed across the replicas.\n", "* Each replica calculates the loss and gradients for the input it received.\n", - "* The gradients are synced across all the replicas by summing them.\n", + "* The gradients are synced across all the replicas by **summing** them.\n", "* After the sync, the same update is made to the copies of the variables on each replica.\n", "\n", - "Note: You can put all the code below inside a single scope. We are dividing it into several code cells for illustration purposes.\n" + "Note: You can put all the code below inside a single scope. This example divides it into several code cells for illustration purposes.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "F2VeZUWUj5S4" }, "outputs": [], "source": [ - "# If the list of devices is not specified in the\n", - "# `tf.distribute.MirroredStrategy` constructor, it will be auto-detected.\n", + "# If the list of devices is not specified in\n", + "# `tf.distribute.MirroredStrategy` constructor, they will be auto-detected.\n", "strategy = tf.distribute.MirroredStrategy()" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ZngeM_2o0_JO" }, "outputs": [], "source": [ - "print ('Number of devices: {}'.format(strategy.num_replicas_in_sync))" + "print('Number of devices: {}'.format(strategy.num_replicas_in_sync))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "k53F5I_IiGyI" }, "source": [ "## Setup input pipeline" ] }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Qb6nDgxiN_n" - }, - "source": [ - "Export the graph and the variables to the platform-agnostic SavedModel format. After your model is saved, you can load it with or without the scope." - ] - }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "jwJtsCQhHK-E" }, "outputs": [], @@ -239,7 +202,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "J7fj3GskHC8g" }, "source": [ @@ -248,16 +210,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WYrMNNDhAvVl" }, "outputs": [], "source": [ - "train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(BUFFER_SIZE).batch(GLOBAL_BATCH_SIZE) \n", - "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE) \n", + "train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(BUFFER_SIZE).batch(GLOBAL_BATCH_SIZE)\n", + "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE)\n", "\n", "train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)\n", "test_dist_dataset = strategy.experimental_distribute_dataset(test_dataset)" @@ -266,34 +226,38 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bAXAo_wWbWSb" }, "source": [ "## Create the model\n", "\n", - "Create a model using `tf.keras.Sequential`. You can also use the Model Subclassing API to do this." + "Create a model using `tf.keras.Sequential`. You can also use the [Model Subclassing API](https://www.tensorflow.org/guide/keras/custom_layers_and_models) or the [functional API](https://www.tensorflow.org/guide/keras/functional) to do this." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "9ODch-OFCaW4" }, "outputs": [], "source": [ "def create_model():\n", + " regularizer = tf.keras.regularizers.L2(1e-5)\n", " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", + " tf.keras.layers.Conv2D(32, 3,\n", + " activation='relu',\n", + " kernel_regularizer=regularizer),\n", " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Conv2D(64, 3, activation='relu'),\n", + " tf.keras.layers.Conv2D(64, 3,\n", + " activation='relu',\n", + " kernel_regularizer=regularizer),\n", " tf.keras.layers.MaxPooling2D(),\n", " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", + " tf.keras.layers.Dense(64,\n", + " activation='relu',\n", + " kernel_regularizer=regularizer),\n", + " tf.keras.layers.Dense(10, kernel_regularizer=regularizer)\n", " ])\n", "\n", " return model" @@ -301,10 +265,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "9iagoTBfijUz" }, "outputs": [], @@ -317,66 +279,117 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "e-wlFFZbP33n" + "id": "0-VVTqDEICrl" }, "source": [ "## Define the loss function\n", "\n", - "Normally, on a single machine with 1 GPU/CPU, loss is divided by the number of examples in the batch of input.\n", + "Recall that the loss function consists of one or two parts:\n", "\n", - "*So, how should the loss be calculated when using a `tf.distribute.Strategy`?*\n", + " * The **prediction loss** measures how far off the model's predictions are from the training labels for a batch of training examples. It is computed for each labeled example and then reduced across the batch by computing the average value.\n", + " * Optionally, **regularization loss** terms can be added to the prediction loss, to steer the model away from overfitting the training data. A common choice is L2 regularization, which adds a small fixed multiple of the sum of squares of all model weights, independent of the number of examples. The model above uses L2 regularization to demonstrate its handling in the training loop below.\n", "\n", - "* For an example, let's say you have 4 GPU's and a batch size of 64. One batch of input is distributed\n", - "across the replicas (4 GPUs), each replica getting an input of size 16.\n", + "For training on a single machine with a single GPU/CPU, this works as follows:\n", "\n", - "* The model on each replica does a forward pass with its respective input and calculates the loss. Now, instead of dividing the loss by the number of examples in its respective input (BATCH_SIZE_PER_REPLICA = 16), the loss should be divided by the GLOBAL_BATCH_SIZE (64).\n", + " * The prediction loss is computed for each example in the batch, summed across the batch, and then divided by the batch size.\n", + " * The regularization loss is added to the prediction loss.\n", + " * The gradient of the total loss is computed w.r.t. each model weight, and the optimizer updates each model weight from the corresponding gradient.\n", "\n", - "*Why do this?*\n", + "With `tf.distribute.Strategy`, the input batch is split between replicas.\n", + "For example, let's say you have 4 GPUs, each with one replica of the model. One batch of 256 input examples is distributed evenly across the 4 replicas, so each replica gets a batch of size 64: We have `256 = 4*64`, or generally `GLOBAL_BATCH_SIZE = num_replicas_in_sync * BATCH_SIZE_PER_REPLICA`.\n", + "\n", + "Each replica computes the loss from the training examples it gets and computes the gradients of the loss w.r.t. each model weight. The optimizer takes care that these **gradients are summed up across replicas** before using them to update the copies of the model weights on each replica.\n", + "\n", + "*So, how should the loss be calculated when using a `tf.distribute.Strategy`?*\n", "\n", - "* This needs to be done because after the gradients are calculated on each replica, they are synced across the replicas by **summing** them.\n", + " * Each replica computes the prediction loss for all examples distributed to it, sums up the results and divides them by `num_replicas_in_sync * BATCH_SIZE_PER_REPLICA`, or equivently, `GLOBAL_BATCH_SIZE`.\n", + " * Each replica compues the regularization loss(es) and divides them by\n", + " `num_replicas_in_sync`.\n", "\n", + "Compared to non-distributed training, all per-replica loss terms are scaled down by a factor of `1/num_replicas_in_sync`. On the other hand, all loss terms -- or rather, their gradients -- are summed across that number of replicas before the optimizer applies them. In effect, the optimizer on each replica uses the same gradients as if a non-distributed computation with `GLOBAL_BATCH_SIZE` had happened. This is consistent with the distributed and undistributed behavior of Keras `Model.fit`. See the [Distributed training with Keras](./keras.ipynb) tutorial on how a larger gloabl batch size enables to scale up the learning rate." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e-wlFFZbP33n" + }, + "source": [ "*How to do this in TensorFlow?*\n", - "* If you're writing a custom training loop, as in this tutorial, you should sum the per example losses and divide the sum by the GLOBAL_BATCH_SIZE: \n", - "`scale_loss = tf.reduce_sum(loss) * (1. / GLOBAL_BATCH_SIZE)`\n", - "or you can use `tf.nn.compute_average_loss` which takes the per example loss,\n", - "optional sample weights, and GLOBAL_BATCH_SIZE as arguments and returns the scaled loss.\n", "\n", - "* If you are using regularization losses in your model then you need to scale\n", - "the loss value by number of replicas. You can do this by using the `tf.nn.scale_regularization_loss` function.\n", + " * Loss reduction and scaling is done automatically in Keras `Model.compile` and `Model.fit`\n", + "\n", + " * If you're writing a custom training loop, as in this tutorial, you should sum the per-example losses and divide the sum by the global batch size using `tf.nn.compute_average_loss`, which takes the per-example losses and\n", + "optional sample weights as arguments and returns the scaled loss.\n", "\n", - "* Using `tf.reduce_mean` is not recommended. Doing so divides the loss by actual per replica batch size which may vary step to step.\n", + " * If using `tf.keras.losses` classes (as in the example below), the loss reduction needs to be explicitly specified to be one of `NONE` or `SUM`. The default `AUTO` and `SUM_OVER_BATCH_SIZE` are disallowed outside `Model.fit`.\n", + " * `AUTO` is disallowed because the user should explicitly think about what reduction they want to make sure it is correct in the distributed case.\n", + " * `SUM_OVER_BATCH_SIZE` is disallowed because currently it would only divide by per replica batch size, and leave the dividing by number of replicas to the user, which might be easy to miss. So, instead, you need to do the reduction yourself explicitly.\n", "\n", - "* This reduction and scaling is done automatically in keras `model.compile` and `model.fit`\n", + " * If you're writing a custom training loop for a model with a non-empty list of `Model.losses` (e.g., weight regularizers), you should sum them up and divide the sum by the number of replicas. You can do this by using the `tf.nn.scale_regularization_loss` function. The model code itself remains unaware of the number of replicas.\n", "\n", - "* If using `tf.keras.losses` classes (as in the example below), the loss reduction needs to be explicitly specified to be one of `NONE` or `SUM`. `AUTO` and `SUM_OVER_BATCH_SIZE` are disallowed when used with `tf.distribute.Strategy`. `AUTO` is disallowed because the user should explicitly think about what reduction they want to make sure it is correct in the distributed case. `SUM_OVER_BATCH_SIZE` is disallowed because currently it would only divide by per replica batch size, and leave the dividing by number of replicas to the user, which might be easy to miss. So instead we ask the user do the reduction themselves explicitly." + " However, models can define input-dependent regularization losses with Keras APIs such as `Layer.add_loss(...)` and `Layer(activity_regularizer=...)`. For `Layer.add_loss(...)`, it falls on the modeling code to perform the division of the summed per-example terms by the per-replica(!) batch size, e.g., by using `tf.math.reduce_mean()`." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "R144Wci782ix" }, "outputs": [], "source": [ "with strategy.scope():\n", - " # Set reduction to `none` so we can do the reduction afterwards and divide by\n", - " # global batch size.\n", + " # Set reduction to `NONE` so you can do the reduction yourself.\n", " loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n", + " from_logits=True,\n", " reduction=tf.keras.losses.Reduction.NONE)\n", - " # or loss_fn = tf.keras.losses.sparse_categorical_crossentropy\n", - " def compute_loss(labels, predictions):\n", + " def compute_loss(labels, predictions, model_losses):\n", " per_example_loss = loss_object(labels, predictions)\n", - " return tf.nn.compute_average_loss(per_example_loss, global_batch_size=GLOBAL_BATCH_SIZE)" + " loss = tf.nn.compute_average_loss(per_example_loss)\n", + " if model_losses:\n", + " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n", + " return loss" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6pM96bqQY52D" + }, + "source": [ + "### Special cases\n", + "\n", + "Advanced users should also consider the following special cases.\n", + "\n", + " * Input batches shorter than `GLOBAL_BATCH_SIZE` create unpleasant corner cases in several places. In practice, it often works best to avoid them by allowing batches to span epoch boundaries using `Dataset.repeat().batch()` and defining approximate epochs by step counts, not dataset ends. Alternatively, `Dataset.batch(drop_remainder=True)` maintains the notion of epoch but drops the last few examples.\n", + "\n", + " For illustration, this example goes the harder route and allows short batches, so that each training epoch contains each training example exactly once.\n", + " \n", + " Which denominator should be used by `tf.nn.compute_average_loss()`?\n", + "\n", + " * By default, in the example code above and equivalently in `Keras.fit()`, the sum of prediction losses is divided by `num_replicas_in_sync` times the actual batch size seen on the replica (with empty batches silently ignored). This preserves the balance between the prediction loss on the one hand and the regularization losses on the other hand. It is particularly appropriate for models that use input-dependent regularization losses. Plain L2 regularization just superimposes weight decay onto the gradients of the prediction loss and is less in need of such a balance.\n", + " * In practice, many custom training loops pass as a constant Python value into `tf.nn.compute_average_loss(..., global_batch_size=GLOBAL_BATCH_SIZE)` to use it as the denominator. This preserves the relative weighting of training examples between batches. Without it, the smaller denominator in short batches effectively upweights the examples in those. (Before TensorFlow 2.13, this was also needed to avoid NaNs in case some replica received an actual batch size of zero.)\n", + " \n", + " Both options are equivalent if short batches are avoided, as suggested above.\n", + "\n", + " * Multi-dimensional `labels` require you to average the `per_example_loss` across the number of predictions in each example. Consider a classification task for all pixels of an input image, with `predictions` of shape `(batch_size, H, W, n_classes)` and `labels` of shape `(batch_size, H, W)`. You will need to update `per_example_loss` like: `per_example_loss /= tf.cast(tf.reduce_prod(tf.shape(labels)[1:]), tf.float32)`\n", + "\n", + " Caution: **Verify the shape of your loss**.\n", + " Loss functions in `tf.losses`/`tf.keras.losses` typically\n", + " return the average over the last dimension of the input. The loss\n", + " classes wrap these functions. Passing `reduction=Reduction.NONE` when\n", + " creating an instance of a loss class means \"no **additional** reduction\".\n", + " For categorical losses with an example input shape of `[batch, W, H, n_classes]` the `n_classes`\n", + " dimension is reduced. For pointwise losses like\n", + " `losses.mean_squared_error` or `losses.binary_crossentropy` include a\n", + " dummy axis so that `[batch, W, H, 1]` is reduced to `[batch, W, H]`. Without\n", + " the dummy axis `[batch, W, H]` will be incorrectly reduced to `[batch, W]`." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "w8y54-o9T2Ni" }, "source": [ @@ -387,10 +400,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "zt3AHb46Tr3w" }, "outputs": [], @@ -407,7 +418,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iuKuNXPORfqJ" }, "source": [ @@ -416,127 +426,116 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "OrMmakq5EqeQ" }, "outputs": [], "source": [ - "# model and optimizer must be created under `strategy.scope`.\n", + "# A model, an optimizer, and a checkpoint must be created under `strategy.scope`.\n", "with strategy.scope():\n", " model = create_model()\n", "\n", - " optimizer = tf.keras.optimizers.Adam()\n", + " optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n", "\n", " checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3UX43wUu04EL" }, "outputs": [], "source": [ - "with strategy.scope():\n", - " def train_step(inputs):\n", - " images, labels = inputs\n", + "def train_step(inputs):\n", + " images, labels = inputs\n", "\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images, training=True)\n", - " loss = compute_loss(labels, predictions)\n", + " with tf.GradientTape() as tape:\n", + " predictions = model(images, training=True)\n", + " loss = compute_loss(labels, predictions, model.losses)\n", "\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", + " gradients = tape.gradient(loss, model.trainable_variables)\n", + " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", "\n", - " train_accuracy.update_state(labels, predictions)\n", - " return loss \n", + " train_accuracy.update_state(labels, predictions)\n", + " return loss\n", "\n", - " def test_step(inputs):\n", - " images, labels = inputs\n", + "def test_step(inputs):\n", + " images, labels = inputs\n", "\n", - " predictions = model(images, training=False)\n", - " t_loss = loss_object(labels, predictions)\n", + " predictions = model(images, training=False)\n", + " t_loss = loss_object(labels, predictions)\n", "\n", - " test_loss.update_state(t_loss)\n", - " test_accuracy.update_state(labels, predictions)" + " test_loss.update_state(t_loss)\n", + " test_accuracy.update_state(labels, predictions)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gX975dMSNw0e" }, "outputs": [], "source": [ - "with strategy.scope():\n", - " # `experimental_run_v2` replicates the provided computation and runs it\n", - " # with the distributed input.\n", - " @tf.function\n", - " def distributed_train_step(dataset_inputs):\n", - " per_replica_losses = strategy.experimental_run_v2(train_step,\n", - " args=(dataset_inputs,))\n", - " return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n", - " axis=None)\n", - " \n", - " @tf.function\n", - " def distributed_test_step(dataset_inputs):\n", - " return strategy.experimental_run_v2(test_step, args=(dataset_inputs,))\n", - "\n", - " for epoch in range(EPOCHS):\n", - " # TRAIN LOOP\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " for x in train_dist_dataset:\n", - " total_loss += distributed_train_step(x)\n", - " num_batches += 1\n", - " train_loss = total_loss / num_batches\n", - "\n", - " # TEST LOOP\n", - " for x in test_dist_dataset:\n", - " distributed_test_step(x)\n", - "\n", - " if epoch % 2 == 0:\n", - " checkpoint.save(checkpoint_prefix)\n", - "\n", - " template = (\"Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, \"\n", - " \"Test Accuracy: {}\")\n", - " print (template.format(epoch+1, train_loss,\n", - " train_accuracy.result()*100, test_loss.result(),\n", - " test_accuracy.result()*100))\n", - "\n", - " test_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_accuracy.reset_states()" + "# `run` replicates the provided computation and runs it\n", + "# with the distributed input.\n", + "@tf.function\n", + "def distributed_train_step(dataset_inputs):\n", + " per_replica_losses = strategy.run(train_step, args=(dataset_inputs,))\n", + " return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n", + " axis=None)\n", + "\n", + "@tf.function\n", + "def distributed_test_step(dataset_inputs):\n", + " return strategy.run(test_step, args=(dataset_inputs,))\n", + "\n", + "for epoch in range(EPOCHS):\n", + " # TRAIN LOOP\n", + " total_loss = 0.0\n", + " num_batches = 0\n", + " for x in train_dist_dataset:\n", + " total_loss += distributed_train_step(x)\n", + " num_batches += 1\n", + " train_loss = total_loss / num_batches\n", + "\n", + " # TEST LOOP\n", + " for x in test_dist_dataset:\n", + " distributed_test_step(x)\n", + "\n", + " if epoch % 2 == 0:\n", + " checkpoint.save(checkpoint_prefix)\n", + "\n", + " template = (\"Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, \"\n", + " \"Test Accuracy: {}\")\n", + " print(template.format(epoch + 1, train_loss,\n", + " train_accuracy.result() * 100, test_loss.result(),\n", + " test_accuracy.result() * 100))\n", + "\n", + " test_loss.reset_states()\n", + " train_accuracy.reset_states()\n", + " test_accuracy.reset_states()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Z1YvXqOpwy08" }, "source": [ - "Things to note in the example above:\n", + "### Things to note in the example above\n", "\n", - "* We are iterating over the `train_dist_dataset` and `test_dist_dataset` using a `for x in ...` construct.\n", + "* Iterate over the `train_dist_dataset` and `test_dist_dataset` using a `for x in ...` construct.\n", "* The scaled loss is the return value of the `distributed_train_step`. This value is aggregated across replicas using the `tf.distribute.Strategy.reduce` call and then across batches by summing the return value of the `tf.distribute.Strategy.reduce` calls.\n", - "* `tf.keras.Metrics` should be updated inside `train_step` and `test_step` that gets executed by `tf.distribute.Strategy.experimental_run_v2`.\n", - "*`tf.distribute.Strategy.experimental_run_v2` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can do `tf.distribute.Strategy.reduce` to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results` to get the list of values contained in the result, one per local replica.\n" + "* `tf.keras.Metrics` should be updated inside `train_step` and `test_step` that gets executed by `tf.distribute.Strategy.run`.\n", + "* `tf.distribute.Strategy.run` returns results from each local replica in the strategy, and there are multiple ways to consume this result. You can do `tf.distribute.Strategy.reduce` to get an aggregated value. You can also do `tf.distribute.Strategy.experimental_local_results` to get the list of values contained in the result, one per local replica.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-q5qp31IQD8t" }, "source": [ @@ -546,7 +545,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WNW2P00bkMGJ" }, "source": [ @@ -555,10 +553,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "pg3B-Cw_cn3a" }, "outputs": [], @@ -574,10 +570,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7qYii7KUYiSM" }, "outputs": [], @@ -590,10 +584,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "LeZ6eeWRoUNq" }, "outputs": [], @@ -604,14 +596,13 @@ "for images, labels in test_dataset:\n", " eval_step(images, labels)\n", "\n", - "print ('Accuracy after restoring the saved model without strategy: {}'.format(\n", - " eval_accuracy.result()*100))" + "print('Accuracy after restoring the saved model without strategy: {}'.format(\n", + " eval_accuracy.result() * 100))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "EbcI87EEzhzg" }, "source": [ @@ -619,82 +610,74 @@ "\n", "### Using iterators\n", "\n", - "If you want to iterate over a given number of steps and not through the entire dataset you can create an iterator using the `iter` call and explicity call `next` on the iterator. You can choose to iterate over the dataset both inside and outside the tf.function. Here is a small snippet demonstrating iteration of the dataset outside the tf.function using an iterator.\n" + "If you want to iterate over a given number of steps and not through the entire dataset, you can create an iterator using the `iter` call and explicitly call `next` on the iterator. You can choose to iterate over the dataset both inside and outside the `tf.function`. Here is a small snippet demonstrating iteration of the dataset outside the `tf.function` using an iterator.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7c73wGC00CzN" }, "outputs": [], "source": [ - "with strategy.scope():\n", - " for _ in range(EPOCHS):\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " train_iter = iter(train_dist_dataset)\n", + "for _ in range(EPOCHS):\n", + " total_loss = 0.0\n", + " num_batches = 0\n", + " train_iter = iter(train_dist_dataset)\n", "\n", - " for _ in range(10):\n", - " total_loss += distributed_train_step(next(train_iter))\n", - " num_batches += 1\n", - " average_train_loss = total_loss / num_batches\n", + " for _ in range(10):\n", + " total_loss += distributed_train_step(next(train_iter))\n", + " num_batches += 1\n", + " average_train_loss = total_loss / num_batches\n", "\n", - " template = (\"Epoch {}, Loss: {}, Accuracy: {}\")\n", - " print (template.format(epoch+1, average_train_loss, train_accuracy.result()*100))\n", - " train_accuracy.reset_states()" + " template = (\"Epoch {}, Loss: {}, Accuracy: {}\")\n", + " print(template.format(epoch + 1, average_train_loss, train_accuracy.result() * 100))\n", + " train_accuracy.reset_states()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GxVp48Oy0m6y" }, "source": [ - "### Iterating inside a tf.function\n", - "You can also iterate over the entire input `train_dist_dataset` inside a tf.function using the `for x in ...` construct or by creating iterators like we did above. The example below demonstrates wrapping one epoch of training in a tf.function and iterating over `train_dist_dataset` inside the function." + "### Iterating inside a `tf.function`\n", + "\n", + "You can also iterate over the entire input `train_dist_dataset` inside a `tf.function` using the `for x in ...` construct or by creating iterators like you did above. The example below demonstrates wrapping one epoch of training with a `@tf.function` decorator and iterating over `train_dist_dataset` inside the function." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "-REzmcXv00qm" }, "outputs": [], "source": [ - "with strategy.scope():\n", - " @tf.function\n", - " def distributed_train_epoch(dataset):\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " for x in dataset:\n", - " per_replica_losses = strategy.experimental_run_v2(train_step,\n", - " args=(x,))\n", - " total_loss += strategy.reduce(\n", - " tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n", - " num_batches += 1\n", - " return total_loss / tf.cast(num_batches, dtype=tf.float32)\n", + "@tf.function\n", + "def distributed_train_epoch(dataset):\n", + " total_loss = 0.0\n", + " num_batches = 0\n", + " for x in dataset:\n", + " per_replica_losses = strategy.run(train_step, args=(x,))\n", + " total_loss += strategy.reduce(\n", + " tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n", + " num_batches += 1\n", + " return total_loss / tf.cast(num_batches, dtype=tf.float32)\n", "\n", - " for epoch in range(EPOCHS):\n", - " train_loss = distributed_train_epoch(train_dist_dataset)\n", + "for epoch in range(EPOCHS):\n", + " train_loss = distributed_train_epoch(train_dist_dataset)\n", "\n", - " template = (\"Epoch {}, Loss: {}, Accuracy: {}\")\n", - " print (template.format(epoch+1, train_loss, train_accuracy.result()*100))\n", + " template = (\"Epoch {}, Loss: {}, Accuracy: {}\")\n", + " print(template.format(epoch + 1, train_loss, train_accuracy.result() * 100))\n", "\n", - " train_accuracy.reset_states()" + " train_accuracy.reset_states()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MuZGXiyC7ABR" }, "source": [ @@ -702,49 +685,51 @@ "\n", "Note: As a general rule, you should use `tf.keras.Metrics` to track per-sample values and avoid values that have been aggregated within a replica.\n", "\n", - "We do *not* recommend using `tf.metrics.Mean` to track the training loss across different replicas, because of the loss scaling computation that is carried out.\n", + "Because of the loss scaling computation that is carried out, it's not recommended to use `tf.keras.metrics.Mean` to track the training loss across different replicas.\n", "\n", "For example, if you run a training job with the following characteristics:\n", + "\n", "* Two replicas\n", "* Two samples are processed on each replica\n", "* Resulting loss values: [2, 3] and [4, 5] on each replica\n", "* Global batch size = 4\n", "\n", - "With loss scaling, you calculate the per-sample value of loss on each replica by adding the loss values, and then dividing by the global batch size. In this case: `(2 + 3) / 4 = 1.25` and `(4 + 5) / 4 = 2.25`. \n", + "With loss scaling, you calculate the per-sample value of loss on each replica by adding the loss values, and then dividing by the global batch size. In this case: `(2 + 3) / 4 = 1.25` and `(4 + 5) / 4 = 2.25`.\n", "\n", - "If you use `tf.metrics.Mean` to track loss across the two replicas, the result is different. In this example, you end up with a `total` of 3.50 and `count` of 2, which results in `total`/`count` = 1.75 when `result()` is called on the metric. Loss calculated with `tf.keras.Metrics` is scaled by an additional factor that is equal to the number of replicas in sync." + "If you use `tf.keras.metrics.Mean` to track loss across the two replicas, the result is different. In this example, you end up with a `total` of 3.50 and `count` of 2, which results in `total`/`count` = 1.75 when `result()` is called on the metric. Loss calculated with `tf.keras.Metrics` is scaled by an additional factor that is equal to the number of replicas in sync." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xisYJaV9KZTN" }, "source": [ - "### Examples and Tutorials\n", + "### Guide and examples\n", + "\n", "Here are some examples for using distribution strategy with custom training loops:\n", "\n", - "1. [Tutorial](../tutorials/distribute/training_loops.ipynb) to train MNIST using `MirroredStrategy`.\n", + "1. [Distributed training guide](../../guide/distributed_training)\n", "2. [DenseNet](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/densenet/distributed_train.py) example using `MirroredStrategy`.\n", - "1. [BERT](https://github.com/tensorflow/models/blob/master/official/bert/run_classifier.py) example trained using `MirroredStrategy` and `TPUStrategy`.\n", + "1. [BERT](https://github.com/tensorflow/models/blob/master/official/legacy/bert/run_classifier.py) example trained using `MirroredStrategy` and `TPUStrategy`.\n", "This example is particularly helpful for understanding how to load from a checkpoint and generate periodic checkpoints during distributed training etc.\n", "2. [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) example trained using `MirroredStrategy` that can be enabled using the `keras_use_ctl` flag.\n", "3. [NMT](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/nmt_with_attention/distributed_train.py) example trained using `MirroredStrategy`.\n", "\n", - "More examples listed in the [Distribution strategy guide](../../guide/distributed_training.ipynb#examples_and_tutorials)" + "You can find more examples listed under _Examples and tutorials_ in the [Distribution strategy guide](../../guide/distributed_training.ipynb)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6hEJNsokjOKs" }, "source": [ "## Next steps\n", "\n", - "Try out the new `tf.distribute.Strategy` API on your models." + "* Try out the new `tf.distribute.Strategy` API on your models.\n", + "* Visit the [Better performance with `tf.function`](../../guide/function.ipynb) and [TensorFlow Profiler](../../guide/profiler.md) guides to learn more about tools to optimize the performance of your TensorFlow models.\n", + "* Check out the [Distributed training in TensorFlow](../../guide/distributed_training.ipynb) guide, which provides an overview of the available distribution strategies." ] } ], @@ -752,10 +737,7 @@ "colab": { "collapsed_sections": [], "name": "custom_training.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/distribute/dtensor_keras_tutorial.ipynb b/site/en/tutorials/distribute/dtensor_keras_tutorial.ipynb new file mode 100644 index 00000000000..84f6478c2b5 --- /dev/null +++ b/site/en/tutorials/distribute/dtensor_keras_tutorial.ipynb @@ -0,0 +1,760 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Tce3stUlHN0L" + }, + "source": [ + "##### Copyright 2019 The TensorFlow Authors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "tuOe1ymfHZPu" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MT-LkFOl2axM" + }, + "source": [ + "# Using DTensors with Keras" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r6P32iYYV27b" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vTe9dcbUAwqx" + }, + "source": [ + "## Overview\n", + "\n", + "In this tutorial, you will learn how to use DTensors with Keras.\n", + "\n", + "Through DTensor integration with Keras, you can reuse your existing Keras layers and models to build and train distributed machine learning models.\n", + "\n", + "You will train a multi-layer classification model with the MNIST data. Setting the layout for subclassing model, Sequential model, and functional model will be demonstrated.\n", + "\n", + "This tutorial assumes that you have already read the [DTensor programing guide](/guide/dtensor_overview), and are familiar with basic DTensor concepts like `Mesh` and `Layout`.\n", + "\n", + "This tutorial is based on [Training a neural network on MNIST with Keras](https://www.tensorflow.org/datasets/keras_example)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "keIyP3IoA1o4" + }, + "source": [ + "## Setup\n", + "\n", + "DTensor (`tf.experimental.dtensor`) has been part of TensorFlow since the 2.9.0 release.\n", + "\n", + "First, install or upgrade TensorFlow Datasets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4dHik7NYA5vm" + }, + "outputs": [], + "source": [ + "!pip install --quiet --upgrade tensorflow-datasets" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VttBMZngDx8x" + }, + "source": [ + "Next, import TensorFlow and `dtensor`, and configure TensorFlow to use 8 virtual CPUs.\n", + "\n", + "Even though this example uses virtual CPUs, DTensor works the same way on CPU, GPU or TPU devices." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CodX6idGBGSm" + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "from tensorflow.experimental import dtensor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aAtvrpasDpDD" + }, + "outputs": [], + "source": [ + "def configure_virtual_cpus(ncpu):\n", + " phy_devices = tf.config.list_physical_devices('CPU')\n", + " tf.config.set_logical_device_configuration(\n", + " phy_devices[0], \n", + " [tf.config.LogicalDeviceConfiguration()] * ncpu)\n", + " \n", + "configure_virtual_cpus(8)\n", + "tf.config.list_logical_devices('CPU')\n", + "\n", + "devices = [f'CPU:{i}' for i in range(8)]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ogULE1OHtyd9" + }, + "source": [ + "## Deterministic pseudo-random number generators\n", + "One thing you should note is that DTensor API requires each of the running client to have the same random seeds, so that it could have deterministic behavior for initializing the weights. You can achieve this by setting the global seeds in keras via `tf.keras.utils.set_random_seed()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9u85YypguL8N" + }, + "outputs": [], + "source": [ + "tf.keras.backend.experimental.enable_tf_random_generator()\n", + "tf.keras.utils.set_random_seed(1337)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tO11XvPDAu3_" + }, + "source": [ + "## Creating a Data Parallel Mesh\n", + "\n", + "This tutorial demonstrates Data Parallel training. Adapting to Model Parallel training and Spatial Parallel training can be as simple as switching to a different set of `Layout` objects. Refer to the [Distributed training with DTensors](dtensor_ml_tutorial.ipynb) tutorial for more information on distributed training beyond Data Parallel.\n", + "\n", + "Data Parallel training is a commonly used parallel training scheme, also used by, for example, `tf.distribute.MirroredStrategy`.\n", + "\n", + "With DTensor, a Data Parallel training loop uses a `Mesh` that consists of a single 'batch' dimension, where each device runs a replica of the model that receives a shard from the global batch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6sT6s6z4j9H-" + }, + "outputs": [], + "source": [ + "mesh = dtensor.create_mesh([(\"batch\", 8)], devices=devices)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rouFcF6FE0aF" + }, + "source": [ + "As each device runs a full replica of the model, the model variables shall be fully replicated across the mesh (unsharded). As an example, a fully replicated Layout for a rank-2 weight on this `Mesh` would be as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "U8OxvkDKE1Nu" + }, + "outputs": [], + "source": [ + "example_weight_layout = dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh) # or\n", + "example_weight_layout = dtensor.Layout.replicated(mesh, rank=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6Bnic98RE0xi" + }, + "source": [ + "A layout for a rank-2 data tensor on this `Mesh` would be sharded along the first dimension (sometimes known as `batch_sharded`)," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PhYp0EKBFfxt" + }, + "outputs": [], + "source": [ + "example_data_layout = dtensor.Layout(['batch', dtensor.UNSHARDED], mesh) # or\n", + "example_data_layout = dtensor.Layout.batch_sharded(mesh, 'batch', rank=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4U-6n0DericV" + }, + "source": [ + "## Create Keras layers with layout\n", + "\n", + "In the data parallel scheme, you usually create your model weights with a fully replicated layout, so that each replica of the model can do calculations with the sharded input data. \n", + "\n", + "In order to configure the layout information for your layers' weights, Keras has exposed an extra parameter in the layer constructor for most of the built-in layers.\n", + "\n", + "The following example builds a small image classification model with fully replicated weight layout. You can specify layout information `kernel` and `bias` in `tf.keras.layers.Dense` via arguments `kernel_layout` and `bias_layout`. Most of the built-in keras layers are ready for explicitly specifying the `Layout` for the layer weights." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Koc5GlA1tFXY" + }, + "outputs": [], + "source": [ + "unsharded_layout_2d = dtensor.Layout.replicated(mesh, 2)\n", + "unsharded_layout_1d = dtensor.Layout.replicated(mesh, 1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GfOGTIxGs5Ql" + }, + "outputs": [], + "source": [ + "model = tf.keras.models.Sequential([\n", + " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", + " tf.keras.layers.Dense(128, \n", + " activation='relu',\n", + " name='d1',\n", + " kernel_layout=unsharded_layout_2d, \n", + " bias_layout=unsharded_layout_1d),\n", + " tf.keras.layers.Dense(10,\n", + " name='d2',\n", + " kernel_layout=unsharded_layout_2d, \n", + " bias_layout=unsharded_layout_1d)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0frf3jsVtx_n" + }, + "source": [ + "You can check the layout information by examining the `layout` property on the weights." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Z_nqv_VdwcXo" + }, + "outputs": [], + "source": [ + "for weight in model.weights:\n", + " print(f'Weight name: {weight.name} with layout: {weight.layout}')\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6FMGB-QsxPtU" + }, + "source": [ + "## Load a dataset and build input pipeline\n", + "\n", + "Load a MNIST dataset and configure some pre-processing input pipeline for it. The dataset itself is not associated with any DTensor layout information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zGt4kwltxOt4" + }, + "outputs": [], + "source": [ + "(ds_train, ds_test), ds_info = tfds.load(\n", + " 'mnist',\n", + " split=['train', 'test'],\n", + " shuffle_files=True,\n", + " as_supervised=True,\n", + " with_info=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HkUaOB_ryaLH" + }, + "outputs": [], + "source": [ + "def normalize_img(image, label):\n", + " \"\"\"Normalizes images: `uint8` -> `float32`.\"\"\"\n", + " return tf.cast(image, tf.float32) / 255., label" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Efm2H1iqydan" + }, + "outputs": [], + "source": [ + "batch_size = 128\n", + "\n", + "ds_train = ds_train.map(\n", + " normalize_img, num_parallel_calls=tf.data.AUTOTUNE)\n", + "ds_train = ds_train.cache()\n", + "ds_train = ds_train.shuffle(ds_info.splits['train'].num_examples)\n", + "ds_train = ds_train.batch(batch_size)\n", + "ds_train = ds_train.prefetch(tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Lcrg6QAtyis4" + }, + "outputs": [], + "source": [ + "ds_test = ds_test.map(\n", + " normalize_img, num_parallel_calls=tf.data.AUTOTUNE)\n", + "ds_test = ds_test.batch(batch_size)\n", + "ds_test = ds_test.cache()\n", + "ds_test = ds_test.prefetch(tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fHEZwib7lhqn" + }, + "source": [ + "## Define the training logic for the model\n", + "\n", + "Next, define the training and evaluation logic for the model. \n", + "\n", + "As of TensorFlow 2.9, you have to write a custom-training-loop for a DTensor-enabled Keras model. This is to pack the input data with proper layout information, which is not integrated with the standard `tf.keras.Model.fit()` or `tf.keras.Model.eval()` functions from Keras. you will get more `tf.data` support in the upcoming release. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CAx11gMjzzjs" + }, + "outputs": [], + "source": [ + "@tf.function\n", + "def train_step(model, x, y, optimizer, metrics):\n", + " with tf.GradientTape() as tape:\n", + " logits = model(x, training=True)\n", + " # tf.reduce_sum sums the batch sharded per-example loss to a replicated\n", + " # global loss (scalar).\n", + " loss = tf.reduce_sum(tf.keras.losses.sparse_categorical_crossentropy(\n", + " y, logits, from_logits=True))\n", + " \n", + " gradients = tape.gradient(loss, model.trainable_variables)\n", + " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", + "\n", + " for metric in metrics.values():\n", + " metric.update_state(y_true=y, y_pred=logits)\n", + "\n", + " loss_per_sample = loss / len(x)\n", + " results = {'loss': loss_per_sample}\n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "maSTWeRemO0P" + }, + "outputs": [], + "source": [ + "@tf.function\n", + "def eval_step(model, x, y, metrics):\n", + " logits = model(x, training=False)\n", + " loss = tf.reduce_sum(tf.keras.losses.sparse_categorical_crossentropy(\n", + " y, logits, from_logits=True))\n", + "\n", + " for metric in metrics.values():\n", + " metric.update_state(y_true=y, y_pred=logits)\n", + "\n", + " loss_per_sample = loss / len(x)\n", + " results = {'eval_loss': loss_per_sample}\n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dt00axcLmvLr" + }, + "outputs": [], + "source": [ + "def pack_dtensor_inputs(images, labels, image_layout, label_layout):\n", + " num_local_devices = image_layout.mesh.num_local_devices()\n", + " images = tf.split(images, num_local_devices)\n", + " labels = tf.split(labels, num_local_devices)\n", + " images = dtensor.pack(images, image_layout)\n", + " labels = dtensor.pack(labels, label_layout)\n", + " return images, labels" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9Eb-qIJGrxB9" + }, + "source": [ + "## Metrics and optimizers\n", + "\n", + "When using DTensor API with Keras `Metric` and `Optimizer`, you will need to provide the extra mesh information, so that any internal state variables and tensors can work with variables in the model.\n", + "\n", + "- For an optimizer, DTensor introduces a new experimental namespace `keras.dtensor.experimental.optimizers`, where many existing Keras Optimizers are extended to receive an additional `mesh` argument. In future releases, it may be merged with Keras core optimizers.\n", + "\n", + "- For metrics, you can directly specify the `mesh` to the constructor as an argument to make it a DTensor compatible `Metric`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1lu_0mz1sxrl" + }, + "outputs": [], + "source": [ + "optimizer = tf.keras.dtensor.experimental.optimizers.Adam(0.01, mesh=mesh)\n", + "metrics = {'accuracy': tf.keras.metrics.SparseCategoricalAccuracy(mesh=mesh)}\n", + "eval_metrics = {'eval_accuracy': tf.keras.metrics.SparseCategoricalAccuracy(mesh=mesh)}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QzufrkistELx" + }, + "source": [ + "## Train the model\n", + "\n", + "The following example demonstrates how to shard the data from input pipeline on the batch dimension, and train with the model, which has fully replicated weights. \n", + "\n", + "After 3 epochs, the model should achieve about 97% of accuracy:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kZW568Dk0vvL" + }, + "outputs": [], + "source": [ + "num_epochs = 3\n", + "\n", + "image_layout = dtensor.Layout.batch_sharded(mesh, 'batch', rank=4)\n", + "label_layout = dtensor.Layout.batch_sharded(mesh, 'batch', rank=1)\n", + "\n", + "for epoch in range(num_epochs):\n", + " print(\"============================\") \n", + " print(\"Epoch: \", epoch)\n", + " for metric in metrics.values():\n", + " metric.reset_state()\n", + " step = 0\n", + " results = {}\n", + " pbar = tf.keras.utils.Progbar(target=None, stateful_metrics=[])\n", + " for input in ds_train:\n", + " images, labels = input[0], input[1]\n", + " images, labels = pack_dtensor_inputs(\n", + " images, labels, image_layout, label_layout)\n", + "\n", + " results.update(train_step(model, images, labels, optimizer, metrics))\n", + " for metric_name, metric in metrics.items():\n", + " results[metric_name] = metric.result()\n", + "\n", + " pbar.update(step, values=results.items(), finalize=False)\n", + " step += 1\n", + " pbar.update(step, values=results.items(), finalize=True)\n", + "\n", + " for metric in eval_metrics.values():\n", + " metric.reset_state()\n", + " for input in ds_test:\n", + " images, labels = input[0], input[1]\n", + " images, labels = pack_dtensor_inputs(\n", + " images, labels, image_layout, label_layout)\n", + " results.update(eval_step(model, images, labels, eval_metrics))\n", + "\n", + " for metric_name, metric in eval_metrics.items():\n", + " results[metric_name] = metric.result()\n", + " \n", + " for metric_name, metric in results.items():\n", + " print(f\"{metric_name}: {metric.numpy()}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HYEXF6qCuoSr" + }, + "source": [ + "## Specify Layout for existing model code\n", + "\n", + "Often you have models that work well for your use case. Specifying `Layout` information to each individual layer within the model will be a large amount of work requiring a lot of edits.\n", + "\n", + "To help you easily convert your existing Keras model to work with DTensor API you can use the new `tf.keras.dtensor.experimental.LayoutMap` API that allow you to specify the `Layout` from a global point of view.\n", + "\n", + "First, you need to create a `LayoutMap` instance, which is a dictionary-like object that contains all the `Layout` you would like to specify for your model weights.\n", + "\n", + "`LayoutMap` needs a `Mesh` instance at init, which can be used to provide default replicated `Layout` for any weights that doesn't have Layout configured. In case you would like all your model weights to be just fully replicated, you can provide empty `LayoutMap`, and the default mesh will be used to create replicated `Layout`.\n", + "\n", + "`LayoutMap` uses a string as key and a `Layout` as value. There is a behavior difference between a normal Python dict and this class. The string key will be treated as a regex when retrieving the value." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SCq5Nl-UP_dS" + }, + "source": [ + "### Subclassed Model\n", + "\n", + "Consider the following model defined using the Keras subclassing Model syntax." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LZ0hRFs8unu0" + }, + "outputs": [], + "source": [ + "class SubclassedModel(tf.keras.Model):\n", + "\n", + " def __init__(self, name=None):\n", + " super().__init__(name=name)\n", + " self.feature = tf.keras.layers.Dense(16)\n", + " self.feature_2 = tf.keras.layers.Dense(24)\n", + " self.dropout = tf.keras.layers.Dropout(0.1)\n", + "\n", + " def call(self, inputs, training=None):\n", + " x = self.feature(inputs)\n", + " x = self.dropout(x, training=training)\n", + " return self.feature_2(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1njxqPB-yS97" + }, + "source": [ + "There are 4 weights in this model, which are `kernel` and `bias` for two `Dense` layers. Each of them are mapped based on the object path:\n", + "\n", + "* `model.feature.kernel`\n", + "* `model.feature.bias`\n", + "* `model.feature_2.kernel`\n", + "* `model.feature_2.bias`\n", + "\n", + "Note: For subclassed Models, the attribute name, rather than the `.name` attribute of the layer, is used as the key to retrieve the Layout from the mapping. This is consistent with the convention followed by `tf.Module` checkpointing. For complex models with more than a few layers, you can [manually inspect checkpoints](https://www.tensorflow.org/guide/checkpoint#manually_inspecting_checkpoints) to view the attribute mappings. \n", + "\n", + "Now define the following `LayoutMap` and apply it to the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "goVX6iIZw468" + }, + "outputs": [], + "source": [ + "layout_map = tf.keras.dtensor.experimental.LayoutMap(mesh=mesh)\n", + "\n", + "layout_map['feature.*kernel'] = dtensor.Layout.batch_sharded(mesh, 'batch', rank=2)\n", + "layout_map['feature.*bias'] = dtensor.Layout.batch_sharded(mesh, 'batch', rank=1)\n", + "\n", + "with layout_map.scope():\n", + " subclassed_model = SubclassedModel()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M32HcSp_PyWs" + }, + "source": [ + "The model weights are created on the first call, so call the model with a DTensor input and confirm the weights have the expected layouts:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "c3CbD9l7qUNq" + }, + "outputs": [], + "source": [ + "dtensor_input = dtensor.copy_to_mesh(tf.zeros((16, 16)), layout=unsharded_layout_2d)\n", + "# Trigger the weights creation for subclass model\n", + "subclassed_model(dtensor_input)\n", + "\n", + "print(subclassed_model.feature.kernel.layout)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZyCnfd-4Q2jk" + }, + "source": [ + "With this, you can quickly map the `Layout` to your models without updating any of your existing code. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6GliUdWTQnKC" + }, + "source": [ + "### Sequential and Functional Models" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6zzvTqAR2Teu" + }, + "source": [ + "For Keras Functional and Sequential models, you can use `tf.keras.dtensor.experimental.LayoutMap` as well.\n", + "\n", + "Note: For Functional and Sequential models, the mappings are slightly different. The layers in the model don't have a public attribute attached to the model (though you can access them via `Model.layers` as a list). Use the string name as the key in this case. The string name is guaranteed to be unique within a model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gXK2EquIRJCC" + }, + "outputs": [], + "source": [ + "layout_map = tf.keras.dtensor.experimental.LayoutMap(mesh=mesh)\n", + "\n", + "layout_map['feature.*kernel'] = dtensor.Layout.batch_sharded(mesh, 'batch', rank=2)\n", + "layout_map['feature.*bias'] = dtensor.Layout.batch_sharded(mesh, 'batch', rank=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cBzwJqrg2TH3" + }, + "outputs": [], + "source": [ + "with layout_map.scope():\n", + " inputs = tf.keras.Input((16,), batch_size=16)\n", + " x = tf.keras.layers.Dense(16, name='feature')(inputs)\n", + " x = tf.keras.layers.Dropout(0.1)(x)\n", + " output = tf.keras.layers.Dense(32, name='feature_2')(x)\n", + " model = tf.keras.Model(inputs, output)\n", + "\n", + "print(model.layers[1].kernel.layout)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pPuh1NlE3-wO" + }, + "outputs": [], + "source": [ + "with layout_map.scope():\n", + " model = tf.keras.Sequential([\n", + " tf.keras.layers.Dense(16, name='feature', input_shape=(16,)),\n", + " tf.keras.layers.Dropout(0.1),\n", + " tf.keras.layers.Dense(32, name='feature_2')\n", + " ])\n", + "\n", + "print(model.layers[2].kernel.layout)" + ] + } + ], + "metadata": { + "colab": { + "name": "dtensor_keras_tutorial.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/distribute/dtensor_ml_tutorial.ipynb b/site/en/tutorials/distribute/dtensor_ml_tutorial.ipynb new file mode 100644 index 00000000000..55557be6368 --- /dev/null +++ b/site/en/tutorials/distribute/dtensor_ml_tutorial.ipynb @@ -0,0 +1,1070 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Tce3stUlHN0L" + }, + "source": [ + "##### Copyright 2019 The TensorFlow Authors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "tuOe1ymfHZPu" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MfBg1C5NB3X0" + }, + "source": [ + "# Distributed training with DTensors" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r6P32iYYV27b" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kiF4jjX4O1mF" + }, + "source": [ + "## Overview\n", + "\n", + "DTensor provides a way for you to distribute the training of your model across devices to improve efficiency, reliability and scalability. For more details, check out the [DTensor concepts](../../guide/dtensor_overview.ipynb) guide.\n", + "\n", + "In this tutorial, you will train a sentiment analysis model using DTensors. The example demonstrates three distributed training schemes:\n", + "\n", + " - Data Parallel training, where the training samples are sharded (partitioned) to devices.\n", + " - Model Parallel training, where the model variables are sharded to devices.\n", + " - Spatial Parallel training, where the features of input data are sharded to devices (also known as [Spatial Partitioning](https://cloud.google.com/blog/products/ai-machine-learning/train-ml-models-on-large-images-and-3d-volumes-with-spatial-partitioning-on-cloud-tpus)).\n", + "\n", + "The training portion of this tutorial is inspired by a Kaggle notebook called [A Kaggle guide on sentiment analysis](https://www.kaggle.com/code/anasofiauzsoy/yelp-review-sentiment-analysis-tensorflow-tfds/notebook). To learn about the complete training and evaluation workflow (without DTensor), refer to that notebook.\n", + "\n", + "This tutorial will walk through the following steps:\n", + "\n", + "- Some data cleaning to obtain a `tf.data.Dataset` of tokenized sentences and their polarity.\n", + "- Then, building an MLP model with custom Dense and BatchNorm layers using a `tf.Module` to track the inference variables. The model constructor will take additional `Layout` arguments to control the sharding of variables.\n", + "- For training, you will first use data parallel training together with `tf.experimental.dtensor`'s checkpoint feature. Then, you will continue with Model Parallel Training and Spatial Parallel Training.\n", + "- The final section briefly describes the interaction between `tf.saved_model` and `tf.experimental.dtensor` as of TensorFlow 2.9." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YD80veeg7QtW" + }, + "source": [ + "## Setup\n", + "\n", + "DTensor (`tf.experimental.dtensor`) has been part of TensorFlow since the 2.9.0 release.\n", + "\n", + "First, install or upgrade TensorFlow Datasets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-RKXLJN-7Yyb" + }, + "outputs": [], + "source": [ + "!pip install --quiet --upgrade tensorflow-datasets" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tcxP4_Zu7ciQ" + }, + "source": [ + "Next, import `tensorflow` and `dtensor`, and configure TensorFlow to use 8 virtual CPUs.\n", + "\n", + "Even though this example uses virtual CPUs, DTensor works the same way on CPU, GPU or TPU devices." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dXcB26oP7dUd" + }, + "outputs": [], + "source": [ + "import tempfile\n", + "import numpy as np\n", + "import tensorflow_datasets as tfds\n", + "\n", + "import tensorflow as tf\n", + "\n", + "from tensorflow.experimental import dtensor\n", + "\n", + "print('TensorFlow version:', tf.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "oHtO6MJLUXlz" + }, + "outputs": [], + "source": [ + "def configure_virtual_cpus(ncpu):\n", + " phy_devices = tf.config.list_physical_devices('CPU')\n", + " tf.config.set_logical_device_configuration(phy_devices[0], [\n", + " tf.config.LogicalDeviceConfiguration(),\n", + " ] * ncpu)\n", + "\n", + "configure_virtual_cpus(8)\n", + "DEVICES = [f'CPU:{i}' for i in range(8)]\n", + "\n", + "tf.config.list_logical_devices('CPU')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "omYd4jbF7j_I" + }, + "source": [ + "## Download the dataset\n", + "\n", + "Download the IMDB reviews data set to train the sentiment analysis model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fW4w4QlFVHhx" + }, + "outputs": [], + "source": [ + "train_data = tfds.load('imdb_reviews', split='train', shuffle_files=True, batch_size=64)\n", + "train_data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ki3mpfi4aZH8" + }, + "source": [ + "## Prepare the data\n", + "\n", + "First tokenize the text. Here use an extension of one-hot encoding, the `'tf_idf'` mode of `tf.keras.layers.TextVectorization`.\n", + "\n", + "- For the sake of speed, limit the number of tokens to 1200.\n", + "- To keep the `tf.Module` simple, run `TextVectorization` as a preprocessing step before the training.\n", + "\n", + "The final result of the data cleaning section is a `Dataset` with the tokenized text as `x` and label as `y`.\n", + "\n", + "**Note**: Running `TextVectorization` as a preprocessing step is **neither a usual practice nor a recommended one** as doing so assumes the training data fits into the client memory, which is not always the case.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zNpxjku_57Lg" + }, + "outputs": [], + "source": [ + "text_vectorization = tf.keras.layers.TextVectorization(output_mode='tf_idf', max_tokens=1200, output_sequence_length=None)\n", + "text_vectorization.adapt(data=train_data.map(lambda x: x['text']))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "q16bjngoVwQp" + }, + "outputs": [], + "source": [ + "def vectorize(features):\n", + " return text_vectorization(features['text']), features['label']\n", + "\n", + "train_data_vec = train_data.map(vectorize)\n", + "train_data_vec" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "atTqL9kE5wz4" + }, + "source": [ + "## Build a neural network with DTensor\n", + "\n", + "Now build a Multi-Layer Perceptron (MLP) network with `DTensor`. The network will use fully connected Dense and BatchNorm layers.\n", + "\n", + "`DTensor` expands TensorFlow through single-program multi-data (SPMD) expansion of regular TensorFlow Ops according to the `dtensor.Layout` attributes of their input `Tensor` and variables.\n", + "\n", + "Variables of `DTensor` aware layers are `dtensor.DVariable`, and the constructors of `DTensor` aware layer objects take additional `Layout` inputs in addition to the usual layer parameters.\n", + "\n", + "Note: As of TensorFlow 2.9, Keras layers such as `tf.keras.layer.Dense`, and `tf.keras.layer.BatchNormalization` accepts `dtensor.Layout` arguments. Refer to the [DTensor Keras Integration Tutorial](/tutorials/distribute/dtensor_keras_tutorial) for more information using Keras with DTensor." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PMCt-Gj3b3Jy" + }, + "source": [ + "### Dense Layer\n", + "\n", + "The following custom Dense layer defines 2 layer variables: $W_{ij}$ is the variable for weights, and $b_i$ is the variable for the biases.\n", + "\n", + "$$\n", + "y_j = \\sigma(\\sum_i x_i W_{ij} + b_j)\n", + "$$\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nYlFUJWNjl4N" + }, + "source": [ + "### Layout deduction\n", + "\n", + "This result comes from the following observations:\n", + "\n", + "- The preferred DTensor sharding for operands to a matrix dot product $t_j = \\sum_i x_i W_{ij}$ is to shard $\\mathbf{W}$ and $\\mathbf{x}$ the same way along the $i$-axis.\n", + "\n", + "- The preferred DTensor sharding for operands to a matrix sum $t_j + b_j$, is to shard $\\mathbf{t}$ and $\\mathbf{b}$ the same way along the $j$-axis.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VpKblz7Yb16G" + }, + "outputs": [], + "source": [ + "class Dense(tf.Module):\n", + "\n", + " def __init__(self, input_size, output_size,\n", + " init_seed, weight_layout, activation=None):\n", + " super().__init__()\n", + "\n", + " random_normal_initializer = tf.function(tf.random.stateless_normal)\n", + "\n", + " self.weight = dtensor.DVariable(\n", + " dtensor.call_with_layout(\n", + " random_normal_initializer, weight_layout,\n", + " shape=[input_size, output_size],\n", + " seed=init_seed\n", + " ))\n", + " if activation is None:\n", + " activation = lambda x:x\n", + " self.activation = activation\n", + " \n", + " # bias is sharded the same way as the last axis of weight.\n", + " bias_layout = weight_layout.delete([0])\n", + "\n", + " self.bias = dtensor.DVariable(\n", + " dtensor.call_with_layout(tf.zeros, bias_layout, [output_size]))\n", + "\n", + " def __call__(self, x):\n", + " y = tf.matmul(x, self.weight) + self.bias\n", + " y = self.activation(y)\n", + "\n", + " return y" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tfVY_vAKbxM0" + }, + "source": [ + "### BatchNorm\n", + "\n", + "A batch normalization layer helps avoid collapsing modes while training. In this case, adding batch normalization layers helps model training avoid producing a model that only produces zeros.\n", + "\n", + "The constructor of the custom `BatchNorm` layer below does not take a `Layout` argument. This is because `BatchNorm` has no layer variables. This still works with DTensor because 'x', the only input to the layer, is already a DTensor that represents the global batch.\n", + "\n", + "Note: With DTensor, the input Tensor 'x' always represents the global batch. Therefore `tf.nn.batch_normalization` is applied to the global batch. This differs from training with `tf.distribute.MirroredStrategy`, where Tensor 'x' only represents the per-replica shard of the batch (the local batch)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "riBA9pfhlPFq" + }, + "outputs": [], + "source": [ + "class BatchNorm(tf.Module):\n", + "\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + " def __call__(self, x, training=True):\n", + " if not training:\n", + " # This branch is not used in the Tutorial.\n", + " pass\n", + " mean, variance = tf.nn.moments(x, axes=[0])\n", + " return tf.nn.batch_normalization(x, mean, variance, 0.0, 1.0, 1e-5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "q4R4MPz5prh4" + }, + "source": [ + "A full featured batch normalization layer (such as `tf.keras.layers.BatchNormalization`) will need Layout arguments for its variables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "unFcP99zprJj" + }, + "outputs": [], + "source": [ + "def make_keras_bn(bn_layout):\n", + " return tf.keras.layers.BatchNormalization(gamma_layout=bn_layout,\n", + " beta_layout=bn_layout,\n", + " moving_mean_layout=bn_layout,\n", + " moving_variance_layout=bn_layout,\n", + " fused=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v8Dj7AJ_lPs0" + }, + "source": [ + "### Putting Layers Together\n", + "\n", + "Next, build a Multi-layer perceptron (MLP) network with the building blocks above. The diagram below shows the axis relationships between the input `x` and the weight matrices for the two `Dense` layers without any DTensor sharding or replication applied." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "udFGAO-NrZw6" + }, + "source": [ + "\"The\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8DCQ0aQ5rQtB" + }, + "source": [ + "The output of the first `Dense` layer is passed into the input of the second `Dense` layer (after the `BatchNorm`). Therefore, the preferred DTensor sharding for the output of first `Dense` layer ($\\mathbf{W_1}$) and the input of second `Dense` layer ($\\mathbf{W_2}$) is to shard $\\mathbf{W_1}$ and $\\mathbf{W_2}$ the same way along the common axis $\\hat{j}$,\n", + "\n", + "$$\n", + "\\mathsf{Layout}[{W_{1,ij}}; i, j] = \\left[\\hat{i}, \\hat{j}\\right] \\\\\n", + "\\mathsf{Layout}[{W_{2,jk}}; j, k] = \\left[\\hat{j}, \\hat{k} \\right]\n", + "$$\n", + "\n", + "Even though the layout deduction shows that the 2 layouts are not independent, for the sake of simplicity of the model interface, `MLP` will take 2 `Layout` arguments, one per Dense layer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "junyS-965opl" + }, + "outputs": [], + "source": [ + "from typing import Tuple\n", + "\n", + "class MLP(tf.Module):\n", + "\n", + " def __init__(self, dense_layouts: Tuple[dtensor.Layout, dtensor.Layout]):\n", + " super().__init__()\n", + "\n", + " self.dense1 = Dense(\n", + " 1200, 48, (1, 2), dense_layouts[0], activation=tf.nn.relu)\n", + " self.bn = BatchNorm()\n", + " self.dense2 = Dense(48, 2, (3, 4), dense_layouts[1])\n", + "\n", + " def __call__(self, x):\n", + " y = x\n", + " y = self.dense1(y)\n", + " y = self.bn(y)\n", + " y = self.dense2(y)\n", + " return y\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9dgLmebHhr7h" + }, + "source": [ + "The trade-off between correctness in layout deduction constraints and simplicity of API is a common design point of APIs that uses DTensor.\n", + "It is also possible to capture the dependency between `Layout`'s with a different API. For example, the `MLPStricter` class creates the `Layout` objects in the constructor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wEZR7UlihsYX" + }, + "outputs": [], + "source": [ + "class MLPStricter(tf.Module):\n", + "\n", + " def __init__(self, mesh, input_mesh_dim, inner_mesh_dim1, output_mesh_dim):\n", + " super().__init__()\n", + "\n", + " self.dense1 = Dense(\n", + " 1200, 48, (1, 2), dtensor.Layout([input_mesh_dim, inner_mesh_dim1], mesh),\n", + " activation=tf.nn.relu)\n", + " self.bn = BatchNorm()\n", + " self.dense2 = Dense(48, 2, (3, 4), dtensor.Layout([inner_mesh_dim1, output_mesh_dim], mesh))\n", + "\n", + "\n", + " def __call__(self, x):\n", + " y = x\n", + " y = self.dense1(y)\n", + " y = self.bn(y)\n", + " y = self.dense2(y)\n", + " return y" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GcQi7D5mal2L" + }, + "source": [ + "To make sure the model runs, probe your model with fully replicated layouts and a fully replicated batch of `'x'` input." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zOPuYeQwallh" + }, + "outputs": [], + "source": [ + "WORLD = dtensor.create_mesh([(\"world\", 8)], devices=DEVICES)\n", + "\n", + "model = MLP([dtensor.Layout.replicated(WORLD, rank=2),\n", + " dtensor.Layout.replicated(WORLD, rank=2)])\n", + "\n", + "sample_x, sample_y = train_data_vec.take(1).get_single_element()\n", + "sample_x = dtensor.copy_to_mesh(sample_x, dtensor.Layout.replicated(WORLD, rank=2))\n", + "print(model(sample_x))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "akrjDstEpDv9" + }, + "source": [ + "## Moving data to the device\n", + "\n", + "Usually, `tf.data` iterators (and other data fetching methods) yield tensor objects backed by the local host device memory. This data must be transferred to the accelerator device memory that backs DTensor's component tensors.\n", + "\n", + "`dtensor.copy_to_mesh` is unsuitable for this situation because it replicates input tensors to all devices due to DTensor's global perspective. So in this tutorial, you will use a helper function `repack_local_tensor`, to facilitate the transfer of data. This helper function uses `dtensor.pack` to send (and only send) the shard of the global batch that is intended for a replica to the device backing the replica.\n", + "\n", + "This simplified function assumes single-client. Determining the correct way to split the local tensor and the mapping between the pieces of the split and the local devices can be laboring in a multi-client application.\n", + "\n", + "Additional DTensor API to simplify `tf.data` integration is planned, supporting both single-client and multi-client applications. Please stay tuned." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3t5WvQR4Hvo4" + }, + "outputs": [], + "source": [ + "def repack_local_tensor(x, layout):\n", + " \"\"\"Repacks a local Tensor-like to a DTensor with layout.\n", + "\n", + " This function assumes a single-client application.\n", + " \"\"\"\n", + " x = tf.convert_to_tensor(x)\n", + " sharded_dims = []\n", + "\n", + " # For every sharded dimension, use tf.split to split the along the dimension.\n", + " # The result is a nested list of split-tensors in queue[0].\n", + " queue = [x]\n", + " for axis, dim in enumerate(layout.sharding_specs):\n", + " if dim == dtensor.UNSHARDED:\n", + " continue\n", + " num_splits = layout.shape[axis]\n", + " queue = tf.nest.map_structure(lambda x: tf.split(x, num_splits, axis=axis), queue)\n", + " sharded_dims.append(dim)\n", + "\n", + " # Now we can build the list of component tensors by looking up the location in\n", + " # the nested list of split-tensors created in queue[0].\n", + " components = []\n", + " for locations in layout.mesh.local_device_locations():\n", + " t = queue[0]\n", + " for dim in sharded_dims:\n", + " split_index = locations[dim] # Only valid on single-client mesh.\n", + " t = t[split_index]\n", + " components.append(t)\n", + "\n", + " return dtensor.pack(components, layout)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2KKCDcjG7zj2" + }, + "source": [ + "## Data parallel training\n", + "\n", + "In this section, you will train your MLP model with data parallel training. The following sections will demonstrate model parallel training and spatial parallel training.\n", + "\n", + "Data parallel training is a commonly used scheme for distributed machine learning:\n", + "\n", + " - Model variables are replicated on N devices each.\n", + " - A global batch is split into N per-replica batches.\n", + " - Each per-replica batch is trained on the replica device.\n", + " - The gradient is reduced before weight up data is collectively performed on all replicas.\n", + "\n", + "Data parallel training provides nearly linear speedup regarding the number of devices." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UMsLUyTGq3oL" + }, + "source": [ + "### Creating a data parallel mesh\n", + "\n", + "A typical data parallelism training loop uses a DTensor `Mesh` that consists of a single `batch` dimension, where each device becomes a replica that receives a shard from the global batch.\n", + "\n", + "\"Data\n", + "\n", + "\n", + "The replicated model runs on the replica, therefore the model variables are fully replicated (unsharded)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "C0IyOlxmeu4I" + }, + "outputs": [], + "source": [ + "mesh = dtensor.create_mesh([(\"batch\", 8)], devices=DEVICES)\n", + "\n", + "model = MLP([dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh),\n", + " dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh),])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OREKwBybo1gZ" + }, + "source": [ + "### Packing training data to DTensors\n", + "\n", + "The training data batch should be packed into DTensors sharded along the `'batch'`(first) axis, such that DTensor will evenly distribute the training data to the `'batch'` mesh dimension.\n", + "\n", + "**Note**: In DTensor, the `batch size` always refers to the global batch size. The batch size should be chosen such that it can be divided evenly by the size of the `batch` mesh dimension." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8xMYkTpGocY8" + }, + "outputs": [], + "source": [ + "def repack_batch(x, y, mesh):\n", + " x = repack_local_tensor(x, layout=dtensor.Layout(['batch', dtensor.UNSHARDED], mesh))\n", + " y = repack_local_tensor(y, layout=dtensor.Layout(['batch'], mesh))\n", + " return x, y\n", + "\n", + "sample_x, sample_y = train_data_vec.take(1).get_single_element()\n", + "sample_x, sample_y = repack_batch(sample_x, sample_y, mesh)\n", + "\n", + "print('x', sample_x[:, 0])\n", + "print('y', sample_y)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uONSiqOIkFL1" + }, + "source": [ + "### Training step\n", + "\n", + "This example uses a Stochastic Gradient Descent optimizer with the Custom Training Loop (CTL). Consult the [Custom Training Loop guide](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) and [Walk through](https://www.tensorflow.org/tutorials/customization/custom_training_walkthrough) for more information on those topics.\n", + "\n", + "The `train_step` is encapsulated as a `tf.function` to indicate this body is to be traced as a TensorFlow Graph. The body of `train_step` consists of a forward inference pass, a backward gradient pass, and the variable update.\n", + "\n", + "Note that the body of `train_step` does not contain any special DTensor annotations. Instead, `train_step` only contains high-level TensorFlow operations that process the input `x` and `y` from the global view of the input batch and the model. All of the DTensor annotations (`Mesh`, `Layout`) are factored out of the train step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BwUFzLGDtQT6" + }, + "outputs": [], + "source": [ + "# Refer to the CTL (custom training loop guide)\n", + "@tf.function\n", + "def train_step(model, x, y, learning_rate=tf.constant(1e-4)):\n", + " with tf.GradientTape() as tape:\n", + " logits = model(x)\n", + " # tf.reduce_sum sums the batch sharded per-example loss to a replicated\n", + " # global loss (scalar).\n", + " loss = tf.reduce_sum(\n", + " tf.nn.sparse_softmax_cross_entropy_with_logits(\n", + " logits=logits, labels=y))\n", + " parameters = model.trainable_variables\n", + " gradients = tape.gradient(loss, parameters)\n", + " for parameter, parameter_gradient in zip(parameters, gradients):\n", + " parameter.assign_sub(learning_rate * parameter_gradient)\n", + "\n", + " # Define some metrics\n", + " accuracy = 1.0 - tf.reduce_sum(tf.cast(tf.argmax(logits, axis=-1, output_type=tf.int64) != y, tf.float32)) / x.shape[0]\n", + " loss_per_sample = loss / len(x)\n", + " return {'loss': loss_per_sample, 'accuracy': accuracy}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0OYTu4j0evWT" + }, + "source": [ + "### Checkpointing\n", + "\n", + "You can checkpoint a DTensor model using `tf.train.Checkpoint` out of the box. Saving and restoring sharded DVariables will perform an efficient sharded save and restore. Currently, when using `tf.train.Checkpoint.save` and `tf.train.Checkpoint.restore`, all DVariables must be on the same host mesh, and DVariables and regular variables cannot be saved together. You can learn more about checkpointing in [this guide](../../guide/checkpoint.ipynb).\n", + "\n", + "When a DTensor checkpoint is restored, `Layout`s of variables can be different from when the checkpoint is saved. That is, saving DTensor models is layout- and mesh-agnostic, and only affects the efficiency of sharded saving. You can save a DTensor model with one mesh and layout and restore it on a different mesh and layout. This tutorial makes use of this feature to continue the training in the Model Parallel training and Spatial Parallel training sections.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rsInFFJg7x9t" + }, + "outputs": [], + "source": [ + "CHECKPOINT_DIR = tempfile.mkdtemp()\n", + "\n", + "def start_checkpoint_manager(model):\n", + " ckpt = tf.train.Checkpoint(root=model)\n", + " manager = tf.train.CheckpointManager(ckpt, CHECKPOINT_DIR, max_to_keep=3)\n", + "\n", + " if manager.latest_checkpoint:\n", + " print(\"Restoring a checkpoint\")\n", + " ckpt.restore(manager.latest_checkpoint).assert_consumed()\n", + " else:\n", + " print(\"New training\")\n", + " return manager\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9r77ky5Jgp1j" + }, + "source": [ + "### Training loop\n", + "\n", + "For the data parallel training scheme, train for epochs and report the progress. 3 epochs is insufficient for training the model -- an accuracy of 50% is as good as randomly guessing.\n", + "\n", + "Enable checkpointing so that you can pick up the training later. In the following section, you will load the checkpoint and train with a different parallel scheme." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UaLn-vGZgqbS" + }, + "outputs": [], + "source": [ + "num_epochs = 2\n", + "manager = start_checkpoint_manager(model)\n", + "\n", + "for epoch in range(num_epochs):\n", + " step = 0\n", + " pbar = tf.keras.utils.Progbar(target=int(train_data_vec.cardinality()), stateful_metrics=[])\n", + " metrics = {'epoch': epoch}\n", + " for x,y in train_data_vec:\n", + "\n", + " x, y = repack_batch(x, y, mesh)\n", + "\n", + " metrics.update(train_step(model, x, y, 1e-2))\n", + "\n", + " pbar.update(step, values=metrics.items(), finalize=False)\n", + " step += 1\n", + " manager.save()\n", + " pbar.update(step, values=metrics.items(), finalize=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YRFJEhum7EGD" + }, + "source": [ + "## Model Parallel Training\n", + "\n", + "If you switch to a 2 dimensional `Mesh`, and shard the model variables along the second mesh dimension, then the training becomes Model Parallel.\n", + "\n", + "In Model Parallel training, each model replica spans multiple devices (2 in this case):\n", + "\n", + "- There are 4 model replicas, and the training data batch is distributed to the 4 replicas.\n", + "- The 2 devices within a single model replica receive replicated training data.\n", + "\n", + "\n", + "\"Model\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5gZE9IT5Dzwl" + }, + "outputs": [], + "source": [ + "mesh = dtensor.create_mesh([(\"batch\", 4), (\"model\", 2)], devices=DEVICES)\n", + "model = MLP([dtensor.Layout([dtensor.UNSHARDED, \"model\"], mesh), \n", + " dtensor.Layout([\"model\", dtensor.UNSHARDED], mesh)])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ihof3DkMFKnf" + }, + "source": [ + "As the training data is still sharded along the batch dimension, you can reuse the same `repack_batch` function as the Data Parallel training case. DTensor will automatically replicate the per-replica batch to all devices inside the replica along the `\"model\"` mesh dimension." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dZf56ynbE_p1" + }, + "outputs": [], + "source": [ + "def repack_batch(x, y, mesh):\n", + " x = repack_local_tensor(x, layout=dtensor.Layout(['batch', dtensor.UNSHARDED], mesh))\n", + " y = repack_local_tensor(y, layout=dtensor.Layout(['batch'], mesh))\n", + " return x, y" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UW3OXdhNFfpv" + }, + "source": [ + "Next run the training loop. The training loop reuses the same checkpoint manager as the Data Parallel training example, and the code looks identical.\n", + "\n", + "You can continue training the data parallel trained model under model parallel training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LLC0wgii7EgA" + }, + "outputs": [], + "source": [ + "num_epochs = 2\n", + "manager = start_checkpoint_manager(model)\n", + "\n", + "for epoch in range(num_epochs):\n", + " step = 0\n", + " pbar = tf.keras.utils.Progbar(target=int(train_data_vec.cardinality()))\n", + " metrics = {'epoch': epoch}\n", + " for x,y in train_data_vec:\n", + " x, y = repack_batch(x, y, mesh)\n", + " metrics.update(train_step(model, x, y, 1e-2))\n", + " pbar.update(step, values=metrics.items(), finalize=False)\n", + " step += 1\n", + " manager.save()\n", + " pbar.update(step, values=metrics.items(), finalize=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BZH-aMrVzi2L" + }, + "source": [ + "## Spatial Parallel Training" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u-bK6IZ9GCS9" + }, + "source": [ + "When training data of very high dimensionality (e.g. a very large image or a video), it may be desirable to shard along the feature dimension. This is called [Spatial Partitioning](https://cloud.google.com/blog/products/ai-machine-learning/train-ml-models-on-large-images-and-3d-volumes-with-spatial-partitioning-on-cloud-tpus), which was first introduced into TensorFlow for training models with large 3-d input samples.\n", + "\n", + "\"Spatial\n", + "\n", + "DTensor also supports this case. The only change you need to do is to create a Mesh that includes a `feature` dimension, and apply the corresponding `Layout`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jpc9mqURGpmK" + }, + "outputs": [], + "source": [ + "mesh = dtensor.create_mesh([(\"batch\", 2), (\"feature\", 2), (\"model\", 2)], devices=DEVICES)\n", + "model = MLP([dtensor.Layout([\"feature\", \"model\"], mesh), \n", + " dtensor.Layout([\"model\", dtensor.UNSHARDED], mesh)])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i07Wrv-jHBc1" + }, + "source": [ + "Shard the input data along the `feature` dimension when packing the input tensors to DTensors. You do this with a slightly different repack function, `repack_batch_for_spt`, where `spt` stands for Spatial Parallel Training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DWR8qF6BGtFL" + }, + "outputs": [], + "source": [ + "def repack_batch_for_spt(x, y, mesh):\n", + " # Shard data on feature dimension, too\n", + " x = repack_local_tensor(x, layout=dtensor.Layout([\"batch\", 'feature'], mesh))\n", + " y = repack_local_tensor(y, layout=dtensor.Layout([\"batch\"], mesh))\n", + " return x, y" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ygl9dqMUHTVN" + }, + "source": [ + "The Spatial parallel training can also continue from a checkpoint created with other parallell training schemes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "p3NnpHSKo-hx" + }, + "outputs": [], + "source": [ + "num_epochs = 2\n", + "\n", + "manager = start_checkpoint_manager(model)\n", + "for epoch in range(num_epochs):\n", + " step = 0\n", + " metrics = {'epoch': epoch}\n", + " pbar = tf.keras.utils.Progbar(target=int(train_data_vec.cardinality()))\n", + "\n", + " for x, y in train_data_vec:\n", + " x, y = repack_batch_for_spt(x, y, mesh)\n", + " metrics.update(train_step(model, x, y, 1e-2))\n", + "\n", + " pbar.update(step, values=metrics.items(), finalize=False)\n", + " step += 1\n", + " manager.save()\n", + " pbar.update(step, values=metrics.items(), finalize=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vp4L59CpJjYr" + }, + "source": [ + "## SavedModel and DTensor\n", + "\n", + "The integration of DTensor and SavedModel is still under development. \n", + "\n", + "As of TensorFlow `2.11`, `tf.saved_model` can save sharded and replicated DTensor models, and saving will do an efficient sharded save on different devices of the mesh. However, after a model is saved, all DTensor annotations are lost and the saved signatures can only be used with regular Tensors, not DTensors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "49HfIq_SJZoj" + }, + "outputs": [], + "source": [ + "mesh = dtensor.create_mesh([(\"world\", 1)], devices=DEVICES[:1])\n", + "mlp = MLP([dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh), \n", + " dtensor.Layout([dtensor.UNSHARDED, dtensor.UNSHARDED], mesh)])\n", + "\n", + "manager = start_checkpoint_manager(mlp)\n", + "\n", + "model_for_saving = tf.keras.Sequential([\n", + " text_vectorization,\n", + " mlp\n", + "])\n", + "\n", + "@tf.function(input_signature=[tf.TensorSpec([None], tf.string)])\n", + "def run(inputs):\n", + " return {'result': model_for_saving(inputs)}\n", + "\n", + "tf.saved_model.save(\n", + " model_for_saving, \"/tmp/saved_model\",\n", + " signatures=run)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h6Csim_VMGxQ" + }, + "source": [ + "As of TensorFlow 2.9.0, you can only call a loaded signature with a regular Tensor, or a fully replicated DTensor (which will be converted to a regular Tensor)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HG_ASSzR4IWW" + }, + "outputs": [], + "source": [ + "sample_batch = train_data.take(1).get_single_element()\n", + "sample_batch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qW8yKPrhKQ5b" + }, + "outputs": [], + "source": [ + "loaded = tf.saved_model.load(\"/tmp/saved_model\")\n", + "\n", + "run_sig = loaded.signatures[\"serving_default\"]\n", + "result = run_sig(sample_batch['text'])['result']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GahGbv0ZmkJb" + }, + "outputs": [], + "source": [ + "np.mean(tf.argmax(result, axis=-1) == sample_batch['label'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ks-Vs9qsH6jO" + }, + "source": [ + "## What's next?\n", + "\n", + "This tutorial demonstrated building and training an MLP sentiment analysis model with DTensor.\n", + "\n", + "Through `Mesh` and `Layout` primitives, DTensor can transform a TensorFlow `tf.function` to a distributed program suitable for a variety of training schemes.\n", + "\n", + "In a real-world machine learning application, evaluation and cross-validation should be applied to avoid producing an over-fitted model. The techniques introduced in this tutorial can also be applied to introduce parallelism to evaluation.\n", + "\n", + "Composing a model with `tf.Module` from scratch is a lot of work, and reusing existing building blocks such as layers and helper functions can drastically speed up model development.\n", + "As of TensorFlow 2.9, all Keras Layers under `tf.keras.layers` accepts DTensor layouts as their arguments, and can be used to build DTensor models. You can even directly reuse a Keras model with DTensor without modifying the model implementation. Refer to the [DTensor Keras Integration Tutorial](https://www.tensorflow.org/tutorials/distribute/dtensor_keras_tutorial) for information on using DTensor Keras. " + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "dtensor_ml_tutorial.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/distribute/images/tensorboard_distributed_training_with_keras.png b/site/en/tutorials/distribute/images/tensorboard_distributed_training_with_keras.png new file mode 100644 index 00000000000..2a51fdf4a82 Binary files /dev/null and b/site/en/tutorials/distribute/images/tensorboard_distributed_training_with_keras.png differ diff --git a/site/en/tutorials/distribute/input.ipynb b/site/en/tutorials/distribute/input.ipynb new file mode 100644 index 00000000000..f779c4f19a6 --- /dev/null +++ b/site/en/tutorials/distribute/input.ipynb @@ -0,0 +1,1108 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "MhoQ0WE77laV" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "_ckMIh7O7s6D" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jYysdyb-CaWM" + }, + "source": [ + "# Distributed Input" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S5Uhzt6vVIB2" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FbVhjPpzn6BM" + }, + "source": [ + "The [tf.distribute](https://www.tensorflow.org/guide/distributed_training) APIs provide an easy way for users to scale their training from a single machine to multiple machines. When scaling their model, users also have to distribute their input across multiple devices. `tf.distribute` provides APIs using which you can automatically distribute your input across devices.\n", + "\n", + "This guide will show you the different ways in which you can create distributed dataset and iterators using `tf.distribute` APIs. Additionally, the following topics will be covered:\n", + "- Usage, sharding and batching options when using `tf.distribute.Strategy.experimental_distribute_dataset` and `tf.distribute.Strategy.distribute_datasets_from_function`.\n", + "- Different ways in which you can iterate over the distributed dataset.\n", + "- Differences between `tf.distribute.Strategy.experimental_distribute_dataset`/`tf.distribute.Strategy.distribute_datasets_from_function` APIs and `tf.data` APIs as well as any limitations that users may come across in their usage.\n", + "\n", + "This guide does not cover usage of distributed input with Keras APIs." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MM6W__qraV55" + }, + "source": [ + "## Distributed datasets" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lNy9GxjSlMKQ" + }, + "source": [ + "To use `tf.distribute` APIs to scale, use `tf.data.Dataset` to represent their input. `tf.distribute` works efficiently with `tf.data.Dataset`—for example, via automatic prefetching onto each accelerator device and regular performance updates. If you have a use case for using something other than `tf.data.Dataset`, please refer to the [Tensor inputs section](#tensorinputs) in this guide.\n", + "In a non-distributed training loop, first create a `tf.data.Dataset` instance and then iterate over the elements. For example:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pCu2Jj-21AEf" + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "\n", + "# Helper libraries\n", + "import numpy as np\n", + "import os\n", + "\n", + "print(tf.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6cnilUtmKwpa" + }, + "outputs": [], + "source": [ + "# Simulate multiple CPUs with virtual devices\n", + "N_VIRTUAL_DEVICES = 2\n", + "physical_devices = tf.config.list_physical_devices(\"CPU\")\n", + "tf.config.set_logical_device_configuration(\n", + " physical_devices[0], [tf.config.LogicalDeviceConfiguration() for _ in range(N_VIRTUAL_DEVICES)])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zd4l1ySeLRk1" + }, + "outputs": [], + "source": [ + "print(\"Available devices:\")\n", + "for i, device in enumerate(tf.config.list_logical_devices()):\n", + " print(\"%d) %s\" % (i, device))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dzLKpmZICaWN" + }, + "outputs": [], + "source": [ + "global_batch_size = 16\n", + "# Create a tf.data.Dataset object.\n", + "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100).batch(global_batch_size)\n", + "\n", + "@tf.function\n", + "def train_step(inputs):\n", + " features, labels = inputs\n", + " return labels - 0.3 * features\n", + "\n", + "# Iterate over the dataset using the for..in construct.\n", + "for inputs in dataset:\n", + " print(train_step(inputs))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ihrhYDYRrVLH" + }, + "source": [ + "To allow users to use `tf.distribute` strategy with minimal changes to a user’s existing code, two APIs were introduced which would distribute a `tf.data.Dataset` instance and return a distributed dataset object. A user could then iterate over this distributed dataset instance and train their model as before. Let us now look at the two APIs - `tf.distribute.Strategy.experimental_distribute_dataset` and `tf.distribute.Strategy.distribute_datasets_from_function` in more detail:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4AXoHhrsbdF3" + }, + "source": [ + "### `tf.distribute.Strategy.experimental_distribute_dataset`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5mVuLZhbem8d" + }, + "source": [ + "#### Usage\n", + "\n", + "This API takes a `tf.data.Dataset` instance as input and returns a `tf.distribute.DistributedDataset` instance. You should batch the input dataset with a value that is equal to the global batch size. This global batch size is the number of samples that you want to process across all devices in 1 step. You can iterate over this distributed dataset in a Pythonic fashion or create an iterator using `iter`. The returned object is not a `tf.data.Dataset` instance and does not support any other APIs that transform or inspect the dataset in any way.\n", + "This is the recommended API if you don’t have specific ways in which you want to shard your input over different replicas.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "F2VeZUWUj5S4" + }, + "outputs": [], + "source": [ + "global_batch_size = 16\n", + "mirrored_strategy = tf.distribute.MirroredStrategy()\n", + "\n", + "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100).batch(global_batch_size)\n", + "# Distribute input using the `experimental_distribute_dataset`.\n", + "dist_dataset = mirrored_strategy.experimental_distribute_dataset(dataset)\n", + "# 1 global batch of data fed to the model in 1 step.\n", + "print(next(iter(dist_dataset)))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QPceDmRht54F" + }, + "source": [ + "#### Properties" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0Qb6nDgxiN_n" + }, + "source": [ + "##### Batching\n", + "\n", + "`tf.distribute` rebatches the input `tf.data.Dataset` instance with a new batch size that is equal to the global batch size divided by the number of replicas in sync. The number of replicas in sync is equal to the number of devices that are taking part in the gradient allreduce during training. When a user calls `next` on the distributed iterator, a per replica batch size of data is returned on each replica. The rebatched dataset cardinality will always be a multiple of the number of replicas. Here are a couple of\n", + "examples:\n", + "* `tf.data.Dataset.range(6).batch(4, drop_remainder=False)`\n", + " * Without distribution:\n", + " * Batch 1: [0, 1, 2, 3]\n", + " * Batch 2: [4, 5]\n", + " * With distribution over 2 replicas.\n", + " The last batch ([4, 5]) is split between 2 replicas.\n", + "\n", + " * Batch 1:\n", + " * Replica 1:[0, 1]\n", + " * Replica 2:[2, 3]\n", + " * Batch 2:\n", + " * Replica 1: [4]\n", + " * Replica 2: [5]\n", + "\n", + "\n", + "\n", + "* `tf.data.Dataset.range(4).batch(4)`\n", + " * Without distribution:\n", + " * Batch 1: [0, 1, 2, 3]\n", + " * With distribution over 5 replicas:\n", + " * Batch 1:\n", + " * Replica 1: [0]\n", + " * Replica 2: [1]\n", + " * Replica 3: [2]\n", + " * Replica 4: [3]\n", + " * Replica 5: []\n", + "\n", + "* `tf.data.Dataset.range(8).batch(4)`\n", + " * Without distribution:\n", + " * Batch 1: [0, 1, 2, 3]\n", + " * Batch 2: [4, 5, 6, 7]\n", + " * With distribution over 3 replicas:\n", + " * Batch 1:\n", + " * Replica 1: [0, 1]\n", + " * Replica 2: [2, 3]\n", + " * Replica 3: []\n", + " * Batch 2:\n", + " * Replica 1: [4, 5]\n", + " * Replica 2: [6, 7]\n", + " * Replica 3: []\n", + "\n", + "Note: The above examples only illustrate how a global batch is split on different replicas. It is not advisable to depend on the actual values that might end up on each replica as it can change depending on the implementation.\n", + "\n", + "Rebatching the dataset has a space complexity that increases linearly with the number of replicas. This means that for the multi-worker training use case the input pipeline can run into OOM errors. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IszBuubdtydp" + }, + "source": [ + "##### Sharding\n", + "\n", + "`tf.distribute` also autoshards the input dataset in multi-worker training with `MultiWorkerMirroredStrategy` and `TPUStrategy`. Each dataset is created on the CPU device of the worker. Autosharding a dataset over a set of workers means that each worker is assigned a subset of the entire dataset (if the right `tf.data.experimental.AutoShardPolicy` is set). This is to ensure that at each step, a global batch size of non-overlapping dataset elements will be processed by each worker. Autosharding has a couple of different options that can be specified using `tf.data.experimental.DistributeOptions`. Note that there is no autosharding in multi-worker training with `ParameterServerStrategy`, and more information on dataset creation with this strategy can be found in the [ParameterServerStrategy tutorial](parameter_server_training.ipynb). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jwJtsCQhHK-E" + }, + "outputs": [], + "source": [ + "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(64).batch(16)\n", + "options = tf.data.Options()\n", + "options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.DATA\n", + "dataset = dataset.with_options(options)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "J7fj3GskHC8g" + }, + "source": [ + "There are three different options that you can set for the `tf.data.experimental.AutoShardPolicy`:\n", + "\n", + "* AUTO: This is the default option which means an attempt will be made to shard by FILE. The attempt to shard by FILE fails if a file-based dataset is not detected. `tf.distribute` will then fall back to sharding by DATA. Note that if the input dataset is file-based but the number of files is less than the number of workers, an `InvalidArgumentError` will be raised. If this happens, explicitly set the policy to `AutoShardPolicy.DATA`, or split your input source into smaller files such that number of files is greater than number of workers.\n", + "* FILE: This is the option if you want to shard the input files over all the workers. You should use this option if the number of input files is much larger than the number of workers and the data in the files is evenly distributed. The downside of this option is having idle workers if the data in the files is not evenly distributed. If the number of files is less than the number of workers, an `InvalidArgumentError` will be raised. If this happens, explicitly set the policy to `AutoShardPolicy.DATA`.\n", + "For example, let us distribute 2 files over 2 workers with 1 replica each. File 1 contains [0, 1, 2, 3, 4, 5] and\n", + "File 2 contains [6, 7, 8, 9, 10, 11]. Let the total number of replicas in sync be 2 and global batch size be 4.\n", + "\n", + " * Worker 0:\n", + " * Batch 1 = Replica 1: [0, 1]\n", + " * Batch 2 = Replica 1: [2, 3]\n", + " * Batch 3 = Replica 1: [4]\n", + " * Batch 4 = Replica 1: [5]\n", + " * Worker 1:\n", + " * Batch 1 = Replica 2: [6, 7]\n", + " * Batch 2 = Replica 2: [8, 9]\n", + " * Batch 3 = Replica 2: [10]\n", + " * Batch 4 = Replica 2: [11]\n", + "\n", + "* DATA: This will autoshard the elements across all the workers. Each of the workers will read the entire dataset and only process the shard assigned to it. All other shards will be discarded. This is generally used if the number of input files is less than the number of workers and you want better sharding of data across all workers. The downside is that the entire dataset will be read on each worker.\n", + "For example, let us distribute 1 files over 2 workers. File 1 contains [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]. Let the total number of replicas in sync be 2.\n", + "\n", + " * Worker 0:\n", + " * Batch 1 = Replica 1: [0, 1]\n", + " * Batch 2 = Replica 1: [4, 5]\n", + " * Batch 3 = Replica 1: [8, 9]\n", + " * Worker 1:\n", + " * Batch 1 = Replica 2: [2, 3]\n", + " * Batch 2 = Replica 2: [6, 7]\n", + " * Batch 3 = Replica 2: [10, 11]\n", + "\n", + "* OFF: If you turn off autosharding, each worker will process all the data.\n", + "For example, let us distribute 1 files over 2 workers. File 1 contains [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]. Let the total number of replicas in sync be 2. Then each worker will see the following distribution:\n", + "\n", + " * Worker 0:\n", + " * Batch 1 = Replica 1: [0, 1]\n", + " * Batch 2 = Replica 1: [2, 3]\n", + " * Batch 3 = Replica 1: [4, 5]\n", + " * Batch 4 = Replica 1: [6, 7]\n", + " * Batch 5 = Replica 1: [8, 9]\n", + " * Batch 6 = Replica 1: [10, 11]\n", + "\n", + " * Worker 1:\n", + " * Batch 1 = Replica 2: [0, 1]\n", + " * Batch 2 = Replica 2: [2, 3]\n", + " * Batch 3 = Replica 2: [4, 5]\n", + " * Batch 4 = Replica 2: [6, 7]\n", + " * Batch 5 = Replica 2: [8, 9]\n", + " * Batch 6 = Replica 2: [10, 11] " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OK46ZJGPH5H2" + }, + "source": [ + "##### Prefetching\n", + "\n", + "By default, `tf.distribute` adds a prefetch transformation at the end of the user provided `tf.data.Dataset` instance. The argument to the prefetch transformation which is `buffer_size` is equal to the number of replicas in sync." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PjiGSY3gtr6_" + }, + "source": [ + "### `tf.distribute.Strategy.distribute_datasets_from_function`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bAXAo_wWbWSb" + }, + "source": [ + "#### Usage\n", + "\n", + "This API takes an input function and returns a `tf.distribute.DistributedDataset` instance. The input function that users pass in has a `tf.distribute.InputContext` argument and should return a `tf.data.Dataset` instance. With this API, `tf.distribute` does not make any further changes to the user’s `tf.data.Dataset` instance returned from the input function. It is the responsibility of the user to batch and shard the dataset. `tf.distribute` calls the input function on the CPU device of each of the workers. Apart from allowing users to specify their own batching and sharding logic, this API also demonstrates better scalability and performance compared to `tf.distribute.Strategy.experimental_distribute_dataset` when used for multi-worker training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9ODch-OFCaW4" + }, + "outputs": [], + "source": [ + "mirrored_strategy = tf.distribute.MirroredStrategy()\n", + "\n", + "def dataset_fn(input_context):\n", + " batch_size = input_context.get_per_replica_batch_size(global_batch_size)\n", + " dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(64).batch(16)\n", + " dataset = dataset.shard(\n", + " input_context.num_input_pipelines, input_context.input_pipeline_id)\n", + " dataset = dataset.batch(batch_size)\n", + " dataset = dataset.prefetch(2) # This prefetches 2 batches per device.\n", + " return dataset\n", + "\n", + "dist_dataset = mirrored_strategy.distribute_datasets_from_function(dataset_fn)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M1bpzPYzt_R7" + }, + "source": [ + "#### Properties" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7cgzhwiiuBvO" + }, + "source": [ + "##### Batching\n", + "\n", + "The `tf.data.Dataset` instance that is the return value of the input function should be batched using the per replica batch size. The per replica batch size is the global batch size divided by the number of replicas that are taking part in sync training. This is because `tf.distribute` calls the input function on the CPU device of each of the workers. The dataset that is created on a given worker should be ready to use by all the replicas on that worker. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e-wlFFZbP33n" + }, + "source": [ + "##### Sharding\n", + "\n", + "The `tf.distribute.InputContext` object that is implicitly passed as an argument to the user’s input function is created by `tf.distribute` under the hood. It has information about the number of workers, current worker ID etc. This input function can handle sharding as per policies set by the user using these properties that are part of the `tf.distribute.InputContext` object.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7TGwnDM-ICHf" + }, + "source": [ + "##### Prefetching\n", + "\n", + "`tf.distribute` does not add a prefetch transformation at the end of the `tf.data.Dataset` returned by the user-provided input function, so you explicitly call `Dataset.prefetch` in the example above." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iOMsf8kyZZpv" + }, + "source": [ + "Note:\n", + "Both `tf.distribute.Strategy.experimental_distribute_dataset` and `tf.distribute.Strategy.distribute_datasets_from_function` return **`tf.distribute.DistributedDataset` instances that are not of type `tf.data.Dataset`**. You can iterate over these instances (as shown in the Distributed Iterators section) and use the `element_spec`\n", + "property. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dL3XbI1gzEjO" + }, + "source": [ + "## Distributed iterators" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "w8y54-o9T2Ni" + }, + "source": [ + "Similar to non-distributed `tf.data.Dataset` instances, you will need to create an iterator on the `tf.distribute.DistributedDataset` instances to iterate over it and access the elements in the `tf.distribute.DistributedDataset`.\n", + "The following are the ways in which you can create a `tf.distribute.DistributedIterator` and use it to train your model:\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FlKh8NV0uOtZ" + }, + "source": [ + "### Usages" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eSZz6EqOuSlB" + }, + "source": [ + "#### Use a Pythonic for loop construct\n", + "\n", + "You can use a user friendly Pythonic loop to iterate over the `tf.distribute.DistributedDataset`. The elements returned from the `tf.distribute.DistributedIterator` can be a single `tf.Tensor` or a `tf.distribute.DistributedValues` which contains a value per replica. Placing the loop inside a `tf.function` will give a performance boost. However, `break` and `return` are currently not supported for a loop over a `tf.distribute.DistributedDataset` that is placed inside of a `tf.function`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zt3AHb46Tr3w" + }, + "outputs": [], + "source": [ + "global_batch_size = 16\n", + "mirrored_strategy = tf.distribute.MirroredStrategy()\n", + "\n", + "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100).batch(global_batch_size)\n", + "dist_dataset = mirrored_strategy.experimental_distribute_dataset(dataset)\n", + "\n", + "@tf.function\n", + "def train_step(inputs):\n", + " features, labels = inputs\n", + " return labels - 0.3 * features\n", + "\n", + "for x in dist_dataset:\n", + " # train_step trains the model using the dataset elements\n", + " loss = mirrored_strategy.run(train_step, args=(x,))\n", + " print(\"Loss is \", loss)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NchPwTEiuSqb" + }, + "source": [ + "#### Use `iter` to create an explicit iterator\n", + "To iterate over the elements in a `tf.distribute.DistributedDataset` instance, you can create a `tf.distribute.DistributedIterator` using the `iter` API on it. With an explicit iterator, you can iterate for a fixed number of steps. In order to get the next element from an `tf.distribute.DistributedIterator` instance `dist_iterator`, you can call `next(dist_iterator)`, `dist_iterator.get_next()`, or `dist_iterator.get_next_as_optional()`. The former two are essentially the same:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OrMmakq5EqeQ" + }, + "outputs": [], + "source": [ + "num_epochs = 10\n", + "steps_per_epoch = 5\n", + "for epoch in range(num_epochs):\n", + " dist_iterator = iter(dist_dataset)\n", + " for step in range(steps_per_epoch):\n", + " # train_step trains the model using the dataset elements\n", + " loss = mirrored_strategy.run(train_step, args=(next(dist_iterator),))\n", + " # which is the same as\n", + " # loss = mirrored_strategy.run(train_step, args=(dist_iterator.get_next(),))\n", + " print(\"Loss is \", loss)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UpJXIlxjqPYg" + }, + "source": [ + "With `next` or `tf.distribute.DistributedIterator.get_next`, if the `tf.distribute.DistributedIterator` has reached its end, an OutOfRange error will be thrown. The client can catch the error on python side and continue doing other work such as checkpointing and evaluation. However, this will not work if you are using a host training loop (i.e., run multiple steps per `tf.function`), which looks like:\n", + "\n", + "```\n", + "@tf.function\n", + "def train_fn(iterator):\n", + " for _ in tf.range(steps_per_loop):\n", + " strategy.run(step_fn, args=(next(iterator),))\n", + "```\n", + "\n", + "This example `train_fn` contains multiple steps by wrapping the step body inside a `tf.range`. In this case, different iterations in the loop with no dependency could start in parallel, so an OutOfRange error can be triggered in later iterations before the computation of previous iterations finishes. Once an OutOfRange error is thrown, all the ops in the function will be terminated right away. If this is some case that you would like to avoid, an alternative that does not throw an OutOfRange error is `tf.distribute.DistributedIterator.get_next_as_optional`. `get_next_as_optional` returns a `tf.experimental.Optional` which contains the next element or no value if the `tf.distribute.DistributedIterator` has reached an end." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Iyjao96Vqwyz" + }, + "outputs": [], + "source": [ + "# You can break the loop with `get_next_as_optional` by checking if the `Optional` contains a value\n", + "global_batch_size = 4\n", + "steps_per_loop = 5\n", + "strategy = tf.distribute.MirroredStrategy()\n", + "\n", + "dataset = tf.data.Dataset.range(9).batch(global_batch_size)\n", + "distributed_iterator = iter(strategy.experimental_distribute_dataset(dataset))\n", + "\n", + "@tf.function\n", + "def train_fn(distributed_iterator):\n", + " for _ in tf.range(steps_per_loop):\n", + " optional_data = distributed_iterator.get_next_as_optional()\n", + " if not optional_data.has_value():\n", + " break\n", + " per_replica_results = strategy.run(lambda x: x, args=(optional_data.get_value(),))\n", + " tf.print(strategy.experimental_local_results(per_replica_results))\n", + "train_fn(distributed_iterator)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LaclbKnqzLjf" + }, + "source": [ + "## Using the `element_spec` property" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z1YvXqOpwy08" + }, + "source": [ + "If you pass the elements of a distributed dataset to a `tf.function` and want a `tf.TypeSpec` guarantee, you can specify the `input_signature` argument of the `tf.function`. The output of a distributed dataset is `tf.distribute.DistributedValues` which can represent the input to a single device or multiple devices. To get the `tf.TypeSpec` corresponding to this distributed value, you can use `tf.distribute.DistributedDataset.element_spec` or `tf.distribute.DistributedIterator.element_spec`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pg3B-Cw_cn3a" + }, + "outputs": [], + "source": [ + "global_batch_size = 16\n", + "epochs = 5\n", + "steps_per_epoch = 5\n", + "mirrored_strategy = tf.distribute.MirroredStrategy()\n", + "\n", + "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100).batch(global_batch_size)\n", + "dist_dataset = mirrored_strategy.experimental_distribute_dataset(dataset)\n", + "\n", + "@tf.function(input_signature=[dist_dataset.element_spec])\n", + "def train_step(per_replica_inputs):\n", + " def step_fn(inputs):\n", + " return 2 * inputs\n", + "\n", + " return mirrored_strategy.run(step_fn, args=(per_replica_inputs,))\n", + "\n", + "for _ in range(epochs):\n", + " iterator = iter(dist_dataset)\n", + " for _ in range(steps_per_epoch):\n", + " output = train_step(next(iterator))\n", + " tf.print(output)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-OAa6svUzuWm" + }, + "source": [ + "## Data preprocessing" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pSMrs3kJQexW" + }, + "source": [ + "So far, you have learned how to distribute a `tf.data.Dataset`. Yet before the data is ready for the model, it needs to be preprocessed, for example by cleansing, transforming, and augmenting it. Two sets of those handy tools are:\n", + "\n", + "* [Keras preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers): a set of Keras layers that allow developers to build Keras-native input processing pipelines. Some Keras preprocessing layers contain non-trainable states, which can be set on initialization or `adapt`ed (refer to the `adapt` section of the [Keras preprocessing layers guide](https://www.tensorflow.org/guide/keras/preprocessing_layers)). When distributing stateful preprocessing layers, the states should be replicated to all workers. To use these layers, you can either make them part of the model or apply them to the datasets.\n", + "\n", + "* [TensorFlow Transform (tf.Transform)](https://www.tensorflow.org/tfx/transform/get_started): a library for TensorFlow that allows you to define both instance-level and full-pass data transformation through data preprocessing pipelines. Tensorflow Transform has two phases. The first is the Analyze phase, where the raw training data is analyzed in a full-pass process to compute the statistics needed for the transformations, and the transformation logic is generated as instance-level operations. The second is the Transform phase, where the raw training data is transformed in an instance-level process.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Pd4aUCFdVlZ1" + }, + "source": [ + "### Keras preprocessing layers vs. Tensorflow Transform \n", + "\n", + "Both Tensorflow Transform and Keras preprocessing layers provide a way to split out preprocessing during training and bundle preprocessing with a model during inference, reducing train/serve skew.\n", + "\n", + "Tensorflow Transform, deeply integrated with [TFX](https://www.tensorflow.org/tfx), provides a scalable map-reduce solution to analyzing and transforming datasets of any size in a job separate from the training pipeline. If you need to run an analysis on a dataset that cannot fit on a single machine, Tensorflow Transform should be your first choice.\n", + "\n", + "Keras preprocessing layers are more geared towards preprocessing applied during training, after reading data from disk. They fit seamlessly with model development in the Keras library. They support analysis of a smaller dataset via [`adapt`](https://www.tensorflow.org/guide/keras/preprocessing_layers#the_adapt_method) and supports use cases like image data augmentation, where each pass over the input dataset will yield different examples for training.\n", + "\n", + "The two libraries can also be mixed, where Tensorflow Transform is used for analysis and static transformations of input data, and Keras preprocessing layers are used for train-time transformations (e.g., one-hot encoding or data augmentation).\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MReKhhZpHUpj" + }, + "source": [ + "### Best Practice with tf.distribute\n", + "\n", + "Working with both tools involves initializing the transformation logic to apply to data, which might create Tensorflow resources. These resources or states should be replicated to all workers to save inter-workers or worker-coordinator communication. To do so, you are recommended to create Keras preprocessing layers, `tft.TFTransformOutput.transform_features_layer`, or `tft.TransformFeaturesLayer` under `tf.distribute.Strategy.scope`, just like you would for any other Keras layers.\n", + "\n", + "The following examples demonstrate usage of the `tf.distribute.Strategy` API with the high-level Keras `Model.fit` API and with a custom training loop separately." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rwEGMWuoX7kJ" + }, + "source": [ + "#### Extra notes for Keras preprocessing layers users:\n", + "\n", + "**Preprocessing layers and large vocabularies**\n", + "\n", + "When dealing with large vocabularies (over one gigabyte) in a multi-worker setting (for example, `tf.distribute.MultiWorkerMirroredStrategy`, `tf.distribute.experimental.ParameterServerStrategy`, `tf.distribute.TPUStrategy`), it is recommended to save the vocabulary to a static file accessible from all workers (for example, with Cloud Storage). This will reduce the time spent replicating the vocabulary to all workers during training.\n", + "\n", + "**Preprocessing in the `tf.data` pipeline versus in the model**\n", + "\n", + "While Keras preprocessing layers can be applied either as part of the model or directly to a `tf.data.Dataset`, each of the options come with their edge:\n", + "\n", + "* Applying the preprocessing layers within the model makes your model portable, and it helps reduce the training/serving skew. (For more details, refer to the _Benefits of doing preprocessing inside the model at inference time_ section in the [Working with preprocessing layers guide](https://www.tensorflow.org/guide/keras/preprocessing_layers#benefits_of_doing_preprocessing_inside_the_model_at_inference_time))\n", + "* Applying within the `tf.data` pipeline allows prefetching or offloading to the CPU, which generally gives better performance when using accelerators.\n", + "\n", + "When running on one or more TPUs, users should almost always place Keras preprocessing layers in the `tf.data` pipeline, as not all layers support TPUs, and string ops do not execute on TPUs. (The two exceptions are `tf.keras.layers.Normalization` and `tf.keras.layers.Rescaling`, which run fine on TPUs and are commonly used as the first layer in an image model.)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hNCYZ9L-BD2R" + }, + "source": [ + "### Preprocessing with `Model.fit`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NhRB2Xe8B6bX" + }, + "source": [ + "When using Keras `Model.fit`, you do not need to distribute data with `tf.distribute.Strategy.experimental_distribute_dataset` nor `tf.distribute.Strategy.distribute_datasets_from_function` themselves. Check out the [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) guide and the [Distributed training with Keras](https://www.tensorflow.org/tutorials/distribute/keras) guide for details. A shortened example may look as below:\n", + "\n", + "```\n", + "strategy = tf.distribute.MirroredStrategy()\n", + "with strategy.scope():\n", + " # Create the layer(s) under scope.\n", + " integer_preprocessing_layer = tf.keras.layers.IntegerLookup(vocabulary=FILE_PATH)\n", + " model = ...\n", + " model.compile(...)\n", + "dataset = dataset.map(lambda x, y: (integer_preprocessing_layer(x), y))\n", + "model.fit(dataset)\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3zL2vzJ-G0yg" + }, + "source": [ + "Users of `tf.distribute.experimental.ParameterServerStrategy` with the `Model.fit` API need to use a `tf.keras.utils.experimental.DatasetCreator` as the input. (See the [Parameter Server Training](https://www.tensorflow.org/tutorials/distribute/parameter_server_training#parameter_server_training_with_modelfit_api) guide for more)\n", + "\n", + "```\n", + "strategy = tf.distribute.experimental.ParameterServerStrategy(\n", + " cluster_resolver,\n", + " variable_partitioner=variable_partitioner)\n", + "\n", + "with strategy.scope():\n", + " preprocessing_layer = tf.keras.layers.StringLookup(vocabulary=FILE_PATH)\n", + " model = ...\n", + " model.compile(...)\n", + "\n", + "def dataset_fn(input_context):\n", + " ...\n", + " dataset = dataset.map(preprocessing_layer)\n", + " ...\n", + " return dataset\n", + "\n", + "dataset_creator = tf.keras.utils.experimental.DatasetCreator(dataset_fn)\n", + "model.fit(dataset_creator, epochs=5, steps_per_epoch=20, callbacks=callbacks)\n", + "\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "imZLQUOYBJyW" + }, + "source": [ + "### Preprocessing with a custom training loop" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r2PX1QH_OwU3" + }, + "source": [ + "When writing a [custom training loop](https://www.tensorflow.org/tutorials/distribute/custom_training), you will distribute your data with either the `tf.distribute.Strategy.experimental_distribute_dataset` API or the `tf.distribute.Strategy.distribute_datasets_from_function` API. If you distribute your dataset through `tf.distribute.Strategy.experimental_distribute_dataset`, applying these preprocessing APIs in your data pipeline will lead the resources automatically co-located with the data pipeline to avoid remote resource access. Thus the examples here will all use `tf.distribute.Strategy.distribute_datasets_from_function`, in which case it is crucial to place initialization of these APIs under `strategy.scope()` for efficiency:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wJS1UmcWQeab" + }, + "outputs": [], + "source": [ + "strategy = tf.distribute.MirroredStrategy()\n", + "vocab = [\"a\", \"b\", \"c\", \"d\", \"f\"]\n", + "\n", + "with strategy.scope():\n", + " # Create the layer(s) under scope.\n", + " layer = tf.keras.layers.StringLookup(vocabulary=vocab)\n", + "\n", + "def dataset_fn(input_context):\n", + " # a tf.data.Dataset\n", + " dataset = tf.data.Dataset.from_tensor_slices([\"a\", \"c\", \"e\"]).repeat()\n", + "\n", + " # Custom your batching, sharding, prefetching, etc.\n", + " global_batch_size = 4\n", + " batch_size = input_context.get_per_replica_batch_size(global_batch_size)\n", + " dataset = dataset.batch(batch_size)\n", + " dataset = dataset.shard(\n", + " input_context.num_input_pipelines,\n", + " input_context.input_pipeline_id)\n", + "\n", + " # Apply the preprocessing layer(s) to the tf.data.Dataset\n", + " def preprocess_with_kpl(input):\n", + " return layer(input)\n", + "\n", + " processed_ds = dataset.map(preprocess_with_kpl)\n", + " return processed_ds\n", + "\n", + "distributed_dataset = strategy.distribute_datasets_from_function(dataset_fn)\n", + "\n", + "# Print out a few example batches.\n", + "distributed_dataset_iterator = iter(distributed_dataset)\n", + "for _ in range(3):\n", + " print(next(distributed_dataset_iterator))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PVl1cblWQy8b" + }, + "source": [ + "Note that if you are training with `tf.distribute.experimental.ParameterServerStrategy`, you'll also call `tf.distribute.experimental.coordinator.ClusterCoordinator.create_per_worker_dataset`\n", + "\n", + "```\n", + "@tf.function\n", + "def per_worker_dataset_fn():\n", + " return strategy.distribute_datasets_from_function(dataset_fn)\n", + "\n", + "per_worker_dataset = coordinator.create_per_worker_dataset(per_worker_dataset_fn)\n", + "per_worker_iterator = iter(per_worker_dataset)\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ol7SmPID1dAt" + }, + "source": [ + "For Tensorflow Transform, as mentioned above, the Analyze stage is done separately from training and thus omitted here. See the [tutorial](https://www.tensorflow.org/tfx/tutorials/transform/census) for a detailed how-to. Usually, this stage includes creating a `tf.Transform` preprocessing function and transforming the data in an [Apache Beam](https://beam.apache.org/) pipeline with this preprocessing function. At the end of the Analyze stage, the output can be exported as a TensorFlow graph which you can use for both training and serving. Our example covers only the training pipeline part:\n", + "\n", + "```\n", + "with strategy.scope():\n", + " # working_dir contains the tf.Transform output.\n", + " tf_transform_output = tft.TFTransformOutput(working_dir)\n", + " # Loading from working_dir to create a Keras layer for applying the tf.Transform output to data\n", + " tft_layer = tf_transform_output.transform_features_layer()\n", + " ...\n", + "\n", + "def dataset_fn(input_context):\n", + " ...\n", + " dataset.map(tft_layer, num_parallel_calls=tf.data.AUTOTUNE)\n", + " ...\n", + " return dataset\n", + "\n", + "distributed_dataset = strategy.distribute_datasets_from_function(dataset_fn)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3_IQxRXxQWof" + }, + "source": [ + "## Partial batches" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hW2_gVkiztUG" + }, + "source": [ + "Partial batches are encountered when: 1) `tf.data.Dataset` instances that users create may contain batch sizes that are not evenly divisible by the number of replicas; or 2) when the cardinality of the dataset instance is not divisible by the batch size. This means that when the dataset is distributed over multiple replicas, the `next` call on some iterators will result in an `tf.errors.OutOfRangeError`. To handle this use case, `tf.distribute` returns dummy batches of batch size `0` on replicas that do not have any more data to process.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rqutdpqtPcCH" + }, + "source": [ + "For the single-worker case, if the data is not returned by the `next` call on the iterator, dummy batches of 0 batch size are created and used along with the real data in the dataset. In the case of partial batches, the last global batch of data will contain real data alongside dummy batches of data. The stopping condition for processing data now checks if any of the replicas have data. If there is no data on any of the replicas, you will get a `tf.errors.OutOfRangeError`.\n", + "\n", + "For the multi-worker case, the boolean value representing presence of data on each of the workers is aggregated using cross replica communication and this is used to identify if all the workers have finished processing the distributed dataset. Since this involves cross worker communication there is some performance penalty involved.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vehLsgljz90Y" + }, + "source": [ + "## Caveats" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Nx4jyN_Az-Dy" + }, + "source": [ + "* When using `tf.distribute.Strategy.experimental_distribute_dataset` APIs with a multi-worker setup, you pass a `tf.data.Dataset` that reads from files. If the `tf.data.experimental.AutoShardPolicy` is set to `AUTO` or `FILE`, the actual per-step batch size may be smaller than the one you defined for the global batch size. This can happen when the remaining elements in the file are less than the global batch size. You can either exhaust the dataset without depending on the number of steps to run, or set `tf.data.experimental.AutoShardPolicy` to `DATA` to work around it.\n", + "\n", + "* Stateful dataset transformations are currently not supported with `tf.distribute` and any stateful ops that the dataset may have are currently ignored. For example, if your dataset has a `map_fn` that uses `tf.random.uniform` to rotate an image, then you have a dataset graph that depends on state (i.e the random seed) on the local machine where the python process is being executed.\n", + "\n", + "* Experimental `tf.data.experimental.OptimizationOptions` that are disabled by default can in certain contexts—such as when used together with `tf.distribute`—cause a performance degradation. You should only enable them after you validate that they benefit the performance of your workload in a distribute setting.\n", + "\n", + "* Please refer to [this guide](https://www.tensorflow.org/guide/data_performance) for how to optimize your input pipeline with `tf.data` in general. A few additional tips:\n", + " * If you have multiple workers and are using `tf.data.Dataset.list_files` to create a dataset from all files matching one or more glob patterns, remember to set the `seed` argument or set `shuffle=False` so that each worker shard the file consistently.\n", + "\n", + " * If your input pipeline includes both shuffling the data on record level and parsing the data, unless the unparsed data is significantly larger than the parsed data (which is usually not the case), shuffle first and then parse, as shown in the following example. This may benefit memory usage and performance.\n", + "```\n", + "d = tf.data.Dataset.list_files(pattern, shuffle=False)\n", + "d = d.shard(num_workers, worker_index)\n", + "d = d.repeat(num_epochs)\n", + "d = d.shuffle(shuffle_buffer_size)\n", + "d = d.interleave(tf.data.TFRecordDataset,\n", + " cycle_length=num_readers, block_length=1)\n", + "d = d.map(parser_fn, num_parallel_calls=num_map_threads)\n", + "```\n", + " * `tf.data.Dataset.shuffle(buffer_size, seed=None, reshuffle_each_iteration=None)` maintain an internal buffer of `buffer_size` elements, and thus reducing `buffer_size` could aleviate OOM issue." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dAC_vRmJyzrB" + }, + "source": [ + "* The order in which the data is processed by the workers when using `tf.distribute.experimental_distribute_dataset` or `tf.distribute.distribute_datasets_from_function` is not guaranteed. This is typically required if you are using `tf.distribute` to scale prediction. You can however insert an index for each element in the batch and order outputs accordingly. The following snippet is an example of how to order outputs.\n", + "\n", + "Note: `tf.distribute.MirroredStrategy` is used here for the sake of convenience. You only need to reorder inputs when you are using multiple workers, but `tf.distribute.MirroredStrategy` is used to distribute training on a single worker." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Zr2xAy-uZZaL" + }, + "outputs": [], + "source": [ + "mirrored_strategy = tf.distribute.MirroredStrategy()\n", + "dataset_size = 24\n", + "batch_size = 6\n", + "dataset = tf.data.Dataset.range(dataset_size).enumerate().batch(batch_size)\n", + "dist_dataset = mirrored_strategy.experimental_distribute_dataset(dataset)\n", + "\n", + "def predict(index, inputs):\n", + " outputs = 2 * inputs\n", + " return index, outputs\n", + "\n", + "result = {}\n", + "for index, inputs in dist_dataset:\n", + " output_index, outputs = mirrored_strategy.run(predict, args=(index, inputs))\n", + " indices = list(mirrored_strategy.experimental_local_results(output_index))\n", + " rindices = []\n", + " for a in indices:\n", + " rindices.extend(a.numpy())\n", + " outputs = list(mirrored_strategy.experimental_local_results(outputs))\n", + " routputs = []\n", + " for a in outputs:\n", + " routputs.extend(a.numpy())\n", + " for i, value in zip(rindices, routputs):\n", + " result[i] = value\n", + "\n", + "print(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nNbn7HXx0YqB" + }, + "source": [ + "\n", + "## Tensor inputs instead of tf.data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dymZixqo0nKK" + }, + "source": [ + "Sometimes users cannot use a `tf.data.Dataset` to represent their input and subsequently\n", + "the above mentioned APIs to distribute the dataset to multiple devices.\n", + " In such cases you can use raw tensors or inputs from a generator.\n", + "\n", + "### Use experimental_distribute_values_from_function for arbitrary tensor inputs\n", + "`strategy.run` accepts `tf.distribute.DistributedValues` which is the output of\n", + "`next(iterator)`. To pass the tensor values, use\n", + "`tf.distribute.Strategy.experimental_distribute_values_from_function` to construct\n", + "`tf.distribute.DistributedValues` from raw tensors. The user will have to specify their own batching and sharding logic in the input function with this option, which can be done using the `tf.distribute.experimental.ValueContext` input object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ajZHNRQs0kqm" + }, + "outputs": [], + "source": [ + "mirrored_strategy = tf.distribute.MirroredStrategy()\n", + "\n", + "def value_fn(ctx):\n", + " return tf.constant(ctx.replica_id_in_sync_group)\n", + "\n", + "distributed_values = mirrored_strategy.experimental_distribute_values_from_function(value_fn)\n", + "for _ in range(4):\n", + " result = mirrored_strategy.run(lambda x: x, args=(distributed_values,))\n", + " print(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P98aFQGf0x_7" + }, + "source": [ + "### Use tf.data.Dataset.from_generator if your input is from a generator" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "emZCWQSi04qT" + }, + "source": [ + "If you have a generator function that you want to use, you can create a `tf.data.Dataset` instance using the `from_generator` API.\n", + "\n", + "Note: This is currently not supported for `tf.distribute.TPUStrategy`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jRhU0X230787" + }, + "outputs": [], + "source": [ + "mirrored_strategy = tf.distribute.MirroredStrategy()\n", + "def input_gen():\n", + " while True:\n", + " yield np.random.rand(4)\n", + "\n", + "# use Dataset.from_generator\n", + "dataset = tf.data.Dataset.from_generator(\n", + " input_gen, output_types=(tf.float32), output_shapes=tf.TensorShape([4]))\n", + "dist_dataset = mirrored_strategy.experimental_distribute_dataset(dataset)\n", + "iterator = iter(dist_dataset)\n", + "for _ in range(4):\n", + " result = mirrored_strategy.run(lambda x: x, args=(next(iterator),))\n", + " print(result)" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "input.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/distribute/keras.ipynb b/site/en/tutorials/distribute/keras.ipynb index 098cfe37285..b96656d4436 100644 --- a/site/en/tutorials/distribute/keras.ipynb +++ b/site/en/tutorials/distribute/keras.ipynb @@ -3,21 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Tce3stUlHN0L" }, "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" + "##### Copyright 2019 The TensorFlow Authors.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "tuOe1ymfHZPu" }, "outputs": [], @@ -38,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MfBg1C5NB3X0" }, "source": [ @@ -48,100 +43,74 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "r6P32iYYV27b" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/distribute/keras\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/distribute/keras.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/distribute/keras.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/distribute/keras.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xHxb-dlhMIzW" }, "source": [ "## Overview\n", "\n", - "The `tf.distribute.Strategy` API provides an abstraction for distributing your training\n", - "across multiple processing units. The goal is to allow users to enable distributed training using existing models and training code, with minimal changes.\n", + "The `tf.distribute.Strategy` API provides an abstraction for distributing your training across multiple processing units. It allows you to carry out distributed training using existing models and training code with minimal changes.\n", "\n", - "This tutorial uses the `tf.distribute.MirroredStrategy`, which\n", - "does in-graph replication with synchronous training on many GPUs on one machine.\n", - "Essentially, it copies all of the model's variables to each processor.\n", - "Then, it uses [all-reduce](http://mpitutorial.com/tutorials/mpi-reduce-and-allreduce/) to combine the gradients from all processors and applies the combined value to all copies of the model.\n", + "This tutorial demonstrates how to use the `tf.distribute.MirroredStrategy` to perform in-graph replication with _synchronous training on many GPUs on one machine_. The strategy essentially copies all of the model's variables to each processor. Then, it uses [all-reduce](http://mpitutorial.com/tutorials/mpi-reduce-and-allreduce/) to combine the gradients from all processors, and applies the combined value to all copies of the model.\n", "\n", - "`MirroredStrategy` is one of several distribution strategy available in TensorFlow core. You can read about more strategies at [distribution strategy guide](../../guide/distributed_training.ipynb).\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUXex9ctTuDB" - }, - "source": [ - "### Keras API\n", + "You will use the `tf.keras` APIs to build the model and `Model.fit` for training it. (To learn about distributed training with a custom training loop and the `MirroredStrategy`, check out [this tutorial](custom_training.ipynb).)\n", + "\n", + "`MirroredStrategy` trains your model on multiple GPUs on a single machine. For _synchronous training on many GPUs on multiple workers_, use the `tf.distribute.MultiWorkerMirroredStrategy` with the [Keras Model.fit](multi_worker_with_keras.ipynb) or [a custom training loop](multi_worker_with_ctl.ipynb). For other options, refer to the [Distributed training guide](../../guide/distributed_training.ipynb).\n", "\n", - "This example uses the `tf.keras` API to build the model and training loop. For custom training loops, see the [tf.distribute.Strategy with training loops](training_loops.ipynb) tutorial." + "To learn about various other strategies, there is the [Distributed training with TensorFlow](../../guide/distributed_training.ipynb) guide." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Dney9v7BsJij" }, "source": [ - "## Import dependencies" + "## Setup" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "r8S3ublR7Ay8" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# Import TensorFlow and TensorFlow Datasets\n", - "try:\n", - " !pip install tf-nightly\n", - "except Exception:\n", - " pass\n", - "\n", "import tensorflow_datasets as tfds\n", "import tensorflow as tf\n", - "tfds.disable_progress_bar()\n", "\n", - "import os" + "import os\n", + "\n", + "# Load the TensorBoard notebook extension.\n", + "%load_ext tensorboard" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "SkocY8tgRd3H" }, "outputs": [], @@ -152,7 +121,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hXhefksNKk2I" }, "source": [ @@ -162,30 +130,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OtnnUwvmB3X5" }, "source": [ - "Download the MNIST dataset and load it from [TensorFlow Datasets](https://www.tensorflow.org/datasets). This returns a dataset in `tf.data` format." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lHAPqG8MtS8M" - }, - "source": [ - "Setting `with_info` to `True` includes the metadata for the entire dataset, which is being saved here to `info`.\n", - "Among other things, this metadata object includes the number of train and test examples. \n" + "Load the MNIST dataset from [TensorFlow Datasets](https://www.tensorflow.org/datasets). This returns a dataset in the `tf.data` format.\n", + "\n", + "Setting the `with_info` argument to `True` includes the metadata for the entire dataset, which is being saved here to `info`. Among other things, this metadata object includes the number of train and test examples." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iXMJ3G9NB3X6" }, "outputs": [], @@ -198,29 +154,25 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GrjVhv-eKuHD" }, "source": [ - "## Define distribution strategy" + "## Define the distribution strategy" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TlH8vx6BB3X9" }, "source": [ - "Create a `MirroredStrategy` object. This will handle distribution, and provides a context manager (`tf.distribute.MirroredStrategy.scope`) to build your model inside." + "Create a `MirroredStrategy` object. This will handle distribution and provide a context manager (`MirroredStrategy.scope`) to build your model inside." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4j0tdf4YB3X9" }, "outputs": [], @@ -230,10 +182,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "cY3KA_h2iVfN" }, "outputs": [], @@ -244,29 +194,25 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lNbPv0yAleW8" }, "source": [ - "## Setup input pipeline" + "## Set up the input pipeline" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "psozqcuptXhK" }, "source": [ - "When training a model with multiple GPUs, you can use the extra computing power effectively by increasing the batch size. In general, use the largest batch size that fits the GPU memory, and tune the learning rate accordingly." + "When training a model with multiple GPUs, you can use the extra computing power effectively by increasing the batch size. In general, use the largest batch size that fits the GPU memory and tune the learning rate accordingly." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "p1xWxKcnhar9" }, "outputs": [], @@ -286,19 +232,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0Wm5rsL2KoDF" }, "source": [ - "Pixel values, which are 0-255, [have to be normalized to the 0-1 range](https://en.wikipedia.org/wiki/Feature_scaling). Define this scale in a function." + "Define a function that normalizes the image pixel values from the `[0, 255]` range to the `[0, 1]` range ([feature scaling](https://en.wikipedia.org/wiki/Feature_scaling)):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Eo9a46ZeJCkm" }, "outputs": [], @@ -313,19 +256,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WZCa5RLc5A91" }, "source": [ - "Apply this function to the training and test data, shuffle the training data, and [batch it for training](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch). Notice we are also keeping an in-memory cache of the training data to improve performance.\n" + "Apply this `scale` function to the training and test data, and then use the `tf.data.Dataset` APIs to shuffle the training data (`Dataset.shuffle`), and batch it (`Dataset.batch`). Notice that you are also keeping an in-memory cache of the training data to improve performance (`Dataset.cache`)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gRZu2maChwdT" }, "outputs": [], @@ -337,29 +277,25 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4xsComp8Kz5H" }, "source": [ - "## Create the model" + "## Create the model and instantiate the optimizer" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1BnQYQTpB3YA" }, "source": [ - "Create and compile the Keras model in the context of `strategy.scope`." + "Within the context of `Strategy.scope`, create and compile the model using the Keras API:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "IexhL_vIB3YA" }, "outputs": [], @@ -370,74 +306,80 @@ " tf.keras.layers.MaxPooling2D(),\n", " tf.keras.layers.Flatten(),\n", " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", + " tf.keras.layers.Dense(10)\n", " ])\n", "\n", - " model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", + " model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "DCDKFcNJzdcd" + }, + "source": [ + "For this toy example with the MNIST dataset, you will be using the Adam optimizer's default learning rate of 0.001.\n", + "\n", + "For larger datasets, the key benefit of distributed training is to learn more in each training step, because each step processes more training data in parallel, which allows for a larger learning rate (within the limits of the model and dataset)." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "8i6OU5W9Vy2u" }, "source": [ - "## Define the callbacks\n", - "\n" + "## Define the callbacks\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YOXO5nvvK3US" }, "source": [ - "The callbacks used here are:\n", + "Define the following [Keras Callbacks](https://www.tensorflow.org/guide/keras/train_and_evaluate):\n", + "\n", + "- `tf.keras.callbacks.TensorBoard`: writes a log for TensorBoard, which allows you to visualize the graphs.\n", + "- `tf.keras.callbacks.ModelCheckpoint`: saves the model at a certain frequency, such as after every epoch.\n", + "- `tf.keras.callbacks.BackupAndRestore`: provides the fault tolerance functionality by backing up the model and current epoch number. Learn more in the _Fault tolerance_ section of the [Multi-worker training with Keras](multi_worker_with_keras.ipynb) tutorial.\n", + "- `tf.keras.callbacks.LearningRateScheduler`: schedules the learning rate to change after, for example, every epoch/batch.\n", "\n", - "* *TensorBoard*: This callback writes a log for TensorBoard which allows you to visualize the graphs.\n", - "* *Model Checkpoint*: This callback saves the model after every epoch.\n", - "* *Learning Rate Scheduler*: Using this callback, you can schedule the learning rate to change after every epoch/batch.\n", + "For illustrative purposes, add a [custom callback](https://www.tensorflow.org/guide/keras/custom_callback) called `PrintLR` to display the *learning rate* in the notebook.\n", "\n", - "For illustrative purposes, add a print callback to display the *learning rate* in the notebook." + "**Note:** Use the `BackupAndRestore` callback instead of `ModelCheckpoint` as the main mechanism to restore the training state upon a restart from a job failure. Since `BackupAndRestore` only supports eager mode, in graph mode consider using `ModelCheckpoint`." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "A9bwLCcXzSgy" }, "outputs": [], "source": [ - "# Define the checkpoint directory to store the checkpoints\n", - "\n", + "# Define the checkpoint directory to store the checkpoints.\n", "checkpoint_dir = './training_checkpoints'\n", - "# Name of the checkpoint files\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")" + "# Define the name of the checkpoint files.\n", + "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch:04d}.weights.h5\")" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wpU-BEdzJDbK" }, "outputs": [], "source": [ - "# Function for decaying the learning rate.\n", + "# Define a function for decaying the learning rate.\n", "# You can define any decay function you need.\n", "def decay(epoch):\n", - " if epoch \u003c 3:\n", + " if epoch < 3:\n", " return 1e-3\n", - " elif epoch \u003e= 3 and epoch \u003c 7:\n", + " elif epoch >= 3 and epoch < 7:\n", " return 1e-4\n", " else:\n", " return 1e-5" @@ -445,31 +387,27 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "jKhiMgXtKq2w" }, "outputs": [], "source": [ - "# Callback for printing the LR at the end of each epoch.\n", + "# Define a callback for printing the learning rate at the end of each epoch.\n", "class PrintLR(tf.keras.callbacks.Callback):\n", " def on_epoch_end(self, epoch, logs=None):\n", - " print('\\nLearning rate for epoch {} is {}'.format(epoch + 1,\n", - " model.optimizer.lr.numpy()))" + " print('\\nLearning rate for epoch {} is {}'.format(epoch + 1, model.optimizer.learning_rate.numpy()))" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YVqAbR6YyNQh" }, "outputs": [], "source": [ + "# Put all the callbacks together.\n", "callbacks = [\n", " tf.keras.callbacks.TensorBoard(log_dir='./logs'),\n", " tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,\n", @@ -482,7 +420,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "70HXgDQmK46q" }, "source": [ @@ -492,99 +429,106 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6EophnOAB3YD" }, "source": [ - "Now, train the model in the usual way, calling `fit` on the model and passing in the dataset created at the beginning of the tutorial. This step is the same whether you are distributing the training or not.\n" + "Now, train the model in the usual way by calling Keras `Model.fit` on the model and passing in the dataset created at the beginning of the tutorial. This step is the same whether you are distributing the training or not." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7MVw_6CqB3YD" }, "outputs": [], "source": [ - "model.fit(train_dataset, epochs=12, callbacks=callbacks)" + "EPOCHS = 12\n", + "\n", + "model.fit(train_dataset, epochs=EPOCHS, callbacks=callbacks)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NUcWAUUupIvG" }, "source": [ - "As you can see below, the checkpoints are getting saved." + "Check for saved checkpoints:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "JQ4zeSTxKEhB" }, "outputs": [], "source": [ - "# check the checkpoint directory\n", + "# Check the checkpoint directory.\n", "!ls {checkpoint_dir}" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qor53h7FpMke" }, "source": [ - "To see how the model perform, load the latest checkpoint and call `evaluate` on the test data.\n", - "\n", - "Call `evaluate` as before using appropriate datasets." + "To check how well the model performs, load the latest checkpoint and call `Model.evaluate` on the test data:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "JtEwxiTgpQoP" }, "outputs": [], "source": [ - "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", + "import pathlib\n", + "latest_checkpoint = sorted(pathlib.Path(checkpoint_dir).glob('*'))[-1]\n", + "\n", + "model.load_weights(latest_checkpoint)\n", "\n", "eval_loss, eval_acc = model.evaluate(eval_dataset)\n", "\n", - "print('Eval loss: {}, Eval Accuracy: {}'.format(eval_loss, eval_acc))" + "print('Eval loss: {}, Eval accuracy: {}'.format(eval_loss, eval_acc))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IIeF2RWfYu4N" }, "source": [ - "To see the output, you can download and view the TensorBoard logs at the terminal.\n", - "\n", - "```\n", - "$ tensorboard --logdir=path/to/log-directory\n", - "```" + "To visualize the output, launch TensorBoard and view the logs:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, + "metadata": { + "id": "vtyAZO0DoKu_" + }, + "outputs": [], + "source": [ + "%tensorboard --logdir=logs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a0a82d26d6bd" + }, + "source": [ + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "LnyscOkvKKBR" }, "outputs": [], @@ -595,65 +539,56 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kBLlogrDvMgg" }, "source": [ - "## Export to SavedModel" + "## Save the model" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Xa87y_A0vRma" }, "source": [ - "Export the graph and the variables to the platform-agnostic SavedModel format. After your model is saved, you can load it with or without the scope.\n" + "Save the model to a `.keras` zip archive using `Model.save`. After your model is saved, you can load it with or without the `Strategy.scope`." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "h8Q4MKOLwG7K" }, "outputs": [], "source": [ - "path = 'saved_model/'" + "path = 'my_model.keras'" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4HvcDmVsvQoa" }, "outputs": [], "source": [ - "model.save(path, save_format='tf')" + "model.save(path)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vKJT4w5JwVPI" }, "source": [ - "Load the model without `strategy.scope`." + "Now, load the model without `Strategy.scope`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "T_gT0RbRvQ3o" }, "outputs": [], @@ -661,7 +596,7 @@ "unreplicated_model = tf.keras.models.load_model(path)\n", "\n", "unreplicated_model.compile(\n", - " loss='sparse_categorical_crossentropy',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " optimizer=tf.keras.optimizers.Adam(),\n", " metrics=['accuracy'])\n", "\n", @@ -673,26 +608,23 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YBLzcRF0wbDe" }, "source": [ - "Load the model with `strategy.scope`." + "Load the model with `Strategy.scope`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "BBVo3WGGwd9a" }, "outputs": [], "source": [ "with strategy.scope():\n", " replicated_model = tf.keras.models.load_model(path)\n", - " replicated_model.compile(loss='sparse_categorical_crossentropy',\n", + " replicated_model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " optimizer=tf.keras.optimizers.Adam(),\n", " metrics=['accuracy'])\n", "\n", @@ -703,31 +635,26 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MUZwaz4AKjtD" }, "source": [ - "### Examples and Tutorials\n", - "Here are some examples for using distribution strategy with keras fit/compile:\n", - "1. [Transformer](https://github.com/tensorflow/models/blob/master/official/transformer/v2/transformer_main.py) example trained using `tf.distribute.MirroredStrategy`\n", - "2. [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) example trained using `tf.distribute.MirroredStrategy`.\n", + "### Additional resources\n", "\n", - "More examples listed in the [Distribution strategy guide](../../guide/distributed_training.ipynb#examples_and_tutorials)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8uNqWRdDMl5S" - }, - "source": [ - "## Next steps\n", + "More examples that use different distribution strategies with the Keras `Model.fit` API:\n", + "\n", + "1. The [Solve GLUE tasks using BERT on TPU](https://www.tensorflow.org/text/tutorials/bert_glue) tutorial uses `tf.distribute.MirroredStrategy` for training on GPUs and `tf.distribute.TPUStrategy` on TPUs.\n", + "1. The [Save and load a model using a distribution strategy](save_and_load.ipynb) tutorial demonstates how to use the SavedModel APIs with `tf.distribute.Strategy`.\n", + "1. The [official TensorFlow models](https://github.com/tensorflow/models/tree/master/official) can be configured to run multiple distribution strategies.\n", + "\n", + "To learn more about TensorFlow distribution strategies:\n", "\n", - "* Read the [distribution strategy guide](../../guide/distributed_training.ipynb).\n", - "* Read the [Distributed Training with Custom Training Loops](training_loops.ipynb) tutorial.\n", + "1. The [Custom training with tf.distribute.Strategy](custom_training.ipynb) tutorial shows how to use the `tf.distribute.MirroredStrategy` for single-worker training with a custom training loop.\n", + "1. The [Multi-worker training with Keras](multi_worker_with_keras.ipynb) tutorial shows how to use the `MultiWorkerMirroredStrategy` with `Model.fit`.\n", + "1. The [Custom training loop with Keras and MultiWorkerMirroredStrategy](multi_worker_with_ctl.ipynb) tutorial shows how to use the `MultiWorkerMirroredStrategy` with Keras and a custom training loop.\n", + "1. The [Distributed training in TensorFlow](https://www.tensorflow.org/guide/distributed_training) guide provides an overview of the available distribution strategies.\n", + "1. The [Better performance with tf.function](../../guide/function.ipynb) guide provides information about other strategies and tools, such as the [TensorFlow Profiler](../../guide/profiler.md) you can use to optimize the performance of your TensorFlow models.\n", "\n", - "Note: `tf.distribute.Strategy` is actively under development and we will be adding more examples and tutorials in the near future. Please give it a try. We welcome your feedback via [issues on GitHub](https://github.com/tensorflow/tensorflow/issues/new)." + "Note: `tf.distribute.Strategy` is actively under development and TensorFlow will be adding more examples and tutorials in the near future. Please give it a try. Your feedback is welcome—feel free to submit it via [issues on GitHub](https://github.com/tensorflow/tensorflow/issues/new)." ] } ], @@ -736,8 +663,6 @@ "colab": { "collapsed_sections": [], "name": "keras.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/distribute/multi_worker_with_ctl.ipynb b/site/en/tutorials/distribute/multi_worker_with_ctl.ipynb new file mode 100644 index 00000000000..0361eea9328 --- /dev/null +++ b/site/en/tutorials/distribute/multi_worker_with_ctl.ipynb @@ -0,0 +1,1100 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Tce3stUlHN0L" + }, + "source": [ + "##### Copyright 2019 The TensorFlow Authors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "tuOe1ymfHZPu" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MfBg1C5NB3X0" + }, + "source": [ + "# Custom training loop with Keras and MultiWorkerMirroredStrategy\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xHxb-dlhMIzW" + }, + "source": [ + "## Overview\n", + "\n", + "This tutorial demonstrates how to perform multi-worker distributed training with a Keras model and with [custom training loops](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) using the `tf.distribute.Strategy` API. The training loop is distributed via `tf.distribute.MultiWorkerMirroredStrategy`, such that a `tf.keras` model—designed to run on [single-worker](custom_training.ipynb)—can seamlessly work on multiple workers with minimal code changes. Custom training loops provide flexibility and a greater control on training, while also making it easier to debug the model. Learn more about [writing a basic training loop](../../guide/basic_training_loops.ipynb), [writing a training loop from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) and [custom training](../customization/custom_training_walkthrough.ipynb).\n", + "\n", + "If you are looking for how to use `MultiWorkerMirroredStrategy` with `tf.keras.Model.fit`, refer to this [tutorial](multi_worker_with_keras.ipynb) instead.\n", + "\n", + "[Distributed Training in TensorFlow](../../guide/distributed_training.ipynb) guide is available for an overview of the distribution strategies TensorFlow supports for those interested in a deeper understanding of `tf.distribute.Strategy` APIs." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MUXex9ctTuDB" + }, + "source": [ + "## Setup\n", + "\n", + "First, some necessary imports." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bnYxvfLD-LW-" + }, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "import sys" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zz0EY91y3mxy" + }, + "source": [ + "Before importing TensorFlow, make a few changes to the environment:\n", + "* Disable all GPUs. This prevents errors caused by all workers trying to use the same GPU. In a real-world application, each worker would be on a different machine." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "685pbYEY3jGC" + }, + "outputs": [], + "source": [ + "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7X1MS6385BWi" + }, + "source": [ + "* Reset the `'TF_CONFIG'` environment variable (you'll see more about this later)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WEJLYa2_7OZF" + }, + "outputs": [], + "source": [ + "os.environ.pop('TF_CONFIG', None)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Rd4L9Ii77SS8" + }, + "source": [ + "* Make sure that the current directory is on Python's path. This allows the notebook to import the files written by `%%writefile` later.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hPBuZUNSZmrQ" + }, + "outputs": [], + "source": [ + "if '.' not in sys.path:\n", + " sys.path.insert(0, '.')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pDhHuMjb7bfU" + }, + "source": [ + "Now import TensorFlow." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vHNvttzV43sA" + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0S2jpf6Sx50i" + }, + "source": [ + "### Dataset and model definition" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fLW6D2TzvC-4" + }, + "source": [ + "Next, create an `mnist.py` file with a simple model and dataset setup. This Python file will be used by the worker-processes in this tutorial:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dma_wUAxZqo2" + }, + "outputs": [], + "source": [ + "%%writefile mnist.py\n", + "\n", + "import os\n", + "import tensorflow as tf\n", + "import numpy as np\n", + "\n", + "def mnist_dataset(batch_size):\n", + " (x_train, y_train), _ = tf.keras.datasets.mnist.load_data()\n", + " # The `x` arrays are in uint8 and have values in the range [0, 255].\n", + " # You need to convert them to float32 with values in the range [0, 1]\n", + " x_train = x_train / np.float32(255)\n", + " y_train = y_train.astype(np.int64)\n", + " train_dataset = tf.data.Dataset.from_tensor_slices(\n", + " (x_train, y_train)).shuffle(60000)\n", + " return train_dataset\n", + "\n", + "def dataset_fn(global_batch_size, input_context):\n", + " batch_size = input_context.get_per_replica_batch_size(global_batch_size)\n", + " dataset = mnist_dataset(batch_size)\n", + " dataset = dataset.shard(input_context.num_input_pipelines,\n", + " input_context.input_pipeline_id)\n", + " dataset = dataset.batch(batch_size)\n", + " return dataset\n", + "\n", + "def build_cnn_model():\n", + " regularizer = tf.keras.regularizers.L2(1e-5)\n", + " return tf.keras.Sequential([\n", + " tf.keras.Input(shape=(28, 28)),\n", + " tf.keras.layers.Reshape(target_shape=(28, 28, 1)),\n", + " tf.keras.layers.Conv2D(32, 3,\n", + " activation='relu',\n", + " kernel_regularizer=regularizer),\n", + " tf.keras.layers.Flatten(),\n", + " tf.keras.layers.Dense(128,\n", + " activation='relu',\n", + " kernel_regularizer=regularizer),\n", + " tf.keras.layers.Dense(10, kernel_regularizer=regularizer)\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JmgZwwymxqt5" + }, + "source": [ + "## Multi-worker configuration\n", + "\n", + "Now let's enter the world of multi-worker training. In TensorFlow, the `'TF_CONFIG'` environment variable is required for training on multiple machines. Each machine may have a different role. The `'TF_CONFIG'` variable used below is a JSON string that specifies the cluster configuration on each worker that is part of the cluster. This is the default method for specifying a cluster, using `cluster_resolver.TFConfigClusterResolver`, but there are other options available in the `distribute.cluster_resolver` module. Learn more about setting up the `'TF_CONFIG'` variable in the [Distributed training guide](../../guide/distributed_training.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SS8WhvRhe_Ya" + }, + "source": [ + "### Describe your cluster\n", + "Here is an example configuration:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XK1eTYvSZiX7" + }, + "outputs": [], + "source": [ + "tf_config = {\n", + " 'cluster': {\n", + " 'worker': ['localhost:12345', 'localhost:23456']\n", + " },\n", + " 'task': {'type': 'worker', 'index': 0}\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JjgwJbPKZkJL" + }, + "source": [ + "Note that `tf_config` is just a local variable in Python. To use it for training configuration, serialize it as a JSON and place it in a `'TF_CONFIG'` environment variable. Here is the same `'TF_CONFIG'` serialized as a JSON string:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yY-T0YDQZjbu" + }, + "outputs": [], + "source": [ + "json.dumps(tf_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AUBmYRZqxthH" + }, + "source": [ + "There are two components of `'TF_CONFIG'`: `'cluster'` and `'task'`.\n", + "\n", + "* `'cluster'` is the same for all workers and provides information about the training cluster, which is a dict consisting of different types of jobs such as `'worker'`. In multi-worker training with `MultiWorkerMirroredStrategy`, there is usually one `'worker'` that takes on a little more responsibility like saving checkpoints and writing summary files for TensorBoard in addition to what a regular `'worker'` does. Such a worker is referred to as the `'chief'` worker, and it is customary that the `'worker'` with `'index'` 0 is appointed as the chief `worker`.\n", + "\n", + "* `'task'` provides information of the current task and is different on each worker. It specifies the `'type'` and `'index'` of that worker." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8YFpxrcsZ2xG" + }, + "source": [ + "In this example, you set the task `'type'` to `'worker'` and the task `'index'` to `0`. This machine is the first worker and will be appointed as the chief worker and do more work than the others. Note that other machines will need to have the `'TF_CONFIG'` environment variable set as well, and it should have the same `'cluster'` dict, but different task `'type'` or task `'index'` depending on what the roles of those machines are.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aogb74kHxynz" + }, + "source": [ + "For illustration purposes, this tutorial shows how one may set a `'TF_CONFIG'` with two workers on `'localhost'`. In practice, users would create multiple workers on external IP addresses/ports, and set `'TF_CONFIG'` on each worker appropriately.\n", + "\n", + "This example uses two workers. The first worker's `'TF_CONFIG'` is shown above. For the second worker, set `tf_config['task']['index']=1`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cIlkfWmjz1PG" + }, + "source": [ + "### Environment variables and subprocesses in notebooks" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FcjAbuGY1ACJ" + }, + "source": [ + "Subprocesses inherit environment variables from their parent. So if you set an environment variable in this Jupyter Notebook process:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PH2gHn2_0_U8" + }, + "outputs": [], + "source": [ + "os.environ['GREETINGS'] = 'Hello TensorFlow!'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gQkIX-cg18md" + }, + "source": [ + "you can then access the environment variable from a subprocess:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pquKO6IA18G5" + }, + "outputs": [], + "source": [ + "%%bash\n", + "echo ${GREETINGS}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "af6BCA-Y2fpz" + }, + "source": [ + "In the next section, you'll use this to pass the `'TF_CONFIG'` to the worker subprocesses. You would never really launch your jobs this way, but it's sufficient for the purposes of this tutorial: To demonstrate a minimal multi-worker example." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UhNtHfuxCGVy" + }, + "source": [ + "## MultiWorkerMirroredStrategy\n", + "\n", + "Before training the model, first create an instance of `tf.distribute.MultiWorkerMirroredStrategy`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1uFSHCJXMrQ-" + }, + "outputs": [], + "source": [ + "strategy = tf.distribute.MultiWorkerMirroredStrategy()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N0iv7SyyAohc" + }, + "source": [ + "Note: `'TF_CONFIG'` is parsed and TensorFlow's GRPC servers are started at the time you call `tf.distribute.MultiWorkerMirroredStrategy.` Therefore, you must set the `'TF_CONFIG'` environment variable before you instantiate a `tf.distribute.Strategy`. To save time in this illustrative example, this is not demonstrated in this tutorial, so that servers do not need to start. You can find a full example in the last section of this tutorial." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TS4S-faBHHam" + }, + "source": [ + "Use `tf.distribute.Strategy.scope` to specify that a strategy should be used when building your model. This allows the strategy to control things like variable placement—it will create copies of all variables in the model's layers on each device across all workers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nXV49tG1_opc" + }, + "outputs": [], + "source": [ + "import mnist\n", + "with strategy.scope():\n", + " # Model building needs to be within `strategy.scope()`.\n", + " multi_worker_model = mnist.build_cnn_model()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DSYkM-on6r3Y" + }, + "source": [ + "## Auto-shard your data across workers\n", + "\n", + "In multi-worker training, _dataset sharding_ is needed to ensure convergence and reproducibility. Sharding means handing each worker a subset of the entire dataset—it helps create the experience similar to training on a single worker. In the example below, you're relying on the default autosharding policy of `tf.distribute`. You can also customize it by setting the `tf.data.experimental.AutoShardPolicy` of the `tf.data.experimental.DistributeOptions`. To learn more, refer to the _Sharding_ section of the [Distributed input tutorial](input.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "65-p36pt6rUF" + }, + "outputs": [], + "source": [ + "per_worker_batch_size = 64\n", + "num_workers = len(tf_config['cluster']['worker'])\n", + "global_batch_size = per_worker_batch_size * num_workers\n", + "\n", + "with strategy.scope():\n", + " multi_worker_dataset = strategy.distribute_datasets_from_function(\n", + " lambda input_context: mnist.dataset_fn(global_batch_size, input_context))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rkNzSR3g60iP" + }, + "source": [ + "## Define a custom training loop and train the model\n", + "Specify an optimizer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NoMr4_zTeKSn" + }, + "outputs": [], + "source": [ + "with strategy.scope():\n", + " # The creation of optimizer and train_accuracy needs to be in\n", + " # `strategy.scope()` as well, since they create variables.\n", + " optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n", + " train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", + " name='train_accuracy')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RmrDcAii4B5O" + }, + "source": [ + "Define a training step with `tf.function`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "znXWN5S3eUDB" + }, + "outputs": [], + "source": [ + "@tf.function\n", + "def train_step(iterator):\n", + " \"\"\"Training step function.\"\"\"\n", + "\n", + " def step_fn(inputs):\n", + " \"\"\"Per-Replica step function.\"\"\"\n", + " x, y = inputs\n", + " with tf.GradientTape() as tape:\n", + " predictions = multi_worker_model(x, training=True)\n", + " per_example_loss = tf.keras.losses.SparseCategoricalCrossentropy(\n", + " from_logits=True,\n", + " reduction=tf.keras.losses.Reduction.NONE)(y, predictions)\n", + " loss = tf.nn.compute_average_loss(per_example_loss)\n", + " model_losses = multi_worker_model.losses\n", + " if model_losses:\n", + " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n", + "\n", + " grads = tape.gradient(loss, multi_worker_model.trainable_variables)\n", + " optimizer.apply_gradients(\n", + " zip(grads, multi_worker_model.trainable_variables))\n", + " train_accuracy.update_state(y, predictions)\n", + " return loss\n", + "\n", + " per_replica_losses = strategy.run(step_fn, args=(next(iterator),))\n", + " return strategy.reduce(\n", + " tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eFXHsUVBy0Rx" + }, + "source": [ + "### Checkpoint saving and restoring\n", + "\n", + "As you write a custom training loop, you need to handle [checkpoint saving](../../guide/checkpoint.ipynb) manually instead of relying on a Keras callback. Note that for `MultiWorkerMirroredStrategy`, saving a checkpoint or a complete model requires the participation of all workers, because attempting to save only on the chief worker could lead to a deadlock. Workers also need to write to different paths to avoid overwriting each other. Here's an example of how to configure the directories:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LcFO6x1KyjhI" + }, + "outputs": [], + "source": [ + "from multiprocessing import util\n", + "checkpoint_dir = os.path.join(util.get_temp_dir(), 'ckpt')\n", + "\n", + "def _is_chief(task_type, task_id, cluster_spec):\n", + " return (task_type is None\n", + " or task_type == 'chief'\n", + " or (task_type == 'worker'\n", + " and task_id == 0\n", + " and \"chief\" not in cluster_spec.as_dict()))\n", + "\n", + "def _get_temp_dir(dirpath, task_id):\n", + " base_dirpath = 'workertemp_' + str(task_id)\n", + " temp_dir = os.path.join(dirpath, base_dirpath)\n", + " tf.io.gfile.makedirs(temp_dir)\n", + " return temp_dir\n", + "\n", + "def write_filepath(filepath, task_type, task_id, cluster_spec):\n", + " dirpath = os.path.dirname(filepath)\n", + " base = os.path.basename(filepath)\n", + " if not _is_chief(task_type, task_id, cluster_spec):\n", + " dirpath = _get_temp_dir(dirpath, task_id)\n", + " return os.path.join(dirpath, base)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nrcdPHtG4ObO" + }, + "source": [ + "Create one `tf.train.Checkpoint` that tracks the model, which is managed by a `tf.train.CheckpointManager`, so that only the latest checkpoints are preserved:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4rURT2pI4aqV" + }, + "outputs": [], + "source": [ + "epoch = tf.Variable(\n", + " initial_value=tf.constant(0, dtype=tf.dtypes.int64), name='epoch')\n", + "step_in_epoch = tf.Variable(\n", + " initial_value=tf.constant(0, dtype=tf.dtypes.int64),\n", + " name='step_in_epoch')\n", + "task_type, task_id = (strategy.cluster_resolver.task_type,\n", + " strategy.cluster_resolver.task_id)\n", + "# Normally, you don't need to manually instantiate a `ClusterSpec`, but in this\n", + "# illustrative example you did not set `'TF_CONFIG'` before initializing the\n", + "# strategy. Check out the next section for \"real-world\" usage.\n", + "cluster_spec = tf.train.ClusterSpec(tf_config['cluster'])\n", + "\n", + "checkpoint = tf.train.Checkpoint(\n", + " model=multi_worker_model, epoch=epoch, step_in_epoch=step_in_epoch)\n", + "\n", + "write_checkpoint_dir = write_filepath(checkpoint_dir, task_type, task_id,\n", + " cluster_spec)\n", + "checkpoint_manager = tf.train.CheckpointManager(\n", + " checkpoint, directory=write_checkpoint_dir, max_to_keep=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RO7cbN40XD5v" + }, + "source": [ + "Now, when you need to restore a checkpoint, you can find the latest checkpoint saved using the convenient `tf.train.latest_checkpoint` function (or by calling `tf.train.CheckpointManager.restore_or_initialize`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gniynaQj6HMV" + }, + "outputs": [], + "source": [ + "latest_checkpoint = tf.train.latest_checkpoint(checkpoint_dir)\n", + "if latest_checkpoint:\n", + " checkpoint.restore(latest_checkpoint)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1j9JuI-h6ObW" + }, + "source": [ + "After restoring the checkpoint, you can continue with training your custom training loop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kZzXZCh45FY6" + }, + "outputs": [], + "source": [ + "num_epochs = 3\n", + "num_steps_per_epoch = 70\n", + "\n", + "while epoch.numpy() < num_epochs:\n", + " iterator = iter(multi_worker_dataset)\n", + " total_loss = 0.0\n", + " num_batches = 0\n", + "\n", + " while step_in_epoch.numpy() < num_steps_per_epoch:\n", + " total_loss += train_step(iterator)\n", + " num_batches += 1\n", + " step_in_epoch.assign_add(1)\n", + "\n", + " train_loss = total_loss / num_batches\n", + " print('Epoch: %d, accuracy: %f, train_loss: %f.'\n", + " %(epoch.numpy(), train_accuracy.result(), train_loss))\n", + "\n", + " train_accuracy.reset_states()\n", + "\n", + " # Once the `CheckpointManager` is set up, you're now ready to save, and remove\n", + " # the checkpoints non-chief workers saved.\n", + " checkpoint_manager.save()\n", + " if not _is_chief(task_type, task_id, cluster_spec):\n", + " tf.io.gfile.rmtree(write_checkpoint_dir)\n", + "\n", + " epoch.assign_add(1)\n", + " step_in_epoch.assign(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0W1Osks466DE" + }, + "source": [ + "## Complete code at a glance" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jfYpmIxO6Jck" + }, + "source": [ + "To sum up all the procedures discussed so far:\n", + "\n", + "1. You create worker processes.\n", + "2. Pass `'TF_CONFIG'`s to the worker processes.\n", + "3. Let each work process run the script below that contains the training code." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MIDCESkVzN6M" + }, + "outputs": [], + "source": [ + "%%writefile main.py\n", + "#@title File: `main.py`\n", + "import os\n", + "import json\n", + "import tensorflow as tf\n", + "import mnist\n", + "from multiprocessing import util\n", + "\n", + "per_worker_batch_size = 64\n", + "tf_config = json.loads(os.environ['TF_CONFIG'])\n", + "num_workers = len(tf_config['cluster']['worker'])\n", + "global_batch_size = per_worker_batch_size * num_workers\n", + "\n", + "num_epochs = 3\n", + "num_steps_per_epoch=70\n", + "\n", + "# Checkpoint saving and restoring\n", + "def _is_chief(task_type, task_id, cluster_spec):\n", + " return (task_type is None\n", + " or task_type == 'chief'\n", + " or (task_type == 'worker'\n", + " and task_id == 0\n", + " and 'chief' not in cluster_spec.as_dict()))\n", + "\n", + "def _get_temp_dir(dirpath, task_id):\n", + " base_dirpath = 'workertemp_' + str(task_id)\n", + " temp_dir = os.path.join(dirpath, base_dirpath)\n", + " tf.io.gfile.makedirs(temp_dir)\n", + " return temp_dir\n", + "\n", + "def write_filepath(filepath, task_type, task_id, cluster_spec):\n", + " dirpath = os.path.dirname(filepath)\n", + " base = os.path.basename(filepath)\n", + " if not _is_chief(task_type, task_id, cluster_spec):\n", + " dirpath = _get_temp_dir(dirpath, task_id)\n", + " return os.path.join(dirpath, base)\n", + "\n", + "checkpoint_dir = os.path.join(util.get_temp_dir(), 'ckpt')\n", + "\n", + "# Define Strategy\n", + "strategy = tf.distribute.MultiWorkerMirroredStrategy()\n", + "\n", + "with strategy.scope():\n", + " # Model building/compiling need to be within `tf.distribute.Strategy.scope`.\n", + " multi_worker_model = mnist.build_cnn_model()\n", + "\n", + " multi_worker_dataset = strategy.distribute_datasets_from_function(\n", + " lambda input_context: mnist.dataset_fn(global_batch_size, input_context))\n", + " optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n", + " train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", + " name='train_accuracy')\n", + "\n", + "@tf.function\n", + "def train_step(iterator):\n", + " \"\"\"Training step function.\"\"\"\n", + "\n", + " def step_fn(inputs):\n", + " \"\"\"Per-Replica step function.\"\"\"\n", + " x, y = inputs\n", + " with tf.GradientTape() as tape:\n", + " predictions = multi_worker_model(x, training=True)\n", + " per_example_loss = tf.keras.losses.SparseCategoricalCrossentropy(\n", + " from_logits=True,\n", + " reduction=tf.keras.losses.Reduction.NONE)(y, predictions)\n", + " loss = tf.nn.compute_average_loss(per_example_loss)\n", + " model_losses = multi_worker_model.losses\n", + " if model_losses:\n", + " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n", + "\n", + " grads = tape.gradient(loss, multi_worker_model.trainable_variables)\n", + " optimizer.apply_gradients(\n", + " zip(grads, multi_worker_model.trainable_variables))\n", + " train_accuracy.update_state(y, predictions)\n", + "\n", + " return loss\n", + "\n", + " per_replica_losses = strategy.run(step_fn, args=(next(iterator),))\n", + " return strategy.reduce(\n", + " tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n", + "\n", + "epoch = tf.Variable(\n", + " initial_value=tf.constant(0, dtype=tf.dtypes.int64), name='epoch')\n", + "step_in_epoch = tf.Variable(\n", + " initial_value=tf.constant(0, dtype=tf.dtypes.int64),\n", + " name='step_in_epoch')\n", + "\n", + "task_type, task_id, cluster_spec = (strategy.cluster_resolver.task_type,\n", + " strategy.cluster_resolver.task_id,\n", + " strategy.cluster_resolver.cluster_spec())\n", + "\n", + "checkpoint = tf.train.Checkpoint(\n", + " model=multi_worker_model, epoch=epoch, step_in_epoch=step_in_epoch)\n", + "\n", + "write_checkpoint_dir = write_filepath(checkpoint_dir, task_type, task_id,\n", + " cluster_spec)\n", + "checkpoint_manager = tf.train.CheckpointManager(\n", + " checkpoint, directory=write_checkpoint_dir, max_to_keep=1)\n", + "\n", + "# Restoring the checkpoint\n", + "latest_checkpoint = tf.train.latest_checkpoint(checkpoint_dir)\n", + "if latest_checkpoint:\n", + " checkpoint.restore(latest_checkpoint)\n", + "\n", + "# Resume our CTL training\n", + "while epoch.numpy() < num_epochs:\n", + " iterator = iter(multi_worker_dataset)\n", + " total_loss = 0.0\n", + " num_batches = 0\n", + "\n", + " while step_in_epoch.numpy() < num_steps_per_epoch:\n", + " total_loss += train_step(iterator)\n", + " num_batches += 1\n", + " step_in_epoch.assign_add(1)\n", + "\n", + " train_loss = total_loss / num_batches\n", + " print('Epoch: %d, accuracy: %f, train_loss: %f.'\n", + " %(epoch.numpy(), train_accuracy.result(), train_loss))\n", + "\n", + " train_accuracy.reset_states()\n", + "\n", + " checkpoint_manager.save()\n", + " if not _is_chief(task_type, task_id, cluster_spec):\n", + " tf.io.gfile.rmtree(write_checkpoint_dir)\n", + "\n", + " epoch.assign_add(1)\n", + " step_in_epoch.assign(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ItVOvPN1qnZ6" + }, + "source": [ + "The current directory now contains both Python files:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bi6x05Sr60O9" + }, + "outputs": [], + "source": [ + "%%bash\n", + "ls *.py" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qmEEStPS6vR_" + }, + "source": [ + "So JSON-serialize the `'TF_CONFIG'` and add it to the environment variables:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9uu3g7vV7Bbt" + }, + "outputs": [], + "source": [ + "os.environ['TF_CONFIG'] = json.dumps(tf_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MsY3dQLK7jdf" + }, + "source": [ + "Now, you can launch a worker process that will run the `main.py` and use the `'TF_CONFIG'`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "txMXaq8d8N_S" + }, + "outputs": [], + "source": [ + "# first kill any previous runs\n", + "%killbgscripts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qnSma_Ck7r-r" + }, + "outputs": [], + "source": [ + "%%bash --bg\n", + "python main.py &> job_0.log" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZChyazqS7v0P" + }, + "source": [ + "There are a few things to note about the above command:\n", + "\n", + "1. It uses the `%%bash` which is a [notebook \"magic\"](https://ipython.readthedocs.io/en/stable/interactive/magics.html) to run some bash commands.\n", + "2. It uses the `--bg` flag to run the `bash` process in the background, because this worker will not terminate. It waits for all the workers before it starts.\n", + "\n", + "The backgrounded worker process won't print the output to this notebook. The `&>` redirects its output to a file, so that you can inspect what happened.\n", + "\n", + "Wait a few seconds for the process to start up:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Hm2yrULE9281" + }, + "outputs": [], + "source": [ + "import time\n", + "time.sleep(20)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZFPoNxg_9_Mx" + }, + "source": [ + "Now, check the output to the worker's log file so far:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vZEOuVgQ9-hn" + }, + "outputs": [], + "source": [ + "%%bash\n", + "cat job_0.log" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RqZhVF7L_KOy" + }, + "source": [ + "The last line of the log file should say: `Started server with target: grpc://localhost:12345`. The first worker is now ready, and is waiting for all the other worker(s) to be ready to proceed." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Pi8vPNNA_l4a" + }, + "source": [ + "Update the `tf_config` for the second worker's process to pick up:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lAiYkkPu_Jqd" + }, + "outputs": [], + "source": [ + "tf_config['task']['index'] = 1\n", + "os.environ['TF_CONFIG'] = json.dumps(tf_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0AshGVO0_x0w" + }, + "source": [ + "Now launch the second worker. This will start the training since all the workers are active (so there's no need to background this process):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_ESVtyQ9_xjx" + }, + "outputs": [], + "source": [ + "%%bash\n", + "python main.py > /dev/null 2>&1" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hX4FA2O2AuAn" + }, + "source": [ + "If you recheck the logs written by the first worker, notice that it participated in training that model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rc6hw3yTBKXX" + }, + "outputs": [], + "source": [ + "%%bash\n", + "cat job_0.log" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sG5_1UgrgniF" + }, + "outputs": [], + "source": [ + "# Delete the `'TF_CONFIG'`, and kill any background tasks so they don't affect the next section.\n", + "os.environ.pop('TF_CONFIG', None)\n", + "%killbgscripts" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bhxMXa0AaZkK" + }, + "source": [ + "## Multi-worker training in depth\n", + "\n", + "This tutorial has demonstrated a custom training loop workflow of the multi-worker setup. Detailed descriptions of other topics is available in the [Multi-worker training with Keras (`tf.keras.Model.fit`)](multi_worker_with_keras.ipynb) tutorial applicable to custom training loops." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ega2hdOQEmy_" + }, + "source": [ + "## Learn more\n", + "\n", + "1. The [Distributed training in TensorFlow](../../guide/distributed_training.ipynb) guide provides an overview of the available distribution strategies.\n", + "2. [Official models](https://github.com/tensorflow/models/tree/master/official), many of which can be configured to run multiple distribution strategies.\n", + "3. The [Performance section](../../guide/function.ipynb) in the `tf.function` guide provides information about other strategies and [tools](../../guide/profiler.md) you can use to optimize the performance of your TensorFlow models.\n" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "multi_worker_with_ctl.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb b/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb index dbd18580467..fcee0618854 100644 --- a/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb +++ b/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb @@ -3,20 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Tce3stUlHN0L" }, "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" + "##### Copyright 2019 The TensorFlow Authors.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", + "cellView": "form", "id": "tuOe1ymfHZPu" }, "outputs": [], @@ -37,50 +34,55 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MfBg1C5NB3X0" }, "source": [ "# Multi-worker training with Estimator\n", "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/distribute/multi_worker_with_estimator\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/distribute/multi_worker_with_estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-_ZO8y69hs-N" + }, + "source": [ + "> Warning: Estimators are not recommended for new code. Estimators run `v1.Session`-style code which is more difficult to write correctly, and can behave unexpectedly, especially when combined with TF 2 code. Estimators do fall under [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities. See the [migration guide](https://tensorflow.org/guide/migrate) for details." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xHxb-dlhMIzW" }, "source": [ "## Overview\n", "\n", - "Note: While you can use Estimators with `tf.distribute` API, we would instead recommend you use Keras with `tf.distribute` (please see [Multi-worker Training with Keras](../../guide/multi_worker_with_keras.ipynb)). Estimator training with `tf.distribute.Strategy` has limited support at this time.\n", + "Note: While you can use Estimators with `tf.distribute` API, it's recommended to use Keras with `tf.distribute`, see [multi-worker training with Keras](multi_worker_with_keras.ipynb). Estimator training with `tf.distribute.Strategy` has limited support.\n", "\n", "\n", "This tutorial demonstrates how `tf.distribute.Strategy` can be used for distributed multi-worker training with `tf.estimator`. If you write your code using `tf.estimator`, and you're interested in scaling beyond a single machine with high performance, this tutorial is for you.\n", "\n", - "Before getting started, please read the [`tf.distribute.Strategy` guide](../../guide/distributed_training.ipynb). The [multi-GPU training tutorial](./keras.ipynb) is also relevant, because this tutorial uses the same model.\n", - "\n" + "Before getting started, please read the [distribution strategy](../../guide/distributed_training.ipynb) guide. The [multi-GPU training tutorial](./keras.ipynb) is also relevant, because this tutorial uses the same model.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MUXex9ctTuDB" }, "source": [ @@ -91,43 +93,41 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IqR2PQG4ZaZ0" + "id": "bnYxvfLD-LW-" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals " + "import tensorflow_datasets as tfds\n", + "import tensorflow as tf\n", + "\n", + "import os, json" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-xicK9byC7hi" + }, + "source": [ + "Note: Starting from TF2.4 multi worker mirrored strategy fails with estimators if run with eager enabled (the default). The error in TF2.4 is `TypeError: cannot pickle '_thread.lock' object`, See [issue #46556](https://github.com/tensorflow/tensorflow/issues/46556) for details. The workaround is to disable eager execution." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bnYxvfLD-LW-" + "id": "5dJ6UYrGDsVs" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "tfds.disable_progress_bar()\n", - "\n", - "import os, json" + "tf.compat.v1.disable_eager_execution()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hPBuZUNSZmrQ" }, "source": [ @@ -138,10 +138,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dma_wUAxZqo2" }, "outputs": [], @@ -170,7 +168,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4BlcVXMhB59T" }, "source": [ @@ -180,7 +177,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8YFpxrcsZ2xG" }, "source": [ @@ -188,11 +184,11 @@ "\n", "One of the key differences in this tutorial (compared to the [multi-GPU training tutorial](./keras.ipynb)) is the multi-worker setup. The `TF_CONFIG` environment variable is the standard way to specify the cluster configuration to each worker that is part of the cluster.\n", "\n", - "There are two components of `TF_CONFIG`: `cluster` and `task`. `cluster` provides information about the entire cluster, namely the workers and parameter servers in the cluster. `task` provides information about the current task. In this example, the task `type` is `worker` and the task `index` is `0`.\n", + "There are two components of `TF_CONFIG`: `cluster` and `task`. `cluster` provides information about the entire cluster, namely the workers and parameter servers in the cluster. `task` provides information about the current task. The first component `cluster` is the same for all workers and parameter servers in the cluster, and the second component `task` is different on each worker and parameter server and specifies its own `type` and `index`. In this example, the task `type` is `worker` and the task `index` is `0`.\n", "\n", - "For illustration purposes, this tutorial shows how to set a `TF_CONFIG` with 2 workers on `localhost`. In practice, you would create multiple workers on an external IP address and port, and set `TF_CONFIG` on each worker appropriately, i.e. modify the task `index`.\n", + "For illustration purposes, this tutorial shows how to set a `TF_CONFIG` with 2 workers on `localhost`. In practice, you would create multiple workers on an external IP address and port, and set `TF_CONFIG` on each worker appropriately, i.e., modify the task `index`.\n", "\n", - "Warning: *Do not execute the following code in Colab.* TensorFlow's runtime will attempt to create a gRPC server at the specified IP address and port, which will likely fail.\n", + "Warning: *Do not execute the following code in Colab.* TensorFlow's runtime will attempt to create a gRPC server at the specified IP address and port, which will likely fail. See the [keras version](multi_worker_with_keras.ipynb) of this tutorial for an example of how you can test run multiple workers on a single machine.\n", "\n", "```\n", "os.environ['TF_CONFIG'] = json.dumps({\n", @@ -201,14 +197,12 @@ " },\n", " 'task': {'type': 'worker', 'index': 0}\n", "})\n", - "```\n", - "\n" + "```\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qDreJzTffAP5" }, "source": [ @@ -219,10 +213,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WNvOn_OeiUYC" }, "outputs": [], @@ -260,7 +252,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "P94PrIW_kSCE" }, "source": [ @@ -270,7 +261,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UhNtHfuxCGVy" }, "source": [ @@ -281,10 +271,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "1uFSHCJXMrQ-" }, "outputs": [], @@ -295,7 +283,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "H47DDcOgfzm7" }, "source": [ @@ -306,10 +293,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "BcsuBYrpgnlS" }, "outputs": [], @@ -328,7 +313,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XVk4ftYx6JAO" }, "source": [ @@ -343,21 +327,21 @@ " * `NCCL` uses [Nvidia's NCCL](https://developer.nvidia.com/nccl) to implement collectives. \n", " * `AUTO` defers the choice to the runtime.\n", " \n", - " The best choice of collective implementation depends upon the number and kind of GPUs, and the network interconnect in the cluster. To override the automatic choice, specify a valid value to the `communication` parameter of `MultiWorkerMirroredStrategy`'s constructor, e.g. `communication=tf.distribute.experimental.CollectiveCommunication.NCCL`.\n" + " The best choice of collective implementation depends upon the number and kind of GPUs, and the network interconnect in the cluster. To override the automatic choice, specify a valid value to the `communication` parameter of `MultiWorkerMirroredStrategy`'s constructor, e.g. `communication=tf.distribute.experimental.CollectiveCommunication.NCCL`.\n", + "\n", + "Visit the [Performance section](../../guide/function.ipynb) in the guide to learn more about other strategies and [tools](../../guide/profiler.md) you can use to optimize the performance of your TensorFlow models.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AW0Hb2xM6EGX" }, "source": [ "## Other code examples\n", "\n", "1. [End to end example](https://github.com/tensorflow/ecosystem/tree/master/distribution_strategy) for multi worker training in tensorflow/ecosystem using Kubernetes templates. This example starts with a Keras model and converts it to an Estimator using the `tf.keras.estimator.model_to_estimator` API.\n", - "2. [Official models](https://github.com/tensorflow/models/tree/master/official), many of which can be configured to run multiple distribution strategies.\n", - "\n" + "2. [Official models](https://github.com/tensorflow/models/tree/master/official), many of which can be configured to run multiple distribution strategies.\n" ] } ], @@ -367,10 +351,7 @@ "Tce3stUlHN0L" ], "name": "multi_worker_with_estimator.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/distribute/multi_worker_with_keras.ipynb b/site/en/tutorials/distribute/multi_worker_with_keras.ipynb index fc59874af27..c972e8b7fb6 100644 --- a/site/en/tutorials/distribute/multi_worker_with_keras.ipynb +++ b/site/en/tutorials/distribute/multi_worker_with_keras.ipynb @@ -3,20 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Tce3stUlHN0L" }, "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" + "##### Copyright 2019 The TensorFlow Authors.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", + "cellView": "form", "id": "tuOe1ymfHZPu" }, "outputs": [], @@ -37,164 +34,247 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MfBg1C5NB3X0" }, "source": [ "# Multi-worker training with Keras\n", "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/distribute/multi_worker_with_keras\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/distribute/multi_worker_with_keras.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/distribute/multi_worker_with_keras.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/distribute/multi_worker_with_keras.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xHxb-dlhMIzW" }, "source": [ "## Overview\n", "\n", - "This tutorial demonstrates multi-worker distributed training with Keras model using `tf.distribute.Strategy` API. With the help of the strategies specifically designed for multi-worker training, a Keras model that was designed to run on single-worker can seamlessly work on multiple workers with minimal code change.\n", + "This tutorial demonstrates how to perform multi-worker distributed training with a Keras model and the `Model.fit` API using the `tf.distribute.MultiWorkerMirroredStrategy` API. With the help of this strategy, a Keras model that was designed to run on a single-worker can seamlessly work on multiple workers with minimal code changes.\n", "\n", - "[Distributed Training in TensorFlow](../../guide/distributed_training.ipynb) guide is available for an overview of the distribution strategies TensorFlow supports for those interested in a deeper understanding of `tf.distribute.Strategy` APIs.\n", - "\n" + "To learn how to use the `MultiWorkerMirroredStrategy` with Keras and a custom training loop, refer to [Custom training loop with Keras and MultiWorkerMirroredStrategy](multi_worker_with_ctl.ipynb).\n", + "\n", + "This tutorial contains a minimal multi-worker example with two workers for demonstration purposes." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JUdRerXg6yz3" + }, + "source": [ + "### Choose the right strategy" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YAiCV_oL63GM" + }, + "source": [ + "Before you dive in, make sure that `tf.distribute.MultiWorkerMirroredStrategy` is the right choice for your accelerator(s) and training. These are two common ways of distributing training with data parallelism:\n", + "\n", + "* _Synchronous training_, where the steps of training are synced across the workers and replicas, such as `tf.distribute.MirroredStrategy`, `tf.distribute.TPUStrategy`, and `tf.distribute.MultiWorkerMirroredStrategy`. All workers train over different slices of input data in sync, and aggregating gradients at each step.\n", + "* _Asynchronous training_, where the training steps are not strictly synced, such as `tf.distribute.experimental.ParameterServerStrategy`. All workers are independently training over the input data and updating variables asynchronously.\n", + "\n", + "If you are looking for multi-worker synchronous training without TPU, then `tf.distribute.MultiWorkerMirroredStrategy` is your choice. It creates copies of all variables in the model's layers on each device across all workers. It uses `CollectiveOps`, a TensorFlow op for collective communication, to aggregate gradients and keeps the variables in sync. For those interested, check out the `tf.distribute.experimental.CommunicationOptions` parameter for the collective implementation options.\n", + "\n", + "For an overview of `tf.distribute.Strategy` APIs, refer to [Distributed training in TensorFlow](../../guide/distributed_training.ipynb)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MUXex9ctTuDB" }, "source": [ "## Setup\n", "\n", - "First, setup TensorFlow and the necessary imports." + "Start with some necessary imports:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IqR2PQG4ZaZ0" + "id": "bnYxvfLD-LW-" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" + "import json\n", + "import os\n", + "import sys" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zz0EY91y3mxy" + }, + "source": [ + "Before importing TensorFlow, make a few changes to the environment:\n", + "\n", + "* In a real-world application, each worker would be on a different machine. For the purposes of this tutorial, all the workers will run on the **this** machine. Therefore, disable all GPUs to prevent errors caused by all workers trying to use the same GPU." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bnYxvfLD-LW-" + "id": "rpEIVI5upIzM" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " !pip install tf-nightly\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "tfds.disable_progress_bar()" + "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"-1\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7X1MS6385BWi" + }, + "source": [ + "* Reset the `TF_CONFIG` environment variable (you'll learn more about this later):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WEJLYa2_7OZF" + }, + "outputs": [], + "source": [ + "os.environ.pop('TF_CONFIG', None)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "Rd4L9Ii77SS8" + }, + "source": [ + "* Make sure that the current directory is on Python's path—this allows the notebook to import the files written by `%%writefile` later:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { "id": "hPBuZUNSZmrQ" }, + "outputs": [], "source": [ - "## Preparing dataset\n", - "\n", - "Now, let's prepare the MNIST dataset from [TensorFlow Datasets](https://www.tensorflow.org/datasets). The [MNIST dataset](http://yann.lecun.com/exdb/mnist/) comprises 60,000\n", - "training examples and 10,000 test examples of the handwritten digits 0–9,\n", - "formatted as 28x28-pixel monochrome images." + "if '.' not in sys.path:\n", + " sys.path.insert(0, '.')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9hLpDZhAz2q-" + }, + "source": [ + "Install `tf-nightly`, as the frequency of checkpoint saving at a particular step with the `save_freq` argument in `tf.keras.callbacks.BackupAndRestore` is introduced from TensorFlow 2.10:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dma_wUAxZqo2" + "id": "-XqozLfzz30N" }, "outputs": [], "source": [ - "BUFFER_SIZE = 10000\n", - "BATCH_SIZE = 64\n", - "\n", - "def make_datasets_unbatched():\n", - " # Scaling MNIST data from (0, 255] to (0., 1.]\n", - " def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - " return image, label\n", - "\n", - " datasets, info = tfds.load(name='mnist',\n", - " with_info=True,\n", - " as_supervised=True)\n", - "\n", - " return datasets['train'].map(scale).cache().shuffle(BUFFER_SIZE)\n", - "\n", - "train_datasets = make_datasets_unbatched().batch(BATCH_SIZE)" + "!pip install tf-nightly" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "o87kcvu8GR4-" + "id": "524e38dab658" }, "source": [ - "## Build the Keras model\n", - "Here we use `tf.keras.Sequential` API to build and compile a simple convolutional neural networks Keras model to train with our MNIST dataset.\n", - "\n", - "Note: For a more comprehensive walkthrough of building Keras model, please see [TensorFlow Keras Guide](https://www.tensorflow.org/guide/keras#sequential_model).\n" + "Finally, import TensorFlow:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vHNvttzV43sA" + }, + "outputs": [], + "source": [ + "import tensorflow as tf" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0S2jpf6Sx50i" + }, + "source": [ + "### Dataset and model definition" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fLW6D2TzvC-4" + }, + "source": [ + "Next, create an `mnist_setup.py` file with a simple model and dataset setup. This Python file will be used by the worker processes in this tutorial:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aVPHl0SfG2v1" + "id": "dma_wUAxZqo2" }, "outputs": [], "source": [ + "%%writefile mnist_setup.py\n", + "\n", + "import os\n", + "import tensorflow as tf\n", + "import numpy as np\n", + "\n", + "def mnist_dataset(batch_size):\n", + " (x_train, y_train), _ = tf.keras.datasets.mnist.load_data()\n", + " # The `x` arrays are in uint8 and have values in the [0, 255] range.\n", + " # You need to convert them to float32 with values in the [0, 1] range.\n", + " x_train = x_train / np.float32(255)\n", + " y_train = y_train.astype(np.int64)\n", + " train_dataset = tf.data.Dataset.from_tensor_slices(\n", + " (x_train, y_train)).shuffle(60000).repeat().batch(batch_size)\n", + " return train_dataset\n", + "\n", "def build_and_compile_cnn_model():\n", " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", + " tf.keras.layers.InputLayer(input_shape=(28, 28)),\n", + " tf.keras.layers.Reshape(target_shape=(28, 28, 1)),\n", + " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", + " tf.keras.layers.Dense(128, activation='relu'),\n", + " tf.keras.layers.Dense(10)\n", " ])\n", " model.compile(\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " optimizer=tf.keras.optimizers.SGD(learning_rate=0.001),\n", " metrics=['accuracy'])\n", " return model" @@ -203,287 +283,1009 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2UL3kisMO90X" }, "source": [ - "Let's first try training the model for a small number of epochs and observe the results in single worker to make sure everything works correctly. You should expect to see the loss dropping and accuracy approaching 1.0 as epoch advances." + "### Model training on a single worker\n", + "\n", + "Try training the model for a small number of epochs and observe the results of _a single worker_ to make sure everything works correctly. As training progresses, the loss should drop and the accuracy should increase." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "6Qe6iAf5O8iJ" }, "outputs": [], "source": [ - "single_worker_model = build_and_compile_cnn_model()\n", - "single_worker_model.fit(x=train_datasets, epochs=3, steps_per_epoch=5)" + "import mnist_setup\n", + "\n", + "batch_size = 64\n", + "single_worker_dataset = mnist_setup.mnist_dataset(batch_size)\n", + "single_worker_model = mnist_setup.build_and_compile_cnn_model()\n", + "single_worker_model.fit(single_worker_dataset, epochs=3, steps_per_epoch=70)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "8YFpxrcsZ2xG" + "id": "JmgZwwymxqt5" }, "source": [ - "## Multi-worker Configuration\n", + "## Multi-worker configuration\n", "\n", - "Now let's enter the world of multi-worker training. In TensorFlow, `TF_CONFIG` environment variable is required for training on multiple machines, each of which possibly has a different role. `TF_CONFIG` is used to specify the cluster configuration on each worker that is part of the cluster.\n", + "Now let's enter the world of multi-worker training.\n", "\n", - "There are two components of `TF_CONFIG`: `cluster` and `task`. `cluster` provides information about the training cluster, which is a dict consisting of different types of jobs such as `worker`. In multi-worker training, there is usually one `worker` that takes on a little more responsibility like saving checkpoint and writing summary file for TensorBoard in addition to what a regular `worker` does. Such worker is referred to as the 'chief' worker, and it is customary that the `worker` with `index` 0 is appointed as the chief `worker` (in fact this is how `tf.distribute.Strategy` is implemented). `task` on the other hand provides information of the current task. \n", + "### A cluster with jobs and tasks\n", "\n", - "In this example, we set the task `type` to `\"worker\"` and the task `index` to `0`. This means the machine that has such setting is the first worker, which will be appointed as the chief worker and do more work than other workers. Note that other machines will need to have `TF_CONFIG` environment variable set as well, and it should have the same `cluster` dict, but different task `type` or task `index` depending on what the roles of those machines are.\n", + "In TensorFlow, distributed training involves a `'cluster'`\n", + "with several jobs, and each of the jobs may have one or more `'task'`s.\n", "\n", - "For illustration purposes, this tutorial shows how one may set a `TF_CONFIG` with 2 workers on `localhost`. In practice, users would create multiple workers on external IP addresses/ports, and set `TF_CONFIG` on each worker appropriately.\n", + "You will need the `TF_CONFIG` configuration environment variable for training on multiple machines, each of which possibly has a different role. `TF_CONFIG` is a JSON string used to specify the cluster configuration for each worker that is part of the cluster.\n", "\n", - "Warning: Do not execute the following code in Colab. TensorFlow's runtime will attempt to create a gRPC server at the specified IP address and port, which will likely fail.\n", + "There are two components of a `TF_CONFIG` variable: `'cluster'` and `'task'`.\n", "\n", - "```\n", - "os.environ['TF_CONFIG'] = json.dumps({\n", + "* A `'cluster'` is the same for all workers and provides information about the training cluster, which is a dict consisting of different types of jobs, such as `'worker'` or `'chief'`.\n", + " - In multi-worker training with `tf.distribute.MultiWorkerMirroredStrategy`, there is usually one `'worker'` that takes on more responsibilities, such as saving a checkpoint and writing a summary file for TensorBoard, in addition to what a regular `'worker'` does. Such `'worker'` is referred to as the chief worker (with a job name `'chief'`).\n", + " - It is customary for the worker with `'index'` `0` to be the `'chief'`.\n", + "\n", + "* A `'task'` provides information on the current task and is different for each worker. It specifies the `'type'` and `'index'` of that worker.\n", + "\n", + "Below is an example configuration:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XK1eTYvSZiX7" + }, + "outputs": [], + "source": [ + "tf_config = {\n", " 'cluster': {\n", - " 'worker': [\"localhost:12345\", \"localhost:23456\"]\n", + " 'worker': ['localhost:12345', 'localhost:23456']\n", " },\n", " 'task': {'type': 'worker', 'index': 0}\n", - "})\n", - "```\n", - "\n" + "}" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "P94PrIW_kSCE" + "id": "JjgwJbPKZkJL" + }, + "source": [ + "Note that `tf_config` is just a local variable in Python. To use it for training configuration, serialize it as a JSON and place it in a `TF_CONFIG` environment variable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yY-T0YDQZjbu" }, + "outputs": [], "source": [ - "Note that while the learning rate is fixed in this example, in general it may be necessary to adjust the learning rate based on the global batch size." + "json.dumps(tf_config)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "UhNtHfuxCGVy" + "id": "8YFpxrcsZ2xG" }, "source": [ - "## Choose the right strategy\n", + "In the example configuration above, you set the task `'type'` to `'worker'` and the task `'index'` to `0`. Therefore, this machine is the _first_ worker. It will be appointed as the `'chief'` worker.\n", "\n", - "In TensorFlow, distributed training consists of synchronous training, where the steps of training are synced across the workers and replicas, and asynchronous training, where the training steps are not strictly synced.\n", - "\n", - "`MultiWorkerMirroredStrategy`, which is the recommended strategy for synchronous multi-worker training, will be demonstrated in this guide.\n", - "To train the model, use an instance of `tf.distribute.experimental.MultiWorkerMirroredStrategy`. `MultiWorkerMirroredStrategy` creates copies of all variables in the model's layers on each device across all workers. It uses `CollectiveOps`, a TensorFlow op for collective communication, to aggregate gradients and keep the variables in sync. The [`tf.distribute.Strategy` guide](../../guide/distributed_training.ipynb) has more details about this strategy." + "Note: Other machines will need to have the `TF_CONFIG` environment variable set as well, and it should have the same `'cluster'` dict, but different task `'type'`s or task `'index'`es, depending on the roles of those machines." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aogb74kHxynz" + }, + "source": [ + "In practice, you would create multiple workers on external IP addresses/ports and set a `TF_CONFIG` variable on each worker accordingly. For illustration purposes, this tutorial shows how you may set up a `TF_CONFIG` variable with two workers on a `localhost`:\n", + "- The first (`'chief'`) worker's `TF_CONFIG` as shown above.\n", + "- For the second worker, you will set `tf_config['task']['index']=1`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cIlkfWmjz1PG" + }, + "source": [ + "### Environment variables and subprocesses in notebooks" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FcjAbuGY1ACJ" + }, + "source": [ + "Subprocesses inherit environment variables from their parent. So if you set an environment variable in this Jupyter Notebook process:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PH2gHn2_0_U8" + }, + "outputs": [], + "source": [ + "os.environ['GREETINGS'] = 'Hello TensorFlow!'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gQkIX-cg18md" + }, + "source": [ + "... then you can access the environment variable from the subprocesses:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pquKO6IA18G5" + }, + "outputs": [], + "source": [ + "%%bash\n", + "echo ${GREETINGS}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "af6BCA-Y2fpz" + }, + "source": [ + "In the next section, you'll use this method to pass the `TF_CONFIG` to the worker subprocesses. You would never really launch your jobs this way in a real-world scenario—this tutorial is just showing how to do it with a minimal multi-worker example." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dnDJmaRA9qnf" + }, + "source": [ + "## Train the model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UhNtHfuxCGVy" + }, + "source": [ + "To train the model, firstly create an instance of the `tf.distribute.MultiWorkerMirroredStrategy`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "1uFSHCJXMrQ-" }, "outputs": [], "source": [ - "strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()" + "strategy = tf.distribute.MultiWorkerMirroredStrategy()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "N0iv7SyyAohc" }, "source": [ - "Note: `TF_CONFIG` is parsed and TensorFlow's GRPC servers are started at the time `MultiWorkerMirroredStrategy.__init__()` is called, so `TF_CONFIG` environment variable must be set before a `tf.distribute.Strategy` instance is created." + "Note: `TF_CONFIG` is parsed and TensorFlow's GRPC servers are started at the time `MultiWorkerMirroredStrategy` is called, so the `TF_CONFIG` environment variable must be set before a `tf.distribute.Strategy` instance is created. Since `TF_CONFIG` is not set yet, the above strategy is effectively single-worker training." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H47DDcOgfzm7" + }, + "source": [ + "With the integration of `tf.distribute.Strategy` API into `tf.keras`, the only change you will make to distribute the training to multiple-workers is enclosing the model building and `model.compile()` call inside `strategy.scope()`. The distribution strategy's scope dictates how and where the variables are created, and in the case of `MultiWorkerMirroredStrategy`, the variables created are `MirroredVariable`s, and they are replicated on each of the workers.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wo6b9wX65glL" + }, + "outputs": [], + "source": [ + "with strategy.scope():\n", + " # Model building/compiling need to be within `strategy.scope()`.\n", + " multi_worker_model = mnist_setup.build_and_compile_cnn_model()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "FMy2VM4Akzpr" + "id": "Mhq3fzyR5hTw" }, "source": [ - "`MultiWorkerMirroredStrategy` provides multiple implementations via the [`CollectiveCommunication`](https://github.com/tensorflow/tensorflow/blob/a385a286a930601211d78530734368ccb415bee4/tensorflow/python/distribute/cross_device_ops.py#L928) parameter. `RING` implements ring-based collectives using gRPC as the cross-host communication layer. `NCCL` uses [Nvidia's NCCL](https://developer.nvidia.com/nccl) to implement collectives. `AUTO` defers the choice to the runtime. The best choice of collective implementation depends upon the number and kind of GPUs, and the network interconnect in the cluster." + "Note: Currently there is a limitation in `MultiWorkerMirroredStrategy` where TensorFlow ops need to be created after the instance of strategy is created. If you encounter `RuntimeError: Collective ops must be configured at program startup`, try creating the instance of `MultiWorkerMirroredStrategy` at the beginning of the program and put the code that may create ops after the strategy is instantiated." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "H47DDcOgfzm7" + "id": "jfYpmIxO6Jck" }, "source": [ - "## Train the model with MultiWorkerMirroredStrategy\n", - "\n", - "With the integration of `tf.distribute.Strategy` API into `tf.keras`, the only change you will make to distribute the training to multi-worker is enclosing the model building and `model.compile()` call inside `strategy.scope()`. The distribution strategy's scope dictates how and where the variables are created, and in the case of `MultiWorkerMirroredStrategy`, the variables created are `MirroredVariable`s, and they are replicated on each of the workers.\n", - "\n", - "Note: Currently there is a limitation in `MultiWorkerMirroredStrategy` where TensorFlow ops need to be created after the instance of strategy is created. If you see `RuntimeError: Collective ops must be configured at program startup`, try creating the instance of `MultiWorkerMirroredStrategy` at the beginning of the program and put the code that may create ops after the strategy is instantiated.\n", - "\n", - "Note: In this Colab, the following code can run with expected result, but however this is effectively single-worker training since `TF_CONFIG` is not set. Once you set `TF_CONFIG` in your own example, you should expect speed-up with training on multiple machines.\n", + "To actually run with `MultiWorkerMirroredStrategy` you'll need to run worker processes and pass a `TF_CONFIG` to them.\n", "\n", - "Note: Since `MultiWorkerMirroredStrategy` does not support last partial batch handling, pass the `steps_per_epoch` argument to `model.fit()` when dataset is imbalanced on multiple workers." + "Like the `mnist_setup.py` file written earlier, here is the `main.py` that each of the workers will run:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "BcsuBYrpgnlS" }, "outputs": [], "source": [ - "NUM_WORKERS = 2\n", - "# Here the batch size scales up by number of workers since \n", - "# `tf.data.Dataset.batch` expects the global batch size. Previously we used 64, \n", - "# and now this becomes 128.\n", - "GLOBAL_BATCH_SIZE = 64 * NUM_WORKERS\n", + "%%writefile main.py\n", + "\n", + "import os\n", + "import json\n", + "\n", + "import tensorflow as tf\n", + "import mnist_setup\n", + "\n", + "per_worker_batch_size = 64\n", + "tf_config = json.loads(os.environ['TF_CONFIG'])\n", + "num_workers = len(tf_config['cluster']['worker'])\n", + "\n", + "strategy = tf.distribute.MultiWorkerMirroredStrategy()\n", + "\n", + "global_batch_size = per_worker_batch_size * num_workers\n", + "multi_worker_dataset = mnist_setup.mnist_dataset(global_batch_size)\n", + "\n", "with strategy.scope():\n", - " # Creation of dataset, and model building/compiling need to be within \n", - " # `strategy.scope()`.\n", - " train_datasets = make_datasets_unbatched().batch(GLOBAL_BATCH_SIZE)\n", - " multi_worker_model = build_and_compile_cnn_model()\n", + " # Model building/compiling need to be within `strategy.scope()`.\n", + " multi_worker_model = mnist_setup.build_and_compile_cnn_model()\n", + "\n", "\n", - "# Keras' `model.fit()` trains the model with specified number of epochs and\n", - "# number of steps per epoch. Note that the numbers here are for demonstration\n", - "# purposes only and may not sufficiently produce a model with good quality.\n", - "multi_worker_model.fit(x=train_datasets, epochs=3, steps_per_epoch=5)" + "multi_worker_model.fit(multi_worker_dataset, epochs=3, steps_per_epoch=70)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Rr14Vl9GR4zq" + "id": "Aom9xelvJQ_6" }, "source": [ - "### Dataset sharding and batch size\n", - "In multi-worker training, sharding data into multiple parts is needed to ensure convergence and performance. However, note that in above code snippet, the datasets are directly sent to `model.fit()` without needing to shard; this is because `tf.distribute.Strategy` API takes care of the dataset sharding automatically in multi-worker trainings.\n", - "\n", - "If you prefer manual sharding for your training, automatic sharding can be turned off via `tf.data.experimental.DistributeOptions` api. Concretely," + "In the code snippet above note that the `global_batch_size`, which gets passed to `Dataset.batch`, is set to `per_worker_batch_size * num_workers`. This ensures that each worker processes batches of `per_worker_batch_size` examples regardless of the number of workers." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lHLhOii67Saa" + }, + "source": [ + "The current directory now contains both Python files:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JxEtdh1vH-TF" + "id": "bi6x05Sr60O9" }, "outputs": [], "source": [ - "options = tf.data.Options()\n", - "options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF\n", - "train_datasets_no_auto_shard = train_datasets.with_options(options)" + "%%bash\n", + "ls *.py" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "NBCtYvmCH-7g" + "id": "qmEEStPS6vR_" }, "source": [ - "Another thing to notice is the batch size for the `datasets`. In the code snippet above, we use `GLOBAL_BATCH_SIZE = 64 * NUM_WORKERS`, which is `NUM_WORKERS` times as large as the case it was for single worker, because the effective per worker batch size is the global batch size (the parameter passed in `tf.data.Dataset.batch()`) divided by the number of workers, and with this change we are keeping the per worker batch size same as before." + "Serialize the `TF_CONFIG` to JSON and add it to the environment variables:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "XVk4ftYx6JAO" + "id": "9uu3g7vV7Bbt" }, + "outputs": [], "source": [ - "## Performance\n", - "\n", - "You now have a Keras model that is all set up to run in multiple workers with `MultiWorkerMirroredStrategy`. You can try the following techniques to tweak performance of multi-worker training.\n", - "\n", - "* `MultiWorkerMirroredStrategy` provides multiple [collective communication implementations](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/distribute/cross_device_ops.py). `RING` implements ring-based collectives using gRPC as the cross-host communication layer. `NCCL` uses [Nvidia's NCCL](https://developer.nvidia.com/nccl) to implement collectives. `AUTO` defers the choice to the runtime. The best choice of collective implementation depends upon the number and kind of GPUs, and the network interconnect in the cluster. To override the automatic choice, specify a valid value to the `communication` parameter of `MultiWorkerMirroredStrategy`'s constructor, e.g. `communication=tf.distribute.experimental.CollectiveCommunication.NCCL`.\n", - "* Cast the variables to `tf.float` if possible. The official ResNet model includes [an example](https://github.com/tensorflow/models/blob/8367cf6dabe11adf7628541706b660821f397dce/official/resnet/resnet_model.py#L466) of how this can be done.\n", - "\n", - "\n", - "\n" + "os.environ['TF_CONFIG'] = json.dumps(tf_config)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "97WhAu8uKw3j" + "id": "MsY3dQLK7jdf" }, "source": [ - "## Fault tolerance\n", - "\n", - "In synchronous training, the cluster would fail if one of the workers fails and no failure-recovery mechanism exists. Using Keras with `tf.distribute.Strategy` comes with the advantage of fault tolerance in cases where workers die or are otherwise unstable. We do this by preserving training state in the distributed file system of your choice, such that upon restart of the instance that previously failed or preempted, the training state is recovered.\n", - "\n", - "Since all the workers are kept in sync in terms of training epochs and steps, other workers would need to wait for the failed or preempted worker to restart to continue.\n", - "\n", - "### ModelCheckpoint callback\n", - "\n", - "To take advantage of fault tolerance in multi-worker training, provide an instance of `tf.keras.callbacks.ModelCheckpoint` at the `tf.keras.Model.fit()` call. The callback will store the checkpoint and training state in the directory corresponding to the `filepath` argument to `ModelCheckpoint`." + "Now, you can launch a worker process that will run the `main.py` and use the `TF_CONFIG`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xIY9vKnUU82o" + "id": "txMXaq8d8N_S" }, "outputs": [], "source": [ - "# Replace the `filepath` argument with a path in the file system\n", - "# accessible by all workers.\n", - "callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath='/tmp/keras-ckpt')]\n", - "with strategy.scope():\n", - " multi_worker_model = build_and_compile_cnn_model()\n", - "multi_worker_model.fit(x=train_datasets,\n", - " epochs=3,\n", - " steps_per_epoch=5,\n", - " callbacks=callbacks)" + "# first kill any previous runs\n", + "%killbgscripts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qnSma_Ck7r-r" + }, + "outputs": [], + "source": [ + "%%bash --bg\n", + "python main.py &> job_0.log" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Ii6VmEdOjkZr" + "id": "ZChyazqS7v0P" }, "source": [ - "If a worker gets preempted, the whole cluster pauses until the preempted worker is restarted. Once the worker rejoins the cluster, other workers will also restart. Now, every worker reads the checkpoint file that was previously saved and picks up its former state, thereby allowing the cluster to get back in sync. Then the training continues.\n", + "There are a few things to note about the above command:\n", "\n", - "If you inspect the directory containing the `filepath` you specified in `ModelCheckpoint`, you may notice some temporarily generated checkpoint files. Those files are needed for recovering the previously lost instances, and they will be removed by the library at the end of `tf.keras.Model.fit()` upon successful exiting of your multi-worker training." + "1. It uses the `%%bash` which is a [notebook \"magic\"](https://ipython.readthedocs.io/en/stable/interactive/magics.html) to run some bash commands.\n", + "2. It uses the `--bg` flag to run the `bash` process in the background, because this worker will not terminate. It waits for all the workers before it starts.\n", + "\n", + "The backgrounded worker process won't print output to this notebook, so the `&>` redirects its output to a file so that you can inspect what happened in a log file later.\n", + "\n", + "So, wait a few seconds for the process to start up:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "ega2hdOQEmy_" + "id": "Hm2yrULE9281" }, + "outputs": [], "source": [ - "## See also\n", - "1. [Distributed Training in TensorFlow](https://www.tensorflow.org/guide/distributed_training) guide provides an overview of the available distribution strategies.\n", - "2. [Official models](https://github.com/tensorflow/models/tree/master/official), many of which can be configured to run multiple distribution strategies.\n" + "import time\n", + "time.sleep(10)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZFPoNxg_9_Mx" + }, + "source": [ + "Now, inspect what's been output to the worker's log file so far:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vZEOuVgQ9-hn" + }, + "outputs": [], + "source": [ + "%%bash\n", + "cat job_0.log" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RqZhVF7L_KOy" + }, + "source": [ + "The last line of the log file should say: `Started server with target: grpc://localhost:12345`. The first worker is now ready and is waiting for all the other worker(s) to be ready to proceed." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Pi8vPNNA_l4a" + }, + "source": [ + "So update the `tf_config` for the second worker's process to pick up:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lAiYkkPu_Jqd" + }, + "outputs": [], + "source": [ + "tf_config['task']['index'] = 1\n", + "os.environ['TF_CONFIG'] = json.dumps(tf_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0AshGVO0_x0w" + }, + "source": [ + "Launch the second worker. This will start the training since all the workers are active (so there's no need to background this process):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_ESVtyQ9_xjx" + }, + "outputs": [], + "source": [ + "%%bash\n", + "python main.py" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hX4FA2O2AuAn" + }, + "source": [ + "If you recheck the logs written by the first worker, you'll learn that it participated in training that model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rc6hw3yTBKXX" + }, + "outputs": [], + "source": [ + "%%bash\n", + "cat job_0.log" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zL79ak5PMzEg" + }, + "source": [ + "Note: This may run slower than the test run at the beginning of this tutorial because running multiple workers on a single machine only adds overhead. The goal here is not to improve the training time but to give an example of multi-worker training.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sG5_1UgrgniF" + }, + "outputs": [], + "source": [ + "# Delete the `TF_CONFIG`, and kill any background tasks so they don't affect the next section.\n", + "os.environ.pop('TF_CONFIG', None)\n", + "%killbgscripts" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9j2FJVHoUIrE" + }, + "source": [ + "## Multi-worker training in depth\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C1hBks_dAZmT" + }, + "source": [ + "So far, you have learned how to perform a basic multi-worker setup. The rest of the tutorial goes over other factors, which may be useful or important for real use cases, in detail." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Rr14Vl9GR4zq" + }, + "source": [ + "### Dataset sharding\n", + "\n", + "In multi-worker training, _dataset sharding_ is needed to ensure convergence and performance.\n", + "\n", + "The example in the previous section relies on the default autosharding provided by the `tf.distribute.Strategy` API. You can control the sharding by setting the `tf.data.experimental.AutoShardPolicy` of the `tf.data.experimental.DistributeOptions`.\n", + "\n", + "To learn more about _auto-sharding_, refer to the [Distributed input guide](https://www.tensorflow.org/tutorials/distribute/input#sharding).\n", + "\n", + "Here is a quick example of how to turn the auto sharding off, so that each replica processes every example (_not recommended_):\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JxEtdh1vH-TF" + }, + "outputs": [], + "source": [ + "options = tf.data.Options()\n", + "options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF\n", + "\n", + "global_batch_size = 64\n", + "multi_worker_dataset = mnist_setup.mnist_dataset(batch_size=64)\n", + "dataset_no_auto_shard = multi_worker_dataset.with_options(options)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "z85hElxsBQsT" + }, + "source": [ + "### Evaluation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gmqvlh5LhAoU" + }, + "source": [ + "If you pass the `validation_data` into `Model.fit` as well, it will alternate between training and evaluation for each epoch. The evaluation work is distributed across the same set of workers, and its results are aggregated and available to all workers.\n", + "\n", + "Similar to training, the validation dataset is automatically sharded at the file level. You need to set a global batch size in the validation dataset and set the `validation_steps`.\n", + "\n", + "A repeated dataset (by calling `tf.data.Dataset.repeat`) is recommended for evaluation.\n", + "\n", + "Alternatively, you can also create another task that periodically reads checkpoints and runs the evaluation. This is what an Estimator does. But this is not a recommended way to perform evaluation and thus its details are omitted." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FNkoxUPJBNTb" + }, + "source": [ + "### Performance" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XVk4ftYx6JAO" + }, + "source": [ + "To tweak the performance of multi-worker training, you can try the following:\n", + "\n", + "- `tf.distribute.MultiWorkerMirroredStrategy` provides multiple [collective communication implementations](https://www.tensorflow.org/api_docs/python/tf/distribute/experimental/CommunicationImplementation):\n", + " - `RING` implements ring-based collectives using gRPC as the cross-host communication layer.\n", + " - `NCCL` uses the [NVIDIA Collective Communication Library](https://developer.nvidia.com/nccl) to implement collectives.\n", + " - `AUTO` defers the choice to the runtime.\n", + " \n", + " The best choice of collective implementation depends upon the number of GPUs, the type of GPUs, and the network interconnects in the cluster. To override the automatic choice, specify the `communication_options` parameter of `MultiWorkerMirroredStrategy`'s constructor. For example:\n", + " \n", + " ```python\n", + " communication_options=tf.distribute.experimental.CommunicationOptions(implementation=tf.distribute.experimental.CommunicationImplementation.NCCL)\n", + " ```\n", + "\n", + "- Cast the variables to `tf.float` if possible:\n", + " - The official ResNet model includes [an example](https://github.com/tensorflow/models/blob/8367cf6dabe11adf7628541706b660821f397dce/official/resnet/resnet_model.py#L466) of how to do this." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "97WhAu8uKw3j" + }, + "source": [ + "### Fault tolerance\n", + "\n", + "In synchronous training, the cluster would fail if one of the workers fails and no failure-recovery mechanism exists.\n", + "\n", + "Using Keras with `tf.distribute.Strategy` comes with the advantage of fault tolerance in cases where workers die or are otherwise unstable. You can do this by preserving the training state in the distributed file system of your choice, such that upon a restart of the instance that previously failed or preempted, the training state is recovered.\n", + "\n", + "When a worker becomes unavailable, other workers will fail (possibly after a timeout). In such cases, the unavailable worker needs to be restarted, as well as other workers that have failed.\n", + "\n", + "Note: Previously, the `ModelCheckpoint` callback provided a mechanism to restore the training state upon a restart from a job failure for multi-worker training. The TensorFlow team is introducing a new [`BackupAndRestore`](#scrollTo=kmH8uCUhfn4w) callback, which also adds the support to single-worker training for a consistent experience, and removed the fault tolerance functionality from existing `ModelCheckpoint` callback. From now on, applications that rely on this behavior should migrate to the new `BackupAndRestore` callback." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KvHPjGlyyFt6" + }, + "source": [ + "#### The `ModelCheckpoint` callback\n", + "\n", + "`ModelCheckpoint` callback no longer provides fault tolerance functionality, please use [`BackupAndRestore`](#scrollTo=kmH8uCUhfn4w) callback instead.\n", + "\n", + "The `ModelCheckpoint` callback can still be used to save checkpoints. But with this, if training was interrupted or successfully finished, in order to continue training from the checkpoint, the user is responsible to load the model manually.\n", + "\n", + "Optionally, users can choose to save and restore model/weights outside `ModelCheckpoint` callback." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EUNV5Utc1d0s" + }, + "source": [ + "### Model saving and loading\n", + "\n", + "To save your model using `model.save` or `tf.saved_model.save`, the saving destination needs to be different for each worker.\n", + "\n", + "- For non-chief workers, you will need to save the model to a temporary directory.\n", + "- For the chief, you will need to save to the provided model directory.\n", + "\n", + "The temporary directories on the worker need to be unique to prevent errors resulting from multiple workers trying to write to the same location.\n", + "\n", + "The model saved in all the directories is identical, and typically only the model saved by the chief should be referenced for restoring or serving.\n", + "\n", + "You should have some cleanup logic that deletes the temporary directories created by the workers once your training has completed.\n", + "\n", + "The reason for saving on the chief and workers at the same time is because you might be aggregating variables during checkpointing, which requires both the chief and workers to participate in the allreduce communication protocol. On the other hand, letting chief and workers save to the same model directory will result in errors due to contention.\n", + "\n", + "Using the `MultiWorkerMirroredStrategy`, the program is run on every worker, and in order to know whether the current worker is the chief, it takes advantage of the cluster resolver object that has attributes `task_type` and `task_id`:\n", + "- `task_type` tells you what the current job is (for example, `'worker'`).\n", + "- `task_id` tells you the identifier of the worker.\n", + "- The worker with `task_id == 0` is designated as the chief worker.\n", + "\n", + "In the code snippet below, the `write_filepath` function provides the file path to write, which depends on the worker's `task_id`:\n", + "\n", + "- For the chief worker (with `task_id == 0`), it writes to the original file path. \n", + "- For other workers, it creates a temporary directory—`temp_dir`—with the `task_id` in the directory path to write in:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XQfGkmg-pfCY" + }, + "outputs": [], + "source": [ + "model_path = '/tmp/keras-model'\n", + "\n", + "def _is_chief(task_type, task_id):\n", + " # Note: there are two possible `TF_CONFIG` configurations.\n", + " # 1) In addition to `worker` tasks, a `chief` task type is use;\n", + " # in this case, this function should be modified to\n", + " # `return task_type == 'chief'`.\n", + " # 2) Only `worker` task type is used; in this case, worker 0 is\n", + " # regarded as the chief. The implementation demonstrated here\n", + " # is for this case.\n", + " # For the purpose of this Colab section, the `task_type` is `None` case\n", + " # is added because it is effectively run with only a single worker.\n", + " return (task_type == 'worker' and task_id == 0) or task_type is None\n", + "\n", + "def _get_temp_dir(dirpath, task_id):\n", + " base_dirpath = 'workertemp_' + str(task_id)\n", + " temp_dir = os.path.join(dirpath, base_dirpath)\n", + " tf.io.gfile.makedirs(temp_dir)\n", + " return temp_dir\n", + "\n", + "def write_filepath(filepath, task_type, task_id):\n", + " dirpath = os.path.dirname(filepath)\n", + " base = os.path.basename(filepath)\n", + " if not _is_chief(task_type, task_id):\n", + " dirpath = _get_temp_dir(dirpath, task_id)\n", + " return os.path.join(dirpath, base)\n", + "\n", + "task_type, task_id = (strategy.cluster_resolver.task_type,\n", + " strategy.cluster_resolver.task_id)\n", + "write_model_path = write_filepath(model_path, task_type, task_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hs0_agYR_qKm" + }, + "source": [ + "With that, you're now ready to save:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XnToxeIcg_6O" + }, + "source": [ + "Deprecated: For Keras objects, it's recommended to use the new high-level `.keras` format and `tf.keras.Model.export`, as demonstrated in the guide [here](https://www.tensorflow.org/guide/keras/save_and_serialize). The low-level SavedModel format continues to be supported for existing code." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "J-yA3BYG_vTs" + }, + "outputs": [], + "source": [ + "multi_worker_model.save(write_model_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8LXUVVl9_v5x" + }, + "source": [ + "As described above, later on the model should only be loaded from the file path the chief worker saved to. Therefore, remove the temporary ones the non-chief workers have saved:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aJTyu-97ABpY" + }, + "outputs": [], + "source": [ + "if not _is_chief(task_type, task_id):\n", + " tf.io.gfile.rmtree(os.path.dirname(write_model_path))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Nr-2PKlHAPBT" + }, + "source": [ + "Now, when it's time to load, use the convenient `tf.keras.models.load_model` API, and continue with further work.\n", + "\n", + "Here, assume only using single worker to load and continue training, in which case you do not call `tf.keras.models.load_model` within another `strategy.scope()` (note that `strategy = tf.distribute.MultiWorkerMirroredStrategy()`, as defined earlier):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "iUZna-JKAOrX" + }, + "outputs": [], + "source": [ + "loaded_model = tf.keras.models.load_model(model_path)\n", + "\n", + "# Now that the model is restored, and can continue with the training.\n", + "loaded_model.fit(single_worker_dataset, epochs=2, steps_per_epoch=20)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YJ1fmxmTpocS" + }, + "source": [ + "### Checkpoint saving and restoring\n", + "\n", + "On the other hand, checkpointing allows you to save your model's weights and restore them without having to save the whole model.\n", + "\n", + "Here, you'll create one `tf.train.Checkpoint` that tracks the model, which is managed by the `tf.train.CheckpointManager`, so that only the latest checkpoint is preserved:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_1-RYaB5xnNH" + }, + "outputs": [], + "source": [ + "checkpoint_dir = '/tmp/ckpt'\n", + "\n", + "checkpoint = tf.train.Checkpoint(model=multi_worker_model)\n", + "write_checkpoint_dir = write_filepath(checkpoint_dir, task_type, task_id)\n", + "checkpoint_manager = tf.train.CheckpointManager(\n", + " checkpoint, directory=write_checkpoint_dir, max_to_keep=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7oBpPCRsW1MF" + }, + "source": [ + "Once the `CheckpointManager` is set up, you're now ready to save and remove the checkpoints the non-chief workers had saved:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "l1ZXG_GbWzLp" + }, + "outputs": [], + "source": [ + "checkpoint_manager.save()\n", + "if not _is_chief(task_type, task_id):\n", + " tf.io.gfile.rmtree(write_checkpoint_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RO7cbN40XD5v" + }, + "source": [ + "Now, when you need to restore the model, you can find the latest checkpoint saved using the convenient `tf.train.latest_checkpoint` function. After restoring the checkpoint, you can continue with training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NJW7vtknXFEH" + }, + "outputs": [], + "source": [ + "latest_checkpoint = tf.train.latest_checkpoint(checkpoint_dir)\n", + "checkpoint.restore(latest_checkpoint)\n", + "multi_worker_model.fit(multi_worker_dataset, epochs=2, steps_per_epoch=20)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kmH8uCUhfn4w" + }, + "source": [ + "#### The `BackupAndRestore` callback\n", + "\n", + "The `tf.keras.callbacks.BackupAndRestore` callback provides the fault tolerance functionality by backing up the model and current training state in a temporary checkpoint file under `backup_dir` argument to `BackupAndRestore`. \n", + "\n", + "Note: In Tensorflow 2.9, the current model and the training state is backed up at epoch boundaries. In the `tf-nightly` version and from TensorFlow 2.10, the `BackupAndRestore` callback can back up the model and the training state at epoch or step boundaries. `BackupAndRestore` accepts an optional `save_freq` argument. `save_freq` accepts either `'epoch'` or an `int` value. If `save_freq` is set to `'epoch'` the model is backed up after every epoch. If `save_freq` is set to an integer value greater than `0`, the model is backed up after every `save_freq` number of batches.\n", + "\n", + "Once the jobs get interrupted and restarted, the `BackupAndRestore` callback restores the last checkpoint, and you can continue training from the beginning of the epoch and step at which the training state was last saved.\n", + "\n", + "To use it, provide an instance of `tf.keras.callbacks.BackupAndRestore` at the `Model.fit` call.\n", + "\n", + "With `MultiWorkerMirroredStrategy`, if a worker gets interrupted, the whole cluster will pause until the interrupted worker is restarted. Other workers will also restart, and the interrupted worker will rejoin the cluster. Then, every worker will read the checkpoint file that was previously saved and pick up its former state, thereby allowing the cluster to get back in sync. Then, the training will continue. The distributed dataset iterator state will be re-initialized and not restored.\n", + "\n", + "The `BackupAndRestore` callback uses the `CheckpointManager` to save and restore the training state, which generates a file called checkpoint that tracks existing checkpoints together with the latest one. For this reason, `backup_dir` should not be re-used to store other checkpoints in order to avoid name collision.\n", + "\n", + "Currently, the `BackupAndRestore` callback supports single-worker training with no strategy—`MirroredStrategy`—and multi-worker training with `MultiWorkerMirroredStrategy`.\n", + "\n", + "Below are two examples for both multi-worker training and single-worker training:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CYdzZi4Qs1jz" + }, + "outputs": [], + "source": [ + "# Multi-worker training with `MultiWorkerMirroredStrategy`\n", + "# and the `BackupAndRestore` callback. The training state \n", + "# is backed up at epoch boundaries by default.\n", + "\n", + "callbacks = [tf.keras.callbacks.BackupAndRestore(backup_dir='/tmp/backup')]\n", + "with strategy.scope():\n", + " multi_worker_model = mnist_setup.build_and_compile_cnn_model()\n", + "multi_worker_model.fit(multi_worker_dataset,\n", + " epochs=3,\n", + " steps_per_epoch=70,\n", + " callbacks=callbacks)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f8e86TAp0Rsl" + }, + "source": [ + "If the `save_freq` argument in the `BackupAndRestore` callback is set to `'epoch'`, the model is backed up after every epoch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rZjQGPsF0aEI" + }, + "outputs": [], + "source": [ + "# The training state is backed up at epoch boundaries because `save_freq` is\n", + "# set to `epoch`.\n", + "\n", + "callbacks = [tf.keras.callbacks.BackupAndRestore(backup_dir='/tmp/backup')]\n", + "with strategy.scope():\n", + " multi_worker_model = mnist_setup.build_and_compile_cnn_model()\n", + "multi_worker_model.fit(multi_worker_dataset,\n", + " epochs=3,\n", + " steps_per_epoch=70,\n", + " callbacks=callbacks)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p-r44kCM0jc6" + }, + "source": [ + "Note: The next code block uses features that are only available in `tf-nightly` until Tensorflow 2.10 is released.\n", + "\n", + "If the `save_freq` argument in the `BackupAndRestore` callback is set to an integer value greater than `0`, the model is backed up after every `save_freq` number of batches." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bSJUyLSF0moC" + }, + "outputs": [], + "source": [ + "# The training state is backed up at every 30 steps because `save_freq` is set\n", + "# to an integer value of `30`.\n", + "\n", + "callbacks = [tf.keras.callbacks.BackupAndRestore(backup_dir='/tmp/backup', save_freq=30)]\n", + "with strategy.scope():\n", + " multi_worker_model = mnist_setup.build_and_compile_cnn_model()\n", + "multi_worker_model.fit(multi_worker_dataset,\n", + " epochs=3,\n", + " steps_per_epoch=70,\n", + " callbacks=callbacks)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rIV5_3ebzXmB" + }, + "source": [ + "If you inspect the directory of `backup_dir` you specified in `BackupAndRestore`, you may notice some temporarily generated checkpoint files. Those files are needed for recovering the previously lost instances, and they will be removed by the library at the end of `Model.fit` upon successful exiting of your training.\n", + "\n", + "Note: Currently the `BackupAndRestore` callback only supports eager mode. In graph mode, consider using `Model.save`/`tf.saved_model.save` and `tf.keras.models.load_model` for saving and restoring models, respectively, as described in the _Model saving and loading_ section above, and by providing `initial_epoch` in `Model.fit` during training." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ega2hdOQEmy_" + }, + "source": [ + "## Additional resources\n", + "\n", + "1. The [Distributed training in TensorFlow](../../guide/distributed_training.ipynb) guide provides an overview of the available distribution strategies.\n", + "1. The [Custom training loop with Keras and MultiWorkerMirroredStrategy](multi_worker_with_ctl.ipynb) tutorial shows how to use the `MultiWorkerMirroredStrategy` with Keras and a custom training loop.\n", + "1. Check out the [official models](https://github.com/tensorflow/models/tree/master/official), many of which can be configured to run multiple distribution strategies.\n", + "1. The [Better performance with tf.function](../../guide/function.ipynb) guide provides information about other strategies and tools, such as the [TensorFlow Profiler](../../guide/profiler.md) you can use to optimize the performance of your TensorFlow models." ] } ], "metadata": { "colab": { - "collapsed_sections": [], "name": "multi_worker_with_keras.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/distribute/parameter_server_training.ipynb b/site/en/tutorials/distribute/parameter_server_training.ipynb new file mode 100644 index 00000000000..2e6bb0cfce2 --- /dev/null +++ b/site/en/tutorials/distribute/parameter_server_training.ipynb @@ -0,0 +1,1395 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "rA5Mubike7OJ" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "fY0a3LRYfHUl" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iNz7xXMSsAQa" + }, + "source": [ + "# Parameter server training with ParameterServerStrategy" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jHyqRIqxsJuc" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6v4D6QfcfTrm" + }, + "source": [ + "## Overview\n", + "\n", + "[Parameter server training](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-li_mu.pdf) is a common data-parallel method to scale up model training on multiple machines.\n", + "\n", + "A parameter server training cluster consists of _workers_ and _parameter servers_. Variables are created on parameter servers and they are read and updated by workers in each step. By default, workers read and update these variables independently without synchronizing with each other. This is why sometimes parameter server-style training is called _asynchronous training_.\n", + "\n", + "In TensorFlow 2, parameter server training is powered by the `tf.distribute.ParameterServerStrategy` class, which distributes the training steps to a cluster that scales up to thousands of workers (accompanied by parameter servers)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W1LGfTdgOF-J" + }, + "source": [ + "### Supported training methods\n", + "\n", + "There are two main supported training methods:\n", + "\n", + "- The Keras `Model.fit` API: if you prefer a high-level abstraction and handling of training. This is generally recommended if you are training a `tf.keras.Model`.\n", + "- A custom training loop: if you prefer to define the details of your training loop (you can refer to guides on [Custom training](../customization/custom_training_walkthrough.ipynb), [Writing a training loop from scratch\n", + "](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) and [Custom training loop with Keras and MultiWorkerMirroredStrategy](multi_worker_with_ctl.ipynb) for more details)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FjbULGvV7NRz" + }, + "source": [ + "### A cluster with jobs and tasks\n", + "\n", + "Regardless of the API of choice (`Model.fit` or a custom training loop), distributed training in TensorFlow 2 involves a `'cluster'` with several `'jobs'`, and each of the jobs may have one or more `'tasks'`.\n", + "\n", + "When using parameter server training, it is recommended to have:\n", + "\n", + "- One _coordinator_ job (which has the job name `chief`)\n", + "- Multiple _worker_ jobs (job name `worker`)\n", + "- Multiple _parameter server_ jobs (job name `ps`)\n", + "\n", + "The _coordinator_ creates resources, dispatches training tasks, writes checkpoints, and deals with task failures. The _workers_ and _parameter servers_ run `tf.distribute.Server` instances that listen for requests from the coordinator." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oLV1FbpLtqtB" + }, + "source": [ + "### Parameter server training with the `Model.fit` API\n", + "\n", + "Parameter server training with the `Model.fit` API requires the coordinator to use a `tf.distribute.ParameterServerStrategy` object. Similar to `Model.fit` usage with no strategy, or with other strategies, the workflow involves creating and compiling the model, preparing the callbacks, and calling `Model.fit`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yJ5AosxFyfzk" + }, + "source": [ + "### Parameter server training with a custom training loop\n", + "\n", + "With custom training loops, the `tf.distribute.coordinator.ClusterCoordinator` class is the key component used for the coordinator.\n", + "\n", + "- The `ClusterCoordinator` class needs to work in conjunction with a `tf.distribute.ParameterServerStrategy` object.\n", + "- This `tf.distribute.Strategy` object is needed to provide the information of the cluster and is used to define a training step, as demonstrated in [Custom training with tf.distribute.Strategy](custom_training.ipynb).\n", + "- The `ClusterCoordinator` object then dispatches the execution of these training steps to remote workers.\n", + "\n", + "The most important API provided by the `ClusterCoordinator` object is `schedule`:\n", + "\n", + "- The `schedule` API enqueues a `tf.function` and returns a future-like `RemoteValue` immediately.\n", + "- The queued functions will be dispatched to remote workers in background threads and their `RemoteValue`s will be filled asynchronously.\n", + "- Since `schedule` doesn’t require worker assignment, the `tf.function` passed in can be executed on any available worker.\n", + "- If the worker it is executed on becomes unavailable before its completion, the function will be retried on another available worker.\n", + "- Because of this fact and the fact that function execution is not atomic, a single function call may be executed more than once.\n", + "\n", + "In addition to dispatching remote functions, the `ClusterCoordinator` also helps\n", + "to create datasets on all the workers and rebuild these datasets when a worker recovers from failure." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MyDnWjmOje5-" + }, + "source": [ + "## Tutorial setup\n", + "\n", + "The tutorial will branch into `Model.fit` and custom training loop paths, and you can choose the one that fits your needs. Sections other than \"Training with X\" are applicable to both paths." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0-V3LUcIs4a-" + }, + "outputs": [], + "source": [ + "!pip install portpicker" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GlI_NAVFae3J" + }, + "outputs": [], + "source": [ + "#@title\n", + "import multiprocessing\n", + "import os\n", + "import random\n", + "import portpicker\n", + "import tensorflow as tf" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uvwgM2rzgzIC" + }, + "source": [ + "## Cluster setup\n", + "\n", + "As mentioned above, a parameter server training cluster requires a coordinator task that runs your training program, one or several workers and parameter server tasks that run TensorFlow servers—`tf.distribute.Server`—and possibly an additional evaluation task that runs sidecar evaluation (refer to the [sidecar evaluation section](#sidecar_evaluation) below). The requirements to set them up are:\n", + "\n", + "- The coordinator task needs to know the addresses and ports of all other TensorFlow servers, except the evaluator.\n", + "- The workers and parameter servers need to know which port they need to listen to. For the sake of simplicity, you can usually pass in the complete cluster information when creating TensorFlow servers on these tasks.\n", + "- The evaluator task doesn’t have to know the setup of the training cluster. If it does, it should not attempt to connect to the training cluster.\n", + "- Workers and parameter servers should have task types as `\"worker\"` and `\"ps\"`, respectively. The coordinator should use `\"chief\"` as the task type for legacy reasons.\n", + "\n", + "In this tutorial, you will create an in-process cluster so that the whole parameter server training can be run in Colab. You will learn how to set up [real clusters](#real_clusters) in a later section." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7UNs7Lm2g19n" + }, + "source": [ + "### In-process cluster\n", + "\n", + "You will start by creating several TensorFlow servers in advance and you will connect to them later. Note that this is only for the purpose of this tutorial's demonstration, and in real training the servers will be started on `\"worker\"` and `\"ps\"` machines." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FbrP5pXuaoVH" + }, + "outputs": [], + "source": [ + "def create_in_process_cluster(num_workers, num_ps):\n", + " \"\"\"Creates and starts local servers and returns the cluster_resolver.\"\"\"\n", + " worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)]\n", + " ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)]\n", + "\n", + " cluster_dict = {}\n", + " cluster_dict[\"worker\"] = [\"localhost:%s\" % port for port in worker_ports]\n", + " if num_ps > 0:\n", + " cluster_dict[\"ps\"] = [\"localhost:%s\" % port for port in ps_ports]\n", + "\n", + " cluster_spec = tf.train.ClusterSpec(cluster_dict)\n", + "\n", + " # Workers need some inter_ops threads to work properly.\n", + " worker_config = tf.compat.v1.ConfigProto()\n", + " if multiprocessing.cpu_count() < num_workers + 1:\n", + " worker_config.inter_op_parallelism_threads = num_workers + 1\n", + "\n", + " for i in range(num_workers):\n", + " tf.distribute.Server(\n", + " cluster_spec,\n", + " job_name=\"worker\",\n", + " task_index=i,\n", + " config=worker_config,\n", + " protocol=\"grpc\")\n", + "\n", + " for i in range(num_ps):\n", + " tf.distribute.Server(\n", + " cluster_spec,\n", + " job_name=\"ps\",\n", + " task_index=i,\n", + " protocol=\"grpc\")\n", + "\n", + " cluster_resolver = tf.distribute.cluster_resolver.SimpleClusterResolver(\n", + " cluster_spec, rpc_layer=\"grpc\")\n", + " return cluster_resolver\n", + "\n", + "# Set the environment variable to allow reporting worker and ps failure to the\n", + "# coordinator. This is a workaround and won't be necessary in the future.\n", + "os.environ[\"GRPC_FAIL_FAST\"] = \"use_caller\"\n", + "\n", + "NUM_WORKERS = 3\n", + "NUM_PS = 2\n", + "cluster_resolver = create_in_process_cluster(NUM_WORKERS, NUM_PS)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pX_91OByt0J2" + }, + "source": [ + "The in-process cluster setup is frequently used in unit testing, such as [here](https://github.com/tensorflow/tensorflow/blob/eb4c40fc91da260199fa2aed6fe67d36ad49fafd/tensorflow/python/distribute/coordinator/cluster_coordinator_test.py#L447).\n", + "\n", + "Another option for local testing is to launch processes on the local machine—check out [Multi-worker training with Keras](multi_worker_with_keras.ipynb) for an example of this approach." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zyby6M2Jqg6J" + }, + "source": [ + "## Instantiate a ParameterServerStrategy\n", + "\n", + "Before you dive into the training code, let's instantiate a `tf.distribute.ParameterServerStrategy` object. Note that this is needed regardless of whether you are proceeding with `Model.fit` or a custom training loop. The `variable_partitioner` argument will be explained in the [Variable sharding section](#variable_sharding)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_YyEPgisrC35" + }, + "outputs": [], + "source": [ + "variable_partitioner = (\n", + " tf.distribute.experimental.partitioners.MinSizePartitioner(\n", + " min_shard_bytes=(256 << 10),\n", + " max_shards=NUM_PS))\n", + "\n", + "strategy = tf.distribute.ParameterServerStrategy(\n", + " cluster_resolver,\n", + " variable_partitioner=variable_partitioner)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WlAQxuMDJ3k9" + }, + "source": [ + "In order to use GPUs for training, allocate GPUs visible to each worker. `ParameterServerStrategy` will use all the available GPUs on each worker, with the restriction that all workers should have the same number of GPUs available." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QMmBLsf6sEXh" + }, + "source": [ + "### Variable sharding\n", + "\n", + "Variable sharding refers to splitting a variable into multiple smaller\n", + "variables, which are called _shards_. Variable sharding may be useful to distribute the network load when accessing these shards. It is also useful to distribute computation and storage of a normal variable across multiple parameter servers, for example, when using very large embeddings\n", + "that may not fit in a single machine's memory.\n", + "\n", + "To enable variable sharding, you can pass in a `variable_partitioner` when\n", + "constructing a `ParameterServerStrategy` object. The `variable_partitioner` will\n", + "be invoked every time when a variable is created and it is expected to return\n", + "the number of shards along each dimension of the variable. Some out-of-box\n", + "`variable_partitioner`s are provided such as\n", + "`tf.distribute.experimental.partitioners.MinSizePartitioner`. It is recommended to use size-based partitioners like\n", + "`tf.distribute.experimental.partitioners.MinSizePartitioner` to avoid\n", + "partitioning small variables, which could have a negative impact on model training\n", + "speed." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1--SxlxtsOb7" + }, + "source": [ + "When a `variable_partitioner` is passed in, and you create a variable directly\n", + "under `Strategy.scope`, the variable will become a container type with a `variables`\n", + "property, which provides access to the list of shards. In most cases, this\n", + "container will be automatically converted to a Tensor by concatenating all the\n", + "shards. As a result, it can be used as a normal variable. On the other hand,\n", + "some TensorFlow methods such as `tf.nn.embedding_lookup` provide efficient\n", + "implementation for this container type and in these methods automatic\n", + "concatenation will be avoided.\n", + "\n", + "Refer to the API docs of `tf.distribute.ParameterServerStrategy` for more details." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jlOq-O-26O1d" + }, + "source": [ + "## Training with `Model.fit`\n", + "    \n", + "\n", + "Keras provides an easy-to-use training API via `Model.fit` that handles the training loop under the hood, with the flexibility of an overridable `train_step`, and callbacks which provide functionalities such as checkpoint saving or summary saving for TensorBoard. With `Model.fit`, the same training code can be used with other strategies with a simple swap of the strategy object." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oMZ9Cu5J6ZGi" + }, + "source": [ + "### Input data\n", + "\n", + "Keras `Model.fit` with `tf.distribute.ParameterServerStrategy` can take input data in the form of a `tf.data.Dataset`, `tf.distribute.DistributedDataset`, or a `tf.keras.utils.experimental.DatasetCreator`, with `Dataset` being the recommended option for ease of use. If you encounter memory issues using `Dataset`, however, you may need to use `DatasetCreator` with a callable `dataset_fn` argument (refer to the `tf.keras.utils.experimental.DatasetCreator` API documentation for details).\n", + "\n", + "If you transform your dataset into a `tf.data.Dataset`, you should use `Dataset.shuffle` and `Dataset.repeat`, as demonstrated in the code example below.\n", + "\n", + "- Keras `Model.fit` with parameter server training assumes that each worker receives the same dataset, except when it is shuffled differently. Therefore, by calling `Dataset.shuffle`, you ensure more even iterations over the data.\n", + "- Because workers do not synchronize, they may finish processing their datasets at different times. Therefore, the easiest way to define epochs with parameter server training is to use `Dataset.repeat`—which repeats a dataset indefinitely when called without an argument—and specify the `steps_per_epoch` argument in the `Model.fit` call.\n", + "\n", + "Refer to the \"Training workflows\" section of the [tf.data guide](../../guide/data.ipynb) for more details on `shuffle` and `repeat`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "shAo1CCS7wU1" + }, + "outputs": [], + "source": [ + "global_batch_size = 64\n", + "\n", + "x = tf.random.uniform((10, 10))\n", + "y = tf.random.uniform((10,))\n", + "\n", + "dataset = tf.data.Dataset.from_tensor_slices((x, y)).shuffle(10).repeat()\n", + "dataset = dataset.batch(global_batch_size)\n", + "dataset = dataset.prefetch(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v_jhF70K7zON" + }, + "source": [ + "If you instead create your dataset with `tf.keras.utils.experimental.DatasetCreator`, the code in `dataset_fn` will be invoked on the input device, which is usually the CPU, on each of the worker machines.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "w60PuWrWwBD4" + }, + "source": [ + "### Model construction and compiling\n", + "\n", + "Now, you will create a `tf.keras.Model`—a trivial `tf.keras.models.Sequential` model for demonstration purposes—followed by a `Model.compile` call to incorporate components, such as an optimizer, metrics, and other parameters such as `steps_per_execution`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PhTHUYaD74vT" + }, + "outputs": [], + "source": [ + "with strategy.scope():\n", + " model = tf.keras.models.Sequential([tf.keras.layers.Dense(10)])\n", + "\n", + " model.compile(tf.keras.optimizers.legacy.SGD(), loss=\"mse\", steps_per_execution=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nWb_Ekm377YX" + }, + "source": [ + "### Callbacks and training\n", + "\n", + " \n", + "\n", + "Before you call Keras `Model.fit` for the actual training, prepare any needed [callbacks](https://www.tensorflow.org/guide/keras/train_and_evaluate) for common tasks, such as:\n", + "\n", + "- `tf.keras.callbacks.ModelCheckpoint`: saves the model at a certain frequency, such as after every epoch.\n", + "- `tf.keras.callbacks.BackupAndRestore`: provides fault tolerance by backing up the model and current epoch number, if the cluster experiences unavailability (such as abort or preemption). You can then restore the training state upon a restart from a job failure, and continue training from the beginning of the interrupted epoch.\n", + "- `tf.keras.callbacks.TensorBoard`: periodically writes model logs in summary files that can be visualized in the TensorBoard tool.\n", + "\n", + "Note: Due to performance considerations, custom callbacks cannot have batch level callbacks overridden when used with `ParameterServerStrategy`. Please modify your custom callbacks to make them epoch level calls, and adjust `steps_per_epoch` to a suitable value. In addition, `steps_per_epoch` is a required argument for `Model.fit` when used with `ParameterServerStrategy`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3ddUvUZk7_wm" + }, + "outputs": [], + "source": [ + "working_dir = \"/tmp/my_working_dir\"\n", + "log_dir = os.path.join(working_dir, \"log\")\n", + "ckpt_filepath = os.path.join(working_dir, \"ckpt\")\n", + "backup_dir = os.path.join(working_dir, \"backup\")\n", + "\n", + "callbacks = [\n", + " tf.keras.callbacks.TensorBoard(log_dir=log_dir),\n", + " tf.keras.callbacks.ModelCheckpoint(filepath=ckpt_filepath),\n", + " tf.keras.callbacks.BackupAndRestore(backup_dir=backup_dir),\n", + "]\n", + "\n", + "model.fit(dataset, epochs=5, steps_per_epoch=20, callbacks=callbacks)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uWgP1h2z8B3j" + }, + "source": [ + "### Direct usage with `ClusterCoordinator` (optional)\n", + "\n", + "Even if you choose the `Model.fit` training path, you can optionally instantiate a `tf.distribute.coordinator.ClusterCoordinator` object to schedule other functions you would like to be executed on the workers. Refer to the [Training with a custom training loop](#training_with_custom_training_loop) section for more details and examples." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GxypEyIthR0z" + }, + "source": [ + "## Training with a custom training loop\n", + "\n", + " \n", + "\n", + "Using custom training loops with `tf.distribute.Strategy` provides great flexibility to define training loops. With the `ParameterServerStrategy` defined above (as `strategy`), you will use a `tf.distribute.coordinator.ClusterCoordinator` to dispatch the execution of training steps to remote workers.\n", + "\n", + "Then, you will create a model, define a dataset, and define a step function, as you have done in the training loop with other `tf.distribute.Strategy`s. You can find more details in the [Custom training with tf.distribute.Strategy](custom_training.ipynb) tutorial.\n", + "\n", + "To ensure efficient dataset prefetching, use the recommended distributed dataset creation APIs mentioned in the [Dispatch training steps to remote workers](#dispatch_training_steps_to_remote_workers) section below. Also, make sure to call `Strategy.run` inside `worker_fn` to take full advantage of GPUs allocated to workers. The rest of the steps are the same for training with or without GPUs.\n", + "\n", + "Let’s create these components in the following steps:\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4QNkCtV8VivM" + }, + "source": [ + "### Set up the data\n", + "\n", + "First, write a function that creates a dataset.\n", + "\n", + "If you would like to preprocess the data with [Keras preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) or [Tensorflow Transform layers](https://www.tensorflow.org/tfx/tutorials/transform/simple), create these layers **outside the `dataset_fn`** and **under `Strategy.scope`**, like you would do for any other Keras layers. This is because the `dataset_fn` will be wrapped into a `tf.function` and then executed on each worker to generate the data pipeline.\n", + "\n", + "If you don't follow the above procedure, creating the layers might create Tensorflow states which will be lifted out of the `tf.function` to the coordinator. Thus, accessing them on workers would incur repetitive RPC calls between coordinator and workers, and cause significant slowdown.\n", + "\n", + "Placing the layers under `Strategy.scope` will instead create them on all workers. Then, you will apply the transformation inside the `dataset_fn` via `tf.data.Dataset.map`. Refer to _Data preprocessing_ in the [Distributed input](input.ipynb) tutorial for more information on data preprocessing with distributed input." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2GUwATssauus" + }, + "outputs": [], + "source": [ + "feature_vocab = [\n", + " \"avenger\", \"ironman\", \"batman\", \"hulk\", \"spiderman\", \"kingkong\", \"wonder_woman\"\n", + "]\n", + "label_vocab = [\"yes\", \"no\"]\n", + "\n", + "with strategy.scope():\n", + " feature_lookup_layer = tf.keras.layers.StringLookup(\n", + " vocabulary=feature_vocab,\n", + " mask_token=None)\n", + " label_lookup_layer = tf.keras.layers.StringLookup(\n", + " vocabulary=label_vocab,\n", + " num_oov_indices=0,\n", + " mask_token=None)\n", + "\n", + " raw_feature_input = tf.keras.layers.Input(\n", + " shape=(3,),\n", + " dtype=tf.string,\n", + " name=\"feature\")\n", + " feature_id_input = feature_lookup_layer(raw_feature_input)\n", + " feature_preprocess_stage = tf.keras.Model(\n", + " {\"features\": raw_feature_input},\n", + " feature_id_input)\n", + "\n", + " raw_label_input = tf.keras.layers.Input(\n", + " shape=(1,),\n", + " dtype=tf.string,\n", + " name=\"label\")\n", + " label_id_input = label_lookup_layer(raw_label_input)\n", + "\n", + " label_preprocess_stage = tf.keras.Model(\n", + " {\"label\": raw_label_input},\n", + " label_id_input)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Jgp8MX_7OR_A" + }, + "source": [ + "Generate toy examples in a dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "chIY4fFANaFH" + }, + "outputs": [], + "source": [ + "def feature_and_label_gen(num_examples=200):\n", + " examples = {\"features\": [], \"label\": []}\n", + " for _ in range(num_examples):\n", + " features = random.sample(feature_vocab, 3)\n", + " label = [\"yes\"] if \"avenger\" in features else [\"no\"]\n", + " examples[\"features\"].append(features)\n", + " examples[\"label\"].append(label)\n", + " return examples\n", + "\n", + "examples = feature_and_label_gen()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2AtZBya7OeyZ" + }, + "source": [ + "Then, create the training dataset wrapped in a `dataset_fn`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Gs0QYRZoNbvw" + }, + "outputs": [], + "source": [ + "def dataset_fn(_):\n", + " raw_dataset = tf.data.Dataset.from_tensor_slices(examples)\n", + "\n", + " train_dataset = raw_dataset.map(\n", + " lambda x: (\n", + " {\"features\": feature_preprocess_stage(x[\"features\"])},\n", + " label_preprocess_stage(x[\"label\"])\n", + " )).shuffle(200).batch(32).repeat()\n", + " return train_dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IT9PQexJiFtB" + }, + "source": [ + "### Build the model\n", + "\n", + "Next, create the model and other objects. Make sure to create all variables under `Strategy.scope`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Quxud1uEazeo" + }, + "outputs": [], + "source": [ + "# These variables created under the `Strategy.scope` will be placed on parameter\n", + "# servers in a round-robin fashion.\n", + "with strategy.scope():\n", + " # Create the model. The input needs to be compatible with Keras processing layers.\n", + " model_input = tf.keras.layers.Input(\n", + " shape=(3,), dtype=tf.int64, name=\"model_input\")\n", + "\n", + " emb_layer = tf.keras.layers.Embedding(\n", + " input_dim=len(feature_lookup_layer.get_vocabulary()), output_dim=16384)\n", + " emb_output = tf.reduce_mean(emb_layer(model_input), axis=1)\n", + " dense_output = tf.keras.layers.Dense(\n", + " units=1, activation=\"sigmoid\",\n", + " kernel_regularizer=tf.keras.regularizers.L2(1e-4),\n", + " )(emb_output)\n", + " model = tf.keras.Model({\"features\": model_input}, dense_output)\n", + "\n", + " optimizer = tf.keras.optimizers.legacy.RMSprop(learning_rate=0.1)\n", + " accuracy = tf.keras.metrics.Accuracy()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iyuxiqCQU50m" + }, + "source": [ + "Let's confirm that the use of `FixedShardsPartitioner` split all variables into two shards and that each shard was assigned to a different parameter server:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "04r1nO4WVDO1" + }, + "outputs": [], + "source": [ + "assert len(emb_layer.weights) == 2\n", + "assert emb_layer.weights[0].shape == (4, 16384)\n", + "assert emb_layer.weights[1].shape == (4, 16384)\n", + "\n", + "print(emb_layer.weights[0].device)\n", + "print(emb_layer.weights[1].device)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lWhfXZLRiHyM" + }, + "source": [ + "### Define the training step\n", + "Third, create the training step wrapped into a `tf.function`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aNNVo0bFa1K9" + }, + "outputs": [], + "source": [ + "@tf.function\n", + "def step_fn(iterator):\n", + "\n", + " def replica_fn(batch_data, labels):\n", + " with tf.GradientTape() as tape:\n", + " pred = model(batch_data, training=True)\n", + " per_example_loss = tf.keras.losses.BinaryCrossentropy(\n", + " reduction=tf.keras.losses.Reduction.NONE)(labels, pred)\n", + " loss = tf.nn.compute_average_loss(per_example_loss)\n", + " model_losses = model.losses\n", + " if model_losses:\n", + " loss += tf.nn.scale_regularization_loss(tf.add_n(model_losses))\n", + " gradients = tape.gradient(loss, model.trainable_variables)\n", + "\n", + " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", + "\n", + " actual_pred = tf.cast(tf.greater(pred, 0.5), tf.int64)\n", + " accuracy.update_state(labels, actual_pred)\n", + " return loss\n", + "\n", + " batch_data, labels = next(iterator)\n", + " losses = strategy.run(replica_fn, args=(batch_data, labels))\n", + " return strategy.reduce(tf.distribute.ReduceOp.SUM, losses, axis=None)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rvrYQUeYiLNy" + }, + "source": [ + "In the above training step function, calling `Strategy.run` and `Strategy.reduce` in the `step_fn` can support multiple GPUs per worker. If the workers have GPUs allocated, `Strategy.run` will distribute the datasets on multiple replicas (GPUs). Their parallel calls to `tf.nn.compute_average_loss()` compute the average of the loss across the replicas (GPUs) of one worker, independent of the total number of workers." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GPJ3PV_L2zAY" + }, + "source": [ + "### Dispatch training steps to remote workers\n", + " \n", + "\n", + "After all the computations are defined by `ParameterServerStrategy`, you will use the `tf.distribute.coordinator.ClusterCoordinator` class to create resources and distribute the training steps to remote workers.\n", + "\n", + "Let’s first create a `ClusterCoordinator` object and pass in the strategy object:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DpcMlH7Pa3DB" + }, + "outputs": [], + "source": [ + "coordinator = tf.distribute.coordinator.ClusterCoordinator(strategy)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-xRIgKxciOSe" + }, + "source": [ + "Then, create a per-worker dataset and an iterator using the `ClusterCoordinator.create_per_worker_dataset` API, which replicates the dataset to all workers. In the `per_worker_dataset_fn` below, wrapping the `dataset_fn` into `strategy.distribute_datasets_from_function` is recommended to allow efficient prefetching to GPUs seamlessly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "h9DCvTJTa4Q2" + }, + "outputs": [], + "source": [ + "@tf.function\n", + "def per_worker_dataset_fn():\n", + " return strategy.distribute_datasets_from_function(dataset_fn)\n", + "\n", + "per_worker_dataset = coordinator.create_per_worker_dataset(per_worker_dataset_fn)\n", + "per_worker_iterator = iter(per_worker_dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i2pnOx78iRwW" + }, + "source": [ + "The final step is to distribute the computation to remote workers using `ClusterCoordinator.schedule`:\n", + "\n", + "- The `schedule` method enqueues a `tf.function` and returns a future-like `RemoteValue` immediately. The queued functions will be dispatched to remote workers in background threads and the `RemoteValue` will be filled asynchronously.\n", + "- The `join` method (`ClusterCoordinator.join`) can be used to wait until all scheduled functions are executed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gmPvactfa6Eh" + }, + "outputs": [], + "source": [ + "num_epochs = 4\n", + "steps_per_epoch = 5\n", + "for i in range(num_epochs):\n", + " accuracy.reset_states()\n", + " for _ in range(steps_per_epoch):\n", + " coordinator.schedule(step_fn, args=(per_worker_iterator,))\n", + " # Wait at epoch boundaries.\n", + " coordinator.join()\n", + " print(\"Finished epoch %d, accuracy is %f.\" % (i, accuracy.result().numpy()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WBn-gn-OP3DR" + }, + "source": [ + "Here is how you can fetch the result of a `RemoteValue`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-15a2I_lQDO1" + }, + "outputs": [], + "source": [ + "loss = coordinator.schedule(step_fn, args=(per_worker_iterator,))\n", + "print(\"Final loss is %f\" % loss.fetch())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "htY4QKc9iXg9" + }, + "source": [ + "Alternatively, you can launch all steps and do something while waiting for\n", + "completion:\n", + "\n", + "```python\n", + "for _ in range(total_steps):\n", + " coordinator.schedule(step_fn, args=(per_worker_iterator,))\n", + "while not coordinator.done():\n", + " time.sleep(10)\n", + " # Do something like logging metrics or writing checkpoints.\n", + "```\n", + "\n", + "For the complete training and serving workflow for this particular example, please check out this [test](https://github.com/keras-team/keras/blob/master/keras/integration_test/parameter_server_keras_preprocessing_test.py).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kzNsj2GR3BGs" + }, + "source": [ + "### More about dataset creation\n", + "\n", + "The dataset in the above code is created using the `ClusterCoordinator.create_per_worker_dataset` API. It creates one dataset per worker and returns a container object. You can call the `iter` method on it to create a per-worker iterator. The per-worker iterator contains one iterator per worker and the corresponding slice of a worker will be substituted in the input argument of the function passed to the `ClusterCoordinator.schedule` method before the function is executed on a particular worker.\n", + "\n", + "The `ClusterCoordinator.schedule` method assumes workers are equivalent and thus assumes the datasets on different workers are the same (except that they may be shuffled differently). Because of this, it is also recommended to repeat datasets, and schedule a finite number of steps instead of relying on receiving an `OutOfRangeError` from a dataset.\n", + "\n", + "Another important note is that `tf.data` datasets don’t support implicit serialization and deserialization across task boundaries. So it is important to create the whole dataset inside the function passed to `ClusterCoordinator.create_per_worker_dataset`. The `create_per_worker_dataset` API can also directly take a `tf.data.Dataset` or `tf.distribute.DistributedDataset` as input." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LcfdI_M83lAM" + }, + "source": [ + "## Evaluation\n", + "\n", + "The two main approaches to performing evaluation with `tf.distribute.ParameterServerStrategy` training are inline evaluation and sidecar evaluation. Each has its own pros and cons as described below. The inline evaluation method is recommended if you don't have a preference. For users using `Model.fit`, `Model.evaluate` uses inline (distributed) evaluation under the hood." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oiG8EhcY3gA1" + }, + "source": [ + "### Inline evaluation\n", + "\n", + "In this method, the coordinator alternates between training and evaluation, and thus it is called _inline evaluation_.\n", + "\n", + "There are several benefits of inline evaluation. For example:\n", + "\n", + "- It can support large evaluation models and evaluation datasets that a single task cannot hold.\n", + "- The evaluation results can be used to make decisions for training the next epoch, for example, whether to stop training early.\n", + "\n", + "There are two ways to implement inline evaluation: direct evaluation and distributed evaluation.\n", + "\n", + "- **Direct evaluation**: For small models and evaluation datasets, the coordinator can run evaluation directly on the distributed model with the evaluation dataset on the coordinator:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WakiAakoaHVn" + }, + "outputs": [], + "source": [ + "eval_dataset = tf.data.Dataset.from_tensor_slices(\n", + " feature_and_label_gen(num_examples=16)).map(\n", + " lambda x: (\n", + " {\"features\": feature_preprocess_stage(x[\"features\"])},\n", + " label_preprocess_stage(x[\"label\"])\n", + " )).batch(8)\n", + "\n", + "eval_accuracy = tf.keras.metrics.Accuracy()\n", + "\n", + "for batch_data, labels in eval_dataset:\n", + " pred = model(batch_data, training=False)\n", + " actual_pred = tf.cast(tf.greater(pred, 0.5), tf.int64)\n", + " eval_accuracy.update_state(labels, actual_pred)\n", + "\n", + "print(\"Evaluation accuracy: %f\" % eval_accuracy.result())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MKGHbdI7aGoJ" + }, + "source": [ + "- **Distributed evaluation**: For large models or datasets that are infeasible to run directly on the coordinator, the coordinator task can distribute evaluation tasks to the workers via the `ClusterCoordinator.schedule`/`ClusterCoordinator.join` methods:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XcHNHJpDgEvK" + }, + "outputs": [], + "source": [ + "with strategy.scope():\n", + " # Define the eval metric on parameter servers.\n", + " eval_accuracy = tf.keras.metrics.Accuracy()\n", + "\n", + "@tf.function\n", + "def eval_step(iterator):\n", + " def replica_fn(batch_data, labels):\n", + " pred = model(batch_data, training=False)\n", + " actual_pred = tf.cast(tf.greater(pred, 0.5), tf.int64)\n", + " eval_accuracy.update_state(labels, actual_pred)\n", + " batch_data, labels = next(iterator)\n", + " strategy.run(replica_fn, args=(batch_data, labels))\n", + "\n", + "def eval_dataset_fn():\n", + " return tf.data.Dataset.from_tensor_slices(\n", + " feature_and_label_gen(num_examples=16)).map(\n", + " lambda x: (\n", + " {\"features\": feature_preprocess_stage(x[\"features\"])},\n", + " label_preprocess_stage(x[\"label\"])\n", + " )).shuffle(16).repeat().batch(8)\n", + "\n", + "per_worker_eval_dataset = coordinator.create_per_worker_dataset(eval_dataset_fn)\n", + "per_worker_eval_iterator = iter(per_worker_eval_dataset)\n", + "\n", + "eval_steps_per_epoch = 2\n", + "for _ in range(eval_steps_per_epoch):\n", + " coordinator.schedule(eval_step, args=(per_worker_eval_iterator,))\n", + "coordinator.join()\n", + "print(\"Evaluation accuracy: %f\" % eval_accuracy.result())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cKrQktZX5z7a" + }, + "source": [ + "#### Enabling exactly-once evaluation\n", + "\n", + "\n", + "The `schedule` and `join` methods of `tf.distribute.coordinator.ClusterCoordinator` don’t support visitation guarantees or exactly-once semantics by default. In other words, in the above example there is no guarantee that all evaluation examples in a dataset will be evaluated exactly once; some may not be visited and some may be evaluated multiple times.\n", + "\n", + "Exactly-once evaluation may be preferred to reduce the variance of evaluation across epochs, and improve model selection done via early stopping, hyperparameter tuning, or other methods. There are different ways to enable exactly-once evaluation:\n", + "\n", + "- With a `Model.fit/.evaluate` workflow, it can be enabled by adding an argument to `Model.compile`. Refer to docs for the `pss_evaluation_shards` argument.\n", + "- The `tf.data` service API can be used to provide exactly-once visitation for evaluation when using `ParameterServerStrategy` (refer to the _Dynamic Sharding_ section of the `tf.data.experimental.service` API documentation).\n", + "- [Sidecar evaluation](#sidecar_evaluation) provides exactly-once evaluation by default, since the evaluation happens on a single machine. However this can be much slower than performing evaluation distributed across many workers.\n", + "\n", + "The first option, using `Model.compile`, is the suggested solution for most users.\n", + "\n", + "Exactly-once evaluation has some limitations:\n", + "\n", + "- It is not supported to write a custom distributed evaluation loop with an exactly-once visitation guarantee. File a GitHub issue if you need support for this.\n", + "- It cannot automatically handle computation of metrics that use the `Layer.add_metric` API. These should be excluded from evaluation, or reworked into `Metric` objects." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H40X-9Gs3i7_" + }, + "source": [ + "### Sidecar evaluation\n", + "\n", + "\n", + "Another method for defining and running an evaluation loop in `tf.distribute.ParameterServerStrategy` training is called _sidecar evaluation_, in which you create a dedicated evaluator task that repeatedly reads checkpoints and runs evaluation on the latest checkpoint (refer to [this guide](../../guide/checkpoint.ipynb) for more details on checkpointing). The coordinator and worker tasks do not spend any time on evaluation, so for a fixed number of iterations the overall training time should be shorter than using other evaluation methods. However, it requires an additional evaluator task and periodic checkpointing to trigger evaluation." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HonyjnXK9-ys" + }, + "source": [ + "To write an evaluation loop for sidecar evaluation, you have two\n", + "options:\n", + "\n", + "1. Use the `tf.keras.utils.SidecarEvaluator` API.\n", + "2. Create a custom evaluation loop.\n", + "\n", + "Refer to the `tf.keras.utils.SidecarEvaluator` API documentation for more details on option 1." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U_c0EiwB88OG" + }, + "source": [ + "Sidecar evaluation is supported only with a single task. This means:\n", + "\n", + "* It is guaranteed that each example is evaluated once. In the event the\n", + " evaluator is preempted or restarted, it simply restarts the\n", + " evaluation loop from the latest checkpoint, and the partial evaluation\n", + " progress made before the restart is discarded.\n", + "\n", + "* However, running evaluation on a single task implies that a full evaluation\n", + " can possibly take a long time.\n", + "\n", + "* If the size of the model is too large to fit into an evaluator's memory,\n", + " single sidecar evaluation is not applicable." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VNJoWVc797B1" + }, + "source": [ + "Another caveat is that the `tf.keras.utils.SidecarEvaluator` implementation, and the custom\n", + "evaluation loop below, may skip some checkpoints because it always picks up the\n", + "latest checkpoint available, and during an evaluation epoch, multiple\n", + "checkpoints can be produced from the training cluster. You can write a custom\n", + "evaluation loop that evaluates every checkpoint, but it is not covered in this\n", + "tutorial. On the other hand, it may sit idle if checkpoints are produced less\n", + "frequently than how long it takes to run evaluation." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G5jopxBd85Ji" + }, + "source": [ + "A custom evaluation loop provides more control over the details, such as choosing which checkpoint to evaluate, or providing any additional logic to run along with evaluation. The following is a possible custom sidecar evaluation loop:\n", + "\n", + "```python\n", + "checkpoint_dir = ...\n", + "eval_model = ...\n", + "eval_data = ...\n", + "checkpoint = tf.train.Checkpoint(model=eval_model)\n", + "\n", + "for latest_checkpoint in tf.train.checkpoints_iterator(\n", + " checkpoint_dir):\n", + " try:\n", + " checkpoint.restore(latest_checkpoint).expect_partial()\n", + " except (tf.errors.OpError,) as e:\n", + " # checkpoint may be deleted by training when it is about to read it.\n", + " continue\n", + "\n", + " # Optionally add callbacks to write summaries.\n", + " eval_model.evaluate(eval_data)\n", + "\n", + " # Evaluation finishes when it has evaluated the last epoch.\n", + " if latest_checkpoint.endswith('-{}'.format(train_epochs)):\n", + " break\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9TkNbtpPhFRQ" + }, + "source": [ + "## Clusters in the real world\n", + "\n", + "\n", + "Note: this section is not necessary for running the tutorial code in this page.\n", + "\n", + "In a real production environment, you will run all tasks in different processes on different machines. The simplest way to configure cluster information on each task is to set `\"TF_CONFIG\"` environment variables and use a `tf.distribute.cluster_resolver.TFConfigClusterResolver` to parse `\"TF_CONFIG\"`.\n", + "\n", + "For a general description of `\"TF_CONFIG\"` environment variables, refer to \"Setting up the `TF_CONFIG` environment variable\" in the [Distributed training](../../guide/distributed_training.ipynb) guide.\n", + "\n", + "If you start your training tasks using Kubernetes or other configuration templates, likely, these templates have already set `“TF_CONFIG\"` for you." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "n7AK9SJGt3tQ" + }, + "source": [ + "### Set the `\"TF_CONFIG\"` environment variable\n", + "\n", + "Suppose you have 3 workers and 2 parameter servers. Then the `\"TF_CONFIG\"` of worker 1 can be:\n", + "\n", + "```python\n", + "os.environ[\"TF_CONFIG\"] = json.dumps({\n", + " \"cluster\": {\n", + " \"worker\": [\"host1:port\", \"host2:port\", \"host3:port\"],\n", + " \"ps\": [\"host4:port\", \"host5:port\"],\n", + " \"chief\": [\"host6:port\"]\n", + " },\n", + " \"task\": {\"type\": \"worker\", \"index\": 1}\n", + "})\n", + "```\n", + "\n", + "The `\"TF_CONFIG\"` of the evaluator can be:\n", + "\n", + "```python\n", + "os.environ[\"TF_CONFIG\"] = json.dumps({\n", + " \"cluster\": {\n", + " \"evaluator\": [\"host7:port\"]\n", + " },\n", + " \"task\": {\"type\": \"evaluator\", \"index\": 0}\n", + "})\n", + "```\n", + "\n", + "The `\"cluster\"` part in the above `\"TF_CONFIG\"` string for the evaluator is optional." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fZRjMS0pt1LM" + }, + "source": [ + "### If you use the same binary for all tasks\n", + "\n", + "If you prefer to run all these tasks using a single binary, you will need to let your program branch into different roles at the very beginning:\n", + "\n", + "```python\n", + "cluster_resolver = tf.distribute.cluster_resolver.TFConfigClusterResolver()\n", + "if cluster_resolver.task_type in (\"worker\", \"ps\"):\n", + " # Start a TensorFlow server and wait.\n", + "elif cluster_resolver.task_type == \"evaluator\":\n", + " # Run sidecar evaluation\n", + "else:\n", + " # Run the coordinator.\n", + "```\n", + "\n", + "The following code starts a TensorFlow server and waits, useful for the `\"worker\"` and `\"ps\"` roles:\n", + "\n", + "```python\n", + "# Set the environment variable to allow reporting worker and ps failure to the\n", + "# coordinator. This is a workaround and won't be necessary in the future.\n", + "os.environ[\"GRPC_FAIL_FAST\"] = \"use_caller\"\n", + "\n", + "server = tf.distribute.Server(\n", + " cluster_resolver.cluster_spec(),\n", + " job_name=cluster_resolver.task_type,\n", + " task_index=cluster_resolver.task_id,\n", + " protocol=cluster_resolver.rpc_layer or \"grpc\",\n", + " start=True)\n", + "server.join()\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZWdYfK593eOL" + }, + "source": [ + "## Handling task failure" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bl9eK5r13cOv" + }, + "source": [ + "### Worker failure\n", + "\n", + "Both the `tf.distribute.coordinator.ClusterCoordinator` custom training loop and `Model.fit` approaches provide built-in fault tolerance for worker failure. Upon worker recovery, the `ClusterCoordinator` invokes dataset re-creation on the workers." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aP0OHZ1-Ne-B" + }, + "source": [ + "### Parameter server or coordinator failure\n", + "\n", + "However, when the coordinator sees a parameter server error, it will raise an `UnavailableError` or `AbortedError` immediately. You can restart the coordinator in this case. The coordinator itself can also become unavailable. Therefore, certain tooling is recommended in order to not lose the training progress:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f7m7Itoz8lsI" + }, + "source": [ + "- For `Model.fit`, you should use a `BackupAndRestore` callback, which handles the progress saving and restoration automatically. See [Callbacks and training](#callbacks-and-training) section above for an example." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-XlLyJp53Z8A" + }, + "source": [ + "- For a custom training loop, you should checkpoint the model variables periodically and load model variables from a checkpoint, if any, before training starts. The training progress can be inferred approximately from `optimizer.iterations` if an optimizer is checkpointed:\n", + "\n", + "```python\n", + "checkpoint_manager = tf.train.CheckpointManager(\n", + " tf.train.Checkpoint(model=model, optimizer=optimizer),\n", + " checkpoint_dir,\n", + " max_to_keep=3)\n", + "if checkpoint_manager.latest_checkpoint:\n", + " checkpoint = checkpoint_manager.checkpoint\n", + " checkpoint.restore(\n", + " checkpoint_manager.latest_checkpoint).assert_existing_objects_matched()\n", + "\n", + "global_steps = int(optimizer.iterations.numpy())\n", + "starting_epoch = global_steps // steps_per_epoch\n", + "\n", + "for _ in range(starting_epoch, num_epochs):\n", + " for _ in range(steps_per_epoch):\n", + " coordinator.schedule(step_fn, args=(per_worker_iterator,))\n", + " coordinator.join()\n", + " checkpoint_manager.save()\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PlN1P7C53XK9" + }, + "source": [ + "### Fetching a `RemoteValue`\n", + "\n", + "Fetching a `RemoteValue` is guaranteed to succeed if a function is executed successfully. This is because currently the return value is immediately copied to the coordinator after a function is executed. If there is any worker failure during the copy, the function will be retried on another available worker. Therefore, if you want to optimize for performance, you can schedule functions without a return value." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iZcR_xNZ3UdU" + }, + "source": [ + "## Error reporting\n", + "\n", + "Once the coordinator sees an error such as `UnavailableError` from parameter servers or other application errors such as an `InvalidArgument` from `tf.debugging.check_numerics`, it will cancel all pending and queued functions before raising the error. Fetching their corresponding `RemoteValue`s will raise a `CancelledError`.\n", + "\n", + "After an error is raised, the coordinator will not raise the same error or any error from cancelled functions." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QfhbXH-j3NVw" + }, + "source": [ + "## Performance improvement\n", + "\n", + "There are several possible reasons you may experience performance issues when you train with `tf.distribute.ParameterServerStrategy` and `tf.distribute.coordinator.ClusterCoordinator`.\n", + "\n", + "One common reason is that the parameter servers have unbalanced load and some heavily-loaded parameter servers have reached capacity. There can also be multiple root causes. Some simple methods to mitigate this issue are to:\n", + "\n", + "1. Shard your large model variables via specifying a `variable_partitioner` when constructing a `ParameterServerStrategy`.\n", + "2. Avoid creating a hotspot variable that is required by all parameter servers in a single step, by both:\n", + "\n", + " 1) Using a constant learning rate or subclass `tf.keras.optimizers.schedules.LearningRateSchedule` in optimizers. This is because the default behavior is that the learning rate will become a variable placed on a particular parameter server, and requested by all other parameter servers in each step); and\n", + "\n", + " 2) Using a `tf.keras.optimizers.legacy.Optimizer` (the standard `tf.keras.optimizers.Optimizer`s could still lead to hotspot variables).\n", + "3. Shuffle your large vocabularies before passing them to Keras preprocessing layers.\n", + "\n", + "Another possible reason for performance issues is the coordinator. The implementation of `schedule`/`join` is Python-based and thus may have threading overhead. Also, the latency between the coordinator and the workers can be large. If this is the case:\n", + "\n", + "- For `Model.fit`, you can set the `steps_per_execution` argument provided at `Model.compile` to a value larger than 1.\n", + "\n", + "- For a custom training loop, you can pack multiple steps into a single `tf.function`:\n", + "\n", + "```python\n", + "steps_per_invocation = 10\n", + "\n", + "@tf.function\n", + "def step_fn(iterator):\n", + " for _ in range(steps_per_invocation):\n", + " features, labels = next(iterator)\n", + " def replica_fn(features, labels):\n", + " ...\n", + "\n", + " strategy.run(replica_fn, args=(features, labels))\n", + "```\n", + "\n", + "As the library is optimized further, hopefully most users won't have to manually pack steps in the future.\n", + "\n", + "In addition, a small trick for performance improvement is to schedule functions without a return value as explained in the [handling task failure section](#handling_task_failure) above." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "chu5F7M_JmVk" + }, + "source": [ + "## Known limitations\n", + "\n", + " \n", + "\n", + "Most of the known limitations are already covered in the above sections. This section provides a summary.\n", + "\n", + "### `ParameterServerStrategy` general\n", + "\n", + "- `os.environment[\"grpc_fail_fast\"]=\"use_caller\"` is needed on every task including the coordinator, to make fault tolerance work properly.\n", + "- Synchronous parameter server training is not supported.\n", + "- It is usually necessary to pack multiple steps into a single function to achieve optimal performance.\n", + "- It is not supported to load a saved_model via `tf.saved_model.load` containing sharded variables. Note loading such a saved_model using TensorFlow Serving is expected to work (refer to the [serving tutorial](https://www.tensorflow.org/tfx/tutorials/serving/rest_simple) for details).\n", + "- It is not supported to recover from parameter server failure without restarting the coordinator task.\n", + "- Creation of `tf.lookup.StaticHashTable`, commonly employed by some Keras preprocessing layers, such as `tf.keras.layers.IntegerLookup`, `tf.keras.layers.StringLookup`, and `tf.keras.layers.TextVectorization`, should be placed under `Strategy.scope`. Otherwise, resources will be placed on the coordinator, and lookup RPCs from workers to the coordinator incur performance implications.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2MKBF0RPSvzB" + }, + "source": [ + "### `Model.fit` specifics\n", + "\n", + "- `steps_per_epoch` argument is required in `Model.fit`. You can select a value that provides appropriate intervals in an epoch.\n", + "- `ParameterServerStrategy` does not have support for custom callbacks that have batch-level calls for performance reasons. You should convert those calls into epoch-level calls with suitably picked `steps_per_epoch`, so that they are called every `steps_per_epoch` number of steps. Built-in callbacks are not affected: their batch-level calls have been modified to be performant. Supporting batch-level calls for `ParameterServerStrategy` is being planned.\n", + "- For the same reason, unlike other strategies, progress bars and metrics are logged only at epoch boundaries.\n", + "- `run_eagerly` is not supported.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wvY-mg35Sx5L" + }, + "source": [ + "### Custom training loop specifics\n", + "\n", + "- `ClusterCoordinator.schedule` doesn't support visitation guarantees for a dataset in general, although a visitation guarantee for evaluation is possible through `Model.fit/.evaluate`. See [Enabling exactly-once evaluation](#exactly_once_evaluation).\n", + "- When `ClusterCoordinator.create_per_worker_dataset` is used with a callable as input, the whole dataset must be created inside the function passed to it.\n", + "- `tf.data.Options` is ignored in a dataset created by `ClusterCoordinator.create_per_worker_dataset`." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "parameter_server_training.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/distribute/save_and_load.ipynb b/site/en/tutorials/distribute/save_and_load.ipynb index ee5a16090ff..c53a9b8bf0b 100644 --- a/site/en/tutorials/distribute/save_and_load.ipynb +++ b/site/en/tutorials/distribute/save_and_load.ipynb @@ -3,21 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1l8bWGmIJuQa" }, "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" + "##### Copyright 2019 The TensorFlow Authors.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "CPSnXS88KFEo" }, "outputs": [], @@ -38,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "89xNCIO5hiCj" }, "source": [ @@ -48,43 +43,45 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9Ejs4QVxIdAm" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/distribute/save_and_load\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/distribute/save_and_load.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/distribute/save_and_load.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/distribute/save_and_load.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", + "\n", + " \n", + " \n", + " \n", + " \n", "\n", - "\u003c/table\u003e" + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "A0lG6qgThxAS" }, "source": [ "## Overview\n", "\n", - "It's common to save and load a model during training. There are two sets of APIs for saving and loading a keras model: a high-level API, and a low-level API. This tutorial demonstrates how you can use the SavedModel APIs when using `tf.distribute.Strategy`. To learn about SavedModel and serialization in general, please read the [saved model guide](../../guide/saved_model.ipynb), and the [Keras model serialization guide](../../guide/keras/save_and_serialize.ipynb). Let's start with a simple example: " + "This tutorial demonstrates how you can save and load models in a SavedModel format with `tf.distribute.Strategy` during or after training. There are two kinds of APIs for saving and loading a Keras model: high-level (`tf.keras.Model.save` and `tf.keras.models.load_model`) and low-level (`tf.saved_model.save` and `tf.saved_model.load`).\n", + "\n", + "To learn about SavedModel and serialization in general, please read the [saved model guide](../../guide/saved_model.ipynb), and the [Keras model serialization guide](https://www.tensorflow.org/guide/keras/save_and_serialize). Let's start with a simple example.\n", + "\n", + "Caution: TensorFlow models are code and it is important to be careful with untrusted code. Learn more in [Using TensorFlow securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md).\n", + "\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FITHltVKQ4eZ" }, "source": [ @@ -93,43 +90,30 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "RWG5HchAiOrZ" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow_datasets as tfds\n", "\n", - "import tensorflow as tf\n", - "tfds.disable_progress_bar()" + "import tensorflow as tf\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qqapWj98ptNV" }, "source": [ - "Prepare the data and model using `tf.distribute.Strategy`:" + "Load and prepare the data with TensorFlow Datasets and `tf.data`, and create the model using `tf.distribute.MirroredStrategy`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "yrYiAf_ziRyw" }, "outputs": [], @@ -137,7 +121,7 @@ "mirrored_strategy = tf.distribute.MirroredStrategy()\n", "\n", "def get_data():\n", - " datasets, ds_info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n", + " datasets = tfds.load(name='mnist', as_supervised=True)\n", " mnist_train, mnist_test = datasets['train'], datasets['test']\n", "\n", " BUFFER_SIZE = 10000\n", @@ -163,31 +147,28 @@ " tf.keras.layers.MaxPooling2D(),\n", " tf.keras.layers.Flatten(),\n", " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", + " tf.keras.layers.Dense(10)\n", " ])\n", "\n", - " model.compile(loss='sparse_categorical_crossentropy',\n", + " model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])\n", + " metrics=[tf.metrics.SparseCategoricalAccuracy()])\n", " return model" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qmU4Y3feS9Na" }, "source": [ - "Train the model: " + "Train the model with `tf.keras.Model.fit`: " ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "zmGurbJmS_vN" }, "outputs": [], @@ -200,58 +181,51 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "L01wjgvRizHS" }, "source": [ "## Save and load the model\n", "\n", - "Now that you have a simple model to work with, let's take a look at the saving/loading APIs. \n", - "There are two sets of APIs available:\n", + "Now that you have a simple model to work with, let's explore the saving/loading APIs. \n", + "There are two kinds of APIs available:\n", "\n", - "* High level keras `model.save` and `tf.keras.models.load_model`\n", - "* Low level `tf.saved_model.save` and `tf.saved_model.load`\n", - "\n" + "* High-level (Keras): `Model.save` and `tf.keras.models.load_model` (`.keras` zip archive format)\n", + "* Low-level: `tf.saved_model.save` and `tf.saved_model.load` (TF SavedModel format)\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FX_IF2F1tvFs" }, "source": [ - "### The Keras APIs" + "### The Keras API" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "O8xfceg4Z3H_" }, "source": [ - "Here is an example of saving and loading a model with the Keras APIs:" + "Here is an example of saving and loading a model with the Keras API:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "LYOStjV5knTQ" }, "outputs": [], "source": [ - "keras_model_path = \"/tmp/keras_save\"\n", - "model.save(keras_model_path) # save() should be called out of strategy scope" + "keras_model_path = '/tmp/keras_save.keras'\n", + "model.save(keras_model_path)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "yvQIdQp3zNMp" }, "source": [ @@ -260,10 +234,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WrXAAVtrzRgv" }, "outputs": [], @@ -275,28 +247,23 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gYAnskzorda-" }, "source": [ - "After restoring the model, you can continue training on it, even without needing to call `compile()` again, since it is already compiled before saving. The model is saved in the TensorFlow's standard `SavedModel` proto format. For more information, please refer to [the guide to `saved_model` format](../../guide/saved_model.ipynb).\n", - "\n", - "It is important to only call the `model.save()` method out of the scope of `tf.distribute.strategy`. Calling it within the scope is not supported.\n", + "After restoring the model, you can continue training on it, even without needing to call `Model.compile` again, since it was already compiled before saving. The model is saved a Keras zip archive format, marked by the `.keras` extension. For more information, please refer to [the guide on Keras saving](https://www.tensorflow.org/guide/keras/save_and_serialize).\n", "\n", - "Now to load the model and train it using a `tf.distribute.Strategy`:" + "Now, restore the model and train it using a `tf.distribute.Strategy`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wROPrJaAqBQz" }, "outputs": [], "source": [ - "another_strategy = tf.distribute.OneDeviceStrategy(\"/cpu:0\")\n", + "another_strategy = tf.distribute.OneDeviceStrategy('/cpu:0')\n", "with another_strategy.scope():\n", " restored_keras_model_ds = tf.keras.models.load_model(keras_model_path)\n", " restored_keras_model_ds.fit(train_dataset, epochs=2)" @@ -305,69 +272,61 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PdiiPmL5tQk5" }, "source": [ - "As you can see, loading works as expected with `tf.distribute.Strategy`. The strategy used here does not have to be the same strategy used before saving. " + "As the `Model.fit` output shows, loading works as expected with `tf.distribute.Strategy`. The strategy used here does not have to be the same strategy used before saving. " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3CrXIbmFt0f6" }, "source": [ - "### The `tf.saved_model` APIs" + "### The `tf.saved_model` API" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HtGzPp6et4Em" }, "source": [ - "Now let's take a look at the lower level APIs. Saving the model is similar to the keras API:" + "Saving the model with lower-level API is similar to the Keras API:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4y6T31APuCqK" }, "outputs": [], "source": [ "model = get_model() # get a fresh model\n", - "saved_model_path = \"/tmp/tf_save\"\n", + "saved_model_path = '/tmp/tf_save'\n", "tf.saved_model.save(model, saved_model_path)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "q1QNRYcwuRll" }, "source": [ - "Loading can be done with `tf.saved_model.load()`. However, since it is an API that is on the lower level (and hence has a wider range of use cases), it does not return a Keras model. Instead, it returns an object that contain functions that can be used to do inference. For example:" + "Loading can be done with `tf.saved_model.load`. However, since it is a lower-level API (and hence has a wider range of use cases), it does not return a Keras model. Instead, it returns an object that contain functions that can be used to do inference. For example:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "aaEKqBSPwAuM" }, "outputs": [], "source": [ - "DEFAULT_FUNCTION_KEY = \"serving_default\"\n", + "DEFAULT_FUNCTION_KEY = 'serving_default'\n", "loaded = tf.saved_model.load(saved_model_path)\n", "inference_func = loaded.signatures[DEFAULT_FUNCTION_KEY]" ] @@ -375,19 +334,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "x65l7AaHUZCA" }, "source": [ - "The loaded object may contain multiple functions, each associated with a key. The `\"serving_default\"` is the default key for the inference function with a saved Keras model. To do an inference with this function: " + "The loaded object may contain multiple functions, each associated with a key. The `\"serving_default\"` key is the default key for the inference function with a saved Keras model. To do inference with this function: " ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "5Ore5q8-UjW1" }, "outputs": [], @@ -400,7 +356,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "osB1LY8WwUJZ" }, "source": [ @@ -409,10 +364,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iDYvu12zYTmT" }, "outputs": [], @@ -427,26 +380,24 @@ "\n", " # Calling the function in a distributed manner\n", " for batch in dist_predict_dataset:\n", - " another_strategy.experimental_run_v2(inference_func, \n", - " args=(batch,))" + " result = another_strategy.run(inference_func, args=(batch,))\n", + " print(result)\n", + " break" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hWGSukoyw3fF" }, "source": [ - "Calling the restored function is just a forward pass on the saved model (predict). What if yout want to continue training the loaded function? Or embed the loaded function into a bigger model? A common practice is to wrap this loaded object to a Keras layer to achieve this. Luckily, [TF Hub](https://www.tensorflow.org/hub) has [hub.KerasLayer](https://github.com/tensorflow/hub/blob/master/tensorflow_hub/keras_layer.py) for this purpose, shown here:" + "Calling the restored function is just a forward pass on the saved model (`tf.keras.Model.predict`). What if you want to continue training the loaded function? Or what if you need to embed the loaded function into a bigger model? A common practice is to wrap this loaded object into a Keras layer to achieve this. Luckily, [TF Hub](https://www.tensorflow.org/hub) has [`hub.KerasLayer`](https://github.com/tensorflow/hub/blob/master/tensorflow_hub/keras_layer.py) for this purpose, shown here:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "clfk3hQoyKu6" }, "outputs": [], @@ -465,26 +416,24 @@ " loaded = tf.saved_model.load(saved_model_path)\n", " model = build_model(loaded)\n", "\n", - " model.compile(loss='sparse_categorical_crossentropy',\n", + " model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])\n", + " metrics=[tf.metrics.SparseCategoricalAccuracy()])\n", " model.fit(train_dataset, epochs=2)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Oe1z_OtSJlu2" }, "source": [ - "As you can see, `hub.KerasLayer` wraps the result loaded back from `tf.saved_model.load()` into a Keras layer that can be used to build another model. This is very useful for transfer learning. " + "In the above example, Tensorflow Hub's `hub.KerasLayer` wraps the result loaded back from `tf.saved_model.load` into a Keras layer that is used to build another model. This is very useful for transfer learning. " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "KFDOZpK5Wa3W" }, "source": [ @@ -494,73 +443,103 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GC6GQ9HDLxD6" }, "source": [ - "For saving, if you are working with a keras model, it is almost always recommended to use the Keras's `model.save()` API. If what you are saving is not a Keras model, then the lower level API is your only choice. \n", + "For saving, if you are working with a Keras model, use the Keras `Model.save` API unless you need the additional control allowed by the low-level API. If what you are saving is not a Keras model, then the lower-level API, `tf.saved_model.save`, is your only choice. \n", "\n", - "For loading, which API you use depends on what you want to get from the loading API. If you cannot (or do not want to) get a Keras model, then use `tf.saved_model.load()`. Otherwise, use `tf.keras.models.load_model()`. Note that you can get a Keras model back only if you saved a Keras model. \n", + "For loading, your API choice depends on what you want to get from the model loading API. If you cannot (or do not want to) get a Keras model, then use `tf.saved_model.load`. Otherwise, use `tf.keras.models.load_model`. Note that you can get a Keras model back only if you saved a Keras model. \n", "\n", - "It is possible to mix and match the APIs. You can save a Keras model with `model.save`, and load a non-Keras model with the low-level API, `tf.saved_model.load`. " + "It is possible to mix and match the APIs. You can save a Keras model with `Model.save`, and load a non-Keras model with the low-level API, `tf.saved_model.load`. " ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Ktwg2GwnXE8v" }, "outputs": [], "source": [ "model = get_model()\n", "\n", - "# Saving the model using Keras's save() API\n", - "model.save(keras_model_path) \n", + "# Saving the model using Keras `Model.save`\n", + "model.save(saved_model_path)\n", "\n", "another_strategy = tf.distribute.MirroredStrategy()\n", - "# Loading the model using lower level API\n", + "# Loading the model using the lower-level API\n", "with another_strategy.scope():\n", - " loaded = tf.saved_model.load(keras_model_path)" + " loaded = tf.saved_model.load(saved_model_path)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "hJTWOnC9iuA3" + "id": "0Z7lSj8nZiW5" }, "source": [ - "### Caveats" + "### Saving/Loading from a local device" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Tzog2ti7YYgy" + "id": "NVAjWcosZodw" }, "source": [ - "A special case is when you have a Keras model that does not have well-defined inputs. For example, a Sequential model can be created without any input shapes (`Sequential([Dense(3), ...]`). Subclassed models also do not have well-defined inputs after initialization. In this case, you should stick with the lower level APIs on both saving and loading, otherwise you will get an error. \n", + "When saving and loading from a local I/O device while training on remote devices—for example, when using a Cloud TPU—you must use the option `experimental_io_device` in `tf.saved_model.SaveOptions` and `tf.saved_model.LoadOptions` to set the I/O device to `localhost`. For example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jFcuzsI94bNA" + }, + "outputs": [], + "source": [ + "model = get_model()\n", "\n", - "To check if your model has well-defined inputs, just check if `model.inputs` is `None`. If it is not `None`, you are all good. Input shapes are automatically defined when the model is used in `.fit`, `.evaluate`, `.predict`, or when calling the model (`model(inputs)`). \n", + "# Saving the model to a path on localhost.\n", + "saved_model_path = '/tmp/tf_save'\n", + "save_options = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n", + "model.save(saved_model_path, options=save_options)\n", "\n", - "Here is an example:" + "# Loading the model from a path on localhost.\n", + "another_strategy = tf.distribute.MirroredStrategy()\n", + "with another_strategy.scope():\n", + " load_options = tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')\n", + " loaded = tf.keras.models.load_model(saved_model_path, options=load_options)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hJTWOnC9iuA3" + }, + "source": [ + "### Caveats" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2cCSZrD7VCxe" + }, + "source": [ + "One special case is when you create Keras models in certain ways, and then save them before training. For example:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gurSIbDFjOBc" }, "outputs": [], "source": [ "class SubclassedModel(tf.keras.Model):\n", + " \"\"\"Example model defined by subclassing `tf.keras.Model`.\"\"\"\n", "\n", " output_name = 'output_layer'\n", "\n", @@ -573,8 +552,89 @@ " return self._dense_layer(inputs)\n", "\n", "my_model = SubclassedModel()\n", - "# my_model.save(keras_model_path) # ERROR! \n", - "tf.saved_model.save(my_model, saved_model_path)" + "try:\n", + " my_model.save(saved_model_path)\n", + "except ValueError as e:\n", + " print(f'{type(e).__name__}: ', *e.args)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D4qMyXFDSPDO" + }, + "source": [ + "A SavedModel saves the `tf.types.experimental.ConcreteFunction` objects generated when you trace a `tf.function` (check _When is a Function tracing?_ in the [Introduction to graphs and tf.function](../../guide/intro_to_graphs.ipynb) guide to learn more). If you get a `ValueError` like this it's because `Model.save` was not able to find or create a traced `ConcreteFunction`.\n", + "\n", + "**Caution:** You shouldn't save a model without at least one `ConcreteFunction`, since the low-level API will otherwise generate a SavedModel with no `ConcreteFunction` signatures ([learn more](../../guide/saved_model.ipynb) about the SavedModel format). For example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "064SE47mYDj8" + }, + "outputs": [], + "source": [ + "tf.saved_model.save(my_model, saved_model_path)\n", + "x = tf.saved_model.load(saved_model_path)\n", + "x.signatures" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LRTxlASJX-cY" + }, + "source": [ + "\n", + "Usually the model's forward pass—the `call` method—will be traced automatically when the model is called for the first time, often via the Keras `Model.fit` method. A `ConcreteFunction` can also be generated by the Keras [Sequential](https://www.tensorflow.org/guide/keras/sequential_model) and [Functional](https://www.tensorflow.org/guide/keras/functional) APIs, if you set the input shape, for example, by making the first layer either a `tf.keras.layers.InputLayer` or another layer type, and passing it the `input_shape` keyword argument. \n", + "\n", + "To verify if your model has any traced `ConcreteFunction`s, check if `Model.save_spec` is `None`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xAXise4eR0YJ" + }, + "outputs": [], + "source": [ + "print(my_model.save_spec() is None)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G2G_FQrWJAO3" + }, + "source": [ + "Let's use `tf.keras.Model.fit` to train the model, and notice that the `save_spec` gets defined and model saving will work:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cv5LTi0zDkKS" + }, + "outputs": [], + "source": [ + "BATCH_SIZE_PER_REPLICA = 4\n", + "BATCH_SIZE = BATCH_SIZE_PER_REPLICA * mirrored_strategy.num_replicas_in_sync\n", + "\n", + "dataset_size = 100\n", + "dataset = tf.data.Dataset.from_tensors(\n", + " (tf.range(5, dtype=tf.float32), tf.range(5, dtype=tf.float32))\n", + " ).repeat(dataset_size).batch(BATCH_SIZE)\n", + "\n", + "my_model.compile(optimizer='adam', loss='mean_squared_error')\n", + "my_model.fit(dataset, epochs=2)\n", + "\n", + "print(my_model.save_spec() is None)\n", + "my_model.save(saved_model_path)" ] } ], @@ -582,15 +642,7 @@ "colab": { "collapsed_sections": [], "name": "save_and_load.ipynb", - "private_outputs": true, - "provenance": [ - { - "file_id": "1QfmRqc5bTaHZhu4-FzRTWDiO2d6gr_r1", - "timestamp": 1560191609957 - } - ], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/estimator/boosted_trees.ipynb b/site/en/tutorials/estimator/boosted_trees.ipynb deleted file mode 100644 index b50ecbca440..00000000000 --- a/site/en/tutorials/estimator/boosted_trees.ipynb +++ /dev/null @@ -1,1093 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7765UFHoyGx6" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "KVtTDrUNyL7x" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xPYxZMrWyA0N" - }, - "source": [ - "# Boosted trees using Estimators" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p_vOREjRx-Y0" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dW3r7qVxzqN5" - }, - "source": [ - "This tutorial is an end-to-end walkthrough of training a Gradient Boosting model using decision trees with the `tf.estimator` API. Boosted Trees models are among the most popular and effective machine learning approaches for both regression and classification. It is an ensemble technique that combines the predictions from several (think 10s, 100s or even 1000s) tree models.\n", - "\n", - "Boosted Trees models are popular with many machine learning practitioners as they can achieve impressive performance with minimal hyperparameter tuning." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eylrTPAN3rJV" - }, - "source": [ - "## Load the titanic dataset\n", - "You will be using the titanic dataset, where the (rather morbid) goal is to predict passenger survival, given characteristics such as gender, age, class, etc." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KuhAiPfZ3rJW" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from IPython.display import clear_output\n", - "from matplotlib import pyplot as plt\n", - "\n", - "# Load dataset.\n", - "dftrain = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/train.csv')\n", - "dfeval = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/eval.csv')\n", - "y_train = dftrain.pop('survived')\n", - "y_eval = dfeval.pop('survived')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NFtnFm1T0kMf" - }, - "outputs": [], - "source": [ - "try:\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "tf.random.set_seed(123)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3ioodHdVJVdA" - }, - "source": [ - "The dataset consists of a training set and an evaluation set:\n", - "\n", - "* `dftrain` and `y_train` are the *training set*—the data the model uses to learn.\n", - "* The model is tested against the *eval set*, `dfeval`, and `y_eval`.\n", - "\n", - "For training you will use the following features:\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Feature NameDescription
    sexGender of passenger
    ageAge of passenger
    n_siblings_spousessiblings and partners aboard
    parchof parents and children aboard
    fareFare passenger paid.
    classPassenger's class on ship
    deckWhich deck passenger was on
    embark_townWhich town passenger embarked from
    aloneIf passenger was alone
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AoPiWsJALr-k" - }, - "source": [ - "## Explore the data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "slcat1yzmzw5" - }, - "source": [ - "Let's first preview some of the data and create summary statistics on the training set." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 204 - }, - "colab_type": "code", - "id": "15PLelXBlxEW", - "outputId": "b300d1e6-b3c2-476e-c7a2-a5a00d3ef03f" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    sexagen_siblings_spousesparchfareclassdeckembark_townalone
    0male22.0107.2500ThirdunknownSouthamptonn
    1female38.01071.2833FirstCCherbourgn
    2female26.0007.9250ThirdunknownSouthamptony
    3female35.01053.1000FirstCSouthamptonn
    4male28.0008.4583ThirdunknownQueenstowny
    \n", - "
    " - ], - "text/plain": [ - " sex age n_siblings_spouses parch ... class deck embark_town alone\n", - "0 male 22.0 1 0 ... Third unknown Southampton n\n", - "1 female 38.0 1 0 ... First C Cherbourg n\n", - "2 female 26.0 0 0 ... Third unknown Southampton y\n", - "3 female 35.0 1 0 ... First C Southampton n\n", - "4 male 28.0 0 0 ... Third unknown Queenstown y\n", - "\n", - "[5 rows x 9 columns]" - ] - }, - "execution_count": 0, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "dftrain.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 297 - }, - "colab_type": "code", - "id": "j2hiM4ETmqP0", - "outputId": "e8b8cc92-0a70-44bd-b7a6-54b16bb0ed63" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    agen_siblings_spousesparchfare
    count627.000000627.000000627.000000627.000000
    mean29.6313080.5454550.37958534.385399
    std12.5118181.1510900.79299954.597730
    min0.7500000.0000000.0000000.000000
    25%23.0000000.0000000.0000007.895800
    50%28.0000000.0000000.00000015.045800
    75%35.0000001.0000000.00000031.387500
    max80.0000008.0000005.000000512.329200
    \n", - "
    " - ], - "text/plain": [ - " age n_siblings_spouses parch fare\n", - "count 627.000000 627.000000 627.000000 627.000000\n", - "mean 29.631308 0.545455 0.379585 34.385399\n", - "std 12.511818 1.151090 0.792999 54.597730\n", - "min 0.750000 0.000000 0.000000 0.000000\n", - "25% 23.000000 0.000000 0.000000 7.895800\n", - "50% 28.000000 0.000000 0.000000 15.045800\n", - "75% 35.000000 1.000000 0.000000 31.387500\n", - "max 80.000000 8.000000 5.000000 512.329200" - ] - }, - "execution_count": 0, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "dftrain.describe()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-IR0e8V-LyJ4" - }, - "source": [ - "There are 627 and 264 examples in the training and evaluation sets, respectively." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 34 - }, - "colab_type": "code", - "id": "_1NwYqGwDjFf", - "outputId": "b9901780-2cbc-4731-c040-21b79a245c3f" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(627, 264)" - ] - }, - "execution_count": 0, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "dftrain.shape[0], dfeval.shape[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "28UFJ4KSMK3V" - }, - "source": [ - "The majority of passengers are in their 20's and 30's." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 279 - }, - "colab_type": "code", - "id": "CaVDmZtuDfux", - "outputId": "df4752dc-9976-4db1-9f09-6a057c1b6f46" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEGCAYAAACHGfl5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFddJREFUeJzt3W1sU+fBxvHLcRKHEQxOljqpw4agH8KLphCCmMQeaU2E\nBisUoX5oeFmzql2l0YSNLmK0S0jKS6cQOqYmgoGmSWiiQdVWYAbBYE2l6ZkmQAtoUWHa2KhgwQHm\nEMxLmjD7fj4g/GDyHp/gY/L/SZWa+/g+vjgnyZVzfHzsMMYYAQDGtZREBwAAJB5lAACgDAAAlAEA\nQJQBAEA2LYNQKKTGxkaFQqFERxkUOa1FTuskQ0aJnFaLJ+eQZdDV1aU33nhDS5Ys0fLly7Vu3Trd\nvHlTknTu3DktX75cixcv1muvvabOzs7ovMGWDecf1NTUlBQbnpzWIad1kiGjRE6rxZNzyDJwOBz6\n3ve+p2PHjunw4cPKz8/X+++/L0nasGGD6urqdPz4cRUXF2vHjh3ReYMtAwDYy5BlMHnyZM2fPz/6\ndWFhoa5evaq2tja5XC7NnTtXklRWVqZjx45J0qDLAAD2M6LXDIwxam5uVklJiQKBgHw+X3SZx+OR\n9OAwZbBlAAD7SR3Jgzdv3qyJEydqzZo1OnHiRJ/lg93ZYqBloVCoT0l0dHSoqKhITqdzJPGeOKfT\nKZ/PR06LkNM6yZBRIqfVnE6nioqK1NHR0WeZ2+2W2+0ecK5juPcmqq+v19///nft2bNHqampamtr\n0zvvvCO/3y9J6uzsVGlpqc6ePTvossc1NjaqqakpZqyoqEjNzc3DiQUAeMzKlSvV2toaM1ZRUaHK\nysoB5wzryGDnzp06f/689u7dq9TUB1PmzJmjnp4etba2qqioSAcOHNCSJUuGXPa48vJyrVixImbs\nYfvevHlXkYi976OXnZ2pYPBOomMMiZzWSoacyZBRIqeVUlIc8ngm6mc/+5nC4XDMssGOCqRhlMHF\nixe1d+9eTZs2TS+//LIkaerUqWpsbFR9fb02bdqk3t5e5efnq6GhQdKDK5C2b9+umpqaPsseN9ih\nSyRibF8GkpIio0ROqyVDzmTIKJHTanl5eSOeM+zTRIkQDN6x/cbPyZmkGzduJzrGkMhprWTImQwZ\nJXJaKSXFoezszNHNtTgLACAJUQYAgJFdWgpI0iT3BGW4Rv+t80XPf3U71G1hIgDxogwwYhmuVC37\n0eFRz/e/v1z2PvMKjD+cJgIAUAYAAMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAA\nIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAg\nygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDK\nAAAgygAAIMoAACDKAAAgKXU4D6qvr9eJEyfU3t6uI0eO6LnnnpMklZSUKCMjQ+np6XI4HKqqqtLC\nhQslSefOnVNtba16enrk8/nU0NCgrKyssfuXAABGbVhHBosWLdKHH34on88XM+5wONTY2KhDhw7p\n4MGD0SKQpA0bNqiurk7Hjx9XcXGxduzYYW1yAIBlhlUGRUVF8nq9MsbEjBtj+oxJUltbm1wul+bO\nnStJKisr07FjxyyICwAYC8M6TTSYqqoqGWM0b948vfXWW8rMzFQgEIg5ivB4PJKkUCgkt9sdMz8U\nCikUCsWMOZ1O5eXlxRsNAMalQCCgcDgcM+Z2u/v8/n1UXGXQ3Nwsr9er+/fva9u2bXr33XfV0NDQ\n72P7O4KQpH379qmpqSlmzOfzqaWlRdnZmfHEe2JyciYlOsKw2CnnYFnslHMwyZAzGTJK5LTa6tWr\n1d7eHjNWUVGhysrKAefEVQZer1eSlJaWplWrVmnt2rWSpLy8vJggnZ2dcjgc/bZSeXm5VqxYETPm\ndDolScHgHUUi/ZeIXeTkTNKNG7cTHWNIVua04gdioCzjcXuOlWTIKJHTSikpDmVnZ2r//v39HhkM\nZtRl0N3drXA4rMzMB3+9Hz16VDNnzpQkzZkzRz09PWptbVVRUZEOHDigJUuW9LueoQ5dAAAjM5rT\n7MMqg61bt+rkyZMKBoP67ne/K4/Ho927d6uyslKRSESRSEQzZsxQbW2tpAdXGW3fvl01NTXq7e1V\nfn7+gKePAACJN6wyqK6uVnV1dZ/xgwcPDjinsLBQfr9/9MkAAE8M70AGAFAGAADKAAAgygAAIMoA\nACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAA\nIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAg\nygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAAKBh\nlEF9fb1KS0tVUFCgixcvRsc///xzlZWVafHixSorK9Ply5eHtQwAYD9DlsGiRYv04YcfyufzxYzX\n1tZqzZo1On78uFatWqWampphLQMA2M+QZVBUVCSv1ytjTHSss7NTFy5c0AsvvCBJWrp0qc6fP6+b\nN28OugwAYE+po5kUCATk9XrlcDgkSSkpKXrmmWfU0dGhSCQy4DKPx9NnXaFQSKFQKGbM6XQqLy9v\nNNEAYNwLBAIKh8MxY263W263e8A5oyoDK+3bt09NTU0xYz6fTy0tLcrOzkxQqpHJyZmU6AjDYqec\ng2WxU87BJEPOZMgokdNqq1evVnt7e8xYRUWFKisrB5wzqjLIy8vTtWvXZIyRw+FQJBLR9evXlZub\nK2PMgMv6U15erhUrVsSMOZ1OSVIweEeRiOlvmm3k5EzSjRu3Ex1jSFbmtOIHYqAs43F7jpVkyCiR\n00opKQ5lZ2dq//79/R4ZDGZEZfDwdYOsrCwVFBTI7/frxRdflN/v16xZs6KngQZb9rihDl0AACMz\nmtPsQ5bB1q1bdfLkSQWDQb366qvyeDzy+/2qq6vTxo0btWvXLk2ePFn19fXROYMtAwDYz5BlUF1d\nrerq6j7j06dP10cffdTvnMGWAQDsh3cgAwAoAwAAZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAA\nEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQ\nZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAAJCUmugAeLImuScow8Vu\nBxCL3wrjTIYrVct+dDiudfjfX25RGgB2wWkiAABlAACgDAAAogwAAKIMAADiaiIkQO/9sHJyJg24\nfLBlD/X0huVKd8aV44ue/+p2qDuudQBPC8oAT1x6mtOSy1utWMftuNYAPD04TQQAoAwAAJQBAECU\nAQBAFryAXFJSooyMDKWnp8vhcKiqqkoLFy7UuXPnVFtbq56eHvl8PjU0NCgrK8uKzONaPDeaG85V\nOgDGp7jLwOFwqLGxUTNmzIgZ37Bhg+rr6zV37lzt3r1bO3bs0HvvvRfv04178d5ojpvMAehP3KeJ\njDEyxsSMtbW1yeVyae7cuZKksrIyHTt2LN6nAgCMEUveZ1BVVSVjjObNm6f169crEAjI5/NFl3s8\nHklSKBSS2+2OmRsKhRQKhWLGnE6n8vLyrIgGAONOIBBQOByOGXO73X1+/z4q7jJobm6W1+vV/fv3\ntW3bNm3evFmLFi3q87jHjx4e2rdvn5qammLGfD6fWlpalJ2dGW+8J4Jz8ckrnn2XDPs9GTJK5LTa\n6tWr1d7eHjNWUVGhysrKAefEXQZer1eSlJaWplWrVmnt2rUqLy+PCdLZ2SmHw9FvK5WXl2vFihUx\nY07ng9sMBIN3FIn0XyJ2kZMzSTduPLn3sSbLN2OyGO2+e9L7fTSSIaNETiulpDiUnZ2p/fv393tk\nMJi4yqC7u1vhcFiZmQ/+gj969KhmzZql2bNnq6enR62trSoqKtKBAwe0ZMmSftcx1KELAGBkRnOa\nPa4y+M9//qN169YpEokoEoloxowZ2rRpkxwOh7Zv366amhr19vYqPz9fDQ0N8TwVAGAMxVUGU6dO\n1cGDB/tdVlhYKL/fH8/qAQBPCO9ABgBQBgAAygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoA\nACDKAAAgygAAIMoAACDKAAAgygAAIAs+9hJIVr33w3F9jOgk9wTdDnVbmAhIHMoA41Z6mlPLfnR4\n1PP97y+XvT8RFxg+ThMBACgDAABlAAAQrxkACTXJPUEZrtH/GH7R819exIYlKAMggTJcqbyIDVvg\nNBEAIDmODCZMSJPD4YhrHd3d92WMsSgRADxdbF8GTmeKUlKdOvK/l0a9juKZXuW4Xfrii/sWJgOA\np4fty0CS7nbf16+PXRj1/MmZ6cqZnWthIsAehnoX9XDeYc2L0JCSpAwA9C/ed1FLvAiNByiDJyje\nywhhL/He2wiwE34zPUHxXkYoPfgrDvZg1V/lgB1waSkAgDIAAFAGAABRBgAAUQYAAFEGAABRBgAA\nUQYAAFEGAABRBgAAcTuKERno3kLcnwbjXbz33ZrknsCdUxOMMhgBKz6iEHga8fGdyY/TRAAAygAA\nMI5OE6W7UjVpUkaiYwCALY2bMnCl81kCgF1Z8UFBfHxnfMZNGQCwLz6+M/HGtAw+//xzbdy4UV1d\nXZoyZYq2b9+ur3zlK2P5lABGiI/vfGA4l8cOtZ2S+ehkTMugtrZWa9as0dKlS/W73/1ONTU12rdv\n31g+JYARelo+vtOKUhvPRydjVgadnZ26cOGCXnjhBUnS0qVLtWXLFt28eVMej2esnhbAOBVvqdmh\n0BJpzC4tDQQC8nq9cjgcD54oJUXPPPOMOjo6xuopAQCjlPAXkEOhkEKhUMyY0+lUXl6eUlIccjik\nSZkuNVT+z6ifIzd7oiTpGc+EuLJasQ4y2CeDFesgg3XreFoypKQ4Rj03MzNDrjhu6/FQIBBQOByO\nGXO73XK73QPOcRhjTNzP3I/Ozk4tXrxYp06dksPhUCQS0YIFC3TixImY00SNjY1qamqKmVtUVKTm\n5uaxiAUAT72VK1eqtbU1ZqyiokKVlZUDTzJj6Dvf+Y45fPiwMcaYQ4cOmVdeeaXPY27dumWuXLkS\n89+ZM2dMWVmZuXr16ljGi9vVq1fN888/T06LkNM6yZDRGHJa7erVq6asrMycOXOmz+/VW7duDTp3\nTE8T1dXVaePGjdq1a5cmT56s+vr6Po8Z6NCltbW1z2GO3YTDYbW3t5PTIuS0TjJklMhptXA4rNbW\nVuXm5io/P39Ec8e0DKZPn66PPvpoLJ8CAGABblQHAKAMAACSs66uri7RIfrjcrm0YMECuVyuREcZ\nFDmtRU7rJENGiZxWG23OMbu0FACQPDhNBACgDAAANrgdxePsetvr+vp6nThxQu3t7Tpy5Iiee+45\nSfbL29XVpQ0bNujKlStKT0/XV7/6Vb377rvyeDw6d+6camtr1dPTI5/Pp4aGBmVlZSUs65tvvqn2\n9nY5HA5NnDhR1dXVKigosN02laSmpiY1NTVF973dtmVJSYkyMjKUnp4uh8OhqqoqLVy40HY5e3t7\n9d577+nPf/6zXC6XCgsLtXnzZlvt8/b2dr355pvR+6rdunVLd+/e1alTp3Tp0iW9/fbbtsgpSZ9+\n+qk++OADGWNkjFFFRYUWLVo0uu35BN4UNyKvvPKK8fv9xhhjDh8+3O+7lhPhL3/5i+no6DAlJSXm\nH//4R3Tcbnm7urrM6dOno1/X19ebn/zkJ8YYYxYtWmRaW1uNMcbs2rXLvP322wnJ+NDt27ej//+H\nP/zBrFixwhhjv2362Wefmddff908//zz0X1vt21ZUlJiLl682Gfcbjm3bNlifvrTn0a/DgaDxhj7\n7fNHbdu2zWzZssUYY7+c8+fPj+73v/3tb2bu3LnGmNHltFUZBINBM3/+fBOJRIwxxoTDYVNcXGw6\nOzsTnOz/PfoLIRny/v73vzevvvqq+etf/2qWLl0aHe/s7DSFhYUJTBbr4MGD5qWXXrLdNu3p6TEv\nv/yy+fe//x3d93bclo9+Xz5kt5x37941xcXF5t69ezHjdtvnj+rt7TVf//rXzYULF2yZc8GCBdGy\nP336tPnWt75lgsGgKS4uHnFOW50mGuy213b8DAS75zXGqLm5WaWlpQoEAvL5fNFlD/OFQqFB72Q4\n1qqrq/WnP/1JkvTLX/7Sdtv0gw8+0PLly2O2nV23ZVVVlYwxmjdvntavX2+7nJcvX9aUKVPU2Nio\nU6dOaeLEifrBD36gjIwMW+3zR33yySfKzc1VQUGBPvvsM9vl3Llzp77//e/rS1/6ku7evau9e/cq\nEAgoNzd3xDl5AfkptnnzZk2cOFFr1qzpd7mxwVXFW7du1aeffqr169dH711lh1ySdO7cObW1tWnl\nypXRsYGyJTpzc3OzDh06pN/85jeKRCLavHlzv49LZM5wOKwrV65ozpw5+u1vf6uqqipVVlbq3r17\nCd9+A/n444/10ksvJTpGv8LhsPbu3atf/OIXamlp0e7du/XDH/5Q9+7dG9X6bFUGeXl5unbtWvQb\nIxKJ6Pr168rNzU1wsv7ZOW99fb0uX76sn//855IeZG1vb48u7+zslMPhSOhfso968cUXderUKVtt\n09OnT+vSpUsqLS1VSUmJrl27ptdff12XL1+23bb0er2SpLS0NK1atUpnz57Vs88+a6uczz77rFJT\nU/Xtb39bkvS1r31NWVlZcrlcun79ui32+aOuX7+uM2fOaNmyZZLs9/N+4cIF3bhxQ4WFhZIe3Pp/\nwoQJcrlco8ppqzLIyspSQUGB/H6/JMnv92vWrFkJP1R83MONbNe8O3fu1Pnz57Vr1y6lpj44Ezhn\nzhz19PRE73F+4MABLVmyJGEZ7927F/Opdy0tLZoyZYqysrI0c+ZMW2zTN954Q3/84x/1ySefqKWl\nRV6vV7/61a/02muv2Wpbdnd3686dO9Gvjx49qlmzZmn27Nm2yunxeLRgwYLoacFLly4pGAxq+vTp\ntvw5+vjjj/XNb35TkydPlmS/n/fc3Fx1dHTo0qVLkqR//vOfCgaDmjZt2qhy2u4dyP/617+0ceNG\nhUKh6G2vp02bluhY2rp1q06ePKlgMKgpU6bI4/HI7/fbLu/Fixe1bNkyTZs2Lfp29KlTp6qxsVFn\nz57Vpk2b1Nvbq/z8/IReZhgMBrV27Vp1d3crJSVFU6ZM0Y9//GPNnDnTdtv0odLSUu3Zsyd6aWlN\nTY0ttuWVK1e0bt06RSIRRSIRzZgxQ9XV1fryl79sq5wPs77zzjvq6upSWlqa3nrrLX3jG9+w5T5f\nvHixampqtHDhwuiY3XIeOXJEe/bskdPplCStW7dOJSUlo8ppuzIAADx5tjpNBABIDMoAAEAZAAAo\nAwCAKAMAgCgDAIAoAwCAKAMAgKT/Az96X9CTJZCZAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "dftrain.age.hist(bins=20)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1pifWiCoMbR5" - }, - "source": [ - "There are approximately twice as male passengers as female passengers aboard." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 274 - }, - "colab_type": "code", - "id": "-WazAq30MO5J", - "outputId": "77a2208b-a25f-4ee8-a31e-8cb3fb1a5f41" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEBCAYAAAC0WehTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEpVJREFUeJzt3HtsU/X/x/FXW8YkG+Uy5xggMWh0gCaCGGMUjAN/gLc5\nBAOCaDIkUbkIGhVFbi7BAV5ZECPeIggSvC5GRZzGazQIGBSJTMXpHDAHs9ucm1vP9w+z/izrtuLe\n7anl+UhIOGen5/PqZ5/1tXMo9TiO4wgAAENetwPEUiAQ0Jo1axQIBNyOEkKm6JApeomYi0zRSeZM\nSV8uxcXFCfeNI1PnyBS9RMxFpugkc6akLhcAgDsoFwCAOcoFAGAu6ctlxIgR8vl8bscI8fl8GjBg\nAJk6QaboJWIuMkUnUTONGDGiy+fxJPNbkevq6pSenu52DAD4z+nq62dSl0uro0frFQwmztPMyEhX\ndXWd2zHCkCk6iZhJSsxcZIpOomXyej3q0yety+fpZpAl4QWDTkKVi6SEyyORKVqJmElKzFxkik4i\nZuqqpP83FwBA/FEuAABzlAsAwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDA\nHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDA\nHOUCADBHuQAAzHVzO0A8ZGSkux2hjczMnm5HaINM0TneTH82Nqs20BCjNEBiOiHKpaBwmw4f5Ycb\n7ih5KE+1bocA4ozbYgAAc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzlAsA\nwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzUZXL\n9u3bdfnll2vixIk6cOBATAMtXLhQGzdujOkYAIDY6hbNQS+99JLmzZuncePGxToPACAJdFouK1as\n0I4dO3TgwAG9+OKLuuOOO7R69WrV19dLkubOnatLLrlEFRUVuvbaa3Xdddfpo48+UmNjo1atWqXN\nmzfrq6++Uo8ePbR27VplZGTou+++07Jly9TQ0KCmpiZdd911mjFjRpux//rrLz3yyCPasWOH/vrr\nL5155plaunSpevToYT8TAAAznZbLwoULtXfvXs2cOVMjRozQjBkz9NRTT+nkk09WVVWVJk2apDff\nfFOSVFNTo5EjR2rBggV6+umnddNNN2nDhg164IEHtGzZMm3YsEHz5s3TwIED9dxzzyklJUV//PGH\nJk+erIsvvliDBw8OG3v9+vXy+/3asmWLJGn16tVat26d5s+f3yZnIBBQIBAI2+fz+ZSdnf2vJwcA\nTlSVlZVqaWkJ2+f3++X3+6N6fFS3xVrt3LlTv/zyi26++WY5jiPp7xfwn376Sb1791ZaWppGjx4t\nSRo6dKj69euns846S5I0bNgwffbZZ5KkhoYGLVmyRPv27ZPX61VVVZX27dvXplxKS0tVX1+vt99+\nW9LfVzI5OTkRsz3//PMqLi4O2zdgwACVlpYez1MEYiIzs2dSjHG8yBSdRMw0bdo0VVRUhO2bPXu2\n5syZE9Xjj6tcJCknJ0cvvPBCm/0VFRXq3r17aNvn8yk1NTVsu7m5WZL08MMPKzMzUytXrpTH41FB\nQYGampranNNxHC1ZskQXXHBBp7luvPFG5efnh+3z+XxRPy8glqqqamN6/szMnjEf43iRKTqJlsnr\n9SgjI10bN26MeOUS9XmOZ9Dhw4frwIED+vzzz0P79uzZE/p769VMZ2pra5WdnS2Px6PvvvtOO3bs\niHhcbm6unn32WTU2NkqS6uvr9f3330c81u/3a+DAgWF/uCUGAP9OdnZ2m9fU4ymXqK5cPB6PpL9f\nwJ944gkVFRVpxYoVampq0qBBg7Ru3bqw4zpzyy236K677tIbb7yhQYMG6fzzz4943KxZs7RmzRpN\nmjRJHo9HXq9Xs2fP1umnnx7VOAAAd3icaC83/sMKCrfp8NEGt2PgBFXyUB63xRIEmTrXelusy+cx\nyAIAQBjKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmOvmdoB4\neHrR/7kdASewPxub3Y4AxN0JUS7V1XUKBh23Y4RkZvZUVVWt2zHCkCk6iZgJSETcFgMAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5jyO4zhuhwAARPZnY7NqAw1xG8/r9SgjI73L5+lmkCXhFRRu0+Gj\n8fvmAICVkofyVOt2iH+B22IAAHOUCwDAHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAA\nc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAA\nc5QLAMBcwpVLcXGxVq5c6XYMAEAXJFy5AAD++7pZniwnJ0e33367tm/frt9//13Lly/XZ599po8+\n+kjNzc167LHHNHjwYP32229asGCB6uvr1dTUpEsuuUR33nlnxHOuX79e27ZtU3Nzs7KyslRYWKiM\njAzL2AAAY+ZXLr169dLWrVt1xx136NZbb9XIkSP16quvKi8vT+vWrZMk+f1+Pfnkk3r55Zf16quv\nas+ePfr444/bnOuNN95QeXm5tmzZoldeeUWjR4/WihUrIo4bCAT0yy+/hP2prKy0fnoAcEKorKxs\n85oaCASifrzplYskTZgwQZI0bNgweb1ejR49OrS9fft2SVJLS4uKioq0a9cuOY6j6upqffvtt7r4\n4ovDzlVaWqpvvvlG11xzTehxfr8/4rjPP/+8iouLw/YNGDBApaWlps8PAOItM7Nn3MecNm2aKioq\nwvbNnj1bc+bMierxpuXi8XiUmpoqSfJ6verevXvoaz6fT83NzZKkZ599VrW1tdq6datSUlK0ePFi\nNTY2tjmf4zi65ZZbNHHixE7HvvHGG5Wfnx+2z+fzdeXpAEBCqKqqjdtYXq9HGRnp2rhxo1paWsK+\n1t4v95GYlovjOB1ut6qtrVVmZqZSUlJ06NAhvffee5o6dWqb43Jzc/XCCy9o7Nix8vv9ampq0g8/\n/KCcnJw2x/r9/uN64gCA9mVnZ3fp8eZXLh1tt7rhhhs0b948TZw4Uf369dOFF14Y8bi8vDzV1NRo\n+vTp8ng8CgaDuv766yOWCwAgcXic9i4vkkhB4TYdPtrgdgwAOG4lD+W5clusy+cxyAIAQBjKBQBg\njnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADmKBcAgDnKBQBg\njnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmPM4juO4HQIAENmfjc2q\nDTTEbTyv16OMjPQun6ebQZaEV11dp2AwcTo0M7Onqqpq3Y4RhkzRScRMUmLmIlN0EjGTBW6LAQDM\nUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADM\nUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADM\ndXM7QDx4vR63I7RBpuiQKXqJmItM0UmkTFZZPI7jOCZnSkB1dXVKT093OwYA/Od09fUzqW+L1dTU\naOrUqaqsrHQ7SkhlZaVyc3PJ1AkyRS8Rc5EpOomaaerUqaqpqenSeZK6XCRp586damlpcTtGSEtL\niyoqKsjUCTJFLxFzkSk6iZpp586dXT5P0pcLACD+KBcAgDnKBQBgzrd06dKlboeIpdTUVF1wwQVK\nTU11O0oImaJDpuglYi4yRSdZMyX1W5EBAO7gthgAwBzlAgAwl7TlcuDAAU2ZMkXjx4/XlClTVF5e\n7kqO3NxcXX755brmmmuUn5+vTz75RJK0e/du5eXlafz48SooKNCRI0dilqGoqEhjxoxRTk6OysrK\nQvs7mqNYz197mdqbLyn2c1ZTU6NZs2ZpwoQJysvL09y5c3X06NFOx45lro4y5eTkKC8vLzRX+/fv\nDz2utLRUEyZM0Lhx47RgwQI1NjaaZZKk2267LTTu9OnTtW/fPknurqn2Mrm5ploVFxeHrXW31lNn\nuUzXlJOkZsyY4ZSUlDiO4zivv/66M2PGDFdy5ObmOmVlZW32X3bZZc7OnTsdx3GctWvXOgsXLoxZ\nhi+//NI5ePCgk5ub6+zfvz+0v6M5ivX8tZepvflynNjPWU1NjfPFF1+EtouKipz77ruv07Fjmauj\nTDk5OU5DQ0Obx9TX1zsXXXSRU15e7jiO49x3331OcXGxWSbHcZza2trQ37dv3+7k5+c7juPummov\n06WXXuramnIcx/nmm2+cmTNnOpdeemlorbu1njrLZbmmkvLK5ciRI/r22291xRVXSJKuvPJK7d27\nN/QbXzw5jiPnmPdM7NmzR6mpqRo+fLgkacqUKXrrrbdilmHEiBHKysoKy9HRHMVj/iJlkiLPlxSf\nOevVq5fOP//80Pa5556rX3/9tcOxY52rvUxS+3P14Ycf6uyzz9app54ak0ySwj5zqra2Vl6v1/U1\ndWwmn88X2nZrTTU1NWn58uX655ty3VxPHeWSbNdUUn4qcmVlpbKysuTx/P3pnl6vV6eccooOHjyo\nPn36xD3PnXfeKcdxdN5552n+/PmqrKzUgAEDQl9vzRQIBOT3++OSqaM5CgaDrs7fP+drwYIFSk9P\nj/ucOY6jTZs2acyYMR2OHc9crZnGjh0rSfJ4PLrhhhvU0tKiUaNGac6cOUpJSWmTqX///jp48KBp\nFklatGhR6BbT+vXrE2JNHZuplVtr6vHHH1deXl7YOImwniLlkmzXVFJeuSSSTZs26bXXXtPWrVsV\nDAa1fPnyiMdF+m3hRHTsfC1btqzdY2M5Z8uXL1daWpqmT59+3GPHKldrpmnTpkmSPvjgA23dulUb\nNmxQWVmZ1q5dG5Nx21NYWKj3339f8+fPV1FRkST313GkTG6tqd27d2vPnj2aOnVqp+eP53qKlKuV\n5ZpKynLJzs7WoUOHQt+UYDCow4cPq1+/fnHPkpWVJUlKSUnR9ddfr127dql///6qqKgIHXPkyBF5\nPJ64XbVIHc+Rm/MXab5a88ZrzoqKilReXq5HH32007HjlevYTNL/z1VaWpomT54c+rDBYzP9+uuv\nys7ONs3zT1dffbU+//zzhFpTrZl+//1319bUF198oR9//FFjxoxRbm6uDh06pJkzZ6q8vNzV9RQp\nV0FBgT799FPTNZWU5dK3b1/l5OSopKREklRSUqKhQ4fG/ZZYQ0OD6urqQttvvvmmhg4dqmHDhqmx\nsTH0jdu8ebMmTJgQl0ytP9wdzZFb8xdpvoYMGSJJOvvss+MyZ4888oj27t2rtWvXqlu3bp2OHY9c\nkTIFAoHQu3Wam5v1zjvvhOZq1KhR+vrrr0Pvxtq8ebPGjx9vluePP/4IuyVSWlqq3r17q2/fvhoy\nZIgra6q9TKmpqa6tqVmzZunDDz/Ue++9p9LSUmVlZemZZ55RQUGBq+upvVznnHOO6ZpK2v+h/8MP\nP+iee+5RIBBQr169VFRUpNNOOy2uGX7++WfNnTtXwWBQwWBQp59+uhYtWqSTTz5Zu3fv1v3336+m\npiYNHDhQq1atUt++fWOSo7CwUO+++66qq6vVu3dv9enTRyUlJR3OUaznL1KmJ554QnPmzIk4X5Ji\nPmdlZWW66qqrdNppp4U+9uLUU0/VmjVrtGvXLi1evDji2LHMdWwmj8ejgQMHqqCgQIsXL5bX61Vz\nc7OGDx+ue++9Vz169JD094vrypUr5TiOhgwZogcffFAnnXSSSabq6mrdeuutamhokNfrVe/evXX3\n3XdryJAhrq2pSJnuuecepaWltfszKMV+Tf3TmDFj9OSTT+qMM87ocNx4Zvpnrrq6OtM1lbTlAgBw\nT1LeFgMAuItyAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADmKBcAgLn/AQvkwWROxmRCAAAAAElF\nTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "dftrain.sex.value_counts().plot(kind='barh')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7_XkxrpmmVU_" - }, - "source": [ - "The majority of passengers were in the \"third\" class." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 274 - }, - "colab_type": "code", - "id": "zZ3PvVy4l4gI", - "outputId": "147730b6-c79a-4864-9a40-b40e97383e8c" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAEBCAYAAABBp2PjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFHBJREFUeJzt3V1wVPX9x/HP7pIHSUhIIqYxBkWjTRAq4LSxFZ/S2kFF\nMQN9kLRwQb1oJUypDCIGlDRTSaLFGVbCTPWCsUjrZIAaHEbKROpM2yEdMVMd7UwzFkNDIikkbgIh\nkN3zv2DI38DmAZJvzp7k/brKnt1z9rPnd9gP5+zO/nyO4zgCAMCI3+0AYyEUCmnr1q0KhUJuR7kq\n5HePl7NL5Hcb+S+YMEUTDAY9Pdjkd4eXs0vkdxv5L5gQRQMAcA9FAwAwRdEAAExNmKKZN2+eAoGA\n2zGuSiAQUHZ2Nvld4OXsEvndNh7yz5s3b8Tb8U2Erzd3dXUpOTnZ7RgA4EkjfQ+dEEVzUXv7aUUi\n3ny5GRnJOnmyy+0YV83L+b2cXSK/27yc3+/3KS0tacTbmTQKWTwjEnE8WzSSPJ1d8nZ+L2eXyO82\nr+cfqQnzGQ0AwB0UDQDAFEUDADBF0QAATFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMA\nMEXRAABMUTQAAFMUDQDAFEUDADA1oeajycjw9iyb06ZNcTvCiFjnP9vTq85Qt+lzALhyE6poVpQf\n0Il23ojGq9qXF6nT7RAALsOlMwCAKYoGAGCKogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApiga\nAIApigYAYIqiAQCYomgAAKaGVTT79+9XUVGRioqK9PDDD2vNmjXWuQZVX1+vxYsXu5oBADA8Q/56\nc1tbm8rKyrR3715lZmZKkv71r3+ZBxuKz+dzOwIAYBiGLJr//e9/iouLU2pqat+yvLw8SdI///lP\nvfTSSzp9+rQkadWqVbrvvvskSe+9956CwaB6e3sVCAS0efNm3XbbbXr//fe1ZcsWRSIRpaenq6ys\nTDk5Oaqvr9dvfvMbfeMb31BDQ4P8fr9++9vf6uabb5YkbdmyRfv371dmZqZmz5496jsCAGBjyKLJ\ny8vT7Nmzdf/99+tb3/qW7rzzTi1atEiBQEDPP/+8fve73+naa69VW1ublixZonfeeUdtbW3asGGD\ndu3apZycHJ0/f17nz5/XqVOn9Mwzz2jnzp26+eabVVNTo6efflpvvfWWJKmxsVGbN29WWVmZtm/f\nrurqalVVVamurk6HDh3S22+/rYSEBP385z833zEAgNExZNH4fD69+uqramxsVH19vQ4ePKjXX39d\na9eu1X//+189+eSTchxHkhQIBPT555+roaFB9913n3JyciRJcXFxiouL0+HDh5Wfn993lrJ48WJt\n2rRJZ86ckSTNmDGj72zpjjvu0KFDhyRd+Ezm4YcfVmJioiRpyZIl2r59e9S8oVBIoVCo37JAIKCs\nrKwr3TcAAEktLS0Kh8P9lqWkpCglJWVY6w97hs3c3Fzl5uZq6dKleuSRRyRdONt54403LntsQ0ND\n1G04jnPZZytfvZ2QkND3dyAQUG9vb996w7Vjxw4Fg8F+y7Kzs1VXVzfsbcC7rKaLZhptd5HfXcXF\nxWpubu63bOXKlSopKRnW+kMWzRdffKGWlhbNmTNHktTa2qr29nbl5ubq6NGjOnz4sAoKCiRJH330\nkWbPnq358+erurpaTU1Nmj59us6dO6fz589rzpw5Ki0t1X/+8x/NmDFDu3fv1syZMzV58uRBM3z7\n29/WK6+8omXLlik+Pl67d+8e8LHLly9XUVFRv2WBQGDIHYHxoa1t9CdznjZtisl2xwr53eXl/H6/\nTxkZydq5c2fUM5rhGrJowuGwtm7dquPHjyshIUGO42j16tXKy8tTdXW1Kioq9OKLL+rcuXOaPn26\ntm/frhtvvFHl5eX65S9/qXA4rEAgoIqKCt16662qrKzU008/rXA4rPT0dFVVVQ0Z8v7771dDQ4Me\nf/xxXXfddSooKNCJEyeiPvZKTucAAEMb6UcPPudKrkt53IryAzrR3u12DBipfXkRZzRRkN9dXs5/\n8YxmxNsZhSwAAAyIogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgA\nAKYoGgCAKYoGAGCKogEAmBr2DJvjweul33c7Agyd7el1OwKAKCZU0Zw82aVIxJvT73h5TgvJ+/kB\nXD0unQEATFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUD\nADBF0QAATFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUD\nADBF0QAATFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUD\nADA1ye0AYykjI9ntCCMybdoUtyOMiJfzD5X9bE+vOkPdY5QG8JYJVTQryg/oRDtvBhh9tS8vUqfb\nIYAYxaUzAIApigYAYIqiAQCYomgAAKYoGgCAKYoGAGCKogEAmKJoAACmKBoAgCmKBgBgiqIBAJii\naAAApsbkRzULCwuVmJio+Ph4+Xw+FRQUKCkpSbm5uXrooYeGvZ3Ozk798Y9/1M9+9jPDtACA0TRm\nv968detW3XLLLUM+znEc+Xy+qPd9+eWXeu211ygaAPCQMSsax3H63X722Wc1a9YsFRcXKxgM6vPP\nP9eZM2d07NgxvfHGG9qyZYvq6+sVHx+vyZMn680339Svf/1rdXV1qaioSImJidq1a9dYxQcAXKUx\nK5pVq1b1XTpbs2bNZfd/8MEH2rNnj1JTU/Xpp5/q73//u959911JFy6ZSdLGjRu1ZMkS7dmzZ8Dn\nCYVCCoVC/ZYFAgFlZWWN4qsBgImjpaVF4XC437KUlBSlpKQMa33XLp3t27ev3/333nuvUlNTJUk5\nOTlyHEfr169XQUGBHnjggWE/z44dOxQMBvsty87OVl1d3QjSA0OL5RlEYznbcJDfXcXFxWpubu63\nbOXKlSopKRnW+q5dOrvU5MmT+/5OTk7Wvn37VF9fr7/97W966aWXtHfv3mE9z/Lly1VUVNRvWSAQ\nuPLAwBVqa4vNOTanTZsSs9mGg/zu8ft9yshI1s6dO6Oe0QxXTE7lfOrUKU2aNEnz58/Xd77zHf3l\nL3/RsWPHNGPGDJ09e1aRSER+f/RvZl/J6RwAYGgj/ehhTIpmoG+RDaS1tVWlpaWKRCIKh8O69957\nNWfOHEnSo48+qoULFyo1NZUvAwCAB/icoa5pjSMryg/oRHu32zEwDtW+vChmL494+dKNRH43Xbx0\nNuLtjEIWAAAGRNEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUDADBF0QAATFE0AABT\nFA0AwBRFAwAwRdEAAEzF5MRnVl4v/b7bETBOne3pdTsCELMmVNGcPNmlSMSb0+94eU4Lydv5vZwd\niAVcOgMAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgAAKYoGgCAKYoG\nAGCKogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgAAKYoGgCAKYoG\nAGCKogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgAAKYoGgCAKZ/j\nOI7bIQAAY+9sT686Q90D3u/3+5SRkTzi55k04i14yIryAzrRPvBOBYCJpPblReocg+fh0hkAwBRF\nAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUDADBF0QAATFE0AABTo/qjmj/8\n4Q91/vx5nTt3TkePHtVtt90mx3HU2dmptLQ01dTUXLbOxx9/rB07dqiqqmpYzxEMBnXmzBmtXbt2\nNKMDAIyMatG89dZbkqTm5mYtWbJEe/bskSTV19ersrIy6jqzZs0asGTC4bACgcBoRgQAjLExu3TW\n29urjRs36rHHHtPjjz+uzz77TNKFElq8eLGkCwV11113KRgMaunSpaqpqVFXV5dWrVqlhQsX6skn\nn1RTU9NYRQYAjIIxK5rGxkYtXbpUb7/9thYsWKDq6uq++3w+X9/fHR0dys3N1Ztvvqkf/ehHevXV\nVzVlyhTt27dPlZWV+sc//jFWkQEAo2DMJj6bMWOG8vLyJEl33HGHDh06FPVxiYmJWrBgQd/tw4cP\na8OGDZKktLQ0Pfjgg4M+TygUUigU6rcsEAgoKytrBOkBYOJqaWlROBzutywlJUUpKSnDWn/MiiYh\nIaHv70AgoN7e3qiPu+aaa/rdvtKZpnfs2KFgMNhvWXZ2turq6q5oOwAwEUybNmXIxxQXF6u5ubnf\nspUrV6qkpGRYz2FWNFdaEAOtd9ddd2n37t2aO3eu2tvbdfDgwX5nPJdavny5ioqK+i3jCwUAEF1b\n28CTOfv9PmVkJGvnzp1Rz2iGy6xovvq5y0jWe+qpp7R+/XotXLhQ2dnZmj9//qDrX8npHABgaCP9\n6MHnXO2phwetKD+gE+3dbscAgJhQ+/KiYZ3RjBS/DAAAMEXRAABMUTQAAFMUDQDAFEUDADBF0QAA\nTFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMTahpAgAA/+9sT686QwNPnTJa\n0wSM2VTOseDkyS5FIt7s1WnTpgw6b0Ss83J+L2eXyO82r+cfDVw6AwCYomgAAKYoGgCAKYoGAGCK\nogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgAAKYoGgCAKYoGAGBq\nQk185vf73I4wIuR3j5ezS+R3m1fzj1buCTGVc1dXl5KTRz4dKQBMRCN9D50Ql846Ojr0xBNPqKWl\nxe0oV6WlpUWFhYXkd4GXs0vkd9t4yP/EE0+oo6NjRNuZEEUjSUeOHFE4HHY7xlUJh8Nqbm4mvwu8\nnF0iv9vGQ/4jR46MeDsTpmgAAO6gaAAApigaAICpwAsvvPCC2yHGQkJCggoKCpSQkOB2lKtCfvd4\nObtEfreRf4J8vRkA4B4unQEATFE0AABT4/4naI4ePap169apo6NDU6dOVWVlpaZPn+52rEEVFhYq\nMTFR8fHx8vl8WrNmje6++241NDTo+eefV09Pj7Kzs1VVVaX09HRXs1ZUVOjAgQNqbm7Wvn37lJub\nK2nw/R5LYzJQ/oHGQFJMjUNHR4fWrl2rY8eOKT4+XjfeeKM2bdqktLS0QXPGymsYLH9eXp6+/vWv\ny+fzyefzqbKyUrfeeqskqa6uTlVVVYpEIrr99tv14osvuvIZyFNPPaXm5mb5fD4lJSWptLRUeXl5\nnjn+B8o/6se/M84tW7bMqa2tdRzHcf70pz85y5YtcznR0AoLC53GxsbLlj/44IPOkSNHHMdxnG3b\ntjnPPvvsWEe7zAcffOC0trY6hYWFzr///e++5YPt91gak4HyDzQGjhNb49DR0eHU19f33a6oqHCe\ne+45x3EGzxkrr2Gw/Hl5eU53d/dl65w+fdq5++67naamJsdxHOe5555zgsHg2AS+RGdnZ9/fBw8e\ndIqKihzH8c7xP1D+Bx54YFSP/3F96ezUqVP69NNP9cgjj0iSFi5cqE8++UTt7e0uJxuc4zhyLvmO\nxkcffaSEhATNnTtXkvTjH/9Y+/fvdyNeP/PmzVNmZma/vIPt91gbk2j5pehjIMXeOKSmpuqb3/xm\n3+05c+bo+PHjg+aMpdcwUH5p4DF4//33NWvWLOXk5EhyN/9Xf/+rs7NTfr/fU8f/pfkDgUDf7dE8\n/sf1pbOWlhZlZmbK57vwC6R+v1/XXXedWltblZaW5nK6wa1Zs0aO4+jOO+/U6tWr1dLSouzs7L77\nL+YPhUJKSUlxK2ZUg+33SCTimTH56hj86le/UnJyckyPg+M42rVrl7773e8OmjNWX8PF/N/73vck\nST6fTz/96U8VDod1zz33qKSkRHFxcZflv/7669Xa2upWbJWWluqvf/2rJOm1117z3PF/af6LRvP4\nH9dnNF61a9cu7d27VzU1NYpEIiorK4v6uGj/48DouHQMNm3aNOBjY2UcysrKlJSUpJ/85CdR7x8s\nZyy8hov5i4uLJUmHDh1STU2Nfv/736uxsVHbtm1zOWF05eXleu+997R69WpVVFRIio39OVzR8o/2\n8T+uiyYrK0tffPFF346IRCI6ceKEvva1r7mcbHCZmZmSpLi4OC1dulQffvihrr/+ejU3N/c95tSp\nU/L5fK7/Lzqawfa7V8Yk2hhIF15bLI5DRUWFmpqa9Morr0gaPGcsvoZL80v/PwZJSUn6wQ9+0Pfj\njpfmP378uLKyssY2cBSPPfaYDh8+7Nnj/2L+L7/8ctSP/3FdNOnp6crLy1Ntba0kqba2VjNnzoy5\nSzRf1d3dra6urr7b77zzjmbOnKnbb79dPT09ff/Y/vCHP+ihhx5yK2ZUF//xDLbfvTAm0cYgPz9f\nkjRr1qyYG4ctW7bok08+0bZt2zRp0oWr4YPljLXXEC1/KBRST0+PJKm3t1fvvvtu3xjcc889+vjj\nj9XU1CTpQv4FCxaMee4zZ870u2RXV1enqVOnKj09Xfn5+TF//A+UPyEhYdSP/3H/ywCfffaZ1q1b\np1AopNTUVFVUVOimm25yO9aAjh07plWrVikSiSgSieiWW25RaWmprr32WjU0NGjDhg06d+6cbrjh\nhpj4enN5ebn+/Oc/6+TJk5o6darS0tJUW1s76H6PpTGJlr+6ulolJSVRx0BSTI1DY2OjHn30Ud10\n0019X+/NycnR1q1b9eGHH2rjxo1Rc8bKa7g0v8/n0w033KAVK1Zo48aN8vv96u3t1dy5c7V+/Xpd\nc801ki68KVZWVspxHOXn52vz5s1KTEwc0+wnT57UL37xC3V3d8vv92vq1Kl65plnlJ+f74njP1r+\ndevWKSkpacD3IOnqjp1xXzQAAHeN60tnAAD3UTQAAFMUDQDAFEUDADBF0QAATFE0AABTFA0AwBRF\nAwAw9X+gI1LNFj+FVgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "dftrain['class'].value_counts().plot(kind='barh')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HM5SlwlxmZMT" - }, - "source": [ - "Most passengers embarked from Southampton." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 274 - }, - "colab_type": "code", - "id": "RVTSrdr4mZaC", - "outputId": "3512101a-0913-48d3-d4b0-6e0fdc2ed34d" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAEBCAYAAAAD5BB0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtUVPXi/vH3zKCkAl7IiMArZnhLMTtKLf0a5clLpXhs\nLRPTymp1EY+30tJMyUw0syNUy45Wapa3xNSysshjJ1FWmZmZhZ0ML2CE2gAqCLN/f/hjjiiXUYGZ\nzud5reWK2TN774cPn+Zh7xn22CzLshAREfkfZ/d2AJM4nU6SkpJwOp3ejuKmTJ7xxUzgm7mUyTPK\n5LnqyqXCq0VOp5Pk5GSfmkzK5BlfzAS+mUuZPKNMnquuXCo8ERExggpPRESMoMITEREjqPBqWdeu\nXXE4HN6O4eZwOAgLC1OmKvhiJvDNXMrkGWXynMPhoGvXrpe9HZv+LKH25OfnExAQ4O0YIiJ/Spf7\nHKrC84LjxwtwuXxn2IODA8jNzfd2jDKUyXO+mEuZPKNMnrHbbTRu3OCyt+NXDVnkIrlclk8VHuBz\neUCZLoYv5lImzyhT7dFreCIiYgQVnoiIGEGFJyIiRlDhiYiIEVR4IiJiBBWeiIgYQYUnIiJGUOGJ\niIgRVHgiImIEFZ6IiBhBhSciIkZQ4XlBQMAV3o4gImIcFZ4X+Pvrmt0iIrVNhSciIkZQ4YmIiBFU\neCIiYgQVnoiIGEGFJyIiRlDhiYiIEVR4IiJihD9t4R0+fJgePXp4O4aIiPxJ/GkLD8Bms3k7goiI\n/En4ROGdf7RWerv0v/Pnzyc2NpZ+/fqxc+fOC9YvKipi7NixJCYmAnDvvfcyZ84chg0bRp8+fZg3\nb577sZmZmdx3333cddddDB48mC+++AKAlStXkpCQAMDu3buJjIxkz549AMyYMYPVq1cDEBkZycKF\nCxkyZAh9+vRh8+bNNTMoIiJSrXyi8ODCo7XS2ydOnKBr166kpKTw2GOPMXfu3DKP++OPPxg1ahTd\nunVj0qRJ7uXZ2dm88847pKSksHr1ajIzMwGYOHEid911F+vXr2fu3Lk88cQTHD9+nOjoaLZv3w7A\n9u3biYqKIi0tDYC0tDSio6Pd2w4MDGTNmjUkJiby3HPPVf9giIhItfOZwqtIgwYN+L//+z8AunTp\nwsGDB933nT59mri4OIYPH87w4cPLrNe3b18AAgICiIiIIDMzk4KCAn744QcGDx4MQEREBO3atePb\nb7+lefPmnD59mqNHj5KWlsaECRNIS0sjOzubM2fOEB4e7t52//793XlycnIoKiq6ILfT6eTQoUNl\n/mVlZVXv4IiIGCQrK+uC51Wn0+nx+j5xFWM/Pz9cLpf7dmFhofvrunXrur+22+2UlJSUua9z586k\npqbSp08f7Pb/9re/v/8F61mWVe7rfqXLevTowZYtW8jNzaVbt27k5OSwZcuWMqdbbTabe9ul+zs3\nU6klS5aQnJxcZllYWBipqakANG0aWNmQ1DpfywPKdDF8MZcyeUaZPBcXF8fhw4fLLBs9ejTx8fEe\nre8ThXfllVdSXFzMwYMHadasGRs2bHDfZ1lWmceee9tut/P888+TkJDA2LFjeemll/Dzq/hbCggI\noF27dqSkpBAbG8vPP//Mjz/+yPXXXw+cLbyXX36ZXr16ARAVFcXrr7/O+PHjPcpzrpEjRxIbG1tm\nmcPhcH+dk5NXYc7a1rRpoE/lAWW6GL6YS5k8o0yesdttBAcHsHz58gsOMIKCgjzejk8UnsPhYMqU\nKdx3332EhYXRvXt3930VvbZ3rmnTppGYmMjo0aNZsGBBpevMnTuXadOm8eabb+Ln58fcuXNp3Lgx\ncLbwsrKyuOmmmwCIjo5m9erVFxzhVZUHzv4QLuYHISIilQsNDb2s9W1WRYcoUqN86TcoX/yNTpk8\n54u5lMkzyuSZ0iO8y95ONWQRERHxeSo8ERExggpPRESMoMITEREjqPBERMQIKjwRETGCCk9ERIyg\nwhMRESOo8ERExAgqPC8oLCz2dgQREeOo8LwgP/+0tyOIiBhHhSciIkZQ4YmIiBFUeCIiYgQVnoiI\nGEGFJyIiRlDhiYiIEVR4IiJiBBWeiIgYQYUnIiJGUOGJiIgRVHgiImIEFZ6IiBhBhSciIkZQ4YmI\niBFUeCIiYgQVnoiIGEGFJyIiRlDhiYiIEVR4IiJiBBWeiIgYQYUnIiJGUOGJiIgRVHgiImIEFZ6I\niBhBhSciIkZQ4YmIiBH8vB3ARMHBAd6OcIGmTQM5XVhMnvOUt6OIiNQIFZ4XjJr5Cb8d971i2TBv\nIHneDiEiUkN0SlNERIygwhMRESOo8ERExAgqPBERMYIKT0REjKDCExERI6jwRETECCo8ERExwiUX\nXlFREbNnz6ZPnz7079+f2NhYPvroo+rMdtmSk5MpLi72dgwREfEBl3yllenTp3P69Gk+/PBD6tSp\nQ0ZGBqNGjaJRo0b06NGjOjNesuTkZEaNGoWfny4oIyJiuktqgiNHjvDRRx/xr3/9izp16gBw7bXX\n8uijj5KcnMxXX31FQUEBkyZNAs4Wz8mTJ3nyySc5c+YM8+fP56uvvuLMmTO0bduW6dOnU69ePfLz\n85k9ezY//fQThYWFdO/enaeeegqbzca9995Lp06d2LVrFzk5OfTt25cJEya4t//hhx/i7++PzWZj\n6dKlvPTSS9hsNoYOHYrdbmfZsmUUFhby7LPPkpmZCcADDzzAoEGD+Pe//82yZctYuHAhubm53Hzz\nzfzjH//g9ttvZ9GiReTl5TFu3DhiYmIYNGgQ27ZtIycnhwceeIC4uLjq+DmIiEgNu6RTmj/99BMt\nWrQgMDCwzPIuXbrw008/AWCz2cpdd9GiRQQFBbFq1SpSUlJo2rQpCxcuBGD27Nn85S9/YdWqVaxb\nt47c3FzWrFnjXjc7O5t33nmHlJQUVq9eTWZmJk6nkzfeeIN169aRkpLC22+/Tf369Zk2bRqWZbFy\n5UpSUlIICAhg5syZtG3blvXr17N48WJefPFF9u/fT7du3fj2228pKSlh+/btREVFkZaWBkBaWho3\n3XSTO8Pp06dZsWIFS5cu5cUXX+TUqfKviel0Ojl06FCZf1lZWZcy3CIiAmRlZV3wvOp0Oj1e/5KO\n8CzLqvC+ioquVGpqKgUFBe7X+86cOUNkZKT7vu+++4433ngDOFsuoaGh7nX79u0LQEBAABEREWRm\nZhIeHk7r1q2ZOHEiPXv2pHfv3tSvX7/crNu2bWPy5MkANG3alN69e7Njxw7atGnDtddey65du9i2\nbRuPP/44c+bM4cyZM+zZs4eoqCj3NgYMGABAWFgYjRo1Ijs7m1atWl3wfS5ZsoTk5OQyy8LCwkhN\nTa10fLytadPAqh9US3wpSylfzAS+mUuZPKNMnouLi+Pw4cNllo0ePZr4+HiP1r+kwmvbti2//vor\nTqeToKAg9/JvvvmGqKgo/Pz8yhRNYWGh+2vLsnj22Wfp3r17udt+5ZVXCA8PL/c+f39/99d2u52S\nkhLsdjurVq1i586dpKWlMXjwYBYvXkzbtm0vWN9ms1VYyN27dyctLY3du3czY8YMgoOD2bhxI+3a\ntaNu3brlZrDZbJSUlJS7vZEjRxIbG1tmmcPhKPexviQnxzc+L6Fp00CfyVLKFzOBb+ZSJs8ok2fs\ndhvBwQEsX778gufcczuoyu1cys7DwsLo27cv06dPp6ioCDh7mnPp0qWMHTuWZs2asWfPHizLIj8/\nny1btrjXjYmJ4c0333SXYEFBAT///LP7vtdffx2XywXA8ePHOXToUKVZCgoKyM3NpVu3bsTHx9O2\nbVsyMjKAs0eCeXn//cHddNNNrFq1CoCcnBy2bt3qfoNNdHQ0a9eu5eqrr8bPz4/o6GiSkpKIjo6+\nlCEiKCiI8PDwMv/OPVoVEZGLExoaesHz6sUU3mW9S3PevHn0798fm83Gb7/9xsqVK4mMjCQiIoJN\nmzZxxx130Lx5czp27Ohe7+GHHyYpKYkhQ4Zgs9mw2+2MHj2aiIgInnrqKebOncvAgQOBs0dTTz/9\nNOHh4RccmZXezs/PJz4+nsLCQlwuFx06dKBPnz4A3H///YwYMYJ69eqxbNkypkyZwrRp07jrrrsA\nmDhxIhEREQB07tyZEydOuF+v69GjB/Pnzy/zjtOKMoiIiO+zWZW9IOeh4uJinn32WbKzs3nttdfK\nnAKUC/nyB8D6yqkMXzyt4ouZwDdzKZNnlMkzpac0L1e1/IGan58fzz//fHVsSkREpEbo0mIiImIE\nFZ6IiBhBhSciIkZQ4YmIiBFUeCIiYgQVnoiIGEGFJyIiRlDhiYiIEfTJqF6weOpfvR2hXKcL9enw\nIvK/S4XnBbm5+bhcl31Ft2rji5cSEhGpbjqlKSIiRlDhiYiIEVR4IiJiBBWeiIgYQYUnIiJGUOGJ\niIgRVHgiImIEFZ6IiBhBhSciIkZQ4YmIiBFUeCIiYgQVnoiIGEGFJyIiRlDhiYiIEVR4IiJiBBWe\niIgYQYUnIiJGUOGJiIgRVHgiImIEFZ6IiBhBhSciIkZQ4YmIiBFUeCIiYgQVnoiIGEGFJyIiRlDh\niYiIEfy8HcBEwcEB3o5wgaZNA70d4QKXm+l0YTF5zlPVlEZE/uxUeF4wauYn/HZcT8Q1bcO8geR5\nO4SI+Ayd0hQRESOo8ERExAgqPBERMYIKT0REjKDCExERI6jwRETECCo8ERExggpPRESMUCt/eF5c\nXMwrr7zChx9+SN26dXG5XPTq1YvWrVvzxRdfsGDBgsveR0xMDK+//jpt2rSphsQiIvK/plYKb/Lk\nyRQVFbFu3Trq1atHSUkJa9eupaioCJvNdlnbtizrsrfhCZfLhd2uA2IRkT+rGi+8X3/9lc8++4wv\nvviCevXqAeBwOLj77rtJSUkhPz+fcePGkZGRQVBQEElJSQQHBwOwaNEiPvnkE4qLiwkJCWHmzJkE\nBweTnJzMr7/+ysmTJzl48CBvv/02AOvXr2fnzp3k5OQwYsQI4uLiANi9ezezZs3i1KlT1KtXjylT\nptCpUyfS09NJTEzkvffeAyhzOz09nVmzZtGtWzf27NnDo48+SmRkJJMmTSI3N5fw8HAsy6Jnz57u\n/YiIiO+q8cLbu3cvLVu2JCCg/Asm79mzh/Xr1xMSEsIzzzzDsmXLGDt2LOvXryczM5NVq1YB8O67\n7/LCCy/w4osvAvD111+TkpJCw4YN3dvKzc3l7bffJjc3l0GDBnHjjTfSqlUr/v73v/PCCy/Qo0cP\n0tLSGDNmDJs3bwa44Ojw3NsZGRkkJCQwdepUAMaMGUOPHj145JFHOHLkCHfeeSc9e/Ys9/tyOp04\nnc4yyxwOB6GhoRczfCIi8v9lZWVRUlJSZllQUBBBQUEerV/jhWdZVqX3R0VFERISAkDnzp1JS0sD\nIDU1le+//55BgwYBUFJSUuab6tWrV5myAxgyZAgAwcHB9O7dm/T0dADq1q1Ljx49AIiOjqZu3br8\n8ssvVWZv0aIF119/vfv2jh073OV3zTXXEB0dXeG6S5YsITk5ucyysLAwUlNTq9yvVJ/q/hQIX/xU\nCfDNXMrkGWXyXFxcHIcPHy6zbPTo0cTHx3u0fo0XXocOHThw4AB5eXkEBl44iP7+/u6vHQ4HxcXF\nwNmifPTRRxk8eHC5261fv36l+y19ba+8wi29z+Fw4HK53MsLCwur3IenrxeOHDmS2NjYMsscDodH\n60r1ycmpvs9LaNo0sFq3V118MZcyeUaZPGO32wgODmD58uXlHuF5vJ3qDna+Fi1aEBMTw7Rp0ygo\nKADOvgFk6dKlnDx5ssL1YmJieOedd9ynBYuKiti3b1+l+0pJSQHg2LFjbN26le7du9O6dWvOnDnj\nPtrbvn07xcXFtGzZkvDwcA4dOkReXh6WZfHBBx9Uuv3u3buzdu1a4Oyh9fbt2yt8bFBQEOHh4WX+\n6XSmiMilCw0NveB59WIKr1bepZmYmEhSUhKDBw+mbt26WJZFr169aNWqVYXrDBw4kBMnTjB8+HBs\nNhsul4thw4YRGRlZ7uNtNhuhoaHExcXx+++/88gjj7j/RGHBggXMnDnT/aaVpKQk/Pz8CAkJ4f77\n7yc2NpZmzZrRqVMn9u/fX2Gmp59+mkmTJrFp0yZat27NDTfcUO5Rq4iI+B6bVdWLbOJWWFiIn58f\nDoeDnJwc7r77bt566y1atmx5UdvRB8DWjg3zBuqUppcok2eUyTOlpzQvlz7x/CIcOHCASZMmYVkW\nJSUljB49+qLLTkREvEOFdxGuu+461q1b5+0YIiJyCXTpEBERMYIKT0REjKDCExERI6jwRETECCo8\nERExggpPRESMoMITEREj6O/wvGDx1L96O4IRThcWezuCiPgQFZ4X5Obm43L5zhXdfPFSQr6YSUT+\n3HRKU0REjKDCExERI6jwRETECCo8ERExggpPRESMoMITEREjqPBERMQIKjwRETGCCk9ERIygwhMR\nESOo8ERExAgqPBERMYIKT0REjKDCExERI6jwRETECCo8ERExggpPRESMoMITEREjqPBERMQIKjwR\nETGCCk9ERIygwhMRESOo8ERExAgqPBERMYIKT0REjKDCExERI9gsy7K8HUJERKSm+Xk7gIlGzfyE\n346f8nYMEZE/hasa12Px1L9e9nZ0SlNERIygwhMRESOo8ERExAgqPBERMYIKT0REjKDCExERI6jw\nRETECB4V3qZNm4iNjSU2Npb+/fszceLES97hvn372LRpU5llkZGRnDrlnb9LS09P58svv/TKvkVE\npPZU+YfnOTk5JCQksG7dOkJCQoCzpXWp9u7dy5YtW+jXr597mc1mu+TtXa709HQKCgq4+eabvZZB\nRERqXpWF9/vvv1OnTh0aNmzoXhYZGQnA1q1bmT9/Pi6XiyZNmpCQkECzZs1ISUnh888/Z8GCBQDu\n2wkJCSQlJVFQUEBsbCzdunVjypQpWJbF0qVL2bx5M3/88QdPPPEEf/3r2b+qnzhxIgcOHKCoqIgW\nLVowa9YsAgMDSU9P5/nnn+f6669n165d1KlThzlz5pCcnExGRgahoaEkJydzxRVXkJyczP79+zl5\n8iRHjhyhdevWzJo1iyNHjrBixQosy2L79u3079+fhx56iHXr1rF48WLsdjvNmzdnxowZNGnShJSU\nFDZu3EhQUBAZGRkEBQWRlJREcHBwTfxsRESkGlV5SjMyMpJOnTrRu3dvxowZw5IlSzhx4gTHjh1j\n0qRJzJs3j/fff58BAwYwYcIE93rnH7XZbDYaNWrEmDFjiI6OJiUlhSlTprjvDwwMZM2aNSQmJjJz\n5kz38qlTp7JmzRrWr19PREQE//znP933/fzzzwwfPpwNGzbQpUsXHnzwQZ5++mk++OAD7HY7Gzdu\ndD/266+/Zvbs2WzcuJGAgABeffVV2rZty9ChQxk4cCApKSk89NBDZGRkMG/ePN566y3ef/992rRp\nw3PPPefezp49e5g8eTIbN24kIiKCZcuWXeSQi4iIN1R5hGez2XjllVfYv38/6enpfPrppyxevJjx\n48fTrl07WrduDcDf/vY3EhISOHny5CUF6d+/PwBdunQhJyeHoqIi6tatS0pKChs2bODMmTOcPn2a\nli1butdp1aoV1113HQDt27fnyJEjXHXVVQB06NCBzMxM92NvueUWmjRpAsCQIUPKlOq5duzYQe/e\nvd1HbaWFWCoqKsp9ardz586kpaWVux2n04nT6SyzzOFwEBoa6vGYiIjIf2VlZVFSUlJmWVBQEEFB\nQR6t7/HFo9u0aUObNm0YNmwYAwYMqPR1N4fDwbkfwlBYWFjptm02G/7+/gDY7WcPOktKSvjqq69Y\nsWIFK1eupFGjRmzcuJFVq1a51ytdp3Sf59+uaL+WZVWY//z7zr99/j6Ki4vL3c6SJUtITk4usyws\nLIzU1NRyHy8iIpWLi4vj8OHDZZaNHj2a+Ph4j9avsvCOHj1KVlYWXbp0ASA7O5vjx48TERHBvn37\n+OWXX2jVqhVr166lffv21K9fn+bNm/Pjjz9y5swZLMvi448/djdwQEAA+fn5ZfZx/icUld7Oy8sj\nMDCQhg0bUlRUxHvvvefRN1WeLVu2cPz4cRo3bkxKSgrdu3d35/ntt9/cj4uOjmbRokXk5uYSHBzM\nqlWruOmmmy56fyNHjiQ2NrbMMofDccn5RURMt3z58nKP8DxVZeGVlJSQlJTEkSNH8Pf3x7Isxo0b\nR8eOHZkzZw4TJkygpKSEJk2aMHfuXODsacno6GjuuOMOwsPDiYiIICcnBzhbKIsXL2bQoEHceOON\nTJkypdzX+wB69erF+vXr6devH1dffTUdO3Zk9+7dHn9z54qOjuapp57i0KFDtG7dmsmTJwNw2223\nER8f7/6Ti4ceeojx48dz3333YbfbadasGQkJCRe9v4s5zBYRkapd7ktCRnwAbHJyMidPnuTJJ5/0\ndhRAn4cnInIx9Hl4IiIiF8GITzwfPXq0tyOIiIiX6QhPRESMoMITEREjqPBERMQIKjwRETGCCk9E\nRIygwhMRESOo8ERExAgqPBERMYIRlxYTEREx4korviY3Nx+Xy3d+z2jaNJCcnDxvxyhDmTzni7mU\nyTPK5Bm73UZwcMDlb6casoiIiPg8FZ6IiBhBhSciIkZQ4YmIiBFUeCIiYgQVnoiIGEGFJyIiRlDh\niYiIEVR4IiJiBBWeiIgYQYUnIiJG0LU0vcBut3k7wgWUyTO+mAl8M5cyeUaZqlZdefRpCbUoPz+f\ngIDLvwCqiIiJLvc5VKc0a9GJEye45557yMrK8nYUt6ysLGJiYpSpCr6YCXwzlzJ5Rpk8l5WVxT33\n3MOJEycuazsqvFq2c+dOSkpKvB3DraSkhMOHDytTFXwxE/hmLmXyjDJ5rqSkhJ07d172dlR4IiJi\nBBWeiIgYQYUnIiJGcEyfPn26t0OYxN/fn+7du+Pv7+/tKG7K5BlfzAS+mUuZPKNMnquOXPqzBBER\nMYJOaYqIiBFUeCIiYgQVXi05cOAAQ4cOpW/fvgwdOpTMzEyv5IiJiaF///4MGjSI2NhYvvzySwB2\n7drFwIED6du3L6NGjeLYsWM1liExMZFbb72VyMhI9u/f715e2RjV9PhVlKmi8YKaH7MTJ07w8MMP\n069fPwYOHMiYMWM4fvx4lfuuyVyVZYqMjGTgwIHuscrIyHCvl5qaSr9+/bj99tsZP348hYWF1ZYJ\n4PHHH3fvd/jw4ezbtw/w7pyqLJc35xVAcnJymbnurflUVa5qn1OW1IoRI0ZYGzZssCzLst5//31r\nxIgRXskRExNj7d+//4Llffr0sXbu3GlZlmW9+uqr1lNPPVVjGb7++msrOzvbiomJsTIyMtzLKxuj\nmh6/ijJVNF6WVfNjduLECSs9Pd19OzEx0ZoyZUqV+67JXJVlioyMtE6dOnXBOgUFBdbNN99sZWZm\nWpZlWVOmTLGSk5OrLZNlWVZeXp77608//dSKjY21LMu7c6qyXLfccovX5tX3339vPfjgg9Ytt9zi\nnuvemk9V5aruOaUjvFpw7NgxfvjhBwYMGADAHXfcwd69e92/Gdcmy7Kwznuf0nfffYe/vz9RUVEA\nDB06lE2bNtVYhq5duxISElImR2VjVBvjV14mKH+8oHbGrGHDhtx4443u2126dOHIkSOV7rumc1WU\nCSoeq61bt9KxY0eaNWtWI5mAMtdXzMvLw263e31OlZfL4XC4b3tjXhUVFZGQkMC5b8735nyqLBdU\n/5zSpyXUgqysLEJCQrDZzl7x2263c9VVV5GdnU3jxo1rPc/EiROxLIsbbriBcePGkZWVRVhYmPv+\n0kxOp5OgoKBayVTZGLlcLq+O37njNX78eAICAmp9zCzL4t133+XWW2+tdN+1mas002233QaAzWbj\n3nvvpaSkhJ49exIfH0+dOnUuyHTNNdeQnZ1drVkApk6d6j41uGjRIp+ZU+fnKuWNebVgwQIGDhxY\nZh++MJ/KywXVP6d0hGeYd999l3Xr1rFmzRpcLhcJCQnlPq6836pMdP54zZgxo8LH1uSYJSQk0KBB\nA4YPH37R+66pXKWZ4uLiANiyZQtr1qzh7bffZv/+/bz66qs1st+KzJw5k88//5xx48aRmJgI+MY8\nLi+XN+bVrl27+O6777jnnnuq3HZtzqfycpWq7jmlwqsFoaGhHD161D1RXC4Xv/32G1dffXWtZwkJ\nCQGgTp06DBs2jG+++YZrrrmGw4cPux9z7NgxbDZbrR3dQeVj5M3xK2+8SvPW1pglJiaSmZnJyy+/\nXOW+ayvX+Zngv2PVoEED7r77bvfFfs/PdOTIEUJDQ6s1z7nuuusuduzY4XNzqjTXH3/84ZV5lZ6e\nzi+//MKtt95KTEwMR48e5cEHHyQzM9Or86m8XKNGjWLbtm3VPqdUeLWgSZMmREZGsmHDBgA2bNhA\n+/bta/105qlTp8jPz3ff/uCDD2jfvj0dOnSgsLDQPZlWrFhBv379aiVT6RNOZWPkrfErb7zatWsH\nQMeOHWtlzObPn8/evXt59dVX8fPzq3LftZGrvExOp9P9Lrni4mI+/vhj91j17NmTPXv2uN8FuWLF\nCvr27VtteU6ePFnmdFZqaiqNGjWiSZMmtGvXzmtzqqJc/v7+XplXDz/8MFu3buWzzz4jNTWVkJAQ\n3njjDUaNGuXV+VRRrk6dOlX7nNKVVmrJf/7zHyZPnozT6aRhw4YkJibSsmXLWs1w8OBBxowZg8vl\nwuVyERERwdSpU7nyyivZtWsXzzzzDEVFRYSHhzN37lyaNGlSIzlmzpzJ5s2byc3NpVGjRjRu3JgN\nGzZUOkY1PX7lZXrttdeIj48vd7yAGh+z/fv3c+edd9KyZUv35ZSaNWtGUlIS33zzDdOmTSt33zWZ\n6/xMNpuN8PBwRo0axbRp07Db7RQXFxMVFcXTTz9NvXr1gLNP9nPmzMGyLNq1a8fs2bO54oorqiVT\nbm4ujz32GKdOncJut9OoUSMmTZpEu3btvDqnyss1efJkGjRoUOH/h1Dz86rUrbfeysKFC2nTpk2l\n+6zN54ZfhbmSAAAAQ0lEQVRzc+Xn51f7nFLhiYiIEXRKU0REjKDCExERI6jwRETECCo8ERExggpP\nRESMoMITEREjqPBERMQIKjwRETHC/wNAPMBcj8HpnQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "dftrain['embark_town'].value_counts().plot(kind='barh')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aTn1niLPob3x" - }, - "source": [ - "Females have a much higher chance of surviving vs. males. This will clearly be a predictive feature for the model." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 292 - }, - "colab_type": "code", - "id": "Eh3KW5oYkaNS", - "outputId": "830984b0-0fbc-4d69-b107-9d4097b0549f" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAETCAYAAABjv5J2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGStJREFUeJzt3XtQlXXix/HP4YiMC/IzWVK8bVkm5tqkaWar0oLkjZvX\n0TK1UHdyTVvdzdE2766pmaWMl7ZWWbTZRVcHwcuq4H3Q8tKqG66psaSS4i0uKnDg+f3hykRgHOPA\n+SLv10wzci7P+YAO784jnmOzLMsSAACG8XD3gJogOztbS5cuVXZ2trun3FNN2Cix09XY6VrsdB1X\nbCRQTsjOzlZMTIzxfxhM3yix09XY6VrsdB1XbCRQAAAjESgAgJEIFADASATKSR06dJDdbnf3jHuy\n2+1q2rSp0RsldroaO12Lna5jt9vVoUOHSh3Dxo+ZVyw3N1c+Pj7ungEANU5lvn8SqPtw/XqeiovN\n/XL5+fno6tVcd8+oEDtdi52uxU7X8PCw6aGHvCt1jDou2lIrFBdbRgdKkvH77mKna7HTtdhpBv4O\nCgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICRCBQAwEgECgBgJAIFADASgQIAGIlAAQCM\nRKAAAEYiUAAAIxEoAICRCBQAwEgECgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICRCBQA\nwEgECgBgpDruHlCT+Pn5uHtChfz967t7gm7nO5STfcvdMwDUcATqPkTP2a7L1/nGW5HERZHKcfcI\nADUep/gAAEYiUAAAIxEoAICRCBQAwEgECgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICR\nCBQAwEgECgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICRCBQAwEgECgBgpAc2UDExMVqw\nYIG7ZwAAfqIHNlAAgJqtjrsHlCcwMFBvvvmmdu7cqe+++06zZs1Samqq9u3bJ4fDoQ8//FAtW7bU\nlStXNHHiROXl5amgoEBBQUH6/e9/X+4xP/74Y23fvl0Oh0ONGjXSnDlz5OfnV82fGQDAWcY+g/q/\n//s/rV+/XpMmTdLYsWPVsWNHbdy4UZGRkVqxYoUkydfXVytXrtQ//vEPbdy4USdOnND+/fvLHGvT\npk3KyMhQfHy8NmzYoO7du2vevHnlPm52drbOnz9f6r/MzMwq/VwB4EGVmZlZ5ntqdna2U/c18hmU\nJPXu3VuS1LZtW3l4eKh79+4lH+/cuVOSVFRUpPnz5+vYsWOyLEtXr15VWlqaunbtWupYKSkp+ve/\n/62oqKiS+/n6+pb7uLGxsYqJiSl1WdOmTZWSkuLSz+9B5+9fv1LXm4KdrsVO16oJO19++WVduHCh\n1GXjxo3TG2+8UeF9jQyUzWaTl5eXJMnDw0N169Ytuc5ut8vhcEiSVq1apZycHK1fv16enp6aNm2a\n8vPzyxzPsiy9/vrr6t+/f4WPPWLECPXr16/UZXa7vTKfTq2UlZVzz+v8/ev/6PWmYKdrsdO1TN/p\n4WGTn5+P1q5dq6KiolLX3esJwg8ZGSjLsn7047tycnLk7+8vT09PXbp0ScnJyRo6dGiZ2wUHBysu\nLk49evSQr6+vCgoKdO7cOQUGBpa5ra+vr9NfPADAjwsICPjJ9zUyUDab7Uc/vuuVV17RhAkT1L9/\nfzVu3FhdunQp93aRkZG6ceOGhg0bJpvNpuLiYr300kvlBgoAYAabda+nJygjes52Xb5+y90zjJe4\nKJJTfNWIna7FTte4e4qvUsdw0RYAAFyKQAEAjESgAABGIlAAACMRKACAkQgUAMBIBAoAYCQCBQAw\nEoECABiJQAEAjESgAABGIlAAACMRKACAkQgUAMBIBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAA\nACMRKACAkQgUAMBIBAoAYKQ67h5Qk3zyxxfdPaFGuJ3vcPcEAA8AAnUfrl7NVXGx5e4Z9+TvX19Z\nWTnungEALsEpPgCAkQgUAMBIBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAAACMRKACAkQgUAMBI\nBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAAACMRKACAkQgUAMBITgfq+vXr5V6ekZHhsjEAANzl\ndKDCw8O1Z8+eUpd9+umnGjRokMtHAQBQx9kbzp07V3/84x8VEhKiV199VbNnz9bly5cVGxtblfsA\nALWU08+ggoKClJiYqCNHjqhXr15q0KCB1q9fr8DAwKrcBwCopZwOVF5enubPn6/c3FyNHDlSe/fu\n1caNG6tyGwCgFnM6UJGRkXI4HNq0aZMmT56s2NhYrVmzRmPGjKnKfQCAWsrpQE2cOFELFy5U/fr1\nJUlt2rTR+vXr9eijj1bZOABA7eV0oPr06SNJKi4u1uXLlyVJXl5emjJlStUsAwDUak4HKjs7W5Mm\nTdJTTz2lF198UZKUnJysxYsXV9k4AEDt5XSgpk+fLh8fH6WkpMjT01OS1L59e23durXKxgEAai+n\n/x1Uamqq9u3bJ09PT9lsNklSw4YNdfXq1SobBwCovZx+BlW/fv0yL3d08eJF+fv7u3wUAABOB2rQ\noEEaP368Dh48qOLiYh07dkyTJ0/WkCFDqnIfAKCWcvoU3+jRo1W3bl3NmjVLDodDU6dO1ZAhQzR8\n+PCq3AcAqKWcfgZ16NAhhYSEaMuWLdq+fbvatWunU6dO6cqVK1W5DwBQSzkdqJkzZ8put0uS5s+f\nr6KiItlsNr3zzjtVNg4AUHs5fYrv0qVLatKkiRwOh/bt26ddu3bJ09NT3bp1q8p9AIBayulA+fj4\n6MqVK/rqq6/0+OOPy9vbWwUFBXI4HFW5DwBQSzkdqGHDhmngwIEqLCzU1KlTJUlHjx5Vy5Ytq2wc\nAKD2cjpQY8aMUWhoqOx2u1q0aCFJatSokebMmVNl4wAAtZfTgZJU5pXLeSVzAEBVcfqn+AAAqE4E\nCgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICRCBQAwEgECgBgJAIFADASgQIAGIlAAQCM\nRKAAAEYiUAAAI9ksy7LcPQIAUPPczncoJ/tWudd5eNjk5+dTqePf1xsW1nbRc7br8vXyfzMAoLZJ\nXBSpnCo8Pqf4AABGIlAAACMRKACAkQgUAMBIBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAAACMR\nKACAkQgUAMBIBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAAACMRKACAkQgUAMBIBAoAYCQCBQAw\nUrUGaufOnerTp4/69++v9PT0Kn2sKVOmaO3atVX6GACAqlOnOh/s73//uyZMmKCePXtW58MCAGqg\nagvUvHnzdPjwYaWnp+vTTz/VpEmT9N577ykvL0+SNH78eAUFBenChQsaMGCABg8erH379ik/P18L\nFy7U3/72N/3rX/9SvXr1tGzZMvn5+en06dOaOXOmbt26pYKCAg0ePFjDhw8v89iFhYVavHixDh8+\nrMLCQj3xxBOaMWOG6tWrV12fPgDgPlVboKZMmaIvv/xSo0aNUocOHTR8+HD9+c9/1s9//nNlZWVp\n4MCB2rx5syTpxo0b6tixoyZOnKhPPvlEI0eO1Jo1azR79mzNnDlTa9as0YQJE9SsWTOtXr1anp6e\nunnzpgYNGqSuXbuqZcuWpR77448/lq+vr+Lj4yVJ7733nlasWKHf/e53ZXZmZ2crOzu71GV2u10B\nAQFV9JUBgAdXZmamioqKSl3m6+srX1/fCu9braf47jp69KjOnz+v0aNHy7IsSXci8N///lcNGjSQ\nt7e3unfvLkl68skn1bhxY7Vu3VqS1LZtW6WmpkqSbt26penTp+vUqVPy8PBQVlaWTp06VSZQKSkp\nysvL07Zt2yTdeUYVGBhY7rbY2FjFxMSUuqxp06ZKSUlx3RcAAB4Q/v71f/T6l19+WRcuXCh12bhx\n4/TGG29UeGy3BEqSAgMDFRcXV+byCxcuqG7duiUf2+12eXl5lfrY4XBIkt5//335+/trwYIFstls\nio6OVkFBQZljWpal6dOnq3PnzhXuGjFihPr161fqMrvd7vTnBQC1SVZWTrmXe3jY5Ofno7Vr15b7\nDMoZbvkx8/bt2ys9PV2HDh0quezEiRMlv777rKoiOTk5CggIkM1m0+nTp3X48OFybxccHKxVq1Yp\nPz9fkpSXl6ezZ8+We1tfX181a9as1H+c3gOAnyYgIKDM91RnA1Wtz6BsNpukOxFYvny55s+fr3nz\n5qmgoEAtWrTQihUrSt2uIq+//rreeustbdq0SS1atFCnTp3Kvd2YMWO0dOlSDRw4UDabTR4eHho3\nbpwee+wx13xiAACXs1nOPl2Bouds1+Xrt9w9AwCMkLgossJTfJXBK0kAAIxEoAAARiJQAAAjESgA\ngJEIFADASAQKAGAkAgUAMBKBAgAYiUABAIxEoAAARiJQAAAjESgAgJEIFADASAQKAGAkAgUAMBKB\nAgAYiUABAIxEoAAARiJQAAAjESgAgJEIFADASAQKAGAkAgUAMJLNsizL3SMAADXP7XyHcrJvlXud\nh4dNfn4+lTp+nUrdu5a5ejVXxcXm9tzfv76ysnLcPaNC7HQtdroWO83BKT4AgJEIFADASAQKAGAk\nAgUAMBKBAgAYiUABAIxEoAAARiJQAAAjESgAgJEIFADASAQKAGAkAgUAMBKBAgAYiUABAIxEoAAA\nRiJQAAAjESgAgJEIFADASAQKAGAkAgUAMBKBAgAYiUABAIxEoAAARiJQAAAjESgAgJHquHtATeLh\nYXP3hArVhI0SO12Nna7FzspzxTabZVmWC7Y80HJzc+Xj4+PuGQBQ41Tm+yen+Jxw48YNDR06VJmZ\nme6eck+ZmZkKDg42eqPETldjp2ux03UyMzM1dOhQ3bhx4ycfg0A56ejRoyoqKnL3jHsqKirShQsX\njN4osdPV2Ola7HSdoqIiHT16tFLHIFAAACMRKACAkQgUAMBI9hkzZsxw94iawMvLS507d5aXl5e7\np9xTTdgosdPV2Ola7HSdym7kx8wBAEbiFB8AwEgECgBgJAL1Penp6RoyZIh69eqlIUOGKCMjo8xt\niouLNXPmTIWGhqpnz55at26dcRsPHDigAQMGqF27dlqwYEG17rvLmZ3Lli1TWFiYoqKiNGDAAO3f\nv9/InRs2bFBERISioqIUERGhuLg4I3fede7cOT399NNu+b13ZmdMTIyef/559evXT/369dPs2bON\n3ClJW7ZsUXh4uMLDwxUREaFr164Zt3Py5MmKiopSv379FBUVpTZt2mjXrl1Gbbx27Zp+85vfKCIi\nQn369NGsWbNUXFxc8cEtlBg+fLiVmJhoWZZlJSQkWMOHDy9zm40bN1rR0dGWZVnW1atXre7du1sX\nLlwwamNGRoaVlpZmffDBB9b8+fOrbdv3ObNz//791u3bty3Lsqy0tDSrY8eOVn5+vnE7c3NzS36d\nl5dn/frXv7b+85//VNtGy3Jup2VZVlFRkTVs2DBr0qRJbvm9d2bn0qVL3fbn8i5ndh4/ftzq27ev\ndfXqVcuyLCsnJ8fIP5/fl5aWZnXu3NkqKCiojnmWZTm3ce7cuSW/5w6Hwxo0aJC1devWCo/NM6j/\nuXbtmtLS0tS3b19JUlhYmL788ktdv3691O22bt2qwYMHS5IaNmyoHj16aNu2bUZtbN68uQIDA2W3\n26tl1w85u/NXv/pVyU/3BAYGSlKZ25iw09vbu+TXN2/elMPhkM1WfS/S6exOSfroo48UHBysRx55\npNr23XU/Oy03/myWsztjY2P12muvqWHDhpIkHx8f1a1b17id37d+/XqFh4fL09PTqI02m015eXmy\nLEu3b9+Ww+FQo0aNKjw+gfqfzMxMNWrUqOQbj4eHhx5++GF9++23pW538eJFNWnSpOTjgICAans9\nLGc3uttP2blx40Y1b97cqT+0rnI/O1NSUhQWFqaQkBBFR0erVatWxu08deqUDhw4oJEjR1bbtu+7\nn6/n1q1bFRkZqejoaH3xxRdG7jx79qwyMjI0bNgw9e/fX8uXLzdy512FhYVKSkrSgAEDjNs4duxY\nff311+ratau6deumrl27qn379hUen0DB7T777DMtXbpUixcvdveUewoODlZSUpL++c9/KiEhQenp\n6e6eVIrD4dC0adM0Y8aMan1291MMHTpUycnJSkhIUHR0tMaOHavvvvvO3bPKcDgcOn36tFavXq24\nuDjt3btXCQkJ7p51Tzt27FCTJk1KzkaYZNu2bQoMDNSBAwe0d+9effbZZ9q+fXuF9yNQ/xMQEKBL\nly6VnHooLi7W5cuX1bhx41K3a9KkiS5evFjycWZmpgICAoza6G73s/PYsWOaPHmyli1bpl/84hfG\n7ryrcePGateunXbv3l1NK53bmZWVpW+++UZjxoxRcHCwYmNjtW7dOk2bNs2onZLk5+dXcvr5+eef\nV+PGjfXVV18Zt7Np06bq2bOn6tSpI29vb4WEhOjEiRPG7bxrw4YN1frsSXJ+45o1axQeHi7pzqnS\nkJAQHTp0qMLjE6j/adiwoQIDA5WYmChJSkxM1JNPPqmHHnqo1O169eql+Ph4WZala9euKTk5WS++\n+KJRG7/PHef6nd15/PhxTZw4UR9++KFb/q/P2Z3nzp0r+fW1a9d06NAhPfHEE0btDAgIUGpqqpKT\nk5WSkqIRI0Zo0KBBmjVrllE7JenSpUslv05LS9PFixf16KOPGrczLCxMBw4ckHTn9Flqaqpat25t\n3E5J+vbbb3XkyJGSCJi2sVmzZtq3b58kqaCgQKmpqc6dJq/0j3A8QM6ePWsNGjTI6tmzpzV48GAr\nPT3dsizLGj16tHXy5EnLsu78lNT06dOtHj16WKGhoVZ8fLxxGw8fPmx1797deuaZZ6wOHTpYQUFB\n1v79+43bOWDAAKtLly5WVFSUFRkZaUVFRVmnT582buef/vQnq2/fviU716xZU60bnd35fe76STln\ndk6ePNkKCwuzIiIirIEDB1p79+41cmdxcbE1b948q3fv3lZYWJj17rvvGrnTsixr+fLl1sSJE6t9\nn7MbMzIyrFdffdUKDw+3+vbta82ePdsqKiqq8Ni81BEAwEic4gMAGIlAAQCMRKAAAEYiUAAAIxEo\nAICRCBQAwEgECqhlVq5cqXfeecfdM4AK8e+gABeaO3euEhIS1LJlSy1ZskQPP/ywJGnTpk06efKk\npk6d6uaFQM3BMyjARY4fP660tDQdOHBAHTp00EcffSRJysnJ0erVqzV+/Phq2eHUG8EBNQCBAlzk\n/PnzeuaZZ+Tp6akuXbrom2++kSR98MEHGjVqlHx8fH70/nv27FHfvn3VoUMHBQUFadWqVZLuvBXJ\nSy+9VOq2gYGBJcefMmWKZsyYoTFjxqh9+/ZauXKlunbtWup1GHfs2KHIyEhJd97R9q233pIkjRo1\nSmvXri117MjISO3cuVPSnbeceO2119S5c2f17t1bW7du/alfHuC+ESjARVq1aqXDhw8rPz9fqamp\nevzxx3Xy5Emlp6erT58+Fd7/7bff1uzZs3X06FElJSXpueeeK7nuh2+h8cOPN2/erLFjx+rYsWOK\njo7Wz372Mx08eLDk+qSkpHJfSDQsLExJSUklH585c0aZmZl64YUXdOvWLUVHRysiIkIHDx7U+++/\nr1mzZuns2bNOf02AyiBQgIu0atVKoaGhGjx4sC5duqRRo0Zp7ty5evvtt/XXv/5Vw4YN0x/+8Afl\n5uaWe/+6devqzJkzys3NVf369dWmTZt7PtYP/+o4JCRETz/9dMlx+vTpU/IK07m5udq7d6/CwsLK\nHCc0NFSnTp0qedPNxMREhYaGqk6dOtq1a5eaNWumqKgo2Ww2tWnTRqGhodX2DtIAgQJcaOTIkUpI\nSNCiRYu0ZcsWderUScXFxVq3bp1iY2PVsmVLrVy5stz7LlmyRLt371ZwcLBeeeWV+3qn2R++/054\neLh27typwsJC7dixQ23bti33fYS8vb0VFBSkzZs3S5K2bNmiiIgISXfePfqLL77Qs88+q2effVad\nOnVSUlKSrly54vQuoDLquHsA8CC6cuWK4uPjFR8fr+TkZLVu3Vp2u13t2rVTXFxcuff55S9/qWXL\nlqmoqEhxcXF68803tXv3btWrV0+3bt0quV1WVlaZ+/7wlN9jjz2mJk2aaM+ePUpKSir32dNdYWFh\niomJUceOHZWfn6/OnTtLuvMeU507d9Ynn3zyU74EQKXxDAqoAu+++67Gjx8vLy8vNWvWTCdOnNDN\nmzd16NAhNW/evMztCwsLlZiYqNzcXNntdnl7e5e862xgYKDOnDmjU6dOqaCgQDExMU69rXtYWJji\n4uJ05MgR9erV65636969uy5evKglS5aU+ruyF154QV9//bUSEhLkcDhUWFioEydO8HdQqDYECnCx\nQ4cOKTc3VyEhIZKkp556SkFBQXrhhRf0+eefa/To0eXeLyEhQSEhIerYsaPi4+O1cOFCSdIjjzyi\n3/72txo5cqR69uypjh07OrWjb9+++vzzz/Xcc8+pQYMG97xd3bp1FRoaqtTU1FLPtLy9vfWXv/xF\nW7ZsUbdu3dStWzctWrRIhYWFzn4pgErhH+oCAIzEMygAgJEIFADASAQKAGAkAgUAMBKBAgAYiUAB\nAIxEoAAARiJQAAAjESgAgJH+HwQ8SRjqGoVPAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "pd.concat([dftrain, y_train], axis=1).groupby('sex').survived.mean().plot(kind='barh').set_xlabel('% survive')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "krkRHuMp3rJn" - }, - "source": [ - "## Create feature columns and input functions\n", - "The Gradient Boosting estimator can utilize both numeric and categorical features. Feature columns work with all TensorFlow estimators and their purpose is to define the features used for modeling. Additionally they provide some feature engineering capabilities like one-hot-encoding, normalization, and bucketization. In this tutorial, the fields in `CATEGORICAL_COLUMNS` are transformed from categorical columns to one-hot-encoded columns ([indicator column](https://www.tensorflow.org/api_docs/python/tf/feature_column/indicator_column)):" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "upaNWxcF3rJn" - }, - "outputs": [], - "source": [ - "fc = tf.feature_column\n", - "CATEGORICAL_COLUMNS = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',\n", - " 'embark_town', 'alone']\n", - "NUMERIC_COLUMNS = ['age', 'fare']\n", - "\n", - "def one_hot_cat_column(feature_name, vocab):\n", - " return tf.feature_column.indicator_column(\n", - " tf.feature_column.categorical_column_with_vocabulary_list(feature_name,\n", - " vocab))\n", - "feature_columns = []\n", - "for feature_name in CATEGORICAL_COLUMNS:\n", - " # Need to one-hot encode categorical features.\n", - " vocabulary = dftrain[feature_name].unique()\n", - " feature_columns.append(one_hot_cat_column(feature_name, vocabulary))\n", - "\n", - "for feature_name in NUMERIC_COLUMNS:\n", - " feature_columns.append(tf.feature_column.numeric_column(feature_name,\n", - " dtype=tf.float32))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "74GNtFpStSAz" - }, - "source": [ - "You can view the transformation that a feature column produces. For example, here is the output when using the `indicator_column` on a single example:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 51 - }, - "colab_type": "code", - "id": "Eaq79D9FtmF8", - "outputId": "d6db9dc1-a533-46c7-b78a-8ce367056a59" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Feature value: \"Third\"\n", - "One-hot encoded: [[ 0. 0. 1.]]\n" - ] - } - ], - "source": [ - "example = dict(dftrain.head(1))\n", - "class_fc = tf.feature_column.indicator_column(tf.feature_column.categorical_column_with_vocabulary_list('class', ('First', 'Second', 'Third')))\n", - "print('Feature value: \"{}\"'.format(example['class'].iloc[0]))\n", - "print('One-hot encoded: ', tf.keras.layers.DenseFeatures([class_fc])(example).numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YbCUn3nCusC3" - }, - "source": [ - "Additionally, you can view all of the feature column transformations together:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 102 - }, - "colab_type": "code", - "id": "omIYcsVws3g0", - "outputId": "165d15ee-4e7f-481d-f623-106f5dbc9d23" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[ 22. , 1. , 0. , 1. , 0. , 0. , 1. , 0. ,\n", - " 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. ,\n", - " 0. , 0. , 7.25, 1. , 0. , 0. , 0. , 0. ,\n", - " 0. , 0. , 1. , 0. , 0. , 0. , 0. , 0. ,\n", - " 1. , 0. ]], dtype=float32)" - ] - }, - "execution_count": 0, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "tf.keras.layers.DenseFeatures(feature_columns)(example).numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-UOlROp33rJo" - }, - "source": [ - "Next you need to create the input functions. These will specify how data will be read into our model for both training and inference. You will use the `from_tensor_slices` method in the [`tf.data`](https://www.tensorflow.org/api_docs/python/tf/data) API to read in data directly from Pandas. This is suitable for smaller, in-memory datasets. For larger datasets, the tf.data API supports a variety of file formats (including [csv](https://www.tensorflow.org/api_docs/python/tf/data/experimental/make_csv_dataset)) so that you can process datasets that do not fit in memory." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9dquwCQB3rJp" - }, - "outputs": [], - "source": [ - "# Use entire batch since this is such a small dataset.\n", - "NUM_EXAMPLES = len(y_train)\n", - "\n", - "def make_input_fn(X, y, n_epochs=None, shuffle=True):\n", - " def input_fn():\n", - " dataset = tf.data.Dataset.from_tensor_slices((dict(X), y))\n", - " if shuffle:\n", - " dataset = dataset.shuffle(NUM_EXAMPLES)\n", - " # For training, cycle thru dataset as many times as need (n_epochs=None).\n", - " dataset = dataset.repeat(n_epochs)\n", - " # In memory training doesn't use batching.\n", - " dataset = dataset.batch(NUM_EXAMPLES)\n", - " return dataset\n", - " return input_fn\n", - "\n", - "# Training and evaluation input functions.\n", - "train_input_fn = make_input_fn(dftrain, y_train)\n", - "eval_input_fn = make_input_fn(dfeval, y_eval, shuffle=False, n_epochs=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HttfNNlN3rJr" - }, - "source": [ - "## Train and evaluate the model\n", - "\n", - "Below you will do the following steps:\n", - "\n", - "1. Initialize the model, specifying the features and hyperparameters.\n", - "2. Feed the training data to the model using the `train_input_fn` and train the model using the `train` function.\n", - "3. You will assess model performance using the evaluation set—in this example, the `dfeval` DataFrame. You will verify that the predictions match the labels from the `y_eval` array.\n", - "\n", - "Before training a Boosted Trees model, let's first train a linear classifier (logistic regression model). It is best practice to start with simpler model to establish a benchmark." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 221 - }, - "colab_type": "code", - "id": "JPOGpmmq3rJr", - "outputId": "0b6bd66c-90fb-40b6-ed15-7130ff63627e" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "accuracy 0.765152\n", - "accuracy_baseline 0.625000\n", - "auc 0.832844\n", - "auc_precision_recall 0.789631\n", - "average_loss 0.478908\n", - "global_step 100.000000\n", - "label/mean 0.375000\n", - "loss 0.478908\n", - "precision 0.703297\n", - "prediction/mean 0.350790\n", - "recall 0.646465\n", - "dtype: float64\n" - ] - } - ], - "source": [ - "linear_est = tf.estimator.LinearClassifier(feature_columns)\n", - "\n", - "# Train model.\n", - "linear_est.train(train_input_fn, max_steps=100)\n", - "\n", - "# Evaluation.\n", - "result = linear_est.evaluate(eval_input_fn)\n", - "clear_output()\n", - "print(pd.Series(result))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BarkNXwA3rJu" - }, - "source": [ - "Next let's train a Boosted Trees model. For boosted trees, regression (`BoostedTreesRegressor`) and classification (`BoostedTreesClassifier`) are supported. Since the goal is to predict a class - survive or not survive, you will use the `BoostedTreesClassifier`.\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 221 - }, - "colab_type": "code", - "id": "tgEzMtlw3rJu", - "outputId": "2ac2d0f9-a7e9-46aa-d8f5-569a65c32251" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "accuracy 0.829545\n", - "accuracy_baseline 0.625000\n", - "auc 0.872788\n", - "auc_precision_recall 0.857807\n", - "average_loss 0.411839\n", - "global_step 100.000000\n", - "label/mean 0.375000\n", - "loss 0.411839\n", - "precision 0.793478\n", - "prediction/mean 0.381942\n", - "recall 0.737374\n", - "dtype: float64\n" - ] - } - ], - "source": [ - "# Since data fits into memory, use entire dataset per layer. It will be faster.\n", - "# Above one batch is defined as the entire dataset.\n", - "n_batches = 1\n", - "est = tf.estimator.BoostedTreesClassifier(feature_columns,\n", - " n_batches_per_layer=n_batches)\n", - "\n", - "# The model will stop training once the specified number of trees is built, not\n", - "# based on the number of steps.\n", - "est.train(train_input_fn, max_steps=100)\n", - "\n", - "# Eval.\n", - "result = est.evaluate(eval_input_fn)\n", - "clear_output()\n", - "print(pd.Series(result))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hEflwznXvuMP" - }, - "source": [ - "Now you can use the train model to make predictions on a passenger from the evaluation set. TensorFlow models are optimized to make predictions on a batch, or collection, of examples at once. Earlier, the `eval_input_fn` is defined using the entire evaluation set." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 289 - }, - "colab_type": "code", - "id": "6zmIjTr73rJ4", - "outputId": "f934b588-cc13-4b5b-e9d1-5253606fa40d" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEQCAYAAABIqvhxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtUVOX+BvBnGG4mjAIHcQCttJLMo0imcTTNuybe8qCk\nKJrXVHTVMTMTITUN8ZJB6tFc65B5NPVkOhXkOdDFW2p5wZZaaSoGAxhgY3KTmff3hz8GJ1D2Zmb2\nDPh81nIt5519+c43nKf97gsqIYQAERGRDC6OLoCIiBoehgcREcnG8CAiItkYHkREJBvDg4iIZGN4\nEBGRbAwPapBef/11rFu3DgDw3XffYfDgwYrsNyQkBFevXlVkX3369MGRI0fqte748eOxe/fuWt/T\n6/UICwtD1VX6dy6r0+kwefLku25XyV6Tc2N4UIPXpUsXpKWl1bncnj17MHbsWKv2pVKprFrfGWi1\nWpw4caLWzzJ06FBs2bLF/PrPYSm119T4MTzI4YxGoyL7EUJY/eVvq3tqlfrM1moMYUn2wfAgu+jT\npw82bdqEIUOGoFu3bli4cCEqKioAAMeOHUOvXr2wefNm9OjRAwsXLgQAfPnllxgxYgSeeuopvPDC\nC/jxxx/N2zt79iyef/55PPnkk3j55ZdRXl5ufq9qe1Xy8vIQGxuL8PBwPP3001i2bBkuXryIhIQE\nnDp1Cp07d0bXrl0BABUVFUhMTETv3r3Ro0cPJCQkmOsEgPfffx89evRAz5498Z///OeeX6bjx4/H\nmjVrEBkZiS5dumDWrFkwGAwAgJycHISEhGD37t3o3bs3Jk6cCADIyMhAREQEunbtigkTJuDixYsW\n2zxz5kytPTQYDJgxYwbCw8PRrVs3zJgxA/n5+RbrZmdn37MWk8lU4zPceXQWHR0NIQSGDRuGsLAw\npKWl1eh1QUEB5syZg/DwcPTr1w9bt241v5eVlYVRo0bhySefRI8ePZCYmHjX3lEDJIjsoHfv3iIi\nIkLk5eWJ33//XURFRYl33nlHCCHE0aNHRfv27cXq1atFRUWFKC8vFz/88IMIDw8XWVlZwmQyiT17\n9ojevXuLiooKUVFRIXr37i1SU1NFZWWlSE9PF0888YTF9nr16iWEEMJoNIphw4aJt99+W5SVlYny\n8nLx/fffCyGE+Pjjj8XYsWMt6ly2bJl46aWXhMFgEDdv3hQzZswQa9asEUII8fXXX4vu3buLCxcu\niNLSUvHKK6+IkJAQkZ2dXetnjo6OFj179jQvHxsbK+bNmyeEEOLXX38V7dq1E6+99pooLS0V5eXl\n4tKlSyI0NFQcPnxYVFZWis2bN4v+/fuLW7du1dnD4uJisX//flFeXi5u3rwp5s6dK2bOnCm5lpCQ\nEGE0Gs3L7tq1q9YetWvXzuLz3tlrk8kkRo4cKdavXy8qKyvF1atXRb9+/cTBgweFEEKMGTNG7N27\nVwghRElJiTh9+rSEnxxqKHjkQXYzfvx4BAQEQKPRYMaMGfjss8/M77m4uCA2NhZubm5wd3fHrl27\nEBUVhb/+9a9QqVQYMWIE3N3dcfr0aZw+fRqVlZWYMGEC1Go1Bg4ciA4dOtS6z9OnT+PatWt49dVX\n4eHhAXd3d4SFhd21xt27d+P111+Ht7c3HnjgAUybNg2ffvopACA9PR3PP/882rZtC09PT8TGxtb5\nmYcPH25efu7cuUhLSzNPdalUKsTGxsLT0xPu7u74/PPP8eyzzyI8PBxqtRqTJ09GWVkZTp48WWcP\nmzdvjv79+8Pd3R0PPPAApk+fju+++05yLbaQlZWF69ev46WXXoJarUZwcDAiIyPNNbq6uiI7OxvF\nxcVo0qQJOnbsaLN9k+O5OroAarwCAgLMfw8KCkJBQYH5ta+vL9zc3Myvc3NzsXfvXnz44YcAbp9b\nqKysNK9z57aqtlebvLw8BAYGwsWl7v8vKioqQmlpKUaNGmUeM5lM5i/YgoICi5AKDAys88u3ZcuW\nFstXVlaiuLi41vcLCgoQGBhofq1SqaDVai2mn+7Ww7KyMixfvhwHDx6EwWCAEAIlJSUW53XqqsVa\nubm5yM/PN08BCiFgMpnw1FNPAQCWL1+OdevWYfDgwWjVqhVmzZqFZ5991mb7J8dieJDd5OXlmf+e\nk5ODFi1amF//+dxBy5YtMWPGDEyfPr3Gdo4fP15jPj83NxetW7eusaxWq4Ver4fJZKoRIH/ep4+P\nD5o0aYJPP/3UorYq/v7+Fp8hNze3zhPIf17ezc0NPj4+KC0trVFDixYt8PPPP1usr9frLb7079bD\nLVu24PLly9i9ezd8fX1x/vx5jBw50iI86qrFWlqtFsHBwfjiiy9qfb9169ZYvXo1AOCLL77AnDlz\ncOzYMXh6etpk/+RYnLYiu9m2bRvy8/Nx/fp1bNq0Cc8999xdlx09ejR27NiBrKwsAEBJSQm+/vpr\nlJSUIDQ0FK6urti6dSuMRiP279+PM2fO1Lqdjh07wt/fH6tWrUJpaSkqKipw4sQJAICfnx/y8vJw\n69YtALe/yCMjI7F8+XIUFRUBAPLz83Hw4EEAwODBg/Hxxx/j4sWLKC0txXvvvVfnZ963b595+Xff\nfReDBg0yf5n/+ahl8ODB+Oqrr/Dtt9+isrISW7ZsgYeHB0JDQ+vsYUlJCTw9PeHl5YXr168jOTnZ\nqlru5i9/+ctd72vp2LEjvLy8sHnzZpSXl8NoNOLnn382/7fZt2+fua/e3t5QqVSSjgipYWjw/yUN\nBgOSk5PNV5Lcz5ytFxEREXjxxRcxYMAAtG7dGi+99NJdl+3QoQOWLl2KJUuWoGvXrhg4cCD27NkD\nAHBzc0NycjI+/vhjdO3aFenp6RgwYECt23FxccHGjRtx8eJFPP300+jZs6f5voSnn34ajz76KHr0\n6IHw8HAAwLx58/Dggw9i9OjR6NKlC1588UVcvnwZANCzZ0/ExMQgJiYGAwcONK9zL8OHD8eCBQvw\nzDPP4NatW3jjjTfM7/35qOXhhx9GUlISli5divDwcHz11VfYuHEjXF1dzcvfrYcxMTEoLS1Ft27d\nEBUVZXEFVNW6VbX06NEDP/74I+bMmVNrLfc6moqNjcX8+fPNfa+t1+fPn0ffvn3xt7/9DXFxcfjj\njz8AAAcOHEBERATCwsKwYsUKrF27Fu7u7nX20J6c7d+II1ndCyXPzpeXl4v4+HgxYMAAMXToUBEX\nFyeEEOLSpUtizJgxYuDAgWLMmDHiypUrkrd59epV8dhjj4mrV6/aq+wGw5l60bt3b3H48GGH7d8R\nvbjzqiVn4kw/F47GXlSztheKnvNYuXIlPD09zXOkVYe08fHxiI6ORkREBPbt24e4uDikpqYqWRoR\nEcmg2LRVSUkJ9u7di7lz55rHfH19UVRUhHPnzmHIkCEAbk91nD171qZXhZDy7sc7k+/Hz0z3L8WO\nPLKzs9G8eXMkJyfj6NGjaNq0KebOnQtPT08EBASY/+G5uLigRYsWyMvLg4+Pj1LlkY1lZGQ4ugTF\nffDBB44ugUgxioWH0WjE1atX0aFDB8yfPx9ZWVmYMWMG1q1bJ/nKD4PBUOPkTl5eHsLCwqBWq+1R\ndoOiVqsRFBTEXoC9uBN7UY29qKZWqxEWFmZxSXcVjUYDjUZzz/VVQuo3t5WKi4vxzDPP4IcffjCP\nRUREYPny5ZgyZQqOHj0KlUoFk8mEbt26Yf/+/TWOPJKTk5GSkmIxFhYWhu3btyvxEYiIGp0XXnjB\nfDl7ldmzZ9f5RAXFjjx8fHzQrVs3HDp0CN27d8elS5dQWFiINm3aICQkBDqdDsOGDYNOp0P79u1r\nnbKKiYnByJEjLcaq/g+iuPgmTCbb5aCfnxcmL9tf7/W3LBqAwsI/bFaPVH5+Xg7ZrzNiL6qxF9XY\ni9tcXFTw8WmKNWvW1HjKc11HHYDCd5gnJCRg4cKFePvtt+Hm5oakpCR4eXkhISEBCxYswPr169Gs\nWbO7Pn3zXodSJpOwaXgAQEGxdXfi2roeZ9+vM2IvqrEX1diLalqttl7rKRoerVq1snhkc5U2bdpg\n586dSpZCRERWaPB3mBMRkfIYHkREJBvDg4iIZGN4EBGRbAwPIiKSjeFBRESyMTyIiEg2hgcREcnG\n8CAiItkYHkREJBvDg4iIZGN4EBGRbAwPIiKSjeFBRESyMTyIiEg2hgcREcnG8CAiItkYHkREJBvD\ng4iIZGN4EBGRbAwPIiKSjeFBRESyMTyIiEg2hgcREcnG8CAiItkYHkREJBvDg4iIZGN4EBGRbK5K\n7qxPnz7w9PSEu7s7VCoV5s2bh+7du+PUqVOIj49HeXk5goKCkJSUBF9fXyVLIyIiGRQND5VKheTk\nZLRt29ZifP78+UhMTETnzp2xYcMGrFq1CsuXL1eyNCIikkHRaSshBIQQFmNnzpyBh4cHOnfuDACI\niopCWlqakmUREZFMih55AMC8efMghMCTTz6Jl19+GXq9HkFBQeb3fXx8AAAGgwEajUbp8oiISAJF\nw2P79u0ICAjArVu38NZbb2HJkiXo379/jeX+fHRSxWAwwGAwWIyp1WpotVq71EtE1Njp9XoYjUaL\nMY1GU+f/vCsaHgEBAQAANzc3jB07FjNnzkRMTAxycnLMyxQVFUGlUtVaeGpqKlJSUizGgoKCkJmZ\nCT8/L/sWXw/+/t731X6dEXtRjb2oxl5UGzdunMV3MADMnj0bsbGx91xPsfAoLS2F0WiEl9ftL/nP\nPvsM7du3xxNPPIHy8nKcOHECYWFh2LFjBwYPHlzrNmJiYjBy5EiLMbVaDQAoLPwDJlPtRyz1YYsf\nrmvXbtigEnn8/b0dsl9nxF5UYy+qsRe3ubio4OfnhW3bttV65FEXxcLjt99+w5w5c2AymWAymdC2\nbVssXrwYKpUKK1euRFxcHCoqKhAcHIykpKRatyHlUIqIiKSr77S/YuHRqlUr7Nmzp9b3QkNDodPp\nlCqFiIisxDvMiYhINoYHERHJxvAgIiLZGB5ERCQbw4OIiGRjeBARkWwMDyIiko3hQUREsjE8iIhI\nNoYHERHJxvAgIiLZGB5ERCQbw4OIiGRjeBARkWwMDyIiko3hQUREsjE8iIhINoYHERHJxvAgIiLZ\nGB5ERCQbw4OIiGRjeBARkWwMDyIiko3hQUREsjE8iIhINoYHERHJxvAgIiLZGB5ERCQbw4OIiGRz\nSHikpKQgJCQEFy5cAACcOnUKw4cPx6BBgzB58mQUFRU5oiwiIpJI8fA4e/YsTp8+jcDAQPPY/Pnz\nkZCQgPT0dHTp0gWrVq1SuiwiIpJB0fCoqKjAkiVLkJCQYB47c+YMPDw80LlzZwBAVFQU0tLSlCyL\niIhkclVyZ++++y6GDx+OoKAg85her7d47ePjAwAwGAzQaDQW6xsMBhgMBosxtVoNrVZrx6qJiBov\nvV4Po9FoMabRaGp8//6ZYuFx6tQpnDlzBvPmzTOPCSFqXfZu46mpqUhJSbEYCwoKQmZmJvz8vGxX\nrI34+3vfV/t1RuxFNfaiGntRbdy4ccjJybEYmz17NmJjY++5nmLhcezYMVy6dAl9+/aFEAL5+fmY\nMmUKxo8fb1F4UVERVCpVrakXExODkSNHWoyp1WoAQGHhHzCZag+d+rDFD9e1azdsUIk8/v7eDtmv\nM2IvqrEX1diL21xcVPDz88K2bdtqPfKoi2LhMW3aNEybNs38uk+fPti8eTPatGmDnTt34sSJEwgL\nC8OOHTswePDgWrch5VCKiIikq++0v6LnPO6kUqkghIBKpcLKlSsRFxeHiooKBAcHIykpyVFlERGR\nBA4Lj4yMDPPfQ0NDodPpHFUKERHJxDvMiYhINoYHERHJxvAgIiLZGB5ERCSb5PD44IMP+MBCIiIC\nICM8Dh8+jL59+2L69On4/PPPUVFRYc+6iIjIiUkOj40bNyIzMxM9e/ZEamoqunfvjjfeeAPHjx+3\nZ31EROSEZJ3z8PHxwbhx4/DRRx9h69atOHPmDCZMmIA+ffpgw4YNuHnzpr3qJCIiJyL7JsEjR45g\n3759yMjIQIcOHTBlyhQEBgbigw8+wNSpU/Hvf//bHnUSEZETkRweiYmJ+Oyzz+Dt7Y3hw4dDp9Mh\nICDA/H6nTp3QtWtXuxRJRETORXJ4lJeXIyUlBR07dqz1fTc3N+zevdtmhRERkfOSHB7Tp0+Hp6en\nxdjvv/+OsrIy8xFI27ZtbVsdERE5JcknzGfOnIm8vDyLsby8PMyePdvmRRERkXOTHB6XLl1Cu3bt\nLMbatWuHX375xeZFERGRc5McHn5+frhy5YrF2JUrV9C8eXObF0VERM5NcniMGjUKsbGx+PLLL3Hh\nwgVkZmZizpw5iIyMtGd9RETkhCSfMJ82bRpcXV2RmJiIvLw8tGzZEpGRkZg0aZI96yMiIickOTxc\nXFwwZcoUTJkyxZ71EBFRAyDrDvNffvkF58+fR0lJicX43//+d5sWRUREzk1yeGzcuBHvvfceQkJC\nLO73UKlUDA8iovuM5PBITU3Frl27EBISYs96iIioAZB8tZWnpyfatGljz1qIiKiBkBwec+fOxbJl\ny1BQUACTyWTxh4iI7i+Sp60WLFgAANi1a5d5TAgBlUqFc+fO2b4yIiJyWpLDIyMjw5512IyLiwqu\n7q4Qov7bUKlsVw8RUWMkOTyCgoIAACaTCb/99htatGhht6KsoVa74EbpLWz4OKve25gTGYpmNqyJ\niKixkRweBoMBb775Jr744gu4urri1KlTyMjIQFZWFl5++WV71ihbSVklsn7+rd7rl5ZX2rAaIqLG\nR/IJ8/j4eHh5eSEzMxNubm4AgM6dOyMtLc1uxRERkXOSfORx5MgRHDhwAG5ublD9/0kBX19fFBYW\nSt7ZrFmzkJOTA5VKhaZNm2LRokUICQnB5cuXsWDBAly/fh3NmzfHypUr0bp1a/mfhoiIFCH5yMPb\n2xvFxcUWY7m5ufD395e8s8TERHzyySfYs2cPJk2ahIULFwK4fVQTHR2N9PR0jB07FnFxcZK3SURE\nypMcHpGRkZgzZw6+/fZbmEwmnDx5Eq+99hqioqIk78zLy8v89xs3bsDFxQVFRUU4d+4chgwZAgCI\niIjA2bNnawQVERE5D8nTVlOnToW7uzuWLFmCyspKLFy4EGPGjEFMTIysHS5atAiHDh0CALz//vvQ\n6/UICAgwT4W5uLigRYsWyMvLg4+Pj8W6BoMBBoPBYkytVkOr1cqqgYiIbtPr9TAajRZjGo0GGo3m\nnutJDg+VSoWJEydi4sSJ9SqwyrJlywAA+/btQ2JiIubOnQsh8aaM1NRUpKSkWIwFBQUhMzMTfn7V\nRzW/3aiwqkaVi21u9PD397bJdhrKfp0Re1GNvajGXlQbN24ccnJyLMZmz56N2NjYe64n64T53YSH\nh0vdjNmwYcMQFxcHrVaL/Px8893qJpMJBQUFaNmyZY11YmJiMHLkSIsxtVoNACgs/AMmk4Cbmxom\na+4QBCBM1q1f5dq1GzbZjhz+/t4O2a8zYi+qsRfV2IvbXFxU8PPzwrZt22o98qiL5PB44403LF4X\nFxfj1q1bCAgIkHT3eUlJCQwGgzkUMjMz0bx5c/j6+uLxxx+HTqfDsGHDoNPp0L59+xpTVoC0Qyki\nIpKuvtP+ksMjMzPT4rXRaMSGDRvQtGlTSeuXlpZi7ty5KC0thYuLC5o3b46NGzcCABISErBgwQKs\nX78ezZo1Q2JiooyPQERESpP1mwTvpFarMWPGDPTq1UvS7zH38/PDRx99VOt7bdq0wc6dO+tbChER\nKUzypbq1OXTokPkqKSIiun9IPvLo1auXRVCUlpaioqIC8fHxdimMiIicl+TwSEpKsnjdpEkTPPzw\nwxY3/hER0f1Bcnh07drVnnU0OhW3jFZfS15WXokbhlIbVUREZDuSw+PVV1+VdH5j5cqVVhXUWLi7\nqTH0H3ut2oZu9XDwanQickaST5hrNBr873//g9FoRMuWLWEymZCRkQGNRoPWrVub/xARUeMn+cjj\n8uXL2LRpE7p06WIe++6777BhwwZs2bLFLsUREVHtvDVN4OlR77stUG7lL72TvOdTp06hU6dOFmOd\nOnXCyZMnrSqAiIjk8/RwtWpqXLd6uFX7lzxt1b59e6xZswZlZWUAgLKyMqxduxaPP/64VQUQEVHD\nI/nIY8WKFZg3bx66dOkCjUYDg8GADh061LiEl4iIGj/J4REcHIwdO3ZAr9ejoKAA/v7+CAwMtGdt\nRETkpGQ9nqS4uBhHjx7FsWPHEBgYiPz8fOTl5dmrNiIiclKSw+PYsWMYNGgQdDod1q9fDwC4cuUK\nEhIS7FUbERE5KcnhsXz5crzzzjvYsmULXF1vz3Z16tQJWVlZdiuOiIick+TwyMnJMf/GwKo7zd3c\n3Gr8BioiImr8JIdH27ZtceDAAYuxw4cP47HHHrN5UURE5NwkX221YMECTJ8+Hc8++yzKysqwePFi\nZGZmms9/EBHR/UPykUdoaCj27duHRx55BKNGjUJwcDB2796Njh072rM+IiJyQpKOPIxGIyZOnIgt\nW7Zg6tSp9q6JiIicnKQjD7VajV9//RUmk8ne9RARUQMgedpq1qxZSEhIQE5ODoxGI0wmk/kPERHd\nXySfMF+0aBEA4JNPPjFfqiuEgEqlwrlz5+xTHREROaU6w+PatWvw9/dHRkaGEvUQEVEDUOe01cCB\nAwEAQUFBCAoKwooVK8x/r/pDRET3lzrDQwhh8frYsWN2K4aIiBqGOsOj6vwGERFRlTrPeRiNRnz7\n7bfmI5DKykqL1wDMz7wiIqL7Q53h4efnh4ULF5pfN2/e3OK1SqXiyXQiovtMneGRmZlpkx1dv34d\n8+fPx9WrV+Hu7o4HH3wQb775Jnx8fHDq1CnEx8ejvLwcQUFBSEpKgq+vr032S0REtifrNwlaQ6VS\nYerUqUhLS8PevXsRHByM1atXAwDmz5+PhIQEpKeno0uXLli1apVSZRERUT0oFh7NmjXDU089ZX4d\nGhqK3NxcnDlzBh4eHujcuTMAICoqCmlpaUqVRURE9SD5DnNbEkJg+/bt6Nu3L/R6vcW9Ij4+PgAA\ng8EAjUZjsZ7BYIDBYLAYU6vV0Gq19i+aiKgR0uv1NX6pn0ajqfH9+2cOCY8lS5agadOmiI6Oxv79\n+2u8/+d7S6qkpqYiJSXFYiwoKAiZmZnw8/Myj/12o8Kq+lQuznN5sr+/tyLrNFbsRTX2ohp7UW3c\nuHHIycmxGJs9ezZiY2PvuZ7i4ZGYmIjs7Gz885//BABotVqLwouKiqBSqWpNvZiYGIwcOdJiTK1W\nAwAKC/+AySTg5qaG6S7hI5UwWbe+LV27dkPW8v7+3rLXaazYi2rsRbXG0gtbBeC2bdtqPfKoi6Lh\nsXbtWpw9exabNm2Cq+vtXXfo0AHl5eU4ceIEwsLCsGPHDgwePLjW9aUcShERkXT1nfZXLDwuXLiA\nTZs24aGHHsKYMWMAAK1atUJycjISExOxePFiVFRUIDg4GElJSUqVRURE9aBYeDzyyCN3fXR7586d\nodPplCqFiIispNilukRE1HgwPIiISDaGBxERycbwICIi2RgeREQkG8ODiIhkY3gQEZFsDA8iIpKN\n4UFERLIxPIiISDaGBxERycbwICIi2RgeREQkG8ODiIhkY3gQEZFsDA8iIpKN4UFERLIxPIiISDaG\nBxERycbwICIi2RgeREQkm6ujC6C7q7hlhL+/t+z1qtYpK6/EDUOprcsiImJ4ODN3NzWG/mNvvdfX\nrR6OGzash4ioCqetiIhINoYHERHJxvAgIiLZGB5ERCQbw4OIiGRTLDwSExPRt29fhISE4MKFC+bx\ny5cvIyoqCoMGDUJUVBSys7OVKomIiOpJsUt1+/fvj4kTJ2Ls2LEW4/Hx8YiOjkZERAT27duHuLg4\npKamKlUWEZHivDVN4OnRsO+UUKz6sLAwAIAQwjxWVFSEc+fOYciQIQCAiIgILF26FMXFxfDx8VGq\nNCIiRXl6uFp1Dxdw+z4uR3LoOQ+9Xo+AgACoVKrbxbi4oEWLFsjLy3NkWUREVIcGddxkMBhgMBgs\nxtRqNbRarYMqIiJq2PR6PYxGo8WYRqOBRqO553oODQ+tVov8/HwIIaBSqWAymVBQUICWLVvWunxq\naipSUlIsxoKCgpCZmQk/Py/z2G83KqyqS+Wismp9Z1KfZ2PdqeKWEe5uaoetby1rP39jwl5UYy+q\njRs3Djk5ORZjs2fPRmxs7D3Xc0h4VJ338PX1RUhICHQ6HYYNGwadTof27dvf9XxHTEwMRo4caTGm\nVt/+Yios/AMmk4CbmxqmO86r1Ks+k3XrO5Nr16x7upW/v7fVz9eytob68vf3dti+nQ17Uc0ZeuFM\n4bVt27Zajzzqolh4LFu2DP/9739RWFiISZMmwcfHBzqdDgkJCViwYAHWr1+PZs2aITEx8a7bkHIo\nRURE0tV32l+x8Fi0aBEWLVpUY7xNmzbYuXOnUmXcV+r7SHeyPVtcmslH7N9mbS8rbhnrXsjONTQG\n9/enb+SsfaQ74PjLARsLW12ayYkn63tpi59pZ6jB0fh4EiIiko3hQUREsnHaiogks3aun+dtGg+G\nBxFJZou5fp63aRw4bUVERLIxPIiISDZOW5HTs2ae3d/fm/Ps/4/3JpAt8SeJnB7n2W2jMTwGnJwH\np62IiEg2HnkQScApn8aDj+2xDf5rIJKAj6NoPPjYHtvgtBUREcnG8CAiItkYHkREJBvDg4iIZGN4\nEBGRbAwPIiKSjZfqUqPXWK7rbyyfgxoHhgc1eo3lun5rP4czfAZqPDhtRUREsjE8iIhINk5bkV1x\nnp6ocWJ4kF01lvMNRGSJ01ZERCQbjzyISDGcxmw8GB5EpBhOYzYenLYiIiLZnCY8Ll++jKioKAwa\nNAhRUVHIzs52dElERHQXThMe8fHxiI6ORnp6OsaOHYu4uDhHl0RERHfhFOFRVFSEc+fOYciQIQCA\niIgInD17FsXFxQ6ujIiIauMU4aHX6xEQEACVSgUAcHFxQYsWLZCXl+fgyoiIqDYN6morg8EAg8Fg\nMaZWq6FIH2abAAAGG0lEQVTVauHiUhU8KrQK8EJS7DP13o/WrykAoIVPk/oXa4P1WQNrYA2swd7b\n0Ov1MBqNFmMajQYajeae66mEEMLqvVupqKgIgwYNwtGjR6FSqWAymdCtWzfs378fPj4+5uWSk5OR\nkpJisW5YWBi2b9+udMlERI3CCy+8gBMnTliMzZ49G7GxsfdeUTiJ8ePHi7179wohhPjkk0/EhAkT\naizz+++/i6tXr1r8OX78uIiKihK5ublKl+x0cnNzRe/evdkLwV7cib2oxl5Uy83NFVFRUeL48eM1\nvld///33Otd3mmmrhIQELFiwAOvXr0ezZs2QmJhYY5m7HUqdOHGixmHX/choNCInJ4e9AHtxJ/ai\nGntRzWg04sSJE2jZsiWCg4Nlr+804dGmTRvs3LnT0WUQEZEETnG1FRERNSwMDyIikk2dkJCQ4Ogi\nrOXh4YFu3brBw8PD0aU4HHtRjb2oxl5UYy+qWdMLp7hUl4iIGhZOWxERkWwMDyIikq3BhIeUR7ab\nTCa8+eab6N+/PwYOHIhdu3Y5oFL7k9KL9evXIyIiAiNGjMCoUaNw8OBBB1Rqf3Ie5f/LL78gNDQU\nK1euVLBC5Ujtxeeff46hQ4di6NChGDZsGIqKihSu1P6k9KKoqAjTp0/HsGHD8Nxzz2HJkiUwmUwO\nqNa+EhMT0bdvX4SEhODChQu1LlOv706738ZoIxMmTBA6nU4IIcTevXtrvQN9z549YvLkyUIIIQoL\nC0XPnj1FTk6OonUqQUovDh48KMrKyoQQQpw7d0506dJFlJeXK1qnEqT0QgghjEajiI6OFv/4xz9E\nYmKikiUqRkovsrKyxJAhQ0RhYaEQQogbN27ctz8Xb731lvlnobKyUkRGRoq0tDRF61TC999/L/Ly\n8kSfPn3Ezz//XOsy9fnubBBHHlIf2Z6WlobRo0cDAHx9fdGvXz+kp6crXq89Se1F9+7dzVdQhISE\nAECje8S9nEf5b9q0CX369MFDDz2kcJXKkNqL1NRUvPjii/D19QUAeHl5wd3dXfF67UlqL1QqFW7e\nvAkhBMrKylBZWYmAgABHlGxXYWFhCAgIgLjHtVH1+e5sEOEh9ZHtubm5CAwMNL/WarXQ6/WK1mpv\n9Xl8/Z49e9CqVatG9w9Dai/Onz+PQ4cOYeLEiQ6oUhlSe3Hx4kVkZ2cjOjoazz//PDZs2OCIcu1K\nai9mzpyJS5cuoUePHnjmmWfQo0cPdO7c2RElO1x9vjsbRHhQ/R07dgzJyclYu3ato0txiMrKSixe\nvBgJCQnmL5P7WWVlJX766Sf861//wtatW/HNN99g7969ji7LIdLT0xESEoJDhw7hm2++wbFjx7B/\n/35Hl9VgNIjw0Gq1yM/PNx92mUwmFBQUoGXLlhbLBQYGIjc31/xar9dDq9UqWqu9Se0FAJw8eRKv\nvfYa1q9fjwcffFDpUu1OSi+uXbuGq1evYtq0aejTpw9SU1Oxa9cuLF682FFl24XUn4ugoCAMHDgQ\nrq6uaNq0Kfr27YszZ844omS7kdqLDz/8EEOHDgVwe/qub9++OHr0qOL1OoP6fHc2iPDw9fVFSEgI\ndDodAECn06F9+/YWv+sDAAYNGoSdO3dCCIGioiJkZGRgwIABjijZbqT2IisrC6+88grWrVtnPufR\n2EjphVarxZEjR5CRkYHMzEzExMQgMjISS5YscVTZdiH15yIiIgKHDh0CANy6dQtHjhxBu3btFK/X\nnqT2Ijg4GAcOHAAAVFRU4MiRI3j00UcVr9cZ1Ou701Zn9O3t4sWLIjIyUgwcOFCMHj1aXL58WQgh\nxNSpU8UPP/wghLh9RU18fLzo16+f6N+/v9i5c6cjS7YbKb0YNWqUCA8PFyNGjBDDhw8XI0aMED/9\n9JMjy7YLKb24U3JycqO92kpKL0wmk1ixYoUYPHiwiIiIEG+//bYjS7YbKb3Izs4WkyZNEkOHDhVD\nhgwRS5cuFUaj0ZFl28XSpUtFz549xRNPPCG6d+8uIiIihBDWf3fy8SRERCRbg5i2IiIi58LwICIi\n2RgeREQkG8ODiIhkY3gQEZFsDA8iIpKN4UFERLIxPIiISLb/A+ehpDT73FBfAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "pred_dicts = list(est.predict(eval_input_fn))\n", - "probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])\n", - "\n", - "probs.plot(kind='hist', bins=20, title='predicted probabilities')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBUaNN1BzJHG" - }, - "source": [ - "Finally you can also look at the receiver operating characteristic (ROC) of the results, which will give us a better idea of the tradeoff between the true positive rate and false positive rate." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 307 - }, - "colab_type": "code", - "id": "NzxghvVz3rJ6", - "outputId": "98bcd7d1-35b8-4593-b22f-ca020c2323fa" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEiCAYAAAA8ij+xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xlc1HX+B/DXzCAIAnIIOOCRdohHHngveCEKCopaXkVh\npqSp66b91DSF1dXSrTVXwrRcw2zdtDTF1NV0KzMSTVdtEzWPQG45HAE5Zubz+4OYaeLwC8PMMPB6\nPh4+iu98Z+Y974fOi8/3+HxkQggBIiIiI8gtXQAREVk/hgkRERmNYUJEREZjmBARkdEYJkREZDSG\nCRERGY1hQkRERrOxdAFE5hYYGIjc3FwoFAo4ODhgyJAhWLVqFezt7XX7nD9/Hps2bcLly5ehUCjQ\nr18/vPrqq3j00Ud1+xQWFmLTpk348ssvce/ePXh4eGD48OGYO3cuXFxcLPHRiCyGIxNqlrZu3Yrz\n58/jwIED+Omnn7B161bdYxcuXMCLL76IUaNG4dtvv8WJEyfQpUsXTJ8+HXfu3AEAlJeXIzIyEjdv\n3sT27dtx/vx57N69Gy4uLrh06ZLJ6tZoNCZ7bSJjMEyoWaqc+MHd3R0BAQFITk7WPfbWW29h0qRJ\niIiIgIODA5ydnfGnP/0JvXr1QmxsLADg888/R1ZWFt5991107twZAODm5oa5c+di6NCh1b7n9evX\nMXPmTAwcOBABAQHYtm0bAOC1117Dpk2bdPslJSVh2LBhup8DAwPx/vvvY/z48ejTpw+2bNmCP/7x\njwav/Ze//AVr164FUDFiWrFiBQICAjBs2DC888474EQXZGoME2rWMjMz8c0336Bjx44AgJKSEly4\ncAHBwcFV9h0zZgxOnz4NAEhMTMSQIUPQsmVLSe9TVFSEF154AUOHDsWpU6dw7NgxDBo0qMb9ZTKZ\nwc+HDx/G+++/j3PnziE8PBynTp1CUVERAECr1eLo0aMYN24cAGDJkiVo0aIFTpw4gf379+O7777D\n3r17JdVJVF8ME2qW5s2bBz8/PwwfPhxt2rTBggULAAD37t2DVquFh4dHled4eHggPz8fAFBQUFDt\nPjX56quv4OnpiRkzZsDW1hYODg7o2bOn5Oc///zz8PLygq2tLby9vdGtWzd8+eWXACqCrfL17t69\ni1OnTmH58uWws7ODm5sbIiMjcejQIcnvRVQfPAFPzVJcXBwGDRqEc+fOYfHixcjPz4ejoyOcnZ0h\nl8uRk5ODTp06GTwnJycHrq6uAAAXFxfk5ORIfr+MjAy0b9++3vW2bdvW4OfQ0FB88cUXCA8Px6FD\nhxAWFgYASE9Ph1qtRkBAAICKw3lCCCiVynq/N5EUHJlQs1R5DqFfv36YMGEC3nzzTQCAvb09evfu\njaNHj1Z5zpEjR/CHP/wBADB48GCcOnUKJSUlkt5PqVQiJSWl2sfs7e0NXkdKSIWEhCApKQlZWVn4\n8ssvdWGiVCphZ2eHM2fOICkpCWfPnsW5c+eQkJAgqU6i+mKYULMXGRmJ7777TncSfvHixdi/fz92\n7dqFoqIi3Lt3Dxs3bsTFixcxb948AEB4eDiUSiUWLFiAmzdvQgiB/Px8bN26Fd98802V9xg+fDju\n3r2LnTt3oqysDEVFRbqrvrp27Yqvv/4a9+7dQ05ODnbu3PnQmt3c3NC/f3+89tpraN++ve4iAA8P\nD/j7+2PdunUoLCyEEAKpqak4e/ZsQ7WLqFpNLkxUKhU2b94MlUpl6VIsjr2o8Ps+/P7ktpubGyZM\nmIC4uDgAQN++fbF9+3b8+9//RkBAAEaOHImrV69i9+7d6NChAwDA1tYWO3bsQOfOnTFz5kz07dsX\nU6dORUFBAXr16lWlhlatWmHHjh04efIkAgICEBwcjDNnzgCoCKYuXbogMDAQs2bNwtixYw2e+/t6\nK4WFhSExMVF34r3S+vXrUV5ejtDQUAwYMAALFy7UjXb4d0KPvdBrkF4IM3nzzTdFYGCg6NKli7h+\n/Xq1+2g0GhETEyOCgoLE6NGjxZ49e+r8PqmpqeKJJ54QqampxpZs9diLCuyDHnuhx17oNUQvzDYy\nGTVqFP75z3/Cx8enxn0OHjyI1NRUHD9+HLt370ZsbCzS09PNVSIREdWT2cLEz88PXl5etd48deTI\nEUyZMgVAxaGHoKCgak+EEhFR49Kozpmkp6fD29tb97NSqURGRoYFKyIiIims+j4TlUpV5YRRZmYm\n/Pz8oFAoLFRV46FQKODj49Pse8E+6LEXeuyFnkKhgJ+fHzIzM6s85uzsDGdn54e+RqMKE29vb6Sn\np6NHjx4AKm70qu0cS3x8vG6upEp+fn7YvXu3Seu0FkqlEidPnrR0GRbHPuixF3rshZ5SqcTu3bsx\nffp0nD9/3uCx+fPn62aIqE2jCpOQkBDs2bMHo0aNQn5+Pk6cOIFdu3bVuH9kZCQmTpxosK3yt4z8\n/CJotZzczt3dEbm5hZYuw+LYBz32Qq8uvcjMK8Y/v7yG4hK1iasyndIyDUrLNfjL7EGwtdGf5ZDL\nZXB1bYW//e1vVWamljIqAcwYJn/5y19w/Phx5ObmYsaMGXB1dUVCQgKioqKwcOFCdO/eHeHh4bh4\n8SJGjx4NmUyGefPmoV27djW+Zm3DL61WMEx+xT5UYB/02As9Kb3IvVeC9R+fh1Yr0OsxdzNUZToe\nLvawkcuq/dzGTLsjE7VdXmXFcnML+Q8GgIeHE3Jy7lu6DItjH/TYC73f9qKopByffnUDpeVV14y5\nkXYPhQ/UWPasH9p7Opq7TJOTy2VwdzfuczWqw1xERJZyK12Fr/+bDhdHW9jaGJ6Ut2thg5ljuzbJ\nIGkoDBMiot94eeKTeMyntaXLsDqN6j4TIiKyThyZEJFZlJVrUFBUZukyDGjkcuQWPAAAFBQ2rtqs\nDcOEiEzu7r0HeGPXeeTfL7V0KQ/VQsEDNvXBMCEik7pfXIa/fXIRJWUaPB/SpVF9WTs7t4RKpV+Y\nzN7OBu29eJK9PhgmRE3IueRsfPr1DWg02hr3kSvk0NbyeEN7UKpBmVqLxVN7oUsHV7O9rxS8TLrh\nMEyImoifbudh68H/wbtNK3RoV/PVSC1btkBJSbnZ6pJBhkHdvRpdkFDDYpgQmcDpyxlITsk33xsK\n4IdrOVC6O2DpM33g0LJFjbvyt3EyBYYJkQkcSvwF+aoSODnU/KXe0Np5OGLuhB61BgmRqTBMiOpB\nKwRyCh7gTnYRSsqqTvz3oFSN3o+3wZzwHhaojsj8GCZED1FarsGdnEKkZhciNevX/+YUorSs6hxO\nv+XsYGumCoksj2FC9BuqojLczlQhNbsQKb8GR1ZeMSqnDLW3U6C9hyMCeijR3ssR7T0d0cq++sNK\nbZxbmq9wIgtjmBD96sdbufj7p5eg1lRER5vWLdHe0xEDunqig5cT2ns6ok3rlpDJZBaulKjxYZgQ\nAbiVocK7+35EW7dWeHbU42jv6QSHlvznQSQV/7VQs1dapsGmvRfhaN8Cr0zpBVcnO0uXRGR1Gs+8\nBkQWcq+oFKricowPeIRBQlRPHJlQs6PWaLHv65soLq24C/xBacVVWQo5z4UQ1RfDhJqdtJwiHE1K\nQauWNmhhUzE493BpCZ82nOCPqL4YJtRszQztij6Pe1i6DKImgedMiIjIaAwTIiIyGsOEiIiMxjAh\nIiKj8QQ8NVl37z3Au/t/RFm5BgqFXLf6YFm5+VYZJGouGCbUZKXfLcYvmffRtaMr3F3sUVqqnyre\nt6MLHvOpeTVCIqobhgk1eZOGdcagXu24uiCRCfGcCRERGY0jE7J65WotfryVq5s6vlJKFkciRObC\nMCGrd+F6Dt478L8aH2/FNdGJTI5hQlatuESN+8UVEzYuqmb6eHs7G7hxxUMik2OYkNX678938fdP\nL+l+9m7TisFBZCEME7JaBfdLAQATh3aGl6s9g4TIghgm1Gj8cDUHB769CfHwXQEAhQ8qDm8N6amE\niyMXtSKyJIYJNRrJv+QjI7cYvR9rI+0JroCLox2cW9matjAieiiGCZndz2n3kPRTVpXtV1ML0NJW\ngXmTnrRAVURkDLOGye3bt7Fs2TIUFBTAxcUFGzZsQIcOHQz2ycvLw2uvvYaMjAyo1WoMGjQIr7/+\nOuRy3l/ZVBw/m4pzydmwt6v61+/xdpzihMgamTVMoqOjERERgbCwMBw8eBArV65EfHy8wT7vvfce\nHn30UWzduhUajQbTp0/HsWPHEBISYs5S6TfuZBciu+BBg71e/v1StHV3wNrZgxrsNYnIsswWJnl5\nebhy5QpCQ0MBAGFhYVizZg3y8/Ph6uqq208mk6GoqAhCCJSUlECtVsPLy8tcZVI1/vqvC7p7ORoK\nJ1kkalrMFiYZGRnw8vKCTCYDAMjlcnh6eiIzM9MgTF5++WUsWLAAAQEBePDgASIiItCnT59qX1Ol\nUkGlUhlsUygUUCqVpvsgTZhao612evbScg0Gd2+L4AHtG+y92rS2b7DXIqKGkZGRAY1GY7DN2dkZ\nzs7OD31uozsBf/ToUfj6+mLnzp0oLCzErFmzcOzYMYwePbrKvvHx8YiNjTXY5uPjg5MnT8Ld3dFc\nJTd6Hh5OD91Ho9HihTXHkP/rvRu/5+PlhL49vBu6NLOS0ofmgr3QYy/0nn32WaSlpRlsmz9/PhYs\nWPDQ55otTJRKJbKysiCEgEwmg1arRXZ2Ntq2bWuw365du7Bu3ToAgKOjI0aOHIkzZ85UGyaRkZGY\nOHGiwTaFQgEAyM0thFYr9Y6FpsvDw0nS1Oul5Rrk3y9Fr0fd0bWjq8FjMpkM/Xw9rXoKd6l9aA7Y\nCz32ooJcLoO7uyM+/vjjakcmUpgtTNzc3ODr64uEhASMHz8eCQkJ6Natm8EhLgBo164dTp06hSef\nfBJlZWVITEysNkgA6cOvpuzuvQfYlvATyso11T5uY6OAWl39Y78lfs3dJ9q7YPSADrXvTERNkjGn\nCGRCCLP9+n7z5k0sW7YMKpUKrVu3xoYNG9CxY0dERUVh4cKF6N69O1JTUxEdHY27d+9Cq9Vi0KBB\nWL58eZ0vDW4uI5P/Xr+Lv392CU+0aw2HambHtbW1QVmZuppnViWXyzAhoBPaeTa9Q4T8DVSPvdBj\nLypUjkyMYdYwMaemFiaqojIcO5sKtcbwBHlOwQNcuH4Xq2b0wyNtq47S+I+lAvugx17osRcVGiJM\nGt0JeKrexZ/v4vD3v8C2hVx3RVwlVyc7uDpxkkMishyGiYU8KFXjWmqB5EkNU7IKAQDrZg/i7LhE\n1OgwTCzk8Pe/4IvEX+r0HLlMBjtbhYkqIiKqP4aJhZSWa2DXQoElz1R/Q2Z1nOxbcAlaImqUGCYW\nJJfL0EnZvC9tJqKmgVPxEhGR0TgyMZNzydk4ciZF93OuqsSC1RARNSyGiZlcvHEXqdmF8O3oAgBo\nZe+IzjzERURNBMPExH7JvI+k5CzczryP1q1aYNGU3pYuiYiowTFMTOzfSSn4/qcs2CjkeLKzm6XL\nISIyCYaJiWmFgJebA96I4qqCRNR08WouIiIymuSRyenTp/HFF18gLy8P7733Hi5fvozCwkIMHjzY\nlPUREZEVkDQy+eijjxATE4NHHnkEZ8+eBQC0bNkSmzZtMmlxRERkHSSFSXx8PHbs2IGoqCjduiKd\nO3fGrVu3TFocERFZB0mHuYqKinQrcFVOf65Wq9GiBeeJqknij5n4z4U0ZOYVo5U9+0RETZukkUn/\n/v2xbds2g207d+7EwIEDTVJUU/DDtRykZheig5cjhvaq/1KYRETWQNLI5PXXX8ecOXOwd+9eFBUV\nITg4GI6OjnjvvfdMXZ9V83BpiVenSZ8VmIjIWkkKE09PT3z22We4fPky0tLSoFQq0bNnzzqvy05E\nRE2TpDSYO3cuZDIZevbsiTFjxqB3796Qy+WYP3++qeuzGveLy3Dldp7uj6q4zNIlERGZjaSRyZkz\nZ6rdnpSU1KDFWLMdh5Px35/vGmx7zKe1haohIjKvWsOk8j6S8vLyKveUpKamwtvb23SVWZnScg3a\nebTCs6Oe0G1r697KghUREZlPrWGSmZkJABBC6P6/klKpxIIFC0xXmRWyt7NBlw6uli6DiMjsag2T\nN954AwDQp08fTJkyxSwFERGR9ZF0zqQySAoLC5Gfn2/wWPv27Ru+KiIisiqSwuTGjRtYvHgxkpOT\nIZPJIITQ3Ql/5coVkxZIRESNn6RLg2NiYjBw4EAkJSXB0dERZ8+exdSpU/Hmm2+auj4iIrICksIk\nOTkZr776KpydnSGEgJOTE5YsWcJZg4mICIDEMLGzs4NarQYAuLq6Ij09HVqtFgUFBSYtzhoUPijH\n1ZR8FJWUW7oUIiKLkXTOpG/fvjhy5AgmTZqE4OBgzJ49G7a2thg0iEvRbjv4P/x4Kw8A0INrvBNR\nMyUpTH57OGvRokV4/PHHUVRUhAkTJpisMGtRUqZBRy8nTBnxKHw8HC1dDhGRRTw0TDQaDWbMmIHt\n27fD1tYWcrkc4eHh5qitUdt+6Cd8979MCAH06OSGro9wVEJEzddDw0ShUODOnTvQarXmqMdq3Llb\nBA8Xewzo6oWej7pbuhwiIouSdAJ+3rx5iImJQVpaGjQaDbRare5Pc9bWzQGThnbmhI5E1OxJXhwL\nAA4cOKDbVnnjIm9aJCIiSWFy4sSJBnmz27dvY9myZSgoKICLiws2bNiADh06VNnv8OHD2LJlC4CK\nNec//PBDuLlZ9pzET7fzcDNdpfv5XmEpWreytWBFRESNh6Qw8fHxaZA3i46ORkREBMLCwnDw4EGs\nXLkS8fHxBvtcvnwZcXFx2LlzJ9zc3FBYWAhbW8t/ae86dg2ZecUG2/r7OlioGiKixkVSmDSEvLw8\nXLlyBaGhoQCAsLAwrFmzBvn5+XB11U/bHh8fj5kzZ+pGIo6ODXu5bU7BA9wrqvsqiKXlGgzo6olZ\nYd1022wUXLaYiAgwY5hkZGTAy8tLN0GkXC6Hp6cnMjMzDcLkxo0baNeuHSIiIlBcXIxRo0Zh7ty5\nDVJDWbkGK97/HmqNqNfzW9raMECIiKphtjCRSq1W49q1a/jwww9RWlqKWbNmwdvbu9p7W1QqFVQq\nlcE2hUIBpVJZ/WtrBNQagWG9vdH3CY8619bZ27nOzyEishYZGRnQaDQG25ydneHs/PDvvjqFSUZG\nBrKystC7d++6VYiKlRmzsrJ0V4FptVpkZ2ejbdu2Bvv5+PggODgYNjY2sLGxwciRI3H58uVqwyQ+\nPh6xsbFVnn/y5Em4u1c9PFb0oGL+rMc6uGHEwEfq/BmslYeHk6VLaBTYBz32Qo+90Hv22WeRlpZm\nsG3+/PmSVtWVFCbp6elYtGiRbj2TCxcu4OjRozh16hTWrl0rqUg3Nzf4+voiISEB48ePR0JCArp1\n62ZwiAuoOJfyzTffIDw8HOXl5UhMTERISEi1rxkZGYmJEycabFMoFACA3NxCaLWGh7OKSyomqywq\nLEFOzn1JdVs7Dw+nZvNZa8M+6LEXeuxFBblcBnd3R3z88cfVjkykkBQmq1atwvDhw/HPf/4TAwcO\nBAD4+/tj/fr1dSo4JiYGy5YtQ1xcHFq3bo0NGzYAAKKiorBw4UJ0794doaGh+PHHHzF27FgoFAoE\nBARg8uTJ1b6e1OGXVgjsPJqMnIKSOtVLRNSc1HSKQAqZEOKhZ6MHDhyIxMREyOVyDBgwAElJSQCA\nfv364dy5c/V+c1P67chEVVSGP23+Fu7OLeHqbIfpIx9HJ2XzOP/B37wqsA967IUee1GhcmRiDEkj\nE3d3d/zyyy/o1KmTbtvPP/9sVIpZwphBHRDo187SZRARNTmSrnOdOXMm5syZg88++wxqtRqHDh3C\nK6+8gtmzZ5u6PiIisgKSRiZPP/00XFxc8Mknn0CpVOLzzz/HwoULERQUZOr6iIjICkgKE41Gg6Cg\nIIYHERFVS9JhLn9/f8TExOCHH34wdT1ERGSFJIXJP/7xDzg4OGDx4sUIDAzE22+/jatXr5q6NiIi\nshKSDnN169YN3bp1w5IlS5CUlIRDhw5hxowZaNOmDRISEkxdIxERNXJ1nrWwU6dOePTRR6FUKqvc\ndk9ERM2TpJGJSqXCv//9bxw6dAgXL16Ev78/Zs2ahZEjR5q6PiIisgKSwmTIkCHo06cPwsLCEBsb\nCycnToxGRER6ksLk+PHj8PT0NHUtRERkpWoMk7Nnz6J///4AKhasunHjRrX7DR482DSVERGR1agx\nTP785z/j0KFDAIAVK1ZUu49MJsOJEydMUxkREVmNGsOkMkgA4OTJk2YphoiIrJOkS4NrWoN9/vz5\nDVpMQyv9dc3317YlAgBkFq6HiKipknQC/syZM9Vur1zXpLEqLC5HRm4xunZ0RUcvJ/R+vO7rvhMR\n0cPVGiabNm0CAJSXl+v+v1Jqaiq8vb1NV1kDGtTNC0N6WUetRETWqNYwyczMBAAIIXT/X0mpVEpa\nZJ6IiJq+WsPkjTfeAAD06dMHU6ZMMUtBRERkfWoMkzt37qBdu4olbgcPHozU1NRq92vfvr1pKiMi\nIqtRY5iMGzcOFy5cAACMGjUKMpkMQgiDfWQyGa5cuWLaComIqNGrMUwqgwQAkpOTzVIMERFZpzpP\nQQ9UXMnF6eeJiKiSpDBZtGgRzp8/DwD47LPPEBoaitDQUOzdu9ekxRnjrX9dQMyOX++D4d2KREQm\nJSlMEhMT0aNHDwDAhx9+iB07dmDv3r14//33TVqcMbLzH6CDlxNG9WuPnp3dLV0OEVGTJukO+PLy\nctja2iIrKwsFBQXo27cvAODu3bsmLc5YAT2VGNy9raXLICJq8iSFSdeuXbF161akpaVh+PDhAICs\nrCw4OjqasjYiIrISkg5zrV27FteuXUNpaSkWLlwIoOJqr3Hjxpm0OCIisg6SRiYdOnTA22+/bbAt\nJCQEISEhJimKiIisi6QwASqu4jpw4ACysrLg5eWF8PBwPPXUU6asjYiIrISkMNmyZQs+//xzzJw5\nE97e3khPT8cHH3yA7OzsGtc6ISKi5kNSmOzduxcfffQRfHx8dNsCAgIQERHBMCEiImkn4B88eAA3\nNzeDbS4uLigpKTFJUQ2F9yoSEZmHpDAZMmQIXn31Vdy8eRMlJSW4ceMGli1bhoCAAFPXV2/+PZTo\n+ojbw3ckIiKjSQqTVatWoVWrVggPD0fv3r0RHh4Oe3t7rFy50tT11Vv4kE5o3crW0mUQETULMvH7\neeVrodVqkZ+fD1dXV8jl9Zoj0mxycwuh1Ur+aE2Wh4cTcnLuW7oMi2Mf9NgLPfaiglwug7u7cTeh\nS06E27dvY+vWrYiNjcXWrVtx+/btOr/Z7du3MW3aNISEhGDatGlISUmpcd+bN2+id+/e2LBhQ53f\nh4iIzEtSmCQkJGDixIm4evUq7O3tce3aNUycOBEJCQl1erPo6GhERETg6NGjeOaZZ2o8TKbVahEd\nHY2goKA6vT4REVmGpEuD33nnHWzbtg39+/fXbTt37hyWLFkieUqVvLw8XLlyBaGhoQCAsLAwrFmz\nRnfY7Le2bduGwMBAFBUVobi4WOpnISIiC5E0MikqKkLv3r0NtvXq1atOX/QZGRnw8vKCTFZxwa5c\nLoenpycyMzMN9ktOTsbp06cxY8YMya9NRESWJWlk8sILL+Bvf/sb/vSnP8HOzg4lJSX4+9//jhde\neKFBi1Gr1Vi1ahXeeOMNXejURqVSQaVSGWxTKBRQKpUNWhcRUXOQkZEBjUZjsM3Z2RnOzs4Pfa6k\nq7mGDRuGu3fvQiaTwdnZGSqVCkIIeHh4GOz31Vdf1fgaeXl5CAkJwZkzZyCTyaDVajFw4EAcO3ZM\nd5grIyMDkyZNgoODA4QQuH+/4iqLMWPGYPXq1VVec/PmzYiNjTXY5uPjg5MnTz70gxMRkaHAwMAq\nS7LPnz8fCxYseOhzJYVJUlKSpEIGDBhQ6+PPP/88nn76aYwfPx4HDhzAvn37EB8fX+P+sbGxKC4u\nxpIlS6p9vLaRCS8NrsBLHyuwD3rshR57UaHy0mBjRiaSDnM9LCSkiomJwbJlyxAXF4fWrVvrLvuN\niorCwoUL0b179zq9ntQPSURED2fMKYI63bRoTTgyqcDfvCqwD3rshR57UcGsNy0SERHVhGFCRERG\nq1OYaLVaZGdnm6oWIiKyUpLCRKVSYfHixejZsydGjx4NADhx4gQ2btxo0uKIiMg6SAqT6OhoODo6\n4uTJk2jRogUAoE+fPjhy5IhJiyMiIusg6dLgxMREnDp1Ci1atNDdme7m5obc3FyTFkdERNZB0sjE\nyckJ+fn5BtvS09Or3AFPRETNk6QwmTx5Mv74xz/i+++/h1arxYULF7B06VJMmzbN1PUREZEVkHSY\na/bs2bC1tcXq1auhVquxfPlyTJ06FZGRkaauj4iIrADvgG/ieIdvBfZBj73QYy8qNMQd8JJPwNdk\n8ODBRhVARETWT1KYrFixwuDn/Px8lJeXw8vLCydOnDBJYUREZD0khcnv1wfRaDTYsmULWrVqZZKi\niIjIutRrbi6FQoE5c+bggw8+aOh6iIjICtV7osfTp09LWlqXiIiaPkmHuYYNG2YQHA8ePEBZWRmi\no6NNVhgREVkPSWHy17/+1eBne3t7dOrUCY6Oxl1KRkRETcNDw0Sj0WDz5s3Yvn07bG1tzVETERFZ\nmYeeM1EoFLhz5w60Wq056iEiIisk6QT8vHnzEBMTg7S0NGg0Gmi1Wt0fIiIiSedMXn/9dQDAgQMH\ndNuEEJDJZLhy5YppKiMiIqshKUx4lzsREdVG0mGuo0ePwsfHp8qfY8eOmbo+IiKyApLC5N133612\n+5YtWxq0GCIisk61HuaqnC1Yq9Xi+++/x29nq79z5w7n5iIiIgAPCZPK2YJLS0uxfPly3XaZTAYP\nDw/diXkiImreag2TytmClyxZgg0bNpilICIisj6SzpkwSIiIqDb1njWYiIioEsOEiIiMxjAhIiKj\nMUyIiMhoDBMiIjIaw4SIiIzGMCEiIqMxTIiIyGiSpqBvKLdv38ayZctQUFAAFxcXbNiwAR06dDDY\nJy4uDoebDyV/AAARKUlEQVQPH4aNjQ0UCgVeeeUVBAQEmLNMIiKqI7OGSXR0NCIiIhAWFoaDBw9i\n5cqViI+PN9inV69eePHFF2FnZ4fk5GQ899xzOH36NNefJyJqxMx2mCsvLw9XrlxBaGgoACAsLAw/\n/fQT8vPzDfbz9/eHnZ0dAMDX1xcAquxDRESNi9lGJhkZGfDy8oJMJgMAyOVyeHp6IjMzE66urtU+\nZ//+/Wjfvj28vLyqfVylUkGlUhlsUygUUCqVDVs8EVEzkJGRAY1GY7DN2dkZzs7OD32uWQ9z1UVS\nUhI2b96MHTt21LhPfHw8YmNjDbb5+Pjg5MmTcHd3NHWJVsPDw8nSJTQK7IMee6HHXug9++yzSEtL\nM9g2f/58LFiw4KHPNVuYKJVKZGVlQQgBmUwGrVaL7OxstG3btsq+Fy5cwNKlS7FlyxZ07NixxteM\njIzExIkTDbYpFAoAQG5uIbRaUd3TmhUPDyfk5Ny3dBkWxz7osRd67EUFuVwGd3dHfPzxx9WOTKQw\nW5i4ubnB19cXCQkJGD9+PBISEtCtW7cqh7guXbqERYsWYdOmTbpzJjWROvwiIqKHM+YUgUz8di1e\nE7t58yaWLVsGlUqF1q1bY8OGDejYsSOioqKwcOFCdO/eHU8//TTS09Ph5eWlG8Vs2LABjz/+eJ3e\niyOTCvzNqwL7oMde6LEXFSpHJsYwa5iYE8OkAv+xVGAf9NgLPfaiQkOECe+AJyIiozFMiIjIaAwT\nIiIyGsOEiIiMxjAhIiKjMUyIiMhoDBMiIjIaw4SIiIzGMCEiIqMxTIiIyGgMEyIiMhrDhIiIjMYw\nISIiozFMiIjIaAwTIiIyGsOEiIiMxjAhIiKjMUyIiMhoDBMiIjIaw4SIiIzGMCEiIqMxTIiIyGgM\nEyIiMhrDhIiIjMYwISIiozFMiIjIaAwTIiIyGsOEiIiMxjAhIiKjMUyIiMhoDBMiIjIaw4SIiIzG\nMCEiIqMxTIiIyGgMEyIiMppZw+T27duYNm0aQkJCMG3aNKSkpFTZR6vV4s9//jNGjRqF4OBg7N27\n15wlEhFRPZg1TKKjoxEREYGjR4/imWeewcqVK6vsc/DgQaSmpuL48ePYvXs3YmNjkZ6ebs4yiYio\njswWJnl5ebhy5QpCQ0MBAGFhYfjpp5+Qn59vsN+RI0cwZcoUAICbmxuCgoJw9OhRc5VJRET1YLYw\nycjIgJeXF2QyWcUby+Xw9PREZmamwX7p6enw9vbW/axUKpGRkWGuMomIqB5sLF2AMVQqFVQqlcE2\nhUIBpVIJuVxmoaoaH/aiAvugx17osRf6HmRkZECj0Rg85uzsDGdn54e+htnCRKlUIisrC0IIyGQy\naLVaZGdno23btgb7eXt7Iz09HT169ABQ8eF8fHyqfc34+HjExsYabPPz88Pu3bvh6trKNB/ECrm7\nO1q6hEaBfdBjL/TYC71Fixbh/PnzBtvmz5+PBQsWPPzJwoyee+45ceDAASGEEJ9//rl4/vnnq+yz\nb98+8eKLLwqtVityc3PFsGHDRGpqarWvd+/ePZGammrw5+zZs2LatGkiPT3dpJ/FGqSnp4sRI0Y0\n+16wD3rshR57oZeeni6mTZsmzp49W+U79d69e5Jew6yHuWJiYrBs2TLExcWhdevW2LBhAwAgKioK\nCxcuRPfu3REeHo6LFy9i9OjRkMlkmDdvHtq1a1ft69U0/Dp//nyVoVpzpNFokJaW1ux7wT7osRd6\n7IWeRqPB+fPn0bZt2xq/bx/GrGHSuXNn7Nmzp8r2bdu26f5fLpcjJibGjFUREZGxeAc8EREZjWFC\nRERGU8Q0wWNKdnZ2GDhwIOzs7CxdisWxFxXYBz32Qo+90DO2FzIhhGjgmoiIqJnhYS4iIjIaw4SI\niIxmtWHC6ez1pPQiLi4OYWFhmDBhAp566il8++23FqjU9KT0otLNmzfRu3dv3f1OTY3UXhw+fBjj\nxo3DuHHjMH78eOTl5Zm5UtOS0oe8vDy89NJLGD9+PMaOHYvVq1dDq9VaoFrTWr9+PUaOHAlfX1/8\n/PPP1e5T7+9Nk95WaULPP/+8SEhIEEIIceDAgWrvpt+/f7948cUXhRBC5ObmiqFDh4q0tDSz1mkO\nUnrx7bffipKSEiGEEFeuXBH9+vUTpaWlZq3THKT0QgghNBqNiIiIEIsXLxbr1683Z4lmI6UXly5d\nEqGhoSI3N1cIIcT9+/eb3N8LKX1Yu3at7u+BWq0WkydPFkeOHDFrnebwww8/iMzMTBEYGCiuX79e\n7T71/d60ypEJp7PXk9oLf39/3VUavr6+AFBlH2sntRdAxY2ygYGBeOSRR8xcpXlI7UV8fDxmzpwJ\nNzc3AICjoyNsbW3NXq+pSO2DTCZDUVERhBAoKSmBWq2Gl5eXJUo2KT8/P3h5eUHUct1Vfb83rTJM\nOJ29ntRe/Nb+/fvRvn37JvePRWovkpOTcfr0acyYMcMCVZqH1F7cuHEDKSkpiIiIwKRJk7BlyxZL\nlGsyUvvw8ssv49atWwgICMCQIUMQEBCAPn36WKJki6vv96ZVhgnVX1JSEjZv3oyNGzdauhSLUKvV\nWLVqFWJiYnRfMM2ZWq3GtWvX8OGHH+Kjjz7CN998gwMHDli6LLM7evQofH19cfr0aXzzzTdISkrC\nsWPHLF2WVbHKMPntdPYAHjqdfaWMjAwolUqz1mpqUnsBABcuXMDSpUsRFxeHjh07mrtUk5PSi5yc\nHKSmpiIqKgqBgYGIj4/H3r17sWrVKkuVbRJS/174+PggODgYNjY2aNWqFUaOHInLly9bomSTkNqH\nXbt2Ydy4cQAqDvWNHDkSZ86cMXu9jUF9vzetMkzc3Nzg6+uLhIQEAEBCQgK6desGV1dXg/1CQkKw\nZ88eCCGQl5eHEydOYPTo0ZYo2WSk9uLSpUtYtGgRNm3apDtn0tRI6YVSqURiYiJOnDiBkydPIjIy\nEpMnT8bq1astVbZJSP17ERYWhtOnTwMAysvLkZiYiC5dupi9XlOR2od27drh1KlTAICysjIkJibi\n8ccfN3u9jUG9vzcb4goBS7hx44aYPHmyCA4OFlOmTBG3b98WQggxe/Zs8eOPPwohKq7YiY6OFkFB\nQWLUqFFiz549lizZZKT04qmnnhKDBw8WEyZMEOHh4WLChAni2rVrlizbJKT04rc2b97cZK/mktIL\nrVYr3njjDTFmzBgRFhYm3nzzTUuWbBJS+pCSkiJeeOEFMW7cOBEaGirWrFkjNBqNJcs2iTVr1oih\nQ4eK7t27C39/fxEWFiaEaJjvTU6nQkRERrPKw1xERNS4MEyIiMhoDBMiIjIaw4SIiIzGMCEiIqMx\nTIiIyGgME2q0bt26hYkTJ6Jv377YtWtXrfumpaXB19fXKqYNDwsLw9mzZ2t8fPbs2fj888/NWBGR\n8XifCTVaK1asgJOTE5YtW/bQfdPS0hAUFIT//e9/kMut53ek2NhYpKSkNMo1VRpzbdT4WM+/Omp2\n0tPT8dhjj1m6jCZJo9FYugRqYhgm1ChFRkbizJkzWL16Nfz8/PDLL7/g66+/1h32GjFiBGJjY2t8\n/r59+xAUFAQ/Pz8EBQXh0KFDusc+/fRTjB07FgMHDsSsWbMMJrX7rcpDZ3v27MGQIUMwZMgQ7Nix\nQ/d4WVkZ1q5diyFDhmDo0KFYt24dysvLAVSsFTNnzhz0798fAwcOREREhO55gYGBSExMxKlTp/De\ne+/h8OHD6NOnDyZMmAAAeO655/Dpp5+irKwM/fv3N1gRLy8vD7169dKthvif//wHEyZMQP/+/TF9\n+nRcvXq1xp74+vri448/RnBwMIKDgwEAa9euxfDhw9G3b1889dRTOHfuHADUWFthYSFWrFiBgIAA\nDBs2DO+8806ta2NQM2KK+V+IGkJERITYu3ev7uekpCTdfGJXr14V/v7+4ssvvxRCCHHnzh3h6+sr\nNBqNKC4uFn5+fro5mHJycsTPP/8shBDi+PHjYvTo0eLmzZtCo9GILVu2iKlTp1b7/nfu3BFdunQR\nixYtEiUlJeLq1ati0KBB4rvvvhNCCPHOO++IqVOniry8PJGXlyemTp0qNm3aJIQQ4u233xbR0dFC\no9EItVotzp07p3vdESNG6F5j8+bN4v/+7/9q/NzLly8XGzdu1D22a9cuMWvWLCGEED/++KMYPHiw\nuHTpktBqtWL//v1ixIgRoqysrNrP06VLFzFz5kyhUql0qykePHhQ3Lt3T2g0GrFjxw7h7++ve6y6\n2ubOnSuio6NFSUmJyM3NFZMnTxaffPJJte9HzQtHJmQ1+vfvr5vJ9YknnsDYsWNrPJGtUChw7do1\nlJaWok2bNnj00UcBAJ988gmioqLQqVMnyOVyREVFITk5udbFfxYsWAA7Ozs88cQTmDRpEr744gsA\nwKFDhzBv3jy4urrC1dUV8+fP160FYmNjg5ycHNy5cwcKhQJ9+/at12cOCwszGFUdOnRIN1X63r17\nMW3aNDz55JOQyWSYMGECbG1tcfHixRpf76WXXoKTk5NuNcVx48bB2dkZcrkcM2bMQFlZGW7dulXt\nc3Nzc3Hq1CksX74cdnZ2cHNzQ2RkpEF91HzZWLoAIqkuXbqEt956C9evX0d5eTnKy8sREhJSZT97\ne3ts3LgR27dvx/Lly9G3b18sXboUnTp1Qnp6OtauXYv169cDAIQQkMlkyMrKqnbNBplMZrD2hbe3\nN65fvw4AyM7ONliRztvbG9nZ2QCAF198EbGxsZg5cyZkMhkmT56MqKioOn/mQYMGobS0FJcuXUKb\nNm2QnJyMoKAgABXnlA4cOKC70k0IAbVarauhOr9fx+Mf//gHPv30U+Tk5AAAioqKalzOOS0tDWq1\nGgEBAbr3E0I0uTWCqH4YJmQ1Fi9ejOeeew7bt29HixYtsG7dOhQUFFS7r7+/P/z9/VFWVoaNGzdi\n5cqV2LVrF9q2bYu5c+ciLCxM0nsKIZCRkYFOnToBqFgoyNPTEwDg6emJtLQ03agnPT1d91irVq2w\ndOlSLF26FDdu3MBzzz2Hnj17YtCgQXX6zDKZDGPGjMGhQ4fQpk0bjBgxAg4ODgAqgmHOnDl46aWX\n6vR6lc6dO4cPPvgAO3fu1F3oMGDAgBrPgSiVStjZ2eHMmTNcpZKq4GEushrFxcVwdnZGixYtcOnS\npSqHVyq/BHNzc3Hy5Ek8ePAANjY2cHBw0F0uPH36dGzdulV3Uvv+/fs4evRore8bFxeHkpISXL9+\nHfv27UNoaCgAIDQ0FFu2bEFeXh7y8vIQFxeH8PBwAMBXX32FlJQUAICDgwMUCgVsbKr+7tamTRuk\npaXVehI7LCwMhw8fRkJCgkEITpkyBf/6179w6dIlXX++/vprFBcX1/p5KhUVFcHGxgYuLi4oKytD\nbGwsioqKaqzNw8MD/v7+WLduHQoLCyGEQGpqaq33zFDzwZEJNVq//+03Ojoab775JtasWYP+/ftj\n7NixUKlUVfbXarXYsWMHli5dCplMBl9fX0RHRwMAgoKCUFxcjFdeeQUZGRlwcnLCH/7wh2oPl1Ua\nMGAARo0aBSEEZs2ahcGDBwMAXn75ZRQVFWH8+PG6EcScOXMAALdv38bq1auRn5+P1q1b49lnn0W/\nfv2qfK6QkBAcPHgQAwcORLt27bBv374qn7tnz55wcHBATk4Ohg4dqtveo0cPrFmzBqtXr0ZKSgrs\n7OzQt29f9O/fX1I/K69QCw4OhoODA2bMmGFwGKy62tavX4+33noLoaGhKC4uRvv27TFr1qwae0fN\nB29aJKqBtd4ISWQJ/BdCVAv+rkUkDcOEqBY80UwkDQ9zERGR0TgyISIiozFMiIjIaAwTIiIyGsOE\niIiMxjAhIiKjMUyIiMho/w/TWJJiO+X3vQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "from sklearn.metrics import roc_curve\n", - "\n", - "fpr, tpr, _ = roc_curve(y_eval, probs)\n", - "plt.plot(fpr, tpr)\n", - "plt.title('ROC curve')\n", - "plt.xlabel('false positive rate')\n", - "plt.ylabel('true positive rate')\n", - "plt.xlim(0,)\n", - "plt.ylim(0,)\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "boosted_trees.ipynb", - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/estimator/boosted_trees_model_understanding.ipynb b/site/en/tutorials/estimator/boosted_trees_model_understanding.ipynb deleted file mode 100644 index 978bed62929..00000000000 --- a/site/en/tutorials/estimator/boosted_trees_model_understanding.ipynb +++ /dev/null @@ -1,1504 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7765UFHoyGx6" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "both", - "colab": {}, - "colab_type": "code", - "id": "KVtTDrUNyL7x" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r0_fqL3ayLHX" - }, - "source": [ - "# Gradient Boosted Trees: Model understanding" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PS6_yKSoyLAl" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dW3r7qVxzqN5" - }, - "source": [ - "\n", - "For an end-to-end walkthrough of training a Gradient Boosting model check out the [boosted trees tutorial](./boosted_trees). In this tutorial you will:\n", - "\n", - "* Learn how to interpret a Boosted Trees model both *locally* and *globally*\n", - "* Gain intution for how a Boosted Trees model fits a dataset\n", - "\n", - "## How to interpret Boosted Trees models both locally and globally\n", - "\n", - "Local interpretability refers to an understanding of a model’s predictions at the individual example level, while global interpretability refers to an understanding of the model as a whole. Such techniques can help machine learning (ML) practitioners detect bias and bugs during the model development stage.\n", - "\n", - "For local interpretability, you will learn how to create and visualize per-instance contributions. To distinguish this from feature importances, we refer to these values as directional feature contributions (DFCs).\n", - "\n", - "For global interpretability you will retrieve and visualize gain-based feature importances, [permutation feature importances](https://www.stat.berkeley.edu/~breiman/randomforest2001.pdf) and also show aggregated DFCs." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eylrTPAN3rJV" - }, - "source": [ - "## Load the titanic dataset\n", - "You will be using the titanic dataset, where the (rather morbid) goal is to predict passenger survival, given characteristics such as gender, age, class, etc." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KuhAiPfZ3rJW" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from IPython.display import clear_output\n", - "\n", - "# Load dataset.\n", - "dftrain = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/train.csv')\n", - "dfeval = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/eval.csv')\n", - "y_train = dftrain.pop('survived')\n", - "y_eval = dfeval.pop('survived')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sp1ShjJJeyH3" - }, - "outputs": [], - "source": [ - "try:\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf\n", - "tf.random.set_seed(123)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3ioodHdVJVdA" - }, - "source": [ - "For a description of the features, please review the prior tutorial." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "krkRHuMp3rJn" - }, - "source": [ - "## Create feature columns, input_fn, and the train the estimator" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JiJ6K3hr1lXW" - }, - "source": [ - "### Preprocess the data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "udMytRJC05oW" - }, - "source": [ - "Create the feature columns, using the original numeric columns as is and one-hot-encoding categorical variables." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "upaNWxcF3rJn" - }, - "outputs": [], - "source": [ - "fc = tf.feature_column\n", - "CATEGORICAL_COLUMNS = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',\n", - " 'embark_town', 'alone']\n", - "NUMERIC_COLUMNS = ['age', 'fare']\n", - "\n", - "def one_hot_cat_column(feature_name, vocab):\n", - " return fc.indicator_column(\n", - " fc.categorical_column_with_vocabulary_list(feature_name,\n", - " vocab))\n", - "feature_columns = []\n", - "for feature_name in CATEGORICAL_COLUMNS:\n", - " # Need to one-hot encode categorical features.\n", - " vocabulary = dftrain[feature_name].unique()\n", - " feature_columns.append(one_hot_cat_column(feature_name, vocabulary))\n", - "\n", - "for feature_name in NUMERIC_COLUMNS:\n", - " feature_columns.append(fc.numeric_column(feature_name,\n", - " dtype=tf.float32))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9rTefnXe1n0v" - }, - "source": [ - "### Build the input pipeline" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-UOlROp33rJo" - }, - "source": [ - "Create the input functions using the `from_tensor_slices` method in the [`tf.data`](https://www.tensorflow.org/api_docs/python/tf/data) API to read in data directly from Pandas." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9dquwCQB3rJp" - }, - "outputs": [], - "source": [ - "# Use entire batch since this is such a small dataset.\n", - "NUM_EXAMPLES = len(y_train)\n", - "\n", - "def make_input_fn(X, y, n_epochs=None, shuffle=True):\n", - " def input_fn():\n", - " dataset = tf.data.Dataset.from_tensor_slices((X.to_dict(orient='list'), y))\n", - " if shuffle:\n", - " dataset = dataset.shuffle(NUM_EXAMPLES)\n", - " # For training, cycle thru dataset as many times as need (n_epochs=None).\n", - " dataset = (dataset\n", - " .repeat(n_epochs)\n", - " .batch(NUM_EXAMPLES))\n", - " return dataset\n", - " return input_fn\n", - "\n", - "# Training and evaluation input functions.\n", - "train_input_fn = make_input_fn(dftrain, y_train)\n", - "eval_input_fn = make_input_fn(dfeval, y_eval, shuffle=False, n_epochs=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HttfNNlN3rJr" - }, - "source": [ - "### Train the model" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 390 - }, - "colab_type": "code", - "id": "tgEzMtlw3rJu", - "outputId": "7e3cb2ae-abc8-425f-ad38-cfb8cdf58fa5" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    0
    accuracy0.806818
    accuracy_baseline0.625000
    auc0.866942
    auc_precision_recall0.845483
    average_loss0.421330
    global_step100.000000
    label/mean0.375000
    loss0.421330
    precision0.755319
    prediction/mean0.385978
    recall0.717172
    \n", - "
    " - ], - "text/plain": [ - " 0\n", - "accuracy 0.806818\n", - "accuracy_baseline 0.625000\n", - "auc 0.866942\n", - "auc_precision_recall 0.845483\n", - "average_loss 0.421330\n", - "global_step 100.000000\n", - "label/mean 0.375000\n", - "loss 0.421330\n", - "precision 0.755319\n", - "prediction/mean 0.385978\n", - "recall 0.717172" - ] - }, - "execution_count": 0, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "params = {\n", - " 'n_trees': 50,\n", - " 'max_depth': 3,\n", - " 'n_batches_per_layer': 1,\n", - " # You must enable center_bias = True to get DFCs. This will force the model to\n", - " # make an initial prediction before using any features (e.g. use the mean of\n", - " # the training labels for regression or log odds for classification when\n", - " # using cross entropy loss).\n", - " 'center_bias': True\n", - "}\n", - "\n", - "est = tf.estimator.BoostedTreesClassifier(feature_columns, **params)\n", - "# Train model.\n", - "est.train(train_input_fn, max_steps=100)\n", - "\n", - "# Evaluation.\n", - "results = est.evaluate(eval_input_fn)\n", - "clear_output()\n", - "pd.Series(results).to_frame()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JgAz3jDa_tRA" - }, - "source": [ - "For performance reasons, when your data fits in memory, we recommend use the `boosted_trees_classifier_train_in_memory` function. However if training time is not of a concern or if you have a very large dataset and want to do distributed training, use the `tf.estimator.BoostedTrees` API shown above.\n", - "\n", - "\n", - "When using this method, you should not batch your input data, as the method operates on the entire dataset.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 54 - }, - "colab_type": "code", - "id": "y7ztzoSk_vjY", - "outputId": "028523bf-556f-4beb-f9eb-1e476aa4ddaa" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'accuracy_baseline': 0.625, 'loss': 0.41725963, 'auc': 0.86810529, 'label/mean': 0.375, 'average_loss': 0.41725963, 'accuracy': 0.80681819, 'global_step': 153, 'precision': 0.75531918, 'prediction/mean': 0.38610166, 'recall': 0.71717173, 'auc_precision_recall': 0.8542586}\n" - ] - } - ], - "source": [ - "in_memory_params = dict(params)\n", - "in_memory_params['n_batches_per_layer'] = 1\n", - "# In-memory input_fn does not use batching.\n", - "def make_inmemory_train_input_fn(X, y):\n", - " y = np.expand_dims(y, axis=1)\n", - " def input_fn():\n", - " return dict(X), y\n", - " return input_fn\n", - "train_input_fn = make_inmemory_train_input_fn(dftrain, y_train)\n", - "\n", - "# Train the model.\n", - "est = tf.estimator.BoostedTreesClassifier(\n", - " feature_columns, \n", - " train_in_memory=True, \n", - " **in_memory_params)\n", - "\n", - "est.train(train_input_fn)\n", - "print(est.evaluate(eval_input_fn))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TSZYqNcRuczV" - }, - "source": [ - "## Model interpretation and plotting" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BjcfLiI3uczW" - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "sns_colors = sns.color_palette('colorblind')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ywTtbBvBuczY" - }, - "source": [ - "## Local interpretability\n", - "Next you will output the directional feature contributions (DFCs) to explain individual predictions using the approach outlined in [Palczewska et al](https://arxiv.org/pdf/1312.1121.pdf) and by Saabas in [Interpreting Random Forests](http://blog.datadive.net/interpreting-random-forests/) (this method is also available in scikit-learn for Random Forests in the [`treeinterpreter`](https://github.com/andosa/treeinterpreter) package). The DFCs are generated with:\n", - "\n", - "`pred_dicts = list(est.experimental_predict_with_explanations(pred_input_fn))`\n", - "\n", - "(Note: The method is named experimental as we may modify the API before dropping the experimental prefix.)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TIL93B4sDRqE" - }, - "outputs": [], - "source": [ - "pred_dicts = list(est.experimental_predict_with_explanations(eval_input_fn))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 328 - }, - "colab_type": "code", - "id": "tDPoRx_ZaY1E", - "outputId": "72c9a915-e8c9-4104-f737-3282c6ff5960" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    countmeanstdmin25%50%75%max
    age264.0-0.0252850.085606-0.172007-0.075237-0.0512900.0119080.466216
    sex264.00.0081380.108147-0.097388-0.074132-0.0726980.1384990.197285
    class264.00.0173170.093436-0.075741-0.045486-0.0444610.0350600.248212
    deck264.0-0.0160940.033232-0.096513-0.042359-0.0267540.0050530.205399
    fare264.00.0164710.089597-0.330205-0.031050-0.0114030.0502120.229989
    embark_town264.0-0.0073200.027672-0.053676-0.015208-0.014375-0.0030160.068483
    n_siblings_spouses264.00.0039730.027914-0.1446800.0035820.0048250.0064120.138123
    parch264.00.0013420.007995-0.0628330.0004050.0005100.0028340.048722
    alone264.00.0000000.0000000.0000000.0000000.0000000.0000000.000000
    \n", - "
    " - ], - "text/plain": [ - " count mean std ... 50% 75% max\n", - "age 264.0 -0.025285 0.085606 ... -0.051290 0.011908 0.466216\n", - "sex 264.0 0.008138 0.108147 ... -0.072698 0.138499 0.197285\n", - "class 264.0 0.017317 0.093436 ... -0.044461 0.035060 0.248212\n", - "deck 264.0 -0.016094 0.033232 ... -0.026754 0.005053 0.205399\n", - "fare 264.0 0.016471 0.089597 ... -0.011403 0.050212 0.229989\n", - "embark_town 264.0 -0.007320 0.027672 ... -0.014375 -0.003016 0.068483\n", - "n_siblings_spouses 264.0 0.003973 0.027914 ... 0.004825 0.006412 0.138123\n", - "parch 264.0 0.001342 0.007995 ... 0.000510 0.002834 0.048722\n", - "alone 264.0 0.000000 0.000000 ... 0.000000 0.000000 0.000000\n", - "\n", - "[9 rows x 8 columns]" - ] - }, - "execution_count": 0, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# Create DFC Pandas dataframe.\n", - "labels = y_eval.values\n", - "probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])\n", - "df_dfc = pd.DataFrame([pred['dfc'] for pred in pred_dicts])\n", - "df_dfc.describe().T" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EUKSaVoraY1C" - }, - "source": [ - "A nice property of DFCs is that the sum of the contributions + the bias is equal to the prediction for a given example." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Hd9VuizRaY1H" - }, - "outputs": [], - "source": [ - "# Sum of DFCs + bias == probabality.\n", - "bias = pred_dicts[0]['bias']\n", - "dfc_prob = df_dfc.sum(axis=1) + bias\n", - "np.testing.assert_almost_equal(dfc_prob.values,\n", - " probs.values)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tx5p4vEhuczg" - }, - "source": [ - "Plot DFCs for an individual passenger. Let's make the plot nice by color coding based on the contributions' directionality and add the feature values on figure." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6z_Tq1Pquczj" - }, - "outputs": [], - "source": [ - "# Boilerplate code for plotting :)\n", - "def _get_color(value):\n", - " \"\"\"To make positive DFCs plot green, negative DFCs plot red.\"\"\"\n", - " green, red = sns.color_palette()[2:4]\n", - " if value >= 0: return green\n", - " return red\n", - "\n", - "def _add_feature_values(feature_values, ax):\n", - " \"\"\"Display feature's values on left of plot.\"\"\"\n", - " x_coord = ax.get_xlim()[0]\n", - " OFFSET = 0.15\n", - " for y_coord, (feat_name, feat_val) in enumerate(feature_values.items()):\n", - " t = plt.text(x_coord, y_coord - OFFSET, '{}'.format(feat_val), size=12)\n", - " t.set_bbox(dict(facecolor='white', alpha=0.5))\n", - " from matplotlib.font_manager import FontProperties\n", - " font = FontProperties()\n", - " font.set_weight('bold')\n", - " t = plt.text(x_coord, y_coord + 1 - OFFSET, 'feature\\nvalue',\n", - " fontproperties=font, size=12)\n", - "\n", - "def plot_example(example):\n", - " TOP_N = 8 # View top 8 features.\n", - " sorted_ix = example.abs().sort_values()[-TOP_N:].index # Sort by magnitude.\n", - " example = example[sorted_ix]\n", - " colors = example.map(_get_color).tolist()\n", - " ax = example.to_frame().plot(kind='barh',\n", - " color=[colors],\n", - " legend=None,\n", - " alpha=0.75,\n", - " figsize=(10,6))\n", - " ax.grid(False, axis='y')\n", - " ax.set_yticklabels(ax.get_yticklabels(), size=14)\n", - "\n", - " # Add feature values.\n", - " _add_feature_values(dfeval.iloc[ID][sorted_ix], ax)\n", - " return ax" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 444 - }, - "colab_type": "code", - "id": "Ht1P2-1euczk", - "outputId": "c666aea0-a8ee-4de1-c8fa-f19cdee75e88" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtcAAAGrCAYAAAALhrt6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl8TNf/x/HXZLIggthjV0qskSD2LbHvS31p8yXa2lKx\nU5SitKWWloraWutXN2urVfq1FK362otWamuLSCRIhYRkkrm/P/xMmyb2iZh4Px8Pj87ce+45n3vP\n0M+cOfdck2EYBiIiIiIi8sicMjsAEREREZGsQsm1iIiIiIidKLkWEREREbETJdciIiIiInai5FpE\nRERExE6UXIuIiIiI2ImSa8lQUVFR9OjRAx8fH7y9vdmxY0dmhyQiIiKSYZRcS4ZasGAB+/bto2TJ\nkvTq1YtixYo9Un3r1q3D29ubnj172ilCEREREftxzuwAJGv7/fffMZlM9OrVi86dOz9yffZ85lFy\ncjLOzvorICIiIvajkWvJMD169ODHH38E4LXXXqNChQqcPn2aGTNm0Lx5c3x9fencuTNbtmyxHfPl\nl1/Spk0b/Pz8qFy5Mi1atODjjz8Gbo1av/baa5hMJvbu3Yu3tzeBgYEAeHt7U6FCBS5cuABAWFgY\n3t7ejBkzxnast7c3L7zwAhMnTsTPz4/58+cDsHXrVrp27Ur16tUJCAjgnXfe4ebNm4/tOomIiEjW\noWE7yTAtW7bk7NmzREdHU69ePcqWLcvUqVPZtWsXlSpVws/Pjx07djBo0CCWLVtGzZo1uXDhAsWL\nF8ff35+EhAQ2b97M5MmTqVSpEmXKlKFevXr88MMPFC5cmBYtWpAnT547tm8ymdJsO3ToELGxsbRr\n147ixYvz/fffM2DAAPLmzUvTpk05deoUS5Ys4erVq7z99tsZeXlEREQkC9LItWSYoKAgSpQoAUC7\ndu3o168fu3btwmw24+vrS65cuShbtixWq5VPP/0UgJdffplOnTqRL18+8uTJQ+HChQH43//+R9Wq\nVWnbti0AJUqUYMyYMYSEhNyx/fSmkLi7u7Nq1SreeOMNOnTowIoVKzCZTFSoUIHcuXNTtWpVANav\nX09iYqJdr4eIiIhkfRq5lscmIiICAKvVysqVK23bTSYTZ8+eBaB///788MMPaUadr1y58kBtpaSk\npLu9bNmy5MyZM01Mu3fvZvfu3bZ4AM6dO0fZsmUfqF0RERF5uim5lsemaNGiALi4uLBz507blI7k\n5GQuX77MtWvXbIn18uXLqVmzJn379mXXrl22UWgnp1s/tvxzVDp79uzcvHmT69evA3DixIl0p4W4\nurqmien06dOMGzeOoKAg2/Zz585RvHhxO525iIiIPC2UXMtjkzdvXlq1asWmTZvo2rUrdevWJTY2\nloMHD9K9e3f69+9Pjhw5uHHjBu+//z65c+e23RB5m5eXFwDHjh3jjTfeoGLFinTt2pUKFSpw6NAh\nJk2aROnSpdm2bdt9rSwSFBTEjh07mDZtGgcPHsTNzY3w8HDi4uJS3WgpIiIicj8051oeq7fffpu+\nffvi5OTE+vXrOXz4MH5+fjRs2BBnZ2emTZuGl5cXx44dI1euXLRs2RL4a6pGzZo1adeuHWazmU8/\n/ZStW7cC8Prrr1OuXDnCw8O5ePEiXbp0wWQypRq9/ud7gIYNGzJ37lwqVKjAzp072bJlC87OzgQH\nBz+mKyIiIiJZicmw58LBIiKSJS1YsIDz588zefJkIiIiCAwM5JdffrFN1XoUkZGRtGnThgMHDqQ7\nnSsjJCYmMnjwYPbv30/9+vWZNWvWY2k3swUEBPDWW29Rp06dzA5FJMvSyLWIZHkBAQH4+Pjg5+eH\nr68vfn5+xMTEPFKde/fupVGjRnaKMPPc73n069ePyZMn294/ShIcEBCQasqXl5cXBw8efGyJNcCm\nTZu4cuUK+/bte2oS60dhsVgYNGgQAQEBeHt7s2/fvlT7k5KSGD9+PPXq1aNWrVqEhIRw8eJF276x\nY8cSEBBA9erV6dy5Mzt37syM0xB5LJRci8hTYcGCBRw8eJBDhw5x8OBBChQo8Ej1GYbxSMngnVa0\nedzu5zyelFjt6cKFC5QqVeqh+jArXo/7UaNGDWbMmJHu351ly5Zx5MgRNmzYwK5du/Dw8OCtt94C\nbl0vLy8vVq5cyYEDBxg0aBBDhgyxPfRLJKtRci0iT4U7zYA7fPgw3bt3p2bNmnTs2JG9e/fa9q1d\nu5bWrVvj5+dHs2bN+OyzzwC4ceMGffv2JTo6OtVI+JgxY5g9e7bt+H+OCgcEBLBo0SLat2+Pr68v\nVquV6OhoBg0aRJ06dWjatCkrVqy44zkkJiYydepUAgICqFmzJkFBQSQlJQG3njTatm1b/P396dmz\nJ6dPn07V7uLFi2nfvj01a9Zk6NChJCUl3fE8wsLCGDRoECNHjqRGjRqsW7eOsLAwRo4cmep6rl69\nmgYNGtCgQQOWLFli23e36/Dqq68SGRlJSEgIfn5+fPTRR0RERODt7Y3VagUgOjqakJAQatWqRYsW\nLVi1apWtrrCwMIYMGcKoUaPw8/OjXbt2/Pzzz7b9CxcupGHDhvj5+dGqVSv27NmT5jrOmTOHuXPn\nsnHjRvz8/FizZg2GYfDBBx8QEBBAvXr1GD16tG31odvxrV69miZNmtCrV690+2f79u107NiRmjVr\n8vzzz/Prr78Ct1YfqlWrFsePHwfg4sWL1K5d2zb6e6fP2d+v3YcffkjdunVp0KABW7ZsYceOHbRo\n0YJatWqxYMGCVNdn0KBBDB06FD8/Pzp37kx4eHi68RqGwcKFC2nWrBm1a9dm6NChxMXFpVvWxcWF\nnj174ufnl+5UoIiICOrXr0/evHlxdXWlTZs2nDx5Eri1mlNoaKjthvTGjRtTrFixVP0mkqUYIiJZ\nXJMmTYzdu3en2R4VFWX4+/sbO3fuNAzDMHbv3m34+/sbV65cMQzDML777jvj3LlzhmEYxr59+wwf\nHx/jl19+MQzDMP73v/8ZjRo1SlXf6NGjjVmzZtne/7NMkyZNjI4dOxpRUVFGYmKiYbVajU6dOhkf\nfPCBkZycbJw7d85o2rSp8f3336d7HhMnTjR69OhhREdHG1ar1Th06JCRlJRknDlzxqhWrZqxe/du\nIzk52Vi0aJHRrFkzw2Kx2Nrt2rWrERMTY1y9etVo1aqV8emnn97xPObMmWNUqlTJ2Lp1q2EYhnHz\n5k1jzpw5xsiRIw3DMIzz588b5cuXN4YNG2bcvHnT+PXXX43atWvbrvH9XIcff/zR9v78+fOGt7e3\nkZKSYhiGYQQFBRmTJk0ykpKSjOPHjxu1a9e2lZ8zZ45RtWpVY+fOnYbVajVmzpxp/Otf/zIMwzDO\nnDljNGrUyIiJiTEMwzAiIiKMs2fPpnst/34+hmEYq1atMpo3b26cP3/eSEhIMEJDQ9Oc76hRo4wb\nN24YiYmJaeo7duyYUadOHePIkSOG1Wo11q1bZzRp0sRISkoyDMMwPv/8c6N169bGjRs3jJdeesmY\nNm2a7dh7fc4qVqxo+4x8/vnnRu3atY3hw4cbCQkJxsmTJ40qVarYjr/dd99++62RnJxsfPTRR0ZA\nQICRnJxsu/a3+2nJkiVGt27djIsXLxpJSUnG+PHjjWHDhqV7vf6uYcOGxt69e1NtO3r0qNG9e3fj\n4sWLRkJCgjFs2DBjypQp6R4fExNjVK1a1Thz5sw92xJxRBq5lieWt7c3FSpU0E+HYhcDBgzA398f\nf39/QkNDAfjyyy9p3LgxDRo0AKBOnTpUrlyZHTt2ANCoUSOKFSsG3PpJvF69euzfv/+R4ujZsyeF\nChXC1dWVo0eP8ueffxISEoLZbKZYsWJ07dqVr7/+Os1xhmGwdu1axo0bR4ECBTCZTFSrVg0XFxe+\n+eYbGjduTJ06dTCbzbz88svcvHmTQ4cOpWo3f/785MqViyZNmthGUe/E19eXgIAAANzc3NItM3Dg\nQNzc3ChXrhydO3dON+47Me7wS0JkZCSHDh1ixIgRuLi44O3tTdeuXfniiy9sZapXr06DBg0wmUx0\n6NDBNkJsNpuxWCycPHmS5ORkihQpct/r1X/11Vf06tWLokWLkj17doYNG8bGjRtto+kmk4mBAweS\nLVu2NOvlA6xatYru3btTpUoVTCYTHTt2xNXVlZ9++gmArl27UrJkSbp27cqlS5cYMmSI7dh7fc5c\nXFzo378/ZrOZ1q1bExsbS3BwMNmzZ6ds2bKULVvWdg0AKleuTLNmzTCbzbz44oskJiZy+PDhNDF/\n/vnnDBkyhIIFC+Li4sKAAQPYvHmz7ZwfRKlSpShSpAgNGzakZs2a/Pbbb7zyyitpyiUnJzNy5Eg6\ndepE6dKlH7gdEUegda5F5KnwwQcfULt27VTbLly4wDfffMP27duBWwlfcnKyrdyOHTv44IMP+P33\n37Fardy8eZPy5cs/UhyFCxe2vY6IiODixYv4+/vb2rdardSsWTPNcbGxsSQlJaWbLEZHR1OkSBHb\ne5PJhJeXl+2GMoB8+fLZXmfPnv2eN3T+Pc70mEymVGWKFClimwbwKGJiYsidOzfZs2dPVfffpxDk\nz5/f9jpbtmwkJiZitVopUaIEr732GnPmzOH06dPUr1+fUaNGUbBgwXu2+89rWLRoUZKTk7l06ZJt\n292uyYULF/jiiy/4z3/+A/z1WYqOjraV6dq1K6+88gqTJk3CxcXFtv1en7M8efLY5oZny5YNSN2f\n2bJlIyEhId04b/fT3+P4e8yhoaGpHs7l7OzMpUuX7uua/d2ECRNISkpi3759ZMuWjUWLFtG7d28+\n//xzWxnDMBg5ciSurq68/vrrD1S/iCNRci0iT4X0Rkq9vLzo2LEjkyZNSrMvKSmJwYMHM336dAID\nA3FycmLAgAG2etK7Ee72k0Jvu1cC6+XlRbFixdi8efM94/f09MTNzY2zZ8+mSfALFiyYJrGNjIy8\nZ4IMd171435u9IuMjLSNPkZGRtoSsntdh7vVXbBgQa5evUpCQgI5cuRIU/e9tGnThjZt2hAfH8/4\n8eOZOXMm77zzzj2PK1iwYKpfySIiInB2diZ//vxERkbeM+7ChQvTv39/+vXrl+7+hIQE3n77bZ57\n7jnCwsJo0aIFuXLluufn7GFERUXZXhuGQVRUFIUKFUpTzsvLi7fffhtfX9+Hbuu2EydOMHToUDw8\nPADo0aMH77//Pn/++aftabyvvfYasbGxLFy4ELPZ/MhtijypNC1EMszo0aPx9vZm6dKltm1jxozB\n29ubxYsX89JLL1G/fn0qV65MzZo1CQkJSfU/hX/65zSRsLAwvL29GTNmjK3M1q1b6dq1K9WrVycg\nIIB33nkn1f/kRf6uffv2bNu2je+//x6r1UpiYiJ79+7l4sWLWCwWLBYLnp6eODk5sWPHDn744Qfb\nsfny5ePPP/+03fQGUKFCBXbs2MHVq1eJiYlh+fLld22/atWq5MyZk0WLFpGYmEhKSgonT57k6NGj\nacqaTCa6dOnC1KlTiY6Oxmq1cvjwYSwWC61ateK7775jz549JCcn89FHH+Hm5ka1atXueQ3SO4/7\nYfz/DYA3b97k5MmTrF27ljZt2tzXdShQoADnz59PUx/cSlJ9fX159913SUpKIjw8nNWrV9O+ffu7\nxgLw22+/sWfPHpKSknBxccHNze2+1+Fu06YNS5cu5fz588THx/Pee+/Rpk2bVKO6d/Ovf/2LTz/9\nlCNHjgC3kukdO3bYRpTffPNNqlSpwuTJk2nUqBHjx48HuOfn7GEcO3aMLVu2kJKSwtKlS3Fzc8PH\nxydNuW7duvHuu+/a/k29cuWK7cFc6UlKSiIxMdH2+vbNtHBrKsr69eu5fv06FouFlStXUqhQIVti\nPX78eH777TfmzZuX7rQakaxEybVkmA4dOgDwzTffALfm2m3btg1nZ2datWpFTEwMDRo0oFu3bhQv\nXpzt27c/8E+Ffx9J2rVrFwMGDCAiIoKmTZvi6enJkiVL0h2VlKfLnUYcCxcuzAcffMCCBQuoU6cO\nTZo0YfHixRiGgbu7O2PHjmXw4MH4+/uzceNGAgMDbcc+88wztGnThsDAQPz9/YmJiaFDhw6UL1+e\ngIAAevfuTevWre8ah5OTE/Pnzyc8PJzAwEDq1q3L66+/fsdEd9SoUZQrV47nnnuOWrVqMXPmTAzD\noHTp0kyfPp3JkydTp04dvvvuO+bPn4+zs/Ndz/9O53G/19Tf359mzZrx4osv0rt3b9uDSe51Hfr0\n6cMHH3yAv7+/bZWRv8c4c+ZMzp8/T4MGDRg0aBCDBw++60NPbh+blJTEzJkzqVOnDg0aNODKlSsM\nGzbsvs7nueeeo0OHDvz73/+mWbNmZM+enXHjxqVp404qV67M5MmTmTRpEv7+/rRo0YJ169YBt770\n//DDD0ycOBG4NfBw/Phxvvrqq3t+zu52vnd6HxgYyMaNG6lZsyYbNmwgLCzMNlL897LBwcEEBgby\n0ksvUb16dbp37277cpCeli1bUq1aNaKjo+nduzc+Pj62xHzUqFG4urrSvHlz6tWrx65duwgLCwNu\nTT/5/PPPOX78OHXr1rWtTPPVV1/d9TxFHJWe0CgZxjAMAgICiIqKYsuWLZw8eZL+/fvTsGFDFi5c\nyB9//MH27duJiYnh8uXLrF+/Hjc3N9sNQN7e3phMJrZu3UqRIkXSvA8LCyMsLIxOnToxZcoU+vbt\ny65du6hbty5lypTBYrHwySef4OTkxKFDh+54U5aISFYRFhbG2bNnmTZtWmaHIvLU0pxryTAmk4n2\n7duzcOFCNm7cyOnTp2130e/fv5+ePXtitVpTjaQkJSVx/fp1cubMec/6//kgh9sjKLt372b37t22\nGODWOrNly5a116mJiIiIpEvTQiRDdejQAcMw2LBhA1u3biVnzpw0bdqUb7/9FqvVSsOGDTl8+HCq\nhybcye3VA27/ZH7ixIlUiXnRokUBGDduHMePH7f92bx5sxJrEREReSw0ci0Z6plnnqFKlSq2G7S6\ndu2Kq6urbSmtn376iUmTJtmeVHY3FSpU4NChQ0yaNInSpUuzbdu2VDcZBQUFsWPHDqZNm8bBgwdx\nc3MjPDycuLg4tmzZkjEnKCLyBLm9hruIZB6NXEuG69ixIyaTCScnJ9tNjkFBQTRr1oykpCT2799P\nSEgIJpPprjfqvP7665QrV47w8HAuXrxIly5dUh3TsGFD5s6dS4UKFdi5cydbtmzB2dmZ4ODgx3ey\nIiIi8lTTDY0iIvLAevToQYcOHXjuuecyO5T7tnfvXkaOHGl7AufdrFu3jlWrVvHxxx8/cDuPcqyI\nOD6NXIuISIZbunQp9evXp2bNmowdOxaLxZJuOYvFwqBBgwgICMDb2zvNlLGwsDAqV66Mn5+fbUm3\nf66ZfTf383Cchyn7KMceP36czp07U61aNbp06UJ4ePhDtysimU/JtYjIU+6fK+/Y265du/jwww9Z\ntmwZ27Zt4+zZs8yZM+eO5WvUqMGMGTMoUKBAuvtbt27NwYMHOXToEAcPHqRYsWIZFXqGs1gsDBgw\ngI4dO7Jv3z46duzIK6+8QnJycmaHJiIPScm1iEgW5O3tzYoVK2jatCl16tRJte7xunXreP7555ky\nZQq1atWyPexj9erVtG7dmlq1atG7d+9UjwP/4YcfaNWqFTVr1mTy5MkPFMv69evp0qULZcqUwcPD\ng1deeYW1a9emW9bFxYWePXvi5+d3309X/Lv27dvz9ddf31fZhQsX0qxZM/z8/Gjbtm2aG5+tVitv\nvvkmNWrUoHXr1vz444+2fdevX2fs2LHUr1+fRo0aMWvWrId6ZPnevXtJSUmhZ8+euLi40KNHDwzD\nYM+ePQ9cl4g8GZRci4hkUVu2bGHdunWsW7eOrVu3snr1atu+I0eOUKJECX788UdCQkLYsmULixYt\nYu7cufz444/UqFHD9nTDK1euMGjQIIYNG8aePXsoXrw4Bw8etNUVGRmJv78/UVFR6cZx6tQpvL29\nbe+9vb25fPkyV69efajz2r59O7Vq1aJdu3Z88sknqfZ9+eWXtsew30vJkiX55JNPOHjwIAMGDGDk\nyJFcunTJtv/2Nfrf//5HaGgoAwcOJC4uDoBXX30VFxcXtm7dyrp169i9ezerVq1Kt53+/fuzaNGi\ndPedPHmS8uXLp9pWvnx5Tp06dV/nICJPHiXXIiJZVN++ffHw8KBw4cIEBwenGtEtVKgQQUFBODk5\n4erqymeffUbfvn0pXbo0Tk5O9O3bl/DwcCIjI9m5cyfPPvsszZo1w2w206tXL9tymgBeXl7s3buX\nwoULpxtHQkICHh4etvceHh4YhkF8fPwDn1OrVq3YuHEje/bsYdKkScydO5eNGzc+cD0ALVq0sJ1H\nq1atKFmyZKrHf+fLl4+ePXtiNptp3bo1pUuX5rvvvuPy5cvs2rWL1157DTc3N/LmzUtwcPAdH+c9\nf/58+vTpk+6+f14bgJw5c9rW8xcRx6N1rkVEsqi/J7tFixYlOjo63X1w6wmnb731Fu+88w4AhmFg\nMpm4ePEi0dHRacp7eXnddxw5cuRIlSxev34dk8mEu7v7A50PQJkyZWyvfX196dmzJ5s2baJ169YP\nXNf69etZunQpERERANy4cYPY2Fjb/kKFCqUqX6RIEaKjo4mIiCA5OZn69esDt66VYRgPdE1u++e1\nAe77KbUi8mRSci0ikkVFRkbaktELFy5QsGBB275/rmbh5eVFSEgIbdu2TVPP77//ztatW9PUfb/K\nli1LeHg4LVu2BG6tjpEvXz5y585933XcycOu6HHhwgVef/11li9fjq+vL3BrTf6/z5u+ePFiqmMi\nIyMJDAzEy8sLNzc3/ve//z3SiiIAzz77LEuXLk217cSJE/z73/9+pHpFJPNoWohkiLi4OObMmWOb\nnyiOQ33n2G73H8BHH31EXFwckZGRLF++/K6ju927d2fBggW2ub7Xrl1j06ZNADRq1IhTp06xZcsW\nUlJSWLZsGZcvX77vmDp27Mjq1as5ffo0V69eZf78+XTp0uWO5ZOSkkhMTLS9TkpKsu3bunWr7bN5\n5MgRVqxYQWBgoG1/QEAA69evv2dMN27cwMnJCU9PT6xWK2vWrOHkyZOpyly+fJkVK1aQnJzMN998\nw5kzZ2jUqBEFChSgXr16vP3221y/fh3DMDh37tx9PWn2n/z9/XFycmLFihVcvnyZPn36YBgGtWvX\nfuC6JHPp307HZs/+U3ItGSIuLo6wsDD9I+OA1HeO7Xb/AQQGBtK5c2c6depEkyZN7vrAl6ZNm9Kn\nTx+GDh1KjRo1aN++Pbt27QLA09OT2bNnM336dGrXrs25c+dso71wa0TXz8/vjjc0NmjQgN69e9Oz\nZ08CAwMpVqxYqsd0t23bNtV85ZYtW1KtWjWio6Pp3bs3Pj4+tpVLNm7caFvhY/To0fTr18/25FeL\nxcLVq1fx8fG553UqU6YML774It26daNevXqcOnUKPz+/VGV8fHz4448/qF27NrNnz2bOnDm20fZ3\n3nkHi8VCmzZt8Pf3Z/DgwcTExKTbVp8+fVi4cGG6+1xcXPjggw9Yt24dAQEB7Ny5k4kTJ+LsrB+W\nHY3+7XRs9uw/PaFRMsT58+cJDAxk69atDr0G7dNIfefYbvefyWTiv//9L8WLF8/skB6bAwcO8PHH\nHzNz5szMDuWh6O+eY1P/OTZ79p++GouISJZQvXp1qlevntlhiMhTTtNCRESyoEe90U5ERB6OkmvJ\nMH5+fpjN5swOQx6Q2WymaNGi6jsHdbv/tm3b9lRNCckK9HfPsan/HJvZbE5z38XD0pxryRBap1VE\nREQcjT3yFyXXkqFiY+OxWvURczT58uXk8mU9Ic5Rqf8cl/rOsan/HJeTkwlPzwd/sFV6dEOjZCir\n1VBy7aDUb45N/ee41HeOTf0nmnMtIiIiImInSq5FREREROxEybWIiIiIiJ0ouRYRERERsRMl1yIi\nIiIidqLkWkRERETETpRci4iIiIjYida5lizJ3ZwCKSmZHYbDSoqNxR1rZochD0n957jUd47Ncl0P\nkBEl15JVpaQQPv3dzI7CYbm4mLFY9OXEUan/HJf6zrFVHjMCpVaiaSEiIiIiInair1eSodzd3TKl\nXWdcKPmvLgAk/fknkd9uzZQ4RERE5Omi5FoylNVqYBjGY2/XjJWk2FgAXD09H3v7IiIi8nTStBAR\nERERETtRci0iIiIiYidKrkVERERE7ETJdRa1b98+unXrhq+vLzVq1KBbt26cOnUKgIMHD9KjRw+q\nVatGw4YNmThxItf/f23OK1euUL9+febOnWurKzw8nKpVq/Ltt99myrkATJjwOvPmzb13QREREZFM\npOQ6C0pJSWHAgAHUqFGDDRs2sGrVKnr27ImTkxMnTpzg5ZdfJjAwkA0bNhAWFkZ4eDhjx44FIG/e\nvEydOpV58+bx008/kZiYyIgRI2jXrh3NmzfP5DMTERERebJptZAs6Pr161y7do0mTZpQrFgxAEqX\nLg3AqFGjaNOmDb169QKgePHiTJgwgU6dOnHlyhXy5s1L/fr1CQoKYvjw4dSsWROLxcK4cePu2F5c\nXBxxcXF4enqS8v9PRTSZTHh4eGTsiYqIiIjYUWRkpC2XuS1XrlzkypXrvutQcp0F5c6dm44dO/LS\nSy9Rp04d6tSpQ8uWLSlcuDA///wzZ8+e5euvv051jMlk4ty5c+TNmxeA4cOHs3PnTr788ks+/fRT\nsmfPfsf2li1bRlhYGFOmTCEiIgK49UEMDg6mTZuW/Otf3fj666+IiDhPixYtGTBgIBMmvM7hw4eo\nUqUq77wzAw8PD0aNGsGhQwdJTEykXLnyjBkzlmeeKZNumzt37mDevLlcuBBBmTJlGTNmLM8+W85O\nV1BERESeRkFBQbZc5rbQ0FAGDhx433Uouc6ipkyZQq9evdi1axdbt25l1qxZhIWFYbVa6dq1q23k\n+u8KFSpke33+/HmioqJsSXeVKlXu2FZwcDCdOnVKM3J927ZtW5k/fxHJyck8/3xXwsPDmTBhEqVL\nlyY0NISnbW1HAAAgAElEQVRPP/2YPn36Ua9eAyZOnIyzszPvvz+LsWPH8Mknn6dp7/jxX5g0aQLv\nvz+XChUq8vXXXzF06CDWrduAi4vLI1w1EREReZqtXLky3ZHrB6HkOgsrX7485cuXp3fv3vTp04f1\n69dTqVIlTp48SfHixe94XHJyMiNHjiQwMJCqVasyYcIE/Pz8KFy4cLrl7/VzSffuz+P5/w9y8fX1\nI2/efJQrd2uUuUmTQPbt2wtA+/YdbMf07duPjz/+D/Hx8bi7u6eqb/36tXTp0pWKFSsB0LZtOxYv\nXsTRo0fw86t+H1dGREREJC0vL69HrkPJdRZ0/vx5PvvsMwICAihUqBBnz57l119/JSgoiMaNG9Ot\nWzcmTJhA9+7dcXd35/Tp02zfvp1JkyYBMGvWLGJjY1m2bBk5c+Zk586djBw5khUrVjxUPHnz5rO9\ndnPLRr58f73Pls2NGzcSsFqthIW9z5Yt/+XPP//EZLo1+v3nn7FpkuvIyEi++moDn332CQCGYZCc\nnExMTMxDxSciIiJiL0qus6Ds2bPz+++/M2TIEGJjY8mfPz8dOnSgd+/emM1mVq5cyaxZs+jRowcp\nKSkUL16cZs2aAbeW8Fu6dClLly4lZ86cAEydOpUOHTqwcOFC+vbtmyExf/PNRnbs+I4FCz7Ey8uL\na9eu0bhxfdJ7cnqhQoV5+eU+vPRS7wyJRURERORhKbnOgvLly8ecOXPuuL9SpUosWrQo3X01a9bk\n2LFjqbblz5+fH374wa4x/tONGwm4ubmSK1cubtxIICxsdqp523/XqVMXRowYir9/LSpXrsKNGwkc\nOHCA6tWrkz17jgyNU0RERORutM61ZKh/5sd3yJdp06YdhQt70bJlU7p27UzVqtXuWGfFihV5/fUJ\nvPPOFBo3rk/Hju3ZsOFLO0YtIiIi8nBMhpHeD+8i9pGQkERmfMRcSeHS7h9vvfb05I/P1zz2GByZ\ni4sZiyXl3gXliaT+c1zqO8dWecwIriZrUoAjcnIykS9fTvvUZZdaREREREREybWIiIiIiL1oWohk\nqEybFuJkgNUKgMnsRNK1+McegyNzdnYiOdma2WHIQ1L/OS71nWNzye7GnzeUVjkie04L0cQgyZKS\nrCbADIAZJ+JxzdyAHEwBTw+uxlzL7DDkIan/HJf6zrEVyJkTbqj/nnaaFiIiIiIiYicauZYM5eRk\nAu6w/t5jkpKin1hFRETk8VByLRkqPj4Rq1Xzz0REROTpoGkhIiIiIiJ2ouRaRERERMROlFyLiIiI\niNiJkmsRERERETtRci0iIiIiYidKrkVERERE7ETJtYiIiIiInSi5FhERERGxEyXXIiIiIiJ2ouRa\nRERERMROlFyLiIiIiNiJkmsRERERETtRci0iIiIiYidKrkVERERE7ETJtYiIiIiInSi5FhERERGx\nE+fMDkDkcXA3p0BKSmaH4TCSYmNxx5rZYchDUv85LvWdY7Ncv57ZIcgTQMm1PB1SUgif/m5mR+Ew\nXFzMWCz6MuKo1H+OS33n2CqPGYFSK9G0EBERERERO9HXK8lQ7u5uj73NlBQrCQlJj71dERERESXX\nkqGsVgPDMB5rm2azfpARERGRzKEsRERERETETpRci4iIiIjYiZJrERERERE7UXKdxezatYugoCD8\n/f2pVasWL7/8MqdPn7bt/+mnn+jcuTNVq1alc+fO7NixA29vb/bt22crc+rUKfr164efnx9169Zl\n+PDhXLp0KTNOR0RERMShKLnOYm7cuEGvXr1Ys2YNK1asIFeuXISEhJCcnExCQgL9+/enTJkyrFu3\njpEjRzJ9+nRMJpPt+JiYGP79739Tvnx51qxZw9KlS0lISCAkJOSh4hk3bgzNmwfSsGFdOnduz/r1\nawGwWCy8+upw2rZtSfXqPhw4sP+u9cTFxTF8+BDq1atF27Yt2bRp40PFIyIiIpKRtFpIFtO8efNU\n79966y1q1KjBkSNHOHnyJFarlbfeegtXV1fKlClD//79GTlypK38J598QoUKFRg2bJht29SpU6lV\nqxZHjx6lSpUqadqMi4sjLi4OT09PUv7/KYgmkwkPDw9eeqkP48e/gYuLC3/88Tt9+ryEt3cFypQp\ni6+vH0FBPXj11RH3PK8pU97E1dWVrVt3EB5+nEGDQilXzptnnnnmYS+ViIiISCqRkZG2XOa2XLly\nkStXrvuuQ8l1FnPu3DlmzZrFkSNHuHLlClarFcMwiIyM5MyZM5QrVw5XV1dbeR8fn1RL5f3888/s\n27cPX1/fVPWaTCbOnTuXbnK9bNkywsLCmDJlChEREcCtD2JwcDDPPPOMrf5b/zVx/vw5vL0r8Pzz\nQQA4OZnS1Pl3N27cYNu2raxevZ5s2bJRrZovjRo14uuvNzBw4OCHuk4iIiIi/xQUFGTLZW4LDQ1l\n4MCB912Hkusspl+/fnh5eTFp0iQKFSqEs7MzrVu3xmKx3Nd601arlcaNGzNq1Kg0+/Lly5fuMcHB\nwXTq1CnNyPVtU6a8xYYNX5CYmIi3dwXq1WvwQOd09uwfmM1mihcvbttWrlx5Dh488ED1iIiIiNzN\nypUr0x25fhBKrrOQP//8kzNnzjBx4kT8/f2BWyPRycnJAJQpU4YvvviCpKQk2+j1Tz/9lCoRrlix\nIps2baJIkSKYzeb7avdeP5eMGTOW0aNf48iRn9i/f1+qkfP7kZCQQM6cHqm25cyZk/j4+AeqR0RE\nRORuvLy8HrkO3dCYheTOnRtPT08+//xzzp49y969e5k4cSLOzre+Q7Vr1w4nJyfGjh3L6dOn2b17\nNwsWLAD+GmkOCgri+vXrDBkyhCNHjnDu3Dl2797N+PHjSUhIeOjYTCYTPj7VuHgxilWrPn+gY3Pk\nyEF8/PVU2+Lj43F3d3/oeEREREQygpLrLMRkMjFr1ix+/fVX2rVrx+TJkxkyZIhtpDhHjhwsWLCA\n06dP06lTJ2bMmMGgQYMwDMNWpmDBgnzyySc4OTnRp08fWz2urq4PPOKcnpSUFM6fP/dAx5QoUZKU\nlBTOnfvruBMnfuWZZ8o8cjwiIiIi9qRpIVlMrVq12LBhQ6ptBw8etL2uWrUqa9eutb3fsmULTk5O\nlChRwratRIkSzJ49+5FjuXLlCjt2fE+DBg1wc8vGnj0/snnzJt5++x3g1nJ8VqvV9vrv01X+Lnv2\n7AQEBDJ//lzGjZvAr7+Gs2PHdyxZsuKRYxQRERGxJyXXT5n169dTrFgxvLy8OHHiBFOmTCEgIIA8\nefJkSHurVn3G229Pxmo18PLyYsSIUTRs2AiATp3aERUVBUBo6K11tDds+AYvLy8WL/6Qw4cP8f77\ncwEYPXosb7wxnqZNG5Mnjyevvfa6luETERGRJ47JuJ8lJCTL+PDDD/n444+5dOkS+fPnp0mTJgwf\nPpwcOXJkSHsJCUn3tUqJPZnNTly7djPVNneSCJ/+7mONw5G5uJixWFLuXVCeSOo/x6W+c2yVx4zg\narLGLR2Rk5OJfPly2qUufQKeMr1796Z3796ZHYaIiIhIlqQbGkVERERE7ETTQiRDPTHTQswpkKKf\nWu+Xs7MTycnWzA5DHpL6z3Gp7xybS3Y3/ryhtMoRaVqIyAOKTzED9/dQHIECnh5cjbmW2WHIQ1L/\nOS71nWMrkDMn3FD/Pe00LURERERExE40ci0ZysnJBJjuWc6eUlL0k6qIiIhkDiXXkqHi4xOxWjX/\nTERERJ4OmhYiIiIiImInSq5FREREROxEybWIiIiIiJ0ouRYRERERsRMl1yIiIiIidqLkWkRERETE\nTpRci4iIiIjYiZJrERERERE7UXItIiIiImInSq5FREREROxEybWIiIiIiJ0ouRYRERERsRMl1yIi\nIiIidqLkWkRERETETpRci4iIiIjYiZJrERERERE7cc7sAEREMoK7OQVSUjI7jEyRFBuLO9bMDkMe\ngvrOsVmuX8/sEOQJoORaRLKmlBTCp7+b2VFkChcXMxbL0/nFwtGp7xxb5TEjUGolmhYiIiIiImIn\n+nolGcrd3S2zQ3goKSlWEhKSMjsMERERcTBKrrMIb29v3n//fZo3b57ZoaRitRoYhpHZYTwws1k/\n6oiIiMiDUwYhIiIiImInSq5FREREROxEybWDWbx4MS1atKBKlSo0btyY9957L91yM2fOpGXLlvj4\n+BAQEMD06dNJSvprDnFUVBSvvPIKtWrVolq1arRu3ZqNGzfa9oeFhREQEECVKlWoX78+o0ePzvBz\nExEREXF0mnPtQGbOnMlnn33GmDFjqFGjBleuXOGXX35Jt2yOHDmYOnUqBQsW5NSpU0ycOBE3NzcG\nDRoEwMSJE7FYLKxYsQJ3d3d+++0327GbN29myZIlvPfee5QrV47Lly9z+PDhx3KOj9OGDV+wbt1a\nFi9eltmhiIiISBah5NpBJCQksGzZMsaNG0enTp0AKF68OD4+PumWDwkJsb0uUqQIffv2ZcmSJbbk\n+sKFC7Ro0YJy5coBULRoUVv5yMhIChYsSL169TCbzRQuXJhKlSo9VNyHDh1k9uz3OH36NM7OZkqV\nKs2IEaOoWLHiQ9VnbyaTKbNDEBERkSxEybWDOHXqFBaLhdq1a99X+U2bNrF8+XLOnj1LfHw8VqsV\nq/Wvp3717NmTiRMnsmvXLmrXrk2zZs1sCXTLli1Zvnw5AQEB1K9fnwYNGhAQEICrq2u6bcXFxREX\nF4enpycp//9EPJPJhMlkYvDgUMaOHU+zZs2xWCwcOnQQV1eXR7waIiIiIvYXGRlpy2Vuy5UrF7ly\n5brvOjTnOgs6fPgww4cPp2HDhsyfP58vvviCIUOGkJycbCvz3HPPsXXrVrp06cIff/xB9+7dCQsL\nA6Bw4cJs2rSJSZMm4eHhwbRp0+jSpQs3b95Mt71ly5YRGBjI5s2bWbZsGcuWLWPt2rX8/vvvmEwm\nmjdvgclkwtXVlVq1alO27LMArF+/ji5dOtKkSQNCQ0OIjIy01Xn69CleeaUfTZo0oHnzAJYs+QgA\ni8XC9Onv0KJFU1q2bMqMGdOwWCwAHDiwn1atmvGf/yynadPGtGjRlC+//MJW59WrVxkyZCANG9al\nZ88gzp8/b98LLyIiIg4tKCiIwMDAVH+WLXuw6aMauXYQZcqUwcXFhR9//JESJUrcteyhQ4coVKgQ\n/fv3t22LiIhIU65QoUJ07dqVrl27smjRIlasWEFoaCgArq6uNGrUiEaNGtGnTx/q1avHwYMHqVu3\nbpp6goOD6dSpU7oj105OZiZMGEfz5i2pWrUqHh63vvlt376NpUsXM3v2HIoXL8GSJR/x2mujWLJk\nOQkJCYSE9CM4uBezZ4eRnGzhzJkzAHz44UJ+/vkYn322GoChQwfx4YcLCQkZAMDly5eJj49n8+at\n7Nmzm5Ejh9OkSQAeHh5MmfIW2bJl57//3c758+cYMKA/RYsWe9CuEBERkSxq5cqV6Y5cPwgl1w7C\n3d2dnj178u677+Li4kLNmjWJjY3l559/5vnnn09VtlSpUkRHR7NhwwaqVavGrl27+Prrr1OVeeut\nt2jYsCGlSpXi+vXr7Nq1i2efvTWivG7dOpKTk/Hx8SFHjhxs3LgRFxcXSpYsmW5sd/u5ZMmSZSxZ\nspi33prEpUuXqF+/AePGjWft2tW8+OLLlCxZCoAXX3yZjz5aRFRUFD/9dIj8+fMTFNQDABcXFypV\nqgzAN99sZPTo18iTJw8Affv25+23J9uSa2dnZ/r06YeTkxP16jUgR44c/PHH71SsWIlt27awevU6\n3NzcKFOmLG3btufQoYMP0RsiIiKSFXl5eT1yHUquHciIESPInTs38+bNY8KECeTPn5+OHTsCqW/M\na9KkCS+//DJTpkzh5s2b1K9fn8GDB/PGG2/YyhiGwZtvvklUVBTu7u7UqVOHUaNGAeDh4cGHH37I\n9OnTsVgslC1blrCwsFQ3Pd6vUqVKM3HiJAD++ON3xo17jRkzphEZGcmMGe/w3nszbPGYTCaioy8S\nFRVF8eLF063v0qUYChf+64Pv5eVFTEyM7X2ePHlwcvprtlO2bNlISEggNjYWq9VKwYKF/nZsESXX\nIiIiYldKrh1Mnz596NOnT5rtx48fT/V+6NChDB06NNW27t27216PGzfujm00bdqUpk2bPmKkaZUs\nWYq2bduzZs0qChcuTO/efWjZsnWacpGRF9i8+Zt06yhQoCCRkRd45pln/r9sJAUKFLhn256enjg5\nOXHxYpRttDwqKvLuB4mIiIg8IN3QKBnmzJkzrFixjOjoi8CtB9ds3vwNVav68Nxz/2Lx4g85c+Y0\nANeuXWPLlm8BaNCgEZcvX+GTT1ZisVhISEjg2LGjALRo0ZKPPlpEbGwssbGxLFq0gNat294zFicn\nJwIDm7JgwTxu3rzJmTOn+eqrLzPozEVERORppZFryTDu7u4cO3aU//xnOdevX8fDw4OGDRsxePAw\ncuTIQUJCAqNHv0pUVBQ5c+akdu3aNG3anBw5cjBv3gKmTZvKggXzcHV144UXgqhcuQq9e/clPj6e\nbt2ew2Qy0axZc3r37nvHGP4+XebVV8cwceLrNG8eSKlSpWjfviP79+97HJdCREREnhImwzCMzA5C\nsq6EhCQc8SNmNjtx7Vr6Sw8+DQoU8CAm5lpmh/FI3EkifPq7mR1GpnBxMWOxpNy7oDxx1HeOrfKY\nEVxN1rilI3JyMpEvX0771GWXWkRERERERMm1iIiIiIi96LcLEcmazGa8Rw7L7CgyhbOzE8nJ1swO\nQx6C+s6xmVxcINnxpkKKfSm5FpEsKT7FDJgzO4xMUcDTg6sOPmf+aaW+c2wFcuaEG+q/p52mhYiI\niIiI2IlGriVDOTmZANM9yz1pUlL0s6yIiIg8OCXXkqHi4xOxWjX/TERERJ4OmhYiIiIiImInSq5F\nREREROxEybWIiIiIiJ0ouRYRERERsRMl1yIiIiIidqLkWkRERETETpRci4iIiIjYiZJrERERERE7\nUXItIiIiImInSq5FREREROxEybWIiIiIiJ0ouRYRERERsRMl1yIiIiIidqLkWkRERETETpRci4iI\niIjYiZJrERERERE7cc7sAEREJH3u5hRISXng45JiY3HHmgERSUZT3zk2y/XrmR2CPAGUXIuIPKlS\nUgif/u4DH+biYsZiefCkXDKf+s6xVR4zAqVWomkhIiIiIiJ2oq9XkqHc3d0yO4QMl5JiJSEhKbPD\nEBERkSeAkusson///nh6ejJlypRHrmvMmDHExsYyf/78R67LajUwDOOR63mSmc36AUhERERuUVYg\nIiIiImInSq5FREREROxEybUDunnzJqNHj8bX15f69euzYMGCVPstFgvTp0+nUaNG+Pr60rVrV77/\n/vtUZc6cOUNISAg1atTA19eX7t27c/LkyXTbCw8Pp379+syaNSvDzklEREQkK1By7YCmTp3Kjz/+\nyNy5c1m6dCm//PIL+/bts+0fPXo0Bw4c4N1332XDhg107NiRkJAQfv31VwCio6N54YUXMJvNLF26\nlPXr1xMUFERKOuvp7t+/n+DgYPr27cuQIUMe2zkCTJjwOvPmzX2sbYqIiIg8Ct3Q6GASEhJYs2YN\nU6dOpW7dugBMmTKFRo0aAXDu3Dk2btzI9u3bKVy4MABBQUHs3r2bzz77jPHjx7Ny5Upy5MjB7Nmz\nMZvNAJQsWTJNW9999x3Dhw9nwoQJtG/f/o4xxcXFERcXh6enpy1BN5lMeHh42PXcRURERDJSZGRk\nmsHGXLlykStXrvuuQ8m1gzl79izJycn4+PjYtuXIkYNy5coB8PPPP2MYBq1bt061SofFYqFOnToA\nHD9+nOrVq9sS6/QcO3aM0NBQZs6cSYsWLe4a07JlywgLC2PKlClEREQAtz6IwcHBD32eIiIiIo9b\nUFCQLZe5LTQ0lIEDB953HUqusxir1YqTkxNr1qzB2Tl197q53Vpz+n6WxitevDj58+dnzZo1NGnS\nBFdX1zuWDQ4OplOnTmlGrgH8/KryxRdfU6xYMeDWVI/ChQsTEjKAAwf2M27cGIKCerB06WLMZmcG\nDBhI+/Yd0rQRHx/P0KGDePbZcowcOYoJE14ne/bsXLhwgUOHDvDMM2V4++2pFC16q52ffjrMjBnT\nOHv2D0qWLMnw4aPw8fFh//59TJs2lc8/XwNA//59SEiIZ/nyjwF46aVggoN70ahRE9q2bUm3bs/z\n1VcbiIqKpG7dekya9BYuLi73vH4iIiLieFauXJnuyPWD0JxrB1OiRAnMZjM//fSTbVtCQoLtZsSK\nFStitVqJiYmhePHiqf4ULFjQVubAgQMkJyffsZ3cuXOzdOlSLl68SGhoKBaL5Y5lc+XKRbFixXB3\nd7f9dHJ7SsjtJPtOLl++THx8PJs3b2X8+AlMnfoW165dS1Xm6tWrhIT0wdfXj5EjR9m2f/vtJvr3\nf4UdO36gWLHizJ07B7g1TWXw4FBeeCGI7dt3ERTUg8GDBxAXF0fVqj6cP3+Oq1evkpKSwpkzp4mO\njubGjQQSExMJDz+On191Wxv//e+3fPDBAr766htOnDjBl19+cdfzEREREcfl5eVFsWLFUv1Rcp3F\n5ciRg+eee44ZM2awe/duTp48ydixY7FarQCUKlWKdu3aMXr0aDZv3sy5c+c4duwYixcvZsuWLQC8\n8MILJCQkMHjwYI4ePcrZs2f5+uuvCQ8PT9VWnjx5WLp0KVFRUYSGhpKU9OBPIbzXKLmzszN9+vTD\nbDZTr14DcuTIwR9//G7bHx0dTZ8+L9K8eUtCQgakOrZJk0AqVqyIk5MTrVu3tt2wuWvXTkqUKEmr\nVm1wcnKiRYtWlCpVmp07v8PV1ZWKFStx8OABfvnlZ8qWLUe1an4cPnyYo0ePUKJESTw8/vpL9MIL\nQeTLlw8Pj1w0bNiIEydSXyMRERGRv9O0EAc0atQobt68SWhoKNmzZ+ff//43N27csO2fOnUq8+bN\nY8aMGURFRZE7d26qVq1K7dq1AShUqBArV65k2rRpBAcHYzKZKFeuHJMnT07TlqenJ8uWLaNXr14M\nGjSIOXPm2HVaRJ48eXBy+us7XrZs2UhISLC9//77neTI4U6XLs+lOTZ//vx/Oy47N27cOi4mJhov\nryKpynp5eREdHQ2An1919u/fS8GChahRowa5cuVi//59uLq6Ur169VTH5c2bL1Vsly7FPMLZioiI\nSFan5NoBZc+enalTpzJ16tR095vNZkJDQwkNDb1jHWXKlEmzPvZt/3yEuqenJ1988XDTIbJly8bN\nm38l/pcvX7KtYnI/Ond+jri4OEJDXyEsbB7Zs2e/5zEFChRk27YtqbZFRUVRr159AKpXr8G7787A\ny8uLF198GQ8PDyZPfgNXV1f+9a/u9x2biIiIyD9pWohkKG/vCnzzzUasVis//PA9Bw8eeOA6Ro0a\nQ6lSpRg8OJTExMR7lq9fvwFnz55l8+ZvSElJYfPmTfz22xkaNLi1XGHVqj788cfv/PzzMSpVqswz\nz5QhMvICx44dTTXfWkRERORBKbmWDDVixKvs3PkdjRvXZ/Pmb2jSJOCu5e90A+S4cRMoXLgww4YN\nvuvNlXDrZszZs8NYvnwZAQENWbFiGbNnzyV37tzArZH/ChUqUqZMWduKKlWr+lCkSBE8PT3vGYuI\niIjInZiM+1mXTeQhJSQk3dfSf47MbHbi2rWbmR2GXRUo4EFMzLV7F5QM5U4S4dPffeDjXFzMWCxp\nn7gqTz71nWOrPGYEV5M149YROTmZyJcvp33qskstIiIiIiKi5FpERERExF7024WIyJPKbMZ75LAH\nPszZ2YnkZGsGBCQZTX3n2EwuLpCctadCyr0puRYReULFp5gB8wMfV8DTg6uaM++Q1HeOrUDOnHBD\n/fe007QQERERERE70ci1ZCgnJxOQtZe0S0nRT7giIiJyi5JryVDx8YlYrZp/JiIiIk8HTQsRERER\nEbETJdciIiIiInai5FpERERExE6UXIuIiIiI2ImSaxERERERO1FyLSIiIiJiJ0quRURERETsRMm1\niIiIiIidKLkWEREREbETJdciIiIiInai5FpERERExE6UXIuIiIiI2ImSaxERERERO1FyLSIiIiJi\nJ0quRURERETsRMm1iIiIiIidOGd2ACIiIvJ4uZtTICUls8PIcizXr2d2CPIEUHItIiLytElJIXz6\nu5kdRZZTecwIlFqJpoWIiIiIiNiJvl5JhnJ3d8vsELKUlBQrCQlJmR2GiIiI3IGS638ICwtj8+bN\nbNiwwe517927l549e7Jnzx7y5Mlj9/qfRFargWEYmR1GlmE268cmERGRJ5n+T/2YmUymhzouIiIC\nb29vfv75ZztHJCIiIiL2ouT6MbFYLI90vGEYD52Yi4iIiMjj4fDJ9aJFi2jWrBk+Pj60b9+eL7/8\nEvhrpHfjxo306NEDHx8fOnXqxK+//srJkyfp3r07vr6+vPDCC0RERKSpd9WqVTRp0gQfHx8GDBhA\nbGysbd/Ro0d5+eWXqV27NtWrV+eFF17g8OHDqY739vZm5cqVDBw4EF9fX9577700bSQlJTFgwAA6\nd+7MlStX7nqeTZs2BaBLly54e3vTs2dP4FbSPXfuXBo3bkyVKlVo164dW7dutR03dOhQJk6caHv/\n3nvv4e3tzZEjR2zbGjVqxFdffQXAmDFj6N+/P8uXL6dhw4b4+/szZswYEhMT7xpfZuvb92XWr1+X\n2WGIiIjIU86hk+v33nuPtWvXMnHiRDZu3Ei/fv2YMGECO3bssJWZM2cOffv2Zf369Xh4eDBixAje\nfPNNhg8fzurVq0lMTOTNN99MVe/58+fZsGED8+bNY+nSpfzxxx+MHTvWtj8+Pp4O/9fenUfXdP3/\nH3/eJEKEmD4SIaGklSg1CxJD0ZiHqrElYi41VSdFSGgNRdEKaqihhn5ozeOnpYOhVNVYDTVPSUWD\nmkE+WMsAACAASURBVELk3vP7I9/cnyuRBDcivB5rZS3nnH32ee+zb1ffZ9999m3Rgq+//ppvv/2W\nF198kTfffJMrV67Y1DNt2jRq167NmjVr6NChg82x69ev061bN65du8bChQvJnz9/qm395ptvMAyD\nOXPmsH37diIiIgCYP38+c+fO5YMPPmDt2rUEBQXRr18/Dh8+DIC/vz+7du2y1rNr1y7y58/Pr7/+\nCsCpU6eIiYmhatWq1jK7d+/m2LFjzJs3j8mTJ7Np0ybmz5+fZn+kZO/ePXTp0olatQKpW7cWXbuG\n8Oeffz5UXUlmzJjOsGFDHqmOjPL777tp1Cgos8MQERGRTJJlk+u4uDjmzZvHxx9/TGBgIEWKFKFJ\nkya0adOGxYsXW8t17dqVmjVrUrx4cbp27crRo0cJDg6mSpUq+Pj40LFjR2uimSQ+Pp7x48fj5+dH\nhQoVGDFiBD/88ANnzpwBoFq1ajRv3pzixYtTvHhxhg4dSrZs2di6datNPY0bN6Z169Z4eXlRpEgR\n6/7Y2Fg6deqEm5sbs2fPJmfOnGm2Nyn5zpMnDwUKFMDNzQ2AOXPm0K1bNxo3bkyxYsXo378/lSpV\nYs6cOUBicn3y5En++ecfbt26xR9//EGXLl2sbd61axdFixalYMGC1mvlzp2b8PBwSpQoQUBAAA0b\nNmTnzp33je3q1aucO3eOGzducPXqVa5evcq1a9e4fv06Awb05fXXO/Dzz9vYuHETb77ZG2fnbGm2\nN6vS9B0REZGsKzo6mnPnztn8Xb169YHqyLKrhRw7dozbt2/TvXt3m/1msxkvLy/rdsmSJa3/LlCg\nACaTKdm+uLg4bt++TfbsicvGeXh44OHhYS1Trlw5HBwcOH78OEWLFuXSpUtMnjyZX3/9ldjYWMxm\nM/Hx8URHR9vEUrp06WRxG4ZBt27dKF26NFOmTMHB4eGfb65fv05MTAwVKlSw2V+pUiW2bNkCgI+P\nDwUKFGDXrl3kzZuXYsWK0aRJE6ZPn47ZbGbXrl02o9ZJ59wdl7u7u800knvNnz+fiIgIxowZY51i\n4+bmRqVKlTCZTNSv3wAAZ2dnqlatZr0PX345ixUrlhMff5uAgEA++GAwrq6u/P77bkJDB7Nhw/fW\nazRt2pDhw0eQkJDAnDmzAfjxxx/w9i7K118vBSA6OoquXUM4evQvypYtx+jRn5AnTx4ABg16j717\n93D79m1KlvRl8OChlCjhA0BY2DBy5MhBVNR59u7dQ8mSvowfP5G5c79k7drVFCjwH8aM+YSSJX2t\nsbRq1YZ169YSG/sPL79clyFDQklISKB//z7cuXOHGjWqYTKZWLFiDXny5GHy5Ils2vQ9JhO88kp9\nBgwYSLZs2axt7dAhmHnz5uDo6ESfPv1o3rzFg3wURERExA46dOiQbLpw37596devX7rryLLJddLy\nbjNmzMDT09PmmJOTExaLxfrvJEkjiintSyqfHh988AGXLl1i6NChFClSBGdnZ0JCQoiPt11/+H4j\n0nXq1GHDhg0cOXKEUqVKpfu695PSSOnd+6pUqcLOnTvJly8fVatWpXDhwuTLl48DBw7w22+/8f77\n79uce/f9SaortfsTEhJCy5YtyZcvH+b/+zldk8mEyWTCwcGRsLBQ6tdvSNmyZcmdO3HEfdWqlaxd\nu4ZZs+aQL18+hg0bwtixo/joo9H3bRNAQEAgXbt259y5s9aySTZu3EBExHQ8PDzo27c3X301j379\nBgAQGFiT8PCPcHJy4vPPJzN06GBrUg6wadN3TJs2gxIlfOjbtzedO3ekd+++vPvu+0yfPpUJE8Yx\nc+aX1vIbNqxn+vQZ5MiRgwED+jF79kx69+7DlCnTGDZsCOvXf2ctO336VA4d+oMlS74FYODA/tby\nkPhNxo0bN/jf/zazc+cvvP/+u9SpU5fcuXPf956LiIiI/S1atMiayyRJmi2QXll2WoiPjw/Ozs6c\nP38eb29vm797k+0HdeHCBS5cuGDd3r9/P4Zh8PzzzwOwZ88egoODqVWrFj4+Pri4uBATE5Ouuk0m\nEwMGDKBdu3Z07tzZOjc6LdmyJU6luLvDc+XKhbu7O7///rtN2d9//90aKyRODfn111/57bff8Pf3\nBxIT7qVLlxITE2Pd97Dc3Nzw8vLC1dUVNzc33NzcyJ07N7ly5WLu3PmYTA6MGjWSevVe5p13BnDp\nUiwbN66nY8dgChcujIuLC/36DeC77/73QA8592revAXe3t44OzsTFFSfv/46YnPMxcWFbNmy0bPn\nm/z11xFu3LhhPV6nTj18ff3Ili0bderUI3v2HDRu3MQ68n53XQDt279OwYLu5M7tRrdu3dm4ccN9\n49qwYT09e/Yib9685M2bl549e7F+/VrrcScnJ3r0eBNHR0cCA2uSM2dOTp8+9dD3QURERB6Op6cn\nXl5eNn8Pmlxn2ZFrV1dXunbtyieffILFYqFKlSrcvHmTffv24ejoSEBAQIrnpecHTZydnRk0aBAf\nfvghcXFxhIeH8/LLL+Pt7Q3Ac889x+rVqylbtiw3btxgwoQJODs7pyvupOsPHDgQgC5dujB37lz8\n/PxSPa9AgQLkyJGDbdu2UaRIEbJnz06uXLno1q0bU6ZMoVixYpQuXZpVq1axZ88eQkNDref6+/sz\nYsQIzp8/b02k/f39GTZsGEWLFsXd3T1dsT+M554rTnj4SABOnz5FaOgQJkwYxz///GPzEOTpWZiE\nhARiY2Mf+loFCvzH+u8cOXJw8+ZNIPFbiYiIz9m06XuuXLmCyZT4kHPlymVcXV3/79wCd52b3eYF\n0+zZ/39dSe6eNuTpWZiLF+//cPXPPxcpVOjutnpy8eJF63bevHltpuHcHbuIiIhkLVk2uQZ4++23\nKViwIHPnzmXEiBHkypWLUqVKWedhpzVd4n68vLxo0qQJvXr14sqVK9SoUYOPPvrIenzMmDEMHz6c\nVq1a4e7uTt++fW2W6kvtOnfvHzhwIIZh0KVLF+bNm4evr+99Y3J0dCQ0NJRp06YxdepUKlWqxFdf\nfUWnTp24efMmEyZM4J9//qF48eJMmTLFpi4fHx8KFixIvnz5yJcvHwBVq1bFYrEkm2+dkYoVe46m\nTZuzbNk3FCxY0GaOenR0FE5OThQoUICLF2O4deuW9ZjZbLa5vw/6wuD69evYsuVnZsyYjaenJ9eu\nXePll2vwKD8c+fff//+bjejoKAoWvP8DSsGC7kRHR1GiRIn/Kx9t8wKpiIiIPD2ydHINiRPP713m\nLklkZKTNdpkyZZLtq1mzps2+vn370rdvXwDatGmTYr2+vr4sWbLEZl/z5s1TvTYkjhbfu/+dd97h\nnXfeSfE692rdujWtW7e22Wcymejduze9e/dO9dx7VzIpUqRIijGOGTMm2b6778mDOHHiBN9/v5n6\n9Rvg7u7B33//zf/+t4GyZctRpsxLzJ8/h4CAQPLmzcfUqVNo0KAhDg4OFC1ajNu3b7N9+1aqVq3O\nl1/OsvkRnvz5C/DrrzvTvTJHXNxNnJ2z4ebmRlzcTSIiPnvkFT2WLv0vNWvWJHv2HMyZ8yUNGjQE\nEkfAr1y5wvXr18mVKxcADRo05MsvZ/Hii4kvuM6aNYPGjZs+0vVFRETkyZTlk2t5crm6uvLHHwdZ\nuPArrl+/Tu7cualVqzYDBryDi4sL//xzke7duxAfH09AQCDvv/8hkDiXfPDgoYwYEY5hWAgJ6WIz\nDSMoqD7r16+lTp2aFCnixaJF/001jqZNm7Fjxy80bPgKefLkoXfvvixb9u0jta1Ro8a89VYv/vnn\nIi+/XJdu3XoAidNgGjZsRPPmjbFYLHz77Uq6d+/JjRs3aNeuNSaTiaCg+nTv3vO+dWspPxERkazL\nZKRnErJkuBkzZvDFF1+keKxKlSrMnDnzMUdkHzdvxqdrnntWkrQsoL//45tSk8TR0YFr126lXfAR\nFSyYm4sXr2X4dSRjqP+yrsfVd67Ec3j8xAy/zrOmzOD3+DdB45ZZkYODiQIFctmlLn0CnhCvv/46\njRs3TvFY0vrbIiIiIvJkU3L9hEhawk6efJq2ISIiIvej5FrkAa1Zc/81rUVEROTZpuRaRETkWePo\niN/76VupStLPlC0bJDxd7xnJg1NyLSIi8oy5YXYEHDM7jKdOwVy5IE4vEz/rsuzPn4uIiIiIPGk0\nci0ZysHBBOgFQHsxmy2ZHYKIiIikQsm1ZKgbN25jsWj+mYiIiDwbNC1ERERERMROlFyLiIiIiNiJ\nkmsRERERETtRci0iIiIiYidKrkVERERE7ETJtYiIiIiInSi5FhERERGxEyXXIiIiIiJ2ouRaRERE\nRMROlFyLiIiIiNiJkmsRERERETtRci0iIiIiYidKrkVERERE7ETJtYiIiIiInSi5FhERERGxEyXX\nIiIiIiJ24pTZAYiIiIg8DW7E38TkkpDZYWQJDjhijjNldhgZQsm1iIiIiB3Em+8wafvszA4jSxgY\n2J2nNQ19OlslTwxX1+yZHUIyZrOFmzfjMzsMEREReQopuZYMZbEYGIaR2WHYcHTUqwYiIiKSMZRl\nPCUMw2D48OFUrVqVUqVK8dtvv2V2SCIiIiLPHI1cPyV+/vlnVqxYwcKFC/Hy8iJPnjyZHZKIiIjI\nM0fJ9VPi1KlTFCxYkHLlyj10HQkJCTg56SMhIiIi8rA0LeQpMHjwYMaOHUt0dDR+fn7Uq1ePrVu3\n0qFDB/z9/alatSrdunXj+PHj1nPOnz+Pn58f69atIyQkhPLly7NkyRIA9uzZQ3BwMOXLl6dWrVqE\nh4dz/fr1zGqeiIiISJZhMp60t83kgV2/fp25c+eyfPlyli1bhslkYvfu3QD4+fkRFxfH9OnTOXTo\nEOvXr8fJyYnz589Tr149ihQpwqBBgyhdujROTk78+++/tGvXjgEDBlCvXj0uX77M6NGj8fDw4LPP\nPnuguOLj4xk2LIxff93JtWtX8fLypk+ffgQG1gDg1q1bTJo0ge+//x6zOYGSJX2ZNWtOsnru3LnD\nmDEf8+uvv6ZYT1RUFM2aNSJnzpwYhoHJZCIkpAvdu/dMMS5HRweuXbv1QG151hQsmJuLF69ldhjy\nkNR/WZf6LmtzymVhwpaZmR1GljAwsDtG3JPzbbmDg4kCBXLZpa4np1Xy0HLlyoWrqysODg7kz58f\ngKCgIJsyo0aNonLlyhw4cICKFSta9wcHB1O/fn3r9sSJE2nSpAmdO3cGwNvbm7CwMFq2bMmlS5es\n9d/t6tWrXL16lXz58mE2mwEwmUw4OTlRqFAhvvxyHoUKFWLr1i18+OH7LF26HE9PTz76aASGYWHF\nitW4ublx5MjhFNuXkJBAoUKe960n6XpbtvyCyfR0LkgvIiIiGS86OtqayyRxc3PDzc0t3XUouX5K\nnT17lsmTJ3PgwAEuXbqExWLBMAyio6NtypUpU8Zm+9ChQ5w5c4Z169bZ7DeZTJw9ezbF5Hr+/PlE\nREQwZswYzp8/DyR+EENCQnjzzd7Wpfhq1qxF4cJFiIz8k/j422zduoWNG78nZ86cAPj5lUqxLS4u\nLvTs2cu6fXc9Scm1YRhYLBYcHR0f5DaJiIiIWHXo0MGayyTp27cv/fr1S3cdSq6fUm+++Saenp6M\nHDkSDw8PnJycaNy4MXfu3LEp5+LiYrNtsVho06aNdeT6bh4eHileKyQkhJYtWyYbub5XbGwsZ8+e\nwcfHh4MHD1KoUCGmT5/KunVrKViwID179qJevVfSbFtsbCxnzpzGx8fHus9kMtG0aUNMJhP+/tV4\n++13yJs3b5p1iYiIiCRZtGhRiiPXD0LJ9VPoypUrnDhxgvDwcPz9/YHEEemEhIQ0z33xxRc5evQo\n3t7e6b5eer4uSUhIIDR0MM2aNadYsefYvHkTx48fIyioPt99t5n9+/cxYEBffHx8eO654mnW07x5\nC4oVew6AfPnysmDBYnx9/fj33yuMGTOKoUM/ZOrUL9LdBhEREZGkb8QfhVYLeQrlyZOHfPnysXTp\nUs6cOcOuXbsIDw9P1zJ7PXr04ODBg4SFhREZGcmZM2f48ccfGT58+EPHYxgGoaFDyJbNmQ8+GAxA\n9uzZyZYtG92798TJyYlKlSpTuXIVdu7c8UD1ALi45KRUqRdxcHAgX778DBo0hJ07d3Dz5s2HjllE\nRETkYSi5fgqZTCYmT57MkSNHaNasGR999BFvv/02zs7Oycrdy9fXl4ULFxIVFUVwcDAtWrRg0qRJ\nFCxY8KHjGTEijCtXLvPpp5Osc6JfeKEkwAP9NHpK9dyPyWR64n52XURERJ5+WopPMtSQIaEcPfoX\n06fPtJnfnZCQQOvWLWnatBldunTj4MED9O/fhwULFlune9xt1KiPUqwH4I8/DpI7d26KFi3Gv//+\ny9ixo7ly5TJffDErxZi0FF/atBxY1qb+y7rUd1mbluJLPy3FJ/IQoqKiWL78W7Jnz05QUB0gcUR5\n6NBhNGzYmIkTP2PkyDDmzZvzf0vzjbIm1nPmzGbfvr18/vlUoqOjU63n/PlzRER8zuXLl3F1zUW1\natUYPXpsZjVbREREnmEauZYMdfNm/BM3PUMj12nT6FnWpv7LutR3WZtGrtPvaR651pxrERERERE7\nUXItIiIiImInSq5FREREROzkyZnsIiIiIpKFOTtmY2Bg98wOI0twwBFz2sWyJCXXIiIiInbg6pyT\nm/8+rSmjfT3Nd0nJtWQoBwcTkPzHajKT2WzJ7BBERETkKaXkWjLUjRu3sVierKX4RERERDKKXmgU\nEREREbETJdciIiIiInai5FpERERExE6UXIuIiIiI2ImSaxERERERO1FyLSIiIiJiJ0quRURERETs\nRMm1iIiIiIidKLkWEREREbETJdciIiIiInai5FpERERExE6UXIuIiIiI2ImSaxERERERO1FyLSIi\nIiJiJ0quRURERETsRMm1iIiIiIidKLkWEREREbETp8wOQERERORpcCP+JiaXhEeuxwFHzHEmO0Qk\nmUHJtYiIiIgdxJvvMGn77EeuZ2Bgd5SiZV3qOclQrq7Z01XObLZw82Z8BkcjIiIikrGUXEuGslgM\nDMNIs5yjo6b/i4iISNaXaRnNihUrqFixonU7IiKCZs2apXrOvWXSc46IiIiIyOOSaSPXTZo0oXbt\n2o9UR7du3QgODrZTRCIiIiIijybTkmtnZ2fy58//SHW4uLjg4uJip4hERERERB5NmtNCgoODGTFi\nBJMmTaJatWoEBATwySefpKvy7777jubNm1OuXDmqVq1KcHAwly5dAmD58uVUqFAh2TnffPMNderU\noVy5cvTp04fLly/ft/57p4UMHjyYXr168dVXX1GrVi38/f0ZPHgwt2/ftpaJi4vjgw8+oEKFCtSo\nUYOZM2fSq1cvBg8enK64U/P333/z1ltvUbVqVcqXL0/jxo1Zv349AOfPn8fPz4+1a9fyxhtvULZs\nWRo1asT27dtt6vjtt99o27YtZcuWJTAwkDFjxnDnzh3r8eDgYD7++GObc5LafXcd7dq1o0KFClSu\nXJl27dpx7Ngx6/E9e/YQHBxM+fLlqVWrFuHh4Vy/fj3d54uIiIhIytI153rt2rU4OTmxZMkShg8f\nzldffWVNGu/nn3/+4Z133uG1115jw4YNLFq0iBYtWliPm0wmTCbbNRzPnTvHmjVrmD59OvPmzeP0\n6dMMHTr0gRq0e/dujh07xrx585g8eTKbNm1i/vz51uNjxoxh9+7dTJs2jfnz53P48GF2796d7rhT\nEx4ezu3bt1mwYAHr1q1jyJAhuLm52ZSZMGECISEhrFq1isDAQN566y1iYmIAuHDhAj179qR06dKs\nXLmS0aNHs27dOiZOnJju9pvNZvr06UPlypVZs2YN33zzDZ06dcLBIbGrjxw5Qrdu3ahXrx5r1qwh\nIiKCw4cPM2TIkHSd/6CWLPmajh1fp1q1yoSHD3+oOkRERESyinRNC/Hx8aFfv34AFCtWjKVLl7Jj\nxw4aN25833NiYmIwm800aNAAT09PAJ5//vlUrxMfH8/48ePx8PAAYMSIEXTo0IEzZ85QtGjRdDUo\nd+7chIeH4+DgQIkSJWjYsCE7d+6kZ8+e3Lx5k+XLlzN+/HiqV68OwKhRo2zmfj9M3EmioqJo0KAB\nJUuWBKBIkSLJyrzxxhs0aNAAgKFDh7J161a+/vprBgwYwOLFi3F3dycsLAyAEiVK8O677xIWFsbb\nb79N9uxpL2t3/fp1rl27Rp06dfDy8gKgePHi1uNz5syhSZMmdO7cGQBvb2/CwsJo2bIlly5dwtHR\nMdXzU3L16lWuXr1Kvnz5MJvNQOLDU+7cuXF3d6dHj5788ssvNt8giIiIiDxpoqOjrblMEjc3t2SD\npalJV3Lt6+trs+3u7k5sbGyq5/j5+VG9enWaNGlCjRo1qF69Og0aNEh1nrWHh4c1sQYoV64cDg4O\nHD9+PN3JtY+Pj80oq7u7OwcOHADgzJkzmM1mXnrpJetxFxcXXnjhhUeKO0mnTp0IDw9n69atVKtW\njaCgIEqXLm1Tply5ctZ/m0wmypUrx/HjxwE4ceIE5cuXtylfqVIl7ty5w+nTp61Je2ry5MnDq6++\nSteuXalevTrVq1enYcOGFCpUCIBDhw5x5swZ1q1bZ3OeyWTi7NmzlCtXLtXzUzJ//nwiIiIYM2YM\n58+fBxI/iCEhIdSpUw/DMDh06JB1hF5ERETkSdShQwdrLpOkb9++1kHm9EhXcp0tWzabbZPJhMVi\nSfUcBwcH5syZw/79+9m2bRvffvstEydOZOHChcmSdXtycrJtUkqx3jsd5W6PEnfr1q2pWbMmW7Zs\n4ZdffqF9+/a8+eab9O3bN12xG4aRYmx373dwcEi2bvTdc7IhcepL586d2bp1K5s3b2bSpElMmzaN\nwMBALBYLbdq0sY5c3y3pwSa181MSEhJCy5Ytk41ci4iIiGQlixYtSnHk+kFk+DrXSS8mLlu2DHd3\n91Tnal+4cIELFy5Yt/fv349hGPj4+NgllqJFi+Lo6GgdyYbEFxyPHj36SHHfzcPDgzZt2jBp0iT6\n9+/P0qVLbY7v37/fZvvAgQPW9vn4+LB3716b47t378bZ2dk6cp8/f34uXrxoU+bIkSPJ4vD19aV7\n9+4sWLAAf39/VqxYAcCLL77I0aNH8fb2Tvbn7Oyc5vkpcXNzw8vLC1dXV+tXJ7lz507rVomIiIg8\nUTw9PfHy8rL5e2KS6/379zN9+nQOHjxIdHQ0mzZt4u+//7aZgnEvZ2dnBg0axOHDh9m7dy/h4eG8\n/PLL6Z4SkpacOXPSqlUrxo8fz44dOzh27BihoaE2I8MPE3eSUaNGsXXrVs6ePUtkZCRbt25Ndt7X\nX3/N//73P06ePMnHH39MdHQ07du3BxLnY8fExBAWFsbx48f56aefmDhxIh07drTOt65WrRpbtmzh\nhx9+4OTJk4wdO5bo6Ghr/efOnePTTz9l7969REVFsXPnTo4cOWKNo0ePHhw8eJCwsDAiIyM5c+YM\nP/74I8OHD0/X+SIiIiJyf2lOC3nYr/dz5crFnj17WLRoEVevXsXT05M+ffrQtGnT+57j5eVFkyZN\n6NWrF1euXKFGjRp89NFHD3X9+xk0aBC3bt3irbfewtXVlZCQEGJjY63J68PEncQwDD7++GP+/vtv\nXF1dqV69OoMGDbIp8+677zJ37lwiIyMpXLgwU6dOtU7H8PDwYNasWYwfP56WLVvi5uZGs2bNGDhw\noPX8Vq1a8ddff1lXUXnjjTcICgqyLlno4uLCqVOnePvtt7l8+TL/+c9/aNGiBd27dwcSR6QXLlzI\n5MmTCQ4Oxmw24+3tTVBQULrOFxEREZH7Mxn3TuB9xsTHx1O3bl26d++e4jxkezl//jz16tVj2bJl\nyV5yfJpduxbHnTt3mDnzC2JiLjBsWDiOjo44OjralHN0dODatVuZFKXcq2DB3Fy8eC2zw5CHpP7L\nutR3WZtTLgsTtsx85HoGBnbHiMu03/l7Jjk4mChQIJdd6nrmei4yMpLjx49TtmxZrl+/zqxZs7hx\n4waNGjXK7NCeSrNnz2TGjOnWb0A2bFhPz5696NmzVxpnioiIiGQ9D51c7969mx49emAymZKtXmEy\nmdizZ88jB5dR5s6dy6lTp3BycsLPz4/FixfbLAF4P02bNk22PAsktnfkyJFpTh15FlfQePPN3kqk\nRURE5Jnx0Ml12bJlWb16tT1jeSxKlSrFsmXLHurcWbNmkZCQkOKxAgUKpHpukSJFiIyMfKjrioiI\niEjW8NDJtbOzM97e3vaM5YmX9IuNIiIiIiIpeebmXIuIiIhkBGfHbAwMfPTVtRxwxJx2MXlCKbkW\nERERsQNX55zc/PfR02Il1llbhv9Co4iIiIjIs0Ij15KhHBxMQNqrpJjNlowPRkRERCSDKbmWDHXj\nxm0slmf6d4pERETkGaJpISIiIiIidqLkWkRERETETpRci4iIiIjYiZJrERERERE7UXItIiIiImIn\nSq5FREREROxEybWIiIiIiJ0ouRYRERERsRMl1yIiIiIidqLkWkRERETETpRci4iIiIjYiZJrERER\nERE7UXItIiIiImInSq5FREREROxEybWIiIiIiJ0ouRYRERERsRMl1yIiIiJ2cCP+JiaXBBxdjMwO\nRTKRkmsRERERO4g332HS9tlYMGd2KJKJlFyLiIiIiNiJU2YHIE83V9fs6SpnNlu4eTM+g6MRERER\nyVhKriVDWSwGhpH23DNHR32JIiIiIlmfMhqxUbduXebOnZvZYYiIiIhkSUquRURERETsRMn1MyQh\nISGzQxARERF5qim5fkIFBwcTFhbGqFGj8Pf3x9/fn3HjxlmPr169mtatW1OxYkUCAgIYMGAAFy5c\nsB7ftWsXfn5+/Pzzz7Rp04aXXnqJ7du3A/DTTz/Rtm1bypUrR9WqVenduzfx8f//ZcJbt24xLkRr\nOQAAGk5JREFUfPhwKlWqRO3atfnyyy8fX8NFREREsjAl10+wtWvXYhgGS5YsYeTIkSxdupR58+YB\niaPQ/fv3Z/Xq1cyYMYMrV67w3nvvJavj008/ZeDAgWzYsIGyZcuyZcsW+vTpQ40aNVi+fDkLFizA\n39/f5qXD+fPn4+vry8qVK+nRowfjx49n//79D9WGJUu+pmPH16lWrTLh4cMfqg4RERGRrEKrhTzB\nChYsSGhoKADFixfn5MmTzJs3j86dO/Paa69Zy3l5eTF8+HCaNGnChQsX8PDwsB7r378/AQEB1u3p\n06fTsGFD+vfvb91XsmRJm+sGBgbSoUMHADp27MiCBQvYsWMH5cqVSzHOq1evcvXqVfLly4fZnLhw\nvslkInfu3Li7u9OjR09++eUXbt++/Yh3RERERCTjREdHW3OZJG5ubri5uaW7DiXXT7Dy5csn2/78\n88+5ceMGp06dYurUqRw+fJgrV65gGAYmk4no6Ghrcm0ymShdurRNHZGRkTaJeUp8fX1ttt3d3YmN\njb1v+fnz5xMREcGYMWM4f/48kPhBDAkJoU6dehiGwaFDh4iJiUl320VEREQetw4dOlhzmSR9+/al\nX79+6a5DyXUWZBgG3bt3JzAwkHHjxlGgQAEuXbpEhw4duHPnjk1ZFxeXB64/W7ZsKV7zfkJCQmjZ\nsmWykWsRERGRrGTRokUpjlw/CCXXT7B75znv27cPd3d3Tp8+zeXLlxk4cCBFihQB4OjRo+lKaEuV\nKsXOnTtp06aN3eJ80K9LRERERJ5Enp6ej1yHXmh8gsXExDB69GhOnjzJxo0bmTNnDl26dMHT0xNn\nZ2cWLlzI2bNn+emnn/j888+TnZ/SaHOvXr3YuHEjkydP5vjx4xw9epR58+ZpPrSIiIiIHSi5foI1\na9YMi8VC27ZtCQsLo02bNoSEhJA/f34++eQTNm/eTNOmTZk2bRqDBw9Odn5KI9m1a9cmIiKCrVu3\n0rJlSzp16sSuXbusZVM6R1M8RERERNJH00KeYE5OToSGhlpXDLlbo0aNaNSokc2+yMhI67/9/f1t\ntu9Wp04d6tSpk+KxzZs3J9v31VdfPUjYNsxmM3fu3MFsNmM2JxAfH4+joyOOjo4PXaeIiIjIk0oj\n15KhZs+eSUCAP/Pnz2XDhvUEBPjz5ZezMjssERERkQyhkesn1NMyFePNN3vTs2evzA5DRERE5LFQ\ncv2EepSpGCIiIiKSOTQtRERERETETpRci4iIiNiBs2M2BgZ2xwG9tP8sU3ItIiIiYgeuzjkx4pww\nxz0d703Jw1FyLSIiIiJiJ3qhUTKUg4MJSPsJ3my2ZHwwIiIiIhlMybVkqBs3bmOxJP8ZdhEREZGn\nkaaFiIiIiIjYiZJrERERERE7UXItIiIiImInSq5FREREROxEybWIiIiIiJ1otRDJUIlL8UlWpL7L\n2tR/WZf6LmtT/2VN9uw3k2EYWidN7O769evkypUrs8MQERERSTd75C+aFiIZ4sqVK7z++utER0dn\ndijygKKjo6lbt676LotS/2Vd6rusTf2XtUVHR/P6669z5cqVR65LybVkmD179mA2mzM7DHlAZrOZ\n8+fPq++yKPVf1qW+y9rUf1mb2Wxmz549dqlLybWIiIiIiJ0ouRYRERERsRMl1yIiIiIiduIYHh4e\nntlByNMpe/bsVK1alezZs2d2KPKA1HdZm/ov61LfZW3qv6zNXv2npfhEREREROxE00JEREREROxE\nybWIiIiIiJ0ouRYRERERsRMl1/LQbt26xcCBA6lfvz6NGzfmp59+um/ZpUuXUr9+ferXr8/HH39s\n3W8YBqNGjaJp06Y0b96cHj16cPHixccQ/bPNHn0HEBkZSceOHWnSpAlNmzZl69atGRy5gP36DyA+\nPp7GjRvTunXrDIxY7maP/tu8eTOvvfYazZo1o1mzZsydO/cxRP7sOnXqFO3bt6dhw4a0b9+eM2fO\nJCtjsVgYMWIEQUFBNGjQgG+++SZdxyRjPWrfTZs2jaZNm/Lqq6/SqlUrtm3blvZFDZGHFBERYYSG\nhhqGYRinTp0yAgMDjZs3byYrd/bsWaNWrVrG5cuXDcMwjK5duxorV640DMMwvv/+e6Ndu3aGxWIx\nDMMwxowZY4wYMeIxteDZZY++u3nzplGvXj1j//79hmEYhtlsNq5cufKYWvBss0f/JRk7dqwxdOhQ\no1WrVhkfuBiGYZ/+279/vxETE2MYhmFcu3bNCAoKMnbv3v2YWvDs6dSpk7FmzRrDMAxj1apVRqdO\nnZKVWbFihdGtWzfDMAwjNjbWqFWrlnH+/Pk0j0nGetS+27Ztm3Hr1i3DMAwjMjLSqFy5snH79u1U\nr6mRa3loGzZsoH379gAUK1aMMmXKsGXLlmTl/ve//xEUFETevHkBaNu2LRs2bADAZDIRHx9PXFwc\nFouFGzduUKhQocfXiGeUPfpu7dq1VK5cmbJlywLg4OBAnjx5HlMLnm326D+A3bt3c/r0aVq0aPF4\nAhfAPv1XtmxZChYsCECuXLkoUaIEUVFRj6kFz5ZLly4RGRlJkyZNAGjatCl//vknly9ftim3YcMG\n2rZtC0D+/Pl55ZVX2LhxY5rHJOPYo+8CAwOtS/P5+fkBJDv/Xkqu5aFFRUVRuHBh67anpyfR0dHJ\nykVHR9+3XN26dalSpQqBgYHUrFmTU6dO0bVr14wP/hlnj747duwYjo6O9OzZk5YtWxIaGsrVq1cz\nPnixS//dvHmTMWPGMGLECAytyPpY2aP/7nb8+HEOHDhAtWrVMibgZ1x0dDQeHh6YTCYgcSDB3d2d\nv//+26Zcav2a3j4X+7JH391txYoVeHt74+Hhkep1newQuzylXnvttWQfLsMwMJlMbN++3S7X+OOP\nPzhx4gTbtm0jZ86cjBo1ijFjxjBs2DC71P+sehx9Zzab2blzJ0uXLqVAgQKMHj2asWPHMnr0aLvU\n/yx7HP03btw4OnToQMGCBTlx4oRd6pREj6P/ksTExNCnTx/CwsKsI9kiYn+7du1iypQp6Xq/Qcm1\n3Nfy5ctTPV6kSBGioqLIly8fkPiEmNLIiaenJ+fPn7duR0dH4+npCcDKlSupVq0arq6uADRv3pyh\nQ4faqwnPrMfRd4ULF6ZatWoUKFAASPy6TX1nH4+j//bs2cPWrVuZOnUqt2/f5t9//6VFixasWrXK\nji15Nj2O/gOIjY2la9eu9OjRgwYNGtgpermXp6cnFy5csD4gWSwWYmJikk1hLFy4MFFRUZQpUwZI\n7K8iRYqkeUwyjj36DmDv3r0MGjSI6dOnU6xYsTSvq2kh8tAaNGjAkiVLgMS3cf/44w9q1qyZrFz9\n+vXZvHkzly9fxmKxsHTpUho1agSAl5cXO3bsICEhAYCff/6ZF1544fE14hn1KH3XsGFDABo1asSB\nAwe4ceMGAFu3brXOR5OMZY/+W716NZs3b2bz5s1MnDgRX19fJdaPiT367/Lly3Tt2pWOHTvSqlWr\nxxr/syZ//vz4+fmxZs0aANasWcOLL75ofThK0rBhQ5YuXYphGFy6dInNmzdTv379NI9JxrFH3x04\ncIB33nmHzz77LN3/j9PPn8tDi4uL48MPPyQyMhJHR0c++OAD6tSpA8Dnn3+Oh4cH7dq1AxKXk5o1\naxYmk4kaNWowbNgw68uM4eHh7Nu3DycnJwoXLszIkSNxd3fPzKY99ezRdwCrVq1i9uzZODg44OXl\nxUcffUT+/PkzrV3PCnv1X5Jdu3Yxbtw4vv3228felmeRPfpv3LhxLF68mOLFi1tH5Tp16kTLli0z\ns2lPrRMnTvDhhx9y9epV8uTJw7hx4yhWrBg9e/ZkwIABlC5dGovFwsiRI9m+fTsmk4kePXrQpk0b\ngFSPScZ61L5r3bo1UVFReHh4WP9bGzduXKoDgUquRURERETsRNNCRERERETsRMm1iIiIiIidKLkW\nEREREbETJdciIiIiInai5FpERERExE6UXIuIiIiI2ImSaxERO9m1axelSpXiypUrQOIv9VWoUCHD\nrle3bt10/RTvs2rjxo02P/qwYsUKKlasmCmx9OrVi8GDB2fKtXft2oWfn5/1c/mwgoOD+fjjjx+o\nTFrbIk8jJdcikiXFxsby8ccfExQUxEsvvUTt2rXp2bMnP//8s12vM3jwYHr16pWushUrVmTbtm3k\nzZsXAJPJlOwHWx5GREQEzZo1S7Z/2bJlvPHGG49cf2rOnz+Pn58fhw4dytDrZIR773+TJk3YtGlT\nus9/mh5e7PE5TI+pU6fyzjvvpPv403SPRZI4ZXYAIiIP6vz587Rv357cuXPz3nvv4evri8ViYceO\nHYwYMYIffvjhsceUkJCAk5MTBQoUeGzXvPcnfDNC0i+SZZY7d+6QLVs2u9Tl7Oz8VP2CqNlsxtHR\nMbPDsOHm5vZIx0WeBhq5FpEsJzw8HJPJxPLly2nQoAHPPfccJUqUoEOHDqxatcpaLjo6mj59+lCx\nYkUqVqxIv379uHDhgvV40ojw+vXrCQoKomLFivTp08f69XlERAQrVqzg559/xs/Pj1KlSvHbb79Z\nR3PXrVtHSEgI5cuXZ8mSJff9+v3HH3+kQYMGlC1blk6dOnH27NlkMdzt7ukkK1asICIigmPHjllj\nWLlyJZB81O9R25uSV155BYBWrVrh5+dHp06dgMSke+rUqbz88su89NJLNGvWjM2bN6fab0nfAkyf\nPp3AwEAqVKjA4MGDiY+Pt5YJDg4mPDycTz75hOrVq1tH5q9fv86wYcMICAigYsWKBAcH88cff9jU\nv3LlSurWrUuFChXo1asX//zzj83xFStWJJum89NPP9G2bVvKlStH1apV6d27N/Hx8QQHBxMVFcW4\nceOs9z3Jnj17CA4Opnz58tSqVYvw8HCuX79uPX7r1i0+/PBDKlSoQI0aNZgxY0aq9+Xu2NLzWVmx\nYgVBQUGULVuWuLg44uPjGTVqFIGBgZQtW5Z27drx+++/J7vGvn37ePXVVylbtiyvvfaazbcRV65c\n4d1336V27dqUK1eOpk2bsnz58mR1JCQkMGrUKPz9/fH392fcuHE2x9Oa9nH38ZTucVxcHJUqVeK7\n776zOW/79u2UKVOGS5cupXkvRTKbkmsRyVL+/fdftm3bRseOHcmRI0ey47lz57b++6233uLSpUss\nWLCABQsWEBMTQ58+fWzKnzt3jg0bNjBt2jTmzp1LZGQkkyZNAqBr1640atSIgIAAfvnlF7Zt22aT\nnE2cOJEOHTqwbt06axJ67yhvfHw8U6dO5ZNPPmHp0qVYLBb69euXahvvns7QuHFjunTpQvHixa0x\nNG7cOMXzHrW9Kfnmm28wDIM5c+awfft2IiIiAJg/fz5z587lgw8+YO3atQQFBdGvXz8OHz6catt2\n7drFkSNHmD9/PhEREWzfvp3x48fblFmzZg0Aixcv5pNPPgGgR48eXLx4kZkzZ7Jq1SqqVKlC586d\nrQn0/v37GTx4MO3bt7cm2Z9//nmK9zbJli1b6NOnDzVq1GD58uUsWLAAf39/DMMgIiKCQoUK0adP\nH7Zv3862bdsAOHLkCN26daNevXqsWbOGiIgIDh8+zJAhQ6z1jh07lh07djB16lTmzZvHn3/+yW+/\n/ZbqfYHEUfq0Pivnzp1j7dq1fP7556xatQpnZ2fGjRvHxo0bGTNmDCtXrqRkyZJ0797d5uHCMAzG\njRvHBx98wPLly/H29ubNN9/k9u3bANy+fZvSpUszc+ZM60NjWFgYO3futLn+6tWrMQyDJUuWMHLk\nSJYuXcq8efPSbFtKUrrHLi4uNGnShGXLltmUXb58OXXr1n2qvnmQp5emhYhIlnL69GkMw6BEiRKp\nltu+fTt//fUXmzZtwtPTE4AJEyZQv359duzYQfXq1QGwWCyMHTsWV1dXANq2bcuKFSsAyJkzJzly\n5CAuLi7F/6kHBwdTv359m9juZTabCQ0NpXz58gCMGzeOV155xSaG1GTPnh1XV1ccHR1TTSzs0d6U\nJF0zT548NlNe5syZQ7du3ayJfv/+/fntt9+YM2dOstHMuzk5OTF27Fhy5MjB888/z3vvvUdoaCjv\nvvuu9WHJy8uLQYMGWc/ZsWMHR44cYefOnTg7O1uv98MPP7Bq1Sq6devGV199RUBAAD179gSgWLFi\nHDhwIFmSdrfp06fTsGFD+vfvb91XsmRJIPG+Ozg44OrqmqzdTZo0oXPnzgB4e3sTFhZGy5YtuXTp\nEjly5GDZsmWMHTuWgIAAAMaMGUPt2rXvG0eS9HxW7ty5w/jx4639EhcXx3//+19Gjx5NrVq1ABgx\nYgQ7d+5k0aJFDBgwwFp/nz59ksW0Zs0aWrdujYeHB127drWWbdOmDTt27GDdunVUq1bNut/d3Z3Q\n0FAAihcvzsmTJ5k3b571fjyIPHnypHiP27ZtS/v27YmJicHd3Z2rV6+yadOmFB+WRJ5EGrkWkafS\niRMncHd3tyaakJgIubu7c/z4ceu+woULWxNNSEweYmNj03WNMmXKpFnGwcGBl156yeZ698ZgD4+j\nvUmuX79OTExMsikWlSpV4tixY6me6+vra/ONQ4UKFbhz5w5nzpyx7itdurTNOX/++SdxcXFUrVqV\nChUqWP+OHTtmnTZx4sQJa1Ka5N7te0VGRtokjulx6NAhVq9ebRPHG2+8gclk4uzZs5w5c4aEhATK\nlStnPSdnzpzWpD016fmsFCpUyOYh68yZM5jNZpu+cHBwoHz58jbnmUymFGNKKmOxWJg+fTrNmze3\n3ufvv/+eqKgomxhTuscXLlzgxo0babYvvcqUKcMLL7xgnf60Zs0a8uTJY314EHnSaeRaRLKUYsWK\nYTKZOHHiRKrlUnsR7+79Tk5OyY5ZLJZ0xeLi4pKucqlJKcaEhIQHrudxtDe1elPblxbDMGy2c+bM\nabNtsVj4z3/+w+LFi5Odm/SgcG8dGcVisdCmTZsUR2o9PDzS/Fw+qns/c0ntftSXTmfPns28efMI\nDQ3lhRdewNXVlU8//TTT5ji3bt2ar776ip49e7Js2TJee+21TH2xVuRBaORaRLKUPHnyUKNGDRYu\nXEhcXFyy49euXQPg+eef58KFCzYjb2fPniUmJobnn38+3dfLli3bQyefkJiMHTx40LodFRVFTEwM\nPj4+QOK0i3tfvPvzzz8fOAZ7tfdeSSt1mM1m675cuXLh7u6e7KW533//Pc1r/fXXX9y6dcu6vXfv\nXpydnSlatOh9zyldujSxsbGYTCa8vb1t/pJGcX18fNi3b5/Nefdu36tUqVLJ5hTfLVu2bDbtBnjx\nxRc5evRosji8vb2t7XB0dGT//v3Wc27evMnRo0dTjQXS/qykpFixYjg5Odn0hcViYd++fbzwwgvW\nfYZhpBhTUt179uyhbt26NGvWDD8/P7y9vTl16lSy691dByTeY3d3d5tvQx5ESvcYoEWLFsTExLBo\n0SIiIyN57bXXHqp+kcyg5FpEspywsDAMw6BVq1Zs3LiRkydPcuLECRYvXkyLFi0ACAgIwNfXl/fe\ne49Dhw5x8OBB3n//fcqUKUPVqlXTfa0iRYpw9OhRTp48yeXLl9McVb53BNXR0ZHRo0ezb98+IiMj\nGTRoECVLlrTOofX39+fff//liy++4OzZs3zzzTfJVkooUqQIUVFR/Pnnn1y+fNlmdY0k9mrvvQoU\nKECOHDnYtm0bsbGx1lUxunXrxpw5c1i3bh2nTp3is88+Y8+ePTbzdlOSkJDAkCFDOHbsGNu3b2fi\nxIm0bds2xZdT725bxYoVeeutt9iyZQvnzp1j7969TJkyxZpUdurUiR07djBz5kxOnz7N0qVL01zT\nulevXmzcuJHJkydz/Phxjh49yrx586wv+Xl5ebF7924uXLjA5cuXgcQXKw8ePEhYWBiRkZGcOXOG\nH3/8keHDhwOJo+6tW7dmwoQJ/PLLLxw9epShQ4em6wEtrc9KSlxcXHj99df59NNP+fnnnzl+/Dhh\nYWHExsby+uuv25SdPn26NaYhQ4bg7OxM06ZNgcT50zt27OD333/n+PHjjBw5knPnziW7XkxMDKNH\nj+bkyZNs3LiROXPm0KVLlzTbdj8p3WNIfIBr0KABY8eOpUqVKqk+fIk8aZRci0iW4+XlxYoVKwgI\nCODTTz+lRYsWdO7cmZ9++omRI0day02bNo38+fPTqVMnOnfujLu7u3W1i/Rq06YNJUqUoFWrVgQE\nBLB3717g/l/D37vf2dmZXr16MWjQINq1a4fJZGLKlCnW4z4+PoSHh7N06VKaN2/Ozp07k/1oTf36\n9alVqxadO3cmICCA9evXp3gte7T3Xo6OjoSGhvLtt99Sq1Yt3nrrLSAxme3WrRsTJkywLsM3ZcoU\nfH19U62vSpUqPP/883Tq1Il+/fpRvXp13n//fevx+93XmTNnUq1aNYYPH06jRo145513OHXqFO7u\n7gCUK1eOUaNG8d///pcWLVqwadOmNFdlqV27NhEREWzdupWWLVvSqVMndu3aZY2hf//+/P333wQF\nBVlfBPT19WXhwoVERUURHBxMixYtmDRpEgULFrTWO2jQIKpWrUrfvn3p3LkzJUuWpHLlymnc6bQ/\nK/fz3nvv0ahRI4YOHUrLli05evQoX375Jf/5z3+sZUwmE++++y5jx46lVatWnDlzhhkzZlgfanr3\n7k3ZsmXp2bMnnTp1ImfOnDRv3tzmOiaTiWbNmmGxWGjbti1hYWG0adOGkJAQmzL3npPadkr3OEnr\n1q25c+cOrVu3TvMeiDxJTMbjmqgmIiLPtMGDB3P58mW++OKLzA7libNixQo++ugj9uzZk9mhPDHW\nr19PeHg4W7duJXv27Jkdjki66YVGEREReWLcunWLc+fOMWPGDNq2bavEWrIcTQsRERGRJ8bs2bN5\n9dVXyZcvH717987scEQemKaFiIiIiIjYiUauRURERETsRMm1iIiIiIidKLkWEREREbETJdciIiIi\nInai5FpERERExE7+HwT8XqNhyV+YAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "# Plot results.\n", - "ID = 182\n", - "example = df_dfc.iloc[ID] # Choose ith example from evaluation set.\n", - "TOP_N = 8 # View top 8 features.\n", - "sorted_ix = example.abs().sort_values()[-TOP_N:].index\n", - "ax = plot_example(example)\n", - "ax.set_title('Feature contributions for example {}\\n pred: {:1.2f}; label: {}'.format(ID, probs[ID], labels[ID]))\n", - "ax.set_xlabel('Contribution to predicted probability', size=14)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aPXgWyFcfzAc" - }, - "source": [ - "The larger magnitude contributions have a larger impact on the model's prediction. Negative contributions indicate the feature value for this given example reduced the model's prediction, while positive values contribute an increase in the prediction." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0swvlkZFaY1Z" - }, - "source": [ - "You can also plot the example's DFCs compare with the entire distribution using a voilin plot." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zo7rNd1v_5e2" - }, - "outputs": [], - "source": [ - "# Boilerplate plotting code.\n", - "def dist_violin_plot(df_dfc, ID):\n", - " # Initialize plot.\n", - " fig, ax = plt.subplots(1, 1, figsize=(10, 6))\n", - "\n", - " # Create example dataframe.\n", - " TOP_N = 8 # View top 8 features.\n", - " example = df_dfc.iloc[ID]\n", - " ix = example.abs().sort_values()[-TOP_N:].index\n", - " example = example[ix]\n", - " example_df = example.to_frame(name='dfc')\n", - "\n", - " # Add contributions of entire distribution.\n", - " parts=ax.violinplot([df_dfc[w] for w in ix],\n", - " vert=False,\n", - " showextrema=False,\n", - " widths=0.7,\n", - " positions=np.arange(len(ix)))\n", - " face_color = sns_colors[0]\n", - " alpha = 0.15\n", - " for pc in parts['bodies']:\n", - " pc.set_facecolor(face_color)\n", - " pc.set_alpha(alpha)\n", - "\n", - " # Add feature values.\n", - " _add_feature_values(dfeval.iloc[ID][sorted_ix], ax)\n", - "\n", - " # Add local contributions.\n", - " ax.scatter(example,\n", - " np.arange(example.shape[0]),\n", - " color=sns.color_palette()[2],\n", - " s=100,\n", - " marker=\"s\",\n", - " label='contributions for example')\n", - "\n", - " # Legend\n", - " # Proxy plot, to show violinplot dist on legend.\n", - " ax.plot([0,0], [1,1], label='eval set contributions\\ndistributions',\n", - " color=face_color, alpha=alpha, linewidth=10)\n", - " legend = ax.legend(loc='lower right', shadow=True, fontsize='x-large',\n", - " frameon=True)\n", - " legend.get_frame().set_facecolor('white')\n", - "\n", - " # Format plot.\n", - " ax.set_yticks(np.arange(example.shape[0]))\n", - " ax.set_yticklabels(example.index)\n", - " ax.grid(False, axis='y')\n", - " ax.set_xlabel('Contribution to predicted probability', size=14)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PiLw2tlm_9aK" - }, - "source": [ - "Plot this example." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 435 - }, - "colab_type": "code", - "id": "VkCqraA2uczm", - "outputId": "71a86715-69c9-4f31-8dc5-383c0018465b" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAArcAAAGiCAYAAAD9bu7CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmcjvX+x/HXdV33MjMMJlmmEkoZJcvYk23GTtaojl+o\nRLLFSbKl6OS0SGVSaJOkRdEhTudQoZAWpU60abGMJYoxy71c1/f3x3Xfl3vMDDO22T7PxwP3cu3X\nmHnP9/5+P19NKaUQQgghhBCiBNAL+wCEEEIIIYQ4WyTcCiGEEEKIEkPCrRBCCCGEKDEk3AohhBBC\niBJDwq0QQgghhCgxJNwKIYQQQogSQ8KtEEIUQ/PmzWPq1KkA7Nmzh4SEBCzLOivbTk1NJTExkfNZ\nKdLn83HnnXfSuHFj7r777vO238KWlJTEpk2bCvswhChRJNwKIc67pKQk6tevT2JiIg0bNiQxMZGD\nBw+e0Ta3bNlCmzZtztIRFp78nsewYcOYMWOG81zTtNPe54kBKz4+ni+//PKMtllQ//73vzl8+DCf\nffYZTz755Hnbb3EVCAQYPXo0SUlJJCQk8Nlnn2V73+/3c//999OyZUuaNWvG8OHD2b9/v/Pe5MmT\nSUpKolGjRvTp04f169cXxmkIcU5IuBVCFIp58+bx5ZdfsnXrVr788ksqVap0RttTSp1RGDNN84z2\nf7bk5zyKyrGeTXv37qVGjRqndQ9L4vXIj8aNG/P444/n+n9n4cKFbNu2jRUrVrBhwwZiY2P5xz/+\nAdjXKz4+nsWLF/PFF18wevRo7r77bvbu3Xu+T0GIc0LCrRCiUOT1kfdXX33FTTfdRJMmTejVqxdb\ntmxx3nvnnXfo2rUriYmJdOjQgTfeeAOAzMxMhg4dyoEDB7K1BE+cOJGnnnrKWf/EVtGkpCQWLFhA\njx49aNiwIZZlceDAAUaPHk2LFi1o3749ixYtyvMcfD4f//znP0lKSqJJkyYMGDAAv98PwNq1a+ne\nvTtNmzZl4MCB/Pzzz9n2++KLL9KjRw+aNGnC2LFj8fv9eZ5HSkoKo0ePZvz48TRu3Jhly5aRkpLC\n+PHjs13PpUuX0qpVK1q1asVLL73kvHey63DvvfeSmprK8OHDSUxM5IUXXsjRzeHAgQMMHz6cZs2a\n0alTJ9566y1nWykpKdx9991MmDCBxMRErr/+ev73v/8578+fP5/WrVuTmJhIly5d2Lx5c47rOGfO\nHJ555hlWrVpFYmIib7/9Nkop5s6dS1JSEi1btuS+++7j2LFjwPFuGEuXLqVdu3YMHjw41/vz4Ycf\n0qtXL5o0acLNN9/M999/D8CuXbto1qwZ27dvB2D//v00b97caf3M6+ss8to9//zzXHvttbRq1Yo1\na9awbt06OnXqRLNmzZg3b1626zN69GjGjh1LYmIiffr0YceOHbker1KK+fPn06FDB5o3b87YsWM5\nevRorsu63W4GDhxIYmIiup7zR/mePXu47rrruOCCC/B4PHTr1o0ff/wRgOjoaEaOHEl8fDwAbdu2\n5ZJLLsl234Qo1pQQQpxn7dq1Uxs3bszx+r59+1TTpk3V+vXrlVJKbdy4UTVt2lQdPnxYKaXURx99\npHbt2qWUUuqzzz5T9evXV999951SSqlPP/1UtWnTJtv27rvvPvXkk086z09cpl27dqpXr15q3759\nyufzKcuyVO/evdXcuXNVMBhUu3btUu3bt1cff/xxrufxwAMPqFtuuUUdOHBAWZaltm7dqvx+v9q5\nc6dq0KCB2rhxowoGg2rBggWqQ4cOKhAIOPvt16+fOnjwoDpy5Ijq0qWLev311/M8jzlz5qirr75a\nrV27VimlVFZWlpozZ44aP368Ukqp3bt3q9q1a6tx48aprKws9f3336vmzZs71zg/12HTpk3O8927\nd6uEhARlmqZSSqkBAwao6dOnK7/fr7Zv366aN2/uLD9nzhxVr149tX79emVZlpo1a5bq37+/Ukqp\nnTt3qjZt2qiDBw8qpZTas2eP+v3333O9lpHno5RSb731lurYsaPavXu3ysjIUCNHjsxxvhMmTFCZ\nmZnK5/Pl2N63336rWrRoobZt26Ysy1LLli1T7dq1U36/Xyml1Jtvvqm6du2qMjMz1W233aYeffRR\nZ91TfZ1dddVVztfIm2++qZo3b67+/ve/q4yMDPXjjz+qa665xlk/fO/+85//qGAwqF544QWVlJSk\ngsGgc+3D9+mll15SN954o9q/f7/y+/3q/vvvV+PGjcv1ekVq3bq12rJlS7bXvvnmG3XTTTep/fv3\nq4yMDDVu3Dg1c+bMXNc/ePCgqlevntq5c+cp9yVEcSAtt0KIQjFixAiaNm1K06ZNGTlyJAD/+te/\naNu2La1atQKgRYsW1K1bl3Xr1gHQpk0bLrnkEsD+SLZly5Z8/vnnZ3QcAwcOpEqVKng8Hr755hv+\n+usvhg8fjmEYXHLJJfTr14/33nsvx3pKKd555x2mTJlCpUqV0DSNBg0a4Ha7Wb16NW3btqVFixYY\nhsHtt99OVlYWW7duzbbfCy+8kHLlytGuXTunFTEvDRs2JCkpCQCv15vrMqNGjcLr9XLllVfSp0+f\nXI87LyqPlvTU1FS2bt3KPffcg9vtJiEhgX79+vHuu+86yzRq1IhWrVqhaRo9e/Z0WkgNwyAQCPDj\njz8SDAa56KKLqFatWr6OZ+XKlQwePJiLL76Y6Ohoxo0bx6pVq5zWZE3TGDVqFFFRUXg8nhzrv/XW\nW9x0001cc801aJpGr1698Hg8fP311wD069eP6tWr069fP/74449sg9hO9XXmdru58847MQyDrl27\n8ueffzJo0CCio6OpVasWtWrVcq4BQN26denQoQOGYXDrrbfi8/n46quvchzzm2++yd13303lypVx\nu92MGDGC999//7QGCtaoUYOLLrqI1q1b06RJE3755RfuuuuuHMsFg0HGjx9P7969qVmzZoH3I0RR\n5CrsAxBClE5z586lefPm2V7bu3cvq1ev5sMPPwTswBUMBp3l1q1bx9y5c/n111+xLIusrCxq1659\nRsdRtWpV5/GePXvYv38/TZs2dfZvWRZNmjTJsd6ff/6J3+/PNawdOHCAiy66yHmuaRrx8fHOgB6A\nihUrOo+jo6NPOaAu8jhzo2latmUuuugi52PoM3Hw4EHKly9PdHR0tm1HfoR94YUXOo+joqLw+XxY\nlsWll17KpEmTmDNnDj///DPXXXcdEyZMoHLlyqfc74nX8OKLLyYYDPLHH384r53smuzdu5d3332X\nV199FTj+tXTgwAFnmX79+nHXXXcxffp03G638/qpvs4qVKjg9A2OiooCst/PqKgoMjIycj3O8H2K\nPI7IYx45cqTTzUAphcvl4o8//sjXNYs0bdo0/H4/n332GVFRUSxYsIAhQ4bw5ptvOssopRg/fjwe\nj8epvCFESSDhVghRKHJrKYyPj6dXr15Mnz49x3t+v58xY8bw2GOPkZycjK7rjBgxwtlObgORoqOj\nycrKcp6fKkDGx8dzySWX8P7775/y+OPi4vB6vfz+++85AnblypVzBMvU1NRTBlTIu+pBfgZapaam\nOq1vqampTiA61XU42bYrV67MkSNHyMjIICYmJse2T6Vbt25069aN9PR07r//fmbNmsUjjzxyyvUq\nV66cbYDTnj17cLlcXHjhhaSmpp7yuKtWrcqdd97JsGHDcn0/IyODhx9+mBtuuIGUlBQ6depEuXLl\nTvl1djr27dvnPFZKsW/fPqpUqZJjufj4eB5++GEaNmx42vsK++GHHxg7diyxsbEA3HLLLTz99NP8\n9ddfVKhQAYBJkybx559/Mn/+fAzDOON9ClFUSLcEIUSR0aNHDz744AM+/vhjLMvC5/OxZcsW9u/f\nTyAQIBAIEBcXh67rrFu3jk8++cRZt2LFivz111/OoCOAOnXqsG7dOo4cOcLBgwd55ZVXTrr/evXq\nUbZsWRYsWIDP58M0TX788Ue++eabHMtqmkbfvn355z//yYEDB7Asi6+++opAIECXLl346KOP2Lx5\nM8FgkBdeeAGv10uDBg1OeQ1yO4/8UKEBWFlZWfz444+88847dOvWLV/XoVKlSuzevTvH9sAOiQ0b\nNuSJJ57A7/ezY8cOli5dSo8ePU56LAC//PILmzdvxu/343a78Xq9uQ5+yk23bt14+eWX2b17N+np\n6cyePZtu3bpla9U8mf79+/P666+zbds2wA6z69atc1pUH3roIa655hpmzJhBmzZtuP/++wFO+XV2\nOr799lvWrFmDaZq8/PLLeL1e6tevn2O5G2+8kSeeeMIJ9YcPH2bt2rV5btfv9+Pz+ZzH4cGMYHeF\nWL58OceOHSMQCLB48WKqVKniBNv777+fX375hWeffTbXbh1CFGcSboUQ511eLW5Vq1Zl7ty5zJs3\njxYtWtCuXTtefPFFlFKUKVOGyZMnM2bMGJo2bcqqVatITk521r3sssvo1q0bycnJNG3alIMHD9Kz\nZ09q165NUlISQ4YMoWvXric9Dl3Xee6559ixYwfJyclce+21TJ06Nc+gOWHCBK688kpuuOEGmjVr\nxqxZs1BKUbNmTR577DFmzJhBixYt+Oijj3juuedwuVwnPf+8ziO/17Rp06Z06NCBW2+9lSFDhtCi\nRQuAU16HO+64g7lz59K0aVOnykLkMc6aNYvdu3fTqlUrRo8ezZgxY5xt53UsYAeuWbNm0aJFC1q1\nasXhw4cZN25cvs7nhhtuoGfPnvzf//0fHTp0IDo6milTpuTYR17q1q3LjBkzmD59Ok2bNqVTp04s\nW7YMsCtZfPLJJzzwwAMA3HfffWzfvp2VK1ee8uvsZOeb1/Pk5GRWrVpFkyZNWLFiBSkpKU5LaeSy\ngwYNIjk5mdtuu41GjRpx0003OeE8N507d6ZBgwYcOHCAIUOGUL9+fScYT5gwAY/HQ8eOHWnZsiUb\nNmwgJSUFsLs/vPnmm2zfvp1rr73WqcyxcuXKk56nEMWFps7ksxYhhBBC5CklJYXff/+dRx99tLAP\nRYhSQ1puS5F9+/Zxyy23UL9+fRISEpwR6EIIIYQQJYWE21Jk3rx5fPbZZ1SvXp3Bgwc7pW5O17Jl\ny0hISGDgwIFn6QiFEEIIIc6MVEsoRX799Vc0TWPw4MH06dPnjLd3Nnu0BINBpz+iEEKUFOEazkKI\n80dabkuJW265hU2bNgF2+Zc6derw888/8/jjj9OxY0caNmxInz59WLNmjbPOv/71L7p160ZiYiJ1\n69alU6dOvPbaa4Ddajtp0iQ0TWPLli0kJCQ4gy4SEhKoU6eOM7AhJSWFhIQEJk6c6KybkJDA3/72\nNx544AESExN57rnnAHugR79+/WjUqBFJSUk88sgj2UoYCSGEEEKcjITbUqJz585OXcWWLVsycOBA\n/vnPf/L8888TGxtLp06dSE1NZfTo0c786nv37qVatWr07NmTbt26sX//fmbMmMHXX3/N5ZdfTsuW\nLVFKUbVqVQYNGsQNN9yQ5/5zG9m8detWPv30U66//nqqVavGxx9/zIgRI9izZw/t27cnLi6Ol156\nKdeap0KI7G655RaWLl1a2IdRIFu2bKFNmzb5WnbZsmX87W9/O639nMm6QojiR8JtKTFgwAAuvfRS\nAK6//nqGDRvGhg0bMAyDhg0bUq5cOWrVqoVlWbz++usA3H777fTu3ZuKFStSoUIFpwD9p59+Sr16\n9ejevTsAl156KRMnTmT48OF57j+3LgxlypThrbfe4sEHH6Rnz54sWrQITdOoU6cO5cuXp169egAs\nX77cqeUohDg7Xn75Za677jqaNGnC5MmTCQQCuS4XCAQYPXo0SUlJJCQkOL/8hqWkpFC3bl0SExOd\nklIn1sw9mfxMTnE6y57Jutu3b6dPnz40aNCAvn37smPHjtPerxDi/JNOjqXUnj17ALAsi8WLFzuv\na5rG77//DsCdd97JJ598kuOHwuHDhwu0L9M0c329Vq1alC1bNscxbdy4kY0bNzrHA7Br1y5q1apV\noP0KUVyZpnlOZ4zasGEDzz//PAsXLqRy5crcddddzJkzJ88atI0bN2bw4MGMGTMm1/e7du1aYkpd\nBQIBRowYweDBg7n55pt5/fXXueuuu/jPf/4j4wKEKCak5baUuvjiiwFwu91s2rSJ7du3s337dr75\n5htSUlJIS0tzgu0rr7zC9u3badWqFXC8FTavmYLCc9CHC9//8MMPubaanDgrTviYpkyZ4hzP9u3b\nef/99yXYimIvISGBRYsW0b59e1q0aJEtDC5btoybb76ZmTNn0qxZM6fY/tKlS+natSvNmjVjyJAh\n2aaj/eSTT+jSpQtNmjRhxowZBTqW5cuX07dvXy6//HJiY2O56667eOedd3Jd1u12M3DgQBITE/M9\nu1ikHj168N577+Vr2fnz59OhQwcSExPp3r17tjEAYP8y/tBDD9G4cWO6du3qjCMA+/vN5MmTue66\n62jTpg1PPvnkaQ163bJlC6ZpMnDgQNxuN7fccgtKKTZv3lzgbQkhCoeE21LqggsuoEuXLgQCAfr1\n68e0adMYPXo0bdu25a233iI6OtqZR/7pp59m5MiR2X6QgD0POthTSz744IO89dZbgD3VJ8D06dOZ\nOnUqH3zwQb5+yAwYMAClFI8++ih///vfmTRpEn369OHWW289m6cuRKFZs2YNy5YtY9myZaxduzZb\nH9lt27Zx6aWXsmnTJoYPH86aNWtYsGABzzzzDJs2baJx48ZOy+rhw4cZPXo048aNY/PmzVSrVo0v\nv/zS2VZqaipNmzZl3759uR7HTz/9REJCgvM8ISGBQ4cOceTIkdM6rw8//JBmzZpx/fXXs2TJkmzv\nhQem5kf16tVZsmQJX375JSNGjGD8+PH88ccfzvvha/Tpp58ycuRIRo0axdGjRwG49957cbvdrF27\nlmXLlrFx40bne9KJ7rzzThYsWJDrez/++CO1a9fO9lrt2rX56aef8nUOQojCJ+G2FHv44YcZOnQo\nuq6zfPlyvvrqKxITE2ndujUul4tHH32U+Ph4vv32W8qVK0fnzp2B410FmjRpwvXXX49hGLz++uvO\nHOhTp07lyiuvZMeOHezfv5++ffuiaVq21tsTnwO0bt2aZ555hjp16rB+/XrWrFmDy+Vi0KBB5+mK\nCHFuDR06lNjYWGcQZmSLZpUqVRgwYAC6ruPxeHjjjTcYOnQoNWvWRNd1hg4dyo4dO0hNTWX9+vVc\nccUVdOjQAcMwGDx4MBdeeKGzrfj4eLZs2eL0kz9RRkYGsbGxzvPY2FiUUqSnpxf4nLp06cKqVavY\nvHkz06dP55lnnmHVqlUF3g5Ap06dnPPo0qUL1atXzzb9bMWKFRk4cCCGYdC1a1dq1qzJRx99xKFD\nh9iwYQOTJk3C6/VywQUXMGjQoDynk33uuee44447cn3vxGsDULZs2TynYBZCFD3SgagUWbRoUbbn\n0dHRjB07lrFjx+a6fPv27Wnfvn221x577DHnsaZp2Z6H1alTh3fffTfbaw899JDzuHfv3vTu3TvX\nfSYnJ59yHnchiqvIsHnxxRdz4MCBXN8Du1rJP/7xDx555BHA7v6jaRr79+/nwIEDOZYPf5KSHzEx\nMdnC2rFjx9A0jTJlyhTofAAuv/xy53HDhg0ZOHAg//73v+natWuBt7V8+XJefvllp/99ZmYmf/75\np/N+uOJL2EUXXcSBAwfYs2cPwWCQ6667DrCvlVKqQNck7MRrA/b1iRwfIIQo2iTcCiHEeZKamuqE\nwb1791K5cmXnvRM/yYiPj2f48OFOVZJIv/76q/NJSeS286tWrVrs2LHD+TRm+/btVKxYkfLly+d7\nG3k53YoGe/fuZerUqbzyyis0bNgQgF69emXr0rR///5s66SmppKcnEx8fDxer5dPP/30jCoqAFxx\nxRW8/PLL2V774Ycf+L//+78z2q4Q4vyRbglCCHGevPDCCxw9epTU1FReeeWVk7Zu3nTTTcybN8/p\n65mWlsa///1vANq0acNPP/3EmjVrME2ThQsXcujQoXwfR69evVi6dCk///wzR44c4bnnnqNv3755\nLu/3+51yfH6/H7/f77y3du1ap9/rtm3bWLRoUbZPX5KSkli+fPkpjykzMxNd14mLi8OyLN5++21+\n/PHHbMscOnSIRYsWEQwGWb16NTt37qRNmzZUqlSJli1b8vDDD3Ps2DGUUuzatStH2bL8aNq0Kbqu\ns2jRIvx+P6+++ioAzZs3L/C2hBCFQ8JtKXD06FHmzJnj/AASxYfcu+LtxPuXnJxMnz596N27N+3a\ntTvpxCft27fnjjvuYOzYsTRu3JgePXqwYcMGAOLi4njqqad47LHHaN68Obt27XJaO8Fu0UxMTMxz\nQFmrVq0YMmQIAwcOJDk5mUsuuSTbNLHdu3fP1l+1c+fONGjQgAMHDjBkyBDq16/vVG5YtWqVU+Hg\nvvvuY9iwYfTs2ROwy2odOXKE+vXrn/JaXX755dx6663ceOONtGzZkp9++onExMRsy9SvX5/ffvuN\n5s2b89RTTzFnzhyntfmRRx4hEAjQrVs3mjZtypgxYzh48GCu+7rjjjuYP39+ru+53W7mzp3LsmXL\naNKkCc899xyPPfaYlAErZuR7Z/F2pvdPU6dTK0UUK7t37yY5OZm1a9dyySWXFPbhiAKQe1e8Rd6/\n9u3b89///pdq1aoV9mGdN1988QWvvfYas2bNKuxDOS3y/6/4kntXvJ3p/ZNfRUW+JCQkoGkaa9eu\n5aKLLirswxFCFAONGjWiUaNGhX0YQohSRrolCCHEeXCmA52EEELkj7TcCiHEebB9+/bCPgQhhCgV\npOW2lLjgggto3759thI3EydOJCEhgRdffJHbbruN6667jrp169KkSROGDx+e52AUsLsp1KlTxxlU\nkpKSQkJCAhMnTnSWWbt2Lf369aNRo0YkJSXxyCOPkJWVdc7OsSQyDIOLL74YwzAK+1DEaZD7V7zJ\n/Su+5N4Vb4Zh5BhQWhASbkuBChUq8MQTTwCwevVqAILBIB988AEul4suXbpw8OBBWrVqxY033ki1\natX48MMPmTp1aoH2E/mx64YNGxgxYgR79uyhffv2xMXF8dJLLzF9+vSzd2KlQHx8PB988MFpFaMX\nhU/uX/Em96/4kntXvMXHx7NkyRIqVKhwWutLt4RSoGzZsjRv3pwqVaqwbds2vv32R3bu/IkjR47Q\nvPm1eDyxPPTQo3z88XoOHTpE9eo1+e677/j00085dCj7TD1//pmO13ssx/OMDD9KKXy+IIcOHeOF\nF15C0zRq1boSrzeGK6+sw//+9z+WL1/OyJF/x+v1nu/LUGxVrFg2x30QxYfcv+JN7l/xJfeu+NJ1\njbi4Mqc9M6CE21JC0zQ6duzKq6++zJo17/Prr7+gaRqdO3dn69atjB49DMuysrW+BgIBjh07RkzM\n8Sk5LQssS+V4bpomYE97aVmKffvs2ZI+++xTPvvsU+cYwC7xUbPmZef8nEuSyGsuih+5f8Wb3L/i\nS+5d6STdEkqRLl26oZTiP//5Nxs2rCMmpgytWrVh3bq1WJZF8+bX8t//bmDevJecdfIqgxwVFQVA\nRob9W/HPP/+ULRjHx9vlwsaMuYf167c4f5YseUeCrRBCCCHOGWm5LUWqV69BQsJV7NjxHQDdu/fC\n4/EQF1cRgP/971tmz36Ur7768pTbuuKK2nz77TaeeOJRqlWrziefrM8WhPv06c+mTZ8wd+5TfPPN\n13i9Xn766UfS0o7y5pvvnpsTFEIIIUSpJy23pUyXLt3RNA1d1+nc2Z7Xvm/f/rRu3ZZAIMDXX29l\n0KDb0TQtR13OyOdjx47nsstq8dNPP/DHHwfo1q1HtnWaN7+WmTMf54orarN580bWr/8IwzDo3//m\n83eyQgghhCh1ZPrdUuTQoWPS/6gYqlQploMH0wr7MMRpkvtXvMn9K77k3hVfuq5RseLpDSYDabkV\nQgghhBAliIRbIYQQQghRYki4FUIIIYQQJYaEWyGEEEIIUWJIuBVCCCGEECWGhFshhBBCCFFiSLgV\nQgghhBAlhoRbIYQQQghRYki4FUIIIYQQJYaEWyGEEEIIUWJIuBVCCCGEECWGhFshhBBCCFFiSLgV\nQgghhBAlhoRbIYQQQghRYki4FUIIIYQQJYaEWyGEEEIIUWJIuBVCCCGEECWGq7APQAghTkXTwFIQ\ntBSWUqiI93RA1zVcmoamgVJ5bUUIIURpIOFWCFGkaJodYv2mIsu08Act/KaJadnBNXu0BS30t6aB\ny9CIMgw8Lg2PoeM19ND7QgghSgsJt0KIIsFvWWQGLdJ9Jn7TwrJOjLG5U+G/FZiWwhewADskG5pG\ntNsgxmMQ5dJx65q07AohRAkn4VYIUSQczgiS7g+ete0pBUGlSPMFSfMFMXSNKJdBWa9BtEvH0KRN\nVwghSiIJt0KIUsG0FOl+O0AbukaMxyDW4yLKJV0XhBCiJJFwW4qUKeMt7EPANC0yMvyFfRiilDMt\nRVpWkGNZQdyGRlmvizJuA69Ll24LQghRzEm4LUUsS6EK+Se3YUj1OVF0KMBvKg5nBPhLCxDlNoj1\nuohx60h77vkTWQ3DVAoz/K8CpZQ9kDA0mFALDR60/2gYGrh0DUPT0DVwaTq6LlUzhCjNJNwKIQR2\nuMrwm2T4TQxdo4zHoKx0WzirwiE2YFoELIXfUgSCFn7TImgpJ8CeTjDVNLtyhqZpuHS7WobbpeM1\n7DJxbkNHl1JxQpQKEm6FEMXCnG0PE7BO3aXFrXsYVW/SGe3LtBRHs4KkZQVxhbotxLjsbgsSdAsm\nYNlB1he0yApa+IIWSimssxwy7WBsPzBD+8Nnv6eHWnk9Lp1ol443VDnDrcsnSUKURBJuRYFNmzaV\nqlWrMnz4iMI+FFGK5CfYFmS5/FBAwFT8Geq24NI1Yjwuol06US4dl5QWc2gaBCxFwFT4TDvE+gIm\n5jkIsgVlKUApMv0mmX4TsAOvoWl4Q32tvYaO29CkXJwQJYCE20KQlZXFhAkT+Pnnn3G5XNSsWZPZ\ns2ezfPmRgRk2AAAgAElEQVRyXnvtNUzTJDY2lgceeIAaNWowd+5ctm/fzpw5c8jMzKR///6MHz+e\n1q1bF/apCFFqKGUH3SOZAY4ARuij72iPQVQpCkbhWeAClrIn27As/AGLrNBEG/mtT1zYLAWWUgR8\nQY757C4Num53afC6dGK9LqJkjIAQxZKE20Lw8ccfk5aWxsqVKwFIS0vj888/Z/Xq1SxevBi32836\n9euZOHEiS5YsYfjw4QwZMoRXX32V7777jrZt20qwFaKQmZYi0zLJDJhOMDJ07BnS3DpuXcel2cuF\nS+oWp+B7fJCXhWlBQCkCpoU/qCJmjCseQTY/FPa9CndpcBm6hFshiikJt4Wgdu3a7Ny5kxkzZtCk\nSRPatm3Lhx9+yPfff0///v1Do4MVaWlpgN1X7LHHHqNnz55cfPHFPPTQQ3lu++jRoxw9epS4uDhM\n03TWj42NpVu3zvTvfyPvvbeSPXt206lTZ0aMGMW0aVP56qutXHNNPR555HFiY2OZMOEetm79Ep/P\nx5VX1mbixMlcdtnlue5z/fp1PPvsM+zdu4fLL6/FxImTueKKK8/+hROiiDoejMAfDGbr65npMjh6\nJBOXbgdew7BH+Bu6hoE9CsoI9QnVNexqAMDJ5pg4WUg+1XpKgYVyWi6Vwu46QKhagaUImoqAFTHl\ncQkKsUKI4iM1NdXJMmHlypWjXLlyJ11Pwm0hqFatGqtWrWLTpk2sX7+e2bNn0759e/r27cuoUaNy\nXWfXrl3ous6RI0fIzMykTJkyuS63cOFCUlJSmDlzJnv27AHsL4RBgwYB8MEHa3nuuQUEg0Fuvrkf\nO3bsYNq06dSsWZORI4fz+uuvcccdw2jZshUPPDADl8vF008/yeTJE1my5M0c+9u+/TumT5/G008/\nQ506V/HeeysZO3Y0y5atwO12n6UrJkTxZAdI7NZOTCD7N2kt9JcWehYe8Y8GOhqabv9LRCUALRSA\n8xKuNuAE0lCYVVbo31BKDVcmIDwQSwghipgBAwY4WSZs5MiReWalMAm3hWD//v2UL1+e5ORkrr32\nWtq0aUO7du2499576d+/P1WqVMGyLLZv387VV1/NkSNHGD9+PLNnz2bjxo1MnTqVJ554ItdtDxo0\niN69e+douQ276aabiYuLA6Bhw0QuuKAiV15pt7K2a5fMZ59tAaBHj57OOkOHDuO1114lPT09R6he\nvvwd+vbtx1VXXQ1A9+7X8+KLC/jmm20kJjY6S1dMiOIt3BKraXYsNXQNQ7dLVOl66I8Geqj1FmU/\nDgddZxvh7Z2iZEO4coATWkPPrVDgtSJqyJqm/TioLEzTft+pLXtWr4IQQhTM4sWLc225PRUJt4Xg\n+++/Z9asWQBYlsWwYcNo3Lgx48aNY/jw4ViWRSAQoHPnzlx99dVMnjyZG264gcTERBo0aMDgwYN5\n4403uPHGG3Ns+1TN9RdcUNF57PVGUbHi8edRUV4yMzOwLIuUlKdZs+a//PXXX84P5b/++jNHuE1N\nTWXlyhW88cYSwP6hGAwGOXjw4BldIyGKIy0UUA0dPIaBx9CoXNZDdNC0JxgwNLtFtgj2wQ0fkzOJ\nggVBpeyBY0ELn6UImpYEXyHEeRMfH39a60m4LQStW7fOdUBY9+7d6d69e47XU1JSnMe6rvPKK6+c\n0+NbvXoV69Z9xLx5zxMfH09aWhpt216X6w/iKlWqcvvtd3DbbUPO6TEJURRFlpOKcut4dbtqQmSJ\nsArRHgLHfNnWK0qhNix8TDp2S7I7ciyVN/sAs4AFftPCH7TwFbMqCacS/gVF6hkLUXxJuBU5ZGZm\n4PV6KFeuHJmZGaSkPJWta0Ok3r37cs89Y2natBl1615DZmYGX3zxBY0aNSI6OuY8H7kQ556ha0S5\nDGI89mQA3lwmAiiK4fVMKWV3i3DrOm4dYly6E3oDoUFoPsvCF7DICoYCbzG4EOGW9iiXgTePX1CE\nEMWLhNtS5MR8mle/vW7drmfjxk/o3Lk95cuXZ/jwkbz99tJcl73qqquYOnUajzwyk127fsfrjaJB\ng4Y0aiT9bcXZ5dY9+Z6h7GwK95GN8RjEuA2i3TqGJsEnTClwaRoul0YUOnjs7y1+0+7S4DOPB95z\nMTNZQYRnKnOCrGHPVOYxct5Pub9CFF+aUvJfuLTIyPBT2LfbMHTS0rIK9RiKm0qVYjl4MK2wD+Oc\n23fMT7o/WNiH4TB0jRi3QVmvPSPZ6X5MXVru38mEuzQETIuAUnZ3hsh6uRFVHM7W/jTskmsel4HX\nZU+969Y03IaOruU/vMr9K77k3hVfuq5RsWLZ015fWm6FECJE18DrMoiNchHjsltoxZkLd2nwGDoe\noIzLcN4LKnums6Blt/SaZvi5halOHnq1UJ9nIzSzmCvUEuuOqEah5RJkpUlHiJJNwq0QolTTsKsY\nxHpdxISm0pXwc/64NA2XoYGR/fXw7xVWqKaZ0uygG1nnV4/43SOveyb3UojSR8KtEKJUMnSNaLdB\n7AndDiQMFQ3h+3B8oguNE/uGyL0SQuRGwq0QotTQNQ2vS6es1x4c5pJuB0IIUeJIuBVCFAnRHp2A\npRE0z+6IekPX8Bg6ZbwGMS4dj3Q7EEKIEk3CrRCiSCjvcVHe47JrpQYtsiLqpSqVvwkCwgX4XaEu\nB1ER5Z7CgVaCrRBClGwSbksRXY+cnb5wmKZVqPsXRZ9X1/F6dMqFytUGLHs62PCoeksprNCXkR1m\n7a9tl65haHbN0hPr0EqgFUKI0kPCbSmSnu7DKswK6kKchnBpp4KQMCuEEKVXznkjhRBCCCGEKKYk\n3AohhBBCiBJDwq0QQgghhCgxJNwKIYQQQogSQ8KtEEIIIYQoMSTcCiGEEEKIEkPCrRBCCCGEKDEk\n3AohhBBCiBJDwq0QQgghhCgxZIYyIUSxomn2DGQW2ach09DQNZmdTAghSjsJt0KIIkvTwG8qApbC\nFzRDjy1MU5FbhjV0cOsGLkPDY2h4DB2PoWFomoReIYQoJSTcCiGKFKUgy7TICJhkBIIETbDymUxN\nC/wEneeaBrqm4TJ0yrh1olwGHkPDpUvYFUKIkkrCrRCi0CkgM2iR7g+S7jexrNxbZgu8XQWmUpiW\niS9gohFA1+0W3RiPQbRLx2PoaGdhX0IIIYoGCbdCiEKhaZAVVKQHghzLChI8S4H2ZBRgWopMyyQz\nYDotu1Eugyi3TrRLxy3dGIQQoliTcFuKlCnjPe/7NE2LjAz/ed+vKLqCSpERMDnqM/EHzUINkeGW\nXbvFOKIbg67hdel4XDouTcPQ7cDr0u02Xj3Xpl4NSykUCkuBUgoLOOYLkhG0UMoO7wr7PQBNO749\nDQ0d+xgMXcOlac7gOSGEEPkn4bYUsSzl/FA9XwxDqs2J490OjvmCZARMTKtoJrbj3RgUvqDlvK6F\nwmcoi6KFXnPWC/2lIrYDdpjN1A0OH83KV6t0eJt66IHL0PDodsh265r9x9ClKoQQQpyEhFshxDnj\nMy3SAybHfEGCeVQ4KA6UCkXV0ziBcGttfvcDdsAGuwuFDwt89uu6Zrf2RrkMvG4db6gahFsGyAkh\nhEPCrRDirFGAL2iRGTQ55g8SMNVZC11ztj1MwDp1Fxe37mFUvUlnZ6dFjGX3aTjejQI77LoNjSi3\nQZRLxxMaMCeEEKWVhFshxGmz69Ba+C1Fht8k028SVGcv0EbKT7AtyHIlQbj/ri9od6M4gt26a2ga\nXrc9SM6r63hcMkhOCFF6SLgtZaZMmcSWLZ+SlZXJhRdeyMCBg+nVqw+BQIDJk+/ju+/+R2pqKvPn\nv0CjRo3z3M7Ro0d58MH72bx5E3FxcYwcOZrOnbuexzMR55umgWWBX1kETEVW0CKzgHVoxblnKft+\nBHxBjvkia/1qTp1fr6Fj6NKdQQhRMkm4LQT33HMPv/76K36/n+rVq/Pwww8TGxvL7NmzWb16NXFx\ncTRp0oRNmzbx9ttvA7B8+XJee+01TNMkNjaWBx54gBo1ahR437fdNoRp0x7E7Xbz22+/cscdt5GQ\nUIfLL69Fw4aJDBhwC/fee88ptzNz5kN4PB7Wrl3Hjh3bGT16JFdemcBll11W4GMSRUd4QJOlIGhZ\nBC0IWBZ+U+EPWvhNy64IIIGo2Mg2SC5gD5ILB15dA49h4HHZlSBcuo4rVK3BkGoNQohiSsJtIZgy\nZQoVKlQA4Mknn2T+/PkkJiaybt06VqxYgdfrZdSoUU6ZoM8//5zVq1ezePFi3G4369evZ+LEiSxZ\nsiTHto8ePcrRo0eJi4vDNE3A7pMXGxsLkC182pUTNHbv3kVCQh1uvnkAAHrudY4cmZmZfPDBWpYu\nXU5UVBQNGjSkTZs2vPfeCkaNGnPG10ecffaXkoZlKUwUlmWXqbJCwSdoKUzT/jdgmZiW/fWhTm8M\nlSjinMALBEy7/y4crwIR/t4TDryGBoahoYfKouka6KHqERrh0mih10PrnlhRIrdjEEKIk0lNTXWy\nTFi5cuUoV67cSdeTcFsIli1bxooVKwgEAmRlZVGjRg0CgQBdunTB67Vr0fbq1Ytnn30WgA8//JDv\nv/+e/v37hwKHIi0tLddtL1y4kJSUFGbOnMmePXsA+wth0KBBzjIzZ/6DFSvexefzkZBQh5YtWxXo\n+H///TcMw6BatWrOa1deWZsvv/yiQNsR50emafFHug/LCvfRhHCZKgkYIpLzNXFitYZcaM5f4X+0\nHGH2ePjVnBXCoVc/sbSas469rBaxjROXidxR5C7zCtNaHsvntk727WmkZQXINK1c1825nxyv5L7c\nKbaTH+pkz/L4f33iy3mVhjzVt4VTX4eTL5Hf88++mZPcw1yW19DwB808vnpP/otXXk72/TK393Kr\nD6Nrdj1rkT8DBgxwskzYyJEjGTVq1EnXk3B7nn3++ee8/vrrvPHGG1SoUIGVK1fyxhtvAHl/Q1BK\n0bdv31PeTIBBgwbRu3fvHC23kSZOnMx9901i27av+fzzz/B4PAU6h4yMDMqWjc32WtmyZUlPTy/Q\ndsT5oQNRLgPTslvrLGVhWnbIsMtUSTcDkT/hQHtimHUCq93sm6NlN9wa7DwGJ11EtvBGBtzcAktk\nVtAi1s9xjBHr5valfdJAG/4rtGK4xnD4+cnWPfGF8Ga0XOLc6YQryC1EKVTk9tXxY4hcVIVfVxGh\nK+K8OI0uKFrkA5XLY0L3Kc/3TrLNXF6IvI6alusi2X7eeVyGM/HKWVHgTeVcQam8f6kQOS1evDjX\nlttTkXB7nqWlpREbG0v58uXx+/28/fbbaJpGs2bNePrppxk4cCAej4d3333XWScpKYkJEybQv39/\nqlSpgmVZbN++nauvvjrH9vPTXA/2N4D69Rvw3nsreOutN7npppvzfQ4xMTGkpx/L9lp6ejplypTJ\n9zbE+eM1dCrF2L/AhL/vKwUWdj9MU0V0TzDtwWJ+S2Galj3TloTfUiFyogqn/62uYRia3S1B15zA\nqmN3X3JmVtNOSC2UgE8FQv9XvC4Dl5PIT3szOZ3m9cklJmd/La8dapH/5pUMT++YTrmds7WPSCrX\nhzmCo1VEJ4wR+RMfH39a60m4Pc9at27Nv/71L7p06ULVqlWpW7cu27Zto127dmzdupWePXtSpUoV\n6tev73Q9aNy4MWPHjmX48OFYlkUgEKBz5865htuCMk2T3bt3FWidSy+tjmma7Nq1y+ma8MMP33PZ\nZZef8fGIcyvy+76Ohq5ruCMXcBsAzkCioFIETUVAKQKmRVZABpUVd+EWVkMPDyazJ4IwtONTDOsR\nvwTlh7RECSGKEgm355lhGMyePTvX94YNG8a4ceNQSjF58mQaNGjgvNe9e3e6d+9+Rvs+fPgwW7Z8\nSuvWrfF6o9i8eRPvv/9vHn74EQACgQCWZTmP/X5/rl0WoqOjSUpK5rnnnmHKlGl8//0O1q37iJde\nWnRGxyeKjnBWcWkaLpdGFIDbQIs+Xg7Mbyqy/CaZQXs6XWkgKXo07BZWl25P8uANfcQeDrN59s2U\neymEKMYk3BYhEyZMYM+ePWRlZVG3bl2GDBlyVrevaRpLl77JzJkPYVmK+Ph47rlnAq1btwGgd+/r\n2bdvHwAjRw4HYMWK1cTHx/Pii8/z1VdbefrpZwC4777JPPjg/bRv35YKFeKYNGmqlAErBZSyW/68\nmo5Xh9hQS6/ftMgyLdL9Jr6ghXkOkq5b9+R7hrLSKtwqG+WyJ3CIMnTchh1uTwysEmCFECWVpuTz\npFIjI8N/3j8+NAydtLSs87rPkqZSpVgOHsy9OkZRo2kQtOwJHo75TTID5jkJusXJBReU4fDhczPY\n0p6cgVCYtSdo8Oj6WevWKIrX/z+Rndy74kvXNSpWLHva60vLrRDirFHKnvq1jNugjNvAVIos0yIt\nyw66MpPZ6bErCtiDuaJcBl63jtc43sVALqsQQhwn4VYIcc4YmkYZl0HZWIOAqTgWMDnmC+I3rWIR\nyCLLWOlauEKAXTUgssJQuDSshcJS9iQZ4Qkwjk9skFvVy8h9HS+tZejg1g27S0GoNdalg1vXc8wa\nVhyuoxBCnE8SboUQ55xSdnmpCl4XFbwGPlOR5reDblHptuAMvjI0okJT0np0HUM/Ph1tWF6B8sSi\n85aliIuLIca0ABUqrZbHvrXjM3zpJ6k5KmFWCCFOTsKtEOI80/AaGt5onQuiXWQGC6fbQjjMelw6\n0W6DKMMuiZXb4CvIX6jMvoxC08Ad6j5QkAKfEmCFEOL0SbgVQhQanePdFvymIiNoku4z8ZvnpuKC\nrmm4DY0Yj2EPwnLlLIklwVIIIYo3CbdCiEKnFLh1jfIeFxW8LgKhigsZfpOsoIlpUeBW3XBZLJeh\nEe0yiHbruHU9xwAsCbNCCFGySLgtRfQCfjR6NpimdV73J4o/pezJI8q6DcqG6ugGLIuApfCbiqBl\nz5oWVFa2Oet1TcOta7gMe9rYvCYrkDArhBAlm4TbUiQ93SfzbItiya3ruHWIifiOpeXye9rp9pUV\nQghRcki4FUIUSxJahRBC5EYv7AMQQgghhBDibJFwK4QQQgghSgwJt0IIIYQQosSQcCuEEEIIIUoM\nCbdCCCGEEKLEkHArhBBCCCFKDAm3QgghhBCixJBwK4QQQgghSgyZxEEIUWrld5YzIYQQxYeEWyFE\niadp4DcVQUvhtyyCocempbAi0qymaRiahkvXMAwNt67h0jQM3X6saRJ+hRCiqJNwK4QokSwUWUFF\nZsAkIxAkaIJSioJmU41w6AW3SyfKpeNx6bg1Dbeho0vgFUKIIkXCrRCixFAKMoIm6X6TjICJaZ15\n6lTYodhSEPCbZPhNAHTNDr2eUOD1unRcmoZHAq8QQhQqCbdCiGJN0yDLtEj3mxzLChK0Ct46ezos\nO/WS6TfJPCHwuo3jLbwxviBBpXDrdgdfCb1CCHFuSbgtRcqU8Rb2IZw207TIyPAX9mGIIkQBGQGT\no74gWQGTs9BIe8bCgde0TLICduA13S7+/CsLQwePYeBx2a27bt3uy+vSpC+vEEKcTRJuSxHLUqhi\n+hPUMKRqncjujww/aVnB89JKeyYUYCmFZULADJIe+h1N00DjeF9er0vHHQq9bt0e1FZM/7sKIUSh\nknArhCieFEU+2J6MUqDI2ZdX00DXNAwdvIaBx6XjNXTcoeoNEniFEOLkJNwKIUQRohSYSmFa4A8G\nwXe8YoPLsAOv163j1XU8Lrt0mQReIYQ4TsLteZSUlMT8+fOpVatWYR9KkbRixbssW/YOL764sLAP\nRZQAc7Y9TMA6dT9tt+5hVL1J5+GITl+4YoM/aAfeNN/xFl6XoRPt0oly2+XJPNKFRwhRykm4LWW2\nbv2Sp59+kp9//hmXy6BGjZrcc88ErrrqqsI+NMBunRLibMhPsC3IckXN8RZeE1/AhEy7WoOhaXhc\nodbdiD68IqfwtxvLgqBSmKGSb1bEY5Xm41BGwKmRrLBb0gn9q+uaUyXD0O0+1Dr2a/ZzGTAoxPkm\n4fYc2bp1K4899hjp6elomsb48eOzvf/SSy+xatUqTNPE4/HwwAMPkJCQQFZWFhMmTAiFTxc1a9Zk\n9uzZ/PLLL0ycOJGsrCxM06RPnz7ceuutBTqm9PR07r57FJMn30+HDh0JBAJs3folHo/7bJ66EKKQ\nhINZwG8PXHMmoNDB6wr337UDV2mpx6tp9nUJz0hnovCbCtMMz1Zn2SE21Ac6x/XwujmSFSjYPkP7\nDf+y7jI03Lrdb9qla7h0+9q7pFqGEOeEhNtz4MiRI4waNYpnnnmG+vXro5QiLS0t2zK9evVywumm\nTZuYNm0ab7zxBh9//DFpaWmsXLkSwFnvtddeo02bNgwfPjzb6wXx22+/omkaHTt2AsDj8dCsWXPn\n/eXLl7Fo0UIOHz7E1VfXZfLk+4mPjwfg559/Ytasx9i+/Tvcbjc33zyAW2+9nUAgwJNPPsGaNf9F\n06B9+46MGTMWt9vNF198zpQpExkw4BZefvlFDMPFiBGj6NGjp3Odpk2bwpdffkGNGjVp0eLaAp+T\nECJvzgQUoUoN+OzXwy2NLt2uyesx7O4NrtBMbJEtjlB0g1f4+OwWbIUZas0OWoqgqQiYiqBl2bWP\n8wqv50CoIpxz4UxL4cPKcex6RPj1GjqVYjzn/uCEKAUk3J4DX331FbVq1aJ+/fqA/UOkXLly2Zb5\n5ptvmD9/PkeOHEHTNH777TcAateuzc6dO5kxYwZNmjShbdu2ADRp0oRHH30Uv99Ps2bNaN68Obk5\nevQoR48eJS4uDtMMj77WiI2NpXr1Gui6wbRpU+jYsTP16tUjNtY+rg8//ICXX36Rp56aQ7Vql/LS\nSy8wadIEXnrpFTIyMhg+fBiDBg3mqadSCAYD7Ny5E4Dnn5/P//73LW+8sRSAsWNH8/zz8xk+fAQA\nhw4dIj09nfffX8vmzRsZP/7vtGuXRGxsLDNn/oOoqGj++98P2b17FyNG3MnFF19yFu+EECI3x+vx\nKnzB46ErR4ujrqHrGm5NwzA0ZxpiXdPsj+RDAdgua3a8vJlGxEf3WuTWI9nBT4WqXijnVeW0pKLA\nCr1mWhFdBkw7yAYtCzM0aYfTbaCIBvEThbuVgB1+g6aiUkwhH5QQRUxqaqqTZcLKlSuXI1OdSEYe\nnAOnqiUbCAQYM2YMU6ZMYcWKFTz//PP4/Xa/v2rVqrFq1SquvfZaNm7cSM+ePfH7/XTs2JElS5ZQ\nvXp1FixYkKObQ9jChQtJTk7m/fffZ+HChSxcuJB33nkHgDJlyvDiiy+jaTr/+Md0kpPbMm7cGA4f\nPsQ77yzl1ltvDwVgnVtvvZ3vv9/Bvn372LBhHRdeeCEDBtyC2+0mOjqGq6+uC8Dq1asYOvROKlSo\nQIUKFRg69E5WrVrpHI/L5eKOO4ZhGAYtW7YiJiaG3377Fcuy+OCDNdx11wi8Xi+XX16L7t17nI3L\nL4Q4TXZNXjtAqlALqD9okWWaZAYs/EELX9DCb1oELEXACj02FQFLEbTsj/8DlsJvKQLK/tdvEfo3\n8g/OMva69r+BUIurvQ17+76ghT9oB/GsoIXPtPAFTYIRgdcJxEKIEmPAgAEkJydn+7Nw4akHnUvL\n7TnQsGFDpkyZwtdff039+vWxLItjx4457/t8PizLokqVKgAsXrzYeW///v2UL1+e5ORkrr32Wtq0\nacORI0fIzMykWrVq9OrVi0svvZRJk3If3T1o0CB69+6do+U2rEaNmjzwwHTA7qYwZcokHn/8UVJT\nU3n88UeYPftxwA7omqZx4MB+9u3bR7Vq1XLd3x9/HKRq1XjneXx8PAcPHnSeV6hQAV0//jtUVFQU\nGRkZ/Pnnn1iWReXKVSLWvYitW788xdUVQpwN4Y/FdY1Q1wQdV7hPqHZ8QJTOueyeUPCBbpHHYnG8\nO4IV0ZIbNO0AHTQtZxKNohh8I1vKXTLoT4gcFi9enGvL7alIuD0HypcvT0pKCjNnziQjIwPDMLj3\n3nudkFm2bFlGjx5N3759ufjii2nVqpWz7vfff8+sWbMAsCyLYcOGUalSJebNm8eKFStwu91omsaU\nKVNy3Xd+muvDqlevQffuPXj77beoWrUqQ4bcQefOXXMsl5q6l/ffX53rNipVqkxq6l4uu+yy0LKp\nVKpU6ZT7jouLQ9d19u/fR/XqNQDYty81X8cthMi/yEFlHsPA69LwuHQM7fhMaHDy4FqUgmHkseiE\nuk1EvBIWHqQV7oMb7sYQdFqHz31f3OPdNbTQALLQoLJQFQtX6L6EB5gVpessRFEQHvdTUBJuz5EG\nDRrw+uuvZ3tt7dq1zuPbb7+d22+/3Xk+dOhQAFq3bk3r1q1zbG/YsGEMGzbsjI7p119/YcOG9XTs\n2InKlauwb98+3n9/NfXq1ee661oxd+4crryyNpdddjlpaWl8+ukm2rfvSKtWbXjiiVksWbKYG27o\nTyAQYOfOn6lb9xo6derMCy8s4KqrrgZgwYJ5dO3a/ZTHous6ycntmTfvWe6//0H27t3DypX/4qKL\nLj6jcxSitAuXA/O6Dbwuu0X2ZOXASmqgCp+XEeovbMsZfsMlwEzr+MC0yH69UYY9NfKJATjcv5jw\njHLhP3poQJ6uYRDxXDv5LxEl9T4IURgk3JYiMTFl+Oabb3j11Vc4duwYsbGxtG7dhjFjxhETE0NG\nRgb33Xcv+/bto2zZsjRv3pz27TsSExPDs8/O49FH/8m8ec/i8Xj5298GULfuNQwZMpT09HRuvPEG\nNE2jQ4eODBkyNM9jiOwice+9E3nggal07JhMjRo16NGjF59//tn5uBRClAjhrgVuQyfabYcwj26X\nnRInFw6TLs1uQc1rBEqlC2KIsczc38xle6f7vhDi7NHUqUY/iRIjI8N/ysFuRZVh6KSlZRX2YRSK\nSpViOXiw4KXfSrqD6X6O+oJ5vl9UZii74IIyHD6cfsbbiZyCNyo0Ba/HsCdqkI+0zx35/1d8yb0r\nvqZv3bsAACAASURBVHRdo2LFsqe9vrTcCiFKpKI+pe7JhFtkDR28RnjyBbu/plvXcgRZCbZCCHGc\nhFshRPFUAgaXh1tjNQ27FdZlB1iProdmtZIgK4QQBSXhVghRLMVFuXHpGmlZQQJW0U58kSPm3dla\nYXV7ljA99ylYJcgKIUTBSbgVQhRLLl0jLspNhSg3GUGTo1lBsgImhZ1znaltDZ0ol86FZTxEB02M\nk4RYkCArhBBni4RbIUSxpgFlXAZlYw18pkV6wOKYL0DAPPeF+8NB1uvSnT8uze5WoOt2YL0gxoOZ\n7nPWkRArhBDnloRbIUSJoBR4dB2PVyfO6yLLtMgMWqT7TYKmhWUpTjdXRs4k5XXZLbIel447VIYr\nt2oFEmKFEKJwSLgtRXRdo7iOwjFNq7APQRQzUYZOlKFzQZSLoKXwmwqfaREwLfymwrSsXLswaNhF\n98OzSR3vGwtuXZe+sUIIUcRJuC1F0tN9WIXdIVGI80wpe5aqaJdGtMuu1B+eS8QKTb0apimN8PwH\nMpOUEEIUTxJuhRClTjig2p9lRHyaIZMhCCFEsSdzNAohhBBCiBJDwq0QQgghhCgxJNwKIYQQQogS\nQ8KtEEIIIYQoMSTcCiGEEEKIEkPCrRBCCCGEKDEk3AohhBBCiBJDwq0QQgghhCgxJNwKIYQQQogS\nQ2YoE0KUWgrICJhkBS10DaJdBtFuXWYpE0KIYkzCrRCiVAooiwPHAvgCJuEsq2kBYjwuLoxx49K0\nk64vhBCiaJJwK4QodXymxb40H0ErexOtUpDuCxIwTSqXjcKrn52AaypFwFSggVvXMCQ4CyHEOSPh\nVghRqvgslWuwjeQPKvYdzaJqrBevcfpDEwKWReqRLHYfycIK9XXQNY1Yr4vyXheusxSehRBCHCfh\nthQpU8Zb2IdwzpmmRUaGv7APQxRRQUtx4FjWSYNt5LKpaT6qlPUS7SpowFUc8Zv8mRGgvG5gRuzP\nVIq/MgMc8wW5sIyHMm6jgNsWQghxMhJuSxHLUqgSPlLGOINWNlGyKeBAuh9/MP//B8xQK2/lsvkP\noaZS/JEZID0ryMn2FLQU+4/5iIvxEOc1AGnFFUKIs0GSgBCiFFAczPCTGTALvKalFAeO+fjTF4ST\nxlXIDFr8P3v3HR9Ftfdx/DOzNQlJILSANOVRAyLKBaQjolQJTZoiBJDgVSnKvdKrINVKEZUmKgpI\nF0VFQMECXkQQleLlCggChpq6deb5Y7NDeoE0kt/7ZWR3dubM2clm97tnzpzzV5yD+GyCrVErHS4n\nuLiQlLP1hRBCZE9aboUQxZqiwMUkD3EOz3WXoelwKcGFw20iLMCC3XxtuDBF8V2gFuv0Eutw53oY\nMR24muTGo+lUCLKgSguuEELcEAm3BSgiIoKffvqJgICAAt1WiJJKUeCSw8OVJDcA83+egVvLvk+2\nRbUyrO64dMsTXV4cbi8Ws8noh+vyajjcmnHB2PVKcHo4mxxwLaqcVBNCiOsl76AFSLmB4X9uZNu8\nMHnyRBYtWliodRAid3QuJbm5nOgyWlNzEmyzW0/Twen2ciXJzZUkN4ku7w0HWz+H28tfsU4S3F5k\ntDAhhLg+0nKbj7744gteffVVSpcuTcuWLY3lBw8e5OWXXyYhIQGA4cOHc//99wOwc+dOFixYgMfj\nwWQyMWvWLO644w7jQjBd15k1axYXLlxg1qxZWCyWgn9iQhRxGjoXEt057vtalPgvNEu0mSljt8hw\nYUIIkUsSbvPJpUuXmDhxImvWrKF69eosWbIEgNjYWKZMmcLixYspV64cMTEx9OjRg08++YSYmBgm\nTpzIhx9+SNWqVXG73bjdvtOpiqLgcDgYPXo0VapU4eWXXy7MpydEkZXg9nI5yY3ToxV2Va6brkOs\nw0OCy0spm5lgq+mGxtsVQoiSRN4t88mBAweoU6cO1atXB6B3797ous6vv/7K6dOniY6OpmvXrkRH\nR2MymTh58iTfffcd999/P1WrVgXAYrEQGBgI+Fpso6OjqVevHqNGjcp0v7GxsZw+fZqEhARiY2OJ\njY0lLi4OgH/8oy6nT5821k3Z1eDHH/fRoUMb3n//XR56qBXt2j3E5s2bMtxHQkICQ4Y8wdy5s41y\nZs2awfDhQ2nRoglRUY9z5sy1/Rw8eIB+/R7j/vub0b//Yxw8eBCAffv+Q69ejxjr/fOf0fTv/5hx\nf9CgKL7+eicAnTq15733VtC7dw/uv78ZY8eOMoK/KD4UJfc/Xl3H4dG47PRwOtbB+TjnTR1sU/Jq\nOleT3JyJdXDqShLnE1xcdnqId3tJ9Gg4vL4fl1fDrWm4NR2vrqOhp5hSWH5u5CflMRRCFKyzZ89y\n+vTpVD+xsbHZbictt/kk7Xiyuq4b/WYjIiJ477330m1z4MCBLMts1KgRu3fvpk+fPpleWLZixQoW\nLFjAzJkzOXPmDAAhISFERUVl22/34sWLJCQk8Pnn29mz5zuef/5fPPBAa4KDg411rl69yrBhT9Gk\nSTOeeuoZY/kXX3zGggVvEhERwcSJ41m4cD4zZswmNjaWESOGMnr0WNq168C2bZ8zYsQzbN78KXXr\n3sPp039y9epVSpUqxf/+dxxVVUlKSkRVTRw5cph//KO+sY9t277gjTfewmq1MGBAfzZv3sQjj/TI\n8jmJm4cOxuQKSvKIARm9ZDVdx6v7/nV5fIHO5dFwem78oq6iStfBret4nB4SXb4zOSZFwaQqycFL\nQVUU1BRhTMU3cq6qXntMVZTkY6oYYzKkPcRGoEvzO1CUDJal2Sa3z8m4nWKZP5b7Hzf+TdPBJKPX\nSEZ1zKiMlGXr6Gi6vw56utEu4p1ukpK/LNlMMpaFEAWpb9++RpbxGzp0KMOGDctyOwm3+aRevXpM\nmDCBU6dOUa1aNT766CMA7rrrLk6cOMHevXtp1KgRAIcOHeLuu++mefPmLFq0yNjG5XLh8XiM1tuh\nQ4fy/vvvEx0dzZtvvkmpUqXS7TcqKopu3bpRpkwZvF7fmJ7+UJvdBA5ms5no6CdRVZVmzVoQGBjI\nyZMnqFPnbgD+/vtvoqMH0rlzVx5/vH+qbR944EFq164NQMeOHXnlFV+3id27d1GtWnU6dHgYgHbt\nOvDhhx+wa9dXdOrUmdq172L//h8pV64c//d/dxASEsKBAwewWCxUq1ad4OAQYx+PPdaXsmXLAtCy\n5f0cO3Ykp78OcRNQAHMOUpKqKJiTA1pA8ql6RfGNWpDo0bia5MbtLT4hV1HAalIJspmxm1TMKpgV\nlbQDKuR7rteN/6UIi3kvXehOfyNzGdQx+7KVLPdRymYhSbqECFEoVq5caWQZv5CQkEzWvkbCbT4J\nCwtj2rRpPPnkk5QuXZoOHToAvl/KokWLmD17NjNnzsTlclGtWjXefPNNqlevzvTp03n22Wfxer2Y\nTCZmz57N7bffbgTU6Oho7HY7gwYNYsmSJel+ySEhITn6xWekdOnSqCk+Me12O4mJicb9b77ZRWBg\nUIatpeXKlUuxXQBJSb7tYmL+plKlyqnWrVSpEn///TcA//hHffbt+4EKFSrSoEEDQkJC2LfvP1it\nVurXr59qu7CwsqnqduFCzHU9T1H86DpYVJVQq0qw1cTV5OG/cjDLbpGlKBBkM1PaZsFuVtKFtWLa\nSC2EEIZKlSpd13YSbvPRQw89xEMPPWTc79/f19pZp06dDLslALRq1YpWrVqlW3748GHjdr9+/ejX\nr1+u62O323E4koz7Fy9eIDw8PMfbd+/eg9jYWIYOfZoFCxblaMzd8uUrsGPHl6mWnTt3jmbNmgNQ\nv34DXnnlJSpVqsTAgU8QHBzMtGlTsVqt9OrVJ8d1E8JPRaGM3YLdbOLveKfR1eFmYjOrlAu0EmDx\nTRYhQVYIIXJOzrWUIBERtdi69VM0TePbb79h//4fc13G6NFjqVGjBiNGDMXpdGa7fvPmLTh16hSf\nf74Vr9fL559/xh9//I8WLXxDn9Wtew8nT57g119/4a676nDbbTU5e/YvfvnlUKr+tkLkVoBZJTzE\njiWfhtLKj4uMFCDEbuaWEHuqWdCEEELknITbEuTf/x7Frl1f0apVcz7/fCsPPNA6y/UzuwBtwoTJ\nhIeHM3LkiGxHLAgNDeX11xfw7rsraN26Je+9t4LXX19IaGgoAAEBAdSqVZuaNf8Ps9l3IqFu3Xuo\nXLkyZcqUybYuQmTFpipUDLEbY8VaVGuOtstsPQWwW0xUCLZROcRO5RA74SE2gqxmbjRDKwqEBVkp\nH2iRi5aEEOIGKHp2VxmJYiMx0ZXtRWU3O5NJJS7OUdjVyFPlywcTExNX2NW4qTk8GmfjnDc0moJZ\nVSgXZCXI4h+H4BpF8e0jJtGN05364oewsCAuXUrIsmxVUahQykqQxXTd9RP5Q/7+bl7yu7t5qapC\n2bLpL5rP8fZ5WBchhCiS7GaVCqWs192NwGYxUSnElhw+0xei675hom4JtlEm0JKr/ZhVhUrBNgm2\nQgiRRyTcCiFKhCCLiTIBuT/lH2AxUamUFWvasbcyoABhdgsVS9lyNG2uzaxSKcSG3SxvxUIIkVdk\ntAQhRIlRxm7Go+nEOjw5Wj/QZqJikJXcDt0fZDFhC7VzMcmd4ZaqAqVsZsoGWnJdthBCiKxJuBVC\nlCAK5QKteHVIcGYdcINsZioGWa87epoVhfAgK/ZgO+7E5CmBFbCbTQRbzRmOXSuEEOLGSbgVQpQo\nClAxyEqMAvEOT7pZthQFQu0Wwq6jC0Nauu5roS0feK2/b9ppZYUQQuQtCbcliKoq5GgKy5uY16sV\ndhXETUABKgRasJlVria58Wg6CmAxmyhjN1PKasrz8ClhVgghCoaE2xIkIcGJdhPO1iRE/lAItZoJ\nsZpxejVMim8KX5AgKoQQNzMJt0KIEk0B7CYZrUAIIYoLeUcXQgghhBDFhoRbIYQQQghRbEi4FUII\nIYQQxYaEWyGEEEIIUWxIuBVCCCGEEMWGhFshhBBCCFFsSLgVQgghhBDFhoRbIYQQQghRbEi4FUII\nIYQQxYbMUCaEENfJo+s4PBqqohBgVlEKu0JCCCEk3AohxPVIcHu5kODCo+kogM1iokKQBYsqJ8SE\nEKIwybuwEELkUoLby9/xTjyaDoAOONxezqVYJoQQonBIuBVCiFxwenViElxklGFdHp3zCS40JOAK\nIURhkXArhBA55NV1YhKceLNonXW4vVxIdIMEXCGEKBTS57YECQqyFXYVihWvVyMx0VXY1RAFRudC\nkhunR8t2zXiHB6tJpbRN3mKFEKKgyTtvCaJpOrourUl5xWSSEx8lyRWnlwSHJ0fr6sDlRBc2eY0I\nIUSBk3deIYTIRqJH43KiK1cdDTQd/o53kuTKWSAWQgiRN6TlVgghsuD06sTEO3n94AzcWvbdUCyq\nlWF1xwHg0XT+inVg8+rYTDIKrhBCFIRi23K7YcMGhg8fnidlRUREkJSUlKN14+LiWLJkSZ7s92Y2\nZMgTbNy4obCrIcQNSfJonItz4NH0HAVbIN16Xh3OxjmIc3uRi8yEECL/FdtwC6AoN9ZS4vV6c13O\n1atXi3S4/emn/Qwc2J+WLZvRunVLBg2K4rfffruhMt96axETJ47LoxrmrR9/3EeHDm0KuxriJqOh\nc9nhNoLtjfJqOjFxTk7HuYh1eXBqGh5dx6vruDUdl1fD4dVwJv/r1nyDid3gW5gQQpRIRbZbws8/\n/8xLL71EQkICAMOHD+f//u//eOSRR+jVqxe7d+/G6XQyd+5cVq1axcGDBwkICOCNN96gbNmygK8V\ndfjw4Zw8eZIyZcowZ84cKlSowLFjx5g6dSpJSUm4XC569epF//79ARg7dixBQUGcOHGCy5cvs27d\nOuMiLF3XmTVrFhcuXGDWrFlYLJZ09Z42bRrx8fF069YNu93Ohx9+yMmTJ5k8eTKXLl3CbDbz3HPP\n0aJFC1avXs3Ro0eZNGkSP//8M7169WLt2rXUqVOHqVOnUrt2bXr27ElERATPPfcc27Zt4+rVq4wa\nNYo2bXIf2BISEnj22WGMHz+JNm3a4na7+emn/Vit6Z9HcaHr+g1/yRHFg9OrE+t0o6BgUkFVFcyq\nggmF5P980+m6NeKdnjyfjEEHnG4vMW4vigIKCooCvrcXPVWbroLvS7XNrBJgMWE3q9jz4OI0/5+C\nrpNi30IIUbwUyXAbFxfH5MmTWbx4MeXKlSMmJoYePXrw1ltvceXKFRo0aMDIkSNZunQpAwYM4P33\n32fatGlMnTqV999/nxEjRgCwf/9+Nm3aRPXq1VmwYAHTp09n3rx5VKlShXfeeQeLxUJiYiI9e/ak\nefPm3HbbbQAcOHCAlStXYrP5hs5SFAWHw8Ho0aOpUqUKL7/8cqZ1nzRpEj169GDDhmun5J9//nn6\n9OlD9+7dOX78OH379mXr1q00adKEFStWALBnzx7q1avH999/T506dfj+++954oknjDKCg4NZu3Yt\n+/fv59lnn8003MbGxhIbG0uZMmVStTwHBwdz8uQJFEWhbdt2AFitVho1agz4QuDSpYvZsGE9LpeT\npk2bMWqUL+j/+OM+JkwYy9at24z9dOrUnkmTpuLxeFi2zNdSvXPnDqpWrcaHH64B4OzZvxg0KIrf\nfz9G3br3MGPGbEJDQwEYPfrf/PTTfpxOJ3fccSdjx47ntttqAjB58kTsdjt//XWGn37azx133Mnc\nua+wfPlStmzZTNmy5Zg5czZ33HGnUZdHHunJJ59s4eLFC7Rq1Zpx4ybg8XgYPvwZ3G43zZs3RlEU\nNmz4mNDQUF577RW+/HIbigIPPdSWESOew2KxGM+1b99+vPPOMkwmM888M4zOnbtk86oVRZ1H04jN\nYLQDxfhfwYU9XU+Os5nsT09eKdHlJdHlC8MWVSHAaibAomJVFcyqipqm3inDq79V2KvruDwaLq+O\nW9PQNB1Nh1I2M+UCiu8XWyHEze/s2bNGlvELCQkhJCQky+2KZLeE/fv3c/r0aaKjo+natSvR0dGY\nTCY8Hg9BQUG0bNkSgNq1axMeHs6dd/pCzl133cWpU6eMcurXr0/16tUB6NmzJ3v37gUgKSmJcePG\nERkZyaOPPkpMTAxHjhwxtmvXrp0RbMEX/KKjo6lXrx6jRo3K1XNJSEjgyJEjdO/eHYCaNWtSq1Yt\nDh48SLVq1XA4HJw/f57vv/+ef/3rX3z//fecO3cOt9tNlSpVjHI6duwIwL333ktMTAwuV8b9/1as\nWMGDDz7I559/zooVK1ixYgXr168HoHr1GqiqicmTJ/Dtt98QFxdrbLdp00a2bPmYxYuXsXnzpyQk\nJDBr1ovG45m1fjZt2oxBgwbTtm07vvlmjxFsAT77bCtTp05n+/avcbvdvPvuO8ZjzZq1YNOmT/jy\ny6+IiKjF+PFjU5X75ZdfMHTocHbu3I3FYmHAgMepXfsudu7czYMPPsRLL81Jtf7WrZ+yaNFbbN78\nCSdPnmDJkrcJCAhg/vw3KF++PN98s4fdu7+nXLlyLFnyNr/++gurV69l1aq1/PrrLyxZ8rZR1sWL\nF0lISODzz7czadJkZs16kbi4uAyfv7j56SSHzSLciqnr4PLqXE1ycz7WyV9xTs7EOTmX4OJikpur\nTg9XnR4uJbk5n+DmrzjfOufinJyLdXIp0U2804PTreH26ng1HV2mCRZCFHF9+/blwQcfTPXjbxTM\nSpFsuQXfRVzvvfdeqmVnzpzBarUa900mU6oQ6g/AmfEHtFdeeYXy5cszZ84cFEXhiSeeSBUWAwMD\n023bqFEjdu/eTZ8+fQgICMjx88hsXFl/XRo3bsxXX33FxYsXadCgATExMXz11Vc0btw41br+56mq\nvu8jab/J+EVFRdGtW7d0LbcAQUFBLFv2Du+8s5wXX3yBCxcu0Lx5CyZMmMRnn33K44/3o3LlygAM\nGzaCXr0eYerU6Tl+rml17tyFqlWrAtCmTVt27fo61WN+Q4Y8yQcfvE9CQgJBQUEAPPDAg9x5Z4Rx\ne+3aNXTs+DAAbdu2Y82aVan21afPo5QvXwGAJ54YzJw5s3nqqWcyrNfWrZ8yZsw4Spcunbz/fzJj\nxjRjfbPZTHT0k6iqSrNmLQgMDOTkyRPUqXP3dR8LUXQVRsttbuVFy63bq+NK0XKrqNJdRwhRtK1c\nuTLDltvsFMlwW69ePU6cOMHevXtp1KgRAIcOHaJMmTK5moRg//79nDp1imrVqrFu3TqjrLi4OCIi\nIlAUhWPHjrFv3z4iIyOzLGvo0KG8//77REdH8+abb1KqVKkM1ytVqhQOhwOv14vJZKJUqVLUqlWL\nDRs20K1bN44fP87Ro0epW7cu4Au3r732mtEaXa9ePd5++21GjhxplJn2OWd1DLJrrq9R41amTHkB\ngJMnTzBhwjheemkOFy5coFKlSsZ6lSpVxuPxcPHixSyPS1bKli1n3Lbb7SQmJgKgaRoLFszjyy+3\nceXKFV//Q0XhypXLRrj195v2bWsjLCzMuG+zXSvLr2LFiqnqHhPzd6b1unAhhvDwlM+1EjExMcb9\n0qVLG18i0tZd3LzMqkqI3VxofW5Tyqs+t2nfClLeNykKpuThx4LMJmO//vWkz60QoqhLmUtyo0iG\n25CQEBYtWsTs2bOZOXMmLpeLatWqMX78+FxdHNSwYUPmzZvH77//blxQBvDUU08xatQoNm/eTLVq\n1WjYsGGW5fj3GR0djd1uZ9CgQSxZsiTDEBkaGkpkZCSRkZGEhoby4YcfMnfuXCZNmsTy5csxm83M\nnTuXMmXKAL5we/bsWZo2bQpAkyZN+Oijj9K13GZUnxtVvXoNOnXqzLp1H1G+fHnOnj1rPHb27F+Y\nzWbKli1LTMzfOBwO4zGv18vly5evuz6ffvoJu3Z9zVtvLaFSpUrExcXRqlXzG/qgPXfufKq6+1tx\nM1K+fAXOnv3L6GN99uxZypcvf/07FzcFm0mhfKA163XwBcEyAWauOjxcSXKTVxlXAawWEyE2Ezaz\niklRUPBN9qDrOlryOjpgUjBaZvMygKYsS4KtEKK4KpLhFqBOnTrpuiUAfP/998bt++67j7Vr1xr3\nu3XrRrdu3dLdTqtWrVp8/PHHGT42c+bMdMsOHz5s3O7Xrx/9+vXLsu4vvPBCqvvVqlXjnXfeyXDd\nChUqpCq/Q4cOdOjQIdP9Z3Q/p06c+IPdu3fRtm07KlSoyLlz5/j8863UrXsPderczYoVy2jatBml\nS5dh4cL5tGvXHlVVqVatOk6nk2+/3U2jRk1YunQxbrfbKDcsrCx79+7J8cgESUmJWK0WQkJCSEpK\nZMGC1284sK9Zs4oWLVpgs9lZtmwp7dq1B3wtwFeuXCE+Pt5obW/Xrj1Lly6mdu27AFi8+C06dux0\nQ/sXxYuKQhm7BbvZxN/xzhtuxTWpCmWDrARbVIw+EP7HUvaLSEMCqBBC5F6RDbci7wUGBnHo0CHe\nf/9d4uPjCQ4OpmXL+xkxYiQBAQFcuBDD4MEDcblcNG3ajOefHwP4ulqMHTueqVOnoOsaUVEDU3UD\naNOmLZ9+uoUHHmjBLbdUYeXKVZlVAYBOnSL5/vvvaN/+IUJDQ3nqqaGsW7c2y22y06FDR55++p9c\nuBBDq1ateeKJaMDXDaN9+w507twRTdNYu3YjgwcPISEhgd69e6AoCm3atGXw4CGZli1DiZVcAWaV\n8GA75+IcWFRrjmcoS8mkQKVgu8xQJoQQBUTRc9OJVRgmT57MwYMHjeCj6zpmszlVS3JRk5joylWf\n5ZuFf1iy++5rVKD7NZlU4uIc2a94g8qXDyYmRkZrKEyJHo3zcY5cd1Ewqwp3VQ8j/mrOZjgURY/8\n/d285Hd381JVhbJlM762KSek5fY6TZ06tbCrIIQoIIFmlTKBVi4luHI8ga6qQIVSNgKsZuLztXZC\nCCFSKpLj3AqRG9JtQBSE0jYTQfactQcoQJlAKwFmeYsVQoiCJi234qb38cdbC7sKokRQKBdgwe3R\ncHq0LNcsZTdT2mYqoHoJIYRISZoVhBAih0yKQvkgG6YsJkCwW0yUC7SQ2QgIQggh8peEWyGEyAWb\nSaF8kJWM8q3VrFAxyIoqwVYIIQqNhFshhMilIIuJCqVsmJMTroKvxTY8xTIhhBCFQ/rcliCqqiCn\nSvOO15t1v0tRvAVZTNhC7Tg8GqqiEGCW9lohhCgKJNyWIAkJTrS8mktUCIFZUShlkQvHhBCiKJFu\nCUIIIYQQotiQcCuEEEIIIYoNCbdCCCGEEKLYkHArhBBCCCGKDQm3QgghhBCi2JBwK4QQQgghig0J\nt0IIIYQQotiQcCuEEEIIIYoNCbdCCCGEEKLYkBnKhBAiG4oCmg4eTUfXdVJOvKwAqgImVUFFQVFA\nl4kAhRCi0Ei4FUKINNyahkvTcXo0XB4dt+bFq/lDq07a7KoAiqKgACaTis2kYDGpWE0qDre3wOsv\nhBAlmYRbIUSJpii+FlmHRyPJo5Ho8uDVdLRctL76Mq9vA4/mxem+9pjLbCL2ahI2swm7RcWmqljN\nCiZFkRZeIYTIBxJuhRAljqKAOznQxju9ODxevLlJs7mgA26vjtvrId7p27eqKFjNKgEWE3aTik3C\nrhBC5BkJt0KIEkNDJ8mjEefI30CbFV0Hr66T5PKS5PL6+uyqClaTSoBVwq4QQtwoCbclSFCQrbCr\nkI7Xq5GY6CrsaohiTAdfC63LQ4KrcAJtVnTAq+kkaV6S3NfCrsWkEmhRsZlNWFQFq0nCrhBC5ISE\n2xJES77SuygxmWQ0OpH3FAWSPBpJbi/xLg9ur37TBEN/2PVq3uSL0dyoioLZBHazCZvZd6Ga1SSt\nu0IIkREJt0KIYkFRwOHVSHJrxDs9uL1ari4KK8o0XcflAZfHA1zrt2s2KdjNJqwmBauqYjYpWFQJ\nvEKIkk3CbQH68ssveeWVV7Db7bzyyivUqFGjsKskxE1NB1xejUSPRsJNEGjn/zwDt5Z9NxyLUbqZ\nIQAAIABJREFUamVY3XGZPu7vt+vVdJxu36i7/sCrKmA1mbCZfV0bzIqCSVUwq77HJPgKIYo7CbcF\naPXq1YwYMYJ27drleBtN01BVOXUvhI+OW9NxenWS3L4Lsjz6zdPlICfBNjfrpWQEXsDt9ZCQXISi\ngJI8uYQ5uS+vxXQt8JqSHzOluK0oqcsVQoibiYTbAjJz5kz27dvHiRMn+OCDDyhfvjwnTpzA5XJR\nvXp1ZsyYQXBwMD/88AMzZsygQYMG/PLLLzz11FPUr1+fWbNmcezYMZxOJ40aNWLs2LEoKT+BcsDt\ndjNjxjT27t1LXFwsVapU5ZlnhtGsWXMAHA4Hr776Etu2bcPr9XDHHXeyePGyDMuZOXN6puX89ddf\nREZ2IDAwEF3XURSFqKiBDB485MYPpCgx/OPPejQdp1fD6dZweL14vLkbg7ak0/XkaSd0X19ep0dL\n9bhvAgrfLV8QBlUFRVF9M68pCirJk1QoGOuQPGmF/23I/27kf1+6dh/Qr01yoaR40D+7G/half1B\nXPVvlOZ5CCFETki4LSBjx47lt99+Y/Dgwdx///1cuXKF0qVLA/Daa6+xePFiRo4cCcDvv//OCy+8\nwIQJEwCYMGEC9913H9OnT0fXdf7973+zdu1aevbsmas6eDwewsMrsXTpO4SHh7N79y7GjHmeNWvW\nU6lSJaZNm4qua2zYsJmQkBCOHj1yXeWA74Ns167vch3ARcnhf2n4T697NfDoOm6vhsur4/L6ZgXT\nbqKW2ZuRzrWZ14w8qQHk/cxqSrob/ptKqtZif2hWFQVF8V04pyj4QnZy94q0ATvlLHHJD2Uq5etJ\nN5Zl/CIzJbqIdXlShXZ/9lb9+/PXLblbiJqmBVxev0IULAm3hWTDhg18/PHHuN1uHA5Hqv631atX\np27dusb9HTt2cOjQIZYt87WiOhwOwsPDMyw3NjaW2NhYypQpg9fr+3BSFIXg4GACAgIYMuSfxrot\nWrSkcuVbOHz4N1wuJ7t37+Kzz7YRGBgIQERErQz3kVU5/nCr6zqapmEyma7j6BRv15P3FUXJ8UgX\nSgatXnnNXxUjGOBrTdV13+9eV/y3feFUI/lff4jVdDzJrxHNv42e37UWhU1Pd8N/M7NffuG/IrxW\nC5fiM+4mkjKsZxTSVQUCLCYqBFkl4ApxHc6ePWtkGb+QkBBCQkKy3E7CbSHYt28fq1atYvXq1ZQu\nXZotW7awZs0a43F/uExp4cKFVKlSJduyV6xYwYIFC5g5cyZnzpwBfC+EqKiodOtevHiRP/88Rc2a\nNTl06BDh4eEsWrSQTz7ZQvny5Rky5J88+OBD2e7z4sWLnDp1kpo1axrLFEWhU6f2KIrCffc15tln\nRxot1Wn5gw/4TklmJiehMCfrZPYhcy2wXQtaaVt19OTbOhihjOTbpFzHWDdF+Vw7RZwqzCWvo/k/\n5rVr6wNcReXylcTsn1gKaY9j2sOiZNCNO+02eopgoWspl197LNXzS5FcUoYY+UwXxVX613nqF7wX\nMKnyFyDE9erbt6+RZfyGDh3KsGHDstxOwm0hiIuLIzg4mNDQUFwuF+vWrcty/datW/P2228zZcoU\nVFXl8uXLJCQkZBh2o6Ki6NatW7qW27Q8Hg8TJowlMrIz1avXYPv2Lzl+/L+0adOWL77YzsGDBxgx\nYig1a9akRo1bM62bv5zOnbtQvXoNAMqUKc17733AnXdGcPXqFWbOfJHx48ewcOGbGZZhSu7XV2Ay\n21VG50yz3eiaVK2ZOuhK6tZI/6nflOul+thL1xrqE1bKisnlznb/Rk0zOJbpwm2OS7smZb18falT\nBvZr3QeM8J/cpUDTwaNr6Lqv1RZjG+luIIo3o4+xEOK6rFy5MsOW2+xIuC1A/tDRsmVLNm/eTIcO\nHQgPD6dOnTr8/PPPmW43btw45syZQ5cuXQCw2WyMGzcuw3Cbk+Z6XdeZMGEcFouVUaPGGmVaLBYG\nDx6CoijUr9+ABg0asmfP95mG24zKAQgICKRWrdoAlCkTxujR42jbtjWJiYkZtkoXt4Dj74OnXDtX\nmfqBXAq2W3BYbu7uHSn7Hmr4Qq43Ofh6dd9FYx6PhlPT8Xi1VCFZlExpLzxLe8rfv/zaRW5GR9xU\nF7qlo6f9ApnmW2UaVlXBZlbTle3vV6vim1HOVw/FuAjPd7GcgqoWv/c4IQqKv6tjbkm4LUDvvvuu\ncfvVV1/NcJ377ruPtWvXploWGBjIlClT8qweU6dO5sqVy8yf/4bRJ/b22+8A/C1yOQtgGZWTmdz0\nGRXFT8pfvYrvA9+SNujbfKFB18GdPEqCW9NweXQcHi8ezddnV15F+e/aFzTfjYzeElKNlGCEUCVF\nIFWMx9QUZfhDnxFKjSB4LRwqpL5QLKMRFbIMsHmoXNkgSulauuXydiZE0SXhtoR58cVpnDjxB4sW\nvY3FYjGW/+Mf9QkPr8SyZUsYOPAJDh36mf37f+S55/6Vq3IAfvnlEMHBwVSrVp2rV68yd+5sGjRo\nSFBQUL4+N3Hz8wcGi+qbaSsAFaygKBY8mo7Lq/tmIXN5cXk1vDImWK6kHPPWpIJJVbEoCqbkcW/9\nLY6m5FSZcgQARQFFV4zQm5tgmV9BsKACpgRZIW4uEm5LkLNnz7J+/VpsNhtt2jwA+D64xo+fSPv2\nHXnlldd54YXJvPPOsuShwV40+tEuW7aEAwd+Yt68hdmWc+bMaRYsmMfly5cJCipF48aNmTFjVmE9\nbVEM6LovcAWYFQLMKmF2Mx5Nx+HRSPJoJLo8eLSi34fXolpzPEPZ9TKGxFLAbFKxmVJM3JA8rJZ/\ntjLIZXBLEWiL+rEWQpRcii7nikuMxERXkesaYDKpxMU5CrsaRVr58sHExMQVdjWKPKdXI8HtJcHl\nwe0tOkE3LCyIS5cS8qVs/1iw5uR+oVazilVVMatgVlWZbjcPyN/fzUt+dzcvVVUoW7bUdW8vLbdC\niGLBZlKxmVTC7BacXo14l5cEZ3KLbmFXLg8o+N7wzaqC3WLCZlaxqArW5BbZjEKsBFshREkk4VYI\nUezYTCq2AJWwAAtJHo14l4dEl/em6qPrb5W1m03YLSr25K4FZjV9kJUQK4QQ10i4FUIUWwoQaFYJ\nNFvxBugkeTTinB4cbs2YOKSoUBUwqdfCrC25i0Ha67aKWLWFEKLIkXArhCgRTIpCKYuJYKsJl1cj\n0aMR7/Ti8hRO0E0ZZgOsJt94qqYMpo4TQgiRKxJuhRAliq6DRVUJtaqUtplxef0jLmg4PN58G0tX\nVRTMJpJbZk3YVAWrhFkhhMhzEm6FECWWP+harCqhNt9saQ6PjtOr4XB7cXp80wannEY5O/6+sqoC\nVpOJsAALthC7cfGXdCsQQoj8JeG2BFHVojfTudebfuYfIQqDrvtm0go0KwSaVRS7GU0Hj6bh1cCt\n63g1HT152mBdJ9UsW+bkSRBUxTcJhf/Cr3KlbOhJLmMfQggh8peE2xIkIcGJdhNdLS5EYdJ131dB\ni6piUcF+HdsLIYQoeNLhSwghhBBCFBsSboUQQgghRLEh4VYIIYQQQhQbEm6FEEIIIUSxIeFWCCGE\nEEIUGxJuhRBCCCFEsSHhVgghhBBCFBsSboUQQgghRLEh4VYIIYQQQhQbMkOZEEIUCB0NUFCK1CTY\nZcsGoarSzpGV8uWDC7sK4jrJ765o0jSNixcT8q18CbdCCJHPnJrGhQQ3bq+GxaRSLtCKzVQ0Iq4E\nWyFEQcvv9x15VxNCiHyU6PRwNtaJw+3Fq+k43F7OxjlwerXCrpoQQhSapUvfZMOGj/KlbAm3QgiR\nTzy6zrl4J15NT7Xcq+n8He/Ek2a5EEKUFOXKlefSpYv5UraEWyGEyBc6FxLdmQZYl1cnJtFVwHUS\nQojiT/rcliBBQbYcref1aiTKh64QNyTW5SXR6cGWxd9dosvLFaeH0jZ5KxZCiLwi76gliKbp6Hr2\np0FNJmnQF+JGuHWNS4luctLp4HKimwCLCZtaNC4wE0KIm52kGCGEyEM6EBPvTtfPNjOarhOT4CIH\n3zvFTah169aMHTu2sKtRLM2fP5+IiIhUy/r160erVq0KtB7yOy56pOVWCCHy0BWHmzk/TsOtZd+1\nx6JaGVZ3HE63l0sON+UCLRJyxQ3TdZ2FCxcSERHBQw89VNjVydaePXvYt28fAwYMoFSpUjneTlGU\nAhvKbtu2bRw9epShQ4dmWA9FkTMvRUmBtdxOmDCBH3/8EYCxY8eycuXKDNdL+diqVatYsWJFQVVR\nCCFuSLzby+Ukd46CLZBqvasON1ednvyqmihBNE1jwYIFbN++vbCrkiN79uxh4cKFxMbG5mq7p59+\nmgMHDuRTrVL74osvWLhwYYaPffbZZ0ybNq1A6iFypsBabqdPn57rbfr06ZMPNSm5Vq/+kM2bN/Hf\n//5O+/YdmTLlhcKukhDFRpJHIyb++rsX6DpcTHBjUhSCLKa8rZwoUXJybUVRktv6OhwO7HY7qqpi\ntVrzqVapZVVHi8VSIHUQOZdty21ERARvvfUWPXr0oE2bNmzbti3L9b/88ksiIyPp1q0bkZGR/Oc/\n/wF8/WC+/vprY70jR44wcOBAOnTowKRJk/B40rdYLFiwgDlz5gCwYcMGnnjiCZ577jk6derEY489\nxsWLvvHR3G43EydOpF27dvTt25dp06YxfPhwAPbv30/37t2N+nz66aeZ1v3SpUsMHDiQzp0707lz\nZ2bNmmXse9CgQQwfPpwuXbowYMAA/v77b8D3DXn27NlERkYSGRnJ7NmzjT+CtM855f0FCxbQsWNH\nunXrRvfu3YmPjwfg559/pn///jzyyCM88sgjxvqZ1S03KlSoQHT0ELp06ZbrbYUQmdGJdXk4F+dA\nu8FQoem+8W9jXR7kLGfeSUpK4pVXXqFNmzbcfffdtGjRgqlTp6ZqKRw6dCiNGjXK9LMoIiKCkydP\nAnDs2DHGjRtHu3btuPfee2nYsCGDBw/m0KFD113HP//8k3/961+0bNnSqOOTTz7J0aNHU613+vRp\nRo0aRbNmzbj77rtp3749S5cuNT53zpw5Q506dVAUhQ0bNhAREUFERAT9+/fPtg7Hjh1jxIgRNG3a\nlLp16/LQQw8xadIkEhMTjXVcLhevvfaacSxbtWrF9OnTjc8wP39/2GPHjjFjxgyaNm3KvffeS3R0\nNH/99Zex3tixY3nrrbcAX9/ViIgIatWqlSo7tGrVij/++IPo6Gjq16/PkCFDUu0jIydOnOCJJ56g\nXr16NGnShBdeeIGkpKRU6/Tr1y/D47J+/XoiIiKMevbr148tW7YAGMezVq1axuMZ9bnN6+MEvhww\nadIkWrduzd13302zZs2Iiopi7969GR6DkixHLbfBwcGsXbuW/fv38+yzz9KmTZtM150/fz5Tpkyh\nfv366Lqe6o8ipZ9//pnVq1djtVqJjo5m9erV9O3bN8t6/PLLL2zevJmKFSsyceJE3nvvPZ599llW\nrVrFuXPn+Oyzz3C73fTr14/w8HAAlixZwoABA+jcuTNAuhdWSps3b+aWW25h+fLlAMTFxRmP7d+/\nn02bNlG9enUWLFjA9OnTmTdvHqtWreLo0aNs3LgRXdcZPHgwq1evzrLVOTY2lmXLlrFnzx6sViuJ\niYnY7Xbi4uKYPHkyixcvply5csTExNCjRw8++eSTLOuWtuzY2FjKlCmD1+sFfP2BgoODeeCBB9F1\nnV9//dUI50KI66MDTo/GZYebJJc3RyMj5ISmw4V4F4kujTIBFmwmAEm618vlchEVFcXx48fp3bs3\nt956K3/88QcrV67k4MGDrF69GovFQmRkJNu3b2fXrl20bt06VRmffvopdevWpXr16gB88803HDt2\njI4dO1K5cmUuXrzI2rVr6d+/P+vXr+fWW2/NVR09Hg+DBg0iKSmJRx99lEqVKnHhwgX27dvH8ePH\nufPOOwE4deoUvXv3JigoiH79+hEWFsbevXuZO3cuf/31FxMnTiQsLIzZs2czevRoGjZsSK9evQAo\nV65clnXYt28fgwcPxm6307t3b6pUqcK5c+f44osvuHLlCoGBgQAMHz6cr7/+mg4dOjBo0CCOHDnC\nypUr+emnn1i1apXRiunvhzpu3DhCQ0N55plniImJYfny5YwaNYr3338f8J2hjY2NZceOHYwfP57S\npUsDULNmTaNuSUlJDBw4kBYtWjBmzBijn21mfV396993332MGjWKAwcO8MEHH3DmzBkjSGclbblP\nP/008+bN48CBA7z00kvGF4mwsLBMy8jr4+Qv88iRI/Tt25dq1apx9epVDh48yG+//UajRo2yfV43\no7NnzxpZxi8kJISQkJAst8tRuO3YsSMA9957LzExMbhcrkxPBTRp0oTZs2fTrl07WrZsye23355p\nmXa7HYCuXbuybdu2bMNtvXr1qFixIgD33HMP33//PQA//PADXbp0QVEUrFYrDz/8sNG/t1GjRrz9\n9tucOXOGZs2aUbdu3UzLv/fee1mxYgVz586lYcOGNG/e3Hisfv36xhtbz549jbC8Z88eunXrhsnk\nO43YvXt3vvzyyyzDbalSpbjtttv497//TYsWLWjVqhWBgYHs37+f06dPEx0dbfzxmEwmTp48mWXd\nUlqxYgULFixg5syZnDlzBvC9EKKiorI8tkJkRAdcmp58KyUlxf99/+a0pVHxb5u8vvEvCoqSeYzT\nwTjlr6Nfu6377udknyn3Z5Sbosxr5aXcp++eput4dd/sYi6PhsPjxaPp+XIBmA4kuDwkuj1YTSoB\nFhNWs4pZVZKPtZLhsfcfQ9IsS132zXXK+katWLGCw4cP89FHH6Vq5WvYsCFPPfUUmzZtokePHjzw\nwAMEBQWxZcuWVOH2119/5X//+x/jx483lj322GMMGjQo1X769OlDx44dWbFiBVOmTMlVHf/73//y\n559/Mm/ePNq2bWss97dQ+k2bNo1SpUqxadMmI2z26tWL8uXL8+677xIVFUW1atXo1KkTo0ePpkqV\nKkRGRma7f13XGT9+PDabjU2bNhmfswDDhg0zbn/99dd89dVX9OvXL9XxqFmzJjNmzGDNmjWpPsd1\nXady5crMmzfPWFa6dGlmz57N8ePHqVmzJvfccw+33347O3bs4MEHH6Ry5crp6hcbG0t0dDSDBw/O\n9rn41+/duzcjR44E4NFHHyUsLIx33nmH3bt306JFixyV49ekSRPWr1/PgQMH6NSpU7br58dxio+P\nZ9++fYwaNSrda68469u3r5Fl/IYOHZrqdZmRbLslKIqCzeYbhNz/bSltik5pzJgxvPjii1itVkaM\nGMFHH2U/b3BO+9v46wG+0Oc/faTreqZXKkZFRbFo0SLKli3LtGnTeP311zMt/95772Xjxo3cdddd\nbNq0KcvTOP79ZbRv/32z2YymXZs/3uXyXTyiqipr1qyhf//+nDt3ju7du3Ps2DHAd8pjw4YNbNy4\nkY0bN7Jjxw7uuuuuHNctKiqK7du3065dO6KiooiKiqJ79+6ZPg8hsqKgY1UV48di/Itvmenaj0VV\nsJrUbH8sJgWLScGs+n5Miu9HzSLY+uoCquL7MSnXtreYst+vf58p95ty+9RlpHg+yc/T4l8v+Tna\nLCpWkwk1n/sOqIqCzaxiM6tYVdU4/pkd+8yOa8of/3L/T3G3detW6tSpQ8WKFbl8+bLxc88992C3\n241GEqvVStu2bdm5c2eq09dbtmzBbDYbjTyA0TADvv6fV65cQdM06tate11dE4KDgwHYtWtXpmc7\n4+Li+Oabb2jbti1OpzPVc2nWrBmaprFnz55c7xvg8OHDnDx5kscffzxVsE1rx44dKIqSLmT26dOH\nUqVKsWPHjlTLFUVJ19DTqFEjdF3nzz//zFUdc3sNTtrPyIEDB6LrOjt37sxVOdcjP46TzWbDYrHw\nww8/cOXKlfx9AkXIypUr2b59e6qfnDTWZdtymzZ4ZhdE//jjD26//XZuv/12EhISOHToED179ky3\n3meffUZUVBRms5nNmzenOw2UG40aNWLz5s20b98ej8fDp59+avyBnjhxgho1alC1alUCAgLYuHFj\npuWcPn2a8PBwOnbsSP369WnXrp3x2P79+zl16hTVqlVj3bp1ximApk2bsmHDBtq3b4+u62zcuJH2\n7dsDULVqVQ4dOsQDDzzAf//7Xw4fPgxAQkICiYmJNGjQgAYNGnDgwAF+//13WrRowYkTJ9i7d69R\n/qFDh7j77ruzrFtKOWmuFyLnlFRttClbCwGjidNo6bzJLmTJiv8Z+/OfCYVr13mplLaBW9OJdXq4\nmuS54f62KamKQukAM6F2M2pmkb8YH/u89L///Q+n00mTJk3SPaYoCpcuXTLuR0ZGsn79erZt22ac\nndu6dSuNGzembNmyxnrx8fG89tprfPbZZ1y4cCFVmVWrVs11HW+55RYGDRrE8uXL2bx5M/Xq1aNZ\ns2Z07tzZ6GL3xx9/oOs6y5YtY+nSpRk+F/91KLl14sQJFEXhjjvuyHK9M2fOEBgYmC4AW61Wqlat\nyunTp9Ntk7Yl1v/5lJuAFhoamqshwoKCgtJ1w6hQoQJBQUEZ1jGv5cdxslgsPP/888yZM4fmzZtT\np04dmjZtSqdOnbjtttvy6ZkUvkqVKl3XdtmG28xaJTPz8ssvc/LkSUwmEyEhIbz44osZbtegQQOe\nfvppzp49m6pf0PXo06cPR48epVOnTlSqVIk6dergcDgAeO+999i7dy8WiwWbzcaECRMyLeeHH35g\n+fLlmEwmdF1n6tSpxmMNGzZk3rx5/P7775QpU8a40K13796cOnWKbt18F2m1aNHCCPPR0dGMGDGC\nXbt2ceedd1K7dm3A98Y4bNgwnE4nmqZx11130aZNG6xWK4sWLWL27NnMnDkTl8tFtWrVePPNN7Os\nmxCi4Ok6mBWFMLuFIIuJ83FO3DmcuCErZlWhYikbdrPMsZMXNE3j3nvvZcSIERl+AQgNDTVuN27c\nmPLly7NlyxY6d+7M3r17OXfuHM8++2yqbUaOHMnevXsZMGAAtWvXJjg4GEVReOutt3LdIuk3atQo\nevbsyc6dO/nuu+9YuHAhixYtYsGCBTRr1syoe58+fVJ1XUjpeoJ1SjkZqzWzdTI7g+rvsncjUraU\n50RWdczJelmdnc6LOlzvcerfvz9t27Zl+/bt7NmzhxUrVvD2228zbdo0I4MIn2zDrb+1MbP7aS1Y\nsCDD5e+++65xe+bMmZlun/KxlIMld+vWLdUvL+V9i8XC2LFjCQoKwuVy8dRTT9GhQwcAJk6cmGV9\nU+revXump/ADAgJ46aWX0i1XVZXRo0czevTodI9VrVqV9evXZ1jemjVrMlxep04d3nvvvVzVLae8\nXi9utxuv14vX68HlcmEymfLkzUeIksxmUqkUauNcrBOX9/oDrllVqBRixypT8eaZ6tWrExcXR+PG\njbNdV1EUHn74Yd5//30uX77Mli1bsNvtqS6ijouLY9euXQwbNoxnnnkm1fZZdXvLiVtvvZVbb72V\nQYMGcf78ebp06WKE26pVqxoXH2XUCp32eeRGjRo10HWdo0ePZhqcwdfC/O2333L+/PlUrZJut5vT\np09zzz335Gq/11vf7MTHx3PhwoVUrbd///03iYmJVKlSxVgWGhqa4ZeRjJblpo75dZwAwsPD6du3\nL3379iUuLo5evXrx6quvSrhNo9g0DQwcOJCuXbvStWtXbr31VulnmoElS96madP7WLFiOVu3fkrT\npvexdOniwq6WEMWCRVGpUMqGSVWwqDkbezPleqria7GVYJu3Hn74YY4fP24M5ZSSpmlcvXo11bLI\nyEi8Xi+bN2/miy++oHXr1gQFBRmPq6qKoiiprqcA2Lt3LwcPHryuOsbHx6drLaxYsSJhYWFG/cLC\nwmjSpAkbNmzg1KlTGZaR8roOm82W40kRatWqRfXq1Vm5ciXnz5/PdL3WrVuj63q6bhGrVq0iPj7+\nursX+i+Oy+0kDllJOwHUsmXLUBSFBx54wFhWvXp1/ve//6XqWhIXF5dho5S/jpmNVJRSfhwnh8OB\n0+lMtSw4OJhbbrklT49bcXFdkzhcunSJQYMGpbuoqk2bNjz99NN5WsGcyqwlNCOTJ0/m4MGDqepv\nNptZu3ZthuunbTW+WT355FMMGfLPwq6GEMWWzaRSNsjK8LrjjL6wYWFBXLqUkOV2ChAWZJGuCPlg\n0KBBfP3114waNYqdO3dSr149wNfPdNu2bTz33HN07drVWP+uu+6iRo0azJs3j8TExHRXxwcFBdGk\nSROWLFlCUlISt956K0ePHmX9+vXGtSa5tWfPHqZMmUL79u259dZbMZlM7Ny5kz/++IMRI0YY602Z\nMoXHHnuMrl270rNnT2rWrElcXBzHjh1j27ZtbNmyxei7effdd/Pdd9+xbNkywsPDCQsLy7T1WlEU\npk+fTnR0NF26dKFXr15UrVqV8+fPs23bNhYtWkTlypW5//77adWqFe+99x4XLlygQYMGHD16lI8+\n+og6depcd/fCu+++G13Xefnll3n44YexWq00btw4y6G2shISEsInn3zC33//zT333MNPP/3Exx9/\nTIsWLVKNlNCrVy+WL1/OgAED6N27N0lJSXz00UfGUGxp67h69WqmTp1Ky5YtMZvNtG7dOsMuE/lx\nnE6cOEH//v1p164dNWvWJCgoiB9++IFvv/2WHj165P4gFXPXFW7DwsKyvDCrqJP+qkKI/BJiNZFk\nNxPnyPlUuoE2M6FW6R6UH6xWKytWrGD58uVs2bKFL7/8EpvNRuXKlencuXOGgS8yMpL58+cTGhpK\ny5Yt0z3+0ksvMXv2bDZt2kRiYiIRERG88cYbbNq0yZh8wC+zsVhTioiIoHXr1nzzzTesW7cOs9lM\njRo1ePHFF1OdhaxWrRrr169n0aJFbNu2jQ8++IDQ0FBq1KjBsGHDUp2GnzJlClOnTmUuqdP9AAAg\nAElEQVT+/Pk4HA4aNmyYZdeMhg0bsmrVKhYuXMiaNWtISkqiYsWKNGvWjDJlyhjrzZ8/n0WLFrF5\n82a2bdtGWFgYffv2ZcSIETmeqSvt8WjcuDFDhw5l7dq1jB8/Hk3TePfdd41wm9Xxy+ixwMBAli9f\nzgsvvMBLL72E1Wrlscce4/nnn0+1XrVq1Xj99dd59dVXmTNnDpUrV2bQoEHY7fZ0o1506dKF3377\njc8//5xPP/0UXdfZvn07lStXzvB3nNfHKTw8nC5durBnzx62bt2KpmlUqVKF0aNH8/jjj+eovJJE\n0eUS2xIjMdGVoyuqTSaVuDhHAdRI5ET58sHExGR/KkwUHV5d5684By6Pnm3LrVlVuCXUjrmQhuUq\nXz64UPYrhCjZlix5hwsXYnjiifRnlFVVoWzZnI+QkdZ1tdwKIYTInElRKBto41w2XxIVBcoFWQst\n2OZUktvLpUQXTo+GJw9GhMiKWfWN7RsWaCXAIq3ZQojck3ArhBD5INCsEmLP+vRjsM1MKaspX2Y5\nyyuXEl1cSHAV2P48mo7H5SXBlUS5ICthgTm7OE8IIfzk6gUhhMgnZe0W7OaMWx/tFhNlAy1FOtgm\nub0FGmzTupDgIsl942OOCiFKFgm3QgiRTxQFwoNt2NIEXKtZoWKQNfPZx4qIS4mFF2yLUh2EEDcX\n6ZZQgqiqAjn4MPV6tWzXEULkjNWsUjnEyqUkDw6PF5tJJSzAgqmI97MFcHoK/72gKNRBCHFzkXBb\ngiQkONHy+WIQIUR6KgrlAiwoStHuhiCEEMWBdEsQQogCcrMFW1sRmFQiv+pw5swZIiIiUo3Zvn79\neiIiIvjrr7/yZZ9pjRkzJt1sVa1bt6Zv374Fsn+/iIgIFixYUKD7FCI/Ff47lxBCiCKpKIxUUJB1\nyMmECxnZtm3bdYVDRVFQ1YL5GF63bl26KWlT1uN6nrcQRZV0SxBCCJGhAIuJckHWQhsxoVxQwY51\n27VrV2P619z44osv2LJlC0OHDs3VdtOnT0fTCqZP8bp16zh//jxRUVHpHjt48CBms8QBUXzIq1kI\nIUSm/JMplIRJHBRFyXWwBXI082NKDocDu92OyWTCZCr8iSqu5zkLUZRJuBVCCJGlAIuJW0IDCrsa\n1+3UqVO8+OKL/PDDD9hsNtq2bZthv9b169czbtw4duzYQeXKlQH4888/ee211/jPf/7D5cuXKV26\nNLVr12bkyJHceeed9OvXj//85z8oikJERATgC8nbt2+ncuXKtG7dmkqVKjFq1Cjmzp3LL7/8QocO\nHZg5cyZjxozhhx9+YMeOHenq8vPPPzNz5kwOHz5MSEgIjzzyCEOHDk0Vhlu3bk2jRo2YOXNmqm3n\nz5/PwoULOXLkiLGevx9xyjoePnzYWDZ06NBULc9xcXG8/vrrbNu2jUuXLhEeHk5kZCT//Oc/U4Xh\nMWPGsHHjRr777jvmzJnDjh078Hg83H///bzwwgsEB1+b3jm7YylEXpFwK4QQoti6fPkyjz32GAkJ\nCfTr148KFSrw+eefM2bMmHT9TNP2PfV4PAwaNIikpCQeffRRKlWqxIULF9i3bx/Hjx/nzjvv5Omn\nn2bevHkcOHCAl156yWjFDQsLM8o5e/YsTz75JF27dqVLly5G4Musr+v58+eJjo4mMjKSyMhIdu3a\nxaJFi7hy5QqTJ0/O9jmnLXf8+PHMnTuX2NhYxo0bl21Ls8vlIioqiiNHjtCjRw9q1arFvn37eOON\nNzh8+DCLFi1Kt68hQ4ZQpUoVRo4cyfHjx1m5ciVWq5XZs2fn+FgKkVck3AohhCi23n77bS5evMji\nxYtp3rw5AI899hiPP/54ttv+97//5c8//2TevHm0bdvWWD5kyBDjdpMmTVi/fj0HDhygU6dOGZZz\n9uxZXnnlFTp06JCjOp85c4YpU6bQu3dvo77Dhw9n1apVPP7449SsWTNH5fg9+OCDLF26FLfbnWkd\nU/roo4/47bffGDNmDAMGDADg0UcfpVy5crz77rt8/fXX3H///am2adCgAWPGjDHu67rOqlWrmDRp\nEkFBQTk6lkLkFRktQQghRLH11f+3d+9xOd7/A8dfd3eKDBPViG/OmkNoOW+ZnOYYIjWEmuYw9hu+\nziM2pDFMxnxtw5xCGoZ+28xYLT82hjlsKEOZUOTQ+b5+f7Sub3d3h7t0UN7Px8Pj4b6uz/W53tfn\nc1fv+3N/rs/144/Ur19fTWwBTExMGDlyZL4jmJkjrMeOHePJkyeFjuHFF180OrEFqFy5Mm5ubnrb\nxowZg6Io/Pjjj4WOw1hHjhzBwsKCN998U2/7W2+9haIoHD582OAYT09PvdcdOnQgPT1dnQ5RVG0p\nhDEkuRVCCFFuRUdHU69ePYPt9evXz/dYW1tbvL29CQ4OpkOHDowaNYr169fz999/FyiGzPm7xrK1\ntTVYvSAz3ps3bxaorsKIjo6mTp06BjeaWVlZUbVqVaKjow2OyX6NVatWBeDBgwdA0bWlEMaQ5FYI\nIUS59jRruE6fPp2DBw/yP//zP1SoUIE1a9bQu3dvwsPDja6jYsWKBTqnsfHmVi49Pb1A5ytI3bmN\ndue26kPW8kXRlkIYQ5JbIYQoRglJqTxOe/pkQxSOra0tUVFRBtsjIyONrqN+/fp4e3uzYcMGvv32\nW8zNzfUe2lDUD0C4efMmaWlpetsy461Tp466rVq1aiQkJBgcf/36dYNtBYnR1taWmzdvkpKiv77x\n3bt3efjwIba2tkbXlV1+bSlEUZDkVgghitHDpDQSktKQB0CVjtdff52oqCh++ukndVt6ejpfffVV\nvgnfo0ePDEZBbWxssLS0VL9uB7CwsAAyls8qCo8fP2bXrl162zZu3IhGo+H1119Xt9nZ2fHbb7/p\nJaE3b97McWkxCwsLo+Pr2rUrjx8/Zvv27XrbN2zYgEajoVu3bgW4mgzGtqUQRUFWSxBCiGKiAIlp\n6erDD7SS4Za4sWPHsn//fiZNmsTIkSOxsbEhNDSU5OTkfI89fvw4fn5+vPHGG9SvXx+tVsuRI0eI\niori3XffVcu1bNmSoKAgFixYgLOzM6ampri4uBR4OkKmOnXqsHLlSq5cuULDhg05duwYR48exd3d\nXW+lBE9PTw4dOsSoUaPo378/cXFxbN++nYYNG3L+/Hm9Olu2bElYWBiLFi2iVatWmJiY0KdPnxzP\nP3ToUIKDg1m6dClXr15VlwI7cOAALi4uBisl5CbrlARj21KIoiDJrRBCFJOUdB2KVotOp5CcrmBh\nKsltSbO0tGTbtm0sWrSILVu2qA9xGDFiBK6urnkea29vj4uLC2FhYQQHB2Nqakq9evVYtGgRgwcP\nVsu5urpy4cIF/vd//5eDBw+qKwpk3mSV1whxTvtsbGxYvnw5S5YsITg4mKpVqzJu3DgmTZqkV65d\nu3b4+fmxYcMG/P39sbOzY968eVy+fNkgufX29ub69evs27ePrVu3oiiKmtxmXxfXzMyMzZs3s3Ll\nSr777jtCQkJ46aWXmDhxIuPGjTPqGrJvN7YthSgKGqWgzw0UZVZSUqpR5dLTdTx5UjrPkheGrKyq\ncOdO0XzdKUrWo9R0UiuYEhf3mOoWFbCsWKG0QzJgZVUl/0JCCFHENmzYyN27d/DxMfzAZGKioUaN\nFwpdt4zcPkd0OsWoZ6BrtTIVW4iikJKuQ/NPPvskJZ0alSogwwlCCFG8JIsRQohioNFAUqpOfZ2a\nriM5XZfHEUIIIYqCJLdCCFEMdLqMkVv1tQKPUyW5FUKI4ibTEsqAwMBAnjx5wvTp00s7FCFEHqYc\nfZ/k9PzvwjfXmvNxlw9KICIhhHj+yMjtM0KnK/4RnaCg7YwY4UmHDk74+c0r9vMJ8bwxJrEtSDkh\nhBAFJyO3RcDe3p533nmHsLAwHjx4wHvvvUfPnj0BmDZtGteuXSMlJQU7OzsWL15MlSpVOHHiBIsX\nL8bJyYnff/+d8ePH88orr7B48WLOnTuHVqvFycmJuXPnAnD79m18fX25ceMGdnZ2rFq1CnNz8wLF\naW1tzdixvvz8889GrfEohBBCCFHWSHJbRLRaLTt27CAqKgoPDw+cnJywtLRk7ty5vPjiiwCsXLmS\n//znP0yZMgWAy5cvs3DhQjWBnTVrFpUrV2b//v0A3L9/X63/999/Jzg4mBdeeAEfHx/27dvH0KFD\nCxRj167dUBSF8+fPExsbWxSXLYQopDRFwVQe6iCEEEVOktsiMmTIECDjudktWrTgzJkzdO3alZCQ\nEPbv309qaipJSUnUq1dPPcbOzg4HBwf19Y8//sjXX3+tvs5MigFee+01XnghY803BwcHbty4kWMc\nCQkJJCQkUL16dfVRhxqNhipVZC1L8fzIWJT+6erIumye8s9r3T+bTAzq1uSwLW+3EpKoVqkCFU1N\nqGCSsYi+hrwX/M8/Zoxa7k8IIcqCW7duGTy2uWrVqlStWjXP4yS5LSJZ/6DodDo0Gg2//PILO3bs\nICgoiBdffJFvvvmGnTt3quUyn0eeSaPR5PqHyczMTP2/VqvNdVrBpk2bCAwMZMmSJURHRwMZb4RR\no0YV+tqEKGt0//wcmagJo3HHKUpGIpuuU9Apit6atDr++zqjPg0ZpTVoUChoSpqmg5Q0HVoTzT+1\n/PNPo6Dhv8m5iUY/cdYpoPwTi0mWJF6nFDwGIYR4lg0fPlzNZTK98847Bk/ry06S2yKyZ88exo0b\nx7Vr17h06RIODg6cOXOGKlWqUK1aNVJSUggODs6zjtdff50NGzao0xTi4+OpXr16geIYNWoUgwYN\nMhi5FeJ5ogH4ZxSzMOOYJmQkjvrZYk4/R4X/2apdtSLmWiOOzzJinPWsmn/2qQl3oSMRQohn09at\nW3Mcuc2PJLdFxMzMDE9PT+7fv88HH3yApaUlzs7O7Nu3j969e/PSSy/RokULzp49m2sds2bNYvHi\nxfTr1w9TU1Patm3LnDlzChSHMcP1QojSZ1RiK4QQz7FatWoV6jhJbouIp6cn3t7eetu0Wi0rVqzI\nsXy7du3YvXu33rYqVaqwZMkSg7LvvPNOnq+NlZ6eTmpqKunp6aSnp5GSkoJWq0Wr1RaqPiGEEEUv\nOjqabt264e/vz8CBAwEICQlh1qxZfPnll3Ts2LFE4li9ejVr1qzh0qVLJXK+wjp69CgrV64kMjKS\nlJQUDh8+TO3atUs7rHKjrLwPspJ1botAWfnaf8OG9XTq1I5Nm77k0KGDdOrUjs8//09phyWEEOXS\ngwcPCAwM5OTJkwU+Nqe/K8Xxt+bGjRsEBgbmmLhoNBpMTJ7tNOH+/fu89957ALz//vsEBARgaWlZ\nylGVLxk36JaNPCeTjNwWgYsXL5Z2CEZ5++3x+PqOK+0whCi3zLXmRj+hTJR/mcktQNu2bY0+ztbW\nljNnzlChQoXiCk118+ZNAgMDqVOnDvb29nr7JkyYgK+vb7HH8DTOnTtHYmIiEyZMoEePHqUdjnhG\nSHIrhBBFJOsjdTUauJGQROUqlYiLe4xGA7ZVK2KufbZHwp6GV/D/kJSWf3Jf0dSczW4rSyCi0lXQ\nZdlSU1PRarWYmJjorZBTnPKKsSTjKKx79+4BqEtlFpWkpCQqVqxYpHWKklN+f8sKIUQpUhSoaPrf\n+eymJppyndgCRiW2BSlXVBITE1m1ahW9e/fGwcGBTp064ePjw6lTp/TKHT16FA8PD9q0acMrr7yC\nj4+PwU3A0dHR2Nvbs2rVKr799lv69++Pg4MDvXr14tChQ2q5EydO0KtXLzQaDYGBgdjb22Nvb8+s\nWbOAjBV27O3tOXr0KMuXL6dLly60atWKv//+Wz1H1nXPM6WlpbFs2TJee+01WrVqxYgRI7hw4YJe\nmcy6Y2JiDI63t7dXR5NDQkLw9vZGo9Ewc+ZMNcbM/atXrzYYzS3qdsq0Y8cOXF1dadOmDU5OTvTv\n35/Vq1cblMvKxcWFmTNnAjBmzBjs7e3x8vJS90dGRjJp0iTat29Pq1atGDx4MHv37s2xnuHDh3Pm\nzBlGjBhB69atWbBgQZ7nBti5cyeDBw+mVatWtG3blgkTJnD16lV1/4MHD+jSpQv9+/cnJSVF3Z6a\nmsrAgQPp3LkzcXFxatmPPvqIQYMG4eTkRKtWrXBzc+Obb74xOG9mX929e5epU6fSrl072rdvz/z5\n80lNTSU5OZkPP/yQzp0707p1ayZNmsSDBw9yrOP27dtMnjwZJycnnJycmDZtmhpTfi5dusTEiRNp\n3749Dg4ODBgwgD179hh1bHGTkVshhCgm5qYmpP3zfwuzsvfrtrhHYktipDc5OZmRI0dy/vx5+vTp\nw4gRI0hJSeH06dOcPHkSR0dHAA4ePMjUqVNp0KABkyZNIjU1lR07djBixAg2bdpEmzZt9Oo9duwY\ne/bswcPDg6pVqxIUFMS0adNo1qwZdnZ2NGzYkJkzZ+Lv70/Pnj3Vr8z/9a9/Af+dP7ts2TIqVaqE\nj48PaWlpWFhY8Pjx4xyvRVEUli9fDoCPjw+PHj1i69atjBo1iuDgYL26jZkj6eTkhK+vL+vXr2fY\nsGE4OTkB0LRp01zrKep2AggODsbPz48ePXrg6ekJQFRUVL5zlefMmcPRo0fZtWsXb7/9No0aNaJm\nzZoAXL9+nWHDhqHRaBgxYgTVq1fnwIEDzJgxg7i4OMaMGaNX161bt3j77bcZOHAgrq6u+T74aNGi\nRWzdupV+/foxdOhQHj58yLZt2/D09CQ4OJi6detSrVo1/P398fb2ZtmyZcyePRuAFStW8Mcff7Bu\n3Tp1fvCNGzc4ePAgPXv2xN3dnZSUFL777jumTZtGWlqaemNh1n7x9fWlQYMGTJkyhZMnT7Jz507M\nzMzUhzxNnDiRy5cvs337dszNzVm2bJlBHW+//Ta1atViypQpXL16le3bt3P16lV27dqFqWnuv7NO\nnTqFj48P//rXvxg7diyVK1fmyJEjzJ49m/v37xvcYF/Syt5vWyGEKCMqmGjIXKGxUoWyN2pb3COx\nJTHS+/nnn3P+/HnmzZunJk6AXnKTnp7O4sWLqVWrFkFBQepX3AMHDqR3794sWrTIYHWbqKgoQkND\nsba2BqBnz564uLiwa9cupk2bRo0aNXBxccHf358mTZrQv3//HOPTarVs27ZNL5HILbkFePLkCfv2\n7VO/Mu/evTuDBw9m5cqVfPzxxwVqm7p169KxY0fWr19PmzZtco0xU3G0E8CRI0do3LhxviO12XXr\n1o2EhAR27dpFhw4d9FaRWL58OY8fP2b37t00a9YMyFjV6M0332TVqlUMGjRI7ymgt27d4uOPP6Z3\n7975nvfMmTN89dVXzJs3jzfffFPd7urqSp8+fVi9ejUBAQEAdOzYkdGjR7Np0ya6du2KVqtl48aN\nDBs2jC5duqjHNm3alB9++EHvw8SoUaMYPXo069ev10tuM7Vv354ZM2YA4OHhwV9//cWWLVvo0aMH\nn3zyiVouLi6O0NBQ/Pz89KZvKIpCw4YN1Q9MkPGU1Q8//JDdu3fj4eGRaxu8//772Nvbs23bNjVm\nT09PJk2axOrVq3F3dy/yqSIFUfZ+2wohRBlhps14gpiJRoPZM37XeXkVGhpKrVq19BLb7H7//Xfu\n3r3LsGHD9P4g29jY0K9fP86fP8+dO3f0junWrZuasAFYWVnRoEEDrl+/XqD4hg4dmucIWU7ls84F\ntbe3p2PHjhw9erRA5y2M4mqnKlWqcPv2bc6cOVMkcep0Oo4dO0aHDh3UxBbA1NSUUaNGkZycTHh4\nuN4xL774olGJLWSMXpuZmdG9e3fi4+PVfxUqVKBly5ZERETolX/vvfdo0qQJM2bMYObMmdSrV0+d\nTpGpQoUKapKYmprKgwcPiI+Pp2PHjkRFRfHkyRODOLInn5nfQmR/rzs6OpKens7Nmzf1tms0Gr1p\nHADu7u5YWFhw5MiRXK//jz/+4OrVq/Tr14/79+/rtYGzszNJSUlF1peFJSO3QghRTLQaDeZaLRVM\nTTDTaijg/UWiCPz111/5rgsbHR2NRqOhQYMGBvsaNWoEZKwqYGVlpW7PaR3VqlWrGsxtzItGo8HW\n1rZA5evXr2+wvX79+vz888/ExcUV6zJYxdVOY8eO5f/+7//w8PDA1taWdu3a0b17d1xcXAoVZ1xc\nHImJiTnG2bBhQxRFMUj0CrIublRUFCkpKTg7Oxvs02g0BmvHm5mZsXTpUlxdXTExMWHnzp053qy2\nceNGgoKCuHbtmt6NfhqNhoSEBCwsLPKMOfMBTtkffJC5/f79+wbnzP5+MjMzw9bW1qB9soqMjATg\ngw8+4IMPPjDYr9Fo1Bv9Soskt88RExP1oZ15Sk/XFX8wQjwHFAUqm5uQbK6VxLYMyGmeamaSkX1f\nbuu/FnSFBHPzol8WLrf5tjpd0fxuL+p2atCgAaGhofz000+Eh4cTFhbGnj17ePXVV/nPf/5T6DVW\n8zou+76CrIyg0+moVKkSa9euNbq/M0fWFUXhzz//pEWLFnr7N2zYwLJly3B1dWX8+PFYWlqi1Wo5\nevQomzZtyrHvcnsAU27bc4q1MG2bWc/kyZMN5lhnaty4cYHrLUqS3D5HHj9ORpf9IfVCiGJVvZIZ\n6Y9LdnUA8V92dnZcvnw5zzK2trYoisKVK1fo3r273r7Mu98LMsKaqagXvlcURR01yyoqKgoLCwt1\n1DZzpC4hIUFvdK+gUyayK652gowRw27dutGtWzcgY87shg0biIiIoFOnTgWqy9LSkkqVKumtXFBU\ncULGeyo8PJwmTZoYNVJ+4cIFVq9eTd++fbl37x6LFi2iXbt21KlTRy1z4MAB2rVrx9KlS/WO/fnn\nnwsdpzEiIyNp1aqV+jolJYXo6Og812XOvHHR3Ny8xJ6WV1AyCUwIIYpRRoJTtp7uU5707t2bmJgY\ngoKCci3TokULrKys2Llzp97NXLdv3+abb76hefPmel+1Gyvza+SEhISCB56L3bt3k5iYqL6+dOkS\nERERejcn2dnZoSgKx48f1zt206ZNBgl35cqVAYyaTlFc7ZTT1+X29vYoipLjvvyYmJjg7OzM8ePH\n9R6ylJaWxubNmzE3N6dz584FrjdT3759URSFVatW5bg/61JaycnJTJs2DSsrK/z8/PD390er1TJ9\n+nS9kVQTExODkdW4uDiCg4MLHWd+FEVh06ZNetuCgoJ48uQJXbt2zfW45s2bU79+fTZv3kx8fLzB\nfmOXEitOMnIrhBCi3PL29ua7777Dz8+PEydO4OjoSGpqKqdPn6Z58+b4+vqi1WqZPXs2U6dOxd3d\nHTc3N1JSUggKCiI9PZ05c+YU6tw1atSgdu3aHDx4EDs7O6pXr06dOnVwcHAACj6FATKSUQ8PDwYN\nGsSjR4/YsmULFhYWTJ48WS3TsGFD2rZty4oVK4iPj8fGxobw8HBiY2MNztmoUSMqVarE9u3bsbCw\noHLlyjRu3DjHr5WLq528vb2pXr06jo6O2NjYcOvWLbZt20bNmjWNSkJzascpU6Zw/PhxRo0axfDh\nw7G0tOTAgQOcPXuWGTNm6K2UUFCOjo6MGTOGjRs3cuXKFbp27UrVqlWJjo7mp59+omnTpixZsgSA\ngIAArl27xpdffkmVKlWoUqUK8+bNY9q0aXz22WeMG5fx1NAePXqwatUqpkyZQocOHYiNjSUoKIha\ntWrlmEAWVG7vtaioKMaNG4ezszNXrlxhx44d2Nvb4+bmlmtdGo2GJUuW4OPjQ9++fRkyZAh169Yl\nPj6e8+fPc+TIEYN1j0uaJLdCCCGKREVTc6PXrS0p5ubmbNmyhXXr1hEaGsq3335L1apVadasmd5X\nr71796Zy5cqsW7eO1atXY2JiQps2bVi1apWajGbKax3Z7NsDAgLw9/cnICCAlJQUBg4cqNZXkDmh\nmdumTJnCiRMn+Pzzz0lISMDBwYFZs2ZRr149vbLLli3Dz8+PzZs3Y2ZmhouLC4sXL6ZDhw56dVtY\nWBAQEMAnn3zCwoULSUtLY+LEiWpymz2O4mgnT09PDhw4wLZt23j48CE1a9aka9eujB8/nmrVquXa\nRnm1lZ2dHdu3b2flypVs27aNpKQkGjZsyNKlSxkwYIBRdeRlxowZtGzZkq1bt7J27Vp0Oh3W1ta8\n8sor6ioGYWFhbN++ndGjR9O+fXv12H79+nHkyBHWrFmDs7MzzZo1Y+zYsaSlpRESEsLhw4epU6cO\n48ePp1KlSur6uE8Tb27vp3Xr1rF48WJWrFiBoij06dOH2bNnGzz6OfvxrVu3Zvfu3axdu5aQkBDu\n37+PpaUljRo1yjHekqZRCvPRUZRJ9+49kjm3ZZCVVRXu3HlY2mGIQnrW+8/KKvfF6t2Dxhtdz85h\nawt87uKuXwiRs1mzZvH1119z/vz5XG/6K24bNmzk7t07+PiMM9hnYqKhRo3Cr5Mrc26FEELkyNgR\n1sKOxBZ3/UKI55NMSxBCCJGjwj7y9lmpXwjxfJKRWyGEEEKI50xRL1X3LJHkVgghhBDiObJkyRIu\nXLhQavNti1v5vCohhBBCCPFckuRWCCGEEEKUG5LcCiGEEEKIckOSWyGEEEIIUW5IciuEEEIIIcoN\nSW6FEEIIIUS5IcmtEEI8x3Q6XWmHIIR4zjx5klis9csTyoQQ4jl2795jAD7/fB01a1qVcjTPHgsL\nM548SSntMEQhSN89vyS5FUIIgaVlDe7evVPaYTxzKlasQFJSammHIQpB+u7ZZ2lZo1jqleRWCCEE\ngwYNLe0QnklWVlW4c+dhaYchCkH67vklc26FEEIIIUS5IcmtEEIIIYQoNyS5FUIIIYQQ5YYkt0II\nIYQQotyQG8qeIyYmmtIOQRSS9F3ZJv1Xtkn/lV3Sd2XT0/abRlEUpYhiEc+oR48e8cILL5R2GEII\nIYQQRits/iLTEp4D9+/fx9PTk1u3bpV2KKKAbt26hYuLi/RdGSX9V7ZJ/5Vd0pqs6OkAABTWSURB\nVHdl261bt/D09OT+/fuFOl6S2+fEqVOnSE9PL+0wRAGlp6cTHR0tfVdGSf+VbdJ/ZZf0XdmWnp7O\nqVOnCn28JLdCCCGEEKLckORWCCGEEEKUG5LcCiGEEEKIckPr5+fnV9pBiOJnbm5O+/btMTc3L+1Q\nRAFJ35Vt0n9lm/Rf2SV9V7Y9Tf/JUmBCCCGEEKLckGkJQgghhBCi3JDkVgghhBBClBuS3JZTSUlJ\nvPfee/Ts2ZM+ffrw448/5lk+JSWFPn36MGTIkJIJUOTK2L47fPgwgwcPpn///vTv358vv/yyZAMV\nqmvXruHh4cEbb7yBh4cH169fNyij0+lYsGABPXr0oFevXuzatasUIhU5Mab/Pv30U/r168fAgQNx\nc3MjLCysFCIV2RnTd5kiIyNp3bo1AQEBJRihyIux/Xfw4EH1b92AAQOIi4vLu2JFlEuBgYHK3Llz\nFUVRlGvXrimdO3dWnjx5kmt5f39/Zc6cOYqbm1tJhShyYWzfnTlzRomNjVUURVEePnyo9OjRQ/nl\nl19KNFaRwcvLS9m/f7+iKIqyd+9excvLy6BMSEiI4uPjoyiKoty7d09xdnZWoqOjSzROkTNj+i8s\nLExJSkpSFEVRLl68qDg5OSnJycklGqcwZEzfKYqipKenKyNGjFCmTp2qLF26tCRDFHkwpv/Onj2r\n9O3bV7l3756iKBl/7/L72ZOR23Lq0KFDeHh4AGBnZ0eLFi04duxYjmV/+eUX/vrrL1xdXUsyRJEL\nY/vOwcEBKysrAF544QUaNGhATExMicYqIC4ujosXL9K3b18A+vXrx4ULF4iPj9crd+jQIdzd3QGw\ntLSke/fuhIaGlni8Qp+x/de5c2f1rm17e3sAgzKiZBnbdwDr16/HxcWFevXqlXCUIjfG9t+mTZvw\n9vbG0tISyPh7Z2ZmlmfdktyWUzExMdSuXVt9XatWrRyfsZ2YmMiSJUtYsGABiiyc8Uwwtu+yunr1\nKmfPnqVDhw7FHZ7I5tatW9jY2KDRaAAwMTHB2tqav//+W69cYfpVFD9j+y+rkJAQ6tati42NTUmF\nKXJgbN9dunSJ8PBwRo8eXQpRitwY239Xr17l+vXrjBgxgsGDB7N27dp86zYtlohFsRs8eLDBH0ZF\nUdBoNISHhxtdT0BAAMOHD8fKyorIyMiiDlPkoKj6LlNsbCwTJ05k/vz56kiuEKJ4nDhxgtWrV8sc\n9zIiLS2NefPmsWTJEjWJEmVLWloaf/75Jxs3biQ5OZm33nqL2rVr5/ltsyS3ZdSePXvy3G9ra0tM\nTAzVq1cHMj4h5TSq9+uvv3Ls2DHWrFlDcnIyDx48wNXVlb179xZL3KLo+g7g3r17eHt7M3bsWHr1\n6lXksYr81apVi9u3b6sfUHQ6HbGxsbz00kt65WrXrk1MTAwtWrQAMvrV1ta2NEIWWRjbfwCnT59m\nxowZrF27Fjs7u1KIVmRlTN/duXOHGzdu4Ovri6IoPHz4EIBHjx6xcOHC0gpdYPzPnq2tLb169cLU\n1BRTU1O6devGuXPn8kxuZVpCOdWrVy+CgoKAjLsRf//9d1577TWDcvv27ePw4cMcPnyYjz/+mKZN\nm0piW8qM7bv4+Hi8vb0ZMWIEbm5uJR2m+IelpSX29vbs378fgP3799OsWTP1w0mmN954g507d6Io\nCnFxcRw+fJiePXuWRsgiC2P77+zZs0yZMoVVq1apc25F6TKm72rVqkVERASHDx/mhx9+YNSoUQwd\nOlQS22eAsT97/fr1U7/VTE1NJSIigqZNm+ZZtzyhrJxKTExk5syZXLx4Ea1Wy/Tp0+natSsAn3zy\nCTY2NgwbNkzvmBMnThAQEMDu3btLI2TxD2P7LiAggG3btlG/fn31k6+XlxeDBg0q5St4/kRGRjJz\n5kwSEhKoVq0aAQEB2NnZ4evry7vvvkvz5s3R6XQsXLiQ8PBwNBoNY8eOZejQoaUdusC4/hsyZAgx\nMTHY2NioP28BAQE0bty4tMN/rhnTd1kFBgby5MkTpk+fXkoRi6yM6T9FUVi6dCnHjh1Dq9Xy6quv\nMmPGjDzrleRWCCGEEEKUGzItQQghhBBClBuS3AohhBBCiHJDklshhBBCCFFuSHIrhBBCCCHKDUlu\nhRBCCCFEuSHJrRBCCCGEKDckuRVClFsnTpzg5Zdf5v79+0DG0+HatGlTbOdzcXGRx7LmITQ0VO8B\nCCEhITg6OpZKLOPGjWPWrFmlcu4TJ05gb2+vvi8La+TIkXz44YcFKpPfayHKA0luhRBF4t69e3z4\n4Yf06NGDli1b0qVLF3x9fTl69GiRnmfWrFmMGzfOqLKOjo6EhYXx4osvAqDRaIrk+fKBgYH079/f\nYHtwcDBvvvnmU9efl+joaOzt7Tl//nyxnqc4ZG//vn378v333xt9fHn68FAU70NjrFmzhilTphi9\nvzy1sXh+mZZ2AEKIsi86OhoPDw+qVKnCtGnTaNq0KTqdjoiICBYsWMAPP/xQ4jGlpaVhampKjRo1\nSuyc2R8bWRwyn45VWlJTU6lQoUKR1GVmZoalpWWR1PUsSE9PR6vVlnYYeqpWrfpU+4Uoi2TkVgjx\n1Pz8/NBoNOzZs4devXpRr149GjRowPDhw9m7d69a7tatW0ycOBFHR0ccHR2ZNGkSt2/fVvdnjoge\nPHiQHj164OjoyMSJE9WvbwMDAwkJCeHo0aPY29vz8ssvc/LkSXU088CBA4waNYrWrVsTFBSU69e/\nR44coVevXjg4OODl5cWNGzcMYsgq63SGkJAQAgMDuXLlihrD119/DRiOej3t9eake/fuALi5uWFv\nb4+XlxeQkfSuWbOG119/nZYtW9K/f38OHz6cZ79ljoKvXbuWzp0706ZNG2bNmkVKSopaZuTIkfj5\n+bF06VI6duyojkw/evSI999/n06dOuHo6MjIkSP5/fff9er/+uuvcXFxoU2bNowbN467d+/q7Q8J\nCTGYJvLjjz/i7u5Oq1ataN++PePHjyclJYWRI0cSExNDQECA2u6ZTp06xciRI2ndujXOzs74+fnx\n6NEjdX9SUhIzZ86kTZs2vPrqq3z22Wd5tkvW2Ix5r4SEhNCjRw8cHBxITEwkJSWFRYsW0blzZxwc\nHBg2bBi//vqrwTl+++03Bg4ciIODA4MHD9Ybjb9//z5Tp06lS5cutGrVin79+rFnzx6DOtLS0li0\naBHt2rWjXbt2BAQE6O3Pb9pB1v05tXFiYiKvvPIK3377rd5x4eHhtGjRgri4uHzbUoiSJsmtEOKp\nPHjwgLCwMEaMGEHFihUN9lepUkX9/4QJE4iLi+Orr77iq6++IjY2lokTJ+qVv3nzJocOHeLTTz/l\nyy+/5OLFi6xYsQIAb29vevfuTadOnfj5558JCwvTS44+/vhjhg8fzoEDB9QkMPsoZ0pKCmvWrGHp\n0qXs3LkTnU7HpEmT8rzGrF+n9+nThzFjxlC/fn01hj59+uR43NNeb0527dqFoih88cUXhIeHExgY\nCMCmTZv48ssvmT59Ot988w09evRg0qRJXLp0Kc9rO3HiBH/88QebNm0iMDCQ8PBwPvroI70y+/fv\nB2Dbtm0sXboUgLFjx3Lnzh3Wr1/P3r17adu2LaNHj1YT2DNnzjBr1iw8PDzUJPeTTz7JsW0zHTt2\njIkTJ/Lqq6+yZ88evvrqK9q1a4eiKAQGBvLSSy8xceJEwsPDCQsLA+CPP/7Ax8eHbt26sX//fgID\nA7l06RKzZ89W6/X39yciIoI1a9awceNGLly4wMmTJ/NsF8gYpc7vvXLz5k2++eYbPvnkE/bu3YuZ\nmRkBAQGEhoayZMkSvv76a5o0acJbb72ll9wrikJAQADTp09nz5491K1bl7fffpvk5GQAkpOTad68\nOevXr1c/tM2fP5/jx4/rnX/fvn0oikJQUBALFy5k586dbNy4Md9ry0lObVypUiX69u1LcHCwXtk9\ne/bg4uJSrkbeRfkh0xKEEE/lr7/+QlEUGjRokGe58PBw/vzzT77//ntq1aoFwLJly+jZsycRERF0\n7NgRAJ1Oh7+/P5UrVwbA3d2dkJAQACwsLKhYsSKJiYk5/lEdOXIkPXv21Istu/T0dObOnUvr1q0B\nCAgIoHv37nox5MXc3JzKlSuj1Wrz/MNeFNebk8xzVqtWTW/KxRdffIGPj4+aaE+ePJmTJ0/yxRdf\nGIzmZWVqaoq/vz8VK1akUaNGTJs2jblz5zJ16lT1w0qdOnWYMWOGekxERAR//PEHx48fx8zMTD3f\nDz/8wN69e/Hx8WHz5s106tQJX19fAOzs7Dh79qxBkpTV2rVreeONN5g8ebK6rUmTJkBGu5uYmFC5\ncmWD6+7bty+jR48GoG7dusyfP59BgwYRFxdHxYoVCQ4Oxt/fn06dOgGwZMkSunTpkmscmYx5r6Sm\npvLRRx+p/ZKYmMiOHTtYvHgxzs7OACxYsIDjx4+zdetW3n33XbX+iRMnGsS0f/9+hgwZgo2NDd7e\n3mrZoUOHEhERwYEDB+jQoYO63dramrlz5wJQv359oqKi2Lhxo9oeBVGtWrUc29jd3R0PDw9iY2Ox\ntrYmISGB77//PscPK0I8C2TkVghRIiIjI7G2tlYTPchIRKytrbl69aq6rXbt2mqiBxl/vO/du2fU\nOVq0aJFvGRMTE1q2bKl3vuwxFIWSuN5Mjx49IjY21uAr/ldeeYUrV67keWzTpk31RtzbtGlDamoq\n169fV7c1b95c75gLFy6QmJhI+/btadOmjfrvypUr6tf2kZGRalKYKfvr7C5evKiXuBnj/Pnz7Nu3\nTy+ON998E41Gw40bN7h+/TppaWm0atVKPcbCwkJNmvNizHvlpZde0vuQc/36ddLT0/X6wsTEhNat\nW+sdp9Focowps4xOp2Pt2rUMGDBAbefvvvuOmJgYvRhzauPbt2/z+PHjfK/PWC1atKBx48bq9Jv9\n+/dTrVo1NXkX4lkjI7dCiKdiZ2eHRqMhMjIyz3J53QiVdbupqanBPp1OZ1QslSpVMqpcXnKKMS0t\nrcD1lMT15lVvXtvyoyiK3msLCwu91zqdjpo1a7Jt2zaDYzMT9ex1FBedTsfQoUNzHKm0sbHJ9335\ntLK/5zKv+2lv+tuwYQMbN25k7ty5NG7cmMqVK7N8+fJSm+M6ZMgQNm/ejK+vL8HBwQwePLhUb2wU\nIi8yciuEeCrVqlXj1VdfZcuWLSQmJhrsf/jwIQCNGjXi9u3beiNPN27cIDY2lkaNGhl9vgoVKhQ6\n+YOMZOjcuXPq65iYGGJjY2nYsCGQ8bV/9hufLly4UOAYiup6s8tcqSA9PV3d9sILL2BtbW1w09Kv\nv/6a77n+/PNPkpKS1NenT5/GzMyMf/3rX7ke07x5c+7du4dGo6Fu3bp6/zJHMRs2bMhvv/2md1z2\n19m9/PLLBnNKs6pQoYLedQM0a9aMy5cvG8RRt25d9Tq0Wi1nzpxRj3ny5AmXL1/OMxbI/72SEzs7\nO0xNTfX6QqfT8dtvv9G4cWN1m6IoOcaUWfepU6dwcXGhf//+2NvbU7duXa5du2Zwvqx1QEYbW1tb\n630bUBA5tTGAq6srsbGxbN26lYsXLzJ48OBC1S9ESZDkVgjx1ObPn4+iKLi5uREaGkpUVBSRkZFs\n27YNV1dXADp16kTTpk2ZNm0a58+f59y5c/z73/+mRYsWtG/f3uhz2dracvnyZaKiooiPj893VDX7\nCKJWq2Xx4sX89ttvXLx4kRkzZtCkSRN1DmW7du148OAB69at48aNG+zatcvgTnFbW1tiYmK4cOEC\n8fHxeqsLZCqq682uRo0aVKxYkbCwMO7du6euCuDj48MXX3zBgQMHuHbtGqtWreLUqVN68zZzkpaW\nxuzZs7ly5Qrh4eF8/PHHuLu753hzYNZrc3R0ZMKECRw7doybN29y+vRpVq9erSZ1Xl5eREREsH79\nev766y927tyZ75q248aNIzQ0lJUrV3L16lUuX77Mxo0b1Zus6tSpwy+//MLt27eJj48HMm5sO3fu\nHPPnz+fixYtcv36dI0eOMG/ePCBj1HnIkCEsW7aMn3/+mcuXLzNnzhyjPiDl917JSaVKlfD09GT5\n8uUcPXqUq1evMn/+fO7du4enp6de2bVr16oxzZ49GzMzM/r16wdkzJ+NiIjg119/5erVqyxcuJCb\nN28anC82NpbFixcTFRVFaGgoX3zxBWPGjMn32nKTUxtDxgeoXr164e/vT9u2bfP88CNEaZPkVgjx\n1OrUqUNISAidOnVi+fLluLq6Mnr0aH788UcWLlyolvv000+xtLTEy8uL0aNHY21trd7tb6yhQ4fS\noEED3Nzc6NSpE6dPnwZy/xo4+3YzMzPGjRvHjBkzGDZsGBqNhtWrV6v7GzZsiJ+fHzt37mTAgAEc\nP37c4KERPXv2xNnZmdGjR9OpUycOHjyY47mK4nqz02q1zJ07l927d+Ps7MyECROAjGTSx8eHZcuW\nqcuArV69mqZNm+ZZX9u2bWnUqBFeXl5MmjSJjh078u9//1vdn1u7rl+/ng4dOjBv3jx69+7NlClT\nuHbtGtbW1gC0atWKRYsWsWPHDlxdXfn+++/zXZWiS5cuBAYG8tNPPzFo0CC8vLw4ceKEGsPkyZP5\n+++/6dGjh3ojVtOmTdmyZQsxMTGMHDkSV1dXVqxYgZWVlVrvjBkzaN++Pe+88w6jR4+mSZMmODk5\n5dPS+b9XcjNt2jR69+7NnDlzGDRoEJcvX+bzzz+nZs2aahmNRsPUqVPx9/fHzc2N69ev89lnn6kf\nKsaPH4+DgwO+vr54eXlhYWHBgAED9M6j0Wjo378/Op0Od3d35s+fz9ChQxk1apRemezH5PU6pzbO\nNGTIEFJTUxkyZEi+bSBEadIoJTUxSgghxDNl1qxZxMfHs27dutIO5ZkTEhLCBx98wKlTp0o7lGfG\nwYMH8fPz46effsLc3Ly0wxEiV3JDmRBCCCFylZSUxM2bN/nss89wd3eXxFY882RaghBCCCFytWHD\nBgYOHEj16tUZP358aYcjRL5kWoIQQgghhCg3ZORWCCGEEEKUG5LcCiGEEEKIckOSWyGEEEIIUW5I\nciuEEEIIIcoNSW6FEEIIIUS5IcmtEEIIIYQoN/4fm+J+Dc88pyYAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "dist_violin_plot(df_dfc, ID)\n", - "plt.title('Feature contributions for example {}\\n pred: {:1.2f}; label: {}'.format(ID, probs[ID], labels[ID]))\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TVJFM85SAWVq" - }, - "source": [ - "Finally, third-party tools, such as [LIME](https://github.com/marcotcr/lime) and [shap](https://github.com/slundberg/shap), can also help understand individual predictions for a model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PnNXH6mZuczr" - }, - "source": [ - "## Global feature importances\n", - "\n", - "Additionally, you might want to understand the model as a whole, rather than studying individual predictions. Below, you will compute and use:\n", - "\n", - "* Gain-based feature importances using `est.experimental_feature_importances`\n", - "* Permutation importances\n", - "* Aggregate DFCs using `est.experimental_predict_with_explanations`\n", - "\n", - "Gain-based feature importances measure the loss change when splitting on a particular feature, while permutation feature importances are computed by evaluating model performance on the evaluation set by shuffling each feature one-by-one and attributing the change in model performance to the shuffled feature.\n", - "\n", - "In general, permutation feature importance are preferred to gain-based feature importance, though both methods can be unreliable in situations where potential predictor variables vary in their scale of measurement or their number of categories and when features are correlated ([source](https://bmcbioinformatics.biomedcentral.com/articles/10.1186/1471-2105-9-307)). Check out [this article](http://explained.ai/rf-importance/index.html) for an in-depth overview and great discussion on different feature importance types." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3ocBcMatuczs" - }, - "source": [ - "### Gain-based feature importances" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gMaxCgPbBJ-j" - }, - "source": [ - "Gain-based feature importances are built into the TensorFlow Boosted Trees estimators using `est.experimental_feature_importances`." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 401 - }, - "colab_type": "code", - "id": "pPTxbAaeuczt", - "outputId": "2bd9bece-7335-4908-93b1-e8c21894c405" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAArsAAAGACAYAAACtNqpxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4lGW+//HPZBJCSUIJCFnAyLpqFKQmhCKEIlIUSGgL\nh1VEzO6iFHGVJgpIESmLQhZXLCwHEYJAaILnLCIgiCBLdUFEIUFigCwtvc79+4PD/IwESBkyyZP3\n67q8ruQp93yfb8bhkzv3PGMzxhgBAAAAFuTh7gIAAACAO4WwCwAAAMsi7AIAAMCyCLsAAACwLMIu\nAAAALIuwCwAAAMsi7AKACyUkJKh58+Yq6l0dP/74Y7Vt21bNmzfX1atXXVzdnfXuu+/q1VdfdXcZ\nAJCHjfvsAkBen376qZYuXaqTJ0+qcuXKqlevnnr37q3/+q//uqOPm5OToxYtWuiTTz7R/fffX6yx\n4uPj1blzZx07dkweHuVvXiMqKkpnzpzR7Nmz3V0KADcrf6+AAHALH374od544w1FRkZq9+7d2r17\nt6ZOnaqDBw8qOzv7jj72f/7zH2VlZenee+8t9ljGGNlstiLPMF8foyzKzc11dwkAShHCLgD8n5SU\nFC1cuFBTpkxRly5dVLlyZUlSUFCQ5syZIy8vL0nSjh07FBERoRYtWqhjx46KiopyjhEfH6+goCA5\nHA5J0pNPPqm3335bgwYNUvPmzTVs2DBduXLlhseOjY1V9+7dJUkhISF6+umnJUk//vijnnnmGYWG\nhqp79+7asmWL85xb1fHkk09KkoKDg9W8eXMdPnxYUVFRevnll29Z6/z58zVo0CA1bdpUZ8+eVUpK\niiZOnKhHHnlEYWFheuutt24agn85/vWx165dqw4dOig0NFQrV67U0aNH1atXL7Vs2VLTpk1znhsT\nE6NBgwZp+vTpCg4OVo8ePbRnzx7n/gsXLmj48OEKDQ1V165d9cknn+R53FGjRunll19WcHCwVq5c\nqb///e/avHmzmjVrpvDwcEnS2rVr1aNHDzVv3lxdunRRdHS0c4x9+/YpLCxMS5YsUZs2bdSuXTut\nXbvWuT8zM1OzZs1Sp06dFBISosGDBysrK0uSdOjQIQ0cOFAhISEKDw/Xvn37nOetXbtWjz76qJo3\nb65HH31UmzZtyrd3AO4gAwAwxhizc+dO07BhQ5Obm3vL4/bt22e+//57Y4wxJ06cMG3btjVbt241\nxhhz9uxZExQU5BzjD3/4g+nSpYuJi4szmZmZ5g9/+IOZN29evuNeP9fhcBhjjElLSzNhYWEmJibG\nOBwOc+zYMdOqVSvzww8/FLiO62MZY8zChQvNyy+/fMPj/bLWjh07mh9++MHk5uaa7OxsM3z4cDN5\n8mSTkZFhLl68aPr372+io6Pzrf+X4589e9Y88MADZvLkySYzM9Ps3r3bPPzww+b55583ly5dMufO\nnTOtW7c233zzjTHGmLVr15qHHnrILF261OTk5JhPP/3UtGjRwly9etUYY8zgwYPN66+/brKysszx\n48dNq1atzJ49e5yP27BhQ/P5558bY4zJzMy84VqNMWb79u3mp59+MsYY880335gmTZqYY8eOGWOM\n2bt3r3nooYfMwoULTU5Ojtm+fbtp0qSJSUpKMsYYM2XKFPPkk0+aCxcuGIfDYQ4ePGiysrLMuXPn\nTMuWLc3OnTuNMcZ89dVXpmXLlubSpUsmLS3NNG/e3MTGxhpjjElMTHT+7ACUHGZ2AeD/XL58WdWq\nVcuzxvX6jF2TJk20f/9+SddmXu+77z5J0v33368ePXrom2++uem4ffr00d13360KFSqoe/fuOn78\n+C3rMP83c/rFF1+oXr16Cg8Pl81m04MPPqguXbros88+K3AdppBLESIiInTvvffKw8NDV69e1Zdf\nfqmJEyfK29tbNWrU0JAhQwo8O2mz2fT888+rQoUKatOmjSpVqqTHH39c1atXV+3atRUcHKxjx445\nj/f399dTTz0lu92uHj16qEGDBtq+fbvOnTungwcP6qWXXpKXl5eCgoLUv39/rV+/3nlus2bN1KlT\nJ0lShQoV8q0nLCxM9erVk3Rtxrtt27bOn6kkeXl56bnnnpPdbldYWJgqV66s06dPyxijtWvXatKk\nSapVq5ZsNpuaNm0qLy8vbdiwQR06dFC7du0kSa1bt1ajRo20Y8cOSZLdbtf333+vzMxM1axZ0yVL\nVAAUjqe7CwCA0qJatWq6cuWKHA6HM/CuXLlSktShQwdncDx8+LDmzZunkydPKjs7W9nZ2erWrdtN\nx61Zs6bz60qVKiktLa1A9fz88886dOiQWrZsKelacM3NzXX+Wb6wdRREnTp1nF/Hx8crJydHjzzy\niPPxjTEKCAgo8Hj+/v7OrytWrJinF97e3nl6Ubt27Tzn/uY3v9GFCxd04cIFVa1aVZUqVcqz79//\n/ne+dd/Mjh07tGjRIsXGxsrhcCgjI0MPPPCAc/+vf9GpWLGiUlNTdfnyZWVlZal+/fo3jPnzzz9r\ny5Yt+uKLLyRd61FOTo5atWqlSpUqaf78+frggw80ceJEtWjRQmPHjtVvf/vb29YKwHUIuwDwf5o1\nayYvLy99/vnn6tKlS559v5whfemll/Tkk0/qgw8+kJeXl2bOnJnvOtziCggIUGhoqD744IN899+q\nDpvNdsPxlSpVUkZGhvP7xMTEG4755XkBAQHy9vbW3r178x3P1c6fP5/n+4SEBHXu3Fl33XWXrl69\nqrS0NOc66oSEBN1111351p2frKwsjR49WnPmzFHnzp3l4eGh559/vkAz39WrV5e3t7fOnDmTJxxL\n13oUHh6u119/Pd9z27Ztq7Zt2yorK0vz58/Xq6++quXLl9/2MQG4DssYAOD/+Pr66vnnn9fUqVP1\nP//zP0pLS5MxRsePH88TEtPS0uTn5ycvLy8dOXLkhj/rF3bpwM3O7dChg06fPq3169crJydH2dnZ\nOnr0qE6dOnXbOmrUqCEPDw+dOXPGue3BBx/UN998o4SEBCUnJ2vx4sW3rKVWrVpq27atZs6cqZSU\nFBlj9NNPP91yycbNrqUgLl26pGXLliknJ0dbtmzRqVOn1KFDB9WpU0fNmjXTX//6V2VlZem7777T\n6tWr1atXr5uOVbNmTcXHxztruD7zXb16dXl4eGjHjh3avXt3geqy2Wzq06ePZs2apQsXLsjhcOjQ\noUPKzs5Wr169tG3bNu3atUsOh0OZmZnat2+fzp8/r4sXL2rbtm1KT0+Xp6enKleuLLvdXqieACg+\nwm45kJSUpIULFyopKcndpZRJ9K/oymLvnn32WY0fP17vv/++c1ZuypQpeumll9SsWTNJ0uTJk7Vg\nwQK1aNFCixYtUo8ePfKM8ctZxsLOiP7y+NzcXHXs2FEbNmxQu3bt1K5dO82bN895F4Bb1VGxYkX9\n+c9/1qBBg9SyZUsdOXJEbdq0UY8ePdSrVy/169dPHTt2vOljX/fmm28qOztbjz/+uFq2bKnRo0fn\nOyN8u2spyPeNGzdWXFycWrVqpbffflsLFy6Un5+fJGnevHk6e/as2rVrp1GjRmn06NFq3br1TR+7\nW7duysnJUZMmTRQeHq4qVapo4sSJGj16tFq2bKnNmzerc+fOBa5/3Lhxuv/++9WvXz+FhoZq3rx5\nMsaoTp06WrRokd599121bt1aHTt21IcffihjjBwOh5YsWaL27durVatW+uabbzR58uQC9c7dyuL/\nu6UJ/SseV/ePD5UoB86ePavOnTvr888/d745AwVH/4qO3hVPeepfTEyMVq9e7dI/8Zen/rkavSse\n+lc8ru4fM7sAAACwLMIuAAAALIu7MQAA3C4iIkIRERHuLgOABTGzW040b96cdwEXkd1uV926delf\nEdC74qF/xUP/io7eFQ/9Kx673a7mzZu7bDzeoFYOpKSkyMfHx91lAAAAFJir8gthtxy5fDlVDgc/\n7qLw9/fRxYsp7i6jTKJ3xUP/iof+FR29Kx76V3QeHjZVr17FZeOxZrcccTgMYbcY6F3R0bvioX/F\nQ/+Kjt4VD/0rHVizCwAAAMsi7AIAAMCyCLsAAACwLMIuAAAALIuwCwAAAMsi7AIAAMCyCLsAAACw\nLD5UAgAAACUqLTNHqUnp+e7z8LDJ3991n/zKh0qUIw1mbFXc5fyfWAAAACXFMbenUkvosVjGAAAA\nAMsi7AIAAMCyCLsAAACwLMIuAAAALIuwCwAAAMsi7AIAAMCyCLtukJGRodGjR+uJJ55QeHi4xowZ\nI0lat26dBgwYoL59++rpp59WbGysJGnRokUaOXKkJCk9PV09e/bUzp073VU+AABAmcF9dt1g165d\nSk5O1qZNmyRJycnJ2r9/v7Zs2aLly5fLy8tLO3fu1IQJE7RixQoNHz5czz77rD766CMdO3ZMHTp0\nUPv27d18FQAAAKUfYdcNHnjgAZ06dUrTpk1TSEiIOnTooC+++EInTpzQgAEDZIyRMUbJycmSJJvN\npjlz5qh3796qW7eupk+fftOxk5KSlJSUlGeb3W5XQEDAHb0mAAAAV0pISFBubm6ebX5+fvLz8yvU\nOIRdN6hfv742b96sPXv2aOfOnZo/f74effRR9e3b17lc4dd++ukneXh46OrVq0pPT1eVKlXyPW7p\n0qWKiorKs61u3bratm2by68DAADgThk8eLDi4+PzbBsxYsRNs9LN2IwxxpWF4fbOnz+vqlWrqmLF\nikpPT1dYWJgWLVqksWPHasWKFapdu7YcDoeOHz+uhg0b6urVq+rfv79mzZqlr776SqdOndJf//rX\nfMe+1cwuHxcMAABKA8fcnkpMTM53n4eHTf7+PszslmUnTpzQvHnzJEkOh0N/+tOfFBwcrBdffFHD\nhw+Xw+FQdna2unXrpoYNG+qVV15Rv3791Lx5czVt2lRPP/20oqOj9fvf//6GsYvyJAAAAChtXLUE\nk5ndcoSZXQAAUBoUZGbXVbj1GAAAACyLsAsAAADLIuwCAADAsgi7AAAAsCzCLgAAACyLsAsAAADL\n4tZjAAAAKFFpmTlKTcr/dqiuvvUYHypRjly8mCKHg99tiqJWLd+b3g8Qt0bviof+FQ/9Kzp6Vzz0\nr/RgGQMAAAAsi7ALAAAAyyLsAgAAwLIIuwAAALAswi4AAAAsi7ALAAAAyyLsAgAAwLIIuwAAALAs\nwi4AAAAsi7ALAAAAyyLsAgAAwLIIuwAAALAswi4AAAAsi7ALAAAAyyLsAgAAwLIIuwAAALAswi4A\nAAAsi7ALAAAAyyLsAgAAwLJsxhjj7iIAAED5kpaZo9SkdHeXccfUquWrxMRkd5dRJnl42OTv7+Oy\n8TxdNhJKvQYztirusnVfWAAAZYdjbk+lursIlAssYwAAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF\n2C1BW7duVY8ePdSnTx/Fxsa6uxwAAADL424MJSg6OlqjR49W165dC3yOw+GQhwe/kwAAABQFYbeE\nvPHGG9q/f79iY2P18ccfq1atWoqNjVVWVpYCAwM1c+ZM+fr6at++fZo5c6aCg4P17bffavjw4WrR\nooVmzZql77//XpmZmQoNDdWECRNks9ncfVkAAAClGlOGJWTChAlq1KiRJk2apKVLl2rSpElavXq1\nNmzYoHvvvVfvvfee89iTJ0+qV69eWrlypcLCwjRr1iy1bNlSq1at0rp163Tx4kWtXr3ajVcDAABQ\nNjCz6yYxMTHauHGjsrOzlZGRoXvuuce5LzAwUI0bN3Z+v23bNh09elQffvihJCkjI0N16tTJd9yk\npCQlJSXl2Wa32xUQEOD6iwAAALhDEhISlJubm2ebn5+f/Pz8CjUOYdcN9u/fr5UrVyo6OlrVqlXT\npk2btGrVKuf+ypUr33DO3/72N9WrV++2Yy9dulRRUVF5ttWtW1fbtm0rfuEAAAAlZPDgwYqPj8+z\nbcSIERo5cmShxiHsukFycrJ8fX1VtWpVZWVlac2aNbc8vlOnTlq8eLGmTJkiDw8PXb58WampqfmG\n3yFDhigiIiLPNrvd7tL6AQAA7rTly5fnO7NbWITdEnT9DWXt27fXhg0b1L17d9WpU0eNGjXSkSNH\nbnrexIkTNXv2bPXu3VuS5O3trYkTJ+YbdosyvQ8AAFDauGoJps0YY1wyEkq9BjO2Ku5yurvLAABA\njrk9lZiY7O4y7phatXwtfX13koeHTf7+Pq4bz2UjAQAAAKUMYRcAAACWRdgFAACAZRF2AQAAYFmE\nXQAAAFgWYRcAAACWxa3HAABAiUvLzFFqknVvh8mtx4rO1bce40MlypGLF1PkcPC7TVHwolV09K54\n6F/x0L+io3ewCpYxAAAAwLIIuwAAALAswi4AAAAsi7ALAAAAyyLsAgAAwLIIuwAAALAswi4AAAAs\ni7ALAAAAyyLsAgAAwLIIuwAAALAswi4AAAAsi7ALAAAAyyLsAgAAwLIIuwAAALAswi4AAAAsi7AL\nAAAAyyLsAgAAwLIIuwAAALAswi4AAAAsy9PdBaDk+Pv7uLuEMq1WLV93l1BmlcfepWXmKDUp3d1l\nAEC5R9gtRxrM2Kq4y/zjC5QEx9yeSnV3EQAAljEAAADAugi7AAAAsCzCLgAAACyLsAsAAADLIuwC\nAADAsgi7AAAAsCxuPeYGL730kmJjY5WVlaXAwEDNnDlTvr6+mj9/vrZs2aLq1asrJCREe/bs0Zo1\nayRJ69at08cff6zc3Fz5+vpqypQpuueee9x7IQAAAKUcYdcNJk2apGrVqkmS3nrrLS1evFjNmzfX\njh07tHHjRnl7e2vkyJGy2WySpP3792vLli1avny5vLy8tHPnTk2YMEErVqxw52UAAACUeoRdN4iJ\nidHGjRuVnZ2tjIwM3XPPPcrOzlb37t3l7e0tSQoPD9c777wjSfriiy904sQJDRgwQMYYGWOUnJyc\n79hJSUlKSkrKs81utysgIODOXhQAAIALJSQkKDc3N882Pz8/+fn5FWocwm4J279/v1auXKno6GhV\nq1ZNmzZtUnR0tCQ5Z3J/zRijvn37auTIkbcdf+nSpYqKisqzrW7dutq2bVvxiwcAACghgwcPVnx8\nfJ5tI0aMKFAe+iXCbglLTk6Wr6+vqlatqqysLK1Zs0Y2m02hoaFasGCBnnrqKVWoUEHr1693ntOp\nUyeNGzdOAwYMUO3ateVwOHT8+HE1bNjwhvGHDBmiiIiIPNvsdvsdvy4AAABXWr58eb4zu4VF2C1h\n7du314YNG9S9e3fVqVNHjRo10pEjR9SxY0cdPHhQvXv3Vu3atdWkSRPnUoXg4GCNGTNGw4cPl8Ph\nUHZ2trp165Zv2C3K9D4AAEBp46olmDZjjHHJSCi21NRUValSRcYYvfLKK6pdu7ZGjx7tsvEbzNiq\nuMvpLhsPwM055vZUYmL+a+sLo1YtX5eMU17Rv6Kjd8VD/4rOw8Mmf38fl43HzG4pMm7cOMXHxysj\nI0ONGjXSs88+6+6SAAAAyjTCbiny6zeWAQAAoHj4BDUAAABYFmEXAAAAlkXYBQAAgGURdgEAAGBZ\nvEGtHDn9yqPuLgEoN9Iyc9xdAgBAhN1y5eLFFDkc3Fa5KLhfYtHROwCAO7GMAQAAAJZF2AUAAIBl\nEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYB\nAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABg\nWZ7uLgAlx9/fx90llGm1avm6u4Qyqyz1Li0zR6lJ6e4uAwDgIoTdcqTBjK2Ku8w/4sCtOOb2VKq7\niwAAuAzLGAAAAGBZhF0AAABYFmEXAAAAlkXYBQAAgGURdktQp06d9MMPP7i7DAAAgHKDsAsAAADL\n4tZjd8jBgwc1Z84cpaamymaz6eWXX86zf8mSJdq8ebNyc3NVoUIFTZkyRUFBQcrIyNC4ceP0448/\nytPTUw0aNND8+fN1+vRpTZgwQRkZGcrNzVWfPn00dOhQN10dAABA2UDYvQOuXr2qkSNH6m9/+5ua\nNGkiY4ySk5PzHBMeHu4Mq3v27NHkyZMVHR2tXbt2KTk5WZs2bZIk53kff/yxwsLCNHz48Dzbfy0p\nKUlJSUl5ttntdgUEBLj0GgEAAO6khIQE5ebm5tnm5+cnPz+/Qo1D2L0DDh06pN/97ndq0qSJJMlm\ns93wgzl69KgWL16sq1evymazKS4uTpL0wAMP6NSpU5o2bZpCQkLUoUMHSVJISIhmz56trKwshYaG\nqlWrVvk+9tKlSxUVFZVnW926dbVt2zYXXyUAAMCdM3jwYMXHx+fZNmLECI0cObJQ4xB27wBjzC33\nZ2dna/To0VqxYoWCgoJ04cIFhYWFSZLq16+vzZs3a8+ePdqxY4fmz5+vjRs36rHHHlOzZs20e/du\nvffee1qzZo3mzJlzw9hDhgxRREREnm12u911FwcAAFACli9fnu/MbmERdu+AZs2aadKkSTp8+LCa\nNGkih8OhlJQU5/7MzEw5HA7Vrl1b0rUf5nXnz59X1apV1blzZ7Vp00ZhYWG6evWq0tPTVb9+fYWH\nh+vuu+/WxIkT833sokzvAwAAlDauWoJJ2L0DqlatqqioKL3xxhtKS0uT3W7X2LFjZbPZJEk+Pj4a\nNWqU+vbtq7p166pdu3bOc0+cOKF58+ZJkhwOh/70pz+pVq1aevfdd7Vx40Z5eXnJZrNp0qRJbrk2\nAACAssRmbvc3d1hGgxlbFXc53d1lAKWaY25PJSbm/wZQd6hVy7dU1VPW0L+io3fFQ/+KzsPDJn9/\nH9eN57KRAAAAgFKGsAsAAADLIuwCAADAsgi7AAAAsCzCLgAAACyLsAsAAADL4j675cjpVx51dwlA\nqZeWmePuEgAALkTYLUcuXkyRw8FtlYuC+yUWHb0DALgTyxgAAABgWYRdAAAAWBZhFwAAAJZF2AUA\nAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBl\nEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJbl6e4CUHL8/X3c\nXUKZVquW722PScvMUWpSeglUAwAACoKwW440mLFVcZcJYneSY25Ppbq7CAAA4MQyBgAAAFgWYRcA\nAACWRdgFAACAZRF2AQAAYFmE3RIUFBSk9PSivUGsOOcCAACUV4TdEmSz2dxyLgAAQHnFrcfuoP/9\n3//V/PnzVa1aNbVv3965/fDhw5o3b55SU6/dpGrUqFEKCwuTJH3xxReKiopSTk6O7Ha7Zs2apfvv\nv1/GGEmSMUazZs3Sf/7zH82aNUteXl4lf2EAAABlBGH3Drl06ZJeffVVrVq1SoGBgXr//fclSUlJ\nSZoyZYree+891axZU4mJierXr58+/fRTJSYm6tVXX9WKFStUv359ZWdnKzs7W9K1md2MjAyNGzdO\n9erV07x58/J93KSkJCUlJeXZZrfbFRAQcGcvGAAAwIUSEhKUm5ubZ5ufn5/8/PwKNQ5h9w45dOiQ\nGjVqpMDAQEnS73//e82dO1f//ve/dfbsWUVGRjpna+12u+Li4nTo0CGFhYWpfv36kiQvLy/nzK0x\nRpGRkXr88cc1dOjQmz7u0qVLFRUVlWdb3bp1tW3btjtxmQAAAHfE4MGDFR8fn2fbiBEjNHLkyEKN\nQ9i9Q64H2V9+f33dbVBQkJYtW3bDOYcOHbrlmKGhofryyy81cOBAVapUKd9jhgwZooiIiDzb7HZ7\nYUoHAABwu+XLl+c7s1tYvEHtDmnWrJmOHTumM2fOSJI++eQTSVLDhg0VGxurvXv3Oo89evSoJOmR\nRx7Rjh07nOdkZWUpLS3NedyIESPUunVrRUZGKiUlJd/H9fPzU7169fL8xxIGAABQ1gQEBNyQaQi7\npUiNGjU0bdo0/elPf9KgQYOcyxH8/Pz0zjvvKCoqSuHh4erRo4f+9re/SZICAwM1ffp0vfDCC+rd\nu7cGDhzonL6/PiscGRmprl276plnnrlhbS4AAADysplf/70dltVgxlbFXeZevXeSY25PJSYmu7uM\nUqVWLV96Ugz0r3joX9HRu+Khf0Xn4WGTv7+P68Zz2UgAAABAKUPYBQAAgGURdgEAAGBZhF0AAABY\nFmEXAAAAlkXYBQAAgGXxCWrlyOlXHnV3CZaXlpnj7hIAAMAvEHbLkYsXU+RwcFvlouB+iQAAlE0s\nYwAAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUA\nAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBl\nEXYBAABgWYRdAAAAWJanuwtAyfH393F3CS6Vlpmj1KR0d5cBAABKMcJuOdJgxlbFXbZOOHTM7alU\ndxcBAABKNZYxAAAAwLIIuwAAALAswi4AAAAsi7ALAAAAy7Js2I2JidGoUaNcMlZQUJDS0wv2xq7k\n5GS9//77LnlcAAAAFI9lw64k2Wy2Yp2fm5tb6HGuXr1K2AUAACglSu2tx44cOaK5c+cqNfXazaVG\njRql3/3ud+rbt68GDBigL7/8UpmZmZozZ45Wrlypw4cPq1KlSlq0aJH8/f0lXZtlHTVqlOLi4lS9\nenXNnj1bd911l77//ntNnTpV6enpysrK0oABA/TUU09JkiZMmKAqVaooNjZWly9f1po1a2SMkSQZ\nYzRr1iz95z//0axZs+Tl5XVD3dOmTVNKSooiIiJUsWJFrVixQnFxcZo8ebIuXbokT09PjRkzRu3a\ntVN0dLROnDih1157TUeOHNGAAQO0evVqNWrUSFOnTtVDDz2k/v37KygoSGPGjNE///lPXb16VWPH\njlWXLl1K6CcBAABQdpXKsJucnKzJkyfrvffeU82aNZWYmKh+/frp3Xff1ZUrVxQcHKwXX3xRH3zw\ngZ5++mn+s1U2AAAXQElEQVR99NFHmjZtmqZOnaqPPvpIo0ePliQdOHBA69evV2BgoKKiojR9+nQt\nWLBA9erV0z/+8Q95eXkpLS1N/fv31yOPPKLf/va3kqRDhw5p+fLl8vb2lnRtZjcjI0Pjxo1TvXr1\nNG/evJvW/tprr6lfv36KiYlxbnv55Zc1cOBA9enTRz/++KMGDx6sLVu2qHXr1lq6dKkk6euvv1az\nZs20Z88eNWrUSHv27NGwYcOcY/j6+mr16tU6cOCAXnjhhZuG3aSkJCUlJeXZZrfbFRAQUISfBAAA\ngHskJCQ4/8p+nZ+fn/z8/Ao1TqkMuwcOHNDZs2cVGRnpnFW12+3KyclRlSpV1L59e0nSQw89pDp1\n6uiBBx6QJDVs2FB79uxxjtOiRQsFBgZKkvr3769evXpJktLT0zV58mR999138vDwUGJior777jtn\n2O3atasz6ErXZnQjIyP1+OOPa+jQoYW6ltTUVH333Xfq06ePJOnee+/Vgw8+qMOHD6tDhw7KyMjQ\n+fPntWfPHv3lL3/RokWL1LNnT2VnZ6tevXrOcXr06CFJatq0qRITE5WVlaUKFSrc8HhLly5VVFRU\nnm1169bVtm3bClU3AACAOw0ePFjx8fF5to0YMUIjR44s1DilMuxK194UtmzZsjzb4uPj8wQ8u92e\nJ5ReD8Q3c33t7V//+lfVqlVLs2fPls1m07Bhw5SVleU8rnLlyjecGxoaqi+//FIDBw5UpUqVCnwd\n18P6zWpp1aqVtm/frosXLyo4OFiJiYnavn27WrVqlefY69fp4XFtmfWvf9O5bsiQIYqIiMizzW63\nF7heAACA0mD58uX5zuwWVql8g1qzZs0UGxurvXv3OrcdPXpUxpibhsf8HDhwQGfOnJEkrVmzRqGh\noZKuLZMICAiQzWbT999/r/379992rBEjRqh169aKjIxUSkrKTY/z8fFRRkaG84fj4+OjBx980Lms\n4ccff9SJEyfUuHFjSdfC7rvvvqvmzZs7r33x4sVq3bq1c8xfX/OteuDn56d69erl+Y8lDAAAoKwJ\nCAi4IdMUJeyWypldPz8/vfPOO3rzzTf1xhtvKCsrS3fffbdeeeWVQt0ZISQkRAsWLNDJkyedb1CT\npOHDh2vs2LHasGGD7r77boWEhNxynOuPGRkZqYoVK+qZZ57R+++/n2/Dq1atqp49e6pnz56qWrWq\nVqxYoTlz5ui1117TkiVL5OnpqTlz5qh69eqSroXdhIQEtWnTRpLUunVrffLJJzfM7OZXDwAAAG7N\nZgozVYoyrcGMrYq7XLD7BZcFjrk9lZiYXCKPVauWb4k9ltXQu+Khf8VD/4qO3hUP/Ss6Dw+b/P19\nXDeey0YCAAAASplSuYyhLJg8ebIOHz7sXFJgjJGnp6dWr17t5soAAABwHWG3iKZOneruEgAAAHAb\nLGMAAACAZRF2AQAAYFmEXQAAAFgWa3bLkdOvPOruElwqLfPmn5YHAAAgEXbLlYsXU+RwcFtlAABQ\nfrCMAQAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZh\nFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAAAJZF2AUAAIBlEXYBAABgWYRdAAAAWBZhFwAA\nAJZF2AUAAIBlEXYBAABgWZ7uLgAlx9/fx90lFEhaZo5Sk9LdXQYAALAAwm450mDGVsVdLv0h0jG3\np1LdXQQAALAEljEAAADAsgi7AAAAsCzCLgAAACyLsAsAAADLKrGwO2nSJP3rX/+SJE2YMEHLly/P\n97hf7lu5cqWWLl1aUiUCAADAYkrsbgzTp08v9DkDBw68A5UAAACgvLht2A0KCtKYMWP0z3/+U1ev\nXtXYsWPVpUuXmx6/detWvf322/L09FROTo5ee+01hYSE6Mknn9Szzz6rsLAwSdJ3332noUOH6ty5\ncwoJCdFrr70mT8+85URFRSktLU1jx45VTEyMNm3aJD8/P508eVJ+fn5auHCh/P39lZ2drddff137\n9u1TzZo1FRQUpMTERC1YsEAHDhzQ9OnTZYxRTk6Ohg8frh49euRb+6VLl/SXv/xFFy9elCS1adNG\n48ePV0xMjDZu3CgfHx/FxcWpevXqmj17tu666y45HA7NmTNHu3btkiQ98sgjGjt2rGw22w3X/Mvv\no6KitHnzZnl7e8tms+m///u/5ePjoyNHjmju3LlKTb12861Ro0YpLCzsprUBAADg5go0s+vr66vV\nq1frwIEDeuGFF24ZdhcuXKgpU6aoRYsWMsYoLS0t3+OOHDmi6OhoVahQQZGRkYqOjtbgwYNvWce3\n336rDRs2qHbt2nr11Ve1bNkyvfDCC1q5cqXOnTunzz77TNnZ2XryySdVp04dSdL777+vp59+Wr16\n9ZIkpaSk3HT8DRs2qG7dulqyZIkkKTk52bnvwIEDWr9+vQIDAxUVFaXp06drwYIFWrlypU6cOKF1\n69bJGKNnn31W0dHRt5yVTkpK0ocffqivv/5aFSpUUFpamipWrKjk5GRNnjxZ7733nmrWrKnExET1\n69dPn3766S1r+/XYSUlJebbZ7XYFBATcsrcAAAClSUJCgnJzc/Ns8/Pzk5+fX6HGKdCa3eszoU2b\nNlViYqKysrJuemzr1q315ptv6oMPPtAPP/ygKlWq3HTMihUrysPDQ+Hh4dq7d+9t62jWrJlq164t\nSWrSpIl++uknSdK+ffvUu3dv2Ww2VahQQY8//rjznNDQUC1evFjvvPOOjhw5Ih+fm3+KWNOmTbV7\n927NmTNH27dvV6VKlZz7WrRoocDAQElS//79nfV+/fXXioiIkN1ul6enp/r06aOvvvrqltfh4+Oj\n3/72t3rppZf0ySefKDU1VR4eHjpw4IDOnj2ryMhIhYeHKzIyUna7XXFxcbes7ZeWLl2qzp075/nv\ndr9EAAAAlDaDBw++IdMU5b1ct53Ztdls8vb2liR5eFzLxr9O2b80fvx4nTx5Ul9//bVGjx6toUOH\nqn///rd8DGNMgYq9Xod0bbYyJyfHeb7NZsv3nCFDhqhTp07as2ePpk2bpkceeUSjR4/O99imTZtq\n3bp12r17t9avX6/Fixfr448/zvfY64+X32Nf/97T01MOh8O5/fovCR4eHlq1apUOHDigPXv2qE+f\nPvrggw8kXVs2smzZsnwfsyC1DRkyRBEREXm22e32fMcDAAAorZYvX57vzG5h3Tbs/jqI3i6Ynj59\nWvfdd5/uu+8+paam6ujRo/mG3c8++0xDhgyRp6enNmzYoE6dOhWy9P8vNDRUGzZsULdu3ZSTk6PN\nmzc7Z4BjY2N1zz33qH79+qpUqZLWrVt303HOnj2rOnXqqEePHmrRooW6du3q3HfgwAGdOXNGd999\nt9asWaPQ0FBJ19bOxsTEqFu3bjLGaN26derWrZskqX79+jp69Kg6duyoH374QcePH5ckpaamKi0t\nTcHBwQoODtahQ4d08uRJtWvXTrGxsdq7d69z/KNHj+rhhx++ZW2/VJTpfQAAgNLGVUswCzSze6vv\nf23evHmKi4uT3W6Xn5+fZsyYke95wcHBeu6555SQkKCQkBANGDCgsLU7DRw4UCdOnNATTzyhgIAA\nNWrUSBkZGZKkZcuWae/evfLy8pK3t7cmTZp003H27dunJUuWyG63yxijqVOnOveFhIRowYIFOnny\npPMNapL0+9//XmfOnHHOprZr184Z7iMjIzV69Gjt3LlTDzzwgB566CFJ19YNjxw5UpmZmXI4HGrY\nsKG6dOmiChUq6J133tGbb76pN954Q1lZWbr77rv197///Za1AQAAIH82U9A1BKVcamqqqlSpoqys\nLA0fPlzdu3dXv379XDJ2TEyMtm/frrffftsl47lLgxlbFXc53d1l3JZjbk8lJub/Bjx3qVXLt9TV\nVFbQu+Khf8VD/4qO3hUP/Ss6Dw+b/P1v/h6rwiqx++zeaUOHDlVWVpaysrLUpk0b9enTx90lAQAA\nwM2KFHYvXbqkZ5555oY3aXXp0kXPPfecSwssqFWrVhX42MmTJ+vw4cN56vf09NTq1avzPT4iIuKG\nN30BAACg9CtS2K1Ro8Yt3+hV2rHeFQAAoHwo0H12AQAAgLKIsAsAAADLsswb1HB7p1951N0lFEha\nZo67SwAAABZB2C1HLl5MkcNhiTvNAQAAFAjLGAAAAGBZhF0AAABYFmEXAAAAlkXYBQAAgGURdgEA\nAGBZhF0AAABYFmEXAAAAlkXYBQAAgGURdgEAAGBZhF0AAABYFmEXAAAAlkXYBQAAgGURdgEAAGBZ\nhF0AAABYFmEXAAAAlkXYBQAAgGURdgEAAGBZhF0AAABYFmEXAAAAluXp7gJQcvz9fQp0XFpmjlKT\n0u9wNQAAAHceYbccaTBjq+Iu3z7EOub2VGoJ1AMAAHCnsYwBAAAAlkXYBQAAgGURdgEAAGBZhF0A\nAABYFmG3DIiKitLs2bPdXQYAAECZQ9gtJRwOh7tLAAAAsBxuPeYCQUFBGjFihHbt2qWrV69qzJgx\neuyxxyRJL730kmJjY5WVlaXAwEDNnDlTvr6+2rdvn2bOnKng4GB9++23Gj58uFq0aKGZM2fq6NGj\nstvtCg4O1qRJkyRJ58+f1x//+Ef99NNPCgwM1Ntvvy1vb293XjYAAECpR9h1EbvdrpUrV+r06dMa\nOHCggoODVaNGDU2aNEnVqlWTJL311lt677339OKLL0qSTp48qddff90ZaCdMmKAqVapo48aNkqQr\nV644x//222+1Zs0a+fj4aNiwYdqwYYP69+9fwlcJAABQthB2XaRfv36SpAYNGqhRo0Y6fPiwOnbs\nqJiYGG3cuFHZ2dnKyMjQPffc4zwnMDBQjRs3dn6/fft2rVu3zvn99ZAsSe3atZOPz7VPQGvcuLF+\n+umnfOtISkpSUlJSnm12u10BAQHFvkYAAICSkpCQoNzc3Dzb/Pz85OfnV6hxCLsuYoxxfu1wOGSz\n2bR//36tXLlS0dHRqlatmjZt2qRVq1Y5j6tcuXKeMWw2W55xfqlChQrOr+12uzIzM/M9bunSpYqK\nisqzrW7dutq2bVuhrwkAAMBdBg8erPj4+DzbRowYoZEjRxZqHMKui6xdu1Z//vOfFRsbq++++06N\nGzfW4cOH5evrq6pVqyorK0tr1qy55RgdOnTQ+++/71zWcPnyZVWvXr1QdQwZMkQRERF5ttnt9sJd\nDAAAgJstX74835ndwiLsukiFChU0aNAgXblyRdOmTVONGjXUvn17bdiwQd27d1edOnXUqFEjHTly\n5KZjTJgwQTNnztQTTzwhT09PhYSE6JVXXilUHUWZ3gcAAChtXLUE02Zu9ndzFFhQUJAOHjyoSpUq\nubuUW2owY6viLqff9jjH3J5KTEwugYrKjlq1fOlJEdG74qF/xUP/io7eFQ/9KzoPD5v8/X1cN57L\nRirHbDabu0sAAABAPljG4ALHjx93dwkAAADIBzO7AAAAsCzCLgAAACyLsAsAAADLIuwCAADAsniD\nWjly+pVHC3RcWmbOHa4EAACgZBB2y5GLF1PkcHBbZQAAUH6wjAEAAACWRdgFAACAZRF2AQAAYFmE\nXQAAAFgWYRcAAACWRdgFAACAZRF2AQAAYFmEXQAAAFgWHypRjnh42NxdQplG/4qO3hUP/Sse+ld0\n9K546F/RuLpvNmMMH6llcSkpKfLx8XF3GQAAAAXmqvzCMoZy4MqVKxo0aJASEhLcXUqZlJCQoE6d\nOtG/IqB3xUP/iof+FR29Kx76VzwJCQkaNGiQrly54pLxCLvlxIEDB5Sbm+vuMsqk3NxcxcfH078i\noHfFQ/+Kh/4VHb0rHvpXPLm5uTpw4IDLxiPsAgAAwLIIuwAAALAswi4AAAAsyz5lypQp7i4Cd563\nt7dCQ0Pl7e3t7lLKJPpXdPSueOhf8dC/oqN3xUP/iseV/ePWYwAAALAsljEAAADAsgi7AAAAsCzC\nbhkWGxurgQMHqlu3bho4cKDOnDlzwzEOh0NTp05Vly5d1LVrV33yyScF2lceFLd/UVFRatOmjSIi\nIhQREaFp06aVZPluV5D+7d69W3379tXDDz+s2bNn59nH8694/SvPz7+C9G7RokV64oknFB4err59\n+2rXrl3OfRkZGRozZowee+wx9ejRQ9u3by/B6t2vuP2bMGGCwsLCnM+9d999tyTLd7uC9G/t2rXq\n1auXwsPD1atXLy1btsy5rzy/9hW3d0V+3TMos5566imzceNGY4wx69evN0899dQNx8TExJhhw4YZ\nY4y5ePGiad++vYmPj7/tvvKguP1buHChefPNN0uu4FKmIP07c+aMOX78uHnrrbdu6BXPv+L1rzw/\n/wrSu127dpmMjAxjjDHHjx83wcHBJjMz0xhjTFRUlJk0aZIxxpjY2FjTtm1bk5aWVkLVu19x+zd+\n/Hjz0UcflVzBpUxB+peSkuL8OjU11XTs2NGcOHHCGFO+X/uK27uivu4xs1tGXbp0ScePH9fjjz8u\nSXriiSd07NgxXb58Oc9xW7Zs0YABAyRJNWrU0KOPPqrPPvvstvuszhX9kyRTTt/fWdD+1a9fX0FB\nQbLb7TeMwfOveP2Tyufzr6C9a9u2rfNd3EFBQTLGOI/ZsmWLBg4cKEkKDAxUo0aNtHPnzhK8Cvdx\nRf/Ks4L2r0qVKs6v09LSlJOTI5vNJqn8vva5ondS0V73CLtlVEJCgmrXru18Anh4eOiuu+7SuXPn\n8hz3888/6ze/+Y3z+4CAAOdndd9qn9W5on/StRet3r17a9iwYTp06FDJFF8KFLR/t8Lzr3j9k8rn\n868ovYuJidHdd9+t2rVrS+K5V9z+SdI//vEP9erVSyNGjNCPP/54x+suLQrTv23btumJJ55Q586d\nNWzYMN13332Syu/zzxW9k4r2uufpmksAyp9BgwZp+PDhstvt+uqrr/Tcc89py5Ytqlq1qrtLQznA\n869g9u3bp4ULF2rJkiXObb+cJcKt5de/MWPG6K677pIkrVu3TpGRkfr888/p66906tRJnTp10rlz\n5/Tcc88pLCxM99xzj7vLKhNu1ruivu4xs1tGBQQE6Pz5887pfIfDoQsXLqhOnTp5jvvNb36jn3/+\n2fl9QkKCAgICbrvP6lzRP39/f+efl9u0aaM6dero5MmTJXQF7lXQ/t0Kz7/i9a+8Pv8K07uDBw9q\n3LhxWrRokQIDA53bee4Vr3/Xg64khYeHKzU1tdB/lSirivL/bp06dfTwww873whZXp9/ruhdUV/3\nCLtlVI0aNRQUFKSNGzdKkjZu3KiHHnpI1atXz3Nct27dtGrVKhljdOnSJX3++ed67LHHbrvP6lzR\nv/PnzzuPO378uH7++Wc1aNCg5C7CjQrav1/69Tornn/F6195ff4VtHdHjhzRiy++qLfffltBQUF5\n9nXt2lXR0dGSrr07/Ntvv1W7du1K5gLczBX9++Vz78svv5Snp2eeJQ5WVtD+nTp1yvn1pUuXtHfv\nXt1///2Syu9rnyt6V9TXPT5BrQw7deqUxo8fr6SkJFWtWlWzZ89WYGCg/vjHP2r06NFq2LChHA6H\nXn/9de3evVs2m02RkZHq37+/JN1yX3lQ3P6NHz9e//73v+Xh4aEKFSpo1KhR5eYfTKlg/fvXv/6l\nF198UampqTLGyNfXVzNmzFDbtm15/hWzf+X5+VeQ3vXr108///yzateuLWOMbDabZs+erfvuu0/p\n6ekaP368jh8/LrvdrrFjx6pjx47uvqwSU9z+DR06VBcvXpTNZpOvr6/Gjh2rxo0bu/uySkxB+vfG\nG29o9+7d8vLykjFG/fv31+DBgyWV7397i9u7or7uEXYBAABgWSxjAAAAgGURdgEAAGBZhF0AAABY\nFmEXAAAAlkXYBQAAgGURdgEAAGBZhF0AAABYFmEXAAAAlvX/ABFRvWGJhKyaAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "importances = est.experimental_feature_importances(normalize=True)\n", - "df_imp = pd.Series(importances)\n", - "\n", - "# Visualize importances.\n", - "N = 8\n", - "ax = (df_imp.iloc[0:N][::-1]\n", - " .plot(kind='barh',\n", - " color=sns_colors[0],\n", - " title='Gain feature importances',\n", - " figsize=(10, 6)))\n", - "ax.grid(False, axis='y')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GvfAcBeGuczw" - }, - "source": [ - "### Average absolute DFCs\n", - "You can also average the absolute values of DFCs to understand impact at a global level." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 401 - }, - "colab_type": "code", - "id": "JkvAWLWLuczx", - "outputId": "080348ec-2de1-45dc-9c91-9756ded3f517" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAArsAAAGACAYAAACtNqpxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlYlXX+//HX4SBugAuZIipa4zcmHVcMadzN3BXX9GuF\nWqQ2ksuUjZOTmmaalZVoi2U5Zi7p4JY6TZm5DGl+yaXJLQ3cSMntIIgHOPfvD3+e8QQi6OEcuHk+\nrqvr6tznvj/nfb9Fz8uPn/u+LYZhGAIAAABMyMfbBQAAAABFhbALAAAA0yLsAgAAwLQIuwAAADAt\nwi4AAABMi7ALAAAA0yLsAgAAwLQIuwBKpQ4dOuj06dM3fT8sLEwnTpyQJE2ePFnvvPOOR+ry1Gd1\n6NBBCQkJeb539epVjRw5UuHh4Ro7dmyR12ImMTExWr16tSQpPj5e//u//+u2sdetW6cnnnjCbePl\nJT4+XhMnTizSzwA8zdfbBQAwvw4dOig1NVXbtm1T5cqVndt79+6tQ4cOafPmzapZs6ZHa7JYLAV+\nf+rUqUVSQ3x8vD777DN9+umnRf5ZhbFp0yadP39e33333S37dCtxcXE6fvy4Xn31VTdV5x0FPY8F\nCxa4vL7d/p06dUodO3bUjz/+KB+fa/NSPXv2VM+ePW9rPKA0Y2YXgEfUqlVLn3/+ufP14cOHdfXq\n1TsOU7frVg+PLOjDJXNycu6oBm+df35Onz6tunXrFova7qS/nubOB5Je/9ngIafAnSPsAvCI3r17\nKz4+3vk6Pj5effr0cdnHbrdr1qxZat++vVq1aqUpU6bIbrdLkmw2m0aOHKnIyEhFRERo5MiROnPm\njPPYxx57TG+99ZYGDx6sZs2a6YknntDFixcLXN8HH3ygVq1aqU2bNlq1apVL0Js4caLeeustSdKu\nXbvUtm1bLViwQK1atdJf//pXSdLXX3+tqKgotWjRQoMHD9ahQ4ecx//yyy+KjY1VZGSkWrZsqenT\np+vo0aOaMmWK9uzZo6ZNm+qBBx7I9VmStGLFCj388MOKiIjQ008/rbNnzzrfCwsL07Jly9S5c2dF\nRETopZdecr534sQJRUdHKyIiQpGRkXr22Wd1+fLlW/Zh7ty5mjdvnjZs2KBmzZpp1apVkqSVK1eq\nW7duioiI0JNPPumyBOTll19Wu3bt1Lx5c/Xr10+7d++WJG3btk3vvvuuNmzYoKZNmyoqKkpS7iUU\ncXFxeu655yRdm9EMCwvTypUr1b59ew0dOlSStGfPHg0aNEgtWrRQVFSUdu3addNzyKvf0rUAOX/+\nfHXo0EF//OMf9Ze//MXZk+ufu3r1arVv316RkZF699138z2Pxx57THPmzNHgwYPVpEkTnTx5Uo89\n9phWrlzprMXhcGj69OkKDw9Xt27dXM47rz5MmDDBObYkhYeHq1mzZtq7d2+uZRGJiYnq37+/WrRo\noQEDBuj77793vpff7we73a7nnntOERERzmPPnz9/034CJR1hF4BHNG7cWOnp6Tp27JgcDoc2bdqk\nXr16ucxczZ49W8nJyVq7dq2++OILnTlzRvPmzZN0LTT069dP33zzjb7++muVK1fOJdxJ0ueff65Z\ns2bp22+/ld1u18KFCwtU29atW/Xxxx9r0aJF+uKLL/Tvf/873/1//fVXpaWl6euvv9a0adP0n//8\nRy+88IKmTZumXbt26ZFHHtGoUaOUlZUlh8OhESNGqFatWtqyZYu2bt2qbt266d5779XUqVPVpEkT\nff/993mGt4SEBL3xxht6++23tX37dtWsWVPjx4932WfLli1atWqVVq9erY0bN2r79u2SrgW7kSNH\naseOHdqwYYPOnDmjuXPn3rIXsbGxGjlypLp166bExET169dPX375pRYsWKB58+YpISFB4eHhLnU0\natRIa9eu1XfffaeePXtq7Nixstvtat26tXOs77//3rmWNS+/nUXevXu3Nm7cqA8//FBnzpzRiBEj\n9Kc//Unfffednn/+ecXGxurChQu5xrlZvyVp1apVWrNmjRYvXqwvv/xS6enpuX6GEhMT9c9//lMf\nffSR5s2bp2PHjuV7HuvWrdP06dOVmJio4ODgXPXs27dPderU0c6dOzV69GjFxsbKZrPd8tfhk08+\ncdaTmJioxo0bu/Tp0qVLGjlypKKjo7Vz504NHTpUI0aM0KVLl5xj3Oz3Q3x8vC5fvqxt27Zp165d\nmjp1qsqWLXvLmoCSirALwGN69+6t1atXa8eOHbrnnnt09913u7y/cuVKTZw4UQEBAapQoYKeeuop\nrV+/XpJUuXJlderUSX5+fqpQoYJGjBjhnEG8rm/fvqpTp478/PzUtWtXHThwoEB1bdq0SX379tW9\n996rcuXKKTY2Nt/9fXx8FBsbqzJlysjPz0+fffaZBg0apD/84Q+yWCyKioqSn5+f9u7dq3379ik1\nNVXPPfecypYtKz8/PzVr1qxAda1fv179+/dXWFiYypQpo/Hjx2vPnj0us6ojRoyQv7+/goODFRER\n4TznOnXqKDIyUr6+vqpSpYqio6P13XffFehzf2v58uV66qmnVK9ePfn4+Oipp57SwYMHlZKSIuna\nWtLAwED5+Pho6NChstvt+vnnn2/rs6RrgS42NlblypWTn5+f1q5dq3bt2ql169aSpMjISDVs2FDf\nfPNNrmPz6/f69es1dOhQhYSEqHz58ho/frw2bNggh8Ph/NzRo0fLz89PYWFhCgsL08GDB/OttU+f\nPrr33nvl4+MjX9/cl8EEBQXp8ccfl9VqVbdu3VSvXj1t2bKlwL242TKGLVu2qG7duurZs6d8fHzU\nvXt33XPPPfr666+d+9zs94Ovr68uXryon3/+WRaLRffff78qVqxY4JqAkoYL1AB4TK9evfToo4/q\n5MmT6t27t8t758+f15UrV9SvXz/nNofD4fyyz8zM1IwZM7R9+3bZbDYZhqGMjAyXda933XWX89jy\n5csrIyOjQHWdPXtWDRs2dL6uWbNmvmslq1atqjJlyjhfnz59WmvWrHHOxhmGoezsbJ09e1YWi0U1\na9Z0XmRUGGfPnlWDBg2crytUqKDKlSvrzJkzzgv6bnbO58+f1/Tp07V7925lZGQoJyfH5eLAwjh9\n+rRefvllzZo1y3l+FotFZ86cUXBwsBYuXKiVK1cqNTVVkpSenp7nrGth1KhRw+XzN27c6Axy1/vb\nsmXLXMelpKTctN9nz551uRAyJCRE2dnZ+vXXX53bbuxnuXLlbvkzdGOdealevbrL65o1a7osRbld\nvz2X62PfuLTnZj8bvXv31i+//KLx48crLS1NvXr10rhx42S1Wu+4LqA4IuwC8JiaNWsqJCREW7du\n1YwZM1zeq1KlisqXL6/169fnmvGVpIULFyopKUkrV65U1apVdfDgQfXp08ctF3lVq1ZNv/zyi/P1\n6dOn8x3zt+/VqFFDI0eO1IgRI3Ltu2fPHqWkpMjhcOQKYLeq++6773aZxc3IyNDFixdvGbAk6fXX\nX5fFYtH69esVGBioL7/80rl2tbCCg4M1atQo9ejRI9d7u3fv1gcffKC///3v+t3vfidJeuCBB/L9\ny0KFChWUmZnpfH09JN/oxt4EBwcrKioq15KDm9V6s37/tp+nTp2Sr6+v7rrrLucsdWHd6tfwxvAp\nXQvjHTt2lJR/Hwrys/HFF1+4bDt9+rTatGlzy5p9fX31pz/9SX/60590+vRpxcTEqF69ei5/0QTM\nhGUMpYDNZtPcuXMLtE4M7kHPb27GjBlatGiRypUr57LdYrFowIABmjFjhvNimTNnzjjXoKanp6tc\nuXLy9/fXxYsXc60/zcnJ0VdffXVbPe/atav+8Y9/6OjRo7py5YpznXBBDRw4UMuWLdO+ffskXQul\n33zzjTIyMtSoUSNVq1ZNr732mq5cuSK73a7ExERJ1/6J+5dfflFWVlae4/bo0UP/+Mc/dPDgQdnt\ndr3xxhtq3LhxnmtDfys9PV0VK1aUv7+/zpw5ow8//LBQ53SjQYMG6b333tNPP/0kSUpLS9OmTZtk\ns9m0bNkyWa1WVa5cWXa7XXFxcUpPT3cee9ddd+nUqVMu4TcsLEyff/65srOztX//fv3zn/90+bzf\nBuVevXpp8+bN2r59uxwOh65evapdu3blCpKS8u139+7d9fHHH+vkyZNKT0/XnDlz1L17d2cozi+g\n53UeBXHu3DktXrxY2dnZ2rhxo44dO6a2bdvesg9Vq1aVj4+Pjh8/7jJeTk6O5s6dq2bNmik5OVmf\nf/65cnJytGHDBh07dkzt27e/ZU07d+7U4cOH5XA4VKFCBfn6+jKrmw/+PPc8d/ecsFsK2Gw2xcXF\n8RvVg+i5qxtnqWrXru3yT/M3vvfss88qNDRUAwcOVHh4uIYPH66kpCRJUnR0tK5cuaKIiAgNGjTI\nGRiuy8nJ0ebNm2+r523atFF0dLSio6PVuXNnRUZGFur4hg0batq0aXrppZf0wAMPqHPnzs47T/j4\n+Ojdd99VcnKy2rVrp7Zt22rjxo2SpJYtW6p+/fpq1apVnp8ZGRmpMWPGKDY2Vq1bt9bJkyf1xhtv\nON/Pb/Zv9OjR+uGHHxQeHq6RI0eqc+fOLu8XZjb8oYceUkxMjMaNG6fw8HD16tVL27Ztk81m07p1\n69S8eXN17txZHTt2VPny5V1mnrt06SLDMBQREaG+fftKksaMGaPjx4/rgQce0Lx583LdOzavmfP5\n8+frvffeU2RkpNq3b6+FCxfmGTzz63f//v3Vu3dvPfroo+rUqZPKly+vSZMm3fRzb3yd13nk1cPf\nbmvcuLGSk5PVsmVLvfXWW5o7d64qVap0yz6UK1dOI0eO1ODBg/XAAw84/yKVnZ2tuLg4WSwWvfvu\nu/rwww/VsmVLffjhh3rvvfecY+f36/vrr7/qmWeeUfPmzdWjRw9FRESoV69eN92/tOPPc89zd88t\nBjfxM72TJ0+qY8eO+uqrr1SrVi1vl1Mq0HPPK2zPO3TooE8++cTjD7MwE37OPa+oex4fH69du3bp\nlVdecfvYJRU/557n7p4zswugVCoOD0wAABQ9LlADUCpFR0crMDDQ22UAxcrvf/97fl/AdAi7pUSz\nZs24AMGDrFarQkJC6LkHFbbnjz/+eBFXZH78nHteUff8+v2F8V/8nHue1Wot8P3IC4I1u6XA5cuX\n5e/v7+0yAAAACsxd+YWwW4pcuJAuh4Nfbk8JCvLXuXOXvV1GqULPPY+eex499zx67lk+PhZVqeK+\np/qxjKEUcTgMwq6H0W/Po+eeR889j557Hj0vubgbAwAAAEyLsAsAAADTIuwCAADAtAi7AAAAMC3C\nLgAAAEyLsAsAAADTIuwCAADAtHioBAAAwG3ItmfowqUcb5dhOj4+FgUFue/JrzxUohQ5sbC+sm3J\n3i4DAABTqDfWLinN22XgFljGAAAAANMi7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIAAMC0\nCLtekJmZqTFjxqhHjx6KiorSuHHjJEmrV6/WwIED1a9fPw0dOlRJSUmSpPnz5ys2NlaSdOXKFfXs\n2VNbt271VvkAAAAlBvfZ9YLt27crLS1N69evlySlpaVp9+7d2rhxo5YsWaIyZcpo69atmjhxopYu\nXapRo0bpySef1CeffKIff/xR7dq1U5s2bbx8FgAAAMUfYdcL7rvvPh07dkzTpk1TixYt1K5dO339\n9dc6dOiQBg4cKMMwZBiG0tKu3ajaYrFo9uzZ6t27t0JCQjR9+vSbjm2z2WSz2Vy2Wa1WBQcHF+k5\nAQAAuFNKSopyclyfUBcYGKjAwMBCjUPY9YLatWtrw4YNSkhI0NatWzVnzhw99NBD6tevn3O5wm+d\nOHFCPj4+unTpkq5cuaKKFSvmud+iRYsUFxfnsi0kJESbN292+3kAAAAUlSFDhujUqVMu20aPHn3T\nrHQzFsMwDHcWhls7c+aMKlWqpHLlyunKlStq27at5s+frwkTJmjp0qWqXr26HA6HDhw4oAYNGujS\npUsaMGCAZs6cqX//+986duyY3njjjTzHzm9ml8cFAwDgPvXG2pWayuOC3c3Hx6KgIH9mdkuyQ4cO\n6fXXX5ckORwOjRgxQuHh4Ro/frxGjRolh8OhrKwsdenSRQ0aNNALL7yg/v37q1mzZmrSpImGDh2q\n5cuX65FHHsk19u38EAAAABQ37lqCycxuKcLMLgAA7sPMbtG4PrPrtvHcNhIAAABQzBB2AQAAYFqE\nXQAAAJgWYRcAAACmRdgFAACAaRF2AQAAYFrcegwAAOA2ZNszdOFSzq13RKG4+9ZjPFSiFDl37rIc\nDv5u4ynVqgVw/0UPo+eeR889j557Hj0v2VjGAAAAANMi7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAA\nANMi7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi\n7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIAAMC0CLsAAAAwLV9vFwDPCQry93YJpU61agHe\nLqHUoeee582eZ9szdOFSjtc+H0DxR9gtRU4srK9sW7K3ywAAt6k31i4pzdtlACjGWMYAAAAA0yLs\nAgAAwLQIuwAAADAtwi4AAABMi7ALAAAA0yLsAgAAwLS49ZgXPPvss0pKSpLdbldoaKhmzJihgIAA\nzZkzRxs3blSVKlXUokULJSQkaNWqVZKk1atX69NPP1VOTo4CAgI0ZcoU1a1b17snAgAAUMwRdr1g\n0qRJqly5siTpzTff1Pvvv69mzZrpm2++0bp161S2bFnFxsbKYrFIknbv3q2NGzdqyZIlKlOmjLZu\n3aqJEydq6dKl3jwNAACAYo+w6wXx8fFat26dsrKylJmZqbp16yorK0tdu3ZV2bJlJUlRUVF65513\nJElff/21Dh06pIEDB8owDBmGobS0vG+ibrPZZLPZXLZZrVYFBwcX7UkBAAC4UUpKinJyXJ+QGBgY\nqMDAwEKNQ9j1sN27d2vZsmVavny5KleurPXr12v58uWS5JzJ/S3DMNSvXz/FxsbecvxFixYpLi7O\nZVtISIg2b95858UDAAB4yJAhQ3Tq1CmXbaNHjy5QHroRYdfD0tLSFBAQoEqVKslut2vVqlWyWCyK\niIjQ22+/rccff1x+fn5as2aN85gOHTro+eef18CBA1W9enU5HA4dOHBADRo0yDV+dHS0+vTp47LN\narUW+XkBAAC405IlS/Kc2S0swq6HtWnTRmvXrlXXrl1Vo0YNNWzYUPv27VP79u31/fffq3fv3qpe\nvboaN27sXKoQHh6ucePGadSoUXI4HMrKylKXLl3yDLu3M70PAABQ3LhrCabFMAzDLSPhjqWnp6ti\nxYoyDEMvvPCCqlevrjFjxrht/BML6yvbluy28QDA2+qNtSs1Ne9rGMyqWrWAUnfO3kbPPcvHx6Kg\nIH+3jcfMbjHy/PPP69SpU8rMzFTDhg315JNPerskAACAEo2wW4z89sIyAAAA3BmeoAYAAADTIuwC\nAADAtAi7AAAAMC3CLgAAAEyLC9RKkdrDj3i7BABwq2x7hrdLAFDMEXZLkXPnLsvh4LbKnsJ9GT2P\nnnsePQdQ3LGMAQAAAKZF2AUAAIBpEXYBAABgWoRdAAAAmBZhFwAAAKZF2AUAAIBpEXYBAABgWoRd\nAAAAmBZhFwAAAKZF2AUAAIBpEXYBAABgWoRdAAAAmBZhFwAAAKZF2AUAAIBpEXYBAABgWoRdAAAA\nmBZhFwAAAKZF2AUAAIBpEXYBAABgWr7eLgCeExTk7+0SSp1q1QK8XUKpQ889z1s9z7Zn6MKlHK98\nNoCSg7BbipxYWF/ZtmRvlwEAblFvrF1SmrfLAFDMsYwBAAAApkXYBQAAgGkRdgEAAGBahF0AAACY\nFmHXgzp06KCffvrJ22UAAACUGoRdAAAAmBa3Hisi33//vWbPnq309HRZLBY999xzLu9/9NFH2rBh\ng3JycuTn56cpU6YoLCxMmZmZev7553X06FH5+vqqXr16mjNnjn7++WdNnDhRmZmZysnJUd++fTVs\n2DAvnR0AAEDJQNgtApcuXVJsbKzmzZunxo0byzAMpaW53gsyKirKGVYTEhI0efJkLV++XNu3b1da\nWprWr18vSc7jPv30U7Vt21ajRo1y2Q4AAICbI+wWgT179uh3v/udGjduLEmyWCwKDAx02Wf//v16\n//33denSJVksFiUnX3vYw3333adjx45p2rRpatGihdq1aydJatGihV599VXZ7XZFRESoZcuWeX62\nzWaTzWZz2Wa1WhUcHOzmswQAACg6KSkpyslxfUpiYGBgrkx1K4TdImAYRr7vZ2VlacyYMVq6dKnC\nwsJ09uxZtW3bVpJUu3ZtbdiwQQkJCfrmm280Z84crVu3Tg8//LCaNm2qHTt2aMGCBVq1apVmz56d\na+xFixYpLi7OZVtISIg2b97svhMEAAAoYkOGDNGpU6dcto0ePVqxsbGFGoewWwSaNm2qSZMmae/e\nvWrcuLEcDocuX77sfP/q1atyOByqXr26JGnJkiXO986cOaNKlSqpY8eOevDBB9W2bVtdunRJV65c\nUe3atRUVFaU6deror3/9a56fHR0drT59+rhss1qtRXCWAAAARWfJkiV5zuwWFmG3CFSqVElxcXF6\n5ZVXlJGRIavVqgkTJshisUiS/P399cwzz6hfv34KCQlR69atncceOnRIr7/+uiTJ4XBoxIgRqlat\nmt577z2tW7dOZcqUkcVi0aRJk/L87NuZ3gcAAChu3LUE02Lc6t/cYRonFtZXti3Z22UAgFvUG2tX\namrpu1i3WrWAUnne3kTPPcvHx6KgIH/3jee2kQAAAIBihrALAAAA0yLsAgAAwLQIuwAAADAtwi4A\nAABMi7ALAAAA0+I+u6VI7eFHvF0CALhNtj3D2yUAKAEIu6XIuXOX5XBwW2VP4b6MnkfPPY+eAyju\nWMYAAAAA0yLsAgAAwLQIuwAAADAtwi4AAABMi7ALAAAA0yLsAgAAwLQIuwAAADAtwi4AAABMi7AL\nAAAA0yLsAgAAwLQIuwAAADAtwi4AAABMi7ALAAAA0yLsAgAAwLQIuwAAADAtwi4AAABMi7ALAAAA\n0yLsAgAAwLQIuwAAADAtX28XAM8JCvL3dgmlTrVqAd4uodSh557nzp5n2zN04VKO28YDAMJuKXJi\nYX1l25K9XQYA3FS9sXZJad4uA4CJsIwBAAAApkXYBQAAgGkRdgEAAGBahF0AAACYFmHXg7788kt1\n69ZNffv2VVJSkrfLAQAAMD3uxuBBy5cv15gxY9S5c+cCH+NwOOTjw99JAAAAbgdh10NeeeUV7d69\nW0lJSfr0009VrVo1JSUlyW63KzQ0VDNmzFBAQIB27dqlGTNmKDw8XD/88INGjRql5s2ba+bMmTp8\n+LCuXr2qiIgITZw4URaLxdunBQAAUKxZDMMwvF1EafHYY4/pySefVNu2bXXx4kVVrlxZkvTmm2/K\n4XBo/Pjx2rVrl4YNG6alS5eqUaNGkqRJkybpgQceUK9evWQYhp599lm1bNlSAwYMyPUZNptNNpvN\nZZvValVwcDD32QVQ7NUba1dqKvfZzU+1agH0yMPouWf5+FgUFOSvlJQU5eS4PmQmMDBQgYGBhRqP\nmV0viY+P17p165SVlaXMzEzVrVvX+V5oaKgz6ErS5s2btX//fi1cuFCSlJmZqRo1auQ57qJFixQX\nF+eyLSQkRJs3b3b/SQAAABSRIUOG6NSpUy7bRo8erdjY2EKNQ9j1gt27d2vZsmVavny5KleurPXr\n12vFihXO9ytUqJDrmHnz5qlWrVq3HDs6Olp9+vRx2Wa1Wu+8aAAAAA9asmRJnjO7hUXY9YK0tDQF\nBASoUqVKstvtWrVqVb77d+jQQe+//76mTJkiHx8fXbhwQenp6XmG39uZ3gcAAChugoOD3TIOl/l7\n0PULytq0aaPatWura9eueuqpp9SgQYN8j/vrX/8qHx8f9e7dWz179lRMTIzOnj3riZIBAABKNC5Q\nK0W4QA1AcccFarfGxVKeR8896/oFam4bz20jAQAAAMUMYRcAAACmRdgFAACAaRF2AQAAYFqEXQAA\nAJgWYRcAAACmxUMlSpHaw494uwQAyFe2PcPbJQAwGcJuKXLu3GU5HNxW2VO4L6Pn0XPPo+cAijuW\nMQAAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIA\nAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIAAMC0\nCLsAAAAwLcIuAAAATMvX2wXAc4KC/L1dQqlTrVqAVz8/256hC5dyvFoDAADeRNgtRU4srK9sW7K3\ny4AH1Rtrl5Tm7TIAAPAaljEAAADAtAi7AAAAMC3CLgAAAEyLsAsAAADTIux6UFhYmK5cueLxYwEA\nAEorwq4HWSwWrxwLAABQWnHrsSL0xRdfaM6cOapcubLatGnj3L537169/vrrSk9PlyQ988wzatu2\nrSTp66+/VlxcnLKzs2W1WjVz5kz9z//8jwzDkCQZhqGZM2fq119/1cyZM1WmTBnPnxgAAEAJQdgt\nIufPn9ff/vY3rVixQqGhofrggw8kSTabTVOmTNGCBQt01113KTU1Vf3799fnn3+u1NRU/e1vf9PS\npUtVu3ZtZWVlKSsrS9K1md3MzEw9//zzqlWrll5//fU8P9dms8lms7lss1qtCg4OLtoTBgAAcKOU\nlBTl5Lg+GCkwMFCBgYGFGoewW0T27Nmjhg0bKjQ0VJL0yCOP6LXXXtN//vMfnTx5UjExMc7ZWqvV\nquTkZO3Zs0dt27ZV7dq1JUllypRxztwahqGYmBh1795dw4YNu+nnLlq0SHFxcS7bQkJCtHnz5qI4\nTQAAgCIxZMgQnTp1ymXb6NGjFRsbW6hxCLtF5HqQvfH19XW3YWFhWrx4ca5j9uzZk++YERER2rZt\nmwYNGqRgLaQYAAAak0lEQVTy5cvnuU90dLT69Onjss1qtRamdAAAAK9bsmRJnjO7hcUFakWkadOm\n+vHHH3X8+HFJ0meffSZJatCggZKSkrRz507nvvv375cktWrVSt98843zGLvdroyMDOd+o0ePVmRk\npGJiYnT58uU8PzcwMFC1atVy+Y8lDAAAoKQJDg7OlWkIu8VI1apVNW3aNI0YMUKDBw92LkcIDAzU\nO++8o7i4OEVFRalbt26aN2+eJCk0NFTTp0/X2LFj1bt3bw0aNMg5fX99VjgmJkadO3fW8OHDc63N\nBQAAgCuL8dt/b4dpnVhYX9m2ZG+XAQ+qN9au1NQ0b5fhMdWqBZSq8y0O6Lnn0XPPo+ee5eNjUVCQ\nv/vGc9tIAAAAQDFD2AUAAIBpEXYBAABgWoRdAAAAmBZhFwAAAKZF2AUAAIBp8QS1UqT28CPeLgEe\nlm3PuPVOAACYGGG3FDl37rIcDm6r7CnclxEAAO9jGQMAAABMi7ALAAAA0yLsAgAAwLQIuwAAADAt\nwi4AAABMi7ALAAAA0yLsAgAAwLQIuwAAADAtwi4AAABMi7ALAAAA0yLsAgAAwLQIuwAAADAtwi4A\nAABMi7ALAAAA0yLsAgAAwLQIuwAAADAtwi4AAABMi7ALAAAA0yLsAgAAwLR8vV0APCcoyN/bJZQ6\n1aoFFHjfbHuGLlzKKcJqAAAofQi7pciJhfWVbUv2dhm4iXpj7ZLSvF0GAACmwjIGAAAAmBZhFwAA\nAKZF2AUAAIBpEXYBAABgWqYNu/Hx8XrmmWfcMlZYWJiuXLlSoH3T0tL0wQcfuOVzAQAAcGdMG3Yl\nyWKx3NHxOTk5hR7n0qVLhF0AAIBiotjeemzfvn167bXXlJ6eLkl65pln9Lvf/U79+vXTwIEDtW3b\nNl29elWzZ8/WsmXLtHfvXpUvX17z589XUFCQpGuzrM8884ySk5NVpUoVvfrqq7r77rt1+PBhTZ06\nVVeuXJHdbtfAgQP1+OOPS5ImTpyoihUrKikpSRcuXNCqVatkGIYkyTAMzZw5U7/++qtmzpypMmXK\n5Kp72rRpunz5svr06aNy5cpp6dKlSk5O1uTJk3X+/Hn5+vpq3Lhxat26tZYvX65Dhw7pxRdf1L59\n+zRw4ECtXLlSDRs21NSpU3X//fdrwIABCgsL07hx4/Svf/1Lly5d0oQJE9SpUycP/UoAAACUXMUy\n7KalpWny5MlasGCB7rrrLqWmpqp///567733dPHiRYWHh2v8+PH68MMPNXToUH3yySeaNm2apk6d\nqk8++URjxoyRJCUmJmrNmjUKDQ1VXFycpk+frrffflu1atXSxx9/rDJlyigjI0MDBgxQq1atdM89\n90iS9uzZoyVLlqhs2bKSrs3sZmZm6vnnn1etWrX0+uuv37T2F198Uf3791d8fLxz23PPPadBgwap\nb9++Onr0qIYMGaKNGzcqMjJSixYtkiR9++23atq0qRISEtSwYUMlJCToiSeecI4REBCglStXKjEx\nUWPHjr1p2LXZbLLZbC7brFargoODb+NXAgAAwDtSUlKc/8p+XWBgoAIDAws1TrEMu4mJiTp58qRi\nYmKcs6pWq1XZ2dmqWLGi2rRpI0m6//77VaNGDd13332SpAYNGighIcE5TvPmzRUaGipJGjBggHr1\n6iVJunLliiZPnqyDBw/Kx8dHqampOnjwoDPsdu7c2Rl0pWszujExMerevbuGDRtWqHNJT0/XwYMH\n1bdvX0nSvffeq9///vfau3ev2rVrp8zMTJ05c0YJCQn685//rPnz56tnz57KyspSrVq1nON069ZN\nktSkSROlpqbKbrfLz88v1+ctWrRIcXFxLttCQkK0efPmQtUNAADgTUOGDNGpU6dcto0ePVqxsbGF\nGqdYhl3p2kVhixcvdtl26tQpl4BntVpdQun1QHwz19fevvHGG6pWrZpeffVVWSwWPfHEE7Lb7c79\nKlSokOvYiIgIbdu2TYMGDVL58uULfB7Xw/rNamnZsqW2bNmic+fOKTw8XKmpqdqyZYtatmzpsu/1\n8/TxubbM+rd/07kuOjpaffr0cdlmtVoLXC8AAEBxsGTJkjxndgurWF6g1rRpUyUlJWnnzp3Obfv3\n75dhGDcNj3lJTEzU8ePHJUmrVq1SRESEpGvLJIKDg2WxWHT48GHt3r37lmONHj1akZGRiomJ0eXL\nl2+6n7+/vzIzM52/OP7+/vr973/vXNZw9OhRHTp0SI0aNZJ0Ley+9957atasmfPc33//fUVGRjrH\n/O0559eDwMBA1apVy+U/ljAAAICSJjg4OFemuZ2wWyxndgMDA/XOO+9o1qxZeuWVV2S321WnTh29\n8MILhbozQosWLfT222/ryJEjzgvUJGnUqFGaMGGC1q5dqzp16qhFixb5jnP9M2NiYlSuXDkNHz5c\nH3zwQZ4Nr1Spknr27KmePXuqUqVKWrp0qWbPnq0XX3xRH330kXx9fTV79mxVqVJF0rWwm5KSogcf\nfFCSFBkZqc8++yzXzG5e9QAAACB/FqMwU6Uo0U4srK9sW7K3y8BN1BtrV2pqmrfLKNGqVQughx5G\nzz2PnnsePfcsHx+LgoL83Tee20YCAAAAipliuYyhJJg8ebL27t3rXFJgGIZ8fX21cuVKL1cGAACA\n6wi7t2nq1KneLgEAAAC3wDIGAAAAmBZhFwAAAKZF2AUAAIBpsWa3FKk9/Ii3S0A+su0Z3i4BAADT\nIeyWIufOXZbDwW2VPYX7MgIA4H0sYwAAAIBpEXYBAABgWoRdAAAAmBZhFwAAAKZF2AUAAIBpEXYB\nAABgWoRdAAAAmBZhFwAAAKZF2AUAAIBpEXYBAABgWoRdAAAAmBZhFwAAAKZF2AUAAIBpEXYBAABg\nWoRdAAAAmBZhFwAAAKZF2AUAAIBpEXYBAABgWoRdAAAAmJavtwuA5wQF+Xu7hCKTbc/QhUs53i4D\nAAAUM4TdUuTEwvrKtiV7u4wiUW+sXVKat8sAAADFDMsYAAAAYFqEXQAAAJgWYRcAAACmRdgFAACA\naXks7E6aNEn/93//J0maOHGilixZkud+N763bNkyLVq0yFMlAgAAwGQ8djeG6dOnF/qYQYMGFUEl\nAAAAKC1uGXbDwsI0btw4/etf/9KlS5c0YcIEderU6ab7f/nll3rrrbfk6+ur7Oxsvfjii2rRooUe\ne+wxPfnkk2rbtq0k6eDBgxo2bJh++eUXtWjRQi+++KJ8fV3LiYuLU0ZGhiZMmKD4+HitX79egYGB\nOnLkiAIDAzV37lwFBQUpKytLL730knbt2qW77rpLYWFhSk1N1dtvv63ExERNnz5dhmEoOztbo0aN\nUrdu3fKs/fz58/rzn/+sc+fOSZIefPBB/eUvf1F8fLzWrVsnf39/JScnq0qVKnr11Vd19913y+Fw\naPbs2dq+fbskqVWrVpowYYIsFkuuc77xdVxcnDZs2KCyZcvKYrHo73//u/z9/bVv3z699tprSk9P\nlyQ988wzatu27U1rAwAAwM0VaGY3ICBAK1euVGJiosaOHZtv2J07d66mTJmi5s2byzAMZWRk5Lnf\nvn37tHz5cvn5+SkmJkbLly/XkCFD8q3jhx9+0Nq1a1W9enX97W9/0+LFizV27FgtW7ZMv/zyizZt\n2qSsrCw99thjqlGjhiTpgw8+0NChQ9WrVy9J0uXLl286/tq1axUSEqKPPvpIkpSW9t/7tiYmJmrN\nmjUKDQ1VXFycpk+frrffflvLli3ToUOHtHr1ahmGoSeffFLLly/Pd1baZrNp4cKF+vbbb+Xn56eM\njAyVK1dOaWlpmjx5shYsWKC77rpLqamp6t+/vz7//PN8a/vt2DabzWWb1WpVcHBwvr0FAAAoTlJS\nUpST4/rAqMDAQAUGBhZqnAKt2b0+E9qkSROlpqbKbrffdN/IyEjNmjVLH374oX766SdVrFjxpmOW\nK1dOPj4+ioqK0s6dO29ZR9OmTVW9enVJUuPGjXXixAlJ0q5du9S7d29ZLBb5+fmpe/fuzmMiIiL0\n/vvv65133tG+ffvk73/zp4g1adJEO3bs0OzZs7VlyxaVL1/e+V7z5s0VGhoqSRowYICz3m+//VZ9\n+vSR1WqVr6+v+vbtq3//+9/5noe/v7/uuecePfvss/rss8+Unp4uHx8fJSYm6uTJk4qJiVFUVJRi\nYmJktVqVnJycb203WrRokTp27Ojy363+EgEAAFDcDBkyJFemuZ1ruW45s2uxWFS2bFlJko/PtWz8\n25R9o7/85S86cuSIvv32W40ZM0bDhg3TgAED8v0MwzAKVOz1OqRrs5XZ2dnO4y0WS57HREdHq0OH\nDkpISNC0adPUqlUrjRkzJs99mzRpotWrV2vHjh1as2aN3n//fX366ad57nv98/L67OuvfX195XA4\nnNuv/yXBx8dHK1asUGJiohISEtS3b199+OGHkq4tG1m8eHGen1mQ2qKjo9WnTx+XbVarNc/xAAAA\niqslS5bkObNbWLcMu78NorcKpj///LPq16+v+vXrKz09Xfv3788z7G7atEnR0dHy9fXV2rVr1aFD\nh0KW/l8RERFau3atunTpouzsbG3YsME5A5yUlKS6deuqdu3aKl++vFavXn3TcU6ePKkaNWqoW7du\nat68uTp37ux8LzExUcePH1edOnW0atUqRURESLq2djY+Pl5dunSRYRhavXq1unTpIkmqXbu29u/f\nr/bt2+unn37SgQMHJEnp6enKyMhQeHi4wsPDtWfPHh05ckStW7dWUlKSdu7c6Rx///79+sMf/pBv\nbTe6nel9AACA4sZdSzALNLOb3+vfev3115WcnCyr1arAwEC9/PLLeR4XHh6up59+WikpKWrRooUG\nDhxY2NqdBg0apEOHDqlHjx4KDg5Ww4YNlZmZKUlavHixdu7cqTJlyqhs2bKaNGnSTcfZtWuXPvro\nI1mtVhmGoalTpzrfa9Gihd5++20dOXLEeYGaJD3yyCM6fvy4cza1devWznAfExOjMWPGaOvWrbrv\nvvt0//33S7q2bjg2NlZXr16Vw+FQgwYN1KlTJ/n5+emdd97RrFmz9Morr8hut6tOnTp69913860N\nAAAAebMYBV1DUMylp6erYsWKstvtGjVqlLp27ar+/fu7Zez4+Hht2bJFb731llvG85YTC+sr25bs\n7TKKRL2xdqWm5n3RnrdUqxZQ7GoyO3ruefTc8+i559Fzz/LxsSgo6ObXWBWWx+6zW9SGDRsmu90u\nu92uBx98UH379vV2SQAAAPCy2wq758+f1/Dhw3NdpNWpUyc9/fTTbi2woFasWFHgfSdPnqy9e/e6\n1O/r66uVK1fmuX+fPn1yXfQFAACA4u+2wm7VqlXzvdCruGO9KwAAQOlQoPvsAgAAACURYRcAAACm\nZZoL1HBrtYcf8XYJRSbbnvdjqQEAQOlG2C1Fzp27LIfDFHeaAwAAKBCWMQAAAMC0CLsAAAAwLcIu\nAAAATIuwCwAAANMi7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIAAMC0CLsAAAAwLcIuAAAA\nTIuwCwAAANMi7AIAAMC0CLsAAAAwLcIuAAAATIuwCwAAANMi7AIAAMC0CLsAAAAwLcIuAAAATMvX\n2wXAc4KC/G/72Gx7hi5cynFjNQAAAEWPsFuKnFhYX9m25Ns6tt5Yu6Q09xYEAABQxFjGAAAAANMi\n7AIAAMC0CLsAAAAwLcIuAAAATIuwWwLExcXp1Vdf9XYZAAAAJQ5ht5hwOBzeLgEAAMB0uPWYG4SF\nhWn06NHavn27Ll26pHHjxunhhx+WJD377LNKSkqS3W5XaGioZsyYoYCAAO3atUszZsxQeHi4fvjh\nB40aNUrNmzfXjBkztH//flmtVoWHh2vSpEmSpDNnzuipp57SiRMnFBoaqrfeektly5b15mkDAAAU\ne4RdN7FarVq2bJl+/vlnDRo0SOHh4apataomTZqkypUrS5LefPNNLViwQOPHj5ckHTlyRC+99JIz\n0E6cOFEVK1bUunXrJEkXL150jv/DDz9o1apV8vf31xNPPKG1a9dqwIABHj5LAACAkoWw6yb9+/eX\nJNWrV08NGzbU3r171b59e8XHx2vdunXKyspSZmam6tat6zwmNDRUjRo1cr7esmWLVq9e7Xx9PSRL\nUuvWreXvf+0JaI0aNdKJEyfyrMNms8lms7lss1qtCg4OvuNzBAAA8JSUlBTl5Lg+vTUwMFCBgYGF\nGoew6yaGYTj/3+FwyGKxaPfu3Vq2bJmWL1+uypUra/369VqxYoVzvwoVKriMYbFYXMa5kZ+fn/P/\nrVarrl69mud+ixYtUlxcnMu2kJAQbd68udDnBAAA4C1DhgzRqVOnXLaNHj1asbGxhRqHsOsm//jH\nPzRy5EglJSXp4MGDatSokfbu3auAgABVqlRJdrtdq1atyneMdu3a6YMPPnAua7hw4YKqVKlSqDqi\no6PVp08fl21Wq7VwJwMAAOBlS5YsyXNmt7AIu27i5+enwYMH6+LFi5o2bZqqVq2qNm3aaO3atera\ntatq1Kihhg0bat++fTcdY+LEiZoxY4Z69OghX19ftWjRQi+88EKh6rid6X0AAIDixl1LMC3Gzf7d\nHAUWFham77//XuXLl/d2Kfk6sbC+sm3Jt3VsvbF2paamubkic6tWLYCeeRg99zx67nn03PPouWf5\n+FgUFOTvvvHcNlIpZrFYvF0CAAAA8sAyBjc4cOCAt0sAAABAHpjZBQAAgGkRdgEAAGBahF0AAACY\nFmEXAAAApsUFaqVI7eFHbvvYbHuGGysBAADwDMJuKXLu3GU5HNxWGQAAlB4sYwAAAIBpEXYBAABg\nWoRdAAAAmBZhFwAAAKZF2AUAAIBpEXYBAABgWoRdAAAAmBZhFwAAAKbFQyVKER8fi7dLKHXouefR\nc8+j555Hzz2PnnuOu3ttMQyDR2qZ3OXLl+Xv7+/tMgAAAArMXfmFZQylwMWLFzV48GClpKR4u5RS\nIyUlRR06dKDnHkTPPY+eex499zx67nkpKSkaPHiwLl686JbxCLulRGJionJycrxdRqmRk5OjU6dO\n0XMPoueeR889j557Hj33vJycHCUmJrptPMIuAAAATIuwCwAAANMi7AIAAMC0rFOmTJni7SJQ9MqW\nLauIiAiVLVvW26WUGvTc8+i559Fzz6PnnkfPPc+dPefWYwAAADAtljEAAADAtAi7AAAAMC3CbgmW\nlJSkQYMGqUuXLho0aJCOHz+eax+Hw6GpU6eqU6dO6ty5sz777LMCvYe83WnP58+frx49eigqKkr9\n+vXT9u3bPVl+iXSnPb/u2LFjatKkiV599VVPlF2iuaPnGzZsUM+ePdWzZ0/16tVL58+f91T5JdKd\n9vz8+fMaMWKEevXqpW7duumll16Sw+Hw5CmUOAXp+Y4dO9SvXz/94Q9/yPVnB9+hhXenPb/t71AD\nJdbjjz9urFu3zjAMw1izZo3x+OOP59onPj7eeOKJJwzDMIxz584Zbdq0MU6dOnXL95C3O+359u3b\njczMTMMwDOPAgQNGeHi4cfXqVQ9VXzLdac8NwzBycnKMRx991Pjzn/9szJo1yzOFl2B32vN9+/YZ\n3bt3N86dO2cYhmGkpaXxc34Ld9rzl19+2fmznZ2dbQwYMMDYuHGjh6ovmQrS8+PHjxsHDhww3nzz\nzVx/dvAdWnh32vPb/Q5lZreEOn/+vA4cOKDu3btLknr06KEff/xRFy5ccNlv48aNGjhwoCSpatWq\neuihh7Rp06Zbvofc3NHzP/7xj84rS8PCwiQp1/H4L3f0XJLef/99dejQQXXr1vVY7SWVO3q+aNEi\nDR8+XFWrVpUk+fv7y8/Pz4NnUbK4o+cWi0Xp6ekyDEOZmZnKzs5W9erVPXsiJUhBe167dm2FhYXJ\narXmGoPv0MJxR89v9zuUsFtCpaSkqHr16rJYLJIkHx8f3X333frll19c9jt9+rRq1qzpfB0cHOx8\nvnd+7yE3d/T8RvHx8apduzZfSPlwR88PHjyoHTt2aOjQoR6ruyRzR8+PHj2q48eP69FHH1Xfvn31\nzjvveO4ESiB39Pzpp5/Wzz//rFatWql169Zq1aqVmjZt6rmTKGEK2vP88B1aOO7o+Y0K8x1K2AW8\nYNeuXZo7d67mzJnj7VJMLTs7Wy+++KKmTJni/AMWRS87O1uHDx/Wxx9/rMWLF2vr1q1as2aNt8sy\ntY0bNyosLEw7duzQ1q1btWvXLn3xxRfeLgsoEoX9DiXsllDBwcE6c+aMjP9/m2SHw6GzZ8+qRo0a\nLvvVrFlTp0+fdr5OSUlRcHDwLd9Dbu7ouSR9//33ev755zV//nyFhoZ6pvgS6k57npqaqhMnTuip\np55Shw4dtGjRIn322Wd68cUXPXoeJYk7fs5DQkLUuXNn+fr6qmLFiurYsaP279/vuZMoYdzR8yVL\nlqhnz56Sri0b6dixo3bu3OmhMyh5Ctrz/PAdWjju6Ll0e9+hhN0SqmrVqgoLC9O6deskSevWrdP9\n99+vKlWquOzXpUsXrVixQoZh6Pz58/rqq6/08MMP3/I95OaOnu/bt0/jx4/XW2+95VxvhJu7054H\nBwcrISFBX331lTZv3qzo6GgNGDBAL730kjdOp0Rwx895jx49tGPHDklSVlaWEhISdN9993n2REqQ\nO+l5586dJUm1atXStm3bJEl2u10JCQmqX7++Z0+kBCloz29k/OYZXHyHFo47en7b36GFv5YOxcXR\no0eNAQMGGJ07dzYGDhxoJCUlGYZhGDExMcYPP/xgGMa1q9AnT55sPPTQQ0anTp2MFStWOI/P7z3k\n7U573q9fPyMyMtKIiooyevfubURFRRmHDx/2yrmUFHfa8xvNnTuXuzEUwJ323OFwGK+88orRtWtX\no0ePHsbMmTO9ch4lyZ32/Pjx48awYcOMnj17Gt27dzemTZtm5OTkeOVcSoqC9Hz37t1GmzZtjObN\nmxvNmjUz2rZta2zfvt0wDL5Db8ed9vx2v0N5XDAAAABMi2UMAAAAMC3CLgAAAEyLsAsAAADTIuwC\nAADAtAi7AAAAMC3CLgAAAEyLsAsAAADTIuwCAADAtP4fEL6gVue8AE8AAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "# Plot.\n", - "dfc_mean = df_dfc.abs().mean()\n", - "N = 8\n", - "sorted_ix = dfc_mean.abs().sort_values()[-N:].index # Average and sort by absolute.\n", - "ax = dfc_mean[sorted_ix].plot(kind='barh',\n", - " color=sns_colors[1],\n", - " title='Mean |directional feature contributions|',\n", - " figsize=(10, 6))\n", - "ax.grid(False, axis='y')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z0k_DvPLaY1o" - }, - "source": [ - "You can also see how DFCs vary as a feature value varies." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 297 - }, - "colab_type": "code", - "id": "ZcIfN1IpaY1o", - "outputId": "abdd429c-8318-4641-aef3-e58e408ac74f" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAEYCAYAAABslZDKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8U2W+P/DPSdKkSdMk3ZuWtSwWKKsgOuhPEZVFZOmg\n1GGbAYfRAWRWbh0HccDlytzRl7IoOjiDjqPMdUCpCoOCc7lylUVQKDu0paVJF5KmaZtmPef3R9qk\nadI2aZOT7ft+vXiNPTk5efJMzvM9z85wHMeBEEII4Ykg3AkghBASXyjwEEII4RUFHkIIIbyiwEMI\nIYRXFHgIIYTwKi4Dj9FoxJYtW2A0GsOdlLCjvHCjvHCjvHCjvHALVl5EXOCpqKhAUVERZsyYgaKi\nIlRWVnqds2fPHsyZMwfz5s3DnDlz8O677wb0GUajEVu3bqUfEigvOqK8cKO8cKO8cAtWXoiClJ6g\n2bBhAxYvXozZs2dj3759WL9+PXbt2uVxzvTp01FYWAgAMJlMmD17NiZPnozhw4eHI8mEEEICEFE1\nHr1ejwsXLuDBBx8EAMyePRvnz59HQ0ODx3lJSUmu/zaZTLDb7WAYhte0EkII6Z2ICjxarRZZWVmu\nICIQCJCZmYmamhqvcw8fPozZs2dj2rRpWLFiBYYNG8Z3cgkhhPRCxDW1+evee+/Fvffei5qaGvz8\n5z/H3XffjUGDBnmdZzQavdoja2pqMGHCBAiFQp5SG7mEQiFyc3MpL0B50RHlhRvlhZtQKMSECRN8\nVgYUCgUUCoVf12Eiaa02vV6PGTNm4NixY2AYBizLYvLkyTh48CBSUlK6fN+GDRswePBg/PjHP/Z6\nbcuWLdi6davHsQkTJuD9998PdvIJISQuPProozh16pTHsdWrV2PNmjV+vT+iajypqanIz89HSUkJ\n5syZg5KSEowcOdIr6JSVlSEvLw+AM1gdO3YM06dP93nNZcuWYf78+R7H2p9cGhpawLIRE3fDJi1N\nDp2uOdzJiAiUF26UF26UF04CAYOUlCS8/PLLcDgcHq/5W9sBIizwAMCzzz6L4uJibN++HUqlEps3\nbwYArFy5EmvXrsWoUaOwe/duHD16FAkJCeA4DkuWLMEPfvADn9frrvrHshwFnjaUD26UF26UF26U\nF25qtbpP74+opja+6XTN9GMCkJGRjPr6pnAnIyJQXrhRXrhRXjgJBAzS0uR9v04Q0kIIIYT4jQIP\nIYQQXlHgIYQQwisKPIQQQnhFgYcQQgivKPAQQgjhFQUeQgghvKLAQwghhFcUeAghhPCKAg8hhBBe\nUeAhhBDCKwo8hBBCeEWBhxBCCK8o8BBCCOEVBR5CCCG8osBDCCGEVxR4CCGE8Critr4mhISevsmM\nD764guZWO+RSEYruG4bU5MRwJ4vECarxEBKHPvjiCuoMrWi12lFnaMXuQ1fCnSQSR6jGQ0gcam61\ng2EYAADDMGgy2cOcosjSuUa46pHx4U5STKEaDyFxSC4VgeM4AADHcZBL6Rm0o841wp37SsOdpJhC\nvzZC4lDRfcOw+9AVNJncfTyxLpB+rc41QmOLlc+kxjwKPITEodTkRDwxbzSvn+lPwR/KQQ/ttRiG\nYWCy2LD70JUu80AuFcFksYFhGHAcB4VMHJQ0BEMsDAyJuMBTUVGB4uJiGAwGqFQqbN68GQMGDPA4\nZ/v27fjss88gEokgFArxy1/+EnfeeWeYUkwI8Yc/BX8gwSFQgfRrda4RrphbAM4WnH6w767W4/WP\nSuFwcBAKGTwxrwDjhmZ0eX7nQGO1szA0W0KSR3yJuMCzYcMGLF68GLNnz8a+ffuwfv167Nq1y+Oc\nsWPHYsWKFZBIJLh48SKWLFmCo0ePQiyOnKcSQognfwr+UA566FyL6a5fq71G2F7ov/L+KYiFTJe1\ni0BqIa9/VAqbnQMDwGbn8PpHpdjxm6ldpqVzMG5osiBV4bx2tA4MiajBBXq9HhcuXMCDDz4IAJg9\nezbOnz+PhoYGj/OmTJkCiUQCAMjPzwcAr3NIeOibzNi+9yw2//00tu89C32TOdxJIhHCnwENoRz0\nUHTfMGSlSCEVi5Cpknr0a3X1u20v9E1mW7fDzgMZnu5wOIMOADBtf3enczAGh6gfGBJRKdZqtcjK\nynJlskAgQGZmJmpqapCSkuLzPXv37kX//v2RlZXl83Wj0Qij0ehxTCgUQq1WBzfxBEBom0pIdPNn\nQEMoBz1016/V1e/W3xpYIDU1oZBx1Xg4ACIh0+W5gHdNbbA6GRKxMKwDQ7RaLRwOh8cxhUIBhULh\n1/sjKvAE6vjx49iyZQv+8pe/dHnOrl27sHXrVo9jubm5OHz4MNLS5KFOYtTIyEgOynWsDg7iBKHr\nb4udC9q1+RJt6Q2lYOZFRkYynsnrui/D33NCoavfbbpKCq2uGYAzQKSrpD7zpP289uDQ1XkAULzs\nNry06wTsDhYJQgH+Y9mkbvN51SPjsXNfKYwtVihkYqyYW4B0lbSP37hvFi1ahOrqao9jq1evxpo1\na/x6P8O119kigF6vx4wZM3Ds2DEwDAOWZTF58mQcPHjQq8Zz+vRp/OpXv8Lrr7/uam7zpbsaj07X\nDJaNmK8fNhkZyaivbwrKtbbvPet6cuQ4DpkqKX4+P3pqPMHMi2Aq0zZi256zsFhZSMQCrCocjTy1\nMqSfGal5EQpd/W71TWbsPnQFFjvXYx9P55patI0084dAwCAtTd7nGk9EBR4AWLp0KRYsWIA5c+bg\n448/xp49e7wGF5w5cwZr167Fq6++ijFjxvT6syjwOAWzgIn2GzBSC9tfb/sKzaYOHeOyBPxpVWhH\nckZqXoRCT7/beMqL7rQHnr6KuMBTVlaG4uJiGI1GKJVKbN68GQMHDsTKlSuxdu1ajBo1CgsWLIBG\no0FWVhY4jgPDMNi8eTOGDQusrZMCjxPdVG6RmherXzkCu4N1/S0SCrD1l/8vpJ8ZqXkRDpQXTsEK\nPBHXx5OXl4d//OMfXsfffPNN139/+OGHfCaJkLCTiAWwmRyuGo9EHFEDUgkJCP16CYkCqwpHQy5L\ngEgogFyWgFWF0dNvRqIfy3KoazDhYqUhKNeLuBoPIXyItmVH8tTKgPt0ou07kvBrbrWhRmeCVt+C\nGr0JtfpW1OhNqGswwe7gkJkixZTx/fr8ORR4SFyKh/lG8fAdSeBsdhZ1hlbU6EyoaQsw7UGmudXW\n7XuFgu7nHPmLAg+JS/GwH008fEfiG8dxMDRbUaNrQU1De5BxBpqbjWb0NKRMmSRGdqoM2WkyZKU4\n/1edKkNWanDmD1HgIXEpkHW7olU8fMd4Z7baUatvdTaN6UyobQ8yDSZYrI5u3ysWCZCVKnMGmLYg\nk53qDDSyRN+/FQHVeAjpvXjYjyYevmM8aK+9aHUt0OpM0OhaXDWYhiZLt+9lAKQpE13BJSvVXXtR\nJUsgYIITSAJFgYfEpXDsR8O3ePiOscTBsqhrq7Fo2oKMtq0fptXSfe1FJhG5aiwdazCZKqnHUkCR\nggJPDIj00UuRnj5C+GR3sNDqTLhR3+ysxdw0Qas3oVZvgqObCe0MA2SopFCnyqBOS3IHmjQZkqUJ\nrv68aECBJwZE+uilSE8fIaHS2GLFjbpmVNU1o6quCVV1LdDqWroNMGKRwBVQctKSoE5PcnXsJ4gi\nr/bSGxR4YkCkj16K9PQR0lcsy0GrN6GypskZZOqdwcbYYu3yPUmJIuSkJ0GdlgR1mrMWk5MmQ6oy\nMWx9L3yhwBOhAmmeiqTRS+3pbmiyQN9kRpoiETqjGTKJCCKRMOzpi0fU1BlcHMehvtGMsupGVNQ0\noUJrxPXaZlhsvvthGAbITpWhf6a8w79kqOTiqGoeCyYqASJUIM1TkTR6qT3dNw1m2OwOWGwsVEkJ\nMFnsyJJJwp6+eERNnX3TarHj+yv1OHW+BmUaI65pGtFk8j3RUiYReQSYfply5KYnRWQHfzhR4IlQ\ngTRPRdLopfZ0O1i2bU8lDgkJIiiSErHuR+PDnby4RE2d/mM5DlqdCWXVjbimMaJM04jqmy0+J1wm\nioUYlJ2MwWoFBqkVGJidjAxlYtzWYgJBgSdCRVLzWSDkUhGMLWbYHRwcLAehgIHd7oBcSU074RKt\nvyU+NLfaUKZpxLVqZ5Ap0zah1eIdmBkGyElPwpAcBfJylBiSo4A6LSloEyrjDf0CI1QkNZ8Foui+\nYXj+nZMQMBwgYCAUMDBZ7FGT/lgUrb+lYGNZDjfqm3G12h1oahtafZ4rlyYgL0fhDDS5Stw2Ogct\nTWaeUxy7KPBEqEhqPgtEanIislKSoEiSuI5JxSLqzA6jaP0t9ZXN7kC5tgmXqwy4cqMRV6sNPidi\nCgUM+mXKMSRHgSE5SuTlKpCpkno0mckSEyjwBBEFnjjUeZTTqkeC2/dCTTskHExmG65WN+JyVSMu\n3zCgQmuE3eHdOaOUizE0V+kMMjnOvhkJdf7zikqEONR5lNPOfaVYPjM/aNenph3Ch4YmCy5XGXD5\nhgFXqhpRXd8MX9My1WkyDOunxLB+Kgzvr0I6DQAIOwo8cajzKKfuJrn1Rrw27ZDQ4dpGm125YcDl\nqkZcuWHAzUbvpi8Bw2BgthzD+qmc//oroZCJw5Bi0h0KPGFUpm3Etj1nYbGykIgFWFU4GnlqJQBn\nc9jre86gTOt8ihMJGfx8fgHGDc3o8+d2bgoL1o1JExVJsNgdLCprm9v6Z5x9NL42KRMnCDAkR4lh\n/ZQY3l+FvBwFEsVUrEU6huN62hIodul0zWC7WTMp1H697Ss0m2zgOA4O1tnJOW5YOoruG4YPvriC\nk5fqPc5PEDHY8Zupff5cfZPZoyls1SPjwdn6PrfjhXdP4Gp1k+vvobkK/G7JxD5fl08ZGcmor2/q\n+cQ4wGdeOFgWFTVNuHi9ARcrDbh6o9HnSgByaYIryAzrp8KALDlEQkHI00e/CyeBgEFamrzP16FH\ngzCyWJ2TLO0sBw7OyWt1hla8c+AiyrXeP3KHj45SIPCaRuemsHSVNCg31bXqpk5/G/t8TRKbWJbD\n9domXKxswMXrzn4aXxuXpSsT2/pmnMEmO1VG/TMxgAJPGEnEAthMDoBzbtjEMAwYhkG5pgl2lvU6\nXyj0fcO9c+ASrlQZwMF5nXcPXMLah8eGNO2+dA6LcVuVJl44joNGZ8K5Mh0uXG/A5Ru+hzZnpkiR\nPyAF+QOcAwFSFdRUG4siLvBUVFSguLgYBoMBKpUKmzdvxoABAzzOOXr0KF5++WVcvnwZS5Yswbp1\n68KU2r5ZVTga2/achbHF2dyWmSIFxzmjUGqyBLVWE+xtpbdIyOCJeQU+r3OtuhHmDs0SV6sb+Ui+\nF5GQ8Ri+KuoiUJLQipS+tlaLHecrGlBarkNpmQ46o/dumenKROQPdAaa/AEpFGjiRMQFng0bNmDx\n4sWYPXs29u3bh/Xr12PXrl0e5wwYMADPP/88/vWvf8Fi6X7r10imkkswJEfpWsk5KVEElVwClVwC\nQ7MFuVnJzoCkkuLn87seJWaxOdAWr8C1/R1M/hZkP59fgNc/KoXDwUHYTaAkoRWuRUE5jkNlbTNK\ny3U4W6bHtepGr31nVHIxRg5KddZqBqqQrpSGPF0k8kRU4NHr9bhw4QIefPBBAMDs2bOxadMmNDQ0\nICUlxXVe//79AQBffPFFWNIZLB0LiGSZGCnJEjwxb7RX539P82DEIgEcDmewYdr+DlU6uyvIxg3N\nCMrgB9I3fC4K2txqw7lyPUrLdCgt16Ox09B8oYDBsH5KjM5LQ0FeGvplJFEfDYmswKPVapGVleX6\nYQoEAmRmZqKmpsYj8ATCaDTCaPTs5BYKhVCr1X1Ob191VUAEOg9maK4SV6obwbIcBAIGQ3OVvKST\nRKZQrhzBshzKa4woLdPjbJkO5RqjV19eujKxLdA4azZSSUQVMyQItFqt62G3nUKhgEKh8Ov9Mf+L\n2LVrF7Zu3epxLDc3F4cPHw7KsMC+SFdJodU1uwqIdJUUGRnJXufdNLRi58elMJqsUMjEWDG3AOkq\ndxPFLxdNxM59pTC2+H7dn+v4+txA0xkrov27rXpkvF+/B39kZCSjwWjGqUt1OHWxDqcv13ntRSMW\nCVAwNB233pKJW0dkISc9Nms10f67CKZFixahurra49jq1auxZs0av94fUfN49Ho9ZsyYgWPHjrXt\n5cJi8uTJOHjwoM8az9atW2EymbodXNBdjSfc83h8Nan56jvZvvesq6mL4zhkpUh71Wbf1XV6mqPg\nbzojpVO7L2i+hnPy5rXqRpTVNuNYqRaVtc1e52SnyjA6Lw2j81IxvL8q5jc6o9+FU/s8npiq8aSm\npiI/Px8lJSWYM2cOSkpKMHLkyG6b2XqKm4FkBt/8bVILVlNXb6/jbzppp8vopWs042y5DqVlepyv\n0MPcaU6NRCzEyIEpKMhLQ8HgVGT0sgZFYkNfuyoiKvAAwLPPPovi4mJs374dSqUSmzdvBgCsXLkS\na9euxahRo/Dtt9/iV7/6FVpaWsBxHPbv34/nn38eU6ZMCXPqQyNYbfahXjWa+oKih93B4lKVAWev\nOQcFaG62eJ0zSK3AiAEqjM5Lw9B+Sl5WCCDxIaKa2vgW7qY2f3XV1BVo01ZX1wlWM0LnpryehoFH\nolhuUmkx23D2mg7fXb2Js2U6rwmcMokIowanoiAvFQWD0zA8Lz1m8yJQsfy7CESwlsyhwBMFgacr\nwer7CdZN5W9fUCSLtQLG0GzBqcv1+PZSPS5VGsB2ut0HZSe39dWkYXBOMoQCd60m1vKiLygvnGit\nNoKGJgtuGsxwsCyEAgESIqApJH4fYyKHrtGMby/X4+SlOly70egx3DlBJMCoQakYNywdY4akQSWX\ndHkdQkKFAk8U0zeZYbM7wDAMbHYH9GHempcGF4RPXYMJ316qx8lL9SjXeo7ilElEGDcsHbcOz8DI\nwam02yYJOwo8USxNkQiLjXVOHGUESAvzOlc0uIBfWl0LTl6sw7eX6lFZ5znkWS5NwITh6Zh4Syby\nB6bQwAAfYmH4f7SiwBPFVHIJrHbW1ccT7maT3oyao5s/MA1NFnxzrgbfnK9FVadgo0wSY8ItGZh4\nSyaG91d69NcQb1RDDx8KPFGmY0EtFjFIkUtgsbF+rekWakX3DQtojTmAbn5/2OwOnL5yE0fP1qC0\nXOfRj5aqkODW4ZmYmJ+BIblKCGJwxYBQ6a6G3vmBaNUj48OVzJhEgSdEQvUk71lQO0eyhWPvHV8C\nXWMOoOa5rnAch4qaJnx1Vovj52vRYnbni1yagMkjsjB5VBaG5ChicnkaPnRXQ+/8QLRzXymWz8wP\nY2pjCwWeEAnVkzyfBTUfzWChntQabRpbrPi6tAZHS7WorndP6hQwDMYMScOU0WqMHZpGfTZB0F0N\nvfN9Zuy06jY1EfdNfN/lIRSqAMFnQc1HM1hvmudijd3B4vurOhw9q8WZazqPuTa56UmYMlqNOwqy\noUwShzGVsae7Gnrn+0wh88x7aiLuGwo8IRKqAMFnQd2b4OnrSdDQbMG2PWdhsbKQiAVYVTgaeWrn\n1g29aZ6LFZW1zqa0b87VornVveKzTCLC5FFZuHO0GoOyk6kpLQw632cr5haAs7l//9RE3Dd+l4YG\ngwFvv/02Lly4AJPJ5PHae++9F/SERav2gvdKpR6Nre4lScbm9W4/oc58FdT6JjPe2X8R5domgAEG\nqxVYOuOWgJbV8XVeb4KnryfBq9WNaDY5r2MzObBtz1n8adWdQcmPaNNksuKb87U4ekbrMQSaYYCC\nwWmYMjob44elI0FEc23CqfN9lq6SeqxcQE3EfeN3bv3617+G1WrFzJkzIZXSyrRdaS94OwYdACj5\nugpjh2e6nvSD/ZlXqhthbxtafeWGwVX197dJwNd5vald+XoStFhZj2MWKxvcDIhwdgeL0jI9jp7V\n4rurNz22g85OlWHK6Gz8oECNlGRaRSBaUBNx3/gdeE6fPo1vvvkGYjG1M3enY8HbWaie9Jtb7WBZ\nzvW5HAdX1d/fJgFf5/WmGczXk6BELIDN5HAdk4hjv2Pc7mBx8XoDjl+ow6nL9TBZ3PkulQgxKT8L\nd45R06i0KBXPTcTB4HfgueWWW1BTU4MBAwaEMj1Rr73g9SVUT/pyqQgCAeOq8TBtxzqmp6cmgWA1\nHfh6EvTVx9MuGKODImWEEctyuFTZgONtqwl07LdhAOQPTMGdY9SYMDyDlq0hcc3v1alfffVVfPrp\npygsLER6errHawsWLAhJ4kItFKtTt6/QfO2GAfpmd8EjYAClXBySGo++yYx3DlxEucZ3H09PK0Zn\nZCTjUll9WFaWDsYK28FapRsIfBViluNw9UYjjl+oxclL9V7Dbof1U+K2EVmYeEsGlFG2ICetyOxG\neeHE++rUJ0+eRFZWFo4ePepxnGGYqA08oZCanIiF04bhnf0XYbI0wGLjwDBAskzs8aQf7M/8xcPj\nunzNn0I4XE0HwRgdxPcII47jUKYx4viFOpy8VIeGJovH63k5CtyWn4mJ+ZlIDfP6eYREIr8Dz7vv\nvhvKdMSUD764gss3DLDYOGcbC5z7nrQPLCjTNnY5vDjeBKOJj48RRu0rCZy4WIcTF+qgM3quBD4w\nKxm3jcjEpPxMpNO20IR0K6A7tLGxEV9++SVqa2uRlZWFqVOnQqmMzwKzO82tzpFcHABwAAfg+2s6\nlGkbkadWYtues1E9vLinPpVAAmt7n5DeaIG+yQKxSIDte88G1NQXqhFGZqsd5ysacObaTXx/TYfG\nZs9mtH4ZSZg0Igu35WciK1UWlM8kpLNI6cMMJr/7eE6fPo2f/exnyMvLQ05ODjQaDcrKyrBjxw6M\nHx+dC+iFagfS7XvP4uSlep+vpSSL0WKywsbCGZEYQCISYPuv7wl6OvwVaPt1T30qv972lSuwchwH\nuSyhx8AazH6avnAIBPjy+HWcuabDxcoG2B2evw91mgyT8jNx24gs5KQn8Z4+PlG/hls48yJS7g0g\nDH08L7zwAjZs2IAHH3zQdeyzzz7Dc889h3/+8599TkgsKbpvWJeBp9lkg61tig8D59BnB8/bdjpY\nFjcNZuiMZhhNVnBMHTR1TTBbHbDZWQgFDIRCBiKBAEIhA3GCECq5GCnJEqQrpWhqCyqA7z6V3szb\nCddMcJPZjqvVBly8bsCZMh00N1s8XhcKGAzrp8TYoekYOzQd2VSzITyLxVUS/A48FRUVmDlzpsex\n6dOnY8OGDUFPVLRLTU6EPFGEZrP3D8Q53JmDQOB8ehEwDHLT/X+C6K7azXEczFYHWsw2tLTa0Wy2\noaXVhhazHXqjGVqdCTV6E2r1Jo9JjL0hFDBIEAkgEgqQmizB+Qo91GlJUMnFvZq3w9dM8CaTFVdu\nNOJSpQGXqwyorGvy2q5bLk3AmCFpGDs0HaMGpUKWSLPSSfjE4ioJfn+DgQMH4tNPP8VDDz3kOnbg\nwAH0798/JAmLdslJCT4DD8dxEIkYqNOSXD+kdKX/7bXtKwwAQGOLBdv+eRYDspNRrjGiRm+C1e7/\nXCEGgFyWgFRFIqRiIaQSERJEArAsB7uDg4PlYHewMFvtMDRb0dhsdS1g6WA5OKwOAA40t9rwXx98\nBwCQJAjAgHE2UTEcpGIRHpk6FDY7iwRR1wEoVP00hmYLLlcZcKnKgMuVBlR3qtEAzpWfB2bLMWpw\nKu6eOAApic55USR6xPKAnVhcJcHvPp5Tp07h8ccfx6BBg5CTk4Pq6mpcv34db7zxBiZMmBDqdIZE\nqObx/HnfOVysavR6jWEAlVyMJdNvwdelNQHPmWluteH5d07gZqPFrxoLwwBJiQlIShRBKZcgO1UG\ndZrM9b9pykQIBQK/268dLIsGowX1hlbUGVpRozc5a1E6E+obW71qDp3TkqGUIjvN+dnqtCRXOpJl\nwVkNg+M46IxmXK5y1mYuVRpQ29DqdZ5IyGCwWoFbBqgwvL8KQ3KUkEqcz2DUr+EWTXnRm37FQERT\nXoQS7308EyZMwOeff45///vfqKurw9SpU3H33XdDpVL1OREdVVRUoLi4GAaDASqVCps3b/ZaLYFl\nWWzatAlfffUVBAIBHnvsMTz88MNBTUdvffDFFVyu9g46AHDr8Az8fL6zU3Dc0Ay/rlfbYMK3l+rx\n/dWbuFrd6FW4MwBGD0lDXo4C/TLkSEmWIClRBLk0AYkSUVB3pBQKBEhXSZGukmJEp9dsdgdqG1rx\nxsfn0GK2wW5nYbOzsDlYcJyzL6uuLWCduabzeK9cmuAMSKltAaktOClkYiSIBBAKGK9lZTiOg6HZ\nihpdCypqm1BWbcRVTaPXyLP2PJKIhZAkCJGdKsUvHxkHMa0cEFPifT3AaBNQY6FSqcTcuXNDlRYA\nwIYNG7B48WLMnj0b+/btw/r167Fr1y6Pc/bt24eqqip8/vnn0Ov1mD9/PqZMmYKcnJyQps0fza32\nLp/8/a0i641mHL9Qh+MXalFR4/mUxTDOpiGGAUQCAXLSZfhFBOxAmiASol+GHDlpMo8ROBnKRDx6\n33Bo9c6akVbX4qoptU+8bG614eqNRly94TtgMwCEQgESRAxEQme/UovZBqvNd+GSKBZiWD8VhvdX\n4uTFenDgOhRKAgo6MSge1wOMZt0GnhUrVmDnzp0AgB/96EddLmYYrG0R9Ho9Lly44Bo5N3v2bGza\ntAkNDQ1ISXFvK7B//3488sgjAIDU1FTcd999OHDgAJYvXx6UdPSFXCoCw8Ar+EhE6LY5zWJ14PiF\nWvxfaQ0uVxnQ8e0quRjjh2Vg7NA0HPlOg5tGs+sGi7SZ8b7ao1OTE5GqSMSoQake57Za7KhtcAYh\nZ5NdC7Rtgx86DmPm4Fx00+4AAM9VvxkAmSlSDMlVYkiuEnlqBfplJkEoEEDfZMbhUzdgMtshFAiQ\nkiyGPID+NBI9VhWO7nI9QBJ5ug088+bNc/03H01ZWq0WWVlZrgAnEAiQmZmJmpoaj8Cj0Wg8ajdq\ntRpardbnNY1GI4xGo8cxoVAItVodgm/gLHibW624WOl+ek+WibC2i1rJ9ZomHDxRiWMX6jz6m+TS\nBEzMz8S2gc5JAAAgAElEQVTkEZkY1l/lajLrlykPqKOxu1Fw7fv4lGmNMFsdkCQIMSRX6VrnrTcC\nWXpHKhFhULYCg7IVHsdZlsPNxla0mO2w2dm2oMPCZuew/5sKGFqsriHfuWlJWFU4xuf13zlwCaZW\nG2wODlawSGgVxETHLPGWp1ZG1STsaKfVauFweD4EKhQKKBSKLt7hqdvA03EEW15eHsaO9S48z5w5\n49cHhcuuXbuwdetWj2O5ubk4fPhwUDrJOsvISMYf197jceymoRU7Py7FvqPXoZCJsXhmPkrLdDjw\nzXVcrTK4zmMAJElFGJCtwPNPTIFI6N1ckJGRjGfy/OsfAoC3P7uIhmYLGIZBQ7MFH31Vgf9YOsn1\n2jWNESazva1WYUdpuQ67v7yGZ1bc3puvHzRZWb5/wP97VgtBh3yxsc488eV6TRPAMG2j6ZxB/RY/\n8q6r68Ujygs3ygu3RYsWobq62uPY6tWrsWbNGr/e73cfz09+8hOcOnXK6/hjjz2G48eP+3uZbqnV\natTW1oLjnG3yLMuirq4O2dnZHue1r5xQUFAAwBl9c3NzfV5z2bJlmD9/vscxodDZxh+qlQs6a595\n7GA5lGsa8fUftR6j0sQJAufoM2kChAIGrINDg9572G9HnWsyPxidjXf/dclrOOlNQ2tbs5Xz8242\ntLpG5zhfYz2a9TgOuFihD8kInmAs/SEWMjDY3G35qiRxl2llWa5tnyLnYASW5Xr8XjR6yY3ywo3y\nwql9VNt7773ns8bjrx4DD8uy4DjO41+7yspKVyEeDKmpqcjPz0dJSQnmzJmDkpISjBw50qOZDQBm\nzJiBf/zjH7j//vvR0NCAQ4cO4W9/+5vPawZS/QsFjuNQ19CKm41mj83AREIBbhuRibvH5eDg8Upo\ndS3QN5rhYFnIEkXQN5m73aL6UqUBdgeLNGUiTBYbXv+oFODgtf5bd5PP2vfx8UwwgBDFYn93Q+1O\nIHMaBquTcaW6ESzLQSgUYLCanlgJCYa+dlX0GHhGjhzp6nMZOXKkx2sCgQCPP/54nxLQ2bPPPovi\n4mJs374dSqUSmzdvBgCsXLkSa9euxahRozB37lx8//33eOCBB8AwDFatWoV+/foFNR195WBZbPjz\n19DoLT5ftztY/F9pDc5cvemcpGljwQBgBAxaLXa8e+ASMpRifHHK3XeVkuyc7yKTiGC1O8BxgL7J\nggyVFA4H52qa6zictLuCuui+YXjnwEWUlunBcs4Rc5IEoauADvbihMFY+iOQPqSlM/ODNvEuFhdq\nJCRcepxAWl1dDY7jsGTJEo9aBcMwSE1NRWJi9N58oWhqe2vfGXx9/mZA7xGLBK4VB8Rts/slYiGa\nTN47mUoSBBAKBQDnnDsjFAqQmSKFVtfiqvEEOoGOSRBh+3+f9hqJFujihD0Vzp2vl6mSuuY1RYqu\nmlT4WKgx0oIbNS+5UV448TaBtL3v5Msvv+zzh8WyFrMNh09VBxx0fOE4rsvmLqFAAIeDRboyEfom\nC0RCATJVUsy7a7BXH4+/0lW+C9FAayg9NaVF89IffCzUGIymSEKigd+DC9atW9fla+3NYfFIbzTj\n4Ikq/M/3Glisjp7f0AOGgas/4kyZ3ut1h8OBREkCkmUSqNOSPJ6K/V0NwV+BLk5oaLbgpsEMtm3x\nU3Gntdl6aibr6xN/KGsMfCzUGEhwi7TaESGB8Ht674ABAzz+SaVSHDlyJG43gtPcbMHbn17Af7zx\nNQ6eqILF6kCiOPCBFnKpCCKhAAqZCCMGpmCwWolRg1KxdGa+z/NZjkOL2YbK2iZc0zTC0Oy7DykY\niu4bhqwUKaRiETJV0h5rKPUNzgEUFpsDJosd9QZzt+d31v7E32q1o87Qit2HrvD6/s70TWZs33sW\nm/9+GlabAylyid950Rtyqcg1eKen4Bbs70oIn/x+bFu9erXXsQULFmDbtm1BTVCku1bdiM++uY7T\nV9xNaookMe6f2A9Tx+figy8u4qtS33vxtFMmifHKmt5NdmM5Bg6Wg4BhQ757aSAd+QBgczicO323\n7fhtswdWA+xrc1awm8M8m76c/TpdTQQOhkCaImNxjxYSP/rUXjBixIigzeGJZBzH4Vy5Hp98fR2X\nO0z4zFRJMWPyAEwZnY0EkbO2M3xAao+Bx9egAX+1z/9pH4zQ5GNRzHBhGMYZcTr+HYC+NmcFuzmM\n78K9q0Dvq1ktFvdoIfHD71/r119/7fG32WzGp59+iqFDhwY9UZGiPeB8/FU5rmncy+4MzE7GrNsH\n4tbhGV7zYP7y2aUeryvws4FTLhWhubX7ws7O8d/e39XnCTqtURfoljZ9HXwQ7MELkVK4+xp0EM0D\nNQjx+056+umnPf6WyWTIz8/Hn/70p6AnKtw4jsO5iraAU+0OOPkDVJj9g0EYMTCly6f5ngZnCxhg\naK5//WK/eGSsx8KH2akyXLhu8DiHYfgZDdUx2NQ2tEAmEUEkEnp8noPlOlZ4At7lNNCmvWC/v7NI\nKdx91byC/V0J4ZPfgefw4cOhTEdE4DgO5ysa8PFX5bjaYU+dEQNTMGfKINwyIKWbd/csf0BKQAVY\n54UPN//9tNc5IiHD/1Bfsx0WG4sMldTj85xbF7iDjdDHWnORqGNQTVdJMe+uQc4VtSOkcI+Umhch\nwRLQL9hoNLo2gsvMzMTdd98d1aPa3v3XJcy8fQBS5BKfASd/gApz7xzcY8DpWHAlCJwLV3Y2JFuK\ndT8a36f0yqUipCYnQN/k7CNiGOCJeQX4v7M1vA71bZ9LBHiOvhqsVuDKDQO4tsEFg9XhW6ooEB2D\nqlbXHHHzZyKl5kVIsATUx7NmzRoMHjwYOTk50Gq12LhxI7Zs2YI77rgjlGkMmZuNrXhz3zmwHDw2\nIfM34LTrWHAJRQLYfOx+eK3GewvmQPtm2gugzE4F0L9PVaPBaAEYZ2EfqqG+7cFNJU9Aq9UBqVjk\nkY6lM27xq4CMtDkokT5CLFJqXoQEi9+BZ9OmTdi4cSNmzZrlOrZ//3784Q9/wIEDB0KSuFDTGS24\nUdfs+vuW/s6Akz8wsCa19oLLbnd0uSumL4H2zfgqgLbvPQtDixWpykTnzosJgpAU4h2fujOUiT6D\nhb8FZKTN0KemLEL45fcdVldXh+nTp3scu//++7F+/fqgJ4ovVptznsnw/irM60XAaddecDU0WV3N\nTP50qwfjSZuvp/VgPnVHWg2jY1Bt7+MhhISO34Fn3rx5eO+997B06VLXsffff99jl9JoI0tMwONz\n8zApPzPgOScdtRdcNw1mSBIEYBgGrZ2Wz8lNl3m9LxhP2tH4tB4Jae6quS/SF4Ms0zZ6bfGcp47e\nflYSn7pdnfpHP/qRq0BmWRZnzpxBWloasrKyUFtbC51Oh7Fjx+Lvf/87bwkOpps3m9D1tw9cxxWM\na3QtEAoEyEiRAgCkYpHX4AJ9k9mrTyTQZrJgXIPvwjYYae6rrlabjvTA8+ttX6HZ1CFoB7AKeaAi\nPS/4RHnhxMvq1A8//LDH34888kifPzCStN+8wdKxyUaWKIJM4szerp7qg9F8FawmMD47/COhszzS\nmvv8ZbGyHum2+BjIQkik6zbwdN4ymnSvY4HqahJpskIiFmDpjFvCnLruRVqHf6hFQnNfb0jEAthM\n7q2/JeLomCtFSEfd3m0fffSRqw/nww8/7PK8BQsWBDdVMeDAN5VIlomhSHIWEP86VhnRBXm01gB6\nK1rnxqwqHO3Vx0NItOk28Hz66aeuwPPxxx/7PIdhGAo8PkRbQR6tNYDeioTmvt7ovJoFIdGo29Ll\nrbfeAuDso3jhhRegVqshEsV2gRQs0VaQR2sNgBASffwqDRmGwUMPPYRTp06FOj28ev7dk1h479CQ\nDEeNtoI8WmsAhJDo4/dj+IgRI1BeXo4hQ4aEMj28amm1BXUjtUhbCoYQQiKR34Hntttuw09/+lPM\nnz8f2dnZHhMuo7WPJ9jDUd85cAlXqgzg4Fy94N0Dl0K6Y2WkocBLCPGH34Hn1KlTyM3N9dpxNJoH\nFwR7OGq51uixB02Z1tjN2bGnqyHZFJAIIR35HXjefffdUKYDZrMZTz31FM6dOweRSIR169bhnnvu\n8TqvtrYWv/3tb3H+/HkMGjSo22HePUmSJmD5gyP6kOpOOGcwc01MDeKqCNGgq5F88TZHiBDSPb8f\n97tak62wsDAoCdm5cyfkcjkOHjyI119/Hb///e/R2uq9lUBSUhKefPLJoOx8+vSSiX4NLNA3mbF9\n71ls/vtpbN97Fvoms8/zBquTIRIJwDCASCTAYHVyn9MYTeRSkWsliI4j+aJtaDkhJLT8DjzXr1/3\nOsZxHG7cuBGUhOzfvx9FRUUAgIEDB6KgoABHjhzxOk8ul2PixImQSqVB+Vx/tD+xt1rtqDO0Yveh\nKz7PWzozHwWDUzFYrcSoQalYOjOftzRGgqL7hiErRQqpWIRMldQ1kq+rgEQIiU89lgDr1q0DANhs\nNtd/t6uursbQoUODkhCNRoOcnBzX32q1Glqtts/XNRqNMBo9+1qEQiHUarXf1/D3iT3ehyR39f1D\nNbSc+o4ICQ+tVguHw3MFfoVCAYXCv12Heww8AwYM8PnfADBhwgTMmDHDrw8qLCz0CiTt/SFHjx71\n6xq9sWvXLmzdutXjWG5uLg4fPuz3KqvpKim0umZX3026SoqMjNhqRgvl98nISMYzeRlBv+7bn11E\nQ7MFDMOgodmCj76qwH8sndTn68ba/7d9QXnhRnnhtmjRIlRXV3scW716NdasWePX+3sMPKtXrwYA\njB07FnfddVcvkui0Z8+ebl/Pzc2FRqNBSopzMzatVovbb7+915/XbtmyZV6LnQqFQgCATtcMlu15\nBMC8uwZ5PLHPu2tQTC2RHq1Lvt80tMLu4NA+iuNmQ2ufv0e05kUoUF64UV44tW+L8N577/ms8fjL\n78b2u+66C2VlZbh48SJMJpPHa8EYTj19+nTs3r0bGzduREVFBUpLS/Hyyy93eT7HcX5taRBI9a8r\n8d6EFqmibVkiQmJFIF0Vvvh9p77xxhvYtm0b8vPzkZjobkcP1jyeFStWoLi4GA888ACEQiE2bdoE\nmcy5a+drr72GrKwsLFy4ECzLYurUqbDZbGhqasI999yDBQsWuGpmJH5E27JEhBCnbncg7eiOO+7A\nX/7yF+Tnx85ILX+b2mIdNSO4UV64UV64UV448bIDaUeJiYnIy8vr8wcSEiw0qo2Q6OT3PJ61a9fi\nueeeQ11dHViW9fhHSDj4O7+KEBJZ/K7xFBcXAwD++7//23WsfTj0hQsXgp8yQnpAKyIQEp38DjyH\nDh0KZToICRiNaiMkOvl9p+bm5gIAWJbFzZs3kZ6eDoEgeCs7Rzvqb+AfjWojJDr5HXiam5uxceNG\nfPbZZ7Db7RCJRHjwwQfx+9//HsnJNKOXVmDmH82vIiQ6+V1lee6559Da2oqSkhKcOXMGJSUlaG1t\nxXPPPRfK9EUN6m8ghBD/+F3j+d///V988cUXrlWhBw8ejBdffBH3339/yBIXTai/gRBC/ON36SiR\nSKDX6119PQDQ0NAAsVgckoRFG+pvCB7qLyMktvkdeBYsWIDly5fjxz/+MXJycqDRaPDXv/4VDz/8\ncCjTFzWovyF4qL+MkNjmd+B54oknkJWVhZKSEtTV1SEzMxOPPfYYBR4SdNRfRkhs83twwfPPP4/B\ngwfjr3/9Kz777DP89a9/xZAhQ/D888+HMn0kDtGOpYTENr8DzyeffIKCggKPYwUFBfjkk0+CnigS\n37raQpsQEhv8fpRkGMZrXTaHw0FrtZGgo/4yQmKb3zWeiRMn4tVXX3UFGpZlsWXLFkycODFkiSOE\nEBJ7/K7xPP300/jZz36GO++8Ezk5OdBqtcjIyMAbb7wRyvQRQgiJMX4HnuzsbOzduxdnzpyBVquF\nWq3GmDFjaL02QgghAQlouJBAIMC4ceMwbty4UKWHEEJIjKNxqiTilGkbsW3PWVisLCRiAVYVjkae\nWhnuZBFCgoQCD4k42/acRbPJue6dzeTAtj1n8adVd4Y7Wb1GSwAR4ok6aEjEsVhZj5ULLNboHrJP\nW3QT4okCD4k4ErHAY+UCiTi6f6a0BBAhnqL7jiYxaVXhaMhlCRAJBZDLErCqMLonk9ISQIR4ipg7\nwGw246mnnsK5c+cgEomwbt063HPPPV7nHTp0CNu2bYPNZgMAFBYW4ic/+QnPqSV90VOfR55aGdV9\nOp3RlhmEeIqYwLNz507I5XIcPHgQ169fx6JFi/D555+7Np5rl5GRgR07diAjIwPNzc0oLCzEmDFj\ncOutt4Yp5SRQ8bbtAS0BRIiniAk8+/fvx0svvQQAGDhwIAoKCnDkyBFMnz7d47wxY8a4/lsulyMv\nLw8ajYYCTwTrXMMxNFuoz4OQOBYxgUej0SAnJ8f1t1qthlar7fY9165dw5kzZ7Bp06YuzzEajTAa\njR7HhEIh1Gp13xJM/Na5htNksiJZJqZtwgmJUlqtFg6Hw+OYQqGAQqHw6/283fGFhYVegYTjODAM\ng6NHjwZ8vbq6OqxatQobNmxARkZGl+ft2rULW7du9TiWm5uLw4cPIy1NHvDnxqqMjOSQXdvq4CBO\nELr+zkpNQrpKCmOLFQqZGCvmFiBdJe3mCvwKZV5EG8oLN8oLt0WLFqG6utrj2OrVq7FmzRq/3s9b\n4NmzZ0+3r+fm5kKj0SAlJQWAM6LefvvtPs/V6XRYvnw5fvrTn3o1xXW2bNkyzJ8/3+OYUChsu04z\nWJbz9yv0SjRMHszISEZ9fVPIri8WMjDYHK4ajipJjOUz812vczZ7SD8/EKHOi2hCeeFGeeEkEDBI\nS5Pjvffe81nj8fs6wU5Yb02fPh27d+8GAFRUVKC0tBR33XWX13kNDQ1Yvnw5Fi9ejB/+8Ic9Xleh\nUKBfv34e//hsZqPJg7SxGyGxRq1We5WrgQSeiGlcX7FiBYqLi/HAAw9AKBRi06ZNkMlkAIDXXnsN\nWVlZWLhwId566y1cv34du3fvxgcffACGYbB06VKvWk2koMmDNKqLEOKJ4dpntsUhPpratu896+pY\n5zgOmSopfj4/sgphakZwC0VeRENzqy/0u3CjvHBqb2rr83WCkBbSDWpmItTcSoiniGlqi1XUzESo\nuTUw0VpDJP6jGg8hIUZrtQWGaoixjwIPISFGza2BoRpi7KNHL0JCjJpbAyOXimCy2GhlixhGNR5C\nSEShGmLso0cJQkhEoRpi7KMaDyGEEF5R4CGEEMIrCjyEEEJ4RYGHEEIIryjwEEII4RUFHkIIIbyi\nwEMIIYRXFHgIIYTwigIPIYQQXlHgIYQQwisKPIQQQnhFgYcQQgivKPAQQgjhFQUeQgghvKLAQwgh\nhFcUeAghhPAqYjaCM5vNeOqpp3Du3DmIRCKsW7cO99xzj9d5Fy9exO9+9ztwHAe73Y7x48dj/fr1\nSEhI4D/RhBBCAhYxgWfnzp2Qy+U4ePAgrl+/jkWLFuHzzz+HVCr1OC8vLw//+Mc/IBI5k/7kk09i\n9+7dWLx4cTiSTQghJEAR09S2f/9+FBUVAQAGDhyIgoICHDlyxOs8sVjsCjpWqxVmsxkMw/CaVkII\nIb0XMTUejUaDnJwc199qtRpardbnuXV1dVi5ciWqqqpw9913Y+HChV1e12g0wmg0ehwTCoVQq9XB\nSTghhMQZrVYLh8PhcUyhUEChUPj1ft4CT2FhoVcg4TgODMPg6NGjAV0rMzMTH330EcxmM37729/i\n4MGDmDVrls9zd+3aha1bt3ocy83NxeHDh5GWJg/sS8SwjIzkcCchYlBeuFFeuFFeuC1atAjV1dUe\nx1avXo01a9b49X7eAs+ePXu6fT03NxcajQYpKSkAnBH19ttv7/Y9iYmJmDlzJkpKSroMPMuWLcP8\n+fM9jgmFQgCATtcMluX8/QoxKyMjGfX1TeFORkSgvHCjvHCjvHASCBikpcnx3nvv+azx+Ctimtqm\nT5+O3bt3Y+PGjaioqEBpaSlefvllr/OqqqqQnZ2NhIQEWK1WHDp0CMOHD+/yuoFU/wghhPSsr10V\nERN4VqxYgeLiYjzwwAMQCoXYtGkTZDIZAOC1115DVlYWFi5ciNOnT+Ott96CUCiEw+HAbbfdhlWr\nVoU59YQQQvzFcBwXt21N1NTmRM0IbpQXbpQXbpQXTu1NbX2+ThDSQgghhPiNAg8hhBBeUeAhhBDC\nKwo8hBBCeEWBhxBCCK8o8BBCCOEVBR5CCCG8osBDCCGEVxR4CCGE8IoCDyGEEF5R4CGEEMIrCjyE\nEEJ4RYGHEEIIryjwEEII4RUFHkIIIbyiwEMIIYRXFHgIIYTwigIPIYQQXlHgIYQQwisKPIQQQnhF\ngYcQQgivKPAQQgjhVcQEHrPZjF/+8pd44IEHMGvWLPz73//u9nyr1YpZs2ZhwYIF/CSQEEJIUERM\n4Nm5cyfkcjkOHjyI119/Hb///e/R2tra5fmvvPIKJkyYwGMKCSGEBEPEBJ79+/ejqKgIADBw4EAU\nFBTgyJEjPs89efIkrl+/jrlz5/KZREIIIUEQMYFHo9EgJyfH9bdarYZWq/U6r7W1FS+++CL+8Ic/\ngOM4PpNICCEkCER8fVBhYaFXIOE4DgzD4OjRo35fZ/PmzVi0aBEyMjJQVlbW4/lGoxFGo9HjmFAo\nhFqthkDA+P25sY7ywo3ywo3ywo3ywp0HWq0WDofD4zWFQgGFQuHXdRguQqoNDz30EP7zP/8To0aN\nAgA8/vjjmD9/PqZPn+5x3pw5c9DS0gIAsFgsaGxsRF5eHj7++GOf192yZQu2bt3qcWzChAl4//33\nQ/AtCCEk9j366KM4deqUx7HVq1djzZo1/l2AixBbtmzh1q9fz3Ecx5WXl3NTpkzhWlpaun3PsWPH\nuB/+8IfdntPY2MhVVVV5/Dtx4gRXVFTEaTSaoKU/Wmk0Gm7q1KmUFxzlRUeUF26UF24ajYYrKiri\nTpw44VWuNjY2+n0d3praerJixQoUFxfjgQcegFAoxKZNmyCTyQAAr732GrKysrBw4cKAr9tV9e/U\nqVNeVcV45HA4UF1dTXkByouOKC/cKC/cHA4HTp06hezsbPTr16/X14mYwCOVSvHqq6/6fO3JJ5/0\nefy2227Dhx9+GMpkEUIICbKIGdVGCCEkPlDgIYQQwivhs88++2y4ExEOEokEkydPhkQiCXdSwo7y\nwo3ywo3ywo3ywi0YeRExw6kJIYTEB2pqI4QQwisKPIQQQngVd4GnoqICRUVFmDFjBoqKilBZWRnu\nJPHGYDBg5cqVmDlzJubOnYsnn3wSDQ0NAIDvvvsOc+fOxYwZM7BixQro9fowp5YfW7duRX5+Pq5e\nvQogPvPBarXi2WefxfTp0zFnzhw888wzAOLzXvnyyy8xf/58zJs3D3PnzsXnn38OID7y4qWXXsK0\nadM87geg++/e63wJ2RTXCLV06VKupKSE4ziO+/jjj7mlS5eGOUX8MRgM3PHjx11/v/TSS9zTTz/N\ncRzH3X///dypU6c4juO47du3c0899VRY0sinc+fOcY899hg3depU7sqVKxzHxWc+bNq0iXvxxRdd\nf+t0Oo7j4vNemTRpEnf16lWO4zju4sWL3Pjx4zmOi4+8+Pbbb7mamhru3nvvdd0PHNf9d+9tvsRV\n4NHpdNykSZM4lmU5juM4h8PBTZw4kdPr9WFOWXj861//4n7yk59wZ86c4WbPnu06rtfruXHjxoUx\nZaFnsVi4hQsXcjdu3HAFnnjMh5aWFm7ixImcyWTyOB6v98rkyZNdDx7Hjx/npk+fzul0Om7ixIlx\nkxcdH8S6+x305TcSMSsX8EGr1SIrKwsM41xhVSAQIDMzEzU1NUhJSQlz6vjFcRzef/99TJs2DVqt\nFrm5ua7X2vPCaDT6vdpstHnttdcwd+5cj+8dj/lQWVkJlUqFLVu24NixY0hKSsLatWuRmJgYl/fK\nK6+8gieeeAIymQwtLS148803odVqkZ2dHXd5AXRfZrIs2+vfSNz18RCnjRs3IikpCYsXL/b5OhfD\no+y/++47nD17Fo8++qjrWFffN5bzAXCuvVVVVYWCggL885//xG9+8xusWbMGJpMp5r97Zw6HA2++\n+SbeeOMNHD58GK+//jp+8YtfwGQyhTtpMSeuajxqtRq1tbWufYBYlkVdXR2ys7PDnTRevfTSS6is\nrMSOHTsAOPOlurra9bperwfDMDH7lH/8+HGUl5dj2rRp4DgOtbW1eOyxx7BkyZK4ygcAyMnJgUgk\nwqxZswAAY8aMQWpqKiQSCerq6uLqXrlw4QLq6+sxbtw4AM7tU6RSKSQSSdyWG92Vme33Tm/yJa5q\nPKmpqcjPz0dJSQkAoKSkBCNHjoz56nJHr7zyCs6fP4/t27dDJHI+dxQUFMBisbj21/jggw8wc+bM\ncCYzpFauXIkjR47g0KFDOHz4MLKysvD2229jxYoVcZUPgLM5cfLkya7NGMvLy6HT6ZCXlxd390p2\ndjZqampQXl4OALh27Rp0Oh0GDRoUd3nRXtvtrszsS3kadysXlJWVobi4GEajEUqlEi+99BIGDRoU\n7mTx4urVq3jooYcwaNAg13IX/fv3x5YtW3D69Gk888wzsFqt6NevH/74xz8iNTU1zCnmx7Rp07Bj\nxw4MHToU3333HdavXx9X+VBVVYXf/e53MBgMSEhIwK9+9SvceeedcXmvfPLJJ9ixYweEQiEA58r4\n9957b1zkxXPPPYfPP/8cOp0OKpUKKSkpKCkp6fa79zZf4i7wEEIICa+4amojhBASfhR4CCGE8IoC\nDyGEEF5R4CGEEMIrCjyEEEJ4RYGHEEIIryjwEMKD8vJyzJ8/H7feeiv+9re/hTs5hIRVXC2ZQ0i4\n/PnPf8bkyZOxd+/ecCeFkLCjGg8hPNBoNBg6dGjA73M4HCFIDSHhRYGHkBBbtmwZjh07ho0bN2LC\nhOHlPWwAAAILSURBVAl45513XM1uU6dOxdatW13nVldXIz8/Hx9++CGmTp2KH//4xwCcK2oXFRVh\n0qRJmDdvHo4fPx6mb0NI31FTGyEhtmvXLixZsgTz5s3DD3/4Q5w4cQJ33HEHhg0bhsuXL2P58uUY\nMWIEpk2b5nrPyZMnsX//fggEAtTW1uJnP/sZ/uu//gt33XUXvv76a6xZswYHDhyI6YUqSeyiGg8h\nPGlfFnHSpEkYNmwYAGD48OGYNWsWTpw44TqPYRisWbMGiYmJEIvF2LdvH+655x7cddddAIA77rgD\nBQUF+J//+R/+vwQhQUA1HkJ49v333+NPf/oTrly5ApvNBpvNhhkzZnic03FPE41Gg/379+PLL78E\n4Axgdrsdt99+O6/pJiRYKPAQwrPf/OY3WLJkCXbu3ImEhAS88MILMBgMHue0bycMODfjmjdvHjZu\n3Mh3UgkJCWpqI4RnJpMJCoUCCQkJOHPmDD755BOP1zvvVDJnzhwcPnwYX331FViWhcViwfHjx1Fb\nW8tnsgkJGgo8hPCgYw3mmWeewWuvvYZbb70V27dvd2077etcwNnstn37duzYsQN33HEHpk6dirff\nftsrQBESLWgjOEIIIbyiGg8hhBBeUeAhhBDCKwo8hBBCeEWBhxBCCK8o8BBCCOEVBR5CCCG8osBD\nCCGEVxR4CCGE8IoCDyGEEF79f73xgK649h0MAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "FEATURE = 'fare'\n", - "feature = pd.Series(df_dfc[FEATURE].values, index=dfeval[FEATURE].values).sort_index()\n", - "ax = sns.regplot(feature.index.values, feature.values, lowess=True)\n", - "ax.set_ylabel('contribution')\n", - "ax.set_xlabel(FEATURE)\n", - "ax.set_xlim(0, 100)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lbpG72ULucz0" - }, - "source": [ - "### Permutation feature importance" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 401 - }, - "colab_type": "code", - "id": "6esOw1VOucz0", - "outputId": "edf89eaf-e61e-4980-8586-3a0c6eb4f626" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm4AAAGACAYAAAAUDu58AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl01PW9//HXzCQQJAlrDDFg9NzemhavlLBEBNmEsKkQ\ndgoSoLLEErlyEUTxlAIiIBRujXhcKuUoIjsWarxWsSJK9aLgSu1CwxJiiGyTbUiY+fz+8MdcBwJk\nmWTyIc/HOZxjPt/vfL7v77znCy8/35mMwxhjBAAAgDrPGeoCAAAAUDEENwAAAEsQ3AAAACxBcAMA\nALAEwQ0AAMASBDcAAABLENwA1Ir27dvr2LFjQZ/3X//6l1JTU9WhQwe98sorQZ+/JuXm5iopKUn8\nViYAFUVwA+qI3r17q127dkpKSlK3bt00d+5clZSUhLosSVJiYqKOHj1a4f3vu+8+bd68OWBs//79\nat26dbBL04svvqjk5GR98sknGjduXLXmKq/umhQXF6dPP/1UDoej1o55OTk5OUpMTJTP5wt1KQCu\ngOAG1CHPPfecPv30U23dulVffvmlnn322UrP4fV6g15XXQgWl3P8+HH96Ec/CnUZkmrmua8NXq9X\nxhg5HA5W/4A6juAG1CEX/tG8/vrrdeedd+pvf/ubJKmwsFCPPfaYunXrph49emjVqlX+fbdt26Yx\nY8boySefVHJysjIzMwPGOnXqpL59+2r//v3atm2bevbsqa5du2r79u3+41680rRt2zb9/Oc/lySN\nGzdOxhjde++9SkpKUlZWltxut6ZNm6YuXbooOTlZ06ZNU15eniRp5cqV+uSTT7Rw4UIlJSVp0aJF\nkgJX7QoLCzV79mx16dJFvXv3DgioF469dOlSde7cWX369NHu3bvLfb7S0tL00UcfacGCBUpKStLh\nw4dVWlqqpUuXqlevXurWrZvmz5+v0tJSSap03eWtQv3wuSrvuZekzZs3a+DAgUpOTtb999+v48eP\nl1v/xfPfd999WrVqlUaPHq327dsrPT1dZ86c0axZs9ShQweNGDEiYK7ExES9/PLL6tOnj7p06aJl\ny5YFvJZWr16t3r17q2vXrnrkkUdUWFgYcNzNmzerV69emjBhgu677z4ZY9SxY0clJSXps88+09Gj\nR5WWlqbk5GR16dJFs2bN8s8hfb9K/NJLL+nee+9Vp06dNHPmTP9zLUlvv/22hgwZog4dOiglJUV7\n9uzx9/9yr2cAV2EA1Am9evUyH374oTHGmOPHj5tBgwaZ3/72t8YYY9LT082vfvUr4/F4zMmTJ82I\nESPMhg0bjDHGbN261fz0pz81r7zyivF6vebcuXNm69atpm3btmbbtm3G5/OZlStXmp49e5oFCxaY\n0tJSs2fPHtO+fXtTXFxsjDFm3LhxZtOmTf5atm7dan7+85/7f77lllvMkSNH/D+fPn3avPXWW+bc\nuXOmqKjIzJgxwzzwwAP+7RfPZ4wxiYmJ/jkefvhh88ADD5ji4mJz7Ngxk5KSYjZv3uw/dtu2bc2m\nTZuMz+czr776qunWrdtln7eLj7Vo0SKTnp5u3G63KSoqMtOmTTO/+c1vqlT3sWPHTGJiovF6veXu\nU95z/6c//cmkpKSYQ4cOGa/Xa5599lkzatSocmu/eP5x48aZlJQUc/ToUVNQUGAGDhxo+vXrZ/bu\n3Wu8Xq+ZPXu2mTt3bkBfxo8fb9xut8nNzTUpKSn+2jZt2mRSUlLMsWPHTHFxsZk+fbp5+OGH/ce9\n5ZZbzJw5c0xJSYk5d+6cvxafz+ef//Dhw+bDDz80ZWVl5tSpU2bcuHFm8eLF/u29evUyI0aMMPn5\n+ebs2bNmwIAB5rXXXjPGGPPZZ5+ZDh06+F/TeXl55tChQ8aYK7+eAVwZK25AHfLLX/5SnTt31tix\nY5WcnKypU6fq5MmTev/99/Xoo4+qYcOGat68udLS0rRz507/42JjYzV27Fg5nU41aNBAkhQfH68h\nQ4bI4XBo4MCB+vbbb/XLX/5S4eHh6tq1q8LDw3X48OEq1dm0aVP17dtXDRo00HXXXaepU6dq3759\nV3yM+f8rKj6fT1lZWfqv//ovNWrUSPHx8Zo0aZJef/11/77x8fEaPny4HA6HUlNT9d133+nkyZMV\nqm3z5s2aO3euoqKidN1112nKlCn+56oqdV/Nxc/9hg0bNGXKFN18881yOp2aMmWK/vrXvyo3N7dC\n8w0dOlStW7dWZGSkunfvrhtvvFG33367nE6n+vfvr4MHDwbsP2XKFEVFRalVq1ZKS0vTH//4R0nS\nzp07NWHCBMXHx6tRo0aaOXOm3njjDf/qnsPhUEZGhiIiIvyvGUkBK1833nijunTporCwMDVr1kxp\naWn63//934Djjx8/Xi1btlR0dLR69erlr2/z5s0aPny4unTpIun7VeSbb765Qq9nAJcXFuoCAPyf\n1atX6/bbbw8Yy8nJ0fnz59WtWzdJ3//DaoxRXFycf59WrVpdMlfLli39/x0RESFJat68ecBYcXFx\nler0eDxavHix9uzZI7fbLWOMiouL/e+TupLTp0/r/PnzuuGGG/xjN9xwg/+WZXm1X5i/RYsWV5z7\n1KlTKikp0bBhw/xjPp/PH0aqU/flXPzcHz9+XE888YSWLl0qSf658/LyAnp2OT88x4YNGwb8XF7P\nfnj8+Ph4nThxQpJ04sSJgOc4Pj5e58+f13fffXfZ2i926tQpLVq0SPv27VNxcbG8Xq+aNm162Xob\nNWqk/Px8SdK3336rHj16XDJnRV7PAC6P4AbUIaac9/nExcWpYcOG+uijjy4bLqr74YHrrrtOHo/H\n//OFf3wv56WXXlJ2drY2b96s5s2b669//atSU1P9IeVK9TRr1kxhYWHKycnRv/3bv0n6PuzExsZW\n6xwuzN2oUSPt3LlT119/fbXrbtSokSSppKREjRs3lqSA4CNd+tzHxcUpPT1dd999d7XPpyJyc3P9\nz2NOTo7/vK+//vqA98Pl5OQoLCxMLVu29K/+/bD28nq2YsUKORwO7dy5U9HR0Xr77bf971m8mlat\nWpX7SeSKvJ4BXB63SoE6LiYmRl27dtXixYtVWFgoY4yOHj16yS2rqykvFF6QmJiot956Sx6PR4cP\nH9aWLVsCtrds2TLgH+GioiJFREQoMjJSZ86c0dNPP33F/X/I6XRqwIABWrVqlYqKipSTk6Pf//73\nGjx4cKXOpzwOh0MjRozQ4sWLderUKUlSXl6e/03xla27efPmio2N1R/+8Af5fD5t3rz5qr8WZfTo\n0Xruuef0j3/8Q5JUUFCgN99887L7X6kvFfG73/1Obrdbubm5evnllzVw4EBJ0qBBg/T73/9ex44d\nU1FRkVauXKlBgwbJ6XSWe9zmzZvL6XTqyJEj/rGioiI1btxYkZGRysvL0+9+97sK1zV8+HBt3bpV\nf/nLX2SMUV5eng4dOhS01zNQX1kX3Nxut55++mm53e5Ql4JKondXdqXVh6VLl6qsrEyDBg1S586d\nNWPGjKuuil1t/h/+PGHCBP973+bOnat77rknYN+MjAw9/PDDuu2227Rt2zZNmDBBJSUlSk5O1ujR\noy+5JTZ+/Hi9+eabSk5O1hNPPHHJ8ebNm6eIiAj16dNH48aN07333htwe/NqtV9p26xZs5SQkKCR\nI0eqY8eOmjRpkrKzsyV9/ynUyta9YMECvfjii7r99tv1z3/+U+3bt79sLZLUp08fTZ48WQ899JA6\nduyoe++9V++//36F6q/KCtRdd92loUOHKjU1Vb169dLw4cMlfR+cBg8erHHjxqlv374KCwtTTEyM\n//q7+FgRERGaNm2axowZo86dO+vzzz/X9OnT9eWXX6pjx46aNm2a+vXrd9naL3bbbbdp8eLFWrx4\nsTp06KDx48f7V/qC8XquT/i7027B7p/DVPd/92rZsWPHdNddd+mdd96pkV/miZpD7+xG/+qexMRE\n/elPf1KbNm2uui/9sxe9s1uw+2fdihsAAEB9RXADAEvx5n6g/uFTpQBgqYt/pxuAa5+VK25JSUly\nuVyhLgOV5HK5FB8fT+8sRf/sRv/sRe/s5nK5lJSUFLT5rPtwQmFhoSIjI0NdBgAAQIUFK79YF9wu\nOH26SD6flaXXay1aROrkycKr74g6if7Zjf7Zi97Zy+l0qFmzxkGbz9r3uPl8huBmKfpmN/pnN/pn\nL3oHydL3uAEAANRHBDcAAABLENwAAAAsQXADAACwBMENAADAEgQ3AAAASxDcAAAALEFwAwAAsATB\nDQAAwBIENwAAAEsQ3AAAACxBcAMAALAEwQ0AAMASBDcAAABLOIwxJtRFVMXJk4Xy+awsvV6Lahah\niLDwUJcBAKhHiktLVXT2XEiO7XQ61KJFZNDmCwvaTEAFRISFy7VmVqjLAADUI96Jy1Wk0AS3YONW\nKQAAgCUIbgAAAJYguAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABYguAGAABgCYIbAACAJQhu\nAAAAliC4AQAAWILgBgAAYIlqBzePx6MZM2bo7rvv1pAhQ/TQQw9JkrZv366RI0dq2LBhmjBhgrKz\nsyVJq1evVkZGhiSppKRE99xzj3bv3l3dMgAAAK55YdWdYM+ePSooKNDOnTslSQUFBdq3b5+ysrK0\nbt06hYeHa/fu3Zo7d67Wr1+v9PR03X///XrllVf09ddfq2fPnurevXu5c7vdbrnd7oAxl8uluLi4\n6pYNAABQa3Jzc+X1egPGoqOjFR0dXal5qh3cbrnlFh06dEgLFy5Up06d1LNnT7377rv65ptvNHLk\nSBljZIxRQUGBJMnhcOipp57S4MGDFR8fr0WLFl127rVr1yozMzNgLD4+Xrt27apu2QAAALVm7Nix\nysnJCRibPn26/y5kRVU7uLVp00ZvvPGG9u7dq927d2vlypXq06ePhg0bdtlijh49KqfTqbNnz6qk\npESNGzcud7+0tDSlpqYGjLlcruqWDAAAUKvWrVtX7opbZTmMMaY6heTl5alJkyaKiIhQSUmJevTo\nodWrV2v27Nlav369YmNj5fP5dPDgQbVt21Znz57ViBEjtGTJEn344Yc6dOiQfvOb31T6uCdPFsrn\nq1bpCIGYmCi51swKdRkAgHrEO3G58vMLQnJsp9OhFi0igzZftVfcvvnmG61YsUKS5PP5NHXqVHXs\n2FEzZ85Uenq6fD6fysrK1L9/f7Vt21aPPfaYhg8frqSkJP3sZz/ThAkTtGHDBo0aNaraJwMAAHAt\nq/aKW6iw4mYnVtwAALXtWlpx4/e4AQAAWILgBgAAYAmCGwAAgCUIbgAAAJYguAEAAFiC4AYAAGAJ\nghsAAIAlCG4AAACWILgBAABYguAGAABgCYIbAACAJQhuAAAAluBL5lGroppFKCIsPNRlAADqkeLS\nUhWdPReSYwf7S+bDgjYTUAERYeHKzy8IdRmoopiYKPpnMfpnL3qHC7hVCgAAYAmCGwAAgCUIbgAA\nAJYguAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABYguAGAABgCYIbAACAJQhuAAAAliC4AQAA\nWILgBgAAYAmCGwAAgCUIbgAAAJYguAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABYguAGAABg\nCYIbAACAJQhuAAAAliC4AQAAWILgBgAAYAmCGwAAgCUIbgAAAJYguAEAAFiC4AYAAGAJghsAAIAl\nCG4AAACWILgBAABYIizUBaB+8ZwvU0xMVKjLQDXU5f4Vl5aq6Oy5UJcBADWG4IZaFREWLteaWaEu\nA9co78TlKhLBDcC1i1ulAAAAliC4AQAAWILgBgAAYAmCGwAAgCUIbgAAAJYguAEAAFiC4AYAAGAJ\nghsAAIAlCG4AAACWILgBAABYosrBrXfv3vrHP/4RzFoAAABwBay4AQAAWKJCXzK/f/9+PfXUUyoq\nKpLD4dDDDz8csH3NmjV644035PV61aBBA82fP1+JiYnyeDyaM2eO/vnPfyosLEw333yzVq5cqX/9\n61+aO3euPB6PvF6vhg4dqokTJ9bICQIAAFwrrhrczp49q4yMDD3zzDNq166djDEqKCgI2GfIkCH+\n4LV371796le/0oYNG7Rnzx4VFBRo586dkuR/3KuvvqoePXooPT09YPxibrdbbrc7YMzlcikuLq6S\npwkAABA6ubm58nq9AWPR0dGKjo6u1DxXDW4HDhzQj370I7Vr106S5HA4LjnIF198oeeff15nz56V\nw+HQ4cOHJUm33HKLDh06pIULF6pTp07q2bOnJKlTp05atmyZSktLlZycrNtvv73cY69du1aZmZkB\nY/Hx8dq1a1elThIAACCUxo4dq5ycnICx6dOnKyMjo1LzXDW4GWOuuL2srEwzZszQ+vXrlZiYqBMn\nTqhHjx6SpDZt2uiNN97Q3r179d5772nlypXasWOHUlJS1L59e33wwQd64YUXtGXLFj311FOXzJ2W\nlqbU1NSAMZfLVZnzAwAACLl169aVu+JWWVcNbu3bt9e8efP02WefqV27dvL5fCosLPRvP3funHw+\nn2JjY/2FXZCXl6cmTZrorrvu0h133KEePXro7NmzKikpUZs2bTRkyBDdeOONevTRR8s9dlWWEAEA\nAOqaYL3N66rBrUmTJsrMzNSTTz6p4uJiuVwuzZ49Ww6HQ5IUGRmpBx98UMOGDVN8fLzuvPNO/2O/\n+eYbrVixQpLk8/k0depUxcTE6LnnntOOHTsUHh4uh8OhefPmBeVkAAAArmUOc7V7oXXUyZOF8vms\nLL1ei4mJkmvNrFCXgWuUd+Jy5eeX/2EnfH/98fzYid7Zy+l0qEWLyODNF7SZAAAAUKMIbgAAAJYg\nuAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABYguAGAABgCYIbAACAJQhuAAAAliC4AQAAWILg\nBgAAYImwUBeA+sVzvkzeictDXQauUcWlpaEuAQBqFMENtSoiLFz5+QWhLgNVFBMTRf8AIIS4VQoA\nAGAJghsAAIAlCG4AAACWILgBAABYguAGAABgCYIbAACAJQhuAAAAliC4AQAAWILgBgAAYAmCGwAA\ngCUIbgAAAJYguAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABYguAGAABgCYIbAACAJQhuAAAA\nliC4AQAAWILgBgAAYAmCGwAAgCUIbgAAAJYguAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABY\nguAGAABgCYIbAACAJQhuAAAAliC4AQAAWCIs1AWgfvGcL1NMTFSoywiK4tJSFZ09F+oyAAD1CMEN\ntSoiLFyuNbNCXUZQeCcuV5EIbgCA2sOtUgAAAEsQ3AAAACxBcAMAALAEwQ0AAMASBDcAAABLENwA\nAAAsQXADAACwBMENAADAEgQ3AAAASxDcAAAALEFwAwAAsATBDQAAwBLV/pL5WbNmKTs7W6WlpUpI\nSNDixYsVFRWllStXKisrS82aNVOnTp20d+9ebdmyRZK0fft2vfrqq/J6vYqKitL8+fN10003VbcU\nAACAa1q1g9u8efPUtGlTSdKqVav0/PPPKykpSe+995527Nihhg0bKiMjQw6HQ5K0b98+ZWVlad26\ndQoPD9fu3bs1d+5crV+//pK53W633G53wJjL5VJcXFx1ywYAAKg1ubm58nq9AWPR0dGKjo6u1DzV\nDm7btm3Tjh07VFZWJo/Ho5tuukllZWUaMGCAGjZsKEkaMmSInn32WUnSu+++q2+++UYjR46UMUbG\nGBUUFJQ799q1a5WZmRkwFh8fr127dlW3bAAAgFozduxY5eTkBIxNnz5dGRkZlZqnWsFt3759eu21\n17RhwwY1bdpUO3fu1IYNGyTJv8J2MWOMhg0bVqFC09LSlJqaGjDmcrmqUzIAAECtW7duXbkrbpVV\nrQ8nFBQUKCoqSk2aNFFpaam2bNkih8Oh5ORkZWVlyePxyOfz6fXXX/c/pnfv3tq+fbvy8vIkST6f\nT1999VW580dHR6t169YBf7hNCgAAbBMXF3dJpqlKcKvWilv37t31hz/8QQMGDFCrVq1066236vPP\nP1evXr20f/9+DR48WLGxsWrXrp3/dmjHjh310EMPKT09XT6fT2VlZerfv7/atm1bnVIAAACueQ5j\njKmJiYuKitS4cWMZY/TYY48pNjZWM2bMCNr8J08WyuerkdJRg2JiouRaMyvUZQSFd+Jy5eeX//7M\na1VMTFS9O+drCf2zF72zl9PpUIsWkUGbr9ofTricOXPmKCcnRx6PR7feeqvuv//+mjoUAABAvVBj\nwe3iT4MCAACgevjmBAAAAEsQ3AAAACxBcAMAALAEwQ0AAMASBDcAAABLENwAAAAsQXADAACwBMEN\nAADAEgQ3AAAASxDcAAAALFFjX3kFlMdzvkzeictDXUZQFJeWhroEAEA9Q3BDrYoIC1d+fkGoywAA\nwErcKgUAALAEwQ0AAMASBDcAAABLENwAAAAsQXADAACwBMENAADAEgQ3AAAASxDcAAAALEFwAwAA\nsATBDQAAwBIENwAAAEsQ3AAAACxBcAMAALAEwQ0AAMASBDcAAABLENwAAAAsQXADAACwBMENAADA\nEgQ3AAAASxDcAAAALEFwAwAAsATBDQAAwBIENwAAAEsQ3AAAACxBcAMAALAEwQ0AAMASBDcAAABL\nENwAAAAsQXADAACwBMENAADAEgQ3AAAASxDcAAAALBEW6gJQv3jOlykmJirUZQRFcWmpis6eC3UZ\nAIB6hOCGWhURFi7XmlmhLiMovBOXq0gENwBA7eFWKQAAgCUIbgAAAJYguAEAAFiC4AYAAGAJghsA\nAIAlCG4AAACWILgBAABYguAGAABgCYIbAACAJQhuAAAAlqhycEtMTFRJSUmtPxYAAKC+qnJwczgc\nVT5odR4LAABQX1X4S+bfeustrVy5Uk2bNlX37t3945999plWrFihoqIiSdKDDz6oHj16SJLeffdd\nZWZm6vz583K5XFqyZIl+/OMfyxgjSTLGaMmSJfruu++0ZMkShYeHB/PcAAAArikVCm6nTp3S448/\nro0bNyohIUEvvviiJMntdmv+/Pl64YUX1LJlS+Xn52v48OH64x//qPz8fD3++ONav3692rRpo7Ky\nMpWVlUn6fsXN4/Fozpw5at26tVasWFHucd1ut9xud8CYy+VSXFxcdc4ZAACgVuXm5srr9QaMRUdH\nKzo6ulLzVCi4HThwQLfeeqsSEhIkSaNGjdLy5cv11Vdf6dixY5o8ebJ/Fc3lcunw4cM6cOCAevTo\noTZt2kiSwsPD/StqxhhNnjxZgwYN0sSJEy973LVr1yozMzNgLD4+Xrt27arUSQIAAITS2LFjlZOT\nEzA2ffp0ZWRkVGqeCgW3C6Hshz9feJ9aYmKiXn755Usec+DAgSvOmZycrPfff1+jR49Wo0aNyt0n\nLS1NqampAWMul6siJQMAANQZ69atK3fFrbIq9OGE9u3b6+uvv9aRI0ckSZs2bZIktW3bVtnZ2fro\no4/8+37xxReSpG7duum9997zP6a0tFTFxcX+/aZPn64uXbpo8uTJKiwsLPe40dHRat26dcAfbpMC\nAADbxMXFXZJpaiy4NW/eXAsXLtTUqVM1ZswY/y3P6OhoPfvss8rMzNSQIUM0cOBAPfPMM5KkhIQE\nLVq0SP/5n/+pwYMHa/To0f4lwgurdZMnT1a/fv00adKkS97LBgAAgEAOc/F9UEucPFkon8/K0uu1\nmJgoudbMCnUZQeGduFz5+QWhLqNWxcRE1btzvpbQP3vRO3s5nQ61aBEZvPmCNhMAAABqFMENAADA\nEgQ3AAAASxDcAAAALEFwAwAAsATBDQAAwBIENwAAAEsQ3AAAACxBcAMAALAEwQ0AAMASBDcAAABL\nENwAAAAsERbqAlC/eM6XyTtxeajLCIri0tJQlwAAqGcIbqhVEWHhys8vCHUZAABYiVulAAAAliC4\nAQAAWILgBgAAYAmCGwAAgCUIbgAAAJYguAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABYguAG\nAABgCYIbAACAJQhuAAAAliC4AQAAWILgBgAAYAmCGwAAgCUIbgAAAJYguAEAAFiC4AYAAGAJghsA\nAIAlCG4AAACWILgBAABYguAGAABgCYIbAACAJQhuAAAAliC4AQAAWILgBgAAYAmCGwAAgCUIbgAA\nAJYguAEAAFiC4AYAAGAJghsAAIAlwkJdAOqOxk0a6roGDWr0GJ7zZTU6PwAA1zKCG/yua9BArjWz\navQY3onLVSBPjR4DAIBrFbdKAQAALEFwAwAAsATBDQAAwBIENwAAAEsQ3AAAACxBcAMAALAEwQ0A\nAMASBDcAAABLENwAAAAsQXADAACwRJWD29tvv62BAwdq6NChys7ODmJJAAAAKE+Vv6t0w4YNmjFj\nhvr161fhx/h8PjmdLPIBAABURZWC25NPPql9+/YpOztbr776qmJiYpSdna3S0lIlJCRo8eLFioqK\n0scff6zFixerY8eO+vLLL5Wenq4OHTpoyZIl+tvf/qZz584pOTlZc+fOlcPhCPa5AQAAXFOqFNzm\nzp2rr7/+Wvfff7969OihM2fOqGnTppKkVatW6YUXXtDMmTMlSX//+9+1YMECzZs3T5I0b948de7c\nWYsWLZIxRrNmzdLmzZs1YsSIS47jdrvldrsDxlwul+Li4qpSNgAAQEjk5ubK6/UGjEVHRys6OrpS\n81T5VukPbdu2TTt27FBZWZk8Ho9uuukm/7aEhATddttt/p937dqlL774Qi+99JIkyePxqFWrVuXO\nu3btWmVmZgaMxcfHa9euXcEoGwAAoFaMHTtWOTk5AWPTp09XRkZGpeapdnDbt2+fXnvtNW3YsEFN\nmzbVzp07tXHjRv/266677pLHPPPMM2rduvVV505LS1NqamrAmMvlqm7JAAAAtWrdunXlrrhVVrWD\nW0FBgaKiotSkSROVlpZqy5YtV9y/d+/eev755zV//nw5nU6dPn1aRUVF5Qa5qiwhAgAA1DXBeptX\nlT/ieeG2scT8AAAHu0lEQVTDBN27d1ebNm00YMAATZkyRW3btr3i4x599FE5nU4NHjxY99xzjyZP\nnqwTJ05UtQwAAIB6w2GMMaEuoipOniyUz2dl6XVWTEyUXGtm1egxvBOXKz+/oEaPgZoTExNF/yxG\n/+xF7+zldDrUokVk8OYL2kwAAACoUQQ3AAAASxDcAAAALEFwAwAAsATBDQAAwBIENwAAAEsQ3AAA\nACxBcAMAALAEwQ0AAMASBDcAAABLENwAAAAsQXADAACwRFioC0DdUVxaKu/E5TV6DM/5shqdHwCA\naxnBDX5FZ8+pSOdq9BgxMVEqkKdGjwEAwLWKW6UAAACWILgBAABYguAGAABgCYIbAACAJQhuAAAA\nliC4AQAAWILgBgAAYAmCGwAAgCUIbgAAAJYguAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABY\nIizUBVSV0+kIdQmoInpnN/pnN/pnL3pnp2D3zWGMMUGdsYYVFhYqMjIy1GUAAABUWLDyi3W3Ss+c\nOaMxY8YoNzc31KWgknJzc9W7d296Zyn6Zzf6Zy96Z7fc3FyNGTNGZ86cCcp81gU3Sfr000/l9XpD\nXQYqyev1Kicnh95Ziv7Zjf7Zi97Zzev16tNPPw3afFYGNwAAgPqI4AYAAGAJghsAAIAlXPPnz58f\n6iIqq2HDhkpOTlbDhg1DXQoqid7Zjf7Zjf7Zi97ZLZj9s+7XgQAAANRX3CoFAACwBMENAADAEnUy\nuHk8Hj300ENKSUnRwIED9ec///my+27cuFEpKSlKSUnRokWL/OMff/yxfvaznyk1NVVDhgzRqFGj\naqHy+is7O1ujR49W//79NXr0aB05cuSSfXw+n37961+rb9++6tevnzZt2lShbah51e1fZmam7rjj\nDqWmpio1NVULFy6szfLrtYr07oMPPtCwYcP0H//xH1q2bFnANq690Kpu/7j2QqcivVu9erXuvvtu\nDRkyRMOGDdOePXv82yqTdQKYOigzM9PMmzfPGGNMdna26dq1qykuLr5kv6NHj5ru3bub06dPG2OM\nmTRpktm+fbsxxpiPPvrIDBs2rPaKrufGjx9vduzYYYwx5vXXXzfjx4+/ZJ9t27aZX/ziF8YYY06e\nPGm6d+9ucnJyrroNNa+6/Xv66afN0qVLa69g+FWkd0eOHDEHDx40q1atuqRPXHuhVd3+ce2FTkV6\nt2fPHuPxeIwxxhw8eNB07NjRnDt3zhhT8axzsTq54paVlaXRo0dLkhISEnTrrbdq9+7dl+z3P//z\nP+rbt6+aNm0qSRo5cqSysrL82w2fu6gVp06d0sGDBzVo0CBJ0t13362vv/5ap0+fDtgvKytLI0eO\nlCQ1b95cffr00ZtvvnnVbahZweifxPUWChXtXZs2bZSYmCiXy3XJHFx7oROM/klce6FQ0d517drV\n/0nSxMREGWP8+1Q061ysTga348eP64YbbvD/HBcXV+53tOXm5l5xv8OHD2vo0KEaNWqUtm/fXrNF\n12O5ubmKjY2Vw+GQJDmdTl1//fX69ttvA/a7Ul8r2nMEXzD6J33/l9DgwYP1i1/8QgcOHKid4uu5\nivbuSrj2QicY/ZO49kKhKr3btm2bbrzxRsXGxkqq+rUXVs3aq2To0KGXFGeMkcPh0AcffBCUY7Rt\n21Z//vOfFRkZqWPHjmnixImKjY1Vly5dgjI/gP8zZswYpaeny+Vy6cMPP9QDDzygrKwsNWnSJNSl\nAdc0rj07fPzxx3r66ae1Zs0a/9iF0FdZIVlx27p1q/bu3Rvw5y9/+Yv27t0rp9Op+Ph4HT9+3L9/\nbm6u4uLiLpknLi5OOTk55e7XuHFjRUZGSpJat26tPn36BPVLXvF/4uLilJeX51+u9/l8OnHihFq1\nahWw3w033HDZvl5pG2pWMPrXokUL/22cO+64Q61atdLf//73WjqD+quivbsSrr3QCUb/uPZCozK9\n279/v+bMmaPVq1crISHBP17Va69O3irt16+fNmzYIOn7T218+eWXuvPOOy/ZLyUlRe+8845Onz4t\nn8+njRs3qn///pKk/Px8/35nzpzRnj179JOf/KR2TqCead68uRITE7Vjxw5J0o4dO/TTn/5UzZo1\nC9ivf//+2rhxo4wxOnXqlN555x2lpKRcdRtqVjD6l5eX59/v4MGDOn78uG6++ebaO4l6qqK9+6GL\n3w/FtRc6wegf115oVLR3n3/+uWbOnKn//u//VmJiYsC2imadi9XJb04oKSnRI488ooMHD8rlcmn2\n7Nnq1auXJOm3v/2tYmNj/b/eY+PGjXrhhRfkcDjUrVs3Pf7443I4HFq3bp3Wr1+v8PBwnT9/Xqmp\nqZo0aVIoT+uadujQIT3yyCNyu91q0qSJli1bpoSEBE2ZMkUzZsxQ27Zt5fP5tGDBAn3wwQdyOBya\nPHmyRowYIUlX3IaaV93+PfLII/rqq6/kdDrVoEEDPfjggxX6CwjVV5HeffLJJ5o5c6aKiopkjFFU\nVJSeeOIJde3alWsvxKrbP6690KlI74YPH67jx48rNjbW/5awZcuW6d///d+vmHWupE4GNwAAAFyq\nTt4qBQAAwKUIbgAAAJYguAEAAFiC4AYAAGAJghsAAIAlCG4AAACWILgBAABYguAGAABgif8HkhdZ\nW/CfFsoAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "def permutation_importances(est, X_eval, y_eval, metric, features):\n", - " \"\"\"Column by column, shuffle values and observe effect on eval set.\n", - "\n", - " source: http://explained.ai/rf-importance/index.html\n", - " A similar approach can be done during training. See \"Drop-column importance\"\n", - " in the above article.\"\"\"\n", - " baseline = metric(est, X_eval, y_eval)\n", - " imp = []\n", - " for col in features:\n", - " save = X_eval[col].copy()\n", - " X_eval[col] = np.random.permutation(X_eval[col])\n", - " m = metric(est, X_eval, y_eval)\n", - " X_eval[col] = save\n", - " imp.append(baseline - m)\n", - " return np.array(imp)\n", - "\n", - "def accuracy_metric(est, X, y):\n", - " \"\"\"TensorFlow estimator accuracy.\"\"\"\n", - " eval_input_fn = make_input_fn(X,\n", - " y=y,\n", - " shuffle=False,\n", - " n_epochs=1)\n", - " return est.evaluate(input_fn=eval_input_fn)['accuracy']\n", - "features = CATEGORICAL_COLUMNS + NUMERIC_COLUMNS\n", - "importances = permutation_importances(est, dfeval, y_eval, accuracy_metric,\n", - " features)\n", - "df_imp = pd.Series(importances, index=features)\n", - "\n", - "sorted_ix = df_imp.abs().sort_values().index\n", - "ax = df_imp[sorted_ix][-5:].plot(kind='barh', color=sns_colors[2], figsize=(10, 6))\n", - "ax.grid(False, axis='y')\n", - "ax.set_title('Permutation feature importance')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E236y3pVEzHg" - }, - "source": [ - "## Visualizing model fitting" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TrcQ-839EzZ6" - }, - "source": [ - "Lets first simulate/create training data using the following formula:\n", - "\n", - "\n", - "$$z=x* e^{-x^2 - y^2}$$\n", - "\n", - "\n", - "Where \\(z\\) is the dependent variable you are trying to predict and \\(x\\) and \\(y\\) are the features." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "e8woaj81GGE9" - }, - "outputs": [], - "source": [ - "from numpy.random import uniform, seed\n", - "from matplotlib.mlab import griddata\n", - "\n", - "# Create fake data\n", - "seed(0)\n", - "npts = 5000\n", - "x = uniform(-2, 2, npts)\n", - "y = uniform(-2, 2, npts)\n", - "z = x*np.exp(-x**2 - y**2)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GRI3KHfLZsGP" - }, - "outputs": [], - "source": [ - "# Prep data for training.\n", - "df = pd.DataFrame({'x': x, 'y': y, 'z': z})\n", - "\n", - "xi = np.linspace(-2.0, 2.0, 200),\n", - "yi = np.linspace(-2.1, 2.1, 210),\n", - "xi,yi = np.meshgrid(xi, yi)\n", - "\n", - "df_predict = pd.DataFrame({\n", - " 'x' : xi.flatten(),\n", - " 'y' : yi.flatten(),\n", - "})\n", - "predict_shape = xi.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "w0JnH4IhZuAb" - }, - "outputs": [], - "source": [ - "def plot_contour(x, y, z, **kwargs):\n", - " # Grid the data.\n", - " plt.figure(figsize=(10, 8))\n", - " # Contour the gridded data, plotting dots at the nonuniform data points.\n", - " CS = plt.contour(x, y, z, 15, linewidths=0.5, colors='k')\n", - " CS = plt.contourf(x, y, z, 15,\n", - " vmax=abs(zi).max(), vmin=-abs(zi).max(), cmap='RdBu_r')\n", - " plt.colorbar() # Draw colorbar.\n", - " # Plot data points.\n", - " plt.xlim(-2, 2)\n", - " plt.ylim(-2, 2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KF7WsIcYGF_E" - }, - "source": [ - "You can visualize the function. Redder colors correspond to larger function values." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 512 - }, - "colab_type": "code", - "id": "WrxuqaaXGFOK", - "outputId": "08121255-36c0-4946-d601-a808b94647a0" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk8AAAHvCAYAAABNMzsdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsfXmYFNW5/ltbd8/CwAzDJsQlIioqCqjBBUGIsojgEkNA\nUC9Go0ajuf40mhvZNIlEk7gQt5i4gybGSDQCXhHURKMoIt5oVHBj3wZm7+6q6vr90V1NdfU5p86p\nqp6Nep/Hx6H71He+c6pmzlvfKlmWZSFChAgRIkSIECECF+T2ViBChAgRIkSIEKEzISJPESJEiBAh\nQoQIAojIU4QIESJEiBAhggAi8hQhQoQIESJEiCCAiDxFiBAhQoQIESIIICJPESJEiBAhQoQIAojI\nU4QIEUqKrVu3YtiwYeCpiiIyNij++te/Yvr06SWfJ0KECF0PEXmKECGHF154Aeeffz6GDh2KkSNH\n4vLLL8d7770XWO7NN9+Mu+++OwQN2x5hEIx+/fphzZo1kCQp1LFhgHeehQsX4sYbbyyxNhEiROgs\nUNtbgQgROgIeeeQRPPzww5g3bx5OPfVUaJqGN954A6+++iqGDx/e3upRYZomFEUpmXzLsjwJRiaT\ngSxH72ERIkTYfxD9xYuw36OpqQn33HMP5syZg29/+9tIJBJQFAWjR4/GDTfcAABIp9P4+c9/jpEj\nR+K0007DL37xC+i6DgB45513MGrUKDzyyCM4+eSTMXLkSDz33HMAgD/96U944YUX8PDDD2PYsGG4\n8sorAQAbNmzAzJkzccIJJ+Dss8/Gq6++mtdn5syZePbZZ/P/dlt/jjjiCDz11FMYN24cxo0bR1zT\nihUrMGnSJJx44om46KKLsGHDhvx3Y8aMwR//+EdMnjwZJ5xwAv77v/8b6XS6SMaGDRswd+5crF27\nFkOHDsWJJ54IIGtJmzt3Li6//HIMHToUb7/9Nl577TWce+65GD58OE4//XQsXLgwL2fz5s044ogj\nkMlk8uu7++67MW3aNAwbNgyXXnop9u7dKzwWAJ5//nmMGTMGI0aMwH333YcxY8bgrbfeIu7J3r17\nccUVV2D48OH47ne/i6+//rrg+5///OcYPXo0hg8fjvPPPx/vvvsuAOCNN97AAw88gJdeeglDhw7F\nOeecAwB47rnnMHHiRAwbNgxnnHEGnnnmGeK8ESJE6IKwIkTYz/H6669bRx11lGWaJnXMXXfdZU2d\nOtWqq6uz6urqrKlTp1p33323ZVmW9fbbb1uDBw+27r33XsswDGvVqlXWscceazU0NFiWZVk33XST\nddddd+Vl6bpunXHGGdaDDz5o6bpuvfXWW9bQoUOtL774wrIsy5oxY4b15z//OT/+ueees6ZPn57/\n9+GHH27NmjXLamhosFKpVJGun3/+uXXcccdZb775pmUYhvX73//eOuOMMyxd1y3LsqzTTz/duuCC\nC6ydO3da9fX11oQJE6ynn36auG733PZ6jj/+eOv999+3LMuyUqmU9c4771iffvqpZVmW9cknn1in\nnHKK9corr1iWZVmbNm2yjjjiiPz+zpgxwzrjjDOsr776ykqlUtaMGTOsX//618JjP/vsM+u4446z\n1qxZY+m6bt1+++3WUUcdZb355pvEtVx33XXWddddZyWTSevTTz+1Ro4cWbC2v/3tb1Z9fb1lmqb1\nyCOPWKecckp+f++9917rhhtuKJC3atUqa+PGjZZlWdbq1autY4891vroo4+Ic0eIEKFrIbI8Rdjv\nsXfvXvTo0YPpenrxxRfxwx/+ENXV1aiursbVV1+NJUuW5L/XNA1XXXUVFEXBqFGjUF5eji+++IIo\na+3atWhpacHll18OVVUxYsQInH766XjxxRe5df7BD36Abt26IRaLFX23dOlSjB49GieddBIURcGl\nl16KZDKJ999/Pz/moosuQm1tLaqqqnD66afj448/5p4bAMaOHYvjjjsOABCLxXDCCSfgsMMOAwAM\nGjQIEydOxOrVq6nXn3feeTjwwAMRi8UwYcIE5vy0scuXL8eYMWMwdOhQqKqKa6+9liojk8ngf//3\nf3HttdciHo/jsMMOw7nnnlsw5uyzz0ZVVRVkWcYll1yCdDpNvYcAMGrUKAwYMAAAcPzxx+OUU07J\nW6siRIjQtRHFPEXY79GjRw/s3buXGbuzY8cOHHDAAfl/H3DAAdixY0eBDOe1iUQCzc3NVFn9+vUr\n+Mwtzwt9+/alfufWVZIk9OvXD9u3b89/1rNnz/zPZWVl2LlzJ/fcpPnXrVuHO++8E5999hl0XYeu\n6xg/fjz1+tra2oL5W1pahMe69zGRSKBHjx5EGXV1dTBNs0Bv5x4BwB//+Ec8++yz+b1obm7Gnj17\nqHq99tpruO+++/Dll18ik8kgmUzi8MMPp46PECFC10FkeYqw32Po0KGIxWJ45ZVXqGP69OmDzZs3\n5/+9ZcsW9O7d29d8vXv3xtatWws+c8orLy9HMpnMf0ciNqwg7t69e2PLli0Fn23dupVJuGigzeP+\n/Prrr8e3v/1tvP7663j33XcxderUkpcb6NWrF7Zt25b/dzKZLIiHcqKmpgaKohTsu3OP3n33XTz8\n8MO45557sHr1aqxevRqVlZXUNaTTaVx77bX4/ve/j7feegurV6/Gaaed1iYlFiJEiND+6FTkqaGh\nAffeey8aGhraW5WSoCuvryOvrbKyEj/60Y8wf/58vPLKK0gmkzAMA6+99hruvPNOAMDEiRNx//33\no66uDnV1dbjvvvswZcqUvIxMJkNdX21tLTZu3Jj/97HHHovy8nL8/ve/h2EYePvtt7Fq1SpMmjQJ\nQDYg/OWXX0YymcRXX32Fv/zlL0LrmTBhAlatWoV//etfMAwDf/jDHxCPx/NuNhH07NkT27ZtQ11d\nHfP+tbS0oKqqCpqmYd26dUUuSBFSwTt2/PjxWLlyJdauXQtd13HPPfdQx8qyjDPPPBMLFy5EMpnE\n+vXr8fzzz+e/37lzJ5LJJBRFQTqdxsKFCwssh7W1tdi8eXNeN9u6Vl1dDVmW8dprr+Gf//wn9xrb\nGh359y8ouvLagK6/vs6KTkeeFi5c2GUfoq68vo6+tksuuQQ33XQT7r//fpx00kkYPXo0Fi9ejG9/\n+9sAgKuuugpHH300Jk+ejClTpuDoo4/GFVdckb/esqyC9TktM9/5znewfv16nHjiibj66quhaRru\nu+8+vP766xgxYgRuvfVW/OpXv8LBBx+c10XTNJxyyim4+eabcfbZZxfo6lU64JBDDsEdd9yBW2+9\nFSeddBJWrVqFBx54AKqqcl3vxIgRI3DYYYfliQft/s2ZMwf33HMPhg8fjvvuuw8TJ06k6uw1P+/Y\ngQMH4pZbbsGPf/xjjBw5Et26dUPPnj2JcWAA8LOf/QzNzc049dRT8dOf/hTnn39+/rujjjoKzc3N\nOP/88zF27FiUlZUVWOrGjx8Py7LwrW99C+eddx4qKirw05/+FNdeey1OPPFEvPTSSxg7dixzXe2J\njv77FwRdeW1A119fp0UYUed79uyxLrvsMmv8+PHW5MmTrWuuucaqq6srGtfa2mpdd9111hlnnGFN\nmDDBWrlypdA8GzdutAYNGpTPcOlq6Mrr68prs6xofR0Bzc3N1uDBg61NmzYJX9sZ1hcEXXl9XXlt\nltX118fCF198YU2dOtUaN26cNXXqVOurr76ijt2wYYN17LHHWgsWLCi4fsaMGdaUKVOsiRMnWvfe\ne29ouoVieZIkCZdddhmWLl2KJUuWYMCAAXl3hxN/+MMfUFlZiZdffhn3338/fvazn6G1tTUMFSJE\niLAfYuXKlUgmk2hpacHtt9+Oww8/HP37929vtSJEiBAC5syZgxkzZmDZsmWYPn06brnlFuK4TCaT\nr9PnxB133IHx48fj+eefx7PPPovnnnsOH374YSi6hUKeunfvjhNOOCH/7+OOO64oIBbIplB/73vf\nAwAcdNBBOProo/H666+HoUKECBH2Q6xYsQIjR47EqFGjsHHjRvzmN79pb5UiRIgQAurq6vDxxx/j\nrLPOAgBMmjQJH330ETED9qGHHsKYMWPyoQ82ZFlGU1MTgGxcpiRJBZnGQRB6qQLLsgpiRZzYsmVL\nQXpwv379iCQLyPp53T7ebdu2YdiwYSVtR9GeUBQF/fv375Lr68prA6L1tRduu+023HbbbYHldNT1\nhYWuvL6uvDYgu75hw4YVZJbaqKqqQlVVVTto5Q+kcx0gr2Pr1q3o06dPPu5RlmX07t0b27ZtQ3V1\ndX7cf/7zH/zzn//E448/jt/97ncFMm6++WZcccUVeOqpp9DY2IgbbrihqESJX4ROnubPn4+Kigpc\neOGFRd+JBKo+9thjBS0eAGDYsGFYvHhxYB07Kvr161fQpqMroSuvDYjW19kRra/zoiuvDciub/Hi\nxZg2bRrWrFlT8N3VV1+Na665JvQ5m+v2oqKGXDMtCGKxGM477zzU19cXfO53HYZhYPbs2fjlL39J\n5BfPPPMMzjnnHMyaNQs7d+7EzJkzcfTRR2PIkCG+12AjVPK0YMECfP3113jwwQeJ3x9wwAHYsmVL\nnjVu3boVI0aMII69+OKLiyoA228We/e2oKIyjh/9ehXuuX40GhuSQqnQkiQVjJckCd2qErj2N6vw\nm2tPg6LISKUMpJK6t7CYgrgqI66pSOkGAOR/TpsZdEvE9umZTMOtpvPalJHhXoMIJAlF87o/p41p\nC7j3gEdfHkgSUK6p+fvdohtF10sSPO8RSUfe70TRFs9DEdIm17CePSuxe3cT9Xv371VbQnTueEJD\nPK4W/J57rY9Xj25VCaG/TfGEBlWVxf7uOCDLMsrKtX3PeXOaOGcY62truPcTADIZC9f99rWC/e2M\na+OFLEuorq7Ab37zG5hm4e9qqaxOFTU9cMep38HeTcXWLr/oMaAvbvjHs1iyZAnXOuzCvlauOXkm\nk8GOHTsKsmB37tyJjRs34vLLL4dlWWhsbASQ7Vc6f/58PP7441ixYgWAbF24ESNG4N133w2FPElW\nSH/tfvvb32Lt2rV46KGHEI/HiWMWLlyIHTt2YP78+fjyyy8xY8YMvPzyyygvLxeaa/fuJsTiGhIJ\nFcm0gaTB98cfABKqgkSs+Dr7cwCYdstSLL51AupbU56HtSQB3cviBdcA+w552nwAUK4qiMVU7vn8\nkAqv9SbTWcLnHNOWRMq9f8m0kdclZZpc+8grm7a/XrJZcnjnEEF7ElkSLMvCjk/XYtDAQ6ljjFgV\ntHgceioFNd22KdXcc0syYGUASYZU2TN/z6ym3dnPeSHlQkUp1wjtRU4XM2NhxpxlwvrYcxlmBhfN\nW45F832spy1g773odyjcTwCQVA2qIof+rNUbFdxj2/JFQZYl9OxZ2SZzOfE/B5+K3V9tCk1ez4MG\n4Odf/kPomosuugjf+c53MHnyZCxZsgTPPfccHnvsMer4hQsXoqWlBTfeeCMAYPLkyZg1axbOOecc\nNDU1Ydq0abjxxhsxcuTIQGsBQgoYX79+PR566CHs2LEDU6dOxTnnnJM3wZ1zzjn5CsmXXnop6uvr\nceaZZ+LKK6/ErbfeKkycbKSSOupbU0LESZKyJGHaLUuRiKlwWvmShpmVlzbyhzjP74Zloega53V5\nuS49EznitLcpxTVfQlXQvSyOhKpwfc5ar/vzgp8Z8koBywJSuf1L5YiTrYutB+u+0WATEJ77SbtH\nTh1pcnjn4IG9Ll4ZAl7wQJAkCX96bgkyGfrhp8WzBFKLx/eRizZRjm9uI1YFqbInjFgVYGWgp7K/\nd3oqJUQ0bDlmWXVWlkMPG2q6AVbTbr6DPaeLZVni+jjWrioyFs0XX09boGDvBb6zkd9PvQlqugFK\n6x7+/RVAd5XcTsmG7RZKlMXQvUcZEmXkemIRwsPcuXPx5JNPYvz48Vi0aBHmz58PALj88svx73//\n2/P622+/HU8//TTOOeccfO9738PEiRNDIU5AiJantkRTcwrxuLjVCeCzYPh58xe5xm2tSKcNtFAs\nHpZFt254WT0SqgJVkaEqMpflKZU2EBewhIUB9/2w/22YGcycuzyvR1zhtzy5ZYZlyWHJkWWAxi14\nQHsuaXP6scQFwWerX8fO3bsx4QxyIcgObXnKWXemz15aaJnxsHjQ5Ey7ZSmenDceiizBatoNQ+sG\nLR4LtnYPaxZN14K1600djjgxrXwCFsB2e75y++6c3yasRJ0d94nXkuVlxdqfLU8dGZ2qwrgNVZVh\nZiyoiiz89u1lZQD8HbQi17itFSTi5LQAsawbtM9ta81F85YDAFIuH7NzH+yfWw0zNCsKD0gWJVsX\nw8wU6MFz32gyw7LksFyjVQn/1jqaZY1mBfRjifMLScr+d9Cx38LnX3xVuokAusWKZcmS5EJLD2ls\nzrrz+JxxAABDq8x/LgSHxcqyLOipFAytElo8FtzqZmWYxIFmnSlYO8962tIqCLCtfKzvnHqWwrLJ\nIWPfvncvmF9PpYk6u++TlyULiKxYnRmdkjwpiowZc5ZBVbLqix4eokHHfr7zAosMsEiFPd4+WAEQ\n5diEa9F8MhFykwr755TJR1LCAI0U0sgSaQ28Mr3g113pvleyj98oks4sghSmq5AFe08qYhr01hZ0\n61ZJPnRCONyIBEGS+V0+OesAcawkZ10+ihz4ALbJitK6B6reBC0ex7r1u3KHaTp8yw/P3grER3m5\nyFh6+AXLjUn6rkjPAG5WEpj7YK+zYN9jBfOr6fri9eTGT59deJ+cBMqdDSZJEhKJ3O94QhXKRo/Q\n/uiU5CmV2ndwxJXSxOhIEvtADSM2iHbosUhFXjfHwerW2wYr3ooVPxX3qJcS5u84i0T6iQHzkkmC\nJAGq4s+a6bxXad30bYEi6WyYGTw5bzwMs/iwEF2jKJzPmCRJSLW2oKJHL/KhE/RwIxAE+4CTVK3o\nQCJeIytEkpE/KLXK8A5g20qUW/eQgbX5QzV0hEUcCPvFi0CkywZLb5fFiXQfheLIWGCQUVZcXNH8\n7vXQrJvIEqiKcgXde5Rl/682o7vajCqlKT9HMmm0W6ZqBH/onOQpFyyeMs2SuC/sg1lVZEyfXSy7\nLdwmXqSCRK5IhIJkreEJIqetyQ9p9OsOY8nz0lVUpuqyZoogaZhoSKYQ05RAz4TbEmiYGSiyBMPM\nENcTRnA6Sxf7GbMsC3V79qK6Zw3VAhLocHMTBIAdCJ2LKym4JmMWkwzXQanqTaEHGhetm2KZK5Xl\nhhuO/UrrBqSKGj4y1NbJAB5uPiEwXLhF8gnr9CRMrnmo1k3GHtpzxPXiqtkROjY6ZcD47t1NyGjZ\nBzDswFl3EDYAouy2DtglwV2niTdlnqd8gf253zm85gqKsOWGIY9XhmgclqibOcg9Id3vKxaswEWn\nJpDJmDh15GmlC9p1BE3TAqGLAofdgdSuf/sKNBYNJGfM5SwlYBl66YKdeXWWFUgVNewg7TD2MCh8\n3gMb2TglRhA/Qb6fdbqvocnglU0KMo8CxjsmOqXlyYmw3Rduqw5NdqndJjzwmzJP050WV2VbmURj\nbUppoSOtoVQxaDS45+OJFyNZ7lh6ixAnHqugfU9IFlXa/X7gJ2PR0tyMcskoSYq4DUOrzLtNiIHQ\npDd4gvvECVGrjW8XFUk3x2eSJBVZI8KCkM4kCx1NFq/LrBTWqEDEqco7iJ8g31kSgQs81iq3bI/n\nkCfIPELHQKcnT0A4aehOOA9SluyOZrMTIQA8FcdJgdCic4RdA4m2hlLGoJHgno8nXoxEJt1y/BJA\nXqJq3xM7NsPWl3a9fb8zyYZshf9SpcJzEiNf8T8+6iYJu6hIuhGy85iB7S5dSqUz7SC3s8rWrd8F\nSdUKSZSPTMB2QW4//AbxOwm8J2jPI20+Lz04gswjdBx0avJUyueqVMSo1L8LfvVmWZncgdA8WW82\ngljovFL2nePaKnWfNJ8s0+PInHCTSaDwurIABFCEqKYzZj42w1nOwa2bUzaQdR+ECudBX3AQ0Q88\n4sEfluWDdBgKyCbpVpCdl3MzepEdIUISakB51lozZGBtQewOLYOxXYuj0hAkiD9EIioKUpkDu4xB\nPKEFkh2hNOgAT7s/eLk/ghygXtf6ld3W1bt5wbI6eAVC86zJD6Gz5Zbl4nNo84tUEg8L7vkymeL5\nafvitmo6r4sHJIA8RNWuSZXWzaL9sq8HQNQ9v69BD0lKCQI13ZArQhhjEwdGbZ2gcB6GvmSTyIuz\nhpMX2WmDA5y4Lhd5tX82zIxYBmOYBCpgOQlRQmNnyD0xdxwfEfUqbMoLisvXLmMQj4fagjZCSOiU\n5EmWZab7Q4SkiJIBv9lm7VHY0P0ZDSzyQSIGTpmlWJMtd936XYjHVOJBDxTei7aOQXPPZ/87ZZqe\n+0Jr3xMGAWRd59QrpiloSJL3i6R7WSKB1mQyMFmxrycSBFHiUCrLByFTz7dswnXMw91BYgwzU5Dy\n7qkzpz5emV9quj7/s2XoxAzGKxasKMpgDJPIBpYVoFyGqsiecU+hknYrg7RuW/mNIpdvKmV4y4jQ\n5uiU5KmyW+GbM0Dv1cY60EnxJl5WDlGy4IyFaavChhUxjRiPwyJ8LPJB+65UFh+7392QgbVYt34X\nYpqCpF7ctsRvJfFSwa45Jnqv3ZafUhFAksWMNcYwM/mYKK3mAOytbwhGKBwHlF1B3h0rIuSCCrl4\nYtiymQcsQ559cM+cuzx8d5jIuqxMlkQ11xWk66f1bBKB86APlci2hztQZF/C1k+SEdNUXLFgBWKa\nWhCoX7+3FamkHkx+hJKgU5KnH/16VcGbs/tQ4Dm4aIcv69qg2Wa0bKywLTaSJHnG45DQFsHxvGtt\nNcw8gZp2y1IktELd7UP9ibnjQiNuIvfBi3j7rdReagLIQ9DsVj4z5y7PPzPlFZVoamoKRiicAdSG\nHigrye94ANwFIgPFs4QdfB4iaFllJLJnxKoKa0LlDvqs9XLfQR+qziVePw3c9zts/XLyHvjJ2CJ5\nnbCS0H6DTlnnSc+5OLxq1Pitd+N1Lel72jVetX9KUa/I3Qy4VLWWROs++dGDVntKtN6Un3looM3f\nEWp/hYUyVUHcsZZNG7/Gf1a/gfOnnB24Bk/g6wNAT1QjpqlI6wa0ZEiFCSnrCVwfSWSfBPeUVC+r\nqEkvQGzcy1xXmPfWlVAgLJs1Pgw9ZQXIhPh7TtCp3qiI6jx1UHRKy1NjQ9KznYcX+QHY7igWnAQN\n8N8qxCsY2g+ShgnT1c6jVO4gEUuc3/goWo+7MN2ForrR5u8Itb/CgJs4AYAky8hk7D5Bwd+02wWy\nUmg1EWhRQgN3014/KFXPOkpZCFapBadFhJnxGELwdH5dWiXMsmpIlT2hJ6qF1sjqcxhGvFKRRS4M\ntNfvRQRf6JTkKYixjJSSz4JXmxIaAXL+n1ZTKZ4LiqaRAD9kypbr1qcU9kVJ4icMQQgPaWyYRMWP\nbn6Jtxfau6xLIkec1q3fVfg8owvUm8mYhYG5Qa0GlH56BSj1gejHPchLihhFH4UyHl09CWlwF+jU\n4vF8CIIdEySSSOAe79kvkRcU+b7QEco7RPCF/erO8VoYvCxKTjn2GzopTb1cU6kWKfvAHjKwFilK\n+xc/ZQ3aKm3fqR/vHCKEh1YryYkw1+aHjIW9t+1dysL5XNvPZb4pNXj7yXTsPylacg+s5rrALjvb\n8mCTsVBjc0R64fmMv/Gyirmb5LJ0ZTUcdsphkixHcUstHoehdYOeSsGyrDzZJcUEEUEKandl0xX1\nSxQBSb4PdLgCoxGE0LH/0oUMHmJBsyjJMl2O8+C1D6Bs5sS++khlDJdeKyF2K4g7r9TuoyD68RAO\nHvIZFpxlHdoz+q+tC32S4H6u7edSkgA9nUY8HmNe32kOg6AWJ8dBHNPUwmw0j+u8YMSq8q4q3n30\n7R6ktmbpzm/NYjUcdhErpkxHccvsmBhUvQlK6x5YTbuzpJdnjTniSQpqd5LMQO5UWtC8oIwOV2A0\nghA6ZcC4szGwH7Bcac5A4GTaQCJXZyimKdyB5XbgsH3duvW7MGRgbVFgMyswva2Cj/2m+Jc6CP2K\nBSvwwE/GhhoU7gYpuL49Ua4qiLVRwDmLMDqfCfs+r/3gQ2zd8BHGnzGGIpAQcNyFYzhEg8G5xuf2\n0MxYmDFnWbB99BFc7bx/2WKlAsHulIbDznUD8JTp2dCXAdpcBfM6mkwHQRhB8zzPRBQw3nGxX9Jd\n2iFMsiixKmxbFtmtZFt+WnQjn27vtnR5teJoi+DjIG4iv/rxBmNnTeLk4phhgFTWoT3jjco1FbEc\n4S5VKQsbpFpgTpD6G0rIQIsxLE/tlF7eXhCy9vBaGXJ7aLuqSlJfigbX/RO2ZlEaDjvl8MhU0/Xk\nMaQ9c8VSOfc4X7xTbyrc+5BA7w9I2HvK/Q6rtUuE9sF+SZ5YcJMCVoVtFvmwx7USSAYtKJcmoxQI\nw03Eo59TLi9Zc5JP0RgpXtgkzT6owiRoou19ZBl5gh7TlAIXcdhxUCKksbC/oQ4pTnj79VEos81Q\n6rkEGhPzEks13ZB3VbV1famiw5ykJ0MeT3A5155R3YlV9M8o2YKhknqCq9H9vXvvPYlsF3/J6MrY\nb8mTSCAyycoiQj5oLU3cQblh6M6Ltggsdx78fsoBOP/PO48IkoaJ5rQeqoVPtL2PJGUJum1lS+tm\nvvI3bc9Ye8dr2eMljXbBzHkPvwNZUQpTyQnZUcINb93jQiI8ftL3A8MPsSDBUbVbeF5esqBQ+qUx\nDnOuPbWvD4u4kp4rSpYjbY+DFjsF+NdesPdAFNfUhbFf3k3Rw5YUF+SXfNCCckulOwuldA26D36g\nNGQtqAXNssKz8Im293HeyxbdQEMya21z6sbbcBjwbpZtQ4Q02jrcce3paGpqLDgc3e4Q4qHN6Fvn\nPpB8B5y7U+EFrS+h1f3hJRZunQPo5CawgN1gOZ3NWiNcrydqIJVXQ0/UMGUXQGBPw+775nyuDK2S\nneVII38B3Z++SNB+5sre37DfkSfRw9ZvAcxSoBQZWaVyDZIOftp+iayD1GQ3bFLmd19F2vukcskI\nzntJ6jVX2JJmAAAgAElEQVRHyuQk3X/Sd15uZZESE5akIJOxmG4S9xu+O92d+VbOUzOJAGIqvFbJ\nd2j5IFo0OYHJGiN+hyovN2767KWQVM1lBYyRr1fUAhcx1QLlhhcRYJBkX3Bc52wn4yvL0ef8znXo\nqTTX8xRZmvYf7HfZdpIEVMQ0SJIEy7LQnNaZ/e9obUD8ZqkFbS3S2VqAeJUBEFkPa2xYjYHD2F/e\n9j5htashfZfOmKhKhNPCRpKA3Vs34/JbnsArT80pzACjZRbRsu8c493ZRsLtTFxzAChqL0I76ESz\nwLzArTthXwytsuhaXnn2OACeMu359XgPxDQFad2ElqwTWyjhfge+j5Q1ka4v+C6kzDluPTiy6Nz6\nmWXV+bNGad0jrG+Ubddxsd+RJ4D/0LLHGWYGhpnJjw16wPq93j50wyIKYclhgdU/EOAnkjbpnD57\nKRbN9x7blsQ2yD7KMtnixJqD1VtRkrJNk1klNvxg68av8MlH/8ZZEyd6kgMaOfIaT/y3B3yRIEof\nN98Hsa2znxR1vYle4oF3LySZTJZc1xfMa7YApkERKAAOkhyKPNcYKjkMG37W4XgegpbvaE/ydP+R\no1H/9ebQ5HU/sD+u/HhVaPLaG/ulXZHH3eZ0g6iKnA+cDcN15sfdx1PRmxRjwyOvVKDtlT13XFHy\nbizDzCCu0HWxXV6PzxkHANSxrKrwXvDjAgyyjwlVQVXC+1qvlkLu7+09j2kKGpLhuJUzkgyjea+n\nFcTpjuIK1HUfJoKHi9Olwx0YTMvM8gE/8T0FerLcYQLZfAUySX3m3C4lHsbOOTfRpeWXiHLGCYXW\nHoVDH9/XRDFPXRr7JXkC9r2ls74nHaSsA5aXSNEsB6zxXoSt6ABlHOqlaEhMAmmv3GtJ56o9z5y7\n3FOXlGlCVWTugGyvNjskiLaQ8buPIq2CvILQVUWGmbGgKtlfZ+eeh3VGqqqKtJ5mLIgS79EGB4ah\nVXK3ErERuMaOI16qKOaIMmcBHHqGUu8nZ+0oInOO+8B9kPOQEUcWmt+iljR47kdI7VHaIiYpjEy/\nCB0TXeLu+End5jlQRQ5S3gOaNM7rWi+LiPuAlWV2YLFXQ+Iw4d5D91pYdbRI0I1M9gAwiv9gumUD\n/ggOr2UvCJEm6cozjqSbqsiYMWfZPvIUYiKD3b4mFo8jndbpA9vrLZszSJdoIfLpVnIHpi+aPyFP\n6kmNaLmsUgH2i5YV5scSSMre452vKOuRBN7SFB7kLmh7FGcrmpIjQKZfoozdEilC+6HTkyfR1G1A\n7ED1Iir24eLXisB7LetAFCEkXg2JSwF3YdFETC2YW+Sw11Q5a65XyY+uLQtAkWswCFGkPUsk3UWL\ngdq6+qk2z7KQBoWzEnlVeTlSubgiGtqlYjIPaQspC8o+0EiVrIt0aKvMK6mwoa5dL8wuVVDk2mId\n5LmMRy9LmlcWGo2AcZem8NqrnOXJ7qfnJyZp3xpiwUoqhJBNSPrO1i+RyBa0jdDx0KnJk2jqtg2e\nt3mb2LhBupY3VibItfb1NLgPWNaBS2tIXGo470mccE9I451wtm7x2ivnvU+ZwSwxbtcYSS/nWC83\nG0tXP9Xmvcinn7+97krkPaoq0Zpkk6esom3vQqGSNj8uK4Z+9oFmmBlmGxKuOcMiU66GujFNyaXw\n1wu5toxYVb5+Es2SZqOwjYurnYqLXLmLqNokj9Z8mMtal7M83bjwDTHLU8HzkCV9tj5cVjPSnnG2\nYvG8zgnHs5NMZgvaRuh46PTZdryp26SDhZYlRcuy87o27M9Yn4cJ9xy8c/oZJ5rp6LcsQZhlDWhZ\neF7Pj3tu0c959AqyD15wN06+79e/xI+vvkJIRhGYqe5pqOn6YPKLZNKzz/zKTOsGYprqmeFHyzYM\nmsqfR0Fmo6uhLivTi5Dh6G4KLKkaVEWml09wlgmgZPUZZgYz5y4vKp+Qnyedhhbj1NkFPVGNmKYi\nrRvQkns8t4q057yf0faep+QEz3WsulH1ehkURUZNTYXnGsNGlG3HRqe2PAF8FhavQ8N+M3e70SRJ\noloDSIeWVxaUyLWs60UhknUXJHaLZ5xopmMipqKMY//coM3jZ09FK33ztPNx9q/zYzniWYeIe5r0\nnbsSuYVgLJ72ph6aC4UoM7zgdbtytx1vo8Xj9Kwvmg6i7jzOGK4iCxDF8kWL/bLHGmYGWjyOi+Yt\nz8rVm8h6U/Sw98lq2g3L0IuKqNoWn7RuFBInhs6kPRGKeZJkSKoGM2NBUrX8+HzzYIrVjCnXbysW\n1hrd11gZJMpi6FaVYK8vQrug05MngJ05xyrO2L0sjnJNLfi/M0bGsizfcTJ+3Dci1/OCdch6BZq7\nSYuobrRxXOQndw/Wrd9V4OIT2QeeeDVe8Fb6ps3tJGBp3SwqUeD1nLpbrvC6+nhcwjwNrpubm1Be\nXk4WwIIXafByofhBqYLXc5W77WQLPZWmu8ZoOgjoRnPv2C6vov1yySpyJTLIgao3wTAzkCQJupHB\novkcerPIBqHifFaneljNdQUElJoF6YeAUOBOqnAG/fuVSys5YZiZQrms63KgvVgkEip+9OtVnuuL\n0PboEuTJT686Zy2crO9cKYqRaU7rgQKqaQcXj768B1+QEgekQHM74NRNWnh0E8lGo8HeGwBI5QLb\neaw9PODVm3U9S44XkoaJhmSq4Fnzc/9E5neTPq85ZNdfBHvPd2/fhgEH9ONbaA7M9iwFqfr1+bid\nsMhOSYLXHfFFWddVI9MCUmTZAABJ5tONabnKtl7h2i+nOy13oD8xdxzxOptgaKpMbH3iGdfFi4zJ\nJimkkgtO8O6hQx6vlUj4uXE+xzkr3cy5y7ksV8710F4skkkD91w/mk+XCG2KTh/zZMej8FSedsKO\nBbGrMPutxkyKO3HGmaRMsyieSKSKNSuuhSeehSfuC9hnveteFsdHX9Rh8CE1nnvBG8skUkXbvTdu\n3UjficJPDJaXHF4444hoMXW8+vHMb4/hiQ10/w4493zezG/ik3VrMGnct/kW6qfydMCYpDYDRxX1\nQPE0DBmFnwvEiHHE2viJxSLFgXGv13m/XT/TdBWSyfgutLgzF/zKpV1Xb1REMU8dFJ3e8sRbedoN\n+428RTcK/i9ygJZxuFTsz9z68lotWBYn3hIHtCrTzkrfTt0GH1LDVcbAabmh6cJbRds5vzsbMb8W\nx3dxxb8VikdvJ/xkwJGus+e75s6VAMiFQUVixHiCxW1ZrHXSLGLO+2FJCtIi1oWCt/20N3HKXVMy\nhFkqgFbgkuWi9FG6gGYF2fe5WHC9YWbw5LzxMEzyPgtbXXJrumLBimI3HM96aXFYNCulYz6STM/a\nTaKFSX08M34tnqzrOqF9Y79Ap7c8AVkSE88dDiRrTiky1uwDad36XRgysLZgTudbPgBqbzdWxpYb\nznE81gR7nN3njDR/mE2PSbqIWtloa3WuwW6TE6S5spfeIt+LyqVZeey1elnXRJ4Xp6xU2kCcYg31\n0rlMVbBl80a89+5qnD/+dO49ANiWibZCqawMrDnCsDyFBkf18bDnD2R5yukmkhWox7uTs+wIGYO8\nzXydMkptmRJF1Nuu46LTW54kiV0xuxQ93JwWC7vYZIGFJGctSJkms9aUn4w158+s+kX2OFWRMX02\nef4Uo5CmKEgWEr9xW04S6l6DiFxevW3LHM1SRIsHounPsmi5LZ7Oed3rclvXRJ5lt6xWjoKctCzB\neEzFdb95FWXl5WJv4yzLRFuhLYpVEuYgB0u3fRFRp1WnFPPbMrXkHn/r5eztZ69DlrN/C4pizAiJ\nByIV3t2FPdukwGmETo1O/1TYhwSpYnZYGWu0OZ0HE2kM65D3m7GWiKmYPjvbrNjpcqNdoyoyFs0n\nk8q4D4sKTUd7zW54peHzVIEnrSFpmEiljeye5K71c39tt6KdbemU5ZUhR9Pfi9w5A89pbjo38bYz\nIUlEmAZi9XOfWYI//8G3IFkQc63lDkY7G8139luQw4vhAgoNjMw60thQwSruSCIBpXCNBlyvJ8ly\nrMP+W0AMeC9IPEjzEyBCGYaSPzMROj06vdvOy/1Fc0X4deWR3Gci1/DoxhoHIG+BYbmtwgxaZyFo\nwDVND2dgdVo30WoUk0/ntckckeLRw/nMdC+L48aFb+BXV4+kypJloCpBLpLJ2kev0gxe17v31l20\nkne/wwiQ/3DdWhh1WzF65Cnc1+xTQOaLeSKgNAUlS+SSaeOAd+c6AHKxzo7ifgoKaoFOEiQ52yya\nUOiTSz6hqCo1SL/E9zxy23VcdGoq7Xxrpx1QQXqPseYTgYhuXuPyP3u4rZzXsGoOBXF7BbXssUoH\nJGIqLpq3HGbGQkwj1x+yr7UtUFcsWFGkh1sn9zOT1k386uqRSOuZAlnONdF6BXrtIyuo3W3ZIl3v\nfj5Spplvm8G7336KlJKQMS347rHFskzYyAUZuz8LzX3iCjouiUuGVfAwbLjWEVrqPWGekowVhLum\nkheIhT555dsgPjP7CrnytpKJ0DXRae8s6eDmyYpyX0crBOk1X1ixVLYFgzanc5zzZ9ohSLvGDdYh\nKlL3KCgJY8VKLZrPLlTq7NGX1s2ce8gssLK4XXFuV5idYRbTZDQks7JIa6LtF+s+sDIQbb28yEzR\nfRfY7yBFSt0oryhHS2urQ7i/ytgkGLEqmGXVzMrXntWYeZCzEoRSQJNE9nIQibUpkskL1zq86ib5\nge+YIV6I3kPedeT2hubeYxXfpMtzFXLNNVFmkXDP7L8InRqdljx5BdfyXMcqBMm6jmSd8PtSTiNh\nvIU0Ra9hXe9HRlgkjCbXq1CpTT73kSAlfz/c98j9zLgtSnYtKtqaaKTD/TnLqkTTiwVSwDmP5SjM\nwPqysjK0tLQA8D4ohQ7S3Fu93YCYx3Li56AOM3DalmWWVRfrkFvP9Nlili0/a3KuI/RgcBELnddY\nufhviW+CyQnafvidV9UbCwu5ehX6pFiruBBZqzoFOvVd8spqY9U+clevFpnPbZ3wW3OIZhkQcYf5\nuUZUHy+EQcJocnnuDa0+FI8FSZQo0WDvlZdVSZTQ0ALSeeHXTeeGJMvIZCzvg1LULZazEliWxWc5\nYcknHNLEa2SFbmXgsKjZsiRJIrYl0VOpfN05VpsOrjV5gZKd5guueXn7zLGseXqiGlJFDfREdcE1\noblOeat4B5jXJlwACggZk7D6bDtUalIZITx0+oBxoLj7ux1c6xUo7n7rFw0itw9Mr+BrllyvekCs\nytKkgGK/wds883qtxx2IHVYNKV6QZJdyPhvOPbPJp1cgPq/FKazAfh6wdNqyeTM+fPNVfPe8KZ5B\nyL6ClB0tKbyQ7e1W2FRWT1ST6/+4dGLVm6Lq7QoKtscZZgaWoRevUVYgVdQwq3mT5qXKE4HPAGZq\n8DkjOLtov9xzu/ehuQ7ImORrvSAr+Wup8wuuk+uaXP2oKxaswAM/Get5L93XGlolV60rO6HCXfOq\nXi+LAsY7KDq15QkoDC4GskG1NAuK+y0+aD0or3IEfuUCdIuBLa+MUDU6DCuD39ICpEBsO8WfFoMU\nNkgHf6mJk/tZo9XO8quXV1Vokj6ikCTve5NKJZFIZNO4WW/chVlJAiTAyngfSrKSk19InCArhT3m\nCBYoNd1AbkjLqgoOsiXAXr/SuofsFqqo4S/N4ErD5wluDi3WirJ2d9q+l960MgiGWgHdyOT+DhgF\n5EfEzUiyXumJGmjxOPY2pYSsSH5611GbP9MgyVRrlRu8/R8jdDx0evLkDC5muWxYLqmgLi/RoGGa\nq81dhJFkRbHH2zWaSBlgQcGylvC4Ge1A7GxxRHoMUingJbcU8zoJTpJRuFRUL8vKylZkKWuV8OHi\n80L+Go9709TYiG6VlWwLkSvOI5TYjZwM+wCVVK04nihj5smK+5AugGucoVWyDy6Wm4dE9lxFQUnN\ndYsgeGBSCZKgS4p1aHPp46V3Tp//fFmX/SepuSUPOSARY1nJxzj2qIyLEw2RsZLMbP7sHmvvq18C\nSiJ3vrNcI5QUobntFixYgJdffhmbN2/Giy++iIEDBxaNWbhwIRYtWoQ+ffoAAIYNG4ZbbrlFeC63\n2w7gc9nwNEfldXnxuoO8XGusdh088trCLcWan/Y5aZwftyKve4vkxuTVPSiCyvW6nqexsh8Xn/Oa\nJ+aOy7u9Se1b1ry7GrKRxugxY5gNacOsK+S0YmnxWH5tAMjyCa6dQnlZd5/d+BpAsUuJ4qLjXY+X\n+5AKHncbq5WJiK40OU4deN1/jHG2PgXzAMIWFdKe7vvMhJasE5InCp59dbpeL5q3HI/PGcdVY8pL\ndkqrRiKhQiuRtZ6FjuC2+/LLL3HTTTdh79696NGjB371q1/hwAMPLBjz3HPP4dFHH4Usy8hkMrjg\nggswc+bM/PcvvfQS7r//fgBZIvroo4+ipqYm8HpCI09r1qxB//79MX36dDz44INU8tTS0oIbb7wx\n0Fwk8sQLVpyTX0LEOyfpgLO/JxVh5F1DW4O2hzwxZCJkyO5ZyNpn+17YvdtYBSt5yEUYxVNFrxMp\nlMlCuaYipilI6yZadINrfncxVVovxA8/eB9mKokHV7Tu6x1Gi4dhHbwCh7LzgLdjldK6AS1VL+7S\nkIp7nwHg67vnU2fR+Bi+eC+PgzwsOSHBGZ8GkAt5coFEjD3IcqjweKad9x0AXzFPL9k5uT/69Src\n/5OxAZT3h45Ani6++GJccMEFmDRpEv72t7/hL3/5Cx577LGCMc3NzaioqAAAtLS0YNKkSXjggQcw\naNAgfPjhh7j55pvx+OOPo6amBk1NTYjFYojFYoHXE5rbbtiwYejTpw+8uFh7x6ez4px4VBN1PzkP\nVVp5hZisCGVgsWpDtQVoe+jW208Mki2vXFPzPQtp+2zfC7vkhB1nRWqFw5PlFiQmy+9jzdJLNOvS\n7SrlgdvlTCs2qqgaki0tBdlD1PgahiWCKyaHkMGlJffAaq7LWh78tHcpkJfel95PioPiXA9pnJ+Y\nFZFYJc+YHc4526rPnpquz86jNwXLsCORpDCJE0dWKA2GVgnDzOTvu0gxT6ZsK4Nk0sA914/mk9PF\nUFdXh48//hhnnXUWAGDSpEn46KOPsGdPoUXXJk5AljwZhpF3dT722GOYNWtW3tJUWVkZCnEC2iHm\naenSpZgyZQouvfRSrF27ljquoaEBmzZtKvhv69atoenhhwQBfIewDdJhTCuvwGryyyO3PRB2HJMt\nzyYAtMbLNiwL+ZITdo0nu8mvDd6ClG0VkyUCL2JFGvvAT8YK13RykntSsVEAiGkxGC1787VuDDMj\ndhByxuQw6zH5OCzJ8hwuR696PYJQ9SYxUuInfZ636KOoHD9wzs0qPtmBg6EDlQfI3b+CauYhri3Z\nmkZjQxIAsHXr1qIzsaGh87Xc4V3H1q1b0adPnzwRkmUZvXv3xrZt24rGvvrqq5g0aRLGjh2LSy+9\nFIcddhgAYMOGDfj6668xY8YMnHfeeXn3XRhoU/I0bdo0rFixAkuWLMGll16Kq666CvX15NiJxx57\nDGPHji3478ILLwSAUNI2g5Agnqw2VjA4Lai9FJYvmowwYDclfmLuuMBFGIFCAuC0IpEaL9tw1twy\nzAxijkwrkYKUYRaUFIHX/WS1F3JXxw+abUkqNmpD0zToup4nIJahix2EPIcnRwYXXXnCnzJOeWFZ\nYfKHsLO2E2edK7+koj3rAjnn5tGjraxdQpBkSKoGM2NBUjVxIupVzTwgevXqhpqarGXlwgsvLDoT\n3S6szoBSrGPMmDF48cUXsXz5cixZsgRffvklAMAwDHz66ad49NFH8cQTT+D111/HkiVLQlgFoIYi\nhRM9e/bM/3zyySejb9+++Oyzz3D88ccXjb344otx7rnnFnym5A7r3bubAIGYJ9qhmTRMYnCs+1pn\n7R57PI9rzT6M07qJqkQ8lEDloId8WEHTsly4L81pXVgG6b4474m7GTANKdNEAipmzl1edI9E9orn\neeABi6S5v+PRkeTKW7d+F4YMrIWV05s0VhQsXSxYkO03ACuTPQh1GarAYUG9xiY1LiLBK5savyMi\nL4QCkwXB0TpnjR849sXnnPkaRHobNiZ2rRdAwdpZrqh2h4tE2/0iF986AX5+ffz8LvBi587GfJ2n\np556CqZZ+De7qqp0pHlwr3K0psOrL1XWqxwAuNfRr18/bN++HZaV7auZyWSwY8cO9O3blzpH3759\nccwxx2DVqlW45JJL0L9/f4wbNw6qqkJVVYwdOxYffvghpkyZEng9bWp52r59e/7njz/+GFu2bMEh\nhxxCHFtVVYUBAwYU/NevXz/hOUkuLucbtQgJEiUrScNEQzKVf5v3W0WcJJdlYaDJCss1lVAVVCXi\n1BgjXhk016O77IKXnqx7JGqNCUqcWOuifSfacsXpqgzbxUivuJ6BBLevkBIszoK7FpDLYiFsnfBw\ne7WZtcNtQQL43HF2PSln2QSBOYVrEIUFP+UNSgm/bXDCcie2wZr79etXdCaWkjyVCrzrqKmpwRFH\nHIEXXngBAPDCCy9g8ODBqK6uLhj3+eef53+uq6vD22+/jUGDBgHIxkn985//BADouo633noLhx9+\neCjrCI083XbbbRg1ahR27NiBSy65BGeffTYA4PLLL8e///1vAMBvf/tbnH322ZgyZQpmz56NO+64\no8AaFTac1hH7kPETL+TXHSJJKOifZpiZvKsrqAWJNt6LlAR1TTkJGCnGSFQG6fB3/tu5HrteFAms\nexS2C84POfVaM6+OkgRq82K/ertBKvTaLR5DRitnXuenSCORYHg1ACa0Q2Eefn4ONh9xRG6i5qWX\nc798BVOL1CAqAUraW08ARc8dg6iS9rlDuhOBNr+fHRFz587Fk08+ifHjx2PRokWYP38+gEJe8cwz\nz2DSpEk499xzMWvWLMycORMnn3wyAOCss85CTU0NJk6ciPPOOw+DBg3CBRdcEIpuXaI9Cwmk+kkp\n0yxKC7cJjh+w3DPuFPC2aFdS6nR8G6VsA8PaNyBbNNIwM6HWaQqqt3M/WeUVvPbN674EqfHl957Z\nz9SUa/6I759Zi0mnn8JMq2am6ZOqUDNS5knf8bZQCYKgafwF1zPKORSWTkgXV04PS9cQ96bDQXAf\nSa19wtQl7GcwmTSQTunt0p7ltdMmonVzeElaZf37YdTrL4Umr73RJaktzTritryUqSqqEtm0eFGw\nLDxuKwNAtviEXa+J17IUdM4gljiWDJJ1xmm1s5uxtlc2HEk/53OQ8KhLxdo3L4uoSPC7+7og7lr7\nmbr9qpNymQ7+Kk7TrFLUt35HPI+zlQqz4rcoeIPMBWVyVZd2x2PZKf2CB7qX1aTLN5p1l7SIx4or\n0OdAbO0TFLQ2PrQm1RyynM9QIqFGFcY7KLq85YnWWNddmDLlkdXlvt7LwuNloShVtWv3PO1dUNMG\n73ppFkO7eGOpLE9+CqRKQEFhTgCeVj/a3DzNpWkFLHl0BeDrebNlvP32arTWbcGoU0/2WAzh7Vuw\nUa4NUmXpsIo7sqxDQa0TQtdzVhb3Sw59F+3sbLBreSWqIctycXXvEuwFrQJ+QUFXzirz7uc6b3lK\nGUgnI8tTR0SXtDwB3jEwmcy+5rV2kUWRmBAvCw9pfiehcVsCwny5cBI0P/Wg3D32gkLE8kEKsrc/\nr29NoTmth06caKn/NP3sOl12EU/7OXA+EyLwep5s/QBwW/zcey5SR8yGM2YwlWpFeXmF90Uki5NI\no9z8AsjxPKHEp+Te7KfPXgpJ1QosBkGtE8LX++1l54bbQkYoNFoS4hRi/8JAyLWXiWlZSw0pji7U\nvXD1cXQWYPVqUm1fT5a1r8ednkohEVcRT2jBdI1QEnRZ8gSw08UBoEU38plLfjLpvA4jWrYYrdJ4\nmEUv/bpqyjX/rkwaRAPVnUH27qbHrEKRfuCuUs5zD5xlEOwinknDzD8TAEJLSiC5gHng3HO7yKjI\n851QFZRr2T/aT8wdhx3btqG2Z7XHVS44SIosyshZh13Qg89Rm8dOU9ficUBWxFx2BMISyOVHkC9a\nWNT971IGQ4fhEgzVrZi7r5ZlEZ+bUPeiyO26rwCrV5Nqrow/x72Px9u0olAETnQp8sRzmLqtMa0+\n0/4BfjcPKz3dXWk8LAuUn8w6WS4skhimBcpJDljZaqTxJASpsu4uVeE39d/W0e3udd5PHmuWUxfS\nZ0FKZSRzPf9E9skmbPYbvKrIaNi9HdU9uvNPngORpHCSilIe/PabvbOthki1ceKhH7Z1w09hURIB\nLJHFKTBRDJtsIntfldY99OcmxL0oej5zsrXkHlgte8guO96MP8e9T6XELNkR2gZdJuaJJ6aGNxtN\nRCYLvPO1VfwT6zMbpOayYcZN8WTZee2B6H3cs6cOu3ftRmNjPRKahsryBDKQceA3B+aDMcO8B7Ys\nu5BlGFmPfu6B6D45kVAVqEo2dsQwM3jysUdx1phTUNuTrxu5O6aIt2BkmyEXAzN99lIsmu+KgfEi\nHB7xM2bGQl1dHXbX1aG+oRHWrq+RsSzIkgT0/AZURYGiKtByhfvKEgkc0K8vNI3invHQhxYvU/qm\nv8HnaStd2xJeaxJZc4NZCUkKp6uGKKKYJza6BHmiHRKkA4f3kLRTzp0yAfEDjGe+UpInP3PJ8r7y\nDWHpZltzaPdJ9JC39DTqdu3Axs1bsG37DtTV7cZ3pn4PsVhxhtMzi55C79qeqO1ZDdPM4JeP/gvX\nXnAMTho9BpJU+ByZZgYP/O5elCUS6HvAAejb7wAMPGwQunXrxlybW19WyQKg7e55kPIGsrwvqeLc\noxtw0tDBGNCPXt03Dxq5INVxakf4Obgty8KWbduw/svN2LhtJ3Zu2wK5cUd+XyUJkCUZPaoqUVvd\nHT169ICqKJCRQSaTgWlmYGYyME0TumHCMAw0tyaxeccuGLnnQJYl9K6pxsH9++LAY0egX999/b2o\ncCtM5CkAACAASURBVBOsUpYncMgOJfWfV9f2Krkg8tzyBqZzrqXeqMhXGG9rROSJjS5BnoDiQ4J0\nONmfeWXW2Ye5bTkIkq1ky2PFX7GIQ6nrQAH+dfOay51ZaGfQkTLmRMjEI79/EBaAfgf0Q22vPjAS\nVeheXYOq7j2Ih4wqSxhUW4nps5fi8TnjoKoyGlp1bMo13CQh2dqCndu3IVO/E599+glM08Ssy35Q\nNM4rq5OU9cibXReWVcr+3g9hs6/58zPP4ISjD8NBB36DQyFyaxLfVoY2IgIk7Ni5E+v+8Qr+s+Fr\n6KYBCRIO6N0Thx58EA7u3xe1VRVQFLK7KdX7SKiV3WEYGSDZiPiOj7lUymQy2L57L77asg2fb9yC\nrTvrYP+ZPmRAPxxzyhgcctCBpU9h96rLZTT7yqL0g/ayUBmxKkiqVpy953FNWLpG5KnjosuQJwAF\nb38kkuDHZWeY2bdFtxUqbDeWk1D4Pex4DlzR9HWSfrxrcha6tPtvOUtDJF195CQJSKfS+L8P12HN\ne+9i0uQp6NM325Jnc2O6YI4BVQlUJTQ0JNkkiDR+W1MKRobvBqqyVDS2f7dYXt+4ZOGS21bg6dvO\nEnLLsu6tyH3nHStKhN1kb9Xf/oSjjjgch36T3E4pO9BFmpwlAHymivs6iAKQrbRu4o0X/4y1/1kP\nWZbQq7oHhg4+DIMPPQiamg3cTfU+EmpFFYzmBjohUjSYBw7HtNlL8eS88VAkQPl6DWCK94C0kclk\n8MWmbVj7n/X4YtNWmJkMBvTphVMnTEZ/H62rWCDuu8PV+ficcVAVOZ+Wb5gZWIZemsKTQPuUXMit\n18xYmDFnmdjcfktQuD6LyFPHRZciTwD7LVvkoAEKD5tk2gjdzUI60OxaPqlcoG8p4rNslxLgXZfI\n1tGODeE5dN37ZsfO2JanD9fvwpGH1EBVZCTTBr7avBkrXl6Out27oWoajj5mCPoMGoLyimxqvJvA\nOC1Ji+ZPwKe7mrjIEIkIseBF0FRZgrR3O57/67NQ1RhOG3sGBh52WMFeAHTSYn/vJpC81ehLFcNH\nGvfmsr/ikIMOxBGDDiNeYx+2AIiHnP290CHrg3D5fev/9K0VeGHN54Ci4rQR38KIA2Jky46iwTxw\nGKbNXobF88czCVGB5SnVhPj2j7j14cXGbTvw2jsfYPOOXYhpKkadcCyOOW18MKsUY99J99n5cyjE\nJkcgnPcSQNtbnnIvA6KWJx7wVs+PyFPHRZciTzzxHSItMAAIWWn8gNWOhJewiRyizrFPzB2XJzBe\nh6lNgESsITYBdBKvhFJM3N5a/R6M8mrU1PYqkkUjMKKWJ1F4ETS3JauxqQkfvbUSGz77DAcedDCm\nTJ6M2poezGdHJHg+jEB7oDjTkPQ9iSy/9+rf0bOmBscecxThon2Hrf1MhVWgkGYB8d0ixoFdu+vw\n9z89he2763DkwG/ijAuvwKxfvpYlRZvXAelW4nVclicbioZU7WFQy7t5j1e0QJapZDqN1975AO9/\n/BlimorTvzUMR4880xeRopFQ+3NnIcgwLU+0wpNW0+59g9rA6hQKcaNZlgCYZdWQJAmWZUFpzWbl\nkZ7diDx1XHQZ8hQkPoclw0Yp4pKcbkZS5XGAn7D5dfWkXK4zmp6ie0uyAO5pbEJZWVm+WnZcU5nk\nx4vAiFqSaHPQZNAImpdexu7N6FlViRse/JAaW+YVfyYSIyXy/PG4YUlked0bLyORiOOE4cOJh5dX\nP7dAcSAFAcrBM5l27tqFpx99GJqq4vwzR6Ffr2wGoU2K9LQBLaZ6uuW4iA6npUqIkHGgNZXGyn+t\nwQefbEBZPI6zRo/AIcePEhNCCEIvOOBb9kAqr+YnxYJZjNl2K84swhL2pWPoAYhb1tiWJQIxdFna\nIstTx0eXIU9A2zSsdR86fuf0CjQG/BMWr89Yn/vRlwXDMPDayhVYt3YtBh1xBI4bPSH/HYm4uD/z\na2HiIVY8smlyWMTKyFgYUJVA9zKNO8sSYBPlMJ5tJwl7ct54KLJEDVZ3P3vvvP02EpKFkaNG0Q8v\njsMxkNUgYCbT3vp6PPWHB6HIMi48+wxUV1cXE5lYGcz+Q/aRHYYFihcFxGj3+uI5BVyBVDDIXHPK\nwN9eeQ3rv96MUScch5PGn+Pbree3LILvcS4XXmjuQYHyD4Cg5Yn0nAJMYkjTKyJPHRddijwB4WSn\nsVLPgwShO+XzWBKCHphhp8OL7O2unTvx0ot/Q93u3Rg99tuoPfQozz/YXoSEF7ykyE/clFsGi+jZ\n39sB5rt27oSmaejeo0f+GhGiHMazzZsA4H523l/zLlTLxH0vN7drnzQ/Fqz6+gY8/ejvkdZ1zJx8\nJmqruzMtPVQLVBC3mqIh1XOg55xUyxNjbp61GM0NiG3/CKtWf4A33l2H4UcNwhnnTROv/A7AiHeH\nFosVERwqRN22slJYmTt3vZ39rKfTUFP1wnrn9ed9hgKUZOCKaXKvk4CIPHVcdKkK40A4af3Epqyu\nPmbOFh2i1Z9Z1zkrZ/O0gKHBb3sWL7158fa/3sTQ0eNxwQ+uQ6+BR3sSJ1WWUJXQMH32UlQlNKjy\nvvEipIYlxwkjY6EhqWPR/AloSOpF7kAeuK9xz2t/v7kxjc2NaWxvMfCnxU/hvnvvxubNmwBA6DkK\n49nm7RHofvYMwwQy5LYXbQmRquPNLS14+N7f4rGHfofvjBuFH198AWqruwOKBrWiCtNmL4NaUZUl\nJg7Ed3wMZfM6aDE1PybVZzDMA4ch1ftI/7p7zfn1GiJxSvU+kj63okGt6E6W61qnpMZw+onHYfZV\nF6F3TQ8suG0eljz1CAyDv4K1EauCFouJVTAvqJSe9q7cXlFDrNw+ZGBtttluLOa/nYufquaSnHez\n8V7jfk6dLjvbYle0zgidCl3O8lQqkGpEkWKWWCBlSwXJnvICqVo4j15B4S4rQENYLjq3LBE5YerA\ne21LUxNWv/IC9tTVYdZlP0BZeTmA8O9DmPjwH/+LWCyGE48nxzx5QsRlxwgI55XxwuLH8MkXX+Oi\nKWfigN61Rd/zxBjlx7Q0AvFKSLIEK2Mhvvl9XxYoX3FNHi69VO8jgUQ3qKoMo6le2KK17pPP8cLK\nN/HNb/TD5OmXIB6L0XVxW4BSaahpfguQp8XHy0IlK6HUleKxPIVatd29ruY6vnVIMur1ssjy1EHR\nKclTXV0zTLXEBeIIoAV2i9Y+Ek0T96urs75SKVyD9Xv34v8+XIdTRp4GgJ80AeG56Giy/MgphSuP\nhbpdO7Hyr4txzY+vL33Bw4D495uvQFFUfOuE44UPLJGDxyvLy0uGaZq4/7d34tgjB+L0E49jK8bj\nhstZcsyDhucPO+Wr9wK570SvpRKgHLGaPmcZFs1jxEpxzPnJFxvx11feQN/aGpw/cxbKysqI44RI\nhJPscrruAiUEhEHQWbpy1GWiQZSM2d8nkwbSKT0iTx0QndJt160q4ashbFA4s59Emr+KuNCcDYN5\nQJNlu4Ie+MlYpmvQj2uvsbERj/z+QTz91BPo/o2BebcUL8Jy0ZFkxVXZlxz7GporT0QGL2pqe+H8\ny35UMuIkItZrbFKrQgqx4ma4noIF3CS0sZwyGhob8fO5s3HW6BHexAngIzGmDpg6jKZ6LJ4/AUZT\nPTfh8j2nC3mX3u71RbKM5gYsmjceRnMDXTbHnIcf8g3cdNl0jD7xONx316/xx9/djcampqJxvG7T\nosbJnE2TveTTvic2aqbBgzhRdSVkkPLO6dabuU7H855IqB3+xWp/Rae0PF25YAXuuX40s4BgqcHb\n/NU93m/ZAb99+rxcg3YtJh69UqkU/vKnp7F3zx6MPPsCYl0mXstLmHWabFkp3UQ8pjBbr/DoF0YJ\nBFHYQeVhIewq5R+8txoxVUFFr8MwZGBt8FpNgmO9ZGxa+088/Ozfcf0l30V1d3ofwkBwWnAo1pxU\nn8F8NZ0EwXS/BawPRcKWHbuw+O+vIh7TMPWS76PakeTgCZaVyW/lbb/zucB6jmiZftxzAqG5tCPL\nU8dHp7Q83XP96AJritOSEhY838QNE6m0ke9950XceIO/SVYq0vp4rVluvdzBya0CQemvvvIyTj1t\nNKb811XUgpaDaisxoCrhKWtTQxKf7moKpcDlpoYkNtQ1IxFTMP2WpeheRg4U59WvrYkTIObyZEGS\nxCydPGMlCZCsDG579N1crItYwLhIkLeaboDVXFc0liXjvZXL8MzSlZh91cV04sSyBvEiR1BoAdyp\nPoOZQeG+4RHgHjZxAoADetfi+v/6LqZOOB2L/vgQFt65ADt37SoeSLECGmYGT84bD8N0PSccFeKz\n1pzu/MpyWrWYFkzSdyxdXXMaWqW4VRZ065X9vCdbw/m7ECF8dEry1NiQzB/2ZT5dTyzwkjEe4uGu\n6Oyln5vcAOTDzW+mH7CP+CViKhKqwn3tkNPGQanpRyQmvFluThgZizuzzQtmxqL/wfapX1vDJlCb\nNn7t63r7uY0r2SSBxbdOQFpnF0FlPUfOZ625sRELrh6dzxYShoCVipqFRJCx7NU38OEuHdfePA8x\nTSXKZGariYJGZBQNanm3fEau0dIYHqnJuecWz/dwz5UAvWp64NqZ5+Pic8fj+UWP4547fon6hqw7\nr+jgt8mIla02rsgSLENA1wICI5ZRx0XQWSSLl4CR5tSbxDP4AG93dDtltEbgQ6ckT7ankVRCIChE\nU/xZbVDcJIxFypzzOK1UrMPNbykDe9941+iMaSJZb2w3l2i8kIiligf23pBmduvXUbG5MY3/W7cO\nzz/3rNB17peImKbgigUrENMULiuq+zkqcz2rzXu2o5uaEcquEoZgGvkrS/6MxmQaL3/RD1pld7Kl\nx8tqIwoakcl9PuTQnlnXmkgfOw6dWKUM2gLVVZW4ctoUfO/71+DhRX/G4r++CEXT8vfKiHUvIFI2\nsQDAb5GxMtBTaSy+dQL2NqXycqngIRuuMSySJWIhLZiTRbwYz7ChVcIwM1mybWZgaG3vmovgH52S\nPAH7YnmSOddZKkecnH/w/Vihglh0bNgkSVVkTJ+dPcxkmU7K3AeVrYcN+3BLEwqq+dHPa43vvvN2\n3vrhdCeRrDdOAiTiigvbEmRkLLToJhRZAiygb2W8aIytH4BQSVvYOOa0M1FeXoHn//Ln/Gderjf3\nSwQtWYAmz51QYMuzn9WGxiZ0qywPuDIPCLz9r3n17/hq02acd9YEtkWmBFYbGpHJfy5AnISsYm1o\ncSJC0dDnGwfjvYZjcNSQoZj7s5/ihnN656plk+sgiVpk1HQ99FQKPSr3WaBI1/EEa1PHeLjj/IBE\nvKjzS3L+ReGaO1cCAGbOXS5mtYrQ7uiUdyqe0IoLSZomVEWGmbGgKnKgOChRi47zIHJarlRFxqL5\n2cMskyETFtJBRUKZqqIqEUc5xTUhCtIaGxsb8bu7f4MdO7YD3fsUxeGQrDe0wpBeCCOzDSgsaLmt\nKRu0f9G85UxCxkPa2tutd9QpY1FRWYnn//JnrmfZfomwXZasZ9jLAmo/v/ZLiWUBLS2tqKioCG+B\nJEhyccwT4TBZ99pSvLX237j8gkn0TDQHSmK18VkSwD2+JDFSpYKDiB418BDcOmsyNq5Zhbvm3oBt\n27bn3MRGQXaaqCsMAFS9iemC57JQ+imGGRQuixNp/jyh0iqhp1K4/ydjkdaNdi8+G0EcnTLbTjfM\nokKS7iwyAEVjSgFSppKzBUYmYyGmKfnvSQ1f7bpCzgKcTsgyUJXYt7aGZAqZkH/H3vzHG3j3X2/h\nqh9eBa28imk98luQ0kuWKEhz8+hDqwsFIN+XLqxMwKD46M0ViEnA0+/Fic+ys3iru30QzeLkVYiV\n9Ew/cs8duO6HPyjZOu3sorRuIKap1J5i/3nzFSx74x1c/1/fzadwh91UtwgcpMhvll2y/3BocQ16\nSkdi83uh6lQyuOaub0njoeXvYOX/NeNvf5wNNNcVu64Ei56G0QA6UGHLEFBYVbyenqFng7AX7dme\n5aMfXgx9547Q5Gm9emPw7x4LTV57o1NanlKpYguO2xUV1PXGA1p8lF2j6eo7VyKmKQXfO3WxLCCV\nNvD4nHHZf1PmiclK3jee1k1P4iTqrnziD78H9BRmz74F19z1DjVbzUkwbLBcdbxB435Aq+/E4zp0\njxlQlcA3q8txeK9KfCNHnEoVWC4qb/DJY3HyyNOoAd3282eXm+Bp8eI1jmS1kiQJmbAZe174vrf0\nmKbiigUroMXjRW/uG95ZhRdXvYX/vuSCfbVvSmy94XGr+c6yUzRosdx6Yyr7Osd3oQbA+4GLtHUv\nj+FHP7gMc384EXN/9lNsXvuPwvGUYpgst5vfek+iY0oJNd0APZXeF/xOsMTls/SieKdOh85JnpI6\nGpLFLokgveD8gHYQOQtU2llPpIPKdtkByMdGuYmPfUBeNG85AKDVow+VqLtSkoArrrgCf/u/RN5U\nrhvFf+xowd00y5FXMHgYMU622y+lmzi0piI/F4uQuQmgTcJURca0W5aiqkxDYyq4O5EEPwHyqixh\nh6ERn+uiFwbO559nXNGzmkigNVkiK5zjUEnr2d8bPZUqOGg2vf8G/rRsJW6Y9b3CZralzETjIWau\nLDtdN/h1yOn+wI1jmLoXkKUO6uqL7/gYJ/aSccvMiVj2j3fw4N2/RmtrK3kwr0uNo2eeJ9rRDZa1\nPBXGghUQuvZwLUYIDZ3ybsUTGqoSZIJgu/DCKl/gJ1PJ+XmLbhC/p8VG0eoy0b6nyeRd97YmHS2G\nhftvHAvDtCBLElp007PpLcAmVOWagoxloVxTiohSWFl2dn2neEzhshSR5jUyFhpTOszMPuK4tTEV\nWg0qG34C5AuD8cn1XtzPn0hzahHUVPfAnr2ETLuQ/uDbh4qW3JM/XOzPmnduwsPP/h03XTYdilI8\nn93Il+ou80swnMSMVnrA1GG0NGLIwFp89EUdNM3DgkTSnRWP5SZLQLuVLfCEqSMe0/CD756NKWNO\nwR2/uA3/efOV4nE+Y6GE0Z5kJEeM8qTa2RDZsd422YcIJUHnJE/xYoLgrqcUhtuO14rDcpHQvnfr\n6DwA3aTHy1JAqv2U4li3JMkYVJs1F3+6qwkb9rQQSQMpuJvmNrOhqTJmzFkGzfV50Cw79/he5TEY\nRiYXC0a3FLHm3VifREvaLCCOYRfKFA2QJ+lLK6Tp9/kWeano26c3tm3bXvCZUEsMHhAOF0NP4647\nF+D/zZoKLU7ut5bqfSTM/kOIbqxUn8GBXFzxHR/DaGmEWt6NKiO+/SMYzQ0YfHC1P0LDGk+wrPkK\ngG9jC9WAvr0w9+pL8M66j/HU7++DO7S21C415rPZRsHjeiqVLyrrLvFh6wegXV2LEfyjU5Ind8wT\nieQEdeGFZb1yZ+I54a7nBNAJGy341z0+aZhI5oKH3TJeW7kCH677AJsb09jerBcczjZoh7o7TsjL\nbVbfqmPRrRNQ31pIFIyMhUafWXZuy5Ft4ZJlCbqRwbamFPVaL/KyMcSK5zSIlHKg6bu5MY11a9/3\nNb/z+RN178ZqB2B3XZ1DWNu4HB69/17MPHciyg49gUyCGG4sOxZp3Ybd/l1cObdcXn6MTODi2z8i\nExrKeBEQyZIAQfMdIxWQcMmyjFnnT8SQww/Fz+fNRkNDY+GAElqcaM9m6ISfASpBdOsXoVOic5Kn\npJ4nHSySEyRQPMx6TwlV4SJFIoSNVEuK1pojnU7hvnvvhmmYqDkk+wfUT6kA95i820wrdpttakji\n051NRdlsA6oS6JbIxhV5kQindYhmOaJZuEjwCm7nJXJuvUQgQhZp+jY2NmLZ319kXut+dpzPH+kZ\nIcXaFVyfSCCVdli+aK6XEEnU+6/+HeUHDMShp02BWtmdHOdDi3mySU+u5ILvit8O+XraoFq47LFO\nJPsPhzngWCT7Dxef10M2N/zESClaUZxVEAw9ciCunXE+7rrjdny++rVAsphwVDinPZslJfxuebRs\nwrZyW0YoKToleQJQFJwtSnJ4LElBrFfuopi8feh41sKKl3LL2LZtG+68/ZeYct75OHzE6ILDXrS/\nHIkopIwMlYQ5yxkMqq3EN7rvy2TrFheLTyKRPZaFiwY/we20sWFXSCfBra8qSzj4uBH4/PMN2Xpc\nBLiJupssAYXPSFzxroSvqip0vfAAd79ZE9/qfR5QzS0t+PsbqzH9okswbfayrGt2/gSiW4xmmdFz\na9TTuljFbxfsmCotpvKTkFgZtLiWyxzUQrFA+QKJXDJ0t0mTWtkd0+csAxLdYB44XMxqRZBf3b0b\n5vzwEryw8k288/ISPythwv3sEa0+JSQt7vm9LFyWrBT8vwBR4HinQJe4S6IkR8RlwSIwLPlViXhB\nph0vwbPXYpc7oOlEi5dyynj/g3V46vFH8b0r/xuo6k0NmOYBiyh4WXSqEhounJMlTDwuO5qViTSP\n28IlCpEYrIKxZVqb98pz3oMzv3sRnnrs0aIxJKsSiZQ7nzPneFol/GQyhUSCQBLtA0hWit7qg7hI\nHrr3t7hy6iSYLY1YPH88kGyE8vV7+wiSV3PcWNm+MgCkIG5Ra0q6VSxQO92KtG7mM26RpmSeicCn\nBchJLpkuPIeVyjAyWDRvAlRVxrTZS7mtViz5iiLj+v/6Lj79chNeeuYJX2shwm1RsgkJgRyxXGki\n83nNz7RwyQpkOVvQWZblffqibd2KEYKhS5AnEYQRy8RboTmmKfmSCiIEz20JcMsHyPFSzu8tCzig\nf3989wc/RjyRCBSozXMtjQjZFqPHZo/LHuSAp7WL5VIkzRMkuFvEfekc29iqB66QHuQedKusxJBj\nj8Ob/3yjYBzNekl6/kiWSlol/GSyFYkE2XpiN/MtqJQM8dYcNv617HkcfvCB6NOzutCqlCMsXjE8\ndgC5njaIZQC4Y4BcZIE7UFvJWppiarb8RUyVA7u+Atd2ylmcmC48h5XKJqtGUz0/YXTLp1jbLjl3\nPBRFxuMP3FsUSO4LrlIX1MbSjvFOMNuouEAc67ZoZUy2hcvKdsGYMWcZVEV2VE2OShd0JnSJuyMa\n/EqzAsmE3aDVXaKRL9Jh5PyOJpdHvnudpLpRzu+blPJ8McEg7VCCtlLZ1pTKvsHeUhiczoKoSzEI\nROba1JBEY0pHt9w6/Or4je5iLj/SPTh8xGhs3by5aCyNqNPOqaRhFtRNI10vZ0zEqsgHjLPAZb61\nik8XSWtrK17513s4+/STyAO8CIDjey2mFpcw4LgeYJAVr0rjvY9Eqv9QmAOOzboNCbFYwlA0qBWU\nmC+X3kxw1MRyk1WhzD5afBhBt7NGjcBRhx+Gu371CxjO2nU+CYPd1iemqWLkw6uNissFTSM3bosW\nM5uQ9rsRxUJ1KnTK9iy7dzcho2UfXJ52EzacbSdSZqHFplzLdqJP6yZadKNovPMgoX3uhLuaOE0P\nWu8x9/de63R//8nOJiLJCdIOJexWKmHDr36kyumssYNqK/MtdT7dRd5nFr5RlUBVmYZ163fhmENr\n882Keed3xpJ1L9OYzyEPynLFWmlyJAn4+IP3Mf/ht/DSwz/OtpVwlhNwtsHQs2txuvNAaGhNw8P3\n/haTxo7EgF7VAMitV/KftTQSY5kKrtm9vogk0Nq5OOXaGXaL54+H8vUavoBtRYN54DCYFjBjzjIs\nnj8ByuYP8i47v21kUr2PBBLdoKoyjKb6omuF5Za6tUusDGb/IZg+Zxkenz0OqiLDaK4n7vUnH67F\nM488gOtv+h/EevQRa6dCCMj205Kl6BpSG5XcPJ7yRVrRcIyN2rN0XHR68gTwkxka+SD1jrMsb7Li\nZ+d4yR5JPmudzc1NqKmqQiLWMXqykRCEfHnBTc5UWcKLz/0JdTt35l0DFiwokoTJ0y5CRWW3/HXl\nmgJNlbFx+240IMY9V2NKx8Z6sX12ki+7NpVlQZhYuuX47d9oP1Pr1u/CkIG1VDn/XrsGVsbAt044\nkXpoGFolJDVbrZ3Wm46FpuZmPPDHJ3HDzf+TJz7mgcOIJMazl5yiIdVzIJ1UuAlEjvjYc9kEipeQ\npNI6vtqyHZ/XW9ja2Iq6ut0wmuqBLZ9mf5dlFcohx+EvKz/DtHFHofuO/6BHQkXPbuXod8zx6FNb\njcrysn16Of5vHjgM0+csw6J5BCLn0pub6IUN137a5AiSlP1b59TNpXPd2ldx16PP4Ee3/ALX3L26\niLCQwCQxjJ55vOTGj3zaNcKEzkWeFEVGTU2Jm3ITEJEnNroEeQL4yAyLfIhYnoIiiFzSOrdv34Y/\nPPgApl75Y1SWl5eMoJBQSkLEwt663fjg3bfx9RcbcP60GTh+4IF5a1BjSkdlXMOGrbuwekcq387j\ntIOq0bcqgW0NSbz+1R4kVBmTj+wDy7IwY84ynHX4Lnz0xSZkMhnEE2U45LDDMfCIwajuWVs0/zdy\nJRf8EFWbfDWlDGxpTGJQbSUunLMUT80Ts2QdWl2OxP9n77rDoyjX75nd2ZZkd9NJh5DQizRBAQHp\nQRFEQJqiF3u93qLXn9KCDa8FlWJHkKJioYgB6QhSpPcSWiAQQkjZ3WTb7M7vj93ZzMxO3QQVL+d5\nfCQz33zz7e7szpnznvd99Vq4PD64ZNr2CIFP5MUaUwPAoYMH4LpyAX16dheZLPC07vMH3kt+c27O\nzVDk5vPFh7PQ996/4flPDodutoIESAlhiIBUhCk4AgqN2+PF2eISHN+xFWdKy+HyBPbrSRJZSbHI\nTIxDSnISEq0x0FLugO8nqDzRHQdDl5gG+4XTsG39HuUuH8orKlBmq0ZppQOO+IbQRpnho3xISU1B\ni8xUtEs1ARltJZUlOSWuzpBRqkSVLwkCyz+mxunCa4tW44EJDyMjPV2aZEgoQ+wxilSpSIiWmjUp\nWavEGt26OBiNJHQKLSn1iRvkSRrXJXlyVLvhlMhGkwLbUM2HRoOwpruRKkxK1qGmrILY2PNFFodu\nGQAAIABJREFU57BowXyMePhZGISyoRQgUgL0e4Ti2Di0Zxf2/rYNfp8fsfHxuKlTF7Rr1RKpVhOc\nHgomPYlSuxvJZgPGTirAwvw8LD96GS7KHyJK/O09GsYhKVoPktTgUlWAVAGAx+VEVFUxzpw8jrxh\nI2sb0aKeQndWI8yGwHsHIKR+VTmVvZfMGp747zrM/nefkFqqFkqJ/LGjR1BZdAID+vYWHUPpLbLK\nk9hTeE2NE7NnvIXnXpomSJaUht4UjxEjBLztlXYH1ixbhpMXr0CrIaAnSTRMikN2SgIaJcchyiCu\nVDJkyVt2EcTuFYGNQTIVtk9vAnrcB0JDgKZpjHrhGzw2IAGHVi6Ex+WCKSoK3ZtmokPfAdzefsxr\nVaDERaJIyb7PcsqY1Ll52ymfD2/NXYK8225Gq+79a8epDM+FhZFFSAyli1Ed4pNCnZUn/hqry0FE\nx+OlD7fizad61Hl9anGDPEnjuiRP3qChNZKVy/k7gGtHmCKB1M3t6OHDWF2wEndPeBIkGVk2jxQB\nkiJV9UEg1KKs9DIuUAZotYFaRXxCtPrkFVS5qDCFiYHQdiOvuKZLoCkyG+1TzaD8NHTOSrjtlcjK\nba6aOPLfu1Pl1chJiMaYiereS3bIscrpRVVlBSxWq6q1ANLXO7Pv2NEjqDp/Ev373C4zmSZwEONz\nYt/4JJ7CP5/1Hgb16IKMlCTlN3ol4+SIl4AniqZpbP5xOTYfPg2zyYA+bXPRJC2JQ6BloTeBvP3+\n2jDghvm15QqE9gEge48PvTeg6QCxOrgW8DjhcLmx9cgZHC4qAQC0zU5D78FDYDIaZFW2SL1WgvMC\ngu+nlCdLDXGjNSRmzv8GHVo2RZcBQ9SHz4LX2NjJBZgX9Fsxx/JJFRGTEApXe92esBYqoueQgpjH\nj5mH/38e+K/Xa4yHXqe9oTz9CXFdkqdIlScl/o5rFaqLBFL+qP1792Ln9l+RN/YhdT/qLEgRICWq\nkpDPqC4EquxSMbZv2YSU9Ex06nobAODgZYfkMWJEyUhqQkSI/W+rkYSb8odUJ+ZYAILziJ1rfeFl\nOI5sx/Gjh9GsZRt0vb0vtCSp+LXy37tIVDxSQ6BpUkyIdC354XukZ2ShVes2itchBfZ3Yd+BA6gq\nPoX+vXtJHiP3lC2kCthsdnw+5wP862/31su6JcFSSvhmZqfLje8Wf4VTl8rQuWkWerbOgVYoBVch\nOOpSkAQJ7guqUvzxdJu+4coVAL+fxv6zF/Hr0TNweihkJsWiz0P/RkrDxoKhs7p4otjEC4BoeM6X\n1RGjJxWEe9NUEDf22MUfvYvUBsnoO+4JxSEvBmwFlPmtDx3LIjfMtSg2v1ripvTa93gp6HWkvOlc\nowURHY9n3t6IOS/0kX3d9Y0b5Eka1yV5EvI8yUGJv0NN5t7vBTEy56ypQblP+c1aDEI3bTWqEkOY\nQiZqlxfnVSgxFOXF5jWrUHauEM2a5KJlx6445lbX74lNjvjgEyQmRHfZVhveW5SfBxoIC+nxzyEU\n9gMA3ZXT2Lr+Z2RkNULvQXdBp+eGcsRIJX97JOSTed/dXh/gpzBjxgw89szfFR0rpzhZTQaMmVSA\n+ZMHYN+e3bDZqnB755skJlTo7wiay5kbzSfTJ+LeQb2RHB+raN11Bd/M/PrfmuPbdyYDPgqDOrVA\n07Sk8INY3iVV0JtESZDgnMw2KeWKh3NXKrDhwEmUOtzIio/B0HtHwhpTazCOWHliwJRvSG8fCCv6\naRiK98qrW2qIm8DYxctXwZTWFIPvGaEutMa7DgWVp1AY2QqdQR8+v9qMO7lrP7j/senr8OELfWS/\nI7VEy3dDefqT4ronT2pCbEpUpT+T8sRA6DUW2z3CgyOA0E2bTapKHG7Jm7pQBpmSLDS3y4WFn8zC\n7QMGYVD3zpLkJRKwCc+i/ICB2R80hy/Kz8NluwvJ5siUJ6ExFkcJos1mJCY3CG0TU5Tq02hvIDXI\niY/G42+uQwvjQYwYcx+ioqVTm9V8FwBg0EPvYPLD3dC5VW7kWVAMWDea95/phCUfvYOnxwxR9mLl\nwOt7JzXuwMVqLN+wBamxMRiWDlH/kqB3SSlUkCBmPN9grua8Z0vLUbD7GOxON7q3aIRedw0NKNN1\nLU+g1cHXsGPopq89t1uRtylS5YkZu2TVRhiMJgwaMUbZOoOqTe11GAzHSZEbFdlzDNkKKVrV5WFK\nVkTKk0B4myFaDrsbcXFRyl5/PeIGeZLGdU2eIiE6SsjWn8nzJIT6JE5SIDUEUmIMisJJQrWL5MgB\nOyQ3uFkSog0kqt0UVhy/Um+voUfDOKRajaAoPzw+P/RaDcccLhbeE4OSMW0aBIiLmIJ3LYz2TOZd\n4akzWL9xA0aOHis6Vk2WHfMd27BxI6IMBnRunSu/GAU+EeZG8t5bb+LBPu0RZzXLzysDd3ILkDFW\nUD5/oJS9yy54w3a63Jgz5xNYo024t/et0Pklvk988rP1G8AhTK7FoJQESRnM1cJHGrBl3xFsOXIG\nN2WnYuio0dBq61YTOUBurGE1m2ShhrgJjP3qp/WwxkSj3z3SBEqoXpPiMJwYJDx7XsoPHakRJ0FC\nYMKGrHFC6/Ia46DXkfB4fXDVeG7UefoT4rqtMB5pmxUlpKiuxIlZi1C3+kjmYaPY7pHtv1afUNrS\n5bzNBZvLizY5iWFVyP1+P6od9tDfBy87OMTJSGpgCvYhM+nJMBN3XbCzuAo0DYzPXw2TnsTa01ex\n7MjlkHLEJkJK1C4lY5jXx1QFn/t/t4fek7q0yREDqSFg0GkxemIBchpn42JxoNyCGNhV8A8UlsEg\n8v1hf8cIAKRP4U1cIXGquFoOwn65ljipqb4tWFncGlgrQYDQEILVuNcv+wGvvfU+7r61Lcb16ihN\nnADA44S37CIW5w+Ex+UG2W0k6I6Dla8TALF7BagN82uz6oSgN0GXmIbH3lwPXWJa7bgIiBPdcTAM\nfR5Aj/F/x0sj+yI9IRZTX38HSxcvlrwu5BCoNr5b2BAuBSWmfomxowb1xtVKGzYu/yb8WKbKt1D1\nb951KFn1WwzsOVgVwClfgDiFVRuXUWVDrWNYhCxs3YQmVCldr9NG7Gm9gWuL/znl6VqDWZPH64NG\nQ4DUaiJaH/u1FaxeDYvFgpTm7SRVC6WKhlwWHXufWpWEfbzLWYMNBT/i4oUi9Bl0F+zmNNHjpEJi\ncmqP3H6xcgRKoURtksT5Q9i3+zeMGP8QjEbTNQnlsec8fuwI0tLTERMjrOYwyqqSzFPmOvxl0yaQ\nPiduubkjbzKV2Uisp/cR7WrQJp5A84ZpkrWCVFUIF1GeyqtsmDXnM7RpmIKBHZorvyExyk9MHMhu\nI6XDbzIqkZwC5e/5APQmIzxOFzSbvlC2PoH1ioUJfztZhNV7jqN7y2z0H3YP9z2IUB1SFJaTKFUg\nWciUhy9+WIWGaQ1w253DAYQrNhEpS2rBasmiqtSBSg8Vs83louBxe28oT39CXLfKEyDew4uBGsJe\nH+Se3xSYCBpS1TYgZs/z245fUXq5BCnN2wmqFoxyYSA1ihSNDIt4TzWhfRdsLpwqr1YcXqL8NCjK\ni5+++xpfz/0EbTt2Rud7H5MkTgCw+VwFlh+9HEZsejSMw10tGqBHwzjB4+T2AwH1idRqMGZiAVIs\nRllli71faH41xwMAMlsjb+gIzP3gHZw6flSwl57U56IE7DmtGTmixInd/9CpoFk18x0jjFGoqanh\n7IuoAzzr6f3Y4YNo3jBNtN+cYH85id50htKj0J7bDcOFvTAU74Wh9ChomsbX8xfgo48+x2MDb0Ve\nxxaKiRPdcTDI2+8PKE2OipAC5S27GEaSOGOFEFSWRk9axVWWWPv1wS4HeqNBXKGSA6OUTcsD5aNB\nt+kb2nVzkyy8NLIvNBoNJr3yFjauWApAXdNhzli5PoESczPbyRiZnn0sPHD3QJw+fwlbf/peULGJ\nSFkCAqE0BQhd77oYgParO59E3zqmL1+otRFqVTKX8/exaNyAetQbeZo+fTr69OmD5s2bo7CwUHCM\n3+/H1KlT0a9fPwwYMABLliyp83nFWpuoaRZsUtlYWGotTDjE4/WBpmnBBsRK5/n33Q2wf/9BdB88\nEgC3Oazd5UVKjAFNE2OQExeFnPhouL0+yea9UiEjUkPAYgrfl2ExIic+WvSmzidpNE1j4cez0bJd\nB3S4ZwLKjeHVuQFhAiKU4ZZiMWLsJGHSI7efGdM53QoQwPzJA1Bic8mqVAxZEpo/UjJ3CTHoOu5p\nHN63Gz999zW8PtaPpwQpFoPQfsZP1TQxJnQts3mCUKhbaRg7KioK1dUs8lSHDvCkx4aqi2cRrwmS\nR6GGtXoT68Zsrb2xyjW39XlD/3kpCq9Mn4HUeDP+MbQXLFEqiKkA2eGE32TGhoEV/hMiXwBk9ysF\ncXAtQNO4b+rqsPUQBIEerRrjpZF94XC6Men1GTh25hyXwIiRGD5ZAqQ/CzFyxdpOUX4szs8TbVTM\nx4Thg3DsTBF++3mpaHNd7pshfV16jXEgouPhNYo/fDHzCF7vKlRXMbJF6S3wGWPDH0RuNAbG2bNn\nMWrUKAwcOBCjRo1CUVFR2JjZs2fjzjvvxNChQ3HPPfdgy5YtYWN27NiBli1bYuHChfW2tnojT/36\n9cOiRYuQnp4uOmb58uU4f/481qxZg8WLF2PmzJm4ePFifS0BAIs0KfRDGYOhiwOFZaoVIiEwT+o1\nXgrVHq/sk70Yjp44gZ9WrUav4fdxtl+wuWB3e2E26mANkh2jXovH31wHg04rqRKxyRefYKXEGAI/\nZNPyZP05bGLFV0sIgkCHeybAHpMq+tqUqEVAgEyV2FxYmJ8nSHrk9jPnSbUaMWZiAUitBjuLBQrh\nIUCy+GQJAGd+AHUicxqNBtm9hqJRbhMc3rcntJ3/uTCkWIywiqlU7M/LoCNh1HEfCmgacAfJvVpC\nHx0djWq28lTHDvDrl32Nfl07hf4O+Gn2wFB6NKBKpLeFN7hWyueHOyFXcKwYqp0uTHntHdx7Wzt0\nadpQ1doAiJMdIVKjgBgBECVfjGoFQJic1dfa2WshCPRr1wz/ubsHDuz4Fe1jDqDk3Gm4E3Lhy+oo\nrEIJEFfJz0KI6AZDeMx2uOzCPioxaHV4ZMSd+O3QcRxZ87Wk8iOrjGq0LG8RKa1A1eV6ZxM4AXKn\nMxhCUQq1DyJ/dUyePBnjxo3DqlWrMGbMGEycODFszE033YTvvvsOS5cuxauvvornnnsOHk+tYldd\nXY23334bPXrUb5X2evuUOnTogAYNGkDKQlVQUICRIwMqSnx8PPr27YtVq1bV1xI4T9aUzy97k2CP\nb5ubCDdrbF3M3swcNK0ss08Iu3/bibseeDwsxEBqCJgNgRukl/Jj0bQ8uDw+zHm+D2wuL9wy3hyh\nkBFz0x2fvxo0DZQ4Aq01hMgWc+PODHps2MSKbwQXghK1iA2xcJ7YfmY+9nkoyo9F+Xm4JKI6MSSr\nc7o1jIyx568rmWPgS2kKOq0ZZxvzuZQ43JLhVyn1kP15ebw+GHXcBwijAo8TA/51qdPr4fZwQwgR\nh0kAnDx3AU0bZXA3Bm+ujCqh05MADdw3dXV4WEdCpSgtr8S06TPw5B3d0DBJRlGQgKjSJDZ26zfy\nYwV8UhzVqp6gdO1ajQYjM4CHW8Vi4Tv5+HrZCox6eSXIGKugAhUiS1dZ0QWJzyKMFAdDeBzSpdBr\nxRzvadAST4+9G+u378WJX9eIvAEKlFG/Dx4vEymghCuDsyB4vcsQHVkCFyRlTJQikgeRvyrKy8tx\n9OhR3HHHHQCAO++8E0eOHEFFBfde0K1bNxgMgdqAzZs3BwDOmDfeeAMPPfQQ4uIi/y0QQt2rLKrA\nxYsXkZZW+wORmpqKS5cuCY612Wyw2bg/ylqtFqmp4ooGO2zm8gSUH5oWLz3AfxJnbihGUgtSq4nY\n7M1ALiwiZXi/ddA9nL8ZMzGf0JRU1IRUIqXgh/SYORdODVekLthcIIN1nti+qkX5edh/5CgWTh0I\nm8uLvZfs/NOIQgnBYEMonCeUJcc3nZfa3aHz7CyuEjwXm2QxNabAG8v+N7+8AR9y+9k4eNkRKmsA\n1H4uYuogM0Zq/wWbC4YaD3Lio3GgsAyfvdgTLk+w0XXwQWHxtDy4gxX6ha5P9nXp9vkC3xOXG0aD\nQPHSCH7oKyorEWsWMcDy1Qog9G8lQfWjm9dh0aY9+M/wPjDpI2tZxIHC8FnEtaDYKlF5CYhggcy6\nhO3YcwNQNJ+ZpPHs0N44kngTjh37BgcOZKF9PClIbNSYvENgkeLF+QPV153iHU+QevzjgRF4/eNF\nGGsyIqNdN+54nlJEilynOlcFaI8WOhnixJ6XgaxBnUXgFk/LA+0VTq4gPTbAqwENiK4TAC5dugQf\nr7OGxWKBxaLCc/gngNLXcenSJTRo0CAkIGg0GiQnJ6OkpESUCP3www/IzMxEgwaBWnubNm2C3W5H\n//79sWHDhnp9Hb8reVKDefPmYebMmZxt6enpWL9+PRISYnDFViN4nIvyhX7wgdobAb+WDUEABm34\nkzijRvn8NOdGozYnUS4TkK168c/Br+PEz85iExoGTD0mu1tZgUo++HMyZIwhbOxK1nNf6o13ZrwH\n2hSD03QsPArvn2xys/zoZdVr5M/Bb8fCJkG9GsUj2WxAmcMtmV3novxwBgl0tZuql3IFajLz+AQK\nCP8s+JDb76b8obIRr776Cp74+z+h0Wg4DxYGrfD1yVyXTGVxZkxNTTWio+qnUN/q775CXo8uovsN\npUcBrQ5a5uaq1YFKyIUvq4PkDfvE1g34Zss+/Gd4H5B1rGmkCiz1aHH+QFAqyQ+xewW8nYZAl5AC\nT88HoDca1JGwmDjR2lOqSJ3HiRap8ZgyeTIWfPklfql24MExo2GuZKlMKkkQp70LixRr1RAnIOBh\nC16/XrcXWp8XGo0Gzz80CtNmz8dTDVsinndDJT020F6NJCEBIKs4CUIJMVJI4JixQkhKqk38GDt2\nLIqLizn7n3rqKTz99NPq168Aia2y4Kuqvyw/rTUewLV7HTt37sQHH3yAuXPnAgDsdjveeeed0N/1\njd81uJqWlsbxOF26dElUSRo/fjzWrVvH+Y8xe129Kh0aYtQm5kbA1LJhvB9SvihGvYrU7A0oq0FF\n04DH62OZy8PHMMZhoTANv62HxajDwVNlsBh1yGR5YSJRpDIsRjRLikHjuChkWIycNZwuPIEXJk5G\nu579kdF1kGLixA/X3ZJhVeR7kprDaqzl/kzIbFF+HkrtrlDrlWSzAb0aiZ/jWtaYkgNzroOXHbh4\n/hyustKC5UoWyO1nwoA339oN27YGDJSMH8/t8wlen4xS6vJQWJSfB1KrCY1xX7mAxIT4OrxaAIQG\nNE3j7MUSNEpPkR7Lu7mKGY8ZXCmvxLx1v+Gfd/f6fYkTIO0xUpI1pzdBl5CC0ZNWQW8yhtd6koC/\n5wMgu98Lf88HhOeVM7LzQOxeAXr3Sox/4AGsLbTijXdn4Pj2X2sHhJRBBSZvjunfAsPVQlmvmii0\nOuiC31Odngx99jqSxL8njMLMd94KUzMAXLsQmEIPlGhomx3ukwj9Xblih6M6YKNYuHBh2D1x/Pjx\ndX4pvzeUvo7U1FRcvnw5ZAXy+/0oLS1FSkr4b8fevXvxwgsvYPbs2WjYMOBxPHHiBMrKyjBixAj0\n7t0bq1atwgcffIDZs2fXy+uo9zpPvXv3xscff4zc3PBKxD/88ANWrlyJTz75BBUVFRg2bBgWLFiA\njIwMgZnEIdfbjq36EAAMLIXH5nLDEkwJ/nLKANHQHJtMRVJxXInyZDUZQiX4mT56jOrEVpsAyNZa\nyrQGxo+eWFvNWml1cDbYzWYXTB0IDUHgRJkDyVE6rPjua9hrXIjrMggarfrMREY1KrXX9pVT246F\nmcPpoWDSkxwFiq1KaQgCKRYDDhSWoXVOouQ55NquXAvwz+l1u7Dj6w8x7tGnYbZY6+08NE1jyUcz\n8Ny/X+Bs54fm+EoUo8wy21Z9vwitW7ZATnajiNbBhDjWr/oJ+qrz6HmzRI88AfDrCbH/ps7tRf70\nGXhhWG9EG4VbrFxTsPrRsUNloj3tBMAoRB6XW7nyFBMHsvu9od82asvXYQqU2nAifx01JUX49PWX\nkZ5gxYhxY0EQRHhtJ4laXF4PBZ2ejLyvnsCcQnMdP3Mea7ftxqPP/rNO51ANtXXOwA33AZAM/dl8\nMbDGmv6Q3nalsyfCV1Veb/NprfFIfmKaqmPuv/9+DB8+HHfddReWLVuG77//HvPmcWtFHThwAM8+\n+yzee+89tG3bVnSuF198Ea1bt8bYseLdF9Sg3h7RXnnlFfTs2ROlpaV44IEHMHhwoN7JI488gsOH\nDwMAhgwZgoyMDPTv3x+jRo3Ck08+qZo4iYH99Mx+qnb5fJxwhd9f64uifH7RbDjG7K2m5IEaME/4\nH77QBxs2bsLWX34JESdG6Rk7OaA2lTjcYSZvPs5XuWBz1nphAOXVwdmg/DTsTi8WT8uDP+ivofw0\niiqrkZTbCgldB6siTmwlhzFfbzxbrtr3xJ5j9ckriNKTHNM5X5XafqESJTYXWuckotQufQ45U7rc\n61ILIdO8zmBEh2F/w4KPPgBF1aEHGQ8EQSAxKRllV7gtbxgVCgiQeFKrCWZu1pYwYNdRq66ugUXM\npyS7iNoQx449+9CjS0f5Y3jgGIxZoSONKQZvvDsHz955W/0TJwVKDae+E6snHXn7/SDiUjFmsjLV\nhzF4azZ9oTzjzlEBj9MVUK+dLsHQnRrTO1up0hsNoLZ+A/3+AjwxqBuSrTHIf/0dVNV4OGqSu0FL\nTt0nAGGmf23xgToTJ0A6y7JZdiYapadgzfeL63weVVBCnHgqE9vILmdqp2kabjdVjwu+vjBlyhQs\nWLAAAwcOxKJFi5Cfnw+Ayyvy8/PhdrsxefJkDB06FHfffTdOnjx5zdd2XVcYZ8B+igYgaPbmq0dK\ne9yxe4Ax6pAc1BxXfKEIS7/7DkP/9iSnwnSGxYgonRY6UoMqpzrlCADHp6RGeWKOsbu9uGSv9dXI\nZdEB4UZuOUUn0l5yYhXDhc7HeJ/UqEpy64pkTj7E3ptEbznWLP8O9z3+d06mZV2qj5ddLsHRHZsw\naiy37AX/OgUgqpR+98WHGD5kMCyWyHrQUXoLqux2fLt4IZ68Q9zvpBSMCvHe9NfQJ9GH5o0bhqs/\nSiFwjCLFRqiaN1C7bVoeQNPyqo+QcqUGEp4ntZB63ZXVTsxcuQV33Pcobu7VD1SNHWSUGWMmr8L8\nSQEVn+l7F1KJauwwXD5SL2tTgvcXfI+B3Tuj8c09xQdFoBZFCqnq4UqUpyoqGhoN8YdUGP8zKE9/\nZlz3BSX4SpNRT+L+qasBIJRRJASlxQHZqpVSmil2HN/75HK5MO/zz3HHfQ+H1e4pcbhBBnsnySlH\n7H3sGkFCZQmkwPY2mQ21fhIlxIlfu0ltSQIlc7LnfebtDaBpcOo2CZUtYMKDStcgV4OqV6M4pFjE\n51T6OsXUrpa5ObjnjoH4deV3oW11rT6e2CAFmVnhtY7416lUTTKfzweiDn34SI8N6776BIM7ZEc8\nBxuG0qNY997LaOwrRbMhE0Defn/A/3P7/aA7DVE8j2BlcKVeISGvE3vblWJZ1Yc5f2jtKvvmAag3\n4iRYCJT12mOjTXhpRF8cXr0E8yc9BX3JYVDVNiyaOjDwWzWpIORJM5QeDZErJZXL6wtPjh6KBct/\nhs0unP0bUVX8SCFSLoHtg6pLuY8b+GNx3ZMn/g2AMbuySUtdQm/s0IWauk/81jFCa/h49kxMeORR\nmAyGsBAb5ac5YTgx1YF9YxUyl6tRK9iNbC9cLgPlpyWJk1BNJcbIrbSApRhJESNfTHbcnBf6wOkJ\nz47jlxhQEx5UUtE82WzEgcIyLJ6Wh1K7O0xpY1cnl4NYRfV3V1zFgL69Aw1/FbbdkUO324QLxLGv\nU6mHA1Krhd9Xt6f1C5cuIys1WXyAisbAVY5qbDt0Anm3dgg11NWbjEHCkwr6ZgUESowkKSx6CQiH\nxTjbpJQkdpjMZMTzs7Zw1xFpixaRc0lBKvzIJnQEQWBcr45onhKHqa+/A+rcXmiL9oByVIUVwySj\nzLUm//p8LRLQajV47oGReP+tN8ObINehKj5zvOJjguMonx8Lpg4M9Ftkg9dw+AauP/wlwnYANwzH\n/7fa0JtQSK8uTYiF1rB540ZoSS0atg2EMJQ2i2X/TWoINE2MCdVditQkzkZVZQUWfzILw+9/CJcg\nLhXzw05iRm6hEJiR1OCuFg1kDeNCoS2lxzIoOVuIo9s2gKIR+lB1BiPSc5qhSfvw8JFcqLHW9O7C\nxrPcUgl3tWiAcZMLMC8YwrgUQViPff5ovTZUHsKg00b8mQJAurlufqDlCz/FwL69kZQo3G5HDjRN\n493pr+H5CaPCd6psEAsA0954Fw/174L4mKhak7PbDb0xQG7b5iQIN+/lr0sqPMciU3JwuNzYt2UX\nLtqqUepwotpT61PRaohAgVJSGyiPQmqh12pg0pFIH/EwGuQ0R0JiMgwGXagpcMR1o9S+xuDrlAw/\nijRCvmqrxgcrt2BCvy7I7dorzDiu2DTOHKe29pMEjp46hw079+KRZ7gGctXNg4MhPkpvAUHqQGo1\nsseqCcvJ4UbY7s+Lvwx5koJS4sPPMGL7pSLxPkmtgaZpXHTw0rFllCI+wSI1BFLNBpgNXLIUqT+m\npPgCflg0D52GT4AxSvzLKkZgkqP16NU4QRGxUZrhJuZ5amA2YPeRk1ixdQ/KLxcjyhKLjr0Hccbt\nEpiXpmn4PG7Qfh90pmgAQCeW+lVceAzHdmwCQerRqOVNyGreGqSOSzzEPFHMayIIYMxE9ZmE7PkB\nhN7jRfl5OFVeLVs9Xg6RECjmQWLNdwvQpVNHZGaIt1+SwslTp3Fsx2bc3e82znbmBosLjxCaAAAg\nAElEQVRge4rF+QOhLdojeRP9ackS+Gk/+rVjVWkP+oXom4dAF5+ijnSIeI2kSIfD5cba1VtwtLQC\nfhowG3TIjjMj3RqF5GgTovU6aKPN8NfYQfn9gfpzwf88Pj88Pj9qvBSqXF5UEAbYGt+E7zaewj09\ncxB/7gC6PzMRL887ia+m5SkigaJ+KSFiJOTvSkoHRflBV1wKvVYlBM5D+fDeis3o07YJbs27U/D8\nvvS2ofPzP9trkZXHYOWm7dBoNOg3bDR3h0LPUy0J8kBn0MPnpzFucsDLRjuuCs9BaEDEJNTWf3Jc\nDWxXqi7x1naDPP158T9BngDl1b4pnx/3T12NRflcklRX5Ylf8oBfCFMOfJXJ7vLCFDSU25xenOcp\nEkoIFHtM4bEj2PzzT2g/bAJInXj4hCEOSpUnKSitxM2Go6oCW5Ythp4kYUlOQ4UxGTGJaSEiVF+g\nPC4kOIpw7thB+LxeNOvUFY1ayqfXM42I1ZQ+kCJjqRaj6lITYp+7EvLEvkbZ1/yaZV+jaW4Omubm\nKFoHHws/mY3+3TohNSWl9ubJurF+OZlrOBZDeZUNs2Z/iufv6S1+svqo0M0jHa61c7F91XrsvngV\nDnfgu3dzRhJaJsdBGxZKJRA94kmYsnLhLCpE9ZJZAKS/i9EjnoIpKxc1507ixGdv4lhmB1yoqoan\n8iqalJ/CwEE9RSumy5EcOXVNklwpeC9pmsZna3ciJyUBA+8Zzt0ppSpqdfBldQiZ6x+bvg4fPt9b\nljyrwfsLvkfebZ2R3UnCQM6ATVx4JMjrdkekPKlRm4SOu0Ge/rz4nyFPUlCadaS23pOYkgWoJ08A\nNxPObNDBTweehBZNy8OJKw5Opp5c6I495tf9h7F903o0HzgqrJceG3zCxNz0+UrU6pNXUOW6Num1\nv50tl1zjNQNNo3tuomKyp5QYKslIbJIQBZqmZV93hsUIi0kHm0h2phx54td+Yn8nVi//FmkNktG6\npYD5V8GT/DtvvIpnX5xaW20aCFccrhZK3jRpmsakV9/Cs4Nvg9kUmXleKeiOg+Ego7F82VJcLDwO\n/7FdaJ4ci45piYgxSHuzNFFmxD8+DaMnr8biqQNQPmci/DVS7YsIaKLMAGjWOAKaqBhQ1TYcKa3E\n9qLLcHl9yEmw4I5BPWGJCr5+PvnZ+o2wgVyCBNVXiHDRpj1ItERj8L33AuDVZao8K3j+a6k8AQDl\n82HqrHn414svSVbIp/RW6Ax60aw40mOr9TspLU8g0ARY9NggWWPq/jHK1g3y9OfFdW8Yrwv4VcXl\nso7UECfGIM6vnwNERpyA2qrR56tcsLm88PsDVdBtTi/HByVnMOaPyW6cgxZ5oyVvzkJmaoYc8I3Z\ndSVONfYq/PrjEtTYA5l0u85VhP6rT+JEADAb5DsUEQAevqUhBrdogPYp3FR9r8cteIwS4qQkI9FF\n+bH/kg3z57wnORepIWA16TBmYgGsJuHPff/evSgtFW6Lw89aBbjfCT1JwuMJJzZKspdomgYIDas+\nkFW4DpAYcQoayed//gUGdWxxzYnTVZcPsxZ8jbseeBl9+/TBo6kaPNqlBXpmp8oSJwDw19jhLCrE\n4qkD4Cwq5BEnhijV/h094knEP54P0x3jEbjaAIZIaQgCrRvE4aGbm+PJW1uiaaIVc5esxisfLcEX\nX/8EW2VFyNzucblBdhspnLGn0vQeCcb07ICqGhe+XbAwVOtpzORVgNEMX/pNgll3TO0mY/HuyKuP\nS4DUavH0uGGY+fZ/RccESJJeMisOQID4sJQpSfBIkpIGwR5voO6fx0vdMJFfB/ifJU/87DelWUdK\nwL4RkVpNwMx98hRsNnvExIkBQ5Iu2Fw4XVGD41e4pQgoPw27WzpLj99cdl+JfEadXOZaJIUm2aBp\nGmcO78WqebOwc9VSUOltcbTcj+MlyhsOqwEB4KEuWXi5X1M83CULBGrJFJ9UxRhI5CbFYNykAuQm\nxeB4iR27zlWA8nqx4Zu52PjtfDiq1L9u/nvKgN12Bgg0xGzWui12/fqL5HxeKpDZ4xUhbg7osHfX\nLsF9QuU12N8JvV4HL8UjxfzsJY02tJ2N8xeK0TA1idXwt4rb/Ffixu5ObgFfVgccvuqDvcaNDjn1\nU1RXCNUuD96btwxfLF6OUX174pcf3ke8zymjGvERIEfVS2ahfM5EVC+ZydkXIErTED3iKTDqkikr\nF6Mnr4YpKxcaCa8hQRDISbBgfIcmeKprK7RPTcTn36zCK4/cj62v/gN6o0FVKxYA9Z4FN6LbTdBo\nCMz7+GPhMgZ6U3hWJUOa6ylUx0dyfCx63NwWBd98WbuRuUaD1zCTQet1e2Qz4VSXO9BopbP8gll8\nel3gnqHXkeozAW/gd8f/TNhOaQZeJK1YhMAOgVRVV+Pd/76JMU/+C1ry2vZiDoX2XOE+qLBxJh2n\nyCQfYtluan1KcjhzeC+O7tyCRi1vgiOpJTRabYjc5CbFoPCKA5/uKJJxjaiD2UDi5X5NMW5SARbk\n5+GVNScwql0acpNiUO3yItqo45z3YdZaPtlRxJnLZa8EdewXuJ016DxgCOKShfs1ioHtk/JSgcy6\najeFFcdrq4LTNI3ti2bhwaf+AZ1eOPyWExcFo14Ll8eHUxXhjbP9fj9WzP8QTzz9d9G1iF3/e3fv\nAumqRPdbOnO2M6ENj5eCXkeG/s8OfyxbOBftW+SicWYaN6NKKLuKt9+X1QH3vrwSzTTb8eJtjaHz\n1+3hIwx6E3yuaixYshpFlQ6MaJONFHMUGGKjljhJeZ3EwnmM3ylwzEzx6YPn4K+L8vmxtrAYJ2Ib\nYvCI0WjXOCOiViyhsF09eMY2HizEuSsVeOTJJ0KeJ6+Hgs6gA0X5AZe93lUmObzxySI89OQziEnK\n5ITjVHmU+Ibw6nLJxsLsLD3K5wdNeTnnkMvMuxG2+/Pif4Le8lUmsSKWcvWgIq3z9NnHH+HBhx6+\n5sSJU+RSoiaQ3+sJjJsoHi6SqrNU3yiLbojk3uNQk9Im1PqFr/bEKAivqYHdTaHwigML8vNQeCWg\nvOUmxeDJN9fBYtKFnffTHUV4Zc2JMOIEAEZzLGJuHgzrLUOwd8MqFJ86rno9KRYjnnhzHQw6LUZP\nLEC0geQoUARBYMjI0Sj44RvB40kNETrWoNMKfvZZsVEBD55EvTOxBwejXgefLjrsaZv02EBXl0Ov\nCzRtZZ6e2U/Ypy9cRHZGkFCyyZJAP7RQq4/gfqrahmFt7Bg+sG+9Eye642DsMjbBK2uPo1liLJ7u\n2ipInACu/0gZ5FQksXCesEolhFrlKmbUs0BQL9WbrRjYLBNPJftxasH7mDxhHLav+1V6Kl6NKaYZ\nMd1pSOTFOllz92qTi+wGCfhk1uxAaO7SEej0gd6bhIbgNnj+nfDoyMH47MNZYSqQqkKVrIbAHi8F\nIjpeXIEKqlpEMJOU1GpAeh1h+5m1kF7HjYKZ1xH+8sqTEpWJIUVS5Qgizbb7ZdNGuFxOtOzaR/Ex\nYlCSQSdnFl/+9UJkZjdG7x49ZDPClJQTqKsSJVROgIGU2qMWBAKEzM7qE8Vsc7gpxBhIQeUpkvN2\nEin8KQXmvWaUJ6/PjysOT1jrmTf/+xZ6Db0XZmts2BxSnz2TrXnbkCew8fv3Ue1VHp4mCODkoQOY\n+OFmFHz2T8E0bb4CxX56fueNV4XrO7HBzrxipbR7vBTeeHc2XhgqXOQzUrgJHWbvK8PPB934ed5L\nqPhwkmqyJAR5FSkSRSsATZQFlkfzQQPQkRo4i04DfipM6aL8fhQcP4/T5Xbc1aIh2vcUbocTpjyV\nl0AXnyJZ1oADBW1t1h84iYvmRnjgsaf+cOUJAJau24L43HbodnvfiGsvAQBlsEKn1wdqiuUmipYu\nEKwPxTKOi6pewTE3lKc/L65r8qQ0xCZFfPh98YTGydV5EltHZUUFvvjsE4x4VDxMohRq+tSJkawf\nv12M1PRM6Bu3AxB5bzkGSms1MbBXXIWr2oGkjIaSpImBEOGJBFIhQP6+r/ZdDJGpup5XLYli1L0h\nLRpgDKtWFlBb8+mDv3dB4RUbzHEJgnNIEewMixFb1q9GekZDNGrSVNGamGv70N7dAO1Dl5s7i99w\nmJsC6+Zgs9nxzbxP8diou2TPxcnOCt5Yv5r3JZplJKNlZgNF61WCq/ZqvDVvOR59+RXkdu6mMFym\nFJGTIzloosyIf+JV+IJZtounDgAA0aw+j8+H5UeLcMlWg7tbZaN179vCyRCvr57SzDvBcSLFNjf5\n0zHtw1XYuOS/0BYfAHxU5P4mtYU0eeNpmsaUmV/g+ZcnwyBS+kEWvNAd5fND66wQzK6jdDGhWlGk\np0qYLPGy8Nhjqmt8N8jTnxTXbdhOTcsVfqsUBvwMI7dPeJxUjzupdRzYvw8DRz8Ytl1tiw01LTrE\nbp4bV/2IuPiEEHGSg1yYTm3vugO/rMW2lUtQWE0qIk5AwC1SVwIDBAhYE5EQID88WJ/nFXqdUu+T\ni/LDRflxiWfMZxvLPWSUKHECIKlMXrC5kNH6ZsQmJilaP/vadnopaNzVikMbDPZvXo2OrZQRNSbz\niiFOfr8fx4pL65U47du8A+/OX4Gnbm2J2F++VRgu+73Az8Tjwl9jh/PcSdB+Govz8+AsKpTI6gP0\nWi2Gt87Gw51b4FdzNj4+akdNq37cSRkypcbjJNH7T6jXX4+bWuC15+7Ge9NfA+2uiZg4hYV1IxhP\nEAQmDL8DX8x5P6I1AOCE7ryUP+Bn0nHJDWMqrw3L6cWN42zSxQvlaTTX7S36L4/r8pMhCIJDepR4\nkYSUISFSJKZkCREwPvnir6NHr9thtlg529Q2ec2wGJETHw2316eqzx0be3dug7OmBpZWXWvXJtFb\njtnXq1G86LqYHnOLp+UJ9phjUFlagh8/ew/G6BjE33YvdEbxWit1hVDpAQLAqHZp8Pn8WDQt4G9i\nEyO+96k+SBMbu85VwO/zYdvKJeieaZXs58dgZ3FVWINjdjYju9+gGiKeYTGibVYDpCTJt1jhX9sm\nygGNxLnEMpD2Hz+Fts1UFNZk3VhXf/8derWOrCinENYWbMKqExfw9+6tEa3XXwOFKDybrn6PJeBc\nOQ+2jyahfPZLqF4yU5FfKspixRP/fhFrC+Pw/tyFWPnzNuGBdWiIzPTBAyDY66+zuxDdrC68/e5M\nRBTsCJY+CPXKk/NLSYzPSk1GlNGAkzs2qF9HEKTHBrqmArpg83YOGWIRIMrnDxXZhN9XS7rcbuFy\nBDxPlcVqhMH4+3rDbkAZrkvyRNO0qBKkFmKqlPB5w/+WWge/LIGSGkxi4w06LU6VV4uG7KTm1mq1\nyOhW27pESjXi7jOgVyPxxr0mfcAkbNKTgorKztVLsXfTKiT2GAlbQjOBWSIHnygJlR4AapWl8fmr\n4aeBr/ZdDJtLygxeH9hzwYYmrdthwUfvY8zLKyWVOoa4dk63cv7u0TAujKCqIeLs60PJAwf/2vb7\nfNCKPQVLNFz1eKmIwyPbjp3DLc0aRnQsHwuXrMLZSgce69ICpEYrQFTEVB9mu7QqBMgbxut2LLse\n1P3w1zAEWt7YzhjVv397NP55/yj4XDWY+uE3KHfwMjL5pEgCnNpQfNIlBI8T7bLT0b1lY7wzY5Z6\nAhVMHuA0H67D+PuG9MeiH9fC51PXLYIBpbeAiIqDx0uFkyEWAaIpL8cErsSczk7CGD2xAIZ6Tpa5\ngfrBdUmeAOWkJ1JVqr7XAYTXV5Izf/PHS/U2o/w03F4fFk/Lg9vr48xNZLTkrbk2DFRq59ZsclF+\nlNoDX/wDhWVINosXbyyxuTD7+T6CdZ8AoH2vgTB3HgJSX79FDYWIklh2HqMsfTk1oCw53FSYOlVf\nYTqpoptnkYjb+t+JttH7UFxRLfh+8Umt1UiKktyTV2tUEXH2taT0gYN9bftpv3iBUnYYg3UTcblc\nkROngh/RPiddXVFUAaWE1hkx44ulMBt0GN46G4AQUTGLqD61apDliddkFSV1xTH5xzrgKj4jGoKr\nCzED2Bl9s9CrVVNM6NQMHyz4EQU/ctUXhhQBkM+6Y4X8wsJ1ImjfOB1dmzfCjPdmqyZQ/LAuAEkF\nSnB8EKRWi1GDeuPruR+rWgMAzsOCXkeCri4PI0McksRXmJQUwGSpVO56VsNvoH5w3ZInQJ70qPFF\n1fc6xIphljjcOFHmUNyrjKkqrsQkbtBp8dj0dZxUdXaIh43N5ypQancj2WwMCyNtPFuOEpsLrXMS\nRYkRM4dYYcxd5yqwv6SO/cVEwBCl+ybXEiWp8BujLH26o0hQnaoL2IU15eZeWqxBXPvbMeXNtwXn\nEqrULlaY1EX5cfDYCcyb2FcREQdqryUX5VNcdoO5tmk/Lem/EHqiPrljI1rlNlJ2Ih5W7z2OAe2V\nq5VM2Ih9s/e1vwPv7ipBzweeRI/sWkWET3IAWpCYsAmLPjoaj/13gwBx4ZIipcUxuQjsN6Znw3nh\nbDBrjgtpYqboHYK/xhFaR4Nx/8Bz/brC5vbirc9/gJenwCgK37FfgYoq5R1yMtC5aRbe/2BORAoU\nA0UeKAmFqlVuI9ira3B+31Z1a+A/LIjVeeJ5mdSC9NhQVemE23VtiofeQN1wXZMnKcj5ka4Vzhed\nQ7HdI6gEMGGWlBiDqjmV3BgZZWHO831CN1Mx4gQEVI5ks0HU8L3xbEWY70YIQsRKqSFcClIqDkOU\n5k0aAA0R8DQRAD7bUYR3N50KC78xylJ91Y4SIkyP3tJQdm4awCkqFo1atMX5E0cE5+YTUimCWnjp\nCr74dqlkKJcPyk+DIDSqHyoCLVbkBnGvhYMnz6B186BZXEVNn0Mb1yC7Qbx4mJAPAa+OnzRgxtwF\n2HI+ATf3vyNMqWGTHDFiwt7uqa7Gh/++nUdchEhReBhNTjXi7M9oJKoqKa8HJQzOeRo2Qfzj+Rj2\n0hvol5uOSXOW4OS+Y4GBKpQkDlSYzjvlZqJ943TMmR2B8gMo80ApuOYeHnEnPv/uJ+HwnQThYUJr\nShIoJCuSy5Cq6zAZ/n8GfznyJNavjl3P6Vrh9KlCbP3lF0EvipqMOaXgz8EoCyculskeK9RuhU+Q\nGGIkZS4HgNMH9+D0ob2h/nNqodS/pAGQYg4Qz6/2XYRWq8GYibVkZUKXLDzXM0dU+akPcziHMN1a\nS5iyE6Nx5mq1ormr4psis2lL0f18Qiqm/HkSG+PooYOC+8T8UIz3qeewf6h6qNBoNKDFSLzIDeCS\nS4ukDn3hSu+oPEtKq8PSHYdw9y1tlC0MELzZv/vJV+jfqR2Wvz9BRKnhkhwxYsJst83+v7D9SkNp\nwuSsVrFSriqpL9wptg7K5w+tOzs9Df/39gdY6k/CTiqQ2RimJNVzGxcA6NK0IZKsMVi/7Af1B8t4\nmpRm5ul1JEYM6IUfFnzO2S7XgoXSW6QLZDKQ8AMqafPyhzRBvwFF+EuRJ6l+db9HCO+Hb5eg+x13\nh5EkNRlzSiF2c1zz0wocObhXUnViwFY1xAiSXEmCnauX4cqFs7gaU2vsVdpwlxmrxL+kAfBy3yb4\nV68cTOrbBNVuCicFKoSLKT/MmupqDmevLTshGmfKagnTR9vOKZ67PtQ5AGjasjWOHz7A2SaVPMB4\n44Z0S4dHRaFMkiTDe9tB4gag0UKjNwQqVxt0irKk3MktcEGXirjmnWDQqVMFiYNrQzf7r75djaYJ\nVmTuXaVCqREjJuKERU0ojUvOwhUrNVXG5YzrUmOY83gvcMOWcU1aYX9NexSWVWHRso2BwUElSSgk\nqgoSxGvwzS2x+fBpVNrlf6/4EPU0qczMa9usMc5dvAybLfj5SRAeRfvZEPEDKpmD0ltgjTXdyLb7\nk+IvQ57EwnSM4nStQ3i/7dyBtu3agdDqOCZvAIoz5pRC7OZ4+sQxlJddgaFxe8VzMYoT36jM3i/k\nu/H7/Vi7+FPEJaeAbHk7LMEvuJhqJEaohIiSkEKUbDbAYgq0d7CYdEg2GzhESEpVYq/poS5ZcNTB\ngMk/z6K9xaE1iBnPxV57fRAoS6tbsW3jOs62lBgDCAKYP3lAGFFnvHErfz0DHalR/F1w6czw+Sju\nj7zIDYDSW4CoOFCUDx8+3xtet1c+Syp4wxv5+GsY/eDDqpSO0M29TV9sXfsrrjrduC07BeLEh08u\n5AiJuGdJOempXYuwYkWHKVJq1qF8TOA8QmHLr/IHYkSvrrBo/Hjr8x/g8/sDIdGk9EBINCldtQIl\nR7wIgsDjeV3x3gcfqZo3BKHrSW1mHoAJ9wzCws8+DC6aTXiEvauyJQdYEMywEyNVDFjfrRvZdn9O\n/GU+FamyAXIlBeoKv9+P9Wt/xtinngcQCJ+RDnfopqU0Y04phLL2qh12rFnxA24Z86Tq+dgEyemh\nMKBJEqdq+OZzFZxK416PG6vnz0GnvneimEjgVOheceRyiAwtyM8LtT8Rq/DtcFOC4a5PdxSFiBQB\nYHDLQDjhyykDYHN6UWIPNNLkG8OFKoPHGEig6hJ6TngHw25vgpOnrsILApbURohv2BSmWOm6R/w2\nLsx5RrVLw0v9mqLwigNf77sImwhxkmpwvOtcRVglcjUtb7RaEtExZlRVVsAaGxci1qMnFmDRtDyU\n8BoEM9fOg3d3QdH5YlgVFMwkiEDYjtJGgYhJqK2OzLsBkMHq4jqDAcP+uQjD2qVBW3wAWo8T0Oqg\nlbqJ+bywl13G3T0aIcpbrdw/w/I7vfdUe6w9W4pnOjeRejW85r2zET3iCdFmvgCX7CyeOgBOTn2o\nABmhaRoX7TU4vPsUKr1eVHl9cFA+0ALtrA0aDbJcsXi4Z3uc2L4ZVocNWg0Rtjbnyvnw19jC1vHY\nfzcE/FecdRChsKH4WsEZyw9bMmO7N0pBqjkKUz78Bi8+PhZRlB8Lpg4MtFVRA9Znszh/ICiRhsOx\n0Sbc1qoxvl2wEMPHjVV3DhEYSo/KX3MsJMZZ4aUo2BzVsMREg/TY4EWgtQphqL3mmerfcLsDhEhJ\n5hwgSLBIjw20VyM8B+u75fZQ0F3jpKcbUA/tlClTpvzRi1ALp9MDWitshnVTPsGQmNS+uoAggJ8L\nfkL7Dh0RFZ8c2s4+jc1NocLpRaWr7imnpIaAn+bOSdM05s2egbaD74PBGJk34VyVCxdsLjRPNmPs\npAKMH9gCp8prQu8X+32jHFVIb9oKZ71mWAwkhrZJxX2TC/C3gS3QLTsB1S4vRvVrhsIrDmw7VwFz\ncMy4SQV46M5W2H6uAh6fP0QschJjcKasOizc5fEFflSY48dOKsDw3k0xfcOp0L7qqyW4dHA7ivdt\nQenJA6gsu4zY9MahOfadvoqismr0bNUILz56D4zpLfDTFQu81ixU2Gpg0XoQFSdOIJg1Dm2Tiluz\nYtG3WTIaxZlwqMQeek2PDG6F2xonIDvOhL3FVZzjxV47G6lWIzZ/vwANW7RFz0bx6JgRiwSTDueq\nlCmUcekNkRxjgk6vh58GDFoNxvRrBptL+JqzuSnQ7hpcuXoVGZmZknMbSS1ijHpcLC4GCBr/+fQI\nRvZrDnhdAGhofG7A64LGx6yVBgUSqcZypCbGITMqlK4n+zq++2wObsu0IKHqrKLXDQDwUfDFpmFo\nv1Z4fdLLeLxJLHRa8RuNJsoMS9/hGD15NUYN7QzPyX0w97wr9Ldr/1bQXi80UWbQ3oDqQHs9IDJy\nMWpoZziLCuHZvwU1Hh9+3nkKK/ecwNZj57H9+AWUXqqARUciMykRzQ1adIgz46ZYM9paYzj/5USb\nEF10ClX7dmPXjyuw7sg5bD12HtvOXkFlgyyYomMQm56NmC69QWTkwnvkt+A6vCDb98SQ25vBU1MD\n168FwVcVIF2WvsOBBlmgqso5a+Widix7buZ1MspXnI5AsyQrZizdgA5dboE5IQm+8ksgLhxW/tlo\ndfCZkzDyjvaBNi4Sx2YlxWHVnmNI07gRl1E/tb0UlQRgIbVtd/xQsAY3dboFGr8X2igzaABjJ60K\nXPOUG1qTGaMnFgT+9jjBJ9oRLFJ0j8bnhgcGGAyk8uSJekT1bxtAu+svY1pjNCH65t71Nt8fjeu6\nt93vAan+eUxfPHu1E0WOa1+LQ6y/3cmjh+HzuuFtoKwNhhSk+tUZSQ06p1vRwGJE4RUHPttRhAld\nstA0OQYU5QdJBgzcC/Lz8O6mUyF1CBBu8ms2kHi5X9OQSvXKmhOiRuuHWec5XV6D2esP48zG70HG\nJqPMmAldbIqkuZIAEBulQ0WN/JNou8YJuFJ4EKXH9kKvJfDggFvx40ESn748CKMn1q51VLs0NEmK\ngc/nx/1TV4u+BiUNjlM8JTh7cBfen/w8xrL62skpUMznZeddE3JNpKsqyrF/888Yc9/9otc3u6fj\n3++Mh9vpQo9ePRU1VJ3/8WwM7X0r4q1BL5RMTzJ3cgu8PXMO/jHhPkXp7nxM/2IF7mySgnRLtOxY\nfvNe7t+zgn/nBMoGfP0eAjc4ArQxCit+3IKjDica9BmI3kOGIkurQel/J4f8AekvvgpLi1awHT2M\n4tdfCm0nLVZQVZWS69JarDC88CqemjoXPVpG4effLuLdl/+GtF++Bu10BHrbPT4trJdd+PZJEAtZ\nis0RAF+VmwWHx4NZ247iHxOGI8mkPFDB6X13cG1go4ya6PJ4Mf279Xhl4r9//7YkwcbU3e96Eivm\nv4pYnR+ULiasqa9oI18x8PrWqUKwh94zb2/EnBfq3lheLVzr5oN21l8FfsJkhrHP/fU23x+Nvwx5\nUtokWAr8OaQaCrNvLIvy83CizFHvqhYbpIZA08QYjJkUfr4MixFmo05xg145CIWNmJs0QYBDkJ7r\nmYNxwZv96bJqZCdGC5IEsSa/csSCOY4A8BKLaB0urkSrNCv2na/Ef5YeqvPznxhomsY/O5pwYs+v\nsDtq8MjDD6ESJnyyoyi0tlHt0gRfAz/cJ0YMmXHe07vQNjUWw+8apOizNJKaUAKIG/4AACAASURB\nVMPgSK7BzGgtLDFRgtd37TkC34FNv2yBzmPDrV26CN8MeDeJGdNfwz8fHAmCIAQb/nKg1eFyVBbu\neewNbPr2rUCRRhVp7z+uWA/K70evxiLVrcMXywtb1f6tibIg/olXMHpSoOmr6/xplMx7C4vX7sFV\njxed4y0Y8OZMWFu2woHCMrTJjsPJp8aDqqoEaY1Fk5nzMGbKz1g0pX9gu61KmFAJLotA4zmLYLSY\n4XF74fW4sHLBfPz25eeI0mpxa4IFPWfM5hA/BnxCKPW6TXeMFxwrRqxcXgofbDuC+9o3QfNe3eU/\nG16DYG95CXTxKbLNhgHgyPnL2HWyCA8/9rD0OQD1TYJl4E5uAZubwheffYLnHp8Q2CjSf04JIeIQ\nLa8jIhLl1sXBaCT/kLDdDfIkjb+E50mK5EQ6B9tkvnhaHtw+n6iPqspZ9+w5OYhVJyc1BMxGXUit\nUOOXEQP/eLahfP7kAViYn4eTVxwosbtReMUR+lvMcwSIV/FmjmEqf7PH0D4KObYjuOP27nAaYrGv\nqAILpg7EoeIqtE6zYtzkVVgwdSAaJkTh7NWasLnFoEaFIggC7+5xITaqC8q91XDvuAgXdJzX9OmO\nIpgNJIfACXmdxNZSOy4Nr772KuiMFqB18lXZ2V61SzaXqmuQ1BCwxESFyDj/+q49hw9unw8enz/w\nqgVuAGJP4wRBBG5wRjN8NACjOWA45t98fV6sXvot5r/9dOAGq+DmzMxR4/bgtwtX8Fx3FaUNwlQZ\nmuNhYvqRrdlyAId2rEHFxoPonWBBskEP0hoLS7Pmod8F+8kTIUWJqqqE7ehhLJrSH7ajh0OEytKi\nVYhQXZZQoEiLFYboqMAD2ZT+OD/xP2h94RxaN0pFNeXD9nIb1owZjeyEONzTrTl02toHSLZniQuG\nGDo4qlL5nEkcPxUgnj1o1JF4tltrzLlC4uHGPZEarZMmQR4nPK5gfzaXC/r4FFnfE4OWmQ3w67Gz\nOPbLOjS/TVxtkSXkahAkYYbSo0jS6mDx23HxwDaktb01sD+SCuFB79+Y4G8m2zelBi6nB14Phfh4\neUX1Bn5fXPfZdvWRSSc0hxKTuYvy4fgV5dXC6wqhauOUnxatQl0fcFSWw3Xit1D21unyGkwLZpcx\nbzX7rVFbO4kGQoZyJkPP7/Xg9JafcPyn+bjlphZ44eODyE2MxvSfj2PUZzvwz+8OYN/5SiyYOhA2\npxcfjemA6UNbK6oYTgB4Y2hrfDWhi+JjaAAVNV4QOj22n7Zh3+mrYWPubZemqF0MH/xxqV0HY823\nCwCIFyZlg19EU2n9MMof6A85f/IAAIBBwifE1McUFKllUq7diU1AkhocPn0VIABf+k2CtXfO79qE\n9NObZJUJfvbW7IUrMaZdrsyrlQO35lLx/h14/fXXUXxyN+69rRtGJVuRbNAD4BIkl82BmMY5SP+/\n10JF5IpffwknnxqP4tf+L2w8Q6i4pyZAWmMFx7ovnAsNiya16JMchwmNUpGtofHO8m347KedqPEw\n3zehMB0r8+7eZzkZfgAEs/rEsgeNZismvTYdIx+dgnJKI511pzdBbzTgsenroDcY4C0vUV5wU2/C\n+N6d8OWG3fD7RX7L1DYJlkBYPSifF/ff1R9fLvtZUR0mUQQN34vy80BqBZoHq5nq+gsO/U/guidP\n9ZFJJzaHkr5111pxkjvfwcsOwSrUSm68cvC4nNj49WcY1Pd2jJlYAK1Wg6/2XQwRpPqq2M3MM+bl\nH/HbmmU4sWohEnNawd1qCBrltsDiaXmwOb2oqPGG1KL/LD2ERxftgcWkw7jJq9AuMxaxUfI/orFR\nOrTLjA0dExelQxzvOAII28bHvtNXa0mU4yqy402y5RaEwB/nM8XilkH3SBYmFStmmhJFIiuGVNQo\nGADcPl/oh515aBB7+FDT185ms8McExW4yUUFDLZtcxMD55pUEHbDc7o9MOi0im6s7EriB387jGg9\niQYxJgQIkEVBDaSwVxYiGFHDn8Tin3fjo2efwaAzB9F15zpUzngl7Iji11/C6RefgSHahDFTfoal\nRSuQFmvw/aDDCBKfUNWeOuCRajJzXoiAiY5lITvahAcbpaJTnBmzf9qJD3/cIfgbxa9cXts77xRM\nd9wvUs5A2Cvlr7GDunQOG76fhRlvvAp7lURIOVi09MPneweUxN+WKWrdwhBjsssw3NO1LeZ+Old4\nYASlCAQhQsJMRgOys9JxruhcnUgPU6JATVmDG7h+8JfItquPTDq1c3y/5BvEZORc0wqwTGYdH36/\nH6dPHEOxr/bpj73uHg3jVGdshZ3D50PBvJmIu3UYWmY1wEN3tgplzzHw+PzIjjOF9h0qsYdlkikB\nM8+wrqmw+fX4xZsN6KNh1GkxtnMWnnhzPYb1zMWPhy7B5a2dv9LpRZs0C54Y0hr7zldixcES2XO5\nvH7OMd1zEvFUr1y0SbNg3bHSkDLF3iaFkgonTD47NiyZi2nP3o0SJ0Lv0d7iKmw/V4Ft5ypCtZ6E\n3h/2OABw0yTuaNlAMOtR7LO1GkkkoAb3PPoGHhk9ABVOr+C1w4ZZrwVoYFS/ZnB5KJAaDWKMeoAO\nJ+mXLhaD9LuRmZmJsFR+Xsbdgc0/IzHWiqyURHj1Fowc0ApUjR1+jyvw72obSMfl0PFrl36PpmlJ\nSI2XecIPZtYx2VtzPvwI4zs2gVajRfSIp2DpNwK6DrdDm8nNIpMCk3037F+LceroRuRWV6J3XDS0\nXg80BiP8buHvj89WBUOLthgz4lbYjh5G1dqVkucRmoe0xiJt/CMYM+VnjBlxKyrXFcDvcomekw8z\nSeKm2BjE6kjM330CJefL0LJxbeIEP0uw+puZcO3fCurkPk7GYSDDULieERveI7/Bd3gHWhLVeHfF\nL+jRoaVoFhhx6QT854/UZtj5eA8PehN3m94EXavuGD1pFUbe2R4JjZph+/rViPXakZiVHf7eVZdB\nYy/lXEeqQftrr0/eNdmkYTo++Xo53pv6MLxuNyubVPVJBDJSlcPt14MgCERF6SM8f+SgzuwHKPnr\nQikInQFk45vqbb4/Gn8JzxOg3iwuZDBXOsfVsjJUVJTXS0aIWFZUptUIsyE8sw4AVi/9FrktWgnO\nx/YnKfFACe2naRprF3+KLgOHochvlfQysWsevRysecSvZaQEn+4owtkSGyprvHhjaGu0y4zFvvOV\n2He+ErP+3Rv7zleissaLOJ5X6T9LDyn2L/GPAYCvJnQJeaeYbYwyxWyTm/uMywy6QV88MultJLbs\njMScwGfDeKLkaj3xYXcLNwQW+2wZM7/TQyHv5gRFFexJDYELNjdSorRwB/t6MQkQQh4/jUYDj8bI\nrfPEBuuJ+viZItzR8xYAAvV2BGrv7D9zEc8Mvk1yvQyI3StA6U3YunINmiVZoddqgwpLDkZPKsCC\nqQMDvh5B/084/DV2LP3sU/TIcGBUjztQfnK3eNYcD8WvvyTpYZKDbEhPIVJNBjyUnYaDVQ5M+34L\nBqUkoFO3QPsfvheK+X9kTYYDqpTVqMd97XPx2iffYdKjI6ARCxWLKImcTDxGjfI44b1agsXT8nCg\nsAxtc1Lw4KAeeH3hj3ilay/h39p6MIuL1YMy6vVIjyZQvO0npLW+WfhgNZl0NxSnvxyu+7BdJKhr\nq5bvv/0GXfOG1XkdYi1WMoMlCQ6eKgtrr1FachFVFeVwxQvXQhGrCC4EsdDQng0FaNK+M4r8gVCE\nmNmb2QdIt0aRw77TV7H39FVU1HjDwmqMz+k/Sw8JepVCfiTIh9r4qKjxhrxT+85XhsKC/G2AfCiP\n0BngbDkYVcWncXb7Gs4+fnizgbm2MbRYRfYZm09j9ckrnFCsWD9ChlCZ9CTsXhpnrkqbUpnrroFJ\ng7mffgyalg9/6/Q6AISiMMbVShsS46y1G9g3Jt5NiqZp+GlaXR0bjxM/nyxG/yYZAFhG5/w80H5a\nFSH4ouA3FC+Zj8Elp1D+Tj6AgHGbMXlzQnJ8CITo1EJJmE4p2lhj8FCjNOypdGDmiu1wUz6IheGk\nK6PLt4BJs0RjUPNMvDVXZV86gSbOobPuWgbv1Utom5MAb9lF6P1e3H1rG3w5d566c8iB75ESIGHu\n5BYY/ug/sWjDHtHWKRH7oW7gL4HrnjypjZrV1WBeUVEO2u+HJTY2bJ+YWVdoO6khEKXTwk/TiNJp\nQ2OY7DnGJ2J316oINE1j6aJ5aNpvuOQahTxQfEj1rGvfcwCuxjSSPAcbahvu2krO49TmwBMn33xd\nWeOFzekN8znxSRXb36TGBM4f++LSQxj12Q68sPRQaMx/eNuUzk8QBOwZXWG0xMFWUptdx35/ql1e\nTvNiId8YAWBClywMaJLEIbZGUhP22fIJVbM27XBk/17R189u7ZNoiYGXFa7he/zY3w1SS8JZU6PY\nu6E0nF1YVIzGKQmKxjJYt2ozbkqND1blDqB6ySyUz34Zto+U9bKjaRrvL9+GVJMB496Zjcavvx/y\nHUWkCLGM36rGMQSMIEDGximbQwKkhsDQ9CT0zs7C9GW/YvPmQyIjxVvXyLeACaBpohUd0hLw0cIf\nw3eKGcoFmjhzzs7zR7VtlIYyWzXObN8sug414BjExYzmQS/UQ9N/gTUhCZcdXi5JUtPb7gb+sriu\nP3W2gsT8Vsv9ZtfVYP7t11+h+53h5EVMRRLbDgA6UoNxk1dBxyIunJIETi/Os3wtm1avxC09+0Bn\nUJbGLrdfTKHac0FdOi0AxQ13z+/aiOL9W2FL6yKYtRYbpYPFpMNj09fBYtKFSBKfVFWyQmlSxIoB\noxzxx1oFwnKMmqVmfjZKjNmwpGRxetp9tqMIH207i2ijTtZYbg4SqrGTCrB363poKA9HJeR/thxC\nld4ch/bsEl0bv+QFzQvvMd8HvjqrJUnA4wjv0SUANQ8kGwsK0K1FuKdF8pjTl9A7h1/TiYa/xiaj\nOAUUFcrnx/Slv6JTnBk3Z2XA0qIVHn9rI0dlUqUICRi/+ftJa6z4uOD2prO+RPb/s3fd4VEU6Pud\n2dnd7Kb3QEKA0HtHBBUUpdgAUU/gFDwb1it6oqd0PVHPu7ODegoCQcGCoASQjiAQOqEmQAgJ6WVL\nts7O/P7Ync30mQ3R3+HxPg/Pw+7OzM5sZnfe+b73e993P5ffhl6EtnX9p19i3seLcd7lwcLv9yKg\nU8spn7unjIFZqUi2mvHF1xvDz2nm2R1Yqy4gFxGqh265Bp9u2qtv6kxt8o4vEI+JF07Z8REWpI/F\nPff+Dnc/Ok9IkrRy6a7ifwJXLHniKkiTZ+WBMpCIt5hhNVK62nH8O+xIfqNYlkVqWhriE5MEzysF\n9Wql29vcfuTK+ERxlgQXQ1oniiTAsiwcdhuIrO76d1gDchWq5obVcpYDcgG4AMDQfhR8vwSUxQpv\np1tAkMG/kbgdxrXNxDonJVLFX0fcauPArxy9MKqL6rJy0Nq+HI6cq8V9vVuF23EPXZONx65th0aP\nXzbH7xWe/cPv+raGgQjm+LXv2Bn7Nq1VrBJy4AgVZTShdbZ6vIWc5QUfctVZkiQRoAO6LhSSa5zK\nBa3a3oi0ePULNB+b8rZjQGZyMwY1ghWVmIdm4t1LwJiMZHSMsYK22+BtdGHhjJHwNrpA223hg9Bb\ncTJnZgvafObMbMHrHGHKmvm6bDuQaxNOmpUHkiTUW4Wi9xZXqrhtPf6PbYjv3hN3dM5B/4RYvPrt\nT3B6tc9bJa8nNdzcMRONPj82rtuu3JYTV6IiMEG1mIwY2acTvsnNVV1OYjsgBo8U0TSjanVgrjoJ\nQ8kBxBoJTBrVDZXl5YJzXxD2e7Xy9D+JK/avzlWQ+D4aJqMB01/frKsdx7KRa58IgsCgW+6UPK9k\nYKn0PIdSuwdnFHyi+O7hnVNi0Cbegq4jJ+jaz0jAr2I0lzgBytodAPB7XDjy9SK0HzIalZYO4edJ\nAP+Y2FvSDuPaZnydE0d6OFIl1iKJW218yOmoJv1nLxZsPK37+NS2L4fwe87KQ6fUmHBrLjrKiH9t\nPyuo0PE1ZTFmCp1SYzBpZh4IgkCXTh1RV1mOcptbt5fXjWNu19w/7vxKTEpCfX2d4DW56qzBYECA\n0TagDQQCAgGx2gXN6/Or5tCJwbIstp+vwPD2rVSWktfrkNYYmDLbY/iEJzD92b8iOz2YQ8k3pjRH\nW/WRlvBbBYlRzmvvwNvoQu6cUfA2usItQCohUaCfiu3UBY4zpyXtQK5NuGLeWDAMK2wVKrUDFapY\nYjIIgkC76ChMaZOON9fuwck92ue8uh5KHhN7tkd+aTVObt0pactpVaL04Nou7XCytAqORgUzXJ3e\nTxwpgsehbXUQMs68b2g3rPnsXenrLHNV+/Q/jCuWPAG8ClLoh97nD2DhjJG62nGXo32S0zAp3c1r\n3eWrTUVxlasps/NgNRowTsH353Lg93lxbNfmyyJOgLrnE2W2gO07EYV2oU7pHxN7oU9WvKQdxrXN\nxKTHyLOnICCsKC0Y31PQyuNDrnI0Y1QXbQ0Tmqpi4laeFvjvWVjtxI8/H8Tnc0ajKOTMrgSHl0Zh\ntRO584MX0qJqJ9r26I/P1m7S1LFxOFbp1L2fnTp3QVlpqeR5sf6JJA0I6LChqG+wITEuRF40Lmjb\nv1+DIV30h8BuzNuOwVmpKlUnZb1OoNGOBS/9DUv+9TziHDZVV3C94BMjc7QVF159CeZoK6bM3Qhr\n157o9O5ipD/5vGD7pa/MkG0Hlr32Es48eT/OP/1A02sq7UAlUTsVn4Co2NhgRd4ajU7vBteNNRnx\ncLtW+KqsGgd3azlyi/VQ2gJyAHhkcBd8fqgQvj3fNrXlVATikWLaTYOwaNF/5F+MxPspRIoMJQd1\nuZOnJcTA5myE1yca2/8VtE+/pBXOVVwernirApZtipBgWcBN69MxNVf7pBTOCygToeb6T3GVq+Vz\nxwKhTLnmRrAorbPtq88x8ObbcfYyw7PVRONHzteBNAp9ShKsRvRoHY+jRTVYMX8sjpTaJOSET0AK\nLtnRs3U8nnxzC97/600R2wrwLQ0SeaRMaT2OmHGWCc3Jz+O/ZztjPf7zzXq0HT5ecz3O/gEIfq5s\nchfUbF+Bjn0GRrgH2mjdvT8yY+U9ZIRWBYQuzUlVdS3Sk0MEX3RBE4+DHzpXiqdu02dRAAC7iivx\n5+t6Kr7O1+usmDsabmssOCLwaV4+OsdYgbfLUSZjYilrOaAR6CsmXq5TBbCfPI7ls0eBIIOGoLlz\nRqHw6Wmo5LUBZbfHsqAbhMSYT5DC0S52W3ifZEkfy8IfYLB0zhgYSCIc9VIZFw/YGvBw+1b4/EIF\nvDuP49rr5e1ORB+CJCxY7PPFRcCYXA5M7d8J/17yHWY8PDG4O71uBggCS2ePVo7e0Yhs4V5PjY9B\ndJQZRbu3oePQEZLFzFUnAZMFBr0tQTWCJcrNGz/yOqzNXYK7p/Ey90TaJ6qFtU+0KQ7xZjO8EaY2\nXMWvgyuePHHgftcjEYDzSZcS+H5QlY1+QTgv5fT+4g7jpXYPKKcXsSaq2REsnA+QOGz21P5dyGib\ng7NufY7UWpDzgxKLwvm5cocvNqBvTjKOljbg2a+PyW6TIyANLj9WPnwNFs4YifpGHxpcfrCALi2S\nOMtOj4YpQYVg6c3G41eriv2JSDBbUXn6ENK79NNcj/8ZEiSJuKRUeF2NMFv/PzOu1M912hSH8oZG\npHRpInlKPjoAECAMoAz67tZ3bPwJvVslqd6JC/U6QRdtS3ZHrPt8CayGA+gRa1UmLjLESeD1tOBl\nULFxsu7hfOLFPU5/8vkmYtPQvKquhCCJQ4YXvIxK/j6FPhvXqQLEdesBj90lIVcGgsDUthnIvVgJ\nz/ZjuHG4eiaglJCK/bOE5Cpj1fvokpqA1d9twvh7gn5Ok2bmBQn06Z8k25f1fFJ5fdIN/fDu9z9h\npgx5Usy7izBAWG47Xdq3wcr1WxEIBGDgtZopnx2sn2xx4sSvaq2YP7Zlt30VLYIrum3XElAjTmJN\nlJaG6XIhbgf6vF54PR4cKnfosh+Qg5IlgaO+FucLDsOX1V9R5B0pWAB2txcee1BDI0ec5GwC/qJA\nnLhtctWiOEvQwoEvGNfSIinZDGitxxGs3JAO5YVRXSStQr3ZeBwa0gag6tQhuOqrI1grCHOfUbqI\nk1Ysj97sOzEIQqPyFPqxf/WjTcjK6Shs0clcuM4HEpE5ZLRuHcyO8+W4MUc8YScFp9dx/7AYluyO\nmPCX5ThVU4vRnSKb6BO3xbJeXiA/TScmXqHHLeXfxN+OpFUnIk5ciw8ACp+ehnOPT5bdB5IgMKVN\nOk7aG7FuyxHV95cKyJ2CFp7cdN6InFY4WdWAyqqqsP7J5/GCGnav8O+t1dKTed1iMiI7JQEF24Re\nakotYk0RuRgqreZRwwZhy3crm5blTd9dFuTafbyq1tXK038nfnPkSS2fK9LtcJqo3GVLUOEM9ru1\nNEzNhZylwQ9frUB9bU34cXNCf+UsCViWxbavliBxyDhFkXdzwDIMCtYuga/RoWhDoGUTIGdGSQCY\nMaoLGIZF7ryxgmqRlhaJe897//olUHMe5sYK+O01oF121DbYVQnB6xtPg2FZPDB3Q1iTFaltgRj+\nbmNx+seVYALa4utIwbczOFVpx5H9ewVkSc02Qwncd4lhGBBqmo7Qj/24oRlIsJg0WyIHC45j1T6/\nLh2M0+NFlJGCgSR1aG+CbTrG5UBj8Rn0ij2N+0feGLGZJb/q4zhzGrGdumibZvLgDwRQUVWNEpcH\nNj/d/HBXUbtPSZ8lJlac86nScRMEgbuz0lDl9eGbHw+q7kKTgPx9iaZMaTpv2oDOeGf5D8D+NaB3\nrYQpyiwlSRqeT0qv3zW0N775WXTDJad5ak6AsIp2anCvrsg/dgpAy5lkqm2H8tlha3DD67l8J/Wr\naHn8Ztp2QLBSFGWiQAcY0AEmbEVwOWHBH8+4HosWfYQAbxu/RMWJszTg2oG1tbVwu12opi7PNA8A\n9pXZgDJbmHwFaD8GjxqPBnN8WOS9bN5YxQgWPWBZFifWLUObAcNxzi0/eq7XUkCsM+IIy/1z1mPp\nnDGyU3Isy8BfXwE2QMOc2jQqzr3ngukDsfXACdSUXQDjc4OhfWBpP8zp7WFtI2//UKewv5HaFvBB\nUiZ0GH4nGmvKEZueFdG6auBXGHPnj8WEXpl4M285Jt16M+wePyqcXsk5pnUec98nj48Gw7CacUSU\nzw40lMFSc1q9VRLwo7DgCFa/PVtZB8PDt2u2YGSH1k3todJiNH75NrTaiB8+/AcMzkhF9ZuHVZdT\nAr8ll/m3v6uKyp10AOsPlqAqNJFoJAjEgkQUQcDJMnCyjGBvCQAUQSAOJOIJEn17t0Ka2agpEJbV\nZ4XWaY7w/daMZGytqsfnefl4YOwgcPolYWsuSEhJa6xsC08cAQMAFiOFMZ2zsPjLPDx4360CEsT/\ne3NxO0rngNzrJsqAnIxkHNmyEX1uGhV+XtIiVtPcqZyfSq1mgiDQu0sOTvy8FT1G3RtuqbH+CGJa\nBBsUtubktsOy7FXR+H8pfjPkiasUTX99M97/602IMgUPjfvx19I2ycFDB/DVqq9w85hbm71fStl1\nfMi1A9euykWnmy7fmkBO70QZTbiIREAk8uZ8miIhUASAaJMB+WuWIaPnYBT71MmeWhadks6II0BL\n5wgJC+N2gik5BFv1JYAgYUpsBQdagWwUXjimvf0TkmLMqHUQgKkLwNNH+xjAWOtGbWgCLqNdcP+d\nZ/aCik3BjG9ZJEabLjtPj4+zjVb0zYnMVRsIWkkMVJi25CqMuSEPm6ff2opOBjJMliqcXsWWs8Ph\nQGyssKLDr7yumD8WHo8LVouOihXDKGtP+Lh0BoFtSzWJEwAU1zsxbkDPpgv3/LEg7nsGzi/egRKB\nOrHnFGx+P3KgUeHji8LFAnFe5UaOtLjoADYcvIhShoaVINCPNGOYUf80mZ9l4QCDepbB9iNlqGED\nYAB0II24ZWC2fItVXE0SabMKn54WscbqxrRE5Nfb8eEP+/Dc58tgycqRFYcre0DJu5X3ykjCgbIa\nHNu5D70AZZIkfo5XmZJ9HcD4IT3x1rfbBOQJgIQQhYkQ7zld56cCsRp7/TV4Z9nX6Dx8XPOE4vw8\nvF9YcH4Vvyx+M+SJZRG2KvDTDGg6IPjxj0KQRHGj1/q2yaLkwgVce+vEZu2T2mSeGJwwnGZYlJdd\nRFx8AqyxEXjOyEApSJZvS8CJvJ1eOqLwWqDJ22nHd1+g++034uMz2mVxtTabWmVKTFgCznoMxRk8\n+OwkVNMxeHTRHvhZ+T40yyJMjgT7TwAfPTYEgzqmIL+oBo8u2oOK4tCFydgB6b4LqNm5AvbkLMR1\nHYrEWAvqQ0L15hInDofP1TaLQCmBi26JokgMzozHB8+PxIcfnMbCR4aFyRL/HONj6eJP8cTTfxQ8\nJ55Gdbs9iOLIk0ogKksawq2SFfPGyN7hO11uWK1W5Qkr3vRVZYMDqdFRwQt3aXHwQkMzoFq1B2mN\nBeOSdzv/7lINpmRnqH9oIuIBAHHdesBReBql82cIS9Yh0uINMNh4oAQXGBpmgkBf0owhxuYNXBgJ\nAkkwIIkwoAMZ/O4wLIuzrB8L952FhSAwrm8bpJjlpyGBpnbdlLkbsXz2KNTExDVLoD4oKR5txv0e\nn65dj+tG/Q59OsiHK8tVmdQwpW9H/HtXAXoMGwRSB1FmB9wBY2omaJoBW1+u6EJuNBjQJTMNB37M\nw4Bb1AXV3uSOTWSptkjz/FSD2WSEgSTht1WCskZHRHhoUxyMZrMgWPsXE5xfxS+O34zmiSAAk9EQ\n9NwIiWbpAIMV88eCDjDN8nM6enA/hl8/LCKNCAc1d3ElcBe1TWu/ReaQ0ZrLa4mD9YQEc5NdnE/T\n/bP1h/ty6/xQYMTE0TeibbJVcx0tKAm5xYSF8UZh7gvPYubiUxjUMQVJblCzgAAAIABJREFUMWYQ\nBJDMC91VArdcUowZgzqm4P4568Pb4F7/+PHrsOHNP+LDv89DVGo2ulduwaTkSxGLxCXvDSDJakSi\n1SirC+NHusghv7gOJ/cJp5bE0S3ccEFsZnts2ndIQNwjaTnzvZ5YhoGBNPA0GlJiT5viwMZlwO+j\nVf12Dlx0ou+EabJicbGh4tdrt+LmjpkACBBEcN8fmLshNKUnfywbtx5Bl9hoRGlM8ol1QnHdeuDY\n+XrEde2OrJdfF4gnf86/iPf2FOGz/HNIIgy4i4rG7VQ0ssiWvf8kCQKdSBMmGmNwg8GCdYdL8c6e\nImzeJx97xOmglswaDcJgQPbf321WvAsVF48bJ96NdUcCyN/+HTxl5xUIklImnhwImGPjMbZzFnK/\n3qC9eEggPv31zSBIQlMPd8fgHliz77j6NsW6J0CoaWoGRg0bhC1rVkXWqlPzhLpKnK5I/GbIk/hO\n2Wyi8MDc4Bc2ECJRkfg5EQRQXnoRH2/x6iY/fFzOZN6YCffAFBX80VAiSPwLphrEU3pKZpicT9OS\nWaNBEsB9fVtrkgRuna//9QDsbj8WTe4vIRdyAnA1yFV12EBTG7GiuAGVF4LVofyiGiydMwb5RTWo\nc3rx0WNDsHn2Lfh4+hDFawdXbdo8+xYsmNIP+3nb4KpTElIV3wb//Ps8LPvJg75tEppNEjlN1/IH\nB2Plw0NkPystAT9BECg+cRjmkGGo0jSlh2bgic/E2TNnwusqncNWqxWNjfLGmoLvC0HwLgAmSVgq\nSBLf7jgPo4mCoeyofEvEYMSJkjL8e22V9OIoM11V6/IiJToKpDUGUZntcSzkC+a+eE72Ik4HGOyt\nc2BYsoaQV0Yn5Cg8jd4dUzBpZh5iO3cBFRcPm5/G+3uKcJGhMYGKxjhjDHJIbW1SxCAImJKFsU8x\nBIlbKCvupqLhA4t/7SnE3vyLklUrP3gTFAFMmhlhvAsPHAn7/sOnkEoS+OTBByI9AJGYv8mw9Jqn\nZ6G4zgmHW2PIxueGz+PFwhkjwQRYeRE5D5QlGn3at8budQoZeYCsAJwzyAQQ2SReCD06tsPxwvMR\nrXM1D++3h98MeQIAH9N0p8xFt3h8NNwit2Q9YFlg/MR7sOKV25ptS1Bq9+BsXWPEk3kVCP4IKREk\npQumEjw0g71532i6iH9x+BIMBhKTZ+qvPj214hAeyz2IOItRMoF2OWP9AOCrL0f1tmXwVl9ARXED\nKoobBOQHAO55azseWbhHsYokhni5F3IPYeTcH/HIwj3hZWodXgExKyp3IL+oBivfnARbo0+WJOoB\np+kiSQKTZ+Whb5sEFFfYUXHyAAB1l3YOBIBbrh2IDkx1uNKkVF2MTUpB1959AKhP2rVqnYnyS+Xq\nO08QYBgWdIDBsrlj4KcZSVhqZXkFpt8zJHiRUrjoeZM7wm53YtXrE6UXR9F01eFN29A2ITh8wOlt\neuUkwXPxLBq//Lfs9j9dn49bM9T9oOTG+sv+/jeUzp8B+8kTyJ0zCrYTBfjyx0NYfqAYoykrrqcs\nIH8p4S5BoPdnH+K6HXnovXihpGpEEAT6Gsy4h4rBeYbGh3uK0Mj7LaMb6mE/WSAf7xLavmzEiwic\nLUKnTd/CQQewc4e+KCI5Z3exhcGUa3tjYe469c2YLMGpvJl5MFIEiGObFBflKpRjHvkrfjx8RnE5\nAIpu4oKKlFKFS2ZCjyAIJCfGo6a2LiJXcUEe3lVc8fjNkCerkUJclBkWKnjBEcdLNGfizkMHmm1L\nQJEEsuKi0CEpulltPzmCxK8qaLXjuG0AQNGRfFjjtH88uWgQOZdwORw+VwsWQHGtS1ar1NyxfpYJ\noP7AOrjOHQbdZjQaPE3aII78PDB3PQZ2SMaq54bj4+lDUOcUEh45jRMgJUY1dq/sso8u2iMgVY8u\n2oO739qOOKup2TYFnKaLb7nQ4PLDTDAoP56v6tLOIcZM4c6xo/Dkyx+FzwslDzCCINChczfNFjIV\nn4rKCnXyZCBJMAEaLO2HgSRAEBDcQdOmONidDiTEWJRFuKEWyuodRQDLql4cAQRz7HKacuy4kXnn\nF2/LLn98zyl4GRZtrOrfN9mxfgBgWZS+MgP7HpmEZ8fdi1TCgAnGGET/wsGvpqREpAzqh8lzNyJl\nUD+YkuSryQaCwHDKghspKz7dfx5r9hWHLRBk410A1YgXNdzZKhnbaxrQ4PZpLiv1egre/PHF5YkE\njbgoEw5t36u8IS3rAg68CqU5LQvtMzNwfLv6uSRpH/MqUn4fjUBmb0kFSs0j6o4R12Lthi2R2xVc\nrTj9ZtBiDfvi4mK88MILaGhoQEJCAt544w1kZ2cLlnnvvfeQm5uL9PR0AED//v0xc+bMy35vkmzS\nO62YPxaeAA2GaR5h4qPMof3DIYesuCjEWYygaQZTZudh+dzI3cjFBGlwZrxgao4TBysRJ27Krqy+\nEWs+2oFWox7UVSmRcwnng2VZuBtqcKY+eEHh3LblJtD0OHmL4W+oQsOBdYjrczPqnTEgiSBhqnN6\nQxNzQfLz+exg/MTkWXlYOmcMkmLMeHTRnvAyRGg9JWKk9FrTcQpF5iyLcAVq6Zwx2FdYg3qXH35H\nLYyx+oXfL6wuQKLVCBZAg8uPBeN7ok/WMPz1pVkAM0Dz83d4aZS7WNw+NEtAnNU8wLRayCnpGag5\ne0KynpzNBydwNQBNF4KQnuPZt9Zj7qPDAAMlL8IN+NFYV437RnWXtyjgXRRXzBsDJ2FCfBRfLK2u\nt1lTXospbdIVX+eg5peUv+8idgbcmEBFw/JLkCaCgCkpEb7apjBmX20davIPIXf2KNTkHxK8Jod4\ngsTdxhicDPjwz71FGGmwoO+gLFmhuGzEi5yNgdhR/bWXMLlNOt7L24eXJgyLwNm9KOzs7i45i7oP\nZ4VF/RN7tMPbuwvQ94bBitvTsi4IfmBCkjVuQGd8uHEfegy/WfVzE4OLc0Fmb0yevR65c3kCcp5W\nSk5Y3rpVK1RtPaZsM6AyVKEImXX+120K9PCKXbt24Z///CfOnDmD+++/H88//3z4tQ8++ADr1q0D\nRVEwGAz485//jOuuu65F9o1gm+3eJsTUqVNxzz334Pbbb8eaNWvw9ddfY8mSJYJl3nvvPbhcLsHB\nNQe1tU4wRuEPm9VIwWQ0wOcPwOWPYNRexQeqsjHydh1FEuEIlxXzx4JlIZm2E9sX8B+LQ1256tGd\n3dLDU3NrTlaqXiyjKDK8/N19XSglE3GR0pg80onzP29AbFomSolWEl+mF1cXSIwv5aJMxM/xH7su\nFMBGp4MwGAXTcHaXD3FWE/KLavDYR3uQGG3Ggin9wpNy/Lab3BRdy5zlkJCyOJTAW30BCQNvl/2h\n446tIRR0zP8cEq1GfPHQNfj97PX4050p+PDHw0jpNUx7HwAQp7ejXZ/BSEpXd93uld7kuaVkm0GR\nBNKjhVU0vseThw7g4P58GD0NuO7aa2TfhzbFYV3eOnTr0B7dopUrtQWF53HuyCGM7p3T9CRvuo6b\ntjp16gyOHT2C2wf3ReOq96Dl6bRl2zHU+HwYkaojOJtrY4nG/nfkl+AE48cdBmvLtej4ZCnUnksZ\n1A81+Ydw9MHHm358ZEiV7HMi0CyLjQEXokHid4PbyZ6DmX/7exMpUnA8p+IT0Om9JWGSVfjUVNC2\nBhxpcKLW58d9owdoHShIa/BcS3p8ftgLqu7DmQLCu/diFVw+GhPGh4iOVq6dGPzlQ/9nB9yBRV98\njckTJyDVU6Z/WyF407oBUbGgKBK00xaunGpZGizdeRLXXj8cWZmZglac3FSdFiTrECRoYwyMoWy7\nmGjtQZiWhmfz52DdeocDtEFYYhE1MjItnR5ecfHiRTQ2NmLDhg3wer0CfrFr1y4MHDgQZrMZp06d\nwv33349du3bBZFKeYNWLFrm1qqurw8mTJ3HbbbcBAG6//XacOHEC9fXSO6EW4moSuPw07B6vbuJE\nENL4Ff5rBEFG7MYMCO/ybW6/pO0n1p5wj90VFxCgpfvuoRndbTr+OhV2Dz57eSQOHj2uSZy0Jrw4\nOCpL4XU0oJQItlLEbbl/TOwl0TeJBeBiHVSy1Sh47EAWiJDOgK9PSoo146l/bMGgjinISY9FrcMr\naa1x0Kt/ag7EFSk7shHVujPqfv4GrPiukXesKx++RvLZ8CtzVFpHlJ0pAMswmn8PFoCv3TWITUhS\nXIYj3XwyLiZOXGu5c0qM4DvA93jiJlQDgQAog/B7ItiWz46Gs0eRQatHzxTs/gld0ptayOLpOuLY\nJoBlsX37VnyZD1iyO4C0xqo6i7Msi921NgxPTdTW9nBtrHcXI/3J58NtrB35JTjN+HFnCxMnvpbJ\nlJyk3J5jWQlxUtNBcaAIArdS0WhDUvjnXqEWioOeqBjaboOj8LSkGtcnIQYVHh+O/XxK42CbnN3l\nvaCCuKZNGg6X18LrpyV/ey1Ilve5w9XKLeeSsXbjJn0u4oBgOXNtESgDiUkz8wRO5EpaKQ539G2D\njV/8R0iQ1KbqlCBahzbFg4hJBgxGBBgWlIam9bcKvbyiTZs26Nq1qyBzkMOwYcNgNgd//7t27QoA\nsrykOWiRv0p5eTnS09PDdz0kSSItLQ0VFRWSZfPy8jBu3Dg89NBDOHxY2fnXbrejtLRU8K+8XF2X\nweisknKkiQqZCPItDKIoA37etgVWowFTZuu3GeCDH+EirjDxtSdmikRclBGTZq5D3nff4JxNuU2o\nN9uOu2juuFCPxVsPIn2weilbz4QXADABGkXbv4O7/Yjwc/yLf8ElO3q0jtfUA/HdwntnxiP3oWvQ\nPzsRv5+9Hn2yEgREh69PqnN48d5zN8Hu8uGrZ4erTtSJdU1q7bmWgM2fjpiOA1C780uwTNPFi08u\nE6NNePLNLZLPhm/NkNlnKDxlhbr+HpQ5CkazPLHXM4mZFReFzqkxsFAGMCwLykCGP0/x5CrLAiav\nDRSlTrCdjY2ItqqbRJbU1KNNaojgyGWbhVoyDfUN+OqNe8IBv3wxshg7dx5H9/hoZP3t75raHio+\nAfE9emPynI2I79EbVHwCduZfxGnGj9sMVmnlRmYKTi/EWiawrO72nF4dFIcc0og7jTFYVFCGw/ml\nwhdVYloAhAllbKcucJw5jbLXXhK8fHdWKr69VA1G541vU5zLe7KvT+zZHou/3aKeayeGUg5e6Hz5\n5p+TcbHoDFhaW2oh0TIF/KAbbfL2GioeUAmxMbA7RJUZ8VSdHojWMZpNeP69nTBSJH4/ez0MIcuN\n8vJyyTXRbr/yxOd6jyMSXqEH3377Ldq0aROWDV0uflWTzEmTJuHxxx+HwWDA7t278cQTTyAvLw/x\n8dKx2iVLluC994RfvszMTGzZsgXJyTGotruatQ9i52RuIo9lg69VXCrFG4t3YMzYsVg+V9tmQKkV\novQcX3vipRnYPX788dZEVNUPgjtUUVLSMmlVnMRu4gmt22pO2PEnvNQiWgq3fIuOw+/E2UbhKcPX\nOr0+vqemvonvFk77/Xhg7mZ8Pns0ls0dg0Pn6iREh9Mn1Tm96JARi6+eHR6OaVn61DD0yE6Ubc3p\n0TW1JOpdCUjqNQK1u1Yh5fr7BMe6bO4Y1Df68P5fb5J8NlxljgDw9F2j0Dc7ESQBTJ7Z9PcAoNv1\nnRs0eOKNzfjg+ZGIokj8uPZb3HJHk1t9mMSHvgPTX9+MhTNGClrYHjogcOWnaRqUUf3nwu+nYdJY\nhmVZGEhSQJQEsR0mC3x7vgV79CfUfRD8QU16fD6mv7kVC/96o6xB4546G6b17KJP28Oy8HMTgwEG\n+/ZdwHHGhzv5xIlrl9XVK7fZdEBOy3T0wcc1W3FK66qCIDD0s4UY27cnXp3xAtzrt+Hagfrif8S6\nKEr02ZlIErekJeGzvHw8dOtgHVtU16ZlJ8Rg3emLqCs+IxvZIgu5cyUETifVK6oR+3/Mw6BRKokQ\nYi1T6DxUimTRQsfsTJzL346cQcPDz3G6QBhjQMQk62rf8c0yfWQi3njqevj8AayYfyuYAANQwJQp\nU1BWJmxLPvXUU3j66acj2me9MLXvDtAt+PtJBW+Mf+3jAIB9+/bh3XffxWeffdZi22wR8tSqVStU\nVlaGc3gYhkFVVRUyMoTtouTkJmHt0KFDkZGRgcLCQgwcOFCyzalTp2LCBGE8CVeWq611AkbtopmS\nnokbt6YDDFx+f7hixbLA6tXf4duPXoTNHcwDUyNOehzExeRK7PJcavfgm3Ub0O/uR0BCPk5FCXyS\npeQmrgU9E15epx3mmHicbZT6G/HbcnpjS15YXQCy6gyGJbmxdM6TsLt8iLUYMaBDMj6ZPgSPhjRN\ntQ6voE3GF2w73H70aZ+Eo0U14dacWODNPVYTjyuhOevU2SxIH3q35FiVNE98hKtUs/Lw+ezRWD5v\nLAqrnbivb+uIXN89NAO3j8bCGSPR6KWDLdxLwkoEn8R7fAF8OGMkfP4A4qLMAhd+8XeHuOzo6KAA\nlh1wB4wprYMXQZ5AmHt+1+pV6JXCkSQCPrcbC2eMhK+xEYxLqAmscroRR1GgSFI9240XveI6VYC4\nbj1QvPsn7KirxD1UTJA4hapMXf/xKlIG9UPd4WNI6tsLk+duRO7sUbpIjxgSsiRuz0Wyrgr4larl\nb/0Df9kyCMi/iGsHtZH9DPhQE9Bz6BBjQYG9ET/vPI5rr++ha//VMLFHO3z64p/wp0fv0xXRA/DE\n5HIv+twY1a8L3l6zU508iabskNm7SdMUIXECgoaZS1avx+ODhksE31q5dRKwTDBmyth0c2+3uQGw\nMJtjsHz5cgREgeJxcZcXTPz/Ab3HoZdXaOHQoUOYMWMGPvzwQ7Rt2/ay9p2PFmnbJSUloWvXrli7\nNmhWtnbtWnTv3h2JicJSc2VlZfj/J0+exKVLl9C+fXvZbcbFxSErK0vwr1WrVrLLykFJz8SyQdNM\nA0mADjCCVp/f74e9sREVPlLSchNDj4O4krcOf7sVZaVIyWgFkiQj8m+Sa82IdVFaVScOn+wtwSs/\nnsHHe+VdjM0xcWhI1xKMqkev8OG6eBI1F85gVVnb8Pg/5300sFMKPn18KDbPvgWfTB+ClDihXomz\nDIi1GDFpZh56d0zB4fPSihUHvi+UWqvvctfhUHlReHHnPhOtz4ZfpTpX58L8H8/gy8OXZH2fOE2U\n3N83iiJhCVVWLSYKURSJmNg4OEQXRK617A0EvwD1dbWCFvYvNeTDkpRsq47fljl2sQJ9c4ITNaQ1\nBiaLBZNm5sFksYRFyZwpY+6WI5i64F8SzyYBROP6ZQtexokn7ser0x7FeComqHHiNEbb1yF1yEBM\nnrsRSX17oe7wMf3VH9kD1k+WIl6X11LkV6pq9x/G7T4ShwNe7OZMNTUsC/Toou5olYyNVXUhci02\nxdSL4HqpMRb4AgE01POOT6t1B4DtdbOiTspoMCDeGoWqOvVQZHPVSRjKjsJoopp8nvRopWSWiYux\nwulyw2+MFdoWKJliammgROsxvAtUq1atJNfEK5E86T0OvbyCD7Gm+ujRo/jLX/6Ct99+O6x5aim0\nmBJtzpw5WLZsGcaMGYPc3FzMmzcPAPDoo4/i+PGghf6//vUv3HHHHRg3bhxmzZqFN998U1CNainI\nCV45RFEGmE0UvDI5dzu3bcXwETfpmrDTGv/WG8+yfeM6pPW/EYAw3FVJGM75PfFJ1oh2SbizW7CP\nu+ZkJfaV2TT3nw8uokUJcjEizYWn/CzcZafgSRkmGP8Pex+dq0Pf9km4f856XNM5FZtmCQmM2DJg\nf1Etpr2/WzGaRSwe75Ch/YN/uYLzcD6eBsTu65z+6eO9JXB6afyub2uQBPD57NHhqqBYo+b3Ciue\ncsMFbdrloKzkguw+cO275cuXhlvYZoP0xoMgCLAadS8twuXx+WAiWHkvH15bprG6AmZ/8PmgAPms\nSIAcNGW0TPsboq8fiazBQ6SeTTyIfZ3Mrdtg4Y8HMcpghTm00/zKjZ9mkDsnSJgO//4R/HTDWByd\nNl394EQfRHN1UpG8h1hQfvTBx8P7ShAExlHROBLwYk/+RclnIHEg19JFIRgdc3dmKj74YZ/EFFPn\nTgvWm9izPT5bGUyB0CUeV9I98XDXtb2wYlmu9q743BLncTWoeT516dAOF0pKJCJxyu8UmGI2RRup\nE56rZppN0MMrDhw4gOHDh2Px4sVYuXIlRowYgV27dgEA5s2bB6/Xi9mzZ2P8+PGYMGECCgsLW2Tf\nWkzzlJOTg5UrV0qe/+ijj8L/X7BgQUu9narFgJzglVsnykThaFENendMAQsICFRBwTHc9dBTuveB\na8HJQW88yy13TEBZQPgjIF6Sa8HxW3rcBbLK4UFabFO7bgDtQ2pMFC7YfDhwoV6zzfNrwltdAte5\ng4jpcht8ziZh56OL9iA51hxutX08fQiWhtqqD8zdEPZx4leXxJ5OStYEfPE4JzbXsi9oCcF5RXED\nMtopT36JbR5eWF0Qrk61RZMWbfLM4N/1i8OXAEg1au+8/x4G33oPzJamlirfAyyKIlFtToKj7Dy6\n9uoj2Af+OfrmPwpgcwePM95iDuugON0TSZJB7QXQ1J4QtSm05EClFdVok5Ig9PLhjZ4TB9bCb4yC\n78Q+YEh33iclyIgJmzLe/Pu5eG/mVNAssGLeWNiOH5UlAPy2lLfRhcODRuCWnL6I/vdH4Z3mV27c\ndgeouCaLh4gqR2p2BC0IPtnjtxT5+8oRqG/pRpCbCpCu0ZrTg1SzCe1TkrD3fCkW/SdEanUGBfPN\nNFfMHY3UlBTQzHk0+Fik8Py9aBX7AiXdE4fkuGg43T54fD5E8cfRZQKAdeucNDyfhg/ohS83rseK\n+c8E9U0sI2s9EFEb76qZJgB9vGLAgAHYvn277PpfffXVL7ZvV+QMpDnKKNuS0wLLAl4fHc6vElel\nnnzmTxGbkmXEmGVbcxRJCKbulJCUkhr+v1zbjmvPjWiXGH6tVVwU9pXZsOZkJbYV1/OIlBe7NuXh\nd88t1R2vooXmVJ2U8uxYvwfvzn0JW+aMklST+E7fjy7ag5FzfsT+s7WKBIavadKqFPHdwcXLkCTQ\nsZW0GqVkgxAJKoob4Ck/C7pRWglUc18/fK4Wp/N3hrVohTwtmlij1rpTD5SclsZocGT7zm7puH1A\nV1QrOIhz5yhBGuDz+sCygM8fCGqM/E2CcZIkEWAC4btnf1Si5C5ay4bkbP7PyEoJEcqQxklcbaiu\nqUFKdNN3SepeHRMehx/V24IOnTpj6rwNYJkAKt9/Q/G9y157CedefAa2Rif+tXQb7nj4D8IJtrBY\nHDDGxmLKPH1TbmLonpK7zOqUXkE5QRAYT0Vjd8CN9XdNk2/N6Yxv4TDMasTm71Zj8d9GyNoRKEHO\nxmBCj3ZYumKNprM4d64AAL31cxAHlPPsbhvYDd990XTBVasaaeqcQkRJrUqVnBAHe1lRU7VIzq6A\nZeD3+q5m2/2GcGWSJ7N8S46DWtvOHcq9kwsKrnDpz74DlFtzfK2TVguQ78MjbrkACBOmtNgoVDm8\nyJ0/FnSAweDM+HBbj7Mx2Hq+FkcKTuKbdx7SFa+iBkeVstmcWtivWp6dwdwG13RO02yHccRIL4HR\nqhSJW33cMiQJbJszCqufH4Ed80aBJIXrtMSkXr0zGvX5ayXEQst93VFZivc3F8hq0fgateqoTJSe\nCbqD8zVyfCLeNi0Bo+8UDl/wQTMsEpOSUN9QD4Jocus3GQ3h7w5JkmDYJhGsyUhh+uubBW0KrRuP\nkup6ZKvZFAA49vNBtE1oIrNKvkENX7wL+4/fw3WqAMtn66imsCy8pRfw4WuvY92SVyWEI0x65myE\nkSKxfFbzdE66SI1ODyct8Nt0aiAJAhOpGGygG1FbUyPZl+bEt1xfcgr/vutWRTsCJYhtDFKio2Dz\n+EDvW61MikLnypQ5oXNFA93apONMWVXwAa9qpFvbFAKfdGl5PsVGW2G3h26SZPROwUqUKSLjzKv4\n70aLOYz/mnA2emE2U4LJIEDYyhM7JIsh1/ZrThyLeOKO7zCeO28sztQ4VQmU2FE8uO9Nk3L8Vt2R\nSgdGd0pVdBo/e+wAvG4XDNl9NYkTASjaEngcDTj30w/wdRkjWSfRasSMUV0k7SYOfNfsZXPH4L7/\n7EW9yx/WAX08fYisK7jgfX6h6ThuGS7qJTHGhNXPjwiX0se/sQ1F5fruoiPZxzjiIhivCzGdhe7c\ncu7rHPwNVUh2nUfOdSqTQyHU7fwSr7z0omRKk3/u1HvU77BP/bwVWdnZ6NK1m+x3J3/vHsQwLgy8\n7qagkV+ohcfSflA+OxxOJ1Yu+QSP3ausWXn9H2/jj3fcECZZ4qk7APg493vcmNNaUH3iWnX8CseP\nW48iABb9E+NkJ8jksDP/IirYAG5Kay1LanovXhhut5169m/6iZPYCVzDLdyUnITrduSFW24/3TC2\n+aLyCOBmGXxNN+JPgzvAGLpTUHIW14NlJRX4w6iBSLDodWuW/h0BoKCyHhUOF+6ecIvimuyAO4CE\nVjAaSfjcHpDbF6u+09Kt+3HbXRPQKjVZ3Slcpp3HPR/I7h9u1RlKDqpWqY6dOYeS8iqMvntK+Dna\nFB8iTD4Yzaamlp2zVnflyUZHgyQJJCfHaC/cwmDO7W9xqwIyRzpZf6Xiiqw8eT1+QegvIJ2uEwcD\ni9FSlFHcmtOrdQLkiRMg9HPiqkoAMLpTKtw+WtFp/MyBn9GY2k0XcVIzYjy7Yy3c2ddJ1lkwvidW\n8Ewt5cwwxcaZ4QiWkKBbq5qkNummJAoHtCtFLMuA8ThRU1OFD/4wAJtn34IZ43qgzhG8Q6xzeCMi\nTnqn8QgCMMZ0hKfyPAJeoTeZ2gSeMSENjbX6zOCMBgPSYkySKU29xqoAkJaeHh4flv3uhA6S8gfP\n2fvnbABlIMOPq6qqkZGi3YbiV6eIA2sl1YaaRo+IOAFyvkHHHY0qYLdIAAAgAElEQVToGRetS+gM\nAH6Gwb6AB8NIsyJR4VdyIiFOkipSaErOy7KoZGgUszT8f30KrZe8j26fvAtfXX1kHk4tBAtBYozB\nikX7zoUroXpsCiQItfnubJWCTzful1tAZhKPUBSZ90hLwPFK9XOUOP0TjEYyNHUZBcSot1Nv6t0J\nG777DkDQQVyuaqTVzqNdDt2C8h4d26Gg8Dxvh8kwYeIqTldbdr8tXJHkCRCSH6U23S9RU5ObmhMT\nJD1ap6qKS5Ln1KwJuBaMxURhQ2G15IJYX1WOuKRUEKS2DowvOhZroxprKmC0RMNgEd7p8DU6nE+W\nkhnmi6sLcPySDdkxLF4f3xNVJQ1hsvHRY0NQpyCyB5T1S82xD/Bf+BneE9/De+J7+E/lwX8xH+aa\n4+iYYghvf+Jb2zH+jW24YdZG7Q1q7KMY/H1e8NwzsB0KThaptT35iIpLgtumrTnL6NANx8+WyJJq\nPV5fAJCU0x3de/QMP5b77rBgFUewa4tPIy2Jp5uRaY8ItslziBa+hz74GRYmUv/P1xf5xbjJYFFv\nLTbDVoBr902avR7niQCW+Dz4sKEOCxvqsKyhAVvrnTgRIBHdsT0em7McXx3Yi/943Xhi/O/wTOeB\n2D31UeWN/wJTeymkAT1IE1btKw4/p8emgL9PXJuv2+w3EENROLKbT0rkSZKcdq1pkwQy46NRUq1C\noJz18Lk9WDF/LHxuD+BUJ1uZyfG4VGdvIkjJHYULaLTzvGndQFljQbsciq06PsjQuRi2FmAZ+PxB\neYjPT1+doPsN4oolT3woTdfpRW1NDfw+7Zadkm+THMSxLHzUVFVi385tgufUIjXEWiibR6bV1ugE\n2Vk7VBZQN8Y8t2sdnJnS9hKAcEXpaJktHCkih3irEUZnJcY/NB99shKQkx6rSIjElSQl/ZIcYVGr\nRFUXHkWDLxp2YzbsxmzYqDawsckoc8WgsMKNpXNGY/exCzixPx9F5Q6wLAP/hT1g/dqGfXqn8fj7\nfPOgzkhIzwIYRlETJkZDUneUHd6luT+N6T1w0mPWXWXiQ2/0EAGAZeWX9Uclot6cCnP7YEle7o7e\n7fEiyhS8QCmNpdfaG5Fo0baFqHN5EW9UuUkQCaCddAAOlkEG2fKBCq6aWrz24svoH3sMRXsOoHut\nD9d6TBjiMWGA14iufgqta91oa4zBxiUzcUe/azGgjsY1Hgox9Y1Y2WDDBw11WFbXABe/ItFCuij+\n9jgi1tVggh0MjnAxLuLqnYqAXGx5cHvn9thQ1UQ4lUiSWLsmxuhOmfjq+22qh0BuXwz6py81W3Yc\nLBYLfISCl5OaCJxPrKyxunVSvTvn4NTuzaGdNYR1gSYjJZlMvYorH79qPMsvCXGcRCT46ssVGD3p\nQRiNyr17vjg8d95YgUu4GuRcyPN3bUds56berx53cP74uRxate+EsggunJ/sLZFonhxVZbAkpCBg\nbCKH4pH6Sf/ZizoNI8w6hxvvLPoGG3IXIL+oRlasTRDAx48NwcBOKcgvDFoHAEHCIRetIiYsdU4v\n3nmgD4oPbYGbiseHhyhUnTmq69jvnbkcKfHRqG5oBBAkWgCQnNEe/rPbwNI+UOndQaZ0VKxU6Il/\nEe9zFdEJ3WLM4QresrljVF3HjbHJaNtFPZuQD71VJg56HPI5RBkpsFRMWMcR1m8EKJiMFN78fDc+\nfe0PgJGSHes+e/EScjIzBEJx8Vj6zq0/o0+GdqXlx58K0Ddewa8rVBmJ69YD9pPHUfbaS/jiwAWM\nTm0NNLTsXX8DG8Biuw19//MNWienoLyqFkYxHSYI3PXdYmQN7oPS/CP4Zty04NMgkMIQSPEG719t\nJIPcBht8BIu2fgPGd2kna0XQLMjYJ4wyWLGKdqIXywqDkGU+P/6PqrjNRzkdSDIacXj3SfQd2k01\nGLhx1ftwW2PDWYXukiI0rnofAItYswlufwB0gAFlULmn16g48TG0cxa2/rAaK+ZNBt1ol1gSmGuL\ngFpIrQr4xMrl0B3Zct2AXvj8u43ofNNEGM1m+PxBp3/OvqA5iHT6+yp+PfwmKk8cmtum8/n9qsQJ\n0KdlokhCcCevNI1XVX4JSRlNUyN6zDG55dTAOU/rgZwxZnRyBhytm6pOBIC2yVbBSL2ej7jhQB4u\nZVyL+97eFdY2ibVOybFmXNM5FffPXo8hnVOREmcWtPbkfjMeXbQHN8/7ETOWHYS5+gh2ff851hwx\n4NHJ48BWnRV+FgSQmhAtf+wswsSJj9qKStgMmbCb2oKlPfAVrAZdIV9dU9NY8Sti4uPWmrITwxgl\njcRpDo7s34viojPhx3pNXIHg8VjMJrz62T6pfiNAw+encf8t7WE2RQEeh+wdfVF9AO1vmwq2182K\nY+mnq23onCLNuRSjxOVFW6t8hUpcGbGZLWh1z3jcvuvHlqnghJBXbcMymw3D3EYkBAi4qnjtVYKA\nNS1o/mtNTULbYQMxZe5GZA3qA2uqPDmMZ0gM9BoxxGOEl2Dx1vmzWL/0ixbRRcnZJxgJAtcaorCS\n174DpJ+fxEwT0jbfqPQk/FjVRGqUg4FZAKxi++7a7DTk/bCt2cfZdMDBlnDvtq1xbPPaJr0Tr4Kk\n2M4LwVx1ErTLAcoaK9VEKVSiYqOtaPT4BBOpbGNds1t1tCkO8QkWmKP0Twhexa+H3xR5ag6cTgei\no4MXWTHxEUNNy5QVF4WcRCu6pDa19eQIl8fjhtks3/a7HInWgQv1qiJwPSANBhCGpgiQBeN7YtHk\n/rC7/bov9r7qC7hrUA6+enYcvv3rCHwS0ieJyQbLCjMG460mDOqYggfmrsfADsnYpKBtevbmDNxI\n7sb02/vjhtt/j1VvTcHuYyUCMkQQwMr5U3B4yTNY9cqUyK+XBIl6Jwt7VAcYUrtEtqpImwUIj7ui\nuCHsJK7U9vwlUEEbUVZSLHhO72ADywIBlsDzU/qFR635+g2jpx7OskKk2oPtGMlYt8GIKqcHs5ec\nhDGlNYhjm2TH0gMsq151AMCE7pCU7sjFlZGv9hXh4eee1fZdigAraxtQaWAwzGNUrDQ9dnIH7lqz\nBK7qOlzYtR/LZ4/ChV37hSRLBiQI5NAUrncb8fXzszG5Ux/kT31M344paKSU7BPak0bYweDY/iZb\nEvHnJ94+FZ8gafNFGUjEUQZUOLiBCBaMyykb36JWmerXOhkHL9VI1okE/JYwSRIgCSDg8wpbyWK9\nk1wsjMEIyhorafmpiswBsDEpCAQCWDpndPDmguEPXURwueV5RZlbwK/vKloe//Pk6diRI+jdp69A\nz6SmbVKqOMVFGUGSBCbNzEOcpelOXky4Cg7uR8/+gwTrR5JppwQ1EbjeihRniCmuOMVZjHgs96Cu\ni73/XD6ee+JhQVadnKC61uHF/rO1YFlg/9nacGvv89ljQBlI3D9bKsZOijHD5K7ESV9/TLt7DJ58\nazX6Tn0H97y8XLDtlPhoDO2VjfvnbMDQXtlIiZevQOlB7fmTQPVZ7QV5+6glJtebAQi0XDROckYW\nykuDOWfc+Q1AcG5WV1Uprk8D8DvrFe+iaa8nqO3gwG91BPyoryjFygV3NVWbREJxHx0Ij8+rYe9P\nJ9BeMo0nBFcZKXn1RXhcLtCF51ukgsOwLD6prwfFAn19RtmgZH6lqe2wgbCmJWP9I89iUbcb8M2d\nU7XfJFS1IkCgm49C13ofFtrrsanKLlhGQpLkNFK85ZQ8ocYYrNhIuwQ+ZGWvvYTCp6cBQJP/E0mq\n+kGNzkjGl7tPczujGt+iVJkiCAIp0VGotjl15dyFwS0r4x3Wp30m9v20U0iWAEE4cCCzt5QMyWmi\ntDyjDEZ06d4TE/74mWASFdAfzRIGbzDDexl+fVfxy+F/njydPnUSSW07NbUwLEbd7QwOXIWJYdhg\nErZb+U6eooyg0zoInpPLI9MLs4HA/gv1iiJwLVsCMZQqTsW1Lo01g7AOnoiD5+vDWXX5hcqCanFL\ni3u850y1rBi71uGFJa0Tcl+5A7uPlaCqvlG2/Vbd0Ijdx0pCovAS2WX0gF/B+mBqL10VLDltFl/U\nThDBSbuWtldzOe2w1ynftZstVrjdLkm7jo9VX65QXD8YzxI8pyK+EADApTMIbFuq6AxdeKkaHZK1\nt3fE5kS/eA3Pm1BlZOf+i+hEGuWJA5+A6Jhq87Is3rfVo63fgBxa+UbEVVUrqDSN+fgtPHZiO8Z8\n8k/tlqGoagWCQAxL4nq3EReNASyrawALyArJJa255CThcpCPmTESBAYZzPiK375jWYBlJXmAiu08\ngkCnmQuQNHY82FsfBGmNVWzNhd5A0ZV8dKc2WH3Bp51zBwAmi3D4gJeP6K+rAHxuDJn0OPY5LfD7\naAER0hMOLKmgajiNI+BHry4dMXmYtWkSlSDlHcd1gPLZYWtww6vh0XYV/z/4n68Hts7Mgtka09TC\ncAdPVD3tDD44g0wDScDLIz9iUW7fwUNk/Z20BOFyuKFtIlZ88j4euf8xfLK3RFYELs5CUzLG5MC3\nJFg2dwweyz2omzgBQOUFmySrTglyrTzOWVxOjF1deBT3zjwqEHuLQRDBypNYFN4c8CtYS+eMBlu1\nGmxKDhhbGQwJWYrrcftf5/QKMvce+2gPFj0afPzivL/jcNINulq1Ab8PLMuAMilXXPYXVSCq8gQG\njx6vuq1IfMj4CDqMs5FndPGhkFcGAAd+PoT+mSmam3DQAcQa9f1snWL8uM0goxkTCagBqGbR7ay0\n40AUjUFeI6IVJg75+GbctLC26bGTOzBl7kYsnz0K1tQk1bYdv2rFX54EgX5eI8oNASzyubCkZ1fZ\nTDt+aw4sq1tw3ok04XjAiXqfH4mhiUhx+85bekHRD4rTSe345DBS9h/FbWAVW3NaSEtNgc0OzZw7\nduA4GJMzACJY7eeWJQ6shT/0mn/QOMQnZWDF+gI89RQFQ9lRGPjbEoUDywrDI8zDy6Fs+Pr4flC3\n3yLItwNPJ6hbPE6QYFn2qmj8vxRXZOWpJU+mUWPGAhC21/T4NMkhI8aMDknR4XZfJKJcAOEgVz1t\nuyiKBOOsw/d7KsNtOjkRuJotAYeas8fD7SGxoDkS4sSBn1WnZiegtr6YONUUHUVqQrSi2BsQVopW\nzp+CGlvziRN3iokrWDVFx8DUngN96ajisXH7L27h8S0bhvbtDqO9VH0fEKxSHThaiNKDO1WXtSSk\noKG6UuOY5FvJ4teV1mVZZZ8nyaoRxGAAQLnDhdZx6uJ4j5+GWae1AhDSUJGkpFIjrtKoZdGtr7bh\niJnGdW59xAkAwLJwVdVKqlCqeqfQB6i2fKuAAb3qA3jxj3/Gwr8MkbQhxSafkRhxjqKsWHHwguA5\nsTBcyQ+KI1rfvDUFZ48cBuNyqIjG1RD8DLITYjH/gQ7KOXeDxsGY0gqTZq0HTTPC4QOTBcbkjGDr\nLikD/toKTL2zP2rKLspuSyt2RRYKjuRA6DsUqjjxq02U3xmRzxNX3Y3S7d5+Fb82rsjKU2xclGLs\nyuWAfxeu944caBKXy1kZRHKXf0PbRKRGm0BRJMptHlXPHg/N4Os132PZO3/RzLGTq0hxcDfUoL6k\nEGibEX7uhdUFqiP0SuAiWDhw4mmu8vLooj26JyJZloG/cAuMnUai9uwxrJw/BUN7ZWP3sRLcO3O5\n7HbElaLmVp44Esa9X79p76Cqvmk7DUwCErw2PNbbh+n33x6uKiVGq9srFJU7sL+oBrnzxqLR5cau\nQ2/hCLJkq098i4hDJZ0x87XXNfaZgNbIwV2Tp4X/L3cuqrUSCRDh1ymfHayfFNxBs7xWhDgOI7ie\n+r6xgHBknvfOXKTH5p0F6BmnL6bC5qcRTZACosSvwAiqNIAsyVhdY0MdFRSGy+mb9ICrQmkRp7u+\nW4y2wwbiwq79WNR9OFyV8i3YaJaA4dudmLZ1GP5gjIKVEAYy8iNiTj33krbxZ4hMorYO8QSJg/ml\n6D8oK7w9gf+Tipt72WsvoTIuHimF57H3bBSuua5HRBUnTidlye6IEccO4sv5z+PpyWOki5ksMCZl\n4GhRTZDA15SDProRRIg4CVp3odifwcZ6/LzkLYyfNEn+rXVaEShBfL6nJMShuroaicaYyKpNnBeU\nqLrr913VPP034oqsPD3z1jbFUOBfEnKVI058mxFjliVK4rt8rmUnri5xonGSJDB5Zh5a6RCOHzp7\nEZ8cs+PjvSWqonC5ihSHiwe2w5bSW7J8OFYF+tyw5aDXiVsO/tMbYcjojpqiY7oF4C2ldRK/nxyn\nMMa0BpwVmPinTzGoYwo+fXyorPu5WNf1Qu4hBBgWjyzYhlbxFsSZ5EkFv33aLzsRRl0VF/VlrDHy\nxEOPro9hGRj47vW8iwFtigMb3xre9O6yolpHoxsxFm1jWSmEwuPTDje6xuqzbthw8CL6kiZpBaau\nHqbkJEGVRk4TtabGBhvJoL9XhTjxLAkUEapCqS0vFpmHTziF5aMYYGB9AB/ZG0DLnZyceHz7OnR9\nSyXsVyQyv56y4qeARrVdyUAzRKyGJsdjd23ko/l8c83WvfrD7gndhIiF4yFy1LtDMvy15SD2fwf4\n3ALtkzj2p3tGAk6VKg9DXBZkzvd+3TujYNemiFzFBTpCXnXX46VbXB95FS2DK5I8vfPsiGY5ictB\nbxiw3ASeuC1X4fTKtkPEd/lybuKcaJxhWOTOH4tyjZZho60B1th4OLy0rChcz4Qdy7LwOBpARct7\n63DVDzU3bIb2wXZkk6TqBOh34haDLjsEMjYDdaG770hI0b0zl8tO4EUCPe9X3dCInteOQe+EYuw9\nWYa+7ZNkSaK4BVljb/pM2vQagtJjewXb5ciquH2KqBj4nDbVv6kpygKfR9shnQ/+eZ2WlhbcB5k/\nNBMIgJQjWaG75K+3FoKKjoM3pZMkE6yqrh5pnMhbZorKRwdkCZzYrZoxmXQ7olexAaSHHMXD5OjB\nx5vIwmcfwlcXquyybJhUAcAP1TbUGBj08ancNMiIu1XBW/6e9bmC5WXbe/zl83Il249iCfTzUvjI\nVi+5uJqSk5A6ZCAmz92I1CEDFcXw4vZldHISWhEG7NtfKk+QeNEschN3QNC2gGZZ+AORmUKKLQwy\nLEZcbDVQVjgeJkf534UORDRlF5MoaNGRJAGGZZVJSIQtZgFkROTdO7TF8aLi4Os6K05iQTnls8Pv\n9SLKTF31efovxRVJnhx2j66WXUtVppS0S3JtOaXW3IWzRTh75qSqLcGOC/X44Uw1vjsR1K4oxbUA\nQGXJOdCtewGQ2hTEmikJmSIBZIj0OXXFp5DcvqvicfOrH33bJCDRakSS1SioRNmPbYUlu6fiNrSC\ngMVgHBVgHJWodwlPTb2kSE0TFQn0vN99c77A16ej8OiHuyIiidxnMnddLcAEwp+nmKy+yPODSm7b\nBYOj6lWnJnN69oPHpf/Yxef1PfdNkgRscwgwDEiDTCQKy8Dv9WHiTZ1wtKgGMMeAssYJMsEuHd2P\n1LhoxViW0poGZKVIKyz8C2rJoX1IYPS16R1+Glb+lz/UupIziwRBwJSSHCZV7MvPo9TIoJ9X/cZD\nYkmgYH4pXr6guB7Z1/bHPeuWC36gvhk3TWBnwN9+9tD+uCePt3yoIhXPkGjnN+DzetGNC8vCTwc9\n1Pw0o+ge7KurR93hY4KW5RDKgnOjbpAlSBIDzfgEWZI1ODEWP2w9ovp5SEHA/cPnYZ3U9V1z8NOR\n4wLbAeHOC4XfXKvO5/GCGnav5Bxrm5qI4jJp0LaWb5MeiHVTZpMRtNz1SWnKTk5HeNXn6b8eVyR5\n0lPGVLoIcPD7/Ti4Pz/8WO2OVk27VGr34Gxdo6a4fO9PW1FnTNS0JeAeiwmWuIWX06s/4jLaAJCK\nwlkgTKY6pcYgzkzh5Zs74bkRHTDr5k7hP3p5wV5UWeUddgGpeHzGqC748uEhyH1wMF4f3xOM14WA\ny4E6Gy/ORSSi1pq444NlAvCf3QEbmSF9rYVIkV7oeT+WBWpdAGGyRkQSWRaoc3qRGh+FD158LEyW\nEkVkNZ6nO7MZUxFvZGR9vDhUW7LC1SM9EJ/XgDRgm7t2MoEADHLkCQDls4HxedG7QzIoisSkWXmC\nTLAaeyOSk5IkHjwcikxt0OuR50OeQKTAXJETHv/wwnPorWVREML6gxfRj5S2iOVaeFx7K3XIQNw9\nYxV+Pn8SI2LTNTVOEYnBQ8uX5h9B744pmDQzD1mDRW7j/PYet/y+I1gxfyyOFtU0uZOLKl6tGQoW\nlsA3NU0Eyldbh4b9B0EyDBr2H5TXPIVadkl9e6Hu8LHglCEAa3IS2vfrg7uey5VYEogn8Fr9eaYs\nyeoaa8UpRySDJlx7dh4st00FQCCNYlB6+qSsE73sFg6sBb1rJUxRZtlz7PoeOdj0ww/ClUwWdd+m\nSKChm9Ky95C0+K76PP3X44okT1ogCOlFQIyLJRdQVRms8OgJ/FWaUMqKixJM2CnB5XQiKuR3suNC\nvWqIq5hgDc6MV61CAUFR+Cs/nsHHe0vCZCp3/lgEAgx+PzALcRZj2MAzLdaMWDOFDtffEXYUVwLn\nhr1g42n0bZOAybPyQJIE+rZJgO/UdniTghl9BAFBxApf+6N34o4gDTD1miC5Q5OLWiEIIC0xWjGC\n5ddEdeHRiEgiJ6TfNPsW9M6Mx/1zmqJvlKJb7LQBnQePUJya5Fq3aueJnEUG/7zmB2x7fTTMhqYb\nEIokQFoTFH/8Dc5qGEoOgnbaJD449U43kmKj5WNZTBaUu7x4ZcVZWLI7Ivp3z4jMFYOeQCUuD7IV\nIlnEqGQDiiHAR//wBPaOn4yj06YLKlF+mkG/+DOYOGQE3NX6jDTF1SItDdSqsVNQ8vNB5M4bgwBL\nYMwn/4Q1PVVxnVW3Bpfv2S4xTNDkKl6d/RRqDSy2VjukppgPPi7btuMfe1LfXuEpQ19tHUa064QR\n2TUSSwIgNHH3zIMwZ+cgtlNn2Ny0hGQRJImk2FjUuvRNKyuFCQdOH4Bn02eK3mASOOsVo3/SE2JR\nY2s6/71p3RDI7A2/n+f/1IKwWsxwuYLvH8yDNONoUY26z5OoxXfV5+m/G79J8sS/CChpoy4UFyMq\nNTMiOwFxS07vuuVlF5GRKfQF0vJz4gjWvjKbLvdxsSj8y8OXwLDA029tRXaiFQ43HTbwvKN7Ol6+\npTOeGd1P9v6aAMLtOU48zlWhcueNBcOw2FtYjoDHBdKSICADA3KSQRDAwA7J6JgRC5IEFj8xFJvn\nyMetiFFzTjgyLBe1wj138LNncGzpn/Ddggd+9eGBy0FYSD97PSgDiaVzmsiSWnQLnyCLwbVu1c6T\nmksl+HnbZsnz/PPaQwfg8dEwmygYSBIBJhiZYiQJvL70gPqPP2c+KMoSa4hvi+TbHgm+lziWxedG\nbUkxVi24C97yEliy2imaK8pP4wnhoGlY1ATSn36Aa1bnovfihfDV1YcrUV99uAj1S1Yj7x6dUSiA\nRAyuqYFiWfww9RmwAQZT529A1tBBmH56Jx4+sVN+HZbFqrFTBARNqeI1wGeEc8INGLjx2yZTTK6y\nJjLUBJQjWwDg3JPP4tx/Pse5+S/IHoMhOhZRcTGYNDMPCTFmOM+dbSJZIV3Ug58ux2aqDbSGGADl\nyJauqQnYu2Gr5vp8hPVQxzbJvs6yrEDkbTQZQbuDpEq2fdfMalT7zFYoLikJtd+CQdq9O6YI7D2k\nOy/9Xl0Vi//34jdJnoDgRcDm9ipqo0ouFKN1m+xmmwZSJAGaYeHQsW7+TzsQ22WQ7GtK4Awzldp8\n+1VsDIAgkWr0+LFwxkg0evx4ZdMZ/GPbWfxr5/kmfVRKNBJEk3Sc7obfnuN+/l5YXYDffbIHUz7b\nh/LSEiyd9zQ+nj4EybFBMvDUm1tgMpL4/ez1MFIkVj07HNvnjsaAjsk4ca4WQzqnqlagqguPCveF\nALpkp0om7bhpOC4CZkjPNlj92v8vgRLvuxr4Qvq9Z6oFZEktukVuapIbDOCqjWou9QbKCIfdprpv\nXNV2+uubYaSCf0vKQMLrp/HSg4Plf/zFP/oBf5OWJL07WLMVD8zfHGylKIBmAHPrdvC53ZKLaCQX\nkOCUnfw5Jqd5Ovrg49h83Wh8+eqbaF0TuacZB10aqFDI44Vd+7Fs1v+x995xUtT3//jzPTPby5W9\nOw4O7ugdbKBo7FggaNQkFkBRE0ViSyKJ4kcBwSQGE5Pv158VNYlISawYoocFu1EEqQrS68G1vbJ7\n26d8/9id2SnvmZ09yCfij+fjkUdkdnZmdnZu5zmv1/P1fF4EjmOylVyWmL9H184DKBUvAP7KEH41\ndw4unPJr5bNRNV4qmEW2QJJwckrCii+1vk8yUgf3IRmJZh9OI104MPcu5TVZF3XXU5vQxToo7uJ0\n0HyhTqkJ4cuG4rPupFEX5LV1qtZdr/ISHGoOZ0Xe8ajSEuU8fnC+kqJz7KwwoLYGuzd8rtMzpTWR\nLWp0y7X/OP6r+M6SJ0Az8WtArKsLPn9WW1GsKabc5htQ5kXA7UA0lbF8b2d7GMFya/dkdaVAP41H\na/MVmqbzuzj43NlWnc/tgNfFoTGa0uijaEG/skhc3Z6TCZYEoC2egQTg8vNOw+znt2PswApIErBm\nZyse+9X5aIumFKHq7X98D2X+/FMXn5vAoREoGnF68cGpWPXoTejoSmom3+RpODkOZ9POVowZVnNE\nGXZHA0LHAds3+ulPf44L5r+De5asL9pPS/7u9VOWz63eb9kOdro8SCbMtSNt4TDS6QySaR5P3TMe\n6YygVG/TgggpETGMXfPOIERPKaSASmulHt/2BiDkpu+o2hWnB1xJGQhDMHlOPZweDzqeX6C5iTZ1\nJVDpslcBaBQF9DJp2VGrLZKE53ftwQnpIxPlFtRAqSpTAPD08HOw96PVWDp/AkRBsqWbUkAhVPHm\nMDJ7DuEX156Lfy1aSnUdN2ifLDygejMcDlkI9HffOhW77vb4VWAAACAASURBVL4Nu2dco7Ev4COd\niO7YhqUPXARnMo5Iu90sQWNkS8DlRDRl82+Dlm9XWaMZUBjdrxfWvp+tSLmatoCPRTB6QAh8LAI+\n1llcjl0B9O/TE3sOHgaQ1zMBEp0gdTO+5Tj+u/jOy/jdHAu3kzOaauoYVTEVJ7lVt+zBiZixYBWe\nvHs8OCZluo0fT/spdkbNt392XRmqg240RpKaNt2S+RM1FSgAEAQeDTu/wU3nnI6BlX7sbOnCs6v3\nGywIrZzFZdPMj7dqHanlMyK353hBxKaGTsPNXW7hqafL1JEk5X4Xfj/1JDz2q/PR3pXG0vkTkeFF\nfLkrjIemnKQxzRQSETBu49OW3vBy/J3P4pt9LcrrV81egqoyHz547BaMHliB1s74ETmKdwdyFIws\nLJcSnRAyCXCVg229Xz4Xn359EP/z/Bvw1I4w3xeAfY0R9K0O4qbTajGw0o894Rj6hXxK9I7PxWH7\n15tQO4Q+/ej0eJBMmFdX3nv3bZx93vlwOHogJQiQJCDBq9reuXgWpfKU+9G/atZrGN+3JHuDETKG\n8W1yeFu2lUIT/aYT4DvbIYkSls2fiMS+7eBbD0Ntjrn2i+3o6yscFBvnBbhpT0o5I8h0uC2rAVJF\nlfCShBgjISge+Q3r1ctvRGjoAIS37DC8po9egSRlDTSrQlQy1N39//iNF/DXt/6JW557HFtuuj2v\nebJD6lXnCcgSqDVrDmLsWEoUkSgidXCf0qYLDhuByNavAQCBQUMQ3b4Ndes+xapUGpddcFK3P5OL\ny1Y9XRaRPNIpl8JR0UsxxZR1Tzwv4rp5bynRLYN6VeDdDdvz227ako1aARTCpMSu6K5hszgWM3As\nC0Fn12Aaa6SbtrMd33Ic/1V8pymulXD8e2ee3a1tqtt8ybSAJ+8eX7Dd5/GaV0T01gUAqG06uTLV\nuHcXEu2tGmsC2uQVYK6RyaSTVOIkj8kDwDXPfo4pf/2Cqr0BgBv+7yea6TJZMC3//y0LP8eGPW0I\neBxYtyuMCx98F/csWa8xzfQnG+Bu30pteem9ltTESd6fKAJBnwszFqxCqd/9v1p5ommx2qM8hCZ7\nMQ9qA9EzhtdAbPwmv21ojUnV3835ZTHlu+8X8mFPa0xDkLd9+Zlp9cvhdCGTNvc1c7ncSCWzFVR5\nE+pN8ZxH++Sc+9F/8tdnw0GgmTgyxF6YTEvxggjxm7WIPD0HbU/cl6s4ac0x9yVSqLMhFn933QGM\nZHRxFjojSEAbkPtyWycGH2HVSd7PD5f/FdM+Xk7VL1ErU5KEeFPrUSFOAOCtKEOfU0/Ely098Vnj\nAaVNN/SPv6XqnvTHL5+nE5c8CxCCkxgnNojWQxB6+wL5vwODh2Bor2rs7CrOd0yP0dUhfPTup+Yr\n6D2ecvl2/PuLILUf1lQ8HSwLUUdMUqGB+dYcJceO3b8OrvDObh27Ju7IJNZIRjGGmv9b4EN14CsH\nHL3/her+2x/pqOI7Q57MfhPMhOMnnNT9pyG5zberPd6tDDzN8VE0Tfo2nbqNt/+bTWj29i6YVweY\nO4t/89bfDcv0nk4izLU3QGELgjKfSzGOPKFfubK+rPX5bOthjOZ2YfXieQr50KOQ15JMsB771XkF\nzTNpE3tHAqrrOWFAOBekdGHtjPpcrN0VRjKTy4iD0ZhU/d18vfojhTDtaY3h6c/3aQiyN1CCWIQe\noWEFjiFwud1IJenfKcMwYFjO0Frg0hEkwocRSFNahbmbkVXRozUSQ5nHCTEeUdo2+umrNOeAx8Qm\nQY0DIo9akiNChMAZKrfU/UiShFZWRKXEFnYKLwA7mieaVskAs4k9G27mMkF7/dGb8fm77yPVGi6o\ne5Ihr7d5bztCY07EiYufgZthkbIyl4TRvkD931KkE0daQzmhZzk2HjYnl9KoCwBC8MLci7Vt4XTC\n4DRugI3WnIZcFYmSgA8dnXmNYUGCdLzidEzhO9G2o7Xm5GXpjHnfXhZ9dwfy++y8nzYirsZH+9qV\n9pwMdcVJ3cZLdLYhODJkmVdnBVHgwVDsCfSeTsXqcPSguYsTkmsNEuCzD1bizlt+gmnz3jbNobPj\ntXTV7CUFM+z0WXXqbDx9680uzFzII0IQwX2fwzno/ILbkFud4WgKLocbYjqJUGlAIUqL501QMgbl\n72bObzdg4ef7MP30OvSr8OGm02rxrKqyGCyvQLStFf4S+k3yR9fdaFjWO+hG0O3ADpcTPG/2vRPw\n6TS1tZBMJuBydS/AtKWzC5U+rc2HYfoqae/hhCD3tM8wOPGFhYqHkZnup741gl4shyvfXILep56A\nfZ+uxauX3WCvxaXZceFQXwCF23O6jDvlWMyWU6Bk6bU1oV4i+D4htgKC0+E2tG3YjNFjTsTk2fVK\nDmCv5gTWrW3AKbTWXQ5yrh3f2QEQkv9vAJUuBxqjcVTbjNXRw8WxSJu5lctVp9n12fbatk8oH0xb\n+XKwLFLpDFxOR+HWnIpcLZs/Id+WtonRQwbgq0/exZmTfpRvdx8nSN8ZHLOVJ7WHkLo1xzDaZU4H\nixkLVhnadoQwBb2dZBSKhLAbGWEFM+sCfWVKJmsSgK4UXzCCRY/2AztR2odujGk1Ji8j8tUHSLfY\nIxp648hyvwtjBlbg2tlvIt1xCBHJf8Q5dHYIllk2Hq31VgxolTHJ6YeUikASC5NadfXOVVWHVPNe\nUxIrfzfrD0XhcRBF66Rv2zakXYi2m9+g3R7tTUyt4fO6XeAF+nETArB8HFKszfDknEql4bYgT1bn\ndfemLaik6Jnk6avOfzxqy6KgM8PDT7J//Ce+sBChMSeiM8kjNOZEMA4HPjnn+4apsn0OETNfW4La\nM0627RRO+3B6IbhlZckCZtUry6qWviKVI2i1PIt9juyDo+lUnQ4brr0Z4bUbNERrFOPEJrFAhJU6\nMFgXHjy6xI/3/r1FtTLRmKAaYXzd53QgQvOMsuEsrkddVRn2HcpLFgztZR300SvFYNTg/ti4bdfx\nSbrvKI5J8uRyOxTzPrWnUzojIOh2wcWymmVP3TNe07YjBLa9nfoUMNC0Y7BpF2YeTnIb7+2tB+DK\n6adoeXZ20LrzKxxm6SPjVmPy8j7PrBLx3ryLDZ5NNCNMfWtPrkZdfyaLmqGn4op7Fynk42i31dRo\n7YxhzdYGA1GzGzhsBjPi5hw0vuhj7EyHkGraA4BOYuXvxh0sR0tzs6FtK0/geUrKEWlroe+EArWG\nz+sPwuHQkiD1d8yzHhBfueEmkEqnrAW9FoWctngK5VQ9U3b6Spm0MwukzWH1+gb0ZTg4y8tQfuIo\nxYNoxoJVKD9xpOEgeEJQ0qcn+o47GZt2tmLZgxNx8IuN9vVHOdJSbKiv2XYA84k900k+C28pNveL\nIEqS5VSdBpKEDdferCFafsIgfgTVkhq3E4cSMvnSatmMHlD010dUlWL1h6tBg9pZfMpckygXFQb1\nqsRX/84SXaVNR6k4yTYFACzJlRU8Lid4Uer+JN3xqbtvNY7Jb8fl0orAk7yASDIFp4NVlqeErM9T\nPMMb/J4kCba8nfqUuBH0mJMsOyaZy5570tZnooUFq5HkRQi8gOGnnQPAmGfnz42uF6pEZRJdYN3W\nJEEvWJbhFaIYPajOEIArm2TqncU128yRq+lPf47ffZjErL9+oZCPI60AWX6W3LbHDqvB2q0NuGp2\nvkpUTOBwMWjdvwfEZFze9DidPnj7nQDAmsR6yiqQ6AzjudX78ecPd+GZ1fs1RPrW8SegvEdNUfuW\nNXyBvkMxZGhe26GOOOIYAqfbTXVJTqczcDrMR7mtvs/2RAplHvOq1ZYvd6GHx4Xe9y+wDKRtkHj0\nJpxmPD+dyvqcJSJd+RBgAGAYiA8/gJsf+yPSvIiRfcuw/7N1eGniFPMD1X0gmbRMePZPlqG+BvG4\nmlRR1jPTRdGWF9JZ9eRZrGwtQoCsm7aT4SMMOjPdiwghhCiTwGZO4ijwev/yIPa0RWGKnLP4ojkX\nA4RkdVAm6F8dwu7GsKmHk7yc86t8n44EorVQ3Axytcpt8bdxHP9dHJPkKZXKR0jID3qiaBSH0yaG\nAOD9Ve8U9HbiGIKAy6E8lUYpJMuOwaYgFA4ztQoLVsMXLME+PlvS1lsRdKX4gpUoSZLgCxlz49Sg\nCZZlNG7bCH/NcEMArnpyTE2qlG2qyNXCW8ahM8Oisiz/w1lZ6sPpI2tBCMHpI2ttV6DsVKvU1SWa\nF5TdwOFi99sdOMvNTSRlHMwE4XC48NPTavHLcwbg5tNqNUR6eO8Qhp80puh9669dfTvcwRI89Pwa\nqkuy2LoPDpPKkyRJlpWnjCDCyXKmrZyWdAaj77oPwWHD84G0qigQGQlJgidH6Dbd+DOsvnwKHCyD\nGQtWwRP0a4TSvoH9sWX7VvyfFWG43Q68dNkNeGlCAeKkIj160rLy5pmmob4aUqMjS96qkP1wYRNv\nJyudVW+ewUHOZtVIP5WoInwjGSfeW3/Q3nYoKHVwaIunTJ3EZZi9XuZxoi1hPfVHNr8LjiXZCo9F\n9cnBshAISxeKsw7AHYAgAZmMiGXzJ3arXXfEUPk+ud2cdmrvOL41ODbJUzKjREiog38LuYoD2afp\nPdu3oXfQbSn25kUJ0VQGowZUIJLM4IAJySpEwuyYJhYKC6aB6P6fVonSrx90O9DvjAmW29VP3akd\nyNPth/HAm42GAFyaOFwNPbl6+sYTNVUmSQI4Nu9mbdOSxla1qlB1qdjAYf1+q8p8mtdkUlWM43gx\ncAQrUdN/kOa7BqAh0nauHzvgBRGL500AL4hIp3ncf+NpWZdkneYpk+FNK08ZXoDTJJw7C+tWTjth\n0f+00/MPMdu3GfLWoH+XJCG2Yxda16zHkzPP1QqlCUH/e2dCFEUsnj8R0XAnDn6yxrA97ca1pCfe\n0qYlLTq7ATNSQ2vxadZraSsc8aKD1QQfBwKB2BO/W03l9SIsDtnQ8JlhSMCDf3+WbXtpncSN+iaa\n0zghhWKaodE/KVN3Zu07gTfVMnEcg9v+8B4cDgZsw8ZutevUIAxbfNtOZWuQTPLHI1q+pTgmp+0I\nIcpT8bIHJyqGfoAynEK9ActP069+sBO/vMsBrsvc2LJ30I2Ay4GoBXGS0d2JPTVoE3dWUGeZLZ4/\nEQBM7Qvkts7ASj827G/Hxt1hg6mmDOupOwkAoVoUqCfH9FCTqw172jB2WI1ifilPun28cS9eeOBi\nfLxxry0yozfRtJqYu2r2ElSW+ooeoiq036XzJ2L93+7Evzfvx9VzluAf87UTfcVCtiUoNOlIM0BV\nT1+OsQiQXr/63/AFAhg8fJSyjDZ1KklZ8pSdYhWQyPDw8DFwaWO8S5rn4TAhSPFkEm6nRUvP4VRa\nNcvmXYxEzhhTRjIeR3rPLowaNgKRb7bg4IN3G7YhSBL16ZxmEOksL0O4xIu3NyUwU5KweNz3TY9N\nht7g0ltZnp9qU1d75OqUbICpe51GqtRGmbT9FNRgFZjg84sEYUlAiFhbPVDdyHVtPMnkPFNBCLjc\n1F1/nwevNsg6PNlJPEuaPbUDkdi/E7GXHkf298XoNG4X5MsV4J0ekHTCYJyph5y9qJmwEzLI5Fq9\nmXQGrIk3WTFwcgy6opGiDTC5dARShkEy4wFzFAaSjuPo45gkT5IkGVp08t+0izVxFEf2N7QzGsPk\nCSMstU5qLdPS+ROpJMuOzYEkZODgtDcOK4JUTMWgK8UbDBLN7AvUVSn1+LsZZi3/irpO2WmXo/kg\n3cOokO+Tmlw9cf0opRLU2hlDZanPluWAGvqKUiE8PvNyqlWBGcwsDPL7zVZkps3LkrfBfSoNZE7i\ns+eDcIUNHgkB5k8YilMHVWDDgQ7MWv6VKcEFYPiuzTy99DgQE9CH5K0zZJuCSDIfMSQ/fCR5QfNg\nQkyqGFaVp0QyDa8FeZIyKctWDiG6UXgKmpNpVJg80Q/9429RMfYktK5Zj003/gzpcBtW/v0VPP/7\nm4ytLkKohMVKyK1+7w9f/xt6nzEWHEew96M1WDl9JrxVIc16r152A7w9KuApz4vfJzzziGJDUNDu\noEjUZVi82R7BdSb+Tmpo3NdzbTz53H1y3Q04nEyjl6fwtax3HW946D5kdL+Van0TjTR3G7mKk2yc\nKbuLK5YFTg/cTg6JZAoe9YxPrnXnUD+U9xiedSE/ApQFA4g2HYCPVBfvHH7c1uBbjWOybQdoW3Sy\nsNXndIBjs2GbRmuC7P83hcNwen2WxpaFtEx2Jux6B90o4WMYWJv3Rzm3b7mlKNwu5EpSvwof9oRj\neG71fgRcnOkNVE207Hg4qQXLavE4w3VfvChJQEtLKyRRUHRGV81eorTAXnxwatHxKlfNXoKTbngU\nACzbd8VM1RECVJX5TFuC+f+W0BlLahzQ9e3B8DerwTesN92PejpRbm3S2qU02CVLerg8PiRi2fOs\nH3iQhAzinR2KSBxQ6QYttpmxqDwlkkl4CuTSZVs1c5B443n6Crrxdz02b25ED2J8DjRrRX268Hm8\nft5V2laXlcgbhQ0u5aoRwxJMmbMSfc8+FTO++Qg3bflYuz1CcO2//4UbPluB6btXw9ujwlI/pT4+\nb1XI/hRfDkGJQZSxWXJVTeXpz92I8gp8uumQ8T2UKUi96zgXLDFcP4X0T3q4OBaJtE3tEa2Fh2yM\nC3feNNSMOR8HGvMTqYp4PDRQGxjsDRSdaadHeUkAbe3tdCJ0fJrumMYx/e3JFSe5hUcIAccyueiU\nvJhcPTUUbmlFRUVVQW8mMy2TnQk7eZ27n/kK1025Bm6Owbl9y1AddBUUhZuh9dB+HNqdzWVSV5L6\nhXyYPq4O9184GDNOr6MM/2qJ1qzc+LvZRJ3+vWrxeNO+4p2r1cjs/hCtOzcrOqOjYRUgSTDdhqxB\nammP4oVXVuLqU1J4f+12anVLTZrW/+1OnD6yFtPm5bcpb0t9zKV+N8bf+awiNleTwspSHyRnCcSu\nZuq+9NOJcmtz0pC2IzYp3fO1kbCVuLPkgnM4kU5nK2L6h4TO9nZ8+MEqapwRAJgpTzK8AM6MPKXS\n8NoI9fVMmqbSPTEFvIC0aJIE9KC0pcyCcQmMra6CDuEF2mNydUoUpGwuJC/i6ntfR339G2ggaWQq\ngpAgITR0AAKhEkyeXY9AqASe8lJL/VT2gPPEbvquz4vSRAEAJxFkiuxZ689daUcUrZJOS5qrMOmn\nIPWu43xnBziGIK0bnqHpm8xQ7ffgcJv9yUGDu7iqGlU7dCT2bt6QXa5zGXe17tAEBlPF4kUQqvLS\nIDr2GKtXx72fjn0ck207NdQ+T7wgIpnWthoYBhp9VI/qHuhR3RP9KvyaVgUNcsVJbtHJ/19owk69\nzuHc9qsCbkX02hhJFS3qbW3Yj/1xDuV9KzWalz3hGPpV+PD1rlaMHliBW8bV4enP9ylPejLRuuae\nl/DnX5yJUq8DHfEMfn/5SJzYp9SyRaQWjy+eN8FU02QbcrBsDkfDKqC1M4aOriSWPThREw5MCPDE\nHeOxe9Mn2N+Uwv2Pv4ZgWSU6wv8CSvpptqF2IJckYNq8t7Bk3kQsmptvLaodymmZe3KbT7/uTXf/\n1qAVUQvoX3ggf16nP/05Ak1r4Q3XGj6nWg/VunsLKvoPNz0nuzZ9ibphJ8Dr5JDkRVw6pBI+F4dY\nisfzjYeQzuRNDw9GkkpbWiIMksk0Nc7ICoIggKO41gNAIpmy1DwBxhYOueYOuGv6Ib5vB/D6Zxr9\nTP6E5Jd1SSL8JsROHwSclEQ4JJ11QK5Vd6QtM1m/JIgi4ldfhDHlHtTWTsS2f6/BZy3NiLMSpG2b\nMfjDT7Pi93Anwlt2GPVRuvahTOxufeQDPHXPeEyeXW9fEwWgp8CgPhzBDyqMU4pWUJ87teWADHWF\naekDF2laq/pWa2+PC/vbYxhcW6OqMtnXN1UHvNj+5Wb0n3Su/Q+g1iypqlEHtmzAlqbcQw3FZVwJ\nDKYQp1TVMHC+IPhYxJaYPFQaxObte7QLVdN0hpBgdfD2cXyrccyTJyCvzQC0QnF1RIt8QwhVVCDo\ndlnqmdSQNSGpjACXg1UIV6H3yeusP5z9cWiMJDFyQAUaI0l8sJeSA1YAkbYw3FVDlX+rNS8zTq/D\n6IEVmDw7Kx5Xa2FkojX9wnI0NzZi1kVDsODtbdQIED304vEjIU4SnwahPLEVq3XSo6LEh1K/GzMW\nrMJjvzpP2VYo6EWsaQe+aB2ApQ9dhhWTfo39DS0ACJDcC3+PvpptqEXgi+Zmheu3/nE5WjtjGFKr\n1TOddMOjmik9Nflas7VBI4j3BcuQTscAV96awWw6UZKABC9CH2QhVwBlsjv7oTcR6jfMVLzrdHsw\nttKFAb0q0NqVgs+Vf3go83vQoGt/yNcxy3JIpNPoTKSKEtfzggiWpVdS23d8Db+b3u6Vp4g0LZyD\ne+Hp3Q+T576Fv8w6Gx6f16CfAaBZhsuuNhcy6wwi32uN5vVR+tiTK36C0JD+CG/ZYf5h9boo9b8l\nCeHmFrzLpdDvzy+gV3klXvy/ryLa3IraXJFfgoT7f3Qtpg4cgY6tO5Vj1BtfqqNYZGL3xMxzEQ13\nKgTPLnryDNa7ujEtpzt3+jNMqzCp36v+dx+vGwf7nYpx02foBOLZLTMFNE89/B6sPmis4nYHlaEQ\nWiP53xtXeCcQhkE8boBZXItFbEuoJIi2Dl3FTBcSLGuheGcQDpcru+xbFBB8HHQc0207NdS+ToC2\nned0sIgks/oouwaZQL79NnVuPdxOFj97eJXSqit2wk52Ce8OcQKAaEcY7qAq1BR5zcvTn+3DzpaY\naVDw3zccwqHDDfjt0u04sU8pJMB2jt2s5V/h6mc/x92vbOjWccsQ2/eBKaszLDezCrDrpWQWDtza\nGUfvIWOx9DeX4eN129GsK/l3Ne1FV9Pe7Dbau7D8nc+VaT/Z90muIq169CZ0dOX1Tc3tMc0xq8mX\nbMYpr5vgyiDGjI7f+uia/OcmkETtk6fePsLr80JIm1dM/T4ffAyPqXPqUeF3IZ7zRYuleEjeElxw\n6eXU9zEsA0EQi55KFEURLEP5KXF6EI0nETARGSfSGXhy/lBKC+cf/0chUq3bvoKHZQ36Gb2mhvXY\nd/dvYUVUCdlj1bfqrvzXC5j28XKDRsnU1JJhNP/enEljFZvC0AgDfwaINLUg2tyq2T8BQU2cYMWW\nr6nHZ9Y+lDVXCweMw9PDs0a5dtt3xVgWWMFNCOK6IZyGh+7DjtuvR8Pv/ie/kKKD6l1ZiRZBohhk\nFnIdz6LS50ZrrPsB7Oq2nbtHH4g5E1u13qkgdFUqCBlTs00ZJQEfOruM2aaGkGBVNapoJ/Lj+K/g\nO1F5olkTqNt5yTQP9f3ITuUIyLffFs29GADw6F3nFSRcMuSKVcDJ4aN9WcJ0JP47Is+D4ejtDwnA\n05/vMw0KjqZ47Nh/CP9YcLdClswm6mjbDofbEN32GVB2WrePX2jbg06pHGALi87VlRw703H66hUh\ngCfdhktv/xMqywIG4qRGV9NeQJLwi/t+i7mLzkNrJF9d09shjL/zWaVNp4a+/ag+HpKOoKza6Ddj\nNp3IeksgxDvB+fNEWV8BhCuAVLQTnIvuY0Ocbuxvbld8wz7a144SN4fOJA+GYQwRLDIYhoUomnuk\nSSaycVEUs/lzqifwZM0pcLgc6OxzMgIu3TnLTT9F4in4nQ4w3gDEeFSpPMReehwJrx/hw41wppPU\n6oa8rG3L10AiCXA29HIMA1SVwXMojtCwAQhv2aG06g6u2Yjep56gtQnI+S7JVaCVN8/UWAmEhg5Q\n/n3PD6vREnJjRGPSVBsmozRDcMgtIi1IcOrWNW0fqqtTklS8pcFRQE/CYf36Q/je2D75hXoxP2XS\nDpIERyyKjgP7sex3P9MIxO1O3XEsc2SWMDoRucRnuhX862reCjg9WRsDG+9nGAYws4lQt+dMqlHH\n8e3FMU+e5NYczZpAP2qthp0/RI4haImnUeLxKW2P/ZGEYR2ajUHQ7cDk+9/A0t9MKsq/qbsoNH21\n7mAnJv/lC4UsFcqxU4PvagPnLzIwVQfiCwEJe0LLQh5OehsBfQtt2ewf4ayTB+Pjddtx6e1/snFw\nBHzZcLTvWQeERiiL9aSIRpxk6AmccmzOINjQAFufGwA4fxn4WIeGPAFa+4gegRIkuzrgq6C7xTMs\niy8OtqPR0aRcd53Jwm0blmURClVQX7OiA5IEpCuHQKjpk9WCdOwF4VgIooR4Mgl/TRUgZEfIpVEX\nKP470RV/QeicSSi/4Waq109aEOAgDNWqoOH394Pc/3skSkox4ifTgBdetQ7RY1mc8e93sOHZhbjj\n57+E08EiGu7EwkFnwBsqRbw5jB/+83kNaVE7gC+Ze5HB1FImX7OvqcXzTzyNfjaIk4x+MQYfeFO4\nSDRWzageUiroCZYdeEWCDklAaQG/Jyv0Ihy2ySHBNB0arHVQkU/eQ9uTszXkqJipu6LdjtQWBdD6\nQBHOQdU7FYJe81To/amqYRBLeoJ3Bgu24mRvp+PE6djAMV0b1EdI0KrX3TVGlO0IKr1OJNOyZkpA\nSkWCzCwL5IrVqeXfaBzDaRN2habu5NdHn2We12QHerIkT9vZmbrjo23ojNq3KaCFBDt6n2L7/VZC\n8kLO4p50G846eTCum/cWzjp5MCrLArn3EVSVm0+2RDs6QDJdgM5J2W58S7FO5TQQAnC+/nCW9TRu\nH/nvb1+EIBU1n3zsUdsfHn+waMLucrtxyWX0lh6Q0yhR2gkSYeDwBzVZYLJjfDKZgJeIypi4o7IG\nk+dkA1xjjBvldeZZZxlBhIMhVKsCLhBEYNAQzPjdv9D/1DEaR2wDCMHJy56DK+jHax/ugtPB4u7H\nPkYgVILQ0P7Kano7AjNTS/U6r152A3414ceIP/MGglWVlF0TBKqMhNQjErhFgi95io6wwGSfcqxF\ntO+qBAbvh43to2IQIgzCkmA6ZQcU1kHRyJG9qTsCoK5q+QAAIABJREFUOGx4TMm7yl1v0imXap3G\ncwaaTN8TkKoaBlfzVvvBv7rJPLAO6/fn1n/l/Z1FuYurcTya5duLY5Y8KUZ+OrPMQnir/s2C6+jt\nCPZ1JrAzHMOu9rjpOnrLgoORJNoSGaVlRwv+LRQGrH69rn/hnrxVMHBZ7SDNerIFwYs3nUbNsVOD\n72oD8Zon2muOgTKG3x2YkRa5KqW2EQAAiAK4lo1obovg43Xb8cLcixWtEyEEKx67Czve+AP+9fhM\n0x8koWwo2LZvNMu6E99SbO4dIUBF0IWFt4zD+7+5FH+4agzKLQito6QK3tKQ6XddXTcAgTKtF5Cb\nY4q2x9AeIwHPeqnj1aLAQ0zGVFoQHjyfjXYRBBGM26voTXhezLdO4hFkWg+bVh0EUQJr8l3JN+lf\nXz0MUkOjIcxWDWd5GUpHDMMXXx3AtEtORDoj4OHbz0I03IlzHro/Tz4AA2kx+DvpiY0kgY8lcNtb\nS/Dg3k9x29tLlOuLEIJb31psWC6jT5ygyS0h1h09Uk7kaTcbr0Jg0MoeQUWDELgrQpBg9HFy1Win\nQ6k6KFhxu0JTdwz819wJz5hzs2SoEHL6pilzV4KU9cyTKNVrr7y/M59rV0R2HTXWxez9ucrWj84b\nWFQosLIvZxAlpR643EfmNXUc/xkck+TJ5XYovk128uzU2LXTYpImB5odQUr3FG/HskDWiNCCfwuF\nAatf71nixg+G9TAN/AXyfk5mwcA1o8YpFSZZgHzbH95Dmc9Z0JhRiHeCuOz57piFBOuz3mRfJTOi\nYUZa5KrU83MuBssweOJXl4MQgI3sheDPButeevufMGjSr3HJbY8AACrLAppq1LD+PamVqGhHOwgf\nB8TueSzZzdvTv2fhLePw7twLMWZACNfPX4nRNSVYZkFoOU8Av/7xeMsQaDXOrivDZcN7YNLgym4b\ntDIsA4ZjqYJWURThCe/IP4ELGSAZBUsAJLqU1Ptl8ydAaj+s8d9JfLzCtOogSBKs7NgaHroPX951\nLw7+9hHLY5c9i+rKOfDhTvx/VaPwt9MvxeJx3y9MPgpUgRJEgt/rx6Azx+Da+e9g0Jlj4K/MEld/\nZYi6XAYBweAog/fYlKmezArF2CsckWhcFRrc4/JLwEc6lepSKhZH/4ce1VagCpiaWuyI4u/FwD/l\nF3D3GYDOBJD2lJhn1snLc/qmpfMmgOMYpdIJf5ny2o/OG1hU6K8sDAdgv1KFrEaK6Txsf3pOmQTN\nC8hdJg9Jx/HfxbFJnlzaVl0xrTm7ZdBCgb+F1ukddKPC68LZdWXU4N9CYcDy67Lh3lSTwF8ZVsHA\nMrGSK0wdOQHy478+X2lJRhIZdJhooAjrBLE5/VEoJBjIk4x1f70Tm1/4hSnRMKvi3PbIchACTJlT\nnzexTIYR7cy2JSRJ0ojE1dWojmgcny2ei71v/YlaieKrTgJybtXFVpG6Y/oZCrgwdlAFrsuFIj8/\nZ4LS8jIjtKVeh2UItBpujkHPoBtTZteDYQiqg2689LeFyuuFzGLzIMik04qgVf0ULUpiVhiruhHJ\n7Qwc3pZ9t9q0UHZ8liQwgKrqoL15Ht56AG7aFJ8MSUJXNAq3DTXMpht/hn//cCrW/XkhIIoIb9lx\nVLydIowItiOGHZ+sxeI5F2LPFxuVCbtoc6uyfMcnaw2Td0DWwLJPgsHHSBtes4NCzufdBiHZXEBo\n3ca9varBBUvQ8NB92H3vnXD5vJpJSCuwhIAXzKovtKk7Av81d8Bd0xeTZ9djxOBatOzZrvVvykHT\npkP+esu0NGDZ/AlIJ1PgvncVpFMuBflyBcS9G7MWBXagb9cVC725qAk0xpkqAXmqG0kCx/GfxzFJ\nnlIpeq6dHRSTUG1HVE5bR27pvfL+DqWqJFsVyG08ANRlany0rx2vb21CSyyNJSY2BDJoYbEyAjKx\nUt2QZy3/CrcsXQcHx2DGglUIehymlaeyU22UylWgjeGriYhMMhiGaAiQGlZVnJaOGNao7ABa2rvy\nbzLBpbf/CadfOw+lAS+um/cWKsr8uP2R9zW6KADoamkACOlWFclMqyV2NUOiPOESAjw05SRIYtaV\nevWOFoyf9w7W7W+n2kjI+rT2eMb0uwaAtarrKcmLOBxJYumDEyGKEhojSaTS2Ru1nZih/LESMJmY\ndrw6B0EQ6dE9Qkb7YKO76WnNQ403z7ggwstaC5yTkgS3nS9HkpBq74C++HKk5GN3hodHIHhywnXY\nvXoj+p16gqZF98TF12J23+/h8Yummm6jNEOQYCXEzSpDVpEsNvRRMmTReEGoKk2j//YU0m3titt4\n/FBjtqokSUgd3Geub6LAxTCmHQL11J2sf2O8frhr+inmwj4hhY7P/ml8s8qGwFHRS1OBIl+uAP/p\ni3C6XZrXHTWDEe8xwtRiQAFrFJYX0+YDbD7cU6wKuHQEnR0JpJLdTxs4jv8cjsl6YCqZQVLMTtFZ\nTdvpIf/GdsenqRjwooTORBpXXjBUqSqZTdwVEvUmeREf7WuHm2PwyS7jjyQBFIsCWjAwAXD1ib3A\nEGDR3Is1N+S94bhSgTrSSBA19GP4QniXwXrg35v34/SRtVg6fyI+3rjX0KIzm7iTSY3sp3TV7CUg\n6QgkVwmsHt4lScKWXYeUClRrexcem3ke1QPKav+FQDP9FDsbQIQM2JIazbpyi3PavJVYNHcC7lm8\nHm1d2kqdTAvKvA7cc9EQxSjzudX74TOxptBDvn6A7PVEQEzDr5uaGtGjh3GKjyDnME3RbfC+Skh1\nJyOVThjaGVa8RpSkrMUB6CPrSUGEy8R8U0YKElzFz2EpB3ekY/4pRkKZSOCrKEf/007AtfPfweI5\nF8JfGUK0uRWSJFErTnr062LwqS+FCyUdkaWYZhruxjY/hywav6KA07gzVI7KcWMweU49ls2fCGeo\nXHEbb27cD4zL6y8LhTarIZMnPyWux2zqLrF/J0b3H4jkgV3gVtfjsCRhZK3u+tTZEBB9ZUrVNs60\nHsr+ZlZUY+qcf+HlBT+i655YB1KhgVo3cZrruB3NlFn1VO0mbmJVoE8nOI5vD44aedq7dy9mzZqF\njo4OlJaW4uGHH0ZtrVZIKIoiHnzwQXzyySdgGAY33XQTrrzyym7tT59rt+zBiaa2BECeZDlZFoNt\nRLMcKRqiKUyY8lN8tK8dZ9eVoTroVjx3ikUk3IJdDfuAEu3Iu9yOG1jpx86WLjy7er/hZiq386bM\nrsfieROw4O1tSvUCgG2/pyOBs2OHgYgo+W8FtE36Kg6N1IQ79iCaEAAbU9iy91NLe9TUAyprntm3\nW9ExtM/TEe5AKSRAR57kFueiufkWZyjgwqb3luOPu6uxeN4EDWliSLZSt3jeBPxrW4spcUpGO9B8\noBNVffoCgIY4yTDT7L360ov42e13Kutp2uK0vy3CgDicuHbe2/jHbyYVJcAVVZom2s0zKYrwFiBP\nGUlCt+S0R4mUpBnAKdpr0VnBKRHwBBAhgVGRQbVpJtXTyc7nyKFCKOA0Tkh2alGSkMkJ/jO8qIjT\nqaL8IvRNLpYoSRA0yP5e6jZu4o1FSOQE5T4nh8ZonPpetQ2BndcdvIDH75pEtRjI2hGUAIKIKXNX\nYuk8uoeTnaiWVNUwSCW9DFYFNDfx41YFRhwprzianEOPo9a2mzt3Lq699lqsXLkSU6ZMwezZsw3r\n/POf/8SBAwfwzjvvYNmyZXjsscdw6BAlqdsm7E7bySRrxoJV+MEPLlEm5FxHMH1kB7woFRSG20G8\nK4JYpNOwXK9z6hEwjvJGUzy+2LoHC6aPwoYDHbjnoiGa6bpi/J5ooNkSqJdJkohEWjAQEUmCwalb\nj6tmL8FJNzyKW/+4XFlGI1ViST+IrD7UhA5ZD6XXRdE+122PLFcm/rozRafs0xmAFG8znCcAuGXh\n5/jxIx8qLc5wNIUtew8pbTsJUNzFeSF7Q/tybxh7tm4ybEtGOhbFod1ZrdHZdWWYNLgSlw3PTm26\nOUYRJ9M0e+q2tjpQO0uiKH9gkgghlcSSBy6mtjSsWhaRXXs1T9X6kfW0mLMqsIAE+zpG+XoHbAQB\n693ETfYhEShkx06LzgplaYKNGW35tJAuq+DnUMFSNK5q1Q195HfoWLsOjCiiY+06DWliSPer9i6G\nQSJj1R2QNMQp28adD8+k6wGQ3ICQBbEwIU4ADJ5PrkNfI7PnSyPpUfRN9eA4BkvnmbTqcuvNePi9\n/NSeHqwDcAfAsCx4CVQxuMHC4Dhx0uBIecXR5hxqHBX20NbWhq1bt2LSpEkAgEsuuQRbtmxBe7u2\nylJfX4+rrroKAFBeXo4LLrgAK1euPKJ925m2k0nWU/eMR99+A7F0/kSkMgIGlPts6T2AYoS1+uOz\nFoYDhb2e0skEXO78hIlsSaDWOcWSGfzynAHU6auF73yJ37z8ERa8vQ2ja7Il+9E1JQX9nQqBZktg\nWAYRIKxtvyQ9Hp95uUF31N1t2QUhBE/eeQHWPncbnvjV5WCY4vVPmu2xDkw6ocJg30AI8PT0cXh5\n5jma5au+acE1z63GPcu/0riLb2roxDXPrca9yzejZftG0/2xDgf4TFoh7gxDMGV2PXrlpjZLVG0T\n/Y0wP2av81CzuP7ZrhZwB9ZTn76tzpWQC9vOQzuyLkoAq7+addEfJLdMFjdbQW1gaSAlLW0aXVEx\npEQ5epstOjOE0gRhp5GYWOmyihe908+VWhReMfYkfPOr+/DJ2ROx6YYZmvUcIMh08wbvYhikbE5F\n0zRQkgTL6Usz6MXkAMCxLPgUxWNLrW/q6jSfrBMyyOTuKZk0b1pt5TgGr318EKmkitjpWnTHCRMd\nR4NX/Cc4h4yjQp4OHz6MHj16KD+8DMOgqqoKjY2NmvUOHTqEXr16Kf/u2bMnDh8+fMT7tyPIk0lW\nPMNjV1sMLgdr6tGkRzHCWsBItPTCcDVZKuT1BADpZBK7OzIgAIIuTmNJ8Nzq/fjzh7vgcztMp69E\nUUBCyNYb5EkujmVsD0eLJk90NFsC/bIyLwcQplsmkmbTa/K2jqQaZIXKsgCih7dhyr0v44xRtRjc\np7LoKTrN5yj1o1eZV3OeCAEGVAeotg4SJE01cNbyrzD5udX4/dvbsssJgWTxg8s5XACfVoi7KErZ\nH+rc1OYv7rzD9JqXq0v6qm6hi4WIPPXp2+pvkzfLxMtBlCQt+aKYM0o6cbMVW9PbASik5LIbDFWm\nozGJVwh6A02HRMDTTkcBUbht0TshGHPjZJzwxouGcyXbOSydexFa16xHujVMbdM5AKS7WXlyWgjG\n9aC1cUVJsu3gnt8pXUzuYBmkeXoL09W8FWzDprztBmC8tlkHHLmHC4eTo177qdBA8LyIW68ai/Zw\nWEOSDNl2x2HA0eAV/ynOAXyLBeORSASRiPbCYlkWPXsa3ZftQv4hT/Fi0eHAemGtGXoH3Qh6HIgk\ntJoqueKk1j990dCptPSWzJ9oKioXBR4Mwyr6JoYgq2GaPxFVARcaoynL6StJFLGrsQteR0aZ5Fq3\nv912u6597ZtAr/GG5eFoCmt3tuKFeROwZkfelkBtVdAWTRU3DqmCHadxWYT+/em/K2qS0grNbRF8\nvfMg/vrAnUosS3f0T+rP4SwbiBd+mTsnXSksvGUcxg6sQCSezp+rrqzmKZK7Rai1aGqx+D2vbTJl\nJQTA9af1xcYPv8HZdWUasfipNSVYMn8ioqprXj880V1xaqpqGJyBUoP+w7LyJEngCuxPfbOkRX9w\nHrdSMVk696JsVcVEnyNB97SYIyX6GBZZV1QoJuVIIBtoDjpzDHZ8shZPXHxtVhxMX9n6OGxO3Hkr\ny+EZMgA/+d07eOmhH8JZXqY5T7Io3MpwlEU323aEwBMIoGVbA1BDjwDSQ6+Bkgr4flFhIiZ3Ox3K\n1Kkeei0TVdtUKNol19abMnclrj7ZCSHWDkBnc2D2AKQWkqtw+PBhCDrNWDAYRDDYDfuE/yK+K5/j\nqJCnnj17oqmpSZkMEEURzc3NqK7WTkX06tULhw4dwsiRIwFkT2JNTQ1tk3j++efx2GNa47yamhq8\n9957CIX8aInQhYN2UWw4sF2iVeJxYPLseiydPwGkQ3uBqPVPS+ZPBBo6C7b0gOyPhs+V9/ZZNPdi\nLJk/EV25Vt3Oli7L6StJEpWbYvcE4iY3apJ7RVL9W8paFZT7XVkyxbBgyupAOpO2J9bUoE2vAUbh\neKEA4GKxpP4LvLH7WWW/ZsdhF8987cUr895BW1dKU3F64YEJ+PEjH2JXY1QhVPfOX4NrLh+pkKUF\nb29TdE+L501Amc+FLhPy5HdxGFxdirv+tRk3/sStIeQykRoUyurD5PBq9fBEZVUVgOyAkHoYw7L0\nxDBgvQFqQKrmMHW6E0GUwBRxN6RFfwjxhKZiMvSPv0XF2JPQumY9Nt34M80BmH0CW2G8RxlqA031\ndJ4BOkH4yptnIt7UvdZgvDmM5O4DeHLmLdnqkp4kmYnCVeAA8MU+pOQqhpmMgL3r1gJtX4Px+gq4\nigOGNi66R+5pYnK3g0MiSSFP+rBfp8c0/FcTEqxHjlwtnTcBbyx/FSlai5ACvZC8sjJvozJ16lQ0\nNDRo1r/99ttxxx132Np2sYgyfojdNValgCEEIdj/HEeDVxTDOYrFUSFP5eXlGDp0KFasWIEf/OAH\nWLFiBYYPH46yMm0rasKECXjxxRdx4YUXor29HatWrcLixYup27z++utxxRVXaJaxOc+XcLgLcBx5\nx9HMo0m/3C7RAqBMqTQ2NaN++T9R871LlNdo+if5hmYlhKwZOBTfrw4qdgO72+JYsaUJvzxnAK6d\nk61A2R1blwXi+sqGNeg/WOoW3QsPTFAIk9qqgDAcHFWD8eKvRmmsCmi/v4TAQFAKTeMtmnsRVry3\n5qgSJyB7nuwch+3tSTCtOO08HM2aZebO5f+5YzoqKvJkSQIU3ZNsKWGmVoumeOzrSOEP904xNV8F\nzCuqP77qGmUyNZ0RdH5q9OuAQEKmq5P6FK6YTp9yqRIITL5cATg9thow+lYbbTRerpgAwJkf1eer\nULoqigizK7lwGO/RhtV0noR8e0qtvVo6fwJu2fJhwak6q0rV2qcXYcQ7H4JtNw6g2AHJTQSC0IOB\n1SvKr8sVw/tuewozrxwK/0Xnw13TTxcGXRhiMZUnHVFHOqFdRkDfr7qiFI+CTSdMK0yFpu1kewNH\npAEAPcRbA8KAcA4IopQNLs4waGmJgmEIQiE/lixZQq3YHGuw+zmOBq8ohnMUi6PWtnvggQcwa9Ys\nPPHEEygpKcHDDz8MAJg+fTp+/vOfY8SIEbjsssuwceNGXHTRRSCE4LbbbkPv3r2p2zvSMp6Z8/hb\n9W9i5Jn0kF3ak7gMu4aZ8YyAoNsBwnAoczJK60QGjSwV8nqqCIUwZlAPTJldj6UPTsTfNxwyNcVU\n+z7J8JZWgO3KmxgSZLPt5MrGrOVfFfj5or9qx00cyJKsQp5J+jacGcFSr3/bI8shSUD7Nx8AxBjK\nagVCSMFqFY3MHQn0ZPPHj3yInYezT9bqc7nxYAcO6siSvmJY0X8E9bsGgEXrG/G9AXXYo7rW9Nec\nWUVVb/8RSaYgitbaJYYQcI1bwHZ66cn0Kt3JsvkTkBl7GRzl1fAcEiFu/MB0u4QQ4371o/G58XqZ\nJGl0O7oqihV5+k9WmWgghOCF635hEJk7RCDJEIQqQog3hzVVMZ6XMG2+iWVBfsOW1gU8JKCto9ut\ndIJsEHTv+xcgMHgIIlu/RsND92kvkFylKThshPJ6ZOvXuG/aiejatw/u887T+HkVrkDlIMlHUGC1\nMZfBEarOE3VQyLvFtlzNW5HqMRycN6CEBxs8nvQVKjOLDiEDSRRtV8w4llH+9iTde45EsvJtQjGf\n40h5RTGco1gcNfLUv39/vPjii4blCxfmoyAYhsEDDzxwxPuiXYhqsmRlnLln104NeZIrTcVqm8yM\nNg9GknDF0+gd8OFvKzbhhp/8tGiyRIMctMqr3qs3xaT5PkkAAj364NQewIbd2R9bOdtOrmyUeh3o\niGcsKlHmf/iaFp0JwtFUQc1QMaaUeqI15eb3TfdNf382KPiskwfj43Xbcentf9LqpSQJkrukKDJn\nB3qyKRMnGfK5dIQ8hsqg3lKietjJ1O9aXjfJi4q+LpHm4XFyBp8xWkVVLxQX1ZeqyWXAMAxESTL3\nd1LrTsKNcISqMePh9/Djk2oAl0lOmfnuVCsQ9LhsEs58eIHSptv0k1vhG9APsR27urfN/wWY6Z0A\nQGAIrvzHMxh03hkK+ZGrYhOe/VNBAXshXyiJmIctmxyshpzyAGp//j8IDhuelSfktGdqQkvTpjU8\ndB92JQX07luOxOljTMOgczsFY0KqCh26NPYyOEI9swRk/gTwskBcRd55pwepDI8Sp1EEDiGTJUYm\nbWgFZponyrqiZLM9rZrCS2d4OH3lcLM80qn//zqMHymvOFqcg7rt/8hW/8MIBN1wc3lHRLUfjVqr\nIWffqddzOVhlak49RVeMtkl+X58S4/QdxxCkeBFJicHlZ9VaapkKQW1u2BJLA4Rgd1sc0RRPrTpY\n5dupoR5/33CgAx3xDH5/+UiN/5MajNPelKEVCtkLWInD9dBP4XkDZWAEuuEpwzAYPqCXZpk+KFgd\nzwIAIATlfU8yTNgVmu6zM/1Hi66RoW53FvLfkiN3bnt4FfW7VuvrfC4Otz68SvEZ+8df8z88tOu8\n2LBtlmUgitbXuJJtt/Z1pJMpPHXPeEiCAD5h/j0zJFstMgMXLIG3pqcyXu8MlWP0X57AacuXUifv\n/hOZAowECEVu2SowmPN5MOi8M3DrIx/kLRJyVTE7U3WWU4KEwOG154cmr6+fZMx4POgxarQSmRLd\nvs3QuqNp0yBJiEUj8Dg4g5+XbqeUjDsbcHqyFc7yauXYMuHGbJtOJxpHOoFoIoWAP38u5ODfVNUw\n23Escn6j3LLTbEMFSZKy2Y82wKUjyKTTcDo4bNrZCrebO+4w/i3FMUme7nzkA4UY6f1ogm6XQasB\n5Nd75b0dikGmXGmS7Qr0poG0cW65QrV5VyuCbgf6qOwL1GSsIZpCOJHulqM4YLQw+GhfO37zznY8\ns3o/GAC3jKtT7Arko7TKt9Nj1vKvcM1zqzFr+VeoC3mVShQtjLZszCTqNtSeTn+77QzLp0I7miG7\n/k16ohVhKhDwGC9lhmGwZ+UjWPP3edj39p/BsiyqyoOaoGBaPIu/R1/DPlo7Y5ZeT3I1jPZ6qFcN\npHRMOQ/yVF13QZCN3BFFEU/dMx6xZAZdukgeAEqwdCzF44m7xytEPpMpHEKrr7JptEe6kGiWYcCb\nOEdLjIrU5XQnTnfWINDj84F3mJ8HBgQCoPF1UoPv7EC84TCWzLkQrWvWA5Kk8SqSdVDKcSJHxqyy\n4uwit42yNEGrqzjyZKZ3EiAhHYsj1tmV/V47uxBvUbUebbYWqSQr1847+bYbClo6yNB7PznLy9AV\nj0HYvxej+pUh8s0WHHzwbup7Gx66Dztuvx4Nv/sfZVlCEOF1WDc7aP5OMlKCACeFhCg+TqMuQKb1\nEEYPCCETPgyy9vX8x1cHUwOIJJII+HLkSR/8yzoMxMgUqoqTfhsyirJYIAwcTicmz67H6IEVSKX4\nozZFfBxHF8ckeXp05rkKMVK3GXhBxOTZ9XA6WESS2idneb0fnT8IkWTG1K5A/n8zbydelBBNZTB6\nYAUmz65HIEe81G0/mYwxTPe6omau5HLFafrpdRhY6aNWmJ5dvV8hWWaQW0JyxenpKScjkshQw2hN\nt6HzKRozMIS/3konUELrDlufuxhRtppoSa5SkJRRuDq0XzUqyvyYPLseFWV+vPvM3djxxh/wr8dn\n4gd3/BmDJv0al9z2CPWzVZb6NPsw85ySYfW6EN4JKR1Xtq03Fi0WpV5HzrKCYPLsevjdDuUakFu3\nPxjWAyVuDhKAziSv8Rkz+yGXpGy2nRkIiDb5PQeGMBApFaxU1TCw/U7UmBOqqwBsqgvJWJfp/liG\noMcdszS+Tnq0vf4m3j97AjbdMMPoVaTTPHkJQYqBLefw/IemEC2V+/hv31iGcJHkCaC7kR/2SBjg\nKYevJHvN+kr8tsw5DaCQLLmd99qHu4zE0sRklHY+JUlC48NzsOP263FwPp04ycegVKRyxqZxQYTX\n6bCsLJll3AFAeyKF2tG6IF+djxPZ/G6WJK15HQaoBOTRRApBf+5v1KzSZNaGprmJW1SrRFECUyBm\nSIHGQDONRLzwg85x/HdwTJKnaCSpIUZym4EXRLpWQ7VeMsMrlSVaPAUAalVKjQOdSUQSWuKlbvtF\nU9ll1/xU685rF/qpvERGwIYP3gKQbc31C/mU0vSe1pimwiQBGg1UQNfOkcXif//pafjjj0YrFaeg\nx4EZS9fh929vK3h8MgF4eeY5iCYyWPbgRGza2YoT+5UrRo9qCK27ANFeC8gu1ESLMASBij6Gdbbs\nOoTW9q6chkDESUP7Ytr8bKuuotRPFYsTQpQK0osPTkU4EkNlqa9gW1H/uuZYUzEQZ/aHmmYsqobQ\ncRCJg+ZPu/KVuLOlC6IoYemDE7FDVWX0uziEuAyuuONZTbtOczwmbSZREPD6Ky+b7tssVoJhKKad\nuSfxV97foU26R74K4Nm92iLrjMBVFoKrb39MeeBtBIeNABc0Btq6AER1XkU0Z2wA8IGB4PfYdw43\niWhR64r6njkWdZ5S7PUWd33T3MgjDgkDu+KFzTn1hI5hEBo+yHJ/8eYw9n6yBlecM0BLLAuYjFLP\nZ6E8O7ULvMrY1HHm+fCXlJlWlmSYtfXC8RQqgrq2OKUlZxnTkkMyzcOt0jxZVppUZInamsu9brYN\nQRAszWD1yBtodm8i8jj+d3BMkicaJClb1rXSarg5Fj+6/DJNNUmv+egddGNAuQ+pjGCpfzpAIV4H\nI0lEkxkE3A7bbuRmULuSE0LQ0rBPc+McMaAOyAQ2AAAgAElEQVQCO1tjeOrzfYb3yqRJ7UTetif7\nB60Wi4/sFcTXhzqVitOMs/qb6p7UUBOAgMeBjXvaMLxfSGP0qDkeTxkYvqvbbuBWWiK5XfbVG4+i\n/pn7DPqA06bMA8+LuH7+W+A4Bovm0Ft1MvoOHq6pIL36u2lKK+7qOdZtRTmLD4CmfSdl4oAjSx4K\nTSiWlYoAQ084lonvM1cNx65tW/HQezsx/+18lZEFUOl3Ys3W3Zj+/V6I69p1hSCKIhjWIl3ZJFaC\nqsnIPYlfOX4Ikk0HjDe0dAJOB4eMQDuurO6ldupP0NnWrtXO6OAhDPhS1VSuhVeRgxAkYzHbzuFm\nES16XdGIaBJOkeCAp/sxG12sBB9PjJUwyr81hI5lMX3nZ7jhsxWYvnt11qDLBH/94fXY/cRiDRHS\nt+Z8A/tr36Q/n7p4HMNynQs8V1KqiMe5shAcLDGtLKl2Sl3eFk8hFDD+Digtuc3vmn72/AdWkXj9\nuaVUmjRkidKaM5ApiiN5VzwBn6/I377jkS3fehyT5EkvGAfyonGXyY+/rHma+8Ju00gWdevN5WCx\nqy1mqEqpoSdVHEMQ9DgwZXY9SjyFY18KQX3DkyRJIUMA8Nt3tuOpz/LESSZMcttm9oWD0b/ci+vm\nZlt74W1fAjCKxWe+shnX5KI/rHRPaugJQCItKCWRZyhZd2WV1Zh349ndyoZTa4le//00w3v17TK9\n+LuxtUPRN3249hvTVh0AlDiSaG1qUCpIa7c2YOywGmXboaC1bYHcRqa179Q/1FaicSEVA+uii3pl\n4nvDnFeRbD2oqTKyAOZdPAS3ntEX5Q4JKZGB28mhOZrS6O5EQQBD6H/2gqh9QiaUe7lZrARNluFq\n3grHwa+QXr3c+CIAl4OjPujIupcnX98GARJ233tnXjujq2oM+/mtGPTUn23reICcJmj4OVh5012W\n65mKrymE5gwm+8Bw2F3cTU+OaGnwijgdzoKZevrXe59+MgKhkqyEIFSC0NABpvtqZgScpvt5VLfm\nEpEurdhe384jBFU/+L6xjaoiTL3vX6CQpeCwEYAkKeLxZEszxHgUsZeeQMfzC0wE4+YQJAmcSftL\nGnWBIb/OsA4l484SerIEgI9H8605gKpz0hOqaCyOgP/ox0gdx38XxyR5UgvGAaNonPYbqtZGmVWT\n9BN3qW5MyckmmZluTtiZwcEymkk69dHLhOn+CwfjltPrMLDSj6m5VPAX5mXF4+nc8dBuL+3xjIFU\nqXVPIp+GxGt77zIBmLVkPcYMrMB1c7NtqDGD8i2pUMCFhbeMQ/2C68C3H8C0ecVnw8nkaMueNowb\n2QevPaQlUPp2WdxhbMNcevufMGjSrzHp1kdMK04ljgQk1gm480/VaV4oOpbFztSgxkSUQFOpE1Nx\nMC76+emIZ/D1oU7Muf4EdElOTbu2LuSF28li8ux6pJJx9OkRwtQ59agKuDRZioRhcOX1N1G3L4ki\nSI48uTkWPqdDmWLVragRjbMsA8Fk2s5JRGRMKsEujl55knUvc28+C01fb0bqYO4hgVLVqDv1FMx6\n/D1l2s5OQDAATHjmEdyy9SNcuXKpJemiia/NCM5ZxIUUI6HZZe9vX7YsuO7tv+L8GdPgJEzBTD39\n6wc/WYNouDM7+RbuRKLV3CG8nRXRkxgfLjfd+DOsvnwKPEG/dnJR184TSoOoGGhso6qtCQKDhyC6\nY5umWiiLx6OfrEK2qngrSq+/p7hpOhAQs+ECfxk1v04DfcYda0OPqtMxpUIDwXkD4ONRJffOoHNS\nxbLIhCqd4eFydX9A5Di+nfjWZttZQS0YB4zeNGbDCUleQFoUcDBiLsKTvW+6A16UIIgSHByDDC92\nLwPKYtt7WmPUSTq1RcHi+ROV9bY3dymGmvINgubxJBMls+iWTHsj/MIhxLihyjI1AVBXoQAo/y1J\nUNp7JwWiWDS3+Gy4lo4Y1m5twLiRfTB5dj3VA0odnUII0G/ICOzZ9rXqWCVLM8wSLg6J9UAs6YdK\nnd/USTc8WrS7uD7KhS2ro64na8fGDqzAmp2tuHfpekSbY2AolSe5ZTeiVwleXt+C9/fHEFIVGXaH\n40ims1Omy1//J0LeUo2Tvew1RgiBw+k0bD97nrI3dPlhRBAlxbCPIKeVIgx4h18TIcGxLHiTkFUn\nxyFtRp4cLFImr8VeehxSJIV9X3yNXiHjTXrpAxehiRAEvD7cckl/JLviltEsyvFIBGxFKeq+NwZf\n7W3H6NNPxpVvLsFL359KL59RxNfxljbEOrsUwqKeiDsXbrzLJcFKIkJp62dTf2UIFSMGYNLUu/Dx\nimfx3EurbGXq6V9fOPB0hIYNxDkP3Ydbtn5k6kAuIWtoSvuMsR27NOJw9eSi7Na+taUJPSTW0EbV\nWxM0PHSf1n1cpZGSq4oz/vA+nvr1eTZNMrNtXA/zT0inXKpMzAF588t0MmXIr9NAr40yE4PLyPk1\nKQaZAITakw3eTwYDzRyhWjTnYoAQpEIDC3y24zhWcUySp2gkCYHT/ggkeQEpQbA0MpTNM3tLDLUd\nJxtfVvtdpk7jVuAYApeDxYwFq/Dk3eMhCRm4WIKUcOQkqq7Ejb4hL/aEY3hWN0mntyjQm2cCAOfy\nIJWKox1e0wqTmbeQo6wasV1fAtVDDa8BWqNMQqAxzZSJ1bIVLpx4/aPdcuu+/N5FeO2haabVHJnc\nyC2+kwZW4om/LMODi7+wHvOVJJSynRCd5RADtQC0laO1WxvQ3F788erJFtdzFAAYzo2sHZs2byWe\nn3Mx3p17Ie6dtwYbHC6DpFtNeq8Zm0FpaRn0mPvWNtSFvNi5vQFDArUIb23SmGXqTTL1IASoqqpS\nHkY4lsk/kAAQHT4InjI4WAabdrZi9MAKSBkGDo4zrS45OdaCPHFIm2ieGK8f3o4upFwecCWl4Ds7\nDDfp6ttngelTi5ffeBeXXxGAZ9wYTJ5TT41mkREUCRrbwji4ZiNGn34yJs+ut3bs1p0gucrkK/Fj\nxoJVeGLmuYb3jpdceMeZAitJKM2YV1b2djTjd/fPxVv/eBQNqzfYz9TTvy6KSLSErc0xbXhR6YOB\n9ZN2B0UeQz54DTu+/sSgP9PH5uhfTwkiHISBGO9COpHAU/eMRzoWgxg3n7aUIROuVz/Yhbvu6pU1\nvszZXmjMLz99EaRLd32rIlm0GXfm34shdiVHjEyDgHVEzBXeCaH2FMWo00qHdhzHLo7Jb9Xshlgo\nzkNu7dE0T2rjS6tJO4Du/wRkq0OpjICn7hmPVEbARytexrk1bsWrSd0+KQZujsH1107GtbPfRL+Q\nj2p+qbYoUGthZASqapDuaAKQ93i6Z/lXtvbPcE5IplNROnPH3H/L7Si5vbfgwxhQaa7HsIIkAVfc\nu6igB5Tc4pu+4COcNnoQejrCKGG7QMR8pZEQgqryrH7BX1EN0VutECcZV89ZgjVbGzBmWA1e+s1U\nVJUdmV6BEKAi6DJYFMjasUVzJ4BjGVw3dyVm/fxWqt5M3VbdcaAJGYexOiUgW4HqUTcA3kCJUnGi\n2V7QMKg6hMlXXQk3xyLJC4ilM8oABgHAOZwgOXuE0QMrFNE4x7HI0CpPrANOB4e0VVXKcF3lTRJ7\nzngA7ssnY/ATixWNjdwCanr8YQQGD8HNCz6G1ymgM5YCCLBozsX00Nscxpb5ECUiXpo4Ffv/vc6W\ncDx7WHmh9oRn/4R9n67FEzPPpb6XgOBC0YXDbhFRTl/9kdDmELE1ICDOAr1efA/Lxl1aUH9VCIXa\nfR2MhFLB4vdH5yQOGCft2iUBFQ6OPmlXYAKvKZVGtdsJxhuA0+vLWsp4fWC8AWTJcsD0vWI8io4d\nWzD5ouH5iTrAWE3SESeqximdAC+Iptopg85J1QIs6P0ki8SFDPhYNusx0REGq79dmGgOj+PYwnf+\nW5Sr1PLT9B0TS7D7wCFNS00tFA+4HIiqdE96mPk/ydtxObK6E5eTRZ+e1bhh9iuoDrpxbt9yjell\nsXCUVGHJbyaZml/SCJMaZbWDMLxflbKuvUDg7kHtZbTwlnFoK7INSpuus9M6U1eNmNI6RMpOhhis\nQwkXR6nUhGBpCVY8dhd2vPmH7GSewwPRZwzsDAV9ilD8rBP6Yv3fihe6y6gaPBoLbxmHd+deiDED\nQpg2T2tRIJPLz7e34IUHJmBv1Py7kUnv5nQQTl/+ZqO3pKgbOgoOV/b6pIVR0yD/Dah1g7IAHshe\nM3wmDUmSFA8aWTROqzzJollu0KmmlSeOZSCw2hai2iQxVF2NSKQTk+fU5zU2kgQ+kh3hlqtQmUQK\nJb6shQJHJHwz839ouwMAVBIWHWz2g730/akFHbtl6HVOK2+eafleAoKLRTciIS++CQjYpvxPRIoj\nuKK8F84iLhCQvP6q3lp/VQhWDuSNrIjzygMGAbgzVG5uV6CbtJNgMllpAw2JFEac0B+M1wdeyEVN\n5aqOdhzFt//tEZR985mmZQcYzS8V6DVOKhLUEulCBWVqD4BGx5RJ8xBqRmttCUzafXqRuEy0Orav\nQbXqnNN80o7j2MR3kjzJf9/q2BYg29prbA1jy74Gzfp6obhsQwBAQ5RoRpim20lkEKrsgZlXDkIy\nzaMq4LL19K+H7DQOACu2NlmaX1rBU1qBYHVt4RVNQFgWkmhOztQo5GVkuR8Lp247uHrOEoy/81lc\nNXsJKkt9kJwB8JUngu8xFuWhKpx1ymBTo0sZahLGC2LB9a2gnIu5K8GxDBbN1VoUyJU6mURZVQNl\n0ls5cBQYVmuKqXebV0Nte2EG+dq10g2ymTjYRLvBg4ZhdPEsqqd3b0U10ibqAOmUS+Ede17upsmA\n8QY0JolSOo1Mhsey+RMR3ZGLAVGJxgFgxx034MBfXkB47QZTc0w1vIRBmuTFknbDgA2VnaZW6/cS\ngh+//jxe3rsRC1/7By4WPbhYcGOC6MEfVizDz775GD/85/PwVoUUUlZ7xsm4sn7J/2Pvy+OjqO/3\nn5mdvbJ3bkhISAg3hCABAeVQEYgWBTw5PFrvo7ZW69UCAm09qm21YrXqV/GAWltFe+BRK14V5EYE\ngXCTg9zZZO/dmd8fszOZ4zPHBmzl9+J5vXiR7Jy72d155nk/7+etTaCMktF1nk+HhcO0l/7YQ5Bo\nWiRMVa8+p5vMDqTns+m9QAZoiMYx5Nofw3/NvWBjEVjAIXGsFgBnmPsEAAfbghhZNZi8czMeJ8k6\nzZ3dyPORjwOkiU/dDlhtDDExXAWNdPFYTgUamHwEBvBle62ctNM4NXFK//VI3zFSwkTqwPN4vHCk\n1B82UmCmkijpzb8TiJSwHwpA/9J+2LR9r9gyLtz9m4Wy5EJ8DaAOwjwRUAACkrKR8Lu9oBxc1Fxo\nm16WUd7ASt1tSUndZmbGAfz74fVlC/Dhk9dj56t3ygkYRaOlO2G6e05IF/9ip7n1tc5R+lps2Nts\nONdO+fobwcw8QylZrz+wF19+9jFxX43dMd2cNA4c32mnyKCx0DRSUu+S5O7dEu1GjDS/Lq0M/PWj\n/XCWVMB95Q959eGKHyP0xtNo+8MiBJ9+AB3vvY3gN7vgGThYlRsktMJ7QOHTBddphmOqn4cGDMjJ\nm7O/j5cnze6VUiV4pZSPg+Nw7MvtYtBs8dhR5OBOjcBOM2DBwZrlRN64M3rynAaUiYQpu2ok2rZ9\nJSefipiCJi6FfDMTEzRyoMKMFfnDqjBvyXuwOZ1iVIFeorgUhzu6UVaQWdq6lip1vKMbxZVj9AlR\nPKI9346wnVbX3X1Pvoc+peU8UVKkh5/Ocjq1cUoaxoEe83c0nhS/7KW+ptXLaxAjdOB5PB7U19cT\n9yklQ1pEiTSJvtjrUBnMPQ4rblyxFePzutDUFcW6Q+1wMDTGFflw0dAClXlX6IaSIppkEUk/h1As\nqVouqA4VeW7UNnfjhQ1H4FIYxTOB0NFV1c+PbUc7cP+anXhI/H0w7luzEw2HdJKFJZCayDOB1ky5\niSNL8J+vjuDyRa9petukxGv18hrc/MiHeOruc2Sdb8pOOC0IZUIz6wtq2cSRJdi0uw6z73+ZP8dE\nCKnOOvG1aOuO6apwFCV//e9bs9PQ5ms0z3ByaQB5LhsYhkZDZxQvfx2G1aX+8hfew7FEkkieKFCa\n52KhabDECwEFh9eHWCJ9ThLzrqAMXHLOAESOHYKzuAzzlvB/N+rKH6L7T08C4EBxHDyDBvd02Ely\ng4SOr1KaweFUHB4dxYk/Hd7X42nrQJBm4WVp2bK5b7+E0rOqcezL7eruO4rC3DUvovSsas1uNim0\nPEjKxwHgjQsW4LK1r2HE2FHEYb4CmdIzhOuhjmGRF4zJDODK7jqZWTxdxpN2Lh7mkqgeoS5xK1/f\novt/Ce/Q4WLXnfAasdGojCQlWxrEzUJvPI1YboHsMSXiKVaWCG4aBFXqaEsHxlWejVTffj2mcAJU\nnXRQm8mF3xEKwnJki6rrbt6UQuTl5IhEiYkHkYCX71aFV5WXdhqnDk5J5YmiKKKqpIwsiBCmw3t8\nPgT1RgtIoDW+RcsvJS3lBaMJrP7FhYDFinWHekgSybyrHAIswMHQcNoY3PzIh3DaGFW5T6k63EgY\nFpwJAllWnFESwMIl7+KMkoDhwGAtKLvKpIjv/ZfhHVcmM+WkkBKvls4wnrr7HJGACapQprEDZtbX\nyqLyO1OgGIc4DNhopl22296r11trnqGgXNI0hfmL1qKP14ESNw27Q65iCu/hKx54G5FQV8alUpqm\nNMp2a+Hy5yDK0UTzLrX5b4hs/Aih13+HyLFDovriKCoTyzdsLKoiS8qhs2PPKMIRzuCGQeLrufOF\nJ3HYysqW5Qyt0C2fGYVXkqDyIKWVLSGkEwCvJL39Et64YKHmMF+pSd3Q4E5Qz44xKXwvx6sygMt+\nl/ibSAOBG9kkipz65XelKijkQEVSKTgstMbYFXO5TydzOG4wlkRO337mSnIKxUlpJpeX6+RahL1p\nNzoP70JulpSk06AYK1IsB4qxni7dncI4Jf9yHMdp5jpFFYRJ+ZnLzy9A0/Hjpo9llNWkVKgK3XYM\nyuW/+Pe2dOP8y/kvQ0FZUpp39bqhokkWTV0xccTGJ/94U3ZsqepwsDWEslzysGCz4ACZmbMjoh2c\nqQWpWZxEEmhfEQJ+p2obaclLSljMhE5KIRCvEQt/i6prnsTli147IQ+VGQhZVMKw6LFDi5Drc4EN\ntYJ28Rcykg9MGZAZbd6GDzbt1n29SWVaabMAx3Go3c4rGsL7TZiB1xCMIhQKw+6Uv/7Ce3jZNUPw\n8Sef6natkmCxWETzLwBZ2c4GFpFYXNO8yyV4gh16/QlEj+5HZXm2rHxDUxQO//IBGVlSdnY5LRbE\nDE5aSghGXjADUVf6dU8TlKs/XYNwMKRZPjPqZiNC6kGSlt3efgkA5GQsN6A5zNesSZ1U2kuAA81R\nsKQ7AOJt7T3lOI1RNqSBwCmA34cWKAoFt/4UFG3By0tmyHKg9nRFMMidBdLYFWmDgJbnKZFiYTU7\nWBcgh2RKl+kM8dWFcjtJaY9oLgcAllWZ7BkLjYVpD+RpnLo4Zct2etD7HrVarZhRc+FJPZ40WHNQ\nrhvzF6/FqmU1aEw/pszZkZbo9LqhJpcGkO+xo6mL366rvRWeZAI003OnJM10uuHMEs3yDQAkIiEU\n4ziOoYD4PNrDCeyo60RVPz921HWiPZxQBWcW9vejUad0JyUJrzw4U6VAWfIGIb77n4CND46Ulry0\nynLK0hlFQbOUpiReeX556GWuz4WWzpCp0l0mUGVRtbTCZ+uJE1D6wAQlSgjIvPHZ9Yi31eOxz5vx\nwvagJnFaNK0UnmQ7svoMwPPpWAop4qEgGg/XomJUNQCI7zeAf69FI2E4HOqYg2PBKA42degnL3OU\n6N2Qgs7ph2SHvAlDKHn4ww0IhUIy8y4xxBAcuv/0JMKK0MQ8mxUtsTgKTarFWlASAoSiiIOBPy9H\nVg6r27QDI6qG9xCkdNnMTHilCMk2AqRE6LUl0wGOMyRjJJO6HpTHyMrLxpb24xiQsIjnpSzHaX1Z\nSst43RyLLAOFRAgwnbd4LV9eXfGouGxvdxjXzawmbmfG83SovQv9A9pRBlIIoZmJlnqV10lYxn1+\ngFiSMwPldvam3TwhK6pUBWiST1A+H5LRUuEJn7PT+G7hlCRP0rLd6uU14mR2LdIktF0LKC0rQ12X\ndso4CYJhXAvCMqVPiqEpUVl6bVmNjDgJPysJlbBMuV1ecX80HT8Kf1HP8E5BdaAAvL6tXva7OijT\ngeO7NwNDL9B8Hvet2YlAllW8KGcaa2A0+JYSht6mR3zk+tTkRisEEzAmW0pidSIeqkwgZFEJx/Y7\nErDkjZKtI/WB5XjUJDPI9YxHIcGfZYWf7cTdv/kH1r70gOrvCwDRrnbkBXLlj0neV9FIBM4s8uy8\nSDQKl92m+rwAgI2hEXF4QLlzxGRxAVarFZEEoWyWSsDjykJ3JKYIKOyBhaYkuTtSZYIPyix02NAY\n5TOC9OClaAQ5Fl6di7yUEJQlLDhoTWGwgqDICJLEByUsM5sHpfRGkZQrM2TMcB0JUVMeI9TUgmYH\niytzeAO3VH0Tg0Tb2smBohJV6hs2jqG0fglZGWAqVQajKRZOq/alJvTGCt2k8f1tXaieeIbu8fkn\nqAjNlHrs0ssuv/9tTO2Xr09wjKDcTmEuNyJkTDwILkFrEqekzSsm+EOjeeM0/vc4JcmTsmxnt6jN\n4wJIxvJMQTKEa0FpKE+yHFFZUqpRSjP4uCIfKAp4eckMcbuiAYMR/3obAPnkc5Jx/DrJ74JCQdEW\ncKzxa3Dv9MGapuVI3R5wbA4onc4bI7O4JW8QAhyL9mCCSG7y/NqqkB7Z0iJWUuVKUKKuXvoeXl5C\nJmu9hZTksaEWWMvOUi0XXhMlyWwJRmAHpWsY7wgncPhoPX79kwsRiibQTVAXy9wAmyJMvU9j+sVz\nQdPk4dkeOgW3Mws+p13ViGFjLIiCEm9YuETPnbHNZkMniTwB8Lic6I6m3wcExSnb6UBrJIYCt7TU\nkh7HUVKB6urP8fYD92o+HwElNIMjbAIjLDq+HAkhmJbvxYqONgxOqAmK8D9JyTEiT3rbqIiQmagE\n0joCYWpuUxE16TGOMiz6JXv+1qpyXFu7KSXqKJfCxdNGgQ3qd9sqU8b50zdzZ6Iu50lxrLMbl+Sb\nyMeTxhO0NcqJenrZkoUVOPh1qPfESQMkJSsSjcFh1yD9OoqTEGewenkNqAhJpT2N7wJO2aKr4G2K\nsynNocBmBgYbgaEpZFktYDkOWVaLZrq4FEqFqj2akOXsCKrSrY9+SMx9EpbPX7QWjIXGl3X8l1Z2\nYRHaGuXlEUBtHM/32DXb121ZHqQi5C8qCjA2iXMsvDim+/ylJAGAyttjyRsILtQk+pwEn5IZf5Jy\nfIqU+GiZy0keqpWLZ8BC03j67tnfig/KWnGO4TpCttMNz6wH190EX2GR6rWXRhf4s6xoPl6PJ946\nAJfDSvS1hYMdyPJqB/AxjBW0hroVj8fg9biIn5dUipUFZEq//K1WBvEE+WJEURRZ2Uv7UvLdDjR3\nyy8QUh9M/zPGo50wzFYJU6ZxBdwshSDNapKY3vicdLfJIFdKExJf02X/fE3um8rPkZGzw0wKF+fI\n3wtSkzjJGK4ECyD3whkYvOJlMeVdE4SU8YZoHH0MVEMjJFkOVovxewDgmxASrY2wZhfKk8XTy/b9\n7UUMzbPrm8R7CwUhO3a8GUUFuRora0BR1juZRvnTOLk4JcmT3cG/8e0WC7wOO+KJFNE8rjcwuMhj\n/gNtZXiDn7WX41U6O+TKkhBB8My95yESV0cQSH1QDRK1ijceqj9Mynb1xq6YZvt6/uDRyI+pgzaF\nmIJn55+BYCShaVp29B2MaP1e08+dZCCnaRovPXovtq38Ed74xQIAPKkx21mnHJ8ifJ+bNZff9vga\nUBQwf/Fa3eOYzZdSIm9gJWiHcYKwlGS66SawuRUyg35HOIGHZ4/An647E4/MHoGOcAL7jtTjuSVz\nNH1toWAHXF5t5UkPDocTDqdb9XnhOCCRTMJmY9IlO7kCYbPZ+IRxjQuS8nrLVV8sdt4NqBqO4wry\nJPXBxOsOIhU1vvt2WiyIZ3ihuTzgwy6bPuHSS+1WQdpNZ2Ybo9BLAqTKVvG4UTi2cbtI1ISk8rnv\nrEQdw6JvilYngkvUN5UPjIAjbgfOurBG1UFnFnu6w5g4TiPc0iQyurexOWHNKSQ2JwDAofrj6Dt6\nqiwN/NvC0YYmFA0zUW5UgIkH00G0p2MMvss4NcmTnQFN96hKNqsFwag63E8o2cVOoGSXZDl0RhJY\ntbwGnZGEYfcdCe+8/qrivPgIgnmL1hIjCADtVOhJsxeo1qUI/2u1r/v69kdn3QFVGKN08KzXacVN\nq7YQ064pmgYo+oTSxqWPScmLWfIjHZ8i3Z6ieGI0+toncetjazTJT1O78XHMJp33lmAJ2wqKnGfY\nJDDeXNncQenfRFCittUHiX9XAX3LB8Hl7d0IoHHjJyC/qJgYlBlLpoBoF/EL3Wa1IuzMNXVB4sZe\nDGtuH/Hi1qcgX0WeAMja2jl+LggxfNE0FKGPAOCkaFg4ChFK5zNtVi1SdNOFW9r1iVEvQy+VylYq\nzt/cWKxWmQp1xMtgTrYx0dnx/Vvw2RTeA6kazwJgW3sLqvJziV4mrecl/TsdC8dQGtBO81ZsrJhx\nRyFmdcJmUnUCoJssDgBJiw1Of446puBkKFHSfVisOFx/HCXFxb3b12mz+Hcep6TnKRZLgmXlqhKr\neK8pAzOjqZRMeXrzjT/jzJmzARibwUnBmEaQ7VNxR2x23hjpcafbA7TKCZVQtluweC1eXVYjGolJ\nygRF0ygbPx3Xnyf31kgHz35d34lDrWHN55ZVOhyOxDF0ob/Bq6BtIJc+JiUvZkIpSSRL6nfq6I7C\n73aI///nqyO4YvFryPH2dNpJj0Pq3h8NYXkAACAASURBVDNjZCd5rHIr9BPUpdtKu+0Wv/sNOMgN\n+tK/iaAC9htxpm4Iar9Bw00dXw+aAo7GArvdgSQsxt1GNies2YXYUdvClyVaGuCzAl0x5bq8WVz0\nwVAU+t73C/iGjVCFL0rhpGiEOBYupWlcp8vsCr8Pf+7oRHXMKls/kxBKQO11uuyfr6I4HXpJCtTs\njZ9KgOBrAvicKGEfggr19/97BXldcdA55MYAGdIDDFVG8tY2JDkOLIDmxx5Eu8LLRAQhJJMDHzdh\nDBruK38IR1EZIkdqEXrjabguuxUHGtswumgUAPMqjFZzAjdmFuivu5GIJ2XmbmXwZW8g3QcAMC4v\nmpLvITtwAoT/NL7TOCWVp1h6YK8y00kKvZIdADQ28mm2eoN+pciEOCn3yVitSCbk3X1m5o1poVoR\npmmUMq1EYXEJqtJhmFJf0/1rdmJnfRDD+/rwyOwRmnK5o+9gROv2mD5fqbdH7zHAfIilNEgTkJOd\nXF8Wbn/sI/H/iSNL8Navrsa2lXeIY1v+vHwBWjpDmgqTGRUskwBPJZSKnFYgplSJAoCc/kNMH0MP\nZrx7Smj5L+w2K8LBDs3cHHGztCpQOSAHidYGUJveJgyapWSDYuksLwIeNxL5hYaloxKKwVE2qVKZ\n9Lw9XopGkuKQEMrhJ0EROvbldhSPHaUbqNmr3CgBaTVMuY83ahbgmaGTsfKhRzHXhOokgJTrBAC7\n2TiG0Tail4kEIa5A+DtFnS5kmcoyouC64g44+g3AjgNtcJZUgMktgLOkAj999M+onlajn91EfFIK\nNdPmRMTmxtuf18FqY2Cp2yGavElz6TKCbB8+8WfaauvpLj6N/+9wSipPUujZHKLJFGIKxUm6oTQd\nfNWymozVJRJI+6wYMhz21kNIFQ5SnN/Jk2aleU9GEMmWwtfky7JiRF8vFi55F68unSnLd5KComl4\nK8+D0TQMQDttXOr3yS0fgpYD35h4lvLtpYSGlC4u/L9xdx3GDi3C7Y99hGfuPQ/zFq0V1SQAmgqT\nkQqmIlgtrcj1FYKyGZMoqSK3/ZjaW0YB4usvLKsqz8wfI4UQhbH6hT/gp3feadg9qowrUJOcHtgd\ndiRbj8rHU2hASxUQIJjF5z/4Hl5ePAPOW5ahyv8atn/4AVY9OE+3dDRmdF+8vfUoLleoTFrkQMCQ\nOINvbEmMjFtPiiIUbmrF3HdWGhIj07lRJo8JioL31nn44ahhGO706uY4KSEbz5LGN2wCt585oGcl\nigKjo0Ap4wo+P3QU47KNvX90lhvO4v49VYKj+5FsaUDkSC1mjS+EKxkmD//NBPEItn72ER6++zJe\ncRL2pwi+zDT3Sb0P3g+4etlMPPbwFnX5zSi/6TuU73Q8lEA8dfLOxWahkePJkAR/h3FKkie7w4pI\nqqeNWu/7QWuZ3eFAdziMYNROHPQLGJfzSCDNxBs2ajTWvvlnDFaQp5MJacq0GTy/4QgONcrDGDvC\nCQQjCaxeXoP2UBwdOvlOVm8uCr3QDcxUlqZufHY98e+RPLoJ2Tl90Naq3a5sBMHvxHEQS3PSMMw3\nfrFAJFRKNUlLYTKjgkkJlg/NGZ2zEOlgzVEkriPzGXd6kMZifMWmDG8YtOI9OI4jfrnbrFYkkknz\n7d/KcorkZ8EsvurBGQBFY97itXjqjpl44eorUPLpu7oKiM/KIOKwEUtQJHIgYGo6toAFOY/JNCT+\nKFPEiOMQbm5DVn6O/np6ZUTJMW25AezvbME/Poph1ZLRms9XmPEnW6ZIG+/kWHgoqqfkpjO3Tgpp\nXMGRcBTzZozRfl5psOFufjzP0hmIHDmA0OtP8Ofw+u8R23YQVKUxATODnR+8hR/cdIOqNKeKGcg0\nA8piVe2jIxxDgFL4rST5TSTvoNHy0/hu4ZQs29ntfBu1g7HA57TDwWQujZYPqMDRgwf4Xwg31WbL\neSQoZ+K53B74snnpnmQO1wNpfY7jkEpkFvKp2gd4sqQ0jXudVtz8yIfwOq2mZ6tpgWQWJ4EpnYDk\n0Y2ASRO6EkLpbetLd+Dpu3kfm5L0KMe2COU+6TLpY2YhEKycgnzAmgXK5lJFM+ht29LSglSkW/a4\ncsZgfzPeFQmk7xlp2GqO0wKLhSEOvAaA440NmvEeFE0jyThBuXOQtMkvZppxBGkwFhqJlH7DhrQk\nKJjFI4f3YvXSGcgKtaMtGFQTJ4KJPBGO4PiGTWqVSWMUiYCBCQZ7bPw5ZtRhp/2ETAdq6pYITa6T\nlZ+Dz4PHcf6IM+TPXWmUl8z4U5rDpfg8FcGc0aXi71pz60jPO9nZga5EEm5T38vpTK/i/ojWHRSJ\nEwDsburA8MLeq61KhGNxw7FVsfyhGXXiydaXEK7de2sxbEDP6yfNb7La7eqZdkbLT+M7h1PyLxRL\nKywnkuE0cNAgBOsO8Hfhi+RDfbWG/WYC5d38+d+bozkAWAta64e7gojt+CDjc5KCAvDE5ZViGzyF\nHoPyip+eqzvLTtmpJ1smIQ5GaeM921CwDjwf3qQ6w8oMSN4jpZcJ4EkOSU3KdFiwChyHxMFPYC2f\nbDjbTwlHcBuU8RPKGYPPzD9D14MmIBWPwt20V/aekTYn7Nx/BIHcfM2B12/99S+aXkGaokBb+CHV\nml/uGn4Rr8eDrojkb6/wr/gcNnRGpe81PjQx9MbT6Fj5CEJvrIBFMcxYUEIGPrVSlj80hLbhjWtv\nkA3ANYPpeV600SzvfToZeUx6SJMdM8OGhXUWLtNYJ02uhj33S5z745sRWfSLnudOIEpK/5erolx1\nzDjHIcpx8NsY8RgFt/4USY7C6mU1prruPm/txEWTRhi+FNJML+lAaIDCV9kVmLbsKVVe08mGSIAK\nhmXmf9LxS31dewiDx0/tWVeR36QqzRktP43vHE5N8hRNGBrCjdC3qBhTzjmPeBdOKr1pwSyxYmgK\nfTQGAJOgF6Tp8voQ7tJP+zWC287gnZV/UJnGlQZlJYSSkkC6+vTvufOXEoeXbpsIigJu+uN6XPr4\nxypjuBJ0VgCWnDL4HfqKGikagGTuPhEzd6bw28Ng+o0FRVtMq21AWkGMdMPilM/tEmYMCli45F0U\nOxII7t+uex5sZxO4SFD1HhOaE2oPHUFeYR8A5AYIQf3RasRIJPlssngiqfpyT7rIUQWx/KHIHns+\n2kvH88cYM0vMeBJQ6HaisVvZ3UnBddmt8F9zL7y3/goFF12KvJ8uFUmS0pwsKCHnjO2H3am4rsqk\nhZExBlvtvVM/TUOiJM18/jem59utXDQDlIXGzOd/I1OLsvKywZT1wQO/XoUb7vkJbAG/+NxJRnmp\n/ysS7MaZa1apFKgvUlGMt/SQVUF1umbZe+DYlGxunRYaonH08xtHFGjNtqOz3Igydtzw6GfEvKZM\n0d4dhs9F2IeUAGV5kAx3mR8YrDNguD3YhYBfrooa5Tedznc6tXBKkicBet12yqRxJWiahs/v17wL\n13pcikxKewIhM4onEGAUpOnLzUe4PTOPjRRdsSRolx+P3lQpU5mMZtkps4dcbAhc+kIqEIddB1tR\nXZGDF2+biGdvHI+/3DVFJFN6YApHgIt0ILe0grhcL3tJWXozmxl1wmAT4MLtsGSXAVCrbXoIONrg\nKBxAXHbfmp2Y98IGdMf4G4Qt279CHPplEE+iHe68IuJ7LJpk0dRYj8K+RaaelvJmhKIpMBZLOleN\nkStPFA2LzUHMzmFcXjz790PoZC2AOyDOHpNeEAeOHo7GLnLK+M2//gg2lwvrvknhKMuJJElrlpqF\nouCgKHST7txpGq6B5NcbACYXeOFnKRxmvr15Ykq16d0b7jIsEb57w12wUBzmL35XpT4Fm1ux4vdP\n4cO/PolIsBvxtp7uXS2j/I7v34INs+fD6XWrOhBTHIfjXApjxqbzidKqE0VbsHLxDFOqU1s8gYBN\nWR5TZjj1QJrpJaCpuRl+C6XOa+olidp5pBEjSgrVCxQEyN6yD5YjW0xHFtibdhPXp7R0YiNFSbFc\nr1HjNP63OKXJE6DsCOL/l3qhSL4o5fuxNx12mZb2GJrCsWBUFU+gpUAZBWkOGz8FzDFtJYIC4CHU\n96WPH80eiZ/8bqWmykSCMnuoubEe7jifON7aFcO2g22orMjFvEVrUVWWLSdTt2oTKKHcZxs0DZQt\nC3kD1XlJemoSqfR2Il4ms8gbPAbWITWyx258dj2mLePLqnrlu9DBrcgqr9Lct9fZ40E7fLAWffuT\nSaWA1oajOAKvZgTGOTWzUFjUT3N73VEQHJBI8kn+8URK/iXPsUjFo+o78PSF6bd316DxwF6gu50Y\nYFiU40NDl1x5EhSJZ356DuLxJJ77xbXYuH4DkpL5anUP/Qz7br8Gdb96QLbtrFH9sCGluOmhaUz4\n/AOM//vrmPDFh4DGiJrLc/yoY1Lopk5C2YSQIK4ypB9vMSwRho+3EBWqFDh8GaBxx49+hB8/8Tmc\nXrdqxIp0HIsIjkNo334isdrExlAtmQ8oKHzzFq8FQ3Fq1YngO/u0pRMXT5F+fuXxE2qTqXq23ccH\nGzC9kEPyo5dBbf4bvxZBtVRBg1ztPnocVedMIy4TCBAAXj3N0f+cqaBQqBLJJCymIhoIkNyUJG1e\n+PxOcaLGaXy3cMqTJwECSXKmO4UEL5TSF6UkU1pjWoxUpUxKe9J9SdUAPQ+UUZCmLycfwdYm4vGE\nQcE/P38QbjizRJY8Ln08Ny8Pne2tGc9Pun/NTjGB3Nl3IGKNteLA4e8//R9sqm3llZd9LSoyJZSx\npN4osz6hTNWkE/Yy6YCigKFjxiLHY1fdHaZzB3XLdxzHAmwKNKN+/ylH5az46bk40tiKmEVf4YxH\nIrBnuWTvFSnppiiq13eytIWC1ULj5kc+hM1qART5NdZwC/EO3N60G4XxBrRs+5g/h81/k10QASDg\nzkJnVF2uDb2xAh0rH4GNoXH7775EMNRtajxIgcOGNo6Vva9dA8qQle3DvEVrkZXtg6uiXJU4LuA6\nXwCbHMme7KfeQMfo/ebF1+LZYVPw7vU/Mb07pYk9BQ6fOxMY0ZGE9WgD/nDXVGIMg55RXkasKApM\ndgAH2QTOGttDsLUUPuE5Cr6z4sWPis+xNZ6QDXqW+pqcJRUSX5M2GoJhFOf6ZYoTSbWUPVUdchWK\nxuFxaTRepJXSE857SqP2SD0G9DOn8EqRtHl7GjIkBnK7gcn9NP43+P+CPEk7hOw2BtFEjxdK6osC\nzJnMtVQlpbpkprQn3VdXaxMibTzhkXZBaXmgjII0+5YPBptSezSUg4KFDhPh8auWrEV5dhYWnT8I\nCy88F8Up80ZtCsBD6Qu7YGJ2DzkLrgivXnEcT6CEAEwZmUqbxpVkKcdD9gmR1Kf/hppkBIoC3nrs\nRnywaBrWLZ2O5wmET1m+a+uOyTrwCoqz4Bl6NnH/pFE5W+qMQwqVJDjTBoXCtB+KBJYF4lLPE0so\nbaUSxItOwOVAR0Ryd04IMCSDE/N+Vi+dgXhLCxId6c+ChmFcwEDain1czzFD+/Yj3NaJ1ctrEG7r\nxMAl92l2nNkpCtd6ffiPM4GUFoEymEtnZAaXzqEz1e0iMbELxGlUjMHkfC9ZXSKMoyHtU+jIq3zx\nD+i6/4e4fNHPVeejpfAJqtTuI53wDhmG4p8/goMUg3KXnORr+Zq0EIon4FSW/QzGruiRK47jNG8Q\nRbN4ToWmfylTfLP/MAaPTX+2zXbNKbvtANFAHssgguY0/nv4/4I8Sc3j8UQKDmvPPLtoMiXOvSOZ\nzEkfKpKqpKVEGZX8pPtKUBbUbfkEQNqD0sUrS01dxiNaSOSqctI0jCvPUz2ulTguPP7K0howDI0F\ni9fiyotmgk5oj2KRggJQmpOlmrfmKChDvLUeXJLvqJIGYCrJFKCOMOA4aHbl5Q2slNVmT5aadCIz\n6QaPHouxFbmgaQrzF69F9UCyMVxIUb/pj+vx4q0T5cORrQ7Ycsh3p8rSaO2ho3D6jaezD58wVfzZ\nDDlXYu5ll2sv5DhYEyFwoTZYo2Qyr9XmnSgcDpRWEhUBQS2wDh0LYmYIejwxvi3/wbF0156WYVzA\n9LEl2J6Sq1lfnHU+1s+6ApEDB5FTXUVMHBfgpywYHWXwOYlAkVQlBZnSy4sy02WnhTg4fOpMoCrK\nYEp+OjJCqS6ZjCMQYMsOwD96JH759FuYdvV8tbqnkTCeDHYiFopgWFk2dtS2wDlwCHaNmoAFK56F\n8m9J8jVpYf2RJkzol696nKRaitAhV9sO1mNk/77qbRTdcvbWWm2/UwZK1IFjDehfWoKkzUeM9iCC\n0G3HxIPo7IiIEzVO47uF/y/IEwCRJNmsFlGBEsp0XkdPmU5pMn9mxe9V+xL8SXtbutHYHTuh6ALp\nvsKMC8H0l9Dk0gDyPQ5E40nkexy66oCRgkDyNmkNBhYe39vEk6uDbRHkDJ9g+DyUpSRVOnnV+XC0\nfUncVkqmAHKEgda4FgDwJg4jp6iXAzZJz8Xk0F8Scgry0XTsADbWtoBlOaxaVoON+8gxDBwHtHXH\n8H+3TER1RQ52HWw17MATIO16rBpSgpLqcwy3KRrQM72eJ+cx0w0KRuA4ji/5CYqT4o6ao2hy2SN9\ngfrrulp1uUWiFuSWliFIaV2geE/MlAlDsTPIE2fdchJ447iXotHOSRQylkWitR3ZVSPF+Xpt277S\nLGtNLvCiMsbgC0cCrIRAqchPfg6xRKeVF0UkVgZKFgA0W1h84Uzgeq8fkwu0L8h642hUSJ/r//36\nMbzw2F3mhv+mwXh9sLucmLdoLSorchEJdeOdz4/CVzEsXZqTmsTVviYtfNPcifHnTiQv1Eka1yJX\nH+/cj/Nnz1FvQOqWIyhOmWY/pVgWnDMAq92WUW4Tk+hWddtlaqk4jf8eTknypOXbUA4LBshlOun7\n0ULTSEgCJ6UKU6HbjkG5bhS67ab9TVJI9yVsk+Vyg4uFxRiCLDujqQ44GNpQQZhcGlB5mwDtxHHh\ncSW5Mhr9QSolSY3mVm8uvJXnolASXUDTQEUfcoeNkiwpCZYUtiE1SBz4BNmBzMIitaA0nuf5XaZU\nqOy8ABJHNoD29sGNz67H+cv/halL3teNYch221FVli1eYLYdbFMlipMg7Xpk7A5Ynfrnp5x3yJNz\nO5q6omLZN2UQVKkHluNApW8aZN4McQWWXPZIX6AumTpAXW6RqAWlPhf219XrnkMfbxaaJEOEtcpJ\nAmaPLsGGlOQ9lf4CaNm4FSP7B9C6aRu2Lbhe95hT870YGmew3pEAlyZQSvIDAMUTx4KjaRRPHNtD\ngnTyomTEyiAIkwWHrfYEGiwp/NAbgJuidctyxC470vpphWrIm6+gpb0dzNOPaL6WJMgJ7C68+tQT\n+NNT96ZLc90GJnGNfbIsr8z2IlsPgIpcpVgWSdCw28jEXKtbTkSGs+8isThsNhusdnvPAOxY3LDL\nTvxMWY39YKfx3cApSZ48XodmqrhUWTKTBVVeUYEjB/cDUHudvM6enxu7Y4b+JilIahVDUxh95kQE\n921FU1cMT99zHkKxJFEdENSmcUU+TeO4QKyU3qZvA8pS0uHWsCoo0+LoucDTNLDuwelYc89UfLJs\nuqq5SY8sKUFZrLANvxhs13H4kkeBlLntiPtKfydLjecr7pqtq0JRFFBa6EKyfgdsw2eBomg+HTwY\nM3wOUpVtU20rrl3xn16fu+ZzgnaqeL6nh3Cvev7pEzwOpZuEbDu+q2fgqgT2pt1gD+0gllsEtWBI\nsg7727qg19IuQLwbNxhY67cx6BKM45JSFgB8NuUCbeKkIBrn5XtRkZArUDLyw3FgGAoLl7wLhjGY\nF9XzJERilZWfo1nG66BZfOpMoDRhwdXZAX5ciomynNIMTlpfUKhmX7cIty8ij1sxQt3DP8eB++/A\noaU/xcF/roHznWcQeuOpXpnEAWB7QxtG9UmTz95mO0m220j1w/jLrtNXjfT8TTpZTiSs3/Y1xlcO\nQSIWQ2VFbnrUikEmn0G6+Om4gu8mTknydMfj63QN39LvAL0sKACoGDgIXfWHAKi9TsGIXG3KJNJA\nua8+Hl7FmjJ2NJoP7RNVgb/taSbGF0jVpi/rOonGcWlHntTbZAQawE0TSvHz8wfh5gml4j2hkfok\nlJLuW7NTFpSp/DMU9vejvMCDbA//hZDtsaO8QP+CqBxpov6dgrXsLDBlk+BNNSK3bIip56o8hlCu\nA4DR1z6JWx9boxumSbFxzB+Zwn0XlOLlJ5eD1mhx14Ogsl274j/gFF1gRjD6mwgdlFqp4iejZAdI\nCQs5Cdlht6HTW4ZUUSXxQuW20vKUcSniERT43WgJRQ3VitIsOw6Fzd3AAMAA2or9XFJVytIkChpE\nY3oer0B94kygg2Zl5Cfc1IpDn3yJ15ZMx6FPvuxRm0yU4kBRmPnc40hxFFYtmymW8Thw2GlLotaa\nwm3eAM7N71H5bDnZ8FefAZam4a8+g6xASXxQWmW8eGsb/vHCi7jvxguBuqNIBjtVsQNG51503y9Q\n/tCT2HnWdJyd6xdLc5maxAWsP9KEaTMmmYslUMLmlG9nc+LLXXvxhw+6e99Bl55bZzb7acuufRg1\nZaY68FKvbKeTLu5w2uDpxYiw0/j2cUqSpyfvmppRqriwHolsFRUVo/7YMfF3aQedmW46PQjbUwC8\nDiu+2t8CX5YNl10yV6YKKC9upIuf1gXwk8Pt+P2af+P9TeaymigAN44vRUWuC12hGCpyXbhpfA+B\n0ruwC6UkZVAmaQZebUMX2rr4L4S2rhhqG7S/PJXddzStji4QyBTt8MI+fBYoxoa8gZWyjjwjE7iy\nXCeYz7XiD/IGVqKsvAR3//BGvLHVYdqvpHrdJCpbVudmJIP64ZmZQOig1EoVFwh3LBxCVpb2a5NI\nxNHcRI6+ECDcAZOSkN1uN6KcRbO80Sfbi4b2oKaaQFEUKMZmqFbMnDwSm9rND5CeUV2C7amYZmCk\nEracbDnRyMkWyck5+V7c7g3gkDWF9Y44wlTPZ0XlbzIzkw49/qlrlr8HLsXi3et/giDFq025KQrX\nBwJglNtyHKwMjYVL3oWVoQ0VI63nHuZYvPPoYxi09s+oe+hnut2LJIgZUEvexf62DkyaPl62PBOT\nOAC0R2Jw2RjYXR5555zbuFtUIE1Udh+kOIAK8J2jia4OvP7L7/Wqg05rbp0ekqkUrNb0ez9Ngohl\nbgWI6eIUDYeDwR2Pr8vovE/jv4NTkjx1BaOaSpIWtIYIW2020Bb5Y1KFSak29WbOnddpFT0vXdEE\nCkvKsUpCjJQlFwdDa6pNJPj7lGDHZ/+SPSaEYSrN5G47g7JcF25+5EP43bwyVJ7rgtvOoLu5Hv6m\nzYbHU5bwSInkhf39mPrg+5j96DpMXvy+7v6U3XflBR7Z7zkeu24OVN7ASuQPqjQ0gWsRJWX8gZSU\ndVnzcThoMZzPZwZcMo5EVyusPnWHpBaMBkB3xZLY19SFS6vCxFRxAa2NdSgs1g7I7GhrxSfrPtJc\nTlEKYq3wcNjz+6Mz2IXVy2uIF6rSUdVoyBmqqyZwyThCh/bqqhVehw3dGXz2GZqCm6LRwqW0W/pz\nc3iCRFEY8tgvxRlubdt2Yshjv5SpUAxF4dpAAAt9fuy0JbHFnkCcEGdgtqNO8E+9ung6Nqz5J/4d\nbECtLYWbvQHMzCNnWsXb2tG6eRtWLZmO5vWbTMUSKJ87x3H4ezKM75/RH6lgp2H3IgmC52l+dQKT\nhw4h/L3Mm8QB4O+7j2DBnGkyL1w8GgNz1uWGwZjW3L64+dF/g7HQ6fIpjUONrShJkfPHVFCqUhl6\nnQCgobkNhblKX1kGA38J8+6i0SSevGuq4bFP47+PUzJ9i/8SN09ipDlQq5fXIJZKyW7WrrvxZtR1\n6V+kAN4A7nVYEYwmMlKjEkl+yGsiyaKhK4Yky4HpjmFrQxcmlwZQ6HWgMb2/Pj4HkikW4IDmUNwU\nebJnuZCIx8CxKVC0RSzlDMp3I5lkcaAtjOc3HBHN4rXN3Vhxz3noiiSxankNkkkWV1b1xfOxJA58\n9g9wuVWgLPxbgwJvFlcSpPvW7BQfl64j/Nx4eD+c4WOobRhmeP7K7rvahi7Z79LAyVcenIlst11F\nYrLddjTt34yax1/B60/ejlyfixhpcPmi15Drc6GlM4Q8P78Ol0ohmYoit6QMlF1dXrzpj+tRXuDR\nVc/MwBncAtdI4645AYmOJhyu34Dysy/QXe+373yOvFgQCZ33irP7OIrLB2ouj0WjcGaRVSEHY4HH\nYUdnRMOwT9Hw5eThJ7/9AO88dSPsrbWqVQry87D3UCfmLX6X94/YnCpzb7HXhb0v/BqlfQslF10K\ndJZbdhHu47ChPhJDX6c5FfDK6v7448YDuMzqJrb0542vRiLJonvXLviHD8WCZe9h1dIaZFeNQJKj\nsGDZe3ht8XRxNhwAuCkaNwYCaONSeL0ziFELL8UVP7odiYP1ePPia4mmchJCFIeHLl2IDrcNju4o\nrgn44dS7wKbPObtqJNq28WrzpM/eQyLJomPTFuz4/i1kJUoRZ/AFG8VI2gaflf+cG3UvauHAL+7H\nJ43tuO/CalPrayHFcuiIxpHn49VGavPfkHQHYDvrct33DACRbD1zz7mIR/ik+0RzHf61aQfmXTVf\nTuQtVhWxj+UPBePy8uNZBJKl8DpZTChP67fvwrhzZ8of5FgkYnGxJMdkOPA3GokjEU8iO/vbm815\nGr3DKak8SWEyX+6EhggDmY9jEZBkOYQTKdAUhXAiJRs+rPQ29fE6MH/RWn6CPU2h0OuAz8F/uRnl\n9AwaPR6elm8A8OrSwDw3vy+aUpnJhU673356ACwHXL30PXGd0jPPh6+JH1UgxBOQvE1CCU+6zm8u\nGSn+/LsbLkCyswn9880pNcruO+nvpGgDJVq7Yug7fBJ+fdfFeHzF/yHevAuFdD18qTrkllaIahLH\nAS3Nx7Hk8uH48VQPrhweh4+tY5HS+gAAIABJREFUB2WxAVY1OaAoiPP59NLPjcAlo0hFgrD6C0xv\n4237GkWjNFq2JchqP4DiIeoxL9L3TGvTcfTRUZ6ikQgcDjV5Em48ljy3ARRNk++cORZOC4UHFlZq\nlkfy/R7UH6jVDTkcd9YZ2NXULiNOggfKfeWPINwwXTR1FD5vNT8Y22GhUUoz2MPKb5BEL9CStQAF\nBEaNQLQ7hNcWT0ciyWL+0vdhZWi8tli71JdNWfCjAQNw+/33YOFdv8ebGz/F5oAF6x1xLLpsAe4c\nPBZf79uDme++gqG//Tn22VLYaE9gvSOO9Y44aq1J5CQp3Gh14JrsgD5xgty/lF01ArljR2Pe4rWg\nLZRxLEEaR9kkOjkW544rkT1u1L1Iwj8bWlBzEi7s6482YUKJIttJY5wPCULjAf3xS2JcQVt3BNm+\nnlIZMXJAR2HKxOsEALVHjqGivEz2WNLmhdVuSxvHTQz8Jfz9T8cVfDdxSpMnrVIcCUbGca0xLQLM\njmMhkSot79TAnCyZt6khGMWq5TVgOQ4syyEST2LGwDzMGpynmfMkXCDLRozGga940tMdS+JAS4jf\nF8upzOSCAhVUhGl2x5IoLqtAd3M92ETclLdJWOfrA62oLPbjjJKAuP7zv7ofo8Pb8cRVww1Jh7L7\nTvm7Xg6UgJv+uAE/erMR77QNwsonf4nNbz2FF379c1C2HsNl3sBK9C8vwZghxXh3jxeP/+pB9Bk3\nF5a8gaBo9ftIWVLsjecJABxtX8I78tyMtomHgrB7jA28Hc3H4c+XDz1VZoPNWXAtbHbtc8/iYvB4\n1KqbcOPx8G2TEI1ENFuufW4Hwge3a15obFYGyaNfE3N4BL9K+YXX4lB7t/g4neWBtbgCCRZw9BsA\n1xU8gfI7behKpjK6qFw0thSbUzGkFAbIlo1bsWppDawMzUeauF3YMHs+OjZtEctiqlKfAvHWNiT3\nHcRHqx/C1WdNxXU2B27yBXCF34tx2W64y4tx/aKXYOtXgMn9++Javx83+7Nxsz8b12QHMC3fS+6o\nIpTjlP6llo1bsXpZDdgUp+vlEtDNsfgsFcG148rVCw26F5VoiycQYVmMmqCXf2TcPQkAm+tacN7M\nyfwvEl+cbjCmEgK5ikewr74ZFX0kZn0tkmTUTaeRmk8C39Qp+TuaLdmlHzfjjTqN7w5OybIdYFyK\nE9aR2TROkMAfC0bBdMc0iZNeWY+0DcWx+ORwu8w0LpAhO0NjxsA8LFjMP7+bH/kQT99znrg8mmRl\nJb9PDrcjt6gUA90JTB0+AGW5LuxvCWHVljoEdbrwnt9whPc7xZK4/swSVOS58Wnu9fjtK39Fu3WK\nobepPZzA1/Wd4vy6l5fMwKtLZ2JnfRCVJTn4ZbQSlndeRsA1Hm3d8u0pCmIJTvozCWaiDYR1pONe\nXnlwJnI8Ttm23Y5ipLyl+NPDY3R9TML3oJHqZYTC/n5EnSNh9ZJTwkml0QGuMFryjedjkQiEVNF8\nbVmNqXTx7q4uFBQWqj4zAH/jEU1xYKPad85utwsHgiburHXGaqxeNhMcI71QcWAsNFIcx3/Ol85A\nJF3CK8ty4FA4ijKXuXZ2iqIw2eLEp6koplqzUPniH5A7djRaNm7Fp5NmYtjvHhYJSWjffuz4/i2y\nMp0RlOtTFIUsUMgKhjHc6cW/Xl6Clo1bsSMYMieXp8tzwjlKy3GyYwkES2uGHUWJ67Ich3eSIdxc\nXc5HHijWY7y+jMjT3xpacUvNWL0nAddlt8FZUoHIkVqE3lgBEPxhLaEosp38jEhuzCxYc/si0VLf\nQ5h0FCctvL91D2686bqeMp1OGc7etBuwWImlOWJJD1CV/5raOpDrl5AeilZ10ZFKdrwyZUciFhOJ\n1urlNeAStGE21Gn8b3HKKk9GpbhMVKlMoKc4ZVLWK/Y68N6qFzChr0tl8o0mWXRGk6IqFYol8fQ9\n56ExGMW4Ih8uGlqAqf2zVeGZY6dfhIKCQnGuXVmOS/ZVJZjIpT8LKpR0Ht6k0UNRNHA4AHnStXQ/\n0oynu/76FbYf68SrS2diR10nrnxhA+766w5sO9qB1x+5FKOmXIBI/QZ+W6FrjoZsZMlzGt11vYHR\nXDnAWMmSdgACwLRl5HXNnqezoEyViwVol0brtn2OolFnGe433HYcgXz5TLreRBUwDIN+fftqfmYo\nmgab0t6P1+NBV8h4zI+K7CnGajBsAvF0mCcb7kLk8D5wLIfVy2pkJvILz6nEl9KuO4oybLMfO7YY\nHVwKMZ9HHlvAsti28Aa5wqQzUFfjiZkbwJs+V12DN0XBVVGunRIuPRbHId7SqkmcpLEL/0iFMdXi\nhEv59zWYE0jC/u4ICuxWuO3aqozZrKe3dx/GwkvONzX8V5lQT/qZZTmE4wlY+58hK9PpluFIniYN\ntYpU/vtk43acNf17AOQKErGLTgBFg2KsvGLFWBFPpIhxBafx3cQpqzwB/B2xUnESPvdGqpQSLc3N\nSNBOWG365TstmC3rAT1E6197rBi+ZT0cxaOIFzipKuVzMIglWVw0tEBUFITZeMIFkqIoRJMsca6d\nYCKvyHOjtpkvjQg/P7/hiGoeXla/wagCsO1Aq6iICKTp3umDUdXPj21HO3Dfmp3gANz91x0q9URq\nKveOPAd+isKymUMwtiIXXZEEAm4bdtS2YGxFLkABVy3hlaKKQg/uuXg4xlbkYmNtC258dn3GquGN\nz65HttuOtu4Y/njTeIxNp3t//+n/pGca6itZ0nLdKw/OJB5fIFh651nY3y8SJOVrBsiT219dOlN8\nvVx5fWFTXGwo8H42aQn2rJEDwKb6q85NqWgaYfLUqfA57ZqfGYqidMtkHo8bQQPylO1xobUrhFyv\n4nlt/huSNieoeASDc/3Y09yJkYU8ueADFz3o6dziDeRudCEiJKanL/7eocMR3P016h7SDnxcOKY/\nXt58CGMJrfsZkaVMICU7WoqSoBC1tYvLI8Fuw2gFPUj9UZdXhTHAF8CYIWrSJu20W/XgdBw3UKA4\njsO/mtrxwBx9cm+c9UQhRNvAshz8LqeKSFMKxUmqSgEg/kxt/hu2HDiGMYMHgHF5cfOj/8Yz95wr\nU6DMIpZTAYai8MqSGT1qlYRQrV42U9zv/qP1uLisv6xUZ0ZBYix8ufjVpTNhs9LgQm1gSEO3T+M7\nh1NWeRIg/Y4U1Ca7xZKxQXx/7T407fvqhM7FbC6UQLTeefo2/GfDl7oXOKE8N2NgniptfN2hdmKc\nwdbGLtVcO6myNDDPjYHpn6VmctI8PCGkUSAAq687U+ZrEnxQ0nEiAqSPURQFf5YV4wbyhCTgtuHm\nRz4UR5Zs3McrRcFwHG/cPQXVA3Jw9dLe+4yEuXIDCntiD6orcvDirRNNVU3MmNSN/FDCqBo97xgp\n9qGqPAf9zpgs25dAfpWjeBirDTaC0RuAKgZDD0ZKLkVR4ngSEmw2OxIJ/ZDWgX1ysa9BI+MqfaE8\n+9zx2N4gJQoc2HBQJE7SEM1Chx31kVhGbfYeK4N+NIPXrv6BoZfp2wAxsFKiEFW9+py43Ol1Y8Ps\n+b0+R8Ef9YOzaTTu3I3zCMQJIHTaGYRl/qctiDOzPbCYaJrRznri/5bvuyuw8MFHeh7V8jgpVCmt\nn2Fz4uOd+zHtexcgEU/imXvPQyKezDjjSSRJi9aCsdA9HaQEj1QqxYKmKVAUhaTVjWSKNacgScp6\nHMchnkjysyNNzME7jf89TvivFI1Gceedd2L69Om44IILsG7dOuJ6X375JaqqqjBnzhzMnj0bV1xx\nxYkeWgapB8phYxBL6RvElag6Ywy2bNZuKTYLsynkx4JR1LaFwbh8CLZphyYapY0rvVIClGnjUmWp\nO8p/4Fctl6tTQglPWt4TICUAyRSr64PSgpQotHXF8NTd54ojS258dj0ue/xjeF02XLXkXTAWGi8v\n6b3PSFCF/nLXFHRHEli9vAY7altQVZZtmowZlfb0CJZ0xp9RLhapNKqElPyaHcUjmMan9HUilTRO\nnzdqqqA04kHEMoWL7OkSMOzsqdjfQJ71JiDgzkKXxvVGWQaae8FEfNramXGb/UXj+mNvKo6mlmbd\n9b4NkAIr5R10I9G27SuZ/+pE8MerrsX6exZhyh7tRgtA0mlnEJaZYFnsCoYw/ZxRJs+AnPVEZ7mB\nvCK89M5m9BtWJS/B6cQRCKqU1s+d7W1w2qyw2p2wpq8HVhuTebq4jCR1ysiXsvy37ZtajB46UFSd\nrl76HgB+0K8RhHVuf+wjvqnC5lOZxk+PZ9GGWf5x/PhxXH311aiursall14qW/bhhx9i7ty5mDVr\nFmbNmoUXX3zR1LFPuGz3wgsvwO124/3338fhw4exYMECfPDBB3A61XfDFRUV+Mtf/nKihyTiROMI\n7HY7opFIz/T4/wKSLIepM7+Hde/+Hd5plxLXMeNfURrHAX5Q7CaFIvX8hiMo8Nhx55QBuGrJWryy\ntAZ/2iYfxqos7z2/4QiqynOw7UCrjAA8/P4eInHSyoUScP+anSjJycLh1jDirRHRLJ7jseOei4eD\nYzmsWlaDDfuace+rWw2Jk5bRXKoKrV5eg85QDMPKcrDtYJtpMqZV2pMeUygPyjoFU3G0ffEmsifM\nFR+TljB7A2VZ1WgUj5R0zxhwHEMnnAO3Xz9IEdBuqkglU6AtEoKeNsRKyxRj/XZijo6AvIAPLUH9\nCwo3ZhZcO7vhmHMhom8+AwBizpOyDORm4wgn+fiPuod+ZlhukuL6seV4auN+zGPcavP0twyluVxJ\nqDI1q5PAcRz+nYrAR9GYNTzfzAZIdnaA8fl1S3h/a2jFBYXSkTPqHC4zYMNdePWZp/Da7+9Tl+g0\n8pyk5V0AxJ/f3rATl8+7Qm0Qz+jseOgZyaXv8U8378APbrlDVJJWLSMYxClC+U5iKn/m3vOIpnGH\n0waH45R213yrMMs/XC4X7rjjDoRCIfz+97+XLcvLy8Ozzz6LvLw8dHd3Y+7cuaisrMSYMWN0j33C\nytPatWtx5ZVXAgBKS0sxYsQIfPLJJ8R1v+28CqM7ZyNUjqpCy/6vT+gcMk0gD+TkIlfRZq6EctSG\nFEplSlCgvvr8QyQi8pBIDkBjVwy1zd14Zan6IkwBKPDYVQoH13kcZY4umUJCIgE0gMcuqdSceUcB\neGj2CDw7/wwsKAkj1vqVqBD9a8n5qB6Qg2uXv4cUy2kSJ6lBWznWRXoNbO2KYduhNlFx8mbZ8fWR\nDlSVZZvKa1IeR+uYwrHE15jjYD32AbwjpqpeeyEXKzvLKprHlYbxMeU5KNQwoCvLqtWE6AoBUtLd\ncLwZLp9x/o8eIpEwXFl8DpaspVpSemATMd3yiOFNSbo0897OJPZ2hEBneVSz7vgy0GJE/rESADAp\n14dPWzr4i38Gs9kcFhrnWJx4JxmSfy+ZSOs+YRDM5TJTeaZmdQVYjsPbyRCKaQazxvU33kBittdT\n8RoifNTD6ImCUZoynEWohXA8ifrPP0DxoU9lJTrlbDoVpKRK8XOKZdHY3oWiAl4BFRQiAOp8JwFG\nipRBuY9lWUSiMWRpBMwCis9LOg5FeCzhCKS77eJg4kH5jDvg9HgWA5jlH263G9XV1URRp7KyEnl5\neeJ65eXlqK+vV62nxAmTp/r6evTt21f8vU+fPmhoaCCue/jwYcydOxdXXHEF1qxZo7vfYDCIY8eO\nyf5p7Vf6nXwi/GzS1HPw6cfrer19sdeBQbluFGc4yHHS+TMxsqDHREvyqWj5orSUqf7DqhDZ/gFx\nG5K3SVCc7pwyAKFoQlQ4rqzqi4eunAzHvk/AJuKaygkF4LFLRmJUsU8zF0pa+rvq4ulw0ilkh7ei\nekCOWKpbuVi7VKckLtJIApLn6O6XNyORZDG8PAfJFIvhpX5TeU3S4zx/8/iMjulo/gzuIRPBeNQX\nYIEovX79eKz6/jg8MnsEAgo/1M+nDcLdUwdg8bSBqg+nUFYVkEzof7ELJd5gPGlIXOqOHRWfO/E1\n6W7hv3gI2TVCR5E90oaUTkceYPD5TJdm/vr72/DJP/8OgCN2azkvvFq8YE+cNBL7QxFwQMYdY2PG\nFmOoxYZ/p9IXYY2hwP8VKEzlvSVwcY7Dn5PdGGtxYOrYEuMNaBrFix6RvW6ksEyW4/B2QwtuuKAn\nmsBsNx0Jb359CHOH95cTIKmvKa8o48HA/95Ri3MqK1SPa4VgEkMzM8TGnXswbmR6e1Kuk+IxypUt\nEqZ5i9bCZmVw8yMfwmq3yT5LTDyoGs/S0NCguiYGzcSDfMdwMp9HJvzDDPbv348dO3Zg/Pjxhusa\nkqe5c+diwoQJsn/jx4/HhAkTwLLm2ymHDx+OdevW4c0338Tjjz+OFStW4IsvvtBcf+XKlTjvvPNk\n/xYsWAAAyMmREo2TF0lgtVoxYaJxezgJvU0gV0IZbmgGJGXKE8gBTdOIdKj9VMqLMCD31LgcVvz2\n4/3407Z6VOS5cc3S93HPnXfAtZ9MxgCeGA3v68OO2hasXl6Dbxq7VERL6f2xVExA0lOIu3/+IP54\n7xRs2Nus6zFSGrQ5Tj+DqSUYw6b9reA4YNP+VtGUbuSjkh6nemCu6WNmdW+HLbcIjoIy1T4pAKU5\nWajq58f8xXzye1U/PzhAfE3WbdyGr7Z+iXmL1sLrtCLfIALhw9XPay4T3kfjinymFN931ryl+1lK\nJBKwWa3aE+A5Fl53FoLd6pE4UnizHOgM6SdFOza+ic6tnxG7tUgX7JE+N3anqIxnswHA1LEl8FI0\nNqSiZDP3fxtmCZx0Jl8aXRyLPye7MYPJQvXYYlPHKv7Zw/AOGYavDrb3vG6EsMx/NLZiWn4ANskc\nUOnfJ1p30HTprjuWQHc8gaFnKUa6SHxNySSrH1lAwObaozjrAgXZ0grB7MXsOhI+2bQdE2tm87+Q\nPhuSx5IpViRMiTg/siWeSIolO+lnSYDHbYc1/XlcsGCB6pq4cuXKXp33/xKZPI+TxT/MoKmpCbfd\ndhuWLFkiKlF6MCymvvnmm7rLi4qKUF9fj0CA/6JpaGggsjaXqyfCv7i4GNOmTcOWLVswYcIE4n6v\nueYazJkzR/aYJf3BbW3tBqy0qaBMJUghgFJUjzsTAEzNupNCGVXQG0hLcKvS4YZaipNymfRnYdmE\nCy/Fur+8jKwpV6pa3JVQemoa06RA+pinZDCyOr5Gi3+4avuOcAI764MYUZ6DjnAcQwo9eGT2CFlb\nPqD2/jiLh2JHdyHOu+bHcI6agdYu7agIkkGb5DkSQFHA/au2it4lwask5D61h2LEmXWy4+zjyafR\nMXMCCURCFFzlZ6jPAz1RBcEI/x5JpljsqOtEezghviaRjX/Fn36zDKsnOhFPsAjFtcvP0a4OuDRK\nVNL30QsPnAOHTrI4wJceaIrS/SwlkkkwDP91wcSD4BK0KvQv4PWgrbMLAZ92onRFn1zsb2zFGQN0\nLu7xCPJcDjR1R5D/xgoxGBMgt79feG4VnqqncCltwctLZiC4e2dGYY8XjeuPP284iPVNDRhCiDD4\nb0JK4FYtmU72Pilm8nVs2oL3r7kBHybDuK26HFkmbyQZrw+eQYPFv3nwm13E1+1AKIIUx2HipBGq\nZaE3ngauuAPO4jJwl91OCMKkweQWINnSowb8ZedBfP/S6cRzEnxN3MhpmpEFJOypa8Kgojyiwqrl\nXcp0dp0SiXQThtXaQ7xInw0mHgSXtIC1eUXCZI11govTsHIsuJj6sySgubkLNE0hJ8eN1157DamU\n/DvB6/320si/aQnpfgdlCpfNgmGF3oyex8niH0ZobW3FD37wA9xwww2YMWOGqW1O2Ik2Y8YMvP76\n61i2bBkOHTqEnTt34je/+Y1qvebmZpHNdXR04LPPPsOdd96puV+v12v4xtAyiWsRJAdjgcPGIBpP\n9toXpQchgbzQbcegXHfGA4SFcS3CsN5xRT6iz4lkEAf4i+a4Ih8KvQ40dcWw7hDgzy/ErMI4zqoe\nJhrAtbjjCxuOIN9jF4kT0JNA3hVLos+IM7Fr7WuoKBmI2iBPcqTZTyP6erGnMYjBhV5ZblFHOCES\nJlKkgcUdgOfs+eBSCRT6HGg81PMFrjSEK4mLnqmblMEkzX1KJFNw2hm0dcUw9cH3IdzIaJEurWMK\nnXVWH/luRZnldONrm9ERSYivAwfAz7YhlV2IunAKFS7gYHuYSHaFrCfm4F4UVo0jHk9ayv1q1x4M\nHFZJXE9AV2cHfP6AbsMF30ghfUD9Ze/3edAe1Fcfhk6chI/XrpWTJ4JB+IIZZ+O9Dz7HJSPKVIpG\n6I0ViKSznwCAcXlQNWk0Zt36B7zz1I04vuJR3XMg4fIzy/DnDQfxx6uuxeT8vv8T4gQA8bZ2RILd\nWL28BpH2IOJt6s+/QLDmLebzgbZ2tmCLjcKdYypgyaDUKPc37cKx5feo1gknU3j/eBt+Pvdswh4o\nMLkFcBb3x7wl78kS4HnQ8N76S9hcLsRDIQSf/hnaIxEkWRYFfp2RLfGIyhxuhL9v3IW7fnyb9goS\nciSkhiMUhOXIll4RJwD4YtsuTKxSE0rlZ0NIEadiMXChIKzCh0uqNJEM5Qr06dNHd/mpgpP5PMzy\nDwEcx6mU+Pb2dvzgBz/AwoULcckll5g+9gl7nq677jp0dnZi+vTpuOWWW7B8+XJkpY2lTz75JF5/\n/XUAwPvvv4/vfe97mDNnDq666irMmTMH556b2awvEpSRBFqlB2WUgdF3jNGsOz2cSPnuy7pOcOlh\nvVIDuACSQdzB0GKZpo9PWGbH1P4BnF1zMbqa63Vb3IVoguvSnidpjpCyxDfk/MuRjEbE7ZTZT4ML\nvdhZHxRLcx3hhOZwYdk50BbQVt4rJpARkiFcIEBGqd5aGUzC4w/84TM47fz7IdtjR3mBR3bMfy0+\nHw8vGC0eU6vMJ4RgktLDBSjLlVLiJODwhg/Q/8xpRD+a+BqhJ+vJ1lmPghLCfLI0hFIuk9sXVeP0\n78TaW1uQk5uj23BhpgPVVzrUkDwV5eeivq1nqK/MICxBca4fDUFl6GbPnDSp74kNd2N61XCMK2wy\nFVWghcvPLIMNFN4+fhQcRcE1cAB5RS1P0kkwm9uyA3B63dh1sA3OgBdVrz4nL91JZvL93/1T8cRv\nf4uWXXtwXWVfnjiZSFqXQvA3kYgTx3FYdfQ4fnjBOEJHIm8W919zL+KRCDEIk8ktgM3l4ktVLheY\n3AK88dVBXHf5THMnZ5I4tXWHkWW3wW4zUXpTlutOABt27MKY8y7UX0nheUoyblUUwemZdr2HWf7B\nsiymTJmCO++8E3v37sXUqVPx1FN89thzzz2Hw4cP4/XXX8fs2bMxZ84cvPXWW4bHPmHlyel04okn\nniAuu+OOO8SfFyxYIHqWThaUSpJeGe9EowzMIpOkcSWaGupxbPsWNBbN0YwmUBrExxX50MfrQDLF\nYuGStVi5eIbYYTZiQC4sFgajzzoXNTUO7CO0uAsX5IF5bqRSrBhhoFXmoxkr/MXlqAJwuDEoKirC\nTLttR+UXroBGgrYRCvv7EciyylK+lYnhG2tbcP+qrWgJqomNVgaT8PivbjkbkRj/fmjriomlO2Wy\nuFZJUCR40E4Pl+K+NTtFhe5P150pW7fE0oKuvmWgGSvRjybAY2dQnp2FVDIJl9UCp9ViGLCqBYam\nkGQ5MDQFa6wLgQB/0df6XDAWi1im0ILP68GBLn3PE221g6PTXzuKuXZJhQLlsDKIJJJwWhnI5qQd\nO6RSO6JvPoOibXvwzw0fYZTfvHFZiQvHlWL95jp8Mvd83HXvPbCmKHxx1vmQypJaCeFas+hUkMyb\nUyLe2oa2bV9hWHUV5i1aKy/dSY6x9sWX8fb11+MCdwADB7jE/ZpNWhehMwz4742tmJjjQ8CpvlGR\nes9WL52BjpWPyEpzAJBsaUA8FOJLVaEQ9u7dA4/dioA7S/+cMsSb/9mB+VfNN7eyzoy7TJBKsWBZ\nTixla0Lhg7La7ZiftmVwCf7G+PRMu97DLP+gaRoff/wxcb177rkH99yjvnkwwikbZUpSkowIUqZR\nBsq6rAAjNcls0rgS+X36Yv+e3fjoQLNmNAHQoyp8WdeJQq8D8xevBcPQeHVpDZpDcTQGoxgxIFck\nX8L6JDVDMIovSO+DFGGghdJCr6ioCDPtHn5/j6x7TGqI7k2o5q9+twKP3DBCJEACubl66buoHpCD\nfxFiCgRohVwKj5/5wFrMfnQdJi9+X1xmlCye7YuCObIWbJz/2+qlh0vBpf8p16UAcN1tKBl7jrgu\nKaRU2AfD0Ljip6swafIkMy8hEUJX6IBAFgbluuHOcqKgkByXIbyuNpsN8bj+387jcfPz7TTMt7H8\noUiVjgFKKnmlSRF8qFQaxvfLw4ajzaCzPHKjeHF/ROsOKtQODrMmDMKGUNT8DYuGSjPl4on4yT33\nYOaVd2HTzu1wDehpANAylZs2m5swhG9beANaN23DqiXT0bbtK5Fk2bIDYEuLcNZFNyGVn42fTBre\nQ5wActJ6hkqUgC/bgnBaaEydMpK4XOk9UxInAcGnf4aOFx9Cx4r78JevDuL6eRfoH1hpDjf4PZb4\nf+x9d3wUdf7+M7MzW7IlPRASSkgIHelVFOkB6UpV1LPBgXqK7U5Bimc7PX/nqeipp6ICdhExgjQV\nBaRI76EmpCekbp/5/bE7uzOzUzfBE755Xi9fkpnPfKbs7M4z7/fzft4+XKpzIUWDwJeDYo87jcLx\nXw8eQ99u4So7JVCe6kDblaAVwYqnApoaH22TL8Bowh8eV6z7lhxRkup3J95OC6ouXcKmzz7ByGm3\nC5anO8xwmGlVPRP3A8694WvFkFFjcXbvVlgHSAsqOXBRhZCXT5ULvxZUCRzHxYJyKeNMvlD8REkt\nVu+7qIk4cVh9oDDCMJMjS/svXEKshW6QQeQPbEdsWLwcdXX1YOJ7oxyx2HWqDCueGg0DSWDmolzZ\nCJFcuo2/XCwWByJ1VQDMQLeDAAAgAElEQVTA+lwwlf6C+ooYJAycApIKpHXV3MP5iEjf1Xvx6owe\nyEoeHNKjAZEmpSwChGp69xbw+Rh8+f9m42KVC+dlyLUSuKrQuS9swhuPDQu98VY53ZLNtbnIrtcc\nB6ZeQQdEkLDbbKhgY+Bv1VOyCz1ljcWMhbm4trkFHkssTEaLorZl4LBBeHHbWdw0dxmc50/BeT4v\n9LCu+/Q11Av0NQRsU+fhvkkE1nywAqPz9il/2RWiNO78czCTBmzPfQefr/4Y3x/ZjxF0DFISkyQd\nwgFp53ApaBKEsyz23XoPun/wHyR074pu772BXQ8+hm+KLyD1//0L61e9BObCORRUVwk2k2qzojsS\nBeBUbT3O1Llw33jldG+dSMwvDQa+skKsOXION3ZoBdogL2bn964j9qxV/RsIaJ3G3jZX+p5TgkTE\nidNCaZnnh1378MAjj4f0TF63W7r5LyAc460FYUoURJrkCjCa8MfGFR15kosk8YXj/P/rQWxcHGpq\nauD1hqvu9NoRROP7lNWhEyryz2B0ZrwmuwK+TYFc9R0fUsaKfJ2NHuIEBKIhaQlC8fjjXx3CzHd2\nIiPJirdv6YVP7uqHqiidtQnaCPM1OYjrORqW2gOgCzbg8VVbMWzJ99hxolST9YBe8MkVy7KweU+C\nLtgcaG7cc3SIOAGRtoBqtxrfaDQuho4wJOUigbc+FVjWLKjt4pbftnQ9GDagjVOCmSIF3mEcuLTy\n8keHweXxY+VSuV52wsguRRlkS4M5zQZhTQJDUtLl334vfHVVWLUsB507d8G+n7eGI00y2hbSHANL\nUgqmPrE2kK5b9z6vT5qw7QcXmXpyxSm07NwFR7zKJEGtH97pP89C3qPz0PHrD3BXn7Y4NWEkDs2a\ngIzXXsKBP/1Zsi+ewOhSBp6KSkH7FVmSFR+HhO5dMfHBd/Hetq04NX0c7nv/Tdx48QgKH58n8GDi\ng+/RpKfnH4cilwc/lFVh3rh+EmvDmrMAxK1XxOuDc9bUo8LpxsDhCjYwot51sMUr/220wM8wOFFU\nia4Drmuw5YAe64KyyirE2qygjaZIXycxxN5PgKzVRxOuLFyR5MlkpkOicLkXKU44HkNTUftAjRw1\nGod/3hT6W4+eKVrfJ4okMGvGdIyZ/ZSkYFwKStoWMUoLIlN3SjobNbAsi8Nr30eHBDYkDH9uYhc4\nLDTirUbMXJSLeKsRrRIbpnMwmK2I7zMO8X0ngGX8oBMtWPTdMUVfqIaieZs4pKQaQdImJF0/E5Qt\nknhqTdtJoXVzR0TLFS4S+P6iUWAYBguGBAT8tVyEcIl8mx4OXPGApzRf0uOJSyvnVdbjRFmtZBpb\nHNklSAN8UponwcPBCMbrjvTUCcJUfgqGc3vQO4nAwQ2fq/v3eJy4tlM2bh9kCKbnqmWjHPw00ti+\n3fHLhYuoUmhUrNoPj2Hgzj8HAHAkJODRZ/+OjacTsGb/bqwz+HCutCRyUjVn8GDKjutfd+COuRJT\nsLjAePFV8QUsfmgBbr3Wgfvmz8f28vbIGDAIlN2hLIjnaZj09vyr8Hjx5cVSPDx+gKxAXN5NXHo9\ny7L4aN8pzJt1o+K+I1K4tZXKf3ucyN1zFDk9sqR9nPRC7AelgM/W/4BJs27XlnKTGBMywtTQ+64J\nf1xckWk7k0nZ24l7a57zvDA1ocUHio8OnTrj22/W4prrR4eqjTg7ArVUXLTCcR/DIr1lBhbNGaX6\nkIwGlcUXYXLWw53eo1HmIwgCXcbdjqNfv4NWOZ1DwvBLTi8q6zyh/P7cwW1lxdR6QNImGBPTAARI\nH51oQfPEwEO4/txBGGIcqKi1gwi+Bcr1vuODZRkwNUVI7ZgJkhaKYw1mK2LayJf660nb8cXlp8rq\n8M7O8/h438UI8rp630UsHJENlmUxc2EuPloaEPC/vfM8BmUmKt4T/GrM3nFHcOPseyXvPW6Z0n0p\nToFLVtyJHg5UXZlk+Tc/JRLrP4rq2JagbpgtSMEIEBSPd2fzsez9NejbN1v2ODnw00j339gP/1y7\nA3dntJAt39faD48jIZ+9MBU1J4/jxM6NWLfnPH7xu0ABaEnSaEVQSCRIxapEuZSdi2VwhPHiLBOw\n8kgnKUzr1Qa28jxQW8vg7txBnQARBChHbKBFDe+cQucYbF0jt32V14dVF0rw1wkDQBsiX9jEAnFx\nui5yfcBK4us9hzEsKw1mrhJOpm8dwOtdJ/47OJ7/t59hcOBsIabMmgUo9aDTAc4PypeYJUwD8vo1\nMgyDyuoaJAWrKrWk3ChPNbyIDaTu4ADlqYaPtqmm+5rwx4Zh8eLFi//XB6EXTqcX00a0h8vjk//x\nZ4GJ12fB4/Vj2vBs5bEKqK+rA9x1sCaEm2tqnaba7UOl04tLLn1RnWq3D7HN0lDv9aOkTtqs00yR\nUZ1PYmo69m7+FlRyK1BG6XQiJ1j2qLTa4EAaKMRndMLW1W/h+Uem4XSlB2sOFGLzsRLc1LMlZi3K\nxZ8ndME3hwrh8mong1waUOs2BG2Cp+QcULIfhupToN0XsWBsRzxz+/Xo2y4Z3+zJBwAwtSWw+c8C\npQdgqD4FQ+1ZxNhoGBNSQRj0v09sOlaCbw4VYu3BIsVx8TE05g/Jwswnv8F17Wj0bd8K47ukooXD\nhN94aTiPn0GbeAvizDRuHp6Nk6W12B7UN6WopIB9DItEC43bRnfE1h9/Qnb3PorjAcBhUo/KFhcV\ngnHVIqN1ZMsP0u8GvC6Qfhd2/rwNA68Rtbsw0CCS22DGou8wdVRnkPUV2H48H//+rhpTx/YAc+EI\n4A9/R9he40B3vhb+uBYgCk/g6JGTSLKaYTOpR/XYYJrdaDDAUlGHTSWV6Oywyo5n3ApFHUGxNeN2\noebnLTB37gF7dnvYOl+Dluf2YWCnDFyTZEFlYQ3yGC/2MW4cYzw4ynhwgfWhkvWjDgzqWRYulkFd\nfT2Ytq0xZFBL/PT5l9i48Xsc9HtwnvWhJUlhUt826NcyER3S4mAkydDx1WzbjEubclG1cZ3scab9\n9e9ocds9cIwcj5SJU2Hq2A012zYDLAvG4w6tDy3nodLjxUcXSvDI+P6IMRpBxjhA0MbQteSuK5Ge\nhekT+8J5/hQ8+7dFXPfw+jzQ2degrvMg7Mw7j+lB0Xnoc41PB3HxeOR5BM0x+Z89/74AEPr7m11H\n0LtdS6RnBUm1mLwY6PAy/r/VQBoE96rXFAsiqQ28RgeoujJs23sQ6c1T0Kod/x5Xc2YmYYixY8bC\nXEwd0QHwuWGw8P72umTncDNGEASBmJjorXOixdGSWnj9jVeWbjSQ6Jii4O91heGKjDy5XV64GOUo\nEv+t2emL3prg+qHDsGunclpISRQeDcFR207OJFMrbrj5Nmz46E2kjrwj4k2Zsy4QC5bFY+wmShAx\noS1W+PrdhKXPPo+/PvxgyF1834XKqCrttFoA8EFZ42DL7gdbdkCvYSe9SLMTmP3UN1j97CR06JiM\nynovnAXFMFgyYGs/AATZ8My1lPGnFLgo1YBmp1HmysKgRCtuWZSLD5dGWkO8vfN8xDW2lRyGO6Un\nTBblFOiP5ypBMT7Ua/SBPXM6DxltZTyNgmAYFqTStQo+nExGGi6PB2Zj+MfenZgFiiDwwVOjAuXh\nHicSrWa8PKt7pIO0hH3B9EnD8d6n63F7L/XoEx99r+2EMxv2YFdlNfrE6/TQEQnKi1//B+zZ7TFz\n8QasXDwSxBPPwZ7dHtVHD8Pw7BPoL/qBqfH5UO724eThYlTDBzdYMCxw9KFHEWu3IyPdgmm9MxCb\nkKDuS6VgJwCE9VtzX9wairSvXDwyFFXja5/4ywHgotONrwvL8PiEAbDQNKw3zwOdngXKQMJ57mRI\nXwaoC8S59QAQd+8STBp7J7Z+9Qaw/RMACH+uy3Lg7TMBxK414VMMisFBEIFMgYR1BQefn8Ghc4W4\n+ZZZgqgQB36UE4BmETgAYfquvgZUjD10L8JAY9ueg1jw1yfV5+FDHJ1l/MK/m/ROVySuSM0ToK1q\nju/xFC0oisKAQVLuugFE2wxYK8SiXymTTC3gjzNbbeh5wxj4Dm8SjCEANLObIkTM4jF39WuFvw7N\nwqKR2QJDzfi4WDyzdDHmPpOL7q3iERcUj3MCaT0QN8zVoyXiUMPQiG3bHaufnSQgb5a09jAmtNBM\nnNRMMPXgnx99hd9qzPg6n43QO/HBIhCB5C8/e2S/KnHiwBTnIbuzdJm5YBzDIHfdN6rjWLDqangA\nSfGxKOOZYMJoCQhxF+aCMpAwlZ8CAPTPTMauFf+MTNlJ2BfEWS2odXvhCwnWpYXJUpg2sheOVNej\n2KWv3ZJYbA2WDemHak/nhYiUnBDbTlFoYzVjRN/WuLFvG0zpm4Gb+2Vgat82GNUxEdkOK7IXPqer\nkbEcuLTi8oeHwFVdE5Hik9M+naipx3fFFfjbxEGw0FQo9UaQBOa8sAmWVpmiZr+cQFzu+gfWM/U1\n+O+/X8Ybzz0EsyUQTYLHCW95UciDjk5oHta88Qizz8fIWldw+HLHQUzs31W6sS9X1bnoO1DW2Kj6\n14WsDIqPCHRQlRUVcNisyi8RMhA0/JX4uwlXHq7IyNPvBX6blzS7MaLfnYkiQ6LwlUtzFLVQei0L\n5CA2ydSiiZKKVKW364ji86fRtZkZB4pdgohTncsr+1Dnqr4g0uNwYueCWj8+/9cdOFVai9bNHag8\nXa454kQAoXYuj41sD5IgsOKpUbqjVnw0xCaBOya9ETCpOeJiaMSxFbh4Ig8dR88EgYC2CdAm1q+v\nLIUtTrt79ZH9v2HslOmq42qqLoX6QimBZVmQGthTcnwcSiurkN48ORQB8Hp8wQdQVUiX0j6jJb75\n6mvkXBMZ8ZKyLxjUphl+OVeM6zLSYJ12PyzpbUKWBWppkwfG9cPfv/wF92Skgtb44JMiHAXPPRmK\nOLnr6jULsaWgFA2KAOfVpBCBEmibJOYS67t2lFfhgtONxyYODEWfOdE9lZaFNx4bBk9dHZh6saiZ\nZ1Yqc/2Pl16Cq7QA3f/ysCCKROxeA2+fCeiW2VwYcRQRZt/BjbJtWarqnDhbUoFb7roL/iAx4qJC\n8HvhTswC/EygEW9tgMRHZYgZHMvvi/ffbzdj4ozZ2ucQQxxhaoo4XdG4YiNPlxtKHeaBQMQpM8EK\nt9evKgpvaHQq0yZ8aPHtCdSgFKnqNWwsjObA2x9Him5ZlAurmcbLP+RJmmpy1WAMw2LlspwI13Jx\ne5HubRM1nSNHUlbf2Q8vTumG7i3jMHNRLkiSwHMbJPQRGqE1pSYHcTVda51Vg9x5LZ+cjZhTP6Hj\nyGmCNivTu7fQEtCB/8Qv6DlUxWCQh4zs9oixqTttV5SVIkmDwSDLMJreuOPbdkb5pSpB6TdtpGAo\nOCBIm1AGeesDABFRhyEjB2NfaS1s0++DpVXbgFlmqyxRZEQaRoMBN6UlY+UFiQo5BfDL/gGAsjtC\nESeTNQan/3q/rGWAABJGlZor4YLpw+zXPkDGv1cg7W/PgIqTILscsWLZkDhcaj3LslhbWAann8G8\ncf0j0vbOdStAGxBoqWKxRFxfgVmpxPV3eX1Ye/Q87rl5uKQBKrFrDXxbVoA4uFF4mnvWBpbvWavY\nluWd73di3py7Iqvj/N7QPTd76XqAZWEqP6VsiKkVfi9q652oc7rQLEXmu6JilNmEqw9Nn7gE5Prg\ncf3u+DYEJtqAvIo6WcPMaC0LOLAsiw/ffAV+UZm41io8tUiVmSLRu3W8wCzzVGmtoDkwB07r9PbO\n83h28yks3RDZg03K9qB720TJknk++CSlSwsHDl+sitBKNTR9Fs32/Gq6aqcXb87sGdGjT2le7rzu\nWPgJljzxV9gtJgFRles3yIfP7YTX40KMXd2nh0PvAdocyMn6S0hKTlEdF2gMrH7vJiUmoKS8MvLh\nJvFAdMSYUVWnrX8Zeo9H1thpyHcTOHCqLNA4N/+sikFjGF0HdEAnRwy2lOrQCIoiPWLCw1kZKCJI\nfqTSc2JyJgUuQjUj+CIR27kb2v37PflUn8L+fAyLFeeL0SbGjGmjeknuj6mvFjiHi9N0Ymdx8fV/\na9dx3DdrLEiSEBIiHtiuwwP9DHtPEO5cpZfd7lMXkNE8EfGxgWOJIEbBe27lEpFtQQOr8ABg1bpN\nuHn2HZLrLndvOpO5cSQDTWhcNKXtJCDnXu73++HzeQGKFtgQuBWITEN63QGB8vDRk6Zi37Z1yBgy\nQX0DCfx4rjLCcRwQpvOAQNRIrqedFiG5HDz1NSD3fwl/5zGh5r9iiEv+xem2aNNn/FRgtOm3x786\nhNaJMXhzZs+IHn1qx1VZ78Wpsjp89docQRpUSe8kBnV+D/qM1P7ZS5ljyqHw4kUMGnyd6jiGZWHQ\nEHlKTkpEaSXfwFOecPVp1wq/nryAEd1VhOBBTcw3x+NRXPIZHnn4Ybgu5KHuY+meVnIYO7Q7Xlu7\nA2fqnMiwxUimt9Sg1d6Ag2J6TkUIDoQJ26qlOfD6GTBgMXuJfKpPbn91Pj9WnC/CuNQk9BzYUWZv\nAQiF4ZFpOjnh+DdHz6NfyxQ0i+PpoTxOoT0BvyBAQjguB4/Pj293H8XTCx8RrhB7iTWSbQEfTpcb\nFVU1SEtNDS8kAj3ofMZY0CYjDpwqQ7esJPXedISG3nW8MQRBwKTyctWE/w3+T0SeGqDFFMyRf/48\nfl0f+KLr6V8Xba87Dq0yMuF2OZHkVTDgU4FUxEmczusVjEBJQW+0hA9jjB3th08Bse9LtHPI/6jx\nxeXidFs0ZpRSqcBoBOgsgLPl9ZJ+TmrH1b1tYkQqE4hMbyqh9/BxiE9JVR0XDWLjYpGckqLqxm/1\n12r6ItFGE3w+Py9tlysr1u05fBQOn1e2dwAQ0sR88sx4OEgGB5/9C2pXKxEneUH5nLH9sKGkEoZ7\nF+gXa3NeSnzCotI7Tq9RpRQKnn0CJ+bdijP3zUb9sUOKc0ntr8jlwXvnivDAmH6qxCmAsHO4dJpO\n7CwOHCmuRJXbg5FjrhfO1GtcIMrUa1xggZJwXAEfbt2DW2/opSn62RiRJj4+zt2CqTnh3pPhSFOA\nOM1YmItuWUmqvem0RKjEY1iWhTtKA+MmXF5c9eRJTbskBXHajpujfbssFOTnw+sJCMf1RJEaKhaf\nOHM2vlr1gbJORAek0nmn9u9GlkXaTFKc1tPrSG52JOCayfegaO8WJFZKV94p6ZPUzCilUmdaUoF6\nIK4c5H7G5Y6L03tJpTK1urpLtdNpTAwfORpWk7HBbvwA74ffmsRL2+XIOj+r6p544FJAM9rF4Mu9\nR5RGKjphG0gSSz9YidVbtmLDzwc0ty2RTIcppMj4kE3PaW3ay7LwXaoMiNY1pPr4Y34sq8LWaiee\nmDQQcRb9XkFqaToAKK51YsPJAvz5lnHCFeKWK0GSROxeA295IbplJipW1XE4X1oJv59Bh8HDdB9/\nQ+H2eFFYWo6W3YOtZUSO+mHLAY9y5Zy4TYuWVi7BMW5X45LBJjQOrmryJCZBWitM+Wk7t8cnmGPi\nlCnY9f3Xl/fAJUDTRowcPxnFuzaqDxZBzs5ALDxv3bErtn6+An6PNIHSEy0RgwBw77VZ+OyVZRjf\nIwP2Cz8rjpXSEMnZHvAjTHw9kphwLfj8YFS2CRz45I6/TwCYwZvXV1uJlmRpVPvg43ITJ0Doxm+k\nDRE6v4jBshOFf/hJ2siLNKlo3WwxqKit13awHidiTEYkWEw4f0m6tQUZY1cUNJMxNsS2zcYRT2/s\n/TkXWz7/VFM0SKpPnObecVLpOY3ES7yNplQjy6KopATvnC1E+ozbsXTDD4if/gA0eU1IoO7T13g9\nBYVwen14b88JPH7npMiokITtRIhAccJxKXd5ET7Ysgd33funqI69ofh8ww+YMpIXTRN7NoUsB5R7\nTUbbyqUJf1xcleSJ+w7zSZDH64fDrP2t2u0PNB12+vwC/VObjExB9On3RGb7jhg2ZrxA06Lm88T1\nOZNrMsxP59FGE26YejvKt30qKfBuSA88ftpv+vjRuGbEJHRvmxhRjSdHhLj9S0WMlFJnSqnAhkC8\nT+5qpfkLYDr9A9LbZDXKfvTCc3ofqi5pF0Vz35E3HhsGjzdQ4u31MTAZhN8Tr9EBmB2RKQfeGzT3\nw0+Bhdvj1eSx0ze7FX49oY+M3zEtB18ePiuxhoBl7GyAIPHBU9KREi6S8vGyMXhg5jQc+s8rWH2h\nGF6VCJhUOqwhKTndTXs1ki2Xn8Fn+SXYXHoJD948DDfedoeuykRpRKbpAMDPsHh9xxE8dOu4cPsV\n8WHzRONSKTxFGC344VAeBnZsIzBd/b3g9flwJr8IbfsIU5ERHk0aSY4Wb6cm/6crB1cdeRKn6Vw+\nP6pdbvW3aok5uAeIyxcgUlwD1fGTJmP3JnVjQa3QU4FnoMJaIzViFI2hZmxiCroOGor6PTKtIKKE\nOO1X6wlcSwLA4I7NQuOisQZQSuk1JmFS2mdFdT0seZtRV3oR/3pmMRbf2E1gIKoXvVppSOdI4MCe\nX+HgpYK03Fvh7wiJOc9vAkkSgu9JIDpFY9k72wXpBC5N5zXHg7AFU5S15WhtY1Fw8aKmhq3dbxiJ\noxeKdZ2jiabQPjkOBwqFGsCQPmdRLigScK57X3J7LpJS/8m/MHNUb1yfFIe3zhTiXL2yJlEqZaYl\njSYFvcRLjWwxLIvNJZVYeaEYg5PiMH9cf5h9LjjP5wVSpwxgGXsboo0+icGyLN7ZfRwTO7VBcqwK\nKQtGnKRSeLLz9xoHZtB0bCsnMHrKTY1yzKoQEfyvNm7DxOEyBsnRRoXUxOQNmbsJvyuuKvIkZzHA\nMNLVc1IgSek5+Nu0zcxCly7q7s1aEK0HVI9UuyoxisZQEwBaZndGi7bZYI79qOuY5MC3OOCn/fh+\nR6/N6IGuaTEov1Stag0ghWidzBsCbp+vrd0KYv8XSOs+CF1vuBHtUuxRCes5MH4/Nq56W/d2l0qK\nkJicEkqfqN1bnPUGEP6OvPHYMLAsK/iesCzgdHuw8M4B4XQCL01npANpP9oUaKrcJq05zhYUafLY\nMRhIMCwLr19jL5kgpk4agQ0n8wURUi36nOAZCdZdM7AjFk6+Fnsqa/BNYRkYuR8IqfSbhoo5Oegh\nXnJki2VZ7K6swVtnCtHCYsLjkwah64AOoe2c694HWAa3LhFHn5Rc2tUd3D/al4feaUnoNaR/YIGa\n6FsqhSeHINEaPv0x3HbPXM3O4A2B2K3cDxLHzpxH+wGXQWcloXe63HYHTWh8EKyaAc8fEOXltWBo\nad5npgwwGym4PL5QpIgD3zFcaVuP1w8jbdA0B+c6Ho2DOEUSyE6yhRzKT5TV6poj3WGG3Uyr9riT\nsinQAr/fB4OBwu4o+udxULI4sJsoPDkiO9Tf7ckvd+G33E9hiU1AXOfr8fZt/UPWANPf2SmIIHEW\nBI0ZVYpmzu5tE+GqroDJHh8iLXfzzjcafRhz7EekZ3dEiwx9vdwOr/sIY2+aDrsjVtO95So+hxZp\n6bBYwg8+qZcFANj/43pYrVb06t4ttMxndIA2meDx+mCkqZAGpKq6Fl98+A7uvvlGTce9c/06FFZU\nI6eXlkqwMLZu+AlF1fUY3b4lbykBUqH/mhp+/ukQNpZUYkpaMlJMRu06o98DomM5UVOPLaWX0CPO\nhrFDr5GtRLPePJ9nNfAqlF3C1R3EP/ztFLISHRg9dggAgO09AXRi0DVcTcMk069OjMO29tifdw4z\np0xqmMGlFhho+Fv1DPdTrK9B7qataJ4Yh97dePekks2AFgsChL8z3HclsH8KREx8wI19WQ7Y2vLQ\nXFU+K0iSQGJitCnX6PHFoULUefS91CjBajRgcpfLUzX8v8BVFXkCIlNsfCgRJ37UykhL66LkKvfS\nHWZkJ+uPHkl5QGlN4VEkgTNl1fh4d56q03g0xAkADIZA1KQhwmWbiUI7GYsDcSrPS1vRZfztSOnQ\nAwXbPsPLy9/G8ocHRaTilHRR0YI/50tTuqnOyddrmR0JggdXQ4T1XmcdKooLdBMnv88Ht8sFezCd\no8VfbPPG7yNcw1k2/D3hP4sJgoBfFB2iPNXwuj1B4hSoNvIZHYhLy0C5X/t3oe/IMfjtdIHm8VyU\nY8jIwThZXo0aN19/KK3P0YpBg7vgsQkDsbGkEhuKK9Di8acbpf9coyAY5brodOOdMxdxwenG3yYN\nwo3DuiuW8IsF30ou4UrrWJbFe3tOoH1ybJg49ZkAOilVczpOC3Gqd3vw2VuvYPbgTpefOAERzYAJ\nUwyeWf4V+gy6LiJFLRUZ0hQ1IkjJajqvOR5ETDw8Xn+TUPwKw1VBnsQeNdHE0vjicp+fiUjbyaUE\n0x1GxFpozFyYi1iLfgdxvgeUWpqFm5sbl2QisOOzd1Tdu9WgRQulh0BxaToCwPTuLeD3M1i5LGxx\nwK0HpImGvVlLdJt4J/IsGbhj4Yv474+HBMJyvZ5P/Oo9uUo+/pzXpMfipSldIwgU465H51SzasuZ\nhgjr6/dvwKBx03RvxxYcRZ9BQldxNX8xr8cDUzDVJob4RYFOSMOlKlFFEUGGfG5okxEgDeGKO6Mp\nMt0ik34hCAI9M9Ox59QF9fMUiY7nzRqLD347pbqdHpgpA/4yfgAyUpLw8qef4ca5r8LeoZM2S4PL\nBIZlcaiqDu+dK8Suyho8NH4AZozqBYOm3xshoVRKb8qtY1kWb+06ju4tEjEiJyigNlpAJzQPOb57\ny4s0kSM1vLruZzww/x4Y8PuRCH4z4C8/XY03nr1fMkUdYTOgwYIgRK5ou7CajiBgpMMv7Gx9ZZNQ\n/AqCYfHixYv/1wehF06nB6wh8KNhpgywmY0gQcBqogE2ek8lH8PC7fODBIHpI9rD5fEJ5iJYYJpo\nOUEABoLElKHt4PezqHR5oXf3DBsgRi0cFsxclIuZI9qj0imcJ91hRguHBWZDuBnxHTd2g4eyIG/P\nNsS1bh/VOV/XOqLYTDEAACAASURBVB690uOQaKFxrkpZMJsaa0Z+WTVIg3zFIpemm9g1FZkJMchM\ntmH2ku8waUg7vP7LWXj9TGh9RrwFewuq4PZL/0garXYkZHYBbY4BAaBdcwcSHGacLalF1xYO/HlC\nF+y7cAlrDxYJ9h8fQ8PlZUJ/PzexC+YPyULXFg4M75AS+vemY+FeZy4vgx4t4zBvYhccOFWGzq3i\n8c2hQjg9frQ2lKPu4CbQNfmIS88EZYquR6Ea0lGB2soKdOnVV/c9fE37TCQ1ax4RgVCa5vi+XejT\nr3/EcoIAbOYAKZo+oj3cPj88Hi/OnzyGzh078Eay8IHC1BEd4HW7Qfpcob9/3roF13ZrF3qLdqd0\nBJHcBl6jA1RdWcQ+23XshA++ysWgjhnyB2y0gO58LWYs+g5Tx/YAc+EILAYCBWcu4JLLgxYOq+I1\n0otWqfEYcstdaJFiwAfLX8eZQweRbjFpbi7cUNT4fNhVUY0tpZdwoKoOcUYasydej95tU0A18Bi8\nR3bBtf9nePZvU13HsCyW7ziKIW1TMXgET0Dt98Ef1wIt0pvBW1EEYre6U7gavtxxEB3TU9C1R48G\nz6UbLAOP14ev132LmyZPBOnnfg9F97mf/zsps44gAbAAQcIQY8eMhbmYOqIDGL8fpKsqMI5l4SVN\nmDa8PTxeHwzeSMsON2MEQRCIifn9qw2PltTC6288VY/RQKJjirKW7krCFR150upRoyfazrLSqT8z\nZYBJQkvFssDhw4dBEgTqvf4GETe5NAu/P57dTKPGHR6X1akr/D4fYqrU39rF0FuN5/O4Ufj9+3DV\nyOs/+JYEGUlWnCmvwwdLwlGnaJzK+cLyu/u1Qo+2iXhv2xFMXbAUr329FazfFxonTr2Jo1RKEauH\nPz+AA/mX0LltIn46fA7eY1tgOLgG9ZUl6DRmFjrlzILZHl0VnBaQJIlH5typWkEpB03uyxog1Z7I\nZrOhtq4uYqy4tJr7Oy4uDmX2jIAAl9ckWM62wGAgQZEk/CrNgqVEx9OmjMS2s8UoZ6Iz95QWSAe0\nP9ZWWRjcMRv3tWDRI86GrwrL8N65Qmwrq0K9hDSgIfCzLI7V1OPj/BK8f64I64sq0NxsxIIJA/DI\nxEGYtvwtNJv/jKT5p34opTfZUGsWxmzFv385jJHt0jBg2MCIkSErAg0tVtRwrrQShRXVGDphUoPn\nkoWK+PyT3C2YljNUV+pM/B3gp/F8tA0+PxN2VBdFemlXJdi6CtCu6HWlTfjf4IpumiP0qPFLVtMp\nCcjV5ubAT9mtWpYDt98vWP/9po2op62ITVTvTg/Ii8vzq12gat0R68TEKr/aBYoMjxs3dRbefOlZ\n9J16D4xm9VYHHPRW49EmM3Jun4/1K5bj2okzcNoVuS+xjkncLy8ap3I+4fpwaQ5sJgqsIx7pA8ag\n9OQBmPJ3gfH7YDGbUXOWxS1v12HVshy8NKUrFnx+UGApAEDwbw5cKu6D3y7CdqQEJUWFyO5zHRhr\nmMQQwWOJNiWnhN6t42GmEpEWb8WfX9iE1x8dFiH05/ci5Ovc9PSy41BfV4cYq3ykxuXzC+7zGKtV\nkjwBED5ogmmLrA4dcevfVuKb5fOAcghsC+T6jnVq2QyHzxehW5sWssdF7FkLn9ECgpceIggCj/z7\nP3jxzXexYPYM+L5+C2rmnLwZJQXSfO3PqiWj4IyxofvAjuiOwG/Dzn3nsDa/EC4/A4ogEG+kEEtT\niDGQMJIkKIIASQAkiIAZeWhv4b99LIsaH4Ny0oALFZdAAMi2WfCnUb1gFfkmSR1PQ7RdWq4LPeFu\nvLL6c8x46Alk15+UH9oIqTqv3493N/6KJX9b0OC55OBO6QjK6oCvrlpSS1XvdCG/uBTT+8wRruCl\n5lYty5HuX8f9zUtdr1qWAwCYuSgXK54aFWrhQom3ZRqXhDfh98EVX23HJ0diUkMQQKwlfCNXOd1R\n6aHE+xGTsLq6Wrzz5huYcvf9qvOkO8xwmOkQCdIDpYq+irJSHNy7Cy16DZHsY6dEjPRW4/m8Xny3\n4nX0GT4OFxAZIVEjGdGQEC0VbH6vB+NaGjC0T9dA6i0zCU9/fwK1bh9I5yUc/2EdPH4GRgOJzs1t\nSIgxgrTG42Jaf8GjVqpCEBLLGuuLw9eTjWufDKuJQp3bh7XHwy7lZorE+I7NMGtRLj5amoOvjxaH\nPjMl8iR3z1B15aitrUVmVjvBcqWK1P/+6wU8OH+O9EoIK4lKSkqwectmzJg8MfygMtCKfcecLjf+\n/dqbeGCceqNiAYwWUDfMxuQFK9E74QzuiasHnNIO5GKQMXYkzF0WIiUVyxeGSElklRrAJ1uugjOo\nXf0KfH4/Kp1uVLo8qPf4UHw8H16GBcOyYBCgcYFrygb+HfybIgm0vX0O2vbtj2akD/WfvQ4l0id9\nPJcHVaDxfpUFh+o64ot/zoRvy4pGIUkRCFbfvfrdDtzYIxvtBw9t/H0AERV1hoIDgfPh3ZNvffoN\nRl3bFy26DYjYXLJKTmYMv/oUQHg7b60+MThBospr+Z9V2y1ZfwwVjVjNnBBD46lRHdQHXiG4IiNP\nXHoiIiLkFJIaqfRDtBC/ifNhtdrQpm1bVJw5ioQM+XJrfvpt5dIcySiTeDx/vdLYhKRkzLppMhxm\nGoW8yIRctEJ4bvqEmRRNY8zt8/H9yrfQqd91KDYJy0/lBNN80sRfr4VMiSNYUjDQRuQWAa3K6tA5\nMylgxhlMFU7v1w1ZNw/EqdJarN53MWyR8FAOnv7+hGBeqUgXgIhljRGB4hMnM0XCEryfP1qaIyC1\n0Xh2KRH1Zs1T0Uw0PtooLYCIt/MW8TGBqGAJT4un0rDVYjbB52fg9ftBK+jqIhBM533x0kzs+f5b\nvP32K7irjzYNoJJ4uu7T1yIiPFwE6MDpCnTLygQ77QHUffwvJNssSLYFI7EtlAsKwnPZkTBtRoi4\nuVSiSVLHczlwpLgSuSfy8ci/3kRCm3YBCwIVX6ZoiBXbaxzopBb4dt06ZI+4CW2uHwxcruo6XkWd\n1+MD0rrB6/GBNlLw1VXDfXoPqmrqJIkTEEzNecnIqBEHcXSqrgJUMKKkuJ1oDo5ccUTM7PLB427q\nbfdHxBWpebI7zDBTBk3kSMm6QC+UyNeN4yfi22++Vqx801I+zkGveSZHzGY9lYvUoH4pGodxrSAN\nBoy85V4ktkjXVIkn1i0RKsvF0FrBxgJ4c/s5PP39Cby98zzu6tcKC0dko21CDG59KqCzAiBIHYoh\nlVpsaGNkKXRODL+7cERJiSCJexECQJKvEnW1kQ9TiiQQQxvAsCxiaINqFahcNalwjMIcEn25CKkH\nhormZHTPDli3S/8DlNPeXMNcQPfUBHx68LTmbeV7t0Xqgpj6GrgKzqBbVhJmLMyFJb0NyBi7qqmk\nFLSbesofj3aoG18CwDdHz2N/UQWWzp0Kx/HN8P38iaJ3U6j6sc8EfYcTNMKccP+7OJWXh68PWRTb\n+DQGTCVHYSg4ANpIYc4Lm0Gb6JAO74O1GzHjjruVJ1AgQHxtk9ftFqbi1IgTaRDaHfCImNlMNZqW\nsQmNiyuSPN3/0tbQD7wWchRtxEnPPUuSJEaOysH5A7/KjqFIQrV8nBvHRagcZnX7Ay5CVe3y4v1F\no0AQQN+02KgdxrWCIAjE2BwwU6SAQPGtCDiIheJ2EwW7idIkIJeaTwkc0eLmnrUoFxRFCoTrb+88\nj79/fwIGghAQN7Eb+up9F0PzNsS/SYx0VGDrZyvAsqygzY4UQeKTXvFnuGndGhiN0nYDNEXilqe+\nA62BNDdGlFYsnCVJAn5eJaXYxVkKPYePxpELRdHZbwSjH8Nzrkec2YQNJ/M1bhgWSGshGLWrX4Hz\n/Okg6cmDZexsJMxdFoWQm4Bz3YogcXstKgKmdT/Wm+cpHqPH78frO44g2WbG3FvGgSCIADEaNDXc\ni06MIAE6kFcOOjEVbG8dBMrjRNnpY+iRcBb33H0v3nh0qGIbn0aDxwlfXTXeeHQovG4vVi0djaJz\np8H4fWiWok2zGoEg2Zm9ZD0ABNJzGuE1x4OwJgjtDhDuE+ly+RpsRdOEy4MrMm33yoIhEe0jGhvR\npDB69OoNlmVxsTb8A8ARGz1aJ70RKm7eolo3spNtmLEwkBY0UyR+PFcZtcO4FvDTggCw51ylpDZI\nHLmZ1r1FaIxSREfJoVw8TpxK4+/zREkgXcdfP7NnGrKSrJixMJyKm847LgCS59FQJDvzcWD3Lxg1\ney4stCEUHRSn6sTXV5x29bicYFkWtETTVB/DosrpxcplOahyRlZvSkEpNQ1A24847y07vXkK8otL\n0bpFM0HF3aqloxX1T9d1zsSPh0/j+i6Z6vuTwc2TR+Ctld/g1wul6NtSy0NR3Vk7DBZ1H/8LzqCB\nZMLcZZi5eD1WLtYj5CZhnXY/LOlt4DyfB4BV2Xf0zulqYvOSWife3XMCt/bIQqdBfQILeb3oVi0d\nDZ9Uas7jhLeiCN2yUgPpKrlxEvD5Gbyy7Ak88vADsBbtAwy0bCGBKlS0dGKYSo6G92egsertD3HH\nnPnR7RsIRV1XLs2RFoTLgTSEfJ4+WDwqFLWiWCaUJnQFNU9N+OPhiow81dY0ThpODlpSGPLbEqGe\nYVzqrWWQ4OiJJElFqMTbiSNUAFDtDJOu45u/AuP3SxKnxkjhSaUFB7ZNAC5dlIwk8aM5/GjT6n0X\nZSM6WiNTcqk/frSIT3zsJgoZidaQwd+ZskAlGX9fcs7o4n3riYo5yo/jzMG9GDHrHpCkcqpOLe1a\nuGsjho0ZL7uv/GoXTpQK76F0hxntk20RLvkclF349f2It01PxekLwcgd38VZJcIwdMJE/Hz0jK59\nSeHumTfiQFE5jpeqt1ZRctaWRiBaxaXeViwaBRCkhua7BMgYB2zT74OlVdvg/jJV9q0eOVKCUnrw\nt4tlWLU/D0/dc1OYOAFhW4hlOfD5WbBdh0ufza418JYVBrRE5UWSY6Tw6rptuPWGXoiLCUZNtZAf\niZSelmimJIL7yztzDolxDsTGNqynHOWtFURdNYHxw+MNRHsZhoncvslp/A+NK5I82eyRLVIaE42S\nwlDwZuKiAFIkiq914kcLpDRQUhEqPunq2rMvDq/7CICQLPHTRHLQQq6kHvwuH4Mftu9C77gj+C2v\nQEBYuMiNnKZICrVuH+pcXqxaloM6lxe1EuM4gjXvhU0RRIfbJ9/13GGiMK17C5AE0LFNAk6V1eGN\nHecijuukisZJq16Lg7X4MMoLL2DIzbcJiIhUqk7q+vLh83pQcrEALVq2VtynlF/Y6DteAG0gdHcb\n0Zs+yOh5LU6cC6fOtDQKBgIkrWvrVBw4e1FxnBY8dPskfHciH/lVMjYLQejXH4XhXLcCFAnMWJSr\nQrw4ErQUVGoGDgaJuzP/rOK+9RO7SIh1XX6Gxar9eThdUYNF994MszGSmBAHNwIsi1uXrFdsvULs\nXgNvRRHopFTgulvl03xBfLxtH/q0a4kOg7U33ZUkSRr8w5TAsiw+WLsBU++4R9d2YoQdxPV/LgKf\npyaydEXhiiRPfM1TQ6C0fUOF5s2stIDYXKgSRpKkyJCc1klJA5Vf7UJeRZ0gusA9MFtnZqFzj16o\n2b0+RJa0iMi1kCsO4gc/QRBw9BiKZgNvxKtvvAHi1C+SD12t+iGbiYLVTGPO85tgNdOSESCOYL3x\n2DBJgsUnOQuHt8MTI7KRnWLDzIW5MBhIfLQ33FeNf1xqx6jH8LN363hkdO6OAWNvllwvl1blri8A\nwWdSsGM9Rk6YEjFeKarJke2RXQzw+tnLku4OgSARG+tAda2ItGhMr0ycMR25e441+DBIksDf7p6C\nVfvzVE005YXjymDqqzURLz4JogwkurZNgOtCXiAFGNI+Re67IcQujLDY/FRZFV766QB6pSXhzhlj\n5SOKMqakEQi2aZmxMBcESSgSrW1HAhFFXUaYciRJRzRTCmu3bMeIAb1hkkh7a4aG9iyqaPJ5uiJx\nRZInseYpGsg1+eWjoQ8XhvELCBM/4sQnQ6YggZHTOomX85HuMCMzwSpblderb3+kJMRj9G1Po3lw\njJKIPJoKPakHP2V1YPTsPyMprRUK17+LbLvIRgLa9ENcNOi1R4fJRoA4gjVjYa4kweJHphwWGrcs\nyoXPx+CjpTk4KZqTf1xqx6hUgcdP53FierM18s1Ua/pU/JkMGTUW6a2FrUy0VGjmV7vg9HpDLwWX\no5BH3Cg1GsErZTCgRYID50or1ZvNcpAZRxsMeOL1d7Cq3o78nmMgTHvxReLRV7NpIV4CEnTuJCqW\nL0Tt6ldC0SillF+0xI6PS0433th5FPuKKrB07jT0vUG6LJ+PkIO4QsUdPE54y4sCJfoMK0u08orK\nsCfvAmb/6XZ9B65AkrRGM8WorKrBkbyz6DdKg8hdiRBJVJk24f8GrkjBeE21C34q+l99Ncdw/riG\nEKhX/vkiJtw+N0LQyydDbq8fmQnWkJBczmWcW97cZkJ2ki0kEFfzjfIxLIaMGoPa2hrsPXAILiJR\nUUQebYWe3HytO3RFeruOAAv0TqCwW8ZrSglqHk9qNgJ8Albt9OLDpYEUoNUQ+FEkoN2PWsux8UXu\nxSKhN/86afHgAiI/k3aJMRFjtHqI1dZUw2azBY9Fe1GEs74eFouIlJEG4Vtz8CHD97tp3zYDx06f\nR8dM5fSiFKbfOgvL1/yEh26ZHfAaUiqZ7z0BdGJz6XG2eDjS2+KYvx92HD8Nx9lSjGyTBH0icTVo\nI15iryYyxq7ROVwrsYsUlnv9DD4/dAZVLg/mzBiDOKv2LgQAIomQSBTO9hoXuPblhSD2b5D0hCqr\nrsXKH/ZicZQO4gKRtxhRCM2Xr16De+97UHVchDkmz4uJg6oHVBOuSlyRkSctb7KKljQaNE1aIlNq\nuGnadHz/6QrJdVy6zUQbItJxStV1EQJxDVV5+dUu9B41EeZmrUJu1EqkiEsV/VpQpek81dJ8BgMF\nAxWOwij5QkkJsLVEqdRSbNz6ZRtP4uUf8mA107r668lB6thsJgoZCRbcIore8a+T3gifnC6Kg9YK\nzeqzxzCgbx9YgsRJa1FERUU5EhMSQn9zJdZec+Cz5Os++G/i/YfnYNveg8qTy8BmdwDmGEz96xrF\nVBDbZwLopFTMWPRdxDiu3N7jcmP1sjG48+YJiIEPb+48Cr/J0mAtkX4ISVDjpOQ4CIXlLAtsPX0R\nr24/jP6tUvDYXVPUiZNKlC/k68TpmnhVeXRCc8lt6t0e/Pubbfjrw/eD0mN+KkYj2Rhs/XUfunfI\nUhWJ+4yxoE2mUE86nzFWEFEVoKHESSq6FU0KsAm/G67KT0cL8VHSNDWk2o6PtPSWiI+PR03+Kcn1\nbh+j2ZIAUBeIq23LaRu09ELrmxarSfcUrRFn79bx6N5C2FtNrwCbDzGJEZMwbj0LoKjG3eiGl3zY\nnYX46xNP4s1HB4eid7FmSnCdAOX0qRRcPkbxs+PfC1LaJ4okcOzIYbz0VXGoybVaUQR379dUV8Nh\nD+6bV2JtpCnAQAl0H/zKo4T4eFyqkfC90SLu9XsxYfwEjMwqg8flFkZAuId8UG/DVU16y4vC43gP\ndiMvanbjuKEY26EVXtywHcd++aGRiIsYgao6Ld5N0aXkIj2p+JqqYobCP3ccg5misGTuNHS/rp/q\njFLESAA+UeJIqoouyuXx4oUvtmDB/XNgFUcugctniikzb53ThR927cfIm2Ypb0+QoE1GzHl+U6gn\nHW0yymub9BAd0Vhxmpu/zGxpgB6rCZcVV3xvOz64H/po+tlxKTru/+KURrQpPL/fjxeffwaz5j8q\nK8xU6lnXGOPlcLBY2sxNqY+aFLSmn8TYu/lblBfmw9zlBljikmA3UeG2KUsj26ZohRZvKI5cNZZ3\nEwB46mpQuzcXsUkp6DNiPGJMNFw+BkPaJCDFboLT44PFSAmuE5fG0+LFpbUBsJKnGFVdjPbtsmTv\na/7f/O/A9p07YaivwOCB/QEEIk9GmoLH6wPtqlTs/fXP5/6OR++cHvpbtkGr2K8n2I9s+NS/YOWr\nf0PC4e8AjzPU1oNL0YX+rigCsWuNYN/cOp8/UDW2aunoUJ82t9eHt1Z9C9BGTO2YBmNDoiICBCJA\ndHoWKAMJ57mTQWLUWD+18ulG1/BZ+OqnnSCdNbitUyxMtMaoarA/IOfr5K0oAp0QmQYVX3v+9mLi\n5Pb68NxnmzAnZyBa9R0csUu1Rr0RUPNzCh6D0rz/fO8TTLvtLk2GmOF73A/afQk+2iZ5j2vpe8el\n+6TSgIQtMdzWpbY8MJy3rKbahYQE+SbelwtNve2UcdVEnrhok8lgELxR69k2hqZCESt+ZKohKTyD\nwYChw0fiyC+bZcfoJUKNQZwAoEuK9BdSr+5JLaUkh55Dx+DaCTPA5u1E+Q+rEecubZSIkFIVHAmg\nuT3gLTOte4uoolwcOALG+P3wHdmC2r3fYtC4aeg3ehJIgyFInOLR3GHCrEW5sBgprD9ZKrhOLh+j\nqbqR8fux4esvVI9JzZ2+WVpLQcSVT5z497k4+mp0V4Omw2/zghJrRLqL85GSEIeSiqDXkoEGzHb4\nWQBmeyhCIFmKHhQKf/nfp/H2228HvIYkoh8hUfOuNRHREm4dW1kYER0x0RTmzx6PIS0T8cbOY/jy\n8Fl4/Q3XrXARIIIkgvYFmY2aEpSyLjhcXInXth/B6ucX4+ZWBtxzTaJ24gQII0jlAeIklQaVFZBL\nRJye/3wz7h41QJI46bUZUPNzcqX1gj/9GrjSesnOu+fwCaQ1S9bmJE6QvOiqAYQt0K8w4h6XqrZT\niCxFjJUSnPOWNTmM/3FxVZAn8Q+92x8gPgBUSQ+37ZznN8FIGwSpOi4SJZfC05rO69O3H3r07NWQ\nUwQgLENXM9rUgnWfr4alUlojpJcQRetgbrHZcd3kWzB0+p9w4eRhLH/3faw9WtygFiicQPwjEQkj\nATw5vB0eHpKJRSPaKdoMqJlfkgDu7d8aC0dk476BrdCmYzeMvGUOrLFxoTFmikSK3RxKKZXUuFHl\nEhJCrWnPizvWI7ODuhGgVGqXu1c481a54gj+fQ4IdYFOygpGXFIt/ltG99ExszWO5p0DALiT2oEK\nto2huHNVeJCayk/BbrNh63EWO48F+tVJmjcGI1KCtBMHjzPw0Jfp09bl2j5YdO/N6No8Aa9tP4Lc\n4xfANOCBxemYWIbFqqU5jZ4S5Ob/7+PX4aM3XsXL329HYU09Hr1zEv5y+wQkmqOLoIWI0e41yhYF\nKg7iNU4Xnvt8M+4dPQAZ/a+THqTHZkB8f0ilE010kJTQ8DlrI+Z1e7xYs2kbJt96p9plCIBHYHx+\nRtA6RW6c1+2Gj7YJU3AicuV1eyIq86RePLhlLqdH2/E24XfHVZO2k0qzaU3fcdt6vH4YaUNE9ZFU\nVVK0HegLaqL7MvBTMQA0t3pRAsuy+PA/r6L/4BvgSmwT9TyAfLWd3u2k5tFboSeXtmtuN+HhIZmh\ne+JMeT1aJcTgVGmtgKyppf0IAAuub4vmDjNmLFRObXIpzZIaF7aelT4PtbRndUUZzm37FjPv/rPm\nayDVFohVEbVaKENIC8W3MmBZ4Lc9u2FwXsLgAX01H0Po+Gtq8PG7b2HuLTfB36onDuSVo1tWUiCt\nUnwEgHIKx53SEYYYO5b87TE8NKAlYuyxghSTb0ugKCNimbgiTCrdJIFt32/DxlMXkZFgx+jsdJii\nKhrhNEkNaeYbCYZlcaCoEjsvXgLAYnibZPQa0r/R5hdAY6sVPsqqa/Hvb7bhsYfmI96hIdrG82xS\nAnd/eD0+0EYq4j5xpfUCbaLhdXthLtgTkeJ75cMvMGHoIKRdM1DX+YAgZdN14nFAIN3GVbyyteWy\nqTqtAvMqnxUkSSAx8fcoZhCiKW2njKsi8gRECsD1uIRz28q9bYrnboignHv71wNBKsZC62r1ogSC\nIHDLPfOxd+cv8J09EPU8ekw1lbaTmodrOhxTdBD2smPwedTJolzarqjGjWpnwK282unF67+cFVTo\ncdEmqe0Zvw/2smNIqD6DQZmJEREljjiJI0dcBE+OOPHHyFU3Hv/+M0yceZvqefPBRZwclvC9onSf\nmiWIExCOUpkoAwiLXbrSiIOMaNZht6Om3hmKNnTLTBQQJ0DZr8dUchTUhd8wb/IN+M/6HdIiZSXh\nstECOjktkIZKTlOtKLt2xLVYPHcqujRLwHv7z+L1HUfwyYHTOFVerSOFwoKpr24AcQoLwl1eH346\nU4TlO45g+Y6j8PQeiYff+wyP/vs/l484AbqJ04WyS3ht3c9Y9NhfpImTVHuVxCxN7VVMJUdhKDgA\n2khJRijNBXtgyN8fIE6AgDht/XUfWqWm6CdOAMD1mVNrvcJLt614ahQAhBzHI7ZvsjS4KnBF+jzJ\nQfy7ptboVAwl7yfxv6WImVZReZrdqCsCJUjFOAM/Clor9NRAEASm3XEP1n22GpaKcrTsO0xXBImf\ndpJqbKu0XarDjJnB7fjVaNw8fdNiQxEZn3cAzh8/hPq938HrdsFstaFt114oolNAkMLIgJLv09Mb\nTyLFbkJRjTs0FoiMNp0qrcWbj12HL9bm4uyP2wGCQGL3PmjTuXtIE9YlMwlF1eGI0nWt45HqMKNQ\nFEHScj3458rflig4gnaduiLGql8w2txmgtfLBO9T+e+Bmu8ZQQBGyoCFb+9C7uDrwHoj35zVRLOU\ngYTH64ver8fvRfOkBLRtnojtx85iANbCZ7QIPIWIPZHLQsfnY/DhktHw6bi3ey54Gv2C0aqSTSux\nddN2bDiZD5YFMhLsGNiqGeIkq6G0NvGVHsewwMVeOdh7Nh9FBSdBntqHPulJePTOyTCYraBuuFW5\nYa9eNMIcJwpK8Pn2g1j8twWBCkwRJCOLOppFAwA8TkGqL+IekjiH4vJKbN93GI88saghp6eZ8FDe\nWhAmngCcUwa2RQAAIABJREFU+640EaarDldN2q4xoDcVx73JS1Xnceulri7Lsqi6dAl1lL4HIr/K\nTq3iTmtFHn+cs+gcenXrEvHwV0M01XbXtY5HstUIiiJRWBXYjj/PrwVVihV/ztoanD64FwV5x8Cy\nLCw2O+JTUlEMB+LSM3VX09lNFBaOyA7t790te3Hk501IaX8NWrbrBDJYicUnh+J/T+jUDDMX5mLl\nshysOaJcociHXHXjda3jcWr/r+jWuz8KgmRPKyiSQHaSDQzLYt4/NmPOqESUVlahTz/pSIXavX9g\nz25QBgJ9evWKJEdSFUOih8UvuV+CMpAY2KNLeKHaw1ICLMviqb+/hPvHDYbdIqFBkdtOR9oOQET1\nGT8NyLIs9v6wE9vPF6Pa7YWBIJDmsKJNgh1t4uxodstfNBhvEqAn3I26mDic3bcbBz/9L0pq60EQ\nBAjahG4z7sZ7P7rw+T+mNigF2ejXRQLf7zuBkxdL8cD9c0GSEr/LwapJ7loazu8Na5G0Vl6K5tNy\n37g8Hix7fQUee2IRYmI0GoPqSKnJQVyJ2pC5m9J2f1z8nyZPYnKjlzzxx/Pf3KucbpgM8nMxDIMX\nn3sGY2b+CfGJSQ0+DzGUStXlxhXVupGdZAvl69eo2BOIoUfzxJGFW57KxYdLhMQoGgduIECmLpUW\nwVVfi4zOPSK2/3Z/Hrav+0zgSUERgNkRjwFjb4oYH2umYDVRqHP7sPZ4qerxmCkSY9sngyQIMCyL\ndcdLdV0/8dxmisSEjs1Cn8eJslrdUcZ0hxkxtAE0ReI/b72NYaNykJgkf78pRU4P/LQBZrMFfXpe\nw9sg/DBQizy5vT78518v4sHbAr39dJep81B+qRqvLX8bj00Zqms7yQiLXNTFaAHbdbgmYuFnGOSX\nV+HA9t9wvs4Lf9dB+OKHPEy5IQv1u7YA3kjiS9AmxA4chc9/LsLCu4ei3aUjaBZDhexMQqSmvAjE\n7jUR24eOW+H4Vc9VgSBquUZ+hsE73+9EWmIsJs+cKXt9AJXPW0SG9IyVA8uy+PubH+LOKWOQ3Emb\nTk+T5YAagi8Sc57fhDceGyave9KIJvL0x8VVkbaLxoNJSmCupWULf5+C8SJ7BKW5SJLEfX95CC+/\n+AJumbcABmP4DVopYiS1ThyNAqCpTYe4nUdRrVtQpcW1AJHzghJDD1Fw+RiU1Ljw4ZJIKwT+v5Xa\nyIgRHxcLiy1sGihOJyYlJWHErHD3dCkhN7c/E0UiJzs59PnFmim4fYx6epJF1L1e+Odqpki0S4zR\nZaAqBb5ZZmFRkSJxApS/QwzLwMDT14kfBrItKoKCW7PdBKcxWImoN10jQmKcA4M6ZmDtr4cxrm9n\nzduJCYBc1IW/3LdlhWQakA8DSaJ1cjxajw+QObb3BDy0IOiR1Ek+usz2God587n9nxSsI/ashbdP\nsOVMr3GS1gBajp9bLjlWpBOTOk+5fZRW1eL1b3/GjOt6oOsNI0UXJPLz1JyuVbg39BDu5avXYNwN\nAzUTJ35VnCDdphdB3dMbjw0LfDdYpvHmbsIfCle8YFzsTaMFUoJvPQJzIHK8kycq1zKXJSYG8+bN\nx5p3X0cLW0A3odTYVWodf1m6w4zsZBua20yaHrpa3cpbmbyN7jNyXet4pNjNKKlx6/aGkptPLDRX\n8qriiNWhvDI0d5gxpE2CYLsqlw917sDnV+f2ocrl0+R9VVrnAUEQKK/3RFV5yKXqJnRqhnSHOfR5\nFNXqS9nx4WNYlJSUICGKCKfg+yQoN5TpJC+hgyJsiaGx6a0zkF9aqa9MXQbDJk7C2ZIKnC+N8v6R\ncsuWWi6zrRxCPfYqilTTYIpNd4PO6VJeS7qO32iRH6vlGCS2++XoGby7aReefPSBCOKk6Mek5XOW\nuzfU7Ap4+HzDj2jXOh2drx0pOyYCDW3wyyuWkBKINzUPvvpwRZMnjgTNXJQLykBqNrKUIzdKLVuk\nIFXhJ7dO6thbt0zDr0XNsHrFOzBRpGwVnZTxobgCL9ZCY+bCXMRaaBTVujW1bJEiS2KyVXDuDPZ8\n9ha8nsgHuNZWLPzx/IhQit2kOIeWKj4lnyQ5ryqXj4HL40O3rCTMWCh9HGuPlyL3RGkoZac0H3ec\nAFBS40aK3ay78pA7l9RYc+hzpEgi1AhailBrxb4fNmD0mLE6j0VkDEvwekpqeRjwCJbPHxCtD772\nWmz6+VcAytV1WnH/fXPx342/oqpOom2LGuSq81Tajch6SUHUYy+hubZjkYtqqRyH5Hq5dika5tJy\nDIyrHm9v2oPiqlosfPxBxIjbreg0vpSD5L3BI1Vejw/+tG6SBO2nPQfgdLlxw4RpuvcrID062q1I\ntVeRbB4sV7Gnsi+T+TK1sGlCg9Bg8vT1119j/Pjx6Ny5Mz766CPFsZ988glGjhyJkSNH4umnn27o\nrkMkaOXSHFAGUtY6QCoixZEbt19IbvQGWZTGq61zeXxY+/q9yMruADvpl40YSUWJ+MtqnF54g9VE\n3mDEQ2uaR21cp2t64sabZ2DHqtfRjA1/8eWIjRwZ4sb3TYvV5F6u1TxSLSIkNb+ZImE2UjhypiLC\naoAPsaGl1Hz840x1mJFiN+nu88fB43LhHy/8I/Q5GlTcwrWCJAkkp6TIrpf6vogjswbSAD8TPnfV\n8m0ewWJ9XrC15WiZaEVhaXl4TAMbvRppCo8vmI9/rvkBbq9PkdhIQS7qIhuNUYjgKPbYixKKUSHR\nev65S22nNpfaMRR+/yGWbTmJIX96CJMefkZ6cCNEFPlziWEqOQpD4RFZu4JjZ85j96HjmH7nnOj3\nG9QnyTYAFkMuChtcpwa1fREEAVMDGpdf7XC5XHjwwQcxcuRIjBkzBlu3bpUct2nTJkyePBnjxo3D\nuHHj8O6770aMqaiowKBBg/DAAw9o2neDyVOnTp3w8ssvY9w45R+s/Px8vPbaa/jkk0+wYcMGnDlz\nBmvWSAghdSIU4ZFJkym1VjEZ9LVdibZBsBy4Y+8zYBBMJhNYlpGNGElFifKrXcirqMOFahfqvX6Q\nBIF6rz9q+wK5h3Nys1Tc9cCj+Pbzj+E7d1CW2CgRKv74XwuqVN3L9bSIicYNvajahQ5tEoJWAxWa\ntlM7zsJql+5mvxy8Hjd+XLkck6fNBEkQcHr9io2jtRKpNLsRs2bfLrte6vshFZl1GR3w+0VkUiX9\nICBYwbGxNisqq7Xp6LQg1mbFvaMH4B9f/wTEJsunueSgEHWRWiYbwQmu65aZCG95obTIOxqoEbBg\nxEmqYa/uuSTAsizWbtuNz3YcwRNL/44lK89GRpX4jvCNEFGUgzulI/ypneD1+IQEzUCjqKwCH3+7\nBX9+8JGG7USODMmRIpkorJgUSZIkJeLFTc+ycDdy4/KrCe+88w5sNhs2bNiA5cuX48knn4TTGXmf\nJycn480338TatWuxatUqrFq1Cnv27BGMWbJkCa6//nrN+24wecrKykJmZqZs01sO69evx4gRIxAX\nFxCNTp06Fbm5uQ3dPYDgj71EmkyttYqa0SV/WUP626kdOx/NrPIRBjEpSneYkZlgFehjonUcV9Jb\nAYDRZMLt8x5EZXkZTFUXI0iCUqRIighpIRZ6SJFejZEW80q9c/14rlLzMfOvj9ftws7Vb+Dm2+6C\n2+xAjdsLu5mW/VzVPiutxErpOyD+PlEGA/xSfd/U3q5FBGvUpKlYt3W7puPTBAONjP7X45bB1+C5\nJQvxwaLh0qmpRoJSBEfQY+/3hFpaTitEhLOyth7PfrYJCfYYPPKXuaD8nsi2J2KNUxT2E5rASwnS\nRgqGggMwlRyFO6UjqhKy8a8vtmLB43+DoaHNnSXIEJ/4SJGgiCismBSRBlmNoBYtlNt1Ga7nVYLc\n3FxMnx5oOt66dWt06dIFP/74Y8S4bt26ITk50NPQZrOhbdu2uHjxYmj9119/jeTkZPTp00fzvn83\nzVNhYSFatAgLMFNTU1FYWCg7vrq6Gvn5+YL/lMYDkURESbitJupWapLa2BEo8X61aFykdFANiTg5\nzDTmvrBJNT00NGccWrXNQqXLi5NltSGSoBYpirZ5cLQ987SkzKKdW20utXmHtAlH6NzOeuz8+E1M\nvf1uJKU0A0USsJvkP1fx524SnacaseJD7TsgrBA1wC9KcetKbwTRqmU6zhcWax6vBP6Du/21QzE9\n24Fn582G79evhAO1RqG0QomcXCbSJoDE+USbluMgTnlu2n8Sb63fgYcemIuhEyYBkIgqiTRO7mad\nNLmFR6WF8nvhq6/BqqU58NXXBK6zgQZrtGD4TfPx0KOPwxyj0TdPhfCLtU984iMbKeITHzEpYvyR\nJCm4rSb38iAKCwsjnonV1VFaKvwP0ZjncfHiRV28AgDy8vJw4MAB9O8f8LwrLi7G+++/jwULFuja\nt2oydfLkyREHw7IsCILAL7/8ohpxihbvv/8+Xn31VcGytLQ0bN68GTabGdVubQ7dnMu40jrxQyPC\nhsDp11WJFy24/U5/8lt8uHiUIiGS0kFFCx/Dwu31443HhsHl0Z728zEsujazhewM1KwFGpOsKCEa\n004g+v58euYb0iYBzR0mHDhVhi6ZSbBeuoCZd81FbHyg4k/tcw2tX5YDt8ePzARryM9LbD+RDkb1\nXtXnws8P3UZZfk2Q6JTVBgdPnEbX7LZadioNiZL2DoOHYQZB4PnPNuORyTfARFONaij5u0Im9aZ4\nPg2IOHFpv+UP9cfrX/2IXhmpWPjXhyLHBtNk8HuFGqf6GlAxdnn7ieDf0fp7hXrbeRnQVgfcKR1h\nLD6CF59/FquWPwWb3QGCplR9lDT7LXH3spgIAaF/R9hy8CC27qA81WB9BlCMP/IYFOZJTg7br8ya\nNQsFBQWC9fPnz8d9990nfx5/QOg5DyX+8fPPP+ved0lJCebNm4ennnoqFIlatGgRHnnkEVgsFl2V\n5ark6YsvvtB9gFJITU0VXLDCwkKkpqbKjr/tttswadIkwTIuJEsaCF3eTnKGlfIO4BKaD52tXqIB\nt99//rk7XnzxRdx4+1xFcppf7ZL1cdIDiiRgog2YsTDoDaUzitW1mQ21NdU4U9+45CMaKLWLUSJH\n0RAuvfOZKRIp9jDhqHZ50bF7r4htNX2uLGA28j6z4Hg+8dJ6r2oZRxsIeGNi4fv/7H15nBxF3f7T\nPd0zs3c2B0lIuMMVIAJBAeUGc3BIgv4EAogCIggoKIfKHV5RQVFQAV/lVZAEARUEQkgkMVxyBwgh\n4QgQhBBCzr1nprunf3/M1Gx1TVV1VXfPZjfu8/nkk2Smuq7u3Xr6ezzfdHPlFz59qMgOEgJyaBxx\n0rdw6/WXxyNPTHAy0Q/a9aAjcJZt4yd/XYDv/b/JGFomBYmVMukDCAkSRXISXU+hB4W1q3DaAR5u\n/eXPcf45ZwiL+rLkh9Zvym+1O7d0SuWaMIIlQsqG1dBSEZ48+cq5uGfmVPzu/kdxzD67YPuRQ2DY\nVjiRj0j4rUI7HLRUCI/fuV7+vBPxWKpNL2EqwM6kleewdm1HRSRz1qxZVdbf5mZ1q68ulv1nEz6J\nUXiexaiyNVxnHWH8Y8yYMfj444/R2lqKs129enXFosRi/fr1OOOMM/DNb34TkydPrnz+6quv4vLL\nL4fv++ju7kY+n8e3vvUt/O53v5OO3Wduu0mTJmHBggXYuHEjisUi7rvvPkyZMkXYvrm5GWPHjg38\nIWQrn1e3/pgm3+UWFsPEi6HqCy32nOuhoXUYvnDoYXj8/rtC28clTqQPnrVDJ7vrqccfw5J/3IlN\na5NxyUQFcR/OZtyHMtkD1cw+Grr9EaLVUyHlHj5sE/9ikgmlEuuS4xYx+7rgPSMxUh9s7MKzzzwd\nug4VGAZgmcDP/vxywF2h43KgD66GpiYMaWnBpxs2xZqXKDh5u/0PxqUXnYeb7p+HdxY/Hz0WKGl3\nn+KYwoy+pGKbyuMQvP3xWlx/2XdR/97zuPLrxwmJk1CKoEyAuPeDvqa+qdf11qXhpvEcuF1tJeHJ\ncq3G+2bdhR3HjMReh07hu8V4iKq3ZJiwM2mc87MFpedfgvDA8HRkzafRo0dXnYm1JE+1QpLrmDx5\nMu69914AwMqVK7F06VIcfPDBVe02btyIM844A6eeeiq+/OUvB757/vnnsWDBAixcuBCXXXYZDjnk\nkFDiBCRQnmXOnDm44YYb0N7ejnQ6jbq6Otxxxx3YaaedcMstt2DkyJE48cSS5sZ9992H3//+9zAM\nAwcddBCuvPLKSG4/1fIsREW84HhI26mAmnhLXe8bSFtPvk+IkQ4MA3jiX//C+nXr8LnJx1c+jxPb\nFAa6b9USLzS6Ozsx76G/oX3TRhx17HRsyAxTui5pdxmv1ImsTh57zQur2oQSByQ4Xqc/ABU185FN\nWZx7wwLcdumRoSVXRPeaLavDq3e4evkryOVyOOgQ9ewRGV5/8TnYmQwm7rNP5NIVbrqlcnhsWr0S\nD8z6E849+fiwyyLDcV3c9KtbsceO2+CLe+6gdW2UWnhJWbVCx445Fun/k3fewN2/+jFGtDTitG+c\nDtsKT4mP4najrwFQcr8VXNhpK7wf2jpVJmsLn30Zn65bh6+c/s1gW9XacRFqzAnr1TH9imo8Vrnq\nFOewOcuznPj75xK3PN37Tb5VKCp6enrwgx/8AMuXL0cqlcKll16Kww8/HAAC/OOGG27A7NmzscMO\nO1Tcfl/72teqvFsPPPAAFi1ahJtvvjl07C22th1LkNpzeVAyNdp17PoCxI1Iz+2vf/sb6hsasOv+\nh0YiNKpgy7zQde5066rlcj14/JEHMaR1KA46crK0xIvIvRWVTLVkLUzeeQS3yG6YWy5rmfjcmBZu\nO/Z61f4A4Eu7j8SMK+dg8k6f4otTj8OYkcO16g7y2rHEim3/tz/8GueedwHsdFpp38Kw9JnHYdk2\n9p+4T6TrebEmN/54Ji458yRYcTOkRCgfvI/97a94/q0PcPaUAzG0sb66HafmGw45DYZpwC/6wJN/\nlpKVmsRV1crFmK5Dz37Tcew3rsEJB2+LM47aB0Pq1QssA4iWUVcmPt62++KcGxZS7rdgoWAaPKL2\n4utv4uU33sZZF1yUSBFfJSgUvgbKVifLhpUy+TFVA6ww8EAgT5sTA1JhXEVxlY1bKvZhKI6KMY1t\nQ9yIdWXiRNyMx02bBsdxMKbJVhZM1BVTZDO04gajZ7N1OPYrJ+OgI0t+5b1GNmKvkdU//Dz3loqq\nuAiHbNeKyTuPQE/BxayZU/FpR6/bjs72k7nleO473jxJfy+sahP2RSQZ3njrHexiPo/dJ+yDNiMT\nKinBy6RkwcvAO+XqUvueznbU1dXBTqcTyww1DMCP+kMk0LP54uf3wz+feSmZCTKgM/GmfPkruPA7\n5+D//vkC5i1+M9COJ6zpT5gEyzLxxnvrYZFnReTCk7nZ4qAGxGltWydu+8cC/O6WmzD75u/hvLO+\noU+cgGhSBFSA+e2XHgEn78jFNDkuwjdWrMRTLy/BmedfGCnTMzI0FPW/du08AIDlcF4YB8uybFEY\nmOQpoyYXICqRois9oHMAqehBsW3o+WTKFic6WH3y1KMBGMg7biih0UlVB8QHtUw3KqrSNUuiWHkD\ngE9eVNCStSrX1qUtrOusLpNCLFAiciaSWxB9/rkxLVKi5zoFrHj8r3jgkUfx1XMuRNPYnUqfh5BR\nlryGgbS/86rJMAxg8YJHccxxx2tpk4U9492pRnnQn0xZWXD4fOawqXh52duhc9MG5+BtaWzA5Zdd\niGzaxs/+thAbO7vFdeCGjcLJV87FhHHD4axfDX+vo8TK5WwcUhxQ9em4n6tez+CtVZ/i5w8swoPP\nLcVpp5+Ky2ZMwbbempoIWYaBxEVlV70sF9NkEgJWvL8S/1jwNM7//mUwRLpJUaB4rUp8n5PPY/bM\nwfp1/y0YkORJJ2BcNZtOBNEBxApukr9VhDfpNnVWqmo+NOljhQvbc3mh1ULFYsFCZmUSxdvEqbWW\nz+Xw0v3/i5auTwAELUI6quJAr1uMZ3Ea3lhdJkUlMFykR8V+Lusra5nYY0Q9ljz0Z3zu4MPx5dPO\ngKGpbUPIKwCl/f6kMw/LMnHi5Y9gw4Z1GLvNGOUXBCWSZRjwwf9BCRMRBPiHj2EY2GmbrbHiP6vY\nLuNBUiZk0glfxnfOPxt3/PMFPPLMSyisXSWpA7caxmvzQy1LRGMJgLw8jKygcNkCVjz064E+VEvO\nBNqlSynXTy17D9ff/ziWfvAJLr7oPJx/wTkYNqS5skdViFiLThtk7BALFiFaHy9eiHvmLMD3f3gF\nTNPkk/EIBErbeiUgRKQfAOoJFIMY8BiY5CkBxVWVIsAiMkQfNvS/VUgZ3WbJinXIlPvlFRkWlc4Y\n08SPY4nqbmOtTCLSFYWcschkszj17POxfMmreP4vt8H7YCm6C73lB1TFNIkFqaSb1GtxmvfOWixa\nuVHLgsRC5XNRX1N3Ho7jdx+JbYfU48zzL8SYbbcTrkFl/1T32y366OhxcO+Pj8W3z/sOikW1FwRV\nK6xJFwYOdKAoIghwD59jvnoa/rFAX6+lAsGBn/l0eUWFmm0/pKkRl192IYY3N+LHN/0Kzz4bVDyv\nCE6+9A+1DLcyKZKRLCkJKlvAzrlhIdJ1Wcy4utxHYyuM1tHwfMBoHS22SFEWNK9pGB5YV4efPr0S\nxaKPa370fcz4+teQScuJUcnNOTFc4LKP8e77H+Cuf8zHJT+6MqAeTpPxSC48hdIokfoZxH8NttiA\n8aTABpazgegAqrL2VDSo6qxUxUXHI3CGATSkbRiGAd/30VXo1e3xPA8fffghrGFbV10HxMvICwtU\nTjJovVgsYvHzz2Dp4pcw8cCDsNe+n5UGlxNkLRPH7DICpmmgWPSxvruArZqqg7dFgedJZveRvvYa\n2VgJtD/l6rm486rJsCwT7T3x9lG3XcFx0aOgZRZcgzx5Imul8NorLyPX3Y1D9tuz6ns6GByAmggh\nhV/d+FOcd+qXUadZSFmW+UVrC2XWLOO3T9lwt9kHk7/xUxy6s4e90x04as/tkTI581AQrQTQGzj+\n+uO97dN1sA7/Wq8+07/uquqL9FPI5WHaaViWCWfdatjDR1d+v7gL76xcFxj3pYfwRuOu+NeLr8L1\nijjmmGOwyy67IrPqFWUdJW+7iZVxUh+8XJsSK5p4ZfkKzHv6BVx46Q9hiTIBFYO5eVAWzAztpzeL\nNNBPAgHtgwHj/RcD0vIkQ5RgbRl4FqGAi43zdq9CR3sULF9WysSpVz8GKxW8TaZpYs5DD6Ljw3e4\n17lFP5pVyDJDLR20lSpq7BOBaZrY78CD8fXzLsJe+5ZqCpG4KF6AOQ3LKu+NZeK5j/iFhmutdN65\naQNem3MP1r68EECv5W/WtVNhWyZmXMnfRx0LnkrNQrq/DGM9UnkWZVZYYpmaecfzSNkW9w3dcjor\nVgAt7SeUDrCpM76JB1/5SM/qIdIc4nyXHzme+aylkjHmdXdg/p9+iEu/ewFGNKTxiwcW4c6FL6K9\nm9lvgcWJtjYZrz/Od9/R1qv1nwRIFQGxdpnP3gsrZZQsGcNGwVm/unTd2lWB6+zhW2PahXfiD/f/\nAzc+9DQ+WPQgzjr9VFxyycW47p6VvYHukv2j4bpF3H3tFLhuEUiFSxbUGoteeBX/fmUpvv/DK8TE\nCYiu3YSQOCZFS1SJgFUTpyQD2lUSpAbR99iiLE+iN2j67TvsLVtVuZxup6N2rgPZXIvFIm7/7a9x\n4BcOwlY77xX4Lop1iFyTdzxk7FRiFhERVK1jD917N4aN2Ar2dnuhrrG3VMEh27VidHMWqzVLsMRF\nx8b1KHywFO+9/SZaW1tx+NHHY8jQoJ6VZRoY1ZgJ7E+YtEAUsLpcLXW2lvSG7Lllf2YWv/QCip6L\nL+wdJDjab+/02zhlNRhvv4gf/OAHyK5+XdnqIbU8jRwPq6EZS1asw4SdhiH1n8XIDxsHq7GlRBBy\nHYEabfSYq9asw333/x1dXV0YN2o4Dt1rJ77EATgyBRIrk//Z42EPHVVtpWLkDXh9otAD1ytiycqP\n8dxbH6B76HbYYbc9cNTBX8BW7loAJRmAJe+ux4Rxw0t7Ura4qexbpfyJqv5SDfHA40+jq7sHJ515\njrgRa9UJ+78GlJ9pw4RX11rxDqR6Nlbir4zGYRVFdB1rGIt2rxEtQ+pgJ1yQXgWDlic5thjyJBK+\npAlI3vOk4pj9QfuJPdBkB5zv+7jz//6AnXfZFdvvXXooo2g0sde8u6ELeYl1Jq4OlA5xMOHj7TeX\n47UXn0NnRwcaGhsx8cCD0NG0NersVOLlYFiXHm0B830fj/7tXoz/zD44aN8JaKlLc9dASA35W7Re\nVQLJa8f2OaYprUXiybOeLwTdfPR39M/Bq0/MQ1NjA/bde0JvQ02XCe9QIp8tW/YmXvz3kzjjiAnC\n67mQ6CHlR44vqVp3tSOzfgUAwNt2Ik6+Sl1faMX9v8aTb7yHjZ3dqEvb2HuXHbDPtiNQR8cQMXPg\n6j4xpAqA1I2HdB16Otux7D+fYPG7q9Dek0PKNDFh+9E49JjjUF+XBVJ2iRAyApS0q7IKKRvetvtW\nxg7sQboO3pgJ/O/6CHc+OA/DhjRjyv87VdgmjNwokR8RudJ5pkXkCYqimgpocxtQV59GY0Pfx1MN\nkic5Bix58tNm1UERFp/U1pOX1rlLWnVc1zqlQ97o/v52370YOWoUdtjnQADxLE+q10S1nOgQL94Y\nHe1tWPzcM9hh3C7YdsdxVdc8+fo7qGtsQrZeL0bAyeewa7aA3MY1eOmV1/DZo47B0OEjtNfAzrkW\nRJPX5/Lnn8RhRxyp1CeJp0uZJrxiEa5XrNRtFP0cvLxwDoYObcXeewVjnpSVk2WHUtlF8sufXodz\nTvwSmhrqlYQYldSuKYLhFFwYqRQsy4Tb2ca/RkIu2pp3wJK33sFLTz+B/MqlAICth7Vg/NiRGDO8\nBa1ROZmWAAAgAElEQVQNdb0VEzikjhcfVVi7ChufuBfvr9mAdz9ZjzUbO1D0fQA+MraN8duMxP6T\npqClsUFtrkD1vjF7qRQr1seWJ9+08Ns/348Ju+6Ez0+dLm4YRm4UyE9k8kWebepv7lhR47A4PzuD\nMU/9FwOSPHV25ZHJ8EkGS1J4hEREZJK0PNF9AQjtV4e88ebpeR5SqRRWdRQARAsa170mamC6CvGK\nSjpeX/wi3l62FD3dXTBQOsh8+PjisdMxcusxVe3nP/R3fPrJx6jLZjF+h23wx/mrcO9NZ2FVjy8d\nT5XUyCxPYVAlaS8+/yw2bdqIoyaJa0XSoJ+1u6+dAt/3YaXMyvPEe75E5KnUYemXflyLwCdrPsXD\n996NM777AyVSJLSg0KCtKddNLZXGueQIqVWFSyA44/luAavWrMOrT/4Lqze0YWNXj9RqbBglkgCv\nnF2asmAUXbQ21mPHkcOw24EHY/SIoaV0fEWEkR3h9zJyGkVBPAa6h+2CW373exx+0Ocxca/dQttz\nnyOKeEgJvSqxYYgM6bPguEjbVqVv0TOt684WtR8kT/0XA5I8Oa6nZSHScWckEb+kkpHHgwp5Y7Pw\nuh2nSj2dEKj+DBnxCnN3JTVOnPHGNmfRXGdXZdOFueh0CCeJY3LcIrodj+sedLwiZv/mRlz8gx9p\nHbrkWXO9IqyUGZoxuvhfj2LIkBbsM2EvfocRDyUWv/3lL/DVb12E7/5mcajrSJU4kDgerXgeDoHY\nXFYZIcgcRWRHlWDy+uwjtHUX8PP7F+DVjdvjwVu+oWSlYYkGlywBQkIfJU6PjmHiWZqiPOts/7yf\nnUHy1H8xIMmTzPLEolbB3GHQtTwRhM2XELMZV83FXVdPDlgMaKgQqFoWGY4KnturVrILbJtPOkup\n9roxYqxVTKWgbxg5C4xx3VS8vZZveVv61D8xdNgwfHZ/9V9K5BkjHiaRK5vGK4vmoqWlWUyeILEI\nAMoBs51dXbj59v/DFVdfG5nkkM8DxGHVkpIbLS456CtyETKOKpHTIXyqbtCk1v/W+x9i1sP/xHk/\n+h8M32orNTLDEo2uDTAahlb+7+Tzvc+g0yl3FWtKGrheEcViMWB5SgqDlqeBhwEpVZDPOWjrySPv\nyYmITnmKpJH3etO/VQQ5CcKIHpFKmD1zasViQMQN6RT1kXUmOj5aEbiWTomPqxReC/BS+KMSJxU5\nALomXL2dwi4jGjG6KTwwUyRGSsZg52yZRiSB0coYPXzB043r1+Gtt94MJU70c8GKuvq+mmAsAAgE\nxitgU7/ddHMpoFYjZbuxoQH77zgcT973ezXrjuggZ1XGSfwR3T5l66tq94EyN12XjwuZTAODzPoV\n8jIoGn2GzksDc596AfOfeRGXXf+rMnEqqEkGUNIErleEazVQUgWFKsFKroyBTiYeJYJppUzY+bbS\nM86rXRcDuhIfg9j8GJDkCSi9LcuIkW79uiRBDqhMKqgMzptjFFQOO0pjiuxHXXk/UqkUXl28GM/N\nfQC+7wfIUpSDnBCAWiJuQWLdvlhdpuXvb0Bz1sY2CoSS6C8Ra5WIjJLPRzVmlNdGrgEg1Xj6+M3X\ncNbZknRuBMmS7GcijLSbpomiSmFg6oCyMxkYhqGt4nzU9JPx9Euvor2zW6m9CKS8h8il5203Efkx\n+8QiAwFCkQSJCiMxZctPgBiGzW1YdWJFFSQlbZTmpQjPK+I3sx6AXyzi3O9dhmx9ffn5SFc9H8JS\nP2Xicto182BnMrDcrjLxaKsiSzxCr0TmKYsp26drN8r7iKJWnoCg5iD6FgOSPBmGEUqMdOrXhY+n\n11aFtMW1itEWg7znIZu2KuVeSgelgZNOORXbbr89/v6HXyMDr0KWAGiRlLHNWew6ohE7ttbX3FKl\nIgqZZF+kTb7gYfwOJfN/k4RQ0p+Pasxgl+GN2KbsjmPJKEtS13YXuPOh+2SvEWFMUxqTpkxFfQMn\nC6sM9lkEov9MmKYJT/OHqOTm8CtWAZ3D4VsXXIhf3fVXfkkYHkSHucBSZDW04OQr58IwjehkoEwo\nZlz9GJBtSqa8iYTE0ESNEEMAfGtQBLIjI5sA5ORKAZs6OnHtb/+Eoz4/EZO+cgrgF1FwSs9jwXGr\nsuaE5VMoQlNwXBgNQ+HapZcNrgWHIfRhZJ4lWIE+Q/qoImcKRCpJQc1B9B0GJHnyfV/pENBxl4mg\nS3JUSBt7qGnE+XLH830gX3AxYdzwKtL22c/tj6+efAqunXktfnrW7hWyJCIWXDXsOrv0y8IyUW+n\ntCxQUaxVScZhqfaVtlNYsmId7rluKjoYQknWILLeNWVtdOSrySht/co7HnYa2oBRjUG3IGuxCnMJ\nAsA2LfzahizYZxEIEm4dOGYWVkOrVhFV33VgW2Y5jqNNa7yhra04/sgv4I6/PRraVtud5Dlwu9pK\ncTBFPzIZIERn9rVTYFkmTr5qbiyrDAGXxAjIUNVnZOwwSxLVL7smFmR/Aai5ADl48/3/4KY/3ofv\nXnwZxn3u8NKHhom0beGcny1AmlWvlymHG2aJ0HRtQNq2qomMiKSrqJGLyBFpGzIv+lolUmSmQgnd\noMJ4/8SADBgnIplJZcbJVJajaj+FzY0ElBccD2k7lbg8AtuXUyhgwT/nYcoxxwmDyUUBzdu2ZNGU\ntUtqzeOG4621yQdIs+jrYHYy1468gw/bqrPnOnIOmspkiQSJh6mIE2QsEzsNbagKMJcFnrPq4WSc\nTMpENp1CwfHQ7bhVYxGwGmN0UDignsBArn/p30/hV399Bw/fdq6ebg0Qyx3xj1l/xNCWJhy+/z78\nBoHA8KlI/UejLhtFNGIhZSM3agLsjA0n7yC76uXY/VXFZnmOVBk8IJLJSCxEzVYk12tn7DF49Inn\n8N5Hq/HNCy4KFPcFFDLfBJIBYVIBUoS4yOKIbPZeW4CdSSvpTbHyBzQGFcb7Lwak5YmAJSfa7rUQ\nq1Ic119Y25zroT2XR9pOJRaXJbO02ek0phxTqrU1pqnaciGKgxrbnEVjxkbO8bDXuOFoY4KXRbE+\nUeKqwvqsJYgljiZOYdYlVRfjiPqS+vddV08WWqY68sF9pUkUvY/ZdOl5SdspocWS91zTlk7dWEDf\nB5x8Add/+wvK9cMqb912vCyh40/5BhYvewfvr/qE34BYWMoBxErxPcz1ScBOlywodtqKZXlirWg8\nVx1NdCqfrV/Bd9OJ1peuK7kafQDZJnHwvKoFi4OC4+KmP90H007jnAsvriJOAMfNxgkQr3zOsQpF\nCrRmy7iEzSmsD+611fFXAVBrSdsW/K4N3PF830c+L35JGsTmw4AmTzR03GukrZUyMeMq+SGi4/rT\nJT/FYnJxWQSqfYxpSgdIFM9dFCg4a6Xw7vqugCCkjCBFDf6OQ7rigmdN66DW8GFbMEicviaMRJ58\n5VxYlhm4FiiRto6cg6aMzSWL9D7++e67kcs75VgPr0rfC+DH3KkUsw5DBgU47b1xH1Ioxpao4tsX\nXYw7/joHXT18kprZtBLwfZx27Txlt1mSmWOEYNx+6RF6LjLO9wEClK6rJkS8vj1Hi+Tkt9od3pgJ\nsCwT5924EJZlIj98Z+F+hMVC8fDBx2sw89Y78aUZZ+KLp35b7rqixC15bi6aiHMJSYwadly3Wtzg\n7fK1UhLGuv+K4vMln+vbEjmDUMMWQZ5Ug7RJOj9pa6VMzJ4ZfoioHDAseVMlUknEZcVBndMBp1By\n47GWFJYAkXp3qhlkOsHfdJp/GOmqNaEi69uptb5icSKkhwSJ00SHljyQkkiO5IBlGmiqk5PFj9pz\nmHXvvRgybDhyXhHtubzQZSeyltLPWZRnzjRNeK6rFscREhcSCqaNbds4/6KLcdOf7qsKICdEwCm4\nJeLQ3aEkBJlE5hiNMIKhRNY48go6Vh/pHMgaqSB3xyni9suOhFNwYNU3yfdDw+L08L/+jQf++RR+\ndPV12GG3PdRItIhwM59bTqe+pYk3rmC8yMHbgjFkJGxQnmBgY4sgTyruNVo+gG6bBHFhyVuUIHNR\nvzqfR0GuJ4d7b78JbR+8BaA6wJolQKxl6JPOvJQgqVicWKuNjHTV2qVHry+bTuHbNy5AnVXSgBJl\n1blFHx15B3deNRmGgaqgcNl6RjVm4LpF3HOdmCyuWvYyerq7cchhpUDbMMUAETminzM6HkoFruvC\nstNqFiXWnaJxMInaDB82FFMO+hxmPfx474cUCbLTFtyeTlj1TeHWpBiuqLB+uVAha+XPWAKkbfWR\nBHznt9o9EORu2yWdONu24HZ3xN6P7p4cfvr72Wisr8P5F1+GtJ0KD9AmEBBu4vb98zWTez/XsAoJ\nnzneeBEtprwxlLPuBuUJBiy2CPIEVB8Y9KHAkhtawDKpgHNCyPIFNxF9KREBS1r4c+sxY3DllVdj\n5Yq3MH/2Hcj1VGvrsLE4rGUoTmC3yE0nsjjV2qVHry9X8HDrJUfCtkzMuFKcVTe2OYumjA3bKh1G\nvLnxsubIek6fOQ++jyqXHgC4a1Zi2Wuv4LRTT9NaRxRrqQyu58FKGaGHYeDQKJfHIP8PPZhCDq99\njjgGXrGIF15/s/QBTYK6O2DVNSpbk6K4opTByWCTkbUqqxT5Xha3pGot4xA3sna3s60yp8yaZbH2\n440VK/GT38/G1755Dg4+9iuVz3WsK7z4J1qgsiJMqeoGDnmeqsZTycZTGSNK1t0gBhwGdLadCLys\nM52iv6pZfGyf5Lq4BYbrrBQyZQLG1huLmv0nAt3nTd/eG7f97n+x736fxc6fPVh6XZLZcNo15crZ\nbx8mmAnCgq5FJ8uqozPm7rp6MizLrKp3x86dXqds7U3Fbsz64x/wWvfe+Mv/HJPI/SbQfZZeXPAI\nRo4Ygb322F1ay4uuAVZdOqOUgcTNYlIsLuz7Pn51w/U45pADMH7c9qUPRdlofVynDQjJYOPNR5DN\nJutHt8ae9pw0kCsU8If756Cxvg4zzjpXq75iGNx0MwzLhpUy9bPrFJ8n2bU68xTXzwvPupNhsDxL\n/8UWR55kh4IKKVIlPmGHT9RixKRfIgvAziMuMePNi5A10ueqjz7EmLHbAODXyKuFjIBO4dxtWkqW\nnjgFg6POjwe2Ph5PbkBVloBgTFMaWSuFjJ2CV/TheUX0JBwXp/MsvbxwDoYObcXee+1Z+kBwwDjZ\nVqRtCwXHhZ3bKK9wX4ZKGxrFYhE33/gTfPHz+2Hv3ajsOspSs1mK+EZM6+eRPmE/UaUDEiaSvu9j\nzhPPYfGyd3DGCVMxaq+ED8UyESc/L37n+tLHCsWndZ+npObLy6iLTODKGCRP/RdbHHkCohMMnbdx\nVjsnatyUzEqWL7jcAzOqvlVUixxNoGpJXFQsUGFFeTcXWAKka2WiMaYprVwAOq7Wmcr1hgG88sRj\naKxvwMR9PhM8DJzOQDo593AzU+JsIt41QGgdMt/3cesvb8T+E3bHAZ8ZHyQg61fE1iaqgiL5iEza\nmP61LE99bGF7+uXX8fizL2PqIftj4pHH9n6RMEmRW3TEulAqBKvPEXFvNid5OuLq+Vi1IV6JJBpj\nhtZj4bWTEutvc2OLiXmiETWDTVXXicSKANAeRxaLRb4j8xdZGqIclryxVLMUxzSlsXWjjbFNJffV\n6++uSzzmSDWeKcn6d1HnKZoX3Ya3FrYeHg9EPkJWAJogLGZJVcNJBjKGmW2B47qVeI4ZV82FYdnB\nWA5eHbB0c6l8hijeg7mGrhsmixUxDAPfvugSLHnrPfzrxSVUXE8LgPilRGjoyBpEjqVi5ijrh/4u\nUcmFECx56z1c89s/oTuXx+XXzAwQp0Sz1MrgxUsRZXGhBSdK3FJfoL/MYxCJYYskT0D0t/Ew4sWr\nFxYGcoixh52MrCVtD+SNpSMC2tnZgV/e8GMce+6tmDBueJWoY1y4RR9OOevMcYvSvpOsf8cDXZuO\nhmqmn4zg8aQOgJIlpd7tCHxGSqiQsjH0PQojvkkkFtBjNDQ2Il+uT+fk8wFSRwfj6tQBI6hc43TC\nzpRc1nYmE3qtYRg447zv4qPVn+Cvf5mF2TOnVIQyEwsIl2XK6dTTiwJZP56TnOSC7LqUjQ8/+RQ/\nvv1uvPn+f/DDK6/BEdNOhBF4C0wuS60KHBevlIyjDyUAdPTLYmqdDaL/4b/yjoa9kYdpPumIDNKH\nmMzKxHPHJA3eWKpWuhGtrfjhj67AWZNG4+prrsELzz2rXrRVARnLRNo2y3WuTGQs+aOpI7qpA1rj\niSY5upl+PIKXsUxuH7mebvz9f2/Ghk9WB8gOedbG7zC0qn/Zc6hqUaQRVlw7ZWfQXc7EJIcT9w2f\ndk/oWAHKKehOPl+p0eh6RaVrTz7zXIy28/if62bixB/cLxeU1IUgU64vLT66c9OBbB3vuS346b2P\n45FXPsB3Lr4UJ5x2JlclPLEstSSvoZ9HSX/KCCsALMFgtt2WiQFNnqIQjCTeyFUJR5VEgqKVKWk5\nAho8vqMS85JNW/jG/zyOadOn48JLLkN7ezvu+e3PgbY1icwr7xaRK3i4/bIjkSt4FUFOVfDIDFvI\nV6UPWuPp3BsWVEhOFHchGwO107AG5B0v0EfHRytw///egnPO+RZ+/uCnVWQn73kBtx2dzCTTc4pK\n8FmQMfxUCoUClTwQJlNQBm1VUoFVaIeTL5TiVVyn14JgUnPjHHqfnzIN5513Hg7aehX+739vQyHX\nozSeCqqsWDUQ2Yw8t/UrolvYeOtI2Vi9dj1u/OP9+OcTT+PFDbvg7PMuQF19g7QrbWuPDuGiaiQq\nXaOgKxaL/OiQOIGUwSAGPgZkwHhXdx5F6BU3BWqT6h8GOiCbuGFkY6rOURTom0SxZB54geW+76NY\nLFbeRkUFh3WQsUwhcdIpQEwHlt9z3VT4PrSkEPKOh4ydkmbQqcIyDew6orFyT1es70JPwcUzj9yP\nYrGIk045DfVpW/g8Ry0irRoMrvK8vbviHXz85ms4dmo54JMNzGVkCejA77jp4nQGn1H05EHETidW\nvrQIsx5+HJ/dc1dMPWT/oIspDLUMDJf1LZAxCCuzEjejkO7j/ZUr8Y+589CYzWDGV76EuqFbR84S\nCwOdym8V2hTaUXOQBF8rSQRoBJW76RbYmXQl+5m01ZFMcO3GSlsAWns6GDDefzEgKXA6QnFTQO2N\nPGl3GXlzB1BROI87x1oIaIatm2flMAwjYMZn6+XxEGYBEhEn1QLExN1XsRSVY6h03W3vbuzG2+tK\n1hJ63KhxXo5bxN3XToHjFjGi3sLG95dj730nYsZpp8M0Tak1M2oRaRUSrWqlKhaLMFPUrwtObS5+\n4HdLtDp35DAzU0jbVqWAKu8t3rBseEUfhlWyAG0/8RBcfs1MtDQ14prf/Amvv/2e0pCRAsPXr4jd\nN++70LlEtX4x7TKfLsfiObNw43VX4PlXluCFDbvinAu+i6am5trFDgWsMWmpUj332ZFYnOh+hVYq\nDQsWIWATxg0PtFXZG2KxAlAmcxkYVul3VRI1HwexeTEg7953frGoEg+hW1BXdkglFWTLg4js8drL\n5iiKZ4kS51KZG7Vu2XWq+7xx5ZtY/uxCFJk6IlFLq4hihYBgcHbe8bDT0IZgmZe1nehmXGWigHC6\nTwJZnBMpjhwGt+ij2/FgGga8YhG+D0zYex/stvv4QDvZ/taiiDSBihva8zykGAFE9gBhA79DDzEV\nFD0UHLdcDNnllNQwYKVMnHr1Y7BSwfntP/l4/Ojqa7Hs3Q/w09/PxsefrhOPEyAjLWrFhYeNUyNb\nIUHnVkMzzrlhYcB1FkqMdOOdUnaAkLmeh0cWPYuZt96FtWvX4ZIfXo4Tv/pV3PeT49WK7sY5+FXJ\nS1g7dg5MexnBUSKGgf4K1W3LJV14IC8NvckP6UA91X6VCTiISBiQbjvH9SpusKQVl4l+UBSXnkwz\niadGHqYTJXK7iMaJom+lqieki1defgmLFi7AiK22wt6HTsaIESMi6TPx3Gg811vGMrHT0AZh/8Td\nptofOz7bbmxzFvV2CrZlok2gKE6DaDeFuWzZ79nPauWWDcOyN5aic/X7+OLhhyq1T1yokNaKYsQH\nC46LtG1JXSFd3d24547fIZcv4MyvHI2mhvqqNvmtdofV2ALXLQK5DrkrTFOsUuZiy42ZCDtjw8k7\nyK56ObQ9O48w4kT6gmHg/136V0zeeSPWv7sUUw6aiAmHTg02VrhPcUQflcZiP1cRVqXbAskSE8E8\nZXOg3YKuV0SxWOx9Rok2msJeD7rt+i8GpOWpoz2nRZyIphH5Nw/EfXHX1ZMBINS9xhtDZvmh3+6J\npcdKmeXgZHGqeZ0kiJclOPTnuq5MmZ5QFOwzcT/88Ic/wldPmI4lT8zDX27/FVatXq0VcE1bnDJ2\nCis3dQtJSt4tSgO6icVJFBAuAi9rjvRjmkapll0dv49isYgPljyP9195FoCc9PCsnrzPak2cRPfd\nc13YlqVsceDVDYsFWmSTKeKati259g+Ahvp6nHXBRTjx62fi13f/HXOeeK4qWzSzfgXg+zjt2nnh\nrjDPgVO2BDoFN5TACKUTUjbs8u8NO20JCwTL5iFFygYy9Zh65i/wk5/8BIdt+wkmT5qESy+/opo4\nAeH3KaIsAReKSQc8ixNvDpVr7caq9rEgIHjCfaAsVq5XxPk//1fwGWVqPg5iYGJAWp46u/LIZNQs\nLMQaU2L/vjTgNm5AuYrlhx3D9YpwvSITSyQv0ZLEPHjzSkIxne6PXuem7lz5c6Oyp7IAc9ZCpGIp\nGtucRXOdLawtx+s3qlK6zPLU2dGOxQsfxSerV+OAz38B+x/4eWnNL95zB2ze5Ab2/r+y+GX4+W4c\nceRRysG7SohxfRwLyHOPPYjHn3sZX582BduPGVX5XMfi4227L865YSFuv/SIWCrmlTG7O5BZsyxS\nHyxcz8PTL7+O55csh984Al849HBM3HdfpL3u2EQ2McsTC61A7mrLJu9aEvBd26B3sdq5k2kJlCvS\nXeeg5an/YkCSJ8f1lNxr9KF097VTkDKNStFS0XVxa8eZZik+RQYyBiFHvLmIigOHIW7NvSTdQvRe\nAvzsyPa2Niz453zsfsChaGoZAqC6BMvKTd3Yfki91OWnU7ZFp46eDMTaRPoY3ZDC7b/5NSzLwrHH\nT8fWY8Yo9xW3mDUPuvUVZc/NinfexicfrsSs563KL3w6iyjKwZTIIRyDfOULBdz9+1vhuh5OnzYZ\njfV1pS9qkXEXljk3cjys+qZYmXPFYhEvLn0LT720BEXfx0H77omJRx5bSupIur5bjerFaT0TzBzY\na8n/a1qqReYmlJAk1XW2e40wDAySp36IAUmeOrvySFmmUnyOjuWJIMnacaL+G9J22Qrjo6vgJErk\nkoyJigviBpIdzB+sXImFj89HR3s7tho5CjtP/Dz2231cINZIpS4cadORc/BhAurjquSKFPHNpi20\ndXTBT6kpz9MQ3ZtaP4uq1/R0d+Nv9/wZ51/wnUrcRqwaYv2oBtmapc/jr/OfQE8uj33H74ID9x7P\njYniQiPmSEiM2PipVUuAgppOVWd3D55/bTleXvY2fN/HfnvsigOnnoC0HZMwqV5bCxIVsU9WssJo\nHFZ5QQ2TRIiKMBIk/T5kneTafN5FY0Mm6amHYpA8yTEgydOGDV1oas4qW2XIAe77tQu4VXH50WPr\nEC0VrR6VQOO+dgPRUF3vJ6tX45mnnkRrayuOnDQp4OILIzNjm7NoqbPhuEV0O15A80nXwrRNcxZN\nHLKWz+XQ9uE72GrkSIwZMxZA796GWTVFiHJvZM9FnHst6/eOm3+G711wnvbbswg1c/9ERLFYxNKn\n5uHZV5ehs7sH2Uwan91zV+y7xy7IpuUSHEIoBpYTguUUXNhpG25XWxXR8mDi7XffxyvL38FHa9aV\nXsLqsvjsnrthwqFTkEqltILoRVC9L/3q/nHIeFzLaJQxowbAh/Vt10AwOQyD5EmOAUme1q/vRLpO\nLCrYF+AdMjrZdqI+dCHrl0fWeDFWcaG6jiTW+/yz/8Z7767AuJ13Qd3IbTFk6DAYRkkyYJcRjZhR\ndtGahoG313ViVGMm1GLFYpuWkgXr5Cvn4sZvTcCCRYvw0YcfAgAy2Sx223089pk4EQ0Nvab0ettC\n2k6h4HjodlztdelYnlSIaC2sjP938w246Pxzgh9uxpinWqO7uwevPvEYFi97G/mCg/psBvvsvjP2\nGb8zGurU5TaU3XvpOnhjP4OTrngUN317b3zw3DyseP8DrNvYBrd+GKxsPXbcdgwmTtgdW48aVVVf\nDgCMxmEVEi891CWZbqqEoKaWwwhZc7qCmklAy7JkpuBaDcqEbtDy1L8xYMlT0TYjW2XigsQj5Qsu\nepj4FPozeg6qlgCdNfEsHiTom1ajznteLMuIDDqHdFJksbN9E95Y/iaWv/kWNmxYB6DUsZUysd02\n26B5yBBsv8OOMIduXRUHtXHjRnS0t8E0U0hZKXR1dMDvWI/hw4djt/F7VAXrL1v+FjzDwNhtthUq\nVSdl1WPj5UTEOMlnSQdc8hSGWh5enMMpkJWX8Dg9PT1Y+vQ/sXjZO+jqycE0DKTTNoYPaUZdNouM\nbSGbSaMum0FdNoOm+nrU12VKz03KhpPvgeO46MkX0N2TQ2dPDp1d3djU0YlP12+CbxjwmkbDMA2M\nHDkKuzZ52G7C5zB8+HCYTcOFRIU+wAEDdiZd+fnnHdJhFiNa0b0S5MygEojtFmEYQKpnY2L3mczP\n9YrwXVfP3Var503Wr4KUAtlTxy3i69fNw6xr1QjnYMxT/8WAJk9hqMXbN+lzU2ceQxozAXIiO9CS\nshawbWiLR4/rBkgSPZ86K55lhAf2IOcRR521RRkvQ/VZLBax5pPVaGvbhKamZowZu03VuO+ueAfL\n33gDXtGD53pobm7CiK1GYdvtt0Nr69DAXGXrIfPRdcOKwNMB42Xg+f7miV0DgD/eciMuPO9bypZh\nJlYAACAASURBVO0TdeuEBAerHPhRxgxz/RQcBxs2bER3Tw/y+QJy+RycT95DV08Ond059ORy8H3A\n933YtoW0ZSGbzaChLovMqB3R1NSIluZmjBg+DKZpCvdMaFUBAhag0pwkLrswi1H5e/I7RGS1ovsI\naBdJ9lI1hor0TZJ8Ij0/CZIo7ee4vIZKvFWhADudDtwjpT3DIHnqzxiw5MlPm7FrxKlkxvH6JL9Y\neBafMJkCUeabaswU3aY9l0dzNniNzPJUi3gnlczBJOOtaOJAdLWiWmBUXZ68/nj3O6qlR7Q/smxF\n2bNbq7i+O27+Gb53/rlqjRN064SmpXdvhFHfGqizF9cCRcYEEG8NUQ5xoShjb9o9AMrahIql5u3/\nbML4HYZW1WLjrS1SkHNVmwIAX+ou0yUftOVp2fsbhOvQEq+MSqYiPsdkDqdcPRd3XlUSHyZimXa+\nTSkea9Bt178xIEUyM1k7tIxKWM2uettCczaDels9M4r0eftlR6LgFMt/l8Q6iUAlKf4rup54flgB\nRJUaY2wbWcmOou9XBDNV65dFQc71kC+4FT0qUd+uV6rt5nrqv8B4ZWxoIVDVNYlqGLKiprQwKX2v\naFTuG0cQlSbCOhDdH/qZosers1JozvKf/yRKDInnqfHgqJbgCANPjJDt23MDJVxiu+6oMUkZqChr\nUBJC5Ak4CkUZ05V9oPfEcjrhd66H7zoYv8NQbi02GknUrOstxdNRdX8C69YR1Sx/R/r2XUe4DuHe\ncsaLJUgZ8Tm2Cu0oOG6FOC1ZsQ5WyoSdL7kgQ/eEWkcmo5+9O4jaY0CSJ8sy4RVLMS6yg0qkxG2a\nqBRZTdsp6IiJ9xZpNSvXkzlkUvKDiz7YeIevSo0xVkWc/T/dr8peJIGekL59v0SeUqZRimNQOIMD\ne8UhmkC8NbGEBUDFgpZJW6i3rap7Se+vqLaiKnmRKdCz8/R9oOB4ZXLgIZXiP/9x6huG4dNP12Cr\nEcMlC6r+VZJIYVnB4cX2bec2wu/aEO6yU1Gbpsb0XSfaGhRIg9ahzuxD1Z74xZL7Ml8Q12Jj+iPz\n1J13oA/2/hhG8HpAiXxU7UV5Pdy9V1T3pq1ycRTRIz3Hhom0bcEwjGoyq0LIqDb5fDJhFoNIFgOS\nPKUEhUB54B3UxSJ9GBXRlNF7U+dZfMIOLvb7POd60Xx5a6IPaXJNmIWplg7asL51iA67V7J9jbMm\nek6+j4oFjZDic362oMqyRPbX9YpV61ElLyKCJXMTkvmk7ZSwEG4tLYxvvfgUJu69N/c7KQkIi4NR\ngPDwYvsOsTjpkJXAmFGsZgpFbXUPdXpOoj2xCm3lz8ODrEWlUHQtLWQuAGA0DK1YAcn1oeQjhAxV\nIWSO7L1LxAKqe115XN/3uWSWLqItglVoR9umHuRz0ZTrB1FbDMiYJ53yLDKkUkBTJnosDhtbEpZu\nzgsKjhIALIsh2lzBxElAFIAN8NXJawEyLh0zxo4piykK2/+o8V86+8HGQyURA3XHzT/Dd889u6RW\nTSNmTIg0DibJzCnDhFfXWhGmTTI7TDcTiyBWQH0ChZaNhqHSwHFlkUzyT/o50Iw7i7QX9BzDMi03\nlySGRHJBZc2D5Vn6Lwak5amQdxNxQclihlTAtudZV2grA/u9qsWKNy5v3rV026ggznj0PrHuSOIq\n7QsySMbtdsTPmOw5CbOwRbUO5b3gfojGyDLxUEnFQBkwqokTEO3NPmmXliICFruEfjhC5ynZj6hu\nTe29YfbXTTcHLESuV6wupqtIgL261kox3sBzoBl3FtgLVddaeY5OthVGw1A42dbQtn2OspuuCkkW\nWB7EZkHsO/bQQw/hS1/6EvbYYw/MmjVL2O6FF17A3nvvjenTp2PatGk48cQTI4/Z1JxFRidQiQNy\nqABQImKqv2vZbDqWzPBcabQriF6X7ODjHaC1ctuorD3OIU32acZVc2GlzCp3JEsIosxPB3S2HS9o\nXeV6GXRjtcje0s8G+5yRv+nnzTSTI9MijSsgGgmI7dLSPWwokldwXBgNQ+MTsyQOwAiZeDpjVhEt\n6vp0OVnmtGvm6c+/3A+J6aED1yPHuPlFfWKYspC2rd71mH2vxK0FyhKViDvxvxy5XA4XXXQRJk2a\nhKOPPhqLFi3itnvzzTdxwgknYPr06TjuuONw1VVXwXF63aHLly/HqaeeimOOOQbHHnssnnrqqdCx\nY5On8ePH45e//CWOO+640Lbjxo3DAw88gAcffBD33ntv5DG/84tFsQ4DWWA1DzrEgJ6TKpkhGXqn\nXTOvsi4VKxKvv6QDw2VrFx3aUbPNZs+cCitlBvoJ67sW2WWkTzpoPOlxdMqlyNZfx8S+kectH5KN\nSfevNt+QCSv+8ieHIwDxQRtysES1vFiFdvhdGyqHbew3/s1xAOqMqZCpGHn+fhFOvgDf9zF75tTe\nLNqYavM6mXluuhlGfSsVv5pApqXGXHXBPrdVLx2D1idt3HHHHWhsbMT8+fNx22234YorrkBPT3Vd\nyB133BH33XcfHnjgATz88MPYtGlThYP09PTgggsuwKWXXoo5c+bgoYcewoQJE0LHjn23xo0bh512\n2kn6ZkqQVHjVLd8/TNmyouP2El2vSgx4B6yIzISRLNU5qpKqKJCtnQ1Yj2Lxovur7BOTASfruxZu\nStInCc5WCVpXWV/UdrL1Z8tK90tWrKvMi0g4ZNIW11VMj6f3UpDE5jKHowSRMq04qCJaRW/Av/Er\nW/rKBEeWqUgIZTRrUSnj0ysW4btO/L1UJIbknhqWjRlXlZI7/O6NauKoCRCUSC5l0XNLaWGJ+sxk\n7dhz3lIxd+5cnHTSSQCA7bbbDnvuuSeefPLJqnbpdBqWVTKUFAoF5HK5yu+0Rx55BPvtt1+FMJmm\niZaWltCx+5TqfvDBBzjhhBNw4okn4sEHH5S2bW9vx0cffRT4s3r1agBAR3tOybKi6/biQYfEiA5Y\nntq4iGTROlFhc6ylpg+Zt2pslW5cEm/utF4WAGGsWNj8ooDNqCMaXqRvHbItWl+UdgD/OaDvwYRx\nw5GXZH2KpBR0CGEiLz66lhrR4SnTDKMPR8GBlYiEgqT/PoFyUH66V82acz2Jfwoc2irroNZupUxp\n1hh7nQw6mXlWysTsmb1aX2FIJI6uPP6SFetKQp7pFrX9kj37kufIMIyKztPq1aurzsT29s1fUFsX\nSa7j448/xtZbb135/+jRoys8gcWnn36KadOm4cADD0RjY2MldGjFihVIpVI4++yzMX36dFxxxRVK\n8wn1WZ1wwglVk/F9H4Zh4N///rfyG+kee+yBRYsWobGxER999BG+8Y1vYOTIkTjwwAO57e+88078\n5je/CXw2ZswYLFy4EEOHNmBtuzwLgD5E7rluKvKep5W6Tx86OdfjXs/2pXLA0nEo91w3FYWiV7E0\n8VSrVciaaH1RwB62vLXz1qpb486itYo4c1ddm8q9CQM7d7rPHrf3XoaNI1I/Z6+jy79kFNdJE1ee\nZZJ+XngyGiyxIq49VeIZ+nOukc3kesVSlhGa9QmMX7JyWETdWlKyhT2wLHp+SVicZP1vbhgmDMuG\nV/SBlA0jM6w6q4s6tO+5bip8J7wcTQUR1q6cUScIsOZpJFlOp9q+c9YaR4KCSJrcc91UgLe3Efrk\n7eXw4Y0lnScLOOWUU7Bq1arApeeffz4uuOCCaOOGYP3K5Vi7RqOuYAiyI1sATNJah4x/PPPMM1rj\nb7XVVnjwwQeRy+VwySWXYP78+Tj66KPheR6ee+453HfffRg2bBiuv/56/PSnP8X1118v7S+UPP39\n73/XmqAIDQ0NlX+PHTsWRx11FBYvXiwkT6effjqmT58e+Ixk+6xf3wmE1LaLY5XgEQGV68MOcjoN\nnggeNmdLZQg8r6h8kMZdH4swyQRRbBWZIx3wPXumGpEjsU2k1hNN2nTdqnHXziM69FwIVN27NDHJ\nF8SuRrI+1XsoI3mkb2K9I59XXcMQrpwi8ZRZnpQPxfIB5hX9WIeYVWiH75i9h0z5TZ13OFa1TRi1\n7j8O6J+xSq06er/ZQxv8PRT2r7P2GOSFfb4spxO+KmkiSJDoWoV2OGipJPmcds288DWV18+9DxDv\n5dq1HTBNA5lMI2bNmgWPqWDR3JxcNmpfQWcdYfxjzJgx+Pjjj9HaWsq0XL16NQ444ADpNdlsFlOn\nTsXDDz+Mo48+GltvvTUOOOAADBtWisU89thjcfnll4euo8/szGvXrq38e9OmTXj66aex++67C9s3\nNzdj7NixgT+jR4/WGlM3eJoXoFyn6RJTsRSl7RQ68vlKTI1hGJUCtzpkKMr6WER14wDVZOeuqycD\nQGgmJEuOeMrsKmtLwm3p+0H17qhEjF1TDxN7JGqneg9FbmE6I5FVYSfXnHJ10L3KymXEgsjlICg7\nEhQOVJc24PUFUK4YNlWetTIl7VILE3Lc3GCyC2+/7MjaCEqqrj1qcD3zfNH3Wxfa7lrJM0PESH3X\nUVuTX6zch4Lj8tuG7Mno0aOrzsSBSJ6SXMfkyZMrgd8rV67E0qVLcfDBB1e1+/DDDyvZdYVCAQsW\nLMAuu+wCAJg6dSqWLFmCrq4uAMBTTz2F3XbbLXTs2L9R5syZg0MPPRSPPfYYbrnlFhx22GF49913\nAQC33HJLZWHz58/Hsccei+nTp+O0007D9OnTccQRR8QdPhSqhwOdDk4ON1KqI4lYWfbQ9Lze//u+\nr3WQsv2qgEc0wlTPdZD3PFgpEzOuUiNfpCaeLBg7qUB+GVj17jj3WlYqhxscr0FgMqmUsCSMjFjl\nCi7+dGWJ1NaVAyajECah245zKMpiS6xCO1I9G0PVlUuDhtQlYw5WUap80ppRof3VkqhpgJAFO7dR\nThpYC0gS8WCS+Wj1HXi+CvIYM8XYIxW46ZbwZ0ZFPZ2aW0BSYTCzLhGceeaZaGtrw6RJk3Duuefi\nuuuuQ319PYAg/3jllVdwwgknYNq0afjyl7+MIUOG4LzzzgNQInNnnXUWTjrpJBx//PFYtmwZfvjD\nH4aOPSAVxtev70QxxG1HI8y1w1N9zqZSFWtQkuKM7FxYC4LOvHXGVFUkVx2TN++sVSodYqVM5Aqu\n0H1HriVz+vM1kyvX6Ox1UmrqtVJlV1EbV93rlrpMxS1K7h8dy8SLlwNKMXbN2d57357LB9THVWAY\nwF9+/xvM+OqX0Ui534ONzIqFR0VxPMzVR753vSK+du08zJ7J70vYj+Z81DdD3p+2UjYdK8aJG4ul\nQi4aRzBWv0Z5vtz9MDRitai+RCBjKD8zinsZ5V5uToXxz5x+Mz5MMOZpm5EteO3O7ybW3+bGFk+e\nVA9GXrukyEsUJHmgk75crwjXK2qVG9Hpjz2oAVTNn16XaRhI2ykUHK8SmE0TAlViUYtA+aSuE32v\ne3/Z9rwyMiKyWm9blX3udsRZSby5knEeevBB7LLtSOy687jQuYYRGjfdAjuTxpIV6zBh3HBuaRCa\noADQKuPCjp8IAaHGkK1Ph6jR/QDgkoIkiF/vOAVYhbbw/ZCUFOGCRx5qSc449yKMZBOorN1oHFZ5\nNsmeiRCLLCtgkDz1XwxY2yHrReB5FXiKy6J+RIrdOnNICjKXj+6YdF9WygxIIZDvdUgD3Z9hGJX5\nERVwEj/kesWq+bProt1lAF+cMgxJkVsV+QEWKjFXIsubrsuR5xJktahE6HZctOfyVcRJpNvFm+fo\nMVtjzdr11MWyeJBqVwbt6rIz6epK83SfjCtQpPdUQUjat9S1ouA+CRU3pOYRVcBSpAEUW5MqME66\nctif87MFXNcXWatX1yp3WZWv47kwa1FaJwDqeeHKFoj2yUyFS0tQ2XQlQiQhD1GkKgaStW8QUgxI\nyxNbGFj2Fi8r9BrHuqNybRyrCK//qPMVXRe3P2J5yntewC0IQFhYV1Tglu2DZKXoFmuOA5nlhVfs\nOUqB37B+da+XFTBWHZ/de55b94UXXkRnexsO3/8zkd62aQsKmXPB8WDnNgAQWWH4b/2qLr+w+akW\nJ9a2/kRw4wB8y1Ml2D3Gocta+nr33w0KS5bXes7PFuC3lxyBlGlwC/zS865ybwHJukmpufHcjoF7\n6HSGWpwKjou0bcnbiyxvtXSrCjBoeeq/GJCWp0xGvX4XEW+k39B5WXW0dSQMKhl5cTPBWEtYnADp\nMJHFqP11FRzkXC8QDE8sTmk7xRXNpMU06XnRfRQcD7dfdmSi9fnCoBJQzwazx5GKiFtGR6WAsQjs\nugDxWoil8to7noOVTqu9vbOfMRaUgMXRMCVWmDQ3KDhsfKUgXlWrQQ0z0GRK3wHrTUzyYRXaAtYU\n8ruwKnCZyggrFn1+DcDyvs24ai4My4bjFntLo3B0mJIgTqwli/4/my3IBVPPz+8qEXaedUyUzSey\npsVTaA/BYFB5v8YWb3kiULHk6FgDSFvyNke/rce1SoSNmVRgc9L9yQKXVcfUjXlKArz7BUBJ/DPp\nefbVunVi/LJWCq8ufgmFXA4HT9xD+rYtfROXBP2GWmFUx9CAVj81DrBmLSi1tN6oxmwB4M6BXA8A\nXtHHeTcuLGkX0fNMar9Yy1/XBhgNQ+V7E2YhEu2vyMoosT7WyvJE+s3lXBTyzqDlqR9iQJKn9es7\n4afNqqw1lRgltg19WKsSHnJNnSXOyKvVobu5D2tetqBoT9n/14pU6s6ZB5E7kc1ArCW5iZr5GBU6\n/b/+zONI2zY+N3Gf8sUc14aOiyssyDjK91EgGgfou/gUzr5pZY9FHDMsCxKA2CVWznAzLBtWyqzd\nPFFNUOKSd1k7rc+TzuSk5kn329Gew9ChgizXGmKQPMkxYMmTjlSBKqJYsMIIWRKZVv0FUSx1cax7\ntZizDIaBikSFiqxDWF+6pJQmlqR0S395RpY9+y8YBnDAZycC0D984qLWsSXsOK5XLgNDj2WmquJ/\nkhu3JViHLoFYp8hgSCqPyAUy+JwOtXlGIb+iuC8B6dUiM6I5aBDqWM+lZA8GLU/9HwPSqRpWZTpq\nFlxYHAove48XYExiZ2SZVqyYZK0y95ICb+1hGYG8mKG4sT5x5qyyx5m0hddXrFMqhiyCKH5KhirV\ndclYff2sZK0UMkNGoNsrr0cSL1QTkUXV+KS4MSLUOIZhBMZysq0wGobCybbGG4MDuoAvgMRinSKD\nGbdq75kMPhXQMUPc+CHOvZPGfQkUupMoPM0jSCJV86iq5WEZiaTfXE9Brd9B9DkGJnnKiA+vuIHa\nqvXkSF06lQBjXqAxXcokiTIjYYh74LKHe7HIDzJmySOvjUw4M8m16AZ1k/a77zA0cl+8Z0D1/hJi\nCaBKTZysWedZSYJkkfXM/NOr6OrpqbwtSw+ppA99hUNRKz1eITjc9/3escxUUB3aTODnlCpjEypZ\nUCsoptZX7b0uSQlZo4hMRdkLIZmJupcq81BWLaflOtSSFQbRfzEgyVM+zz+8eAdX0m/pouw9gH/A\n8g47UsqEXJ9EmREZkiJnrNVIJSMw6Rp1pG1YzUGZfpcM7L2h74dKX+wzAOhbv7JpC1+7dl5lPvSa\n41i/oqBC9v/nK/jk448rv9BrWcaDB2lWk8ZBq/rGn+rZ2DtW0UPBIS9NbjzXHUsWGCKSdKaaCEpk\ns0wSefda6/77RTj5An+N4Fi2KtdE3AuR5SiK7lRS2YPMM0rvxyBJGpgYmOQp53A/Vyk4K4IOaRFZ\nXQC1+mZVxWFjpLyHzTuOJAEP7PzYQGpVS1OU+ZG2pOag6L7yiuSqIqxQsEpfIgkGHevX7JnV5Eu1\neHTS9zznevDtDDo3rg1+0Ye/9N10c3XaPDUPpQNOQ56gynWV2wi/a0NQFynKGhqHSUU8+4SUKuxD\nlZtS4CZTAe2WrFqj5N4lshdRLFhMm0TmwayTFBbuq5ePQSSPAUmeZCAHV97zavqWzrO6ENCkQYVk\nqVpH2DWoKlzHIWcqoOeha+nRmZ/vA/mCiwnjhkvjgVT1u0SfxS0UzAaL6+6JjHxFsX4lc88NCIsD\n1xpJaTvFsSIYZmyLE1kDcckG5kDPpdakNGwfknRTiu6dKkmKuxea91xopUrgnlStc9DiNKAxIMlT\n2C9x348Xo6IKXpwPC9lhR89JJZWeHkNn3rUM0ubNg5d9KIPO/HpcT3pfVV2nontGrlcV6VQltLoE\nRka+dK1fvHlGQeTE3LixO6oHoELsTRQrgptuiV9uhI6nch31OdQo7km6D0m6KaPeu4SSAoJ1/cK1\nvGoeczZImLYYDEjy1NScRdYKtwr0xVu6ComJWyNPRFB03UG1AC/Gh4aqVU9GblmE3Vee65TObgy7\nZ6pkjl1bmNp9VETVe1Ih96pwHAeWLa6fJ0JSdc5CSU/IQRdVsZuuxRf3QFVVw+bOuRaQ7EMSbkoC\nXcIqWrf2frBZgWH3rgbq6MoYVBMfcBiQd+w7v1iEbFqteGyUt3QdyEiMUnBwDNdb1HnXIoieZIlF\nsY5FyZxUJYu87EYV4qlCeOm11TOFkXlEMgriEp+k4p/y+RyyZVVp9cETfpOXaOJID1WVeQg+szMZ\nLClLVzj5Qt8FcfeFFSQMrMUpzhxU90207ij7EYEMJRpzprhfNSfJg6gJBiR5uuX7h1VqqCX1hh/H\nMsMjMSqHXhKuN91511IWQWYdc70iMin1GoBJBj3zMujiujJZ2Yp0OZA9bafQka8mklGQxB7oWChl\n/edzeWSzmuSpL97kFVPJufMI09wpX0dqwvGKFOtAekiy6wCS27uyNlOc6xM/4HVrCUZ8liKRobh7\nDQ1CxN73JGQwBtEnGJDkqVB+q69lELQuWIsTe+iZvN/pfex60z2MdeO/eGshhWVPu2YeV0yzVoV3\nVeYWdz9Z2QpSsb4pk4GVMquEUJOadxiiuCHDSHUun0M2k1WdegVRRQRDPyNQPFQD8wiQgZa+sfCE\nkTzOOpKwgrjpZnh1rZGJD9knw7Ix46oIexQmfsmBaN2R96OP3G/aOk7luZH7zi3EPIh+iwFJntKM\nDk5U1Cp5iD306iwLzdkM6jkxI3R2YK2hcxgnkYEYNmbYfJIKdOdZmpK697RsRb7gVoiUlTIrcgOi\nfeYRahaiPUhSJFaFVOd6etTcdioigoLDhHeoqrzBKx+q5aK4QTKQFpMvym1nZzJw0y3y/mOSPO46\nYlpB7EwGhmFEI4cU4SPPs7LVJ674JSnJQl1X+bwWSFCdXlfHiWiYkQzHzeamHYQWBuQdyufd0IOJ\nB/pQqLWqNzn0TKByoKbtFPfA1NGjkkGFEKgQEl4pFh1Vb50xw+YT1zrEK5eT9L0na2AzAWXrqrfF\nhJqFrAQQDREJCltvGIk1DKCQLyAT4rarOixVrA6ymBbZQcux3AhB2popLhkQki/KbRcWdKyydiUk\nSQ7K8/d9X0kDS3Q9uVY1YF+kZ8XrUyWOTSWmLQ5oK2RkiHScnE5+e3bORW/zBasPIhIGJnnKOdoW\nCfoAkb1pJ+nKAkpWsk2d+UpsTJF9CQ9561cdL+yApEmbqlAjmXO9bccmG2H6TbUAL/staQFJAl4Q\nv8ziFEaoRdB1daquV0RiyXPluwW55YkhOtz0/qo2YqVtWRkQnfgb0paIPpIU/CoyIHH3hVoRIqy9\nLy0LVsqUFq6V7adKhqAo0J2rZyVDhJI1sWOxmIy8OC4zloS7dqNW1mBfq/YPIh4GJHkC9A5c9gAB\nwuuy8aBrsSCH2ZDGDAoFF91OdQaW7K1fdbywA1LHykFAx/MQs3+tysfUAllO9puO/lcchPVZLAZV\nzFlCHda3jqtTV4SUBv1cwS8iLSNPDNHhpvcH2hSkStsEVZ/pkJBy25Lgaa/oo9+9US4XwCBUDVp7\n7X1kWVDMMoxVu00S6M7VsxKMF6lkTRKElCodQ9yziWQVCiypKrFvgxgYMPzIynebD+vXd6JY9IGM\nuiWk3raQtlNwvSJcr1iJMaLLi7TUlR7qe66biraefFUQuOx7GVR0etg2uuNly3XPiAo1gWkCzdne\nftpzea3DOmulYKVMWCmzqu++QBSNI3bv6DWTfcoXXPT08VpYmCYqhI4F73kggp9R5i/bR9l3ZLyF\njz+O5jQwcZ/PhAxkVmKLSrEfHIuHShtBvwCC1zmd0gOHtC04LtK2Vfm7MibVb2xorL2voLLHWvdB\n9Xp6ncyaq9obJozGYZWfV79zPUhB5opcgmDfeH1F2V/tPVAYh+4TAPffYWO1uQ0wTQPDhjWqLSRB\nfOb0m/HhmngZpjS2GdmC1+78bmL9bW4MWMsTACCvFkRL3CRe0a9kfbXUZQKp82Fv6GFikDKoHP5s\nm0wqVTF7q1hIRG4X2srhuEWkNVNhc66HroK+m1QVsrIpOpY+mfuKECfakpKpsRVNpe+0yV8fu276\n/1HnLyNH0iy78nPl9bTBtm2FgTiFgwVv16puCtbNQa4DIHXZ0OrSdm4j/O6NgaDcxNPvVdYVlzhp\nWkVU9lh6r6KOUSG61S5MXpkSYl1yvSJcu7G6lqHAXUr3Fed+6rjMVMep9Ol0BqxNltM56J7bAjCw\nyVMZhiE+BGj3DQmcdL0iN41cVbkaiK/hE7qeCBmFogOyxy2Rva9dWyKOdQpuQLZf3ZIrKggrmxKl\nNiGJH9LN/EsCOqRP5GqtY9bNxm3lE5y/ajyU7wOu48KyNBXGqew24UEjOBTpf3PdPHZTIBOu6tCv\nimVpgVHfSrlL3drGINXAuhRFYVt5Lir3iu2XuZ6FVJ2daU8Cq0+7Zl5onBPpO6AYH9eFp2qxUh2H\n3ntBPF9/R/faj9C5ZmVif7rXfrS5l5QoBjx5qhxUgmKw5PO0nUK3U7KguF4xoDhNQ+VAihLgHUUz\nSZZRGFWDacmKdVKrhcrBn0S2Gu/gZj9TIQrkmlOunouUaQZiu5IiRzpK8Sz5kRESXnB3piy2ScdD\n0e16EqxTqBUPBR/ahYEjHGhVBzjv4DHMyqFMBCyrDiNBHFLaTuHt/2wsu+4U08n7Q9q4ccj/twAA\nHt9JREFU5l6GCXJW/Tus/4jij8rq7KpxTqK5MvF0uiV4lEmpQuwarz8iRzBobdpy0A9+K0SHYRiV\numWOW6wcOAQ8943v8xWnVSFzp8msX0loJpXWzC8SrNJfvuBiwrjhUkIWdvAnVb+Nd3Czn6kQBd8H\n8gUXs66dCtsypRlsUTLtdK1ImXLcmQohEQV3Txg3HMve31AO1q+u1ZekxUxVS8uIHIyrESTNORS5\nhV2Zg1J0INGuGDooeJdtW4Pp5JIDrd+UzmD3UgYJERJKAEjulZb4IxMgr6POTt+vMNcnb66l+5zX\ny5qLoPAtnZssIF5HALM/EPZBSDGg75Dv+xULjW2Z5ayaoCRBEu4b1pLFc6fxtJFkn6utL/h/npWt\nToOYhZERQkRE+yLKYIsKnnClLlHIWqkKYQnLYIty36NYkXTEPXnZcvmCi/E7DK1SQ1clcroIs2xm\nrRSMbBNcq167b62CvhyCQLLlxBpLYbob5GBtow7zIBETQjOzj/vvBKEa6yUrRyOTAOBaR8rXzLhq\nLgyrFPP252smc8mwKDYtcmkUyb3h9h3FdRdV4Vs0N6GlNCGr4SD6DQY0eQKow7bg4vbLjgzUvGPF\nEbnXhRxwrMiiyJ3GaiM1Z4PXsJ/rgj7IactXRtOSEpaxR4gI1+JFuUDbc+K903UpstY0HaJAk5se\n10V7Ll+RhGDnQVxqecWsQR2yxbMiRQWP5KoQuaTEP3maaDf95RWkbDvg5lGGij5QGayuUMEp/VwX\nHFeeBq6A3r4VM4gUrT1ago4yKK5DZe2iIG6Za4xrHSlfM3vm1Iq13kqZ1eKPIgsOqxSe0B5U+mb+\nH7n+XYIK37KAeBUXMb2P2q7yQfQZBjx5AsqHXPnAIcSCJlEiaw+bCs6C56IKU8uma53R1/A+114j\nVWSXzCGpAGjR4Uz+JrFhf75mciCDjb4e0D/ASXu6Fpyqa5BHbmhJAjZjjcQT6WSqxbEixQGvL9cr\n4u5rp8D1qn/5JiX+KdJEu/Crn4HveXqBxdKBJASIIkm0PlMimkmagbqh1h4NQUcZlPa0KrYnxH0X\nUgKGzgaT3Q/STrrnAguOjjp4Es9VZKHJpBW+2YB4jRJCAVX3gack9F+DLYI8EfBIlIq1R5T1JXJR\nyZ5nNsCXtCUSAYR8hBE3Hnjq1cTNk01bsRXAWZcdLwjaSplV2X9RMuSA4EFN14IT7WHYnhCw5Iv+\n/4Rxw5HXJJqytn31Yuj7JfKUMg24XjFUSiNqkgGvn5zrYeOajzC8wYqf1VQZiO/eCG1TRqJqzApr\nkBE9njWHpNyrjh9WhoYmFsruO9G66HgmooJtN4aSo7A951lwlNXBk3qumPVxoWOtSwokm09hXYPB\n5QMDWxR5IiC/8FWsPWFZX2EuKh54sTwi8hFFtZydf5jrTjUwmnbZqQZBR8mQo9ciqgVH7p+uxUdE\netlA9CRQ6/qILFSlNHjf66jV8/ppa2tDS3NTPKsPgzCNHm6wOI0E0r2VrB0haw5Yc5iUe924G34Z\nmhYusYhdtoQhLEr6Q4zCdxUoC07BccUlWiTK5LVUYI8knZHQmE62Vcmyph1cPojNgi2SPBGoWDDo\ng5VYJNiMOh1Fbrpf3hj0PJJwtYRZHKJqDqkGQYsy5FS1qUS14AgJ0iEmPNILlDS5ACQq9FmrGnlh\nUCGkLFTnKntWbNtGwXEAJPyGLgqoZXSaIlsiQoK8Va0dvDIxVeso/60T3yLrn16/at0/AIECyNJ1\nxdAfCq2HV7ZAkeSaqhgpztg1r+2mYOHT6UtnzFL2rCCmSqJrNhjz1H8xsMuzEISUaVHN2iLlTYj8\nweyZemVYwsCbh6isShJ980q8APx2DWkbhmHA9310FZwAwVNZP90u7prilMKhx857XuR+ZHOLu86k\n5QZUEDZXds8BBNrOueePOPKwQzByqxE1mR+vPEZNyoZEaBPpmhDRxUhzS1mAx1Q3EJQ+qSpDI4OG\nQCQhu9xyKhHWqFVOJaytZsmUqM9Z1PZLVqyjsj3blObT1e1ttvIsOx99CT5YvT6x/rYbPQzvPHpj\nYv1tbmwZlqe8/OAiGV0i8FxPMoHKqIiT9Relb9YqlEnxLQuyeBrV9felNU0G1pKVpKI4a5mJcu+S\ncvXp7mvYXNmEBPb+NTTUo6u7O9acZeAWBY5TxkLRqqQlpaDRbxISCAGdqmwrjPpWONlW8ThUv2nb\nUo+bUSAvqppQovnHGbtq/LDvNS2IWrFWEWQHSmTWq9a8EvTFznHQ+tQ/sWWQpxCEHVhRxBl1EPbs\n19IKkaPcaDJSkySJk8VH6c47ynzoMZNal4gU6ty7pFx9UQlY2FzZZAv6/jU2NKCzsyvahEUQub7K\n4pgkkDkSdNxngjge7oGdUMyX1tzMVDDrUCTkyM6tmMzvL94hr5M9Vovxed8TLSqVGDb631r3U6Kh\nJZt32k5Vk1lZ5mS532xdGk3NWfmcBrFZsMWTJ9UDK0mdHhqig46VAqgleOrdIkuVCDqB6Lxg7ygH\nvuo96Ks9jGvFYvuIgqRjrdjr6YzVgM5U81bo6GTiVmJAZEkgnyeReRVIxdedk0LqvrK1ijMGAHWr\nWtFDwSFacW41KdKZWxSICENf1WcLITiEYN919eSKFlU0/S+1PWPbC61iCmSWmzlJaallsxa+84tF\nSvMaRN9iiydPOode0hYg0UFHiES9bfWrbC0Rwgr48trTwd61DK4OK1WTZEZcElasuMWldZ5nei90\nRTXZfhsaG9HVlZDlSURMqM+FWVpUHyqopOIr1mJTzv7SEP6UjRE2JwI7txF+1wbY+aDAp8g6lghq\nSco41iPhd7LxqT21UiaU6xWyY+numaJgq+q+BYLEKfdoLufilu8fpje3QfQJtjjyxB4YJFBWVVU6\nyTnwDjpCJGZcVTLlnvOzBf0mW4vINLCfhRXwrdrzkMw91QB0lTayUjW1IG2JWSQjluwB1EhcQHuL\nQ5J09yaTyaAnl4ue9UZDYskgn/uuIzx0dIreKluwksj+ChtPw0XElW6wGoKfJamNpDB+UqSM7VtZ\noZ03PrOnVfUKFayAsSQByir4FaugQK8srA9aH4y+n7meAjrac9HnN4iaYYsiT7wDgyhX66hKJzUH\nQFz89a6rJwMAfnPx4YkHpkdB1kqhIW1XHbI84iMjQ6LvVA78KCrltOo2q3fFE/7sLyDzdtxiJEmG\nMBJMEyNRrJZIPoOHQqEAq15Np0YFImLClmepXpwJw7LhFf1SrTVZjJJmPIuopIkywtxLYbpV1Brl\n0g2ZSmZZTbSRakXKeOtgZBXild0J1ivUsgLGkMJI21a5rqoFN90S6WeErMF3nUrBa7KOAZgQ/1+B\nLYY8iQ4MWrm61s+galBx3vMCdaJUNZFqBTJvwzCqLEwi0cSwMjUiTSgRoqiUkyxB3/dhpcwqYUxZ\nrb7NiUwqBStlwvd92JapZRlTIZZVGl0CAsneJ2nfjoOGpsZkDtQwV0kYyUmZeOO99bBSJly7qdKn\nSuZSKGISkMB4QiuYXLeKxPDQBXhdu5HryuzTGKcYoBMA2DggWb09IcKsOwlaAaUo93P7ZUfCyedh\nZ9LRf0bK1k7Sz6BIZv/GFkOe/JwnPDCSzJyTzkHRPZV0Cn1ckPn4vl8la9CQtoXFlZOKH9NRKWdd\nizQRpdXgN5eIZRiq1OY5bl3RdYZRIg5esUQWZWuiiZGM6KpITBgG4HkOVm8oVFwLURHbVVI+rCaM\nGx4kIrIsqL4KbKbmGNkKxsTw0HXnZIKTSSPp8jcyJXNuvT1ObBOB6jMkqwUpzBiM8FIQkJSIS8gG\nRTIHDLYMkUwKRjYVOBBEisthq6bb6IoaRhGW7A+gf06JYOLd105ByjQSFQvlgRVxlAmKul4RrlcM\nWEx4ApBJCZAmDdFaw9aRL7gV16Su6KdIRJVVdRft1wtP/gtbjRqFPfeaAN91ImsvqYgrqqBUsiRd\nLVRIkaW4IpuRIVungtCjazcmLhjKG6cvSWWc+dPXAgbsTLoiOCkT6DQsuxxEHhxTNJdE9jhqADoz\n90GRzP6PLcbyBKCskC0PNlZxe4QF24ahPxEiHZB4Jtoy5vu+UkxMXITFRNGWEcMwAoHWomvzXu/n\nYfPWXZdO+zBNLTqRgLX8qNYYlEE1W1Jqocp3wHK6kOrZGP1gSbQuXhvfQkH6VC1RUgvI1qkQJA5U\nyxgkaQlKJFCaoNb7ylhiiFuMCE6K4uKEljqFTM+oz0tsbbIy2Hudydqx+htEbbDFkKdsXRotQ+oq\n6f88GQDTDHflqATb6iBKHbG4SILgkIO0q+CoxcTEABsbxhuHJXQFxwsEWotq+mVS4QRYd1067UVt\n2fmKXLmqNQZF0M2WFLqai0UY8PnZRBqIRAIEAoRSuYCGoZUsqFoWmhUhdqYeDzqp96rjiAQ3FaBa\nfDgWMWGIaO+/C2L9rvI1s2fyyauI2GoTe9XMzig/L+XxDcNAJmPpXz+ImmOLIE+GYSCbJRkPqUr8\nCy0DQLSH2KBiHmhfeZxsLZlApgopi0KC2HR9GVSUz5Msu8IDT6dJNA4hDt2OU7nPvLnoEGCddYUR\nD157nT3LuR7y5dqKMouQznOomy0pXItpwvM4sTxRoEFkuCn7illUWiVKqOsTAyeIWdZWK1ia05fS\nvaHGKTgujIah0e6lICOQOx82SFyTyPJiogBfulYZeRWJXALVFj8RVGPa4v68+L6PfD6aoO4gaost\ngjz5vo9czsXtlx1ZIUcFx8Ptlx1ZUXLOpi0sWbEOaTsl1Xxi67zRpVpUCYPK2z1dRyyT4gtN6lp5\nSHbZ6yvWhR7Wuv3XIsg9iiaU7wPFYngblWwznXXRliyV9lH0rQwDVXIL9DzZtqrQzZbkjZPJpFFw\n3b51hamm7NPgqTqrCmom6c7S7Vux5Anpx6trDfalYd2xCu3wuzZUSr5EzQyj97lKjDQkSFwbjMUz\ndn1BkcilCjh7zZWh0LS4tbkNVX8AIJ9z1OY1iD7FFkGeACDXU0Dbph50d+YDf+e6ChW9H5KhE6b5\nxHvTVyUbqocskSc47Zp5SkKTYdYp+pq9xg0HAC4pE/WvgqTqxBHE0YQKa6OababSV1U2oCdvTyvI\n0wKtKpY+HSKnG4fHjh9G5uhxrKYRyMGurStMIZ1clLJPgyYhNRHU1F1TSN9s0V1ZX6QfwzCCfenG\nkzESAVHuJV3+RkmDSlBDUGWvlQsTxySBSvvAqUnHlaGQZYAy8yREaRADB7F/Q8ycORNTp07FtGnT\nMGPGDCxdulTY9re//S2++MUvYtKkSbj11lvjDl0FkjjI/o28hx7X07IC0N/ruNlUD1kdoUkiG8Ae\nluRwqytLCdDWLNlcN4dUgmjPomhC8dqI1qnaX5jsAs/1xQN5BmgXcoajfi4Ct64ch1izyvRh5FqX\ncLHP8lYjhmHm756I5gpTgIjksLpJ0pR9GmVLhYi0xBXU1FqTzG0VUQXd9/2qvpTirOKWXOHpKwn2\nTiQFoKwoTl2jUpi4lpZDFqyYq+j+8lyEXl1JbDZvtwYsTIPQRy6Xw0UXXYRJkybh6KOPxqJFi6Tt\nC4UCjj76aHzlK18J9HHxxRfjuOOOw3HHHYfvfe976O7uDh07Nnk69NBD8cgjj+DBBx/E2WefjYsu\nuojb7qWXXsL8+fMxZ84cPPzww3jsscfw0ksvxR1eHfleC4SuKGUc/SbZoawiNJn3PGkW1pIV65Ap\nx8iQa8ibuWyu7NgqJUKiBoyHXSeSk+CB93lY8WVZH6pWtzBrE+mLPAO0CzlHSQyoxEmx8gH02nwf\nFZd0wfGkVlGaXOtaGtlnuXXYCNx06fHBAqeC2BttKLjiyN/CQGDuIuhDrVexWSqomRQxVHVb6aqg\nl/sRZj1qlnyJG39WmRdLZjgp+7yiz6LMN3ZNoYWJo1oOYwazkz6kRJR57ipixFlrUMcpJu644w40\nNjZi/vz5uO2223DFFVegp6dH2P6Xv/wl9t1338Bn9957L1zXxcMPP4yHH34YruvinnvuCR07dhj/\noYceWvn33nvvjTVr1nDbPfroo5g2bRrS6TQAYNq0aZg7dy7222+/uFNQR95Dpi6NbNZCLuci11NQ\nvjSX95D//+2dfUxT1xvHv22h5UVBahwUWNjilrjBEkY2N2Uy5aUtykvLMqkzMUMyXyZjMSYbMbgI\nugxYfjMKW7IYFpcfxMiWDW2wbobAzwmLsJH9wWQJZMEXis4oL25SoPT8/gCulL7d8tL2ds8nuWnv\nPc+55/me59xzz729PffRpMup8p3ZicViWCzWndV8qzVrVuLevYdWaSbRzEnMZAYzTXFp4+LHP0We\nPZbJlWsCP19nU0NWyCANlGBicgqP/h63aysSiRAUIrMpyxXW+bbyyhfkIEbzt892PPb84rMPALza\ngtW+TNZ209NjzJYXCJNpEqZ/Jjg/xmbSYYFNDF3ptlfnACCWiTBlYRCLRRBPMgSFBNjon5/XNG7b\nhlwxty0HioPw4ouJM20zlPPVPGWBedLi1rFkV7vpsX+myWCH9RMQMH1ym2IS/MPnit08hWCYIZNJ\nYWIRnJ/2ylsZFgyJROR237AYTbM+isbGZo4Lx5octen5zLbJ+dvCZUHYceTizBxhK3gdv9b5H7cn\nZ/nt+Tk3/3+Pqrh6AWBVRw41uqijNWtW4uHf4/zqe76/fOPEU6sz24CZCTu5Y1Go89r4CAaDAZWV\nlQCAuLg4JCQk4MqVK1CpVDa2v/zyC27cuIGCggL88ccf3HaRSASTyYTJyZk+e2wMUVFRLste0mee\n6urqsHnzZrtpRqMR0dHR3LpCocDg4KDDfY2OjuL27dtWy6y9WCxa0CKRiBEUFIDi/7QiKCgAEonY\nrfwiEb+y7dmFrpQhLDwIoStlTvPa0zcxPomHoyZMjE9abR83TXL/BhwfN1uVy9fXgAAJpIESFP+n\nFdJACQICJA41jY/bL8tVXczms0xZXOZzFKP520NCp6emCAoOtPGL7z74tAVnbSY4ZNqH4FDpjE0L\nZzOrc/bTUQydleOozgMkYhw88T9ulnF7NvO3T5icl8+nLc+2zbm+imf+6eruscS3jc+vH7FI5Nbx\nK5GIIZPZxm9+eRKJGBKJeMF9w0I0Oapnd9vh3IVrkyFSm/1bZu5Iu3P8unv8O/Jzbv4ps4Wrl7l1\n5EqjM58BuFXfC43TQmIy1/bQySsIkIjx98Nxt8oDgMHBQZtz4ujo8k3+GhMpR5xi9ZItMZHyJdfB\nd1wxNjaGTz75BGVlZTYDVp1Oh5CQECQnJ2PTpk0ICwvDtm3bXJbtcobxvLw8G2dmr2zb29u5q/+m\npibU1NSgvr4ecrncZj/79u2DVqvlRoQGgwF6vd7hs0/V1dWoqamx2paUlMTrdhpBEARB+BM7duxA\nV1eX1baioiK89957XvLIfUwmE1JSUjAyMmK13ZEOZ+OPtrY2vPTSS2hubkZERAQAoKysDHFxcXj7\n7bet8pSVleGFF15AXl4erl27hk8//RTffvstAKC1tRV6vR4VFRVgjOHQoUNISkpCQUGBczFsCfjx\nxx9ZRkYGMxqNDm3KysrYV199xa3X1tay8vJyh/YjIyPs1q1bVktnZyfT6XROyxEyRqORbdmyxS/1\n+bM2xkif0CF9wsWftTE2rU+n07HOzk6bc+LIyIi33XMLe+f1xejIyspi3d3d3PrevXvZpUuXbOyy\ns7NZamoqS01NZcnJySwhIYHl5ORweQwGA2fb1NTE9u7d67LsRT/z1NLSgoqKCpw5cwYKhcKhnVqt\nxscff4ydO3fCYrGgsbERH330kUP7sLAwhIXZPpjY1dWFKTcf+BYKU1NTGBgY8Et9/qwNIH1Ch/QJ\nF3/WBkzr6+rqQlRUFGJjY73tzqJwdF5fKCqVCufOnUN5eTn6+/vR3d2Nzz77zMbuwoUL3PeOjg5U\nVVVxd55iY2Nx9epVqNVqWCwW/PTTT3j22Wddlr3oZ54OHz4Ms9mM4uJiaDQaaLVa7pZcaWkpWlpa\nAADr169HRkYGtm3bhuzsbKhUKs8+LE4QBEEQhN9QWFiIkZERKJVK7N+/H8eOHUNISAgA4NSpUzh3\n7pzLfRw4cAAjIyPIyspCbm4uJicnsW/fPpf5Fn3n6eeff3aYdvz4cav1oqIiFBUVLbZIgiAIgiD+\n5QQHB+PkyZN204qLi+1uX79+PXfXCQAiIiJQXV3tdtl+M8M4QRAEQRCEJ5AcPXr0qLedcAeZTIZX\nXnkFMr7vIRIY/qzPn7UBpE/okD7h4s/aAP/XJ0RcTlVAEARBEARBPIZ+tiMIgiAIgnADGjwRBEEQ\nBEG4AQ2eCIIgCIIg3MDnB0/l5eXIzMyERqPBW2+9he7uboe2n3/+OTIyMqBUKh2+9sXXuHDhAnJy\nchAfH4/6+nqHdh0dHUhMTIRWq4VGo0F+fr4HvVwYfLUBQENDA5RKJZRKpc0UF76KyWTCwYMHoVQq\nsXXrVrS2ttq1E1Ls+vv7odPpoFarodPpcPPmTRsbi8WCsrIyZGRkQKVS4ZtvvvGCpwuDj76amhps\n3LgRWq0WWq0Wx44d84Kn7lNZWYm0tDSsW7cOfX19dm2EHDs++oQau+HhYezZsweZmZnIzc1FcXEx\nhoaGbOz49jmEB1jQnOgepLW1lZnNZsYYYy0tLSw9Pd2uXWdnJ8vJyWHj4+PMZDKx7Oxs1tnZ6UlX\nF0Rvby/r6+tjH374Iaurq3Nod+3aNfbGG2940LPFw1fbrVu3WEpKChsaGmKMMbZ7927W2NjoKTcX\nTE1NDSstLWWMMdbf38+Sk5PZo0ePbOyEFLtdu3YxvV7PGGPs/PnzbNeuXTY233//PSssLGSMMXb/\n/n2WkpLCBgYGPOrnQuGjr7q6mlVWVnratUXz66+/sjt37rDU1FTW29tr10bIseOjT6ixGx4eZh0d\nHdx6ZWUlO3z4sI0d3z6HWH58/s7T66+/DolEAgBITEzE3bt37dpdvHgRGo0GUqkUMpkMGo0GBoPB\nk64uiGeeeQZr167lXrDsDCawP0by1fbDDz8gIyMDq1atAgBs375dELEzGAzQ6XQAgLi4OCQkJODK\nlSt2bYUQuwcPHqCnp4d7o3hWVhauX79ucwVsMBiwfft2AIBcLkd6ejouXbrkcX/dha8+QBjxmk9S\nUhIiIyOd+i7U2AH89AHCjF14eDhefvllbj0xMdHmhbiAe30Osbz4/OBpLnV1ddi8ebPdNKPRiOjo\naG5doVDYbXxC5saNG8jLy0N+fj4aGxu97c6SMTg4KMjYudPmhBC7wcFBREZGcoNdsViMJ554Anfu\n3LGyE+qxxlcfMH2Sys3NRWFhIX777TdPu7psCDV27iD02DHGcPbsWaSlpdmk/RviJxQW/XqWxZKX\nl2cTfMYYRCIR2tvbuY6uqakJTU1NLp+d8TX46nNFfHw8WltbsWLFCty+fRsFBQWIjIzEhg0blsNt\nXiyVNl/Fmb62tjbe+/HF2BGO2bFjB/bv3w+JRIL29na8++67MBgMCA8P97ZrhAv8IXbl5eUIDQ3F\nzp07bdKE3qf6E14fPH333XcubS5fvoyTJ0/i66+/hlwut2sTHR0No9HIrQ8ODkKhUCyZnwuFjz4+\nhIaGct9jY2ORnp6Orq4ur56Al0qbQqHAwMAAty6U2MXExMBoNCIiIgLAtN+vvvqqjZ0vxs4eCoUC\nd+/e5QaIFosFf/31F6KioqzsZo+1hIQEANO6Y2JivOGyW/DVt3r1au77xo0bERUVhd7eXr94kblQ\nY8cXoceusrISN2/exJdffmk3fTZ+rvocYvnx+Z/tWlpaUFFRgdraWqcnVLVajcbGRkxMTMBkMqGx\nsRGZmZke9HR5uXfvHvd9eHgYV69exXPPPedFj5YOpVKJ5uZmDA0NwWKxoKGhAWq12ttuuUSlUnFv\n7e7v70d3dzc2bdpkYyeU2Mnlcqxbtw56vR4AoNfr8fzzz3Md9SxqtRoNDQ1gjOHBgwdobm6GUqn0\nhstuwVff3Ocqe3p6YDQa8fTTT3vU1+VCqLHji5Bjd+LECVy/fh1ffPEFAgLs39fg2+cQy4/Pv55l\nw4YNkEqlkMvl3BXjmTNnEB4ejtLSUqSlpWHLli0Apv+mev78eQCARqPBgQMHvOk6L5qamlBVVYXR\n0VFIpVIEBwejtrYWa9euxalTpxAZGYn8/HzU19fj7NmzCAwMhNlshlarxe7du73tvlP4agOmpyo4\nffo0RCIRXnvtNRw5csTnb1GPjY2hpKQEPT09kEgk+OCDD7i2KNTY/fnnnygpKcHo6CjCw8NRVVWF\nuLg47NmzB++//z7i4+NhsVhQXl6OtrY2iEQivPPOO3jzzTe97Tov+OgrKSnB77//DrFYDKlUiuLi\nYkGcoI4fP47Lly/j/v37WLVqFSIiIqDX6/0mdnz0CTV2fX19yM7OxlNPPcW9v+7JJ59EdXU1NBoN\nTp8+jTVr1jjtcwjP4vODJ4IgCIIgCF/C53+2IwiCIAiC8CVo8EQQBEEQBOEGNHgiCIIgCIJwAxo8\nEQRBEARBuAENngiCIAiCINyABk8EQRAEQRBuQIMngiAIgiAIN/g/hWLfhf9T7/kAAAAASUVORK5C\nYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "zi = griddata(x, y, z, xi, yi, interp='linear')\n", - "plot_contour(xi, yi, zi)\n", - "plt.scatter(df.x, df.y, marker='.')\n", - "plt.title('Contour on training data')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hoANr0f2GFrM" - }, - "outputs": [], - "source": [ - "fc = [tf.feature_column.numeric_column('x'),\n", - " tf.feature_column.numeric_column('y')]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xVRWyoY3ayTK" - }, - "outputs": [], - "source": [ - "def predict(est):\n", - " \"\"\"Predictions from a given estimator.\"\"\"\n", - " predict_input_fn = lambda: tf.data.Dataset.from_tensors(dict(df_predict))\n", - " preds = np.array([p['predictions'][0] for p in est.predict(predict_input_fn)])\n", - " return preds.reshape(predict_shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uyPu5618GU7K" - }, - "source": [ - "First let's try to fit a linear model to the data." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zUIV2IVgGVSk" - }, - "outputs": [], - "source": [ - "train_input_fn = make_input_fn(df, df.z)\n", - "est = tf.estimator.LinearRegressor(fc)\n", - "est.train(train_input_fn, max_steps=500);" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 502 - }, - "colab_type": "code", - "id": "_u4WAcCqfbco", - "outputId": "c67fa79f-319a-4c80-e2d2-dacf439133c6" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk8AAAHlCAYAAADsqBh7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXt0lOW1/7/JJCE3EhIkFwJa0bYU0VK0rYIevJAQISQk\nWIwNp7S19SwV7LJ/eFjn6A+K2gs9S5cWbUuXp6VL6hGtlaRtEArFC1ip9crFCiigzOR+mYTc8zy/\nPwZp0wQyM5nMN++792etrkUnr/nOzjvzvvvdz372N8Zaa6EoiqIoiqIERSz7DSiKoiiKojgJTZ4U\nRVEURVFCQJMnRVEURVGUENDkSVEURVEUJQQclTz5/X785Cc/gd/vZ7+VUcHN8bk5NkDjczoan3Nx\nc2yA++NzKo5LnjZs2ODaD5Gb43NzbIDG53Q0Pufi5tgA98fnVCKSPLW0tOC2227DjTfeiJKSEtx1\n111obm4edFxXVxfuvvtuFBQUYOHChdi9e3ck5BVFURRFcRnHjh1DeXk5CgsLUV5ejhMnTgw6Zs+e\nPVi6dCkuvfRSrF+/ftDP//jHP2Lx4sVYvHgxiouL0dTUFJH3FheJXxITE4Nvf/vb+OIXvwgAWL9+\nPf7nf/4HDz744IDjnnjiCaSmpmL79u04fvw4KioqsGPHDiQlJUXibSiKoiiK4hLWrFmD5cuXo6io\nCJWVlbjvvvuwadOmAcecf/75ePDBB/HCCy+gu7t7wM/effddPP744/j1r3+NzMxMtLe3IyEhISLv\nLSKVp/T09DOJEwDMmjULPp9v0HHV1dUoLy8HAFxwwQWYOXMmXnrppUi8BUVRFEVRXEJTUxMOHTqE\nRYsWAQCKiopw8ODBQataU6dOxfTp0+HxeAb9jk2bNuGb3/wmMjMzAQCpqakRS54iUnn6Z6y1eOqp\npzB//vxBP/N6vZg8efKZ/5+bmztkkgUE1nn/dY23pqYGs2fPHvKP5AY8Hg/y8vJcGZ+bYwM0Pqej\n8TkXN8cGBOKbPXs2ampqBv0sLS0NaWlphHcVHkPd14Gh4/D5fMjOzkZMTAwAIDY2FllZWaipqUFG\nRkZQekePHsWUKVOwfPlydHR0ID8/H7fffvvIA8EoJE/r1q1DSkoKKioqBv3skz9CMGzatAkbNmwY\n8Nrs2bPx1FNPjfg9jlVyc3Oxa9cu9tsYFdwcG6DxOR2Nz7m4OTYgEN9TTz2FW265BW+88caAn61c\nuRKrVq2KuKbt6UJMQmLEf29CQgLKysrQ2to64PXRiqOvrw/vv/8+fvWrX6G7uxvf+ta3MHnyZJSU\nlIz4d0c0efrRj36EEydO4Oc///mQP588eTK8Xu+ZrNHn8+HKK68c8tgVK1agtLR0wGufPFls+3sd\nOnr7g35fh/+yC7PnFQR9fKQ48voezPjCFRiXlBx17bde/hP+bX5h1HX3vbgDc6+9AbHRfgq0Fnt3\nbcf1C26Mri6Al3ZUo6BwUdR1u7q68MZf9uD6Iaq8o82vf/kEvvn1FVHXbWpqxsu7d2LJ4uj/vf/3\nl7/CrRXLoq7rq6nF22+/jYXXzo269hO/eQbfuin6f+ujH3nx4Qcf4vorLo269i+e/QO+eeM1Udc9\neMwLf3MjvvyZCwa8HpOYgnFzl+Khhx5Cf//A+95oVZ1iEhLRvec52K72yP3OxFQkzi3D1q1bg4oj\nNzcXtbW1sNYiJiYGxhjU1dUhJycnaM28vDwsWLAAcXFxiIuLww033IB33313bCVPDz/8MA4ePIiN\nGzciLm7oX7tgwQI8/fTTWLduHY4dO4b9+/fjoYceGvLYc5UjO3r7caon+OTpyOHD+NycG4I+PlJ8\n+OEH+NSMzwPxJuraRw6/jyuvIySMhw/jS9dcB09MdKdg9Pf348iRI5iXH32f6/ffP4z5C6Kv29Pb\ngw+PfQiGt3drSwtA0O3r7cGp9nbABP/9jxStLS1Af1/UdXu6utDR3gb090Zdu7W5CejrHv7ACNPT\n0Y6u9lagpzPq2q2NDUDXqajrdrW1oLO1GbYzc8if5+bmRvX92K522M62iP/eYOPIzMzE9OnTUVVV\nheLiYlRVVWHGjBnnXLL712thUVERXnrpJZSUlKC3txevvvoqCgsjU1SIyB3uyJEj2LhxI+rq6nDz\nzTdjyZIlZ0pwS5YsQX19PQDg1ltvRWtrKwoKCnD77bfj/vvvR3Ly6FdlQlkujCTWWMTEckZpxYAU\nsyXFfPrphAHt82V52ryYmeeZIgtjDWLFxWwRG+UHsE+gxWwsYmNJ4mOUtWvX4sknn0RhYSF+85vf\nYN26dQCA2267DQcOHAAA/O1vf8O8efPwq1/9Clu2bMG1116LPXv2AAAWLVqEzMxMLFy4EGVlZfjM\nZz6Dr3zlKxF5bxGpPF188cU4dOjQkD97/vnnz/w7KSkJjzzySCQkQ4LxZA4AFpaXxIATsyHd3Jg3\nVdrnS2TMoN3dSCFTk2RezJaWxNBiJt4vxirTpk3Dli1bBr2+cePGM/++/PLL8eKLLw7538fExGD1\n6tVYvXp1xN+boyaMhwuv8mQQQ3qSYH4JGX9vw6p4gfz5oj2dc2IWWYUxEmO24mK2xkILT85BRPJE\ng1gZEAexzM/CWotYUsLIgllt4yEvZqsxK2McEVfeqxYUU3Tn5i9CXHxkBnKFSv7i0uEPGgUKizm6\nnrg4LChaTNEuKV1K0U1JTUXBggUUbdayHfOBhLlsxyokc5ft9DwrY5eIz3kai0zMju4uhX/oBr+l\nMtJk5U4e/qBRIGdyHkXX4/EgO4dznifncWKOj49HWnY2RZu2bGcsPB5pjcQGnljOAEjqEhar5YEU\nc39HGzzCKslORs+Uoighw6o8GWvAejxnVSSMwOZpY4mbbQSeZyV0NHlSFCVkmLv8WH1evARGasyy\nKk/WgtYkr4SOJk+KooQMrfJkjLwqjDG0VhitPEUP1pgXJTw0eVIUJWRoF3mBQzIlxsyc88SMWXEO\nIpKnvdu2UnR3/+F3FN2+3l7sqq6iaFc//1uK7qn2duzavo2i/fxvn6HoNjU24s8kQ1TmkExxu7AA\ngYNBrbiYAeJDiRIyIpKnproaim5DLUe3v78PjXW1FO3aGh9Ft6+3F42nbYCijc/HibmrqwuNjY0U\nbe6QTFn9P8YYgf0/8oZkGmLMSuiISJ6kIXGQIM1Tj4iF0CGZwsYwW4EDYPtPtYnzeWMuVSqh4/pv\npMh1ZOLSBgtjeY3ELAL2LLKCNhJjFnhTZTaMs7CWa6ulhIb7kyeRF1sjrwpj5MVsjMAqjJFXhbGG\nt1TJgjkkk4UhjmdQQsf130iRfkHWint+kfjUJnZ5VlbI1CZ5FhbyvEokfp+djOvtWWJjPbiyoIii\nPW9hCUV3XGISrplfSNFmedulTZiAa667nqLN8rbLzsnBpIw0ijYLC4G9bSL7vCCuCsNskldCx/XJ\nU0xMDDKzOJ5nk3I4/nKeuDhkknz1WN52CQkJSE/OomizvO0SExORlJpE0eYOyZS1hT3QzycwZpFD\nMjnaSujIeoRTFCUiUO1ZhG1hDyxJc6DGLG1IJqQtVDobTZ4URQkZ2pBMw+sLYQ6M1JijqU2RpQ4G\nVUJHkydFUUKGOiRTmEkuuk+Ji1lmhVF7npyEJk+KojgGiTuSLATutpPaJC/sPDsZ1ydPPV2deH33\ndoo2y9vO39KMV1/keJ6xvO3qa2vw2t5XKNosb7sTx4/jr/v2UbR53na8BiBaI7HIJnnekExtGFeC\nwfXJU39fH1obOZ5nLG+7vt4etDQ1ULRZ3nZdXZ1oaWqiaLO87TpOtaO1tZWiTVu2Iw7JpC3nEAdG\n8vz8BMYssKrqZFyfPFkwm/DkOc+zHttsYG2DpM2MmSJNrTxJq8JYyKvCSIxZoq2Wk3F/8kTcncO6\ns1lrEMOyc2Dd2AyzkZjYPE2rwvAeSKRVJCzRtoP5DCYuZmjDuJNw/ZBMUEuh8p7OeZUnZkVCY44W\nIvt/dDxDlLUpsoHzzJEekpiUNMR4PJH7hYkpkftdYwD3V56YVRhW5ckY3k4V2gWPd1OlXeQFVtvQ\n20n7PjMrErzPNkWWmjwxh2RKs6RxMq6vPCWmjMesuddRtFnedhMmTsIVV11D0WZ52+XkTUFuFsee\nheVtd+FFFyEehqLNqzxZxMUJrEhIrMLIKp7DMJsYlZBxffIUFx+P9PGTKNosb7uEceOQns4pkbK8\n7ZKSkpE0PpWizfK2S00djyRSIsHbbWfk9cIYZm8bRRbGCt1VqT1PjsH1y3aKorgHS5z/w4K4kZSG\nFTjzKLA8y34XSrBo8qQoimNg9v+wkDhVHcRRBSyoI2aUkNHkSVEUxxAYGCnrssUcGMlCZMzWQljI\njkbWVUhRFEfDtGdhwRwYySLQsy0sZoFL0k7G9cmTv6kB+/ftoWizvO3qvB/j7ddfo2izvO2OHT2C\nd996k6LN8rY7dOAADh08SNFmYYjN0yyMMYgRV23jbQxgYQFxZshOxvW77Xp7utHWwvE8a6gl+bx1\ndqKttYWiXevzUnQ7TrWj51QbRdvr5cTcfcqP/jjOV5g2GBTEuVpU2w6SNG1IpsSYx9aQTOXcuP9x\nhtpsyZtsTouZOfFa2pBM6iBBHVUQLbg2PBTZgCWNyJg1fXIKrk+euOvIPNsOaY9tzJ0qTHsWaWbI\nzIchrcJEU1fikEzdbeckZCRPHnn2LLS+EKI9izSTXGPl2bMEJk/LqkgYqg0PRRZGoD0LM2YldNyf\nPBkjbgeDFThVj1ptIyF3kKCsoMXuPBN2niHw++xkXN8wPmFSDrLPy6Ros7ztcqaej+QLplK0Wd52\n0z79WYwjPQqwvO0uu+zziIuk63kIMJcqpS3niIwZoD30UpdnhSXJTsb1yVNCYiLSEjmeZyxvu6Tk\nFKQlck4ty9sudfx4JMVxsieWt92ECRMougC3YdxDShiZ3nYeact2IjcGyBsM6mRcv2ynyICVOEmF\nVnkyvA0gvEZigU3yENgkDx1V4CT0jqMoSsjQbuawtEGC1C3swipP1liB1TZ51kNORs+Uoighw6o8\nGWPEVWECOww52rxqmwFtt7LA3jYldDR5UhQlZJgJjLQt7PZ0+zQDZtImbM4vtUleCR3XJ0/1J0/g\nyH6O5xnL2+7E0fdx+OB+ijbL2+7AO2/j6OH3Kdosb7t9r72G48ePUbS5u+0kViQ05uhpU2S18uQw\nXL/brrvzFPrbOJ5nDbU1FN1TbW0wHs4VoLaG4+fnJ3n5AYDPx4m5qakJEzIyKNq8G5uhWVgw+39E\n7jwTdp6ZMSuh4/rKkyX2C7Cw1ogbGGkEDsk01kgLWeTwxMAuLFkxcz1JeUiM2alELHn60Y9+hBtu\nuAHTp0/HkSNHhjxmw4YNmDNnDkpLS1FaWor7778/UvLnQN5NVeQNRmLJmzhUj7dsJ+8GIzJmiLts\n05YLxzLHjh1DeXk5CgsLUV5ejhMnTgw6Zs+ePVi6dCkuvfRSrF+/fsDPHn/8cRQVFWHJkiVYunQp\nXnnllYi9t4gt2+Xn5+PrX/86vvrVr57zuCVLluCee+6JlOywWGNoXlgskuNied52JKh+fiSMMYgR\n523H83ljERgMKi1medv2pSWLwbBmzRosX74cRUVFqKysxH333YdNmzYNOOb888/Hgw8+iBdeeAHd\n3d0Dfvb5z38et956K8aNG4f33nsP//7v/449e/YgISFhxO8tYp/O2bNnIzs7e9gn0mg/sQaeYGR9\nKq1W20TAbaoljSqwvFEFLIzAkoSxQj1JlTM0NTXh0KFDWLRoEQCgqKgIBw8eRHNz84Djpk6diunT\npw/pPDB37lyMGzcOADB9+nQAGPTfh0vUG8arq6uxd+9enHfeeVi1ahVmzZo15HF+vx9+v3/Aax6P\nB7m5uSHpZZ9/IVLjOF9ClrfdxdNn0J7aWN52n599OeIj8DQRDixvuzlz5yI5OZmizdwJJa0iAeqQ\nTOZ5jr62tbz+suH+1D6fD/39/QNeS0tLQ1pa2ii+q8gTbBw+nw/Z2dlnPoOxsbHIyspCTU0NMsLY\nKPO73/0OU6dORXZ2dvhv/p+IavJ0yy234Pbbb4fH48HevXtxxx13oLq6Gunp6YOO3bRpEzZs2DDg\ntby8POzatSskzcTkVKSQfN5Y3nbj0wb/PaMFy9sufQJn1xnA87abOHEiRRdg2rPwmuS5QzJlVRhZ\nScxYHkhaUVGBkydPDnht5cqVWLVq1Si+q8jDiGPfvn34yU9+gl/+8pcR+51RzSr++WI/Z84c5OTk\n4PDhw7jiiisGHbtixQqUlg6sYrAMQRVFGQjzZi5xSKasBSzeBhALS+udHC7ezZs3D1mxGS08KWlA\nfHzkfmFCEoDg48jNzUVtbe2Z77wxBnV1dcjJyQlJ9s0338R//ud/4qc//SkuuOCC8N//vxDV5Km2\ntvZMyezQoUPwer248MILhzzWieVIRZECrSJBnLatAyOjqcuJeSxXnkJtWRmrBBtHZmYmpk+fjqqq\nKhQXF6OqqgozZsw455Ldv16X3nnnHXz3u9/FI488cqbnKVJELHl64IEHsGPHDjQ2NuLrX/86MjIy\nUFVVhdtuuw3f+c53cMkll+Dhhx/GgQMHEBsbi4SEBPz4xz+mLj0oihIevN12lrbzjGkYKy3mgDYh\neRJY2RzLrF27FqtXr8bjjz+O9PT0M6MI/jmv+Nvf/obvfve7OHXqFKy1qK6uxoMPPoi5c+di3bp1\n6O7uxpo1a84k5OvXr8enP/3pEb+3iCVP9957L+69995Br2/cuPHMv3/4wx9GSi5o0kn9ToqiRB6m\nMTALpkmuOJjJk57jQUybNg1btmwZ9Po/5xWXX345XnzxxSH/+2effXbU3pvrt60c//tBfHyU43nG\n8rY78NbfcJLkecbyttv36l7U+LwUbZa33a6dO9HY2EjRZi3bMW9utK3kxCGZtNPMkQ30l1ENoBWn\n4PrkqaPdj67OUxRtlredv6UFXV2dFG2Wt11zYwN6/mVAWrRgedvV19ejr7eXok1btrNjt6F3tAgM\nBiVt2yfFzKrBGKP+ckpwuD55Ys7tYD0/WWt5Vx/m1maB27lZdzduzBRpYhWGcw2jfqcoqlwrHB2S\n6SxkJE+xrBEHPOd5DytmYkVCrUqih8yYKbKnrUo4O89of2uKaqC/jPG3BrRh3Gm4P3kiDtVjPT8Z\nQ7RnETZU7xNtii4kVtvkVQZo2/aJVji8yhPzOkKRVcLE/ckT8QbDXF+gLVUSkzbeEyMxZtq2atZN\nlRkzRfa0NmPZDry/NUWVHLNWnhyF6/fxT/vcZfDEccJkedtddsWXMC4xkaLN8ra78uprkDqeM1SV\n5W03P78A40mDZLn2LLIqA7wqjLzKE7XappUnR+H65CmF6PPG8rZLz8ik6AI8b7uJ502i6AI8b7tI\nGVyGA3NIJq+3jSJLrMLI221H3IOhlSeH4fplO0VR3AOr/4c214pIYGOjrDt64Dxr5UkZHk2eFEVx\nDMYaSk8Kc9s+C2MMYkh9hCwMtXeSIquEiSZPiqI4B2LliTWck4UF5MVM3IShOAtZ3wxFURwNa1SB\noY484SAxZmafly7bOQvXJ09/f+uvaPCdpGizvO3+uuclNDXUU7RZ3na7/7QdbX4/RZvlbVdVWYlu\nkiUNC2s5gxsNSZeJxGob055FWqLqdFz/zWhraUZvD+cG01DL83ljeZ7Vksx562tr0d/fR9H2ejkx\n+3xerk2KIF0wG4kpquQqDEc2YM6rowqUIHB98sRt9ORNf6Y1ehKHJ0qzZ2H2Z7Bipv6tWY3EFFVe\nlQ/QIZnK2EdA8mSIn0rW07nhbTEmPT4Zw4uZVQ1h2vBIqzwZI88k1xheKYQZs9qzKMHg+uSJtTsn\nAPHGJmzSG9PzjFkNoe08k1blY9rCUFRPxyys8gQQB4Nq5clRuD554g564/k5MP3WWLryTHJJhrFU\nixRiZVNg5UlczxP1OkKRVcLE9fYsn7v8y0hMTqFoz1u4hKL75X+7DuPTObY0hSVlFN0bFtyIxKQk\nivaSsps4uqWl8Hg8Udc1xsirPDFtYSiqgeV/1m47VsyG2kdIkVXCxPXJU1rGRJo2y9suk+jzxvK2\ny8rJoegCPG+7PJKuxGnbzJ1nLJir/yyYy/+Ks3D9sp2iKBGGuL5AWyIFsZGYokpewqKoBux/pHU8\nKOGhyZOiKCHBHBjJuplTlyopqtw+L17MumynBIcmT4qihAS3qZbVnM+7ufEGRjJ3N/LQhnElGDR5\nUhQlJEQ2jBOtSqhDMoVVnpgPBlp5chauT57e3vsi2lubKdosb7uXdlSjs+MURZvlbfeHrb9Df39/\n1HWNMXj+uWejrgsAzz7D8dTjmqfyBs/ypudzkDoYVCtPAWJT0hA7fkLk/peSxg4porg+eWptrKfc\nVAGgobaGo1tXS/si1tZw/PxqvBzzZ2MM6mo459lH8hEMJE8CK0/C7FkAeTFTh6Fq5clRuD55spbX\n9MjCGt6OESaU8yx2276smJkVCRZMyyMW1MGgY6zypJwb9ydPhveUzMIYQxvox4JlGWIsb3giC2bP\nEwtjeAMjWRhDNBgnwTXbpsgqYeL6q4ElehWxYM6kkYbEKozIgZFSryPCgpb4fVbCw/3Jk8gxuTol\nN1pIvNhaKy85lzh5musLyoG5cqbLds7C9cnT7GtuQBLN266EonvdjUWIi4+naBcWl1J0Fy3heOrF\nx8djYVExRbtsKcdTzwpcqgz0EcqLmdUwzsIQYxaWmzse13vbTTgvi6bN8rablJ1L0QV43na5JF2P\nx4OcXM7fW6K3HW9UAbGRmCNLXbbjWdLwqm1aeXIWsh6lFEUZMRKHZFItaSiqZKsSiuon51krT8rw\naPKkKEpIyBySKbEKw+sXpZoha+VJCQJNnhRFCQlL3LZPNQaWWIURFnOgn09LQMrwaPKkKEpIUHth\nWI/nUqswEmMmaSvOwvXJ02t/+iN6e3oo2ixvu+2Vz1F0e3t68Kc/VlG0q37H8Zdrb2vDn7Zvo2iz\nvO0CwxOFVZ4E2rNYIy9marVNszZH4frkqamO4zsG8Lzt6kn+cv39fWioq6Vo1/o4Mff29qKhvp6i\nzfW2o0jTEDnbSuCwXYlz25TwcH3yxPIeoy0vIHDRo+gylzaIjcSs52SRzdMSY6aoyhzP0N/Rrt52\nSlC4Pnmy1lAaAJkD5mi7RYwV10hsrYHHIyxmY+CJ9VC0eQ3jlneeKaqnRxVIG89geV6oWvByFgKS\nJ07p2YLZYEp6OhfYSGyMkRezyMqTAa3CSFEFjCV+timq7DEcHF0lPGQkT4SnJ2oVhnSRN4ZT5QOY\nlSfQkmRmAiPvPMtrnrYWAmPWhnElOFyfPF21oJhywY31eHD1gqKo6wJA/mKOv1xScjLm5RdStFne\ndhMyMjDv2usp2ixvO5HVNiOw58kYXgsARZVrAK2VJ2fhem+7iSSft9jYWGSQtLNyOZ56cXHxmJSV\nTdFmedslJCQgPYvjn8jytgNx5xm12iZtYCR4f29ezxNxGKpWnhyF6ytPiqJEFrG77URu2ydpc2SZ\ns1AVh6HJk6IoIWGsRaywHYbGGlrMLGx3B2/nGUVV5zwpwSPraqAoyohh9jyxsMSeJxZih6FKC1oJ\ni4glTz/60Y9www03YPr06Thy5MiQxxhj8L3vfQ/5+flYsGABniHZSyiKMgKozvO6bBctmKNHWFjL\n3d2oDOTYsWMoLy9HYWEhysvLceLEiUHHnCuvaGpqwn/8x3+guLgYCxcuxLp162CMich7i1jylJ+f\nj9/85jfnbGKtrKzERx99hB07duCpp57Chg0b4PWOrsXE3m1bR/X3n42uzg68upPjecbytmttacae\n3bso2ixvu9oaH/bueZmizfO2M7zhicxlO9K2fRbGWHhI55mFsbyYheWpQbFmzRosX74c27Ztw1e/\n+lXcd999g445V17xs5/9DBdddBEqKytRVVWF/fv3Y/v27RF5bxH7lMyePRvZ2dnnfDKsrq7GsmXL\nAACZmZmYP38+tm0b3QSjsZbk89bXh5bGBop2XQ3H86y3uxvNTZyYa0Y5CT8bXZ2daGlupmh7vScp\nuhKHoVK3sFNUycNQKaqBmHkWUxTZMUtTUxMOHTqERYsWAQCKiopw8OBBNP/L9fZceUVMTAxOnToF\nay26urrQ19eH7OzI7AiP6qgCr9eLyZP/sY0+NzcXvrMYuvr9fvj9/gGveTwe5OaGtv2fVnZmDhKk\nzWaRaM8iMWbi8ETqeZa1bR8CYx7LQzJ9Ph/6+/sHvJaWloa0tLRRfFeRJ9g4fD4fsrOzz3znY2Nj\nkZWVhZqaGmRkZJw57lx5xR133IFVq1bh6quvRmdnJ5YvX44vfOELEYljzM552rRpEzZs2DDgtby8\nPOzaFdqyENPOgffURoqZOUhQomEs0ZJGnAE0M2aKamDZTlrMY9mepaKiAidPDqw2r1y5EqtWrRqd\n95OQDHjiI/cLT/+uaMZRXV2N6dOn49e//jXa29vxrW99C9u3b0dBQcGIf3dUk6fJkyfD6/Vi5syZ\nAAKZ5dl6pFasWIHS0oGTsj2e0M1ImU/nvG2+TNsOaVUYeTFDYrWNuFTJG5LJSySoowpoA2DP/fPN\nmzcPWbFxGsHGkZubi9ra2jMPqMYY1NXVIScnZ8Bx58orNm/ejO9///sAgNTUVNxwww147bXXnJc8\nFRYWYsuWLcjPz0dzczN27tyJJ598cshjI1WO1MpT9DCWaOdANQamSKsZcjTp7kRMGA9vkYBbhRFW\neQKvcXu4j3aoLStjlWDjyMzMxPTp01FVVYXi4mJUVVVhxowZA5bsgKHzis2bNwMApkyZgpdffhmX\nXnopenp68Oqrr0YkcQIi2DD+wAMPYN68eairq8PXv/51LF68GABw22234cCBAwCAkpISTJkyBQUF\nBSgvL8edd96JKVOmROotDMmcwpJR/f1nIzk1DVdccx1Fu2Axx+ctc+IkfHHu1RTtotKlFN28KVNx\nxZe+TNFeetNXKLqGaZJL221H3IVFUeXutmPGzNtJSpEd06xduxZPPvkkCgsL8Zvf/Abr1q0DMHxe\n8Unl6b+/ahy6AAAgAElEQVT+67/w+uuvo7i4GGVlZZg2bdqZ5vKRErHK07333ot777130OsbN248\n8+/Y2FisXbs2UpJBwfK2i4uPR3oqx/OM5W03LjERSeOTKdosb7uk5GQkxaVStFnedhIHCUocDMrc\nbcdC4mDQscy0adOwZcuWQa8Hm1dMnToV//u//zsq703WEA9FUUaMpW7nZo4qoEjTlrACw1BJ0ixd\n6gBYiqwSJpo8KYoSEtYYcQ3jxhrexgCK6unKk8CYeUvSFFklTDR5UhQlJCSOZ6Ba0lBUydv2ObKn\n7Vm08qQMjyZPiqKEhIW8hnHmsh13YKSsW0Rg0K9WnpThcf03g+Vt19JQj7f+8gpFm+Vt5zv5Ed56\nfR9Fm+Vt9+HRI3j7zTcp2ixvu9j+PnGNxAFvO9dfLgdgBDZPG6YzsOIoxuyE8UjRVFdD0e3t6YK/\nheN5Vl/D8fPr6uiAv6WFol17Fpuf0eZUezt6Otop2j4fx88vsJwjK5Fg2nawoFoPUVRBtaRRnIWs\nK2AU4e7O4VmVSHtqM8RhqCxExky0KmERsFtiv4voInE8gxIemjyNEoEJ4/LsWaQtbViB83+sMbSe\nJxZW6rKdtJiNVp6U4JD1zYgiEgcJMmeksAhUGDVmtyP3+yyLgD2LtKiVcNDkabQQerGVVucXe1MV\nF7O4FWmh55l3onVUgbNwfcP4VQuKKbqZWTnInTSRop2/uJSim3f+pxB/wVSK9qIlHD+/T392OsZ5\nOFfbsqU3UXQNcUgmC2NkLtvJW57VUQVKcLg+eWJ52yUkJiFt3HiKNsvbLjklBUnxnBsMy9sudfx4\nJMVxYqZ520GgPQuIg0EpqmSrEooqd0laK0/OQtajlKJEGFbixMQaC49HmD2L4W2G4G3bB28YKkVV\nK09K8Mi78iuKMiIMcYchrfLEjJmiyh1JwYuZaEmjlSdHocmToighwVzOYd7M5dmz8P7etJiJy7Na\neXIWmjwpihISEo2BqTFTVGVWnqwh9nlp5clRuD55Ynnb1Xx0DO+98wZFm+Vtd/T9Q3j/4AGKNsvb\n7p233sSRw4cp2ixvO2MNYoQZAzN3GLIKEsyBkbSYiTsMtfLkLFy/247lbdfd0YHeNj9Fm+Vt19He\njp4YzuMTy9uutbWVdkNnedtB5JBMeTc3mXOeeA4NirNwfeVJ5tZmgUsbxOUccdv2JZ5n5ngGimoA\ncct20IZxJThcnzzRvvzGIDbWQ9FmPTnFx4K3nZt2MzfweEjnmbiE5WF9tpnfZ3HLdkbesh1zqVIL\nXo7C9cmTyKdzVuVJ4BZ2Q20wJcVsDfHpnPV95t3cmAMjpVXbLECsJFNklTBxffLEfFLlbfMlVSQs\ncZAg8zwLa562FogRdp4Ns8JIUT0ds7DBoNZYeAjfZ+ZwTiU8XN8wzvK2m3zhxUglWZWwvO2mXzKT\ntlTJ8rabfcUXkZCQQNFmedsxHwxYBCqMsmAOjGTBGs8g8W/tdFyfPLG87ZJSUpE6jvPnZXnbjU9L\np+gCPG+7CRkZFF2A6G1nrThjYGstrdrGQupuO0YFaCxWnsy4FKC/L3K/0BMHzqP16CDraqAoyohh\nDk9kYSXGbOQlT4aUMLJ0lfDR5ElRlJCwApcYRFZhMPaqIaMNc0yBtM+X09HkaZRIJy3ZKcpoI/FC\nHxieKAvmbjtpSHwgcTqaPCmKEhKBHYayLh3GGNpOUhbMOU/SYM6XUsLD9VcDlrfd0UP7ceLo+xRt\nlrfdO2+8jo+Of0jRZnnbvbrnFfi8HJsUlredyIZxWNpIChYWoPi8sWZ5MbHgzYtTwsP1V8DGWo7n\n2ak2Pzo7TlG062o4N/PWlmZ0d3VRtGtICUxTYyN6enoo2l7vSYquyCGZzGGoFNVA5YmxPMu1/6HI\n6rKdA3F98sSceB1Ds3NgDYwkxky0KuG5sLNiljcMlblsRxsYSdo+z9x5xkpgdLed83B98qT2LFHU\nFWoYKy5miWbIkGjPwvlsS6w8QeAmDKfj+uSJW3mSZc/C7AuhnmdhVRgr0YaH2OfFvKVykifOoEqA\nmSBbCGupczyuT554T+dEk1ymMbAwk1yNOcraNANo5veZA6sKwxzCSotZ4EBSp+P6YURzCksoup+Z\nOQueOM6ft2Axx+ftC1+8EolJSRTtotKlFN1r5l2H1PHjKdpLb/oKRdfCiuvzCpghsyrJHGhVGMP8\nfFFktefJgbg+eWJ526USfd5Y3nYTMjMpugDP227ieedRdAGetx3zKVlaJZm5bZ+684wjTYxZ3hBW\np+P6ZTtFUSKLIQ7JpO4wFNY8Ta3C0D5fFNnTfYSaPjkJTZ4URQkNgUsMEneesWBWnlgEYpYWtbPR\n5ElRlJAITEMmaQsbScEcnkhdwhLWMB74fHG0lfDQ5ElRlJCQOCSTNZ7BWAtPrCfqugBz2c7AI2zZ\njvmdUsLD9WeL5W134G+voc77MUWb5W33l5dfRGN9HUWb5W33p+3b4Pf7KdpMbztpS0ksjDG0TmJa\nFcbIq7YZgUuVTsf1yVNTXQ1F19/SjF6S51l9DcfPr7mxAX19fRTtWh8n5vq6Opj+foq2z8fx82MO\nyRSHBW84J3FgpLQmeRAHgyrhoVfAUYJpYcGCOUiQhdSYpX22WXBNmDm6EgdG6pwn56HJ0yjBNIxl\nwbQqYWGMvJhZhrHMmUcsJI4qkLgsrA3jzsP1V33eBdeC1axANQYWZtsBocbA0rbt686zKOqCNzCS\nOyRTsycn4frkiTpUj7VjhPQlZFbbeOdZasyEnWfGEI2nKbKBv7WwyhP3O0WRpfr5KeHhenuWqxYU\nU3Q//6U5SEpJoWjnLy6l6F593Q0Yn86xpVm0hOPnV7hwERITOX5+ZUtvougyK0+s5mkWFhKXheVV\nYQS2yDqeiH0rjx07hvLychQWFqK8vBwnTpwYdMyGDRswZ84clJaWorS0FPfff3+k5M8Ky9suPXMi\nEsYlUrRZ3nYTJ2UhPj6Bos3ytsvOyYXHw5nDw/K2YyVPzKZa6rZ9jjQtZu5SOEVW5NT+YAgmrzDG\n4Hvf+x7y8/OxYMECPDPECJcPPvgAs2bNwvr16yP23iJWeVqzZg2WL1+OoqIiVFZW4r777sOmTZsG\nHbdkyRLcc889kZJVFCXK0G5sEpdzSEukAHtgpLDzbC08mjwNIpi8orKyEh999BF27NiBpqYmlJaW\nYu7cuZg8OVBEMMZgzZo1mD9/fkTfW0S+lU1NTTh06BAWLVoEACgqKsLBgwfR3Nw86FiJO2YUxU1I\na1QPaFNkA83TwmJm9v8wh2QKW6kclmDziurqaixbtgwAkJmZifnz52Pbtm1nfr5x40Zcf/31+NSn\nPhXR9xeRypPP50N2dvaZD3xsbCyysrJQU1ODjIyMAcdWV1dj7969OO+887Bq1SrMmjVryN/p9/sH\nTW72eDzIzeUswymKEoC5uzBG2sBI0lgIgBkzb2DkWI7Z5/Oh/18G8qalpSEtLW0031rECTaOYPMK\nr9d7psoEALm5ufCdHpr83nvvYc+ePfj1r3+Nxx57LKJxRLVh/JZbbsHtt98Oj8eDvXv34o477kB1\ndTXSh2gy3rRpEzZs2DDgtby8POzatStab1dRlCFgVZ4CA0kp0sSBkQKrMBJjDqKqWlFRgZMnTw54\nbeXKlVi1atWovCebkAyYCLonnPZojFYcfX19+H//7//hBz/4wah8niKSPOXm5qK2tvZMWd0Yg7q6\nOuTk5Aw4buLEiWf+PWfOHOTk5ODw4cO44oorBv3OFStWoLR04K6xcBpz927bijmFJSH/dyPlry/v\nwvTLZmN8+oSoa2+vfA4FxdHffbZr2x9w1bzrkJSUHHXtqt89i8Wl0d999vxzz6J4SVnU+1L6+/vx\n3NatKFu6NKq6ALHyBIjr/wFxOYdnz8LTZp7n4aQ3b948ZMXGaQQbR7B5xeTJk+H1ejFz5kwAgYpV\nXl4e6uvr8dFHH+G2226DtRZtbW0AgPb2dqxbt27EcUQkecrMzMT06dNRVVWF4uJiVFVVYcaMGYOW\n7Gpra5GdnQ0AOHToELxeLy688MIhf2ekypGNtRzPs5bGBvSTfN7qajieZ/V1tbCG8+hW4+XEXEPy\n1DPGoKaG49tI63kSWJGQ2PMUEJYVczDDUN3SshJsHMHmFYWFhdiyZQvy8/PR3NyMnTt34sknn0Ru\nbi5effXVM8dt2LABHR0dEduwFrFlu7Vr12L16tV4/PHHkZ6efmZL4G233YbvfOc7uOSSS/Dwww/j\nwIEDiI2NRUJCAn784x8PqEaNBsydQbSBfqwLD3WIIa8Ph1ENsVbocE5hvTCBmGVV22TuttMhmUMR\nTF5RUlKCt99+GwUFBYiJicGdd96JKVOmjPp7i1jyNG3aNGzZsmXQ6xs3bjzz7x/+8IeRkhvzMJtb\nWUgcYsiCadvBQqLZdiA5FxYzeE3yLCx4TfJjmWDyitjYWKxdu3bY37Vy5cpIvjX327PwtlUTlxhY\n3nZEN3Tm9nmWrry/tbwlLImDQQ31OkKRPR0zR1sJD9cnT7xlO56tAs3bzspbtqMuC9OWc3ifL2kN\n41biUqWVFzNzJIUSHq5Pnljedl++Lp+y6wzgedsVFJXA4+HYJbK87UpKo7/bDQASxo3D4sWLKdos\nAtU29ruILpa49YzZPC0vZrVncRquNwZmedtlTsqm6AI8b7usHN5uEJa33WSSv5zH40GOS3bfBI3A\nnjqJQzIhMGbmrkolPGRdiRRFcSzM4YkspFqViItZe54chyZPiqI4AmtB66ljYS3EbduXWG2TuMPQ\n6WjypChKSNDsWazhzTBjWpWIi5lXhaH1PJnhJ4wrYwtNnhRFCQnmDkNxu+2sJe5gpciK3FVpiOdZ\nCQ/XJ097t22l6L78QhX6ensp2tsrn6PoVj//W4puT3c3Xvh9JUX7+d8+Q9H1+/14Yds2ijYLiTuS\nZMbMfgfRx1pLqzAq4eH65InlbddYy/Mdq6/hxFzr4/jL9Zt+1NfVUrS9JE+9vt5eNDTUU7S5QzIp\n0iK3sDO97aTFbDrbtWHcYbg+eWIO9GNoM8v8tNkszIGkRJ83iUMyeTFTZGEENk9rzIoTcH3yxHpK\nZj09MZ3nWY9tMq1KNOboalNkNeaoa1NkRXpVOh3XJ0/MLyGjGmJBNCSm/a1lVmF4TbW8apu0bftG\npD2LvMqTjipwHq5PnqQZAzPNebmVJ4o0ubLJkpZVzT0tzUFgzNzvM0fXGIFd8g7H9cnTnMISiu68\nhUsoup64OFy/kON5VljC8ZdLSkrGDQtupGgvKbuJopuRmYnrrr+Bos2rPPHsWbhb2IXFbHjVc2bx\nh1VVVcJDve1GiUk5HH+52NhYTCLFnEPyl4uLj8ekLI6XIMvbLiFhHNKTEynatGoutAoTXW2OLiAv\nZqOjChyH6ytPiqJEFuoOQ3E9T/KqbX3tbQJ727TnyWlo8qQoiiMQOTCSWIVhYSGvCiPxPDsdTZ4U\nRQkJ7pBMict2smKW2SQPbsOVEjKaPCmKEhJMbzuPsOZpK3DZzhCTJ+YOVl22cxauTp76+/rwlx2/\np2jv/sPvKLqdHafw8g6O5xnL2665qREv795F0WZ52/m8Xux55RWKNgtjjbinc2sthK1giRwYGfC2\nU5yEq5Mnaw1aGuoo2g0kb7v+vj40NzVQtGtJnnq9PT1obmqkaPt8nJi7ujrR0tJC0WZhraUNBmXB\nHIbKwljeYFAWxlodVeAwXP2tpFqVkDDGgPaoSmoYMELPs7yY5T2dCyw8UQ2gWVjIO89Ox9VzngL3\ncmkfSeKuDeIVT9zuHIlXW4G77Qyxz4uFxAoj1RniLNj4pMg+EI+x+EaKyz+hzA+kPPNUNQaOqrC4\nmCUOyYzp6RAXs8T+H+ZnWwkPVydP1jD7BVg7kuQZAweGJwozybWGuAuLaQysO8+iBdWeRVj/j7EQ\n1+fldFy9bBc3bhyuuG4BRXveQo6nXmpaOq66luN5VlhcStE9b1IWMudeQ9EuKV1K0Z069XxMzeVY\n0rCQufNMXkVC5JBMog2PEh6uTp48njikn5dF0WZ528XFxyN9/CSKNsvbblxiIpJSkynaLG+7pORk\nJMWlULSpQzJZFV3awEjm7CGerrylSnl9m07H1ct2iqJEHuayncfjoWjzlu0MPLESY5a1VGl1VIHj\n0ORJUZSQ4FWejLgqjDG85RxaFYa484x2ngUuzzodTZ4URQkJ5i4/aVYlImMG7zOm9ixKsGjypChK\nSNAqTz0dtLsbsyIhbOYtdQAst/LE0VbCw9XJU0dbK956heN5xvK2a6qvxeuvvkzRZnnbnfzoBN74\n62sUbZa33ZHDh/HWm29StJk3NokVCXkx87SZ1TZxW0kdjqt32/X19sLf3ETRbqjleJ71dHfD39xM\n0a71eSm6XR0d8Le2UrS9Xk7Mp9rb0dPZRtHm9TzJ64XhDkOlyFJHFVAHg2ru5ChcXXmyljc8kTYk\nkxkzcWAkazAoc0gmbxcWcbedwCGZ4mI28mLWnifn4erkCQKzeYFzBEU+tUk0vZY4MFKiAbTI7zNx\nSVoJD1cnT1xjYN7Shry9zdwdYCxdjTma2hRZav8PdwlL2HmGPD8/p+Py5Emgtx0zZuaynThvO57/\nF3PZjhczRZa8JE2RFevnp0MynYWrG8ZTJ2Ri5pevpmizvO0m5UzGlGyOPQvL2+78T01DHC6gaLO8\n7aZ/bgYSOS1PNKyV6nmmMbsdiTE7HVcnT/EJ45CWyPH/YnnbjUtMQnrieIo2y9suOSUFSXGcp3OW\nt11WRhpFl4kFiJVkEhbiGomtxJghL2anI+xKpCiKUwk0T7PfRXQJLNvJClpiFcYY7XlyGpo8KYri\nDKwVV3myFuJ6YUTutoPutnMasq5EiqKMGNZuO2MseBsxKLIirUp0SKbiBDR5UhQlJJiDQVlVGN15\nFj0CO89k7TBkGkAr4eHqs9VU68V7b+yjaLO87U4e/xAH336Dos3ytjvy90N478B+ijbL2+7NN9/A\nkcOHKdo65ym6wtJiZprkSoxZCQ9X77br7erCqbYWinZDbQ1Ft/NUO/o7OJ5ntTUcP782vx+JpN12\nPh8n5pbmFnH2LFZgRcIQbTuoMQusMOpuO2cRsSvRsWPHUF5ejsLCQpSXl+PEiRODjjHG4Hvf+x7y\n8/OxYMECPPPM6D61W/B6JHjI26mSEAtIO8/WWqJvIwed8yQEkedZG8aHYqR5xWjmHBG7+q5ZswbL\nly/Htm3b8NWvfhX33XffoGMqKyvx0UcfYceOHXjqqaewYcOG0XWlp154WEsbxC8hbTlHoD0LMUnm\nxkyRVquSKOvKi5m1Ljy2GWleMZo5R0SSp6amJhw6dAiLFi0CABQVFeHgwYNobm4ecFx1dTWWLVsG\nAMjMzMT8+fOxbdu2SLyFITES7VmMPHsWawxiPbLsWSzVeZ5pzyJs2Y5o28G1pJEVsyUuVY5VIpFX\njGbOEZErkc/nQ3Z29pkPfGxsLLKyslBTM7Dvx+v1YvLkf0zezs3NHdWeEWuIrpqkyhN1qB5rCzsx\nZt62fXkxi6xIEFsP1Aw5irqAuKXK4YhEXjGaOceYbRj3+/3w+/0DXvN4PMjNzQ36d5w3eSqSp3Ls\nM+YtXELRPX/ap5GawHk6Lywpo+jOmHkZPB5O8/SSspsould88YtITEykaDOTNmnN08yKhMRt+9yY\nzy3u8/nQ398/4LW0tDSkpTnLqsktcUQkecrNzUVtbe2Z9XljDOrq6pCTkzPguMmTJ8Pr9WLmzJkA\nAn/EvLN4g23atAkbNmwY8FpeXh527doV9Psal5SM1EROfsjytktOHY/xpJhZ3nZp6ekUXYDnbZeZ\nmUnRBZiVJ4EDI428nicdVTA0FRUVOHny5IDXVq5ciVWrVo3Ke+qLiTtd+YwMMTExGIfg44hEXhFK\nzhEqEbnLZmZmYvr06aiqqkJxcTGqqqowY8YMZGRkDDiusLAQW7ZsQX5+Ppqbm7Fz5048+eSTQ/7O\nFStWoLS0dMBrrOqCoij/gNfzJHVUgayYJVbbAtrnFt+8efOQFRunEWwckcgrQsk5QiViJYq1a9di\n9erVePzxx5Geno7169cDAG677TZ85zvfwSWXXIKSkhK8/fbbKCgoQExMDO68805MmTJlyN/nxDKe\nokiAOSSTtilBoG0HNWaBfV7DEUrLylgmlDhGmleEknOESsSSp2nTpmHLli2DXt+4ceOZf8fGxmLt\n2rWRklSGIJ20ZKfIgbnbTlrPkyHunmXuMIwRVnnSEU9DM9K8YjRzDllT9hRFcSzM2VYsJE7/YRkD\n66wlJRRcnTz5PjyMY38/QNFmedv9/cA7OHbkfYo2y9vujb/uw0cnjlO0Wd52L7/8EmpI1jAsRE5h\nFjhhnLU6K3G5UAkfVydPXR3t6DzVTtFmedu1+VvR2dFB0WZ527W2NKO7s5OizfK2a2xoRE9PD0Wb\nhTUGHtIwVBaGOAyVhbWcmCV66inh4+pvpbUCB4+JtGdhDkPlINHzzFgDaR6G1KG3JFgDYCV6Jyrh\n4+rkCZbXeMjCMmNmDk8U+HQuzxgYIs+zRNsOTvIEcfcLJXxcfSViDtWjGQMTbTuYlSdpViUS7VkC\nMVOkecMTBQ7JZLX/GGtpdSfteXIerk6ekuNjEUMaMEfzoyLtVAFArTzxjESJ51lazNTzTJGletvR\ntu1zZAHteVJCwNVDgc7/9OdoH8p5C0soup+79AuIi4+naBcWlw5/0ChwxZevQlJSEkW7pHQpRffa\na69DOmmILG1LN7XCSJGl9TAyt+1zK0/CPl9K2Lg6eUpOHU/TZnnbpU2YQNEFeN52GUSfN5a33aRJ\nkyi6AHFIpkCrksCQTFLzNKvKR1H9xP5HK09KcLh62U5RFPdA3QxBwsJSpqpL3HkWaHlQlODQ5ElR\nlJCQ2SRPkaU1jFteqxVt2U6id6ISPpo8KYoSErpsFz1YYziMJXrqUVRP/62FbUhQwkeTJ0VRQoLb\nME6T5uhyZE8PGCZps3SJA4a18uQ8XJ08Hd3/FmpOfEjRZnnbvbXvL6j1nqRos7ztXnnxz2hqaKBo\ns7zttlVXo62tjaLNHFVAq4YI27ZvBVaejDVaeVKCxtXJU7u/Fd1dXRTthlqO51lLcyN6erop2rU+\nL0W3qaEBvX29FG2vlxNzbW0t+vv7Kdq8nid5PSnM/h/ebCuSLtHlSStPzsPVyRPAtDbgfQtZfSHc\nIZmsioTEKgwxZtbOM2GVJ2OI/WUUVe40d608OQ9XJ0/WMI1ESU/nAh+fuLYdLEsagfYspJip9j8U\n1U92npG0ObLUqf1aeXIerk6eDNU8lfQlJA3VA0CsPBH7M1g7z6hPyazPNmnnmTHyqjDMnY0U1dOf\nL608KUHi6uSJuTuHBXNWCQuRA/2I1RAWrJiZy3YsRMYMeTEr4eNqe5bPfP5yxCckUrRZ3naXX3U1\nUki2NCxvu3k35GN8OsfnjeVtV1RUhMREzmebCSV5IukykZmcQ9xDmBI+rk6eUtMzaNosb7uMiedR\ndAGet915k7IougDP2y4nN5eiK5HAUri7i/T/irUWHmExG2vE2f8o4SPr26EoihIiTFsYFsyeOhZa\neVJCQZMnRVGChjZdnIi1EFl5kpY8GWLDuOI8ZF0RFEUZEdRt+8wRCRRl3rZ91lgIgBcziA3jAp9J\nHI+re54URYkszOSJmbTRYqaocqdtcweD6qiCT+jqs4G5gREiNgZIidhv4+PqytOBv+5FS0MdRZvl\nbbf3z39CW2sLRZvlbbf9j79HV2cnRZvlbffbZ5+FMSbquhIrT1KHZLL6f3RIpuIEXJ08+Zsa0NfL\n8Txjeds1NdTRPM9Y3nZ1NT7ajZXlbef1nuQNjBQ2kNQaS9uFxas88ayt1J5FcQKuTp4CpWdZn0qJ\njZ4SY2Yh8W9tLG/COAuJ51mHZCqh4PIrgsRp29CYBSByCUtizBRVoct2xIdtXbZzHq5OnphLDDRv\nO6LPGyuBMQK97XieevL+1sYIXMKyFrEeYd52xMGgwp79XIGrkydu6Zm0rZo50I/1+ETy4WLOPBJZ\nhWHGLKwKY4zA8QyEDRifoJUn5+HqUQWXXfVvSE7leJ7NW7iEonvN/EIkJSVTtAtLyji6i0sQFxdP\n0WZ52y296SsUXWuZ27l5t/MYVkWCosodDMqNWRvGleBwdfKUnsnzeWN5252XlU3RBXjedjm5nL91\nTEwMzdsuj6RriQOAaEMyDXF4IkcW1hpxMRtmhVErT47D1ct2iqJEFh2SGV2Y/T+sRIIaM2swqFae\nHIcmT4qiBI32PEVZm6LK3cFKHZKplSclSDR5UhQlaIwxxB1JvN2cHmE7z4wlnmeKamAYqvY8KcGi\nyZOiKEFjmbs5SVgjb14cs8+LhRE4GFQJH1cnT397cTu6Ok5RtFnedn/6/fPo7+uLuq61luZtV/W7\nZym6XV1d+OPvKynazz7D8dSzgLhEgrlsR0NgIsHseVKch6uTp+b6OljLmd3RUFtD0a0neepZa1FH\nirnWx4m5v78P9XUc42kfyUeQO3iWQyBmWXdVQxxJwSIQs6zPthI+7v6kBLoe2e8iqrC2kgeezoWh\n3okiCPROC4tZ4nmWtzqrjABXJ0+WONyOBmm4nbVG3N86cIORF7O0ioTUmGOEVdskJoxK+Lj6ym8N\ns1eBNNDPchp6JW5hD1jhUKR52/aJDePc8yxr2z51YCRFlX0No8gqI8DdyRP16Yk1YI705Te8fgHe\njc2KM8llVnOZno3y7FnkmSEzWw+04OU8XJ08ffH6QsTFJ1C05y0soejmLy6l6MbFx2P+osUU7UVL\nOJ56KckpmL+gkKJdtvQmim5gC7uwyhNxFxatCsM8zxTV0w8GWnlSgsTV3naZWTk0bZa3XRbJ5y02\nNhZZObkU7VySp15cfDyySF6CLG875s4z2pBMY3hVVYpqYPmfZgBNUQX62tvg0SGZSpC4uvKkKEqE\nEZx75tsAACAASURBVDjzSGIjsRW4k5TZ56U4D02eFEUJGokbA5hb2JnN0/K87cCLWZftHMeIk6eu\nri7cfffdKCgowMKFC7F79+4hj9u3bx9mzZqF0tJSLFmyBDfffPNIpRVFiTLGCmySt4Y2kkKX7aJH\nYNOLLts5iWDzDwDYsmULCgoKUFBQgAceeGDQz3t6erBw4ULcdFNw/aQj7nl64oknkJqaiu3bt+P4\n8eOoqKjAjh07kJSUNOjYiy++GM8+y7HSUBRl5FArErTKk8DmaeKyHbdhnKStlaewCDb/+Pjjj/HY\nY49h69atmDBhAm699VZs3boVJSX/2Nj18MMPY/bs2XjvvfeC0h7x41R1dTXKy8sBABdccAFmzpyJ\nl156achjo33x27tta1T1PqGvtxcvv1BF0d5e+RxFt6O9Hbt3bKNos7ztmhoa8NKfd1G0Wd52xhh4\nhFWejLHweGRVnpiDQXkxQytPDiPY/OOFF15Afn4+JkyYAABYtmwZqqurz/z89ddfx/HjxwckU8Mx\n4sqT1+vF5Mn/2OGVm5sL31m8xo4fP46ysjLEx8fjlltuwZIlS876e/1+P/x+/4DXPB4PcnOD39HV\nSPJ5M6YfzfUcz7O6Go7nWW9fL5oaGijaNV5OzN093WhqaqRoe70nKbpih2TqwMjoaVNUeQOGgeAq\nTz6fD/39/QNeS0tLQ1pa2ii9q9EhknEEm3/4fL6zHtfR0YEf/OAH+NnPfoYPP/wwaO1hk6eysrJB\nb+aTL9aePXuCFrrkkkuwe/dupKam4uOPP8Y3vvENZGdn46qrrhry+E2bNmHDhg0DXsvLy8OuXcE/\n6TOH6rEeJWi7RYgXW5HnmTkkU2OOGjokM3owlyqDka2oqMDJkwMfmlauXIlVq1aN0rsaHUKJI1L5\nx7lYv349KioqMGnSJHzwwQdB/3fDJk/PPXfuZaC8vDx4vV5kZGQACGR4V1555aDjUlJSzvx7ypQp\nmD9/Pt54442zJk8rVqxAaenAgY8ej2e4tzsA7u4c3oWeokscJMhC7s4zaTELHJLZ3UGbqs6ttrG0\nhz9m8+bNQ1ZsRouufoN+E7mz8ckMrVDiiFT+kZubOyBh8/l8Z1ax3njjDbz88st47LHH0N3djdbW\nVpSUlGDr1nO3/Yz427FgwQI8/fTTAIBjx45h//79uOaaawYdV19ff+bfLS0teOWVV/C5z33urL83\nLS0NU6ZMGfC/UJbsAKm7c3iDBKWZ5Borz5LGGCPPMNbIs2cxljgMlaIa6G0byz1Pubm5g+6JTluy\nAyIbR7D5R0FBAXbu3Inm5mYYY7BlyxYUFgbcISorK7Fz507s3LkTDz30ED772c8OmzgBEeh5uvXW\nW7F69WoUFBTA4/Hg/vvvR3JyMgDg0UcfRXZ2Nm6++WZs374dTz31FOLj49HX14fS0lJcf/31I5Uf\nk0iswgDyhupJHJ7IHJLJrHhJG54ocUgmc3lWCY9g84+pU6fijjvuwLJlyxATE4Orr746pObwoRhx\n8pSUlIRHHnlkyJ/dddddZ/5dUVGBioqKkcqFxFULiqOq9wkJiYm46gaO5xnL2258ejrmXstJhlne\ndpOysnHe1YOfcqIBy9tOYsIo8WGIaZLLwlpiz6gSFsHmH0Bgh92yZcvO+fu+9KUvBT1OydXedhOz\nOV5rHk8c0idxPM9Y3nbx8QlIm5RF0WZ52yUmJiIpNZmiTfO2sybk3kOnwxwMykJqzNKWpJXwkfXt\nUBRlRBgjtAojL2iZ55n9JhTHoMmToihBE0gkZF02rDHyqjBGXuXJWksbAKs4D/2kKIoSNBKHZNru\nDh0YGUV4MY/tUQXK2EKTJ0VRgoa5hMW8mbNuqtSBkaxdlRRV9mebIquMAFcnTyxvu3Z/C/76Msfz\njOVt11BXi317XqZos7ztPjpxHK/ve42izfK2Y27nVmPgKOpKHAxqucmq4ixcnTw11dVQdPt6euFv\nbqJo19dw/Py6u7rgb22haNeexUtxtOns6IC/tZWi7fNx/PwCvTCyKk/GGN4wVIoqd7cdtfJE+2xT\nZJUR4OrkiYUl9guwkBkzz9uOhcyYxYV8ehiqLCQOQ1XCx9XJk8wyPy9m1jMj6zwzzZCpMQs7zxa8\n7Im7bCcsZjCn2FNklRHg6uSJaudAc2EnxWyIXlis5RxraFubmUtYHo+smK0hnmeKqsxlO2PsGfPa\naCOusukCXJ08MZ/OeU+qrMoTN1llYAzxPLNiZm5hJ55naVUYkTFTq6oUWWUEuNqeZU7hyIz/wiUt\ncyK+cCXH86xgMcfnLXvyZORknUfRLipdStH91LRpiLeGor30pq9QdGP7+xAbz7lsMCuM4qowllhJ\npqie7tvUypMSJK5OnljedvEJ45A+PoWizfK2S0xKRlJ8KkWb5W2XkpKKpDjOTZXlbSext41p28Hd\nti+r8qSjCpRQcPWynaIokcWC2AtDbOaNEVZ5giWOpKCofvLZ1sqTEhyaPCmKEjQBexaSNq3yJNOq\nRNoOQ2O050kJHk2eFEUJGmshsPJkEUvbPcuBuVTJXCJVexYlWDR5UhQlaAzRGJgFc+cZC+a4FRaW\n2fSkOA5XJ08sb7sG30kc+BvH84zlbXfiw6PY/9abFG2Wt93fDx3EewcPULRZ3naBUQWuvmwMgrnb\njoUh9jyxYFYYFefh6t12LG+77q5OtPk5nmcsb7uOU6fQ3e6naLO87XpOtaGPdFNleduBaBjLQqI9\nSyCRkJUwWgtNnpSgkfXtiBrySt5MqxIWMpc2eH0hLJjjGVhYgUmyEfh9VsJHk6dRwArskRC5tGE0\nZglYgTEH+ryExWyNVp6UoHH1t0ONgaMpLM+qhGoYSx0YKSxm8JbtuMbAJG2OLJgFRh1V4DxcnTzx\nqj+8pzbWjc0Y4nZu4lWe1RdCNb0WOapA1pBM6jBUiioCg0F1VIESJK5uGJ9TWEzRnZR3PpKnTKFo\n5xeXUnQv/PRnkBjHuQKwvO1mXnoZYj2cGwzL2y6w244iTTQGFjgk08izpGH2PGnlyXm4OnnKzOJ4\n2yUmJWP8OM6fNiuH422XkjoeSfGcRCKH5OeXlp5O0QWAyZM5MVsj1Z5F3pBMaZUn5m67sVh56uoz\n6DORy+riXDb6wtXLdoqiRBZqP5/as0RPV2DPk1aelFDQ5EmJCKyqkxJdAjuSZFWeqP18FFXubjta\nzMTkaSxWnpRzo3c8RVGCRuL8H5HzvCDvPEscAKuEjyZPiqIEjSXu22eOpJC5bCctZs5uZdbnWhkZ\nrk6eWN52H39wGEcP7ados7ztDr37Nj488j5Fm+Vt9/q+13Di+HGKNsvbjjkwUpftoodhbtunqH4y\n6Df66sZaeIQNYXUDrt5t11jL8TzrbG+DjeU8TdTVcDzP/K2ttCeoGi8n5ubmJqRPmEDR9npPUnRl\nNowTh6FSVCVXnqIPcyyEEj6uTnepgwRpT206PDFaBO6psmKW2DBuidZDtG37AqttrMGgzGVhJXxc\nnTzJ3NosMGbWeTbyBkZSt7CLtFviILPPi6Qr0AvVDbg6eWJ9IFPiYgVWnnjaIiuMxP4faRVGkT1P\nxoobDMqca6W5k/NwdfLEIvAAI+vbIHc7t6yYIfE8S41Z2DWMhf6tnYmrG8ZZ3nYXfuZztIsty9tu\n5qwvICEhgaLN8rb78pVzkJySQtGmedsZXs8TC2N5VRgWxlrKzjOJ2/aZOxuV8HF18sTytksZn0bR\nBXjedukTMii6AM/bLnPiRIouQPS2kzo8UVhlgFVtY3rqsbACk3M3IOtTqijKiAjsMJR1oZe4bAdS\nD6MRWnkS9ulyBZo8KYoSNMYYxHo87LcRVYy18HhkXSppAyMNZ7mQidVlO0ci64qgKMqICIxnkDWS\nwhjD28VKUeVVGJnN07RRBbwZrMoI0ORJUZSgYT4lU0dS6Lb9qBAYVMn6fFFktfLkUFydPO2pfp6i\n+947b8B74hhF+4Wtv6Xo/u0ve1FDsgypfI7j8/bSn3ehsaGBov3MM1soukaoPYsOjIySLnFgJCtm\no6UnR+Lq5Km5vpai297aiu7OTop2Q20NRbeluQk9PT0U7boaTsz19fXo6+ulaNf4OL6NEgeDMnee\nias8UT9fFFlYC608ORBXJ0+0J1Xidm6ePYtA2w4QDWMlVmGYMRPSGO7fmiJLrmxSZKnXTiV8XJ08\n0b6ExvAsLEjPqszhiczzLDFmkfYslJ1nnB1vAK8KQ7XCYcVsjVaeHIirkycaFogRNoUZlud5xkLk\nQD/Ia25l2fAw/SJZSBzCKvE8uwFXX/lpW5utEbdsJ7GR2FCbWzXmaOpSkieidyJ1CUvcqAIrzQrV\nFYw4eaqsrERxcTEuueQSbN68+ZzHbtmyBQUFBSgoKMADDzwwUulhmXtjyahrDMWML3wRk3LzKNoF\nxWUU3S/NvQYTz5tE0V5cdhNF9/r8fIxP41jx3PSVZRTdwFKSrGU7lrYxhjcigfXwRxySyVu2k1fN\njRRdXV24++67UVBQgIULF2L37t1nPfZs+Ye1Fg8++CCKiopQXFyMb3/726ivrx9We8RXwRkzZuDh\nhx/G4sWLz3ncxx9/jMceewxbtmzB9u3b8eGHH2Lr1q0jlT8nLG+7tAkZSExKpmizvO0yJ56HhHHj\nKNosb7usrGzEx8dTtGnedgIrjDTHEOJyDq0KQ92EQZE93eahyVM4PPHEE0hNTcX27dvx05/+FPfe\ney86h9jpfq78Y+fOnXj33XdRVVWFyspKXHTRRfjpT386rPaIk6eLL74YF1100bAn/4UXXkB+fj4m\nTJgAAFi2bBmqq6tHKq8oShSROKqA2kgsrArD3LZPHQyqyVNYVFdXo7y8HABwwQUXYObMmXjppZcG\nHXeu/CMmJgY9PT3o7OyEMQanTp1CTk7OsNpxEYzjnPh8vgFPy7m5ufCdY1aN3++H3+8f8JrH40Fu\nLqeapCgKd+YRC+4WdllVGEO1/6HInu4XHf44n8+H/v7+Aa+lpaUhjdQ6EC6RjMPr9QaVV5wr/7j+\n+uuxb98+zJ07F8nJyZg2bRrWrFkzrPawyVNZWdmgN/PJl3rv3r2j9kHftGkTNmzYMOC1vLw87Nq1\na1T0FEUZHtYNXe62fVZ/GUUWFgIrT0Eu21VUVODkyYEuDitXrsSqVatG662NCqHEca78Y8+ePRF5\nP/v378cHH3yAV155BcnJyXjwwQfxgx/8APfdd985/7thk6fnnnsuIm8wNzd3wB/M5/Ods4q0YsUK\nlJaWDnjNI8zNXVHGGtTKk7ClDYnDEwOVJ/a7iC7GBmc8vXnz5iErNqNFd59FT7+J2O9L8AQeBEKJ\nY7j8Iy8vD16vFxkZGQACecWVV1456Lhz5R/PP/88rrzySqSkpAAAiouL8d///d/DxhO1x5qCggLs\n3LkTzc3NMMZgy5YtKCwsPOvxaWlpmDJlyoD/hbpkt3fb6Dakn423/vIKmhvqKNrbKyOT7IbKSzu3\no83fStGu+t2zFN0//r4SXV1dFO1nn+H4+bF22zGHsLKwUmenSTvPQe4wzM3NHXRPdNqSHRDZOBYs\nWICnn34aAHDs2DHs378f11xzzaDjhso/brzxRgDAlClT8Oqrr6Kvrw8A8OKLL+LTn/70sNoj/pT+\n4Q9/wLx587Bt2zY8+uijuPbaa3H06FEAwKOPPnomsKlTp+KOO+7AsmXLUFhYiAsuuAAlJaM7SqCp\njuTz1tSAvl6O51l9DcfzrLG+Hv19/cMfOArUknzean0+Wi+Oz+el6AKcxm1L0mXC9LZjIXHmkYW4\nkCPGrbfeitbWVhQUFOD222/H/fffj+TkwE734fKP4uJiAIFlxKysLBQXF6O4uBgHDhzA6tWrh9Ue\nccP4okWLsGjRoiF/dtdddw34/8uWLcOyZZz5NFFF7BKDsJgh74ZOQ+ASlsjvlMBp2xKXZyNFUlIS\nHnnkkSF/Fmz+kZCQgO9///sha8uqj0YJkaVnyxvox4LZxCwNYwxihfU8GoH2P8zxDCyY7gxK+Lj6\nm8m0c6ANeiPZs8gcnsixkuBu22fZwgTXVDsacEcVsLR5utJGFTBjVsLH1ckT03metq2adYMxlmaG\nzDzPjIZeiYMqAybMsrawM5vkuTELO89qz+JIojYkk8GcwmKK7uy512J8+gSKdn5x6fAHjQLXLrgR\nSUlJFO2i0qUU3cVLyijjM2JiYlC2lOPnxzPbljcwEsTmaZ49C3feEktYUyfn4erkieVtl0EyyAV4\n3naTsrIpugDP2y6X5C8XExND87ajLS9Qq20U2dMVCVmVJ2sMsYJNkT3d26bpk9Nw9bKdoiiRhdpf\nJqzyJDNmeZWnwHdKkyenocmToihBw7qZMz3PqPYs0vq8rPY8Kc5AkydFUcY8EnckWcjbwi7yPMsb\nC+gKNHlSFCVoaMt2xESCumzHkdalyugqi0sY3YCrkyeWt92rO7ehq6ODos3ytnuh8nn0n/YGiibW\nWpq33fO/5fjLdXZ24vdVVRRt5rKdhzQwktc8zfO2k9g8TV2e1eTJcbg6eWqs5XieNTXUwRiOz1td\nDcfzjKVrjEFdDcfD0OvlxNzf34e6ulqKNq3yROx5YlUkmP0/tMqTwGGoOmHcmbg6eeINT7SIjeVY\nSbAuPMZajn0G0cKCeTOXF7O8KoylfrYpsrCAuMqTtdDKkwNxdfLE21ZteMPtWPYspMqAscyKBO9v\nzfqAcYdkUqR5FQkjMWaplSeOthI+rk6eqE/JrO22rKyNubwg0KpEoj0L67PNrDzJi5mnTUtgdMK4\nI3F18kRD17Cjh8C/NTN5YiExZom7sESOZxAYsxtwdfLE8rabm78IcfEJFG2Wt92NJWUUXU9cHG4s\nKqFoLynj+MulpKZiQWEhRZuFNQYej6svV4MwxiJWYsykPi8WgeVZTZ6chnrbjQITs3MougDP2y6b\n5C/n8XiQlcP5e7O87eLj45GWzfMSZCDxBsPs/2FhrRHX/2M629XbzoHISvEVRXEkgY0Qsm4wxHY+\nGtYS+zZJMHvblPDR5ElRlDGPMUbgco7UmGUlEsYCHmExuwFZ30xFUUYEb/yHvG37Iu1ZwKs86ZBM\nJRQ0eVIUJWhoF3mBAyMhdDBojLghmZo8ORFXJ097qp+n6P759xx/ub7eXuz6YyVF+48kf7n2tjbs\nfKGaov27Zznedo0NDfjzrl0UbdqQTMPrC5FYkaAOBhV2nplVVSV8XJ08Nddz/L8aSb5jxvSjsb6O\nol1Xy/GX6+/rQ1NjI0W7pobjndjd3Y2mpiaKNu1mDpkVCZZtB+9mzkskmJY02jDuPFybPLGekE+r\nU1QN0TyV9dgm0p5FYMzMzza3IqExR0+bIquVJ4fi2jlPljnnn+g7xuqRoNmzEHfnsC7yzF1YvJjl\n9TwZa8WZ5PadakNCHOe2xD3PY6+O4e/uQ3efidjvGxc39mIcCe6K5p8R6BcksfGQmyRzsPJGHp3+\nPssK2kockkk0Q2bB3FWphI9rkyeuXxCt/sv7EtKWsJhbm5nb9iXGTJEmLucwjZgpstRhqBLPsxI+\nrl22i4mJxZUFiyna1y7i+MuNS0zCNfk3UrRvXLKUops2YQL+7fobKNqlS79C0c3OyUFWRhpFm7Zs\nZ424bfuBmGUt2wWWZ4XFTNwYoISPa5On2NhYpGdxPM/Oy+Z46nni4pCZxfE8Y3nbJSQkID15EkWb\n5W2XmJiIpNQkirbMahtFVmbM0FEFijNw7bKdoijuwRh5T+diYxZmVaITxp2JJk+Koox9BN5gJG4A\nAQT2/1h5+z/cgCZPiqIEDXNIprRRBTE9HeKGZEocDBr4bGv65DQ0eVIUZcxDHQBLwggcw2GMvDKM\nIe4YVsLHtclTd2cH/vrnFyjaLG87f0szXn2R43nG8rarq/HhL3tepmizvO2OHzuGv+7bR9FmYYy8\nJSwrtP8nNsa1t6Uhkdjb5gZc+yk1/f3wNzVQtFnedn29PWghxczytuvu7kJrczNF2+fzUnQ7O06h\ntbWVos1CYv+P3JjZ7yK6WG16ciSuTZ6Y/l8srJH31GYNz5KGN/OIZ5LLwlqeJQ0LY42477PEmUfM\nPi8lfFz7zbQWPJ83EgHneWExj1FfqNHESrQqkViRkNfyJHRXpcAdhi7AtXcda5m+UKRBgsymWtKE\nOWMjZ1wZKjIHRkqMmSIrM2YItKRRbztH4t7kyfCsDVgL2NRha8SkjRUz7SJveEtYtJh7OhAb66Fo\nU207hI1nMMbItGcRtgzvBlxrz5KUmoZZc6+jaF+7aAlFN2PiJGTNuYaifWNJGUU3J28KJmdnUbSX\nlN1E0Z128cWIB6fixqo8GWMRFyesIkGsJLNiNsQlaWa1TTvGnYdrk6e4+Hikj+d4nrG87RLGjUN6\nYgpFm+Vtl5SUjKS4VIo2y9suJSUVSaREgrlcKG54osiYIa7ypA3jzsS1y3aKokQeWuWJuHuWWoUR\nFjOzV1WNgZVQ0ORJUZSg4W1I4GkzKxLyYiba4TB7njR7chyaPCmKEjS62y66usI2z+p5VhyDJk+K\nogQNbTAocfcsryJhEEMaksmswkirtlF3SSth49rkyd/UgP2vvULRZnnb1Xo/xtt//QtFm+Vtd+zo\nEbz71psUbZa33dH3DuDggQMUbRZWoOeZFbiFXaIljcTBoG7Atbvtenu60dbK8TxrrCP5vHV2os3P\n8Tyrq/FRdDtOtaPnVBtFm+Vt1+b3Iy6O89XlNYwLXM4xArftS1yqhA4qcCIjfpSrrKxEcXExLrnk\nEmzevPmsx+3btw+zZs1CaWkplixZgptvvnmk0ufEGqYvFOuCZ3j2LLTlHIHedoZ3nrmDQeUt50iL\n2VoLj7jBoLyYnU5XVxfuvvtuFBQUYOHChdi9e/eQx9XW1uJrX/sarrjiCtx008D5fDt37kRZWRkW\nL16MxYsX45e//GVQ2iN+fJ0xYwYefvhh/OIXvxj22IsvvhjPPhut5R1mFx6xqZb1DMN6bINalURb\nm6UrrvIkMGYjcFSB0VEFYfPEE08gNTUV27dvx/Hjx1FRUYEdO3YgKSlpwHEpKSm46667cOrUKfzk\nJz8Z8LNJkybh5z//OSZNmoT29naUlZXhsssuw+WXX35O7RGnuxdffDEuuuiioL7k0bzwBoyBZdmz\nwBKNgYk3c15TLSlmMCsSvBubtOZp7mebIku9bhOncGjPU5hUV1ejvLwcAHDBBRdg5syZeOmllwYd\nl5qaiiuuuGJQUgUAl112GSZNmnTmuGnTpsHrHb4lI6qNE8ePH0dZWRni4+Nxyy23YMmSs9uY+P1+\n+P3+Aa95PB7k5gY3vdsapjEwB4lPMMYYeIQFbYi9MCwCO8/Y7yK6iPw+M6vnJILtbfP5fOjv7x/w\nWlpaGtLS0kbrrY0KkYzD6/Vi8j85PeTm5sLnC7//9ujRo3jnnXdw//33D3vssMlTWVnZoDfzSTl5\n7969QWfMl1xyCXbv3o3U1FR8/PHH+MY3voHs7GxcddVVQx6/adMmbNiwYcBreXl52LVrV1B6Eybl\nIGfSxKCOjTQsb7vJUz+F5E+dT9Fmedtd9JnpSPRwLrYsb7tZs2bB4+GY5LKwRJNcGgJ3YUmctm0R\nXMwVFRU4efLkgNdWrlyJVatWjcr7auvpR2dv//AHBkmSCVyzQonjXPnHnj17IvbeAKCurg533nkn\n1qxZc6YSdS6GTZ6eey4y2+5TUv7huTZlyhTMnz8fb7zxxlmTpxUrVqC0tHTAa6HcMBISEzE+keN5\nxvK2S0xOxvhEzi4slrdd6vjxSIrj3FRZ3nbp6ekUXSbWSE0kNGa3Y4Ocnr958+YhKzZOI5Q4hss/\n8vLy4PV6kZGRASBQ1bryyitDfk+NjY345je/iW9/+9tYsGBBUP9N1O609fX1Z7K5lpYWvPLKK7j7\n7rvPerwTy5FK9GElTkp0MdaIs7CQaNsh0SQ32PMcbMvKWCeScSxYsABPP/001q1bh2PHjmH//v14\n6KGHznq8tXZQ73VzczO++c1vYvny5Vi6dGnQ2iNOnv7whz9g/fr18Pv92LVrF37xi1/giSeewEUX\nXYRHH30U2dnZuPnmm7F9+3Y89dRTiI+PR19fH0pLS3H99dePVF5RFAFYgR4WEpcqrYUOBlWC5tZb\nb8Xq1atRUFAAj8eD+++/H8nJyQAwIP8wxuC6665Db28v2tracO211+Kmm27CypUr8Ytf/ALHjx/H\n008/jf/7v/9DTEwMvva1rw1a+fpXRpw8LVq0CIsWLRryZ3fdddeZf1dUVKCiomKkcoqiENFRBdHD\nGF7/D3NUAW9OHkVWZJ9XpEhKSsIjjzwy5M/+Of+IjY3Fiy++OORx99xzD+65556QtWU91iiKMiJ4\ng0F5VRj1tosegfMsa1SBxOVZN+Da5Kn+4+M4/C7H84zlbXf8yN9x+OB+ijbL227/22/h6OH3Kdos\nb7vXXvsLjh8/RtGmVp4oytT5r+IqTwGrEmmVJ17MSvi41tuuu7MD/e0cz7PGulqKbkd7GyxpB3td\nLcfPz9/aSquG1JD8/JqbmvH/2zv3mCivbo0/MAoiyAhWYdSG9rMnsdUY4ke91WuVAbwO3kBtbKut\n6QVpbBNL1Cb1EqP+4RcLmjTGRBONkfazKEVa+QxUiz1V69c0VM+JnkZtnQEtXvDCiPru88e0c8oZ\nkIEBHt53r19iwjDbWXuxZ95Z79p7rScuLp5im3c2Q7/ME1MMmeqzZpmnYFsVCF0Ly2aeuPvIvLtz\n3W5VFfH2nCeSy2sYSfOZ2KqAK9uhl8/M6zZXDFmiJ7Nh2eCJq23HsduzWxivOocokqubVAkUiBkJ\nogyPbhkJHbMwGkrSSNxkTiwbPCmDd9iSl3ki7p0Tb9t4ZyRYWRhDv2wbUW6JV21naJd50rHCkHam\nTggJ6wZP1AoG1gXP4IkhE6uwWGLIvMozg3j+h+QzsecRNwujV+aJVXnG3C6UzJM5seyB8YSkS319\naQAAE2RJREFUvyGmO+diy9K2+48XhtK2c1jadsl//zu6R0RQbLO07caOHYuef5E70gINGwnq2DyR\n5bPSUJBYCA3LBk89esYgmqTzxtK2i+nFk7NhadvZe8dR7AI8bbv4PhzBa4DdJJNiWsuDxMytJEYQ\no2GtjRAilt22EwSh/eFt22nYMJKo58drkskpADGUAZtm28JCaEjwJAhC0PAyT/plYXSUpFHgrLNB\nTD1J5smcWDZ4spO27ATBynAPyeuWhdGvVQGzi7xuWT4hNCwbPAmCYB20PDz9h1iJ0PEoYuNZwZxY\nNni6/N/n8Nv/cDTPWNp2Vf/+AVdJmmcsbbtTJytR7XFTbLO07Y4d+xdqa2sptlkwpUpYKGLmSTdE\nX05oLZa9Gt2/Wwdv/T2KbZa23Z3bt+D11lNss7Ttbt6oRcODBxTbLG2736//jkcPH1Jss9Ay80Qs\n29cNrpyXYEYsGzxxO4xz8FUk6XUFMJRBa5LJwjB09Vm39zanYSSzCSsLZhNWwZxY9t3CqtqgouXd\nuX7rLFkYPWD5zBQkZsGUhRHMiXWDJ8U8bMlsJKhXpzdmZ2Ba2T6YJezSqqDz7BK7bbP+1hSrPsu6\naScKoWHh4MlAuI3lHulDaBBTz6yLLbWEnegzrayap9uoYxNDSubJ4OmCsv7UVJ8l42VKLNsM6W/P\nD4OtG8c9lrbdsJQRiIyKothmaduNGTcB0b16UWyztO2caenoFcuR4mFlngyDd56P2TCSZVm3zBM3\nm0sxK4SIZYOn6Fg7zTZL284eF0+xC/C07eKfeopiF+Bp2/Xr149iF+BunbEOyevXMFK/zBNzW1gy\nT+bEstt2giC0P7TMkzIoX6zMsn2WZd/hab0yT8wD45J5MicSPAmCEDTMg+qsw9Os5pzMzBOrLQSv\nxIcXMErmyZxYdttOEIT2h5aJYZXtM89aUaz+0aqAVfRCsfpn1S7LNslwC9x58Aj3Gh632+s9Mrqo\no21EMk+CIAQNTxiYU0nqa56oVxaGKsJMseoLnnSs5hTajmWDp//69ylc9/xGsc3StjtdeRw3fr9O\nsc3Stisv+xp1t29TbLO07Q4fOoQHJEkaJrr1PGKim86bNMkUWotlg6e7t2/hUUMDxXbtNZLOWy1P\n8+waS+ft2jUYRvullluDhyRIXF3tITar1MsuiM12qVtYrMPTHLNQULxeddbazdIGywZPinhWgdck\nk6j/xdrOIWrb8baw9GuSSf1ba7aFxdoiBaRJpmAerBs8Ee8kWCiixAALpjwLC18yhLOFpRu6aifq\nhlL6bVUKoWHZ4AnEEmMWvhJj3XyGdmroiqQAr+P5H0PxsnwsFIgyTzR4hQGCObHsJ0QZGmaeiM3t\nWCjF3J7lwCqf1zF40tFnHQ9PGwavVYFgTizb5+n5lNHo0bMnxTZL227UhJfRy86RpWFp201Jn4bI\nHj0otlnadrPnzOGU7RuGdhkJZeiYzdUwaw9eY1DBnFg2eIol6ryxtO3in+pLsQvwtO36JSRQ7AI8\nbbv+JLs6ZmGYlWcsdPTZl3nSzGkhJPS6vRAEoc3oGDz5zjzpdZn0VXNq5jOxGapgTvT6hAiC0GZ0\nDJ509Vm3JAxTnkUwJxI8CYIQFMxAgteck1d3wmsYSRSAplj1wXtvU8wKISLBkyAIQcE8MM5tSKpb\nw0hiE1aKVa4AtGaJTctg2eDpx8py3L19k2KbpW33zdEjqL9/j2KbpW1XUnQQjx93vjyLYRgoOsjx\n+bPPCil2qbIdtNtzomwHxSq3MaiWPkvmyZRYNni6feN3GIZBsV17rYZj9/o12gfxWg1Hz6/a46Zc\n9AzDwPUazjpXezg6gr7mnDaKbV7mScFm0yvzpJQBm27yLErBxpLhkcyTKbFs8KQMnqAn6/7Jp+dH\nMU27fWKdw1HKIAqJcv7WzK0Nqs+aCQMbxAbDNJ8VcZ0l82RKLBs8gVolw/uCoTX0I94+cYIn/URy\nmQfGtfSZYpW7PUu7ihALAyTzZE4sGzwpMBu9ke6SiXdPzMwTy65uWRgdzzwxryO8U17E9zbFKvvz\nTDErhIh1gyfDQBit0RvpQ2gQBT11y0gwK5KoPutVbceUZ+FVnin9qu2oWVWKWSFELBs8DR+fih7R\n0RTbLG27l6fOgK0bR3GHpW03PXMOxW73iAhMnTGTYnvO3HkUuwrQ7krva56ol8/cIw8cdJSkEULD\nstp2vYk6byxtu74kuwBP2y6RZNdmsyEhkfP31lHbTsutSo5ZTbftIAfGhVZh2cyTIAjti2EYvLJ9\nVqsCpWCzkdozUKz+sW2n21YlUdtOMl7mRIInQRCCQhHL9ln4WhXohWHot4Wl5fasEBIhB0/r1q1D\nRkYGXC4XFi5ciKqqqmbHbt++HampqXA6ndixY0eopgVB6ER8TTL1ut+itv8goaBohQEsFDHzJLQd\nr9eLFStWwOl0YurUqaioqGhyXE1NDRYvXoyUlBTMnTs34Pnz58/jlVdewbRp0zB9+nScOHGiRdsh\nn3maMGECVq9eDZvNhoqKCqxYsQJlZWUB486cOYOjR4+ipKQESinMmzcPI0aMQEpKSqhTEAShEzCY\nKrkklKHf4WmD2WyXhKF4DWCFtrNr1y7ExMTg6NGjuHz5MhYtWoSysjJERUU1GhcdHY3c3Fzcu3cP\n+fn5jZ6rr6/H8uXLsXXrVgwbNgyGYeDOnTst2g759mLChAn+MwHJycmoaUay4siRI3C5XIiIiEBk\nZCRcLhdKS0tDNd8s/1n2JR49fNhhr/8kWNp2Xx/6J8Xuw4YGlJUcptg+fPAzit07dXX419GvKLaZ\n2na63Z37sjB6+QwFDTNP0G+dLUBpaSmys7MBAElJSRg6dCiOHz8eMC4mJgYpKSkBQRUAfPnll0hJ\nScGwYcMAAOHh4bDb7S3abtdqu71792LixIlNPud2uzFy5Ej/Y4fDgTNnzjT7WnV1dairq2v0O5vN\nBofDgZ7dWz7AGfbwAXp2t8EWxNj2RjV4Edmt8y8+jx/UI4JwoFeFKTx64EU3whdrQ309RZMqTBl4\n4PVSLrj19+9T7pJt4eGIiYmhZJ9i7XaAoKtn69Yd0TG9AFvnFybH9o4DbN073W63Hj0QFdML6BbZ\n6bZj4/oAEYFfcB1N9/BwhMXagR6d397G3qcvwqJ6Nft82B9z8ng8ASLosbGxiI2N7ZB5BfM925bX\na08/3G53o+pjh8MBTyu1Py9evAibzYZly5bh+vXrGDJkCFauXNnifMJUCzXAs2fPDpjMnyXLJ0+e\n9F/ES0pKUFBQgH379iE+Pj7gdd566y1kZmYiLS0NgC9iLC4ubvbsU35+PgoKChr9bvjw4di/f/8T\nHRIEQRAEq7FgwQKcPXu20e9ycnKwfPly0oxaj9frxfjx43H79u1Gv2/OjyfFH5WVlUhJScGxY8cQ\nFxcHAFi7di2SkpLw2muvNWn/1KlT2LJlCz7//HP/7zZs2IDy8nIUFhaiT58+2LhxI+7evYuNGzc+\n0ZcWb6cOHmx5C6qsrAzbtm3Dnj17mgycAF9vGrfb7X/s8XjgcDTfJ+fVV19FZmZmo99VV1djwYIF\n2Lp16xP/r1nxeDxYtGgR9u3bZzn/rOwbIP6ZHfHPvFjZN8Dn3/vvv48PPvgAiYmJjZ7rqKxTR9HQ\n0NBkTNGcHy3FHwMGDIDb7fYHTx6PB6NGjWrVnPr3749Ro0ahT58+AIDp06dj9erVLf6/kPd4ysvL\nsWnTJuzateuJb9z09HQUFRWhoaEBXq8XRUVFyMjIaHZ8bGwsBg4c2OhfYmIizp49G5DyswqPHz/G\n1atXLemflX0DxD+zI/6ZFyv7Bvj8O3v2LBITEwO+E80WPDX1vR6KH2lpaThw4AAA4NKlS6iqqsK4\nceOaHa+UCmi4m5GRgZ9++gn37t0DAJw4cQKDBw9u0XbIG/mrVq1CREQEcnNz/em03bt3w263Y82a\nNZg8eTImTZqEESNGIDU1FdOmTQMAuFwuqbQTBEEQBKFNLF26FHl5eXA6nbDZbFi/fj169uwJAPjk\nk0+QkJCArKwsGIaBSZMm4eHDh7hz5w4mTpyIuXPnIicnBw6HA2+88Qays7MRHh6OgQMHYv369S3a\nDjl4+u6775p9bsOGDY0e5+TkICcnJ1STgiAIgiBoTlRUFLZt29bkc7m5uf6fw8PD8c033zT7OrNm\nzcKsWbNaZVuvelRBEARBEIQQsX388ccfsyfRGiIjIzFy5EhERnZ+GW1nYGX/rOwbIP6ZHfHPvFjZ\nN8D6/pmRFlsVCIIgCIIgCP+HbNsJgiAIgiC0AgmeBEEQBEEQWoEET4IgCIIgCK2gywdP69atQ0ZG\nBlwuFxYuXIiqqqpmx27fvh2pqalwOp3Nyr50NQ4fPoyZM2diyJAh2LdvX7PjTp06heTkZGRmZsLl\nciErK6sTZ9k2gvUNAAoLC+F0OuF0OgNaXHRVvF4vVqxYAafTialTp6KioqLJcWZau0uXLiE7Oxvp\n6enIzs7GlStXAsYYhoG1a9ciNTUVaWlp+OwzjjhzWwjGv4KCAowZMwaZmZnIzMwMqudLV2Dz5s2Y\nPHkyBg8ejIsXLzY5xsxrF4x/Zl27W7duYdmyZcjIyMCsWbOQm5uLmzdvBowL9pojdAKqi1NRUaEe\nPXqklFKqvLxcTZkypclxp0+fVjNnzlQPHjxQXq9XzZgxQ50+fbozp9omLly4oC5evKg+/PBDtXfv\n3mbHff/992rOnDmdOLPQCda3X3/9VY0fP17dvHlTKaXUkiVLVFFRUWdNs80UFBSoNWvWKKWUunTp\nknrppZfU/fv3A8aZae0WL16siouLlVJKHTp0SC1evDhgzBdffKGWLl2qlFKqtrZWjR8/Xl29erVT\n59lWgvEvPz9fbd68ubOnFjI//PCDqq6uVi+//LK6cOFCk2PMvHbB+GfWtbt165Y6deqU//HmzZvV\nqlWrAsYFe80ROp4un3maMGECbDafGnNycjJqamqaHHfkyBG4XC5EREQgMjISLpcLpaWlnTnVNvHc\nc89h0KBBfoHlJ6FMVhgZrG9ff/01UlNT0bt3bwDA/PnzTbF2paWlyM7OBgAkJSVh6NChOH78eJNj\nzbB2N27cwPnz5/0qANOnT8e5c+cC7oBLS0sxf/58AEB8fDymTJmCr776qtPn21qC9Q8wx3r9f4YP\nH46EhIQnzt2sawcE5x9gzrWz2+148cUX/Y+Tk5MDBHGB1l1zhI6lywdPf2Xv3r2YOHFik8+53W70\n79/f/9jhcDT55jMzly9fxuzZs5GVlYWioiL2dNoNj8djyrVrzXvODGvn8XiQkJDgD3bDw8PRr18/\nVFdXNxpn1s9asP4Bvi+pWbNmYenSpfjxxx87e6odhlnXrjWYfe2UUti/fz8mT54c8JwO62cWQpZn\nCZXZs2cHLL76QyPv5MmT/gtdSUkJSkpKWjw709UI1r+WGDJkCCoqKhATE4PffvsNr7/+OhISEjB6\n9OiOmHZQtJdvXZUn+VdZWRn063TFtROaZ8GCBXj77bdhs9lw8uRJvPPOOygtLYXdbmdPTWgBK6zd\nunXrEB0djUWLFgU8Z/ZrqpWgB08HDx5scUxZWRm2bduGPXv2ID4+vskx/fv3h9vt9j/2eDxwOBzt\nNs+2Eox/wRAdHe3/eeDAgZgyZQrOnj1L/QJuL98cDgeuXr3qf2yWtRswYADcbjfi4uIA+OY9atSo\ngHFdce2awuFwoKamxh8gGoaBa9euITExsdG4Pz9rQ4cOBeDze8CAAYwpt4pg/evTp4//5zFjxiAx\nMREXLlywhJC5WdcuWMy+dps3b8aVK1fw6aefNvn8n+vX0jVH6Hi6/LZdeXk5Nm3ahF27dj3xCzU9\nPR1FRUVoaGiA1+tFUVERMjIyOnGmHcv169f9P9+6dQvffvstnn/+eeKM2g+n04ljx47h5s2bMAwD\nhYWFSE9PZ0+rRdLS0nDgwAEAviquqqoqjBs3LmCcWdYuPj4egwcPRnFxMQCguLgYL7zwgv9C/Sfp\n6ekoLCyEUgo3btzAsWPH4HQ6GVNuFcH699dzlefPn4fb7cazzz7bqXPtKMy6dsFi5rX7xz/+gXPn\nzmHHjh3o1q3pvEaw1xyh4+ny8iyjR49GREQE4uPj/XeMu3fvht1ux5o1azB58mRMmjQJgK9M9dCh\nQwAAl8uFd999lzn1oCgpKcGWLVtQV1eHiIgIREVFYdeuXRg0aBA++eQTJCQkICsrC/v27cP+/fvR\nvXt3PHr0CJmZmViyZAl7+k8kWN8AX6uCnTt3IiwsDGPHjsVHH33U5VPU9fX1yMvLw/nz52Gz2bBy\n5Ur/e9Gsa/fLL78gLy8PdXV1sNvt2LJlC5KSkrBs2TK89957GDJkCAzDwLp161BZWYmwsDC8+eab\nmDdvHnvqQRGMf3l5efj5558RHh6OiIgI5ObmmuILasOGDSgrK0NtbS169+6NuLg4FBcXW2btgvHP\nrGt38eJFzJgxA88884xfv+7pp59Gfn4+XC4Xdu7cib59+z7xmiN0Ll0+eBIEQRAEQehKdPltO0EQ\nBEEQhK6EBE+CIAiCIAitQIInQRAEQRCEViDBkyAIgiAIQiuQ4EkQBEEQBKEVSPAkCIIgCILQCiR4\nEgRBEARBaAX/C6r7iNk12uGzAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "plot_contour(xi, yi, predict(est))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XD_fMAUtSCSa" - }, - "source": [ - "It's not a very good fit. Next let's try to fit a GBDT model to it and try to understand how the model fits the function." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": { - "height": 523 - }, - "colab_type": "code", - "id": "-dHlKFlFgHDQ", - "outputId": "3d916f4f-834b-4567-addd-c101c96d4c1e" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk8AAAH6CAYAAAAeKKg1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd8U+X+B/DPyW7apnsvKAUKZdSyh+wyq4KIoKDiwgXX\nq/dexZ+iDL1XUeEqXFRcoOBAZVVBZI+yN7JXoXTRmabNPCfn90clkCZtkzbJadrv+/Xqi/Y5zznn\nm5Im3zznOd+H4XmeByGEEEIIcYhI6AAIIYQQQryJROgAhBIZGYnCwkKhwyBuFBERgYKCAqHDIIQQ\n0swwLfWyHcMwQodAPKCFPr0JIYS4EV22I4QQQghxAiVPhBBCCCFOoOSJEEIIIcQJlDwRQgghhDiB\nkqd6yOVyaLVavP/++5a2zz77DOXl5U5POuc4Dlu3bnV1iF6jS5cueOutt7B7927k5uZCr9cjJycH\nK1euRGpqqt19RowYgY8//hhHjx5FSUkJtFotzp49iwULFiAsLMzDj4AQQgihu+3qNXToUGzatAkZ\nGRn4/fffAQAXL17E6dOnMXbsWKfOyXEcduzYgaFDhzodb3Owd+9e9OzZE0eOHMGBAwdQWVmJ1NRU\njBgxAizLYsKECVi/fr2lv0wmg06ng8FgwK5du3DixAmIxWIMGTIEXbt2RWFhIfr3748rV67Ues4W\n+vQmhBDiRi22zpOjhgwZAo7jsHv3bgBAfHw8EhMTsWjRIoEj8z4rVqzA5MmTcfXqVav2SZMmYeXK\nlfj888/x22+/geM4ANXJ5uuvv44lS5agoqLCap8lS5bgmWeewYIFC5xOYgkhhJDGoMt2Nfj6+iIx\nMRGJiYlo06YN0tPTcfbsWURERCAxMRETJ04Ez/PIzs629JPL5XUe89FHHwXHceB5HoMGDQLHcZav\nWbNmAahOyjiOw5dffomkpCT88MMPKCgoAMuyuPvuuy3HCgwMxL///W+cPn0aVVVVKCsrw+bNmzFs\n2LBazz9p0iRs3brVctnr9OnT+L//+z9IpVKbvv3798f69etx/fp16HQ65OXlYe/evZY4G2PJkiU2\niRMA/PDDD7h48SJCQkLQuXNnSzvHcXj33XdtEicAmDt3LgBg0KBBjY6LEEIIcQaNPNUwfvx4fP31\n11ZtPM/j4sWLVj+vWbPG8v3gwYMtI1P2HDt2DLNnz8bs2bORnZ2NZcuWWbbt2LHDqm9SUhIOHDiA\n8+fPY8WKFfDx8bEkD3Fxcdi5cyfi4+Oxe/dubNy4Eb6+vpZLitOmTcNXX31ldbwvvvgCjz/+OHJy\ncvDLL7+gvLwcvXv3xrx58zBkyBCkp6dbLm2NGDECv/76K9RqNdavX4/c3FwEBwejQ4cOeO655zBv\n3jzLcePj43H16lVkZ2ejTZs2jv+Ca2EymQAALMs61N9oNDrVnxBCCHEZvoUCYPcrNjaWHzduHD9u\n3Dj+gw8+4FmW5V977TVLm0aj4Tdv3mz5edy4cXxwcHCtx7vzi+M4fuvWrXa3xcfH8xzH8SzL8nPn\nzrXbZ/v27bzJZOIfeOABq3Z/f3/+6NGjfGVlJR8aGmppf+yxx3iO4/hVq1bxUqnUap9Zs2bxLMvy\n06dPt7T9/PPPPMuyfEpKis25g4KCbOJlWZa/dOmSQ4+9rq+ePXvyHMfx2dnZDu/zyiuv8BzH8d9+\n+22d/QghhBBXa7HvLo68Qb///vu8TqfjFQoFD4BPSkriOY7jn3766QYlCY4kT7m5ubxEIrHZ3rlz\nZ57jOP6HH36wu/8999zDsyzLP/PMM5a2o0eP8nq9nvf397fpzzAMf/PmTX7fvn2Wtp9++olnWZZv\n06ZNvY9FLBbzbdu25Vu1atWoxCkgIIA/f/48z7Isf//99zu0T/fu3fnKykq+rKys3vMTQgghrkaX\n7eowZMgQHDp0CHq9HkD1/Bqe57Fr1y63nfPEiRN2L0X16dMHABAQEIA333zTZnt4eDgYhkGHDh0A\nAAqFAl26dEFRURFeeuklm/4Mw8BgMFj6A8DKlSsxbtw4HDx4ED/++CO2b9+OrKws5OXl2ezPcZzV\npcyG8PHxQWZmJtq0aYP33nsPq1evrneftm3bIjMzExKJBA8++CCys7MbFQMhhBDiLEqe7jBgwADL\nBGSRSIQuXbrg8OHDlmRl9OjR4DgOkyZNsswTujVx2VUKCgrstoeEhAAA0tPTkZ6ebrcPz/Pw9fUF\nAAQFBYFhGISFhdlNtu7c55a1a9ciIyMD//jHP/D4449j2rRpYBgGR44cwWuvvebSGlU+Pj7YsGED\n+vbtiw8//BCvv/56vfskJSVh+/btCAwMxMSJE7FhwwaXxUMIIYQ4TNiBL+HAziWeN998k2dZttav\nW3OSbn2ZTCaXX7b78ssv7W5//vnneY7j+BdeeMGhcymVSp7jOP7QoUMNupymUCj4gQMH8u+//z5f\nVVXFa7Vavn379o26RHfry9fXl9+5cyfPsiz/zjvvOLRPcnIyn5uby2u1Wn7MmDEOn4sQQghxNSpV\ncIe5c+dCIpFAIpFg4cKF0Ov1kMvlkEgkSElJAQA888wzlj72bvWvi9lshlgsblBs+/fvBwCrsgV1\nuVWSICUlBQEBAU6fT6/XY+fOnfjXv/6Ff//735DJZBg1apTTx6nJ398fmzdvRr9+/fD22287NOLU\nqVMn7NixA4GBgbj//vvx22+/NToOQgghpKEoearF4MGDsX//fkvBxsGDB4PneezcubPBxywpKUFc\nXFyD9j169Ch2796N+++/H1OnTrXbJyUlBaGhoZafFyxYALlcjq+//hoqlcqmf0BAgNWyKIMHD7Zb\nsyoyMhJAdUJ2i1gsRrt27dC6dWuHH0NAQAC2bt2Knj174s0338Ts2bPr3adr167Yvn07fH19ce+9\n91qqvBNCCCFCoeVZ7AgICEBxcTHmzJmDt99+G0B1Icf+/fsjNja2wedcuXIlJk6ciN9++w1HjhwB\ny7LYtWsX9uzZY6mbtGzZMjz55JN294+OjsbWrVvRtm1bnDx5EgcOHEB5eTliY2PRpUsXpKSkoE+f\nPjh06JBln48//hjPP/88ysrKsGnTJly/fh3BwcFo3bo1BgwYgK+++govvPACgOoErVWrVtixYwey\ns7NhNBrRrVs3DBkyBFevXkVaWpql5lRD6jxt27YNAwcOxKVLl7By5Uq7fdasWYNTp04BqP5/uHz5\nMgIDA7F161ZkZWXZ3WfhwoXQaDR2t7XQpzchhBB3EvaqoXBQxzyZe++9l2dZlu/fv7+lLTc3t96a\nQvV9hYaG8t9++y2fl5fHG41GnmVZftasWZY5TyzL8l988UW9c5leffVV/uDBg7xareYrKyv5S5cu\n8evXr+efeOIJS1mFO79GjRrFr1u3js/Pz+d1Oh2fm5vL79u3j589ezbftm1bS7/x48fzK1as4M+d\nO8er1Wq+rKyMP3HiBD9nzhybWlYNqfN05cqVOueUsSzLP/LIIzbnqO8rLi6O5jwRQgjxGBp5Is1a\nC316E0IIcSOa80QIIYQQ4gRKngghhBBCnEDJEyGEEEKIE7wqeaqoqMCiRYssd3wR0lQ09+cmPT7v\n1pwfX3N+bEDzf3zeyuuSp8WLF9OTiDQ5zf25SY/PuzXnx9ecHxvQ/B+ft3LJ2nbl5eV45ZVXkJOT\nA5lMhoSEBMyZMwdBQUFW/fR6PV577TWcPn0aEokEr7zyimUtOUIIIYQQb+CSkSeGYfD0009j48aN\nWLduHWJjY/HBBx/Y9Pvyyy/h5+eHP/74A5988gneeOMN6HQ6V4TgtIiICEHOSzyH/o8JIYS4g0uS\np4CAAPTo0cPyc2pqKvLz8236bdy4EZMmTQIAJCQkoFOnTti1a5crQnBaQUEBeJ5vUl85OTlo164d\ncnJyBI+lOTy2goICQZ5bhBBCmjeXXLa7E8/z+P777zFs2DCbbXl5eYiOjrb8HBUVZTfJAqqv89a8\nxltQUIC0tLQGL67b1InFYsTExDTLx9ecHxtAj8/b0ePzXs35sQHVjy8tLc3uh0GVSmV33VLifi6v\nMD5nzhwUFRVh8eLFNtu6deuGLVu2WOZCzZkzBwkJCXYXul20aJHNMdLS0vD999+7MlxCCCGkyXvo\noYdw9OhRq7bp06djxowZLj+XrrQcPsGBLj9uc+LSkaf33nsP169fx2effWZ3e3R0NPLy8izJU35+\nPnr37m2372OPPYZx48ZZtd36ZGGsKAF4swsjd9xX36zEhKnPNGjfi0f3QCSSILVLJwDAshXfYdwj\nT0HEeNVNj6SZ+v3nlRiVPhR+/n519qvtb6BcXYZD2zfh/nsz3BUiIW6n1+tx38RHsHH1DxDVGM2q\n6/X/p2Wf4YlHJzvc31EihkGQnwILFiwAx3FW29w16uQTHIgVwyZBk+u6qQ/+MZGYsuUHlx1PaC5L\nnhYuXIgzZ85g6dKlkEjsH3bEiBH48ccfMXfuXGRnZ+PPP//EggUL7PatcziSNwNmzv42N6tQq2Fu\n4GCdXqeDWCyxxF6hVsNs5gERrb9GhKfRVMDMsfX+bdX2N8CyHCo1GsH+NglxBZ5jkZubC97MATWW\nQK3r9b9CrbZ57jfm/aKmqKgolxzHUZrcAqiv53r0nN7EJUMely5dwtKlS3Hz5k1MnDgRY8eOtQwl\njh07FkVFRQCAJ598Emq1GsOHD8dzzz2HefPmQalUuiIEQgghhBCPcMnIU1JSEs6ePWt329q1ay3f\n+/j44KOPPnLFKQkhhBBCBEGTbQghhBBCnEDJEyGEEEKIEyh5csLFy1cQGRHe4P0D4trh4qXLlp85\nzgyGYerYgxBCCCFNDSVPTvhl3a8YOm5Sg/ePjYvH9Ru5VrebUvJECCGEeBeXVxhvLj5ashQ164f2\n79MLMpm8Ucft26sHjhw/iZ7d7mrUcQghhBAiDEqeasHzPB7/279cf2BVOAxVJQBo1IkQQgjxRnTZ\njhBCCCHECZQ8EUIIIYQ4gS7b1YLl3LfExMk/z4BhGJSr1W47ByGEEELcg5InO44cOwF/v7oXR22o\nTp06AwBYnsf4R592yzkIIYQ0XM2bhQipiZKnGi5dvoptu/bg6ZdmuuX4UpkMd6V1c8uxCSGENA7D\nMOBByROpG815qmHtbxvx5Iuv0J1whBDSAjEMQyNPpF6UPNUgEYshFouFDoMQQogQ6IMzcQAlT4QQ\nQgghTqDkiRBCCCHECZQ8EUIIIX+5Nd/V3rzX2krYFF86CYWPwqad5k41X3S3HSGEEPIXmVSKyRMf\ngEhkO7YQHxuDrWu+R2CAytJWUHgTpWXlePbJxzwZJhEYJU+EEELIHd7418t228fdMxpnzl2waouP\njUVSm9aeCIs0IZQ8EUIIIQ6QSqXo2jnF4f5U8qb5ojlPhBBCCCFOoOSJEEIIIcQJlDwRQgghhDiB\nkidCCCGEECfQhHFCCCGENDnZ2dmYOXMmysvLERgYiPnz5yM+Pt6qz+rVq7Fs2TKIRCKYzWZMmDAB\njzzyiGX7hg0b8MknnwConsC/bNkyBAcHNzq2Fp08nTp9Ftt27rZqu3jlikDREOId6ir8R0UBCbmt\ntqKaxDFvvfUWpkyZgoyMDKxfvx6zZs3C8uXLrfqMGDEC999/PwBAq9UiIyMDvXr1Qrt27XDq1Cks\nWbIE33zzDYKDg1FZWQmZTOaS2Lwyefr0y68hl0isFnCUSiRI7dIJHZPb2y1udiez2Yxvf/gJDMPg\n0RdepttJCXFCbX8vEokULMt6OBpCmq62bVrjxp8HEdupp9CheJ3S0lKcPXsWY8aMAQBkZGRg3rx5\nKCsrQ1BQkKWfr6+v5XutVguWZS2vUcuXL8cTTzxhGWny8/NzWXxemTw99NQLUJerrdoMRgMuH92H\nzdt21psMmUwmjEwfivguvdwZJiEtikwmg9FoFDoMQpqMkelDsfizL/AUJU8W+fn54GqMyKlUKqhU\nKpt+ERERlvdzkUiE8PBwFBQUWCVPALBt2zYsWLAAOTk5ePnll9G2bVsAwOXLlxEbG4spU6ZAq9Ui\nPT0dzz33nEseh1cmTyJGhIDAQJv28JFj0WekAAERQiAWiWA202U7Qm6Ry2QQi8RCh9GkTJ48Gbm5\nuVZt06dPx4wZMxp8zCFDhmDIkCEoKCjA888/j4EDB6JVq1ZgWRYXLlzAsmXLYDAY8NRTTyE6Ohr3\n3XdfYx+GdyZPhBBCCHGfjmFK6Iyuu8zlE6YEAKxcudLuyFNNUVFRKCwsBM/zYBgGZrMZN2/eRGRk\nZK3niIyMROfOnbFjxw5MnToVMTExGDFiBCQSCSQSCYYOHYpTp065JHmiUgWEEEII8YioqCjExsZa\nfdlLnoKDg5GcnIzMzEwAQGZmJjp27Ghzye7KHTd5lZaW4sCBA2jXrh2A6nlSWVlZAKqn6+zbtw/t\n27d3yeOgkSdCCCGENDmzZ8/GzJkzsWTJEgQEBGD+/PkAgGnTpuHFF19ESkoKfvzxR2RlZUEqlYLn\neTzyyCPo27cvAGDMmDH4888/MXr0aIjFYvTv3x8TJkxwSWyUPBFCAAABKhWKiouhUvnX2Y/KERDi\nODNvFjoEr5WYmIhVq1bZtC9dutTy/WuvvVbr/gzDYObMmZg5c6bLY6PLdoQQAEDPYRnYtHV7nX04\njoNYTC8bhDiCPmg0X/QqSAgBUD1ps0qrq7OP3mCAXK7wUESEeLeS0jKEuKCaNWl6KHkihFhIJXVf\nyTebzTTyRIiDWJaFTCYVOgziBvQqSAghhBDiBEqeCCGEEEKcQMkTIYQQQogTKHkihBBCCHECJU+E\nEEIIIU6g5IkQ4jCRSASOs1/0T6vVwseHyhgQcoufny+yr+UIHQZxA6owTghxmI9CAb1eb3db/vnj\naNsm0cMREdJ0+fn6onePbtiy+jvcNWB4nX1lcjn8/euu7k+aDkqeCCEOE4lEMJvtjzxdyc7GwH59\nPRwRIU3b4AH9sXn7TuzdtK7OftdybmDMyHS0TqW/IW9AyRMhxCkMw9htN5lYyOVyD0dDSNOXPnhg\nvX3OnDuPG7l5aO2BeEjj0ZwnQogFZ+bqXY+rtu1GoxG15FWEkHowjAgVGo3QYRAHUfJECLHoltoV\nu7L21bq9tsRJp9OhoPAmgoOC3BUaIc1acrsklJaVY8+GX6DT6Sxftc0xJMKiy3aEEIvUgSOx6D+z\n0S6pDaIiI2y2G00myGQyq7aKa2fx+fIVeO7JqZ4JkpBmiGEYTHv8UWzftQdrV3xhafdXqfD4088J\nGBmxx2XJ03vvvYc//vgDubm5+PXXX5GUlGTTZ/Hixfjuu+8QEVH9opyWloZZs2a5KgRCSCMxDIOJ\nTz6HbVs3YPLEB2y2sywLqdT6ZePntZl445WXIZPSAqiENNbgAf0xeED/2w0isXDBkFq5LHlKT0/H\n1KlT8fDDD9fZb+zYsXjllVdcdVpCiIuJnXyxZhiGEidCSIvisuQpLS0NQO1zIm6pbzshhBBCSFPm\n8TlPGzduxN69exEaGooZM2YgNTXVbr+KigpUVFRYtYnFYkRFRXkiTEIIIaTJyM/PB8dxVm0qlQoq\nlUqgiFo2jyZPDz30EJ577jmIxWLs3bsXzz//PDZu3IiAgACbvsuXL8fixYut2mJiYrBt2zZPhUsI\nIYQ0CZMnT0Zubq5V2/Tp0zFjxgyBImrZPJo8hYSEWL7v27cvIiMjcfHiRXTv3t2m72OPPYZx48ZZ\ntYnFNHGOEEJIy7Ny5Uq7I09EGB5NngoLCy132p09exZ5eXlo3dp+PVUajiREGCKRCFqdzu42o9EI\nkYjKwxHiaTRlpWlxWfL09ttvY/PmzSgpKcHUqVMRFBSEzMxMTJs2DS+++CJSUlKwcOFCnD59GiKR\nCDKZDO+//77VaBQhRHjBISEoV1cgNz8fMTVesH9asx6jR6QLFBkhhDQNDO+Ft7+VaHQwe1/YhHgN\ng8GAj955C2Ghtz/c8DwPmUyGKc/93arvVx/Nx0vTn/V0iIS0DCIxZAFhHj/tzgGjocvNd9nxfGKi\nMHDXBpcdT2g0/k4IsWEyGWHmzVZtDMOgsqrSptwIZ7buRwghzR0tz0IIscKyLD5f+B7+9eILNmvV\n7T1wEFvXfIdh90+2tNFiwK5XVFyMX1etREWlFgwDmM08Jo0egsjOvYUOjRACSp4IITWc3b8dQwbe\nbXeR3769euK9hR9j2F8/67Ra+CqVng2wBfjg3f/glaceQkRI9f+BVqfHvE+/xTxKnghpEuiyHSHE\nCmfm4OfrW+t2uUxu+Z7nebr7zg1iIkItiRMAKH0UCAuyrYdHCBEGjTwRQqyEBAVh647d6No5xaa2\nWklpGUrKyiw/S6QSXLx8BcUlpQgNCfZ0qB7B8zyOb9+As1eue+R8LMtCqVDY3XazqAjhYZ6fPExa\nnrDOYTBFuu7GLGkze97S3XakSTEYDOB5Hoq/3jwKC/JxYP8+xMTEoluPnjb9T504jj27d9m0d+rc\nBXcPHNTo/sePHcW+rD1WbRKJBAMGDUb75A4OPirvk3/2CH5asx7+fn5W7UajEZOf/Rv8/PwtbRUV\nFfj2k/9ibMZodExu5+lQ3UatrsCWdatw8doN9OiUjLSObT127tCgADA1JpPp9AYsXP4TFHIZeB4w\n82ZEhAQjOjwEDG73lckk6D40A1JarLl5EOhuuzMvPAZT0U2XHU8aFo6O/1vusuMJjZIn0qSUFBdj\nzS8/wWgwgOU4hIdHIDG1F1SBgfBXNY3LFkaDAQGMEcF2apQd2LcXR48cBsuymPbcC5DL5XaO0Pxw\nHIfF787BmzP/KXQoTtu+7kecPH/FauI7y5nhp1RgeL8eSIqPES64OvA8j8KSMhQUl1q1V2n12HX4\nBLqltMeQsRMFio64DCVPTRJdtiNNQq7GWP2NXIVRDz8pbDD1kMnl0EF+O+Y7xHbqjthO3XF+/w5c\nvXwJyR1TBIjQ88RiMVT+/vV3bIKOn7uEfz0xSegwnMYwDCJDgxEZanu5tF9aJ7z7+XcYMlaAwAhp\nASh5IsQNfKMScPnyuRaTPHkrs9kMEdM8J7yndkjCR/P/YzWiFqTyxyPPTBcuKEKaCUqeiOBOnTiO\n4MSOQofhUgGBQbhcXi50GKQeOr0evkr7k7O93cj+PTGyv/U8wQ+/XiVQNIQ0L83zIxfxKvYmcHs7\nhY8S/foPEDoMQqxQQVNCXINGnghxA6lMhpjWrYUOgxBCiBvQyBMhhBBCiBMoeSIuZTQasDdrt9Xi\nsRzH2e1bWanBoQP7oamo8FR4gvl22VfYtOE3XLp4sdbfh7fT6nRCh+AUjuOw7rvliIsMFzoUj+HM\nZqxb+TWMJpPQoRDi1ajOE3GpY0cOI/fGDZw5/SeGDR+BfVl70Kp1Isbce59N37zcXFy+dBGtUns3\n+yU+dNoq3MzPR0XuFRw7egRPP/s8QptRxd1DW3+FTqvDPaNHCB2KQ/7cvQm//LELE0cNRqe2Levy\n6p8Xr2LTnkNW85/MZh5xUeEY/+hTwgVG7KM6T00SJU/E5XI1RpiMRlw4tButu/aCskaV6pZOr9Pi\nly8W45GpTyAqOlrocBpt3YovoPL3w31jRtlUxW6Kvvl0EZQ+Cjw4clCzT9qdsXD5T5j85DN2F4Qm\nAqLkqUmiVw7iUrcKR0plMqT0G0qJkx0KHyVeePHvCAhoGhXTG6u0tAxjM0Z7ReIEAOWaSkwaPYQS\npxraxMWgtJTKaxDiCLrbjhAB+PpSUkkIId6KPnoRQgghhDiBRp4IIc2GXq+HiWWt2mpO6zSarLcT\nQoizKHkiDaLRaGAy3l4Y12w2I3PdGqQ/+JjXzH0RUq7GiBh/GcrKSvHDyhVW23izGYltkjByTIZA\n0dWttLQE33/+Pyh9fMByHFrFxwodEnLz87Hqm68gFongp1TabL/zKdmlXaIHI/Mendsn4utlX2DU\ngN6QR7Wx20culyFApYKvr7LWv3NfZe3bCGku6G470iAbfl0PTYXGqq3/gAFAQIRAETUvvmwVApvg\nXU86rRb/e/9tvPrSDPj5+nr8/DzPY/7bc+GjkFm1B6n8MWHkIPgpfTweU3NiNLHIOnoKnNlsd7ve\nYIRaU4XKWmp68WYe+UUl+Ns/XkFAgMqdobYcdLddk0QjT8Qpt+6m6zpwpMCRNG9VEl8ECh2EHT8t\n+xR/e/ZpQRInADi8JRM9OidjeL/ugpy/uZNJJRjc665GHaNcU4mFH87HrLlvuygqQpoemjBOHHYr\ncSItF8dyCAsNEez8VTo9IkKb3ogcuS3Q3w8B/sIk14R4CiVPxCFGo6HZLitCCCGEOIOSJ+KQ7Vu2\nIPd6ttBhEEIIIYKj5Ik4xMzbn0BK3OfqlctQl1PFZ0IIaWooeSIOOXfmDKJi44QOo0W5WViIixfO\nCx2GFS+8OZcQQlyO7rYjtWJZFp9/ugQcyyKpbTtIpbL6dyIu4xMRj8Nbf0NsXDyUSiVUzWQtPEII\n8XaUPBG7bt1Zd+9jzwocScsVHBqGiIhI7Mvag+PHjmDOO+8KHRJ4CDvydPbyNaQmJwkaA6mf2Uwj\nlKR5o+SJkCaKYRikDh4FAEjr3kPgaIR3s6gIDMMgNIhG4Jq6QJUfysrLERTYFKuVEdJ4lDy1UIWF\nBcjNybFqKykphp+fP/r06y9QVKQ2Ca1aCR0CAICBe5bdKFerodFU2t1WodHg+J5tOHv5Gl5/dopb\nzk9cq33rOFw6tBs90u8ROhTixbKzszFz5kyUl5cjMDAQ8+fPR3x8vFWfJUuWYMOGDZBIJBCLxXjp\npZfQv7/1e9iBAwfw+OOP4/XXX8fkyZNdEhslT83Yru3boNVq7a6Rxpt5mEwmqzZZcBRi2yVTMcwm\n6NZaeLeYzWbs2bUTAwYN9mgc7lizTFNZiQ/+8290S2lnd7uvjwLD+/XAY2NHuPzcxD2UCgUqqrRC\nh0G83FtvvYUpU6YgIyMD69evx6xZs7B8ufUSL127dsWTTz4JuVyOc+fO4ZFHHkFWVhZksurXy6qq\nKnz44YcYMGCAS2Oj5KkZO3hgPx564Z/2kyG/EMR2Eq5SNHFezf9HjuPw67q1yLhvrMdicMfddt9/\n9RlefvwDkDo5AAAgAElEQVRBhAfTJZ7mQumjwJ6jpyANPwKm9AbKNZUoVVeA46pLnvS9qxMSewwU\nOErSlJWWluLs2bMYM2YMACAjIwPz5s1DWVkZgu5Y97Nfv36W75OTkwEAZWVliIioXmf13XffxVNP\nPYXt27e7ND5KnpoxXz8/oUMgbtSu5wCs+3qJR8/pjpEnvcFIiVMzk9w6DhWVVeDKboARidAmPhrd\n/NtBJpWA54Hfdu5H5va9GDN+EqRS+29DMdFRkEjoLaq5yc/Pt1mtQqVSQaVS2fSLiIiwvOaIRCKE\nh4ejoKDAKnm605o1axAXF2dJnHbu3AmNRoPhw4dT8kQIuYMbkhlPo9JRzQ/DMOjVpUOt2x+9bzjK\n1BpsP7DD7v+/2WzG1+cv49mJ9yKyc283Rko8bfLkycjNzbVqmz59OmbMmNGo4x48eBCLFi3C119/\nDQDQaDRYsGCB5WdXo+SJEC/mjpEgT2sGD4E0QFCAP+5Pr30eyl0d2+Lw6fPIoORJEKEp8eDUrrt6\nIQ4IBgCsXLnS7shTTVFRUSgsLATP82AYBmazGTdv3kRkZKRN32PHjuHVV1/FJ598goSEBADAhQsX\nUFxcjAkTJoDneZSVlWH79u1Qq9V4/vnnG/14KHlqxjqmpAgdAiGENIi77uwkwoqKinKoX3BwMJKT\nk5GZmYl7770XmZmZ6Nixo80lu5MnT+Lll1/GRx99ZJnzBADdunVDVlaW5efXXnsNnTp1orvtSP0G\nDh7arO+cy8/NwbXLF23ao2PjEZ9oW0jx+pVLyLbTP65VIlq3bW/TXpB7A2UlxZDKrCurBwaHIDQ8\nohGRu86T057x6PncMWGcCioSe/z9lFBrqoQOgwho9uzZmDlzJpYsWYKAgADMnz8fADBt2jS8+OKL\nSElJwdy5c2EwGPDWW29ZRqnmz5+Ptm3bujU2Sp6aMW9LnM6fPolDWbssP996o+7Y5S5062Nbe0oq\nlaFM7G/zCZXVi6AutK0ZVMUqYAqKtWkvMPug0k7/8jIDfCo1YNkaJR3kcgC2ydOpo4dwdP9e+KlU\n6HRXN0RGx0IVGOTWS2tFBgYxNVbNuXzpIv74fSN4sxm+fn54eMqjNglgQ7njsXjDpcfPP/0cpZU6\nj1xi5PnqOT9hAX7o2LM3GIZBVFgIWsXYXq5ozsKCAlBUSgtjt2SJiYlYtWqVTfvSpUst3//8888O\nHes///mPy+ICKHnyOgaDAV99/plVm8loRFBwMB6Z+oSlrakkTpoKNb799GMEBAZb2sKjopF+zzir\nfqcKK4HQRHS9L9HucU7ZSW4AX8Qm1T4p1aZ3QCB8Axy/qyswLAIIi4C8Rru6tnhiOiBtfAdoNWqU\n51/A+T9PoUJdhl79B6Ftx04On9dZNf+vFREJlmV12JI8fLzwQ6hUKgwfORoJrVu7LY6GyDmehehw\nYUtmHNy0ASey86zaOM4MtVYPgIfZzKNX+wQ82sGzv7vCcg2u3rgMAFizYwf0RhO6to5BWHJnAEBM\nRCiiwkK8IvlsCIZhYOZ5bFz1LeQyGfx9lUgbMhpisVjo0AgBw3vhMuklGh3M3hd2ozWVhMgZmatW\nIqLbEPj4+QsdiuA6RwhXOsJkNEIkFiM+0KdRx/n64/fx9xdcd6lw0Qfv4ukJGfBTNi6uhtj/+2/Y\neOQsOidEoW+HVlZJiJhhoFIqmlRiwnJmnLqWjyq9ETx45BarUVCuAWA96V5vZNG7fQKG3z9eoEhd\nh+PMOHvlGnieR0l5BfafOANfHwWCVNavJz27dGi+daNEYsgCwjx+2ptLZoFTl7rseOKAYIQ/P89l\nxxMajTwRt7rnwcm1jBoRT3LVZbvGfNZasfR/KFVXWLVFhYUIkjjN+feH6JwQhVfHD4FYJPL4+RtC\nIhbhrsQYh/puOXEBC//7P7z09xfcHJV7icUidGp7e8RvUM9UVGp1qNLprfotXrkGbzXX5Ik0SZQ8\nEbeixOm2U4WVgo4+uUJjRmJK1RV4eeqDLoym4XzkUozp0VHoMNxmWNd2OH29QOgw3MJP6WOTcAep\nvPvvingf7/jIRbB+7WqhQyAucOeEeEIIId6JkicvkXvjhtAhEBe4cPqUoOc/d/aMoOcnhJDmgJIn\nQlqQ7Vu3CB0CIYR4PZclT++99x6GDh2K5ORkXLp0yW4fs9mMOXPmID09HSNGjMBPP/3kqtOTJmjf\njq1Ch0BqaOzNtQ3d32A0Qm9oGneLaqq0kNeyGG1zwnJmFBSXorCkzPKlrTHRurkIUvmjqLhY6DBI\nC+KyV5D09HRMnToVDz/8cK191q9fj5ycHGzevBmlpaUYN24c+vXrh+joaFeFQZoInudx6fwZpHbo\nJXQo5A6NvfW+ofv/vPwLTBw1pFHndpXN69ZiQEobocNwu7G9O+H3tWut2i7mFWNCv65IHTpCoKjc\n4+7uXbB3Uybum/y40KGQFsJlyVNaWhqAuj+Zbty4EQ8+WH23TXBwMIYNG4bff/8dTzzxRK37kOqK\n0SEhwhYSdFRFeTmytv2Bgrwb6N73bqHDaXIMBj2uX72M+NbCvHmzLGtZwqBB+9dY0NNRV3LyMGFE\n07iV/GJeEUZ3c7y4qrdqExmKNpGhVm1mM4+5P/7R7JKnNnHRWLtlj9BhkBbEo2PXeXl5VqNMUVFR\nyM/Pt9u3oqICFRXWNWHEYrHDiwo2J5FRUeg9clz9HT2gtLgIIpEIgcG2ydyuzRtRmJeL8NQB6N5v\ntADRNX09HnwG8XbKFRj0euzZusnyc4cuqYiOS3D5+du2bYcL58+hfXLDkgeZTAqdTgcfH+dqMz0z\n40V8+MliPPVAhlV7gJ8SSh9Fg2JpKIZhmlTxS08SiRgE+Xm+rpa7MQwDkah5/5/m5+eDq/HhRaVS\nQaVSCRRRy9ZkL/wvX74cixcvtmqLiYnBtm3bBIrIfTQaDTQVFeDBQyKRICLCeg0rX18/lJuFny/y\n5ccfICwiCr3uHmS3flNQl7sR1EWAwLyMvd+dmeMgjv1rRXCex55tfyCpfUek9e7n0nMnde+Hqrzs\nBu/fu0d3HDx6DAP79XVqv/CwMDw4cjC2Hzhq1X71RgFef3ZKg+Nx1OmdW5B58DREIgbXi1r2emm+\nCjne++Bjq6rkPA8MSElEn1EZte9IBDV58mTk5uZatU2fPh0zZswQKKKWzaPJU3R0NPLy8tCpU/U6\nX/n5+YiJsV8x97HHHsO4cdajLc1xTaPLly7i51U/WEYCVP4qRKRbJ09NZVkWhcIHrQfdh5tCB9IM\nicRihETe/luQKYYj/899Lj+Pj9IXSXfd1eD9RYGR0F5pWLmDNj0HoU3PQVZtH7/vusU6eZ6H0cSC\nB39HG/D9N99CZ2QxI+NuSMR0g/FT6bbzEHmex0eZu6GU/4GuQ4YLEBWpz8qVK+2OPBFheDR5Gjly\nJFatWoX09HSUlZVh69atWLFihd2+LWE4MvdGDjLXrsFDz//TKjFsKskSEY5SFYCuI8cIHYbbOXvz\n3uY1q7H37FX4yKU2x2EYQCoW21y+6d+hNToltLzL/c5gGAZ/y7gb763ehqjUnggPdnwBbeIZLXHK\nSlPmsuTp7bffxubNm1FSUoKpU6ciKCgImZmZmDZtGl588UWkpKTgvvvuw4kTJzB8+HAwDIMXXngB\nsbGxrgrB6/j6+iHjkaeb5YgaaRyxWAJfv+a/5ISzU4+OXbmBmQ8MdU8wLZxIxGBgSiKu3sin5ImQ\nergseXrjjTfwxhtv2LQvXbrU8r1IJMLs2bNddUqvFxgUhCovGmVK690PrNBBEEIIIQKjCQDEYR26\npAodAiGEECI4Sp4IaWEOHdgvdAiEEOLVKHkipIU5fOig0CEQQohXo+SJkCaoSl2O7b//KnQYhBBC\n7KDkSUDHjhzGjWtXhQ6DNFEadcsu5kgIIU0VJU8CUqvV0Ot0QofhsMP7dgsdQouhVAVAXV4mdBiE\nEELsoOSJOOz8qZNCh9BitNS11wghxBtQ8kRIEyVi3PPn2b2n7fIchBBCHNdkFwYmhLhHD4GSJ71e\nj3J1hVWb0eRc2VWz2cn1XIjTdHoDqnR6u9t85DKIRPSZmxBKnghposY+/KjQIdhSF0Kp9HFqF57n\n8ev3y3H2yjW0irFe9Hp4v+4AAL3RCIPRZHd/TaUWh7dvwansfCRGhjQs7lriyikuR9bOgyiusk4W\nOJ6HgeVq2dPz7kwZb13QDfP1QWKIP9qkdcadF3lFIhEiA/1t1vhzROdWUfgp6yBOH7RfC6xSbwTP\n82AYxu7SOmYzj2Fd26H78FFOn7sxTCzrtpFaQuyh5ElAqXelodh7VmchHubr5y90CDYKbt5Eu6Q2\ntW43GI3Y89tqnLpwBUD1emlGE4thfbphXPrddvcxm814be77aB8dZne7UiFDl4QopKe2h0Ts/Buk\nwcTi9407ce5m9d2LdyYiUf5KdIkMRkRr64RQxAByibjJzj3jeR6FlTpcLdVgx7Z9VttMHI+bVTrw\nPA+RE/G3Cw3A2PuG4olhPRsV139+3oqE7n0Q5sH18S7n5KFNfLTHzkcIJU8CorXtiLcpLStHUGDt\nb4qf/vdDDO6VipcemwCxg4nOlRv56N0uAWN7d3JVmFbe+fwXDEuKwdM9k51KJpoyhmEQ6a9EpL/S\nZcf84+INrFu/FWPvG9aouF68ZwC+/Ppb/OsfM1wWW32u591ETMduHjtfS6Bq3xa8TuOy4zE+Te/D\nYGPQOCdxGK1tRwDUOeeFYYC0ju0cTpwAgOPMkEnFrgjNLl+ZBF2igptN4uQuSSEqmLjGzynzVcjA\nw7Nz03ied+o5R0hj0ciTG+n1ehw5dBCaCutJsp1TUxETE+v285vNZpQUFVp+Vvr62b0UVF5agvKy\nUpiMRpSVFKO8tARmM4ek5BQkJXd0a4w8z0NTWoyi3GswGQ0AgLZ39YJYbPvU/HPvdhj1OpiMBugq\nNTAZquepDJ30JERi2zffrT98CbPZbNM+dOIT9ffn+epMgOdrPf6Rrb8hMCwCwZExkPtUjwBIpDJI\n5QqXXu75+ZsvIRKL0WfgEETFxjf6eHuzdqNvP/uX0OojFonAcTT+SAhp2Sh5ckJJcTF279yO3Bs3\nrN5Me/fpi7u6dbfpf/TIISgUCogDw63aq0Q+yHXz5brS4iKsWvY5EtokQSSqjrVN+w52k6GK8jLc\nyL4CiVSKUsYXfq07QyQWo0yuxKnCSqfOW1FShOwzJ3DzRjbMHAeGYTDogUchlSts+u74aTlMRgNU\nwaEIjYnH9arqODXX1ejRunpi8OFrtwtFVoiCIFKFQyyVQaH0g5+s+phHb1TYHBsAAvrcb7fdFf15\nnoc2MBGBjBaXjh+CUV9d7JQ1GTHg/ik2/c1mM7Z+/wV4nofSX4VOfYcgMCzC7vluufW7Txo2HgZt\nFY4fysIf61dDIpFCoVRi/JTH69y/NufPnEGHjikICgp2et+oyAjkFRQiPMz+/CRCCGkJKHlygkZT\ngZgOqegx/D6bbfaSoYQuwtwSbjIZsWH1j+gx4WnIFLcnwuoA+8mQbyQCOlXfBeVbz7H3b1yNjr0G\nQBUcarMtK3MVRCIGVUGJUPXsApFECgA4UaD76+zW/Hrea/m+DEBwwO1tdyZNt6iiEuqJznMYhoFv\naCTKADCqRMj/apcDOHLd/rIqQf0nAAD0FaU4e3AXNGWl8AsMRt+MCTZ9NWUlyL96Ee3SekMslkDp\nHwBl39G4c7zy1v9l5wg/p2K/a/BIbNrwGyZNfsSp/QAgML4dcs4eR2pn+/OTGlJKwJ2XeHieh9bJ\ncggtFQMGJjsjtQ3Bca45jqPEYhHyzx1DSodkj56XtFyUPNXjzqRIGhoLb7ifQyqVoet9U11+XDPH\nQVNWggsaMaCxTW7kXdIBAJ67x8Y7KVTBQKdhuDXuYy9RBEQoO3cK7dJ613u8U4WVTiVQoeGRKC0t\ndbj/naRSGUxs7ckIyzl/e3+VVg9fuaxB8dypQqvH7KU/I8LP+s654W3df4m8OUgI8sPq09kuORYP\nQKvTQ+ljO+LsDkN63YWfNu3E++/MhVwmvR3HX1ff4yLDEd+pOyIiwhEaHGT3MjwASCWSJnuHJWla\nKHlqRpy9xOasSnUZ/AOdv9RD3K+0uAjBoY5dSjNzHKSShv3pG41GyO54c7rTtSO7kRjn/MeL4nMn\nEejnXO0oe/77bSb+1rcTAn0an4jVVKLVY//+cy4/LgCIwaBfvw7wd0EC2RgihkG3mBBs2bgTw0YN\nbNSxJt19F+a+918E+Skhl0oQqvJFsJ/ScmNA96HDEejv3IhpXUQiESaOGmx3G8eZkVNwE1dzL2H/\niYMoLleD521HO3kA5RWVeO3N2ZRAkXpR8lSHQwf2o4ITI7lzV6FDqZNer4PJYIC7/zuNei3kSl80\nndKB5JaNq1dh8rQXHOprNptr/eRd/74cJHYm8wPVtXaSWzs/od1gYqGQ2k/InCEXi9ySOAHAqm0n\n0NpXAWUDf291KTWasG77SUwZaTtv0tP6J0Ri2dGLaHixgmoxIQGY8/BIAIDeaEKJRosSTRVYzoz8\n0gr8+vMvmPL4Y40P2AFisQitYiJtCrTa88n366DV6eCrdF0JCNI8UfJUB5Zl4Q2rQRTm5uL61csI\n7NxP6FCIFxCJxRgyLN0tx26uH9gZAMl+Siglrk+ebhqMOF7u3lFjh7nh/08hkyImJAAxIdWTGnMD\n1dh3Ltv1J3IBpgFV2UnLRMkTcRhrMkHuo4RW6EBIo4jFYsQn1l4l3NNMHAdxI9+09EZTg6qPO3Rs\nlkOuzgCZm9Z0U0kkyNcb8cHavbj1a+B5IEQuRahMCgZAuEKGHv3cWzaEEOI4Sp6IwyIT2uAGaM6T\np/QYdo/QIXiEWqtHoG/j5jwd2LEf7UMD6u/oJBNnxvx1e/FQXAQkbhqVUIhFeCzB+pISz/MoMbIo\n+Wu9v11F5TDvOY1e/VPcEoM1LxhuJ0RgVJKVkCYqMLz+ORrNgUZngJ+PvP6OdbhQrEZymOvv8/x0\nw0GMiw5DqLzxc7KcwTAMQuVStPdXor2/EpPjI7CpsGF3SDpDxDC4UqpBTrH9chuu4COX4uS1fOgM\nTW9pKplEAr3eIHQYxAvQyBMhzUBQiG3dLW/B82j00imsmYfMDZftTGYzIhTC3gUHVP9+/Nww38re\neV4fnIpfN+1BUaUO0hq/0y6RwRiVYf+uNkcF+ynx7Mi+eGf+f9EuJhyyWh5XxgMPwE/Z+LswnZHa\nIQl/7tmCgffa1l4j5E6UPNWhe89eyK80CR0GAODqxfPIy7mGnncPglRq/WLu6+cHhY9nX2RI0zJ6\n/ESPnau2opYiEQPWw8URb7F367kraFlhHo+QfGVSTOySaHfbT6euYNOGHRgxelCjzhEdrMIbD6Yj\np7jc7rNJXaXD99+swNPPPt2o8zirU9tELPluLRpXqIG0BJQ81UEqlUIsaRrX/7duWI+E3uk4U6SD\nSFxzuNsPiqQ0QeLyBM5kBG/mIJJIIbJzm3zO4R2oKMyxaY/rNhCqSNtb568f3g5N4Q07/QdBFRln\n0158+TT0FbcvmUh9fBGW1NlSQd0bbdrwG0aMHuP0fjKZDEaj/cstXe5Ox+qVy3BXhySnjukjk0Bn\nNEEubfjLkYE1Q+7CkRme57Eocz96BjedleAlDAMDy7n0cTrrgU6tsfTgOYxwwbEYhkF8WJD9jWFB\n2H7qkgvO4hyZVAIz3/ISZuI8Sp68QEV5OYJDwxDZyrk3JVfSayvBuOM+5r9UFuVBczMXABDSOhky\npT80hTeQvf8PiGVyiKVyRHbohoCY1pZ9jl8pqf4muHP1Vw1XtABu9blTcJfqLwf7mzRimE2330Rb\n+zI4u+kHJPYbBZ9A77xcdvlyw96YfJRKVGltl9oBgOCgIJSpNU4f008hh0ZnaNSkcSPHNfiy3fG9\nZ7HlZhnunA9ebmIxIiIYbf2aTr2fWB8Fsss0aO+GuV2OYhiGppMTAkqevELxzQJERMcIGkPB1Usw\nGQ1AcDu3HF8i98G1cjOMZflg9VrEdRuI64e3g+04Btxfo01XDbCfDLmZVGVduTsPADrEwicwxK3n\nvXTiMPwDgxGRYP8SSmM0tIKyVCoFW8fyLLIGFLuUSyUwsY0vvdrQx3RcXYkxkSEI8fCkcGcFSiXI\n/vMa2g8WdgEkqoRECCVPXsOdoz6OB+G+GBSqICiikiCSKQBGffuUtVSzbgmMei1Yk+uWsPAEby2S\n6Q1xMwyoigAhTQSVKvACIeERaN3WPSM+TVnSwHuFDsFrlBTdhNlMczUIIcQTKHmqw6ED+3Hu1Amh\nw0BAYBCiYp1fM8wbiZUBCPxrXpPcz/VFD5urrb+tA1fH5TRCCCGu03KviTjAW9a2c7eK0mKc2LMF\nvUaOg7r+7rXSlZfAzJrAsSbw5uo5LqxBh8DYNhD/VX5B7OMP/wj3ziVypdJrF+AbHA65v+vnofgF\nBOHwlkyYDHq0Skmts69UKoVer4NU5lhNohGjRjcoJpPJBEkti+Pq9XoUFDtfyLFKb4RC5vhL0U+r\nN+NKaYVVW0Gl/UnsdSms1GHj7j9xQaPFoFBh5xE5IlAqQWZ+MU6v2wuguj5WmFyKELnU6rK+hGEw\neEAnmxpNrvTuF7/YnUjAAxicGIVeQ/o2+hz5ZRqUVVQiSOXZS9eaKuefS6TlYXh3FUhxoxKNDmYP\nhL0vaw8qeQk6dm0aZQBOFTq3eChrMkEskdhMpNVXVWL32u9sJnoolH64e+xDOHytzKqdMxnBMIzD\nt+bzPG938m7O4R3gWBNEEglEIjHAMBBJZCiUJ4Dx0tv+OW0F/PIPgzXqrdpj0+5GQFSrxh/fZAQj\nEqNnYt139TG552A0GpDW27HFoWP8G1b48fzBnTAYDLi7b2+r9vyCQny2+CO88PA4RIU5t4TPO+/9\nF/8cN6jeflqDER8sW4decWHoFRfu8PF5nsfBrDO4+FeCpeXMKDOaECqXomeQqkkUwWwInudRZDSh\n1Ghdi07HmXFSXQUJwyC4RlIqFTGIkMvQMS3JJrkKVMgavT4gz/NYtO8MxndqhZR+PRp1LHWVDot/\ny0JsaIBVIU25VIIJUybXmsQ31s5DJ3DqwhXERYXDZKoezWUYBgMz7kdggACj4SIxZAFh9fdzMf3W\nb8DrnL97tjaMjz8UQx912fGERslTHbw9edq/cTWMUZ2gDPLMH562rAgXt61GmwH34JKmccttNEep\niSHI3r8ZnFGPNgOcX7eue0ItNXEAmAx6XNy6GhMfn+bQsRqaPP3x8wr06dkdsTHRAIArh3Zi1e/b\nERkajIfGDIWvj8Kp47Echw8X/g9/v3dAnf1KKqrwwTfr8WT39gj1tX8OE2fG++v2QlEjAeB4oJVS\ngXb+PhAzDOQiEQIaUVPKW+g5M6pq3MVoNJtRaDDipsEE9o7XUJ4HsrV6vDSmJwIbuVSOiTPjg90n\n8e8XJkHcyMWUeZ5HcUWV1Tz5InUlfj96Dv/3yt8bdey6XMnJg5nnIRGLwTAMTCyLHzdsx9hh/dG+\nz1C3ndcuSp6apOb/CtKCVVWUw7+95xbyrSzKQ1VoR0qcanH8SgkQngbp2Q0uP7ZUrgBrcn81fL1B\nD4XidvJy7sp1TBw1GG0TYht0PBPLwUdW/6hjoboSveLCak2cAMDAcgiRSTEuxvNvNE2RQiyySSQB\nIKqW5GhPsRrFVfpGJ09SsQiRfkqwnLnRyRPDMAgLsL5sFx7ghy0nLjTquPVJjIu2aXv4nmE4duYC\n2vdx66mJl6DkqQ61LUPhacWFBahQlwMBTr5Bed+gImmEyJiGJTCEEEKcQ8lTHXr26oNcjfArbOfm\nXAPDMGCcvNxuNpvBNPKTnzN05UWQ+EZ47Hxeq4FJbcG1ywCAyIQ2drcPHXNfg0MihBDiOCpVUAeJ\nRAJJE5jIfDM/D2XSht0N1NCqyw1RWZQPaVCkx87nrRixGGbW+Uts50uMyLlw2uH+xw7sRWFerk27\nRl2OPbt2On1+Qggh1Sh58gLqslL4BdQ+Wbg2Mrlzk3cbKy5tABiGnlL1iUsbADPn/HIkyqAwqItv\n1rr9VGGl1Rcbnojf1/5k04/ngevXsp0+PyGEkGr0Tuc1nB9BGjj+ETfEUTtVVIJHz+et/CPiIGlI\nYutkYqpQ+tkdOfWGpUgIISQ7OxuTJk3CyJEjMWnSJFy/ft2mT1ZWFsaPH4/OnTtj/vz5VtuWLFmC\njIwMjB07FuPHj8eePXtcFhslT16AZVmI3FTThHiX0oJc7F77fb39Cq9dQc6FM2BEtpmS2WyGqIFz\n4cxm6xpevj4KlFc4V0LjTrzZ7FAyx3FmiOrpaDKbIbHzeIljJAxg5LxjiR8h7oXhOK7BfzekYd56\n6y1MmTIFv//+Ox5++GHMmjXLpk98fDzeeecdPPXUUzbbunbtil9++QVr167FO++8g5deeglGo9El\nsdGEcS/QrU9/6BuwWn1zYCzLh/7GefDmO5Ye4Xn4te8DsY9t5eHyY3+As6lNwiPwrhEQ+/jb9Nec\n3wezwbaisF+7XhArfG3a1Se3gedYyMMTIA9PgEjq/AjS8SslSE28XUX95vnjCGvXtd75aQzDoNW9\nz4EzGXH4WplN3afOEbd/H/pL5TCxRgycZDv6qNNqofS1fWyO0OsNUChu38ree8S9+PSjBejROblB\nxzMYTZA7UHMp9/R5qOR116bKr9AivJ4+pHbRPnL8eeIKOg7v1uhjuXt0U4jR0zK1BkEq29cQ4h6l\npaU4e/YsxowZAwDIyMjAvHnzUFZWhqCg2699cXFxAIAtW7bYHKNfv9tFg5OTq1+jysrKEBHR+Bub\nKHmqw8+rfkBKnyFQBQq7dEPbDilOF8j0JuoTW+ATnwJZUJTNNlZTgkpRHFAjedTmG8CI7KzlFtQT\nsDM9rKiQA1Bu027mYwCp7adtbZ4ejMh2Ujfv2wm8SQ+pqRzlhzdAogqBKmVg7Q+uFsevlFi+j1f4\n4B64tu0AACAASURBVPzmVUgePrHe/UQSqUOV3nv0q73opE5bBaVS6VigNRiMBsjvWAJGLpfD1Ig1\n9fRGE+SS+l+GNAYTwuqo8QQA505eRUwjaxS1ZNEKObJKGrMAkzV3FjIWYuSpVK1BRPuunj9xM5Of\nnw+uxpxPlUoFlUpl0y8iIsLyoVIkEiE8PBwFBQVWyZOj1qxZg7i4OJckTgAlT3ViTaYmU+upOWOr\n1ChV+wBq2+QGiIWoYYMkDhEpVPV3ugMjloERy6CBCoiKB5+3tdExXOdCITU5P5Rsb/TJEeFR0QiO\na9hdkbyZh7jGJWRZo6p183YvLdbEmc2Q1HPJxGTmIaPLKg0mETHgXJSVuPsuXyFGnjiOg8SBRL+5\nkLXuCLAuLNUjqf5gM3nyZOTmWt8FPH36dMyYMcN156rh4MGDWLRoEb7++muXHbPlPBNaIKPecwtc\nqvOzwRkNAOpeg625UXXsL3QIFof37Ub3PnfX28/Xzx+hDVyehRBCGmPlypV2R55qioqKQmFhoWWt\nVLPZjJs3byIy0rkPfseOHcOrr76KTz75BAkJrrupiT6mNWM7V6/w2LlYnRbGKtetg+QtpIHCFQXV\nlhWhvKjQ8vP5UycFi4UQQhwRFRWF2NhYqy97yVNwcDCSk5ORmZkJAMjMzETHjh3rvGRXc6nekydP\n4uWXX8ZHH31kmfPkKpQ8EeKlqorzUZKfI3QYhBDiFrNnz8aKFSswcuRIfPfdd5g7dy4AYNq0aTh9\nurpg8JEjRzBw4EAsW7YMq1atwqBBg5CVlQUAmDt3LgwGA9566y2MHTsW48aNw8WLF10Sm8su22Vn\nZ2PmzJkoLy9HYGAg5s+fj/j4eKs+ixcvxnfffWeZsJWWlmb31sOmory8HEqlGyfcOMCg1+PIvj3w\n79jb+Z09MKvy4vY1MGorweq1aDPgHsDJwSezyQiJXxDcv6Rt09Z28Din9xHLFDDotJafqyorsHLp\n/wAAQ0bfg6jY+Np2bZRbw+i3VGr1DT5WlU7v0MLAxVo9An3qvtRYZmKhklBJj8bgXPSSoZSKodEZ\nHPq/bQiz2fNzURNiInHhcBaS27X1+LlbqsTERKxatcqmfenSpZbvu3Xrhp077a+Y8PPPP7stNpcl\nT7fqMWRkZGD9+vWYNWsWli9fbtNv7NixeOWVV1x1Wrd6+tnnUaB1vhK0K+3Y9Bs6dO4K190D4zrH\nr5QACbfv6rpkJ3Ey3MxGxZk9EMl8ENB5MCT+wVbbRVIZdKo0d4fqNgXZ5Yhs1fi7MWVK27IL9fEL\ni0LJ+dtF3/pMvj3hshhAcWGlVfkCANjy61oMyxjb4DgZEWNVJ+rkzo3o0j6xwcfLPnoAMcH1T9pX\n600IUNSdPGlYDv6Nmrxe43gmFjuO3UCOmQUPwF1pGY/qErh3/nunW21tRFIM6h7n1knxYgYwchxk\njawr1yrIHyf3HsGwUc7fieoIlVKBsopKBKmc/7tpqJSkVvjlj13IqPHhgbRMLnmlcbQeA2B7TbIp\nq76rSJjkSaetwr6d25B7PRvRvUc4vb9Rr4NU5r7btu+81b7WGErzUHnxINiE0WAYEYpLAJTYu6PO\n+1VePARD0XXAzCKkf/0lB2qqWfvJETKlP0or6x7qq1nioiDvBniex6/r1iLjPueTKJlMBqPRhH/9\n4x9Yu7U6cXv35WlgOQ6SBrzhZheWYmRa3XMRzmQdQoiy7udycZUeSrHzSYXRbMa3h67CyPNWNfzN\nAHwYETqIpEiTyOst0OluHM/jMm/CskNXYear7wE2A5AyDEIYEZg7oleAwage8Q16g+8VrMKizAN4\nfkxP+DQiEW0TrMLCrFPQGLYgrV83xIUEQuTCAqZ3JcZg/x8bMeqBCS47piNG9O+BD/49D/I7RtS0\negNeeOmf8PfzXCJHhOeS5MmZegwbN27E3r17ERoaihkzZiA1NdXuMSsqKlBRUWHVJhaLERVlWwvI\nXXI1rqlE6gyOZfHDV59BLBaj96ChCEsb3KDjlBXmI7pNe1TU39VhlcX58FEF49QNx2pOiRS+EPsG\nNvv17gqyywFpWyC6LfzYS9Bmn4SyVRehw6oVwzANXtsuPDQUN4uLsfCjj7Dwr7YL+7Zi9uJleP6h\n+xAd7tzdlkXqSoQF+CGnuBxFavvPq1+OX8aMvimWn1mzGQbW+kPNl38cwYOx4U6de9eh6zjMGZAu\nViJU1LQv94kZBu0YGdqJrEffTDyPUt76d1HEc/j0wGU806uN00lfWz8lVBIJPv71AAKkEgTUSKA6\nqXzRuU/9E2+DlXK8PvguZJdqsGv7fuRVaC0jawCgUsjw+IMjoWjgZb0uraKx6NfdGNWgvRuuV5cO\n6NWlg1VbqboCC9/7D/7v/9l77zC5qivd+z2hcnVV59xSdytnoYyQQAglEEkCg0B4ZMBgGzAe7LE/\n5s7MBY/vvc/YM2Nfe7DngrEN2JgMBhsjokEiCqEckdRSS51jdXVVVzhhf3+0WupWV3WdWHH/nqcf\nxKm999pVdeqcddZe+10P/dBUKQOl+kiU5JBUqYJbbrkF3/rWt8BxHD7++GPcc889eOONN+D1eke1\nffLJJ/HII4+MOFZVVYX33nsvWdNNKsMjBLOv2wJAdfrQCMrG1+NMLLVIjciSiGPvvgTMvxmMwmUD\nzuEB0aBflMn0cxPAn3o9ac5T/Sx1S55Wqw2RcFizUE5pSTHaOzpRO67m3LHJF1+B781ajKce+xW+\n89Ub4vb94M+v4sNDJ2Hhz58/zrNLcY89vxWXjI+9BfmWORNgH5bL9PPXPoXrgtymBQV5o46NxfbP\nz+CkLGIT787oJRgLw6CMGXkZLwMPL8PhdzsacOfiCarHLLNbcXttBXqjAsLyMAFZAmxt78HAhwex\neNmM+AOcxc5zmFqaj6mlo5e1m/qC+N+Pv4Trp9di/gr1+Zw8x6bNKkah14PbN16J3/3qF7jr/u+a\nZicV+kiU+BjiPCnVYygqOr8ssXTpUpSXl+PYsWNYsGDBqDG3bNmCDRtGJtFeKM5HSR6yKMBZVI6Q\ninwLhmFQsPDqwchMjsAwDFirQ1NfWRQUqYcPZ+KcharaW6xWiKKg2WEQ7PlgIqMz8PLc7lFPxRdy\nrKUTt69aiKK80Zsw3FYLloxTFjliGGBjVYmyCcehg4iYy1oz2nEaixqWxy5JeyI/ABTEiAotL/ai\nNaz/gaja68L6KePQ5A9CazEYTsMyrVnUVVcgFDFQUDIGSvWRKMnBEOdpuB7DtddeG1ePob29/dxO\nu8OHD6OlpQV1dXUxx0x1ODIYDEASWXAmK8oeO3QA4XAIqJqWuHEKSZenvEygcNG1mvodfvNZzFg/\nuhZdOjHWeZDoFKFnEIWinWSmrFASY5hn8PDDD+PBBx/Er371K3i9XvzkJz8BMKjH8J3vfAczZszA\nz372Mxw8eBAsy8JqteLf//3fR0Sj0onf/voxXLflm6bbaTh2FPbaWShM3DSl+JpOwFtZi+Rplmcu\njMroUTK5fN3VsDu01bUbIl6wJlEQR5ZlcGlSPkUkg+VIKBQKRQuGOU9K9Bj+7d/+zShzptLn88Hh\ncJgedQKA9pYmXKRhN12y6fxyL8ITr1CtqppLS3aZQH7h4MPKNRp22gH6IpCiJINPk6WWKAisoM4T\nhULRBq1tF4Nt7/8Nsy5ZmRRbrEl5XP293TBSQL5q7jKcDCurh0aIDIDJ2nwSsyBExuGtz4C3OTDp\ncu1aTEqortEooNnfCdewaHFUELDr3dexbedeLJs/dpJ8IBxfNFGNS2aEkKOfyHBmufNkxjJpkdWC\nL3qNKcNUlufA60fP4MjjL416zReO4hs3rUVtafyYPCGjBVtTSVQQUz0FShKhzlMMTjeewkUrr0qK\nLcakC/gnr7+IwuU3qe538pM3Ubt4Nfae6r3glTxF/Ykso3v7H5G/4Gp0dao2nxVoFc4Upl0FYfju\nJgWc2LcTE2aP3nBhFk0trZg7exbu//a3MXFcJViWxYIZU/APd2yCNYEukCSTmMt2/aEw3FZll6L2\nQAhFCtvGIyRJ4Bkm5dpNZmPGu/NaePSL0tnvUp+FAocN31s+K+Zrx7v6sPOjL1C7YXXc/nkOG/zB\nAXjdqa0CMcTMSXV47reP4qbb704bh45iHtR5igHDJC9qQkx4PhQiYVhs6gUyg91tECOhGI6TcsIt\nX8JRMzNnHach2k754GGb4RyXeEv3EAzDAiqXtRr270qq89Tb64PL5cS0+nH49m0bFffzBwbgdcXe\nhdja40d5nrI8rE8+PYKpCtvG44MvmjCdVRZFzWTMStCfmufE0U4fppcZJ4VyIeUeJz490zFmm6VT\na/Hf//04RFlGTXE+ttx5u2nzUcLVKy7Gx7sP4KWnHseNW+5K6Vwo5pMeCQhpxqQpU5Jma9Md3zB8\nTEKIpuVASRBgd+srNUIkEb4eGr4GgFDTEfONpOIJlxDV28QlSYI1Th+ZQHEUSCYEvE7RVQlA9rtO\n5kSeAMDGshBNri3HMQwSmZhWU4YHrrsM91x5CQIhc2UClDJ9Yi2COmo9UjIH6jzFYPXa5OnW8mbt\nzNJwbRvU6EptLb+sQk6CE6kygfsvL/xRp7khLTf1J5gRt1uJ6PcXZRDTlsvTCbPcG44B2o80mTT6\nIAwGHWVF82EZdPqDps5HKTzHIRCie5JzAbpsl4XwVhuikTDUZgK4ispwZtcHgLoKG5Q4MJwFUjgI\nzp4eORkA4OvtAQD86aUXcP0N6uuCWa1WEEIQFQRV/ew2K8JR/c6kXxSRr7P4bxeRMSNNJBPMhGUY\nnAiEMDw1qdBqGVVyRS1T81z4/ek27H/141Gv+QUJt62YjYnFo6tGqMHGcwiLyh7k7FbL4BLerx7D\nt+65W5ddvbidDoiihA8++hg2ixXjx9WgorwspXOimAN1nrIQlmXhKVCvn8XyFsiSvhsc5/SAsaWv\nzlEyCefPB3PgfRQsWK+67+7jHagUmlA2bezyK2qlA4bat7a2qp4TAHg9efD396sNeA06Tyodrlj0\nCSLyVJRhGU5YkkFAECQyHFlebxEAlnF2HD7UPuLYUVnAdxdP1JXTaedY3FVXGfO144EBHNzTgImr\nLtI8PnA271RF+8tmToCV5/A/fvhj5LvP59YJooz777sbeS59eXJquOeW67DnyHEAwB9+81d8/5/+\nZ9JsU5IHdZ6ylMVXbsTORvWJ39PWbFJc+DcWtpJxYINU2wkAWIcXnOxGqPkoHFXq8ugYlkP7kS9Q\nPGEGOGv85P9pi5ZBEkXFmmQ2m32wtp1GBEGA1WpVvXQmSpIhAplq8qOG0xmJ4tFdp1DF8hjP5sZl\nr5DhUHhB7qOHYfHUjpPYsrjeFJulNisOpGgJ7eKptbh4au2IY+/s/RInm9owe4o57zcWDrsNF88d\n3CjyyZ6DiESjsFlzIcsut8iNq4hKmvuTV8w2UT2wZDN4o9buPKUSseMoII+ObnAlk8Fwqbl4Dbjm\nAALQp0G+oH7Zeux96VHYPAXgbXZMvuLGURGDdlslalSIuZaWV6CjrQWRcBhnTjfCYrGiXEXZB39/\nADarFVaLuuhiIBhCnkP9DtAL0Rov2bqnCTdaXPAyuV0fcxJrRZAQ/OGzBiyZMbL2aI3DDptOEVOP\nhUe/kD7XtAKXAx2H9wBJdJ6Gc8m8WfjszVdx6TXql8gp6U3OOk+EEDzx+GMIhUJghj0Rr1y1Gu4q\n9ZXItRDo9+P1F5/F9KtuTYo9Jexp6E71FEYgth+C1PklwJ6/WVvql4O1D9Y97Dy279xxJtQFkNE6\nSUUlsaM+0cOvx1x+sk67clA2IF57WQRfPgNc8URV70Wt/tPxfhvYxZshACgRm+A7cxwF4yapsnkh\nlePGo/n0KVx59TXYu3s3PF6vKudJlmX0+vpQ6FWm+zVER48PxZ7YuV/NB47AHUc8czj+cBT2BDd3\niZBRicYvft4IAmhynHqIhD/5/AgxJGkXywtPSQaADIAnDMaJLOYVuQEAxQyrafltLmdDgyzg2KHz\neiICCN6SRdyzRN05HQujEtVlQiDLBKwOPamJFcV46m87scqgOallwYwp+I/fPYdLr0nRBCimkbPO\n0+nGRixeegm84yanbA7vv/k6yuetSJq9vpZTECMhBDqbEexqO5f/MnnlRlgcxiQ1G1mORWzdDwDw\nW+tGbrE6cypme+KInene1XA4tgG+Jvbx4wfGbk8ICiIBRPa+AOusjWBY5TfltlM+lFRYEe06o2op\nr4Wvxtxx+utADnhrMHt8PZwuFy46+5AwFGmtylMWnev1+VCYr65o95c7PkZdWez5N/YGcElt4qTa\nV/62F8uK4ici9wkiHvmiAWUXOEnTWSvqWHWRMpEQ/N7ngwxgepSHg6R+d14UBGcsEt7p7YfEAD5W\nxje8BbBocKDqY3weYRLCJ5+fwcUL4/wuFMIygCjL4HUu004s8mLnB59i0eUXax7D63LAyvNo6+pB\neXHyK4hyHAuOZREVBNXRWkp6k/1Zk3EYX1ubUscJAHo6O1BYHjvxUi+x8p0i/T5EB/rRzpUjMnnN\noKL1tKtwsDWMPQ3daRd16g0Q9AYM2JtuNAyD3gEW/PglIMEu1d07WiIIHPtcdb9434+a3DaO5+F0\nxYkAKVyu7u8PwKMyAbe114+KgtjRqs5gCKUue8IxOiICKsZY+ntuVyM28W6s510j/tQ6TgDweF8v\n6gQO8yOWtHCcAMAKBhMEHtMEHjOjPGZHeDzpM+5h5RLOji8k/XpJpTYrOgL6t+vPKMvH4Q797+/m\n5XPx4jOj664mi7nTJuLg4SRovlGSSs46T8nMa4oHx5kT+Ovv7YYknH9/Q45Ri6UG7Y4JsBZUqIqW\nKCXYsMvwMdOZns5esHnqtyEzDAPWGlttO5VEwmEc2L9vzDYMw4D1Ncdc1hwLOU5pluHjJoJL0EQm\nBE6DdtExAPLl9L48eggLiTFOzYllGEOWIpQIXCqBZ1nFWk9j4bRZIaQwt9RqsUBSKLtAyRzS++pA\n0cTRLz5B2N8DQojqrex6CLedTJotykg+/evo4qpqiUYj2Ls7txxgCoVC0ULO5jylA4XFJaaO7ztz\nHKG+bsClL8mYkv70+3pSPQUKhULJGXIy8pTsiEw8rtx4U6qnYDDGfKZElkCiA4aMlW007dqW6ilk\nHTIh8BFpxF+mVGdM/VXMXNLgMk2hxCQnI0+vv/YqyibNQGXN+FRPxRQYlgWRZYBhBv+bYZBgF6Te\nRgCJk4gzlbwpSzT162ttRLVO2688/QQ2bP6azlGyh6d8PjAEGJ4FWCVnhh5UOmbSODgWwah+Nfl8\nhxU9BhT8tXBsSnOeKNlJTjpPdRMmoKO1EchS54njOMiSBJbjQXSWW0kF8kAPWGchEEnz6JMwADnk\nA+tQJ34JANaiKhMmpIyBYHoUUU0HBEIwwBAsjWSmAnShzOLFbh/mFJzf/egEi6oUqqh7LTxO7m/E\nlJXqfxfDsXIcRAMe/vSUoqFQ4pGTy3ZWqxXRaOp325mFy1sAzmoFITKQxBpernp99ayGkHsb0dOV\n/iVeGDkKqfNoUm2KkRDaDu1E26GdEKPny6yoiTCSOIs9Npsd02fMHLsvIbCU12MgpK7EiyjJusuz\nJFrC0bLC87TPh+nRzH2GnBzlYCUMDveEzv0909+neTwjVsk8PAe/aMxDm1Fuj2zE9j+N2G1WRNvp\nZppsI3OvGjpoOH4ceZV1SbG17e03cOZkw4hjBARXXHUtKqrHmWJz0txF6GvsRVfDIfD25G2Jt5dP\nAHSKZBIig4hhwJb+kQBi9UIOnNHUd7jauBQOgrU5FT0hC5OuwMmeTjAshzLLec2jpVcrL/8Qr+Sq\n1WbDRfMXjN2XYVBWUoLjuz5RbA8AQlEBDgUq4smkj8iIIv0lCcaCA4Nx4sglxg5ORoQQ2FIUcXHx\nHIJieqULMAwDSZLB6Sw/o4XifA/2Hm1I3DDNEIvGA7KBy50sh/S/qisnJ52nlpZmXLrgkqTYWr5q\nHQ50jF4m6QLQ1a68hlxfdwd2vPnqqOOewmIsXrcBwGixxOL66dh9osuwpzezkQMdEBo/BV81D8iA\nyBMYBpAEEFnSpJs1pMbuFo6BdxfCXpG4LBDvygfvyj9r/vw3685Xrp4cL/KklIJ8L3r9/ar6yERf\nmY2BqAiHwTe+5319mJPBUad4eGQGPURCBZOa95ZnYOTJZeXRGxhAgVudKOuF1JUVoqGpBZPG680Y\nVE9ZcSFatu9Iul2KuWTflUMBFoslaarVRq23e4tKUXDJjTFfG0thOh3X+4ksAbIIhj8fOTlXo85a\nmxmO01n4cYshNn4CS90yzWP4hWK4Ok8qcp6MIF7kSSmiJIHj1DmLek9DXzgCj8XYy5XIENjSRD3c\nSByEwa7uINaXxC9lEw8GZx1dHV+YhWUhGbRN7rK6Cvyf37yCYpcdVo7DN269Ck4NUek5tZXY8d67\nmHT7FkPmpQaXw47Gljb84bFfjjh+1WVLUDhlftLnQzGGnHSebttye1oojOciUm8jxNOfwzLpinPO\n0/DivplGT0cXPOHOxA3HgHHkQ2xVptM0MS8Cd4m+kj4bbtV+AyGEQJIk8CYo1I+FPyLAzRtnM0hk\n2LPQcQIAh8zAx2lbNvMwLHyCiMI0WWKtyXfjn1YO5lJ2BcN46NEX8e/3qy+kXltWiJc+Sd115qF7\nv4Zw5Pw9pz84gFfe2Y47qfOUseSk80RJHWSgF/1MEUjzGQDa8oXSDUv9cl39B6ODym7kjTvexYz1\nX9Vlz+l2a+47FMlMdkCTkMGyH2OhZkpREFiy1HliAWjNOOLBGBY1Mppilx3Fzvi1DRPBpyDfaQiX\nww6X47z0SoHHjZABMgyU1JG5mZKUtEIK9SPUnNydZ+kC6ypO9RQMwdfTjYYTx1M9DQqFQkl7qPNk\nMqKoXyxOK0SWk6akLkfDEHxtSbGVjfDuAl39P3z1Gd1zoM4ThUKhKCOnnKdwOAx/X19SS7M899vH\nDBnn4KcfqO5zeuffIPjaDbFPMRfv7JWK2smigL7mk4gGR+52CwWV79ykUCgUij5yynn6j3/73/jz\nq68kzXnq8/XC7fHoHocQgqZjh1X3iw70g7O7dNsfi55PX0H3xy/Bv/9vsJXVJ2zPlc8AsanfBZQJ\niM17IAc6TLVRt3Qd+juaIYQukL8w4Jy2WKyIhBPnYciyrHoXp97pnTnYCOcYOSsDogSrijl91hWA\nK72kiAzDLTPoY7V94AUMiwN79UeQzbrELqwuwS9//5qmvumUysUwDGSSpSdgjpBTzlNhUTFWbLgV\nrcHklCw5+eVR1E2aonuc7tYzKKpQr08SDfjB2rUnB49F2ykf2k75EC2/HELlFRCq16A3kBe3feex\nfeg8tg9dDYeBFJaOMIOh99brj0Jo2AY5ZJ7UwvF+G7q803DMf8FuKBWOw8tPPxHzeJ43H35/YnVq\nSZLBqlQL15tg3haOotwef4v6zt3NqFaha9TCy6iQMqN+nVqsYCAwBLIGb2EKa8UhWf9OZIaBKQ+p\n86qKEYgIEKXMdjzSUUKGoo6ccp6STVvLGfTZinSP03HmFHqsJeo7Mgz9kSYT3g6/rR5Cw/bk21Zx\noxqIs8TndLsQ6B9b/HLwhqhe8FLvfbQnwfb5TiKhlFHuDEUZkrVSBQBQLLFoI+rVoS0MY0iJFjfP\noT9iTr5nicuBTn/mL1OnUySMop7sCgEkINmOBCFE9RN6vHEYA8YxijYVJVjE9sPwdbQBdv1OZEbA\n8kmtJ3gOFed2PJFMhmEVlWfRgu6fHhlbioAAYFWIFWSv2zSIPrkC/TAwxgmLBctqczzS7Tky3eZD\nUUf63JGTQF1d4pwcI7FYrCl1eliVKtBKCLeeUJfXIwkAXdtPiODvHCzkrJHl192iew4cx2HBosW6\nx6FQKJRsJ6ecp3Xrr06qvVVXX4+8Av0Rl5pJ0+EqKlPdb9o69Uq8iZAG+kCEsOHjZhNcsfoyK/1H\nPgYk5cssshBF14mD5/7f7jInt41CoVAoo8kp5ylT8RSVwOqMn4xNSS/40qmm2yCSgN7Tx0y3Q6FQ\nKJTRUOeJQskx9NS2o1AoFAp1niiqUZupSbeUpBt6attRKBRjoLvtMhvqPFFUEe1pAessVNx+UBRT\nX+mRnIAQddtvTLjyiqKA3bu+SGCWIBAcGFHkVAl6NX8kQsbcIRcgBC4Vnx+9b1FSSa8/ALfLkepp\nUHSQM1IFnR0d4C08wCfvqTs0EEzcyERkyXgxUPfkxYj0xhcrvBCG5bNOFDMRncf2oXj8RIDlwfDK\nPis+r0ilxAEZIRuw/U9/xPLr9W0QiITDOLBvHy6aN3/Mdl1f7kV5sXIH2iixxLFkEkJEhjMVEhEU\nikJCkSj+47fPwuWwQxAlfGXtZameEkUHOXNX2/XFTuRV1GL8hIlJs/nSH36HOdd9Tfc4+z96F3LF\nHNXSA4e3PgNMu0q3/eFYvKVAr3Kdp85j+wy1nynI/a0gQhh8+QxF7T3Tl6san+GtKKybdu7/wwbU\nthMFARZL4ktCb58fddUViscdCEfgstn0TI1q4lAynhe2vo/NV6/CuHnqfuuU9CR3HtUIAaNSFVkv\n8cQI1dLZdBp0oSGz6G06aWpSA2uxoXD85PMHDPMuxh7nXPRHzRJZEn57akenvhgl2UiShLz6Wame\nBsUgcsd5olAoAOLXtqNQKBSKMqjzRDENMwvkUrQTCqY2F49CoVAyHeo8UUxDOPlRqqdAUYHNbseM\nmTNTPQ2KQWhdNCYwJsnf3K346gcnxLjNC1rgeQ5HPv0b/P5++P39KZ0LRT854zxVVFbC6XQl1abd\nkdqtqEbXtpPCQfQfpg6RIlgLGKvy71/o69Rlbvn1+mvb2R1OzJozV/c4lNRjIQyiGt2nPIZFn6C8\nVFAsbCyDiIpyQ2pwWy0IhCKq+9UU5+NUS7sJM1LGLeuvQCAYwstP/w7PPvEY/vj4f6dsLhT9ZFF9\nnAAAIABJREFU5IzzNHvuRSguK0+qzRu+eoch48xcejkYDduwDa9tR2RIYbrkowRiLwBXpLzGnf/g\nNlVPorIQRnfDoXP/b3emr/BlMp6wqXTrSKwEONAzoKlvGcNh394WXfYdHIdARNA1Rjzy7Vac3ndY\ndb9Fk8fhwze3mjAjZfAchysvXYyv37ge99xyPZraOiGKxsvJUJJDzjhPmUxpTS0Yln5V2QzvLoDY\n36W8A8OiI0UyEAzLgsiy4vahSBQOq7mqKHT33EjshEGY0eYiFjEceojy7zcWDo5FSDQn8mTlOAiS\n+vl5nXYEwlETZqQNjqPX9EwmZ3SeKJR0JkDKwbYch8VToqg9y1shC9puBHpr2/EcC1HFzavb50eh\n26nLppGQBGrl2YBTZjBg0eY8eRkWPhXOcSxsLIuISc6TIMvgNTgezd19qCr0mDAj5XT19qGzx4dw\nJAq30wGep7fgTMUw1/fUqVPYtGkT1q1bh02bNuH06dOj2siyjB/+8IdYvXo11q5dixdeeMEo85Q0\nhLWn9kKVSbCecgi9KpdKNGo76altRwgBx7KQVOSzBIIDcDv0iWQmQo2bIALgSHa7T1YAEY2RJwcY\nhHRGniwsg46jzbrGiIdMCDwTalX36+4fQPmM1Ob0/fSJF3CquQ1tXT245fa7UzqXTECvX2Gmz2GY\n8/TQQw/htttuw9atW3HrrbfiX/7lX0a1ee2113DmzBm8/fbbeOaZZ/DII4+gpUXf2jolfbHUUyVd\npTAMmxaFQj/f8dmYrzMMM2aZlLj9TI71ZLcrpB5GxyfOMPq/LbO/D60zTPV5UlzgweobbsXqG26F\n10sfLhOh168w0+cwxHnq6enB4cOHsX79egDA1VdfjUOHDqG3t3dEuzfeeAM33XQTAKCwsBCrVq3C\n1q3JSeD78uiRpNgZTrbVtmNtTuRNvdjQMbMWQiAP9CZuNwxrofKSJxfy8V9eQDQc1tx/iJ0JnCcK\nhUJJBkb4FWb6HIY4T62trSgrKzv3RMqyLEpLS9HW1jaiXUtLCyorK8/9f0VFBVpbW2OO6ff70dTU\nNOIvXlslvPPWm5r7auXlPzxhyDhfvPdXTf0Ob33GEPtDMCwHzpFn6JjZC4F4+lNVPfKmLlXVvmzq\nvHP/FiL6HScKhZK+tLa2jron+v3+VE9LNUrfhxF+hRqfQy1pm6325JNP4pFHHhlxrKqqCu+9916K\nZqQeYtCG6J7WJhQo3/VOSRdMXoYrnnC+6LBRcgCJxiGEqLaVBquRFErGs3nzZjQ3j8wju++++/Dt\nb3/bFHv9rBuyxry5WLAMgyIk/32YhSHOU0VFBdrb2wd3sTAMZFlGR0cHystH6ipVVlaipaUFM8+q\nGLe2tqKqqirmmFu2bMGGDRtGHOMMFn00GyPyPCRRBJtGOzLaTtGSK4pgWBCdSbdqEIUoeKtVUduX\n/vA73HDb7aOOy7KcMJ+JYRgIogSeV/dbNKxuMYWSozz99NOjNmp4PJmXN6X0fRjhV6jxOdRiyLJd\nYWEhpk6dij//+c8AgD//+c+YPn06CgoKRrRbt24dnn/+eRBC0NPTg3fffRdr1qyJOabH40F1dfWI\nv4oK7TkhqcCIyFMo4IfTnXlLZYQQyMHuVE8jxSQv5kJkGaxCLbB4uXj9/j54vd6x7RCCqCDAarGo\nniOFQtFORUXFqHtiJjpPSt+HEX6FGp9DLYaFNB5++GE8+OCD+NWvfgWv14uf/OQnAIC7774b3/nO\ndzBjxgxcd9112Lt3L9asWQOGYXDvvfeiurraqCmMiZYdQrptGhB5EqNR8FZzt3krgRAZUKP9IosQ\nm3YCbGY5vMaSxHNOxfkd77zkeR7z5i9IYIaBTAgV+KNQKKaj168w0+cwzHmqr6/H888/P+r4Y489\ndu7fLMvi4YcfNsqkKiZPnpJ0m6mvbWfccp8U7EOwYTfgmZe4MQUAwDoLVbWP9rTAWliZuGEMLtt4\nm6Z+w3G581BVUaR7HAqFQjECvX6FmT5Hzjw+XrFmbdJtGlHbzpVfgMnzlmjqO22d/mKxFO1YatXJ\nOvQf+URV+/bDu87922pPraNOoVAouUTOOE+ZisVqg7eoNNXToKQhXQ0HUz0FCoVCyUmo80RRBMNy\nIJKKKukMCyRxt1mmQ2QpacWfN2z+mua+g3XhGMiy8mR4nmMhmFTnbAg1qfmMyva5iN7Px8zPV6sq\nh3x211YqMft3QEke1HmiKIJ15EEa6FfcnmE5MI58E2eUXZBoAJxde805NThdLs19GYaB02HDgAo1\n86qyEjT39Gm2aTQ8w0AyUL8mHSEggMb6fUY4GQQAa5KfIhMCTsPgEUGEzZq6XaJHTp7GxHHGbJOn\npB7qPFEUwTAMWIsyHaEhLONpKRelkEgQbIrV29uam9B0ZnThzeEQQmAvr8dASLnzVFqYj86+1JYq\nyjVkaL+4S9C/k0gmxLS9phIhimU5hhMVRNgUaqGZwbuf7MLK676SMvsUY8kZ5ykna9uJKpbZFFCw\n6FpDx8t25GCX4rYMb4XFU6zZ1gcv/15z3yG62tvQcoHy74UwDAOO5yCpkK0YLCasd3YJbJjcPtcw\n5vMx71PWcj4RELBmhcMUEIlGkedOTnSZYj454zy9+/ZbSbdpRG07Qoj22nZvPqvbPkU7QqPyIrus\nqxj2iomqxh9e286IosBKGCzPoqWf8XMZMb7J7XMJ+tlQKInJGecpk+lpGzsaQMlNhte2Syak6zR4\nAzXEjIBGnkYiAdBazCoCArvOT0gkBCVTtGmWUSiZAHWe0pxU7w65kPJamgSe6bz89BO6+ocjUdht\nynNHIlEBVpW18Cj6iDIEVo0hJD+RkcfouzVEZQJbhtUipVDUQJ0nEzGitt3gQJkZSJcDHameAiUG\noaC+XDxBEmFR4QyFo1HYTd7llJm/EPOQGIDTuNsuZEDkSdK4I45CyRTSK/aeZRhR225woPS4CBFJ\nhCxEFLcXTn8OWGpMnBHFSEorKlHqHNspGoqEqo2Imn0Gp8cvJHsw4vNMt6g5hWIkORN5SkltO6cz\n6TaHY2RtOwAQ+3sQbNiVuCEFAMC6lNe2I0IIYqBXsy0jatuVVlSiqpo6u9kAgXYHSAIBq9N9kgkx\n7eYyJNSqFkGUwaVoKTEwEEJnjy8ltinmkDPOU0pq2912uyHjzFt5laZ+tLZdalGjcyUHuxBuO6Fq\nfFrbjhKPKAPNOU/9REb9jBJd9iMygd1izsJGWJTgsKofu72vH+VFBSbMKDE/f+ol/P33H0yJbYo5\n5IzzlMkUlVNVWspoaG07SjwEEFg0Ro/6iQyvTscnLMmwm7RJICxKsGqYXzgqwmG3mTCjxNhtVhQV\npsZxo5gDdZ4olBxDT207SmZAGGByobZopIzBEja67IOANSvniQCcBoXxVKZg0fSv7IM6TxR1SMoL\nW7Ju7YrZFPPQU9tOCyQNd4um34yMRc/7M+KzMbUwsMbRU3kapuFPgKIT6jxRFMN7ikGI8rIclnGL\nUTJptokzSl/Uvm+vM6SrPIsRtLc0J6xtp4WoIMJmUv6LVrI9EDBYn07buwwSGS6dS24RSYbDYs6y\nXSAqwuNIzfIbhTJEzjhPRw4fSvoTcDQSgRBJTtmMWEhC1NDxGJaFd/ZKKpQ5FkIQXrYHUre65G8w\nDKxF1ZrNGlHbrrOtNWFtOy0EQ2E4VYhqUvQjMgQ2jc6TDIDTuc4Uls3LefKHo/A47ar7cSwDSVL+\n8Gck6Rh9pegjZ5ynD7d9AFEwtlBuIo4e3Aeu/bjucT5/+zVN/Y689Zxu2xR1eIRm8JVzwRVNUNXP\nPWkhGJXSEqmobaeFaFSAhc+ZS01aMBh50oZRUTmzdJ6IxrHtFgvCUWMfKJUiU+cp68iZK1oqBNuM\nEsn0dbYbMg4lCVgcYB3JicylqradFqhgYuaQ7t+U1vml8hTkuZy51eYMOfONOp1O9Pv7kmvT7Yav\nt0f3OAzDQBaTGzVLRHltPgrcATgH9p/7K44jDZPteU8lk2ajePwEeITTKXMSiKx8OeKlP/wu5nGb\nFILDMfYOLS3LDzzHQdS5XBKW5DFtiwAEFXMTMLidP1sJsARODfXpBlTkNI6FIJvz2R5o70UwKmrq\n29M/AAuf/Nw7Qgi6ev1Jt0sxl/TK4jSRiy9Zhr17PkXhqmuSZrNu0hS885dXMKFwHDxFxbA73ZrG\nGT9tNphAI3z5E1X1Y1gWRBJVLwcphXcXgOUHc1kIkdH72asgddeAueCiHT3xPrzs0JwsgMUOhrOC\nK50KhrOg89g+U+aXTBirC7bpV5tuZ259UZxXlN2sCCEIDwzEfO3g/n34xr3fHrO/FuewpDAfnX0B\n1f2GM8frxs7efiws9MR8fRlnx3tSCGt5Zar+X/Xm4yn4sDRsAZf2sRZ19DMyCACPBufpAymEGy8a\nr8++IMJpUL5TWBDx7L4GEAL4I1HUFuTh4W/epHqcQ2faUVdepEkfSi9/3fYZrl25NOl2KeaSM5Gn\n2rp6nG5sTKpNhmGw+pqN4HtOY7xDxqwybc5T3YyLcOrQXtX9nAUlEPv1R77iwdldsOSXwpJfCmtB\nOTyzVsAZGD1PS+0lsFQvgKV6AbjyGeA8VWDsHuDsxT3bI1OJUJOAf+Tt50cdE8IDsDmUyQ9EQkE4\n4kgVXLricvAJnsyHoj9qIlB5Lgf6Q8prIsbiqpVzsGcMB2zuwmqEQRRHTrwMi1lRHtsd6RXRVYuP\nlbHdHsVnw/522UV8NV/90rFMCAYIQbFNXxHnD7v7sPGyWbrGGGLbqTYsqi7BPbddjX+660ZsuflK\nsBoKDr+1+whuum2zIXNSy+7Dx7AgiQ/tlOSQM5EnAFi0eEnSbdZNmoK6Sfrq6vEWi6blEovdCZLE\n3SXW4nGwFtUgdHpkiJrhLAA3eEFmAIBWEhlBpOMULPllYK2JPxgpOtoJ4W0OLLx2kyJbkiiB52Pf\nHGfNmZuwP8MwqnP5GIbRnW/CMAwcCfJGCsAiDAKlFSVXlHpw1Gfew0UyCLAEq/PcmMrp380oA7Ab\nsOwcECUUadgNF4v+iICJC+folrpgGCYlUScAcKZI1ZxiLjkTeQKAxRdnbuj0kmvUh6qr5i6DtaDc\nhNnEhmEYMBqUf3OdcFsD5EhIc3+GYcBb9EULKBQKhaIceqfLEJx53lRPwTTkkC+jJXilnpOpngKF\nQqFQkgh1nigpR2z8BJlcMENsO5TqKSjG4c7Dmus2pnoaFAqFktHkVM5Tqtn29hsomL081dOg5DAs\ny8Jup0lnFAplbNqDAqIG5sxaORZFedlz7aGRpyRy5mRDqqdAyXCmrP6KaWN/8Ld3TRubQqFQsomc\nijx9+vFH2L3rC4QGBnDV5juR50luHlFoIIj+3m7kFcTT6jEWWRRM1Xm6EEIIej55Cahcpaofm18D\nT9cJgOVhnbgCjNWVMdpPhUVeyH3aBPCGJArCrePBWhPvToqn8VSDXuzb/gVmL1f+uXefOIiPPtw+\n4pjNZsNll18xZj9CCCw8B0FQJ1RoRG0vQgbHiac1xTNnhTJVbBizEAYDDIGTZJfWkxaMWjiXYZxK\nuUwINCgTjCJVKZWEEMgmCYZSUktOOU9Lll6CJUsvQXt7G7a+/ipWfeXvkmr/tm/ch9898jNctvYq\ndBIXCssrFffdt/0d9J4ZnZg899I1KKkeLWq3s7EXvuYG8Ic+greq7txxluXQ4Z50TtzSSBiGAWRJ\ndT++fCb48pkjjg1pP6WzE1VUVQOh4QNYZ1ynuu9wbSd7hbo6eBey/+P3sPy6WxS1HdIamz33Isye\ne5Eme4UTZqLjy+R/LwVWHr2CiEJr7J2FpQyPdiKhTMVl7Sv5Xjzr68PCSObuVjRK1D4KAqsBbo8o\nE1gMKkcSjIpwGbDVP1WlWVo7u1FZmpyHZUpyySnnaYiysnL4enshiSK4JMr12x1O3Hn/P+CLTz9C\nWRGHqWVu7G9XprysJqpw6tBeBAZ4FIybDIvdheHPlLIoorK8BGyMaNTRd15AqH6lYjux4D0liAS7\nwbqMuWCUTJqdOgeKSPBET4PhrbBOvXL0y7IE6/Srk1KSJb6yOCBGI7A5lYlkGkFZaSn2fNilshcD\nUZJ11fiqdTpwMhiO6zzNvagCL31xGmokV/POCrXuso0Uy7QSBlOiHCxprj4eYAhsBs3RKOeJgIA1\n6DcRESXYDdBnSlXk6dCJRkybUJsa4xRTyUnnCQAumjcfvaeOonhicourWm02XHyZPgclEdFwCGG/\nCHdJJfLKqhX3E8PatYaGsBbXgI/0ox/GOE/yQC8gBAFL8pyDIbykE3ztxWDdpTFfZ1htJShKSo0r\nmyMJUVWO0/tvvo6pM+egamq9ZpvFRYX4dO8h+PwB8DyHDauWo6qseMw+S6aMx0eHT+KymaOjbDae\nQ1gQE94kly2fjl/+ZQfmF+TFfD2P59FBJByQRgqJ1rOWMeu8fT0/H5ELFq0+7Qxgh11AocRiqsAZ\nVuTbSNo5Cf0sQS1rTNSsj8jwaijpYjZGPJykKvLU1dOHyQsvTY1xiqmk3y8lSVxy6WWYPTexonIm\nYrXZUZ+v/sbMWayQhagu25b8Ugi+dl1jDIfheHgtsWuxmUlhkReMzR3XcdKD/8D7IHLinCFCZDB7\nXobl8F/RdviLmG0igT64PMpLcThd+opVD6qFM/jFf/0X7vnu97H569/E6x98iv/83fOQxtiZs3z9\nNfj82JmYr1V7XTjTF0xo28pxEBOEEL46Zxyqpxaf+yufUoR3pbEfChiGgZ1hR/ytKPXg3vxCFMgM\ntjkE9DPJU+pXQoCRcdwi4esayrDEo5fImDQ9TnXvDCdVkSdffwBeb+x6jJTMJmedJ47jkrLckgpY\nngeR1V/sLU435Ki+6JOSEiNqYGx5IELY0DGVIPedAVc23ZyxRQGsRUH5ClmG1ZWHGeu/ivJp82M2\nIURWJaBqs9kgRvU5yMCg5AHLsshzu3HHvd+By2FHZIxxWZaFhY99uXFbLQgqTEDnEvxkKxw2TPO4\nzv3N8roh67hzrivxYmOeB+18ejlPPRzBWk+eodewKAhsWVohIFWX+qgowkrV/7OS7PylUFIGy1uR\nf9HaVE9DN6y3Cgyf/jWpXIVlmLn0ctX9enq6TZgNhUKh5AbUeUohn3+0DQP9fameBiUGXGEdGEv2\nCLpdyHN/fDrVU6BQKJSMhTpPKaSrvQ2iICRuqJLS6lpUT1K/5FR38VpwztjJuJQUwLConL0k1bOg\nUCgUygXk7G47APD19kJmHWBTtM7vcDoRCvjhKRx7p5Ja7K5BLR/4elX1Y3kLLppQgj0NabakI0Xg\nwZlByYDJqwGo138a0o0CADnQCbF598gGsgDGXQpLzULd000En1eoqB3DsvBW1Bpqe+ZFCwAAJ/Z9\nrqk/ISSmUGVxgRdN7V2YXBt/d2c8oUyeZSAoLAPBgIFECDgVSSxRDIktakt88TIsmnkZXVw0KXvu\nCACWMLCTQckEABAYgiA7+PkxGHxPSw3eGRcgMly8th2kw0mU1K8USSboC+vPz2vp8cOaJEmazh4f\n/rrtM3T19oFhgM7ePnCc/s+Ukn7ktPPU1tqCd956c4TWU093Nzbf931YrMaLSF7IzIsW4LPt7wPj\ntG8bT1fKa/PR2tCteTv/cGyzRheyHeEMhfshNHww7NXBGw5ry4NlwuhtwoyrCJaJK0YeZHkwSdqm\n7Z2lLEdpLG2nIRaML1Ble+hc15ponJfnRn8gAE/eyAjlimtuwItP/XZM54lhGMgyAXuBZPTEebPw\n0bYdiuzXuew4EQhhcp5T8ZznclZ8IUewkFOQpB8DD8PiO15lDq9RSIQgAIIwGXQqrQyDfLCmbnLx\nETmuhpZSIpIM3qA5PrP3BDbOqNU1RlSU8OjWj/Hw//iuIXOKByEEP//9S3A57Fh9/U2orqww1R4l\n9eS08zR1+gxMnT5S56m3twdutw0dYfP3thaXlaOnqwPjTLeUfAgh4E+/Aan2atNtsfY82KYrt8Mw\nLMCZ7xxfyHBVcaOIhAbA8RbwKnf0aC2XUlJchM6u7lHOU77XC39wbEmJK2ZPwi/+sn3EzqeoKOG+\n9cvQGVS2o3LZxdPw6gf7VDlPly4ch198ehwLMygAwDEMvGCSprskEgKLQqfneGAA73T0wh0jSiUS\ngitK1Dn0sWj2B8EwwILLL47bZteJJry5+yictsFzf+iUHv42REnGHasXw27yw/Brf/sYi2dPw8LV\n15pqh5I+5LTzFIuCgrNPmAaEi5XAGhCZSUcYhgFrdUB9sRaKGr7c9QlKqmtRPl5diZfCQm0ipjzP\nQ5Jif6uJ1MPnr74S81ePVGr/yX/+AhaOg6iw/pfTwiOqoVZY8l3lzEKC8s+oKyLgpuWzMLXU+IeB\nIfxhATXescVfz3T5sOXvbkVddeqjPO1dvbh0/egIOSV7oQnjKWbtdTekegojiA4EDCniSklvNm2+\nLdVToFAolIyFOk8ppris3JRxd77zZ039Tmx7DSDpJQiYLYhBH4iofHdlw4evmzgbCoVCoWiFOk9Z\nSm9Hm/bONPJkCoEvd0AK+RW3D/VpL6MSj707P8PhfXsMH5eSO9CrA4VCc55iEgqFYAsHELErL3uR\ndmh0gIrqZyDafxRd+foLJnN2F4xXscociosk9H7+F7B29+D34fSAcynLExnHdsFfVGb4nGRZhizT\nTLRcpotI+FAMjZBdiAKYqqDAsF8QscsXwOp8t2q7kkwgERmEALtburGjqRNWjo0p/yDKBDfMrB1z\nvObuPqwv0p+cbgQdPb1wOrJXVJcyGt3OUzgcxj/+4z/i4MGD4HkeP/jBD7BixYpR7Xbs2IG7774b\ndXV1IITAZrPhueee02veFBiGwWP//Uvc/M2/h92hfFdPWqFxu3Dp5Dk4sf11WFr/CovDhckrzydB\nqtV/yp+3Dm2nfJrmkemU1+Yj0nEKRZfeApZXnq5MJBFEEnF6/7uYc8M3E7a32hzgOPoMRBlNpyzh\nBBn5+NJJJPAAbl9YD3uCBP8j/QPojZ7vTwDs8QXw/WuXwGlVfs6FBBGvHmpERzCMvLNSCNNK8/HP\nd904SrZCKYQQBCNRuBza5CeM5ODxU5hcW0P1nHIM3Vfd3/zmN3C73XjrrbfQ2NiIzZs34+2334Yj\nhhc+ceJEvPjii3pNmo7dbscNN92MLz/fjtmXml+nbVaZG/vbA8YOqmPpbcLy9TGPz60vSj8BzTTG\nVlqrqv3c+iK0H96F3jPHUH35BjAKxFunLFiqaW49Pd2ad9xRMoOPpBCumVMzIrKTx3PwWJRd9t/t\n6MUtl80acWxdvht2hf2HePSzI7hxVh1mLjNOgLbTH0Rlocew8fTw5oc78K2//4dUT4OSZHTnPL3x\nxhvYtGkTAGD8+PGYOXMmtm3bFrNtJu3icjqdkETzlzdOHD2M/bu0qT2PSarKiFN0UTZtHqauuRnu\nYnO3X9PadtkPA6DaYUPVsD+ljhMAeC0cppbkj/hT6zgBgMPCGeo4AYAky+DTKNJjUamzRsl8dEee\nWlpaUFlZee7/Kyoq0NraGrNtY2MjNm7cCIvFgltuuQXXX3993HH9fj/8/pHJtRzHoaIiOZoekXAk\nKT8ISRIxEAzA6MXBhauuwfGQsWN2nTgAMKnXVKFkFwyj/MGKZQfLs6hFAmKWlclm9DyqijJB7Gwk\nhbYJwZ7WbggSgT9ifOYjISQppXKUkKyYQGtr6yiNNY/HA48nPSJwuUZC52njxo2jnKGhi9BHH32k\n2NCMGTPw/vvvw+12o6mpCbfffjvKyspw8cWxFWSffPJJPPLIIyOOVVVV4b333lNsUw9f7PwcUxct\nN91OeWUNjuzfi7rJxj6Z5ZeWA43qatslItDZggpvFK228YaOS0kes+cNnmcn9poQ7dQIx7KQFd6B\n7DyHqKxeSqOOteAEETGRyZ0IgZ57+t6+AGYmEKkci7AoYeuXTbhx3TJ8ff5MHTOJTUQQYddZSkYL\noXAEH+zcO+JYR7ex19l4bN68Gc3NzSOO3Xffffj2t7+dFPuUkSR0nl5++eUxX6+qqkJLSwsKCgZ3\nPbS2tmLJktGV4F2u8z/E6upqrFq1Crt27YrrPG3ZsgUbNmwYcSyZCXkLFi2C3SQNpuF48vPh78uM\npOraJWuw70+PA7OVOU9SKJBzT/tDSAN+cM70eyLkklQgNd1Ys2AcHv+8AQelCAIgWME5UMVm72ch\nqiyefCEH/UE8cG380iiJiEoyagvysHCSOcWnQlFR0xKiXrbt3AdBEDF+9qJzx7616LKk2H766adj\nRp4oqUH32bd27Vo899xz+Nd//VecOnUKBw4cwE9/+tNR7To7O1FSUgIA8Pl8+PDDD/HAAw/EHTfV\n4cgJEyehuT85JVqYtAlAJ4az2KD0ub9vz1tA+Qogg96fUfj2vI2iperU47tOHETxBHUSEVpr2+Ua\nPMvgm4sHS9g0hyLYsb8VVVms1BIFgV3H745lAE7jTjhgcHWCNfGhSSajC0wnA38giAWXr0NNdVXS\nbScrZYWiDN1XjzvvvBMPPvgg1qxZA47j8KMf/QhO52AGzy9+8QuUlZXh5ptvxltvvYVnnnkGFosF\noihiw4YNWLlype43QElzJBFQsVU/GyBEBhHVO97th79Q7Twd/eITlNaor213roZjjhLOoM0rmQj9\ndCnZjm7nyeFw4Oc//3nM1+6///5z/968eTM2b96s11xWsuHWLTgRTPUslFE6ZS7CPfvRXTgrYdu8\nactADm0HZBFgeTjHz4Sjasqodq0ne8AkqXq8GZTX5kPo74Z///uDpW0YBo66OarGKBs4jj6Leiez\nu+UMJs9brLqf1tp2Ru+Y1TJcjcOOP5weqaBvY1l8pbpUUf8ymxUsgOeFACawPOZzqdcKMhJCCLaK\nA1ii8X3t6PGjQGck83CHD/WFebrGGIuuvgAK3cnX4PP1B+DJM+99UdShVGeyvb0d3/9winT5AAAg\nAElEQVT+93Ho0CHU1taOkEx699138ctf/hKCMLixYePGjbj99tsT2s7euHUG4XS7gaCxOk/BPh/4\nkzsg1i1K3FgFJRNngRCCmgtC8oQQRPp9ONJ1flHPkl+qaOnKI59EuPUEwHIACDi7CwPOGWDt6b+e\nX147qBhuyStSvUw3s8qJpt3bMdDbiWhhGaau3aSqPyEEkVAQdqd6tWc9GJnDpmWor6yZN+rYU298\njhOBECa4E6s88yyDmxfXAQC27mjEC0IAs7mRjqsdDMYxfFrn6xFCcFQW8OUFQph9RMYlnB3zF1ar\nHrM9HMWJYAjf0ZHvBAA7m7vwz3fdqGuMsTjS3IG77rrDtPHj4Q8MwOtN/+tSrqBUZ9LlcuH+++9H\nMBjEf/3Xf414raSkBI8++ihKSkoQCASwceNGzJ49G/Pnzx/TNnWe0oRZZYM3wEg4jHf+8idMmz0X\nQa/6i98QLm8+OppOodBg5wmIc/MkMo68/TzESVeAd6srmeCaMB+uCedPVDHQC3L0E8jdAQAM7JWT\n4RoWyRlSLSdSFCQ6cH5eLD/ogLEcGG50FKeklEW449So45wjD/ayulHHxf4eDJzaByKLI47zecVw\n1c9V9R4vZG59EWRJQsmk2eCsdtjzlJVtGU6wqxUl1bWq+uz+7GO48vJQtXjsC0M8GIaBHGe3myBK\nqjcIDEWeIpKM3lAkZhsHzyVMDt68dgF+9PKH+LqzAlYF4qJDrFs0HksFETt3t4w43gkJX0iD8yli\nOExiLShizNmwIoCglYhokSWIZxe8rGDgYlhYMZgTKYCgh0jwExkMBpfFBACTWQu+vqh+VHaTlnwj\nUSZ4qbkT/991+hwnALCwjKk5SeGoCGcaqItTUssbb7yBH//4xwBG6kyuXTtS3NrtdmPBggXYsWPH\nqDFmz549ol19fT1aWlqo86SVYDCAHW++gUVrr0uqXYvViqWXr8IbLz+P+UuXIVpcr3msZD41MyyH\nmddswf5XHoc8dwNYi/YLG+8uQMH8qwAM5g9JAyP1voaiPWJ/N4InG84eJSCSBCIJsHiK4Z4weilL\nCgfBWkdHJuKVT2HtTjjGzwRzQfkThjMmOZvlOLiKtO/orOIC4MsrEzccRjgUgqdAez0wC89DFMWY\nr02pq8GRhtOYNkG5lMXQKbq4pgRvH2uO2aYzGEaxy46NM2phiVNShGMZ3FBVgkcbWlDrGnnuzfC4\nUO+KH5HyWHisXBR/V1h7OIqP9rVgrxzbudMLDwYVDIeVc6tgO+v4hSUZAVFC+KyjamEYLLFZkG8x\nJxpGCMEfzrTh+spiOAzYxWZmsjiQOg3gNA5EZgxG6lWp0ZlUwokTJ7Bv3z786Ec/StiWOk9xcLnc\naG46k3S7LMuioKgYt3z9W/h///F/sHjTN2Gx2pI+Dy3wVjumXXkrDm99FlhwsyFjMgwLPk4xXT6v\nCN7ZyjcdcHYXHJWTFLdnLXZdTqDpMAyKKmpUdRGEKCy8dueP53lEhdiih5esuw5/+uOTqpynocjT\n2qtWjNlu9wef4T+378eDK+Lnks26eComRCciMuzCTAjw6vv70BKKYFmx+ugeAJTZrdi4qFZTX614\nLYDxZaEH+cPpNhAy2hG4uNCLuUunmWTVWFKV70/3GehHjV6VUTqTSujo6MC9996Lhx566JwywFhQ\n52kMUqmJwzAMyiqrIIsioNV5SsFjkt1TCKsrD8ZrClMuZNJcjUuyDIOe7m4UFqmvbceOsSTmdDoR\niar75pWeohddthhvHWtK2M5p5eG84LL2lVVz8Ye3dqmaV7bzveu11URMF2jkyXyOdAURjBpXosxl\n5TC93KNKr8oonclEdHd344477sBdd901askvHpm7xSkHsFozc4u/3ZPb2+AzgeeeobXtKBRK8qmo\nqEB1dfWIP62ajkM6kwDO6UwuXx6/MgghZNSO4d7eXtxxxx247bbbcMMNyjf9UOcpjbn6K7fC5tRe\nIuGSa24ycDbKmbB8fUrsZgrhthNoPfBZqqdBoVAoGc2dd96Jvr4+rFmzBt/61rdG6UwOOVayLOOy\nyy7DAw88gC+//BIrVqw4V/7t17/+NRobG/Hcc8/h+uuvx4YNG/DKK68ktE2X7bIYZ54X6ElO3SVK\nZrD08lVgGBZ7Uj0RCoVC0YlSnUmWZfHBBx/EbPeDH/wAP/jBD1TbppGnMfjq1xILZVEomQTPW5Ja\nI5JCoVCyEeo8jYHbTZVkKemLr7MdQiSc6mlQKBRKzkGX7RJQlWdNWoHgWMwqc+PksaPoc5aldPef\nWubWF6Hhw9fhKqlEk1QE1jaks8OktWqz0cytH7mjrff0MTQ270Te5A26x97/0XuYf8VVsNjUyylo\nrW0XiURgi7ORoa2tHUX56hI/k7H1285xaAlHcKR/AA6OhZVhUG63ZvV5SAjB2x296IiMvHaJhGC8\nyeKSvaEIPHZzN7ukQjJg/5cNcNgyQzaGYj6ZczdOMb6ebnA8jzyPN+m2CSHY++oT4C/Q51l7/Y0o\nLh2tBvOnZ57ComWXodtahFpLALvf3zp8MIBhUFxZg4tWrBvVVxQE7GkxplRMzfwV6D1zHHltOyFG\nQoPmZRlT19wMLkYdt8NvPgsA4K02WF0eeCtrcSqqTZsnVQh9nSgaOAl7Xj4qZ49WanYVlWPW9V+P\n+f7VEgr4B/PaNKC1tl1HVxfmzY2ttfTun1/GxtXxd7rEIhn+C8+x+OcNl+D1v+2DTAhCkoyt7T1w\n8hxmelyY7HbAokKVPN3xRUU829SOFSUFuG3dgqTb//R0B65YqX67uBrMPm9aO3twpq0DHd29ECUJ\noXAELR3duP/7D5prmJIxUOdJAVV5Vgy0+vH8E3/E1777T0m3Xz95KuonT1Xcfv0Nm/DHx/8b8264\nE8WVNVh9612K+7aePIa+3Z/Cu0R/ZMTicKF08hyUTlZWJHfKFTeAEAJJiCAS8KO38Us4+nYhVK9c\nCDPZ1DsHcHrne+cKG3vyi1E29aK4yuFWV2YvBff0+FBYENuh7QsEUFygzplLVgTBynHYsOqiEccC\nUQHbPjyEF5o7IceYCAsGt44zS6rSGF5s6kBIkkc4EzaWxfeuWQKX1RglfLU0+gLYVFFsqg2zz5v/\n9f+ews1XrkTd3CXg+UFV9xvHj8vqaCVFHdR5UsikyVM0iQqmAovVCl7jEl/N5Ok4stNY5ValsGcj\na5zFCqszD3mlVQCAPQ3dKZmPEjzlNZh59ZaU2GY0REteePJxbNz8NQDaIl+EkLgJ55yG+aTyXuS2\nWnDVyjm4Ks7rP33146TORwthWU47wUsG5peGMvu8mVBTiaVX6n+ApGQv2ROrplAoCZHi1KWjUCgU\ninKo80ShZCilNbWpngKFQqHkJNR5ylJKh1WapmQnc5av1ty3pzt9l0IpFAol3aHOkwpuvmVzqqeg\nmNVXa1+vHzdlpoEzoaQjtLYdhUKhaIc6TyrIlIRxvUyZP3qLfaoQ01wEsmn39lRPgUKhUChJhjpP\nlLTm6DsvpHoKY9LXcirVU1DFDX93B1hanoVCoVB0QaUKKJQcwmKAOCeFQqHkOjTyRKFkKB1nTqZ6\nChQKhZKTUOcpS2lrbkr1FCgms3f7O5r7FhbmRv4ehUKhmAF1nlTw9FNPIBJO7wTmId55/U+a++75\n4E2Edm8d8TeO7TNwdsoRQgHIQuoKMxNCQOLUgphVk4ewvyfJMzoPkSXNfW++VdvO0XifBQBEBfUC\nnOGoOOaYw5EJwY4znZDk5NR0CYgSXm7uxB5fAGFJTopNNTQOhNEZEVJiOySIGIiO/gtEBfjC5v5e\nJVlGIGSeDX9gQHOFBkruQM8QFaxedyV+99j/xaZ7vjeqSG+6QWTtF/sZF6+AJJy/KBNCsOPNP+HS\njbfhi9M+I6anmMlX3IhgVytK6mcDGCweLIuDc2MYBrzNATAMJl52LVhu9OncdmgnokE/Ap0tI27S\n06+6LWYJiYOv//78/5xtz3Acpl852tkghGDODd/Q9f6GI0bDiAb7AQzWwFsyqWJUmxP7dsLf04Vo\nOARPUemI12aVuc/9+72/voauzvZR/SvWXYPisth19/QyfUItPt17CEvmTFfc54o5k/DMtt249bJ5\nCdt+/46NeGfrNjy64zC4C767iCRj/ZQaTCjyqJ53PP7nDcsRiAj4+OPDeLW1C4KO35TREAJUOmz4\nZxNKs7T4g2jqG0B7YADScMeWAB3BMARJhsvKj/oOhvjKrDrD5nKirQvbDzYAAKKihIGIgGA4ik2X\nzjXMxnAIIfjpk8/jvgf+wZTxKdkDQ5Q+9qUR3f2hmIU8k0HTmdN48jePo7B4sPDltRs2oqqqelS7\nP738IsaPr0Xp5NnJniIA4Pknfo0Jl2+A1W43fOydjb2Gj6kFIkvnpAx4uzOmM9TTeBS8zQl3SWVa\n7zIjsozmN36DqomDBaAnzV2E/NLRTk5fdweEs++5oKwS3FmHcbjjBABCNBozosNbLKjxaj8nfvvz\nn+CB+74Z+z0Qgv/10L/goXu3gFVR5+53v/4tplaXYt6E0b8jpQiShCef34rmviDs/Mjvuchpx8aZ\ntWBpUdcxaeoL4rl9DagtcKO2IA+T5s8Cf8FvpijPCZtF/TO3LBP8ZechBMMRAAADBgzLgGUYxPpW\nZELQ0NaN2tJCXH3jDeA4Fhaeh8th/PVsOE+8shVzp03EzOVrTbWjCpaD1VuSdLMvH2hFMKo9un0h\nLiuHjTNHPxBmKjTypJLqmnH4p4f/NWG7a67bgF/+/Ge4IUXOkyc/H6Gg3xTnKV1gWA4Wh2vMNoXj\npyRpNvqQhCgKyiqwcM21Y7bzXhBtiofFas6uurEKvjIMg4qSIoQiUVU3uS13fg3/+bNf6nKeLByH\nr9+yPuZrv3jyVfRHBHjtdKfhWHx4qg0P/N01KHQ7DR/bHwrjVHsP/u5rtwEYdLRlWYY8xhLsLSWF\no5w3s+n2+dPLcaKkLdR5MgmO40y7gVGyFI2BEVEQcGDXTsyct8DY+WhBw3sYyyEzAitHUzuVYmZ0\nzuuyo6yowLTxjYAGJylKoVeVLKW4rBy8xZbqaVAUwvI8Js5eqKmvLIk4enCf4vZnTjdqskOhUCiU\nQajzlKUsuHg5XB5vqqdBUQjL8efynczmL6+9mhQ7FAqFkq1Q58lE6usnpHoKhuLrbMehHbSWG4VC\noVByG+o8mcjaq2InsGYqkiggHOhP9TQoFAqFQkkp1HmiKIa3WnF4x4dgTnyCQv9JBLpaUz2lrGHB\neO2JtAP9fvAW5bpjpqqTaByaaO2oAJ5lEYimRkwykxBl2bSE8VBUAKdCvoJCSXfobrssZlaZG/vb\nA4aN5y0qxc3f+yF8na3wdbZjWokdZeNG3/TTRQcqXakm3TjwyfuQJREMy6GybjIw/jLN49XncVhw\n7UYDZ6gdSZZVaTwBg7vtzPTnNm1chR//9hV8d9ks03f2ZTL+iACP0xxpkz99egCbv3qrKWNTKKmA\nOk9ZzPEjh4CCcYaOyVssKK4ch+LK+OMyJz5BT+vZ2noMMyiHzDCYf8V6FJZVjmqfTc5WNelG0/HD\nWLDqmlGv+Tra8Pnbr4GtqMalGzbDanfotjcojulO2G4448fX6rYbD0mW0y7C4LbbsHpiNX78wT4U\nOKwocTmwcWZtqqeVM/gHwhAlCSWF+ameCoViGNR5ymI+2/Y3zL5uS9Ltzl95leK2R7/4BChOzi6z\nZLDvw3dx+U1fi/lafmk5Vm++O7kTisH6a68zbWxCCFhWfXTH7IDQpWuW4dI1g//+8eMvmWssQ4lX\nbkUvTd19mFqtTNyVQskUqPNkErIso+HECTjKx6d6KmnN6aMHUJBFzhPDsrBYqb4WhULJbPY396Fn\nwLhcwUKnJavKs6RXfD2LkGUZf3vnrVRPg0IxlETJ5vl5bvT00R2ZFAolu6GRJwqFYhizp0zA7kPH\nsHaZOrX0SFTEly2dMV8r8bhQYEK9NQqFQtEKdZ5MgmEYSLKc0jnIJLX2lVA+vh6t259DceU4zDub\nK/Xha88i1O+HxWqDfc5q8Db9idVmMlxmYOex8qTa3v72VlSOG48JU6YlxV5enhudXV0oKS6O+fqM\nZavx0v/8Z8yaXIfK0thtYnHVgmk43toV87WXP96HYo8LVy2YjspCj6Z5D6fYZcdjO47g0rpyTCry\ngtOQo5XuDERFfHy6HZ3B8IjjEVFCRJRi9qnJV7fxYCx2Hj+D9/Ydh83CIRwVcd3imYaNbSZm7vqk\nZBfUeTIJjuMgp9h5mjRtRkrtK2HWJVdg1iVXjPisLrnmZjAMA393Jz587VkUVVSjtKYWdTMuOtdm\naIeeJEQRDfaDYRnwNgc4q33M7ehiJIyQb/RNmrPa4CwoGXU8OtAPf+vIWnCEAHPqywYlBi4g1i47\nsyCE4PD+PVi2Sl0V+IYTx1E/YaImm2s23opnnv4N7v/mXTFfZ1kW//gvD+H//uTfsGj2VBTne2G1\nWjB9wvj/v707j2vi2vsH/kkmC2vYxLApKmhV0LriVhUXQFyBVtBitVZrF9G29z611nrbW217a+/z\n67116X281Lb2kcdKW6VSRLEqtW51Qdu6tK6gQlhUIGwhJJnfH1xTYwIkkGQy4ft+vXy9yJmTme84\nMPnmnDPntHpdhsbEY2gL2xIB3KmsRlbmVyitqoGfpzsCfDwN6nTz88aAHuaNp1g0dxrqG9XIyfkB\nh2+UQseyra5n7C4RI1LuA38P2zzGb4qnRAxPqdjg/6xRo8XdehW0OhY6FrhRWYPfK6qM3qtjWUhE\nDMaEyhET+5jBNrGIgZvUtguW7zz+C+obm7D6tZd5NzWETedAI06FkicnNnLcRKvO82RLD84NdP+G\nK/Pzx9SFy3Cn5JbRTfh+a0/1nTL8dvVXsDodGlX1UKuav2n7BgSbfOpPee8OrpeUGJV7SP0QHmqc\nDNUphShpdMwB4JKK64gYPNTiD6gff8hHV7kcHh6ebVd+iJe3NzQaDVQqFVxcTCcTUqkUK1a/iVP7\nd6Oqpha19Q3Ylr0fry2eC2/P9rVudPHxwuLnFgNoTqTuVikNthccPoSzN4oxf8Iws/bnJpVgdlKM\nWXUra+txPP8nnC2+a1nQ7cSCRa1aA6VKbVAuYRj4uUkhYpr/Vrp7e2D69ET9a3v5V+4xNDY1mfy9\nY1kWQ8O7YXKCY8w7Zonq2jrIPNy5DoPwBCVPNtS7j/GHMbFcl6BuLW7z6iLHiCmJZu9L5tsFg6LN\nb6lxl3mh96Aos+vbk7KqEl3llj+9IpW6QK1Wt12xBT5eXmhsVLeYPAHNCXBU7B9TIqibPkOVsrbd\nydODuvh4oYuP4aLXj/Scjw0b/oWyqhrIvS1PClvj4+GGqdMnWHWffKbWaPDaf73EdRhWp1Y3wcXG\nrXLEedDTdjYUExfPdQiEdBrzF85H5pFzXIdBCOkEKHkihKfcPT0hdXXswfT25OXhDh2NWSGE2AF1\n2xHCUxGDWhpiTYjt8D0/VTdpTD6JrFKracA4MRslTzYW7ClBU1MTPtn8L30Zy7IQCAQQiURY/NwL\nKKm13Yrvd8/mo/hmkVH5+LipCAntaVSev/e7NuvzZRC6o9LpdHCrvAk3dw8Em1hn7tzJE7h88Veo\n1Y0AAAEEYMFidPRk9OrTsdnYgz0l8Pf3h6dn+x/5L1aUws3NshavyN49kXf0NJYkT2/3cc3hKhHj\no+zD+tcsC3i4SLBg0nCIGcamx3YGLMuiqLwSGhNPCrMsi70FvyGiu32n43hQg6oRFZXV0Gg0YFmg\nUa1GbYMKdfUNqK1vQL2q0aA+y7JQNapRXVsHdVMTWBaQSsQQmfhd0Ol0mDLWMcc3EscjYDuYau/e\nvRuffPIJrl27hlWrViE1NbXFupmZmfjkk08AAOPGjcPq1avbdcy7NQ1O3Tx/9fJl5O3LbU6wGAZj\nxo2HrFtvh3zsd883O1B59w5YsNBqNHBxdcMjEQPBBj0CkVjMdXicaV6wt9mvBadw4VwBNJrmJFmn\n1SK8bwQGjxgFVzfjp3vqa2shFDFwMXPhYJdGJXZ98xXUarXB70jXrnI8npzSwTMx9OuR/airq0Pc\n5IkWvzc3839xpei2QZnMwx2Ln5hmrfBMunT4AL44dBrjI8PgIhHBQypFZGiAQ/49calW1YhNOUfR\nQ+4LjxYGTg8OC0HPkeM6fKy6BhV+u34Tt0rLTW5nWeBWaTlUjWowQqF+fXEXqQT+vt4Qi5q/97tI\nxHB3c4VLYBg8PNzh5upqtFCiq4sLPD09IOHr/UjIQOJlPI2Krb297zerL8/yVpwTLcXV0eTp6tWr\nEAgESE9Px4ABA1pMnm7fvo3U1FR8++238Pb2xqJFizBz5kzMmmX5IqXOnjw9SKVS4YdDB3GzqBBT\nUxdzHU6b6uvqcO33i+g74FGIxX/cgDtLa9WDSdN9Wk1zN8GD/x/Wcr9lk2V1kNhhTb3N//0uXn05\nzWqJx0cf/A1/XphslX21RqvV4czFy9BqdSj8+TQEECBx1ACbH5dPPvjmIF54fpHRk4ztUVNXj+Xv\nbcCA3j3BstAnPw8mQf3DQhHYbygELUxSGhQghyuN6aPkyUF1uNsuPLx5sr22bqb79u1DTEwMvL29\nAQDJycnYtWtXu5KnzsTFxQVx8VNx5PAPXIdiFjd3dwwYYtnSHM7k4J7dmDh1pkEZIxLBlh1GYjt+\no5ZIJFZtsbFX4w/DCBE1oPnG/UjPEHzzZaZ9DswjUonIKokTAGi0OowdMgDzn19mlf0R4mjsNuZJ\noVAgKChI/zowMBAKhaLF+kqlEkql4UR4DMMgMNB5VmW2xGPjxqO4pv1z83DpbkU5gM6xNlnJ7Ztc\nh2BTWp3ppT0IIbalUCig1Rr+/clkMshkHV+yiFiuzeQpKSnJKMm5P+D52LFjNhs3sHXrVmzcuNGg\nLDg4GAcPHrTJ8Yjt/Hb+ZxReuYy62hqMi52KJv9eXIdkM4JWF/rgP18fH5SVV0De1f7dCIR0Zqmp\nqSguLjYoS0tLw7Jl1LrHhTaTp507d1rlQIGBgQYXXqFQtNqKtGDBAiQmGs4czdDTMrw0ZkIMxkyI\ngUrVgKz/+wIR05w3eWJh37F4DfX12PzxRkgfmu3bx8cXc1LnWf14Y6bMwt68bCx40roD0QkhrcvI\nyDDZ8kS4Ybduu9jYWDz11FNIS0uDl5cXMjMzMWNGy4uoUnOk83FxcYVWo+E6DJuyd8vTPa0Is597\n2eS24ho1gj2tO0hdLg9AZaXxYrSEENvqrENWHFWHZxjPycnB+PHjsXfvXqxfvx7R0dG4du0aAGD9\n+vXYsWMHAKBbt2548cUXkZycjClTpiA0NJQGi3dCkYNpYkdCCCH81uGWp2nTpmHaNNPztCxfvtzg\ndXJyMpKTbf9YsjP65dxZ+IVFcB1Ghz06fKRTT1swcWrLramEODJrzq6t0WhomAVxarS2HU8cPfIj\n1yEQMwSGdOc6BEIsptOxgBW7nBsa1XBzsf28Y4RwhZZnIcRJ6XQ6fJq+Gc88+5xV92vNFgp1k/3H\nwLlKpahtaGy7ohPLPnkBxfeq9a9rG9ToG2K9JyjrVSq4ubq0XZEQnqLkiRCeuz8o/NjRH/Hz2bP6\n8kaVCpNi46x+PGtOTxLWLQiXC2+jT48Qq+2zLa4uUtSq+DlnmjXsP3cZDCPEc88brlggteJkq7V1\nDfCwcP1DQviEkidCeOzBp+lGjxmL0WPG2vR41l51Pvbxufhi8ya7Jk8AENrVBzcrKtHd38eux+Va\nY5MGZ67ewlur/mzT49Q1qCANcN4pSQihMU/Ern4tOMV1CKSDrNny5O7mhiYOpq/wdnNBfaP11u3i\nC7VGg67enjY/jo5lacA4cWqUPPHEwEcHcR2CVfx8+ieuQ7Cp/dm7uA6BdzrJGt8Ow16Lqlu7lZIQ\nR0LddjwxZuw4o7KK8nJ8nfklRAyDqfOe5SAqy9TX1cHFxfnGQXgoi/FrwWnodFo0qe07lubc2QLI\nZDL0Cgu363EJP3m4SFFTr7L5cUSMkNZBJE6Nkice8+/aFS+kLcfH6//JdSitOnfyBC6cO4Oqe3cx\nZmKsWe+J8HfDpV/OQhf4iI2j6xidVou83Tsxd9ELAAB3T9t3iTxIWV0NkYjff8Y2Wh6TmGCrtUgf\nJhaJoCkvAjDYLscjnZNKpcLrr7+OCxcuQCQSYcWKFYiOjjaqd+DAAWzatAlNTc1d9UlJSVi4cKFB\nnXv37mHGjBkYNmwYPvroozaPze+7Lmnm4J8+Qd26gwmNtOg9QqEQZ08ex6OzHDt5aqitweCoUZB5\ne3MdCm9R74592SOBEjEMVI2d94lGYh9btmyBh4cH8vLyUFRUhNTUVOzfvx+uroY9HP7+/ti8eTP8\n/f1RW1uLpKQkDBw4EEOH/rHixdtvv43x48ejrq7OrGPTmCcnYK9vk+3VNTCoXe+z9zpx7eHu5Y2o\nsdFch8FrDv7r63Ts8f/t6Pck4hxyc3MxZ84cAEBoaCgiIyNx+PBho3oDBw6Ev3/zPGYeHh7o1asX\nSkpK9Nt3794Nf39/DB8+3OxjU8uTE5DLA8CyrEPfsAbIPfQ/X7t8CSfyDwIAdKwOYFl4+/qh38DB\nCO/bX19PWV0JTVMTRFacf8baHjwvLlRVVSI0tAenMRD+0el0EArpuzOxP4VCAa3WcDycTCaDTCaz\neF8lJSUICvrjy3lgYCAUCkWr77l27Rp++eUXrF27FgBQVlaGrVu3Ytu2bdi7d6/Zx6bkyQkkzU5G\ncQ1/msjD+vRDWJ9+BmX37lQY3cwTn3wa+779AoOTnuEsMdRptdA9NPBVIBCCEYk4T5wAoKiwEDNm\nJXIdBuERT1cX1NQ3wMvD3abHod5YYkpqaiqKi4sNytLS0rBs2TKjuklJSUbJ0P2GgqNHj1p87PLy\ncixduhRvvfWWviXqzTffxKuvvgpXV1eLnhCl5MlJBHtKeJVAPcy3i/HSEAHBIVIO1TcAACAASURB\nVBgUNRLFx3MRn2i8oHThtSs4eiDPqLx7r3CMnWw8s/aNK7/j2KHvjcp7hPc2OZD9+uXfcDz/AMQS\niWH9sN4O01WX+tQCuyeWdfX1Vt1fxb0qVNfW2fzD/EF+MndkHjmHLjJ3yL094e5ieI2FAgF8PFzh\n5+kOHw9XeLm5Qih0rJZdjVaHmgYVVA8tcaPWaHHhZikuF1cYddGxLFBaqYROq7NpbIXFpQgNktv0\nGMS2Lt6sQqnSek9mBsial+vJyMgw2fJkys6dO1vdZ3BwMEpKSuDj0zzZrUKhwMiRI03WvXv3Lp55\n5hk8++yziIv74/Ph3LlzeOONN8CyLOrr69HY2IjnnnsOmzdvbvXYApaHk3HcrWmw21wlfMLn5IlY\n7sHZxe0pZ/tniOjXFwMjm7tYO5q8VVZV4Yt//6vFsTjNi9YCbq5SjB4ciYF9elmty0mlVqP8bhXq\nGgw/JLRaLcovnsMdZR0qa+uhrG8Ea6e2lIdvbQKBYdn91yJGCE9XKaRikcE1EAmF6NdNjoHRMWAY\n+3fNHTp5DoW3FXjquTS7H9spCRlIvKy37qC5UtJPWD152vGs6cSmvTZu3Ijy8nKsWbMGhYWFmDdv\nHvLy8uDm5mZQr7KyEk8//TTmzp2rHyNlyq5du5Cfn09P2xHibLhKmB4Un7IAO7duRv6PR6HVaaFp\n0iBq2BBEjx3Trv35eHvjpRWvt1mvprYWJ/Kysf/YaTAPJU8arQ4BXXwxf5Z5U2Hc5yKRoHtgV9Mb\ne/e0aF/OoODiZRw8cbbFVjaWbX3AuU7HwtfLE/OfN+6CIcTaFi1ahJUrVyI2NhYMw2Dt2rX6xGn9\n+vWQy+VISUlBeno6ioqKsGPHDnz55ZcQCASYP38+EhPbP+SBWp6cCLU8Ob+SiwUYPsK6396s4bP1\nf8fLS5/jNIb1f/8b/vS0cfcuMd+Hn2di+attJ7LEjqjlySHR4xZO4mZREdchEDs4feok1yEQQkin\nR8mTk8jJ/pbrEAghhJBOgZInQgghhBALUPLkJIQCAS4eO4DCa1dwp7yM63CIDQiUFahRKrkOw6T7\na0ZxSSIW4161Y/7/8AHLsqiuMW9pCkI6O3razkk8t3QZzp45jTtlN+Ej7oZgz25Gdc7/+gt+OHgA\nWp0OoT16ICgoGAAQFBKC4OAQowHnFWUK3LtTAQBwdXNH18AguLi4Gu2X2J6/lMXmr3bg5f96letQ\nTPLz80XFnTvw79KFsxhmz1+ETRv/iSmPRbVaz8fLE+Hdg+0UFX/k/HACk0cPbbsiIYSetuusCm9c\nx52K5sQoKDgEQcHGHyaFN67j1s2bAIDa2hooSkpQVHgD89JWwM2D+9m1+e5Yzte4c+eOUfkTyXPg\n37WFx+cdVOGN67h27ickzZzGaRxXThxERWV1q3Vyf/wJby1dABcJ99M+OJI1H3+BVW+9zXUY5GH0\ntJ1DopanTqpHz17o0bOXxXUa6utRoXasmZb5avacJ7kOwWrc3N2hamzkOgz0HjkRvduoc+l6ETQa\nLUC5kwEPNxeuQyCEN2jME7GIq5sbuntT111HOcJkl9bUUN8AoQMvTP0wHja42xz9lxBiPmp5Iu1y\n/8NfrW7Eln83rwHEsizEIhGCgkMQPmwMPDwtXyW7M3C2xEmtbsTXX6Rjxcv8WI7DR+aJqppaeLq7\ntV25k9DpdA63dh8hjoySJ9IhEokUL6Qt179uamrCzaJCnDtyAI8np+jLnXn2c1PJ0Gef/BsNDQ0A\nDNd+e3rRs3aLqyOamprw2fq/QywWG5U/PW8OJPI/unO3fPR3PP/MAni4229h346Q+/mgtOIeugXw\na1yZLSnr6imZJMQClDwRqxKLxQgL742w8LZGnji3hYuXcB1Ch5SXlaJHaHfMTpxpUF6sUCDvQD6m\nP/lH8iSRiBEUGGDvENvNNSgctcVXuA7Doaga1XCVSrkOgxDeoDFPhHTQvtw9XIdgE6a6cSQPtUQB\ngFDAr9uIgEdjswghjolangjpoOvXrnIdgtW5u3ugysSEkzeKbkEuN+zu0rE6e4VlFfKu/jh75CAm\nRA3iOhSbKrtbid9v3DIoq62rx+Wi29BotAbJcaO6CeOHP2rvEAnhLUqeiE3pdDr88vM5+IdHch0K\nsYC3jw8qq4znSzr4w49Y8qfXDcr49uRaUGAA6hoacKeyGl18vLgOx2Y+25mL6CkzDMpC3FwRPSvZ\nZAsiIcR8lDwRm2JZFiePH8M0Sp545+Herbr6eni4u4NhGINyoUAIlmV51R22YMlSZGz5Hyybl8R1\nKDYjEAgwbIhzt64RwhV+DVYghNgNIzRMkpqamuBu4oksoZB/txGZzBNNGg3XYdiUWMS0XYkQ0i78\nu+sRQgghhHCIkidCOigmLp7rEAghhNgRjXkiNtfQ0AD1Q+ueSVqZU0an06G+rhZisQRSF8dfbyu8\nt3POaWVyIDjPBoe3xolOxQjfBvETwjeUPBGbYhgGffr2xaFd/6cvE4lEmL9wkVHdJrUa6Zv/BaFQ\nCA8PD5SXlWFY1AiEDR1jz5ANsCxr9EEkEAj0g6OdbamV+3Q6HRpUhiuqyzw9UVpewVFE1qfRanHp\n+k3069Wd61Cs7v++O4AJI4ZwHQYhTouSJ2Jz8dNmtF0JgFgiwYvLXjIoO3Tge1w7cxTjoicY1T/1\n0wmcPnXSqHxY1AgMjxphVH7yxHGcOX3KqDxqxEgMHR5lVP7T8WMoOHPaaMHb4SNGYsiw4W2eD5+d\nPpiDiePHGpQJhUKEBAei8sYF+PSM4Cgy63l5xev47F/rcf7ydcyeEs11OFZz47YC1bW1GDAujutQ\nCHFalDwRhzZh0uQWtw0fMRLDR4w0e19RI0chauQos+uPGDUaI0aNNru+M/nl/AX8Ke0Fo/Ipkydi\nT94BJDhB8sQwDBanvYItm/6J26UVCAnw5zokq9i5/0e8+Mp/cR0GIU6NBowTQowwQsbkvE0ikcjp\nxtP4ecmg1fFrlvTWCAQwWtCZEGJdlDwRQgghhFiAkidCCCGEEAvQmCdCSKfGMAy+2ptvsM6dWCRC\n//Ae6B8WCqmEX11gTtarSohDouSJENKpTU2eh2ql0qBMpWrE7yd/wMaMAohEDBYmxkPmYbw0DZc0\nWi227tqHe0olmAeWyBEK+bPGICF8RckTIcRsLQ0W59vCwA9iGAa+Pj5G5UGzUjABQMWdO/ifTzaD\nEQqNFku2hSaNFkP698bEEUPAMH8kRfUNKpy5eBm19SpoNBqc/PU3zJsRg7CoaNsHRQgxQMkTIcRs\n5RV34N/Fz6AsathgHPjhR8RMGM9RVLbl36UL/rTyDbsdj2VZHN+bhf/3eSaEAgGEQgE0Wh1cpRIM\ni3wEwf2HAgBikuZCJKJbOLGNittKlN6rt9r+GF/HarntKPrLI4QYEQoF0Gq1YBjGoPz3K1fRPWKo\nQVm/kROx4W9/xeTocbxtfXIkAoEAo+MTMTo+ketQCCEtoKftCCFGXFxcjJZnAQBFaRkCg4INygQC\nAcLDeqJYUWqv8AghhFMdTp52796NmTNnIiIiAhkZGS3WO3nyJAYNGoTExEQkJCQgJSWlo4cmhNhI\nay1IpraJRSJ6zIsQ0ml0uNuuf//++Mc//oH09PQ264aHh+Prr7/u6CEJIYQQQjjT4eQpPDwcQOvf\nVO9ztmUdCCGEENL52HXAeFFREZKSkiAWizF37lwkJCS0WFepVEL50NwrDMMgMDDQ1mES0ump1U0Q\nCox79QUCIbRajVG5VquDgOYXIsRmFAoFtFqtQZlMJoNMJuMoos6tzeQpKSkJCoXCoOz+nC7Hjh0z\n++maiIgI5Ofnw8PDA7dv38bChQshl8sxapTpVe63bt2KjRs3GpQFBwfj4MGDZh2PENI+LMuiWqmE\nm5ur0bbeYT1R+ts5hA19zKBcUVoGub+/vUIkpNNJTU1FcXGxQVlaWhqWLVvGUUSdW5vJ086dO61y\nIHd3d/3PISEhmDx5MgoKClpMnhYsWIDERMNHdR9+bJoQYn1XT/+IYYMfNblt0MBI7MreY5Q8aXVa\nmnOIEBvKyMgw2fJEuGG3u11FRQX8//PNtKqqCkeOHMErr7zSYn1qjiSEG79dvoK4yRNMbvPx9oZS\nWWNUTvM7EWJbNGTFsXQ4ecrJycEHH3wApVKJgwcPIj09HVu2bEFYWBjWr18PuVyOlJQU5OXlYfv2\n7RCLxdBoNEhMTMTEiROtcQ6EECtqVKshlUhNblOr1XBxMd7GCBk0NTVBLObXIrqEENIeHU6epk2b\nhmnTppnctnz5cv3PqampSE1N7ejhCCE21qTRtNgFp25qMrnNv4sf7lVWQd6Vxj0RQpwfzTBOCDHS\nWi+cyS466rYjhHQilDwRQgxUVVW32PLECBloHhq0CgAyTw+UlpXbOjRCCHEIlDwRQvT2f7MNgwdG\ntjh2ydXVBaoG4zXvRsUlIOu7PTQRLiGkU6DkiRACACg8dxzVyhrETW75QQ6BQGAyQRKJRIiZOB47\nvsmyZYiEEOIQKHkihAAAgvsPRdGt22hoaGixTk1tLdzc3Exuixg9Gd5eXsjes89WIRJCiEOg5IkQ\nAgAQSyR46vnl2PzpFy3WuXzlGh7pHdbi9lFTEvDrxUvQ6XS2CJEQQhwCJU+EEL2uXeWtbq9WKiH2\nC261jq+PN9RqtTXDIoQQh0LJEyGEEEKIBSh5IoToVVbeQ1NTU4vbxWIxmqhViRDSydFKnoQQAEBt\nbQ22fPR3vPan5S3W8fH2QklpiR2jIsR5KUrLcObsz2BZFhKJGMOGDIafrw/XYREzUPJECAEAfP3Z\nZvxp2QvwcHdvsY6nhwdqaq7ZMSpCnFPegUO4cu0GRk6aCkbEgKm9g8ydWZB37YqkmaaXPCOOg5In\nQggAgGVZ+Hh7t1pHKBQCoIkwCekoZU0NJsx8AsHBIf8pCcP00L74ftd2TuMi5qExT4QQQgghFqDk\niRBCCCHEApQ8EUIs0tb6dbS+HSFta+nPREd/P7xAY54IIWYTiUTQaLSt1mlQqSCVSu0UESH8pCgt\nxQT/rgZlnjIZ7t2r5Cgi/lGpVHj99ddx4cIFiEQirFixAtHR0Ub1fvvtN6xatQosy0Kj0WDw4MH4\ny1/+ol8A/dKlS3j33XdRWVkJgUCA1157DWPHjm312JQ8EULMJpVKoGpsbLOeQCCwQzSE8FeTRgOx\nRGJQdn/h7a927daXuXt4ICH5SXuHxwtbtmyBh4cH8vLyUFRUhNTUVOzfvx+urq4G9Xr16oXMzEyI\nRM0pz/Lly7Fjxw7MmzcPDQ0NWLZsGT788EMMHDgQOp0ONTU1bR6bkidCiNmaW540rdYRgBInQtoi\nFJgeNTP/xVdQWflH65NYTB/TLcnNzcW6desAAKGhoYiMjMThw4cRFxdnUE/yQJKqVquhUqn0X/C+\n++47DBs2DAMHDgTQ/ESxl5dXm8emq0IIMZs5LUrU6kRI21r6O3F1c4Orm5v+tdDJ/p4UCgW0WsOu\nf5lMBplMZvG+SkpKEBQUpH8dGBgIhUJhsm55eTmWLFmCW7duYfz48UhJSQEAXL16FQzDYMmSJaio\nqEBERARWrFjRZjyUPBFCCCHEwN3CS6goq7ba/lzkXgBikZqaiuLiYoNtaWlpWLZsmdF7kpKSjJIh\nlmUhEAhw9OhRi47ftWtXZGVlQaVS4dVXX0VeXh6mTp0KrVaLEydOIDMzE35+fnjvvffw/vvv4733\n3mt1f5Q8EUIIIcQuMjIyTLY8mbJz585W9xUcHIySkhL4+DQvaaNQKDBy5MhW3+Pi4oL4+HhkZ2dj\n6tSpCAoKwsiRI+Hn5wcAmD59Ot544402z4OSJ0IIgOaxANag1bX+NB4hnQnLsjj20ynU1taCZYFG\ndSPq6uo7bfd2YGCg1fYVFxeHHTt2YM2aNSgsLMT58+fx4YcfGtW7desWAgICIBaLoVarceDAAfTp\n0wcAEB8fjyVLlqCurg7u7u748ccf0bdv3zaPzcvkKfOzzVBWVxn88lVVV+PxWTPQfeAIDiMjhB++\n35mBW7cNF/gN7d7NKvvW6Vh90zohnY1Op0O1UgkA+P3KNez7/iCix46BT+gjAAAXqQtc3d3g59eF\nyzCdwqJFi7By5UrExsaCYRisXbsWbv8ZL7Z+/XrI5XKkpKTg7NmzSE9PB8Mw0Gq1iIqKwtKlSwE0\nJ3OLFy/GnDlzIBQKERISgrVr17Z5bAHLwxnt1NUVwEPfblmWxf9s+RyDHx2IfiMncBQZIfzwyT/f\nx5+XvWjx+2pqa/Hl17uQvGhpi3Xyd+9Av0f6oE94WEdCJIR3tFot3vngQ3QPCYZAKERwUCCGT5re\noS8SQoEAfp6ubVe0skcXfIRbVhzz1E3uhZ+3vmS1/XGNly1PpggEAryweCH+uWkzJU+EtIERMu16\nnzkfAl7BYSivuEPJE+l0GhvVCJDLkfT081yHQmyMlmchhBBCrIAF7zpySDtR8kRIJ9TeQd1qtVo/\nS29LevTqhZ9On2nX/gnhs+w9+xA1dDDXYRA7cJpuO0KI+fo90gf/3LTZoMxL5omFT7W+DERdXT1k\nnh6t1vHx8UWP7t3x47ETiOzfF0KBEDKZJw0gJ7xXV1+PG4VFJrfdraxEtbIGvYaMsXNUhAuUPBHS\nCY2JT8KYeMOyf/+/1ieFA4D6hga4uLi0WW9S4lzs/yYD124UQqfT4eJvv+O/3327veES4hDW/O2/\nMWnCOJNfBIQCAZ5YSGOdOgtKngghAKBfYbwt5i7REvvEPP1r5fq/tzsuQhxFYIAcI2Jmch0GcQA0\n5okQQgghxAKUPBFCCCGEWICSJ0IIIYQQC1DyRAghhBBiAUqeCCE2x8NVoAgxwLIsLXpN9DrN03Ys\nyyLrf9NRVVUNlmUxZ/FSuLm7cx0WIQ7D3ASnPYnQoIGR+GzbdjyZ/LhBuYhhwDDtWyqGEFvJO3AI\nNbW1+tdNTRpcvnoN06fEchgVcSSdJnn6Mn0DRkUNx6MDIqAoLcOu/01H6vMvcx0WIQ7DnCkIGCED\nVmd58jRkwlR4nPoBW7ZuMyivq6vHqy+nWbw/Qmzpp9MFeHz+4j8KBALEzX4KQiF11pBmnSJ5Kisr\nhUgkwqMDIgA0z9VRV1fPcVSEOBZzWpQEAkDH6tq1/z7Dx6PP8PEGZZ9+9EG79kWIrWg0Gnh5yRAc\n0o3rUIgD6xRptFajhZdMZlBm7oSAhHQWZi2fYuUlVmjJFuKI6PeStMXpkidVY6NRWWXRJaP1uISM\nEPV1dfYKixCH16TRtF2JBn4TJ6fVasFQ9xxpg9N12/XqEYrPHloKQiqV4PlFTxuUzU6Yiaxtn+DJ\n514y2gfLsjh/9HuUlZcblD8aGQn/Po9aO2RC7O7L9I1oaGgwKHs0sn+b72tUN0EqkdgqLEI4d+fe\nPfj5+nIdBnFwTpc8JSfNMqteYIAcLMvi7p078OvSRV9+7cwR7Nn/PcaOHomooUMM3vPl17uw5M+U\nPBF+qygvB8MI8fLS5yx+b6O6ERJKnogTq7hzF/5d/LgOgzg4p0ueLBEe1gsV5WUGydOJU6fx6ktp\nJr9d0zgp4gzuVFQgtHs7B8OyoCeOiFNTq5sglUq5DoM4OLoLEkIIIYRYgJInQgghhBALUPJECCGE\nEGIBSp4IIYQQQizQ4QHja9aswfHjxyGVSuHm5oZVq1YhMjLSZN1NmzYhKysLAoEACQkJePHFFzt6\neEIIIYQQu+pw8jR+/Hi88cYbYBgG+fn5eOWVV7B//36jeqdPn0ZeXh5ycnLAsixmz56NqKgoDBs2\nrKMhENJp6XQ6/PLzOei05q32rqsuw5HjP2HW9Ph2HU8qlaJGobDoPSzLouLKzzh89DhqamoNtqnV\n6nbFQYitqNVqerKatMkqydN9gwYNQllZmcl6e/bsQUJCgn6OmISEBOTm5lLyREgHfPXpxwgKCICP\nj7dZ9V29vfHGq69AJGrfn37fPuH4Lncfxk5rvZ5Go8HfVr+KkKAg6FgdevUIReKMqfD18WnXcQmx\nlyvXriN6xhNch0EcnFXnedq2bRuio6NNbispKcGIESP0rwMDA3H69OkW96VUKqFUKg3KGIZBYGAg\nIBBaZbSWq5s7pFIJhA+sYySTeUEgZAAhY1Rf5uVlUJcQLl0pOIrQ0B6ImzzBbscUChnIAwOgblTB\nxcW1xXrFlwrwRGICRkbRlyPCLzoW6OLXpe2KdnL/M0ehUED7UAuzTCaD7KF1W60lqIt192vt/XFN\nwLaxlHpSUhIUDzXTsywLgUCAY8eO6RdQzMnJwcaNG5GRkQFfE1PbP//880hMTERcXBwAIDc3F9nZ\n2fj4449NHnfDhg3YuHGjQdmQIUOwfft288+OEEIIcQJz585FQUGBQVlaWhqWLVvGUUSdHGsFeXl5\nbExMDFtSUtJinbfffpv99NNP9a+3bNnCrlmzpsX61dXV7K1btwz+nTp1ip0zZ06rx+GzkpISdsKE\nCU55fs58bixL58d3dH785cznxrLN5zdnzhz21KlTRp+J1dXVXIfXaXW42+7QoUN4//338fnnnzd3\nqbVgypQpePfdd5GamgqdToesrCy8+eabLdZvqTmyoKDAqOnSWWi1WhQXFzvl+TnzuQF0fnxH58df\nznxuQPP5FRQUICAgACEhIVyHQ/6jw8nTqlWrIJFIsHz5cn133ueffw4vLy+sXr0akyZNwoQJExAV\nFYWYmBhMm9Y80jQhIYEGixNCCCGEdzqcPB0/frzFbe+8847B67S0NKSlpXX0kIQQQgghnKEZxgkh\nhBBCLMD89a9//SvXQVhCKpVixIgRkEqlXIdiE858fs58bgCdH9/R+fGXM58b4Pznx0dtTlVACCGE\nEEL+QN12hBBCCCEWoOSJEEIIIcQClDwRQgghhFjA4ZOnNWvWID4+HgkJCXjyySdx/vz5Futu2rQJ\nMTExiI2NbXHZF0eze/duzJw5ExEREcjIyGix3smTJzFo0CAkJiYiISEBKSkpdoyyfcw9NwDIzMxE\nbGwsYmNjjaa4cFQqlQqvvPIKYmNjMXXqVOTn55usx6drV1hYiDlz5mDKlCmYM2cObt68aVRHp9Ph\n7bffRkxMDOLi4vDVV19xEGn7mHN+GzduxOjRo5GYmIjExESsXbuWg0gtt27dOkyaNAl9+/bF1atX\nTdbh87Uz5/z4eu2qqqqwZMkSxMfHY9asWVi+fDkqKyuN6pl7zyF2wPEM523Kz89nNRoNy7Ise+jQ\nIXby5Mkm6506dYqdOXMm29jYyKpUKnbGjBnsqVOn7Blqu1y5coW9evUq+9prr7Hbtm1rsd5PP/3E\nPv7443aMrOPMPbdbt26x48aNYysrK1mWZdlnnnmGzcrKsleY7bZx40Z29erVLMuybGFhITtmzBi2\nvr7eqB6frt38+fPZ7OxslmVZ9ttvv2Xnz59vVGfXrl3sokWLWJZl2bt377Ljxo1ji4uL7Rpne5lz\nfhs2bGDXrVtn79A67MyZM2xpaSk7ceJE9sqVKybr8PnamXN+fL12VVVV7MmTJ/Wv161bx65atcqo\nnrn3HGJ7Dt/yNH78eDAMAwAYNGgQysrKTNbbs2cPEhISIJFIIJVKkZCQgNzcXHuG2i7h4eEICwvT\nL7DcGpZnD0aae2779u1DTEwMvL29AQDJycm8uHa5ubmYM2cOACA0NBSRkZE4fPiwybp8uHb37t3D\npUuX9KsATJ8+HRcvXjT6Bpybm4vk5GQAgK+vLyZPnoy9e/faPV5LmXt+AD+u18OGDBkCuVzeaux8\nvXaAeecH8PPaeXl5Yfjw4frXgwYNgkKhMKpnyT2H2JbDJ08P2rZtG6Kjo01uKykpQVBQkP51YGCg\nyV8+PisqKkJSUhJSUlKQlZXFdThWo1AoeHntLPmd48O1UygUkMvl+mRXKBSia9euKC0tNajH1781\nc88PaP6QmjVrFhYtWoRz587ZO1Sb4eu1swTfrx3Lsti+fTsmTZpktK0zXD++6PDyLB2VlJRkdPHZ\n/6yRd+zYMf2NLicnBzk5OW2OnXE05p5fWyIiIpCfnw8PDw/cvn0bCxcuhFwux6hRo2wRtlmsdW6O\nqrXzO3r0qNn7ccRrR1o2d+5cvPDCC2AYBseOHcOLL76I3NxceHl5cR0aaYMzXLs1a9bA3d0dqamp\nRtv4fk91JpwnTzt37myzzv79+/HRRx9h69at8PX1NVknKCgIJSUl+tcKhQKBgYFWi7O9zDk/c7i7\nu+t/DgkJweTJk1FQUMDpB7C1zi0wMBDFxcX613y5dsHBwSgpKYGPjw+A5rhHjhxpVM8Rr50pgYGB\nKCsr0yeIOp0O5eXlCAgIMKh3/28tMjISQPN5BwcHcxGyRcw9Pz8/P/3Po0ePRkBAAK5cueIUC5nz\n9dqZi+/Xbt26dbh58yY2b95scvv969fWPYfYnsN32x06dAjvv/8+tmzZ0uoH6pQpU5CVlQW1Wg2V\nSoWsrCzEx8fbMVLbqqio0P9cVVWFI0eOoF+/fhxGZD2xsbE4cOAAKisrodPpkJmZiSlTpnAdVpvi\n4uKwY8cOAM1PcZ0/fx5jx441qseXa+fr64u+ffsiOzsbAJCdnY3+/fvrb9T3TZkyBZmZmWBZFvfu\n3cOBAwcQGxvLRcgWMff8HhxXeenSJZSUlKBnz552jdVW+HrtzMXna/ePf/wDFy9exMcffwyRyHS7\nhrn3HGJ7Dr88y6hRoyCRSODr66v/xvj555/Dy8sLq1evxqRJkzBhwgQAzY+pfvvttwCAhIQELF26\nlMvQzZKTk4MPPvgASqUSEokErq6u2LJlC8LCwrB+/XrI5XKkpKQgIyMD27dvh1gshkajQWJiIp55\n5hmuw2+VuecGNE9VkJ6eDoFAgMceewx/+ctfHL6JuqGhAStXrsSlS5fAMAxWrFih/13k67W7fv06\nVq5cCaVSCS8vL3zwwQcIDQ3FkiVL8NJLLyEiIgI6nQ5r1qzB0aNHIRAIMomqmgAAALNJREFU8Oyz\nz2L27Nlch24Wc85v5cqVuHDhAoRCISQSCZYvX86LD6h33nkH+/fvx927d+Ht7Q0fHx9kZ2c7zbUz\n5/z4eu2uXr2KGTNmoEePHvr167p164YNGzYgISEB6enp8Pf3b/WeQ+zL4ZMnQgghhBBH4vDddoQQ\nQgghjoSSJ0IIIYQQC1DyRAghhBBiAUqeCCGEEEIsQMkTIYQQQogFKHkihBBCCLEAJU+EEEIIIRb4\n/ylMBXehJVQ0AAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - }, - "output_type": "display_data" - } - ], - "source": [ - "n_trees = 22 #@param {type: \"slider\", min: 1, max: 80, step: 1}\n", - "\n", - "est = tf.estimator.BoostedTreesRegressor(fc, n_batches_per_layer=1, n_trees=n_trees)\n", - "est.train(train_input_fn, max_steps=500)\n", - "clear_output()\n", - "plot_contour(xi, yi, predict(est))\n", - "plt.text(-1.8, 2.1, '# trees: {}'.format(n_trees), color='w', backgroundcolor='black', size=20)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5WcZ9fubh1wT" - }, - "source": [ - "As you increase the number of trees, the model's predictions better approximates the underlying function." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cj8u3NCG-IKX" - }, - "source": [ - "![](https://www.tensorflow.org/images/boosted_trees/boosted_trees_ntrees.gif)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SMKoEZnCdrsp" - }, - "source": [ - "## Conclusion" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZSZUSrjXdw9g" - }, - "source": [ - "In this tutorial you learned how to interpret Boosted Trees models using directional feature contributions and feature importance techniques. These techniques provide insight into how the features impact a model's predictions. Finally, you also gained intution for how a Boosted Tree model fits a complex function by viewing the decision surface for several models." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "boosted_trees_model_understanding.ipynb", - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/estimator/keras_model_to_estimator.ipynb b/site/en/tutorials/estimator/keras_model_to_estimator.ipynb index c1a0aa56994..be97a38b6eb 100644 --- a/site/en/tutorials/estimator/keras_model_to_estimator.ipynb +++ b/site/en/tutorials/estimator/keras_model_to_estimator.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "d6p8EySq1zXZ" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "KsOkK8O69PyT" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "F1xIRPtY0E1w" }, "source": [ @@ -47,84 +43,60 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "r61fkA2i9Y3_" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/estimator/keras_model_to_estimator\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/estimator/keras_model_to_estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/estimator/keras_model_to_estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/estimator/keras_model_to_estimator.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "ZaGcclVLwqDS" + "id": "Dhcq8Ds4mCtm" }, "source": [ - "## Overview\n", - "\n", - "TensorFlow Estimators are fully supported in TensorFlow, and can be created from new and existing `tf.keras` models. This tutorial contains a complete, minimal example of that process." + "> Warning: TensorFlow 2.15 included the final release of the `tf-estimator` package. Estimators will not be available in TensorFlow 2.16 or after. See the [migration guide](https://tensorflow.org/guide/migrate/migrating_estimator) for more information about how to convert off of Estimators." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "epgfaZmO2vF0" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "oS6nyl8EvevY" + "id": "ZaGcclVLwqDS" }, - "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" + "## Overview\n", + "\n", + "TensorFlow Estimators are supported in TensorFlow, and can be created from new and existing `tf.keras` models. This tutorial contains a complete, minimal example of that process.\n", + "\n", + "Note: If you have a Keras model, you can use it directly with [`tf.distribute` strategies](https://tensorflow.org/guide/migrate/guide/distributed_training) without converting it to an estimator. As such, `model_to_estimator` is no longer recommended." ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5z45VJDkvcql" + "id": "epgfaZmO2vF0" }, - "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" + "## Setup" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Qmq4FzaztASN" }, "outputs": [], @@ -138,7 +110,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9ZUATGJGtQIU" }, "source": [ @@ -148,7 +119,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rR-zPidHyzcb" }, "source": [ @@ -161,10 +131,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "p5NSx38itD1a" }, "outputs": [], @@ -172,14 +140,13 @@ "model = tf.keras.models.Sequential([\n", " tf.keras.layers.Dense(16, activation='relu', input_shape=(4,)),\n", " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", + " tf.keras.layers.Dense(3)\n", "])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ABgo9-8BtYNs" }, "source": [ @@ -188,22 +155,20 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "nViACuBDtVEC" }, "outputs": [], "source": [ - "model.compile(loss='categorical_crossentropy', optimizer='adam')\n", + "model.compile(loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " optimizer='adam')\n", "model.summary()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pM3Cx5Fm_sHI" }, "source": [ @@ -217,10 +182,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "H0DpLEop_x0o" }, "outputs": [], @@ -236,7 +199,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UR1vRw1bBFjo" }, "source": [ @@ -245,10 +207,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WO94bGYKBKRv" }, "outputs": [], @@ -261,7 +221,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "svdhkQ4Otcv0" }, "source": [ @@ -274,16 +233,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "roChngg8t7il", - "toc_visible": true + "id": "roChngg8t7il" }, "outputs": [], "source": [ - "model_dir = \"/tmp/tfkeras_example/\"\n", + "import tempfile\n", + "model_dir = tempfile.mkdtemp()\n", "keras_estimator = tf.keras.estimator.model_to_estimator(\n", " keras_model=model, model_dir=model_dir)" ] @@ -291,7 +248,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "U-8ekW5It_2w" }, "source": [ @@ -300,15 +256,13 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ouIkVtp9uAg5" }, "outputs": [], "source": [ - "keras_estimator.train(input_fn=input_fn, steps=25)\n", + "keras_estimator.train(input_fn=input_fn, steps=500)\n", "eval_result = keras_estimator.evaluate(input_fn=input_fn, steps=10)\n", "print('Eval result: {}'.format(eval_result))" ] @@ -318,13 +272,10 @@ "colab": { "collapsed_sections": [], "name": "keras_model_to_estimator.ipynb", - "private_outputs": true, - "provenance": [], - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", - "language": "python", "name": "python3" } }, diff --git a/site/en/tutorials/estimator/linear.ipynb b/site/en/tutorials/estimator/linear.ipynb index e2a13fb317b..a26ffe2df4f 100644 --- a/site/en/tutorials/estimator/linear.ipynb +++ b/site/en/tutorials/estimator/linear.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OoasdhSAp0zJ" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "cIrwotvGqsYh" }, "outputs": [], @@ -37,44 +34,52 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "C81KT2D_j-xR" }, "source": [ "# Build a linear model with Estimators\n", "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/estimator/linear\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/estimator/linear.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/estimator/linear.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/estimator/linear.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JOccPOFMm5Tc" + }, + "source": [ + "> Warning: TensorFlow 2.15 included the final release of the `tf-estimator` package. Estimators will not be available in TensorFlow 2.16 or after. See the [migration guide](https://tensorflow.org/guide/migrate/migrating_estimator) for more information about how to convert off of Estimators." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tUP8LMdYtWPz" }, "source": [ "## Overview\n", "\n", - "This end-to-end walkthrough trains a logistic regression model using the `tf.estimator` API. The model is often used as a baseline for other, more complex, algorithms.\n" + "This end-to-end walkthrough trains a logistic regression model using the `tf.estimator` API. The model is often used as a baseline for other, more complex, algorithms.\n", + "\n", + "Note: A Keras logistic regression example is [available](https://tensorflow.org/guide/migrate/tutorials/keras/regression) and is recommended over this tutorial.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vkC_j6VpqrDw" }, "source": [ @@ -83,29 +88,23 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "rutbJGmpqvm3" }, "outputs": [], "source": [ - "!pip install sklearn" + "!pip install sklearn\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "54mb4J9PqqDh" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "import os\n", "import sys\n", "\n", @@ -119,7 +118,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "fsjkwfsGOBMT" }, "source": [ @@ -129,19 +127,12 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "bNiwh-APcRVD" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow.compat.v2.feature_column as fc\n", "\n", "import tensorflow as tf" @@ -149,10 +140,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "DSeMKcx03d5R" }, "outputs": [], @@ -167,7 +156,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jjm4Qj0u7_cp" }, "source": [ @@ -177,7 +165,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UrQzxKKh4d6u" }, "source": [ @@ -186,10 +173,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "rTjugo3n308g" }, "outputs": [], @@ -199,10 +184,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "y86q1fj44lZs" }, "outputs": [], @@ -213,7 +196,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8JSa_duD4tFZ" }, "source": [ @@ -222,10 +204,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Fs3Nu5pV4v5J" }, "outputs": [], @@ -236,7 +216,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RxCA4Nr45AfF" }, "source": [ @@ -245,10 +224,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "RYeCMm7K40ZN" }, "outputs": [], @@ -259,7 +236,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DItSwJ_B5B0f" }, "source": [ @@ -268,10 +244,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "b03dVV9q5Dv2" }, "outputs": [], @@ -282,7 +256,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rK6WQ29q5Jf5" }, "source": [ @@ -291,10 +264,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dgpJVeCq5Fgd" }, "outputs": [], @@ -305,7 +276,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FXJhGGL85TLp" }, "source": [ @@ -314,10 +284,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "lSZYa7c45Ttt" }, "outputs": [], @@ -328,12 +296,28 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "qCHvgeorEsHa" + }, + "source": [ + "## Feature Engineering for the Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Dhcq8Ds4mCtm" + }, + "source": [ + "> Warning: The tf.feature_columns module described in this tutorial is not recommended for new code. Keras preprocessing layers cover this functionality, for migration instructions see the [Migrating feature columns guide](https://www.tensorflow.org/guide/migrate/migrating_feature_columns). The tf.feature_columns module was designed for use with TF1 Estimators. It does fall under our [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "VqDKQLZn8L-B" }, "source": [ - "## Feature Engineering for the Model\n", - "Estimators use a system called [feature columns](https://www.tensorflow.org/guide/feature_columns) to describe how the model should interpret each of the raw input features. An Estimator expects a vector of numeric inputs, and *feature columns* describe how the model should convert each feature.\n", + "Estimators use a system called [feature columns](https://www.tensorflow.org/tutorials/structured_data/feature_columns) to describe how the model should interpret each of the raw input features. An Estimator expects a vector of numeric inputs, and *feature columns* describe how the model should convert each feature.\n", "\n", "Selecting and crafting the right set of feature columns is key to learning an effective model. A feature column can be either one of the raw inputs in the original features `dict` (a *base feature column*), or any new columns created using transformations defined over one or multiple base columns (a *derived feature columns*).\n", "\n", @@ -343,7 +327,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "puZFOhTDkblt" }, "source": [ @@ -352,10 +335,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "GpveXYSsADS6" }, "outputs": [], @@ -376,19 +357,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Gt8HMtwOh9lJ" }, "source": [ - "The `input_function` specifies how data is converted to a `tf.data.Dataset` that feeds the input pipeline in a streaming fashion. `tf.data.Dataset` take take in multiple sources such as a dataframe, a csv-formatted file, and more." + "The `input_function` specifies how data is converted to a `tf.data.Dataset` that feeds the input pipeline in a streaming fashion. `tf.data.Dataset` can take in multiple sources such as a dataframe, a csv-formatted file, and more." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "qVtrIHFnAe7w" }, "outputs": [], @@ -409,7 +387,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "P7UMVkQnkrgb" }, "source": [ @@ -418,10 +395,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8ZcG_3KiCb1M" }, "outputs": [], @@ -438,7 +413,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lMNBMyodjlW3" }, "source": [ @@ -447,10 +421,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "IMjlmbPlDmkB" }, "outputs": [], @@ -462,7 +434,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "f4zrAdCIjr3s" }, "source": [ @@ -471,10 +442,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "1VXmXFTSFEvv" }, "outputs": [], @@ -486,7 +455,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MEp59g5UkHYY" }, "source": [ @@ -495,10 +463,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "aGXjdnqqdgIs" }, "outputs": [], @@ -514,7 +480,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3tOan4hDsG6d" }, "source": [ @@ -524,21 +489,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NOG2FSTHlAMu" }, "source": [ - "Now you reached an accuracy of 75%. Using each base feature column separately may not be enough to explain the data. For example, the correlation between gender and the label may be different for different gender. Therefore, if you only learn a single model weight for `gender=\"Male\"` and `gender=\"Female\"`, you won't capture every age-gender combination (e.g. distinguishing between `gender=\"Male\"` AND `age=\"30\"` AND `gender=\"Male\"` AND `age=\"40\"`).\n", + "Now you reached an accuracy of 75%. Using each base feature column separately may not be enough to explain the data. For example, the correlation between age and the label may be different for different gender. Therefore, if you only learn a single model weight for `gender=\"Male\"` and `gender=\"Female\"`, you won't capture every age-gender combination (e.g. distinguishing between `gender=\"Male\"` AND `age=\"30\"` AND `gender=\"Male\"` AND `age=\"40\"`).\n", "\n", "To learn the differences between different feature combinations, you can add *crossed feature columns* to the model (you can also bucketize age column before the cross column):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "AM-RsDzNfGlu" }, "outputs": [], @@ -549,7 +511,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DqDFyPKQmGTN" }, "source": [ @@ -558,10 +519,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "s8FV9oPQfS-g" }, "outputs": [], @@ -578,7 +537,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rwfdZj7ImLwb" }, "source": [ @@ -588,7 +546,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8_eyb9d-ncjH" }, "source": [ @@ -597,10 +554,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wiScyBcef6Dq" }, "outputs": [], @@ -614,7 +569,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UEHRCd4sqrLs" }, "source": [ @@ -623,10 +577,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "kqEjsezIokIe" }, "outputs": [], @@ -648,10 +600,7 @@ "colab": { "collapsed_sections": [], "name": "linear.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/estimator/premade.ipynb b/site/en/tutorials/estimator/premade.ipynb index 62e285754bc..dc81847c7cd 100644 --- a/site/en/tutorials/estimator/premade.ipynb +++ b/site/en/tutorials/estimator/premade.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1Z6Wtb_jisbA" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "QUyRGn9riopB" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "H1yCdGFW4j_F" }, "source": [ @@ -47,46 +43,50 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PS6_yKSoyLAl" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/estimator/premade\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/estimator/premade.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/estimator/premade.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/estimator/premade.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "stQiPWL6ni6_" + }, + "source": [ + "> Warning: TensorFlow 2.15 included the final release of the `tf-estimator` package. Estimators will not be available in TensorFlow 2.16 or after. See the [migration guide](https://tensorflow.org/guide/migrate/migrating_estimator) for more information about how to convert off of Estimators." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "R4YZ_ievcY7p" }, "source": [ - "\n", "This tutorial shows you\n", - "how to solve the Iris classification problem in TensorFlow using Estimators. An Estimator is TensorFlow's high-level representation of a complete model, and it has been designed for easy scaling and asynchronous training. For more details see\n", + "how to solve the Iris classification problem in TensorFlow using Estimators. An Estimator is a legacy TensorFlow high-level representation of a complete model. For more details see\n", "[Estimators](https://www.tensorflow.org/guide/estimator).\n", "\n", - "Note that in TensorFlow 2.0, the [Keras API](https://www.tensorflow.org/guide/keras) can accomplish many of these same tasks, and is believed to be an easier API to learn. If you are starting fresh, we would recommend you start with Keras. For more information about the available high level APIs in TensorFlow 2.0, see [Standardizing on Keras](https://medium.com/tensorflow/standardizing-on-keras-guidance-on-high-level-apis-in-tensorflow-2-0-bad2b04c819a).\n", - "\n" + "Note: In TensorFlow 2.0, the [Keras API](https://www.tensorflow.org/guide/keras) can accomplish these same tasks, and is believed to be an easier API to learn. If you are starting fresh, it is recommended you start with Keras.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8IFct0yedsTy" }, "source": [ @@ -97,22 +97,12 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "jPo5bQwndr9P" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", "import tensorflow as tf\n", "\n", "import pandas as pd" @@ -121,7 +111,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "c5w4m5gncnGh" }, "source": [ @@ -148,10 +137,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "lSyrXp_He_UE" }, "outputs": [], @@ -163,7 +150,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "j6mTfIQzfC9w" }, "source": [ @@ -172,10 +158,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "PumyCN8VdGGc" }, "outputs": [], @@ -192,7 +176,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wHFxNLszhQjz" }, "source": [ @@ -201,10 +184,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WOJt-ML4hAwI" }, "outputs": [], @@ -215,7 +196,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jQJEYfVvfznP" }, "source": [ @@ -224,10 +204,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "zM0wz2TueuA6" }, "outputs": [], @@ -242,7 +220,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jZx1L_1Vcmxv" }, "source": [ @@ -253,8 +230,8 @@ "`tf.estimator`\n", "(for example, `LinearRegressor`) to implement common ML algorithms. Beyond\n", "those, you may write your own\n", - "[custom Estimators](https://www.tensorflow.org/guide/custom_estimators).\n", - "We recommend using pre-made Estimators when just getting started.\n", + "[custom Estimators](https://www.tensorflow.org/guide/estimator#custom_estimators).\n", + "It is recommended using pre-made Estimators when just getting started.\n", "\n", "To write a TensorFlow program based on pre-made Estimators, you must perform the\n", "following tasks:\n", @@ -272,7 +249,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2OcguDfBcmmg" }, "source": [ @@ -297,10 +273,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "nzr5vRr5caGF" }, "outputs": [], @@ -317,12 +291,11 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NpXvGjfnjHgY" }, "source": [ "Your input function may generate the `features` dictionary and `label` list any\n", - "way you like. However, we recommend using TensorFlow's [Dataset API](https://www.tensorflow.org/guide/datasets), which can\n", + "way you like. However, It is recommended using TensorFlow's [Dataset API](https://www.tensorflow.org/guide/datasets), which can\n", "parse all sorts of data.\n", "\n", "The Dataset API can handle a lot of common cases for you. For example,\n", @@ -336,10 +309,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "T20u1anCi8NP" }, "outputs": [], @@ -359,7 +330,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xIwcFT4MlZEi" }, "source": [ @@ -372,7 +342,7 @@ "The `tf.feature_column` module provides many options for representing data\n", "to the model.\n", "\n", - "For Iris, the 4 raw features are numeric values, so we'll build a list of\n", + "For Iris, the 4 raw features are numeric values, so you'll build a list of\n", "feature columns to tell the Estimator model to represent each of the four\n", "features as 32-bit floating-point values. Therefore, the code to create the\n", "feature column is:" @@ -380,10 +350,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ZTTriO8FlSML" }, "outputs": [], @@ -397,11 +365,10 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jpKkhMoZljco" }, "source": [ - "Feature columns can be far more sophisticated than those we're showing here. You can read more about Feature Columns in [this guide](https://www.tensorflow.org/guide/feature_columns).\n", + "Feature columns can be far more sophisticated than those shown here. You can read more about Feature Columns in [this guide](https://www.tensorflow.org/guide/feature_columns).\n", "\n", "Now that you have the description of how you want the model to represent the raw\n", "features, you can build the estimator." @@ -410,7 +377,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kuE59XHEl22K" }, "source": [ @@ -421,7 +387,7 @@ "\n", "* `tf.estimator.DNNClassifier` for deep models that perform multi-class\n", " classification.\n", - "* `tf.estimator.DNNLinearCombinedClassifier` for wide \u0026 deep models.\n", + "* `tf.estimator.DNNLinearCombinedClassifier` for wide & deep models.\n", "* `tf.estimator.LinearClassifier` for classifiers based on linear models.\n", "\n", "For the Iris problem, `tf.estimator.DNNClassifier` seems like the best choice.\n", @@ -430,10 +396,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "qnf4o2V5lcPn" }, "outputs": [], @@ -450,7 +414,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tzzt5nUpmEe3" }, "source": [ @@ -466,7 +429,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rnihuLdWmE75" }, "source": [ @@ -477,10 +439,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4jW08YtPl1iS" }, "outputs": [], @@ -494,7 +454,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ybiTFDmlmes8" }, "source": [ @@ -508,7 +467,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HNvJLH8hmsdf" }, "source": [ @@ -521,10 +479,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "A169XuO4mKxF" }, "outputs": [], @@ -538,7 +494,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VnPMP5EHph17" }, "source": [ @@ -553,7 +508,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ur624ibpp52X" }, "source": [ @@ -567,10 +521,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wltc0jpgng38" }, "outputs": [], @@ -596,7 +548,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JsETKQo0rHvi" }, "source": [ @@ -607,10 +558,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Efm4mLzkrCxp" }, "outputs": [], @@ -628,10 +577,7 @@ "colab": { "collapsed_sections": [], "name": "premade.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/generative/adversarial_fgsm.ipynb b/site/en/tutorials/generative/adversarial_fgsm.ipynb index a5dcb363a10..0cce917ebf7 100644 --- a/site/en/tutorials/generative/adversarial_fgsm.ipynb +++ b/site/en/tutorials/generative/adversarial_fgsm.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XGCdmDAKpLuf" }, "source": [ @@ -14,15 +13,13 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "GF4d1XplpLGF" }, "outputs": [], - "source": [ + "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", "# You may obtain a copy of the License at\n", @@ -39,32 +36,30 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "W1L3zJP6pPGD" }, "source": [ "# Adversarial example using FGSM\n", "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/generative/adversarial_fgsm\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/generative/adversarial_fgsm.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/adversarial_fgsm.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/generative/adversarial_fgsm.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8dn1-g8BpPDx" }, "source": [ @@ -91,28 +86,19 @@ "* $\\theta$ : Model parameters.\n", "* $J$ : Loss.\n", "\n", - "An intriguing property here, is the fact that the gradients are taken with respect to the input image. This is done because the objective is to create an image that maximises the loss. A method to accomplish this is to find how much each pixel in the image contributes to the loss value, and add a perturbation accordingly. This works pretty fast because it is easy find how each input pixel contributes to the loss, by using the chain rule, and finding the required gradients. Hence, the gradients are used with respect to the image. In addition, since the model is no longer being trained (thus the gradient is not taken with respect to the trainable variables, i.e., the model parameters), and so the model parameters remain constant. The only goal is to fool an already trained model.\n", + "An intriguing property here, is the fact that the gradients are taken with respect to the input image. This is done because the objective is to create an image that maximises the loss. A method to accomplish this is to find how much each pixel in the image contributes to the loss value, and add a perturbation accordingly. This works pretty fast because it is easy to find how each input pixel contributes to the loss by using the chain rule and finding the required gradients. Hence, the gradients are taken with respect to the image. In addition, since the model is no longer being trained (thus the gradient is not taken with respect to the trainable variables, i.e., the model parameters), and so the model parameters remain constant. The only goal is to fool an already trained model.\n", "\n", "So let's try and fool a pretrained model. In this tutorial, the model is [MobileNetV2](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/applications/MobileNetV2) model, pretrained on [ImageNet](http://www.image-net.org/)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "vag2WYR6yTOC" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", "import matplotlib as mpl\n", "import matplotlib.pyplot as plt\n", @@ -124,19 +110,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wiTHY8dqxzx7" }, "source": [ - "Let's load the pretained MobileNetV2 model and the ImageNet class names." + "Let's load the pretrained MobileNetV2 model and the ImageNet class names." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "nqhk2vYx6Ag0" }, "outputs": [], @@ -151,10 +134,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "f2cLrJH0zpfC" }, "outputs": [], @@ -162,8 +143,8 @@ "# Helper function to preprocess the image so that it can be inputted in MobileNetV2\n", "def preprocess(image):\n", " image = tf.cast(image, tf.float32)\n", - " image = image/255\n", " image = tf.image.resize(image, (224, 224))\n", + " image = tf.keras.applications.mobilenet_v2.preprocess_input(image)\n", " image = image[None, ...]\n", " return image\n", "\n", @@ -175,20 +156,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iEZaMVFgSUA-" }, "source": [ "## Original image\n", - "Let's use a sample image of a [Labrador Retriever](https://commons.wikimedia.org/wiki/File:YellowLabradorLooking_new.jpg) -by Mirko [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) from Wikimedia Common and create adversarial examples from it. The first step is to preprocess it so that it can be fed as an input to the MobileNetV2 model." + "Let's use a sample image of a [Labrador Retriever](https://commons.wikimedia.org/wiki/File:YellowLabradorLooking_new.jpg) by Mirko [CC-BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/) from Wikimedia Common and create adversarial examples from it. The first step is to preprocess it so that it can be fed as an input to the MobileNetV2 model." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wpYrQ4OQSYWk" }, "outputs": [], @@ -204,7 +182,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mvPlta_uSbuI" }, "source": [ @@ -213,16 +190,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "99Jc-SNoSZot" }, "outputs": [], "source": [ "plt.figure()\n", - "plt.imshow(image[0])\n", + "plt.imshow(image[0] * 0.5 + 0.5) # To change [-1, 1] to [0,1]\n", "_, image_class, class_confidence = get_imagenet_label(image_probs)\n", "plt.title('{} : {:.2f}% Confidence'.format(image_class, class_confidence*100))\n", "plt.show()" @@ -231,7 +206,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kElVTbF690CF" }, "source": [ @@ -243,10 +217,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "FhZxlOnuBCVr" }, "outputs": [], @@ -269,7 +241,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RbuftX0eSlDQ" }, "source": [ @@ -278,10 +249,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "rVjnb6M7Smv4" }, "outputs": [], @@ -289,27 +258,25 @@ "# Get the input label of the image.\n", "labrador_retriever_index = 208\n", "label = tf.one_hot(labrador_retriever_index, image_probs.shape[-1])\n", + "label = tf.reshape(label, (1, image_probs.shape[-1]))\n", "\n", "perturbations = create_adversarial_pattern(image, label)\n", - "plt.imshow(perturbations[0])" + "plt.imshow(perturbations[0] * 0.5 + 0.5); # To change [-1, 1] to [0,1]" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DKKSFHjwCyQH" }, "source": [ - "Let's try this out for different values of epsilon and observe the resultant image. You'll notice that as the value of epsilon is increased, it becomes easier to fool the network, however, this comes as a trade-off which results in the perturbations becoming more identifiable." + "Let's try this out for different values of epsilon and observe the resultant image. You'll notice that as the value of epsilon is increased, it becomes easier to fool the network. However, this comes as a trade-off which results in the perturbations becoming more identifiable." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dBtG0Kl5SspV" }, "outputs": [], @@ -317,7 +284,7 @@ "def display_images(image, description):\n", " _, label, confidence = get_imagenet_label(pretrained_model.predict(image))\n", " plt.figure()\n", - " plt.imshow(image[0])\n", + " plt.imshow(image[0]*0.5+0.5)\n", " plt.title('{} \\n {} : {:.2f}% Confidence'.format(description,\n", " label, confidence*100))\n", " plt.show()" @@ -325,10 +292,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3DA8g-Zp69J4" }, "outputs": [], @@ -339,14 +304,13 @@ "\n", "for i, eps in enumerate(epsilons):\n", " adv_x = image + eps*perturbations\n", - " adv_x = tf.clip_by_value(adv_x, 0, 1)\n", + " adv_x = tf.clip_by_value(adv_x, -1, 1)\n", " display_images(adv_x, descriptions[i])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "fxt5VfnXHQT6" }, "source": [ @@ -365,10 +329,7 @@ "colab": { "collapsed_sections": [], "name": "adversarial_fgsm.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/generative/autoencoder.ipynb b/site/en/tutorials/generative/autoencoder.ipynb new file mode 100644 index 00000000000..1b2a6fcd2a8 --- /dev/null +++ b/site/en/tutorials/generative/autoencoder.ipynb @@ -0,0 +1,1023 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Ndo4ERqnwQOU" + }, + "source": [ + "##### Copyright 2024 The TensorFlow Authors." + ] + }, + { + "metadata": { + "id": "13rwRG5Jec7n" + }, + "cell_type": "markdown", + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "MTKwbguKwT4R" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xfNT-mlFwxVM" + }, + "source": [ + "# Intro to Autoencoders" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0TD5ZrvEMbhZ" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ITZuApL56Mny" + }, + "source": [ + "This tutorial introduces autoencoders with three examples: the basics, image denoising, and anomaly detection.\n", + "\n", + "An autoencoder is a special type of neural network that is trained to copy its input to its output. For example, given an image of a handwritten digit, an autoencoder first encodes the image into a lower dimensional latent representation, then decodes the latent representation back to an image. An autoencoder learns to compress the data while minimizing the reconstruction error.\n", + "\n", + "To learn more about autoencoders, please consider reading chapter 14 from [Deep Learning](https://www.deeplearningbook.org/) by Ian Goodfellow, Yoshua Bengio, and Aaron Courville." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e1_Y75QXJS6h" + }, + "source": [ + "## Import TensorFlow and other libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YfIk2es3hJEd" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "\n", + "from sklearn.metrics import accuracy_score, precision_score, recall_score\n", + "from sklearn.model_selection import train_test_split\n", + "from tensorflow.keras import layers, losses\n", + "from tensorflow.keras.datasets import fashion_mnist\n", + "from tensorflow.keras.models import Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iYn4MdZnKCey" + }, + "source": [ + "## Load the dataset\n", + "To start, you will train the basic autoencoder using the Fashion MNIST dataset. Each image in this dataset is 28x28 pixels." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YZm503-I_tji" + }, + "outputs": [], + "source": [ + "(x_train, _), (x_test, _) = fashion_mnist.load_data()\n", + "\n", + "x_train = x_train.astype('float32') / 255.\n", + "x_test = x_test.astype('float32') / 255.\n", + "\n", + "print (x_train.shape)\n", + "print (x_test.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VEdCXSwCoKok" + }, + "source": [ + "## First example: Basic autoencoder\n", + "![Basic autoencoder results](images/intro_autoencoder_result.png)\n", + "\n", + "Define an autoencoder with two Dense layers: an `encoder`, which compresses the images into a 64 dimensional latent vector, and a `decoder`, that reconstructs the original image from the latent space.\n", + "\n", + "To define your model, use the [Keras Model Subclassing API](https://www.tensorflow.org/guide/keras/custom_layers_and_models).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0MUxidpyChjX" + }, + "outputs": [], + "source": [ + "class Autoencoder(Model):\n", + " def __init__(self, latent_dim, shape):\n", + " super(Autoencoder, self).__init__()\n", + " self.latent_dim = latent_dim\n", + " self.shape = shape\n", + " self.encoder = tf.keras.Sequential([\n", + " layers.Flatten(),\n", + " layers.Dense(latent_dim, activation='relu'),\n", + " ])\n", + " self.decoder = tf.keras.Sequential([\n", + " layers.Dense(tf.math.reduce_prod(shape).numpy(), activation='sigmoid'),\n", + " layers.Reshape(shape)\n", + " ])\n", + "\n", + " def call(self, x):\n", + " encoded = self.encoder(x)\n", + " decoded = self.decoder(encoded)\n", + " return decoded\n", + "\n", + "\n", + "shape = x_test.shape[1:]\n", + "latent_dim = 64\n", + "autoencoder = Autoencoder(latent_dim, shape)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9I1JlqEIDCI4" + }, + "outputs": [], + "source": [ + "autoencoder.compile(optimizer='adam', loss=losses.MeanSquaredError())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7oJSeMTroABs" + }, + "source": [ + "Train the model using `x_train` as both the input and the target. The `encoder` will learn to compress the dataset from 784 dimensions to the latent space, and the `decoder` will learn to reconstruct the original images.\n", + "." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "h1RI9OfHDBsK" + }, + "outputs": [], + "source": [ + "autoencoder.fit(x_train, x_train,\n", + " epochs=10,\n", + " shuffle=True,\n", + " validation_data=(x_test, x_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wAM1QBhtoC-n" + }, + "source": [ + "Now that the model is trained, let's test it by encoding and decoding images from the test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Pbr5WCj7FQUi" + }, + "outputs": [], + "source": [ + "encoded_imgs = autoencoder.encoder(x_test).numpy()\n", + "decoded_imgs = autoencoder.decoder(encoded_imgs).numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "s4LlDOS6FUA1" + }, + "outputs": [], + "source": [ + "n = 10\n", + "plt.figure(figsize=(20, 4))\n", + "for i in range(n):\n", + " # display original\n", + " ax = plt.subplot(2, n, i + 1)\n", + " plt.imshow(x_test[i])\n", + " plt.title(\"original\")\n", + " plt.gray()\n", + " ax.get_xaxis().set_visible(False)\n", + " ax.get_yaxis().set_visible(False)\n", + "\n", + " # display reconstruction\n", + " ax = plt.subplot(2, n, i + 1 + n)\n", + " plt.imshow(decoded_imgs[i])\n", + " plt.title(\"reconstructed\")\n", + " plt.gray()\n", + " ax.get_xaxis().set_visible(False)\n", + " ax.get_yaxis().set_visible(False)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r4gv6G8PoRQE" + }, + "source": [ + "## Second example: Image denoising\n", + "\n", + "\n", + "![Image denoising results](images/image_denoise_fmnist_results.png)\n", + "\n", + "An autoencoder can also be trained to remove noise from images. In the following section, you will create a noisy version of the Fashion MNIST dataset by applying random noise to each image. You will then train an autoencoder using the noisy image as input, and the original image as the target.\n", + "\n", + "Let's reimport the dataset to omit the modifications made earlier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gDYHJA2PCQ3m" + }, + "outputs": [], + "source": [ + "(x_train, _), (x_test, _) = fashion_mnist.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uJZ-TcaqDBr5" + }, + "outputs": [], + "source": [ + "x_train = x_train.astype('float32') / 255.\n", + "x_test = x_test.astype('float32') / 255.\n", + "\n", + "x_train = x_train[..., tf.newaxis]\n", + "x_test = x_test[..., tf.newaxis]\n", + "\n", + "print(x_train.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aPZl_6P65_8R" + }, + "source": [ + "Adding random noise to the images" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "axSMyxC354fc" + }, + "outputs": [], + "source": [ + "noise_factor = 0.2\n", + "x_train_noisy = x_train + noise_factor * tf.random.normal(shape=x_train.shape)\n", + "x_test_noisy = x_test + noise_factor * tf.random.normal(shape=x_test.shape)\n", + "\n", + "x_train_noisy = tf.clip_by_value(x_train_noisy, clip_value_min=0., clip_value_max=1.)\n", + "x_test_noisy = tf.clip_by_value(x_test_noisy, clip_value_min=0., clip_value_max=1.)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wRxHe4XXltNd" + }, + "source": [ + "Plot the noisy images.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "thKUmbVVCQpt" + }, + "outputs": [], + "source": [ + "n = 10\n", + "plt.figure(figsize=(20, 2))\n", + "for i in range(n):\n", + " ax = plt.subplot(1, n, i + 1)\n", + " plt.title(\"original + noise\")\n", + " plt.imshow(tf.squeeze(x_test_noisy[i]))\n", + " plt.gray()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Sy9SY8jGl5aP" + }, + "source": [ + "### Define a convolutional autoencoder" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vT_BhZngWMwp" + }, + "source": [ + "In this example, you will train a convolutional autoencoder using [Conv2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2D) layers in the `encoder`, and [Conv2DTranspose](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2DTranspose) layers in the `decoder`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "R5KjoIlYCQko" + }, + "outputs": [], + "source": [ + "class Denoise(Model):\n", + " def __init__(self):\n", + " super(Denoise, self).__init__()\n", + " self.encoder = tf.keras.Sequential([\n", + " layers.Input(shape=(28, 28, 1)),\n", + " layers.Conv2D(16, (3, 3), activation='relu', padding='same', strides=2),\n", + " layers.Conv2D(8, (3, 3), activation='relu', padding='same', strides=2)])\n", + "\n", + " self.decoder = tf.keras.Sequential([\n", + " layers.Conv2DTranspose(8, kernel_size=3, strides=2, activation='relu', padding='same'),\n", + " layers.Conv2DTranspose(16, kernel_size=3, strides=2, activation='relu', padding='same'),\n", + " layers.Conv2D(1, kernel_size=(3, 3), activation='sigmoid', padding='same')])\n", + "\n", + " def call(self, x):\n", + " encoded = self.encoder(x)\n", + " decoded = self.decoder(encoded)\n", + " return decoded\n", + "\n", + "autoencoder = Denoise()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QYKbiDFYCQfj" + }, + "outputs": [], + "source": [ + "autoencoder.compile(optimizer='adam', loss=losses.MeanSquaredError())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IssFr1BNCQX3" + }, + "outputs": [], + "source": [ + "autoencoder.fit(x_train_noisy, x_train,\n", + " epochs=10,\n", + " shuffle=True,\n", + " validation_data=(x_test_noisy, x_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G85xUVBGTAKp" + }, + "source": [ + "Let's take a look at a summary of the encoder. Notice how the images are downsampled from 28x28 to 7x7." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "oEpxlX6sTEQz" + }, + "outputs": [], + "source": [ + "autoencoder.encoder.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DDZBfMx1UtXx" + }, + "source": [ + "The decoder upsamples the images back from 7x7 to 28x28." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pbeQtYMaUpro" + }, + "outputs": [], + "source": [ + "autoencoder.decoder.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "A7-VAuEy_N6M" + }, + "source": [ + "Plotting both the noisy images and the denoised images produced by the autoencoder." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t5IyPi1fCQQz" + }, + "outputs": [], + "source": [ + "encoded_imgs = autoencoder.encoder(x_test_noisy).numpy()\n", + "decoded_imgs = autoencoder.decoder(encoded_imgs).numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sfxr9NdBCP_x" + }, + "outputs": [], + "source": [ + "n = 10\n", + "plt.figure(figsize=(20, 4))\n", + "for i in range(n):\n", + "\n", + " # display original + noise\n", + " ax = plt.subplot(2, n, i + 1)\n", + " plt.title(\"original + noise\")\n", + " plt.imshow(tf.squeeze(x_test_noisy[i]))\n", + " plt.gray()\n", + " ax.get_xaxis().set_visible(False)\n", + " ax.get_yaxis().set_visible(False)\n", + "\n", + " # display reconstruction\n", + " bx = plt.subplot(2, n, i + n + 1)\n", + " plt.title(\"reconstructed\")\n", + " plt.imshow(tf.squeeze(decoded_imgs[i]))\n", + " plt.gray()\n", + " bx.get_xaxis().set_visible(False)\n", + " bx.get_yaxis().set_visible(False)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ErGrTnWHoUYl" + }, + "source": [ + "## Third example: Anomaly detection\n", + "\n", + "## Overview\n", + "\n", + "\n", + "In this example, you will train an autoencoder to detect anomalies on the [ECG5000 dataset](http://www.timeseriesclassification.com/description.php?Dataset=ECG5000). This dataset contains 5,000 [Electrocardiograms](https://en.wikipedia.org/wiki/Electrocardiography), each with 140 data points. You will use a simplified version of the dataset, where each example has been labeled either `0` (corresponding to an abnormal rhythm), or `1` (corresponding to a normal rhythm). You are interested in identifying the abnormal rhythms.\n", + "\n", + "Note: This is a labeled dataset, so you could phrase this as a supervised learning problem. The goal of this example is to illustrate anomaly detection concepts you can apply to larger datasets, where you do not have labels available (for example, if you had many thousands of normal rhythms, and only a small number of abnormal rhythms).\n", + "\n", + "How will you detect anomalies using an autoencoder? Recall that an autoencoder is trained to minimize reconstruction error. You will train an autoencoder on the normal rhythms only, then use it to reconstruct all the data. Our hypothesis is that the abnormal rhythms will have higher reconstruction error. You will then classify a rhythm as an anomaly if the reconstruction error surpasses a fixed threshold." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i5estNaur_Mh" + }, + "source": [ + "### Load ECG data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "y35nsXLPsDNX" + }, + "source": [ + "The dataset you will use is based on one from [timeseriesclassification.com](http://www.timeseriesclassification.com/description.php?Dataset=ECG5000).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KmKRDJWgsFYa" + }, + "outputs": [], + "source": [ + "# Download the dataset\n", + "dataframe = pd.read_csv('http://storage.googleapis.com/download.tensorflow.org/data/ecg.csv', header=None)\n", + "raw_data = dataframe.values\n", + "dataframe.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UmuCPVYKsKKx" + }, + "outputs": [], + "source": [ + "# The last element contains the labels\n", + "labels = raw_data[:, -1]\n", + "\n", + "# The other data points are the electrocadriogram data\n", + "data = raw_data[:, 0:-1]\n", + "\n", + "train_data, test_data, train_labels, test_labels = train_test_split(\n", + " data, labels, test_size=0.2, random_state=21\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "byK2vP7hsMbz" + }, + "source": [ + "Normalize the data to `[0,1]`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tgMZVWRKsPx6" + }, + "outputs": [], + "source": [ + "min_val = tf.reduce_min(train_data)\n", + "max_val = tf.reduce_max(train_data)\n", + "\n", + "train_data = (train_data - min_val) / (max_val - min_val)\n", + "test_data = (test_data - min_val) / (max_val - min_val)\n", + "\n", + "train_data = tf.cast(train_data, tf.float32)\n", + "test_data = tf.cast(test_data, tf.float32)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BdSYr2IPsTiz" + }, + "source": [ + "You will train the autoencoder using only the normal rhythms, which are labeled in this dataset as `1`. Separate the normal rhythms from the abnormal rhythms." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VvK4NRe8sVhE" + }, + "outputs": [], + "source": [ + "train_labels = train_labels.astype(bool)\n", + "test_labels = test_labels.astype(bool)\n", + "\n", + "normal_train_data = train_data[train_labels]\n", + "normal_test_data = test_data[test_labels]\n", + "\n", + "anomalous_train_data = train_data[~train_labels]\n", + "anomalous_test_data = test_data[~test_labels]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wVcTBDo-CqFS" + }, + "source": [ + "Plot a normal ECG." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZTlMIrpmseYe" + }, + "outputs": [], + "source": [ + "plt.grid()\n", + "plt.plot(np.arange(140), normal_train_data[0])\n", + "plt.title(\"A Normal ECG\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QpI9by2ZA0NN" + }, + "source": [ + "Plot an anomalous ECG." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zrpXREF2siBr" + }, + "outputs": [], + "source": [ + "plt.grid()\n", + "plt.plot(np.arange(140), anomalous_train_data[0])\n", + "plt.title(\"An Anomalous ECG\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0DS6QKZJslZz" + }, + "source": [ + "### Build the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bf6owZQDsp9y" + }, + "outputs": [], + "source": [ + "class AnomalyDetector(Model):\n", + " def __init__(self):\n", + " super(AnomalyDetector, self).__init__()\n", + " self.encoder = tf.keras.Sequential([\n", + " layers.Dense(32, activation=\"relu\"),\n", + " layers.Dense(16, activation=\"relu\"),\n", + " layers.Dense(8, activation=\"relu\")])\n", + "\n", + " self.decoder = tf.keras.Sequential([\n", + " layers.Dense(16, activation=\"relu\"),\n", + " layers.Dense(32, activation=\"relu\"),\n", + " layers.Dense(140, activation=\"sigmoid\")])\n", + "\n", + " def call(self, x):\n", + " encoded = self.encoder(x)\n", + " decoded = self.decoder(encoded)\n", + " return decoded\n", + "\n", + "autoencoder = AnomalyDetector()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gwRpBBbg463S" + }, + "outputs": [], + "source": [ + "autoencoder.compile(optimizer='adam', loss='mae')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zuTy60STBEy4" + }, + "source": [ + "Notice that the autoencoder is trained using only the normal ECGs, but is evaluated using the full test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "V6NFSs-jsty2" + }, + "outputs": [], + "source": [ + "history = autoencoder.fit(normal_train_data, normal_train_data,\n", + " epochs=20,\n", + " batch_size=512,\n", + " validation_data=(test_data, test_data),\n", + " shuffle=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OEexphFwwTQS" + }, + "outputs": [], + "source": [ + "plt.plot(history.history[\"loss\"], label=\"Training Loss\")\n", + "plt.plot(history.history[\"val_loss\"], label=\"Validation Loss\")\n", + "plt.legend()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ceI5lKv1BT-A" + }, + "source": [ + "You will soon classify an ECG as anomalous if the reconstruction error is greater than one standard deviation from the normal training examples. First, let's plot a normal ECG from the training set, the reconstruction after it's encoded and decoded by the autoencoder, and the reconstruction error." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hmsk4DuktxJ2" + }, + "outputs": [], + "source": [ + "encoded_data = autoencoder.encoder(normal_test_data).numpy()\n", + "decoded_data = autoencoder.decoder(encoded_data).numpy()\n", + "\n", + "plt.plot(normal_test_data[0], 'b')\n", + "plt.plot(decoded_data[0], 'r')\n", + "plt.fill_between(np.arange(140), decoded_data[0], normal_test_data[0], color='lightcoral')\n", + "plt.legend(labels=[\"Input\", \"Reconstruction\", \"Error\"])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ocA_q9ufB_aF" + }, + "source": [ + "Create a similar plot, this time for an anomalous test example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vNFTuPhLwTBn" + }, + "outputs": [], + "source": [ + "encoded_data = autoencoder.encoder(anomalous_test_data).numpy()\n", + "decoded_data = autoencoder.decoder(encoded_data).numpy()\n", + "\n", + "plt.plot(anomalous_test_data[0], 'b')\n", + "plt.plot(decoded_data[0], 'r')\n", + "plt.fill_between(np.arange(140), decoded_data[0], anomalous_test_data[0], color='lightcoral')\n", + "plt.legend(labels=[\"Input\", \"Reconstruction\", \"Error\"])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ocimg3MBswdS" + }, + "source": [ + "### Detect anomalies" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Xnh8wmkDsypN" + }, + "source": [ + "Detect anomalies by calculating whether the reconstruction loss is greater than a fixed threshold. In this tutorial, you will calculate the mean average error for normal examples from the training set, then classify future examples as anomalous if the reconstruction error is higher than one standard deviation from the training set.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TeuT8uTA5Y_w" + }, + "source": [ + "Plot the reconstruction error on normal ECGs from the training set" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "N7FltOnHu4-l" + }, + "outputs": [], + "source": [ + "reconstructions = autoencoder.predict(normal_train_data)\n", + "train_loss = tf.keras.losses.mae(reconstructions, normal_train_data)\n", + "\n", + "plt.hist(train_loss[None,:], bins=50)\n", + "plt.xlabel(\"Train loss\")\n", + "plt.ylabel(\"No of examples\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mh-3ChEF5hog" + }, + "source": [ + "Choose a threshold value that is one standard deviations above the mean." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "82hkl0Chs3P_" + }, + "outputs": [], + "source": [ + "threshold = np.mean(train_loss) + np.std(train_loss)\n", + "print(\"Threshold: \", threshold)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uEGlA1Be50Nj" + }, + "source": [ + "Note: There are other strategies you could use to select a threshold value above which test examples should be classified as anomalous, the correct approach will depend on your dataset. You can learn more with the links at the end of this tutorial." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zpLSDAeb51D_" + }, + "source": [ + "If you examine the reconstruction error for the anomalous examples in the test set, you'll notice most have greater reconstruction error than the threshold. By varing the threshold, you can adjust the [precision](https://developers.google.com/machine-learning/glossary#precision) and [recall](https://developers.google.com/machine-learning/glossary#recall) of your classifier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sKVwjQK955Wy" + }, + "outputs": [], + "source": [ + "reconstructions = autoencoder.predict(anomalous_test_data)\n", + "test_loss = tf.keras.losses.mae(reconstructions, anomalous_test_data)\n", + "\n", + "plt.hist(test_loss[None, :], bins=50)\n", + "plt.xlabel(\"Test loss\")\n", + "plt.ylabel(\"No of examples\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PFVk_XGE6AX2" + }, + "source": [ + "Classify an ECG as an anomaly if the reconstruction error is greater than the threshold." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mkgJZfhh6CHr" + }, + "outputs": [], + "source": [ + "def predict(model, data, threshold):\n", + " reconstructions = model(data)\n", + " loss = tf.keras.losses.mae(reconstructions, data)\n", + " return tf.math.less(loss, threshold)\n", + "\n", + "def print_stats(predictions, labels):\n", + " print(\"Accuracy = {}\".format(accuracy_score(labels, predictions)))\n", + " print(\"Precision = {}\".format(precision_score(labels, predictions)))\n", + " print(\"Recall = {}\".format(recall_score(labels, predictions)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sOcfXfXq6FBd" + }, + "outputs": [], + "source": [ + "preds = predict(autoencoder, test_data, threshold)\n", + "print_stats(preds, test_labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HrJRef8Ln945" + }, + "source": [ + "## Next steps\n", + "\n", + "To learn more about anomaly detection with autoencoders, check out this excellent [interactive example](https://anomagram.fastforwardlabs.com/#/) built with TensorFlow.js by Victor Dibia. For a real-world use case, you can learn how [Airbus Detects Anomalies in ISS Telemetry Data](https://blog.tensorflow.org/2020/04/how-airbus-detects-anomalies-iss-telemetry-data-tfx.html) using TensorFlow. To learn more about the basics, consider reading this [blog post](https://blog.keras.io/building-autoencoders-in-keras.html) by François Chollet. For more details, check out chapter 14 from [Deep Learning](https://www.deeplearningbook.org/) by Ian Goodfellow, Yoshua Bengio, and Aaron Courville.\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "private_outputs": true, + "provenance": [ + { + "file_id": "17gKB2bKebV2DzoYIMFzyEXA5uDnwWOvT", + "timestamp": 1712793165979 + }, + { + "file_id": "https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/autoencoder.ipynb", + "timestamp": 1712792176273 + } + ], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/generative/cvae.ipynb b/site/en/tutorials/generative/cvae.ipynb index 994823ad7f4..02856018590 100644 --- a/site/en/tutorials/generative/cvae.ipynb +++ b/site/en/tutorials/generative/cvae.ipynb @@ -3,20 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ndo4ERqnwQOU" }, "source": [ - "##### Copyright 2018 The TensorFlow Authors." + "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "MTKwbguKwT4R" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xfNT-mlFwxVM" }, "source": [ @@ -47,116 +43,100 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0TD5ZrvEMbhZ" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/generative/cvae\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/generative/cvae.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/cvae.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/generative/cvae.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ITZuApL56Mny" }, "source": [ - "![evolution of output during training](https://tensorflow.org/images/autoencoders/cvae.gif)\n", + "This notebook demonstrates how to train a Variational Autoencoder (VAE) ([1](https://arxiv.org/abs/1312.6114), [2](https://arxiv.org/abs/1401.4082)) on the MNIST dataset. A VAE is a probabilistic take on the autoencoder, a model which takes high dimensional input data and compresses it into a smaller representation. Unlike a traditional autoencoder, which maps the input onto a latent vector, a VAE maps the input data into the parameters of a probability distribution, such as the mean and variance of a Gaussian. This approach produces a continuous, structured latent space, which is useful for image generation.\n", "\n", - "This notebook demonstrates how to generate images of handwritten digits by training a Variational Autoencoder ([1](https://arxiv.org/abs/1312.6114), [2](https://arxiv.org/abs/1401.4082)).\n", - "\n" + "![CVAE image latent space](images/cvae_latent_space.jpg)" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "P-JuIu2N_SQf" + "id": "e1_Y75QXJS6h" }, - "outputs": [], "source": [ - "# to generate gifs\n", - "!pip install imageio" + "## Setup" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "e1_Y75QXJS6h" + "id": "P-JuIu2N_SQf" }, + "outputs": [], "source": [ - "## Import TensorFlow and other libraries" + "!pip install tensorflow-probability\n", + "\n", + "# to generate gifs\n", + "!pip install imageio\n", + "!pip install git+https://github.com/tensorflow/docs" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YfIk2es3hJEd" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", + "from IPython import display\n", "\n", - "import os\n", - "import time\n", - "import numpy as np\n", "import glob\n", + "import imageio\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "import PIL\n", - "import imageio\n", - "\n", - "from IPython import display" + "import tensorflow as tf\n", + "import tensorflow_probability as tfp\n", + "import time" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iYn4MdZnKCey" }, "source": [ "## Load the MNIST dataset\n", - "Each MNIST image is originally a vector of 784 integers, each of which is between 0-255 and represents the intensity of a pixel. We model each pixel with a Bernoulli distribution in our model, and we statically binarize the dataset." + "Each MNIST image is originally a vector of 784 integers, each of which is between 0-255 and represents the intensity of a pixel. Model each pixel with a Bernoulli distribution in our model, and statically binarize the dataset." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "a4fYMGxGhrna" }, "outputs": [], @@ -166,139 +146,133 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "NFC2ghIdiZYE" }, "outputs": [], "source": [ - "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n", - "test_images = test_images.reshape(test_images.shape[0], 28, 28, 1).astype('float32')\n", - "\n", - "# Normalizing the images to the range of [0., 1.]\n", - "train_images /= 255.\n", - "test_images /= 255.\n", + "def preprocess_images(images):\n", + " images = images.reshape((images.shape[0], 28, 28, 1)) / 255.\n", + " return np.where(images > .5, 1.0, 0.0).astype('float32')\n", "\n", - "# Binarization\n", - "train_images[train_images \u003e= .5] = 1.\n", - "train_images[train_images \u003c .5] = 0.\n", - "test_images[test_images \u003e= .5] = 1.\n", - "test_images[test_images \u003c .5] = 0." + "train_images = preprocess_images(train_images)\n", + "test_images = preprocess_images(test_images)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "S4PIDhoDLbsZ" }, "outputs": [], "source": [ - "TRAIN_BUF = 60000\n", - "BATCH_SIZE = 100\n", - "\n", - "TEST_BUF = 10000" + "train_size = 60000\n", + "batch_size = 32\n", + "test_size = 10000" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PIGN6ouoQxt3" }, "source": [ - "## Use *tf.data* to create batches and shuffle the dataset" + "## Use *tf.data* to batch and shuffle the data" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "-yKCCQOoJ7cn" }, "outputs": [], "source": [ - "train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(TRAIN_BUF).batch(BATCH_SIZE)\n", - "test_dataset = tf.data.Dataset.from_tensor_slices(test_images).shuffle(TEST_BUF).batch(BATCH_SIZE)" + "train_dataset = (tf.data.Dataset.from_tensor_slices(train_images)\n", + " .shuffle(train_size).batch(batch_size))\n", + "test_dataset = (tf.data.Dataset.from_tensor_slices(test_images)\n", + " .shuffle(test_size).batch(batch_size))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "THY-sZMiQ4UV" }, "source": [ - "## Wire up the generative and inference network with *tf.keras.Sequential*\n", + "## Define the encoder and decoder networks with *tf.keras.Sequential*\n", + "\n", + "In this VAE example, use two small ConvNets for the encoder and decoder networks. In the literature, these networks are also referred to as inference/recognition and generative models respectively. Use `tf.keras.Sequential` to simplify implementation. Let $x$ and $z$ denote the observation and latent variable respectively in the following descriptions.\n", + "\n", + "### Encoder network\n", + "This defines the approximate posterior distribution $q(z|x)$, which takes as input an observation and outputs a set of parameters for specifying the conditional distribution of the latent representation $z$. \n", + "In this example, simply model the distribution as a diagonal Gaussian, and the network outputs the mean and log-variance parameters of a factorized Gaussian. \n", + "Output log-variance instead of the variance directly for numerical stability.\n", "\n", - "In our VAE example, we use two small ConvNets for the generative and inference network. Since these neural nets are small, we use `tf.keras.Sequential` to simplify our code. Let $x$ and $z$ denote the observation and latent variable respectively in the following descriptions.\n", + "### Decoder network \n", + "This defines the conditional distribution of the observation $p(x|z)$, which takes a latent sample $z$ as input and outputs the parameters for a conditional distribution of the observation.\n", + "Model the latent distribution prior $p(z)$ as a unit Gaussian.\n", "\n", - "### Generative Network\n", - "This defines the generative model which takes a latent encoding as input, and outputs the parameters for a conditional distribution of the observation, i.e. $p(x|z)$. Additionally, we use a unit Gaussian prior $p(z)$ for the latent variable.\n", + "### Reparameterization trick\n", + "To generate a sample $z$ for the decoder during training, you can sample from the latent distribution defined by the parameters outputted by the encoder, given an input observation $x$.\n", + "However, this sampling operation creates a bottleneck because backpropagation cannot flow through a random node.\n", "\n", - "### Inference Network\n", - "This defines an approximate posterior distribution $q(z|x)$, which takes as input an observation and outputs a set of parameters for the conditional distribution of the latent representation. In this example, we simply model this distribution as a diagonal Gaussian. In this case, the inference network outputs the mean and log-variance parameters of a factorized Gaussian (log-variance instead of the variance directly is for numerical stability).\n", + "To address this, use a reparameterization trick.\n", + "In our example, you approximate $z$ using the decoder parameters and another parameter $\\epsilon$ as follows:\n", "\n", - "### Reparameterization Trick\n", - "During optimization, we can sample from $q(z|x)$ by first sampling from a unit Gaussian, and then multiplying by the standard deviation and adding the mean. This ensures the gradients could pass through the sample to the inference network parameters.\n", + "$$z = \\mu + \\sigma \\odot \\epsilon$$\n", + "\n", + "where $\\mu$ and $\\sigma$ represent the mean and standard deviation of a Gaussian distribution respectively. They can be derived from the decoder output. The $\\epsilon$ can be thought of as a random noise used to maintain stochasticity of $z$. Generate $\\epsilon$ from a standard normal distribution.\n", + "\n", + "The latent variable $z$ is now generated by a function of $\\mu$, $\\sigma$ and $\\epsilon$, which would enable the model to backpropagate gradients in the encoder through $\\mu$ and $\\sigma$ respectively, while maintaining stochasticity through $\\epsilon$.\n", "\n", "### Network architecture\n", - "For the inference network, we use two convolutional layers followed by a fully-connected layer. In the generative network, we mirror this architecture by using a fully-connected layer followed by three convolution transpose layers (a.k.a. deconvolutional layers in some contexts). Note, it's common practice to avoid using batch normalization when training VAEs, since the additional stochasticity due to using mini-batches may aggravate instability on top of the stochasticity from sampling." + "For the encoder network, use two convolutional layers followed by a fully-connected layer. In the decoder network, mirror this architecture by using a fully-connected layer followed by three convolution transpose layers (a.k.a. deconvolutional layers in some contexts). Note, it's common practice to avoid using batch normalization when training VAEs, since the additional stochasticity due to using mini-batches may aggravate instability on top of the stochasticity from sampling.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "VGLbvBEmjK0a" }, "outputs": [], "source": [ "class CVAE(tf.keras.Model):\n", + " \"\"\"Convolutional variational autoencoder.\"\"\"\n", + "\n", " def __init__(self, latent_dim):\n", " super(CVAE, self).__init__()\n", " self.latent_dim = latent_dim\n", - " self.inference_net = tf.keras.Sequential(\n", - " [\n", - " tf.keras.layers.InputLayer(input_shape=(28, 28, 1)),\n", - " tf.keras.layers.Conv2D(\n", - " filters=32, kernel_size=3, strides=(2, 2), activation='relu'),\n", - " tf.keras.layers.Conv2D(\n", - " filters=64, kernel_size=3, strides=(2, 2), activation='relu'),\n", - " tf.keras.layers.Flatten(),\n", - " # No activation\n", - " tf.keras.layers.Dense(latent_dim + latent_dim),\n", - " ]\n", + " self.encoder = tf.keras.Sequential(\n", + " [\n", + " tf.keras.layers.InputLayer(input_shape=(28, 28, 1)),\n", + " tf.keras.layers.Conv2D(\n", + " filters=32, kernel_size=3, strides=(2, 2), activation='relu'),\n", + " tf.keras.layers.Conv2D(\n", + " filters=64, kernel_size=3, strides=(2, 2), activation='relu'),\n", + " tf.keras.layers.Flatten(),\n", + " # No activation\n", + " tf.keras.layers.Dense(latent_dim + latent_dim),\n", + " ]\n", " )\n", "\n", - " self.generative_net = tf.keras.Sequential(\n", + " self.decoder = tf.keras.Sequential(\n", " [\n", - " tf.keras.layers.InputLayer(input_shape=(latent_dim,)),\n", - " tf.keras.layers.Dense(units=7*7*32, activation=tf.nn.relu),\n", - " tf.keras.layers.Reshape(target_shape=(7, 7, 32)),\n", - " tf.keras.layers.Conv2DTranspose(\n", - " filters=64,\n", - " kernel_size=3,\n", - " strides=(2, 2),\n", - " padding=\"SAME\",\n", - " activation='relu'),\n", - " tf.keras.layers.Conv2DTranspose(\n", - " filters=32,\n", - " kernel_size=3,\n", - " strides=(2, 2),\n", - " padding=\"SAME\",\n", - " activation='relu'),\n", - " # No activation\n", - " tf.keras.layers.Conv2DTranspose(\n", - " filters=1, kernel_size=3, strides=(1, 1), padding=\"SAME\"),\n", + " tf.keras.layers.InputLayer(input_shape=(latent_dim,)),\n", + " tf.keras.layers.Dense(units=7*7*32, activation=tf.nn.relu),\n", + " tf.keras.layers.Reshape(target_shape=(7, 7, 32)),\n", + " tf.keras.layers.Conv2DTranspose(\n", + " filters=64, kernel_size=3, strides=2, padding='same',\n", + " activation='relu'),\n", + " tf.keras.layers.Conv2DTranspose(\n", + " filters=32, kernel_size=3, strides=2, padding='same',\n", + " activation='relu'),\n", + " # No activation\n", + " tf.keras.layers.Conv2DTranspose(\n", + " filters=1, kernel_size=3, strides=1, padding='same'),\n", " ]\n", " )\n", "\n", @@ -309,7 +283,7 @@ " return self.decode(eps, apply_sigmoid=True)\n", "\n", " def encode(self, x):\n", - " mean, logvar = tf.split(self.inference_net(x), num_or_size_splits=2, axis=1)\n", + " mean, logvar = tf.split(self.encoder(x), num_or_size_splits=2, axis=1)\n", " return mean, logvar\n", "\n", " def reparameterize(self, mean, logvar):\n", @@ -317,18 +291,16 @@ " return eps * tf.exp(logvar * .5) + mean\n", "\n", " def decode(self, z, apply_sigmoid=False):\n", - " logits = self.generative_net(z)\n", + " logits = self.decoder(z)\n", " if apply_sigmoid:\n", " probs = tf.sigmoid(logits)\n", " return probs\n", - "\n", " return logits" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0FMYgY_mPfTi" }, "source": [ @@ -338,46 +310,50 @@ "\n", "$$\\log p(x) \\ge \\text{ELBO} = \\mathbb{E}_{q(z|x)}\\left[\\log \\frac{p(x, z)}{q(z|x)}\\right].$$\n", "\n", - "In practice, we optimize the single sample Monte Carlo estimate of this expectation:\n", + "In practice, optimize the single sample Monte Carlo estimate of this expectation:\n", "\n", "$$\\log p(x| z) + \\log p(z) - \\log q(z|x),$$\n", "where $z$ is sampled from $q(z|x)$.\n", "\n", - "**Note**: we could also analytically compute the KL term, but here we incorporate all three terms in the Monte Carlo estimator for simplicity." + "Note: You could also analytically compute the KL term, but here you incorporate all three terms in the Monte Carlo estimator for simplicity." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iWCn_PVdEJZ7" }, "outputs": [], "source": [ "optimizer = tf.keras.optimizers.Adam(1e-4)\n", "\n", + "\n", "def log_normal_pdf(sample, mean, logvar, raxis=1):\n", " log2pi = tf.math.log(2. * np.pi)\n", " return tf.reduce_sum(\n", " -.5 * ((sample - mean) ** 2. * tf.exp(-logvar) + logvar + log2pi),\n", " axis=raxis)\n", "\n", - "@tf.function\n", + "\n", "def compute_loss(model, x):\n", " mean, logvar = model.encode(x)\n", " z = model.reparameterize(mean, logvar)\n", " x_logit = model.decode(z)\n", - "\n", " cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, labels=x)\n", " logpx_z = -tf.reduce_sum(cross_ent, axis=[1, 2, 3])\n", " logpz = log_normal_pdf(z, 0., 0.)\n", " logqz_x = log_normal_pdf(z, mean, logvar)\n", " return -tf.reduce_mean(logpx_z + logpz - logqz_x)\n", "\n", + "\n", "@tf.function\n", - "def compute_apply_gradients(model, x, optimizer):\n", + "def train_step(model, x, optimizer):\n", + " \"\"\"Executes one training step and returns the loss.\n", + "\n", + " This function computes the loss and gradients, and uses the latter to\n", + " update the model's parameters.\n", + " \"\"\"\n", " with tf.GradientTape() as tape:\n", " loss = compute_loss(model, x)\n", " gradients = tape.gradient(loss, model.trainable_variables)\n", @@ -387,38 +363,36 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Rw1fkAczTQYh" }, "source": [ "## Training\n", "\n", - "* We start by iterating over the dataset\n", - "* During each iteration, we pass the image to the encoder to obtain a set of mean and log-variance parameters of the approximate posterior $q(z|x)$\n", - "* We then apply the *reparameterization trick* to sample from $q(z|x)$\n", - "* Finally, we pass the reparameterized samples to the decoder to obtain the logits of the generative distribution $p(x|z)$\n", - "* **Note:** Since we use the dataset loaded by keras with 60k datapoints in the training set and 10k datapoints in the test set, our resulting ELBO on the test set is slightly higher than reported results in the literature which uses dynamic binarization of Larochelle's MNIST.\n", + "* Start by iterating over the dataset\n", + "* During each iteration, pass the image to the encoder to obtain a set of mean and log-variance parameters of the approximate posterior $q(z|x)$\n", + "* then apply the *reparameterization trick* to sample from $q(z|x)$\n", + "* Finally, pass the reparameterized samples to the decoder to obtain the logits of the generative distribution $p(x|z)$\n", + "* Note: Since you use the dataset loaded by keras with 60k datapoints in the training set and 10k datapoints in the test set, our resulting ELBO on the test set is slightly higher than reported results in the literature which uses dynamic binarization of Larochelle's MNIST.\n", "\n", - "## Generate Images\n", + "### Generating images\n", "\n", "* After training, it is time to generate some images\n", - "* We start by sampling a set of latent vectors from the unit Gaussian prior distribution $p(z)$\n", + "* Start by sampling a set of latent vectors from the unit Gaussian prior distribution $p(z)$\n", "* The generator will then convert the latent sample $z$ to logits of the observation, giving a distribution $p(x|z)$\n", - "* Here we plot the probabilities of Bernoulli distributions\n" + "* Here, plot the probabilities of Bernoulli distributions\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "NS2GWywBbAWo" }, "outputs": [], "source": [ - "epochs = 100\n", - "latent_dim = 50\n", + "epochs = 10\n", + "# set the dimensionality of the latent space to a plane for visualization later\n", + "latent_dim = 2\n", "num_examples_to_generate = 16\n", "\n", "# keeping the random vector constant for generation (prediction) so\n", @@ -430,22 +404,22 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "RmdVsmvhPxyy" }, "outputs": [], "source": [ - "def generate_and_save_images(model, epoch, test_input):\n", - " predictions = model.sample(test_input)\n", - " fig = plt.figure(figsize=(4,4))\n", + "def generate_and_save_images(model, epoch, test_sample):\n", + " mean, logvar = model.encode(test_sample)\n", + " z = model.reparameterize(mean, logvar)\n", + " predictions = model.sample(z)\n", + " fig = plt.figure(figsize=(4, 4))\n", "\n", " for i in range(predictions.shape[0]):\n", - " plt.subplot(4, 4, i+1)\n", - " plt.imshow(predictions[i, :, :, 0], cmap='gray')\n", - " plt.axis('off')\n", + " plt.subplot(4, 4, i + 1)\n", + " plt.imshow(predictions[i, :, :, 0], cmap='gray')\n", + " plt.axis('off')\n", "\n", " # tight_layout minimizes the overlap between 2 sub-plots\n", " plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n", @@ -454,52 +428,57 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, + "metadata": { + "id": "swCyrbqQQ-Ri" + }, + "outputs": [], + "source": [ + "# Pick a sample of the test set for generating output images\n", + "assert batch_size >= num_examples_to_generate\n", + "for test_batch in test_dataset.take(1):\n", + " test_sample = test_batch[0:num_examples_to_generate, :, :, :]" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "2M7LmLtGEMQJ" }, "outputs": [], "source": [ - "generate_and_save_images(model, 0, random_vector_for_generation)\n", + "generate_and_save_images(model, 0, test_sample)\n", "\n", "for epoch in range(1, epochs + 1):\n", " start_time = time.time()\n", " for train_x in train_dataset:\n", - " compute_apply_gradients(model, train_x, optimizer)\n", + " train_step(model, train_x, optimizer)\n", " end_time = time.time()\n", "\n", - " if epoch % 1 == 0:\n", - " loss = tf.keras.metrics.Mean()\n", - " for test_x in test_dataset:\n", - " loss(compute_loss(model, test_x))\n", - " elbo = -loss.result()\n", - " display.clear_output(wait=False)\n", - " print('Epoch: {}, Test set ELBO: {}, '\n", - " 'time elapse for current epoch {}'.format(epoch,\n", - " elbo,\n", - " end_time - start_time))\n", - " generate_and_save_images(\n", - " model, epoch, random_vector_for_generation)" + " loss = tf.keras.metrics.Mean()\n", + " for test_x in test_dataset:\n", + " loss(compute_loss(model, test_x))\n", + " elbo = -loss.result()\n", + " display.clear_output(wait=False)\n", + " print('Epoch: {}, Test set ELBO: {}, time elapse for current epoch: {}'\n", + " .format(epoch, elbo, end_time - start_time))\n", + " generate_and_save_images(model, epoch, test_sample)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "P4M_vIbUi7c0" }, "source": [ - "### Display an image using the epoch number" + "### Display a generated image from the last training epoch" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WfO5wCdclHGL" }, "outputs": [], @@ -510,34 +489,29 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "5x3q9_Oe5q0A" }, "outputs": [], "source": [ - "plt.imshow(display_image(epochs))\n", - "plt.axis('off')# Display images" + "plt.imshow(display_image(epoch))\n", + "plt.axis('off') # Display images" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NywiH3nL8guF" }, "source": [ - "### Generate a GIF of all the saved images." + "### Display an animated GIF of all the saved images" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "IGKQgENQ8lEI" }, "outputs": [], @@ -547,49 +521,102 @@ "with imageio.get_writer(anim_file, mode='I') as writer:\n", " filenames = glob.glob('image*.png')\n", " filenames = sorted(filenames)\n", - " last = -1\n", - " for i,filename in enumerate(filenames):\n", - " frame = 2*(i**0.5)\n", - " if round(frame) \u003e round(last):\n", - " last = frame\n", - " else:\n", - " continue\n", + " for filename in filenames:\n", " image = imageio.imread(filename)\n", " writer.append_data(image)\n", " image = imageio.imread(filename)\n", - " writer.append_data(image)\n", - "\n", - "import IPython\n", - "if IPython.version_info \u003e= (6,2,0,''):\n", - " display.Image(filename=anim_file)" + " writer.append_data(image)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2ZqAEtdqUmJF" + }, + "outputs": [], + "source": [ + "import tensorflow_docs.vis.embed as embed\n", + "embed.embed_file(anim_file)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "yQXO_dlXkKsT" + "id": "PeunRU6TSumT" + }, + "source": [ + "### Display a 2D manifold of digits from the latent space\n", + "\n", + "Running the code below will show a continuous distribution of the different digit classes, with each digit morphing into another across the 2D latent space. Use [TensorFlow Probability](https://www.tensorflow.org/probability) to generate a standard normal distribution for the latent space." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "code", + "id": "mNcaaYPBS3mj" }, + "outputs": [], "source": [ - "If you're working in Colab you can download the animation with the code below:" + "def plot_latent_images(model, n, digit_size=28):\n", + " \"\"\"Plots n x n digit images decoded from the latent space.\"\"\"\n", + "\n", + " norm = tfp.distributions.Normal(0, 1)\n", + " grid_x = norm.quantile(np.linspace(0.05, 0.95, n))\n", + " grid_y = norm.quantile(np.linspace(0.05, 0.95, n))\n", + " image_width = digit_size*n\n", + " image_height = image_width\n", + " image = np.zeros((image_height, image_width))\n", + "\n", + " for i, yi in enumerate(grid_x):\n", + " for j, xi in enumerate(grid_y):\n", + " z = np.array([[xi, yi]])\n", + " x_decoded = model.sample(z)\n", + " digit = tf.reshape(x_decoded[0], (digit_size, digit_size))\n", + " image[i * digit_size: (i + 1) * digit_size,\n", + " j * digit_size: (j + 1) * digit_size] = digit.numpy()\n", + "\n", + " plt.figure(figsize=(10, 10))\n", + " plt.imshow(image, cmap='Greys_r')\n", + " plt.axis('Off')\n", + " plt.show()" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4fSJS3m5HLFM" + "id": "F-ZG69QCZnGY" }, "outputs": [], "source": [ - "try:\n", - " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download(anim_file)" + "plot_latent_images(model, 20)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HrJRef8Ln945" + }, + "source": [ + "## Next steps\n", + "\n", + "This tutorial has demonstrated how to implement a convolutional variational autoencoder using TensorFlow. \n", + "\n", + "As a next step, you could try to improve the model output by increasing the network size. \n", + "For instance, you could try setting the `filter` parameters for each of the `Conv2D` and `Conv2DTranspose` layers to 512. \n", + "Note that in order to generate the final 2D latent image plot, you would need to keep `latent_dim` to 2. Also, the training time would increase as the network size increases.\n", + "\n", + "You could also try implementing a VAE using a different dataset, such as CIFAR-10.\n", + "\n", + "VAEs can be implemented in several different styles and of varying complexity. You can find additional implementations in the following sources:\n", + "- [Variational AutoEncoder (keras.io)](https://keras.io/examples/generative/vae/)\n", + "- [VAE example from \"Writing custom layers and models\" guide (tensorflow.org)](https://www.tensorflow.org/guide/keras/custom_layers_and_models#putting_it_all_together_an_end-to-end_example)\n", + "- [TFP Probabilistic Layers: Variational Auto Encoder](https://www.tensorflow.org/probability/examples/Probabilistic_Layers_VAE)\n", + "\n", + "If you'd like to learn more about the details of VAEs, please refer to [An Introduction to Variational Autoencoders](https://arxiv.org/abs/1906.02691)." ] } ], @@ -598,10 +625,7 @@ "colab": { "collapsed_sections": [], "name": "cvae.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/generative/cyclegan.ipynb b/site/en/tutorials/generative/cyclegan.ipynb index 29063d964e4..313be519591 100644 --- a/site/en/tutorials/generative/cyclegan.ipynb +++ b/site/en/tutorials/generative/cyclegan.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "v1CUZ0dkOo_F" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "qmkj-80IHxnd" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_xnMOsbqHz61" }, "source": [ @@ -47,30 +43,28 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ds4o1h4WHz9U" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/generative/cyclegan\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/generative/cyclegan.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/cyclegan.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/generative/cyclegan.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ITZuApL56Mny" }, "source": [ @@ -89,7 +83,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "e1_Y75QXJS6h" }, "source": [ @@ -99,7 +92,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5fGHWOKPX4ta" }, "source": [ @@ -108,10 +100,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "bJ1ROiQxJ-vY" }, "outputs": [], @@ -121,34 +111,23 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "lhSsUx9Nyb3t" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YfIk2es3hJEd" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "import tensorflow_datasets as tfds\n", "from tensorflow_examples.models.pix2pix import pix2pix\n", "\n", @@ -157,35 +136,31 @@ "import matplotlib.pyplot as plt\n", "from IPython.display import clear_output\n", "\n", - "tfds.disable_progress_bar()\n", - "AUTOTUNE = tf.data.experimental.AUTOTUNE" + "AUTOTUNE = tf.data.AUTOTUNE" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iYn4MdZnKCey" }, "source": [ "## Input Pipeline\n", "\n", - "This tutorial trains a model to translate from images of horses, to images of zebras. You can find this dataset and similar ones [here](https://www.tensorflow.org/datasets/datasets#cycle_gan). \n", + "This tutorial trains a model to translate from images of horses, to images of zebras. You can find this dataset and similar ones [here](https://www.tensorflow.org/datasets/catalog/cycle_gan). \n", "\n", "As mentioned in the [paper](https://arxiv.org/abs/1703.10593), apply random jittering and mirroring to the training dataset. These are some of the image augmentation techniques that avoids overfitting.\n", "\n", "This is similar to what was done in [pix2pix](https://www.tensorflow.org/tutorials/generative/pix2pix#load_the_dataset)\n", "\n", "* In random jittering, the image is resized to `286 x 286` and then randomly cropped to `256 x 256`.\n", - "* In random mirroring, the image is randomly flipped horizontally i.e left to right." + "* In random mirroring, the image is randomly flipped horizontally i.e., left to right." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iuGVPOo7Cce0" }, "outputs": [], @@ -199,10 +174,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "2CbTEt448b4R" }, "outputs": [], @@ -215,10 +188,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Yn3IwqhiIszt" }, "outputs": [], @@ -232,10 +203,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "muhR2cgbLKWW" }, "outputs": [], @@ -249,10 +218,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "fVQOjcPVLrUc" }, "outputs": [], @@ -273,10 +240,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "tyaP4hLJ8b4W" }, "outputs": [], @@ -289,10 +254,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "VB3Z6D_zKSru" }, "outputs": [], @@ -304,37 +267,33 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "RsajGXxd5JkZ" }, "outputs": [], "source": [ - "train_horses = train_horses.map(\n", - " preprocess_image_train, num_parallel_calls=AUTOTUNE).cache().shuffle(\n", - " BUFFER_SIZE).batch(1)\n", + "train_horses = train_horses.cache().map(\n", + " preprocess_image_train, num_parallel_calls=AUTOTUNE).shuffle(\n", + " BUFFER_SIZE).batch(BATCH_SIZE)\n", "\n", - "train_zebras = train_zebras.map(\n", - " preprocess_image_train, num_parallel_calls=AUTOTUNE).cache().shuffle(\n", - " BUFFER_SIZE).batch(1)\n", + "train_zebras = train_zebras.cache().map(\n", + " preprocess_image_train, num_parallel_calls=AUTOTUNE).shuffle(\n", + " BUFFER_SIZE).batch(BATCH_SIZE)\n", "\n", "test_horses = test_horses.map(\n", " preprocess_image_test, num_parallel_calls=AUTOTUNE).cache().shuffle(\n", - " BUFFER_SIZE).batch(1)\n", + " BUFFER_SIZE).batch(BATCH_SIZE)\n", "\n", "test_zebras = test_zebras.map(\n", " preprocess_image_test, num_parallel_calls=AUTOTUNE).cache().shuffle(\n", - " BUFFER_SIZE).batch(1)" + " BUFFER_SIZE).batch(BATCH_SIZE)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "e3MhJ3zVLPan" }, "outputs": [], @@ -345,10 +304,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4pOYjMk_KfIB" }, "outputs": [], @@ -364,10 +321,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0KJyB9ENLb2y" }, "outputs": [], @@ -384,7 +339,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hvX8sKsfMaio" }, "source": [ @@ -394,7 +348,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "cGrL73uCd-_M" }, "source": [ @@ -407,8 +360,8 @@ "\n", "There are 2 generators (G and F) and 2 discriminators (X and Y) being trained here. \n", "\n", - "* Generator `G` learns to transform image `X` to image `Y`. $(G: X -\u003e Y)$\n", - "* Generator `F` learns to transform image `Y` to image `X`. $(F: Y -\u003e X)$\n", + "* Generator `G` learns to transform image `X` to image `Y`. $(G: X -> Y)$\n", + "* Generator `F` learns to transform image `Y` to image `X`. $(F: Y -> X)$\n", "* Discriminator `D_X` learns to differentiate between image `X` and generated image `X` (`F(Y)`).\n", "* Discriminator `D_Y` learns to differentiate between image `Y` and generated image `Y` (`G(X)`).\n", "\n", @@ -417,10 +370,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8ju9Wyw87MRW" }, "outputs": [], @@ -436,10 +387,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wDaGZ3WpZUyw" }, "outputs": [], @@ -464,10 +413,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "O5MhJmxyZiy9" }, "outputs": [], @@ -488,7 +435,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0FMYgY_mPfTi" }, "source": [ @@ -498,21 +444,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JRqt02lupRn8" }, "source": [ "In CycleGAN, there is no paired data to train on, hence there is no guarantee that the input `x` and the target `y` pair are meaningful during training. Thus in order to enforce that the network learns the correct mapping, the authors propose the cycle consistency loss.\n", "\n", - "The discriminator loss and the generator loss are similar to the ones used in [pix2pix](https://www.tensorflow.org/tutorials/generative/pix2pix#define_the_loss_functions_and_the_optimizer)." + "The discriminator loss and the generator loss are similar to the ones used in [pix2pix](https://www.tensorflow.org/tutorials/generative/pix2pix#build_the_generator)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "cyhxTuvJyIHV" }, "outputs": [], @@ -522,10 +465,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Q1Xbz5OaLj5C" }, "outputs": [], @@ -535,10 +476,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wkMNfBWlT-PV" }, "outputs": [], @@ -555,10 +494,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "90BIcCKcDMxz" }, "outputs": [], @@ -570,7 +507,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5iIWQzVF7f9e" }, "source": [ @@ -582,9 +518,9 @@ "* Generated image $\\hat{Y}$ is passed via generator $F$ that yields cycled image $\\hat{X}$.\n", "* Mean absolute error is calculated between $X$ and $\\hat{X}$.\n", "\n", - "$$forward\\ cycle\\ consistency\\ loss: X -\u003e G(X) -\u003e F(G(X)) \\sim \\hat{X}$$\n", + "$$forward\\ cycle\\ consistency\\ loss: X -> G(X) -> F(G(X)) \\sim \\hat{X}$$\n", "\n", - "$$backward\\ cycle\\ consistency\\ loss: Y -\u003e F(Y) -\u003e G(F(Y)) \\sim \\hat{Y}$$\n", + "$$backward\\ cycle\\ consistency\\ loss: Y -> F(Y) -> G(F(Y)) \\sim \\hat{Y}$$\n", "\n", "\n", "![Cycle loss](images/cycle_loss.png)" @@ -592,10 +528,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "NMpVGj_sW6Vo" }, "outputs": [], @@ -609,21 +543,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "U-tJL-fX0Mq7" }, "source": [ "As shown above, generator $G$ is responsible for translating image $X$ to image $Y$. Identity loss says that, if you fed image $Y$ to generator $G$, it should yield the real image $Y$ or something close to image $Y$.\n", "\n", + "If you run the zebra-to-horse model on a horse or the horse-to-zebra model on a zebra, it should not modify the image much since the image already contains the target class.\n", + "\n", "$$Identity\\ loss = |G(Y) - Y| + |F(X) - X|$$" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "05ywEH680Aud" }, "outputs": [], @@ -636,7 +569,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "G-vjRM7IffTT" }, "source": [ @@ -645,10 +577,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iWCn_PVdEJZ7" }, "outputs": [], @@ -663,7 +593,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "aKUZnDiqQrAh" }, "source": [ @@ -672,10 +601,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WJnftd5sQsv6" }, "outputs": [], @@ -702,34 +629,29 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Rw1fkAczTQYh" }, "source": [ "## Training\n", "\n", - "Note: This example model is trained for fewer epochs (40) than the paper (200) to keep training time reasonable for this tutorial. Predictions may be less accurate. " + "Note: This example model is trained for fewer epochs (10) than the paper (200) to keep training time reasonable for this tutorial. The generated images will have much lower quality." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "NS2GWywBbAWo" }, "outputs": [], "source": [ - "EPOCHS = 40" + "EPOCHS = 10" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "RmdVsmvhPxyy" }, "outputs": [], @@ -754,7 +676,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kE47ERn5fyLC" }, "source": [ @@ -768,10 +689,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "KBKUV2sKXDbY" }, "outputs": [], @@ -781,8 +700,8 @@ " # persistent is set to True because the tape is used more than\n", " # once to calculate the gradients.\n", " with tf.GradientTape(persistent=True) as tape:\n", - " # Generator G translates X -\u003e Y\n", - " # Generator F translates Y -\u003e X.\n", + " # Generator G translates X -> Y\n", + " # Generator F translates Y -> X.\n", " \n", " fake_y = generator_g(real_x, training=True)\n", " cycled_x = generator_f(fake_y, training=True)\n", @@ -840,10 +759,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "2M7LmLtGEMQJ" }, "outputs": [], @@ -856,7 +773,7 @@ " train_step(image_x, image_y)\n", " if n % 10 == 0:\n", " print ('.', end='')\n", - " n+=1\n", + " n += 1\n", "\n", " clear_output(wait=True)\n", " # Using a consistent image (sample_horse) so that the progress of the model\n", @@ -875,7 +792,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1RGysMU_BZhx" }, "source": [ @@ -884,10 +800,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "KUgSnmy2nqSP" }, "outputs": [], @@ -900,13 +814,12 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ABGiHY6fE02b" }, "source": [ "## Next steps\n", "\n", - "This tutorial has shown how to implement CycleGAN starting from the generator and discriminator implemented in the [Pix2Pix](https://www.tensorflow.org/tutorials/generative/pix2pix) tutorial. As a next step, you could try using a different dataset from [TensorFlow Datasets](https://www.tensorflow.org/datasets/datasets#cycle_gan). \n", + "This tutorial has shown how to implement CycleGAN starting from the generator and discriminator implemented in the [Pix2Pix](https://www.tensorflow.org/tutorials/generative/pix2pix) tutorial. As a next step, you could try using a different dataset from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/cycle_gan). \n", "\n", "You could also train for a larger number of epochs to improve the results, or you could implement the modified ResNet generator used in the [paper](https://arxiv.org/abs/1703.10593) instead of the U-Net generator used here." ] @@ -917,26 +830,11 @@ "colab": { "collapsed_sections": [], "name": "cyclegan.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", - "language": "python", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.2" } }, "nbformat": 4, diff --git a/site/en/tutorials/generative/data_compression.ipynb b/site/en/tutorials/generative/data_compression.ipynb new file mode 100644 index 00000000000..f756f088acd --- /dev/null +++ b/site/en/tutorials/generative/data_compression.ipynb @@ -0,0 +1,901 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Tce3stUlHN0L" + }, + "source": [ + "##### Copyright 2022 The TensorFlow Compression Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "tuOe1ymfHZPu" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qFdPvlXBOdUN" + }, + "source": [ + "# Learned data compression" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MfBg1C5NB3X0" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xHxb-dlhMIzW" + }, + "source": [ + "## Overview\n", + "\n", + "This notebook shows how to do lossy data compression using neural networks and [TensorFlow Compression](https://github.com/tensorflow/compression).\n", + "\n", + "Lossy compression involves making a trade-off between **rate**, the expected number of bits needed to encode a sample, and **distortion**, the expected error in the reconstruction of the sample.\n", + "\n", + "The examples below use an autoencoder-like model to compress images from the MNIST dataset. The method is based on the paper [End-to-end Optimized Image Compression](https://arxiv.org/abs/1611.01704).\n", + "\n", + "More background on learned data compression can be found in [this paper](https://arxiv.org/abs/2007.03034) targeted at people familiar with classical data compression, or [this survey](https://arxiv.org/abs/2202.06533) targeted at a machine learning audience.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MUXex9ctTuDB" + }, + "source": [ + "## Setup\n", + "\n", + "Install Tensorflow Compression via `pip`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "K489KsEgxuLI" + }, + "outputs": [], + "source": [ + "%%bash\n", + "# Installs the latest version of TFC compatible with the installed TF version.\n", + "\n", + "read MAJOR MINOR <<< \"$(pip show tensorflow | perl -p -0777 -e 's/.*Version: (\\d+)\\.(\\d+).*/\\1 \\2/sg')\"\n", + "pip install \"tensorflow-compression<$MAJOR.$(($MINOR+1))\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WfVAmHCVxpTS" + }, + "source": [ + "Import library dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IqR2PQG4ZaZ0" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import tensorflow as tf\n", + "import tensorflow_compression as tfc\n", + "import tensorflow_datasets as tfds\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wsncKT2iymgQ" + }, + "source": [ + "## Define the trainer model.\n", + "\n", + "Because the model resembles an autoencoder, and we need to perform a different set of functions during training and inference, the setup is a little different from, say, a classifier.\n", + "\n", + "The training model consists of three parts:\n", + "- the **analysis** (or encoder) transform, converting from the image into a latent space,\n", + "- the **synthesis** (or decoder) transform, converting from the latent space back into image space, and\n", + "- a **prior** and entropy model, modeling the marginal probabilities of the latents.\n", + "\n", + "First, define the transforms:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8yZESLgW-vp1" + }, + "outputs": [], + "source": [ + "def make_analysis_transform(latent_dims):\n", + " \"\"\"Creates the analysis (encoder) transform.\"\"\"\n", + " return tf.keras.Sequential([\n", + " tf.keras.layers.Conv2D(\n", + " 20, 5, use_bias=True, strides=2, padding=\"same\",\n", + " activation=\"leaky_relu\", name=\"conv_1\"),\n", + " tf.keras.layers.Conv2D(\n", + " 50, 5, use_bias=True, strides=2, padding=\"same\",\n", + " activation=\"leaky_relu\", name=\"conv_2\"),\n", + " tf.keras.layers.Flatten(),\n", + " tf.keras.layers.Dense(\n", + " 500, use_bias=True, activation=\"leaky_relu\", name=\"fc_1\"),\n", + " tf.keras.layers.Dense(\n", + " latent_dims, use_bias=True, activation=None, name=\"fc_2\"),\n", + " ], name=\"analysis_transform\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2sHdYBzF2xcu" + }, + "outputs": [], + "source": [ + "def make_synthesis_transform():\n", + " \"\"\"Creates the synthesis (decoder) transform.\"\"\"\n", + " return tf.keras.Sequential([\n", + " tf.keras.layers.Dense(\n", + " 500, use_bias=True, activation=\"leaky_relu\", name=\"fc_1\"),\n", + " tf.keras.layers.Dense(\n", + " 2450, use_bias=True, activation=\"leaky_relu\", name=\"fc_2\"),\n", + " tf.keras.layers.Reshape((7, 7, 50)),\n", + " tf.keras.layers.Conv2DTranspose(\n", + " 20, 5, use_bias=True, strides=2, padding=\"same\",\n", + " activation=\"leaky_relu\", name=\"conv_1\"),\n", + " tf.keras.layers.Conv2DTranspose(\n", + " 1, 5, use_bias=True, strides=2, padding=\"same\",\n", + " activation=\"leaky_relu\", name=\"conv_2\"),\n", + " ], name=\"synthesis_transform\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lYC8tHhkxTlK" + }, + "source": [ + "The trainer holds an instance of both transforms, as well as the parameters of the prior.\n", + "\n", + "Its `call` method is set up to compute:\n", + "- **rate**, an estimate of the number of bits needed to represent the batch of digits, and\n", + "- **distortion**, the mean absolute difference between the pixels of the original digits and their reconstructions.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ROn2DbzsBirI" + }, + "outputs": [], + "source": [ + "class MNISTCompressionTrainer(tf.keras.Model):\n", + " \"\"\"Model that trains a compressor/decompressor for MNIST.\"\"\"\n", + "\n", + " def __init__(self, latent_dims):\n", + " super().__init__()\n", + " self.analysis_transform = make_analysis_transform(latent_dims)\n", + " self.synthesis_transform = make_synthesis_transform()\n", + " self.prior_log_scales = tf.Variable(tf.zeros((latent_dims,)))\n", + "\n", + " @property\n", + " def prior(self):\n", + " return tfc.NoisyLogistic(loc=0., scale=tf.exp(self.prior_log_scales))\n", + "\n", + " def call(self, x, training):\n", + " \"\"\"Computes rate and distortion losses.\"\"\"\n", + " # Ensure inputs are floats in the range (0, 1).\n", + " x = tf.cast(x, self.compute_dtype) / 255.\n", + " x = tf.reshape(x, (-1, 28, 28, 1))\n", + "\n", + " # Compute latent space representation y, perturb it and model its entropy,\n", + " # then compute the reconstructed pixel-level representation x_hat.\n", + " y = self.analysis_transform(x)\n", + " entropy_model = tfc.ContinuousBatchedEntropyModel(\n", + " self.prior, coding_rank=1, compression=False)\n", + " y_tilde, rate = entropy_model(y, training=training)\n", + " x_tilde = self.synthesis_transform(y_tilde)\n", + "\n", + " # Average number of bits per MNIST digit.\n", + " rate = tf.reduce_mean(rate)\n", + "\n", + " # Mean absolute difference across pixels.\n", + " distortion = tf.reduce_mean(abs(x - x_tilde))\n", + "\n", + " return dict(rate=rate, distortion=distortion)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vEXbp9RV3kRX" + }, + "source": [ + "### Compute rate and distortion.\n", + "\n", + "Let's walk through this step by step, using one image from the training set. Load the MNIST dataset for training and validation:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7FV99WTrIBen" + }, + "outputs": [], + "source": [ + "training_dataset, validation_dataset = tfds.load(\n", + " \"mnist\",\n", + " split=[\"train\", \"test\"],\n", + " shuffle_files=True,\n", + " as_supervised=True,\n", + " with_info=False,\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SwKgNTg_QfjH" + }, + "source": [ + "And extract one image $x$:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "O-BSdeHcPBBf" + }, + "outputs": [], + "source": [ + "(x, _), = validation_dataset.take(1)\n", + "\n", + "plt.imshow(tf.squeeze(x))\n", + "print(f\"Data type: {x.dtype}\")\n", + "print(f\"Shape: {x.shape}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "V8IvuFkrRJIa" + }, + "source": [ + "To get the latent representation $y$, we need to cast it to `float32`, add a batch dimension, and pass it through the analysis transform." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jA0DOWq23lEq" + }, + "outputs": [], + "source": [ + "x = tf.cast(x, tf.float32) / 255.\n", + "x = tf.reshape(x, (-1, 28, 28, 1))\n", + "y = make_analysis_transform(10)(x)\n", + "\n", + "print(\"y:\", y)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rTojJQvZT8SX" + }, + "source": [ + "The latents will be quantized at test time. To model this in a differentiable way during training, we add uniform noise in the interval $(-.5, .5)$ and call the result $\\tilde y$. This is the same terminology as used in the paper [End-to-end Optimized Image Compression](https://arxiv.org/abs/1611.01704)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Spr3503OUOFQ" + }, + "outputs": [], + "source": [ + "y_tilde = y + tf.random.uniform(y.shape, -.5, .5)\n", + "\n", + "print(\"y_tilde:\", y_tilde)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7hRN89R7SA3U" + }, + "source": [ + "The \"prior\" is a probability density that we train to model the marginal distribution of the noisy latents. For example, it could be a set of independent [logistic distributions](https://en.wikipedia.org/wiki/Logistic_distribution) with different scales for each latent dimension. `tfc.NoisyLogistic` accounts for the fact that the latents have additive noise. As the scale approaches zero, a logistic distribution approaches a dirac delta (spike), but the added noise causes the \"noisy\" distribution to approach the uniform distribution instead." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2tmA1Bw7ReMY" + }, + "outputs": [], + "source": [ + "prior = tfc.NoisyLogistic(loc=0., scale=tf.linspace(.01, 2., 10))\n", + "\n", + "_ = tf.linspace(-6., 6., 501)[:, None]\n", + "plt.plot(_, prior.prob(_));\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2NSWtBZmUvVY" + }, + "source": [ + "During training, `tfc.ContinuousBatchedEntropyModel` adds uniform noise, and uses the noise and the prior to compute a (differentiable) upper bound on the rate (the average number of bits necessary to encode the latent representation). That bound can be minimized as a loss." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hFuGlyJuThBC" + }, + "outputs": [], + "source": [ + "entropy_model = tfc.ContinuousBatchedEntropyModel(\n", + " prior, coding_rank=1, compression=False)\n", + "y_tilde, rate = entropy_model(y, training=True)\n", + "\n", + "print(\"rate:\", rate)\n", + "print(\"y_tilde:\", y_tilde)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Cyr8DGgmWd32" + }, + "source": [ + "Lastly, the noisy latents are passed back through the synthesis transform to produce an image reconstruction $\\tilde x$. Distortion is the error between original image and reconstruction. Obviously, with the transforms untrained, the reconstruction is not very useful." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gtmI0xGEVym0" + }, + "outputs": [], + "source": [ + "x_tilde = make_synthesis_transform()(y_tilde)\n", + "\n", + "# Mean absolute difference across pixels.\n", + "distortion = tf.reduce_mean(abs(x - x_tilde))\n", + "print(\"distortion:\", distortion)\n", + "\n", + "x_tilde = tf.saturate_cast(x_tilde[0] * 255, tf.uint8)\n", + "plt.imshow(tf.squeeze(x_tilde))\n", + "print(f\"Data type: {x_tilde.dtype}\")\n", + "print(f\"Shape: {x_tilde.shape}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UVz3I7E8ecij" + }, + "source": [ + "For every batch of digits, calling the `MNISTCompressionTrainer` produces the rate and distortion as an average over that batch:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ICJnjj1LeB8L" + }, + "outputs": [], + "source": [ + "(example_batch, _), = validation_dataset.batch(32).take(1)\n", + "trainer = MNISTCompressionTrainer(10)\n", + "example_output = trainer(example_batch)\n", + "\n", + "print(\"rate: \", example_output[\"rate\"])\n", + "print(\"distortion: \", example_output[\"distortion\"])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lgdfRtmee5Mn" + }, + "source": [ + "In the next section, we set up the model to do gradient descent on these two losses." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fKGVwv5MAq6w" + }, + "source": [ + "## Train the model.\n", + "\n", + "We compile the trainer in a way that it optimizes the rate–distortion Lagrangian, that is, a sum of rate and distortion, where one of the terms is weighted by Lagrange parameter $\\lambda$.\n", + "\n", + "This loss function affects the different parts of the model differently:\n", + "- The analysis transform is trained to produce a latent representation that achieves the desired trade-off between rate and distortion.\n", + "- The synthesis transform is trained to minimize distortion, given the latent representation.\n", + "- The parameters of the prior are trained to minimize the rate given the latent representation. This is identical to fitting the prior to the marginal distribution of latents in a maximum likelihood sense." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "k5mm1aDkcgAf" + }, + "outputs": [], + "source": [ + "def pass_through_loss(_, x):\n", + " # Since rate and distortion are unsupervised, the loss doesn't need a target.\n", + " return x\n", + "\n", + "def make_mnist_compression_trainer(lmbda, latent_dims=50):\n", + " trainer = MNISTCompressionTrainer(latent_dims)\n", + " trainer.compile(\n", + " optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),\n", + " # Just pass through rate and distortion as losses/metrics.\n", + " loss=dict(rate=pass_through_loss, distortion=pass_through_loss),\n", + " metrics=dict(rate=pass_through_loss, distortion=pass_through_loss),\n", + " loss_weights=dict(rate=1., distortion=lmbda),\n", + " )\n", + " return trainer\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DPwd4DTs3Mfr" + }, + "source": [ + "Next, train the model. The human annotations are not necessary here, since we just want to compress the images, so we drop them using a `map` and instead add \"dummy\" targets for rate and distortion." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QNBpCTgzAV7M" + }, + "outputs": [], + "source": [ + "def add_rd_targets(image, label):\n", + " # Training is unsupervised, so labels aren't necessary here. However, we\n", + " # need to add \"dummy\" targets for rate and distortion.\n", + " return image, dict(rate=0., distortion=0.)\n", + "\n", + "def train_mnist_model(lmbda):\n", + " trainer = make_mnist_compression_trainer(lmbda)\n", + " trainer.fit(\n", + " training_dataset.map(add_rd_targets).batch(128).prefetch(8),\n", + " epochs=15,\n", + " validation_data=validation_dataset.map(add_rd_targets).batch(128).cache(),\n", + " validation_freq=1,\n", + " verbose=1,\n", + " )\n", + " return trainer\n", + "\n", + "trainer = train_mnist_model(lmbda=2000)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Td4xuttmCd7T" + }, + "source": [ + "## Compress some MNIST images.\n", + "\n", + "For compression and decompression at test time, we split the trained model in two parts:\n", + "\n", + "- The encoder side consists of the analysis transform and the entropy model.\n", + "- The decoder side consists of the synthesis transform and the same entropy model.\n", + "\n", + "At test time, the latents will not have additive noise, but they will be quantized and then losslessly compressed, so we give them new names. We call them and the image reconstruction $\\hat x$ and $\\hat y$, respectively (following [End-to-end Optimized Image Compression](https://arxiv.org/abs/1611.01704))." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sBRAPa5jksss" + }, + "outputs": [], + "source": [ + "class MNISTCompressor(tf.keras.Model):\n", + " \"\"\"Compresses MNIST images to strings.\"\"\"\n", + "\n", + " def __init__(self, analysis_transform, entropy_model):\n", + " super().__init__()\n", + " self.analysis_transform = analysis_transform\n", + " self.entropy_model = entropy_model\n", + "\n", + " def call(self, x):\n", + " # Ensure inputs are floats in the range (0, 1).\n", + " x = tf.cast(x, self.compute_dtype) / 255.\n", + " y = self.analysis_transform(x)\n", + " # Also return the exact information content of each digit.\n", + " _, bits = self.entropy_model(y, training=False)\n", + " return self.entropy_model.compress(y), bits\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sSZ0X2xPnkN-" + }, + "outputs": [], + "source": [ + "class MNISTDecompressor(tf.keras.Model):\n", + " \"\"\"Decompresses MNIST images from strings.\"\"\"\n", + "\n", + " def __init__(self, entropy_model, synthesis_transform):\n", + " super().__init__()\n", + " self.entropy_model = entropy_model\n", + " self.synthesis_transform = synthesis_transform\n", + "\n", + " def call(self, string):\n", + " y_hat = self.entropy_model.decompress(string, ())\n", + " x_hat = self.synthesis_transform(y_hat)\n", + " # Scale and cast back to 8-bit integer.\n", + " return tf.saturate_cast(tf.round(x_hat * 255.), tf.uint8)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GI7rxeOUDnaC" + }, + "source": [ + "When instantiated with `compression=True`, the entropy model converts the learned prior into tables for a range coding algorithm. When calling `compress()`, this algorithm is invoked to convert the latent space vector into bit sequences. The length of each binary string approximates the information content of the latent (the negative log likelihood of the latent under the prior).\n", + "\n", + "The entropy model for compression and decompression must be the same instance, because the range coding tables need to be exactly identical on both sides. Otherwise, decoding errors can occur." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Dnm_p7mbnigo" + }, + "outputs": [], + "source": [ + "def make_mnist_codec(trainer, **kwargs):\n", + " # The entropy model must be created with `compression=True` and the same\n", + " # instance must be shared between compressor and decompressor.\n", + " entropy_model = tfc.ContinuousBatchedEntropyModel(\n", + " trainer.prior, coding_rank=1, compression=True, **kwargs)\n", + " compressor = MNISTCompressor(trainer.analysis_transform, entropy_model)\n", + " decompressor = MNISTDecompressor(entropy_model, trainer.synthesis_transform)\n", + " return compressor, decompressor\n", + "\n", + "compressor, decompressor = make_mnist_codec(trainer)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SYu5sVVH3YMv" + }, + "source": [ + "Grab 16 images from the validation dataset. You can select a different subset by changing the argument to `skip`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qAxArlU728K5" + }, + "outputs": [], + "source": [ + "(originals, _), = validation_dataset.batch(16).skip(3).take(1)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CHeN_ny929YS" + }, + "source": [ + "Compress them to strings, and keep track of each of their information content in bits." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "smOk42gQ3IXv" + }, + "outputs": [], + "source": [ + "strings, entropies = compressor(originals)\n", + "\n", + "print(f\"String representation of first digit in hexadecimal: 0x{strings[0].numpy().hex()}\")\n", + "print(f\"Number of bits actually needed to represent it: {entropies[0]:0.2f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5j9R4bTT3Qhl" + }, + "source": [ + "Decompress the images back from the strings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yOP6pEqU3P0w" + }, + "outputs": [], + "source": [ + "reconstructions = decompressor(strings)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JWo0Q-vy23tt" + }, + "source": [ + "Display each of the 16 original digits together with its compressed binary representation, and the reconstructed digit." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "jU5IqzZzeEpf" + }, + "outputs": [], + "source": [ + "#@title\n", + "\n", + "def display_digits(originals, strings, entropies, reconstructions):\n", + " \"\"\"Visualizes 16 digits together with their reconstructions.\"\"\"\n", + " fig, axes = plt.subplots(4, 4, sharex=True, sharey=True, figsize=(12.5, 5))\n", + " axes = axes.ravel()\n", + " for i in range(len(axes)):\n", + " image = tf.concat([\n", + " tf.squeeze(originals[i]),\n", + " tf.zeros((28, 14), tf.uint8),\n", + " tf.squeeze(reconstructions[i]),\n", + " ], 1)\n", + " axes[i].imshow(image)\n", + " axes[i].text(\n", + " .5, .5, f\"→ 0x{strings[i].numpy().hex()} →\\n{entropies[i]:0.2f} bits\",\n", + " ha=\"center\", va=\"top\", color=\"white\", fontsize=\"small\",\n", + " transform=axes[i].transAxes)\n", + " axes[i].axis(\"off\")\n", + " plt.subplots_adjust(wspace=0, hspace=0, left=0, right=1, bottom=0, top=1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "km9PqVEtPJPc" + }, + "outputs": [], + "source": [ + "display_digits(originals, strings, entropies, reconstructions)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EzlrIOiYOzJc" + }, + "source": [ + "Note that the length of the encoded string differs from the information content of each digit.\n", + "\n", + "This is because the range coding process works with discrete probabilities, and has a small amount of overhead. So, especially for short strings, the correspondence is only approximate. However, range coding is **asymptotically optimal**: in the limit, the expected bit count will approach the cross entropy (the expected information content), for which the rate term in the training model is an upper bound." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "78qIG8t8FvJW" + }, + "source": [ + "## The rate–distortion trade-off\n", + "\n", + "Above, the model was trained for a specific trade-off (given by `lmbda=2000`) between the average number of bits used to represent each digit and the incurred error in the reconstruction.\n", + "\n", + "What happens when we repeat the experiment with different values?\n", + "\n", + "Let's start by reducing $\\lambda$ to 500." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1iFcAD0WF78p" + }, + "outputs": [], + "source": [ + "def train_and_visualize_model(lmbda):\n", + " trainer = train_mnist_model(lmbda=lmbda)\n", + " compressor, decompressor = make_mnist_codec(trainer)\n", + " strings, entropies = compressor(originals)\n", + " reconstructions = decompressor(strings)\n", + " display_digits(originals, strings, entropies, reconstructions)\n", + "\n", + "train_and_visualize_model(lmbda=500)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Uy5OkgJMObMc" + }, + "source": [ + "The bit rate of our code goes down, as does the fidelity of the digits. However, most of the digits remain recognizable.\n", + "\n", + "Let's reduce $\\lambda$ further." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NQp9_9_5GcxH" + }, + "outputs": [], + "source": [ + "train_and_visualize_model(lmbda=300)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3ELLMANN1OwMQ" + }, + "source": [ + "The strings begin to get much shorter now, on the order of one byte per digit. However, this comes at a cost. More digits are becoming unrecognizable.\n", + "\n", + "This demonstrates that this model is agnostic to human perceptions of error, it just measures the absolute deviation in terms of pixel values. To achieve a better perceived image quality, we would need to replace the pixel loss with a perceptual loss." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v9cWHtH0LP_r" + }, + "source": [ + "## Use the decoder as a generative model.\n", + "\n", + "If we feed the decoder random bits, this will effectively sample from the distribution that the model learned to represent digits.\n", + "\n", + "First, re-instantiate the compressor/decompressor without a sanity check that would detect if the input string isn't completely decoded." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qnic8YsM0_ke" + }, + "outputs": [], + "source": [ + "compressor, decompressor = make_mnist_codec(trainer, decode_sanity_check=False)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "86uc9_Is1eeo" + }, + "source": [ + "Now, feed long enough random strings into the decompressor so that it can decode/sample digits from them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "o4fP7BkqKCHY" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "strings = tf.constant([os.urandom(8) for _ in range(16)])\n", + "samples = decompressor(strings)\n", + "\n", + "fig, axes = plt.subplots(4, 4, sharex=True, sharey=True, figsize=(5, 5))\n", + "axes = axes.ravel()\n", + "for i in range(len(axes)):\n", + " axes[i].imshow(tf.squeeze(samples[i]))\n", + " axes[i].axis(\"off\")\n", + "plt.subplots_adjust(wspace=0, hspace=0, left=0, right=1, bottom=0, top=1)\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "data_compression.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/generative/dcgan.ipynb b/site/en/tutorials/generative/dcgan.ipynb index 5d7156c7f5e..e1a852ba815 100644 --- a/site/en/tutorials/generative/dcgan.ipynb +++ b/site/en/tutorials/generative/dcgan.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_jQ1tEQCxwRx" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "V_sgB_5dx1f1" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rF2x3qooyBTI" }, "source": [ @@ -47,36 +43,34 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0TD5ZrvEMbhZ" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/generative/dcgan\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/generative/dcgan.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/dcgan.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/generative/dcgan.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ITZuApL56Mny" }, "source": [ @@ -86,7 +80,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2MbKJY38Puy9" }, "source": [ @@ -103,55 +96,22 @@ "\n", "![sample output](https://tensorflow.org/images/gan/dcgan.gif)\n", "\n", - "To learn more about GANs, we recommend MIT's [Intro to Deep Learning](http://introtodeeplearning.com/) course." + "To learn more about GANs, see MIT's [Intro to Deep Learning](http://introtodeeplearning.com/) course." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "e1_Y75QXJS6h" }, "source": [ - "### Import TensorFlow and other libraries" + "### Setup" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "J5oue0oqCkZZ" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "g5RstiiB8V-z" - }, - "outputs": [], - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", "id": "WZKbyU2-AiY-" }, "outputs": [], @@ -161,10 +121,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wx-zNbLqB4K8" }, "outputs": [], @@ -174,24 +132,21 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YzTlj4YdCip_" }, "outputs": [], "source": [ "# To generate GIFs\n", - "!pip install imageio" + "!pip install imageio\n", + "!pip install git+https://github.com/tensorflow/docs" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YfIk2es3hJEd" }, "outputs": [], @@ -211,7 +166,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iYn4MdZnKCey" }, "source": [ @@ -222,10 +176,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "a4fYMGxGhrna" }, "outputs": [], @@ -235,24 +187,20 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "NFC2ghIdiZYE" }, "outputs": [], "source": [ "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n", - "train_images = (train_images - 127.5) / 127.5 # Normalize the images to [-1, 1]" + "train_images = (train_images - 127.5) / 127.5 # Normalize the images to [-1, 1]" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "S4PIDhoDLbsZ" }, "outputs": [], @@ -263,10 +211,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "-yKCCQOoJ7cn" }, "outputs": [], @@ -278,7 +224,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "THY-sZMiQ4UV" }, "source": [ @@ -290,7 +235,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-tEyxE-GMC48" }, "source": [ @@ -301,10 +245,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "6bpTcDqoLWjY" }, "outputs": [], @@ -316,7 +258,7 @@ " model.add(layers.LeakyReLU())\n", "\n", " model.add(layers.Reshape((7, 7, 256)))\n", - " assert model.output_shape == (None, 7, 7, 256) # Note: None is the batch size\n", + " assert model.output_shape == (None, 7, 7, 256) # Note: None is the batch size\n", "\n", " model.add(layers.Conv2DTranspose(128, (5, 5), strides=(1, 1), padding='same', use_bias=False))\n", " assert model.output_shape == (None, 7, 7, 128)\n", @@ -337,7 +279,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GyWgG09LCSJl" }, "source": [ @@ -346,10 +287,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gl7jcC7TdPTG" }, "outputs": [], @@ -365,7 +304,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "D0IKnaCtg6WE" }, "source": [ @@ -376,10 +314,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dw2tPLmk2pEP" }, "outputs": [], @@ -404,7 +340,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QhPneagzCaQv" }, "source": [ @@ -413,10 +348,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gDkA05NE6QMs" }, "outputs": [], @@ -429,7 +362,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0FMYgY_mPfTi" }, "source": [ @@ -440,10 +372,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "psQfmXxYKU3X" }, "outputs": [], @@ -455,7 +385,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PKY_iPSPNWoj" }, "source": [ @@ -466,10 +395,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wkMNfBWlT-PV" }, "outputs": [], @@ -484,20 +411,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Jd-3GCUEiKtv" }, "source": [ "### Generator loss\n", - "The generator's loss quantifies how well it was able to trick the discriminator. Intuitively, if the generator is performing well, the discriminator will classify the fake images as real (or 1). Here, we will compare the discriminators decisions on the generated images to an array of 1s." + "The generator's loss quantifies how well it was able to trick the discriminator. Intuitively, if the generator is performing well, the discriminator will classify the fake images as real (or 1). Here, compare the discriminators decisions on the generated images to an array of 1s." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "90BIcCKcDMxz" }, "outputs": [], @@ -509,19 +433,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MgIc7i0th_Iu" }, "source": [ - "The discriminator and the generator optimizers are different since we will train two networks separately." + "The discriminator and the generator optimizers are different since you will train two networks separately." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iWCn_PVdEJZ7" }, "outputs": [], @@ -533,7 +454,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mWtinsGDPJlV" }, "source": [ @@ -543,10 +463,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "CA1w-7s2POEy" }, "outputs": [], @@ -562,20 +480,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Rw1fkAczTQYh" }, "source": [ - "## Define the training loop\n", - "\n" + "## Define the training loop\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "NS2GWywBbAWo" }, "outputs": [], @@ -584,7 +498,7 @@ "noise_dim = 100\n", "num_examples_to_generate = 16\n", "\n", - "# We will reuse this seed overtime (so it's easier)\n", + "# You will reuse this seed overtime (so it's easier)\n", "# to visualize progress in the animated GIF)\n", "seed = tf.random.normal([num_examples_to_generate, noise_dim])" ] @@ -592,7 +506,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jylSonrqSWfi" }, "source": [ @@ -601,10 +514,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3t5ibNo05jCB" }, "outputs": [], @@ -633,10 +544,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "2M7LmLtGEMQJ" }, "outputs": [], @@ -648,7 +557,7 @@ " for image_batch in dataset:\n", " train_step(image_batch)\n", "\n", - " # Produce images for the GIF as we go\n", + " # Produce images for the GIF as you go\n", " display.clear_output(wait=True)\n", " generate_and_save_images(generator,\n", " epoch + 1,\n", @@ -670,20 +579,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2aFF7Hk3XdeW" }, "source": [ - "**Generate and save images**\n", - "\n" + "**Generate and save images**\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "RmdVsmvhPxyy" }, "outputs": [], @@ -693,7 +598,7 @@ " # This is so all layers run in inference mode (batchnorm).\n", " predictions = model(test_input, training=False)\n", "\n", - " fig = plt.figure(figsize=(4,4))\n", + " fig = plt.figure(figsize=(4, 4))\n", "\n", " for i in range(predictions.shape[0]):\n", " plt.subplot(4, 4, i+1)\n", @@ -707,7 +612,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "dZrd4CdjR-Fp" }, "source": [ @@ -719,10 +623,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Ly3UN0SLLY2l" }, "outputs": [], @@ -733,7 +635,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rfM4YcPVPkNO" }, "source": [ @@ -742,10 +643,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "XhXsd0srPo8c" }, "outputs": [], @@ -756,7 +655,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "P4M_vIbUi7c0" }, "source": [ @@ -765,10 +663,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WfO5wCdclHGL" }, "outputs": [], @@ -780,10 +676,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "5x3q9_Oe5q0A" }, "outputs": [], @@ -794,7 +688,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NywiH3nL8guF" }, "source": [ @@ -803,10 +696,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "IGKQgENQ8lEI" }, "outputs": [], @@ -816,55 +707,28 @@ "with imageio.get_writer(anim_file, mode='I') as writer:\n", " filenames = glob.glob('image*.png')\n", " filenames = sorted(filenames)\n", - " last = -1\n", - " for i,filename in enumerate(filenames):\n", - " frame = 2*(i**0.5)\n", - " if round(frame) \u003e round(last):\n", - " last = frame\n", - " else:\n", - " continue\n", + " for filename in filenames:\n", " image = imageio.imread(filename)\n", " writer.append_data(image)\n", " image = imageio.imread(filename)\n", - " writer.append_data(image)\n", - "\n", - "import IPython\n", - "if IPython.version_info \u003e (6,2,0,''):\n", - " display.Image(filename=anim_file)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cGhC3-fMWSwl" - }, - "source": [ - "If you're working in Colab you can download the animation with the code below:" + " writer.append_data(image)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uV0yiKpzNP1b" + "id": "ZBwyU6t2Wf3g" }, "outputs": [], "source": [ - "try:\n", - " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download(anim_file)" + "import tensorflow_docs.vis.embed as embed\n", + "embed.embed_file(anim_file)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "k6qC-SbjK0yW" }, "source": [ @@ -874,11 +738,10 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xjjkT9KAK6H7" }, "source": [ - "This tutorial has shown the complete code necessary to write and train a GAN. As a next step, you might like to experiment with a different dataset, for example the Large-scale Celeb Faces Attributes (CelebA) dataset [available on Kaggle](https://www.kaggle.com/jessicali9530/celeba-dataset/home). To learn more about GANs we recommend the [NIPS 2016 Tutorial: Generative Adversarial Networks](https://arxiv.org/abs/1701.00160).\n" + "This tutorial has shown the complete code necessary to write and train a GAN. As a next step, you might like to experiment with a different dataset, for example the Large-scale Celeb Faces Attributes (CelebA) dataset [available on Kaggle](https://www.kaggle.com/jessicali9530/celeba-dataset). To learn more about GANs see the [NIPS 2016 Tutorial: Generative Adversarial Networks](https://arxiv.org/abs/1701.00160).\n" ] } ], @@ -887,8 +750,6 @@ "colab": { "collapsed_sections": [], "name": "dcgan.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/generative/deepdream.ipynb b/site/en/tutorials/generative/deepdream.ipynb index 0ca4f309721..e4a675ebed6 100644 --- a/site/en/tutorials/generative/deepdream.ipynb +++ b/site/en/tutorials/generative/deepdream.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1SgrstLXNbG_" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "k7gifg92NbG9" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "dCMqzy7BNbG9" }, "source": [ @@ -47,30 +43,28 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2yqCPS8SNbG8" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/generative/deepdream\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/generative/deepdream.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/deepdream.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/generative/deepdream.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XPDKhwPcNbG7" }, "source": [ @@ -78,7 +72,7 @@ "\n", "DeepDream is an experiment that visualizes the patterns learned by a neural network. Similar to when a child watches clouds and tries to interpret random shapes, DeepDream over-interprets and enhances the patterns it sees in an image.\n", "\n", - "It does so by forwarding an image through the network, then calculating the gradient of the image with respect to the activations of a particular layer. The image is then modified to increase these activations, enhancing the patterns seen by the network, and resulting in a dream-like image. This process was dubbed \"Inceptionism\" (a reference to [InceptionNet](https://arxiv.org/pdf/1409.4842.pdf), and the [movie](https://en.wikipedia.org/wiki/Inception) Inception.\n", + "It does so by forwarding an image through the network, then calculating the gradient of the image with respect to the activations of a particular layer. The image is then modified to increase these activations, enhancing the patterns seen by the network, and resulting in a dream-like image. This process was dubbed \"Inceptionism\" (a reference to [InceptionNet](https://arxiv.org/pdf/1409.4842.pdf), and the [movie](https://en.wikipedia.org/wiki/Inception) Inception).\n", "\n", "Let's demonstrate how you can make a neural network \"dream\" and enhance the surreal patterns it sees in an image.\n", "\n", @@ -87,41 +81,19 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GUSXlFNkxrqh" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", "id": "Sc5Yq_Rgxreb" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "g_Qp173_NbG5" }, "outputs": [], @@ -131,15 +103,12 @@ "import matplotlib as mpl\n", "\n", "import IPython.display as display\n", - "import PIL.Image\n", - "\n", - "from tensorflow.keras.preprocessing import image" + "import PIL.Image" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wgeIJg82NbG4" }, "source": [ @@ -149,7 +118,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "yt6zam_9NbG4" }, "source": [ @@ -158,10 +126,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0lclzk9sNbG2" }, "outputs": [], @@ -171,10 +137,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Y5BPgc8NNbG0" }, "outputs": [], @@ -201,13 +165,12 @@ "# Downsizing the image makes it easier to work with.\n", "original_img = download(url, max_dim=500)\n", "show(original_img)\n", - "display.display(display.HTML('Image cc-by: \u003ca \"href=https://commons.wikimedia.org/wiki/File:Felis_catus-cat_on_snow.jpg\"\u003eVon.grzanka\u003c/a\u003e'))" + "display.display(display.HTML('Image cc-by: Von.grzanka'))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "F4RBFfIWNbG0" }, "source": [ @@ -217,19 +180,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "cruNQmMDNbGz" }, "source": [ - "Download and prepare a pre-trained image classification model. You will use [InceptionV3](https://keras.io/applications/#inceptionv3) which is similar to the model originally used in DeepDream. Note that any [pre-trained model](https://keras.io/applications/#models-for-image-classification-with-weights-trained-on-imagenet) will work, although you will have to adjust the layer names below if you change this." + "Download and prepare a pre-trained image classification model. You will use [InceptionV3](https://keras.io/api/applications/inceptionv3/) which is similar to the model originally used in DeepDream. Note that any [pre-trained model](https://keras.io/api/applications/#available-models) will work, although you will have to adjust the layer names below if you change this." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "GlLi48GKNbGy" }, "outputs": [], @@ -240,7 +200,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Bujb0jPNNbGx" }, "source": [ @@ -250,19 +209,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qOVmDO4LNbGv" }, "source": [ - "The InceptionV3 architecture is quite large (for a graph of the model architecture see TensorFlow's [research repo](https://github.com/tensorflow/models/tree/master/research/inception)). For DeepDream, the layers of interest are those where the convolutions are concatenated. There are 11 of these layers in InceptionV3, named 'mixed0' though 'mixed10'. Using different layers will result in different dream-like images. Deeper layers respond to higher-level features (such as eyes and faces), while earlier layers respond to simpler features (such as edges, shapes, and textures). Feel free to experiment with the layers selected below, but keep in mind that deeper layers (those with a higher index) will take longer to train on since the gradient computation is deeper." + "The InceptionV3 architecture is quite large (for a graph of the model architecture see TensorFlow's [research repo](https://github.com/tensorflow/models/tree/master/research/slim)). For DeepDream, the layers of interest are those where the convolutions are concatenated. There are 11 of these layers in InceptionV3, named 'mixed0' though 'mixed10'. Using different layers will result in different dream-like images. Deeper layers respond to higher-level features (such as eyes and faces), while earlier layers respond to simpler features (such as edges, shapes, and textures). Feel free to experiment with the layers selected below, but keep in mind that deeper layers (those with a higher index) will take longer to train on since the gradient computation is deeper." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "08KB502ONbGt" }, "outputs": [], @@ -278,21 +234,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "sb7u31B4NbGt" }, "source": [ "## Calculate loss\n", "\n", - "The loss is the sum of the activations in the chosen layers. The loss is normalizaed at each layer so the contribution from larger layers does not outweigh smaller layers. Normally, loss is a quantity you wish to minimize via gradient descent. In DeepDream, you will maximize this loss via gradient ascent." + "The loss is the sum of the activations in the chosen layers. The loss is normalized at each layer so the contribution from larger layers does not outweigh smaller layers. Normally, loss is a quantity you wish to minimize via gradient descent. In DeepDream, you will maximize this loss via gradient ascent." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8MhfSweXXiuq" }, "outputs": [], @@ -302,6 +255,8 @@ " # Converts the image into a batch of size 1.\n", " img_batch = tf.expand_dims(img, axis=0)\n", " layer_activations = model(img_batch)\n", + " if len(layer_activations) == 1:\n", + " layer_activations = [layer_activations]\n", "\n", " losses = []\n", " for act in layer_activations:\n", @@ -314,7 +269,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "k4TCNsAUO9kI" }, "source": [ @@ -324,15 +278,13 @@ "\n", "Adding the gradients to the image enhances the patterns seen by the network. At each step, you will have created an image that increasingly excites the activations of certain layers in the network.\n", "\n", - "The method that does this, below, is wrapped in a `tf.function` for performance. It uses an `input_signature` to ensure that the function is not retraced for different image sizes or `steps`/`step_size` values. See the [Concrete functions guide](../../guide/concrete_function.ipynb) for details." + "The method that does this, below, is wrapped in a `tf.function` for performance. It uses an `input_signature` to ensure that the function is not retraced for different image sizes or `steps`/`step_size` values. See the [Concrete functions guide](../../guide/function.ipynb) for details." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "qRScWg_VNqvj" }, "outputs": [], @@ -373,10 +325,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "yB9pTqn6xfuK" }, "outputs": [], @@ -387,7 +337,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XLArRTVHZFAi" }, "source": [ @@ -396,10 +345,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "9vHEcy7dTysi" }, "outputs": [], @@ -412,7 +359,7 @@ " steps_remaining = steps\n", " step = 0\n", " while steps_remaining:\n", - " if steps_remaining\u003e100:\n", + " if steps_remaining>100:\n", " run_steps = tf.constant(100)\n", " else:\n", " run_steps = tf.constant(steps_remaining)\n", @@ -435,10 +382,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "tEfd00rr0j8Z" }, "outputs": [], @@ -450,7 +395,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2PbfXEVFNbGp" }, "source": [ @@ -464,16 +408,13 @@ " \n", "One approach that addresses all these problems is applying gradient ascent at different scales. This will allow patterns generated at smaller scales to be incorporated into patterns at higher scales and filled in with additional detail.\n", "\n", - "To do this you can perform the previous gradient ascent approach, then increase the size of the image (which is reffered to as an octave), and repeat this process for multiple octaves.\n", - "\n" + "To do this you can perform the previous gradient ascent approach, then increase the size of the image (which is referred to as an octave), and repeat this process for multiple octaves.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0eGDSdatLT-8" }, "outputs": [], @@ -506,7 +447,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "s9xqyeuwLZFy" }, "source": [ @@ -523,10 +463,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "oGgLHk7o80ac" }, "outputs": [], @@ -534,29 +472,25 @@ "def random_roll(img, maxroll):\n", " # Randomly shift the image to avoid tiled boundaries.\n", " shift = tf.random.uniform(shape=[2], minval=-maxroll, maxval=maxroll, dtype=tf.int32)\n", - " shift_down, shift_right = shift[0],shift[1] \n", - " img_rolled = tf.roll(tf.roll(img, shift_right, axis=1), shift_down, axis=0)\n", - " return shift_down, shift_right, img_rolled" + " img_rolled = tf.roll(img, shift=shift, axis=[0,1])\n", + " return shift, img_rolled" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "sKsiqWfA9H41" }, "outputs": [], "source": [ - "shift_down, shift_right, img_rolled = random_roll(np.array(original_img), 512)\n", + "shift, img_rolled = random_roll(np.array(original_img), 512)\n", "show(img_rolled)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tGIjA3UhhAt8" }, "source": [ @@ -565,10 +499,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "x__TZ0uqNbGm" }, "outputs": [], @@ -580,19 +512,20 @@ " @tf.function(\n", " input_signature=(\n", " tf.TensorSpec(shape=[None,None,3], dtype=tf.float32),\n", + " tf.TensorSpec(shape=[2], dtype=tf.int32),\n", " tf.TensorSpec(shape=[], dtype=tf.int32),)\n", " )\n", - " def __call__(self, img, tile_size=512):\n", - " shift_down, shift_right, img_rolled = random_roll(img, tile_size)\n", + " def __call__(self, img, img_size, tile_size=512):\n", + " shift, img_rolled = random_roll(img, tile_size)\n", "\n", " # Initialize the image gradients to zero.\n", " gradients = tf.zeros_like(img_rolled)\n", " \n", " # Skip the last tile, unless there's only one tile.\n", - " xs = tf.range(0, img_rolled.shape[0], tile_size)[:-1]\n", + " xs = tf.range(0, img_size[1], tile_size)[:-1]\n", " if not tf.cast(len(xs), bool):\n", " xs = tf.constant([0])\n", - " ys = tf.range(0, img_rolled.shape[1], tile_size)[:-1]\n", + " ys = tf.range(0, img_size[0], tile_size)[:-1]\n", " if not tf.cast(len(ys), bool):\n", " ys = tf.constant([0])\n", "\n", @@ -605,14 +538,14 @@ " tape.watch(img_rolled)\n", "\n", " # Extract a tile out of the image.\n", - " img_tile = img_rolled[x:x+tile_size, y:y+tile_size]\n", + " img_tile = img_rolled[y:y+tile_size, x:x+tile_size]\n", " loss = calc_loss(img_tile, self.model)\n", "\n", " # Update the image gradients for this tile.\n", " gradients = gradients + tape.gradient(loss, img_rolled)\n", "\n", " # Undo the random shift applied to the image and its gradients.\n", - " gradients = tf.roll(tf.roll(gradients, -shift_right, axis=1), -shift_down, axis=0)\n", + " gradients = tf.roll(gradients, shift=-shift, axis=[0,1])\n", "\n", " # Normalize the gradients.\n", " gradients /= tf.math.reduce_std(gradients) + 1e-8 \n", @@ -622,10 +555,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Vcq4GubA2e5J" }, "outputs": [], @@ -636,7 +567,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hYnTTs_qiaND" }, "source": [ @@ -645,10 +575,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gA-15DM4NbGk" }, "outputs": [], @@ -656,7 +584,7 @@ "def run_deep_dream_with_octaves(img, steps_per_octave=100, step_size=0.01, \n", " octaves=range(-2,3), octave_scale=1.3):\n", " base_shape = tf.shape(img)\n", - " img = tf.keras.preprocessing.image.img_to_array(img)\n", + " img = tf.keras.utils.img_to_array(img)\n", " img = tf.keras.applications.inception_v3.preprocess_input(img)\n", "\n", " initial_shape = img.shape[:-1]\n", @@ -664,10 +592,11 @@ " for octave in octaves:\n", " # Scale the image based on the octave\n", " new_size = tf.cast(tf.convert_to_tensor(base_shape[:-1]), tf.float32)*(octave_scale**octave)\n", - " img = tf.image.resize(img, tf.cast(new_size, tf.int32))\n", + " new_size = tf.cast(new_size, tf.int32)\n", + " img = tf.image.resize(img, new_size)\n", "\n", " for step in range(steps_per_octave):\n", - " gradients = get_tiled_gradients(img)\n", + " gradients = get_tiled_gradients(img, new_size)\n", " img = img + gradients*step_size\n", " img = tf.clip_by_value(img, -1, 1)\n", "\n", @@ -682,10 +611,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "T7PbRLV74RrU" }, "outputs": [], @@ -701,7 +628,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0Og0-qLwNbGg" }, "source": [ @@ -716,8 +642,6 @@ "colab": { "collapsed_sections": [], "name": "deepdream.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/generative/images/cvae_latent_space.jpg b/site/en/tutorials/generative/images/cvae_latent_space.jpg new file mode 100644 index 00000000000..fd2392d4030 Binary files /dev/null and b/site/en/tutorials/generative/images/cvae_latent_space.jpg differ diff --git a/site/en/tutorials/generative/images/image_denoise_fmnist_results.png b/site/en/tutorials/generative/images/image_denoise_fmnist_results.png new file mode 100644 index 00000000000..440a02b7405 Binary files /dev/null and b/site/en/tutorials/generative/images/image_denoise_fmnist_results.png differ diff --git a/site/en/tutorials/generative/images/intro_autoencoder_result.png b/site/en/tutorials/generative/images/intro_autoencoder_result.png new file mode 100644 index 00000000000..83b1e4213b6 Binary files /dev/null and b/site/en/tutorials/generative/images/intro_autoencoder_result.png differ diff --git a/site/en/tutorials/generative/pix2pix.ipynb b/site/en/tutorials/generative/pix2pix.ipynb index 5b38d7f5059..e380924d04d 100644 --- a/site/en/tutorials/generative/pix2pix.ipynb +++ b/site/en/tutorials/generative/pix2pix.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "v1CUZ0dkOo_F" }, "source": [ @@ -14,11 +13,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "qmkj-80IHxnd" }, "outputs": [], @@ -39,50 +36,54 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_xnMOsbqHz61" }, "source": [ - "# Pix2Pix" + "# pix2pix: Image-to-image translation with a conditional GAN" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ds4o1h4WHz9U" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/generative/pix2pix\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/generative/pix2pix.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/pix2pix.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/generative/pix2pix.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ITZuApL56Mny" }, "source": [ - "This notebook demonstrates image to image translation using conditional GAN's, as described in [Image-to-Image Translation with Conditional Adversarial Networks](https://arxiv.org/abs/1611.07004). Using this technique we can colorize black and white photos, convert google maps to google earth, etc. Here, we convert building facades to real buildings.\n", + "This tutorial demonstrates how to build and train a conditional generative adversarial network (cGAN) called pix2pix that learns a mapping from input images to output images, as described in [Image-to-image translation with conditional adversarial networks](https://arxiv.org/abs/1611.07004) by Isola et al. (2017). pix2pix is not application specific—it can be applied to a wide range of tasks, including synthesizing photos from label maps, generating colorized photos from black and white images, turning Google Maps photos into aerial images, and even transforming sketches into photos.\n", "\n", - "In example, we will use the [CMP Facade Database](http://cmp.felk.cvut.cz/~tylecr1/facade/), helpfully provided by the [Center for Machine Perception](http://cmp.felk.cvut.cz/) at the [Czech Technical University in Prague](https://www.cvut.cz/). To keep our example short, we will use a preprocessed [copy](https://people.eecs.berkeley.edu/~tinghuiz/projects/pix2pix/datasets/) of this dataset, created by the authors of the [paper](https://arxiv.org/abs/1611.07004) above.\n", + "In this example, your network will generate images of building facades using the [CMP Facade Database](http://cmp.felk.cvut.cz/~tylecr1/facade/) provided by the [Center for Machine Perception](http://cmp.felk.cvut.cz/) at the [Czech Technical University in Prague](https://www.cvut.cz/). To keep it short, you will use a [preprocessed copy](https://efrosgans.eecs.berkeley.edu/pix2pix/datasets/) of this dataset created by the pix2pix authors.\n", "\n", - "Each epoch takes around 15 seconds on a single V100 GPU.\n", + "In the pix2pix cGAN, you condition on input images and generate corresponding output images. cGANs were first proposed in [Conditional Generative Adversarial Nets](https://arxiv.org/abs/1411.1784) (Mirza and Osindero, 2014)\n", "\n", - "Below is the output generated after training the model for 200 epochs.\n", + "The architecture of your network will contain:\n", + "\n", + "- A generator with a [U-Net](https://arxiv.org/abs/1505.04597)-based architecture.\n", + "- A discriminator represented by a convolutional PatchGAN classifier (proposed in the [pix2pix paper](https://arxiv.org/abs/1611.07004)).\n", + "\n", + "Note that each epoch can take around 15 seconds on a single V100 GPU.\n", + "\n", + "Below are some examples of the output generated by the pix2pix cGAN after training for 200 epochs on the facades dataset (80k steps).\n", "\n", "![sample output_1](https://www.tensorflow.org/images/gan/pix2pix_1.png)\n", "![sample output_2](https://www.tensorflow.org/images/gan/pix2pix_2.png)" @@ -91,7 +92,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "e1_Y75QXJS6h" }, "source": [ @@ -100,143 +100,213 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YfIk2es3hJEd" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", "\n", "import os\n", + "import pathlib\n", "import time\n", + "import datetime\n", "\n", "from matplotlib import pyplot as plt\n", "from IPython import display" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "iYn4MdZnKCey" + }, + "source": [ + "## Load the dataset\n", + "\n", + "Download the CMP Facade Database data (30MB). Additional datasets are available in the same format [here](http://efrosgans.eecs.berkeley.edu/pix2pix/datasets/). In Colab you can select other datasets from the drop-down menu. Note that some of the other datasets are significantly larger (`edges2handbags` is 8GB in size). " + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wifwThPoEj7e" + "id": "qp6IAZvEShNf" }, "outputs": [], "source": [ - "!pip install -U tensorboard" + "dataset_name = \"facades\" #@param [\"cityscapes\", \"edges2handbags\", \"edges2shoes\", \"facades\", \"maps\", \"night2day\"]\n" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "iYn4MdZnKCey" + "id": "Kn-k8kTXuAlv" }, + "outputs": [], "source": [ - "## Load the dataset\n", + "_URL = f'http://efrosgans.eecs.berkeley.edu/pix2pix/datasets/{dataset_name}.tar.gz'\n", + "\n", + "path_to_zip = tf.keras.utils.get_file(\n", + " fname=f\"{dataset_name}.tar.gz\",\n", + " origin=_URL,\n", + " extract=True)\n", "\n", - "You can download this dataset and similar datasets from [here](https://people.eecs.berkeley.edu/~tinghuiz/projects/pix2pix/datasets). As mentioned in the [paper](https://arxiv.org/abs/1611.07004) we apply random jittering and mirroring to the training dataset.\n", + "path_to_zip = pathlib.Path(path_to_zip)\n", "\n", - "* In random jittering, the image is resized to `286 x 286` and then randomly cropped to `256 x 256`\n", - "* In random mirroring, the image is randomly flipped horizontally i.e left to right." + "extraction_dir = f'{dataset_name}_extracted/{dataset_name}'\n", + "\n", + "PATH = path_to_zip.parent/extraction_dir" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Kn-k8kTXuAlv" + "id": "V67lt3BFb2iN" }, "outputs": [], "source": [ - "_URL = 'https://people.eecs.berkeley.edu/~tinghuiz/projects/pix2pix/datasets/facades.tar.gz'\n", - "\n", - "path_to_zip = tf.keras.utils.get_file('facades.tar.gz',\n", - " origin=_URL,\n", - " extract=True)\n", - "\n", - "PATH = os.path.join(os.path.dirname(path_to_zip), 'facades/')" + "list(PATH.parent.iterdir())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1fUzsnerj1P3" + }, + "source": [ + "Each original image is of size `256 x 512` containing two `256 x 256` images:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2CbTEt448b4R" + "id": "XGY1kiptguTQ" }, "outputs": [], "source": [ - "BUFFER_SIZE = 400\n", - "BATCH_SIZE = 1\n", - "IMG_WIDTH = 256\n", - "IMG_HEIGHT = 256" + "sample_image = tf.io.read_file(str(PATH / 'train/1.jpg'))\n", + "sample_image = tf.io.decode_jpeg(sample_image)\n", + "print(sample_image.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vJ2sO8Izg7QV" + }, + "outputs": [], + "source": [ + "plt.figure()\n", + "plt.imshow(sample_image)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2A5SU-qxPAqd" + }, + "source": [ + "You need to separate real building facade images from the architecture label images—all of which will be of size `256 x 256`.\n", + "\n", + "Define a function that loads image files and outputs two image tensors:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "aO9ZAGH5K3SY" }, "outputs": [], "source": [ "def load(image_file):\n", + " # Read and decode an image file to a uint8 tensor\n", " image = tf.io.read_file(image_file)\n", - " image = tf.image.decode_jpeg(image)\n", + " image = tf.io.decode_jpeg(image)\n", "\n", + " # Split each image tensor into two tensors:\n", + " # - one with a real building facade image\n", + " # - one with an architecture label image \n", " w = tf.shape(image)[1]\n", - "\n", " w = w // 2\n", - " real_image = image[:, :w, :]\n", " input_image = image[:, w:, :]\n", + " real_image = image[:, :w, :]\n", "\n", + " # Convert both images to float32 tensors\n", " input_image = tf.cast(input_image, tf.float32)\n", " real_image = tf.cast(real_image, tf.float32)\n", "\n", " return input_image, real_image" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "r5ByHTlfE06P" + }, + "source": [ + "Plot a sample of the input (architecture label image) and real (building facade photo) images:" + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4OLHMpsQ5aOv" }, "outputs": [], "source": [ - "inp, re = load(PATH+'train/100.jpg')\n", - "# casting to int for matplotlib to show the image\n", + "inp, re = load(str(PATH / 'train/100.jpg'))\n", + "# Casting to int for matplotlib to display the images\n", "plt.figure()\n", - "plt.imshow(inp/255.0)\n", + "plt.imshow(inp / 255.0)\n", "plt.figure()\n", - "plt.imshow(re/255.0)" + "plt.imshow(re / 255.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PVuZQTfI_c-s" + }, + "source": [ + "As described in the [pix2pix paper](https://arxiv.org/abs/1611.07004), you need to apply random jittering and mirroring to preprocess the training set.\n", + "\n", + "Define several functions that:\n", + "\n", + "1. Resize each `256 x 256` image to a larger height and width—`286 x 286`.\n", + "2. Randomly crop it back to `256 x 256`.\n", + "3. Randomly flip the image horizontally i.e., left to right (random mirroring).\n", + "4. Normalize the images to the `[-1, 1]` range." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, + "metadata": { + "id": "2CbTEt448b4R" + }, + "outputs": [], + "source": [ + "# The facade training set consist of 400 images\n", + "BUFFER_SIZE = 400\n", + "# The batch size of 1 produced better results for the U-Net in the original pix2pix experiment\n", + "BATCH_SIZE = 1\n", + "# Each image is 256x256 in size\n", + "IMG_WIDTH = 256\n", + "IMG_HEIGHT = 256" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "rwwYQpu9FzDu" }, "outputs": [], @@ -252,10 +322,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Yn3IwqhiIszt" }, "outputs": [], @@ -270,16 +338,13 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "muhR2cgbLKWW" }, "outputs": [], "source": [ - "# normalizing the images to [-1, 1]\n", - "\n", + "# Normalizing the images to [-1, 1]\n", "def normalize(input_image, real_image):\n", " input_image = (input_image / 127.5) - 1\n", " real_image = (real_image / 127.5) - 1\n", @@ -289,24 +354,22 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "fVQOjcPVLrUc" }, "outputs": [], "source": [ "@tf.function()\n", "def random_jitter(input_image, real_image):\n", - " # resizing to 286 x 286 x 3\n", + " # Resizing to 286x286\n", " input_image, real_image = resize(input_image, real_image, 286, 286)\n", "\n", - " # randomly cropping to 256 x 256 x 3\n", + " # Random cropping back to 256x256\n", " input_image, real_image = random_crop(input_image, real_image)\n", "\n", - " if tf.random.uniform(()) \u003e 0.5:\n", - " # random mirroring\n", + " if tf.random.uniform(()) > 0.5:\n", + " # Random mirroring\n", " input_image = tf.image.flip_left_right(input_image)\n", " real_image = tf.image.flip_left_right(real_image)\n", "\n", @@ -316,46 +379,42 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wfAQbzy799UV" }, "source": [ - "As you can see in the images below\n", - "that they are going through random jittering\n", - "Random jittering as described in the paper is to\n", - "\n", - "1. Resize an image to bigger height and width\n", - "2. Randomly crop to the target size\n", - "3. Randomly flip the image horizontally" + "You can inspect some of the preprocessed output:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "n0OGdi6D92kM" }, "outputs": [], "source": [ - "\n", - "\n", "plt.figure(figsize=(6, 6))\n", "for i in range(4):\n", " rj_inp, rj_re = random_jitter(inp, re)\n", - " plt.subplot(2, 2, i+1)\n", - " plt.imshow(rj_inp/255.0)\n", + " plt.subplot(2, 2, i + 1)\n", + " plt.imshow(rj_inp / 255.0)\n", " plt.axis('off')\n", "plt.show()" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "3E9LGq3WBmsh" + }, + "source": [ + "Having checked that the loading and preprocessing works, let's define a couple of helper functions that load and preprocess the training and test sets:" + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "tyaP4hLJ8b4W" }, "outputs": [], @@ -370,10 +429,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "VB3Z6D_zKSru" }, "outputs": [], @@ -390,41 +447,39 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PIGN6ouoQxt3" }, "source": [ - "## Input Pipeline" + "## Build an input pipeline with `tf.data`" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "SQHmYSmk8b4b" }, "outputs": [], "source": [ - "train_dataset = tf.data.Dataset.list_files(PATH+'train/*.jpg')\n", + "train_dataset = tf.data.Dataset.list_files(str(PATH / 'train/*.jpg'))\n", "train_dataset = train_dataset.map(load_image_train,\n", - " num_parallel_calls=tf.data.experimental.AUTOTUNE)\n", + " num_parallel_calls=tf.data.AUTOTUNE)\n", "train_dataset = train_dataset.shuffle(BUFFER_SIZE)\n", "train_dataset = train_dataset.batch(BATCH_SIZE)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "MS9J0yA58b4g" }, "outputs": [], "source": [ - "test_dataset = tf.data.Dataset.list_files(PATH+'test/*.jpg')\n", + "try:\n", + " test_dataset = tf.data.Dataset.list_files(str(PATH / 'test/*.jpg'))\n", + "except tf.errors.InvalidArgumentError:\n", + " test_dataset = tf.data.Dataset.list_files(str(PATH / 'val/*.jpg'))\n", "test_dataset = test_dataset.map(load_image_test)\n", "test_dataset = test_dataset.batch(BATCH_SIZE)" ] @@ -432,24 +487,31 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "THY-sZMiQ4UV" }, "source": [ - "## Build the Generator\n", - " * The architecture of generator is a modified U-Net.\n", - " * Each block in the encoder is (Conv -\u003e Batchnorm -\u003e Leaky ReLU)\n", - " * Each block in the decoder is (Transposed Conv -\u003e Batchnorm -\u003e Dropout(applied to the first 3 blocks) -\u003e ReLU)\n", - " * There are skip connections between the encoder and decoder (as in U-Net).\n", - "\n" + "## Build the generator\n", + "\n", + "The generator of your pix2pix cGAN is a _modified_ [U-Net](https://arxiv.org/abs/1505.04597). A U-Net consists of an encoder (downsampler) and decoder (upsampler). (You can find out more about it in the [Image segmentation](../images/segmentation.ipynb) tutorial and on the [U-Net project website](https://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/).)\n", + "\n", + "- Each block in the encoder is: Convolution -> Batch normalization -> Leaky ReLU\n", + "- Each block in the decoder is: Transposed convolution -> Batch normalization -> Dropout (applied to the first 3 blocks) -> ReLU\n", + "- There are skip connections between the encoder and decoder (as in the U-Net)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4MQPuBCgtldI" + }, + "source": [ + "Define the downsampler (encoder):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "tqqvWxlw8b4l" }, "outputs": [], @@ -459,10 +521,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3R09ATE_SH9P" }, "outputs": [], @@ -485,10 +545,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "a6_uCZCppTh7" }, "outputs": [], @@ -498,12 +556,19 @@ "print (down_result.shape)" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "aFI_Pa52tjLl" + }, + "source": [ + "Define the upsampler (decoder):" + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "nhgDsHClSQzP" }, "outputs": [], @@ -530,10 +595,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "mz-ahSdsq0Oc" }, "outputs": [], @@ -543,38 +606,45 @@ "print (up_result.shape)" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "ueEJyRVrtZ-p" + }, + "source": [ + "Define the generator with the downsampler and the upsampler:" + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "lFPI4Nu-8b4q" }, "outputs": [], "source": [ "def Generator():\n", - " inputs = tf.keras.layers.Input(shape=[256,256,3])\n", + " inputs = tf.keras.layers.Input(shape=[256, 256, 3])\n", "\n", " down_stack = [\n", - " downsample(64, 4, apply_batchnorm=False), # (bs, 128, 128, 64)\n", - " downsample(128, 4), # (bs, 64, 64, 128)\n", - " downsample(256, 4), # (bs, 32, 32, 256)\n", - " downsample(512, 4), # (bs, 16, 16, 512)\n", - " downsample(512, 4), # (bs, 8, 8, 512)\n", - " downsample(512, 4), # (bs, 4, 4, 512)\n", - " downsample(512, 4), # (bs, 2, 2, 512)\n", - " downsample(512, 4), # (bs, 1, 1, 512)\n", + " downsample(64, 4, apply_batchnorm=False), # (batch_size, 128, 128, 64)\n", + " downsample(128, 4), # (batch_size, 64, 64, 128)\n", + " downsample(256, 4), # (batch_size, 32, 32, 256)\n", + " downsample(512, 4), # (batch_size, 16, 16, 512)\n", + " downsample(512, 4), # (batch_size, 8, 8, 512)\n", + " downsample(512, 4), # (batch_size, 4, 4, 512)\n", + " downsample(512, 4), # (batch_size, 2, 2, 512)\n", + " downsample(512, 4), # (batch_size, 1, 1, 512)\n", " ]\n", "\n", " up_stack = [\n", - " upsample(512, 4, apply_dropout=True), # (bs, 2, 2, 1024)\n", - " upsample(512, 4, apply_dropout=True), # (bs, 4, 4, 1024)\n", - " upsample(512, 4, apply_dropout=True), # (bs, 8, 8, 1024)\n", - " upsample(512, 4), # (bs, 16, 16, 1024)\n", - " upsample(256, 4), # (bs, 32, 32, 512)\n", - " upsample(128, 4), # (bs, 64, 64, 256)\n", - " upsample(64, 4), # (bs, 128, 128, 128)\n", + " upsample(512, 4, apply_dropout=True), # (batch_size, 2, 2, 1024)\n", + " upsample(512, 4, apply_dropout=True), # (batch_size, 4, 4, 1024)\n", + " upsample(512, 4, apply_dropout=True), # (batch_size, 8, 8, 1024)\n", + " upsample(512, 4), # (batch_size, 16, 16, 1024)\n", + " upsample(256, 4), # (batch_size, 32, 32, 512)\n", + " upsample(128, 4), # (batch_size, 64, 64, 256)\n", + " upsample(64, 4), # (batch_size, 128, 128, 128)\n", " ]\n", "\n", " initializer = tf.random_normal_initializer(0., 0.02)\n", @@ -582,7 +652,7 @@ " strides=2,\n", " padding='same',\n", " kernel_initializer=initializer,\n", - " activation='tanh') # (bs, 256, 256, 3)\n", + " activation='tanh') # (batch_size, 256, 256, 3)\n", "\n", " x = inputs\n", "\n", @@ -604,12 +674,19 @@ " return tf.keras.Model(inputs=inputs, outputs=x)" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z4PKwrcQFYvF" + }, + "source": [ + "Visualize the generator model architecture:" + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dIbRPFzjmV85" }, "outputs": [], @@ -619,49 +696,46 @@ ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U1N1_obwtdQH" + "id": "Z8kbgTK8FcPo" }, - "outputs": [], "source": [ - "gen_output = generator(inp[tf.newaxis,...], training=False)\n", - "plt.imshow(gen_output[0,...])" + "Test the generator:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "dpDPEQXIAiQO" + "id": "U1N1_obwtdQH" }, + "outputs": [], "source": [ - "* **Generator loss**\n", - " * It is a sigmoid cross entropy loss of the generated images and an **array of ones**.\n", - " * The [paper](https://arxiv.org/abs/1611.07004) also includes L1 loss which is MAE (mean absolute error) between the generated image and the target image.\n", - " * This allows the generated image to become structurally similar to the target image.\n", - " * The formula to calculate the total generator loss = gan_loss + LAMBDA * l1_loss, where LAMBDA = 100. This value was decided by the authors of the [paper](https://arxiv.org/abs/1611.07004)." + "gen_output = generator(inp[tf.newaxis, ...], training=False)\n", + "plt.imshow(gen_output[0, ...])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "fSZbDgESHIV6" + "id": "dpDPEQXIAiQO" }, "source": [ - "The training procedure for the generator is shown below:" + "### Define the generator loss\n", + "\n", + "GANs learn a loss that adapts to the data, while cGANs learn a structured loss that penalizes a possible structure that differs from the network output and the target image, as described in the [pix2pix paper](https://arxiv.org/abs/1611.07004).\n", + "\n", + "- The generator loss is a sigmoid cross-entropy loss of the generated images and an **array of ones**.\n", + "- The pix2pix paper also mentions the L1 loss, which is a MAE (mean absolute error) between the generated image and the target image.\n", + "- This allows the generated image to become structurally similar to the target image.\n", + "- The formula to calculate the total generator loss is `gan_loss + LAMBDA * l1_loss`, where `LAMBDA = 100`. This value was decided by the authors of the paper." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "cyhxTuvJyIHV" }, "outputs": [], @@ -670,21 +744,20 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "Yn-NhtQj1b4a" + "id": "Q1Xbz5OaLj5C" }, + "outputs": [], "source": [ - "" + "loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=True)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "90BIcCKcDMxz" }, "outputs": [], @@ -692,7 +765,7 @@ "def generator_loss(disc_generated_output, gen_output, target):\n", " gan_loss = loss_object(tf.ones_like(disc_generated_output), disc_generated_output)\n", "\n", - " # mean absolute error\n", + " # Mean absolute error\n", " l1_loss = tf.reduce_mean(tf.abs(target - gen_output))\n", "\n", " total_gen_loss = gan_loss + (LAMBDA * l1_loss)\n", @@ -703,39 +776,53 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "fSZbDgESHIV6" + }, + "source": [ + "The training procedure for the generator is as follows:" + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "TlB-XMY5Awj9" }, "source": [ - "\n", - "\n", "![Generator Update Image](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/images/gen.png?raw=1)\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ZTKZfoaoEF22" }, "source": [ - "## Build the Discriminator\n", - " * The Discriminator is a PatchGAN.\n", - " * Each block in the discriminator is (Conv -\u003e BatchNorm -\u003e Leaky ReLU)\n", - " * The shape of the output after the last layer is (batch_size, 30, 30, 1)\n", - " * Each 30x30 patch of the output classifies a 70x70 portion of the input image (such an architecture is called a PatchGAN).\n", - " * Discriminator receives 2 inputs.\n", - " * Input image and the target image, which it should classify as real.\n", - " * Input image and the generated image (output of generator), which it should classify as fake.\n", - " * We concatenate these 2 inputs together in the code (`tf.concat([inp, tar], axis=-1)`)" + "## Build the discriminator\n", + "\n", + "The discriminator in the pix2pix cGAN is a convolutional PatchGAN classifier—it tries to classify if each image _patch_ is real or not real, as described in the [pix2pix paper](https://arxiv.org/abs/1611.07004).\n", + "\n", + "- Each block in the discriminator is: Convolution -> Batch normalization -> Leaky ReLU.\n", + "- The shape of the output after the last layer is `(batch_size, 30, 30, 1)`.\n", + "- Each `30 x 30` image patch of the output classifies a `70 x 70` portion of the input image.\n", + "- The discriminator receives 2 inputs: \n", + " - The input image and the target image, which it should classify as real.\n", + " - The input image and the generated image (the output of the generator), which it should classify as fake.\n", + " - Use `tf.concat([inp, tar], axis=-1)` to concatenate these 2 inputs together." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XIuTeGL5v45m" + }, + "source": [ + "Let's define the discriminator:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ll6aNeQx8b4v" }, "outputs": [], @@ -746,91 +833,90 @@ " inp = tf.keras.layers.Input(shape=[256, 256, 3], name='input_image')\n", " tar = tf.keras.layers.Input(shape=[256, 256, 3], name='target_image')\n", "\n", - " x = tf.keras.layers.concatenate([inp, tar]) # (bs, 256, 256, channels*2)\n", + " x = tf.keras.layers.concatenate([inp, tar]) # (batch_size, 256, 256, channels*2)\n", "\n", - " down1 = downsample(64, 4, False)(x) # (bs, 128, 128, 64)\n", - " down2 = downsample(128, 4)(down1) # (bs, 64, 64, 128)\n", - " down3 = downsample(256, 4)(down2) # (bs, 32, 32, 256)\n", + " down1 = downsample(64, 4, False)(x) # (batch_size, 128, 128, 64)\n", + " down2 = downsample(128, 4)(down1) # (batch_size, 64, 64, 128)\n", + " down3 = downsample(256, 4)(down2) # (batch_size, 32, 32, 256)\n", "\n", - " zero_pad1 = tf.keras.layers.ZeroPadding2D()(down3) # (bs, 34, 34, 256)\n", + " zero_pad1 = tf.keras.layers.ZeroPadding2D()(down3) # (batch_size, 34, 34, 256)\n", " conv = tf.keras.layers.Conv2D(512, 4, strides=1,\n", " kernel_initializer=initializer,\n", - " use_bias=False)(zero_pad1) # (bs, 31, 31, 512)\n", + " use_bias=False)(zero_pad1) # (batch_size, 31, 31, 512)\n", "\n", " batchnorm1 = tf.keras.layers.BatchNormalization()(conv)\n", "\n", " leaky_relu = tf.keras.layers.LeakyReLU()(batchnorm1)\n", "\n", - " zero_pad2 = tf.keras.layers.ZeroPadding2D()(leaky_relu) # (bs, 33, 33, 512)\n", + " zero_pad2 = tf.keras.layers.ZeroPadding2D()(leaky_relu) # (batch_size, 33, 33, 512)\n", "\n", " last = tf.keras.layers.Conv2D(1, 4, strides=1,\n", - " kernel_initializer=initializer)(zero_pad2) # (bs, 30, 30, 1)\n", + " kernel_initializer=initializer)(zero_pad2) # (batch_size, 30, 30, 1)\n", "\n", " return tf.keras.Model(inputs=[inp, tar], outputs=last)" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YHoUui4om-Ev" + "id": "HdV9yAbBHNkg" }, - "outputs": [], "source": [ - "discriminator = Discriminator()\n", - "tf.keras.utils.plot_model(discriminator, show_shapes=True, dpi=64)" + "Visualize the discriminator model architecture:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gDkA05NE6QMs" + "id": "YHoUui4om-Ev" }, "outputs": [], "source": [ - "disc_out = discriminator([inp[tf.newaxis,...], gen_output], training=False)\n", - "plt.imshow(disc_out[0,...,-1], vmin=-20, vmax=20, cmap='RdBu_r')\n", - "plt.colorbar()" + "discriminator = Discriminator()\n", + "tf.keras.utils.plot_model(discriminator, show_shapes=True, dpi=64)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "AOqg1dhUAWoD" + "id": "ps7nIHigHYc7" }, "source": [ - "**Discriminator loss**\n", - " * The discriminator loss function takes 2 inputs; **real images, generated images**\n", - " * real_loss is a sigmoid cross entropy loss of the **real images** and an **array of ones(since these are the real images)**\n", - " * generated_loss is a sigmoid cross entropy loss of the **generated images** and an **array of zeros(since these are the fake images)**\n", - " * Then the total_loss is the sum of real_loss and the generated_loss\n" + "Test the discriminator:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Q1Xbz5OaLj5C" + "id": "gDkA05NE6QMs" }, "outputs": [], "source": [ - "loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=True)" + "disc_out = discriminator([inp[tf.newaxis, ...], gen_output], training=False)\n", + "plt.imshow(disc_out[0, ..., -1], vmin=-20, vmax=20, cmap='RdBu_r')\n", + "plt.colorbar()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AOqg1dhUAWoD" + }, + "source": [ + "### Define the discriminator loss\n", + "\n", + "- The `discriminator_loss` function takes 2 inputs: **real images** and **generated images**.\n", + "- `real_loss` is a sigmoid cross-entropy loss of the **real images** and an **array of ones(since these are the real images)**.\n", + "- `generated_loss` is a sigmoid cross-entropy loss of the **generated images** and an **array of zeros (since these are the fake images)**.\n", + "- The `total_loss` is the sum of `real_loss` and `generated_loss`." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wkMNfBWlT-PV" }, "outputs": [], @@ -848,44 +934,36 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-ede4p2YELFa" }, "source": [ "The training procedure for the discriminator is shown below.\n", "\n", - "To learn more about the architecture and the hyperparameters you can refer the [paper](https://arxiv.org/abs/1611.07004)." + "To learn more about the architecture and the hyperparameters you can refer to the [pix2pix paper](https://arxiv.org/abs/1611.07004)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IS9sHa-1BoAF" }, "source": [ - "\n", "![Discriminator Update Image](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/images/dis.png?raw=1)\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0FMYgY_mPfTi" }, "source": [ - "## Define the Optimizers and Checkpoint-saver\n", - "\n", - "\n" + "## Define the optimizers and a checkpoint-saver\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "lbHFNexF0x6O" }, "outputs": [], @@ -896,10 +974,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WJnftd5sQsv6" }, "outputs": [], @@ -915,46 +991,38 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Rw1fkAczTQYh" }, "source": [ - "## Generate Images\n", + "## Generate images\n", "\n", "Write a function to plot some images during training.\n", "\n", - "* We pass images from the test dataset to the generator.\n", - "* The generator will then translate the input image into the output.\n", - "* Last step is to plot the predictions and **voila!**" + "- Pass images from the test set to the generator.\n", + "- The generator will then translate the input image into the output.\n", + "- The last step is to plot the predictions and _voila_!" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Rb0QQFHF-JfS" }, "source": [ - "Note: The `training=True` is intentional here since\n", - "we want the batch statistics while running the model\n", - "on the test dataset. If we use training=False, we will get\n", - "the accumulated statistics learned from the training dataset\n", - "(which we don't want)" + "Note: The `training=True` is intentional here since you want the batch statistics, while running the model on the test dataset. If you use `training=False`, you get the accumulated statistics learned from the training dataset (which you don't want)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "RmdVsmvhPxyy" }, "outputs": [], "source": [ "def generate_images(model, test_input, tar):\n", " prediction = model(test_input, training=True)\n", - " plt.figure(figsize=(15,15))\n", + " plt.figure(figsize=(15, 15))\n", "\n", " display_list = [test_input[0], tar[0], prediction[0]]\n", " title = ['Input Image', 'Ground Truth', 'Predicted Image']\n", @@ -962,18 +1030,25 @@ " for i in range(3):\n", " plt.subplot(1, 3, i+1)\n", " plt.title(title[i])\n", - " # getting the pixel values between [0, 1] to plot it.\n", + " # Getting the pixel values in the [0, 1] range to plot.\n", " plt.imshow(display_list[i] * 0.5 + 0.5)\n", " plt.axis('off')\n", " plt.show()" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "gipsSEoZIG1a" + }, + "source": [ + "Test the function:" + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8Fc4NzT-DgEx" }, "outputs": [], @@ -985,43 +1060,26 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NLKOG55MErD0" }, "source": [ "## Training\n", "\n", - "* For each example input generate an output.\n", - "* The discriminator receives the input_image and the generated image as the first input. The second input is the input_image and the target_image.\n", - "* Next, we calculate the generator and the discriminator loss.\n", - "* Then, we calculate the gradients of loss with respect to both the generator and the discriminator variables(inputs) and apply those to the optimizer.\n", - "* Then log the losses to TensorBoard." + "- For each example input generates an output.\n", + "- The discriminator receives the `input_image` and the generated image as the first input. The second input is the `input_image` and the `target_image`.\n", + "- Next, calculate the generator and the discriminator loss.\n", + "- Then, calculate the gradients of loss with respect to both the generator and the discriminator variables(inputs) and apply those to the optimizer.\n", + "- Finally, log the losses to TensorBoard." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NS2GWywBbAWo" - }, - "outputs": [], - "source": [ - "EPOCHS = 150" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", "id": "xNNMDBNH12q-" }, "outputs": [], "source": [ - "import datetime\n", "log_dir=\"logs/\"\n", "\n", "summary_writer = tf.summary.create_file_writer(\n", @@ -1030,16 +1088,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "KBKUV2sKXDbY" }, "outputs": [], "source": [ "@tf.function\n", - "def train_step(input_image, target, epoch):\n", + "def train_step(input_image, target, step):\n", " with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n", " gen_output = generator(input_image, training=True)\n", "\n", @@ -1060,192 +1116,145 @@ " discriminator.trainable_variables))\n", "\n", " with summary_writer.as_default():\n", - " tf.summary.scalar('gen_total_loss', gen_total_loss, step=epoch)\n", - " tf.summary.scalar('gen_gan_loss', gen_gan_loss, step=epoch)\n", - " tf.summary.scalar('gen_l1_loss', gen_l1_loss, step=epoch)\n", - " tf.summary.scalar('disc_loss', disc_loss, step=epoch)" + " tf.summary.scalar('gen_total_loss', gen_total_loss, step=step//1000)\n", + " tf.summary.scalar('gen_gan_loss', gen_gan_loss, step=step//1000)\n", + " tf.summary.scalar('gen_l1_loss', gen_l1_loss, step=step//1000)\n", + " tf.summary.scalar('disc_loss', disc_loss, step=step//1000)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hx7s-vBHFKdh" }, "source": [ - "The actual training loop:\n", + "The actual training loop. Since this tutorial can run of more than one dataset, and the datasets vary greatly in size the training loop is setup to work in steps instead of epochs.\n", "\n", - "* Iterates over the number of epochs.\n", - "* On each epoch it clears the display, and runs `generate_images` to show it's progress.\n", - "* On each epoch it iterates over the training dataset, printing a '.' for each example.\n", - "* It saves a checkpoint every 20 epochs." + "- Iterates over the number of steps.\n", + "- Every 10 steps print a dot (`.`).\n", + "- Every 1k steps: clear the display and run `generate_images` to show the progress.\n", + "- Every 5k steps: save a checkpoint." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2M7LmLtGEMQJ" + "id": "GFyPlBWv1B5j" }, "outputs": [], "source": [ - "def fit(train_ds, epochs, test_ds):\n", - " for epoch in range(epochs):\n", - " start = time.time()\n", + "def fit(train_ds, test_ds, steps):\n", + " example_input, example_target = next(iter(test_ds.take(1)))\n", + " start = time.time()\n", "\n", - " display.clear_output(wait=True)\n", + " for step, (input_image, target) in train_ds.repeat().take(steps).enumerate():\n", + " if (step) % 1000 == 0:\n", + " display.clear_output(wait=True)\n", + "\n", + " if step != 0:\n", + " print(f'Time taken for 1000 steps: {time.time()-start:.2f} sec\\n')\n", + "\n", + " start = time.time()\n", "\n", - " for example_input, example_target in test_ds.take(1):\n", " generate_images(generator, example_input, example_target)\n", - " print(\"Epoch: \", epoch)\n", + " print(f\"Step: {step//1000}k\")\n", "\n", - " # Train\n", - " for n, (input_image, target) in train_ds.enumerate():\n", - " print('.', end='')\n", - " if (n+1) % 100 == 0:\n", - " print()\n", - " train_step(input_image, target, epoch)\n", - " print()\n", + " train_step(input_image, target, step)\n", "\n", - " # saving (checkpoint) the model every 20 epochs\n", - " if (epoch + 1) % 20 == 0:\n", - " checkpoint.save(file_prefix = checkpoint_prefix)\n", + " # Training step\n", + " if (step+1) % 10 == 0:\n", + " print('.', end='', flush=True)\n", "\n", - " print ('Time taken for epoch {} is {} sec\\n'.format(epoch + 1,\n", - " time.time()-start))\n", - " checkpoint.save(file_prefix = checkpoint_prefix)" + "\n", + " # Save (checkpoint) the model every 5k steps\n", + " if (step + 1) % 5000 == 0:\n", + " checkpoint.save(file_prefix=checkpoint_prefix)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wozqyTh2wmCu" }, "source": [ - "This training loop saves logs you can easily view in TensorBoard to monitor the training progress. Working locally you would launch a separate tensorboard process. In a notebook, if you want to monitor with TensorBoard it's easiest to launch the viewer before starting the training.\n", + "This training loop saves logs that you can view in TensorBoard to monitor the training progress.\n", "\n", - "To launch the viewer paste the following into a code-cell:" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BhNh-9x53GLT" - }, - "source": [ - "```\n", - "%load_ext tensorboard\n", - "%tensorboard --logdir {log_dir}\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Pe0-8Bzg22ox" - }, - "source": [ - "Now run the training loop:" + "If you work on a local machine, you would launch a separate TensorBoard process. When working in a notebook, launch the viewer before starting the training to monitor with TensorBoard.\n", + "\n", + "Launch the TensorBoard viewer (Sorry, this doesn't\n", + "display on tensorflow.org):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "a1zZmKmvOH85" + "id": "Ot22ujrlLhOd" }, "outputs": [], "source": [ - "fit(train_dataset, EPOCHS, test_dataset)" + "%load_ext tensorboard\n", + "%tensorboard --logdir {log_dir}" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "oeq9sByu86-B" + "id": "fyjixlMlBybN" }, "source": [ - "If you want to share the TensorBoard results _publicly_ you can upload the logs to [TensorBoard.dev](https://tensorboard.dev/) by copying the following into a code-cell.\n", - "\n", - "Note: This requires a Google account.\n", - "\n", - "```\n", - "!tensorboard dev upload --logdir {log_dir}\n", - "```\n", - "\n", - "Caution: This command does not terminate. It's designed to continuously upload the results of long-running experiments. Once your data is uploaded you need to stop it using the \"interrupt execution\" option in your notebook tool." + "You can view the [results of a previous run](https://tensorboard.dev/experiment/lZ0C6FONROaUMfjYkVyJqw) of this notebook on [TensorBoard.dev](https://tensorboard.dev/)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "-lGhS_LfwQoL" + "id": "Pe0-8Bzg22ox" }, "source": [ - "You can view the [results of a previous run](https://tensorboard.dev/experiment/lZ0C6FONROaUMfjYkVyJqw) of this notebook on [TensorBoard.dev](https://tensorboard.dev/).\n", - "\n", - "TensorBoard.dev is a managed experience for hosting, tracking, and sharing ML experiments with everyone.\n", - "\n", - "It can also included inline using an `\u003ciframe\u003e`:" + "Finally, run the training loop:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8IS4c93guQ8E" + "id": "a1zZmKmvOH85" }, "outputs": [], "source": [ - "display.IFrame(\n", - " src=\"https://tensorboard.dev/experiment/lZ0C6FONROaUMfjYkVyJqw\",\n", - " width=\"100%\",\n", - " height=\"1000px\")" + "fit(train_dataset, test_dataset, steps=40000)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DMTm4peo3cem" }, "source": [ - "Interpreting the logs from a GAN is more subtle than a simple classification or regression model. Things to look for::\n", + "Interpreting the logs is more subtle when training a GAN (or a cGAN like pix2pix) compared to a simple classification or regression model. Things to look for:\n", "\n", - "* Check that neither model has \"won\". If either the `gen_gan_loss` or the `disc_loss` gets very low it's an indicator that this model is dominating the other, and you are not successfully training the combined model.\n", - "* The value `log(2) = 0.69` is a good reference point for these losses, as it indicates a perplexity of 2: That the discriminator is on average equally uncertain about the two options.\n", - "* For the `disc_loss` a value below `0.69` means the discriminator is doing better than random, on the combined set of real+generated images.\n", - "* For the `gen_gan_loss` a value below `0.69` means the generator i doing better than random at foolding the descriminator.\n", - "* As training progresses the `gen_l1_loss` should go down." + "- Check that neither the generator nor the discriminator model has \"won\". If either the `gen_gan_loss` or the `disc_loss` gets very low, it's an indicator that this model is dominating the other, and you are not successfully training the combined model.\n", + "- The value `log(2) = 0.69` is a good reference point for these losses, as it indicates a perplexity of 2 - the discriminator is, on average, equally uncertain about the two options.\n", + "- For the `disc_loss`, a value below `0.69` means the discriminator is doing better than random on the combined set of real and generated images.\n", + "- For the `gen_gan_loss`, a value below `0.69` means the generator is doing better than random at fooling the discriminator.\n", + "- As training progresses, the `gen_l1_loss` should go down." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kz80bY3aQ1VZ" }, "source": [ - "## Restore the latest checkpoint and test" + "## Restore the latest checkpoint and test the network" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "HSSm4kfvJiqv" }, "outputs": [], @@ -1255,39 +1264,34 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4t4x69adQ5xb" }, "outputs": [], "source": [ - "# restoring the latest checkpoint in checkpoint_dir\n", + "# Restoring the latest checkpoint in checkpoint_dir\n", "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1RGysMU_BZhx" }, "source": [ - "## Generate using test dataset" + "## Generate some images using the test set" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "KUgSnmy2nqSP" }, "outputs": [], "source": [ - "# Run the trained model on a few examples from the test dataset\n", + "# Run the trained model on a few examples from the test set\n", "for inp, tar in test_dataset.take(5):\n", " generate_images(generator, inp, tar)" ] @@ -1296,11 +1300,8 @@ "metadata": { "accelerator": "GPU", "colab": { - "collapsed_sections": [], "name": "pix2pix.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/generative/style_transfer.ipynb b/site/en/tutorials/generative/style_transfer.ipynb index 9c2434de7e2..c8f1376624e 100644 --- a/site/en/tutorials/generative/style_transfer.ipynb +++ b/site/en/tutorials/generative/style_transfer.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "g_nWetWWd_ns" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "2pHVBk_seED1" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6msVLevwcRhm" }, "source": [ @@ -47,69 +43,82 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ds4o1h4WHz9U" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/generative/style_transfer\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/generative/style_transfer.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/style_transfer.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/generative/style_transfer.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View on GitHub\n", + " \n", + " Download notebook\n", + " \n", + " See TF Hub model\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "aDyGj8DmXCJI" }, "source": [ - "This tutorial uses deep learning to compose one image in the style of another image (ever wish you could paint like Picasso or Van Gogh?). This is known as *neural style transfer* and the technique is outlined in \u003ca href=\"https://arxiv.org/abs/1508.06576\" class=\"external\"\u003eA Neural Algorithm of Artistic Style\u003c/a\u003e (Gatys et al.). \n", + "This tutorial uses deep learning to compose one image in the style of another image (ever wish you could paint like Picasso or Van Gogh?). This is known as *neural style transfer* and the technique is outlined in A Neural Algorithm of Artistic Style (Gatys et al.). \n", "\n", - "Note: This tutorial demonstrates the original style-transfer algorithm. It optimizes the image content to a particular style. Modern approaches train a model to generate the stylized image directly (similar to [cyclegan](cyclegan.ipynb)). This approach is much faster (up to 1000x). A pretrained [Arbitrary Image Stylization module](https://colab.sandbox.google.com/github/tensorflow/hub/blob/master/examples/colab/tf2_arbitrary_image_stylization.ipynb) is available in [TensorFlow Hub](https://tensorflow.org/hub), and for [TensorFlow Lite](https://www.tensorflow.org/lite/models/style_transfer/overview).\n", + "Note: This tutorial demonstrates the original style-transfer algorithm. It optimizes the image content to a particular style. Modern approaches train a model to generate the stylized image directly (similar to [CycleGAN](./cyclegan.ipynb)). This approach is much faster (up to 1000x).\n", "\n", + "For a simple application of style transfer with a pretrained model from [TensorFlow Hub](https://tfhub.dev), check out the [Fast style transfer for arbitrary styles](https://www.tensorflow.org/hub/tutorials/tf2_arbitrary_image_stylization) tutorial that uses an [arbitrary image stylization model](https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2). For an example of style transfer with [TensorFlow Lite](https://www.tensorflow.org/lite), refer to [Artistic style transfer with TensorFlow Lite](https://www.tensorflow.org/lite/examples/style_transfer/overview)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1b3XwN9V1nvR" + }, + "source": [ "Neural style transfer is an optimization technique used to take two images—a *content* image and a *style reference* image (such as an artwork by a famous painter)—and blend them together so the output image looks like the content image, but “painted” in the style of the style reference image.\n", "\n", - "This is implemented by optimizing the output image to match the content statistics of the content image and the style statistics of the style reference image. These statistics are extracted from the images using a convolutional network.\n", - "\n", - "\n" + "This is implemented by optimizing the output image to match the content statistics of the content image and the style statistics of the style reference image. These statistics are extracted from the images using a convolutional network." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "28W8ggyO1KER" + }, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { "id": "3kb_UJY-jCEl" }, "source": [ "For example, let’s take an image of this dog and Wassily Kandinsky's Composition 7:\n", "\n", - "\u003cimg src=\"https://storage.googleapis.com/download.tensorflow.org/example_images/YellowLabradorLooking_new.jpg\" width=\"500px\"/\u003e\n", + "\n", "\n", - "[Yellow Labrador Looking](https://commons.wikimedia.org/wiki/File:YellowLabradorLooking_new.jpg), from Wikimedia Commons\n", + "[Yellow Labrador Looking](https://commons.wikimedia.org/wiki/File:YellowLabradorLooking_new.jpg), from Wikimedia Commons by [Elf](https://en.wikipedia.org/wiki/User:Elf). License [CC BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0/deed.en)\n", "\n", - "\u003cimg src=\"https://storage.googleapis.com/download.tensorflow.org/example_images/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg\" width=\"500px\"/\u003e\n", + "\n", "\n", "\n", - "Now how would it look like if Kandinsky decided to paint the picture of this Dog exclusively with this style? Something like this?\n", + "Now, what would it look like if Kandinsky decided to paint the picture of this Dog exclusively with this style? Something like this?\n", "\n", - "\u003cimg src=\"https://tensorflow.org/tutorials/generative/images/stylized-image.png\" style=\"width: 500px;\"/\u003e" + "" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "U8ajP_u73s6m" }, "source": [ @@ -119,7 +128,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "eqxUicSPUOP6" }, "source": [ @@ -128,41 +136,22 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2Mdpou0qzCm6" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", "id": "NyftRTSMuwue" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" + "import os\n", + "import tensorflow as tf\n", + "# Load compressed models from tensorflow_hub\n", + "os.environ['TFHUB_MODEL_LOAD_FORMAT'] = 'COMPRESSED'" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "sc1OLbOWhPCO" }, "outputs": [], @@ -171,7 +160,7 @@ "\n", "import matplotlib.pyplot as plt\n", "import matplotlib as mpl\n", - "mpl.rcParams['figure.figsize'] = (12,12)\n", + "mpl.rcParams['figure.figsize'] = (12, 12)\n", "mpl.rcParams['axes.grid'] = False\n", "\n", "import numpy as np\n", @@ -182,10 +171,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "GM6VEGrGLh62" }, "outputs": [], @@ -193,7 +180,7 @@ "def tensor_to_image(tensor):\n", " tensor = tensor*255\n", " tensor = np.array(tensor, dtype=np.uint8)\n", - " if np.ndim(tensor)\u003e3:\n", + " if np.ndim(tensor)>3:\n", " assert tensor.shape[0] == 1\n", " tensor = tensor[0]\n", " return PIL.Image.fromarray(tensor)" @@ -202,7 +189,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "oeXebYusyHwC" }, "source": [ @@ -211,25 +197,19 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wqc0OJHwyFAk" }, "outputs": [], "source": [ - "\n", "content_path = tf.keras.utils.get_file('YellowLabradorLooking_new.jpg', 'https://storage.googleapis.com/download.tensorflow.org/example_images/YellowLabradorLooking_new.jpg')\n", - "\n", - "# https://commons.wikimedia.org/wiki/File:Vassily_Kandinsky,_1913_-_Composition_7.jpg\n", "style_path = tf.keras.utils.get_file('kandinsky5.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xE4Yt8nArTeR" }, "source": [ @@ -239,7 +219,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "klh6ObK2t_vH" }, "source": [ @@ -248,10 +227,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3TLljcwv5qZs" }, "outputs": [], @@ -276,7 +253,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2yAlRzJZrWM3" }, "source": [ @@ -285,16 +261,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "cBX-eNT8PAK_" }, "outputs": [], "source": [ "def imshow(image, title=None):\n", - " if len(image.shape) \u003e 3:\n", + " if len(image.shape) > 3:\n", " image = tf.squeeze(image, axis=0)\n", "\n", " plt.imshow(image)\n", @@ -304,10 +278,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "_UWQmeEaiKkP" }, "outputs": [], @@ -325,35 +297,31 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YMzChXSlKTA2" }, "source": [ "## Fast Style Transfer using TF-Hub\n", "\n", - "This tutorial demonstrates the original style-transfer algorithm. Which optimizes the image content to a particular style. Before getting into the details let's see how the [TensorFlow Hub](https://tensorflow.org/hub) module does:" + "This tutorial demonstrates the original style-transfer algorithm, which optimizes the image content to a particular style. Before getting into the details, let's see how the [TensorFlow Hub model](https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2) does this:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iYSLexgRKSh-" }, "outputs": [], "source": [ "import tensorflow_hub as hub\n", - "hub_module = hub.load('https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/1')\n", - "stylized_image = hub_module(tf.constant(content_image), tf.constant(style_image))[0]\n", + "hub_model = hub.load('https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2')\n", + "stylized_image = hub_model(tf.constant(content_image), tf.constant(style_image))[0]\n", "tensor_to_image(stylized_image)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GEwZ7FlwrjoZ" }, "source": [ @@ -365,19 +333,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LP_7zrziuiJk" }, "source": [ - "Load a [VGG19](https://keras.io/applications/#vgg19) and test run it on our image to ensure it's used correctly:" + "Load a [VGG19](https://keras.io/api/applications/vgg/#vgg19-function) and test run it on our image to ensure it's used correctly:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "fMbzrr7BCTq0" }, "outputs": [], @@ -391,10 +356,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "1_FyCm0dYnvl" }, "outputs": [], @@ -406,7 +369,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ljpoYk-0f6HS" }, "source": [ @@ -415,10 +377,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Yh_AV6220ebD" }, "outputs": [], @@ -433,7 +393,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Wt-tASys0eJv" }, "source": [ @@ -442,18 +401,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ArfX_6iA0WAX" }, "outputs": [], "source": [ - "# Content layer where will pull our feature maps\n", "content_layers = ['block5_conv2'] \n", "\n", - "# Style layer of interest\n", "style_layers = ['block1_conv1',\n", " 'block2_conv1',\n", " 'block3_conv1', \n", @@ -467,7 +422,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2o4nSwuN0U3X" }, "source": [ @@ -483,7 +437,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Jt3i3RRrJiOX" }, "source": [ @@ -500,17 +453,15 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "nfec6MuMAbPx" }, "outputs": [], "source": [ "def vgg_layers(layer_names):\n", - " \"\"\" Creates a vgg model that returns a list of intermediate output values.\"\"\"\n", - " # Load our model. Load pretrained VGG, trained on imagenet data\n", + " \"\"\" Creates a VGG model that returns a list of intermediate output values.\"\"\"\n", + " # Load our model. Load pretrained VGG, trained on ImageNet data\n", " vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\n", " vgg.trainable = False\n", " \n", @@ -523,7 +474,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jbaIvZf5wWn_" }, "source": [ @@ -532,10 +482,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "LkyvPpBHSfVi" }, "outputs": [], @@ -556,7 +504,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lGUfttK9F8d5" }, "source": [ @@ -564,7 +511,7 @@ "\n", "The content of an image is represented by the values of the intermediate feature maps.\n", "\n", - "It turns out, the style of an image can be described by the means and correlations across the different feature maps. Calculate a Gram matrix that includes this information by taking the outer product of the feature vector with itself at each location, and averaging that outer product over all locations. This Gram matrix can be calcualted for a particular layer as:\n", + "It turns out, the style of an image can be described by the means and correlations across the different feature maps. Calculate a Gram matrix that includes this information by taking the outer product of the feature vector with itself at each location, and averaging that outer product over all locations. This Gram matrix can be calculated for a particular layer as:\n", "\n", "$$G^l_{cd} = \\frac{\\sum_{ij} F^l_{ijc}(x)F^l_{ijd}(x)}{IJ}$$\n", "\n", @@ -573,16 +520,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "HAy1iGPdoEpZ" }, "outputs": [], "source": [ "def gram_matrix(input_tensor):\n", - " result = tf.linalg.einsum('bijc,bijd-\u003ebcd', input_tensor, input_tensor)\n", + " result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor)\n", " input_shape = tf.shape(input_tensor)\n", " num_locations = tf.cast(input_shape[1]*input_shape[2], tf.float32)\n", " return result/(num_locations)" @@ -591,7 +536,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pXIUX6czZABh" }, "source": [ @@ -601,7 +545,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1HGHvwlJ1nkn" }, "source": [ @@ -610,10 +553,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Sr6QALY-I1ja" }, "outputs": [], @@ -621,7 +562,7 @@ "class StyleContentModel(tf.keras.models.Model):\n", " def __init__(self, style_layers, content_layers):\n", " super(StyleContentModel, self).__init__()\n", - " self.vgg = vgg_layers(style_layers + content_layers)\n", + " self.vgg = vgg_layers(style_layers + content_layers)\n", " self.style_layers = style_layers\n", " self.content_layers = content_layers\n", " self.num_style_layers = len(style_layers)\n", @@ -632,27 +573,26 @@ " inputs = inputs*255.0\n", " preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs)\n", " outputs = self.vgg(preprocessed_input)\n", - " style_outputs, content_outputs = (outputs[:self.num_style_layers], \n", + " style_outputs, content_outputs = (outputs[:self.num_style_layers],\n", " outputs[self.num_style_layers:])\n", "\n", " style_outputs = [gram_matrix(style_output)\n", " for style_output in style_outputs]\n", "\n", - " content_dict = {content_name:value \n", - " for content_name, value \n", + " content_dict = {content_name: value\n", + " for content_name, value\n", " in zip(self.content_layers, content_outputs)}\n", "\n", - " style_dict = {style_name:value\n", + " style_dict = {style_name: value\n", " for style_name, value\n", " in zip(self.style_layers, style_outputs)}\n", - " \n", - " return {'content':content_dict, 'style':style_dict}" + "\n", + " return {'content': content_dict, 'style': style_dict}" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Xuj1o33t1edl" }, "source": [ @@ -661,10 +601,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "rkjO-DoNDU0A" }, "outputs": [], @@ -673,8 +611,6 @@ "\n", "results = extractor(tf.constant(content_image))\n", "\n", - "style_results = results['style']\n", - "\n", "print('Styles:')\n", "for name, output in sorted(results['style'].items()):\n", " print(\" \", name)\n", @@ -696,7 +632,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "y9r8Lyjb_m0u" }, "source": [ @@ -709,10 +644,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "PgkNOnGUFcKa" }, "outputs": [], @@ -724,7 +657,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CNPrpl-e_w9A" }, "source": [ @@ -733,10 +665,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "J0vKxF8ZO6G8" }, "outputs": [], @@ -747,7 +677,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "M6L8ojmn_6rH" }, "source": [ @@ -756,10 +685,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "kdgpTJwL_vE2" }, "outputs": [], @@ -771,30 +698,26 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MBU5RFpcAo7W" }, "source": [ - "Create an optimizer. The paper recommends LBFGS, but `Adam` works okay, too:" + "Create an optimizer. The paper recommends LBFGS, but Adam works okay, too:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "r4XZjqUk_5Eu" }, "outputs": [], "source": [ - "opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)" + "opt = tf.keras.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "As-evbBiA2qT" }, "source": [ @@ -803,10 +726,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Dt4pxarvA4I4" }, "outputs": [], @@ -817,10 +738,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0ggx2Na8oROH" }, "outputs": [], @@ -842,7 +761,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vbF2WnP9BI5M" }, "source": [ @@ -851,10 +769,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0t0umkajFIuh" }, "outputs": [], @@ -873,7 +789,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5FHMJq4UBRIQ" }, "source": [ @@ -882,10 +797,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Y542mxi-O2a2" }, "outputs": [], @@ -899,7 +812,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mNzE-mTbBVgY" }, "source": [ @@ -908,10 +820,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "rQW1tXYoLbUS" }, "outputs": [], @@ -927,7 +837,7 @@ " for m in range(steps_per_epoch):\n", " step += 1\n", " train_step(image)\n", - " print(\".\", end='')\n", + " print(\".\", end='', flush=True)\n", " display.clear_output(wait=True)\n", " display.display(tensor_to_image(image))\n", " print(\"Train step: {}\".format(step))\n", @@ -939,7 +849,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "GWVB3anJMY2v" }, "source": [ @@ -950,53 +859,48 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7szUUybCQMB3" }, "outputs": [], "source": [ "def high_pass_x_y(image):\n", - " x_var = image[:,:,1:,:] - image[:,:,:-1,:]\n", - " y_var = image[:,1:,:,:] - image[:,:-1,:,:]\n", + " x_var = image[:, :, 1:, :] - image[:, :, :-1, :]\n", + " y_var = image[:, 1:, :, :] - image[:, :-1, :, :]\n", "\n", " return x_var, y_var" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Atc2oL29PXu_" }, "outputs": [], "source": [ "x_deltas, y_deltas = high_pass_x_y(content_image)\n", "\n", - "plt.figure(figsize=(14,10))\n", - "plt.subplot(2,2,1)\n", + "plt.figure(figsize=(14, 10))\n", + "plt.subplot(2, 2, 1)\n", "imshow(clip_0_1(2*y_deltas+0.5), \"Horizontal Deltas: Original\")\n", "\n", - "plt.subplot(2,2,2)\n", + "plt.subplot(2, 2, 2)\n", "imshow(clip_0_1(2*x_deltas+0.5), \"Vertical Deltas: Original\")\n", "\n", "x_deltas, y_deltas = high_pass_x_y(image)\n", "\n", - "plt.subplot(2,2,3)\n", + "plt.subplot(2, 2, 3)\n", "imshow(clip_0_1(2*y_deltas+0.5), \"Horizontal Deltas: Styled\")\n", "\n", - "plt.subplot(2,2,4)\n", + "plt.subplot(2, 2, 4)\n", "imshow(clip_0_1(2*x_deltas+0.5), \"Vertical Deltas: Styled\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lqHElVgBkgkz" }, "source": [ @@ -1007,27 +911,24 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "HyvqCiywiUfL" }, "outputs": [], "source": [ - "plt.figure(figsize=(14,10))\n", + "plt.figure(figsize=(14, 10))\n", "\n", "sobel = tf.image.sobel_edges(content_image)\n", - "plt.subplot(1,2,1)\n", - "imshow(clip_0_1(sobel[...,0]/4+0.5), \"Horizontal Sobel-edges\")\n", - "plt.subplot(1,2,2)\n", - "imshow(clip_0_1(sobel[...,1]/4+0.5), \"Vertical Sobel-edges\")" + "plt.subplot(1, 2, 1)\n", + "imshow(clip_0_1(sobel[..., 0]/4+0.5), \"Horizontal Sobel-edges\")\n", + "plt.subplot(1, 2, 2)\n", + "imshow(clip_0_1(sobel[..., 1]/4+0.5), \"Vertical Sobel-edges\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vv5bKlSDnPP7" }, "source": [ @@ -1036,10 +937,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "mP-92lXMIYPn" }, "outputs": [], @@ -1051,10 +950,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "s4OYBUX2KQ25" }, "outputs": [], @@ -1065,7 +962,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pu2hJ8zOKMc1" }, "source": [ @@ -1074,10 +970,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YQjWW04NKLfJ" }, "outputs": [], @@ -1088,7 +982,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "nTessd-DCdcC" }, "source": [ @@ -1099,10 +992,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "tGeRLD4GoAd4" }, "outputs": [], @@ -1113,7 +1004,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kG1-T4kJsoAv" }, "source": [ @@ -1122,10 +1012,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "BzmfcyyYUyWq" }, "outputs": [], @@ -1145,30 +1033,27 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lcLWBQChsutQ" }, "source": [ - "Reinitialize the optimization variable:" + "Reinitialize the image-variable and the optimizer:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "a-dPRr8BqexB" }, "outputs": [], "source": [ + "opt = tf.keras.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)\n", "image = tf.Variable(content_image)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "BEflRstmtGBu" }, "source": [ @@ -1177,10 +1062,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "q3Cc3bLtoOWy" }, "outputs": [], @@ -1196,7 +1079,7 @@ " for m in range(steps_per_epoch):\n", " step += 1\n", " train_step(image)\n", - " print(\".\", end='')\n", + " print(\".\", end='', flush=True)\n", " display.clear_output(wait=True)\n", " display.display(tensor_to_image(image))\n", " print(\"Train step: {}\".format(step))\n", @@ -1208,7 +1091,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "KKox7K46tKxy" }, "source": [ @@ -1217,10 +1099,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "SSH6OpyyQn7w" }, "outputs": [], @@ -1230,10 +1110,20 @@ "\n", "try:\n", " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download(file_name)" + " files.download(file_name)\n", + "except (ImportError, AttributeError):\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tNlwRXagxQZk" + }, + "source": [ + "## Learn more\n", + "\n", + "This tutorial demonstrates the original style-transfer algorithm. For a simple application of style transfer check out this [tutorial](https://www.tensorflow.org/hub/tutorials/tf2_arbitrary_image_stylization) to learn more about how to use the arbitrary image style transfer model from [TensorFlow Hub](https://tfhub.dev)." ] } ], @@ -1241,23 +1131,8 @@ "accelerator": "GPU", "colab": { "collapsed_sections": [], - "last_runtime": { - "build_target": "", - "kind": "local" - }, "name": "style_transfer.ipynb", - "private_outputs": true, - "provenance": [ - { - "file_id": "1ny1S8yiStnIk7WpVKw0NWcM2I2V4yEkV", - "timestamp": 1570221686081 - }, - { - "file_id": "https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/style_transfer.ipynb", - "timestamp": 1570217494261 - } - ], - "toc_visible": true + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/images/classification.ipynb b/site/en/tutorials/images/classification.ipynb index 755dee9a6c2..f006b81d756 100644 --- a/site/en/tutorials/images/classification.ipynb +++ b/site/en/tutorials/images/classification.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TBFXQGKYUc4X" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "1z4xy2gTUc4a" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FE7KNzPPVrVV" }, "source": [ @@ -47,38 +43,36 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "KwQtSOz0VrVX" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/images/classification\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/images/classification.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/images/classification.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/images/classification.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gN7G9GFmVrVY" }, "source": [ - "This tutorial shows how to classify cats or dogs from images. It builds an image classifier using a `tf.keras.Sequential` model and load data using `tf.keras.preprocessing.image.ImageDataGenerator`. You will get some practical experience and develop intuition for the following concepts:\n", + "This tutorial shows how to classify images of flowers using a `tf.keras.Sequential` model and load data using `tf.keras.utils.image_dataset_from_directory`. It demonstrates the following concepts:\n", "\n", - "* Building _data input pipelines_ using the `tf.keras.preprocessing.image.ImageDataGenerator` class to efficiently work with data on disk to use with the model.\n", - "* _Overfitting_ —How to identify and prevent it.\n", - "* _Data augmentation_ and _dropout_ —Key techniques to fight overfitting in computer vision tasks to incorporate into the data pipeline and image classifier model.\n", + "\n", + "* Efficiently loading a dataset off disk.\n", + "* Identifying overfitting and applying techniques to mitigate it, including data augmentation and dropout.\n", "\n", "This tutorial follows a basic machine learning workflow:\n", "\n", @@ -87,522 +81,517 @@ "3. Build the model\n", "4. Train the model\n", "5. Test the model\n", - "6. Improve the model and repeat the process" + "6. Improve the model and repeat the process\n", + "\n", + "In addition, the notebook demonstrates how to convert a [saved model](../../../guide/saved_model.ipynb) to a [TensorFlow Lite](https://www.tensorflow.org/lite/) model for on-device machine learning on mobile, embedded, and IoT devices." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "zF9uvbXNVrVY" }, "source": [ - "## Import packages" + "## Setup\n", + "\n", + "Import TensorFlow and other necessary libraries:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "VddxeYBEVrVZ" + "id": "L1WtoaOHVrVh" }, + "outputs": [], "source": [ - "Let's start by importing the required packages. The `os` package is used to read files and directory structure, NumPy is used to convert python list to numpy array and to perform required matrix operations and `matplotlib.pyplot` to plot the graph and display images in the training and validation data." + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import PIL\n", + "import tensorflow as tf\n", + "\n", + "from tensorflow import keras\n", + "from tensorflow.keras import layers\n", + "from tensorflow.keras.models import Sequential" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "rtPGh2MAVrVa" + "id": "UZZI6lNkVrVm" }, - "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" + "## Download and explore the dataset" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Jlchl4x2VrVg" + "id": "DPHx8-t-VrVo" }, "source": [ - "Import Tensorflow and the Keras classes needed to construct our model." + "This tutorial uses a dataset of about 3,700 photos of flowers. The dataset contains five sub-directories, one per class:\n", + "\n", + "```\n", + "flower_photo/\n", + " daisy/\n", + " dandelion/\n", + " roses/\n", + " sunflowers/\n", + " tulips/\n", + "```" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "E82grprdYPI0" + "id": "57CcilYSG0zv" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" + "import pathlib\n", + "\n", + "dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + "data_dir = tf.keras.utils.get_file(\"flower_photos.tgz\", origin=dataset_url, extract=True)\n", + "data_dir = pathlib.Path(data_dir).parent / \"flower_photos_extracted\" / \"flower_photos\"" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "L1WtoaOHVrVh" + "id": "VpmywIlsVrVx" }, - "outputs": [], "source": [ - "from tensorflow.keras.models import Sequential\n", - "from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPooling2D\n", - "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", - "\n", - "import os\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt" + "After downloading, you should now have a copy of the dataset available. There are 3,670 total images:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "UZZI6lNkVrVm" + "id": "SbtTDYhOHZb6" }, + "outputs": [], "source": [ - "## Load data" + "image_count = len(list(data_dir.glob('*/*.jpg')))\n", + "print(image_count)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "DPHx8-t-VrVo" + "id": "PVmwkOSdHZ5A" }, "source": [ - "Begin by downloading the dataset. This tutorial uses a filtered version of \u003ca href=\"https://www.kaggle.com/c/dogs-vs-cats/data\" target=\"_blank\"\u003eDogs vs Cats\u003c/a\u003e dataset from Kaggle. Download the archive version of the dataset and store it in the \"/tmp/\" directory." + "Here are some roses:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "C1nqr-CYY6uw" + "id": "N1loMlbYHeiJ" }, "outputs": [], "source": [ - "_URL = 'https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip'\n", - "\n", - "path_to_zip = tf.keras.utils.get_file('cats_and_dogs.zip', origin=_URL, extract=True)\n", - "\n", - "PATH = os.path.join(os.path.dirname(path_to_zip), 'cats_and_dogs_filtered')" + "roses = list(data_dir.glob('roses/*'))\n", + "PIL.Image.open(str(roses[0]))" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "Giv0wMQzVrVw" + "id": "RQbZBOTLHiUP" }, + "outputs": [], "source": [ - "The dataset has the following directory structure:\n", - "\n", - "\u003cpre\u003e\n", - "\u003cb\u003ecats_and_dogs_filtered\u003c/b\u003e\n", - "|__ \u003cb\u003etrain\u003c/b\u003e\n", - " |______ \u003cb\u003ecats\u003c/b\u003e: [cat.0.jpg, cat.1.jpg, cat.2.jpg ....]\n", - " |______ \u003cb\u003edogs\u003c/b\u003e: [dog.0.jpg, dog.1.jpg, dog.2.jpg ...]\n", - "|__ \u003cb\u003evalidation\u003c/b\u003e\n", - " |______ \u003cb\u003ecats\u003c/b\u003e: [cat.2000.jpg, cat.2001.jpg, cat.2002.jpg ....]\n", - " |______ \u003cb\u003edogs\u003c/b\u003e: [dog.2000.jpg, dog.2001.jpg, dog.2002.jpg ...]\n", - "\u003c/pre\u003e" + "PIL.Image.open(str(roses[1]))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "VpmywIlsVrVx" + "id": "DGEqiBbRHnyI" }, "source": [ - "After extracting its contents, assign variables with the proper file path for the training and validation set." + "And some tulips:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sRucI3QqVrVy" + "id": "HyQkfPGdHilw" }, "outputs": [], "source": [ - "train_dir = os.path.join(PATH, 'train')\n", - "validation_dir = os.path.join(PATH, 'validation')" + "tulips = list(data_dir.glob('tulips/*'))\n", + "PIL.Image.open(str(tulips[0]))" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Utv3nryxVrV0" + "id": "wtlhWJPAHivf" }, "outputs": [], "source": [ - "train_cats_dir = os.path.join(train_dir, 'cats') # directory with our training cat pictures\n", - "train_dogs_dir = os.path.join(train_dir, 'dogs') # directory with our training dog pictures\n", - "validation_cats_dir = os.path.join(validation_dir, 'cats') # directory with our validation cat pictures\n", - "validation_dogs_dir = os.path.join(validation_dir, 'dogs') # directory with our validation dog pictures" + "PIL.Image.open(str(tulips[1]))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "ZdrHHTy2VrV3" + "id": "gIjgz7_JIo_m" }, "source": [ - "### Understand the data" + "## Load data using a Keras utility\n", + "\n", + "Next, load these images off disk using the helpful `tf.keras.utils.image_dataset_from_directory` utility. This will take you from a directory of images on disk to a `tf.data.Dataset` in just a couple lines of code. If you like, you can also write your own data loading code from scratch by visiting the [Load and preprocess images](../load_data/images.ipynb) tutorial." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "LblUYjl-VrV3" + "id": "xyDNn9MbIzfT" }, "source": [ - "Let's look at how many cats and dogs images are in the training and validation directory:" + "### Create a dataset" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vc4u8e9hVrV4" + "id": "anqiK_AGI086" }, - "outputs": [], "source": [ - "num_cats_tr = len(os.listdir(train_cats_dir))\n", - "num_dogs_tr = len(os.listdir(train_dogs_dir))\n", - "\n", - "num_cats_val = len(os.listdir(validation_cats_dir))\n", - "num_dogs_val = len(os.listdir(validation_dogs_dir))\n", - "\n", - "total_train = num_cats_tr + num_dogs_tr\n", - "total_val = num_cats_val + num_dogs_val" + "Define some parameters for the loader:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "g4GGzGt0VrV7" + "id": "H74l2DoDI2XD" }, "outputs": [], "source": [ - "print('total training cat images:', num_cats_tr)\n", - "print('total training dog images:', num_dogs_tr)\n", - "\n", - "print('total validation cat images:', num_cats_val)\n", - "print('total validation dog images:', num_dogs_val)\n", - "print(\"--\")\n", - "print(\"Total training images:\", total_train)\n", - "print(\"Total validation images:\", total_val)" + "batch_size = 32\n", + "img_height = 180\n", + "img_width = 180" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "8Lp-0ejxOtP1" + "id": "pFBhRrrEI49z" }, "source": [ - "For convenience, set up variables to use while pre-processing the dataset and training the network." + "It's good practice to use a validation split when developing your model. Use 80% of the images for training and 20% for validation." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3NqNselLVrWA" + "id": "fIR0kRZiI_AT" }, "outputs": [], "source": [ - "batch_size = 128\n", - "epochs = 15\n", - "IMG_HEIGHT = 150\n", - "IMG_WIDTH = 150" + "train_ds = tf.keras.utils.image_dataset_from_directory(\n", + " data_dir,\n", + " validation_split=0.2,\n", + " subset=\"training\",\n", + " seed=123,\n", + " image_size=(img_height, img_width),\n", + " batch_size=batch_size)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "INn-cOn1VrWC" + "id": "iscU3UoVJBXj" }, + "outputs": [], "source": [ - "## Data preparation" + "val_ds = tf.keras.utils.image_dataset_from_directory(\n", + " data_dir,\n", + " validation_split=0.2,\n", + " subset=\"validation\",\n", + " seed=123,\n", + " image_size=(img_height, img_width),\n", + " batch_size=batch_size)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "5Jfk6aSAVrWD" + "id": "WLQULyAvJC3X" }, "source": [ - "Format the images into appropriately pre-processed floating point tensors before feeding to the network:\n", - "\n", - "1. Read images from the disk.\n", - "2. Decode contents of these images and convert it into proper grid format as per their RGB content.\n", - "3. Convert them into floating point tensors.\n", - "4. Rescale the tensors from values between 0 and 255 to values between 0 and 1, as neural networks prefer to deal with small input values.\n", - "\n", - "Fortunately, all these tasks can be done with the `ImageDataGenerator` class provided by `tf.keras`. It can read images from disk and preprocess them into proper tensors. It will also set up generators that convert these images into batches of tensors—helpful when training the network." + "You can find the class names in the `class_names` attribute on these datasets. These correspond to the directory names in alphabetical order." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "syDdF_LWVrWE" + "id": "ZHAxkHX5JD3k" }, "outputs": [], "source": [ - "train_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our training data\n", - "validation_image_generator = ImageDataGenerator(rescale=1./255) # Generator for our validation data" + "class_names = train_ds.class_names\n", + "print(class_names)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "RLciCR_FVrWH" + "id": "_uoVvxSLJW9m" }, "source": [ - "After defining the generators for training and validation images, the `flow_from_directory` method load images from the disk, applies rescaling, and resizes the images into the required dimensions." + "## Visualize the data\n", + "\n", + "Here are the first nine images from the training dataset:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Pw94ajOOVrWI" + "id": "wBmEA9c0JYes" }, "outputs": [], "source": [ - "train_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" + "import matplotlib.pyplot as plt\n", + "\n", + "plt.figure(figsize=(10, 10))\n", + "for images, labels in train_ds.take(1):\n", + " for i in range(9):\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " plt.imshow(images[i].numpy().astype(\"uint8\"))\n", + " plt.title(class_names[labels[i]])\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5M6BXtXFJdW0" + }, + "source": [ + "You will pass these datasets to the Keras `Model.fit` method for training later in this tutorial. If you like, you can also manually iterate over the dataset and retrieve batches of images:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2oUoKUzRVrWM" + "id": "2-MfMoenJi8s" }, "outputs": [], "source": [ - "val_data_gen = validation_image_generator.flow_from_directory(batch_size=batch_size,\n", - " directory=validation_dir,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" + "for image_batch, labels_batch in train_ds:\n", + " print(image_batch.shape)\n", + " print(labels_batch.shape)\n", + " break" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "hyexPJ8CVrWP" + "id": "Wj4FrKxxJkoW" }, "source": [ - "### Visualize training images" + "The `image_batch` is a tensor of the shape `(32, 180, 180, 3)`. This is a batch of 32 images of shape `180x180x3` (the last dimension refers to color channels RGB). The `label_batch` is a tensor of the shape `(32,)`, these are corresponding labels to the 32 images.\n", + "\n", + "You can call `.numpy()` on the `image_batch` and `labels_batch` tensors to convert them to a `numpy.ndarray`.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "60CnhEL4VrWQ" + "id": "4Dr0at41KcAU" }, "source": [ - "Visualize the training images by extracting a batch of images from the training generator—which is 32 images in this example—then plot five of them with `matplotlib`." + "## Configure the dataset for performance\n", + "\n", + "Make sure to use buffered prefetching, so you can yield data from disk without having I/O become blocking. These are two important methods you should use when loading data:\n", + "\n", + "- `Dataset.cache` keeps the images in memory after they're loaded off disk during the first epoch. This will ensure the dataset does not become a bottleneck while training your model. If your dataset is too large to fit into memory, you can also use this method to create a performant on-disk cache.\n", + "- `Dataset.prefetch` overlaps data preprocessing and model execution while training.\n", + "\n", + "Interested readers can learn more about both methods, as well as how to cache data to disk in the *Prefetching* section of the [Better performance with the tf.data API](../../guide/data_performance.ipynb) guide." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3f0Z7NZgVrWQ" + "id": "nOjJSm7DKoZA" }, "outputs": [], "source": [ - "sample_training_images, _ = next(train_data_gen)" + "AUTOTUNE = tf.data.AUTOTUNE\n", + "\n", + "train_ds = train_ds.cache().shuffle(1000).prefetch(buffer_size=AUTOTUNE)\n", + "val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "49weMt5YVrWT" + "id": "8GUnmPF4JvEf" }, "source": [ - "The `next` function returns a batch from the dataset. The return value of `next` function is in form of `(x_train, y_train)` where x_train is training features and y_train, its labels. Discard the labels to only visualize the training images." + "## Standardize the data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e56VXHMWJxYT" + }, + "source": [ + "The RGB channel values are in the `[0, 255]` range. This is not ideal for a neural network; in general you should seek to make your input values small.\n", + "\n", + "Here, you will standardize values to be in the `[0, 1]` range by using `tf.keras.layers.Rescaling`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JMt2RES_VrWU" + "id": "PEYxo2CTJvY9" }, "outputs": [], "source": [ - "# This function will plot images in the form of a grid with 1 row and 5 columns where images are placed in each column.\n", - "def plotImages(images_arr):\n", - " fig, axes = plt.subplots(1, 5, figsize=(20,20))\n", - " axes = axes.flatten()\n", - " for img, ax in zip( images_arr, axes):\n", - " ax.imshow(img)\n", - " ax.axis('off')\n", - " plt.tight_layout()\n", - " plt.show()" + "normalization_layer = layers.Rescaling(1./255)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bl4RmanbJ4g0" + }, + "source": [ + "There are two ways to use this layer. You can apply it to the dataset by calling `Dataset.map`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "d_VVg_gEVrWW" + "id": "X9o9ESaJJ502" }, "outputs": [], "source": [ - "plotImages(sample_training_images[:5])" + "normalized_ds = train_ds.map(lambda x, y: (normalization_layer(x), y))\n", + "image_batch, labels_batch = next(iter(normalized_ds))\n", + "first_image = image_batch[0]\n", + "# Notice the pixel values are now in `[0,1]`.\n", + "print(np.min(first_image), np.max(first_image))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "b5Ej-HLGVrWZ" + "id": "XWEOmRSBJ9J8" }, "source": [ - "## Create the model" + "Or, you can include the layer inside your model definition, which can simplify deployment. Use the second approach here." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "wEgW4i18VrWZ" + "id": "XsRk1xCwKZR4" }, "source": [ - "The model consists of three convolution blocks with a max pool layer in each of them. There's a fully connected layer with 512 units on top of it that is activated by a `relu` activation function. The model outputs class probabilities based on binary classification by the `sigmoid` activation function." + "Note: You previously resized images using the `image_size` argument of `tf.keras.utils.image_dataset_from_directory`. If you want to include the resizing logic in your model as well, you can use the `tf.keras.layers.Resizing` layer." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WcUTyDOPKucd" + }, + "source": [ + "## A basic Keras model\n", + "\n", + "### Create the model\n", + "\n", + "The Keras [Sequential](https://www.tensorflow.org/guide/keras/sequential_model) model consists of three convolution blocks (`tf.keras.layers.Conv2D`) with a max pooling layer (`tf.keras.layers.MaxPooling2D`) in each of them. There's a fully-connected layer (`tf.keras.layers.Dense`) with 128 units on top of it that is activated by a ReLU activation function (`'relu'`). This model has not been tuned for high accuracy; the goal of this tutorial is to show a standard approach." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "F15-uwLPVrWa" + "id": "QR6argA1K074" }, "outputs": [], "source": [ + "num_classes = len(class_names)\n", + "\n", "model = Sequential([\n", - " Conv2D(16, 3, padding='same', activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3)),\n", - " MaxPooling2D(),\n", - " Conv2D(32, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Conv2D(64, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Flatten(),\n", - " Dense(512, activation='relu'),\n", - " Dense(1, activation='sigmoid')\n", + " layers.Rescaling(1./255, input_shape=(img_height, img_width, 3)),\n", + " layers.Conv2D(16, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Conv2D(32, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Conv2D(64, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Flatten(),\n", + " layers.Dense(128, activation='relu'),\n", + " layers.Dense(num_classes)\n", "])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "PI5cdkMQVrWc" + "id": "EaKFzz72Lqpg" }, "source": [ "### Compile the model\n", "\n", - "For this tutorial, choose the *ADAM* optimizer and *binary cross entropy* loss function. To view training and validation accuracy for each training epoch, pass the `metrics` argument." + "For this tutorial, choose the `tf.keras.optimizers.Adam` optimizer and `tf.keras.losses.SparseCategoricalCrossentropy` loss function. To view training and validation accuracy for each training epoch, pass the `metrics` argument to `Model.compile`." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6Mg7_TXOVrWd" + "id": "jloGNS1MLx3A" }, "outputs": [], "source": [ "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "2YmQZ3TAVrWg" + "id": "aMJ4DnuJL55A" }, "source": [ "### Model summary\n", "\n", - "View all the layers of the network using the model's `summary` method:" + "View all the layers of the network using the Keras `Model.summary` method:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Vtny8hmBVrWh" + "id": "llLYH-BXL7Xe" }, "outputs": [], "source": [ @@ -612,8 +601,7 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "N06iqE8VVrWj" + "id": "NiYHcbvaL9H-" }, "source": [ "### Train the model" @@ -622,59 +610,51 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "oub9RtoFVrWk" + "id": "j30F69T4sIVN" }, "source": [ - "Use the `fit_generator` method of the `ImageDataGenerator` class to train the network." + "Train the model for 10 epochs with the Keras `Model.fit` method:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KSF2HqhDVrWk" + "id": "5fWToCqYMErH" }, "outputs": [], "source": [ - "history = model.fit_generator(\n", - " train_data_gen,\n", - " steps_per_epoch=total_train // batch_size,\n", - " epochs=epochs,\n", - " validation_data=val_data_gen,\n", - " validation_steps=total_val // batch_size\n", + "epochs=10\n", + "history = model.fit(\n", + " train_ds,\n", + " validation_data=val_ds,\n", + " epochs=epochs\n", ")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "ojJNteAGVrWo" + "id": "SyFKdQpXMJT4" }, "source": [ - "### Visualize training results" + "## Visualize training results" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "LZPYT-EmVrWo" + "id": "dFvOvmAmMK9w" }, "source": [ - "Now visualize the results after training the network." + "Create plots of the loss and accuracy on the training and validation sets:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "K6oA77ADVrWp" + "id": "jWnopEChMMCn" }, "outputs": [], "source": [ @@ -704,20 +684,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "kDnr50l2VrWu" + "id": "hO_jT7HwMrEn" }, "source": [ - "As you can see from the plots, training accuracy and validation accuracy are off by large margin and the model has achieved only around **70%** accuracy on the validation set.\n", + "The plots show that training accuracy and validation accuracy are off by large margins, and the model has achieved only around 60% accuracy on the validation set.\n", "\n", - "Let's look at what went wrong and try to increase overall performance of the model." + "The following tutorial sections show how to inspect what went wrong and try to increase the overall performance of the model." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "rLO7yhLlVrWu" + "id": "hqtyGodAMvNV" }, "source": [ "## Overfitting" @@ -726,22 +704,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "hNyx3Lp4VrWv" + "id": "ixsz9XFfMxcu" }, "source": [ - "In the plots above, the training accuracy is increasing linearly over time, whereas validation accuracy stalls around 70% in the training process. Also, the difference in accuracy between training and validation accuracy is noticeable—a sign of *overfitting*.\n", + "In the plots above, the training accuracy is increasing linearly over time, whereas validation accuracy stalls around 60% in the training process. Also, the difference in accuracy between training and validation accuracy is noticeable—a sign of [overfitting](https://www.tensorflow.org/tutorials/keras/overfit_and_underfit).\n", "\n", "When there are a small number of training examples, the model sometimes learns from noises or unwanted details from training examples—to an extent that it negatively impacts the performance of the model on new examples. This phenomenon is known as overfitting. It means that the model will have a difficult time generalizing on a new dataset.\n", "\n", - "There are multiple ways to fight overfitting in the training process. In this tutorial, you'll use *data augmentation* and add *dropout* to our model." + "There are multiple ways to fight overfitting in the training process. In this tutorial, you'll use *data augmentation* and add *dropout* to your model." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "UOoVpxFwVrWy" + "id": "BDMfYqwmM1C-" }, "source": [ "## Data augmentation" @@ -750,580 +726,449 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Wn_QLciWVrWy" + "id": "GxYwix81M2YO" }, "source": [ - "Overfitting generally occurs when there are a small number of training examples. One way to fix this problem is to augment the dataset so that it has a sufficient number of training examples. Data augmentation takes the approach of generating more training data from existing training samples by augmenting the samples using random transformations that yield believable-looking images. The goal is the model will never see the exact same picture twice during training. This helps expose the model to more aspects of the data and generalize better.\n", + "Overfitting generally occurs when there are a small number of training examples. [Data augmentation](./data_augmentation.ipynb) takes the approach of generating additional training data from your existing examples by augmenting them using random transformations that yield believable-looking images. This helps expose the model to more aspects of the data and generalize better.\n", "\n", - "Implement this in `tf.keras` using the `ImageDataGenerator` class. Pass different transformations to the dataset and it will take care of applying it during the training process." + "You will implement data augmentation using the following Keras preprocessing layers: `tf.keras.layers.RandomFlip`, `tf.keras.layers.RandomRotation`, and `tf.keras.layers.RandomZoom`. These can be included inside your model like other layers, and run on the GPU." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "2uJ1G030VrWz" + "id": "9J80BAbIMs21" }, + "outputs": [], "source": [ - "### Augment and visualize data" + "data_augmentation = keras.Sequential(\n", + " [\n", + " layers.RandomFlip(\"horizontal\",\n", + " input_shape=(img_height,\n", + " img_width,\n", + " 3)),\n", + " layers.RandomRotation(0.1),\n", + " layers.RandomZoom(0.1),\n", + " ]\n", + ")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "hvX7hHlgVrW0" + "id": "PN4k1dK3S6eV" }, "source": [ - "Begin by applying random horizontal flip augmentation to the dataset and see how individual images look like after the transformation." + "Visualize a few augmented examples by applying data augmentation to the same image several times:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "rlVj6VqaVrW0" + "id": "7Z90k539S838" }, + "outputs": [], "source": [ - "### Apply horizontal flip" + "plt.figure(figsize=(10, 10))\n", + "for images, _ in train_ds.take(1):\n", + " for i in range(9):\n", + " augmented_images = data_augmentation(images)\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " plt.imshow(augmented_images[0].numpy().astype(\"uint8\"))\n", + " plt.axis(\"off\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "xcdvx4TVVrW1" + "id": "tsjXCBLYYNs5" }, "source": [ - "Pass `horizontal_flip` as an argument to the `ImageDataGenerator` class and set it to `True` to apply this augmentation." + "You will add data augmentation to your model before training in the next step." ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Bi1_vHyBVrW2" + "id": "ZeD3bXepYKXs" }, - "outputs": [], "source": [ - "image_gen = ImageDataGenerator(rescale=1./255, horizontal_flip=True)" + "## Dropout\n", + "\n", + "Another technique to reduce overfitting is to introduce [dropout](https://developers.google.com/machine-learning/glossary#dropout_regularization) regularization to the network.\n", + "\n", + "When you apply dropout to a layer, it randomly drops out (by setting the activation to zero) a number of output units from the layer during the training process. Dropout takes a fractional number as its input value, in the form such as 0.1, 0.2, 0.4, etc. This means dropping out 10%, 20% or 40% of the output units randomly from the applied layer.\n", + "\n", + "Create a new neural network with `tf.keras.layers.Dropout` before training it using the augmented images:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zvwqmefgVrW3" + "id": "2Zeg8zsqXCsm" }, "outputs": [], "source": [ - "train_data_gen = image_gen.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH))" + "model = Sequential([\n", + " data_augmentation,\n", + " layers.Rescaling(1./255),\n", + " layers.Conv2D(16, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Conv2D(32, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Conv2D(64, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Dropout(0.2),\n", + " layers.Flatten(),\n", + " layers.Dense(128, activation='relu'),\n", + " layers.Dense(num_classes, name=\"outputs\")\n", + "])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "zJpRSxJ-VrW7" + "id": "L4nEcuqgZLbi" }, "source": [ - "Take one sample image from the training examples and repeat it five times so that the augmentation is applied to the same image five times." + "## Compile and train the model" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RrKGd_jjVrW7" + "id": "EvyAINs9ZOmJ" }, "outputs": [], "source": [ - "augmented_images = [train_data_gen[0][0][0] for i in range(5)]" + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EvBZoQ9xVrW9" + "id": "wWLkKoKjZSoC" }, "outputs": [], "source": [ - "# Re-use the same custom plotting function defined and used\n", - "# above to visualize the training images\n", - "plotImages(augmented_images)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "i7n9xcqCVrXB" - }, - "source": [ - "### Randomly rotate the image" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qXnwkzFuVrXB" - }, - "source": [ - "Let's take a look at a different augmentation called rotation and apply 45 degrees of rotation randomly to the training examples." + "model.summary()" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1zip35pDVrXB" + "id": "LWS-vvNaZDag" }, "outputs": [], "source": [ - "image_gen = ImageDataGenerator(rescale=1./255, rotation_range=45)" + "epochs = 15\n", + "history = model.fit(\n", + " train_ds,\n", + " validation_data=val_ds,\n", + " epochs=epochs\n", + ")" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kVoWh4OIVrXD" + "id": "Lkdl8VsBbZOu" }, - "outputs": [], "source": [ - "train_data_gen = image_gen.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH))\n", + "## Visualize training results\n", "\n", - "augmented_images = [train_data_gen[0][0][0] for i in range(5)]" + "After applying data augmentation and `tf.keras.layers.Dropout`, there is less overfitting than before, and training and validation accuracy are closer aligned:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wmBx8NhrVrXK" + "id": "dduoLfKsZVIA" }, "outputs": [], "source": [ - "plotImages(augmented_images)" + "acc = history.history['accuracy']\n", + "val_acc = history.history['val_accuracy']\n", + "\n", + "loss = history.history['loss']\n", + "val_loss = history.history['val_loss']\n", + "\n", + "epochs_range = range(epochs)\n", + "\n", + "plt.figure(figsize=(8, 8))\n", + "plt.subplot(1, 2, 1)\n", + "plt.plot(epochs_range, acc, label='Training Accuracy')\n", + "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", + "plt.legend(loc='lower right')\n", + "plt.title('Training and Validation Accuracy')\n", + "\n", + "plt.subplot(1, 2, 2)\n", + "plt.plot(epochs_range, loss, label='Training Loss')\n", + "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", + "plt.legend(loc='upper right')\n", + "plt.title('Training and Validation Loss')\n", + "plt.show()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "FOqGPL76VrXM" + "id": "dtv5VbaVb-3W" }, "source": [ - "### Apply zoom augmentation" + "## Predict on new data" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "NvqXaD8BVrXN" + "id": "10buWpJbcCQz" }, "source": [ - "Apply a zoom augmentation to the dataset to zoom images up to 50% randomly." + "Use your model to classify an image that wasn't included in the training or validation sets." ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tGNKLa_YVrXR" + "id": "NKgMZ4bDcHf7" }, - "outputs": [], "source": [ - "# zoom_range from 0 - 1 where 1 = 100%.\n", - "image_gen = ImageDataGenerator(rescale=1./255, zoom_range=0.5) # " + "Note: Data augmentation and dropout layers are inactive at inference time." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VOvTs32FVrXU" + "id": "dC40sRITBSsQ" }, "outputs": [], "source": [ - "train_data_gen = image_gen.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH))\n", + "sunflower_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/592px-Red_sunflower.jpg\"\n", + "sunflower_path = tf.keras.utils.get_file('Red_sunflower', origin=sunflower_url)\n", "\n", - "augmented_images = [train_data_gen[0][0][0] for i in range(5)]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-KQWw8IZVrXZ" - }, - "outputs": [], - "source": [ - "plotImages(augmented_images)" + "img = tf.keras.utils.load_img(\n", + " sunflower_path, target_size=(img_height, img_width)\n", + ")\n", + "img_array = tf.keras.utils.img_to_array(img)\n", + "img_array = tf.expand_dims(img_array, 0) # Create a batch\n", + "\n", + "predictions = model.predict(img_array)\n", + "score = tf.nn.softmax(predictions[0])\n", + "\n", + "print(\n", + " \"This image most likely belongs to {} with a {:.2f} percent confidence.\"\n", + " .format(class_names[np.argmax(score)], 100 * np.max(score))\n", + ")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "usS13KCNVrXd" + "id": "aOc3PZ2N2r18" }, "source": [ - "### Put it all together" + "## Use TensorFlow Lite\n", + "\n", + "TensorFlow Lite is a set of tools that enables on-device machine learning by helping developers run their models on mobile, embedded, and edge devices." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "OC8fIsalVrXd" + "id": "cThu25rh4LPP" }, "source": [ - "Apply all the previous augmentations. Here, you applied rescale, 45 degree rotation, width shift, height shift, horizontal flip and zoom augmentation to the training images." + "### Convert the Keras Sequential model to a TensorFlow Lite model\n", + "\n", + "To use the trained model with on-device applications, first [convert it](https://www.tensorflow.org/lite/models/convert) to a smaller and more efficient model format called a [TensorFlow Lite](https://www.tensorflow.org/lite/) model.\n", + "\n", + "In this example, take the trained Keras Sequential model and use `tf.lite.TFLiteConverter.from_keras_model` to generate a [TensorFlow Lite](https://www.tensorflow.org/lite/) model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gnr2xujaVrXe" + "id": "mXo6ftuL2ufx" }, "outputs": [], "source": [ - "image_gen_train = ImageDataGenerator(\n", - " rescale=1./255,\n", - " rotation_range=45,\n", - " width_shift_range=.15,\n", - " height_shift_range=.15,\n", - " horizontal_flip=True,\n", - " zoom_range=0.5\n", - " )" + "# Convert the model.\n", + "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n", + "tflite_model = converter.convert()\n", + "\n", + "# Save the model.\n", + "with open('model.tflite', 'wb') as f:\n", + " f.write(tflite_model)" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "K0Efxy7EVrXh" + "id": "4R26OU4gGKhh" }, - "outputs": [], "source": [ - "train_data_gen = image_gen_train.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" + "The TensorFlow Lite model you saved in the previous step can contain several function signatures. The Keras model converter API uses the default signature automatically. Learn more about [TensorFlow Lite signatures](https://www.tensorflow.org/lite/guide/signatures)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "AW-pV5awVrXl" + "id": "7fjQfXaV2l-5" }, "source": [ - "Visualize how a single image would look five different times when passing these augmentations randomly to the dataset." + "### Run the TensorFlow Lite model\n", + "\n", + "You can access the TensorFlow Lite saved model signatures in Python via the `tf.lite.Interpreter` class.\n", + "\n", + "Load the model with the `Interpreter`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "z2m68eMhVrXm" + "id": "cHYcip_FOaHq" }, "outputs": [], "source": [ - "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", - "plotImages(augmented_images)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J8cUd7FXVrXq" - }, - "source": [ - "### Create validation data generator" + "TF_MODEL_FILE_PATH = 'model.tflite' # The default path to the saved TensorFlow Lite model\n", + "\n", + "interpreter = tf.lite.Interpreter(model_path=TF_MODEL_FILE_PATH)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "a99fDBt7VrXr" + "id": "nPUXY6BdHDHo" }, "source": [ - "Generally, only apply data augmentation to the training examples. In this case, only rescale the validation images and convert them into batches using `ImageDataGenerator`." + "Print the signatures from the converted model to obtain the names of the inputs (and outputs):\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "54x0aNbKVrXr" + "id": "ZdDl00E2OaHq" }, "outputs": [], "source": [ - "image_gen_val = ImageDataGenerator(rescale=1./255)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1PCHKzI8VrXv" - }, - "outputs": [], - "source": [ - "val_data_gen = image_gen_val.flow_from_directory(batch_size=batch_size,\n", - " directory=validation_dir,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" + "interpreter.get_signature_list()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "yQGhdqHFVrXx" + "id": "4eVFqT0je3YG" }, "source": [ - "## Dropout" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2Iq5TAH_VrXx" - }, - "source": [ - "Another technique to reduce overfitting is to introduce *dropout* to the network. It is a form of *regularization* that forces the weights in the network to take only small values, which makes the distribution of weight values more regular and the network can reduce overfitting on small training examples. Dropout is one of the regularization technique used in this tutorial\n", - "\n", - "When you apply dropout to a layer it randomly drops out (set to zero) number of output units from the applied layer during the training process. Dropout takes a fractional number as its input value, in the form such as 0.1, 0.2, 0.4, etc. This means dropping out 10%, 20% or 40% of the output units randomly from the applied layer.\n", + "In this example, you have one default signature called `serving_default`. In addition, the name of the `'inputs'` is `'sequential_1_input'`, while the `'outputs'` are called `'outputs'`. You can look up these first and last Keras layer names when running `Model.summary`, as demonstrated earlier in this tutorial.\n", "\n", - "When appling 0.1 dropout to a certain layer, it randomly kills 10% of the output units in each training epoch.\n", - "\n", - "Create a network architecture with this new dropout feature and apply it to different convolutions and fully-connected layers." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DyxxXRmVVrXy" - }, - "source": [ - "## Creating a new network with Dropouts" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1Ba2LjtkVrXy" - }, - "source": [ - "Here, you apply dropout to first and last max pool layers. Applying dropout will randomly set 20% of the neurons to zero during each training epoch. This helps to avoid overfitting on the training dataset." + "Now you can test the loaded TensorFlow Model by performing inference on a sample image with `tf.lite.Interpreter.get_signature_runner` by passing the signature name as follows:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2fjio8EsVrXz" + "id": "yFoT_7W_OaHq" }, "outputs": [], "source": [ - "model_new = Sequential([\n", - " Conv2D(16, 3, padding='same', activation='relu', \n", - " input_shape=(IMG_HEIGHT, IMG_WIDTH ,3)),\n", - " MaxPooling2D(),\n", - " Dropout(0.2),\n", - " Conv2D(32, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Conv2D(64, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Dropout(0.2),\n", - " Flatten(),\n", - " Dense(512, activation='relu'),\n", - " Dense(1, activation='sigmoid')\n", - "])" + "classify_lite = interpreter.get_signature_runner('serving_default')\n", + "print(classify_lite.get_input_details())\n", + "print(classify_lite.get_output_details())" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "tpTgIxWAVrX0" + "id": "b1mfRcBOnEx0" }, "source": [ - "### Compile the model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1osvc_iTVrX1" - }, - "source": [ - "After introducing dropouts to the network, compile the model and view the layers summary." + "Similar to what you did earlier in the tutorial, you can use the TensorFlow Lite model to classify images that weren't included in the training or validation sets.\n", + "\n", + "You have already tensorized that image and saved it as `img_array`. Now, pass it to the first argument (the name of the `'inputs'`) of the loaded TensorFlow Lite model (`predictions_lite`), compute softmax activations, and then print the prediction for the class with the highest computed probability." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OkIJhS-WVrX1" + "id": "sEqR27YcnFvc" }, "outputs": [], "source": [ - "model_new.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "model_new.summary()" + "predictions_lite = classify_lite(keras_tensor_15=img_array)['output_0']\n", + "score_lite = tf.nn.softmax(predictions_lite)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "7KiDshEUVrX6" + "id": "ZKP_GFeKUWb5" }, + "outputs": [], "source": [ - "### Train the model" + "print(\n", + " \"This image most likely belongs to {} with a {:.2f} percent confidence.\"\n", + " .format(class_names[np.argmax(score_lite)], 100 * np.max(score_lite))\n", + ")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "NFj0oVqVVrX6" + "id": "Poz_iYgeUg_U" }, "source": [ - "After successfully introducing data augmentations to the training examples and adding dropouts to the network, train this new network:" + "The prediction generated by the lite model should be almost identical to the predictions generated by the original model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GWxHs_luVrX7" + "id": "InXXDJL8UYC1" }, "outputs": [], "source": [ - "history = model_new.fit_generator(\n", - " train_data_gen,\n", - " steps_per_epoch=total_train // batch_size,\n", - " epochs=epochs,\n", - " validation_data=val_data_gen,\n", - " validation_steps=total_val // batch_size\n", - ")" + "print(np.max(np.abs(predictions - predictions_lite)))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "bbdyqZdxVrYA" + "id": "5hJzY8XijM7N" }, "source": [ - "### Visualize the model" + "Of the five classes—`'daisy'`, `'dandelion'`, `'roses'`, `'sunflowers'`, and `'tulips'`—the model should predict the image belongs to sunflowers, which is the same result as before the TensorFlow Lite conversion.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "OgvF2nt7OtR7" - }, - "source": [ - "Visualize the new model after training, you can see that there is significantly less overfitting than before. The accuracy should go up after training the model for more epochs." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7BTeMuNAVrYC" + "id": "1RlfCY9v2_ir" }, - "outputs": [], "source": [ - "acc = history.history['accuracy']\n", - "val_acc = history.history['val_accuracy']\n", + "## Next steps\n", "\n", - "loss = history.history['loss']\n", - "val_loss = history.history['val_loss']\n", + "This tutorial showed how to train a model for image classification, test it, convert it to the TensorFlow Lite format for on-device applications (such as an image classification app), and perform inference with the TensorFlow Lite model with the Python API.\n", "\n", - "epochs_range = range(epochs)\n", - "\n", - "plt.figure(figsize=(8, 8))\n", - "plt.subplot(1, 2, 1)\n", - "plt.plot(epochs_range, acc, label='Training Accuracy')\n", - "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", - "plt.legend(loc='lower right')\n", - "plt.title('Training and Validation Accuracy')\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.plot(epochs_range, loss, label='Training Loss')\n", - "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", - "plt.legend(loc='upper right')\n", - "plt.title('Training and Validation Loss')\n", - "plt.show()" + "You can learn more about TensorFlow Lite through [tutorials](https://www.tensorflow.org/lite/tutorials) and [guides](https://www.tensorflow.org/lite/guide)." ] } ], @@ -1332,10 +1177,7 @@ "colab": { "collapsed_sections": [], "name": "classification.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/images/cnn.ipynb b/site/en/tutorials/images/cnn.ipynb index 22351130901..bf6cbfee4b9 100644 --- a/site/en/tutorials/images/cnn.ipynb +++ b/site/en/tutorials/images/cnn.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "x4HI2mpwlrcn" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "679Lmwt3l1Bk" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DSPCom-KmApV" }, "source": [ @@ -47,46 +43,43 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "klAltGp8ycek" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/images/cnn\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/images/cnn.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/images/cnn.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/images/cnn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qLGkt5qiyz4E" }, "source": [ - "This tutorial demonstrates training a simple [Convolutional Neural Network](https://developers.google.com/machine-learning/glossary/#convolutional_neural_network) (CNN) to classify [CIFAR images](https://www.cs.toronto.edu/~kriz/cifar.html). Because this tutorial uses the [Keras Sequential API](https://www.tensorflow.org/guide/keras/overview), creating and training our model will take just a few lines of code.\n" + "This tutorial demonstrates training a simple [Convolutional Neural Network](https://developers.google.com/machine-learning/glossary/#convolutional_neural_network) (CNN) to classify [CIFAR images](https://www.cs.toronto.edu/~kriz/cifar.html). Because this tutorial uses the [Keras Sequential API](https://www.tensorflow.org/guide/keras/overview), creating and training your model will take just a few lines of code.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "m7KBpffWzlxH" }, "source": [ @@ -95,21 +88,12 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iAve6DCL4JH4" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", "\n", "from tensorflow.keras import datasets, layers, models\n", @@ -119,7 +103,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jRFxccghyMVo" }, "source": [ @@ -131,10 +114,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "JWoEqyMuXFF4" }, "outputs": [], @@ -148,21 +129,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7wArwCTJJlUa" }, "source": [ "### Verify the data\n", "\n", - "To verify that the dataset looks correct, let's plot the first 25 images from the training set and display the class name below each image.\n" + "To verify that the dataset looks correct, let's plot the first 25 images from the training set and display the class name below each image:\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "K3PAELE2eSU9" }, "outputs": [], @@ -176,7 +154,7 @@ " plt.xticks([])\n", " plt.yticks([])\n", " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", + " plt.imshow(train_images[i])\n", " # The CIFAR labels happen to be arrays, \n", " # which is why you need the extra index\n", " plt.xlabel(class_names[train_labels[i][0]])\n", @@ -186,7 +164,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Oewp-wYg31t9" }, "source": [ @@ -196,22 +173,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3hQvqXpNyN3x" }, "source": [ "The 6 lines of code below define the convolutional base using a common pattern: a stack of [Conv2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2D) and [MaxPooling2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/MaxPool2D) layers.\n", "\n", - "As input, a CNN takes tensors of shape (image_height, image_width, color_channels), ignoring the batch size. If you are new to these dimensions, color_channels refers to (R,G,B). In this example, you will configure our CNN to process inputs of shape (32, 32, 3), which is the format of CIFAR images. You can do this by passing the argument `input_shape` to our first layer.\n", - "\n" + "As input, a CNN takes tensors of shape (image_height, image_width, color_channels), ignoring the batch size. If you are new to these dimensions, color_channels refers to (R,G,B). In this example, you will configure your CNN to process inputs of shape (32, 32, 3), which is the format of CIFAR images. You can do this by passing the argument `input_shape` to your first layer.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "L9YmGQBQPrdn" }, "outputs": [], @@ -227,19 +200,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lvDVFkg-2DPm" }, "source": [ - "Let's display the architecture of our model so far." + "Let's display the architecture of your model so far:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8-C4XBg4UTJy" }, "outputs": [], @@ -250,7 +220,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_j-AXYeZ2GO5" }, "source": [ @@ -260,45 +229,40 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_v8sVOtG37bT" }, "source": [ "### Add Dense layers on top\n", - "To complete our model, you will feed the last output tensor from the convolutional base (of shape (3, 3, 64)) into one or more Dense layers to perform classification. Dense layers take vectors as input (which are 1D), while the current output is a 3D tensor. First, you will flatten (or unroll) the 3D output to 1D, then add one or more Dense layers on top. CIFAR has 10 output classes, so you use a final Dense layer with 10 outputs and a softmax activation." + "\n", + "To complete the model, you will feed the last output tensor from the convolutional base (of shape (4, 4, 64)) into one or more Dense layers to perform classification. Dense layers take vectors as input (which are 1D), while the current output is a 3D tensor. First, you will flatten (or unroll) the 3D output to 1D, then add one or more Dense layers on top. CIFAR has 10 output classes, so you use a final Dense layer with 10 outputs." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "mRs95d6LUVEi" }, "outputs": [], "source": [ "model.add(layers.Flatten())\n", "model.add(layers.Dense(64, activation='relu'))\n", - "model.add(layers.Dense(10, activation='softmax'))" + "model.add(layers.Dense(10))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ipGiQMcR4Gtq" }, "source": [ - "Here's the complete architecture of our model." + "Here's the complete architecture of your model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8Yu_m-TZUWGX" }, "outputs": [], @@ -309,17 +273,15 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xNKXi-Gy3RO-" }, "source": [ - "As you can see, our (3, 3, 64) outputs were flattened into vectors of shape (576) before going through two Dense layers." + "The network summary shows that (4, 4, 64) outputs were flattened into vectors of shape (1024) before going through two Dense layers." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "P3odqfHP4M67" }, "source": [ @@ -328,16 +290,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "MdDzI75PUXrG" }, "outputs": [], "source": [ "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "\n", "history = model.fit(train_images, train_labels, epochs=10, \n", @@ -347,7 +307,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jKgyC5K_4O0d" }, "source": [ @@ -356,10 +315,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gtyDF0MKUcM7" }, "outputs": [], @@ -376,10 +333,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0LvwaKhtUdOo" }, "outputs": [], @@ -390,11 +345,10 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8cfJ8AR03gT5" }, "source": [ - "Our simple CNN has achieved a test accuracy of over 70%. Not bad for a few lines of code! For another CNN style, see an example using the Keras subclassing API and a `tf.GradientTape` [here](https://www.tensorflow.org/tutorials/quickstart/advanced)." + "Your simple CNN has achieved a test accuracy of over 70%. Not bad for a few lines of code! For another CNN style, check out the [TensorFlow 2 quickstart for experts](https://www.tensorflow.org/tutorials/quickstart/advanced) example that uses the Keras subclassing API and `tf.GradientTape`." ] } ], @@ -403,10 +357,7 @@ "colab": { "collapsed_sections": [], "name": "cnn.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/images/data_augmentation.ipynb b/site/en/tutorials/images/data_augmentation.ipynb new file mode 100644 index 00000000000..8a1eaaabec4 --- /dev/null +++ b/site/en/tutorials/images/data_augmentation.ipynb @@ -0,0 +1,1378 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "4EFY9e5wRn7v" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "pkTRazeVRwDe" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VyOckJu6Rs-i" + }, + "source": [ + "# Data augmentation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0HEsULqDR7AH" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PxIOE5RnSQtj" + }, + "source": [ + "## Overview\n", + "\n", + "This tutorial demonstrates data augmentation: a technique to increase the diversity of your training set by applying random (but realistic) transformations, such as image rotation.\n", + "\n", + "You will learn how to apply data augmentation in two ways:\n", + "\n", + "- Use the Keras preprocessing layers, such as `tf.keras.layers.Resizing`, `tf.keras.layers.Rescaling`, `tf.keras.layers.RandomFlip`, and `tf.keras.layers.RandomRotation`.\n", + "- Use the `tf.image` methods, such as `tf.image.flip_left_right`, `tf.image.rgb_to_grayscale`, `tf.image.adjust_brightness`, `tf.image.central_crop`, and `tf.image.stateless_random*`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-UxHAqXmSXN5" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "C2Q5rPenTAJP" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "\n", + "from tensorflow.keras import layers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ydx3SSoF4wpG" + }, + "source": [ + "## Download a dataset\n", + "\n", + "This tutorial uses the [tf_flowers](https://www.tensorflow.org/datasets/catalog/tf_flowers) dataset. For convenience, download the dataset using [TensorFlow Datasets](https://www.tensorflow.org/datasets). If you would like to learn about other ways of importing data, check out the [load images](https://www.tensorflow.org/tutorials/load_data/images) tutorial.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ytHhsYmO52zy" + }, + "outputs": [], + "source": [ + "(train_ds, val_ds, test_ds), metadata = tfds.load(\n", + " 'tf_flowers',\n", + " split=['train[:80%]', 'train[80%:90%]', 'train[90%:]'],\n", + " with_info=True,\n", + " as_supervised=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MjxEJtCwsnmm" + }, + "source": [ + "The flowers dataset has five classes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wKwx7vQuspxz" + }, + "outputs": [], + "source": [ + "num_classes = metadata.features['label'].num_classes\n", + "print(num_classes)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zZAQW44949uw" + }, + "source": [ + "Let's retrieve an image from the dataset and use it to demonstrate data augmentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kXlx1lCr5Bip" + }, + "outputs": [], + "source": [ + "get_label_name = metadata.features['label'].int2str\n", + "\n", + "image, label = next(iter(train_ds))\n", + "_ = plt.imshow(image)\n", + "_ = plt.title(get_label_name(label))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vdJ6XA4q2nqK" + }, + "source": [ + "## Use Keras preprocessing layers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GRMPnfzBB2hw" + }, + "source": [ + "### Resizing and rescaling\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jhG7gSWmUMJx" + }, + "source": [ + "You can use the Keras preprocessing layers to resize your images to a consistent shape (with `tf.keras.layers.Resizing`), and to rescale pixel values (with `tf.keras.layers.Rescaling`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jMM3b85e3yhd" + }, + "outputs": [], + "source": [ + "IMG_SIZE = 180\n", + "\n", + "resize_and_rescale = tf.keras.Sequential([\n", + " layers.Resizing(IMG_SIZE, IMG_SIZE),\n", + " layers.Rescaling(1./255)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4z8AV1WgnYNW" + }, + "source": [ + "Note: The rescaling layer above standardizes pixel values to the `[0, 1]` range. If instead you wanted it to be `[-1, 1]`, you would write `tf.keras.layers.Rescaling(1./127.5, offset=-1)`.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MQiTwsHJDHAD" + }, + "source": [ + "You can visualize the result of applying these layers to an image. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "X9OLuR1bC1Pd" + }, + "outputs": [], + "source": [ + "result = resize_and_rescale(image)\n", + "_ = plt.imshow(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yxAMg8Zql5lw" + }, + "source": [ + "Verify that the pixels are in the `[0, 1]` range:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DPTB8IQmSeKM" + }, + "outputs": [], + "source": [ + "print(\"Min and max pixel values:\", result.numpy().min(), result.numpy().max())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fL6M7fuivAw4" + }, + "source": [ + "### Data augmentation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SL4Suj46ScfU" + }, + "source": [ + "You can use the Keras preprocessing layers for data augmentation as well, such as `tf.keras.layers.RandomFlip` and `tf.keras.layers.RandomRotation`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "V-4PugTE-4sl" + }, + "source": [ + "Let's create a few preprocessing layers and apply them repeatedly to the same image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Svu_5yfa_Jb7" + }, + "outputs": [], + "source": [ + "data_augmentation = tf.keras.Sequential([\n", + " layers.RandomFlip(\"horizontal_and_vertical\"),\n", + " layers.RandomRotation(0.2),\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kfzEuaNg69iU" + }, + "outputs": [], + "source": [ + "# Add the image to a batch.\n", + "image = tf.cast(tf.expand_dims(image, 0), tf.float32)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eR4wwi5Q_UZK" + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(10, 10))\n", + "for i in range(9):\n", + " augmented_image = data_augmentation(image)\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " plt.imshow(augmented_image[0])\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jA17pEeS_2_-" + }, + "source": [ + "There are a variety of preprocessing layers you can use for data augmentation including `tf.keras.layers.RandomContrast`, `tf.keras.layers.RandomCrop`, `tf.keras.layers.RandomZoom`, and others." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GG5RhIJtE0ng" + }, + "source": [ + "### Two options to use the Keras preprocessing layers\n", + "\n", + "There are two ways you can use these preprocessing layers, with important trade-offs." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MxGvUT727Po6" + }, + "source": [ + "#### Option 1: Make the preprocessing layers part of your model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ULGJQjP6hHvu" + }, + "outputs": [], + "source": [ + "model = tf.keras.Sequential([\n", + " # Add the preprocessing layers you created earlier.\n", + " resize_and_rescale,\n", + " data_augmentation,\n", + " layers.Conv2D(16, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " # Rest of your model.\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pc6ELneyhJN9" + }, + "source": [ + "There are two important points to be aware of in this case:\n", + "\n", + "* Data augmentation will run on-device, synchronously with the rest of your layers, and benefit from GPU acceleration.\n", + "\n", + "* When you export your model using `model.save`, the preprocessing layers will be saved along with the rest of your model. If you later deploy this model, it will automatically standardize images (according to the configuration of your layers). This can save you from the effort of having to reimplement that logic server-side." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "syZwDSpiRXZP" + }, + "source": [ + "Note: Data augmentation is inactive at test time so input images will only be augmented during calls to `Model.fit` (not `Model.evaluate` or `Model.predict`)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B2X3JTeY_vfv" + }, + "source": [ + "#### Option 2: Apply the preprocessing layers to your dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "r1Bt7w5VhVDY" + }, + "outputs": [], + "source": [ + "aug_ds = train_ds.map(\n", + " lambda x, y: (resize_and_rescale(x, training=True), y))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HKqeahG2hVdV" + }, + "source": [ + "With this approach, you use `Dataset.map` to create a dataset that yields batches of augmented images. In this case:\n", + "\n", + "* Data augmentation will happen asynchronously on the CPU, and is non-blocking. You can overlap the training of your model on the GPU with data preprocessing, using `Dataset.prefetch`, shown below.\n", + "* In this case the preprocessing layers will not be exported with the model when you call `Model.save`. You will need to attach them to your model before saving it or reimplement them server-side. After training, you can attach the preprocessing layers before export.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cgj51k9J7jfc" + }, + "source": [ + "You can find an example of the first option in the [Image classification](classification.ipynb) tutorial. Let's demonstrate the second option here." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "31YwMQdrXKBP" + }, + "source": [ + "### Apply the preprocessing layers to the datasets" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WUgW-2LOGiOT" + }, + "source": [ + "Configure the training, validation, and test datasets with the Keras preprocessing layers you created earlier. You will also configure the datasets for performance, using parallel reads and buffered prefetching to yield batches from disk without I/O become blocking. (Learn more dataset performance in the [Better performance with the tf.data API](https://www.tensorflow.org/guide/data_performance) guide.)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eI7VdyqK767y" + }, + "source": [ + "Note: Data augmentation should only be applied to the training set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "R5fGVMqlFxF7" + }, + "outputs": [], + "source": [ + "batch_size = 32\n", + "AUTOTUNE = tf.data.AUTOTUNE\n", + "\n", + "def prepare(ds, shuffle=False, augment=False):\n", + " # Resize and rescale all datasets.\n", + " ds = ds.map(lambda x, y: (resize_and_rescale(x), y), \n", + " num_parallel_calls=AUTOTUNE)\n", + "\n", + " if shuffle:\n", + " ds = ds.shuffle(1000)\n", + "\n", + " # Batch all datasets.\n", + " ds = ds.batch(batch_size)\n", + "\n", + " # Use data augmentation only on the training set.\n", + " if augment:\n", + " ds = ds.map(lambda x, y: (data_augmentation(x, training=True), y), \n", + " num_parallel_calls=AUTOTUNE)\n", + "\n", + " # Use buffered prefetching on all datasets.\n", + " return ds.prefetch(buffer_size=AUTOTUNE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "N86SFGMBHcx-" + }, + "outputs": [], + "source": [ + "train_ds = prepare(train_ds, shuffle=True, augment=True)\n", + "val_ds = prepare(val_ds)\n", + "test_ds = prepare(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9gplDz4ZV6kk" + }, + "source": [ + "### Train a model\n", + "\n", + "For completeness, you will now train a model using the datasets you have just prepared.\n", + "\n", + "The [Sequential](https://www.tensorflow.org/guide/keras/sequential_model) model consists of three convolution blocks (`tf.keras.layers.Conv2D`) with a max pooling layer (`tf.keras.layers.MaxPooling2D`) in each of them. There's a fully-connected layer (`tf.keras.layers.Dense`) with 128 units on top of it that is activated by a ReLU activation function (`'relu'`). This model has not been tuned for accuracy (the goal is to show you the mechanics)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IODSymGhq9N6" + }, + "outputs": [], + "source": [ + "model = tf.keras.Sequential([\n", + " layers.Conv2D(16, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Conv2D(32, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Conv2D(64, 3, padding='same', activation='relu'),\n", + " layers.MaxPooling2D(),\n", + " layers.Flatten(),\n", + " layers.Dense(128, activation='relu'),\n", + " layers.Dense(num_classes)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "86454855f7d9" + }, + "source": [ + "Choose the `tf.keras.optimizers.Adam` optimizer and `tf.keras.losses.SparseCategoricalCrossentropy` loss function. To view training and validation accuracy for each training epoch, pass the `metrics` argument to `Model.compile`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZnRJr95WY68k" + }, + "outputs": [], + "source": [ + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "976f718cabc8" + }, + "source": [ + "Train for a few epochs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "i_sDl9uZY9Mh" + }, + "outputs": [], + "source": [ + "epochs=5\n", + "history = model.fit(\n", + " train_ds,\n", + " validation_data=val_ds,\n", + " epochs=epochs\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "V9PSf4qgiQJG" + }, + "outputs": [], + "source": [ + "loss, acc = model.evaluate(test_ds)\n", + "print(\"Accuracy\", acc)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0BkRvvsXb6SI" + }, + "source": [ + "### Custom data augmentation\n", + "\n", + "You can also create custom data augmentation layers.\n", + "\n", + "This section of the tutorial shows two ways of doing so:\n", + "\n", + "- First, you will create a `tf.keras.layers.Lambda` layer. This is a good way to write concise code.\n", + "- Next, you will write a new layer via [subclassing](https://www.tensorflow.org/guide/keras/custom_layers_and_models), which gives you more control.\n", + "\n", + "Both layers will randomly invert the colors in an image, according to some probability." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nMxEhIVXmAH0" + }, + "outputs": [], + "source": [ + "def random_invert_img(x, p=0.5):\n", + " if tf.random.uniform([]) < p:\n", + " x = (255-x)\n", + " else:\n", + " x\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "C0huNpxdmDKu" + }, + "outputs": [], + "source": [ + "def random_invert(factor=0.5):\n", + " return layers.Lambda(lambda x: random_invert_img(x, factor))\n", + "\n", + "random_invert = random_invert()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wAcOluP0TNG6" + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(10, 10))\n", + "for i in range(9):\n", + " augmented_image = random_invert(image)\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " plt.imshow(augmented_image[0].numpy().astype(\"uint8\"))\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Xd9XG2PLM5ZJ" + }, + "source": [ + "Next, implement a custom layer by [subclassing](https://www.tensorflow.org/guide/keras/custom_layers_and_models):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "d11eExc-Ke-7" + }, + "outputs": [], + "source": [ + "class RandomInvert(layers.Layer):\n", + " def __init__(self, factor=0.5, **kwargs):\n", + " super().__init__(**kwargs)\n", + " self.factor = factor\n", + "\n", + " def call(self, x):\n", + " return random_invert_img(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qX-VQgkRL6fc" + }, + "outputs": [], + "source": [ + "_ = plt.imshow(RandomInvert()(image)[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B0nmllnXZO6T" + }, + "source": [ + "Both of these layers can be used as described in options 1 and 2 above." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j7-k__2dAfX6" + }, + "source": [ + "## Using tf.image" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NJco2x35EAMs" + }, + "source": [ + "The above Keras preprocessing utilities are convenient. But, for finer control, you can write your own data augmentation pipelines or layers using `tf.data` and `tf.image`. (You may also want to check out [TensorFlow Addons Image: Operations](https://www.tensorflow.org/addons/tutorials/image_ops) and [TensorFlow I/O: Color Space Conversions](https://www.tensorflow.org/io/tutorials/colorspace).)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xR1RvjYkdd_i" + }, + "source": [ + "Since the flowers dataset was previously configured with data augmentation, let's reimport it to start fresh:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JB-lAS0z9ZJY" + }, + "outputs": [], + "source": [ + "(train_ds, val_ds, test_ds), metadata = tfds.load(\n", + " 'tf_flowers',\n", + " split=['train[:80%]', 'train[80%:90%]', 'train[90%:]'],\n", + " with_info=True,\n", + " as_supervised=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rQ3pqBTS9hNj" + }, + "source": [ + "Retrieve an image to work with:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dDsPaAi8de_j" + }, + "outputs": [], + "source": [ + "image, label = next(iter(train_ds))\n", + "_ = plt.imshow(image)\n", + "_ = plt.title(get_label_name(label))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "chelxcPtFiTF" + }, + "source": [ + "Let's use the following function to visualize and compare the original and augmented images side-by-side:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sN1ykjJCHikc" + }, + "outputs": [], + "source": [ + "def visualize(original, augmented):\n", + " fig = plt.figure()\n", + " plt.subplot(1,2,1)\n", + " plt.title('Original image')\n", + " plt.imshow(original)\n", + "\n", + " plt.subplot(1,2,2)\n", + " plt.title('Augmented image')\n", + " plt.imshow(augmented)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C5X4ijQYHmlt" + }, + "source": [ + "### Data augmentation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RRD9oujLHo6c" + }, + "source": [ + "#### Flip an image\n", + "\n", + "Flip an image either vertically or horizontally with `tf.image.flip_left_right`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1ZjVI24nIH0S" + }, + "outputs": [], + "source": [ + "flipped = tf.image.flip_left_right(image)\n", + "visualize(image, flipped)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6iD_lLibIL9q" + }, + "source": [ + "#### Grayscale an image\n", + "\n", + "You can grayscale an image with `tf.image.rgb_to_grayscale`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ikaMj0guIRtL" + }, + "outputs": [], + "source": [ + "grayscaled = tf.image.rgb_to_grayscale(image)\n", + "visualize(image, tf.squeeze(grayscaled))\n", + "_ = plt.colorbar()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f-5yjIs4IZ7v" + }, + "source": [ + "#### Saturate an image\n", + "\n", + "Saturate an image with `tf.image.adjust_saturation` by providing a saturation factor:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PHz-NosiInmz" + }, + "outputs": [], + "source": [ + "saturated = tf.image.adjust_saturation(image, 3)\n", + "visualize(image, saturated)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FWXiy8qfIqdC" + }, + "source": [ + "#### Change image brightness\n", + "\n", + "Change the brightness of image with `tf.image.adjust_brightness` by providing a brightness factor:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1hdG-j46I0nJ" + }, + "outputs": [], + "source": [ + "bright = tf.image.adjust_brightness(image, 0.4)\n", + "visualize(image, bright)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vjEOFEITJOr2" + }, + "source": [ + "#### Center crop an image\n", + "\n", + "Crop the image from center up to the image part you desire using `tf.image.central_crop`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RWkK5GFHJUKT" + }, + "outputs": [], + "source": [ + "cropped = tf.image.central_crop(image, central_fraction=0.5)\n", + "visualize(image, cropped)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "unt76GebI3Gc" + }, + "source": [ + "#### Rotate an image\n", + "\n", + "Rotate an image by 90 degrees with `tf.image.rot90`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "b19KuAhkJKR-" + }, + "outputs": [], + "source": [ + "rotated = tf.image.rot90(image)\n", + "visualize(image, rotated)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5CPP0vEKB56X" + }, + "source": [ + "### Random transformations\n", + "\n", + "Warning: There are two sets of random image operations: `tf.image.random*` and `tf.image.stateless_random*`. Using `tf.image.random*` operations is strongly discouraged as they use the old RNGs from TF 1.x. Instead, please use the random image operations introduced in this tutorial. For more information, refer to [Random number generation](../../guide/random_numbers.ipynb).\n", + "\n", + "Applying random transformations to the images can further help generalize and expand the dataset. The current `tf.image` API provides eight such random image operations (ops):\n", + "\n", + "* [`tf.image.stateless_random_brightness`](https://www.tensorflow.org/api_docs/python/tf/image/stateless_random_brightness)\n", + "* [`tf.image.stateless_random_contrast`](https://www.tensorflow.org/api_docs/python/tf/image/stateless_random_contrast)\n", + "* [`tf.image.stateless_random_crop`](https://www.tensorflow.org/api_docs/python/tf/image/stateless_random_crop)\n", + "* [`tf.image.stateless_random_flip_left_right`](https://www.tensorflow.org/api_docs/python/tf/image/stateless_random_flip_left_right)\n", + "* [`tf.image.stateless_random_flip_up_down`](https://www.tensorflow.org/api_docs/python/tf/image/stateless_random_flip_up_down)\n", + "* [`tf.image.stateless_random_hue`](https://www.tensorflow.org/api_docs/python/tf/image/stateless_random_hue)\n", + "* [`tf.image.stateless_random_jpeg_quality`](https://www.tensorflow.org/api_docs/python/tf/image/stateless_random_jpeg_quality)\n", + "* [`tf.image.stateless_random_saturation`](https://www.tensorflow.org/api_docs/python/tf/image/stateless_random_saturation)\n", + "\n", + "These random image ops are purely functional: the output only depends on the input. This makes them simple to use in high performance, deterministic input pipelines. They require a `seed` value be input each step. Given the same `seed`, they return the same results independent of how many times they are called.\n", + "\n", + "Note: `seed` is a `Tensor` of shape `(2,)` whose values are any integers.\n", + "\n", + "In the following sections, you will:\n", + "1. Go over examples of using random image operations to transform an image.\n", + "2. Demonstrate how to apply random transformations to a training dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "251Wy-MqE4La" + }, + "source": [ + "#### Randomly change image brightness\n", + "\n", + "Randomly change the brightness of `image` using `tf.image.stateless_random_brightness` by providing a brightness factor and `seed`. The brightness factor is chosen randomly in the range `[-max_delta, max_delta)` and is associated with the given `seed`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-fFd1kh7Fr-_" + }, + "outputs": [], + "source": [ + "for i in range(3):\n", + " seed = (i, 0) # tuple of size (2,)\n", + " stateless_random_brightness = tf.image.stateless_random_brightness(\n", + " image, max_delta=0.95, seed=seed)\n", + " visualize(image, stateless_random_brightness)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uLaDEmooUfYJ" + }, + "source": [ + "#### Randomly change image contrast\n", + "\n", + "Randomly change the contrast of `image` using `tf.image.stateless_random_contrast` by providing a contrast range and `seed`. The contrast range is chosen randomly in the interval `[lower, upper]` and is associated with the given `seed`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GmcYoQHaUoke" + }, + "outputs": [], + "source": [ + "for i in range(3):\n", + " seed = (i, 0) # tuple of size (2,)\n", + " stateless_random_contrast = tf.image.stateless_random_contrast(\n", + " image, lower=0.1, upper=0.9, seed=seed)\n", + " visualize(image, stateless_random_contrast)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wxb-MP-KVPNz" + }, + "source": [ + "#### Randomly crop an image\n", + "\n", + "Randomly crop `image` using `tf.image.stateless_random_crop` by providing target `size` and `seed`. The portion that gets cropped out of `image` is at a randomly chosen offset and is associated with the given `seed`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vtZQbUw0VOm5" + }, + "outputs": [], + "source": [ + "for i in range(3):\n", + " seed = (i, 0) # tuple of size (2,)\n", + " stateless_random_crop = tf.image.stateless_random_crop(\n", + " image, size=[210, 300, 3], seed=seed)\n", + " visualize(image, stateless_random_crop)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "isrM-MZtpxTq" + }, + "source": [ + "### Apply augmentation to a dataset\n", + "\n", + "Let's first download the image dataset again in case they are modified in the previous sections." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xC80NQP809Uo" + }, + "outputs": [], + "source": [ + "(train_datasets, val_ds, test_ds), metadata = tfds.load(\n", + " 'tf_flowers',\n", + " split=['train[:80%]', 'train[80%:90%]', 'train[90%:]'],\n", + " with_info=True,\n", + " as_supervised=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SMo9HTDV0Gaz" + }, + "source": [ + "Next, define a utility function for resizing and rescaling the images. This function will be used in unifying the size and scale of images in the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1JKmx06lfcFr" + }, + "outputs": [], + "source": [ + "def resize_and_rescale(image, label):\n", + " image = tf.cast(image, tf.float32)\n", + " image = tf.image.resize(image, [IMG_SIZE, IMG_SIZE])\n", + " image = (image / 255.0)\n", + " return image, label" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M7OpE_-jWq-I" + }, + "source": [ + "Let's also define the `augment` function that can apply the random transformations to the images. This function will be used on the dataset in the next step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KitLdvlpVxPa" + }, + "outputs": [], + "source": [ + "def augment(image_label, seed):\n", + " image, label = image_label\n", + " image, label = resize_and_rescale(image, label)\n", + " image = tf.image.resize_with_crop_or_pad(image, IMG_SIZE + 6, IMG_SIZE + 6)\n", + " # Make a new seed.\n", + " new_seed = tf.random.split(seed, num=1)[0, :]\n", + " # Random crop back to the original size.\n", + " image = tf.image.stateless_random_crop(\n", + " image, size=[IMG_SIZE, IMG_SIZE, 3], seed=seed)\n", + " # Random brightness.\n", + " image = tf.image.stateless_random_brightness(\n", + " image, max_delta=0.5, seed=new_seed)\n", + " image = tf.clip_by_value(image, 0, 1)\n", + " return image, label" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SlXRsVp70hg8" + }, + "source": [ + "#### Option 1: Using tf.data.experimental.Counter\n", + "\n", + "Create a `tf.data.experimental.Counter` object (let's call it `counter`) and `Dataset.zip` the dataset with `(counter, counter)`. This will ensure that each image in the dataset gets associated with a unique value (of shape `(2,)`) based on `counter` which later can get passed into the `augment` function as the `seed` value for random transformations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SZ6Qq0IWznfi" + }, + "outputs": [], + "source": [ + "# Create a `Counter` object and `Dataset.zip` it together with the training set.\n", + "counter = tf.data.experimental.Counter()\n", + "train_ds = tf.data.Dataset.zip((train_datasets, (counter, counter)))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eF9ybVQ94X9f" + }, + "source": [ + "Map the `augment` function to the training dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wQK9BDKk1_3N" + }, + "outputs": [], + "source": [ + "train_ds = (\n", + " train_ds\n", + " .shuffle(1000)\n", + " .map(augment, num_parallel_calls=AUTOTUNE)\n", + " .batch(batch_size)\n", + " .prefetch(AUTOTUNE)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3AQoyA-k3ELk" + }, + "outputs": [], + "source": [ + "val_ds = (\n", + " val_ds\n", + " .map(resize_and_rescale, num_parallel_calls=AUTOTUNE)\n", + " .batch(batch_size)\n", + " .prefetch(AUTOTUNE)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "p2IQN3NN3G_M" + }, + "outputs": [], + "source": [ + "test_ds = (\n", + " test_ds\n", + " .map(resize_and_rescale, num_parallel_calls=AUTOTUNE)\n", + " .batch(batch_size)\n", + " .prefetch(AUTOTUNE)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pvTVY8BY2LpD" + }, + "source": [ + "#### Option 2: Using tf.random.Generator\n", + "\n", + "- Create a `tf.random.Generator` object with an initial `seed` value. Calling the `make_seeds` function on the same generator object always returns a new, unique `seed` value.\n", + "- Define a wrapper function that: 1) calls the `make_seeds` function; and 2) passes the newly generated `seed` value into the `augment` function for random transformations.\n", + "\n", + "Note: `tf.random.Generator` objects store RNG state in a `tf.Variable`, which means it can be saved as a [checkpoint](../../guide/checkpoint.ipynb) or in a [SavedModel](../../guide/saved_model.ipynb). For more details, please refer to [Random number generation](../../guide/random_numbers.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BQDvedZ33eAy" + }, + "outputs": [], + "source": [ + "# Create a generator.\n", + "rng = tf.random.Generator.from_seed(123, alg='philox')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eDEkO1nt2ta0" + }, + "outputs": [], + "source": [ + "# Create a wrapper function for updating seeds.\n", + "def f(x, y):\n", + " seed = rng.make_seeds(1)[:, 0]\n", + " image, label = augment((x, y), seed)\n", + " return image, label" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PyPC4vUM4MT0" + }, + "source": [ + "Map the wrapper function `f` to the training dataset, and the `resize_and_rescale` function—to the validation and test sets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Pu2uB7k12xKw" + }, + "outputs": [], + "source": [ + "train_ds = (\n", + " train_datasets\n", + " .shuffle(1000)\n", + " .map(f, num_parallel_calls=AUTOTUNE)\n", + " .batch(batch_size)\n", + " .prefetch(AUTOTUNE)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e6caldPi2HAP" + }, + "outputs": [], + "source": [ + "val_ds = (\n", + " val_ds\n", + " .map(resize_and_rescale, num_parallel_calls=AUTOTUNE)\n", + " .batch(batch_size)\n", + " .prefetch(AUTOTUNE)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ceaCdJnh2I-r" + }, + "outputs": [], + "source": [ + "test_ds = (\n", + " test_ds\n", + " .map(resize_and_rescale, num_parallel_calls=AUTOTUNE)\n", + " .batch(batch_size)\n", + " .prefetch(AUTOTUNE)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hKwCA6AOjTrc" + }, + "source": [ + "These datasets can now be used to train a model as shown previously." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YypDihDlj0no" + }, + "source": [ + "## Next steps\n", + "\n", + "This tutorial demonstrated data augmentation using Keras preprocessing layers and `tf.image`.\n", + "\n", + "- To learn how to include preprocessing layers inside your model, refer to the [Image classification](classification.ipynb) tutorial.\n", + "- You may also be interested in learning how preprocessing layers can help you classify text, as shown in the [Basic text classification](../keras/text_classification.ipynb) tutorial.\n", + "- You can learn more about `tf.data` in this [guide](../../guide/data.ipynb), and you can learn how to configure your input pipelines for performance [here](../../guide/data_performance.ipynb)." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "data_augmentation.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/images/images/tensorboard_transfer_learning_with_hub.png b/site/en/tutorials/images/images/tensorboard_transfer_learning_with_hub.png new file mode 100644 index 00000000000..5b31cddb5eb Binary files /dev/null and b/site/en/tutorials/images/images/tensorboard_transfer_learning_with_hub.png differ diff --git a/site/en/tutorials/images/index.md b/site/en/tutorials/images/index.md new file mode 100644 index 00000000000..d1bf3e01839 --- /dev/null +++ b/site/en/tutorials/images/index.md @@ -0,0 +1,113 @@ +# Computer vision with TensorFlow + +TensorFlow provides a number of computer vision (CV) and image classification +tools. This document introduces some of these tools and provides an overview of +resources to help you get started with common CV tasks. + +## Vision libraries and tools + +TensorFlow provides CV tools through the higher-level Keras libraries and the +lower-level `tf.image` module. For most use cases, the Keras libraries +will be more convenient than the built-in TensorFlow alternatives. +But if the Keras options don't fit your use case, or you want lower-level +control over image preprocessing, you might need the lower-level TensorFlow +tools. + +### KerasCV + +If you're just getting started with a CV project, and you're not sure which +libraries and tools you'll need, [KerasCV](https://keras.io/keras_cv/) is a good +place to start. KerasCV is a library of modular CV components built on Keras +Core. KerasCV includes models, layers, metrics, callbacks, and other tools that +extend the high-level Keras API for CV tasks. The KerasCV APIs can help with +data augmentation, classification, object detection, segmentation, +image generation, and other common CV workflows. You can use KerasCV to quickly +assemble production-grade, state-of-the-art training and inference pipelines. + +### Keras utilities + +`tf.keras.utils` provides several high-level image preprocessing utilities. For +example, `tf.keras.utils.image_dataset_from_directory` generates a +`tf.data.Dataset` from a directory of images on disk. + +### `tf.image` + +If KerasCV doesn't fit your use case, you can use `tf.image` and `tf.data` to +write your own data augmentation pipelines or layers. + +The `tf.image` module contains various functions for image processing, such as +`tf.image.flip_left_right`, `tf.image.rgb_to_grayscale`, +`tf.image.adjust_brightness`, `tf.image.central_crop`, and +`tf.image.stateless_random*`. + +The `tf.data` API enables you to build complex input pipelines from simple, +reusable pieces. + +### TensorFlow Datasets + +[TensorFlow Datasets](https://www.tensorflow.org/datasets) is a collection of +datasets ready to use with TensorFlow. Many of the datasets (for example, +[MNIST](https://www.tensorflow.org/datasets/catalog/mnist), +[Fashion-MNIST](https://www.tensorflow.org/datasets/catalog/fashion_mnist), and +[TF Flowers](https://www.tensorflow.org/datasets/catalog/tf_flowers)) can be +used to develop and test computer vision algorithms. + +## Where to start + +The following resources will help you get up and running with TensorFlow and +Keras CV tools. + +* [KerasCV](https://keras.io/keras_cv/): Documentation and resources for + KerasCV. +* [KerasCV developer guides](https://keras.io/guides/keras_cv/): Guides to + performing common CV tasks using KerasCV. If you're new to KerasCV, + [Classification with KerasCV](https://keras.io/guides/keras_cv/classification_with_keras_cv/) + is a good place to start. +* [TensorFlow tutorials](https://www.tensorflow.org/tutorials): The core + TensorFlow documentation (this guide) includes a number of CV and image + processing tutorials. + * [Basic classification: Classify images of clothing](https://www.tensorflow.org/tutorials/keras/classification): + Train a neural network model to classify images of clothing, like sneakers + and shirts. + * [Load and preprocess images](https://www.tensorflow.org/tutorials/load_data/images): + Load and preprocess an image dataset in three ways: + + 1. Use high-level Keras preprocessing utilities to read a directory of + images on disk. + 2. Write your own input pipeline from scratch + [using `tf.data`](https://www.tensorflow.org/guide/data). + 3. Download a dataset from the large + [catalog](https://www.tensorflow.org/datasets/catalog/overview) + available in + [TensorFlow Datasets](https://www.tensorflow.org/datasets). + + * [Load video data](https://www.tensorflow.org/tutorials/load_data/video): + Load and preprocess AVI video data using the + [UCF101 human action dataset](https://www.tensorflow.org/datasets/catalog/ucf101). + * [Convolutional Neural Network (CNN)](https://www.tensorflow.org/tutorials/images/cnn): + Train a simple [Convolutional Neural Network](https://developers.google.com/machine-learning/glossary/#convolutional_neural_network) + (CNN) to classify + [CIFAR images](https://www.cs.toronto.edu/~kriz/cifar.html) + using the + [Keras API](https://www.tensorflow.org/guide/keras/overview). + * [Image classification](https://www.tensorflow.org/tutorials/images/classification): + Classify images of flowers using a `tf.keras.Sequential` model and load data + using `tf.keras.utils.image_dataset_from_directory`. + * [Transfer learning and fine-tuning](https://www.tensorflow.org/tutorials/images/transfer_learning): + Classify images of cats and dogs by using transfer learning from a + pre-trained network. + * [Data augmentation](https://www.tensorflow.org/tutorials/images/data_augmentation): + Increase the diversity of your training set by applying random (but + realistic) transformations, such as image rotation. + * [Image segmentation](https://www.tensorflow.org/tutorials/images/segmentation): + Perform image segmentation, using a modified + [U-Net](https://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/){: .external}. + * [Video classification with a 3D convolutional neural network](https://www.tensorflow.org/tutorials/video/video_classification): + Train a 3D convolutional neural network (CNN) for video classification using + the [UCF101](https://www.crcv.ucf.edu/data/UCF101.php){: .external} action + recognition dataset. + * [Transfer learning for video classification with MoViNet](https://www.tensorflow.org/tutorials/video/transfer_learning_with_movinet): + Use a pre-trained MoViNet model and the + [UCF101 dataset](https://www.crcv.ucf.edu/data/UCF101.php){: .external} to + classify videos for an action recognition task. + diff --git a/site/en/tutorials/images/segmentation.ipynb b/site/en/tutorials/images/segmentation.ipynb index 85aab2d3542..285ef538664 100644 --- a/site/en/tutorials/images/segmentation.ipynb +++ b/site/en/tutorials/images/segmentation.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "cZCM65CBt1CJ" }, "source": [ @@ -14,11 +13,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "JOgMcEajtkmg" }, "outputs": [], @@ -39,7 +36,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rCSP-dbMw88x" }, "source": [ @@ -49,99 +45,88 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NEWs8JXRuGex" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/images/segmentation\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/images/segmentation.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/images/segmentation.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/images/segmentation.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "sMP7mglMuGT2" }, "source": [ - "This tutorial focuses on the task of image segmentation, using a modified [U-Net](https://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/).\n", + "This tutorial focuses on the task of image segmentation, using a modified U-Net.\n", "\n", "## What is image segmentation?\n", - "So far you have seen image classification, where the task of the network is to assign a label or class to an input image. However, suppose you want to know where an object is located in the image, the shape of that object, which pixel belongs to which object, etc. In this case you will want to segment the image, i.e., each pixel of the image is given a label. Thus, the task of image segmentation is to train a neural network to output a pixel-wise mask of the image. This helps in understanding the image at a much lower level, i.e., the pixel level. Image segmentation has many applications in medical imaging, self-driving cars and satellite imaging to name a few.\n", "\n", - "The dataset that will be used for this tutorial is the [Oxford-IIIT Pet Dataset](https://www.robots.ox.ac.uk/~vgg/data/pets/), created by Parkhi *et al*. The dataset consists of images, their corresponding labels, and pixel-wise masks. The masks are basically labels for each pixel. Each pixel is given one of three categories :\n", + "In an image classification task, the network assigns a label (or class) to each input image. However, suppose you want to know the shape of that object, which pixel belongs to which object, etc. In this case, you need to assign a class to each pixel of the image—this task is known as segmentation. A segmentation model returns much more detailed information about the image. Image segmentation has many applications in medical imaging, self-driving cars and satellite imaging, just to name a few.\n", "\n", - "* Class 1 : Pixel belonging to the pet.\n", - "* Class 2 : Pixel bordering the pet.\n", - "* Class 3 : None of the above/ Surrounding pixel." + "This tutorial uses the [Oxford-IIIT Pet Dataset](https://www.robots.ox.ac.uk/~vgg/data/pets/) ([Parkhi et al, 2012](https://www.robots.ox.ac.uk/~vgg/publications/2012/parkhi12a/parkhi12a.pdf)). The dataset consists of images of 37 pet breeds, with 200 images per breed (~100 each in the training and test splits). Each image includes the corresponding labels, and pixel-wise masks. The masks are class-labels for each pixel. Each pixel is given one of three categories:\n", + "\n", + "- Class 1: Pixel belonging to the pet.\n", + "- Class 2: Pixel bordering the pet.\n", + "- Class 3: None of the above/a surrounding pixel." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "MQmKthrSBCld" }, "outputs": [], "source": [ - "!pip install git+https://github.com/tensorflow/examples.git" + "!pip install git+https://github.com/tensorflow/examples.git\n", + "!pip install -U keras\n", + "!pip install -q tensorflow_datasets\n", + "!pip install -q -U tensorflow-text tensorflow" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YQX7R4bhZy5h" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" + "import numpy as np\n", + "\n", + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "g87--n2AtyO_" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "from tensorflow_examples.models.pix2pix import pix2pix\n", "\n", - "import tensorflow_datasets as tfds\n", - "tfds.disable_progress_bar()\n", - "\n", "from IPython.display import clear_output\n", "import matplotlib.pyplot as plt" ] @@ -149,21 +134,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "oWe0_rQM4JbC" }, "source": [ "## Download the Oxford-IIIT Pets dataset\n", "\n", - "The dataset is already included in TensorFlow datasets, all that is needed to do is download it. The segmentation masks are included in version 3+." + "The dataset is [available from TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/oxford_iiit_pet). The segmentation masks are included in version 3+." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "40ITeStwDwZb" }, "outputs": [], @@ -174,19 +156,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rJcVdj_U4vzf" }, "source": [ - "The following code performs a simple augmentation of flipping an image. In addition, image is normalized to [0,1]. Finally, as mentioned above the pixels in the segmentation mask are labeled either {1, 2, 3}. For the sake of convinience, let's subtract 1 from the segmentation mask, resulting in labels that are : {0, 1, 2}." + " In addition, the image color values are normalized to the `[0, 1]` range. Finally, as mentioned above the pixels in the segmentation mask are labeled either {1, 2, 3}. For the sake of convenience, subtract 1 from the segmentation mask, resulting in labels that are : {0, 1, 2}." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "FD60EbcAQqov" }, "outputs": [], @@ -199,41 +178,19 @@ }, { "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2NPlCnBXQwb1" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def load_image_train(datapoint):\n", - " input_image = tf.image.resize(datapoint['image'], (128, 128))\n", - " input_mask = tf.image.resize(datapoint['segmentation_mask'], (128, 128))\n", - "\n", - " if tf.random.uniform(()) \u003e 0.5:\n", - " input_image = tf.image.flip_left_right(input_image)\n", - " input_mask = tf.image.flip_left_right(input_mask)\n", - "\n", - " input_image, input_mask = normalize(input_image, input_mask)\n", - "\n", - " return input_image, input_mask" - ] - }, - { - "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Zf0S67hJRp3D" }, "outputs": [], "source": [ - "def load_image_test(datapoint):\n", + "def load_image(datapoint):\n", " input_image = tf.image.resize(datapoint['image'], (128, 128))\n", - " input_mask = tf.image.resize(datapoint['segmentation_mask'], (128, 128))\n", + " input_mask = tf.image.resize(\n", + " datapoint['segmentation_mask'],\n", + " (128, 128),\n", + " method = tf.image.ResizeMethod.NEAREST_NEIGHBOR,\n", + " )\n", "\n", " input_image, input_mask = normalize(input_image, input_mask)\n", "\n", @@ -243,19 +200,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "65-qHTjX5VZh" }, "source": [ - "The dataset already contains the required splits of test and train and so let's continue to use the same split." + "The dataset already contains the required training and test splits, so continue to use the same splits:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "yHwj2-8SaQli" }, "outputs": [], @@ -268,49 +222,89 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "39fYScNz9lmo" }, "outputs": [], "source": [ - "train = dataset['train'].map(load_image_train, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n", - "test = dataset['test'].map(load_image_test)" + "train_images = dataset['train'].map(load_image, num_parallel_calls=tf.data.AUTOTUNE)\n", + "test_images = dataset['test'].map(load_image, num_parallel_calls=tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T9hGHyg8L3Y1" + }, + "source": [ + "The following class performs a simple augmentation by randomly-flipping an image.\n", + "Go to the [Image augmentation](data_augmentation.ipynb) tutorial to learn more.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DeFwFDN6EVoI" + "id": "fUWdDJRTL0PP" }, "outputs": [], "source": [ - "train_dataset = train.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()\n", - "train_dataset = train_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n", - "test_dataset = test.batch(BATCH_SIZE)" + "class Augment(tf.keras.layers.Layer):\n", + " def __init__(self, seed=42):\n", + " super().__init__()\n", + " # both use the same seed, so they'll make the same random changes.\n", + " self.augment_inputs = tf.keras.layers.RandomFlip(mode=\"horizontal\", seed=seed)\n", + " self.augment_labels = tf.keras.layers.RandomFlip(mode=\"horizontal\", seed=seed)\n", + "\n", + " def call(self, inputs, labels):\n", + " inputs = self.augment_inputs(inputs)\n", + " labels = self.augment_labels(labels)\n", + " return inputs, labels" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xTIbNIBdcgL3" + }, + "source": [ + "Build the input pipeline, applying the augmentation after batching the inputs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VPscskQcNCx4" + }, + "outputs": [], + "source": [ + "train_batches = (\n", + " train_images\n", + " .cache()\n", + " .shuffle(BUFFER_SIZE)\n", + " .batch(BATCH_SIZE)\n", + " .repeat()\n", + " .map(Augment())\n", + " .prefetch(buffer_size=tf.data.AUTOTUNE))\n", + "\n", + "test_batches = test_images.batch(BATCH_SIZE)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Xa3gMAE_9qNa" }, "source": [ - "Let's take a look at an image example and it's correponding mask from the dataset." + "Visualize an image example and its corresponding mask from the dataset:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3N2RPAAW9q4W" }, "outputs": [], @@ -323,68 +317,47 @@ " for i in range(len(display_list)):\n", " plt.subplot(1, len(display_list), i+1)\n", " plt.title(title[i])\n", - " plt.imshow(tf.keras.preprocessing.image.array_to_img(display_list[i]))\n", + " plt.imshow(tf.keras.utils.array_to_img(display_list[i]))\n", " plt.axis('off')\n", " plt.show()" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "a6u_Rblkteqb" }, "outputs": [], "source": [ - "for image, mask in train.take(1):\n", - " sample_image, sample_mask = image, mask\n", - "display([sample_image, sample_mask])" + "for images, masks in train_batches.take(2):\n", + " sample_image, sample_mask = images[0], masks[0]\n", + " display([sample_image, sample_mask])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FAOe93FRMk3w" }, "source": [ "## Define the model\n", - "The model being used here is a modified U-Net. A U-Net consists of an encoder (downsampler) and decoder (upsampler). In-order to learn robust features, and reduce the number of trainable parameters, a pretrained model can be used as the encoder. Thus, the encoder for this task will be a pretrained MobileNetV2 model, whose intermediate outputs will be used, and the decoder will be the upsample block already implemented in TensorFlow Examples in the [Pix2pix tutorial](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/pix2pix/pix2pix.py). \n", - "\n", - "The reason to output three channels is because there are three possible labels for each pixel. Think of this as multi-classification where each pixel is being classified into three classes." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "c6iB4iMvMkX9" - }, - "outputs": [], - "source": [ - "OUTPUT_CHANNELS = 3" + "The model being used here is a modified [U-Net](https://arxiv.org/abs/1505.04597). A U-Net consists of an encoder (downsampler) and decoder (upsampler). To learn robust features and reduce the number of trainable parameters, use a pretrained model—[MobileNetV2](https://arxiv.org/abs/1801.04381)—as the encoder. For the decoder, you will use the upsample block, which is already implemented in the [pix2pix](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/pix2pix/pix2pix.py) example in the TensorFlow Examples repo. (Check out the [pix2pix: Image-to-image translation with a conditional GAN](../generative/pix2pix.ipynb) tutorial in a notebook.)\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "W4mQle3lthit" }, "source": [ - "As mentioned, the encoder will be a pretrained MobileNetV2 model which is prepared and ready to use in [tf.keras.applications](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/applications). The encoder consists of specific outputs from intermediate layers in the model. Note that the encoder will not be trained during the training process." + "As mentioned, the encoder is a pretrained MobileNetV2 model. You will use the model from `tf.keras.applications`. The encoder consists of specific outputs from intermediate layers in the model. Note that the encoder will not be trained during the training process." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "liCeLH0ctjq7" }, "outputs": [], @@ -399,10 +372,10 @@ " 'block_13_expand_relu', # 8x8\n", " 'block_16_project', # 4x4\n", "]\n", - "layers = [base_model.get_layer(name).output for name in layer_names]\n", + "base_model_outputs = [base_model.get_layer(name).output for name in layer_names]\n", "\n", "# Create the feature extraction model\n", - "down_stack = tf.keras.Model(inputs=base_model.input, outputs=layers)\n", + "down_stack = tf.keras.Model(inputs=base_model.input, outputs=base_model_outputs)\n", "\n", "down_stack.trainable = False" ] @@ -410,53 +383,41 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "KPw8Lzra5_T9" }, "source": [ - "The decoder/upsampler is simply a series of upsample blocks implemented in TensorFlow examples." + "The decoder/upsampler is simply a series of upsample blocks implemented in TensorFlow examples:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "p0ZbfywEbZpJ" }, "outputs": [], "source": [ "up_stack = [\n", - " pix2pix.upsample(512, 3), # 4x4 -\u003e 8x8\n", - " pix2pix.upsample(256, 3), # 8x8 -\u003e 16x16\n", - " pix2pix.upsample(128, 3), # 16x16 -\u003e 32x32\n", - " pix2pix.upsample(64, 3), # 32x32 -\u003e 64x64\n", + " pix2pix.upsample(512, 3), # 4x4 -> 8x8\n", + " pix2pix.upsample(256, 3), # 8x8 -> 16x16\n", + " pix2pix.upsample(128, 3), # 16x16 -> 32x32\n", + " pix2pix.upsample(64, 3), # 32x32 -> 64x64\n", "]" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "45HByxpVtrPF" }, "outputs": [], "source": [ - "def unet_model(output_channels):\n", - "\n", - " # This is the last layer of the model\n", - " last = tf.keras.layers.Conv2DTranspose(\n", - " output_channels, 3, strides=2,\n", - " padding='same', activation='softmax') #64x64 -\u003e 128x128\n", - "\n", + "def unet_model(output_channels:int):\n", " inputs = tf.keras.layers.Input(shape=[128, 128, 3])\n", - " x = inputs\n", "\n", " # Downsampling through the model\n", - " skips = down_stack(x)\n", + " skips = down_stack(inputs)\n", " x = skips[-1]\n", " skips = reversed(skips[:-1])\n", "\n", @@ -466,6 +427,11 @@ " concat = tf.keras.layers.Concatenate()\n", " x = concat([x, skip])\n", "\n", + " # This is the last layer of the model\n", + " last = tf.keras.layers.Conv2DTranspose(\n", + " filters=output_channels, kernel_size=3, strides=2,\n", + " padding='same') #64x64 -> 128x128\n", + "\n", " x = last(x)\n", "\n", " return tf.keras.Model(inputs=inputs, outputs=x)" @@ -474,84 +440,90 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "LRsjdZuEnZfA" + }, + "source": [ + "Note that the number of filters on the last layer is set to the number of `output_channels`. This will be one output channel per class." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "j0DGH_4T0VYn" }, "source": [ "## Train the model\n", - "Now, all that is left to do is to compile and train the model. The loss being used here is losses.sparse_categorical_crossentropy. The reason to use this loss function is because the network is trying to assign each pixel a label, just like multi-class prediction. In the true segmentation mask, each pixel has either a {0,1,2}. The network here is outputting three channels. Essentially, each channel is trying to learn to predict a class, and losses.sparse_categorical_crossentropy is the recommended loss for such a scenario. Using the output of the network, the label assigned to the pixel is the channel with the highest value. This is what the create_mask function is doing." + "\n", + "Now, all that is left to do is to compile and train the model.\n", + "\n", + "Since this is a multiclass classification problem, use the `tf.keras.losses.SparseCategoricalCrossentropy` loss function with the `from_logits` argument set to `True`, since the labels are scalar integers instead of vectors of scores for each pixel of every class.\n", + "\n", + "When running inference, the label assigned to the pixel is the channel with the highest value. This is what the `create_mask` function is doing." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "6he36HK5uKAc" }, "outputs": [], "source": [ - "model = unet_model(OUTPUT_CHANNELS)\n", - "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',\n", + "OUTPUT_CLASSES = 3\n", + "\n", + "model = unet_model(output_channels=OUTPUT_CLASSES)\n", + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xVMzbIZLcyEF" }, "source": [ - "Have a quick look at the resulting model architecture:" + "Plot the resulting model architecture:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "sw82qF1Gcovr" }, "outputs": [], "source": [ - "tf.keras.utils.plot_model(model, show_shapes=True)" + "tf.keras.utils.plot_model(model, show_shapes=True, expand_nested=True, dpi=64)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Tc3MiEO2twLS" }, "source": [ - "Let's try out the model to see what it predicts before training." + "Try out the model to check what it predicts before training:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "UwvIKLZPtxV_" }, "outputs": [], "source": [ "def create_mask(pred_mask):\n", - " pred_mask = tf.argmax(pred_mask, axis=-1)\n", + " pred_mask = tf.math.argmax(pred_mask, axis=-1)\n", " pred_mask = pred_mask[..., tf.newaxis]\n", " return pred_mask[0]" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YLNsrynNtx4d" }, "outputs": [], @@ -568,10 +540,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "X_1CC0T4dho3" }, "outputs": [], @@ -582,19 +552,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "22AyVYWQdkgk" }, "source": [ - "Let's observe how the model improves while it is training. To accomplish this task, a callback function is defined below. " + "The callback defined below is used to observe how the model improves while it is training:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wHrHsqijdmL6" }, "outputs": [], @@ -608,10 +575,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "StKDH_B9t4SD" }, "outputs": [], @@ -620,19 +585,17 @@ "VAL_SUBSPLITS = 5\n", "VALIDATION_STEPS = info.splits['test'].num_examples//BATCH_SIZE//VAL_SUBSPLITS\n", "\n", - "model_history = model.fit(train_dataset, epochs=EPOCHS,\n", + "model_history = model.fit(train_batches, epochs=EPOCHS,\n", " steps_per_epoch=STEPS_PER_EPOCH,\n", " validation_steps=VALIDATION_STEPS,\n", - " validation_data=test_dataset,\n", + " validation_data=test_batches,\n", " callbacks=[DisplayCallback()])" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "P_mu0SAbt40Q" }, "outputs": [], @@ -640,11 +603,9 @@ "loss = model_history.history['loss']\n", "val_loss = model_history.history['val_loss']\n", "\n", - "epochs = range(EPOCHS)\n", - "\n", "plt.figure()\n", - "plt.plot(epochs, loss, 'r', label='Training loss')\n", - "plt.plot(epochs, val_loss, 'bo', label='Validation loss')\n", + "plt.plot(model_history.epoch, loss, 'r', label='Training loss')\n", + "plt.plot(model_history.epoch, val_loss, 'bo', label='Validation loss')\n", "plt.title('Training and Validation Loss')\n", "plt.xlabel('Epoch')\n", "plt.ylabel('Loss Value')\n", @@ -656,7 +617,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "unP3cnxo_N72" }, "source": [ @@ -666,47 +626,198 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7BVXldSo-0mW" }, "source": [ - "Let's make some predictions. In the interest of saving time, the number of epochs was kept small, but you may set this higher to achieve more accurate results." + "Now, make some predictions. In the interest of saving time, the number of epochs was kept small, but you may set this higher to achieve more accurate results." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ikrzoG24qwf5" }, "outputs": [], "source": [ - "show_predictions(test_dataset, 3)" + "show_predictions(test_batches, 3)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QAwvlgSNoK3o" + }, + "source": [ + "## Optional: Imbalanced classes and class weights" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eqtFPqqu2kxP" + }, + "source": [ + "Semantic segmentation datasets can be highly imbalanced meaning that particular class pixels can be present more inside images than that of other classes. Since segmentation problems can be treated as per-pixel classification problems, you can deal with the imbalance problem by weighing the loss function to account for this. It's a simple and elegant way to deal with this problem. Refer to the [Classification on imbalanced data](../structured_data/imbalanced_data.ipynb) tutorial to learn more.\n", + "\n", + "To [avoid ambiguity](https://github.com/keras-team/keras/issues/3653#issuecomment-243939748), `Model.fit` does not support the `class_weight` argument for targets with 3+ dimensions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aHt90UEQsZDn" + }, + "outputs": [], + "source": [ + "try:\n", + " model_history = model.fit(train_batches, epochs=EPOCHS,\n", + " steps_per_epoch=STEPS_PER_EPOCH,\n", + " class_weight = {0:2.0, 1:2.0, 2:1.0})\n", + " assert False\n", + "except Exception as e:\n", + " print(f\"Expected {type(e).__name__}: {e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "brbhYODCsvbe" + }, + "source": [ + "So, in this case you need to implement the weighting yourself. You'll do this using sample weights: In addition to `(data, label)` pairs, `Model.fit` also accepts `(data, label, sample_weight)` triples.\n", + "\n", + "Keras `Model.fit` propagates the `sample_weight` to the losses and metrics, which also accept a `sample_weight` argument. The sample weight is multiplied by the sample's value before the reduction step. For example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EmHtImJn5Kk-" + }, + "outputs": [], + "source": [ + "label = np.array([0,0])\n", + "prediction = np.array([[-3., 0], [-3, 0]])\n", + "sample_weight = [1, 10]\n", + "\n", + "loss = tf.keras.losses.SparseCategoricalCrossentropy(\n", + " from_logits=True,\n", + " reduction=tf.keras.losses.Reduction.NONE\n", + ")\n", + "loss(label, prediction, sample_weight).numpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Gbwo3DZ-9TxM" + }, + "source": [ + "So, to make sample weights for this tutorial, you need a function that takes a `(data, label)` pair and returns a `(data, label, sample_weight)` triple where the `sample_weight` is a 1-channel image containing the class weight for each pixel.\n", + "\n", + "The simplest possible implementation is to use the label as an index into a `class_weight` list:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DlG-n2Ugo8Jc" + }, + "outputs": [], + "source": [ + "def add_sample_weights(image, label):\n", + " # The weights for each class, with the constraint that:\n", + " # sum(class_weights) == 1.0\n", + " class_weights = tf.constant([2.0, 2.0, 1.0])\n", + " class_weights = class_weights/tf.reduce_sum(class_weights)\n", + "\n", + " # Create an image of `sample_weights` by using the label at each pixel as an\n", + " # index into the `class weights` .\n", + " sample_weights = tf.gather(class_weights, indices=tf.cast(label, tf.int32))\n", + "\n", + " return image, label, sample_weights" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hLH_NvH2UrXU" + }, + "source": [ + "The resulting dataset elements contain 3 images each:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SE_ezRSFRCnE" + }, + "outputs": [], + "source": [ + "train_batches.map(add_sample_weights).element_spec" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Yc-EpIzaRbSL" + }, + "source": [ + "Now, you can train a model on this weighted dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QDWipedAoOQe" + }, + "outputs": [], + "source": [ + "weighted_model = unet_model(OUTPUT_CLASSES)\n", + "weighted_model.compile(\n", + " optimizer='adam',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "btEFKc1xodGR" + }, + "outputs": [], + "source": [ + "weighted_model.fit(\n", + " train_batches.map(add_sample_weights),\n", + " epochs=1,\n", + " steps_per_epoch=10)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "R24tahEqmSCk" }, "source": [ "## Next steps\n", - "Now that you have an understanding of what image segmentation is and how it works, you can try this tutorial out with different intermediate layer outputs, or even different pretrained model. You may also challenge yourself by trying out the [Carvana](https://www.kaggle.com/c/carvana-image-masking-challenge/overview) image masking challenge hosted on Kaggle.\n", "\n", - "You may also want to see the [Tensorflow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) for another model you can retrain on your own data." + "Now that you have an understanding of what image segmentation is and how it works, you can try this tutorial out with different intermediate layer outputs, or even different pretrained models. You may also challenge yourself by trying out the [Carvana](https://www.kaggle.com/c/carvana-image-masking-challenge/overview) image masking challenge hosted on Kaggle.\n", + "\n", + "You may also want to see the [Tensorflow Object Detection API](https://github.com/tensorflow/models/blob/master/research/object_detection/README.md) for another model you can retrain on your own data. Pretrained models are available on [TensorFlow Hub](https://www.tensorflow.org/hub/tutorials/tf2_object_detection#optional)." ] } ], "metadata": { "accelerator": "GPU", "colab": { - "collapsed_sections": [], "name": "segmentation.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/images/transfer_learning.ipynb b/site/en/tutorials/images/transfer_learning.ipynb index e0c78257940..172bb2700b4 100644 --- a/site/en/tutorials/images/transfer_learning.ipynb +++ b/site/en/tutorials/images/transfer_learning.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "77gENRVX40S7" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "d8jyt37T42Vf" }, "outputs": [], @@ -36,11 +33,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "aPxHdjwW5P2j" }, "outputs": [], @@ -71,112 +66,82 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hRTa3Ee15WsJ" }, "source": [ - "# Transfer learning with a pretrained ConvNet" + "# Transfer learning and fine-tuning" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "dQHMcypT3vDT" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/images/transfer_learning\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/images/transfer_learning.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/images/transfer_learning.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/images/transfer_learning.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2X4KyhORdSeO" }, "source": [ - "In this tutorial you will learn how to classify cats vs dogs images by using transfer learning from a pre-trained network.\n", + "In this tutorial, you will learn how to classify images of cats and dogs by using transfer learning from a pre-trained network.\n", "\n", + "A pre-trained model is a saved network that was previously trained on a large dataset, typically on a large-scale image-classification task. You either use the pretrained model as is or use transfer learning to customize this model to a given task.\n", "\n", - "A **pre-trained model** is a saved network that was previously trained on a large dataset, typically on a large-scale image-classification task. You either use the pretrained model as it is, or use **transfer learning** to customize this model to a given task.\n", - "\n", - "The intuition behind transfer learning is that if a model trained on a large and general enough dataset, this model will effectively serve as a generic model of the visual world. You can then take advantage of these learned feature maps without having to start from scratch training a large model on a large dataset.\n", + "The intuition behind transfer learning for image classification is that if a model is trained on a large and general enough dataset, this model will effectively serve as a generic model of the visual world. You can then take advantage of these learned feature maps without having to start from scratch by training a large model on a large dataset.\n", "\n", "In this notebook, you will try two ways to customize a pretrained model:\n", "\n", - "1. **Feature Extraction**: Use the representations learned by a previous network to extract meaningful features from new samples. You simply add a new classifier, which will be trained from scratch, on top of the pretrained model so that you can repurpose the feature maps learned previously for our dataset.\n", + "1. Feature Extraction: Use the representations learned by a previous network to extract meaningful features from new samples. You simply add a new classifier, which will be trained from scratch, on top of the pretrained model so that you can repurpose the feature maps learned previously for the dataset.\n", "\n", - " You do not need to (re)train the entire model. The base convolutional network already contains features that are generically useful for classifying pictures. However, the final, classification part of the pretrained model is specific to original classification task, and subsequently specific to the set of classes on which the model was trained.\n", + " You do not need to (re)train the entire model. The base convolutional network already contains features that are generically useful for classifying pictures. However, the final, classification part of the pretrained model is specific to the original classification task, and subsequently specific to the set of classes on which the model was trained.\n", "\n", - "2. **Fine-Tuning**: Unfreezing a few of the top layers of a frozen model base and jointly training both the newly-added classifier layers and the last layers of the base model. This allows us to \"fine tune\" the higher-order feature representations in the base model in order to make them more relevant for the specific task.\n", + "1. Fine-Tuning: Unfreeze a few of the top layers of a frozen model base and jointly train both the newly-added classifier layers and the last layers of the base model. This allows us to \"fine-tune\" the higher-order feature representations in the base model in order to make them more relevant for the specific task.\n", "\n", "You will follow the general machine learning workflow.\n", "\n", "1. Examine and understand the data\n", - "2. Build an input pipeline, in this case using Keras `ImageDataGenerator`\n", - "3. Compose our model\n", - " * Load in our pretrained base model (and pretrained weights)\n", - " * Stack our classification layers on top\n", - "4. Train our model\n", - "5. Evaluate model\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iBMcobPHdD8O" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "\n", - "import numpy as np\n", - "\n", - "import matplotlib.pyplot as plt" + "1. Build an input pipeline, in this case using Keras ImageDataGenerator\n", + "1. Compose the model\n", + " * Load in the pretrained base model (and pretrained weights)\n", + " * Stack the classification layers on top\n", + "1. Train the model\n", + "1. Evaluate model\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "TqOt6Sv7AsMi" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "keras = tf.keras" + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import os\n", + "import tensorflow as tf" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "v77rlkCKW0IJ" }, "source": [ @@ -186,7 +151,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0GoKGm1duzgk" }, "source": [ @@ -196,265 +160,288 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vHP9qMJxt2oz" }, "source": [ - "Use [TensorFlow Datasets](http://tensorflow.org/datasets) to load the cats and dogs dataset.\n", + "In this tutorial, you will use a dataset containing several thousand images of cats and dogs. Download and extract a zip file containing the images, then create a `tf.data.Dataset` for training and validation using the `tf.keras.utils.image_dataset_from_directory` utility. You can learn more about loading images in this [tutorial](https://www.tensorflow.org/tutorials/load_data/images)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ro4oYaEmxe4r" + }, + "outputs": [], + "source": [ + "_URL = 'https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip'\n", + "path_to_zip = tf.keras.utils.get_file('cats_and_dogs.zip', origin=_URL, extract=True)\n", + "PATH = os.path.join(os.path.dirname(path_to_zip), 'cats_and_dogs_filtered')\n", + "\n", + "train_dir = os.path.join(PATH, 'train')\n", + "validation_dir = os.path.join(PATH, 'validation')\n", "\n", - "This `tfds` package is the easiest way to load pre-defined data. If you have your own data, and are interested in importing using it with TensorFlow see [loading image data](../load_data/images.ipynb)\n" + "BATCH_SIZE = 32\n", + "IMG_SIZE = (160, 160)\n", + "\n", + "train_dataset = tf.keras.utils.image_dataset_from_directory(train_dir,\n", + " shuffle=True,\n", + " batch_size=BATCH_SIZE,\n", + " image_size=IMG_SIZE)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KVh7rDVAuW8Y" + "id": "cAvtLwi7_J__" }, "outputs": [], "source": [ - "import tensorflow_datasets as tfds\n", - "tfds.disable_progress_bar()" + "validation_dataset = tf.keras.utils.image_dataset_from_directory(validation_dir,\n", + " shuffle=True,\n", + " batch_size=BATCH_SIZE,\n", + " image_size=IMG_SIZE)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Nsoic6bGuwQ-" + "id": "yO1Q2JaW5sIy" }, "source": [ - "The `tfds.load` method downloads and caches the data, and returns a `tf.data.Dataset` object. These objects provide powerful, efficient methods for manipulating data and piping it into your model.\n", - "\n", - "Since `\"cats_vs_dog\"` doesn't define standard splits, use the subsplit feature to divide it into (train, validation, test) with 80%, 10%, 10% of the data respectively." + "Show the first nine images and labels from the training set:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ro4oYaEmxe4r" + "id": "K5BeQyKThC_Y" }, "outputs": [], "source": [ - "SPLIT_WEIGHTS = (8, 1, 1)\n", - "splits = tfds.Split.TRAIN.subsplit(weighted=SPLIT_WEIGHTS)\n", + "class_names = train_dataset.class_names\n", "\n", - "(raw_train, raw_validation, raw_test), metadata = tfds.load(\n", - " 'cats_vs_dogs', split=list(splits),\n", - " with_info=True, as_supervised=True)" + "plt.figure(figsize=(10, 10))\n", + "for images, labels in train_dataset.take(1):\n", + " for i in range(9):\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " plt.imshow(images[i].numpy().astype(\"uint8\"))\n", + " plt.title(class_names[labels[i]])\n", + " plt.axis(\"off\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "o29EfE-p0g5X" + "id": "EZqCX_mpV3Mx" }, "source": [ - "The resulting `tf.data.Dataset` objects contain `(image, label)` pairs. Where the images have variable shape and 3 channels, and the label is a scalar." + "As the original dataset doesn't contain a test set, you will create one. To do so, determine how many batches of data are available in the validation set using `tf.data.experimental.cardinality`, then move 20% of them to a test set." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GIys1_zY1S9b" + "id": "uFFIYrTFV9RO" }, "outputs": [], "source": [ - "print(raw_train)\n", - "print(raw_validation)\n", - "print(raw_test)" + "val_batches = tf.data.experimental.cardinality(validation_dataset)\n", + "test_dataset = validation_dataset.take(val_batches // 5)\n", + "validation_dataset = validation_dataset.skip(val_batches // 5)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "yO1Q2JaW5sIy" + "id": "Q9pFlFWgBKgH" }, + "outputs": [], "source": [ - "Show the first two images and labels from the training set:" + "print('Number of validation batches: %d' % tf.data.experimental.cardinality(validation_dataset))\n", + "print('Number of test batches: %d' % tf.data.experimental.cardinality(test_dataset))" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "K5BeQyKThC_Y" + "id": "MakSrdd--RKg" }, - "outputs": [], "source": [ - "get_label_name = metadata.features['label'].int2str\n", - "\n", - "for image, label in raw_train.take(2):\n", - " plt.figure()\n", - " plt.imshow(image)\n", - " plt.title(get_label_name(label))" + "### Configure the dataset for performance" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "wvidPx6jeFzf" + "id": "22XWC7yjkZu4" }, "source": [ - "### Format the Data\n", - "\n", - "Use the `tf.image` module to format the images for the task.\n", - "\n", - "Resize the images to a fixes input size, and rescale the input channels to a range of `[-1,1]`\n", - "\n", - "\u003c!-- TODO(markdaoust): fix the keras_applications preprocessing functions to work in tf2 --\u003e" + "Use buffered prefetching to load images from disk without having I/O become blocking. To learn more about this method see the [data performance](https://www.tensorflow.org/guide/data_performance) guide." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "y3PM6GVHcC31" + "id": "p3UUPdm86LNC" }, "outputs": [], "source": [ - "IMG_SIZE = 160 # All images will be resized to 160x160\n", + "AUTOTUNE = tf.data.AUTOTUNE\n", "\n", - "def format_example(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image = (image/127.5) - 1\n", - " image = tf.image.resize(image, (IMG_SIZE, IMG_SIZE))\n", - " return image, label" + "train_dataset = train_dataset.prefetch(buffer_size=AUTOTUNE)\n", + "validation_dataset = validation_dataset.prefetch(buffer_size=AUTOTUNE)\n", + "test_dataset = test_dataset.prefetch(buffer_size=AUTOTUNE)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "i2MRh_AeBtOM" + "id": "MYfcVwYLiR98" }, "source": [ - "Apply this function to each item in the dataset using the map method:" + "### Use data augmentation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bDWc5Oad1daX" + }, + "source": [ + "When you don't have a large image dataset, it's a good practice to artificially introduce sample diversity by applying random, yet realistic, transformations to the training images, such as rotation and horizontal flipping. This helps expose the model to different aspects of the training data and reduce [overfitting](https://www.tensorflow.org/tutorials/keras/overfit_and_underfit). You can learn more about data augmentation in this [tutorial](https://www.tensorflow.org/tutorials/images/data_augmentation)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SFZ6ZW7KSXP9" + "id": "3P99QiMGit1A" }, "outputs": [], "source": [ - "train = raw_train.map(format_example)\n", - "validation = raw_validation.map(format_example)\n", - "test = raw_test.map(format_example)" + "data_augmentation = tf.keras.Sequential([\n", + " tf.keras.layers.RandomFlip('horizontal'),\n", + " tf.keras.layers.RandomRotation(0.2),\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s9SlcbhrarOO" + }, + "source": [ + "Note: These layers are active only during training, when you call `Model.fit`. They are inactive when the model is used in inference mode in `Model.evaluate`, `Model.predict`, or `Model.call`." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "E5ifgXDuBfOC" + "id": "9mD3rE2Lm7-d" }, "source": [ - "Now shuffle and batch the data." + "Let's repeatedly apply these layers to the same image and see the result." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Yic-I66m6Isv" + "id": "aQullOUHkm67" }, "outputs": [], "source": [ - "BATCH_SIZE = 32\n", - "SHUFFLE_BUFFER_SIZE = 1000" + "for image, _ in train_dataset.take(1):\n", + " plt.figure(figsize=(10, 10))\n", + " first_image = image[0]\n", + " for i in range(9):\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " augmented_image = data_augmentation(tf.expand_dims(first_image, 0))\n", + " plt.imshow(augmented_image[0] / 255)\n", + " plt.axis('off')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bAywKtuVn8uK" + }, + "source": [ + "### Rescale pixel values\n", + "\n", + "In a moment, you will download `tf.keras.applications.MobileNetV2` for use as your base model. This model expects pixel values in `[-1, 1]`, but at this point, the pixel values in your images are in `[0, 255]`. To rescale them, use the preprocessing method included with the model." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "p3UUPdm86LNC" + "id": "cO0HM9JAQUFq" }, "outputs": [], "source": [ - "train_batches = train.shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE)\n", - "validation_batches = validation.batch(BATCH_SIZE)\n", - "test_batches = test.batch(BATCH_SIZE)" + "preprocess_input = tf.keras.applications.mobilenet_v2.preprocess_input" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "02rJpcFtChP0" + "id": "xnr81qRMzcs5" }, "source": [ - "Inspect a batch of data:" + "Note: Alternatively, you could rescale pixel values from `[0, 255]` to `[-1, 1]` using `tf.keras.layers.Rescaling`." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iknFo3ELBVho" + "id": "R2NyJn4KQMux" }, "outputs": [], "source": [ - "for image_batch, label_batch in train_batches.take(1):\n", - " pass\n", - "\n", - "image_batch.shape" + "rescale = tf.keras.layers.Rescaling(1./127.5, offset=-1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wz7qgImhTxw4" + }, + "source": [ + "Note: If using other `tf.keras.applications`, be sure to check the API doc to determine if they expect pixels in `[-1, 1]` or `[0, 1]`, or use the included `preprocess_input` function." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OkH-kazQecHB" }, "source": [ "## Create the base model from the pre-trained convnets\n", - "You will create the base model from the **MobileNet V2** model developed at Google. This is pre-trained on the ImageNet dataset, a large dataset of 1.4M images and 1000 classes of web images. ImageNet has a fairly arbitrary research training dataset with categories like `jackfruit` and `syringe`, but this base of knowledge will help us tell apart cats and dogs from our specific dataset.\n", + "You will create the base model from the **MobileNet V2** model developed at Google. This is pre-trained on the ImageNet dataset, a large dataset consisting of 1.4M images and 1000 classes. ImageNet is a research training dataset with a wide variety of categories like `jackfruit` and `syringe`. This base of knowledge will help us classify cats and dogs from our specific dataset.\n", "\n", - "First, you need to pick which layer of MobileNet V2 you will use for feature extraction. Obviously, the very last classification layer (on \"top\", as most diagrams of machine learning models go from bottom to top) is not very useful. Instead, you will follow the common practice to instead depend on the very last layer before the flatten operation. This layer is called the \"bottleneck layer\". The bottleneck features retain much generality as compared to the final/top layer.\n", + "First, you need to pick which layer of MobileNet V2 you will use for feature extraction. The very last classification layer (on \"top\", as most diagrams of machine learning models go from bottom to top) is not very useful. Instead, you will follow the common practice to depend on the very last layer before the flatten operation. This layer is called the \"bottleneck layer\". The bottleneck layer features retain more generality as compared to the final/top layer.\n", "\n", "First, instantiate a MobileNet V2 model pre-loaded with weights trained on ImageNet. By specifying the **include_top=False** argument, you load a network that doesn't include the classification layers at the top, which is ideal for feature extraction." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "19IQ2gqneqmS" }, "outputs": [], "source": [ - "IMG_SHAPE = (IMG_SIZE, IMG_SIZE, 3)\n", - "\n", "# Create the base model from the pre-trained model MobileNet V2\n", + "IMG_SHAPE = IMG_SIZE + (3,)\n", "base_model = tf.keras.applications.MobileNetV2(input_shape=IMG_SHAPE,\n", " include_top=False,\n", " weights='imagenet')" @@ -463,23 +450,21 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AqcsxoJIEVXZ" }, "source": [ - "This feature extractor converts each `160x160x3` image to a `5x5x1280` block of features. See what it does to the example batch of images:" + "This feature extractor converts each `160x160x3` image into a `5x5x1280` block of features. Let's see what it does to an example batch of images:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Y-2LJL0EEUcx" }, "outputs": [], "source": [ + "image_batch, label_batch = next(iter(train_dataset))\n", "feature_batch = base_model(image_batch)\n", "print(feature_batch.shape)" ] @@ -487,31 +472,35 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rlx56nQtfe8Y" }, "source": [ "## Feature extraction\n", - "You will freeze the convolutional base created from the previous step and use that as a feature extractor, add a classifier on top of it and train the top-level classifier." + "In this step, you will freeze the convolutional base created from the previous step and to use as a feature extractor. Additionally, you add a classifier on top of it and train the top-level classifier." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CnMLieHBCwil" }, "source": [ - "### Freeze the convolutional base\n", - "It's important to freeze the convolutional based before you compile and train the model. By freezing (or setting `layer.trainable = False`), you prevent the weights in a given layer from being updated during training. MobileNet V2 has many layers, so setting the entire model's trainable flag to `False` will freeze all the layers." + "### Freeze the convolutional base" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7fL6upiN3ekS" + }, + "source": [ + "It is important to freeze the convolutional base before you compile and train the model. Freezing (by setting layer.trainable = False) prevents the weights in a given layer from being updated during training. MobileNet V2 has many layers, so setting the entire model's `trainable` flag to False will freeze all of them." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "OTCJH4bphOeo" }, "outputs": [], @@ -519,12 +508,27 @@ "base_model.trainable = False" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "jsNHwpm7BeVM" + }, + "source": [ + "### Important note about BatchNormalization layers\n", + "\n", + "Many models contain `tf.keras.layers.BatchNormalization` layers. This layer is a special case and precautions should be taken in the context of fine-tuning, as shown later in this tutorial.\n", + "\n", + "When you set `layer.trainable = False`, the `BatchNormalization` layer will run in inference mode, and will not update its mean and variance statistics.\n", + "\n", + "When you unfreeze a model that contains BatchNormalization layers in order to do fine-tuning, you should keep the BatchNormalization layers in inference mode by passing `training = False` when calling the base model. Otherwise, the updates applied to the non-trainable weights will destroy what the model has learned.\n", + "\n", + "For more details, see the [Transfer learning guide](https://www.tensorflow.org/guide/keras/transfer_learning)." + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "KpbzSmPkDa-N" }, "outputs": [], @@ -536,7 +540,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wdMRM8YModbk" }, "source": [ @@ -546,7 +549,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QBc31c4tMOdH" }, "source": [ @@ -555,10 +557,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dLnpMF5KOALm" }, "outputs": [], @@ -571,24 +571,21 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "O1p0OJBR6dOT" }, "source": [ - "Apply a `tf.keras.layers.Dense` layer to convert these features into a single prediction per image. You don't need an activation function here because this prediction will be treated as a `logit`, or a raw prediction value. Positive numbers predict class 1, negative numbers predict class 0." + "Apply a `tf.keras.layers.Dense` layer to convert these features into a single prediction per image. You don't need an activation function here because this prediction will be treated as a `logit`, or a raw prediction value. Positive numbers predict class 1, negative numbers predict class 0." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Wv4afXKj6cVa" }, "outputs": [], "source": [ - "prediction_layer = keras.layers.Dense(1)\n", + "prediction_layer = tf.keras.layers.Dense(1, activation='sigmoid')\n", "prediction_batch = prediction_layer(feature_batch_average)\n", "print(prediction_batch.shape)" ] @@ -596,147 +593,125 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "0iqnBeZrfoIc" + "id": "HXvz-ZkTa9b3" }, "source": [ - "Now stack the feature extractor, and these two layers using a `tf.keras.Sequential` model:" + "Build a model by chaining together the data augmentation, rescaling, `base_model` and feature extractor layers using the [Keras Functional API](https://www.tensorflow.org/guide/keras/functional). As previously mentioned, use `training=False` as our model contains a `BatchNormalization` layer." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eApvroIyn1K0" + "id": "DgzQX6Veb2WT" }, "outputs": [], "source": [ - "model = tf.keras.Sequential([\n", - " base_model,\n", - " global_average_layer,\n", - " prediction_layer\n", - "])" + "inputs = tf.keras.Input(shape=(160, 160, 3))\n", + "x = data_augmentation(inputs)\n", + "x = preprocess_input(x)\n", + "x = base_model(x, training=False)\n", + "x = global_average_layer(x)\n", + "x = tf.keras.layers.Dropout(0.2)(x)\n", + "outputs = prediction_layer(x)\n", + "model = tf.keras.Model(inputs, outputs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "I8ARiyMFsgbH" + }, + "outputs": [], + "source": [ + "model.summary()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "g0ylJXE_kRLi" + "id": "lxOcmVr0ydFZ" }, "source": [ - "### Compile the model\n", - "\n", - "You must compile the model before training it. Since there are two classes, use a binary cross-entropy loss." + "The 8+ million parameters in MobileNet are frozen, but there are 1.2 thousand _trainable_ parameters in the Dense layer. These are divided between two `tf.Variable` objects, the weights and biases." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RpR8HdyMhukJ" + "id": "krvBumovycVA" }, "outputs": [], "source": [ - "base_learning_rate = 0.0001\n", - "model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=base_learning_rate),\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" + "len(model.trainable_variables)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "I8ARiyMFsgbH" + "id": "jeGk93R2ahav" }, "outputs": [], "source": [ - "model.summary()" + "tf.keras.utils.plot_model(model, show_shapes=True)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "lxOcmVr0ydFZ" + "id": "g0ylJXE_kRLi" }, "source": [ - "The 2.5M parameters in MobileNet are frozen, but there are 1.2K _trainable_ parameters in the Dense layer. These are divided between two `tf.Variable` objects, the weights and biases." + "### Compile the model\n", + "\n", + "Compile the model before training it. Since there are two classes and a sigmoid oputput, use the `BinaryAccuracy`." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "krvBumovycVA" + "id": "RpR8HdyMhukJ" }, "outputs": [], "source": [ - "len(model.trainable_variables)" + "base_learning_rate = 0.0001\n", + "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),\n", + " loss=tf.keras.losses.BinaryCrossentropy(),\n", + " metrics=[tf.keras.metrics.BinaryAccuracy(threshold=0.5, name='accuracy')])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RxvgOYTDSWTx" }, "source": [ "### Train the model\n", "\n", - "After training for 10 epochs, you should see ~96% accuracy.\n", - "\n", - "\u003c!-- TODO(markdaoust): delete steps_per_epoch in TensorFlow r1.14/r2.0 --\u003e" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hlHEavK7DUI7" - }, - "outputs": [], - "source": [ - "num_train, num_val, num_test = (\n", - " metadata.splits['train'].num_examples*weight/10\n", - " for weight in SPLIT_WEIGHTS\n", - ")" + "After training for 10 epochs, you should see ~96% accuracy on the validation set.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Om4O3EESkab1" }, "outputs": [], "source": [ "initial_epochs = 10\n", - "steps_per_epoch = round(num_train)//BATCH_SIZE\n", - "validation_steps = 20\n", "\n", - "loss0,accuracy0 = model.evaluate(validation_batches, steps = validation_steps)" + "loss0, accuracy0 = model.evaluate(validation_dataset)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8cYT1c48CuSd" }, "outputs": [], @@ -747,37 +722,32 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "JsaRFlZ9B6WK" }, "outputs": [], "source": [ - "history = model.fit(train_batches,\n", + "history = model.fit(train_dataset,\n", " epochs=initial_epochs,\n", - " validation_data=validation_batches)" + " validation_data=validation_dataset)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Hd94CKImf8vi" }, "source": [ "### Learning curves\n", "\n", - "Let's take a look at the learning curves of the training and validation accuracy/loss when using the MobileNet V2 base model as a fixed feature extractor." + "Let's take a look at the learning curves of the training and validation accuracy/loss when using the MobileNetV2 base model as a fixed feature extractor." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "53OTCh3jnbwV" }, "outputs": [], @@ -811,7 +781,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "foWMyyUHbc1j" }, "source": [ @@ -823,24 +792,22 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CqwV-CRdS6Nv" }, "source": [ "## Fine tuning\n", - "In our feature extraction experiment, you were only training a few layers on top of an MobileNet V2 base model. The weights of the pre-trained network were **not** updated during training.\n", + "In the feature extraction experiment, you were only training a few layers on top of an MobileNetV2 base model. The weights of the pre-trained network were **not** updated during training.\n", "\n", - "One way to increase performance even further is to train (or \"fine-tune\") the weights of the top layers of the pre-trained model alongside the training of the classifier you added. The training process will force the weights to be tuned from generic features maps to features associated specifically to our dataset.\n", + "One way to increase performance even further is to train (or \"fine-tune\") the weights of the top layers of the pre-trained model alongside the training of the classifier you added. The training process will force the weights to be tuned from generic feature maps to features associated specifically with the dataset.\n", "\n", "Note: This should only be attempted after you have trained the top-level classifier with the pre-trained model set to non-trainable. If you add a randomly initialized classifier on top of a pre-trained model and attempt to train all layers jointly, the magnitude of the gradient updates will be too large (due to the random weights from the classifier) and your pre-trained model will forget what it has learned.\n", "\n", - "Also, you should try to fine-tune a small number of top layers rather than the whole MobileNet model. In most convolutional networks, the higher up a layer is, the more specialized it is. The first few layers learn very simple and generic features which generalize to almost all types of images. As you go higher up, the features are increasingly more specific to the dataset on which the model was trained. The goal of fine-tuning is to adapt these specialized features to work with the new dataset, rather than overwrite the generic learning." + "Also, you should try to fine-tune a small number of top layers rather than the whole MobileNet model. In most convolutional networks, the higher up a layer is, the more specialized it is. The first few layers learn very simple and generic features that generalize to almost all types of images. As you go higher up, the features are increasingly more specific to the dataset on which the model was trained. The goal of fine-tuning is to adapt these specialized features to work with the new dataset, rather than overwrite the generic learning." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CPXnzUK0QonF" }, "source": [ @@ -850,19 +817,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rfxv_ifotQak" }, "source": [ - "All you need to do is unfreeze the `base_model` and set the bottom layers be un-trainable. Then, you should recompile the model (necessary for these changes to take effect), and resume training." + "All you need to do is unfreeze the `base_model` and set the bottom layers to be un-trainable. Then, you should recompile the model (necessary for these changes to take effect), and resume training." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4nzcagVitLQm" }, "outputs": [], @@ -872,10 +836,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "-4HgVAacRs5v" }, "outputs": [], @@ -883,47 +845,42 @@ "# Let's take a look to see how many layers are in the base model\n", "print(\"Number of layers in the base model: \", len(base_model.layers))\n", "\n", - "# Fine tune from this layer onwards\n", + "# Fine-tune from this layer onwards\n", "fine_tune_at = 100\n", "\n", "# Freeze all the layers before the `fine_tune_at` layer\n", "for layer in base_model.layers[:fine_tune_at]:\n", - " layer.trainable = False" + " layer.trainable = False" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4Uk1dgsxT0IS" }, "source": [ "### Compile the model\n", "\n", - "Compile the model using a much lower training rate." + "As you are training a much larger model and want to readapt the pretrained weights, it is important to use a lower learning rate at this stage. Otherwise, your model could overfit very quickly." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "NtUnaz0WUDva" }, "outputs": [], "source": [ - "model.compile(loss='binary_crossentropy',\n", - " optimizer = tf.keras.optimizers.RMSprop(lr=base_learning_rate/10),\n", - " metrics=['accuracy'])" + "model.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n", + " optimizer = tf.keras.optimizers.RMSprop(learning_rate=base_learning_rate/10),\n", + " metrics=[tf.keras.metrics.BinaryAccuracy(threshold=0.5, name='accuracy')])" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WwBWy7J2kZvA" }, "outputs": [], @@ -933,10 +890,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "bNXelbMQtonr" }, "outputs": [], @@ -947,29 +902,25 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4G5O4jd6TuAG" }, "source": [ - "### Continue Train the model" + "### Continue training the model" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0foWUN-yDLo_" }, "source": [ - "If you trained to convergence earlier, this will get you a few percent more accuracy." + "If you trained to convergence earlier, this step will improve your accuracy by a few percentage points." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ECQLkAsFTlun" }, "outputs": [], @@ -977,40 +928,36 @@ "fine_tune_epochs = 10\n", "total_epochs = initial_epochs + fine_tune_epochs\n", "\n", - "history_fine = model.fit(train_batches,\n", + "history_fine = model.fit(train_dataset,\n", " epochs=total_epochs,\n", - " initial_epoch = history.epoch[-1],\n", - " validation_data=validation_batches)" + " initial_epoch=len(history.epoch),\n", + " validation_data=validation_dataset)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TfXEmsxQf6eP" }, "source": [ - "Let's take a look at the learning curves of the training and validation accuracy/loss, when fine tuning the last few layers of the MobileNet V2 base model and training the classifier on top of it. The validation loss is much higher than the training loss, so you may get some overfitting.\n", + "Let's take a look at the learning curves of the training and validation accuracy/loss when fine-tuning the last few layers of the MobileNetV2 base model and training the classifier on top of it. The validation loss is much higher than the training loss, so you may get some overfitting.\n", "\n", - "You may also get some overfitting as the new training set is relatively small and similar to the original MobileNet V2 datasets.\n" + "You may also get some overfitting as the new training set is relatively small and similar to the original MobileNetV2 datasets.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DNtfNZKlInGT" }, "source": [ - "After fine tuning the model nearly reaches 98% accuracy." + "After fine tuning the model nearly reaches 98% accuracy on the validation set." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "PpA8PlpQKygw" }, "outputs": [], @@ -1024,10 +971,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "chW103JUItdk" }, "outputs": [], @@ -1057,27 +1002,88 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "R6cWgjgfrsn5" + }, + "source": [ + "### Evaluation and prediction" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PSXH7PRMxOi5" + }, + "source": [ + "Finally you can verify the performance of the model on new data using test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2KyNhagHwfar" + }, + "outputs": [], + "source": [ + "loss, accuracy = model.evaluate(test_dataset)\n", + "print('Test accuracy :', accuracy)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8UjS5ukZfOcR" + }, + "source": [ + "And now you are all set to use this model to predict if your pet is a cat or dog." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RUNoQNgtfNgt" + }, + "outputs": [], + "source": [ + "# Retrieve a batch of images from the test set\n", + "image_batch, label_batch = test_dataset.as_numpy_iterator().next()\n", + "predictions = model.predict_on_batch(image_batch).flatten()\n", + "predictions = tf.where(predictions < 0.5, 0, 1)\n", + "\n", + "print('Predictions:\\n', predictions.numpy())\n", + "print('Labels:\\n', label_batch)\n", + "\n", + "plt.figure(figsize=(10, 10))\n", + "for i in range(9):\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " plt.imshow(image_batch[i].astype(\"uint8\"))\n", + " plt.title(class_names[predictions[i]])\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "_TZTwG7nhm0C" }, "source": [ - "## Summary:\n", + "## Summary\n", "\n", - "* **Using a pre-trained model for feature extraction**: When working with a small dataset, it is common to take advantage of features learned by a model trained on a larger dataset in the same domain. This is done by instantiating the pre-trained model and adding a fully-connected classifier on top. The pre-trained model is \"frozen\" and only the weights of the classifier get updated during training.\n", + "* **Using a pre-trained model for feature extraction**: When working with a small dataset, it is a common practice to take advantage of features learned by a model trained on a larger dataset in the same domain. This is done by instantiating the pre-trained model and adding a fully-connected classifier on top. The pre-trained model is \"frozen\" and only the weights of the classifier get updated during training.\n", "In this case, the convolutional base extracted all the features associated with each image and you just trained a classifier that determines the image class given that set of extracted features.\n", "\n", "* **Fine-tuning a pre-trained model**: To further improve performance, one might want to repurpose the top-level layers of the pre-trained models to the new dataset via fine-tuning.\n", - "In this case, you tuned your weights such that your model learned high-level features specific to the dataset. This technique is usually recommended when the training dataset is large and very similar to the original dataset that the pre-trained model was trained on.\n" + "In this case, you tuned your weights such that your model learned high-level features specific to the dataset. This technique is usually recommended when the training dataset is large and very similar to the original dataset that the pre-trained model was trained on.\n", + "\n", + "To learn more, visit the [Transfer learning guide](https://www.tensorflow.org/guide/keras/transfer_learning).\n" ] } ], "metadata": { "accelerator": "GPU", "colab": { - "collapsed_sections": [], "name": "transfer_learning.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/images/transfer_learning_with_hub.ipynb b/site/en/tutorials/images/transfer_learning_with_hub.ipynb index a60fc87902a..58ebfc2d20f 100644 --- a/site/en/tutorials/images/transfer_learning_with_hub.ipynb +++ b/site/en/tutorials/images/transfer_learning_with_hub.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "W_tvPdyfA-BL" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "0O_LFhwSBCjm" }, "outputs": [], @@ -37,46 +34,48 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PWUmcKKjtwXL" }, "source": [ "# Transfer learning with TensorFlow Hub\n", "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/images/transfer_learning_with_hub\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/images/transfer_learning_with_hub.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/images/transfer_learning_with_hub.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/images/transfer_learning_with_hub.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View on GitHub\n", + " \n", + " Download notebook\n", + " \n", + " See TF Hub model\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "crU-iluJIEzw" }, "source": [ - "[TensorFlow Hub](http://tensorflow.org/hub) is a way to share pretrained model components. See the [TensorFlow Module Hub](https://tfhub.dev/) for a searchable listing of pre-trained models. This tutorial demonstrates:\n", + "[TensorFlow Hub](https://tfhub.dev/) is a repository of pre-trained TensorFlow models.\n", "\n", - "1. How to use TensorFlow Hub with `tf.keras`.\n", - "1. How to do image classification using TensorFlow Hub.\n", - "1. How to do simple transfer learning." + "This tutorial demonstrates how to:\n", + "\n", + "1. Use models from TensorFlow Hub with `tf.keras`.\n", + "1. Use an image classification model from TensorFlow Hub.\n", + "1. Do simple transfer learning to fine-tune a model for your own image classes." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CKFUvuEho9Th" }, "source": [ @@ -85,84 +84,66 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "OGNpmn43C0O6" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", + "import numpy as np\n", + "import time\n", "\n", + "import PIL.Image as Image\n", "import matplotlib.pylab as plt\n", "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " !pip install tf-nightly\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "s4Z9vFE1IQ2Q" - }, - "outputs": [], - "source": [ - "!pip install -U tf-hub-nightly\n", + "import tensorflow as tf\n", "import tensorflow_hub as hub\n", "\n", - "from tensorflow.keras import layers" + "import datetime\n", + "\n", + "%load_ext tensorboard" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "s4YuF5HvpM1W" }, "source": [ - "## An ImageNet classifier" + "## An ImageNet classifier\n", + "\n", + "You'll start by using a classifier model pre-trained on the [ImageNet](https://en.wikipedia.org/wiki/ImageNet) benchmark dataset—no initial training required!" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xEY_Ow5loN6q" }, "source": [ "### Download the classifier\n", "\n", - "Use `hub.module` to load a mobilenet, and `tf.keras.layers.Lambda` to wrap it up as a keras layer. Any [TensorFlow 2 compatible image classifier URL](https://tfhub.dev/s?q=tf2\u0026module-type=image-classification) from tfhub.dev will work here." + "Select a MobileNetV2 pre-trained model [from TensorFlow Hub](https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/2) and wrap it as a Keras layer with [`hub.KerasLayer`](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer). Any compatible image classifier model from TensorFlow Hub will work here, including the examples provided in the drop-down below." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "cellView": "both", - "colab": {}, - "colab_type": "code", "id": "feiXojVXAbI9" }, "outputs": [], "source": [ - "classifier_url =\"https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/2\" #@param {type:\"string\"}" + "mobilenet_v2 =\"https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/4\"\n", + "inception_v3 = \"https://tfhub.dev/google/imagenet/inception_v3/classification/5\"\n", + "\n", + "classifier_model = mobilenet_v2 #@param [\"mobilenet_v2\", \"inception_v3\"] {type:\"raw\"}" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "y_6bGjoPtzau" }, "outputs": [], @@ -170,14 +151,13 @@ "IMAGE_SHAPE = (224, 224)\n", "\n", "classifier = tf.keras.Sequential([\n", - " hub.KerasLayer(classifier_url, input_shape=IMAGE_SHAPE+(3,))\n", + " hub.KerasLayer(classifier_model, input_shape=IMAGE_SHAPE+(3,))\n", "])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pwZXaoV0uXp2" }, "source": [ @@ -187,26 +167,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TQItP1i55-di" }, "source": [ - "Download a single image to try the model on." + "Download a single image to try the model on:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "w5wDjXNjuXGD" }, "outputs": [], "source": [ - "import numpy as np\n", - "import PIL.Image as Image\n", - "\n", "grace_hopper = tf.keras.utils.get_file('image.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg')\n", "grace_hopper = Image.open(grace_hopper).resize(IMAGE_SHAPE)\n", "grace_hopper" @@ -214,10 +188,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "BEmmBnGbLxPp" }, "outputs": [], @@ -229,19 +201,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0Ic8OEEo2b73" }, "source": [ - "Add a batch dimension, and pass the image to the model." + "Add a batch dimension (with `np.newaxis`) and pass the image to the model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "EMquyn29v8q3" }, "outputs": [], @@ -253,48 +222,41 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "NKzjqENF6jDF" }, "source": [ - "The result is a 1001 element vector of logits, rating the probability of each class for the image.\n", + "The result is a 1001-element vector of logits, rating the probability of each class for the image.\n", "\n", - "So the top class ID can be found with argmax:" + "The top class ID can be found with `tf.math.argmax`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "rgXb44vt6goJ" }, "outputs": [], "source": [ - "predicted_class = np.argmax(result[0], axis=-1)\n", + "predicted_class = tf.math.argmax(result[0], axis=-1)\n", "predicted_class" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YrxLMajMoxkf" }, "source": [ "### Decode the predictions\n", "\n", - "We have the predicted class ID,\n", - "Fetch the `ImageNet` labels, and decode the predictions" + "Take the `predicted_class` ID (such as `653`) and fetch the ImageNet dataset labels to decode the predictions:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ij6SrDxcxzry" }, "outputs": [], @@ -305,10 +267,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "uzziRK3Z2VQo" }, "outputs": [], @@ -322,7 +282,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "amfzqn1Oo7Om" }, "source": [ @@ -332,98 +291,180 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "K-nIpVJ94xrw" }, "source": [ - "Using TF Hub it is simple to retrain the top layer of the model to recognize the classes in our dataset." + "But what if you want to create a custom classifier using your own dataset that has classes that aren't included in the original ImageNet dataset (that the pre-trained model was trained on)?\n", + "\n", + "To do that, you can:\n", + "\n", + "1. Select a pre-trained model from TensorFlow Hub; and\n", + "2. Retrain the top (last) layer to recognize the classes from your custom dataset." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Z93vvAdGxDMD" }, "source": [ "### Dataset\n", "\n", - " For this example you will use the TensorFlow flowers dataset:" + "In this example, you will use the TensorFlow flowers dataset:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "DrIUV3V0xDL_" }, "outputs": [], "source": [ - "data_root = tf.keras.utils.get_file(\n", - " 'flower_photos','https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", - " untar=True)" + "import pathlib\n", + "\n", + "data_file = tf.keras.utils.get_file(\n", + " 'flower_photos.tgz',\n", + " 'https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", + " cache_dir='.',\n", + " extract=True)\n", + "\n", + "data_root = pathlib.Path(data_file).with_suffix('')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jFHdp18ccah7" }, "source": [ - "The simplest way to load this data into our model is using `tf.keras.preprocessing.image.ImageDataGenerator`,\n", + "First, load this data into the model using the image data off disk with `tf.keras.utils.image_dataset_from_directory`, which will generate a `tf.data.Dataset`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mqnsczfLgcwv" + }, + "outputs": [], + "source": [ + "batch_size = 32\n", + "img_height = 224\n", + "img_width = 224\n", "\n", - "All of TensorFlow Hub's image modules expect float inputs in the `[0, 1]` range. Use the `ImageDataGenerator`'s `rescale` parameter to achieve this.\n", + "train_ds = tf.keras.utils.image_dataset_from_directory(\n", + " str(data_root),\n", + " validation_split=0.2,\n", + " subset=\"training\",\n", + " seed=123,\n", + " image_size=(img_height, img_width),\n", + " batch_size=batch_size\n", + ")\n", "\n", - "The image size will be handled later." + "val_ds = tf.keras.utils.image_dataset_from_directory(\n", + " str(data_root),\n", + " validation_split=0.2,\n", + " subset=\"validation\",\n", + " seed=123,\n", + " image_size=(img_height, img_width),\n", + " batch_size=batch_size\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cCrSRlomEIZ4" + }, + "source": [ + "The flowers dataset has five classes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AFgDHs6VEFRD" + }, + "outputs": [], + "source": [ + "class_names = np.array(train_ds.class_names)\n", + "print(class_names)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L0Btd0V3C8h4" + }, + "source": [ + "Second, because TensorFlow Hub's convention for image models is to expect float inputs in the `[0, 1]` range, use the `tf.keras.layers.Rescaling` preprocessing layer to achieve this." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Rs6gfO-ApTQW" + }, + "source": [ + "Note: You could also include the `tf.keras.layers.Rescaling` layer inside the model. Refer to the [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) guide for a discussion of the tradeoffs." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2PwQ_wYDcii9" + "id": "8NzDDWEMCL20" }, "outputs": [], "source": [ - "image_generator = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1/255)\n", - "image_data = image_generator.flow_from_directory(str(data_root), target_size=IMAGE_SHAPE)" + "normalization_layer = tf.keras.layers.Rescaling(1./255)\n", + "train_ds = train_ds.map(lambda x, y: (normalization_layer(x), y)) # Where x—images, y—labels.\n", + "val_ds = val_ds.map(lambda x, y: (normalization_layer(x), y)) # Where x—images, y—labels." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "0p7iDOhIcqY2" + "id": "IW-BUJ-NC7y-" }, "source": [ - "The resulting object is an iterator that returns `image_batch, label_batch` pairs." + "Third, finish the input pipeline by using buffered prefetching with `Dataset.prefetch`, so you can yield the data from disk without I/O blocking issues.\n", + "\n", + "These are some of the most important `tf.data` methods you should use when loading data. Interested readers can learn more about them, as well as how to cache data to disk and other techniques, in the [Better performance with the tf.data API](https://www.tensorflow.org/guide/data_performance#prefetching) guide." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "W4lDPkn2cpWZ" + "id": "ZmJMKFw7C4ki" }, "outputs": [], "source": [ - "for image_batch, label_batch in image_data:\n", - " print(\"Image batch shape: \", image_batch.shape)\n", - " print(\"Label batch shape: \", label_batch.shape)\n", + "AUTOTUNE = tf.data.AUTOTUNE\n", + "train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n", + "val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "m0JyiEZ0imgf" + }, + "outputs": [], + "source": [ + "for image_batch, labels_batch in train_ds:\n", + " print(image_batch.shape)\n", + " print(labels_batch.shape)\n", " break" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0gTN7M_GxDLx" }, "source": [ @@ -433,57 +474,48 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "O3fvrZR8xDLv" }, "source": [ - "Now run the classifier on the image batch." + "Now, run the classifier on an image batch:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nbyg6tcyxDLh" + "id": "pcFeNcrehEue" }, "outputs": [], "source": [ - "result_batch = classifier.predict(image_batch)\n", - "result_batch.shape" + "result_batch = classifier.predict(train_ds)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Kv7ZwuR4xDLc" + "id": "-wK2ky45hlyS" }, "outputs": [], "source": [ - "predicted_class_names = imagenet_labels[np.argmax(result_batch, axis=-1)]\n", + "predicted_class_names = imagenet_labels[tf.math.argmax(result_batch, axis=-1)]\n", "predicted_class_names" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QmvSWg9nxDLa" }, "source": [ - "Now check how these predictions line up with the images:" + "Check how these predictions line up with the images:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "IXTB22SpxDLP" }, "outputs": [], @@ -501,83 +533,77 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FUa3YkvhxDLM" }, "source": [ - "See the `LICENSE.txt` file for image attributions.\n", + "Note: all images are licensed CC-BY, creators are listed in the LICENSE.txt file.\n", "\n", - "The results are far from perfect, but reasonable considering that these are not the classes the model was trained for (except \"daisy\")." + "The results are far from perfect, but reasonable considering that these are not the classes the model was trained for (except for \"daisy\")." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JzV457OXreQP" }, "source": [ "### Download the headless model\n", "\n", - "TensorFlow Hub also distributes models without the top classification layer. These can be used to easily do transfer learning.\n", + "TensorFlow Hub also distributes models without the top classification layer. These can be used to easily perform transfer learning.\n", "\n", - "Any [Tensorflow 2 compatible image feature vector URL](https://tfhub.dev/s?module-type=image-feature-vector\u0026q=tf2) from tfhub.dev will work here." + "Select a MobileNetV2 pre-trained model from TensorFlow Hub. Any compatible image feature vector model from TensorFlow Hub will work here, including the examples from the drop-down menu." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "cellView": "both", - "colab": {}, - "colab_type": "code", "id": "4bw8Jf94DSnP" }, "outputs": [], "source": [ - "feature_extractor_url = \"https://tfhub.dev/google/tf2-preview/mobilenet_v2/feature_vector/2\" #@param {type:\"string\"}" + "mobilenet_v2 = \"https://tfhub.dev/google/tf2-preview/mobilenet_v2/feature_vector/4\"\n", + "inception_v3 = \"https://tfhub.dev/google/tf2-preview/inception_v3/feature_vector/4\"\n", + "\n", + "feature_extractor_model = mobilenet_v2 #@param [\"mobilenet_v2\", \"inception_v3\"] {type:\"raw\"}" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "sgwmHugQF-PD" }, "source": [ - "Create the feature extractor." + "Create the feature extractor by wrapping the pre-trained model as a Keras layer with [`hub.KerasLayer`](https://www.tensorflow.org/hub/api_docs/python/hub/KerasLayer). Use the `trainable=False` argument to freeze the variables, so that the training only modifies the new classifier layer:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "5wB030nezBwI" }, "outputs": [], "source": [ - "feature_extractor_layer = hub.KerasLayer(feature_extractor_url,\n", - " input_shape=(224,224,3))" + "feature_extractor_layer = hub.KerasLayer(\n", + " feature_extractor_model,\n", + " input_shape=(224, 224, 3),\n", + " trainable=False)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0QzVdu4ZhcDE" }, "source": [ - "It returns a 1280-length vector for each image:" + "The feature extractor returns a 1280-long vector for each image (the image batch size remains at 32 in this example):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Of7i-35F09ls" }, "outputs": [], @@ -589,51 +615,27 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "CtFmF7A5E4tk" - }, - "source": [ - "Freeze the variables in the feature extractor layer, so that the training only modifies the new classifier layer." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Jg5ar6rcE4H-" - }, - "outputs": [], - "source": [ - "feature_extractor_layer.trainable = False" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", "id": "RPVeouTksO9q" }, "source": [ "### Attach a classification head\n", "\n", - "Now wrap the hub layer in a `tf.keras.Sequential` model, and add a new classification layer." + "To complete the model, wrap the feature extractor layer in a `tf.keras.Sequential` model and add a fully-connected layer for classification:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mGcY27fY1q3Q" + "id": "vQq_kCWzlqSu" }, "outputs": [], "source": [ + "num_classes = len(class_names)\n", + "\n", "model = tf.keras.Sequential([\n", " feature_extractor_layer,\n", - " layers.Dense(image_data.num_classes, activation='softmax')\n", + " tf.keras.layers.Dense(num_classes)\n", "])\n", "\n", "model.summary()" @@ -641,11 +643,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "G9VkAz00HOJx" + "id": "IyhX4VCFmzVS" }, "outputs": [], "source": [ @@ -654,11 +654,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sB7sVGJ23vrY" + "id": "FQdUaTkzm3jQ" }, "outputs": [], "source": [ @@ -668,250 +666,165 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OHbXQqIquFxQ" }, "source": [ "### Train the model\n", "\n", - "Use compile to configure the training process:" + "Use `Model.compile` to configure the training process and add a `tf.keras.callbacks.TensorBoard` callback to create and store logs:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3n0Wb9ylKd8R" + "id": "4xRx8Rjzm67O" }, "outputs": [], "source": [ "model.compile(\n", " optimizer=tf.keras.optimizers.Adam(),\n", - " loss='categorical_crossentropy',\n", - " metrics=['acc'])" + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['acc'])\n", + "\n", + "log_dir = \"logs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n", + "tensorboard_callback = tf.keras.callbacks.TensorBoard(\n", + " log_dir=log_dir,\n", + " histogram_freq=1) # Enable histogram computation for every epoch." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "58-BLV7dupJA" }, "source": [ - "Now use the `.fit` method to train the model.\n", - "\n", - "To keep this example short train just 2 epochs. To visualize the training progress, use a custom callback to log the loss and accuracy of each batch individually, instead of the epoch average." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jZ54Gubac4Lu" - }, - "outputs": [], - "source": [ - "class CollectBatchStats(tf.keras.callbacks.Callback):\n", - " def __init__(self):\n", - " self.batch_losses = []\n", - " self.batch_acc = []\n", + "Now use the `Model.fit` method to train the model.\n", "\n", - " def on_train_batch_end(self, batch, logs=None):\n", - " self.batch_losses.append(logs['loss'])\n", - " self.batch_acc.append(logs['acc'])\n", - " self.model.reset_metrics()" + "To keep this example short, you'll be training for just 10 epochs. To visualize the training progress in TensorBoard later, create and store logs an a [TensorBoard callback](https://www.tensorflow.org/tensorboard/get_started#using_tensorboard_with_keras_modelfit)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EyMDJxt2HdHr" + "id": "JI0yAKd-nARd" }, "outputs": [], "source": [ - "steps_per_epoch = np.ceil(image_data.samples/image_data.batch_size)\n", + "NUM_EPOCHS = 10\n", "\n", - "batch_stats_callback = CollectBatchStats()\n", - "\n", - "history = model.fit_generator(image_data, epochs=2,\n", - " steps_per_epoch=steps_per_epoch,\n", - " callbacks = [batch_stats_callback])" + "history = model.fit(train_ds,\n", + " validation_data=val_ds,\n", + " epochs=NUM_EPOCHS,\n", + " callbacks=tensorboard_callback)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Kd0N272B9Q0b" + "id": "tiDbmiAK_h03" }, "source": [ - "Now after, even just a few training iterations, we can already see that the model is making progress on the task." + "Start the TensorBoard to view how the metrics change with each epoch and to track other scalar values:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "A5RfS1QIIP-P" + "id": "-yVJar0MiT2t" }, "outputs": [], "source": [ - "plt.figure()\n", - "plt.ylabel(\"Loss\")\n", - "plt.xlabel(\"Training Steps\")\n", - "plt.ylim([0,2])\n", - "plt.plot(batch_stats_callback.batch_losses)" + "%tensorboard --logdir logs/fit" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3uvX11avTiDg" + "id": "36a9d7cab8c8" }, - "outputs": [], "source": [ - "plt.figure()\n", - "plt.ylabel(\"Accuracy\")\n", - "plt.xlabel(\"Training Steps\")\n", - "plt.ylim([0,1])\n", - "plt.plot(batch_stats_callback.batch_acc)" + "" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kb__ZN8uFn-D" }, "source": [ "### Check the predictions\n", "\n", - "To redo the plot from before, first get the ordered list of class names:" + "Obtain the ordered list of class names from the model predictions:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "JGbEf5l1I4jz" }, "outputs": [], - "source": [ - "class_names = sorted(image_data.class_indices.items(), key=lambda pair:pair[1])\n", - "class_names = np.array([key.title() for key, value in class_names])\n", - "class_names" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4Olg6MsNGJTL" - }, - "source": [ - "Run the image batch through the model and convert the indices to class names." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fCLVCpEjJ_VP" - }, - "outputs": [], "source": [ "predicted_batch = model.predict(image_batch)\n", - "predicted_id = np.argmax(predicted_batch, axis=-1)\n", - "predicted_label_batch = class_names[predicted_id]" + "predicted_id = tf.math.argmax(predicted_batch, axis=-1)\n", + "predicted_label_batch = class_names[predicted_id]\n", + "print(predicted_label_batch)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "CkGbZxl9GZs-" }, "source": [ - "Plot the result" + "Plot the model predictions:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "rpFQR1MPMtT1" - }, - "outputs": [], - "source": [ - "label_id = np.argmax(label_batch, axis=-1)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wC_AYRJU9NQe" + "id": "hW3Ic_ZlwtrZ" }, "outputs": [], "source": [ "plt.figure(figsize=(10,9))\n", "plt.subplots_adjust(hspace=0.5)\n", + "\n", "for n in range(30):\n", " plt.subplot(6,5,n+1)\n", " plt.imshow(image_batch[n])\n", - " color = \"green\" if predicted_id[n] == label_id[n] else \"red\"\n", - " plt.title(predicted_label_batch[n].title(), color=color)\n", + " plt.title(predicted_label_batch[n].title())\n", " plt.axis('off')\n", - "_ = plt.suptitle(\"Model predictions (green: correct, red: incorrect)\")" + "_ = plt.suptitle(\"Model predictions\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uRcJnAABr22x" }, "source": [ - "## Export your model\n", + "## Export and reload your model\n", "\n", - "Now that you've trained the model, export it as a saved model:" + "Now that you've trained the model, export it as a SavedModel for reusing it later." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "PLcqg-RmsLno" }, "outputs": [], "source": [ - "import time\n", "t = time.time()\n", "\n", "export_path = \"/tmp/saved_models/{}\".format(int(t))\n", - "model.save(export_path, save_format='tf')\n", + "model.save(export_path)\n", "\n", "export_path" ] @@ -919,19 +832,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AhQ9liIUsPsi" }, "source": [ - "Now confirm that we can reload it, and it still gives the same results:" + "Confirm that you can reload the SavedModel and that the model is able to output the same results:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7nI5fvkAQvbS" }, "outputs": [], @@ -941,11 +851,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jor83-LqI8xW" + "id": "dnZO14taYPH6" }, "outputs": [], "source": [ @@ -955,26 +863,57 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dnZO14taYPH6" + "id": "wtjsIPjQnPyM" }, "outputs": [], "source": [ "abs(reloaded_result_batch - result_batch).max()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jor83-LqI8xW" + }, + "outputs": [], + "source": [ + "reloaded_predicted_id = tf.math.argmax(reloaded_result_batch, axis=-1)\n", + "reloaded_predicted_label_batch = class_names[reloaded_predicted_id]\n", + "print(reloaded_predicted_label_batch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RkQIBksVkxPO" + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(10,9))\n", + "plt.subplots_adjust(hspace=0.5)\n", + "for n in range(30):\n", + " plt.subplot(6,5,n+1)\n", + " plt.imshow(image_batch[n])\n", + " plt.title(reloaded_predicted_label_batch[n].title())\n", + " plt.axis('off')\n", + "_ = plt.suptitle(\"Model predictions\")" + ] + }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "TYZd4MNiV3Rc" + "id": "mSBRrW-MqBbk" }, "source": [ - "This saved model can loaded for inference later, or converted to [TFLite](https://www.tensorflow.org/lite/convert/) or [TFjs](https://github.com/tensorflow/tfjs-converter).\n", - "\n" + "## Next steps\n", + "\n", + "You can use the SavedModel to load for inference or convert it to a [TensorFlow Lite](https://www.tensorflow.org/lite/models/convert/) model (for on-device machine learning) or a [TensorFlow.js](https://www.tensorflow.org/js/tutorials#convert_pretrained_models_to_tensorflowjs) model (for machine learning in JavaScript).\n", + "\n", + "Discover [more tutorials](https://www.tensorflow.org/hub/tutorials) to learn how to use pre-trained models from TensorFlow Hub on image, text, audio, and video tasks." ] } ], @@ -985,9 +924,7 @@ "W_tvPdyfA-BL" ], "name": "transfer_learning_with_hub.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/interpretability/images/IG_fireboat.png b/site/en/tutorials/interpretability/images/IG_fireboat.png new file mode 100644 index 00000000000..9b0f2efe10f Binary files /dev/null and b/site/en/tutorials/interpretability/images/IG_fireboat.png differ diff --git a/site/en/tutorials/interpretability/integrated_gradients.ipynb b/site/en/tutorials/interpretability/integrated_gradients.ipynb new file mode 100644 index 00000000000..e63c8cdb7a2 --- /dev/null +++ b/site/en/tutorials/interpretability/integrated_gradients.ipynb @@ -0,0 +1,1162 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Ic4_occAAiAT" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "ioaprt5q5US7" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sPrjeJhFQBmu" + }, + "source": [ + "# Integrated gradients" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hKY4XMc9o8iB" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View on GitHub\n", + " \n", + " Download notebook\n", + " \n", + " See TF Hub model\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NG17_Wp6ikKf" + }, + "source": [ + "This tutorial demonstrates how to implement **Integrated Gradients (IG)**, an [Explainable AI](https://en.wikipedia.org/wiki/Explainable_artificial_intelligence) technique introduced in the paper [Axiomatic Attribution for Deep Networks](https://arxiv.org/abs/1703.01365). IG aims to explain the relationship between a model's predictions in terms of its features. It has many use cases including understanding feature importances, identifying data skew, and debugging model performance.\n", + "\n", + "IG has become a popular interpretability technique due to its broad applicability to any differentiable model (e.g. images, text, structured data), ease of implementation, theoretical justifications, and computational efficiency relative to alternative approaches that allow it to scale to large networks and feature spaces such as images.\n", + "\n", + "In this tutorial, you will walk through an implementation of IG step-by-step to understand the pixel feature importances of an image classifier. As an example, consider this [image](https://commons.wikimedia.org/wiki/File:San_Francisco_fireboat_showing_off.jpg) of a fireboat spraying jets of water. You would classify this image as a fireboat and might highlight the pixels making up the boat and water cannons as being important to your decision. Your model will also classify this image as a fireboat later on in this tutorial; however, does it highlight the same pixels as important when explaining its decision?\n", + "\n", + "In the images below titled \"IG Attribution Mask\" and \"Original + IG Mask Overlay\" you can see that your model instead highlights (in purple) the pixels comprising the boat's water cannons and jets of water as being more important than the boat itself to its decision. How will your model generalize to new fireboats? What about fireboats without water jets? Read on to learn more about how IG works and how to apply IG to your models to better understand the relationship between their predictions and underlying features.\n", + "\n", + "![Output Image 1](images/IG_fireboat.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ppydw6ZbKzM1" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cbUMIubipgg0" + }, + "outputs": [], + "source": [ + "import matplotlib.pylab as plt\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "import tensorflow_hub as hub" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GVVV4BGrABkA" + }, + "source": [ + "### Download a pretrained image classifier from TF-Hub" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7kwwJ35xmtoK" + }, + "source": [ + "IG can be applied to any differentiable model. In the spirit of the original paper, you will use a pre-trained version of the same model, Inception V1, which you will download from [TensorFlow Hub](https://tfhub.dev/google/imagenet/inception_v1/classification/4)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "14APZcfHolKj" + }, + "outputs": [], + "source": [ + "model = tf.keras.Sequential([\n", + " hub.KerasLayer(\n", + " name='inception_v1',\n", + " handle='https://tfhub.dev/google/imagenet/inception_v1/classification/4',\n", + " trainable=False),\n", + "])\n", + "model.build([None, 224, 224, 3])\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GjLRn2e5xFOb" + }, + "source": [ + "From the module page, you need to keep in mind the following about Inception V1:\n", + "\n", + "**Inputs**: The expected input shape for the model is `(None, 224, 224, 3)`. This is a dense 4D tensor of dtype float32 and shape `(batch_size, height, width, RGB channels)` whose elements are RGB color values of pixels normalized to the range [0, 1]. The first element is `None` to indicate that the model can take any integer batch size.\n", + "\n", + "**Outputs**: A `tf.Tensor` of logits in the shape of `(batch_size, 1001)`. Each row represents the model's predicted score for 1,001 classes from ImageNet. For the model's top predicted class index you can use `tf.math.argmax(predictions, axis=-1)`. Furthermore, you can also convert the model's logit output to predicted probabilities across all classes using `tf.nn.softmax(predictions, axis=-1)` to quantify the model's uncertainty and explore similar predicted classes for debugging." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "huZnb_O0L9mw" + }, + "outputs": [], + "source": [ + "def load_imagenet_labels(file_path):\n", + " labels_file = tf.keras.utils.get_file('ImageNetLabels.txt', file_path)\n", + " with open(labels_file) as reader:\n", + " f = reader.read()\n", + " labels = f.splitlines()\n", + " return np.array(labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Rtrl-u7T6NEk" + }, + "outputs": [], + "source": [ + "imagenet_labels = load_imagenet_labels('https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "STpIJr1Z5r_u" + }, + "source": [ + "### Load and preprocess images with `tf.image`\n", + "\n", + "You will illustrate IG using two images from [Wikimedia Commons](https://commons.wikimedia.org/wiki/Main_Page): a [Fireboat](https://commons.wikimedia.org/wiki/File:San_Francisco_fireboat_showing_off.jpg), and a [Giant Panda](https://commons.wikimedia.org/wiki/File:Giant_Panda_2.JPG)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YOb0Adq-rU5J" + }, + "outputs": [], + "source": [ + "def read_image(file_name):\n", + " image = tf.io.read_file(file_name)\n", + " image = tf.io.decode_jpeg(image, channels=3)\n", + " image = tf.image.convert_image_dtype(image, tf.float32)\n", + " image = tf.image.resize_with_pad(image, target_height=224, target_width=224)\n", + " return image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_khLTN75CLMJ" + }, + "outputs": [], + "source": [ + "img_url = {\n", + " 'Fireboat': 'http://storage.googleapis.com/download.tensorflow.org/example_images/San_Francisco_fireboat_showing_off.jpg',\n", + " 'Giant Panda': 'http://storage.googleapis.com/download.tensorflow.org/example_images/Giant_Panda_2.jpeg',\n", + "}\n", + "\n", + "img_paths = {name: tf.keras.utils.get_file(name, url) for (name, url) in img_url.items()}\n", + "img_name_tensors = {name: read_image(img_path) for (name, img_path) in img_paths.items()}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AYIeu8rMLN-8" + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(8, 8))\n", + "for n, (name, img_tensors) in enumerate(img_name_tensors.items()):\n", + " ax = plt.subplot(1, 2, n+1)\n", + " ax.imshow(img_tensors)\n", + " ax.set_title(name)\n", + " ax.axis('off')\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QvTYc8IZKbeO" + }, + "source": [ + "### Classify images\n", + "Start by classifying these images and displaying the top 3 most confident predictions. The following is a utility function to retrieve the top k predicted labels and probabilities." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5gsO7ILHZ0By" + }, + "outputs": [], + "source": [ + "def top_k_predictions(img, k=3):\n", + " image_batch = tf.expand_dims(img, 0)\n", + " predictions = model(image_batch)\n", + " probs = tf.nn.softmax(predictions, axis=-1)\n", + " top_probs, top_idxs = tf.math.top_k(input=probs, k=k)\n", + " top_labels = imagenet_labels[tuple(top_idxs)]\n", + " return top_labels, top_probs[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "l80a8N2vcIP-" + }, + "outputs": [], + "source": [ + "for (name, img_tensor) in img_name_tensors.items():\n", + " plt.imshow(img_tensor)\n", + " plt.title(name, fontweight='bold')\n", + " plt.axis('off')\n", + " plt.show()\n", + "\n", + " pred_label, pred_prob = top_k_predictions(img_tensor)\n", + " for label, prob in zip(pred_label, pred_prob):\n", + " print(f'{label}: {prob:0.1%}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v-lH1N4timM2" + }, + "source": [ + "## Calculate Integrated Gradients" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FV7pfZHANaz1" + }, + "source": [ + "Your model, Inception V1, is a learned function that describes a mapping between your input feature space, image pixel values, and an output space defined by ImageNet class probability values between 0 and 1. Early interpretability methods for neural networks assigned feature importance scores using gradients, which tell you which pixels have the steepest local relative to your model's prediction at a given point along your model's prediction function. However, gradients only describe *local* changes in your model's prediction function with respect to pixel values and do not fully describe your entire model prediction function. As your model fully \"learns\" the relationship between the range of an individual pixel and the correct ImageNet class, the gradient for this pixel will *saturate*, meaning become increasingly small and even go to zero. Consider the simple model function below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0AUkIUvpkaO8" + }, + "outputs": [], + "source": [ + "def f(x):\n", + " \"\"\"A simplified model function.\"\"\"\n", + " return tf.where(x < 0.8, x, 0.8)\n", + "\n", + "def interpolated_path(x):\n", + " \"\"\"A straight line path.\"\"\"\n", + " return tf.zeros_like(x)\n", + "\n", + "x = tf.linspace(start=0.0, stop=1.0, num=6)\n", + "y = f(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kMdAKooulVRE" + }, + "outputs": [], + "source": [ + "#@title\n", + "fig = plt.figure(figsize=(12, 5))\n", + "ax0 = fig.add_subplot(121)\n", + "ax0.plot(x, f(x), marker='o')\n", + "ax0.set_title('Gradients saturate over F(x)', fontweight='bold')\n", + "ax0.text(0.2, 0.5, 'Gradients > 0 = \\n x is important')\n", + "ax0.text(0.7, 0.85, 'Gradients = 0 \\n x not important')\n", + "ax0.set_yticks(tf.range(0, 1.5, 0.5))\n", + "ax0.set_xticks(tf.range(0, 1.5, 0.5))\n", + "ax0.set_ylabel('F(x) - model true class predicted probability')\n", + "ax0.set_xlabel('x - (pixel value)')\n", + "\n", + "ax1 = fig.add_subplot(122)\n", + "ax1.plot(x, f(x), marker='o')\n", + "ax1.plot(x, interpolated_path(x), marker='>')\n", + "ax1.set_title('IG intuition', fontweight='bold')\n", + "ax1.text(0.25, 0.1, 'Accumulate gradients along path')\n", + "ax1.set_ylabel('F(x) - model true class predicted probability')\n", + "ax1.set_xlabel('x - (pixel value)')\n", + "ax1.set_yticks(tf.range(0, 1.5, 0.5))\n", + "ax1.set_xticks(tf.range(0, 1.5, 0.5))\n", + "ax1.annotate('Baseline', xy=(0.0, 0.0), xytext=(0.0, 0.2),\n", + " arrowprops=dict(facecolor='black', shrink=0.1))\n", + "ax1.annotate('Input', xy=(1.0, 0.0), xytext=(0.95, 0.2),\n", + " arrowprops=dict(facecolor='black', shrink=0.1))\n", + "plt.show();" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LswhGFd0xZBr" + }, + "source": [ + "* **left**: Your model's gradients for pixel `x` are positive between 0.0 and 0.8 but go to 0.0 between 0.8 and 1.0. Pixel `x` clearly has a significant impact on pushing your model toward 80% predicted probability on the true class. *Does it make sense that pixel `x`'s importance is small or discontinuous?*\n", + "\n", + "* **right**: The intuition behind IG is to accumulate pixel `x`'s local gradients and attribute its importance as a score for how much it adds or subtracts to your model's overall output class probability. You can break down and compute IG in 3 parts: \n", + " 1. interpolate small steps along a straight line in the feature space between 0 (a baseline or starting point) and 1 (input pixel's value) \n", + " 2. compute gradients at each step between your model's predictions with respect to each step \n", + " 3. approximate the integral between your baseline and input by accumulating (cumulative average) these local gradients.\n", + "\n", + "To reinforce this intuition, you will walk through these 3 parts by applying IG to the example \"Fireboat\" image below. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "28MU35BCLM-s" + }, + "source": [ + "### Establish a baseline" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MIPG5yYfkydQ" + }, + "source": [ + "A baseline is an input image used as a starting point for calculating feature importance. Intuitively, you can think of the baseline's explanatory role as representing the impact of the absence of each pixel on the \"Fireboat\" prediction to contrast with its impact of each pixel on the \"Fireboat\" prediction when present in the input image. As a result, the choice of the baseline plays a central role in interpreting and visualizing pixel feature importances. For additional discussion of baseline selection, see the resources in the \"Next steps\" section at the bottom of this tutorial. Here, you will use a black image whose pixel values are all zero.\n", + "\n", + "Other choices you could experiment with include an all white image, or a random image, which you can create with `tf.random.uniform(shape=(224,224,3), minval=0.0, maxval=1.0)`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wxvpwGkj4G4J" + }, + "outputs": [], + "source": [ + "baseline = tf.zeros(shape=(224,224,3))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vXRYwBWQS19B" + }, + "outputs": [], + "source": [ + "plt.imshow(baseline)\n", + "plt.title(\"Baseline\")\n", + "plt.axis('off')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xphryu2mGAk8" + }, + "source": [ + "### Unpack formulas into code" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QstMR0IcbfFA" + }, + "source": [ + "The formula for Integrated Gradients is as follows:\n", + "\n", + "$IntegratedGradients_{i}(x) ::= (x_{i} - x'_{i})\\times\\int_{\\alpha=0}^1\\frac{\\partial F(x'+\\alpha \\times (x - x'))}{\\partial x_i}{d\\alpha}$\n", + "\n", + "where:\n", + "\n", + "$_{i}$ = feature \n", + "$x$ = input \n", + "$x'$ = baseline \n", + "$\\alpha$ = interpolation constant to perturb features by" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_7w8SD5YMqvi" + }, + "source": [ + "In practice, computing a definite integral is not always numerically possible and can be computationally costly, so you compute the following numerical approximation:\n", + "\n", + "$IntegratedGrads^{approx}_{i}(x)::=(x_{i}-x'_{i})\\times\\sum_{k=1}^{m}\\frac{\\partial F(x' + \\frac{k}{m}\\times(x - x'))}{\\partial x_{i}} \\times \\frac{1}{m}$\n", + "\n", + "where:\n", + "\n", + "$_{i}$ = feature (individual pixel) \n", + "$x$ = input (image tensor) \n", + "$x'$ = baseline (image tensor) \n", + "$k$ = scaled feature perturbation constant \n", + "$m$ = number of steps in the Riemann sum approximation of the integral \n", + "$(x_{i}-x'_{i})$ = a term for the difference from the baseline. This is necessary to scale the integrated gradients and keep them in terms of the original image. The path from the baseline image to the input is in pixel space. Since with IG you are integrating in a straight line (linear transformation) this ends up being roughly equivalent to the integral term of the derivative of the interpolated image function with respect to $\\alpha$ with enough steps. The integral sums each pixel's gradient times the change in the pixel along the path. It's simpler to implement this integration as uniform steps from one image to the other, substituting $x := (x' + \\alpha(x-x'))$. So the change of variables gives $dx = (x-x')d\\alpha$. The $(x-x')$ term is constant and is factored out of the integral." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5aPG68RssS2h" + }, + "source": [ + "### Interpolate images" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pPrldEYsIR4M" + }, + "source": [ + "$IntegratedGrads^{approx}_{i}(x)::=(x_{i}-x'_{i})\\times\\sum_{k=1}^{m}\\frac{\\partial F(\\overbrace{x' + \\frac{k}{m}\\times(x - x')}^\\text{interpolate m images at k intervals})}{\\partial x_{i}} \\times \\frac{1}{m}$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "y4r-ZrIIsdbI" + }, + "source": [ + "First, you will generate a [linear interpolation](https://en.wikipedia.org/wiki/Linear_interpolation) between the baseline and the original image. You can think of interpolated images as small steps in the feature space between your baseline and input, represented by $\\alpha$ in the original equation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "I42mBKXyjcIc" + }, + "outputs": [], + "source": [ + "m_steps=50\n", + "alphas = tf.linspace(start=0.0, stop=1.0, num=m_steps+1) # Generate m_steps intervals for integral_approximation() below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7SWLSFOHsbgh" + }, + "outputs": [], + "source": [ + "def interpolate_images(baseline,\n", + " image,\n", + " alphas):\n", + " alphas_x = alphas[:, tf.newaxis, tf.newaxis, tf.newaxis]\n", + " baseline_x = tf.expand_dims(baseline, axis=0)\n", + " input_x = tf.expand_dims(image, axis=0)\n", + " delta = input_x - baseline_x\n", + " images = baseline_x + alphas_x * delta\n", + " return images" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s4zFzbUBj684" + }, + "source": [ + "Use the above function to generate interpolated images along a linear path at alpha intervals between a black baseline image and the example \"Fireboat\" image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NgVx8swDQtTl" + }, + "outputs": [], + "source": [ + "interpolated_images = interpolate_images(\n", + " baseline=baseline,\n", + " image=img_name_tensors['Fireboat'],\n", + " alphas=alphas)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QABFsuCvkO1h" + }, + "source": [ + "Visualize the interpolated images. Note: another way of thinking about the $\\alpha$ constant is that it is consistently increasing each interpolated image's intensity." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9tmBGdnHAupk" + }, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(20, 20))\n", + "\n", + "i = 0\n", + "for alpha, image in zip(alphas[0::10], interpolated_images[0::10]):\n", + " i += 1\n", + " plt.subplot(1, len(alphas[0::10]), i)\n", + " plt.title(f'alpha: {alpha:.1f}')\n", + " plt.imshow(image)\n", + " plt.axis('off')\n", + "\n", + "plt.tight_layout();" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h7T0f1cqsaxA" + }, + "source": [ + "### Compute gradients" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tps0eWc0REqL" + }, + "source": [ + "This section explains how to calculate the gradients to measure the relationship between changes to a feature and changes in the model's predictions. In the case of images, the gradient tells us which pixels have the strongest effect on the model's predicted class probabilities." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ouuVIsdfgukW" + }, + "source": [ + "$IntegratedGrads^{approx}_{i}(x)::=(x_{i}-x'_{i})\\times\\sum_{k=1}^{m}\\frac{\\overbrace{\\partial F(\\text{interpolated images})}^\\text{compute gradients}}{\\partial x_{i}} \\times \\frac{1}{m}$\n", + "\n", + "where: \n", + "$F()$ = your model's prediction function \n", + "$\\frac{\\partial{F}}{\\partial{x_i}}$ = gradient (vector of partial derivatives $\\partial$) of your model F's prediction function relative to each feature $x_i$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hY_Ok3CoJW1W" + }, + "source": [ + "TensorFlow makes computing gradients easy for you with a [`tf.GradientTape`](https://www.tensorflow.org/api_docs/python/tf/GradientTape)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JW1O9qEsxZOP" + }, + "outputs": [], + "source": [ + "def compute_gradients(images, target_class_idx):\n", + " with tf.GradientTape() as tape:\n", + " tape.watch(images)\n", + " logits = model(images)\n", + " probs = tf.nn.softmax(logits, axis=-1)[:, target_class_idx]\n", + " return tape.gradient(probs, images)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9BfRuzx4-c87" + }, + "source": [ + "Compute the gradients for each image along the interpolation path with respect to the correct output. Recall that your model returns a `(1, 1001)` shaped `Tensor` with logits that you convert to predicted probabilities for each class. You need to pass the correct ImageNet target class index to the `compute_gradients` function for your image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kHIR58rNJ3q_" + }, + "outputs": [], + "source": [ + "path_gradients = compute_gradients(\n", + " images=interpolated_images,\n", + " target_class_idx=555)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JJodSbDUQ3T_" + }, + "source": [ + "Note the output shape of `(n_interpolated_images, img_height, img_width, RGB)`, which gives us the gradient for every pixel of every image along the interpolation path. You can think of these gradients as measuring the change in your model's predictions for each small step in the feature space." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "v2rpO2JTbQId" + }, + "outputs": [], + "source": [ + "print(path_gradients.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zmaPpr5bUtbr" + }, + "source": [ + "**Visualizing gradient saturation**\n", + "\n", + "Recall that the gradients you just calculated above describe *local* changes to your model's predicted probability of \"Fireboat\" and can *saturate*.\n", + "\n", + "These concepts are visualized using the gradients you calculated above in the 2 plots below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FQWwcI0Wr0AX" + }, + "outputs": [], + "source": [ + "pred = model(interpolated_images)\n", + "pred_proba = tf.nn.softmax(pred, axis=-1)[:, 555]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mCH8sAf3TTJ2" + }, + "outputs": [], + "source": [ + "#@title\n", + "plt.figure(figsize=(10, 4))\n", + "ax1 = plt.subplot(1, 2, 1)\n", + "ax1.plot(alphas, pred_proba)\n", + "ax1.set_title('Target class predicted probability over alpha')\n", + "ax1.set_ylabel('model p(target class)')\n", + "ax1.set_xlabel('alpha')\n", + "ax1.set_ylim([0, 1])\n", + "\n", + "ax2 = plt.subplot(1, 2, 2)\n", + "# Average across interpolation steps\n", + "average_grads = tf.reduce_mean(path_gradients, axis=[1, 2, 3])\n", + "# Normalize gradients to 0 to 1 scale. E.g., (x - min(x))/(max(x)-min(x))\n", + "average_grads_norm = (average_grads-tf.math.reduce_min(average_grads))/(tf.math.reduce_max(average_grads)-tf.reduce_min(average_grads))\n", + "ax2.plot(alphas, average_grads_norm)\n", + "ax2.set_title('Average pixel gradients (normalized) over alpha')\n", + "ax2.set_ylabel('Average pixel gradients')\n", + "ax2.set_xlabel('alpha')\n", + "ax2.set_ylim([0, 1]);" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-ntMpA87jNN6" + }, + "source": [ + "* **left**: This plot shows how your model's confidence in the \"Fireboat\" class varies across alphas. Notice how the gradients, or slope of the line, largely flattens or saturates between 0.6 and 1.0 before settling at the final \"Fireboat\" predicted probability of about 40%.\n", + "\n", + "* **right**: The right plot shows the average gradients magnitudes over alpha more directly. Note how the values sharply approach and even briefly dip below zero. In fact, your model \"learns\" the most from gradients at lower values of alpha before saturating. Intuitively, you can think of this as your model has learned the pixels e.g. water cannons to make the correct prediction, sending these pixel gradients to zero, but is still quite uncertain and focused on spurious bridge or water jet pixels as the alpha values approach the original input image.\n", + "\n", + "To make sure these important water cannon pixels are reflected as important to the \"Fireboat\" prediction, you will continue on below to learn how to accumulate these gradients to accurately approximate how each pixel impacts your \"Fireboat\" predicted probability.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LQdACCM6sJdW" + }, + "source": [ + "### Accumulate gradients (integral approximation)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QHopk9evmO5P" + }, + "source": [ + "There are many different ways you can go about computing the numerical approximation of an integral for IG with different tradeoffs in accuracy and convergence across varying functions. A popular class of methods is called [Riemann sums](https://en.wikipedia.org/wiki/Riemann_sum). Here, you will use the Trapezoidal rule (you can find additional code to explore different approximation methods at the end of this tutorial)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GshPZQgROs80" + }, + "source": [ + "$IntegratedGrads^{approx}_{i}(x)::=(x_{i}-x'_{i})\\times \\overbrace{\\sum_{k=1}^{m}}^\\text{Sum m local gradients}\\text{gradients(interpolated images)} \\times \\overbrace{\\frac{1}{m}}^\\text{Divide by m steps}$\n", + "\n", + "From the equation, you can see you are summing over `m` gradients and dividing by `m` steps. You can implement the two operations together for part 3 as an *average of the local gradients of `m` interpolated predictions and input images*." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1cMVl-Grx3lp" + }, + "outputs": [], + "source": [ + "def integral_approximation(gradients):\n", + " # riemann_trapezoidal\n", + " grads = (gradients[:-1] + gradients[1:]) / tf.constant(2.0)\n", + " integrated_gradients = tf.math.reduce_mean(grads, axis=0)\n", + " return integrated_gradients" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QVQAHunkW79t" + }, + "source": [ + "The `integral_approximation` function takes the gradients of the predicted probability of the target class with respect to the interpolated images between the baseline and the original image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JeF01fydNq0I" + }, + "outputs": [], + "source": [ + "ig = integral_approximation(\n", + " gradients=path_gradients)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7XVItLpurOAM" + }, + "source": [ + "You can confirm averaging across the gradients of `m` interpolated images returns an integrated gradients tensor with the same shape as the original \"Giant Panda\" image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "z1bP6l3ahfyn" + }, + "outputs": [], + "source": [ + "print(ig.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C1ODXUevyGxL" + }, + "source": [ + "### Putting it all together" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NcaTR-x8v1At" + }, + "source": [ + "Now you will combine the 3 previous general parts together into an `IntegratedGradients` function and utilize a [@tf.function](https://www.tensorflow.org/guide/function) decorator to compile it into a high performance callable TensorFlow graph. This is implemented as 5 smaller steps below:\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5YdWHscoovhk" + }, + "source": [ + "$IntegratedGrads^{approx}_{i}(x)::=\\overbrace{(x_{i}-x'_{i})}^\\text{5.}\\times \\overbrace{\\sum_{k=1}^{m}}^\\text{4.} \\frac{\\partial \\overbrace{F(\\overbrace{x' + \\overbrace{\\frac{k}{m}}^\\text{1.}\\times(x - x'))}^\\text{2.}}^\\text{3.}}{\\partial x_{i}} \\times \\overbrace{\\frac{1}{m}}^\\text{4.}$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pjdfjp_3pHiY" + }, + "source": [ + "1. Generate alphas $\\alpha$\n", + "\n", + "2. Generate interpolated images = $(x' + \\frac{k}{m}\\times(x - x'))$\n", + "\n", + "3. Compute gradients between model $F$ output predictions with respect to input features = $\\frac{\\partial F(\\text{interpolated path inputs})}{\\partial x_{i}}$\n", + "\n", + "4. Integral approximation through averaging gradients = $\\sum_{k=1}^m \\text{gradients} \\times \\frac{1}{m}$\n", + "\n", + "5. Scale integrated gradients with respect to original image = $(x_{i}-x'_{i}) \\times \\text{integrated gradients}$. The reason this step is necessary is to make sure that the attribution values accumulated across multiple interpolated images are in the same units and faithfully represent the pixel importances on the original image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "O_H3k9Eu7Rl5" + }, + "outputs": [], + "source": [ + "def integrated_gradients(baseline,\n", + " image,\n", + " target_class_idx,\n", + " m_steps=50,\n", + " batch_size=32):\n", + " # Generate alphas.\n", + " alphas = tf.linspace(start=0.0, stop=1.0, num=m_steps+1)\n", + "\n", + " # Collect gradients. \n", + " gradient_batches = []\n", + " \n", + " # Iterate alphas range and batch computation for speed, memory efficiency, and scaling to larger m_steps.\n", + " for alpha in tf.range(0, len(alphas), batch_size):\n", + " from_ = alpha\n", + " to = tf.minimum(from_ + batch_size, len(alphas))\n", + " alpha_batch = alphas[from_:to]\n", + "\n", + " gradient_batch = one_batch(baseline, image, alpha_batch, target_class_idx)\n", + " gradient_batches.append(gradient_batch)\n", + " \n", + " # Concatenate path gradients together row-wise into single tensor.\n", + " total_gradients = tf.concat(gradient_batches, axis=0)\n", + "\n", + " # Integral approximation through averaging gradients.\n", + " avg_gradients = integral_approximation(gradients=total_gradients)\n", + "\n", + " # Scale integrated gradients with respect to input.\n", + " integrated_gradients = (image - baseline) * avg_gradients\n", + "\n", + " return integrated_gradients" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dszwB_Sp0CX0" + }, + "outputs": [], + "source": [ + "@tf.function\n", + "def one_batch(baseline, image, alpha_batch, target_class_idx):\n", + " # Generate interpolated inputs between baseline and input.\n", + " interpolated_path_input_batch = interpolate_images(baseline=baseline,\n", + " image=image,\n", + " alphas=alpha_batch)\n", + "\n", + " # Compute gradients between model outputs and interpolated inputs.\n", + " gradient_batch = compute_gradients(images=interpolated_path_input_batch,\n", + " target_class_idx=target_class_idx)\n", + " return gradient_batch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8G0ELl_wRrd0" + }, + "outputs": [], + "source": [ + "ig_attributions = integrated_gradients(baseline=baseline,\n", + " image=img_name_tensors['Fireboat'],\n", + " target_class_idx=555,\n", + " m_steps=240)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P-LSHD2sajFf" + }, + "source": [ + "Again, you can check that the IG feature attributions have the same shape as the input \"Fireboat\" image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "beoEunC5aZOa" + }, + "outputs": [], + "source": [ + "print(ig_attributions.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fLaB21txJhMq" + }, + "source": [ + "The paper suggests the number of steps to range between 20 to 300 depending upon the example (although in practice this can be higher in the 1,000s to accurately approximate the integral). You can find additional code to check for the appropriate number of steps in the \"Next steps\" resources at the end of this tutorial." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o55W6NYXGSZ8" + }, + "source": [ + "### Visualize attributions" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XSQ6Y-DZvrQu" + }, + "source": [ + "You are ready to visualize attributions, and overlay them on the original image. The code below sums the absolute values of the integrated gradients across the color channels to produce an attribution mask. This plotting method captures the relative impact of pixels on the model's predictions. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4QN2cEA_WFym" + }, + "outputs": [], + "source": [ + "#@title\n", + "def plot_img_attributions(baseline,\n", + " image,\n", + " target_class_idx,\n", + " m_steps=50,\n", + " cmap=None,\n", + " overlay_alpha=0.4):\n", + "\n", + " attributions = integrated_gradients(baseline=baseline,\n", + " image=image,\n", + " target_class_idx=target_class_idx,\n", + " m_steps=m_steps)\n", + "\n", + " # Sum of the attributions across color channels for visualization.\n", + " # The attribution mask shape is a grayscale image with height and width\n", + " # equal to the original image.\n", + " attribution_mask = tf.reduce_sum(tf.math.abs(attributions), axis=-1)\n", + "\n", + " fig, axs = plt.subplots(nrows=2, ncols=2, squeeze=False, figsize=(8, 8))\n", + "\n", + " axs[0, 0].set_title('Baseline image')\n", + " axs[0, 0].imshow(baseline)\n", + " axs[0, 0].axis('off')\n", + "\n", + " axs[0, 1].set_title('Original image')\n", + " axs[0, 1].imshow(image)\n", + " axs[0, 1].axis('off')\n", + "\n", + " axs[1, 0].set_title('Attribution mask')\n", + " axs[1, 0].imshow(attribution_mask, cmap=cmap)\n", + " axs[1, 0].axis('off')\n", + "\n", + " axs[1, 1].set_title('Overlay')\n", + " axs[1, 1].imshow(attribution_mask, cmap=cmap)\n", + " axs[1, 1].imshow(image, alpha=overlay_alpha)\n", + " axs[1, 1].axis('off')\n", + "\n", + " plt.tight_layout()\n", + " return fig" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "n73VxzbxeMvD" + }, + "source": [ + "Looking at the attributions on the \"Fireboat\" image, you can see the model identifies the water cannons and spouts as contributing to its correct prediction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vxCQFx96iDVs" + }, + "outputs": [], + "source": [ + "_ = plot_img_attributions(image=img_name_tensors['Fireboat'],\n", + " baseline=baseline,\n", + " target_class_idx=555,\n", + " m_steps=240,\n", + " cmap=plt.cm.inferno,\n", + " overlay_alpha=0.4)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Lo4SncDZfTw0" + }, + "source": [ + "On the \"Giant Panda\" image, the attributions highlight the texture, nose, and the fur of the Panda's face." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TcpGLJWuHnYl" + }, + "outputs": [], + "source": [ + "_ = plot_img_attributions(image=img_name_tensors['Giant Panda'],\n", + " baseline=baseline,\n", + " target_class_idx=389,\n", + " m_steps=55,\n", + " cmap=plt.cm.viridis,\n", + " overlay_alpha=0.5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H3etJZHuI6hX" + }, + "source": [ + "## Uses and limitations" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xEX4Jh2uLvxA" + }, + "source": [ + "Use cases\n", + "* Employing techniques like Integrated Gradients before deploying your model can help you develop intuition for how and why it works. Do the features highlighted by this technique match your intuition? If not, that may be indicative of a bug in your model or dataset, or overfitting.\n", + "\n", + "Limitations\n", + "* The Integrated Gradients technique provides feature importances on individual examples. However, it does not provide global feature importances across an entire dataset.\n", + "\n", + "* The Integrated Gradients technique provides individual feature importances, but it does not explain feature interactions and combinations." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ejc2Ho_8i162" + }, + "source": [ + "## Next steps" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G8Ra5ijj7pEc" + }, + "source": [ + "This tutorial presented a basic implementation of Integrated Gradients. As a next step, you can use this notebook to try this technique with different models and images yourself.\n", + "\n", + "For interested readers, there is a lengthier version of this tutorial (which includes code for different baselines, to compute integral approximations, and to determine a sufficient number of steps) which you can find [here](https://github.com/GoogleCloudPlatform/training-data-analyst/tree/master/blogs/integrated_gradients).\n", + "\n", + "To deepen your understanding, check out the paper [Axiomatic Attribution for Deep Networks](https://arxiv.org/abs/1703.01365) and [Github repository](https://github.com/ankurtaly/Integrated-Gradients), which contains an implementation in a previous version of TensorFlow. You can also explore feature attribution, and the impact of different baselines, on [distill.pub](https://distill.pub/2020/attribution-baselines/).\n", + "\n", + "Interested in incorporating IG into your production machine learning workflows for feature importances, model error analysis, and data skew monitoring? Check out Google Cloud's [Explainable AI](https://cloud.google.com/explainable-ai) product that supports IG attributions. The Google AI PAIR research group also open-sourced the [What-if tool](https://pair-code.github.io/what-if-tool/index.html#about) which can be used for model debugging, including visualizing IG feature attributions." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "integrated_gradients.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/keras/classification.ipynb b/site/en/tutorials/keras/classification.ipynb index 43ff65d1944..dee10f107c3 100644 --- a/site/en/tutorials/keras/classification.ipynb +++ b/site/en/tutorials/keras/classification.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MhoQ0WE77laV" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "_ckMIh7O7s6D" }, "outputs": [], @@ -36,11 +33,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "vasWnqRgy1H4" }, "outputs": [], @@ -71,7 +66,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jYysdyb-CaWM" }, "source": [ @@ -81,30 +75,28 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "S5Uhzt6vVIB2" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/keras/classification\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/keras/classification.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/classification.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/keras/classification.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FbVhjPpzn6BM" }, "source": [ @@ -115,36 +107,14 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jL3OqFKZ9dFg" - }, - "outputs": [], - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", "id": "dzLKpmZICaWN" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "# TensorFlow and tf.keras\n", "import tensorflow as tf\n", - "from tensorflow import keras\n", "\n", "# Helper libraries\n", "import numpy as np\n", @@ -156,7 +126,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "yR0EdgrLCaWR" }, "source": [ @@ -166,40 +135,37 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DLdCchMdCaWQ" }, "source": [ "This guide uses the [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset which contains 70,000 grayscale images in 10 categories. The images show individual articles of clothing at low resolution (28 by 28 pixels), as seen here:\n", "\n", - "\u003ctable\u003e\n", - " \u003ctr\u003e\u003ctd\u003e\n", - " \u003cimg src=\"https://tensorflow.org/images/fashion-mnist-sprite.png\"\n", - " alt=\"Fashion MNIST sprite\" width=\"600\"\u003e\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\"\u003e\n", - " \u003cb\u003eFigure 1.\u003c/b\u003e \u003ca href=\"https://github.com/zalandoresearch/fashion-mnist\"\u003eFashion-MNIST samples\u003c/a\u003e (by Zalando, MIT License).\u003cbr/\u003e\u0026nbsp;\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e\n", + "\n", + " \n", + " \n", + "
    \n", + " \"Fashion\n", + "
    \n", + " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
     \n", + "
    \n", "\n", "Fashion MNIST is intended as a drop-in replacement for the classic [MNIST](http://yann.lecun.com/exdb/mnist/) dataset—often used as the \"Hello, World\" of machine learning programs for computer vision. The MNIST dataset contains images of handwritten digits (0, 1, 2, etc.) in a format identical to that of the articles of clothing you'll use here.\n", "\n", "This guide uses Fashion MNIST for variety, and because it's a slightly more challenging problem than regular MNIST. Both datasets are relatively small and are used to verify that an algorithm works as expected. They're good starting points to test and debug code.\n", "\n", - "Here, 60,000 images are used to train the network and 10,000 images to evaluate how accurately the network learned to classify images. You can access the Fashion MNIST directly from TensorFlow. Import and load the Fashion MNIST data directly from TensorFlow:" + "Here, 60,000 images are used to train the network and 10,000 images to evaluate how accurately the network learned to classify images. You can access the Fashion MNIST directly from TensorFlow. Import and [load the Fashion MNIST data](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/fashion_mnist/load_data) directly from TensorFlow:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7MqDQO0KCaWS" }, "outputs": [], "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", + "fashion_mnist = tf.keras.datasets.fashion_mnist\n", "\n", "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" ] @@ -207,7 +173,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "t9FDsUlxCaWW" }, "source": [ @@ -218,62 +183,60 @@ "\n", "The images are 28x28 NumPy arrays, with pixel values ranging from 0 to 255. The *labels* are an array of integers, ranging from 0 to 9. These correspond to the *class* of clothing the image represents:\n", "\n", - "\u003ctable\u003e\n", - " \u003ctr\u003e\n", - " \u003cth\u003eLabel\u003c/th\u003e\n", - " \u003cth\u003eClass\u003c/th\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e0\u003c/td\u003e\n", - " \u003ctd\u003eT-shirt/top\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e1\u003c/td\u003e\n", - " \u003ctd\u003eTrouser\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e2\u003c/td\u003e\n", - " \u003ctd\u003ePullover\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e3\u003c/td\u003e\n", - " \u003ctd\u003eDress\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e4\u003c/td\u003e\n", - " \u003ctd\u003eCoat\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e5\u003c/td\u003e\n", - " \u003ctd\u003eSandal\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e6\u003c/td\u003e\n", - " \u003ctd\u003eShirt\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e7\u003c/td\u003e\n", - " \u003ctd\u003eSneaker\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e8\u003c/td\u003e\n", - " \u003ctd\u003eBag\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e9\u003c/td\u003e\n", - " \u003ctd\u003eAnkle boot\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - "\u003c/table\u003e\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    LabelClass
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", "\n", "Each image is mapped to a single label. Since the *class names* are not included with the dataset, store them here to use later when plotting the images:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "IjnLH5S2CaWx" }, "outputs": [], @@ -285,7 +248,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Brm0b_KACaWX" }, "source": [ @@ -296,10 +258,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "zW5k_xz1CaWX" }, "outputs": [], @@ -310,7 +270,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "cIAcvQqMCaWf" }, "source": [ @@ -319,10 +278,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "TRFYHB2mCaWb" }, "outputs": [], @@ -333,7 +290,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YSlYxFuRCaWk" }, "source": [ @@ -342,10 +298,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "XKnCTHz4CaWg" }, "outputs": [], @@ -356,7 +310,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TMPI88iZpO2T" }, "source": [ @@ -365,10 +318,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "2KFnYlcwCaWl" }, "outputs": [], @@ -379,7 +330,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rd0A0Iu0CaWq" }, "source": [ @@ -388,10 +338,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "iJmPr5-ACaWn" }, "outputs": [], @@ -402,7 +350,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ES6uQoLKCaWr" }, "source": [ @@ -413,10 +360,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "m4VEw8Ud9Quh" }, "outputs": [], @@ -431,7 +376,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Wz7l27Lz9S1P" }, "source": [ @@ -440,10 +384,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "bW5WzIPlCaWv" }, "outputs": [], @@ -456,7 +398,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ee638AlnCaWz" }, "source": [ @@ -465,10 +406,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "oZTImqg_CaW1" }, "outputs": [], @@ -487,7 +426,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "59veuiEZCaW4" }, "source": [ @@ -499,73 +437,66 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Gxg1XGm0eOBy" }, "source": [ "### Set up the layers\n", "\n", - "The basic building block of a neural network is the *layer*. Layers extract representations from the data fed into them. Hopefully, these representations are meaningful for the problem at hand.\n", + "The basic building block of a neural network is the [*layer*](https://www.tensorflow.org/api_docs/python/tf/keras/layers). Layers extract representations from the data fed into them. Hopefully, these representations are meaningful for the problem at hand.\n", "\n", "Most of deep learning consists of chaining together simple layers. Most layers, such as `tf.keras.layers.Dense`, have parameters that are learned during training." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "9ODch-OFCaW4" }, "outputs": [], "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation='relu'),\n", - " keras.layers.Dense(10, activation='softmax')\n", + "model = tf.keras.Sequential([\n", + " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", + " tf.keras.layers.Dense(128, activation='relu'),\n", + " tf.keras.layers.Dense(10)\n", "])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gut8A_7rCaW6" }, "source": [ "The first layer in this network, `tf.keras.layers.Flatten`, transforms the format of the images from a two-dimensional array (of 28 by 28 pixels) to a one-dimensional array (of 28 * 28 = 784 pixels). Think of this layer as unstacking rows of pixels in the image and lining them up. This layer has no parameters to learn; it only reformats the data.\n", "\n", - "After the pixels are flattened, the network consists of a sequence of two `tf.keras.layers.Dense` layers. These are densely connected, or fully connected, neural layers. The first `Dense` layer has 128 nodes (or neurons). The second (and last) layer is a 10-node *softmax* layer that returns an array of 10 probability scores that sum to 1. Each node contains a score that indicates the probability that the current image belongs to one of the 10 classes.\n", + "After the pixels are flattened, the network consists of a sequence of two `tf.keras.layers.Dense` layers. These are densely connected, or fully connected, neural layers. The first `Dense` layer has 128 nodes (or neurons). The second (and last) layer returns a logits array with length of 10. Each node contains a score that indicates the current image belongs to one of the 10 classes.\n", "\n", "### Compile the model\n", "\n", - "Before the model is ready for training, it needs a few more settings. These are added during the model's *compile* step:\n", + "Before the model is ready for training, it needs a few more settings. These are added during the model's [*compile*](https://www.tensorflow.org/api_docs/python/tf/keras/Model#compile) step:\n", "\n", - "* *Loss function* —This measures how accurate the model is during training. You want to minimize this function to \"steer\" the model in the right direction.\n", - "* *Optimizer* —This is how the model is updated based on the data it sees and its loss function.\n", - "* *Metrics* —Used to monitor the training and testing steps. The following example uses *accuracy*, the fraction of the images that are correctly classified." + "* [*Optimizer*](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers) —This is how the model is updated based on the data it sees and its loss function.\n", + "* [*Loss function*](https://www.tensorflow.org/api_docs/python/tf/keras/losses) —This measures how accurate the model is during training. You want to minimize this function to \"steer\" the model in the right direction.\n", + "* [*Metrics*](https://www.tensorflow.org/api_docs/python/tf/keras/metrics) —Used to monitor the training and testing steps. The following example uses *accuracy*, the fraction of the images that are correctly classified." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Lhan11blCaW7" }, "outputs": [], "source": [ "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qKF6uW-BCaW-" }, "source": [ @@ -575,17 +506,25 @@ "\n", "1. Feed the training data to the model. In this example, the training data is in the `train_images` and `train_labels` arrays.\n", "2. The model learns to associate images and labels.\n", - "3. You ask the model to make predictions about a test set—in this example, the `test_images` array. Verify that the predictions match the labels from the `test_labels` array.\n", + "3. You ask the model to make predictions about a test set—in this example, the `test_images` array.\n", + "4. Verify that the predictions match the labels from the `test_labels` array.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z4P4zIV7E28Z" + }, + "source": [ + "### Feed the model\n", "\n", - "To start training, call the `model.fit` method—so called because it \"fits\" the model to the training data:" + "To start training, call the [`model.fit`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#fit) method—so called because it \"fits\" the model to the training data:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "xvwvpA64CaW_" }, "outputs": [], @@ -596,31 +535,27 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "W3ZVOhugCaXA" }, "source": [ - "As the model trains, the loss and accuracy metrics are displayed. This model reaches an accuracy of about 0.88 (or 88%) on the training data." + "As the model trains, the loss and accuracy metrics are displayed. This model reaches an accuracy of about 0.91 (or 91%) on the training data." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" + "id": "wCpr6DGyE28h" }, "source": [ - "## Evaluate accuracy\n", + "### Evaluate accuracy\n", "\n", "Next, compare how the model performs on the test dataset:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "VflXLEeECaXC" }, "outputs": [], @@ -633,42 +568,52 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "yWfgsmVXCaXG" }, "source": [ - "It turns out that the accuracy on the test dataset is a little less than the accuracy on the training dataset. This gap between training accuracy and test accuracy represents *overfitting*. Overfitting is when a machine learning model performs worse on new, previously unseen inputs than on the training data." + "It turns out that the accuracy on the test dataset is a little less than the accuracy on the training dataset. This gap between training accuracy and test accuracy represents *overfitting*. Overfitting happens when a machine learning model performs worse on new, previously unseen inputs than it does on the training data. An overfitted model \"memorizes\" the noise and details in the training dataset to a point where it negatively impacts the performance of the model on the new data. For more information, see the following:\n", + "* [Demonstrate overfitting](https://www.tensorflow.org/tutorials/keras/overfit_and_underfit#demonstrate_overfitting)\n", + "* [Strategies to prevent overfitting](https://www.tensorflow.org/tutorials/keras/overfit_and_underfit#strategies_to_prevent_overfitting)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" + "id": "v-PyD1SYE28q" }, "source": [ - "## Make predictions\n", + "### Make predictions\n", "\n", - "With the model trained, you can use it to make predictions about some images." + "With the model trained, you can use it to make predictions about some images.\n", + "Attach a softmax layer to convert the model's linear outputs—[logits](https://developers.google.com/machine-learning/glossary#logits)—to probabilities, which should be easier to interpret." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, + "metadata": { + "id": "DnfNA0CrQLSD" + }, + "outputs": [], + "source": [ + "probability_model = tf.keras.Sequential([model, \n", + " tf.keras.layers.Softmax()])" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Gl91RPhdCaXI" }, "outputs": [], "source": [ - "predictions = model.predict(test_images)" + "predictions = probability_model.predict(test_images)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "x9Kk1voUCaXJ" }, "source": [ @@ -677,10 +622,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3DmJEUinCaXK" }, "outputs": [], @@ -691,7 +634,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-hw1hgeSCaXN" }, "source": [ @@ -700,10 +642,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "qsqenuPnCaXO" }, "outputs": [], @@ -714,7 +654,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "E51yS7iCCaXO" }, "source": [ @@ -723,10 +662,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Sd7Pgsu6CaXP" }, "outputs": [], @@ -737,25 +674,22 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ygh2yYC972ne" }, "source": [ - "Graph this to look at the full set of 10 class predictions." + "Define functions to graph the full set of 10 class predictions." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "DvYmmrpIy6Y1" }, "outputs": [], "source": [ "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array, true_label[i], img[i]\n", + " true_label, img = true_label[i], img[i]\n", " plt.grid(False)\n", " plt.xticks([])\n", " plt.yticks([])\n", @@ -774,7 +708,7 @@ " color=color)\n", "\n", "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array, true_label[i]\n", + " true_label = true_label[i]\n", " plt.grid(False)\n", " plt.xticks(range(10))\n", " plt.yticks([])\n", @@ -789,7 +723,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "Zh9yABaME29S" + }, + "source": [ + "### Verify predictions\n", + "\n", + "With the model trained, you can use it to make predictions about some images." + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "d4Ov9OFDMmOD" }, "source": [ @@ -798,10 +742,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "HV5jw-5HwSmO" }, "outputs": [], @@ -817,10 +759,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Ko-uzOufSCSe" }, "outputs": [], @@ -837,7 +777,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kgdvGD52CaXR" }, "source": [ @@ -846,10 +785,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "hQlnbqaw2Qu_" }, "outputs": [], @@ -872,19 +809,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "R32zteKHCaXT" }, "source": [ + "## Use the trained model\n", + "\n", "Finally, use the trained model to make a prediction about a single image." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "yRJ7JU7JCaXT" }, "outputs": [], @@ -898,7 +834,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vz3bVp21CaXV" }, "source": [ @@ -907,10 +842,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "lDFh5yF_CaXW" }, "outputs": [], @@ -924,7 +857,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "EQ5wLTkcCaXY" }, "source": [ @@ -933,49 +865,43 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "o_rzNSdrCaXY" }, "outputs": [], "source": [ - "predictions_single = model.predict(img)\n", + "predictions_single = probability_model.predict(img)\n", "\n", "print(predictions_single)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "6Ai-cpLjO-3A" }, "outputs": [], "source": [ "plot_value_array(1, predictions_single[0], test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" + "_ = plt.xticks(range(10), class_names, rotation=45)\n", + "plt.show()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "cU1Y2OAMCaXb" }, "source": [ - "`model.predict` returns a list of lists—one list for each image in the batch of data. Grab the predictions for our (only) image in the batch:" + "`tf.keras.Model.predict` returns a list of lists—one list for each image in the batch of data. Grab the predictions for our (only) image in the batch:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "2tRmdq_8CaXb" }, "outputs": [], @@ -986,11 +912,12 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YFc2HbEVCaXd" }, "source": [ - "And the model predicts a label as expected." + "And the model predicts a label as expected.\n", + "\n", + "To learn more about building models with Keras, see the [Keras guides](https://www.tensorflow.org/guide/keras)." ] } ], @@ -998,8 +925,6 @@ "colab": { "collapsed_sections": [], "name": "classification.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/keras/keras_tuner.ipynb b/site/en/tutorials/keras/keras_tuner.ipynb new file mode 100644 index 00000000000..7e20fefb869 --- /dev/null +++ b/site/en/tutorials/keras/keras_tuner.ipynb @@ -0,0 +1,428 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Tce3stUlHN0L" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "tuOe1ymfHZPu" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qFdPvlXBOdUN" + }, + "source": [ + "# Introduction to the Keras Tuner" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MfBg1C5NB3X0" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xHxb-dlhMIzW" + }, + "source": [ + "## Overview\n", + "\n", + "The Keras Tuner is a library that helps you pick the optimal set of hyperparameters for your TensorFlow program. The process of selecting the right set of hyperparameters for your machine learning (ML) application is called *hyperparameter tuning* or *hypertuning*.\n", + "\n", + "Hyperparameters are the variables that govern the training process and the topology of an ML model. These variables remain constant over the training process and directly impact the performance of your ML program. Hyperparameters are of two types:\n", + "1. **Model hyperparameters** which influence model selection such as the number and width of hidden layers\n", + "2. **Algorithm hyperparameters** which influence the speed and quality of the learning algorithm such as the learning rate for Stochastic Gradient Descent (SGD) and the number of nearest neighbors for a k Nearest Neighbors (KNN) classifier\n", + "\n", + "In this tutorial, you will use the Keras Tuner to perform hypertuning for an image classification application." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MUXex9ctTuDB" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IqR2PQG4ZaZ0" + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "from tensorflow import keras" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "g83Lwsy-Aq2_" + }, + "source": [ + "Install and import the Keras Tuner." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hpMLpbt9jcO6" + }, + "outputs": [], + "source": [ + "!pip install -q -U keras-tuner" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_leAIdFKAxAD" + }, + "outputs": [], + "source": [ + "import keras_tuner as kt" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ReV_UXOgCZvx" + }, + "source": [ + "## Download and prepare the dataset\n", + "\n", + "In this tutorial, you will use the Keras Tuner to find the best hyperparameters for a machine learning model that classifies images of clothing from the [Fashion MNIST dataset](https://github.com/zalandoresearch/fashion-mnist)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HljH_ENLEdHa" + }, + "source": [ + "Load the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OHlHs9Wj_PUM" + }, + "outputs": [], + "source": [ + "(img_train, label_train), (img_test, label_test) = keras.datasets.fashion_mnist.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bLVhXs3xrUD0" + }, + "outputs": [], + "source": [ + "# Normalize pixel values between 0 and 1\n", + "img_train = img_train.astype('float32') / 255.0\n", + "img_test = img_test.astype('float32') / 255.0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K5YEL2H2Ax3e" + }, + "source": [ + "## Define the model\n", + "\n", + "When you build a model for hypertuning, you also define the hyperparameter search space in addition to the model architecture. The model you set up for hypertuning is called a *hypermodel*.\n", + "\n", + "You can define a hypermodel through two approaches:\n", + "\n", + "* By using a model builder function\n", + "* By subclassing the `HyperModel` class of the Keras Tuner API\n", + "\n", + "You can also use two pre-defined [HyperModel](https://keras.io/api/keras_tuner/hypermodels/) classes - [HyperXception](https://keras.io/api/keras_tuner/hypermodels/hyper_xception/) and [HyperResNet](https://keras.io/api/keras_tuner/hypermodels/hyper_resnet/) for computer vision applications.\n", + "\n", + "In this tutorial, you use a model builder function to define the image classification model. The model builder function returns a compiled model and uses hyperparameters you define inline to hypertune the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZQKodC-jtsva" + }, + "outputs": [], + "source": [ + "def model_builder(hp):\n", + " model = keras.Sequential()\n", + " model.add(keras.layers.Flatten(input_shape=(28, 28)))\n", + "\n", + " # Tune the number of units in the first Dense layer\n", + " # Choose an optimal value between 32-512\n", + " hp_units = hp.Int('units', min_value=32, max_value=512, step=32)\n", + " model.add(keras.layers.Dense(units=hp_units, activation='relu'))\n", + " model.add(keras.layers.Dense(10))\n", + "\n", + " # Tune the learning rate for the optimizer\n", + " # Choose an optimal value from 0.01, 0.001, or 0.0001\n", + " hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])\n", + "\n", + " model.compile(optimizer=keras.optimizers.Adam(learning_rate=hp_learning_rate),\n", + " loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])\n", + "\n", + " return model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0J1VYw4q3x0b" + }, + "source": [ + "## Instantiate the tuner and perform hypertuning\n", + "\n", + "Instantiate the tuner to perform the hypertuning. The Keras Tuner has four tuners available - `RandomSearch`, `Hyperband`, `BayesianOptimization`, and `Sklearn`. In this tutorial, you use the [Hyperband](https://arxiv.org/pdf/1603.06560.pdf) tuner.\n", + "\n", + "To instantiate the Hyperband tuner, you must specify the hypermodel, the `objective` to optimize and the maximum number of epochs to train (`max_epochs`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "oichQFly6Y46" + }, + "outputs": [], + "source": [ + "tuner = kt.Hyperband(model_builder,\n", + " objective='val_accuracy',\n", + " max_epochs=10,\n", + " factor=3,\n", + " directory='my_dir',\n", + " project_name='intro_to_kt')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VaIhhdKf9VtI" + }, + "source": [ + "The Hyperband tuning algorithm uses adaptive resource allocation and early-stopping to quickly converge on a high-performing model. This is done using a sports championship style bracket. The algorithm trains a large number of models for a few epochs and carries forward only the top-performing half of models to the next round. Hyperband determines the number of models to train in a bracket by computing 1 + log`factor`(`max_epochs`) and rounding it up to the nearest integer." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cwhBdXx0Ekj8" + }, + "source": [ + "Create a callback to stop training early after reaching a certain value for the validation loss." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WT9IkS9NEjLc" + }, + "outputs": [], + "source": [ + "stop_early = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UKghEo15Tduy" + }, + "source": [ + "Run the hyperparameter search. The arguments for the search method are the same as those used for `tf.keras.model.fit` in addition to the callback above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dSBQcTHF9cKt" + }, + "outputs": [], + "source": [ + "tuner.search(img_train, label_train, epochs=50, validation_split=0.2, callbacks=[stop_early])\n", + "\n", + "# Get the optimal hyperparameters\n", + "best_hps=tuner.get_best_hyperparameters(num_trials=1)[0]\n", + "\n", + "print(f\"\"\"\n", + "The hyperparameter search is complete. The optimal number of units in the first densely-connected\n", + "layer is {best_hps.get('units')} and the optimal learning rate for the optimizer\n", + "is {best_hps.get('learning_rate')}.\n", + "\"\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Lak_ylf88xBv" + }, + "source": [ + "## Train the model\n", + "\n", + "Find the optimal number of epochs to train the model with the hyperparameters obtained from the search." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "McO82AXOuxXh" + }, + "outputs": [], + "source": [ + "# Build the model with the optimal hyperparameters and train it on the data for 50 epochs\n", + "model = tuner.hypermodel.build(best_hps)\n", + "history = model.fit(img_train, label_train, epochs=50, validation_split=0.2)\n", + "\n", + "val_acc_per_epoch = history.history['val_accuracy']\n", + "best_epoch = val_acc_per_epoch.index(max(val_acc_per_epoch)) + 1\n", + "print('Best epoch: %d' % (best_epoch,))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uOTSirSTI3Gp" + }, + "source": [ + "Re-instantiate the hypermodel and train it with the optimal number of epochs from above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NoiPUEHmMhCe" + }, + "outputs": [], + "source": [ + "hypermodel = tuner.hypermodel.build(best_hps)\n", + "\n", + "# Retrain the model\n", + "hypermodel.fit(img_train, label_train, epochs=best_epoch, validation_split=0.2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MqU5ZVAaag2v" + }, + "source": [ + "To finish this tutorial, evaluate the hypermodel on the test data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9E0BTp9Ealjb" + }, + "outputs": [], + "source": [ + "eval_result = hypermodel.evaluate(img_test, label_test)\n", + "print(\"[test loss, test accuracy]:\", eval_result)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EQRpPHZsz-eC" + }, + "source": [ + "The `my_dir/intro_to_kt` directory contains detailed logs and checkpoints for every trial (model configuration) run during the hyperparameter search. If you re-run the hyperparameter search, the Keras Tuner uses the existing state from these logs to resume the search. To disable this behavior, pass an additional `overwrite=True` argument while instantiating the tuner." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sKwLOzKpFGAj" + }, + "source": [ + "## Summary\n", + "\n", + "In this tutorial, you learned how to use the Keras Tuner to tune hyperparameters for a model. To learn more about the Keras Tuner, check out these additional resources:\n", + "\n", + "* [Keras Tuner on the TensorFlow blog](https://blog.tensorflow.org/2020/01/hyperparameter-tuning-with-keras-tuner.html)\n", + "* [Keras Tuner website](https://keras-team.github.io/keras-tuner/)\n", + "\n", + "Also check out the [HParams Dashboard](https://www.tensorflow.org/tensorboard/hyperparameter_tuning_with_hparams) in TensorBoard to interactively tune your model hyperparameters." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [ + "Tce3stUlHN0L" + ], + "name": "keras_tuner.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/keras/overfit_and_underfit.ipynb b/site/en/tutorials/keras/overfit_and_underfit.ipynb index 3e47a82be00..29fbfc49c80 100644 --- a/site/en/tutorials/keras/overfit_and_underfit.ipynb +++ b/site/en/tutorials/keras/overfit_and_underfit.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "fTFj8ft5dlbS" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "lzyBOpYMdp3F" }, "outputs": [], @@ -36,11 +33,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "m_x4KfSJ7Vt7" }, "outputs": [], @@ -71,7 +66,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "C9HmC2T4ld5B" }, "source": [ @@ -81,54 +75,51 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kRTxFhXAlnl1" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/keras/overfit_and_underfit\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/keras/overfit_and_underfit.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/overfit_and_underfit.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/keras/overfit_and_underfit.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "19rPukKZsPG6" }, "source": [ "As always, the code in this example will use the `tf.keras` API, which you can learn more about in the TensorFlow [Keras guide](https://www.tensorflow.org/guide/keras).\n", "\n", - "In both of the previous examples—[classifying text](https://www.tensorflow.org/tutorials/keras/text_classification_with_hub) and [predicting fuel efficiency](https://www.tensorflow.org/tutorials/keras/regression) — we saw that the accuracy of our model on the validation data would peak after training for a number of epochs, and would then stagnate or start decreasing.\n", + "In both of the previous examples—[classifying text](text_classification_with_hub.ipynb) and [predicting fuel efficiency](regression.ipynb)—the accuracy of models on the validation data would peak after training for a number of epochs and then stagnate or start decreasing.\n", "\n", - "In other words, our model would *overfit* to the training data. Learning how to deal with overfitting is important. Although it's often possible to achieve high accuracy on the *training set*, what we really want is to develop models that generalize well to a *testing set* (or data they haven't seen before).\n", + "In other words, your model would *overfit* to the training data. Learning how to deal with overfitting is important. Although it's often possible to achieve high accuracy on the *training set*, what you really want is to develop models that generalize well to a *testing set* (or data they haven't seen before).\n", "\n", - "The opposite of overfitting is *underfitting*. Underfitting occurs when there is still room for improvement on the test data. This can happen for a number of reasons: If the model is not powerful enough, is over-regularized, or has simply not been trained long enough. This means the network has not learned the relevant patterns in the training data.\n", + "The opposite of overfitting is *underfitting*. Underfitting occurs when there is still room for improvement on the train data. This can happen for a number of reasons: If the model is not powerful enough, is over-regularized, or has simply not been trained long enough. This means the network has not learned the relevant patterns in the training data.\n", "\n", - "If you train for too long though, the model will start to overfit and learn patterns from the training data that don't generalize to the test data. We need to strike a balance. Understanding how to train for an appropriate number of epochs as we'll explore below is a useful skill.\n", + "If you train for too long though, the model will start to overfit and learn patterns from the training data that don't generalize to the test data. You need to strike a balance. Understanding how to train for an appropriate number of epochs as you'll explore below is a useful skill.\n", "\n", "To prevent overfitting, the best solution is to use more complete training data. The dataset should cover the full range of inputs that the model is expected to handle. Additional data may only be useful if it covers new and interesting cases.\n", "\n", "A model trained on more complete data will naturally generalize better. When that is no longer possible, the next best solution is to use techniques like regularization. These place constraints on the quantity and type of information your model can store. If a network can only afford to memorize a small number of patterns, the optimization process will force it to focus on the most prominent patterns, which have a better chance of generalizing well.\n", "\n", - "In this notebook, we'll explore several common regularization techniques, and use them to improve on a classification model." + "In this notebook, you'll explore several common regularization techniques, and use them to improve on a classification model." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WL8UoOTmGGsL" }, "source": [ @@ -138,7 +129,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9FklhSI0Gg9R" }, "source": [ @@ -147,21 +137,12 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "5pZ8A2liqvgk" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", "\n", "from tensorflow.keras import layers\n", @@ -172,10 +153,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "QnAtAjqRYVXe" }, "outputs": [], @@ -189,10 +168,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "-pnOU-ctX27Q" }, "outputs": [], @@ -209,10 +186,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "jj6I4dvTtbUe" }, "outputs": [], @@ -224,35 +199,29 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1cweoTiruj8O" }, "source": [ + "## The Higgs dataset\n", "\n", - "## The Higgs Dataset\n", - "\n", - "The goal of this tutorial is not to do particle physics, so don't dwell on the details of the dataset. It contains 11\u0026#x202F;000\u0026#x202F;000 examples, each with 28 features, and a binary class label." + "The goal of this tutorial is not to do particle physics, so don't dwell on the details of the dataset. It contains 11,000,000 examples, each with 28 features, and a binary class label." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YPjAvwb-6dFd" }, "outputs": [], "source": [ - "gz = tf.keras.utils.get_file('HIGGS.csv.gz', 'https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz')" + "gz = tf.keras.utils.get_file('HIGGS.csv.gz', 'http://mlphysics.ics.uci.edu/data/higgs/HIGGS.csv.gz')" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "AkiyUdaWIrww" }, "outputs": [], @@ -263,7 +232,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SFggl9gYKKRJ" }, "source": [ @@ -272,10 +240,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "QHz4sLVQEVIU" }, "outputs": [], @@ -286,7 +252,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HzahEELTKlSV" }, "source": [ @@ -295,10 +260,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "zPD6ICDlF6Wf" }, "outputs": [], @@ -312,21 +275,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4oa8tLuwLsbO" }, "source": [ "TensorFlow is most efficient when operating on large batches of data.\n", "\n", - "So instead of repacking each row individually make a new `Dataset` that takes batches of 10000-examples, applies the `pack_row` function to each batch, and then splits the batches back up into individual records:" + "So, instead of repacking each row individually make a new `tf.data.Dataset` that takes batches of 10,000 examples, applies the `pack_row` function to each batch, and then splits the batches back up into individual records:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "-w-VHTwwGVoZ" }, "outputs": [], @@ -337,21 +297,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lUbxc5bxNSXV" }, "source": [ - "Have a look at some of the records from this new `packed_ds`.\n", + "Inspect some of the records from this new `packed_ds`.\n", "\n", "The features are not perfectly normalized, but this is sufficient for this tutorial." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "TfcXuv33Fvka" }, "outputs": [], @@ -364,19 +321,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ICKZRY7gN-QM" }, "source": [ - "To keep this tutorial relatively short use just the first 1000 samples for validation, and the next 10 000 for training:" + "To keep this tutorial relatively short, use just the first 1,000 samples for validation, and the next 10,000 for training:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "hmk49OqZIFZP" }, "outputs": [], @@ -391,21 +345,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FP3M9DmvON32" }, "source": [ "The `Dataset.skip` and `Dataset.take` methods make this easy.\n", "\n", - "At the same time, use the `Dataset.cache` method to ensure that the loader doesn't need to re-read the data form the file on each epoch:" + "At the same time, use the `Dataset.cache` method to ensure that the loader doesn't need to re-read the data from the file on each epoch:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "H8H_ZzpBOOk-" }, "outputs": [], @@ -416,10 +367,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "9zAOqk2_Px7K" }, "outputs": [], @@ -430,19 +379,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6PMliHoVO3OL" }, "source": [ - "These datasets return individual examples. Use the `.batch` method to create batches of an appropriate size for training. Before batching also remember to `.shuffle` and `.repeat` the training set." + "These datasets return individual examples. Use the `Dataset.batch` method to create batches of an appropriate size for training. Before batching, also remember to use `Dataset.shuffle` and `Dataset.repeat` on the training set." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Y7I4J355O223" }, "outputs": [], @@ -454,7 +400,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lglk41MwvU5o" }, "source": [ @@ -472,13 +417,12 @@ "\n", "To find an appropriate model size, it's best to start with relatively few layers and parameters, then begin increasing the size of the layers or adding new layers until you see diminishing returns on the validation loss.\n", "\n", - "Start with a simple model using only `layers.Dense` as a baseline, then create larger versions, and compare them." + "Start with a simple model using only densely-connected layers (`tf.keras.layers.Dense`) as a baseline, then create larger models, and compare them." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_ReKHdC2EgVu" }, "source": [ @@ -488,19 +432,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pNzkSkkXSP5l" }, "source": [ - "Many models train better if you gradually reduce the learning rate during training. Use `optimizers.schedules` to reduce the learning rate over time:" + "Many models train better if you gradually reduce the learning rate during training. Use `tf.keras.optimizers.schedules` to reduce the learning rate over time:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "LwQp-ERhAD6F" }, "outputs": [], @@ -518,19 +459,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kANLx6OYTQ8B" }, "source": [ - "The code above sets a `schedules.InverseTimeDecay` to hyperbolically decrease the learning rate to 1/2 of the base rate at 1000 epochs, 1/3 at 2000 epochs and so on." + "The code above sets a `tf.keras.optimizers.schedules.InverseTimeDecay` to hyperbolically decrease the learning rate to 1/2 of the base rate at 1,000 epochs, 1/3 at 2,000 epochs, and so on." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "HIo_yPjEAFgn" }, "outputs": [], @@ -547,26 +485,22 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ya7x7gr9UjU0" }, "source": [ "Each model in this tutorial will use the same training configuration. So set these up in a reusable way, starting with the list of callbacks.\n", "\n", - "The training for this tutorial runs for many short epochs. To reduce the logging noise use the `tfdocs.EpochDots` which simply a `.` for each epoch and, and a full set of metrics every 100 epochs.\n", + "The training for this tutorial runs for many short epochs. To reduce the logging noise use the `tfdocs.EpochDots` which simply prints a `.` for each epoch, and a full set of metrics every 100 epochs.\n", "\n", - "Next include `callbacks.EarlyStopping` to avoid long and unnecessary training times. Note that this callback is set to monitor the `val_binary_crossentropy`, not the `val_loss`. This difference will be important later.\n", + "Next include `tf.keras.callbacks.EarlyStopping` to avoid long and unnecessary training times. Note that this callback is set to monitor the `val_binary_crossentropy`, not the `val_loss`. This difference will be important later.\n", "\n", - "Use `callbacks.TensorBoard` to generate TensorBoard logs for the training.\n", - "\n" + "Use `callbacks.TensorBoard` to generate TensorBoard logs for the training.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "vSv8rfw_T85n" }, "outputs": [], @@ -582,7 +516,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VhctzKhBWVDD" }, "source": [ @@ -591,10 +524,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "xRCGwU3YH5sT" }, "outputs": [], @@ -603,16 +534,19 @@ " if optimizer is None:\n", " optimizer = get_optimizer()\n", " model.compile(optimizer=optimizer,\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " metrics=[\n", + " tf.keras.metrics.BinaryCrossentropy(\n", + " from_logits=True, name='binary_crossentropy'),\n", + " 'accuracy'])\n", "\n", " model.summary()\n", "\n", " history = model.fit(\n", - " train_ds,\n", + " train_ds.map(lambda x, y: (x, tf.expand_dims(y, axis=-1))),\n", " steps_per_epoch = STEPS_PER_EPOCH,\n", " epochs=max_epochs,\n", - " validation_data=validate_ds,\n", + " validation_data=validate_ds.map(lambda x, y: (x, tf.expand_dims(y, axis=-1))),\n", " callbacks=get_callbacks(name),\n", " verbose=0)\n", " return history" @@ -621,7 +555,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mxBeiLUiWHJV" }, "source": [ @@ -631,35 +564,30 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "a6JDv12scLTI" }, "source": [ - "Start by training a linear model:" + "Start by training a model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "EZh-QFjKHb70" }, "outputs": [], "source": [ "tiny_model = tf.keras.Sequential([\n", " layers.Dense(16, activation='elu', input_shape=(FEATURES,)),\n", - " layers.Dense(1, activation='sigmoid')\n", + " layers.Dense(1)\n", "])" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "X72IUdWYipIS" }, "outputs": [], @@ -669,10 +597,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "bdOcJtPGHhJ5" }, "outputs": [], @@ -683,7 +609,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rS_QGT6icwdI" }, "source": [ @@ -692,10 +617,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dkEvb2x5XsjE" }, "outputs": [], @@ -708,7 +631,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LGxGzh_FWOJ8" }, "source": [ @@ -718,21 +640,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YjMb6E72f2pN" }, "source": [ - "To see if you can beat the performance of the small model, progressively train some larger models.\n", + "To check if you can beat the performance of the small model, progressively train some larger models.\n", "\n", "Try two hidden layers with 16 units each:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "QKgdXPx9usBa" }, "outputs": [], @@ -741,16 +660,14 @@ " # `input_shape` is only required here so that `.summary` works.\n", " layers.Dense(16, activation='elu', input_shape=(FEATURES,)),\n", " layers.Dense(16, activation='elu'),\n", - " layers.Dense(1, activation='sigmoid')\n", + " layers.Dense(1)\n", "])" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "LqG3MXF5xSjR" }, "outputs": [], @@ -761,7 +678,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "L-DGRBbGxI6G" }, "source": [ @@ -771,19 +687,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SrfoVQheYSO5" }, "source": [ - "Now try 3 hidden layers with 64 units each:" + "Now try three hidden layers with 64 units each:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "jksi-XtaxDAh" }, "outputs": [], @@ -792,14 +705,13 @@ " layers.Dense(64, activation='elu', input_shape=(FEATURES,)),\n", " layers.Dense(64, activation='elu'),\n", " layers.Dense(64, activation='elu'),\n", - " layers.Dense(1, activation='sigmoid')\n", + " layers.Dense(1)\n", "])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jbngCZliYdma" }, "source": [ @@ -808,10 +720,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Ofn1AwDhx-Fe" }, "outputs": [], @@ -822,21 +732,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vIPuf23FFaVn" }, "source": [ "### Large model\n", "\n", - "As an exercise, you can create an even larger model, and see how quickly it begins overfitting. Next, let's add to this benchmark a network that has much more capacity, far more than the problem would warrant:" + "As an exercise, you can create an even larger model and check how quickly it begins overfitting. Next, add to this benchmark a network that has much more capacity, far more than the problem would warrant:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ghQwwqwqvQM9" }, "outputs": [], @@ -846,14 +753,13 @@ " layers.Dense(512, activation='elu'),\n", " layers.Dense(512, activation='elu'),\n", " layers.Dense(512, activation='elu'),\n", - " layers.Dense(1, activation='sigmoid')\n", + " layers.Dense(1)\n", "])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "D-d-i5DaYmr7" }, "source": [ @@ -862,10 +768,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "U1A99dhqvepf" }, "outputs": [], @@ -876,7 +780,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Fy3CMUZpzH3d" }, "source": [ @@ -886,7 +789,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HSlo1F4xHuuM" }, "source": [ @@ -896,13 +798,12 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OLhL1AszdLfM" }, "source": [ "While building a larger model gives it more power, if this power is not constrained somehow it can easily overfit to the training set.\n", "\n", - "In this example, typically, only the `\"Tiny\"` model manages to avoid overfitting altogether, and each of the larger models overfit the data more quickly. This becomes so severe for the `\"large\"` model that you need to switch the plot to a log-scale to really see what's happening.\n", + "In this example, typically, only the `\"Tiny\"` model manages to avoid overfitting altogether, and each of the larger models overfit the data more quickly. This becomes so severe for the `\"large\"` model that you need to switch the plot to a log-scale to really figure out what's happening.\n", "\n", "This is apparent if you plot and compare the validation metrics to the training metrics.\n", "\n", @@ -914,10 +815,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0XmKDtOWzOpk" }, "outputs": [], @@ -932,7 +831,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UekcaQdmZxnW" }, "source": [ @@ -942,7 +840,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DEQNKadHA0M3" }, "source": [ @@ -950,64 +847,37 @@ "\n", "These models all wrote TensorBoard logs during training.\n", "\n", - "To open an embedded TensorBoard viewer inside a notebook, copy the following into a code-cell:\n", - "\n", - "```\n", - "%tensorboard --logdir {logdir}/sizes\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fjqx3bywDPjf" - }, - "source": [ - "You can view the [results of a previous run](https://tensorboard.dev/experiment/vW7jmmF9TmKmy3rbheMQpw/#scalars\u0026_smoothingWeight=0.97) of this notebook on [TensorBoard.dev](https://tensorboard.dev/).\n", - "\n", - "TensorBoard.dev is a managed experience for hosting, tracking, and sharing ML experiments with everyone.\n", - "\n", - "It's also included in an `\u003ciframe\u003e` for convenience:" + "Open an embedded TensorBoard viewer inside a notebook (Sorry, this doesn't\n", + "display on tensorflow.org):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dX5fcgrADwym" + "id": "6oa1lkJddZ-m" }, "outputs": [], "source": [ - "display.IFrame(\n", - " src=\"https://tensorboard.dev/experiment/vW7jmmF9TmKmy3rbheMQpw/#scalars\u0026_smoothingWeight=0.97\",\n", - " width=\"100%\", height=\"800px\")" + "# Load the TensorBoard notebook extension\n", + "%load_ext tensorboard\n", + "\n", + "# Open an embedded TensorBoard viewer\n", + "%tensorboard --logdir {logdir}/sizes" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "RDQDBKYZBXF_" + "id": "HkIIzE5rBBY_" }, "source": [ - "If you want to share TensorBoard results you can upload the logs to [TensorBoard.dev](https://tensorboard.dev/) by copying the following into a code-cell.\n", - "\n", - "Note: This step requires a Google account.\n", - "\n", - "```\n", - "!tensorboard dev upload --logdir {logdir}/sizes\n", - "```\n", - "\n", - "Caution: This command does not terminate. It's designed to continuously upload the results of long-running experiments. Once your data is uploaded you need to stop it using the \"interrupt execution\" option in your notebook tool." + "You can view the [results of a previous run](https://tensorboard.dev/experiment/vW7jmmF9TmKmy3rbheMQpw/#scalars&_smoothingWeight=0.97) of this notebook on [TensorBoard.dev](https://tensorboard.dev/)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ASdv7nsgEFhx" }, "source": [ @@ -1017,7 +887,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YN512ksslaxJ" }, "source": [ @@ -1026,10 +895,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "40k1eBtnQzNo" }, "outputs": [], @@ -1040,10 +907,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "vFWMeFo7jLpN" }, "outputs": [], @@ -1055,40 +920,35 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4rHoVWcswFLa" }, "source": [ - "### Add weight regularization\n", - "\n" + "### Add weight regularization\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kRxWepNawbBK" }, "source": [ "You may be familiar with Occam's Razor principle: given two explanations for something, the explanation most likely to be correct is the \"simplest\" one, the one that makes the least amount of assumptions. This also applies to the models learned by neural networks: given some training data and a network architecture, there are multiple sets of weights values (multiple models) that could explain the data, and simpler models are less likely to overfit than complex ones.\n", "\n", - "A \"simple model\" in this context is a model where the distribution of parameter values has less entropy (or a model with fewer parameters altogether, as we saw in the section above). Thus a common way to mitigate overfitting is to put constraints on the complexity of a network by forcing its weights only to take small values, which makes the distribution of weight values more \"regular\". This is called \"weight regularization\", and it is done by adding to the loss function of the network a cost associated with having large weights. This cost comes in two flavors:\n", + "A \"simple model\" in this context is a model where the distribution of parameter values has less entropy (or a model with fewer parameters altogether, as demonstrated in the section above). Thus a common way to mitigate overfitting is to put constraints on the complexity of a network by forcing its weights only to take small values, which makes the distribution of weight values more \"regular\". This is called \"weight regularization\", and it is done by adding to the loss function of the network a cost associated with having large weights. This cost comes in two flavors:\n", "\n", "* [L1 regularization](https://developers.google.com/machine-learning/glossary/#L1_regularization), where the cost added is proportional to the absolute value of the weights coefficients (i.e. to what is called the \"L1 norm\" of the weights).\n", "\n", "* [L2 regularization](https://developers.google.com/machine-learning/glossary/#L2_regularization), where the cost added is proportional to the square of the value of the weights coefficients (i.e. to what is called the squared \"L2 norm\" of the weights). L2 regularization is also called weight decay in the context of neural networks. Don't let the different name confuse you: weight decay is mathematically the exact same as L2 regularization.\n", "\n", - "L1 regularization pushes weights towards exactly zero encouraging a sparse model. L2 regularization will penalize the weights parameters without making them sparse since the penalty goes to zero for small weights. one reason why L2 is more common.\n", + "L1 regularization pushes weights towards exactly zero, encouraging a sparse model. L2 regularization will penalize the weights parameters without making them sparse since the penalty goes to zero for small weights—one reason why L2 is more common.\n", "\n", - "In `tf.keras`, weight regularization is added by passing weight regularizer instances to layers as keyword arguments. Let's add L2 weight regularization now." + "In `tf.keras`, weight regularization is added by passing weight regularizer instances to layers as keyword arguments. Add L2 weight regularization:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "HFGmcwduwVyQ" }, "outputs": [], @@ -1103,7 +963,7 @@ " kernel_regularizer=regularizers.l2(0.001)),\n", " layers.Dense(512, activation='elu',\n", " kernel_regularizer=regularizers.l2(0.001)),\n", - " layers.Dense(1, activation='sigmoid')\n", + " layers.Dense(1)\n", "])\n", "\n", "regularizer_histories['l2'] = compile_and_fit(l2_model, \"regularizers/l2\")" @@ -1112,24 +972,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bUUHoXb7w-_C" }, "source": [ "`l2(0.001)` means that every coefficient in the weight matrix of the layer will add `0.001 * weight_coefficient_value**2` to the total **loss** of the network.\n", "\n", - "That is why we're monitoring the `binary_crossentropy` directly. Because it doesn't have this regularization component mixed in.\n", + "That is why you need to monitor the `binary_crossentropy` directly. Because it doesn't have this regularization component mixed in.\n", "\n", - "So, that same `\"Large\"` model with an `L2` regularization penalty performs much better:\n", - "\n" + "So, that same `\"Large\"` model with an `L2` regularization penalty performs much better:\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7wkfLyxBZdh_" }, "outputs": [], @@ -1141,57 +997,51 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Kx1YHMsVxWjP" }, "source": [ - "As you can see, the `\"L2\"` regularized model is now much more competitive with the the `\"Tiny\"` model. This `\"L2\"` model is also much more resistant to overfitting than the `\"Large\"` model it was based on despite having the same number of parameters." + "As demonstrated in the diagram above, the `\"L2\"` regularized model is now much more competitive with the `\"Tiny\"` model. This `\"L2\"` model is also much more resistant to overfitting than the `\"Large\"` model it was based on despite having the same number of parameters." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JheBk6f8jMQ7" }, "source": [ "#### More info\n", "\n", - "There are two important things to note about this sort of regularization.\n", + "There are two important things to note about this sort of regularization:\n", "\n", - "**First:** if you are writing your own training loop, then you need to be sure to ask the model for it's regularization losses." + "1. If you are writing your own training loop, then you need to be sure to ask the model for its regularization losses." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "apDHQNybjaML" }, "outputs": [], "source": [ "result = l2_model(features)\n", - "regularization_loss = tf.add_n(l2_model.losses)" + "regularization_loss=tf.add_n(l2_model.losses)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MLhG6fMSjE-J" }, "source": [ - "**Second:** This implementation works by adding the weight penalties to the model's loss, and then applying a standard optimization procedure after that.\n", + "2. This implementation works by adding the weight penalties to the model's loss, and then applying a standard optimization procedure after that.\n", "\n", - "There is a second approach that instead only runs the optimizer on the raw loss, and then while applying the calculated step the optimizer also applies some weight decay. This \"Decoupled Weight Decay\" Is seen in optimizers like `optimizers.FTRL` and `optimizers.AdamW`." + "There is a second approach that instead only runs the optimizer on the raw loss, and then while applying the calculated step the optimizer also applies some weight decay. This \"decoupled weight decay\" is used in optimizers like `tf.keras.optimizers.Ftrl` and `tfa.optimizers.AdamW`." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HmnBNOOVxiG8" }, "source": [ @@ -1201,22 +1051,19 @@ "\n", "The intuitive explanation for dropout is that because individual nodes in the network cannot rely on the output of the others, each node must output features that are useful on their own.\n", "\n", - "Dropout, applied to a layer, consists of randomly \"dropping out\" (i.e. set to zero) a number of output features of the layer during training. Let's say a given layer would normally have returned a vector [0.2, 0.5, 1.3, 0.8, 1.1] for a given input sample during training; after applying dropout, this vector will have a few zero entries distributed at random, e.g. [0, 0.5,\n", - "1.3, 0, 1.1].\n", + "Dropout, applied to a layer, consists of randomly \"dropping out\" (i.e. set to zero) a number of output features of the layer during training. For example, a given layer would normally have returned a vector `[0.2, 0.5, 1.3, 0.8, 1.1]` for a given input sample during training; after applying dropout, this vector will have a few zero entries distributed at random, e.g. `[0, 0.5, 1.3, 0, 1.1]`.\n", "\n", "The \"dropout rate\" is the fraction of the features that are being zeroed-out; it is usually set between 0.2 and 0.5. At test time, no units are dropped out, and instead the layer's output values are scaled down by a factor equal to the dropout rate, so as to balance for the fact that more units are active than at training time.\n", "\n", - "In `tf.keras` you can introduce dropout in a network via the Dropout layer, which gets applied to the output of layer right before.\n", + "In Keras, you can introduce dropout in a network via the `tf.keras.layers.Dropout` layer, which gets applied to the output of layer right before.\n", "\n", - "Let's add two Dropout layers in our network to see how well they do at reducing overfitting:" + "Add two dropout layers to your network to check how well they do at reducing overfitting:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "OFEYvtrHxSWS" }, "outputs": [], @@ -1230,7 +1077,7 @@ " layers.Dropout(0.5),\n", " layers.Dense(512, activation='elu'),\n", " layers.Dropout(0.5),\n", - " layers.Dense(1, activation='sigmoid')\n", + " layers.Dense(1)\n", "])\n", "\n", "regularizer_histories['dropout'] = compile_and_fit(dropout_model, \"regularizers/dropout\")" @@ -1238,10 +1085,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "SPZqwVchx5xp" }, "outputs": [], @@ -1253,7 +1098,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4zlHr4iaI1U6" }, "source": [ @@ -1265,7 +1109,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "u7qMg_7Nwy5t" }, "source": [ @@ -1274,10 +1117,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7zfs_qQIw1cz" }, "outputs": [], @@ -1295,7 +1136,7 @@ " layers.Dense(512, kernel_regularizer=regularizers.l2(0.0001),\n", " activation='elu'),\n", " layers.Dropout(0.5),\n", - " layers.Dense(1, activation='sigmoid')\n", + " layers.Dense(1)\n", "])\n", "\n", "regularizer_histories['combined'] = compile_and_fit(combined_model, \"regularizers/combined\")" @@ -1303,10 +1144,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "qDqBBxfI0Yd8" }, "outputs": [], @@ -1318,7 +1157,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tE0OoNCQNTJv" }, "source": [ @@ -1328,7 +1166,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-dw23T03FEO1" }, "source": [ @@ -1336,59 +1173,32 @@ "\n", "These models also recorded TensorBoard logs.\n", "\n", - "To open an embedded tensorboard viewer inside a notebook, copy the following into a code-cell:\n", - "\n", - "```\n", - "%tensorboard --logdir {logdir}/regularizers\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KX3Voac-FEO4" - }, - "source": [ - "You can view the [results of a previous run](https://tensorboard.dev/experiment/fGInKDo8TXes1z7HQku9mw/#scalars\u0026_smoothingWeight=0.97) of this notebook on [TensorDoard.dev](https://tensorboard.dev/).\n", - "\n", - "It's also included in an `\u003ciframe\u003e` for convenience:" + "To open an embedded run the following into a code-cell (Sorry, this doesn't display on tensorflow.org):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "doMtyYoqFEO5" + "id": "Op4vLqVWBK_y" }, "outputs": [], "source": [ - "display.IFrame(\n", - " src=\"https://tensorboard.dev/experiment/fGInKDo8TXes1z7HQku9mw/#scalars\u0026_smoothingWeight=0.97\",\n", - " width = \"100%\",\n", - " height=\"800px\")\n" + "%tensorboard --logdir {logdir}/regularizers" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "mds5RXGjIcSu" + "id": "_rx5b294BXBd" }, "source": [ - "This was uploaded with:\n", - "\n", - "```\n", - "!tensorboard dev upload --logdir {logdir}/regularizers\n", - "```" + "You can view the [results of a previous run](https://tensorboard.dev/experiment/vW7jmmF9TmKmy3rbheMQpw/#scalars&_smoothingWeight=0.97) of this notebook on [TensorBoard.dev](https://tensorboard.dev/)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uXJxtwBWIhjG" }, "source": [ @@ -1398,11 +1208,10 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gjfnkEeQyAFG" }, "source": [ - "To recap: here are the most common ways to prevent overfitting in neural networks:\n", + "To recap, here are the most common ways to prevent overfitting in neural networks:\n", "\n", "* Get more training data.\n", "* Reduce the capacity of the network.\n", @@ -1411,21 +1220,16 @@ "\n", "Two important approaches not covered in this guide are:\n", "\n", - "* data-augmentation\n", - "* batch normalization\n", + "* [Data augmentation](../images/data_augmentation.ipynb)\n", + "* Batch normalization (`tf.keras.layers.BatchNormalization`)\n", "\n", "Remember that each method can help on its own, but often combining them can be even more effective." ] } ], "metadata": { - "accelerator": "GPU", "colab": { - "collapsed_sections": [], - "machine_shape": "hm", "name": "overfit_and_underfit.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/keras/regression.ipynb b/site/en/tutorials/keras/regression.ipynb index 2eab2c3f46a..3010eb9afe0 100644 --- a/site/en/tutorials/keras/regression.ipynb +++ b/site/en/tutorials/keras/regression.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FhGuhbZ6M5tl" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "AwOEIRJC6Une" }, "outputs": [], @@ -36,11 +33,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "KyPEtTqk6VdG" }, "outputs": [], @@ -71,7 +66,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "EIdT9iu_Z4Rb" }, "source": [ @@ -81,92 +75,75 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bBIlTPscrIT9" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/keras/regression\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/keras/regression.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/regression.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/keras/regression.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AHp3M9ZmrIxj" }, "source": [ - "In a *regression* problem, we aim to predict the output of a continuous value, like a price or a probability. Contrast this with a *classification* problem, where we aim to select a class from a list of classes (for example, where a picture contains an apple or an orange, recognizing which fruit is in the picture).\n", + "In a *regression* problem, the aim is to predict the output of a continuous value, like a price or a probability. Contrast this with a *classification* problem, where the aim is to select a class from a list of classes (for example, where a picture contains an apple or an orange, recognizing which fruit is in the picture).\n", "\n", - "This notebook uses the classic [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) Dataset and builds a model to predict the fuel efficiency of late-1970s and early 1980s automobiles. To do this, we'll provide the model with a description of many automobiles from that time period. This description includes attributes like: cylinders, displacement, horsepower, and weight.\n", + "This tutorial uses the classic [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) dataset and demonstrates how to build models to predict the fuel efficiency of the late-1970s and early 1980s automobiles. To do this, you will provide the models with a description of many automobiles from that time period. This description includes attributes like cylinders, displacement, horsepower, and weight.\n", "\n", - "This example uses the `tf.keras` API, see [this guide](https://www.tensorflow.org/guide/keras) for details." + "This example uses the Keras API. (Visit the Keras [tutorials](https://www.tensorflow.org/tutorials/keras) and [guides](https://www.tensorflow.org/guide/keras) to learn more.)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "moB4tpEHxKB3" }, "outputs": [], "source": [ - "# Use seaborn for pairplot\n", - "!pip install seaborn\n", - "\n", - "# Use some functions from tensorflow_docs\n", - "!pip install git+https://github.com/tensorflow/docs" + "# Use seaborn for pairplot.\n", + "!pip install -q seaborn" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "1rRo8oNqZ-Rj" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", - "import seaborn as sns" + "import seaborn as sns\n", + "\n", + "# Make NumPy printouts easier to read.\n", + "np.set_printoptions(precision=3, suppress=True)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "9xQKvCJ85kCQ" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", "\n", "from tensorflow import keras\n", @@ -175,85 +152,52 @@ "print(tf.__version__)" ] }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Qz4HfsgRQUiV" - }, - "outputs": [], - "source": [ - "import tensorflow_docs as tfdocs\n", - "import tensorflow_docs.plots\n", - "import tensorflow_docs.modeling" - ] - }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "F_72b0LCNbjx" }, "source": [ "## The Auto MPG dataset\n", "\n", - "The dataset is available from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/).\n", - "\n" + "The dataset is available from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/).\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gFh9ne3FZ-On" }, "source": [ "### Get the data\n", - "First download the dataset." + "First download and import the dataset using pandas:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "p9kxxgzvzlyz" + "id": "CiX2FI4gZtTt" }, "outputs": [], "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "Import it using pandas" + "url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data'\n", + "column_names = ['MPG', 'Cylinders', 'Displacement', 'Horsepower', 'Weight',\n", + " 'Acceleration', 'Model Year', 'Origin']\n", + "\n", + "raw_dataset = pd.read_csv(url, names=column_names,\n", + " na_values='?', comment='\\t',\n", + " sep=' ', skipinitialspace=True)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "CiX2FI4gZtTt" + "id": "2oY3pMPagJrO" }, "outputs": [], "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", "dataset = raw_dataset.copy()\n", "dataset.tail()" ] @@ -261,21 +205,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3MWuJTKEDM-f" }, "source": [ "### Clean the data\n", "\n", - "The dataset contains a few unknown values." + "The dataset contains a few unknown values:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "JEJHhN65a2VV" }, "outputs": [], @@ -286,19 +227,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9UPN0KBHa_WI" }, "source": [ - "To keep this initial tutorial simple drop those rows." + "Drop those rows to keep this initial tutorial simple:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4ZUDosChC1UN" }, "outputs": [], @@ -309,365 +247,423 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8XKitwaH4v8h" }, "source": [ - "The `\"Origin\"` column is really categorical, not numeric. So convert that to a one-hot:" + "The `\"Origin\"` column is categorical, not numeric. So the next step is to one-hot encode the values in the column with [pd.get_dummies](https://pandas.pydata.org/docs/reference/api/pandas.get_dummies.html).\n", + "\n", + "Neglecting to specify a data type by way of a `dtype` argument will leave you with boolean values, causing errors during normalization when instantiating the Tensor object if the feature values are not cast to a uniform type when passing the array into `tf.keras.layers.Normalization.adapt()`. [Tensor objects](https://www.tensorflow.org/guide/tensor) must house uniform data types.\n", + "\n", + "Note: You can set up the `tf.keras.Model` to do this kind of transformation for you but that's beyond the scope of this tutorial. Check out the [Classify structured data using Keras preprocessing layers](../structured_data/preprocessing_layers.ipynb) or [Load CSV data](../load_data/csv.ipynb) tutorials for examples." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gWNTD2QjBWFJ" }, "outputs": [], "source": [ - "dataset['Origin'] = dataset['Origin'].map(lambda x: {1: 'USA', 2: 'Europe', 3: 'Japan'}.get(x))" + "dataset['Origin'] = dataset['Origin'].map({1: 'USA', 2: 'Europe', 3: 'Japan'})" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ulXz4J7PAUzk" }, "outputs": [], "source": [ - "dataset = pd.get_dummies(dataset, prefix='', prefix_sep='')\n", + "dataset = pd.get_dummies(dataset, columns=['Origin'], prefix='', prefix_sep='', dtype=float)\n", "dataset.tail()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Cuym4yvk76vU" }, "source": [ - "### Split the data into train and test\n", - "\n", - "Now split the dataset into a training set and a test set.\n", + "### Split the data into training and test sets\n", "\n", - "We will use the test set in the final evaluation of our model." + "Now, split the dataset into a training set and a test set. You will use the test set in the final evaluation of your models." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "qn-IGhUE7_1H" }, "outputs": [], "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", + "train_dataset = dataset.sample(frac=0.8, random_state=0)\n", "test_dataset = dataset.drop(train_dataset.index)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "J4ubs136WLNp" }, "source": [ "### Inspect the data\n", "\n", - "Have a quick look at the joint distribution of a few pairs of columns from the training set." + "Review the joint distribution of a few pairs of columns from the training set.\n", + "\n", + "The top row suggests that the fuel efficiency (MPG) is a function of all the other parameters. The other rows indicate they are functions of each other." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "oRKO_x8gWKv-" }, "outputs": [], "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" + "sns.pairplot(train_dataset[['MPG', 'Cylinders', 'Displacement', 'Weight']], diag_kind='kde')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gavKO_6DWRMP" }, "source": [ - "Also look at the overall statistics:" + "Let's also check the overall statistics. Note how each feature covers a very different range:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "yi2FzC3T21jR" }, "outputs": [], "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" + "train_dataset.describe().transpose()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Db7Auq1yXUvh" }, "source": [ "### Split features from labels\n", "\n", - "Separate the target value, or \"label\", from the features. This label is the value that you will train the model to predict." + "Separate the target value—the \"label\"—from the features. This label is the value that you will train the model to predict." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "t2sluJdCW7jN" }, "outputs": [], "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" + "train_features = train_dataset.copy()\n", + "test_features = test_dataset.copy()\n", + "\n", + "train_labels = train_features.pop('MPG')\n", + "test_labels = test_features.pop('MPG')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mRklxK5s388r" }, "source": [ - "### Normalize the data\n", + "## Normalization\n", "\n", - "Look again at the `train_stats` block above and note how different the ranges of each feature are." + "In the table of statistics it's easy to see how different the ranges of each feature are:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IcmY6lKKbkw8" + }, + "outputs": [], + "source": [ + "train_dataset.describe().transpose()[['mean', 'std']]" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-ywmerQ6dSox" }, "source": [ - "It is good practice to normalize features that use different scales and ranges. Although the model *might* converge without feature normalization, it makes training more difficult, and it makes the resulting model dependent on the choice of units used in the input.\n", + "It is good practice to normalize features that use different scales and ranges.\n", + "\n", + "One reason this is important is because the features are multiplied by the model weights. So, the scale of the outputs and the scale of the gradients are affected by the scale of the inputs.\n", + "\n", + "Although a model *might* converge without feature normalization, normalization makes training much more stable.\n", + "\n", + "Note: There is no advantage to normalizing the one-hot features—it is done here for simplicity. For more details on how to use the preprocessing layers, refer to the [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) guide and the [Classify structured data using Keras preprocessing layers](../structured_data/preprocessing_layers.ipynb) tutorial." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aFJ6ISropeoo" + }, + "source": [ + "### The Normalization layer\n", "\n", - "Note: Although we intentionally generate these statistics from only the training dataset, these statistics will also be used to normalize the test dataset. We need to do that to project the test dataset into the same distribution that the model has been trained on." + "The `tf.keras.layers.Normalization` is a clean and simple way to add feature normalization into your model.\n", + "\n", + "The first step is to create the layer:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "JlC5ooJrgjQF" }, "outputs": [], "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" + "normalizer = tf.keras.layers.Normalization(axis=-1)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" + "id": "XYA2Ap6nVOha" }, "source": [ - "This normalized data is what we will use to train the model.\n", - "\n", - "Caution: The statistics used to normalize the inputs here (mean and standard deviation) need to be applied to any other data that is fed to the model, along with the one-hot encoding that we did earlier. That includes the test set as well as live data when the model is used in production." + "Then, fit the state of the preprocessing layer to the data by calling `Normalization.adapt`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CrBbbjbwV91f" + }, + "outputs": [], + "source": [ + "normalizer.adapt(np.array(train_features))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" + "id": "oZccMR5yV9YV" + }, + "source": [ + "Calculate the mean and variance, and store them in the layer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GGn-ukwxSPtx" }, + "outputs": [], "source": [ - "## The model" + "print(normalizer.mean.numpy())" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" + "id": "oGWKaF9GSRuN" }, "source": [ - "### Build the model\n", - "\n", - "Let's build our model. Here, we'll use a `Sequential` model with two densely connected hidden layers, and an output layer that returns a single, continuous value. The model building steps are wrapped in a function, `build_model`, since we'll create a second model, later on." + "When the layer is called, it returns the input data, with each feature independently normalized:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "c26juK7ZG8j-" + "id": "2l7zFL_XWIRu" }, "outputs": [], "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation='relu', input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation='relu'),\n", - " layers.Dense(1)\n", - " ])\n", + "first = np.array(train_features[:1])\n", "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", + "with np.printoptions(precision=2, suppress=True):\n", + " print('First example:', first)\n", + " print()\n", + " print('Normalized:', normalizer(first).numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6o3CrycBXA2s" + }, + "source": [ + "## Linear regression\n", "\n", - " model.compile(loss='mse',\n", - " optimizer=optimizer,\n", - " metrics=['mae', 'mse'])\n", - " return model" + "Before building a deep neural network model, start with linear regression using one and several variables." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lFby9n0tnHkw" + }, + "source": [ + "### Linear regression with one variable\n", + "\n", + "Begin with a single-variable linear regression to predict `'MPG'` from `'Horsepower'`.\n", + "\n", + "Training a model with `tf.keras` typically starts by defining the model architecture. Use a `tf.keras.Sequential` model, which [represents a sequence of steps](https://www.tensorflow.org/guide/keras/sequential_model).\n", + "\n", + "There are two steps in your single-variable linear regression model:\n", + "\n", + "- Normalize the `'Horsepower'` input features using the `tf.keras.layers.Normalization` preprocessing layer.\n", + "- Apply a linear transformation ($y = mx+b$) to produce 1 output using a linear layer (`tf.keras.layers.Dense`).\n", + "\n", + "The number of _inputs_ can either be set by the `input_shape` argument, or automatically when the model is run for the first time." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Xp3gAFn3TPv8" + }, + "source": [ + "First, create a NumPy array made of the `'Horsepower'` features. Then, instantiate the `tf.keras.layers.Normalization` and fit its state to the `horsepower` data:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cGbPb-PHGbhs" + "id": "1gJAy0fKs1TS" }, "outputs": [], "source": [ - "model = build_model()" + "horsepower = np.array(train_features['Horsepower'])\n", + "\n", + "horsepower_normalizer = layers.Normalization(input_shape=[1,], axis=None)\n", + "horsepower_normalizer.adapt(horsepower)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" + "id": "4NVlHJY2TWlC" }, "source": [ - "### Inspect the model\n", - "\n", - "Use the `.summary` method to print a simple description of the model" + "Build the Keras Sequential model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ReAD0n6MsFK-" + "id": "c0sXM7qLlKfZ" }, "outputs": [], "source": [ - "model.summary()" + "horsepower_model = tf.keras.Sequential([\n", + " horsepower_normalizer,\n", + " layers.Dense(units=1)\n", + "])\n", + "\n", + "horsepower_model.summary()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" + "id": "eObQu9fDnXGL" }, "source": [ + "This model will predict `'MPG'` from `'Horsepower'`.\n", "\n", - "Now try out the model. Take a batch of `10` examples from the training data and call `model.predict` on it." + "Run the untrained model on the first 10 'Horsepower' values. The output won't be good, but notice that it has the expected shape of `(10, 1)`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-d-gBaVtGTSC" + "id": "UfV1HS6bns-s" }, "outputs": [], "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" + "horsepower_model.predict(horsepower[:10])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" + "id": "CSkanJlmmFBX" + }, + "source": [ + "Once the model is built, configure the training procedure using the Keras `Model.compile` method. The most important arguments to compile are the `loss` and the `optimizer`, since these define what will be optimized (`mean_absolute_error`) and how (using the `tf.keras.optimizers.Adam`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JxA_3lpOm-SK" }, + "outputs": [], "source": [ - "It seems to be working, and it produces a result of the expected shape and type." + "horsepower_model.compile(\n", + " optimizer=tf.keras.optimizers.Adam(learning_rate=0.1),\n", + " loss='mean_absolute_error')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" + "id": "Z3q1I9TwnRSC" }, "source": [ - "### Train the model\n", - "\n", - "Train the model for 1000 epochs, and record the training and validation accuracy in the `history` object." + "Use Keras `Model.fit` to execute the training for 100 epochs:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sD7qHCmNIOY0" + "id": "-iSrNy59nRAp" }, "outputs": [], "source": [ - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[tfdocs.modeling.EpochDots()])" + "%%time\n", + "history = horsepower_model.fit(\n", + " train_features['Horsepower'],\n", + " train_labels,\n", + " epochs=100,\n", + " # Suppress logging.\n", + " verbose=0,\n", + " # Calculate validation results on 20% of the training data.\n", + " validation_split = 0.2)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tQm3pc0FYPQB" }, "source": [ - "Visualize the model's training progress using the stats stored in the `history` object." + "Visualize the model's training progress using the stats stored in the `history` object:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4Xj91b-dymEy" + "id": "YCAwD_y4AdC3" }, "outputs": [], "source": [ @@ -678,144 +674,602 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, + "metadata": { + "id": "9E54UoZunqhc" + }, + "outputs": [], + "source": [ + "def plot_loss(history):\n", + " plt.plot(history.history['loss'], label='loss')\n", + " plt.plot(history.history['val_loss'], label='val_loss')\n", + " plt.ylim([0, 10])\n", + " plt.xlabel('Epoch')\n", + " plt.ylabel('Error [MPG]')\n", + " plt.legend()\n", + " plt.grid(True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "czYtZS9A6D-X" + "id": "yYsQYrIZyqjz" }, "outputs": [], "source": [ - "plotter = tfdocs.plots.HistoryPlotter(smoothing_std=2)" + "plot_loss(history)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CMNrt8X2ebXd" + }, + "source": [ + "Collect the results on the test set for later:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nMCWKskbUTvG" + "id": "kDZ8EvNYrDtx" }, "outputs": [], "source": [ - "plotter.plot({'Basic': history}, metric = \"mae\")\n", - "plt.ylim([0, 10])\n", - "plt.ylabel('MAE [MPG]')" + "test_results = {}\n", + "\n", + "test_results['horsepower_model'] = horsepower_model.evaluate(\n", + " test_features['Horsepower'],\n", + " test_labels, verbose=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F0qutYAKwoda" + }, + "source": [ + "Since this is a single variable regression, it's easy to view the model's predictions as a function of the input:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "N9u74b1tXMd9" + "id": "xDS2JEtOn9Jn" }, "outputs": [], "source": [ - "plotter.plot({'Basic': history}, metric = \"mse\")\n", - "plt.ylim([0, 20])\n", - "plt.ylabel('MSE [MPG^2]')" + "x = tf.linspace(0.0, 250, 251)\n", + "y = horsepower_model.predict(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rttFCTU8czsI" + }, + "outputs": [], + "source": [ + "def plot_horsepower(x, y):\n", + " plt.scatter(train_features['Horsepower'], train_labels, label='Data')\n", + " plt.plot(x, y, color='k', label='Predictions')\n", + " plt.xlabel('Horsepower')\n", + " plt.ylabel('MPG')\n", + " plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7l9ZiAOEUNBL" + }, + "outputs": [], + "source": [ + "plot_horsepower(x, y)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" + "id": "Yk2RmlqPoM9u" }, "source": [ - "This graph shows little improvement, or even degradation in the validation error after about 100 epochs. Let's update the `model.fit` call to automatically stop training when the validation score doesn't improve. We'll use an *EarlyStopping callback* that tests a training condition for every epoch. If a set amount of epochs elapses without showing improvement, then automatically stop the training.\n", + "### Linear regression with multiple inputs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PribnwDHUksC" + }, + "source": [ + "You can use an almost identical setup to make predictions based on multiple inputs. This model still does the same $y = mx+b$ except that $m$ is a matrix and $x$ is a vector.\n", "\n", - "You can learn more about this callback [here](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping)." + "Create a two-step Keras Sequential model again with the first layer being `normalizer` (`tf.keras.layers.Normalization(axis=-1)`) you defined earlier and adapted to the whole dataset:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fdMZuhUgzMZ4" + "id": "ssnVcKg7oMe6" }, "outputs": [], "source": [ - "model = build_model()\n", - "\n", - "# The patience parameter is the amount of epochs to check for improvement\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "early_history = model.fit(normed_train_data, train_labels, \n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0, \n", - " callbacks=[early_stop, tfdocs.modeling.EpochDots()])" + "linear_model = tf.keras.Sequential([\n", + " normalizer,\n", + " layers.Dense(units=1)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IHlx6WeIWyAr" + }, + "source": [ + "When you call `Model.predict` on a batch of inputs, it produces `units=1` outputs for each example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DynfJV18WiuT" + }, + "outputs": [], + "source": [ + "linear_model.predict(train_features[:10])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hvHKH3rPXHmq" + }, + "source": [ + "When you call the model, its weight matrices will be built—check that the `kernel` weights (the $m$ in $y=mx+b$) have a shape of `(9, 1)`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LcopvQh3X-kX" + "id": "DwJ4Fq0RXBQf" }, "outputs": [], "source": [ - "plotter.plot({'Early Stopping': early_history}, metric = \"mae\")\n", - "plt.ylim([0, 10])\n", - "plt.ylabel('MAE [MPG]')" + "linear_model.layers[1].kernel" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eINAc6rZXzOt" + }, + "source": [ + "Configure the model with Keras `Model.compile` and train with `Model.fit` for 100 epochs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "A0Sv_Ybr0szp" + }, + "outputs": [], + "source": [ + "linear_model.compile(\n", + " optimizer=tf.keras.optimizers.Adam(learning_rate=0.1),\n", + " loss='mean_absolute_error')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EZoOYORvoTSe" + }, + "outputs": [], + "source": [ + "%%time\n", + "history = linear_model.fit(\n", + " train_features,\n", + " train_labels,\n", + " epochs=100,\n", + " # Suppress logging.\n", + " verbose=0,\n", + " # Calculate validation results on 20% of the training data.\n", + " validation_split = 0.2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EdxiCbiNYK2F" + }, + "source": [ + "Using all the inputs in this regression model achieves a much lower training and validation error than the `horsepower_model`, which had one input:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4sWO3W0koYgu" + }, + "outputs": [], + "source": [ + "plot_loss(history)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NyN49hIWe_NH" + }, + "source": [ + "Collect the results on the test set for later:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jNC3D1DGsGgK" + }, + "outputs": [], + "source": [ + "test_results['linear_model'] = linear_model.evaluate(\n", + " test_features, test_labels, verbose=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SmjdzxKzEu1-" + }, + "source": [ + "## Regression with a deep neural network (DNN)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DT_aHPsrzO1t" + }, + "source": [ + "In the previous section, you implemented two linear models for single and multiple inputs.\n", + "\n", + "Here, you will implement single-input and multiple-input DNN models.\n", + "\n", + "The code is basically the same except the model is expanded to include some \"hidden\" non-linear layers. The name \"hidden\" here just means not directly connected to the inputs or outputs." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" + "id": "6SWtkIjhrZwa" }, "source": [ - "The graph shows that on the validation set, the average error is usually around +/- 2 MPG. Is this good? We'll leave that decision up to you.\n", + "These models will contain a few more layers than the linear model:\n", "\n", - "Let's see how well the model generalizes by using the **test** set, which we did not use when training the model. This tells us how well we can expect the model to predict when we use it in the real world." + "* The normalization layer, as before (with `horsepower_normalizer` for a single-input model and `normalizer` for a multiple-input model).\n", + "* Two hidden, non-linear, `Dense` layers with the ReLU (`relu`) activation function nonlinearity.\n", + "* A linear `Dense` single-output layer.\n", + "\n", + "Both models will use the same training procedure, so the `compile` method is included in the `build_and_compile_model` function below." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jl_yNr5n1kms" + "id": "c26juK7ZG8j-" }, "outputs": [], "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", + "def build_and_compile_model(norm):\n", + " model = keras.Sequential([\n", + " norm,\n", + " layers.Dense(64, activation='relu'),\n", + " layers.Dense(64, activation='relu'),\n", + " layers.Dense(1)\n", + " ])\n", "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" + " model.compile(loss='mean_absolute_error',\n", + " optimizer=tf.keras.optimizers.Adam(0.001))\n", + " return model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6c51caebbc0d" + }, + "source": [ + "### Regression using a DNN and a single input" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xvu9gtxTZR5V" + }, + "source": [ + "Create a DNN model with only `'Horsepower'` as input and `horsepower_normalizer` (defined earlier) as the normalization layer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cGbPb-PHGbhs" + }, + "outputs": [], + "source": [ + "dnn_horsepower_model = build_and_compile_model(horsepower_normalizer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Sj49Og4YGULr" + }, + "source": [ + "This model has quite a few more trainable parameters than the linear models:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ReAD0n6MsFK-" + }, + "outputs": [], + "source": [ + "dnn_horsepower_model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0-qWCsh6DlyH" + }, + "source": [ + "Train the model with Keras `Model.fit`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sD7qHCmNIOY0" + }, + "outputs": [], + "source": [ + "%%time\n", + "history = dnn_horsepower_model.fit(\n", + " train_features['Horsepower'],\n", + " train_labels,\n", + " validation_split=0.2,\n", + " verbose=0, epochs=100)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dArGGxHxcKjN" + }, + "source": [ + "This model does slightly better than the linear single-input `horsepower_model`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NcF6UWjdCU8T" + }, + "outputs": [], + "source": [ + "plot_loss(history)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TG1snlpR2QCK" + }, + "source": [ + "If you plot the predictions as a function of `'Horsepower'`, you should notice how this model takes advantage of the nonlinearity provided by the hidden layers:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hPF53Rem14NS" + }, + "outputs": [], + "source": [ + "x = tf.linspace(0.0, 250, 251)\n", + "y = dnn_horsepower_model.predict(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rsf9rD8I17Wq" + }, + "outputs": [], + "source": [ + "plot_horsepower(x, y)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WxCJKIUpe4io" + }, + "source": [ + "Collect the results on the test set for later:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bJjM0dU52XtN" + }, + "outputs": [], + "source": [ + "test_results['dnn_horsepower_model'] = dnn_horsepower_model.evaluate(\n", + " test_features['Horsepower'], test_labels,\n", + " verbose=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S_2Btebp2e64" + }, + "source": [ + "### Regression using a DNN and multiple inputs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aKFtezDldLSf" + }, + "source": [ + "Repeat the previous process using all the inputs. The model's performance slightly improves on the validation dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "c0mhscXh2k36" + }, + "outputs": [], + "source": [ + "dnn_model = build_and_compile_model(normalizer)\n", + "dnn_model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CXDENACl2tuW" + }, + "outputs": [], + "source": [ + "%%time\n", + "history = dnn_model.fit(\n", + " train_features,\n", + " train_labels,\n", + " validation_split=0.2,\n", + " verbose=0, epochs=100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-9Dbj0fX23RQ" + }, + "outputs": [], + "source": [ + "plot_loss(history)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hWoVYS34fJPZ" + }, + "source": [ + "Collect the results on the test set:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-bZIa96W3c7K" + }, + "outputs": [], + "source": [ + "test_results['dnn_model'] = dnn_model.evaluate(test_features, test_labels, verbose=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uiCucdPLfMkZ" + }, + "source": [ + "## Performance" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rDf1xebEfWBw" + }, + "source": [ + "Since all models have been trained, you can review their test set performance:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e5_ooufM5iH2" + }, + "outputs": [], + "source": [ + "pd.DataFrame(test_results, index=['Mean absolute error [MPG]']).T" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DABIVzsCf-QI" + }, + "source": [ + "These results match the validation error observed during training." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ft603OzXuEZC" }, "source": [ "### Make predictions\n", "\n", - "Finally, predict MPG values using data in the testing set:" + "You can now make predictions with the `dnn_model` on the test set using Keras `Model.predict` and review the loss:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Xe7RXH3N3CWU" }, "outputs": [], "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", + "test_predictions = dnn_model.predict(test_features).flatten()\n", "\n", "a = plt.axes(aspect='equal')\n", "plt.scatter(test_labels, test_predictions)\n", @@ -830,55 +1284,96 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "19wyogbOSU5t" }, "source": [ - "It looks like our model predicts reasonably well. Let's take a look at the error distribution." + "It appears that the model predicts reasonably well.\n", + "\n", + "Now, check the error distribution:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "f-OHX4DiXd8x" }, "outputs": [], "source": [ "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" + "plt.hist(error, bins=25)\n", + "plt.xlabel('Prediction Error [MPG]')\n", + "_ = plt.ylabel('Count')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "m0CB5tBjSU5w" + "id": "KSyaHUfDT-mZ" }, "source": [ - "It's not quite gaussian, but we might expect that because the number of samples is very small." + "If you're happy with the model, save it for later use with `Model.save`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4-WwLlmfT-mb" + }, + "outputs": [], + "source": [ + "dnn_model.save('dnn_model.keras')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Benlnl8UT-me" + }, + "source": [ + "If you reload the model, it gives identical output:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dyyyj2zVT-mf" + }, + "outputs": [], + "source": [ + "reloaded = tf.keras.models.load_model('dnn_model.keras')\n", + "\n", + "test_results['reloaded'] = reloaded.evaluate(\n", + " test_features, test_labels, verbose=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "f_GchJ2tg-2o" + }, + "outputs": [], + "source": [ + "pd.DataFrame(test_results, index=['Mean absolute error [MPG]']).T" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vgGQuV-yqYZH" }, "source": [ "## Conclusion\n", "\n", - "This notebook introduced a few techniques to handle a regression problem.\n", + "This notebook introduced a few techniques to handle a regression problem. Here are a few more tips that may help:\n", "\n", - "* Mean Squared Error (MSE) is a common loss function used for regression problems (different loss functions are used for classification problems).\n", - "* Similarly, evaluation metrics used for regression differ from classification. A common regression metric is Mean Absolute Error (MAE).\n", - "* When numeric input data features have values with different ranges, each feature should be scaled independently to the same range.\n", - "* If there is not much training data, one technique is to prefer a small network with few hidden layers to avoid overfitting.\n", - "* Early stopping is a useful technique to prevent overfitting." + "- Mean squared error (MSE) (`tf.keras.losses.MeanSquaredError`) and mean absolute error (MAE) (`tf.keras.losses.MeanAbsoluteError`) are common loss functions used for regression problems. MAE is less sensitive to outliers. Different loss functions are used for classification problems.\n", + "- Similarly, evaluation metrics used for regression differ from classification.\n", + "- When numeric input data features have values with different ranges, each feature should be scaled independently to the same range.\n", + "- Overfitting is a common problem for DNN models, though it wasn't a problem for this tutorial. Visit the [Overfit and underfit](overfit_and_underfit.ipynb) tutorial for more help with this." ] } ], @@ -886,8 +1381,6 @@ "colab": { "collapsed_sections": [], "name": "regression.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/keras/save_and_load.ipynb b/site/en/tutorials/keras/save_and_load.ipynb index 4864a26af65..140ea1b59ac 100644 --- a/site/en/tutorials/keras/save_and_load.ipynb +++ b/site/en/tutorials/keras/save_and_load.ipynb @@ -1,25 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "save_and_load.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "g_nWetWWd_ns" }, "source": [ @@ -28,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} + "id": "2pHVBk_seED1" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -46,18 +29,16 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} + "id": "N_fMsQ-N8I7j" }, + "outputs": [], "source": [ "#@title MIT License\n", "#\n", @@ -80,14 +61,11 @@ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pZJ3uY9O17VN" }, "source": [ @@ -97,7 +75,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "M4Ata7_wMul1" }, "source": [ @@ -120,28 +97,26 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mBdde4YJeJKF" }, "source": [ - "Model progress can be saved during—and after—training. This means a model can resume where it left off and avoid long training times. Saving also means you can share your model and others can recreate your work. When publishing research models and techniques, most machine learning practitioners share:\n", + "Model progress can be saved during and after training. This means a model can resume where it left off and avoid long training times. Saving also means you can share your model and others can recreate your work. When publishing research models and techniques, most machine learning practitioners share:\n", "\n", "* code to create the model, and\n", "* the trained weights, or parameters, for the model\n", "\n", "Sharing this data helps others understand how the model works and try it themselves with new data.\n", "\n", - "Caution: Be careful with untrusted code—TensorFlow models are code. See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) for details.\n", + "Caution: TensorFlow models are code and it is important to be careful with untrusted code. See [Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) for details.\n", "\n", "### Options\n", "\n", - "There are different ways to save TensorFlow models—depending on the API you're using. This guide uses [tf.keras](https://www.tensorflow.org/guide/keras), a high-level API to build and train models in TensorFlow. For other approaches, see the TensorFlow [Save and Restore](https://www.tensorflow.org/guide/saved_model) guide or [Saving in eager](https://www.tensorflow.org/guide/eager#object-based_saving)." + "There are different ways to save TensorFlow models depending on the API you're using. This guide uses [tf.keras](https://www.tensorflow.org/guide/keras)—a high-level API to build and train models in TensorFlow. The new, high-level `.keras` format used in this tutorial is recommended for saving Keras objects, as it provides robust, efficient name-based saving that is often easier to debug than low-level or legacy formats. For more advanced saving or serialization workflows, especially those involving custom objects, please refer to the [Save and load Keras models guide](https://www.tensorflow.org/guide/keras/save_and_serialize). For other approaches, refer to the [Using the SavedModel format guide](../../guide/saved_model.ipynb)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xCUREq7WXgvg" }, "source": [ @@ -153,7 +128,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7l0MiTOrXtNv" }, "source": [ @@ -162,47 +136,34 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} + "id": "RzIOVSdnMYyO" }, + "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "!pip install pyyaml h5py # Required to save models in HDF5 format" - ], - "execution_count": 0, - "outputs": [] + "!pip install pyyaml h5py # Required to save models in HDF5 format." + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} + "id": "7Nm7Tyb-gRt-" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "import os\n", "\n", "import tensorflow as tf\n", "from tensorflow import keras\n", "\n", "print(tf.version.VERSION)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SbGsznErXWt6" }, "source": [ @@ -213,11 +174,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} + "id": "9rGfFwE9XVwz" }, + "outputs": [], "source": [ "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", "\n", @@ -226,14 +187,11 @@ "\n", "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "anG3iVoXyZGI" }, "source": [ @@ -243,7 +201,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wynsOBfby0Pa" }, "source": [ @@ -252,23 +209,23 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} + "id": "0HZbJIjxyX1S" }, + "outputs": [], "source": [ "# Define a simple sequential model\n", "def create_model():\n", - " model = tf.keras.models.Sequential([\n", + " model = tf.keras.Sequential([\n", " keras.layers.Dense(512, activation='relu', input_shape=(784,)),\n", " keras.layers.Dropout(0.2),\n", - " keras.layers.Dense(10, activation='softmax')\n", + " keras.layers.Dense(10)\n", " ])\n", "\n", " model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n", "\n", " return model\n", "\n", @@ -277,14 +234,11 @@ "\n", "# Display the model's architecture\n", "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "soDE0W_KH8rG" }, "source": [ @@ -294,11 +248,10 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mRyd5qQQIXZm" }, "source": [ - "You can use a trained model without having to retrain it, or pick-up training where you left off—in case the training process was interrupted. The `tf.keras.callbacks.ModelCheckpoint` callback allows to continually save the model both *during* and at *the end* of training.\n", + "You can use a trained model without having to retrain it, or pick-up training where you left off in case the training process was interrupted. The `tf.keras.callbacks.ModelCheckpoint` callback allows you to continually save the model both *during* and at *the end* of training.\n", "\n", "### Checkpoint callback usage\n", "\n", @@ -307,13 +260,13 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} + "id": "IFPuhwntH8VH" }, + "outputs": [], "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", + "checkpoint_path = \"training_1/cp.weights.h5\" # Since you're only saving weights, you should use the .weights.h5 extension. If you're saving the whole model, you would use the .keras extension instead\n", "checkpoint_dir = os.path.dirname(checkpoint_path)\n", "\n", "# Create a callback that saves the model's weights\n", @@ -322,23 +275,20 @@ " verbose=1)\n", "\n", "# Train the model with the new callback\n", - "model.fit(train_images, \n", - " train_labels, \n", + "model.fit(train_images,\n", + " train_labels,\n", " epochs=10,\n", - " validation_data=(test_images,test_labels),\n", + " validation_data=(test_images, test_labels),\n", " callbacks=[cp_callback]) # Pass callback to training\n", "\n", "# This may generate warnings related to saving the state of the optimizer.\n", "# These warnings (and similar warnings throughout this notebook)\n", "# are in place to discourage outdated usage, and can be ignored." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rlM-sgyJO084" }, "source": [ @@ -347,51 +297,45 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} + "id": "gXG5FVKFOVQ3" }, + "outputs": [], "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] + "os.listdir(checkpoint_dir)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wlRN_f56Pqa9" }, "source": [ - "Create a new, untrained model. When restoring a model from weights-only, you must have a model with the same architecture as the original model. Since it's the same model architecture, you can share weights despite that it's a different *instance* of the model.\n", + "As long as two models share the same architecture you can share weights between them. So, when restoring a model from weights-only, create a model with the same architecture as the original model and then set its weights.\n", "\n", - "Now rebuild a fresh, untrained model, and evaluate it on the test set. An untrained model will perform at chance levels (~10% accuracy):" + "Now rebuild a fresh, untrained model and evaluate it on the test set. An untrained model will perform at chance levels (~10% accuracy):" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} + "id": "Fp5gbuiaPqCT" }, + "outputs": [], "source": [ "# Create a basic model instance\n", "model = create_model()\n", "\n", "# Evaluate the model\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Untrained model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", + "print(\"Untrained model, accuracy: {:5.2f}%\".format(100 * acc))" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1DTKpZssRSo3" }, "source": [ @@ -400,26 +344,23 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} + "id": "2IZxbwiRRSD2" }, + "outputs": [], "source": [ "# Loads the weights\n", "model.load_weights(checkpoint_path)\n", "\n", "# Re-evaluate the model\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", + "print(\"Restored model, accuracy: {:5.2f}%\".format(100 * acc))" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bpAbKkAyVPV8" }, "source": [ @@ -432,22 +373,30 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} + "id": "mQF_dlgIVOvq" }, + "outputs": [], "source": [ "# Include the epoch in the file name (uses `str.format`)\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", + "checkpoint_path = \"training_2/cp-{epoch:04d}.weights.h5\"\n", "checkpoint_dir = os.path.dirname(checkpoint_path)\n", + "os.mkdir(checkpoint_dir)\n", + "\n", + "batch_size = 32\n", + "\n", + "# Calculate the number of batches per epoch\n", + "import math\n", + "n_batches = len(train_images) / batch_size\n", + "n_batches = math.ceil(n_batches) # round up the number of batches to the nearest whole integer\n", "\n", "# Create a callback that saves the model's weights every 5 epochs\n", "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " filepath=checkpoint_path, \n", - " verbose=1, \n", + " filepath=checkpoint_path,\n", + " verbose=1,\n", " save_weights_only=True,\n", - " period=5)\n", + " save_freq=5*n_batches)\n", "\n", "# Create a new model instance\n", "model = create_model()\n", @@ -456,72 +405,69 @@ "model.save_weights(checkpoint_path.format(epoch=0))\n", "\n", "# Train the model with the new callback\n", - "model.fit(train_images, \n", - " train_labels,\n", - " epochs=50, \n", - " callbacks=[cp_callback],\n", - " validation_data=(test_images,test_labels),\n", - " verbose=0)" - ], - "execution_count": 0, - "outputs": [] + "model.fit(train_images,\n", + " train_labels,\n", + " epochs=50,\n", + " batch_size=batch_size,\n", + " callbacks=[cp_callback],\n", + " validation_data=(test_images, test_labels),\n", + " verbose=0)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1zFrKTjjavWI" }, "source": [ - "Now, look at the resulting checkpoints and choose the latest one:" + "Now, review the resulting checkpoints and choose the latest one:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} + "id": "p64q3-V4sXt0" }, + "outputs": [], "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] + "os.listdir(checkpoint_dir)" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} + "id": "1AN_fnuyR41H" }, + "outputs": [], "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", + "def load_latest_checkpoint(checkpoint_dir):\n", + " latest = max(os.listdir(checkpoint_dir), key=lambda f: int(f.split('-')[1].split('.')[0]))\n", + " return os.path.join(checkpoint_dir, latest)\n", + "\n", + "latest = load_latest_checkpoint(checkpoint_dir)\n", "latest" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Zk2ciGbKg561" }, "source": [ - "Note: the default tensorflow format only saves the 5 most recent checkpoints.\n", + "Note: The default TensorFlow format only saves the 5 most recent checkpoints.\n", "\n", - "To test, reset the model and load the latest checkpoint:" + "To test, reset the model, and load the latest checkpoint:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} + "id": "3M04jyK-H3QK" }, + "outputs": [], "source": [ "# Create a new model instance\n", "model = create_model()\n", @@ -530,16 +476,13 @@ "model.load_weights(latest)\n", "\n", "# Re-evaluate the model\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", + "print(\"Restored model, accuracy: {:5.2f}%\".format(100 * acc))" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "c2OxsJOTHxia" }, "source": [ @@ -549,171 +492,165 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "JtdYhvWnH2ib" }, "source": [ - "The above code stores the weights to a collection of [checkpoint](https://www.tensorflow.org/guide/saved_model#save_and_restore_variables)-formatted files that contain only the trained weights in a binary format. Checkpoints contain:\n", + "The above code stores the weights to a collection of [checkpoint](../../guide/checkpoint.ipynb)-formatted files that contain only the trained weights in a binary format. Checkpoints contain:\n", "* One or more shards that contain your model's weights.\n", - "* An index file that indicates which weights are stored in a which shard.\n", + "* An index file that indicates which weights are stored in which shard.\n", "\n", - "If you are only training a model on a single machine, you'll have one shard with the suffix: `.data-00000-of-00001`" + "If you are training a model on a single machine, you'll have one shard with the suffix: `.data-00000-of-00001`" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "S_FA-ZvxuXQV" }, "source": [ "## Manually save weights\n", "\n", - "You saw how to load the weights into a model. Manually saving them is just as simple with the `Model.save_weights` method. By default, `tf.keras`—and `save_weights` in particular—uses the TensorFlow [checkpoints](../../guide/keras/checkpoints) format with a `.ckpt` extension (saving in [HDF5](https://js.tensorflow.org/tutorials/import-keras.html) with a `.h5` extension is covered in the [Save and serialize models](../../guide/keras/save_and_serialize#weights-only_saving_in_savedmodel_format) guide):" + "To save weights manually, use `tf.keras.Model.save_weights`. You have to use .weights.h5 extension to save the weights. You can refer to the [Save and load models](https://www.tensorflow.org/guide/keras/save_and_serialize) guide." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} + "id": "R7W5plyZ-u9X" }, + "outputs": [], "source": [ "# Save the weights\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", + "os.mkdir('./checkpoints')\n", + "model.save_weights('./checkpoints/my_checkpoint.weights.h5')\n", "\n", "# Create a new model instance\n", "model = create_model()\n", "\n", "# Restore the weights\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", + "model.load_weights('./checkpoints/my_checkpoint.weights.h5')\n", "\n", "# Evaluate the model\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] + "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", + "print(\"Restored model, accuracy: {:5.2f}%\".format(100 * acc))" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kOGlxPRBEvV1" }, "source": [ "## Save the entire model\n", "\n", - "Call [`model.save`](https://www.tensorflow.org/api_docs/python/tf/keras/Model#save) to save the a model's architecture, weights, and training configuration in a single file/folder. This allows you to export a model so it can be used without access to the original Python code*. Since the optimizer-state is recovered, you can resume training from exactly where you left off.\n", + "Call `tf.keras.Model.save` to save a model's architecture, weights, and training configuration in a single `model.keras` zip archive.\n", + "\n", + "An entire model can be saved in three different file formats (the new `.keras` format and two legacy formats: `SavedModel`, and `HDF5`). Saving a model as `path/to/model.keras` automatically saves in the latest format.\n", + "\n", + "**Note:** For Keras objects it's recommended to use the new high-level `.keras` format for richer, name-based saving and reloading, which is easier to debug. The low-level SavedModel format and legacy H5 format continue to be supported for existing code.\n", + "\n", + "You can switch to the SavedModel format by:\n", "\n", - "Saving a fully-functional model is very useful—you can load them in TensorFlow.js ([HDF5](https://js.tensorflow.org/tutorials/import-keras.html), [Saved Model](https://js.tensorflow.org/tutorials/import-saved-model.html)) and then train and run them in web browsers, or convert them to run on mobile devices using TensorFlow Lite ([HDF5](https://www.tensorflow.org/lite/convert/python_api#exporting_a_tfkeras_file_), [Saved Model](https://www.tensorflow.org/lite/convert/python_api#exporting_a_savedmodel_))\n", + "- Passing `save_format='tf'` to `save()`\n", + "- Passing a filename without an extension\n", "\n", - "\\*Custom objects (e.g. subclassed models or layers) require special attention when saving and loading. See the **Saving custom objects** section below " + "You can switch to the H5 format by:\n", + "- Passing `save_format='h5'` to `save()`\n", + "- Passing a filename that ends in `.h5`\n", + "\n", + "Saving a fully-functional model is very useful—you can load them in TensorFlow.js ([Saved Model](https://www.tensorflow.org/js/tutorials/conversion/import_saved_model), [HDF5](https://www.tensorflow.org/js/tutorials/conversion/import_keras)) and then train and run them in web browsers, or convert them to run on mobile devices using TensorFlow Lite ([Saved Model](https://www.tensorflow.org/lite/models/convert/#convert_a_savedmodel_recommended_), [HDF5](https://www.tensorflow.org/lite/models/convert/#convert_a_keras_model_))\n", + "\n", + "\\*Custom objects (for example, subclassed models or layers) require special attention when saving and loading. Refer to the **Saving custom objects** section below." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" + "id": "0fRGnlHMrkI7" }, "source": [ - "### HDF5 format\n", + "### New high-level `.keras` format" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eqO8jj7GsCDn" + }, + "source": [ + "The new Keras v3 saving format, marked by the `.keras` extension, is a more\n", + "simple, efficient format that implements name-based saving, ensuring what you load is exactly what you saved, from Python's perspective. This makes debugging much easier, and it is the recommended format for Keras.\n", "\n", - "Keras provides a basic save format using the [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) standard. " + "The section below illustrates how to save and restore the model in the `.keras` format." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} + "id": "3f55mAXwukUX" }, + "outputs": [], "source": [ "# Create and train a new model instance.\n", "model = create_model()\n", "model.fit(train_images, train_labels, epochs=5)\n", "\n", - "# Save the entire model to a HDF5 file.\n", - "# The '.h5' extension indicates that the model shuold be saved to HDF5.\n", - "model.save('my_model.h5') " - ], - "execution_count": 0, - "outputs": [] + "# Save the entire model as a `.keras` zip archive.\n", + "model.save('my_model.keras')" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" + "id": "iHqwaun5g8lD" }, "source": [ - "Now, recreate the model from that file:" + "Reload a fresh Keras model from the `.keras` zip archive:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} + "id": "HyfUMOZwux_-" }, + "outputs": [], "source": [ - "# Recreate the exact same model, including its weights and the optimizer\n", - "new_model = tf.keras.models.load_model('my_model.h5')\n", + "new_model = tf.keras.models.load_model('my_model.keras')\n", "\n", "# Show the model architecture\n", "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" + "id": "9Cn3pSBqvJ5f" }, "source": [ - "Check its accuracy:" + "Try running evaluate and predict with the loaded model:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} + "id": "8BT4mHNIvMdW" }, + "outputs": [], "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print('Restored model, accuracy: {:5.2f}%'.format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "This technique saves everything:\n", - "\n", - "* The weight values\n", - "* The model's configuration(architecture)\n", - "* The optimizer configuration\n", + "# Evaluate the restored model\n", + "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", + "print('Restored model, accuracy: {:5.2f}%'.format(100 * acc))\n", "\n", - "Keras saves models by inspecting the architecture. Currently, it is not able to save TensorFlow optimizers (from `tf.train`). When using those you will need to re-compile the model after loading, and you will lose the state of the optimizer.\n" + "print(new_model.predict(test_images).shape)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kPyhgcoVzqUB" }, "source": [ @@ -723,20 +660,19 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LtcN4VIb7JkK" }, "source": [ - "The SavedModel format is another way to serialize models. Models saved in this format can be restored using `tf.keras.models.load_model` and are compatible with TensorFlow Serving. The [SavedModel guide](https://www.tensorflow.org/guide/saved_model) goes into detail about how to serve/inspect the SavedModel. The section below illustrates the steps to saving and restoring the model." + "The SavedModel format is another way to serialize models. Models saved in this format can directly be used with TFLite/TFServing/etc for inferencing. The [SavedModel guide](../../guide/saved_model.ipynb) goes into detail about how to `serve/inspect` the SavedModel. The section below illustrates the steps to save and restore the model." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} + "id": "sI1YvCDFzpl3" }, + "outputs": [], "source": [ "# Create and train a new model instance.\n", "model = create_model()\n", @@ -744,111 +680,178 @@ "\n", "# Save the entire model as a SavedModel.\n", "!mkdir -p saved_model\n", - "model.save('saved_model/my_model') " - ], - "execution_count": 0, - "outputs": [] + "tf.saved_model.save(model, 'saved_model/my_model')" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iUvT_3qE8hV5" }, "source": [ - "The SavedModel format is a directory containing a protobuf binary and a Tensorflow checkpoint. Inspect the saved model directory:" + "The SavedModel format is a directory containing a protobuf binary and a TensorFlow checkpoint. Inspect the saved model directory:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "colab": {} + "id": "sq8fPglI1RWA" }, + "outputs": [], "source": [ "# my_model directory\n", "!ls saved_model\n", "\n", "# Contains an assets folder, saved_model.pb, and variables folder.\n", "!ls saved_model/my_model" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "B7qfpvpY9HCe" }, "source": [ - "Reload a fresh Keras model from the saved model:" + "Reload the saved SavedModel file:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} + "id": "0YofwHdN0pxa" }, + "outputs": [], "source": [ - "new_model = tf.keras.models.load_model('saved_model/my_model')\n", + "saved_model = tf.saved_model.load('saved_model/my_model')\n", + "saved_model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SkGwf-50zLNn" + }, + "source": [ + "### HDF5 format\n", + "\n", + "Keras provides a basic legacy high-level save format using the [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) standard." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "m2dkmJVCGUia" + }, + "outputs": [], + "source": [ + "# Create and train a new model instance.\n", + "model = create_model()\n", + "model.fit(train_images, train_labels, epochs=5)\n", + "\n", + "# Save the entire model to a HDF5 file.\n", + "# The '.h5' extension indicates that the model should be saved to HDF5.\n", + "model.save('my_model.h5')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GWmttMOqS68S" + }, + "source": [ + "Now, recreate the model from that file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5NDMO_7kS6Do" + }, + "outputs": [], + "source": [ + "# Recreate the exact same model, including its weights and the optimizer\n", + "new_model = tf.keras.models.load_model('my_model.h5')\n", "\n", - "# Check its architecture\n", + "# Show the model architecture\n", "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" + "id": "JXQpbTicTBwt" }, "source": [ - "The restored model is compiled with the same arguments as the original model. Try running evaluate and predict with the loaded model:" + "Check its accuracy:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Yh5Mu0yOgE5J", - "colab": {} + "id": "jwEaj9DnTCVA" }, + "outputs": [], "source": [ - "# Evaluate the restored model\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print('Restored model, accuracy: {:5.2f}%'.format(100*acc))\n", + "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", + "print('Restored model, accuracy: {:5.2f}%'.format(100 * acc))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dGXqd4wWJl8O" + }, + "source": [ + "Keras saves models by inspecting their architectures. This technique saves everything:\n", "\n", - "print(new_model.predict(test_images).shape)" - ], - "execution_count": 0, - "outputs": [] + "* The weight values\n", + "* The model's architecture\n", + "* The model's training configuration (what you pass to the `.compile()` method)\n", + "* The optimizer and its state, if any (this enables you to restart training where you left off)\n", + "\n", + "Keras is not able to save the `v1.x` optimizers (from `tf.compat.v1.train`) since they aren't compatible with checkpoints. For v1.x optimizers, you need to re-compile the model after loading—losing the state of the optimizer.\n" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kAUKJQyGqTNH" }, "source": [ "### Saving custom objects\n", "\n", - "If you are using the SavedModel format, you can skip this section. The key difference between HDF5 and SavedModel is that HDF5 uses object configs to save the model architecture, while SavedModel saves the execution graph. Thus, SavedModels are able to save custom objects like subclassed models and custom layers without requiring the orginal code.\n", + "If you are using the SavedModel format, you can skip this section. The key difference between high-level `.keras`/HDF5 formats and the low-level SavedModel format is that the `.keras`/HDF5 formats uses object configs to save the model architecture, while SavedModel saves the execution graph. Thus, SavedModels are able to save custom objects like subclassed models and custom layers without requiring the original code. However, debugging low-level SavedModels can be more difficult as a result, and the recommended approach is using the high-level `.keras` format instead due to its name-based, Keras built-in nature.\n", "\n", - "To save custom objects to HDF5, you must do the following:\n", + "To save custom objects to `.keras` and HDF5, you must do the following:\n", "\n", "1. Define a `get_config` method in your object, and optionally a `from_config` classmethod.\n", " * `get_config(self)` returns a JSON-serializable dictionary of parameters needed to recreate the object.\n", " * `from_config(cls, config)` uses the returned config from `get_config` to create a new object. By default, this function will use the config as initialization kwargs (`return cls(**config)`).\n", - "2. Pass the object to the `custom_objects` argument when loading the model. The argument must be a dictionary mapping the string class name to the Python class. E.g. `tf.keras.models.load_model(path, custom_objects={'CustomLayer': CustomLayer})`\n", + "2. Pass the custom objects to the model in one of three ways:\n", + " - Register the custom object with the `@tf.keras.utils.register_keras_serializable` decorator. **(recommended)**\n", + " - Directly pass the object to the `custom_objects` argument when loading the model. The argument must be a dictionary mapping the string class name to the Python class. E.g., `tf.keras.models.load_model(path, custom_objects={'CustomLayer': CustomLayer})`\n", + " - Use a `tf.keras.utils.custom_object_scope` with the object included in the `custom_objects` dictionary argument, and place a `tf.keras.models.load_model(path)` call within the scope.\n", "\n", - "See the [Writing layers and models from scratch](https://www.tensorflow.org/guide/keras/custom_layers_and_models) tutorial for examples of custom objects and `get_config`.\n" + "Refer to the [Writing layers and models from scratch](https://www.tensorflow.org/guide/keras/custom_layers_and_models) tutorial for examples of custom objects and `get_config`.\n" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "save_and_load.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/keras/text_classification.ipynb b/site/en/tutorials/keras/text_classification.ipynb index 49fecfafdc1..58f48cc2be0 100644 --- a/site/en/tutorials/keras/text_classification.ipynb +++ b/site/en/tutorials/keras/text_classification.ipynb @@ -1,38 +1,22 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_classification.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ic4_occAAiAT" }, "source": [ - "##### Copyright 2018 The TensorFlow Authors." + "##### Copyright 2019 The TensorFlow Authors." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} + "id": "ioaprt5q5US7" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -45,18 +29,16 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} + "id": "yCl0eTNH5RS3" }, + "outputs": [], "source": [ "#@title MIT License\n", "#\n", @@ -79,24 +61,20 @@ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ItXfxkxvosLH" }, "source": [ - "# Text classification with preprocessed text: Movie reviews" + "# Basic text classification" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hKY4XMc9o8iB" }, "source": [ @@ -119,543 +97,691 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Eg62Pmz3o83v" }, "source": [ + "This tutorial demonstrates text classification starting from plain text files stored on disk. You'll train a binary classifier to perform sentiment analysis on an IMDB dataset. At the end of the notebook, there is an exercise for you to try, in which you'll train a multi-class classifier to predict the tag for a programming question on Stack Overflow.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8RZOuS9LWQvv" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import os\n", + "import re\n", + "import shutil\n", + "import string\n", + "import tensorflow as tf\n", + "\n", + "from tensorflow.keras import layers\n", + "from tensorflow.keras import losses\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6-tTFS04dChr" + }, + "outputs": [], + "source": [ + "print(tf.__version__)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NBTI1bi8qdFV" + }, + "source": [ + "## Sentiment analysis\n", "\n", - "This notebook classifies movie reviews as *positive* or *negative* using the text of the review. This is an example of *binary*—or two-class—classification, an important and widely applicable kind of machine learning problem.\n", - "\n", - "We'll use the [IMDB dataset](https://www.tensorflow.org/datasets/catalog/imdb_reviews) that contains the text of 50,000 movie reviews from the [Internet Movie Database](https://www.imdb.com/). These are split into 25,000 reviews for training and 25,000 reviews for testing. The training and testing sets are *balanced*, meaning they contain an equal number of positive and negative reviews.\n", + "This notebook trains a sentiment analysis model to classify movie reviews as *positive* or *negative*, based on the text of the review. This is an example of *binary*—or two-class—classification, an important and widely applicable kind of machine learning problem.\n", "\n", - "This notebook uses [tf.keras](https://www.tensorflow.org/guide/keras), a high-level API to build and train models in TensorFlow. For a more advanced text classification tutorial using `tf.keras`, see the [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)." + "You'll use the [Large Movie Review Dataset](https://ai.stanford.edu/~amaas/data/sentiment/) that contains the text of 50,000 movie reviews from the [Internet Movie Database](https://www.imdb.com/). These are split into 25,000 reviews for training and 25,000 reviews for testing. The training and testing sets are *balanced*, meaning they contain an equal number of positive and negative reviews.\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "8vdyFn79gt1L" + "id": "iAsKG535pHep" }, "source": [ - "## Setup" + "### Download and explore the IMDB dataset\n", + "\n", + "Let's download and extract the dataset, then explore the directory structure." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Nh0KjNGMWNlL", - "colab": {} + "id": "k7ZYnuajVlFN" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ], - "execution_count": 0, - "outputs": [] + "url = \"https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\"\n", + "\n", + "dataset = tf.keras.utils.get_file(\"aclImdb_v1\", url,\n", + " untar=True, cache_dir='.',\n", + " cache_subdir='')\n", + "\n", + "dataset_dir = os.path.join(os.path.dirname(dataset), 'aclImdb_v1')" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "8RZOuS9LWQvv", - "colab": {} + "id": "355CfOvsV1pl" }, + "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] + "os.listdir(dataset_dir)" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} + "id": "7ASND15oXpF1" }, + "outputs": [], "source": [ - "from tensorflow import keras\n", - "\n", - "!pip install tensorflow-datasets\n", - "import tensorflow_datasets as tfds\n", - "tfds.disable_progress_bar()\n", + "train_dir = os.path.join(dataset_dir, 'aclImdb', 'train')\n", + "test_dir = os.path.join(dataset_dir, 'aclImdb', 'test')\n", + "os.listdir(train_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ysMNMI1CWDFD" + }, + "source": [ + "The `aclImdb/train/pos` and `aclImdb/train/neg` directories contain many text files, each of which is a single movie review. Let's take a look at one of them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "R7g8hFvzWLIZ" + }, + "outputs": [], + "source": [ + "sample_file = os.path.join(train_dir, 'pos', '1181_9.txt')\n", + "with open(sample_file) as f:\n", + " print(f.read())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mk20TEm6ZRFP" + }, + "source": [ + "### Load the dataset\n", "\n", - "import numpy as np\n", + "Next, you will load the data off disk and prepare it into a format suitable for training. To do so, you will use the helpful [text_dataset_from_directory](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text_dataset_from_directory) utility, which expects a directory structure as follows.\n", "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] + "```\n", + "main_directory/\n", + "...class_a/\n", + "......a_text_1.txt\n", + "......a_text_2.txt\n", + "...class_b/\n", + "......b_text_1.txt\n", + "......b_text_2.txt\n", + "```" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" + "id": "nQauv38Lnok3" }, "source": [ - "\n", - "\n", - "## Download the IMDB dataset\n", + "To prepare a dataset for binary classification, you will need two folders on disk, corresponding to `class_a` and `class_b`. These will be the positive and negative movie reviews, which can be found in `aclImdb/train/pos` and `aclImdb/train/neg`. As the IMDB dataset contains additional folders, you will remove them before using this utility." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VhejsClzaWfl" + }, + "outputs": [], + "source": [ + "remove_dir = os.path.join(train_dir, 'unsup')\n", + "shutil.rmtree(remove_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "95kkUdRoaeMw" + }, + "source": [ + "Next, you will use the `text_dataset_from_directory` utility to create a labeled `tf.data.Dataset`. [tf.data](https://www.tensorflow.org/guide/data) is a powerful collection of tools for working with data.\n", "\n", - "The IMDB movie reviews dataset comes packaged in `tfds`. It has already been preprocessed so that the reviews (sequences of words) have been converted to sequences of integers, where each integer represents a specific word in a dictionary.\n", + "When running a machine learning experiment, it is a best practice to divide your dataset into three splits: [train](https://developers.google.com/machine-learning/glossary#training_set), [validation](https://developers.google.com/machine-learning/glossary#validation_set), and [test](https://developers.google.com/machine-learning/glossary#test-set).\n", "\n", - "The following code downloads the IMDB dataset to your machine (or uses a cached copy if you've already downloaded it):\n", + "The IMDB dataset has already been divided into train and test, but it lacks a validation set. Let's create a validation set using an 80:20 split of the training data by using the `validation_split` argument below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nOrK-MTYaw3C" + }, + "outputs": [], + "source": [ + "batch_size = 32\n", + "seed = 42\n", "\n", - "To encode your own text see the [Loading text tutorial](../load_data/text.ipynb)" + "raw_train_ds = tf.keras.utils.text_dataset_from_directory(\n", + " train_dir,\n", + " batch_size=batch_size,\n", + " validation_split=0.2,\n", + " subset='training',\n", + " seed=seed)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5Y33oxOUpYkh" + }, + "source": [ + "As you can see above, there are 25,000 examples in the training folder, of which you will use 80% (or 20,000) for training. As you will see in a moment, you can train a model by passing a dataset directly to `model.fit`. If you're new to `tf.data`, you can also iterate over the dataset and print out a few examples as follows." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "wbIQ2wSeXSme", - "colab": {} + "id": "51wNaPPApk1K" }, + "outputs": [], "source": [ - "(train_data, test_data), info = tfds.load(\n", - " # Use the version pre-encoded with an ~8k vocabulary.\n", - " 'imdb_reviews/subwords8k', \n", - " # Return the train/test datasets as a tuple.\n", - " split = (tfds.Split.TRAIN, tfds.Split.TEST),\n", - " # Return (example, label) pairs from the dataset (instead of a dictionary).\n", - " as_supervised=True,\n", - " # Also return the `info` structure. \n", - " with_info=True)" - ], - "execution_count": 0, - "outputs": [] + "for text_batch, label_batch in raw_train_ds.take(1):\n", + " for i in range(3):\n", + " print(\"Review\", text_batch.numpy()[i])\n", + " print(\"Label\", label_batch.numpy()[i])" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "qvA8HYDJj8OU" + "id": "JWq1SUIrp1a-" }, "source": [ - "\n", - "\n", - "## Try the encoder\n", + "Notice the reviews contain raw text (with punctuation and occasional HTML tags like `
    `). You will show how to handle these in the following section.\n", "\n", - " The dataset `info` includes the text encoder (a `tfds.features.text.SubwordTextEncoder`)." + "The labels are 0 or 1. To see which of these correspond to positive and negative movie reviews, you can check the `class_names` property on the dataset.\n" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "EplYp5pNnW1S", - "colab": {} + "id": "MlICTG8spyO2" }, + "outputs": [], "source": [ - "encoder = info.features['text'].encoder" - ], - "execution_count": 0, - "outputs": [] + "print(\"Label 0 corresponds to\", raw_train_ds.class_names[0])\n", + "print(\"Label 1 corresponds to\", raw_train_ds.class_names[1])" + ] }, { - "cell_type": "code", + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "e7ACuHM5hFp3", - "colab": {} + "id": "pbdO39vYqdJr" }, "source": [ - "print ('Vocabulary size: {}'.format(encoder.vocab_size))" - ], - "execution_count": 0, - "outputs": [] + "Next, you will create a validation and test dataset. You will use the remaining 5,000 reviews from the training set for validation." + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "tAfGg8YRe6fu" + "id": "SzxazN8Hq1pF" }, "source": [ - "This text encoder will reversibly encode any string:" + "Note: When using the `validation_split` and `subset` arguments, make sure to either specify a random seed, or to pass `shuffle=False`, so that the validation and training splits have no overlap." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Bq6xDmf2SAs-", - "colab": {} + "id": "JsMwwhOoqjKF" }, + "outputs": [], "source": [ - "sample_string = 'Hello TensorFlow.'\n", + "raw_val_ds = tf.keras.utils.text_dataset_from_directory(\n", + " train_dir,\n", + " batch_size=batch_size,\n", + " validation_split=0.2,\n", + " subset='validation',\n", + " seed=seed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rdSr0Nt3q_ns" + }, + "outputs": [], + "source": [ + "raw_test_ds = tf.keras.utils.text_dataset_from_directory(\n", + " test_dir,\n", + " batch_size=batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qJmTiO0IYAjm" + }, + "source": [ + "### Prepare the dataset for training\n", "\n", - "encoded_string = encoder.encode(sample_string)\n", - "print ('Encoded string is {}'.format(encoded_string))\n", + "Next, you will standardize, tokenize, and vectorize the data using the helpful `tf.keras.layers.TextVectorization` layer.\n", "\n", - "original_string = encoder.decode(encoded_string)\n", - "print ('The original string: \"{}\"'.format(original_string))\n", + "Standardization refers to preprocessing the text, typically to remove punctuation or HTML elements to simplify the dataset. Tokenization refers to splitting strings into tokens (for example, splitting a sentence into individual words, by splitting on whitespace). Vectorization refers to converting tokens into numbers so they can be fed into a neural network. All of these tasks can be accomplished with this layer.\n", "\n", - "assert original_string == sample_string" - ], - "execution_count": 0, - "outputs": [] + "As you saw above, the reviews contain various HTML tags like `
    `. These tags will not be removed by the default standardizer in the `TextVectorization` layer (which converts text to lowercase and strips punctuation by default, but doesn't strip HTML). You will write a custom standardization function to remove the HTML." + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "TbhM970AVA8w" + "id": "ZVcHl-SLrH-u" }, "source": [ - "The encoder encodes the string by breaking it into subwords or characters if the word is not in its dictionary. So the more a string resembles the dataset, the shorter the encoded representation will be." + "Note: To prevent [training-testing skew](https://developers.google.com/machine-learning/guides/rules-of-ml#training-serving_skew) (also known as training-serving skew), it is important to preprocess the data identically at train and test time. To facilitate this, the `TextVectorization` layer can be included directly inside your model, as shown later in this tutorial." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "GUIRWSO8yxT5", - "colab": {} + "id": "SDRI_s_tX1Hk" }, + "outputs": [], "source": [ - "for ts in encoded_string:\n", - " print ('{} ----> {}'.format(ts, encoder.decode([ts])))" - ], - "execution_count": 0, - "outputs": [] + "def custom_standardization(input_data):\n", + " lowercase = tf.strings.lower(input_data)\n", + " stripped_html = tf.strings.regex_replace(lowercase, '
    ', ' ')\n", + " return tf.strings.regex_replace(stripped_html,\n", + " '[%s]' % re.escape(string.punctuation),\n", + " '')" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" + "id": "d2d3Aw8dsUux" }, "source": [ - "## Explore the data\n", - "\n", - "Let's take a moment to understand the format of the data. The dataset comes preprocessed: each example is an array of integers representing the words of the movie review. \n", - "\n", - "The text of reviews have been converted to integers, where each integer represents a specific word-piece in the dictionary. \n", + "Next, you will create a `TextVectorization` layer. You will use this layer to standardize, tokenize, and vectorize our data. You set the `output_mode` to `int` to create unique integer indices for each token.\n", "\n", - "Each label is an integer value of either 0 or 1, where 0 is a negative review, and 1 is a positive review.\n", + "Note that you're using the default split function, and the custom standardization function you defined above. You'll also define some constants for the model, like an explicit maximum `sequence_length`, which will cause the layer to pad or truncate sequences to exactly `sequence_length` values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-c76RvSzsMnX" + }, + "outputs": [], + "source": [ + "max_features = 10000\n", + "sequence_length = 250\n", "\n", - "Here's what the first review looks like:" + "vectorize_layer = layers.TextVectorization(\n", + " standardize=custom_standardization,\n", + " max_tokens=max_features,\n", + " output_mode='int',\n", + " output_sequence_length=sequence_length)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vlFOpfF6scT6" + }, + "source": [ + "Next, you will call `adapt` to fit the state of the preprocessing layer to the dataset. This will cause the model to build an index of strings to integers." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lAhdjK7AtroA" + }, + "source": [ + "Note: It's important to only use your training data when calling adapt (using the test set would leak information)." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "cxnWQJijdGA1", - "colab": {} + "id": "GH4_2ZGJsa_X" }, + "outputs": [], "source": [ - "for train_example, train_label in train_data.take(1):\n", - " print('Encoded text:', train_example[:10].numpy())\n", - " print('Label:', train_label.numpy())" - ], - "execution_count": 0, - "outputs": [] + "# Make a text-only dataset (without labels), then call adapt\n", + "train_text = raw_train_ds.map(lambda x, y: x)\n", + "vectorize_layer.adapt(train_text)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "wy0v9Hs4v41q" + "id": "SHQVEFzNt-K_" + }, + "source": [ + "Let's create a function to see the result of using this layer to preprocess some data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SCIg_T50wOCU" }, + "outputs": [], "source": [ - "The `info` structure contains the encoder/decoder. The encoder can be used to recover the original text:" + "def vectorize_text(text, label):\n", + " text = tf.expand_dims(text, -1)\n", + " return vectorize_layer(text), label" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "34VUXtgxsVpf", - "colab": {} + "id": "XULcm6B3xQIO" }, + "outputs": [], "source": [ - "encoder.decode(train_example)" - ], - "execution_count": 0, - "outputs": [] + "# retrieve a batch (of 32 reviews and labels) from the dataset\n", + "text_batch, label_batch = next(iter(raw_train_ds))\n", + "first_review, first_label = text_batch[0], label_batch[0]\n", + "print(\"Review\", first_review)\n", + "print(\"Label\", raw_train_ds.class_names[first_label])\n", + "print(\"Vectorized review\", vectorize_text(first_review, first_label))" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "qJmTiO0IYAjm" + "id": "6u5EX0hxyNZT" }, "source": [ - "## Prepare the data for training\n", - "\n", - "You will want to create batches of training data for your model. The reviews are all different lengths, so use `padded_batch` to zero pad the sequences while batching:" + "As you can see above, each token has been replaced by an integer. You can lookup the token (string) that each integer corresponds to by calling `.get_vocabulary()` on the layer." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "SDRI_s_tX1Hk", - "colab": {} + "id": "kRq9hTQzhVhW" }, + "outputs": [], "source": [ - "BUFFER_SIZE = 1000\n", - "\n", - "train_batches = (\n", - " train_data\n", - " .shuffle(BUFFER_SIZE)\n", - " .padded_batch(32, train_data.output_shapes))\n", - "\n", - "test_batches = (\n", - " test_data\n", - " .padded_batch(32, train_data.output_shapes))" - ], - "execution_count": 0, - "outputs": [] + "print(\"1287 ---> \",vectorize_layer.get_vocabulary()[1287])\n", + "print(\" 313 ---> \",vectorize_layer.get_vocabulary()[313])\n", + "print('Vocabulary size: {}'.format(len(vectorize_layer.get_vocabulary())))" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "9D9pIr0JwvRl" + "id": "XD2H6utRydGv" }, "source": [ - "Each batch will have a shape of `(batch_size, sequence_length)` because the padding is dynamic each batch will have a different length:" + "You are nearly ready to train your model. As a final preprocessing step, you will apply the TextVectorization layer you created earlier to the train, validation, and test dataset." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "sXXne4DreQfv", - "colab": {} + "id": "2zhmpeViI1iG" }, + "outputs": [], "source": [ - "for example_batch, label_batch in train_batches.take(2):\n", - " print(\"Batch shape:\", example_batch.shape)\n", - " print(\"label shape:\", label_batch.shape)\n", - " " - ], - "execution_count": 0, - "outputs": [] + "train_ds = raw_train_ds.map(vectorize_text)\n", + "val_ds = raw_val_ds.map(vectorize_text)\n", + "test_ds = raw_test_ds.map(vectorize_text)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" + "id": "YsVQyPMizjuO" }, "source": [ - "## Build the model\n", + "### Configure the dataset for performance\n", "\n", - "The neural network is created by stacking layers—this requires two main architectural decisions:\n", + "These are two important methods you should use when loading data to make sure that I/O does not become blocking.\n", "\n", - "* How many layers to use in the model?\n", - "* How many *hidden units* to use for each layer?\n", + "`.cache()` keeps data in memory after it's loaded off disk. This will ensure the dataset does not become a bottleneck while training your model. If your dataset is too large to fit into memory, you can also use this method to create a performant on-disk cache, which is more efficient to read than many small files.\n", "\n", - "In this example, the input data consists of an array of word-indices. The labels to predict are either 0 or 1. Let's build a \"Continuous bag of words\" style model for this problem:\n", + "`.prefetch()` overlaps data preprocessing and model execution while training.\n", "\n", - "Caution: This model doesn't use masking, so the zero-padding is used as part of the input, so the padding length may affect the output. To fix this, see the [masking and padding guide](../../guide/keras/masking_and_padding)." + "You can learn more about both methods, as well as how to cache data to disk in the [data performance guide](https://www.tensorflow.org/guide/data_performance)." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} + "id": "wMcs_H7izm5m" }, + "outputs": [], "source": [ - "model = keras.Sequential([\n", - " keras.layers.Embedding(encoder.vocab_size, 16),\n", - " keras.layers.GlobalAveragePooling1D(),\n", - " keras.layers.Dense(1, activation='sigmoid')])\n", + "AUTOTUNE = tf.data.AUTOTUNE\n", "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] + "train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n", + "val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)\n", + "test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" + "id": "LLC02j2g-llC" }, "source": [ - "The layers are stacked sequentially to build the classifier:\n", + "### Create the model\n", "\n", - "1. The first layer is an `Embedding` layer. This layer takes the integer-encoded vocabulary and looks up the embedding vector for each word-index. These vectors are learned as the model trains. The vectors add a dimension to the output array. The resulting dimensions are: `(batch, sequence, embedding)`.\n", - "2. Next, a `GlobalAveragePooling1D` layer returns a fixed-length output vector for each example by averaging over the sequence dimension. This allows the model to handle input of variable length, in the simplest way possible.\n", - "3. This fixed-length output vector is piped through a fully-connected (`Dense`) layer with 16 hidden units.\n", - "4. The last layer is densely connected with a single output node. Using the `sigmoid` activation function, this value is a float between 0 and 1, representing a probability, or confidence level." + "It's time to create your neural network:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dkQP6in8yUBR" + }, + "outputs": [], + "source": [ + "embedding_dim = 16" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" + "id": "xpKOoWgu-llD" }, + "outputs": [], "source": [ - "### Hidden units\n", + "model = tf.keras.Sequential([\n", + " layers.Embedding(max_features, embedding_dim),\n", + " layers.Dropout(0.2),\n", + " layers.GlobalAveragePooling1D(),\n", + " layers.Dropout(0.2),\n", + " layers.Dense(1, activation='sigmoid')])\n", "\n", - "The above model has two intermediate or \"hidden\" layers, between the input and output. The number of outputs (units, nodes, or neurons) is the dimension of the representational space for the layer. In other words, the amount of freedom the network is allowed when learning an internal representation.\n", + "model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6PbKQ6mucuKL" + }, + "source": [ + "The layers are stacked sequentially to build the classifier:\n", "\n", - "If a model has more hidden units (a higher-dimensional representation space), and/or more layers, then the network can learn more complex representations. However, it makes the network more computationally expensive and may lead to learning unwanted patterns—patterns that improve performance on training data but not on the test data. This is called *overfitting*, and we'll explore it later." + "1. The first layer is an `Embedding` layer. This layer takes the integer-encoded reviews and looks up an embedding vector for each word-index. These vectors are learned as the model trains. The vectors add a dimension to the output array. The resulting dimensions are: `(batch, sequence, embedding)`. To learn more about embeddings, check out the [Word embeddings](https://www.tensorflow.org/text/guide/word_embeddings) tutorial.\n", + "2. Next, a `GlobalAveragePooling1D` layer returns a fixed-length output vector for each example by averaging over the sequence dimension. This allows the model to handle input of variable length, in the simplest way possible.\n", + "3. The last layer is densely connected with a single output node." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "L4EqVWg4-llM" }, "source": [ "### Loss function and optimizer\n", "\n", - "A model needs a loss function and an optimizer for training. Since this is a binary classification problem and the model outputs a probability (a single-unit layer with a sigmoid activation), we'll use the `binary_crossentropy` loss function.\n", - "\n", - "This isn't the only choice for a loss function, you could, for instance, choose `mean_squared_error`. But, generally, `binary_crossentropy` is better for dealing with probabilities—it measures the \"distance\" between probability distributions, or in our case, between the ground-truth distribution and the predictions.\n", - "\n", - "Later, when we are exploring regression problems (say, to predict the price of a house), we will see how to use another loss function called mean squared error.\n", + "A model needs a loss function and an optimizer for training. Since this is a binary classification problem and the model outputs a probability (a single-unit layer with a sigmoid activation), you'll use `losses.BinaryCrossentropy` loss function.\n", "\n", "Now, configure the model to use an optimizer and a loss function:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} + "id": "Mr0GP-cQ-llN" }, + "outputs": [], "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] + "model.compile(loss=losses.BinaryCrossentropy(),\n", + " optimizer='adam',\n", + " metrics=[tf.metrics.BinaryAccuracy(threshold=0.5)])" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "35jv_fzP-llU" }, "source": [ - "## Train the model\n", + "### Train the model\n", "\n", - "Train the model by passing the `Dataset` object to the model's fit function. Set the number of epochs." + "You will train the model by passing the `dataset` object to the fit method." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} + "id": "tXSGrjWZ-llW" }, + "outputs": [], "source": [ - "history = model.fit(train_batches,\n", - " epochs=10,\n", - " validation_data=test_batches,\n", - " validation_steps=30)" - ], - "execution_count": 0, - "outputs": [] + "epochs = 10\n", + "history = model.fit(\n", + " train_ds,\n", + " validation_data=val_ds,\n", + " epochs=epochs)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9EEGuDVuzb5r" }, "source": [ - "## Evaluate the model\n", + "### Evaluate the model\n", "\n", - "And let's see how the model performs. Two values will be returned. Loss (a number which represents our error, lower values are better), and accuracy." + "Let's see how the model performs. Two values will be returned. Loss (a number which represents our error, lower values are better), and accuracy." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} + "id": "zOMKywn4zReN" }, + "outputs": [], "source": [ - "loss, accuracy = model.evaluate(test_batches)\n", + "loss, accuracy = model.evaluate(test_ds)\n", "\n", "print(\"Loss: \", loss)\n", "print(\"Accuracy: \", accuracy)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "z1iEXVTR0Z2t" }, "source": [ - "This fairly naive approach achieves an accuracy of about 87%. With more advanced approaches, the model should get closer to 95%." + "This fairly naive approach achieves an accuracy of about 86%." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" + "id": "ldbQqCw2Xc1W" }, "source": [ - "## Create a graph of accuracy and loss over time\n", + "### Create a plot of accuracy and loss over time\n", "\n", "`model.fit()` returns a `History` object that contains a dictionary with everything that happened during training:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} + "id": "-YcvZsdvWfDf" }, + "outputs": [], "source": [ "history_dict = history.history\n", "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" + "id": "1_CH32qJXruI" }, "source": [ - "There are four entries: one for each monitored metric during training and validation. We can use these to plot the training and validation loss for comparison, as well as the training and validation accuracy:" + "There are four entries: one for each monitored metric during training and validation. You can use these to plot the training and validation loss for comparison, as well as the training and validation accuracy:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} + "id": "2SEMeQ5YXs8z" }, + "outputs": [], "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history_dict['accuracy']\n", - "val_acc = history_dict['val_accuracy']\n", + "acc = history_dict['binary_accuracy']\n", + "val_acc = history_dict['val_binary_accuracy']\n", "loss = history_dict['loss']\n", "val_loss = history_dict['val_loss']\n", "\n", @@ -671,20 +797,16 @@ "plt.legend()\n", "\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} + "id": "Z3PJemLPXwz_" }, + "outputs": [], "source": [ - "plt.clf() # clear figure\n", - "\n", "plt.plot(epochs, acc, 'bo', label='Training acc')\n", "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", "plt.title('Training and validation accuracy')\n", @@ -693,26 +815,168 @@ "plt.legend(loc='lower right')\n", "\n", "plt.show()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" + "id": "hFFyCuJoXy7r" }, "source": [ - "\n", "In this plot, the dots represent the training loss and accuracy, and the solid lines are the validation loss and accuracy.\n", "\n", "Notice the training loss *decreases* with each epoch and the training accuracy *increases* with each epoch. This is expected when using a gradient descent optimization—it should minimize the desired quantity on every iteration.\n", "\n", - "This isn't the case for the validation loss and accuracy—they seem to peak after about twenty epochs. This is an example of overfitting: the model performs better on the training data than it does on data it has never seen before. After this point, the model over-optimizes and learns representations *specific* to the training data that do not *generalize* to test data.\n", + "This isn't the case for the validation loss and accuracy—they seem to peak before the training accuracy. This is an example of overfitting: the model performs better on the training data than it does on data it has never seen before. After this point, the model over-optimizes and learns representations *specific* to the training data that do not *generalize* to test data.\n", + "\n", + "For this particular case, you could prevent overfitting by simply stopping the training when the validation accuracy is no longer increasing. One way to do so is to use the `tf.keras.callbacks.EarlyStopping` callback." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-to23J3Vy5d3" + }, + "source": [ + "## Export the model\n", + "\n", + "In the code above, you applied the `TextVectorization` layer to the dataset before feeding text to the model. If you want to make your model capable of processing raw strings (for example, to simplify deploying it), you can include the `TextVectorization` layer inside your model. To do so, you can create a new model using the weights you just trained." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FWXsMvryuZuq" + }, + "outputs": [], + "source": [ + "export_model = tf.keras.Sequential([\n", + " vectorize_layer,\n", + " model,\n", + " layers.Activation('sigmoid')\n", + "])\n", + "\n", + "export_model.compile(\n", + " loss=losses.BinaryCrossentropy(from_logits=False), optimizer=\"adam\", metrics=['accuracy']\n", + ")\n", + "\n", + "# Test it with `raw_test_ds`, which yields raw strings\n", + "metrics = export_model.evaluate(raw_test_ds, return_dict=True)\n", + "print(metrics)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TwQgoN88LoEF" + }, + "source": [ + "### Inference on new data\n", "\n", - "For this particular case, we could prevent overfitting by simply stopping the training after twenty or so epochs. Later, you'll see how to do this automatically with a callback." + "To get predictions for new examples, you can simply call `model.predict()`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QW355HH5L49K" + }, + "outputs": [], + "source": [ + "examples = tf.constant([\n", + " \"The movie was great!\",\n", + " \"The movie was okay.\",\n", + " \"The movie was terrible...\"\n", + "])\n", + "\n", + "export_model.predict(examples)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MaxlpFWpzR6c" + }, + "source": [ + "Including the text preprocessing logic inside your model enables you to export a model for production that simplifies deployment, and reduces the potential for [train/test skew](https://developers.google.com/machine-learning/guides/rules-of-ml#training-serving_skew).\n", + "\n", + "There is a performance difference to keep in mind when choosing where to apply your TextVectorization layer. Using it outside of your model enables you to do asynchronous CPU processing and buffering of your data when training on GPU. So, if you're training your model on the GPU, you probably want to go with this option to get the best performance while developing your model, then switch to including the TextVectorization layer inside your model when you're ready to prepare for deployment.\n", + "\n", + "Visit this [tutorial](https://www.tensorflow.org/tutorials/keras/save_and_load) to learn more about saving models." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eSSuci_6nCEG" + }, + "source": [ + "## Exercise: multi-class classification on Stack Overflow questions\n", + "\n", + "This tutorial showed how to train a binary classifier from scratch on the IMDB dataset. As an exercise, you can modify this notebook to train a multi-class classifier to predict the tag of a programming question on [Stack Overflow](http://stackoverflow.com/).\n", + "\n", + "A [dataset](https://storage.googleapis.com/download.tensorflow.org/data/stack_overflow_16k.tar.gz) has been prepared for you to use containing the body of several thousand programming questions (for example, \"How can I sort a dictionary by value in Python?\") posted to Stack Overflow. Each of these is labeled with exactly one tag (either Python, CSharp, JavaScript, or Java). Your task is to take a question as input, and predict the appropriate tag, in this case, Python.\n", + "\n", + "The dataset you will work with contains several thousand questions extracted from the much larger public Stack Overflow dataset on [BigQuery](https://console.cloud.google.com/marketplace/details/stack-exchange/stack-overflow), which contains more than 17 million posts.\n", + "\n", + "After downloading the dataset, you will find it has a similar directory structure to the IMDB dataset you worked with previously:\n", + "\n", + "```\n", + "train/\n", + "...python/\n", + "......0.txt\n", + "......1.txt\n", + "...javascript/\n", + "......0.txt\n", + "......1.txt\n", + "...csharp/\n", + "......0.txt\n", + "......1.txt\n", + "...java/\n", + "......0.txt\n", + "......1.txt\n", + "```\n", + "\n", + "Note: To increase the difficulty of the classification problem, occurrences of the words Python, CSharp, JavaScript, or Java in the programming questions have been replaced with the word *blank* (as many questions contain the language they're about).\n", + "\n", + "To complete this exercise, you should modify this notebook to work with the Stack Overflow dataset by making the following modifications:\n", + "\n", + "1. At the top of your notebook, update the code that downloads the IMDB dataset with code to download the [Stack Overflow dataset](https://storage.googleapis.com/download.tensorflow.org/data/stack_overflow_16k.tar.gz) that has already been prepared. As the Stack Overflow dataset has a similar directory structure, you will not need to make many modifications.\n", + "\n", + "1. Modify the last layer of your model to `Dense(4)`, as there are now four output classes.\n", + "\n", + "1. When compiling the model, change the loss to `tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)`. This is the correct loss function to use for a multi-class classification problem, when the labels for each class are integers (in this case, they can be 0, *1*, *2*, or *3*). In addition, change the metrics to `metrics=['accuracy']`, since this is a multi-class classification problem (`tf.metrics.BinaryAccuracy` is only used for binary classifiers).\n", + "\n", + "1. When plotting accuracy over time, change `binary_accuracy` and `val_binary_accuracy` to `accuracy` and `val_accuracy`, respectively.\n", + "\n", + "1. Once these changes are complete, you will be able to train a multi-class classifier." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F0T5SIwSm7uc" + }, + "source": [ + "## Learning more\n", + "\n", + "This tutorial introduced text classification from scratch. To learn more about the text classification workflow in general, check out the [Text classification guide](https://developers.google.com/machine-learning/guides/text-classification/) from Google Developers.\n" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "text_classification.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/keras/text_classification_with_hub.ipynb b/site/en/tutorials/keras/text_classification_with_hub.ipynb index 6a732010ec6..e7e4288ef32 100644 --- a/site/en/tutorials/keras/text_classification_with_hub.ipynb +++ b/site/en/tutorials/keras/text_classification_with_hub.ipynb @@ -1,24 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_classification_with_hub.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ic4_occAAiAT" }, "source": [ @@ -27,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} + "id": "ioaprt5q5US7" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -45,18 +29,16 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} + "id": "yCl0eTNH5RS3" }, + "outputs": [], "source": [ "#@title MIT License\n", "#\n", @@ -79,14 +61,11 @@ "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ItXfxkxvosLH" }, "source": [ @@ -96,7 +75,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hKY4XMc9o8iB" }, "source": [ @@ -108,99 +86,97 @@ " Run in Google Colab\n", " \n", " \n", - " View source on GitHub\n", + " View on GitHub\n", " \n", " \n", " Download notebook\n", " \n", + " \n", + " See TF Hub models\n", + " \n", "" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Eg62Pmz3o83v" }, "source": [ - "\n", "This notebook classifies movie reviews as *positive* or *negative* using the text of the review. This is an example of *binary*—or two-class—classification, an important and widely applicable kind of machine learning problem.\n", "\n", - "The tutorial demonstrates the basic application of transfer learning with TensorFlow Hub and Keras.\n", + "The tutorial demonstrates the basic application of transfer learning with [TensorFlow Hub](https://tfhub.dev) and Keras.\n", "\n", - "We'll use the [IMDB dataset](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) that contains the text of 50,000 movie reviews from the [Internet Movie Database](https://www.imdb.com/). These are split into 25,000 reviews for training and 25,000 reviews for testing. The training and testing sets are *balanced*, meaning they contain an equal number of positive and negative reviews. \n", + "It uses the [IMDB dataset](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) that contains the text of 50,000 movie reviews from the [Internet Movie Database](https://www.imdb.com/). These are split into 25,000 reviews for training and 25,000 reviews for testing. The training and testing sets are *balanced*, meaning they contain an equal number of positive and negative reviews. \n", "\n", - "This notebook uses [tf.keras](https://www.tensorflow.org/guide/keras), a high-level API to build and train models in TensorFlow, and [TensorFlow Hub](https://www.tensorflow.org/hub), a library and platform for transfer learning. For a more advanced text classification tutorial using `tf.keras`, see the [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)." + "This notebook uses [`tf.keras`](https://www.tensorflow.org/guide/keras), a high-level API to build and train models in TensorFlow, and [`tensorflow_hub`](https://www.tensorflow.org/hub), a library for loading trained models from [TFHub](https://tfhub.dev) in a single line of code. For a more advanced text classification tutorial using `tf.keras`, see the [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} + "id": "IHTzYqKZ7auw" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", + "!pip install tensorflow-hub\n", + "!pip install tensorflow-datasets\n", + "!pip install tf-keras" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2ew7HTbPpCJH" + }, + "outputs": [], + "source": [ + "import os\n", "import numpy as np\n", "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", - "\n", - "!pip install tensorflow-hub\n", - "!pip install tensorflow-datasets\n", "import tensorflow_hub as hub\n", "import tensorflow_datasets as tfds\n", + "import tf_keras as keras\n", "\n", "print(\"Version: \", tf.__version__)\n", "print(\"Eager mode: \", tf.executing_eagerly())\n", "print(\"Hub version: \", hub.__version__)\n", - "print(\"GPU is\", \"available\" if tf.config.experimental.list_physical_devices(\"GPU\") else \"NOT AVAILABLE\")" - ], - "execution_count": 0, - "outputs": [] + "print(\"GPU is\", \"available\" if tf.config.list_physical_devices(\"GPU\") else \"NOT AVAILABLE\")" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iAsKG535pHep" }, "source": [ "## Download the IMDB dataset\n", "\n", - "The IMDB dataset is available on [imdb reviews](https://github.com/tensorflow/datasets/blob/master/docs/datasets.md#imdb_reviews) or on [TensorFlow datasets](https://github.com/tensorflow/datasets). The following code downloads the IMDB dataset to your machine (or the colab runtime):" + "The IMDB dataset is available on [imdb reviews](https://www.tensorflow.org/datasets/catalog/imdb_reviews) or on [TensorFlow datasets](https://www.tensorflow.org/datasets). The following code downloads the IMDB dataset to your machine (or the colab runtime):" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} + "id": "zXXx5Oc3pOmN" }, + "outputs": [], "source": [ - "# Split the training set into 60% and 40%, so we'll end up with 15,000 examples\n", + "# Split the training set into 60% and 40% to end up with 15,000 examples\n", "# for training, 10,000 examples for validation and 25,000 examples for testing.\n", - "train_validation_split = tfds.Split.TRAIN.subsplit([6, 4])\n", - "\n", - "(train_data, validation_data), test_data = tfds.load(\n", + "train_data, validation_data, test_data = tfds.load(\n", " name=\"imdb_reviews\", \n", - " split=(train_validation_split, tfds.Split.TEST),\n", + " split=('train[:60%]', 'train[60%:]', 'test'),\n", " as_supervised=True)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "l50X3GfjpU4r" }, "source": [ @@ -213,22 +189,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} + "id": "QtTS4kpEpjbi" }, + "outputs": [], "source": [ "train_examples_batch, train_labels_batch = next(iter(train_data.batch(10)))\n", "train_examples_batch" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IFtaCHTdc-GY" }, "source": [ @@ -237,21 +210,18 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "tvAjVXOWc6Mj", - "colab": {} + "id": "tvAjVXOWc6Mj" }, + "outputs": [], "source": [ "train_labels_batch" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LLC02j2g-llC" }, "source": [ @@ -265,25 +235,26 @@ "\n", "In this example, the input data consists of sentences. The labels to predict are either 0 or 1.\n", "\n", - "One way to represent the text is to convert sentences into embeddings vectors. We can use a pre-trained text embedding as the first layer, which will have three advantages:\n", + "One way to represent the text is to convert sentences into embeddings vectors. Use a pre-trained text embedding as the first layer, which will have three advantages:\n", "\n", - "* we don't have to worry about text preprocessing,\n", - "* we can benefit from transfer learning,\n", + "* You don't have to worry about text preprocessing,\n", + "* Benefit from transfer learning,\n", "* the embedding has a fixed size, so it's simpler to process.\n", "\n", - "For this example we will use a **pre-trained text embedding model** from [TensorFlow Hub](https://www.tensorflow.org/hub) called [google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1).\n", + "For this example you use a **pre-trained text embedding model** from [TensorFlow Hub](https://tfhub.dev) called [google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2).\n", + "\n", + "There are many other pre-trained text embeddings from TFHub that can be used in this tutorial:\n", "\n", - "There are three other pre-trained models to test for the sake of this tutorial:\n", + "* [google/nnlm-en-dim128/2](https://tfhub.dev/google/nnlm-en-dim128/2) - trained with the same NNLM architecture on the same data as [google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2), but with a larger embedding dimension. Larger dimensional embeddings can improve on your task but it may take longer to train your model.\n", + "* [google/nnlm-en-dim128-with-normalization/2](https://tfhub.dev/google/nnlm-en-dim128-with-normalization/2) - the same as [google/nnlm-en-dim128/2](https://tfhub.dev/google/nnlm-en-dim128/2), but with additional text normalization such as removing punctuation. This can help if the text in your task contains additional characters or punctuation.\n", + "* [google/universal-sentence-encoder/4](https://tfhub.dev/google/universal-sentence-encoder/4) - a much larger model yielding 512 dimensional embeddings trained with a deep averaging network (DAN) encoder.\n", "\n", - "* [google/tf2-preview/gnews-swivel-20dim-with-oov/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim-with-oov/1) - same as [google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1), but with 2.5% vocabulary converted to OOV buckets. This can help if vocabulary of the task and vocabulary of the model don't fully overlap.\n", - "* [google/tf2-preview/nnlm-en-dim50/1](https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1) - A much larger model with ~1M vocabulary size and 50 dimensions.\n", - "* [google/tf2-preview/nnlm-en-dim128/1](https://tfhub.dev/google/tf2-preview/nnlm-en-dim128/1) - Even larger model with ~1M vocabulary size and 128 dimensions." + "And many more! Find more [text embedding models](https://tfhub.dev/s?module-type=text-embedding) on TFHub." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "In2nDpTLkgKa" }, "source": [ @@ -292,24 +263,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "_NUbzVeYkgcO", - "colab": {} + "id": "_NUbzVeYkgcO" }, + "outputs": [], "source": [ - "embedding = \"https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1\"\n", + "embedding = \"https://tfhub.dev/google/nnlm-en-dim50/2\"\n", "hub_layer = hub.KerasLayer(embedding, input_shape=[], \n", " dtype=tf.string, trainable=True)\n", "hub_layer(train_examples_batch[:3])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "dfSbV6igl1EH" }, "source": [ @@ -318,34 +286,31 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} + "id": "xpKOoWgu-llD" }, + "outputs": [], "source": [ - "model = tf.keras.Sequential()\n", + "model = keras.Sequential()\n", "model.add(hub_layer)\n", - "model.add(tf.keras.layers.Dense(16, activation='relu'))\n", - "model.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n", + "model.add(keras.layers.Dense(16, activation='relu'))\n", + "model.add(keras.layers.Dense(1))\n", "\n", "model.summary()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6PbKQ6mucuKL" }, "source": [ "The layers are stacked sequentially to build the classifier:\n", "\n", - "1. The first layer is a TensorFlow Hub layer. This layer uses a pre-trained Saved Model to map a sentence into its embedding vector. The pre-trained text embedding model that we are using ([google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1)) splits the sentence into tokens, embeds each token and then combines the embedding. The resulting dimensions are: `(num_examples, embedding_dimension)`.\n", + "1. The first layer is a TensorFlow Hub layer. This layer uses a pre-trained Saved Model to map a sentence into its embedding vector. The pre-trained text embedding model that you are using ([google/nnlm-en-dim50/2](https://tfhub.dev/google/nnlm-en-dim50/2)) splits the sentence into tokens, embeds each token and then combines the embedding. The resulting dimensions are: `(num_examples, embedding_dimension)`. For this NNLM model, the `embedding_dimension` is 50.\n", "2. This fixed-length output vector is piped through a fully-connected (`Dense`) layer with 16 hidden units.\n", - "3. The last layer is densely connected with a single output node. Using the `sigmoid` activation function, this value is a float between 0 and 1, representing a probability, or confidence level.\n", + "3. The last layer is densely connected with a single output node.\n", "\n", "Let's compile the model." ] @@ -353,68 +318,61 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "L4EqVWg4-llM" }, "source": [ "### Loss function and optimizer\n", "\n", - "A model needs a loss function and an optimizer for training. Since this is a binary classification problem and the model outputs a probability (a single-unit layer with a sigmoid activation), we'll use the `binary_crossentropy` loss function. \n", + "A model needs a loss function and an optimizer for training. Since this is a binary classification problem and the model outputs logits (a single-unit layer with a linear activation), you'll use the `binary_crossentropy` loss function.\n", "\n", "This isn't the only choice for a loss function, you could, for instance, choose `mean_squared_error`. But, generally, `binary_crossentropy` is better for dealing with probabilities—it measures the \"distance\" between probability distributions, or in our case, between the ground-truth distribution and the predictions.\n", "\n", - "Later, when we are exploring regression problems (say, to predict the price of a house), we will see how to use another loss function called mean squared error.\n", + "Later, when you are exploring regression problems (say, to predict the price of a house), you'll see how to use another loss function called mean squared error.\n", "\n", "Now, configure the model to use an optimizer and a loss function:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} + "id": "Mr0GP-cQ-llN" }, + "outputs": [], "source": [ "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", + " loss=keras.losses.BinaryCrossentropy(from_logits=True),\n", " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "35jv_fzP-llU" }, "source": [ "## Train the model\n", "\n", - "Train the model for 20 epochs in mini-batches of 512 samples. This is 20 iterations over all samples in the `x_train` and `y_train` tensors. While training, monitor the model's loss and accuracy on the 10,000 samples from the validation set:" + "Train the model for 10 epochs in mini-batches of 512 samples. This is 10 iterations over all samples in the `x_train` and `y_train` tensors. While training, monitor the model's loss and accuracy on the 10,000 samples from the validation set:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} + "id": "tXSGrjWZ-llW" }, + "outputs": [], "source": [ "history = model.fit(train_data.shuffle(10000).batch(512),\n", - " epochs=20,\n", + " epochs=10,\n", " validation_data=validation_data.batch(512),\n", " verbose=1)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "9EEGuDVuzb5r" }, "source": [ @@ -425,24 +383,21 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} + "id": "zOMKywn4zReN" }, + "outputs": [], "source": [ "results = model.evaluate(test_data.batch(512), verbose=2)\n", "\n", "for name, value in zip(model.metrics_names, results):\n", " print(\"%s: %.3f\" % (name, value))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "z1iEXVTR0Z2t" }, "source": [ @@ -452,14 +407,27 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5KggXVeL-llZ" }, "source": [ "## Further reading\n", "\n", - "For a more general way to work with string inputs and for a more detailed analysis of the progress of accuracy and loss during training, take a look [here](https://www.tensorflow.org/tutorials/keras/basic_text_classification)." + "* For a more general way to work with string inputs and for a more detailed analysis of the progress of accuracy and loss during training, see the [Text classification with preprocessed text](./text_classification.ipynb) tutorial.\n", + "* Try out more [text-related tutorials](https://www.tensorflow.org/hub/tutorials#text-related-tutorials) using trained models from TFHub." ] } - ] + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "text_classification_with_hub.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/site/en/tutorials/load_data/csv.ipynb b/site/en/tutorials/load_data/csv.ipynb index 377398f79cd..7778af974b3 100644 --- a/site/en/tutorials/load_data/csv.ipynb +++ b/site/en/tutorials/load_data/csv.ipynb @@ -3,21 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DweYe9FcbMK_" }, "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" + "##### Copyright 2019 The TensorFlow Authors.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "AVV2e0XKbJeX" }, "outputs": [], @@ -38,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "sUtoed20cRJJ" }, "source": [ @@ -48,964 +43,1834 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1ap_W4aQcgNT" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/load_data/csv\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/load_data/csv.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/load_data/csv.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/load_data/csv.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "C-3Xbt0FfGfs" }, "source": [ - "This tutorial provides an example of how to load CSV data from a file into a `tf.data.Dataset`.\n", + "This tutorial provides examples of how to use CSV data with TensorFlow.\n", "\n", - "The data used in this tutorial are taken from the Titanic passenger list. The model will predict the likelihood a passenger survived based on characteristics like age, gender, ticket class, and whether the person was traveling alone." + "There are two main parts to this:\n", + "\n", + "1. **Loading the data off disk**\n", + "2. **Pre-processing it into a form suitable for training.**\n", + "\n", + "This tutorial focuses on the loading, and gives some quick examples of preprocessing. To learn more about the preprocessing aspect, check out the [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) guide and the [Classify structured data using Keras preprocessing layers](../structured_data/preprocessing_layers.ipynb) tutorial.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fgZ9gjmPfSnK" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "baYFZMW_bJHh" + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "# Make numpy values easier to read.\n", + "np.set_printoptions(precision=3, suppress=True)\n", + "\n", + "import tensorflow as tf\n", + "from tensorflow.keras import layers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1ZhJYbJxHNGJ" + }, + "source": [ + "## In memory data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ny5TEgcmHjVx" + }, + "source": [ + "For any small CSV dataset the simplest way to train a TensorFlow model on it is to load it into memory as a [pandas `DataFrame`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) or a NumPy array.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LgpBOuU8PGFf" + }, + "source": [ + "A relatively simple example is the [abalone dataset](https://archive.ics.uci.edu/ml/datasets/abalone).\n", + "\n", + "* The dataset is small.\n", + "* All the input features are limited-range floating point values.\n", + "\n", + "Here is how to download the data into a `DataFrame`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IZVExo9DKoNz" + }, + "outputs": [], + "source": [ + "abalone_train = pd.read_csv(\n", + " \"https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\",\n", + " names=[\"Length\", \"Diameter\", \"Height\", \"Whole weight\", \"Shucked weight\",\n", + " \"Viscera weight\", \"Shell weight\", \"Age\"])\n", + "\n", + "abalone_train.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hP22mdyPQ1_t" + }, + "source": [ + "The dataset contains a set of measurements of [abalone](https://en.wikipedia.org/wiki/Abalone), a type of sea snail.\n", + "\n", + "![an abalone shell](https://tensorflow.org/images/abalone_shell.jpg)\n", + "\n", + " [“Abalone shell”](https://www.flickr.com/photos/thenickster/16641048623/) (by [Nicki Dugan Pogue](https://www.flickr.com/photos/thenickster/), CC BY-SA 2.0)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vlfGrk_9N-wf" + }, + "source": [ + "The nominal task for this dataset is to predict the age from the other measurements, so separate the features and labels for training:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "udOnDJOxNi7p" + }, + "outputs": [], + "source": [ + "abalone_features = abalone_train.copy()\n", + "abalone_labels = abalone_features.pop('Age')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "seK9n71-UBfT" + }, + "source": [ + "For this dataset you will treat all features identically. Pack the features into a single NumPy array.:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Dp3N5McbUMwb" + }, + "outputs": [], + "source": [ + "abalone_features = np.array(abalone_features)\n", + "abalone_features" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1C1yFOxLOdxh" + }, + "source": [ + "Next make a regression model predict the age. Since there is only a single input tensor, a `tf.keras.Sequential` model is sufficient here." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "d8zzNrZqOmfB" + }, + "outputs": [], + "source": [ + "abalone_model = tf.keras.Sequential([\n", + " layers.Dense(64, activation='relu'),\n", + " layers.Dense(1)\n", + "])\n", + "\n", + "abalone_model.compile(loss = tf.keras.losses.MeanSquaredError(),\n", + " optimizer = tf.keras.optimizers.Adam())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j6IWeP78O2wE" + }, + "source": [ + "To train that model, pass the features and labels to `Model.fit`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uZdpCD92SN3Z" + }, + "outputs": [], + "source": [ + "abalone_model.fit(abalone_features, abalone_labels, epochs=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GapLOj1OOTQH" + }, + "source": [ + "You have just seen the most basic way to train a model using CSV data. Next, you will learn how to apply preprocessing to normalize numeric columns." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B87Rd1SOUv02" + }, + "source": [ + "## Basic preprocessing" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yCrB2Jd-U0Vt" + }, + "source": [ + "It's good practice to normalize the inputs to your model. The Keras preprocessing layers provide a convenient way to build this normalization into your model.\n", + "\n", + "The `tf.keras.layers.Normalization` layer precomputes the mean and variance of each column, and uses these to normalize the data.\n", + "\n", + "First, create the layer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "H2WQpDU5VRk7" + }, + "outputs": [], + "source": [ + "normalize = layers.Normalization()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hGgEZE-7Vpt6" + }, + "source": [ + "Then, use the `Normalization.adapt` method to adapt the normalization layer to your data.\n", + "\n", + "Note: Only use your training data with the `PreprocessingLayer.adapt` method. Do not use your validation or test data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2WgOPIiOVpLg" + }, + "outputs": [], + "source": [ + "normalize.adapt(abalone_features)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rE6vh0byV7cE" + }, + "source": [ + "Then, use the normalization layer in your model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "quPcZ9dTWA9A" + }, + "outputs": [], + "source": [ + "norm_abalone_model = tf.keras.Sequential([\n", + " normalize,\n", + " layers.Dense(64, activation='relu'),\n", + " layers.Dense(1)\n", + "])\n", + "\n", + "norm_abalone_model.compile(loss = tf.keras.losses.MeanSquaredError(),\n", + " optimizer = tf.keras.optimizers.Adam())\n", + "\n", + "norm_abalone_model.fit(abalone_features, abalone_labels, epochs=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wuqj601Qw0Ml" + }, + "source": [ + "## Mixed data types\n", + "\n", + "In the previous sections, you worked with a dataset where all the features were limited-range floating point values. But not all datasets are limited to a single data type.\n", + "\n", + "The \"Titanic\" dataset contains information about the passengers on the Titanic. The nominal task on this dataset is to predict who survived.\n", + "\n", + "![The Titanic](images/csv/Titanic.jpg)\n", + "\n", + "Image [from Wikimedia](https://commons.wikimedia.org/wiki/File:RMS_Titanic_3.jpg)\n", + "\n", + "The raw data can easily be loaded as a Pandas `DataFrame`, but is not immediately usable as input to a TensorFlow model.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GS-dBMpuYMnz" + }, + "outputs": [], + "source": [ + "titanic = pd.read_csv(\"https://storage.googleapis.com/tf-datasets/titanic/train.csv\")\n", + "titanic.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "D8rCGIK1ZzKx" + }, + "outputs": [], + "source": [ + "titanic_features = titanic.copy()\n", + "titanic_labels = titanic_features.pop('survived')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "urHOwpCDYtcI" + }, + "source": [ + "Because of the different data types and ranges, you can't simply stack the features into a NumPy array and pass it to a `tf.keras.Sequential` model. Each column needs to be handled individually.\n", + "\n", + "As one option, you could preprocess your data offline (using any tool you like) to convert categorical columns to numeric columns, then pass the processed output to your TensorFlow model. The disadvantage to that approach is that if you save and export your model the preprocessing is not saved with it. The Keras preprocessing layers avoid this problem because they're part of the model.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Bta4Sx0Zau5v" + }, + "source": [ + "In this example, you'll build a model that implements the preprocessing logic using [Keras functional API](https://www.tensorflow.org/guide/keras/functional). You could also do it by [subclassing](https://www.tensorflow.org/guide/keras/custom_layers_and_models).\n", + "\n", + "The functional API operates on \"symbolic\" tensors. Normal \"eager\" tensors have a value. In contrast these \"symbolic\" tensors do not. Instead they keep track of which operations are run on them, and build a representation of the calculation, that you can run later. Here's a quick example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "730F16_97D-3" + }, + "outputs": [], + "source": [ + "# Create a symbolic input\n", + "input = tf.keras.Input(shape=(), dtype=tf.float32)\n", + "\n", + "# Perform a calculation using the input\n", + "result = 2*input + 1\n", + "\n", + "# the result doesn't have a value\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RtcNXWB18kMJ" + }, + "outputs": [], + "source": [ + "calc = tf.keras.Model(inputs=input, outputs=result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fUGQOUqZ8sa-" + }, + "outputs": [], + "source": [ + "print(calc(np.array([1])).numpy())\n", + "print(calc(np.array([2])).numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rNS9lT7f6_U2" + }, + "source": [ + "To build the preprocessing model, start by building a set of symbolic `tf.keras.Input` objects, matching the names and data-types of the CSV columns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5WODe_1da3yw" + }, + "outputs": [], + "source": [ + "inputs = {}\n", + "\n", + "for name, column in titanic_features.items():\n", + " dtype = column.dtype\n", + " if dtype == object:\n", + " dtype = tf.string\n", + " else:\n", + " dtype = tf.float32\n", + "\n", + " inputs[name] = tf.keras.Input(shape=(1,), name=name, dtype=dtype)\n", + "\n", + "inputs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aaheJFmymq8l" + }, + "source": [ + "The first step in your preprocessing logic is to concatenate the numeric inputs together, and run them through a normalization layer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wPRC_E6rkp8D" + }, + "outputs": [], + "source": [ + "numeric_inputs = {name:input for name,input in inputs.items()\n", + " if input.dtype==tf.float32}\n", + "\n", + "x = layers.Concatenate()(list(numeric_inputs.values()))\n", + "norm = layers.Normalization()\n", + "norm.adapt(np.array(titanic[numeric_inputs.keys()]))\n", + "all_numeric_inputs = norm(x)\n", + "\n", + "all_numeric_inputs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-JoR45Uj712l" + }, + "source": [ + "Collect all the symbolic preprocessing results, to concatenate them later:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "M7jIJw5XntdN" + }, + "outputs": [], + "source": [ + "preprocessed_inputs = [all_numeric_inputs]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r0Hryylyosfm" + }, + "source": [ + "For the string inputs use the `tf.keras.layers.StringLookup` function to map from strings to integer indices in a vocabulary. Next, use `tf.keras.layers.CategoryEncoding` to convert the indexes into `float32` data appropriate for the model.\n", + "\n", + "The default settings for the `tf.keras.layers.CategoryEncoding` layer create a one-hot vector for each input. A `tf.keras.layers.Embedding` would also work. Check out the [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) guide and the [Classify structured data using Keras preprocessing layers](../structured_data/preprocessing_layers.ipynb) tutorial for more on this topic." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "79fi1Cgan2YV" + }, + "outputs": [], + "source": [ + "for name, input in inputs.items():\n", + " if input.dtype == tf.float32:\n", + " continue\n", + " \n", + " lookup = layers.StringLookup(vocabulary=np.unique(titanic_features[name]))\n", + " one_hot = layers.CategoryEncoding(num_tokens=lookup.vocabulary_size())\n", + "\n", + " x = lookup(input)\n", + " x = one_hot(x)\n", + " preprocessed_inputs.append(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wnhv0T7itnc7" + }, + "source": [ + "With the collection of `inputs` and `preprocessed_inputs`, you can concatenate all the preprocessed inputs together, and build a model that handles the preprocessing:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XJRzUTe8ukXc" + }, + "outputs": [], + "source": [ + "preprocessed_inputs_cat = layers.Concatenate()(preprocessed_inputs)\n", + "\n", + "titanic_preprocessing = tf.keras.Model(inputs, preprocessed_inputs_cat)\n", + "\n", + "tf.keras.utils.plot_model(model = titanic_preprocessing , rankdir=\"LR\", dpi=72, show_shapes=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PNHxrNW8vdda" + }, + "source": [ + "This model just contains the input preprocessing. You can run it to see what it does to your data. Keras models don't automatically convert pandas `DataFrame`s because it's not clear if it should be converted to one tensor or to a dictionary of tensors. So, convert it to a dictionary of tensors:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5YjdYyMEacwQ" + }, + "outputs": [], + "source": [ + "titanic_features_dict = {name: np.array(value) \n", + " for name, value in titanic_features.items()}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0nKJYoPByada" + }, + "source": [ + "Slice out the first training example and pass it to this preprocessing model, you see the numeric features and string one-hots all concatenated together:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SjnmU8PSv8T3" + }, + "outputs": [], + "source": [ + "features_dict = {name:values[:1] for name, values in titanic_features_dict.items()}\n", + "titanic_preprocessing(features_dict)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qkBf4LvmzMDp" + }, + "source": [ + "Now, build the model on top of this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "coIPtGaCzUV7" + }, + "outputs": [], + "source": [ + "def titanic_model(preprocessing_head, inputs):\n", + " body = tf.keras.Sequential([\n", + " layers.Dense(64, activation='relu'),\n", + " layers.Dense(1)\n", + " ])\n", + "\n", + " preprocessed_inputs = preprocessing_head(inputs)\n", + " result = body(preprocessed_inputs)\n", + " model = tf.keras.Model(inputs, result)\n", + "\n", + " model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " optimizer=tf.keras.optimizers.Adam())\n", + " return model\n", + "\n", + "titanic_model = titanic_model(titanic_preprocessing, inputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LK5uBQQF2KbZ" + }, + "source": [ + "When you train the model, pass the dictionary of features as `x`, and the label as `y`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "D1gVfwJ61ejz" + }, + "outputs": [], + "source": [ + "titanic_model.fit(x=titanic_features_dict, y=titanic_labels, epochs=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LxgJarZk3bfH" + }, + "source": [ + "Since the preprocessing is part of the model, you can save the model and reload it somewhere else and get identical results:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ay-8ymNA2ZCh" + }, + "outputs": [], + "source": [ + "titanic_model.save('test.keras')\n", + "reloaded = tf.keras.models.load_model('test.keras')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Qm6jMTpD20lK" + }, + "outputs": [], + "source": [ + "features_dict = {name:values[:1] for name, values in titanic_features_dict.items()}\n", + "\n", + "before = titanic_model(features_dict)\n", + "after = reloaded(features_dict)\n", + "assert (before-after)<1e-3\n", + "print(before)\n", + "print(after)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7VsPlxIRZpXf" + }, + "source": [ + "## Using tf.data\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" + "id": "NyVDCwGzR5HW" }, "source": [ - "## Setup" + "In the previous section you relied on the model's built-in data shuffling and batching while training the model.\n", + "\n", + "If you need more control over the input data pipeline or need to use data that doesn't easily fit into memory: use `tf.data`.\n", + "\n", + "For more examples, refer to the [`tf.data`: Build TensorFlow input pipelines](../../guide/data.ipynb) guide." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gP5Y1jM2Sor0" + }, + "source": [ + "### On in memory data\n", + "\n", + "As a first example of applying `tf.data` to CSV data, consider the following code to manually slice up the dictionary of features from the previous section. For each index, it takes that index for each feature:\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "I4dwMQVQMQWD" + "id": "i8wE-MVuVu7_" }, "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" + "import itertools\n", + "\n", + "def slices(features):\n", + " for i in itertools.count():\n", + " # For each feature take index `i`\n", + " example = {name:values[i] for name, values in features.items()}\n", + " yield example" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cQ3RTbS9YEal" + }, + "source": [ + "Run this and print the first example:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "baYFZMW_bJHh" + "id": "Wwq8XK88WwFk" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "import functools\n", - "\n", - "import numpy as np\n", - "import tensorflow as tf" + "for example in slices(titanic_features_dict):\n", + " for name, value in example.items():\n", + " print(f\"{name:19s}: {value}\")\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vvp8Dct6YOIE" + }, + "source": [ + "The most basic `tf.data.Dataset` in memory data loader is the `Dataset.from_tensor_slices` constructor. This returns a `tf.data.Dataset` that implements a generalized version of the above `slices` function, in TensorFlow." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Ncf5t6tgL5ZI" + "id": "2gEJthslYxeV" }, "outputs": [], "source": [ - "TRAIN_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/train.csv\"\n", - "TEST_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/eval.csv\"\n", - "\n", - "train_file_path = tf.keras.utils.get_file(\"train.csv\", TRAIN_DATA_URL)\n", - "test_file_path = tf.keras.utils.get_file(\"eval.csv\", TEST_DATA_URL)" + "features_ds = tf.data.Dataset.from_tensor_slices(titanic_features_dict)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-ZC0rTpMZMZK" + }, + "source": [ + "You can iterate over a `tf.data.Dataset` like any other python iterable:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4ONE94qulk6S" + "id": "gOHbiefaY4ag" }, "outputs": [], "source": [ - "# Make numpy values easier to read.\n", - "np.set_printoptions(precision=3, suppress=True)" + "for example in features_ds:\n", + " for name, value in example.items():\n", + " print(f\"{name:19s}: {value}\")\n", + " break" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Wuqj601Qw0Ml" + "id": "uwcFoVJWZY5F" }, "source": [ - "## Load data\n", - "\n", - "To start, let's look at the top of the CSV file to see how it is formatted." + "The `from_tensor_slices` function can handle any structure of nested dictionaries or tuples. The following code makes a dataset of `(features_dict, labels)` pairs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xIHGBy76Zcrx" + }, + "outputs": [], + "source": [ + "titanic_ds = tf.data.Dataset.from_tensor_slices((titanic_features_dict, titanic_labels))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gQwxitt8c2GK" + }, + "source": [ + "To train a model using this `Dataset`, you'll need to at least `shuffle` and `batch` the data." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "54Dv7mCrf9Yw" + "id": "SbJcbldhddeC" }, "outputs": [], "source": [ - "!head {train_file_path}" + "titanic_batches = titanic_ds.shuffle(len(titanic_labels)).batch(32)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "jC9lRhV-q_R3" + "id": "-4FRqhRFuoJx" + }, + "source": [ + "Instead of passing `features` and `labels` to `Model.fit`, you pass the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8yXkNPumdBtB" }, + "outputs": [], "source": [ - "You can [load this using pandas](pandas.ipynb), and pass the NumPy arrays to TensorFlow. If you need to scale up to a large set of files, or need a loader that integrates with [TensorFlow and tf.data](../../guide/data.ipynb) then use the `tf.data.experimental.make_csv_dataset` function:" + "titanic_model.fit(titanic_batches, epochs=5)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "67mfwr4v-mN_" + "id": "qXuibiv9exT7" }, "source": [ - "The only column you need to identify explicitly is the one with the value that the model is intended to predict. " + "### From a single file\n", + "\n", + "So far this tutorial has worked with in-memory data. `tf.data` is a highly scalable toolkit for building data pipelines, and provides a few functions for loading CSV files. " ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iXROZm5f3V4E" + "id": "Ncf5t6tgL5ZI" }, "outputs": [], "source": [ - "LABEL_COLUMN = 'survived'\n", - "LABELS = [0, 1]" + "titanic_file_path = tf.keras.utils.get_file(\"train.csv\", \"https://storage.googleapis.com/tf-datasets/titanic/train.csv\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "t4N-plO4tDXd" }, "source": [ - "Now read the CSV data from the file and create a dataset. \n", + "Now read the CSV data from the file and create a `tf.data.Dataset`.\n", "\n", "(For the full documentation, see `tf.data.experimental.make_csv_dataset`)\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "yIbUscB9sqha" }, "outputs": [], "source": [ - "def get_dataset(file_path, **kwargs):\n", - " dataset = tf.data.experimental.make_csv_dataset(\n", - " file_path,\n", - " batch_size=5, # Artificially small to make examples easier to show.\n", - " label_name=LABEL_COLUMN,\n", - " na_value=\"?\",\n", - " num_epochs=1,\n", - " ignore_errors=True, \n", - " **kwargs)\n", - " return dataset\n", + "titanic_csv_ds = tf.data.experimental.make_csv_dataset(\n", + " titanic_file_path,\n", + " batch_size=5, # Artificially small to make examples easier to show.\n", + " label_name='survived',\n", + " num_epochs=1,\n", + " ignore_errors=True,)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Sf3v3BKgy4AG" + }, + "source": [ + "This function includes many convenient features, so the data is easy to work with. This includes:\n", "\n", - "raw_train_data = get_dataset(train_file_path)\n", - "raw_test_data = get_dataset(test_file_path)" + "* Using the column headers as dictionary keys.\n", + "* Automatically determining the type of each column.\n", + "\n", + "Caution: Make sure to set the `num_epochs` argument in `tf.data.experimental.make_csv_dataset`, otherwise the default behavior for `tf.data.Dataset` is to loop endlessly." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "v4oMO9MIxgTG" }, "outputs": [], "source": [ - "def show_batch(dataset):\n", - " for batch, label in dataset.take(1):\n", - " for key, value in batch.items():\n", - " print(\"{:20s}: {}\".format(key,value.numpy()))" + "for batch, label in titanic_csv_ds.take(1):\n", + " for key, value in batch.items():\n", + " print(f\"{key:20s}: {value}\")\n", + " print()\n", + " print(f\"{'label':20s}: {label}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "k-TgA6o2Ja6U" + }, + "source": [ + "Note: If you run the above cell twice it will produce different results. The default settings for `tf.data.experimental.make_csv_dataset` include `shuffle_buffer_size=1000`, which is more than sufficient for this small dataset, but may not be for a real-world dataset." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "vHUQFKoQI6G7" + "id": "d6uviU_KCCWD" }, "source": [ - "Each item in the dataset is a batch, represented as a tuple of (*many examples*, *many labels*). The data from the examples is organized in column-based tensors (rather than row-based tensors), each with as many elements as the batch size (5 in this case).\n", + "It can also decompress the data on the fly. Here's a gzipped CSV file containing the [metro interstate traffic dataset](https://archive.ics.uci.edu/ml/datasets/Metro+Interstate+Traffic+Volume).\n", + "\n", + "![A traffic jam.](images/csv/traffic.jpg)\n", "\n", - "It might help to see this yourself." + "Image [from Wikimedia](https://commons.wikimedia.org/wiki/File:Trafficjam.jpg)\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HjrkJROoxoll" + "id": "kT7oZI2E46Q8" }, "outputs": [], "source": [ - "show_batch(raw_train_data)" + "traffic_volume_csv_gz = tf.keras.utils.get_file(\n", + " 'Metro_Interstate_Traffic_Volume.csv.gz', \n", + " \"https://archive.ics.uci.edu/ml/machine-learning-databases/00492/Metro_Interstate_Traffic_Volume.csv.gz\",\n", + " cache_dir='.', cache_subdir='traffic')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "YOYKQKmMj3D6" + "id": "F-IOsFHbCw0i" }, "source": [ - "As you can see, the columns in the CSV are named. The dataset constructor will pick these names up automatically. If the file you are working with does not contain the column names in the first line, pass them in a list of strings to the `column_names` argument in the `make_csv_dataset` function." + "Set the `compression_type` argument to read directly from the compressed file:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2Av8_9L3tUg1" + "id": "ar0MPEVJ5NeA" }, "outputs": [], "source": [ - "CSV_COLUMNS = ['survived', 'sex', 'age', 'n_siblings_spouses', 'parch', 'fare', 'class', 'deck', 'embark_town', 'alone']\n", + "traffic_volume_csv_gz_ds = tf.data.experimental.make_csv_dataset(\n", + " traffic_volume_csv_gz,\n", + " batch_size=256,\n", + " label_name='traffic_volume',\n", + " num_epochs=1,\n", + " compression_type=\"GZIP\")\n", + "\n", + "for batch, label in traffic_volume_csv_gz_ds.take(1):\n", + " for key, value in batch.items():\n", + " print(f\"{key:20s}: {value[:5]}\")\n", + " print()\n", + " print(f\"{'label':20s}: {label[:5]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p12Y6tGq8D6M" + }, + "source": [ + "Note: If you need to parse those date-time strings in the `tf.data` pipeline, you can use `tfa.text.parse_time`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EtrAXzYGP3l0" + }, + "source": [ + "### Caching" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fN2dL_LRP83r" + }, + "source": [ + "There is some overhead to parsing the CSV data. For small models this can be the bottleneck in training.\n", + "\n", + "Depending on your use case, it may be a good idea to use `Dataset.cache` or `tf.data.Dataset.snapshot`, so that the CSV data is only parsed on the first epoch.\n", "\n", - "temp_dataset = get_dataset(train_file_path, column_names=CSV_COLUMNS)\n", + "The main difference between the `cache` and `snapshot` methods is that `cache` files can only be used by the TensorFlow process that created them, but `snapshot` files can be read by other processes.\n", "\n", - "show_batch(temp_dataset)" + "For example, iterating over the `traffic_volume_csv_gz_ds` 20 times may take around 15 seconds without caching, or about two seconds with caching." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Qk38Sw4MO4eh" + }, + "outputs": [], + "source": [ + "%%time\n", + "for i, (batch, label) in enumerate(traffic_volume_csv_gz_ds.repeat(20)):\n", + " if i % 40 == 0:\n", + " print('.', end='')\n", + "print()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "gZfhoX7bR9u4" + "id": "pN3HtDONh5TX" }, "source": [ - "This example is going to use all the available columns. If you need to omit some columns from the dataset, create a list of just the columns you plan to use, and pass it into the (optional) `select_columns` argument of the constructor.\n" + "Note: `Dataset.cache` stores the data from the first epoch and replays it in order. So, using the `cache` method disables any shuffles earlier in the pipeline. Below, `Dataset.shuffle` is added back in after `Dataset.cache`." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "S1TzSkUKwsNP" + "id": "r5Jj72MrPbnh" }, "outputs": [], "source": [ - "SELECT_COLUMNS = ['survived', 'age', 'n_siblings_spouses', 'class', 'deck', 'alone']\n", + "%%time\n", + "caching = traffic_volume_csv_gz_ds.cache().shuffle(1000)\n", "\n", - "temp_dataset = get_dataset(train_file_path, select_columns=SELECT_COLUMNS)\n", - "\n", - "show_batch(temp_dataset)" + "for i, (batch, label) in enumerate(caching.shuffle(1000).repeat(20)):\n", + " if i % 40 == 0:\n", + " print('.', end='')\n", + "print()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "9cryz31lxs3e" + "id": "wN7uUBjmgNZ9" }, "source": [ - "## Data preprocessing\n", - "\n", - "A CSV file can contain a variety of data types. Typically you want to convert from those mixed types to a fixed length vector before feeding the data into your model.\n", + "Note: The `tf.data.Dataset.snapshot` files are meant for *temporary* storage of a dataset while in use. This is *not* a format for long term storage. The file format is considered an internal detail, and not guaranteed between TensorFlow versions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PHGD1E8ktUvW" + }, + "outputs": [], + "source": [ + "%%time\n", + "snapshotting = traffic_volume_csv_gz_ds.snapshot('titanic.tfsnap').shuffle(1000)\n", "\n", - "TensorFlow has a built-in system for describing common input conversions: `tf.feature_column`, see [this tutorial](../keras/feature_columns) for details.\n", + "for i, (batch, label) in enumerate(snapshotting.shuffle(1000).repeat(20)):\n", + " if i % 40 == 0:\n", + " print('.', end='')\n", + "print()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fUSSegnMCGRz" + }, + "source": [ + "If your data loading is slowed by loading CSV files, and `Dataset.cache` and `tf.data.Dataset.snapshot` are insufficient for your use case, consider re-encoding your data into a more streamlined format." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M0iGXv9pC5kr" + }, + "source": [ + "### Multiple files" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9FFzHQrCDH4w" + }, + "source": [ + "All the examples so far in this section could easily be done without `tf.data`. One place where `tf.data` can really simplify things is when dealing with collections of files.\n", "\n", + "For example, the [character font images](https://archive.ics.uci.edu/ml/datasets/Character+Font+Images) dataset is distributed as a collection of csv files, one per font.\n", "\n", - "You can preprocess your data using any tool you like (like [nltk](https://www.nltk.org/) or [sklearn](https://scikit-learn.org/stable/)), and just pass the processed output to TensorFlow. \n", + "![Fonts](images/csv/fonts.jpg)\n", "\n", + "Image by Willi Heidelbach from Pixabay\n", "\n", - "The primary advantage of doing the preprocessing inside your model is that when you export the model it includes the preprocessing. This way you can pass the raw data directly to your model." + "Download the dataset, and review the files inside:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "9AsbaFmCeJtF" + "id": "RmVknMdJh5ks" }, + "outputs": [], "source": [ - "### Continuous data" + "fonts_zip = tf.keras.utils.get_file(\n", + " 'fonts.zip', \"https://archive.ics.uci.edu/ml/machine-learning-databases/00417/fonts.zip\",\n", + " cache_dir='.', cache_subdir='fonts',\n", + " extract=True)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "Xl0Q0DcfA_rt" + "id": "xsDlMCnyi55e" }, + "outputs": [], "source": [ - "If your data is already in an appropriate numeric format, you can pack the data into a vector before passing it off to the model:" + "import pathlib\n", + "font_csvs = sorted(str(p) for p in pathlib.Path('fonts').glob(\"*.csv\"))\n", + "\n", + "font_csvs[:10]" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4Yfji3J5BMxz" + "id": "lRAEJx9ROAGl" }, "outputs": [], "source": [ - "SELECT_COLUMNS = ['survived', 'age', 'n_siblings_spouses', 'parch', 'fare']\n", - "DEFAULTS = [0, 0.0, 0.0, 0.0, 0.0]\n", - "temp_dataset = get_dataset(train_file_path, \n", - " select_columns=SELECT_COLUMNS,\n", - " column_defaults = DEFAULTS)\n", + "len(font_csvs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "19Udrw9iG-FS" + }, + "source": [ + "When dealing with a bunch of files, you can pass a glob-style `file_pattern` to the `tf.data.experimental.make_csv_dataset` function. The order of the files is shuffled each iteration.\n", "\n", - "show_batch(temp_dataset)" + "Use the `num_parallel_reads` argument to set how many files are read in parallel and interleaved together." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zEUhI8kZCfq8" + "id": "6TSUNdT6iG58" }, "outputs": [], "source": [ - "example_batch, labels_batch = next(iter(temp_dataset)) " + "fonts_ds = tf.data.experimental.make_csv_dataset(\n", + " file_pattern = \"fonts/*.csv\",\n", + " batch_size=10, num_epochs=1,\n", + " num_parallel_reads=20,\n", + " shuffle_buffer_size=10000)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "IP45_2FbEKzn" + "id": "XMoexinLHYFa" }, "source": [ - "Here's a simple function that will pack together all the columns:" + "These CSV files have the images flattened out into a single row. The column names are formatted `r{row}c{column}`. Here's the first batch:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JQ0hNSL8CC3a" + "id": "RmFvBWxxi3pq" }, "outputs": [], "source": [ - "def pack(features, label):\n", - " return tf.stack(list(features.values()), axis=-1), label" + "for features in fonts_ds.take(1):\n", + " for i, (name, value) in enumerate(features.items()):\n", + " if i>15:\n", + " break\n", + " print(f\"{name:20s}: {value}\")\n", + "print('...')\n", + "print(f\"[total: {len(features)} features]\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "75LA9DisEIoE" + "id": "xrC3sKdeOhb5" }, "source": [ - "Apply this to each element of the dataset:" + "#### Optional: Packing fields\n", + "\n", + "You probably don't want to work with each pixel in separate columns like this. Before trying to use this dataset be sure to pack the pixels into an image-tensor.\n", + "\n", + "Here is code that parses the column names to build images for each example:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VnP2Z2lwCTRl" + "id": "hct5EMEWNyfH" }, "outputs": [], "source": [ - "packed_dataset = temp_dataset.map(pack)\n", + "import re\n", "\n", - "for features, labels in packed_dataset.take(1):\n", - " print(features.numpy())\n", - " print()\n", - " print(labels.numpy())" + "def make_images(features):\n", + " image = [None]*400\n", + " new_feats = {}\n", + "\n", + " for name, value in features.items():\n", + " match = re.match('r(\\d+)c(\\d+)', name)\n", + " if match:\n", + " image[int(match.group(1))*20+int(match.group(2))] = value\n", + " else:\n", + " new_feats[name] = value\n", + "\n", + " image = tf.stack(image, axis=0)\n", + " image = tf.reshape(image, [20, 20, -1])\n", + " new_feats['image'] = image\n", + "\n", + " return new_feats" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "1VBvmaFrFU6J" + "id": "61qy8utAwARP" }, "source": [ - "If you have mixed datatypes you may want to separate out these simple-numeric fields. The `tf.feature_column` api can handle them, but this incurs some overhead and should be avoided unless really necessary. Switch back to the mixed dataset:" + "Apply that function to each batch in the dataset:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ad-IQ_JPFQge" + "id": "DJnnfIW9baE4" }, "outputs": [], "source": [ - "show_batch(raw_train_data)" + "fonts_image_ds = fonts_ds.map(make_images)\n", + "\n", + "for features in fonts_image_ds.take(1):\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_ThqrthGwHSm" + }, + "source": [ + "Plot the resulting images:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HSrYNKKcIdav" + "id": "I5dcey31T_tk" }, "outputs": [], "source": [ - "example_batch, labels_batch = next(iter(temp_dataset)) " + "from matplotlib import pyplot as plt\n", + "\n", + "plt.figure(figsize=(6,6), dpi=120)\n", + "\n", + "for n in range(9):\n", + " plt.subplot(3,3,n+1)\n", + " plt.imshow(features['image'][..., n])\n", + " plt.title(chr(features['m_label'][n]))\n", + " plt.axis('off')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "p5VtThKfGPaQ" + "id": "7-nNR0Nncdd1" }, "source": [ - "So define a more general preprocessor that selects a list of numeric features and packs them into a single column:" + "## Lower level functions" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5DRishYYGS-m" + "id": "3jiGZeUijJNd" }, - "outputs": [], "source": [ - "class PackNumericFeatures(object):\n", - " def __init__(self, names):\n", - " self.names = names\n", + "So far this tutorial has focused on the highest-level utilities for reading csv data. There are two other APIs that may be helpful for advanced users if your use-case doesn't fit the basic patterns.\n", "\n", - " def __call__(self, features, labels):\n", - " numeric_features = [features.pop(name) for name in self.names]\n", - " numeric_features = [tf.cast(feat, tf.float32) for feat in numeric_features]\n", - " numeric_features = tf.stack(numeric_features, axis=-1)\n", - " features['numeric'] = numeric_features\n", + "* `tf.io.decode_csv`: a function for parsing lines of text into a list of CSV column tensors.\n", + "* `tf.data.experimental.CsvDataset`: a lower-level CSV dataset constructor.\n", "\n", - " return features, labels" + "This section recreates functionality provided by `tf.data.experimental.make_csv_dataset`, to demonstrate how this lower-level functionality can be used.\n" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1SeZka9AHfqD" + "id": "LL_ixywomOHW" }, - "outputs": [], "source": [ - "NUMERIC_FEATURES = ['age','n_siblings_spouses','parch', 'fare']\n", + "### `tf.io.decode_csv`\n", + "\n", + "This function decodes a string, or list of strings into a list of columns.\n", "\n", - "packed_train_data = raw_train_data.map(\n", - " PackNumericFeatures(NUMERIC_FEATURES))\n", + "Unlike `tf.data.experimental.make_csv_dataset` this function does not try to guess column data-types. You specify the column types by providing a list of `record_defaults` containing a value of the correct type, for each column.\n", "\n", - "packed_test_data = raw_test_data.map(\n", - " PackNumericFeatures(NUMERIC_FEATURES))" + "To read the Titanic data **as strings** using `tf.io.decode_csv` you would say:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wFrw0YobIbUB" + "id": "m1D2C-qdlqeW" }, "outputs": [], "source": [ - "show_batch(packed_train_data)" + "text = pathlib.Path(titanic_file_path).read_text()\n", + "lines = text.split('\\n')[1:-1]\n", + "\n", + "all_strings = [str()]*10\n", + "all_strings" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_EPUS8fPLUb1" + "id": "9W4UeJYyHPx5" }, "outputs": [], "source": [ - "example_batch, labels_batch = next(iter(packed_train_data)) " + "features = tf.io.decode_csv(lines, record_defaults=all_strings) \n", + "\n", + "for f in features:\n", + " print(f\"type: {f.dtype.name}, shape: {f.shape}\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "o2maE8d2ijsq" + "id": "j8TaHSQFoQL4" }, "source": [ - "#### Data Normalization\n", - "\n", - "Continuous data should always be normalized." + "To parse them with their actual types, create a list of `record_defaults` of the corresponding types: " ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WKT1ASWpwH46" + "id": "rzUjR59yoUe1" }, "outputs": [], "source": [ - "import pandas as pd\n", - "desc = pd.read_csv(train_file_path)[NUMERIC_FEATURES].describe()\n", - "desc" + "print(lines[0])" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cHHstcKPsMXM" + "id": "7sPTunxwoeWU" }, "outputs": [], "source": [ - "MEAN = np.array(desc.T['mean'])\n", - "STD = np.array(desc.T['std'])" + "titanic_types = [int(), str(), float(), int(), int(), float(), str(), str(), str(), str()]\n", + "titanic_types" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "REKqO_xHPNx0" + "id": "n3NlViCzoB7F" }, "outputs": [], "source": [ - "def normalize_numeric_data(data, mean, std):\n", - " # Center the data\n", - " return (data-mean)/std\n" + "features = tf.io.decode_csv(lines, record_defaults=titanic_types) \n", + "\n", + "for f in features:\n", + " print(f\"type: {f.dtype.name}, shape: {f.shape}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "m-LkTUTnpn2P" + }, + "source": [ + "Note: It is more efficient to call `tf.io.decode_csv` on large batches of lines than on individual lines of CSV text." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "VPsoMUgRCpUM" + "id": "Yp1UItJmqGqw" }, "source": [ - "Now create a numeric column. The `tf.feature_columns.numeric_column` API accepts a `normalizer_fn` argument, which will be run on each batch.\n", + "### `tf.data.experimental.CsvDataset`\n", "\n", - "Bind the `MEAN` and `STD` to the normalizer fn using [`functools.partial`](https://docs.python.org/3/library/functools.html#functools.partial)." + "The `tf.data.experimental.CsvDataset` class provides a minimal CSV `Dataset` interface without the convenience features of the `tf.data.experimental.make_csv_dataset` function: column header parsing, column type-inference, automatic shuffling, file interleaving.\n", + "\n", + "This constructor uses `record_defaults` the same way as `tf.io.decode_csv`:\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Bw0I35xRS57V" + "id": "9OzZLp3krP-t" }, "outputs": [], "source": [ - "# See what you just created.\n", - "normalizer = functools.partial(normalize_numeric_data, mean=MEAN, std=STD)\n", + "simple_titanic = tf.data.experimental.CsvDataset(titanic_file_path, record_defaults=titanic_types, header=True)\n", "\n", - "numeric_column = tf.feature_column.numeric_column('numeric', normalizer_fn=normalizer, shape=[len(NUMERIC_FEATURES)])\n", - "numeric_columns = [numeric_column]\n", - "numeric_column" + "for example in simple_titanic.take(1):\n", + " print([e.numpy() for e in example])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "HZxcHXc6LCa7" + "id": "_HBmfI-Ks7dw" }, "source": [ - "When you train the model, include this feature column to select and center this block of numeric data:" + "The above code is basically equivalent to:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "b61NM76Ot_kb" + "id": "E5O5d69Yq7gG" }, "outputs": [], "source": [ - "example_batch['numeric']" + "def decode_titanic_line(line):\n", + " return tf.io.decode_csv(line, titanic_types)\n", + "\n", + "manual_titanic = (\n", + " # Load the lines of text\n", + " tf.data.TextLineDataset(titanic_file_path)\n", + " # Skip the header row.\n", + " .skip(1)\n", + " # Decode the line.\n", + " .map(decode_titanic_line)\n", + ")\n", + "\n", + "for example in manual_titanic.take(1):\n", + " print([e.numpy() for e in example])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5R3ralsnt2AC" + }, + "source": [ + "#### Multiple files\n", + "\n", + "To parse the fonts dataset using `tf.data.experimental.CsvDataset`, you first need to determine the column types for the `record_defaults`. Start by inspecting the first row of one file:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "j-r_4EAJAZoI" + "id": "3tlFOTjCvAI5" }, "outputs": [], "source": [ - "numeric_layer = tf.keras.layers.DenseFeatures(numeric_columns)\n", - "numeric_layer(example_batch).numpy()" + "font_line = pathlib.Path(font_csvs[0]).read_text().splitlines()[1]\n", + "print(font_line)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "M37oD2VcCO4R" + "id": "etyGu8K_ySRz" + }, + "source": [ + "Only the first two fields are strings, the rest are integers or floats, and you can get the total number of features by counting the commas:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "crgZZn0BzkSB" }, + "outputs": [], "source": [ - "The mean based normalization used here requires knowing the means of each column ahead of time." + "num_font_features = font_line.count(',')+1\n", + "font_column_types = [str(), str()] + [float()]*(num_font_features-2)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "tSyrkSQwYHKi" + "id": "YeK2Pw540RNj" }, "source": [ - "### Categorical data\n", - "\n", - "Some of the columns in the CSV data are categorical columns. That is, the content should be one of a limited set of options.\n", - "\n", - "Use the `tf.feature_column` API to create a collection with a `tf.feature_column.indicator_column` for each categorical column.\n", - "\n" + "The `tf.data.experimental.CsvDataset` constructor can take a list of input files, but reads them sequentially. The first file in the list of CSVs is `AGENCY.csv`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mWDniduKMw-C" + "id": "_SvL5Uvl0r0N" }, "outputs": [], "source": [ - "CATEGORIES = {\n", - " 'sex': ['male', 'female'],\n", - " 'class' : ['First', 'Second', 'Third'],\n", - " 'deck' : ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'],\n", - " 'embark_town' : ['Cherbourg', 'Southhampton', 'Queenstown'],\n", - " 'alone' : ['y', 'n']\n", - "}\n" + "font_csvs[0]" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kkxLdrsLwHPT" + "id": "EfAX3G8Xywy6" }, - "outputs": [], "source": [ - "categorical_columns = []\n", - "for feature, vocab in CATEGORIES.items():\n", - " cat_col = tf.feature_column.categorical_column_with_vocabulary_list(\n", - " key=feature, vocabulary_list=vocab)\n", - " categorical_columns.append(tf.feature_column.indicator_column(cat_col))" + "So, when you pass the list of files to `CsvDataset`, the records from `AGENCY.csv` are read first:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "H18CxpHY_Nma" + "id": "Gtr1E66VmBqj" }, "outputs": [], "source": [ - "# See what you just created.\n", - "categorical_columns" + "simple_font_ds = tf.data.experimental.CsvDataset(\n", + " font_csvs, \n", + " record_defaults=font_column_types, \n", + " header=True)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "p7mACuOsArUH" + "id": "k750Mgq4yt_o" }, "outputs": [], "source": [ - "categorical_layer = tf.keras.layers.DenseFeatures(categorical_columns)\n", - "print(categorical_layer(example_batch).numpy()[0])" + "for row in simple_font_ds.take(10):\n", + " print(row[0].numpy())" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "R7-1QG99_1sN" + "id": "NiqWKQV21FrE" }, "source": [ - "This will be become part of a data processing input later when you build the model." + "To interleave multiple files, use `Dataset.interleave`.\n", + "\n", + "Here's an initial dataset that contains the CSV file names: " ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "kPWkC4_1l3IG" + "id": "t9dS3SNb23W8" }, + "outputs": [], "source": [ - "### Combined preprocessing layer" + "font_files = tf.data.Dataset.list_files(\"fonts/*.csv\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "R3QAjo1qD4p9" + "id": "TNiLHMXpzHy5" }, "source": [ - "Add the two feature column collections and pass them to a `tf.keras.layers.DenseFeatures` to create an input layer that will extract and preprocess both input types:" + "This shuffles the file names each epoch:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3-OYK7GnaH0r" + "id": "zNd-TYyNzIgg" }, "outputs": [], "source": [ - "preprocessing_layer = tf.keras.layers.DenseFeatures(categorical_columns+numeric_columns)" + "print('Epoch 1:')\n", + "for f in list(font_files)[:5]:\n", + " print(\" \", f.numpy())\n", + "print(' ...')\n", + "print()\n", + "\n", + "print('Epoch 2:')\n", + "for f in list(font_files)[:5]:\n", + " print(\" \", f.numpy())\n", + "print(' ...')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "B0QB1PtU3WAN" + }, + "source": [ + "The `interleave` method takes a `map_func` that creates a child-`Dataset` for each element of the parent-`Dataset`.\n", + "\n", + "Here, you want to create a `tf.data.experimental.CsvDataset` from each element of the dataset of files:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "m7_U_K0UMSVS" + "id": "QWp4rH0Q4uPh" }, "outputs": [], "source": [ - "print(preprocessing_layer(example_batch).numpy()[0])" + "def make_font_csv_ds(path):\n", + " return tf.data.experimental.CsvDataset(\n", + " path, \n", + " record_defaults=font_column_types, \n", + " header=True)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "DlF_omQqtnOP" + "id": "VxRGdLMB5nRF" }, "source": [ - "## Build the model" + "The `Dataset` returned by interleave returns elements by cycling over a number of the child-`Dataset`s. Note, below, how the dataset cycles over `cycle_length=3` three font files:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "lQoFh16LxtT_" + "id": "OePMNF_x1_Cc" }, + "outputs": [], "source": [ - "Build a `tf.keras.Sequential`, starting with the `preprocessing_layer`." + "font_rows = font_files.interleave(make_font_csv_ds,\n", + " cycle_length=3)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3mSGsHTFPvFo" + "id": "UORIGWLy54-E" }, "outputs": [], "source": [ - "model = tf.keras.Sequential([\n", - " preprocessing_layer,\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid'),\n", - "])\n", + "fonts_dict = {'font_name':[], 'character':[]}\n", + "\n", + "for row in font_rows.take(10):\n", + " fonts_dict['font_name'].append(row[0].numpy().decode())\n", + " fonts_dict['character'].append(chr(int(row[2].numpy())))\n", "\n", - "model.compile(\n", - " loss='binary_crossentropy',\n", - " optimizer='adam',\n", - " metrics=['accuracy'])" + "pd.DataFrame(fonts_dict)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "hPdtI2ie0lEZ" + "id": "mkKZa_HX8zAm" }, "source": [ - "## Train, evaluate, and predict" + "#### Performance\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "8gvw1RE9zXkD" + "id": "8BtGHraUApdJ" }, "source": [ - "Now the model can be instantiated and trained." + "Earlier, it was noted that `tf.io.decode_csv` is more efficient when run on a batch of strings.\n", + "\n", + "It is possible to take advantage of this fact, when using large batch sizes, to improve CSV loading performance (but try [caching](#caching) first)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d35zWMH7MDL1" + }, + "source": [ + "With the built-in loader 20, 2048-example batches take about 17s. " ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sW-4XlLeEQ2B" + "id": "ieUVAPryjpJS" }, "outputs": [], "source": [ - "train_data = packed_train_data.shuffle(500)\n", - "test_data = packed_test_data" + "BATCH_SIZE=2048\n", + "fonts_ds = tf.data.experimental.make_csv_dataset(\n", + " file_pattern = \"fonts/*.csv\",\n", + " batch_size=BATCH_SIZE, num_epochs=1,\n", + " num_parallel_reads=100)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Q_nm28IzNDTO" + "id": "MUC2KW4LkQIz" }, "outputs": [], "source": [ - "model.fit(train_data, epochs=20)" + "%%time\n", + "for i,batch in enumerate(fonts_ds.take(20)):\n", + " print('.',end='')\n", + "\n", + "print()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "QyDMgBurzqQo" + "id": "5lhnh6rZEDS2" }, "source": [ - "Once the model is trained, you can check its accuracy on the `test_data` set." + "Passing **batches of text lines** to`decode_csv` runs faster, in about 5s:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eB3R3ViVONOp" + "id": "4XbPZV1okVF9" }, "outputs": [], "source": [ - "test_loss, test_accuracy = model.evaluate(test_data)\n", + "fonts_files = tf.data.Dataset.list_files(\"fonts/*.csv\")\n", + "fonts_lines = fonts_files.interleave(\n", + " lambda fname:tf.data.TextLineDataset(fname).skip(1), \n", + " cycle_length=100).batch(BATCH_SIZE)\n", "\n", - "print('\\n\\nTest Loss {}, Test Accuracy {}'.format(test_loss, test_accuracy))" + "fonts_fast = fonts_lines.map(lambda x: tf.io.decode_csv(x, record_defaults=font_column_types))" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "sTrn_pD90gdJ" + "id": "te9C2km-qO8W" }, + "outputs": [], "source": [ - "Use `tf.keras.Model.predict` to infer labels on a batch or a dataset of batches." + "%%time\n", + "for i,batch in enumerate(fonts_fast.take(20)):\n", + " print('.',end='')\n", + "\n", + "print()" ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Qwcx74F3ojqe" + "id": "aebC1plsMeOi" }, - "outputs": [], "source": [ - "predictions = model.predict(test_data)\n", + "For another example of increasing CSV performance by using large batches, refer to the [Overfit and underfit tutorial](../keras/overfit_and_underfit.ipynb).\n", "\n", - "# Show some results\n", - "for prediction, survived in zip(predictions[:10], list(test_data)[0][1][:10]):\n", - " print(\"Predicted survival: {:.2%}\".format(prediction[0]),\n", - " \" | Actual outcome: \",\n", - " (\"SURVIVED\" if bool(survived) else \"DIED\"))\n", - "\n" + "This sort of approach may work, but consider other options like `Dataset.cache` and `tf.data.Dataset.snapshot`, or re-encoding your data into a more streamlined format." ] } ], "metadata": { "colab": { - "collapsed_sections": [], "name": "csv.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/load_data/images.ipynb b/site/en/tutorials/load_data/images.ipynb index 5ba7b4d89c9..557666b92f9 100644 --- a/site/en/tutorials/load_data/images.ipynb +++ b/site/en/tutorials/load_data/images.ipynb @@ -1,51 +1,22 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "images.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mt9dL5dIir8X" }, "source": [ - "##### Copyright 2019 The TensorFlow Authors." + "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "ufPx7EiCiqgR", - "colab": {} + "id": "ufPx7EiCiqgR" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -58,24 +29,20 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License.\n" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ucMoYase6URl" }, "source": [ - "# Load images" + "# Load and preprocess images" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_Wwu5SXZmEkB" }, "source": [ @@ -98,19 +65,19 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Oxw4WahM7DU9" }, "source": [ - "This tutorial provides a simple example of how to load an image dataset using `tf.data`.\n", + "This tutorial shows how to load and preprocess an image dataset in three ways:\n", "\n", - "The dataset used in this example is distributed as directories of images, with one class of image per directory." + "- First, you will use high-level Keras preprocessing utilities (such as `tf.keras.utils.image_dataset_from_directory`) and layers (such as `tf.keras.layers.Rescaling`) to read a directory of images on disk.\n", + "- Next, you will write your own input pipeline from scratch [using tf.data](../../guide/data.ipynb).\n", + "- Finally, you will download a dataset from the large [catalog](https://www.tensorflow.org/datasets/catalog/overview) available in [TensorFlow Datasets](https://www.tensorflow.org/datasets)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hoQQiZDB6URn" }, "source": [ @@ -119,423 +86,680 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3vhAMaIOBIee" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import os\n", + "import PIL\n", + "import PIL.Image\n", + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Qnp9Z2sT5dWj" + }, + "outputs": [], + "source": [ + "print(tf.__version__)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wO0InzL66URu" + }, + "source": [ + "### Download the flowers dataset\n", + "\n", + "This tutorial uses a dataset of several thousand photos of flowers. The flowers dataset contains five sub-directories, one per class:\n", + "\n", + "```\n", + "flowers_photos/\n", + " daisy/\n", + " dandelion/\n", + " roses/\n", + " sunflowers/\n", + " tulips/\n", + "```" + ] + }, + { + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "3vhAMaIOBIee", - "colab": {} + "id": "Ju2yXtdV5YaT" }, "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ], - "execution_count": 0, - "outputs": [] + "Note: all images are licensed CC-BY, creators are listed in the LICENSE.txt file." + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "QGXxBuPyKJw1", - "colab": {} + "id": "rN-Pc6Zd6awg" }, + "outputs": [], "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " !pip install tf-nightly\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] + "import pathlib\n", + "dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz\"\n", + "archive = tf.keras.utils.get_file(origin=dataset_url, extract=True)\n", + "data_dir = pathlib.Path(archive).with_suffix('')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rFkFK74oO--g" + }, + "source": [ + "After downloading (218MB), you should now have a copy of the flower photos available. There are 3,670 total images:" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "KT6CcaqgQewg", - "colab": {} + "id": "QhewYCxhXQBX" }, + "outputs": [], "source": [ - "AUTOTUNE = tf.data.experimental.AUTOTUNE" - ], - "execution_count": 0, - "outputs": [] + "image_count = len(list(data_dir.glob('*/*.jpg')))\n", + "print(image_count)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZUFusk44d9GW" + }, + "source": [ + "Each directory contains images of that type of flower. Here are some roses:" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "gIksPgtT8B6B", - "colab": {} + "id": "crs7ZjEp60Ot" }, + "outputs": [], "source": [ - "import IPython.display as display\n", - "from PIL import Image\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import os" - ], - "execution_count": 0, - "outputs": [] + "roses = list(data_dir.glob('roses/*'))\n", + "PIL.Image.open(str(roses[0]))" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "ZJ20R66fzktl", - "colab": {} + "id": "oV9PtjdKKWyI" }, + "outputs": [], "source": [ - "tf.__version__" - ], - "execution_count": 0, - "outputs": [] + "roses = list(data_dir.glob('roses/*'))\n", + "PIL.Image.open(str(roses[1]))" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "wO0InzL66URu" + "id": "9_kge08gSCan" }, "source": [ - "### Retrieve the images\n", - "\n", - "Before you start any training, you will need a set of images to teach the network about the new classes you want to recognize. You can use an archive of creative-commons licensed flower photos from Google.\n", + "## Load data using a Keras utility\n", "\n", - "Note: all images are licensed CC-BY, creators are listed in the `LICENSE.txt` file." + "Let's load these images off disk using the helpful `tf.keras.utils.image_dataset_from_directory` utility." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6jobDTUs8Wxu" + }, + "source": [ + "### Create a dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lAmtzsnjDNhB" + }, + "source": [ + "Define some parameters for the loader:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "rN-Pc6Zd6awg", - "colab": {} + "id": "qJdpyqK541ty" }, + "outputs": [], "source": [ - "import pathlib\n", - "data_dir = tf.keras.utils.get_file(origin='https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", - " fname='flower_photos', untar=True)\n", - "data_dir = pathlib.Path(data_dir)" - ], - "execution_count": 0, - "outputs": [] + "batch_size = 32\n", + "img_height = 180\n", + "img_width = 180" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "rFkFK74oO--g" + "id": "ehhW308g8soJ" }, "source": [ - "After downloading (218MB), you should now have a copy of the flower photos available.\n", - "\n", - "The directory contains 5 sub-directories, one per class:" + "It's good practice to use a validation split when developing your model. You will use 80% of the images for training and 20% for validation." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "QhewYCxhXQBX", - "colab": {} + "id": "chqakIP14PDm" }, + "outputs": [], "source": [ - "image_count = len(list(data_dir.glob('*/*.jpg')))\n", - "image_count" - ], - "execution_count": 0, - "outputs": [] + "train_ds = tf.keras.utils.image_dataset_from_directory(\n", + " data_dir,\n", + " validation_split=0.2,\n", + " subset=\"training\",\n", + " seed=123,\n", + " image_size=(img_height, img_width),\n", + " batch_size=batch_size)" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "sJ1HKKdR4A7c", - "colab": {} + "id": "pb2Af2lsUShk" }, + "outputs": [], "source": [ - "CLASS_NAMES = np.array([item.name for item in data_dir.glob('*') if item.name != \"LICENSE.txt\"])\n", - "CLASS_NAMES" - ], - "execution_count": 0, - "outputs": [] + "val_ds = tf.keras.utils.image_dataset_from_directory(\n", + " data_dir,\n", + " validation_split=0.2,\n", + " subset=\"validation\",\n", + " seed=123,\n", + " image_size=(img_height, img_width),\n", + " batch_size=batch_size)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "IVxsk4OW61TY" + "id": "Ug3ITsz0b_cF" }, "source": [ - "Each directory contains images of that type of flower. Here are some roses:" + "You can find the class names in the `class_names` attribute on these datasets." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "crs7ZjEp60Ot", - "colab": {} + "id": "R7z2yKt7VDPJ" }, + "outputs": [], "source": [ - "roses = list(data_dir.glob('roses/*'))\n", + "class_names = train_ds.class_names\n", + "print(class_names)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bK6CQCqIctCd" + }, + "source": [ + "### Visualize the data\n", + "\n", + "Here are the first nine images from the training dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AAY3LJN28Kuy" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", "\n", - "for image_path in roses[:3]:\n", - " display.display(Image.open(str(image_path)))" - ], - "execution_count": 0, - "outputs": [] + "plt.figure(figsize=(10, 10))\n", + "for images, labels in train_ds.take(1):\n", + " for i in range(9):\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " plt.imshow(images[i].numpy().astype(\"uint8\"))\n", + " plt.title(class_names[labels[i]])\n", + " plt.axis(\"off\")" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "6jobDTUs8Wxu" + "id": "jUI0fr7igPtA" + }, + "source": [ + "You can train a model using these datasets by passing them to `model.fit` (shown later in this tutorial). If you like, you can also manually iterate over the dataset and retrieve batches of images:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BdPHeHXt9sjA" }, + "outputs": [], "source": [ - "## Load using `keras.preprocessing`" + "for image_batch, labels_batch in train_ds:\n", + " print(image_batch.shape)\n", + " print(labels_batch.shape)\n", + " break" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "ehhW308g8soJ" + "id": "2ZgIZeXaDUsF" + }, + "source": [ + "The `image_batch` is a tensor of the shape `(32, 180, 180, 3)`. This is a batch of 32 images of shape `180x180x3` (the last dimension refers to color channels RGB). The `label_batch` is a tensor of the shape `(32,)`, these are corresponding labels to the 32 images.\n", + "\n", + "You can call `.numpy()` on either of these tensors to convert them to a `numpy.ndarray`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ybl6a2YCg1rV" }, "source": [ - "A simple way to load images is to use `tf.keras.preprocessing`." + "### Standardize the data\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IdogGjM2K6OU" + }, + "source": [ + "The RGB channel values are in the `[0, 255]` range. This is not ideal for a neural network; in general you should seek to make your input values small.\n", + "\n", + "Here, you will standardize values to be in the `[0, 1]` range by using `tf.keras.layers.Rescaling`:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "syDdF_LWVrWE", - "colab": {} + "id": "16yNdZXdExyM" }, + "outputs": [], "source": [ - "# The 1./255 is to convert from uint8 to float32 in range [0,1].\n", - "image_generator = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255)" - ], - "execution_count": 0, - "outputs": [] + "normalization_layer = tf.keras.layers.Rescaling(1./255)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "lAmtzsnjDNhB" + "id": "Nd0_enkb8uxZ" }, "source": [ - "Define some parameters for the loader:" + "There are two ways to use this layer. You can apply it to the dataset by calling `Dataset.map`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QgOnza-U_z5Y" + }, + "outputs": [], + "source": [ + "normalized_ds = train_ds.map(lambda x, y: (normalization_layer(x), y))\n", + "image_batch, labels_batch = next(iter(normalized_ds))\n", + "first_image = image_batch[0]\n", + "# Notice the pixel values are now in `[0,1]`.\n", + "print(np.min(first_image), np.max(first_image))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "z39nXayj9ioS" + }, + "source": [ + "Or, you can include the layer inside your model definition to simplify deployment. You will use the second approach here." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hXLd3wMpDIkp" + }, + "source": [ + "Note: If you would like to scale pixel values to `[-1,1]` you can instead write `tf.keras.layers.Rescaling(1./127.5, offset=-1)`" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LeNWVa8qRBGm" + }, + "source": [ + "Note: You previously resized images using the `image_size` argument of `tf.keras.utils.image_dataset_from_directory`. If you want to include the resizing logic in your model as well, you can use the `tf.keras.layers.Resizing` layer." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ti8avTlLofoJ" + }, + "source": [ + "### Configure the dataset for performance\n", + "\n", + "Let's make sure to use buffered prefetching so you can yield data from disk without having I/O become blocking. These are two important methods you should use when loading data:\n", + "\n", + "- `Dataset.cache` keeps the images in memory after they're loaded off disk during the first epoch. This will ensure the dataset does not become a bottleneck while training your model. If your dataset is too large to fit into memory, you can also use this method to create a performant on-disk cache.\n", + "- `Dataset.prefetch` overlaps data preprocessing and model execution while training.\n", + "\n", + "Interested readers can learn more about both methods, as well as how to cache data to disk in the *Prefetching* section of the [Better performance with the tf.data API](../../guide/data_performance.ipynb) guide." ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ea3kbMe-pGDw" + }, + "outputs": [], + "source": [ + "AUTOTUNE = tf.data.AUTOTUNE\n", + "\n", + "train_ds = train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n", + "val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "1zf695or-Flq", - "colab": {} + "id": "XqHjIr6cplwY" }, "source": [ - "BATCH_SIZE = 32\n", - "IMG_HEIGHT = 224\n", - "IMG_WIDTH = 224\n", - "STEPS_PER_EPOCH = np.ceil(image_count/BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] + "### Train a model\n", + "\n", + "For completeness, you will show how to train a simple model using the datasets you have just prepared.\n", + "\n", + "The [Sequential](https://www.tensorflow.org/guide/keras/sequential_model) model consists of three convolution blocks (`tf.keras.layers.Conv2D`) with a max pooling layer (`tf.keras.layers.MaxPooling2D`) in each of them. There's a fully-connected layer (`tf.keras.layers.Dense`) with 128 units on top of it that is activated by a ReLU activation function (`'relu'`). This model has not been tuned in any way—the goal is to show you the mechanics using the datasets you just created. To learn more about image classification, visit the [Image classification](../images/classification.ipynb) tutorial." + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Pw94ajOOVrWI", - "colab": {} + "id": "LdR0BzCcqxw0" }, + "outputs": [], "source": [ - "train_data_gen = image_generator.flow_from_directory(directory=str(data_dir),\n", - " batch_size=BATCH_SIZE,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " classes = list(CLASS_NAMES))" - ], - "execution_count": 0, - "outputs": [] + "num_classes = 5\n", + "\n", + "model = tf.keras.Sequential([\n", + " tf.keras.layers.Rescaling(1./255),\n", + " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", + " tf.keras.layers.MaxPooling2D(),\n", + " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", + " tf.keras.layers.MaxPooling2D(),\n", + " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", + " tf.keras.layers.MaxPooling2D(),\n", + " tf.keras.layers.Flatten(),\n", + " tf.keras.layers.Dense(128, activation='relu'),\n", + " tf.keras.layers.Dense(num_classes)\n", + "])" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "2ZgIZeXaDUsF" + "id": "d83f5aa7f3fb" }, "source": [ - "Inspect a batch:" + "Choose the `tf.keras.optimizers.Adam` optimizer and `tf.keras.losses.SparseCategoricalCrossentropy` loss function. To view training and validation accuracy for each training epoch, pass the `metrics` argument to `Model.compile`." ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t_BlmsnmsEr4" + }, + "outputs": [], + "source": [ + "model.compile(\n", + " optimizer='adam',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "nLp0XVG_Vgi2", - "colab": {} + "id": "ffwd44ldNMOE" }, "source": [ - "def show_batch(image_batch, label_batch):\n", - " plt.figure(figsize=(10,10))\n", - " for n in range(25):\n", - " ax = plt.subplot(5,5,n+1)\n", - " plt.imshow(image_batch[n])\n", - " plt.title(CLASS_NAMES[label_batch[n]==1][0].title())\n", - " plt.axis('off')" - ], - "execution_count": 0, - "outputs": [] + "Note: You will only train for a few epochs so this tutorial runs quickly. " + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "S08ZKKODsnGW" + }, + "outputs": [], + "source": [ + "model.fit(\n", + " train_ds,\n", + " validation_data=val_ds,\n", + " epochs=3\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MEtT9YGjSAOK" + }, + "source": [ + "Note: You can also write a custom training loop instead of using `Model.fit`. To learn more, visit the [Writing a training loop from scratch](https://www.tensorflow.org/guide/keras/writing_a_training_loop_from_scratch) tutorial." + ] + }, + { + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "suh6Sjv68rY3", - "colab": {} + "id": "BaW4wx5L7hrZ" }, "source": [ - "image_batch, label_batch = next(train_data_gen)\n", - "show_batch(image_batch, label_batch)" - ], - "execution_count": 0, - "outputs": [] + "You may notice the validation accuracy is low compared to the training accuracy, indicating your model is overfitting. You can learn more about overfitting and how to reduce it in this [tutorial](https://www.tensorflow.org/tutorials/keras/overfit_and_underfit)." + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "AxS1cLzM8mEp" }, "source": [ - "## Load using `tf.data`" + "## Using tf.data for finer control" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ylj9fgkamgWZ" }, "source": [ - "The above `keras.preprocessing` method is convienient, but has two downsides: \n", + "The above Keras preprocessing utility—`tf.keras.utils.image_dataset_from_directory`—is a convenient way to create a `tf.data.Dataset` from a directory of images.\n", "\n", - "1. It's slow. See the performance section below.\n", - "1. It lacks fine-grained control.\n", - "1. It is not well integrated with the rest of TensorFlow." + "For finer grain control, you can write your own input pipeline using `tf.data`. This section shows how to do just that, beginning with the file paths from the TGZ file you downloaded earlier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lAkQp5uxoINu" + }, + "outputs": [], + "source": [ + "list_ds = tf.data.Dataset.list_files(str(data_dir/'*/*'), shuffle=False)\n", + "list_ds = list_ds.shuffle(image_count, reshuffle_each_iteration=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "coORvEH-NGwc" + }, + "outputs": [], + "source": [ + "for f in list_ds.take(5):\n", + " print(f.numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6NLQ_VJhWO4z" + }, + "source": [ + "The tree structure of the files can be used to compile a `class_names` list." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uRPHzDGhKACK" + }, + "outputs": [], + "source": [ + "class_names = np.array(sorted([item.name for item in data_dir.glob('*') if item.name != \"LICENSE.txt\"]))\n", + "print(class_names)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "IIG5CPaULegg" + "id": "CiptrWmAlmAa" }, "source": [ - "To load the files as a `tf.data.Dataset` first create a dataset of the file paths:" + "Split the dataset into training and validation sets:" ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GWHNPzXclpVr" + }, + "outputs": [], + "source": [ + "val_size = int(image_count * 0.2)\n", + "train_ds = list_ds.skip(val_size)\n", + "val_ds = list_ds.take(val_size)" + ] + }, + { + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "lAkQp5uxoINu", - "colab": {} + "id": "rkB-IR4-pS3U" }, "source": [ - "list_ds = tf.data.Dataset.list_files(str(data_dir/'*/*'))" - ], - "execution_count": 0, - "outputs": [] + "You can print the length of each dataset as follows:" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "coORvEH-NGwc", - "colab": {} + "id": "SiKQrb9ppS-7" }, + "outputs": [], "source": [ - "for f in list_ds.take(5):\n", - " print(f.numpy())" - ], - "execution_count": 0, - "outputs": [] + "print(tf.data.experimental.cardinality(train_ds).numpy())\n", + "print(tf.data.experimental.cardinality(val_ds).numpy())" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "91CPfUUJ_8SZ" }, "source": [ - "Write a short pure-tensorflow function that converts a file paths to an (image_data, label) pair:" + "Write a short function that converts a file path to an `(img, label)` pair:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "arSQzIey-4D4", - "colab": {} + "id": "arSQzIey-4D4" }, + "outputs": [], "source": [ "def get_label(file_path):\n", - " # convert the path to a list of path components\n", + " # Convert the path to a list of path components\n", " parts = tf.strings.split(file_path, os.path.sep)\n", " # The second to last is the class-directory\n", - " return parts[-2] == CLASS_NAMES" - ], - "execution_count": 0, - "outputs": [] + " one_hot = parts[-2] == class_names\n", + " # Integer encode the label\n", + " return tf.argmax(one_hot)" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "MGlq4IP4Aktb", - "colab": {} + "id": "MGlq4IP4Aktb" }, + "outputs": [], "source": [ "def decode_img(img):\n", - " # convert the compressed string to a 3D uint8 tensor\n", - " img = tf.image.decode_jpeg(img, channels=3)\n", - " # Use `convert_image_dtype` to convert to floats in the [0,1] range.\n", - " img = tf.image.convert_image_dtype(img, tf.float32)\n", - " # resize the image to the desired size.\n", - " return tf.image.resize(img, [IMG_WIDTH, IMG_HEIGHT])" - ], - "execution_count": 0, - "outputs": [] + " # Convert the compressed string to a 3D uint8 tensor\n", + " img = tf.io.decode_jpeg(img, channels=3)\n", + " # Resize the image to the desired size\n", + " return tf.image.resize(img, [img_height, img_width])" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "-xhBRgvNqRRe", - "colab": {} + "id": "-xhBRgvNqRRe" }, + "outputs": [], "source": [ "def process_path(file_path):\n", " label = get_label(file_path)\n", - " # load the raw data from the file as a string\n", + " # Load the raw data from the file as a string\n", " img = tf.io.read_file(file_path)\n", " img = decode_img(img)\n", " return img, label" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "S9a5GpsUOBx8" }, "source": [ @@ -544,47 +768,42 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3SDhbo8lOBQv", - "colab": {} + "id": "3SDhbo8lOBQv" }, + "outputs": [], "source": [ "# Set `num_parallel_calls` so multiple images are loaded/processed in parallel.\n", - "labeled_ds = list_ds.map(process_path, num_parallel_calls=AUTOTUNE)" - ], - "execution_count": 0, - "outputs": [] + "train_ds = train_ds.map(process_path, num_parallel_calls=AUTOTUNE)\n", + "val_ds = val_ds.map(process_path, num_parallel_calls=AUTOTUNE)" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "kxrl0lGdnpRz", - "colab": {} + "id": "kxrl0lGdnpRz" }, + "outputs": [], "source": [ - "for image, label in labeled_ds.take(1):\n", + "for image, label in train_ds.take(1):\n", " print(\"Image shape: \", image.numpy().shape)\n", " print(\"Label: \", label.numpy())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vYGCgJuR_9Qp" }, "source": [ - "### Basic methods for training" + "### Configure dataset for performance" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "wwZavzgsIytz" }, "source": [ @@ -594,208 +813,227 @@ "* To be batched.\n", "* Batches to be available as soon as possible.\n", "\n", - "These features can be easily added using the `tf.data` api." + "These features can be added using the `tf.data` API. For more details, visit the [Input Pipeline Performance](../../guide/performance/datasets.ipynb) guide." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "uZmZJx8ePw_5", - "colab": {} + "id": "uZmZJx8ePw_5" }, + "outputs": [], "source": [ - "\n", - "\n", - "def prepare_for_training(ds, cache=True, shuffle_buffer_size=1000):\n", - " # This is a small dataset, only load it once, and keep it in memory.\n", - " # use `.cache(filename)` to cache preprocessing work for datasets that don't\n", - " # fit in memory.\n", - " if cache:\n", - " if isinstance(cache, str):\n", - " ds = ds.cache(cache)\n", - " else:\n", - " ds = ds.cache()\n", - "\n", - " ds = ds.shuffle(buffer_size=shuffle_buffer_size)\n", - "\n", - " # Repeat forever\n", - " ds = ds.repeat()\n", - "\n", - " ds = ds.batch(BATCH_SIZE)\n", - "\n", - " # `prefetch` lets the dataset fetch batches in the background while the model\n", - " # is training.\n", + "def configure_for_performance(ds):\n", + " ds = ds.cache()\n", + " ds = ds.shuffle(buffer_size=1000)\n", + " ds = ds.batch(batch_size)\n", " ds = ds.prefetch(buffer_size=AUTOTUNE)\n", + " return ds\n", "\n", - " return ds" - ], - "execution_count": 0, - "outputs": [] + "train_ds = configure_for_performance(train_ds)\n", + "val_ds = configure_for_performance(val_ds)" + ] }, { - "cell_type": "code", + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "-YKnrfAeZV10", - "colab": {} + "id": "45P7OvzRWzOB" }, "source": [ - "train_ds = prepare_for_training(labeled_ds)\n", + "### Visualize the data\n", "\n", - "image_batch, label_batch = next(iter(train_ds))" - ], - "execution_count": 0, - "outputs": [] + "You can visualize this dataset similarly to the one you created previously:" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "UN_Dnl72YNIj", - "colab": {} + "id": "UN_Dnl72YNIj" }, + "outputs": [], "source": [ - "show_batch(image_batch.numpy(), label_batch.numpy())" - ], - "execution_count": 0, - "outputs": [] + "image_batch, label_batch = next(iter(train_ds))\n", + "\n", + "plt.figure(figsize=(10, 10))\n", + "for i in range(9):\n", + " ax = plt.subplot(3, 3, i + 1)\n", + " plt.imshow(image_batch[i].numpy().astype(\"uint8\"))\n", + " label = label_batch[i]\n", + " plt.title(class_names[label])\n", + " plt.axis(\"off\")" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "UMVnoBcG_NlQ" + "id": "fMT8kh_uXPRU" }, "source": [ - "## Performance\n", + "### Continue training the model\n", "\n", - "Note: This section just shows a couple of easy tricks that may help performance. For an in depth guide see [Input Pipeline Performance](../../guide/performance/datasets)." + "You have now manually built a similar `tf.data.Dataset` to the one created by `tf.keras.utils.image_dataset_from_directory` above. You can continue training the model with it. As before, you will train for just a few epochs to keep the running time short." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "oNmQqgGhLWie" + "id": "Vm_bi7NKXOzW" }, + "outputs": [], "source": [ - "To investigate, first here's a function to check the performance of our datasets:" + "model.fit(\n", + " train_ds,\n", + " validation_data=val_ds,\n", + " epochs=3\n", + ")" ] }, { - "cell_type": "code", + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "_gFVe1rp_MYr", - "colab": {} + "id": "EDJXAexrwsx8" }, "source": [ - "import time\n", - "default_timeit_steps = 1000\n", + "## Using TensorFlow Datasets\n", "\n", - "def timeit(ds, steps=default_timeit_steps):\n", - " start = time.time()\n", - " it = iter(ds)\n", - " for i in range(steps):\n", - " batch = next(it)\n", - " if i%10 == 0:\n", - " print('.',end='')\n", - " print()\n", - " end = time.time()\n", + "So far, this tutorial has focused on loading data off disk. You can also find a dataset to use by exploring the large [catalog](https://www.tensorflow.org/datasets/catalog/overview) of easy-to-download datasets at [TensorFlow Datasets](https://www.tensorflow.org/datasets).\n", "\n", - " duration = end-start\n", - " print(\"{} batches: {} s\".format(steps, duration))\n", - " print(\"{:0.5f} Images/s\".format(BATCH_SIZE*steps/duration))" - ], - "execution_count": 0, - "outputs": [] + "As you have previously loaded the Flowers dataset off disk, let's now import it with TensorFlow Datasets." + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "TYiOr4vdLcNX" + "id": "Qyu9wWDf1gfH" }, "source": [ - "Let's compare the speed of the two data generators:" + "Download the Flowers [dataset](https://www.tensorflow.org/datasets/catalog/tf_flowers) using TensorFlow Datasets:" ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NTQ-53DNwv8o" + }, + "outputs": [], + "source": [ + "(train_ds, val_ds, test_ds), metadata = tfds.load(\n", + " 'tf_flowers',\n", + " split=['train[:80%]', 'train[80%:90%]', 'train[90%:]'],\n", + " with_info=True,\n", + " as_supervised=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "85Yc-jZnVjvm", - "colab": {} + "id": "3hxXSgtj1iLV" }, "source": [ - "# `keras.preprocessing`\n", - "timeit(train_data_gen)" - ], - "execution_count": 0, - "outputs": [] + "The flowers dataset has five classes:" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "IjouTJadRxyp", - "colab": {} + "id": "kJvt6qzF1i4L" }, + "outputs": [], "source": [ - "# `tf.data`\n", - "timeit(train_ds)" - ], - "execution_count": 0, - "outputs": [] + "num_classes = metadata.features['label'].num_classes\n", + "print(num_classes)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "ZB2TjJR62BJ3" + "id": "6dbvEz_F1lgE" }, "source": [ - "A large part of the performance gain comes from the use of `.cache`." + " Retrieve an image from the dataset:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Oq1V854E2Nh4", - "colab": {} + "id": "1lF3IUAO1ogi" }, + "outputs": [], "source": [ - "uncached_ds = prepare_for_training(labeled_ds, cache=False)\n", - "timeit(uncached_ds)" - ], - "execution_count": 0, - "outputs": [] + "get_label_name = metadata.features['label'].int2str\n", + "\n", + "image, label = next(iter(train_ds))\n", + "_ = plt.imshow(image)\n", + "_ = plt.title(get_label_name(label))" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "-JCHymejWSPZ" + "id": "lHOOH_4TwaUb" }, "source": [ - "If the dataset doesn't fit in memory use a cache file to maintain some of the advantages:" + "As before, remember to batch, shuffle, and configure the training, validation, and test sets for performance:" ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AMV6GtZiwfGP" + }, + "outputs": [], + "source": [ + "train_ds = configure_for_performance(train_ds)\n", + "val_ds = configure_for_performance(val_ds)\n", + "test_ds = configure_for_performance(test_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gmR7kT8l1w20" + }, + "source": [ + "You can find a complete example of working with the Flowers dataset and TensorFlow Datasets by visiting the [Data augmentation](../images/data_augmentation.ipynb) tutorial." + ] + }, + { + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "RqHFQFwxWNbu", - "colab": {} + "id": "6cqkPenZIaHl" }, "source": [ - "filecache_ds = prepare_for_training(labeled_ds, cache=\"./flowers.tfcache\")\n", - "timeit(filecache_ds)" - ], - "execution_count": 0, - "outputs": [] + "## Next steps\n", + "\n", + "This tutorial showed two ways of loading images off disk. First, you learned how to load and preprocess an image dataset using Keras preprocessing layers and utilities. Next, you learned how to write an input pipeline from scratch using `tf.data`. Finally, you learned how to download a dataset from TensorFlow Datasets.\n", + "\n", + "For your next steps:\n", + "\n", + "- You can learn [how to add data augmentation](https://www.tensorflow.org/tutorials/images/data_augmentation).\n", + "- To learn more about `tf.data`, you can visit the [tf.data: Build TensorFlow input pipelines](https://www.tensorflow.org/guide/data) guide." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "images.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } - ] -} \ No newline at end of file + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/load_data/images/csv/Titanic.jpg b/site/en/tutorials/load_data/images/csv/Titanic.jpg new file mode 100644 index 00000000000..dc3a91a9eae Binary files /dev/null and b/site/en/tutorials/load_data/images/csv/Titanic.jpg differ diff --git a/site/en/tutorials/load_data/images/csv/fonts.jpg b/site/en/tutorials/load_data/images/csv/fonts.jpg new file mode 100644 index 00000000000..e9567e7f9bb Binary files /dev/null and b/site/en/tutorials/load_data/images/csv/fonts.jpg differ diff --git a/site/en/tutorials/load_data/images/csv/traffic.jpg b/site/en/tutorials/load_data/images/csv/traffic.jpg new file mode 100644 index 00000000000..503f5a18818 Binary files /dev/null and b/site/en/tutorials/load_data/images/csv/traffic.jpg differ diff --git a/site/en/tutorials/load_data/numpy.ipynb b/site/en/tutorials/load_data/numpy.ipynb index 9542d84a22a..86d226059bf 100644 --- a/site/en/tutorials/load_data/numpy.ipynb +++ b/site/en/tutorials/load_data/numpy.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pixDvex9KBqt" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "K16pBM8mKK7a" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TfRdquslKbO3" }, "source": [ @@ -47,30 +43,28 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-uq3F0ggKlZb" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/load_data/numpy\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/load_data/numpy.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/load_data/numpy.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/load_data/numpy.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-0tqX8qkXZEj" }, "source": [ @@ -82,7 +76,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-Ze5IBx9clLB" }, "source": [ @@ -91,32 +84,12 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "D1gtCQrnNk6b" - }, - "outputs": [], - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", "id": "k6J3JzK5NxQ6" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", " \n", "import numpy as np\n", "import tensorflow as tf" @@ -125,7 +98,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "G0yWiN8-cpDb" }, "source": [ @@ -134,10 +106,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "GLHNrFM6RWoM" }, "outputs": [], @@ -155,7 +125,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "cCeCkvrDgCMM" }, "source": [ @@ -165,7 +134,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "tslB0tJPgB-2" }, "source": [ @@ -174,10 +142,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "QN_8wwc5R7Qm" }, "outputs": [], @@ -189,7 +155,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6Rco85bbkDfN" }, "source": [ @@ -199,7 +164,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0dvl1uUukc4K" }, "source": [ @@ -208,10 +172,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "GTXdRMPcSXZj" }, "outputs": [], @@ -226,7 +188,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "w69Jl8k6lilg" }, "source": [ @@ -235,10 +196,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Uhxr8py4DkDN" }, "outputs": [], @@ -246,20 +205,18 @@ "model = tf.keras.Sequential([\n", " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", + " tf.keras.layers.Dense(10)\n", "])\n", "\n", "model.compile(optimizer=tf.keras.optimizers.RMSprop(),\n", - " loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n", - " metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])" + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['sparse_categorical_accuracy'])" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "XLDzlPGgOHBx" }, "outputs": [], @@ -269,10 +226,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "2q82yN8mmKIE" }, "outputs": [], @@ -285,10 +240,7 @@ "colab": { "collapsed_sections": [], "name": "numpy.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/load_data/pandas_dataframe.ipynb b/site/en/tutorials/load_data/pandas_dataframe.ipynb index db0c1395282..b9d0763e068 100644 --- a/site/en/tutorials/load_data/pandas_dataframe.ipynb +++ b/site/en/tutorials/load_data/pandas_dataframe.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "zwBCE43Cv3PH" }, "source": [ @@ -14,11 +13,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "fOad0I2cv569" }, "outputs": [], @@ -39,52 +36,48 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YQB7yiF6v9GR" }, "source": [ - "# Load a pandas.DataFrame" + "# Load a pandas DataFrame" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Oqa952X4wQKK" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/load_data/pandas_dataframe\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/load_data/pandas_dataframe.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/load_data/pandas_dataframe.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/load_data/pandas_dataframe.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UmyEaf4Awl2v" }, "source": [ - "This tutorial provides an example of how to load pandas dataframes into a `tf.data.Dataset`.\n", + "This tutorial provides examples of how to load pandas DataFrames into TensorFlow.\n", "\n", - "This tutorials uses a small [dataset](https://archive.ics.uci.edu/ml/datasets/heart+Disease) provided by the Cleveland Clinic Foundation for Heart Disease. There are several hundred rows in the CSV. Each row describes a patient, and each column describes an attribute. We will use this information to predict whether a patient has heart disease, which in this dataset is a binary classification task." + "You will use a small heart disease dataset provided by the UCI Machine Learning Repository. There are several hundred rows in the CSV. Each row describes a patient, and each column describes an attribute. You will use this information to predict whether a patient has heart disease, which is a binary classification task." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iiyC7HkqxlUD" }, "source": [ @@ -93,64 +86,53 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "5IoRbCA2n0_V" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", + "import numpy as np\n", "import pandas as pd\n", - "import tensorflow as tf" + "import tensorflow as tf\n", + "\n", + "SHUFFLE_BUFFER = 500\n", + "BATCH_SIZE = 2" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "-2kBGy_pxn47" }, "source": [ - "Download the csv file containing the heart dataset." + "Download the CSV file containing the heart disease dataset:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "VS4w2LePn9g3" }, "outputs": [], "source": [ - "csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/applied-dl/heart.csv')" + "csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/download.tensorflow.org/data/heart.csv')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6BXRPD2-xtQ1" }, "source": [ - "Read the csv file using pandas." + "Read the CSV file using pandas:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "UEfJ8TcMpe-2" }, "outputs": [], @@ -158,12 +140,19 @@ "df = pd.read_csv(csv_file)" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "4K873P-Pp8c7" + }, + "source": [ + "This is what the data looks like:" + ] + }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "8FkK6QIRpjd4" }, "outputs": [], @@ -173,10 +162,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "_MOAKz654CT5" }, "outputs": [], @@ -187,312 +174,1112 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "ww4lRDCS3qPh" + "id": "jVyGjKvnqGlb" }, "source": [ - "Convert `thal` column which is an `object` in the dataframe to a discrete numerical value." + "You will build models to predict the label contained in the `target` column." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LmCl5R5C2IKo" + "id": "2wwhILm1ycSp" }, "outputs": [], "source": [ - "df['thal'] = pd.Categorical(df['thal'])\n", - "df['thal'] = df.thal.cat.codes" + "target = df.pop('target')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vFGv9fgjDeao" + }, + "source": [ + "## A DataFrame as an array" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xNxJ41MafiB-" + }, + "source": [ + "If your data has a uniform datatype, or `dtype`, it's possible to use a pandas DataFrame anywhere you could use a NumPy array. This works because the `pandas.DataFrame` class supports the `__array__` protocol, and TensorFlow's `tf.convert_to_tensor` function accepts objects that support the protocol.\n", + "\n", + "Take the numeric features from the dataset (skip the categorical features for now):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "s4XA1SNW2QyI" + "id": "b9VlFGAie3K0" }, "outputs": [], "source": [ - "df.head()" + "numeric_feature_names = ['age', 'thalach', 'trestbps', 'chol', 'oldpeak']\n", + "numeric_features = df[numeric_feature_names]\n", + "numeric_features.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Xe1CMRvSpR_R" + }, + "source": [ + "The DataFrame can be converted to a NumPy array using the `DataFrame.values` property or `numpy.array(df)`. To convert it to a tensor, use `tf.convert_to_tensor`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OVv6Nwc9oDBU" + }, + "outputs": [], + "source": [ + "tf.convert_to_tensor(numeric_features)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7iRYvoTrr1_G" + }, + "source": [ + "In general, if an object can be converted to a tensor with `tf.convert_to_tensor` it can be passed anywhere you can pass a `tf.Tensor`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RVF7_Z-Mp-qD" + }, + "source": [ + "### With Model.fit" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "WWRhH6r4xxQu" + "id": "Vqkc9gIapQNu" }, "source": [ - "## Load data using `tf.data.Dataset`" + "A DataFrame, interpreted as a single tensor, can be used directly as an argument to the `Model.fit` method.\n", + "\n", + "Below is an example of training a model on the numeric features of the dataset." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "GuqmVVH_yApQ" + "id": "u8M3oYHZgH_t" }, "source": [ - "Use `tf.data.Dataset.from_tensor_slices` to read the values from a pandas dataframe. \n", + "The first step is to normalize the input ranges. Use a `tf.keras.layers.Normalization` layer for that.\n", "\n", - "One of the advantages of using `tf.data.Dataset` is it allows you to write simple, highly efficient data pipelines. Read the [loading data guide](https://www.tensorflow.org/guide/data) to find out more." + "To set the layer's mean and standard-deviation before running it be sure to call the `Normalization.adapt` method:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2wwhILm1ycSp" + "id": "88XTmyEdgkJn" }, "outputs": [], "source": [ - "target = df.pop('target')" + "normalizer = tf.keras.layers.Normalization(axis=-1)\n", + "normalizer.adapt(np.array(numeric_features))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_D7JqUtnYCnb" + }, + "source": [ + "Call the layer on the first three rows of the DataFrame to visualize an example of the output from this layer:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "W6Yc-D3aqyBb" + "id": "jOwzIG-DhB0y" }, "outputs": [], "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((df.values, target.values))" + "normalizer(numeric_features.iloc[:3])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KWKcuVZJh-HY" + }, + "source": [ + "Use the normalization layer as the first layer of a simple model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "chEnp_Swsf0a" + "id": "lu-bni-nh6mX" }, "outputs": [], "source": [ - "for feat, targ in dataset.take(5):\n", - " print ('Features: {}, Target: {}'.format(feat, targ))" + "def get_basic_model():\n", + " model = tf.keras.Sequential([\n", + " normalizer,\n", + " tf.keras.layers.Dense(10, activation='relu'),\n", + " tf.keras.layers.Dense(10, activation='relu'),\n", + " tf.keras.layers.Dense(1)\n", + " ])\n", + "\n", + " model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])\n", + " return model" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "GzwlAhX6xH9Q" + "id": "ntGi6ngYitob" }, "source": [ - "Since a `pd.Series` implements the `__array__` protocol it can be used transparently nearly anywhere you would use a `np.array` or a `tf.Tensor`." + "When you pass the DataFrame as the `x` argument to `Model.fit`, Keras treats the DataFrame as it would a NumPy array:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GnpHHkpktl5y" + "id": "XMjM-eddiNNT" }, "outputs": [], "source": [ - "tf.constant(df['thal'])" + "model = get_basic_model()\n", + "model.fit(numeric_features, target, epochs=15, batch_size=BATCH_SIZE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EjtQbsRPEoJT" + }, + "source": [ + "### With tf.data" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "9XLxRHS10Ylp" + "id": "nSjV5gy3EsVv" }, "source": [ - "Shuffle and batch the dataset." + "If you want to apply `tf.data` transformations to a DataFrame of a uniform `dtype`, the `Dataset.from_tensor_slices` method will create a dataset that iterates over the rows of the DataFrame. Each row is initially a vector of values. To train a model, you need `(inputs, labels)` pairs, so pass `(features, labels)` and `Dataset.from_tensor_slices` will return the needed pairs of slices:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FCphpgdRGikx" + }, + "outputs": [], + "source": [ + "numeric_dataset = tf.data.Dataset.from_tensor_slices((numeric_features, target))\n", + "\n", + "for row in numeric_dataset.take(3):\n", + " print(row)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "R3dQ-83Ztsgl" + "id": "lStkN86gEkCe" }, "outputs": [], "source": [ - "train_dataset = dataset.shuffle(len(df)).batch(1)" + "numeric_batches = numeric_dataset.shuffle(1000).batch(BATCH_SIZE)\n", + "\n", + "model = get_basic_model()\n", + "model.fit(numeric_batches, epochs=15)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "bB9C0XJkyQEk" + "id": "NRASs9IIESWQ" + }, + "source": [ + "## A DataFrame as a dictionary" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NQcp7kiPF8TP" }, "source": [ - "## Create and train a model" + "When you start dealing with heterogeneous data, it is no longer possible to treat the DataFrame as if it were a single array. TensorFlow tensors require that all elements have the same `dtype`.\n", + "\n", + "So, in this case, you need to start treating it as a dictionary of columns, where each column has a uniform `dtype`. A DataFrame is a lot like a dictionary of arrays, so typically all you need to do is cast the DataFrame to a Python dict. Many important TensorFlow APIs support (nested-)dictionaries of arrays as inputs." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "hvFUYZr7OLcL" + "id": "9y5UMKL8bury" }, "source": [ - "" + "`tf.data` input pipelines handle this quite well. All `tf.data` operations handle dictionaries and tuples automatically. So, to make a dataset of dictionary-examples from a DataFrame, just cast it to a dict before slicing it with `Dataset.from_tensor_slices`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FQd9PcPRpkP4" + "id": "voDoA447GBC3" }, "outputs": [], "source": [ - "def get_compiled_model():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", - " ])\n", + "numeric_features_dict = {key: value.to_numpy()[:, tf.newaxis] for key, value in dict(numeric_features).items()}\n", + "target_array = target.to_numpy()[:, tf.newaxis]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "U3QDo-jwHYXc" + }, + "outputs": [], + "source": [ + "numeric_dict_ds = tf.data.Dataset.from_tensor_slices((numeric_features_dict , target_array))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HL4Bf1b7M7DT" + }, + "outputs": [], + "source": [ + "len(numeric_features_dict)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yyEERK9ldIi_" + }, + "source": [ + "Here are the first three examples from that dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "q0tDwk0VdH6D" + }, + "outputs": [], + "source": [ + "for row in numeric_dict_ds.take(3):\n", + " print(row)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dnoyoWLWx07i" + }, + "source": [ + "Typically, Keras models and layers expect a single input tensor, but these classes can accept and return nested structures of dictionaries, tuples and tensors. These structures are known as \"nests\" (refer to the `tf.nest` module for details).\n", "\n", - " model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - " return model" + "There are two equivalent ways you can write a Keras model that accepts a dictionary as input." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5xUTrm0apDTr" + }, + "source": [ + "### 1. The Model-subclass style\n", + "\n", + "You write a subclass of `tf.keras.Model` (or `tf.keras.Layer`). You directly handle the inputs, and create the outputs:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ybDzNUheqxJw" + "id": "Rz4Cg6WpzNzi" + }, + "outputs": [], + "source": [ + "#@title\n", + "class MyModel(tf.keras.Model):\n", + " def __init__(self):\n", + " # Create all the internal layers in init.\n", + " super().__init__()\n", + "\n", + " self.normalizer = tf.keras.layers.Normalization(axis=-1)\n", + "\n", + " self.seq = tf.keras.Sequential([\n", + " self.normalizer,\n", + " tf.keras.layers.Dense(10, activation='relu'),\n", + " tf.keras.layers.Dense(10, activation='relu'),\n", + " tf.keras.layers.Dense(1)\n", + " ])\n", + "\n", + " self.concat = tf.keras.layers.Concatenate(axis=1)\n", + "\n", + " def _stack(self, input_dict):\n", + " values = []\n", + " for key, value in sorted(input_dict.items()):\n", + " values.append(value)\n", + "\n", + " return self.concat(values)\n", + "\n", + " def adapt(self, inputs):\n", + " # Stack the inputs and `adapt` the normalization layer.\n", + " inputs = self._stack(inputs)\n", + " self.normalizer.adapt(inputs)\n", + "\n", + " def call(self, inputs):\n", + " # Stack the inputs\n", + " inputs = self._stack(inputs)\n", + " # Run them through all the layers.\n", + " result = self.seq(inputs)\n", + "\n", + " return result\n", + "\n", + "model = MyModel()\n", + "\n", + "model.adapt(numeric_features_dict)\n", + "\n", + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " metrics=['accuracy'],\n", + " run_eagerly=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hMLXNEDF_tu2" + }, + "source": [ + "This model can accept either a dictionary of columns or a dataset of dictionary-elements for training:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "v3xEjtHY8gZG" + }, + "outputs": [], + "source": [ + "model.fit(numeric_features_dict, target_array, epochs=5, batch_size=BATCH_SIZE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "73wgiTaVAA2F" }, "outputs": [], "source": [ - "model = get_compiled_model()\n", - "model.fit(train_dataset, epochs=15)" + "numeric_dict_batches = numeric_dict_ds.shuffle(SHUFFLE_BUFFER).batch(BATCH_SIZE)\n", + "model.fit(numeric_dict_batches, epochs=5)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "d6V_6F_MBiG9" + "id": "-xDB3HLZGzAb" + }, + "source": [ + "Here are the predictions for the first three examples:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xtolTQA-GpBW" }, + "outputs": [], "source": [ - "## Alternative to feature columns" + "model.predict(dict(numeric_features.iloc[:3]))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "X63B9vDsD8Ly" + "id": "QIIdxIYm13Ik" + }, + "source": [ + "### 2. The Keras functional style" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DG_bmO0sS_G5" }, + "outputs": [], "source": [ - "Passing a dictionary as an input to a model is as easy as creating a matching dictionary of `tf.keras.layers.Input` layers, applying any pre-processing and stacking them up using the [functional api](../../guide/keras/functional.ipynb). You can use this as an alternative to [feature columns](../keras/feature_columns.ipynb)." + "inputs = {}\n", + "for name, column in numeric_features.items():\n", + " inputs[name] = tf.keras.Input(\n", + " shape=(1,), name=name, dtype=tf.float32)\n", + "\n", + "inputs" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FwQ47_WmOBnY" + "id": "9iXU9oem12dL" }, "outputs": [], "source": [ - "inputs = {key: tf.keras.layers.Input(shape=(), name=key) for key in df.keys()}\n", - "x = tf.stack(list(inputs.values()), axis=-1)\n", + "xs = [value for key, value in sorted(inputs.items())]\n", + "\n", + "concat = tf.keras.layers.Concatenate(axis=1)\n", + "x = concat(xs)\n", "\n", + "normalizer = tf.keras.layers.Normalization(axis=-1)\n", + "normalizer.adapt(np.concatenate([value for key, value in sorted(numeric_features_dict.items())], axis=1))\n", + "\n", + "x = normalizer(x)\n", + "x = tf.keras.layers.Dense(10, activation='relu')(x)\n", "x = tf.keras.layers.Dense(10, activation='relu')(x)\n", - "output = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n", + "x = tf.keras.layers.Dense(1)(x)\n", "\n", - "model_func = tf.keras.Model(inputs=inputs, outputs=output)\n", + "model = tf.keras.Model(inputs, x)\n", "\n", - "model_func.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " metrics=['accuracy'],\n", + " run_eagerly=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xrAxmuJrEwnf" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(model, rankdir=\"LR\", show_shapes=True, show_layer_names=True)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "qSCN5f_vUURE" + "id": "UYtoAOIzCFY1" }, "source": [ - "The easiest way to preserve the column structure of a `pd.DataFrame` when used with `tf.data` is to convert the `pd.DataFrame` to a `dict`, and slice that dictionary." + "You can train the functional model the same way as the model subclass:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wUjRKgEhPZqK" + "id": "yAwjPq7I_ehX" }, "outputs": [], "source": [ - "dict_slices = tf.data.Dataset.from_tensor_slices((df.to_dict('list'), target.values)).batch(16)" + "model.fit(numeric_features_dict, target, epochs=5, batch_size=BATCH_SIZE)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WWRaiwxeyA9Z" + "id": "brwodxxVApO_" }, "outputs": [], "source": [ - "for dict_slice in dict_slices.take(1):\n", - " print (dict_slice)" + "numeric_dict_batches = numeric_dict_ds.shuffle(SHUFFLE_BUFFER).batch(BATCH_SIZE)\n", + "model.fit(numeric_dict_batches, epochs=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xhn0Bt_Xw4nO" + }, + "source": [ + "## Full example" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zYQ5fDaRxRWQ" + }, + "source": [ + "If you're passing a heterogeneous DataFrame to Keras, each column may need unique preprocessing. You could do this preprocessing directly in the DataFrame, but for a model to work correctly, inputs always need to be preprocessed the same way. So, the best approach is to build the preprocessing into the model. [Keras preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) cover many common tasks." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BFsDZeu-BQ-h" + }, + "source": [ + "### Build the preprocessing head" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C6aVQN4Gw-Va" + }, + "source": [ + "In this dataset some of the \"integer\" features in the raw data are actually Categorical indices. These indices are not really ordered numeric values (refer to the the dataset description for details). Because these are unordered they are inappropriate to feed directly to the model; the model would interpret them as being ordered. To use these inputs you'll need to encode them, either as one-hot vectors or embedding vectors. The same applies to string-categorical features.\n", + "\n", + "Note: If you have many features that need identical preprocessing it's more efficient to concatenate them together before applying the preprocessing.\n", + "\n", + "Binary features on the other hand do not generally need to be encoded or normalized.\n", + "\n", + "Start by by creating a list of the features that fall into each group:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IH2VCyLBPYX8" + }, + "outputs": [], + "source": [ + "binary_feature_names = ['sex', 'fbs', 'exang']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Pxh4FPucOpDz" + }, + "outputs": [], + "source": [ + "categorical_feature_names = ['cp', 'restecg', 'slope', 'thal', 'ca']" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HRcC8WkyamJb" + }, + "source": [ + "The next step is to build a preprocessing model that will apply appropriate preprocessing to each input and concatenate the results.\n", + "\n", + "This section uses the [Keras Functional API](https://www.tensorflow.org/guide/keras/functional) to implement the preprocessing. You start by creating one `tf.keras.Input` for each column of the dataframe:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "D3OeiteJbWvI" + }, + "outputs": [], + "source": [ + "inputs = {}\n", + "for name, column in df.items():\n", + " if type(column[0]) == str:\n", + " dtype = tf.string\n", + " elif (name in categorical_feature_names or\n", + " name in binary_feature_names):\n", + " dtype = tf.int64\n", + " else:\n", + " dtype = tf.float32\n", + "\n", + " inputs[name] = tf.keras.Input(shape=(1,), name=name, dtype=dtype)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5N3vBMjidpx6" + }, + "outputs": [], + "source": [ + "inputs" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_EEmzxinyhI4" + }, + "source": [ + "For each input you'll apply some transformations using Keras layers and TensorFlow ops. Each feature starts as a batch of scalars (`shape=(batch,)`). The output for each should be a batch of `tf.float32` vectors (`shape=(batch, n)`). The last step will concatenate all those vectors together.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ubBDazjNFWiF" + }, + "source": [ + "#### Binary inputs\n", + "\n", + "Since the binary inputs don't need any preprocessing, just add the vector axis, cast them to `float32` and add them to the list of preprocessed inputs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tmAIkOIid-Mp" + }, + "outputs": [], + "source": [ + "preprocessed = []\n", + "\n", + "for name in binary_feature_names:\n", + " inp = inputs[name]\n", + " preprocessed.append(inp)\n", + "\n", + "preprocessed" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZHQcdtG1GN7E" + }, + "source": [ + "#### Numeric inputs\n", + "\n", + "Like in the earlier section you'll want to run these numeric inputs through a `tf.keras.layers.Normalization` layer before using them. The difference is that this time they're input as a dict. The code below collects the numeric features from the DataFrame, stacks them together and passes those to the `Normalization.adapt` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UC9LaIBNIK5V" + }, + "outputs": [], + "source": [ + "normalizer = tf.keras.layers.Normalization(axis=-1)\n", + "normalizer.adapt(np.concatenate([value for key, value in sorted(numeric_features_dict.items())], axis=1))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S537tideIpeh" + }, + "source": [ + "The code below stacks the numeric features and runs them through the normalization layer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "U8MJiFpPK5uD" + }, + "outputs": [], + "source": [ + "numeric_inputs = []\n", + "for name in numeric_feature_names:\n", + " numeric_inputs.append(inputs[name])\n", + "\n", + "numeric_inputs = tf.keras.layers.Concatenate(axis=-1)(numeric_inputs)\n", + "numeric_normalized = normalizer(numeric_inputs)\n", + "\n", + "preprocessed.append(numeric_normalized)\n", + "\n", + "preprocessed" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G5f-VzASKPF7" + }, + "source": [ + "#### Categorical features" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z3wcFs1oKVao" + }, + "source": [ + "To use categorical features you'll first need to encode them into either binary vectors or embeddings. Since these features only contain a small number of categories, convert the inputs directly to one-hot vectors using the `output_mode='one_hot'` option, supported by both the `tf.keras.layers.StringLookup` and `tf.keras.layers.IntegerLookup` layers.\n", + "\n", + "Here is an example of how these layers work:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vXleJfBRS9xr" + }, + "outputs": [], + "source": [ + "vocab = ['a','b','c']\n", + "lookup = tf.keras.layers.StringLookup(vocabulary=vocab, output_mode='one_hot')\n", + "lookup(['c','a','a','b','zzz'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kRnsFYJiSVmH" + }, + "outputs": [], + "source": [ + "vocab = [1,4,7,99]\n", + "lookup = tf.keras.layers.IntegerLookup(vocabulary=vocab, output_mode='one_hot')\n", + "\n", + "lookup([-1,4,1])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "est6aCFBZDVs" + }, + "source": [ + "To determine the vocabulary for each input, create a layer to convert that vocabulary to a one-hot vector:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HELhoFlo0H9Q" + }, + "outputs": [], + "source": [ + "for name in categorical_feature_names:\n", + " vocab = sorted(set(df[name]))\n", + " print(f'name: {name}')\n", + " print(f'vocab: {vocab}\\n')\n", + "\n", + " if type(vocab[0]) is str:\n", + " lookup = tf.keras.layers.StringLookup(vocabulary=vocab, output_mode='one_hot')\n", + " else:\n", + " lookup = tf.keras.layers.IntegerLookup(vocabulary=vocab, output_mode='one_hot')\n", + "\n", + " x = inputs[name]\n", + " x = lookup(x)\n", + " preprocessed.append(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PzMMkwNBa2pK" + }, + "source": [ + "#### Assemble the preprocessing head" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GaQ-_pEQbCE8" + }, + "source": [ + "At this point `preprocessed` is just a Python list of all the preprocessing results, each result has a shape of `(batch_size, depth)`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LlLaq_BVRlnO" + }, + "outputs": [], + "source": [ + "preprocessed" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U9lYYHIXbYv-" + }, + "source": [ + "Concatenate all the preprocessed features along the `depth` axis, so each dictionary-example is converted into a single vector. The vector contains categorical features, numeric features, and categorical one-hot features:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "j2I8vpQh313w" + }, + "outputs": [], + "source": [ + "preprocessed_result = tf.keras.layers.Concatenate(axis=1)(preprocessed)\n", + "preprocessed_result" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OBFowyJtb0WB" + }, + "source": [ + "Now create a model out of that calculation so it can be reused:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rHQBFHwE37TO" + }, + "outputs": [], + "source": [ + "preprocessor = tf.keras.Model(inputs, preprocessed_result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ViMARQ-f6zfx" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(preprocessor, rankdir=\"LR\", show_shapes=True, show_layer_names=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IURRtL_WZbht" + }, + "source": [ + "To test the preprocessor, use the DataFrame.iloc accessor to slice the first example from the DataFrame. Then convert it to a dictionary and pass the dictionary to the preprocessor. The result is a single vector containing the binary features, normalized numeric features and the one-hot categorical features, in that order:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QjBzCKsZUj0y" + }, + "outputs": [], + "source": [ + "preprocessor(dict(df.iloc[:1]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bB9C0XJkyQEk" + }, + "source": [ + "### Create and train a model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WfU_FFXMbKGM" + }, + "source": [ + "Now build the main body of the model. Use the same configuration as in the previous example: A couple of `Dense` rectified-linear layers and a `Dense(1)` output layer for the classification." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "75OxXTnfboKN" + }, + "outputs": [], + "source": [ + "body = tf.keras.Sequential([\n", + " tf.keras.layers.Dense(10, activation='relu'),\n", + " tf.keras.layers.Dense(10, activation='relu'),\n", + " tf.keras.layers.Dense(1)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MpD6WNX5_zh5" + }, + "source": [ + "Now put the two pieces together using the Keras functional API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_TY_BuVMbNcB" + }, + "outputs": [], + "source": [ + "inputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "iin2kvA9bDpz" + }, + "outputs": [], + "source": [ + "x = preprocessor(inputs)\n", + "x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FQd9PcPRpkP4" + }, + "outputs": [], + "source": [ + "result = body(x)\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "v_KerrXabhgP" + }, + "outputs": [], + "source": [ + "model = tf.keras.Model(inputs, result)\n", + "\n", + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "i_Z2C2ZcZ3oC" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(model, show_shapes=True, show_layer_names=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S1MR-XD9kC6C" + }, + "source": [ + "This model expects a dictionary of inputs. The simplest way to pass it the data is to convert the DataFrame to a dict and pass that dict as the `x` argument to `Model.fit`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ybDzNUheqxJw" + }, + "outputs": [], + "source": [ + "history = model.fit(dict(df), target, epochs=5, batch_size=BATCH_SIZE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dacoEIB_BSsL" + }, + "source": [ + "Using `tf.data` works as well:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rYadV3wwE4G3" + }, + "outputs": [], + "source": [ + "ds = tf.data.Dataset.from_tensor_slices((\n", + " dict(df),\n", + " target\n", + "))\n", + "\n", + "ds = ds.batch(BATCH_SIZE)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2YIpp2r0bv-6" + }, + "outputs": [], + "source": [ + "import pprint\n", + "\n", + "for x, y in ds.take(1):\n", + " pprint.pprint(x)\n", + " print()\n", + " print(y)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8nTrfczNyKup" + "id": "NMT-AevGFmdu" }, "outputs": [], "source": [ - "model_func.fit(dict_slices, epochs=15)" + "history = model.fit(ds, epochs=5)" ] } ], "metadata": { "colab": { - "collapsed_sections": [], "name": "pandas_dataframe.ipynb", - "private_outputs": true, - "provenance": [ - { - "file_id": "1XwmbuQRh8Xo93CV_op6Q_ty2TAjGoBjc", - "timestamp": 1558919304199 - } - ], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/load_data/text.ipynb b/site/en/tutorials/load_data/text.ipynb index 21378e350aa..35004db9b94 100644 --- a/site/en/tutorials/load_data/text.ipynb +++ b/site/en/tutorials/load_data/text.ipynb @@ -3,21 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DweYe9FcbMK_" }, "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" + "##### Copyright 2018 The TensorFlow Authors.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "AVV2e0XKbJeX" }, "outputs": [], @@ -38,8 +34,7 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "sUtoed20cRJJ" + "id": "sZfSvVcDo6GQ" }, "source": [ "# Load text" @@ -48,614 +43,1839 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "1ap_W4aQcgNT" + "id": "giK0nMbZFnoR" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/load_data/text\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/load_data/text.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/load_data/text.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/load_data/text.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "NWeQAo0Ec_BL" + "id": "dwlfPb11GH8J" }, "source": [ - "This tutorial provides an example of how to use `tf.data.TextLineDataset` to load examples from text files. `TextLineDataset` is designed to create a dataset from a text file, in which each example is a line of text from the original file. This is potentially useful for any text data that is primarily line-based (for example, poetry or error logs).\n", + "This tutorial demonstrates two ways to load and preprocess text.\n", "\n", - "In this tutorial, we'll use three different English translations of the same work, Homer's Illiad, and train a model to identify the translator given a single line of text." + "- First, you will use Keras utilities and preprocessing layers. These include `tf.keras.utils.text_dataset_from_directory` to turn data into a `tf.data.Dataset` and `tf.keras.layers.TextVectorization` for data standardization, tokenization, and vectorization. If you are new to TensorFlow, you should start with these.\n", + "- Then, you will use lower-level utilities like `tf.data.TextLineDataset` to load text files, `tf.lookup` for custom in-model lookup tables, and [TensorFlow Text](https://www.tensorflow.org/text) APIs, such as `text.UnicodeScriptTokenizer` and `text.case_fold_utf8`, to preprocess the data for finer-grain control." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" + "id": "sa6IKWvADqH7" }, + "outputs": [], "source": [ - "## Setup" + "!pip install \"tensorflow-text==2.13.*\"" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "baYFZMW_bJHh" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", + "import collections\n", + "import pathlib\n", "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", + "import matplotlib.pyplot as plt\n", "import tensorflow as tf\n", "\n", + "from tensorflow.keras import layers\n", + "from tensorflow.keras import losses\n", + "from tensorflow.keras import utils\n", + "from tensorflow.keras.layers import TextVectorization\n", + "\n", "import tensorflow_datasets as tfds\n", - "import os" + "import tensorflow_text as tf_text" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "YWVWjyIkffau" + "id": "Az-d_K5_HQ5k" }, "source": [ - "The texts of the three translations are by:\n", + "## Example 1: Predict the tag for a Stack Overflow question\n", + "\n", + "As a first example, you will download a dataset of programming questions from Stack Overflow. Each question (_\"How do I sort a dictionary by value?\"_) is labeled with exactly one tag (`Python`, `CSharp`, `JavaScript`, or `Java`). Your task is to develop a model that predicts the tag for a question. This is an example of multi-class classification—an important and widely applicable kind of machine learning problem." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bU4PRq7Z81B0" + }, + "source": [ + "To implement this task, you'll start with the simplest tools:\n", "\n", - " - [William Cowper](https://en.wikipedia.org/wiki/William_Cowper) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/cowper.txt)\n", + "* `keras.utils.text_datasaet_from_directory`: for loading text-file examples.\n", + "* `keras.layers.TextVectorization`: for converting strings to token indices.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tjC3yLa5IjP7" + }, + "source": [ + "### Download and explore the dataset\n", "\n", - " - [Edward, Earl of Derby](https://en.wikipedia.org/wiki/Edward_Smith-Stanley,_14th_Earl_of_Derby) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/derby.txt)\n", + "Begin by downloading the Stack Overflow dataset using `tf.keras.utils.get_file`, and exploring the directory structure:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8ELgzA6SHTuV" + }, + "outputs": [], + "source": [ + "data_url = 'https://storage.googleapis.com/download.tensorflow.org/data/stack_overflow_16k.tar.gz'\n", "\n", - "- [Samuel Butler](https://en.wikipedia.org/wiki/Samuel_Butler_%28novelist%29) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/butler.txt)\n", + "dataset_dir = utils.get_file(\n", + " origin=data_url,\n", + " untar=True,\n", + " cache_dir='stack_overflow',\n", + " cache_subdir='')\n", "\n", - "The text files used in this tutorial have undergone some typical preprocessing tasks, mostly removing stuff — document header and footer, line numbers, chapter titles. Download these lightly munged files locally." + "dataset_dir = pathlib.Path(dataset_dir).parent" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4YlKQthEYlFw" + "id": "jIrPl5fUH2gb" }, "outputs": [], "source": [ - "DIRECTORY_URL = 'https://storage.googleapis.com/download.tensorflow.org/data/illiad/'\n", - "FILE_NAMES = ['cowper.txt', 'derby.txt', 'butler.txt']\n", + "list(dataset_dir.iterdir())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fEoV7YByJoWQ" + }, + "outputs": [], + "source": [ + "train_dir = dataset_dir/'train'\n", + "list(train_dir.iterdir())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3mxAN17MhEh0" + }, + "source": [ + "The `train/csharp`, `train/java`, `train/python` and `train/javascript` directories contain many text files, each of which is a Stack Overflow question.\n", "\n", - "for name in FILE_NAMES:\n", - " text_dir = tf.keras.utils.get_file(name, origin=DIRECTORY_URL+name)\n", - " \n", - "parent_dir = os.path.dirname(text_dir)\n", + "Print an example file and inspect the data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Go1vTSGdJu08" + }, + "outputs": [], + "source": [ + "sample_file = train_dir/'python/1755.txt'\n", + "\n", + "with open(sample_file) as f:\n", + " print(f.read())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "deWBTkpJiO7D" + }, + "source": [ + "### Load the dataset\n", + "\n", + "Next, you will load the data off-disk and prepare it into a format suitable for training. To do so, you will use the `tf.keras.utils.text_dataset_from_directory` utility to create a labeled `tf.data.Dataset`. If you're new to `tf.data`, it's a powerful collection of tools for building input pipelines. (Learn more in the [tf.data: Build TensorFlow input pipelines](../../guide/data.ipynb) guide.)\n", "\n", - "parent_dir" + "The `tf.keras.utils.text_dataset_from_directory` API expects a directory structure as follows:\n", + "\n", + "```\n", + "train/\n", + "...csharp/\n", + "......1.txt\n", + "......2.txt\n", + "...java/\n", + "......1.txt\n", + "......2.txt\n", + "...javascript/\n", + "......1.txt\n", + "......2.txt\n", + "...python/\n", + "......1.txt\n", + "......2.txt\n", + "```" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "q3sDy6nuXoNp" + "id": "Dyl6JTAjlbQV" }, "source": [ - "## Load text into datasets\n", + "When running a machine learning experiment, it is a best practice to divide your dataset into three splits: [training](https://developers.google.com/machine-learning/glossary#training_set), [validation](https://developers.google.com/machine-learning/glossary#validation_set), and [test](https://developers.google.com/machine-learning/glossary#test-set).\n", "\n", - "Iterate through the files, loading each one into its own dataset.\n", + "The Stack Overflow dataset has already been divided into training and test sets, but it lacks a validation set.\n", "\n", - "Each example needs to be individually labeled, so use `tf.data.Dataset.map` to apply a labeler function to each one. This will iterate over every example in the dataset, returning (`example, label`) pairs." + "Create a validation set using an 80:20 split of the training data by using `tf.keras.utils.text_dataset_from_directory` with `validation_split` set to `0.2` (i.e. 20%):" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "K0BjCOpOh7Ch" + "id": "qqyliMw8N-az" }, "outputs": [], "source": [ - "def labeler(example, index):\n", - " return example, tf.cast(index, tf.int64) \n", + "batch_size = 32\n", + "seed = 42\n", "\n", - "labeled_data_sets = []\n", + "raw_train_ds = utils.text_dataset_from_directory(\n", + " train_dir,\n", + " batch_size=batch_size,\n", + " validation_split=0.2,\n", + " subset='training',\n", + " seed=seed)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DMI_gPLfloD7" + }, + "source": [ + "As the previous cell output suggests, there are 8,000 examples in the training folder, of which you will use 80% (or 6,400) for training. You will learn in a moment that you can train a model by passing a `tf.data.Dataset` directly to `Model.fit`.\n", "\n", - "for i, file_name in enumerate(FILE_NAMES):\n", - " lines_dataset = tf.data.TextLineDataset(os.path.join(parent_dir, file_name))\n", - " labeled_dataset = lines_dataset.map(lambda ex: labeler(ex, i))\n", - " labeled_data_sets.append(labeled_dataset)" + "First, iterate over the dataset and print out a few examples, to get a feel for the data.\n", + "\n", + "Note: To increase the difficulty of the classification problem, the dataset author replaced occurrences of the words *Python*, *CSharp*, *JavaScript*, or *Java* in the programming question with the word *blank*." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_JMTyZ6Glt_C" + }, + "outputs": [], + "source": [ + "for text_batch, label_batch in raw_train_ds.take(1):\n", + " for i in range(10):\n", + " print(\"Question: \", text_batch.numpy()[i])\n", + " print(\"Label:\", label_batch.numpy()[i])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "M8PHK5J_cXE5" + "id": "jCZGl4Q5l2sS" }, "source": [ - "Combine these labeled datasets into a single dataset, and shuffle it.\n" + "The labels are `0`, `1`, `2` or `3`. To check which of these correspond to which string label, you can inspect the `class_names` property on the dataset:\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6jAeYkTIi9-2" + "id": "gIpCS7YjmGkj" }, "outputs": [], "source": [ - "BUFFER_SIZE = 50000\n", - "BATCH_SIZE = 64\n", - "TAKE_SIZE = 5000" + "for i, label in enumerate(raw_train_ds.class_names):\n", + " print(\"Label\", i, \"corresponds to\", label)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oUsdn-37qol9" + }, + "source": [ + "Next, you will create a validation and a test set using `tf.keras.utils.text_dataset_from_directory`. You will use the remaining 1,600 reviews from the training set for validation.\n", + "\n", + "Note: When using the `validation_split` and `subset` arguments of `tf.keras.utils.text_dataset_from_directory`, make sure to either specify a random seed or pass `shuffle=False`, so that the validation and training splits have no overlap." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Qd544E-Sh63L" + "id": "x7m6sCWJQuYt" }, "outputs": [], "source": [ - "all_labeled_data = labeled_data_sets[0]\n", - "for labeled_dataset in labeled_data_sets[1:]:\n", - " all_labeled_data = all_labeled_data.concatenate(labeled_dataset)\n", - " \n", - "all_labeled_data = all_labeled_data.shuffle(\n", - " BUFFER_SIZE, reshuffle_each_iteration=False)" + "# Create a validation set.\n", + "raw_val_ds = utils.text_dataset_from_directory(\n", + " train_dir,\n", + " batch_size=batch_size,\n", + " validation_split=0.2,\n", + " subset='validation',\n", + " seed=seed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BXMZc7fMQwKE" + }, + "outputs": [], + "source": [ + "test_dir = dataset_dir/'test'\n", + "\n", + "# Create a test set.\n", + "raw_test_ds = utils.text_dataset_from_directory(\n", + " test_dir,\n", + " batch_size=batch_size)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "r4JEHrJXeG5k" + "id": "NHuAF8hYfP5Z" }, "source": [ - "You can use `tf.data.Dataset.take` and `print` to see what the `(example, label)` pairs look like. The `numpy` property shows each Tensor's value." + "### Configure the datasets for performance\n", + "\n", + "These are two important methods you should use when loading data to make sure that I/O does not become blocking.\n", + "\n", + "- `Dataset.cache` keeps data in memory after it's loaded off-disk. This will ensure the dataset does not become a bottleneck while training your model. If your dataset is too large to fit into memory, you can also use this method to create a performant on-disk cache, which is more efficient to read than many small files.\n", + "- `Dataset.prefetch` overlaps data preprocessing and model execution while training.\n", + "\n", + "You can learn more about both methods, as well as how to cache data to disk in the *Prefetching* section of the [Better performance with the tf.data API](../../guide/data_performance.ipynb) guide." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gywKlN0xh6u5" + "id": "h14NxXx13WRD" }, "outputs": [], "source": [ - "for ex in all_labeled_data.take(5):\n", - " print(ex)" + "raw_train_ds = raw_train_ds.cache().prefetch(buffer_size=tf.data.AUTOTUNE)\n", + "raw_val_ds = raw_val_ds.cache().prefetch(buffer_size=tf.data.AUTOTUNE)\n", + "raw_test_ds = raw_test_ds.prefetch(buffer_size=tf.data.AUTOTUNE)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "5rrpU2_sfDh0" + "id": "8Ph91k8n3yPB" + }, + "source": [ + "This first example will keep things simple by integrating the text processing into the model. But you may be able to increase performance by moving the text processing into the dataset pipeline, this is demonstrated in [Example 2](#example2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Xdt-ATrGRGDL" + }, + "source": [ + "### Prepare the dataset for training" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N6fRti45Rlj8" }, "source": [ - "## Encode text lines as numbers\n", + "Next, you will standardize, tokenize, and vectorize the data using the `tf.keras.layers.TextVectorization` layer.\n", + "\n", + "- _Standardization_ refers to preprocessing the text, typically to remove punctuation or HTML elements to simplify the dataset.\n", + "- _Tokenization_ refers to splitting strings into tokens (for example, splitting a sentence into individual words by splitting on whitespace).\n", + "- _Vectorization_ refers to converting tokens into numbers so they can be fed into a neural network.\n", "\n", - "Machine learning models work on numbers, not words, so the string values need to be converted into lists of numbers. To do that, map each unique word to a unique integer.\n", + "All of these tasks can be accomplished with this layer. (You can learn more about each of these in the `tf.keras.layers.TextVectorization` API docs.)\n", "\n", - "### Build vocabulary\n", + "Note that:\n", "\n", - "First, build a vocabulary by tokenizing the text into a collection of individual unique words. There are a few ways to do this in both TensorFlow and Python. For this tutorial:\n", + "- The default standardization converts text to lowercase and removes punctuation (`standardize='lower_and_strip_punctuation'`).\n", + "- The default tokenizer splits on whitespace (`split='whitespace'`).\n", + "- The default vectorization mode is `'int'` (`output_mode='int'`). This outputs integer indices (one per token). This mode can be used to build models that take word order into account. You can also use other modes—like `'binary'`—to build [bag-of-words](https://developers.google.com/machine-learning/glossary#bag-of-words) models.\n", "\n", - "1. Iterate over each example's `numpy` value.\n", - "2. Use `tfds.features.text.Tokenizer` to split it into tokens.\n", - "3. Collect these tokens into a Python set, to remove duplicates.\n", - "4. Get the size of the vocabulary for later use." + "You will build two models to learn more about standardization, tokenization, and vectorization with `TextVectorization`:\n", + "\n", + "- First, you will use the `'binary'` vectorization mode to build a bag-of-words model.\n", + "- Then, you will use the `'int'` mode with a 1D ConvNet." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YkHtbGnDh6mg" + "id": "voaC43rZR0jc" }, "outputs": [], "source": [ - "tokenizer = tfds.features.text.Tokenizer()\n", + "VOCAB_SIZE = 10000\n", "\n", - "vocabulary_set = set()\n", - "for text_tensor, _ in all_labeled_data:\n", - " some_tokens = tokenizer.tokenize(text_tensor.numpy())\n", - " vocabulary_set.update(some_tokens)\n", + "binary_vectorize_layer = TextVectorization(\n", + " max_tokens=VOCAB_SIZE,\n", + " output_mode='binary')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ifDPFxuf2Hfz" + }, + "source": [ + "For the `'int'` mode, in addition to maximum vocabulary size, you need to set an explicit maximum sequence length (`MAX_SEQUENCE_LENGTH`), which will cause the layer to pad or truncate sequences to exactly `output_sequence_length` values:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XWsY01Zl2aRe" + }, + "outputs": [], + "source": [ + "MAX_SEQUENCE_LENGTH = 250\n", "\n", - "vocab_size = len(vocabulary_set)\n", - "vocab_size" + "int_vectorize_layer = TextVectorization(\n", + " max_tokens=VOCAB_SIZE,\n", + " output_mode='int',\n", + " output_sequence_length=MAX_SEQUENCE_LENGTH)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "0W35VJqAh9zs" + "id": "ts6h9b5atD-Y" }, "source": [ - "### Encode examples\n", + "Next, call `TextVectorization.adapt` to fit the state of the preprocessing layer to the dataset. This will cause the model to build an index of strings to integers.\n", "\n", - "Create an encoder by passing the `vocabulary_set` to `tfds.features.text.TokenTextEncoder`. The encoder's `encode` method takes in a string of text and returns a list of integers." + "Note: It's important to only use your training data when calling `TextVectorization.adapt`, as using the test set would leak information." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gkxJIVAth6j0" + "id": "yTXsdDEqSf9e" }, "outputs": [], "source": [ - "encoder = tfds.features.text.TokenTextEncoder(vocabulary_set)" + "# Make a text-only dataset (without labels), then call `TextVectorization.adapt`.\n", + "train_text = raw_train_ds.map(lambda text, labels: text)\n", + "binary_vectorize_layer.adapt(train_text)\n", + "int_vectorize_layer.adapt(train_text)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "v6S5Qyabi-vo" + "id": "XKVO6Jg7Sls0" }, "source": [ - "You can try this on a single line to see what the output looks like." + "Print the result of using these layers to preprocess data:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jgxPZaxUuTbk" + "id": "Vi_sElMiSmXe" + }, + "outputs": [], + "source": [ + "# Retrieve a batch (of 32 reviews and labels) from the dataset.\n", + "text_batch, label_batch = next(iter(raw_train_ds))\n", + "first_question, first_label = text_batch[0], label_batch[0]\n", + "print(\"Question:\", first_question)\n", + "print(\"Label:\", first_label)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a2nTgt9e1KjK" + }, + "source": [ + "The binary vectorization layer returns a multi-hot vector, with a 1 in the location for each token that was in the input string." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UGukZoYv2v3v" }, "outputs": [], "source": [ - "example_text = next(iter(all_labeled_data))[0].numpy()\n", - "print(example_text)" + "print(\"'binary' vectorized question:\",\n", + " list(binary_vectorize_layer(first_question).numpy()))\n", + "\n", + "plt.plot(binary_vectorize_layer(first_question).numpy())\n", + "plt.xlim(0,1000)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "XoVpKR3qj5yb" + "id": "Lu07FsIw2yH5" }, "outputs": [], "source": [ - "encoded_example = encoder.encode(example_text)\n", - "print(encoded_example)" + "print(\"'int' vectorized question:\",\n", + " int_vectorize_layer(first_question).numpy())" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "p9qHM0v8k_Mg" + "id": "wgjeF9PdS7tN" }, "source": [ - "Now run the encoder on the dataset by wrapping it in `tf.py_function` and passing that to the dataset's `map` method." + "As shown above, `TextVectorization`'s `'binary'` mode returns an array denoting which tokens exist at least once in the input, while the `'int'` mode replaces each token by an integer, thus preserving their order.\n", + "\n", + "You can lookup the token (string) that each integer corresponds to by calling `TextVectorization.get_vocabulary` on the layer:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HcIQ7LOTh6eT" + "id": "WpBnTZilS8wt" }, "outputs": [], "source": [ - "def encode(text_tensor, label):\n", - " encoded_text = encoder.encode(text_tensor.numpy())\n", - " return encoded_text, label\n", + "print(\"1289 ---> \", int_vectorize_layer.get_vocabulary()[1289])\n", + "print(\"313 ---> \", int_vectorize_layer.get_vocabulary()[313])\n", + "print(\"Vocabulary size: {}\".format(len(int_vectorize_layer.get_vocabulary())))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NYGb7z_bfpGm" + }, + "source": [ + "### Train the model\n", "\n", - "def encode_map_fn(text, label):\n", - " return tf.py_function(encode, inp=[text, label], Tout=(tf.int64, tf.int64))\n", + "It's time to create your neural network.\n", "\n", - "all_encoded_data = all_labeled_data.map(encode_map_fn)" + "For the `'binary'` vectorized data, define a simple bag-of-words linear model, then configure and train it:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "_YZToSXSm0qr" + "id": "L9D0Gngw8Cwr" }, + "outputs": [], "source": [ - "## Split the dataset into test and train batches\n", + "binary_model = tf.keras.Sequential([\n", + " binary_vectorize_layer,\n", + " layers.Dense(4)])\n", "\n", - "Use `tf.data.Dataset.take` and `tf.data.Dataset.skip` to create a small test dataset and a larger training set.\n", + "binary_model.compile(\n", + " loss=losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " optimizer='adam',\n", + " metrics=['accuracy'])\n", "\n", - "Before being passed into the model, the datasets need to be batched. Typically, the examples inside of a batch need to be the same size and shape. But, the examples in these datasets are not all the same size — each line of text had a different number of words. So use `tf.data.Dataset.padded_batch` (instead of `batch`) to pad the examples to the same size." + "tf.keras.utils.plot_model(binary_model, show_shapes=True)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "r-rmbijQh6bf" + "id": "n12Cnhtc_bct" }, "outputs": [], "source": [ - "train_data = all_encoded_data.skip(TAKE_SIZE).shuffle(BUFFER_SIZE)\n", - "train_data = train_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))\n", + "bin_history = binary_model.fit(\n", + " raw_train_ds, validation_data=raw_val_ds, epochs=10)\n", "\n", - "test_data = all_encoded_data.take(TAKE_SIZE)\n", - "test_data = test_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))" + "print()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "Xdz7SVwmqi1l" + "id": "EwidD-SwNIkz" + }, + "source": [ + "Next, you will use the `'int'` vectorized layer to build a 1D ConvNet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nxLCInw7AUFr" }, + "outputs": [], "source": [ - "Now, `test_data` and `train_data` are not collections of (`example, label`) pairs, but collections of batches. Each batch is a pair of (*many examples*, *many labels*) represented as arrays.\n", + "def create_model(vocab_size, num_labels, vectorizer=None):\n", + " my_layers =[]\n", + " if vectorizer is not None:\n", + " my_layers = [vectorizer]\n", "\n", - "To illustrate:" + " my_layers.extend([\n", + " layers.Embedding(vocab_size, 64, mask_zero=True),\n", + " layers.Dropout(0.5),\n", + " layers.Conv1D(64, 5, padding=\"valid\", activation=\"relu\", strides=2),\n", + " layers.GlobalMaxPooling1D(),\n", + " layers.Dense(num_labels)\n", + " ])\n", + "\n", + " model = tf.keras.Sequential(my_layers)\n", + " return model" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kMslWfuwoqpB" + "id": "_igpRY7p8NTK" }, "outputs": [], "source": [ - "sample_text, sample_labels = next(iter(test_data))\n", + "# `vocab_size` is `VOCAB_SIZE + 1` since `0` is used additionally for padding.\n", + "int_model = create_model(vocab_size=VOCAB_SIZE + 1, num_labels=4, vectorizer=int_vectorize_layer)\n", "\n", - "sample_text[0], sample_labels[0]" + "tf.keras.utils.plot_model(int_model, show_shapes=True)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "UI4I6_Sa0vWu" + "id": "C4UA3h8GAUFs" }, + "outputs": [], "source": [ - "Since we have introduced a new token encoding (the zero used for padding), the vocabulary size has increased by one." + "\n", + "int_model.compile(\n", + " loss=losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " optimizer='adam',\n", + " metrics=['accuracy'])\n", + "int_history = int_model.fit(raw_train_ds, validation_data=raw_val_ds, epochs=10)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IlD1Lli91vuc" + "id": "hHTVwxNAovn7" }, "outputs": [], "source": [ - "vocab_size += 1" + "loss = plt.plot(bin_history.epoch, bin_history.history['loss'], label='bin-loss')\n", + "plt.plot(bin_history.epoch, bin_history.history['val_loss'], '--', color=loss[0].get_color(), label='bin-val_loss')\n", + "\n", + "loss = plt.plot(int_history.epoch, int_history.history['loss'], label='int-loss')\n", + "plt.plot(int_history.epoch, int_history.history['val_loss'], '--', color=loss[0].get_color(), label='int-val_loss')\n", + "\n", + "plt.legend()\n", + "plt.xlabel('Epoch')\n", + "plt.ylabel('CE/token')" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "K8SUhGFNsmRi" + "id": "0kHgPE_YwHvp" }, "source": [ - "## Build the model\n", - "\n" + "You are nearly ready to train your model.\n", + "\n", + "As a final preprocessing step, you will apply the `TextVectorization` layers you created earlier to the training, validation, and test sets:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QJgI1pow2YR9" + "id": "46LeHmnD55wJ" }, "outputs": [], "source": [ - "model = tf.keras.Sequential()" + "binary_train_ds = raw_train_ds.map(lambda x,y: (binary_vectorize_layer(x), y))\n", + "binary_val_ds = raw_val_ds.map(lambda x,y: (binary_vectorize_layer(x), y))\n", + "binary_test_ds = raw_test_ds.map(lambda x,y: (binary_vectorize_layer(x), y))\n", + "\n", + "int_train_ds = raw_train_ds.map(lambda x,y: (int_vectorize_layer(x), y))\n", + "int_val_ds = raw_val_ds.map(lambda x,y: (int_vectorize_layer(x), y))\n", + "int_test_ds = raw_test_ds.map(lambda x,y: (int_vectorize_layer(x), y))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "wi0iiKLTKdoF" + "id": "NZo_OTxD3ItL" }, "source": [ - "The first layer converts integer representations to dense vector embeddings. See the [word embeddings tutorial](../text/word_embeddings.ipynb) or more details. " + "### Export the model" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DR6-ctbY638P" + "id": "HYteWp9fykbP" }, "outputs": [], "source": [ - "model.add(tf.keras.layers.Embedding(vocab_size, 64))" + "binary_model.export('bin.tf')" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OPD0WE_vy-dT" + }, + "outputs": [], + "source": [ + "loaded = tf.saved_model.load('bin.tf')" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "text", - "id": "_8OJOPohKh1q" + "id": "xFoBg53DzE2s" }, + "outputs": [], "source": [ - "The next layer is a [Long Short-Term Memory](http://colah.github.io/posts/2015-08-Understanding-LSTMs/) layer, which lets the model understand words in their context with other words. A bidirectional wrapper on the LSTM helps it to learn about the datapoints in relationship to the datapoints that came before it and after it." + "binary_model.predict(['How do you sort a list?'])" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "x6rnq6DN_WUs" + "id": "Cx5aIBm3zVQ0" }, "outputs": [], "source": [ - "model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))" + "loaded.serve(tf.constant(['How do you sort a list?'])).numpy()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "cdffbMr5LF1g" + "id": "0QDVfii_4slI" }, "source": [ - "Finally we'll have a series of one or more densely connected layers, with the last one being the output layer. The output layer produces a probability for all the labels. The one with the highest probability is the models prediction of an example's label." + "\n", + "Including the text preprocessing logic inside your model enables you to export a model for production that simplifies deployment, and reduces the potential for [train/test skew](https://developers.google.com/machine-learning/guides/rules-of-ml#training-serving_skew).\n", + "\n", + "There is a performance difference to keep in mind when choosing where to apply `tf.keras.layers.TextVectorization`. Using it outside of your model enables you to do asynchronous CPU processing and buffering of your data when training on GPU. So, if you're training your model on the GPU, you probably want to go with this option to get the best performance while developing your model, then switch to including the `TextVectorization` layer inside your model when you're ready to prepare for deployment.\n", + "\n", + "Visit the [Save and load models](../keras/save_and_load.ipynb) tutorial to learn more about saving models." ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QTEaNSnLCsv5" + "id": "p4cvuFzavTRy" }, - "outputs": [], "source": [ - "# One or more dense layers.\n", - "# Edit the list in the `for` line to experiment with layer sizes.\n", - "for units in [64, 64]:\n", - " model.add(tf.keras.layers.Dense(units, activation='relu'))\n", + "\n", "\n", - "# Output layer. The first argument is the number of labels.\n", - "model.add(tf.keras.layers.Dense(3, activation='softmax'))" + "## Example 2: Predict the author of Iliad translations" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "zLHPU8q5DLi_" + "id": "fOlJ22508RIe" }, "source": [ - "Finally, compile the model. For a softmax categorization model, use `sparse_categorical_crossentropy` as the loss function. You can try other optimizers, but `adam` is very common." + "The following provides an example of using `tf.data.TextLineDataset` to load examples from text files, and [TensorFlow Text](https://www.tensorflow.org/text) to preprocess the data. You will use three different English translations of the same work, Homer's Iliad, and train a model to identify the translator given a single line of text." ] }, { - "cell_type": "code", - "execution_count": 0, + "cell_type": "markdown", "metadata": { - "colab": {}, - "colab_type": "code", - "id": "pkTBUVO4h6Y5" + "id": "cu2ruJMG9vnY" }, - "outputs": [], "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" + "To implement this task you'll use some lower level tools.\n", + "\n", + "* You'll use `tf.data.TextLineDataset` to load text-lines from files.\n", + "* You'll implement your own version of `keras.layers.TextVectorization` using:\n", + " * `text.UnicodeScriptTokenizer` - to convert strings to tokens.\n", + " * `tf.lookup.StaticVocabularyTable` - to convert tokens to integer ids.\n", + "* You'll maximize performance by placing the text processing in the dataset pipeline, so it can run in parallel with the model training." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "DM-HLo5NDhql" + "id": "-pCgKbOSk7kU" }, "source": [ - "## Train the model\n", + "### Download and explore the dataset\n", "\n", - "This model running on this data produces decent results (about 83%)." + "The texts of the three translations are by:\n", + "\n", + "- [William Cowper](https://en.wikipedia.org/wiki/William_Cowper): [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/cowper.txt)\n", + "- [Edward, Earl of Derby](https://en.wikipedia.org/wiki/Edward_Smith-Stanley,_14th_Earl_of_Derby): [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/derby.txt)\n", + "- [Samuel Butler](https://en.wikipedia.org/wiki/Samuel_Butler_%28novelist%29): [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/butler.txt)\n", + "\n", + "The text files used in this tutorial have undergone some typical preprocessing tasks like removing document header and footer, line numbers and chapter titles.\n", + "\n", + "Download these lightly munged files locally:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aLtO33tNh6V8" + "id": "4YlKQthEYlFw" }, "outputs": [], "source": [ - "model.fit(train_data, epochs=3, validation_data=test_data)" + "DIRECTORY_URL = 'https://storage.googleapis.com/download.tensorflow.org/data/illiad/'\n", + "FILE_NAMES = ['cowper.txt', 'derby.txt', 'butler.txt']\n", + "\n", + "for name in FILE_NAMES:\n", + " text_dir = utils.get_file(name, origin=DIRECTORY_URL + name)\n", + "\n", + "parent_dir = pathlib.Path(text_dir).parent\n", + "list(parent_dir.iterdir())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M8PHK5J_cXE5" + }, + "source": [ + "### Load the dataset\n", + "\n", + "Previously, with `tf.keras.utils.text_dataset_from_directory` all contents of a file were treated as a single example. Here, you will use `tf.data.TextLineDataset`, which is designed to create a `tf.data.Dataset` from a text file where each example is a line of text from the original file. `TextLineDataset` is useful for text data that is primarily line-based (for example, poetry or error logs).\n", + "\n", + "Iterate through these files, loading each one into its own dataset. Each example needs to be individually labeled, so use `Dataset.map` to apply a labeler function to each one. This will iterate over every example in the dataset, returning (`example, label`) pairs." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KTPCYf_Jh6TH" + "id": "YIIWIdPXgk7I" }, "outputs": [], "source": [ - "eval_loss, eval_acc = model.evaluate(test_data)\n", - "\n", - "print('\\nEval loss: {:.3f}, Eval accuracy: {:.3f}'.format(eval_loss, eval_acc))" + "def labeler(example, index):\n", + " return example, tf.cast(index, tf.int64)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Q7IWg9UBNUNM" + "id": "8Ajx7AmZnEg3" }, "outputs": [], "source": [ - "" + "labeled_data_sets = []\n", + "\n", + "for i, file_name in enumerate(FILE_NAMES):\n", + " lines_dataset = tf.data.TextLineDataset(str(parent_dir/file_name))\n", + " labeled_dataset = lines_dataset.map(lambda ex: labeler(ex, i))\n", + " labeled_data_sets.append(labeled_dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wPOsVK1e9NGM" + }, + "source": [ + "Next, you'll combine these labeled datasets into a single dataset using `Dataset.concatenate`, and shuffle it with `Dataset.shuffle`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6jAeYkTIi9-2" + }, + "outputs": [], + "source": [ + "BUFFER_SIZE = 50000\n", + "BATCH_SIZE = 64\n", + "VALIDATION_SIZE = 5000" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Qd544E-Sh63L" + }, + "outputs": [], + "source": [ + "all_labeled_data = labeled_data_sets[0]\n", + "for labeled_dataset in labeled_data_sets[1:]:\n", + " all_labeled_data = all_labeled_data.concatenate(labeled_dataset)\n", + "\n", + "all_labeled_data = all_labeled_data.shuffle(\n", + " BUFFER_SIZE, reshuffle_each_iteration=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "r4JEHrJXeG5k" + }, + "source": [ + "Print out a few examples as before. The dataset hasn't been batched yet, hence each entry in `all_labeled_data` corresponds to one data point:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gywKlN0xh6u5" + }, + "outputs": [], + "source": [ + "for text, label in all_labeled_data.take(10):\n", + " print(\"Sentence: \", text.numpy())\n", + " print(\"Label:\", label.numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5rrpU2_sfDh0" + }, + "source": [ + "### Prepare the dataset for training\n", + "\n", + "Instead of using `tf.keras.layers.TextVectorization` to preprocess the text dataset, you will now use the lower-level TensorFlow Text APIs to standardize and tokenize the data, build a vocabulary and use `tf.lookup.StaticVocabularyTable` to map tokens to integers to feed to the model. (Learn more about [TensorFlow Text](https://www.tensorflow.org/text)).\n", + "\n", + "Define a function to convert the text to lower-case and tokenize it:\n", + "\n", + "- TensorFlow Text provides various tokenizers. In this example, you will use the `text.UnicodeScriptTokenizer` to tokenize the dataset.\n", + "- You will use `Dataset.map` to apply the tokenization to the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gX6opNxnjgkH" + }, + "outputs": [], + "source": [ + "class MyTokenizer(tf.keras.layers.Layer):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.tokenizer = tf_text.UnicodeScriptTokenizer()\n", + "\n", + " def call(self, text):\n", + " lower_case = tf_text.case_fold_utf8(text)\n", + " result = self.tokenizer.tokenize(lower_case)\n", + " # If you pass a batch of strings, it will return a RaggedTensor.\n", + " if isinstance(result, tf.RaggedTensor):\n", + " # Convert to dense 0-padded.\n", + " result = result.to_tensor()\n", + " return result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "v4DpQW-Y12rm" + }, + "outputs": [], + "source": [ + "tokenizer = MyTokenizer()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vzUrAzOq31QL" + }, + "outputs": [], + "source": [ + "tokenized_ds = all_labeled_data.map(lambda text, label: (tokenizer(text), label))\n", + "tokenized_ds" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jx4Q2i8XLV7o" + }, + "source": [ + "You can iterate over the dataset and print out a few tokenized examples:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "S0cWZcWjkQAc" + }, + "outputs": [], + "source": [ + "for tokens, label in tokenized_ds.take(1):\n", + " break\n", + "\n", + "print(tokens)\n", + "print()\n", + "print(label)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JPd4PsskJ_Xt" + }, + "source": [ + "Next, you will build a vocabulary by sorting tokens by frequency and keeping the top `VOCAB_SIZE` tokens:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FqYVv8LMnHKW" + }, + "outputs": [], + "source": [ + "tokenized_ds = tokenized_ds.cache().prefetch(tf.data.AUTOTUNE)\n", + "\n", + "vocab_count = collections.Counter()\n", + "for toks, labels in tokenized_ds.ragged_batch(1000):\n", + " toks = tf.reshape(toks, [-1])\n", + " for tok in toks.numpy():\n", + " vocab_count[tok] += 1\n", + "\n", + "vocab = [tok for tok, count in vocab_count.most_common(VOCAB_SIZE)]\n", + "\n", + "print(\"First five vocab entries:\", vocab[:5])\n", + "print()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PyKSsaNAKi17" + }, + "source": [ + "To convert the tokens into integers, use the `vocab` set to create a `tf.lookup.StaticVocabularyTable`. You will map tokens to integers with `0` reserved for padding, and `n+1` reserved to denote an out-of-vocabulary (OOV) token." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UYB3t-jUhdYB" + }, + "outputs": [], + "source": [ + "class MyVocabTable(tf.keras.layers.Layer):\n", + " def __init__(self, vocab):\n", + " super().__init__()\n", + " self.keys = [''] + vocab\n", + " self.values = range(len(self.keys))\n", + "\n", + " self.init = tf.lookup.KeyValueTensorInitializer(\n", + " self.keys, self.values, key_dtype=tf.string, value_dtype=tf.int64)\n", + "\n", + " num_oov_buckets = 1\n", + "\n", + " self.table = tf.lookup.StaticVocabularyTable(self.init, num_oov_buckets)\n", + "\n", + " def call(self, x):\n", + " result = self.table.lookup(x)\n", + " return result" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "laUBoirZ55Sp" + }, + "source": [ + "Try it on a dummy vocabulary:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kCBo2yFHD7y62" + }, + "outputs": [], + "source": [ + "vocab_table = MyVocabTable(['a','b','c'])\n", + "vocab_table(tf.constant([''] + list('abcdefghi')))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yvXh1oGo6CEq" + }, + "source": [ + "Create one for the real vocabulary:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zPsRw1UD6u-q" + }, + "outputs": [], + "source": [ + "vocab_table = MyVocabTable(vocab)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z5F-EiBpOADE" + }, + "source": [ + "Finally, define a layer to standardize, tokenize and vectorize the dataset using the tokenizer and lookup table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8ClQM4xxpo-s" + }, + "outputs": [], + "source": [ + "preprocess_text = tf.keras.Sequential([\n", + " tokenizer,\n", + " vocab_table\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v6S5Qyabi-vo" + }, + "source": [ + "You can try this on a single example to print the output:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jgxPZaxUuTbk" + }, + "outputs": [], + "source": [ + "example_text, example_label = next(iter(all_labeled_data))\n", + "print(\"Sentence: \", example_text.numpy())\n", + "vectorized_text = preprocess_text(example_text)\n", + "print(\"Vectorized sentence: \", vectorized_text.numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p9qHM0v8k_Mg" + }, + "source": [ + "Now create a dataset pipeline that will process the text on the fly using `Dataset.map`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KmQVsAgJ-RM0" + }, + "outputs": [], + "source": [ + "all_encoded_data = all_labeled_data.map(lambda text, labels:(preprocess_text(text), labels))\n", + "\n", + "for ids, label in all_encoded_data.take(1):\n", + " break\n", + "\n", + "print(\"Ids: \", ids.numpy())\n", + "print(\"Label: \", label.numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_YZToSXSm0qr" + }, + "source": [ + "### Split the dataset into training and test sets\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "itxIJwkrUXgv" + }, + "source": [ + "The Keras `TextVectorization` layer also batches and pads the vectorized data. Padding is required because the examples inside of a batch need to be the same size and shape, but the examples in these datasets are not all the same size—each line of text has a different number of words.\n", + "\n", + "`tf.data.Dataset` supports splitting and padded-batching datasets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "r-rmbijQh6bf" + }, + "outputs": [], + "source": [ + "train_data = all_encoded_data.skip(VALIDATION_SIZE).shuffle(BUFFER_SIZE).padded_batch(BATCH_SIZE)\n", + "validation_data = all_encoded_data.take(VALIDATION_SIZE).padded_batch(BATCH_SIZE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "m-wmFq8uW1zS" + }, + "source": [ + "Now, `validation_data` and `train_data` are not collections of (`example, label`) pairs, but collections of batches. Each batch is a pair of (*many examples*, *many labels*) represented as arrays.\n", + "\n", + "To illustrate this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kMslWfuwoqpB" + }, + "outputs": [], + "source": [ + "sample_text, sample_labels = next(iter(validation_data))\n", + "print(\"Text batch shape: \", sample_text.shape)\n", + "print(\"Label batch shape: \", sample_labels.shape)\n", + "print(\"First text example: \", sample_text[0])\n", + "print(\"First label example: \", sample_labels[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h44Ox11OYLP-" + }, + "source": [ + "Configure the datasets for better performance as before:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BpT0b_7mYRXV" + }, + "outputs": [], + "source": [ + "train_data = train_data.prefetch(tf.data.AUTOTUNE)\n", + "validation_data = validation_data.prefetch(tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K8SUhGFNsmRi" + }, + "source": [ + "### Train the model\n", + "\n", + "You can train a model on this dataset as before:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UI4I6_Sa0vWu" + }, + "source": [ + "Since this text vectorization adds `0` for padding and `n+1` for out-of-vocabulary (OOV) tokens, the vocabulary size has increased by two:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ClUFxHh47dtJ" + }, + "outputs": [], + "source": [ + "model = create_model(vocab_size=VOCAB_SIZE+2, num_labels=3)\n", + "\n", + "model.compile(\n", + " optimizer='adam',\n", + " loss=losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xHpqtP6b7fPl" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(model, show_shapes=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QJgI1pow2YR9" + }, + "outputs": [], + "source": [ + "history = model.fit(train_data, validation_data=validation_data, epochs=3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KTPCYf_Jh6TH" + }, + "outputs": [], + "source": [ + "metrics = model.evaluate(validation_data, return_dict=True)\n", + "\n", + "print(\"Loss: \", metrics['loss'])\n", + "print(\"Accuracy: {:2.2%}\".format(metrics['accuracy']))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_knIsO-r4pHb" + }, + "source": [ + "### Export the model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FEuMLJA_Xiwo" + }, + "source": [ + "To make the model capable of taking raw strings as input, pack both the text processor and the model into a `keras.Sequential`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ZpFRX39rt6UL" + }, + "outputs": [], + "source": [ + "export_model = tf.keras.Sequential([\n", + " preprocess_text,\n", + " model\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "G-Cvd27y4qwt" + }, + "outputs": [], + "source": [ + "export_model.compile(\n", + " loss=losses.SparseCategoricalCrossentropy(from_logits=False),\n", + " optimizer='adam',\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AmU7iV9vtnfY" + }, + "source": [ + "This model you can run directly on batches of strings:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UVwz4FADul8p" + }, + "outputs": [], + "source": [ + "# Create a test dataset of raw strings.\n", + "test_ds = all_labeled_data.take(VALIDATION_SIZE).batch(BATCH_SIZE)\n", + "test_ds = test_ds.cache().prefetch(tf.data.AUTOTUNE)\n", + "test_ds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Pyg0B4zsc-UD" + }, + "outputs": [], + "source": [ + "loss, accuracy = export_model.evaluate(test_ds)\n", + "\n", + "print(\"Loss: \", loss)\n", + "print(\"Accuracy: {:2.2%}\".format(accuracy))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jenLg-ANtusp" + }, + "source": [ + "Use `saved_model.save` to export it.\n", + "\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RrxPxqXYwp5t" + }, + "outputs": [], + "source": [ + "tf.saved_model.save(export_model, 'export.tf')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wdpOqe2nxTDP" + }, + "outputs": [], + "source": [ + "loaded = tf.saved_model.load('export.tf')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sFvoII8lnpyi" + }, + "outputs": [], + "source": [ + "export_model(tf.constant(['The field bristled with the long and deadly spears which they bore.'])).numpy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nig380T3siW_" + }, + "outputs": [], + "source": [ + "loaded(tf.constant(['The field bristled with the long and deadly spears which they bore.'])).numpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o6Mm0Y9QYQwE" + }, + "source": [ + "The loss and accuracy for the model on encoded validation set and the exported model on the raw validation set are the same, as expected." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Stk2BP8GE-qo" + }, + "source": [ + "### Run inference on new data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-w1fQGJPD2Yh" + }, + "outputs": [], + "source": [ + "inputs = [\n", + " \"Join'd to th' Ionians with their flowing robes,\", # Label: 1\n", + " \"the allies, and his armour flashed about him so that he seemed to all\", # Label: 2\n", + " \"And with loud clangor of his arms he fell.\", # Label: 0\n", + "]\n", + "\n", + "predicted_scores = export_model.predict(inputs)\n", + "predicted_labels = tf.math.argmax(predicted_scores, axis=1)\n", + "\n", + "for input, label in zip(inputs, predicted_labels):\n", + " print(\"Question: \", input)\n", + " print(\"Predicted label: \", label.numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9eA8TVdnA-3L" + }, + "source": [ + "## Download more datasets using TensorFlow Datasets (TFDS)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2QFSxfZ3Vqsn" + }, + "source": [ + "You can download many more datasets from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview).\n", + "\n", + "In this example, you will use the [IMDB Large Movie Review dataset](https://www.tensorflow.org/datasets/catalog/imdb_reviews) to train a model for sentiment classification:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NzC65LOaVw0B" + }, + "outputs": [], + "source": [ + "# Training set.\n", + "train_ds = tfds.load(\n", + " 'imdb_reviews',\n", + " split='train[:80%]',\n", + " batch_size=BATCH_SIZE,\n", + " shuffle_files=True,\n", + " as_supervised=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XKGkgPBkFh0k" + }, + "outputs": [], + "source": [ + "# Validation set.\n", + "val_ds = tfds.load(\n", + " 'imdb_reviews',\n", + " split='train[80%:]',\n", + " batch_size=BATCH_SIZE,\n", + " shuffle_files=True,\n", + " as_supervised=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BQjf3YZAb5Ne" + }, + "source": [ + "Print a few examples:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Bq1w8MnfWt2C" + }, + "outputs": [], + "source": [ + "for review_batch, label_batch in val_ds.take(1):\n", + " for i in range(5):\n", + " print(\"Review: \", review_batch[i].numpy())\n", + " print(\"Label: \", label_batch[i].numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "q-lVaukyb75k" + }, + "source": [ + "You can now preprocess the data and train a model as before.\n", + "\n", + "Note: You will use `tf.keras.losses.BinaryCrossentropy` instead of `tf.keras.losses.SparseCategoricalCrossentropy` for your model, since this is a binary classification problem." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ciz2CxAsZw3Z" + }, + "source": [ + "### Prepare the dataset for training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UzT_t9ihZLH4" + }, + "outputs": [], + "source": [ + "vectorize_layer = TextVectorization(\n", + " max_tokens=VOCAB_SIZE,\n", + " output_mode='int',\n", + " output_sequence_length=MAX_SEQUENCE_LENGTH)\n", + "\n", + "# Make a text-only dataset (without labels), then call `TextVectorization.adapt`.\n", + "train_text = train_ds.map(lambda text, labels: text)\n", + "vectorize_layer.adapt(train_text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zz-Xrd_ZZ4tB" + }, + "outputs": [], + "source": [ + "def vectorize_text(text, label):\n", + " text = tf.expand_dims(text, -1)\n", + " return vectorize_layer(text), label" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ycn0Itd6g5aF" + }, + "outputs": [], + "source": [ + "train_ds = train_ds.map(vectorize_text)\n", + "val_ds = val_ds.map(vectorize_text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jc11jQTlZ5lj" + }, + "outputs": [], + "source": [ + "# Configure datasets for performance as before.\n", + "train_ds = train_ds.cache().prefetch(tf.data.AUTOTUNE)\n", + "val_ds = val_ds.cache().prefetch(tf.data.AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SQzoYkaGZ82Z" + }, + "source": [ + "### Create, configure and train the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "B9IOTLkyZ-a7" + }, + "outputs": [], + "source": [ + "model = create_model(vocab_size=VOCAB_SIZE, num_labels=1)\n", + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xLnDs5dhaBAk" + }, + "outputs": [], + "source": [ + "model.compile(\n", + " loss=losses.BinaryCrossentropy(from_logits=True),\n", + " optimizer='adam',\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rq59QpNzaDMa" + }, + "outputs": [], + "source": [ + "history = model.fit(train_ds, validation_data=val_ds, epochs=3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gCMWCEtyaEbR" + }, + "outputs": [], + "source": [ + "loss, accuracy = model.evaluate(val_ds)\n", + "\n", + "print(\"Loss: \", loss)\n", + "print(\"Accuracy: {:2.2%}\".format(accuracy))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jGtqLXVnaaFy" + }, + "source": [ + "### Export the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yE9WZARZaZr1" + }, + "outputs": [], + "source": [ + "export_model = tf.keras.Sequential(\n", + " [vectorize_layer, model,\n", + " layers.Activation('sigmoid')])\n", + "\n", + "export_model.compile(\n", + " loss=losses.SparseCategoricalCrossentropy(from_logits=False),\n", + " optimizer='adam',\n", + " metrics=['accuracy'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bhF8tDH-afoC" + }, + "outputs": [], + "source": [ + "# 0 --> negative review\n", + "# 1 --> positive review\n", + "inputs = [\n", + " \"This is a fantastic movie.\",\n", + " \"This is a bad movie.\",\n", + " \"This movie was so bad that it was good.\",\n", + " \"I will never say yes to watching this movie.\",\n", + "]\n", + "\n", + "predicted_scores = export_model.predict(inputs)\n", + "predicted_labels = [int(round(x[0])) for x in predicted_scores]\n", + "\n", + "for input, label in zip(inputs, predicted_labels):\n", + " print(\"Question: \", input)\n", + " print(\"Predicted label: \", label)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "q1KSXDFPWiPN" + }, + "source": [ + "## Conclusion\n", + "\n", + "This tutorial demonstrated several ways to load and preprocess text. As a next step, you can explore additional text preprocessing [TensorFlow Text](https://www.tensorflow.org/text) tutorials, such as:\n", + "\n", + "- [BERT Preprocessing with TF Text](https://www.tensorflow.org/text/guide/bert_preprocessing_guide)\n", + "- [Tokenizing with TF Text](https://www.tensorflow.org/text/guide/tokenizers)\n", + "- [Subword tokenizers](https://www.tensorflow.org/text/guide/subwords_tokenizer)\n", + "\n", + "You can also find new datasets on [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). And, to learn more about `tf.data`, check out the guide on [building input pipelines](../../guide/data.ipynb)." ] } ], "metadata": { + "accelerator": "GPU", "colab": { - "collapsed_sections": [], - "name": "text.ipynb", + "gpuType": "T4", "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "provenance": [ + { + "file_id": "1XHr3IzoXn_Si4HSDpY43hdTirjwMDvON", + "timestamp": 1694267031282 + }, + { + "file_id": "https://github.com/tensorflow/docs/blob/4daf8a1645ddb4dd97ef31dca2e456ffaedbc8a1/site/en/tutorials/load_data/text.ipynb", + "timestamp": 1694092268316 + } + ], + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/load_data/tfrecord.ipynb b/site/en/tutorials/load_data/tfrecord.ipynb index 8f2eb2b726e..905e1a3745b 100644 --- a/site/en/tutorials/load_data/tfrecord.ipynb +++ b/site/en/tutorials/load_data/tfrecord.ipynb @@ -1,39 +1,8 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "tfrecord.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "pL--_KGdYoBz" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - } - }, "cells": [ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pL--_KGdYoBz" }, "source": [ @@ -42,12 +11,12 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "cellView": "form", - "colab_type": "code", - "id": "uBDvXpYzYnGj", - "colab": {} + "id": "uBDvXpYzYnGj" }, + "outputs": [], "source": [ "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -60,18 +29,15 @@ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", "# See the License for the specific language governing permissions and\n", "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "HQzaEQuJiW_d" }, "source": [ - "# TFRecord and tf.Example\n", + "# TFRecord and tf.train.Example\n", "\n", "\n", " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + +
    + + + + +
    +

    Nested Classes

    +
    \n", @@ -92,38 +58,34 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3pkUd_9IZCFO" }, "source": [ - "To read data efficiently it can be helpful to serialize your data and store it in a set of files (100-200MB each) that can each be read linearly. This is especially true if the data is being streamed over a network. This can also be useful for caching any data-preprocessing.\n", - "\n", "The TFRecord format is a simple format for storing a sequence of binary records.\n", "\n", "[Protocol buffers](https://developers.google.com/protocol-buffers/) are a cross-platform, cross-language library for efficient serialization of structured data.\n", "\n", "Protocol messages are defined by `.proto` files, these are often the easiest way to understand a message type.\n", "\n", - "The `tf.Example` message (or protobuf) is a flexible message type that represents a `{\"string\": value}` mapping. It is designed for use with TensorFlow and is used throughout the higher-level APIs such as [TFX](https://www.tensorflow.org/tfx/)." + "The `tf.train.Example` message (or protobuf) is a flexible message type that represents a `{\"string\": value}` mapping. It is designed for use with TensorFlow and is used throughout the higher-level APIs such as [TFX](https://www.tensorflow.org/tfx/)." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Ac83J0QxjhFt" }, "source": [ + "This notebook demonstrates how to create, parse, and use the `tf.train.Example` message, and then serialize, write, and read `tf.train.Example` messages to and from `.tfrecord` files.\n", "\n", - "This notebook will demonstrate how to create, parse, and use the `tf.Example` message, and then serialize, write, and read `tf.Example` messages to and from `.tfrecord` files.\n", + "Note: While useful, these structures are optional. There is no need to convert existing code to use TFRecords, unless you are [using tf.data](https://www.tensorflow.org/guide/data) and reading data is still the bottleneck to training. You can refer to [Better performance with the tf.data API](https://www.tensorflow.org/guide/data_performance) for dataset performance tips.\n", "\n", - "Note: While useful, these structures are optional. There is no need to convert existing code to use TFRecords, unless you are using [`tf.data`](https://www.tensorflow.org/guide/datasets) and reading data is still the bottleneck to training. See [Data Input Pipeline Performance](https://www.tensorflow.org/guide/performance/datasets) for dataset performance tips." + "Note: In general, you should shard your data across multiple files so that you can parallelize I/O (within a single host or across multiple hosts). The rule of thumb is to have at least 10 times as many files as there will be hosts reading data. At the same time, each file should be large enough (at least 10 MB+ and ideally 100 MB+) so that you can benefit from I/O prefetching. For example, say you have `X` GB of data and you plan to train on up to `N` hosts. Ideally, you should shard the data to ~`10*N` files, as long as ~`X/(10*N)` is 10 MB+ (and ideally 100 MB+). If it is less than that, you might need to create fewer shards to trade off parallelism benefits and I/O prefetching benefits." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WkRreBf1eDVc" }, "source": [ @@ -132,55 +94,43 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Ja7sezsmnXph", - "colab": {} + "id": "Ja7sezsmnXph" }, + "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " !pip install tf-nightly\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", "\n", "import numpy as np\n", "import IPython.display as display" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "e5Kq88ccUWQV" }, "source": [ - "## `tf.Example`" + "## `tf.train.Example`" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VrdQHgvNijTi" }, "source": [ - "### Data types for `tf.Example`" + "### Data types for `tf.train.Example`" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "lZw57Qrn4CTE" }, "source": [ - "Fundamentally, a `tf.Example` is a `{\"string\": tf.train.Feature}` mapping.\n", + "Fundamentally, a `tf.train.Example` is a `{\"string\": tf.train.Feature}` mapping.\n", "\n", "The `tf.train.Feature` message type can accept one of the following three types (See the [`.proto` file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto) for reference). Most other generic types can be coerced into one of these:\n", "\n", @@ -207,23 +157,22 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_e3g9ExathXP" }, "source": [ - "In order to convert a standard TensorFlow type to a `tf.Example`-compatible `tf.train.Feature`, you can use the shortcut functions below. Note that each function takes a scalar input value and returns a `tf.train.Feature` containing one of the three `list` types above:" + "In order to convert a standard TensorFlow type to a `tf.train.Example`-compatible `tf.train.Feature`, you can use the shortcut functions below. Note that each function takes a scalar input value and returns a `tf.train.Feature` containing one of the three `list` types above:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "mbsPOUpVtYxA", - "colab": {} + "id": "mbsPOUpVtYxA" }, + "outputs": [], "source": [ "# The following functions can be used to convert a value to a type compatible\n", - "# with tf.Example.\n", + "# with tf.train.Example.\n", "\n", "def _bytes_feature(value):\n", " \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n", @@ -238,37 +187,33 @@ "def _int64_feature(value):\n", " \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n", " return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Wst0v9O8hgzy" }, "source": [ - "Note: To stay simple, this example only uses scalar inputs. The simplest way to handle non-scalar features is to use `tf.serialize_tensor` to convert tensors to binary-strings. Strings are scalars in tensorflow. Use `tf.parse_tensor` to convert the binary-string back to a tensor." + "Note: To stay simple, this example only uses scalar inputs. The simplest way to handle non-scalar features is to use `tf.io.serialize_tensor` to convert tensors to binary-strings. Strings are scalars in TensorFlow. Use `tf.io.parse_tensor` to convert the binary-string back to a tensor." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "vsMbkkC8xxtB" }, "source": [ - "Below are some examples of how these functions work. Note the varying input types and the standardized output types. If the input type for a function does not match one of the coercible types stated above, the function will raise an exception (e.g. `_int64_feature(1.0)` will error out, since `1.0` is a float, so should be used with the `_float_feature` function instead):" + "Below are some examples of how these functions work. Note the varying input types and the standardized output types. If the input type for a function does not match one of the coercible types stated above, the function will raise an exception (e.g. `_int64_feature(1.0)` will error out because `1.0` is a float—therefore, it should be used with the `_float_feature` function instead):" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "hZzyLGr0u73y", - "colab": {} + "id": "hZzyLGr0u73y" }, + "outputs": [], "source": [ "print(_bytes_feature(b'test_string'))\n", "print(_bytes_feature(u'test_bytes'.encode('utf-8')))\n", @@ -277,14 +222,11 @@ "\n", "print(_int64_feature(True))\n", "print(_int64_feature(1))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "nj1qpfQU5qmi" }, "source": [ @@ -293,37 +235,33 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "5afZkORT5pjm", - "colab": {} + "id": "5afZkORT5pjm" }, + "outputs": [], "source": [ "feature = _float_feature(np.exp(1))\n", "\n", "feature.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "laKnw9F3hL-W" }, "source": [ - "### Creating a `tf.Example` message" + "### Creating a `tf.train.Example` message" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "b_MEnhxchQPC" }, "source": [ - "Suppose you want to create a `tf.Example` message from existing data. In practice, the dataset may come from anywhere, but the procedure of creating the `tf.Example` message from a single observation will be the same:\n", + "Suppose you want to create a `tf.train.Example` message from existing data. In practice, the dataset may come from anywhere, but the procedure of creating the `tf.train.Example` message from a single observation will be the same:\n", "\n", "1. Within each observation, each value needs to be converted to a `tf.train.Feature` containing one of the 3 compatible types, using one of the functions above.\n", "\n", @@ -335,7 +273,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4EgFQ2uHtchc" }, "source": [ @@ -353,11 +290,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "CnrguFAy3YQv", - "colab": {} + "id": "CnrguFAy3YQv" }, + "outputs": [], "source": [ "# The number of observations in the dataset.\n", "n_observations = int(1e4)\n", @@ -368,39 +305,37 @@ "# Integer feature, random from 0 to 4.\n", "feature1 = np.random.randint(0, 5, n_observations)\n", "\n", - "# String feature\n", + "# String feature.\n", "strings = np.array([b'cat', b'dog', b'chicken', b'horse', b'goat'])\n", "feature2 = strings[feature1]\n", "\n", - "# Float feature, from a standard normal distribution\n", + "# Float feature, from a standard normal distribution.\n", "feature3 = np.random.randn(n_observations)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "aGrscehJr7Jd" }, "source": [ - "Each of these features can be coerced into a `tf.Example`-compatible type using one of `_bytes_feature`, `_float_feature`, `_int64_feature`. You can then create a `tf.Example` message from these encoded features:" + "Each of these features can be coerced into a `tf.train.Example`-compatible type using one of `_bytes_feature`, `_float_feature`, `_int64_feature`. You can then create a `tf.train.Example` message from these encoded features:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "RTCS49Ij_kUw", - "colab": {} + "id": "RTCS49Ij_kUw" }, + "outputs": [], "source": [ + "@tf.py_function(Tout=tf.string)\n", "def serialize_example(feature0, feature1, feature2, feature3):\n", " \"\"\"\n", - " Creates a tf.Example message ready to be written to a file.\n", + " Creates a tf.train.Example message ready to be written to a file.\n", " \"\"\"\n", - " # Create a dictionary mapping the feature name to the tf.Example-compatible\n", + " # Create a dictionary mapping the feature name to the tf.train.Example-compatible\n", " # data type.\n", " feature = {\n", " 'feature0': _int64_feature(feature0),\n", @@ -413,42 +348,36 @@ "\n", " example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n", " return example_proto.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XftzX9CN_uGT" }, "source": [ - "For example, suppose you have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. You can create and print the `tf.Example` message for this observation using `create_message()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.Example` [message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message:" + "For example, suppose you have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. You can create and print the `tf.train.Example` message for this observation using `serialize_example()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.train.Example` [message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "N8BtSx2RjYcb", - "colab": {} + "id": "N8BtSx2RjYcb" }, + "outputs": [], "source": [ "# This is an example observation from the dataset.\n", "\n", - "example_observation = []\n", - "\n", - "serialized_example = serialize_example(False, 4, b'goat', 0.9876)\n", + "example_observation = [False, 4, b'goat', 0.9876]\n", + "serialized_example = serialize_example(*example_observation)\n", "serialized_example" - ], - "execution_count": 0, - "outputs": [] + + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_pbGATlG6u-4" }, "source": [ @@ -457,22 +386,19 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "dGim-mEm6vit", - "colab": {} + "id": "dGim-mEm6vit" }, + "outputs": [], "source": [ - "example_proto = tf.train.Example.FromString(serialized_example)\n", + "example_proto = tf.train.Example.FromString(serialized_example.numpy())\n", "example_proto" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "o6qxofy89obI" }, "source": [ @@ -480,7 +406,7 @@ "\n", "A TFRecord file contains a sequence of records. The file can only be read sequentially.\n", "\n", - "Each record contains a byte-string, for the data-payload, plus the data-length, and CRC32C (32-bit CRC using the Castagnoli polynomial) hashes for integrity checking.\n", + "Each record contains a byte-string, for the data-payload, plus the data-length, and CRC-32C ([32-bit CRC](https://en.wikipedia.org/wiki/Cyclic_redundancy_check#CRC-32_algorithm) using the [Castagnoli polynomial](https://en.wikipedia.org/wiki/Cyclic_redundancy_check#Standards_and_common_use)) hashes for integrity checking.\n", "\n", "Each record is stored in the following formats:\n", "\n", @@ -493,280 +419,205 @@ "[described here](https://en.wikipedia.org/wiki/Cyclic_redundancy_check), and\n", "the mask of a CRC is:\n", "\n", - " masked_crc = ((crc >> 15) | (crc << 17)) + 0xa282ead8ul\n", - "\n", - "Note: There is no requirement to use `tf.Example` in TFRecord files. `tf.Example` is just a method of serializing dictionaries to byte-strings. Lines of text, encoded image data, or serialized tensors (using `tf.io.serialize_tensor`, and\n", - "`tf.io.parse_tensor` when loading). See the `tf.io` module for more options." + " masked_crc = ((crc >> 15) | (crc << 17)) + 0xa282ead8ul\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "y-Hjmee-fbLH" + "id": "-0iHagLQCJv6" }, "source": [ - "## TFRecord files using `tf.data`" + "Note: There is no requirement to use `tf.train.Example` in TFRecord files. `tf.train.Example` is just a method of serializing dictionaries to byte-strings. Any byte-string that can be decoded in TensorFlow could be stored in a TFRecord file. Examples include: lines of text, JSON (using `tf.io.decode_json_example`), encoded image data, or serialized `tf.Tensors` (using `tf.io.serialize_tensor`/`tf.io.parse_tensor`). See the `tf.io` module for more options." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "GmehkCCT81Ez" + "id": "jyg1g3gU7DNn" }, "source": [ - "The `tf.data` module also provides tools for reading and writing data in TensorFlow." + "## Reading and writing TFRecord files" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "1FISEuz8ubu3" + "id": "3FXG3miA7Kf1" }, "source": [ - "### Writing a TFRecord file\n", - "\n", - "The easiest way to get the data into a dataset is to use the `from_tensor_slices` method.\n", - "\n", - "Applied to an array, it returns a dataset of scalars:" + "The `tf.io` module also contains pure-Python functions for reading and writing TFRecord files." ] }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mXeaukvwu5_-", - "colab": {} - }, - "source": [ - "tf.data.Dataset.from_tensor_slices(feature1)" - ], - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "f-q0VKyZvcad" + "id": "CKn5uql2lAaN" }, "source": [ - "Applies to a tuple of arrays, it returns a dataset of tuples:" + "### Writing a TFRecord file" ] }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "H5sWyu1kxnvg", - "colab": {} - }, - "source": [ - "features_dataset = tf.data.Dataset.from_tensor_slices((feature0, feature1, feature2, feature3))\n", - "features_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m1C-t71Nywze", - "colab": {} - }, - "source": [ - "# Use `take(1)` to only pull one example from the dataset.\n", - "for f0,f1,f2,f3 in features_dataset.take(1):\n", - " print(f0)\n", - " print(f1)\n", - " print(f2)\n", - " print(f3)" - ], - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "mhIe63awyZYd" + "id": "LNW_FA-GQWXs" }, "source": [ - "Use the `tf.data.Dataset.map` method to apply a function to each element of a `Dataset`.\n", - "\n", - "The mapped function must operate in TensorFlow graph mode—it must operate on and return `tf.Tensors`. A non-tensor function, like `serialize_example`, can be wrapped with `tf.py_function` to make it compatible.\n", - "\n", - "Using `tf.py_function` requires to specify the shape and type information that is otherwise unavailable:" + "Next, write the 10,000 observations to the file `test.tfrecord`. Each observation is converted to a `tf.train.Example` message, then written to file. You can then verify that the file `test.tfrecord` has been created:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "apB5KYrJzjPI", - "colab": {} + "id": "gxB_cwlN0DLy" }, + "outputs": [], "source": [ - "def tf_serialize_example(f0,f1,f2,f3):\n", - " tf_string = tf.py_function(\n", - " serialize_example,\n", - " (f0,f1,f2,f3), # pass these args to the above function.\n", - " tf.string) # the return type is `tf.string`.\n", - " return tf.reshape(tf_string, ()) # The result is a scalar" - ], - "execution_count": 0, - "outputs": [] + "filename = 'test.tfrecord'" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "lHFjW4u4Npz9", - "colab": {} - }, - "source": [ - "tf_serialize_example(f0,f1,f2,f3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CrFZ9avE3HUF" + "id": "MKPHzoGv7q44" }, + "outputs": [], "source": [ - "Apply this function to each element in the dataset:" + "# Write the `tf.train.Example` observations to the file.\n", + "with tf.io.TFRecordWriter(filename) as writer:\n", + " for i in range(n_observations):\n", + " example = serialize_example(feature0[i], feature1[i], feature2[i], feature3[i])\n", + " writer.write(example.numpy())" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "VDeqYVbW3ww9", - "colab": {} + "id": "EjdFHHJMpUUo" }, + "outputs": [], "source": [ - "serialized_features_dataset = features_dataset.map(tf_serialize_example)\n", - "serialized_features_dataset" - ], - "execution_count": 0, - "outputs": [] + "!du -sh {filename}" + ] }, { - "cell_type": "code", + "cell_type": "markdown", "metadata": { - "colab_type": "code", - "id": "DlDfuh46bRf6", - "colab": {} + "id": "2osVRnYNni-E" }, "source": [ - "def generator():\n", - " for features in features_dataset:\n", - " yield serialize_example(*features)" - ], - "execution_count": 0, - "outputs": [] + "### Reading a TFRecord file in python\n", + "\n", + "These serialized tensors can be easily parsed using `tf.train.Example.ParseFromString`:" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "iv9oXKrcbhvX", - "colab": {} + "id": "U3tnd3LerOtV" }, + "outputs": [], "source": [ - "serialized_features_dataset = tf.data.Dataset.from_generator(\n", - " generator, output_types=tf.string, output_shapes=())" - ], - "execution_count": 0, - "outputs": [] + "filenames = [filename]\n", + "raw_dataset = tf.data.TFRecordDataset(filenames)\n", + "raw_dataset" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "Dqz8C4D5cIj9", - "colab": {} + "id": "nsEAACHcnm3f" }, + "outputs": [], "source": [ - "serialized_features_dataset" - ], - "execution_count": 0, - "outputs": [] + "for raw_record in raw_dataset.take(1):\n", + " example = tf.train.Example()\n", + " example.ParseFromString(raw_record.numpy())\n", + " print(example)" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "p6lw5VYpjZZC" + "id": "yhnZZmhm1miG" }, "source": [ - "And write them to a TFRecord file:" + "That returns a `tf.train.Example` proto which is dificult to use as is, but it's fundamentally a representation of a:\n", + "\n", + "```\n", + "Dict[str,\n", + " Union[List[float],\n", + " List[int],\n", + " List[str]]]\n", + "```\n", + "\n", + "The following code manually converts the `Example` to a dictionary of NumPy arrays, without using TensorFlow Ops. Refer to [the PROTO file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto) for details." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "vP1VgTO44UIE", - "colab": {} + "id": "Ziv9tiNE1l6J" }, + "outputs": [], "source": [ - "filename = 'test.tfrecord'\n", - "writer = tf.data.experimental.TFRecordWriter(filename)\n", - "writer.write(serialized_features_dataset)" - ], - "execution_count": 0, - "outputs": [] + "result = {}\n", + "# example.features.feature is the dictionary\n", + "for key, feature in example.features.feature.items():\n", + " # The values are the Feature objects which contain a `kind` which contains:\n", + " # one of three fields: bytes_list, float_list, int64_list\n", + "\n", + " kind = feature.WhichOneof('kind')\n", + " result[key] = np.array(getattr(feature, kind).value)\n", + "\n", + "result" + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6aV0GQhV8tmp" }, "source": [ - "### Reading a TFRecord file" + "### Reading a TFRecord file Using tf.data" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "o3J5D4gcSy8N" }, "source": [ "You can also read the TFRecord file using the `tf.data.TFRecordDataset` class.\n", "\n", - "More information on consuming TFRecord files using `tf.data` can be found [here](https://www.tensorflow.org/guide/datasets#consuming_tfrecord_data).\n", + "More information on consuming TFRecord files using `tf.data` can be found in the [tf.data: Build TensorFlow input pipelines](https://www.tensorflow.org/guide/data#consuming_tfrecord_data) guide.\n", "\n", "Using `TFRecordDataset`s can be useful for standardizing input data and optimizing performance." ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "6OjX6UZl-bHC", - "colab": {} + "id": "6OjX6UZl-bHC" }, + "outputs": [], "source": [ "filenames = [filename]\n", "raw_dataset = tf.data.TFRecordDataset(filenames)\n", "raw_dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6_EQ9i2E_-Fz" }, "source": [ @@ -779,35 +630,32 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "hxVXpLz_AJlm", - "colab": {} + "id": "hxVXpLz_AJlm" }, + "outputs": [], "source": [ "for raw_record in raw_dataset.take(10):\n", " print(repr(raw_record))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "W-6oNzM4luFQ" }, "source": [ - "These tensors can be parsed using the function below. Note that the `feature_description` is necessary here because datasets use graph-execution, and need this description to build their shape and type signature:" + "These tensors can be parsed using the function below. Note that the `feature_description` is necessary here because `tf.data.Dataset`s use graph-execution, and need this description to build their shape and type signature:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "zQjbIR1nleiy", - "colab": {} + "id": "zQjbIR1nleiy" }, + "outputs": [], "source": [ "# Create a description of the features.\n", "feature_description = {\n", @@ -818,40 +666,34 @@ "}\n", "\n", "def _parse_function(example_proto):\n", - " # Parse the input `tf.Example` proto using the dictionary above.\n", + " # Parse the input `tf.train.Example` proto using the dictionary above.\n", " return tf.io.parse_single_example(example_proto, feature_description)" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gWETjUqhEQZf" }, "source": [ - "Alternatively, use `tf.parse example` to parse the whole batch at once. Apply this function to each item in the dataset using the `tf.data.Dataset.map` method:" + "Alternatively, use `tf.parse_example` to parse the whole batch at once. Apply this function to each item in the dataset using the `tf.data.Dataset.map` method:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "6Ob7D-zmBm1w", - "colab": {} + "id": "6Ob7D-zmBm1w" }, + "outputs": [], "source": [ "parsed_dataset = raw_dataset.map(_parse_function)\n", "parsed_dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "sNV-XclGnOvn" }, "source": [ @@ -860,145 +702,28 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "x2LT2JCqhoD_", - "colab": {} + "id": "x2LT2JCqhoD_" }, + "outputs": [], "source": [ "for parsed_record in parsed_dataset.take(10):\n", " print(repr(parsed_record))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cig9EodTlDmg" - }, - "source": [ - "Here, the `tf.parse_example` function unpacks the `tf.Example` fields into standard tensors." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jyg1g3gU7DNn" - }, - "source": [ - "## TFRecord files in Python" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "3FXG3miA7Kf1" - }, - "source": [ - "The `tf.io` module also contains pure-Python functions for reading and writing TFRecord files." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CKn5uql2lAaN" - }, - "source": [ - "### Writing a TFRecord file" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LNW_FA-GQWXs" - }, - "source": [ - "Next, write the 10,000 observations to the file `test.tfrecord`. Each observation is converted to a `tf.Example` message, then written to file. You can then verify that the file `test.tfrecord` has been created:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MKPHzoGv7q44", - "colab": {} - }, - "source": [ - "# Write the `tf.Example` observations to the file.\n", - "with tf.io.TFRecordWriter(filename) as writer:\n", - " for i in range(n_observations):\n", - " example = serialize_example(feature0[i], feature1[i], feature2[i], feature3[i])\n", - " writer.write(example)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EjdFHHJMpUUo", - "colab": {} - }, - "source": [ - "!du -sh {filename}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2osVRnYNni-E" + "id": "Cig9EodTlDmg" }, "source": [ - "### Reading a TFRecord file\n", - "\n", - "These serialized tensors can be easily parsed using `tf.train.Example.ParseFromString`:" + "Here, the `tf.parse_example` function unpacks the `tf.train.Example` fields into standard tensors." ] }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "U3tnd3LerOtV", - "colab": {} - }, - "source": [ - "filenames = [filename]\n", - "raw_dataset = tf.data.TFRecordDataset(filenames)\n", - "raw_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nsEAACHcnm3f", - "colab": {} - }, - "source": [ - "for raw_record in raw_dataset.take(1):\n", - " example = tf.train.Example()\n", - " example.ParseFromString(raw_record.numpy())\n", - " print(example)" - ], - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "S0tFDrwdoj3q" }, "source": [ @@ -1008,7 +733,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rjN2LFxFpcR9" }, "source": [ @@ -1022,7 +746,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5Lk2qrKvN0yu" }, "source": [ @@ -1031,50 +754,48 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "3a0fmwg8lHdF", - "colab": {} + "id": "3a0fmwg8lHdF" }, + "outputs": [], "source": [ - "cat_in_snow = tf.keras.utils.get_file('320px-Felis_catus-cat_on_snow.jpg', 'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg')\n", - "williamsburg_bridge = tf.keras.utils.get_file('194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg')" - ], - "execution_count": 0, - "outputs": [] + "cat_in_snow = tf.keras.utils.get_file(\n", + " '320px-Felis_catus-cat_on_snow.jpg',\n", + " 'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg')\n", + "\n", + "williamsburg_bridge = tf.keras.utils.get_file(\n", + " '194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg',\n", + " 'https://storage.googleapis.com/download.tensorflow.org/example_images/194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg')" + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "7aJJh7vENeE4", - "colab": {} + "id": "7aJJh7vENeE4" }, + "outputs": [], "source": [ "display.display(display.Image(filename=cat_in_snow))\n", "display.display(display.HTML('Image cc-by: Von.grzanka'))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "KkW0uuhcXZqA", - "colab": {} + "id": "KkW0uuhcXZqA" }, + "outputs": [], "source": [ "display.display(display.Image(filename=williamsburg_bridge))\n", "display.display(display.HTML('From Wikimedia'))" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VSOgJSwoN5TQ" }, "source": [ @@ -1084,36 +805,33 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Azx83ryQEU6T" }, "source": [ - "As before, encode the features as types compatible with `tf.Example`. This stores the raw image string feature, as well as the height, width, depth, and arbitrary `label` feature. The latter is used when you write the file to distinguish between the cat image and the bridge image. Use `0` for the cat image, and `1` for the bridge image:" + "As before, encode the features as types compatible with `tf.train.Example`. This stores the raw image string feature, as well as the height, width, depth, and arbitrary `label` feature. The latter is used when you write the file to distinguish between the cat image and the bridge image. Use `0` for the cat image, and `1` for the bridge image:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "kC4TS1ZEONHr", - "colab": {} + "id": "kC4TS1ZEONHr" }, + "outputs": [], "source": [ "image_labels = {\n", " cat_in_snow : 0,\n", " williamsburg_bridge : 1,\n", "}" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "c5njMSYNEhNZ", - "colab": {} + "id": "c5njMSYNEhNZ" }, + "outputs": [], "source": [ "# This is an example, just using the cat image.\n", "image_string = open(cat_in_snow, 'rb').read()\n", @@ -1122,7 +840,7 @@ "\n", "# Create a dictionary with features that may be relevant.\n", "def image_example(image_string, label):\n", - " image_shape = tf.image.decode_jpeg(image_string).shape\n", + " image_shape = tf.io.decode_jpeg(image_string).shape\n", "\n", " feature = {\n", " 'height': _int64_feature(image_shape[0]),\n", @@ -1137,30 +855,27 @@ "for line in str(image_example(image_string, label)).split('\\n')[:15]:\n", " print(line)\n", "print('...')" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2G_o3O9MN0Qx" }, "source": [ - "Notice that all of the features are now stored in the `tf.Example` message. Next, functionalize the code above and write the example messages to a file named `images.tfrecords`:" + "Notice that all of the features are now stored in the `tf.train.Example` message. Next, functionalize the code above and write the example messages to a file named `images.tfrecords`:" ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "qcw06lQCOCZU", - "colab": {} + "id": "qcw06lQCOCZU" }, + "outputs": [], "source": [ "# Write the raw image files to `images.tfrecords`.\n", - "# First, process the two images into `tf.Example` messages.\n", + "# First, process the two images into `tf.train.Example` messages.\n", "# Then, write to a `.tfrecords` file.\n", "record_file = 'images.tfrecords'\n", "with tf.io.TFRecordWriter(record_file) as writer:\n", @@ -1168,27 +883,22 @@ " image_string = open(filename, 'rb').read()\n", " tf_example = image_example(image_string, label)\n", " writer.write(tf_example.SerializeToString())" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "yJrTe6tHPCfs", - "colab": {} + "id": "yJrTe6tHPCfs" }, + "outputs": [], "source": [ "!du -sh {record_file}" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "jJSsCkZLPH6K" }, "source": [ @@ -1199,11 +909,11 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "M6Cnfd3cTKHN", - "colab": {} + "id": "M6Cnfd3cTKHN" }, + "outputs": [], "source": [ "raw_image_dataset = tf.data.TFRecordDataset('images.tfrecords')\n", "\n", @@ -1217,19 +927,16 @@ "}\n", "\n", "def _parse_image_function(example_proto):\n", - " # Parse the input tf.Example proto using the dictionary above.\n", + " # Parse the input tf.train.Example proto using the dictionary above.\n", " return tf.io.parse_single_example(example_proto, image_feature_description)\n", "\n", "parsed_image_dataset = raw_image_dataset.map(_parse_image_function)\n", "parsed_image_dataset" - ], - "execution_count": 0, - "outputs": [] + ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0PEEFPk4NEg1" }, "source": [ @@ -1238,18 +945,33 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { - "colab_type": "code", - "id": "yZf8jOyEIjSF", - "colab": {} + "id": "yZf8jOyEIjSF" }, + "outputs": [], "source": [ "for image_features in parsed_image_dataset:\n", " image_raw = image_features['image_raw'].numpy()\n", " display.display(display.Image(data=image_raw))" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "pL--_KGdYoBz" ], - "execution_count": 0, - "outputs": [] + "name": "tfrecord.ipynb", + "private_outputs": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } - ] + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/site/en/tutorials/load_data/unicode.ipynb b/site/en/tutorials/load_data/unicode.ipynb deleted file mode 100644 index 12151b67f04..00000000000 --- a/site/en/tutorials/load_data/unicode.ipynb +++ /dev/null @@ -1,836 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "unicode.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "oL9KopJirB2g" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oL9KopJirB2g" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "SKaX3Hd3ra6C", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AXH1bmUctMld" - }, - "source": [ - "# Unicode strings\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LrHJrKYis06U" - }, - "source": [ - "## Introduction\n", - "\n", - "Models that process natural language often handle different languages with different character sets. *Unicode* is a standard encoding system that is used to represent character from almost all languages. Each character is encoded using a unique integer [code point](https://en.wikipedia.org/wiki/Code_point) between `0` and `0x10FFFF`. A *Unicode string* is a sequence of zero or more code points.\n", - "\n", - "This tutorial shows how to represent Unicode strings in TensorFlow and manipulate them using Unicode equivalents of standard string ops. It separates Unicode strings into tokens based on script detection." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OIKHl5Lvn4gh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n-LkcI-vtWNj" - }, - "source": [ - "## The `tf.string` data type\n", - "\n", - "The basic TensorFlow `tf.string` `dtype` allows you to build tensors of byte strings.\n", - "Unicode strings are utf-8 encoded by default." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3yo-Qv6ntaFr", - "colab": {} - }, - "source": [ - "tf.constant(u\"Thanks 😊\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2kA1ziG2tyCT" - }, - "source": [ - "A `tf.string` tensor can hold byte strings of varying lengths because the byte strings are treated as atomic units. The string length is not included in the tensor dimensions.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eyINCmTztyyS", - "colab": {} - }, - "source": [ - "tf.constant([u\"You're\", u\"welcome!\"]).shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jsMPnjb6UDJ1" - }, - "source": [ - "Note: When using python to construct strings, the handling of unicode differs betweeen v2 and v3. In v2, unicode strings are indicated by the \"u\" prefix, as above. In v3, strings are unicode-encoded by default." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hUFZ7B1Lk-uj" - }, - "source": [ - "## Representing Unicode\n", - "\n", - "There are two standard ways to represent a Unicode string in TensorFlow:\n", - "\n", - "* `string` scalar — where the sequence of code points is encoded using a known [character encoding](https://en.wikipedia.org/wiki/Character_encoding).\n", - "* `int32` vector — where each position contains a single code point.\n", - "\n", - "For example, the following three values all represent the Unicode string `\"语言处理\"` (which means \"language processing\" in Chinese):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cjQIkfJWvC_u", - "colab": {} - }, - "source": [ - "# Unicode string, represented as a UTF-8 encoded string scalar.\n", - "text_utf8 = tf.constant(u\"语言处理\")\n", - "text_utf8" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yQqcUECcvF2r", - "colab": {} - }, - "source": [ - "# Unicode string, represented as a UTF-16-BE encoded string scalar.\n", - "text_utf16be = tf.constant(u\"语言处理\".encode(\"UTF-16-BE\"))\n", - "text_utf16be" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ExdBr1t7vMuS", - "colab": {} - }, - "source": [ - "# Unicode string, represented as a vector of Unicode code points.\n", - "text_chars = tf.constant([ord(char) for char in u\"语言处理\"])\n", - "text_chars" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B8czv4JNpBnZ" - }, - "source": [ - "### Converting between representations\n", - "\n", - "TensorFlow provides operations to convert between these different representations:\n", - "\n", - "* `tf.strings.unicode_decode`: Converts an encoded string scalar to a vector of code points.\n", - "* `tf.strings.unicode_encode`: Converts a vector of code points to an encoded string scalar.\n", - "* `tf.strings.unicode_transcode`: Converts an encoded string scalar to a different encoding." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qb-UQ_oLpAJg", - "colab": {} - }, - "source": [ - "tf.strings.unicode_decode(text_utf8,\n", - " input_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kEBUcunnp-9n", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(text_chars,\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0MLhWcLZrph-", - "colab": {} - }, - "source": [ - "tf.strings.unicode_transcode(text_utf8,\n", - " input_encoding='UTF8',\n", - " output_encoding='UTF-16-BE')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QVeLeVohqN7I" - }, - "source": [ - "### Batch dimensions\n", - "\n", - "When decoding multiple strings, the number of characters in each string may not be equal. The return result is a [`tf.RaggedTensor`](../../guide/ragged_tensor.ipynb), where the length of the innermost dimension varies depending on the number of characters in each string:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N2jVzPymr_Mm", - "colab": {} - }, - "source": [ - "# A batch of Unicode strings, each represented as a UTF8-encoded string.\n", - "batch_utf8 = [s.encode('UTF-8') for s in\n", - " [u'hÃllo', u'What is the weather tomorrow', u'Göödnight', u'😊']]\n", - "batch_chars_ragged = tf.strings.unicode_decode(batch_utf8,\n", - " input_encoding='UTF-8')\n", - "for sentence_chars in batch_chars_ragged.to_list():\n", - " print(sentence_chars)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iRh3n1hPsJ9v" - }, - "source": [ - "You can use this `tf.RaggedTensor` directly, or convert it to a dense `tf.Tensor` with padding or a `tf.SparseTensor` using the methods `tf.RaggedTensor.to_tensor` and `tf.RaggedTensor.to_sparse`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yz17yeSMsUid", - "colab": {} - }, - "source": [ - "batch_chars_padded = batch_chars_ragged.to_tensor(default_value=-1)\n", - "print(batch_chars_padded.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kBjsPQp3rhfm", - "colab": {} - }, - "source": [ - "batch_chars_sparse = batch_chars_ragged.to_sparse()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GCCkZh-nwlbL" - }, - "source": [ - "When encoding multiple strings with the same lengths, a `tf.Tensor` may be used as input:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_lP62YUAwjK9", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode([[99, 97, 116], [100, 111, 103], [ 99, 111, 119]],\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w58CMRg9tamW" - }, - "source": [ - "When encoding multiple strings with varyling length, a `tf.RaggedTensor` should be used as input:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "d7GtOtrltaMl", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(batch_chars_ragged, output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T2Nh5Aj9xob3" - }, - "source": [ - "If you have a tensor with multiple strings in padded or sparse format, then convert it to a `tf.RaggedTensor` before calling `unicode_encode`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R2bYCYl0u-Ue", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(\n", - " tf.RaggedTensor.from_sparse(batch_chars_sparse),\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UlV2znh_u_zm", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(\n", - " tf.RaggedTensor.from_tensor(batch_chars_padded, padding=-1),\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hQOOGkscvDpc" - }, - "source": [ - "## Unicode operations" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NkmtsA_yvMB0" - }, - "source": [ - "### Character length\n", - "\n", - "The `tf.strings.length` operation has a parameter `unit`, which indicates how lengths should be computed. `unit` defaults to `\"BYTE\"`, but it can be set to other values, such as `\"UTF8_CHAR\"` or `\"UTF16_CHAR\"`, to determine the number of Unicode codepoints in each encoded `string`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1ZzMe59mvLHr", - "colab": {} - }, - "source": [ - "# Note that the final character takes up 4 bytes in UTF8.\n", - "thanks = u'Thanks 😊'.encode('UTF-8')\n", - "num_bytes = tf.strings.length(thanks).numpy()\n", - "num_chars = tf.strings.length(thanks, unit='UTF8_CHAR').numpy()\n", - "print('{} bytes; {} UTF-8 characters'.format(num_bytes, num_chars))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fHG85gxlvVU0" - }, - "source": [ - "### Character substrings\n", - "\n", - "Similarly, the `tf.strings.substr` operation accepts the \"`unit`\" parameter, and uses it to determine what kind of offsets the \"`pos`\" and \"`len`\" paremeters contain." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WlWRLV-4xWYq", - "colab": {} - }, - "source": [ - "# default: unit='BYTE'. With len=1, we return a single byte.\n", - "tf.strings.substr(thanks, pos=7, len=1).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JfNUVDPwxkCS", - "colab": {} - }, - "source": [ - "# Specifying unit='UTF8_CHAR', we return a single character, which in this case\n", - "# is 4 bytes.\n", - "print(tf.strings.substr(thanks, pos=7, len=1, unit='UTF8_CHAR').numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zJUEsVSyeIa3" - }, - "source": [ - "### Split Unicode strings\n", - "\n", - "The `tf.strings.unicode_split` operation splits unicode strings into substrings of individual characters:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dDjkh5G1ejMt", - "colab": {} - }, - "source": [ - "tf.strings.unicode_split(thanks, 'UTF-8').numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HQqEEZEbdG9O" - }, - "source": [ - "### Byte offsets for characters\n", - "\n", - "To align the character tensor generated by `tf.strings.unicode_decode` with the original string, it's useful to know the offset for where each character begins. The method `tf.strings.unicode_decode_with_offsets` is similar to `unicode_decode`, except that it returns a second tensor containing the start offset of each character." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Cug7cmwYdowd", - "colab": {} - }, - "source": [ - "codepoints, offsets = tf.strings.unicode_decode_with_offsets(u\"🎈🎉🎊\", 'UTF-8')\n", - "\n", - "for (codepoint, offset) in zip(codepoints.numpy(), offsets.numpy()):\n", - " print(\"At byte offset {}: codepoint {}\".format(offset, codepoint))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2ZnCNxOvx66T" - }, - "source": [ - "## Unicode scripts" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRRHqkqNyGZ6" - }, - "source": [ - "Each Unicode code point belongs to a single collection of codepoints known as a [script](https://en.wikipedia.org/wiki/Script_%28Unicode%29) . A character's script is helpful in determining which language the character might be in. For example, knowing that 'Б' is in Cyrillic script indicates that modern text containing that character is likely from a Slavic language such as Russian or Ukrainian.\n", - "\n", - "TensorFlow provides the `tf.strings.unicode_script` operation to determine which script a given codepoint uses. The script codes are `int32` values corresponding to [International Components for\n", - "Unicode](http://site.icu-project.org/home) (ICU) [`UScriptCode`](http://icu-project.org/apiref/icu4c/uscript_8h.html) values.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K7DeYHrRyFPy", - "colab": {} - }, - "source": [ - "uscript = tf.strings.unicode_script([33464, 1041]) # ['芸', 'Б']\n", - "\n", - "print(uscript.numpy()) # [17, 8] == [USCRIPT_HAN, USCRIPT_CYRILLIC]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2fW992a1lIY6" - }, - "source": [ - "The `tf.strings.unicode_script` operation can also be applied to multidimensional `tf.Tensor`s or `tf.RaggedTensor`s of codepoints:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uR7b8meLlFnp", - "colab": {} - }, - "source": [ - "print(tf.strings.unicode_script(batch_chars_ragged))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mx7HEFpBzEsB" - }, - "source": [ - "## Example: Simple segmentation\n", - "\n", - "Segmentation is the task of splitting text into word-like units. This is often easy when space characters are used to separate words, but some languages (like Chinese and Japanese) do not use spaces, and some languages (like German) contain long compounds that must be split in order to analyze their meaning. In web text, different languages and scripts are frequently mixed together, as in \"NY株価\" (New York Stock Exchange).\n", - "\n", - "We can perform very rough segmentation (without implementing any ML models) by using changes in script to approximate word boundaries. This will work for strings like the \"NY株価\" example above. It will also work for most languages that use spaces, as the space characters of various scripts are all classified as USCRIPT_COMMON, a special script code that differs from that of any actual text." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "grsvFiC4BoPb", - "colab": {} - }, - "source": [ - "# dtype: string; shape: [num_sentences]\n", - "#\n", - "# The sentences to process. Edit this line to try out different inputs!\n", - "sentence_texts = [u'Hello, world.', u'世界こんにちは']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CapnbShuGU8i" - }, - "source": [ - "First, we decode the sentences into character codepoints, and find the script identifeir for each character." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReQVcDQh1MB8", - "colab": {} - }, - "source": [ - "# dtype: int32; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_codepoint[i, j] is the codepoint for the j'th character in\n", - "# the i'th sentence.\n", - "sentence_char_codepoint = tf.strings.unicode_decode(sentence_texts, 'UTF-8')\n", - "print(sentence_char_codepoint)\n", - "\n", - "# dtype: int32; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_scripts[i, j] is the unicode script of the j'th character in\n", - "# the i'th sentence.\n", - "sentence_char_script = tf.strings.unicode_script(sentence_char_codepoint)\n", - "print(sentence_char_script)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O2fapF5UGcUc" - }, - "source": [ - "Next, we use those script identifiers to determine where word boundaries should be added. We add a word boundary at the beginning of each sentence, and for each character whose script differs from the previous character:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7v5W6MOr1Rlc", - "colab": {} - }, - "source": [ - "# dtype: bool; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_starts_word[i, j] is True if the j'th character in the i'th\n", - "# sentence is the start of a word.\n", - "sentence_char_starts_word = tf.concat(\n", - " [tf.fill([sentence_char_script.nrows(), 1], True),\n", - " tf.not_equal(sentence_char_script[:, 1:], sentence_char_script[:, :-1])],\n", - " axis=1)\n", - "\n", - "# dtype: int64; shape: [num_words]\n", - "#\n", - "# word_starts[i] is the index of the character that starts the i'th word (in\n", - "# the flattened list of characters from all sentences).\n", - "word_starts = tf.squeeze(tf.where(sentence_char_starts_word.values), axis=1)\n", - "print(word_starts)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LAwh-1QkGuC9" - }, - "source": [ - "We can then use those start offsets to build a `RaggedTensor` containing the list of words from all batches:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bNiA1O_eBBCL", - "colab": {} - }, - "source": [ - "# dtype: int32; shape: [num_words, (num_chars_per_word)]\n", - "#\n", - "# word_char_codepoint[i, j] is the codepoint for the j'th character in the\n", - "# i'th word.\n", - "word_char_codepoint = tf.RaggedTensor.from_row_starts(\n", - " values=sentence_char_codepoint.values,\n", - " row_starts=word_starts)\n", - "print(word_char_codepoint)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "66a2ZnYmG2ao" - }, - "source": [ - "And finally, we can segment the word codepoints `RaggedTensor` back into sentences:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NCfwcqLSEjZb", - "colab": {} - }, - "source": [ - "# dtype: int64; shape: [num_sentences]\n", - "#\n", - "# sentence_num_words[i] is the number of words in the i'th sentence.\n", - "sentence_num_words = tf.reduce_sum(\n", - " tf.cast(sentence_char_starts_word, tf.int64),\n", - " axis=1)\n", - "\n", - "# dtype: int32; shape: [num_sentences, (num_words_per_sentence), (num_chars_per_word)]\n", - "#\n", - "# sentence_word_char_codepoint[i, j, k] is the codepoint for the k'th character\n", - "# in the j'th word in the i'th sentence.\n", - "sentence_word_char_codepoint = tf.RaggedTensor.from_row_lengths(\n", - " values=word_char_codepoint,\n", - " row_lengths=sentence_num_words)\n", - "print(sentence_word_char_codepoint)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xWaX8WcbHyqY" - }, - "source": [ - "To make the final result easier to read, we can encode it back into UTF-8 strings:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HSivquOgFr3C", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(sentence_word_char_codepoint, 'UTF-8').to_list()" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/en/tutorials/load_data/video.ipynb b/site/en/tutorials/load_data/video.ipynb new file mode 100644 index 00000000000..42439404948 --- /dev/null +++ b/site/en/tutorials/load_data/video.ipynb @@ -0,0 +1,1026 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "mt9dL5dIir8X" + }, + "source": [ + "##### Copyright 2022 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "ufPx7EiCiqgR" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4StGz9ynOEL6" + }, + "source": [ + "# Load video data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KwQtSOz0VrVX" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F-SqCosJ6-0H" + }, + "source": [ + "This tutorial demonstrates how to load and preprocess [AVI](https://en.wikipedia.org/wiki/Audio_Video_Interleave) video data using the [UCF101 human action dataset](https://www.tensorflow.org/datasets/catalog/ucf101). Once you have preprocessed the data, it can be used for such tasks as video classification/recognition, captioning or clustering. The original dataset contains realistic action videos collected from YouTube with 101 categories, including playing cello, brushing teeth, and applying eye makeup. You will learn how to:\n", + "\n", + "* Load the data from a zip file.\n", + "\n", + "* Read sequences of frames out of the video files.\n", + "\n", + "* Visualize the video data.\n", + "\n", + "* Wrap the frame-generator [`tf.data.Dataset`](https://www.tensorflow.org/guide/data).\n", + "\n", + "This video loading and preprocessing tutorial is the first part in a series of TensorFlow video tutorials. Here are the other three tutorials:\n", + "\n", + "- [Build a 3D CNN model for video classification](https://www.tensorflow.org/tutorials/video/video_classification): Note that this tutorial uses a (2+1)D CNN that decomposes the spatial and temporal aspects of 3D data; if you are using volumetric data such as an MRI scan, consider using a 3D CNN instead of a (2+1)D CNN.\n", + "- [MoViNet for streaming action recognition](https://www.tensorflow.org/hub/tutorials/movinet): Get familiar with the MoViNet models that are available on TF Hub.\n", + "- [Transfer learning for video classification with MoViNet](https://www.tensorflow.org/tutorials/video/transfer_learning_with_movinet): This tutorial explains how to use a pre-trained video classification model trained on a different dataset with the UCF-101 dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PnpPjKVD68eH" + }, + "source": [ + "## Setup\n", + "\n", + "Begin by installing and importing some necessary libraries, including:\n", + "[remotezip](https://github.com/gtsystem/python-remotezip) to inspect the contents of a ZIP file, [tqdm](https://github.com/tqdm/tqdm) to use a progress bar, [OpenCV](https://opencv.org/) to process video files, and [`tensorflow_docs`](https://github.com/tensorflow/docs/tree/master/tools/tensorflow_docs) for embedding data in a Jupyter notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SjI3AaaO16bd" + }, + "outputs": [], + "source": [ + "# The way this tutorial uses the `TimeDistributed` layer requires TF>=2.10\n", + "!pip install -U \"tensorflow>=2.10.0\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "P5SBasQcbwQA" + }, + "outputs": [], + "source": [ + "!pip install remotezip tqdm opencv-python\n", + "!pip install -q git+https://github.com/tensorflow/docs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9RYQIJ9C6BVH" + }, + "outputs": [], + "source": [ + "import tqdm\n", + "import random\n", + "import pathlib\n", + "import itertools\n", + "import collections\n", + "\n", + "import os\n", + "import cv2\n", + "import numpy as np\n", + "import remotezip as rz\n", + "\n", + "import tensorflow as tf\n", + "\n", + "# Some modules to display an animation using imageio.\n", + "import imageio\n", + "from IPython import display\n", + "from urllib import request\n", + "from tensorflow_docs.vis import embed" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KbhwWLLM7FXo" + }, + "source": [ + "## Download a subset of the UCF101 dataset\n", + "\n", + "The [UCF101 dataset](https://www.tensorflow.org/datasets/catalog/ucf101) contains 101 categories of different actions in video, primarily used in action recognition. You will use a subset of these categories in this demo." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gVIgj-jIA8U8" + }, + "outputs": [], + "source": [ + "URL = 'https://storage.googleapis.com/thumos14_files/UCF101_videos.zip'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2tm8aBzw6Md7" + }, + "source": [ + "The above URL contains a zip file with the UCF 101 dataset. Create a function that uses the `remotezip` library to examine the contents of the zip file in that URL:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lY-x7TaZlK6O" + }, + "outputs": [], + "source": [ + "def list_files_from_zip_url(zip_url):\n", + " \"\"\" List the files in each class of the dataset given a URL with the zip file.\n", + "\n", + " Args:\n", + " zip_url: A URL from which the files can be extracted from.\n", + "\n", + " Returns:\n", + " List of files in each of the classes.\n", + " \"\"\"\n", + " files = []\n", + " with rz.RemoteZip(zip_url) as zip:\n", + " for zip_info in zip.infolist():\n", + " files.append(zip_info.filename)\n", + " return files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lYErXAdUr-rk" + }, + "outputs": [], + "source": [ + "files = list_files_from_zip_url(URL)\n", + "files = [f for f in files if f.endswith('.avi')]\n", + "files[:10]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rQ4l8D9dFPS7" + }, + "source": [ + "Begin with a few videos and a limited number of classes for training. After running the above code block, notice that the class name is included in the filename of each video.\n", + "\n", + "Define the `get_class` function that retrieves the class name from a filename. Then, create a function called `get_files_per_class` which converts the list of all files (`files` above) into a dictionary listing the files for each class:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yyyivOX0sO19" + }, + "outputs": [], + "source": [ + "def get_class(fname):\n", + " \"\"\" Retrieve the name of the class given a filename.\n", + "\n", + " Args:\n", + " fname: Name of the file in the UCF101 dataset.\n", + "\n", + " Returns:\n", + " Class that the file belongs to.\n", + " \"\"\"\n", + " return fname.split('_')[-3]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1qnH0xKzlyw_" + }, + "outputs": [], + "source": [ + "def get_files_per_class(files):\n", + " \"\"\" Retrieve the files that belong to each class.\n", + "\n", + " Args:\n", + " files: List of files in the dataset.\n", + "\n", + " Returns:\n", + " Dictionary of class names (key) and files (values). \n", + " \"\"\"\n", + " files_for_class = collections.defaultdict(list)\n", + " for fname in files:\n", + " class_name = get_class(fname)\n", + " files_for_class[class_name].append(fname)\n", + " return files_for_class" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VxSt5YgSGrWn" + }, + "source": [ + "Once you have the list of files per class, you can choose how many classes you would like to use and how many videos you would like per class in order to create your dataset. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qPdURg74uUTk" + }, + "outputs": [], + "source": [ + "NUM_CLASSES = 10\n", + "FILES_PER_CLASS = 50" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GUs0xtXsr9i3" + }, + "outputs": [], + "source": [ + "files_for_class = get_files_per_class(files)\n", + "classes = list(files_for_class.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-YqFARvqwon9" + }, + "outputs": [], + "source": [ + "print('Num classes:', len(classes))\n", + "print('Num videos for class[0]:', len(files_for_class[classes[0]]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yFAFqKqE92bQ" + }, + "source": [ + "Create a new function called `select_subset_of_classes` that selects a subset of the classes present within the dataset and a particular number of files per class:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "O3jek4QimIj-" + }, + "outputs": [], + "source": [ + "def select_subset_of_classes(files_for_class, classes, files_per_class):\n", + " \"\"\" Create a dictionary with the class name and a subset of the files in that class.\n", + "\n", + " Args:\n", + " files_for_class: Dictionary of class names (key) and files (values).\n", + " classes: List of classes.\n", + " files_per_class: Number of files per class of interest.\n", + "\n", + " Returns:\n", + " Dictionary with class as key and list of specified number of video files in that class.\n", + " \"\"\"\n", + " files_subset = dict()\n", + "\n", + " for class_name in classes:\n", + " class_files = files_for_class[class_name]\n", + " files_subset[class_name] = class_files[:files_per_class]\n", + "\n", + " return files_subset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5cjcz6Gpcb-W" + }, + "outputs": [], + "source": [ + "files_subset = select_subset_of_classes(files_for_class, classes[:NUM_CLASSES], FILES_PER_CLASS)\n", + "list(files_subset.keys())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ALrlDS1lZx3E" + }, + "source": [ + "Define helper functions that split the videos into training, validation, and test sets. The videos are downloaded from a URL with the zip file, and placed into their respective subdirectiories." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AH9sWS_6nRz3" + }, + "outputs": [], + "source": [ + "def download_from_zip(zip_url, to_dir, file_names):\n", + " \"\"\" Download the contents of the zip file from the zip URL.\n", + "\n", + " Args:\n", + " zip_url: A URL with a zip file containing data.\n", + " to_dir: A directory to download data to.\n", + " file_names: Names of files to download.\n", + " \"\"\"\n", + " with rz.RemoteZip(zip_url) as zip:\n", + " for fn in tqdm.tqdm(file_names):\n", + " class_name = get_class(fn)\n", + " zip.extract(fn, str(to_dir / class_name))\n", + " unzipped_file = to_dir / class_name / fn\n", + "\n", + " fn = pathlib.Path(fn).parts[-1]\n", + " output_file = to_dir / class_name / fn\n", + " unzipped_file.rename(output_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pejRTChA6mrp" + }, + "source": [ + "The following function returns the remaining data that hasn't already been placed into a subset of data. It allows you to place that remaining data in the next specified subset of data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6ARYc-WLqqNF" + }, + "outputs": [], + "source": [ + "def split_class_lists(files_for_class, count):\n", + " \"\"\" Returns the list of files belonging to a subset of data as well as the remainder of\n", + " files that need to be downloaded.\n", + " \n", + " Args:\n", + " files_for_class: Files belonging to a particular class of data.\n", + " count: Number of files to download.\n", + "\n", + " Returns:\n", + " Files belonging to the subset of data and dictionary of the remainder of files that need to be downloaded.\n", + " \"\"\"\n", + " split_files = []\n", + " remainder = {}\n", + " for cls in files_for_class:\n", + " split_files.extend(files_for_class[cls][:count])\n", + " remainder[cls] = files_for_class[cls][count:]\n", + " return split_files, remainder" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LlEQ_I0TLd1X" + }, + "source": [ + "The following `download_ucf_101_subset` function allows you to download a subset of the UCF101 dataset and split it into the training, validation, and test sets. You can specify the number of classes that you would like to use. The `splits` argument allows you to pass in a dictionary in which the key values are the name of subset (example: \"train\") and the number of videos you would like to have per class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IHH2Y1M06xoz" + }, + "outputs": [], + "source": [ + "def download_ucf_101_subset(zip_url, num_classes, splits, download_dir):\n", + " \"\"\" Download a subset of the UCF101 dataset and split them into various parts, such as\n", + " training, validation, and test.\n", + "\n", + " Args:\n", + " zip_url: A URL with a ZIP file with the data.\n", + " num_classes: Number of labels.\n", + " splits: Dictionary specifying the training, validation, test, etc. (key) division of data \n", + " (value is number of files per split).\n", + " download_dir: Directory to download data to.\n", + "\n", + " Return:\n", + " Mapping of the directories containing the subsections of data.\n", + " \"\"\"\n", + " files = list_files_from_zip_url(zip_url)\n", + " for f in files:\n", + " path = os.path.normpath(f)\n", + " tokens = path.split(os.sep)\n", + " if len(tokens) <= 2:\n", + " files.remove(f) # Remove that item from the list if it does not have a filename\n", + " \n", + " files_for_class = get_files_per_class(files)\n", + "\n", + " classes = list(files_for_class.keys())[:num_classes]\n", + "\n", + " for cls in classes:\n", + " random.shuffle(files_for_class[cls])\n", + " \n", + " # Only use the number of classes you want in the dictionary\n", + " files_for_class = {x: files_for_class[x] for x in classes}\n", + "\n", + " dirs = {}\n", + " for split_name, split_count in splits.items():\n", + " print(split_name, \":\")\n", + " split_dir = download_dir / split_name\n", + " split_files, files_for_class = split_class_lists(files_for_class, split_count)\n", + " download_from_zip(zip_url, split_dir, split_files)\n", + " dirs[split_name] = split_dir\n", + "\n", + " return dirs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NuD-xU8Q66Vm" + }, + "outputs": [], + "source": [ + "download_dir = pathlib.Path('./UCF101_subset/')\n", + "subset_paths = download_ucf_101_subset(URL,\n", + " num_classes = NUM_CLASSES,\n", + " splits = {\"train\": 30, \"val\": 10, \"test\": 10},\n", + " download_dir = download_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MBMRm9Ub3Zrk" + }, + "source": [ + "After downloading the data, you should now have a copy of a subset of the UCF101 dataset. Run the following code to print the total number of videos you have amongst all your subsets of data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zupvOLYP4D4q" + }, + "outputs": [], + "source": [ + "video_count_train = len(list(download_dir.glob('train/*/*.avi')))\n", + "video_count_val = len(list(download_dir.glob('val/*/*.avi')))\n", + "video_count_test = len(list(download_dir.glob('test/*/*.avi')))\n", + "video_total = video_count_train + video_count_val + video_count_test\n", + "print(f\"Total videos: {video_total}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JmJG1SlXiOX8" + }, + "source": [ + "You can also preview the directory of data files now." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "y9be0WlDiNM0" + }, + "outputs": [], + "source": [ + "!find ./UCF101_subset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U4uslY4dScyu" + }, + "source": [ + "## Create frames from each video file" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D1vvyT0F7JAZ" + }, + "source": [ + "The `frames_from_video_file` function splits the videos into frames, reads a randomly chosen span of `n_frames` out of a video file, and returns them as a NumPy `array`.\n", + "To reduce memory and computation overhead, choose a **small** number of frames. In addition, pick the **same** number of frames from each video, which makes it easier to work on batches of data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vNBCiV3bMzpD" + }, + "outputs": [], + "source": [ + "def format_frames(frame, output_size):\n", + " \"\"\"\n", + " Pad and resize an image from a video.\n", + " \n", + " Args:\n", + " frame: Image that needs to resized and padded. \n", + " output_size: Pixel size of the output frame image.\n", + "\n", + " Return:\n", + " Formatted frame with padding of specified output size.\n", + " \"\"\"\n", + " frame = tf.image.convert_image_dtype(frame, tf.float32)\n", + " frame = tf.image.resize_with_pad(frame, *output_size)\n", + " return frame" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9ujLDC9G7JyE" + }, + "outputs": [], + "source": [ + "def frames_from_video_file(video_path, n_frames, output_size = (224,224), frame_step = 15):\n", + " \"\"\"\n", + " Creates frames from each video file present for each category.\n", + "\n", + " Args:\n", + " video_path: File path to the video.\n", + " n_frames: Number of frames to be created per video file.\n", + " output_size: Pixel size of the output frame image.\n", + "\n", + " Return:\n", + " An NumPy array of frames in the shape of (n_frames, height, width, channels).\n", + " \"\"\"\n", + " # Read each video frame by frame\n", + " result = []\n", + " src = cv2.VideoCapture(str(video_path)) \n", + "\n", + " video_length = src.get(cv2.CAP_PROP_FRAME_COUNT)\n", + "\n", + " need_length = 1 + (n_frames - 1) * frame_step\n", + "\n", + " if need_length > video_length:\n", + " start = 0\n", + " else:\n", + " max_start = video_length - need_length\n", + " start = random.randint(0, max_start + 1)\n", + "\n", + " src.set(cv2.CAP_PROP_POS_FRAMES, start)\n", + " # ret is a boolean indicating whether read was successful, frame is the image itself\n", + " ret, frame = src.read()\n", + " result.append(format_frames(frame, output_size))\n", + "\n", + " for _ in range(n_frames - 1):\n", + " for _ in range(frame_step):\n", + " ret, frame = src.read()\n", + " if ret:\n", + " frame = format_frames(frame, output_size)\n", + " result.append(frame)\n", + " else:\n", + " result.append(np.zeros_like(result[0]))\n", + " src.release()\n", + " result = np.array(result)[..., [2, 1, 0]]\n", + "\n", + " return result" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1ENtlwhxwyTe" + }, + "source": [ + "## Visualize video data\n", + "\n", + "The `frames_from_video_file` function that returns a set of frames as a NumPy array. Try using this function on a new video from [Wikimedia](https://commons.wikimedia.org/wiki/Category:Videos_of_sports) by Patrick Gillett:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Z2hgSghlykzA" + }, + "outputs": [], + "source": [ + "!curl -O https://upload.wikimedia.org/wikipedia/commons/8/86/End_of_a_jam.ogv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xdHvHw3hym-U" + }, + "outputs": [], + "source": [ + "video_path = \"End_of_a_jam.ogv\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "u845YODXyqo5" + }, + "outputs": [], + "source": [ + "sample_video = frames_from_video_file(video_path, n_frames = 10)\n", + "sample_video.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zFHGHiFgGjv2" + }, + "outputs": [], + "source": [ + "def to_gif(images):\n", + " converted_images = np.clip(images * 255, 0, 255).astype(np.uint8)\n", + " imageio.mimsave('./animation.gif', converted_images, fps=10)\n", + " return embed.embed_file('./animation.gif')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7hiwUJenEN3p" + }, + "outputs": [], + "source": [ + "to_gif(sample_video)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3dktTnDVG7xf" + }, + "source": [ + "In addition to examining this video, you can also display the UCF-101 data. To do this, run the following code:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MghJzJsWme0t" + }, + "outputs": [], + "source": [ + "# docs-infra: no-execute\n", + "ucf_sample_video = frames_from_video_file(next(subset_paths['train'].glob('*/*.avi')), 50)\n", + "to_gif(ucf_sample_video)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NlvuC5_E7XrF" + }, + "source": [ + "Next, define the `FrameGenerator` class in order to create an iterable object that can feed data into the TensorFlow data pipeline. The generator (`__call__`) function yields the frame array produced by `frames_from_video_file` and a one-hot encoded vector of the label associated with the set of frames." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MVmfLTlw7Ues" + }, + "outputs": [], + "source": [ + "class FrameGenerator:\n", + " def __init__(self, path, n_frames, training = False):\n", + " \"\"\" Returns a set of frames with their associated label. \n", + "\n", + " Args:\n", + " path: Video file paths.\n", + " n_frames: Number of frames. \n", + " training: Boolean to determine if training dataset is being created.\n", + " \"\"\"\n", + " self.path = path\n", + " self.n_frames = n_frames\n", + " self.training = training\n", + " self.class_names = sorted(set(p.name for p in self.path.iterdir() if p.is_dir()))\n", + " self.class_ids_for_name = dict((name, idx) for idx, name in enumerate(self.class_names))\n", + "\n", + " def get_files_and_class_names(self):\n", + " video_paths = list(self.path.glob('*/*.avi'))\n", + " classes = [p.parent.name for p in video_paths] \n", + " return video_paths, classes\n", + "\n", + " def __call__(self):\n", + " video_paths, classes = self.get_files_and_class_names()\n", + "\n", + " pairs = list(zip(video_paths, classes))\n", + "\n", + " if self.training:\n", + " random.shuffle(pairs)\n", + "\n", + " for path, name in pairs:\n", + " video_frames = frames_from_video_file(path, self.n_frames) \n", + " label = self.class_ids_for_name[name] # Encode labels\n", + " yield video_frames, label" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xsvhPIkpzx-r" + }, + "source": [ + "Test out the `FrameGenerator` object before wrapping it as a TensorFlow Dataset object. Moreover, for the training dataset, ensure you enable training mode so that the data will be shuffled." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "P5jwagZxzxOf" + }, + "outputs": [], + "source": [ + "fg = FrameGenerator(subset_paths['train'], 10, training=True)\n", + "\n", + "frames, label = next(fg())\n", + "\n", + "print(f\"Shape: {frames.shape}\")\n", + "print(f\"Label: {label}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "E7MRRFSks7l1" + }, + "source": [ + "Finally, create a TensorFlow data input pipeline. This pipeline that you create from the generator object allows you to feed in data to your deep learning model. In this video pipeline, each element is a single set of frames and its associated label. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HM4NboJr7ck4" + }, + "outputs": [], + "source": [ + "# Create the training set\n", + "output_signature = (tf.TensorSpec(shape = (None, None, None, 3), dtype = tf.float32),\n", + " tf.TensorSpec(shape = (), dtype = tf.int16))\n", + "train_ds = tf.data.Dataset.from_generator(FrameGenerator(subset_paths['train'], 10, training=True),\n", + " output_signature = output_signature)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9oF_8m8IZvcY" + }, + "source": [ + "Check to see that the labels are shuffled. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3XYVmsgiZsJD" + }, + "outputs": [], + "source": [ + "for frames, labels in train_ds.take(10):\n", + " print(labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Pi8-WkOkEXw5" + }, + "outputs": [], + "source": [ + "# Create the validation set\n", + "val_ds = tf.data.Dataset.from_generator(FrameGenerator(subset_paths['val'], 10),\n", + " output_signature = output_signature)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "V6qXc-6i7eyK" + }, + "outputs": [], + "source": [ + "# Print the shapes of the data\n", + "train_frames, train_labels = next(iter(train_ds))\n", + "print(f'Shape of training set of frames: {train_frames.shape}')\n", + "print(f'Shape of training labels: {train_labels.shape}')\n", + "\n", + "val_frames, val_labels = next(iter(val_ds))\n", + "print(f'Shape of validation set of frames: {val_frames.shape}')\n", + "print(f'Shape of validation labels: {val_labels.shape}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bIrFpUIxvTLe" + }, + "source": [ + "## Configure the dataset for performance\n", + "\n", + "Use buffered prefetching such that you can yield data from the disk without having I/O become blocking. Two important functions to use while loading data are:\n", + "\n", + "* `Dataset.cache`: keeps the sets of frames in memory after they're loaded off the disk during the first epoch. This function ensures that the dataset does not become a bottleneck while training your model. If your dataset is too large to fit into memory, you can also use this method to create a performant on-disk cache.\n", + "\n", + "* `Dataset.prefetch`: overlaps data preprocessing and model execution while training.\n", + "Refer to [Better performance with the `tf.data`](https://www.tensorflow.org/guide/data_performance) for details." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QSxjFtxAvY3_" + }, + "outputs": [], + "source": [ + "AUTOTUNE = tf.data.AUTOTUNE\n", + "\n", + "train_ds = train_ds.cache().shuffle(1000).prefetch(buffer_size = AUTOTUNE)\n", + "val_ds = val_ds.cache().shuffle(1000).prefetch(buffer_size = AUTOTUNE)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VaY-hyr-Fbfr" + }, + "source": [ + "To prepare the data to be fed into the model, use batching as shown below. Notice that when working with video data, such as AVI files, the data should be shaped as a five dimensional object. These dimensions are as follows: `[batch_size, number_of_frames, height, width, channels]`. In comparison, an image would have four dimensions: `[batch_size, height, width, channels]`. The image below is an illustration of how the shape of video data is represented.\n", + "\n", + "![Video data shape](https://www.tensorflow.org/images/tutorials/video/video_data_shape.png)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pp2Qc6XSFmeB" + }, + "outputs": [], + "source": [ + "train_ds = train_ds.batch(2)\n", + "val_ds = val_ds.batch(2)\n", + "\n", + "train_frames, train_labels = next(iter(train_ds))\n", + "print(f'Shape of training set of frames: {train_frames.shape}')\n", + "print(f'Shape of training labels: {train_labels.shape}')\n", + "\n", + "val_frames, val_labels = next(iter(val_ds))\n", + "print(f'Shape of validation set of frames: {val_frames.shape}')\n", + "print(f'Shape of validation labels: {val_labels.shape}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hqjXn1FgsMqZ" + }, + "source": [ + "## Next steps\n", + "\n", + "Now that you have created a TensorFlow `Dataset` of video frames with their labels, you can use it with a deep learning model. The following classification model that uses a pre-trained [EfficientNet](https://arxiv.org/abs/1905.11946) trains to high accuracy in a few minutes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qzqgPBUuForj" + }, + "outputs": [], + "source": [ + "net = tf.keras.applications.EfficientNetB0(include_top = False)\n", + "net.trainable = False\n", + "\n", + "model = tf.keras.Sequential([\n", + " tf.keras.layers.Rescaling(scale=255),\n", + " tf.keras.layers.TimeDistributed(net),\n", + " tf.keras.layers.Dense(10),\n", + " tf.keras.layers.GlobalAveragePooling3D()\n", + "])\n", + "\n", + "model.compile(optimizer = 'adam',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits = True),\n", + " metrics=['accuracy'])\n", + "\n", + "model.fit(train_ds, \n", + " epochs = 10,\n", + " validation_data = val_ds,\n", + " callbacks = tf.keras.callbacks.EarlyStopping(patience = 2, monitor = 'val_loss'))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DdJm7ojgGxtT" + }, + "source": [ + "To learn more about working with video data in TensorFlow, check out the following tutorials:\n", + "\n", + "* [Build a 3D CNN model for video classification](https://www.tensorflow.org/tutorials/video/video_classification)\n", + "* [MoViNet for streaming action recognition](https://www.tensorflow.org/hub/tutorials/movinet)\n", + "* [Transfer learning for video classification with MoViNet](https://www.tensorflow.org/tutorials/video/transfer_learning_with_movinet)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "video.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/optimization/compression.ipynb b/site/en/tutorials/optimization/compression.ipynb new file mode 100644 index 00000000000..b94ecaf6476 --- /dev/null +++ b/site/en/tutorials/optimization/compression.ipynb @@ -0,0 +1,1187 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Tce3stUlHN0L" + }, + "source": [ + "##### Copyright 2022 The TensorFlow Compression Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "tuOe1ymfHZPu" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qFdPvlXBOdUN" + }, + "source": [ + "# Scalable model compression" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MfBg1C5NB3X0" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xHxb-dlhMIzW" + }, + "source": [ + "## Overview\n", + "\n", + "This notebook shows how to compress a model using [TensorFlow Compression](https://github.com/tensorflow/compression).\n", + "\n", + "In the example below, we compress the weights of an MNIST classifier to a much smaller size than their floating point representation, while retaining classification accuracy. This is done by a two step process, based on the paper [Scalable Model Compression by Entropy Penalized Reparameterization](https://arxiv.org/abs/1906.06624):\n", + "\n", + "- Training a \"compressible\" model with an explicit **entropy penalty** during training, which encourages compressibility of the model parameters. The weight on this penalty, $\\lambda$, enables continuously controlling the trade-off between the compressed model size and its accuracy.\n", + "\n", + "- Encoding the compressible model into a compressed model using a coding scheme that is matched with the penalty, meaning that the penalty is a good predictor for model size. This ensures that the method doesn't require multiple iterations of training, compressing, and re-training the model for fine-tuning.\n", + "\n", + "This method is strictly concerned with compressed model size, not with computational complexity. It can be combined with a technique like model pruning to reduce size and complexity.\n", + "\n", + "Example compression results on various models:\n", + "\n", + "Model (dataset) | Model size | Comp. ratio | Top-1 error comp. (uncomp.)\n", + "------------------------|------------|-------------|------------\n", + "LeNet300-100 (MNIST) | 8.56 KB | 124x | 1.9% (1.6%)\n", + "LeNet5-Caffe (MNIST) | 2.84 KB | 606x | 1.0% (0.7%)\n", + "VGG-16 (CIFAR-10) | 101 KB | 590x | 10.0% (6.6%)\n", + "ResNet-20-4 (CIFAR-10) | 128 KB | 134x | 8.8% (5.0%)\n", + "ResNet-18 (ImageNet) | 1.97 MB | 24x | 30.0% (30.0%)\n", + "ResNet-50 (ImageNet) | 5.49 MB | 19x | 26.0% (25.0%)\n", + "\n", + "Applications include:\n", + "- Deploying/broadcasting models to edge devices on a large scale, saving bandwidth in transit.\n", + "- Communicating global model state to clients in federated learning. The model architecture (number of hidden units, etc.) is unchanged from the initial model, and clients can continue learning on the decompressed model.\n", + "- Performing inference on extremely memory limited clients. During inference, the weights of each layer can be sequentially decompressed, and discarded right after the activations are computed." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MUXex9ctTuDB" + }, + "source": [ + "## Setup\n", + "\n", + "Install Tensorflow Compression via `pip`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "K489KsEgxuLI" + }, + "outputs": [], + "source": [ + "%%bash\n", + "# Installs the latest version of TFC compatible with the installed TF version.\n", + "\n", + "read MAJOR MINOR <<< \"$(pip show tensorflow | perl -p -0777 -e 's/.*Version: (\\d+)\\.(\\d+).*/\\1 \\2/sg')\"\n", + "pip install \"tensorflow-compression<$MAJOR.$(($MINOR+1))\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WfVAmHCVxpTS" + }, + "source": [ + "Import library dependencies." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IqR2PQG4ZaZ0" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import tensorflow as tf\n", + "import tensorflow_compression as tfc\n", + "import tensorflow_datasets as tfds\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wsncKT2iymgQ" + }, + "source": [ + "## Define and train a basic MNIST classifier\n", + "\n", + "In order to effectively compress dense and convolutional layers, we need to define custom layer classes. These are analogous to the layers under `tf.keras.layers`, but we will subclass them later to effectively implement Entropy Penalized Reparameterization (EPR). For this purpose, we also add a copy constructor.\n", + "\n", + "First, we define a standard dense layer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "n_7ZRqiaO1WQ" + }, + "outputs": [], + "source": [ + "class CustomDense(tf.keras.layers.Layer):\n", + "\n", + " def __init__(self, filters, name=\"dense\"):\n", + " super().__init__(name=name)\n", + " self.filters = filters\n", + "\n", + " @classmethod\n", + " def copy(cls, other, **kwargs):\n", + " \"\"\"Returns an instantiated and built layer, initialized from `other`.\"\"\"\n", + " self = cls(filters=other.filters, name=other.name, **kwargs)\n", + " self.build(None, other=other)\n", + " return self\n", + "\n", + " def build(self, input_shape, other=None):\n", + " \"\"\"Instantiates weights, optionally initializing them from `other`.\"\"\"\n", + " if other is None:\n", + " kernel_shape = (input_shape[-1], self.filters)\n", + " kernel = tf.keras.initializers.GlorotUniform()(shape=kernel_shape)\n", + " bias = tf.keras.initializers.Zeros()(shape=(self.filters,))\n", + " else:\n", + " kernel, bias = other.kernel, other.bias\n", + " self.kernel = tf.Variable(\n", + " tf.cast(kernel, self.variable_dtype), name=\"kernel\")\n", + " self.bias = tf.Variable(\n", + " tf.cast(bias, self.variable_dtype), name=\"bias\")\n", + " self.built = True\n", + "\n", + " def call(self, inputs):\n", + " outputs = tf.linalg.matvec(self.kernel, inputs, transpose_a=True)\n", + " outputs = tf.nn.bias_add(outputs, self.bias)\n", + " return tf.nn.leaky_relu(outputs)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RUZkcXegc0yR" + }, + "source": [ + "And similarly, a 2D convolutional layer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RDibtb8EWCSj" + }, + "outputs": [], + "source": [ + "class CustomConv2D(tf.keras.layers.Layer):\n", + "\n", + " def __init__(self, filters, kernel_size,\n", + " strides=1, padding=\"SAME\", name=\"conv2d\"):\n", + " super().__init__(name=name)\n", + " self.filters = filters\n", + " self.kernel_size = kernel_size\n", + " self.strides = strides\n", + " self.padding = padding\n", + "\n", + " @classmethod\n", + " def copy(cls, other, **kwargs):\n", + " \"\"\"Returns an instantiated and built layer, initialized from `other`.\"\"\"\n", + " self = cls(filters=other.filters, kernel_size=other.kernel_size,\n", + " strides=other.strides, padding=other.padding, name=other.name,\n", + " **kwargs)\n", + " self.build(None, other=other)\n", + " return self\n", + "\n", + " def build(self, input_shape, other=None):\n", + " \"\"\"Instantiates weights, optionally initializing them from `other`.\"\"\"\n", + " if other is None:\n", + " kernel_shape = 2 * (self.kernel_size,) + (input_shape[-1], self.filters)\n", + " kernel = tf.keras.initializers.GlorotUniform()(shape=kernel_shape)\n", + " bias = tf.keras.initializers.Zeros()(shape=(self.filters,))\n", + " else:\n", + " kernel, bias = other.kernel, other.bias\n", + " self.kernel = tf.Variable(\n", + " tf.cast(kernel, self.variable_dtype), name=\"kernel\")\n", + " self.bias = tf.Variable(\n", + " tf.cast(bias, self.variable_dtype), name=\"bias\")\n", + " self.built = True\n", + "\n", + " def call(self, inputs):\n", + " outputs = tf.nn.convolution(\n", + " inputs, self.kernel, strides=self.strides, padding=self.padding)\n", + " outputs = tf.nn.bias_add(outputs, self.bias)\n", + " return tf.nn.leaky_relu(outputs)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6xWa1hHMdCpG" + }, + "source": [ + "Before we continue with model compression, let's check that we can successfully train a regular classifier.\n", + "\n", + "Define the model architecture:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8yZESLgW-vp1" + }, + "outputs": [], + "source": [ + "classifier = tf.keras.Sequential([\n", + " CustomConv2D(20, 5, strides=2, name=\"conv_1\"),\n", + " CustomConv2D(50, 5, strides=2, name=\"conv_2\"),\n", + " tf.keras.layers.Flatten(),\n", + " CustomDense(500, name=\"fc_1\"),\n", + " CustomDense(10, name=\"fc_2\"),\n", + "], name=\"classifier\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9iRSvt_CdUuY" + }, + "source": [ + "Load the training data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "L4bsA3HFF2k0" + }, + "outputs": [], + "source": [ + "def normalize_img(image, label):\n", + " \"\"\"Normalizes images: `uint8` -> `float32`.\"\"\"\n", + " return tf.cast(image, tf.float32) / 255., label\n", + "\n", + "training_dataset, validation_dataset = tfds.load(\n", + " \"mnist\",\n", + " split=[\"train\", \"test\"],\n", + " shuffle_files=True,\n", + " as_supervised=True,\n", + " with_info=False,\n", + ")\n", + "training_dataset = training_dataset.map(normalize_img)\n", + "validation_dataset = validation_dataset.map(normalize_img)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rR9WYjt_daRG" + }, + "source": [ + "Finally, train the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ROn2DbzsBirI" + }, + "outputs": [], + "source": [ + "def train_model(model, training_data, validation_data, **kwargs):\n", + " model.compile(\n", + " optimizer=tf.keras.optimizers.Adam(learning_rate=1e-3),\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", + " metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],\n", + " # Uncomment this to ease debugging:\n", + " # run_eagerly=True,\n", + " )\n", + " kwargs.setdefault(\"epochs\", 5)\n", + " kwargs.setdefault(\"verbose\", 1)\n", + " log = model.fit(\n", + " training_data.batch(128).prefetch(8),\n", + " validation_data=validation_data.batch(128).cache(),\n", + " validation_freq=1,\n", + " **kwargs,\n", + " )\n", + " return log.history[\"val_sparse_categorical_accuracy\"][-1]\n", + "\n", + "classifier_accuracy = train_model(\n", + " classifier, training_dataset, validation_dataset)\n", + "\n", + "print(f\"Accuracy: {classifier_accuracy:0.4f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QupWKZ91di-y" + }, + "source": [ + "Success! The model trained fine, and reached an accuracy of over 98% on the validation set within 5 epochs." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yRqZFwb5dqQm" + }, + "source": [ + "## Train a compressible classifier\n", + "\n", + "Entropy Penalized Reparameterization (EPR) has two main ingredients:\n", + "\n", + "- Applying a **penalty** to the model weights during training which corresponds to their entropy under a probabilistic model, which is matched with the encoding scheme of the weights. Below, we define a Keras `Regularizer` which implements this penalty.\n", + "\n", + "- **Reparameterizing** the weights, i.e. bringing them into a latent representation which is more compressible (yields a better trade-off between compressibility and model performance). For convolutional kernels, [it has been shown](https://arxiv.org/abs/1906.06624) that the Fourier domain is a good representation. For other parameters, the below example simply uses scalar quantization (rounding) with a varying quantization step size." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e4jmnqEmO6eB" + }, + "source": [ + "First, define the penalty.\n", + "\n", + "The example below uses a code/probabilistic model implemented in the `tfc.PowerLawEntropyModel` class, inspired by the paper [Optimizing the Communication-Accuracy Trade-off in Federated Learning with Rate-Distortion Theory](https://arxiv.org/abs/2201.02664). The penalty is defined as:\n", + "$$ \\log \\Bigl(\\frac {|x| + \\alpha} \\alpha\\Bigr), $$\n", + "where $x$ is one element of the model parameter or its latent representation, and $\\alpha$ is a small constant for numerical stability around values of 0." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hh57nxjuwocc" + }, + "outputs": [], + "source": [ + "_ = tf.linspace(-5., 5., 501)\n", + "plt.plot(_, tfc.PowerLawEntropyModel(0).penalty(_));\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Gr3-6vLrwo-H" + }, + "source": [ + "The penalty is effectively a regularization loss (sometimes called \"weight loss\"). The fact that it is concave with a cusp at zero encourages weight sparsity. The coding scheme applied for compressing the weights, an [Elias gamma code](https://en.wikipedia.org/wiki/Elias_gamma_coding), produces codes of length $ 1 + \\lfloor \\log_2 |x| \\rfloor $ bits for the magnitude of the element. That is, it is matched to the penalty, and applying the penalty thus minimizes the expected code length." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "H1Yt6e1ub6pU" + }, + "outputs": [], + "source": [ + "class PowerLawRegularizer(tf.keras.regularizers.Regularizer):\n", + "\n", + " def __init__(self, lmbda):\n", + " super().__init__()\n", + " self.lmbda = lmbda\n", + "\n", + " def __call__(self, variable):\n", + " em = tfc.PowerLawEntropyModel(coding_rank=variable.shape.rank)\n", + " return self.lmbda * em.penalty(variable)\n", + "\n", + "# Normalizing the weight of the penalty by the number of model parameters is a\n", + "# good rule of thumb to produce comparable results across models.\n", + "regularizer = PowerLawRegularizer(lmbda=2./classifier.count_params())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kyQc35QTf8Aq" + }, + "source": [ + "Second, define subclasses of `CustomDense` and `CustomConv2D` which have the following additional functionality:\n", + "\n", + "- They take an instance of the above regularizer and apply it to the kernels and biases during training.\n", + "- They define kernel and bias as a `@property`, which perform quantization with straight-through gradients whenever the variables are accessed. This accurately reflects the computation that is carried out later in the compressed model.\n", + "- They define additional `log_step` variables, which represent the logarithm of the quantization step size. The coarser the quantization, the smaller the model size, but the lower the accuracy. The quantization step sizes are trainable for each model parameter, so that performing optimization on the penalized loss function will determine what quantization step size is best.\n", + "\n", + "The quantization step is defined as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "60fMt3avgSFw" + }, + "outputs": [], + "source": [ + "def quantize(latent, log_step):\n", + " step = tf.exp(log_step)\n", + " return tfc.round_st(latent / step) * step\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "stKrchp7mB0b" + }, + "source": [ + "With that, we can define the dense layer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ciz1F1WsXre_" + }, + "outputs": [], + "source": [ + "class CompressibleDense(CustomDense):\n", + "\n", + " def __init__(self, regularizer, *args, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " self.regularizer = regularizer\n", + "\n", + " def build(self, input_shape, other=None):\n", + " \"\"\"Instantiates weights, optionally initializing them from `other`.\"\"\"\n", + " super().build(input_shape, other=other)\n", + " if other is not None and hasattr(other, \"kernel_log_step\"):\n", + " kernel_log_step = other.kernel_log_step\n", + " bias_log_step = other.bias_log_step\n", + " else:\n", + " kernel_log_step = bias_log_step = -4.\n", + " self.kernel_log_step = tf.Variable(\n", + " tf.cast(kernel_log_step, self.variable_dtype), name=\"kernel_log_step\")\n", + " self.bias_log_step = tf.Variable(\n", + " tf.cast(bias_log_step, self.variable_dtype), name=\"bias_log_step\")\n", + " self.add_loss(lambda: self.regularizer(\n", + " self.kernel_latent / tf.exp(self.kernel_log_step)))\n", + " self.add_loss(lambda: self.regularizer(\n", + " self.bias_latent / tf.exp(self.bias_log_step)))\n", + "\n", + " @property\n", + " def kernel(self):\n", + " return quantize(self.kernel_latent, self.kernel_log_step)\n", + "\n", + " @kernel.setter\n", + " def kernel(self, kernel):\n", + " self.kernel_latent = tf.Variable(kernel, name=\"kernel_latent\")\n", + "\n", + " @property\n", + " def bias(self):\n", + " return quantize(self.bias_latent, self.bias_log_step)\n", + "\n", + " @bias.setter\n", + " def bias(self, bias):\n", + " self.bias_latent = tf.Variable(bias, name=\"bias_latent\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CsykbQO0hxzW" + }, + "source": [ + "The convolutional layer is analogous. In addition, the convolution kernel is stored as its real-valued discrete Fourier transform (RDFT) whenever the kernel is set, and the transform is inverted whenever the kernel is used. Since the different frequency components of the kernel tend to be more or less compressible, each of them gets its own quantization step size assigned.\n", + "\n", + "Define the Fourier transform and its inverse as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rUFMKGHDguJS" + }, + "outputs": [], + "source": [ + "def to_rdft(kernel, kernel_size):\n", + " # The kernel has shape (H, W, I, O) -> transpose to take DFT over last two\n", + " # dimensions.\n", + " kernel = tf.transpose(kernel, (2, 3, 0, 1))\n", + " # The RDFT has type complex64 and shape (I, O, FH, FW).\n", + " kernel_rdft = tf.signal.rfft2d(kernel)\n", + " # Map real and imaginary parts into regular floats. The result is float32\n", + " # and has shape (I, O, FH, FW, 2).\n", + " kernel_rdft = tf.stack(\n", + " [tf.math.real(kernel_rdft), tf.math.imag(kernel_rdft)], axis=-1)\n", + " # Divide by kernel size to make the DFT orthonormal (length-preserving).\n", + " return kernel_rdft / kernel_size\n", + "\n", + "def from_rdft(kernel_rdft, kernel_size):\n", + " # Undoes the transformations in to_rdft.\n", + " kernel_rdft *= kernel_size\n", + " kernel_rdft = tf.dtypes.complex(*tf.unstack(kernel_rdft, axis=-1))\n", + " kernel = tf.signal.irfft2d(kernel_rdft, fft_length=2 * (kernel_size,))\n", + " return tf.transpose(kernel, (2, 3, 0, 1))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "esZZrJ5ImVDY" + }, + "source": [ + "With that, define the convolutional layer as:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YKzXBNCO7bjB" + }, + "outputs": [], + "source": [ + "class CompressibleConv2D(CustomConv2D):\n", + "\n", + " def __init__(self, regularizer, *args, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " self.regularizer = regularizer\n", + "\n", + " def build(self, input_shape, other=None):\n", + " \"\"\"Instantiates weights, optionally initializing them from `other`.\"\"\"\n", + " super().build(input_shape, other=other)\n", + " if other is not None and hasattr(other, \"kernel_log_step\"):\n", + " kernel_log_step = other.kernel_log_step\n", + " bias_log_step = other.bias_log_step\n", + " else:\n", + " kernel_log_step = tf.fill(self.kernel_latent.shape[2:], -4.)\n", + " bias_log_step = -4.\n", + " self.kernel_log_step = tf.Variable(\n", + " tf.cast(kernel_log_step, self.variable_dtype), name=\"kernel_log_step\")\n", + " self.bias_log_step = tf.Variable(\n", + " tf.cast(bias_log_step, self.variable_dtype), name=\"bias_log_step\")\n", + " self.add_loss(lambda: self.regularizer(\n", + " self.kernel_latent / tf.exp(self.kernel_log_step)))\n", + " self.add_loss(lambda: self.regularizer(\n", + " self.bias_latent / tf.exp(self.bias_log_step)))\n", + "\n", + " @property\n", + " def kernel(self):\n", + " kernel_rdft = quantize(self.kernel_latent, self.kernel_log_step)\n", + " return from_rdft(kernel_rdft, self.kernel_size)\n", + "\n", + " @kernel.setter\n", + " def kernel(self, kernel):\n", + " kernel_rdft = to_rdft(kernel, self.kernel_size)\n", + " self.kernel_latent = tf.Variable(kernel_rdft, name=\"kernel_latent\")\n", + "\n", + " @property\n", + " def bias(self):\n", + " return quantize(self.bias_latent, self.bias_log_step)\n", + "\n", + " @bias.setter\n", + " def bias(self, bias):\n", + " self.bias_latent = tf.Variable(bias, name=\"bias_latent\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1-ekDDQ9jidI" + }, + "source": [ + "Define a classifier model with the same architecture as above, but using these modified layers:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TQgp84L7qalw" + }, + "outputs": [], + "source": [ + "def make_mnist_classifier(regularizer):\n", + " return tf.keras.Sequential([\n", + " CompressibleConv2D(regularizer, 20, 5, strides=2, name=\"conv_1\"),\n", + " CompressibleConv2D(regularizer, 50, 5, strides=2, name=\"conv_2\"),\n", + " tf.keras.layers.Flatten(),\n", + " CompressibleDense(regularizer, 500, name=\"fc_1\"),\n", + " CompressibleDense(regularizer, 10, name=\"fc_2\"),\n", + " ], name=\"classifier\")\n", + "\n", + "compressible_classifier = make_mnist_classifier(regularizer)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hJ-TMHE1kNFc" + }, + "source": [ + "And train the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6L5ZJAX4EiXW" + }, + "outputs": [], + "source": [ + "penalized_accuracy = train_model(\n", + " compressible_classifier, training_dataset, validation_dataset)\n", + "\n", + "print(f\"Accuracy: {penalized_accuracy:0.4f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZuE4NeY_kTDz" + }, + "source": [ + "The compressible model has reached a similar accuracy as the plain classifier.\n", + "\n", + "However, the model is not actually compressed yet. To do this, we define another set of subclasses which store the kernels and biases in their compressed form – as a sequence of bits." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AZhj8A2gnBkD" + }, + "source": [ + "## Compress the classifier\n", + "\n", + "The subclasses of `CustomDense` and `CustomConv2D` defined below convert the weights of a compressible dense layer into binary strings. In addition, they store the logarithm of the quantization step size at half precision to save space. Whenever the kernel or bias is accessed through the `@property`, they are decompressed from their string representation and dequantized.\n", + "\n", + "First, define functions to compress and decompress a model parameter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xS19FhDajeto" + }, + "outputs": [], + "source": [ + "def compress_latent(latent, log_step, name):\n", + " em = tfc.PowerLawEntropyModel(latent.shape.rank)\n", + " compressed = em.compress(latent / tf.exp(log_step))\n", + " compressed = tf.Variable(compressed, name=f\"{name}_compressed\")\n", + " log_step = tf.cast(log_step, tf.float16)\n", + " log_step = tf.Variable(log_step, name=f\"{name}_log_step\")\n", + " return compressed, log_step\n", + "\n", + "def decompress_latent(compressed, shape, log_step):\n", + " latent = tfc.PowerLawEntropyModel(len(shape)).decompress(compressed, shape)\n", + " step = tf.exp(tf.cast(log_step, latent.dtype))\n", + " return latent * step\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bPPABE9fjqHJ" + }, + "source": [ + "With these, we can define `CompressedDense`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CnaiNzhgaZ7s" + }, + "outputs": [], + "source": [ + "class CompressedDense(CustomDense):\n", + "\n", + " def build(self, input_shape, other=None):\n", + " assert isinstance(other, CompressibleDense)\n", + " self.input_channels = other.kernel.shape[0]\n", + " self.kernel_compressed, self.kernel_log_step = compress_latent(\n", + " other.kernel_latent, other.kernel_log_step, \"kernel\")\n", + " self.bias_compressed, self.bias_log_step = compress_latent(\n", + " other.bias_latent, other.bias_log_step, \"bias\")\n", + " self.built = True\n", + "\n", + " @property\n", + " def kernel(self):\n", + " kernel_shape = (self.input_channels, self.filters)\n", + " return decompress_latent(\n", + " self.kernel_compressed, kernel_shape, self.kernel_log_step)\n", + "\n", + " @property\n", + " def bias(self):\n", + " bias_shape = (self.filters,)\n", + " return decompress_latent(\n", + " self.bias_compressed, bias_shape, self.bias_log_step)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tzvMCM0El2iW" + }, + "source": [ + "The convolutional layer class is analogous to the above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hS-2ADA6iWeQ" + }, + "outputs": [], + "source": [ + "class CompressedConv2D(CustomConv2D):\n", + "\n", + " def build(self, input_shape, other=None):\n", + " assert isinstance(other, CompressibleConv2D)\n", + " self.input_channels = other.kernel.shape[2]\n", + " self.kernel_compressed, self.kernel_log_step = compress_latent(\n", + " other.kernel_latent, other.kernel_log_step, \"kernel\")\n", + " self.bias_compressed, self.bias_log_step = compress_latent(\n", + " other.bias_latent, other.bias_log_step, \"bias\")\n", + " self.built = True\n", + "\n", + " @property\n", + " def kernel(self):\n", + " rdft_shape = (self.input_channels, self.filters,\n", + " self.kernel_size, self.kernel_size // 2 + 1, 2)\n", + " kernel_rdft = decompress_latent(\n", + " self.kernel_compressed, rdft_shape, self.kernel_log_step)\n", + " return from_rdft(kernel_rdft, self.kernel_size)\n", + "\n", + " @property\n", + " def bias(self):\n", + " bias_shape = (self.filters,)\n", + " return decompress_latent(\n", + " self.bias_compressed, bias_shape, self.bias_log_step)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cJLCPoe3l8jG" + }, + "source": [ + "To turn the compressible model into a compressed one, we can conveniently use the `clone_model` function. `compress_layer` converts any compressible layer into a compressed one, and simply passes through any other types of layers (such as `Flatten`, etc.).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WEHroUyhG56m" + }, + "outputs": [], + "source": [ + "def compress_layer(layer):\n", + " if isinstance(layer, CompressibleDense):\n", + " return CompressedDense.copy(layer)\n", + " if isinstance(layer, CompressibleConv2D):\n", + " return CompressedConv2D.copy(layer)\n", + " return type(layer).from_config(layer.get_config())\n", + "\n", + "compressed_classifier = tf.keras.models.clone_model(\n", + " compressible_classifier, clone_function=compress_layer)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "b3wbN1XQmkDg" + }, + "source": [ + "Now, let's validate that the compressed model still performs as expected:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "R95kuURITpa9" + }, + "outputs": [], + "source": [ + "compressed_classifier.compile(metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n", + "_, compressed_accuracy = compressed_classifier.evaluate(validation_dataset.batch(128))\n", + "\n", + "print(f\"Accuracy of the compressible classifier: {penalized_accuracy:0.4f}\")\n", + "print(f\"Accuracy of the compressed classifier: {compressed_accuracy:0.4f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KtFhpXh6uaIY" + }, + "source": [ + "The classification accuracy of the compressed model is identical to the one achieved during training!\n", + "\n", + "In addition, the size of the compressed model weights is much smaller than the original model size:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Qp-ecfuYufbs" + }, + "outputs": [], + "source": [ + "def get_weight_size_in_bytes(weight):\n", + " if weight.dtype == tf.string:\n", + " return tf.reduce_sum(tf.strings.length(weight, unit=\"BYTE\"))\n", + " else:\n", + " return tf.size(weight) * weight.dtype.size\n", + "\n", + "original_size = sum(map(get_weight_size_in_bytes, classifier.weights))\n", + "compressed_size = sum(map(get_weight_size_in_bytes, compressed_classifier.weights))\n", + "\n", + "print(f\"Size of original model weights: {original_size} bytes\")\n", + "print(f\"Size of compressed model weights: {compressed_size} bytes\")\n", + "print(f\"Compression ratio: {(original_size/compressed_size):0.0f}x\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K8A8v0df6TR2" + }, + "source": [ + "Storing the models on disk requires some overhead for storing the model architecture, function graphs, etc.\n", + "\n", + "Lossless compression methods such as ZIP are good at compressing this type of data, but not the weights themselves. That is why there is still a significant benefit of EPR when counting model size inclusive of that overhead, after also applying ZIP compression:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4hunDYxH1zqb" + }, + "outputs": [], + "source": [ + "import os\n", + "import shutil\n", + "\n", + "def get_disk_size(model, path):\n", + " model.save(path)\n", + " zip_path = shutil.make_archive(path, \"zip\", path)\n", + " return os.path.getsize(zip_path)\n", + "\n", + "original_zip_size = get_disk_size(classifier, \"/tmp/classifier\")\n", + "compressed_zip_size = get_disk_size(\n", + " compressed_classifier, \"/tmp/compressed_classifier\")\n", + "\n", + "print(f\"Original on-disk size (ZIP compressed): {original_zip_size} bytes\")\n", + "print(f\"Compressed on-disk size (ZIP compressed): {compressed_zip_size} bytes\")\n", + "print(f\"Compression ratio: {(original_zip_size/compressed_zip_size):0.0f}x\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FSITvJrlAhZs" + }, + "source": [ + "## Regularization effect and size–accuracy trade-off\n", + "\n", + "Above, the $\\lambda$ hyperparameter was set to 2 (normalized by the number of parameters in the model). As we increase $\\lambda$, the model weights are more and more heavily penalized for compressibility.\n", + "\n", + "For low values, the penalty can act like a weight regularizer. It actually has a beneficial effect on the generalization performance of the classifier, and can lead to a slightly higher accuracy on the validation dataset:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "4rhmKu98FdPJ" + }, + "outputs": [], + "source": [ + "#@title\n", + "\n", + "print(f\"Accuracy of the vanilla classifier: {classifier_accuracy:0.4f}\")\n", + "print(f\"Accuracy of the penalized classifier: {penalized_accuracy:0.4f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9UCfC4LQFdjL" + }, + "source": [ + "For higher values, we see a smaller and smaller model size, but also a gradually diminishing accuracy. To see this, let's train a few models and plot their size vs. accuracy:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "diApPKHbAIqa" + }, + "outputs": [], + "source": [ + "def compress_and_evaluate_model(lmbda):\n", + " print(f\"lambda={lmbda:0.0f}: training...\", flush=True)\n", + " regularizer = PowerLawRegularizer(lmbda=lmbda/classifier.count_params())\n", + " compressible_classifier = make_mnist_classifier(regularizer)\n", + " train_model(\n", + " compressible_classifier, training_dataset, validation_dataset, verbose=0)\n", + " print(\"compressing...\", flush=True)\n", + " compressed_classifier = tf.keras.models.clone_model(\n", + " compressible_classifier, clone_function=compress_layer)\n", + " compressed_size = sum(map(\n", + " get_weight_size_in_bytes, compressed_classifier.weights))\n", + " compressed_zip_size = float(get_disk_size(\n", + " compressed_classifier, \"/tmp/compressed_classifier\"))\n", + " print(\"evaluating...\", flush=True)\n", + " compressed_classifier = tf.keras.models.load_model(\n", + " \"/tmp/compressed_classifier\")\n", + " compressed_classifier.compile(\n", + " metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])\n", + " _, compressed_accuracy = compressed_classifier.evaluate(\n", + " validation_dataset.batch(128), verbose=0)\n", + " print()\n", + " return compressed_size, compressed_zip_size, compressed_accuracy\n", + "\n", + "lambdas = (2., 5., 10., 20., 50.)\n", + "metrics = [compress_and_evaluate_model(l) for l in lambdas]\n", + "metrics = tf.convert_to_tensor(metrics, tf.float32)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "bhAi85KzGqTz" + }, + "outputs": [], + "source": [ + "#@title\n", + "\n", + "def plot_broken_xaxis(ax, compressed_sizes, original_size, original_accuracy):\n", + " xticks = list(range(\n", + " int(tf.math.floor(min(compressed_sizes) / 5) * 5),\n", + " int(tf.math.ceil(max(compressed_sizes) / 5) * 5) + 1,\n", + " 5))\n", + " xticks.append(xticks[-1] + 10)\n", + " ax.set_xlim(xticks[0], xticks[-1] + 2)\n", + " ax.set_xticks(xticks[1:])\n", + " ax.set_xticklabels(xticks[1:-1] + [f\"{original_size:0.2f}\"])\n", + " ax.plot(xticks[-1], original_accuracy, \"o\", label=\"float32\")\n", + "\n", + "sizes, zip_sizes, accuracies = tf.transpose(metrics)\n", + "sizes /= 1024\n", + "zip_sizes /= 1024\n", + "\n", + "fig, (axl, axr) = plt.subplots(1, 2, sharey=True, figsize=(10, 4))\n", + "axl.plot(sizes, accuracies, \"o-\", label=\"EPR compressed\")\n", + "axr.plot(zip_sizes, accuracies, \"o-\", label=\"EPR compressed\")\n", + "plot_broken_xaxis(axl, sizes, original_size/1024, classifier_accuracy)\n", + "plot_broken_xaxis(axr, zip_sizes, original_zip_size/1024, classifier_accuracy)\n", + "\n", + "axl.set_xlabel(\"size of model weights [kbytes]\")\n", + "axr.set_xlabel(\"ZIP compressed on-disk model size [kbytes]\")\n", + "axl.set_ylabel(\"accuracy\")\n", + "axl.legend(loc=\"lower right\")\n", + "axr.legend(loc=\"lower right\")\n", + "axl.grid()\n", + "axr.grid()\n", + "for i in range(len(lambdas)):\n", + " axl.annotate(f\"$\\lambda = {lambdas[i]:0.0f}$\", (sizes[i], accuracies[i]),\n", + " xytext=(10, -5), xycoords=\"data\", textcoords=\"offset points\")\n", + " axr.annotate(f\"$\\lambda = {lambdas[i]:0.0f}$\", (zip_sizes[i], accuracies[i]),\n", + " xytext=(10, -5), xycoords=\"data\", textcoords=\"offset points\")\n", + "plt.tight_layout()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ajrHaFTAaLd2" + }, + "source": [ + "The plot should ideally show an elbow-shaped size–accuracy trade-off, but it is normal for accuracy metrics to be somewhat noisy. Depending on initialization, the curve can exhibit some kinks.\n", + "\n", + "Due to the regularization effect, the EPR compressed model is more accurate on the test set than the original model for small values of $\\lambda$. The EPR compressed model is also many times smaller, even if we compare the sizes after additional ZIP compression." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-RBhdXZTzoWw" + }, + "source": [ + "## Decompress the classifier\n", + "\n", + "`CompressedDense` and `CompressedConv2D` decompress their weights on every forward pass. This makes them ideal for memory-limited devices, but the decompression can be computationally expensive, especially for small batch sizes.\n", + "\n", + "To decompress the model once, and use it for further training or inference, we can convert it back into a model using regular or compressible layers. This can be useful in model deployment or federated learning scenarios.\n", + "\n", + "First, converting back into a plain model, we can do inference, and/or continue regular training without a compression penalty:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QBB2-X5XzvwB" + }, + "outputs": [], + "source": [ + "def decompress_layer(layer):\n", + " if isinstance(layer, CompressedDense):\n", + " return CustomDense.copy(layer)\n", + " if isinstance(layer, CompressedConv2D):\n", + " return CustomConv2D.copy(layer)\n", + " return type(layer).from_config(layer.get_config())\n", + "\n", + "decompressed_classifier = tf.keras.models.clone_model(\n", + " compressed_classifier, clone_function=decompress_layer)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ehE2ov8U0p0G" + }, + "outputs": [], + "source": [ + "decompressed_accuracy = train_model(\n", + " decompressed_classifier, training_dataset, validation_dataset, epochs=1)\n", + "\n", + "print(f\"Accuracy of the compressed classifier: {compressed_accuracy:0.4f}\")\n", + "print(f\"Accuracy of the decompressed classifier after one more epoch of training: {decompressed_accuracy:0.4f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jiSCvemQ04o8" + }, + "source": [ + "Note that the validation accuracy drops after training for an additional epoch, since the training is done without regularization.\n", + "\n", + "Alternatively, we can convert the model back into a \"compressible\" one, for inference and/or further training with a compression penalty:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JDppVUdx1BvY" + }, + "outputs": [], + "source": [ + "def decompress_layer_with_penalty(layer):\n", + " if isinstance(layer, CompressedDense):\n", + " return CompressibleDense.copy(layer, regularizer=regularizer)\n", + " if isinstance(layer, CompressedConv2D):\n", + " return CompressibleConv2D.copy(layer, regularizer=regularizer)\n", + " return type(layer).from_config(layer.get_config())\n", + "\n", + "decompressed_classifier = tf.keras.models.clone_model(\n", + " compressed_classifier, clone_function=decompress_layer_with_penalty)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AJcnyOFW2IcK" + }, + "outputs": [], + "source": [ + "decompressed_accuracy = train_model(\n", + " decompressed_classifier, training_dataset, validation_dataset, epochs=1)\n", + "\n", + "print(f\"Accuracy of the compressed classifier: {compressed_accuracy:0.4f}\")\n", + "print(f\"Accuracy of the decompressed classifier after one more epoch of training: {decompressed_accuracy:0.4f}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ciol315T_TwQ" + }, + "source": [ + "Here, the accuracy improves after training for an additional epoch." + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "Tce3stUlHN0L", + "xHxb-dlhMIzW" + ], + "name": "compression.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/quickstart/advanced.ipynb b/site/en/tutorials/quickstart/advanced.ipynb index ab9f59f91dd..7cc134b2613 100644 --- a/site/en/tutorials/quickstart/advanced.ipynb +++ b/site/en/tutorials/quickstart/advanced.ipynb @@ -1,418 +1,358 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "rX8mhOLljYeM" + }, + "source": [ + "##### Copyright 2019 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "BZSlp3DAjdYf" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3wF5wszaj97Y" + }, + "source": [ + "# TensorFlow 2 quickstart for experts" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DUNzJc4jTj6G" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hiH7AC-NTniF" + }, + "source": [ + "This is a [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) notebook file. Python programs are run directly in the browser—a great way to learn and use TensorFlow. To follow this tutorial, run the notebook in Google Colab by clicking the button at the top of this page.\n", + "\n", + "1. In Colab, connect to a Python runtime: At the top-right of the menu bar, select *CONNECT*.\n", + "2. Run all the notebook code cells: Select *Runtime* > *Run all*." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eOsVdx6GGHmU" + }, + "source": [ + "Download and install TensorFlow 2. Import TensorFlow into your program:\n", + "\n", + "Note: Upgrade `pip` to install the TensorFlow 2 package. See the [install guide](https://www.tensorflow.org/install) for details." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QS7DDTiZGRTo" + }, + "source": [ + "Import TensorFlow into your program:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0trJmd6DjqBZ" + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "print(\"TensorFlow version:\", tf.__version__)\n", + "\n", + "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", + "from tensorflow.keras import Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7NAbSZiaoJ4z" + }, + "source": [ + "Load and prepare the [MNIST dataset](http://yann.lecun.com/exdb/mnist/)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JqFRS6K07jJs" + }, + "outputs": [], + "source": [ + "mnist = tf.keras.datasets.mnist\n", + "\n", + "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", + "x_train, x_test = x_train / 255.0, x_test / 255.0\n", + "\n", + "# Add a channels dimension\n", + "x_train = x_train[..., tf.newaxis].astype(\"float32\")\n", + "x_test = x_test[..., tf.newaxis].astype(\"float32\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "k1Evqx0S22r_" + }, + "source": [ + "Use `tf.data` to batch and shuffle the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8Iu_quO024c2" + }, + "outputs": [], + "source": [ + "train_ds = tf.data.Dataset.from_tensor_slices(\n", + " (x_train, y_train)).shuffle(10000).batch(32)\n", + "\n", + "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BPZ68wASog_I" + }, + "source": [ + "Build the `tf.keras` model using the Keras [model subclassing API](https://www.tensorflow.org/guide/keras/custom_layers_and_models):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "h3IKyzTCDNGo" + }, + "outputs": [], + "source": [ + "class MyModel(Model):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.conv1 = Conv2D(32, 3, activation='relu')\n", + " self.flatten = Flatten()\n", + " self.d1 = Dense(128, activation='relu')\n", + " self.d2 = Dense(10)\n", + "\n", + " def call(self, x):\n", + " x = self.conv1(x)\n", + " x = self.flatten(x)\n", + " x = self.d1(x)\n", + " return self.d2(x)\n", + "\n", + "# Create an instance of the model\n", + "model = MyModel()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uGih-c2LgbJu" + }, + "source": [ + "Choose an optimizer and loss function for training:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "u48C9WQ774n4" + }, + "outputs": [], + "source": [ + "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", + "\n", + "optimizer = tf.keras.optimizers.Adam()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JB6A1vcigsIe" + }, + "source": [ + "Select metrics to measure the loss and the accuracy of the model. These metrics accumulate the values over epochs and then print the overall result." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "N0MqHFb4F_qn" + }, + "outputs": [], + "source": [ + "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", + "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", + "\n", + "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", + "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ix4mEL65on-w" + }, + "source": [ + "Use `tf.GradientTape` to train the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OZACiVqA8KQV" + }, + "outputs": [], + "source": [ + "@tf.function\n", + "def train_step(images, labels):\n", + " with tf.GradientTape() as tape:\n", + " # training=True is only needed if there are layers with different\n", + " # behavior during training versus inference (e.g. Dropout).\n", + " predictions = model(images, training=True)\n", + " loss = loss_object(labels, predictions)\n", + " gradients = tape.gradient(loss, model.trainable_variables)\n", + " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", + "\n", + " train_loss(loss)\n", + " train_accuracy(labels, predictions)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z8YT7UmFgpjV" + }, + "source": [ + "Test the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xIKdEzHAJGt7" + }, + "outputs": [], + "source": [ + "@tf.function\n", + "def test_step(images, labels):\n", + " # training=False is only needed if there are layers with different\n", + " # behavior during training versus inference (e.g. Dropout).\n", + " predictions = model(images, training=False)\n", + " t_loss = loss_object(labels, predictions)\n", + "\n", + " test_loss(t_loss)\n", + " test_accuracy(labels, predictions)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "i-2pkctU_Ci7" + }, + "outputs": [], + "source": [ + "EPOCHS = 5\n", + "\n", + "for epoch in range(EPOCHS):\n", + " # Reset the metrics at the start of the next epoch\n", + " train_loss.reset_state()\n", + " train_accuracy.reset_state()\n", + " test_loss.reset_state()\n", + " test_accuracy.reset_state()\n", + "\n", + " for images, labels in train_ds:\n", + " train_step(images, labels)\n", + "\n", + " for test_images, test_labels in test_ds:\n", + " test_step(test_images, test_labels)\n", + "\n", + " print(\n", + " f'Epoch {epoch + 1}, '\n", + " f'Loss: {train_loss.result():0.2f}, '\n", + " f'Accuracy: {train_accuracy.result() * 100:0.2f}, '\n", + " f'Test Loss: {test_loss.result():0.2f}, '\n", + " f'Test Accuracy: {test_accuracy.result() * 100:0.2f}'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T4JfEh7kvx6m" + }, + "source": [ + "The image classifier is now trained to ~98% accuracy on this dataset. To learn more, read the [TensorFlow tutorials](https://www.tensorflow.org/tutorials)." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "advanced.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "BZSlp3DAjdYf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# TensorFlow 2 quickstart for experts" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "This is a [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) notebook file. Python programs are run directly in the browser—a great way to learn and use TensorFlow. To follow this tutorial, run the notebook in Google Colab by clicking the button at the top of this page.\n", - "\n", - "1. In Colab, connect to a Python runtime: At the top-right of the menu bar, select *CONNECT*.\n", - "2. Run all the notebook code cells: Select *Runtime* > *Run all*." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eOsVdx6GGHmU" - }, - "source": [ - "Download and install the TensorFlow 2 package:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ioLbtB3uGKPX" - }, - "outputs": [], - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QS7DDTiZGRTo" - }, - "source": [ - "Import TensorFlow into your program:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0trJmd6DjqBZ" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Load and prepare the [MNIST dataset](http://yann.lecun.com/exdb/mnist/)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JqFRS6K07jJs" - }, - "outputs": [], - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# Add a channels dimension\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "Use `tf.data` to batch and shuffle the dataset:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8Iu_quO024c2" - }, - "outputs": [], - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Build the `tf.keras` model using the Keras [model subclassing API](https://www.tensorflow.org/guide/keras#model_subclassing):" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "h3IKyzTCDNGo" - }, - "outputs": [], - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "# Create an instance of the model\n", - "model = MyModel()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "Choose an optimizer and loss function for training: " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "u48C9WQ774n4" - }, - "outputs": [], - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "Select metrics to measure the loss and the accuracy of the model. These metrics accumulate the values over epochs and then print the overall result." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "N0MqHFb4F_qn" - }, - "outputs": [], - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Use `tf.GradientTape` to train the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OZACiVqA8KQV" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z8YT7UmFgpjV" - }, - "source": [ - "Test the model:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xIKdEzHAJGt7" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "i-2pkctU_Ci7" - }, - "outputs": [], - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " # Reset the metrics at the start of the next epoch\n", - " train_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_loss.reset_states()\n", - " test_accuracy.reset_states()\n", - "\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n", - " print(template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "The image classifier is now trained to ~98% accuracy on this dataset. To learn more, read the [TensorFlow tutorials](https://www.tensorflow.org/tutorials)." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "advanced.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.5rc1" - } - }, - "nbformat": 4, - "nbformat_minor": 1 + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/site/en/tutorials/quickstart/beginner.ipynb b/site/en/tutorials/quickstart/beginner.ipynb index ad40362ddf9..6a7811cfde4 100644 --- a/site/en/tutorials/quickstart/beginner.ipynb +++ b/site/en/tutorials/quickstart/beginner.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rX8mhOLljYeM" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "BZSlp3DAjdYf" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3wF5wszaj97Y" }, "source": [ @@ -47,101 +43,95 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DUNzJc4jTj6G" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/quickstart/beginner\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/quickstart/beginner.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "04QgGZc9bF5D" }, "source": [ "This short introduction uses [Keras](https://www.tensorflow.org/guide/keras/overview) to:\n", "\n", - "1. Build a neural network that classifies images.\n", + "1. Load a prebuilt dataset.\n", + "1. Build a neural network machine learning model that classifies images.\n", "2. Train this neural network.\n", - "3. And, finally, evaluate the accuracy of the model." + "3. Evaluate the accuracy of the model." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hiH7AC-NTniF" }, "source": [ - "This is a [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) notebook file. Python programs are run directly in the browser—a great way to learn and use TensorFlow. To follow this tutorial, run the notebook in Google Colab by clicking the button at the top of this page.\n", + "This tutorial is a [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) notebook. Python programs are run directly in the browser—a great way to learn and use TensorFlow. To follow this tutorial, run the notebook in Google Colab by clicking the button at the top of this page.\n", "\n", "1. In Colab, connect to a Python runtime: At the top-right of the menu bar, select *CONNECT*.\n", - "2. Run all the notebook code cells: Select *Runtime* \u003e *Run all*." + "2. To run all the code in the notebook, select **Runtime** > **Run all**. To run the code cells one at a time, hover over each cell and select the **Run cell** icon.\n", + "\n", + "![Run cell icon](images/beginner/run_cell_icon.png)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "nnrWf3PCEzXL" }, "source": [ - "Download and install the TensorFlow 2 package. Import TensorFlow into your program:" + "## Set up TensorFlow\n", + "\n", + "Import TensorFlow into your program to get started:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0trJmd6DjqBZ" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# Install TensorFlow\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf" + "import tensorflow as tf\n", + "print(\"TensorFlow version:\", tf.__version__)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7NAbSZiaoJ4z" }, "source": [ - "Load and prepare the [MNIST dataset](http://yann.lecun.com/exdb/mnist/). Convert the samples from integers to floating-point numbers:" + "If you are following along in your own development environment, rather than [Colab](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb), see the [install guide](https://www.tensorflow.org/install) for setting up TensorFlow for development.\n", + "\n", + "Note: Make sure you have upgraded to the latest `pip` to install the TensorFlow 2 package if you are using your own development environment. See the [install guide](https://www.tensorflow.org/install) for details.\n", + "\n", + "## Load a dataset\n", + "\n", + "Load and prepare the MNIST dataset. The pixel values of the images range from 0 through 255. Scale these values to a range of 0 to 1 by dividing the values by `255.0`. This also converts the sample data from integers to floating-point numbers:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7FP5258xjs-v" }, "outputs": [], @@ -155,19 +145,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "BPZ68wASog_I" }, "source": [ - "Build the `tf.keras.Sequential` model by stacking layers. Choose an optimizer and loss function for training:" + "## Build a machine learning model\n", + "\n", + "Build a `tf.keras.Sequential` model:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "h3IKyzTCDNGo" }, "outputs": [], @@ -176,58 +165,228 @@ " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", " tf.keras.layers.Dense(128, activation='relu'),\n", " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", + " tf.keras.layers.Dense(10)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "l2hiez2eIUz8" + }, + "source": [ + "[`Sequential`](https://www.tensorflow.org/guide/keras/sequential_model) is useful for stacking layers where each layer has one input [tensor](https://www.tensorflow.org/guide/tensor) and one output tensor. Layers are functions with a known mathematical structure that can be reused and have trainable variables. Most TensorFlow models are composed of layers. This model uses the [`Flatten`](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Flatten), [`Dense`](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense), and [`Dropout`](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dropout) layers.\n", + "\n", + "For each example, the model returns a vector of [logits](https://developers.google.com/machine-learning/glossary#logits) or [log-odds](https://developers.google.com/machine-learning/glossary#log-odds) scores, one for each class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OeOrNdnkEEcR" + }, + "outputs": [], + "source": [ + "predictions = model(x_train[:1]).numpy()\n", + "predictions" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tgjhDQGcIniO" + }, + "source": [ + "The `tf.nn.softmax` function converts these logits to *probabilities* for each class: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "zWSRnQ0WI5eq" + }, + "outputs": [], + "source": [ + "tf.nn.softmax(predictions).numpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "he5u_okAYS4a" + }, + "source": [ + "Note: It is possible to bake the `tf.nn.softmax` function into the activation function for the last layer of the network. While this can make the model output more directly interpretable, this approach is discouraged as it's impossible to provide an exact and numerically stable loss calculation for all models when using a softmax output. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hQyugpgRIyrA" + }, + "source": [ + "Define a loss function for training using `losses.SparseCategoricalCrossentropy`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RSkzdv8MD0tT" + }, + "outputs": [], + "source": [ + "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SfR4MsSDU880" + }, + "source": [ + "The loss function takes a vector of ground truth values and a vector of logits and returns a scalar loss for each example. This loss is equal to the negative log probability of the true class: The loss is zero if the model is sure of the correct class.\n", "\n", + "This untrained model gives probabilities close to random (1/10 for each class), so the initial loss should be close to `-tf.math.log(1/10) ~= 2.3`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NJWqEVrrJ7ZB" + }, + "outputs": [], + "source": [ + "loss_fn(y_train[:1], predictions).numpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ada44eb947d4" + }, + "source": [ + "Before you start training, configure and compile the model using Keras `Model.compile`. Set the [`optimizer`](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers) class to `adam`, set the `loss` to the `loss_fn` function you defined earlier, and specify a metric to be evaluated for the model by setting the `metrics` parameter to `accuracy`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9foNKHzTD2Vo" + }, + "outputs": [], + "source": [ "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", + " loss=loss_fn,\n", " metrics=['accuracy'])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ix4mEL65on-w" }, "source": [ - "Train and evaluate the model:" + "## Train and evaluate your model\n", + "\n", + "Use the `Model.fit` method to adjust your model parameters and minimize the loss: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "y7suUbJXVLqP" + }, + "outputs": [], + "source": [ + "model.fit(x_train, y_train, epochs=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4mDAAPFqVVgn" + }, + "source": [ + "The `Model.evaluate` method checks the model's performance, usually on a [validation set](https://developers.google.com/machine-learning/glossary#validation-set) or [test set](https://developers.google.com/machine-learning/glossary#test-set)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "F7dTAzgHDUh7" }, "outputs": [], "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", "model.evaluate(x_test, y_test, verbose=2)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "T4JfEh7kvx6m" }, "source": [ "The image classifier is now trained to ~98% accuracy on this dataset. To learn more, read the [TensorFlow tutorials](https://www.tensorflow.org/tutorials/)." ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Aj8NrlzlJqDG" + }, + "source": [ + "If you want your model to return a probability, you can wrap the trained model, and attach the softmax to it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rYb6DrEH0GMv" + }, + "outputs": [], + "source": [ + "probability_model = tf.keras.Sequential([\n", + " model,\n", + " tf.keras.layers.Softmax()\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cnqOZtUp1YR_" + }, + "outputs": [], + "source": [ + "probability_model(x_test[:5])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-47O6_GLdRuT" + }, + "source": [ + "## Conclusion\n", + "\n", + "Congratulations! You have trained a machine learning model using a prebuilt dataset using the [Keras](https://www.tensorflow.org/guide/keras/overview) API.\n", + "\n", + "For more examples of using Keras, check out the [tutorials](https://www.tensorflow.org/tutorials/keras/). To learn more about building models with Keras, read the [guides](https://www.tensorflow.org/guide/keras). If you want learn more about loading and preparing data, see the tutorials on [image data loading](https://www.tensorflow.org/tutorials/load_data/images) or [CSV data loading](https://www.tensorflow.org/tutorials/load_data/csv).\n" + ] } ], "metadata": { "colab": { - "collapsed_sections": [ - "rX8mhOLljYeM" - ], "name": "beginner.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/quickstart/images/beginner/run_cell_icon.png b/site/en/tutorials/quickstart/images/beginner/run_cell_icon.png new file mode 100644 index 00000000000..92f811bb95d Binary files /dev/null and b/site/en/tutorials/quickstart/images/beginner/run_cell_icon.png differ diff --git a/site/en/tutorials/reinforcement_learning/actor_critic.ipynb b/site/en/tutorials/reinforcement_learning/actor_critic.ipynb new file mode 100644 index 00000000000..a13c3fe44e4 --- /dev/null +++ b/site/en/tutorials/reinforcement_learning/actor_critic.ipynb @@ -0,0 +1,762 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "_jQ1tEQCxwRx" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "V_sgB_5dx1f1" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p62G8M_viUJp" + }, + "source": [ + "# Playing CartPole with the Actor-Critic method\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-mJ2i6jvZ3sK" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kFgN7h_wiUJq" + }, + "source": [ + "This tutorial demonstrates how to implement the [Actor-Critic](https://papers.nips.cc/paper/1786-actor-critic-algorithms.pdf) method using TensorFlow to train an agent on the [Open AI Gym](https://www.gymlibrary.dev/) [`CartPole-v0`](https://www.gymlibrary.dev/environments/classic_control/cart_pole/) environment.\n", + "The reader is assumed to have some familiarity with [policy gradient methods](https://papers.nips.cc/paper/1713-policy-gradient-methods-for-reinforcement-learning-with-function-approximation.pdf) of [(deep) reinforcement learning](https://en.wikipedia.org/wiki/Deep_reinforcement_learning).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_kA10ZKRR0hi" + }, + "source": [ + "**Actor-Critic methods**\n", + "\n", + "Actor-Critic methods are [temporal difference (TD) learning](https://en.wikipedia.org/wiki/Temporal_difference_learning) methods that represent the policy function independent of the value function.\n", + "\n", + "A policy function (or policy) returns a probability distribution over actions that the agent can take based on the given state.\n", + "A value function determines the expected return for an agent starting at a given state and acting according to a particular policy forever after.\n", + "\n", + "In the Actor-Critic method, the policy is referred to as the *actor* that proposes a set of possible actions given a state, and the estimated value function is referred to as the *critic*, which evaluates actions taken by the *actor* based on the given policy.\n", + "\n", + "In this tutorial, both the *Actor* and *Critic* will be represented using one neural network with two outputs.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rBfiafKSRs2k" + }, + "source": [ + "**`CartPole-v0`**\n", + "\n", + "In the [`CartPole-v0` environment](https://www.gymlibrary.dev/environments/classic_control/cart_pole/), a pole is attached to a cart moving along a frictionless track.\n", + "The pole starts upright and the goal of the agent is to prevent it from falling over by applying a force of `-1` or `+1` to the cart.\n", + "A reward of `+1` is given for every time step the pole remains upright.\n", + "An episode ends when: 1) the pole is more than 15 degrees from vertical; or 2) the cart moves more than 2.4 units from the center.\n", + "\n", + "
    \n", + "
    \n", + " \n", + "
    \n", + " Trained actor-critic model in Cartpole-v0 environment\n", + "
    \n", + "
    \n", + "
    \n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XSNVK0AeRoJd" + }, + "source": [ + "The problem is considered \"solved\" when the average total reward for the episode reaches 195 over 100 consecutive trials." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "glLwIctHiUJq" + }, + "source": [ + "## Setup\n", + "\n", + "Import necessary packages and configure global settings.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "13l6BbxKhCKp" + }, + "outputs": [], + "source": [ + "!pip install gym[classic_control]\n", + "!pip install pyglet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WBeQhPi2S4m5" + }, + "outputs": [], + "source": [ + "%%bash\n", + "# Install additional packages for visualization\n", + "sudo apt-get install -y python-opengl > /dev/null 2>&1\n", + "pip install git+https://github.com/tensorflow/docs > /dev/null 2>&1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tT4N3qYviUJr" + }, + "outputs": [], + "source": [ + "import collections\n", + "import gym\n", + "import numpy as np\n", + "import statistics\n", + "import tensorflow as tf\n", + "import tqdm\n", + "\n", + "from matplotlib import pyplot as plt\n", + "from tensorflow.keras import layers\n", + "from typing import Any, List, Sequence, Tuple\n", + "\n", + "\n", + "# Create the environment\n", + "env = gym.make(\"CartPole-v1\")\n", + "\n", + "# Set seed for experiment reproducibility\n", + "seed = 42\n", + "tf.random.set_seed(seed)\n", + "np.random.seed(seed)\n", + "\n", + "# Small epsilon value for stabilizing division operations\n", + "eps = np.finfo(np.float32).eps.item()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AOUCe2D0iUJu" + }, + "source": [ + "## The model\n", + "\n", + "The *Actor* and *Critic* will be modeled using one neural network that generates the action probabilities and Critic value respectively. This tutorial uses model subclassing to define the model.\n", + "\n", + "During the forward pass, the model will take in the state as the input and will output both action probabilities and critic value $V$, which models the state-dependent [value function](https://spinningup.openai.com/en/latest/spinningup/rl_intro.html#value-functions). The goal is to train a model that chooses actions based on a policy $\\pi$ that maximizes expected [return](https://spinningup.openai.com/en/latest/spinningup/rl_intro.html#reward-and-return).\n", + "\n", + "For `CartPole-v0`, there are four values representing the state: cart position, cart-velocity, pole angle and pole velocity respectively. The agent can take two actions to push the cart left (`0`) and right (`1`), respectively.\n", + "\n", + "Refer to [Gym's Cart Pole documentation page](https://www.gymlibrary.dev/environments/classic_control/cart_pole/) and [_Neuronlike adaptive elements that can solve difficult learning control problems_](http://www.derongliu.org/adp/adp-cdrom/Barto1983.pdf) by Barto, Sutton and Anderson (1983) for more information.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aXKbbMC-kmuv" + }, + "outputs": [], + "source": [ + "class ActorCritic(tf.keras.Model):\n", + " \"\"\"Combined actor-critic network.\"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " num_actions: int,\n", + " num_hidden_units: int):\n", + " \"\"\"Initialize.\"\"\"\n", + " super().__init__()\n", + "\n", + " self.common = layers.Dense(num_hidden_units, activation=\"relu\")\n", + " self.actor = layers.Dense(num_actions)\n", + " self.critic = layers.Dense(1)\n", + "\n", + " def call(self, inputs: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]:\n", + " x = self.common(inputs)\n", + " return self.actor(x), self.critic(x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nWyxJgjLn68c" + }, + "outputs": [], + "source": [ + "num_actions = env.action_space.n # 2\n", + "num_hidden_units = 128\n", + "\n", + "model = ActorCritic(num_actions, num_hidden_units)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hk92njFziUJw" + }, + "source": [ + "## Train the agent\n", + "\n", + "To train the agent, you will follow these steps:\n", + "\n", + "1. Run the agent on the environment to collect training data per episode.\n", + "2. Compute expected return at each time step.\n", + "3. Compute the loss for the combined Actor-Critic model.\n", + "4. Compute gradients and update network parameters.\n", + "5. Repeat 1-4 until either success criterion or max episodes has been reached.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R2nde2XDs8Gh" + }, + "source": [ + "### 1. Collect training data\n", + "\n", + "As in supervised learning, in order to train the actor-critic model, you need\n", + "to have training data. However, in order to collect such data, the model would\n", + "need to be \"run\" in the environment.\n", + "\n", + "Training data is collected for each episode. Then at each time step, the model's forward pass will be run on the environment's state in order to generate action probabilities and the critic value based on the current policy parameterized by the model's weights.\n", + "\n", + "The next action will be sampled from the action probabilities generated by the model, which would then be applied to the environment, causing the next state and reward to be generated.\n", + "\n", + "This process is implemented in the `run_episode` function, which uses TensorFlow operations so that it can later be compiled into a TensorFlow graph for faster training. Note that `tf.TensorArray`s were used to support Tensor iteration on variable length arrays." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5URrbGlDSAGx" + }, + "outputs": [], + "source": [ + "# Wrap Gym's `env.step` call as an operation in a TensorFlow function.\n", + "# This would allow it to be included in a callable TensorFlow graph.\n", + "\n", + "@tf.numpy_function(Tout=[tf.float32, tf.int32, tf.int32])\n", + "def env_step(action: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n", + " \"\"\"Returns state, reward and done flag given an action.\"\"\"\n", + "\n", + " state, reward, done, truncated, info = env.step(action)\n", + " return (state.astype(np.float32),\n", + " np.array(reward, np.int32),\n", + " np.array(done, np.int32))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "a4qVRV063Cl9" + }, + "outputs": [], + "source": [ + "def run_episode(\n", + " initial_state: tf.Tensor,\n", + " model: tf.keras.Model,\n", + " max_steps: int) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor]:\n", + " \"\"\"Runs a single episode to collect training data.\"\"\"\n", + "\n", + " action_probs = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)\n", + " values = tf.TensorArray(dtype=tf.float32, size=0, dynamic_size=True)\n", + " rewards = tf.TensorArray(dtype=tf.int32, size=0, dynamic_size=True)\n", + "\n", + " initial_state_shape = initial_state.shape\n", + " state = initial_state\n", + "\n", + " for t in tf.range(max_steps):\n", + " # Convert state into a batched tensor (batch size = 1)\n", + " state = tf.expand_dims(state, 0)\n", + "\n", + " # Run the model and to get action probabilities and critic value\n", + " action_logits_t, value = model(state)\n", + "\n", + " # Sample next action from the action probability distribution\n", + " action = tf.random.categorical(action_logits_t, 1)[0, 0]\n", + " action_probs_t = tf.nn.softmax(action_logits_t)\n", + "\n", + " # Store critic values\n", + " values = values.write(t, tf.squeeze(value))\n", + "\n", + " # Store log probability of the action chosen\n", + " action_probs = action_probs.write(t, action_probs_t[0, action])\n", + "\n", + " # Apply action to the environment to get next state and reward\n", + " state, reward, done = env_step(action)\n", + " state.set_shape(initial_state_shape)\n", + "\n", + " # Store reward\n", + " rewards = rewards.write(t, reward)\n", + "\n", + " if tf.cast(done, tf.bool):\n", + " break\n", + "\n", + " action_probs = action_probs.stack()\n", + " values = values.stack()\n", + " rewards = rewards.stack()\n", + "\n", + " return action_probs, values, rewards" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lBnIHdz22dIx" + }, + "source": [ + "### 2. Compute the expected returns\n", + "\n", + "The sequence of rewards for each timestep $t$, $\\{r_{t}\\}^{T}_{t=1}$ collected during one episode is converted into a sequence of expected returns $\\{G_{t}\\}^{T}_{t=1}$ in which the sum of rewards is taken from the current timestep $t$ to $T$ and each reward is multiplied with an exponentially decaying discount factor $\\gamma$:\n", + "\n", + "$$G_{t} = \\sum^{T}_{t'=t} \\gamma^{t'-t}r_{t'}$$\n", + "\n", + "Since $\\gamma\\in(0,1)$, rewards further out from the current timestep are given less weight.\n", + "\n", + "Intuitively, expected return simply implies that rewards now are better than rewards later. In a mathematical sense, it is to ensure that the sum of the rewards converges.\n", + "\n", + "To stabilize training, the resulting sequence of returns is also standardized (i.e. to have zero mean and unit standard deviation).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jpEwFyl315dl" + }, + "outputs": [], + "source": [ + "def get_expected_return(\n", + " rewards: tf.Tensor,\n", + " gamma: float,\n", + " standardize: bool = True) -> tf.Tensor:\n", + " \"\"\"Compute expected returns per timestep.\"\"\"\n", + "\n", + " n = tf.shape(rewards)[0]\n", + " returns = tf.TensorArray(dtype=tf.float32, size=n)\n", + "\n", + " # Start from the end of `rewards` and accumulate reward sums\n", + " # into the `returns` array\n", + " rewards = tf.cast(rewards[::-1], dtype=tf.float32)\n", + " discounted_sum = tf.constant(0.0)\n", + " discounted_sum_shape = discounted_sum.shape\n", + " for i in tf.range(n):\n", + " reward = rewards[i]\n", + " discounted_sum = reward + gamma * discounted_sum\n", + " discounted_sum.set_shape(discounted_sum_shape)\n", + " returns = returns.write(i, discounted_sum)\n", + " returns = returns.stack()[::-1]\n", + "\n", + " if standardize:\n", + " returns = ((returns - tf.math.reduce_mean(returns)) /\n", + " (tf.math.reduce_std(returns) + eps))\n", + "\n", + " return returns" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qhr50_Czxazw" + }, + "source": [ + "### 3. The Actor-Critic loss\n", + "\n", + "Since you're using a hybrid Actor-Critic model, the chosen loss function is a combination of Actor and Critic losses for training, as shown below:\n", + "\n", + "$$L = L_{actor} + L_{critic}$$" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nOQIJuG1xdTH" + }, + "source": [ + "#### The Actor loss\n", + "\n", + "The Actor loss is based on [policy gradients with the Critic as a state dependent baseline](https://www.youtube.com/watch?v=EKqxumCuAAY&t=62m23s) and computed with single-sample (per-episode) estimates.\n", + "\n", + "$$L_{actor} = -\\sum^{T}_{t=1} \\log\\pi_{\\theta}(a_{t} | s_{t})[G(s_{t}, a_{t}) - V^{\\pi}_{\\theta}(s_{t})]$$\n", + "\n", + "where:\n", + "- $T$: the number of timesteps per episode, which can vary per episode\n", + "- $s_{t}$: the state at timestep $t$\n", + "- $a_{t}$: chosen action at timestep $t$ given state $s$\n", + "- $\\pi_{\\theta}$: is the policy (Actor) parameterized by $\\theta$\n", + "- $V^{\\pi}_{\\theta}$: is the value function (Critic) also parameterized by $\\theta$\n", + "- $G = G_{t}$: the expected return for a given state, action pair at timestep $t$\n", + "\n", + "A negative term is added to the sum since the idea is to maximize the probabilities of actions yielding higher rewards by minimizing the combined loss.\n", + "\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Y304O4OAxiAv" + }, + "source": [ + "##### The Advantage\n", + "\n", + "The $G - V$ term in our $L_{actor}$ formulation is called the [Advantage](https://spinningup.openai.com/en/latest/spinningup/rl_intro.html#advantage-functions), which indicates how much better an action is given a particular state over a random action selected according to the policy $\\pi$ for that state.\n", + "\n", + "While it's possible to exclude a baseline, this may result in high variance during training. And the nice thing about choosing the critic $V$ as a baseline is that it trained to be as close as possible to $G$, leading to a lower variance.\n", + "\n", + "In addition, without the Critic, the algorithm would try to increase probabilities for actions taken on a particular state based on expected return, which may not make much of a difference if the relative probabilities between actions remain the same.\n", + "\n", + "For instance, suppose that two actions for a given state would yield the same expected return. Without the Critic, the algorithm would try to raise the probability of these actions based on the objective $J$. With the Critic, it may turn out that there's no Advantage ($G - V = 0$), and thus no benefit gained in increasing the actions' probabilities and the algorithm would set the gradients to zero.\n", + "\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1hrPLrgGxlvb" + }, + "source": [ + "#### The Critic loss\n", + "\n", + "Training $V$ to be as close possible to $G$ can be set up as a regression problem with the following loss function:\n", + "\n", + "$$L_{critic} = L_{\\delta}(G, V^{\\pi}_{\\theta})$$\n", + "\n", + "where $L_{\\delta}$ is the [Huber loss](https://en.wikipedia.org/wiki/Huber_loss), which is less sensitive to outliers in data than squared-error loss.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9EXwbEez6n9m" + }, + "outputs": [], + "source": [ + "huber_loss = tf.keras.losses.Huber(reduction=tf.keras.losses.Reduction.SUM)\n", + "\n", + "def compute_loss(\n", + " action_probs: tf.Tensor,\n", + " values: tf.Tensor,\n", + " returns: tf.Tensor) -> tf.Tensor:\n", + " \"\"\"Computes the combined Actor-Critic loss.\"\"\"\n", + "\n", + " advantage = returns - values\n", + "\n", + " action_log_probs = tf.math.log(action_probs)\n", + " actor_loss = -tf.math.reduce_sum(action_log_probs * advantage)\n", + "\n", + " critic_loss = huber_loss(values, returns)\n", + "\n", + " return actor_loss + critic_loss" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HSYkQOmRfV75" + }, + "source": [ + "### 4. Define the training step to update parameters\n", + "\n", + "All of the steps above are combined into a training step that is run every episode. All steps leading up to the loss function are executed with the `tf.GradientTape` context to enable automatic differentiation.\n", + "\n", + "This tutorial uses the Adam optimizer to apply the gradients to the model parameters.\n", + "\n", + "The sum of the undiscounted rewards, `episode_reward`, is also computed in this step. This value will be used later on to evaluate if the success criterion is met.\n", + "\n", + "The `tf.function` context is applied to the `train_step` function so that it can be compiled into a callable TensorFlow graph, which can lead to 10x speedup in training.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QoccrkF3IFCg" + }, + "outputs": [], + "source": [ + "optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)\n", + "\n", + "\n", + "@tf.function\n", + "def train_step(\n", + " initial_state: tf.Tensor,\n", + " model: tf.keras.Model,\n", + " optimizer: tf.keras.optimizers.Optimizer,\n", + " gamma: float,\n", + " max_steps_per_episode: int) -> tf.Tensor:\n", + " \"\"\"Runs a model training step.\"\"\"\n", + "\n", + " with tf.GradientTape() as tape:\n", + "\n", + " # Run the model for one episode to collect training data\n", + " action_probs, values, rewards = run_episode(\n", + " initial_state, model, max_steps_per_episode)\n", + "\n", + " # Calculate the expected returns\n", + " returns = get_expected_return(rewards, gamma)\n", + "\n", + " # Convert training data to appropriate TF tensor shapes\n", + " action_probs, values, returns = [\n", + " tf.expand_dims(x, 1) for x in [action_probs, values, returns]]\n", + "\n", + " # Calculate the loss values to update our network\n", + " loss = compute_loss(action_probs, values, returns)\n", + "\n", + " # Compute the gradients from the loss\n", + " grads = tape.gradient(loss, model.trainable_variables)\n", + "\n", + " # Apply the gradients to the model's parameters\n", + " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", + "\n", + " episode_reward = tf.math.reduce_sum(rewards)\n", + "\n", + " return episode_reward" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HFvZiDoAflGK" + }, + "source": [ + "### 5. Run the training loop\n", + "\n", + "Training is executed by running the training step until either the success criterion or maximum number of episodes is reached. \n", + "\n", + "A running record of episode rewards is kept in a queue. Once 100 trials are reached, the oldest reward is removed at the left (tail) end of the queue and the newest one is added at the head (right). A running sum of the rewards is also maintained for computational efficiency.\n", + "\n", + "Depending on your runtime, training can finish in less than a minute." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kbmBxnzLiUJx" + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "min_episodes_criterion = 100\n", + "max_episodes = 10000\n", + "max_steps_per_episode = 500\n", + "\n", + "# `CartPole-v1` is considered solved if average reward is >= 475 over 500\n", + "# consecutive trials\n", + "reward_threshold = 475\n", + "running_reward = 0\n", + "\n", + "# The discount factor for future rewards\n", + "gamma = 0.99\n", + "\n", + "# Keep the last episodes reward\n", + "episodes_reward: collections.deque = collections.deque(maxlen=min_episodes_criterion)\n", + "\n", + "t = tqdm.trange(max_episodes)\n", + "for i in t:\n", + " initial_state, info = env.reset()\n", + " initial_state = tf.constant(initial_state, dtype=tf.float32)\n", + " episode_reward = int(train_step(\n", + " initial_state, model, optimizer, gamma, max_steps_per_episode))\n", + "\n", + " episodes_reward.append(episode_reward)\n", + " running_reward = statistics.mean(episodes_reward)\n", + "\n", + "\n", + " t.set_postfix(\n", + " episode_reward=episode_reward, running_reward=running_reward)\n", + "\n", + " # Show the average episode reward every 10 episodes\n", + " if i % 10 == 0:\n", + " pass # print(f'Episode {i}: average reward: {avg_reward}')\n", + "\n", + " if running_reward > reward_threshold and i >= min_episodes_criterion:\n", + " break\n", + "\n", + "print(f'\\nSolved at episode {i}: average reward: {running_reward:.2f}!')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ru8BEwS1EmAv" + }, + "source": [ + "## Visualization\n", + "\n", + "After training, it would be good to visualize how the model performs in the environment. You can run the cells below to generate a GIF animation of one episode run of the model. Note that additional packages need to be installed for Gym to render the environment's images correctly in Colab." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qbIMMkfmRHyC" + }, + "outputs": [], + "source": [ + "# Render an episode and save as a GIF file\n", + "\n", + "from IPython import display as ipythondisplay\n", + "from PIL import Image\n", + "\n", + "render_env = gym.make(\"CartPole-v1\", render_mode='rgb_array')\n", + "\n", + "def render_episode(env: gym.Env, model: tf.keras.Model, max_steps: int):\n", + " state, info = env.reset()\n", + " state = tf.constant(state, dtype=tf.float32)\n", + " screen = env.render()\n", + " images = [Image.fromarray(screen)]\n", + "\n", + " for i in range(1, max_steps + 1):\n", + " state = tf.expand_dims(state, 0)\n", + " action_probs, _ = model(state)\n", + " action = np.argmax(np.squeeze(action_probs))\n", + "\n", + " state, reward, done, truncated, info = env.step(action)\n", + " state = tf.constant(state, dtype=tf.float32)\n", + "\n", + " # Render screen every 10 steps\n", + " if i % 10 == 0:\n", + " screen = env.render()\n", + " images.append(Image.fromarray(screen))\n", + "\n", + " if done:\n", + " break\n", + "\n", + " return images\n", + "\n", + "\n", + "# Save GIF image\n", + "images = render_episode(render_env, model, max_steps_per_episode)\n", + "image_file = 'cartpole-v1.gif'\n", + "# loop=0: loop forever, duration=1: play each frame for 1ms\n", + "images[0].save(\n", + " image_file, save_all=True, append_images=images[1:], loop=0, duration=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TLd720SejKmf" + }, + "outputs": [], + "source": [ + "import tensorflow_docs.vis.embed as embed\n", + "embed.embed_file(image_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lnq9Hzo1Po6X" + }, + "source": [ + "## Next steps\n", + "\n", + "This tutorial demonstrated how to implement the Actor-Critic method using Tensorflow.\n", + "\n", + "As a next step, you could try training a model on a different environment in Gym.\n", + "\n", + "For additional information regarding Actor-Critic methods and the Cartpole-v0 problem, you may refer to the following resources:\n", + "\n", + "- [The Actor-Critic method](https://hal.inria.fr/hal-00840470/document)\n", + "- [The Actor-Critic lecture (CAL)](https://www.youtube.com/watch?v=EKqxumCuAAY&list=PLkFD6_40KJIwhWJpGazJ9VSj9CFMkb79A&index=7&t=0s)\n", + "- [Cart Pole learning control problem \\[Barto, et al. 1983\\]](http://www.derongliu.org/adp/adp-cdrom/Barto1983.pdf)\n", + "\n", + "For more reinforcement learning examples in TensorFlow, you can check the following resources:\n", + "- [Reinforcement learning code examples (keras.io)](https://keras.io/examples/rl/)\n", + "- [TF-Agents reinforcement learning library](https://www.tensorflow.org/agents)\n" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "_jQ1tEQCxwRx" + ], + "name": "actor_critic.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/reinforcement_learning/images/cartpole-v0.gif b/site/en/tutorials/reinforcement_learning/images/cartpole-v0.gif new file mode 100644 index 00000000000..f89e9de95be Binary files /dev/null and b/site/en/tutorials/reinforcement_learning/images/cartpole-v0.gif differ diff --git a/site/en/tutorials/structured_data/feature_columns.ipynb b/site/en/tutorials/structured_data/feature_columns.ipynb index 02273422c6d..4448b763ca6 100644 --- a/site/en/tutorials/structured_data/feature_columns.ipynb +++ b/site/en/tutorials/structured_data/feature_columns.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "rNdWfPXCjTjY" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "I1dUQ0GejU8N" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "c05P9g5WjizZ" }, "source": [ @@ -47,40 +43,40 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "zofH_gCzgplN" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/structured_data/feature_columns\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/structured_data/feature_columns.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/feature_columns.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/structured_data/feature_columns.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "K1y4OHpGgss7" }, "source": [ - "This tutorial demonstrates how to classify structured data (e.g. tabular data in a CSV). We will use [Keras](https://www.tensorflow.org/guide/keras) to define the model, and [feature columns](https://www.tensorflow.org/guide/feature_columns) as a bridge to map from columns in a CSV to features used to train the model. This tutorial contains complete code to:\n", + "> Warning: The `tf.feature_columns` module described in this tutorial is not recommended for new code. [Keras preprocessing layers](https://www.tensorflow.org/tutorials/structured_data/preprocessing_layers) cover this functionality, for migration instructions see the [Migrating feature columns](../../guide/migrate/migrating_feature_columns.ipynb) guide. The `tf.feature_columns` module was designed for use with TF1 `Estimators`. It does fall under our [compatibility guarantees](https://tensorflow.org/guide/versions), but will receive no fixes other than security vulnerabilities.\n", + "\n", + "This tutorial demonstrates how to classify structured data (e.g. tabular data in a CSV). We will use [Keras](https://www.tensorflow.org/guide/keras) to define the model, and `tf.feature_column` as a bridge to map from columns in a CSV to features used to train the model. This tutorial contains complete code to:\n", "\n", "* Load a CSV file using [Pandas](https://pandas.pydata.org/).\n", "* Build an input pipeline to batch and shuffle the rows using [tf.data](https://www.tensorflow.org/guide/datasets).\n", @@ -89,32 +85,31 @@ "\n", "## The Dataset\n", "\n", - "We will use a small [dataset](https://archive.ics.uci.edu/ml/datasets/heart+Disease) provided by the Cleveland Clinic Foundation for Heart Disease. There are several hundred rows in the CSV. Each row describes a patient, and each column describes an attribute. We will use this information to predict whether a patient has heart disease, which in this dataset is a binary classification task.\n", + "We will use a simplified version of the PetFinder [dataset](https://www.kaggle.com/c/petfinder-adoption-prediction). There are several thousand rows in the CSV. Each row describes a pet, and each column describes an attribute. We will use this information to predict the speed at which the pet will be adopted.\n", "\n", - "Following is a [description](https://archive.ics.uci.edu/ml/machine-learning-databases/heart-disease/heart-disease.names) of this dataset. Notice there are both numeric and categorical columns.\n", + "Following is a description of this dataset. Notice there are both numeric and categorical columns. There is a free text column which we will not use in this tutorial.\n", "\n", - "\u003eColumn| Description| Feature Type | Data Type\n", - "\u003e------------|--------------------|----------------------|-----------------\n", - "\u003eAge | Age in years | Numerical | integer\n", - "\u003eSex | (1 = male; 0 = female) | Categorical | integer\n", - "\u003eCP | Chest pain type (0, 1, 2, 3, 4) | Categorical | integer\n", - "\u003eTrestbpd | Resting blood pressure (in mm Hg on admission to the hospital) | Numerical | integer\n", - "\u003eChol | Serum cholestoral in mg/dl | Numerical | integer\n", - "\u003eFBS | (fasting blood sugar \u003e 120 mg/dl) (1 = true; 0 = false) | Categorical | integer\n", - "\u003eRestECG | Resting electrocardiographic results (0, 1, 2) | Categorical | integer\n", - "\u003eThalach | Maximum heart rate achieved | Numerical | integer\n", - "\u003eExang | Exercise induced angina (1 = yes; 0 = no) | Categorical | integer\n", - "\u003eOldpeak | ST depression induced by exercise relative to rest | Numerical | integer\n", - "\u003eSlope | The slope of the peak exercise ST segment | Numerical | float\n", - "\u003eCA | Number of major vessels (0-3) colored by flourosopy | Numerical | integer\n", - "\u003eThal | 3 = normal; 6 = fixed defect; 7 = reversable defect | Categorical | string\n", - "\u003eTarget | Diagnosis of heart disease (1 = true; 0 = false) | Classification | integer" + "Column | Description| Feature Type | Data Type\n", + "------------|--------------------|----------------------|-----------------\n", + "Type | Type of animal (Dog, Cat) | Categorical | string\n", + "Age | Age of the pet | Numerical | integer\n", + "Breed1 | Primary breed of the pet | Categorical | string\n", + "Color1 | Color 1 of pet | Categorical | string\n", + "Color2 | Color 2 of pet | Categorical | string\n", + "MaturitySize | Size at maturity | Categorical | string\n", + "FurLength | Fur length | Categorical | string\n", + "Vaccinated | Pet has been vaccinated | Categorical | string\n", + "Sterilized | Pet has been sterilized | Categorical | string\n", + "Health | Health Condition | Categorical | string\n", + "Fee | Adoption Fee | Numerical | integer\n", + "Description | Profile write-up for this pet | Text | string\n", + "PhotoAmt | Total uploaded photos for this pet | Numerical | integer\n", + "AdoptionSpeed | Speed of adoption | Classification | integer" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "VxyBFc_kKazA" }, "source": [ @@ -123,10 +118,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "LuOWVJBz8a6G" }, "outputs": [], @@ -136,24 +129,15 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "9dEreb4QKizj" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", "import numpy as np\n", "import pandas as pd\n", "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", "import tensorflow as tf\n", "\n", "from tensorflow import feature_column\n", @@ -164,7 +148,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "KCEhSZcULZ9n" }, "source": [ @@ -175,23 +158,64 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "REZ57BXCLdfG" }, "outputs": [], "source": [ - "URL = 'https://storage.googleapis.com/applied-dl/heart.csv'\n", - "dataframe = pd.read_csv(URL)\n", + "import pathlib\n", + "\n", + "dataset_url = 'http://storage.googleapis.com/download.tensorflow.org/data/petfinder-mini.zip'\n", + "csv_file = 'datasets/petfinder-mini/petfinder-mini.csv'\n", + "\n", + "tf.keras.utils.get_file('petfinder_mini.zip', dataset_url,\n", + " extract=True, cache_dir='.')\n", + "dataframe = pd.read_csv(csv_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "o8QIi0_jT5LM" + }, + "outputs": [], + "source": [ "dataframe.head()" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", + "id": "awGiBeBWbQC8" + }, + "source": [ + "## Create target variable\n", + "\n", + "The task in the original dataset is to predict the speed at which a pet will be adopted (e.g., in the first week, the first month, the first three months, and so on). Let's simplify this for our tutorial. Here, we will transform this into a binary classification problem, and simply predict whether the pet was adopted, or not.\n", + "\n", + "After modifying the label column, 0 will indicate the pet was not adopted, and 1 will indicate it was." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xcbTpEXWbMDz" + }, + "outputs": [], + "source": [ + "# In the original dataset \"4\" indicates the pet was not adopted.\n", + "dataframe['target'] = np.where(dataframe['AdoptionSpeed']==4, 0, 1)\n", + "\n", + "# Drop un-used columns.\n", + "dataframe = dataframe.drop(columns=['AdoptionSpeed', 'Description'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { "id": "u0zhLtQqMPem" }, "source": [ @@ -202,10 +226,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YEOpw7LhMYsI" }, "outputs": [], @@ -220,7 +242,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "84ef46LXMfvu" }, "source": [ @@ -231,10 +252,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "NkcaMYP-MsRe" }, "outputs": [], @@ -252,10 +271,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "CXbbXkJvMy34" }, "outputs": [], @@ -269,7 +286,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qRLGSMDzM-dl" }, "source": [ @@ -280,24 +296,21 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "CSBo3dUVNFc9" }, "outputs": [], "source": [ "for feature_batch, label_batch in train_ds.take(1):\n", " print('Every feature:', list(feature_batch.keys()))\n", - " print('A batch of ages:', feature_batch['age'])\n", + " print('A batch of ages:', feature_batch['Age'])\n", " print('A batch of targets:', label_batch )" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "OT5N6Se-NQsC" }, "source": [ @@ -307,20 +320,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ttIvgLRaNoOQ" }, "source": [ - "## Demonstrate several types of feature column\n", + "## Demonstrate several types of feature columns\n", "TensorFlow provides many types of feature columns. In this section, we will create several types of feature columns, and demonstrate how they transform a column from the dataframe." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "mxwiHFHuNhmf" }, "outputs": [], @@ -331,10 +341,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "0wfLB8Q3N3UH" }, "outputs": [], @@ -349,7 +357,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Q7OEKe82N-Qb" }, "source": [ @@ -359,32 +366,28 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "QZTZ0HnHOCxC" }, "outputs": [], "source": [ - "age = feature_column.numeric_column(\"age\")\n", - "demo(age)" + "photo_count = feature_column.numeric_column('PhotoAmt')\n", + "demo(photo_count)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7a6ddSyzOKpq" }, "source": [ - "In the heart disease dataset, most columns from the dataframe are numeric." + "In the PetFinder dataset, most columns from the dataframe are categorical." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IcSxUoYgOlA1" }, "source": [ @@ -394,60 +397,45 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "wJ4Wt3SAOpTQ" }, "outputs": [], "source": [ - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", + "age = feature_column.numeric_column('Age')\n", + "age_buckets = feature_column.bucketized_column(age, boundaries=[1, 3, 5])\n", "demo(age_buckets)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "r1tArzewPb-b" }, "source": [ "### Categorical columns\n", - "In this dataset, thal is represented as a string (e.g. 'fixed', 'normal', or 'reversible'). We cannot feed strings directly to a model. Instead, we must first map them to numeric values. The categorical vocabulary columns provide a way to represent strings as a one-hot vector (much like you have seen above with age buckets). The vocabulary can be passed as a list using [categorical_column_with_vocabulary_list](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_list), or loaded from a file using [categorical_column_with_vocabulary_file](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_file)." + "In this dataset, Type is represented as a string (e.g. 'Dog', or 'Cat'). We cannot feed strings directly to a model. Instead, we must first map them to numeric values. The categorical vocabulary columns provide a way to represent strings as a one-hot vector (much like you have seen above with age buckets). The vocabulary can be passed as a list using [categorical_column_with_vocabulary_list](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_list), or loaded from a file using [categorical_column_with_vocabulary_file](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_file)." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "DJ6QnSHkPtOC" }, "outputs": [], "source": [ - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", + "animal_type = feature_column.categorical_column_with_vocabulary_list(\n", + " 'Type', ['Cat', 'Dog'])\n", "\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "demo(thal_one_hot)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dxQloQ9jOoXL" - }, - "source": [ - "In a more complex dataset, many columns would be categorical (e.g. strings). Feature columns are most valuable when working with categorical data. Although there is only one categorical column in this dataset, we will use it to demonstrate several important types of feature columns that you could use when working with other datasets." + "animal_type_one_hot = feature_column.indicator_column(animal_type)\n", + "demo(animal_type_one_hot)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "LEFPjUr6QmwS" }, "source": [ @@ -459,24 +447,23 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "hSlohmr2Q_UU" }, "outputs": [], "source": [ "# Notice the input to the embedding column is the categorical column\n", "# we previously created\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "demo(thal_embedding)" + "breed1 = feature_column.categorical_column_with_vocabulary_list(\n", + " 'Breed1', dataframe.Breed1.unique())\n", + "breed1_embedding = feature_column.embedding_column(breed1, dimension=8)\n", + "demo(breed1_embedding)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "urFCAvTVRMpB" }, "source": [ @@ -489,48 +476,42 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YHU_Aj2nRRDC" }, "outputs": [], "source": [ - "thal_hashed = feature_column.categorical_column_with_hash_bucket(\n", - " 'thal', hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(thal_hashed))" + "breed1_hashed = feature_column.categorical_column_with_hash_bucket(\n", + " 'Breed1', hash_bucket_size=10)\n", + "demo(feature_column.indicator_column(breed1_hashed))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "fB94M27DRXtZ" }, "source": [ "### Crossed feature columns\n", - "Combining features into a single feature, better known as [feature crosses](https://developers.google.com/machine-learning/glossary/#feature_cross), enables a model to learn separate weights for each combination of features. Here, we will create a new feature that is the cross of age and thal. Note that `crossed_column` does not build the full table of all possible combinations (which could be very large). Instead, it is backed by a `hashed_column`, so you can choose how large the table is." + "Combining features into a single feature, better known as [feature crosses](https://developers.google.com/machine-learning/glossary/#feature_cross), enables a model to learn separate weights for each combination of features. Here, we will create a new feature that is the cross of Age and Type. Note that `crossed_column` does not build the full table of all possible combinations (which could be very large). Instead, it is backed by a `hashed_column`, so you can choose how large the table is." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "oaPVERd9Rep6" }, "outputs": [], "source": [ - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", + "crossed_feature = feature_column.crossed_column([age_buckets, animal_type], hash_bucket_size=10)\n", "demo(feature_column.indicator_column(crossed_feature))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ypkI9zx6Rj1q" }, "source": [ @@ -542,10 +523,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "4PlLY7fORuzA" }, "outputs": [], @@ -553,33 +532,73 @@ "feature_columns = []\n", "\n", "# numeric cols\n", - "for header in ['age', 'trestbps', 'chol', 'thalach', 'oldpeak', 'slope', 'ca']:\n", - " feature_columns.append(feature_column.numeric_column(header))\n", - "\n", + "for header in ['PhotoAmt', 'Fee', 'Age']:\n", + " feature_columns.append(feature_column.numeric_column(header))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jdF4rXkcDmBl" + }, + "outputs": [], + "source": [ "# bucketized cols\n", - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "feature_columns.append(age_buckets)\n", - "\n", - "# indicator cols\n", - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "feature_columns.append(thal_one_hot)\n", - "\n", - "# embedding cols\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "feature_columns.append(thal_embedding)\n", - "\n", - "# crossed cols\n", - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "crossed_feature = feature_column.indicator_column(crossed_feature)\n", - "feature_columns.append(crossed_feature)" + "age = feature_column.numeric_column('Age')\n", + "age_buckets = feature_column.bucketized_column(age, boundaries=[1, 2, 3, 4, 5])\n", + "feature_columns.append(age_buckets)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RsteO7FGDmNc" + }, + "outputs": [], + "source": [ + "# indicator_columns\n", + "indicator_column_names = ['Type', 'Color1', 'Color2', 'Gender', 'MaturitySize',\n", + " 'FurLength', 'Vaccinated', 'Sterilized', 'Health']\n", + "for col_name in indicator_column_names:\n", + " categorical_column = feature_column.categorical_column_with_vocabulary_list(\n", + " col_name, dataframe[col_name].unique())\n", + " indicator_column = feature_column.indicator_column(categorical_column)\n", + " feature_columns.append(indicator_column)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6MhdqQ5uDmYU" + }, + "outputs": [], + "source": [ + "# embedding columns\n", + "breed1 = feature_column.categorical_column_with_vocabulary_list(\n", + " 'Breed1', dataframe.Breed1.unique())\n", + "breed1_embedding = feature_column.embedding_column(breed1, dimension=8)\n", + "feature_columns.append(breed1_embedding)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qkzRNfCLDsQf" + }, + "outputs": [], + "source": [ + "# crossed columns\n", + "age_type_feature = feature_column.crossed_column([age_buckets, animal_type], hash_bucket_size=100)\n", + "feature_columns.append(feature_column.indicator_column(age_type_feature))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "M-nDp8krS_ts" }, "source": [ @@ -589,10 +608,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "6o-El1R2TGQP" }, "outputs": [], @@ -603,7 +620,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8cf6vKfgTH0U" }, "source": [ @@ -612,10 +628,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "gcemszoGSse_" }, "outputs": [], @@ -629,7 +643,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bBx4Xu0eTXWq" }, "source": [ @@ -638,10 +651,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "_YJPPb3xTPeZ" }, "outputs": [], @@ -650,24 +661,23 @@ " feature_layer,\n", " layers.Dense(128, activation='relu'),\n", " layers.Dense(128, activation='relu'),\n", - " layers.Dense(1, activation='sigmoid')\n", + " layers.Dropout(.1),\n", + " layers.Dense(1)\n", "])\n", "\n", "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", " metrics=['accuracy'])\n", "\n", "model.fit(train_ds,\n", " validation_data=val_ds,\n", - " epochs=5)" + " epochs=10)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "GnFmMOW0Tcaa" }, "outputs": [], @@ -679,7 +689,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3bdfbq20V6zu" }, "source": [ @@ -689,7 +698,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SotnhVWuHQCw" }, "source": [ @@ -699,13 +707,11 @@ } ], "metadata": { + "accelerator": "GPU", "colab": { "collapsed_sections": [], "name": "feature_columns.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/site/en/tutorials/structured_data/images/baseline.png b/site/en/tutorials/structured_data/images/baseline.png new file mode 100644 index 00000000000..06b49ede487 Binary files /dev/null and b/site/en/tutorials/structured_data/images/baseline.png differ diff --git a/site/en/tutorials/structured_data/images/conv_window.png b/site/en/tutorials/structured_data/images/conv_window.png new file mode 100644 index 00000000000..33519256dd5 Binary files /dev/null and b/site/en/tutorials/structured_data/images/conv_window.png differ diff --git a/site/en/tutorials/structured_data/images/last_window.png b/site/en/tutorials/structured_data/images/last_window.png new file mode 100644 index 00000000000..6c465f2b884 Binary files /dev/null and b/site/en/tutorials/structured_data/images/last_window.png differ diff --git a/site/en/tutorials/structured_data/images/lstm_1_window.png b/site/en/tutorials/structured_data/images/lstm_1_window.png new file mode 100644 index 00000000000..de4476e4ddf Binary files /dev/null and b/site/en/tutorials/structured_data/images/lstm_1_window.png differ diff --git a/site/en/tutorials/structured_data/images/lstm_many_window.png b/site/en/tutorials/structured_data/images/lstm_many_window.png new file mode 100644 index 00000000000..ba88af8542c Binary files /dev/null and b/site/en/tutorials/structured_data/images/lstm_many_window.png differ diff --git a/site/en/tutorials/structured_data/images/multistep_autoregressive.png b/site/en/tutorials/structured_data/images/multistep_autoregressive.png new file mode 100644 index 00000000000..8658fdae3e3 Binary files /dev/null and b/site/en/tutorials/structured_data/images/multistep_autoregressive.png differ diff --git a/site/en/tutorials/structured_data/images/multistep_conv.png b/site/en/tutorials/structured_data/images/multistep_conv.png new file mode 100644 index 00000000000..425d3e015d8 Binary files /dev/null and b/site/en/tutorials/structured_data/images/multistep_conv.png differ diff --git a/site/en/tutorials/structured_data/images/multistep_dense.png b/site/en/tutorials/structured_data/images/multistep_dense.png new file mode 100644 index 00000000000..87e43ecb490 Binary files /dev/null and b/site/en/tutorials/structured_data/images/multistep_dense.png differ diff --git a/site/en/tutorials/structured_data/images/multistep_last.png b/site/en/tutorials/structured_data/images/multistep_last.png new file mode 100644 index 00000000000..926fc3b1e94 Binary files /dev/null and b/site/en/tutorials/structured_data/images/multistep_last.png differ diff --git a/site/en/tutorials/structured_data/images/multistep_lstm.png b/site/en/tutorials/structured_data/images/multistep_lstm.png new file mode 100644 index 00000000000..4b937a358b5 Binary files /dev/null and b/site/en/tutorials/structured_data/images/multistep_lstm.png differ diff --git a/site/en/tutorials/structured_data/images/multistep_repeat.png b/site/en/tutorials/structured_data/images/multistep_repeat.png new file mode 100644 index 00000000000..81ed373a283 Binary files /dev/null and b/site/en/tutorials/structured_data/images/multistep_repeat.png differ diff --git a/site/en/tutorials/structured_data/images/narrow_window.png b/site/en/tutorials/structured_data/images/narrow_window.png new file mode 100644 index 00000000000..4d6dba9d50a Binary files /dev/null and b/site/en/tutorials/structured_data/images/narrow_window.png differ diff --git a/site/en/tutorials/structured_data/images/raw_window.png b/site/en/tutorials/structured_data/images/raw_window.png new file mode 100644 index 00000000000..f57cf5ea2c0 Binary files /dev/null and b/site/en/tutorials/structured_data/images/raw_window.png differ diff --git a/site/en/tutorials/structured_data/images/raw_window_1h.png b/site/en/tutorials/structured_data/images/raw_window_1h.png new file mode 100644 index 00000000000..2eee1324a67 Binary files /dev/null and b/site/en/tutorials/structured_data/images/raw_window_1h.png differ diff --git a/site/en/tutorials/structured_data/images/raw_window_24h.png b/site/en/tutorials/structured_data/images/raw_window_24h.png new file mode 100644 index 00000000000..ecc21a66fd8 Binary files /dev/null and b/site/en/tutorials/structured_data/images/raw_window_24h.png differ diff --git a/site/en/tutorials/structured_data/images/residual.png b/site/en/tutorials/structured_data/images/residual.png new file mode 100644 index 00000000000..af0f8c6a0a7 Binary files /dev/null and b/site/en/tutorials/structured_data/images/residual.png differ diff --git a/site/en/tutorials/structured_data/images/split_window.png b/site/en/tutorials/structured_data/images/split_window.png new file mode 100644 index 00000000000..9a3e8fe9edb Binary files /dev/null and b/site/en/tutorials/structured_data/images/split_window.png differ diff --git a/site/en/tutorials/structured_data/images/time_series.png b/site/en/tutorials/structured_data/images/time_series.png deleted file mode 100644 index 64af2935412..00000000000 Binary files a/site/en/tutorials/structured_data/images/time_series.png and /dev/null differ diff --git a/site/en/tutorials/structured_data/images/wide_conv_window.png b/site/en/tutorials/structured_data/images/wide_conv_window.png new file mode 100644 index 00000000000..09b3b322529 Binary files /dev/null and b/site/en/tutorials/structured_data/images/wide_conv_window.png differ diff --git a/site/en/tutorials/structured_data/images/wide_window.png b/site/en/tutorials/structured_data/images/wide_window.png new file mode 100644 index 00000000000..85ebcd00db7 Binary files /dev/null and b/site/en/tutorials/structured_data/images/wide_window.png differ diff --git a/site/en/tutorials/structured_data/imbalanced_data.ipynb b/site/en/tutorials/structured_data/imbalanced_data.ipynb index f2c04fdb35d..25b55071817 100644 --- a/site/en/tutorials/structured_data/imbalanced_data.ipynb +++ b/site/en/tutorials/structured_data/imbalanced_data.ipynb @@ -3,7 +3,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "dUeKVCYTbcyT" }, "source": [ @@ -12,11 +11,9 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "4ellrPx7tdxq" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "7JfLUlawto_D" }, "source": [ @@ -47,34 +43,32 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "DwdpaTKJOoPu" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/structured_data/imbalanced_data\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/structured_data/imbalanced_data.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/imbalanced_data.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/structured_data/imbalanced_data.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "mthoSGBAOoX-" }, "source": [ - "This tutorial demonstrates how to classify a highly imbalanced dataset in which the number of examples in one class greatly outnumbers the examples in another. You will work with the [Credit Card Fraud Detection](https://www.kaggle.com/mlg-ulb/creditcardfraud) dataset hosted on Kaggle. The aim is to detect a mere 492 fraudulent transactions from 284,807 transactions in total. You will use [Keras](../../guide/keras/overview.ipynb) to define the model and [class weights](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model) to help the model learn from the imbalanced data. .\n", + "This tutorial demonstrates how to classify a highly imbalanced dataset in which the number of examples in one class greatly outnumbers the examples in another. You will work with the [Credit Card Fraud Detection](https://www.kaggle.com/mlg-ulb/creditcardfraud) dataset hosted on Kaggle. The aim is to detect a mere 492 fraudulent transactions from 284,807 transactions in total. You will use [Keras](https://www.tensorflow.org/guide/keras/overview) to define the model and [class weights](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model) to help the model learn from the imbalanced data. .\n", "\n", "This tutorial contains complete code to:\n", "\n", @@ -82,15 +76,13 @@ "* Create train, validation, and test sets.\n", "* Define and train a model using Keras (including setting class weights).\n", "* Evaluate the model using various metrics (including precision and recall).\n", - "* Try common tecniques for dealing with imbalanced data like:\n", - " * Class weighting \n", - " * Oversampling\n" + "* Select a threshold for a probabilistic classifier to get a deterministic classifier.\n", + "* Try and compare with class weighted modelling and oversampling." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "kRHmSyHxEIhN" }, "source": [ @@ -99,40 +91,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yJHVo_K_v20i" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fYBlUQ5FvzxP" - }, - "outputs": [], - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", "id": "JM7hDSNClfoK" }, "outputs": [], @@ -157,10 +117,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "c8o1FHzD-_y_" }, "outputs": [], @@ -172,7 +130,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Z3iZVjziKHmX" }, "source": [ @@ -182,23 +139,20 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4sA9WOcmzH2D" }, "source": [ "### Download the Kaggle Credit Card Fraud data set\n", "\n", - "Pandas is a Python library with many helpful utilities for loading and working with structured data and can be used to download CSVs into a dataframe.\n", + "Pandas is a Python library with many helpful utilities for loading and working with structured data. It can be used to download CSVs into a Pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html#pandas.DataFrame).\n", "\n", "Note: This dataset has been collected and analysed during a research collaboration of Worldline and the [Machine Learning Group](http://mlg.ulb.ac.be) of ULB (Université Libre de Bruxelles) on big data mining and fraud detection. More details on current and past projects on related topics are available [here](https://www.researchgate.net/project/Fraud-detection-5) and the page of the [DefeatFraud](https://mlg.ulb.ac.be/wordpress/portfolio_page/defeatfraud-assessment-and-validation-of-deep-feature-engineering-and-learning-solutions-for-fraud-detection/) project" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "pR_SnbMArXr7" }, "outputs": [], @@ -210,10 +164,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "-fgdQgmwUFuj" }, "outputs": [], @@ -224,7 +176,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xWKB_CVZFLpB" }, "source": [ @@ -235,10 +186,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "HCJFrtuY2iLF" }, "outputs": [], @@ -252,7 +201,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "KnLKFQDsCBUg" }, "source": [ @@ -262,7 +210,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "6qox6ryyzwdr" }, "source": [ @@ -273,10 +220,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Ef42jTuxEjnj" }, "outputs": [], @@ -287,14 +232,13 @@ "cleaned_df.pop('Time')\n", "\n", "# The `Amount` column covers a huge range. Convert to log-space.\n", - "eps=0.001 # 0 =\u003e 0.1¢\n", - "cleaned_df['Log Ammount'] = np.log(cleaned_df.pop('Amount')+eps)" + "eps = 0.001 # 0 => 0.1¢\n", + "cleaned_df['Log Amount'] = np.log(cleaned_df.pop('Amount')+eps)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uSNgdQFFFQ6u" }, "source": [ @@ -303,23 +247,21 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "xfxhKg7Yr1-b" }, "outputs": [], "source": [ - "# Use a utility from sklearn to split and shuffle our dataset.\n", + "# Use a utility from sklearn to split and shuffle your dataset.\n", "train_df, test_df = train_test_split(cleaned_df, test_size=0.2)\n", "train_df, val_df = train_test_split(train_df, test_size=0.2)\n", "\n", "# Form np arrays of labels and features.\n", - "train_labels = np.array(train_df.pop('Class'))\n", - "bool_train_labels = train_labels != 0\n", - "val_labels = np.array(val_df.pop('Class'))\n", - "test_labels = np.array(test_df.pop('Class'))\n", + "train_labels = np.array(train_df.pop('Class')).reshape(-1, 1)\n", + "bool_train_labels = train_labels[:, 0] != 0\n", + "val_labels = np.array(val_df.pop('Class')).reshape(-1, 1)\n", + "test_labels = np.array(test_df.pop('Class')).reshape(-1, 1)\n", "\n", "train_features = np.array(train_df)\n", "val_features = np.array(val_df)\n", @@ -329,22 +271,43 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "8a_Z_kBmr7Oh" }, "source": [ + "We check whether the distribution of the classes in the three sets is about the same or not." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "96520cffee66" + }, + "outputs": [], + "source": [ + "print(f'Average class probability in training set: {train_labels.mean():.4f}')\n", + "print(f'Average class probability in validation set: {val_labels.mean():.4f}')\n", + "print(f'Average class probability in test set: {test_labels.mean():.4f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ueKV4cmcoRnf" + }, + "source": [ + "Given the small number of positive labels, this seems about right.\n", + "\n", "Normalize the input features using the sklearn StandardScaler.\n", "This will set the mean to 0 and standard deviation to 1.\n", "\n", - "Note: The `StandardScaler` is only fit using the `train_features` to be sure the model is not peeking at the validation or test sets. " + "Note: The `StandardScaler` is only fit using the `train_features` to be sure the model is not peeking at the validation or test sets." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "IO-qEUmJ5JQg" }, "outputs": [], @@ -372,7 +335,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XF2nNfWKJ33w" }, "source": [ @@ -382,7 +344,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uQ7m9nqDC3W6" }, "source": [ @@ -390,68 +351,66 @@ "\n", "Next compare the distributions of the positive and negative examples over a few features. Good questions to ask yourself at this point are:\n", "\n", - "* Do these distributions make sense? \n", + "* Do these distributions make sense?\n", " * Yes. You've normalized the input and these are mostly concentrated in the `+/- 2` range.\n", - "* Can you see the difference between the ditributions?\n", + "* Can you see the difference between the distributions?\n", " * Yes the positive examples contain a much higher rate of extreme values." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "raK7hyjd_vf6" }, "outputs": [], "source": [ - "pos_df = pd.DataFrame(train_features[ bool_train_labels], columns = train_df.columns)\n", - "neg_df = pd.DataFrame(train_features[~bool_train_labels], columns = train_df.columns)\n", + "pos_df = pd.DataFrame(train_features[ bool_train_labels], columns=train_df.columns)\n", + "neg_df = pd.DataFrame(train_features[~bool_train_labels], columns=train_df.columns)\n", "\n", - "sns.jointplot(pos_df['V5'], pos_df['V6'],\n", - " kind='hex', xlim = (-5,5), ylim = (-5,5))\n", + "sns.jointplot(x=pos_df['V5'], y=pos_df['V6'],\n", + " kind='hex', xlim=(-5,5), ylim=(-5,5))\n", "plt.suptitle(\"Positive distribution\")\n", "\n", - "sns.jointplot(neg_df['V5'], neg_df['V6'],\n", - " kind='hex', xlim = (-5,5), ylim = (-5,5))\n", + "sns.jointplot(x=neg_df['V5'], y=neg_df['V6'],\n", + " kind='hex', xlim=(-5,5), ylim=(-5,5))\n", "_ = plt.suptitle(\"Negative distribution\")" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qFK1u4JX16D8" }, "source": [ "## Define the model and metrics\n", "\n", - "Define a function that creates a simple neural network with a densly connected hidden layer, a [dropout](https://developers.google.com/machine-learning/glossary/#dropout_regularization) layer to reduce overfitting, and an output sigmoid layer that returns the probability of a transaction being fraudulent: " + "Define a function that creates a simple neural network with a densly connected hidden layer, a [dropout](https://developers.google.com/machine-learning/glossary/#dropout_regularization) layer to reduce overfitting, and an output sigmoid layer that returns the probability of a transaction being fraudulent:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3JQDzUqT3UYG" }, "outputs": [], "source": [ "METRICS = [\n", + " keras.metrics.BinaryCrossentropy(name='cross entropy'), # same as model's loss\n", + " keras.metrics.MeanSquaredError(name='Brier score'),\n", " keras.metrics.TruePositives(name='tp'),\n", " keras.metrics.FalsePositives(name='fp'),\n", " keras.metrics.TrueNegatives(name='tn'),\n", - " keras.metrics.FalseNegatives(name='fn'), \n", + " keras.metrics.FalseNegatives(name='fn'),\n", " keras.metrics.BinaryAccuracy(name='accuracy'),\n", " keras.metrics.Precision(name='precision'),\n", " keras.metrics.Recall(name='recall'),\n", " keras.metrics.AUC(name='auc'),\n", + " keras.metrics.AUC(name='prc', curve='PR'), # precision-recall curve\n", "]\n", "\n", - "def make_model(metrics = METRICS, output_bias=None):\n", + "def make_model(metrics=METRICS, output_bias=None):\n", " if output_bias is not None:\n", " output_bias = tf.keras.initializers.Constant(output_bias)\n", " model = keras.Sequential([\n", @@ -464,7 +423,7 @@ " ])\n", "\n", " model.compile(\n", - " optimizer=keras.optimizers.Adam(lr=1e-3),\n", + " optimizer=keras.optimizers.Adam(learning_rate=1e-3),\n", " loss=keras.losses.BinaryCrossentropy(),\n", " metrics=metrics)\n", "\n", @@ -474,39 +433,60 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "SU0GX6E6mieP" }, "source": [ "### Understanding useful metrics\n", "\n", "Notice that there are a few metrics defined above that can be computed by the model that will be helpful when evaluating the performance.\n", + "These can be divided into three groups.\n", "\n", + "#### Metrics for probability predictions\n", "\n", + "As we train our network with the cross entropy as a loss function, it is fully capable of predicting class probabilities, i.e., it is a probabilistic classifier.\n", + "Good metrics to assess probabilistic predictions are, in fact, **proper scoring rules**. Their key property is that predicting the true probability is optimal. We give two well-known examples:\n", + "\n", + "* **cross entropy** also known as log loss\n", + "* **Mean squared error** also known as the Brier score\n", + "\n", + "#### Metrics for deterministic 0/1 predictions\n", + "\n", + "In the end, one often wants to predict a class label, 0 or 1, *no fraud* or *fraud*.\n", + "This is called a deterministic classifier.\n", + "To get a label prediction from our probabilistic classifier, one needs to choose a probability threshold $t$.\n", + "The default is to predict label 1 (fraud) if the predicted probability is larger than $t=50\\%$ and all the following metrics implicitly use this default.\n", "\n", "* **False** negatives and **false** positives are samples that were **incorrectly** classified\n", "* **True** negatives and **true** positives are samples that were **correctly** classified\n", "* **Accuracy** is the percentage of examples correctly classified\n", - "\u003e $\\frac{\\text{true samples}}{\\text{total samples}}$\n", + "> $\\frac{\\text{true samples}}{\\text{total samples}}$\n", "* **Precision** is the percentage of **predicted** positives that were correctly classified\n", - "\u003e $\\frac{\\text{true positives}}{\\text{true positives + false positives}}$\n", + "> $\\frac{\\text{true positives}}{\\text{true positives + false positives}}$\n", "* **Recall** is the percentage of **actual** positives that were correctly classified\n", - "\u003e $\\frac{\\text{true positives}}{\\text{true positives + false negatives}}$\n", - "* **AUC** refers to the Area Under the Curve of a Receiver Operating Characteristic curve (ROC-AUC). This metric is equal to the probability that a classifier will rank a random positive sample higher than than a random negative sample.\n", + "> $\\frac{\\text{true positives}}{\\text{true positives + false negatives}}$\n", + "\n", + "**Note:** Accuracy is not a helpful metric for this task. You can have 99.8%+ accuracy on this task by predicting False all the time. \n", "\n", - "Note: Accuracy is not a helpful metric for this task. You can 99.8%+ accuracy on this task by predicting False all the time. \n", + "#### Other metrices\n", "\n", - "Read more:\n", - "* [True vs. False and Positive vs. Negative](https://developers.google.com/machine-learning/crash-course/classification/true-false-positive-negative)\n", - "* [Accuracy](https://developers.google.com/machine-learning/crash-course/classification/accuracy)\n", + "The following metrics take into account all possible choices of thresholds $t$.\n", + "\n", + "* **AUC** refers to the Area Under the Curve of a Receiver Operating Characteristic curve (ROC-AUC). This metric is equal to the probability that a classifier will rank a random positive sample higher than a random negative sample.\n", + "* **AUPRC** refers to Area Under the Curve of the Precision-Recall Curve. This metric computes precision-recall pairs for different probability thresholds.\n", + "\n", + "\n", + "#### Read more:\n", + "* [Strictly Proper Scoring Rules, Prediction, and Estimation](https://www.stat.washington.edu/people/raftery/Research/PDF/Gneiting2007jasa.pdf)\n", + "* [True vs. False and Positive vs. Negative](https://developers.google.com/machine-learning/crash-course/classification/true-false-positive-negative)\n", + "* [Accuracy](https://developers.google.com/machine-learning/crash-course/classification/accuracy)\n", "* [Precision and Recall](https://developers.google.com/machine-learning/crash-course/classification/precision-and-recall)\n", - "* [ROC-AUC](https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc)" + "* [ROC-AUC](https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc)\n", + "* [Relationship between Precision-Recall and ROC Curves](https://www.biostat.wisc.edu/~page/rocpr.pdf)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FYdhSAoaF_TK" }, "source": [ @@ -516,7 +496,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IDbltVPg2m2q" }, "source": [ @@ -525,15 +504,13 @@ "Now create and train your model using the function that was defined earlier. Notice that the model is fit using a larger than default batch size of 2048, this is important to ensure that each batch has a decent chance of containing a few positive samples. If the batch size was too small, they would likely have no fraudulent transactions to learn from.\n", "\n", "\n", - "Note: this model will not handle the class imbalance well. You will improve it later in this tutorial." + "Note: Fitting this model will not handle the class imbalance efficiently. You will improve it later in this tutorial." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "ouUkwPcGQsy3" }, "outputs": [], @@ -541,8 +518,9 @@ "EPOCHS = 100\n", "BATCH_SIZE = 2048\n", "\n", - "early_stopping = tf.keras.callbacks.EarlyStopping(\n", - " monitor='val_auc', \n", + "def early_stopping():\n", + " return tf.keras.callbacks.EarlyStopping(\n", + " monitor='val_prc',\n", " verbose=1,\n", " patience=10,\n", " mode='max',\n", @@ -551,10 +529,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "1xlR_dekzw7C" }, "outputs": [], @@ -566,7 +542,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Wx7ND3_SqckO" }, "source": [ @@ -575,10 +550,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "LopSd-yQqO3a" }, "outputs": [], @@ -589,7 +562,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YKIgWqHms_03" }, "source": [ @@ -599,29 +571,25 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qk_3Ry6EoYDq" }, "source": [ - "These are initial guesses are not great. You know the dataset is imbalanced. Set the output layer's bias to reflect that (See: [A Recipe for Training Neural Networks: \"init well\"](http://karpathy.github.io/2019/04/25/recipe/#2-set-up-the-end-to-end-trainingevaluation-skeleton--get-dumb-baselines)). This can help with initial convergence." + "These initial guesses are not great. You know the dataset is imbalanced. Set the output layer's bias to reflect that, see [A Recipe for Training Neural Networks: \"init well\"](http://karpathy.github.io/2019/04/25/recipe/#2-set-up-the-end-to-end-trainingevaluation-skeleton--get-dumb-baselines). This can help with initial convergence." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PdbfWDuVpo6k" }, "source": [ - "With the default bias initialization the loss should be about `math.log(2) = 0.69314` " + "With the default bias initialization the loss should be about `math.log(2) = 0.69314`" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "H-oPqh3SoGXk" }, "outputs": [], @@ -633,12 +601,9 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hE-JRzfKqfhB" }, "source": [ - "\n", - "\n", "The correct bias to set can be derived from:\n", "\n", "$$ p_0 = pos/(pos + neg) = 1/(1+e^{-b_0}) $$\n", @@ -648,10 +613,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "F5KWPSjjstUS" }, "outputs": [], @@ -663,33 +626,29 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "d1juXI9yY1KD" }, "source": [ - "Set that as the initial bias, and the model will give much more reasonable initial guesses. \n", + "Set that as the initial bias, and the model will give much more reasonable initial guesses.\n", "\n", "It should be near: `pos/total = 0.0018`" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "50oyu1uss0i-" }, "outputs": [], "source": [ - "model = make_model(output_bias = initial_bias)\n", + "model = make_model(output_bias=initial_bias)\n", "model.predict(train_features[:10])" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "4xqFYb2KqRHQ" }, "source": [ @@ -700,10 +659,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "xVDqCWXDqHSc" }, "outputs": [], @@ -715,45 +672,40 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "FrDC8hvNr9yw" }, "source": [ - "This initial loss is about 50 times less than if would have been with naive initilization.\n", + "This initial loss is about 50 times less than it would have been with naive initialization.\n", "\n", - "This way the model doesn't need to spend the first few epochs just learning that positive examples are unlikely. This also makes it easier to read plots of the loss during training." + "This way the model doesn't need to spend the first few epochs just learning that positive examples are unlikely. It also makes it easier to read plots of the loss during training." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "0EJj9ixKVBMT" }, "source": [ "### Checkpoint the initial weights\n", "\n", - "To make the various training runs more comparable, keep this initial model's weights in a checkpoint file, and load them into each model before training." + "To make the various training runs more comparable, keep this initial model's weights in a checkpoint file, and load them into each model before training:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "_tSUm4yAVIif" }, "outputs": [], "source": [ - "initial_weights = os.path.join(tempfile.mkdtemp(),'initial_weights')\n", + "initial_weights = os.path.join(tempfile.mkdtemp(), 'initial.weights.h5')\n", "model.save_weights(initial_weights)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "EVXiLyqyZ8AX" }, "source": [ @@ -761,15 +713,13 @@ "\n", "Before moving on, confirm quick that the careful bias initialization actually helped.\n", "\n", - "Train the model for 20 epochs, with and without this careful initialization, and compare the losses: " + "Train the model for 20 epochs, with and without this careful initialization, and compare the losses:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "Dm4-4K5RZ63Q" }, "outputs": [], @@ -782,16 +732,14 @@ " train_labels,\n", " batch_size=BATCH_SIZE,\n", " epochs=20,\n", - " validation_data=(val_features, val_labels), \n", + " validation_data=(val_features, val_labels),\n", " verbose=0)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "j8DsLXHQaSql" }, "outputs": [], @@ -803,39 +751,34 @@ " train_labels,\n", " batch_size=BATCH_SIZE,\n", " epochs=20,\n", - " validation_data=(val_features, val_labels), \n", + " validation_data=(val_features, val_labels),\n", " verbose=0)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "E3XsMBjhauFV" }, "outputs": [], "source": [ "def plot_loss(history, label, n):\n", - " # Use a log scale to show the wide range of values.\n", - " plt.semilogy(history.epoch, history.history['loss'],\n", - " color=colors[n], label='Train '+label)\n", - " plt.semilogy(history.epoch, history.history['val_loss'],\n", - " color=colors[n], label='Val '+label,\n", - " linestyle=\"--\")\n", + " # Use a log scale on y-axis to show the wide range of values.\n", + " plt.semilogy(history.epoch, history.history['loss'],\n", + " color=colors[n], label='Train ' + label)\n", + " plt.semilogy(history.epoch, history.history['val_loss'],\n", + " color=colors[n], label='Val ' + label,\n", + " linestyle=\"--\")\n", " plt.xlabel('Epoch')\n", " plt.ylabel('Loss')\n", - " \n", " plt.legend()" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "dxFaskm7beC7" }, "outputs": [], @@ -847,17 +790,15 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "fKMioV0ddG3R" }, "source": [ - "The above figure makes it clear: In terms of validation loss, on this problem, this careful initialization gives a clear advantage. " + "The above figure makes it clear: In terms of validation loss, on this problem, this careful initialization gives a clear advantage." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RsA_7SEntRaV" }, "source": [ @@ -866,10 +807,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "yZKAc8NCDnoR" }, "outputs": [], @@ -881,39 +820,37 @@ " train_labels,\n", " batch_size=BATCH_SIZE,\n", " epochs=EPOCHS,\n", - " callbacks = [early_stopping],\n", + " callbacks=[early_stopping()],\n", " validation_data=(val_features, val_labels))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "iSaDBYU9xtP6" }, "source": [ "### Check training history\n", - "In this section, you will produce plots of your model's accuracy and loss on the training and validation set. These are useful to check for overfitting, which you can learn more about in this [tutorial](https://www.tensorflow.org/tutorials/keras/overfit_and_underfit).\n", + "\n", + "In this section, you will produce plots of your model's accuracy and loss on the training and validation set. These are useful to check for overfitting, which you can learn more about in the [Overfit and underfit](https://www.tensorflow.org/tutorials/keras/overfit_and_underfit) tutorial.\n", "\n", "Additionally, you can produce these plots for any of the metrics you created above. False negatives are included as an example." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "WTSkhT1jyGu6" }, "outputs": [], "source": [ "def plot_metrics(history):\n", - " metrics = ['loss', 'auc', 'precision', 'recall']\n", + " metrics = ['loss', 'prc', 'precision', 'recall']\n", " for n, metric in enumerate(metrics):\n", " name = metric.replace(\"_\",\" \").capitalize()\n", " plt.subplot(2,2,n+1)\n", - " plt.plot(history.epoch, history.history[metric], color=colors[0], label='Train')\n", + " plt.plot(history.epoch, history.history[metric], color=colors[0], label='Train')\n", " plt.plot(history.epoch, history.history['val_'+metric],\n", " color=colors[0], linestyle=\"--\", label='Val')\n", " plt.xlabel('Epoch')\n", @@ -925,15 +862,13 @@ " else:\n", " plt.ylim([0,1])\n", "\n", - " plt.legend()\n" + " plt.legend()" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "u6LReDsqlZlk" }, "outputs": [], @@ -944,7 +879,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UCa4iWo6WDKR" }, "source": [ @@ -954,21 +888,18 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "aJC1booryouo" }, "source": [ "### Evaluate metrics\n", "\n", - "You can use a [confusion matrix](https://developers.google.com/machine-learning/glossary/#confusion_matrix) to summarize the actual vs. predicted labels where the X axis is the predicted label and the Y axis is the actual label." + "You can use a [confusion matrix](https://developers.google.com/machine-learning/glossary/#confusion_matrix) to summarize the actual vs. predicted labels, where the X axis is the predicted label and the Y axis is the actual label:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "aNS796IJKrev" }, "outputs": [], @@ -979,19 +910,17 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "MVWBGfADwbWI" }, "outputs": [], "source": [ - "def plot_cm(labels, predictions, p=0.5):\n", - " cm = confusion_matrix(labels, predictions \u003e p)\n", + "def plot_cm(labels, predictions, threshold=0.5):\n", + " cm = confusion_matrix(labels, predictions > threshold)\n", " plt.figure(figsize=(5,5))\n", " sns.heatmap(cm, annot=True, fmt=\"d\")\n", - " plt.title('Confusion matrix @{:.2f}'.format(p))\n", + " plt.title('Confusion matrix @{:.2f}'.format(threshold))\n", " plt.ylabel('Actual label')\n", " plt.xlabel('Predicted label')\n", "\n", @@ -1005,19 +934,16 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "nOTjD5Z5Wp1U" }, "source": [ - "Evaluate your model on the test dataset and display the results for the metrics you created above." + "Evaluate your model on the test dataset and display the results for the metrics you created above:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "poh_hZngt2_9" }, "outputs": [], @@ -1034,31 +960,55 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PyZtSr1v6L4t" }, "source": [ - "If the model had predicted everything perfectly, this would be a [diagonal matrix](https://en.wikipedia.org/wiki/Diagonal_matrix) where values off the main diagonal, indicating incorrect predictions, would be zero. In this case the matrix shows that you have relatively few false positives, meaning that there were relatively few legitimate transactions that were incorrectly flagged. However, you would likely want to have even fewer false negatives despite the cost of increasing the number of false positives. This trade off may be preferable because false negatives would allow fraudulent transactions to go through, whereas false positives may cause an email to be sent to a customer to ask them to verify their card activity." + "If the model had predicted everything perfectly (impossible with true randomness), this would be a [diagonal matrix](https://en.wikipedia.org/wiki/Diagonal_matrix) where values off the main diagonal, indicating incorrect predictions, would be zero. In this case, the matrix shows that you have relatively few false positives, meaning that there were relatively few legitimate transactions that were incorrectly flagged." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "P-QpQsip_F2Q" }, + "source": [ + "### Changing the threshold\n", + "\n", + "The default threshold of $t=50\\%$ corresponds to equal costs of false negatives and false positives.\n", + "In the case of fraud detection, however, you would likely associate higher costs to false negatives than to false positives.\n", + "This trade off may be preferable because false negatives would allow fraudulent transactions to go through, whereas false positives may cause an email to be sent to a customer to ask them to verify their card activity.\n", + "\n", + "By decreasing the threshold, we attribute higher cost to false negatives, thereby increasing missed transactions at the price of more false positives.\n", + "We test thresholds at 10% and at 1%." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "52bd793e04bb" + }, + "outputs": [], + "source": [ + "plot_cm(test_labels, test_predictions_baseline, threshold=0.1)\n", + "plot_cm(test_labels, test_predictions_baseline, threshold=0.01)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kF8k-g9goRni" + }, "source": [ "### Plot the ROC\n", "\n", - "Now plot the [ROC](https://developers.google.com/machine-learning/glossary#ROC). This plot is useful because it shows, at a glance, the range of performance the model can reach just by tuning the output threshold." + "Now plot the [ROC](https://developers.google.com/machine-learning/glossary#ROC). This plot is useful because it shows, at a glance, the range of performance the model can reach by tuning the output threshold over its full range (0 to 1). So each point corresponds to a single value of the threshold." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "lhaxsLSvANF9" }, "outputs": [], @@ -1078,23 +1028,63 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "DfHHspttKJE0" }, "outputs": [], "source": [ "plot_roc(\"Train Baseline\", train_labels, train_predictions_baseline, color=colors[0])\n", "plot_roc(\"Test Baseline\", test_labels, test_predictions_baseline, color=colors[0], linestyle='--')\n", - "plt.legend(loc='lower right')" + "plt.legend(loc='lower right');" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Y5twGRLfNwmO" + }, + "source": [ + "### Plot the PRC\n", + "\n", + "Now plot the [AUPRC](https://developers.google.com/machine-learning/glossary?hl=en#PR_AUC). Area under the interpolated precision-recall curve, obtained by plotting (recall, precision) points for different values of the classification threshold. Depending on how it's calculated, PR AUC may be equivalent to the average precision of the model.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XV6JSlFGEqGI" + }, + "outputs": [], + "source": [ + "def plot_prc(name, labels, predictions, **kwargs):\n", + " precision, recall, _ = sklearn.metrics.precision_recall_curve(labels, predictions)\n", + "\n", + " plt.plot(precision, recall, label=name, linewidth=2, **kwargs)\n", + " plt.xlabel('Precision')\n", + " plt.ylabel('Recall')\n", + " plt.grid(True)\n", + " ax = plt.gca()\n", + " ax.set_aspect('equal')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FdQs_PcqEsiL" + }, + "outputs": [], + "source": [ + "plot_prc(\"Train Baseline\", train_labels, train_predictions_baseline, color=colors[0])\n", + "plot_prc(\"Test Baseline\", test_labels, test_predictions_baseline, color=colors[0], linestyle='--')\n", + "plt.legend(loc='lower right');" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "gpdsFyp64DhY" }, "source": [ @@ -1104,7 +1094,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "cveQoiMyGQCo" }, "source": [ @@ -1114,29 +1103,26 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "ePGp6GUE1WfH" }, "source": [ "### Calculate class weights\n", "\n", - "The goal is to identify fradulent transactions, but you don't have very many of those positive samples to work with, so you would want to have the classifier heavily weight the few examples that are available. You can do this by passing Keras weights for each class through a parameter. These will cause the model to \"pay more attention\" to examples from an under-represented class." + "The goal is to identify fraudulent transactions, but you don't have very many of those positive samples to work with, so you would want to have the classifier heavily weight the few examples that are available. You can do this by passing Keras weights for each class through a parameter. These will cause the model to \"pay more attention\" to examples from an under-represented class. Note, however, that this does not increase in any way the amount of information of your dataset. In the end, using class weights is more or less equivalent to changing the output bias or to changing the threshold. Let's see how it works out." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "qjGWErngGny7" }, "outputs": [], "source": [ "# Scaling by total/2 helps keep the loss to a similar magnitude.\n", "# The sum of the weights of all examples stays the same.\n", - "weight_for_0 = (1 / neg)*(total)/2.0 \n", - "weight_for_1 = (1 / pos)*(total)/2.0\n", + "weight_for_0 = (1 / neg) * (total / 2.0)\n", + "weight_for_1 = (1 / pos) * (total / 2.0)\n", "\n", "class_weight = {0: weight_for_0, 1: weight_for_1}\n", "\n", @@ -1147,7 +1133,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Mk1OOE2ZSHzy" }, "source": [ @@ -1155,15 +1140,13 @@ "\n", "Now try re-training and evaluating the model with class weights to see how that affects the predictions.\n", "\n", - "Note: Using `class_weights` changes the range of the loss. This may affect the stability of the training depending on the optimizer. Optimizers whose step size is dependent on the magnitude of the gradient, like `optimizers.SGD`, may fail. The optimizer used here, `optimizers.Adam`, is unaffected by the scaling change. Also note that because of the weighting, the total losses are not comparable between the two models." + "Note: Using `class_weights` changes the range of the loss. This may affect the stability of the training depending on the optimizer. Optimizers whose step size is dependent on the magnitude of the gradient, like `tf.keras.optimizers.SGD`, may fail. The optimizer used here, `tf.keras.optimizers.Adam`, is unaffected by the scaling change. Also note that because of the weighting, the total losses are not comparable between the two models." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "UJ589fn8ST3x" }, "outputs": [], @@ -1176,16 +1159,15 @@ " train_labels,\n", " batch_size=BATCH_SIZE,\n", " epochs=EPOCHS,\n", - " callbacks = [early_stopping],\n", + " callbacks=[early_stopping()],\n", " validation_data=(val_features, val_labels),\n", " # The class weights go here\n", - " class_weight=class_weight) " + " class_weight=class_weight)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "R0ynYRO0G3Lx" }, "source": [ @@ -1194,10 +1176,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "BBe9FMO5ucTC" }, "outputs": [], @@ -1208,7 +1188,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "REy6WClTZIwQ" }, "source": [ @@ -1217,10 +1196,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "nifqscPGw-5w" }, "outputs": [], @@ -1231,10 +1208,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "owKL2vdMBJr6" }, "outputs": [], @@ -1251,17 +1226,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "PTh1rtDn8r4-" }, "source": [ - "Here you can see that with class weights the accuracy and precision are lower because there are more false positives, but conversely the recall and AUC are higher because the model also found more true positives. Despite having lower accuracy, this model has higher recall (and identifies more fraudulent transactions). Of course, there is a cost to both types of error (you wouldn't want to bug users by flagging too many legitimate transactions as fraudulent, either). Carefully consider the trade offs between these different types of errors for your application." + "Here you can see that with class weights the accuracy and precision are lower because there are more false positives, but conversely the recall and AUC are higher because the model also found more true positives. Despite having lower accuracy, this model has higher recall (and identifies more fraudulent transactions than the baseline model at threshold 50%). Of course, there is a cost to both types of error (you wouldn't want to bug users by flagging too many legitimate transactions as fraudulent, either). Carefully consider the trade-offs between these different types of errors for your application.\n", + "\n", + "Compared to the baseline model with changed threshold, the class weighted model is clearly inferior. The superiority of the baseline model is further confirmed by the lower test loss value (cross entropy and mean squared error) and additionally can be seen by plotting the ROC curves of both models together." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "hXDAwyr0HYdX" }, "source": [ @@ -1270,10 +1245,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "3hzScIVZS1Xm" }, "outputs": [], @@ -1285,13 +1258,39 @@ "plot_roc(\"Test Weighted\", test_labels, test_predictions_weighted, color=colors[1], linestyle='--')\n", "\n", "\n", - "plt.legend(loc='lower right')" + "plt.legend(loc='lower right');" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_0krS8g1OTbD" + }, + "source": [ + "### Plot the PRC" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7jHnmVebOWOC" + }, + "outputs": [], + "source": [ + "plot_prc(\"Train Baseline\", train_labels, train_predictions_baseline, color=colors[0])\n", + "plot_prc(\"Test Baseline\", test_labels, test_predictions_baseline, color=colors[0], linestyle='--')\n", + "\n", + "plot_prc(\"Train Weighted\", train_labels, train_predictions_weighted, color=colors[1])\n", + "plot_prc(\"Test Weighted\", test_labels, test_predictions_weighted, color=colors[1], linestyle='--')\n", + "\n", + "\n", + "plt.legend(loc='lower right');" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "5ysRtr6xHnXP" }, "source": [ @@ -1301,7 +1300,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "18VUHNc-UF5w" }, "source": [ @@ -1312,10 +1310,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "sHirNp6u7OWp" }, "outputs": [], @@ -1330,22 +1326,19 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "WgBVbX7P7QrL" }, "source": [ "#### Using NumPy\n", "\n", - "You can balance the dataset manually by choosing the right number of random \n", + "You can balance the dataset manually by choosing the right number of random\n", "indices from the positive examples:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "BUzGjSkwqT88" }, "outputs": [], @@ -1361,10 +1354,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "7ie_FFet6cep" }, "outputs": [], @@ -1383,7 +1374,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "IYfJe2Kc-FAz" }, "source": [ @@ -1393,7 +1383,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "usyixaST8v5P" }, "source": [ @@ -1402,10 +1391,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "yF4OZ-rI6xb6" }, "outputs": [], @@ -1424,7 +1411,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "RNQUx-OA-oJc" }, "source": [ @@ -1433,10 +1419,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "llXc9rNH7Fbz" }, "outputs": [], @@ -1450,33 +1434,28 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "sLEfjZO0-vbN" }, "source": [ - "Merge the two together using `experimental.sample_from_datasets`:" + "Merge the two together using `tf.data.Dataset.sample_from_datasets`:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "e7w9UQPT9wzE" }, "outputs": [], "source": [ - "resampled_ds = tf.data.experimental.sample_from_datasets([pos_ds, neg_ds], weights=[0.5, 0.5])\n", + "resampled_ds = tf.data.Dataset.sample_from_datasets([pos_ds, neg_ds], weights=[0.5, 0.5])\n", "resampled_ds = resampled_ds.batch(BATCH_SIZE).prefetch(2)" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "EWXARdTdAuQK" }, "outputs": [], @@ -1488,7 +1467,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "irgqf3YxAyN0" }, "source": [ @@ -1499,22 +1477,19 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "xH-7K46AAxpq" }, "outputs": [], "source": [ - "resampled_steps_per_epoch = np.ceil(2.0*neg/BATCH_SIZE)\n", + "resampled_steps_per_epoch = int(np.ceil(2.0*neg/BATCH_SIZE))\n", "resampled_steps_per_epoch" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "XZ1BvEpcBVHP" }, "source": [ @@ -1522,15 +1497,13 @@ "\n", "Now try training the model with the resampled data set instead of using class weights to see how these methods compare.\n", "\n", - "Note: Because the data was balanced by replicating the positive examples, the total dataset size is larger, and each epoch runs for more training steps. " + "Note: Because the data was balanced by replicating the positive examples, the total dataset size is larger, and each epoch runs for more training steps." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "soRQ89JYqd6b" }, "outputs": [], @@ -1539,30 +1512,29 @@ "resampled_model.load_weights(initial_weights)\n", "\n", "# Reset the bias to zero, since this dataset is balanced.\n", - "output_layer = resampled_model.layers[-1] \n", + "output_layer = resampled_model.layers[-1]\n", "output_layer.bias.assign([0])\n", "\n", "val_ds = tf.data.Dataset.from_tensor_slices((val_features, val_labels)).cache()\n", - "val_ds = val_ds.batch(BATCH_SIZE).prefetch(2) \n", + "val_ds = val_ds.batch(BATCH_SIZE).prefetch(2)\n", "\n", "resampled_history = resampled_model.fit(\n", " resampled_ds,\n", " epochs=EPOCHS,\n", " steps_per_epoch=resampled_steps_per_epoch,\n", - " callbacks = [early_stopping],\n", + " callbacks=[early_stopping()],\n", " validation_data=val_ds)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "avALvzUp3T_c" }, "source": [ "If the training process were considering the whole dataset on each gradient update, this oversampling would be basically identical to the class weighting.\n", "\n", - "But when training the model batch-wise, as you did here, the oversampled data provides a smoother gradient signal: Instead of each positive example being shown in one batch with a large weight, they're shown in many different batches each time with a small weight. \n", + "But when training the model batch-wise, as you did here, the oversampled data provides a smoother gradient signal: Instead of each positive example being shown in one batch with a large weight, they're shown in many different batches each time with a small weight.\n", "\n", "This smoother gradient signal makes it easier to train the model." ] @@ -1570,57 +1542,49 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "klHZ0HV76VC5" }, "source": [ "### Check training history\n", "\n", - "Note that the distributions of metrics will be different here, because the training data has a totally different distribution from the validation and test data. " + "Note that the distributions of metrics will be different here, because the training data has a totally different distribution from the validation and test data." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "YoUGfr1vuivl" }, "outputs": [], "source": [ - "plot_metrics(resampled_history )" + "plot_metrics(resampled_history)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1PuH3A2vnwrh" }, "source": [ - "### Re-train\n", - "\n" + "### Re-train\n" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "KFLxRL8eoDE5" }, "source": [ - "Because training is easier on the balanced data, the above training procedure may overfit quickly. \n", + "Because training is easier on the balanced data, the above training procedure may overfit quickly.\n", "\n", - "So break up the epochs to give the `callbacks.EarlyStopping` finer control over when to stop training." + "So break up the epochs to give the `tf.keras.callbacks.EarlyStopping` finer control over when to stop training." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "e_yn9I26qAHU" }, "outputs": [], @@ -1629,22 +1593,21 @@ "resampled_model.load_weights(initial_weights)\n", "\n", "# Reset the bias to zero, since this dataset is balanced.\n", - "output_layer = resampled_model.layers[-1] \n", + "output_layer = resampled_model.layers[-1]\n", "output_layer.bias.assign([0])\n", "\n", "resampled_history = resampled_model.fit(\n", " resampled_ds,\n", " # These are not real epochs\n", - " steps_per_epoch = 20,\n", + " steps_per_epoch=20,\n", " epochs=10*EPOCHS,\n", - " callbacks = [early_stopping],\n", + " callbacks=[early_stopping()],\n", " validation_data=(val_ds))" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UuJYKv0gpBK1" }, "source": [ @@ -1653,10 +1616,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "FMycrpJwn39w" }, "outputs": [], @@ -1667,7 +1628,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "bUuE5HOWZiwP" }, "source": [ @@ -1676,10 +1636,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "C0fmHSgXxFdW" }, "outputs": [], @@ -1690,10 +1648,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "FO0mMOYUDWFk" }, "outputs": [], @@ -1703,14 +1659,12 @@ "for name, value in zip(resampled_model.metrics_names, resampled_results):\n", " print(name, ': ', value)\n", "print()\n", - "\n", - "plot_cm(test_labels, test_predictions_weighted)" + "plot_cm(test_labels, test_predictions_resampled)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "_xYozM1IIITq" }, "source": [ @@ -1719,48 +1673,64 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "fye_CiuYrZ1U" }, "outputs": [], "source": [ "plot_roc(\"Train Baseline\", train_labels, train_predictions_baseline, color=colors[0])\n", "plot_roc(\"Test Baseline\", test_labels, test_predictions_baseline, color=colors[0], linestyle='--')\n", - "\n", "plot_roc(\"Train Weighted\", train_labels, train_predictions_weighted, color=colors[1])\n", "plot_roc(\"Test Weighted\", test_labels, test_predictions_weighted, color=colors[1], linestyle='--')\n", + "plot_roc(\"Train Resampled\", train_labels, train_predictions_resampled, color=colors[2])\n", + "plot_roc(\"Test Resampled\", test_labels, test_predictions_resampled, color=colors[2], linestyle='--')\n", + "plt.legend(loc='lower right');" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vayGnv0VOe_v" + }, + "source": [ + "### Plot the AUPRC\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wgWXQ8aeOhCZ" + }, + "outputs": [], + "source": [ + "plot_prc(\"Train Baseline\", train_labels, train_predictions_baseline, color=colors[0])\n", + "plot_prc(\"Test Baseline\", test_labels, test_predictions_baseline, color=colors[0], linestyle='--')\n", + "\n", + "plot_prc(\"Train Weighted\", train_labels, train_predictions_weighted, color=colors[1])\n", + "plot_prc(\"Test Weighted\", test_labels, test_predictions_weighted, color=colors[1], linestyle='--')\n", "\n", - "plot_roc(\"Train Resampled\", train_labels, train_predictions_resampled, color=colors[2])\n", - "plot_roc(\"Test Resampled\", test_labels, test_predictions_resampled, color=colors[2], linestyle='--')\n", - "plt.legend(loc='lower right')" + "plot_prc(\"Train Resampled\", train_labels, train_predictions_resampled, color=colors[2])\n", + "plot_prc(\"Test Resampled\", test_labels, test_predictions_resampled, color=colors[2], linestyle='--')\n", + "plt.legend(loc='lower right');" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "3o3f0ywl8uqW" }, "source": [ "## Applying this tutorial to your problem\n", "\n", - "Imbalanced data classification is an inherantly difficult task since there are so few samples to learn from. You should always start with the data first and do your best to collect as many samples as possible and give substantial thought to what features may be relevant so the model can get the most out of your minority class. At some point your model may struggle to improve and yield the results you want, so it is important to keep in mind the context of your problem and the trade offs between different types of errors." + "Imbalanced data classification is an inherently difficult task since there are so few samples to learn from. You should always start with the data first and do your best to collect as many samples as possible and give substantial thought to what features may be relevant so the model can get the most out of your minority class. At some point your model may struggle to improve and yield the results you want, so it is important to keep in mind the context of your problem and the trade offs between different types of errors." ] } ], "metadata": { "colab": { - "collapsed_sections": [], - "last_runtime": { - "build_target": "", - "kind": "local" - }, "name": "imbalanced_data.ipynb", - "private_outputs": true, - "provenance": [], "toc_visible": true }, "kernelspec": { diff --git a/site/en/tutorials/structured_data/preprocessing_layers.ipynb b/site/en/tutorials/structured_data/preprocessing_layers.ipynb new file mode 100644 index 00000000000..d05df3c6d21 --- /dev/null +++ b/site/en/tutorials/structured_data/preprocessing_layers.ipynb @@ -0,0 +1,894 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "zg02FZzDyEqd" + }, + "source": [ + "##### Copyright 2019 The TensorFlow Authors.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "2mapZ9afGJ69" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sMYQvJuBi7MS" + }, + "source": [ + "# Classify structured data using Keras preprocessing layers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8FaL4wnr22oy" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " \n", + " \n", + " View on TensorFlow.org\n", + " \n", + " \n", + " \n", + " Run in Google Colab\n", + " \n", + " \n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Nna1tOKxyEqe" + }, + "source": [ + "This tutorial demonstrates how to classify structured data, such as tabular data, using a simplified version of the PetFinder dataset from a Kaggle competition stored in a CSV file.\n", + "\n", + "You will use [Keras](https://www.tensorflow.org/guide/keras) to define the model, and [Keras preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) as a bridge to map from columns in a CSV file to features used to train the model. The goal is to predict if a pet will be adopted.\n", + "\n", + "This tutorial contains complete code for:\n", + "\n", + "* Loading a CSV file into a DataFrame using pandas.\n", + "* Building an input pipeline to batch and shuffle the rows using `tf.data`. (Visit [tf.data: Build TensorFlow input pipelines](../../guide/data.ipynb) for more details.)\n", + "* Mapping from columns in the CSV file to features used to train the model with the Keras preprocessing layers.\n", + "* Building, training, and evaluating a model using the Keras built-in methods." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h5xkXCicjFQD" + }, + "source": [ + "Note: This tutorial is similar to [Classify structured data with feature columns](../structured_data/feature_columns.ipynb). This version uses the [Keras preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) instead of the `tf.feature_column` API, as the former are more intuitive and can be easily included inside your model to simplify deployment." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZHxU1FMNpomc" + }, + "source": [ + "## The PetFinder.my mini dataset\n", + "\n", + "There are several thousand rows in the PetFinder.my mini's CSV dataset file, where each row describes a pet (a dog or a cat) and each column describes an attribute, such as age, breed, color, and so on.\n", + "\n", + "In the dataset's summary below, notice there are mostly numerical and categorical columns. In this tutorial, you will only be dealing with those two feature types, dropping `Description` (a free text feature) and `AdoptionSpeed` (a classification feature) during data preprocessing.\n", + "\n", + "| Column | Pet description | Feature type | Data type |\n", + "| --------------- | ----------------------------- | -------------- | --------- |\n", + "| `Type` | Type of animal (`Dog`, `Cat`) | Categorical | String |\n", + "| `Age` | Age | Numerical | Integer |\n", + "| `Breed1` | Primary breed | Categorical | String |\n", + "| `Color1` | Color 1 | Categorical | String |\n", + "| `Color2` | Color 2 | Categorical | String |\n", + "| `MaturitySize` | Size at maturity | Categorical | String |\n", + "| `FurLength` | Fur length | Categorical | String |\n", + "| `Vaccinated` | Pet has been vaccinated | Categorical | String |\n", + "| `Sterilized` | Pet has been sterilized | Categorical | String |\n", + "| `Health` | Health condition | Categorical | String |\n", + "| `Fee` | Adoption fee | Numerical | Integer |\n", + "| `Description` | Profile write-up | Text | String |\n", + "| `PhotoAmt` | Total uploaded photos | Numerical | Integer |\n", + "| `AdoptionSpeed` | Categorical speed of adoption | Classification | Integer |" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vjFbdBldyEqf" + }, + "source": [ + "## Import TensorFlow and other libraries\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LklnLlt6yEqf" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import tensorflow as tf\n", + "\n", + "from tensorflow.keras import layers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TKU7RyoQGVKB" + }, + "outputs": [], + "source": [ + "tf.__version__" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UXvBvobayEqi" + }, + "source": [ + "## Load the dataset and read it into a pandas DataFrame\n", + "\n", + "pandas is a Python library with many helpful utilities for loading and working with structured data. Use `tf.keras.utils.get_file` to download and extract the CSV file with the PetFinder.my mini dataset, and load it into a DataFrame with `pandas.read_csv`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qJ4Ajn-YyEqj" + }, + "outputs": [], + "source": [ + "dataset_url = 'http://storage.googleapis.com/download.tensorflow.org/data/petfinder-mini.zip'\n", + "csv_file = 'datasets/petfinder-mini/petfinder-mini.csv'\n", + "\n", + "tf.keras.utils.get_file('petfinder_mini.zip', dataset_url,\n", + " extract=True, cache_dir='.')\n", + "dataframe = pd.read_csv(csv_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "efa6910dfa5f" + }, + "source": [ + "Inspect the dataset by checking the first five rows of the DataFrame:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3uiq4hoIGyXI" + }, + "outputs": [], + "source": [ + "dataframe.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C3zDbrozyEqq" + }, + "source": [ + "## Create a target variable\n", + "\n", + "The original task in Kaggle's PetFinder.my Adoption Prediction competition was to predict the speed at which a pet will be adopted (e.g. in the first week, the first month, the first three months, and so on).\n", + "\n", + "In this tutorial, you will simplify the task by transforming it into a binary classification problem, where you simply have to predict whether a pet was adopted or not.\n", + "\n", + "After modifying the `AdoptionSpeed` column, `0` will indicate the pet was not adopted, and `1` will indicate it was." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wmMDc46-yEqq" + }, + "outputs": [], + "source": [ + "# In the original dataset, `'AdoptionSpeed'` of `4` indicates\n", + "# a pet was not adopted.\n", + "dataframe['target'] = np.where(dataframe['AdoptionSpeed']==4, 0, 1)\n", + "\n", + "# Drop unused features.\n", + "dataframe = dataframe.drop(columns=['AdoptionSpeed', 'Description'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sp0NCbswyEqs" + }, + "source": [ + "## Split the DataFrame into training, validation, and test sets\n", + "\n", + "The dataset is in a single pandas DataFrame. Split it into training, validation, and test sets using a, for example, 80:10:10 ratio, respectively:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XvSinthO8oMj" + }, + "outputs": [], + "source": [ + "train, val, test = np.split(dataframe.sample(frac=1), [int(0.8*len(dataframe)), int(0.9*len(dataframe))])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "U02Q1moWoPwQ" + }, + "outputs": [], + "source": [ + "print(len(train), 'training examples')\n", + "print(len(val), 'validation examples')\n", + "print(len(test), 'test examples')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C_7uVu-xyEqv" + }, + "source": [ + "## Create an input pipeline using tf.data\n", + "\n", + "Next, create a utility function that converts each training, validation, and test set DataFrame into a `tf.data.Dataset`, then shuffles and batches the data.\n", + "\n", + "Note: If you were working with a very large CSV file (so large that it does not fit into memory), you would use the `tf.data` API to read it from disk directly. That is not covered in this tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7r4j-1lRyEqw" + }, + "outputs": [], + "source": [ + "def df_to_dataset(dataframe, shuffle=True, batch_size=32):\n", + " df = dataframe.copy()\n", + " labels = df.pop('target')\n", + " df = {key: value.to_numpy()[:,tf.newaxis] for key, value in dataframe.items()}\n", + " ds = tf.data.Dataset.from_tensor_slices((dict(df), labels))\n", + " if shuffle:\n", + " ds = ds.shuffle(buffer_size=len(dataframe))\n", + " ds = ds.batch(batch_size)\n", + " ds = ds.prefetch(batch_size)\n", + " return ds" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PYxIXH579uS9" + }, + "source": [ + "Now, use the newly created function (`df_to_dataset`) to check the format of the data the input pipeline helper function returns by calling it on the training data, and use a small batch size to keep the output readable:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tYiNH-QI96Jo" + }, + "outputs": [], + "source": [ + "batch_size = 5\n", + "train_ds = df_to_dataset(train, batch_size=batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nFYir6S8HgIJ" + }, + "outputs": [], + "source": [ + "[(train_features, label_batch)] = train_ds.take(1)\n", + "print('Every feature:', list(train_features.keys()))\n", + "print('A batch of ages:', train_features['Age'])\n", + "print('A batch of targets:', label_batch )" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "geqHWW54Hmte" + }, + "source": [ + "As the output demonstrates, the training set returns a dictionary of column names (from the DataFrame) that map to column values from rows." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-v50jBIuj4gb" + }, + "source": [ + "## Apply the Keras preprocessing layers\n", + "\n", + "The Keras preprocessing layers allow you to build Keras-native input processing pipelines, which can be used as independent preprocessing code in non-Keras workflows, combined directly with Keras models, and exported as part of a Keras SavedModel.\n", + "\n", + "In this tutorial, you will use the following four preprocessing layers to demonstrate how to perform preprocessing, structured data encoding, and feature engineering:\n", + "\n", + "- `tf.keras.layers.Normalization`: Performs feature-wise normalization of input features.\n", + "- `tf.keras.layers.CategoryEncoding`: Turns integer categorical features into one-hot, multi-hot, or tf-idf\n", + "dense representations.\n", + "- `tf.keras.layers.StringLookup`: Turns string categorical values into integer indices.\n", + "- `tf.keras.layers.IntegerLookup`: Turns integer categorical values into integer indices.\n", + "\n", + "You can learn more about the available layers in the [Working with preprocessing layers](https://www.tensorflow.org/guide/keras/preprocessing_layers) guide.\n", + "\n", + "- For _numerical features_ of the PetFinder.my mini dataset, you will use a `tf.keras.layers.Normalization` layer to standardize the distribution of the data.\n", + "- For _categorical features_, such as pet `Type`s (`Dog` and `Cat` strings), you will transform them to multi-hot encoded tensors with `tf.keras.layers.CategoryEncoding`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "twXBSxnT66o8" + }, + "source": [ + "### Numerical columns\n", + "\n", + "For each numeric feature in the PetFinder.my mini dataset, you will use a `tf.keras.layers.Normalization` layer to standardize the distribution of the data.\n", + "\n", + "Define a new utility function that returns a layer which applies feature-wise normalization to numerical features using that Keras preprocessing layer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "D6OuEKMMyEq1" + }, + "outputs": [], + "source": [ + "def get_normalization_layer(name, dataset):\n", + " # Create a Normalization layer for the feature.\n", + " normalizer = layers.Normalization(axis=None)\n", + "\n", + " # Prepare a Dataset that only yields the feature.\n", + " feature_ds = dataset.map(lambda x, y: x[name])\n", + "\n", + " # Learn the statistics of the data.\n", + " normalizer.adapt(feature_ds)\n", + "\n", + " return normalizer" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lL4TRreQCPjV" + }, + "source": [ + "Next, test the new function by calling it on the total uploaded pet photo features to normalize `'PhotoAmt'`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MpKgUDyk69bM" + }, + "outputs": [], + "source": [ + "photo_count_col = train_features['PhotoAmt']\n", + "layer = get_normalization_layer('PhotoAmt', train_ds)\n", + "layer(photo_count_col)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "foWY00YBUx9N" + }, + "source": [ + "Note: If you have many numeric features (hundreds, or more), it is more efficient to concatenate them first and use a single `tf.keras.layers.Normalization` layer." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yVD--2WZ7vmh" + }, + "source": [ + "### Categorical columns\n", + "\n", + "Pet `Type`s in the dataset are represented as strings—`Dog`s and `Cat`s—which need to be multi-hot encoded before being fed into the model. The `Age` feature\n", + "\n", + "Define another new utility function that returns a layer which maps values from a vocabulary to integer indices and multi-hot encodes the features using the `tf.keras.layers.StringLookup`, `tf.keras.layers.IntegerLookup`, and `tf.keras.CategoryEncoding` preprocessing layers:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GmgaeRjlDoUO" + }, + "outputs": [], + "source": [ + "def get_category_encoding_layer(name, dataset, dtype, max_tokens=None):\n", + " # Create a layer that turns strings into integer indices.\n", + " if dtype == 'string':\n", + " index = layers.StringLookup(max_tokens=max_tokens)\n", + " # Otherwise, create a layer that turns integer values into integer indices.\n", + " else:\n", + " index = layers.IntegerLookup(max_tokens=max_tokens)\n", + "\n", + " # Prepare a `tf.data.Dataset` that only yields the feature.\n", + " feature_ds = dataset.map(lambda x, y: x[name])\n", + "\n", + " # Learn the set of possible values and assign them a fixed integer index.\n", + " index.adapt(feature_ds)\n", + "\n", + " # Encode the integer indices.\n", + " encoder = layers.CategoryEncoding(num_tokens=index.vocabulary_size())\n", + "\n", + " # Apply multi-hot encoding to the indices. The lambda function captures the\n", + " # layer, so you can use them, or include them in the Keras Functional model later.\n", + " return lambda feature: encoder(index(feature))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7b3DwtTeCPjX" + }, + "source": [ + "Test the `get_category_encoding_layer` function by calling it on pet `'Type'` features to turn them into multi-hot encoded tensors:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "X2t2ff9K8PcT" + }, + "outputs": [], + "source": [ + "test_type_col = train_features['Type']\n", + "test_type_layer = get_category_encoding_layer(name='Type',\n", + " dataset=train_ds,\n", + " dtype='string')\n", + "test_type_layer(test_type_col)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j6eDongw8knz" + }, + "source": [ + "Repeat the process on the pet `'Age'` features:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7FjBioQ38oNE" + }, + "outputs": [], + "source": [ + "test_age_col = train_features['Age']\n", + "test_age_layer = get_category_encoding_layer(name='Age',\n", + " dataset=train_ds,\n", + " dtype='int64',\n", + " max_tokens=5)\n", + "test_age_layer(test_age_col)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SiE0glOPkMyh" + }, + "source": [ + "## Preprocess selected features to train the model on\n", + "\n", + "You have learned how to use several types of Keras preprocessing layers. Next, you will:\n", + "\n", + "- Apply the preprocessing utility functions defined earlier on 13 numerical and categorical features from the PetFinder.my mini dataset.\n", + "- Add all the feature inputs to a list.\n", + "\n", + "As mentioned in the beginning, to train the model, you will use the PetFinder.my mini dataset's numerical (`'PhotoAmt'`, `'Fee'`) and categorical (`'Age'`, `'Type'`, `'Color1'`, `'Color2'`, `'Gender'`, `'MaturitySize'`, `'FurLength'`, `'Vaccinated'`, `'Sterilized'`, `'Health'`, `'Breed1'`) features.\n", + "\n", + "Note: If your aim is to build an accurate model, try a larger dataset of your own, and think carefully about which features are the most meaningful to include, and how they should be represented." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Uj1GoHSZ9R3H" + }, + "source": [ + "Earlier, you used a small batch size to demonstrate the input pipeline. Let's now create a new input pipeline with a larger batch size of 256:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Rcv2kQTTo23h" + }, + "outputs": [], + "source": [ + "batch_size = 256\n", + "train_ds = df_to_dataset(train, batch_size=batch_size)\n", + "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", + "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5bIGNYN2V7iR" + }, + "source": [ + "Normalize the numerical features (the number of pet photos and the adoption fee), and add them to one list of inputs called `encoded_features`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Q3RBa51VkaAn" + }, + "outputs": [], + "source": [ + "all_inputs = {}\n", + "encoded_features = []\n", + "\n", + "# Numerical features.\n", + "for header in ['PhotoAmt', 'Fee']:\n", + " numeric_col = tf.keras.Input(shape=(1,), name=header)\n", + " normalization_layer = get_normalization_layer(header, train_ds)\n", + " encoded_numeric_col = normalization_layer(numeric_col)\n", + " all_inputs[header] = numeric_col\n", + " encoded_features.append(encoded_numeric_col)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qVcUAFd6bvlT" + }, + "source": [ + "Turn the integer categorical values from the dataset (the pet age) into integer indices, perform multi-hot encoding, and add the resulting feature inputs to `encoded_features`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1FOMGfZflhoA" + }, + "outputs": [], + "source": [ + "age_col = tf.keras.Input(shape=(1,), name='Age', dtype='int64')\n", + "\n", + "encoding_layer = get_category_encoding_layer(name='Age',\n", + " dataset=train_ds,\n", + " dtype='int64',\n", + " max_tokens=5)\n", + "encoded_age_col = encoding_layer(age_col)\n", + "all_inputs['Age'] = age_col\n", + "encoded_features.append(encoded_age_col)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QYzynk6wdqKe" + }, + "source": [ + "Repeat the same step for the string categorical values:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "K8C8xyiXm-Ie" + }, + "outputs": [], + "source": [ + "categorical_cols = ['Type', 'Color1', 'Color2', 'Gender', 'MaturitySize',\n", + " 'FurLength', 'Vaccinated', 'Sterilized', 'Health', 'Breed1']\n", + "\n", + "for header in categorical_cols:\n", + " categorical_col = tf.keras.Input(shape=(1,), name=header, dtype='string')\n", + " encoding_layer = get_category_encoding_layer(name=header,\n", + " dataset=train_ds,\n", + " dtype='string',\n", + " max_tokens=5)\n", + " encoded_categorical_col = encoding_layer(categorical_col)\n", + " all_inputs[header] = categorical_col\n", + " encoded_features.append(encoded_categorical_col)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YHSnhz2fyEq3" + }, + "source": [ + "## Create, compile, and train the model\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IDGyN_wpo0XS" + }, + "source": [ + "The next step is to create a model using the [Keras Functional API](https://www.tensorflow.org/guide/keras/functional). For the first layer in your model, merge the list of feature inputs—`encoded_features`—into one vector via concatenation with `tf.keras.layers.concatenate`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EtkwHC-akvcv" + }, + "outputs": [], + "source": [ + "encoded_features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6Yrj-_pr6jyL" + }, + "outputs": [], + "source": [ + "all_features = tf.keras.layers.concatenate(encoded_features)\n", + "x = tf.keras.layers.Dense(32, activation=\"relu\")(all_features)\n", + "x = tf.keras.layers.Dropout(0.5)(x)\n", + "output = tf.keras.layers.Dense(1)(x)\n", + "\n", + "model = tf.keras.Model(all_inputs, output)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NRLDRcYAefTA" + }, + "source": [ + "Configure the model with Keras `Model.compile`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GZDb_lJdelSg" + }, + "outputs": [], + "source": [ + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n", + " metrics=[\"accuracy\"],\n", + " run_eagerly=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f6mNMfG6yEq5" + }, + "source": [ + "Let's visualize the connectivity graph:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Y7Bkx4c7yEq5" + }, + "outputs": [], + "source": [ + "# Use `rankdir='LR'` to make the graph horizontal.\n", + "tf.keras.utils.plot_model(model, show_shapes=True, show_layer_names=True, rankdir=\"LR\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CED6OStLyEq7" + }, + "source": [ + "Next, train and test the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OQfE3PC6yEq8" + }, + "outputs": [], + "source": [ + "model.fit(train_ds, epochs=10, validation_data=val_ds)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "T8N2uAdU2Cni" + }, + "outputs": [], + "source": [ + "result = model.evaluate(test_ds, return_dict=True)\n", + "print(result)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LmZMnTKaCZda" + }, + "source": [ + "## Perform inference\n", + "\n", + "The model you have developed can now classify a row from a CSV file directly after you've included the preprocessing layers inside the model itself.\n", + "\n", + "You can now [save and reload the Keras model](../keras/save_and_load.ipynb) with `Model.save` and `Model.load_model` before performing inference on new data:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QH9Zy1sBvwOH" + }, + "outputs": [], + "source": [ + "model.save('my_pet_classifier.keras')\n", + "reloaded_model = tf.keras.models.load_model('my_pet_classifier.keras')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D973plJrdwQ9" + }, + "source": [ + "To get a prediction for a new sample, you can simply call the Keras `Model.predict` method. There are just two things you need to do:\n", + "\n", + "1. Wrap scalars into a list so as to have a batch dimension (`Model`s only process batches of data, not single samples).\n", + "2. Call `tf.convert_to_tensor` on each feature." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rKq4pxtdDa7i" + }, + "outputs": [], + "source": [ + "sample = {\n", + " 'Type': 'Cat',\n", + " 'Age': 3,\n", + " 'Breed1': 'Tabby',\n", + " 'Gender': 'Male',\n", + " 'Color1': 'Black',\n", + " 'Color2': 'White',\n", + " 'MaturitySize': 'Small',\n", + " 'FurLength': 'Short',\n", + " 'Vaccinated': 'No',\n", + " 'Sterilized': 'No',\n", + " 'Health': 'Healthy',\n", + " 'Fee': 100,\n", + " 'PhotoAmt': 2,\n", + "}\n", + "\n", + "input_dict = {name: tf.convert_to_tensor([value]) for name, value in sample.items()}\n", + "predictions = reloaded_model.predict(input_dict)\n", + "prob = tf.nn.sigmoid(predictions[0])\n", + "\n", + "print(\n", + " \"This particular pet had a %.1f percent probability \"\n", + " \"of getting adopted.\" % (100 * prob)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XJQQZEiH2FaB" + }, + "source": [ + "Note: You will typically have better results with deep learning with larger and more complex datasets. When working with a small dataset, such as the simplified PetFinder.my one, you can use a decision tree or a random forest as a strong baseline. The goal of this tutorial is to demonstrate the mechanics of working with structured data, so you have a starting point when working with your own datasets in the future.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "k0QAY2Tb2HYG" + }, + "source": [ + "## Next steps\n", + "\n", + "To learn more about classifying structured data, try working with other datasets. To improve accuracy during training and testing your models, think carefully about which features to include in your model and how they should be represented.\n", + "\n", + "Below are some suggestions for datasets:\n", + "\n", + "- [TensorFlow Datasets: MovieLens](https://www.tensorflow.org/datasets/catalog/movie_lens): A set of movie ratings from a movie recommendation service.\n", + "- [TensorFlow Datasets: Wine Quality](https://www.tensorflow.org/datasets/catalog/wine_quality): Two datasets related to red and white variants of the Portuguese \"Vinho Verde\" wine. You can also find the Red Wine Quality dataset on Kaggle.\n", + "- Kaggle: arXiv Dataset: A corpus of 1.7 million scholarly articles from arXiv, covering physics, computer science, math, statistics, electrical engineering, quantitative biology, and economics.\n" + ] + } + ], + "metadata": { + "colab": { + "name": "preprocessing_layers.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/structured_data/time_series.ipynb b/site/en/tutorials/structured_data/time_series.ipynb index ae0752c86ec..31aab384859 100644 --- a/site/en/tutorials/structured_data/time_series.ipynb +++ b/site/en/tutorials/structured_data/time_series.ipynb @@ -1,1346 +1,2928 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2Pmxv2ioyCRw" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "b-2ShX25yNWf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pa49bUnKyRgF" - }, - "source": [ - "# Time series forecasting" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "11Ilg92myRcw" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GU8C5qm_4vZb" - }, - "source": [ - "This tutorial is an introduction to time series forecasting using Recurrent Neural Networks (RNNs). This is covered in two parts: first, you will forecast a univariate time series, then you will forecast a multivariate time series." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7rZnJaGTWQw0" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import matplotlib as mpl\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import os\n", - "import pandas as pd\n", - "\n", - "mpl.rcParams['figure.figsize'] = (8, 6)\n", - "mpl.rcParams['axes.grid'] = False" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TokBlnUhWFw9" - }, - "source": [ - "## The weather dataset\n", - "This tutorial uses a [weather time series dataset](https://www.bgc-jena.mpg.de/wetter/) recorded by the [Max-Planck-Institute for Biogeochemistry](https://www.bgc-jena.mpg.de/index.php/Main/HomePage).\n", - "\n", - "This dataset contains 14 different features such as air temperature, atmospheric pressure, and humidity. These were collected every 10 minutes, beginning in 2003. For efficiency, you will use only the data collected between 2009 and 2016. This section of the dataset was prepared by François Chollet for his book [Deep Learning with Python](https://www.manning.com/books/deep-learning-with-python)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xyv_i85IWInT" - }, - "outputs": [], - "source": [ - "zip_path = tf.keras.utils.get_file(\n", - " origin='https://storage.googleapis.com/tensorflow/tf-keras-datasets/jena_climate_2009_2016.csv.zip',\n", - " fname='jena_climate_2009_2016.csv.zip',\n", - " extract=True)\n", - "csv_path, _ = os.path.splitext(zip_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TX6uGeeeWIkG" - }, - "outputs": [], - "source": [ - "df = pd.read_csv(csv_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VdbOWXiTWM2T" - }, - "source": [ - "Let's take a glance at the data." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ojHE-iCCWIhz" - }, - "outputs": [], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qfbpcV0MWQzl" - }, - "source": [ - "As you can see above, an observation is recorded every 10 mintues. This means that, for a single hour, you will have 6 observations. Similarly, a single day will contain 144 (6x24) observations. \n", - "\n", - "Given a specific time, let's say you want to predict the temperature 6 hours in the future. In order to make this prediction, you choose to use 5 days of observations. Thus, you would create a window containing the last 720(5x144) observations to train the model. Many such configurations are possible, making this dataset a good one to experiment with.\n", - "\n", - "The function below returns the above described windows of time for the model to train on. The parameter `history_size` is the size of the past window of information. The `target_size` is how far in the future does the model need to learn to predict. The `target_size` is the label that needs to be predicted." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7AoxQuTrWIbi" - }, - "outputs": [], - "source": [ - "def univariate_data(dataset, start_index, end_index, history_size, target_size):\n", - " data = []\n", - " labels = []\n", - "\n", - " start_index = start_index + history_size\n", - " if end_index is None:\n", - " end_index = len(dataset) - target_size\n", - "\n", - " for i in range(start_index, end_index):\n", - " indices = range(i-history_size, i)\n", - " # Reshape data from (history_size,) to (history_size, 1)\n", - " data.append(np.reshape(dataset[indices], (history_size, 1)))\n", - " labels.append(dataset[i+target_size])\n", - " return np.array(data), np.array(labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qoFJZmXBaxCc" - }, - "source": [ - "In both the following tutorials, the first 300,000 rows of the data will be the training dataset, and there remaining will be the validation dataset. This amounts to ~2100 days worth of training data." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ia-MPAHxbInX" - }, - "outputs": [], - "source": [ - "TRAIN_SPLIT = 300000" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EowWDtaNnH1y" - }, - "source": [ - "Setting seed to ensure reproducibility." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-x-GgENynHdx" - }, - "outputs": [], - "source": [ - "tf.random.set_seed(13)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8YEwr-NoWUpV" - }, - "source": [ - "## Part 1: Forecast a univariate time series\n", - "First, you will train a model using only a single feature (temperature), and use it to make predictions for that value in the future.\n", - "\n", - "Let's first extract only the temperature from the dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nbdcnm1_WIY9" - }, - "outputs": [], - "source": [ - "uni_data = df['T (degC)']\n", - "uni_data.index = df['Date Time']\n", - "uni_data.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aQB-46MyWZMm" - }, - "source": [ - "Let's observe how this data looks across time." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ftOExwAqWXSU" - }, - "outputs": [], - "source": [ - "uni_data.plot(subplots=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ejSEiDqBWXQa" - }, - "outputs": [], - "source": [ - "uni_data = uni_data.values" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-eFckdUUHWmT" - }, - "source": [ - "It is important to scale features before training a neural network. Standardization is a common way of doing this scaling by subtracting the mean and dividing by the standard deviation of each feature.", - "You could also use a `tf.keras.utils.normalize` method that rescales the values into a range of [0,1]." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mxbIic5TMlxx" - }, - "source": [ - "Note: The mean and standard deviation should only be computed using the training data." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Eji6njXvHusN" - }, - "outputs": [], - "source": [ - "uni_train_mean = uni_data[:TRAIN_SPLIT].mean()\n", - "uni_train_std = uni_data[:TRAIN_SPLIT].std()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8Gob1YJYH0cH" - }, - "source": [ - "Let's standardize the data." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BO55yRD6H0Dx" - }, - "outputs": [], - "source": [ - "uni_data = (uni_data-uni_train_mean)/uni_train_std" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gn8A_nrccKtn" - }, - "source": [ - "Let's now create the data for the univariate model. For part 1, the model will be given the last 20 recorded temperature observations, and needs to learn to predict the temperature at the next time step. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aJJ-T49vWXOZ" - }, - "outputs": [], - "source": [ - "univariate_past_history = 20\n", - "univariate_future_target = 0\n", - "\n", - "x_train_uni, y_train_uni = univariate_data(uni_data, 0, TRAIN_SPLIT,\n", - " univariate_past_history,\n", - " univariate_future_target)\n", - "x_val_uni, y_val_uni = univariate_data(uni_data, TRAIN_SPLIT, None,\n", - " univariate_past_history,\n", - " univariate_future_target)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aWpVMENsdp0N" - }, - "source": [ - "This is what the `univariate_data` function returns." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "feDd95XFdz5H" - }, - "outputs": [], - "source": [ - "print ('Single window of past history')\n", - "print (x_train_uni[0])\n", - "print ('\\n Target temperature to predict')\n", - "print (y_train_uni[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hni3Jt9OMR1_" - }, - "source": [ - "Now that the data has been created, let's take a look at a single example. The information given to the network is given in blue, and it must predict the value at the red cross." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qVukM9dRipop" - }, - "outputs": [], - "source": [ - "def create_time_steps(length):\n", - " return list(range(-length, 0))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QQeGvh7cWXMR" - }, - "outputs": [], - "source": [ - "def show_plot(plot_data, delta, title):\n", - " labels = ['History', 'True Future', 'Model Prediction']\n", - " marker = ['.-', 'rx', 'go']\n", - " time_steps = create_time_steps(plot_data[0].shape[0])\n", - " if delta:\n", - " future = delta\n", - " else:\n", - " future = 0\n", - "\n", - " plt.title(title)\n", - " for i, x in enumerate(plot_data):\n", - " if i:\n", - " plt.plot(future, plot_data[i], marker[i], markersize=10,\n", - " label=labels[i])\n", - " else:\n", - " plt.plot(time_steps, plot_data[i].flatten(), marker[i], label=labels[i])\n", - " plt.legend()\n", - " plt.xlim([time_steps[0], (future+5)*2])\n", - " plt.xlabel('Time-Step')\n", - " return plt" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Pd05iV-UWXKL" - }, - "outputs": [], - "source": [ - "show_plot([x_train_uni[0], y_train_uni[0]], 0, 'Sample Example')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b5rUJ_2YMWzG" - }, - "source": [ - "### Baseline\n", - "Before proceeding to train a model, let's first set a simple baseline. Given an input point, the baseline method looks at all the history and predicts the next point to be the average of the last 20 observations." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "P9nYWcxMMWnr" - }, - "outputs": [], - "source": [ - "def baseline(history):\n", - " return np.mean(history)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KMcdFYKQMWlm" - }, - "outputs": [], - "source": [ - "show_plot([x_train_uni[0], y_train_uni[0], baseline(x_train_uni[0])], 0,\n", - " 'Baseline Prediction Example')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "067m6t8cMakb" - }, - "source": [ - "Let's see if you can beat this baseline using a recurrent neural network." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H4crpOcoMlSe" - }, - "source": [ - "### Recurrent neural network\n", - "\n", - "A Recurrent Neural Network (RNN) is a type of neural network well-suited to time series data. RNNs process a time series step-by-step, maintaining an internal state summarizing the information they've seen so far. For more details, read the [RNN tutorial](https://www.tensorflow.org/tutorials/sequences/recurrent). In this tutorial, you will use a specialized RNN layer called Long Short Term Memory ([LSTM](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/LSTM))\n", - "\n", - "Let's now use `tf.data` to shuffle, batch, and cache the dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kk-evkrmMWh9" - }, - "outputs": [], - "source": [ - "BATCH_SIZE = 256\n", - "BUFFER_SIZE = 10000\n", - "\n", - "train_univariate = tf.data.Dataset.from_tensor_slices((x_train_uni, y_train_uni))\n", - "train_univariate = train_univariate.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()\n", - "\n", - "val_univariate = tf.data.Dataset.from_tensor_slices((x_val_uni, y_val_uni))\n", - "val_univariate = val_univariate.batch(BATCH_SIZE).repeat()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n2AmKkyVS5Ht" - }, - "source": [ - "The following visualisation should help you understand how the data is represented after batching.\n", - "\n", - "![Time Series](images/time_series.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4nagdTRNfPuZ" - }, - "source": [ - "You will see the LSTM requires the input shape of the data it is being given." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IDbpHosCMWZO" - }, - "outputs": [], - "source": [ - "simple_lstm_model = tf.keras.models.Sequential([\n", - " tf.keras.layers.LSTM(8, input_shape=x_train_uni.shape[-2:]),\n", - " tf.keras.layers.Dense(1)\n", - "])\n", - "\n", - "simple_lstm_model.compile(optimizer='adam', loss='mae')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NOGZtDAqMtSi" - }, - "source": [ - "Let's make a sample prediction, to check the output of the model. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2mPZbIKCMtLR" - }, - "outputs": [], - "source": [ - "for x, y in val_univariate.take(1):\n", - " print(simple_lstm_model.predict(x).shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QYz6RN_mMyau" - }, - "source": [ - "Let's train the model now. Due to the large size of the dataset, in the interest of saving time, each epoch will only run for 200 steps, instead of the complete training data as normally done." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0opH9xi5MtIk" - }, - "outputs": [], - "source": [ - "EVALUATION_INTERVAL = 200\n", - "EPOCHS = 10\n", - "\n", - "simple_lstm_model.fit(train_univariate, epochs=EPOCHS,\n", - " steps_per_epoch=EVALUATION_INTERVAL,\n", - " validation_data=val_univariate, validation_steps=50)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "euyPo_lyNryZ" - }, - "source": [ - "#### Predict using the simple LSTM model\n", - "Now that you have trained your simple LSTM, let's try and make a few predictions." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "S2rRLrs8MtGU" - }, - "outputs": [], - "source": [ - "for x, y in val_univariate.take(3):\n", - " plot = show_plot([x[0].numpy(), y[0].numpy(),\n", - " simple_lstm_model.predict(x)[0]], 0, 'Simple LSTM model')\n", - " plot.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q-AVEJyRNvt0" - }, - "source": [ - "This looks better than the baseline. Now that you have seen the basics, let's move on to part two, where you will work with a multivariate time series." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VlJYi3_HXcw8" - }, - "source": [ - "## Part 2: Forecast a multivariate time series" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hoxNZ2GM7DPm" - }, - "source": [ - "The original dataset contains fourteen features. For simplicity, this section considers only three of the original fourteen. The features used are air temperature, atmospheric pressure, and air density. \n", - "\n", - "To use more features, add their names to this list." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DphrB7bxSNDd" - }, - "outputs": [], - "source": [ - "features_considered = ['p (mbar)', 'T (degC)', 'rho (g/m**3)']" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IfQUSiJfUpXJ" - }, - "outputs": [], - "source": [ - "features = df[features_considered]\n", - "features.index = df['Date Time']\n", - "features.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qSfhTZi5r15R" - }, - "source": [ - "Let's have a look at how each of these features vary across time." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QdgC8zvGr21X" - }, - "outputs": [], - "source": [ - "features.plot(subplots=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cqStgZ-O1b3_" - }, - "source": [ - "As mentioned, the first step will be to standardize the dataset using the mean and standard deviation of the training data." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "W7VuNIwfHRHx" - }, - "outputs": [], - "source": [ - "dataset = features.values\n", - "data_mean = dataset[:TRAIN_SPLIT].mean(axis=0)\n", - "data_std = dataset[:TRAIN_SPLIT].std(axis=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eJUeWDqploCt" - }, - "outputs": [], - "source": [ - "dataset = (dataset-data_mean)/data_std" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LyuGuJUgjUK3" - }, - "source": [ - "### Single step model\n", - "In a single step setup, the model learns to predict a single point in the future based on some history provided.\n", - "\n", - "The below function performs the same windowing task as below, however, here it samples the past observation based on the step size given." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "d-rVX4d3OF86" - }, - "outputs": [], - "source": [ - "def multivariate_data(dataset, target, start_index, end_index, history_size,\n", - " target_size, step, single_step=False):\n", - " data = []\n", - " labels = []\n", - "\n", - " start_index = start_index + history_size\n", - " if end_index is None:\n", - " end_index = len(dataset) - target_size\n", - "\n", - " for i in range(start_index, end_index):\n", - " indices = range(i-history_size, i, step)\n", - " data.append(dataset[indices])\n", - "\n", - " if single_step:\n", - " labels.append(target[i+target_size])\n", - " else:\n", - " labels.append(target[i:i+target_size])\n", - "\n", - " return np.array(data), np.array(labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HWVGYwbN2ITI" - }, - "source": [ - "In this tutorial, the network is shown data from the last five (5) days, i.e. 720 observations that are sampled every hour. The sampling is done every one hour since a drastic change is not expected within 60 minutes. Thus, 120 observation represent history of the last five days. For the single step prediction model, the label for a datapoint is the temperature 12 hours into the future. In order to create a label for this, the temperature after 72(12*6) observations is used." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HlhVGzPhmMYI" - }, - "outputs": [], - "source": [ - "past_history = 720\n", - "future_target = 72\n", - "STEP = 6\n", - "\n", - "x_train_single, y_train_single = multivariate_data(dataset, dataset[:, 1], 0,\n", - " TRAIN_SPLIT, past_history,\n", - " future_target, STEP,\n", - " single_step=True)\n", - "x_val_single, y_val_single = multivariate_data(dataset, dataset[:, 1],\n", - " TRAIN_SPLIT, None, past_history,\n", - " future_target, STEP,\n", - " single_step=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CamMObrwPhnp" - }, - "source": [ - "Let's look at a single data-point.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_tVKm-ZIPls0" - }, - "outputs": [], - "source": [ - "print ('Single window of past history : {}'.format(x_train_single[0].shape))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eCWG4xgQ3O6E" - }, - "outputs": [], - "source": [ - "train_data_single = tf.data.Dataset.from_tensor_slices((x_train_single, y_train_single))\n", - "train_data_single = train_data_single.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()\n", - "\n", - "val_data_single = tf.data.Dataset.from_tensor_slices((x_val_single, y_val_single))\n", - "val_data_single = val_data_single.batch(BATCH_SIZE).repeat()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0aWec9_nlxBl" - }, - "outputs": [], - "source": [ - "single_step_model = tf.keras.models.Sequential()\n", - "single_step_model.add(tf.keras.layers.LSTM(32,\n", - " input_shape=x_train_single.shape[-2:]))\n", - "single_step_model.add(tf.keras.layers.Dense(1))\n", - "\n", - "single_step_model.compile(optimizer=tf.keras.optimizers.RMSprop(), loss='mae')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oYhUfWjwOPFN" - }, - "source": [ - "Let's check out a sample prediction." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yY7FodHVOPsH" - }, - "outputs": [], - "source": [ - "for x, y in val_data_single.take(1):\n", - " print(single_step_model.predict(x).shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U0jnt2l2mwkl" - }, - "outputs": [], - "source": [ - "single_step_history = single_step_model.fit(train_data_single, epochs=EPOCHS,\n", - " steps_per_epoch=EVALUATION_INTERVAL,\n", - " validation_data=val_data_single,\n", - " validation_steps=50)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-ZAdeAnP5c72" - }, - "outputs": [], - "source": [ - "def plot_train_history(history, title):\n", - " loss = history.history['loss']\n", - " val_loss = history.history['val_loss']\n", - "\n", - " epochs = range(len(loss))\n", - "\n", - " plt.figure()\n", - "\n", - " plt.plot(epochs, loss, 'b', label='Training loss')\n", - " plt.plot(epochs, val_loss, 'r', label='Validation loss')\n", - " plt.title(title)\n", - " plt.legend()\n", - "\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l8lBKA-z5yYV" - }, - "outputs": [], - "source": [ - "plot_train_history(single_step_history,\n", - " 'Single Step Training and validation loss')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DfjrGAlEUp7i" - }, - "source": [ - "#### Predict a single step future\n", - "Now that the model is trained, let's make a few sample predictions. The model is given the history of three features over the past five days sampled every hour (120 data-points), since the goal is to predict the temperature, the plot only displays the past temperature. The prediction is made one day into the future (hence the gap between the history and prediction). " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "h1qmPLLVUpuN" - }, - "outputs": [], - "source": [ - "for x, y in val_data_single.take(3):\n", - " plot = show_plot([x[0][:, 1].numpy(), y[0].numpy(),\n", - " single_step_model.predict(x)[0]], 12,\n", - " 'Single Step Prediction')\n", - " plot.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2GnE087bJYSu" - }, - "source": [ - "### Multi-Step model\n", - "In a multi-step prediction model, given a past history, the model needs to learn to predict a range of future values. Thus, unlike a single step model, where only a single future point is predicted, a multi-step model predict a sequence of the future.\n", - "\n", - "For the multi-step model, the training data again consists of recordings over the past five days sampled every hour. However, here, the model needs to learn to predict the temperature for the next 12 hours. Since an obversation is taken every 10 minutes, the output is 72 predictions. For this task, the dataset needs to be prepared accordingly, thus the first step is just to create it again, but with a different target window." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kZCk9fqyJZqX" - }, - "outputs": [], - "source": [ - "future_target = 72\n", - "x_train_multi, y_train_multi = multivariate_data(dataset, dataset[:, 1], 0,\n", - " TRAIN_SPLIT, past_history,\n", - " future_target, STEP)\n", - "x_val_multi, y_val_multi = multivariate_data(dataset, dataset[:, 1],\n", - " TRAIN_SPLIT, None, past_history,\n", - " future_target, STEP)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LImXPwAGRtWy" - }, - "source": [ - "Let's check out a sample data-point." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SpWDcBkQRwS-" - }, - "outputs": [], - "source": [ - "print ('Single window of past history : {}'.format(x_train_multi[0].shape))\n", - "print ('\\n Target temperature to predict : {}'.format(y_train_multi[0].shape))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cjR4PJArMOpA" - }, - "outputs": [], - "source": [ - "train_data_multi = tf.data.Dataset.from_tensor_slices((x_train_multi, y_train_multi))\n", - "train_data_multi = train_data_multi.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()\n", - "\n", - "val_data_multi = tf.data.Dataset.from_tensor_slices((x_val_multi, y_val_multi))\n", - "val_data_multi = val_data_multi.batch(BATCH_SIZE).repeat()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IZcg8FWpSG8K" - }, - "source": [ - "Plotting a sample data-point." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ksXKVbwBV7D3" - }, - "outputs": [], - "source": [ - "def multi_step_plot(history, true_future, prediction):\n", - " plt.figure(figsize=(12, 6))\n", - " num_in = create_time_steps(len(history))\n", - " num_out = len(true_future)\n", - "\n", - " plt.plot(num_in, np.array(history[:, 1]), label='History')\n", - " plt.plot(np.arange(num_out)/STEP, np.array(true_future), 'bo',\n", - " label='True Future')\n", - " if prediction.any():\n", - " plt.plot(np.arange(num_out)/STEP, np.array(prediction), 'ro',\n", - " label='Predicted Future')\n", - " plt.legend(loc='upper left')\n", - " plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LCQKetflZRMF" - }, - "source": [ - "In this plot and subsequent similar plots, the history and the future data are sampled every hour." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "R6G8bacQR4w2" - }, - "outputs": [], - "source": [ - "for x, y in train_data_multi.take(1):\n", - " multi_step_plot(x[0], y[0], np.array([0]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XOjz8DzZ4HFS" - }, - "source": [ - "Since the task here is a bit more complicated than the previous task, the model now consists of two LSTM layers. Finally, since 72 predictions are made, the dense layer outputs 72 predictions." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "byAl0NKSNBP6" - }, - "outputs": [], - "source": [ - "multi_step_model = tf.keras.models.Sequential()\n", - "multi_step_model.add(tf.keras.layers.LSTM(32,\n", - " return_sequences=True,\n", - " input_shape=x_train_multi.shape[-2:]))\n", - "multi_step_model.add(tf.keras.layers.LSTM(16, activation='relu'))\n", - "multi_step_model.add(tf.keras.layers.Dense(72))\n", - "\n", - "multi_step_model.compile(optimizer=tf.keras.optimizers.RMSprop(clipvalue=1.0), loss='mae')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UvB7zBqVSMyl" - }, - "source": [ - "Let's see how the model predicts before it trains." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "13_ZWvB9SRlZ" - }, - "outputs": [], - "source": [ - "for x, y in val_data_multi.take(1):\n", - " print (multi_step_model.predict(x).shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7uwOhXo3Oems" - }, - "outputs": [], - "source": [ - "multi_step_history = multi_step_model.fit(train_data_multi, epochs=EPOCHS,\n", - " steps_per_epoch=EVALUATION_INTERVAL,\n", - " validation_data=val_data_multi,\n", - " validation_steps=50)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UKfQoBjQ5l7U" - }, - "outputs": [], - "source": [ - "plot_train_history(multi_step_history, 'Multi-Step Training and validation loss')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oDg94-yq4pas" - }, - "source": [ - "#### Predict a multi-step future\n", - "Let's now have a look at how well your network has learnt to predict the future." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dt22wq6fyIBU" - }, - "outputs": [], - "source": [ - "for x, y in val_data_multi.take(3):\n", - " multi_step_plot(x[0], y[0], multi_step_model.predict(x)[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pOzaIRYBhqwg" - }, - "source": [ - "## Next steps\n", - "This tutorial was a quick introduction to time series forecasting using an RNN. You may now try to predict the stock market and become a billionaire.\n", - "\n", - "In addition, you may also write a generator to yield data (instead of the uni/multivariate_data function), which would be more memory efficient. You may also check out this [time series windowing](https://www.tensorflow.org/guide/data#time_series_windowing) guide and use it in this tutorial.\n", - "\n", - "For further understanding, you may read Chapter 15 of [Hands-on Machine Learning with Scikit-Learn, Keras, and TensorFlow](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/), 2nd Edition and Chapter 6 of [Deep Learning with Python](https://www.manning.com/books/deep-learning-with-python)." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "time_series.ipynb", - "private_outputs": true, - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "2Pmxv2ioyCRw" + }, + "source": [ + "##### Copyright 2019 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "b-2ShX25yNWf" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pa49bUnKyRgF" + }, + "source": [ + "# Time series forecasting" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "11Ilg92myRcw" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GU8C5qm_4vZb" + }, + "source": [ + "This tutorial is an introduction to time series forecasting using TensorFlow. It builds a few different styles of models including Convolutional and Recurrent Neural Networks (CNNs and RNNs).\n", + "\n", + "This is covered in two main parts, with subsections:\n", + "\n", + "* Forecast for a single time step:\n", + " * A single feature.\n", + " * All features.\n", + "* Forecast multiple steps:\n", + " * Single-shot: Make the predictions all at once.\n", + " * Autoregressive: Make one prediction at a time and feed the output back to the model." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XVhK72Pu1cJL" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7rZnJaGTWQw0" + }, + "outputs": [], + "source": [ + "import os\n", + "import datetime\n", + "\n", + "import IPython\n", + "import IPython.display\n", + "import matplotlib as mpl\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "import tensorflow as tf\n", + "\n", + "mpl.rcParams['figure.figsize'] = (8, 6)\n", + "mpl.rcParams['axes.grid'] = False" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TokBlnUhWFw9" + }, + "source": [ + "## The weather dataset\n", + "\n", + "This tutorial uses a weather time series dataset recorded by the Max Planck Institute for Biogeochemistry.\n", + "\n", + "This dataset contains 14 different features such as air temperature, atmospheric pressure, and humidity. These were collected every 10 minutes, beginning in 2003. For efficiency, you will use only the data collected between 2009 and 2016. This section of the dataset was prepared by François Chollet for his book Deep Learning with Python." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xyv_i85IWInT" + }, + "outputs": [], + "source": [ + "zip_path = tf.keras.utils.get_file(\n", + " origin='https://storage.googleapis.com/tensorflow/tf-keras-datasets/jena_climate_2009_2016.csv.zip',\n", + " fname='jena_climate_2009_2016.csv.zip',\n", + " extract=True)\n", + "csv_path, _ = os.path.splitext(zip_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R81Wx8WP4c3G" + }, + "source": [ + "This tutorial will just deal with **hourly predictions**, so start by sub-sampling the data from 10-minute intervals to one-hour intervals:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TX6uGeeeWIkG" + }, + "outputs": [], + "source": [ + "df = pd.read_csv(csv_path)\n", + "# Slice [start:stop:step], starting from index 5 take every 6th record.\n", + "df = df[5::6]\n", + "\n", + "date_time = pd.to_datetime(df.pop('Date Time'), format='%d.%m.%Y %H:%M:%S')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VdbOWXiTWM2T" + }, + "source": [ + "Let's take a glance at the data. Here are the first few rows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ojHE-iCCWIhz" + }, + "outputs": [], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WRzj1inMfgcO" + }, + "source": [ + "Here is the evolution of a few features over time:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Vg5XIc5tfNlG" + }, + "outputs": [], + "source": [ + "plot_cols = ['T (degC)', 'p (mbar)', 'rho (g/m**3)']\n", + "plot_features = df[plot_cols]\n", + "plot_features.index = date_time\n", + "_ = plot_features.plot(subplots=True)\n", + "\n", + "plot_features = df[plot_cols][:480]\n", + "plot_features.index = date_time[:480]\n", + "_ = plot_features.plot(subplots=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wXWLG0_WBhZS" + }, + "source": [ + "### Inspect and cleanup" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yhmZXJew6GlS" + }, + "source": [ + "Next, look at the statistics of the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "h510pgKVrrai" + }, + "outputs": [], + "source": [ + "df.describe().transpose()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TzOTnWOoWMGK" + }, + "source": [ + "#### Wind velocity" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i47LiW5DCVsP" + }, + "source": [ + "One thing that should stand out is the `min` value of the wind velocity (`wv (m/s)`) and the maximum value (`max. wv (m/s)`) columns. This `-9999` is likely erroneous.\n", + "\n", + "There's a separate wind direction column, so the velocity should be greater than zero (`>=0`). Replace it with zeros:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qFOq0_80vF4d" + }, + "outputs": [], + "source": [ + "wv = df['wv (m/s)']\n", + "bad_wv = wv == -9999.0\n", + "wv[bad_wv] = 0.0\n", + "\n", + "max_wv = df['max. wv (m/s)']\n", + "bad_max_wv = max_wv == -9999.0\n", + "max_wv[bad_max_wv] = 0.0\n", + "\n", + "# The above inplace edits are reflected in the DataFrame.\n", + "df['wv (m/s)'].min()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vtmu2IBPgPG8" + }, + "source": [ + "### Feature engineering\n", + "\n", + "Before diving in to build a model, it's important to understand your data and be sure that you're passing the model appropriately formatted data." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FYyEaqiD6j4s" + }, + "source": [ + "#### Wind\n", + "The last column of the data, `wd (deg)`—gives the wind direction in units of degrees. Angles do not make good model inputs: 360° and 0° should be close to each other and wrap around smoothly. Direction shouldn't matter if the wind is not blowing.\n", + "\n", + "Right now the distribution of wind data looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YO7JGTcWQG2z" + }, + "outputs": [], + "source": [ + "plt.hist2d(df['wd (deg)'], df['wv (m/s)'], bins=(50, 50), vmax=400)\n", + "plt.colorbar()\n", + "plt.xlabel('Wind Direction [deg]')\n", + "plt.ylabel('Wind Velocity [m/s]')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yWnf5dwMU1_g" + }, + "source": [ + "But this will be easier for the model to interpret if you convert the wind direction and velocity columns to a wind **vector**:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6GmSTHXw6lI1" + }, + "outputs": [], + "source": [ + "wv = df.pop('wv (m/s)')\n", + "max_wv = df.pop('max. wv (m/s)')\n", + "\n", + "# Convert to radians.\n", + "wd_rad = df.pop('wd (deg)')*np.pi / 180\n", + "\n", + "# Calculate the wind x and y components.\n", + "df['Wx'] = wv*np.cos(wd_rad)\n", + "df['Wy'] = wv*np.sin(wd_rad)\n", + "\n", + "# Calculate the max wind x and y components.\n", + "df['max Wx'] = max_wv*np.cos(wd_rad)\n", + "df['max Wy'] = max_wv*np.sin(wd_rad)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7iI0zDoxWDyB" + }, + "source": [ + "The distribution of wind vectors is much simpler for the model to correctly interpret:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bMgCG5o2SYKD" + }, + "outputs": [], + "source": [ + "plt.hist2d(df['Wx'], df['Wy'], bins=(50, 50), vmax=400)\n", + "plt.colorbar()\n", + "plt.xlabel('Wind X [m/s]')\n", + "plt.ylabel('Wind Y [m/s]')\n", + "ax = plt.gca()\n", + "ax.axis('tight')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_8im1ttOWlRB" + }, + "source": [ + "#### Time" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7YE21HKK40zQ" + }, + "source": [ + "Similarly, the `Date Time` column is very useful, but not in this string form. Start by converting it to seconds:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LIFf-VjMfnh3" + }, + "outputs": [], + "source": [ + "timestamp_s = date_time.map(pd.Timestamp.timestamp)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EC_pnM1D5Sgc" + }, + "source": [ + "Similar to the wind direction, the time in seconds is not a useful model input. Being weather data, it has clear daily and yearly periodicity. There are many ways you could deal with periodicity.\n", + "\n", + "You can get usable signals by using sine and cosine transforms to clear \"Time of day\" and \"Time of year\" signals:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MBfX6CDwax73" + }, + "outputs": [], + "source": [ + "day = 24*60*60\n", + "year = (365.2425)*day\n", + "\n", + "df['Day sin'] = np.sin(timestamp_s * (2 * np.pi / day))\n", + "df['Day cos'] = np.cos(timestamp_s * (2 * np.pi / day))\n", + "df['Year sin'] = np.sin(timestamp_s * (2 * np.pi / year))\n", + "df['Year cos'] = np.cos(timestamp_s * (2 * np.pi / year))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mXBbTJZfuuTC" + }, + "outputs": [], + "source": [ + "plt.plot(np.array(df['Day sin'])[:25])\n", + "plt.plot(np.array(df['Day cos'])[:25])\n", + "plt.xlabel('Time [h]')\n", + "plt.title('Time of day signal')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HiurzTGQgf_D" + }, + "source": [ + "This gives the model access to the most important frequency features. In this case you knew ahead of time which frequencies were important.\n", + "\n", + "If you don't have that information, you can determine which frequencies are important by extracting features with Fast Fourier Transform. To check the assumptions, here is the `tf.signal.rfft` of the temperature over time. Note the obvious peaks at frequencies near `1/year` and `1/day`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EN4U1fcMiTYs" + }, + "outputs": [], + "source": [ + "fft = tf.signal.rfft(df['T (degC)'])\n", + "f_per_dataset = np.arange(0, len(fft))\n", + "\n", + "n_samples_h = len(df['T (degC)'])\n", + "hours_per_year = 24*365.2524\n", + "years_per_dataset = n_samples_h/(hours_per_year)\n", + "\n", + "f_per_year = f_per_dataset/years_per_dataset\n", + "plt.step(f_per_year, np.abs(fft))\n", + "plt.xscale('log')\n", + "plt.ylim(0, 400000)\n", + "plt.xlim([0.1, max(plt.xlim())])\n", + "plt.xticks([1, 365.2524], labels=['1/Year', '1/day'])\n", + "_ = plt.xlabel('Frequency (log scale)')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2rbL8bSGDHy3" + }, + "source": [ + "### Split the data" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qoFJZmXBaxCc" + }, + "source": [ + "You'll use a `(70%, 20%, 10%)` split for the training, validation, and test sets. Note the data is **not** being randomly shuffled before splitting. This is for two reasons:\n", + "\n", + "1. It ensures that chopping the data into windows of consecutive samples is still possible.\n", + "2. It ensures that the validation/test results are more realistic, being evaluated on the data collected after the model was trained." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ia-MPAHxbInX" + }, + "outputs": [], + "source": [ + "column_indices = {name: i for i, name in enumerate(df.columns)}\n", + "\n", + "n = len(df)\n", + "train_df = df[0:int(n*0.7)]\n", + "val_df = df[int(n*0.7):int(n*0.9)]\n", + "test_df = df[int(n*0.9):]\n", + "\n", + "num_features = df.shape[1]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-eFckdUUHWmT" + }, + "source": [ + "### Normalize the data\n", + "\n", + "It is important to scale features before training a neural network. Normalization is a common way of doing this scaling: subtract the mean and divide by the standard deviation of each feature." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mxbIic5TMlxx" + }, + "source": [ + "The mean and standard deviation should only be computed using the training data so that the models have no access to the values in the validation and test sets.\n", + "\n", + "It's also arguable that the model shouldn't have access to future values in the training set when training, and that this normalization should be done using moving averages. That's not the focus of this tutorial, and the validation and test sets ensure that you get (somewhat) honest metrics. So, in the interest of simplicity this tutorial uses a simple average." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Eji6njXvHusN" + }, + "outputs": [], + "source": [ + "train_mean = train_df.mean()\n", + "train_std = train_df.std()\n", + "\n", + "train_df = (train_df - train_mean) / train_std\n", + "val_df = (val_df - train_mean) / train_std\n", + "test_df = (test_df - train_mean) / train_std" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G6ufs8kk9JQw" + }, + "source": [ + "Now, peek at the distribution of the features. Some features do have long tails, but there are no obvious errors like the `-9999` wind velocity value." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "T0UYEnkwm8Fe" + }, + "outputs": [], + "source": [ + "df_std = (df - train_mean) / train_std\n", + "df_std = df_std.melt(var_name='Column', value_name='Normalized')\n", + "plt.figure(figsize=(12, 6))\n", + "ax = sns.violinplot(x='Column', y='Normalized', data=df_std)\n", + "_ = ax.set_xticklabels(df.keys(), rotation=90)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZBBmdxZ2HgfJ" + }, + "source": [ + "## Data windowing\n", + "\n", + "The models in this tutorial will make a set of predictions based on a window of consecutive samples from the data.\n", + "\n", + "The main features of the input windows are:\n", + "\n", + "- The width (number of time steps) of the input and label windows.\n", + "- The time offset between them.\n", + "- Which features are used as inputs, labels, or both.\n", + "\n", + "This tutorial builds a variety of models (including Linear, DNN, CNN and RNN models), and uses them for both:\n", + "\n", + "- *Single-output*, and *multi-output* predictions.\n", + "- *Single-time-step* and *multi-time-step* predictions.\n", + "\n", + "This section focuses on implementing the data windowing so that it can be reused for all of those models.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YAhGUVx1jtOy" + }, + "source": [ + "Depending on the task and type of model you may want to generate a variety of data windows. Here are some examples:\n", + "\n", + "1. For example, to make a single prediction 24 hours into the future, given 24 hours of history, you might define a window like this:\n", + "\n", + " ![One prediction 24 hours into the future.](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/raw_window_24h.png?raw=1)\n", + "\n", + "2. A model that makes a prediction one hour into the future, given six hours of history, would need a window like this:\n", + "\n", + " ![One prediction one hour into the future.](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/raw_window_1h.png?raw=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sa2BbfNZt8wy" + }, + "source": [ + "The rest of this section defines a `WindowGenerator` class. This class can:\n", + "\n", + "1. Handle the indexes and offsets as shown in the diagrams above.\n", + "1. Split windows of features into `(features, labels)` pairs.\n", + "2. Plot the content of the resulting windows.\n", + "3. Efficiently generate batches of these windows from the training, evaluation, and test data, using `tf.data.Dataset`s." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rfx3jGjyziUF" + }, + "source": [ + "### 1. Indexes and offsets\n", + "\n", + "Start by creating the `WindowGenerator` class. The `__init__` method includes all the necessary logic for the input and label indices.\n", + "\n", + "It also takes the training, evaluation, and test DataFrames as input. These will be converted to `tf.data.Dataset`s of windows later." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Kem30j8QHxyW" + }, + "outputs": [], + "source": [ + "class WindowGenerator():\n", + " def __init__(self, input_width, label_width, shift,\n", + " train_df=train_df, val_df=val_df, test_df=test_df,\n", + " label_columns=None):\n", + " # Store the raw data.\n", + " self.train_df = train_df\n", + " self.val_df = val_df\n", + " self.test_df = test_df\n", + "\n", + " # Work out the label column indices.\n", + " self.label_columns = label_columns\n", + " if label_columns is not None:\n", + " self.label_columns_indices = {name: i for i, name in\n", + " enumerate(label_columns)}\n", + " self.column_indices = {name: i for i, name in\n", + " enumerate(train_df.columns)}\n", + "\n", + " # Work out the window parameters.\n", + " self.input_width = input_width\n", + " self.label_width = label_width\n", + " self.shift = shift\n", + "\n", + " self.total_window_size = input_width + shift\n", + "\n", + " self.input_slice = slice(0, input_width)\n", + " self.input_indices = np.arange(self.total_window_size)[self.input_slice]\n", + "\n", + " self.label_start = self.total_window_size - self.label_width\n", + " self.labels_slice = slice(self.label_start, None)\n", + " self.label_indices = np.arange(self.total_window_size)[self.labels_slice]\n", + "\n", + " def __repr__(self):\n", + " return '\\n'.join([\n", + " f'Total window size: {self.total_window_size}',\n", + " f'Input indices: {self.input_indices}',\n", + " f'Label indices: {self.label_indices}',\n", + " f'Label column name(s): {self.label_columns}'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yVJgblsYzL1g" + }, + "source": [ + "Here is code to create the 2 windows shown in the diagrams at the start of this section:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IsM5kRkz0UwK" + }, + "outputs": [], + "source": [ + "w1 = WindowGenerator(input_width=24, label_width=1, shift=24,\n", + " label_columns=['T (degC)'])\n", + "w1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "viwKsYeAKFUn" + }, + "outputs": [], + "source": [ + "w2 = WindowGenerator(input_width=6, label_width=1, shift=1,\n", + " label_columns=['T (degC)'])\n", + "w2" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kJaUyTWQJd-L" + }, + "source": [ + "### 2. Split\n", + "\n", + "Given a list of consecutive inputs, the `split_window` method will convert them to a window of inputs and a window of labels.\n", + "\n", + "The example `w2` you define earlier will be split like this:\n", + "\n", + "![The initial window is all consecutive samples, this splits it into an (inputs, labels) pairs](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/split_window.png?raw=1)\n", + "\n", + "This diagram doesn't show the `features` axis of the data, but this `split_window` function also handles the `label_columns` so it can be used for both the single output and multi-output examples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "W4KbxfzqkXPW" + }, + "outputs": [], + "source": [ + "def split_window(self, features):\n", + " inputs = features[:, self.input_slice, :]\n", + " labels = features[:, self.labels_slice, :]\n", + " if self.label_columns is not None:\n", + " labels = tf.stack(\n", + " [labels[:, :, self.column_indices[name]] for name in self.label_columns],\n", + " axis=-1)\n", + "\n", + " # Slicing doesn't preserve static shape information, so set the shapes\n", + " # manually. This way the `tf.data.Datasets` are easier to inspect.\n", + " inputs.set_shape([None, self.input_width, None])\n", + " labels.set_shape([None, self.label_width, None])\n", + "\n", + " return inputs, labels\n", + "\n", + "WindowGenerator.split_window = split_window" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G6U6VtVuM15s" + }, + "source": [ + "Try it out:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YeCWbq6KLmL7" + }, + "outputs": [], + "source": [ + "# Stack three slices, the length of the total window.\n", + "example_window = tf.stack([np.array(train_df[:w2.total_window_size]),\n", + " np.array(train_df[100:100+w2.total_window_size]),\n", + " np.array(train_df[200:200+w2.total_window_size])])\n", + "\n", + "example_inputs, example_labels = w2.split_window(example_window)\n", + "\n", + "print('All shapes are: (batch, time, features)')\n", + "print(f'Window shape: {example_window.shape}')\n", + "print(f'Inputs shape: {example_inputs.shape}')\n", + "print(f'Labels shape: {example_labels.shape}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xtMk1ffk2Mmd" + }, + "source": [ + "Typically, data in TensorFlow is packed into arrays where the outermost index is across examples (the \"batch\" dimension). The middle indices are the \"time\" or \"space\" (width, height) dimension(s). The innermost indices are the features.\n", + "\n", + "The code above took a batch of three 7-time step windows with 19 features at each time step. It splits them into a batch of 6-time step 19-feature inputs, and a 1-time step 1-feature label. The label only has one feature because the `WindowGenerator` was initialized with `label_columns=['T (degC)']`. Initially, this tutorial will build models that predict single output labels." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tFZukGXrJoGo" + }, + "source": [ + "### 3. Plot\n", + "\n", + "Here is a plot method that allows a simple visualization of the split window:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fmgd1qkYUWT7" + }, + "outputs": [], + "source": [ + "w2.example = example_inputs, example_labels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jIrYccI-Hm3B" + }, + "outputs": [], + "source": [ + "def plot(self, model=None, plot_col='T (degC)', max_subplots=3):\n", + " inputs, labels = self.example\n", + " plt.figure(figsize=(12, 8))\n", + " plot_col_index = self.column_indices[plot_col]\n", + " max_n = min(max_subplots, len(inputs))\n", + " for n in range(max_n):\n", + " plt.subplot(max_n, 1, n+1)\n", + " plt.ylabel(f'{plot_col} [normed]')\n", + " plt.plot(self.input_indices, inputs[n, :, plot_col_index],\n", + " label='Inputs', marker='.', zorder=-10)\n", + "\n", + " if self.label_columns:\n", + " label_col_index = self.label_columns_indices.get(plot_col, None)\n", + " else:\n", + " label_col_index = plot_col_index\n", + "\n", + " if label_col_index is None:\n", + " continue\n", + "\n", + " plt.scatter(self.label_indices, labels[n, :, label_col_index],\n", + " edgecolors='k', label='Labels', c='#2ca02c', s=64)\n", + " if model is not None:\n", + " predictions = model(inputs)\n", + " plt.scatter(self.label_indices, predictions[n, :, label_col_index],\n", + " marker='X', edgecolors='k', label='Predictions',\n", + " c='#ff7f0e', s=64)\n", + "\n", + " if n == 0:\n", + " plt.legend()\n", + "\n", + " plt.xlabel('Time [h]')\n", + "\n", + "WindowGenerator.plot = plot" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HXvctEuK68vX" + }, + "source": [ + "This plot aligns inputs, labels, and (later) predictions based on the time that the item refers to:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XjTqUnglOOni" + }, + "outputs": [], + "source": [ + "w2.plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UqiqcPOldPG6" + }, + "source": [ + "You can plot the other columns, but the example window `w2` configuration only has labels for the `T (degC)` column." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EBRe4wnlfCH8" + }, + "outputs": [], + "source": [ + "w2.plot(plot_col='p (mbar)')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xCvD-UaUzYMw" + }, + "source": [ + "### 4. Create `tf.data.Dataset`s" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kLO3SFR9Osdf" + }, + "source": [ + "Finally, this `make_dataset` method will take a time series DataFrame and convert it to a `tf.data.Dataset` of `(input_window, label_window)` pairs using the `tf.keras.utils.timeseries_dataset_from_array` function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "35qoSQeRVfJg" + }, + "outputs": [], + "source": [ + "def make_dataset(self, data):\n", + " data = np.array(data, dtype=np.float32)\n", + " ds = tf.keras.utils.timeseries_dataset_from_array(\n", + " data=data,\n", + " targets=None,\n", + " sequence_length=self.total_window_size,\n", + " sequence_stride=1,\n", + " shuffle=True,\n", + " batch_size=32,)\n", + "\n", + " ds = ds.map(self.split_window)\n", + "\n", + " return ds\n", + "\n", + "WindowGenerator.make_dataset = make_dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LvsxQwJaCift" + }, + "source": [ + "The `WindowGenerator` object holds training, validation, and test data.\n", + "\n", + "Add properties for accessing them as `tf.data.Dataset`s using the `make_dataset` method you defined earlier. Also, add a standard example batch for easy access and plotting:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2jZ2KkqGCfzu" + }, + "outputs": [], + "source": [ + "@property\n", + "def train(self):\n", + " return self.make_dataset(self.train_df)\n", + "\n", + "@property\n", + "def val(self):\n", + " return self.make_dataset(self.val_df)\n", + "\n", + "@property\n", + "def test(self):\n", + " return self.make_dataset(self.test_df)\n", + "\n", + "@property\n", + "def example(self):\n", + " \"\"\"Get and cache an example batch of `inputs, labels` for plotting.\"\"\"\n", + " result = getattr(self, '_example', None)\n", + " if result is None:\n", + " # No example batch was found, so get one from the `.train` dataset\n", + " result = next(iter(self.train))\n", + " # And cache it for next time\n", + " self._example = result\n", + " return result\n", + "\n", + "WindowGenerator.train = train\n", + "WindowGenerator.val = val\n", + "WindowGenerator.test = test\n", + "WindowGenerator.example = example" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fF_Vj6Iw3Y2w" + }, + "source": [ + "Now, the `WindowGenerator` object gives you access to the `tf.data.Dataset` objects, so you can easily iterate over the data.\n", + "\n", + "The `Dataset.element_spec` property tells you the structure, data types, and shapes of the dataset elements." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "daJ0-U383YVs" + }, + "outputs": [], + "source": [ + "# Each element is an (inputs, label) pair.\n", + "w2.train.element_spec" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XKTx3_Z7ua-n" + }, + "source": [ + "Iterating over a `Dataset` yields concrete batches:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6gtKXEgf4Iml" + }, + "outputs": [], + "source": [ + "for example_inputs, example_labels in w2.train.take(1):\n", + " print(f'Inputs shape (batch, time, features): {example_inputs.shape}')\n", + " print(f'Labels shape (batch, time, features): {example_labels.shape}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LyuGuJUgjUK3" + }, + "source": [ + "## Single step models\n", + "\n", + "The simplest model you can build on this sort of data is one that predicts a single feature's value—1 time step (one hour) into the future based only on the current conditions.\n", + "\n", + "So, start by building models to predict the `T (degC)` value one hour into the future.\n", + "\n", + "![Predict the next time step](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/narrow_window.png?raw=1)\n", + "\n", + "Configure a `WindowGenerator` object to produce these single-step `(input, label)` pairs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "G5QX1G1JTPCr" + }, + "outputs": [], + "source": [ + "single_step_window = WindowGenerator(\n", + " input_width=1, label_width=1, shift=1,\n", + " label_columns=['T (degC)'])\n", + "single_step_window" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RKTm8ajVGw4N" + }, + "source": [ + "The `window` object creates `tf.data.Dataset`s from the training, validation, and test sets, allowing you to easily iterate over batches of data.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Do4ILUaBF8oc" + }, + "outputs": [], + "source": [ + "for example_inputs, example_labels in single_step_window.train.take(1):\n", + " print(f'Inputs shape (batch, time, features): {example_inputs.shape}')\n", + " print(f'Labels shape (batch, time, features): {example_labels.shape}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D1bbPiR3VAm_" + }, + "source": [ + "### Baseline\n", + "\n", + "Before building a trainable model it would be good to have a performance baseline as a point for comparison with the later more complicated models.\n", + "\n", + "This first task is to predict temperature one hour into the future, given the current value of all features. The current values include the current temperature.\n", + "\n", + "So, start with a model that just returns the current temperature as the prediction, predicting \"No change\". This is a reasonable baseline since temperature changes slowly. Of course, this baseline will work less well if you make a prediction further in the future.\n", + "\n", + "![Send the input to the output](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/baseline.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9TybQaIsi3yg" + }, + "outputs": [], + "source": [ + "class Baseline(tf.keras.Model):\n", + " def __init__(self, label_index=None):\n", + " super().__init__()\n", + " self.label_index = label_index\n", + "\n", + " def call(self, inputs):\n", + " if self.label_index is None:\n", + " return inputs\n", + " result = inputs[:, :, self.label_index]\n", + " return result[:, :, tf.newaxis]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0vb3f948i8p8" + }, + "source": [ + "Instantiate and evaluate this model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IS3-QKc4sX0D" + }, + "outputs": [], + "source": [ + "baseline = Baseline(label_index=column_indices['T (degC)'])\n", + "\n", + "baseline.compile(loss=tf.keras.losses.MeanSquaredError(),\n", + " metrics=[tf.keras.metrics.MeanAbsoluteError()])\n", + "\n", + "val_performance = {}\n", + "performance = {}\n", + "val_performance['Baseline'] = baseline.evaluate(single_step_window.val, return_dict=True)\n", + "performance['Baseline'] = baseline.evaluate(single_step_window.test, verbose=0, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nhBxQcCSs7Ec" + }, + "source": [ + "That printed some performance metrics, but those don't give you a feeling for how well the model is doing.\n", + "\n", + "The `WindowGenerator` has a plot method, but the plots won't be very interesting with only a single sample.\n", + "\n", + "So, create a wider `WindowGenerator` that generates windows 24 hours of consecutive inputs and labels at a time. The new `wide_window` variable doesn't change the way the model operates. The model still makes predictions one hour into the future based on a single input time step. Here, the `time` axis acts like the `batch` axis: each prediction is made independently with no interaction between time steps:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "C8jNR5uuJ5Zp" + }, + "outputs": [], + "source": [ + "wide_window = WindowGenerator(\n", + " input_width=24, label_width=24, shift=1,\n", + " label_columns=['T (degC)'])\n", + "\n", + "wide_window" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZAnj7CFZkuYv" + }, + "source": [ + "This expanded window can be passed directly to the same `baseline` model without any code changes. This is possible because the inputs and labels have the same number of time steps, and the baseline just forwards the input to the output:\n", + "\n", + "![One prediction 1h into the future, ever hour.](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/last_window.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sGKdvdg087qs" + }, + "outputs": [], + "source": [ + "print('Input shape:', wide_window.example[0].shape)\n", + "print('Output shape:', baseline(wide_window.example[0]).shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SKqQHX1K0JW-" + }, + "source": [ + "By plotting the baseline model's predictions, notice that it is simply the labels shifted right by one hour:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jQyAPVLgWTOZ" + }, + "outputs": [], + "source": [ + "wide_window.plot(baseline)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e93TLUhfAVg2" + }, + "source": [ + "In the above plots of three examples the single step model is run over the course of 24 hours. This deserves some explanation:\n", + "\n", + "- The blue `Inputs` line shows the input temperature at each time step. The model receives all features, this plot only shows the temperature.\n", + "- The green `Labels` dots show the target prediction value. These dots are shown at the prediction time, not the input time. That is why the range of labels is shifted 1 step relative to the inputs.\n", + "- The orange `Predictions` crosses are the model's prediction's for each output time step. If the model were predicting perfectly the predictions would land directly on the `Labels`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "E4aOJScj52Yu" + }, + "source": [ + "### Linear model\n", + "\n", + "The simplest **trainable** model you can apply to this task is to insert linear transformation between the input and output. In this case the output from a time step only depends on that step:\n", + "\n", + "![A single step prediction](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/narrow_window.png?raw=1)\n", + "\n", + "A `tf.keras.layers.Dense` layer with no `activation` set is a linear model. The layer only transforms the last axis of the data from `(batch, time, inputs)` to `(batch, time, units)`; it is applied independently to every item across the `batch` and `time` axes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6341OXuQ5xA9" + }, + "outputs": [], + "source": [ + "linear = tf.keras.Sequential([\n", + " tf.keras.layers.Dense(units=1)\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KwaOM8RucUSn" + }, + "outputs": [], + "source": [ + "print('Input shape:', single_step_window.example[0].shape)\n", + "print('Output shape:', linear(single_step_window.example[0]).shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OMZTYIj3bYLg" + }, + "source": [ + "This tutorial trains many models, so package the training procedure into a function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CbCL6VIrk-Gt" + }, + "outputs": [], + "source": [ + "MAX_EPOCHS = 20\n", + "\n", + "def compile_and_fit(model, window, patience=2):\n", + " early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss',\n", + " patience=patience,\n", + " mode='min')\n", + "\n", + " model.compile(loss=tf.keras.losses.MeanSquaredError(),\n", + " optimizer=tf.keras.optimizers.Adam(),\n", + " metrics=[tf.keras.metrics.MeanAbsoluteError()])\n", + "\n", + " history = model.fit(window.train, epochs=MAX_EPOCHS,\n", + " validation_data=window.val,\n", + " callbacks=[early_stopping])\n", + " return history" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OobVjM-schwj" + }, + "source": [ + "Train the model and evaluate its performance:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9agbz2qB9bLS" + }, + "outputs": [], + "source": [ + "history = compile_and_fit(linear, single_step_window)\n", + "\n", + "val_performance['Linear'] = linear.evaluate(single_step_window.val, return_dict=True)\n", + "performance['Linear'] = linear.evaluate(single_step_window.test, verbose=0, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7U9XukYh8beN" + }, + "source": [ + "Like the `baseline` model, the linear model can be called on batches of wide windows. Used this way the model makes a set of independent predictions on consecutive time steps. The `time` axis acts like another `batch` axis. There are no interactions between the predictions at each time step.\n", + "\n", + "![A single step prediction](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/wide_window.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "K9UVM5Sw9KQN" + }, + "outputs": [], + "source": [ + "print('Input shape:', wide_window.example[0].shape)\n", + "print('Output shape:', linear(wide_window.example[0]).shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "X-CGj85oKaOG" + }, + "source": [ + "Here is the plot of its example predictions on the `wide_window`, note how in many cases the prediction is clearly better than just returning the input temperature, but in a few cases it's worse:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bCC8VVo-OvwV" + }, + "outputs": [], + "source": [ + "wide_window.plot(linear)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Is51vU8EMl6c" + }, + "source": [ + "One advantage to linear models is that they're relatively simple to interpret.\n", + "You can pull out the layer's weights and visualize the weight assigned to each input:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "d4uCTbsmK8VI" + }, + "outputs": [], + "source": [ + "plt.bar(x = range(len(train_df.columns)),\n", + " height=linear.layers[0].kernel[:,0].numpy())\n", + "axis = plt.gca()\n", + "axis.set_xticks(range(len(train_df.columns)))\n", + "_ = axis.set_xticklabels(train_df.columns, rotation=90)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ylng7215boIY" + }, + "source": [ + "Sometimes the model doesn't even place the most weight on the input `T (degC)`. This is one of the risks of random initialization." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W18e6da1cNbw" + }, + "source": [ + "### Dense\n", + "\n", + "Before applying models that actually operate on multiple time-steps, it's worth checking the performance of deeper, more powerful, single input step models.\n", + "\n", + "Here's a model similar to the `linear` model, except it stacks several a few `Dense` layers between the input and the output:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Z86WkYp7cNAD" + }, + "outputs": [], + "source": [ + "dense = tf.keras.Sequential([\n", + " tf.keras.layers.Dense(units=64, activation='relu'),\n", + " tf.keras.layers.Dense(units=64, activation='relu'),\n", + " tf.keras.layers.Dense(units=1)\n", + "])\n", + "\n", + "history = compile_and_fit(dense, single_step_window)\n", + "\n", + "val_performance['Dense'] = dense.evaluate(single_step_window.val, return_dict=True)\n", + "performance['Dense'] = dense.evaluate(single_step_window.test, verbose=0, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j5dv_whJdswH" + }, + "source": [ + "### Multi-step dense\n", + "\n", + "A single-time-step model has no context for the current values of its inputs. It can't see how the input features are changing over time. To address this issue the model needs access to multiple time steps when making predictions:\n", + "\n", + "![Three time steps are used for each prediction.](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/conv_window.png?raw=1)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zac-ti8agbJ7" + }, + "source": [ + "The `baseline`, `linear` and `dense` models handled each time step independently. Here the model will take multiple time steps as input to produce a single output.\n", + "\n", + "Create a `WindowGenerator` that will produce batches of three-hour inputs and one-hour labels:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gtN4BwZ37niR" + }, + "source": [ + "Note that the `Window`'s `shift` parameter is relative to the end of the two windows.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lBh0j5djUKY2" + }, + "outputs": [], + "source": [ + "CONV_WIDTH = 3\n", + "conv_window = WindowGenerator(\n", + " input_width=CONV_WIDTH,\n", + " label_width=1,\n", + " shift=1,\n", + " label_columns=['T (degC)'])\n", + "\n", + "conv_window" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dCQ5gvs68Xkd" + }, + "outputs": [], + "source": [ + "conv_window.plot()\n", + "plt.suptitle(\"Given 3 hours of inputs, predict 1 hour into the future.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "We0HdMxKeqB_" + }, + "source": [ + "You could train a `dense` model on a multiple-input-step window by adding a `tf.keras.layers.Flatten` as the first layer of the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "oNQnUOkOnC1G" + }, + "outputs": [], + "source": [ + "multi_step_dense = tf.keras.Sequential([\n", + " # Shape: (time, features) => (time*features)\n", + " tf.keras.layers.Flatten(),\n", + " tf.keras.layers.Dense(units=32, activation='relu'),\n", + " tf.keras.layers.Dense(units=32, activation='relu'),\n", + " tf.keras.layers.Dense(units=1),\n", + " # Add back the time dimension.\n", + " # Shape: (outputs) => (1, outputs)\n", + " tf.keras.layers.Reshape([1, -1]),\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cayD74luo4Vq" + }, + "outputs": [], + "source": [ + "print('Input shape:', conv_window.example[0].shape)\n", + "print('Output shape:', multi_step_dense(conv_window.example[0]).shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fu91yEbRo9-J" + }, + "outputs": [], + "source": [ + "history = compile_and_fit(multi_step_dense, conv_window)\n", + "\n", + "IPython.display.clear_output()\n", + "val_performance['Multi step dense'] = multi_step_dense.evaluate(conv_window.val, return_dict=True)\n", + "performance['Multi step dense'] = multi_step_dense.evaluate(conv_window.test, verbose=0, return_dict=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tnqdXYT6pkEh" + }, + "outputs": [], + "source": [ + "conv_window.plot(multi_step_dense)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gWfrsP8mq8lV" + }, + "source": [ + "The main down-side of this approach is that the resulting model can only be executed on input windows of exactly this shape." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "j-q6tz5Yq8Jk" + }, + "outputs": [], + "source": [ + "print('Input shape:', wide_window.example[0].shape)\n", + "try:\n", + " print('Output shape:', multi_step_dense(wide_window.example[0]).shape)\n", + "except Exception as e:\n", + " print(f'\\n{type(e).__name__}:{e}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bvvajm3ip_8V" + }, + "source": [ + "The convolutional models in the next section fix this problem." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CrpU6gwSJome" + }, + "source": [ + "### Convolution neural network\n", + "\n", + "A convolution layer (`tf.keras.layers.Conv1D`) also takes multiple time steps as input to each prediction." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cdLBwoaHmsWb" + }, + "source": [ + "Below is the **same** model as `multi_step_dense`, re-written with a convolution.\n", + "\n", + "Note the changes:\n", + "* The `tf.keras.layers.Flatten` and the first `tf.keras.layers.Dense` are replaced by a `tf.keras.layers.Conv1D`.\n", + "* The `tf.keras.layers.Reshape` is no longer necessary since the convolution keeps the time axis in its output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5azaMBj4ac9t" + }, + "outputs": [], + "source": [ + "conv_model = tf.keras.Sequential([\n", + " tf.keras.layers.Conv1D(filters=32,\n", + " kernel_size=(CONV_WIDTH,),\n", + " activation='relu'),\n", + " tf.keras.layers.Dense(units=32, activation='relu'),\n", + " tf.keras.layers.Dense(units=1),\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ftaH6B5ECRiK" + }, + "source": [ + "Run it on an example batch to check that the model produces outputs with the expected shape:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5YNgt1-e98lH" + }, + "outputs": [], + "source": [ + "print(\"Conv model on `conv_window`\")\n", + "print('Input shape:', conv_window.example[0].shape)\n", + "print('Output shape:', conv_model(conv_window.example[0]).shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5m4kC-jGCY3x" + }, + "source": [ + "Train and evaluate it on the ` conv_window` and it should give performance similar to the `multi_step_dense` model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QDVWdm4paUW7" + }, + "outputs": [], + "source": [ + "history = compile_and_fit(conv_model, conv_window)\n", + "\n", + "IPython.display.clear_output()\n", + "val_performance['Conv'] = conv_model.evaluate(conv_window.val, return_dict=True)\n", + "performance['Conv'] = conv_model.evaluate(conv_window.test, verbose=0, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sYRipDeXs0Kr" + }, + "source": [ + "The difference between this `conv_model` and the `multi_step_dense` model is that the `conv_model` can be run on inputs of any length. The convolutional layer is applied to a sliding window of inputs:\n", + "\n", + "![Executing a convolutional model on a sequence](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/wide_conv_window.png?raw=1)\n", + "\n", + "If you run it on wider input, it produces wider output:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hoqccxx9r5jF" + }, + "outputs": [], + "source": [ + "print(\"Wide window\")\n", + "print('Input shape:', wide_window.example[0].shape)\n", + "print('Labels shape:', wide_window.example[1].shape)\n", + "print('Output shape:', conv_model(wide_window.example[0]).shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "h_WGxtLIHhRF" + }, + "source": [ + "Note that the output is shorter than the input. To make training or plotting work, you need the labels, and prediction to have the same length. So build a `WindowGenerator` to produce wide windows with a few extra input time steps so the label and prediction lengths match:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_VPvJ_VwTc0f" + }, + "outputs": [], + "source": [ + "LABEL_WIDTH = 24\n", + "INPUT_WIDTH = LABEL_WIDTH + (CONV_WIDTH - 1)\n", + "wide_conv_window = WindowGenerator(\n", + " input_width=INPUT_WIDTH,\n", + " label_width=LABEL_WIDTH,\n", + " shift=1,\n", + " label_columns=['T (degC)'])\n", + "\n", + "wide_conv_window" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gtqlWYXeKXej" + }, + "outputs": [], + "source": [ + "print(\"Wide conv window\")\n", + "print('Input shape:', wide_conv_window.example[0].shape)\n", + "print('Labels shape:', wide_conv_window.example[1].shape)\n", + "print('Output shape:', conv_model(wide_conv_window.example[0]).shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yzxbbS56cSBV" + }, + "source": [ + "Now, you can plot the model's predictions on a wider window. Note the 3 input time steps before the first prediction. Every prediction here is based on the 3 preceding time steps:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gR7VyL45UuEe" + }, + "outputs": [], + "source": [ + "wide_conv_window.plot(conv_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H4crpOcoMlSe" + }, + "source": [ + "### Recurrent neural network\n", + "\n", + "A Recurrent Neural Network (RNN) is a type of neural network well-suited to time series data. RNNs process a time series step-by-step, maintaining an internal state from time-step to time-step.\n", + "\n", + "You can learn more in the [Text generation with an RNN](https://www.tensorflow.org/text/tutorials/text_generation) tutorial and the [Recurrent Neural Networks (RNN) with Keras](https://www.tensorflow.org/guide/keras/rnn) guide.\n", + "\n", + "In this tutorial, you will use an RNN layer called Long Short-Term Memory (`tf.keras.layers.LSTM`)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vfQbHSMb1ATa" + }, + "source": [ + "An important constructor argument for all Keras RNN layers, such as `tf.keras.layers.LSTM`, is the `return_sequences` argument. This setting can configure the layer in one of two ways:\n", + "\n", + "1. If `False`, the default, the layer only returns the output of the final time step, giving the model time to warm up its internal state before making a single prediction:\n", + "\n", + "![An LSTM warming up and making a single prediction](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/lstm_1_window.png?raw=1)\n", + "\n", + "2. If `True`, the layer returns an output for each input. This is useful for:\n", + " * Stacking RNN layers.\n", + " * Training a model on multiple time steps simultaneously.\n", + "\n", + "![An LSTM making a prediction after every time step](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/lstm_many_window.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DXKLCJy8nWNU" + }, + "outputs": [], + "source": [ + "lstm_model = tf.keras.models.Sequential([\n", + " # Shape [batch, time, features] => [batch, time, lstm_units]\n", + " tf.keras.layers.LSTM(32, return_sequences=True),\n", + " # Shape => [batch, time, features]\n", + " tf.keras.layers.Dense(units=1)\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F124B00KZcLC" + }, + "source": [ + "With `return_sequences=True`, the model can be trained on 24 hours of data at a time.\n", + "\n", + "Note: This will give a pessimistic view of the model's performance. On the first time step, the model has no access to previous steps and, therefore, can't do any better than the simple `linear` and `dense` models shown earlier." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eZEROCQVYV6q" + }, + "outputs": [], + "source": [ + "print('Input shape:', wide_window.example[0].shape)\n", + "print('Output shape:', lstm_model(wide_window.example[0]).shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uvdWRl1e9WJl" + }, + "outputs": [], + "source": [ + "history = compile_and_fit(lstm_model, wide_window)\n", + "\n", + "IPython.display.clear_output()\n", + "val_performance['LSTM'] = lstm_model.evaluate(wide_window.val, return_dict=True)\n", + "performance['LSTM'] = lstm_model.evaluate(wide_window.test, verbose=0, return_dict=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NwAOWCVgB26e" + }, + "outputs": [], + "source": [ + "wide_window.plot(lstm_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pYglOCKehi8F" + }, + "source": [ + "### Performance" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2pCk0_rwhi8H" + }, + "source": [ + "With this dataset typically each of the models does slightly better than the one before it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dMPev9Nzd4mD" + }, + "outputs": [], + "source": [ + "cm = lstm_model.metrics[1]\n", + "cm.metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6is3g113eIIa" + }, + "outputs": [], + "source": [ + "val_performance" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "JjEkt488hi8I" + }, + "outputs": [], + "source": [ + "x = np.arange(len(performance))\n", + "width = 0.3\n", + "metric_name = 'mean_absolute_error'\n", + "val_mae = [v[metric_name] for v in val_performance.values()]\n", + "test_mae = [v[metric_name] for v in performance.values()]\n", + "\n", + "plt.ylabel('mean_absolute_error [T (degC), normalized]')\n", + "plt.bar(x - 0.17, val_mae, width, label='Validation')\n", + "plt.bar(x + 0.17, test_mae, width, label='Test')\n", + "plt.xticks(ticks=x, labels=performance.keys(),\n", + " rotation=45)\n", + "_ = plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cBMCpsdphi8L" + }, + "outputs": [], + "source": [ + "for name, value in performance.items():\n", + " print(f'{name:12s}: {value[metric_name]:0.4f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "b5rUJ_2YMWzG" + }, + "source": [ + "### Multi-output models\n", + "\n", + "The models so far all predicted a single output feature, `T (degC)`, for a single time step.\n", + "\n", + "All of these models can be converted to predict multiple features just by changing the number of units in the output layer and adjusting the training windows to include all features in the `labels` (`example_labels`):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9Gk0Z91xjOwv" + }, + "outputs": [], + "source": [ + "single_step_window = WindowGenerator(\n", + " # `WindowGenerator` returns all features as labels if you\n", + " # don't set the `label_columns` argument.\n", + " input_width=1, label_width=1, shift=1)\n", + "\n", + "wide_window = WindowGenerator(\n", + " input_width=24, label_width=24, shift=1)\n", + "\n", + "for example_inputs, example_labels in wide_window.train.take(1):\n", + " print(f'Inputs shape (batch, time, features): {example_inputs.shape}')\n", + " print(f'Labels shape (batch, time, features): {example_labels.shape}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XmcjHfDskX1N" + }, + "source": [ + "Note above that the `features` axis of the labels now has the same depth as the inputs, instead of `1`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9k7S5IHNhSNF" + }, + "source": [ + "#### Baseline\n", + "\n", + "The same baseline model (`Baseline`) can be used here, but this time repeating all features instead of selecting a specific `label_index`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sqqB9W-pjr5i" + }, + "outputs": [], + "source": [ + "baseline = Baseline()\n", + "baseline.compile(loss=tf.keras.losses.MeanSquaredError(),\n", + " metrics=[tf.keras.metrics.MeanAbsoluteError()])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ltQdgaqQjQWu" + }, + "outputs": [], + "source": [ + "val_performance = {}\n", + "performance = {}\n", + "val_performance['Baseline'] = baseline.evaluate(wide_window.val, return_dict=True)\n", + "performance['Baseline'] = baseline.evaluate(wide_window.test, verbose=0, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dfbCrf5q3P6n" + }, + "source": [ + "#### Dense" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NdpzH1dYjdIN" + }, + "outputs": [], + "source": [ + "dense = tf.keras.Sequential([\n", + " tf.keras.layers.Dense(units=64, activation='relu'),\n", + " tf.keras.layers.Dense(units=64, activation='relu'),\n", + " tf.keras.layers.Dense(units=num_features)\n", + "])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6uHuU9Cd3PTo" + }, + "outputs": [], + "source": [ + "history = compile_and_fit(dense, single_step_window)\n", + "\n", + "IPython.display.clear_output()\n", + "val_performance['Dense'] = dense.evaluate(single_step_window.val, return_dict=True)\n", + "performance['Dense'] = dense.evaluate(single_step_window.test, verbose=0, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dsc9pur_mHsx" + }, + "source": [ + "#### RNN\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4QbGLMyomXaz" + }, + "outputs": [], + "source": [ + "%%time\n", + "wide_window = WindowGenerator(\n", + " input_width=24, label_width=24, shift=1)\n", + "\n", + "lstm_model = tf.keras.models.Sequential([\n", + " # Shape [batch, time, features] => [batch, time, lstm_units]\n", + " tf.keras.layers.LSTM(32, return_sequences=True),\n", + " # Shape => [batch, time, features]\n", + " tf.keras.layers.Dense(units=num_features)\n", + "])\n", + "\n", + "history = compile_and_fit(lstm_model, wide_window)\n", + "\n", + "IPython.display.clear_output()\n", + "val_performance['LSTM'] = lstm_model.evaluate( wide_window.val, return_dict=True)\n", + "performance['LSTM'] = lstm_model.evaluate( wide_window.test, verbose=0, return_dict=True)\n", + "\n", + "print()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UwhY2f_Nn0_K" + }, + "source": [ + "\n", + "\n", + "#### Advanced: Residual connections\n", + "\n", + "The `Baseline` model from earlier took advantage of the fact that the sequence doesn't change drastically from time step to time step. Every model trained in this tutorial so far was randomly initialized, and then had to learn that the output is a a small change from the previous time step.\n", + "\n", + "While you can get around this issue with careful initialization, it's simpler to build this into the model structure.\n", + "\n", + "It's common in time series analysis to build models that instead of predicting the next value, predict how the value will change in the next time step. Similarly, residual networks—or ResNets—in deep learning refer to architectures where each layer adds to the model's accumulating result.\n", + "\n", + "That is how you take advantage of the knowledge that the change should be small.\n", + "\n", + "![A model with a residual connection](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/residual.png?raw=1)\n", + "\n", + "Essentially, this initializes the model to match the `Baseline`. For this task it helps models converge faster, with slightly better performance." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yP58A_ORx0kM" + }, + "source": [ + "This approach can be used in conjunction with any model discussed in this tutorial.\n", + "\n", + "Here, it is being applied to the LSTM model, note the use of the `tf.initializers.zeros` to ensure that the initial predicted changes are small, and don't overpower the residual connection. There are no symmetry-breaking concerns for the gradients here, since the `zeros` are only used on the last layer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7YlfnDQC22TQ" + }, + "outputs": [], + "source": [ + "class ResidualWrapper(tf.keras.Model):\n", + " def __init__(self, model):\n", + " super().__init__()\n", + " self.model = model\n", + "\n", + " def call(self, inputs, *args, **kwargs):\n", + " delta = self.model(inputs, *args, **kwargs)\n", + "\n", + " # The prediction for each time step is the input\n", + " # from the previous time step plus the delta\n", + " # calculated by the model.\n", + " return inputs + delta" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NNeH02pspc9B" + }, + "outputs": [], + "source": [ + "%%time\n", + "residual_lstm = ResidualWrapper(\n", + " tf.keras.Sequential([\n", + " tf.keras.layers.LSTM(32, return_sequences=True),\n", + " tf.keras.layers.Dense(\n", + " num_features,\n", + " # The predicted deltas should start small.\n", + " # Therefore, initialize the output layer with zeros.\n", + " kernel_initializer=tf.initializers.zeros())\n", + "]))\n", + "\n", + "history = compile_and_fit(residual_lstm, wide_window)\n", + "\n", + "IPython.display.clear_output()\n", + "val_performance['Residual LSTM'] = residual_lstm.evaluate(wide_window.val, return_dict=True)\n", + "performance['Residual LSTM'] = residual_lstm.evaluate(wide_window.test, verbose=0, return_dict=True)\n", + "print()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "I42Er9Du6co1" + }, + "source": [ + "#### Performance" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "LZxR38P_6pUi" + }, + "source": [ + "Here is the overall performance for these multi-output models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6XgTK9tnr7rc" + }, + "outputs": [], + "source": [ + "x = np.arange(len(performance))\n", + "width = 0.3\n", + "\n", + "metric_name = 'mean_absolute_error'\n", + "val_mae = [v[metric_name] for v in val_performance.values()]\n", + "test_mae = [v[metric_name] for v in performance.values()]\n", + "\n", + "plt.bar(x - 0.17, val_mae, width, label='Validation')\n", + "plt.bar(x + 0.17, test_mae, width, label='Test')\n", + "plt.xticks(ticks=x, labels=performance.keys(),\n", + " rotation=45)\n", + "plt.ylabel('MAE (average over all outputs)')\n", + "_ = plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "URz3ajCc6kBj" + }, + "outputs": [], + "source": [ + "for name, value in performance.items():\n", + " print(f'{name:15s}: {value[metric_name]:0.4f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_Vt2MJhNxwPU" + }, + "source": [ + "The above performances are averaged across all model outputs." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eYokb7Om2YbK" + }, + "source": [ + "## Multi-step models\n", + "\n", + "Both the single-output and multiple-output models in the previous sections made **single time step predictions**, one hour into the future.\n", + "\n", + "This section looks at how to expand these models to make **multiple time step predictions**.\n", + "\n", + "In a multi-step prediction, the model needs to learn to predict a range of future values. Thus, unlike a single step model, where only a single future point is predicted, a multi-step model predicts a sequence of the future values.\n", + "\n", + "There are two rough approaches to this:\n", + "\n", + "1. Single shot predictions where the entire time series is predicted at once.\n", + "2. Autoregressive predictions where the model only makes single step predictions and its output is fed back as its input.\n", + "\n", + "In this section all the models will predict **all the features across all output time steps**.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WFsDAwVt4_rq" + }, + "source": [ + "For the multi-step model, the training data again consists of hourly samples. However, here, the models will learn to predict 24 hours into the future, given 24 hours of the past.\n", + "\n", + "Here is a `Window` object that generates these slices from the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1cFYtsz6XiGw" + }, + "outputs": [], + "source": [ + "OUT_STEPS = 24\n", + "multi_window = WindowGenerator(input_width=24,\n", + " label_width=OUT_STEPS,\n", + " shift=OUT_STEPS)\n", + "\n", + "multi_window.plot()\n", + "multi_window" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5lg8SInh9Jzd" + }, + "source": [ + "### Baselines" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "axwpoWYOApJL" + }, + "source": [ + "A simple baseline for this task is to repeat the last input time step for the required number of output time steps:\n", + "\n", + "![Repeat the last input, for each output step](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/multistep_last.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_5iaHSaJ9Rxv" + }, + "outputs": [], + "source": [ + "class MultiStepLastBaseline(tf.keras.Model):\n", + " def call(self, inputs):\n", + " return tf.tile(inputs[:, -1:, :], [1, OUT_STEPS, 1])\n", + "\n", + "last_baseline = MultiStepLastBaseline()\n", + "last_baseline.compile(loss=tf.keras.losses.MeanSquaredError(),\n", + " metrics=[tf.keras.metrics.MeanAbsoluteError()])\n", + "\n", + "multi_val_performance = {}\n", + "multi_performance = {}\n", + "\n", + "multi_val_performance['Last'] = last_baseline.evaluate(multi_window.val, return_dict=True)\n", + "multi_performance['Last'] = last_baseline.evaluate(multi_window.test, verbose=0, return_dict=True)\n", + "multi_window.plot(last_baseline)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AvHZ93ObAfMA" + }, + "source": [ + "Since this task is to predict 24 hours into the future, given 24 hours of the past, another simple approach is to repeat the previous day, assuming tomorrow will be similar:\n", + "\n", + "![Repeat the previous day](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/multistep_repeat.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "L8Y1uMhGwIRs" + }, + "outputs": [], + "source": [ + "class RepeatBaseline(tf.keras.Model):\n", + " def call(self, inputs):\n", + " return inputs\n", + "\n", + "repeat_baseline = RepeatBaseline()\n", + "repeat_baseline.compile(loss=tf.keras.losses.MeanSquaredError(),\n", + " metrics=[tf.keras.metrics.MeanAbsoluteError()])\n", + "\n", + "multi_val_performance['Repeat'] = repeat_baseline.evaluate(multi_window.val, return_dict=True)\n", + "multi_performance['Repeat'] = repeat_baseline.evaluate(multi_window.test, verbose=0, return_dict=True)\n", + "multi_window.plot(repeat_baseline)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tbndS-ct9C2Q" + }, + "source": [ + "### Single-shot models\n", + "\n", + "One high-level approach to this problem is to use a \"single-shot\" model, where the model makes the entire sequence prediction in a single step.\n", + "\n", + "This can be implemented efficiently as a `tf.keras.layers.Dense` with `OUT_STEPS*features` output units. The model just needs to reshape that output to the required `(OUTPUT_STEPS, features)`." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NCKS4m1VKrDQ" + }, + "source": [ + "#### Linear\n", + "\n", + "A simple linear model based on the last input time step does better than either baseline, but is underpowered. The model needs to predict `OUTPUT_STEPS` time steps, from a single input time step with a linear projection. It can only capture a low-dimensional slice of the behavior, likely based mainly on the time of day and time of year.\n", + "\n", + "![Predict all timesteps from the last time-step](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/multistep_dense.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kfRz_WVhIQcd" + }, + "outputs": [], + "source": [ + "multi_linear_model = tf.keras.Sequential([\n", + " # Take the last time-step.\n", + " # Shape [batch, time, features] => [batch, 1, features]\n", + " tf.keras.layers.Lambda(lambda x: x[:, -1:, :]),\n", + " # Shape => [batch, 1, out_steps*features]\n", + " tf.keras.layers.Dense(OUT_STEPS*num_features,\n", + " kernel_initializer=tf.initializers.zeros()),\n", + " # Shape => [batch, out_steps, features]\n", + " tf.keras.layers.Reshape([OUT_STEPS, num_features])\n", + "])\n", + "\n", + "history = compile_and_fit(multi_linear_model, multi_window)\n", + "\n", + "IPython.display.clear_output()\n", + "multi_val_performance['Linear'] = multi_linear_model.evaluate(multi_window.val, return_dict=True)\n", + "multi_performance['Linear'] = multi_linear_model.evaluate(multi_window.test, verbose=0, return_dict=True)\n", + "multi_window.plot(multi_linear_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zi2TMHk2IRrh" + }, + "source": [ + "#### Dense\n", + "\n", + "Adding a `tf.keras.layers.Dense` between the input and output gives the linear model more power, but is still only based on a single input time step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jezm-BKaGj91" + }, + "outputs": [], + "source": [ + "multi_dense_model = tf.keras.Sequential([\n", + " # Take the last time step.\n", + " # Shape [batch, time, features] => [batch, 1, features]\n", + " tf.keras.layers.Lambda(lambda x: x[:, -1:, :]),\n", + " # Shape => [batch, 1, dense_units]\n", + " tf.keras.layers.Dense(512, activation='relu'),\n", + " # Shape => [batch, out_steps*features]\n", + " tf.keras.layers.Dense(OUT_STEPS*num_features,\n", + " kernel_initializer=tf.initializers.zeros()),\n", + " # Shape => [batch, out_steps, features]\n", + " tf.keras.layers.Reshape([OUT_STEPS, num_features])\n", + "])\n", + "\n", + "history = compile_and_fit(multi_dense_model, multi_window)\n", + "\n", + "IPython.display.clear_output()\n", + "multi_val_performance['Dense'] = multi_dense_model.evaluate(multi_window.val, return_dict=True)\n", + "multi_performance['Dense'] = multi_dense_model.evaluate(multi_window.test, verbose=0, return_dict=True)\n", + "multi_window.plot(multi_dense_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "icsBAjCzMaMl" + }, + "source": [ + "#### CNN" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "34lCZrWYNBwd" + }, + "source": [ + "A convolutional model makes predictions based on a fixed-width history, which may lead to better performance than the dense model since it can see how things are changing over time:\n", + "\n", + "![A convolutional model sees how things change over time](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/multistep_conv.png?raw=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0xJoIP6PMWMI" + }, + "outputs": [], + "source": [ + "CONV_WIDTH = 3\n", + "multi_conv_model = tf.keras.Sequential([\n", + " # Shape [batch, time, features] => [batch, CONV_WIDTH, features]\n", + " tf.keras.layers.Lambda(lambda x: x[:, -CONV_WIDTH:, :]),\n", + " # Shape => [batch, 1, conv_units]\n", + " tf.keras.layers.Conv1D(256, activation='relu', kernel_size=(CONV_WIDTH)),\n", + " # Shape => [batch, 1, out_steps*features]\n", + " tf.keras.layers.Dense(OUT_STEPS*num_features,\n", + " kernel_initializer=tf.initializers.zeros()),\n", + " # Shape => [batch, out_steps, features]\n", + " tf.keras.layers.Reshape([OUT_STEPS, num_features])\n", + "])\n", + "\n", + "history = compile_and_fit(multi_conv_model, multi_window)\n", + "\n", + "IPython.display.clear_output()\n", + "\n", + "multi_val_performance['Conv'] = multi_conv_model.evaluate(multi_window.val, return_dict=True)\n", + "multi_performance['Conv'] = multi_conv_model.evaluate(multi_window.test, verbose=0, return_dict=True)\n", + "multi_window.plot(multi_conv_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "weBjeZAFJOP4" + }, + "source": [ + "#### RNN" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8022xOKxOO92" + }, + "source": [ + "A recurrent model can learn to use a long history of inputs, if it's relevant to the predictions the model is making. Here the model will accumulate internal state for 24 hours, before making a single prediction for the next 24 hours.\n", + "\n", + "In this single-shot format, the LSTM only needs to produce an output at the last time step, so set `return_sequences=False` in `tf.keras.layers.LSTM`.\n", + "\n", + "![The LSTM accumulates state over the input window, and makes a single prediction for the next 24 hours](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/multistep_lstm.png?raw=1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Bf1ks6RTzF64" + }, + "outputs": [], + "source": [ + "multi_lstm_model = tf.keras.Sequential([\n", + " # Shape [batch, time, features] => [batch, lstm_units].\n", + " # Adding more `lstm_units` just overfits more quickly.\n", + " tf.keras.layers.LSTM(32, return_sequences=False),\n", + " # Shape => [batch, out_steps*features].\n", + " tf.keras.layers.Dense(OUT_STEPS*num_features,\n", + " kernel_initializer=tf.initializers.zeros()),\n", + " # Shape => [batch, out_steps, features].\n", + " tf.keras.layers.Reshape([OUT_STEPS, num_features])\n", + "])\n", + "\n", + "history = compile_and_fit(multi_lstm_model, multi_window)\n", + "\n", + "IPython.display.clear_output()\n", + "\n", + "multi_val_performance['LSTM'] = multi_lstm_model.evaluate(multi_window.val, return_dict=True)\n", + "multi_performance['LSTM'] = multi_lstm_model.evaluate(multi_window.test, verbose=0, return_dict=True)\n", + "multi_window.plot(multi_lstm_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d5n-1cDW12Vo" + }, + "source": [ + "### Advanced: Autoregressive model\n", + "\n", + "The above models all predict the entire output sequence in a single step.\n", + "\n", + "In some cases it may be helpful for the model to decompose this prediction into individual time steps. Then, each model's output can be fed back into itself at each step and predictions can be made conditioned on the previous one, like in the classic Generating Sequences With Recurrent Neural Networks.\n", + "\n", + "One clear advantage to this style of model is that it can be set up to produce output with a varying length.\n", + "\n", + "You could take any of the single-step multi-output models trained in the first half of this tutorial and run in an autoregressive feedback loop, but here you'll focus on building a model that's been explicitly trained to do that.\n", + "\n", + "![Feedback a model's output to its input](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/images/multistep_autoregressive.png?raw=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PKRreBbULRXY" + }, + "source": [ + "#### RNN\n", + "\n", + "This tutorial only builds an autoregressive RNN model, but this pattern could be applied to any model that was designed to output a single time step.\n", + "\n", + "The model will have the same basic form as the single-step LSTM models from earlier: a `tf.keras.layers.LSTM` layer followed by a `tf.keras.layers.Dense` layer that converts the `LSTM` layer's outputs to model predictions.\n", + "\n", + "A `tf.keras.layers.LSTM` is a `tf.keras.layers.LSTMCell` wrapped in the higher level `tf.keras.layers.RNN` that manages the state and sequence results for you (Check out the [Recurrent Neural Networks (RNN) with Keras](https://www.tensorflow.org/guide/keras/rnn) guide for details).\n", + "\n", + "In this case, the model has to manually manage the inputs for each step, so it uses `tf.keras.layers.LSTMCell` directly for the lower level, single time step interface." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "s5tz3Nu0R5JG" + }, + "outputs": [], + "source": [ + "class FeedBack(tf.keras.Model):\n", + " def __init__(self, units, out_steps):\n", + " super().__init__()\n", + " self.out_steps = out_steps\n", + " self.units = units\n", + " self.lstm_cell = tf.keras.layers.LSTMCell(units)\n", + " # Also wrap the LSTMCell in an RNN to simplify the `warmup` method.\n", + " self.lstm_rnn = tf.keras.layers.RNN(self.lstm_cell, return_state=True)\n", + " self.dense = tf.keras.layers.Dense(num_features)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2OXVM9G1U7xR" + }, + "outputs": [], + "source": [ + "feedback_model = FeedBack(units=32, out_steps=OUT_STEPS)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ph5uFSfTUNho" + }, + "source": [ + "The first method this model needs is a `warmup` method to initialize its internal state based on the inputs. Once trained, this state will capture the relevant parts of the input history. This is equivalent to the single-step `LSTM` model from earlier:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vM2K_LLdRjDZ" + }, + "outputs": [], + "source": [ + "def warmup(self, inputs):\n", + " # inputs.shape => (batch, time, features)\n", + " # x.shape => (batch, lstm_units)\n", + " x, *state = self.lstm_rnn(inputs)\n", + "\n", + " # predictions.shape => (batch, features)\n", + " prediction = self.dense(x)\n", + " return prediction, state\n", + "\n", + "FeedBack.warmup = warmup" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6JkaSYaZ9eB7" + }, + "source": [ + "This method returns a single time-step prediction and the internal state of the `LSTM`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "w9Fz6NTKXXwU" + }, + "outputs": [], + "source": [ + "prediction, state = feedback_model.warmup(multi_window.example[0])\n", + "prediction.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S_ZdvPjdX3y3" + }, + "source": [ + "With the `RNN`'s state, and an initial prediction you can now continue iterating the model feeding the predictions at each step back as the input.\n", + "\n", + "The simplest approach for collecting the output predictions is to use a Python list and a `tf.stack` after the loop." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yotTad3nZXQU" + }, + "source": [ + "Note: Stacking a Python list like this only works with eager-execution, using `Model.compile(..., run_eagerly=True)` for training, or with a fixed length output. For a dynamic output length, you would need to use a `tf.TensorArray` instead of a Python list, and `tf.range` instead of the Python `range`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "g1GRDu3mZtr9" + }, + "outputs": [], + "source": [ + "def call(self, inputs, training=None):\n", + " # Use a TensorArray to capture dynamically unrolled outputs.\n", + " predictions = []\n", + " # Initialize the LSTM state.\n", + " prediction, state = self.warmup(inputs)\n", + "\n", + " # Insert the first prediction.\n", + " predictions.append(prediction)\n", + "\n", + " # Run the rest of the prediction steps.\n", + " for n in range(1, self.out_steps):\n", + " # Use the last prediction as input.\n", + " x = prediction\n", + " # Execute one lstm step.\n", + " x, state = self.lstm_cell(x, states=state,\n", + " training=training)\n", + " # Convert the lstm output to a prediction.\n", + " prediction = self.dense(x)\n", + " # Add the prediction to the output.\n", + " predictions.append(prediction)\n", + "\n", + " # predictions.shape => (time, batch, features)\n", + " predictions = tf.stack(predictions)\n", + " # predictions.shape => (batch, time, features)\n", + " predictions = tf.transpose(predictions, [1, 0, 2])\n", + " return predictions\n", + "\n", + "FeedBack.call = call" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ubop-YWp15XW" + }, + "source": [ + "Test run this model on the example inputs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Xja83zEYaM2D" + }, + "outputs": [], + "source": [ + "print('Output shape (batch, time, features): ', feedback_model(multi_window.example[0]).shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qMs0rYB8be9M" + }, + "source": [ + "Now, train the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VBRVG2hnNyrO" + }, + "outputs": [], + "source": [ + "history = compile_and_fit(feedback_model, multi_window)\n", + "\n", + "IPython.display.clear_output()\n", + "\n", + "multi_val_performance['AR LSTM'] = feedback_model.evaluate(multi_window.val, return_dict=True)\n", + "multi_performance['AR LSTM'] = feedback_model.evaluate(multi_window.test, verbose=0, return_dict=True)\n", + "multi_window.plot(feedback_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hGjcJsAQJUkI" + }, + "source": [ + "### Performance" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sODAwr2ndtDB" + }, + "source": [ + "There are clearly diminishing returns as a function of model complexity on this problem:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WZwWBA8S6B3L" + }, + "outputs": [], + "source": [ + "x = np.arange(len(multi_performance))\n", + "width = 0.3\n", + "\n", + "metric_name = 'mean_absolute_error'\n", + "val_mae = [v[metric_name] for v in multi_val_performance.values()]\n", + "test_mae = [v[metric_name] for v in multi_performance.values()]\n", + "\n", + "plt.bar(x - 0.17, val_mae, width, label='Validation')\n", + "plt.bar(x + 0.17, test_mae, width, label='Test')\n", + "plt.xticks(ticks=x, labels=multi_performance.keys(),\n", + " rotation=45)\n", + "plt.ylabel(f'MAE (average over all times and outputs)')\n", + "_ = plt.legend()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zq3hUsedCEmJ" + }, + "source": [ + "The metrics for the multi-output models in the first half of this tutorial show the performance averaged across all output features. These performances are similar but also averaged across output time steps." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jKq3eAIvH4Db" + }, + "outputs": [], + "source": [ + "for name, value in multi_performance.items():\n", + " print(f'{name:8s}: {value[metric_name]:0.4f}')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MpBFwfnaHP23" + }, + "source": [ + "The gains achieved going from a dense model to convolutional and recurrent models are only a few percent (if any), and the autoregressive model performed clearly worse. So these more complex approaches may not be worth while on **this** problem, but there was no way to know without trying, and these models could be helpful for **your** problem." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pOzaIRYBhqwg" + }, + "source": [ + "## Next steps\n", + "\n", + "This tutorial was a quick introduction to time series forecasting using TensorFlow.\n", + "\n", + "To learn more, refer to:\n", + "\n", + "- Chapter 15 of Hands-on Machine Learning with Scikit-Learn, Keras, and TensorFlow, 2nd Edition.\n", + "- Chapter 6 of Deep Learning with Python.\n", + "- Lesson 8 of Udacity's intro to TensorFlow for deep learning, including the exercise notebooks.\n", + "\n", + "Also, remember that you can implement any classical time series model in TensorFlow—this tutorial just focuses on TensorFlow's built-in functionality.\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "time_series.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/site/en/tutorials/text/image_captioning.ipynb b/site/en/tutorials/text/image_captioning.ipynb deleted file mode 100644 index 1b1d7cb9c3e..00000000000 --- a/site/en/tutorials/text/image_captioning.ipynb +++ /dev/null @@ -1,1107 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K2s1A9eLRPEj" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "VRLVEKiTEn04" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cffg2i257iMS" - }, - "source": [ - "# Image captioning with visual attention\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/image_captioning\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/image_captioning.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/image_captioning.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/text/image_captioning.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QASbY_HGo4Lq" - }, - "source": [ - "Given an image like the example below, our goal is to generate a caption such as \"a surfer riding on a wave\".\n", - "\n", - "![Man Surfing](https://tensorflow.org/images/surf.jpg)\n", - "\n", - "*[Image Source](https://commons.wikimedia.org/wiki/Surfing#/media/File:Surfing_in_Hawaii.jpg); License: Public Domain*\n", - "\n", - "To accomplish this, you'll use an attention-based model, which enables us to see what parts of the image the model focuses on as it generates a caption.\n", - "\n", - "![Prediction](https://tensorflow.org/images/imcap_prediction.png)\n", - "\n", - "The model architecture is similar to [Show, Attend and Tell: Neural Image Caption Generation with Visual Attention](https://arxiv.org/abs/1502.03044).\n", - "\n", - "This notebook is an end-to-end example. When you run the notebook, it downloads the [MS-COCO](http://cocodataset.org/#home) dataset, preprocesses and caches a subset of images using Inception V3, trains an encoder-decoder model, and generates captions on new images using the trained model.\n", - "\n", - "In this example, you will train a model on a relatively small amount of data—the first 30,000 captions for about 20,000 images (because there are multiple captions per image in the dataset)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6svGEEek67ds" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U8l4RJ0XRPEm" - }, - "outputs": [], - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "# You'll generate plots of attention in order to see which parts of an image\n", - "# our model focuses on during captioning\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# Scikit-learn includes many helpful utilities\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.utils import shuffle\n", - "\n", - "import re\n", - "import numpy as np\n", - "import os\n", - "import time\n", - "import json\n", - "from glob import glob\n", - "from PIL import Image\n", - "import pickle" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b6qbGw8MRPE5" - }, - "source": [ - "## Download and prepare the MS-COCO dataset\n", - "\n", - "You will use the [MS-COCO dataset](http://cocodataset.org/#home) to train our model. The dataset contains over 82,000 images, each of which has at least 5 different caption annotations. The code below downloads and extracts the dataset automatically.\n", - "\n", - "**Caution: large download ahead**. You'll use the training set, which is a 13GB file." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "krQuPYTtRPE7" - }, - "outputs": [], - "source": [ - "annotation_zip = tf.keras.utils.get_file('captions.zip',\n", - " cache_subdir=os.path.abspath('.'),\n", - " origin = 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',\n", - " extract = True)\n", - "annotation_file = os.path.dirname(annotation_zip)+'/annotations/captions_train2014.json'\n", - "\n", - "name_of_zip = 'train2014.zip'\n", - "if not os.path.exists(os.path.abspath('.') + '/' + name_of_zip):\n", - " image_zip = tf.keras.utils.get_file(name_of_zip,\n", - " cache_subdir=os.path.abspath('.'),\n", - " origin = 'http://images.cocodataset.org/zips/train2014.zip',\n", - " extract = True)\n", - " PATH = os.path.dirname(image_zip)+'/train2014/'\n", - "else:\n", - " PATH = os.path.abspath('.')+'/train2014/'" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aANEzb5WwSzg" - }, - "source": [ - "## Optional: limit the size of the training set \n", - "To speed up training for this tutorial, you'll use a subset of 30,000 captions and their corresponding images to train our model. Choosing to use more data would result in improved captioning quality." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4G3b8x8_RPFD" - }, - "outputs": [], - "source": [ - "# Read the json file\n", - "with open(annotation_file, 'r') as f:\n", - " annotations = json.load(f)\n", - "\n", - "# Store captions and image names in vectors\n", - "all_captions = []\n", - "all_img_name_vector = []\n", - "\n", - "for annot in annotations['annotations']:\n", - " caption = '\u003cstart\u003e ' + annot['caption'] + ' \u003cend\u003e'\n", - " image_id = annot['image_id']\n", - " full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n", - "\n", - " all_img_name_vector.append(full_coco_image_path)\n", - " all_captions.append(caption)\n", - "\n", - "# Shuffle captions and image_names together\n", - "# Set a random state\n", - "train_captions, img_name_vector = shuffle(all_captions,\n", - " all_img_name_vector,\n", - " random_state=1)\n", - "\n", - "# Select the first 30000 captions from the shuffled set\n", - "num_examples = 30000\n", - "train_captions = train_captions[:num_examples]\n", - "img_name_vector = img_name_vector[:num_examples]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mPBMgK34RPFL" - }, - "outputs": [], - "source": [ - "len(train_captions), len(all_captions)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8cSW4u-ORPFQ" - }, - "source": [ - "## Preprocess the images using InceptionV3\n", - "Next, you will use InceptionV3 (which is pretrained on Imagenet) to classify each image. You will extract features from the last convolutional layer.\n", - "\n", - "First, you will convert the images into InceptionV3's expected format by:\n", - "* Resizing the image to 299px by 299px\n", - "* [Preprocess the images](https://cloud.google.com/tpu/docs/inception-v3-advanced#preprocessing_stage) using the [preprocess_input](https://www.tensorflow.org/api_docs/python/tf/keras/applications/inception_v3/preprocess_input) method to normalize the image so that it contains pixels in the range of -1 to 1, which matches the format of the images used to train InceptionV3." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zXR0217aRPFR" - }, - "outputs": [], - "source": [ - "def load_image(image_path):\n", - " img = tf.io.read_file(image_path)\n", - " img = tf.image.decode_jpeg(img, channels=3)\n", - " img = tf.image.resize(img, (299, 299))\n", - " img = tf.keras.applications.inception_v3.preprocess_input(img)\n", - " return img, image_path" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MDvIu4sXRPFV" - }, - "source": [ - "## Initialize InceptionV3 and load the pretrained Imagenet weights\n", - "\n", - "Now you'll create a tf.keras model where the output layer is the last convolutional layer in the InceptionV3 architecture. The shape of the output of this layer is ```8x8x2048```. You use the last convolutional layer because you are using attention in this example. You don't perform this initialization during training because it could become a bottleneck.\n", - "\n", - "* You forward each image through the network and store the resulting vector in a dictionary (image_name --\u003e feature_vector).\n", - "* After all the images are passed through the network, you pickle the dictionary and save it to disk.\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RD3vW4SsRPFW" - }, - "outputs": [], - "source": [ - "image_model = tf.keras.applications.InceptionV3(include_top=False,\n", - " weights='imagenet')\n", - "new_input = image_model.input\n", - "hidden_layer = image_model.layers[-1].output\n", - "\n", - "image_features_extract_model = tf.keras.Model(new_input, hidden_layer)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rERqlR3WRPGO" - }, - "source": [ - "## Caching the features extracted from InceptionV3\n", - "\n", - "You will pre-process each image with InceptionV3 and cache the output to disk. Caching the output in RAM would be faster but also memory intensive, requiring 8 \\* 8 \\* 2048 floats per image. At the time of writing, this exceeds the memory limitations of Colab (currently 12GB of memory).\n", - "\n", - "Performance could be improved with a more sophisticated caching strategy (for example, by sharding the images to reduce random access disk I/O), but that would require more code.\n", - "\n", - "The caching will take about 10 minutes to run in Colab with a GPU. If you'd like to see a progress bar, you can: \n", - "\n", - "1. install [tqdm](https://github.com/tqdm/tqdm):\n", - "\n", - " `!pip install tqdm`\n", - "\n", - "2. Import tqdm:\n", - "\n", - " `from tqdm import tqdm`\n", - "\n", - "3. Change the following line:\n", - "\n", - " `for img, path in image_dataset:`\n", - "\n", - " to:\n", - "\n", - " `for img, path in tqdm(image_dataset):`\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Dx_fvbVgRPGQ" - }, - "outputs": [], - "source": [ - "# Get unique images\n", - "encode_train = sorted(set(img_name_vector))\n", - "\n", - "# Feel free to change batch_size according to your system configuration\n", - "image_dataset = tf.data.Dataset.from_tensor_slices(encode_train)\n", - "image_dataset = image_dataset.map(\n", - " load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(16)\n", - "\n", - "for img, path in image_dataset:\n", - " batch_features = image_features_extract_model(img)\n", - " batch_features = tf.reshape(batch_features,\n", - " (batch_features.shape[0], -1, batch_features.shape[3]))\n", - "\n", - " for bf, p in zip(batch_features, path):\n", - " path_of_feature = p.numpy().decode(\"utf-8\")\n", - " np.save(path_of_feature, bf.numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nyqH3zFwRPFi" - }, - "source": [ - "## Preprocess and tokenize the captions\n", - "\n", - "* First, you'll tokenize the captions (for example, by splitting on spaces). This gives us a vocabulary of all of the unique words in the data (for example, \"surfing\", \"football\", and so on).\n", - "* Next, you'll limit the vocabulary size to the top 5,000 words (to save memory). You'll replace all other words with the token \"UNK\" (unknown).\n", - "* You then create word-to-index and index-to-word mappings.\n", - "* Finally, you pad all sequences to be the same length as the longest one." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HZfK8RhQRPFj" - }, - "outputs": [], - "source": [ - "# Find the maximum length of any caption in our dataset\n", - "def calc_max_length(tensor):\n", - " return max(len(t) for t in tensor)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "oJGE34aiRPFo" - }, - "outputs": [], - "source": [ - "# Choose the top 5000 words from the vocabulary\n", - "top_k = 5000\n", - "tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k,\n", - " oov_token=\"\u003cunk\u003e\",\n", - " filters='!\"#$%\u0026()*+.,-/:;=?@[\\]^_`{|}~ ')\n", - "tokenizer.fit_on_texts(train_captions)\n", - "train_seqs = tokenizer.texts_to_sequences(train_captions)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8Q44tNQVRPFt" - }, - "outputs": [], - "source": [ - "tokenizer.word_index['\u003cpad\u003e'] = 0\n", - "tokenizer.index_word[0] = '\u003cpad\u003e'" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0fpJb5ojRPFv" - }, - "outputs": [], - "source": [ - "# Create the tokenized vectors\n", - "train_seqs = tokenizer.texts_to_sequences(train_captions)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AidglIZVRPF4" - }, - "outputs": [], - "source": [ - "# Pad each vector to the max_length of the captions\n", - "# If you do not provide a max_length value, pad_sequences calculates it automatically\n", - "cap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gL0wkttkRPGA" - }, - "outputs": [], - "source": [ - "# Calculates the max_length, which is used to store the attention weights\n", - "max_length = calc_max_length(train_seqs)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M3CD75nDpvTI" - }, - "source": [ - "## Split the data into training and testing" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iS7DDMszRPGF" - }, - "outputs": [], - "source": [ - "# Create training and validation sets using an 80-20 split\n", - "img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector,\n", - " cap_vector,\n", - " test_size=0.2,\n", - " random_state=0)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "XmViPkRFRPGH" - }, - "outputs": [], - "source": [ - "len(img_name_train), len(cap_train), len(img_name_val), len(cap_val)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uEWM9xrYcg45" - }, - "source": [ - "## Create a tf.data dataset for training\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "horagNvhhZiy" - }, - "source": [ - " Our images and captions are ready! Next, let's create a tf.data dataset to use for training our model." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Q3TnZ1ToRPGV" - }, - "outputs": [], - "source": [ - "# Feel free to change these parameters according to your system's configuration\n", - "\n", - "BATCH_SIZE = 64\n", - "BUFFER_SIZE = 1000\n", - "embedding_dim = 256\n", - "units = 512\n", - "vocab_size = top_k + 1\n", - "num_steps = len(img_name_train) // BATCH_SIZE\n", - "# Shape of the vector extracted from InceptionV3 is (64, 2048)\n", - "# These two variables represent that vector shape\n", - "features_shape = 2048\n", - "attention_features_shape = 64" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SmZS2N0bXG3T" - }, - "outputs": [], - "source": [ - "# Load the numpy files\n", - "def map_func(img_name, cap):\n", - " img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n", - " return img_tensor, cap" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FDF_Nm3tRPGZ" - }, - "outputs": [], - "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\n", - "\n", - "# Use map to load the numpy files in parallel\n", - "dataset = dataset.map(lambda item1, item2: tf.numpy_function(\n", - " map_func, [item1, item2], [tf.float32, tf.int32]),\n", - " num_parallel_calls=tf.data.experimental.AUTOTUNE)\n", - "\n", - "# Shuffle and batch\n", - "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", - "dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nrvoDphgRPGd" - }, - "source": [ - "## Model\n", - "\n", - "Fun fact: the decoder below is identical to the one in the example for [Neural Machine Translation with Attention](../sequences/nmt_with_attention.ipynb).\n", - "\n", - "The model architecture is inspired by the [Show, Attend and Tell](https://arxiv.org/pdf/1502.03044.pdf) paper.\n", - "\n", - "* In this example, you extract the features from the lower convolutional layer of InceptionV3 giving us a vector of shape (8, 8, 2048).\n", - "* You squash that to a shape of (64, 2048).\n", - "* This vector is then passed through the CNN Encoder (which consists of a single Fully connected layer).\n", - "* The RNN (here GRU) attends over the image to predict the next word." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ja2LFTMSdeV3" - }, - "outputs": [], - "source": [ - "class BahdanauAttention(tf.keras.Model):\n", - " def __init__(self, units):\n", - " super(BahdanauAttention, self).__init__()\n", - " self.W1 = tf.keras.layers.Dense(units)\n", - " self.W2 = tf.keras.layers.Dense(units)\n", - " self.V = tf.keras.layers.Dense(1)\n", - "\n", - " def call(self, features, hidden):\n", - " # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n", - "\n", - " # hidden shape == (batch_size, hidden_size)\n", - " # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n", - " hidden_with_time_axis = tf.expand_dims(hidden, 1)\n", - "\n", - " # score shape == (batch_size, 64, hidden_size)\n", - " score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n", - "\n", - " # attention_weights shape == (batch_size, 64, 1)\n", - " # you get 1 at the last axis because you are applying score to self.V\n", - " attention_weights = tf.nn.softmax(self.V(score), axis=1)\n", - "\n", - " # context_vector shape after sum == (batch_size, hidden_size)\n", - " context_vector = attention_weights * features\n", - " context_vector = tf.reduce_sum(context_vector, axis=1)\n", - "\n", - " return context_vector, attention_weights" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AZ7R1RxHRPGf" - }, - "outputs": [], - "source": [ - "class CNN_Encoder(tf.keras.Model):\n", - " # Since you have already extracted the features and dumped it using pickle\n", - " # This encoder passes those features through a Fully connected layer\n", - " def __init__(self, embedding_dim):\n", - " super(CNN_Encoder, self).__init__()\n", - " # shape after fc == (batch_size, 64, embedding_dim)\n", - " self.fc = tf.keras.layers.Dense(embedding_dim)\n", - "\n", - " def call(self, x):\n", - " x = self.fc(x)\n", - " x = tf.nn.relu(x)\n", - " return x" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "V9UbGQmERPGi" - }, - "outputs": [], - "source": [ - "class RNN_Decoder(tf.keras.Model):\n", - " def __init__(self, embedding_dim, units, vocab_size):\n", - " super(RNN_Decoder, self).__init__()\n", - " self.units = units\n", - "\n", - " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", - " self.gru = tf.keras.layers.GRU(self.units,\n", - " return_sequences=True,\n", - " return_state=True,\n", - " recurrent_initializer='glorot_uniform')\n", - " self.fc1 = tf.keras.layers.Dense(self.units)\n", - " self.fc2 = tf.keras.layers.Dense(vocab_size)\n", - "\n", - " self.attention = BahdanauAttention(self.units)\n", - "\n", - " def call(self, x, features, hidden):\n", - " # defining attention as a separate model\n", - " context_vector, attention_weights = self.attention(features, hidden)\n", - "\n", - " # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n", - " x = self.embedding(x)\n", - "\n", - " # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n", - " x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n", - "\n", - " # passing the concatenated vector to the GRU\n", - " output, state = self.gru(x)\n", - "\n", - " # shape == (batch_size, max_length, hidden_size)\n", - " x = self.fc1(output)\n", - "\n", - " # x shape == (batch_size * max_length, hidden_size)\n", - " x = tf.reshape(x, (-1, x.shape[2]))\n", - "\n", - " # output shape == (batch_size * max_length, vocab)\n", - " x = self.fc2(x)\n", - "\n", - " return x, state, attention_weights\n", - "\n", - " def reset_state(self, batch_size):\n", - " return tf.zeros((batch_size, self.units))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Qs_Sr03wRPGk" - }, - "outputs": [], - "source": [ - "encoder = CNN_Encoder(embedding_dim)\n", - "decoder = RNN_Decoder(embedding_dim, units, vocab_size)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-bYN7xA0RPGl" - }, - "outputs": [], - "source": [ - "optimizer = tf.keras.optimizers.Adam()\n", - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " from_logits=True, reduction='none')\n", - "\n", - "def loss_function(real, pred):\n", - " mask = tf.math.logical_not(tf.math.equal(real, 0))\n", - " loss_ = loss_object(real, pred)\n", - "\n", - " mask = tf.cast(mask, dtype=loss_.dtype)\n", - " loss_ *= mask\n", - "\n", - " return tf.reduce_mean(loss_)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6A3Ni64joyab" - }, - "source": [ - "## Checkpoint" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "PpJAqPMWo0uE" - }, - "outputs": [], - "source": [ - "checkpoint_path = \"./checkpoints/train\"\n", - "ckpt = tf.train.Checkpoint(encoder=encoder,\n", - " decoder=decoder,\n", - " optimizer = optimizer)\n", - "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fUkbqhc_uObw" - }, - "outputs": [], - "source": [ - "start_epoch = 0\n", - "if ckpt_manager.latest_checkpoint:\n", - " start_epoch = int(ckpt_manager.latest_checkpoint.split('-')[-1])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PHod7t72RPGn" - }, - "source": [ - "## Training\n", - "\n", - "* You extract the features stored in the respective `.npy` files and then pass those features through the encoder.\n", - "* The encoder output, hidden state(initialized to 0) and the decoder input (which is the start token) is passed to the decoder.\n", - "* The decoder returns the predictions and the decoder hidden state.\n", - "* The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n", - "* Use teacher forcing to decide the next input to the decoder.\n", - "* Teacher forcing is the technique where the target word is passed as the next input to the decoder.\n", - "* The final step is to calculate the gradients and apply it to the optimizer and backpropagate.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Vt4WZ5mhJE-E" - }, - "outputs": [], - "source": [ - "# adding this in a separate cell because if you run the training cell\n", - "# many times, the loss_plot array will be reset\n", - "loss_plot = []" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sqgyz2ANKlpU" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def train_step(img_tensor, target):\n", - " loss = 0\n", - "\n", - " # initializing the hidden state for each batch\n", - " # because the captions are not related from image to image\n", - " hidden = decoder.reset_state(batch_size=target.shape[0])\n", - "\n", - " dec_input = tf.expand_dims([tokenizer.word_index['\u003cstart\u003e']] * target.shape[0], 1)\n", - "\n", - " with tf.GradientTape() as tape:\n", - " features = encoder(img_tensor)\n", - "\n", - " for i in range(1, target.shape[1]):\n", - " # passing the features through the decoder\n", - " predictions, hidden, _ = decoder(dec_input, features, hidden)\n", - "\n", - " loss += loss_function(target[:, i], predictions)\n", - "\n", - " # using teacher forcing\n", - " dec_input = tf.expand_dims(target[:, i], 1)\n", - "\n", - " total_loss = (loss / int(target.shape[1]))\n", - "\n", - " trainable_variables = encoder.trainable_variables + decoder.trainable_variables\n", - "\n", - " gradients = tape.gradient(loss, trainable_variables)\n", - "\n", - " optimizer.apply_gradients(zip(gradients, trainable_variables))\n", - "\n", - " return loss, total_loss" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UlA4VIQpRPGo" - }, - "outputs": [], - "source": [ - "EPOCHS = 20\n", - "\n", - "for epoch in range(start_epoch, EPOCHS):\n", - " start = time.time()\n", - " total_loss = 0\n", - "\n", - " for (batch, (img_tensor, target)) in enumerate(dataset):\n", - " batch_loss, t_loss = train_step(img_tensor, target)\n", - " total_loss += t_loss\n", - "\n", - " if batch % 100 == 0:\n", - " print ('Epoch {} Batch {} Loss {:.4f}'.format(\n", - " epoch + 1, batch, batch_loss.numpy() / int(target.shape[1])))\n", - " # storing the epoch end loss value to plot later\n", - " loss_plot.append(total_loss / num_steps)\n", - "\n", - " if epoch % 5 == 0:\n", - " ckpt_manager.save()\n", - "\n", - " print ('Epoch {} Loss {:.6f}'.format(epoch + 1,\n", - " total_loss/num_steps))\n", - " print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1Wm83G-ZBPcC" - }, - "outputs": [], - "source": [ - "plt.plot(loss_plot)\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.title('Loss Plot')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xGvOcLQKghXN" - }, - "source": [ - "## Caption!\n", - "\n", - "* The evaluate function is similar to the training loop, except you don't use teacher forcing here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n", - "* Stop predicting when the model predicts the end token.\n", - "* And store the attention weights for every time step." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RCWpDtyNRPGs" - }, - "outputs": [], - "source": [ - "def evaluate(image):\n", - " attention_plot = np.zeros((max_length, attention_features_shape))\n", - "\n", - " hidden = decoder.reset_state(batch_size=1)\n", - "\n", - " temp_input = tf.expand_dims(load_image(image)[0], 0)\n", - " img_tensor_val = image_features_extract_model(temp_input)\n", - " img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n", - "\n", - " features = encoder(img_tensor_val)\n", - "\n", - " dec_input = tf.expand_dims([tokenizer.word_index['\u003cstart\u003e']], 0)\n", - " result = []\n", - "\n", - " for i in range(max_length):\n", - " predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n", - "\n", - " attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n", - "\n", - " predicted_id = tf.random.categorical(predictions, 1)[0][0].numpy()\n", - " result.append(tokenizer.index_word[predicted_id])\n", - "\n", - " if tokenizer.index_word[predicted_id] == '\u003cend\u003e':\n", - " return result, attention_plot\n", - "\n", - " dec_input = tf.expand_dims([predicted_id], 0)\n", - "\n", - " attention_plot = attention_plot[:len(result), :]\n", - " return result, attention_plot" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fD_y7PD6RPGt" - }, - "outputs": [], - "source": [ - "def plot_attention(image, result, attention_plot):\n", - " temp_image = np.array(Image.open(image))\n", - "\n", - " fig = plt.figure(figsize=(10, 10))\n", - "\n", - " len_result = len(result)\n", - " for l in range(len_result):\n", - " temp_att = np.resize(attention_plot[l], (8, 8))\n", - " ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n", - " ax.set_title(result[l])\n", - " img = ax.imshow(temp_image)\n", - " ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n", - "\n", - " plt.tight_layout()\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7x8RiPHe_4qI" - }, - "outputs": [], - "source": [ - "# captions on the validation set\n", - "rid = np.random.randint(0, len(img_name_val))\n", - "image = img_name_val[rid]\n", - "real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n", - "result, attention_plot = evaluate(image)\n", - "\n", - "print ('Real Caption:', real_caption)\n", - "print ('Prediction Caption:', ' '.join(result))\n", - "plot_attention(image, result, attention_plot)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Rprk3HEvZuxb" - }, - "source": [ - "## Try it on your own images\n", - "For fun, below we've provided a method you can use to caption your own images with the model we've just trained. Keep in mind, it was trained on a relatively small amount of data, and your images may be different from the training data (so be prepared for weird results!)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9Psd1quzaAWg" - }, - "outputs": [], - "source": [ - "image_url = 'https://tensorflow.org/images/surf.jpg'\n", - "image_extension = image_url[-4:]\n", - "image_path = tf.keras.utils.get_file('image'+image_extension,\n", - " origin=image_url)\n", - "\n", - "result, attention_plot = evaluate(image_path)\n", - "print ('Prediction Caption:', ' '.join(result))\n", - "plot_attention(image_path, result, attention_plot)\n", - "# opening the image\n", - "Image.open(image_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VJZXyJco6uLO" - }, - "source": [ - "# Next steps\n", - "\n", - "Congrats! You've just trained an image captioning model with attention. Next, take a look at this example [Neural Machine Translation with Attention](../sequences/nmt_with_attention.ipynb). It uses a similar architecture to translate between Spanish and English sentences. You can also experiment with training the code in this notebook on a different dataset." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "image_captioning.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/text/images/embedding.jpg b/site/en/tutorials/text/images/embedding.jpg deleted file mode 100644 index 3f9cf1e9234..00000000000 Binary files a/site/en/tutorials/text/images/embedding.jpg and /dev/null differ diff --git a/site/en/tutorials/text/images/embedding2.png b/site/en/tutorials/text/images/embedding2.png deleted file mode 100644 index a0f1666961d..00000000000 Binary files a/site/en/tutorials/text/images/embedding2.png and /dev/null differ diff --git a/site/en/tutorials/text/images/one-hot.png b/site/en/tutorials/text/images/one-hot.png deleted file mode 100644 index adfe9b9b733..00000000000 Binary files a/site/en/tutorials/text/images/one-hot.png and /dev/null differ diff --git a/site/en/tutorials/text/images/text_generation_sampling.png b/site/en/tutorials/text/images/text_generation_sampling.png deleted file mode 100644 index 817cb89d9ee..00000000000 Binary files a/site/en/tutorials/text/images/text_generation_sampling.png and /dev/null differ diff --git a/site/en/tutorials/text/images/text_generation_training.png b/site/en/tutorials/text/images/text_generation_training.png deleted file mode 100644 index 575d3d4195b..00000000000 Binary files a/site/en/tutorials/text/images/text_generation_training.png and /dev/null differ diff --git a/site/en/tutorials/text/index.md b/site/en/tutorials/text/index.md new file mode 100644 index 00000000000..5f8e5b0e36c --- /dev/null +++ b/site/en/tutorials/text/index.md @@ -0,0 +1,91 @@ +# Text and natural language processing with TensorFlow + +Before you can train a model on text data, you'll typically need to process +(or preprocess) the text. In many cases, text needs to be tokenized and +vectorized before it can be fed to a model, and in some cases the text requires +additional preprocessing steps such as normalization and feature selection. + +After text is processed into a suitable format, you can use it in natural +language processing (NLP) workflows such as text classification, text +generation, summarization, and translation. + +TensorFlow provides two libraries for text and natural language processing: +KerasNLP ([GitHub](https://github.com/keras-team/keras-nlp)) and +TensorFlow Text ([GitHub](https://github.com/tensorflow/text)). + +KerasNLP is a high-level NLP modeling library that includes all the latest +transformer-based models as well as lower-level tokenization utilities. It's the +recommended solution for most NLP use cases. Built on TensorFlow Text, KerasNLP +abstracts low-level text processing operations into an API that's designed for +ease of use. But if you prefer not to work with the Keras API, or you need +access to the lower-level text processing ops, you can use TensorFlow Text +directly. + +## KerasNLP + +The easiest way to get started processing text in TensorFlow is to use +[KerasNLP](https://keras.io/keras_nlp/). KerasNLP is a natural language +processing library that supports workflows built from modular components that +have state-of-the-art preset weights and architectures. You can use KerasNLP +components with their out-of-the-box configuration. If you need more control, +you can easily customize components. KerasNLP provides in-graph computation for +all workflows so you can expect easy productionization using the TensorFlow +ecosystem. + +KerasNLP contains end-to-end implementations of popular +[model architectures](https://keras.io/api/keras_nlp/models/) like +[BERT](https://keras.io/api/keras_nlp/models/bert/) and +[FNet](https://keras.io/api/keras_nlp/models/f_net/). Using KerasNLP models, +layers, and tokenizers, you can complete many state-of-the-art NLP workflows, +including +[machine translation](https://keras.io/examples/nlp/neural_machine_translation_with_keras_nlp/), +[text generation](https://keras.io/examples/generative/text_generation_gpt/), +[text classification](https://keras.io/examples/nlp/fnet_classification_with_keras_nlp/), +and +[transformer model training](https://keras.io/guides/keras_nlp/transformer_pretraining/). + +KerasNLP is an extension of the core Keras API, and every high-level KerasNLP +module is a `Layer` or `Model`. If you're familiar with Keras, you already +understand most of KerasNLP. + +## TensorFlow Text + +KerasNLP provides high-level text processing modules that are available as +layers or models. If you need access to lower-level tools, you can use +[TensorFlow Text](https://www.tensorflow.org/text/guide/tf_text_intro). +TensorFlow Text provides operations and libraries to help you work with raw text +strings and documents. TensorFlow Text can perform the preprocessing regularly +required by text-based models, and it also includes other features useful for +sequence modeling. + +Using TensorFlow Text, you can do the following: + +* Apply feature-rich tokenizers that can split strings on whitespace, separate + words and punctuation, and return byte offsets with tokens, so that you know + where a string can be found in the source text. +* Check if a token matches a specified string pattern. You can check for + capitalization, punctuation, numerical data, and other token features. +* Combine tokens into n-grams. +* Process text within the TensorFlow graph, so that tokenization during training + matches tokenization at inference. + +## Where to start + +The following resources will help you get started with TensorFlow text +processing: + +* [TensorFlow Text](https://www.tensorflow.org/text): Tutorials, guides, and + other resources to help you process text using TensorFlow Text and KerasNLP. +* [KerasNLP](https://keras.io/keras_nlp/): Documentation and resources for + KerasNLP. + * [Getting Started with KerasNLP](https://keras.io/guides/keras_nlp/getting_started/) + * [Pretraining a Transformer from scratch with KerasNLP](https://keras.io/guides/keras_nlp/transformer_pretraining/) +* [TensorFlow tutorials](https://www.tensorflow.org/tutorials): The core + TensorFlow documentation (this guide) includes several text processing + tutorials. + * [Basic text classification](https://www.tensorflow.org/tutorials/keras/text_classification) + * [Text classification with TensorFlow Hub: Movie reviews](https://www.tensorflow.org/tutorials/keras/text_classification_with_hub) + * [Load text](https://www.tensorflow.org/tutorials/load_data/text) +* [Google Machine Learning: Text Classification guide](https://developers.google.com/machine-learning/guides/text-classification): + A step-by-step introduction to text classification. This is a good place to + start if you're new to machine learning. \ No newline at end of file diff --git a/site/en/tutorials/text/nmt_with_attention.ipynb b/site/en/tutorials/text/nmt_with_attention.ipynb deleted file mode 100644 index 9282df2e7b7..00000000000 --- a/site/en/tutorials/text/nmt_with_attention.ipynb +++ /dev/null @@ -1,1023 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "s_qNSzzyaCbD" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "jmjh290raIky" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J0Qjg6vuaHNt" - }, - "source": [ - "# Neural machine translation with attention" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AOpGoE2T-YXS" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/nmt_with_attention\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/nmt_with_attention.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/nmt_with_attention.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/text/nmt_with_attention.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CiwtNgENbx2g" - }, - "source": [ - "This notebook trains a sequence to sequence (seq2seq) model for Spanish to English translation. This is an advanced example that assumes some knowledge of sequence to sequence models.\n", - "\n", - "After training the model in this notebook, you will be able to input a Spanish sentence, such as *\"¿todavia estan en casa?\"*, and return the English translation: *\"are you still at home?\"*\n", - "\n", - "The translation quality is reasonable for a toy example, but the generated attention plot is perhaps more interesting. This shows which parts of the input sentence has the model's attention while translating:\n", - "\n", - "\u003cimg src=\"https://tensorflow.org/images/spanish-english.png\" alt=\"spanish-english attention plot\"\u003e\n", - "\n", - "Note: This example takes approximately 10 mintues to run on a single P100 GPU." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tnxXKDjq3jEL" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.ticker as ticker\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "import unicodedata\n", - "import re\n", - "import numpy as np\n", - "import os\n", - "import io\n", - "import time" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfodePkj3jEa" - }, - "source": [ - "## Download and prepare the dataset\n", - "\n", - "We'll use a language dataset provided by http://www.manythings.org/anki/. This dataset contains language translation pairs in the format:\n", - "\n", - "```\n", - "May I borrow this book?\t¿Puedo tomar prestado este libro?\n", - "```\n", - "\n", - "There are a variety of languages available, but we'll use the English-Spanish dataset. For convenience, we've hosted a copy of this dataset on Google Cloud, but you can also download your own copy. After downloading the dataset, here are the steps we'll take to prepare the data:\n", - "\n", - "1. Add a *start* and *end* token to each sentence.\n", - "2. Clean the sentences by removing special characters.\n", - "3. Create a word index and reverse word index (dictionaries mapping from word → id and id → word).\n", - "4. Pad each sentence to a maximum length." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kRVATYOgJs1b" - }, - "outputs": [], - "source": [ - "# Download the file\n", - "path_to_zip = tf.keras.utils.get_file(\n", - " 'spa-eng.zip', origin='http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip',\n", - " extract=True)\n", - "\n", - "path_to_file = os.path.dirname(path_to_zip)+\"/spa-eng/spa.txt\"" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "rd0jw-eC3jEh" - }, - "outputs": [], - "source": [ - "# Converts the unicode file to ascii\n", - "def unicode_to_ascii(s):\n", - " return ''.join(c for c in unicodedata.normalize('NFD', s)\n", - " if unicodedata.category(c) != 'Mn')\n", - "\n", - "\n", - "def preprocess_sentence(w):\n", - " w = unicode_to_ascii(w.lower().strip())\n", - "\n", - " # creating a space between a word and the punctuation following it\n", - " # eg: \"he is a boy.\" =\u003e \"he is a boy .\"\n", - " # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n", - " w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n", - " w = re.sub(r'[\" \"]+', \" \", w)\n", - "\n", - " # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n", - " w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n", - "\n", - " w = w.rstrip().strip()\n", - "\n", - " # adding a start and an end token to the sentence\n", - " # so that the model know when to start and stop predicting.\n", - " w = '\u003cstart\u003e ' + w + ' \u003cend\u003e'\n", - " return w" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "opI2GzOt479E" - }, - "outputs": [], - "source": [ - "en_sentence = u\"May I borrow this book?\"\n", - "sp_sentence = u\"¿Puedo tomar prestado este libro?\"\n", - "print(preprocess_sentence(en_sentence))\n", - "print(preprocess_sentence(sp_sentence).encode('utf-8'))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OHn4Dct23jEm" - }, - "outputs": [], - "source": [ - "# 1. Remove the accents\n", - "# 2. Clean the sentences\n", - "# 3. Return word pairs in the format: [ENGLISH, SPANISH]\n", - "def create_dataset(path, num_examples):\n", - " lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n", - "\n", - " word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')] for l in lines[:num_examples]]\n", - "\n", - " return zip(*word_pairs)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cTbSbBz55QtF" - }, - "outputs": [], - "source": [ - "en, sp = create_dataset(path_to_file, None)\n", - "print(en[-1])\n", - "print(sp[-1])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OmMZQpdO60dt" - }, - "outputs": [], - "source": [ - "def max_length(tensor):\n", - " return max(len(t) for t in tensor)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bIOn8RCNDJXG" - }, - "outputs": [], - "source": [ - "def tokenize(lang):\n", - " lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(\n", - " filters='')\n", - " lang_tokenizer.fit_on_texts(lang)\n", - "\n", - " tensor = lang_tokenizer.texts_to_sequences(lang)\n", - "\n", - " tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n", - " padding='post')\n", - "\n", - " return tensor, lang_tokenizer" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eAY9k49G3jE_" - }, - "outputs": [], - "source": [ - "def load_dataset(path, num_examples=None):\n", - " # creating cleaned input, output pairs\n", - " targ_lang, inp_lang = create_dataset(path, num_examples)\n", - "\n", - " input_tensor, inp_lang_tokenizer = tokenize(inp_lang)\n", - " target_tensor, targ_lang_tokenizer = tokenize(targ_lang)\n", - "\n", - " return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GOi42V79Ydlr" - }, - "source": [ - "### Limit the size of the dataset to experiment faster (optional)\n", - "\n", - "Training on the complete dataset of \u003e100,000 sentences will take a long time. To train faster, we can limit the size of the dataset to 30,000 sentences (of course, translation quality degrades with less data):" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cnxC7q-j3jFD" - }, - "outputs": [], - "source": [ - "# Try experimenting with the size of that dataset\n", - "num_examples = 30000\n", - "input_tensor, target_tensor, inp_lang, targ_lang = load_dataset(path_to_file, num_examples)\n", - "\n", - "# Calculate max_length of the target tensors\n", - "max_length_targ, max_length_inp = max_length(target_tensor), max_length(input_tensor)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4QILQkOs3jFG" - }, - "outputs": [], - "source": [ - "# Creating training and validation sets using an 80-20 split\n", - "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n", - "\n", - "# Show length\n", - "print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lJPmLZGMeD5q" - }, - "outputs": [], - "source": [ - "def convert(lang, tensor):\n", - " for t in tensor:\n", - " if t!=0:\n", - " print (\"%d ----\u003e %s\" % (t, lang.index_word[t]))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VXukARTDd7MT" - }, - "outputs": [], - "source": [ - "print (\"Input Language; index to word mapping\")\n", - "convert(inp_lang, input_tensor_train[0])\n", - "print ()\n", - "print (\"Target Language; index to word mapping\")\n", - "convert(targ_lang, target_tensor_train[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rgCLkfv5uO3d" - }, - "source": [ - "### Create a tf.data dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TqHsArVZ3jFS" - }, - "outputs": [], - "source": [ - "BUFFER_SIZE = len(input_tensor_train)\n", - "BATCH_SIZE = 64\n", - "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n", - "embedding_dim = 256\n", - "units = 1024\n", - "vocab_inp_size = len(inp_lang.word_index)+1\n", - "vocab_tar_size = len(targ_lang.word_index)+1\n", - "\n", - "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n", - "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qc6-NK1GtWQt" - }, - "outputs": [], - "source": [ - "example_input_batch, example_target_batch = next(iter(dataset))\n", - "example_input_batch.shape, example_target_batch.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TNfHIF71ulLu" - }, - "source": [ - "## Write the encoder and decoder model\n", - "\n", - "Implement an encoder-decoder model with attention which you can read about in the TensorFlow [Neural Machine Translation (seq2seq) tutorial](https://github.com/tensorflow/nmt). This example uses a more recent set of APIs. This notebook implements the [attention equations](https://github.com/tensorflow/nmt#background-on-the-attention-mechanism) from the seq2seq tutorial. The following diagram shows that each input words is assigned a weight by the attention mechanism which is then used by the decoder to predict the next word in the sentence. The below picture and formulas are an example of attention mechanism from [Luong's paper](https://arxiv.org/abs/1508.04025v5). \n", - "\n", - "\u003cimg src=\"https://www.tensorflow.org/images/seq2seq/attention_mechanism.jpg\" width=\"500\" alt=\"attention mechanism\"\u003e\n", - "\n", - "The input is put through an encoder model which gives us the encoder output of shape *(batch_size, max_length, hidden_size)* and the encoder hidden state of shape *(batch_size, hidden_size)*.\n", - "\n", - "Here are the equations that are implemented:\n", - "\n", - "\u003cimg src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_0.jpg\" alt=\"attention equation 0\" width=\"800\"\u003e\n", - "\u003cimg src=\"https://www.tensorflow.org/images/seq2seq/attention_equation_1.jpg\" alt=\"attention equation 1\" width=\"800\"\u003e\n", - "\n", - "This tutorial uses [Bahdanau attention](https://arxiv.org/pdf/1409.0473.pdf) for the encoder. Let's decide on notation before writing the simplified form:\n", - "\n", - "* FC = Fully connected (dense) layer\n", - "* EO = Encoder output\n", - "* H = hidden state\n", - "* X = input to the decoder\n", - "\n", - "And the pseudo-code:\n", - "\n", - "* `score = FC(tanh(FC(EO) + FC(H)))`\n", - "* `attention weights = softmax(score, axis = 1)`. Softmax by default is applied on the last axis but here we want to apply it on the *1st axis*, since the shape of score is *(batch_size, max_length, hidden_size)*. `Max_length` is the length of our input. Since we are trying to assign a weight to each input, softmax should be applied on that axis.\n", - "* `context vector = sum(attention weights * EO, axis = 1)`. Same reason as above for choosing axis as 1.\n", - "* `embedding output` = The input to the decoder X is passed through an embedding layer.\n", - "* `merged vector = concat(embedding output, context vector)`\n", - "* This merged vector is then given to the GRU\n", - "\n", - "The shapes of all the vectors at each step have been specified in the comments in the code:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nZ2rI24i3jFg" - }, - "outputs": [], - "source": [ - "class Encoder(tf.keras.Model):\n", - " def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n", - " super(Encoder, self).__init__()\n", - " self.batch_sz = batch_sz\n", - " self.enc_units = enc_units\n", - " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", - " self.gru = tf.keras.layers.GRU(self.enc_units,\n", - " return_sequences=True,\n", - " return_state=True,\n", - " recurrent_initializer='glorot_uniform')\n", - "\n", - " def call(self, x, hidden):\n", - " x = self.embedding(x)\n", - " output, state = self.gru(x, initial_state = hidden)\n", - " return output, state\n", - "\n", - " def initialize_hidden_state(self):\n", - " return tf.zeros((self.batch_sz, self.enc_units))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "60gSVh05Jl6l" - }, - "outputs": [], - "source": [ - "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n", - "\n", - "# sample input\n", - "sample_hidden = encoder.initialize_hidden_state()\n", - "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n", - "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n", - "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "umohpBN2OM94" - }, - "outputs": [], - "source": [ - "class BahdanauAttention(tf.keras.layers.Layer):\n", - " def __init__(self, units):\n", - " super(BahdanauAttention, self).__init__()\n", - " self.W1 = tf.keras.layers.Dense(units)\n", - " self.W2 = tf.keras.layers.Dense(units)\n", - " self.V = tf.keras.layers.Dense(1)\n", - "\n", - " def call(self, query, values):\n", - " # hidden shape == (batch_size, hidden size)\n", - " # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n", - " # we are doing this to perform addition to calculate the score\n", - " hidden_with_time_axis = tf.expand_dims(query, 1)\n", - "\n", - " # score shape == (batch_size, max_length, 1)\n", - " # we get 1 at the last axis because we are applying score to self.V\n", - " # the shape of the tensor before applying self.V is (batch_size, max_length, units)\n", - " score = self.V(tf.nn.tanh(\n", - " self.W1(values) + self.W2(hidden_with_time_axis)))\n", - "\n", - " # attention_weights shape == (batch_size, max_length, 1)\n", - " attention_weights = tf.nn.softmax(score, axis=1)\n", - "\n", - " # context_vector shape after sum == (batch_size, hidden_size)\n", - " context_vector = attention_weights * values\n", - " context_vector = tf.reduce_sum(context_vector, axis=1)\n", - "\n", - " return context_vector, attention_weights" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "k534zTHiDjQU" - }, - "outputs": [], - "source": [ - "attention_layer = BahdanauAttention(10)\n", - "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n", - "\n", - "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n", - "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yJ_B3mhW3jFk" - }, - "outputs": [], - "source": [ - "class Decoder(tf.keras.Model):\n", - " def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n", - " super(Decoder, self).__init__()\n", - " self.batch_sz = batch_sz\n", - " self.dec_units = dec_units\n", - " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", - " self.gru = tf.keras.layers.GRU(self.dec_units,\n", - " return_sequences=True,\n", - " return_state=True,\n", - " recurrent_initializer='glorot_uniform')\n", - " self.fc = tf.keras.layers.Dense(vocab_size)\n", - "\n", - " # used for attention\n", - " self.attention = BahdanauAttention(self.dec_units)\n", - "\n", - " def call(self, x, hidden, enc_output):\n", - " # enc_output shape == (batch_size, max_length, hidden_size)\n", - " context_vector, attention_weights = self.attention(hidden, enc_output)\n", - "\n", - " # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n", - " x = self.embedding(x)\n", - "\n", - " # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n", - " x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n", - "\n", - " # passing the concatenated vector to the GRU\n", - " output, state = self.gru(x)\n", - "\n", - " # output shape == (batch_size * 1, hidden_size)\n", - " output = tf.reshape(output, (-1, output.shape[2]))\n", - "\n", - " # output shape == (batch_size, vocab)\n", - " x = self.fc(output)\n", - "\n", - " return x, state, attention_weights" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "P5UY8wko3jFp" - }, - "outputs": [], - "source": [ - "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n", - "\n", - "sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)),\n", - " sample_hidden, sample_output)\n", - "\n", - "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ch_71VbIRfK" - }, - "source": [ - "## Define the optimizer and the loss function" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WmTHr5iV3jFr" - }, - "outputs": [], - "source": [ - "optimizer = tf.keras.optimizers.Adam()\n", - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " from_logits=True, reduction='none')\n", - "\n", - "def loss_function(real, pred):\n", - " mask = tf.math.logical_not(tf.math.equal(real, 0))\n", - " loss_ = loss_object(real, pred)\n", - "\n", - " mask = tf.cast(mask, dtype=loss_.dtype)\n", - " loss_ *= mask\n", - "\n", - " return tf.reduce_mean(loss_)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DMVWzzsfNl4e" - }, - "source": [ - "## Checkpoints (Object-based saving)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Zj8bXQTgNwrF" - }, - "outputs": [], - "source": [ - "checkpoint_dir = './training_checkpoints'\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n", - " encoder=encoder,\n", - " decoder=decoder)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hpObfY22IddU" - }, - "source": [ - "## Training\n", - "\n", - "1. Pass the *input* through the *encoder* which return *encoder output* and the *encoder hidden state*.\n", - "2. The encoder output, encoder hidden state and the decoder input (which is the *start token*) is passed to the decoder.\n", - "3. The decoder returns the *predictions* and the *decoder hidden state*.\n", - "4. The decoder hidden state is then passed back into the model and the predictions are used to calculate the loss.\n", - "5. Use *teacher forcing* to decide the next input to the decoder.\n", - "6. *Teacher forcing* is the technique where the *target word* is passed as the *next input* to the decoder.\n", - "7. The final step is to calculate the gradients and apply it to the optimizer and backpropagate." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sC9ArXSsVfqn" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def train_step(inp, targ, enc_hidden):\n", - " loss = 0\n", - "\n", - " with tf.GradientTape() as tape:\n", - " enc_output, enc_hidden = encoder(inp, enc_hidden)\n", - "\n", - " dec_hidden = enc_hidden\n", - "\n", - " dec_input = tf.expand_dims([targ_lang.word_index['\u003cstart\u003e']] * BATCH_SIZE, 1)\n", - "\n", - " # Teacher forcing - feeding the target as the next input\n", - " for t in range(1, targ.shape[1]):\n", - " # passing enc_output to the decoder\n", - " predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n", - "\n", - " loss += loss_function(targ[:, t], predictions)\n", - "\n", - " # using teacher forcing\n", - " dec_input = tf.expand_dims(targ[:, t], 1)\n", - "\n", - " batch_loss = (loss / int(targ.shape[1]))\n", - "\n", - " variables = encoder.trainable_variables + decoder.trainable_variables\n", - "\n", - " gradients = tape.gradient(loss, variables)\n", - "\n", - " optimizer.apply_gradients(zip(gradients, variables))\n", - "\n", - " return batch_loss" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ddefjBMa3jF0" - }, - "outputs": [], - "source": [ - "EPOCHS = 10\n", - "\n", - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - "\n", - " enc_hidden = encoder.initialize_hidden_state()\n", - " total_loss = 0\n", - "\n", - " for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n", - " batch_loss = train_step(inp, targ, enc_hidden)\n", - " total_loss += batch_loss\n", - "\n", - " if batch % 100 == 0:\n", - " print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n", - " batch,\n", - " batch_loss.numpy()))\n", - " # saving (checkpoint) the model every 2 epochs\n", - " if (epoch + 1) % 2 == 0:\n", - " checkpoint.save(file_prefix = checkpoint_prefix)\n", - "\n", - " print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n", - " total_loss / steps_per_epoch))\n", - " print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mU3Ce8M6I3rz" - }, - "source": [ - "## Translate\n", - "\n", - "* The evaluate function is similar to the training loop, except we don't use *teacher forcing* here. The input to the decoder at each time step is its previous predictions along with the hidden state and the encoder output.\n", - "* Stop predicting when the model predicts the *end token*.\n", - "* And store the *attention weights for every time step*.\n", - "\n", - "Note: The encoder output is calculated only once for one input." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EbQpyYs13jF_" - }, - "outputs": [], - "source": [ - "def evaluate(sentence):\n", - " attention_plot = np.zeros((max_length_targ, max_length_inp))\n", - "\n", - " sentence = preprocess_sentence(sentence)\n", - "\n", - " inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n", - " inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n", - " maxlen=max_length_inp,\n", - " padding='post')\n", - " inputs = tf.convert_to_tensor(inputs)\n", - "\n", - " result = ''\n", - "\n", - " hidden = [tf.zeros((1, units))]\n", - " enc_out, enc_hidden = encoder(inputs, hidden)\n", - "\n", - " dec_hidden = enc_hidden\n", - " dec_input = tf.expand_dims([targ_lang.word_index['\u003cstart\u003e']], 0)\n", - "\n", - " for t in range(max_length_targ):\n", - " predictions, dec_hidden, attention_weights = decoder(dec_input,\n", - " dec_hidden,\n", - " enc_out)\n", - "\n", - " # storing the attention weights to plot later on\n", - " attention_weights = tf.reshape(attention_weights, (-1, ))\n", - " attention_plot[t] = attention_weights.numpy()\n", - "\n", - " predicted_id = tf.argmax(predictions[0]).numpy()\n", - "\n", - " result += targ_lang.index_word[predicted_id] + ' '\n", - "\n", - " if targ_lang.index_word[predicted_id] == '\u003cend\u003e':\n", - " return result, sentence, attention_plot\n", - "\n", - " # the predicted ID is fed back into the model\n", - " dec_input = tf.expand_dims([predicted_id], 0)\n", - "\n", - " return result, sentence, attention_plot" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "s5hQWlbN3jGF" - }, - "outputs": [], - "source": [ - "# function for plotting the attention weights\n", - "def plot_attention(attention, sentence, predicted_sentence):\n", - " fig = plt.figure(figsize=(10,10))\n", - " ax = fig.add_subplot(1, 1, 1)\n", - " ax.matshow(attention, cmap='viridis')\n", - "\n", - " fontdict = {'fontsize': 14}\n", - "\n", - " ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n", - " ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n", - "\n", - " ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n", - " ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n", - "\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sl9zUHzg3jGI" - }, - "outputs": [], - "source": [ - "def translate(sentence):\n", - " result, sentence, attention_plot = evaluate(sentence)\n", - "\n", - " print('Input: %s' % (sentence))\n", - " print('Predicted translation: {}'.format(result))\n", - "\n", - " attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n", - " plot_attention(attention_plot, sentence.split(' '), result.split(' '))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n250XbnjOaqP" - }, - "source": [ - "## Restore the latest checkpoint and test" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UJpT9D5_OgP6" - }, - "outputs": [], - "source": [ - "# restoring the latest checkpoint in checkpoint_dir\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WrAM0FDomq3E" - }, - "outputs": [], - "source": [ - "translate(u'hace mucho frio aqui.')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zSx2iM36EZQZ" - }, - "outputs": [], - "source": [ - "translate(u'esta es mi vida.')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "A3LLCx3ZE0Ls" - }, - "outputs": [], - "source": [ - "translate(u'¿todavia estan en casa?')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DUQVLVqUE1YW" - }, - "outputs": [], - "source": [ - "# wrong translation\n", - "translate(u'trata de averiguarlo.')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RTe5P5ioMJwN" - }, - "source": [ - "## Next steps\n", - "\n", - "* [Download a different dataset](http://www.manythings.org/anki/) to experiment with translations, for example, English to German, or English to French.\n", - "* Experiment with training on a larger dataset, or using more epochs\n" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "nmt_with_attention.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/text/text_classification_rnn.ipynb b/site/en/tutorials/text/text_classification_rnn.ipynb deleted file mode 100644 index 285db848c57..00000000000 --- a/site/en/tutorials/text/text_classification_rnn.ipynb +++ /dev/null @@ -1,719 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hX4n9TsbGw-f" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "0nbI5DtDGw-i" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9TnJztDZGw-n" - }, - "source": [ - "# Text classification with an RNN" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AfN3bMR5Gw-o" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/text_classification_rnn\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/text_classification_rnn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/text_classification_rnn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/text/text_classification_rnn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lUWearf0Gw-p" - }, - "source": [ - "This text classification tutorial trains a [recurrent neural network](https://developers.google.com/machine-learning/glossary/#recurrent_neural_network) on the [IMDB large movie review dataset](http://ai.stanford.edu/~amaas/data/sentiment/) for sentiment analysis." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_2VQo4bajwUU" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "z682XYsrjkY9" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1rXHa-w9JZhb" - }, - "source": [ - "Import `matplotlib` and create a helper function to plot graphs:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Mp1Z7P9pYRSK" - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "def plot_graphs(history, metric):\n", - " plt.plot(history.history[metric])\n", - " plt.plot(history.history['val_'+metric], '')\n", - " plt.xlabel(\"Epochs\")\n", - " plt.ylabel(metric)\n", - " plt.legend([metric, 'val_'+metric])\n", - " plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pRmMubr0jrE2" - }, - "source": [ - "## Setup input pipeline\n", - "\n", - "\n", - "The IMDB large movie review dataset is a *binary classification* dataset—all the reviews have either a *positive* or *negative* sentiment.\n", - "\n", - "Download the dataset using [TFDS](https://www.tensorflow.org/datasets).\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SHRwRoP2nVHX" - }, - "outputs": [], - "source": [ - "dataset, info = tfds.load('imdb_reviews/subwords8k', with_info=True,\n", - " as_supervised=True)\n", - "train_dataset, test_dataset = dataset['train'], dataset['test']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MCorLciXSDJE" - }, - "source": [ - " The dataset `info` includes the encoder (a `tfds.features.text.SubwordTextEncoder`)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EplYp5pNnW1S" - }, - "outputs": [], - "source": [ - "encoder = info.features['text'].encoder" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "e7ACuHM5hFp3" - }, - "outputs": [], - "source": [ - "print('Vocabulary size: {}'.format(encoder.vocab_size))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tAfGg8YRe6fu" - }, - "source": [ - "This text encoder will reversibly encode any string, falling back to byte-encoding if necessary." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Bq6xDmf2SAs-" - }, - "outputs": [], - "source": [ - "sample_string = 'Hello TensorFlow.'\n", - "\n", - "encoded_string = encoder.encode(sample_string)\n", - "print('Encoded string is {}'.format(encoded_string))\n", - "\n", - "original_string = encoder.decode(encoded_string)\n", - "print('The original string: \"{}\"'.format(original_string))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TN7QbKaM4-5H" - }, - "outputs": [], - "source": [ - "assert original_string == sample_string" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MDVc6UGO5Dh6" - }, - "outputs": [], - "source": [ - "for index in encoded_string:\n", - " print('{} ----\u003e {}'.format(index, encoder.decode([index])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GlYWqhTVlUyQ" - }, - "source": [ - "## Prepare the data for training" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z2qVJzcEluH_" - }, - "source": [ - "Next create batches of these encoded strings. Use the `padded_batch` method to zero-pad the sequences to the length of the longest string in the batch:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dDsCaZCDYZgm" - }, - "outputs": [], - "source": [ - "BUFFER_SIZE = 10000\n", - "BATCH_SIZE = 64" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VznrltNOnUc5" - }, - "outputs": [], - "source": [ - "train_dataset = train_dataset.shuffle(BUFFER_SIZE)\n", - "train_dataset = train_dataset.padded_batch(BATCH_SIZE, train_dataset.output_shapes)\n", - "\n", - "test_dataset = test_dataset.padded_batch(BATCH_SIZE, test_dataset.output_shapes)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bjUqGVBxGw-t" - }, - "source": [ - "## Create the model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bgs6nnSTGw-t" - }, - "source": [ - "Build a `tf.keras.Sequential` model and start with an embedding layer. An embedding layer stores one vector per word. When called, it converts the sequences of word indices to sequences of vectors. These vectors are trainable. After training (on enough data), words with similar meanings often have similar vectors.\n", - "\n", - "This index-lookup is much more efficient than the equivalent operation of passing a one-hot encoded vector through a `tf.keras.layers.Dense` layer.\n", - "\n", - "A recurrent neural network (RNN) processes sequence input by iterating through the elements. RNNs pass the outputs from one timestep to their input—and then to the next.\n", - "\n", - "The `tf.keras.layers.Bidirectional` wrapper can also be used with an RNN layer. This propagates the input forward and backwards through the RNN layer and then concatenates the output. This helps the RNN to learn long range dependencies." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LwfoBkmRYcP3" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(encoder.vocab_size, 64),\n", - " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", - "])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sRI776ZcH3Tf" - }, - "source": [ - "Compile the Keras model to configure the training process:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kj2xei41YZjC" - }, - "outputs": [], - "source": [ - "model.compile(loss='binary_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(1e-4),\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zIwH3nto596k" - }, - "source": [ - "## Train the model" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hw86wWS4YgR2" - }, - "outputs": [], - "source": [ - "history = model.fit(train_dataset, epochs=10,\n", - " validation_data=test_dataset, \n", - " validation_steps=30)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BaNbXi43YgUT" - }, - "outputs": [], - "source": [ - "test_loss, test_acc = model.evaluate(test_dataset)\n", - "\n", - "print('Test Loss: {}'.format(test_loss))\n", - "print('Test Accuracy: {}'.format(test_acc))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DwSE_386uhxD" - }, - "source": [ - "The above model does not mask the padding applied to the sequences. This can lead to skew if trained on padded sequences and test on un-padded sequences. Ideally you would [use masking](../../guide/keras/masking_and_padding) to avoid this, but as you can see below it only have a small effect on the output.\n", - "\n", - "If the prediction is \u003e= 0.5, it is positive else it is negative." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8w0dseJMiEUh" - }, - "outputs": [], - "source": [ - "def pad_to_size(vec, size):\n", - " zeros = [0] * (size - len(vec))\n", - " vec.extend(zeros)\n", - " return vec" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Y-E4cgkIvmVu" - }, - "outputs": [], - "source": [ - "def sample_predict(sample_pred_text, pad):\n", - " encoded_sample_pred_text = encoder.encode(sample_pred_text)\n", - "\n", - " if pad:\n", - " encoded_sample_pred_text = pad_to_size(encoded_sample_pred_text, 64)\n", - " encoded_sample_pred_text = tf.cast(encoded_sample_pred_text, tf.float32)\n", - " predictions = model.predict(tf.expand_dims(encoded_sample_pred_text, 0))\n", - "\n", - " return (predictions)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "O41gw3KfWHus" - }, - "outputs": [], - "source": [ - "# predict on a sample text without padding.\n", - "\n", - "sample_pred_text = ('The movie was cool. The animation and the graphics '\n", - " 'were out of this world. I would recommend this movie.')\n", - "predictions = sample_predict(sample_pred_text, pad=False)\n", - "print(predictions)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kFh4xLARucTy" - }, - "outputs": [], - "source": [ - "# predict on a sample text with padding\n", - "\n", - "sample_pred_text = ('The movie was cool. The animation and the graphics '\n", - " 'were out of this world. I would recommend this movie.')\n", - "predictions = sample_predict(sample_pred_text, pad=True)\n", - "print(predictions)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ZfIVoxiNmKBF" - }, - "outputs": [], - "source": [ - "plot_graphs(history, 'accuracy')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IUzgkqnhmKD2" - }, - "outputs": [], - "source": [ - "plot_graphs(history, 'loss')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7g1evcaRpTKm" - }, - "source": [ - "## Stack two or more LSTM layers\n", - "\n", - "Keras recurrent layers have two available modes that are controlled by the `return_sequences` constructor argument:\n", - "\n", - "* Return either the full sequences of successive outputs for each timestep (a 3D tensor of shape `(batch_size, timesteps, output_features)`).\n", - "* Return only the last output for each input sequence (a 2D tensor of shape (batch_size, output_features))." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jo1jjO3vn0jo" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(encoder.vocab_size, 64),\n", - " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)),\n", - " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dropout(0.5),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", - "])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hEPV5jVGp-is" - }, - "outputs": [], - "source": [ - "model.compile(loss='binary_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(1e-4),\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LeSE-YjdqAeN" - }, - "outputs": [], - "source": [ - "history = model.fit(train_dataset, epochs=10,\n", - " validation_data=test_dataset,\n", - " validation_steps=30)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_LdwilM1qPM3" - }, - "outputs": [], - "source": [ - "test_loss, test_acc = model.evaluate(test_dataset)\n", - "\n", - "print('Test Loss: {}'.format(test_loss))\n", - "print('Test Accuracy: {}'.format(test_acc))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ykUKnAoqbycW" - }, - "outputs": [], - "source": [ - "# predict on a sample text without padding.\n", - "\n", - "sample_pred_text = ('The movie was not good. The animation and the graphics '\n", - " 'were terrible. I would not recommend this movie.')\n", - "predictions = sample_predict(sample_pred_text, pad=False)\n", - "print(predictions)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2RiC-94zvdZO" - }, - "outputs": [], - "source": [ - "# predict on a sample text with padding\n", - "\n", - "sample_pred_text = ('The movie was not good. The animation and the graphics '\n", - " 'were terrible. I would not recommend this movie.')\n", - "predictions = sample_predict(sample_pred_text, pad=True)\n", - "print(predictions)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_YYub0EDtwCu" - }, - "outputs": [], - "source": [ - "plot_graphs(history, 'accuracy')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DPV3Nn9xtwFM" - }, - "outputs": [], - "source": [ - "plot_graphs(history, 'loss')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9xvpE3BaGw_V" - }, - "source": [ - "Check out other existing recurrent layers such as [GRU layers](https://www.tensorflow.org/api_docs/python/tf/keras/layers/GRU).\n", - "\n", - "If you're interestied in building custom RNNs, see the [Keras RNN Guide](../../guide/keras/rnn.ipynb).\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UQK2Na3Z_aMx" - }, - "outputs": [], - "source": [ - "" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "text_classification_rnn.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/text/text_generation.ipynb b/site/en/tutorials/text/text_generation.ipynb deleted file mode 100644 index 97ccec2d7e9..00000000000 --- a/site/en/tutorials/text/text_generation.ipynb +++ /dev/null @@ -1,1237 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "GCCk8_dHpuNf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ovpZyIhNIgoq" - }, - "source": [ - "# Text generation with an RNN" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hcD2nPQvPOFM" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/text_generation\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/text_generation.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/text_generation.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/text/text_generation.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BwpJ5IffzRG6" - }, - "source": [ - "This tutorial demonstrates how to generate text using a character-based RNN. We will work with a dataset of Shakespeare's writing from Andrej Karpathy's [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/). Given a sequence of characters from this data (\"Shakespear\"), train a model to predict the next character in the sequence (\"e\"). Longer sequences of text can be generated by calling the model repeatedly.\n", - "\n", - "Note: Enable GPU acceleration to execute this notebook faster. In Colab: *Runtime \u003e Change runtime type \u003e Hardware acclerator \u003e GPU*. If running locally make sure TensorFlow version \u003e= 1.11.\n", - "\n", - "This tutorial includes runnable code implemented using [tf.keras](https://www.tensorflow.org/programmers_guide/keras) and [eager execution](https://www.tensorflow.org/programmers_guide/eager). The following is sample output when the model in this tutorial trained for 30 epochs, and started with the string \"Q\":\n", - "\n", - "\u003cpre\u003e\n", - "QUEENE:\n", - "I had thought thou hadst a Roman; for the oracle,\n", - "Thus by All bids the man against the word,\n", - "Which are so weak of care, by old care done;\n", - "Your children were in your holy love,\n", - "And the precipitation through the bleeding throne.\n", - "\n", - "BISHOP OF ELY:\n", - "Marry, and will, my lord, to weep in such a one were prettiest;\n", - "Yet now I was adopted heir\n", - "Of the world's lamentable day,\n", - "To watch the next way with his father with his face?\n", - "\n", - "ESCALUS:\n", - "The cause why then we are all resolved more sons.\n", - "\n", - "VOLUMNIA:\n", - "O, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, it is no sin it should be dead,\n", - "And love and pale as any will to that word.\n", - "\n", - "QUEEN ELIZABETH:\n", - "But how long have I heard the soul for this world,\n", - "And show his hands of life be proved to stand.\n", - "\n", - "PETRUCHIO:\n", - "I say he look'd on, if I must be content\n", - "To stay him from the fatal of our country's bliss.\n", - "His lordship pluck'd from this sentence then for prey,\n", - "And then let us twain, being the moon,\n", - "were she such a case as fills m\n", - "\u003c/pre\u003e\n", - "\n", - "While some of the sentences are grammatical, most do not make sense. The model has not learned the meaning of words, but consider:\n", - "\n", - "* The model is character-based. When training started, the model did not know how to spell an English word, or that words were even a unit of text.\n", - "\n", - "* The structure of the output resembles a play—blocks of text generally begin with a speaker name, in all capital letters similar to the dataset.\n", - "\n", - "* As demonstrated below, the model is trained on small batches of text (100 characters each), and is still able to generate a longer sequence of text with coherent structure." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "srXC6pLGLwS6" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WGyKZj3bzf9p" - }, - "source": [ - "### Import TensorFlow and other libraries" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yG_n40gFzf9s" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import numpy as np\n", - "import os\n", - "import time" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EHDoRoc5PKWz" - }, - "source": [ - "### Download the Shakespeare dataset\n", - "\n", - "Change the following line to run this code on your own data." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "pD_55cOxLkAb" - }, - "outputs": [], - "source": [ - "path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UHjdCjDuSvX_" - }, - "source": [ - "### Read the data\n", - "\n", - "First, look in the text:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aavnuByVymwK" - }, - "outputs": [], - "source": [ - "# Read, then decode for py2 compat.\n", - "text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n", - "# length of text is the number of characters in it\n", - "print ('Length of text: {} characters'.format(len(text)))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Duhg9NrUymwO" - }, - "outputs": [], - "source": [ - "# Take a look at the first 250 characters in text\n", - "print(text[:250])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IlCgQBRVymwR" - }, - "outputs": [], - "source": [ - "# The unique characters in the file\n", - "vocab = sorted(set(text))\n", - "print ('{} unique characters'.format(len(vocab)))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rNnrKn_lL-IJ" - }, - "source": [ - "## Process the text" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LFjSVAlWzf-N" - }, - "source": [ - "### Vectorize the text\n", - "\n", - "Before training, we need to map strings to a numerical representation. Create two lookup tables: one mapping characters to numbers, and another for numbers to characters." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IalZLbvOzf-F" - }, - "outputs": [], - "source": [ - "# Creating a mapping from unique characters to indices\n", - "char2idx = {u:i for i, u in enumerate(vocab)}\n", - "idx2char = np.array(vocab)\n", - "\n", - "text_as_int = np.array([char2idx[c] for c in text])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tZfqhkYCymwX" - }, - "source": [ - "Now we have an integer representation for each character. Notice that we mapped the character as indexes from 0 to `len(unique)`." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FYyNlCNXymwY" - }, - "outputs": [], - "source": [ - "print('{')\n", - "for char,_ in zip(char2idx, range(20)):\n", - " print(' {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n", - "print(' ...\\n}')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l1VKcQHcymwb" - }, - "outputs": [], - "source": [ - "# Show how the first 13 characters from the text are mapped to integers\n", - "print ('{} ---- characters mapped to int ---- \u003e {}'.format(repr(text[:13]), text_as_int[:13]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bbmsf23Bymwe" - }, - "source": [ - "### The prediction task" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wssHQ1oGymwe" - }, - "source": [ - "Given a character, or a sequence of characters, what is the most probable next character? This is the task we're training the model to perform. The input to the model will be a sequence of characters, and we train the model to predict the output—the following character at each time step.\n", - "\n", - "Since RNNs maintain an internal state that depends on the previously seen elements, given all the characters computed until this moment, what is the next character?\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hgsVvVxnymwf" - }, - "source": [ - "### Create training examples and targets\n", - "\n", - "Next divide the text into example sequences. Each input sequence will contain `seq_length` characters from the text.\n", - "\n", - "For each input sequence, the corresponding targets contain the same length of text, except shifted one character to the right.\n", - "\n", - "So break the text into chunks of `seq_length+1`. For example, say `seq_length` is 4 and our text is \"Hello\". The input sequence would be \"Hell\", and the target sequence \"ello\".\n", - "\n", - "To do this first use the `tf.data.Dataset.from_tensor_slices` function to convert the text vector into a stream of character indices." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0UHJDA39zf-O" - }, - "outputs": [], - "source": [ - "# The maximum length sentence we want for a single input in characters\n", - "seq_length = 100\n", - "examples_per_epoch = len(text)//(seq_length+1)\n", - "\n", - "# Create training examples / targets\n", - "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n", - "\n", - "for i in char_dataset.take(5):\n", - " print(idx2char[i.numpy()])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ZSYAcQV8OGP" - }, - "source": [ - "The `batch` method lets us easily convert these individual characters to sequences of the desired size." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l4hkDU3i7ozi" - }, - "outputs": [], - "source": [ - "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n", - "\n", - "for item in sequences.take(5):\n", - " print(repr(''.join(idx2char[item.numpy()])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UbLcIPBj_mWZ" - }, - "source": [ - "For each sequence, duplicate and shift it to form the input and target text by using the `map` method to apply a simple function to each batch:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9NGu-FkO_kYU" - }, - "outputs": [], - "source": [ - "def split_input_target(chunk):\n", - " input_text = chunk[:-1]\n", - " target_text = chunk[1:]\n", - " return input_text, target_text\n", - "\n", - "dataset = sequences.map(split_input_target)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiCopyGZymwi" - }, - "source": [ - "Print the first examples input and target values:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GNbw-iR0ymwj" - }, - "outputs": [], - "source": [ - "for input_example, target_example in dataset.take(1):\n", - " print ('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n", - " print ('Target data:', repr(''.join(idx2char[target_example.numpy()])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_33OHL3b84i0" - }, - "source": [ - "Each index of these vectors are processed as one time step. For the input at time step 0, the model receives the index for \"F\" and trys to predict the index for \"i\" as the next character. At the next timestep, it does the same thing but the `RNN` considers the previous step context in addition to the current input character." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0eBu9WZG84i0" - }, - "outputs": [], - "source": [ - "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n", - " print(\"Step {:4d}\".format(i))\n", - " print(\" input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n", - " print(\" expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MJdfPmdqzf-R" - }, - "source": [ - "### Create training batches\n", - "\n", - "We used `tf.data` to split the text into manageable sequences. But before feeding this data into the model, we need to shuffle the data and pack it into batches." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "p2pGotuNzf-S" - }, - "outputs": [], - "source": [ - "# Batch size\n", - "BATCH_SIZE = 64\n", - "\n", - "# Buffer size to shuffle the dataset\n", - "# (TF data is designed to work with possibly infinite sequences,\n", - "# so it doesn't attempt to shuffle the entire sequence in memory. Instead,\n", - "# it maintains a buffer in which it shuffles elements).\n", - "BUFFER_SIZE = 10000\n", - "\n", - "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n", - "\n", - "dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r6oUuElIMgVx" - }, - "source": [ - "## Build The Model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m8gPwEjRzf-Z" - }, - "source": [ - "Use `tf.keras.Sequential` to define the model. For this simple example three layers are used to define our model:\n", - "\n", - "* `tf.keras.layers.Embedding`: The input layer. A trainable lookup table that will map the numbers of each character to a vector with `embedding_dim` dimensions;\n", - "* `tf.keras.layers.GRU`: A type of RNN with size `units=rnn_units` (You can also use a LSTM layer here.)\n", - "* `tf.keras.layers.Dense`: The output layer, with `vocab_size` outputs." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zHT8cLh7EAsg" - }, - "outputs": [], - "source": [ - "# Length of the vocabulary in chars\n", - "vocab_size = len(vocab)\n", - "\n", - "# The embedding dimension\n", - "embedding_dim = 256\n", - "\n", - "# Number of RNN units\n", - "rnn_units = 1024" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MtCrdfzEI2N0" - }, - "outputs": [], - "source": [ - "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(vocab_size, embedding_dim,\n", - " batch_input_shape=[batch_size, None]),\n", - " tf.keras.layers.GRU(rnn_units,\n", - " return_sequences=True,\n", - " stateful=True,\n", - " recurrent_initializer='glorot_uniform'),\n", - " tf.keras.layers.Dense(vocab_size)\n", - " ])\n", - " return model" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wwsrpOik5zhv" - }, - "outputs": [], - "source": [ - "model = build_model(\n", - " vocab_size = len(vocab),\n", - " embedding_dim=embedding_dim,\n", - " rnn_units=rnn_units,\n", - " batch_size=BATCH_SIZE)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RkA5upJIJ7W7" - }, - "source": [ - "For each character the model looks up the embedding, runs the GRU one timestep with the embedding as input, and applies the dense layer to generate logits predicting the log-likelihood of the next character:\n", - "\n", - "![A drawing of the data passing through the model](images/text_generation_training.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ubPo0_9Prjb" - }, - "source": [ - "## Try the model\n", - "\n", - "Now run the model to see that it behaves as expected.\n", - "\n", - "First check the shape of the output:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "C-_70kKAPrPU" - }, - "outputs": [], - "source": [ - "for input_example_batch, target_example_batch in dataset.take(1):\n", - " example_batch_predictions = model(input_example_batch)\n", - " print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q6NzLBi4VM4o" - }, - "source": [ - "In the above example the sequence length of the input is `100` but the model can be run on inputs of any length:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vPGmAAXmVLGC" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uwv0gEkURfx1" - }, - "source": [ - "To get actual predictions from the model we need to sample from the output distribution, to get actual character indices. This distribution is defined by the logits over the character vocabulary.\n", - "\n", - "Note: It is important to _sample_ from this distribution as taking the _argmax_ of the distribution can easily get the model stuck in a loop.\n", - "\n", - "Try it for the first example in the batch:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4V4MfFg0RQJg" - }, - "outputs": [], - "source": [ - "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n", - "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QM1Vbxs_URw5" - }, - "source": [ - "This gives us, at each timestep, a prediction of the next character index:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YqFMUQc_UFgM" - }, - "outputs": [], - "source": [ - "sampled_indices" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LfLtsP3mUhCG" - }, - "source": [ - "Decode these to see the text predicted by this untrained model:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xWcFwPwLSo05" - }, - "outputs": [], - "source": [ - "print(\"Input: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n", - "print()\n", - "print(\"Next Char Predictions: \\n\", repr(\"\".join(idx2char[sampled_indices ])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LJL0Q0YPY6Ee" - }, - "source": [ - "## Train the model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YCbHQHiaa4Ic" - }, - "source": [ - "At this point the problem can be treated as a standard classification problem. Given the previous RNN state, and the input this time step, predict the class of the next character." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "trpqTWyvk0nr" - }, - "source": [ - "### Attach an optimizer, and a loss function" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UAjbjY03eiQ4" - }, - "source": [ - "The standard `tf.keras.losses.sparse_categorical_crossentropy` loss function works in this case because it is applied across the last dimension of the predictions.\n", - "\n", - "Because our model returns logits, we need to set the `from_logits` flag.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4HrXTACTdzY-" - }, - "outputs": [], - "source": [ - "def loss(labels, logits):\n", - " return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n", - "\n", - "example_batch_loss = loss(target_example_batch, example_batch_predictions)\n", - "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n", - "print(\"scalar_loss: \", example_batch_loss.numpy().mean())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jeOXriLcymww" - }, - "source": [ - "Configure the training procedure using the `tf.keras.Model.compile` method. We'll use `tf.keras.optimizers.Adam` with default arguments and the loss function." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DDl1_Een6rL0" - }, - "outputs": [], - "source": [ - "model.compile(optimizer='adam', loss=loss)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ieSJdchZggUj" - }, - "source": [ - "### Configure checkpoints" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C6XBUUavgF56" - }, - "source": [ - "Use a `tf.keras.callbacks.ModelCheckpoint` to ensure that checkpoints are saved during training:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "W6fWTriUZP-n" - }, - "outputs": [], - "source": [ - "# Directory where the checkpoints will be saved\n", - "checkpoint_dir = './training_checkpoints'\n", - "# Name of the checkpoint files\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n", - "\n", - "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n", - " filepath=checkpoint_prefix,\n", - " save_weights_only=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Ky3F_BhgkTW" - }, - "source": [ - "### Execute the training" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IxdOA-rgyGvs" - }, - "source": [ - "To keep training time reasonable, use 10 epochs to train the model. In Colab, set the runtime to GPU for faster training." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7yGBE2zxMMHs" - }, - "outputs": [], - "source": [ - "EPOCHS=10" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UK-hmKjYVoll" - }, - "outputs": [], - "source": [ - "history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKkD5M6eoSiN" - }, - "source": [ - "## Generate text" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JIPcXllKjkdr" - }, - "source": [ - "### Restore the latest checkpoint" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LyeYRiuVjodY" - }, - "source": [ - "To keep this prediction step simple, use a batch size of 1.\n", - "\n", - "Because of the way the RNN state is passed from timestep to timestep, the model only accepts a fixed batch size once built.\n", - "\n", - "To run the model with a different `batch_size`, we need to rebuild the model and restore the weights from the checkpoint.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zk2WJ2-XjkGz" - }, - "outputs": [], - "source": [ - "tf.train.latest_checkpoint(checkpoint_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LycQ-ot_jjyu" - }, - "outputs": [], - "source": [ - "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n", - "\n", - "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "model.build(tf.TensorShape([1, None]))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "71xa6jnYVrAN" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DjGz1tDkzf-u" - }, - "source": [ - "### The prediction loop\n", - "\n", - "The following code block generates the text:\n", - "\n", - "* It Starts by choosing a start string, initializing the RNN state and setting the number of characters to generate.\n", - "\n", - "* Get the prediction distribution of the next character using the start string and the RNN state.\n", - "\n", - "* Then, use a categorical distribution to calculate the index of the predicted character. Use this predicted character as our next input to the model.\n", - "\n", - "* The RNN state returned by the model is fed back into the model so that it now has more context, instead than only one word. After predicting the next word, the modified RNN states are again fed back into the model, which is how it learns as it gets more context from the previously predicted words.\n", - "\n", - "\n", - "![To generate text the model's output is fed back to the input](images/text_generation_sampling.png)\n", - "\n", - "Looking at the generated text, you'll see the model knows when to capitalize, make paragraphs and imitates a Shakespeare-like writing vocabulary. With the small number of training epochs, it has not yet learned to form coherent sentences." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WvuwZBX5Ogfd" - }, - "outputs": [], - "source": [ - "def generate_text(model, start_string):\n", - " # Evaluation step (generating text using the learned model)\n", - "\n", - " # Number of characters to generate\n", - " num_generate = 1000\n", - "\n", - " # Converting our start string to numbers (vectorizing)\n", - " input_eval = [char2idx[s] for s in start_string]\n", - " input_eval = tf.expand_dims(input_eval, 0)\n", - "\n", - " # Empty string to store our results\n", - " text_generated = []\n", - "\n", - " # Low temperatures results in more predictable text.\n", - " # Higher temperatures results in more surprising text.\n", - " # Experiment to find the best setting.\n", - " temperature = 1.0\n", - "\n", - " # Here batch size == 1\n", - " model.reset_states()\n", - " for i in range(num_generate):\n", - " predictions = model(input_eval)\n", - " # remove the batch dimension\n", - " predictions = tf.squeeze(predictions, 0)\n", - "\n", - " # using a categorical distribution to predict the word returned by the model\n", - " predictions = predictions / temperature\n", - " predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n", - "\n", - " # We pass the predicted word as the next input to the model\n", - " # along with the previous hidden state\n", - " input_eval = tf.expand_dims([predicted_id], 0)\n", - "\n", - " text_generated.append(idx2char[predicted_id])\n", - "\n", - " return (start_string + ''.join(text_generated))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ktovv0RFhrkn" - }, - "outputs": [], - "source": [ - "print(generate_text(model, start_string=u\"ROMEO: \"))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AM2Uma_-yVIq" - }, - "source": [ - "The easiest thing you can do to improve the results it to train it for longer (try `EPOCHS=30`).\n", - "\n", - "You can also experiment with a different start string, or try adding another RNN layer to improve the model's accuracy, or adjusting the temperature parameter to generate more or less random predictions." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y4QwTjAM6A2O" - }, - "source": [ - "## Advanced: Customized Training\n", - "\n", - "The above training procedure is simple, but does not give you much control.\n", - "\n", - "So now that you've seen how to run the model manually let's unpack the training loop, and implement it ourselves. This gives a starting point if, for example, to implement _curriculum learning_ to help stabilize the model's open-loop output.\n", - "\n", - "We will use `tf.GradientTape` to track the gradients. You can learn more about this approach by reading the [eager execution guide](https://www.tensorflow.org/guide/eager).\n", - "\n", - "The procedure works as follows:\n", - "\n", - "* First, initialize the RNN state. We do this by calling the `tf.keras.Model.reset_states` method.\n", - "\n", - "* Next, iterate over the dataset (batch by batch) and calculate the *predictions* associated with each.\n", - "\n", - "* Open a `tf.GradientTape`, and calculate the predictions and loss in that context.\n", - "\n", - "* Calculate the gradients of the loss with respect to the model variables using the `tf.GradientTape.grads` method.\n", - "\n", - "* Finally, take a step downwards by using the optimizer's `tf.train.Optimizer.apply_gradients` method.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_XAm7eCoKULT" - }, - "outputs": [], - "source": [ - "model = build_model(\n", - " vocab_size = len(vocab),\n", - " embedding_dim=embedding_dim,\n", - " rnn_units=rnn_units,\n", - " batch_size=BATCH_SIZE)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qUKhnZtMVpoJ" - }, - "outputs": [], - "source": [ - "optimizer = tf.keras.optimizers.Adam()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "b4kH1o0leVIp" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def train_step(inp, target):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(inp)\n", - " loss = tf.reduce_mean(\n", - " tf.keras.losses.sparse_categorical_crossentropy(\n", - " target, predictions, from_logits=True))\n", - " grads = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " return loss" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "d4tSNwymzf-q" - }, - "outputs": [], - "source": [ - "# Training step\n", - "EPOCHS = 10\n", - "\n", - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - "\n", - " # initializing the hidden state at the start of every epoch\n", - " # initally hidden is None\n", - " hidden = model.reset_states()\n", - "\n", - " for (batch_n, (inp, target)) in enumerate(dataset):\n", - " loss = train_step(inp, target)\n", - "\n", - " if batch_n % 100 == 0:\n", - " template = 'Epoch {} Batch {} Loss {}'\n", - " print(template.format(epoch+1, batch_n, loss))\n", - "\n", - " # saving (checkpoint) the model every 5 epochs\n", - " if (epoch + 1) % 5 == 0:\n", - " model.save_weights(checkpoint_prefix.format(epoch=epoch))\n", - "\n", - " print ('Epoch {} Loss {:.4f}'.format(epoch+1, loss))\n", - " print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n", - "\n", - "model.save_weights(checkpoint_prefix.format(epoch=epoch))" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "text_generation.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/text/transformer.ipynb b/site/en/tutorials/text/transformer.ipynb deleted file mode 100644 index e9a5f7e8a8c..00000000000 --- a/site/en/tutorials/text/transformer.ipynb +++ /dev/null @@ -1,2050 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "s_qNSzzyaCbD" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "jmjh290raIky" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J0Qjg6vuaHNt" - }, - "source": [ - "# Transformer model for language understanding" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AOpGoE2T-YXS" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/transformer\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/transformer.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/transformer.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/text/transformer.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M-f8TnGpE_ex" - }, - "source": [ - "This tutorial trains a \u003ca href=\"https://arxiv.org/abs/1706.03762\" class=\"external\"\u003eTransformer model\u003c/a\u003e to translate Portuguese to English. This is an advanced example that assumes knowledge of [text generation](text_generation.ipynb) and [attention](nmt_with_attention.ipynb).\n", - "\n", - "The core idea behind the Transformer model is *self-attention*—the ability to attend to different positions of the input sequence to compute a representation of that sequence. Transformer creates stacks of self-attention layers and is explained below in the sections *Scaled dot product attention* and *Multi-head attention*.\n", - "\n", - "A transformer model handles variable-sized input using stacks of self-attention layers instead of [RNNs](text_classification_rnn.ipynb) or [CNNs](../images/intro_to_cnns.ipynb). This general architecture has a number of advantages:\n", - "\n", - "* It make no assumptions about the temporal/spatial relationships across the data. This is ideal for processing a set of objects (for example, [StarCraft units](https://deepmind.com/blog/alphastar-mastering-real-time-strategy-game-starcraft-ii/#block-8)).\n", - "* Layer outputs can be calculated in parallel, instead of a series like an RNN.\n", - "* Distant items can affect each other's output without passing through many RNN-steps, or convolution layers (see [Scene Memory Transformer](https://arxiv.org/pdf/1903.03878.pdf) for example).\n", - "* It can learn long-range dependencies. This is a challenge in many sequence tasks.\n", - "\n", - "The downsides of this architecture are:\n", - "\n", - "* For a time-series, the output for a time-step is calculated from the *entire history* instead of only the inputs and current hidden-state. This _may_ be less efficient. \n", - "* If the input *does* have a temporal/spatial relationship, like text, some positional encoding must be added or the model will effectively see a bag of words. \n", - "\n", - "After training the model in this notebook, you will be able to input a Portuguese sentence and return the English translation.\n", - "\n", - "\u003cimg src=\"https://www.tensorflow.org/images/tutorials/transformer/attention_map_portuguese.png\" width=\"800\" alt=\"Attention heatmap\"\u003e" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JjJJyJTZYebt" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "\n", - "import time\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fd1NWMxjfsDd" - }, - "source": [ - "## Setup input pipeline" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t4_Qt8W1hJE_" - }, - "source": [ - "Use [TFDS](https://www.tensorflow.org/datasets) to load the [Portugese-English translation dataset](https://github.com/neulab/word-embeddings-for-nmt) from the [TED Talks Open Translation Project](https://www.ted.com/participate/translate).\n", - "\n", - "This dataset contains approximately 50000 training examples, 1100 validation examples, and 2000 test examples." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8q9t4FmN96eN" - }, - "outputs": [], - "source": [ - "examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True,\n", - " as_supervised=True)\n", - "train_examples, val_examples = examples['train'], examples['validation']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RCEKotqosGfq" - }, - "source": [ - "Create a custom subwords tokenizer from the training dataset. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KVBg5Q8tBk5z" - }, - "outputs": [], - "source": [ - "tokenizer_en = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n", - " (en.numpy() for pt, en in train_examples), target_vocab_size=2**13)\n", - "\n", - "tokenizer_pt = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n", - " (pt.numpy() for pt, en in train_examples), target_vocab_size=2**13)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4DYWukNFkGQN" - }, - "outputs": [], - "source": [ - "sample_string = 'Transformer is awesome.'\n", - "\n", - "tokenized_string = tokenizer_en.encode(sample_string)\n", - "print ('Tokenized string is {}'.format(tokenized_string))\n", - "\n", - "original_string = tokenizer_en.decode(tokenized_string)\n", - "print ('The original string: {}'.format(original_string))\n", - "\n", - "assert original_string == sample_string" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o9KJWJjrsZ4Y" - }, - "source": [ - "The tokenizer encodes the string by breaking it into subwords if the word is not in its dictionary." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bf2ntBxjkqK6" - }, - "outputs": [], - "source": [ - "for ts in tokenized_string:\n", - " print ('{} ----\u003e {}'.format(ts, tokenizer_en.decode([ts])))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bcRp7VcQ5m6g" - }, - "outputs": [], - "source": [ - "BUFFER_SIZE = 20000\n", - "BATCH_SIZE = 64" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kGi4PoVakxdc" - }, - "source": [ - "Add a start and end token to the input and target. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UZwnPr4R055s" - }, - "outputs": [], - "source": [ - "def encode(lang1, lang2):\n", - " lang1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode(\n", - " lang1.numpy()) + [tokenizer_pt.vocab_size+1]\n", - "\n", - " lang2 = [tokenizer_en.vocab_size] + tokenizer_en.encode(\n", - " lang2.numpy()) + [tokenizer_en.vocab_size+1]\n", - " \n", - " return lang1, lang2" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6JrGp5Gek6Ql" - }, - "source": [ - "Note: To keep this example small and relatively fast, drop examples with a length of over 40 tokens." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2QEgbjntk6Yf" - }, - "outputs": [], - "source": [ - "MAX_LENGTH = 40" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "c081xPGv1CPI" - }, - "outputs": [], - "source": [ - "def filter_max_length(x, y, max_length=MAX_LENGTH):\n", - " return tf.logical_and(tf.size(x) \u003c= max_length,\n", - " tf.size(y) \u003c= max_length)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tx1sFbR-9fRs" - }, - "source": [ - "Operations inside `.map()` run in graph mode and receive a graph tensor that do not have a numpy attribute. The `tokenizer` expects a string or Unicode symbol to encode it into integers. Hence, you need to run the encoding inside a `tf.py_function`, which receives an eager tensor having a numpy attribute that contains the string value." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Mah1cS-P70Iz" - }, - "outputs": [], - "source": [ - "def tf_encode(pt, en):\n", - " return tf.py_function(encode, [pt, en], [tf.int64, tf.int64])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9mk9AZdZ5bcS" - }, - "outputs": [], - "source": [ - "train_dataset = train_examples.map(tf_encode)\n", - "train_dataset = train_dataset.filter(filter_max_length)\n", - "# cache the dataset to memory to get a speedup while reading from it.\n", - "train_dataset = train_dataset.cache()\n", - "train_dataset = train_dataset.shuffle(BUFFER_SIZE).padded_batch(\n", - " BATCH_SIZE, padded_shapes=([-1], [-1]))\n", - "train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)\n", - "\n", - "\n", - "val_dataset = val_examples.map(tf_encode)\n", - "val_dataset = val_dataset.filter(filter_max_length).padded_batch(\n", - " BATCH_SIZE, padded_shapes=([-1], [-1]))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_fXvfYVfQr2n" - }, - "outputs": [], - "source": [ - "pt_batch, en_batch = next(iter(val_dataset))\n", - "pt_batch, en_batch" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nBQuibYA4n0n" - }, - "source": [ - "## Positional encoding\n", - "\n", - "Since this model doesn't contain any recurrence or convolution, positional encoding is added to give the model some information about the relative position of the words in the sentence. \n", - "\n", - "The positional encoding vector is added to the embedding vector. Embeddings represent a token in a d-dimensional space where tokens with similar meaning will be closer to each other. But the embeddings do not encode the relative position of words in a sentence. So after adding the positional encoding, words will be closer to each other based on the *similarity of their meaning and their position in the sentence*, in the d-dimensional space.\n", - "\n", - "See the notebook on [positional encoding](https://github.com/tensorflow/examples/blob/master/community/en/position_encoding.ipynb) to learn more about it. The formula for calculating the positional encoding is as follows:\n", - "\n", - "$$\\Large{PE_{(pos, 2i)} = sin(pos / 10000^{2i / d_{model}})} $$\n", - "$$\\Large{PE_{(pos, 2i+1)} = cos(pos / 10000^{2i / d_{model}})} $$" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WhIOZjMNKujn" - }, - "outputs": [], - "source": [ - "def get_angles(pos, i, d_model):\n", - " angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n", - " return pos * angle_rates" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1Rz82wEs5biZ" - }, - "outputs": [], - "source": [ - "def positional_encoding(position, d_model):\n", - " angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n", - " np.arange(d_model)[np.newaxis, :],\n", - " d_model)\n", - " \n", - " # apply sin to even indices in the array; 2i\n", - " angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n", - " \n", - " # apply cos to odd indices in the array; 2i+1\n", - " angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n", - " \n", - " pos_encoding = angle_rads[np.newaxis, ...]\n", - " \n", - " return tf.cast(pos_encoding, dtype=tf.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1kLCla68EloE" - }, - "outputs": [], - "source": [ - "pos_encoding = positional_encoding(50, 512)\n", - "print (pos_encoding.shape)\n", - "\n", - "plt.pcolormesh(pos_encoding[0], cmap='RdBu')\n", - "plt.xlabel('Depth')\n", - "plt.xlim((0, 512))\n", - "plt.ylabel('Position')\n", - "plt.colorbar()\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "a_b4ou4TYqUN" - }, - "source": [ - "## Masking" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "s42Uydjkv0hF" - }, - "source": [ - "Mask all the pad tokens in the batch of sequence. It ensures that the model does not treat padding as the input. The mask indicates where pad value `0` is present: it outputs a `1` at those locations, and a `0` otherwise." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U2i8-e1s8ti9" - }, - "outputs": [], - "source": [ - "def create_padding_mask(seq):\n", - " seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n", - " \n", - " # add extra dimensions to add the padding\n", - " # to the attention logits.\n", - " return seq[:, tf.newaxis, tf.newaxis, :] # (batch_size, 1, 1, seq_len)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "A7BYeBCNvi7n" - }, - "outputs": [], - "source": [ - "x = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])\n", - "create_padding_mask(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z0hzukDBgVom" - }, - "source": [ - "The look-ahead mask is used to mask the future tokens in a sequence. In other words, the mask indicates which entries should not be used.\n", - "\n", - "This means that to predict the third word, only the first and second word will be used. Similarly to predict the fourth word, only the first, second and the third word will be used and so on." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dVxS8OPI9uI0" - }, - "outputs": [], - "source": [ - "def create_look_ahead_mask(size):\n", - " mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n", - " return mask # (seq_len, seq_len)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yxKGuXxaBeeE" - }, - "outputs": [], - "source": [ - "x = tf.random.uniform((1, 3))\n", - "temp = create_look_ahead_mask(x.shape[1])\n", - "temp" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xluDl5cXYy4y" - }, - "source": [ - "## Scaled dot product attention" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vsxEE_-Wa1gF" - }, - "source": [ - "\u003cimg src=\"https://www.tensorflow.org/images/tutorials/transformer/scaled_attention.png\" width=\"500\" alt=\"scaled_dot_product_attention\"\u003e\n", - "\n", - "The attention function used by the transformer takes three inputs: Q (query), K (key), V (value). The equation used to calculate the attention weights is:\n", - "\n", - "$$\\Large{Attention(Q, K, V) = softmax_k(\\frac{QK^T}{\\sqrt{d_k}}) V} $$\n", - "\n", - "The dot-product attention is scaled by a factor of square root of the depth. This is done because for large values of depth, the dot product grows large in magnitude pushing the softmax function where it has small gradients resulting in a very hard softmax. \n", - "\n", - "For example, consider that `Q` and `K` have a mean of 0 and variance of 1. Their matrix multiplication will have a mean of 0 and variance of `dk`. Hence, *square root of `dk`* is used for scaling (and not any other number) because the matmul of `Q` and `K` should have a mean of 0 and variance of 1, and you get a gentler softmax.\n", - "\n", - "The mask is multiplied with -1e9 (close to negative infinity). This is done because the mask is summed with the scaled matrix multiplication of Q and K and is applied immediately before a softmax. The goal is to zero out these cells, and large negative inputs to softmax are near zero in the output." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LazzUq3bJ5SH" - }, - "outputs": [], - "source": [ - "def scaled_dot_product_attention(q, k, v, mask):\n", - " \"\"\"Calculate the attention weights.\n", - " q, k, v must have matching leading dimensions.\n", - " k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n", - " The mask has different shapes depending on its type(padding or look ahead) \n", - " but it must be broadcastable for addition.\n", - " \n", - " Args:\n", - " q: query shape == (..., seq_len_q, depth)\n", - " k: key shape == (..., seq_len_k, depth)\n", - " v: value shape == (..., seq_len_v, depth_v)\n", - " mask: Float tensor with shape broadcastable \n", - " to (..., seq_len_q, seq_len_k). Defaults to None.\n", - " \n", - " Returns:\n", - " output, attention_weights\n", - " \"\"\"\n", - "\n", - " matmul_qk = tf.matmul(q, k, transpose_b=True) # (..., seq_len_q, seq_len_k)\n", - " \n", - " # scale matmul_qk\n", - " dk = tf.cast(tf.shape(k)[-1], tf.float32)\n", - " scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n", - "\n", - " # add the mask to the scaled tensor.\n", - " if mask is not None:\n", - " scaled_attention_logits += (mask * -1e9) \n", - "\n", - " # softmax is normalized on the last axis (seq_len_k) so that the scores\n", - " # add up to 1.\n", - " attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) # (..., seq_len_q, seq_len_k)\n", - "\n", - " output = tf.matmul(attention_weights, v) # (..., seq_len_q, depth_v)\n", - "\n", - " return output, attention_weights" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FiqETnhCkoXh" - }, - "source": [ - "As the softmax normalization is done on K, its values decide the amount of importance given to Q.\n", - "\n", - "The output represents the multiplication of the attention weights and the V (value) vector. This ensures that the words you want to focus on are kept as-is and the irrelevant words are flushed out." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "n90YjClyInFy" - }, - "outputs": [], - "source": [ - "def print_out(q, k, v):\n", - " temp_out, temp_attn = scaled_dot_product_attention(\n", - " q, k, v, None)\n", - " print ('Attention weights are:')\n", - " print (temp_attn)\n", - " print ('Output is:')\n", - " print (temp_out)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yAzUAf2DPlNt" - }, - "outputs": [], - "source": [ - "np.set_printoptions(suppress=True)\n", - "\n", - "temp_k = tf.constant([[10,0,0],\n", - " [0,10,0],\n", - " [0,0,10],\n", - " [0,0,10]], dtype=tf.float32) # (4, 3)\n", - "\n", - "temp_v = tf.constant([[ 1,0],\n", - " [ 10,0],\n", - " [ 100,5],\n", - " [1000,6]], dtype=tf.float32) # (4, 2)\n", - "\n", - "# This `query` aligns with the second `key`,\n", - "# so the second `value` is returned.\n", - "temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32) # (1, 3)\n", - "print_out(temp_q, temp_k, temp_v)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zg6k-fGhgXra" - }, - "outputs": [], - "source": [ - "# This query aligns with a repeated key (third and fourth), \n", - "# so all associated values get averaged.\n", - "temp_q = tf.constant([[0, 0, 10]], dtype=tf.float32) # (1, 3)\n", - "print_out(temp_q, temp_k, temp_v)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UAq3YOzUgXhb" - }, - "outputs": [], - "source": [ - "# This query aligns equally with the first and second key, \n", - "# so their values get averaged.\n", - "temp_q = tf.constant([[10, 10, 0]], dtype=tf.float32) # (1, 3)\n", - "print_out(temp_q, temp_k, temp_v)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aOz-4_XIhaTP" - }, - "source": [ - "Pass all the queries together." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6dlU8Tm-hYrF" - }, - "outputs": [], - "source": [ - "temp_q = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32) # (3, 3)\n", - "print_out(temp_q, temp_k, temp_v)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kmzGPEy64qmA" - }, - "source": [ - "## Multi-head attention" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fz5BMC8Kaoqo" - }, - "source": [ - "\u003cimg src=\"https://www.tensorflow.org/images/tutorials/transformer/multi_head_attention.png\" width=\"500\" alt=\"multi-head attention\"\u003e\n", - "\n", - "\n", - "Multi-head attention consists of four parts:\n", - "* Linear layers and split into heads.\n", - "* Scaled dot-product attention.\n", - "* Concatenation of heads.\n", - "* Final linear layer." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JPmbr6F1C-v_" - }, - "source": [ - "Each multi-head attention block gets three inputs; Q (query), K (key), V (value). These are put through linear (Dense) layers and split up into multiple heads. \n", - "\n", - "The `scaled_dot_product_attention` defined above is applied to each head (broadcasted for efficiency). An appropriate mask must be used in the attention step. The attention output for each head is then concatenated (using `tf.transpose`, and `tf.reshape`) and put through a final `Dense` layer.\n", - "\n", - "Instead of one single attention head, Q, K, and V are split into multiple heads because it allows the model to jointly attend to information at different positions from different representational spaces. After the split each head has a reduced dimensionality, so the total computation cost is the same as a single head attention with full dimensionality." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BSV3PPKsYecw" - }, - "outputs": [], - "source": [ - "class MultiHeadAttention(tf.keras.layers.Layer):\n", - " def __init__(self, d_model, num_heads):\n", - " super(MultiHeadAttention, self).__init__()\n", - " self.num_heads = num_heads\n", - " self.d_model = d_model\n", - " \n", - " assert d_model % self.num_heads == 0\n", - " \n", - " self.depth = d_model // self.num_heads\n", - " \n", - " self.wq = tf.keras.layers.Dense(d_model)\n", - " self.wk = tf.keras.layers.Dense(d_model)\n", - " self.wv = tf.keras.layers.Dense(d_model)\n", - " \n", - " self.dense = tf.keras.layers.Dense(d_model)\n", - " \n", - " def split_heads(self, x, batch_size):\n", - " \"\"\"Split the last dimension into (num_heads, depth).\n", - " Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n", - " \"\"\"\n", - " x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n", - " return tf.transpose(x, perm=[0, 2, 1, 3])\n", - " \n", - " def call(self, v, k, q, mask):\n", - " batch_size = tf.shape(q)[0]\n", - " \n", - " q = self.wq(q) # (batch_size, seq_len, d_model)\n", - " k = self.wk(k) # (batch_size, seq_len, d_model)\n", - " v = self.wv(v) # (batch_size, seq_len, d_model)\n", - " \n", - " q = self.split_heads(q, batch_size) # (batch_size, num_heads, seq_len_q, depth)\n", - " k = self.split_heads(k, batch_size) # (batch_size, num_heads, seq_len_k, depth)\n", - " v = self.split_heads(v, batch_size) # (batch_size, num_heads, seq_len_v, depth)\n", - " \n", - " # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n", - " # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n", - " scaled_attention, attention_weights = scaled_dot_product_attention(\n", - " q, k, v, mask)\n", - " \n", - " scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3]) # (batch_size, seq_len_q, num_heads, depth)\n", - "\n", - " concat_attention = tf.reshape(scaled_attention, \n", - " (batch_size, -1, self.d_model)) # (batch_size, seq_len_q, d_model)\n", - "\n", - " output = self.dense(concat_attention) # (batch_size, seq_len_q, d_model)\n", - " \n", - " return output, attention_weights" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0D8FJue5lDyZ" - }, - "source": [ - "Create a `MultiHeadAttention` layer to try out. At each location in the sequence, `y`, the `MultiHeadAttention` runs all 8 attention heads across all other locations in the sequence, returning a new vector of the same length at each location." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Hu94p-_-2_BX" - }, - "outputs": [], - "source": [ - "temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\n", - "y = tf.random.uniform((1, 60, 512)) # (batch_size, encoder_sequence, d_model)\n", - "out, attn = temp_mha(y, k=y, q=y, mask=None)\n", - "out.shape, attn.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RdDqGayx67vv" - }, - "source": [ - "## Point wise feed forward network" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gBqzJXGfHK3X" - }, - "source": [ - "Point wise feed forward network consists of two fully-connected layers with a ReLU activation in between." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ET7xLt0yCT6Z" - }, - "outputs": [], - "source": [ - "def point_wise_feed_forward_network(d_model, dff):\n", - " return tf.keras.Sequential([\n", - " tf.keras.layers.Dense(dff, activation='relu'), # (batch_size, seq_len, dff)\n", - " tf.keras.layers.Dense(d_model) # (batch_size, seq_len, d_model)\n", - " ])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mytb1lPyOHLB" - }, - "outputs": [], - "source": [ - "sample_ffn = point_wise_feed_forward_network(512, 2048)\n", - "sample_ffn(tf.random.uniform((64, 50, 512))).shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7e7hKcxn6-zd" - }, - "source": [ - "## Encoder and decoder" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yScbC0MUH8dS" - }, - "source": [ - "\u003cimg src=\"https://www.tensorflow.org/images/tutorials/transformer/transformer.png\" width=\"600\" alt=\"transformer\"\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfYJG-Kvgwy2" - }, - "source": [ - "The transformer model follows the same general pattern as a standard [sequence to sequence with attention model](nmt_with_attention.ipynb). \n", - "\n", - "* The input sentence is passed through `N` encoder layers that generates an output for each word/token in the sequence.\n", - "* The decoder attends on the encoder's output and its own input (self-attention) to predict the next word. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QFv-FNYUmvpn" - }, - "source": [ - "### Encoder layer\n", - "\n", - "Each encoder layer consists of sublayers:\n", - "\n", - "1. Multi-head attention (with padding mask) \n", - "2. Point wise feed forward networks. \n", - "\n", - "Each of these sublayers has a residual connection around it followed by a layer normalization. Residual connections help in avoiding the vanishing gradient problem in deep networks.\n", - "\n", - "The output of each sublayer is `LayerNorm(x + Sublayer(x))`. The normalization is done on the `d_model` (last) axis. There are N encoder layers in the transformer." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ncyS-Ms3i2x_" - }, - "outputs": [], - "source": [ - "class EncoderLayer(tf.keras.layers.Layer):\n", - " def __init__(self, d_model, num_heads, dff, rate=0.1):\n", - " super(EncoderLayer, self).__init__()\n", - "\n", - " self.mha = MultiHeadAttention(d_model, num_heads)\n", - " self.ffn = point_wise_feed_forward_network(d_model, dff)\n", - "\n", - " self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " \n", - " self.dropout1 = tf.keras.layers.Dropout(rate)\n", - " self.dropout2 = tf.keras.layers.Dropout(rate)\n", - " \n", - " def call(self, x, training, mask):\n", - "\n", - " attn_output, _ = self.mha(x, x, x, mask) # (batch_size, input_seq_len, d_model)\n", - " attn_output = self.dropout1(attn_output, training=training)\n", - " out1 = self.layernorm1(x + attn_output) # (batch_size, input_seq_len, d_model)\n", - " \n", - " ffn_output = self.ffn(out1) # (batch_size, input_seq_len, d_model)\n", - " ffn_output = self.dropout2(ffn_output, training=training)\n", - " out2 = self.layernorm2(out1 + ffn_output) # (batch_size, input_seq_len, d_model)\n", - " \n", - " return out2" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AzZRXdO0mI48" - }, - "outputs": [], - "source": [ - "sample_encoder_layer = EncoderLayer(512, 8, 2048)\n", - "\n", - "sample_encoder_layer_output = sample_encoder_layer(\n", - " tf.random.uniform((64, 43, 512)), False, None)\n", - "\n", - "sample_encoder_layer_output.shape # (batch_size, input_seq_len, d_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6LO_48Owmx_o" - }, - "source": [ - "### Decoder layer\n", - "\n", - "Each decoder layer consists of sublayers:\n", - "\n", - "1. Masked multi-head attention (with look ahead mask and padding mask)\n", - "2. Multi-head attention (with padding mask). V (value) and K (key) receive the *encoder output* as inputs. Q (query) receives the *output from the masked multi-head attention sublayer.*\n", - "3. Point wise feed forward networks\n", - "\n", - "Each of these sublayers has a residual connection around it followed by a layer normalization. The output of each sublayer is `LayerNorm(x + Sublayer(x))`. The normalization is done on the `d_model` (last) axis.\n", - "\n", - "There are N decoder layers in the transformer.\n", - "\n", - "As Q receives the output from decoder's first attention block, and K receives the encoder output, the attention weights represent the importance given to the decoder's input based on the encoder's output. In other words, the decoder predicts the next word by looking at the encoder output and self-attending to its own output. See the demonstration above in the scaled dot product attention section." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9SoX0-vd1hue" - }, - "outputs": [], - "source": [ - "class DecoderLayer(tf.keras.layers.Layer):\n", - " def __init__(self, d_model, num_heads, dff, rate=0.1):\n", - " super(DecoderLayer, self).__init__()\n", - "\n", - " self.mha1 = MultiHeadAttention(d_model, num_heads)\n", - " self.mha2 = MultiHeadAttention(d_model, num_heads)\n", - "\n", - " self.ffn = point_wise_feed_forward_network(d_model, dff)\n", - " \n", - " self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " \n", - " self.dropout1 = tf.keras.layers.Dropout(rate)\n", - " self.dropout2 = tf.keras.layers.Dropout(rate)\n", - " self.dropout3 = tf.keras.layers.Dropout(rate)\n", - " \n", - " \n", - " def call(self, x, enc_output, training, \n", - " look_ahead_mask, padding_mask):\n", - " # enc_output.shape == (batch_size, input_seq_len, d_model)\n", - "\n", - " attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask) # (batch_size, target_seq_len, d_model)\n", - " attn1 = self.dropout1(attn1, training=training)\n", - " out1 = self.layernorm1(attn1 + x)\n", - " \n", - " attn2, attn_weights_block2 = self.mha2(\n", - " enc_output, enc_output, out1, padding_mask) # (batch_size, target_seq_len, d_model)\n", - " attn2 = self.dropout2(attn2, training=training)\n", - " out2 = self.layernorm2(attn2 + out1) # (batch_size, target_seq_len, d_model)\n", - " \n", - " ffn_output = self.ffn(out2) # (batch_size, target_seq_len, d_model)\n", - " ffn_output = self.dropout3(ffn_output, training=training)\n", - " out3 = self.layernorm3(ffn_output + out2) # (batch_size, target_seq_len, d_model)\n", - " \n", - " return out3, attn_weights_block1, attn_weights_block2" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Ne2Bqx8k71l0" - }, - "outputs": [], - "source": [ - "sample_decoder_layer = DecoderLayer(512, 8, 2048)\n", - "\n", - "sample_decoder_layer_output, _, _ = sample_decoder_layer(\n", - " tf.random.uniform((64, 50, 512)), sample_encoder_layer_output, \n", - " False, None, None)\n", - "\n", - "sample_decoder_layer_output.shape # (batch_size, target_seq_len, d_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SE1H51Ajm0q1" - }, - "source": [ - "### Encoder\n", - "\n", - "The `Encoder` consists of:\n", - "1. Input Embedding\n", - "2. Positional Encoding\n", - "3. N encoder layers\n", - "\n", - "The input is put through an embedding which is summed with the positional encoding. The output of this summation is the input to the encoder layers. The output of the encoder is the input to the decoder." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jpEox7gJ8FCI" - }, - "outputs": [], - "source": [ - "class Encoder(tf.keras.layers.Layer):\n", - " def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n", - " maximum_position_encoding, rate=0.1):\n", - " super(Encoder, self).__init__()\n", - "\n", - " self.d_model = d_model\n", - " self.num_layers = num_layers\n", - " \n", - " self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n", - " self.pos_encoding = positional_encoding(maximum_position_encoding, \n", - " self.d_model)\n", - " \n", - " \n", - " self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n", - " for _ in range(num_layers)]\n", - " \n", - " self.dropout = tf.keras.layers.Dropout(rate)\n", - " \n", - " def call(self, x, training, mask):\n", - "\n", - " seq_len = tf.shape(x)[1]\n", - " \n", - " # adding embedding and position encoding.\n", - " x = self.embedding(x) # (batch_size, input_seq_len, d_model)\n", - " x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n", - " x += self.pos_encoding[:, :seq_len, :]\n", - "\n", - " x = self.dropout(x, training=training)\n", - " \n", - " for i in range(self.num_layers):\n", - " x = self.enc_layers[i](x, training, mask)\n", - " \n", - " return x # (batch_size, input_seq_len, d_model)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8QG9nueFQKXx" - }, - "outputs": [], - "source": [ - "sample_encoder = Encoder(num_layers=2, d_model=512, num_heads=8, \n", - " dff=2048, input_vocab_size=8500,\n", - " maximum_position_encoding=10000)\n", - "temp_input = tf.random.uniform((64, 62), dtype=tf.int64, minval=0, maxval=200)\n", - "\n", - "sample_encoder_output = sample_encoder(temp_input, training=False, mask=None)\n", - "\n", - "print (sample_encoder_output.shape) # (batch_size, input_seq_len, d_model)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p-uO6ls8m2O5" - }, - "source": [ - "### Decoder" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZtT7PKzrXkNr" - }, - "source": [ - " The `Decoder` consists of:\n", - "1. Output Embedding\n", - "2. Positional Encoding\n", - "3. N decoder layers\n", - "\n", - "The target is put through an embedding which is summed with the positional encoding. The output of this summation is the input to the decoder layers. The output of the decoder is the input to the final linear layer." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "d5_d5-PLQXwY" - }, - "outputs": [], - "source": [ - "class Decoder(tf.keras.layers.Layer):\n", - " def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n", - " maximum_position_encoding, rate=0.1):\n", - " super(Decoder, self).__init__()\n", - "\n", - " self.d_model = d_model\n", - " self.num_layers = num_layers\n", - " \n", - " self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n", - " self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n", - " \n", - " self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n", - " for _ in range(num_layers)]\n", - " self.dropout = tf.keras.layers.Dropout(rate)\n", - " \n", - " def call(self, x, enc_output, training, \n", - " look_ahead_mask, padding_mask):\n", - "\n", - " seq_len = tf.shape(x)[1]\n", - " attention_weights = {}\n", - " \n", - " x = self.embedding(x) # (batch_size, target_seq_len, d_model)\n", - " x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n", - " x += self.pos_encoding[:, :seq_len, :]\n", - " \n", - " x = self.dropout(x, training=training)\n", - "\n", - " for i in range(self.num_layers):\n", - " x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n", - " look_ahead_mask, padding_mask)\n", - " \n", - " attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n", - " attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n", - " \n", - " # x.shape == (batch_size, target_seq_len, d_model)\n", - " return x, attention_weights" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "a1jXoAMRZyvu" - }, - "outputs": [], - "source": [ - "sample_decoder = Decoder(num_layers=2, d_model=512, num_heads=8, \n", - " dff=2048, target_vocab_size=8000,\n", - " maximum_position_encoding=5000)\n", - "temp_input = tf.random.uniform((64, 26), dtype=tf.int64, minval=0, maxval=200)\n", - "\n", - "output, attn = sample_decoder(temp_input, \n", - " enc_output=sample_encoder_output, \n", - " training=False,\n", - " look_ahead_mask=None, \n", - " padding_mask=None)\n", - "\n", - "output.shape, attn['decoder_layer2_block2'].shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y54xnJnuYgJ7" - }, - "source": [ - "## Create the Transformer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uERO1y54cOKq" - }, - "source": [ - "Transformer consists of the encoder, decoder and a final linear layer. The output of the decoder is the input to the linear layer and its output is returned." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "PED3bIpOYkBu" - }, - "outputs": [], - "source": [ - "class Transformer(tf.keras.Model):\n", - " def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n", - " target_vocab_size, pe_input, pe_target, rate=0.1):\n", - " super(Transformer, self).__init__()\n", - "\n", - " self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n", - " input_vocab_size, pe_input, rate)\n", - "\n", - " self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n", - " target_vocab_size, pe_target, rate)\n", - "\n", - " self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n", - " \n", - " def call(self, inp, tar, training, enc_padding_mask, \n", - " look_ahead_mask, dec_padding_mask):\n", - "\n", - " enc_output = self.encoder(inp, training, enc_padding_mask) # (batch_size, inp_seq_len, d_model)\n", - " \n", - " # dec_output.shape == (batch_size, tar_seq_len, d_model)\n", - " dec_output, attention_weights = self.decoder(\n", - " tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n", - " \n", - " final_output = self.final_layer(dec_output) # (batch_size, tar_seq_len, target_vocab_size)\n", - " \n", - " return final_output, attention_weights" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tJ4fbQcIkHW1" - }, - "outputs": [], - "source": [ - "sample_transformer = Transformer(\n", - " num_layers=2, d_model=512, num_heads=8, dff=2048, \n", - " input_vocab_size=8500, target_vocab_size=8000, \n", - " pe_input=10000, pe_target=6000)\n", - "\n", - "temp_input = tf.random.uniform((64, 38), dtype=tf.int64, minval=0, maxval=200)\n", - "temp_target = tf.random.uniform((64, 36), dtype=tf.int64, minval=0, maxval=200)\n", - "\n", - "fn_out, _ = sample_transformer(temp_input, temp_target, training=False, \n", - " enc_padding_mask=None, \n", - " look_ahead_mask=None,\n", - " dec_padding_mask=None)\n", - "\n", - "fn_out.shape # (batch_size, tar_seq_len, target_vocab_size)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wsINyf1VEQLC" - }, - "source": [ - "## Set hyperparameters" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zVjWCxFNcgbt" - }, - "source": [ - "To keep this example small and relatively fast, the values for *num_layers, d_model, and dff* have been reduced. \n", - "\n", - "The values used in the base model of transformer were; *num_layers=6*, *d_model = 512*, *dff = 2048*. See the [paper](https://arxiv.org/abs/1706.03762) for all the other versions of the transformer.\n", - "\n", - "Note: By changing the values below, you can get the model that achieved state of the art on many tasks." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lnJn5SLA2ahP" - }, - "outputs": [], - "source": [ - "num_layers = 4\n", - "d_model = 128\n", - "dff = 512\n", - "num_heads = 8\n", - "\n", - "input_vocab_size = tokenizer_pt.vocab_size + 2\n", - "target_vocab_size = tokenizer_en.vocab_size + 2\n", - "dropout_rate = 0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xYEGhEOtzn5W" - }, - "source": [ - "## Optimizer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GOmWW--yP3zx" - }, - "source": [ - "Use the Adam optimizer with a custom learning rate scheduler according to the formula in the [paper](https://arxiv.org/abs/1706.03762).\n", - "\n", - "$$\\Large{lrate = d_{model}^{-0.5} * min(step{\\_}num^{-0.5}, step{\\_}num * warmup{\\_}steps^{-1.5})}$$\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iYQdOO1axwEI" - }, - "outputs": [], - "source": [ - "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n", - " def __init__(self, d_model, warmup_steps=4000):\n", - " super(CustomSchedule, self).__init__()\n", - " \n", - " self.d_model = d_model\n", - " self.d_model = tf.cast(self.d_model, tf.float32)\n", - "\n", - " self.warmup_steps = warmup_steps\n", - " \n", - " def __call__(self, step):\n", - " arg1 = tf.math.rsqrt(step)\n", - " arg2 = step * (self.warmup_steps ** -1.5)\n", - " \n", - " return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7r4scdulztRx" - }, - "outputs": [], - "source": [ - "learning_rate = CustomSchedule(d_model)\n", - "\n", - "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n", - " epsilon=1e-9)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "f33ZCgvHpPdG" - }, - "outputs": [], - "source": [ - "temp_learning_rate_schedule = CustomSchedule(d_model)\n", - "\n", - "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n", - "plt.ylabel(\"Learning Rate\")\n", - "plt.xlabel(\"Train Step\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YgkDE7hzo8r5" - }, - "source": [ - "## Loss and metrics" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oxGJtoDuYIHL" - }, - "source": [ - "Since the target sequences are padded, it is important to apply a padding mask when calculating the loss." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MlhsJMm0TW_B" - }, - "outputs": [], - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " from_logits=True, reduction='none')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "67oqVHiT0Eiu" - }, - "outputs": [], - "source": [ - "def loss_function(real, pred):\n", - " mask = tf.math.logical_not(tf.math.equal(real, 0))\n", - " loss_ = loss_object(real, pred)\n", - "\n", - " mask = tf.cast(mask, dtype=loss_.dtype)\n", - " loss_ *= mask\n", - " \n", - " return tf.reduce_mean(loss_)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "phlyxMnm-Tpx" - }, - "outputs": [], - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", - " name='train_accuracy')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aeHumfr7zmMa" - }, - "source": [ - "## Training and checkpointing" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UiysUa--4tOU" - }, - "outputs": [], - "source": [ - "transformer = Transformer(num_layers, d_model, num_heads, dff,\n", - " input_vocab_size, target_vocab_size, \n", - " pe_input=input_vocab_size, \n", - " pe_target=target_vocab_size,\n", - " rate=dropout_rate)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ZOJUSB1T8GjM" - }, - "outputs": [], - "source": [ - "def create_masks(inp, tar):\n", - " # Encoder padding mask\n", - " enc_padding_mask = create_padding_mask(inp)\n", - " \n", - " # Used in the 2nd attention block in the decoder.\n", - " # This padding mask is used to mask the encoder outputs.\n", - " dec_padding_mask = create_padding_mask(inp)\n", - " \n", - " # Used in the 1st attention block in the decoder.\n", - " # It is used to pad and mask future tokens in the input received by \n", - " # the decoder.\n", - " look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n", - " dec_target_padding_mask = create_padding_mask(tar)\n", - " combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n", - " \n", - " return enc_padding_mask, combined_mask, dec_padding_mask" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fzuf06YZp66w" - }, - "source": [ - "Create the checkpoint path and the checkpoint manager. This will be used to save checkpoints every `n` epochs." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hNhuYfllndLZ" - }, - "outputs": [], - "source": [ - "checkpoint_path = \"./checkpoints/train\"\n", - "\n", - "ckpt = tf.train.Checkpoint(transformer=transformer,\n", - " optimizer=optimizer)\n", - "\n", - "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n", - "\n", - "# if a checkpoint exists, restore the latest checkpoint.\n", - "if ckpt_manager.latest_checkpoint:\n", - " ckpt.restore(ckpt_manager.latest_checkpoint)\n", - " print ('Latest checkpoint restored!!')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Di_Yaa1gf9r" - }, - "source": [ - "The target is divided into tar_inp and tar_real. tar_inp is passed as an input to the decoder. `tar_real` is that same input shifted by 1: At each location in `tar_input`, `tar_real` contains the next token that should be predicted.\n", - "\n", - "For example, `sentence` = \"SOS A lion in the jungle is sleeping EOS\"\n", - "\n", - "`tar_inp` = \"SOS A lion in the jungle is sleeping\"\n", - "\n", - "`tar_real` = \"A lion in the jungle is sleeping EOS\"\n", - "\n", - "The transformer is an auto-regressive model: it makes predictions one part at a time, and uses its output so far to decide what to do next. \n", - "\n", - "During training this example uses teacher-forcing (like in the [text generation tutorial](./text_generation.ipynb)). Teacher forcing is passing the true output to the next time step regardless of what the model predicts at the current time step.\n", - "\n", - "As the transformer predicts each word, *self-attention* allows it to look at the previous words in the input sequence to better predict the next word.\n", - "\n", - "To prevent the model from peaking at the expected output the model uses a look-ahead mask." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LKpoA6q1sJFj" - }, - "outputs": [], - "source": [ - "EPOCHS = 20" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iJwmp9OE29oj" - }, - "outputs": [], - "source": [ - "# The @tf.function trace-compiles train_step into a TF graph for faster\n", - "# execution. The function specializes to the precise shape of the argument\n", - "# tensors. To avoid re-tracing due to the variable sequence lengths or variable\n", - "# batch sizes (the last batch is smaller), use input_signature to specify\n", - "# more generic shapes.\n", - "\n", - "train_step_signature = [\n", - " tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n", - " tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n", - "]\n", - "\n", - "@tf.function(input_signature=train_step_signature)\n", - "def train_step(inp, tar):\n", - " tar_inp = tar[:, :-1]\n", - " tar_real = tar[:, 1:]\n", - " \n", - " enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n", - " \n", - " with tf.GradientTape() as tape:\n", - " predictions, _ = transformer(inp, tar_inp, \n", - " True, \n", - " enc_padding_mask, \n", - " combined_mask, \n", - " dec_padding_mask)\n", - " loss = loss_function(tar_real, predictions)\n", - "\n", - " gradients = tape.gradient(loss, transformer.trainable_variables) \n", - " optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n", - " \n", - " train_loss(loss)\n", - " train_accuracy(tar_real, predictions)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qM2PDWGDJ_8V" - }, - "source": [ - "Portuguese is used as the input language and English is the target language." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bbvmaKNiznHZ" - }, - "outputs": [], - "source": [ - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - " \n", - " train_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " \n", - " # inp -\u003e portuguese, tar -\u003e english\n", - " for (batch, (inp, tar)) in enumerate(train_dataset):\n", - " train_step(inp, tar)\n", - " \n", - " if batch % 50 == 0:\n", - " print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n", - " epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n", - " \n", - " if (epoch + 1) % 5 == 0:\n", - " ckpt_save_path = ckpt_manager.save()\n", - " print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,\n", - " ckpt_save_path))\n", - " \n", - " print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, \n", - " train_loss.result(), \n", - " train_accuracy.result()))\n", - "\n", - " print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QfcsSWswSdGV" - }, - "source": [ - "## Evaluate" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y6APsFrgImLW" - }, - "source": [ - "The following steps are used for evaluation:\n", - "\n", - "* Encode the input sentence using the Portuguese tokenizer (`tokenizer_pt`). Moreover, add the start and end token so the input is equivalent to what the model is trained with. This is the encoder input.\n", - "* The decoder input is the `start token == tokenizer_en.vocab_size`.\n", - "* Calculate the padding masks and the look ahead masks.\n", - "* The `decoder` then outputs the predictions by looking at the `encoder output` and its own output (self-attention).\n", - "* Select the last word and calculate the argmax of that.\n", - "* Concatentate the predicted word to the decoder input as pass it to the decoder.\n", - "* In this approach, the decoder predicts the next word based on the previous words it predicted.\n", - "\n", - "Note: The model used here has less capacity to keep the example relatively faster so the predictions maybe less right. To reproduce the results in the paper, use the entire dataset and base transformer model or transformer XL, by changing the hyperparameters above." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5buvMlnvyrFm" - }, - "outputs": [], - "source": [ - "def evaluate(inp_sentence):\n", - " start_token = [tokenizer_pt.vocab_size]\n", - " end_token = [tokenizer_pt.vocab_size + 1]\n", - " \n", - " # inp sentence is portuguese, hence adding the start and end token\n", - " inp_sentence = start_token + tokenizer_pt.encode(inp_sentence) + end_token\n", - " encoder_input = tf.expand_dims(inp_sentence, 0)\n", - " \n", - " # as the target is english, the first word to the transformer should be the\n", - " # english start token.\n", - " decoder_input = [tokenizer_en.vocab_size]\n", - " output = tf.expand_dims(decoder_input, 0)\n", - " \n", - " for i in range(MAX_LENGTH):\n", - " enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n", - " encoder_input, output)\n", - " \n", - " # predictions.shape == (batch_size, seq_len, vocab_size)\n", - " predictions, attention_weights = transformer(encoder_input, \n", - " output,\n", - " False,\n", - " enc_padding_mask,\n", - " combined_mask,\n", - " dec_padding_mask)\n", - " \n", - " # select the last word from the seq_len dimension\n", - " predictions = predictions[: ,-1:, :] # (batch_size, 1, vocab_size)\n", - "\n", - " predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n", - " \n", - " # return the result if the predicted_id is equal to the end token\n", - " if predicted_id == tokenizer_en.vocab_size+1:\n", - " return tf.squeeze(output, axis=0), attention_weights\n", - " \n", - " # concatentate the predicted_id to the output which is given to the decoder\n", - " # as its input.\n", - " output = tf.concat([output, predicted_id], axis=-1)\n", - "\n", - " return tf.squeeze(output, axis=0), attention_weights" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "CN-BV43FMBej" - }, - "outputs": [], - "source": [ - "def plot_attention_weights(attention, sentence, result, layer):\n", - " fig = plt.figure(figsize=(16, 8))\n", - " \n", - " sentence = tokenizer_pt.encode(sentence)\n", - " \n", - " attention = tf.squeeze(attention[layer], axis=0)\n", - " \n", - " for head in range(attention.shape[0]):\n", - " ax = fig.add_subplot(2, 4, head+1)\n", - " \n", - " # plot the attention weights\n", - " ax.matshow(attention[head][:-1, :], cmap='viridis')\n", - "\n", - " fontdict = {'fontsize': 10}\n", - " \n", - " ax.set_xticks(range(len(sentence)+2))\n", - " ax.set_yticks(range(len(result)))\n", - " \n", - " ax.set_ylim(len(result)-1.5, -0.5)\n", - " \n", - " ax.set_xticklabels(\n", - " ['\u003cstart\u003e']+[tokenizer_pt.decode([i]) for i in sentence]+['\u003cend\u003e'], \n", - " fontdict=fontdict, rotation=90)\n", - " \n", - " ax.set_yticklabels([tokenizer_en.decode([i]) for i in result \n", - " if i \u003c tokenizer_en.vocab_size], \n", - " fontdict=fontdict)\n", - " \n", - " ax.set_xlabel('Head {}'.format(head+1))\n", - " \n", - " plt.tight_layout()\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lU2_yG_vBGza" - }, - "outputs": [], - "source": [ - "def translate(sentence, plot=''):\n", - " result, attention_weights = evaluate(sentence)\n", - " \n", - " predicted_sentence = tokenizer_en.decode([i for i in result \n", - " if i \u003c tokenizer_en.vocab_size]) \n", - "\n", - " print('Input: {}'.format(sentence))\n", - " print('Predicted translation: {}'.format(predicted_sentence))\n", - " \n", - " if plot:\n", - " plot_attention_weights(attention_weights, sentence, result, plot)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YsxrAlvFG8SZ" - }, - "outputs": [], - "source": [ - "translate(\"este é um problema que temos que resolver.\")\n", - "print (\"Real translation: this is a problem we have to solve .\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7EH5y_aqI4t1" - }, - "outputs": [], - "source": [ - "translate(\"os meus vizinhos ouviram sobre esta ideia.\")\n", - "print (\"Real translation: and my neighboring homes heard about this idea .\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "J-hVCTSUMlkb" - }, - "outputs": [], - "source": [ - "translate(\"vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.\")\n", - "print (\"Real translation: so i 'll just share with you some stories very quickly of some magical things that have happened .\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_1MxkSZvz0jX" - }, - "source": [ - "You can pass different layers and attention blocks of the decoder to the `plot` parameter." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "t-kFyiOLH0xg" - }, - "outputs": [], - "source": [ - "translate(\"este é o primeiro livro que eu fiz.\", plot='decoder_layer4_block2')\n", - "print (\"Real translation: this is the first book i've ever done.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RqQ1fIsLwkGE" - }, - "source": [ - "## Summary\n", - "\n", - "In this tutorial, you learned about positional encoding, multi-head attention, the importance of masking and how to create a transformer.\n", - "\n", - "Try using a different dataset to train the transformer. You can also create the base transformer or transformer XL by changing the hyperparameters above. You can also use the layers defined here to create [BERT](https://arxiv.org/abs/1810.04805) and train state of the art models. Futhermore, you can implement beam search to get better predictions." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [ - "s_qNSzzyaCbD" - ], - "name": "transformer.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/text/word_embeddings.ipynb b/site/en/tutorials/text/word_embeddings.ipynb deleted file mode 100644 index 1c8b008c311..00000000000 --- a/site/en/tutorials/text/word_embeddings.ipynb +++ /dev/null @@ -1,670 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GE91qWZkm8ZQ" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "YS3NA-i6nAFC" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7SN5USFEIIK3" - }, - "source": [ - "# Word embeddings" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Aojnnc7sXrab" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/word_embeddings\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/word_embeddings.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/word_embeddings.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/text/word_embeddings.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q6mJg1g3apaz" - }, - "source": [ - "This tutorial introduces word embeddings. It contains complete code to train word embeddings from scratch on a small dataset, and to visualize these embeddings using the [Embedding Projector](http://projector.tensorflow.org) (shown in the image below).\n", - "\n", - "\u003cimg src=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/images/embedding.jpg?raw=1\" alt=\"Screenshot of the embedding projector\" width=\"400\"/\u003e\n", - "\n", - "## Representing text as numbers\n", - "\n", - "Machine learning models take vectors (arrays of numbers) as input. When working with text, the first thing we must do come up with a strategy to convert strings to numbers (or to \"vectorize\" the text) before feeding it to the model. In this section, we will look at three strategies for doing so.\n", - "\n", - "### One-hot encodings\n", - "\n", - "As a first idea, we might \"one-hot\" encode each word in our vocabulary. Consider the sentence \"The cat sat on the mat\". The vocabulary (or unique words) in this sentence is (cat, mat, on, sat, the). To represent each word, we will create a zero vector with length equal to the vocabulary, then place a one in the index that corresponds to the word. This approach is shown in the following diagram.\n", - "\n", - "\u003cimg src=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/images/one-hot.png?raw=1\" alt=\"Diagram of one-hot encodings\" width=\"400\" /\u003e\n", - "\n", - "To create a vector that contains the encoding of the sentence, we could then concatenate the one-hot vectors for each word.\n", - "\n", - "Key point: This approach is inefficient. A one-hot encoded vector is sparse (meaning, most indicices are zero). Imagine we have 10,000 words in the vocabulary. To one-hot encode each word, we would create a vector where 99.99% of the elements are zero.\n", - "\n", - "### Encode each word with a unique number\n", - "\n", - "A second approach we might try is to encode each word using a unique number. Continuing the example above, we could assign 1 to \"cat\", 2 to \"mat\", and so on. We could then encode the sentence \"The cat sat on the mat\" as a dense vector like [5, 1, 4, 3, 5, 2]. This appoach is efficient. Instead of a sparse vector, we now have a dense one (where all elements are full).\n", - "\n", - "There are two downsides to this approach, however:\n", - "\n", - "* The integer-encoding is arbitrary (it does not capture any relationship between words).\n", - "\n", - "* An integer-encoding can be challenging for a model to interpret. A linear classifier, for example, learns a single weight for each feature. Because there is no relationship between the similarity of any two words and the similarity of their encodings, this feature-weight combination is not meaningful.\n", - "\n", - "### Word embeddings\n", - "\n", - "Word embeddings give us a way to use an efficient, dense representation in which similar words have a similar encoding. Importantly, we do not have to specify this encoding by hand. An embedding is a dense vector of floating point values (the length of the vector is a parameter you specify). Instead of specifying the values for the embedding manually, they are trainable parameters (weights learned by the model during training, in the same way a model learns weights for a dense layer). It is common to see word embeddings that are 8-dimensional (for small datasets), up to 1024-dimensions when working with large datasets. A higher dimensional embedding can capture fine-grained relationships between words, but takes more data to learn.\n", - "\n", - "\u003cimg src=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/images/embedding2.png?raw=1\" alt=\"Diagram of an embedding\" width=\"400\"/\u003e\n", - "\n", - "Above is a diagram for a word embedding. Each word is represented as a 4-dimensional vector of floating point values. Another way to think of an embedding is as \"lookup table\". After these weights have been learned, we can encode each word by looking up the dense vector it corresponds to in the table." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SZUQErGewZxE" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SIXEk5ON5P7h" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RutaI-Tpev3T" - }, - "outputs": [], - "source": [ - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "import tensorflow_datasets as tfds\n", - "tfds.disable_progress_bar()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eqBazMiVQkj1" - }, - "source": [ - "## Using the Embedding layer\n", - "\n", - "Keras makes it easy to use word embeddings. Let's take a look at the [Embedding](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Embedding) layer.\n", - "\n", - "The Embedding layer can be understood as a lookup table that maps from integer indices (which stand for specific words) to dense vectors (their embeddings). The dimensionality (or width) of the embedding is a parameter you can experiment with to see what works well for your problem, much in the same way you would experiment with the number of neurons in a Dense layer.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-OjxLVrMvWUE" - }, - "outputs": [], - "source": [ - "embedding_layer = layers.Embedding(1000, 5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2dKKV1L2Rk7e" - }, - "source": [ - "\n", - "\n", - "When you create an Embedding layer, the weights for the embedding are randomly initialized (just like any other layer). During training, they are gradually adjusted via backpropagation. Once trained, the learned word embeddings will roughly encode similarities between words (as they were learned for the specific problem your model is trained on).\n", - "\n", - "If you pass an integer to an embedding layer, the result replaces each integer with the vector from the embedding table:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0YUjPgP7w0PO" - }, - "outputs": [], - "source": [ - "result = embedding_layer(tf.constant([1,2,3]))\n", - "result.numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O4PC4QzsxTGx" - }, - "source": [ - "For text or sequence problems, the Embedding layer takes a 2D tensor of integers, of shape `(samples, sequence_length)`, where each entry is a sequence of integers. It can embed sequences of variable lengths. You could feed into the embedding layer above batches with shapes `(32, 10)` (batch of 32 sequences of length 10) or `(64, 15)` (batch of 64 sequences of length 15).\n", - "\n", - "The returned tensor has one more axis than the input, the embedding vectors are aligned along the new last axis. Pass it a `(2, 3)` input batch and the output is `(2, 3, N)`\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vwSYepRjyRGy" - }, - "outputs": [], - "source": [ - "result = embedding_layer(tf.constant([[0,1,2],[3,4,5]]))\n", - "result.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WGQp2N92yOyB" - }, - "source": [ - "When given a batch of sequences as input, an embedding layer returns a 3D floating point tensor, of shape `(samples, sequence_length, embedding_dimensionality)`. To convert from this sequence of variable length to a fixed representation there are a variety of standard approaches. You could use an RNN, Attention, or pooling layer before passing it to a Dense layer. This tutorial uses pooling because it's simplest. The [Text Classification with an RNN](text_classification_rnn.ipynb) tutorial is a good next step." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aGicgV5qT0wh" - }, - "source": [ - "## Learning embeddings from scratch" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_Bh8B1TUT6mV" - }, - "source": [ - "In this tutorial you will train a sentiment classifier on IMDB movie reviews. In the process, the model will learn embeddings from scratch. We will use to a preprocessed dataset.\n", - "\n", - "To load a text dataset from scratch see the [Loading text tutorial](../load_data/text.ipynb)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yg6tyxPtp1TE" - }, - "outputs": [], - "source": [ - "(train_data, test_data), info = tfds.load(\n", - " 'imdb_reviews/subwords8k', \n", - " split = (tfds.Split.TRAIN, tfds.Split.TEST), \n", - " with_info=True, as_supervised=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jjnBsFXaLVPL" - }, - "source": [ - "Get the encoder (`tfds.features.text.SubwordTextEncoder`), and have a quick look at the vocabulary. \n", - "\n", - "The \"\\_\" in the vocabulary represent spaces. Note how the vocabulary includes whole words (ending with \"\\_\") and partial words which it can use to build larger words:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MYrsTgxhLBfl" - }, - "outputs": [], - "source": [ - "encoder = info.features['text'].encoder\n", - "encoder.subwords[:20]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GwCTfSG63Qth" - }, - "source": [ - "Movie reviews can be different lengths. We will use the `padded_batch` method to standardize the lengths of the reviews." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LwSCxER_2Lef" - }, - "outputs": [], - "source": [ - "padded_shapes = ([None],())\n", - "train_batches = train_data.shuffle(1000).padded_batch(10, padded_shapes = padded_shapes)\n", - "test_batches = test_data.shuffle(1000).padded_batch(10, padded_shapes = padded_shapes)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dF8ORMt2U9lj" - }, - "source": [ - "As imported, the text of reviews is integer-encoded (each integer represents a specific word or word-part in the vocabulary).\n", - "\n", - "Note the trailing zeros, because the batch is padded to the longest example." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Se-phCknsoan" - }, - "outputs": [], - "source": [ - "train_batch, train_labels = next(iter(train_batches))\n", - "train_batch.numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zI9_wLIiWO8Z" - }, - "source": [ - "### Create a simple model\n", - "\n", - "We will use the [Keras Sequential API](../../guide/keras) to define our model. In this case it is a \"Continuous bag of words\" style model.\n", - "\n", - "* Next the Embedding layer takes the integer-encoded vocabulary and looks up the embedding vector for each word-index. These vectors are learned as the model trains. The vectors add a dimension to the output array. The resulting dimensions are: `(batch, sequence, embedding)`.\n", - "\n", - "* Next, a GlobalAveragePooling1D layer returns a fixed-length output vector for each example by averaging over the sequence dimension. This allows the model to handle input of variable length, in the simplest way possible.\n", - "\n", - "* This fixed-length output vector is piped through a fully-connected (Dense) layer with 16 hidden units.\n", - "\n", - "* The last layer is densely connected with a single output node. Using the sigmoid activation function, this value is a float between 0 and 1, representing a probability (or confidence level) that the review is positive.\n", - "\n", - "Caution: This model doesn't use masking, so the zero-padding is used as part of the input, so the padding length may affect the output. To fix this, see the [masking and padding guide](../../guide/keras/masking_and_padding)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "pHLcFtn5Wsqj" - }, - "outputs": [], - "source": [ - "embedding_dim=16\n", - "\n", - "model = keras.Sequential([\n", - " layers.Embedding(encoder.vocab_size, embedding_dim),\n", - " layers.GlobalAveragePooling1D(),\n", - " layers.Dense(16, activation='relu'),\n", - " layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JjLNgKO7W2fe" - }, - "source": [ - "### Compile and train the model" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lCUgdP69Wzix" - }, - "outputs": [], - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "history = model.fit(\n", - " train_batches,\n", - " epochs=10,\n", - " validation_data=test_batches, validation_steps=20)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LQjpKVYTXU-1" - }, - "source": [ - "With this approach our model reaches a validation accuracy of around 88% (note the model is overfitting, training accuracy is significantly higher)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0D3OTmOT1z1O" - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "history_dict = history.history\n", - "\n", - "acc = history_dict['accuracy']\n", - "val_acc = history_dict['val_accuracy']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "plt.figure(figsize=(12,9))\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "plt.show()\n", - "\n", - "plt.figure(figsize=(12,9))\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend(loc='lower right')\n", - "plt.ylim((0.5,1))\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KCoA6qwqP836" - }, - "source": [ - "## Retrieve the learned embeddings\n", - "\n", - "Next, let's retrieve the word embeddings learned during training. This will be a matrix of shape `(vocab_size, embedding-dimension)`." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "t8WwbsXCXtpa" - }, - "outputs": [], - "source": [ - "e = model.layers[0]\n", - "weights = e.get_weights()[0]\n", - "print(weights.shape) # shape: (vocab_size, embedding_dim)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J8MiCA77X8B8" - }, - "source": [ - "We will now write the weights to disk. To use the [Embedding Projector](http://projector.tensorflow.org), we will upload two files in tab separated format: a file of vectors (containing the embedding), and a file of meta data (containing the words)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GsjempweP9Lq" - }, - "outputs": [], - "source": [ - "import io\n", - "\n", - "encoder = info.features['text'].encoder\n", - "\n", - "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n", - "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n", - "\n", - "for num, word in enumerate(encoder.subwords):\n", - " vec = weights[num+1] # skip 0, it's padding.\n", - " out_m.write(word + \"\\n\")\n", - " out_v.write('\\t'.join([str(x) for x in vec]) + \"\\n\")\n", - "out_v.close()\n", - "out_m.close()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JQyMZWyxYjMr" - }, - "source": [ - "If you are running this tutorial in [Colaboratory](https://colab.research.google.com), you can use the following snippet to download these files to your local machine (or use the file browser, *View -\u003e Table of contents -\u003e File browser*)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-gFbbMmvYvhp" - }, - "outputs": [], - "source": [ - "try:\n", - " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download('vecs.tsv')\n", - " files.download('meta.tsv')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PXLfFA54Yz-o" - }, - "source": [ - "## Visualize the embeddings\n", - "\n", - "To visualize our embeddings we will upload them to the embedding projector.\n", - "\n", - "Open the [Embedding Projector](http://projector.tensorflow.org/) (this can also run in a local TensorBoard instance).\n", - "\n", - "* Click on \"Load data\".\n", - "\n", - "* Upload the two files we created above: `vecs.tsv` and `meta.tsv`.\n", - "\n", - "The embeddings you have trained will now be displayed. You can search for words to find their closest neighbors. For example, try searching for \"beautiful\". You may see neighbors like \"wonderful\". \n", - "\n", - "Note: your results may be a bit different, depending on how weights were randomly initialized before training the embedding layer.\n", - "\n", - "Note: experimentally, you may be able to produce more interpretable embeddings by using a simpler model. Try deleting the `Dense(16)` layer, retraining the model, and visualizing the embeddings again.\n", - "\n", - "\u003cimg src=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/images/embedding.jpg?raw=1\" alt=\"Screenshot of the embedding projector\" width=\"400\"/\u003e\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iS_uMeMw3Xpj" - }, - "source": [ - "## Next steps\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BSgAZpwF5xF_" - }, - "source": [ - "This tutorial has shown you how to train and visualize word embeddings from scratch on a small dataset.\n", - "\n", - "* To learn about recurrent networks see the [Keras RNN Guide](../../guide/keras/rnn.ipynb).\n", - "\n", - "* To learn more about text classification (including the overall workflow, and if you're curious about when to use embeddings vs one-hot encodings) we recommend this practical text classification [guide](https://developers.google.com/machine-learning/guides/text-classification/step-2-5)." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "word_embeddings.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/en/tutorials/understanding/images/sngp.png b/site/en/tutorials/understanding/images/sngp.png new file mode 100644 index 00000000000..c1cb406ffaa Binary files /dev/null and b/site/en/tutorials/understanding/images/sngp.png differ diff --git a/site/en/tutorials/understanding/sngp.ipynb b/site/en/tutorials/understanding/sngp.ipynb new file mode 100644 index 00000000000..a5be4cbd54c --- /dev/null +++ b/site/en/tutorials/understanding/sngp.ipynb @@ -0,0 +1,1567 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "9teObmxrP0FE" + }, + "source": [ + "##### Copyright 2021 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "mz8tfSwOP4fW" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CBwpoARnQ3H-" + }, + "source": [ + "# Uncertainty-aware Deep Learning with SNGP" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2dL6_obQRBGQ" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UvW1QEMtP7Gy" + }, + "source": [ + "In AI applications that are safety-critical, such as medical decision making and autonomous driving, or where the data is inherently noisy (for example, natural language understanding), it is important for a deep classifier to reliably quantify its uncertainty. The deep classifier should be able to be aware of its own limitations and when it should hand control over to the human experts. This tutorial shows how to improve a deep classifier's ability in quantifying uncertainty using a technique called **Spectral-normalized Neural Gaussian Process ([SNGP](https://arxiv.org/abs/2006.10108){.external})**.\n", + "\n", + "The core idea of SNGP is to improve a deep classifier's _**distance awareness**_ by applying simple modifications to the network. A model's _distance awareness_ is a measure of how its predictive probability reflects the distance between the test example and the training data. This is a desirable property that is common for gold-standard probabilistic models (for example, the [Gaussian process](https://en.wikipedia.org/wiki/Gaussian_process){.external} with RBF kernels) but is lacking in models with deep neural networks. SNGP provides a simple way to inject this Gaussian-process behavior into a deep classifier while maintaining its predictive accuracy.\n", + "\n", + "This tutorial implements a deep residual network (ResNet)-based SNGP model on [scikit-learn’s two moons](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_moons.html){.external} dataset, and compares its uncertainty surface with that of two other popular uncertainty approaches: [Monte Carlo dropout](https://arxiv.org/abs/1506.02142){.external} and [Deep ensemble](https://arxiv.org/abs/1612.01474){.external}.\n", + "\n", + "This tutorial illustrates the SNGP model on a toy 2D dataset. For an example of applying SNGP to a real-world natural language understanding task using a BERT-base, check out the [SNGP-BERT tutorial](https://www.tensorflow.org/text/tutorials/uncertainty_quantification_with_sngp_bert). For high-quality implementations of an SNGP model (and many other uncertainty methods) on a wide variety of benchmark datasets (such as [CIFAR-100](https://www.tensorflow.org/datasets/catalog/cifar100), [ImageNet](https://www.tensorflow.org/datasets/catalog/imagenet2012), [Jigsaw toxicity detection](https://www.tensorflow.org/datasets/catalog/wikipedia_toxicity_subtypes), etc), refer to the [Uncertainty Baselines](https://github.com/google/uncertainty-baselines){.external} benchmark." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "K-tZzAIlvMv_" + }, + "source": [ + "## About SNGP" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ysyslHCyvYi-" + }, + "source": [ + "SNGP is a simple approach to improve a deep classifier's uncertainty quality while maintaining a similar level of accuracy and latency. Given a deep residual network, SNGP makes two simple changes to the model:\n", + "\n", + "* It applies spectral normalization to the hidden residual layers.\n", + "* It replaces the Dense output layer with a Gaussian process layer." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ffU8rJ_eWHou" + }, + "source": [ + ">![SNGP](http://tensorflow.org/tutorials/understanding/images/sngp.png)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2L88PoKr6XaE" + }, + "source": [ + "Compared to other uncertainty approaches (such as Monte Carlo dropout or Deep ensemble), SNGP has several advantages:\n", + "\n", + "* It works for a wide range of state-of-the-art residual-based architectures (for example, (Wide) ResNet, DenseNet, or BERT).\n", + "* It is a single-model method—it does not rely on ensemble averaging). Therefore, SNGP has a similar level of latency as a single deterministic network, and can be scaled easily to large datasets like [ImageNet](https://github.com/google/uncertainty-baselines/tree/main/baselines/imagenet){.external} and [Jigsaw Toxic Comments classification](https://github.com/google/uncertainty-baselines/tree/main/baselines/toxic_comments){.external}.\n", + "* It has strong out-of-domain detection performance due to the _distance-awareness_ property.\n", + "\n", + "The downsides of this method are:\n", + "\n", + "* The predictive uncertainty of SNGP is computed using the [Laplace approximation](http://www.gaussianprocess.org/gpml/chapters/RW3.pdf){.external}. Therefore, theoretically, the posterior uncertainty of SNGP is different from that of an exact Gaussian process.\n", + "\n", + "* SNGP training needs a covariance reset step at the beginning of a new epoch. This can add a tiny amount of extra complexity to a training pipeline. This tutorial shows a simple way to implement this using Keras callbacks." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ck_O7S8r1boS" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "MOS9qFlW2o3J" + }, + "outputs": [], + "source": [ + "!pip install -U -q --use-deprecated=legacy-resolver tf-models-official tensorflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RCoGSYz-PIR7" + }, + "outputs": [], + "source": [ + "# refresh pkg_resources so it takes the changes into account.\n", + "import pkg_resources\n", + "import importlib\n", + "importlib.reload(pkg_resources)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nJKtMbtYNXHn" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import matplotlib.colors as colors\n", + "\n", + "import sklearn.datasets\n", + "\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "\n", + "import official.nlp.modeling.layers as nlp_layers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8RCyTrQO4ZXo" + }, + "source": [ + "Define visualization macros" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "o5VTrcxo3rI-" + }, + "outputs": [], + "source": [ + "plt.rcParams['figure.dpi'] = 140\n", + "\n", + "DEFAULT_X_RANGE = (-3.5, 3.5)\n", + "DEFAULT_Y_RANGE = (-2.5, 2.5)\n", + "DEFAULT_CMAP = colors.ListedColormap([\"#377eb8\", \"#ff7f00\"])\n", + "DEFAULT_NORM = colors.Normalize(vmin=0, vmax=1,)\n", + "DEFAULT_N_GRID = 100" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lL5HzdYT5x5J" + }, + "source": [ + "## The two moon dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nqazrSzhd24R" + }, + "source": [ + "Create the training and evaluation datasets from the [scikit-learn two moon dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_moons.html){.external}." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xJ_ul9Ii52mh" + }, + "outputs": [], + "source": [ + "def make_training_data(sample_size=500):\n", + " \"\"\"Create two moon training dataset.\"\"\"\n", + " train_examples, train_labels = sklearn.datasets.make_moons(\n", + " n_samples=2 * sample_size, noise=0.1)\n", + "\n", + " # Adjust data position slightly.\n", + " train_examples[train_labels == 0] += [-0.1, 0.2]\n", + " train_examples[train_labels == 1] += [0.1, -0.2]\n", + "\n", + " return train_examples, train_labels" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "goQkrxR_fFGd" + }, + "source": [ + "Evaluate the model's predictive behavior over the entire 2D input space." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Cj7ldkNw5-cT" + }, + "outputs": [], + "source": [ + "def make_testing_data(x_range=DEFAULT_X_RANGE, y_range=DEFAULT_Y_RANGE, n_grid=DEFAULT_N_GRID):\n", + " \"\"\"Create a mesh grid in 2D space.\"\"\"\n", + " # testing data (mesh grid over data space)\n", + " x = np.linspace(x_range[0], x_range[1], n_grid)\n", + " y = np.linspace(y_range[0], y_range[1], n_grid)\n", + " xv, yv = np.meshgrid(x, y)\n", + " return np.stack([xv.flatten(), yv.flatten()], axis=-1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "G9BYe4yqfeFa" + }, + "source": [ + "To evaluate model uncertainty, add an out-of-domain (OOD) dataset that belongs to a third class. The model never observes these OOD examples during training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5UHz2SU4feSI" + }, + "outputs": [], + "source": [ + "def make_ood_data(sample_size=500, means=(2.5, -1.75), vars=(0.01, 0.01)):\n", + " return np.random.multivariate_normal(\n", + " means, cov=np.diag(vars), size=sample_size)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "l1-jmJb45_at" + }, + "outputs": [], + "source": [ + "# Load the train, test and OOD datasets.\n", + "train_examples, train_labels = make_training_data(\n", + " sample_size=500)\n", + "test_examples = make_testing_data()\n", + "ood_examples = make_ood_data(sample_size=500)\n", + "\n", + "# Visualize\n", + "pos_examples = train_examples[train_labels == 0]\n", + "neg_examples = train_examples[train_labels == 1]\n", + "\n", + "plt.figure(figsize=(7, 5.5))\n", + "\n", + "plt.scatter(pos_examples[:, 0], pos_examples[:, 1], c=\"#377eb8\", alpha=0.5)\n", + "plt.scatter(neg_examples[:, 0], neg_examples[:, 1], c=\"#ff7f00\", alpha=0.5)\n", + "plt.scatter(ood_examples[:, 0], ood_examples[:, 1], c=\"red\", alpha=0.1)\n", + "\n", + "plt.legend([\"Positive\", \"Negative\", \"Out-of-Domain\"])\n", + "\n", + "plt.ylim(DEFAULT_Y_RANGE)\n", + "plt.xlim(DEFAULT_X_RANGE)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nlzxsnBBkybB" + }, + "source": [ + "Here, the blue and orange represent the positive and negative classes, and the red represents the OOD data. A model that quantifies the uncertainty well is expected to be confident when close to training data (i.e., $p(x_{test})$ close to 0 or 1), and be uncertain when far away from the training data regions (i.e., $p(x_{test})$ close to 0.5)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "RJ3i4n8li-Mv" + }, + "source": [ + "## The deterministic model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Utncgxs3lc4u" + }, + "source": [ + "### Define model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-z83-nq6jPlb" + }, + "source": [ + "Start from the (baseline) deterministic model: a multi-layer residual network (ResNet) with dropout regularization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "wCBRm8fc6CgY" + }, + "outputs": [], + "source": [ + "#@title\n", + "class DeepResNet(tf.keras.Model):\n", + " \"\"\"Defines a multi-layer residual network.\"\"\"\n", + " def __init__(self, num_classes, num_layers=3, num_hidden=128,\n", + " dropout_rate=0.1, **classifier_kwargs):\n", + " super().__init__()\n", + " # Defines class meta data.\n", + " self.num_hidden = num_hidden\n", + " self.num_layers = num_layers\n", + " self.dropout_rate = dropout_rate\n", + " self.classifier_kwargs = classifier_kwargs\n", + "\n", + " # Defines the hidden layers.\n", + " self.input_layer = tf.keras.layers.Dense(self.num_hidden, trainable=False)\n", + " self.dense_layers = [self.make_dense_layer() for _ in range(num_layers)]\n", + "\n", + " # Defines the output layer.\n", + " self.classifier = self.make_output_layer(num_classes)\n", + "\n", + " def call(self, inputs):\n", + " # Projects the 2d input data to high dimension.\n", + " hidden = self.input_layer(inputs)\n", + "\n", + " # Computes the ResNet hidden representations.\n", + " for i in range(self.num_layers):\n", + " resid = self.dense_layers[i](hidden)\n", + " resid = tf.keras.layers.Dropout(self.dropout_rate)(resid)\n", + " hidden += resid\n", + "\n", + " return self.classifier(hidden)\n", + "\n", + " def make_dense_layer(self):\n", + " \"\"\"Uses the Dense layer as the hidden layer.\"\"\"\n", + " return tf.keras.layers.Dense(self.num_hidden, activation=\"relu\")\n", + "\n", + " def make_output_layer(self, num_classes):\n", + " \"\"\"Uses the Dense layer as the output layer.\"\"\"\n", + " return tf.keras.layers.Dense(\n", + " num_classes, **self.classifier_kwargs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4u870GAen2aO" + }, + "source": [ + "This tutorial uses a six-layer ResNet with 128 hidden units." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bWL9wCnGpc4h" + }, + "outputs": [], + "source": [ + "resnet_config = dict(num_classes=2, num_layers=6, num_hidden=128)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "I47RY26wurgg" + }, + "outputs": [], + "source": [ + "resnet_model = DeepResNet(**resnet_config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "okQXf2F1ur16" + }, + "outputs": [], + "source": [ + "resnet_model.build((None, 2))\n", + "resnet_model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8ZzueLoImW0t" + }, + "source": [ + "### Train model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JfQ1k-IUukLt" + }, + "source": [ + "Configure the training parameters to use `SparseCategoricalCrossentropy` as the loss function and the Adam optimizer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "a9ZkfNXAnumV" + }, + "outputs": [], + "source": [ + "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", + "metrics = tf.keras.metrics.SparseCategoricalAccuracy(),\n", + "optimizer = tf.keras.optimizers.legacy.Adam(learning_rate=1e-4)\n", + "\n", + "train_config = dict(loss=loss, metrics=metrics, optimizer=optimizer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wEWzfHvf6A5_" + }, + "source": [ + "Train the model for 100 epochs with batch size 128." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1Fx6EtGXpzVr" + }, + "outputs": [], + "source": [ + "fit_config = dict(batch_size=128, epochs=100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cFwkbKIuqj7Y" + }, + "outputs": [], + "source": [ + "resnet_model.compile(**train_config)\n", + "resnet_model.fit(train_examples, train_labels, **fit_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eo6tKKd1rvBh" + }, + "source": [ + "### Visualize uncertainty" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "HZDMX7gZrZ-5" + }, + "outputs": [], + "source": [ + "#@title\n", + "def plot_uncertainty_surface(test_uncertainty, ax, cmap=None):\n", + " \"\"\"Visualizes the 2D uncertainty surface.\n", + " \n", + " For simplicity, assume these objects already exist in the memory:\n", + "\n", + " test_examples: Array of test examples, shape (num_test, 2).\n", + " train_labels: Array of train labels, shape (num_train, ).\n", + " train_examples: Array of train examples, shape (num_train, 2).\n", + " \n", + " Arguments:\n", + " test_uncertainty: Array of uncertainty scores, shape (num_test,).\n", + " ax: A matplotlib Axes object that specifies a matplotlib figure.\n", + " cmap: A matplotlib colormap object specifying the palette of the\n", + " predictive surface.\n", + "\n", + " Returns:\n", + " pcm: A matplotlib PathCollection object that contains the palette\n", + " information of the uncertainty plot.\n", + " \"\"\"\n", + " # Normalize uncertainty for better visualization.\n", + " test_uncertainty = test_uncertainty / np.max(test_uncertainty)\n", + "\n", + " # Set view limits.\n", + " ax.set_ylim(DEFAULT_Y_RANGE)\n", + " ax.set_xlim(DEFAULT_X_RANGE)\n", + "\n", + " # Plot normalized uncertainty surface.\n", + " pcm = ax.imshow(\n", + " np.reshape(test_uncertainty, [DEFAULT_N_GRID, DEFAULT_N_GRID]),\n", + " cmap=cmap,\n", + " origin=\"lower\",\n", + " extent=DEFAULT_X_RANGE + DEFAULT_Y_RANGE,\n", + " vmin=DEFAULT_NORM.vmin,\n", + " vmax=DEFAULT_NORM.vmax,\n", + " interpolation='bicubic',\n", + " aspect='auto')\n", + "\n", + " # Plot training data.\n", + " ax.scatter(train_examples[:, 0], train_examples[:, 1],\n", + " c=train_labels, cmap=DEFAULT_CMAP, alpha=0.5)\n", + " ax.scatter(ood_examples[:, 0], ood_examples[:, 1], c=\"red\", alpha=0.1)\n", + "\n", + " return pcm" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a1age2y0339T" + }, + "source": [ + "Now visualize the predictions of the deterministic model. First plot the class probability:\n", + "$$p(x) = softmax(logit(x))$$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4aqFgQOD40lb" + }, + "outputs": [], + "source": [ + "resnet_logits = resnet_model(test_examples)\n", + "resnet_probs = tf.nn.softmax(resnet_logits, axis=-1)[:, 0] # Take the probability for class 0." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rpflH2Qj33oN" + }, + "outputs": [], + "source": [ + "_, ax = plt.subplots(figsize=(7, 5.5))\n", + "\n", + "pcm = plot_uncertainty_surface(resnet_probs, ax=ax)\n", + "\n", + "plt.colorbar(pcm, ax=ax)\n", + "plt.title(\"Class Probability, Deterministic Model\")\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7ShGAB7FNYgU" + }, + "source": [ + "In this plot, the yellow and purple are the predictive probabilities for the two classes. The deterministic model did a good job in classifying the two known classes—blue and orange—with a nonlinear decision boundary. However, it is not **distance-aware**, and classified the never-observed red out-of-domain (OOD) examples confidently as the orange class.\n", + "\n", + "Visualize the model uncertainty by computing the [predictive variance](https://en.wikipedia.org/wiki/Bernoulli_distribution#Variance):\n", + "$$var(x) = p(x) * (1 - p(x))$$" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VxMce5D15Qwt" + }, + "outputs": [], + "source": [ + "resnet_uncertainty = resnet_probs * (1 - resnet_probs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fwdEfb5_6woh" + }, + "outputs": [], + "source": [ + "_, ax = plt.subplots(figsize=(7, 5.5))\n", + "\n", + "pcm = plot_uncertainty_surface(resnet_uncertainty, ax=ax)\n", + "\n", + "plt.colorbar(pcm, ax=ax)\n", + "plt.title(\"Predictive Uncertainty, Deterministic Model\")\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "alVunBEXhD-1" + }, + "source": [ + "In this plot, the yellow indicates high uncertainty, and the purple indicates low uncertainty. A deterministic ResNet's uncertainty depends only on the test examples' distance from the decision boundary. This leads the model to be over-confident when out of the training domain. The next section shows how SNGP behaves differently on this dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CwDa80iOh32J" + }, + "source": [ + "## The SNGP model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IbjNrmLu8oXv" + }, + "source": [ + "### Define SNGP model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "urtuCrk1Hf5P" + }, + "source": [ + "Let's now implement the SNGP model. Both the SNGP components, `SpectralNormalization` and `RandomFeatureGaussianProcess`, are available at the tensorflow_model's [built-in layers](https://github.com/tensorflow/models/tree/master/official/nlp/modeling/layers). " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7IlQwKCEGwpk" + }, + "source": [ + ">![SNGP](http://tensorflow.org/tutorials/understanding/images/sngp.png)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rp2O2iv8LSke" + }, + "source": [ + "Let's inspect these two components in more detail. (You can also jump to [the full SNGP model](#full-sngp-model) section to learn how SNGP is implemented.)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5n4NIt3QjKwl" + }, + "source": [ + "#### `SpectralNormalization` wrapper" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tE-Va7J2jR2X" + }, + "source": [ + "[`SpectralNormalization`](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/spectral_normalization.py){.external} is a Keras layer wrapper. It can be applied to an existing Dense layer like this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Dp8vqJBWLSq3" + }, + "outputs": [], + "source": [ + "dense = tf.keras.layers.Dense(units=10)\n", + "dense = nlp_layers.SpectralNormalization(dense, norm_multiplier=0.9)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "E9q25_6fRJRh" + }, + "source": [ + "Spectral normalization regularizes the hidden weight $W$ by gradually guiding its spectral norm (that is, the largest eigenvalue of $W$) toward the target value `norm_multiplier`).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cqt-DVzvqyAE" + }, + "source": [ + "Note: Usually it is preferable to set `norm_multiplier` to a value smaller than 1. However in practice, it can be also relaxed to a larger value to ensure the deep network has enough expressive power." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yqvxJUXBjBhV" + }, + "source": [ + "#### The Gaussian Process (GP) layer" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7rYfIgtrjHnB" + }, + "source": [ + "[`RandomFeatureGaussianProcess`](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/gaussian_process.py){.external} implements a [random-feature based approximation](https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf){.external} to a Gaussian process model that is end-to-end trainable with a deep neural network. Under the hood, the Gaussian process layer implements a two-layer network:\n", + "\n", + "$$logits(x) = \\Phi(x) \\beta, \\quad \\Phi(x)=\\sqrt{\\frac{2}{M}} * cos(Wx + b)$$\n", + "\n", + "Here, $x$ is the input, and $W$ and $b$ are frozen weights initialized randomly from Gaussian and Uniform distributions, respectively. (Therefore, $\\Phi(x)$ are called \"random features\".) $\\beta$ is the learnable kernel weight similar to that of a Dense layer. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NqnU39ui3wAE" + }, + "outputs": [], + "source": [ + "batch_size = 32\n", + "input_dim = 1024\n", + "num_classes = 10" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LrlVd-foRJno" + }, + "outputs": [], + "source": [ + "gp_layer = nlp_layers.RandomFeatureGaussianProcess(units=num_classes,\n", + " num_inducing=1024,\n", + " normalize_input=False,\n", + " scale_random_features=True,\n", + " gp_cov_momentum=-1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xxb8sSAg5AGf" + }, + "source": [ + "The main parameters of the GP layers are:\n", + "\n", + "* `units`: The dimension of the output logits.\n", + "* `num_inducing`: The dimension $M$ of the hidden weight $W$. Default to 1024.\n", + "* `normalize_input`: Whether to apply layer normalization to the input $x$.\n", + "* `scale_random_features`: Whether to apply the scale $\\sqrt{2/M}$ to the hidden output.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xgzw09gS03ae" + }, + "source": [ + "Note: For a deep neural network that is sensitive to the learning rate (for example, ResNet-50 and ResNet-110), it is generally recommended to set `normalize_input=True` to stabilize training, and set `scale_random_features=False` to avoid the learning rate from being modified in unexpected ways when passing through the GP layer." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pZkcKw-u7XRp" + }, + "source": [ + "* `gp_cov_momentum` controls how the model covariance is computed. If set to a positive value (for example, `0.999`), the covariance matrix is computed using the momentum-based moving average update (similar to batch normalization). If set to `-1`, the covariance matrix is updated without momentum." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "P13X7Adt-c2d" + }, + "source": [ + "Note: The momentum-based update method can be sensitive to batch size. Therefore it is generally recommended to set `gp_cov_momentum=-1` to compute the covariance exactly. For this to work properly, the covariance matrix estimator needs to be reset at the beginning of a new epoch in order to avoid counting the same data twice. For `RandomFeatureGaussianProcess`, this can be done by calling its `reset_covariance_matrix()`. The next section shows an easy implementation of this using Keras' built-in API.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0AA492qA1biZ" + }, + "source": [ + "Given a batch input with shape `(batch_size, input_dim)`, the GP layer returns a `logits` tensor (shape `(batch_size, num_classes)`) for prediction, and also `covmat` tensor (shape `(batch_size, batch_size)`) which is the posterior covariance matrix of the batch logits." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yOXxBYnMi1v4" + }, + "outputs": [], + "source": [ + "embedding = tf.random.normal(shape=(batch_size, input_dim))\n", + "\n", + "logits, covmat = gp_layer(embedding)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ALBqcAtwDNiO" + }, + "source": [ + "Note: Notice that under this implementation of the SNGP model, the predictive logits $logit(x_{test})$ for all classes share the same covariance matrix $var(x_{test})$, which describes the distance between $x_{test}$ from the training data.\n", + "\n", + "Theoretically, it is possible to extend the algorithm to compute different variance values for different classes (as introduced in the [original SNGP paper](https://arxiv.org/abs/2006.10108){.external}). However, this is difficult to scale to problems with large output spaces (such as classification with ImageNet or language modeling)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "II3GVzjJhu5Z" + }, + "source": [ + "\n", + "#### The full SNGP model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F5Fm0NUlLTHd" + }, + "source": [ + "Given the base class `DeepResNet`, the SNGP model can be implemented easily by modifying the residual network's hidden and output layers. For compatibility with Keras `model.fit()` API, also modify the model's `call()` method so it only outputs `logits` during training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Dzx4FsO97QZv" + }, + "outputs": [], + "source": [ + "class DeepResNetSNGP(DeepResNet):\n", + " def __init__(self, spec_norm_bound=0.9, **kwargs):\n", + " self.spec_norm_bound = spec_norm_bound\n", + " super().__init__(**kwargs)\n", + "\n", + " def make_dense_layer(self):\n", + " \"\"\"Applies spectral normalization to the hidden layer.\"\"\"\n", + " dense_layer = super().make_dense_layer()\n", + " return nlp_layers.SpectralNormalization(\n", + " dense_layer, norm_multiplier=self.spec_norm_bound)\n", + "\n", + " def make_output_layer(self, num_classes):\n", + " \"\"\"Uses Gaussian process as the output layer.\"\"\"\n", + " return nlp_layers.RandomFeatureGaussianProcess(\n", + " num_classes,\n", + " gp_cov_momentum=-1,\n", + " **self.classifier_kwargs)\n", + "\n", + " def call(self, inputs, training=False, return_covmat=False):\n", + " # Gets logits and a covariance matrix from the GP layer.\n", + " logits, covmat = super().call(inputs)\n", + "\n", + " # Returns only logits during training.\n", + " if not training and return_covmat:\n", + " return logits, covmat\n", + "\n", + " return logits" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "02SvlnDFyszm" + }, + "source": [ + "Use the same architecture as the deterministic model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QiDcC5ipyqMU" + }, + "outputs": [], + "source": [ + "resnet_config" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "D9-imQtDyZL5" + }, + "outputs": [], + "source": [ + "sngp_model = DeepResNetSNGP(**resnet_config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "12P9UxlCyq6k" + }, + "outputs": [], + "source": [ + "sngp_model.build((None, 2))\n", + "sngp_model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BlCgMRrwkXgN" + }, + "source": [ + "\n", + "Implement a Keras callback to reset the covariance matrix at the beginning of a new epoch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "W6Wr8D_0n-cQ" + }, + "outputs": [], + "source": [ + "class ResetCovarianceCallback(tf.keras.callbacks.Callback):\n", + "\n", + " def on_epoch_begin(self, epoch, logs=None):\n", + " \"\"\"Resets covariance matrix at the beginning of the epoch.\"\"\"\n", + " if epoch > 0:\n", + " self.model.classifier.reset_covariance_matrix()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NMiz_mYawEPh" + }, + "source": [ + "Add this callback to the `DeepResNetSNGP` model class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yrSi0ZD5zaDf" + }, + "outputs": [], + "source": [ + "class DeepResNetSNGPWithCovReset(DeepResNetSNGP):\n", + " def fit(self, *args, **kwargs):\n", + " \"\"\"Adds ResetCovarianceCallback to model callbacks.\"\"\"\n", + " kwargs[\"callbacks\"] = list(kwargs.get(\"callbacks\", []))\n", + " kwargs[\"callbacks\"].append(ResetCovarianceCallback())\n", + "\n", + " return super().fit(*args, **kwargs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "asIwYqGlwJcP" + }, + "source": [ + "### Train model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4YRzayOCopt9" + }, + "source": [ + "Use `tf.keras.model.fit` to train the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "coazo53nwJqv" + }, + "outputs": [], + "source": [ + "sngp_model = DeepResNetSNGPWithCovReset(**resnet_config)\n", + "sngp_model.compile(**train_config)\n", + "sngp_model.fit(train_examples, train_labels, **fit_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KTONd8vowgEP" + }, + "source": [ + "### Visualize uncertainty" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yhkpsiy10l9d" + }, + "source": [ + "First compute the predictive logits and variances." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bqRPqeEavi4Z" + }, + "outputs": [], + "source": [ + "sngp_logits, sngp_covmat = sngp_model(test_examples, return_covmat=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "p7w0iW_L0cU8" + }, + "outputs": [], + "source": [ + "sngp_variance = tf.linalg.diag_part(sngp_covmat)[:, None]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YLbz_EZF1Gay" + }, + "source": [ + "\n", + "Now compute the posterior predictive probability. The classic method for computing the predictive probability of a probabilistic model is to use Monte Carlo sampling, i.e.,\n", + "\n", + "$$E(p(x)) = \\frac{1}{M} \\sum_{m=1}^M logit_m(x), $$\n", + "\n", + "where $M$ is the sample size, and $logit_m(x)$ are random samples from the SNGP posterior $MultivariateNormal$(`sngp_logits`,`sngp_covmat`). However, this approach can be slow for latency-sensitive applications such as autonomous driving or real-time bidding. Instead, you can approximate $E(p(x))$ using the [mean-field method](https://arxiv.org/abs/2006.07584){.external}:\n", + "\n", + "$$E(p(x)) \\approx softmax(\\frac{logit(x)}{\\sqrt{1+ \\lambda * \\sigma^2(x)}})$$\n", + "\n", + "where $\\sigma^2(x)$ is the SNGP variance, and $\\lambda$ is often chosen as $\\pi/8$ or $3/\\pi^2$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1A9NYMhd0iZ-" + }, + "outputs": [], + "source": [ + "sngp_logits_adjusted = sngp_logits / tf.sqrt(1. + (np.pi / 8.) * sngp_variance)\n", + "sngp_probs = tf.nn.softmax(sngp_logits_adjusted, axis=-1)[:, 0]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bNVs_KO-5HdL" + }, + "source": [ + "Note: Instead of fixing $\\lambda$ to a fixed value, you can also treat it as a hyperparameter, and tune it to optimize the model's calibration performance. This is known as [temperature scaling](http://proceedings.mlr.press/v70/guo17a.html){.external} in the deep learning uncertainty literature. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DlYPlUbJfBFa" + }, + "source": [ + "This mean-field method is implemented as a built-in function `layers.gaussian_process.mean_field_logits`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hgb3WSaY8iQY" + }, + "outputs": [], + "source": [ + "def compute_posterior_mean_probability(logits, covmat, lambda_param=np.pi / 8.):\n", + " # Computes uncertainty-adjusted logits using the built-in method.\n", + " logits_adjusted = nlp_layers.gaussian_process.mean_field_logits(\n", + " logits, covmat, mean_field_factor=lambda_param)\n", + " \n", + " return tf.nn.softmax(logits_adjusted, axis=-1)[:, 0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kVToZpG7QqS3" + }, + "outputs": [], + "source": [ + "sngp_logits, sngp_covmat = sngp_model(test_examples, return_covmat=True)\n", + "sngp_probs = compute_posterior_mean_probability(sngp_logits, sngp_covmat)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bVi_Whpwe3O4" + }, + "source": [ + "### SNGP Summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "dRmmxuO41BV4" + }, + "outputs": [], + "source": [ + "#@title\n", + "\n", + "def plot_predictions(pred_probs, model_name=\"\"):\n", + " \"\"\"Plot normalized class probabilities and predictive uncertainties.\"\"\"\n", + " # Compute predictive uncertainty.\n", + " uncertainty = pred_probs * (1. - pred_probs)\n", + "\n", + " # Initialize the plot axes.\n", + " fig, axs = plt.subplots(1, 2, figsize=(14, 5))\n", + "\n", + " # Plots the class probability.\n", + " pcm_0 = plot_uncertainty_surface(pred_probs, ax=axs[0])\n", + " # Plots the predictive uncertainty.\n", + " pcm_1 = plot_uncertainty_surface(uncertainty, ax=axs[1])\n", + "\n", + " # Adds color bars and titles.\n", + " fig.colorbar(pcm_0, ax=axs[0])\n", + " fig.colorbar(pcm_1, ax=axs[1])\n", + "\n", + " axs[0].set_title(f\"Class Probability, {model_name}\")\n", + " axs[1].set_title(f\"(Normalized) Predictive Uncertainty, {model_name}\")\n", + "\n", + " plt.show() " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R9kY5dJg8fEi" + }, + "source": [ + "You can now put everything together. The entire procedure—training, evaluation and uncertainty computation—can be done in just five lines:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NQtUAG4-ftqe" + }, + "outputs": [], + "source": [ + "def train_and_test_sngp(train_examples, test_examples):\n", + " sngp_model = DeepResNetSNGPWithCovReset(**resnet_config)\n", + "\n", + " sngp_model.compile(**train_config)\n", + " sngp_model.fit(train_examples, train_labels, verbose=0, **fit_config)\n", + "\n", + " sngp_logits, sngp_covmat = sngp_model(test_examples, return_covmat=True)\n", + " sngp_probs = compute_posterior_mean_probability(sngp_logits, sngp_covmat)\n", + "\n", + " return sngp_probs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Dl3N7kHJ283w" + }, + "outputs": [], + "source": [ + "sngp_probs = train_and_test_sngp(train_examples, test_examples)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OAUGrXSv6k3R" + }, + "source": [ + "Visualize the class probability (left) and the predictive uncertainty (right) of the SNGP model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rxt3CY51A_jq" + }, + "outputs": [], + "source": [ + "plot_predictions(sngp_probs, model_name=\"SNGP\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "raHP5Vuiuku9" + }, + "source": [ + "Remember that in the class probability plot (left), the yellow and purple are class probabilities. When close to the training data domain, SNGP correctly classifies the examples with high confidence (i.e., assigning near 0 or 1 probability). When far away from the training data, SNGP gradually becomes less confident, and its predictive probability becomes close to 0.5 while the (normalized) model uncertainty rises to 1." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u4_VRi9w7Km3" + }, + "source": [ + "Compare this to the uncertainty surface of the deterministic model: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aMAdstYZ7T_w" + }, + "outputs": [], + "source": [ + "plot_predictions(resnet_probs, model_name=\"Deterministic\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mao9L-LYE1Nl" + }, + "source": [ + "As mentioned earlier, a deterministic model is not _distance-aware_. Its uncertainty is defined by the distance of the test example from the decision boundary. This leads the model to produce overconfident predictions for the out-of-domain examples (red)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EKURpzOf0oNq" + }, + "source": [ + "## Comparison with other uncertainty approaches" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S1DPELWE6LL8" + }, + "source": [ + "This section compares the uncertainty of SNGP with [Monte Carlo dropout](https://arxiv.org/abs/1506.02142){.external} and [Deep ensemble](https://arxiv.org/abs/1612.01474){.external}.\n", + "\n", + "Both of these methods are based on Monte Carlo averaging of multiple forward passes of deterministic models. First, set the ensemble size $M$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xLqkihbk8Dey" + }, + "outputs": [], + "source": [ + "num_ensemble = 10" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jM5AQmtIAatd" + }, + "source": [ + "### Monte Carlo dropout" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZBzp2LBt7-kj" + }, + "source": [ + "Given a trained neural network with Dropout layers, Monte Carlo dropout computes the mean predictive probability\n", + "\n", + "$$E(p(x)) = \\frac{1}{M}\\sum_{m=1}^M softmax(logit_m(x))$$\n", + "\n", + "by averaging over multiple Dropout-enabled forward passes $\\{logit_m(x)\\}_{m=1}^M$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "I7R2WBgq4-OC" + }, + "outputs": [], + "source": [ + "def mc_dropout_sampling(test_examples):\n", + " # Enable dropout during inference.\n", + " return resnet_model(test_examples, training=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "r6oXgaDZAiD0" + }, + "outputs": [], + "source": [ + "# Monte Carlo dropout inference.\n", + "dropout_logit_samples = [mc_dropout_sampling(test_examples) for _ in range(num_ensemble)]\n", + "dropout_prob_samples = [tf.nn.softmax(dropout_logits, axis=-1)[:, 0] for dropout_logits in dropout_logit_samples]\n", + "dropout_probs = tf.reduce_mean(dropout_prob_samples, axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_oUNtbVG-YuI" + }, + "outputs": [], + "source": [ + "dropout_probs = tf.reduce_mean(dropout_prob_samples, axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "J-mhyp8hAiPn" + }, + "outputs": [], + "source": [ + "plot_predictions(dropout_probs, model_name=\"MC Dropout\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mwtj2vTB75cF" + }, + "source": [ + "### Deep ensemble" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L-Z2veGJ9ZgY" + }, + "source": [ + "Deep ensemble is a state-of-the-art (but expensive) method for deep learning uncertainty. To train a Deep ensemble, first train $M$ ensemble members." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "a43hxiJC3Kla" + }, + "outputs": [], + "source": [ + "# Deep ensemble training\n", + "resnet_ensemble = []\n", + "for _ in range(num_ensemble):\n", + " resnet_model = DeepResNet(**resnet_config)\n", + " resnet_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n", + " resnet_model.fit(train_examples, train_labels, verbose=0, **fit_config)\n", + "\n", + " resnet_ensemble.append(resnet_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Al7uM-fn_ZE1" + }, + "source": [ + "Collect logits and compute the mean predictive probability $E(p(x)) = \\frac{1}{M}\\sum_{m=1}^M softmax(logit_m(x))$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "c6E9PntV3Mue" + }, + "outputs": [], + "source": [ + "# Deep ensemble inference\n", + "ensemble_logit_samples = [model(test_examples) for model in resnet_ensemble]\n", + "ensemble_prob_samples = [tf.nn.softmax(logits, axis=-1)[:, 0] for logits in ensemble_logit_samples]\n", + "ensemble_probs = tf.reduce_mean(ensemble_prob_samples, axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Z_JhYftG-NKR" + }, + "outputs": [], + "source": [ + "plot_predictions(ensemble_probs, model_name=\"Deep ensemble\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GH33oVvV5-ez" + }, + "source": [ + "Both the Monte Carlo Dropout and Deep ensemble methods improve the model's uncertainty ability by making the decision boundary less certain. However, they both inherit the deterministic deep network's limitation in lacking distance awareness." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "x9nAaWuYfD03" + }, + "source": [ + "## Summary" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-ryMkRllFHOC" + }, + "source": [ + "In this tutorial, you have:\n", + "* Implemented the SNGP model on a deep classifier to improve its distance awareness.\n", + "* Trained the SNGP model end-to-end using Keras `Model.fit` API.\n", + "* Visualized the uncertainty behavior of SNGP.\n", + "* Compared the uncertainty behavior between SNGP, Monte Carlo dropout and deep ensemble models." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UoTekiQmkZXF" + }, + "source": [ + "## Resources and further reading" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HoIikRybke-b" + }, + "source": [ + "* Check out the [SNGP-BERT tutorial](https://www.tensorflow.org/text/tutorials/uncertainty_quantification_with_sngp_bert) for an example of applying SNGP on a BERT model for uncertainty-aware natural language understanding.\n", + "* Go to the [Uncertainty Baselines GitHub repo](https://github.com/google/uncertainty-baselines){.external} for the implementation of SNGP model (and many other uncertainty methods) on a wide variety of benchmark datasets (for example, [CIFAR](https://www.tensorflow.org/datasets/catalog/cifar100), [ImageNet](https://www.tensorflow.org/datasets/catalog/imagenet2012), [Jigsaw toxicity detection](https://www.tensorflow.org/datasets/catalog/wikipedia_toxicity_subtypes), etc).\n", + "* For a deeper understanding of the SNGP method, check out the paper titled [Simple and Principled Uncertainty Estimation with Deterministic Deep Learning via Distance Awareness](https://arxiv.org/abs/2006.10108){.external}.\n" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "sngp.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/video/transfer_learning_with_movinet.ipynb b/site/en/tutorials/video/transfer_learning_with_movinet.ipynb new file mode 100644 index 00000000000..df395295ba9 --- /dev/null +++ b/site/en/tutorials/video/transfer_learning_with_movinet.ipynb @@ -0,0 +1,825 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "EOgDUDMAG6mn" + }, + "source": [ + "##### Copyright 2022 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "B3PsBDmGG_W8" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ifkGYxdCHIof" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sWxDDkRwLVMC" + }, + "source": [ + "# Transfer learning for video classification with MoViNet\n", + "\n", + "MoViNets (Mobile Video Networks) provide a family of efficient video classification models, supporting inference on streaming video. In this tutorial, you will use a pre-trained MoViNet model to classify videos, specifically for an action recognition task, from the [UCF101 dataset](https://www.crcv.ucf.edu/data/UCF101.php). A pre-trained model is a saved network that was previously trained on a larger dataset. You can find more details about MoViNets in the [MoViNets: Mobile Video Networks for Efficient Video Recognition](https://arxiv.org/abs/2103.11511) paper by Kondratyuk, D. et al. (2021). In this tutorial, you will: \n", + "\n", + "* Learn how to download a pre-trained MoViNet model\n", + "* Create a new model using a pre-trained model with a new classifier by freezing the convolutional base of the MoViNet model\n", + "* Replace the classifier head with the number of labels of a new dataset\n", + "* Perform transfer learning on the [UCF101 dataset](https://www.crcv.ucf.edu/data/UCF101.php)\n", + "\n", + "The model downloaded in this tutorial is from [official/projects/movinet](https://github.com/tensorflow/models/tree/master/official/projects/movinet). This repository contains a collection of MoViNet models that TF Hub uses in the TensorFlow 2 SavedModel format.\n", + "\n", + "This transfer learning tutorial is the third part in a series of TensorFlow video tutorials. Here are the other three tutorials:\n", + "\n", + "- [Load video data](https://www.tensorflow.org/tutorials/load_data/video): This tutorial explains much of the code used in this document; in particular, how to preprocess and load data through the `FrameGenerator` class is explained in more detail.\n", + "- [Build a 3D CNN model for video classification](https://www.tensorflow.org/tutorials/video/video_classification). Note that this tutorial uses a (2+1)D CNN that decomposes the spatial and temporal aspects of 3D data; if you are using volumetric data such as an MRI scan, consider using a 3D CNN instead of a (2+1)D CNN.\n", + "- [MoViNet for streaming action recognition](https://www.tensorflow.org/hub/tutorials/movinet): Get familiar with the MoViNet models that are available on TF Hub." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GidiisyXwK--" + }, + "source": [ + "## Setup\n", + "\n", + "Begin by installing and importing some necessary libraries, including:\n", + "[remotezip](https://github.com/gtsystem/python-remotezip) to inspect the contents of a ZIP file, [tqdm](https://github.com/tqdm/tqdm) to use a progress bar, [OpenCV](https://opencv.org/) to process video files (ensure that `opencv-python` and `opencv-python-headless` are the same version), and TensorFlow models ([`tf-models-official`](https://github.com/tensorflow/models/tree/master/official)) to download the pre-trained MoViNet model. The TensorFlow models package are a collection of models that use TensorFlow’s high-level APIs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nubWhqYdwEXD" + }, + "outputs": [], + "source": [ + "!pip install remotezip tqdm opencv-python==4.5.2.52 opencv-python-headless==4.5.2.52 tf-models-official" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QImPsudoK9JI" + }, + "outputs": [], + "source": [ + "import tqdm\n", + "import random\n", + "import pathlib\n", + "import itertools\n", + "import collections\n", + "\n", + "import cv2\n", + "import numpy as np\n", + "import remotezip as rz\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import keras\n", + "import tensorflow as tf\n", + "import tensorflow_hub as hub\n", + "from tensorflow.keras import layers\n", + "from tensorflow.keras.optimizers import Adam\n", + "from tensorflow.keras.losses import SparseCategoricalCrossentropy\n", + "\n", + "# Import the MoViNet model from TensorFlow Models (tf-models-official) for the MoViNet model\n", + "from official.projects.movinet.modeling import movinet\n", + "from official.projects.movinet.modeling import movinet_model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2w3H4dfOPfnm" + }, + "source": [ + "## Load data\n", + " \n", + "The hidden cell below defines helper functions to download a slice of data from the UCF-101 dataset, and load it into a `tf.data.Dataset`. The [Loading video data tutorial](https://www.tensorflow.org/tutorials/load_data/video) provides a detailed walkthrough of this code.\n", + "\n", + "The `FrameGenerator` class at the end of the hidden block is the most important utility here. It creates an iterable object that can feed data into the TensorFlow data pipeline. Specifically, this class contains a Python generator that loads the video frames along with its encoded label. The generator (`__call__`) function yields the frame array produced by `frames_from_video_file` and a one-hot encoded vector of the label associated with the set of frames.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "fwEhJ13_PSy6" + }, + "outputs": [], + "source": [ + "#@title \n", + "\n", + "def list_files_per_class(zip_url):\n", + " \"\"\"\n", + " List the files in each class of the dataset given the zip URL.\n", + "\n", + " Args:\n", + " zip_url: URL from which the files can be unzipped. \n", + "\n", + " Return:\n", + " files: List of files in each of the classes.\n", + " \"\"\"\n", + " files = []\n", + " with rz.RemoteZip(URL) as zip:\n", + " for zip_info in zip.infolist():\n", + " files.append(zip_info.filename)\n", + " return files\n", + "\n", + "def get_class(fname):\n", + " \"\"\"\n", + " Retrieve the name of the class given a filename.\n", + "\n", + " Args:\n", + " fname: Name of the file in the UCF101 dataset.\n", + "\n", + " Return:\n", + " Class that the file belongs to.\n", + " \"\"\"\n", + " return fname.split('_')[-3]\n", + "\n", + "def get_files_per_class(files):\n", + " \"\"\"\n", + " Retrieve the files that belong to each class. \n", + "\n", + " Args:\n", + " files: List of files in the dataset.\n", + "\n", + " Return:\n", + " Dictionary of class names (key) and files (values).\n", + " \"\"\"\n", + " files_for_class = collections.defaultdict(list)\n", + " for fname in files:\n", + " class_name = get_class(fname)\n", + " files_for_class[class_name].append(fname)\n", + " return files_for_class\n", + "\n", + "def download_from_zip(zip_url, to_dir, file_names):\n", + " \"\"\"\n", + " Download the contents of the zip file from the zip URL.\n", + "\n", + " Args:\n", + " zip_url: Zip URL containing data.\n", + " to_dir: Directory to download data to.\n", + " file_names: Names of files to download.\n", + " \"\"\"\n", + " with rz.RemoteZip(zip_url) as zip:\n", + " for fn in tqdm.tqdm(file_names):\n", + " class_name = get_class(fn)\n", + " zip.extract(fn, str(to_dir / class_name))\n", + " unzipped_file = to_dir / class_name / fn\n", + "\n", + " fn = pathlib.Path(fn).parts[-1]\n", + " output_file = to_dir / class_name / fn\n", + " unzipped_file.rename(output_file,)\n", + "\n", + "def split_class_lists(files_for_class, count):\n", + " \"\"\"\n", + " Returns the list of files belonging to a subset of data as well as the remainder of\n", + " files that need to be downloaded.\n", + "\n", + " Args:\n", + " files_for_class: Files belonging to a particular class of data.\n", + " count: Number of files to download.\n", + "\n", + " Return:\n", + " split_files: Files belonging to the subset of data.\n", + " remainder: Dictionary of the remainder of files that need to be downloaded.\n", + " \"\"\"\n", + " split_files = []\n", + " remainder = {}\n", + " for cls in files_for_class:\n", + " split_files.extend(files_for_class[cls][:count])\n", + " remainder[cls] = files_for_class[cls][count:]\n", + " return split_files, remainder\n", + "\n", + "def download_ufc_101_subset(zip_url, num_classes, splits, download_dir):\n", + " \"\"\"\n", + " Download a subset of the UFC101 dataset and split them into various parts, such as\n", + " training, validation, and test. \n", + "\n", + " Args:\n", + " zip_url: Zip URL containing data.\n", + " num_classes: Number of labels.\n", + " splits: Dictionary specifying the training, validation, test, etc. (key) division of data \n", + " (value is number of files per split).\n", + " download_dir: Directory to download data to.\n", + "\n", + " Return:\n", + " dir: Posix path of the resulting directories containing the splits of data.\n", + " \"\"\"\n", + " files = list_files_per_class(zip_url)\n", + " for f in files:\n", + " tokens = f.split('/')\n", + " if len(tokens) <= 2:\n", + " files.remove(f) # Remove that item from the list if it does not have a filename\n", + "\n", + " files_for_class = get_files_per_class(files)\n", + "\n", + " classes = list(files_for_class.keys())[:num_classes]\n", + "\n", + " for cls in classes:\n", + " new_files_for_class = files_for_class[cls]\n", + " random.shuffle(new_files_for_class)\n", + " files_for_class[cls] = new_files_for_class\n", + "\n", + " # Only use the number of classes you want in the dictionary\n", + " files_for_class = {x: files_for_class[x] for x in list(files_for_class)[:num_classes]}\n", + "\n", + " dirs = {}\n", + " for split_name, split_count in splits.items():\n", + " print(split_name, \":\")\n", + " split_dir = download_dir / split_name\n", + " split_files, files_for_class = split_class_lists(files_for_class, split_count)\n", + " download_from_zip(zip_url, split_dir, split_files)\n", + " dirs[split_name] = split_dir\n", + "\n", + " return dirs\n", + "\n", + "def format_frames(frame, output_size):\n", + " \"\"\"\n", + " Pad and resize an image from a video.\n", + "\n", + " Args:\n", + " frame: Image that needs to resized and padded. \n", + " output_size: Pixel size of the output frame image.\n", + "\n", + " Return:\n", + " Formatted frame with padding of specified output size.\n", + " \"\"\"\n", + " frame = tf.image.convert_image_dtype(frame, tf.float32)\n", + " frame = tf.image.resize_with_pad(frame, *output_size)\n", + " return frame\n", + "\n", + "def frames_from_video_file(video_path, n_frames, output_size = (224,224), frame_step = 15):\n", + " \"\"\"\n", + " Creates frames from each video file present for each category.\n", + "\n", + " Args:\n", + " video_path: File path to the video.\n", + " n_frames: Number of frames to be created per video file.\n", + " output_size: Pixel size of the output frame image.\n", + "\n", + " Return:\n", + " An NumPy array of frames in the shape of (n_frames, height, width, channels).\n", + " \"\"\"\n", + " # Read each video frame by frame\n", + " result = []\n", + " src = cv2.VideoCapture(str(video_path)) \n", + "\n", + " video_length = src.get(cv2.CAP_PROP_FRAME_COUNT)\n", + "\n", + " need_length = 1 + (n_frames - 1) * frame_step\n", + "\n", + " if need_length > video_length:\n", + " start = 0\n", + " else:\n", + " max_start = video_length - need_length\n", + " start = random.randint(0, max_start + 1)\n", + "\n", + " src.set(cv2.CAP_PROP_POS_FRAMES, start)\n", + " # ret is a boolean indicating whether read was successful, frame is the image itself\n", + " ret, frame = src.read()\n", + " result.append(format_frames(frame, output_size))\n", + "\n", + " for _ in range(n_frames - 1):\n", + " for _ in range(frame_step):\n", + " ret, frame = src.read()\n", + " if ret:\n", + " frame = format_frames(frame, output_size)\n", + " result.append(frame)\n", + " else:\n", + " result.append(np.zeros_like(result[0]))\n", + " src.release()\n", + " result = np.array(result)[..., [2, 1, 0]]\n", + "\n", + " return result\n", + "\n", + "class FrameGenerator:\n", + " def __init__(self, path, n_frames, training = False):\n", + " \"\"\" Returns a set of frames with their associated label. \n", + "\n", + " Args:\n", + " path: Video file paths.\n", + " n_frames: Number of frames. \n", + " training: Boolean to determine if training dataset is being created.\n", + " \"\"\"\n", + " self.path = path\n", + " self.n_frames = n_frames\n", + " self.training = training\n", + " self.class_names = sorted(set(p.name for p in self.path.iterdir() if p.is_dir()))\n", + " self.class_ids_for_name = dict((name, idx) for idx, name in enumerate(self.class_names))\n", + "\n", + " def get_files_and_class_names(self):\n", + " video_paths = list(self.path.glob('*/*.avi'))\n", + " classes = [p.parent.name for p in video_paths] \n", + " return video_paths, classes\n", + "\n", + " def __call__(self):\n", + " video_paths, classes = self.get_files_and_class_names()\n", + "\n", + " pairs = list(zip(video_paths, classes))\n", + "\n", + " if self.training:\n", + " random.shuffle(pairs)\n", + "\n", + " for path, name in pairs:\n", + " video_frames = frames_from_video_file(path, self.n_frames) \n", + " label = self.class_ids_for_name[name] # Encode labels\n", + " yield video_frames, label" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vDHrNLZkPSR9" + }, + "outputs": [], + "source": [ + "URL = 'https://storage.googleapis.com/thumos14_files/UCF101_videos.zip'\n", + "download_dir = pathlib.Path('./UCF101_subset/')\n", + "subset_paths = download_ufc_101_subset(URL, \n", + " num_classes = 10, \n", + " splits = {\"train\": 30, \"test\": 20}, \n", + " download_dir = download_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aYYShfhMx9DW" + }, + "source": [ + "Create the training and test datasets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-twTu3_Bx-iJ" + }, + "outputs": [], + "source": [ + "batch_size = 8\n", + "num_frames = 8\n", + "\n", + "output_signature = (tf.TensorSpec(shape = (None, None, None, 3), dtype = tf.float32),\n", + " tf.TensorSpec(shape = (), dtype = tf.int16))\n", + "\n", + "train_ds = tf.data.Dataset.from_generator(FrameGenerator(subset_paths['train'], num_frames, training = True),\n", + " output_signature = output_signature)\n", + "train_ds = train_ds.batch(batch_size)\n", + "\n", + "test_ds = tf.data.Dataset.from_generator(FrameGenerator(subset_paths['test'], num_frames),\n", + " output_signature = output_signature)\n", + "test_ds = test_ds.batch(batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d7stgmuBCGQT" + }, + "source": [ + "The labels generated here represent the encoding of the classes. For instance, 'ApplyEyeMakeup' is mapped to the integer Take a look at the labels of the training data to ensure that the dataset has been sufficiently shuffled. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "k9L2-toXCOQq" + }, + "outputs": [], + "source": [ + "for frames, labels in train_ds.take(10):\n", + " print(labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YZ3qwZnpfy9c" + }, + "source": [ + "Take a look at the shape of the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "b6MqP4m2fyQT" + }, + "outputs": [], + "source": [ + "print(f\"Shape: {frames.shape}\")\n", + "print(f\"Label: {labels.shape}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lxbhPqXGvc_F" + }, + "source": [ + "## What are MoViNets?\n", + "\n", + "As mentioned previously, [MoViNets](https://arxiv.org/abs/2103.11511) are video classification models used for streaming video or online inference in tasks, such as action recognition. Consider using MoViNets to classify your video data for action recognition.\n", + "\n", + "A 2D frame based classifier is efficient and simple to run over whole videos, or streaming one frame at a time. Because they can't take temporal context into account they have limited accuracy and may give inconsistent outputs from frame to frame.\n", + "\n", + "A simple 3D CNN uses bidirectional temporal context which can increase accuracy and temporal consistency. These networks may require more resources and because they look into the future they can't be used for streaming data.\n", + "\n", + "![Standard convolution](https://www.tensorflow.org/images/tutorials/video/standard_convolution.png)\n", + "\n", + "The MoViNet architecture uses 3D convolutions that are \"causal\" along the time axis (like `layers.Conv1D` with `padding=\"causal\"`). This gives some of the advantages of both approaches, mainly it allow for efficient streaming.\n", + "\n", + "![Causal convolution](https://www.tensorflow.org/images/tutorials/video/causal_convolution.png)\n", + "\n", + "Causal convolution ensures that the output at time *t* is computed using only inputs up to time *t*. To demonstrate how this can make streaming more efficient, start with a simpler example you may be familiar with: an RNN. The RNN passes state forward through time:\n", + "\n", + "![RNN model](https://www.tensorflow.org/images/tutorials/video/rnn_comparison.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dMvDkgfFZC6a" + }, + "outputs": [], + "source": [ + "gru = layers.GRU(units=4, return_sequences=True, return_state=True)\n", + "\n", + "inputs = tf.random.normal(shape=[1, 10, 8]) # (batch, sequence, channels)\n", + "\n", + "result, state = gru(inputs) # Run it all at once" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T7xyb5C4bTs7" + }, + "source": [ + "By setting the RNN's `return_sequences=True` argument you ask it to return the state at the end of the computation. This allows you to pause and then continue where you left off, to get exactly the same result:\n", + "\n", + "![States passing in RNNs](https://www.tensorflow.org/images/tutorials/video/rnn_state_passing.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bI8FOPRRXXPa" + }, + "outputs": [], + "source": [ + "first_half, state = gru(inputs[:, :5, :]) # run the first half, and capture the state\n", + "second_half, _ = gru(inputs[:,5:, :], initial_state=state) # Use the state to continue where you left off.\n", + "\n", + "print(np.allclose(result[:, :5,:], first_half))\n", + "print(np.allclose(result[:, 5:,:], second_half))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KM3MArumY_Qk" + }, + "source": [ + "Causal convolutions can be used the same way, if handled with care. This technique was used in the [Fast Wavenet Generation Algorithm](https://arxiv.org/abs/1611.09482) by Le Paine et al. In the [MoVinet paper](https://arxiv.org/abs/2103.11511), the `state` is referred to as the \"Stream Buffer\".\n", + "\n", + "![States passed in causal convolution](https://www.tensorflow.org/images/tutorials/video/causal_conv_states.png)\n", + "\n", + "By passing this little bit of state forward, you can avoid recalculating the whole receptive field that shown above. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1UsxiPs8yA2e" + }, + "source": [ + "## Download a pre-trained MoViNet model\n", + "\n", + "In this section, you will:\n", + "\n", + "1. You can create a MoViNet model using the open source code provided in [`official/projects/movinet`](https://github.com/tensorflow/models/tree/master/official/projects/movinet) from TensorFlow models.\n", + "2. Load the pretrained weights. \n", + "3. Freeze the convolutional base, or all other layers except the final classifier head, to speed up fine-tuning.\n", + "\n", + "To build the model, you can start with the `a0` configuration because it is the fastest to train when benchmarked against other models. Check out the [available MoViNet models on TensorFlow Model Garden](https://github.com/tensorflow/models/blob/master/official/projects/movinet/configs/movinet.py) to find what might work for your use case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rhSCM6cee05F" + }, + "outputs": [], + "source": [ + "model_id = 'a0'\n", + "resolution = 224\n", + "\n", + "tf.keras.backend.clear_session()\n", + "\n", + "backbone = movinet.Movinet(model_id=model_id)\n", + "backbone.trainable = False\n", + "\n", + "# Set num_classes=600 to load the pre-trained weights from the original model\n", + "model = movinet_model.MovinetClassifier(backbone=backbone, num_classes=600)\n", + "model.build([None, None, None, None, 3])\n", + "\n", + "# Load pre-trained weights\n", + "!wget https://storage.googleapis.com/tf_model_garden/vision/movinet/movinet_a0_base.tar.gz -O movinet_a0_base.tar.gz -q\n", + "!tar -xvf movinet_a0_base.tar.gz\n", + "\n", + "checkpoint_dir = f'movinet_{model_id}_base'\n", + "checkpoint_path = tf.train.latest_checkpoint(checkpoint_dir)\n", + "checkpoint = tf.train.Checkpoint(model=model)\n", + "status = checkpoint.restore(checkpoint_path)\n", + "status.assert_existing_objects_matched()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BW23HVNtCXff" + }, + "source": [ + "To build a classifier, create a function that takes the backbone and the number of classes in a dataset. The `build_classifier` function will take the backbone and the number of classes in a dataset to build the classifier. In this case, the new classifier will take a `num_classes` outputs (10 classes for this subset of UCF101)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6cfAelbU5Gi3" + }, + "outputs": [], + "source": [ + "def build_classifier(batch_size, num_frames, resolution, backbone, num_classes):\n", + " \"\"\"Builds a classifier on top of a backbone model.\"\"\"\n", + " model = movinet_model.MovinetClassifier(\n", + " backbone=backbone,\n", + " num_classes=num_classes)\n", + " model.build([batch_size, num_frames, resolution, resolution, 3])\n", + "\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9HWSk-u7oPUZ" + }, + "outputs": [], + "source": [ + "model = build_classifier(batch_size, num_frames, resolution, backbone, 10)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JhbX7qdTN8lc" + }, + "source": [ + "For this tutorial, choose the `tf.keras.optimizers.Adam` optimizer and the `tf.keras.losses.SparseCategoricalCrossentropy` loss function. Use the metrics argument to the view the accuracy of the model performance at every step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dVqBLrn1tBsd" + }, + "outputs": [], + "source": [ + "num_epochs = 2\n", + "\n", + "loss_obj = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", + "\n", + "optimizer = tf.keras.optimizers.Adam(learning_rate = 0.001)\n", + "\n", + "model.compile(loss=loss_obj, optimizer=optimizer, metrics=['accuracy'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VflEr_t6CuQu" + }, + "source": [ + "Train the model. After two epochs, observe a low loss with high accuracy for both the training and test sets. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "background_save": true + }, + "id": "9ZeiYzI0tqQG" + }, + "outputs": [], + "source": [ + "results = model.fit(train_ds,\n", + " validation_data=test_ds,\n", + " epochs=num_epochs,\n", + " validation_freq=1,\n", + " verbose=1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KkLl2zF8G9W0" + }, + "source": [ + "## Evaluate the model\n", + "\n", + "The model achieved high accuracy on the training dataset. Next, use Keras `Model.evaluate` to evaluate it on the test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NqgbzOiKuxxT" + }, + "outputs": [], + "source": [ + "model.evaluate(test_ds, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OkFst2gsHBwD" + }, + "source": [ + "To visualize model performance further, use a [confusion matrix](https://www.tensorflow.org/api_docs/python/tf/math/confusion_matrix). The confusion matrix allows you to assess the performance of the classification model beyond accuracy. To build the confusion matrix for this multi-class classification problem, get the actual values in the test set and the predicted values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hssSdW9XHF_j" + }, + "outputs": [], + "source": [ + "def get_actual_predicted_labels(dataset):\n", + " \"\"\"\n", + " Create a list of actual ground truth values and the predictions from the model.\n", + "\n", + " Args:\n", + " dataset: An iterable data structure, such as a TensorFlow Dataset, with features and labels.\n", + "\n", + " Return:\n", + " Ground truth and predicted values for a particular dataset.\n", + " \"\"\"\n", + " actual = [labels for _, labels in dataset.unbatch()]\n", + " predicted = model.predict(dataset)\n", + "\n", + " actual = tf.stack(actual, axis=0)\n", + " predicted = tf.concat(predicted, axis=0)\n", + " predicted = tf.argmax(predicted, axis=1)\n", + "\n", + " return actual, predicted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2TmTue6THGWO" + }, + "outputs": [], + "source": [ + "def plot_confusion_matrix(actual, predicted, labels, ds_type):\n", + " cm = tf.math.confusion_matrix(actual, predicted)\n", + " ax = sns.heatmap(cm, annot=True, fmt='g')\n", + " sns.set(rc={'figure.figsize':(12, 12)})\n", + " sns.set(font_scale=1.4)\n", + " ax.set_title('Confusion matrix of action recognition for ' + ds_type)\n", + " ax.set_xlabel('Predicted Action')\n", + " ax.set_ylabel('Actual Action')\n", + " plt.xticks(rotation=90)\n", + " plt.yticks(rotation=0)\n", + " ax.xaxis.set_ticklabels(labels)\n", + " ax.yaxis.set_ticklabels(labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4RK1A1C1HH6V" + }, + "outputs": [], + "source": [ + "fg = FrameGenerator(subset_paths['train'], num_frames, training = True)\n", + "label_names = list(fg.class_ids_for_name.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "r4AFi2e5HKEO" + }, + "outputs": [], + "source": [ + "actual, predicted = get_actual_predicted_labels(test_ds)\n", + "plot_confusion_matrix(actual, predicted, label_names, 'test')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ddQG9sYxa1Ib" + }, + "source": [ + "## Next steps\n", + "\n", + "Now that you have some familiarity with the MoViNet model and how to leverage various TensorFlow APIs (for example, for transfer learning), try using the code in this tutorial with your own dataset. The data does not have to be limited to video data. Volumetric data, such as MRI scans, can also be used with 3D CNNs. The NUSDAT and IMH datasets mentioned in [Brain MRI-based 3D Convolutional Neural Networks for Classification of Schizophrenia and Controls](https://arxiv.org/pdf/2003.08818.pdf) could be two such sources for MRI data.\n", + "\n", + "In particular, using the `FrameGenerator` class used in this tutorial and the other video data and classification tutorials will help you load data into your models.\n", + "\n", + "To learn more about working with video data in TensorFlow, check out the following tutorials:\n", + "\n", + "* [Load video data](https://www.tensorflow.org/tutorials/load_data/video)\n", + "* [Build a 3D CNN model for video classification](https://www.tensorflow.org/tutorials/video/video_classification)\n", + "* [MoViNet for streaming action recognition](https://www.tensorflow.org/hub/tutorials/movinet)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "transfer_learning_with_movinet.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/tutorials/video/video_classification.ipynb b/site/en/tutorials/video/video_classification.ipynb new file mode 100644 index 00000000000..4265b6387e3 --- /dev/null +++ b/site/en/tutorials/video/video_classification.ipynb @@ -0,0 +1,1058 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "TBFXQGKYUc4X" + }, + "source": [ + "##### Copyright 2022 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "1z4xy2gTUc4a" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KwQtSOz0VrVX" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L2MHy42s5wl6" + }, + "source": [ + "# Video classification with a 3D convolutional neural network\n", + "\n", + "This tutorial demonstrates training a 3D convolutional neural network (CNN) for video classification using the [UCF101](https://www.crcv.ucf.edu/data/UCF101.php) action recognition dataset. A 3D CNN uses a three-dimensional filter to perform convolutions. The kernel is able to slide in three directions, whereas in a 2D CNN it can slide in two dimensions. The model is based on the work published in [A Closer Look at Spatiotemporal Convolutions for Action Recognition](https://arxiv.org/abs/1711.11248v3) by D. Tran et al. (2017). In this tutorial, you will:\n", + "\n", + "* Build an input pipeline\n", + "* Build a 3D convolutional neural network model with residual connections using Keras functional API\n", + "* Train the model\n", + "* Evaluate and test the model\n", + "\n", + "This video classification tutorial is the second part in a series of TensorFlow video tutorials. Here are the other three tutorials:\n", + "\n", + "- [Load video data](https://www.tensorflow.org/tutorials/load_data/video): This tutorial explains much of the code used in this document.\n", + "- [MoViNet for streaming action recognition](https://www.tensorflow.org/hub/tutorials/movinet): Get familiar with the MoViNet models that are available on TF Hub.\n", + "- [Transfer learning for video classification with MoViNet](https://www.tensorflow.org/tutorials/video/transfer_learning_with_movinet): This tutorial explains how to use a pre-trained video classification model trained on a different dataset with the UCF-101 dataset." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_Ih_df2q0kw4" + }, + "source": [ + "## Setup\n", + "\n", + "Begin by installing and importing some necessary libraries, including:\n", + "[remotezip](https://github.com/gtsystem/python-remotezip) to inspect the contents of a ZIP file, [tqdm](https://github.com/tqdm/tqdm) to use a progress bar, [OpenCV](https://opencv.org/) to process video files, [einops](https://github.com/arogozhnikov/einops/tree/master/docs) for performing more complex tensor operations, and [`tensorflow_docs`](https://github.com/tensorflow/docs/tree/master/tools/tensorflow_docs) for embedding data in a Jupyter notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KEbL4Mwi01PV" + }, + "outputs": [], + "source": [ + "!pip install remotezip tqdm opencv-python einops \n", + "!pip install -U tensorflow keras" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gg0otuqb0hIf" + }, + "outputs": [], + "source": [ + "import tqdm\n", + "import random\n", + "import pathlib\n", + "import itertools\n", + "import collections\n", + "\n", + "import cv2\n", + "import einops\n", + "import numpy as np\n", + "import remotezip as rz\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import tensorflow as tf\n", + "import keras\n", + "from keras import layers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Ctk9A57-6ABq" + }, + "source": [ + "## Load and preprocess video data\n", + "\n", + "The hidden cell below defines helper functions to download a slice of data from the UCF-101 dataset, and load it into a `tf.data.Dataset`. You can learn more about the specific preprocessing steps in the [Loading video data tutorial](../load_data/video.ipynb), which walks you through this code in more detail.\n", + "\n", + "The `FrameGenerator` class at the end of the hidden block is the most important utility here. It creates an iterable object that can feed data into the TensorFlow data pipeline. Specifically, this class contains a Python generator that loads the video frames along with its encoded label. The generator (`__call__`) function yields the frame array produced by `frames_from_video_file` and a one-hot encoded vector of the label associated with the set of frames." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nB2aOTU35r9_" + }, + "outputs": [], + "source": [ + "#@title\n", + "\n", + "def list_files_per_class(zip_url):\n", + " \"\"\"\n", + " List the files in each class of the dataset given the zip URL.\n", + "\n", + " Args:\n", + " zip_url: URL from which the files can be unzipped. \n", + "\n", + " Return:\n", + " files: List of files in each of the classes.\n", + " \"\"\"\n", + " files = []\n", + " with rz.RemoteZip(URL) as zip:\n", + " for zip_info in zip.infolist():\n", + " files.append(zip_info.filename)\n", + " return files\n", + "\n", + "def get_class(fname):\n", + " \"\"\"\n", + " Retrieve the name of the class given a filename.\n", + "\n", + " Args:\n", + " fname: Name of the file in the UCF101 dataset.\n", + "\n", + " Return:\n", + " Class that the file belongs to.\n", + " \"\"\"\n", + " return fname.split('_')[-3]\n", + "\n", + "def get_files_per_class(files):\n", + " \"\"\"\n", + " Retrieve the files that belong to each class. \n", + "\n", + " Args:\n", + " files: List of files in the dataset.\n", + "\n", + " Return:\n", + " Dictionary of class names (key) and files (values).\n", + " \"\"\"\n", + " files_for_class = collections.defaultdict(list)\n", + " for fname in files:\n", + " class_name = get_class(fname)\n", + " files_for_class[class_name].append(fname)\n", + " return files_for_class\n", + "\n", + "def download_from_zip(zip_url, to_dir, file_names):\n", + " \"\"\"\n", + " Download the contents of the zip file from the zip URL.\n", + "\n", + " Args:\n", + " zip_url: Zip URL containing data.\n", + " to_dir: Directory to download data to.\n", + " file_names: Names of files to download.\n", + " \"\"\"\n", + " with rz.RemoteZip(zip_url) as zip:\n", + " for fn in tqdm.tqdm(file_names):\n", + " class_name = get_class(fn)\n", + " zip.extract(fn, str(to_dir / class_name))\n", + " unzipped_file = to_dir / class_name / fn\n", + "\n", + " fn = pathlib.Path(fn).parts[-1]\n", + " output_file = to_dir / class_name / fn\n", + " unzipped_file.rename(output_file,)\n", + "\n", + "def split_class_lists(files_for_class, count):\n", + " \"\"\"\n", + " Returns the list of files belonging to a subset of data as well as the remainder of\n", + " files that need to be downloaded.\n", + " \n", + " Args:\n", + " files_for_class: Files belonging to a particular class of data.\n", + " count: Number of files to download.\n", + "\n", + " Return:\n", + " split_files: Files belonging to the subset of data.\n", + " remainder: Dictionary of the remainder of files that need to be downloaded.\n", + " \"\"\"\n", + " split_files = []\n", + " remainder = {}\n", + " for cls in files_for_class:\n", + " split_files.extend(files_for_class[cls][:count])\n", + " remainder[cls] = files_for_class[cls][count:]\n", + " return split_files, remainder\n", + "\n", + "def download_ufc_101_subset(zip_url, num_classes, splits, download_dir):\n", + " \"\"\"\n", + " Download a subset of the UFC101 dataset and split them into various parts, such as\n", + " training, validation, and test. \n", + "\n", + " Args:\n", + " zip_url: Zip URL containing data.\n", + " num_classes: Number of labels.\n", + " splits: Dictionary specifying the training, validation, test, etc. (key) division of data \n", + " (value is number of files per split).\n", + " download_dir: Directory to download data to.\n", + "\n", + " Return:\n", + " dir: Posix path of the resulting directories containing the splits of data.\n", + " \"\"\"\n", + " files = list_files_per_class(zip_url)\n", + " for f in files:\n", + " tokens = f.split('/')\n", + " if len(tokens) <= 2:\n", + " files.remove(f) # Remove that item from the list if it does not have a filename\n", + " \n", + " files_for_class = get_files_per_class(files)\n", + "\n", + " classes = list(files_for_class.keys())[:num_classes]\n", + "\n", + " for cls in classes:\n", + " new_files_for_class = files_for_class[cls]\n", + " random.shuffle(new_files_for_class)\n", + " files_for_class[cls] = new_files_for_class\n", + " \n", + " # Only use the number of classes you want in the dictionary\n", + " files_for_class = {x: files_for_class[x] for x in list(files_for_class)[:num_classes]}\n", + "\n", + " dirs = {}\n", + " for split_name, split_count in splits.items():\n", + " print(split_name, \":\")\n", + " split_dir = download_dir / split_name\n", + " split_files, files_for_class = split_class_lists(files_for_class, split_count)\n", + " download_from_zip(zip_url, split_dir, split_files)\n", + " dirs[split_name] = split_dir\n", + "\n", + " return dirs\n", + "\n", + "def format_frames(frame, output_size):\n", + " \"\"\"\n", + " Pad and resize an image from a video.\n", + " \n", + " Args:\n", + " frame: Image that needs to resized and padded. \n", + " output_size: Pixel size of the output frame image.\n", + "\n", + " Return:\n", + " Formatted frame with padding of specified output size.\n", + " \"\"\"\n", + " frame = tf.image.convert_image_dtype(frame, tf.float32)\n", + " frame = tf.image.resize_with_pad(frame, *output_size)\n", + " return frame\n", + "\n", + "def frames_from_video_file(video_path, n_frames, output_size = (224,224), frame_step = 15):\n", + " \"\"\"\n", + " Creates frames from each video file present for each category.\n", + "\n", + " Args:\n", + " video_path: File path to the video.\n", + " n_frames: Number of frames to be created per video file.\n", + " output_size: Pixel size of the output frame image.\n", + "\n", + " Return:\n", + " An NumPy array of frames in the shape of (n_frames, height, width, channels).\n", + " \"\"\"\n", + " # Read each video frame by frame\n", + " result = []\n", + " src = cv2.VideoCapture(str(video_path)) \n", + "\n", + " video_length = src.get(cv2.CAP_PROP_FRAME_COUNT)\n", + "\n", + " need_length = 1 + (n_frames - 1) * frame_step\n", + "\n", + " if need_length > video_length:\n", + " start = 0\n", + " else:\n", + " max_start = video_length - need_length\n", + " start = random.randint(0, max_start + 1)\n", + "\n", + " src.set(cv2.CAP_PROP_POS_FRAMES, start)\n", + " # ret is a boolean indicating whether read was successful, frame is the image itself\n", + " ret, frame = src.read()\n", + " result.append(format_frames(frame, output_size))\n", + "\n", + " for _ in range(n_frames - 1):\n", + " for _ in range(frame_step):\n", + " ret, frame = src.read()\n", + " if ret:\n", + " frame = format_frames(frame, output_size)\n", + " result.append(frame)\n", + " else:\n", + " result.append(np.zeros_like(result[0]))\n", + " src.release()\n", + " result = np.array(result)[..., [2, 1, 0]]\n", + "\n", + " return result\n", + "\n", + "class FrameGenerator:\n", + " def __init__(self, path, n_frames, training = False):\n", + " \"\"\" Returns a set of frames with their associated label. \n", + "\n", + " Args:\n", + " path: Video file paths.\n", + " n_frames: Number of frames. \n", + " training: Boolean to determine if training dataset is being created.\n", + " \"\"\"\n", + " self.path = path\n", + " self.n_frames = n_frames\n", + " self.training = training\n", + " self.class_names = sorted(set(p.name for p in self.path.iterdir() if p.is_dir()))\n", + " self.class_ids_for_name = dict((name, idx) for idx, name in enumerate(self.class_names))\n", + "\n", + " def get_files_and_class_names(self):\n", + " video_paths = list(self.path.glob('*/*.avi'))\n", + " classes = [p.parent.name for p in video_paths] \n", + " return video_paths, classes\n", + "\n", + " def __call__(self):\n", + " video_paths, classes = self.get_files_and_class_names()\n", + "\n", + " pairs = list(zip(video_paths, classes))\n", + "\n", + " if self.training:\n", + " random.shuffle(pairs)\n", + "\n", + " for path, name in pairs:\n", + " video_frames = frames_from_video_file(path, self.n_frames) \n", + " label = self.class_ids_for_name[name] # Encode labels\n", + " yield video_frames, label" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OYY7PkdJFM4Z" + }, + "outputs": [], + "source": [ + "URL = 'https://storage.googleapis.com/thumos14_files/UCF101_videos.zip'\n", + "download_dir = pathlib.Path('./UCF101_subset/')\n", + "subset_paths = download_ufc_101_subset(URL, \n", + " num_classes = 10, \n", + " splits = {\"train\": 30, \"val\": 10, \"test\": 10},\n", + " download_dir = download_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C0O3ttIzpFZJ" + }, + "source": [ + "Create the training, validation, and test sets (`train_ds`, `val_ds`, and `test_ds`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "lq86IyGDJjTX" + }, + "outputs": [], + "source": [ + "n_frames = 10\n", + "batch_size = 8\n", + "\n", + "output_signature = (tf.TensorSpec(shape = (None, None, None, 3), dtype = tf.float32),\n", + " tf.TensorSpec(shape = (), dtype = tf.int16))\n", + "\n", + "train_ds = tf.data.Dataset.from_generator(FrameGenerator(subset_paths['train'], n_frames, training=True),\n", + " output_signature = output_signature)\n", + "\n", + "\n", + "# Batch the data\n", + "train_ds = train_ds.batch(batch_size)\n", + "\n", + "val_ds = tf.data.Dataset.from_generator(FrameGenerator(subset_paths['val'], n_frames),\n", + " output_signature = output_signature)\n", + "val_ds = val_ds.batch(batch_size)\n", + "\n", + "test_ds = tf.data.Dataset.from_generator(FrameGenerator(subset_paths['test'], n_frames),\n", + " output_signature = output_signature)\n", + "\n", + "test_ds = test_ds.batch(batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nzogoGA4pQW0" + }, + "source": [ + "## Create the model\n", + "\n", + "The following 3D convolutional neural network model is based off the paper [A Closer Look at Spatiotemporal Convolutions for Action Recognition](https://arxiv.org/abs/1711.11248v3) by D. Tran et al. (2017). The paper compares several versions of 3D ResNets. Instead of operating on a single image with dimensions `(height, width)`, like standard ResNets, these operate on video volume `(time, height, width)`. The most obvious approach to this problem would be replace each 2D convolution (`layers.Conv2D`) with a 3D convolution (`layers.Conv3D`).\n", + "\n", + "This tutorial uses a (2 + 1)D convolution with [residual connections](https://arxiv.org/abs/1512.03385). The (2 + 1)D convolution allows for the decomposition of the spatial and temporal dimensions, therefore creating two separate steps. An advantage of this approach is that factorizing the convolutions into spatial and temporal dimensions saves parameters. \n", + "\n", + "For each output location a 3D convolution combines all the vectors from a 3D patch of the volume to create one vector in the output volume.\n", + "\n", + "![3D convolutions](https://www.tensorflow.org/images/tutorials/video/3DCNN.png)\n", + "\n", + "This operation is takes `time * height * width * channels` inputs and produces `channels` outputs (assuming the number of input and output channels are the same. So a 3D convolution layer with a kernel size of `(3 x 3 x 3)` would need a weight-matrix with `27 * channels ** 2` entries. The reference paper found that a more effective & efficient approach was to factorize the convolution. Instead of a single 3D convolution to process the time and space dimensions, they proposed a \\\"(2+1)D\\\" convolution which processes the space and time dimensions separately. The figure below shows the factored spatial and temporal convolutions of a (2 + 1)D convolution.\n", + "\n", + "![(2+1)D convolutions](https://www.tensorflow.org/images/tutorials/video/2plus1CNN.png)\n", + "\n", + "The main advantage of this approach is that it reduces the number of parameters. In the (2 + 1)D convolution the spatial convolution takes in data of the shape `(1, width, height)`, while the temporal convolution takes in data of the shape `(time, 1, 1)`. For example, a (2 + 1)D convolution with kernel size `(3 x 3 x 3)` would need weight matrices of size `(9 * channels**2) + (3 * channels**2)`, less than half as many as the full 3D convolution. This tutorial implements (2 + 1)D ResNet18, where each convolution in the resnet is replaced by a (2+1)D convolution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GZcB_7dg-EZJ" + }, + "outputs": [], + "source": [ + "# Define the dimensions of one frame in the set of frames created\n", + "HEIGHT = 224\n", + "WIDTH = 224" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yD_sDIBlNu7K" + }, + "outputs": [], + "source": [ + "class Conv2Plus1D(keras.layers.Layer):\n", + " def __init__(self, filters, kernel_size, padding):\n", + " \"\"\"\n", + " A sequence of convolutional layers that first apply the convolution operation over the\n", + " spatial dimensions, and then the temporal dimension. \n", + " \"\"\"\n", + " super().__init__()\n", + " self.seq = keras.Sequential([ \n", + " # Spatial decomposition\n", + " layers.Conv3D(filters=filters,\n", + " kernel_size=(1, kernel_size[1], kernel_size[2]),\n", + " padding=padding),\n", + " # Temporal decomposition\n", + " layers.Conv3D(filters=filters, \n", + " kernel_size=(kernel_size[0], 1, 1),\n", + " padding=padding)\n", + " ])\n", + " \n", + " def call(self, x):\n", + " return self.seq(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "I-fCAddqEORZ" + }, + "source": [ + "A ResNet model is made from a sequence of residual blocks.\n", + "A residual block has two branches. The main branch performs the calculation, but is difficult for gradients to flow through.\n", + "The residual branch bypasses the main calculation and mostly just adds the input to the output of the main branch.\n", + "Gradients flow easily through this branch.\n", + "Therefore, an easy path from the loss function to any of the residual block's main branch will be present.\n", + "This avoids the vanishing gradient problem.\n", + "\n", + "Create the main branch of the residual block with the following class. In contrast to the standard ResNet structure this uses the custom `Conv2Plus1D` layer instead of `layers.Conv2D`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tjxAKHwn6mTJ" + }, + "outputs": [], + "source": [ + "class ResidualMain(keras.layers.Layer):\n", + " \"\"\"\n", + " Residual block of the model with convolution, layer normalization, and the\n", + " activation function, ReLU.\n", + " \"\"\"\n", + " def __init__(self, filters, kernel_size):\n", + " super().__init__()\n", + " self.seq = keras.Sequential([\n", + " Conv2Plus1D(filters=filters,\n", + " kernel_size=kernel_size,\n", + " padding='same'),\n", + " layers.LayerNormalization(),\n", + " layers.ReLU(),\n", + " Conv2Plus1D(filters=filters, \n", + " kernel_size=kernel_size,\n", + " padding='same'),\n", + " layers.LayerNormalization()\n", + " ])\n", + " \n", + " def call(self, x):\n", + " return self.seq(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CevmZ9qsdpWC" + }, + "source": [ + "To add the residual branch to the main branch it needs to have the same size. The `Project` layer below deals with cases where the number of channels is changed on the branch. In particular, a sequence of densely-connected layer followed by normalization is added. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "znrk5BrL6kuq" + }, + "outputs": [], + "source": [ + "class Project(keras.layers.Layer):\n", + " \"\"\"\n", + " Project certain dimensions of the tensor as the data is passed through different \n", + " sized filters and downsampled. \n", + " \"\"\"\n", + " def __init__(self, units):\n", + " super().__init__()\n", + " self.seq = keras.Sequential([\n", + " layers.Dense(units),\n", + " layers.LayerNormalization()\n", + " ])\n", + "\n", + " def call(self, x):\n", + " return self.seq(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "S8zycXGvfnak" + }, + "source": [ + "Use `add_residual_block` to introduce a skip connection between the layers of the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "urjVgqvw-TlB" + }, + "outputs": [], + "source": [ + "def add_residual_block(input, filters, kernel_size):\n", + " \"\"\"\n", + " Add residual blocks to the model. If the last dimensions of the input data\n", + " and filter size does not match, project it such that last dimension matches.\n", + " \"\"\"\n", + " out = ResidualMain(filters, \n", + " kernel_size)(input)\n", + " \n", + " res = input\n", + " # Using the Keras functional APIs, project the last dimension of the tensor to\n", + " # match the new filter size\n", + " if out.shape[-1] != input.shape[-1]:\n", + " res = Project(out.shape[-1])(res)\n", + "\n", + " return layers.add([res, out])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bozog_0hFKrD" + }, + "source": [ + "Resizing the video is necessary to perform downsampling of the data. In particular, downsampling the video frames allow for the model to examine specific parts of frames to detect patterns that may be specific to a certain action. Through downsampling, non-essential information can be discarded. Moreoever, resizing the video will allow for dimensionality reduction and therefore faster processing through the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lQOWuc2I-QqK" + }, + "outputs": [], + "source": [ + "class ResizeVideo(keras.layers.Layer):\n", + " def __init__(self, height, width):\n", + " super().__init__()\n", + " self.height = height\n", + " self.width = width\n", + " self.resizing_layer = layers.Resizing(self.height, self.width)\n", + "\n", + " def call(self, video):\n", + " \"\"\"\n", + " Use the einops library to resize the tensor. \n", + " \n", + " Args:\n", + " video: Tensor representation of the video, in the form of a set of frames.\n", + " \n", + " Return:\n", + " A downsampled size of the video according to the new height and width it should be resized to.\n", + " \"\"\"\n", + " # b stands for batch size, t stands for time, h stands for height, \n", + " # w stands for width, and c stands for the number of channels.\n", + " old_shape = einops.parse_shape(video, 'b t h w c')\n", + " images = einops.rearrange(video, 'b t h w c -> (b t) h w c')\n", + " images = self.resizing_layer(images)\n", + " videos = einops.rearrange(\n", + " images, '(b t) h w c -> b t h w c',\n", + " t = old_shape['t'])\n", + " return videos" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z9IqzCq--Uu9" + }, + "source": [ + "Use the [Keras functional API](https://www.tensorflow.org/guide/keras/functional) to build the residual network." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_bROfh_K-Wxs" + }, + "outputs": [], + "source": [ + "input_shape = (None, 10, HEIGHT, WIDTH, 3)\n", + "input = layers.Input(shape=(input_shape[1:]))\n", + "x = input\n", + "\n", + "x = Conv2Plus1D(filters=16, kernel_size=(3, 7, 7), padding='same')(x)\n", + "x = layers.BatchNormalization()(x)\n", + "x = layers.ReLU()(x)\n", + "x = ResizeVideo(HEIGHT // 2, WIDTH // 2)(x)\n", + "\n", + "# Block 1\n", + "x = add_residual_block(x, 16, (3, 3, 3))\n", + "x = ResizeVideo(HEIGHT // 4, WIDTH // 4)(x)\n", + "\n", + "# Block 2\n", + "x = add_residual_block(x, 32, (3, 3, 3))\n", + "x = ResizeVideo(HEIGHT // 8, WIDTH // 8)(x)\n", + "\n", + "# Block 3\n", + "x = add_residual_block(x, 64, (3, 3, 3))\n", + "x = ResizeVideo(HEIGHT // 16, WIDTH // 16)(x)\n", + "\n", + "# Block 4\n", + "x = add_residual_block(x, 128, (3, 3, 3))\n", + "\n", + "x = layers.GlobalAveragePooling3D()(x)\n", + "x = layers.Flatten()(x)\n", + "x = layers.Dense(10)(x)\n", + "\n", + "model = keras.Model(input, x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TiO0WylG-ZHM" + }, + "outputs": [], + "source": [ + "frames, label = next(iter(train_ds))\n", + "model.build(frames)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GAsKrM8r-bKM" + }, + "outputs": [], + "source": [ + "# Visualize the model\n", + "keras.utils.plot_model(model, expand_nested=True, dpi=60, show_shapes=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1yvJJPnY-dMP" + }, + "source": [ + "## Train the model\n", + "\n", + "For this tutorial, choose the `tf.keras.optimizers.Adam` optimizer and the `tf.keras.losses.SparseCategoricalCrossentropy` loss function. Use the `metrics` argument to the view the accuracy of the model performance at every step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ejrbyebDp2tA" + }, + "outputs": [], + "source": [ + "model.compile(loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n", + " optimizer = keras.optimizers.Adam(learning_rate = 0.0001), \n", + " metrics = ['accuracy'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nZT1Xlx9stP2" + }, + "source": [ + "Train the model for 50 epoches with the Keras `Model.fit` method.\n", + "\n", + "Note: This example model is trained on fewer data points (300 training and 100 validation examples) to keep training time reasonable for this tutorial. Moreover, this example model may take over one hour to train." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VMrMUl2hOqMs" + }, + "outputs": [], + "source": [ + "history = model.fit(x = train_ds,\n", + " epochs = 50, \n", + " validation_data = val_ds)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KKUfMNVns2hu" + }, + "source": [ + "### Visualize the results\n", + "\n", + "Create plots of the loss and accuracy on the training and validation sets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Cd5tpNrtOrs7" + }, + "outputs": [], + "source": [ + "def plot_history(history):\n", + " \"\"\"\n", + " Plotting training and validation learning curves.\n", + "\n", + " Args:\n", + " history: model history with all the metric measures\n", + " \"\"\"\n", + " fig, (ax1, ax2) = plt.subplots(2)\n", + "\n", + " fig.set_size_inches(18.5, 10.5)\n", + "\n", + " # Plot loss\n", + " ax1.set_title('Loss')\n", + " ax1.plot(history.history['loss'], label = 'train')\n", + " ax1.plot(history.history['val_loss'], label = 'test')\n", + " ax1.set_ylabel('Loss')\n", + " \n", + " # Determine upper bound of y-axis\n", + " max_loss = max(history.history['loss'] + history.history['val_loss'])\n", + "\n", + " ax1.set_ylim([0, np.ceil(max_loss)])\n", + " ax1.set_xlabel('Epoch')\n", + " ax1.legend(['Train', 'Validation']) \n", + "\n", + " # Plot accuracy\n", + " ax2.set_title('Accuracy')\n", + " ax2.plot(history.history['accuracy'], label = 'train')\n", + " ax2.plot(history.history['val_accuracy'], label = 'test')\n", + " ax2.set_ylabel('Accuracy')\n", + " ax2.set_ylim([0, 1])\n", + " ax2.set_xlabel('Epoch')\n", + " ax2.legend(['Train', 'Validation'])\n", + "\n", + " plt.show()\n", + "\n", + "plot_history(history)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EJrGF0Sss8E0" + }, + "source": [ + "## Evaluate the model\n", + "\n", + "Use Keras `Model.evaluate` to get the loss and accuracy on the test dataset. \n", + "\n", + "Note: The example model in this tutorial uses a subset of the UCF101 dataset to keep training time reasonable. The accuracy and loss can be improved with further hyperparameter tuning or more training data. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Hev0hMCxOtfy" + }, + "outputs": [], + "source": [ + "model.evaluate(test_ds, return_dict=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-F73GxD1-yc8" + }, + "source": [ + "To visualize model performance further, use a [confusion matrix](https://www.tensorflow.org/api_docs/python/tf/math/confusion_matrix). The confusion matrix allows you to assess the performance of the classification model beyond accuracy. In order to build the confusion matrix for this multi-class classification problem, get the actual values in the test set and the predicted values. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Yw-6rG5V-0L-" + }, + "outputs": [], + "source": [ + "def get_actual_predicted_labels(dataset): \n", + " \"\"\"\n", + " Create a list of actual ground truth values and the predictions from the model.\n", + "\n", + " Args:\n", + " dataset: An iterable data structure, such as a TensorFlow Dataset, with features and labels.\n", + "\n", + " Return:\n", + " Ground truth and predicted values for a particular dataset.\n", + " \"\"\"\n", + " actual = [labels for _, labels in dataset.unbatch()]\n", + " predicted = model.predict(dataset)\n", + "\n", + " actual = tf.stack(actual, axis=0)\n", + " predicted = tf.concat(predicted, axis=0)\n", + " predicted = tf.argmax(predicted, axis=1)\n", + "\n", + " return actual, predicted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aln6qWW_-2dk" + }, + "outputs": [], + "source": [ + "def plot_confusion_matrix(actual, predicted, labels, ds_type):\n", + " cm = tf.math.confusion_matrix(actual, predicted)\n", + " ax = sns.heatmap(cm, annot=True, fmt='g')\n", + " sns.set(rc={'figure.figsize':(12, 12)})\n", + " sns.set(font_scale=1.4)\n", + " ax.set_title('Confusion matrix of action recognition for ' + ds_type)\n", + " ax.set_xlabel('Predicted Action')\n", + " ax.set_ylabel('Actual Action')\n", + " plt.xticks(rotation=90)\n", + " plt.yticks(rotation=0)\n", + " ax.xaxis.set_ticklabels(labels)\n", + " ax.yaxis.set_ticklabels(labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tfQ3VAGd-4Az" + }, + "outputs": [], + "source": [ + "fg = FrameGenerator(subset_paths['train'], n_frames, training=True)\n", + "labels = list(fg.class_ids_for_name.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1ucGpbiA-5qi" + }, + "outputs": [], + "source": [ + "actual, predicted = get_actual_predicted_labels(train_ds)\n", + "plot_confusion_matrix(actual, predicted, labels, 'training')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Mfr7AT5T-7ZD" + }, + "outputs": [], + "source": [ + "actual, predicted = get_actual_predicted_labels(test_ds)\n", + "plot_confusion_matrix(actual, predicted, labels, 'test')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FefzeIZz-9aI" + }, + "source": [ + "The precision and recall values for each class can also be calculated using a confusion matrix." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dq95-56Z-_E2" + }, + "outputs": [], + "source": [ + "def calculate_classification_metrics(y_actual, y_pred, labels):\n", + " \"\"\"\n", + " Calculate the precision and recall of a classification model using the ground truth and\n", + " predicted values. \n", + "\n", + " Args:\n", + " y_actual: Ground truth labels.\n", + " y_pred: Predicted labels.\n", + " labels: List of classification labels.\n", + "\n", + " Return:\n", + " Precision and recall measures.\n", + " \"\"\"\n", + " cm = tf.math.confusion_matrix(y_actual, y_pred)\n", + " tp = np.diag(cm) # Diagonal represents true positives\n", + " precision = dict()\n", + " recall = dict()\n", + " for i in range(len(labels)):\n", + " col = cm[:, i]\n", + " fp = np.sum(col) - tp[i] # Sum of column minus true positive is false negative\n", + " \n", + " row = cm[i, :]\n", + " fn = np.sum(row) - tp[i] # Sum of row minus true positive, is false negative\n", + " \n", + " precision[labels[i]] = tp[i] / (tp[i] + fp) # Precision \n", + " \n", + " recall[labels[i]] = tp[i] / (tp[i] + fn) # Recall\n", + " \n", + " return precision, recall" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4jSEonYQ_BZt" + }, + "outputs": [], + "source": [ + "precision, recall = calculate_classification_metrics(actual, predicted, labels) # Test dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "hXvTW1Df_DV8" + }, + "outputs": [], + "source": [ + "precision" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "be1yrQl5_EYF" + }, + "outputs": [], + "source": [ + "recall" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d4WsP4Z2HZ6L" + }, + "source": [ + "## Next steps\n", + "\n", + "To learn more about working with video data in TensorFlow, check out the following tutorials:\n", + "\n", + "* [Load video data](https://www.tensorflow.org/tutorials/load_data/video)\n", + "* [MoViNet for streaming action recognition](https://www.tensorflow.org/hub/tutorials/movinet)\n", + "* [Transfer learning for video classification with MoViNet](https://www.tensorflow.org/tutorials/video/transfer_learning_with_movinet)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "video_classification.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/site/en/xla/README.md b/site/en/xla/README.md deleted file mode 100644 index f7eda17bc22..00000000000 --- a/site/en/xla/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Welcome to the warp zone! - -# XLA: Accelerated Linear Algebra - -These docs are available here: https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/xla/g3doc diff --git a/site/es/README.md b/site/es/README.md deleted file mode 100644 index 7e7babfdf8f..00000000000 --- a/site/es/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# Documentacion de TensorFlow - -## Traducciones de la Comunidad - -Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad -son basados en el "mejor esfuerzo", no hay ninguna garantia que esta sea un reflejo preciso y actual -de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en). -Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un "Pull request" -al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs). -Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad -por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Por favor enfoquen sus esfuerzos de traduccion en -[TensorFlow 2](https://www.tensorflow.org) que se encuentra en el siguiente -directorio. [site/en/](https://github.com/tensorflow/docs/tree/master/site/en/) -Documentos de la comunidad en la version TF 1.x ya no sera mantneida por ninguna -comunidad mientras se prepara para el lanzamiento de la version 2.0. - -Por favor revisar [El Anuncio](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). -Igualmente, p por favor no traduzcan la seccion de `install/`, cualquier archivo `*.yaml`, o archivo `index.md`. -Revisar [El Anuncio](https://groups.google.com/a/tensorflow.org/forum/#!msg/docs-zh-cn/mhLp-egzNyE/EhGSeIBqAQAJ). - -Note: el siguiente directorio -[site/ko/swift](https://github.com/tensorflow/docs/tree/master/site/ko/swift) -es el "Home" para las traducciones -[Swift para Tensorflow](https://www.tensorflow.org/swift)(S4TF). -Los archivos originales estan en el directorio -[docs/site](https://github.com/tensorflow/swift/tree/master/docs/site) del repositorio -tensorflow/swif. Los "notebooks" S4TF deben tener los resultados guardados. - -## Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Please focus translation efforts on -[TensorFlow 2](https://www.tensorflow.org) in the -[site/en/r2](https://github.com/tensorflow/docs/tree/master/site/en/r2) -directory. TF 1.x community docs will no longer be updated as we prepare for the -2.0 release. See -[the announcement](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). -Also, please do not translate the `install/` section, any `*.yaml` files, or `index.md` files. -See [the announcement](https://groups.google.com/a/tensorflow.org/forum/#!msg/docs-zh-cn/mhLp-egzNyE/EhGSeIBqAQAJ). - -Note: The -[site/es/swift](https://github.com/tensorflow/docs/tree/master/site/es/swift) -directory is the home for -[Swift for TensorFlow](https://www.tensorflow.org/swift)(S4TF) translations. -Original files are in the -[docs/site](https://github.com/tensorflow/swift/tree/master/docs/site) directory -of the tensorflow/swift repository. S4TF notebooks must have the outputs saved. - -## Para nuevos contribuidores - -Muchas gracias por unirse a el esfuerzo de traduccion! -Por favor lean los documentos existentes en -[Es documentos](https://github.com/tensorflow/docs/tree/master/site/es) -Antes de comenzar su traduccion. -Por favor no utilicen coloquismos como Guey, Parcero, Ueon, Che o palabras Soeces. -No agregar acentos, dieresis o tildes ya que pueden afectar la ejecucion de los codigos. -Sigan el estilo de los documentos lo mas posible. - -Una vez la traduccion este completa por favor notifiquen a los contribuidores -[Contribuidores en Espanol de Documentacion en TensorFlow ](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs) -para coordinar la traduccion. - -Copien un archivo en el folder `es` desde el folder `en`, asegurensen que el directorio de folders sea el adecuado. -`site/es/r1` es para TensorFlow 1.x. -`site/es/` es para TensorFlow 2.x. - -Deben traducir los comentarios y los "Markdowns" no deben correr las celdas. -La estructura de Archivos y folders no debe ser cambiado esto dificulta la revision en GitHub. -Usen un editor de texto para editar "Jupyter Notebooks". -Prueben el "Notebook" en su repositorio con Colab luego de que terminen la traduccion para comprobar. -Pueden pedir revision si no hay ningun error. - -Si tienen alguna pregunta sobre traducciones, por favor envien un mensaje a -[Contribuidores en Espanol de Documentacion en TensorFlow ](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs) - -Gracias! - -## For new contributors - -Thanks for joining the translation effort. -Please read the existing -[ES documents](https://github.com/tensorflow/docs/tree/master/site/es) -before starting your translation. -Do not use country Coloquialisms. -Follow the style of existing documents, as possible as you can. - -After your translation is complete, notify the -[Spanish TensorFlow Documentation Contributors](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs) -mailing list to coordinate a review. - -Copy a file in `en` folder to same location under `ko` folder if anybody doesn't work on the file, -and get it start. -`site/es/r1` are for TensorFlow 1.x. -`site/es/` are for TensorFlow 2.x. - -You should translate markdown and comments. You should not run code cells. -Whole file structure can be changed even if you modify only a chunk in the notebook. -It is hard to review such a file in GitHub. -You should use a text editor when you edit a few words of existing notebook. -You should test the notebook in your repository with Colab after you finish the translation. -You can request for review if there is no error. - -If you have any question about translation, feel free to contact -[Spanish TensorFlow Documentation Contributors](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs) -mailing list. - -Thanks! - -## Guia de Traduccion en Espanol/Spanish translation guide - -Algunas palabras Tecnicas en ingles no tienen una traduccion natural al Espanol por favor *NO* -traduzcan las siguientes palabras. - -* mini-batch -* batch -* label -* class -* helper -* hyperparameter -* optimizer -* one-hot encoding -* epoch -* callback -* sequence -* dictionary (in python) -* embedding -* padding -* unit -* node -* neuron -* target -* checkpoint -* compile -* dropout -* penalty -* eager execution -* eagerly diff --git a/site/es/REVIEWERS b/site/es/REVIEWERS deleted file mode 100644 index d06ee5008b3..00000000000 --- a/site/es/REVIEWERS +++ /dev/null @@ -1,9 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/ko directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs -crispindev -gogasca -enricsoler diff --git a/site/es/guide/checkpoint.ipynb b/site/es/guide/checkpoint.ipynb deleted file mode 100644 index 9654fc4ef9a..00000000000 --- a/site/es/guide/checkpoint.ipynb +++ /dev/null @@ -1,683 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "checkpoint.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Pnn4rDWGqDZL" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "l534d35Gp68G", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3TI3Q3XBesaS" - }, - "source": [ - "# Entrenar checkpoints" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yw_a0iGucY8z" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Ver en TensorFlow.org\n", - " \n", - " Ejecutar en Google Colab\n", - " \n", - " Ver fuente en GitHub\n", - " \n", - " Descargar notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4HYmIbxDQOCF", - "colab_type": "text" - }, - "source": [ - "Note: Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad\n", - "son basados en el \"mejor esfuerzo\", no hay ninguna garantia que esta sea un reflejo preciso y actual \n", - "de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en).\n", - "Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un \"Pull request\"\n", - "al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs).\n", - "Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad\n", - "por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LeDp7dovcbus" - }, - "source": [ - "La frase \"Saving a TensorFlow model\" significa tipicamente una de las dos cosas: (1) Checkpoints, O (2) SavedModel.\n", - "\n", - "Los Checkpoints capturan el valor exacto de todos los parametros (objetos `tf.Variable`) usados por un modelo. Los Checkpoints no almacenan ninguna descripcion del computo utilizado por el modelo. Por lo mismo, los checkpoints solo son utiles cuando el codigo que usara los parametros almacenados esta disponible.\n", - "\n", - "Por otro lado, el formato SavedModel incluye una descripcion serializada del computo definido por el modelo ademas de los valores de los parametros (checkpoint). Con este formato, los modelos son independientes al codigo que creo el mismo. Por ende son idoneos para el despliegue de los modelos a traves de TensorFlos Serving, TensorFlow Lite, TensorFlow.js, o programas en otros lenguajes de programacion (las APIs de TensorFlow para C, C++, Java, Go, Rust, C# etc.)\n", - "\n", - "Esta guia cubre las APIs para leer y escribir checkpoints." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5vsq3-pffo1I" - }, - "source": [ - "## Guardando de las APIs de entrenamiento de `tf.keras`\n", - "\n", - "Pueden referirse a la [ guia para guardar y restaurar de `tf.keras`](https://www.tensorflow.org/tutorials/keras/save_and_restore_models), NOTA: al momento esta en ingles.\n", - "\n", - "`tf.keras.Model.save_weights` tambien permite la opcion de guardar en el formato TensorFlow checkpoint. Esta guia explica el formato a mayor detalle e introduce las APIs para administrar los checkpoints en bucles de entrenamiento personalizados." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XseWX5jDg4lQ" - }, - "source": [ - "## Definir checkpoints manualmente" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1jpZPz76ZP3K" - }, - "source": [ - "El estado persistente de un modelo de TensorFlow es almacenado en objectos `tf.Variable`. Estos objetos pueden ser construidos directamente, pero comunmente con creados mediante APIs de alto nivel tales como `tf.keras.layers`.\n", - "\n", - "La manera mas sencilla de admistrar las variables es asociandolas a objetos de Python, y despues referenciando dichos objetos. Las subclases de `tf.train.Checkpoint`, `tf.keras.layers.Layer`, y `tf.keras.Model` rastrean automaticamente las variables asociadas a sus atributos. El ejemplo a contunuacion construye un modelo linear simple, y posteriormente escribe checkpoints que contienen valores para todas las variables del modelo." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VEvpMYAKsC4z", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "try:\n", - " # %tensorflow_version solo existe en Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BR5dChK7rXnj", - "colab": {} - }, - "source": [ - "class Net(tf.keras.Model):\n", - " \"\"\"Un Modelo Linear simple.\"\"\"\n", - "\n", - " def __init__(self):\n", - " super(Net, self).__init__()\n", - " self.l1 = tf.keras.layers.Dense(5)\n", - "\n", - " def call(self, x):\n", - " return self.l1(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fNjf9KaLdIRP" - }, - "source": [ - "Este ejemplo necesita datos y un paso de optimizacion para poder ser ejecutable aunque esta guia no se trate de esos temas. El modelo entrenara por slices de un dataset en memoria." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tSNyP4IJ9nkU", - "colab": {} - }, - "source": [ - "def toy_dataset():\n", - " inputs = tf.range(10.)[:, None]\n", - " labels = inputs * 5. + tf.range(5.)[None, :]\n", - " return tf.data.Dataset.from_tensor_slices(\n", - " dict(x=inputs, y=labels)).repeat(10).batch(2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ICm1cufh_JH8", - "colab": {} - }, - "source": [ - "def train_step(net, example, optimizer):\n", - " \"\"\"Entrena `net` en `example` usando `optimizer`.\"\"\"\n", - " with tf.GradientTape() as tape:\n", - " output = net(example['x'])\n", - " loss = tf.reduce_mean(tf.abs(output - example['y']))\n", - " variables = net.trainable_variables\n", - " gradients = tape.gradient(loss, variables)\n", - " optimizer.apply_gradients(zip(gradients, variables))\n", - " return loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NP9IySmCeCkn" - }, - "source": [ - "El siguiente buckle crea una instancia del modelo y de un optimizer, despues los recolecta en un objeto `tf.train.Checkpoint`. Llama el paso de entrenamiento en un ciclo para cada batch de datos, y escribe periodicamente checkpoints en disco." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BbCS5A6K1VSH", - "colab": {} - }, - "source": [ - "opt = tf.keras.optimizers.Adam(0.1)\n", - "net = Net()\n", - "ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=net)\n", - "manager = tf.train.CheckpointManager(ckpt, './tf_ckpts', max_to_keep=3)\n", - "ckpt.restore(manager.latest_checkpoint)\n", - "if manager.latest_checkpoint:\n", - " print(\"Restaurado de {}\".format(manager.latest_checkpoint))\n", - "else:\n", - " print(\"Inicializando desde cero.\")\n", - "\n", - "for example in toy_dataset():\n", - " loss = train_step(net, example, opt)\n", - " ckpt.step.assign_add(1)\n", - " if int(ckpt.step) % 10 == 0:\n", - " save_path = manager.save()\n", - " print(\"Checkpoint almacenado para el paso {}: {}\".format(int(ckpt.step), save_path))\n", - " print(\"loss {:1.2f}\".format(loss.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lw1QeyRBgsLE" - }, - "source": [ - "El snippet anterior inicializara aleatoriamente las variables del modelo en su primera corrida. Posterior a esta, reanudara el entrenamiendo en donde se quedo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UjilkTOV2PBK", - "colab": {} - }, - "source": [ - "opt = tf.keras.optimizers.Adam(0.1)\n", - "net = Net()\n", - "ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=net)\n", - "manager = tf.train.CheckpointManager(ckpt, './tf_ckpts', max_to_keep=3)\n", - "ckpt.restore(manager.latest_checkpoint)\n", - "if manager.latest_checkpoint:\n", - " print(\"Restaurado de {}\".format(manager.latest_checkpoint))\n", - "else:\n", - " print(\"Inicializando desde cero\")\n", - "\n", - "for example in toy_dataset():\n", - " loss = train_step(net, example, opt)\n", - " ckpt.step.assign_add(1)\n", - " if int(ckpt.step) % 10 == 0:\n", - " save_path = manager.save()\n", - " print(\"Checkpoint almacenado para el paso {}: {}\".format(int(ckpt.step), save_path))\n", - " print(\"loss {:1.2f}\".format(loss.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dxJT9vV-2PnZ" - }, - "source": [ - "El objeto `tf.train.CheckpointManager` elimina checkpoints viejos. Arriba ha sido configurado para conservar los tres checkpoints mas recientes." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3zmM0a-F5XqC", - "colab": {} - }, - "source": [ - "print(manager.checkpoints) # Lista los tres checkpoints restantes" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qwlYDyjemY4P" - }, - "source": [ - "Estos paths, e.g. `'./tf_ckpts/ckpt-10'`, no son archivos en disco. Son prefijos para un archivo tipo `index` y uno o mas archivos de datos que contienen los valores de las variables. Estos prefijos estan agrupados en un unico archivo de `checkpoint` (`'./tf_ckpts/checkpoint'`) donde el `CheckpointManager` guarda su estado." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t1feej9JntV_", - "colab": {} - }, - "source": [ - "!ls ./tf_ckpts" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DR2wQc9x6b3X" - }, - "source": [ - "\n", - "## Mecanica de carga\n", - "\n", - "TensorFlow hace coincider las variables con los valores almacenados como *checkpoint* atravesando un grafo dirigido con aristas nombradas, comenzando con el objeto que este siendo cargado. Los nombres de las aristas vienen de los nombres de los atributos en los objetos, por ejemplo el `\"l1\"` en `self.l1 = tf.keras.layers.Dense(5)`. `tf.train.Checkpoint` usa sus argumentos de palabras clave como nombre, tal como el `\"step\"` en `tf.train.Checkpoint(step=...)`.\n", - "\n", - "El grafo de dependencias del ejemplo anterior se ve asi:\n", - "\n", - "![Visualization of the dependency graph for the example training loop](http://tensorflow.org/images/guide/whole_checkpoint.svg)\n", - "\n", - "Con el *optimizer* en rojo, las variables regulares en azul, y variables slot del *optimizer* en naranja. Los otros nodos, por ejemplo el que representa el `tf.train.Checkpoint`, son negros.\n", - "\n", - "Las variables *Slot* son parte del estado del *optimizer*, pero son creadas para una variable especifica. Por ejemplo las aristas `'m'` de arriba corresponden a un momentum, los cuales son rastreados por el *Adam's* *optimizer* para cada variable. Las variables *Slot* solo son almacenadas en un *checkpoint* si la variable y el optimizer serian ambas almacenadas, por eso los aristas punteados." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VpY5IuanUEQ0" - }, - "source": [ - "La llamada `restore()` de un objeto `tf.train.Checkpoint` hace cola las restauraciones requeridas, restaurando los valores de las variables tan pronto como se encuentre un *path* correspondiente en el objeto `Checkpoint`. Por ejemplo podemos cargar solo el kernel del model que definimos anteriormente recosntruyendo un path a el mediante la red y la capa (layer)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wmX2AuyH7TVt", - "colab": {} - }, - "source": [ - "to_restore = tf.Variable(tf.zeros([5]))\n", - "print(to_restore.numpy()) # Puros ceros\n", - "fake_layer = tf.train.Checkpoint(bias=to_restore)\n", - "fake_net = tf.train.Checkpoint(l1=fake_layer)\n", - "new_root = tf.train.Checkpoint(net=fake_net)\n", - "status = new_root.restore(tf.train.latest_checkpoint('./tf_ckpts/'))\n", - "print(to_restore.numpy()) # Ahora obtenemos el valor restaurado" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GqEW-_pJDAnE" - }, - "source": [ - "El grafo de dependencias para estos objetos nuevos es un sub-grafo del checkpoint que escribimos anteriormente. Solo incluye el *bias* y un *save counter* que el `tf.train.Checkpoint` usa para enumerar los *checkpoints*.\n", - "\n", - "![Visualization of a subgraph for the bias variable](http://tensorflow.org/images/guide/partial_checkpoint.svg)\n", - "\n", - "`restore()` regresa el estado del objeto, que tiene afirmaciones (assertions) opcionales. Todos los objetos que hemos creado en nuestro nuevo `Checkpoint` han sido restaurados, asi que `status.assert_existing_objects_matched()` pasa exitosamente." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "P9TQXl81Dq5r", - "colab": {} - }, - "source": [ - "status.assert_existing_objects_matched()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GoMwf8CFDu9r" - }, - "source": [ - "Hay muchos objetos en el *checkpoint* que no han sido emparejados, incluyendo el *kernel* de la capa y las variables del optimized. `status.assert_consumed()` solo pasa si el *checkpoint* y el programa empatan exactamente, y arrojara una excepcion en este caso." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KCcmJ-2j9RUP" - }, - "source": [ - "### Restauraciones retrasadas\n", - "\n", - "Los objetos `Layer` en TensorFlow pueden retrasar la creacion de variables para su primera llamada, cuando las dimensiones de entrada estan disponibles. Por ejemplo, las dimensiones de un *layer kernel* `Dense` dependen tanto de las entradas de la capa como de las dimensiones de salida, y por ende solo la dimension de salida que es requerida como argumento de construccion no es suficiente informacion para la creacion de las variables. Como la llamada a `Layer` tambien lee el valor de la variable, una restauracion debe pasa entre las variables de creacion y su primer uso.\n", - "\n", - "Para dar soporte a este idioma, `tf.train.Checkpoint` forma una lista de restauranciones que no tienen una vatiable que empate aun.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TXYUCO3v-I72", - "colab": {} - }, - "source": [ - "delayed_restore = tf.Variable(tf.zeros([1, 5]))\n", - "print(delayed_restore.numpy()) # No restaurado; siguen siendo ceros\n", - "fake_layer.kernel = delayed_restore\n", - "print(delayed_restore.numpy()) # Restaurado" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-DWhJ3glyobN" - }, - "source": [ - "### Revision manual de los checkpoints\n", - "\n", - "`tf.train.list_variables` lista las *checkpoint keys* y las dimensiones de las variables en un checkpoint. Las *Checkpoint keys* son los *paths* del grafo mostrado anteriormente." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RlRsADTezoBD", - "colab": {} - }, - "source": [ - "tf.train.list_variables(tf.train.latest_checkpoint('./tf_ckpts/'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5fxk_BnZ4W1b" - }, - "source": [ - "### Rastreo de listas y diccionarios\n", - "List and dictionary tracking\n", - "\n", - "Igual que en las asignaciones directas de atributos, e.g. `self.l1 = tf.keras.layers.Dense(5)`, la asignacion de listas y diccionarios a atributos rastreara sus contenidos." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rfaIbDtDHAr_", - "colab": {} - }, - "source": [ - "save = tf.train.Checkpoint()\n", - "save.listed = [tf.Variable(1.)]\n", - "save.listed.append(tf.Variable(2.))\n", - "save.mapped = {'one': save.listed[0]}\n", - "save.mapped['two'] = save.listed[1]\n", - "save_path = save.save('./tf_list_example')\n", - "\n", - "restore = tf.train.Checkpoint()\n", - "v2 = tf.Variable(0.)\n", - "assert 0. == v2.numpy() # No ha sido restaurado aun\n", - "restore.mapped = {'two': v2}\n", - "restore.restore(save_path)\n", - "assert 2. == v2.numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UTKvbxHcI3T2" - }, - "source": [ - "Puede notar que existen *wrappers* de objetos para listas y diccionarios. Estos *wrappers* pueden ser incluidos en versiones *checkpoint* de las estructuras de datos subyacientes. Asi como la carga basada en atributos, estos *wrappers* restauran el valor de una variable al momento de ser agregada al contenedor.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s0Uq1Hv5JCmm", - "colab": {} - }, - "source": [ - "restore.listed = []\n", - "print(restore.listed) # ListWrapper([])\n", - "v1 = tf.Variable(0.)\n", - "restore.listed.append(v1) # Restaurar v1, del restore() de la celda anterior\n", - "assert 1. == v1.numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OxCIf2J6JyQ8" - }, - "source": [ - "El mismo rastreo es aplicado automaticamente a subclases de `tf.keras.Model`, y puede ser usado para rastrear listas de capas por ejemplo.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zGG1tOM0L6iM" - }, - "source": [ - "## Guardar checkponts basados en objetos con Estimator\n", - "\n", - "Ver la [guia a Estimator](https://www.tensorflow.org/guide/estimator). NOTA: documentacion en ingles.\n", - "\n", - "Los Estimators guardan checkpoints por default con nombres de variables en lugar de el ogjeto grafo descrito en las secciones anteriores. `tf.train.Checkpoint` aceptara ckeckpoints basadon en nombres, pero los nombres de las variables podrian cambiar al movr las pertes del modelo fuera del `model_fn` del Estimator. Guardar checkpoints basados en objetos facilita el entrenamiento de un modelo dentro de un Estimator y su posterior uso fuera de el.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-8AMJeueNyoM", - "colab": {} - }, - "source": [ - "import tensorflow.compat.v1 as tf_compat" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "T6fQsBzJQN2y", - "colab": {} - }, - "source": [ - "def model_fn(features, labels, mode):\n", - " net = Net()\n", - " opt = tf.keras.optimizers.Adam(0.1)\n", - " ckpt = tf.train.Checkpoint(step=tf_compat.train.get_global_step(),\n", - " optimizer=opt, net=net)\n", - " with tf.GradientTape() as tape:\n", - " output = net(features['x'])\n", - " loss = tf.reduce_mean(tf.abs(output - features['y']))\n", - " variables = net.trainable_variables\n", - " gradients = tape.gradient(loss, variables)\n", - " return tf.estimator.EstimatorSpec(\n", - " mode,\n", - " loss=loss,\n", - " train_op=tf.group(opt.apply_gradients(zip(gradients, variables)),\n", - " ckpt.step.assign_add(1)),\n", - " # Decirle al Estimator gue guarde \"ckpt\" en un formato basado en objeto.\n", - " scaffold=tf_compat.train.Scaffold(saver=ckpt))\n", - "\n", - "tf.keras.backend.clear_session()\n", - "est = tf.estimator.Estimator(model_fn, './tf_estimator_example/')\n", - "est.train(toy_dataset, steps=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tObYHnrrb_mL" - }, - "source": [ - "`tf.train.Checkpoint` puede cargar los checkpoints del Estimator de su `model_dir`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Q6IP3Y_wb-fs", - "colab": {} - }, - "source": [ - "opt = tf.keras.optimizers.Adam(0.1)\n", - "net = Net()\n", - "ckpt = tf.train.Checkpoint(\n", - " step=tf.Variable(1, dtype=tf.int64), optimizer=opt, net=net)\n", - "ckpt.restore(tf.train.latest_checkpoint('./tf_estimator_example/'))\n", - "ckpt.step.numpy() # De est.train(..., steps=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "knyUFMrJg8y4" - }, - "source": [ - "## Resumen\n", - "\n", - "Los objetos de TensorFlow proveen un mecanismo facil y automatico para guardar y restaurar los valores de las variables que usan.\n" - ] - } - ] -} \ No newline at end of file diff --git a/site/es/guide/keras/custom_callback.ipynb b/site/es/guide/keras/custom_callback.ipynb deleted file mode 100644 index 9260aab253b..00000000000 --- a/site/es/guide/keras/custom_callback.ipynb +++ /dev/null @@ -1,587 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_callback", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "9xMkm699JzK8" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "597OjogAI3fy" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5bSCD8SyJC2g", - "colab": {}, - "cellView": "form" - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Rd7BF9BMTJ9l", - "colab_type": "text" - }, - "source": [ - "# Escribir callbacks de Keras personalizados" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E_ceEiH7g0MY" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Ver en TensorFlow.org\n", - " \n", - " Ejecutar en Google Colab\n", - " \n", - " Ver fuente en GitHub\n", - " \n", - " Descargar notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YLYPmTSZTM5V", - "colab_type": "text" - }, - "source": [ - "Note: Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad\n", - "son basados en el \"mejor esfuerzo\", no hay ninguna garantia que esta sea un reflejo preciso y actual \n", - "de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en).\n", - "Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un \"Pull request\"\n", - "al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs).\n", - "Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad\n", - "por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1OykC-6lI4gv" - }, - "source": [ - "Un callback personalizado es una herramienta poderosa para personalizar el comportamiento de un modelo de Keras durante el entrenamiento, evaluacion o inferencia, incluyendo la lectura/cambio del modelo de Keras. Ejemplos incluyen `tf.keras.callbacks.TensorBoard`, donde se pueden exportar y visualizar el progreso del entrenamiento y los resultados con TensorBoard, o `tf.keras.callbacks.ModelCheckpoint` donde el modelo es automaticamente guardado durante el entrenamiento, entre otros. En esta guia aprenderas que es un callback de Keras, cuando se llama, que puede hacer y como puedes construir una propia. Al final de la guia habra demos para la creacion de aplicaciones simples de callback para ayudarte a empezar tu propio callback personalizados." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d5zZ8rZD69VW" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7BazS4qD6-2n", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0c_TYhQOUe1j" - }, - "source": [ - "## Introduccion a los callbacks de Keras\n", - "En Keras 'Callback' es una clase de python destinada a ser subclase para proporcionar una funcionalidad específica, con un conjunto de métodos llamados en varias etapas de entrenamiento (incluyendo el inicio y fin de los batch/epoch), pruebas y predicciones. Los Callbacks son útiles para tener visibilidad de los estados internos y las estadísticas del modelo durante el entrenamiento. Puedes pasar una lista de callbacks (como argumento de palabra clave `callbacks`) a cualquiera de los siguientes metodos ` tf.keras.Model.fit () `,` tf.keras.Model.evaluate () `y` tf.keras.Model .predict () `. Los metodos de los callbacks se llamaran en diferentes etapas del entrenamiento/evaluación/inferencia.\n", - "\n", - "Para comenzar, importemos TensorDlow y definamos un modelo secuencial sencillo en Keras:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ct0VCSI2dt3a", - "colab": {} - }, - "source": [ - "# Definir el modelo de Keras model al que se le agregaran los callbacks\n", - "def get_model():\n", - " model = tf.keras.Sequential()\n", - " model.add(tf.keras.layers.Dense(1, activation = 'linear', input_dim = 784))\n", - " model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.1), loss='mean_squared_error', metrics=['mae'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ySzdG1IqNgah" - }, - "source": [ - "Luego, cara el dataset de MNIST para entrenamiento y pruebas de la APLI de datasetws de Keras:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fwo9LMKGNPWr", - "colab": {} - }, - "source": [ - "# Cargar los datos de ejemplo de MNIST data y preprocesarlos\n", - "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n", - "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n", - "x_test = x_test.reshape(10000, 784).astype('float32') / 255" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kHVK7kceNqH2" - }, - "source": [ - "Ahora, define un callback simple y personalizado para rastrear el inicio y fin de cada batch de datos. Durante esas llamadas, imprime el indice del batch actual." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-dfuGTMINKRR", - "colab": {} - }, - "source": [ - "import datetime\n", - "\n", - "class MyCustomCallback(tf.keras.callbacks.Callback):\n", - "\n", - " def on_train_batch_begin(self, batch, logs=None):\n", - " print('Entrenamiento: batch {} comienza en {}'.format(batch, datetime.datetime.now().time()))\n", - "\n", - " def on_train_batch_end(self, batch, logs=None):\n", - " print('Entrenamiento: batch {} termina en {}'.format(batch, datetime.datetime.now().time()))\n", - "\n", - " def on_test_batch_begin(self, batch, logs=None):\n", - " print('Evaluacion: batch {} comienza en {}'.format(batch, datetime.datetime.now().time()))\n", - "\n", - " def on_test_batch_end(self, batch, logs=None):\n", - " print('Evaluacion: batch {} termina en {}'.format(batch, datetime.datetime.now().time()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z4FTUUIBN3WG" - }, - "source": [ - "Dar un callback mara los metodos del modelo tales como `tf.keras.Model.fit()` aseguran que los metodos son llamados en dichas etapas:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NJV6Tj3sNGzg", - "colab": {} - }, - "source": [ - "model = get_model()\n", - "_ = model.fit(x_train, y_train,\n", - " batch_size=64,\n", - " epochs=1,\n", - " steps_per_epoch=5,\n", - " verbose=0,\n", - " callbacks=[MyCustomCallback()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fIy5JKMlZNmh" - }, - "source": [ - "## Metodos del Modelo que aceptan callbacks\n", - "Los usuarios pueden dar una lista de callbacks para los siguientes metodos de `tf.keras.Model`:\n", - "#### [`fit()`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model#fit), [`fit_generator()`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model#fit_generator)\n", - "Entrena el modelo por una cantidad determinada de epochs (iteraciones en un dataset, o para los datos determinados por un generador de Python que va batch-por-batch).\n", - "#### [`evaluate()`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model#evaluate), [`evaluate_generator()`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model#evaluate_generator)\n", - "Evalua el modelo para determinados datos o generador de datos. Regresa la perdida (loss) y valores metricos para la evaluacion.\n", - "#### [`predict()`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model#predict), [`predict_generator()`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model#predict_generator)\n", - "Genera las predicciones a regresar para los datos ingresados o el generador de datos.\n", - "NOTA: Toda la documentacion esta en ingles.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J00bXBbqdnJe", - "colab": {} - }, - "source": [ - "_ = model.evaluate(x_test, y_test, batch_size=128, verbose=0, steps=5,\n", - " callbacks=[MyCustomCallback()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "13n44LVkYQsV" - }, - "source": [ - "## Una revision de los metodos de callback\n", - "\n", - "\n", - "### Metodos comunes para entrenamiento/pruebas/prediccion\n", - "Para entrenamiento, pruebas y prediccion, los siguientes metodos se han previsto para ser sobreescritos.\n", - "#### `on_(train|test|predict)_begin(self, logs=None)`\n", - "Llamado al inicio de `fit`/`evaluate`/`predict`.\n", - "#### `on_(train|test|predict)_end(self, logs=None)`\n", - "Llamado al fin de `fit`/`evaluate`/`predict`.\n", - "#### `on_(train|test|predict)_batch_begin(self, batch, logs=None)`\n", - "Llamado justo antes de procesar un batch durante entrenamiento/pruebas/prediccion. Dentro de este metodo, `logs` es un diccionario con las llaves `batch` y `size` disponibles, representando el numero de batch actual y las dimensiones del mismo.\n", - "#### `on_(train|test|predict)_batch_end(self, batch, logs=None)`\n", - "Llamado al final del entrenamiento/pruebas/prediccion de un batch. dentro de este metodo, `logs` es un diccionario que contiene resultados metricos con estado.\n", - "\n", - "### Entrenamiento de metodos especificos\n", - "Adicionalmente, para el entrenamiento, los siguientes metodos son provistos.\n", - "#### on_epoch_begin(self, epoch, logs=None)\n", - "Llamado al inicio de una epoch durante el entrenamiento.\n", - "#### on_epoch_end(self, epoch, logs=None)\n", - "Llamado al final de una epoch durante el entrenamiento.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SWf3mXYoceCz" - }, - "source": [ - "### Uso del diccionario `logs`\n", - "El diccionario `logs` contiene el valor de perdida (loss), y todas las metricas pertinentes al final de un batch o epoch. El ejemplo a continuacion incluye la perdidad (loss) y el MAE (Mean Absolute Error)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u4wIdcF9BjJH", - "colab": {} - }, - "source": [ - "class LossAndErrorPrintingCallback(tf.keras.callbacks.Callback):\n", - "\n", - " def on_train_batch_end(self, batch, logs=None):\n", - " print('Para el batch {}, la perdida (loss) es {:7.2f}.'.format(batch, logs['loss']))\n", - "\n", - " def on_test_batch_end(self, batch, logs=None):\n", - " print('Para el batch {}, la perdida (loss) es {:7.2f}.'.format(batch, logs['loss']))\n", - "\n", - " def on_epoch_end(self, epoch, logs=None):\n", - " print('La perdida promedio para la epoch {} es {:7.2f} y el MAE es {:7.2f}.'.format(epoch, logs['loss'], logs['mae']))\n", - "\n", - "model = get_model()\n", - "_ = model.fit(x_train, y_train,\n", - " batch_size=64,\n", - " steps_per_epoch=5,\n", - " epochs=3,\n", - " verbose=0,\n", - " callbacks=[LossAndErrorPrintingCallback()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LbXqvC8FHqeu" - }, - "source": [ - "De manera similar, uno puede proveer callbacks en las llamadas a `evaluate()`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jzTKYPQHwcxF", - "colab": {} - }, - "source": [ - "_ = model.evaluate(x_test, y_test, batch_size=128, verbose=0, steps=20,\n", - " callbacks=[LossAndErrorPrintingCallback()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HnSljqtsXKfb" - }, - "source": [ - "## Ejemplos de aplicaciones de callbacks de Keras\n", - "La siguiente seccion te guiara en la creacion de una aplicacion de callback simple." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kptNF0--Lznv" - }, - "source": [ - "### Detencion anticipada con perdida minima.\n", - "El primer ejemplo muestra la creacion de un `Callback` que detiene el entrenamiento de Keras cuando se alcanza el minimo de perdida mutando el atributo` model.stop_training` (boolean). Opcionalmente, el usuario puede proporcionar el argumento `patience` para especificar cuantas epochs debe esperar el entrenamiento antes de detenerse.\n", - "\n", - "`tf.keras.callbacks.EarlyStopping` proporciona una implementación mas completa y general." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BM31gfAV4mks", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "class EarlyStoppingAtMinLoss(tf.keras.callbacks.Callback):\n", - " \"\"\"Detener el entrenamiento cuando la perdida (loss) esta en su minimo, i.e. la perdida (loss) deja de disminuir.\n", - "\n", - " Arguments:\n", - " patience: Numero de epochs a esperar despues de que el min ha sido alcanzaado. Despues de este numero\n", - " de no mejoras, el entrenamiento para.\n", - " \"\"\"\n", - "\n", - " def __init__(self, patience=0):\n", - " super(EarlyStoppingAtMinLoss, self).__init__()\n", - "\n", - " self.patience = patience\n", - "\n", - " # best_weights para almacenar los pesos en los cuales ocurre la perdida minima.\n", - " self.best_weights = None\n", - "\n", - " def on_train_begin(self, logs=None):\n", - " # El numero de epoch que ha esperado cuando la perdida ya no es minima.\n", - " self.wait = 0\n", - " # El epoch en el que en entrenamiento se detiene.\n", - " self.stopped_epoch = 0\n", - " # Initialize el best como infinito.\n", - " self.best = np.Inf\n", - "\n", - " def on_epoch_end(self, epoch, logs=None):\n", - " current = logs.get('loss')\n", - " if np.less(current, self.best):\n", - " self.best = current\n", - " self.wait = 0\n", - " # Guardar los mejores pesos si el resultado actual es mejor (menos).\n", - " self.best_weights = self.model.get_weights()\n", - " else:\n", - " self.wait += 1\n", - " if self.wait >= self.patience:\n", - " self.stopped_epoch = epoch\n", - " self.model.stop_training = True\n", - " print('Restaurando los pesos del modelo del final de la mejor epoch.')\n", - " self.model.set_weights(self.best_weights)\n", - "\n", - " def on_train_end(self, logs=None):\n", - " if self.stopped_epoch > 0:\n", - " print('Epoch %05d: Detencion anticipada' % (self.stopped_epoch + 1))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xS4fa-7PFzzc", - "colab": {} - }, - "source": [ - "model = get_model()\n", - "_ = model.fit(x_train, y_train,\n", - " batch_size=64,\n", - " steps_per_epoch=5,\n", - " epochs=30,\n", - " verbose=0,\n", - " callbacks=[LossAndErrorPrintingCallback(), EarlyStoppingAtMinLoss()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SpVDjs_Dkkdh" - }, - "source": [ - "### Programacion del Learning Rate\n", - "\n", - "Algo que es hecho comunmente en el entrenamiento de un modelo es cambiar el learning rate conforme pasan mas epochs. El backend de Keras expone la API `get_value` la cual puede ser usada para definir las variables. En este ejemplo estamos mostrando como un Callback personalizado puede ser usado para cambiar dinamicamente el learning rate.\n", - "\n", - "Nota: este es solo una implementacion de ejemplo, `callbacks.LearningRateScheduler` y `keras.optimizers.schedules` contienen implementaciones mas generales." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "PGowEUC8klSz", - "colab": {} - }, - "source": [ - "class LearningRateScheduler(tf.keras.callbacks.Callback):\n", - " \"\"\"Planificador de Learning rate que define el learning rate deacuerdo a lo programado.\n", - "\n", - " Arguments:\n", - " schedule: una funcion que toma el indice del epoch\n", - " (entero, indexado desde 0) y el learning rate actual\n", - " como entradas y regresa un nuevo learning rate como salida (float).\n", - " \"\"\"\n", - "\n", - " def __init__(self, schedule):\n", - " super(LearningRateScheduler, self).__init__()\n", - " self.schedule = schedule\n", - "\n", - " def on_epoch_begin(self, epoch, logs=None):\n", - " if not hasattr(self.model.optimizer, 'lr'):\n", - " raise ValueError('Optimizer must have a \"lr\" attribute.')\n", - " # Obtener el learning rate actua del optimizer del modelo.\n", - " lr = float(tf.keras.backend.get_value(self.model.optimizer.lr))\n", - " # Llamar la funcion schedule para obtener el learning rate programado.\n", - " scheduled_lr = self.schedule(epoch, lr)\n", - " # Definir el valor en el optimized antes de que la epoch comience\n", - " tf.keras.backend.set_value(self.model.optimizer.lr, scheduled_lr)\n", - " print('\\nEpoch %05d: Learning rate is %6.4f.' % (epoch, scheduled_lr))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1jL3pI5Ep5F8", - "colab": {} - }, - "source": [ - "LR_SCHEDULE = [\n", - " # (epoch a comenzar, learning rate) tupla\n", - " (3, 0.05), (6, 0.01), (9, 0.005), (12, 0.001)\n", - "]\n", - "\n", - "def lr_schedule(epoch, lr):\n", - " \"\"\"Funcion de ayuda para recuperar el learning rate programado basado en la epoch.\"\"\"\n", - " if epoch < LR_SCHEDULE[0][0] or epoch > LR_SCHEDULE[-1][0]:\n", - " return lr\n", - " for i in range(len(LR_SCHEDULE)):\n", - " if epoch == LR_SCHEDULE[i][0]:\n", - " return LR_SCHEDULE[i][1]\n", - " return lr\n", - "\n", - "model = get_model()\n", - "_ = model.fit(x_train, y_train,\n", - " batch_size=64,\n", - " steps_per_epoch=5,\n", - " epochs=15,\n", - " verbose=0,\n", - " callbacks=[LossAndErrorPrintingCallback(), LearningRateScheduler(lr_schedule)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9xMkm699JzK8" - }, - "source": [ - "### Callbacks de Keras estandar\n", - "Asegurate de revisar los callbacks de Keras preexistentes [visitando la documentacion de la api](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/callbacks). Las aplicaciones incluyen el registro a CSV, guardar el modelo, visualizar en TensorBoard y mucho mas.\n", - "\n", - "NOTA: La documentacion aun esta en ingles" - ] - } - ] -} \ No newline at end of file diff --git a/site/es/guide/keras/functional.ipynb b/site/es/guide/keras/functional.ipynb deleted file mode 100644 index c30cbd5429d..00000000000 --- a/site/es/guide/keras/functional.ipynb +++ /dev/null @@ -1,1531 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "functional.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-zMKQx6DkKwt" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "J307vsiDkMMW", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vCMYwDIE9dTT" - }, - "source": [ - "# La API funcional \"Keras\" en TensorFlow" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lAJfkZ-K9flj" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Ver en TensorFlow.org\n", - " \n", - " Correr en Google Colab\n", - " \n", - " Ver código fuente en GitHub\n", - " \n", - " Descargar notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dt4DDXp2Vca-", - "colab_type": "text" - }, - "source": [ - "Note: Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad\n", - "son basados en el \"mejor esfuerzo\", no hay ninguna garantia que esta sea un reflejo preciso y actual \n", - "de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en).\n", - "Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un \"Pull request\"\n", - "al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs).\n", - "Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad\n", - "por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ITh3wzORxgpw" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HFbM9dcfxh4l", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version de Colab\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "tf.keras.backend.clear_session() # Reseteo sencillo" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZI47-lpfkZ5c" - }, - "source": [ - "\n", - "## Introduccion\n", - "\n", - "Ya estás familiarizado con el uso del metodo `keras.Sequential()` para crear modelos.\n", - "La API funcional es una forma de crear modelos mas dinamicos que con ` Sequential `: La API funcional puede manejar modelos con topología no lineal, modelos con capas compartidas y modelos con múltiples entradas o salidas.\n", - "\n", - "Se basa en la idea de que un modelo de aprendizaje profundo\n", - "suele ser un gráfico acíclico dirigido (DAG) de capas.\n", - "La API funcional es un conjunto de herramientas para **construir gráficos de capas**.\n", - "\n", - "Considera el siguiente modelo:\n", - "\n", - "```\n", - "(input: 784-vectores dimensionales)\n", - " ↧\n", - "[Dense (64 units, activacion relu)]\n", - " ↧\n", - "[Dense (64 units, activacion relu)]\n", - " ↧\n", - "[Dense (10 units, activacion softmax)]\n", - " ↧\n", - "(output: distribución de probabilidad en 10 clases)\n", - "```\n", - "\n", - "Es una simple grafica de tres capas.\n", - "\n", - "Para construir este modelo con la API funcional,\n", - "comenzarías creando un nodo de entrada:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Yxi0LaSHkDT-", - "colab": {} - }, - "source": [ - "from tensorflow import keras\n", - "\n", - "inputs = keras.Input(shape=(784,))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Mr3Z_Pxcnf-H" - }, - "source": [ - "Aqui solo especificamos el tipo de nuestra data set: 784-vectores dimensionales.\n", - "Nota que el tamaño del batch siempre debe ser omitido, solo se incluye el tipo de la data set.\n", - "Para una input de tipo imágen ` (31,32,3) ` hubiese sido:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0-2Q2nJNneIO", - "colab": {} - }, - "source": [ - "img_inputs = keras.Input(shape=(32, 32, 3))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HoMFNu-pnkgF" - }, - "source": [ - "Lo que se devuelve, ` input `, contiene información sobre la forma y el tipo de dato que se espera ingresa en tu modelo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ddIr9LPJnibj", - "colab": {} - }, - "source": [ - "inputs.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lZkLJeQonmTe", - "colab": {} - }, - "source": [ - "inputs.dtype" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kZnhhndTnrzC" - }, - "source": [ - "Puedes crear un nuevo nodo en el grafico de capas mandando a llamar al objeto ` input `." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sMyyMTqDnpYV", - "colab": {} - }, - "source": [ - "from tensorflow.keras import layers\n", - "\n", - "dense = layers.Dense(64, activation='relu')\n", - "x = dense(inputs)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "besm-lgFnveV" - }, - "source": [ - "La acción \"layer call\" es como dibujar una flecha desde \"entradas\" a la capa que creamos.\n", - "Estamos \"pasando\" las entradas a la capa `dense`, y afuera obtenemos` x`.\n", - "\n", - "Agreguemos algunas capas más a nuestro gráfico de capas:\n", - "\n", - "La acción \"llamada a la capa\" es como dibujar una flecha de \"entradas\" a la capa que creamos.\n", - "\n", - "Estamos pasando las entradas a una capa mas densa, y respecto a la salida obtenemos una ` x `." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DbF-MIO2ntf7", - "colab": {} - }, - "source": [ - "x = layers.Dense(64, activation='relu')(x)\n", - "outputs = layers.Dense(10, activation='softmax')(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B38UlEIlnz_8" - }, - "source": [ - "LLegados a este punto, podemos crear un ` Modelo ` especificando sus entradas y salidas en las capas de graficas." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MrSfwvl-nx9s", - "colab": {} - }, - "source": [ - "model = keras.Model(inputs=inputs, outputs=outputs)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5EeeV1xJn3jW" - }, - "source": [ - "Recapitulando, esta es nuestra definción completa del proceso:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xkz7oqj2n1-q", - "colab": {} - }, - "source": [ - "inputs = keras.Input(shape=(784,), name='img')\n", - "x = layers.Dense(64, activation='relu')(inputs)\n", - "x = layers.Dense(64, activation='relu')(x)\n", - "outputs = layers.Dense(10, activation='softmax')(x)\n", - "\n", - "model = keras.Model(inputs=inputs, outputs=outputs, name='mnist_model')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jJzocCbdn6qj" - }, - "source": [ - "Veamos como se muestra el model summary:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GirC9odQn5Ep", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mbNqYAlOn-vA" - }, - "source": [ - "También podemos trazar el modelo como un gráfico:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JYh2wLain8Oi", - "colab": {} - }, - "source": [ - "keras.utils.plot_model(model, 'my_first_model.png')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QtgX2RoGoDZo" - }, - "source": [ - "Y opcionalmente mostrar la entrada y la salida de la forma de cada capa en la gráfica ploteada:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FGesSSuoAG5", - "colab": {} - }, - "source": [ - "keras.utils.plot_model(model, 'my_first_model_with_shape_info.png', show_shapes=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PBZ9XE6LoWvi" - }, - "source": [ - "Esta figura y el código que escribimos son prácticamente idénticos. En la versión de código, las flechas de conexión simplemente se reemplazan por la operación de llamada.\n", - "\n", - "Un \"gráfico de capas\" es una imagen mental muy intuitiva para un modelo de aprendizaje profundo, y la API funcional es una forma de crear modelos que reflejan de cerca esta imagen mental.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WUUHMaKLoZDn" - }, - "source": [ - "\n", - "\n", - "## Entrenamiento, evaluación e inferencia.\n", - "\n", - "El entrenamiento, la evaluación y la inferencia funcionan exactamente de la misma manera para los modelos construidos\n", - "utilizando la API funcional como para los modelos secuenciales.\n", - "\n", - "Aquí hay una demostración rápida.\n", - "\n", - "Aquí cargamos datos de imagen MNIST, los rediseñamos en vectores,\n", - "ajustar el modelo en los datos (mientras se monitorea el rendimiento en una división de validación),\n", - "y finalmente evaluamos nuestro modelo en los datos de prueba:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DnHvkD22oFEY", - "colab": {} - }, - "source": [ - "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n", - "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n", - "x_test = x_test.reshape(10000, 784).astype('float32') / 255\n", - "\n", - "model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=keras.optimizers.RMSprop(),\n", - " metrics=['accuracy'])\n", - "history = model.fit(x_train, y_train,\n", - " batch_size=64,\n", - " epochs=5,\n", - " validation_split=0.2)\n", - "test_scores = model.evaluate(x_test, y_test, verbose=2)\n", - "print('Test loss:', test_scores[0])\n", - "print('Test accuracy:', test_scores[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c3nq2fjiLCkE" - }, - "source": [ - "Para obtener una guía completa sobre el entrenamiento y evaluación de modelos, consulta [Guía de entrenamiento y evaluación](./train_and_evaluate.ipynb)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XOsL56zDorLh" - }, - "source": [ - "## Almacenado y serialización\n", - "\n", - "El almacenado y la serialización funcionan exactamente de la misma manera para los modelos construidos\n", - "utilizando la API funcional como para los modelos secuenciales.\n", - "\n", - "Una forma estándar de guardar un modelo funcional es llamar a `model.save ()` para guardar todo el modelo en un solo archivo.\n", - "Posteriormente, puede volver a crear el mismo modelo a partir de este archivo, incluso si ya no tiene acceso al código.\n", - "eso creó el modelo.\n", - "\n", - "Este archivo incluye:\n", - "- La arquitectura del modelo.\n", - "- Los valores de peso del modelo (que se aprendieron durante el entrenamiento)\n", - "- La configuración de entrenamiento del modelo (lo que pasó a `compilar`), si corresponde\n", - "- El optimizador y su estado, si corresponde (esto le permite reiniciar el entrenamiento donde lo dejó)" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kN-AO7xvobtr", - "colab": {} - }, - "source": [ - "model.save('path_to_my_model.h5')\n", - "del model\n", - "# Recrea el mismo modelo, desde el archivo:\n", - "model = keras.models.load_model('path_to_my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u0J0tFPHK4pb" - }, - "source": [ - "Para obtener una guía completa sobre el guardado de modelos, consulta [Guía para guardar y serializar modelos](./save_and_serialize.ipynb)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lKz1WWr2LUzF" - }, - "source": [ - "## Usando el mismo gráfico de capas para definir múltiples modelos\n", - "\n", - "\n", - "En la API funcional, los modelos se crean especificando sus entradas\n", - "y salidas en un gráfico de capas. Eso significa que un solo gráfico de capas\n", - "Se puede utilizar para generar múltiples modelos.\n", - "\n", - "En el siguiente ejemplo, usamos la misma arquitectura de capas para crear instancias de dos modelos:\n", - "un modelo de \"codificador\" que convierte las entradas de imagen en vectores de 16 dimensiones,\n", - "y un modelo completo de `autoencoder` para entrenamiento.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WItZQr6LuVbF", - "colab": {} - }, - "source": [ - "encoder_input = keras.Input(shape=(28, 28, 1), name='img')\n", - "x = layers.Conv2D(16, 3, activation='relu')(encoder_input)\n", - "x = layers.Conv2D(32, 3, activation='relu')(x)\n", - "x = layers.MaxPooling2D(3)(x)\n", - "x = layers.Conv2D(32, 3, activation='relu')(x)\n", - "x = layers.Conv2D(16, 3, activation='relu')(x)\n", - "encoder_output = layers.GlobalMaxPooling2D()(x)\n", - "\n", - "encoder = keras.Model(encoder_input, encoder_output, name='encoder')\n", - "encoder.summary()\n", - "\n", - "x = layers.Reshape((4, 4, 1))(encoder_output)\n", - "x = layers.Conv2DTranspose(16, 3, activation='relu')(x)\n", - "x = layers.Conv2DTranspose(32, 3, activation='relu')(x)\n", - "x = layers.UpSampling2D(3)(x)\n", - "x = layers.Conv2DTranspose(16, 3, activation='relu')(x)\n", - "decoder_output = layers.Conv2DTranspose(1, 3, activation='relu')(x)\n", - "\n", - "autoencoder = keras.Model(encoder_input, decoder_output, name='autoencoder')\n", - "autoencoder.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oNeg3WWFuYZK" - }, - "source": [ - "Tenga en cuenta que hacemos que la arquitectura de decodificación sea estrictamente simétrica a la arquitectura de codificación,\n", - "para que obtengamos una forma de salida que sea igual a la forma de entrada `(28, 28, 1)`.\n", - "El reverso de una capa `Conv2D` es una capa` Conv2DTranspose`, y el reverso de una capa `MaxPooling2D`\n", - "La capa es una capa `UpSampling2D`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "h1FVW4j-uc6Y" - }, - "source": [ - "\n", - "## Todos los modelos son invocables, al igual que las capas.\n", - "\n", - "Puede tratar cualquier modelo como si fuera una capa, llamándolo en una `Entrada` o en la salida de otra capa.\n", - "Tenga en cuenta que al llamar a un modelo no solo está reutilizando la arquitectura del modelo, también está reutilizando sus pesos.\n", - "\n", - "Veamos esto en acción. Aquí hay una versión diferente del ejemplo de autoencoder que crea un modelo de codificador, un modelo de decodificador,\n", - "y encadenarlos en dos llamadas para obtener el modelo de autoencoder:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ld7KdsQ_uZbr", - "colab": {} - }, - "source": [ - "encoder_input = keras.Input(shape=(28, 28, 1), name='original_img')\n", - "x = layers.Conv2D(16, 3, activation='relu')(encoder_input)\n", - "x = layers.Conv2D(32, 3, activation='relu')(x)\n", - "x = layers.MaxPooling2D(3)(x)\n", - "x = layers.Conv2D(32, 3, activation='relu')(x)\n", - "x = layers.Conv2D(16, 3, activation='relu')(x)\n", - "encoder_output = layers.GlobalMaxPooling2D()(x)\n", - "\n", - "encoder = keras.Model(encoder_input, encoder_output, name='encoder')\n", - "encoder.summary()\n", - "\n", - "decoder_input = keras.Input(shape=(16,), name='encoded_img')\n", - "x = layers.Reshape((4, 4, 1))(decoder_input)\n", - "x = layers.Conv2DTranspose(16, 3, activation='relu')(x)\n", - "x = layers.Conv2DTranspose(32, 3, activation='relu')(x)\n", - "x = layers.UpSampling2D(3)(x)\n", - "x = layers.Conv2DTranspose(16, 3, activation='relu')(x)\n", - "decoder_output = layers.Conv2DTranspose(1, 3, activation='relu')(x)\n", - "\n", - "decoder = keras.Model(decoder_input, decoder_output, name='decoder')\n", - "decoder.summary()\n", - "\n", - "autoencoder_input = keras.Input(shape=(28, 28, 1), name='img')\n", - "encoded_img = encoder(autoencoder_input)\n", - "decoded_img = decoder(encoded_img)\n", - "autoencoder = keras.Model(autoencoder_input, decoded_img, name='autoencoder')\n", - "autoencoder.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "icQFny_huiXC" - }, - "source": [ - "Como puede ver, el modelo puede estar anidado: un modelo puede contener submodelos (ya que un modelo es como una capa).\n", - "\n", - "Un caso de uso común para la anidación de modelos es * ensamblaje *.\n", - "Como ejemplo, a continuación se explica cómo agrupar un conjunto de modelos en un solo modelo que promedia sus predicciones:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZBlZbRn5uk-9", - "colab": {} - }, - "source": [ - "def get_model():\n", - " inputs = keras.Input(shape=(128,))\n", - " outputs = layers.Dense(1, activation='sigmoid')(inputs)\n", - " return keras.Model(inputs, outputs)\n", - "\n", - "model1 = get_model()\n", - "model2 = get_model()\n", - "model3 = get_model()\n", - "\n", - "inputs = keras.Input(shape=(128,))\n", - "y1 = model1(inputs)\n", - "y2 = model2(inputs)\n", - "y3 = model3(inputs)\n", - "outputs = layers.average([y1, y2, y3])\n", - "ensemble_model = keras.Model(inputs=inputs, outputs=outputs)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e1za1TZxuoId" - }, - "source": [ - "## Manipulación de topologías gráficas complejas\n", - "\n", - "\n", - "### Modelos con múltiples entradas y salidas\n", - "\n", - "La API funcional facilita la manipulación de múltiples entradas y salidas.\n", - "Esto no se puede manejar con la API secuencial.\n", - "\n", - "Aquí hay un ejemplo simple.\n", - "\n", - "Supongamos que está creando un sistema para clasificar los tickets de emisión personalizados por prioridad y enrutarlos al departamento correcto.\n", - "\n", - "Tu modelo tendrá 3 entradas:\n", - "\n", - "- Título del ticket (entrada de texto)\n", - "- Cuerpo del texto del ticket (entrada de texto)\n", - "- Cualquier etiqueta agregada por el usuario (entrada categórica)\n", - "\n", - "Tendrá dos salidas:\n", - "\n", - "- Puntuación de prioridad entre 0 y 1 (salida sigmoidea escalar)\n", - "- El departamento que debe manejar el ticket (salida softmax sobre el conjunto de departamentos)\n", - "\n", - "Construyamos este modelo en pocas líneas con la API funcional." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gt91OtzbutJy", - "colab": {} - }, - "source": [ - "num_tags = 12 # Número de etiquetas de problemas únicos\n", - "num_words = 10000 # Tamaño del vocabulario obtenido al preprocesar datos de texto\n", - "num_departments = 4 # Número de departamentos para predicciones.\n", - "\n", - "title_input = keras.Input(shape=(None,), name='title') # Secuencia de longitud variable de entradas\n", - "body_input = keras.Input(shape=(None,), name='body') # Secuencia de longitud variable de entradas\n", - "tags_input = keras.Input(shape=(num_tags,), name='tags') # Vectores binarios de tamaño `num_tags`\n", - "\n", - "# Ingresa cada palabra en el título en un vector de 64 dimensiones\n", - "title_features = layers.Embedding(num_words, 64)(title_input)\n", - "# Ingresa cada palabra en el texto en un vector de 64 dimensiones\n", - "body_features = layers.Embedding(num_words, 64)(body_input)\n", - "\n", - "# Reduce la secuencia de palabras ingresadas en el título en un solo vector de 128 dimensiones\n", - "title_features = layers.LSTM(128)(title_features)\n", - "# Reduce la secuencia de palabras ingresadas en el cuerpo en un solo vector de 32 dimensiones\n", - "body_features = layers.LSTM(32)(body_features)\n", - "\n", - "# Combina todas las funciones disponibles en un solo vector grande mediante concatenación\n", - "x = layers.concatenate([title_features, body_features, tags_input])\n", - "\n", - "# Pegua una regresión logística para la predicción de prioridad en la parte superior de las características\n", - "priority_pred = layers.Dense(1, activation='sigmoid', name='priority')(x)\n", - "# Stick a department classifier on top of the features\n", - "department_pred = layers.Dense(num_departments, activation='softmax', name='department')(x)\n", - "\n", - "# Instancia un modelo de extremo a extremo que prediga tanto la prioridad como el departamento\n", - "model = keras.Model(inputs=[title_input, body_input, tags_input],\n", - " outputs=[priority_pred, department_pred])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KIS7lqW0uwh-" - }, - "source": [ - "Ploteando el modelo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IMij4gzhuzYV", - "colab": {} - }, - "source": [ - "keras.utils.plot_model(model, 'multi_input_and_output_model.png', show_shapes=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oOyuig2Hu00p" - }, - "source": [ - "Al compilar este modelo, podemos asignar diferentes pérdidas a cada salida.\n", - "Incluso puede asignar diferentes pesos a cada pérdida, para modular su\n", - "contribución a la pérdida total de entrenamiento." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Crtdpi5Uu2cX", - "colab": {} - }, - "source": [ - "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n", - " loss=['binary_crossentropy', 'categorical_crossentropy'],\n", - " loss_weights=[1., 0.2])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t42Jrn0Yu5jL" - }, - "source": [ - "Como dimos nombres a nuestras capas de salida, también podríamos especificar la pérdida de esta manera:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dPM0EwW_u6mV", - "colab": {} - }, - "source": [ - "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n", - " loss={'priority': 'binary_crossentropy',\n", - " 'department': 'categorical_crossentropy'},\n", - " loss_weights=[1., 0.2])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpTx2sXnu3-W" - }, - "source": [ - "Podemos entrenar el modelo pasando listas de matrices Numpy de entradas y objetivos:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nB-upOoGu_k4", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "# Datos de entrada ficticios\n", - "title_data = np.random.randint(num_words, size=(1280, 10))\n", - "body_data = np.random.randint(num_words, size=(1280, 100))\n", - "tags_data = np.random.randint(2, size=(1280, num_tags)).astype('float32')\n", - "# Datos objetivo ficticios\n", - "priority_targets = np.random.random(size=(1280, 1))\n", - "dept_targets = np.random.randint(2, size=(1280, num_departments))\n", - "\n", - "model.fit({'title': title_data, 'body': body_data, 'tags': tags_data},\n", - " {'priority': priority_targets, 'department': dept_targets},\n", - " epochs=2,\n", - " batch_size=32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qNguhBWuvCtz" - }, - "source": [ - "Al llamar al ajuste con un objeto `Dataset`, debería producir un\n", - "tupla de listas como `([title_data, body_data, tags_data], [priority_targets, dept_targets])`\n", - "o una tupla de diccionarios como\n", - "`({'title': title_data, 'body': body_data, 'tags': tags_data}, {'priority': priority_targets, 'department': dept_targets})`.\n", - "\n", - "Para obtener una explicación más detallada, consulta la guía completa [Guía de entrenamiento y evaluación](./train_and_evaluate.ipynb)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tR0X5tTOvPyg" - }, - "source": [ - "### Un modelo de Red neuronal residual de juguete\n", - "\n", - "Además de los modelos con múltiples entradas y salidas,\n", - "La API funcional facilita la manipulación de topologías de conectividad no lineal,\n", - "es decir, modelos donde las capas no están conectadas secuencialmente.\n", - "Esto tampoco se puede manejar con la API secuencial (como su nombre lo indica).\n", - "\n", - "Un caso de uso común para esto son las conexiones residuales.\n", - "\n", - "Construyamos un modelo de ResNet de juguete para CIFAR10 para demostrar esto." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VzMoYrMNvXrm", - "colab": {} - }, - "source": [ - "inputs = keras.Input(shape=(32, 32, 3), name='img')\n", - "x = layers.Conv2D(32, 3, activation='relu')(inputs)\n", - "x = layers.Conv2D(64, 3, activation='relu')(x)\n", - "block_1_output = layers.MaxPooling2D(3)(x)\n", - "\n", - "x = layers.Conv2D(64, 3, activation='relu', padding='same')(block_1_output)\n", - "x = layers.Conv2D(64, 3, activation='relu', padding='same')(x)\n", - "block_2_output = layers.add([x, block_1_output])\n", - "\n", - "x = layers.Conv2D(64, 3, activation='relu', padding='same')(block_2_output)\n", - "x = layers.Conv2D(64, 3, activation='relu', padding='same')(x)\n", - "block_3_output = layers.add([x, block_2_output])\n", - "\n", - "x = layers.Conv2D(64, 3, activation='relu')(block_3_output)\n", - "x = layers.GlobalAveragePooling2D()(x)\n", - "x = layers.Dense(256, activation='relu')(x)\n", - "x = layers.Dropout(0.5)(x)\n", - "outputs = layers.Dense(10, activation='softmax')(x)\n", - "\n", - "model = keras.Model(inputs, outputs, name='toy_resnet')\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISQX32bgrkis" - }, - "source": [ - "Ploteando el modelo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pNFVkAd3rlCM", - "colab": {} - }, - "source": [ - "keras.utils.plot_model(model, 'mini_resnet.png', show_shapes=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ECcG87yZrxp5" - }, - "source": [ - "Vamos a entrenarlo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_iXGz5XEryou", - "colab": {} - }, - "source": [ - "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar10.load_data()\n", - "x_train = x_train.astype('float32') / 255.\n", - "x_test = x_test.astype('float32') / 255.\n", - "y_train = keras.utils.to_categorical(y_train, 10)\n", - "y_test = keras.utils.to_categorical(y_test, 10)\n", - "\n", - "model.compile(optimizer=keras.optimizers.RMSprop(1e-3),\n", - " loss='categorical_crossentropy',\n", - " metrics=['acc'])\n", - "model.fit(x_train, y_train,\n", - " batch_size=64,\n", - " epochs=1,\n", - " validation_split=0.2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XQfg0JUkr7SH" - }, - "source": [ - "## Compartir capas\n", - "\n", - "Otro buen uso de la API funcional son los modelos que usan capas compartidas. Las capas compartidas son instancias de capa que se reutilizan varias veces en un mismo modelo: aprenden características que corresponden a múltiples rutas en el gráfico de capas.\n", - "\n", - "Las capas compartidas a menudo se usan para codificar entradas que provienen de espacios similares (por ejemplo, dos piezas de texto diferentes que presentan un vocabulario similar), ya que permiten compartir información entre estas diferentes entradas y hacen posible entrenar un modelo de este tipo en menos datos. Si se ve una palabra determinada en una de las entradas, eso beneficiará el procesamiento de todas las entradas que pasan por la capa compartida.\n", - "\n", - "Para compartir una capa en la API funcional, simplemente llame a la misma instancia de capa varias veces. Por ejemplo, aquí hay una capa `Ingresa (del ingles Embedding)` compartida entre dos entradas de texto diferentes:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R9pAPQCnKuMR", - "colab": {} - }, - "source": [ - "# Ingreso de 1000 palabras únicas asignadas a vectores de 128 dimensiones\n", - "shared_embedding = layers.Embedding(1000, 128)\n", - "\n", - "# Secuencia de longitud variable de enteros\n", - "text_input_a = keras.Input(shape=(None,), dtype='int32')\n", - "\n", - "# Secuencia de longitud variable de enteros\n", - "text_input_b = keras.Input(shape=(None,), dtype='int32')\n", - "\n", - "# Reutilizamos la misma capa para codificar ambas entradas\n", - "encoded_input_a = shared_embedding(text_input_a)\n", - "encoded_input_b = shared_embedding(text_input_b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xNEKvfUpr-Kf" - }, - "source": [ - "## Extracción y reutilización de nodos en el gráfico de capas" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JHVGI6bEr-ze" - }, - "source": [ - "Debido a que el gráfico de capas que está manipulando en la API funcional es una estructura de datos estática, se puede acceder e inspeccionarlo. Así es como podemos trazar modelos funcionales como imágenes, por ejemplo.\n", - "\n", - "Esto también significa que podemos acceder a las activaciones de capas intermedias (\"nodos\" en el gráfico) y reutilizarlas en otros lugares. ¡Esto es extremadamente útil para la extracción de características, por ejemplo!\n", - "\n", - "Veamos un ejemplo. Este es un modelo VGG19 con pesas pre-entrenadas en ImageNet:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c-gl3xHBH-oX", - "colab": {} - }, - "source": [ - "from tensorflow.keras.applications import VGG19\n", - "\n", - "vgg19 = VGG19()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AKefin_xIGBP" - }, - "source": [ - "Y estas son las activaciones intermedias del modelo, obtenidas al consultar la estructura de datos del gráfico:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1_Ap05fgIRgE", - "colab": {} - }, - "source": [ - "features_list = [layer.output for layer in vgg19.layers]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H1zx5qM7IYu4" - }, - "source": [ - "Podemos usar estas características para crear un nuevo modelo de extracción de características, que devuelve los valores de las activaciones de la capa intermedia, y podemos hacer todo esto en 3 líneas." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NrU82Pa8Igwo", - "colab": {} - }, - "source": [ - "feat_extraction_model = keras.Model(inputs=vgg19.input, outputs=features_list)\n", - "\n", - "img = np.random.random((1, 224, 224, 3)).astype('float32')\n", - "extracted_features = feat_extraction_model(img)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "G-e2-jNCLIqy" - }, - "source": [ - "Esto es útil cuando [implementa la transferencia de estilo neural] (https://medium.com/tensorflow/neural-style-transfer-creating-art-with-deep-learning-using-tf-keras-and-eager-execution- 7d541ac31398), entre otras cosas." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9M2Uvi3sBy0" - }, - "source": [ - "## Extendiendo la API escribiendo capas personalizadas\n", - "\n", - "tf.keras tiene una amplia gama de capas incorporadas. Aquí están algunos ejemplos:\n", - "\n", - "- Capas convolucionales: `Conv1D`,` Conv2D`, `Conv3D`,` Conv2DTranspose`, etc.\n", - "- Capas de agrupación: `MaxPooling1D`,` MaxPooling2D`, `MaxPooling3D`,` AveragePooling1D`, etc.\n", - "- Capas RNN: `GRU`,` LSTM`, `ConvLSTM2D`, etc.\n", - "- `BatchNormalization`,` Dropout`, `Embedded`, etc.\n", - "\n", - "Si no encuentras lo que necesitas, es fácil extender la API creando tus propias capas.\n", - "\n", - "Todas las capas subclasifican la clase `Layer` e implementan:\n", - "- Un método `call`, que especifica el cálculo realizado por la capa.\n", - "- Un método `build`, que crea los pesos de la capa (tenga en cuenta que esto es solo una convención de estilo; también puede crear pesos en` __init__`).\n", - "\n", - "Para obtener más información sobre cómo crear capas desde cero, consulta la guía [Guía para escribir capas y modelos desde cero](./custom_layers_and_models.ipynb).\n", - "\n", - "Aquí hay una implementación simple de una capa `Densa`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ztAmarbgNV6V", - "colab": {} - }, - "source": [ - "class CustomDense(layers.Layer):\n", - "\n", - " def __init__(self, units=32):\n", - " super(CustomDense, self).__init__()\n", - " self.units = units\n", - "\n", - " def build(self, input_shape):\n", - " self.w = self.add_weight(shape=(input_shape[-1], self.units),\n", - " initializer='random_normal',\n", - " trainable=True)\n", - " self.b = self.add_weight(shape=(self.units,),\n", - " initializer='random_normal',\n", - " trainable=True)\n", - "\n", - " def call(self, inputs):\n", - " return tf.matmul(inputs, self.w) + self.b\n", - "\n", - "inputs = keras.Input((4,))\n", - "outputs = CustomDense(10)(inputs)\n", - "\n", - "model = keras.Model(inputs, outputs)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NXxp_32bNWTy" - }, - "source": [ - "Si deseas que tu capa personalizada admita la serialización, también debes definir un método `get_config`,\n", - "que devuelve los argumentos del constructor de la instancia de capa:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K3OQ4XxzNfAZ", - "colab": {} - }, - "source": [ - "class CustomDense(layers.Layer):\n", - "\n", - " def __init__(self, units=32):\n", - " super(CustomDense, self).__init__()\n", - " self.units = units\n", - "\n", - " def build(self, input_shape):\n", - " self.w = self.add_weight(shape=(input_shape[-1], self.units),\n", - " initializer='random_normal',\n", - " trainable=True)\n", - " self.b = self.add_weight(shape=(self.units,),\n", - " initializer='random_normal',\n", - " trainable=True)\n", - "\n", - " def call(self, inputs):\n", - " return tf.matmul(inputs, self.w) + self.b\n", - "\n", - " def get_config(self):\n", - " return {'units': self.units}\n", - "\n", - "\n", - "inputs = keras.Input((4,))\n", - "outputs = CustomDense(10)(inputs)\n", - "\n", - "model = keras.Model(inputs, outputs)\n", - "config = model.get_config()\n", - "\n", - "new_model = keras.Model.from_config(\n", - " config, custom_objects={'CustomDense': CustomDense})" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kXg6hZN_NfN8" - }, - "source": [ - "Opcionalmente, también podría implementar el método de clase `from_config (cls, config)`, que se encarga de recrear una instancia de capa dado su diccionario de configuración. La implementación predeterminada de `from_config` es:\n", - "\n", - "```python\n", - "def from_config(cls, config):\n", - " return cls(**config)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ifOVqn84sCNU" - }, - "source": [ - "## Cuándo usar la API funcional\n", - "\n", - "¿Cómo decidir si usar la API funcional para crear un nuevo modelo o simplemente subclasificar la clase `Modelo` directamente?\n", - "\n", - "En general, la API funcional es de nivel superior, más fácil y segura de usar, y tiene una serie de características que los modelos de subclases no admiten.\n", - "\n", - "Sin embargo, la subclasificación de modelos le brinda una mayor flexibilidad al crear modelos que no se pueden expresar fácilmente como gráficos acíclicos dirigidos de capas (por ejemplo, no podría implementar un Tree-RNN con la API funcional, tendría que subclasificar `Model` directamente).\n", - "\n", - "\n", - "### Estas son las fortalezas de la API funcional:\n", - "\n", - "Las propiedades enumeradas a continuación también son ciertas para los modelos secuenciales (que también son estructuras de datos), pero no son ciertas para los modelos subclasificados (que son bytecode de Python, no estructuras de datos).\n", - "\n", - "\n", - "#### Es menos detallado.\n", - "\n", - "No ` super (MyClass, self) .__ init __ (...)`, no `def call (self, ...): `, etc.\n", - "\n", - "Comparar:\n", - "\n", - "```pitón\n", - "input = keras.Input (shape = (32,))\n", - "x = capas. Denso (64, activación = 'relu') (entradas)\n", - "salidas = capas. Denso (10) (x)\n", - "mlp = keras.Model (entradas, salidas)\n", - "```\n", - "\n", - "Con la versión subclaseada:\n", - "\n", - "```pitón\n", - "clase MLP (keras.Model):\n", - "\n", - "  def __init __ (self, ** kwargs):\n", - "    super (MLP, self) .__ init __ (** kwargs)\n", - "    self.dense_1 = capas.Dense (64, activación = 'relu')\n", - "    self.dense_2 = layers.Dense (10)\n", - "\n", - "  llamada def (auto, entradas):\n", - "    x = self.dense_1 (entradas)\n", - "    return self.dense_2 (x)\n", - "\n", - "# Instanciar el modelo.\n", - "mlp = MLP ()\n", - "# Necesario para crear el estado del modelo.\n", - "# El modelo no tiene un estado hasta que se llama al menos una vez.\n", - "_ = mlp (tf.zeros ((1, 32)))\n", - "```\n", - "\n", - "\n", - "#### Valida su modelo mientras lo está definiendo.\n", - "\n", - "En la API funcional, su especificación de entrada (forma y dtype) se crea de antemano (a través de `Input`), y cada vez que llama a una capa, la capa comprueba que la especificación que se le pasa coincide con sus supuestos, y generará un mensaje de error útil si no.\n", - "\n", - "Esto garantiza que se ejecutará cualquier modelo que pueda construir con la API funcional. Toda la depuración (que no sea la depuración relacionada con la convergencia) ocurrirá estáticamente durante la construcción del modelo, y no en el momento de la ejecución. Esto es similar a la comprobación de tipo en un compilador.\n", - "\n", - "\n", - "#### Tu modelo funcional es trazable e inspeccionable.\n", - "\n", - "Puedes trazar el modelo como un gráfico, y puedes acceder fácilmente a los nodos intermedios en este gráfico, por ejemplo, para extraer y reutilizar las activaciones de las capas intermedias, como vimos en un ejemplo anterior:\n", - "\n", - "```pitón\n", - "features_list = [layer.output para la capa en vgg19.layers]\n", - "feat_extraction_model = keras.Model (input = vgg19.input, salidas = features_list)\n", - "```\n", - "\n", - "\n", - "#### Su modelo funcional puede ser serializado o clonado.\n", - "\n", - "Debido a que un modelo funcional es una estructura de datos en lugar de un fragmento de código, es serializable de forma segura y se puede guardar como un único archivo que le permite recrear exactamente el mismo modelo sin tener acceso a ninguno de los códigos originales. Consulta nuestra [guía de guardado y serialización] (./save_and_serialize.ipynb) para obtener más detalles.\n", - "\n", - "\n", - "### Estas son las debilidades de la API funcional:\n", - "\n", - "\n", - "#### No admite arquitecturas dinámicas.\n", - "\n", - "La API funcional trata los modelos como DAG de capas. Esto es cierto para la mayoría de las arquitecturas de aprendizaje profundo, pero no para todas: por ejemplo, las redes recursivas o los RNN de árbol no siguen este supuesto y no se pueden implementar en la API funcional.\n", - "\n", - "\n", - "#### A veces, solo necesitas escribir todo desde cero.\n", - "\n", - "Al escribir actividades avanzadas, es posible que desee hacer cosas que están fuera del alcance de \"definir un DAG de capas\": por ejemplo, es posible que desee exponer múltiples métodos personalizados de entrenamiento e inferencia en su instancia de modelo. Esto requiere subclases.\n", - "\n", - "\n", - "---\n", - "\n", - "\n", - "Para profundizar más en las diferencias entre la API funcional y la subclasificación de modelos, puede leer [¿Qué son las API simbólicas e imperativas en TensorFlow 2.0?] (Https://medium.com/tensorflow/what-are-symbolic-and -imperative-apis-in-tensorflow-2-0-dfccecb01021)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ym1jrCqusGvj" - }, - "source": [ - "## Mezcla y combina diferentes estilos de API\n", - "\n", - "Es importante destacar que elegir entre la subclasificación de API funcional o modelo no es una decisión binaria que lo restringe a una categoría de modelos. Todos los modelos en la API tf.keras pueden interactuar con cada uno, ya sean modelos secuenciales, modelos funcionales o modelos / capas subclasificados escritos desde cero.\n", - "\n", - "Siempre puede usar un modelo funcional o modelo secuencial como parte de un modelo / capa subclasificado:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9zF5YTLy_vGZ", - "colab": {} - }, - "source": [ - "units = 32\n", - "timesteps = 10\n", - "input_dim = 5\n", - "\n", - "# Define a Functional model\n", - "inputs = keras.Input((None, units))\n", - "x = layers.GlobalAveragePooling1D()(inputs)\n", - "outputs = layers.Dense(1, activation='sigmoid')(x)\n", - "model = keras.Model(inputs, outputs)\n", - "\n", - "\n", - "class CustomRNN(layers.Layer):\n", - "\n", - " def __init__(self):\n", - " super(CustomRNN, self).__init__()\n", - " self.units = units\n", - " self.projection_1 = layers.Dense(units=units, activation='tanh')\n", - " self.projection_2 = layers.Dense(units=units, activation='tanh')\n", - " # Our previously-defined Functional model\n", - " self.classifier = model\n", - "\n", - " def call(self, inputs):\n", - " outputs = []\n", - " state = tf.zeros(shape=(inputs.shape[0], self.units))\n", - " for t in range(inputs.shape[1]):\n", - " x = inputs[:, t, :]\n", - " h = self.projection_1(x)\n", - " y = h + self.projection_2(state)\n", - " state = y\n", - " outputs.append(y)\n", - " features = tf.stack(outputs, axis=1)\n", - " print(features.shape)\n", - " return self.classifier(features)\n", - "\n", - "rnn_model = CustomRNN()\n", - "_ = rnn_model(tf.zeros((1, timesteps, input_dim)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oxW1d0a8_ufg" - }, - "source": [ - "Inversamente, puede usar cualquier Capa o Modelo subclasificado en la API Funcional siempre que implemente un método `call` que siga uno de los siguientes patrones:\n", - "\n", - "- `call (self, input, ** kwargs)` donde `input` es un tensor o una estructura anidada de tensores (por ejemplo, una lista de tensores), y donde` ** kwargs` son argumentos no tensoriales (no input )\n", - "- `call (self, input, training = None, ** kwargs)` donde `training` es un valor booleano que indica si la capa debe comportarse en modo de entrenamiento y modo de inferencia.\n", - "- `call (self, input, mask = None, ** kwargs)` donde `mask` es un tensor de máscara booleano (útil para RNN, por ejemplo).\n", - "- `call (self, input, training = None, mask = None, ** kwargs)` - por supuesto, puede tener tanto un comportamiento específico de enmascaramiento como de entrenamiento al mismo tiempo.\n", - "\n", - "Además, si implementa el método `get_config` en su Capa o Modelo personalizado, los modelos funcionales que cree con él seguirán siendo serializables y clonables.\n", - "\n", - "Aquí hay un ejemplo rápido en el que usamos un RNN personalizado escrito desde cero en un modelo funcional:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TmTEZ6F3ArJR", - "colab": {} - }, - "source": [ - "units = 32\n", - "timesteps = 10\n", - "input_dim = 5\n", - "batch_size = 16\n", - "\n", - "\n", - "class CustomRNN(layers.Layer):\n", - "\n", - " def __init__(self):\n", - " super(CustomRNN, self).__init__()\n", - " self.units = units\n", - " self.projection_1 = layers.Dense(units=units, activation='tanh')\n", - " self.projection_2 = layers.Dense(units=units, activation='tanh')\n", - " self.classifier = layers.Dense(1, activation='sigmoid')\n", - "\n", - " def call(self, inputs):\n", - " outputs = []\n", - " state = tf.zeros(shape=(inputs.shape[0], self.units))\n", - " for t in range(inputs.shape[1]):\n", - " x = inputs[:, t, :]\n", - " h = self.projection_1(x)\n", - " y = h + self.projection_2(state)\n", - " state = y\n", - " outputs.append(y)\n", - " features = tf.stack(outputs, axis=1)\n", - " return self.classifier(features)\n", - "\n", - "# Tenga en cuenta que especificamos un tamaño de lote estático para las entradas con `batch_shape`\n", - "# arg, porque el cálculo interno de `CustomRNN` requiere un tamaño de lote estático\n", - "# (cuando creamos el tensor de ceros `estado`).\n", - "inputs = keras.Input(batch_shape=(batch_size, timesteps, input_dim))\n", - "x = layers.Conv1D(32, 3)(inputs)\n", - "outputs = CustomRNN()(x)\n", - "\n", - "model = keras.Model(inputs, outputs)\n", - "\n", - "rnn_model = CustomRNN()\n", - "_ = rnn_model(tf.zeros((1, 10, 5)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6VxcYb4qArlb" - }, - "source": [ - "¡Esto concluye nuestra guía sobre la API funcional \"Keras\"!\n", - "\n", - "Ahora tienes a tu alcance un poderoso conjunto de herramientas para construir modelos de aprendizaje profundo." - ] - } - ] -} \ No newline at end of file diff --git a/site/es/guide/keras/index.md b/site/es/guide/keras/index.md deleted file mode 100644 index 1a407b9ada5..00000000000 --- a/site/es/guide/keras/index.md +++ /dev/null @@ -1,21 +0,0 @@ -# Keras - -`tf.keras` es la API de alto nivel de TensorFlow para construir y entrenar modelos de aprendizaje profundo. Se utiliza para la creacion rapida de prototipos, la investigacion de vanguardia (estado-del-arte) y en produccion, con tres ventajas clave: - -- *Amigable al usuario*
    Keras tiene una interfaz simple y consistente optimizada para casos de uso comun. Proporciona informacion clara y procesable sobre los errores del usuario. -- *Modular y configurable*
    Los modelos en Keras se fabrican conectando bloques de construccion configurables entre si, con pocas restricciones. -- *Facil de extender*
    Escribir bloques de construccion personalizados para expresar nuevas ideas para la investigacion. Crea nuevas capas, metricas, funciones de perdida y desarrolla modelos de estado-del-arte. - -La guia [Keras: Una visión aápida](./overview.ipynb) te ayudara a empezar. - -Para una introduccion amigable a principiantes sobre aprendizaje maquina con `tf.keras`, ve [este conjunto de tutoriales para principiantes] (https://www.tensorflow.org/tutorials/keras). - -Para profundizar mas en la API, consulta el siguiente conjunto de guías que cubren lo siguiente que necesitas saber como super usuario de TensorFlow Keras: - -- [Guía de la API funcional de Keras](./functional.ipynb) -- [Guia de capacitación y evaluacion](./train_and_evaluate.ipynb) -- [Guia para escribir capas y modelos desde cero con subclases](./custom_layers_and_models.ipynb) -- [Guia de la API de red neuronal recurrente](./rnn.ipynb) -- [Guía para enmascarar y acolchar (padding)](./masking_and_padding.ipynb) -- [Guia para guardar y serializar modelos](./save_and_serialize.ipynb) -- [Guia para escribir llamadas de retorno pequick overviewrsonalizadas](./custom_callback.ipynb) diff --git a/site/es/guide/keras/save_and_serialize.ipynb b/site/es/guide/keras/save_and_serialize.ipynb deleted file mode 100644 index e6f47ff2119..00000000000 --- a/site/es/guide/keras/save_and_serialize.ipynb +++ /dev/null @@ -1,772 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "save_and_serialize.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mHDxn9VHjxKn" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "3x19oys5j89H", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hFDUpbtvv_3u" - }, - "source": [ - "# Guardando y Serializando Modelos con TensorFlow Keras" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "V94_3U2k9rWV" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Ver en TensorFlow.org\n", - " \n", - " Ejecutar en Google Colab\n", - " \n", - " Ver fuente en GitHub\n", - " \n", - " Descargar notas\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Px0f1zDWUik3", - "colab_type": "text" - }, - "source": [ - "Note: Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad\n", - "son basados en el \"mejor esfuerzo\", no hay ninguna garantia que esta sea un reflejo preciso y actual \n", - "de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en).\n", - "Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un \"Pull request\"\n", - "al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs).\n", - "Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad\n", - "por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZwiVWAQc9tk7" - }, - "source": [ - "La primera parte de esta guia cubre el guardado y serialización para modelos secuenciales y modelos creados con la API funcional y para modelos secuenciales. Las APIs de guardado y serialización son exactamente las mismas para ambos tipos de modelos.\n", - "\n", - "El guardado para las subclases personalizadas de Modelo se explica en la sección \"Guardar modelos de subclases\". Las APIs en este caso son ligeramente diferentes a las de los modelos secuenciales o funcionales." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uqSgPMHguAAs" - }, - "source": [ - "## Configurar" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bx5w4U5muDAo", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version solo existe en Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "tf.keras.backend.clear_session() # Para restablecer fácilmente el estado del portátil." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wwCxkE6RyyPy" - }, - "source": [ - "## Parte I: Guardar modelos secuenciales o modelos funcionales\n", - "\n", - "Consideremos el siguiente modelo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ILmySACTvSA9", - "colab": {} - }, - "source": [ - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "inputs = keras.Input(shape=(784,), name='digits')\n", - "x = layers.Dense(64, activation='relu', name='dense_1')(inputs)\n", - "x = layers.Dense(64, activation='relu', name='dense_2')(x)\n", - "outputs = layers.Dense(10, activation='softmax', name='predictions')(x)\n", - "\n", - "model = keras.Model(inputs=inputs, outputs=outputs, name='3_layer_mlp')\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xPRqbd0yw8hY" - }, - "source": [ - "Opcionalmente, entrenaremos este modelo, solo para que tenga valores de peso para guardarlos, así como un estado optimizador. Por supuesto, tambien puede guardar modelos que nunca ha entrenado, pero obviamente eso es menos interesante." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gCygTeGQw74g", - "colab": {} - }, - "source": [ - "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n", - "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n", - "x_test = x_test.reshape(10000, 784).astype('float32') / 255\n", - "\n", - "model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=keras.optimizers.RMSprop())\n", - "history = model.fit(x_train, y_train,\n", - " batch_size=64,\n", - " epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "htnmbhz-iOwh", - "colab": {} - }, - "source": [ - "# Guardar predicciones para futuras verificaciones\n", - "predictions = model.predict(x_test)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "opP1KROHwWwd" - }, - "source": [ - "\n", - "### Modelo-Completo Guardando\n", - "\n", - "Puede guardar un modelo creado con la API funcional en un solo archivo. Posteriormente, puede volver a crear el mismo modelo a partir de este archivo, incluso si ya no tiene acceso al codigo que creo el modelo.\n", - "\n", - "Este archivo incluye:\n", - "\n", - "- Los modelos de arquitectura\n", - "- Los valores de peso del modelo (que se aprendieron durante el entrenamiento)\n", - "- La configuración de entrenamiento del modelo (lo que pasó a 'compilar'), si corresponde\n", - "- El optimizador y su estado, si corresponde (esto le permite reiniciar el entrenamiento donde lo dejó)" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HqHvq6Igw3wx", - "colab": {} - }, - "source": [ - "# Guardar el Modelo\n", - "model.save('path_to_my_model.h5')\n", - "\n", - "# Recrea exactamente el mismo modelo solo desde el archivo\n", - "new_model = keras.models.load_model('path_to_my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mmIcF6UOItJE", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "# Verifique que el estado esté preservado\n", - "new_predictions = new_model.predict(x_test)\n", - "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n", - "\n", - "# Tenga en cuenta que el estado del optimizador también se conserva:\n", - "# puede reanudar el entrenamiento donde lo dejó." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-WEPW3n8ICyz" - }, - "source": [ - "### Exportar a 'SavedModel'\n", - "\n", - "Tambien puede exportar un modelo completo al formato 'SavedModel' de TensorFlow. 'SavedModel' es un formato de serialización independiente para objetos de Tensorflow, compatible con el servicio de TensorFlow y las implementaciones de TensorFlow que no sean Python.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cKASRTKCU5nv", - "colab": {} - }, - "source": [ - "# Exportar el modelo a 'SavedModel'\n", - "keras.experimental.export_saved_model(model, 'path_to_saved_model')\n", - "\n", - "# Recrea exactamente el mismo modelo\n", - "new_model = keras.experimental.load_from_saved_model('path_to_saved_model')\n", - "\n", - "# Verifique que el estado esté guardado\n", - "new_predictions = new_model.predict(x_test)\n", - "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n", - "\n", - "# Tenga en cuenta que el estado del optimizador también se conserva:\n", - "# puede reanudar el entrenamiento donde lo dejó." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4AWgwkKWIhfj" - }, - "source": [ - "Los archivos 'SavedModel' que se crearon contienen:\n", - "\n", - "- Un punto de control TensorFlow que contiene los pesos del modelo.\n", - "- Un prototipo 'SavedModel' que contiene el grafico subyacente de Tensorflow. Separar los graficos que se guardan para prediccion (servicio), capacitacion y evaluacion. Si el modelo no se compilo antes, solo el grafico de inferencia se exporta\n", - "- La configuracion de arquitectura del modelo, si esta disponible." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GkY8XP_XxgMI" - }, - "source": [ - "### Solo arquitectura de guardado\n", - "\n", - "A veces, solo esta interesado en la arquitectura del modelo y no necesita guardar los valores de peso o el optimizador. En este caso, puede recuperar la \"configuracion\" del modelo mediante el metodo get_config (). La configuracion es un dict de Python que le permite recrear el mismo modelo, inicializado desde cero, sin ninguna de la información aprendida previamente durante el entrenamiento." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yQGGvo2Fw4o-", - "colab": {} - }, - "source": [ - "config = model.get_config()\n", - "reinitialized_model = keras.Model.from_config(config)\n", - "\n", - "# ¡Tenga en cuenta que el estado del modelo no se conserva! Solo guardamos la arquitectura.\n", - "new_predictions = reinitialized_model.predict(x_test)\n", - "assert abs(np.sum(predictions - new_predictions)) > 0." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WsNBBvDgxsTS" - }, - "source": [ - "Alternativamente, puede usar 'to_json()' de 'from_json ()', que usa una cadena JSON para almacenar la configuracion en lugar de un 'dict' de Python. Esto es util para guardar la configuracion en el disco." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5a0z7_6XxqWV", - "colab": {} - }, - "source": [ - "json_config = model.to_json()\n", - "reinitialized_model = keras.models.model_from_json(json_config)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vq0NONs-2QGL", - "colab_type": "text" - }, - "source": [ - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SGC7R6IIxy0o" - }, - "source": [ - "### Guardando solo con pesos\n", - "\n", - "A veces, solo le interesa el estado del modelo, sus valores de peso, y no la arquitectura. En este caso, puede recuperar los valores de pesos como una lista de matrices Numpy a traves de 'get_weights()', y establecer el estado del modelo a través de 'set_weights':" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B8tHwEvkxw5E", - "colab": {} - }, - "source": [ - "weights = model.get_weights() # Recupera el estado del modelo.\n", - "model.set_weights(weights) # Establece el estado del modelo." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ydwtw-u2x7xC" - }, - "source": [ - "Puede combinar get_config()/from_config() y get_weights()/set_weights() para recrear su modelo en el mismo estado. Sin embargo, a diferencia de model.save(), esto no incluira la configuracion de entrenamiento y el optimizado tendría que volver a llamar a compile() antes de usar el modelo para el entrenamiento." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LWVtuxtrx5lb", - "colab": {} - }, - "source": [ - "config = model.get_config()\n", - "weights = model.get_weights()\n", - "\n", - "new_model = keras.Model.from_config(config)\n", - "new_model.set_weights(weights)\n", - "\n", - "# Verifique que el estado esté preservado\n", - "new_predictions = new_model.predict(x_test)\n", - "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n", - "\n", - "# Tenga en cuenta que el optimizador no se conserva\n", - "# entonces el modelo debe compilarse nuevamente antes de entrenar\n", - "# (y el optimizador comenzará desde un estado en blanco)." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "prk0GzwCyIYy" - }, - "source": [ - "La alternativa de guardar en disco a 'get_weights()' y 'set_weights(weights)' es 'save_weights(fpath)' y 'load_weights(fpath)'.\n", - "\n", - "Aquí hay un ejemplo que guarda en el disco:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2irLnOUbyHlI", - "colab": {} - }, - "source": [ - "# Guardar configuración JSON en el disco\n", - "json_config = model.to_json()\n", - "with open('model_config.json', 'w') as json_file:\n", - " json_file.write(json_config)\n", - "# Guardar pesos en el disco\n", - "model.save_weights('path_to_my_weights.h5')\n", - "\n", - "# Recargue el modelo de los 2 archivos que guardamos\n", - "with open('model_config.json') as json_file:\n", - " json_config = json_file.read()\n", - "new_model = keras.models.model_from_json(json_config)\n", - "new_model.load_weights('path_to_my_weights.h5')\n", - "\n", - "# Verifique que el estado esté preservado\n", - "new_predictions = new_model.predict(x_test)\n", - "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n", - "\n", - "# Tenga en cuenta que el optimizador no se conservo." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KBxcFAPHyYi5" - }, - "source": [ - "Pero recuerde que la forma más simple y recomendada es solo esto:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DE4b3ndNyQh3", - "colab": {} - }, - "source": [ - "model.save('path_to_my_model.h5')\n", - "del model\n", - "model = keras.models.load_model('path_to_my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yKikmbdC3O_i" - }, - "source": [ - "### Weights-only Guardando en formato 'SavedModel'\n", - "\n", - "Tenga en cuenta que 'save_weights' puede crear archivos en el formato Keras HDF5 o en el formato TensorFlow 'SavedModel'. El formato se infiere de la extension de archivo que proporciona: si es \".h5\" o \".keras\", el marco utiliza el formato Keras HDF5. Cualquier otra cosa por defecto es 'SavedModel'." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0pYKb6LV3h2l", - "colab": {} - }, - "source": [ - "model.save_weights('path_to_my_tf_savedmodel')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZFwKv6JC3kyu" - }, - "source": [ - "Para una total explicidad, el formato se puede pasar explicitamente a traves del argumento 'save_format', que puede tomar el valor \"tf\" o \"h5\":" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oN9vOaWU34lA", - "colab": {} - }, - "source": [ - "model.save_weights('path_to_my_tf_savedmodel', save_format='tf')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xXgtNRCSyuIW" - }, - "source": [ - "## Guardar modelos subclasificados" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mJqOn0snzCRy" - }, - "source": [ - "Los modelos secuenciales y los modelos funcionales son estructuras de datos que representan un DAG de capas. Como tal, se pueden serializar y deserializar de forma segura.\n", - "\n", - "Un modelo subclasificado difiere en que no es una estructura de datos, es una pieza de código. La arquitectura del modelo se define a través del cuerpo del método de llamada. Esto significa que la arquitectura del modelo no se puede serializar de forma segura. Para cargar un modelo, deberá tener acceso al código que lo creó (el código de la subclase de modelo). Alternativamente, podría estar serializando este código como bytecode (por ejemplo, mediante pickling), pero eso no es seguro y, en general, no es portátil.\n", - "\n", - "Para obtener más información sobre estas diferencias, vea el artículo [\"¿Qué son las API simbólicas e imperativas en TensorFlow 2.0?\"](https://medium.com/tensorflow/what-are-symbolic-and-imperative-apis-in-tensorflow-2-0-dfccecb01021)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Pkwyu5dVz12P" - }, - "source": [ - "\n", - "Consideremos el siguiente modelo subclasificado, que sigue la misma estructura que el modelo de la primera seccion:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Onp-8rGyeQG", - "colab": {} - }, - "source": [ - "class ThreeLayerMLP(keras.Model):\n", - "\n", - " def __init__(self, name=None):\n", - " super(ThreeLayerMLP, self).__init__(name=name)\n", - " self.dense_1 = layers.Dense(64, activation='relu', name='dense_1')\n", - " self.dense_2 = layers.Dense(64, activation='relu', name='dense_2')\n", - " self.pred_layer = layers.Dense(10, activation='softmax', name='predictions')\n", - "\n", - " def call(self, inputs):\n", - " x = self.dense_1(inputs)\n", - " x = self.dense_2(x)\n", - " return self.pred_layer(x)\n", - "\n", - "def get_model():\n", - " return ThreeLayerMLP(name='3_layer_mlp')\n", - "\n", - "model = get_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wwT_YoKA0yQW" - }, - "source": [ - "En primer lugar, *un modelo subclasificado que nunca se ha utilizado no se puede guardar.*.\n", - "\n", - "Esto se debe a que es necesario invocar un modelo subclasificado en algunos datos para crear sus pesos.\n", - "\n", - "Hasta que se haya llamado al modelo, no conoce la forma y el tipo de datos de entrada que debería ser\n", - "esperando, y por lo tanto no puede crear sus variables de peso. Puede recordar que en el modelo funcional de la primera sección, la forma y el tipo de las entradas se especificaron de antemano (a través de 'keras.Input (...)'), por eso los modelos funcionales tienen un estado tan pronto como Estás instanciado.\n", - "\n", - "Vamos a entrenar el modelo para darle un estado:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xqP4kIFN0fTZ", - "colab": {} - }, - "source": [ - "(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n", - "x_train = x_train.reshape(60000, 784).astype('float32') / 255\n", - "x_test = x_test.reshape(10000, 784).astype('float32') / 255\n", - "\n", - "model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=keras.optimizers.RMSprop())\n", - "history = model.fit(x_train, y_train,\n", - " batch_size=64,\n", - " epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rvGCpyX72HOC" - }, - "source": [ - "La forma recomendada de guardar un modelo subclasificado es usar 'save_weights' para crear un punto de control TensorFlow 'SavedModel', que contendra el valor de todas las variables asociadas con el modelo:\n", - "- Los pesos de las capas\n", - "- El estado del optimizador\n", - "- Cualquier variable asociada con métricas de modelo con estado (si las hay)\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gMg87Tz01cxQ", - "colab": {} - }, - "source": [ - "model.save_weights('path_to_my_weights', save_format='tf')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KOKNBojtsl0F", - "colab": {} - }, - "source": [ - "# Guardar predicciones para futuras verificaciones\n", - "predictions = model.predict(x_test)\n", - "# También guarde la pérdida en el primer lote\n", - "# para luego afirmar que el estado del optimizador fue preservado\n", - "first_batch_loss = model.train_on_batch(x_train[:64], y_train[:64])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "h2PM_PL1SzPo" - }, - "source": [ - "Para restaurar su modelo, necesitará acceso al codigo que creó el objeto modelo.\n", - "\n", - "Tenga en cuenta que para restaurar el estado del optimizador y el estado de cualquier métrica con estado, debe compila el modelo (con los mismos argumentos que antes) y llámalo con algunos datos antes de llamar a 'load_weights':" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OOSGiSkHTERy", - "colab": {} - }, - "source": [ - "# Recrea el modelo\n", - "new_model = get_model()\n", - "new_model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=keras.optimizers.RMSprop())\n", - "\n", - "# Esto inicializa las variables utilizadas por los optimizadores,\n", - "# así como cualquier variable métrica con estado\n", - "new_model.train_on_batch(x_train[:1], y_train[:1])\n", - "\n", - "# Cargue el estado del modelo anterior.\n", - "new_model.load_weights('path_to_my_weights')\n", - "\n", - "# Compruebe que se ha conservado el estado del modelo.\n", - "new_predictions = new_model.predict(x_test)\n", - "np.testing.assert_allclose(predictions, new_predictions, rtol=1e-6, atol=1e-6)\n", - "\n", - "# El estado del optimizador también se conserva,\n", - "# para que pueda reanudar el entrenamiento donde lo dejó\n", - "new_first_batch_loss = new_model.train_on_batch(x_train[:64], y_train[:64])\n", - "assert first_batch_loss == new_first_batch_loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2_XaE5Oqv7Rh" - }, - "source": [ - "You've reached the end of this guide! This covers everything you need to know about saving and serializing models with tf.keras in TensorFlow 2.0." - ] - } - ] -} \ No newline at end of file diff --git a/site/es/guide/variable.md b/site/es/guide/variable.md deleted file mode 100644 index f0493067a1a..00000000000 --- a/site/es/guide/variable.md +++ /dev/null @@ -1,113 +0,0 @@ -# Variables - -Una **variable** de TensorFlow es la mejor manera de representar un estado compartido y persistente -manipulado por su programa. - -Variables son manipuladas via la clase `tf.Variable`. Una `tf.Variable` -representa un el valor de un tensor que puede ser cambiado corriendo operaciones -en el. Operaciones especificas te permiten leer y modificar los valores del -tensor. Librerias de alto nivel como `tf.keras` usan `tf.Variable` para guardar -parametros de modelo. Esta guia cubre como crear, actualizar y administrar -`tf.Variable`s en TensorFlow. - -## Creando una Variable - -Para crear una variable, simplemente provee el valor inicial - -``` python -my_variable = tf.Variable(tf.zeros([1., 2., 3.])) -``` - -Esto crea una variable la cual es un tensor 3D con forma `[1, 2, -3]` llenado con ceros. Esta variable, por default, tendra el `dtype` -`tf.float32`. El dtype es, si no se especifica, inferido del valor -inicial. - -Si existe un `tf.device` scope activo, la variable sera colocada en ese -dispositivo; de otra manera la variable sera colocada en el dispositivo "mas rapido" -con su dtype (esto significa que la mayoria de las variables son colocadas automaticamente -en una GPU si esta disponible). Por ejemplo, el siguiente codigo crea una variable -llamada `v` y la coloca en el segundo GPU: - -``` python -with tf.device("/device:GPU:1"): - v = tf.Variable(tf.zeros([10, 10])) -``` - -Idealmente se deberia usar la API `tf.distribute`, ya que eso permite que -se escriba el codigo una sola vez y funcione en diferentes configuraciones. - -## Usar variables - -Para usar el valor de una `tf.Variable` en una grafica TensorFlow, simplemente se -trata como un `tf.Tensor` normal: - -``` python -v = tf.Variable(0.0) -w = v + 1 # w es un tf.Tensor que se computa basado en el valor de v. - # Cualuier momento en que una variable es usada en una expresion - # Es convertida automaticamente en un tf.Tensor representando su - # valor -``` - -Para asignar un valor a una variable, se usan los metodos `assign`, `assign_add`, y -friends en la clase `tf.Variable`. Por ejemplo, asi es como se pueden llamar estos -metodos: - -``` python -v = tf.Variable(0.0) -v.assign_add(1) -``` - -La mayoria de los optimizadores de TensorFlow tienen operaciones especializadas que -eficientemente actualizan los valores de las variables de acuerdo con algunos -algoritmos de descenso de gradiente. Revisa `tf.keras.optimizers.Optimizer` para -una explicación de como usar optimizadores. - -Tambien se puede leer explicitamente el valor actual de una variable, usando -`read_value`: - -```python -v = tf.Variable(0.0) -v.assign_add(1) -v.read_value() # 1.0 -``` - -Cuando la ultima referencia a una `tf.Variable` se sale del scope la memoria utilizada -es liberada. - -### Seguimiento de variables - -Una variable en TensorFlow es un objeto de Python. Conforme creas capas, modelos, -optimizadores, y otras herramientas relacionadas, probablemente sea necesario -obtener una lista de todas las variables en un (digamos) modelo. - -Un caso de uso comun es [implementando `Layer` subclases]( -https://www.tensorflow.org/guide/keras/custom_layers_and_models#the_layer_class). -La clase `Layer` rastrea recursivamente las variables establecidas como atributos de instancia: - -```python -class MyLayer(tf.keras.layers.Layer): - - def __init__(self): - super(MyLayer, self).__init__() - self.my_var = tf.Variable(1.0) - self.my_var_list = [tf.Variable(x) for x in range(10)] - -class MyOtherLayer(tf.keras.layers.Layer): - - def __init__(self): - super(MyOtherLayer, self).__init__() - self.sublayer = MyLayer() - self.my_other_var = tf.Variable(10.0) - -m = MyOtherLayer() -print(len(m.variables)) # 12 (11 de MyLayer, mas my_other_var) -``` - -Si no se esta desarrollando una nueva `Layer`, TensorFlow tambien posee una -case generica base `tf.Module` que _solo_ implementa rastreo de variables. -Instancias de `tf.Module` tienen las propiedades `variables` y -`trainable_variables` que regresan todas las (trainable) variables de ese modelo -potencialmente navegando a traves de otros modelos (similar al rastreo hecho por -la clase`Layer`). diff --git a/site/es/tutorials/README.md b/site/es/tutorials/README.md deleted file mode 100644 index d03d269a31c..00000000000 --- a/site/es/tutorials/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# TensorFlow 2.x docs - -See https://www.tensorflow.org/overview diff --git a/site/es/tutorials/keras/classification.ipynb b/site/es/tutorials/keras/classification.ipynb deleted file mode 100644 index b9e7250f8ae..00000000000 --- a/site/es/tutorials/keras/classification.ipynb +++ /dev/null @@ -1,1036 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YenH_9hJbFk1" - }, - "source": [ - "# Clasificacion Basica: Predecir una imagen de moda" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Uo47Ynr8gNAU", - "colab_type": "text" - }, - "source": [ - "Note: Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad\n", - "son basados en el \"mejor esfuerzo\", no hay ninguna garantia que esta sea un reflejo preciso y actual \n", - "de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en).\n", - "Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un \"Pull request\"\n", - "al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs).\n", - "Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad\n", - "por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "Esta Guia entrena un modelo de red neuronal para clasificar imagenes de ropa como, tennis y camisetas. No hay problema sino entiende todos los detalles; es un repaso rapido de un programa completo de Tensorflow con los detalles explicados a medida que avanza.\n", - "\n", - "Esta Guia usa [tf.keras](https://www.tensorflow.org/guide/keras), un API de alto nivel para construir y entrenar modelos en Tensorflow." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jL3OqFKZ9dFg", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow y tf.keras\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# Librerias de ayuda\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## Importar el set de datos de moda de MNIST" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "Esta guia usa el set de datos de [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist)\n", - "que contiene mas de 70,000 imagenes en 10 categorias. Las imagenes muestran articulos individuales de ropa a una resolucion baja (28 por 28 pixeles) como se ve aca:\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
     \n", - "
    \n", - "\n", - "Moda MNIST esta construida como un reemplazo para el set de datos clasico [MNIST](http://yann.lecun.com/exdb/mnist/) \n", - "casi siempre utilizado como el \"Hola Mundo\" de programas de aprendizaje automatico (ML) para computo de vision. El set de datos de MNIST contiene imagenes de digitos escrito a mano (0, 1, 2, etc.) en un formato identico al de los articulos de ropa que va a utilizar aca.\n", - "\n", - "Esta guia utiliza Moda MNIST para variedad y por que es un poco mas retador que la regular MNIST. Ambos set de datos son relativamente pequenos y son usados para verificar que el algoritmo funciona como debe.\n", - "\n", - "Aca, 60,000 imagenes son usadas para entrenar la red neuronal y 10,000 imagenes son usadas para evaluar que tan exacto aprendia la red a clasificar imagenes. Pueden acceder al set de moda de MNIST directamente desde TensorFlow. Para importar y cargar el set de datos de MNIST directamente de TensorFlow:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "[link text](https://)Al cargar el set de datos retorna cuatro arreglos en NumPy:\n", - "\n", - "* El arreglo `train_images` y `train_labels` son los arreglos que *training set*—el modelo de datos usa para aprender.\n", - "* el modelo es probado contra los arreglos *test set*, el `test_images`, y `test_labels`.\n", - "\n", - "Las imagenes son 28x28 arreglos de NumPy, con valores de pixel que varian de 0 a 255. Los *labels* son un arreglo de integros, que van del 0 al 9. Estos corresponden a la *class* de ropa que la imagen representa.\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    LabelClass
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "Cada imagen es mapeada a una unica etiqueta. Ya que los *Class names* no estan incluidoen el dataset, almacenelo aca para usarlos luego cuando se visualicen las imagenes:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## Explore el set de datos\n", - "\n", - "Explore el formato de el set de datos antes de entrenar el modelo. Lo siguiente muestra que hay 60,000 imagenes en el set de entrenamiento, con cada imagen representada por pixeles de 28x28:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "Asimismo, hay 60,000 etiquetas en el set de entrenamiento:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "Cada etiqueta es un integro entre 0 y 9:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "Hay 10,000 imagenes en el set de pruebas. Otra vez, cada imagen es representada como pixeles de 28x28:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "Y el set de pruebas contiene 10,000 etiquetas de imagen:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## Pre-procese el set de datos\n", - "\n", - "El set de datos debe ser pre-procesada antes de entrenar la red. Si usted inspecciona la primera imagen en el set de entrenamiento, va a encontrar que los valores de los pixeles estan entre 0 y 255:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "Escale estos valores en un rango de 0 a 1 antes de alimentarlos al modelo de la red neuronal. Para hacero, divida los valores por 255. Es importante que el *training set* y el *testing set* se pre-procesen de la misma forma:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "Para verificar que el set de datos esta en el formato adecuado y que estan listos para construir y entrenar la red, vamos a desplegar las primeras 25 imagenes de el *training set* y despleguemos el nombre de cada clase debajo de cada imagen." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## Construir el Modelo\n", - "\n", - "Construir la red neuronal requiere configurar las capas del modelo y luego compilar el modelo." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### Configurar las Capas\n", - "\n", - "Los bloques de construccion basicos de una red neuronal son las *capas* o *layers*. Las capas extraen representaciones de el set de datos que se les alimentan. Con suerte, estas representaciones son considerables para el problema que estamos solucionando.\n", - "\n", - "La mayoria de aprendizaje profundo consiste de unir capas sencillas. \n", - "La mayoria de las capas como `tf.keras.layers.Dense`, tienen parametros que son aprendidos durante el entrenamiento." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation='relu'),\n", - " keras.layers.Dense(10, activation='softmax')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "La primera capa de esta red, `tf.keras.layers.Flatten`, \n", - "transforma el formato de las imagenes de un arreglo bi-dimensional (de 28 por 28 pixeles) a un arreglo uni dimensional (de 28*28 pixeles = 784 pixeles). Observe esta capa como una capa no apilada de filas de pixeles en la misma imagen y alineandolo. Esta capa no tiene parametros que aprender; solo reformatea el set de datos.\n", - "\n", - "Despues de que los pixeles estan \"aplanados\", la secuencia consiste de dos capas`tf.keras.layers.Dense`. Estas estan densamente conectadas, o completamente conectadas. La primera capa `Dense` tiene 128 nodos (o neuronas). La segunda (y ultima) capa es una capa de 10 nodos *softmax* que devuelve un arreglo de 10 probabilidades que suman a 1. Cada nodo contiene una calificacion que indica la probabilidad que la actual imagen pertenece a una de las 10 clases.\n", - "\n", - "### Compile el modelo\n", - "\n", - "Antes de que el modelo este listo para entrenar , se necesitan algunas configuraciones mas. Estas son agregadas durante el paso de compilacion del modelo:\n", - "\n", - "* *Loss function* —Esto mide que tan exacto es el modelo durante el entrenamiento. Quiere minimizar esta funcion para dirigir el modelo en la direccion adecuada.\n", - "* *Optimizer* — Esto es como el modelo se actualiza basado en el set de datos que ve y la funcion de perdida.\n", - "* *Metrics* — Se usan para monitorear los pasos de entrenamiento y de pruebas.\n", - "El siguiente ejemplo usa *accuracy* (exactitud) , la fraccion de la imagenes que son correctamente clasificadas." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## Entrenar el Modelo\n", - "\n", - "Entrenar el modelo de red neuronal requiere de los siguientes pasos:\n", - "\n", - "1. Entregue los datos de entrenamiento al modelo. En este ejemplo , el set de datos de entrenamiento estan en los arreglos `train_images` y `train_labels`.\n", - "2. el modelo aprende a asociar imagenes y etiquetas.\n", - "3. Usted le pregunta al modelo que haga predicciones sobre un set de datos que se encuentran en el ejemplo,incluido en el arreglo `test_images`. Verifique que las predicciones sean iguales a las etiquetas de el arreglo`test_labels`.\n", - "\n", - "Para comenzar a entrenar, llame el metodo `model.fit`, es llamado asi por que *fit* (ajusta) el modelo a el set de datos de entrenamiento:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "A medida que el modelo entrena, la perdida y la exactitud son desplegadas. Este modelo alcanza una exactitud de 0.88 (o 88%) sobre el set de datos de entrenamiento." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## Evaluar Exactitud\n", - "\n", - "Siguente, compare como el rendimiento del modelo sobre el set de datos:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('\\nTest accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "Resulta que la exactitud sobre el set de datos es un poco menor que la exactitud sobre el set de entrenamiento. Esta diferencia entre el entrenamiento y el test se debe a *overfitting* (sobre ajuste). Sobre ajuste sucede cuando un modelo de aprendizaje de maquina (ML) tiene un rendimiento peor sobre un set de datos nuevo, que nunca antes ha visto comparado con el de entrenamiento." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## Hacer predicciones\n", - "\n", - "Con el modelo entrenado usted puede usarlo para hacer predicciones sobre imagenes." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "Aca, el modelo ha predecido la etiqueta para cada imagen en el set de datos de *test* (prueba). Miremos la primera prediccion:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "*una* prediccion es un arreglo de 10 numeros. Estos representan el nivel de \"confianza\" del modelo sobre las imagenes de cada uno de los 10 articulos de moda/ropa. Ustedes pueden revisar cual tiene el nivel mas alto de confianza:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "Entonces,el modelo tiene mayor confianza que esta imagen es un bota de tobillo \"ankle boot\" o `class_names[9]`. Examinando las etiquetas de *test* o de pruebas muestra que esta clasificaion es correcta:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "**Grafique** esto para poder ver todo el set de la prediccion de las 10 clases." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DvYmmrpIy6Y1", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array, true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array, true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks(range(10))\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d4Ov9OFDMmOD" - }, - "source": [ - "Miremos la imagen [0], sus predicciones y el arreglo de predicciones. Las etiquetas de prediccion correctas estan en azul y las incorrectas estan en rojo. El numero entrega el porcentaje (sobre 100) para la etiqueta predecida." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HV5jw-5HwSmO", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions[i], test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions[i], test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ko-uzOufSCSe", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions[i], test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions[i], test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "Vamos a graficar multiples imagenes con sus predicciones. Notese que el modelo puede estar equivocado aun cuando tiene mucha confianza." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hQlnbqaw2Qu_", - "colab": {} - }, - "source": [ - "# Plot the first X test images, their predicted labels, and the true labels.\n", - "# Color correct predictions in blue and incorrect predictions in red.\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions[i], test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions[i], test_labels)\n", - "plt.tight_layout()\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "Finalmente, usamos el modelo entrenado para hacer una prediccion sobre una unica imagen." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yRJ7JU7JCaXT", - "colab": {} - }, - "source": [ - "# Grab an image from the test dataset.\n", - "img = test_images[1]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "Los modelos de `tf.keras` son optimizados sobre *batch* o bloques, \n", - "o coleciones de ejemplos por vez.\n", - "De acuerdo a esto, aunque use una unica imagen toca agregarla a una lista:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# Add the image to a batch where it's the only member.\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "Ahora prediga la etiqueta correcta para esta imagen:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ai-cpLjO-3A", - "colab": {} - }, - "source": [ - "plot_value_array(1, predictions_single[0], test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "`model.predict` retorna una lista de listas para cada imagen dentro del *batch* o bloque de datos. Tome la prediccion para nuestra unica imagen dentro del *batch* o bloque:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "np.argmax(predictions_single[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "Y el modelo predice una etiqueta de 2." - ] - } - ] -} \ No newline at end of file diff --git a/site/es/tutorials/keras/regression.ipynb b/site/es/tutorials/keras/regression.ipynb deleted file mode 100644 index a273cb97229..00000000000 --- a/site/es/tutorials/keras/regression.ipynb +++ /dev/null @@ -1,878 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# Regresion Basica: Predecir eficiencia de gasolina" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Au0QDqNXgRb_", - "colab_type": "text" - }, - "source": [ - "Note: Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad\n", - "son basados en el \"mejor esfuerzo\", no hay ninguna garantia que esta sea un reflejo preciso y actual \n", - "de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en).\n", - "Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un \"Pull request\"\n", - "al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs).\n", - "Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad\n", - "por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "En un problema de *regresion*, buscamos predecir la salida de un valor continuo como la probabilidad de un precio. En contraste en un problema de *Clasificacion*, buscamos seleccionar una clase de una lista de clases (por ejemplo, en donde una imagen contenga una manzana o una naranja queremos reconocer cual es la fruta en la imagen).\n", - "\n", - "Este libro usa el set de datos clasico [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) y construye un modelo para predecir la eficiencia de vehiculos de 1970 y 1980. Para hacer esto proveeremos el modelo con una descripcion de muchos automoviles de ese periodo. Esta descripcion incluye atributos como: Cilindros, desplazamiento, potencia y peso.\n", - "\n", - "Este ejemplo usa el API `tf.keras` , revise [Esta Guia](https://www.tensorflow.org/guide/keras) para obtener mas detalles." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# Use seaborn for pairplot\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## El set de Datos de MPG\n", - "\n", - "el set de datos esta disponible de el siguiente repositorio [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/).\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### Obtenga la data\n", - "Primero descargue el set de datos." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "Importelo usando pandas." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### Limpie la data\n", - "\n", - "El set de datos contiene algunos valores desconocidos." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "**Para** Mantener este tutorial inicial sencillo eliminemos las siguientes filas." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "La columna de `\"Origin\"` realmente es categorica, no numerica. Entonces conviertala a un \"one-hot\":" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### Dividamos la data en entrenamiento y prueba\n", - "\n", - "Ahora divida el set de datos en un set de entrenamiento y otro de pruebas.\n", - "\n", - "Usaremos el set de pruebas en la evaluacion final de nuestro modelo." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### Inspeccione la data\n", - "\n", - "Revise rapidamente la distribucion conjunta de un par de columnas de el set de entrenamiento." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "Tambien revise las estadisticas generales:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### Separe las caracteristicas de las etiquetas.\n", - "\n", - "Separe el valor objetivo, o la \"etiqueta\" \n", - "de las caracteristicas. Esta etiqueta es el valor que entrenara el modelo para predecir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### Normalice la data\n", - "\n", - "Revise otra vez el bloque de `train_stats` que se presento antes y note la diferencia de rangos de cada caracteristica." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "Es una buena práctica normalizar funciones que utilizan diferentes escalas y rangos. Aunque el modelo * podría * converger sin normalización de características, dificulta el entrenamiento y hace que el modelo resultante dependa de la elección de las unidades utilizadas en la entrada.\n", - "\n", - "Nota: Aunque generamos intencionalmente estas estadísticas solo del conjunto de datos de entrenamiento, estas estadísticas también se utilizarán para normalizar el conjunto de datos de prueba. Necesitamos hacer eso para proyectar el conjunto de datos de prueba en la misma distribución en la que el modelo ha sido entrenado." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "Estos datos normalizados es lo que usaremos para entrenar el modelo.\n", - "\n", - "Precaución: las estadísticas utilizadas para normalizar las entradas aquí (media y desviación estándar) deben aplicarse a cualquier otro dato que se alimente al modelo, junto con la codificación de un punto que hicimos anteriormente. Eso incluye el conjunto de pruebas, así como los datos en vivo cuando el modelo se usa en producción." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## El modelo" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### Construye el modelo\n", - "\n", - "Construyamos nuestro modelo. Aquí, utilizaremos un modelo `secuencial` con dos capas ocultas densamente conectadas y una capa de salida que devuelve un único valor continuo. Los pasos de construcción del modelo se envuelven en una función, `build_model`, ya que crearemos un segundo modelo, más adelante." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation='relu', input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation='relu'),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mse',\n", - " optimizer=optimizer,\n", - " metrics=['mae', 'mse'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### Inspeccione el modelo\n", - "\n", - "Use el método `.summary` para imprimir una descripción simple del modelo" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "\n", - "Ahora pruebe el modelo. Tome un lote de ejemplos `10` de los datos de entrenamiento y llame a` model.predict` en él." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "Parece estar funcionando, y produce un resultado de la forma y tipo esperados." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### Entrenar a la modelo\n", - "\n", - "Entrene el modelo durante 1000 épocas y registre la precisión de entrenamiento y validación en el objeto `history`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# Display training progress by printing a single dot for each completed epoch\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "Visualice el progreso de entrenamiento del modelo usando las estadísticas almacenadas en el objeto `history`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mae'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mae'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mse'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mse'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "Este gráfico muestra poca mejora, o incluso degradación en el error de validación después de aproximadamente 100 épocas. Actualicemos la llamada `model.fit` para detener automáticamente el entrenamiento cuando el puntaje de validación no mejore. Utilizaremos una * devolución de llamada de EarlyStopping * que pruebe una condición de entrenamiento para cada época. Si transcurre una cantidad determinada de épocas sin mostrar mejoría, entonces detiene automáticamente el entrenamiento.\n", - "\n", - "Puedes obtener más información sobre esta devolución de llamada [Aca](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# The patience parameter is the amount of epochs to check for improvement\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "El gráfico muestra que en el conjunto de validación, el error promedio generalmente es de alrededor de +/- 2 MPG. ¿Es esto bueno? Le dejaremos esa decisión a usted.\n", - "\n", - "Veamos qué tan bien generaliza el modelo al usar el conjunto ** test **, que no usamos al entrenar el modelo. Esto nos dice qué tan bien podemos esperar que el modelo prediga cuándo lo usamos en el mundo real." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "### Haga Predicciones\n", - "\n", - "Finalmente, prediga los valores de MPG utilizando datos en el conjunto de pruebas:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19wyogbOSU5t" - }, - "source": [ - "Parece que nuestro modelo predice razonablemente bien. Echemos un vistazo a la distribución de errores." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m0CB5tBjSU5w" - }, - "source": [ - "No es del todo gaussiano, pero podríamos esperar eso porque el número de muestras es muy pequeño." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## Conclusion\n", - "\n", - "Este cuaderno introdujo algunas técnicas para manejar un problema de regresión.\n", - "\n", - "* El error cuadrático medio (MSE) es una función de pérdida común utilizada para problemas de regresión (se utilizan diferentes funciones de pérdida para problemas de clasificación).\n", - "* Del mismo modo, las métricas de evaluación utilizadas para la regresión difieren de la clasificación. Una métrica de regresión común es el error absoluto medio (MAE).\n", - "* Cuando las características de datos de entrada numéricos tienen valores con diferentes rangos, cada característica debe escalarse independientemente al mismo rango.\n", - "* Si no hay muchos datos de entrenamiento, una técnica es preferir una red pequeña con pocas capas ocultas para evitar el sobreajuste.\n", - "* La detención temprana es una técnica útil para evitar el sobreajuste." - ] - } - ] -} \ No newline at end of file diff --git a/site/es/tutorials/quickstart/advanced.ipynb b/site/es/tutorials/quickstart/advanced.ipynb deleted file mode 100644 index 21fc1c349bc..00000000000 --- a/site/es/tutorials/quickstart/advanced.ipynb +++ /dev/null @@ -1,437 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "advanced.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# Guia inicial de TensorFlow 2.0 para expertos" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Ver en TensorFlow.org\n", - " \n", - " Ejecutar en Google Colab\n", - " \n", - " Ver codigo en GitHub\n", - " \n", - " Descargar notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CJEtYUgQnivP", - "colab_type": "text" - }, - "source": [ - "Note: Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad\n", - "son basados en el \"mejor esfuerzo\", no hay ninguna garantia que esta sea un reflejo preciso y actual \n", - "de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en).\n", - "Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un \"Pull request\"\n", - "al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs).\n", - "Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad\n", - "por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Este es un notebook de Google Colaboratory. Los programas de Python se executan directamente en tu navegador —una gran manera de aprender y utilizar TensorFlow. Para poder seguir este tutorial, ejecuta este notebook en Google Colab presionando el boton en la parte superior de esta pagina.\n", - "\n", - "En Colab, selecciona \"connect to a Python runtime\": En la parte superior derecha de la barra de menus selecciona: CONNECT.\n", - "Para ejecutar todas las celdas de este notebook: Selecciona Runtime > Run all." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eOsVdx6GGHmU" - }, - "source": [ - "Descarga e installa el paquete TensorFlow 2.0 version. \n", - "\n", - "Importa TensorFlow en tu programa:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ioLbtB3uGKPX", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version solo existe in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QS7DDTiZGRTo" - }, - "source": [ - "Import TensorFlow into your program:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Carga y prepara el conjunto de datos [MNIST](http://yann.lecun.com/exdb/mnist/)" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JqFRS6K07jJs", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# Agrega una dimension de canales\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "Utiliza `tf.data` to separar por lotes y mezclar el conjunto de datos:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Iu_quO024c2", - "colab": {} - }, - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Construye el modelo `tf.keras` utilizando la API de Keras [model subclassing API](https://www.tensorflow.org/guide/keras#model_subclassing):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "# Crea una instancia del modelo\n", - "model = MyModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "Escoge un optimizador y una funcion de perdida para el entrenamiento de tu modelo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u48C9WQ774n4", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "Escoge metricas para medir la perdida y exactitud del modelo.\n", - "Estas metricas acumulan los valores cada epoch y despues imprimen el resultado total." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N0MqHFb4F_qn", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Utiliza `tf.GradientTape` para entrenar el modelo." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OZACiVqA8KQV", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z8YT7UmFgpjV" - }, - "source": [ - "Prueba el modelo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIKdEzHAJGt7", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "i-2pkctU_Ci7", - "colab": {} - }, - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = 'Epoch {}, Perdida: {}, Exactitud: {}, Perdida de prueba: {}, Exactitud de prueba: {}'\n", - " print(template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))\n", - "\n", - " # Reinicia las metricas para el siguiente epoch.\n", - " train_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_loss.reset_states()\n", - " test_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "El model de clasificacion de images fue entrenado y alcanzo una exactitud de ~98% en este conjunto de datos. Para aprender mas, lee los [tutoriales de TensorFlow](https://www.tensorflow.org/tutorials)." - ] - } - ] -} \ No newline at end of file diff --git a/site/es/tutorials/quickstart/beginner.ipynb b/site/es/tutorials/quickstart/beginner.ipynb deleted file mode 100644 index 2b2572bfe42..00000000000 --- a/site/es/tutorials/quickstart/beginner.ipynb +++ /dev/null @@ -1,243 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "beginner.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# Guia inicial de TensorFlow 2.0 para principiantes" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Ver en TensorFlow.org\n", - " \n", - " Ejecutar en Google Colab\n", - " \n", - " Ver codigo en GitHub\n", - " \n", - " Descargar notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gw8qUkhemiPf", - "colab_type": "text" - }, - "source": [ - "Note: Nuestra comunidad de Tensorflow ha traducido estos documentos. Como las traducciones de la comunidad\n", - "son basados en el \"mejor esfuerzo\", no hay ninguna garantia que esta sea un reflejo preciso y actual \n", - "de la [Documentacion Oficial en Ingles](https://www.tensorflow.org/?hl=en).\n", - "Si tienen sugerencias sobre como mejorar esta traduccion, por favor envian un \"Pull request\"\n", - "al siguiente repositorio [tensorflow/docs](https://github.com/tensorflow/docs).\n", - "Para ofrecerse como voluntario o hacer revision de las traducciones de la Comunidad\n", - "por favor contacten al siguiente grupo [docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Este es un notebook de [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb). Los programas de Python se executan directamente en tu navegador— una gran manera de aprender y utilizar TensorFlow. \n", - "Para poder seguir este tutorial, ejecuta este notebook en Google Colab presionando el boton en la parte superior de esta pagina.\n", - "\n", - "1. En Colab, selecciona \"connect to a Python runtime\": En la parte superior derecha de la barra de menus selecciona: *CONNECT*.\n", - "2. Para ejecutar todas las celdas de este notebook:\n", - " Selecciona *Runtime* > *Run all*." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nnrWf3PCEzXL" - }, - "source": [ - "Descarga e installa el paquete TensorFlow 2.0 version. Importa TensorFlow en tu programa:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# Installa TensorFlow\n", - "try:\n", - " # %tensorflow_version solo existe in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Carga y prepara el conjunto de datos [MNIST](http://yann.lecun.com/exdb/mnist/). Convierte los ejemplos de numeros enteros a numeros de punto flotante:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FP5258xjs-v", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Construye un modelo `tf.keras.Sequential` apilando capas. Escoge un optimizador y una funcion de perdida para el entrenamiento de tu modelo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "model = tf.keras.models.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Entrena y evalua el modelo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F7dTAzgHDUh7", - "colab": {} - }, - "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", - "model.evaluate(x_test, y_test, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "El model de clasificacion de images fue entrenado y alcanzo una exactitud de ~98% en este conjunto de datos. Para aprender mas, lee los [tutoriales de TensorFlow]." - ] - } - ] -} \ No newline at end of file diff --git a/site/fr/README.md b/site/fr/README.md deleted file mode 100644 index 951aff92f1c..00000000000 --- a/site/fr/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Traductions communautaires - -Notre communauté `TensorFlow` a traduit ces documents. -Étant donné que les traductions effectuées dans la communauté sont plus optimales, -rien ne garantit qu'il s'agit d'un reflet exact et à jour de la documentation anglaise officielle. -Si vous avez des suggestions pour améliorer cette traduction, -veuillez envoyer une demande d'extraction ou `pull request`) à [tensorflow/docs](https://github.com/tensorflow/docs). -Pour vous porter volontaire pour écrire ou réviser les traductions de la communauté, -contactez la [liste docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Remarque : concentrez vos efforts de traduction de TensorFlow 2 dans le -répertoire [site/en/](https://github.com/tensorflow/docs/tree/master/site/en/). -Les documents de la communauté TF 1.x ne seront plus mis à jour lors de la -préparation de la version 2.0. Voir -[l'annonce](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). - -# Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Please focus translation efforts on -[TensorFlow 2](https://www.tensorflow.org) in the -[site/en/](https://github.com/tensorflow/docs/tree/master/site/en/) -directory. TF 1.x community docs will no longer be updated as we prepare for the -2.0 release. See -[the announcement](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). diff --git a/site/fr/REVIEWERS b/site/fr/REVIEWERS deleted file mode 100644 index c81daf40e62..00000000000 --- a/site/fr/REVIEWERS +++ /dev/null @@ -1,11 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/fr directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs -# -ArmanSoltani -TahiriNadia -bayethiernodiop -SamuelAlev diff --git a/site/fr/about/bib.md b/site/fr/about/bib.md deleted file mode 100644 index e70123ed77d..00000000000 --- a/site/fr/about/bib.md +++ /dev/null @@ -1,96 +0,0 @@ -# Livres blancs TensorFlow - -Ce document identifie les livres blancs sur TensorFlow. - -## Apprentissage automatique à grande échelle sur des systèmes distribués hétérogènes - -[Accéder à ce livre blanc.](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/45166.pdf) - -**Résumé:** TensorFlow est une interface pour décrire des algorithmes d'apprentissage machine. et une implémentation pour exécuter ces algorithmes. -Un calcul exprimé en utilisant TensorFlow peut être -exécuté avec peu ou pas de changement sur une grande variété de systèmes hétérogènes, allant des appareils mobiles tels que les téléphones et des tablettes jusqu'à des systèmes distribués à grande échelle de plusieurs centaines de machines et de milliers d'appareils de calcul tels que des cartes graphiques. Le système est flexible et peut être utilisé pour décrire une grande variété d'algorithmes, y compris l'entraînement et l'inférence de modèles de réseau d'apprentissage profond, et cela a été utilisés pour mener des recherches et pour déployer l'apprentissage automatique en production dans plus d'une douzaine de secteurs de l'industrie de l'informatique et d'autres domaines, y compris la reconnaissance vocale, la vision par ordinateur, la robotique, la recherche d'information, le traitement automatique du langage naturel, l'extraction d'information géographique, et la découverte computationnelle de médicaments. Ce livre blanc décrit l'interface TensorFlow et une version de cette interface que nous avons construite chez Google. L'API TensorFlow et une version de référence ont été publiées sous forme de paquet open-source sous licence Apache 2.0 en novembre 2015 et sont disponibles sur www.tensorflow.org. - - -### Au format BibTeX - -Si vous utilisez TensorFlow dans vos recherches et souhaitez citer le système TensorFlow, nous vous suggérons de citer ce livre blanc. - -
    -@misc{tensorflow2015-whitepaper,
-title={ {TensorFlow}: Large-Scale Machine Learning on Heterogeneous Systems},
-url={https://www.tensorflow.org/},
-note={Software available from tensorflow.org},
-author={
-    Mart\'{\i}n~Abadi and
-    Ashish~Agarwal and
-    Paul~Barham and
-    Eugene~Brevdo and
-    Zhifeng~Chen and
-    Craig~Citro and
-    Greg~S.~Corrado and
-    Andy~Davis and
-    Jeffrey~Dean and
-    Matthieu~Devin and
-    Sanjay~Ghemawat and
-    Ian~Goodfellow and
-    Andrew~Harp and
-    Geoffrey~Irving and
-    Michael~Isard and
-    Yangqing Jia and
-    Rafal~Jozefowicz and
-    Lukasz~Kaiser and
-    Manjunath~Kudlur and
-    Josh~Levenberg and
-    Dandelion~Man\'{e} and
-    Rajat~Monga and
-    Sherry~Moore and
-    Derek~Murray and
-    Chris~Olah and
-    Mike~Schuster and
-    Jonathon~Shlens and
-    Benoit~Steiner and
-    Ilya~Sutskever and
-    Kunal~Talwar and
-    Paul~Tucker and
-    Vincent~Vanhoucke and
-    Vijay~Vasudevan and
-    Fernanda~Vi\'{e}gas and
-    Oriol~Vinyals and
-    Pete~Warden and
-    Martin~Wattenberg and
-    Martin~Wicke and
-    Yuan~Yu and
-    Xiaoqiang~Zheng},
-  year={2015},
-}
-
    - -Ou sous forme textuelle: - -
    -Martín Abadi, Ashish Agarwal, Paul Barham, Eugene Brevdo,
-Zhifeng Chen, Craig Citro, Greg S. Corrado, Andy Davis,
-Jeffrey Dean, Matthieu Devin, Sanjay Ghemawat, Ian Goodfellow,
-Andrew Harp, Geoffrey Irving, Michael Isard, Rafal Jozefowicz, Yangqing Jia,
-Lukasz Kaiser, Manjunath Kudlur, Josh Levenberg, Dan Mané, Mike Schuster,
-Rajat Monga, Sherry Moore, Derek Murray, Chris Olah, Jonathon Shlens,
-Benoit Steiner, Ilya Sutskever, Kunal Talwar, Paul Tucker,
-Vincent Vanhoucke, Vijay Vasudevan, Fernanda Viégas,
-Oriol Vinyals, Pete Warden, Martin Wattenberg, Martin Wicke,
-Yuan Yu, and Xiaoqiang Zheng.
-TensorFlow: Large-scale machine learning on heterogeneous systems,
-2015. Software available from tensorflow.org.
-
    - - - -## TensorFlow: un système pour l'apprentissage automatique à grande échelle - -[Accéder à ce livre blanc.](https://www.usenix.org/system/files/conference/osdi16/osdi16-abadi.pdf) - -**Résumé:** TensorFlow est un système d'apprentissage automatique qui fonctionne à grande échelle et dans des environnements hétérogènes. TensorFlow utilise des graphes de flux de données pour représenter le calcul, l'état partagé et les opérations qui font muter cet état. -Il mappe les nœuds d'un graphe de flux de données sur de nombreuses machines d'un cluster, et au sein d'une machine sur plusieurs périphériques de calcul, y compris les processeurs multicœurs, les GPU polyvalents et les ASIC personnalisés appelés Tensor Processing Units (TPU). -Cette architecture donne de la flexibilité au développeur d'applications : alors que dans les précédentes conceptions “serveur de paramètres”, la gestion des états partagés est intégrée au système, TensorFlow permet aux développeurs d'expérimenter de nouvelles optimisations et algorithmes d'apprentissage. -TensorFlow supporte un grand nombre d'applications, avec un accent mis sur l'entraînement et l'inférence des réseaux d'apprentissage profond. -Plusieurs services Google utilisent TensorFlow en production, nous l'avons publié en tant que projet open-source, et il est devenu largement utilisé pour la recherche en apprentissage automatique. -Dans ce livre blanc, nous décrivons le modèle de flux de données de TensorFlow et démontrons les performances intéressantes que TensorFlow permet d'atteindre dans plusieurs applications du monde réel. diff --git a/site/id/CONTRIBUTING.md b/site/id/CONTRIBUTING.md deleted file mode 100644 index eff210b5c01..00000000000 --- a/site/id/CONTRIBUTING.md +++ /dev/null @@ -1,81 +0,0 @@ -# Contribution guideline - -## Directory structure - -Struktur direktori dokumen : - -``` -site -├── en -│ ├── ... -│ ├── ... -│ └── ... -├── id -│ ├── ... -│ ├── ... -│ └── ... -├── ... -└── ... -``` - -Dokumen di bawah `situs / en` adalah dokumen yang belum diterjemahkan. -Terjemahkan dapat diletakkan di bawah `site / id` dalam konfigurasi yang sama -dengan` site / en` dan kirim pull request. - -## Pull request title - -Disarankan bahwa judul (title) pull request adalah "ID: ...". - -Sehingga Peninjau (reviewer) dapat menggunakan kata kunci "ID" untuk mencarinya, -jadi diharapkan kerja samanya untuk proses peninjauan dokumen. - -## Pull request size - -Disarankan untuk membuat pull request dengan membaginya menjadi 1 file untuk 1 -pull request. Ini untuk mengurangi beban peninjau dan untuk mempermudah proses -penyatuan kode, jadi dimohon kerja samanya. - -## Proofreading tool - -[Proofing tool] (https://github.com/tfug/proofreading) digunakan sebagai -penanggulangan fluktuasi notasi kata maupun ejaan, dikarenakan otomatisasi -beberapa ulasan dan terjemahan oleh banyak orang. - -Ini tidak begitu penting, karena peninjau juga memeriksa pada saat proses -peninjauan, tetapi jika Anda memeriksanya terlebih dahulu, Anda akan memerlukan -lebih sedikit koreksi pada proses pull request. - -Kami juga menerima pull request ke Proofing tool. - -## Frequently asked questions - -### Haruskah saya menerjemahkan komentar dalam kode sumber? - -Disarankan untuk menerjemahkan juga bagian komentar dalam kode sumber. - -### Dokumen mana yang harus saya terjemahkan? - -[Terjemahan TF 1.0 telah dinyatakan sebagai akhir pembaruan](https://groups.google.com/a/tensorflow.org/forum/#!msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ) -Jadi tolong terjemahkan dokumentasi TF 2.0. Tidak ada batasan lain. - -### Dapatkah saya melihat siapa yang menerjemahkan dokumen mana? - -Sulit untuk mengetahui dengan pasti, jadi mohon periksa beberapa hal berikut : - -* Cari keyword "ID" untuk mencari tahu apa yang sedang dibuat -* Jika Ada yang ingin di diskusikan, hubungi via Slack atau lainnya. - -### Bagaimana saya bisa menjadi peninjau dokumen ini ? - -Saat ini sedang dibahas kriteria kriteria nya, beberapa kriteria yang sudah ada : - -* Dokumen sudah diterjemahkan dan pull request telah dilakukan - -Jika Anda memenuhi persyaratan di atas dan ingin menjadi peninjau -[REVIEWERS](https://github.com/tensorflow/docs/blob/master/site/ja/REVIEWERS) -Tambahkan ID GitHub Anda ke dan kirim pull request. - -### Apa yang harus saya lakukan jika sisi dokumen bahasa Inggris adalah tautan ke repositori eksternal? - -Tempatkan di dalam `site / id` dengan konfigurasi yang sama dengan repositori -yang ditautkan. diff --git a/site/id/README.md b/site/id/README.md deleted file mode 100644 index 6fe7f1589bc..00000000000 --- a/site/id/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Dokumentasi TensorFlow - -## Translasi Komunitas - -Komunitas TensorFlow kami telah menerjemahkan dokumen-dokumen ini. Tidak ada jaminan bahwa translasi ini akurat, dan translasi terbaru dari [Official Dokumentasi - Bahasa Inggris](https://www.tensorflow.org/?hl=en) karena komunitas translasi ini adalah usaha terbaik dari komunitas translasi. -Jika Anda memiliki saran untuk meningkatkan terjemahan ini, silakan kirim pull request ke [tensorflow/docs](https://github.com/tensorflow/docs) repositori GitHub. -Untuk menjadi sukarelawan untuk menulis atau memeriksa terjemahan komunitas, hubungi -[daftar docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -## Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -# Daftar Istilah - -Glosarium ini bertujuan untuk membakukan kata yang diterjemahkan (atau tidak diterjemahkan tetapi mempertahankan bahasa Inggris asli), dan menjelaskan arti kata-kata itu. - -| Bahasa Inggris | Bahasa Indonesia| Penjelasan | -| - | - | - | -| activation function | | | -| backpropagation | | | -| batch | | | -| callback | | | -| checkpoint | | | -| class | | | -| compile | | | -| dictionary (in python) | | | -| dimension | | | -| dropout | | | -| define-by-run | | | -| eager execution | | | -| embedding | | | -| epoch | | | -| helper | | | -| hyperparameter | | | -| layer | | | -| label | | | -| loss function | | | -| mini-batch | | | -| neuron | | | -| node | | | -| one-hot encoding | | | -| optimizer | | | -| padding | | | -| penalty | | | -| sequence || | -| training loop | | | -| training set | | | -| target | | | -| unit | | | -| utility function | | | diff --git a/site/id/REVIEWERS b/site/id/REVIEWERS deleted file mode 100644 index 3e63ceb0b59..00000000000 --- a/site/id/REVIEWERS +++ /dev/null @@ -1,9 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/id directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs -# -febriaroo -rendrat diff --git a/site/id/tutorials/load_data/csv.ipynb b/site/id/tutorials/load_data/csv.ipynb deleted file mode 100644 index b89c1728d89..00000000000 --- a/site/id/tutorials/load_data/csv.ipynb +++ /dev/null @@ -1,1041 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "csv.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DweYe9FcbMK_" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AVV2e0XKbJeX", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sUtoed20cRJJ" - }, - "source": [ - "# Muat data CSV" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1ap_W4aQcgNT" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Lihat di TensorFlow.org\n", - " \n", - " Jalankan di Google Colab\n", - " \n", - " Lihat kode di GitHub\n", - " \n", - " Unduh notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yPWYwGI7pIkH", - "colab_type": "text" - }, - "source": [ - "Note: Komunitas TensorFlow kami telah menerjemahkan dokumen-dokumen ini. Tidak ada jaminan bahwa translasi ini akurat, dan translasi terbaru dari [Official Dokumentasi - Bahasa Inggris](https://www.tensorflow.org/?hl=en) karena komunitas translasi ini adalah usaha terbaik dari komunitas translasi.\n", - "Jika Anda memiliki saran untuk meningkatkan terjemahan ini, silakan kirim pull request ke [tensorflow/docs](https://github.com/tensorflow/docs) repositori GitHub.\n", - "Untuk menjadi sukarelawan untuk menulis atau memeriksa terjemahan komunitas, hubungi\n", - "[daftar docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C-3Xbt0FfGfs" - }, - "source": [ - "Tutorial ini memberikan contoh cara memuat data CSV dari file ke `tf.data.Dataset`.\n", - "\n", - "Data yang digunakan dalam tutorial ini diambil dari daftar penumpang Titanic. Model akan memprediksi kemungkinan penumpang selamat berdasarkan karakteristik seperti usia, jenis kelamin, kelas tiket, dan apakah orang tersebut bepergian sendirian." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "I4dwMQVQMQWD", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version hanya ada di Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "baYFZMW_bJHh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "import functools\n", - "\n", - "import numpy as np\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ncf5t6tgL5ZI", - "colab": {} - }, - "source": [ - "TRAIN_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/train.csv\"\n", - "TEST_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/eval.csv\"\n", - "\n", - "train_file_path = tf.keras.utils.get_file(\"train.csv\", TRAIN_DATA_URL)\n", - "test_file_path = tf.keras.utils.get_file(\"eval.csv\", TEST_DATA_URL)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ONE94qulk6S", - "colab": {} - }, - "source": [ - "# Membuat nilai numpy agar lebih mudah dibaca.\n", - "np.set_printoptions(precision=3, suppress=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wuqj601Qw0Ml" - }, - "source": [ - "## Memuat data\n", - "\n", - "Untuk memulai, mari kita lihat bagian atas file CSV untuk melihat formatnya.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "54Dv7mCrf9Yw", - "colab": {} - }, - "source": [ - "!head {train_file_path}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jC9lRhV-q_R3" - }, - "source": [ - "Anda dapat [memuat ini menggunakan panda] (pandas.ipynb), dan meneruskan array NumPy ke TensorFlow. Jika Anda perlu meningkatkan ke file dengan skala besar, atau membutuhkan loader yang terintegrasi dengan [TensorFlow dan tf.data] (../../panduan/data.ipynb) kemudian gunakan fungsi `tf.data.experimental.make_csv_dataset`:" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "67mfwr4v-mN_" - }, - "source": [ - "Satu-satunya kolom yang perlu Anda identifikasi secara eksplisit adalah kolom dengan nilai yang dimaksudkan untuk diprediksi oleh model." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iXROZm5f3V4E", - "colab": {} - }, - "source": [ - "LABEL_COLUMN = 'survived'\n", - "LABELS = [0, 1]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t4N-plO4tDXd" - }, - "source": [ - "Sekarang baca data CSV dari file dan buat sebuah dataset.\n", - "\n", - "(Untuk dokumentasi lengkap, lihat `tf.data.experimental.make_csv_dataset`)\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yIbUscB9sqha", - "colab": {} - }, - "source": [ - "def get_dataset(file_path, **kwargs):\n", - " dataset = tf.data.experimental.make_csv_dataset(\n", - " file_path,\n", - " batch_size=5, # Artifisial kecil untuk membuat contoh lebih mudah ditampilkan.\n", - " label_name=LABEL_COLUMN,\n", - " na_value=\"?\",\n", - " num_epochs=1,\n", - " ignore_errors=True, \n", - " **kwargs)\n", - " return dataset\n", - "\n", - "raw_train_data = get_dataset(train_file_path)\n", - "raw_test_data = get_dataset(test_file_path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "v4oMO9MIxgTG", - "colab": {} - }, - "source": [ - "def show_batch(dataset):\n", - " for batch, label in dataset.take(1):\n", - " for key, value in batch.items():\n", - " print(\"{:20s}: {}\".format(key,value.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vHUQFKoQI6G7" - }, - "source": [ - "Setiap item dalam dataset adalah *batch*, direpresentasikan sebagai *tuple* dari (*banyak contoh*, *banyak label*). Data dari contoh-contoh tersebut disusun dalam tensor berbasis kolom (bukan tensor berbasis baris), masing-masing dengan elemen sebanyak ukuran batch (dalam kasus ini 5).\n", - "\n", - "Dengan melihat sendiri, mungkin akan membantu Anda untuk memahami." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HjrkJROoxoll", - "colab": {} - }, - "source": [ - "show_batch(raw_train_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YOYKQKmMj3D6" - }, - "source": [ - "Seperti yang Anda lihat, kolom dalam file CSV diberi nama. Konstruktor dataset akan mengambil nama-nama ini secara otomatis. Jika file yang sedang Anda kerjakan tidak mengandung nama kolom di baris pertama, berikan mereka dalam daftar string ke argumen `column_names` dalam fungsi` make_csv_dataset`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2Av8_9L3tUg1", - "colab": {} - }, - "source": [ - "CSV_COLUMNS = ['survived', 'sex', 'age', 'n_siblings_spouses', 'parch', 'fare', 'class', 'deck', 'embark_town', 'alone']\n", - "\n", - "temp_dataset = get_dataset(train_file_path, column_names=CSV_COLUMNS)\n", - "\n", - "show_batch(temp_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gZfhoX7bR9u4" - }, - "source": [ - "Contoh ini akan menggunakan semua kolom yang tersedia. Jika Anda perlu menghilangkan beberapa kolom dari dataset, buat daftar hanya kolom yang Anda rencanakan untuk digunakan, dan kirimkan ke argumen `select_columns` (opsional) dari konstruktor.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "S1TzSkUKwsNP", - "colab": {} - }, - "source": [ - "SELECT_COLUMNS = ['survived', 'age', 'n_siblings_spouses', 'class', 'deck', 'alone']\n", - "\n", - "temp_dataset = get_dataset(train_file_path, select_columns=SELECT_COLUMNS)\n", - "\n", - "show_batch(temp_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9cryz31lxs3e" - }, - "source": [ - "## Data pre-processing\n", - "\n", - "File CSV dapat berisi berbagai tipe data. Biasanya Anda ingin mengonversi dari berbagai tipe ke vektor dengan panjang tetap sebelum memasukkan data ke dalam model Anda.\n", - "\n", - "TensorFlow memiliki sistem bawaan untuk menjelaskan konversi input umum: `tf.feature_column`, lihat [tutorial ini](../keras/feature_columns) untuk detailnya.\n", - "\n", - "Anda dapat memproses data Anda menggunakan alat apa pun yang Anda suka (seperti [nltk](https://www.nltk.org/) atau [sklearn](https://scikit-learn.org/stable/)), dan kemudian memberikan output yang telah diproses ke TensorFlow.\n", - "\n", - "Keuntungan utama melakukan preprocessing di dalam model Anda adalah ketika Anda mengekspor model itu termasuk dengan proses preprocessing. Dengan cara ini Anda bisa mengirimkan data mentah langsung ke model Anda." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9AsbaFmCeJtF" - }, - "source": [ - "### Continuous data" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xl0Q0DcfA_rt" - }, - "source": [ - "Jika data Anda sudah dalam format numerik yang sesuai, Anda bisa mengemas data ke dalam vektor sebelum meneruskannya ke model:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Yfji3J5BMxz", - "colab": {} - }, - "source": [ - "SELECT_COLUMNS = ['survived', 'age', 'n_siblings_spouses', 'parch', 'fare']\n", - "DEFAULTS = [0, 0.0, 0.0, 0.0, 0.0]\n", - "temp_dataset = get_dataset(train_file_path, \n", - " select_columns=SELECT_COLUMNS,\n", - " column_defaults = DEFAULTS)\n", - "\n", - "show_batch(temp_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zEUhI8kZCfq8", - "colab": {} - }, - "source": [ - "example_batch, labels_batch = next(iter(temp_dataset)) " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IP45_2FbEKzn" - }, - "source": [ - "Berikut adalah fungsi sederhana yang akan menyatukan semua\n", - "\n", - "---\n", - "\n", - "kolom:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JQ0hNSL8CC3a", - "colab": {} - }, - "source": [ - "def pack(features, label):\n", - " return tf.stack(list(features.values()), axis=-1), label" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "75LA9DisEIoE" - }, - "source": [ - "Terapkan ini ke setiap elemen dataset:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VnP2Z2lwCTRl", - "colab": {} - }, - "source": [ - "packed_dataset = temp_dataset.map(pack)\n", - "\n", - "for features, labels in packed_dataset.take(1):\n", - " print(features.numpy())\n", - " print()\n", - " print(labels.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1VBvmaFrFU6J" - }, - "source": [ - "Jika Anda memiliki data dengan berbagai tipe, Anda mungkin ingin memisahkan data simple-numeric fields. Api `tf.feature_column` dapat menanganinya, tetapi hal ini menimbulkan beberapa overhead dan harus dihindari kecuali benar-benar diperlukan. Kembali ke kumpulan data campuran:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ad-IQ_JPFQge", - "colab": {} - }, - "source": [ - "show_batch(raw_train_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HSrYNKKcIdav", - "colab": {} - }, - "source": [ - "example_batch, labels_batch = next(iter(temp_dataset)) " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p5VtThKfGPaQ" - }, - "source": [ - "Jadi tentukan preprosesor yang lebih umum yang memilih daftar fitur numerik dan mengemasnya ke dalam satu kolom:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5DRishYYGS-m", - "colab": {} - }, - "source": [ - "class PackNumericFeatures(object):\n", - " def __init__(self, names):\n", - " self.names = names\n", - "\n", - " def __call__(self, features, labels):\n", - " numeric_freatures = [features.pop(name) for name in self.names]\n", - " numeric_features = [tf.cast(feat, tf.float32) for feat in numeric_freatures]\n", - " numeric_features = tf.stack(numeric_features, axis=-1)\n", - " features['numeric'] = numeric_features\n", - "\n", - " return features, labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1SeZka9AHfqD", - "colab": {} - }, - "source": [ - "NUMERIC_FEATURES = ['age','n_siblings_spouses','parch', 'fare']\n", - "\n", - "packed_train_data = raw_train_data.map(\n", - " PackNumericFeatures(NUMERIC_FEATURES))\n", - "\n", - "packed_test_data = raw_test_data.map(\n", - " PackNumericFeatures(NUMERIC_FEATURES))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wFrw0YobIbUB", - "colab": {} - }, - "source": [ - "show_batch(packed_train_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_EPUS8fPLUb1", - "colab": {} - }, - "source": [ - "example_batch, labels_batch = next(iter(packed_train_data)) " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o2maE8d2ijsq" - }, - "source": [ - "#### Normalisasi Data\n", - "\n", - "Data kontinu (continues data) harus selalu dinormalisasi." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WKT1ASWpwH46", - "colab": {} - }, - "source": [ - "import pandas as pd\n", - "desc = pd.read_csv(train_file_path)[NUMERIC_FEATURES].describe()\n", - "desc" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cHHstcKPsMXM", - "colab": {} - }, - "source": [ - "MEAN = np.array(desc.T['mean'])\n", - "STD = np.array(desc.T['std'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "REKqO_xHPNx0", - "colab": {} - }, - "source": [ - "def normalize_numeric_data(data, mean, std):\n", - " # Center the data\n", - " return (data-mean)/std" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VPsoMUgRCpUM" - }, - "source": [ - "Sekarang buat kolom angka. API `tf.feature_columns.numeric_column` menerima argumen `normalizer_fn`, yang akan dijalankan pada setiap batch.\n", - "\n", - "Bundlekan `MEAN` dan `STD` ke normalizer fn menggunakan [`functools.partial`](https://docs.python.org/3/library/functools.html#functools.partial)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Bw0I35xRS57V", - "colab": {} - }, - "source": [ - "# Lihat apa yang baru saja Anda buat.\n", - "normalizer = functools.partial(normalize_numeric_data, mean=MEAN, std=STD)\n", - "\n", - "numeric_column = tf.feature_column.numeric_column('numeric', normalizer_fn=normalizer, shape=[len(NUMERIC_FEATURES)])\n", - "numeric_columns = [numeric_column]\n", - "numeric_column" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HZxcHXc6LCa7" - }, - "source": [ - "Saat Anda melatih model, sertakan kolom fitur ini untuk memilih dan memusatkan blok data numerik ini:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "b61NM76Ot_kb", - "colab": {} - }, - "source": [ - "example_batch['numeric']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "j-r_4EAJAZoI", - "colab": {} - }, - "source": [ - "numeric_layer = tf.keras.layers.DenseFeatures(numeric_columns)\n", - "numeric_layer(example_batch).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M37oD2VcCO4R" - }, - "source": [ - "Normalisasi berdasarkan rata-rata yang digunakan di sini mewajibkan kita untuk mengetahui rata-rata setiap kolom sebelumnya." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tSyrkSQwYHKi" - }, - "source": [ - "### Kategori data\n", - "\n", - "Beberapa kolom dalam data CSV adalah kolom kategorikal. Artinya, kontennya harus menjadi salah satu dari opsi yang ada.\n", - "\n", - "Gunakan API `tf.feature_column` untuk membuat koleksi dengan `tf.feature_column.indicator_column` untuk setiap kolom kategorikal." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mWDniduKMw-C", - "colab": {} - }, - "source": [ - "CATEGORIES = {\n", - " 'sex': ['male', 'female'],\n", - " 'class' : ['First', 'Second', 'Third'],\n", - " 'deck' : ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'],\n", - " 'embark_town' : ['Cherbourg', 'Southhampton', 'Queenstown'],\n", - " 'alone' : ['y', 'n']\n", - "}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kkxLdrsLwHPT", - "colab": {} - }, - "source": [ - "categorical_columns = []\n", - "for feature, vocab in CATEGORIES.items():\n", - " cat_col = tf.feature_column.categorical_column_with_vocabulary_list(\n", - " key=feature, vocabulary_list=vocab)\n", - " categorical_columns.append(tf.feature_column.indicator_column(cat_col))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "H18CxpHY_Nma", - "colab": {} - }, - "source": [ - "# See what you just created.\n", - "categorical_columns" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p7mACuOsArUH", - "colab": {} - }, - "source": [ - "categorical_layer = tf.keras.layers.DenseFeatures(categorical_columns)\n", - "print(categorical_layer(example_batch).numpy()[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R7-1QG99_1sN" - }, - "source": [ - "Ini akan menjadi bagian dari pemrosesan input data ketika Anda membangun model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPWkC4_1l3IG" - }, - "source": [ - "### Layer preprocessing gabungan" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R3QAjo1qD4p9" - }, - "source": [ - "Tambahkan dua koleksi kolom fitur dan teruskan ke `tf.keras.layers.DenseFeatures` untuk membuat lapisan input yang akan mengekstraksi dan memproses kedua jenis input:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3-OYK7GnaH0r", - "colab": {} - }, - "source": [ - "preprocessing_layer = tf.keras.layers.DenseFeatures(categorical_columns+numeric_columns)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m7_U_K0UMSVS", - "colab": {} - }, - "source": [ - "print(preprocessing_layer(example_batch).numpy()[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DlF_omQqtnOP" - }, - "source": [ - "## Membangun Model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lQoFh16LxtT_" - }, - "source": [ - "Jalankan `tf.keras.Sequential`, mulai dari `preprocessing_layer`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3mSGsHTFPvFo", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " preprocessing_layer,\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid'),\n", - "])\n", - "\n", - "model.compile(\n", - " loss='binary_crossentropy',\n", - " optimizer='adam',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hPdtI2ie0lEZ" - }, - "source": [ - "## Latih, evaluasi, dan prediksi" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8gvw1RE9zXkD" - }, - "source": [ - "Sekarang model dapat dipakai dan dilatih." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sW-4XlLeEQ2B", - "colab": {} - }, - "source": [ - "train_data = packed_train_data.shuffle(500)\n", - "test_data = packed_test_data" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Q_nm28IzNDTO", - "colab": {} - }, - "source": [ - "model.fit(train_data, epochs=20)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QyDMgBurzqQo" - }, - "source": [ - "Setelah model dilatih, Anda dapat memeriksa akurasinya pada set `test_data`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eB3R3ViVONOp", - "colab": {} - }, - "source": [ - "test_loss, test_accuracy = model.evaluate(test_data)\n", - "\n", - "print('\\n\\nTest Loss {}, Test Accuracy {}'.format(test_loss, test_accuracy))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sTrn_pD90gdJ" - }, - "source": [ - "Gunakan `tf.keras.Model.predict` untuk menyimpulkan pada label batch atau dataset batch.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qwcx74F3ojqe", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_data)\n", - "\n", - "# Tampilkan beberapa hasil\n", - "for prediction, survived in zip(predictions[:10], list(test_data)[0][1][:10]):\n", - " print(\"Predicted survival: {:.2%}\".format(prediction[0]),\n", - " \" | Actual outcome: \",\n", - " (\"SURVIVED\" if bool(survived) else \"DIED\"))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/id/tutorials/quickstart/advanced.ipynb b/site/id/tutorials/quickstart/advanced.ipynb deleted file mode 100644 index 10b483b27e6..00000000000 --- a/site/id/tutorials/quickstart/advanced.ipynb +++ /dev/null @@ -1,424 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "orig_nbformat": 2, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "npconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": 3, - "colab": { - "name": "advanced.ipynb", - "provenance": [], - "private_outputs": true, - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# TensorFlow 2 quickstart untuk tingkat lanjut" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Lihat di TensorFlow.org\n", - " \n", - " Jalankan di Google Colab\n", - " \n", - " Lihat sumber kode di GitHub\n", - " \n", - " Unduh notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "LKIURqSTZU4U", - "colab_type": "text" - }, - "source": [ - "Note: Komunitas TensorFlow kami telah menerjemahkan dokumen-dokumen ini. Tidak ada jaminan bahwa translasi ini akurat, dan translasi terbaru dari [Official Dokumentasi - Bahasa Inggris](https://www.tensorflow.org/?hl=en) karena komunitas translasi ini adalah usaha terbaik dari komunitas translasi.\n", - "Jika Anda memiliki saran untuk meningkatkan terjemahan ini, silakan kirim pull request ke [tensorflow/docs](https://github.com/tensorflow/docs) repositori GitHub.\n", - "Untuk menjadi sukarelawan untuk menulis atau memeriksa terjemahan komunitas, hubungi\n", - "[daftar docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Ini adalah file notebook [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb). Program python akan dijalankan langsung dari browser — cara yang bagus untuk mempelajari dan menggunakan TensorFlow. Untuk mengikuti tutorial ini, jalankan notebook di Google Colab dengan mengklik tombol di bagian atas halaman ini.\n", - "\n", - "1. Di halaman Colab, sambungkan ke runtime Python: Di menu sebelah kanan atas, pilih * CONNECT *.\n", - "2. Untuk menjalankan semua sel kode pada notebook: Pilih * Runtime *> * Run all *." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eOsVdx6GGHmU" - }, - "source": [ - "Download dan instal TensorFlow 2 dan impor TensorFlow ke dalam program Anda:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ioLbtB3uGKPX", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version hanya ada di Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QS7DDTiZGRTo" - }, - "source": [ - "Impor TensorFlow ke dalam program Anda:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Siapkan [dataset MNIST](http://yann.lecun.com/exdb/mnist/)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JqFRS6K07jJs", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# Tambahkan dimensi chanel\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "Gunakan `tf.data` untuk mengelompokkan dan mengatur kembali dataset:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Iu_quO024c2", - "colab": {} - }, - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Buat model `tf.keras` menggunakan Keras [model subclassing API](https://www.tensorflow.org/guide/keras#model_subclassing):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "# Buat sebuah contoh dari model\n", - "model = MyModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "Pilih fungsi untuk mengoptimalkan dan fungsi untuk menilai loss dari hasil pelatihan:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u48C9WQ774n4", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "Pilih metrik untuk mengukur loss dan keakuratan model. Metrik ini mengakumulasi nilai di atas epochs dan kemudian mencetak hasil secara keseluruhan." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N0MqHFb4F_qn", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Gunakan `tf.GradientTape` untuk melatih model:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OZACiVqA8KQV", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "LnQ3nUVuXf6B", - "colab_type": "text" - }, - "source": [ - "Tes modelnya:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIKdEzHAJGt7", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "i-2pkctU_Ci7", - "colab": {} - }, - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n", - " print(template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))\n", - "\n", - " # Menghitung ulang metrik untuk epoch selanjutnya\n", - " train_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_loss.reset_states()\n", - " test_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "Penggolong gambar tersebut, sekarang dilatih untuk akurasi ~ 98% pada dataset ini. Untuk mempelajari lebih lanjut, baca [tutorial TensorFlow](https://www.tensorflow.org/tutorials/)." - ] - } - ] -} \ No newline at end of file diff --git a/site/id/tutorials/quickstart/beginner.ipynb b/site/id/tutorials/quickstart/beginner.ipynb deleted file mode 100644 index cb7a9ea92a4..00000000000 --- a/site/id/tutorials/quickstart/beginner.ipynb +++ /dev/null @@ -1,257 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "orig_nbformat": 2, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "npconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": 3, - "colab": { - "name": "beginner.ipynb", - "provenance": [], - "private_outputs": true, - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# TensorFlow 2 quickstart for beginners" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Lihat di TensorFlow.org\n", - " \n", - " Jalankan di Google Colab\n", - " \n", - " Lihat sumber kode di GitHub\n", - " \n", - " Unduh notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DJELZj47ZL1o", - "colab_type": "text" - }, - "source": [ - "Note: Komunitas TensorFlow kami telah menerjemahkan dokumen-dokumen ini. Tidak ada jaminan bahwa translasi ini akurat, dan translasi terbaru dari [Official Dokumentasi - Bahasa Inggris](https://www.tensorflow.org/?hl=en) karena komunitas translasi ini adalah usaha terbaik dari komunitas translasi.\n", - "Jika Anda memiliki saran untuk meningkatkan terjemahan ini, silakan kirim pull request ke [tensorflow/docs](https://github.com/tensorflow/docs) repositori GitHub.\n", - "Untuk menjadi sukarelawan untuk menulis atau memeriksa terjemahan komunitas, hubungi\n", - "[daftar docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "04QgGZc9bF5D" - }, - "source": [ - "Panduan singkat ini akan menggunakan [Keras](https://www.tensorflow.org/guide/keras/overview) untuk:\n", - "\n", - "1. Membangun jaringan saraf tiruan yang mengklasifikasikan gambar.\n", - "2. Melatih jaringan saraf tiruan tersebut.\n", - "3. Dan, pada akhirnya, mengevaluasi keakuratan dari model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Ini adalah file notebook [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb). Program python akan dijalankan langsung dari browser — cara yang bagus untuk mempelajari dan menggunakan TensorFlow. Untuk mengikuti tutorial ini, jalankan notebook di Google Colab dengan mengklik tombol di bagian atas halaman ini.\n", - "\n", - "1. Di halaman Colab, sambungkan ke runtime Python: Di menu sebelah kanan atas, pilih * CONNECT *.\n", - "2. Untuk menjalankan semua sel kode pada notebook: Pilih * Runtime *> * Run all *." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nnrWf3PCEzXL" - }, - "source": [ - "Download dan instal TensorFlow 2 dan impor TensorFlow ke dalam program Anda:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# Install TensorFlow\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Siapkan [dataset MNIST](http://yann.lecun.com/exdb/mnist/). Ubah sampel dari bilangan bulat menjadi angka floating-point (desimal):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FP5258xjs-v", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Build model `tf.keras.Sequential` dengan cara menumpuk lapisan layer. Untuk melatih data, pilih fungsi untuk mengoptimalkan dan fungsi untuk menghitung kerugian:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "model = tf.keras.models.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Melatih dan mengevaluasi model:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F7dTAzgHDUh7", - "colab": {} - }, - "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", - "model.evaluate(x_test, y_test, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "Penggolong gambar tersebut, sekarang dilatih untuk akurasi ~ 98% pada dataset ini. Untuk mempelajari lebih lanjut, baca [tutorial TensorFlow](https://www.tensorflow.org/tutorials/)." - ] - } - ] -} \ No newline at end of file diff --git a/site/it/README.md b/site/it/README.md deleted file mode 100644 index 1430d5016e2..00000000000 --- a/site/it/README.md +++ /dev/null @@ -1,40 +0,0 @@ -## Traduzioni della comunità - -La nostra comunità di Tensorflow ha tradotto questi documenti. Poichè queste traduzioni sono *best-effort*, non è garantito che rispecchino in maniera precisa e aggiornata la [documentazione ufficiale in inglese](https://www.tensorflow.org/?hl=en). -Se avete suggerimenti per migliorare questa traduzione, mandate per favore una pull request al repository Github [tensorflow/docs](https://github.com/tensorflow/docs). -Per proporsi come volontari alla scrittura o alla review delle traduzioni della comunità contattate la -[mailing list docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -## Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -## Guida alla traduzione italiana - -Alcune parole fanno parte di un vocabolario tecnico inglese e pertanto non hanno una traduzione naturale. Per favore *non* traducete le seguenti parole: - -* batch(es) -* browser -* cluster(s) -* dataflow -* deep learning -* layer(s) -* machine learning -* multicore -* notebook(s) -* overfit(ting) -* pattern(s) -* pixel(s) -* skill(s) -* tablet(s) -* test(s) -* tutorials(s) -* underfit(ting) -* white paper(s) diff --git a/site/it/REVIEWERS b/site/it/REVIEWERS deleted file mode 100644 index 437accead50..00000000000 --- a/site/it/REVIEWERS +++ /dev/null @@ -1,11 +0,0 @@ -# Aggiungi il tuo username GitHub per essere taggato nei commenti su -# tutte le pull requests alla cartella site/it. Rimuovi il tuo -# username quando desideri. -# -# Notifiche per le pull request create sono mandate anche alla lista -# docs@tensorflow.org: https://groups.google.com/a/tensorflow.org/forum/#!forum/docs -# -mcaci -bhack -codeadmin_peritiae -omonimus1 diff --git a/site/it/api_docs/index.md b/site/it/api_docs/index.md deleted file mode 100644 index 5252e752828..00000000000 --- a/site/it/api_docs/index.md +++ /dev/null @@ -1,37 +0,0 @@ -# Documentazione API - -Note: La nostra comunità di Tensorflow ha tradotto questi documenti. Poichè queste traduzioni sono *best-effort*, non è garantito che rispecchino in maniera precisa e aggiornata la [documentazione ufficiale in inglese](https://www.tensorflow.org/?hl=en). -Se avete suggerimenti per migliorare questa traduzione, mandate per favore una pull request al repository Github [tensorflow/docs](https://github.com/tensorflow/docs). -Per proporsi come volontari alla scrittura o alla review delle traduzioni della comunità contattate la "[mailing list docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs) - -TensorFlow dispone di API in diversi linguaggi sia per costruire, sia per -eseguire un grafo TensorFlow. Al momento, le API Python API sono le più complete -e facili da usare, ma possono essere facilmente integrate nei progetti API in altri linguaggi, -che possono offrire vantaggi in terini di prestazioni nell'esecuzione dei grafi. - -Una nota di attenzione: le API in linguaggi diversi da Python non sono ancora coperte -dal vincolo di stabilità delle [API](https://www.tensorflow.org/guide/versions). - -* [Python](https://www.tensorflow.org/api_docs/python/) -* [JavaScript](https://js.tensorflow.org/api/latest/) -* [C++](https://www.tensorflow.org/api_docs/cc/) -* [Java](https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/package-summary) -* [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go) -* [Swift (Early Release)](https://www.tensorflow.org/swift) - - -Noi incoraggiamo la comunità a sviluppare e mantenere il supporto per altri linguaggi -con l' [approccio raccomandato ai manutentori di -TensorFlow](https://github.com/tensorflow/docs/blob/master/site/en/r1/guide/extend/bindings.md). -Per esempio, vedere i seguenti collegamenti per: - -* C#: [TensorFlowSharp](https://github.com/migueldeicaza/TensorFlowSharp) e [TensorFlow.NET](https://github.com/SciSharp/TensorFlow.NET), -* [Haskell](https://github.com/tensorflow/haskell), -* [Julia](https://github.com/malmaud/TensorFlow.jl), -* [Ruby](https://github.com/somaticio/tensorflow.rb), -* [Rust](https://github.com/tensorflow/rust), and -* [Scala](https://github.com/eaplatanios/tensorflow_scala). - -Inoltre, sono disponibili i riferimenti per le API C++ di TensorFlow Serving: - -* [TensorFlow Serving](https://www.tensorflow.org/tfx/guide/serving) diff --git a/site/it/tutorials/quickstart/advanced.ipynb b/site/it/tutorials/quickstart/advanced.ipynb deleted file mode 100644 index 6fd7e0651e0..00000000000 --- a/site/it/tutorials/quickstart/advanced.ipynb +++ /dev/null @@ -1,430 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "advanced.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# Introduzione a TensorFlow 2 per esperti" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Visualizza su TensorFlow.org\n", - " \n", - " Esegui in Google Colab\n", - " \n", - " Visualizza il sorgente su GitHub\n", - " \n", - " Scarica il notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BYzaKBe8YXg0", - "colab_type": "text" - }, - "source": [ - "Note: La nostra comunità di Tensorflow ha tradotto questi documenti. Poichè queste traduzioni sono *best-effort*, non è garantito che rispecchino in maniera precisa e aggiornata la [documentazione ufficiale in inglese](https://www.tensorflow.org/?hl=en). \n", - "Se avete suggerimenti per migliorare questa traduzione, mandate per favore una pull request al repository Github [tensorflow/docs](https://github.com/tensorflow/docs). \n", - "Per proporsi come volontari alla scrittura o alla review delle traduzioni della comunità contattate la \n", - "[mailing list docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Questo è un [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) notebook file. I programmi Python sono eseguiti direttamente nel browser—un ottimo modo per imparare e utilizzare TensorFlow. Per seguire questo tutorial, esegui il file notebook in Google Colab cliccando sul bottone in cima a questa pagina.\n", - "\n", - "1. All'interno di Colab, connettiti al runtime di Python: In alto a destra della barra dei menu, seleziona *CONNECT*.\n", - "2. Esegui tutte le celle di codice di notebook: Seleziona *Runtime* > *Run all*." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eOsVdx6GGHmU" - }, - "source": [ - "Scarica e installa il pacchetto TensorFlow 2:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ioLbtB3uGKPX", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QS7DDTiZGRTo" - }, - "source": [ - "Importa TensorFlow nel tuo codice:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Carica e prepara il [dataset MNIST](http://yann.lecun.com/exdb/mnist/)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JqFRS6K07jJs", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# Add a channels dimension\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "Usa `tf.data` per raggruppare e mischiare il dataset:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Iu_quO024c2", - "colab": {} - }, - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Costrusci il modello `tf.keras` usando l'[API Keras per creare sottoclassi di modelli](https://www.tensorflow.org/guide/keras#model_subclassing):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "# Create an instance of the model\n", - "model = MyModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "Scegli un metodo di ottimizzazione e una funzione obiettivo per l'addestramento:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u48C9WQ774n4", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "Seleziona delle metriche per misurare la pertita e l'accuratezza del modello. Queste metriche accumulano i valori alle varie epoche e alla fine stampano il risultato globale." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N0MqHFb4F_qn", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Usa `tf.GradientTape` per addestrare il modello:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OZACiVqA8KQV", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z8YT7UmFgpjV" - }, - "source": [ - "Testa il modello:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIKdEzHAJGt7", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "i-2pkctU_Ci7", - "colab": {} - }, - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n", - " print(template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))\n", - "\n", - " # Reset the metrics for the next epoch\n", - " train_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_loss.reset_states()\n", - " test_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "Il classificatore di immagini è ora addestrato per circa il 98% di accuratezza su questo insieme di dati. Per approfondire, leggi i [tutorials di TensorFlow](https://www.tensorflow.org/tutorials)." - ] - } - ] -} diff --git a/site/it/tutorials/quickstart/beginner.ipynb b/site/it/tutorials/quickstart/beginner.ipynb deleted file mode 100644 index 1b6435cf670..00000000000 --- a/site/it/tutorials/quickstart/beginner.ipynb +++ /dev/null @@ -1,253 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "beginner.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "rX8mhOLljYeM" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# Guida rapida a Tensorflow 2 per principianti" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Visualizza su TensorFlow.org\n", - " \n", - " Esegui in Google Colab\n", - " \n", - " Visualizza il sorgente su GitHub\n", - " \n", - " Scarica il notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "uWZYm5nCXhO1", - "colab_type": "text" - }, - "source": [ - "Note: La nostra comunità di Tensorflow ha tradotto questi documenti. Poichè queste traduzioni sono *best-effort*, non è garantito che rispecchino in maniera precisa e aggiornata la [documentazione ufficiale in inglese](https://www.tensorflow.org/?hl=en). \n", - "Se avete suggerimenti per migliorare questa traduzione, mandate per favore una pull request al repository Github [tensorflow/docs](https://github.com/tensorflow/docs). \n", - "Per proporsi come volontari alla scrittura o alla review delle traduzioni della comunità contattate la \n", - "[mailing list docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "04QgGZc9bF5D" - }, - "source": [ - "Questa breve introduzione usa [Keras](https://www.tensorflow.org/guide/keras/overview) per:\n", - "\n", - "1. Costruire una rete neurale che classifica immagini.\n", - "2. Addestrare questa rete neurale.\n", - "3. E, infine, valutare l'accuratezza del modello." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Questo è un [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) notebook file. I programmi Python sono eseguiti direttamente nel browser—un ottimo modo per imparare e utilizzare TensorFlow. Per seguire questo tutorial, esegui il file notebook in Google Colab cliccando sul bottone in cima a questa pagina.\n", - "\n", - "1. All'interno di Colab, connettiti al runtime di Python: In alto a destra della barra dei menu, seleziona *CONNECT*.\n", - "2. Esegui tutte le celle di codice di notebook: Seleziona *Runtime* > *Run all*." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nnrWf3PCEzXL" - }, - "source": [ - "Scarica e installa il package TensorFlow 2. Importa TensorFlow nel tuo codice:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# Install TensorFlow\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Carica e prepara il [dataset MNIST](http://yann.lecun.com/exdb/mnist/). Converti gli esempi da numeri di tipo integer a floating-point:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FP5258xjs-v", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Costruisci il modello `tf.keras.Sequential` tramite layer. Scegli un metodo di ottimizzazione e una funzione obiettivo per l'addestramento:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "model = tf.keras.models.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Addestra e valuta il modello:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F7dTAzgHDUh7", - "colab": {} - }, - "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", - "model.evaluate(x_test, y_test, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "Il classificatore di immagini è ora addestrato per circa il 98% di accuratezza su questo insieme di dati. Per approfondire, leggi il [tutorial di TensorFlow](https://www.tensorflow.org/tutorials/)." - ] - } - ] -} diff --git a/site/ja/CONTRIBUTING.md b/site/ja/CONTRIBUTING.md deleted file mode 100644 index 6ba21a06940..00000000000 --- a/site/ja/CONTRIBUTING.md +++ /dev/null @@ -1,69 +0,0 @@ -# Contribution guideline - -## Directory structure - -ドキュメントのディレクトリ構成は次のようになっています: - -``` -site -├── en -│ ├── ... -│ ├── ... -│ └── ... -├── ja -│ ├── ... -│ ├── ... -│ └── ... -├── ... -└── ... -``` - -`site/en` 以下にあるけれど `site/ja` 以下に無いものは、まだ翻訳が済んでいないドキュメントです。 -それらを翻訳して `site/en` 以下と同じ構成で `site/ja` 以下に配置して pull request を送ってください。 - -## Pull request title - -Pull request のタイトルは "JA: ..." とすることを推奨しています。 -レビュアーが "JA" で検索することもあるので、見落としを防ぐためご協力お願いします。 - -## Pull request size - -1 ファイル 1 pull request 程度の大きさに分けて pull request を作成することを推奨しています。 -これはレビュアーの負担を軽減して迅速にマージを行うためなので、ご協力お願いします。 - -## Proofreading tool - -一部レビューの自動化や多人数の翻訳による表記ゆれ対策として[校正ツール](https://github.com/tfug/proofreading)を使用しています。 -レビュー時にレビュアー側でも確認を行うので必須ではありませんが、予めチェックを行っておくと pull request の修正が少なく済みます。 - -また、校正ツールへの pull request も歓迎します。 - -## Frequently asked questions - -### ソースコードのコメントは翻訳した方が良いですか? - -基本的に翻訳する方針です。 - -### どのドキュメントを翻訳すれば良いですか? - -[TF 1.0 の翻訳はアップデートの終了が宣言された](https://groups.google.com/a/tensorflow.org/forum/#!msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ)ので TF 2.0 のドキュメントを翻訳してください。 -それ以外は特に制限はありません。 - -### 誰がどのドキュメントを翻訳中か確認できますか? - -正確に把握することは難しいので、必要に応じて次のように確認してください: - -* "JA" で検索して現在作成されている pull request を調べる -* 心配であれば Slack 上などで呼びかける - -### どうすればレビュアーになれますか? - -現在議論中ですが、次の条件を目安としています: - -* 既に 1 つドキュメントを翻訳して pull request が取り込まれている - -上記の条件を充たしており、レビュアーになりたい場合は [`REVIEWERS`](https://github.com/tensorflow/docs/blob/master/site/ja/REVIEWERS) に自分の GitHub ID を書き加えて pull request を送ってください。 - -### 英語ドキュメント側が外部リポジトリへのリンクになっている場合はどうすれば良いですか? - -リンク先のリポジトリと同じ構成で `site/ja` 以下に配置してください。 diff --git a/site/ja/README.md b/site/ja/README.md deleted file mode 100644 index 199c30d017c..00000000000 --- a/site/ja/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# コミュニティによる翻訳について - -これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 - -翻訳やレビューに参加して頂ける方は以下のコミュニティにご連絡ください: - -* Slack - * Slack の #docs_translation チャンネルで議論をしています - * [TensorFlow User Group の公式ページ](https://tfug.jp/)から参加可能です -* Google Groups - * [docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs) - * [docs-ja@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja) -* [Gitter](https://gitter.im/tensorflow/docs) - -また、翻訳を行う際には [CONTRIBUTING.md](CONTRIBUTING.md) をお読みください。 - -# Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Please focus translation efforts on -[TensorFlow 2](https://www.tensorflow.org) in the -[site/en/](https://github.com/tensorflow/docs/tree/master/site/en/) -directory. TF 1.x community docs will no longer be updated as we prepare for the -2.0 release. See -[the announcement](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). - -# Japanese translation guide - -Some technical words in English do not have a natural translation. Do *not* -translate the following words, use katakana otherwise: - -* (mini-) batch -* estimator -* eager execution -* label -* class -* helper -* hyperparameter -* optimizer -* one-hot encoding -* epoch -* callback -* sequence -* dictionary (in Python) -* embedding -* padding -* unit -* node -* neuron -* target -* import -* checkpoint -* compile -* dropout -* penalty -* scalar -* tensor -* decode -* tuple -* protocol buffer diff --git a/site/ja/REVIEWERS b/site/ja/REVIEWERS deleted file mode 100644 index 7401dabc510..00000000000 --- a/site/ja/REVIEWERS +++ /dev/null @@ -1,15 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/ja directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs-ja@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja -# -ohtaman -sfujiwara -masa-ita -AseiSugiyama -yukinagae -nuka137 -chie8842 -kiszk diff --git a/site/ja/guide/eager.ipynb b/site/ja/guide/eager.ipynb deleted file mode 100644 index 3b3cc2e517a..00000000000 --- a/site/ja/guide/eager.ipynb +++ /dev/null @@ -1,1250 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "eager.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCQY7jpBfMur" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "z6X9omPnfO_h", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2QQJJyDzqGRb" - }, - "source": [ - "# Eager Execution の基本" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B1xdylywqUSX" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0v-hhRA6Nrzt", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EGjDcGxIqEfX" - }, - "source": [ - "\n", - "\n", - "TensorFlow の Eager Execution は、計算グラフの作成と評価を同時におこなう命令的なプログラミングを行うための環境です：\n", - "オペレーションはあとで実行するための計算グラフでなく、具体的な計算結果の値を返します。\n", - "この方法を用いることにより、初心者にとって TensorFlow を始めやすくなり、またモデルのデバッグも行いやすくなります。\n", - "さらにコードの記述量も削減されます。\n", - "このガイドの内容を実行するためには、対話的インタープリタ `python` を起動し、以下のコードサンプルを実行してください。\n", - "\n", - "Eager Execution は研究や実験のための柔軟な機械学習環境として、以下を提供します。\n", - "\n", - "* *直感的なインタフェース*— Python のデータ構造を使用して、コードを自然に記述することができます。小規模なモデルとデータに対してすばやく実験を繰り返すことができます。\n", - "* *より簡単なデバッグ*— ops を直接呼び出すことで、実行中のモデルを調査したり、変更をテストすることができます。 Python 標準のデバッグツールを用いて即座にエラーのレポーティングができます。\n", - "* *自然な制御フロー*— TensorFlow のグラフ制御フローの代わりに Python の制御フローを利用するため、動的なモデルの作成をシンプルに行うことができます。\n", - " \n", - "Eager Execution は TensorFlow のほとんどのオペレーションとGPUアクセラレーションをサポートします。\n", - "\n", - "Note: いくつかのモデルは Eager Execution を有効化することでオーバヘッドが増える可能性があります。\n", - "パフォーマンス改善を行っていますが、もしも問題を発見したら、バグ報告してベンチマークを共有してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RBAeIwOMrYk8" - }, - "source": [ - "## セットアップと基本的な使い方" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ByNsp4VqqEfa", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x #gpu\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import cProfile" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "48P3-8q4qEfe" - }, - "source": [ - "TensorFlow 2.0 では、 Eager Execution はデフォルトで有効化されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "7aFsD8csqEff", - "colab": {} - }, - "source": [ - "tf.executing_eagerly()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x_G1zZT5qEfh" - }, - "source": [ - "これで TensorFlow のオペレーションを実行してみましょう。結果はすぐに返されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "9gsI54pbqEfj", - "colab": {} - }, - "source": [ - "x = [[2.]]\n", - "m = tf.matmul(x, x)\n", - "print(\"hello, {}\".format(m))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ajFn6qsdqEfl" - }, - "source": [ - "Eager Execution を有効化することで、 TensorFlow の挙動は変わります—TensorFlowは即座に式を評価して結果をPythonに返すようになります。\n", - "`tf.Tensor` オブジェクトは計算グラフのノードへのシンボリックハンドルの代わりに具体的な値を参照します。\n", - "セッションの中で構築して実行する計算グラフが存在しないため、`print（）`やデバッガを使って容易に結果を調べることができます。\n", - "勾配計算を遮ることなくテンソル値を評価、出力、およびチェックすることができます。\n", - "\n", - "Eager Execution は、[NumPy](http://www.numpy.org/)と一緒に使うことができます。\n", - "NumPy のオペレーションは、`tf.Tensor`を引数として受け取ることができます。\n", - "TensorFlow [math operations](https://www.tensorflow.org/api_guides/python/math_ops) はPython オブジェクトと Numpy array を `tf.Tensor` に変換します。\n", - "`tf.Tensor.numpy` メソッドはオブジェクトの値を NumPy の `ndarray` 形式で返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sTO0_5TYqz1n", - "colab": {} - }, - "source": [ - "a = tf.constant([[1, 2],\n", - " [3, 4]])\n", - "print(a)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Dp14YT8Gq4r1", - "colab": {} - }, - "source": [ - "# ブロードキャストのサポート\n", - "b = tf.add(a, 1)\n", - "print(b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "69p3waMfq8cQ", - "colab": {} - }, - "source": [ - "# オペレータのオーバーロードがサポートされている\n", - "print(a * b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Ui025t1qqEfm", - "colab": {} - }, - "source": [ - "# NumPy valueの使用\n", - "import numpy as np\n", - "\n", - "c = np.multiply(a, b)\n", - "print(c)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tq_aFRzWrCua", - "colab": {} - }, - "source": [ - "# Tensor から numpy の値を得る\n", - "print(a.numpy())\n", - "# => [[1 2]\n", - "# [3 4]]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H08f9ss9qEft" - }, - "source": [ - "## 動的な制御フロー\n", - "\n", - "Eager Execution の主要なメリットは、モデルを実行する際にホスト言語のすべての機能性が利用できることです。\n", - "たとえば、[fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz)が簡単に書けます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "0fudRMeUqEfu", - "colab": {} - }, - "source": [ - "def fizzbuzz(max_num):\n", - " counter = tf.constant(0)\n", - " max_num = tf.convert_to_tensor(max_num)\n", - " for num in range(1, max_num.numpy()+1):\n", - " num = tf.constant(num)\n", - " if int(num % 3) == 0 and int(num % 5) == 0:\n", - " print('FizzBuzz')\n", - " elif int(num % 3) == 0:\n", - " print('Fizz')\n", - " elif int(num % 5) == 0:\n", - " print('Buzz')\n", - " else:\n", - " print(num.numpy())\n", - " counter += 1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "P2cKknQWrJLB", - "colab": {} - }, - "source": [ - "fizzbuzz(15)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7kA-aC3BqEfy" - }, - "source": [ - "この関数はテンソル値に依存する条件式を持ち、実行時にこれらの値を表示します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8huKpuuAwICq" - }, - "source": [ - "## Eager Execution による学習" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mp2lCCZYrxHd" - }, - "source": [ - "### 勾配の計算\n", - "\n", - "[自動微分](https://en.wikipedia.org/wiki/Automatic_differentiation)はニューラルネットワークの学習で利用される[バックプロパゲーション](https://en.wikipedia.org/wiki/Backpropagation)などの機械学習アルゴリズムの実装を行う上で便利です。\n", - "Eager Executionでは、勾配計算をあとで行うためのオペレーションをトレースするために`tf.GradientTape` を利用します。\n", - "\n", - "Eager Execution では、学習や勾配計算に, `tf.GradientTape` を利用できます。これは複雑な学習ループを実行するときに特に役立ちます。\n", - "\n", - "各呼び出し中に異なるオペレーションが発生する可能性があるため、すべての forward-pass オペレーションは一つの「テープ」に記録されます。勾配を計算するには、テープを逆方向に再生してから破棄します。特定の `tf.GradientTape`は一つのグラデーションしか計算できません。後続の呼び出しは実行時エラーをスローします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "7g1yWiSXqEf-", - "colab": {} - }, - "source": [ - "w = tf.Variable([[1.0]])\n", - "with tf.GradientTape() as tape:\n", - " loss = w * w\n", - "\n", - "grad = tape.gradient(loss, w)\n", - "print(grad) # => tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkHs32GqweYS" - }, - "source": [ - "### モデル学習\n", - "\n", - "以下の example は MNIST という手書き数字分類を行うマルチレイヤーモデルを作成します。\n", - "Eager Execution 環境における学習可能なグラフを構築するためのオプティマイザーとレイヤーAPIを提示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "38kymXZowhhz", - "colab": {} - }, - "source": [ - "# mnist データのを取得し、フォーマットする\n", - "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n", - "\n", - "dataset = tf.data.Dataset.from_tensor_slices(\n", - " (tf.cast(mnist_images[...,tf.newaxis]/255, tf.float32),\n", - " tf.cast(mnist_labels,tf.int64)))\n", - "dataset = dataset.shuffle(1000).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rl1K8rOowmwT", - "colab": {} - }, - "source": [ - "# モデルを構築する\n", - "mnist_model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu',\n", - " input_shape=(None, None, 1)),\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(10)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fvyk-HgGwxwl" - }, - "source": [ - "\n", - "学習を行わずとも、モデルを呼び出して、 Eager Execution により、出力を検査することができます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BsxystjBwxLS", - "colab": {} - }, - "source": [ - "for images,labels in dataset.take(1):\n", - " print(\"Logits: \", mnist_model(images[0:1]).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y3PGa8G7qEgB" - }, - "source": [ - "keras モデルは組み込みで学習のループを回すメソッド `fit` がありますが、よりカスタマイズが必要な場合もあるでしょう。 Eager Executionを用いて実装された学習ループのサンプルを以下に示します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bzRhM7JDnaEG", - "colab": {} - }, - "source": [ - "optimizer = tf.keras.optimizers.Adam()\n", - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", - "\n", - "loss_history = []" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tXaupYXRI2YM" - }, - "source": [ - "Note: モデルの状況を確認したいときは、 `tf.debugging` にある assert 機能を利用してください。この機能は Eager Execution と Graph Execution のどちらでも利用できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DDHrigtiCIA4", - "colab": {} - }, - "source": [ - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " logits = mnist_model(images, training=True)\n", - " \n", - " # assertを入れて出力の型をチェックする。\n", - " tf.debugging.assert_equal(logits.shape, (32, 10))\n", - " \n", - " loss_value = loss_object(labels, logits)\n", - "\n", - " loss_history.append(loss_value.numpy().mean())\n", - " grads = tape.gradient(loss_value, mnist_model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, mnist_model.trainable_variables))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "0m1xAXrmqEgJ", - "colab": {} - }, - "source": [ - "def train():\n", - " for epoch in range(3):\n", - " for (batch, (images, labels)) in enumerate(dataset):\n", - " train_step(images, labels)\n", - " print ('Epoch {} finished'.format(epoch))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "C5dGz0p_nf4W", - "colab": {} - }, - "source": [ - "train()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5vG5ql_2vYB5", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.plot(loss_history)\n", - "plt.xlabel('Batch #')\n", - "plt.ylabel('Loss [entropy]')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKpOlHPLqEgl" - }, - "source": [ - "### Variablesとオプティマイザ\n", - "\n", - "`tf.Variable` オブジェクトは、学習中にアクセスされるミュータブルな `tf.Tensor` 値を格納し、自動微分を容易にします。\n", - "モデルのパラメータは、変数としてクラスにカプセル化できます。\n", - "\n", - "`tf.GradientTape` と共に `tf.Variable` を使うことでモデルパラメータはよりカプセル化されます。たとえば、上の\n", - "の自動微分の例は以下のように書き換えることができます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "nnQLBYmEqEgm", - "colab": {} - }, - "source": [ - "class Model(tf.keras.Model):\n", - " def __init__(self):\n", - " super(Model, self).__init__()\n", - " self.W = tf.Variable(5., name='weight')\n", - " self.B = tf.Variable(10., name='bias')\n", - " def call(self, inputs):\n", - " return inputs * self.W + self.B\n", - "\n", - "# 3 ＊ x + 2を近似するトイデータセット\n", - "NUM_EXAMPLES = 2000\n", - "training_inputs = tf.random.normal([NUM_EXAMPLES])\n", - "noise = tf.random.normal([NUM_EXAMPLES])\n", - "training_outputs = training_inputs * 3 + 2 + noise\n", - "\n", - "# 最適化対象のloss関数\n", - "def loss(model, inputs, targets):\n", - " error = model(inputs) - targets\n", - " return tf.reduce_mean(tf.square(error))\n", - "\n", - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return tape.gradient(loss_value, [model.W, model.B])\n", - "\n", - "# 定義：\n", - "# 1. モデル\n", - "# 2. モデルパラメータに関する損失関数の導関数\n", - "# 3. 導関数に基づいて変数を更新するストラテジ。\n", - "model = Model()\n", - "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)\n", - "\n", - "print(\"Initial loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", - "\n", - "# 学習ループ\n", - "for i in range(300):\n", - " grads = grad(model, training_inputs, training_outputs)\n", - " optimizer.apply_gradients(zip(grads, [model.W, model.B]))\n", - " if i % 20 == 0:\n", - " print(\"Loss at step {:03d}: {:.3f}\".format(i, loss(model, training_inputs, training_outputs)))\n", - "\n", - "print(\"Final loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", - "print(\"W = {}, B = {}\".format(model.W.numpy(), model.B.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rPjb8nRWqEgr" - }, - "source": [ - "## Eager Execution の途中でオブジェクトのステータスを使用する\n", - "\n", - "TF 1.x の Graph Execution では、プログラムの状態（Variableなど）は global collection に格納され、それらの存続期間は `tf.Session` オブジェクトによって管理されます。\n", - "対照的に、 Eager Execution の間、状態オブジェクトの存続期間は、対応する Python オブジェクトの存続期間によって決定されます。\n", - "\n", - "### 変数とオブジェクト\n", - "\n", - "Eager Execution の間、変数はオブジェクトへの最後の参照が削除され、その後削除されるまで存続します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "A2boS674qEgs", - "colab": {} - }, - "source": [ - "if tf.test.is_gpu_available():\n", - " with tf.device(\"gpu:0\"):\n", - " v = tf.Variable(tf.random.normal([1000, 1000]))\n", - " v = None # v は GPU メモリを利用しなくなる" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "scMjg6L6qEgv" - }, - "source": [ - "### オブジェクトベースの保存\n", - "\n", - "このセクションは、[チェックポイントの学習の手引き]（./checkpoint.ipynb） の省略版です。\n", - "\n", - "`tf.train.Checkpoint` はチェックポイントを用いて `tf.Variable` を保存および復元することができます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7z5xRfdHzZOQ", - "colab": {} - }, - "source": [ - "x = tf.Variable(10.)\n", - "checkpoint = tf.train.Checkpoint(x=x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IffrUVG7zyVb", - "colab": {} - }, - "source": [ - "x.assign(2.) # 新しい値を変数に代入して保存する。\n", - "checkpoint_path = './ckpt/'\n", - "checkpoint.save('./ckpt/')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "eMT9koCoqEgw", - "colab": {} - }, - "source": [ - "x.assign(11.) # 保存後に変数の値を変える。\n", - "\n", - "# チェックポイントから変数を復元する\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_path))\n", - "\n", - "print(x) # => 2.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbFnP-yLqEgx" - }, - "source": [ - "モデルを保存して読み込むために、 `tf.train.Checkpoint` は隠れ変数なしにオブジェクトの内部状態を保存します。 `モデル`、 `オプティマイザ` 、そしてグローバルステップの状態を記録するには、それらを `tf.train.Checkpoint` に渡します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "hWZHyAXMqEg0", - "colab": {} - }, - "source": [ - "import os\n", - "\n", - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(10)\n", - "])\n", - "optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n", - "checkpoint_dir = 'path/to/model_dir'\n", - "if not os.path.exists(checkpoint_dir):\n", - " os.makedirs(checkpoint_dir)\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "root = tf.train.Checkpoint(optimizer=optimizer,\n", - " model=model)\n", - "\n", - "root.save(checkpoint_prefix)\n", - "root.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R-ITwkBCF6GJ" - }, - "source": [ - "多くの学習ループでは、変数は `tf.train..Checkpoint.restore` が呼ばれたあとに作成されます。これらの変数は作成されてすぐに復元され、チェックポイントがすべてロードされたことを確認するためのアサーションが利用可能になります。詳しくは、 [guide to training checkpoints](./checkpoint.ipynb) を見てください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3yoD0VJ7qEg3" - }, - "source": [ - "### オブジェクト指向メトリクス\n", - "\n", - "`tfe.keras.metrics`はオブジェクトとして保存されます。新しいデータを呼び出し可能オブジェクトに渡してメトリクスを更新し、 `tfe.keras.metrics.result`メソッドを使って結果を取得します。次に例を示します:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "9ccu0iAaqEg5", - "colab": {} - }, - "source": [ - "m = tf.keras.metrics.Mean(\"loss\")\n", - "m(0)\n", - "m(5)\n", - "m.result() # => 2.5\n", - "m([8, 9])\n", - "m.result() # => 5.5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xEL4yJe5qEhD" - }, - "source": [ - "## 高度な自動微分トピック\n", - "\n", - "### 動的なモデル\n", - "\n", - "`tf.GradientTape` は動的モデルでも使うことができます。 \n", - "以下の [バックトラックライン検索](https://wikipedia.org/wiki/Backtracking_line_search)\n", - "アルゴリズムの例は、複雑な制御フローにもかかわらず\n", - "勾配があり、微分可能であることを除いて、通常の NumPy コードのように見えます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "L518n5dkqEhE", - "colab": {} - }, - "source": [ - "def line_search_step(fn, init_x, rate=1.0):\n", - " with tf.GradientTape() as tape:\n", - " # 変数は自動的に記録されるが、Tensorは手動でウォッチする\n", - " tape.watch(init_x)\n", - " value = fn(init_x)\n", - " grad = tape.gradient(value, init_x)\n", - " grad_norm = tf.reduce_sum(grad * grad)\n", - " init_value = value\n", - " while value > init_value - rate * grad_norm:\n", - " x = init_x - rate * grad\n", - " value = fn(x)\n", - " rate /= 2.0\n", - " return x, value" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gieGOf_DqEhK" - }, - "source": [ - "### カスタム勾配\n", - "\n", - "カスタム勾配は、勾配を上書きする簡単な方法です。 フォワード関数では、\n", - "入力、出力、または中間結果に関する勾配を定義します。たとえば、逆方向パスにおいて勾配のノルムを制限する簡単な方法は次のとおりです：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "-OwwsWUAqEhK", - "colab": {} - }, - "source": [ - "@tf.custom_gradient\n", - "def clip_gradient_by_norm(x, norm):\n", - " y = tf.identity(x)\n", - " def grad_fn(dresult):\n", - " return [tf.clip_by_norm(dresult, norm), None]\n", - " return y, grad_fn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JPLDHkF_qEhN" - }, - "source": [ - "カスタム勾配は、一連の演算に対して数値的に安定した勾配を提供するために共通的に使用されます。:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "24WiLROnqEhO", - "colab": {} - }, - "source": [ - "def log1pexp(x):\n", - " return tf.math.log(1 + tf.exp(x))\n", - "\n", - "def grad_log1pexp(x):\n", - " with tf.GradientTape() as tape:\n", - " tape.watch(x)\n", - " value = log1pexp(x)\n", - " return tape.gradient(value, x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "n8fq69r9-B-c", - "colab": {} - }, - "source": [ - "# 勾配計算は x = 0 のときはうまくいく。\n", - "grad_log1pexp(tf.constant(0.)).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_VFSU0mG-FSp", - "colab": {} - }, - "source": [ - "# しかし、x = 100のときは数値的不安定により失敗する。\n", - "grad_log1pexp(tf.constant(100.)).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-VcTR34rqEhQ" - }, - "source": [ - "ここで、 `log1pexp` 関数はカスタム勾配を用いて解析的に単純化することができます。\n", - "以下の実装は、フォワードパスの間に計算された `tf.exp（x）` の値を\n", - "再利用します—冗長な計算を排除することでより効率的になります：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Q7nvfx_-qEhS", - "colab": {} - }, - "source": [ - "@tf.custom_gradient\n", - "def log1pexp(x):\n", - " e = tf.exp(x)\n", - " def grad(dy):\n", - " return dy * (1 - 1 / (1 + e))\n", - " return tf.math.log(1 + e), grad\n", - "\n", - "def grad_log1pexp(x):\n", - " with tf.GradientTape() as tape:\n", - " tape.watch(x)\n", - " value = log1pexp(x)\n", - " return tape.gradient(value, x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5gHPKMfl-Kge", - "colab": {} - }, - "source": [ - "# 上と同様に、勾配計算はx = 0のときにはうまくいきます。\n", - "grad_log1pexp(tf.constant(0.)).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u38MOfz3-MDE", - "colab": {} - }, - "source": [ - "# また、勾配計算はx = 100でも機能します。\n", - "grad_log1pexp(tf.constant(100.)).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rnZXjfQzqEhV" - }, - "source": [ - "## パフォーマンス\n", - "\n", - "Eager Executionの間、計算は自動的にGPUにオフロードされます。計算を実行するデバイスを指定したい場合は、\n", - "`tf.device（ '/ gpu：0'）` ブロック（もしくはCPUを指定するブロック）で囲むことで指定できます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Ac9Y64H-qEhX", - "colab": {} - }, - "source": [ - "import time\n", - "\n", - "def measure(x, steps):\n", - " # TensorFlowはGPUを初めて使用するときに初期化するため、時間計測対象からは除外する。\n", - " tf.matmul(x, x)\n", - " start = time.time()\n", - " for i in range(steps):\n", - " x = tf.matmul(x, x)\n", - " # ｔｆ．ｍａｔｍｕｌは、行列乗算が完了する前に戻ることができる。\n", - " # （たとえば、ＣＵＤＡストリームにオペレーションをエンキューした後に戻すことができる）。\n", - " # 以下のx.numpy（）呼び出しは、すべてのキューに入れられたオペレーションが完了したことを確認する。\n", - " # （そして結果をホストメモリにコピーするため、計算時間は単純なmatmulオペレーションよりも多くのことを含む時間になる。）\n", - " _ = x.numpy()\n", - " end = time.time()\n", - " return end - start\n", - "\n", - "shape = (1000, 1000)\n", - "steps = 200\n", - "print(\"Time to multiply a {} matrix by itself {} times:\".format(shape, steps))\n", - "\n", - "# CPU上で実行するとき：\n", - "with tf.device(\"/cpu:0\"):\n", - " print(\"CPU: {} secs\".format(measure(tf.random.normal(shape), steps)))\n", - "\n", - "# GPU上で実行するとき（GPUが利用できれば）：\n", - "if tf.test.is_gpu_available():\n", - " with tf.device(\"/gpu:0\"):\n", - " print(\"GPU: {} secs\".format(measure(tf.random.normal(shape), steps)))\n", - "else:\n", - " print(\"GPU: not found\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RLw3IS7UqEhe" - }, - "source": [ - "`tf.Tensor` オブジェクトはそのオブジェクトに対するオペレーションを実行するために別のデバイスにコピーすることができます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "ny6LX2BVqEhf", - "colab": {} - }, - "source": [ - "if tf.test.is_gpu_available():\n", - " x = tf.random.normal([10, 10])\n", - "\n", - " x_gpu0 = x.gpu()\n", - " x_cpu = x.cpu()\n", - "\n", - " _ = tf.matmul(x_cpu, x_cpu) # CPU上で実行するとき\n", - " _ = tf.matmul(x_gpu0, x_gpu0) # GPU:0上で実行するとき" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oA_qaII3-p6c" - }, - "source": [ - "### ベンチマーク\n", - "\n", - "GPUでの\n", - "[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)\n", - "の学習のような、計算量の多いモデルの場合は、Eager Executionのパフォーマンスは `tf.function` のパフォーマンスに匹敵します。\n", - "しかし、この2つの環境下のパフォーマンスの違いは計算量の少ないモデルではより大きくなり、小さなたくさんのオペレーションからなるモデルでホットコードパスを最適化するためにやるべきことがあります。\n", - "\n", - "## functionsの利用\n", - "\n", - "Eager Execution は開発とデバッグをより対話的にしますが、\n", - "TensorFlow 1.x スタイルの Graph Execution は分散学習、パフォーマンスの最適化、そしてプロダクション環境へのデプロイの観点で利点があります。\n", - "\n", - "2つの手法のギャップを埋めるために、 TensorFlow 2.0 は `ｔｆ．function` という機能を導入しています。\n", - "詳しくは、 [Autograph](./function.ipynb) のガイドを見てください。\n" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/guide/function.ipynb b/site/ja/guide/function.ipynb deleted file mode 100644 index 5ffce8531a4..00000000000 --- a/site/ja/guide/function.ipynb +++ /dev/null @@ -1,734 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "function.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "Jxv6goXm7oGF" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Jxv6goXm7oGF" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\");" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "llMNufAK7nfK", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\"); { display-mode: \"form\" }\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8Byow2J6LaPl" - }, - "source": [ - "# TensorFlow 2.0 での tf.function と AutoGraph" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kGXS3UWBBNoc" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TPrI2hpDnNgG", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CydFK2CL7ZHA" - }, - "source": [ - "TensorFlow 2.0 では Eager Execution の使いやすさとTensorFlow 1.0 のパワーとを同時に提供します。この統合の中核となるのは `tf.function` です。これは Python の構文のサブセットを移植可能でハイパフォーマンスな TensorFlow のグラフに変換します。\n", - "\n", - "`tf.function`の魅力的な特徴に AutoGraph があります。これはグラフを Python の構文そのものを用いて記述できるようにします。\n", - "AutoGraph で利用可能な Python の機能の一覧は、[AutoGraph Capabilities and Limitations (Autograph の性能と制限事項)](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/limitations.md) で確認できます。また、`tf.function`の詳細については RFC [TF 2.0: Functions, not Sessions](https://github.com/tensorflow/community/blob/master/rfcs/20180918-functions-not-sessions-20.md) を参照してください。AutoGraph の詳細については `tf.autograph` を参照してください。\n", - "\n", - "このチュートリアルでは `tf.function` と AutoGraph の基本的な特徴についてひととおり確認します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n4EKOpw9mObL" - }, - "source": [ - "## セットアップ\n", - "\n", - "TensorFlow 2.0 をインポートして、TF 2.0 モードを有効にしてください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "V9oECvVSI1Kj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "import numpy as np" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mT7meGqrZTz9", - "colab": {} - }, - "source": [ - "try:\n", - " !pip install tf-nightly-2.0-preview\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "77AsVr1GGtBP" - }, - "source": [ - "## `tf.function` デコレータ\n", - "\n", - "`tf.function`を用いてある関数にアノテーションを付けたとしても、一般の関数と変わらずに呼び出せます。一方、実行時にはその関数はグラフへとコンパイルされます。これにより、より高速な実行や、 GPU や TPU での実行、SavedModel へのエクスポートといった利点が得られます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FhIg7-z6HNWj", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def simple_nn_layer(x, y):\n", - " return tf.nn.relu(tf.matmul(x, y))\n", - "\n", - "\n", - "x = tf.random.uniform((3, 3))\n", - "y = tf.random.uniform((3, 3))\n", - "\n", - "simple_nn_layer(x, y)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U-LAE4pMNR9g" - }, - "source": [ - "アノテーションの結果を調べてみると、 TensorFlow ランタイムとのやり取りのすべてを処理する特別な呼び出し可能オブジェクトを確認できます。 " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q4t2iuS7Nqc0", - "colab": {} - }, - "source": [ - "simple_nn_layer" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DqeefLGNXjZQ" - }, - "source": [ - "記述したコードで複数の関数を利用していたとしても、すべての関数にアノテーションを付ける必要はありません。アノテーションをつけた関数から呼び出されるすべての関数は、グラフモードで実行されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3VGF7tlVXiZY", - "colab": {} - }, - "source": [ - "def linear_layer(x):\n", - " return 2 * x + 1\n", - "\n", - "\n", - "@tf.function\n", - "def deep_net(x):\n", - " return tf.nn.relu(linear_layer(x))\n", - "\n", - "\n", - "deep_net(tf.constant((1, 2, 3)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yQvg6ZSKWyqE" - }, - "source": [ - "グラフが大量の軽量な演算から構成される場合、関数は Eager Execution で実行するコードよりも高速になる場合があります。しかし、 graph が少量の (畳み込み演算のような) 計算に時間のかかる演算からなる場合、高速化はそれほど見込めないでしょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0EL6lVwEWuFo", - "colab": {} - }, - "source": [ - "import timeit\n", - "conv_layer = tf.keras.layers.Conv2D(100, 3)\n", - "\n", - "@tf.function\n", - "def conv_fn(image):\n", - " return conv_layer(image)\n", - "\n", - "image = tf.zeros([1, 200, 200, 100])\n", - "# warm up\n", - "conv_layer(image); conv_fn(image)\n", - "print(\"Eager conv:\", timeit.timeit(lambda: conv_layer(image), number=10))\n", - "print(\"Function conv:\", timeit.timeit(lambda: conv_fn(image), number=10))\n", - "print(\"Note how there's not much difference in performance for convolutions\")\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "L4zj-jpH0jKH", - "colab": {} - }, - "source": [ - "lstm_cell = tf.keras.layers.LSTMCell(10)\n", - "\n", - "@tf.function\n", - "def lstm_fn(input, state):\n", - " return lstm_cell(input, state)\n", - "\n", - "input = tf.zeros([10, 10])\n", - "state = [tf.zeros([10, 10])] * 2\n", - "# warm up\n", - "lstm_cell(input, state); lstm_fn(input, state)\n", - "print(\"eager lstm:\", timeit.timeit(lambda: lstm_cell(input, state), number=10))\n", - "print(\"function lstm:\", timeit.timeit(lambda: lstm_fn(input, state), number=10))\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ohbSnA79mcJV" - }, - "source": [ - "## Python の制御フローの利用\n", - "\n", - "`tf.function`の内部でデータに依存した制御フローを用いる場合、Pythonの制御フロー構文を用いることができます。AutoGraph はそれらの構文を TensorFlow の Ops に書き換えます。たとえば、 `Tensor` に依存する `if` 文は、`tf.cond()` に変換されます。\n", - "\n", - "次の例では `x` は `Tensor` です。ですが、 `if` 文は期待するどおりに動作しています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aA3gOodCBkOw", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def square_if_positive(x):\n", - " if x > 0:\n", - " x = x * x\n", - " else:\n", - " x = 0\n", - " return x\n", - "\n", - "\n", - "print('square_if_positive(2) = {}'.format(square_if_positive(tf.constant(2))))\n", - "print('square_if_positive(-2) = {}'.format(square_if_positive(tf.constant(-2))))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GMiCUkdyoq98" - }, - "source": [ - "Note: この例ではスカラー値を用いた単純な条件を用いています。実利用する場合、典型的には    バッチ処理が用いられます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m-jWmsCmByyw" - }, - "source": [ - "AutoGraph は `while`, `for`, `if`, `break`, `continue`, `return` といった典型的なPythonの構文をサポートしています。また、これらを入れ子にして利用する場合もサポートしています。つまり、`Tensor`を返す式を`while` 文や `if` 文の条件式として用いることが可能です。また、`for` 文で `Tensor` の要素に渡って反復することも可能です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "toxKBOXbB1ro", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def sum_even(items):\n", - " s = 0\n", - " for c in items:\n", - " if c % 2 > 0:\n", - " print(c)\n", - " continue\n", - " s += c\n", - " return s\n", - "\n", - "\n", - "sum_even(tf.constant([10, 12, 15, 20]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AtDaLrbySw4j" - }, - "source": [ - "より高度な使い方をするユーザーのために、AutoGraph は低レベルAPIも提供しています。次の例では AutoGraph が生成したコードを確認できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aRsde3x_SjTQ", - "colab": {} - }, - "source": [ - "print(tf.autograph.to_code(sum_even.python_function))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rvJXCfk8VkLf" - }, - "source": [ - "次はより複雑な制御フローの例です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h-Z87IJqVlKl", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def fizzbuzz(n):\n", - " msg = tf.constant('')\n", - " for i in tf.range(n):\n", - " if i % 3 == 0:\n", - " tf.print('Fizz')\n", - " elif i % 5 == 0:\n", - " tf.print('Buzz')\n", - " else:\n", - " tf.print(i)\n", - "\n", - "fizzbuzz(tf.constant(15))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "h_Y4uC1R1B55" - }, - "source": [ - "## Keras での AutoGraph の利用\n", - "\n", - "`tf.function` はオブジェクトのメソッドに対しても利用できます。たとえば、カスタムしたKeras モデルにデコレーターを適用できます、典型的には `call` 関数にアノテーションを付けることで実現できるでしょう。より詳細が必要な場合、`tf.keras` を確認してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cR6mpLKP1HLe", - "colab": {} - }, - "source": [ - "class CustomModel(tf.keras.models.Model):\n", - "\n", - " @tf.function\n", - " def call(self, input_data):\n", - " if tf.reduce_mean(input_data) > 0:\n", - " return input_data\n", - " else:\n", - " return input_data // 2\n", - "\n", - "\n", - "model = CustomModel()\n", - "\n", - "model(tf.constant([-2, -4]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NTEvpBK9f8kj" - }, - "source": [ - "## 副作用\n", - "\n", - "Eager モードのように、通常の場合 `tf.function` の中で、`tf.assign` や `tf.print` といった副作用のある命令を実行できます。また、実行時の順序を保つために、処理順について必要な依存関係を書き加えます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-Wd6i8S9gcuC", - "colab": {} - }, - "source": [ - "v = tf.Variable(5)\n", - "\n", - "@tf.function\n", - "def find_next_odd():\n", - " v.assign(v + 1)\n", - " if v % 2 == 0:\n", - " v.assign(v + 1)\n", - "\n", - "\n", - "find_next_odd()\n", - "v" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4LfnJjm0Bm0B" - }, - "source": [ - "## 例: シンプルなモデルの学習\n", - "\n", - "AutoGraph はこれまで見てきたよりもずっと多くの演算を TensorFlow の内部で実行できます。たとえば、学習のためのループ処理は単に制御フローなので、実際にそれを TensorFlow に持ち込んで処理できます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Em5dzSUOtLRP" - }, - "source": [ - "### データのダウンロード" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xqoxumv0ssQW", - "colab": {} - }, - "source": [ - "def prepare_mnist_features_and_labels(x, y):\n", - " x = tf.cast(x, tf.float32) / 255.0\n", - " y = tf.cast(y, tf.int64)\n", - " return x, y\n", - "\n", - "def mnist_dataset():\n", - " (x, y), _ = tf.keras.datasets.mnist.load_data()\n", - " ds = tf.data.Dataset.from_tensor_slices((x, y))\n", - " ds = ds.map(prepare_mnist_features_and_labels)\n", - " ds = ds.take(20000).shuffle(20000).batch(100)\n", - " return ds\n", - "\n", - "train_dataset = mnist_dataset()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "znmy4l8ntMvW" - }, - "source": [ - "### モデルの定義" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ltxyJVWTqNAO", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential((\n", - " tf.keras.layers.Reshape(target_shape=(28 * 28,), input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(10)))\n", - "model.build()\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oeYV6mKnJGMr" - }, - "source": [ - "### 学習のためのループ処理の定義" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3xtg_MMhJETd", - "colab": {} - }, - "source": [ - "compute_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", - "\n", - "compute_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n", - "\n", - "\n", - "def train_one_step(model, optimizer, x, y):\n", - " with tf.GradientTape() as tape:\n", - " logits = model(x)\n", - " loss = compute_loss(y, logits)\n", - "\n", - " grads = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " compute_accuracy(y, logits)\n", - " return loss\n", - "\n", - "\n", - "@tf.function\n", - "def train(model, optimizer):\n", - " train_ds = mnist_dataset()\n", - " step = 0\n", - " loss = 0.0\n", - " accuracy = 0.0\n", - " for x, y in train_ds:\n", - " step += 1\n", - " loss = train_one_step(model, optimizer, x, y)\n", - " if step % 10 == 0:\n", - " tf.print('Step', step, ': loss', loss, '; accuracy', compute_accuracy.result())\n", - " return step, loss, accuracy\n", - "\n", - "step, loss, accuracy = train(model, optimizer)\n", - "print('Final step', step, ': loss', loss, '; accuracy', compute_accuracy.result())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SnsumiP6eRYL" - }, - "source": [ - "## バッチ処理\n", - "\n", - "実際のアプリケーションにおいて、処理をバッチにまとめることはパフォーマンスの観点から重要です。AutoGraphを用いるのにもっとも適しているコードは、制御フローを _バッチ_ の単位で決定するようなコードです。もし、個々の _要素_ の単位で制御を決定する場合、パフォーマンスを保つために batch API を試してみてください。\n", - "\n", - "一例として、次の Python コードががあったとします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t31QoERiNccJ", - "colab": {} - }, - "source": [ - "def square_if_positive(x):\n", - " return [i ** 2 if i > 0 else i for i in x]\n", - "\n", - "\n", - "square_if_positive(range(-5, 5))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kSeEJ76uNgwD" - }, - "source": [ - "TensorFlowに同等の処理を行わせる場合、次のように記述したくなるかもしれません。 (これは実際には動作します!)" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RqR8WzSzNf87", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def square_if_positive_naive(x):\n", - " result = tf.TensorArray(tf.int32, size=x.shape[0])\n", - " for i in tf.range(x.shape[0]):\n", - " if x[i] > 0:\n", - " result = result.write(i, x[i] ** 2)\n", - " else:\n", - " result = result.write(i, x[i])\n", - " return result.stack()\n", - "\n", - "\n", - "square_if_positive_naive(tf.range(-5, 5))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gTcyWXVGN3gS" - }, - "source": [ - "しかし、この場合、次のように書くこともできます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VO2f6x-lNfVj", - "colab": {} - }, - "source": [ - "def square_if_positive_vectorized(x):\n", - " return tf.where(x > 0, x ** 2, x)\n", - "\n", - "\n", - "square_if_positive_vectorized(tf.range(-5, 5))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ja/js/tutorials/index.md b/site/ja/js/tutorials/index.md deleted file mode 100644 index 6e44c62a86a..00000000000 --- a/site/ja/js/tutorials/index.md +++ /dev/null @@ -1,58 +0,0 @@ -# はじめに - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -TensorFlow.jsは ブラウザとNode.js内で 機械学習モデルの訓練とデプロイを行うための JavaScriptライブラリです。 - -さまざまな利用目的に合わせて以下のセクションを参照してください。 - - -## Tensorを直接使用せずにMLプログラムをコーディングする - -TensorやOptimizerのような低レベルな詳細については考えずに機械学習を始めたい？ - -TensorFlow.jsを土台として作成されたml5.jsライブラリを利用すると、簡潔でわかりやすいAPIを通じてブラウザ上で機械学習のアルゴリズムとモデルを使用できます。 - -ml5.jsを取得 - - -## TensorFlow.jsを準備する - -テンソルやレイヤー、オプティマイザ、損失関数などの概念に馴染みがある？（もしくはそれらに詳しくなりたい？） TensorFlow.jsでは -JavaScriptでニューラルネットワークプログラミングを行うための柔軟な構成要素が利用できます。 - -ブラウザやNode.jsでTensorFlow.jsのコードを用意して使用する方法については以下を参照してください。 - -準備 - -### 学習済みモデルをTensorFlow.js用に変換する - -Pythonの 学習済みモデルをTensorFlow.jsに変換する方法は以下を参照してください。 - -Kerasモデル -GraphDefモデル - -## TensorFlow.jsの既存コードを元にして学ぶ - -tfjs-examplesにTensorFlow.jsを使用してさまざまなMLタスクを実装した小さなコード例があります。 - -GitHubで見る - -## 自身のTensorFlow.jsモデルの振る舞いを可視化する - -tfjs-visはブラウザ上で可視化を行う小さなライブラリで、TensorFlow.jsと合わせて利用するために作られています。 - -GitHubで見る -デモを表示 - -## 自身のデータを TensorFlow.jsで処理できるように準備する - -TensorFlow.jsにはMLのベストプラクティスを使用してデータを処理するためのサポートがあります。 - -ドキュメントを見る diff --git a/site/ja/js/tutorials/learning/ml.md b/site/ja/js/tutorials/learning/ml.md deleted file mode 100644 index beeb60e9fac..00000000000 --- a/site/ja/js/tutorials/learning/ml.md +++ /dev/null @@ -1,60 +0,0 @@ -# 機械学習資料 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -TensorFlow.jsで機械学習を行うには、特に機械学習やニューラルネットワーク、JavaScript、そしてNode.JSまたはブラウザベース開発などのさまざまなドメインの知識が必要です。あなたのバックグラウンドによっては、それらの領域のいくつかに馴染みがにないということもあるでしょう。ウェブ上にはすばらしい学習資料が数多くあります。このページではニューラルネットワークを使用した機械学習の知識獲得の第一歩となる資料をいくつか紹介します。 - -## 高度な紹介 - -深層学習とTensorFlow.jsに関する高度な説明が必要なら次の動画をおすすめします。 - -- [But what is a Neural Network?](https://www.youtube.com/watch?v=aircAruvnKk) - by 3blue1brown -- [Deep Learning in JS](https://www.youtube.com/watch?v=SV-cgdobtTA) by Ashi - Krishnan - -## TensorFlow.js中心の説明 - -次の動画資料はTensorFlow.jsが中心で、機械学習の初心者も対象にしています。 - -- [TensorFlow.js](https://www.youtube.com/playlist?list=PLs6AluHXaQnjeI6jzDkpKXvbPj31i4GgF) - by TensorFlow -- [TensorFlow.js: Intelligence and Learning Series](https://www.youtube.com/playlist?list=PLRqwX-V7Uu6YIeVA3dNxbR9PYj4wV31oQ) - by Coding Train -- [TensorFlow.js Deep Learning with JavaScript](https://www.youtube.com/playlist?list=PLZbbT5o_s2xr83l8w44N_g3pygvajLrJ-) - by Deeplizard - -## 包括的なコース - -深層学習もカバーした包括的なオンラインコースがあります。ほとんどのコースは説明のためにPythonを主に使用しています。しかし構文は違いますが、基本的な考え方はTensorFlow.jsの利用にも適用できます。 - -- [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/ml-intro) - by Google -- [Deep Learning Specialization](https://www.coursera.org/specializations/deep-learning) - by Coursera - -## 包括的な書籍 - -- [Neural Networks and Deep Learning](http://neuralnetworksanddeeplearning.com/)（邦訳「[ニューラルネットワークと深層学習](https://nnadl-ja.github.io/nnadl_site_ja/)」） - by Michael Nielsen -- [Deep Learning with Python](https://www.manning.com/books/deep-learning-with-python) - （邦訳「[PythonとKerasによるディープラーニング](https://book.mynavi.jp/ec/products/detail/id=90124)」）by - Francois Chollet - -## 数学的な概念 - -機械学習は数学が重要な分野です。単に機械学習モデルを利用するだけであれば数学を理解する必要はありませんが、機械学習モデルを修正したりはじめから新しく構築する予定があるなら、基礎となる数学的な概念に馴染んでおくことは意味があります。事前に数学的なことをすべて理解しておく必要はありませんが、それでも遭遇するかもしれない馴染みのない概念を先に見ておくことはできるでしょう。 - -- [Essence of Linear Algebra](https://www.youtube.com/playlist?list=PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab) - by 3blue1brown -- [Essence of Calculus](https://www.youtube.com/playlist?list=PLZHQObOWTQDMsr9K-rj53DwVRMYO3t5Yr) - by 3blue1brown -- [Linear Algebra](https://www.khanacademy.org/math/linear-algebra) by Khan - Academy -- [Calculus](https://www.khanacademy.org/math/calculus-home) by Khan Academy diff --git a/site/ja/js/tutorials/setup.md b/site/ja/js/tutorials/setup.md deleted file mode 100644 index 2cf031276de..00000000000 --- a/site/ja/js/tutorials/setup.md +++ /dev/null @@ -1,162 +0,0 @@ -# 準備 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -## ブラウザの準備 - -ブラウザベースのプロジェクトでTensorFlow.jsを利用する方法は主に2つあります。 - -- [scriptタグ](https://developer.mozilla.org/en-US/docs/Learn/HTML/Howto/Use_JavaScript_within_a_webpage)を使用する -- [NPM](https://www.npmjs.com)からインストールし、[Parcel](https://parceljs.org/)や[WebPack](https://webpack.js.org/)、[Rollup](https://rollupjs.org/guide/en)などのビルドツールを使用する - -ウェブ開発に馴染みがなかったり、webpackやparcelなどのツールについて聞いたことがなければ、_scriptタグを使用することをおすすめします。_経験が豊富だったり、より大きなプログラムを書こうとしている場合には、ビルドツールの利用を試みる価値があるでしょう。 - -### scriptタグから使用する - -以下のscriptタグをメインのHTMLファイルに追加してください。 - -```html - -``` - - - -### NPMからインストール - -[npm cli](https://docs.npmjs.com/cli/npm)ツールと[yarn](https://yarnpkg.com/en/)のどちらでもTensorFlow.jsをインストールできます。 - -``` -yarn add @tensorflow/tfjs -``` - -_または_ - -``` -npm install @tensorflow/tfjs -``` - - - - -## Node.jsの準備 - -[npm cli](https://docs.npmjs.com/cli/npm)ツールと[yarn](https://yarnpkg.com/en/)のどちらでもTensorFlow.jsをインストールできます。 - -**オプション1:** ネイティブC++バインディングの付属するTensorFlow.jsをインストール - -``` -yarn add @tensorflow/tfjs-node -``` - -_または_ - -``` -npm install @tensorflow/tfjs-node -``` - -**オプション2:** (Linuxのみ) -システムに[CUDA support](https://www.tensorflow.org/install/install_linux#NVIDIARequirements)のあるNVIDIA® -GPUがあれば、より高いパフォーマンスを実現するためにGPUパッケージを使用 - -``` -yarn add @tensorflow/tfjs-node-gpu -``` - -_または_ - -``` -npm install @tensorflow/tfjs-node-gpu -``` - -**オプション3:** ピュアJavaScriptバージョンをインストール。パフォーマンスの観点からは最も遅い選択肢です。 - -``` -yarn add @tensorflow/tfjs -``` - -_または_ - -``` -npm install @tensorflow/tfjs -``` - - - - -### TypeScript - -TypeScriptを使用していて、プロジェクトで厳密なnullチェックを使用していると`tsconfig.json`ファイルで`skipLibCheck: -true`を設定する必要があるでしょう。そうしなければコンパイル中にエラーが発生します。 diff --git a/site/ja/js/tutorials/training/handwritten_digit_cnn.md b/site/ja/js/tutorials/training/handwritten_digit_cnn.md deleted file mode 100644 index be2bce25d91..00000000000 --- a/site/ja/js/tutorials/training/handwritten_digit_cnn.md +++ /dev/null @@ -1,13 +0,0 @@ -# CNNを使用した手書き数字認識 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -このチュートリアルでは、畳み込みニューラルネットワークを使用して手書きの数字を認識するTensorFlow.jsモデルを構築します。はじめに、数千の手書きの数字とそのラベルを「見せて」分類器を訓練します。次にモデルがまだ見たことのないテストデータを使用して分類器の精度を評価します。 - -CodeLabを開く diff --git a/site/ja/js/tutorials/training/linear_regression.md b/site/ja/js/tutorials/training/linear_regression.md deleted file mode 100644 index 4ed564c320b..00000000000 --- a/site/ja/js/tutorials/training/linear_regression.md +++ /dev/null @@ -1,15 +0,0 @@ -# 2次元データを予測する - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -このチュートリアルでは、一連の自動車を表現した数値データを予測するモデルを訓練します。 - -この演習は様々な種類のモデルの訓練に共通する手順を紹介するものですが、使用するのは小さなデータセットと単純な（浅い）モデルです。主な目的はTensorFlow.jsでモデルを訓練する際に使用する基本的な用語や概念、構文に慣れる手助けをし、さらに調査と学習を進める足がかりを提供することです。 - -CodeLabを開く diff --git a/site/ja/js/tutorials/transfer/audio_recognizer.md b/site/ja/js/tutorials/transfer/audio_recognizer.md deleted file mode 100644 index 68da45075a3..00000000000 --- a/site/ja/js/tutorials/transfer/audio_recognizer.md +++ /dev/null @@ -1,15 +0,0 @@ -# 音声識別器の転移学習 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -このチュートリアルでは、TensorFlow.jsを使用してブラウザ上で学習し、独自の音声分類器を構築する方法について学びます。 - -転移学習を使用して、比較的少ない学習データで短い音を分類するモデルを作成します。ここでは[音声コマンド認識](https://github.com/tensorflow/tfjs-models/tree/master/speech-commands)のための学習済みモデルを使用します。このモデルを土台として新しいモデルを訓練して独自の音声クラスを認識します。 - -このチュートリアルはコードラボとして提供されます。[次のリンクに従ってコードラボを開いてください。](https://codelabs.developers.google.com/codelabs/tensorflowjs-audio-codelab/index.html) diff --git a/site/ja/js/tutorials/transfer/image_classification.md b/site/ja/js/tutorials/transfer/image_classification.md deleted file mode 100644 index 2bad4cb19b7..00000000000 --- a/site/ja/js/tutorials/transfer/image_classification.md +++ /dev/null @@ -1,15 +0,0 @@ -# 画像分類器の転移学習 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -このチュートリアルでは、TensorFlow.jsを使用してブラウザ上でその場で独自の画像分類器を構築する方法について学びます。 - -最小の学習データで非常に高精度なモデルを作成するために転移学習を利用できます。ここでは学習済みモデルとしてMobileNetという画像分類器を使用します。このモデルを土台としてモデルを訓練し、認識する画像クラスをカスタマイズします。 - -このチュートリアルはコードラボとして提供されます。[次のリンクに従ってコードラボを開いてください。](https://codelabs.developers.google.com/codelabs/tensorflowjs-teachablemachine-codelab/index.html) diff --git a/site/ja/js/tutorials/transfer/what_is_transfer_learning.md b/site/ja/js/tutorials/transfer/what_is_transfer_learning.md deleted file mode 100644 index e3ce979efc0..00000000000 --- a/site/ja/js/tutorials/transfer/what_is_transfer_learning.md +++ /dev/null @@ -1,20 +0,0 @@ -# 転移学習とは？ - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -高度な深層学習モデルには大量のパラメータ（重み）があり、それらをなにもないところから訓練するには大量の計算資源のデータが必要です。転移学習は関連するタスクについてすでに学習済みのモデルの一部を取り出して、新しいモデルの中で再利用することでそのような資源の大部分を省略するテクニックです。 - -例えば、このセクションの次のチュートリアルではすでに画像の中の数千種類の物体を認識できるように訓練されたモデルを利用して、独自の画像認識器を構築する方法を紹介します。学習済みモデルの既存の知識を適用すると、元のモデルで必要としたものよりもずっと少ない訓練データで独自の画像クラスを検出できます。 - -この手法はブラウザやモバイルデバイスのような資源が限られた環境でモデルをカスタマイズする際だけでなく、新しいモデルを高速に開発する際にも有効です。 - -転移学習を行う際、元のモデルの重みを調節しないこともよくあります。そのかわりに最終レイヤを取り除き、切り詰められたモデルの出力で新しい（通常は非常に浅い）モデルを訓練します。このセクションのチュートリアルで、このテクニックを紹介します。 - -- [画像分類器を元にした転移学習](image_classification) -- [音声認識を元にした転移学習](audio_recognizer) diff --git a/site/ja/lite/convert/cmdline.md b/site/ja/lite/convert/cmdline.md deleted file mode 100644 index dd2110e357a..00000000000 --- a/site/ja/lite/convert/cmdline.md +++ /dev/null @@ -1,40 +0,0 @@ -# コンバータのコマンドラインリファレンス - -このページでは、TensorFlow 2.0 でコマンドラインから [TensorFlow Lite コンバータ](index.md) を使用する方法を説明します。 ただし、より好ましいのは [Python API](python_api.md) を使用する方法です。 - -[TOC] - -## 概要の概要 - -TensorFlow Lite コンバータには基本的なモデルをサポートするコマンドラインツール `tflite_convert` がありますが、量子化やその他のパラメータ (SavedModel のシグネチャや Keras モデルのカスタムオブジェクトなど) を含む場合には、 `TFLiteConverter`[Python API](python_api.md) を使用してください。 - -## 使い方 - -以下のフラグで入力ファイルと出力ファイルを指定します。 - -* `--output_file` 文字列型。 出力ファイルのパスを指定する。 -* `--saved_model_dir` 文字列型。 TensorFlow 1.x もしくは 2.0 で構築した SavedModel を含むディレクトリのパスを指定する。 -* `--keras_model_file` 文字列型。 TensorFlow 1.x もしくは 2.0 で構築した tf.keras モデルを含む HDF5ファイルのパスを指定する。 - - -使用例は以下のとおりです。 - -``` -tflite_convert \ - --saved_model_dir=/tmp/mobilenet_saved_model \ - --output_file=/tmp/mobilenet.tflite -``` - -## 追加の説明 - -### ソースからビルドする - -TensorFlow Lite Converter の最新バージョンを利用するには [pip](https://www.tensorflow.org/install/pip) を使用してナイトリービルドをインストールする方法に加えて、[TensorFlow リポジトリを clone](https://www.tensorflow.org/install/source) して `bazel` を使う方法があります。 - -使用例は以下のとおりです。 - -``` -bazel run //third_party/tensorflow/lite/python:tflite_convert -- \ - --saved_model_dir=/tmp/mobilenet_saved_model \ - --output_file=/tmp/mobilenet.tflite -``` diff --git a/site/ja/lite/convert/concrete_function.md b/site/ja/lite/convert/concrete_function.md deleted file mode 100644 index ef53fa23ffb..00000000000 --- a/site/ja/lite/convert/concrete_function.md +++ /dev/null @@ -1,181 +0,0 @@ -# 具象関数の生成 - -TensorFlow 2.0 モデルを TensorFlow Lite に変換するには、モデルを具象関数 (concrete function) としてエクスポートする必要があります。 このドキュメントでは、具象関数とは何か、既存のモデルからどのように具象関数を生成するか、について概説します。 - -[TOC] - -## 背景 - -TensorFlow 2.0 では、Eager Execution がデフォルトでオンになっています。 TensorFlow において Eager Execution とは、グラフを作成せずにオペレーションを即時評価する命令型プログラミング環境のことです。各オペレーションは、後で実行するために計算グラフを構築するのではなく、具体的な値を返します。 Eager Execution に関する詳細なガイドは[こちら](https://github.com/tensorflow/docs/blob/master/site/en/guide/eager.ipynb)にあります。 - -Eager Execution で命令的に実行すると開発とデバッグがより対話的になりますが、デバイスへのデプロイはできなくなります。 `tf.function` API はモデルをグラフとして保存することを可能にします。 これは TensorFlow2.0 で TensorFlow Lite を実行するために必要です。 `tf.function` デコレータでラップされたオペレーションはすべてグラフとしてエクスポートできるので、 TensorFlow Lite FlatBuffer フォーマットに変換できます。 - -## 用語 - -この文書では次の用語を使用します。 - -* **シグネチャ** - 一連のオペレーションの入力と出力. -* **具象関数** - 単一のシグネチャを持つグラフ. -* **多相関数** - いくつかの具象関数グラフを1つの関数内にカプセル化した Python の呼び出し可能オブジェクト. - -## 方法論 - -この章では、具象関数をエクスポートする方法を解説します。 - -### 関数に `tf.function` デコーレータを付与する - -関数に `tf.function` デコレータを付与すると、その関数のオペレーションを含む *多相関数* が生成されます。 `tf.function` のデコレータが付けられていないオペレーションはすべて Eager Execution で評価されます。 以下の例は `tf.function` の使い方を示しています。 - -```python -@tf.function -def pow(x): - return x ** 2 -``` - -```python -tf.function(lambda x : x ** 2) -``` - -### 保存したいオブジェクトを生成する - -`tf.function` は、` tf.Module` オブジェクトの一部として保存することもできます。 その際、変数は `tf.Module` 内で一度だけ定義されるべきです。 以下の例は `Checkpoint` を派生するクラスを作成するための2つの異なるアプローチを示しています。 - -```python -class BasicModel(tf.Module): - - def __init__(self): - self.const = None - - @tf.function - def pow(self, x): - if self.const is None: - self.const = tf.Variable(2.) - return x ** self.const - -root = BasicModel() -``` - -```python -root = tf.Module() -root.const = tf.Variable(2.) -root.pow = tf.function(lambda x : x ** root.const) -``` - -### 具象関数をエクスポートする - -具象関数は、TensorFlow Lite モデルに変換したり、SavedModel にエクスポートしたりできるグラフを定義します。 -多相関数から具象関数をエクスポートするには、シグネチャを定義する必要があります。 -シグネチャは次のようにして定義できます。 - -* `tf.function` に ` input_signature` パラメータを定義します。 -* `tf.TensorSpec` を ` get_concrete_function` に渡します: 例) `tf.TensorSpec(shape=[1], dtype = tf.float32)` -* サンプルの入力テンソルを `get_concrete_function` に渡します: 例) `tf.constant(1., shape=[1])` - -次の例は `tf.function` の ` input_signature` パラメータを定義する方法を示しています。 - -```python -class BasicModel(tf.Module): - - def __init__(self): - self.const = None - - @tf.function(input_signature=[tf.TensorSpec(shape=[1], dtype=tf.float32)]) - def pow(self, x): - if self.const is None: - self.const = tf.Variable(2.) - return x ** self.const - -# tf.Module オブジェクトを生成 -root = BasicModel() - -# 具象関数を取得 -concrete_func = root.pow.get_concrete_function() -``` - -以下の例では、サンプルの入力テンソルを `get_concrete_function` に渡しています。 - -```python -# tf.Module オブジェクトを生成 -root = tf.Module() -root.const = tf.Variable(2.) -root.pow = tf.function(lambda x : x ** root.const) - -# 具象関数を取得 -input_data = tf.constant(1., shape=[1]) -concrete_func = root.pow.get_concrete_function(input_data) -``` - -## プログラム例 - -```python -import tensorflow as tf - -# tf.Module オブジェクトを初期化 -root = tf.Module() - -# 変数を一度だけインスタンス化する -root.var = None - -# 演算が事前に計算されないように関数を定義 -@tf.function -def exported_function(x): - # 変数は一度だけ定義できます。変数は関数内で定義できますが、関数外の参照を含める必要があります。 - if root.var is None: - root.var = tf.Variable(tf.random.uniform([2, 2])) - root.const = tf.constant([[37.0, -23.0], [1.0, 4.0]]) - root.mult = tf.matmul(root.const, root.var) - return root.mult * x - -# tf.Moduleオブジェクトの一部として関数を保存 -root.func = exported_function - -# 具象関数を取得 -concrete_func = root.func.get_concrete_function( - tf.TensorSpec([1, 1], tf.float32)) -``` - -## よくある質問 - -### 具象関数を SavedModel として保存するにはどうすればいいですか？ - -TensorFlow Lite に変換する前に TensorFlow モデルを保存したい場合は、 SavedModel として保存する必要があります。 -具象関数を取得したあとで `tf.saved_model.save` を呼び出すことでモデルを保存できます。上述の例の場合、以下のようにして保存できます。 - -```python -tf.saved_model.save(root, export_dir, concrete_func) -``` - -SavedModel の使用方法の詳細については、[SavedModel ガイド](https://github.com/tensorflow/docs/blob/master/site/en/guide/saved_model.ipynb) を参照してください。 - - -### SavedModel から具象関数を得るにはどうすればいいですか？ - -SavedModel 内の各象徴関数は、シグネチャキーによって識別できます。 -デフォルトのシグネチャキーは `tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY` です。以下の例は、モデルから具象関数を取得する方法を示しています。 - -```python -model = tf.saved_model.load(export_dir) -concrete_func = model.signatures[ - tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY] -``` - -### `tf.Keras` モデルから具象関数を得るにはどうしたらいいですか？ - -方法が2つあります: - -1. モデルを SavedModel として保存します。保存処理中に具象関数が生成されるので、上述の要領でモデルをロードして具象関数を取得することができます。 -2. 下記のようにモデルを `tf.function` でラップします。 - - -```python -model = tf.keras.Sequential([tf.keras.layers.Dense(units=1, input_shape=[1])]) -model.compile(optimizer='sgd', loss='mean_squared_error') -model.fit(x=[-1, 0, 1, 2, 3, 4], y=[-3, -1, 1, 3, 5, 7], epochs=50) - -# 具象関数を Keras モデルから取得 -run_model = tf.function(lambda x : model(x)) - -# 具象関数を保存 -concrete_func = run_model.get_concrete_function( - tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype)) -``` diff --git a/site/ja/lite/convert/index.md b/site/ja/lite/convert/index.md deleted file mode 100644 index 27c25f4f345..00000000000 --- a/site/ja/lite/convert/index.md +++ /dev/null @@ -1,22 +0,0 @@ -# TensorFlow Lite コンバータ - -TensorFlow Lite コンバータは、 TensorFlow モデルを入力として TensorFlow Lite [`FlatBuffer`](https://google.github.io/flatbuffers/) ファイルを生成します。 - -コンバータは [SavedModel ディレクトリ](https://www.tensorflow.org/guide/saved_model)、 -[`tf.keras` モデル](https://www.tensorflow.org/guide/keras/overview)、 -[具象関数](https://tensorflow.org/guide/concrete_function) をサポートしています。 - -Note: このページは TensorFlow 2.0 のコンバータ API に関するドキュメントです。TensorFlow 1.X の API については [こちら](https://www.tensorflow.org/lite/convert/) をご覧ください。 - -## デバイスへのデプロイ - -TensorFlow Lite `FlatBuffer` ファイルは、クライアントデバイス（モバイルデバイスや組み込みデバイス）にデプロイし、TensorFlow Lite インタープリタを使ってローカルで実行できます。この変換プロセスは下図のとおりです。 - -![TFLite converter workflow](../images/convert/workflow.svg) - -## モデルを変換する - -TensorFlow Lite コンバータは [Python API](python_api.md) を使うべきです。 -Python API を使うことで、モデルの変換をデプロイパイプラインの中に組み込むことが簡単になり、[互換性](../../guide/ops_compatibility.md) の問題に早い段階で対処しやすくなります。 - -代わりの方法として [コマンドラインツール](cmdline.md) を使って基本的なモデルを変換することもできます。 diff --git a/site/ja/lite/convert/python_api.md b/site/ja/lite/convert/python_api.md deleted file mode 100644 index c90a106b6bf..00000000000 --- a/site/ja/lite/convert/python_api.md +++ /dev/null @@ -1,237 +0,0 @@ -# コンバータ Python API ガイド - -このページでは、TensorFlow 2.0 の Python API による [TensorFlow Lite コンバータ](index.md) の使用例を説明します。 - -[TOC] - -## Python API - -TensorFlow 2.0 において、TensorFlow モデルを TensorFlow Lite に変換する Python API は `tf.lite.TFLiteConverter` です。 - `TFLiteConverter` には、元のモデルフォーマットに基づいてモデルを変換する以下のクラスメソッドがあります： - -* `TFLiteConverter.from_saved_model()`: - [SavedModel ディレクトリ](https://www.tensorflow.org/guide/saved_model) を変換します。 -* `TFLiteConverter.from_keras_model()`: - [`tf.keras` モデル](https://www.tensorflow.org/guide/keras/overview) を変換します。 -* `TFLiteConverter.from_concrete_functions()`: - [具象関数](https://tensorflow.org/guide/concrete_function) を変換します。 - -Node: TensorFlow Lite 2.0 alpha には、 [`from_concrete_function`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/lite/TFLiteConverter#from_concrete_function) だけを含むような、異なるバージョンの `TFLiteConverter` API があります。 -このドキュメントで記述されている API は、[`tf-nightly-2.0-preview`](#installing_the_tensorflow_20_nightly_) を PIP でインストールすることで使えるようになります。 - - -このドキュメントでは API の [使用例](＃examples) 、 [1.X と 2.0 の間の API の変更点の詳細なリスト](#differences) 、 異なるバージョンの TensorFlow で実行する [方法](#versioning) を含みます。 - -## 例 - -### SavedModel を変換する - -以下の例は [SavedModel](https://www.tensorflow.org/guide/saved_model) を TensorFlow Lite [`FlatBuffer`](https://google.github.io/flatbuffers/) に変換する方法を示しています。 - -```python -import tensorflow as tf - -# 基本的な関数を構築 -root = tf.train.Checkpoint() -root.v1 = tf.Variable(3.) -root.v2 = tf.Variable(2.) -root.f = tf.function(lambda x: root.v1 * root.v2 * x) - -# モデルを保存 -export_dir = "/tmp/test_saved_model" -input_data = tf.constant(1., shape=[1, 1]) -to_save = root.f.get_concrete_function(input_data) -tf.saved_model.save(root, export_dir, to_save) - -# モデルを変換 -converter = tf.lite.TFLiteConverter.from_saved_model(export_dir) -tflite_model = converter.convert() -``` - -### Keras モデルを変換する - -以下の例は [`tf.keras` モデル](https://www.tensorflow.org/guide/keras/overview) を TensorFlow Lite [`FlatBuffer`](https://google.github.io/flatbuffers/) に変換する方法を示しています. - - -```python -import tensorflow as tf - -# シンプルな Keras モデルを構築 -x = [-1, 0, 1, 2, 3, 4] -y = [-3, -1, 1, 3, 5, 7] - -model = tf.keras.models.Sequential( - [tf.keras.layers.Dense(units=1, input_shape=[1])]) -model.compile(optimizer='sgd', loss='mean_squared_error') -model.fit(x, y, epochs=50) - -# モデルを変換 -converter = tf.lite.TFLiteConverter.from_keras_model(model) -tflite_model = converter.convert() -``` - -### 具象関数を変換する - -以下の例は TensorFlow の[具象関数](https://tensorflow.org/guide/concrete_function)を -TensorFlow Lite [`FlatBuffer`](https://google.github.io/flatbuffers/) -に変換する方法を示しています。 - -```python -import tensorflow as tf - -# 基本的な関数を構築 -root = tf.train.Checkpoint() -root.v1 = tf.Variable(3.) -root.v2 = tf.Variable(2.) -root.f = tf.function(lambda x: root.v1 * root.v2 * x) - -# 具象関数を生成 -input_data = tf.constant(1., shape=[1, 1]) -concrete_func = root.f.get_concrete_function(input_data) - -# モデルを変換 -# -# `from_concrete_function` は具象関数のリストを引数に取りますが、 -# 現在のところは1つの関数のみをサポートしています。 複数関数の変換は開発中です -converter = tf.lite.TFLiteConverter.from_concrete_functions([concrete_func]) -tflite_model = converter.convert() -``` - -### End-to-end な MobileNet の変換 - -以下の例は、訓練済みの `tf.keras` MobileNet モデルを TensorFlow Lite に変換して実行する方法を示しています。 -また、 元の TensorFlow モデルと TensorFlow Lite モデルの結果をランダムデータで比較しています。 -モデルをファイルからロードするために、 `model_content` の代わりに ` model_path` を使用します。 - - -```python -import numpy as np -import tensorflow as tf - -# MobileNet tf.keras モデルをロード -model = tf.keras.applications.MobileNetV2( - weights="imagenet", input_shape=(224, 224, 3)) - -# モデルを変換 -converter = tf.lite.TFLiteConverter.from_keras_model(model) -tflite_model = converter.convert() - -# TFLite モデルを変換し、テンソルを割当てる -interpreter = tf.lite.Interpreter(model_content=tflite_model) -interpreter.allocate_tensors() - -# 入出力テンソルを取得 -input_details = interpreter.get_input_details() -output_details = interpreter.get_output_details() - -# TensorFlow Lite モデルをランダムな入力データでテスト -input_shape = input_details[0]['shape'] -input_data = np.array(np.random.random_sample(input_shape), dtype=np.float32) -interpreter.set_tensor(input_details[0]['index'], input_data) - -interpreter.invoke() -# `get_tensor()` はテンソルのコピーを返す -# テンソルのポインタを取得したい場合は `tensor()` を使う -tflite_results = interpreter.get_tensor(output_details[0]['index']) - -# 元の TensorFlow モデルをランダムな入力データでテスト -tf_results = model(tf.constant(input_data)) - -# 結果を比較 -for tf_result, tflite_result in zip(tf_results, tflite_results): - np.testing.assert_almost_equal(tf_result, tflite_result, decimal=5) -``` - -## 1.X から 2.0 への Python API の変更点まとめ - -以降の章では、Python API の 1.X から 2.0 への変更点についてまとめていますが、 -もしなにか懸念が生じた場合は GitHub の [issue](https://github.com/tensorflow/tensorflow/issues) を出してください。 - -### `TFLiteConverter` のサポートしているフォーマット - -2.0の `TFLiteConverter` は 1.X と 2.0 で生成された SavedModel と Keras -モデルファイルをサポートしますが、1.X で生成された frozen `GraphDefs` はサポートしません。 frozen `GraphDefs` を -TensorFlow Lite に変換したい場合は `tf.compat.v1.lite.TFLiteConverter` を使う必要があります。 - -### Quantization-aware training - -[quantization-aware training](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/quantize) に関連する以下の属性とメソッドは、 TensorFlow 2.0 の `TFLiteConverter` から削除されました: - - -* `inference_type` -* `inference_input_type` -* `quantized_input_stats` -* `default_ranges_stats` -* `reorder_across_fake_quant` -* `change_concat_input_ranges` -* `post_training_quantize` - 1.X API で非推奨 -* `get_input_arrays()` - -quantization-aware training をサポートしていた計算グラフの書き換え関数は、TensorFlow -2.0によるモデルをサポートしません。 また、TensorFlow Lite の quantization API は、Keras API を通じて -quantization-aware training をサポートする方向で作り直しと合理化を勧めている最中です。 新しい quantization API -がローンチされるまでは、これらの属性は 2.0 API から削除されます。 書き換え関数によってモデルを変換したい場合は -`tf.compat.v1.lite.TFLiteConverter` を使ってください。 - -### `TFLiteConverter` の属性に対する変更点 - -`target_ops` 属性は `TargetSpec` の属性となり、将来追加される予定の最適化フレームワークに合わせて `supported_ops` にリネームされました。 -また、以下の属性が削除されています: - -* `drop_control_dependency` (default: `True`) - 現在のところコントロールフローは TFLite でサポートされていないので、常に `True` です。 -* _Graph visualization_ - TensorFlow 2.0 において、 TensorFlow Lite グラフの可視化で推奨されるのは [visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/visualize.py) を使うことです。 GraphViz と違い、 post training quantization が施された後のグラフを可視化できます。 また、グラフの可視化に関する以下の属性は削除される予定です: - * `output_format` - * `dump_graphviz_dir` - * `dump_graphviz_video` - -### 一般的な API に対する変更点 - -#### 変換方法 - -1.X で既に非推奨となっていた以下のメソッドは、 2.0 では削除されています: - -* `lite.toco_convert` -* `lite.TocoConverter` - -#### `lite.constants` - -`lite.constants` API は、 TensorFlow と TensorFlow Lite の間の重複を減らすために 2.0 で削除されました。 -`lite.constant` の型と TensorFlow の型の対応は以下のとおりです。 - -* `lite.constants.FLOAT`: `tf.float32` -* `lite.constants.INT8`: `tf.int8` -* `lite.constants.INT32`: `tf.int32` -* `lite.constants.INT64`: `tf.int64` -* `lite.constants.STRING`: `tf.string` -* `lite.constants.QUANTIZED_UINT8`: `tf.uint8` - -また、`lite.constants.TFLITE` と `lite.constants.GRAPHVIZ_DOT` は、 `TFLiteConverter` の `output_format` の廃止に伴い削除されました。 - -#### `lite.OpHint` - -`OpHint` API は、2.0 API との互換性がないため、現在 2.0 では利用できません。 -この API は LSTM ベースのモデルの変換を可能にするものですが、2.0 における LSTM のサポートは検証中のため、関連する `lite.experimental` API はすべて削除されています。 - -## TensorFlow のインストール - -### TensorFlow 2.0 nightly のインストール - -TensorFlow 2.0 nightly は以下のコマンドでインストールできます: - -``` -pip install tf-nightly-2.0-preview -``` - -### インストール済の TensorFlow 1.X から 2.0 を使う - -TensorFlow 2.0 は、最近の 1.X から以下のようにして利用できます。 - -```python -import tensorflow.compat.v2 as tf - -tf.enable_v2_behavior() -``` - -### ソースコードからのビルド - -TensorFlow Lite コンバータ Python API の最新バージョンを実行するには、 [pip](https://www.tensorflow.org/install/pip) (推奨) または [Docker](https://www.tensorflow.org/install/docker) を使用してナイトリービルドをインストールするか、[ソースから pip パッケージをビルド](https://www.tensorflow.org/install/source) してください。 diff --git a/site/ja/lite/images/convert/workflow.svg b/site/ja/lite/images/convert/workflow.svg deleted file mode 100644 index 2d8339f35f3..00000000000 --- a/site/ja/lite/images/convert/workflow.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/site/ja/probability/overview.md b/site/ja/probability/overview.md deleted file mode 100644 index 8bdc89cfea6..00000000000 --- a/site/ja/probability/overview.md +++ /dev/null @@ -1,88 +0,0 @@ -# TensorFlow Probability - - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -TensorFlow Probability は TensorFlow における確率的推論と統計的分析のためのライブラリです。 -TensorFlow エコシステムの一部として、TensorFlow Probability は確率的手法とさまざまな手法や機能との統合を提供します。 -たとえば、深層ネットワークを用いた確率的な手法、自動微分を用いた勾配に基づく推論、GPU のようなハードウェア高速化や分散処理による大きなデータセットやモデルに対するスケーラビリティなどです。 - -TensorFlow Probability を始めるためには、[インストールガイド](./install) や -[Python notebook チュートリアル](https://github.com/tensorflow/probability/blob/master/tensorflow_probability/examples/jupyter_notebooks/)を参照してください。 - -## コンポーネント - -我々の確率的機械学習ツール群は以下のような構造になっています: - -### Layer 0: TensorFlow - -*数値処理*—特に、`LinearOperator` -クラスが可能にする、効率的な演算のための特定の構造 (対角、低ランク) を開発できるようにする行列フリーな実装。 -TensorFlow Probability チームによりメンテナンスされていて、TensorFlow コアの [`tf.linalg`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/ops/linalg) の一部です。 - -### Layer 1: 確率的なブロックの構築 - -* *Distributions* ([`tfp.distributions`](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/python/distributions), - [`tf.distributions`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/ops/distributions)): - バッチや[ブロードキャスティング](https://docs.scipy.org/doc/numpy-1.14.0/user/basics.broadcasting.html)の仕組みを取り入れた、多くの確率分布やそれに関連する統計量の処理 -* *Bijectors* ([`tfp.bijectors`](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/python/bijectors)): - ランダム変数の可逆で組み立て可能な変換。 Bijectors は変換された分布の豊富なクラスを提供します。それは、古典的な - [対数正規分布](https://en.wikipedia.org/wiki/Log-normal_distribution)から - [masked autoregressive flows](https://arxiv.org/abs/1705.07057) のような洗練された深層学習モデルにまで及びます。 - -### Layer 2: モデル構築 - -* *Edward2* - ([`tfp.edward2`](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/python/edward2)): - プログラムとして柔軟な確率的モデルを定義するための確率的プログラミング言語 -* *Probabilistic layers* - ([`tfp.layers`](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/python/layers)): - TensorFlow の layers を拡張して、それらが表現する関数の不確実性を出力できるニューラルネットワーク層 -* *Trainable distributions* - ([`tfp.trainable_distributions`](https://github.com/tensorflow/probability/blob/master/tensorflow_probability/python/trainable_distributions)): - 確率分布を出力するニューラルネットワークを構築することを簡単にする、1つの Tensor によるパラメータをもつ確率分布 - -### Layer 3: 確率的推論 - -* *Markov chain Monte Carlo* - ([`tfp.mcmc`](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/python/mcmc)): - サンプリングによる積分近似のためのアルゴリズム - [ハミルトンモンテカルロ法](https://en.wikipedia.org/wiki/Hamiltonian_Monte_Carlo)、 - ランダムウォークメトロポリス・ヘイスティング法、カスタム遷移カーネルを構築することができる機能を含みます。 -* *Variational Inference* - ([`tfp.vi`](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/python/vi)): - 最適化による積分近似のためのアルゴリズム -* *Optimizers* - ([`tfp.optimizer`](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/python/optimizer)): - TensorFlow Optimizers を拡張した、確率的最適化モジュール。 - [Stochastic Gradient Langevin Dynamics](http://www.icml-2011.org/papers/398_icmlpaper.pdf) を含みます。 -* *Monte Carlo* - ([`tfp.monte_carlo`](https://github.com/tensorflow/probability/blob/master/tensorflow_probability/python/monte_carlo)): - モンテカルロ法を用いたツール群 - -TensorFlow Probability は開発中であるため、インタフェースは変更される可能性があります。 - -## 使用例 - -ナビゲーションに載っている [Python notebook チュートリアル](https://github.com/tensorflow/probability/blob/master/tensorflow_probability/examples/jupyter_notebooks/) -に加えて、いくつかのスクリプト例が利用できます: - -* [Variational Autoencoders](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/examples/vae.py) - —latent code と変分推論による表現学習 -* [Vector-Quantized Autoencoder](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/examples/vq_vae.py) - —ベクトル量子化による離散表現学習 -* [ベイジアンニューラルネットワーク](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/examples/bayesian_neural_network.py) - —重みの不確実性を出力するニューラルネットワーク -* [ベイズロジスティック回帰](https://github.com/tensorflow/probability/tree/master/tensorflow_probability/examples/logistic_regression.py) - —二値分類のためのベイズ推論 - -## issue の報告 - -バグ報告や機能要望は -[TensorFlow Probability issue tracker](https://github.com/tensorflow/probability/issues) を使用してください。 diff --git a/site/ja/r1/README.md b/site/ja/r1/README.md deleted file mode 100644 index 66f97f98f75..00000000000 --- a/site/ja/r1/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# TensorFlow 1.x - -This archive of the TensorFlow 1.x docs is in *maintenance mode* only. - -For docs contributors, please update the source files in `site/en/` and read the -[TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs). - -For community translations, read the instructions in `site/ja/README.md`. diff --git a/site/ja/r1/guide/eager.ipynb b/site/ja/r1/guide/eager.ipynb deleted file mode 100644 index 8ad6d4f7b73..00000000000 --- a/site/ja/r1/guide/eager.ipynb +++ /dev/null @@ -1,1653 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "eager.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "CCQY7jpBfMur" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCQY7jpBfMur" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "z6X9omPnfO_h", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2QQJJyDzqGRb" - }, - "source": [ - "# Eager Execution\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B1xdylywqUSX" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dfJ5KndxaefN" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EGjDcGxIqEfX" - }, - "source": [ - "TensorflowのEager Executionは、計算グラフの作成と評価を同時におこなう命令的なプログラミングを行うための環境です：\n", - "オペレーションはあとで実行するための計算グラフでなく、具体的な計算結果の値を返します。\n", - "この方法を用いることにより、初心者にとってTensorFlowを始めやすくなり、またモデルのデバッグも行いやすくなります。\n", - "さらにコードの記述量も削減されます。\n", - "このガイドの内容を実行するためには、対話的インタープリタ`python`を起動し、以下のコードサンプルを実行してください。\n", - "\n", - "Eager Executionは研究や実験のための柔軟な機械学習環境として、以下を提供します。\n", - "\n", - "* *直感的なインタフェース*—Pythonのデータ構造を使用して、コードをナチュラルに記述することができます。スモールなモデルとデータに対してすばやく実験を繰り返すことができます。\n", - "* *より簡単なデバッグ*—opsを直接呼び出すことで、実行中のモデルを調査したり、変更をテストすることができます。Python標準のデバッグツールを用いて即座にエラーのレポーティングができます。\n", - "* *自然な制御フロー*—TensorFlowのグラフ制御フローの代わりにPythonの制御フローを利用するため、動的なモデルのパラメータ変更をシンプルに行うことができます。\n", - " \n", - "Eager ExecutionはTensorflowのほとんどのオペレーションとGPUアクセラレーションをサポートします。\n", - "Eager Executionの実行例については、以下を参照してください。\n", - "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).\n", - "\n", - "Note: いくつかのモデルはEager Executionを有効化することでオーバヘッドが増える可能性があります。\n", - "パフォーマンス改善を行っていますが、もしも問題を発見したら、バグ報告してベンチマークを共有してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RBAeIwOMrYk8" - }, - "source": [ - "## セットアップと基本的な使い方" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "48P3-8q4qEfe" - }, - "source": [ - "Eager Executionをはじめるためには、プログラムやコンソールセッションの最初に、`tf.enable_eager_execution()`を追加してください。\n", - "プログラムが呼び出すほかのモジュールにこのオペレーションを追加しないでください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "7aFsD8csqEff", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x_G1zZT5qEfh" - }, - "source": [ - "これでTensorFlowのオペレーションを実行してみましょう。結果はすぐに返されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5hien2IEqwLQ", - "colab": {} - }, - "source": [ - "tf.executing_eagerly()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "9gsI54pbqEfj", - "colab": {} - }, - "source": [ - "x = [[2.]]\n", - "m = tf.matmul(x, x)\n", - "print(\"hello, {}\".format(m))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ajFn6qsdqEfl" - }, - "source": [ - "Eager Executionを有効化することで、TensorFlowの挙動は変わります—TensorFlowは即座に式を評価して結果をPythonに返すようになります。\n", - "`tf.Tensor` オブジェクトは計算グラフのノードへのシンボリックハンドルの代わりに具体的な値を参照します。\n", - "セッションの中で構築して実行する計算グラフが存在しないため、`print（）`やデバッガを使って容易に結果を調べることができます。\n", - "勾配計算を終了することなくテンソル値を評価、出力、およびチェックすることができます。\n", - "\n", - "Eager Executionは、[NumPy](http://www.numpy.org/)と一緒に使うことができます。\n", - "NumPyのオペレーションは、`tf.Tensor`を引数として受け取ることができます。\n", - "TensorFlow [math operations](https://www.tensorflow.org/api_guides/python/math_ops) はPythonオブジェクトとNumpy arrayを`tf.Tensor`にコンバートします。\n", - "`tf.Tensor.numpy`メソッドはオブジェクトの値をNumPyの`ndarray`形式で返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sTO0_5TYqz1n", - "colab": {} - }, - "source": [ - "a = tf.constant([[1, 2],\n", - " [3, 4]])\n", - "print(a)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Dp14YT8Gq4r1", - "colab": {} - }, - "source": [ - "# ブロードキャストのサポート\n", - "b = tf.add(a, 1)\n", - "print(b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "69p3waMfq8cQ", - "colab": {} - }, - "source": [ - "# オペレータのオーバーロードがサポートされている\n", - "print(a * b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Ui025t1qqEfm", - "colab": {} - }, - "source": [ - "# NumPy valueの使用\n", - "import numpy as np\n", - "\n", - "c = np.multiply(a, b)\n", - "print(c)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tq_aFRzWrCua", - "colab": {} - }, - "source": [ - "# Tensorからnumpyの値を得る\n", - "print(a.numpy())\n", - "# => [[1 2]\n", - "# [3 4]]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Jdg3nZFwqEfp" - }, - "source": [ - "`tf.contrib.eager` モジュールは、Eager ExecutionとGraph Executionの両方の環境で利用可能なシンボルが含まれており、[Graph Execution](#work_with_graphs)方式での記述に便利です:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "N-2lSGiHqEfq", - "colab": {} - }, - "source": [ - "tfe = tf.contrib.eager" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H08f9ss9qEft" - }, - "source": [ - "## 動的な制御フロー\n", - "\n", - "Eager Executionの主要なメリットは、モデルを実行する際にホスト言語のすべての機能性が利用できることです。\n", - "たとえば、[fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz)が簡単に書けます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "0fudRMeUqEfu", - "colab": {} - }, - "source": [ - "def fizzbuzz(max_num):\n", - " counter = tf.constant(0)\n", - " max_num = tf.convert_to_tensor(max_num)\n", - " for num in range(1, max_num.numpy()+1):\n", - " num = tf.constant(num)\n", - " if int(num % 3) == 0 and int(num % 5) == 0:\n", - " print('FizzBuzz')\n", - " elif int(num % 3) == 0:\n", - " print('Fizz')\n", - " elif int(num % 5) == 0:\n", - " print('Buzz')\n", - " else:\n", - " print(num.numpy())\n", - " counter += 1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "P2cKknQWrJLB", - "colab": {} - }, - "source": [ - "fizzbuzz(15)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7kA-aC3BqEfy" - }, - "source": [ - "この関数はテンソル値に依存する条件式を持ち、実行時にこれらの値を表示します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zn3mp-j6rjnk" - }, - "source": [ - "## モデルの構築\n", - "\n", - "多くの機械学習モデルはレイヤーを積み重ねによって成り立っています。Eager ExecutionでTensorFlowを使うときは、自分でレイヤーの内容を記述してもいいし、もしくは `tf.keras.layers`パッケージで提供されるレイヤーを使うこともできます。\n", - "\n", - "レイヤーを表現するためには任意のPythonオブジェクトを使用できますが、\n", - "TensorFlowには便利な基本クラスとして `tf.keras.layers.Layer`があります。\n", - "このクラスを継承した独自のレイヤーを実装してみます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "kvXiJsmyqEfz", - "colab": {} - }, - "source": [ - "class MySimpleLayer(tf.keras.layers.Layer):\n", - " def __init__(self, output_units):\n", - " super(MySimpleLayer, self).__init__()\n", - " self.output_units = output_units\n", - "\n", - " def build(self, input_shape):\n", - " # buildメソッドは、レイヤーが初めて使われたときに呼ばれます\n", - " # build（）で変数を作成すると、それらのshapeを入力のshapeに依存させることができ、\n", - " # ユーザがshapeを完全に指定する必要はありません。\n", - " # 既に完全なshapeが決まっている場合は、__init__()の中で変数を作成することもできます。\n", - " self.kernel = self.add_variable(\n", - " \"kernel\", [input_shape[-1], self.output_units])\n", - "\n", - " def call(self, input):\n", - " # __call__の代わりにcall()を上書きします。\n", - " return tf.matmul(input, self.kernel)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eo3qgrVCqEf2" - }, - "source": [ - " `MySimpleLayer`の代わりに、その機能のスーパーセットを持っている`tf.keras.layers.Dense`レイヤーを使用してください\n", - " （このレイヤーはバイアスを加えることもできるもできます）。\n", - "\n", - "レイヤーをモデルに組み立てるとき、レイヤーの線形スタックである\n", - "モデルを表すために `tf.keras.Sequential`を使うことができます。この書き方は基本的なモデルを扱いやすいです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "VrfLnhNPqEf3", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, input_shape=(784,)), # 入力のshapeを指定する必要がある\n", - " tf.keras.layers.Dense(10)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dms3mduTqEf6" - }, - "source": [ - "もしくは、 `tf.keras.Model`を継承してモデルをクラスにまとめます。 \n", - "これはレイヤー自身であるレイヤーのコンテナで、 `tf.keras.Model`オブジェクトが他の` tf.keras.Model`オブジェクトを含むことを可能にします。\n", - "\n", - "Alternatively, organize models in classes by inheriting from `tf.keras.Model`.\n", - "This is a container for layers that is a layer itself, allowing `tf.keras.Model`\n", - "objects to contain other `tf.keras.Model` objects." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "MwWxQmNOqEf7", - "colab": {} - }, - "source": [ - "class MNISTModel(tf.keras.Model):\n", - " def __init__(self):\n", - " super(MNISTModel, self).__init__()\n", - " self.dense1 = tf.keras.layers.Dense(units=10)\n", - " self.dense2 = tf.keras.layers.Dense(units=10)\n", - "\n", - " def call(self, input):\n", - " \"\"\"Run the model.\"\"\"\n", - " result = self.dense1(input)\n", - " result = self.dense2(result)\n", - " result = self.dense2(result) # dense2レイヤーを再利用します reuse variables from dense2 layer\n", - " return result\n", - "\n", - "model = MNISTModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "a639YaF4qEf-" - }, - "source": [ - "入力のshapeは最初のレイヤーに初めて入力データを渡すときにセットされるため、\n", - "モデル構築時に`tf.keras.Model`クラスに設定する必要はありません。\n", - "\n", - "`tf.keras.layers`クラスは独自のモデル変数を作成し、包含します。このモデル変数は、それを含むレイヤーオブジェクトのライフタイムにひもづきます。レイヤー変数を共有するには、それらのオブジェクトを共有します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8huKpuuAwICq" - }, - "source": [ - "## Eager Executionにおける学習" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mp2lCCZYrxHd" - }, - "source": [ - "### 勾配の計算\n", - "\n", - "[自動微分](https://en.wikipedia.org/wiki/Automatic_differentiation)はニューラルネットワークの学習で利用される[バックプロパゲーション](https://en.wikipedia.org/wiki/Backpropagation)などの機械学習アルゴリズムの実装を行う上で便利です。\n", - "Eager Executionでは、勾配計算をあとで行うためのオペレーションをトレースするために`tf.GradientTape` を利用します。\n", - "\n", - "`tf.GradientTape` はトレースしない場合に最大のパフォーマンスを提供するオプトイン機能です。各呼び出し中に異なるオペレーションが発生する可能性があるため、すべてのforward-passオペレーションは一つの「テープ」に記録されます。勾配を計算するには、テープを逆方向に再生してから破棄します。特定の `tf.GradientTape`は一つのグラデーションしか計算できません。後続の呼び出しは実行時エラーをスローします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "7g1yWiSXqEf-", - "colab": {} - }, - "source": [ - "w = tf.Variable([[1.0]])\n", - "with tf.GradientTape() as tape:\n", - " loss = w * w\n", - "\n", - "grad = tape.gradient(loss, w)\n", - "print(grad) # => tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkHs32GqweYS" - }, - "source": [ - "### モデル学習\n", - "\n", - "以下のexampleはMNISTという手書き数字分類を行うマルチレイヤーモデルを作成します。\n", - "Eager Execution環境における学習可能なグラフを構築するためのオプティマイザーとレイヤーAPIを提示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "38kymXZowhhz", - "colab": {} - }, - "source": [ - "# mnistデータのを取得し、フォーマットする\n", - "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n", - "\n", - "dataset = tf.data.Dataset.from_tensor_slices(\n", - " (tf.cast(mnist_images[...,tf.newaxis]/255, tf.float32),\n", - " tf.cast(mnist_labels,tf.int64)))\n", - "dataset = dataset.shuffle(1000).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rl1K8rOowmwT", - "colab": {} - }, - "source": [ - "# モデルを構築する\n", - "mnist_model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(10)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fvyk-HgGwxwl" - }, - "source": [ - "学習を行わずとも、モデルを呼び出して、Eager Executionにより、出力を検査することができます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BsxystjBwxLS", - "colab": {} - }, - "source": [ - "for images,labels in dataset.take(1):\n", - " print(\"Logits: \", mnist_model(images[0:1]).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y3PGa8G7qEgB" - }, - "source": [ - "kerasモデルは組み込みで学習のループを回すメソッド`fit`がありますが、よりカスタマイズが必要な場合もあるでしょう。 Eager Executionを用いて実装された学習ループのサンプルを以下に示します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bzRhM7JDnaEG", - "colab": {} - }, - "source": [ - "optimizer = tf.train.AdamOptimizer()\n", - "\n", - "loss_history = []" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "0m1xAXrmqEgJ", - "colab": {} - }, - "source": [ - "for (batch, (images, labels)) in enumerate(dataset.take(400)):\n", - " if batch % 10 == 0:\n", - " print('.', end='')\n", - " with tf.GradientTape() as tape:\n", - " logits = mnist_model(images, training=True)\n", - " loss_value = tf.losses.sparse_softmax_cross_entropy(labels, logits)\n", - "\n", - " loss_history.append(loss_value.numpy())\n", - " grads = tape.gradient(loss_value, mnist_model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, mnist_model.trainable_variables),\n", - " global_step=tf.train.get_or_create_global_step())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5vG5ql_2vYB5", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.plot(loss_history)\n", - "plt.xlabel('Batch #')\n", - "plt.ylabel('Loss [entropy]')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKpOlHPLqEgl" - }, - "source": [ - "### 値とオプティマイザ\n", - "\n", - "`tf.Variable` オブジェクトは、学習中にアクセスされるミュータブルな`tf.Tensor`値を格納し、自動微分を容易にします。\n", - "モデルのパラメータは、変数としてクラスにカプセル化できます。\n", - "\n", - "`tf.GradientTape`と共に` tf.Variable`を使うことでモデルパラメータはよりカプセル化されます。たとえば、上の\n", - "の自動微分の例は以下のように書き換えることができます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "nnQLBYmEqEgm", - "colab": {} - }, - "source": [ - "class Model(tf.keras.Model):\n", - " def __init__(self):\n", - " super(Model, self).__init__()\n", - " self.W = tf.Variable(5., name='weight')\n", - " self.B = tf.Variable(10., name='bias')\n", - " def call(self, inputs):\n", - " return inputs * self.W + self.B\n", - "\n", - "# 3 ＊ ２ + 2を近似するトイデータセット\n", - "NUM_EXAMPLES = 2000\n", - "training_inputs = tf.random_normal([NUM_EXAMPLES])\n", - "noise = tf.random_normal([NUM_EXAMPLES])\n", - "training_outputs = training_inputs * 3 + 2 + noise\n", - "\n", - "# オプティマイズ対象のloss関数\n", - "def loss(model, inputs, targets):\n", - " error = model(inputs) - targets\n", - " return tf.reduce_mean(tf.square(error))\n", - "\n", - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return tape.gradient(loss_value, [model.W, model.B])\n", - "\n", - "# 定義：\n", - "# 1. モデル\n", - "# 2. モデルパラメータに関する損失関数の導関数\n", - "# 3. 導関数に基づいて変数を更新するストラテジ。\n", - "model = Model()\n", - "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n", - "\n", - "print(\"Initial loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", - "\n", - "# 学習ループ\n", - "for i in range(300):\n", - " grads = grad(model, training_inputs, training_outputs)\n", - " optimizer.apply_gradients(zip(grads, [model.W, model.B]),\n", - " global_step=tf.train.get_or_create_global_step())\n", - " if i % 20 == 0:\n", - " print(\"Loss at step {:03d}: {:.3f}\".format(i, loss(model, training_inputs, training_outputs)))\n", - "\n", - "print(\"Final loss: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", - "print(\"W = {}, B = {}\".format(model.W.numpy(), model.B.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rPjb8nRWqEgr" - }, - "source": [ - "## Eager Executionの途中でオブジェクトのステータスを使用する\n", - "\n", - "Graph Executionでは、プログラムの状態（変数など）はglobal collectionに格納され、それらの存続期間は `tf.Session`オブジェクトによって管理されます。\n", - "対照的に、Eager Executionの間、状態オブジェクトの存続期間は、対応するPythonオブジェクトの存続期間によって決定されます。\n", - "\n", - "### 変数とオブジェクト\n", - "\n", - "Eager Executionの間、変数はオブジェクトへの最後の参照が削除され、その後削除されるまで存続します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "A2boS674qEgs", - "colab": {} - }, - "source": [ - "if tf.test.is_gpu_available():\n", - " with tf.device(\"gpu:0\"):\n", - " v = tf.Variable(tf.random_normal([1000, 1000]))\n", - " v = None # vは既にGPUメモリ上を使用しないようにする" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "scMjg6L6qEgv" - }, - "source": [ - "### オブジェクトベースの保存\n", - "\n", - "`tf.train.Checkpoint`はチェックポイントを用いて`tf.Variable`を保存および復元することができます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7z5xRfdHzZOQ", - "colab": {} - }, - "source": [ - "x = tf.Variable(10.)\n", - "checkpoint = tf.train.Checkpoint(x=x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IffrUVG7zyVb", - "colab": {} - }, - "source": [ - "x.assign(2.) # 変数に新しい値を割り当てて保存する\n", - "checkpoint_path = './ckpt/'\n", - "checkpoint.save('./ckpt/')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "eMT9koCoqEgw", - "colab": {} - }, - "source": [ - "x.assign(11.) # 保存後に変数の値を変更する\n", - "\n", - "# チェックポイントから値を復元する\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_path))\n", - "\n", - "print(x) # => 2.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbFnP-yLqEgx" - }, - "source": [ - "モデルを保存して読み込むために、 `tf.train.Checkpoint`は隠れ変数なしにオブジェクトの内部状態を保存します。 `モデル`、 `オプティマイザ`、そしてグローバルステップの状態を記録するには、それらを `tf.train.Checkpoint`に渡します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "hWZHyAXMqEg0", - "colab": {} - }, - "source": [ - "import os\n", - "import tempfile\n", - "\n", - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(10)\n", - "])\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=0.001)\n", - "checkpoint_dir = tempfile.mkdtemp()\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "root = tf.train.Checkpoint(optimizer=optimizer,\n", - " model=model,\n", - " optimizer_step=tf.train.get_or_create_global_step())\n", - "\n", - "root.save(checkpoint_prefix)\n", - "root.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3yoD0VJ7qEg3" - }, - "source": [ - "### オブジェクト指向メトリクス\n", - "\n", - "`tfe.metrics`はオブジェクトとして保存されます。新しいデータを呼び出し可能オブジェクトに渡してメトリクスを更新し、 `tfe.metrics.result`メソッドを使って結果を取得します。次に例を示します:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "9ccu0iAaqEg5", - "colab": {} - }, - "source": [ - "m = tfe.metrics.Mean(\"loss\")\n", - "m(0)\n", - "m(5)\n", - "m.result() # => 2.5\n", - "m([8, 9])\n", - "m.result() # => 5.5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BE8cXErYqEg8" - }, - "source": [ - "#### サマリとTensorBoard\n", - "\n", - "[TensorBoard](../guide/summaries_and_tensorboard.md) はモデルの学習プロセスを理解、デバッグ、最適化するための可視化ツールです。プログラムの実行中に書き込まれるサマリイベントを使用します。\n", - "\n", - "`tf.contrib.summary`はEager ExecutionとGraph Executionの両方の環境と互換性があります。 \n", - "`tf.contrib.summary.scalar`のようなサマリオペレーションはモデル構築の間に挿入されます。\n", - "たとえば、100のグローバルステップごとにサマリを記録するには、次のようにします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "3PXAJB1GqEg9", - "colab": {} - }, - "source": [ - "global_step = tf.train.get_or_create_global_step()\n", - "\n", - "logdir = \"./tb/\"\n", - "writer = tf.contrib.summary.create_file_writer(logdir)\n", - "writer.set_as_default()\n", - "\n", - "for _ in range(10):\n", - " global_step.assign_add(1)\n", - " # record_summariesメソッドをincludeする必要がある\n", - " with tf.contrib.summary.record_summaries_every_n_global_steps(100):\n", - " # ここにモデルのコードを記述する\n", - " tf.contrib.summary.scalar('global_step', global_step)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6TJSs_wG8Spg", - "colab": {} - }, - "source": [ - "!ls tb/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xEL4yJe5qEhD" - }, - "source": [ - "## 高度な自動分類トピック\n", - "\n", - "### 動的なモデル\n", - "\n", - "`tf.GradientTape`は動的モデルでも使うことができます。 \n", - "以下の[バックトラックライン検索](https://wikipedia.org/wiki/Backtracking_line_search)\n", - "アルゴリズムの例は、複雑な制御フローにも関わらず\n", - "勾配があり、微分可能であることを除いて、通常のNumPyコードのように見えます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "L518n5dkqEhE", - "colab": {} - }, - "source": [ - "def line_search_step(fn, init_x, rate=1.0):\n", - " with tf.GradientTape() as tape:\n", - " # 変数は自動的に記録されるが、手動でTensorを監視する\n", - " tape.watch(init_x)\n", - " value = fn(init_x)\n", - " grad = tape.gradient(value, init_x)\n", - " grad_norm = tf.reduce_sum(grad * grad)\n", - " init_value = value\n", - " while value > init_value - rate * grad_norm:\n", - " x = init_x - rate * grad\n", - " value = fn(x)\n", - " rate /= 2.0\n", - " return x, value" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "03owpCFrqEhH" - }, - "source": [ - "### 勾配計算のための追加機能\n", - "\n", - "`tf.GradientTape`は強力な勾配計算インタフェースですが、\n", - "自動微分に利用できる別の[Autograd](https://github.com/HIPS/autograd)スタイルのAPIもあります。\n", - "これらの関数はテンソルと勾配関数のみを使って、`tf.variables`を使わずに数式コードを書く場合に便利です：\n", - "\n", - "* `tfe.gradients_function`—引数をとり、入力関数パラメータの導関数を計算する関数を返します。 \n", - " 入力パラメータはスカラ値を返さなければなりません。返された関数が\n", - " されると、 `tf.Tensor`オブジェクトのリストを返します：入力関数のそれぞれの\n", - " 引数に対して一つの要素。重要なものすべてを関数パラメータとして渡さなければならないので、\n", - " 多くのtrainableパラメータに依存している場合、これは扱いにくくなります。\n", - "* `tfe.value_and_gradients_function`—` tfe.gradients_function`に似ていますが、\n", - " 返された関数が呼び出されると、その引数に関する入力関数の導関数のリストに加えて、入力関数からの値を返します。\n", - "\n", - "次の例では、 `tfe.gradients_function`は引数として` square` 関数を取り、その入力に関して `square`の偏微分\n", - "導関数を計算する関数を返します。 `3`における` square`の微分を計算するために、 `grad（3.0）`は `6`を返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QDPFUG-68i-B", - "colab": {} - }, - "source": [ - "def square(x):\n", - " return tf.multiply(x, x)\n", - "\n", - "grad = tfe.gradients_function(square)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_Ow9LHre8k_3", - "colab": {} - }, - "source": [ - "square(3.).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Rg5ea0kC8nEQ", - "colab": {} - }, - "source": [ - "grad(3.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "41D1LzcG87p8", - "colab": {} - }, - "source": [ - "# 平方の二次導関数：\n", - "gradgrad = tfe.gradients_function(lambda x: grad(x)[0])\n", - "gradgrad(3.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "DotEGcx5qEhH", - "colab": {} - }, - "source": [ - "# 3次導関数はNoneになる：\n", - "gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0])\n", - "gradgradgrad(3.)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9DRAews99F4-", - "colab": {} - }, - "source": [ - "# フロー制御：\n", - "def abs(x):\n", - " return x if x > 0. else -x\n", - "\n", - "grad = tfe.gradients_function(abs)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hTA1jmu_9gyB", - "colab": {} - }, - "source": [ - "grad(3.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7zs8JpKw9kir", - "colab": {} - }, - "source": [ - "grad(-3.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gieGOf_DqEhK" - }, - "source": [ - "### カスタム勾配\n", - "\n", - "カスタム勾配は、Eager ExecutionとGraph Executionの両方の環境で、勾配を上書きする簡単な方法です。 フォワード関数では、\n", - "入力、出力、または中間結果に関する勾配を定義します。たとえば、逆方向パスにおいて勾配のノルムを切り取る簡単な方法は次のとおりです：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "-OwwsWUAqEhK", - "colab": {} - }, - "source": [ - "@tf.custom_gradient\n", - "def clip_gradient_by_norm(x, norm):\n", - " y = tf.identity(x)\n", - " def grad_fn(dresult):\n", - " return [tf.clip_by_norm(dresult, norm), None]\n", - " return y, grad_fn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JPLDHkF_qEhN" - }, - "source": [ - "カスタム勾配は、一連の演算に対して数値的に安定した勾配を提供するために共通的に使用されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "24WiLROnqEhO", - "colab": {} - }, - "source": [ - "def log1pexp(x):\n", - " return tf.log(1 + tf.exp(x))\n", - "grad_log1pexp = tfe.gradients_function(log1pexp)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "n8fq69r9-B-c", - "colab": {} - }, - "source": [ - "# 勾配計算はx = 0のときにはうまくいきます。\n", - "grad_log1pexp(0.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_VFSU0mG-FSp", - "colab": {} - }, - "source": [ - "# しかし、x = 100のときは数値的不安定により失敗します。\n", - "grad_log1pexp(100.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-VcTR34rqEhQ" - }, - "source": [ - "ここで、 `log1pexp`関数はカスタム勾配を用いて解析的に単純化することができます。\n", - "以下の実装は、フォワードパスの間に計算された `tf.exp（x）`の値を\n", - "再利用します—冗長な計算を排除することでより効率的になります：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Q7nvfx_-qEhS", - "colab": {} - }, - "source": [ - "@tf.custom_gradient\n", - "def log1pexp(x):\n", - " e = tf.exp(x)\n", - " def grad(dy):\n", - " return dy * (1 - 1 / (1 + e))\n", - " return tf.log(1 + e), grad\n", - "\n", - "grad_log1pexp = tfe.gradients_function(log1pexp)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5gHPKMfl-Kge", - "colab": {} - }, - "source": [ - "# 上と同様に、勾配計算はx = 0のときにはうまくいきます。\n", - "grad_log1pexp(0.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u38MOfz3-MDE", - "colab": {} - }, - "source": [ - "# また、勾配計算はx = 100でも機能します。\n", - "grad_log1pexp(100.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rnZXjfQzqEhV" - }, - "source": [ - "## パフォーマンス\n", - "\n", - "Eager Executionの間、計算は自動的にGPUにオフロードされます。計算を実行するデバイスを指定したい場合は、\n", - "`tf.device（ '/ gpu：0'）`ブロック（もしくはCPUを指定するブロック）で囲むことで指定できます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Ac9Y64H-qEhX", - "colab": {} - }, - "source": [ - "import time\n", - "\n", - "def measure(x, steps):\n", - " \n", - " # TensorFlowはGPUを初めて使用するときに初期化するため、時間計測対象からは除外する。\n", - " tf.matmul(x, x)\n", - " start = time.time()\n", - " for i in range(steps):\n", - " x = tf.matmul(x, x)\n", - " \n", - " # ｔｆ．ｍａｔｍｕｌは、行列乗算が完了する前に戻ることができます。\n", - " # （たとえば、ＣＵＤＡストリームにオペレーションをエンキューした後に戻すことができます）。\n", - " # 以下のx.numpy（）呼び出しは、すべてのキューに入れられたオペレーションが完了したことを確認します。\n", - " # （そして結果をホストメモリにコピーするため、計算時間は単純なmatmulオペレーションよりも多くのことを含む時間になります。）\n", - " _ = x.numpy()\n", - " end = time.time()\n", - " return end - start\n", - "\n", - "shape = (1000, 1000)\n", - "steps = 200\n", - "print(\"Time to multiply a {} matrix by itself {} times:\".format(shape, steps))\n", - "\n", - "# CPU上で実行するとき：\n", - "with tf.device(\"/cpu:0\"):\n", - " print(\"CPU: {} secs\".format(measure(tf.random_normal(shape), steps)))\n", - "\n", - "# GPU上で実行するとき（GPUが利用できれば）：\n", - "if tfe.num_gpus() > 0:\n", - " with tf.device(\"/gpu:0\"):\n", - " print(\"GPU: {} secs\".format(measure(tf.random_normal(shape), steps)))\n", - "else:\n", - " print(\"GPU: not found\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RLw3IS7UqEhe" - }, - "source": [ - "`tf.Tensor`オブジェクトはそのオブジェクトに対するオペレーションを実行するために別のデバイスにコピーすることができます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "ny6LX2BVqEhf", - "colab": {} - }, - "source": [ - "if tf.test.is_gpu_available():\n", - " x = tf.random_normal([10, 10])\n", - "\n", - " x_gpu0 = x.gpu()\n", - " x_cpu = x.cpu()\n", - "\n", - " _ = tf.matmul(x_cpu, x_cpu) # CPU上で実行するとき\n", - " _ = tf.matmul(x_gpu0, x_gpu0) # GPU:0上で実行するとき\n", - "\n", - " if tfe.num_gpus() > 1:\n", - " x_gpu1 = x.gpu(1)\n", - " _ = tf.matmul(x_gpu1, x_gpu1) # GPU:1で実行するとき" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oA_qaII3-p6c" - }, - "source": [ - "### ベンチマーク\n", - "\n", - "GPUでの\n", - "[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)\n", - "の学習のような、計算量の多いモデルの場合は、Eager ExecutionのパフォーマンスはGraph Executionのパフォーマンスに匹敵します。\n", - "しかし、この2つの環境下のパフォーマンスの違いは計算量の少ないモデルではより大きくなり、小さなたくさんのオペレーションからなるモデルでホットコードパスを最適化するためにやるべきことがあります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TjMTyFIp-sSE" - }, - "source": [ - "## Graph Executionの実行\n", - "\n", - "Eager Executionは開発とデバッグをより対話的にしますが、\n", - "TensorFlowのGraph Executionは分散学習、パフォーマンスの最適化、そしてプロダクション環境へのデプロイの観点で利点があります。\n", - "しかし、Graph Executionのコードの記述方法、標準的なのPythonコードの書き方と異なり、デバッグがより難しく感じるかもしれません。\n", - "\n", - "Graph Execution形式のモデルの構築と学習のために、Pythonプログラムは最初に計算グラフを構築し、\n", - "それからC++ベースのランタイムで実行するために`Session.run`を呼び出し、グラフを渡します。この機能の特徴は以下のとおりです：\n", - "\n", - "* 静的なautodiffによる自動微分\n", - "* プラットフォームに依存しないサーバーへの簡単なデプロイ\n", - "* グラフベースの最適化（共通的な部分式の削除、定数の畳み込みなど）\n", - "* コンパイルとカーネルフュージョン\n", - "* 自動分散とレプリケーション（分散システムへのノード配置）\n", - "\n", - "Eager Executionのコードは、Graph Executionのコードよりもデプロイが難しいです：モデルから\n", - "計算グラフを生成するか、またはサーバ上で直接Pythonランタイムからコードを実行する必要があります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hll3ZbE5qEhh" - }, - "source": [ - "### 互換性のあるコードの記述\n", - "\n", - "Eager Execution環境で記述されたコードは、Eager Executionが有効になっていない新しいPythonセッションでおなじコードを実行するだけで\n", - "おなじコードのままGraph Executionで実行することができます。\n", - "\n", - "ほとんどのTensorFlowオペレーションはEager Executionで動作しますが、注意すべき点がいくつかあります：\n", - "\n", - "* 入力処理にはキューの代わりに `tf.data`を使います。この方法はより高速で簡単です。\n", - "* `tf.keras.layers`や`tf.keras.Model`のような、オブジェクト指向のレイヤーAPIを使用します—これらのAPIは変数のための明示的なストレージを持っているためです。\n", - "* ほとんどのモデルコードは、Eager ExecutionとGraph Executionにおなじように機能しますが、例外があります。\n", - " （たとえば、Pythonによる制御フローで入力に基づいて演算を変更する動的モデルなど）\n", - "* 一度`tf.enable_eager_execution`によってEager Executionが有効化されると、それを無効化することはできません。\n", - " Graph Executionに戻すには、新しいPythonセッションを開始する必要があります。\n", - "\n", - "以上が、Eager Execution *と* Graph Executionの両方のためのコードを書くためのベストプラクティスです。これによって、\n", - "Eager Executionによる対話的な実験とデバッガビリティを享受することができ、かつGraph Executionによる分散パフォーマンスの恩恵を受けることができます。\n", - "\n", - "Eager Executionを用いてコードを記述、デバッグ、実験を繰り返したのちにプロダクションへのデプロイのためにモデルパスをimportします。\n", - "モデル変数を保存および復元するには `tf.train.Checkpoint`を使います。これはEager ExecutionとGraph Executionの両環境の互換性を担保します。\n", - "以下にEager Executionのサンプル集があります： \n", - "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sPoSUqmL-ts5" - }, - "source": [ - "### Graph Execution環境でEager Executionを使う\n", - "\n", - "`tfe.py_func`を使ってTensorFlowGraph Execution環境でEager Executionを選択的に可能にすることができます。\n", - "この機能は、 `tf.enable_eager_execution（）`が呼ばれていないときに使うことができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Lks-3LB0qEhi", - "colab": {} - }, - "source": [ - "def my_py_func(x):\n", - " x = tf.matmul(x, x) # tfオペレーションを使用することができる\n", - " print(x) # しかしEager Executionで実行される！\n", - " return x\n", - "\n", - "with tf.Session() as sess:\n", - " x = tf.placeholder(dtype=tf.float32)\n", - " # Graph Execution環境でEager Executionを呼び出す\n", - " pf = tfe.py_func(my_py_func, [x], tf.float32)\n", - "\n", - " sess.run(pf, feed_dict={x: [[2.0]]}) # [[4.0]]" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/guide/keras.ipynb b/site/ja/r1/guide/keras.ipynb deleted file mode 100644 index ec9f4910cc5..00000000000 --- a/site/ja/r1/guide/keras.ipynb +++ /dev/null @@ -1,1334 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "keras.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCQY7jpBfMur" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "z6X9omPnfO_h", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F1xIRPtY0E1w" - }, - "source": [ - "# Keras" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VyOjQZHhZxaA" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EhmkAPm44hhF", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ViAXWoKlZ8s6" - }, - "source": [ - "Kerasは、深層学習モデルを構築・学習するための高水準APIです。 \n", - "迅速なプロトタイピングから先端研究、実運用にも使用されており、3つの特徴があります：\n", - "\n", - "- ユーザーフレンドリー
    \n", - " 一般的なユースケースに最適化したKerasのAPIは、シンプルで統一性があります。誤った使い方をした場合のエラー出力も明快で、どう対応すべきか一目瞭然です。\n", - "- モジュール性
    \n", - " Kerasのモデルは、設定可能なモジュールをつなぎ合わせて作られます。モジュールのつなぎ方には、ほとんど制約がありません。\n", - "- 拡張性
    \n", - " 簡単にモジュールをカスタマイズできるため、研究の新しいアイデアを試すのに最適です。新しい層、損失関数を自作し、最高水準のモデルを開発しましょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IsK5aF2xZ-40" - }, - "source": [ - "## tf.keras のインポート\n", - "\n", - "`tf.keras` は、TensorFlow版 [Keras API 仕様](https://keras.io) です。 \n", - "モデルを構築・学習するための高水準APIであり、TensorFlow特有の機能である\n", - " [Eagerモード](#eager_execution)や`tf.data` パイプライン、 [Estimators](./estimators.md) にも正式に対応しています。\n", - "`tf.keras` は、TensorFlowの柔軟性やパフォーマンスを損ねることなく使いやすさを向上しています。\n", - "\n", - "TensorFlowプログラムの準備として、先ずは `tf.keras` をインポートしましょう：" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "HOXBMnKbd0jL", - "colab_type": "code", - "colab": {} - }, - "source": [ - "!pip install tensorflow==\"1.*\"" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qoc055N3wiUG", - "colab": {} - }, - "source": [ - "!pip install pyyaml # YAML形式でモデルを保存する際に必要です。" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TgPcBFru0E1z", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.version.VERSION)\n", - "print(tf.keras.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lj03RamP0E13" - }, - "source": [ - "`tf.keras` ではKerasと互換性のあるコードを実行できますが、注意点もあります：\n", - "\n", - "* 最新リリースのTensorFlowに同梱されている `tf.keras` のバージョンと、pipインストールした最新の `keras` のバージョンが同一とは限りません。バージョンは `tf.keras.__version__` の出力をご確認ください。\n", - "* [モデルの重みを保存](#weights_only)する場合、\n", - "`tf.keras` のデフォルトの保存形式は [チェックポイント形式](./checkpoints.md)です。\n", - "HDF5形式にて保存する場合は、 `save_format='h5'` オプションを指定してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7e1LPcXx0gR6" - }, - "source": [ - "## 単純なモデルの構築\n", - "\n", - "### シーケンシャル モデル\n", - "\n", - "Kerasでは、層を組み合わせてモデルを構築します。\n", - "モデルは通常、複数の層から成るグラフ構造をしています。\n", - "最も一般的なモデルは、単純に層を積み重ねる類の `tf.keras.Sequential` モデルです。\n", - "\n", - "単純な全結合ネットワーク（いわゆる マルチ レイヤー パーセプトロン）を構築してみましょう：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WM-DUVQB0E14", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()\n", - "# ユニット数が64の全結合層をモデルに追加します：\n", - "model.add(layers.Dense(64, activation='relu'))\n", - "# 全結合層をもう一つ追加します：\n", - "model.add(layers.Dense(64, activation='relu'))\n", - "# 出力ユニット数が10のソフトマックス層を追加します：\n", - "model.add(layers.Dense(10, activation='softmax'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ztyTipu0E18" - }, - "source": [ - "### 層の設定\n", - "\n", - "`tf.keras.layers` はさまざまな層を提供していますが、共通のコンストラクタ引数があります：\n", - "\n", - "* `activation`： 層の活性化関数を設定します。組み込み関数、もしくは呼び出し可能オブジェクトの名前で指定します。デフォルト値は、活性化関数なし。\n", - "* `kernel_initializer` ・ `bias_initializer`： 層の重み（カーネルとバイアス）の初期化方式。名前、もしくは呼び出し可能オブジェクトで指定します。デフォルト値は、 `\"Glorot uniform\"` 。\n", - "* `kernel_regularizer` ・ `bias_regularizer`：層の重み（カーネルとバイアス）に適用する、L1やL2等の正則化方式。デフォルト値は、正則化なし。\n", - "\n", - "コンストラクタ引数を使って `tf.keras.layers.Dense` 層をインスタンス化する例を以下に示します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MlL7PBtp0E19", - "colab": {} - }, - "source": [ - "# シグモイド層を１層作る場合：\n", - "layers.Dense(64, activation='sigmoid')\n", - "# 別の記法：\n", - "layers.Dense(64, activation=tf.sigmoid)\n", - "\n", - "# カーネル行列に係数0,01のL1正則化を課した全結合層：\n", - "layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l1(0.01))\n", - "\n", - "# バイアスベクトルに係数0,01のL2正則化を課した全結合層：\n", - "layers.Dense(64, bias_regularizer=tf.keras.regularizers.l2(0.01))\n", - "\n", - "# カーネルをランダム直交行列で初期化した全結合層：\n", - "layers.Dense(64, kernel_initializer='orthogonal')\n", - "\n", - "# バイアスベクトルを2.0で初期化した全結合層：\n", - "layers.Dense(64, bias_initializer=tf.keras.initializers.constant(2.0))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9NR6reyk0E2A" - }, - "source": [ - "## 学習と評価\n", - "\n", - "### 学習の準備\n", - "\n", - "モデルを構築したあとは、`compile` メソッドを呼んで学習方法を構成します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sJ4AOn090E2A", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - "# ユニット数64の全結合層をモデルに追加する：\n", - "layers.Dense(64, activation='relu', input_shape=(32,)),\n", - "# もう１層追加する：\n", - "layers.Dense(64, activation='relu'),\n", - "# 出力ユニット数10のソフトマックス層を追加する：\n", - "layers.Dense(10, activation='softmax')])\n", - "\n", - "model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HG-RAa9F0E2D" - }, - "source": [ - "`tf.keras.Model.compile` には3つの重要な引数があります：\n", - "\n", - "* `optimizer`: このオブジェクトが訓練方式を規定します。 `tf.train` モジュールから\n", - " `tf.train.AdamOptimizer`や `tf.train.RMSPropOptimizer`、\n", - " `tf.train.GradientDescentOptimizer`等のオプティマイザ インスタンスを指定します。\n", - "* `loss`: 最適化の過程で最小化する関数を指定します。平均二乗誤差（`mse`）や`categorical_crossentropy`、\n", - " `binary_crossentropy`等が好んで使われます。損失関数は名前、もしくは `tf.keras.losses` モジュールから呼び出し可能オブジェクトとして指定できます。\n", - "* `metrics`: 学習の監視に使用します。 名前、もしくは`tf.keras.metrics` モジュールから呼び出し可能オブジェクトとして指定できます。\n", - "\n", - "学習用モデルの構成例を2つ、以下に示します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "St4Mgdar0E2E", - "colab": {} - }, - "source": [ - "# 平均二乗誤差 回帰モデルを構成する。\n", - "model.compile(optimizer=tf.train.AdamOptimizer(0.01),\n", - " loss='mse', # 平均二乗誤差\n", - " metrics=['mae']) # 平均絶対誤差\n", - "\n", - "# 多クラス分類モデルを構成する。\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.01),\n", - " loss=tf.keras.losses.categorical_crossentropy,\n", - " metrics=[tf.keras.metrics.categorical_accuracy])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yjI5rbi80E2G" - }, - "source": [ - "### NumPy データの入力\n", - "\n", - "小規模なデータセットであれば、モデルを学習・評価する際にインメモリの [NumPy](https://www.numpy.org/)配列を使いましょう。\n", - "モデルは `fit` メソッドを使って学習データに適合させます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3CvP6L-m0E2I", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "def random_one_hot_labels(shape):\n", - " n, n_class = shape\n", - " classes = np.random.randint(0, n_class, n)\n", - " labels = np.zeros((n, n_class))\n", - " labels[np.arange(n), classes] = 1\n", - " return labels\n", - "\n", - "data = np.random.random((1000, 32))\n", - "labels = random_one_hot_labels((1000, 10))\n", - "\n", - "model.fit(data, labels, epochs=10, batch_size=32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N-pnVaFe0E2N" - }, - "source": [ - "`tf.keras.Model.fit` は3つの重要な引数があります：\n", - "\n", - "* `epochs`: **エポック** は学習の構成単位で、（バッチに分割した）全入力データを一巡したものを1エポックと換算します。\n", - "* `batch_size`: NumPyデータを渡されたモデルは、データをバッチに分割し、それを順繰りに舐めて学習を行います。一つのバッチに配分するサンプル数を、バッチサイズとして整数で指定します。全サンプル数がバッチサイズで割り切れない場合、最後のバッチだけ小さくなる可能性があることに注意しましょう。\n", - "* `validation_data`: モデルの試作中に評価データを使って簡単にパフォーマンスを監視したい場合は、この引数に入力とラベルの対を渡すことで、各エポックの最後に推論モードで評価データの損失と評価指標を表示することができます。\n", - "\n", - "`validation_data` の使用例：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gFcXzVQa0E2N", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "data = np.random.random((1000, 32))\n", - "labels = random_one_hot_labels((1000, 10))\n", - "\n", - "val_data = np.random.random((100, 32))\n", - "val_labels = random_one_hot_labels((100, 10))\n", - "\n", - "model.fit(data, labels, epochs=10, batch_size=32,\n", - " validation_data=(val_data, val_labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-6ImyXzz0E2Q" - }, - "source": [ - "### tf.data データセットの入力\n", - "\n", - "大規模なデータセット、もしくは複数デバイスを用いた学習を行う際は [Datasets API](./datasets.md) を使いましょう。 `fit`メソッドに`tf.data.Dataset` インスタンスを渡します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OziqhpIj0E2R", - "colab": {} - }, - "source": [ - "# データセットのインスタンス化の例：\n", - "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n", - "dataset = dataset.batch(32)\n", - "dataset = dataset.repeat()\n", - "\n", - "# `fit` にデータセットを渡す際は、`steps_per_epoch` の指定をお忘れなく：\n", - "model.fit(dataset, epochs=10, steps_per_epoch=30)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "I7BcMHkB0E2U" - }, - "source": [ - "`fit` メソッドの引数 `steps_per_epoch` には、1エポックあたりの学習ステップ数を指定します。\n", - "`Dataset` がバッチを生成するため `batch_size`の指定は不要です。\n", - "\n", - "`Dataset` は評価データにも使えます：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YPMb3A0N0E2V", - "colab": {} - }, - "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n", - "dataset = dataset.batch(32).repeat()\n", - "\n", - "val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_labels))\n", - "val_dataset = val_dataset.batch(32).repeat()\n", - "\n", - "model.fit(dataset, epochs=10, steps_per_epoch=30,\n", - " validation_data=val_dataset,\n", - " validation_steps=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IgGdlXso0E2X" - }, - "source": [ - "### 評価と推論\n", - "\n", - "`tf.keras.Model.evaluate` と `tf.keras.Model.predict` メソッドは、NumPyデータと`tf.data.Dataset`に使えます。\n", - "\n", - "推論モードでデータの損失と評価指標を**評価**する例を示します： " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mhDbOHEK0E2Y", - "colab": {} - }, - "source": [ - "data = np.random.random((1000, 32))\n", - "labels = random_one_hot_labels((1000, 10))\n", - "\n", - "model.evaluate(data, labels, batch_size=32)\n", - "\n", - "model.evaluate(dataset, steps=30)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UXUTmDfb0E2b" - }, - "source": [ - "**推論** 結果を最終層のNumPy配列として出力する例を示します:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9e3JsSoQ0E2c", - "colab": {} - }, - "source": [ - "result = model.predict(data, batch_size=32)\n", - "print(result.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fzEOW4Cn0E2h" - }, - "source": [ - "## 高度なモデルの構築\n", - "\n", - "### Functional API\n", - "\n", - "`tf.keras.Sequential` モデルは層を積み重ねる単純なつくりであり、あらゆるモデルに対応しているわけではありません。\n", - "以下に挙げる複雑な構成のモデルを構築するには\n", - "[Keras functional API](https://keras.io/getting-started/functional-api-guide/)\n", - "を使いましょう：\n", - "\n", - "* 入力ヘッドが複数あるモデル\n", - "* 出力ヘッドが複数あるモデル\n", - "* 共有層（おなじ層が複数回呼び出される）を含むモデル\n", - "* （残差結合のように）データの流れが分岐するモデル \n", - "\n", - "Functional API を用いたモデル構築の流れ：\n", - "\n", - "1. 層のインスタンスは呼び出し可能で、テンソルを返します。\n", - "2. 入力テンソルと出力テンソルを使って`tf.keras.Model`インスタンスを定義します。\n", - "3. モデルは`Sequential`モデルと同様の方法で学習します。\n", - "\n", - "Functional API を使って全結合ネットワークを構築する例を示します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mROj832r0E2i", - "colab": {} - }, - "source": [ - "inputs = tf.keras.Input(shape=(32,)) # プレイスホルダのテンソルを返します。\n", - "\n", - "# 層のインスタンスは呼び出し可能で、テンソルを返します。\n", - "x = layers.Dense(64, activation='relu')(inputs)\n", - "x = layers.Dense(64, activation='relu')(x)\n", - "predictions = layers.Dense(10, activation='softmax')(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AFmspHeG1_W7" - }, - "source": [ - "inputsとoutputsを引数にモデルをインスタンス化します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5k5uzlyu16HM", - "colab": {} - }, - "source": [ - "model = tf.keras.Model(inputs=inputs, outputs=predictions)\n", - "\n", - "# コンパイル時に学習方法を指定します。\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# 5エポック学習します。\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EcKSLH3i0E2k" - }, - "source": [ - "### モデルの派生\n", - "\n", - "`tf.keras.Model` を継承し順伝播を定義することでカスタムモデルを構築できます。\n", - "`__init__` メソッドにクラス インスタンスの属性として層をつくります。\n", - "`call` メソッドに順伝播を定義します。 \n", - "\n", - "順伝播を命令型で記載できるため、モデルの派生は\n", - "[Eagerモード](./eager.md) でより威力を発揮します。\n", - "\n", - "キーポイント：目的にあったAPIを選択しましょう。派生モデルは柔軟性を与えてくれますが、その代償にモデルはより複雑になりエラーを起こしやすくなります。目的がFunctional APIで賄えるのであれば、そちらを使いましょう。\n", - "\n", - "`tf.keras.Model`を継承して順伝播をカスタマイズした例を以下に示します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KLiHWzcn2Fzk", - "colab": {} - }, - "source": [ - "class MyModel(tf.keras.Model):\n", - "\n", - " def __init__(self, num_classes=10):\n", - " super(MyModel, self).__init__(name='my_model')\n", - " self.num_classes = num_classes\n", - " # 層をここに定義します。\n", - " self.dense_1 = layers.Dense(32, activation='relu')\n", - " self.dense_2 = layers.Dense(num_classes, activation='sigmoid')\n", - "\n", - " def call(self, inputs):\n", - " # （`__init__`）にてあらかじめ定義した層を使って\n", - " # 順伝播をここに定義します。\n", - " x = self.dense_1(inputs)\n", - " return self.dense_2(x)\n", - "\n", - " def compute_output_shape(self, input_shape):\n", - " # 派生モデルを使用する場合、\n", - " # このメソッドをオーバーライドすることになります。\n", - " # 派生モデルを使用しない場合、このメソッドは省略可能です。\n", - " shape = tf.TensorShape(input_shape).as_list()\n", - " shape[-1] = self.num_classes\n", - " return tf.TensorShape(shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ShDD4fv72KGc" - }, - "source": [ - "今定義した派生モデルをインスンス化します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "42C-qQHm0E2l", - "colab": {} - }, - "source": [ - "model = MyModel(num_classes=10)\n", - "\n", - "# コンパイル時に学習方法を指定します。\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# 5エポック学習します。\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yqRQiKj20E2o" - }, - "source": [ - "### 層のカスタマイズ\n", - "\n", - "`tf.keras.layers.Layer`を継承して層をカスタマイズするには、以下のメソッドを実装します： \n", - "\n", - "* `build`： 層の重みを定義します。`add_weight`メソッドで重みを追加します。\n", - "* `call`： 順伝播を定義します。\n", - "* `compute_output_shape`: 入力の形状をもとに出力の形状を算出する方法を指定します。\n", - "* 必須ではありませんが、`get_config`メソッド と `from_config` クラスメソッドを実装することで層をシリアライズすることができます。\n", - "\n", - "入力のカーネル行列を `matmul` （行列乗算）するカスタム層の実装例：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "l7BFnIHr2WNc", - "colab": {} - }, - "source": [ - "class MyLayer(layers.Layer):\n", - "\n", - " def __init__(self, output_dim, **kwargs):\n", - " self.output_dim = output_dim\n", - " super(MyLayer, self).__init__(**kwargs)\n", - "\n", - " def build(self, input_shape):\n", - " shape = tf.TensorShape((input_shape[1], self.output_dim))\n", - " # 学習可能な重みを指定します。\n", - " self.kernel = self.add_weight(name='kernel',\n", - " shape=shape,\n", - " initializer='uniform',\n", - " trainable=True)\n", - " # 最後に`build` メソッドを呼ぶのをお忘れなく。\n", - " super(MyLayer, self).build(input_shape)\n", - "\n", - " def call(self, inputs):\n", - " return tf.matmul(inputs, self.kernel)\n", - "\n", - " def compute_output_shape(self, input_shape):\n", - " shape = tf.TensorShape(input_shape).as_list()\n", - " shape[-1] = self.output_dim\n", - " return tf.TensorShape(shape)\n", - "\n", - " def get_config(self):\n", - " base_config = super(MyLayer, self).get_config()\n", - " base_config['output_dim'] = self.output_dim\n", - " return base_config\n", - "\n", - " @classmethod\n", - " def from_config(cls, config):\n", - " return cls(**config)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8wXDRgXV2ZrF" - }, - "source": [ - "カスタマイズした層を使ってモデルを構築します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uqH-cY0h0E2p", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " MyLayer(10),\n", - " layers.Activation('softmax')])\n", - "\n", - "# コンパイル時に学習方法を指定します。\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# 5エポック学習します。\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Lu8cc3AJ0E2v" - }, - "source": [ - "## コールバック\n", - "\n", - "コールバックは、学習中のモデルの挙動をカスタマイズするためにモデルに渡されるオブジェクトです。\n", - "コールバック関数は自作する、もしくは以下に示す`tf.keras.callbacks`が提供する組み込み関数を利用できます：\n", - "\n", - "* `tf.keras.callbacks.ModelCheckpoint`：モデルのチェックポイントを一定間隔で保存します。\n", - "* `tf.keras.callbacks.LearningRateScheduler`：学習率を動的に変更します。\n", - "* `tf.keras.callbacks.EarlyStopping`：評価パフォーマンスが向上しなくなったら学習を中断させます。\n", - "* `tf.keras.callbacks.TensorBoard`： モデルの挙動を\n", - " [TensorBoard](./summaries_and_tensorboard.md)で監視します。\n", - "\n", - "`tf.keras.callbacks.Callback`を使用するには、モデルの `fit` メソッドにコールバック関数を渡します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rdYwzSYV0E2v", - "colab": {} - }, - "source": [ - "callbacks = [\n", - " # `val_loss` が2エポック経っても改善しなければ学習を中断させます。\n", - " tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'),\n", - " # TensorBoard用ログを`./logs` ディレクトリに書き込みます。\n", - " tf.keras.callbacks.TensorBoard(log_dir='./logs')\n", - "]\n", - "model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks,\n", - " validation_data=(val_data, val_labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ghhaGfX62abv" - }, - "source": [ - "\n", - "## 保存と復元" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qnl7K-aI0E2z" - }, - "source": [ - "### 重みのみ\n", - "\n", - "`tf.keras.Model.save_weights`を使ってモデルの重みの保存やロードを行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uQIANjB94fLB", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - "layers.Dense(64, activation='relu', input_shape=(32,)),\n", - "layers.Dense(10, activation='softmax')])\n", - "\n", - "model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4eoHJ-ny0E21", - "colab": {} - }, - "source": [ - "# TensorFlow チェックポイント ファイルに重みを保存します。\n", - "model.save_weights('./weights/my_model')\n", - "\n", - "# モデルの状態を復元します。\n", - "# 復元対象のモデルと保存されていた重みのモデル構造が同一である必要があります。\n", - "model.load_weights('./weights/my_model')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u25Id3xe0E25" - }, - "source": [ - "デフォルトでは、モデルの重みは\n", - "[TensorFlow チェックポイント](./checkpoints.md) 形式で保存されます。\n", - "重みはKerasのHDF5形式でも保存できます（マルチバックエンド実装のKerasではHDF5形式がデフォルト）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JSAYoFEd0E26", - "colab": {} - }, - "source": [ - "# 重みをHDF5形式で保存します。\n", - "model.save_weights('my_model.h5', save_format='h5')\n", - "\n", - "# モデルの状態を復元します。\n", - "model.load_weights('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mje_yKL10E29" - }, - "source": [ - "### 構成のみ\n", - "\n", - "モデルの構成も保存可能です。\n", - "モデル構造を重み抜きでシリアライズします。\n", - "元のモデルのコードがなくとも、保存された構成で再構築できます。\n", - "Kerasがサポートしているシリアライズ形式は、JSONとYAMLです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EbET0oJTzGkq", - "colab": {} - }, - "source": [ - "# JSON形式にモデルをシリアライズします\n", - "json_string = model.to_json()\n", - "json_string" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pX_badhH3yWV", - "colab": {} - }, - "source": [ - "import json\n", - "import pprint\n", - "pprint.pprint(json.loads(json_string))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q7CIa05r4yTb" - }, - "source": [ - "JSONから（新たに初期化して）モデルを再構築します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J9UFv9k00E2_", - "colab": {} - }, - "source": [ - "fresh_model = tf.keras.models.model_from_json(json_string)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t5NHtICh4uHK" - }, - "source": [ - "YAML形式でモデルを保存するには、\n", - "**TensorFlowをインポートする前に** あらかじめ`pyyaml`をインストールしておく必要があります：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aj24KB3Z36S4", - "colab": {} - }, - "source": [ - "yaml_string = model.to_yaml()\n", - "print(yaml_string)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O53Kerfl43v7" - }, - "source": [ - "YAMLからモデルを再構築します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "77yRuwg03_MG", - "colab": {} - }, - "source": [ - "fresh_model = tf.keras.models.model_from_yaml(yaml_string)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xPvOSSzM0E3B" - }, - "source": [ - "注意：`call`メソッド内ににPythonコードでモデル構造を定義するため、派生モデルはシリアライズできません。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iu8qMwld4-71" - }, - "source": [ - "\n", - "### モデル全体\n", - "\n", - "モデルの重み、構成からオプティマイザ設定までモデル全体をファイルに保存できます。\n", - "そうすることで、元のコードなしに、チェックポイントで保存したときと全く同じ状態から学習を再開できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "45oNY34Z0E3C", - "colab": {} - }, - "source": [ - "# 層の浅いモデルを構築します。\n", - "model = tf.keras.Sequential([\n", - " layers.Dense(64, activation='relu', input_shape=(32,)),\n", - " layers.Dense(10, activation='softmax')\n", - "])\n", - "model.compile(optimizer='rmsprop',\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "model.fit(data, labels, batch_size=32, epochs=5)\n", - "\n", - "\n", - "# HDF5ファイルにモデル全体を保存します。\n", - "model.save('my_model.h5')\n", - "\n", - "# 重みとオプティマイザを含む 全く同一のモデルを再構築します。\n", - "model = tf.keras.models.load_model('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PMOWhDOB0E3E" - }, - "source": [ - "## Eagerモード\n", - "\n", - "[Eagerモード](./eager.md) は、オペレーションを即時に評価する命令型のプログラミング環境です。\n", - "Kerasでは必要ありませんが、`tf.keras`でサポートされておりプログラムを検査しデバッグするのに便利です。\n", - "\n", - "すべての`tf.keras`モデル構築用APIは、Eagerモード互換性があります。\n", - "`Sequential` や Functional APIも使用できますが、\n", - "Eagerモードは特に**派生モデル** の構築や\n", - "**層のカスタマイズ**に有益です。\n", - "（既存の層の組み合わせでモデルを作成するAPIの代わりに）\n", - "順伝播をコードで実装する必要があります。\n", - "\n", - "詳しくは [Eagerモード ガイド](./eager.md#build_a_model) \n", - "（カスタマイズした学習ループと`tf.GradientTape`を使ったKerasモデルの適用事例）をご参照ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2wG3NVco5B5V" - }, - "source": [ - "## 分散\n", - "\n", - "### Estimators\n", - "\n", - "[Estimators](./estimators.md) は分散学習を行うためのAPIです。\n", - "実運用に耐えるモデルを巨大なデータセットを用いて分散学習するといった産業利用を目的にすえています。\n", - "\n", - "`tf.keras.Model`で`tf.estimator` APIによる学習を行うには、\n", - "`tf.keras.estimator.model_to_estimator`を使ってKerasモデルを `tf.estimator.Estimator`オブジェクトに変換する必要があります。\n", - "\n", - "[KerasモデルからEstimatorsを作成する](./estimators.md#creating_estimators_from_keras_models)をご参照ください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cVg0vfTO0E3E", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([layers.Dense(64, activation='relu', input_shape=(32,)),\n", - " layers.Dense(10,activation='softmax')])\n", - "\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "estimator = tf.keras.estimator.model_to_estimator(model)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S7FKvikO0E3H" - }, - "source": [ - "注意：[Estimator input functions](./premade_estimators.md#create_input_functions)をデバッグしてデータの検査を行うには[Eagerモード](./eager.md)で実行してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PJZ6e9J5JHF" - }, - "source": [ - "### マルチGPU\n", - "\n", - "`tf.keras`モデルは`tf.contrib.distribute.DistributionStrategy`を使用することでマルチGPU上で実行できます。\n", - "このAPIを使えば、既存コードをほとんど改変することなく分散学習へ移行できます。\n", - "\n", - "目下、分散方式として`tf.contrib.distribute.MirroredStrategy`のみサポートしています。\n", - "`MirroredStrategy` は、シングルマシン上でAllReduce を使った同期学習によりin-grapnレプリケーションを行います。\n", - "Kerasで`DistributionStrategy`を使用する場合は、`tf.keras.estimator.model_to_estimator`を使って\n", - "`tf.keras.Model` を`tf.estimator.Estimator`に変換し、Estimatorインスタンスを使って分散学習を行います。\n", - "\n", - "以下の例では、シングルマシンのマルチGPUに`tf.keras.Model`を分散します。\n", - "\n", - "まず、単純なモデルを定義します：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sbaRr7g-0E3I", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()\n", - "model.add(layers.Dense(16, activation='relu', input_shape=(10,)))\n", - "model.add(layers.Dense(1, activation='sigmoid'))\n", - "\n", - "optimizer = tf.train.GradientDescentOptimizer(0.2)\n", - "\n", - "model.compile(loss='binary_crossentropy', optimizer=optimizer)\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yw4hSJme0E3L" - }, - "source": [ - "**入力パイプライン**を定義します。`input_fn` は、複数デバイスにデータを配置するのに使用する `tf.data.Dataset` を返します。\n", - "各デバイスは、入力バッチの一部（デバイス間で均等に分割）を処理します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CxJW1QMH0E3L", - "colab": {} - }, - "source": [ - "def input_fn():\n", - " x = np.random.random((1024, 10))\n", - " y = np.random.randint(2, size=(1024, 1))\n", - " x = tf.cast(x, tf.float32)\n", - " dataset = tf.data.Dataset.from_tensor_slices((x, y))\n", - " dataset = dataset.repeat(10)\n", - " dataset = dataset.batch(32)\n", - " return dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rO9MiL6X0E3O" - }, - "source": [ - "次に、 `tf.estimator.RunConfig`を作成し、 `train_distribute` 引数に`tf.contrib.distribute.MirroredStrategy` インスタンスを設定します。`MirroredStrategy`を作成する際、デバイスの一覧を指定する、もしくは引数で`num_gpus`（GPU数）を設定することができます。デフォルトでは、使用可能なすべてのGPUを使用する設定になっています：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BEwFq4PM0E3P", - "colab": {} - }, - "source": [ - "strategy = tf.contrib.distribute.MirroredStrategy()\n", - "config = tf.estimator.RunConfig(train_distribute=strategy)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TcnwYVun0E3R" - }, - "source": [ - "Kerasモデルを `tf.estimator.Estimator` インスタンスへ変換します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VSvbuIID0E3S", - "colab": {} - }, - "source": [ - "keras_estimator = tf.keras.estimator.model_to_estimator(\n", - " keras_model=model,\n", - " config=config,\n", - " model_dir='/tmp/model_dir')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N6BXU5F90E3U" - }, - "source": [ - "最後に、`input_fn` と `steps`引数を指定して `Estimator` インスタンスを学習します： " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKoJ2wUH0E3U", - "colab": {} - }, - "source": [ - "keras_estimator.train(input_fn=input_fn, steps=10)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/images/hub_with_keras.ipynb b/site/ja/r1/tutorials/images/hub_with_keras.ipynb deleted file mode 100644 index adbce2400fd..00000000000 --- a/site/ja/r1/tutorials/images/hub_with_keras.ipynb +++ /dev/null @@ -1,1032 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "hub_with_keras.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "W_tvPdyfA-BL" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W_tvPdyfA-BL" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "0O_LFhwSBCjm", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PWUmcKKjtwXL" - }, - "source": [ - "# Hub with Keras" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bAF5fnbcq5_Q", - "colab_type": "text" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MzXvBPp0IkoD", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "crU-iluJIEzw" - }, - "source": [ - "[TensorFlow Hub](http://tensorflow.org/hub) は事前学習済みモデルのコンポーネントを共有する一つの方法です。事前学習済みモデルの検索可能なリストは [TensorFlow Module Hub](https://tfhub.dev/) をご覧ください。\n", - "\n", - "このチュートリアルでは以下の事を説明します。\n", - "\n", - "1. tf.keras での TensorFlow Hub の使い方\n", - "1. TensorFlow Hub を使って画像を分類する方法\n", - "1. シンプルな転移学習の方法" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CKFUvuEho9Th" - }, - "source": [ - "## セットアップ" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7RVsYZLEpEWs" - }, - "source": [ - "### インポート" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nSiOCtv_Rwi_", - "colab": {} - }, - "source": [ - "!pip install -U tensorflow_hub\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OGNpmn43C0O6", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import matplotlib.pylab as plt\n", - "\n", - "import tensorflow as tf\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-V4l8oN8Lw2q", - "colab": {} - }, - "source": [ - "import tensorflow_hub as hub\n", - "\n", - "from tensorflow.keras import layers" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "s4YuF5HvpM1W" - }, - "source": [ - "## ImageNet 分類器" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xEY_Ow5loN6q" - }, - "source": [ - "### 分類器をダウンロードする\n", - "\n", - "mobilenet をロードするには hub.module を、それを keras レイヤーとしてまとめるには tf.keras.layers.Lambda を使ってください。\n", - "\n", - "tfhub.dev にある [TensorFlow 1.x の画像分類器 URL](https://tfhub.dev/s?module-type=image-classification) であればすべてここで動作します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "both", - "colab_type": "code", - "id": "feiXojVXAbI9", - "colab": {} - }, - "source": [ - "classifier_url =\"https://tfhub.dev/google/tf2-preview/mobilenet_v2/classification/2\" #@param {type:\"string\"}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y_6bGjoPtzau", - "colab": {} - }, - "source": [ - "IMAGE_SHAPE = (224, 224)\n", - "\n", - "classifier = tf.keras.Sequential([\n", - " hub.KerasLayer(classifier_url, input_shape=IMAGE_SHAPE+(3,))\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pwZXaoV0uXp2" - }, - "source": [ - "### 単一の画像で実行する" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TQItP1i55-di" - }, - "source": [ - "モデルを試すために単一の画像をダウンロードしてください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "w5wDjXNjuXGD", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "import PIL.Image as Image\n", - "\n", - "grace_hopper = tf.keras.utils.get_file('image.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg')\n", - "grace_hopper = Image.open(grace_hopper).resize(IMAGE_SHAPE)\n", - "grace_hopper" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BEmmBnGbLxPp", - "colab": {} - }, - "source": [ - "grace_hopper = np.array(grace_hopper)/255.0\n", - "grace_hopper.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Ic8OEEo2b73" - }, - "source": [ - "バッチ次元を一つ追加し、画像をモデルに渡してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EMquyn29v8q3", - "colab": {} - }, - "source": [ - "result = classifier.predict(grace_hopper[np.newaxis, ...])\n", - "result.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NKzjqENF6jDF" - }, - "source": [ - "結果は 1001 の要素をもつロジットベクトルで、画像がそれぞれのクラスに属する確率を表します。\n", - "\n", - "そのため、もっとも確率の高いクラスの ID は argmax でみつけることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rgXb44vt6goJ", - "colab": {} - }, - "source": [ - "predicted_class = np.argmax(result[0], axis=-1)\n", - "predicted_class" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YrxLMajMoxkf" - }, - "source": [ - "### 予測結果をデコードする\n", - "\n", - "予測されたクラスの ID を ImageNet のラベルと突き合わせて、予測結果をデコードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ij6SrDxcxzry", - "colab": {} - }, - "source": [ - "labels_path = tf.keras.utils.get_file('ImageNetLabels.txt','https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt')\n", - "imagenet_labels = np.array(open(labels_path).read().splitlines())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uzziRK3Z2VQo", - "colab": {} - }, - "source": [ - "plt.imshow(grace_hopper)\n", - "plt.axis('off')\n", - "predicted_class_name = imagenet_labels[predicted_class]\n", - "_ = plt.title(\"Prediction: \" + predicted_class_name.title())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "amfzqn1Oo7Om" - }, - "source": [ - "## シンプルな転移学習" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K-nIpVJ94xrw" - }, - "source": [ - "TF Hub を利用すると、私たちが用意したデータセット内のクラスを認識するために、モデルの最上位層を再学習する事が容易になります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z93vvAdGxDMD" - }, - "source": [ - "### データセット\n", - "\n", - "この例では、TensorFlow の花データセットを使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DrIUV3V0xDL_", - "colab": {} - }, - "source": [ - "data_root = tf.keras.utils.get_file(\n", - " 'flower_photos','https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", - " untar=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jFHdp18ccah7" - }, - "source": [ - "このデータをモデルに読み込むもっとも簡単な方法は、 tf.keras.preprocessing.image.ImageDataGenerator を使用することです。\n", - "\n", - "TensorFlow Hub のすべての画像モジュールは、 [0, 1] の範囲の float 入力を想定しています。これを実現するには ImageDataGenerator の rescale パラメータを使用してください。\n", - "\n", - "画像のサイズについては後で処理されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2PwQ_wYDcii9", - "colab": {} - }, - "source": [ - "image_generator = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1/255)\n", - "image_data = image_generator.flow_from_directory(str(data_root), target_size=IMAGE_SHAPE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0p7iDOhIcqY2" - }, - "source": [ - "結果のオブジェクトは、image_batch と label_batch のペアを返すイテレータです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W4lDPkn2cpWZ", - "colab": {} - }, - "source": [ - "for image_batch, label_batch in image_data:\n", - " print(\"Image batch shape: \", image_batch.shape)\n", - " print(\"Label batch shape: \", label_batch.shape)\n", - " break" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0gTN7M_GxDLx" - }, - "source": [ - "### 画像のバッチに対して分類器を実行する" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O3fvrZR8xDLv" - }, - "source": [ - "それでは、画像のバッチで分類器を実行します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nbyg6tcyxDLh", - "colab": {} - }, - "source": [ - "result_batch = classifier.predict(image_batch)\n", - "result_batch.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Kv7ZwuR4xDLc", - "colab": {} - }, - "source": [ - "predicted_class_names = imagenet_labels[np.argmax(result_batch, axis=-1)]\n", - "predicted_class_names" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QmvSWg9nxDLa" - }, - "source": [ - "これらの予測結果と実際の画像がどの程度一致しているか確認してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IXTB22SpxDLP", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,9))\n", - "plt.subplots_adjust(hspace=0.5)\n", - "for n in range(30):\n", - " plt.subplot(6,5,n+1)\n", - " plt.imshow(image_batch[n])\n", - " plt.title(predicted_class_names[n])\n", - " plt.axis('off')\n", - "_ = plt.suptitle(\"ImageNet predictions\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FUa3YkvhxDLM" - }, - "source": [ - "画像の属性については、 LICENSE.txt ファイルを参照してください。\n", - "\n", - "結果は完全には程遠いですが、これらがこのモデルのために学習されたクラスでない事を考えると悪くはないです（ 「デイジー」を除く）。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JzV457OXreQP" - }, - "source": [ - "### ヘッドレスモデルをダウンロードする\n", - "\n", - "TensorFlow Hub はさらに、最上位の分類層がないモデルを配布することもしています。これらは転移学習を簡単に行うために使用することができます。\n", - "\n", - "tfhub.dev からの [TensorFlow 1.x の画像特徴ベクトルの URL](https://tfhub.dev/s?module-type=image-feature-vector) はすべてここで動作します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "both", - "colab_type": "code", - "id": "4bw8Jf94DSnP", - "colab": {} - }, - "source": [ - "feature_extractor_url = \"https://tfhub.dev/google/tf2-preview/mobilenet_v2/feature_vector/2\" #@param {type:\"string\"}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sgwmHugQF-PD" - }, - "source": [ - "モジュールを作成し、入力として期待される画像サイズを確認します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5wB030nezBwI", - "colab": {} - }, - "source": [ - "feature_extractor_layer = hub.KerasLayer(feature_extractor_url,\n", - " input_shape=(224,224,3))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GUY-5Eyzuzlu" - }, - "source": [ - "特徴抽出器は、各画像に対して 1280 要素のベクトルを返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Of7i-35F09ls", - "colab": {} - }, - "source": [ - "feature_batch = feature_extractor_layer(image_batch)\n", - "print(feature_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CtFmF7A5E4tk" - }, - "source": [ - "学習が新しい分類層のみを修正するように、特徴抽出層の変数を freeze（固定）します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Jg5ar6rcE4H-", - "colab": {} - }, - "source": [ - "feature_extractor_layer.trainable = False" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RPVeouTksO9q" - }, - "source": [ - "### 分類ヘッドを追加する\n", - "\n", - "hub レイヤーを tf.keras.Sequential モデルでラップし、新しい分類層を一つ追加します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mGcY27fY1q3Q", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " feature_extractor_layer,\n", - " layers.Dense(image_data.num_classes, activation='softmax')\n", - "])\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "G9VkAz00HOJx", - "colab": {} - }, - "source": [ - "predictions = model(image_batch)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sB7sVGJ23vrY", - "colab": {} - }, - "source": [ - "predictions.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OHbXQqIquFxQ" - }, - "source": [ - "### モデルを学習する\n", - "\n", - "学習プロセスを構成するために、 compile を使用してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3n0Wb9ylKd8R", - "colab": {} - }, - "source": [ - "model.compile(\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " loss='categorical_crossentropy',\n", - " metrics=['acc'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "58-BLV7dupJA" - }, - "source": [ - "今度は .fit メソッドを使ってモデルを学習します。\n", - "\n", - "この例を短くするために、2エポックだけ学習を実行します。学習の進行状況を可視化するには、エポック平均ではなく、カスタムコールバックを使用して各バッチの損失と精度を個別に記録します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jZ54Gubac4Lu", - "colab": {} - }, - "source": [ - "class CollectBatchStats(tf.keras.callbacks.Callback):\n", - " def __init__(self):\n", - " self.batch_losses = []\n", - " self.batch_acc = []\n", - "\n", - " def on_train_batch_end(self, batch, logs=None):\n", - " self.batch_losses.append(logs['loss'])\n", - " self.batch_acc.append(logs['acc'])\n", - " self.model.reset_metrics()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EyMDJxt2HdHr", - "colab": {} - }, - "source": [ - "steps_per_epoch = np.ceil(image_data.samples/image_data.batch_size)\n", - "\n", - "batch_stats_callback = CollectBatchStats()\n", - "\n", - "history = model.fit(image_data, epochs=2,\n", - " steps_per_epoch=steps_per_epoch,\n", - " callbacks = [batch_stats_callback])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Kd0N272B9Q0b" - }, - "source": [ - "これで、ほんの数回の学習の繰り返しでも、モデルがタスクを進めていることがわかったと思います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "A5RfS1QIIP-P", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.ylabel(\"Loss\")\n", - "plt.xlabel(\"Training Steps\")\n", - "plt.ylim([0,2])\n", - "plt.plot(batch_stats_callback.batch_losses)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3uvX11avTiDg", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.ylabel(\"Accuracy\")\n", - "plt.xlabel(\"Training Steps\")\n", - "plt.ylim([0,1])\n", - "plt.plot(batch_stats_callback.batch_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kb__ZN8uFn-D" - }, - "source": [ - "### 予測結果を確認する\n", - "\n", - "前からプロットをやり直すには、まずクラス名の順序付きリストを取得します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JGbEf5l1I4jz", - "colab": {} - }, - "source": [ - "class_names = sorted(image_data.class_indices.items(), key=lambda pair:pair[1])\n", - "class_names = np.array([key.title() for key, value in class_names])\n", - "class_names" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4Olg6MsNGJTL" - }, - "source": [ - "モデルを介してイメージバッチを実行し、インデックスをクラス名に変換します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fCLVCpEjJ_VP", - "colab": {} - }, - "source": [ - "predicted_batch = model.predict(image_batch)\n", - "predicted_id = np.argmax(predicted_batch, axis=-1)\n", - "predicted_label_batch = class_names[predicted_id]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CkGbZxl9GZs-" - }, - "source": [ - "結果をプロットします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rpFQR1MPMtT1", - "colab": {} - }, - "source": [ - "label_id = np.argmax(label_batch, axis=-1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wC_AYRJU9NQe", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,9))\n", - "plt.subplots_adjust(hspace=0.5)\n", - "for n in range(30):\n", - " plt.subplot(6,5,n+1)\n", - " plt.imshow(image_batch[n])\n", - " color = \"green\" if predicted_id[n] == label_id[n] else \"red\"\n", - " plt.title(predicted_label_batch[n].title(), color=color)\n", - " plt.axis('off')\n", - "_ = plt.suptitle(\"Model predictions (green: correct, red: incorrect)\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uRcJnAABr22x" - }, - "source": [ - "## モデルをエクスポートする\n", - "\n", - "モデルの学習が完了したので、saved model としてエクスポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "PLcqg-RmsLno", - "colab": {} - }, - "source": [ - "import time\n", - "t = time.time()\n", - "\n", - "export_path = \"/tmp/saved_models/{}\".format(int(t))\n", - "tf.keras.experimental.export_saved_model(model, export_path)\n", - "\n", - "export_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AhQ9liIUsPsi" - }, - "source": [ - "それではモデルをリロードできることを確認してください。リロードしたモデルでもおなじ結果が得られます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7nI5fvkAQvbS", - "colab": {} - }, - "source": [ - "reloaded = tf.keras.experimental.load_from_saved_model(export_path, custom_objects={'KerasLayer':hub.KerasLayer})" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jor83-LqI8xW", - "colab": {} - }, - "source": [ - "result_batch = model.predict(image_batch)\n", - "reloaded_result_batch = reloaded.predict(image_batch)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dnZO14taYPH6", - "colab": {} - }, - "source": [ - "abs(reloaded_result_batch - result_batch).max()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TYZd4MNiV3Rc" - }, - "source": [ - "この saved model は後で推論のためにロードするか、もしくは [TFLite](https://www.tensorflow.org/lite/convert/) か [TFjs](https://github.com/tensorflow/tfjs-converter).に変換することができます。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/keras/README.md b/site/ja/r1/tutorials/keras/README.md deleted file mode 100644 index 5d610c4057f..00000000000 --- a/site/ja/r1/tutorials/keras/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# 機械学習の学習と実践 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -この一連のノートブックは、書籍 -*[Deep Learning with Python](https://books.google.com/books?id=Yo3CAQAACAAJ)* -（邦訳 -「[PythonとKerasによるディープラーニング](https://book.mynavi.jp/ec/products/detail/id=90124)」） -に触発された ものです。 これらのチュートリアルでは、TensorFlowでディープラーニングモデルの 構築と訓練を行うためのハイレベルなPython -APIである`tf.keras`を 使用しています。KerasをTensorFlowとともに使う方法の詳細は、 -[TensorFlow Keras Guide](../../guide/keras.ipynb)を参照してください。 - -出版社から：*Deep Learning with Python*では、Python言語と強力なKeras -ライブラリを使ってディープラーニングを紹介しています。 -著者はKerasの作者でGoogleのAI研究者でもあるFrançois Cholletです。 -この本では、直感的な説明と実践的な例を通して理解を深めることができます。 - -機械学習の基礎と概念を学ぶには、[Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/)をおすすめします。 - -1. [分類問題の基本](./basic_classification.ipynb) -2. [テキスト分類](./basic_text_classification.ipynb) -3. [回帰](./basic_regression.ipynb) -4. [過学習と学習不足](./overfit_and_underfit.ipynb) -5. [モデルの保存と復元](./save_and_restore_models.ipynb) diff --git a/site/ja/r1/tutorials/keras/basic_classification.ipynb b/site/ja/r1/tutorials/keras/basic_classification.ipynb deleted file mode 100644 index 00ca9a4bca3..00000000000 --- a/site/ja/r1/tutorials/keras/basic_classification.ipynb +++ /dev/null @@ -1,1014 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# はじめてのニューラルネットワーク：分類問題の初歩" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RSywPQ2n736s", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "このガイドでは、スニーカーやシャツなど、身に着けるものの写真を分類するニューラルネットワークのモデルを訓練します。すべての詳細を理解できなくても問題ありません。TensorFlowの全体を早足で掴むためのもので、詳細についてはあとから見ていくことになります。\n", - "\n", - "このガイドでは、TensorFlowのモデルを構築し訓練するためのハイレベルのAPIである [tf.keras](https://www.tensorflow.org/r1/guide/keras)を使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow と tf.keras のインポート\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# ヘルパーライブラリのインポート\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## ファッションMNISTデータセットのロード" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "このガイドでは、[Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist)を使用します。Fashion MNISTには10カテゴリーの白黒画像70,000枚が含まれています。それぞれは下図のような1枚に付き1種類の衣料品が写っている低解像度（28×28ピクセル）の画像です。\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
     \n", - "
    \n", - "\n", - "Fashion MNISTは、画像処理のための機械学習での\"Hello, World\"としてしばしば登場する[MNIST](http://yann.lecun.com/exdb/mnist/) データセットの代替として開発されたものです。MNISTデータセットは手書きの数字（0, 1, 2 など）から構成されており、そのフォーマットはこれから使うFashion MNISTと全く同じです。\n", - "\n", - "Fashion MNISTを使うのは、目先を変える意味もありますが、普通のMNISTよりも少しだけ手応えがあるからでもあります。どちらのデータセットも比較的小さく、アルゴリズムが期待したとおりに機能するかどうかを確かめるために使われます。プログラムのテストやデバッグのためには、よい出発点になります。\n", - "\n", - "ここでは、60,000枚の画像を訓練に、10,000枚の画像を、ネットワークが学習した画像分類の正確性を評価するのに使います。TensorFlowを使うと、下記のようにFashion MNISTのデータを簡単にインポートし、ロードすることが出来ます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "ロードしたデータセットは、NumPy配列になります。\n", - "\n", - "* `train_images` と `train_labels` の2つの配列は、モデルの訓練に使用される**訓練用データセット**です。\n", - "* 訓練されたモデルは、 `test_images` と `test_labels` 配列からなる**テスト用データセット**を使ってテストします。\n", - "\n", - "画像は28×28のNumPy配列から構成されています。それぞれのピクセルの値は0から255の間の整数です。**ラベル**（label）は、0から9までの整数の配列です。それぞれの数字が下表のように、衣料品の**クラス**（class）に対応しています。\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    LabelClass
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "画像はそれぞれ単一のラベルに分類されます。データセットには上記の**クラス名**が含まれていないため、後ほど画像を出力するときのために、クラス名を保存しておきます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## データの観察\n", - "\n", - "モデルの訓練を行う前に、データセットのフォーマットを見てみましょう。下記のように、訓練用データセットには28×28ピクセルの画像が60,000枚含まれています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "同様に、訓練用データセットには60,000個のラベルが含まれます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "ラベルはそれぞれ、0から9までの間の整数です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "テスト用データセットには、10,000枚の画像が含まれます。画像は28×28ピクセルで構成されています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "テスト用データセットには10,000個のラベルが含まれます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## データの前処理\n", - "\n", - "ネットワークを訓練する前に、データを前処理する必要があります。最初の画像を調べてみればわかるように、ピクセルの値は0から255の間の数値です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.gca().grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "ニューラルネットワークにデータを投入する前に、これらの値を0から1までの範囲にスケールします。そのためには、画素の値を255で割ります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3jCZdQNNCaWv" - }, - "source": [ - "**訓練用データセット**と**テスト用データセット**は、同じように前処理することが重要です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "**訓練用データセット**の最初の25枚の画像を、クラス名付きで表示してみましょう。ネットワークを構築・訓練する前に、データが正しいフォーマットになっていることを確認します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## モデルの構築\n", - "\n", - "ニューラルネットワークを構築するには、まずモデルの階層を定義し、その後モデルをコンパイルします。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### 層の設定\n", - "\n", - "ニューラルネットワークを形作る基本的な構成要素は**層**（layer）です。層は、入力されたデータから「表現」を抽出します。それらの「表現」は、今取り組もうとしている問題に対して、より「意味のある」ものであることが期待されます。\n", - "\n", - "ディープラーニングモデルのほとんどは、単純な層の積み重ねで構成されています。`tf.keras.layers.Dense` のような層のほとんどには、訓練中に学習されるパラメータが存在します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation=tf.nn.relu),\n", - " keras.layers.Dense(10, activation=tf.nn.softmax)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "このネットワークの最初の層は、`tf.keras.layers.Flatten` です。この層は、画像を（28×28ピクセルの）2次元配列から、28×28＝784ピクセルの、1次元配列に変換します。この層が、画像の中に積まれているピクセルの行を取り崩し、横に並べると考えてください。この層には学習すべきパラメータはなく、ただデータのフォーマット変換を行うだけです。\n", - "\n", - "ピクセルが１次元化されたあと、ネットワークは2つの `tf.keras.layers.Dense` 層となります。これらの層は、密結合あるいは全結合されたニューロンの層となります。最初の `Dense` 層には、128個のノード（あるはニューロン）があります。最後の層でもある2番めの層は、10ノードの**softmax**層です。この層は、合計が1になる10個の確率の配列を返します。それぞれのノードは、今見ている画像が10個のクラスのひとつひとつに属する確率を出力します。\n", - "\n", - "### モデルのコンパイル\n", - "\n", - "モデルが訓練できるようになるには、いくつかの設定を追加する必要があります。それらの設定は、モデルの**コンパイル**(compile）時に追加されます。\n", - "\n", - "* **損失関数**（loss function） —訓練中にモデルがどれくらい正確かを測定します。この関数の値を最小化することにより、訓練中のモデルを正しい方向に向かわせようというわけです。\n", - "* **オプティマイザ**（optimizer）—モデルが見ているデータと、損失関数の値から、どのようにモデルを更新するかを決定します。\n", - "* **メトリクス**（metrics） —訓練とテストのステップを監視するのに使用します。下記の例では*accuracy* （正解率）、つまり、画像が正しく分類された比率を使用しています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## モデルの訓練\n", - "\n", - "ニューラルネットワークの訓練には次のようなステップが必要です。\n", - "\n", - "1. モデルに訓練用データを投入します—この例では `train_images` と `train_labels` の２つの配列です。\n", - "2. モデルは、画像とラベルの対応関係を学習します。\n", - "3. モデルにテスト用データセットの予測（分類）を行わせます—この例では `test_images` 配列です。その後、予測結果と `test_labels` 配列を照合します。\n", - "\n", - "訓練を開始するには、`model.fit` メソッドを呼び出します。モデルを訓練用データに \"fit\"（適合）させるという意味です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "モデルの訓練の進行とともに、損失値と正解率が表示されます。このモデルの場合、訓練用データでは0.88（すなわち88%）の正解率に達します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## 正解率の評価\n", - "\n", - "次に、テスト用データセットに対するモデルの性能を比較します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('Test accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "ご覧の通り、テスト用データセットでの正解率は、訓練用データセットでの正解率よりも少し低くなります。この訓練時の正解率とテスト時の正解率の差は、**過学習**（over fitting）の一例です。過学習とは、新しいデータに対する機械学習モデルの性能が、訓練時と比較して低下する現象です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## 予測する\n", - "\n", - "モデルの訓練が終わったら、そのモデルを使って画像の分類予測を行うことが出来ます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "これは、モデルがテスト用データセットの画像のひとつひとつを分類予測した結果です。最初の予測を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "予測結果は、10個の数字の配列です。これは、その画像が10の衣料品の種類のそれぞれに該当するかの「確信度」を表しています。どのラベルが一番確信度が高いかを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "というわけで、このモデルは、この画像が、アンクルブーツ、`class_names[9]` である可能性が最も高いと判断したことになります。これが正しいかどうか、テスト用ラベルを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "10チャンネルすべてをグラフ化してみることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VsRq6uZiG7eT", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " \n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - " \n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - " \n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - " \n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aZ_jDyLZG7eW" - }, - "source": [ - "0番目の画像と、予測、予測配列を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UH_jgCxEG7eW", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5_7K0ZL7G7eY", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Lduh0pbfG7eb" - }, - "source": [ - "予測の中のいくつかの画像を、予測値とともに表示してみましょう。正しい予測は青で、誤っている予測は赤でラベルを表示します。数字は予測したラベルのパーセント（100分率）を示します。自信があるように見えても間違っていることがあることに注意してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YGBDAiziCaXR", - "colab": {} - }, - "source": [ - "# X個のテスト画像、予測されたラベル、正解ラベルを表示します。\n", - "# 正しい予測は青で、間違った予測は赤で表示しています。\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "最後に、訓練済みモデルを使って1枚の画像に対する予測を行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yRJ7JU7JCaXT", - "colab": {} - }, - "source": [ - "# テスト用データセットから画像を1枚取り出す\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "`tf.keras` モデルは、サンプルの中の**バッチ**（batch）あるいは「集まり」について予測を行うように作られています。そのため、1枚の画像を使う場合でも、リスト化する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# 画像を1枚だけのバッチのメンバーにする\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "そして、予測を行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "scrolled": true, - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6o3nwO-KG7ex", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "`model.predict` メソッドの戻り値は、リストのリストです。リストの要素のそれぞれが、バッチの中の画像に対応します。バッチの中から、（といってもバッチの中身は１つだけですが）予測を取り出します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "prediction = predictions[0]\n", - "\n", - "np.argmax(prediction)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "というわけで、モデルは9というラベルを予測しました。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/keras/basic_regression.ipynb b/site/ja/r1/tutorials/keras/basic_regression.ipynb deleted file mode 100644 index 9258f44fb04..00000000000 --- a/site/ja/r1/tutorials/keras/basic_regression.ipynb +++ /dev/null @@ -1,858 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# 回帰: 燃費を予測する" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gDpivKCZ9aTm", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "回帰問題では、価格や確率といった連続的な値の出力を予測することが目的となります。これは、分類問題の目的が、（例えば、写真にリンゴが写っているかオレンジが写っているかといった）離散的なラベルを予測することであるのとは対照的です。\n", - "\n", - "このノートブックでは、古典的な[Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg)データセットを使用し、1970年代後半から1980年台初めの自動車の燃費を予測するモデルを構築します。この目的のため、モデルにはこの時期の多数の自動車の仕様を読み込ませます。仕様には、気筒数、排気量、馬力、重量などが含まれています。\n", - "\n", - "このサンプルでは`tf.keras` APIを使用しています。詳細は[このガイド](https://www.tensorflow.org/r1/guide/keras)を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# ペアプロットのためseabornを使用します\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Auto MPG データセット\n", - "\n", - "このデータセットは[UCI Machine Learning Repository](https://archive.ics.uci.edu/)から入手可能です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### データの取得\n", - "\n", - "まず、データセットをダウンロードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "pandasを使ってデータをインポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### データのクレンジング\n", - "\n", - "このデータには、いくつか欠損値があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "この最初のチュートリアルでは簡単化のためこれらの行を削除します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "`\"Origin\"`の列は数値ではなくカテゴリーです。このため、ワンホットエンコーディングを行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### データを訓練用セットとテスト用セットに分割\n", - "\n", - "データセットを訓練用セットとテスト用セットに分割しましょう。\n", - "\n", - "テスト用データセットは、作成したモデルの最終評価に使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### データの調査\n", - "\n", - "訓練用セットのいくつかの列の組み合わせの同時分布を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "全体の統計値も見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### ラベルと特徴量の分離\n", - "\n", - "ラベル、すなわち目的変数を特徴量から切り離しましょう。このラベルは、モデルに予測させたい数量です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### データの正規化\n", - "\n", - "上の`train_stats`のブロックをもう一度見て、それぞれの特徴量の範囲がどれほど違っているかに注目してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "スケールや値の範囲が異なる特徴量を正規化するのは良い習慣です。特徴量の正規化なしでもモデルは収束する**かもしれませんが**、モデルの訓練はより難しくなり、結果として得られたモデルも入力で使われる単位に依存することになります。\n", - "\n", - "注：（正規化に使用する）統計量は意図的に訓練用データセットだけを使って算出していますが、これらはテスト用データセットの正規化にも使うことになります。テスト用のデータセットを、モデルの訓練に使用した分布と同じ分布に射影する必要があるのです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "この正規化したデータを使ってモデルを訓練することになります。\n", - "\n", - "注意：ここで入力の正規化に使った統計量（平均と標準偏差）は、先程実施したワンホットエンコーディングとともに、モデルに供給する他のどんなデータにも適用する必要があります。テスト用データセットだけでなく、モデルを本番で使用する際の生のデータも同様です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## モデル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### モデルの構築\n", - "\n", - "それではモデルを構築しましょう。ここでは、2つの全結合の隠れ層と、1つの連続値を返す出力層からなる、`Sequential`モデルを使います。モデルを構築するステップは`build_model`という1つの関数の中に組み込みます。あとから2つ目のモデルを構築するためです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation=tf.nn.relu, input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation=tf.nn.relu),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mean_squared_error',\n", - " optimizer=optimizer,\n", - " metrics=['mean_absolute_error', 'mean_squared_error'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### モデルの検証\n", - "\n", - "`.summary`メソッドを使って、モデルの簡単な説明を表示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "では、モデルを試してみましょう。訓練用データのうち`10`個のサンプルからなるバッチを取り出し、それを使って`model.predict`メソッドを呼び出します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "うまく動作しているようです。予定通りの型と形状の出力が得られています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### モデルの訓練\n", - "\n", - "モデルを1000エポック訓練し、訓練と検証の正解率を`history`オブジェクトに記録します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# エポックが終わるごとにドットを一つ出力することで進捗を表示\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "`history`オブジェクトに保存された数値を使ってモデルの訓練の様子を可視化します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mean_absolute_error'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mean_absolute_error'],\n", - " label = 'Val Error')\n", - " plt.legend()\n", - " plt.ylim([0,5])\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mean_squared_error'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mean_squared_error'],\n", - " label = 'Val Error')\n", - " plt.legend()\n", - " plt.ylim([0,20])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "このグラフを見ると、検証エラーは100エポックを過ぎたあたりで改善が見られなくなり、むしろ悪化しているようです。検証スコアの改善が見られなくなったら自動的に訓練を停止するように、`model.fit`メソッド呼び出しを変更します。ここでは、エポック毎に訓練状態をチェックする*EarlyStopping*コールバックを使用します。設定したエポック数の間に改善が見られない場合、訓練を自動的に停止します。\n", - "\n", - "このコールバックについての詳細は[ここ](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping)を参照ください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# The patience parameter is the amount of epochs to check for improvement\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "検証用データセットでのグラフを見ると、平均誤差は+/- 2 MPG（マイル/ガロン）前後です。これは良い精度でしょうか？その判断はおまかせします。\n", - "\n", - "モデルの訓練に使用していない**テスト用**データセットを使って、モデルがどれくらい汎化できているか見てみましょう。これによって、モデルが実際の現場でどれくらい正確に予測できるかがわかります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "### モデルを使った予測\n", - "\n", - "最後に、テストデータを使ってMPG値を予測します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OrkHGKZcusUo" - }, - "source": [ - "そこそこ良い予測ができているように見えます。誤差の分布を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r9_kI6MHu1UU" - }, - "source": [ - "とても正規分布には見えませんが、サンプル数が非常に小さいからだと考えられます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## 結論\n", - "\n", - "このノートブックでは、回帰問題を扱うためのテクニックをいくつか紹介しました。\n", - "\n", - "* 平均二乗誤差（MSE: Mean Squared Error）は回帰問題に使われる一般的な損失関数です（分類問題には異なる損失関数が使われます）。\n", - "* 同様に、回帰問題に使われる評価指標も分類問題とは異なります。回帰問題の一般的な評価指標は平均絶対誤差（MAE: Mean Absolute Error）です。\n", - "* 入力数値特徴量の範囲が異なっている場合、特徴量毎に同じ範囲に正規化するべきです。\n", - "* 訓練用データが多くない場合、過学習を避けるために少ない隠れ層を持つ小さいネットワークを使うというのが良い方策の1つです。\n", - "* Early Stoppingは過学習を防止するための便利な手法の一つです。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/keras/basic_text_classification.ipynb b/site/ja/r1/tutorials/keras/basic_text_classification.ipynb deleted file mode 100644 index f9585645374..00000000000 --- a/site/ja/r1/tutorials/keras/basic_text_classification.ipynb +++ /dev/null @@ -1,736 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# 映画レビューのテキスト分類" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nBk9hleL9kon" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "ここでは、映画のレビューをそのテキストを使って**肯定的**か**否定的**かに分類します。これは、二値分類あるいは2クラス分類という問題の例であり、機械学習において重要でいろいろな応用が可能なものです。\n", - "\n", - "ここでは、[Internet Movie Database](https://www.imdb.com/)から抽出した50,000件の映画レビューを含む、 [IMDB dataset](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) を使います。レビューは訓練用とテスト用に25,000件ずつに分割されています。訓練用とテスト用のデータは**均衡**しています。言い換えると、それぞれが同数の肯定的及び否定的なレビューを含んでいます。\n", - "\n", - "ここでは、TensorFlowを使ってモデルを構築・訓練するためのハイレベルなAPIである [tf.keras](https://www.tensorflow.org/r1/guide/keras)を使用します。`tf.keras`を使ったもう少し高度なテキスト分類のチュートリアルについては、 [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "k6HBIXfqD6ZB", - "colab": {} - }, - "source": [ - "# keras.datasets.imdb is broken in 1.13 and 1.14, by np 1.16.3\n", - "!pip install tf_nightly" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## IMDB datasetのダウンロード\n", - "\n", - "IMDBデータセットは、TensorFlowにパッケージ化されています。それは前処理済みのものであり、（単語の連なりである）レビューが、整数の配列に変換されています。そこでは整数が辞書中の特定の単語を表します。\n", - "\n", - "次のコードは、IMDBデータセットをあなたのパソコンにダウンロードします。（すでにダウンロードしていれば、キャッシュされたコピーを使用します）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "`num_words=10000`という引数は、訓練データ中に出てくる単語のうち、最も頻繁に出現する10,000個を保持するためのものです。データサイズを管理可能にするため、稀にしか出現しない単語は破棄されます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## データを調べる\n", - "\n", - "データの形式を理解するために少し時間を割いてみましょう。このデータセットは前処理済みで、サンプルそれぞれが、映画レビューの中の単語を表す整数の配列になっています。ラベルはそれぞれ、0または1の整数値で、0が否定的レビュー、1が肯定的なレビューを示しています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} - }, - "source": [ - "print(\"Training entries: {}, labels: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "レビューのテキストは複数の整数に変換されており、それぞれの整数が辞書の中の特定の単語を表します。最初のレビューがどのようなものか見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "映画のレビューはそれぞれ長さが異なっていることでしょう。次のコードで、最初と2つ目のレビューの単語の数を見てみます。ニューラルネットワークへの入力は同じ長さでなければならないため、後ほどその問題を解決する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### 整数を単語に戻してみる\n", - "\n", - "整数をテキストに戻す方法を知っていると便利です。整数を文字列にマッピングする辞書オブジェクトを検索するためのヘルパー関数を定義します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} - }, - "source": [ - "# 単語を整数にマッピングする辞書\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# インデックスの最初の方は予約済み\n", - "word_index = {k:(v+3) for k,v in word_index.items()}\n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # unknown\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "`decode_review`を使うと、最初のレビューのテキストを表示できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} - }, - "source": [ - "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## データの準備\n", - "\n", - "レビュー（整数の配列）は、ニューラルネットワークに投入する前に、テンソルに変換する必要があります。これには2つの方法があります。\n", - "\n", - "* 配列をワンホット（one-hot）エンコーディングと同じように、単語の出現を表す0と1のベクトルに変換します。例えば、[3, 5]という配列は、インデックス3と5を除いてすべてゼロの10,000次元のベクトルになります。そして、これをネットワークの最初の層、すなわち、浮動小数点のベクトルデータを扱うことができるDense（全結合）層とします。ただし、これは単語数×レビュー数の行列が必要なメモリ集約的な方法です。\n", - "* もう一つの方法では、配列をパディングによって同じ長さに揃え、`サンプル数 * 長さの最大値`の形の整数テンソルにします。そして、この形式を扱うことができるEmbedding（埋め込み）層をネットワークの最初の層にします。\n", - "\n", - "このチュートリアルでは、後者を採用することにします。\n", - "\n", - "映画レビューは同じ長さでなければならないので、長さを標準化する [pad_sequences](https://www.tensorflow.org/versions/r1.10/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences) 関数を使うことにします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} - }, - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "サンプルの長さを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "次に、パディング済みの最初のサンプルを確認します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## モデルの構築\n", - "\n", - "ニューラルネットワークは、層を積み重ねることで構成されます。この際、２つの大きな決定が必要です。\n", - "\n", - "* モデルにいくつの**層**を設けるか？\n", - "* 層ごとに何個の**隠れユニット**を使用するか？\n", - "\n", - "この例では、入力データは単語インデックスの配列で構成されています。推定の対象となるラベルは、0または1です。この問題のためのモデルを構築しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "# 入力の形式は映画レビューで使われている語彙数（10,000語）\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation=tf.nn.relu))\n", - "model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "これらの層は、分類器を構成するため一列に積み重ねられます。\n", - "\n", - "1. 最初の層は`Embedding`（埋め込み）層です。この層は、整数にエンコードされた語彙を受け取り、それぞれの単語インデックスに対応する埋め込みベクトルを検索します。埋め込みベクトルは、モデルの訓練の中で学習されます。ベクトル化のために、出力行列には次元が１つ追加されます。その結果、次元は、`(batch, sequence, embedding)`となります。\n", - "2. 次は、`GlobalAveragePooling1D`（１次元のグローバル平均プーリング）層です。この層は、それぞれのサンプルについて、シーケンスの次元方向に平均値をもとめ、固定長のベクトルを返します。この結果、モデルは最も単純な形で、可変長の入力を扱うことができるようになります。\n", - "3. この固定長の出力ベクトルは、16個の隠れユニットを持つ全結合（`Dense`）層に受け渡されます。\n", - "4. 最後の層は、1個の出力ノードに全結合されます。シグモイド（`sigmoid`）活性化関数を使うことで、値は確率あるいは確信度を表す0と1の間の浮動小数点数となります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### 隠れユニット\n", - "\n", - "上記のモデルには、入力と出力の間に、2つの中間層あるいは「隠れ」層があります。出力（ユニット、ノード、またはニューロン）は、その層の内部表現の次元数です。言い換えると、このネットワークが学習によって内部表現を獲得する際の自由度ということです。\n", - "\n", - "モデルにより多くの隠れユニットがある場合（内部表現空間の次元数がより大きい場合）、または、より多くの層がある場合、あるいはその両方の場合、ネットワークはより複雑な内部表現を学習することができます。しかしながら、その結果として、ネットワークの計算量が多くなるほか、学習してほしくないパターンを学習するようになります。学習してほしくないパターンとは、訓練データでの性能は向上するものの、テスト用データの性能が向上しないパターンです。この問題を**過学習**（*overfitting*）といいます。この問題は後ほど検証することになります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### 損失関数とオプティマイザ\n", - "\n", - "モデルを訓練するには、損失関数とオプティマイザが必要です。今回の問題は二値分類問題であり、モデルの出力は確率（1ユニットの層とシグモイド活性化関数）であるため、損失関数として`binary_crossentropy`（2値のクロスエントロピー）関数を使用することにします。\n", - "\n", - "損失関数の候補はこれだけではありません。例えば、`mean_squared_error`（平均二乗誤差）を使うこともできます。しかし、一般的には、確率を扱うには`binary_crossentropy`の方が適しています。`binary_crossentropy`は、確率分布の間の「距離」を測定する尺度です。今回の場合には、真の分布と予測値の分布の間の距離ということになります。\n", - "\n", - "後ほど、回帰問題を検証する際には（例えば家屋の値段を推定するとか）、もう一つの損失関数である`mean_squared_error`（平均二乗誤差）の使い方を目にすることになります。\n", - "\n", - "さて、モデルのオプティマイザと損失関数を設定しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "## 検証用データを作る\n", - "\n", - "訓練を行う際、モデルが見ていないデータでの正解率を検証したいと思います。もとの訓練用データから、10,000個のサンプルを取り分けて**検証用データ**（*validation set*）を作ります。（なぜ、ここでテスト用データを使わないのでしょう？ 今回の目的は、訓練用データだけを使って、モデルの開発とチューニングを行うことです。その後、テスト用データを1回だけ使い、正解率を検証するのです。）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} - }, - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## モデルの訓練\n", - "\n", - "512個のサンプルからなるミニバッチを使って、40エポックモデルを訓練します。この結果、`x_train`と`y_train`に含まれるすべてのサンプルを40回繰り返すことになります。訓練中、検証用データの10,000サンプルを用いて、モデルの損失と正解率をモニタリングします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "scrolled": false, - "colab": {} - }, - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## モデルの評価\n", - "\n", - "さて、モデルの性能を見てみましょう。2つの値が返されます。損失（エラーを示す数値であり、小さい方が良い）と正解率です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "この、かなり素朴なアプローチでも87%前後の正解率を達成しました。もっと高度なアプローチを使えば、モデルの正解率は95%に近づけることもできるでしょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## 正解率と損失の時系列グラフを描く\n", - "\n", - "`model.fit()` は、訓練中に発生したすべてのことを記録した辞書を含む`History` オブジェクトを返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} - }, - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "4つのエントリがあります。それぞれが、訓練と検証の際にモニターしていた指標を示します。これを使って、訓練時と検証時の損失を比較するグラフと、訓練時と検証時の正解率を比較するグラフを作成することができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "\n", - "acc = history.history['acc']\n", - "val_acc = history.history['val_acc']\n", - "loss = history.history['loss']\n", - "val_loss = history.history['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# \"bo\" は青いドット\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "# ”b\" は青い実線\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} - }, - "source": [ - "plt.clf() # 図のクリア\n", - "acc_values = history_dict['acc']\n", - "val_acc_values = history_dict['val_acc']\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "上記のグラフでは、点が訓練時の損失と正解率を、実線が検証時の損失と正解率を表しています。\n", - "\n", - "訓練時の損失がエポックごとに**減少**し、訓練時の正解率がエポックごとに**上昇**していることに気がつくはずです。繰り返すごとに指定された数値指標を最小化する勾配降下法を最適化に使用している場合に期待される動きです。\n", - "\n", - "これは、検証時の損失と正解率には当てはまりません。20エポックを過ぎたあたりから、横ばいになっているようです。これが、過学習の一例です。モデルの性能が、訓練用データでは高い一方で、見たことの無いデータではそれほど高くないというものです。このポイントをすぎると、モデルが最適化しすぎて、訓練用データでは特徴的であるが、テスト用データには一般化できない内部表現を学習しています。\n", - "\n", - "このケースの場合、20エポックを過ぎたあたりで訓練をやめることで、過学習を防止することが出来ます。後ほど、コールバックを使って、これを自動化する方法を紹介します。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/keras/overfit_and_underfit.ipynb b/site/ja/r1/tutorials/keras/overfit_and_underfit.ipynb deleted file mode 100644 index e0632ced073..00000000000 --- a/site/ja/r1/tutorials/keras/overfit_and_underfit.ipynb +++ /dev/null @@ -1,687 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "overfit_and_underfit.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "fTFj8ft5dlbS" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fTFj8ft5dlbS" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "lzyBOpYMdp3F", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m_x4KfSJ7Vt7", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C9HmC2T4ld5B" - }, - "source": [ - "# 過学習と学習不足について知る" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRTxFhXAlnl1" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "82CTPwRf9si8", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19rPukKZsPG6" - }, - "source": [ - "いつものように、この例のプログラムは`tf.keras` APIを使用します。詳しくはTensorFlowの[Keras guide](https://www.tensorflow.org/r1/guide/keras)を参照してください。\n", - "\n", - "これまでの例、つまり、映画レビューの分類と燃費の推定では、検証用データでのモデルの正解率が、数エポックでピークを迎え、その後低下するという現象が見られました。\n", - "\n", - "言い換えると、モデルが訓練用データを**過学習**したと考えられます。過学習への対処の仕方を学ぶことは重要です。**訓練用データセット**で高い正解率を達成することは難しくありませんが、我々は、（これまで見たこともない）**テスト用データ**に汎化したモデルを開発したいのです。\n", - "\n", - "過学習の反対語は**学習不足**(underfitting)です。学習不足は、モデルがテストデータに対してまだ改善の余地がある場合に発生します。学習不足の原因は様々です。モデルが十分強力でないとか、正則化のしすぎだとか、単に訓練時間が短すぎるといった理由があります。学習不足は、訓練用データの中の関連したパターンを学習しきっていないということを意味します。\n", - "\n", - "モデルの訓練をやりすぎると、モデルは過学習を始め、訓練用データの中のパターンで、テストデータには一般的ではないパターンを学習します。我々は、過学習と学習不足の中間を目指す必要があります。これから見ていくように、ちょうどよいエポック数だけ訓練を行うというのは必要なスキルなのです。\n", - "\n", - "過学習を防止するための、最良の解決策は、より多くの訓練用データを使うことです。多くのデータで訓練を行えば行うほど、モデルは自然により汎化していく様になります。これが不可能な場合、次善の策は正則化のようなテクニックを使うことです。正則化は、モデルに保存される情報の量とタイプに制約を課すものです。ネットワークが少数のパターンしか記憶できなければ、最適化プロセスにより、最も主要なパターンのみを学習することになり、より汎化される可能性が高くなります。\n", - "\n", - "このノートブックでは、重みの正則化とドロップアウトという、よく使われる2つの正則化テクニックをご紹介します。これらを使って、IMDBの映画レビューを分類するノートブックの改善を図ります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5pZ8A2liqvgk", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1cweoTiruj8O" - }, - "source": [ - "## IMDBデータセットのダウンロード\n", - "\n", - "以前のノートブックで使用したエンベディングの代わりに、ここでは文をマルチホットエンコードします。このモデルは、訓練用データセットをすぐに過学習します。このモデルを使って、過学習がいつ起きるかということと、どうやって過学習と戦うかをデモします。\n", - "\n", - "リストをマルチホットエンコードすると言うのは、0と1のベクトルにするということです。具体的にいうと、例えば`[3, 5]`というシーケンスを、インデックス3と5の値が1で、それ以外がすべて0の、10,000次元のベクトルに変換するということを意味します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QpzE4iqZtJly", - "colab": {} - }, - "source": [ - "NUM_WORDS = 10000\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = keras.datasets.imdb.load_data(num_words=NUM_WORDS)\n", - "\n", - "def multi_hot_sequences(sequences, dimension):\n", - " # 形状が (len(sequences), dimension)ですべて0の行列を作る\n", - " results = np.zeros((len(sequences), dimension))\n", - " for i, word_indices in enumerate(sequences):\n", - " results[i, word_indices] = 1.0 # 特定のインデックスに対してresults[i] を１に設定する\n", - " return results\n", - "\n", - "\n", - "train_data = multi_hot_sequences(train_data, dimension=NUM_WORDS)\n", - "test_data = multi_hot_sequences(test_data, dimension=NUM_WORDS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MzWVeXe3NBTn" - }, - "source": [ - "結果として得られるマルチホットベクトルの1つを見てみましょう。単語のインデックスは頻度順にソートされています。このため、インデックスが0に近いほど１が多く出現するはずです。分布を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "71kr5rG4LkGM", - "colab": {} - }, - "source": [ - "plt.plot(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lglk41MwvU5o" - }, - "source": [ - "## 過学習のデモ\n", - "\n", - "過学習を防止するための最も単純な方法は、モデルのサイズ、すなわち、モデル内の学習可能なパラメータの数を小さくすることです（学習パラメータの数は、層の数と層ごとのユニット数で決まります）。ディープラーニングでは、モデルの学習可能なパラメータ数を、しばしばモデルの「キャパシティ」と呼びます。直感的に考えれば、パラメータ数の多いモデルほど「記憶容量」が大きくなり、訓練用のサンプルとその目的変数の間の辞書のようなマッピングをたやすく学習することができます。このマッピングには汎化能力がまったくなく、これまで見たことが無いデータを使って予測をする際には役に立ちません。\n", - "\n", - "ディープラーニングのモデルは訓練用データに適応しやすいけれど、本当のチャレレンジは汎化であって適応ではないということを、肝に銘じておく必要があります。\n", - "\n", - "一方、ネットワークの記憶容量が限られている場合、前述のようなマッピングを簡単に学習することはできません。損失を減らすためには、より予測能力が高い圧縮された表現を学習しなければなりません。同時に、モデルを小さくしすぎると、訓練用データに適応するのが難しくなります。「多すぎる容量」と「容量不足」の間にちょうどよい容量があるのです。\n", - "\n", - "残念ながら、（層の数や、層ごとの大きさといった）モデルの適切なサイズやアーキテクチャを決める魔法の方程式はありません。一連の異なるアーキテクチャを使って実験を行う必要があります。\n", - "\n", - "適切なモデルのサイズを見つけるには、比較的少ない層の数とパラメータから始めるのがベストです。それから、検証用データでの損失値の改善が見られなくなるまで、徐々に層の大きさを増やしたり、新たな層を加えたりします。映画レビューの分類ネットワークでこれを試してみましょう。\n", - "\n", - "比較基準として、```Dense```層だけを使ったシンプルなモデルを構築し、その後、それより小さいバージョンと大きいバージョンを作って比較します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ReKHdC2EgVu" - }, - "source": [ - "### 比較基準を作る" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QKgdXPx9usBa", - "colab": {} - }, - "source": [ - "baseline_model = keras.Sequential([\n", - " # `.summary` を見るために`input_shape`が必要\n", - " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "baseline_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "baseline_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LqG3MXF5xSjR", - "colab": {} - }, - "source": [ - "baseline_history = baseline_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L-DGRBbGxI6G" - }, - "source": [ - "### より小さいモデルの構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SrfoVQheYSO5" - }, - "source": [ - "今作成したばかりの比較基準となるモデルに比べて隠れユニット数が少ないモデルを作りましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jksi-XtaxDAh", - "colab": {} - }, - "source": [ - "smaller_model = keras.Sequential([\n", - " keras.layers.Dense(4, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(4, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "smaller_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "smaller_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jbngCZliYdma" - }, - "source": [ - "同じデータを使って訓練します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ofn1AwDhx-Fe", - "colab": {} - }, - "source": [ - "smaller_history = smaller_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vIPuf23FFaVn" - }, - "source": [ - "### より大きなモデルの構築\n", - "\n", - "練習として、より大きなモデルを作成し、どれほど急速に過学習が起きるかを見ることもできます。次はこのベンチマークに、この問題が必要とするよりはるかに容量の大きなネットワークを追加しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ghQwwqwqvQM9", - "colab": {} - }, - "source": [ - "bigger_model = keras.models.Sequential([\n", - " keras.layers.Dense(512, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(512, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "bigger_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "bigger_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D-d-i5DaYmr7" - }, - "source": [ - "このモデルもまた同じデータを使って訓練します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "U1A99dhqvepf", - "colab": {} - }, - "source": [ - "bigger_history = bigger_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fy3CMUZpzH3d" - }, - "source": [ - "### 訓練時と検証時の損失をグラフにする\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HSlo1F4xHuuM" - }, - "source": [ - "実線は訓練用データセットの損失、破線は検証用データセットでの損失です（検証用データでの損失が小さい方が良いモデルです）。これをみると、小さいネットワークのほうが比較基準のモデルよりも過学習が始まるのが遅いことがわかります（4エポックではなく6エポック後）。また、過学習が始まっても性能の低下がよりゆっくりしています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0XmKDtOWzOpk", - "colab": {} - }, - "source": [ - "def plot_history(histories, key='binary_crossentropy'):\n", - " plt.figure(figsize=(16,10))\n", - "\n", - " for name, history in histories:\n", - " val = plt.plot(history.epoch, history.history['val_'+key],\n", - " '--', label=name.title()+' Val')\n", - " plt.plot(history.epoch, history.history[key], color=val[0].get_color(),\n", - " label=name.title()+' Train')\n", - "\n", - " plt.xlabel('Epochs')\n", - " plt.ylabel(key.replace('_',' ').title())\n", - " plt.legend()\n", - "\n", - " plt.xlim([0,max(history.epoch)])\n", - "\n", - "\n", - "plot_history([('baseline', baseline_history),\n", - " ('smaller', smaller_history),\n", - " ('bigger', bigger_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bi6hBhdnSfjA" - }, - "source": [ - "より大きなネットワークでは、すぐに、1エポックで過学習が始まり、その度合も強いことに注目してください。ネットワークの容量が大きいほど訓練用データをモデル化するスピードが早くなり（結果として訓練時の損失値が小さくなり）ますが、より過学習しやすく（結果として訓練時の損失値と検証時の損失値が大きく乖離しやすく）なります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ASdv7nsgEFhx" - }, - "source": [ - "## 戦略" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4rHoVWcswFLa" - }, - "source": [ - "### 重みの正則化を加える\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRxWepNawbBK" - }, - "source": [ - "「オッカムの剃刀」の原則をご存知でしょうか。何かの説明が2つあるとすると、最も正しいと考えられる説明は、仮定の数が最も少ない「一番単純な」説明だというものです。この原則は、ニューラルネットワークを使って学習されたモデルにも当てはまります。ある訓練用データとネットワーク構造があって、そのデータを説明できる重みの集合が複数ある時（つまり、複数のモデルがある時）、単純なモデルのほうが複雑なものよりも過学習しにくいのです。\n", - "\n", - "ここで言う「単純なモデル」とは、パラメータ値の分布のエントロピーが小さいもの（あるいは、上記で見たように、そもそもパラメータの数が少ないもの）です。したがって、過学習を緩和するための一般的な手法は、重みが小さい値のみをとることで、重み値の分布がより整然となる（正則）様に制約を与えるものです。これを「重みの正則化」と呼ばれ、ネットワークの損失関数に、重みの大きさに関連するコストを加えることで行われます。このコストには2つの種類があります。\n", - "\n", - "* [L1正則化](https://developers.google.com/machine-learning/glossary/#L1_regularization) 重み係数の絶対値に比例するコストを加える（重みの「L1ノルム」と呼ばれる）。\n", - "\n", - "* [L2正則化](https://developers.google.com/machine-learning/glossary/#L2_regularization) 重み係数の二乗に比例するコストを加える（重み係数の二乗「L2ノルム」と呼ばれる）。L2正則化はニューラルネットワーク用語では重み減衰（Weight Decay）と呼ばれる。呼び方が違うので混乱しないように。重み減衰は数学的にはL2正則化と同義である。\n", - "\n", - "`tf.keras`では、重みの正則化をするために、重み正則化のインスタンスをキーワード引数として層に加えます。ここでは、L2正則化を追加してみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HFGmcwduwVyQ", - "colab": {} - }, - "source": [ - "l2_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "l2_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "l2_model_history = l2_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bUUHoXb7w-_C" - }, - "source": [ - "```l2(0.001)```というのは、層の重み行列の係数全てに対して```0.001 * 重み係数の値 **2```をネットワークの損失値合計に加えることを意味します。このペナルティは訓練時のみに加えられるため、このネットワークの損失値は、訓練時にはテスト時に比べて大きくなることに注意してください。\n", - "\n", - "L2正則化の影響を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7wkfLyxBZdh_", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('l2', l2_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Kx1YHMsVxWjP" - }, - "source": [ - "ご覧のように、L2正則化ありのモデルは比較基準のモデルに比べて過学習しにくくなっています。両方のモデルのパラメータ数は同じであるにもかかわらずです。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HmnBNOOVxiG8" - }, - "source": [ - "### ドロップアウトを追加する\n", - "\n", - "ドロップアウトは、ニューラルネットワークの正則化テクニックとして最もよく使われる手法の一つです。この手法は、トロント大学のヒントンと彼の学生が開発したものです。ドロップアウトは層に適用するもので、訓練時に層から出力された特徴量に対してランダムに「ドロップアウト（つまりゼロ化）」を行うものです。例えば、ある層が訓練時にある入力サンプルに対して、普通は`[0.2, 0.5, 1.3, 0.8, 1.1]` というベクトルを出力するとします。ドロップアウトを適用すると、このベクトルは例えば`[0, 0.5, 1.3, 0, 1.1]`のようにランダムに散らばったいくつかのゼロを含むようになります。「ドロップアウト率」はゼロ化される特徴の割合で、通常は0.2から0.5の間に設定します。テスト時は、どのユニットもドロップアウトされず、代わりに出力値がドロップアウト率と同じ比率でスケールダウンされます。これは、訓練時に比べてたくさんのユニットがアクティブであることに対してバランスをとるためです。\n", - "\n", - "`tf.keras`では、Dropout層を使ってドロップアウトをネットワークに導入できます。ドロップアウト層は、その直前の層の出力に対してドロップアウトを適用します。\n", - "\n", - "それでは、IMDBネットワークに2つのドロップアウト層を追加しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OFEYvtrHxSWS", - "colab": {} - }, - "source": [ - "dpt_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dropout(rate=0.5),\n", - " keras.layers.Dense(16, activation=tf.nn.relu),\n", - " keras.layers.Dropout(rate=0.5),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "dpt_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "dpt_model_history = dpt_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SPZqwVchx5xp", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('dropout', dpt_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QioQ5e9GN0bM" - }, - "source": [ - "ドロップアウトを追加することで、比較対象モデルより明らかに改善が見られます。\n", - "\n", - "まとめ：ニューラルネットワークにおいて過学習を防ぐ最も一般的な方法は次のとおりです。\n", - "\n", - "* 訓練データを増やす\n", - "* ネットワークの容量をへらす\n", - "* 重みの正則化を行う\n", - "* ドロップアウトを追加する\n", - "\n", - "このガイドで触れていない２つの重要なアプローチがあります。データ拡張とバッチ正規化です。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/keras/save_and_restore_models.ipynb b/site/ja/r1/tutorials/keras/save_and_restore_models.ipynb deleted file mode 100644 index 607077081ee..00000000000 --- a/site/ja/r1/tutorials/keras/save_and_restore_models.ipynb +++ /dev/null @@ -1,859 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "save_and_restore_models.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pZJ3uY9O17VN" - }, - "source": [ - "# モデルの保存と復元" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M4Ata7_wMul1" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jqFf_t5y9xep", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBdde4YJeJKF" - }, - "source": [ - "モデルは訓練中にも、訓練が終わったあとも保存できます。このことは、長い訓練時間を掛けなくても、やめたところから再開できるということを意味します。モデルが保存可能であることは、あなたが作ったモデルを他の人と共有できるということでもあります。研究結果であるモデルや手法を公開する際、機械学習の実務家はほとんど次のものを共有します。\n", - "\n", - "* モデルを構築するプログラム\n", - "* 学習済みモデルの重みあるいはパラメータ\n", - "\n", - "このデータを共有することで、他の人がモデルだどの様に動作するかを理解したり、新しいデータに試してみたりすることが容易になります。\n", - "\n", - "注意：信頼できないプログラムには気をつけましょう。TensorFlowのモデルもプログラムです。詳しくは、[Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)を参照してください。\n", - "\n", - "### オプション\n", - "\n", - "TensorFlowのモデルを保存する方法は、使っているAPIによって異なります。このガイドはTensorFlowのモデルを構築し訓練するためのハイレベルなAPIである[tf.keras](https://www.tensorflow.org/r1/guide/keras)を使っています。この他のアプローチについては、TensorFlowの [Save and Restore](https://www.tensorflow.org/r1/guide/saved_model) ガイド、あるいは、[Saving in eager](https://www.tensorflow.org/r1/guide/eager#object-based_saving)を参照してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xCUREq7WXgvg" - }, - "source": [ - "## 設定\n", - "\n", - "### インストールとインポート" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7l0MiTOrXtNv" - }, - "source": [ - "TensorFlowと依存関係のライブラリをインストールし、インポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} - }, - "source": [ - "!pip install h5py pyyaml" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbGsznErXWt6" - }, - "source": [ - "### サンプルデータセットの取得\n", - "\n", - "ここでは、モデルを訓練し重みの保存をデモするために、 [MNIST dataset](http://yann.lecun.com/exdb/mnist/) を使います。デモの実行を速くするため、最初の1,000件のサンプルだけを使います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "tf.__version__" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_labels = train_labels[:1000]\n", - "test_labels = test_labels[:1000]\n", - "\n", - "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", - "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "anG3iVoXyZGI" - }, - "source": [ - "### モデルの定義" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wynsOBfby0Pa" - }, - "source": [ - "重みの保存と読み込みのデモを行うための簡単なモデルを定義しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} - }, - "source": [ - "# 短いシーケンシャルモデルを返す関数\n", - "def create_model():\n", - " model = tf.keras.models.Sequential([\n", - " keras.layers.Dense(512, activation=tf.nn.relu, input_shape=(784,)),\n", - " keras.layers.Dropout(rate=0.2),\n", - " keras.layers.Dense(10, activation=tf.keras.activations.softmax)\n", - " ])\n", - "\n", - " model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - "\n", - " return model\n", - "\n", - "\n", - "# 基本的なモデルのインスタンスを作る\n", - "model = create_model()\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "soDE0W_KH8rG" - }, - "source": [ - "## 訓練中にチェックポイントを保存する" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRyd5qQQIXZm" - }, - "source": [ - "主な用途は訓練の**途中**あるいは**終了後**にチェックポイントを自動的に保存することです。こうすることにより、再び訓練を行うことなくモデルを使用することができ、また、訓練が中断された場合に、中止したところから再開できます。\n", - "\n", - "`tf.keras.callbacks.ModelCheckpoint`がこれを行うためのコールバックです。このコールバックにはチェックポイントを構成するためのいくつかの引数があります。\n", - "\n", - "### チェックポイントコールバックの使い方\n", - "\n", - "モデルの訓練時に、`ModelCheckpoint`を渡します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} - }, - "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# チェックポイントコールバックを作る\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,\n", - " save_weights_only=True,\n", - " verbose=1)\n", - "\n", - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs = 10,\n", - " validation_data = (test_images,test_labels),\n", - " callbacks = [cp_callback]) # 訓練にコールバックを渡す\n", - "\n", - "# オプティマイザの状態保存についての警告が表示されるかもしれません。\n", - "# これらの警告は（このノートブックで発生する同様な警告を含めて）\n", - "# 古い用法を非推奨にするためのもので、無視して構いません。" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlM-sgyJO084" - }, - "source": [ - "この結果、エポックごとに更新される一連のTensorFlowチェックポイントファイルが作成されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} - }, - "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wlRN_f56Pqa9" - }, - "source": [ - "訓練していない新しいモデルを作ります。重みだけからモデルを復元する場合には、元のモデルと同じアーキテクチャのモデルが必要です。モデルのアーキテクチャが同じであるため、モデルの異なる**インスタンス**であっても重みを共有することができるのです。\n", - "\n", - "訓練していない全く新しいモデルを作り、テストデータセットで評価します。訓練をしていないモデルは偶然のレベル（正解率10%以下）の性能しか無いはずです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Untrained model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1DTKpZssRSo3" - }, - "source": [ - "次に、チェックポイントから重みをロードし、再び評価します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} - }, - "source": [ - "model.load_weights(checkpoint_path)\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpAbKkAyVPV8" - }, - "source": [ - "### チェックポイントコールバックのオプション\n", - "\n", - "このコールバックには、チェックポイントに一意な名前をつけたり、チェックポイントの頻度を調整するためのオプションがあります。\n", - "\n", - "新しいモデルを訓練し、5エポックごとに一意な名前のチェックポイントを保存します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} - }, - "source": [ - "# ファイル名に(`str.format`を使って)エポック数を埋め込みます\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " checkpoint_path, verbose=1, save_weights_only=True,\n", - " # 重みを5エポックごとに保存します\n", - " period=5)\n", - "\n", - "model = create_model()\n", - "model.fit(train_images, train_labels,\n", - " epochs = 50, callbacks = [cp_callback],\n", - " validation_data = (test_images,test_labels),\n", - " verbose=0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1zFrKTjjavWI" - }, - "source": [ - "次に、出来上がったチェックポイントを確認し、最後のものを選択します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} - }, - "source": [ - "! ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} - }, - "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", - "latest" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zk2ciGbKg561" - }, - "source": [ - "注意：デフォルトのtensorflowフォーマットは、直近の5つのチェックポイントのみを保存します。\n", - "\n", - "テストのため、モデルをリセットし最後のチェックポイントをロードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "model.load_weights(latest)\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c2OxsJOTHxia" - }, - "source": [ - "## これらのファイルは何？" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtdYhvWnH2ib" - }, - "source": [ - "上記のコードでは、重みだけをバイナリで[checkpoint](https://www.tensorflow.org/r1/guide/saved_model#save_and_restore_variables)形式の一連のファイルに保存します。チェックポイントには、次のものが含まれます。\n", - "\n", - "* １つ以上のモデルの重みの断片\n", - "* どの重みがどの断片に保存されているかを示すインデックスファイル\n", - "\n", - "1台のマシンだけでモデルの訓練を行っている場合には、`.data-00000-of-00001`のようなサフィックスのついたファイルが1つだけ作成されます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S_FA-ZvxuXQV" - }, - "source": [ - "## 手動で重みを保存する\n", - "\n", - "上記では重みをモデルにロードする方法を見ました。\n", - "\n", - "手動で重みを保存するのも同じ様に簡単です。`Model.save_weights` メソッドを使います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} - }, - "source": [ - "# 重みの保存\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# 重みの復元\n", - "model = create_model()\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", - "\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kOGlxPRBEvV1" - }, - "source": [ - "## モデル全体の保存\n", - "\n", - "重みの値、モデルの設定、そして（構成によりますが※訳注1）オプティマイザの設定までも含んだモデル全体をファイルに保存することができます。これにより、モデルのある時点の状態を保存し、オリジナルのPythonコードにアクセスしなくとも、中断したところから訓練を再開することができます。\n", - "\n", - "（※訳注1：tf.trainモジュールに含まれるオプティマイザではないオプティマイザを使用しているモデルをHDF5ファイルに保存する場合にはオプティマイザの設定を保存できます。）\n", - "\n", - "完全に機能するモデルを保存できるのは便利です。保存したモデルをTensorFlow.js ([HDF5](https://js.tensorflow.org/r1/tutorials/import-keras.html), [Saved Model](https://js.tensorflow.org/r1/tutorials/import-saved-model.html))でロードし、ブラウザで訓練したり、実行したりすることができるほか、TensorFlow Lite ([HDF5](https://www.tensorflow.org/lite/convert/python_api#exporting_a_tfkeras_file_), [Saved Model](https://www.tensorflow.org/lite/convert/python_api#exporting_a_savedmodel_))\n", - "を使ってモバイルデバイスで実行できるように変換することも可能です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" - }, - "source": [ - "### HDF5ファイルとして\n", - "\n", - "Kerasでは、[HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) 標準を使った基本的なファイルフォーマットが利用できます。ここでの利用目的では、保存されたモデルは単独のバイナリラージオブジェクト（blob）として扱うことができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)\n", - "\n", - "# モデル全体を１つのHDF5ファイルに保存します。\n", - "model.save('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" - }, - "source": [ - "保存したファイルを使ってモデルを再作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} - }, - "source": [ - "# 重みとオプティマイザを含む全く同じモデルを再作成\n", - "new_model = keras.models.load_model('my_model.h5')\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" - }, - "source": [ - "正解率を検査します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} - }, - "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "この方法では、次のすべてが保存されます。\n", - "\n", - "* 重みの値\n", - "* モデルの設定（アーキテクチャ）\n", - "* オプティマイザの設定\n", - "\n", - "Kerasは保存する際にアーキテクチャを調べます。いまのところ、TensorFlowのオプティマイザ（`tf.train`に含まれるもの）を保存することはできません。TensorFlowのオプティマイザを使用している場合には、モデルをロードしたあと再コンパイルする必要があり、オプティマイザの状態は失われます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPyhgcoVzqUB" - }, - "source": [ - "### `saved_model`として" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LtcN4VIb7JkK" - }, - "source": [ - "注意：この手法による`tf.keras`モデルの保存は実験的なもので、将来のバージョンで変更される可能性があります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DSWiSB0Q8c46" - }, - "source": [ - "新しいモデルを作ります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iUvT_3qE8hV5" - }, - "source": [ - "`saved_model`を作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "scrolled": true, - "colab": {} - }, - "source": [ - "saved_model_path = tf.contrib.saved_model.save_keras_model(model, \"./saved_models\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MjpmyPfh8-1n" - }, - "source": [ - "SavedModel はタイムスタンプ付きのディレクトリに保存されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZtOvxA7V0iTv", - "colab": {} - }, - "source": [ - "!ls saved_models/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B7qfpvpY9HCe" - }, - "source": [ - "保存されたモデル(SavedModel)から新しいKerasモデルをリロードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} - }, - "source": [ - "new_model = tf.contrib.saved_model.load_keras_model(saved_model_path)\n", - "new_model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" - }, - "source": [ - "復元されたモデルを実行します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Pc9e6G6w1AWG", - "scrolled": true, - "colab": {} - }, - "source": [ - "# モデルを評価する前にコンパイルする必要があります。\n", - "# モデルをデプロイするだけであればこのステップは不要です。\n", - "\n", - "new_model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - "\n", - "# モデルを評価します。\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eUYTzSz5VxL2" - }, - "source": [ - "## この先は？\n", - "\n", - "`tf.keras`を使った保存とロードのクイックガイドでした。\n", - "\n", - "* [tf.keras guide](https://www.tensorflow.org/r1/guide/keras) には`tf.keras`での保存とロードについて、もう少し記載されています\n", - "\n", - "* Eager Executionでの保存については[Saving in eager](https://www.tensorflow.org/r1/guide/eager#object_based_saving) を参照ください\n", - "\n", - "* [Save and Restore](https://www.tensorflow.org/r1/guide/saved_model)ガイドには、TensorFlowでの保存についてローレベルの詳細が記載されています" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/load_data/images.ipynb b/site/ja/r1/tutorials/load_data/images.ipynb deleted file mode 100644 index c2c2cb5a5aa..00000000000 --- a/site/ja/r1/tutorials/load_data/images.ipynb +++ /dev/null @@ -1,1662 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "images.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mt9dL5dIir8X" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ufPx7EiCiqgR", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ucMoYase6URl" - }, - "source": [ - "# tf.dataを使って画像をロードする" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_Wwu5SXZmEkB" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "uybAm_B9LvF7", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oxw4WahM7DU9" - }, - "source": [ - "このチュートリアルでは、'tf.data'を使って画像データセットをロードする簡単な例を示します。\n", - "\n", - "このチュートリアルで使用するデータセットは、クラスごとに別々のディレクトリに別れた形で配布されています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hoQQiZDB6URn" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DHz3JONNEHlj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", - "tf.__version__" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KT6CcaqgQewg", - "colab": {} - }, - "source": [ - "AUTOTUNE = tf.data.experimental.AUTOTUNE" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rxndJHNC8YPM" - }, - "source": [ - "## データセットのダウンロードと検査" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wO0InzL66URu" - }, - "source": [ - "### 画像の取得\n", - "\n", - "訓練を始める前に、ネットワークに認識すべき新しいクラスを教えるために画像のセットが必要です。最初に使うためのクリエイティブ・コモンズでライセンスされた花の画像のアーカイブを作成してあります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rN-Pc6Zd6awg", - "colab": {} - }, - "source": [ - "import pathlib\n", - "data_root = tf.keras.utils.get_file('flower_photos','https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz', untar=True)\n", - "data_root = pathlib.Path(data_root)\n", - "print(data_root)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rFkFK74oO--g" - }, - "source": [ - "218MBをダウンロードすると、花の画像のコピーが使えるようになっているはずです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7onR_lWE7Njj", - "colab": {} - }, - "source": [ - "for item in data_root.iterdir():\n", - " print(item)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4yYX3ZRqGOuq", - "colab": {} - }, - "source": [ - "import random\n", - "all_image_paths = list(data_root.glob('*/*'))\n", - "all_image_paths = [str(path) for path in all_image_paths]\n", - "random.shuffle(all_image_paths)\n", - "\n", - "image_count = len(all_image_paths)\n", - "image_count" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t_BbYnLjbltQ", - "colab": {} - }, - "source": [ - "all_image_paths" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkM-IpB-6URx" - }, - "source": [ - "### 画像の検査\n", - "\n", - "扱っている画像について知るために、画像のいくつかを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wNGateQJ6UR1", - "colab": {} - }, - "source": [ - "import os\n", - "attributions = (data_root/\"LICENSE.txt\").open(encoding='utf-8').readlines()[4:]\n", - "attributions = [line.split(' CC-BY') for line in attributions]\n", - "attributions = dict(attributions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jgowG2xu88Io", - "colab": {} - }, - "source": [ - "import IPython.display as display\n", - "\n", - "def caption_image(image_path):\n", - " image_rel = pathlib.Path(image_path).relative_to(data_root)\n", - " return \"Image (CC BY 2.0) \" + ' - '.join(attributions[str(image_rel)].split(' - ')[:-1])\n", - " " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YIjLi-nX0txI", - "colab": {} - }, - "source": [ - "for n in range(3):\n", - " image_path = random.choice(all_image_paths)\n", - " display.display(display.Image(image_path))\n", - " print(caption_image(image_path))\n", - " print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OaNOr-co3WKk" - }, - "source": [ - "### 各画像のラベルの決定" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-weOQpDw2Jnu" - }, - "source": [ - "ラベルを一覧してみます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ssUZ7Qh96UR3", - "colab": {} - }, - "source": [ - "label_names = sorted(item.name for item in data_root.glob('*/') if item.is_dir())\n", - "label_names" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9l_JEBql2OzS" - }, - "source": [ - "ラベルにインデックスを割り当てます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y8pCV46CzPlp", - "colab": {} - }, - "source": [ - "label_to_index = dict((name, index) for index,name in enumerate(label_names))\n", - "label_to_index" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VkXsHg162T9F" - }, - "source": [ - "ファイルとラベルのインデックスの一覧を作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q62i1RBP4Q02", - "colab": {} - }, - "source": [ - "all_image_labels = [label_to_index[pathlib.Path(path).parent.name]\n", - " for path in all_image_paths]\n", - "\n", - "print(\"First 10 labels indices: \", all_image_labels[:10])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "i5L09icm9iph" - }, - "source": [ - "### 画像の読み込みと整形" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbqqRUS79ooq" - }, - "source": [ - "TensorFlowには画像を読み込んで処理するために必要なツールが備わっています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jQZdySHvksOu", - "colab": {} - }, - "source": [ - "img_path = all_image_paths[0]\n", - "img_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2t2h2XCcmK1Y" - }, - "source": [ - "以下は生のデータです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LJfkyC_Qkt7A", - "colab": {} - }, - "source": [ - "img_raw = tf.read_file(img_path)\n", - "print(repr(img_raw)[:100]+\"...\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "opN8AVc8mSbz" - }, - "source": [ - "画像のテンソルにデコードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tm0tdrlfk0Bb", - "colab": {} - }, - "source": [ - "img_tensor = tf.image.decode_image(img_raw)\n", - "\n", - "print(img_tensor.shape)\n", - "print(img_tensor.dtype)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3k-Of2Tfmbeq" - }, - "source": [ - "モデルに合わせてリサイズします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XFpz-3_vlJgp", - "colab": {} - }, - "source": [ - "img_final = tf.image.resize_images(img_tensor, [192, 192])\n", - "img_final = img_final/255.0\n", - "print(img_final.shape)\n", - "print(img_final.numpy().min())\n", - "print(img_final.numpy().max())\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aCsAa4Psl4AQ" - }, - "source": [ - "このあと使用するために、簡単な関数にまとめます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HmUiZJNU73vA", - "colab": {} - }, - "source": [ - "def preprocess_image(image):\n", - " image = tf.image.decode_jpeg(image, channels=3)\n", - " image = tf.image.resize_images(image, [192, 192])\n", - " image /= 255.0 # normalize to [0,1] range\n", - "\n", - " return image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "einETrJnO-em", - "colab": {} - }, - "source": [ - "def load_and_preprocess_image(path):\n", - " image = tf.read_file(path)\n", - " return preprocess_image(image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3brWQcdtz78y", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "image_path = all_image_paths[0]\n", - "label = all_image_labels[0]\n", - "\n", - "plt.imshow(load_and_preprocess_image(img_path))\n", - "plt.grid(False)\n", - "plt.xlabel(caption_image(img_path))\n", - "plt.title(label_names[label].title())\n", - "print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n2TCr1TQ8pA3" - }, - "source": [ - "## `tf.data.Dataset`の構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6H9Z5Mq63nSH" - }, - "source": [ - "### 画像のデータセット" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GN-s04s-6Luq" - }, - "source": [ - "`tf.data.Dataset`を構築する最も簡単な方法は、`from_tensor_slices`メソッドを使うことです。\n", - "\n", - "文字列の配列をスライスすると、文字列のデータセットが出来上がります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6oRPG3Jz3ie_", - "colab": {} - }, - "source": [ - "path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uML4JeMmIAvO" - }, - "source": [ - "`output_shapes`と`output_types`という2つのフィールドが、データセット中の要素の中身を示しています。この場合には、バイナリ文字列というスカラーのセットです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mIsNflFbIK34", - "colab": {} - }, - "source": [ - "print('shape: ', repr(path_ds.output_shapes))\n", - "print('type: ', path_ds.output_types)\n", - "print()\n", - "print(path_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZjyGcM8OwBJ2" - }, - "source": [ - "`preprocess_image`をファイルパスのデータセットにマップすることで、画像を実行時にロードし整形する新しいデータセットを作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "D1iba6f4khu-", - "colab": {} - }, - "source": [ - "image_ds = path_ds.map(load_and_preprocess_image, num_parallel_calls=AUTOTUNE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JLUPs2a-lEEJ", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.figure(figsize=(8,8))\n", - "for n,image in enumerate(image_ds.take(4)):\n", - " plt.subplot(2,2,n+1)\n", - " plt.imshow(image)\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.xlabel(caption_image(all_image_paths[n]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P6FNqPbxkbdx" - }, - "source": [ - "### `(image, label)`のペアのデータセット" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YgvrWLKG67-x" - }, - "source": [ - "同じ`from_tensor_slices`メソッドを使ってラベルのデータセットを作ることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AgBsAiV06udj", - "colab": {} - }, - "source": [ - "label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(all_image_labels, tf.int64))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HEsk5nN0vyeX", - "colab": {} - }, - "source": [ - "for label in label_ds.take(10):\n", - " print(label_names[label.numpy()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jHjgrEeTxyYz" - }, - "source": [ - "これらのデータセットは同じ順番なので、zipすることで`(image, label)`というペアのデータセットができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AOEWNMdQwsbN", - "colab": {} - }, - "source": [ - "image_label_ds = tf.data.Dataset.zip((image_ds, label_ds))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yA2F09SJLMuM" - }, - "source": [ - "新しいデータセットの`shapes`と`types`は、それぞれのフィールドを示すシェイプと型のタプルです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DuVYNinrLL-N", - "colab": {} - }, - "source": [ - "print('image shape: ', image_label_ds.output_shapes[0])\n", - "print('label shape: ', image_label_ds.output_shapes[1])\n", - "print('types: ', image_label_ds.output_types)\n", - "print()\n", - "print(image_label_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2WYMikoPWOQX" - }, - "source": [ - "注：`all_image_labels`や`all_image_paths`のような配列がある場合、`tf.data.dataset.Dataset.zip`メソッドの代わりとなるのは、配列のペアをスライスすることです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HOFwZI-2WhzV", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.from_tensor_slices((all_image_paths, all_image_labels))\n", - "\n", - "# The tuples are unpacked into the positional arguments of the mapped function\n", - "# タプルは展開され、マップ関数の位置引数に割り当てられます\n", - "def load_and_preprocess_from_path_label(path, label):\n", - " return load_and_preprocess_image(path), label\n", - "\n", - "image_label_ds = ds.map(load_and_preprocess_from_path_label)\n", - "image_label_ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vYGCgJuR_9Qp" - }, - "source": [ - "### 基本的な訓練手法" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wwZavzgsIytz" - }, - "source": [ - "このデータセットを使ってモデルの訓練を行うには、データが\n", - "\n", - "* よくシャッフルされ\n", - "* バッチ化され\n", - "* 限りなく繰り返され\n", - "* バッチが出来るだけ早く利用できる\n", - "\n", - "ことが必要です。\n", - "\n", - "これらの特性は`tf.data`APIを使えば簡単に付け加えることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uZmZJx8ePw_5", - "colab": {} - }, - "source": [ - "BATCH_SIZE = 32\n", - "\n", - "# シャッフルバッファのサイズをデータセットと同じに設定することで、データが完全にシャッフルされる\n", - "# ようにできます。\n", - "ds = image_label_ds.shuffle(buffer_size=image_count)\n", - "ds = ds.repeat()\n", - "ds = ds.batch(BATCH_SIZE)\n", - "# `prefetch`を使うことで、モデルの訓練中にバックグラウンドでデータセットがバッチを取得できます。\n", - "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6JsM-xHiFCuW" - }, - "source": [ - "注意すべきことがいくつかあります。\n", - "\n", - "1. 順番が重要です。\n", - "\n", - " * `.repeat`の前に`.shuffle`すると、エポックの境界を越えて要素がシャッフルされます。（他の要素がすべて出現する前に2回出現する要素があるかもしれません）\n", - " * `.batch`の後に`.shuffle`すると、バッチの順番がシャッフルされますが、要素がバッチを越えてシャッフルされることはありません。\n", - "\n", - "1. 完全なシャッフルのため、`buffer_size`をデータセットと同じサイズに設定しています。データセットのサイズ未満の場合、値が大きいほど良くランダム化されますが、より多くのメモリーを使用します。\n", - "\n", - "1. シャッフルバッファがいっぱいになってから要素が取り出されます。そのため、大きな`buffer_size`が`Dataset`を使い始める際の遅延の原因になります。\n", - "\n", - "1. シャッフルされたデータセットは、シャッフルバッファが完全に空になるまでデータセットが終わりであることを伝えません。`.repeat`によって`Dataset`が再起動されると、シャッフルバッファが一杯になるまでもう一つの待ち時間が発生します。\n", - "\n", - "最後の問題は、`tf.data.Dataset.apply`メソッドを、融合された`tf.data.experimental.shuffle_and_repeat`関数と組み合わせることで対処できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ocr6PybXNDoO", - "colab": {} - }, - "source": [ - "ds = image_label_ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE)\n", - "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GBBZMSuAmQVL" - }, - "source": [ - "### データセットをモデルにつなぐ\n", - "\n", - "`tf.keras.applications`からMobileNet v2のコピーを取得します。\n", - "\n", - "これを簡単な転移学習のサンプルに使用します。\n", - "\n", - "MobileNetの重みを訓練不可に設定します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KbJrXn9omO_g", - "colab": {} - }, - "source": [ - "mobile_net = tf.keras.applications.MobileNetV2(input_shape=(192, 192, 3), include_top=False)\n", - "mobile_net.trainable=False" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y7NVWiLF3Vbf" - }, - "source": [ - "このモデルは、入力が`[-1,1]`の範囲に正規化されていることを想定しています。\n", - "\n", - "```\n", - "help(keras_applications.mobilenet_v2.preprocess_input)\n", - "```\n", - "\n", - "
    \n",
-        "...\n",
-        "This function applies the \"Inception\" preprocessing which converts\n",
-        "the RGB values from [0, 255] to [-1, 1] \n",
-        "...\n",
-        "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CboYya6LmdQI" - }, - "source": [ - "このため、データをMobileNetモデルに渡す前に、入力を`[0,1]`の範囲から`[-1,1]`の範囲に変換する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SNOkHUGv3FYq", - "colab": {} - }, - "source": [ - "def change_range(image,label):\n", - " return 2*image-1, label\n", - "\n", - "keras_ds = ds.map(change_range)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QDzZ3Nye5Rpv" - }, - "source": [ - "MobileNetは画像ごとに`6x6`の特徴量の空間を返します。\n", - "\n", - "バッチを１つ渡してみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OzAhGkEK6WuE", - "colab": {} - }, - "source": [ - "# シャッフルバッファがいっぱいになるまで、データセットは何秒かかかります。\n", - "image_batch, label_batch = next(iter(keras_ds))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LcFdiWpO5WbV", - "colab": {} - }, - "source": [ - "feature_map_batch = mobile_net(image_batch)\n", - "print(feature_map_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vrbjEvaC5XmU" - }, - "source": [ - "MobileNetをラップしたモデルを作り、出力層である`tf.keras.layers.Dense`の前に、`tf.keras.layers.GlobalAveragePooling2D`で空間の軸に沿って平均値を求めます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X0ooIU9fNjPJ", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " mobile_net,\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(len(label_names))])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "foQYUJs97V4V" - }, - "source": [ - "期待したとおりの形状の出力が得られます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1nwYxvpj7ZEf", - "colab": {} - }, - "source": [ - "logit_batch = model(image_batch).numpy()\n", - "\n", - "print(\"min logit:\", logit_batch.min())\n", - "print(\"max logit:\", logit_batch.max())\n", - "print()\n", - "\n", - "print(\"Shape:\", logit_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pFc4I_J2nNOJ" - }, - "source": [ - "訓練手法を記述するためにモデルをコンパイルします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZWGqLEWYRNvv", - "colab": {} - }, - "source": [ - "model.compile(optimizer=tf.train.AdamOptimizer(), \n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=[\"accuracy\"])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tF1mO6haBOSd" - }, - "source": [ - "訓練可能な変数は2つ、全結合層の`weights`と`bias`です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pPQ5yqyKBJMm", - "colab": {} - }, - "source": [ - "len(model.trainable_variables) " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kug5Wg66UJjl", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f_glpYZ-nYC_" - }, - "source": [ - "モデルを訓練します。\n", - "\n", - "普通は、エポックごとの本当のステップ数を指定しますが、ここではデモの目的なので3ステップだけとします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AnXPRNWoTypI", - "colab": {} - }, - "source": [ - "steps_per_epoch=tf.ceil(len(all_image_paths)/BATCH_SIZE).numpy()\n", - "steps_per_epoch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q_8sabaaSGAp", - "colab": {} - }, - "source": [ - "model.fit(ds, epochs=1, steps_per_epoch=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UMVnoBcG_NlQ" - }, - "source": [ - "## 性能\n", - "\n", - "注：このセクションでは性能の向上に役立ちそうな簡単なトリックをいくつか紹介します。詳しくは、[Input Pipeline Performance](https://www.tensorflow.org/r1/guide/performance/datasets)を参照してください。\n", - "\n", - "上記の単純なパイプラインは、エポックごとにそれぞれのファイルを一つずつ読み込みます。これは、CPUを使ったローカルでの訓練では問題になりませんが、GPUを使った訓練では十分ではなく、いかなる分散訓練でも使うべきではありません。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oNmQqgGhLWie" - }, - "source": [ - "調査のため、まず、データセットの性能をチェックする簡単な関数を定義します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_gFVe1rp_MYr", - "colab": {} - }, - "source": [ - "import time\n", - "\n", - "def timeit(ds, batches=2*steps_per_epoch+1):\n", - " overall_start = time.time()\n", - " # タイマーをスタートする前に、パイプラインの初期化の（シャッフルバッファを埋める）ため、\n", - " # バッチを１つ取得します\n", - " it = iter(ds.take(batches+1))\n", - " next(it)\n", - "\n", - " start = time.time()\n", - " for i,(images,labels) in enumerate(it):\n", - " if i%10 == 0:\n", - " print('.',end='')\n", - " print()\n", - " end = time.time()\n", - "\n", - " duration = end-start\n", - " print(\"{} batches: {} s\".format(batches, duration))\n", - " print(\"{:0.5f} Images/s\".format(BATCH_SIZE*batches/duration))\n", - " print(\"Total time: {}s\".format(end-overall_start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TYiOr4vdLcNX" - }, - "source": [ - "現在のデータセットの性能は次のとおりです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZDxLwMJOReVe", - "colab": {} - }, - "source": [ - "ds = image_label_ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjouTJadRxyp", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HsLlXMO7EWBR" - }, - "source": [ - "### キャッシュ" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lV1NOn2zE2lR" - }, - "source": [ - "`tf.data.Dataset.cache`を使うと、エポックを越えて計算結果を簡単にキャッシュできます。特に、データがメモリに収まるときには効果的です。\n", - "\n", - "ここでは、画像が前処理（デコードとリサイズ）された後でキャッシュされます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qj_U09xpDvOg", - "colab": {} - }, - "source": [ - "ds = image_label_ds.cache()\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rdxpvQ7VEo3y", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "usIv7MqqZQps" - }, - "source": [ - "メモリキャッシュを使う際の欠点のひとつは、実行の都度キャッシュを再構築しなければならないことです。このため、データセットがスタートするたびに同じだけ起動のための遅延が発生します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eKX6ergKb_xd", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jUzpG4lYNkN-" - }, - "source": [ - "データがメモリに収まらない場合には、キャッシュファイルを使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vIvF8K4GMq0g", - "colab": {} - }, - "source": [ - "ds = image_label_ds.cache(filename='./cache.tf-data')\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(1)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eTIj6IOmM4yA", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qqo3dyB0Z4t2" - }, - "source": [ - "キャッシュファイルには、キャッシュを再構築することなくデータセットを再起動できるという利点もあります。2回めがどれほど早いか見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hZhVdR8MbaUj", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WqOVlf8tFrDU" - }, - "source": [ - "### TFRecord ファイル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y1llOTwWFzmR" - }, - "source": [ - "#### 生の画像データ\n", - "\n", - "TFRecordファイルは、バイナリの大きなオブジェクトのシーケンスを保存するための単純なフォーマットです。複数のサンプルを同じファイルに詰め込むことで、TensorFlowは複数のサンプルを一度に読み込むことができます。これは、特にGCSのようなリモートストレージサービスを使用する際の性能にとって重要です。\n", - "\n", - "最初に、生の画像データからTFRecordファイルを構築します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EqtARqKuHQLu", - "colab": {} - }, - "source": [ - "image_ds = tf.data.Dataset.from_tensor_slices(all_image_paths).map(tf.read_file)\n", - "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", - "tfrec.write(image_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "flR2GXWFKcO1" - }, - "source": [ - "次に、TFRecordファイルを読み込み、以前定義した`preprocess_image`関数を使って画像のデコード/リフォーマットを行うデータセットを構築します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "j9PVUL2SFufn", - "colab": {} - }, - "source": [ - "image_ds = tf.data.TFRecordDataset('images.tfrec').map(preprocess_image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cRp1eZDRKzyN" - }, - "source": [ - "これを、前に定義済みのラベルデータセットとzipし、期待通りの`(image,label)`のペアを得ます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7XI_nDU2KuhS", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.zip((image_ds, label_ds))\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3ReSapoPK22E", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wb7VyoKNOMms" - }, - "source": [ - "これは、`cache`バージョンよりも低速です。前処理をキャッシュしていないからです。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NF9W-CTKkM-f" - }, - "source": [ - "#### シリアライズしたテンソル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J9HzljSPkxt0" - }, - "source": [ - "前処理をTFRecordファイルに保存するには、前やったように前処理した画像のデータセットを作ります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OzS0Azukkjyw", - "colab": {} - }, - "source": [ - "paths_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)\n", - "image_ds = paths_ds.map(load_and_preprocess_image)\n", - "image_ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "onWOwLpYlzJQ" - }, - "source": [ - "`.jpeg`文字列のデータセットではなく、これはテンソルのデータセットです。\n", - "\n", - "これをTFRecordファイルにシリアライズするには、まず、テンソルのデータセットを文字列のデータセットに変換します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xxZSwnRllyf0", - "colab": {} - }, - "source": [ - "ds = image_ds.map(tf.serialize_tensor)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "w9N6hJWAkKPC", - "colab": {} - }, - "source": [ - "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", - "tfrec.write(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OlFc9dJSmcx0" - }, - "source": [ - "前処理をキャッシュしたことにより、データはTFRecordファイルから非常に効率的にロードできます。テンソルを使用する前にデシリアライズすることを忘れないでください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BsqFyTBFmSCZ", - "colab": {} - }, - "source": [ - "RESTORE_TYPE = image_ds.output_types\n", - "RESTORE_SHAPE = image_ds.output_shapes\n", - "\n", - "ds = tf.data.TFRecordDataset('images.tfrec')\n", - "\n", - "def parse(x):\n", - " result = tf.parse_tensor(x, out_type=RESTORE_TYPE)\n", - " result = tf.reshape(result, RESTORE_SHAPE)\n", - " return result\n", - "\n", - "ds = ds.map(parse, num_parallel_calls=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OPs_sLV9pQg5" - }, - "source": [ - "次にラベルを追加し、以前と同じような標準的な処理を適用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XYxBwaLYnGop", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.zip((ds, label_ds))\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W8X6RmGan1-P", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/load_data/tf_records.ipynb b/site/ja/r1/tutorials/load_data/tf_records.ipynb deleted file mode 100644 index 79609ce4a15..00000000000 --- a/site/ja/r1/tutorials/load_data/tf_records.ipynb +++ /dev/null @@ -1,1285 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "tf_records.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "pL--_KGdYoBz" - ], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pL--_KGdYoBz" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "both", - "colab_type": "code", - "id": "uBDvXpYzYnGj", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HQzaEQuJiW_d" - }, - "source": [ - "# TFRecords と `tf.Example` の使用法" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hWk5m9autbk8", - "colab_type": "text" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AvXexNxlLoFx", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3pkUd_9IZCFO" - }, - "source": [ - "データの読み込みを効率的にするには、データをシリアライズし、連続的に読み込めるファイルのセット（各ファイルは100-200MB）に保存することが有効です。データをネットワーク経由で流そうとする場合には、特にそうです。また、データの前処理をキャッシュする際にも役立ちます。\n", - "\n", - "TFRecord形式は、バイナリレコードの系列を保存するための単純な形式です。\n", - "\n", - "[プロトコルバッファ](https://developers.google.com/protocol-buffers/) は、構造化データを効率的にシリアライズする、プラットフォームや言語に依存しないライブラリです。\n", - "\n", - "プロトコルメッセージは`.proto`という拡張子のファイルで定義されます。メッセージ型を識別する最も簡単な方法です。\n", - "\n", - "`tf.Example`メッセージ（あるいはプロトコルバッファ）は、`{\"string\": value}`形式のマッピングを表現する柔軟なメッセージ型です。これは、TensorFlow用に設計され、[TFX](https://www.tensorflow.org/tfx/)のような上位レベルのAPIで共通に使用されています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ac83J0QxjhFt" - }, - "source": [ - "このノートブックでは、`tf.Example`メッセージの作成、パースと使用法をデモし、その後、`tf.Example`メッセージをパースして、`.tfrecord`に書き出し、その後読み取る方法を示します。\n", - "\n", - "注：こうした構造は有用ですが必ずそうしなければならなというものではありません。[`tf.data`](https://www.tensorflow.org/r1/guide/datasets) を使っていて、それでもなおデータの読み込みが訓練のボトルネックである場合でなければ、既存のコードをTFRecordsを使用するために変更する必要はありません。データセットの性能改善のヒントは、 [Data Input Pipeline Performance](https://www.tensorflow.org/r1/guide/performance/datasets)を参照ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WkRreBf1eDVc" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ja7sezsmnXph", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "tf.enable_eager_execution()\n", - "\n", - "import numpy as np\n", - "import IPython.display as display" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e5Kq88ccUWQV" - }, - "source": [ - "## `tf.Example`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VrdQHgvNijTi" - }, - "source": [ - "### `tf.Example`用のデータ型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lZw57Qrn4CTE" - }, - "source": [ - "基本的には`tf.Example`は`{\"string\": tf.train.Feature}`というマッピングです。\n", - "\n", - "`tf.train.Feature`メッセージ型は次の3つの型のうち1つをとることができます（[.proto file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto)を参照）。一般的なデータ型の多くは、これらの型のいずれかに強制的に変換することができます。\n", - "\n", - "1. `tf.train.BytesList` (次の型のデータを扱うことが可能)\n", - " - `string`\n", - " - `byte` \n", - "1. `tf.train.FloatList` (次の型のデータを扱うことが可能)\n", - " - `float` (`float32`)\n", - " - `double` (`float64`) \n", - "1. `tf.train.Int64List` (次の型のデータを扱うことが可能)\n", - " - `bool`\n", - " - `enum`\n", - " - `int32`\n", - " - `uint32`\n", - " - `int64`\n", - " - `uint64`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_e3g9ExathXP" - }, - "source": [ - "通常のTensorFlowの型を`tf.Example`互換の `tf.train.Feature`に変換するには、次のショートカット関数を使うことができます。\n", - "\n", - "どの関数も、1個のスカラー値を入力とし、上記の3つの`list`型のうちの一つを含む`tf.train.Feature`を返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mbsPOUpVtYxA", - "colab": {} - }, - "source": [ - "# 下記の関数を使うと値を tf.Exampleと互換性の有る型に変換できる\n", - "\n", - "def _bytes_feature(value):\n", - " \"\"\"string / byte 型から byte_listを返す\"\"\"\n", - " return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n", - "\n", - "def _float_feature(value):\n", - " \"\"\"float / double 型から float_listを返す\"\"\"\n", - " return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n", - "\n", - "def _int64_feature(value):\n", - " \"\"\"bool / enum / int / uint 型から Int64_listを返す\"\"\"\n", - " return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wst0v9O8hgzy" - }, - "source": [ - "注：単純化のため、このサンプルではスカラー値の入力のみを扱っています。スカラー値ではない特徴を扱う最も簡単な方法は、`tf.serialize_tensor`を使ってテンソルをバイナリ文字列に変換する方法です。TensorFlowでは文字列はスカラー値として扱います。バイナリ文字列をテンソルに戻すには、`tf.parse_tensor`を使用します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vsMbkkC8xxtB" - }, - "source": [ - "上記の関数の使用例を下記に示します。入力が様々な型であるのに対して、出力が標準化されていることに注目してください。入力が、強制変換できない型であった場合、例外が発生します。（例：`_int64_feature(1.0)`はエラーとなります。`1.0`が浮動小数点数であるためで、代わりに`_float_feature`関数を使用すべきです）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hZzyLGr0u73y", - "colab": {} - }, - "source": [ - "print(_bytes_feature(b'test_string'))\n", - "print(_bytes_feature(u'test_bytes'.encode('utf-8')))\n", - "\n", - "print(_float_feature(np.exp(1)))\n", - "\n", - "print(_int64_feature(True))\n", - "print(_int64_feature(1))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nj1qpfQU5qmi" - }, - "source": [ - "メッセージはすべて`.SerializeToString` を使ってバイナリ文字列にシリアライズすることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5afZkORT5pjm", - "colab": {} - }, - "source": [ - "feature = _float_feature(np.exp(1))\n", - "\n", - "feature.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "laKnw9F3hL-W" - }, - "source": [ - "### `tf.Example` メッセージの作成" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b_MEnhxchQPC" - }, - "source": [ - "既存のデータから`tf.Example`を作成したいとします。実際には、データセットの出処はどこでも良いのですが、1件の観測記録から`tf.Example`メッセージを作る手順は同じです。\n", - "\n", - "1. 観測記録それぞれにおいて、各値は上記の関数を使って3種類の互換性のある型のうち1つだけを含む`tf.train.Feature`に変換する必要があります。\n", - "\n", - "1. 次に、特徴の名前を表す文字列と、#1で作ったエンコード済みの特徴量を対応させたマップ（ディクショナリ）を作成します。\n", - "\n", - "1. #2で作成したマップを[Featuresメッセージ](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto#L85)に変換します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4EgFQ2uHtchc" - }, - "source": [ - "このノートブックでは、NumPyを使ってデータセットを作成します。\n", - "\n", - "このデータセットには4つの特徴量があります。\n", - "- `False` または `True`を表す論理値。出現確率は等しいものとします。\n", - "- `[0,5)`の範囲から一様にサンプリングした整数値。\n", - "- 整数特徴量をインデックスとした文字列テーブルを使って生成した文字列特徴量\n", - "- 標準正規分布からサンプリングした浮動小数点数。\n", - "\n", - "サンプルは上記の分布から独立して同じ様に分布した10,000件の観測記録からなるものとします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CnrguFAy3YQv", - "colab": {} - }, - "source": [ - "# データセットに含まれる観測結果の件数\n", - "n_observations = int(1e4)\n", - "\n", - "# ブール特徴量 FalseまたはTrueとしてエンコードされている\n", - "feature0 = np.random.choice([False, True], n_observations)\n", - "\n", - "# 整数特徴量 0以上 5未満の乱数\n", - "feature1 = np.random.randint(0, 5, n_observations)\n", - "\n", - "# バイト文字列特徴量\n", - "strings = np.array([b'cat', b'dog', b'chicken', b'horse', b'goat'])\n", - "feature2 = strings[feature1]\n", - "\n", - "# 浮動小数点数特徴量 標準正規分布から発生\n", - "feature3 = np.random.randn(n_observations)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aGrscehJr7Jd" - }, - "source": [ - "これらの特徴量は、`_bytes_feature`, `_float_feature`, `_int64_feature`のいずれかを使って`tf.Example`互換の型に強制変換されます。その後、エンコード済みの特徴量から`tf.Example`メッセージを作成できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RTCS49Ij_kUw", - "colab": {} - }, - "source": [ - "def serialize_example(feature0, feature1, feature2, feature3):\n", - " \"\"\"\n", - " Creates a tf.Example message ready to be written to a file.\n", - " ファイル出力可能なtf.Exampleメッセージを作成する\n", - " \"\"\"\n", - "\n", - " # 特徴量名とtf.Example互換データ型との対応ディクショナリを作成\n", - "\n", - " feature = {\n", - " 'feature0': _int64_feature(feature0),\n", - " 'feature1': _int64_feature(feature1),\n", - " 'feature2': _bytes_feature(feature2),\n", - " 'feature3': _float_feature(feature3),\n", - " }\n", - "\n", - " # tf.train.Exampleを用いて特徴メッセージを作成\n", - "\n", - " example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n", - " return example_proto.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XftzX9CN_uGT" - }, - "source": [ - "例えば、データセットに`[False, 4, bytes('goat'), 0.9876]`という1つの観測記録があるとします。`create_message()`を使うとこの観測記録から`tf.Example`メッセージを作成し印字できます。上記のように、観測記録一つ一つが`Features`メッセージとして書かれています。`tf.Example` [メッセージ](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88)は、この`Features` メッセージを包むラッパーに過ぎないことに注意してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N8BtSx2RjYcb", - "colab": {} - }, - "source": [ - "# データセットからの観測記録の例\n", - "\n", - "example_observation = []\n", - "\n", - "serialized_example = serialize_example(False, 4, b'goat', 0.9876)\n", - "serialized_example" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_pbGATlG6u-4" - }, - "source": [ - "メッセージをデコードするには、`tf.train.Example.FromString`メソッドを使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dGim-mEm6vit", - "colab": {} - }, - "source": [ - "example_proto = tf.train.Example.FromString(serialized_example)\n", - "example_proto" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZHV-ff6YHtDG", - "colab_type": "text" - }, - "source": [ - "## TFRecordフォーマットの詳細\n", - "\n", - "TFRecordファイルにはレコードのシーケンスが含まれます。このファイルはシーケンシャル読み取りのみが可能です。\n", - "\n", - "それぞれのレコードには、データを格納するためのバイト文字列とデータ長、そして整合性チェックのためのCRC32C（Castagnoli多項式を使った32ビットのCRC）ハッシュ値が含まれます。\n", - "\n", - "各レコードのフォーマットは下記の通りです。\n", - "\n", - " uint64 長さ\n", - " uint32 長さのマスク済みcrc32ハッシュ値\n", - " byte data[長さ]\n", - " uint32 データのマスク済みcrc32ハッシュ値\n", - "\n", - "複数のレコードが結合されてファイルを構成します。CRCについては[ここ](https://en.wikipedia.org/wiki/Cyclic_redundancy_check)に說明があります。CRCのマスクは下記のとおりです。\n", - "\n", - " masked_crc = ((crc >> 15) | (crc << 17)) + 0xa282ead8ul\n", - " \n", - "注：TFRecordファイルを作るのに、`tf.Example`を使わなければならないということはありません。`tf.Example`は、ディクショナリをバイト文字列にシリアライズする方法の1つです。エンコードされた画像データや、（`tf.io.serialize_tensor`を使ってシリアライズされ、`tf.io.parse_tensor`で読み込まれる）シリアライズされたテンソルもあります。その他のオプションについては、`tf.io`モジュールを参照してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y-Hjmee-fbLH" - }, - "source": [ - "## `tf.data`を使用したTFRecordファイル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GmehkCCT81Ez" - }, - "source": [ - "`tf.data`モジュールには、TensorFlowでデータを読み書きするツールが含まれます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1FISEuz8ubu3" - }, - "source": [ - "### TFRecordファイルの書き出し\n", - "\n", - "データをデータセットにする最も簡単な方法は`from_tensor_slices`メソッドです。\n", - "\n", - "配列に適用すると、このメソッドはスカラー値のデータセットを返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mXeaukvwu5_-", - "colab": {} - }, - "source": [ - "tf.data.Dataset.from_tensor_slices(feature1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f-q0VKyZvcad" - }, - "source": [ - "配列のタプルに適用すると、タプルのデータセットが返されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "H5sWyu1kxnvg", - "colab": {} - }, - "source": [ - "features_dataset = tf.data.Dataset.from_tensor_slices((feature0, feature1, feature2, feature3))\n", - "features_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m1C-t71Nywze", - "colab": {} - }, - "source": [ - "# データセットから1つのサンプルだけを取り出すには`take(1)` を使います。\n", - "for f0,f1,f2,f3 in features_dataset.take(1):\n", - " print(f0)\n", - " print(f1)\n", - " print(f2)\n", - " print(f3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mhIe63awyZYd" - }, - "source": [ - "`Dataset`のそれぞれの要素に関数を適用するには、`tf.data.Dataset.map`メソッドを使用します。\n", - "\n", - "マップされる関数はTensorFlowのグラフモードで動作する必要があります。関数は`tf.Tensors`を処理し、返す必要があります。`create_example`のような非テンソル関数は、互換性のため`tf.py_func`でラップすることができます。\n", - "\n", - "`tf.py_func`を使用する際には、シェイプと型は取得できないため、指定する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "apB5KYrJzjPI", - "colab": {} - }, - "source": [ - "def tf_serialize_example(f0,f1,f2,f3):\n", - " tf_string = tf.py_func(\n", - " serialize_example, \n", - " (f0,f1,f2,f3), # pass these args to the above function.\n", - " tf.string) # the return type is `tf.string`.\n", - " return tf.reshape(tf_string, ()) # The result is a scalar" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CrFZ9avE3HUF" - }, - "source": [ - "この関数をデータセットのそれぞれの要素に適用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VDeqYVbW3ww9", - "colab": {} - }, - "source": [ - "serialized_features_dataset = features_dataset.map(tf_serialize_example)\n", - "serialized_features_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p6lw5VYpjZZC" - }, - "source": [ - "TFRecordファイルに書き出します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vP1VgTO44UIE", - "colab": {} - }, - "source": [ - "filename = 'test.tfrecord'\n", - "writer = tf.data.experimental.TFRecordWriter(filename)\n", - "writer.write(serialized_features_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6aV0GQhV8tmp" - }, - "source": [ - "### TFRecordファイルの読み込み" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o3J5D4gcSy8N" - }, - "source": [ - "`tf.data.TFRecordDataset`クラスを使ってTFRecordファイルを読み込むこともできます。\n", - "\n", - "`tf.data`を使ってTFRecordファイルを取り扱う際の詳細については、[こちら](https://www.tensorflow.org/r1/guide/datasets#consuming_tfrecord_data)を参照ください。 \n", - "\n", - "`TFRecordDataset`を使うことは、入力データを標準化し、パフォーマンスを最適化するのに有用です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6OjX6UZl-bHC", - "colab": {} - }, - "source": [ - "filenames = [filename]\n", - "raw_dataset = tf.data.TFRecordDataset(filenames)\n", - "raw_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6_EQ9i2E_-Fz" - }, - "source": [ - "この時点で、データセットにはシリアライズされた`tf.train.Example`メッセージが含まれています。データセットをイテレートすると、スカラーの文字列テンソルが返ってきます。\n", - "\n", - "`.take`メソッドを使って最初の10レコードだけを表示します。\n", - "\n", - "注：`tf.data.Dataset`をイテレートできるのは、Eager Executionが有効になっている場合のみです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hxVXpLz_AJlm", - "colab": {} - }, - "source": [ - "for raw_record in raw_dataset.take(10):\n", - " print(repr(raw_record))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W-6oNzM4luFQ" - }, - "source": [ - "これらのテンソルは下記の関数でパースできます。\n", - "\n", - "注：ここでは、`feature_description`が必要です。データセットはグラフ実行を使用するため、この記述を使ってシェイプと型を構築するのです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zQjbIR1nleiy", - "colab": {} - }, - "source": [ - "# 特徴の記述\n", - "feature_description = {\n", - " 'feature0': tf.FixedLenFeature([], tf.int64, default_value=0),\n", - " 'feature1': tf.FixedLenFeature([], tf.int64, default_value=0),\n", - " 'feature2': tf.FixedLenFeature([], tf.string, default_value=''),\n", - " 'feature3': tf.FixedLenFeature([], tf.float32, default_value=0.0),\n", - "}\n", - "\n", - "def _parse_function(example_proto):\n", - " # 上記の記述を使って入力のtf.Exampleを処理\n", - " return tf.parse_single_example(example_proto, feature_description)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gWETjUqhEQZf" - }, - "source": [ - "あるいは、`tf.parse example`を使ってバッチ全体を一度にパースします。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AH73hav6Bnmg" - }, - "source": [ - "`tf.data.Dataset.map`メソッドを使って、データセットの各アイテムにこの関数を適用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ob7D-zmBm1w", - "colab": {} - }, - "source": [ - "parsed_dataset = raw_dataset.map(_parse_function)\n", - "parsed_dataset " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sNV-XclGnOvn" - }, - "source": [ - "Eager Execution を使ってデータセット中の観測記録を表示します。このデータセットには10,000件の観測記録がありますが、最初の10個だけ表示します。 \n", - "データは特徴量のディクショナリの形で表示されます。それぞれの項目は`tf.Tensor`であり、このテンソルの`numpy` 要素は特徴量を表します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x2LT2JCqhoD_", - "colab": {} - }, - "source": [ - "for parsed_record in parsed_dataset.take(10):\n", - " print(repr(raw_record))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cig9EodTlDmg" - }, - "source": [ - "ここでは、`tf.parse_example` が`tf.Example`のフィールドを通常のテンソルに展開しています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jyg1g3gU7DNn" - }, - "source": [ - "## tf.python_ioを使ったTFRecordファイル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3FXG3miA7Kf1" - }, - "source": [ - "`tf.python_io`モジュールには、TFRecordファイルの読み書きのための純粋なPython関数も含まれています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CKn5uql2lAaN" - }, - "source": [ - "### TFRecordファイルの書き出し" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LNW_FA-GQWXs" - }, - "source": [ - "次にこの10,000件の観測記録を`test.tfrecords`ファイルに出力します。観測記録はそれぞれ`tf.Example`メッセージに変換され、ファイルに出力されます。その後、`test.tfrecords`ファイルが作成されたことを確認することができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MKPHzoGv7q44", - "colab": {} - }, - "source": [ - "# `tf.Example`観測記録をファイルに出力\n", - "with tf.python_io.TFRecordWriter(filename) as writer:\n", - " for i in range(n_observations):\n", - " example = serialize_example(feature0[i], feature1[i], feature2[i], feature3[i])\n", - " writer.write(example)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EjdFHHJMpUUo", - "colab": {} - }, - "source": [ - "!ls" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wtQ7k0YWQ1cz" - }, - "source": [ - "### TFRecordファイルの読み込み" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "utkozytkQ-2K" - }, - "source": [ - "モデルに入力にするため、このデータを読み込みたいとしましょう。\n", - "\n", - "次の例では、データをそのまま、`tf.Example`メッセージとしてインポートします。これは、ファイルが期待されるデータを含んでいるかを確認するのに役に立ちます。これは、また、入力データがTFRecordとして保存されているが、[この](https://www.tensorflow.org/r1/guide/datasets#consuming_numpy_arrays)例のようにNumPyデータ（またはそれ以外のデータ型）として入力したい場合に有用です。このコーディング例では値そのものを読み取れるからです。\n", - "\n", - "入力ファイルの中のTFRecordをイテレートして、`tf.Example`メッセージを取り出し、その中の値を読み取って保存できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "36ltP9B8OezA", - "colab": {} - }, - "source": [ - "record_iterator = tf.python_io.tf_record_iterator(path=filename)\n", - "\n", - "for string_record in record_iterator:\n", - " example = tf.train.Example()\n", - " example.ParseFromString(string_record)\n", - " \n", - " print(example)\n", - " \n", - " # Exit after 1 iteration as this is purely demonstrative.\n", - " # 純粋にデモであるため、イテレーションの1回目で終了\n", - " break" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "i3uquiiGTZTK" - }, - "source": [ - "（上記で作成した`tf.Example`型の）`example`オブジェクトの特徴量は（他のプロトコルバッファメッセージと同様に）ゲッターを使ってアクセス可能です。`example.features`は`repeated feature`メッセージを返し、`feature`メッセージをを取得すると（Pythonのディクショナリとして保存された）特徴量の名前と特徴量の値のマッピングが得られます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-UNzS7vsUBs0", - "colab": {} - }, - "source": [ - "print(dict(example.features.feature))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u1M-WrbqUUVW" - }, - "source": [ - "このディクショナリから、指定した値をディクショナリとして得ることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2yCBu70IUb2H", - "colab": {} - }, - "source": [ - "print(example.features.feature['feature3'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4dw6_OI9UiNZ" - }, - "source": [ - "次に、ゲッターを使って値にアクセスできます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BdDYjDnDUlFe", - "colab": {} - }, - "source": [ - "print(example.features.feature['feature3'].float_list.value)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S0tFDrwdoj3q" - }, - "source": [ - "## ウォークスルー： 画像データの読み書き" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rjN2LFxFpcR9" - }, - "source": [ - "以下は、TFRecordを使って画像データを読み書きする方法の例です。この例の目的は、データ（この場合は画像）を入力し、そのデータをTFRecordファイルに書き込んで、再びそのファイルを読み込み、画像を表示するという手順を最初から最後まで示すことです。\n", - "\n", - "これは、例えば、同じ入力データセットを使って複数のモデルを構築するといった場合に役立ちます。画像データをそのまま保存する代わりに、TFRecord形式に前処理しておき、その後の処理やモデル構築に使用することができます。\n", - "\n", - "まずは、雪の中の猫の[画像](https://commons.wikimedia.org/wiki/File:Felis_catus-cat_on_snow.jpg)と、ニューヨーク市にあるウイリアムズバーグ橋の [写真](https://upload.wikimedia.org/wikipedia/commons/f/fe/New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg)をダウンロードしましょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5Lk2qrKvN0yu" - }, - "source": [ - "### 画像の取得" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3a0fmwg8lHdF", - "colab": {} - }, - "source": [ - "cat_in_snow = tf.keras.utils.get_file('320px-Felis_catus-cat_on_snow.jpg', 'https://upload.wikimedia.org/wikipedia/commons/thumb/b/b6/Felis_catus-cat_on_snow.jpg/320px-Felis_catus-cat_on_snow.jpg')\n", - "williamsburg_bridge = tf.keras.utils.get_file('194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg','https://upload.wikimedia.org/wikipedia/commons/thumb/f/fe/New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg/194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7aJJh7vENeE4", - "colab": {} - }, - "source": [ - "display.display(display.Image(filename=cat_in_snow))\n", - "display.display(display.HTML('Image cc-by: Von.grzanka'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KkW0uuhcXZqA", - "colab": {} - }, - "source": [ - "display.display(display.Image(filename=williamsburg_bridge))\n", - "display.display(display.HTML('source'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VSOgJSwoN5TQ" - }, - "source": [ - "### TFRecordファイルの書き出し" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Azx83ryQEU6T" - }, - "source": [ - "上記で行ったように、この特徴量を`tf.Example`と互換のデータ型にエンコードできます。この場合には、生画像のバイト文字列を特徴として保存するだけではなく、縦、横のサイズにチャネル数、更に画像を保存する際に猫の画像と橋の画像を区別するための`label`特徴量を付け加えます。猫の画像には`0`を、橋の画像には`1`を使うことにしましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kC4TS1ZEONHr", - "colab": {} - }, - "source": [ - "image_labels = {\n", - " cat_in_snow : 0,\n", - " williamsburg_bridge : 1,\n", - "}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c5njMSYNEhNZ", - "colab": {} - }, - "source": [ - "# 猫の画像を使った例\n", - "image_string = open(cat_in_snow, 'rb').read()\n", - "\n", - "label = image_labels[cat_in_snow]\n", - "\n", - "# 関連する特徴量のディクショナリを作成\n", - "def image_example(image_string, label):\n", - " image_shape = tf.image.decode_jpeg(image_string).shape\n", - "\n", - " feature = {\n", - " 'height': _int64_feature(image_shape[0]),\n", - " 'width': _int64_feature(image_shape[1]),\n", - " 'depth': _int64_feature(image_shape[2]),\n", - " 'label': _int64_feature(label),\n", - " 'image_raw': _bytes_feature(image_string),\n", - " }\n", - "\n", - " return tf.train.Example(features=tf.train.Features(feature=feature))\n", - "\n", - "for line in str(image_example(image_string, label)).split('\\n')[:15]:\n", - " print(line)\n", - "print('...')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2G_o3O9MN0Qx" - }, - "source": [ - "ご覧のように、すべての特徴量が`tf.Example`メッセージに保存されました。上記のコードを関数化し、このサンプルメッセージを`images.tfrecords`ファイルに書き込みます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qcw06lQCOCZU", - "colab": {} - }, - "source": [ - "# 生の画像をimages.tfrecordsファイルに書き出す\n", - "# まず、2つの画像をtf.Exampleメッセージに変換し、\n", - "# 次に.tfrecordsファイルに書き出す\n", - "with tf.python_io.TFRecordWriter('images.tfrecords') as writer:\n", - " for filename, label in image_labels.items():\n", - " image_string = open(filename, 'rb').read()\n", - " tf_example = image_example(image_string, label)\n", - " writer.write(tf_example.SerializeToString())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yJrTe6tHPCfs", - "colab": {} - }, - "source": [ - "!ls" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jJSsCkZLPH6K" - }, - "source": [ - "### TFRecordファイルの読み込み\n", - "\n", - "これで、`images.tfrecords`ファイルができました。このファイルの中のレコードをイテレートし、書き込んだものを読み出します。このユースケースでは、画像を復元するだけなので、必要なのは生画像の文字列だけです。上記のゲッター、すなわち、`example.features.feature['image_raw'].bytes_list.value[0]`を使って抽出することができます。猫と橋のどちらであるかを決めるため、ラベルも使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "M6Cnfd3cTKHN", - "colab": {} - }, - "source": [ - "raw_image_dataset = tf.data.TFRecordDataset('images.tfrecords')\n", - "\n", - "# 特徴量を記述するディクショナリを作成\n", - "image_feature_description = {\n", - " 'height': tf.FixedLenFeature([], tf.int64),\n", - " 'width': tf.FixedLenFeature([], tf.int64),\n", - " 'depth': tf.FixedLenFeature([], tf.int64),\n", - " 'label': tf.FixedLenFeature([], tf.int64),\n", - " 'image_raw': tf.FixedLenFeature([], tf.string),\n", - "}\n", - "\n", - "def _parse_image_function(example_proto):\n", - " # 入力のtf.Exampleのプロトコルバッファを上記のディクショナリを使って解釈\n", - " return tf.parse_single_example(example_proto, image_feature_description)\n", - "\n", - "parsed_image_dataset = raw_image_dataset.map(_parse_image_function)\n", - "parsed_image_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0PEEFPk4NEg1" - }, - "source": [ - "TFRecordファイルから画像を復元しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yZf8jOyEIjSF", - "colab": {} - }, - "source": [ - "for image_features in parsed_image_dataset:\n", - " image_raw = image_features['image_raw'].numpy()\n", - " display.display(display.Image(data=image_raw))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/r1/tutorials/non-ml/mandelbrot.md b/site/ja/r1/tutorials/non-ml/mandelbrot.md deleted file mode 100644 index 0b81c9f9ae6..00000000000 --- a/site/ja/r1/tutorials/non-ml/mandelbrot.md +++ /dev/null @@ -1,120 +0,0 @@ -# マンデルブロ集合 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -[マンデルブロ集合](https://en.wikipedia.org/wiki/Mandelbrot_set)の可視化は -機械学習とは何の関係もありませんが、 -一般的な数学に対してTensorFlowを使う面白い例として役立ちます。 -実際には可視化の非常に単純な実装ですが、この例は要点を示しています。 -（最終的にはより美しい画像を生成するためにもっと複雑な実装を -提供することになるかもしれません。） - -## 基本的な設定 - -始めにいくつかインポートが必要です。 - -```python -# シミュレーションのためにライブラリをインポート -import tensorflow as tf -import numpy as np - -# 可視化のためにインポート -import PIL.Image -from io import BytesIO -from IPython.display import Image, display -``` - -今度は、反復回数を受け取って実際に画像を表示する関数を定義します。 - -```python -def DisplayFractal(a, fmt='jpeg'): - """色彩豊かなフラクタルの画像として - 反復回数の配列を表示します""" - a_cyclic = (6.28*a/20.0).reshape(list(a.shape)+[1]) - img = np.concatenate([10+20*np.cos(a_cyclic), - 30+50*np.sin(a_cyclic), - 155-80*np.cos(a_cyclic)], 2) - img[a==a.max()] = 0 - a = img - a = np.uint8(np.clip(a, 0, 255)) - f = BytesIO() - PIL.Image.fromarray(a).save(f, fmt) - display(Image(data=f.getvalue())) -``` - -## セッションと変数の初期化 - -実際に試すために、私たちはよく対話型セッションを使いますが、 -通常のセッションでも同様にうまく動きます。 - -```python -sess = tf.InteractiveSession() -``` - -NumPyとTensorFlowを自由に組み合わせることができて便利です。 - -```python -# Use NumPy to create a 2D array of complex numbers - -Y, X = np.mgrid[-1.3:1.3:0.005, -2:1:0.005] -Z = X+1j*Y -``` - -それでは、TensorFlowのテンソルの定義と初期化をします。 - -```python -xs = tf.constant(Z.astype(np.complex64)) -zs = tf.Variable(xs) -ns = tf.Variable(tf.zeros_like(xs, tf.float32)) -``` - -TensorFlowではテンソルの使用するために明示的に初期化する必要があります。 - -```python -tf.global_variables_initializer().run() -``` - -## 計算の定義と実行 - -それでは、追加の計算を記述します... - -```python -# 新しいzの値を計算します z: z^2 + x -zs_ = zs*zs + xs - -# 新しい値は発散しているか？ -not_diverged = tf.abs(zs_) < 4 - -# zsと反復回数を更新する演算 -# -# 注意: 新しいzsの値が発散していてもzsを更新し続けている！これは大変無駄である！ -# 多少単純ではなくなるが、これを実行するより良い方法はある -# -step = tf.group( - zs.assign(zs_), - ns.assign_add(tf.cast(not_diverged, tf.float32)) - ) -``` - -... そして200ステップ実行します。 - -```python -for i in range(200): step.run() -``` - -結果を見てみましょう。 - -```python -DisplayFractal(ns.eval()) -``` - -![jpeg](https://www.tensorflow.org/images/mandelbrot_output.jpg) - -なかなか悪くないですね！ - diff --git a/site/ja/r1/tutorials/non-ml/pdes.md b/site/ja/r1/tutorials/non-ml/pdes.md deleted file mode 100644 index 45f44cb3ef0..00000000000 --- a/site/ja/r1/tutorials/non-ml/pdes.md +++ /dev/null @@ -1,146 +0,0 @@ -# 偏微分方程式 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -TensorFlowは機械学習だけのものではありません。ここでは[偏微分方程式]( -https://en.wikipedia.org/wiki/Partial_differential_equation)の振る舞いを -シミュレートするためにTensorFlowを使った(ややありきたりな)例を示します。 -いくつかの雨粒が落ちた場合における正方形の池の表面をシミュレートします。 - -## 基本的な設定 - -いくつかのインポートが必要になります。 - -```python -# シミュレーションのためにライブラリをインポート -import tensorflow as tf -import numpy as np - -# 可視化のためにインポート -import PIL.Image -from io import BytesIO -from IPython.display import clear_output, Image, display -``` - -以下は画像として池の表面の状態を表示する関数です。 - -```python -def DisplayArray(a, fmt='jpeg', rng=[0,1]): - """画像として配列を表示する""" - a = (a - rng[0])/float(rng[1] - rng[0])*255 - a = np.uint8(np.clip(a, 0, 255)) - f = BytesIO() - PIL.Image.fromarray(a).save(f, fmt) - clear_output(wait = True) - display(Image(data=f.getvalue())) -``` - -ここで私たちは遊んで便利に対話型TensorFlowセッションを開始します。 -実行可能な.pyファイルでこれを行っている場合は、 -通常のセッションも同様に機能します。 - -```python -sess = tf.InteractiveSession() -``` - -## 計算の便宜関数 - - -```python -def make_kernel(a): - """畳込みカーネルに2次元配列を変形する""" - a = np.asarray(a) - a = a.reshape(list(a.shape) + [1,1]) - return tf.constant(a, dtype=1) - -def simple_conv(x, k): - """簡略化された2次元畳込み演算""" - x = tf.expand_dims(tf.expand_dims(x, 0), -1) - y = tf.nn.depthwise_conv2d(x, k, [1, 1, 1, 1], padding='SAME') - return y[0, :, :, 0] - -def laplace(x): - """配列の2次元ラプラシアンを計算する""" - laplace_k = make_kernel([[0.5, 1.0, 0.5], - [1.0, -6., 1.0], - [0.5, 1.0, 0.5]]) - return simple_conv(x, laplace_k) -``` - -## PDEの定義 - -私たちの池は完全500 x 500の正方形で、 -自然界に見られるほとんどの池のケースと同様なものとなっています。 - -```python -N = 500 -``` - -ここで私たちは池を作り、そこにいくつかの雨粒を降らせます。 - -```python -# 初期条件 -- いくつかの雨粒を池に降らせる - -# すべてを0で設定 -u_init = np.zeros([N, N], dtype=np.float32) -ut_init = np.zeros([N, N], dtype=np.float32) - -# いくつかの雨粒をランダムなポイントで池に降らせる -for n in range(40): - a,b = np.random.randint(0, N, 2) - u_init[a,b] = np.random.uniform() - -DisplayArray(u_init, rng=[-0.1, 0.1]) -``` - -![jpeg](https://www.tensorflow.org/images/pde_output_1.jpg) - - -それでは、微分方程式の詳細を記述しましょう。 - - -```python -# パラメータ: -# eps -- 時間分解能 -# damping -- 波減衰 -eps = tf.placeholder(tf.float32, shape=()) -damping = tf.placeholder(tf.float32, shape=()) - -# シミュレーションの状態のための変数を生成する -U = tf.Variable(u_init) -Ut = tf.Variable(ut_init) - -# 離散化したPDEの更新ルール -U_ = U + eps * Ut -Ut_ = Ut + eps * (laplace(U) - damping * Ut) - -# 状態を更新するための演算 -step = tf.group( - U.assign(U_), - Ut.assign(Ut_)) -``` - -## シミュレーションの実行 - -ここがおもしろくなるところです -- 単純なforループで時間を進めてみましょう。 - -```python -# 状態を初期条件で初期化する -tf.global_variables_initializer().run() - -# PDEを1000ステップ実行 -for i in range(1000): - # シミュレーションのステップ実行 - step.run({eps: 0.03, damping: 0.04}) - DisplayArray(U.eval(), rng=[-0.1, 0.1]) -``` - -![jpeg](https://www.tensorflow.org/images/pde_output_2.jpg) - -見てください! さざ波です! diff --git a/site/ja/tfx/guide/index.md b/site/ja/tfx/guide/index.md deleted file mode 100644 index 8cd43f8f23b..00000000000 --- a/site/ja/tfx/guide/index.md +++ /dev/null @@ -1,585 +0,0 @@ -# TFX ユーザーガイド - -## イントロダクション - -TFX は TensorFlow 上で動作する、Google のプロダクションスケールの機械学習プラットフォームです。 -これは機械学習システムを定義、起動、監視するために必要な設定フレームワークと共有ライブラリを提供し、 -それらを機械学習システムに統合できるようにします。 - -## インストール - -[![Python](https://img.shields.io/pypi/pyversions/tfx.svg?style=plastic)](https://github.com/tensorflow/tfx) -[![PyPI](https://badge.fury.io/py/tfx.svg)](https://badge.fury.io/py/tfx) - -``` -pip install tensorflow -pip install tfx -``` - -Note: [TensorFlow Serving](https://www.tensorflow.org/serving/), -[TensorFlow JS](https://js.tensorflow.org/), -[TensorFlow Lite](https://www.tensorflow.org/lite) オプションのコンポーネントです。これらのインストールについては、それぞれのドキュメントを確認してください。 - -Note: このコマンドは [Apache Beam](beam.md) を Direct runner と同時にインストールします。 -別途、[Flink](https://flink.apache.org/)のようなストリーミングランナーをインストールする必要があります。 - -## コアコンポーネント - -### TFX パイプライン - -TFX パイプラインはいくつかのコンポーネントを用いて、特定の機械学習タスク (たとえば、 -特定のデータを用いた回帰モデルの構築とデプロイ) を最終的に実行するためのデータフローを定義します。 -パイプラインのコンポーネントは TFX のライブラリを用いて構築されます。 -パイプラインの最終結果は、推論を要求する TFX デプロイメントターゲットかサービス、またはその両方です。 - -## TFX パイプライン コンポーネント - -TFX パイプラインは一連のコンポーネントを連結したもので、スケーラブルでハイパフォーマンスな機械学習を実現するように設計されています。 -これにはモデリング、訓練、推論のサービング、そしてオンライン、ネイティブモバイルアプリ、JavaScript へのデプロイの管理が含まれています。 - -TFX パイプラインには典型的には次のコンポーネントが含まれます : - -- [**ExampleGen**](examplegen.md) はパイプラインの先頭に来るコンポーネントで、 - データセットの取り込みと、必要な場合には分割を行います。 - -- [**StatisticsGen**](statsgen.md) はデータセットの統計量を計算します。 - -- [**SchemaGen**](schemagen.md) は統計量を確認し、データのスキーマを生成します。 - -- [**ExampleValidator**](exampleval.md) はデータセットに異常値や欠損値が含まれないかを検査します。 - -- [**Transform**](transform.md) はデータセットに対して特徴量エンジニアリングを行います。 - -- [**Trainer**](trainer.md) はモデルを訓練します。 - -- [**Evaluator**](evaluator.md) は訓練させた結果について深く分析を行います。 - -- [**ModelValidator**](modelval.md) は出力されたモデルのバリデーションを手助けし、 - プロダクション環境に適用するのに「十分良さそう」であることを保証します。 - -- [**Pusher**](pusher.md) はサービスを提供するインフラストラクチャにモデルをデプロイします。 - -次の図はこれらのコンポーネント間でのデータのやり取りをあらわしています。 - -![Component Flow](diag_all.svg) - -### コンポーネントの内部構造 - -TFX のコンポーネントは次の3つの主要な部分から成り立ちます。 - -- Driver -- Executor -- Publisher - -Component Anatomy - -#### Driver と Publisher - -Driver はメタデータストアに対してクエリを発行し、 Executor にメタデータを提供します。 -一方、Publisher は Executor の出力を受け取りメタデータとして保存します。 -典型的には、開発者は Driver や Publisher を直接扱う必要はありませんが、Driver や Publisher が提供するログメッセージはデバッグの役に立つでしょう。 -詳細は [トラブルシューティング](#トラブルシューティング) で改めて取り上げます。 - -#### Executor - -Executor はコンポーネントが処理を行う箇所です。 -開発者はそれぞれのコンポーネントを実装しているクラスの仕様に従ったコードを記述することで、Executor の内部で実行される処理を記述できます。 -たとえば [Transform コンポーネント](transform.md) を利用する場合、 `preprocessing_fn` を実装する必要が生じるでしょう。 - -## TFX ライブラリ - -TFX はライブラリとパイプラインのコンポーネントの両方を含んでいます。次の図は TFX の提供するライブラリとコンポーネントの関係を表しています。 - -![Libraries and Components](libraries_components.svg) - -TFX はパイプラインのコンポーネントを作成するために必要ないくつかのライブラリを Python パッケージとして提供します。 -これらのライブラリは、実装したいパイプラインに特有な側面に集中できるように、パイプラインのコンポーネントを作成するときに利用できます。 - -TFX のライブラリは次のものを含んでいます: - -- [**TensorFlow Data Validation (TFDV)**](tfdv.md) は機械学習で用いるデータの解析や検証のためのライブラリです。 - これは高いスケーラビリティを持ち、 TensorFlow 及び TFX とうまく連携できるように設計されています。 - TFDV は次の内容を含みます: - - * 学習データとテストデータの要約統計量のスケーラブルな算出 - * データの分布や統計量、データセットの組み合わせに対する多面的な比較を行うビューワーとの統合 - * 必須になる値、値域、語彙などのデータに期待できる内容を説明する、データのスキーマを自動的に生成 - * スキーマの検査を補助するためのスキーマビューワー - * 欠損値、値域を超えた値、誤った特徴量の型といった異常値を特定するための異常値検知 - * どの特徴量で異常値が生じたのか確認し、それを修正するのに必要な知見を得るための異常値ビューワー - -- [**TensorFlow Transform (TFT)**](tft.md) は TensorFlow でデータの前処理を行うためのライブラリです。 - TensorFlow Transform はデータセット全体を通じた処理が必要な特徴量の算出に役立ちます。たとえば次のような処理です: - - * 平均と標準偏差を用いた入力値の正規化 - * すべての入力値から語彙を生成し、文字列を整数に変換 - * 観測されたデータの分布をもとにして区間を設定し、実数値 (float) をそれぞれの区間を表す整数値に変換 - -- [**TensorFlow**](train.md) は TFX のモデルを学習させるために利用されます。 - これは学習データとモデルのコードを入力すると、 SaveModel を出力します。 - また、TFT で作成された特徴量エンジニアリングのパイプラインを用いて、入力値の前処理を行うこともできます。 - -- [**TensorFlow Model Analysis (TFMA)**](tfma.md) は TensorFlow のモデルを評価するためのライブラリです。 - これは TensorFlow と同時に利用することで EvalSavedModel を生成することもできます。これは TFMA の分析の基本になります。 - TFMA を用いることで、作成したモデルを大量のデータに対して分散処理を行い、学習時に定義したものとおなじ指標で評価することができます。 - これらの指標をデータの異なるスライスに対して計算し、Jupyter notebook を用いて可視化できます。 - -- [**TensorFlow Metadata (TFMD)**](https://github.com/tensorflow/metadata) は - 機械学習モデルを TensorFlow で学習させるときに役立つメタデータについての標準的な表現形式を提供します。 - メタデータは手動で作成することも、入力データの解析を通じて自動的に生成されることもあるでしょう。 - また、データの検証、探索、変形に使われるかもしれません。 - メタデータのシリアライズ形式は次のものを含みます。 - - * 表形式のデータに対するスキーマの記述 (たとえば tf.Example) - * データセット全体に対する要約統計量の一式 - -- [**ML Metadata (MLMD)**](mlmd.md) は機械学習デベロッパーとデータサイエンティストのワークフローに関係するメタデータを記録・検索するためのライブラリです。 - 大概の場合、メタデータは TFMD の表現を利用します。 - MLMD は [SQL-Lite](https://www.sqlite.org/index.html) や [MySQL](https://www.mysql.com/)、その他の類似した永続的なデータストアの管理を行います。 - -### TFX を支える技術 - -#### 必須なもの - -- [**Apache Beam**](beam.md) はオープンソースで、バッチ処理とストリーミング処理の両方に対し統一的なデータ並列処理パイプラインのモデルとなるものです。 - TFX は Beam をデータ並列なパイプラインを実装するために利用しています。 - パイプラインは Beam がサポートする分散処理基盤をバックエンドに利用して動作します。 - サポートする分散処理基盤には Apache Flink, Google Cloud Dataflow などが含まれます。 - -#### オプション - -Apache Airflow, Kubeflow のようなオーケストレーターは機械学習パイプラインの設定、オペレーション、監視、メンテナンスをより簡易にします。 - -- [**Apache Airflow**](orchestra.md) はワークフローをプログラムで記述し、ワークフローのスケジューリング、監視を行うプラットフォームです。 - TFX は Airflow をワークフローをタスクの有向非巡回グラフ (directed acyclic graphs: DAGs) で書き表すために利用しています。 - Airflow のスケジューラーは指定された依存関係にしたがって、ワーカー上でタスクを実行していきます。 - 豊富なコマンドラインユーティリティは DAG の複雑な "手術" を簡単に実行できるようにします。 - リッチなユーザーインターフェイスはパイプラインのプロダクション環境での実行状況の可視化、進行状況の監視、また問題が発生したときのトラブルシューティングを容易にします。 - ワークフローがコードで定義されている場合、保守やバージョン管理、テスト、コラボレーションがよりよいものになるでしょう。 - -- [**Kubeflow**](https://www.kubeflow.org/) はシンプルでポータブルかつスケーラブルであり、 - Kubernetes 上での機械学習ワークフローの作成に特化しています。 - Kubeflow の目的はほかのサービスを再作成することなく、オープンソースのベスト・オブ・ブリードな - 機械学習システムを多様なインフラストラクチャ上で展開するための、単純な方法を提供することです。 - [Kubeflow Pipelines](https://www.kubeflow.org/docs/pipelines/pipelines-overview/) - は Kubeflow 上で再現性のあるワークフローを実験や Notebook ベースの経験と統合された形で - 構築・実行することを可能にします。 - Kubernetes 上に構築された Kubeflow Pipeline にはホスティングされたメタデータストア、 - コンテナベースのオーケストレーションエンジン、ノートブックサーバー、そしてユーザーが複雑なパイプラインを - 開発、実行、管理するのを助ける UI が含まれています。 - Kubeflow Pipeline SDK はコンポーネントの作成や共有、パイプラインの構築をプログラムで実行できるようにします。 - -### オーケストレーションとポータビリティ - -TFX は Apache Airflow や Kubeflow といった複数の環境やオーケストレーションフレームワークの間で -の移植性が高くなるように設計されています。また、ベアメタル環境や Google Cloud Platform (GCP) -のような異なるコンピューティングプラットフォームの間でも移植可能です。 - -Note: 現在のバージョンのユーザーガイドではベアメタルシステムのうち、 -主に Apache Airflow をオーケストレーションに用いた場合について述べています。 - -### Model vs. SavedModel - -#### Model - -モデル (Model) は学習プロセスの成果物で、学習プロセスを通じて得られた重みがシリアライズされたレコードです。 -重みはその後、新たな入力に対する予測を計算するために利用されます。 -TFX と TensorFlow において、「モデル」はある時点で学習された重みを含むチェックポイントを指します。 - -「モデル」は予測がどのように行われるか書き表した TensorFlow のコンピュテーショングラフの定義 -(別の言い方をすると Python ファイル) を指す場合もあることに注意してください。 -この2つの意味のどちらを指すかは文脈に応じて変わります。 - -#### SavedModel - -- **[SavedModel](https://www.tensorflow.org/api_docs/python/tf/saved_model)とは** - 普遍的で、言語に依存しない、密閉された、元の状態に回復可能な TensorFlow モデルのシリアライゼーションです。 -- **なぜ重要なのか** : SavedModel は高水準なシステムで単一の抽象を利用して、 - TensorFlow のモデルを作成し、変形し、利用することを可能にするためです。 - -SavedModel は TensorFlow のモデルのシリアライゼーション形式です。 -これはモデルをプロダクション環境でサービス提供する、または訓練したモデルをネイティブモバイルや JavaScript で利用する場合に推奨されます。 -たとえば、モデルから推論結果を提供する REST サービスを作成する場合、モデルを SavedModel に -シリアライズし、 TensorFlow Serving でサービス提供できます。詳細は [Serving a TensorFlow -Model](https://www.tensorflow.org/serving/tutorials/Serving_REST_simple) をご確認ください。 - -### スキーマ - -TFX のコンポーネントのうちのいくつかは、_スキーマ_ と呼ばれる入力データについての記述を扱います。 -スキーマは [schema.proto](https://github.com/tensorflow/metadata/tree/master/tensorflow_metadata/proto/v0) のインスタンスです。 -スキーマは [protocol buffer](https://developers.google.com/protocol-buffers/) 、より一般には "protobuf" として知られているものの一種です。 -スキーマは特徴量のデータ型、その特徴量が必ず含まれなければならないかどうか、入力値の許される範囲、などの事柄を指定できます。 -TensorFlow Data Validation (TFDV) を利用する利点の一つは、型やカテゴリ、入力値の範囲を推論し、 -自動的にスキーマを推測してくれることです。 - -schema protobuf からの抜粋を次に示します。: - -```proto -... -feature { - name: "age" - value_count { - min: 1 - max: 1 - } - type: FLOAT - presence { - min_fraction: 1 - min_count: 1 - } -} -feature { - name: "capital-gain" - value_count { - min: 1 - max: 1 - } - type: FLOAT - presence { - min_fraction: 1 - min_count: 1 - } -} -... -``` - -スキーマを利用しているコンポーネントを次に示します : - -- TensorFlow Data Validation -- TensorFlow Transform - -典型的な TFX のパイプラインでは TensorFlow Data Validation がスキーマを生成し、 -ほかのコンポーネントはそれを利用します。 - -Note: 自動生成されたスキーマは「ベストエフォート」のものであり、単にデータの基本的な特徴の推測を試みるものです。 -生成されたスキーマについては、開発者が確認し、必要な場合は修正を加えることが期待されています。 - -## TFX を使った開発 - -TFX は機械学習のプロジェクトやリサーチ、実験、ローカルのマシン上での開発、デプロイを行うまでのすべてのフェーズにおいて、強力なプラットフォームを提供します。 -コードの重複を避け、[学習時とサービス提供時の歪み](#training-serving-skew-detection) の可能性を排除するために、 -学習時と学習済みモデルのデプロイ時の両方で TFX のパイプラインを実装し、[TensorFlow Transform](tft.md) ライブラリを -を活用するために [Transform](transform.md) コンポーネントを学習時と推論時の両方で利用することを強く推奨します。 -このようにすることで、前処理や解析を行う同一のコードを一貫して利用することができ、 -学習に利用するデータとプロダクション環境で学習済みモデルに与えられるデータの間で差異が生じることを避けられます。 -また、コードの記述も一度で済みます。 - -### データ探索、可視化、クリーニング - -![Data Exploration, Visualization, and Cleaning](wrangling.svg) - -TFX パイプラインは典型的には [ExampleGen](examplegen.md) コンポーネントから始まります。 -ExampleGen コンポーネントは入力されたデータを受け付け、 tf.Examples の形式に整形します。 -これはデータセットが学習用と評価用に分割されたあとで実行されることもよくあるため、学習用と評価用に、2つの ExampleGen コンポーネントのコピーが存在する場合もあります。 -また、一般的には次に [StatisticsGen](statsgen.md) コンポーネントと [SchemaGen](schemagen.md) -コンポーネントが続きます。 -これらのコンポーネントはデータを確認し、データのスキーマと統計量を推定します。 -スキーマと統計量は [ExampleValidator](exampleval.md) コンポーネントに入力されます。 -このコンポーネントはデータの中に異常値や欠損値、誤ったデータ型が存在しないか検査します。 -これらのコンポーネントは [TensorFlow Data Validation](tfdv.md) の能力を活用しています。 - -[TensorFlow Data Validation (TFDV)](tfdv.md) はデータセットの探索、可視化、クリーニングを -行う際に役に立つツールです。 -TFDV はデータを確認してデータの型、カテゴリ、値域を推定し、その後、自動的に異常値や欠損値を特定するのを手助けします。 -また、可視化ツールも用意されており、データセットの内容理解に役立つでしょう。 -パイプラインのコンポーネントの処理が完了したあと、 [MLMD](mlmd.md) からメタデータを読み込み、 -データを分析するために TFDV の可視化ツールを Jupyter ノートブック上で利用できます。 - -一度モデルをデプロイしたあとであれば、TFDV をデプロイされたモデルへの推論リクエストに含まれる -新たなデータを監視し、異常値や傾向の変化を検出するために利用できます。 -これは時系列データでトレンドや季節性があり、時間の経過にしたがって変化するものに対してはとくに有効で、 -データの問題やモデルを新しいデータで再学習させる必要があるときに、通知を行う手助けになります。 - -### データの可視化 - -パイプラインのうち、TFDV を利用するコンポーネント (典型的には StatisticsGen, SchemaGen, ExampleValidator) では、データを処理した結果を Jupyter notebook で可視化できます。 -データがモデルやアプリケーションに最適化されるまで何度も実行して結果を比較し、調整を行うことも可能です。 - -これらのコンポーネントの出力結果を [**ML Metadata (MLMD)**](mlmd.md) にクエリを発行して取得したあと notebook で可視化するために、 TFDV の可視化をサポートする API を利用できます。 -これには [tfdv.load_statistics()](`tfdv.load_statistics`) や [tfdv.visualize_statistics()](`tfdv.visualize_statistics`) が含まれます。 -これらの可視化を利用することで、データセットの特徴についてよりよい理解を得ることや、もし必要な場合は要求にそって修正することもできます。 - -### モデルの開発と学習 - -![Feature Engineering](feature_eng.svg) - -典型的な TFX パイプラインは [Transform](transform.md) コンポーネントを含みます。 -これは [TensorFlow Transform (TFT)](tft.md) ライブラリを活用して特徴量エンジニアリングを行います。 -Transform コンポーネントは SchemaGen コンポーネントの作成したスキーマを入力とし、[data transformations](https://www.tensorflow.org/tfx/transform/api_docs/python/tft) を適用してモデルの学習に利用する特徴量の作成、組み合わせ、変換を行います。 -欠損値の除去や型の変換についても、それらが推論時のリクエストにも含まれる可能性のある場合、 Transform コンポーネントで実行すべきです。 -TensorFlow のコードを TFX で学習させるよう設計するときには[いくつか重要な検討事項があります。](train.md) - -![Modeling and Training](train.svg) - -Transform コンポーネントは SavedModel を生成します。 -これは [Trainer](trainer.md) コンポーネントの中で TensorFlow にインポートされ、モデルを記述するコードで利用されます。 -この SavedModel には Transform コンポーネントの作成した、データエンジニアリングで行う変換がすべて含まれています。 -このため、学習時と推論時でまったくおなじコードを用いた同一の変換がなされます。 -Transform コンポーネントで生成された SavedModel を含む、モデルを記述するコードを利用することで、学習用と評価用、両方のデータを利用し、モデルの訓練ができるようになります。 - -モデルを記述するコードの最後のセクションで、モデルを SavedModel と EvalSavedModel の両方の形式で保存すべきです。 EvalSavedModel 形式で保存するためには Trainer コンポーネントで [TensorFlow Model Analysis (TFMA)](tfma.md) ライブラリをインポートして適用することが必要になるでしょう。 - -```python -import tensorflow_model_analysis as tfma -... - -tfma.export.export_eval_savedmodel( - estimator=estimator, - export_dir_base=eval_model_dir, - eval_input_receiver_fn=receiver_fn) -``` - -### モデルの振る舞いの分析と理解 - -![Model Analysis](analysis.svg) - -モデルの開発を始め学習させる場合、モデルの振る舞いについて分析し、真に理解することは極めて重要です。 -典型的な TFX パイプラインは [Evaluator](evaluator.md) コンポーネントを含みます。 -このコンポーネントは開発のこのフェーズにおける強力なツールセットを提供する [TensorFlow Model Analysis (TFMA)](tfma.md) ライブラリの能力を活用します。 -Evaluator コンポーネントは保存した EvalSavedModel を入力として受け付けます。また、モデルの振る舞いを可視化し分析する際に利用する `SliceSpecs` のリストを指定できます。 -それぞれの SliceSpec は学習データ中の、カテゴリカルな特徴量における特定のカテゴリ、数値的な特徴量における特定の値域といった、確認したい学習データの切り口を定義します。 - -たとえば、年間購入金額や地域データ、年齢層、性別といった異なるセグメントの顧客に対するモデルの振る舞いを理解しようと試みることは重要になりえるでしょう。 -データセットがロングテールな分布をしる場合に、これはとくに重要になりえます。少数でも重要なグループに対する容認できない振る舞いが、多数派に対する振る舞いによって覆い隠されてしまう場合があるためです。 -たとえば、モデルが平均的な従業員に対してはうまく機能するものの、企業の幹部に対しては本当にひどい過ちを犯すような場合、それを知っておくことは重要になるかもしれません。 - -### モデルの分析と可視化 - -モデルの学習を終え、学習結果を [Evaluator](evaluator.md) コンポーネント (これは [TFMA](tfma.md) を活用しています) に入力して処理が完了したあと、結果を Jupyter スタイルのノートブックで可視化できます。 -2回目以降であれば、これまでの結果を比較して、結果がモデルとアプリケーションにとって最適化されるまで何度も調整できます。 - -ノートブック上で可視化を行うためには、まず[**ML Metadata (MLMD)**](mlmd.md) にクエリを発行し、コンポーネント群の実行結果を取得します。 -次に、 TFMA の可視化用 API を利用すると結果の可視化ができます。 -可視化用 API には [tfma.load_eval_results()](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/load_eval_results)や [tfma.view.render_slicing_metrics()](`tfma/view/render_slicing_metrics`) が含まれます。 -可視化を行うことでモデルの特徴についてよりよい理解を得ることができ、必要な場合には修正もできるようになるでしょう。 - -## デプロイメントターゲット - -モデルの開発と学習を終えてその結果に満足しているなら、推論リクエストを受け付ける場所である、1つまたは複数のデプロイメントターゲットにモデルをデプロイするタイミングです。 -TFX は3つのクラスのデプロイメントターゲットをサポートしています。 -学習済みモデルを SavedModel としてエクスポートすると、これら3つのうちのどれか、またはすべてのデプロイメントターゲットにモデルをデプロイできます。 - -![Component Flow](diag_all.svg) - -### 推論: TensorFlow Serving - -[TensorFlow Serving (TFS)](serving.md) はプロダクション環境向けに設計された、柔軟でハイパフォーマンスな機械学習モデルのサービングシステムです。 -これは SavedModel を読み込むと、REST や gRPC インターフェースでの推論リクエストを受け付けるようになります。 -また、複数のプロセスを同期・分散処理のためのいくつかの高度なアーキテクチャを用いて、1つないしは複数のネットワークサーバー上で走らせます。 -TFS の開発やデプロイに関する詳細については [TFS のドキュメント](serving.md)を参照してください。 - -典型的なパイプラインでは [Pusher](pusher.md) コンポーネントが Trainer コンポーネントで訓練された SavedModel を読み込み、TFS のインフラストラクチャにデプロイします。 -これは複数のバージョンやモデルのアップデートの処理を含みます。 - -### モバイルネイティブや IoT アプリケーションでの推論: TensorFlow Lite - -[TensorFlow Lite](https://www.tensorflow.org/lite) は学習済みの TensorFlow モデルをネイティブモバイルや IoT アプリケーションで使うためのツール一式をデベロッパーに提供します。 -TensorFlow Serving と同様に、これは SavedModel を読み込み、量子化や枝刈りのような最適化手法を用いて、モバイルや IoT デバイスで動作するようにサイズとパフォーマンスの最適化を試みます。 -TensorFlow Lite の利用についての詳細は TensorFlow Lite のドキュメントを参照してください。 - -### JavaScript での推論: TensorFlow JS - -[TensorFlow JS](https://js.tensorflow.org/) は機械学習モデルの学習とデプロイをブラウザや Node.js 上で行う JavaScript ライブラリです。 -これは TensorFlow Serving や TensorFlow Lite と同様に SavedModel を読み込み、それを TensorFlow.js Web フォーマットに変換します。 -TensorFlow JS の利用についての詳細は TensorFlow JS のドキュメントを参照してください。 - -## Airflow を用いた TFX パイプラインの作成 - -### インストール - -Airflow は PyPI からインストールできます: - -```python -# Airflow -# set this to avoid the GPL version; no functionality difference either way -export SLUGIFY_USES_TEXT_UNIDECODE=yes -pip install apache-airflow -``` - -### DAG の作成 - -Python を用いて TFX パイプラインを作成するためには、`tfx.runtimes.tfx_airflow.PipelineDecorator` でデコレートされた関数を定義して、 -パイプラインで利用するコンポーネントを作成し、パイプラインに合わせてそれらをつなぎ合わせる必要があります。 -その後、ファイルのグローバルコンテキストで `create_pipeline()` を呼び出します。 -たとえば、典型的なパイプラインは次のようになるでしょう : - -```python -@PipelineDecorator( - pipeline_name='tfx_example_solution', - schedule_interval=None, - start_date=datetime.datetime(2018, 1, 1), - enable_cache=True, - additional_pipeline_args={'logger_args': logging_utils.LoggerConfig( - log_root='/var/tmp/tfx/logs', log_level=logging.INFO)}, - metadata_db_root=os.path.join(home_dir, 'data/tfx/metadata'), - pipeline_root=pipeline_root) -def create_pipeline(): - """Implements the example pipeline with TFX.""" - examples = csv_input(os.path.join(base_dir, 'no_split/span_1')) - example_gen = CsvExampleGen(input_data=examples) - statistics_gen = StatisticsGen(input_data=example_gen.outputs.output) - infer_schema = SchemaGen(stats=statistics_gen.outputs.output) - validate_stats = ExampleValidator( # pylint: disable=unused-variable - stats=statistics_gen.outputs.output, - schema=infer_schema.outputs.output) - transform = Transform( - input_data=example_gen.outputs.output, - schema=infer_schema.outputs.output, - module_file=transforms) - trainer = Trainer( - module_file=model, - transformed_examples=transform.outputs.transformed_examples, - schema=infer_schema.outputs.output, - transform_output=transform.outputs.transform_output, - train_steps=10000, - eval_steps=5000, - warm_starting=True) - model_analyzer = Evaluator( - examples=example_gen.outputs.output, - model_exports=trainer.outputs.output) - model_validator = ModelValidator( - examples=example_gen.outputs.output, - model=trainer.outputs.output) - pusher = Pusher( - model_export=trainer.outputs.output, - model_blessing=model_validator.outputs.blessing, - serving_model_dir=serving_model_dir) - - return [ - example_gen, statistics_gen, infer_schema, validate_stats, transform, - trainer, model_analyzer, model_validator, pusher - ] - -pipeline = TfxRunner().run(create_pipeline()) -``` - -### Airflow を用いる場合の TFX パイプラインの初期化 - -[Apache Airflow](orchestra.md) のインストールの際に、パイプラインが作成される `$AIRFLOW_HOME` ディレクトリ (デフォルトでは `~/airflow`) が初期化されます。 -完了後、パイプラインのコードを保持するためのディレクトリを作成する必要があります: - -```bash -mkdir -p ~/airflow/dags # or $AIRFLOW_HOME/dags -mkdir -p ~/airflow/data # or $AIRFLOW_HOME/data -mkdir -p ~/airflow/plugins # or $AIRFLOW_HOME/plugins -``` - -#### パイプラインの設定 - -実際のコードの構造に対する唯一の要件は、`create_pipeline()` 関数 (これは "パイプラインの設定" そのものです) が記された Python ファイルは `dags` フォルダ配下に置かれていなければいけない、というものです。 -Python ファイルの名前は DAG の名前と一致させることを推奨します。DAG が `taxi` という名前のときには、そのファイル名も `taxi` とすべきです。 - -パイプラインの設定ファイル中の `create_pipeline()` 関数は `PipelineDecorator` でデコレートされ、`pipeline_name` などの値が設定されます。 -これらは Airflow の web UI を用いてパイプラインを名前で識別する際や、パイプラインのログファイルを配置する際に重要となります。 - -#### パイプラインのコードのデプロイ - -上記のセクションで記述しているようにパイプラインの設定ファイルを配置し、ファイル名を変更します。 - -`data` フォルダーと `plugins` フォルダー配下に新しくフォルダーを作成しパイプラインのコードを配置することで、パイプラインコードの残りのコードをデプロイしましょう。 -これらの出力先フォルダーの名前をパイプラインで設定した名前 (`PipelineDecorator` で設定したもの) と一致させることはグッドプラクティスです。 -たとえば、パイプラインの名前が `taxi` のとき、出力フォルダーの名前は次のようになります: - -```bash -mkdir -p ~/airflow/data/taxi # or $AIRFLOW_HOME/data/taxi -mkdir -p ~/airflow/plugins/taxi # or $AIRFLOW_HOME/plugins/taxi - -``` - -```python -home_dir = os.path.join(os.environ['HOME'], 'airflow/') -base_dir = os.path.join(home_dir, 'data/taxi/') -output_dir = os.path.join(base_dir, 'pipelines/') -``` - -データセットをファイルとして保存し利用する場合、 `data` フォルダー配下にファイルをコピーすべきです: - -```bash -cp data.csv ~/airflow/data/taxi # or $AIRFLOW_HOME/data/taxi -``` - -### サンプルコード - -慣習的に、パイプラインの設定の記述にはパイプラインの設定に必要な最小限のコードのみを含むべきです。[これはそのようなコードのサンプルです](https://github.com/tensorflow/tfx/blob/master/tfx/examples/chicago_taxi_pipeline/taxi_pipeline_simple.py)。 - -コードで呼び出している、TensorFlow のモデルや Transform の `preprocessing_fn` などの処理の記述はすべて単一のファイルに含めるべきです。 -[これはそのようなコードのサンプルです](https://github.com/tensorflow/tfx/blob/master/tfx/examples/chicago_taxi_pipeline/taxi_utils.py)。 - -## TFX パイプラインのデプロイとオペレーション - -新しいパイプラインの処理を開始させるためには Airflow の web UI でパイプラインを有効にしなければいけません。 -多くの場合、web UI から処理を開始する必要もあるでしょう。 -もし実行を止めたい場合は、パイプラインを無効にすることも web UI から可能です。 -また、パイプラインの現在の状態や、過去の履歴の確認、ログの閲覧も web UI で可能です。 - -### パイプラインの開始と更新 - -もしパイプラインをまだ実行していない場合、 Airflow をコマンドラインから実行する必要があります: - -```bash -airflow webserver -p 8080 -airflow scheduler -``` - -#### コードの更新 - -パイプラインのデプロイを行ったあとでも、パイプラインのコードに変更を加えることができます。 -変更を加えた場合、 Airflow がリフレッシュを行う次のタイミング (デフォルトは1分です) まで待ち、Airflow の web UI ページを更新して変更を確認する必要があります。 -パイプラインの `pipeline_name` を変更した場合には、古い名前が残るかもしれませんが、そのうち "missing" (行方不明) と表示されるようになります。 - -#### ノートブックを用いた可視化 - -パイプラインに含まれる TFX のコンポーネントの入力と出力を調べ、実行結果の比較を行うために、Jupyter スタイルのノートブックはとても有用なツールです。 -加えて、TFDV と TFMA はどちらも強力な可視化のサポートを備えており、開発者がデータセットを探索し、モデルの出力結果を詳細に解析できるようになっています。 - -## トラブルシューティング - -### Log ファイル中のエラーの探し方 - -TFX はログを PipelineDecotator の追加の引数である LoggerConfig で設定された場所に出力します。 -デフォルトは `/var/tmp/tfx/logs/tfx.log` です。 -加えて、オーケストレーター (たとえば Airflow や Kubeflow) もまたログファイルを出力します。 -パイプラインのエラーの分析を試みるときには、これらのログファイルがとても有効です。 -`taxi` という名前のパイプラインで特に LoggerConfig が設定されていない場合、TFX のログは `/var/tmp/tfx/logs/tfx.log` に出力されます。 -これは [logging_utils.LoggerConfig](https://github.com/tensorflow/tfx/blob/master/tfx/utils/logging_utils.py) オブジェクトを生成し、パイプラインの設定の `logger_args` と呼ばれるパラメータに与えることで、設定の変更が可能です。 - -```python -@PipelineDecorator( - pipeline_name='tfx_example_solution', - schedule_interval=None, - start_date=datetime.datetime(2018, 1, 1), - enable_cache=True, - additional_pipeline_args={'logger_args': logging_utils.LoggerConfig( - log_root='/var/tmp/tfx/logs', log_level=logging.INFO)}, - metadata_db_root=os.path.join(home_dir, 'data/tfx/metadata'), - pipeline_root=pipeline_root) -``` - -Note: Docker や Kubeflow のリモート環境上で Executor を動かしている場合、Executor のログはリモートワーカー上に出力されます。 - -Airflow を利用している場合、ログは Airflow が出力するログにも記録されます。 -デフォルトの Airflow のログの出力先は `$AIRFLOW_HOME/logs` で、次のファイルが含まれます: - -``` -$AIRFLOW_HOME/logs/scheduler/{DATE}/taxi.py.log -$AIRFLOW_HOME/logs/scheduler/latest/taxi.py.log -$AIRFLOW_HOME/logs/taxi -$AIRFLOW_HOME/logs/taxi.COMPONENT_NAME -``` - -### パイプラインは表示されているが、Airflow で実行しようとするとみつからない - -ウェブサーバーとスケジューラーを再起動してみてください。 - -## Kubeflow を用いた TFX パイプラインの作成 - -### セットアップ - -Kubeflow はパイプラインを大規模に実行するために Kubernetes クラスタを必要とします。 -Kubeflow のデプロイを行うための一連の手引については次のデプロイメントガイドラインを参照してください ([deplopying the Kubeflow cluster.](https://www.kubeflow.org/docs/started/getting-started-gke/)) - -### TFX パイプラインの設定と実行 - -Kubeflow 上で TFX のサンプルパイプラインを実行するためには Kubeflow パイプラインの[手順書](https://github.com/kubeflow/pipelines/tree/master/samples/tfx-oss)にしたがってください。 -TFX コンポーネントは Kubeflow pipeline を構築するためにコンテナ化されています。 -また、サンプルは大規模なパブリックのデータセットを読み込み、学習を行って、クラウド上で大規模にデータを処理するステップを解説しています。 diff --git a/site/ja/tutorials/customization/autodiff.ipynb b/site/ja/tutorials/customization/autodiff.ipynb deleted file mode 100644 index 541108af8f3..00000000000 --- a/site/ja/tutorials/customization/autodiff.ipynb +++ /dev/null @@ -1,382 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "autodiff.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "GCCk8_dHpuNf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xh8WkEwWpnm7" - }, - "source": [ - "# 自動微分と勾配テープ" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idv0bPeCp325" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Qk0cqFB6lPZ9", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJ4XzMqodTy" - }, - "source": [ - "これまでのチュートリアルでは、テンソルとその演算について紹介しました。このチュートリアルでは、機械学習モデルの最適化の鍵となる手法である[自動微分](https://ja.wikipedia.org/wiki/自動微分)を扱います。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GQJysDM__Qb0" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "O8u4t_ptlu_u", - "colab_type": "code", - "colab": {} - }, - "source": [ - "try:\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cxzaxo6ff2y3", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1CLWJl0QliB0" - }, - "source": [ - "## 勾配テープ\n", - "\n", - "TensorFlow には、自動微分、すなわち、入力変数に対する計算結果の勾配を計算するための[tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) API があります。TensorFlow は、`tf.GradientTape` のコンテキスト内で行われる演算すべてを「テープ」に「記録」します。その後 TensorFlow は、そのテープと、そこに記録された演算ひとつひとつに関連する勾配を使い、[トップダウン型自動微分（リバースモード）](https://ja.wikipedia.org/wiki/自動微分)を使用して、「記録」された計算の勾配を計算します。\n", - "\n", - "例を示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bAFeIE8EuVIq", - "colab": {} - }, - "source": [ - "x = tf.ones((2, 2))\n", - " \n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# 元の入力テンソル x に対する z の微分\n", - "dz_dx = t.gradient(z, x)\n", - "for i in [0, 1]:\n", - " for j in [0, 1]:\n", - " assert dz_dx[i][j].numpy() == 8.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N4VlqKFzzGaC" - }, - "source": [ - "「記録された」`tf.GradientTape` のコンテキスト中の、中間の値に対する出力の勾配を求めることもできます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7XaPRAwUyYms", - "colab": {} - }, - "source": [ - "x = tf.ones((2, 2))\n", - " \n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# テープを使って中間値 y に対する z の微分を計算\n", - "dz_dy = t.gradient(z, y)\n", - "assert dz_dy.numpy() == 8.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISkXuY7YzIcS" - }, - "source": [ - "デフォルトでは、ある GrandientTape に保持されたリソースは、GradientTape.gradient() メソッドが呼び出されると解放されます。一つの計算で複数の勾配を計算したい場合には、永続的(`persistent`)な勾配テープを作成します。こうすると、リソースはテープオブジェクトがガベージコレクションによって廃棄されるときに解放されるため、`gradient()` メソッドを複数回呼び出すことができます。 \n", - "\n", - "例を示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zZaCm3-9zVCi", - "colab": {} - }, - "source": [ - "x = tf.constant(3.0)\n", - "with tf.GradientTape(persistent=True) as t:\n", - " t.watch(x)\n", - " y = x * x\n", - " z = y * y\n", - "dz_dx = t.gradient(z, x) # 108.0 (4*x^3 at x = 3)\n", - "dy_dx = t.gradient(y, x) # 6.0\n", - "del t # テープへの参照を削除" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6kADybtQzYj4" - }, - "source": [ - "### 制御フローの記録\n", - "\n", - "勾配テープは演算を実行の都度記録するため、（たとえば `if` や `while` を使った）Python の制御フローも自然に扱われます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9FViq92UX7P8", - "colab": {} - }, - "source": [ - "def f(x, y):\n", - " output = 1.0\n", - " for i in range(y):\n", - " if i > 1 and i < 5:\n", - " output = tf.multiply(output, x)\n", - " return output\n", - "\n", - "def grad(x, y):\n", - " with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " out = f(x, y)\n", - " return t.gradient(out, x) \n", - "\n", - "x = tf.convert_to_tensor(2.0)\n", - "\n", - "assert grad(x, 6).numpy() == 12.0\n", - "assert grad(x, 5).numpy() == 12.0\n", - "assert grad(x, 4).numpy() == 4.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DK05KXrAAld3" - }, - "source": [ - "### 高次勾配\n", - "\n", - "`GradientTape` コンテキストマネージャの内側で行われた演算は、自動微分のために記録されます。このコンテキスト内で勾配が計算された場合、勾配計算も同様に記録されます。結果として、まったくおなじAPIが高次勾配にも使えることになります。例を示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cPQgthZ7ugRJ", - "colab": {} - }, - "source": [ - "x = tf.Variable(1.0) # 1.0 で初期化された TensorFlow 変数を作成\n", - "\n", - "with tf.GradientTape() as t:\n", - " with tf.GradientTape() as t2:\n", - " y = x * x * x\n", - " # ’t’ コンテキストマネジャー内で勾配を計算\n", - " # これは勾配計算も同様に微分可能であるということ\n", - " dy_dx = t2.gradient(y, x)\n", - "d2y_dx2 = t.gradient(dy_dx, x)\n", - "\n", - "assert dy_dx.numpy() == 3.0\n", - "assert d2y_dx2.numpy() == 6.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4U1KKzUpNl58" - }, - "source": [ - "## 次のステップ\n", - "\n", - "このチュートリアルでは、TensorFlow の勾配計算を扱いました。これで、ニューラルネットワークの構築と訓練のために必要な構成要素が揃いました。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/customization/basics.ipynb b/site/ja/tutorials/customization/basics.ipynb deleted file mode 100644 index de2877e8668..00000000000 --- a/site/ja/tutorials/customization/basics.ipynb +++ /dev/null @@ -1,479 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basics.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iPpI7RaYoZuE" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "hro2InpHobKk", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# テンソルと演算" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Hndw-YcxoOJK" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RSywPQ2n736s" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6sILUVbHoSgH" - }, - "source": [ - "これは、下記の手法を示す TensorFlow の入門チュートリアルです。\n", - "\n", - "* 必要なパッケージのインポート\n", - "* テンソルの作成と使用\n", - "* GPUによる高速化の使用\n", - "* `tf.data.Dataset`のデモ" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "miTaGiqV9RjO", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1JcS5iBXMRO" - }, - "source": [ - "## TensorFlowのインポート\n", - "\n", - "はじめに、`tensorflow` モジュールをインポートします。TensorFlow 2.0 では、eager execution が既定でオンとなっています。\n", - "これにより、TensorFlow のフロントエンドがよりインタラクティブになります。詳細は後述します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vjBPmYjLdFmk", - "colab": {} - }, - "source": [ - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H9UySOPLXdaw" - }, - "source": [ - "## テンソル\n", - "\n", - "テンソルは多次元配列です。NumPy の `ndarray` オブジェクトと同様に、`tf.Tensor` にはデータ型と形状があります。これに加えて、`tf.Tensor` は（ GPU のような）アクセラレータのメモリに置くことができます。TensorFlow には、`tf.Tensor` を使用し生成するたくさんの演算([tf.add](https://www.tensorflow.org/api_docs/python/tf/add), [tf.matmul](https://www.tensorflow.org/api_docs/python/tf/matmul), [tf.linalg.inv](https://www.tensorflow.org/api_docs/python/tf/linalg/inv) など)のライブラリが存在します。これらの演算では、ネイティブな Python データ型が自動変換されます。例を示します。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "code", - "colab_type": "code", - "id": "ngUe237Wt48W", - "colab": {} - }, - "source": [ - "print(tf.add(1, 2))\n", - "print(tf.add([1, 2], [3, 4]))\n", - "print(tf.square(5))\n", - "print(tf.reduce_sum([1, 2, 3]))\n", - "\n", - "# Operator overloading is also supported\n", - "print(tf.square(2) + tf.square(3))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDY4WsYRhP81" - }, - "source": [ - "それぞれの`tf.Tensor`には、形状とデータ型があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "srYWH1MdJNG7", - "colab": {} - }, - "source": [ - "x = tf.matmul([[1]], [[2, 3]])\n", - "print(x)\n", - "print(x.shape)\n", - "print(x.dtype)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eBPw8e8vrsom" - }, - "source": [ - "NumPy 配列と `tf.Tensor` の間のもっとも明確な違いは\n", - "\n", - "1. テンソルは（ GPU や TPU などの）アクセラレータメモリを使用できる\n", - "2. テンソルは変更不可" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dwi1tdW3JBw6" - }, - "source": [ - "### NumPy互換性\n", - "\n", - "TensorFlow の`tf.Tensor`と NumPy の `ndarray` 間の変換は簡単です。\n", - "\n", - "* TensorFlow の演算により NumPy の ndarray は自動的にテンソルに変換される\n", - "* NumPy の演算によりテンソルは自動的に NuｍPy の ndarray に変換される\n", - "\n", - "テンソルは `.numpy()` メソッドを使って明示的に NumPy の ndarray に変換されます。NumPy のndarray と `tf.Tensor` はその下敷きとなるメモリ上の表現が、できるかぎり共通化されているので、通常この変換のコストは小さいです。しかし、NumPy 配列はホスト側のメモリに置かれる一方、`tf.Tensor` はGPU のメモリに置かれる可能性もあるため、下層の表現をいつも共通化できるとは限りません。また、変換にはGPU からホスト側メモリへのコピーも関わってきます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lCUWzso6mbqR", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "ndarray = np.ones([3, 3])\n", - "\n", - "print(\"TensorFlow演算によりnumpy配列は自動的にテンソルに変換される\")\n", - "tensor = tf.multiply(ndarray, 42)\n", - "print(tensor)\n", - "\n", - "\n", - "print(\"またNumPy演算によりテンソルは自動的にnumpy配列に変換される\")\n", - "print(np.add(tensor, 1))\n", - "\n", - "print(\".numpy()メソッドによりテンソルは明示的にnumpy配列に変換される\")\n", - "print(tensor.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PBNP8yTRfu_X" - }, - "source": [ - "## GPU による高速化\n", - "\n", - "TensorFlow の演算の多くは、GPU を計算に使用することで高速化されます。TensorFlow は演算に注釈をつけなくとも、自動的に GPU と CPU のどちらかを選択し、必要であればテンソルを GPU メモリと CPU メモリの間でコピーして実行します。演算で生成されたテンソルは通常演算を実行したデバイスのメモリに置かれます。例を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "code", - "colab_type": "code", - "id": "3Twf_Rw-gQFM", - "colab": {} - }, - "source": [ - "x = tf.random.uniform([3, 3])\n", - "\n", - "print(\"利用できるGPUはあるか: \"),\n", - "print(tf.config.experimental.list_physical_devices(\"GPU\"))\n", - "\n", - "print(\"テンソルはGPU #0にあるか: \"),\n", - "print(x.device.endswith('GPU:0'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vpgYzgVXW2Ud" - }, - "source": [ - "### デバイス名\n", - "\n", - "`Tensor.device` プロパティにより、そのテンソルの内容を保持しているデバイスの完全な名前文字列を得ることができます。この名前には、プログラムを実行中のホストのネットワークアドレスや、ホスト上のデバイスについての詳細がエンコードされています。この情報は、TensorFlow プログラムの分散実行に必要なものです。テンソルがホスト上の `N` 番目のGPUにある場合、文字列の最後は `GPU:` となります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZWZQCimzuqyP" - }, - "source": [ - "### 明示的デバイス配置\n", - "\n", - "TensorFlowでいう**配置**は、個々の演算を実行するためにどのようにデバイスにアサイン（配置）されるかを指します。前述のとおり、明示的な示唆がなければ、TensorFlow は演算を実行するデバイスを自動的に決め、必要であればテンソルをそのデバイスにコピーします。しかし、`tf.device` コンテキストマネジャーを使うことで、TensorFlow の演算を特定のデバイスに配置することができます。例を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RjkNZTuauy-Q", - "colab": {} - }, - "source": [ - "import time\n", - "\n", - "def time_matmul(x):\n", - " start = time.time()\n", - " for loop in range(10):\n", - " tf.matmul(x, x)\n", - "\n", - " result = time.time()-start\n", - " \n", - " print(\"10 loops: {:0.2f}ms\".format(1000*result))\n", - "\n", - "# CPUでの実行を強制\n", - "print(\"On CPU:\")\n", - "with tf.device(\"CPU:0\"):\n", - " x = tf.random.uniform([1000, 1000])\n", - " assert x.device.endswith(\"CPU:0\")\n", - " time_matmul(x)\n", - "\n", - "# GPU #0があればその上での実行を強制\n", - "if tf.config.experimental.list_physical_devices(\"GPU\"):\n", - " print(\"On GPU:\")\n", - " with tf.device(\"GPU:0\"): # 2番めのGPUなら GPU:1, 3番目なら GPU:2 など\n", - " x = tf.random.uniform([1000, 1000])\n", - " assert x.device.endswith(\"GPU:0\")\n", - " time_matmul(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o1K4dlhhHtQj" - }, - "source": [ - "## データセット\n", - "\n", - "このセクションでは [`tf.data.Dataset` API](https://www.tensorflow.org/guide/datasets) を使って、モデルにデータを供給するためのパイプラインを構築します。`tf.data.Dataset` APIは、単純で再利用可能な部品をもとに、モデルの訓練あるいは評価ループにデータを供給する高性能で複雑な入力パイプラインを構築するために使われます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zI0fmOynH-Ne" - }, - "source": [ - "### ソース`Dataset`の作成\n", - "\n", - "\n", - "[Dataset.from_tensors](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors) や[Dataset.from_tensor_slices](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices) といったファクトリー関数または [TextLineDataset](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) あるいは[TFRecordDataset](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset) のようなファイルを読み込むオブジェクトを使って、 **元となる**データセットを作成しましょう。詳しくは、[TensorFlow Dataset guide](https://www.tensorflow.org/guide/datasets#reading_input_data) を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F04fVOHQIBiG", - "colab": {} - }, - "source": [ - "ds_tensors = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", - "\n", - "# CSVファイルを作成\n", - "import tempfile\n", - "_, filename = tempfile.mkstemp()\n", - "\n", - "with open(filename, 'w') as f:\n", - " f.write(\"\"\"Line 1\n", - "Line 2\n", - "Line 3\n", - " \"\"\")\n", - "\n", - "ds_file = tf.data.TextLineDataset(filename)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbxIhC-5IPdf" - }, - "source": [ - "### 変換の適用\n", - "\n", - "[map](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map), [batch](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch), [shuffle](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle) などの変換関数を使って、データセットレコードに変換を適用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uXSDZWE-ISsd", - "colab": {} - }, - "source": [ - "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", - "\n", - "ds_file = ds_file.batch(2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "A8X1GNfoIZKJ" - }, - "source": [ - "### イテレート\n", - "\n", - "`tf.data.Dataset` オブジェクトは、中のレコードを繰り返し利用するためのイテレーションをサポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ws-WKRk5Ic6-", - "colab": {} - }, - "source": [ - "print('ds_tensors の要素:')\n", - "for x in ds_tensors:\n", - " print(x)\n", - "\n", - "print('\\nds_file の要素:')\n", - "for x in ds_file:\n", - " print(x)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ja/tutorials/customization/custom_layers.ipynb b/site/ja/tutorials/customization/custom_layers.ipynb deleted file mode 100644 index 1fafe9a55c8..00000000000 --- a/site/ja/tutorials/customization/custom_layers.ipynb +++ /dev/null @@ -1,399 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_layers.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tDnwEv8FtJm7" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "JlknJBWQtKkI", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "60RdWsg1tETW" - }, - "source": [ - "# カスタムレイヤー" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BcJg7Enms86w" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YRXLphinx2fF" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UEu3q4jmpKVT" - }, - "source": [ - "ニューラルネットワークの構築には、ハイレベルの API である `tf.keras` を使うことを推奨します。しかしながら、TensorFlow API のほとんどは、eager execution でも使用可能です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-sXDg19Q691F", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Py0m-N6VgQFJ", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zSFfVVjkrrsI" - }, - "source": [ - "## レイヤー：有用な演算の共通セット\n", - "\n", - "機械学習モデルのコーディングでは、個々の演算やひとつひとつの変数のオペレーションよりは、より高度に抽象化されたオペレーションを行いたいのがほとんどだと思います。\n", - "\n", - "多くの機械学習モデルは、比較的単純なレイヤーの組み合わせや積み重ねによって表現可能です。TensorFlow では、多くの一般的なレイヤーのセットに加えて、アプリケーションに特有なレイヤーを最初から記述したり、既存のレイヤーの組み合わせによって作るための、簡単な方法が提供されています。\n", - "\n", - "TensorFlow には、tf.keras パッケージに[Keras](https://keras.io) APIのすべてが含まれています。Keras のレイヤーは、独自のモデルを構築する際に大変便利です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8PyXlPl-4TzQ", - "colab": {} - }, - "source": [ - "# tf.keras.layers パッケージの中では、レイヤーはオブジェクトです。\n", - "# レイヤーを構築するためにすることは、単にオブジェクトを作成するだけです。\n", - "# ほとんどのレイヤーでは、最初の引数が出力の次元あるいはチャネル数を表します。\n", - "layer = tf.keras.layers.Dense(100)\n", - "# 入力の次元数は多くの場合不要となっています。それは、レイヤーが最初に使われる際に\n", - "# 推定可能だからです。ただし、引数として渡すことで手動で指定することも可能です。\n", - "# これは複雑なモデルを構築する場合に役に立つでしょう。\n", - "layer = tf.keras.layers.Dense(10, input_shape=(None, 5))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fn69xxPO5Psr" - }, - "source": [ - "既存のレイヤーのすべての一覧は、[ドキュメント](https://www.tensorflow.org/api_docs/python/tf/keras/layers)を参照してください。Dense（全結合レイヤー）、Conv2D、LSTM、BatchNormalization、Dropoutなどのたくさんのレイヤーが含まれています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "E3XKNknP5Mhb", - "colab": {} - }, - "source": [ - "# レイヤーを使うには、単純にcallします。\n", - "layer(tf.zeros([10, 5]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Wt_Nsv-L5t2s", - "colab": {} - }, - "source": [ - "# レイヤーにはたくさんの便利なメソッドがあります。例えば、`layer.variables`を使って\n", - "# レイヤーのすべての変数を調べることができます。訓練可能な変数は、 `layer.trainable_variables`\n", - "# でわかります。この例では、全結合レイヤーには重みとバイアスの変数があります。\n", - "layer.variables" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6ilvKjz8_4MQ", - "colab": {} - }, - "source": [ - "# これらの変数には便利なアクセサを使ってアクセス可能です。\n", - "layer.kernel, layer.bias" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O0kDbE54-5VS" - }, - "source": [ - "## カスタムレイヤーの実装\n", - "\n", - "独自のレイヤーを実装する最良の方法は、tf.keras.Layer クラスを拡張し、下記のメソッドを実装することです。\n", - " * `__init__` , 入力に依存しないすべての初期化を行う\n", - " * `build`, 入力の `shape` を知った上で、残りの初期化を行う\n", - " * `call`, フォワード計算を行う\n", - "\n", - "`build` が呼ばれるまで変数の生成を待つ必要はなく、`__init__` で作成できることに注意してください。しかしながら、`build` で変数を生成することの優位な点は、レイヤーがオペレーションをしようとする入力の `shape` に基づいて、後から定義できる点です。これに対して、`__init__` で変数を生成するということは、そのために必要な `shape` を明示的に指定する必要があるということです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5Byl3n1k5kIy", - "colab": {} - }, - "source": [ - "class MyDenseLayer(tf.keras.layers.Layer):\n", - " def __init__(self, num_outputs):\n", - " super(MyDenseLayer, self).__init__()\n", - " self.num_outputs = num_outputs\n", - " \n", - " def build(self, input_shape):\n", - " self.kernel = self.add_variable(\"kernel\", \n", - " shape=[int(input_shape[-1]), \n", - " self.num_outputs])\n", - " \n", - " def call(self, input):\n", - " return tf.matmul(input, self.kernel)\n", - " \n", - "layer = MyDenseLayer(10)\n", - "print(layer(tf.zeros([10, 5])))\n", - "print(layer.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tk8E2vY0-z4Z" - }, - "source": [ - "できるだけ標準のレイヤーを使ったほうが、概してコードは読みやすく保守しやすくなります。コードを読む人は標準的なレイヤーの振る舞いに慣れているからです。`tf.keras.layers` にはないレイヤーを使いたい場合には、[githubのイシュー](http://github.com/tensorflow/tensorflow/issues/new)を登録するか、もっとよいのはプルリクエストを送ることです。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Qhg4KlbKrs3G" - }, - "source": [ - "## モデル：レイヤーの組み合わせ\n", - "\n", - "機械学習では、多くのレイヤーに類するものが、既存のレイヤーを組み合わせることで実装されています。例えば、ResNetの残差ブロックは、畳込み、バッチ正規化とショートカットの組み合わせです。\n", - "\n", - "他のレイヤーからなるレイヤーに類するものを定義する際の主役は、tf.keras.Model クラスです。このクラスを継承することで実装できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N30DTXiRASlb", - "colab": {} - }, - "source": [ - "class ResnetIdentityBlock(tf.keras.Model):\n", - " def __init__(self, kernel_size, filters):\n", - " super(ResnetIdentityBlock, self).__init__(name='')\n", - " filters1, filters2, filters3 = filters\n", - "\n", - " self.conv2a = tf.keras.layers.Conv2D(filters1, (1, 1))\n", - " self.bn2a = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2b = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same')\n", - " self.bn2b = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2c = tf.keras.layers.Conv2D(filters3, (1, 1))\n", - " self.bn2c = tf.keras.layers.BatchNormalization()\n", - "\n", - " def call(self, input_tensor, training=False):\n", - " x = self.conv2a(input_tensor)\n", - " x = self.bn2a(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2b(x)\n", - " x = self.bn2b(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2c(x)\n", - " x = self.bn2c(x, training=training)\n", - "\n", - " x += input_tensor\n", - " return tf.nn.relu(x)\n", - "\n", - " \n", - "block = ResnetIdentityBlock(1, [1, 2, 3])\n", - "print(block(tf.zeros([1, 2, 3, 3])))\n", - "print([x.name for x in block.trainable_variables])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wYfucVw65PMj" - }, - "source": [ - "しかし、ほとんどの場合には、モデルはレイヤーを次々に呼び出すことで構成されます。tf.keras.Sequential クラスを使うことで、これをかなり短いコードで実装できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "L9frk7Ur4uvJ", - "colab": {} - }, - "source": [ - "my_seq = tf.keras.Sequential([tf.keras.layers.Conv2D(1, (1, 1), \n", - " input_shape=(\n", - " None, None, 3)),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(2, 1,\n", - " padding='same'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(3, (1, 1)),\n", - " tf.keras.layers.BatchNormalization()])\n", - "my_seq(tf.zeros([1, 2, 3, 3]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c5YwYcnuK-wc" - }, - "source": [ - "# 次のステップ\n", - "\n", - "それでは、前出のノートブックに戻り、線形回帰の例を、レイヤーとモデルを使って、より構造化された形で実装してみてください。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/customization/custom_training.ipynb b/site/ja/tutorials/customization/custom_training.ipynb deleted file mode 100644 index fd5f7446ae4..00000000000 --- a/site/ja/tutorials/customization/custom_training.ipynb +++ /dev/null @@ -1,448 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rmpybwysXGV" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "m8y3rGtQsYP2" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hrXv0rU9sIma" - }, - "source": [ - "# カスタム訓練：基本" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7S0BwJ_8sLu7" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k2o3TTG4TFpt" - }, - "source": [ - "前のチュートリアルでは、機械学習の基本構成ブロックの1つである自動微分について TensorFlow の API を学習しました。\n", - "このチュートリアルでは、これまでのチュートリアルに出てきた TensorFlow の基本要素を使って、単純な機械学習を実行します。\n", - "\n", - "TensorFlow には `tf.keras` が含まれています。`tf.keras`は、抽象化により決まり切った記述を削減し、柔軟さと性能を犠牲にすることなく TensorFlow をやさしく使えるようにする、高度なニューラルネットワーク API です。開発には [tf.Keras API](../../guide/keras/overview.ipynb) を使うことを強くおすすめします。しかしながら、この短いチュートリアルでは、しっかりした基礎を身につけていただくために、ニューラルネットワークの訓練についていちから学ぶことにします。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3LXMVuV0VhDr" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NiolgWMPgpwI" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eMAWbDJFVmMk" - }, - "source": [ - "## 変数\n", - "\n", - "TensorFlow のテンソルはイミュータブルでステートレスなオブジェクトです。しかしながら、機械学習モデルには変化する状態が必要です。モデルの訓練が進むにつれて、推論を行うおなじコードが異なる振る舞いをする必要があります（望むべくはより損失の少なくなるように）。この計算が進むにつれて変化する必要がある状態を表現するために、Python が状態を保つプログラミング言語であることを利用することができます。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VkJwtLS_Jbn8" - }, - "outputs": [], - "source": [ - "# Python の状態を使う\n", - "x = tf.zeros([10, 10])\n", - "x += 2 # これは x = x + 2 と等価で, x の元の値を変えているわけではない\n", - "\n", - "print(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfneTXy7JcUz" - }, - "source": [ - "TensorFlow にはステートフルな演算が組み込まれているので、状態を表現するのに低レベルの Python による表現を使うよりは簡単なことがしばしばあります。\n", - "\n", - "`tf.Variable`オブジェクトは値を保持し、何も指示しなくともこの保存された値を読み出します。TensorFlow の変数に保持された値を操作する演算（`tf.assign_sub`, `tf.scatter_update`, など）が用意されています。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "itxmrMil6DQi" - }, - "outputs": [], - "source": [ - "v = tf.Variable(1.0)\n", - "# Python の `assert` を条件をテストするデバッグ文として使用\n", - "assert v.numpy() == 1.0\n", - "\n", - "# `v` に値を再代入\n", - "v.assign(3.0)\n", - "assert v.numpy() == 3.0\n", - "\n", - "# `v` に TensorFlow の `tf.square()` 演算を適用し再代入\n", - "v.assign(tf.square(v))\n", - "assert v.numpy() == 9.0\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-paSaeq1JzwC" - }, - "source": [ - "`tf.Variable`を使った計算は、勾配計算の際に自動的にトレースされます。埋め込みを表す変数では、TensorFlow は既定でスパースな更新を行います。これは計算量やメモリ使用量においてより効率的です。\n", - "\n", - "`tf.Variable`はあなたのコードを読む人にその状態の一部がミュータブルであることを示す方法でもあります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BMiFcDzE7Qu3" - }, - "source": [ - "## 線形モデルの適合\n", - "\n", - "これまでに学んだ `Tensor`、 `Variable`、 そして `GradientTape`という概念を使って、簡単なモデルの構築と訓練を行ってみましょう。通常、これには次のようないくつかの手順が含まれます。\n", - "\n", - "1. モデルの定義\n", - "2. 損失関数の定義\n", - "3. 訓練データの取得\n", - "4. 訓練データを使って実行し、\"optimizer\" を使って変数をデータに適合\n", - "\n", - "ここでは、`f(x) = x * W + b`という簡単な線形モデルを作ります。このモデルには `W` (重み) と `b` (バイアス) の2つの変数があります。十分訓練されたモデルが `W = 3.0` と `b = 2.0` になるようなデータを人工的に作ります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFzH64Jn9PIm" - }, - "source": [ - "### モデルの定義\n", - "\n", - "変数と計算をカプセル化する単純なクラスを定義してみましょう。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_WRu7Pze7wk8" - }, - "outputs": [], - "source": [ - "class Model(object):\n", - " def __init__(self):\n", - " # 重みを `5.0` に、バイアスを `0.0` に初期化\n", - " # 実際には、これらの値は乱数で初期化するべき（例えば `tf.random.normal` を使って）\n", - " self.W = tf.Variable(5.0)\n", - " self.b = tf.Variable(0.0)\n", - "\n", - " def __call__(self, x):\n", - " return self.W * x + self.b\n", - "\n", - "model = Model()\n", - "\n", - "assert model(3.0).numpy() == 15.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xa6j_yXa-j79" - }, - "source": [ - "### 損失関数の定義\n", - "\n", - "損失関数は、ある入力値に対するモデルの出力がどれだけ出力の目的値に近いかを測るものです。訓練を通じて、この差異を最小化するのがゴールとなります。最小二乗誤差とも呼ばれる L2 損失を使ってみましょう。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Y0ysUFGY924U" - }, - "outputs": [], - "source": [ - "def loss(predicted_y, target_y):\n", - " return tf.reduce_mean(tf.square(predicted_y - target_y))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qutT_fkl_CBc" - }, - "source": [ - "### 訓練データの取得\n", - "\n", - "最初に、入力にランダムなガウス（正規）分布のノイズを加えることで、訓練用データを生成します。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gxPTb-kt_N5m" - }, - "outputs": [], - "source": [ - "TRUE_W = 3.0\n", - "TRUE_b = 2.0\n", - "NUM_EXAMPLES = 1000\n", - "\n", - "inputs = tf.random.normal(shape=[NUM_EXAMPLES])\n", - "noise = tf.random.normal(shape=[NUM_EXAMPLES])\n", - "outputs = inputs * TRUE_W + TRUE_b + noise" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-50nq-wPBsAW" - }, - "source": [ - "モデルを訓練する前に、モデルの予測値を赤で、訓練データを青でプロットすることで、損失を可視化します。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_eb83LtrB4nt" - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(inputs, outputs, c='b')\n", - "plt.scatter(inputs, model(inputs), c='r')\n", - "plt.show()\n", - "\n", - "print('Current loss: %1.6f' % loss(model(inputs), outputs).numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sSDP-yeq_4jE" - }, - "source": [ - "### 訓練ループの定義\n", - "\n", - "ネットワークと訓練データが準備できたところで、損失が少なくなるように、重み変数 (`W`) とバイアス変数 (`b`) を更新するために、[gradient descent （勾配降下法）](https://en.wikipedia.org/wiki/Gradient_descent) を使ってモデルを訓練します。勾配降下法にはさまざまな変種があり、我々の推奨する実装である `tf.train.Optimizer` にも含まれています。しかし、ここでは基本原理から構築するという精神で、自動微分を行う `tf.GradientTape` と、値を減少させる `tf.assign_sub` （これは、`tf.assign` と `tf.sub` の組み合わせですが）の力を借りて、この基本計算を実装してみましょう。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MBIACgdnA55X" - }, - "outputs": [], - "source": [ - "def train(model, inputs, outputs, learning_rate):\n", - " with tf.GradientTape() as t:\n", - " current_loss = loss(model(inputs), outputs)\n", - " dW, db = t.gradient(current_loss, [model.W, model.b])\n", - " model.W.assign_sub(learning_rate * dW)\n", - " model.b.assign_sub(learning_rate * db)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RwWPaJryD2aN" - }, - "source": [ - "最後に、訓練データ全体に対して繰り返し実行し、`W` と `b` がどのように変化するかを見てみましょう。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "XdfkR223D9dW" - }, - "outputs": [], - "source": [ - "model = Model()\n", - "\n", - "# 後ほどプロットするために、W 値と b 値の履歴を集める\n", - "Ws, bs = [], []\n", - "epochs = range(10)\n", - "for epoch in epochs:\n", - " Ws.append(model.W.numpy())\n", - " bs.append(model.b.numpy())\n", - " current_loss = loss(model(inputs), outputs)\n", - "\n", - " train(model, inputs, outputs, learning_rate=0.1)\n", - " print('Epoch %2d: W=%1.2f b=%1.2f, loss=%2.5f' %\n", - " (epoch, Ws[-1], bs[-1], current_loss))\n", - "\n", - "# すべてをプロット\n", - "plt.plot(epochs, Ws, 'r',\n", - " epochs, bs, 'b')\n", - "plt.plot([TRUE_W] * len(epochs), 'r--',\n", - " [TRUE_b] * len(epochs), 'b--')\n", - "plt.legend(['W', 'b', 'True W', 'True b'])\n", - "plt.show()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vPnIVuaSJwWz" - }, - "source": [ - "## 次のステップ\n", - "\n", - "このチュートリアルでは `tf.Variable` を使って単純な線形モデルの構築と訓練を行いました。\n", - "\n", - "実際にニューラルネットワークを構築する際には、`tf.keras` のような高レベルな API のほうが遥かに便利です。`tf.keras` は、（「レイヤー」と呼ばれる）高レベルの部品、状態を保存・復元するためのユーティリティ、さまざまな損失関数、さまざまな最適化戦略などを提供しています。詳しく知るには [TensorFlow Keras guide](https://www.tensorflow.org/guide/keras/overview) を参照してください。\n" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "custom_training.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/site/ja/tutorials/customization/custom_training_walkthrough.ipynb b/site/ja/tutorials/customization/custom_training_walkthrough.ipynb deleted file mode 100644 index 334b2982c46..00000000000 --- a/site/ja/tutorials/customization/custom_training_walkthrough.ipynb +++ /dev/null @@ -1,1152 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training_walkthrough", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rwxGnsA92emp" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "CPII1rGR2rF9", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtEZ1pCPn--z" - }, - "source": [ - "# カスタム訓練：ウォークスルー" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GV1F7tVTN3Dn" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "SQ8UtWX18HNR", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LDrzLFXE8T1l" - }, - "source": [ - "このガイドでは機械学習を使ってアヤメの花を品種によって*分類*します。TensorFlowを使って下記のことを行います。\n", - "\n", - "1. モデルを構築し、\n", - "2. このモデルをサンプルデータを使って訓練し、\n", - "3. このモデルを使って未知のデータに対する予測を行う\n", - "\n", - "## TensorFlow プログラミング\n", - "\n", - "このガイドでは、次のような TensorFlow の高レベルの概念を用います。\n", - "\n", - "* TensorFlow の既定の [Eager Execution](https://www.tensorflow.org/guide/eager) 開発環境を用いて、\n", - "* データを [Datasets API](https://www.tensorflow.org/guide/datasets) を使ってインポートし、\n", - "* TensorFlow の [Keras API](https://keras.io/getting-started/sequential-model-guide/) を使ってモデルと層を構築する\n", - "\n", - "このチュートリアルは、多くの TensorFlow のプログラムと同様に下記のように構成されています。\n", - "\n", - "1. データセットのインポートとパース\n", - "2. モデルのタイプの選択\n", - "3. モデルの訓練\n", - "4. モデルの有効性の評価\n", - "5. 訓練済みモデルを使った予測" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yNr7H-AIoLOR" - }, - "source": [ - "## プログラムの設定" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1J3AuPBT9gyR" - }, - "source": [ - "### インポートの構成\n", - "\n", - "TensorFlow とその他必要な Python のモジュールをインポートします。TensorFlow は既定で [Eager Execution](https://www.tensorflow.org/guide/eager) を使って演算結果を即時に評価し、後で実行される [計算グラフ](https://www.tensorflow.org/guide/graphs) を構築する代わりに具体的な値を返します。REPL や `python` の対話的コンソールを使っている方にとっては、馴染みのあるものです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jElLULrDhQZR", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "import matplotlib.pyplot as plt" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bfV2Dai0Ow2o", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "g4Wzg69bnwK2", - "colab": {} - }, - "source": [ - "print(\"TensorFlow version: {}\".format(tf.__version__))\n", - "print(\"Eager execution: {}\".format(tf.executing_eagerly()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zx7wc0LuuxaJ" - }, - "source": [ - "## アヤメ分類問題\n", - "\n", - "あなたが植物学者で、発見したアヤメの花を分類する自動的な方法を探しているとします。機械学習には花を統計的に分類するためのアルゴリズムがたくさんあります。たとえば、洗練された機械学習プログラムを使うと、写真を元に花を分類することができます。私達の願望はもう少し控えめです。アヤメの花を [sepals（萼片）](https://en.wikipedia.org/wiki/Sepal) と [petals（花弁）](https://en.wikipedia.org/wiki/Petal) の長さと幅を使って分類することにしましょう。\n", - "\n", - "アヤメ属にはおよそ300の種がありますが、ここで扱うプログラムは次の3つの種類のみを分類します。\n", - "\n", - "* Iris setosa\n", - "* Iris virginica\n", - "* Iris versicolor\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Petal\n", - "
    \n", - " 図1. Iris setosa (by Radomil, CC BY-SA 3.0), Iris versicolor, (by Dlanglois, CC BY-SA 3.0), and Iris virginica (by Frank Mayfield, CC BY-SA 2.0).
     \n", - "
    \n", - "\n", - "幸いにして、萼片と花弁の計測値を使った [120のアヤメのデータセット](https://en.wikipedia.org/wiki/Iris_flower_data_set) を作ってくれた方がいます。これは初歩の機械学習による分類問題でよく使われる古典的なデータセットです。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Px6KAg0Jowz" - }, - "source": [ - "## 訓練用データセットのインポートとパース\n", - "\n", - "データセットファイルをダウンロードし、この Python プログラムで使えるような構造に変換します。\n", - "\n", - "### データセットのダウンロード\n", - "\n", - "[tf.keras.utils.get_file](https://www.tensorflow.org/api_docs/python/tf/keras/utils/get_file) 関数を使って訓練用データセットをダウンロードします。この関数は、ダウンロードしたファイルのパスを返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J6c7uEU9rjRM", - "colab": {} - }, - "source": [ - "train_dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv\"\n", - "\n", - "train_dataset_fp = tf.keras.utils.get_file(fname=os.path.basename(train_dataset_url),\n", - " origin=train_dataset_url)\n", - "\n", - "print(\"Local copy of the dataset file: {}\".format(train_dataset_fp))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qnX1-aLors4S" - }, - "source": [ - "### データの観察\n", - "\n", - "このデータセット `iris_training.csv` は、表形式のデータをコンマ区切り値（CSV）形式で格納したプレインテキストファイルです。`head -n5` コマンドを使って最初の5つのエントリを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FQvb_JYdrpPm", - "colab": {} - }, - "source": [ - "!head -n5 {train_dataset_fp}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kQhzD6P-uBoq" - }, - "source": [ - "こうしてデータセットを見ることで次のことがわかります。\n", - "\n", - "1. 最初の行はデータセットに関する情報を含むヘッダである\n", - " * 全体で120のサンプルがある。各サンプルには4つの特徴量があり、3つのラベルの1つが当てはまる。\n", - "2. 続く行はデータレコードで、1行が1つの [*サンプル(example)*](https://developers.google.com/machine-learning/glossary/#example) である。ここでは、\n", - " * 最初の4つのフィールドはサンプルの特徴を示す [*特徴量(features)*](https://developers.google.com/machine-learning/glossary/#feature) である。ここで各フィールドは花の計測値を表す浮動小数点値をもつ。\n", - " * 最後の列は予想したい [*ラベル(label)*](https://developers.google.com/machine-learning/glossary/#label) である。このデータセットでは、花の名前に対応する整数値、0、1、2 のいずれかである。\n", - "\n", - "コードを書いてみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9Edhevw7exl6", - "colab": {} - }, - "source": [ - "# CSV ファイルの列の順序\n", - "column_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']\n", - "\n", - "feature_names = column_names[:-1]\n", - "label_name = column_names[-1]\n", - "\n", - "print(\"Features: {}\".format(feature_names))\n", - "print(\"Label: {}\".format(label_name))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCtwLoJhhDNc" - }, - "source": [ - "ラベルはそれぞれ文字列の名前（たとえば、\"setosa\"）に関連付けられていますが、機械学習は大抵の場合、数値に依存しています。ラベルの数値は名前の表現にマップされます。たとえば次のようになります。\n", - "\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica\n", - "\n", - "特徴量とラベルについてもっと知りたい場合には、 [ML Terminology section of the Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/framing/ml-terminology) を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sVNlJlUOhkoX", - "colab": {} - }, - "source": [ - "class_names = ['Iris setosa', 'Iris versicolor', 'Iris virginica']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dqPkQExM2Pwt" - }, - "source": [ - "### `tf.data.Dataset` の作成\n", - "\n", - "TensorFlow の [Dataset API](https://www.tensorflow.org/guide/datasets) はデータのモデルへのロードについて、多くの一般的なケースを扱います。この API は、データを読み込んで訓練に使われる形式へ変換するための高レベル API です。詳しくは、 [Datasets Quick Start guide](https://www.tensorflow.org/get_started/datasets_quickstart) を参照してください。\n", - "\n", - "今回のデータセットは CSV 形式のテキストファイルなので、データをパースして適切なフォーマットに変換するため、 [make_csv_dataset](https://www.tensorflow.org/api_docs/python/tf/data/experimental/make_csv_dataset) 関数を使います。この関数はモデルの訓練のためのデータを生成するので、既定の動作はデータをシャッフルし (`shuffle=True, shuffle_buffer_size=10000`) 、データセットを永遠に繰り返すこと (`num_epochs=None`) です。また、 [batch_size](https://developers.google.com/machine-learning/glossary/#batch_size) パラメータも設定します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WsxHnz1ebJ2S", - "colab": {} - }, - "source": [ - "batch_size = 32\n", - "\n", - "train_dataset = tf.data.experimental.make_csv_dataset(\n", - " train_dataset_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name=label_name,\n", - " num_epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gB_RSn62c-3G" - }, - "source": [ - "`make_csv_dataset` 関数は、 `(features, label)` というペアの `tf.data.Dataset` を返します。ここで、 `features` は、 `{'feature_name': value}` というディクショナリです。\n", - "\n", - "この `Dataset` オブジェクトは反復可能オブジェクトです。特徴量のバッチを1つ見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iDuG94H-C122", - "colab": {} - }, - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "print(features)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E63mArnQaAGz" - }, - "source": [ - "「特徴量のようなもの」はグループ化、言い換えると *バッチ化* されることに注意してください。それぞれのサンプル行のフィールドは、対応する特徴量配列に追加されます。これらの特徴量配列に保存されるサンプルの数を設定するには、 `batch_size` を変更します。\n", - "\n", - "上記のバッチから特徴量のいくつかをプロットしてみると、いくつかのクラスタが見られます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "me5Wn-9FcyyO", - "colab": {} - }, - "source": [ - "plt.scatter(features['petal_length'],\n", - " features['sepal_length'],\n", - " c=labels,\n", - " cmap='viridis')\n", - "\n", - "plt.xlabel(\"Petal length\")\n", - "plt.ylabel(\"Sepal length\")\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YlxpSyHlhT6M" - }, - "source": [ - "モデル構築のステップを単純化するため、特徴量のディクショナリを `(batch_size, num_features)` という形状の単一の配列にパッケージし直す関数を作成します。\n", - "\n", - "この関数は、テンソルのリストから値を受け取り、指定された次元で結合されたテンソルを作成する [tf.stack](https://www.tensorflow.org/api_docs/python/tf/stack) メソッドを使っています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jm932WINcaGU", - "colab": {} - }, - "source": [ - "def pack_features_vector(features, labels):\n", - " \"\"\"特徴量を1つの配列にパックする\"\"\"\n", - " features = tf.stack(list(features.values()), axis=1)\n", - " return features, labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "V1Vuph_eDl8x" - }, - "source": [ - "次に、 [tf.data.Dataset.map](https://www.tensorflow.org/api_docs/python/tf/data/dataset/map) メソッドを使って `(features,label)` ペアそれぞれの `features` を訓練用データセットにパックします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZbDkzGZIkpXf", - "colab": {} - }, - "source": [ - "train_dataset = train_dataset.map(pack_features_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NLy0Q1xCldVO" - }, - "source": [ - "この `Dataset` の特徴量要素は、`(batch_size, num_features)` の形状をした配列になっています。 最初のサンプルをいくつか見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kex9ibEek6Tr", - "colab": {} - }, - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "print(features[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LsaVrtNM3Tx5" - }, - "source": [ - "## モデルのタイプの選択\n", - "\n", - "### なぜモデルか？\n", - "\n", - "[*モデル(model)*](https://developers.google.com/machine-learning/crash-course/glossary#model) は特徴量とラベルの間の関係です。アヤメの分類問題の場合、モデルは萼片と花弁の計測値と予測されるアヤメの種類の関係を定義します。単純なモデルであれば、2、3行の数式で記述できますが、複雑な機械学習モデルには多数のパラメータがあり、かんたんに要約できるものではありません。\n", - "\n", - "機械学習を*使わずに*これらの4つの特徴量とアヤメの品種の関係を決定することは可能でしょうか？言い換えると、従来のプログラミング技術（たとえば、たくさんの条件文）を使ってモデルを作ることは可能でしょうか？おそらく、十分に時間をかけてデータセットを分析して、萼片と花弁の計測値と特定の品種との関係を決定すれば可能でしょう。もっと複雑なデータセットの場合には、これは困難であり、おそらく不可能です。機械学習を使ったよいアプローチでは、*あなたに代わってモデルを決定*してくれます。適切なタイプの機械学習モデルに、十分な量の典型的なサンプルを投入すれば、プログラムがあなたのためにこの関係性をみつけ出してくれるでしょう。\n", - "\n", - "### モデルの選択\n", - "\n", - "訓練すべきモデルの種類を決める必要があります。モデルにはたくさんの種類があり、よいものを選択するには経験が必要です。このチュートリアルでは、アヤメの分類問題を解くために、ニューラルネットワークを使用します。[*ニューラルネットワーク(Neural networks)*](https://developers.google.com/machine-learning/glossary/#neural_network) は、特徴量とラベルの間の複雑な関係をみつけることができます。ニューラルネットワークは、1つかそれ以上の [*隠れ層(hidden layers)*](https://developers.google.com/machine-learning/glossary/#hidden_layer) で構成された高度なグラフ構造です。それぞれの隠れ層には1つ以上の [*ニューロン(neurons)*](https://developers.google.com/machine-learning/glossary/#neuron) があります。ニューラルネットワークにはいくつかのカテゴリーがありますが、このプログラムでは、[*全結合ニューラルネットワーク(fully-connected neural network または dense neural network)*](https://developers.google.com/machine-learning/glossary/#fully_connected_layer) を使用します。全結合ニューラルネットワークでは、1つの層の中のニューロンすべてが、前の層の*すべての*ニューロンからの入力を受け取ります。例として、図2に入力層、2つの隠れ層、そして、出力層からなる密なニューラルネットワークを示します。\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \n", - "
    \n", - " 図2. 特徴量と隠れ層、予測をもつニューラルネットワーク
     \n", - "
    \n", - "\n", - "図2のモデルが訓練されてラベルの付いていないサンプルを受け取ったとき、モデルは3つの予測値を返します。予測値はサンプルの花が与えられた3つの品種のそれぞれである可能性を示します。この予測は、 [*推論(inference)*](https://developers.google.com/machine-learning/crash-course/glossary#inference) とも呼ばれます。このサンプルでは、出力の予測値の合計は 1.0 になります。図2では、この予測値は *Iris setosa* が `0.02` 、 *Iris versicolor* が `0.95` 、 *Iris virginica* が `0.03` となっています。これは、モデルがこのラベルのない花を、95% の確率で *Iris versicolor* であると予測したことを意味します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W23DIMVPQEBt" - }, - "source": [ - "### Keras を使ったモデル構築\n", - "\n", - "TensorFlow の [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras) API は、モデルと層を作成するためのおすすめの方法です。Keras がすべてを結びつけるという複雑さを引き受けてくれるため、モデルや実験の構築がかんたんになります。\n", - "\n", - "[tf.keras.Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential) モデルは層が一列に積み上げられたものです。このクラスのコンストラクタは、レイヤーインスタンスのリストを引数として受け取ります。今回の場合、それぞれ 10 のノードをもつ 2つの [Dense](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense) レイヤーと、ラベルの予測値を表す3つのノードからなる出力レイヤーです。最初のレイヤーの `input_shape` パラメータがデータセットの特徴量の数に対応しており、これは必須パラメータです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2fZ6oL2ig3ZK", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(4,)), # input shape required\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu),\n", - " tf.keras.layers.Dense(3)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FHcbEzMpxbHL" - }, - "source": [ - "[*活性化関数(activation function)*](https://developers.google.com/machine-learning/crash-course/glossary#activation_function) は、そのレイヤーの各ノードの出力の形を決定します。この関数の非線形性は重要であり、それがなければモデルは 1層しかないものと等価になってしまいます。[利用可能な活性化関数](https://www.tensorflow.org/api_docs/python/tf/keras/activations) はたくさんありますが、隠れ層では [ReLU](https://developers.google.com/machine-learning/crash-course/glossary#ReLU) が一般的です。\n", - "\n", - "理想的な隠れ層の数やニューロンの数は問題やデータセットによって異なります。機械学習のさまざまな側面と同様に、ニューラルネットワークの最良の形を選択するには、知識と経験の両方が必要です。経験則から、一般的には隠れ層やニューロンの数を増やすとより強力なモデルを作ることができますが、効果的に訓練を行うためにより多くのデータを必要とします。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2wFKnhWCpDSS" - }, - "source": [ - "### モデルの使用\n", - "\n", - "それでは、このモデルが特徴量のバッチに対して何を行うかを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xe6SQ5NrpB-I", - "colab": {} - }, - "source": [ - "predictions = model(features)\n", - "predictions[:5]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wxyXOhwVr5S3" - }, - "source": [ - "ご覧のように、サンプルのそれぞれは、各クラスの [ロジット(logit)](https://developers.google.com/machine-learning/crash-course/glossary#logits) 値を返します。\n", - "\n", - "これらのロジット値を各クラスの確率に変換するためには、 [softmax](https://developers.google.com/machine-learning/crash-course/glossary#softmax) 関数を使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_tRwHZmTNTX2", - "colab": {} - }, - "source": [ - "tf.nn.softmax(predictions[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uRZmchElo481" - }, - "source": [ - "クラス間で `tf.argmax` を取ると、予測されたクラスのインデックスが得られます。しかし、このモデルはまだ訓練されていないので、予測はよいものではありません。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-Jzm_GoErz8B", - "colab": {} - }, - "source": [ - "print(\"Prediction: {}\".format(tf.argmax(predictions, axis=1)))\n", - "print(\" Labels: {}\".format(labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vzq2E5J2QMtw" - }, - "source": [ - "## モデルの訓練\n", - "\n", - "[*訓練(Training)*](https://developers.google.com/machine-learning/crash-course/glossary#training) は、機械学習において、モデルが徐々に最適化されていく、あるいはモデルがデータセットを*学習する*段階です。目的は、見たことのないデータについて予測を行うため、訓練用データセットの構造を十分に学習することです。訓練用データセットを学習*しすぎる*と、予測は見たことのあるデータに対してしか有効ではなく、一般化できません。この問題は [*過学習(overfitting)*](https://developers.google.com/machine-learning/crash-course/glossary#overfitting) と呼ばれ、問題の解き方を理解するのではなく答えを丸暗記するようなものです。\n", - "\n", - "アヤメの分類問題は、 [*教師あり学習(supervised machine learning)*](https://developers.google.com/machine-learning/glossary/#supervised_machine_learning) の1種で、モデルはラベルの付いたサンプルを学習します。[*教師なし学習(unsupervised machine learning)*](https://developers.google.com/machine-learning/glossary/#unsupervised_machine_learning) では、サンプルにラベルはありません。そのかわり、一般的にはモデルが特徴量からパターンを発見します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RaKp8aEjKX6B" - }, - "source": [ - "### 損失関数と勾配関数の定義\n", - "\n", - "訓練段階と評価段階では、モデルの [*損失(loss)*](https://developers.google.com/machine-learning/crash-course/glossary#loss) を計算する必要があります。損失とは、モデルの予測が望ましいラベルからどれくらい離れているかを測定するものです。言い換えると、どれくらいモデルの性能が悪いかを示します。この値を最小化、または最適化したいのです。\n", - "\n", - "今回のモデルでは、 `tf.keras.losses.SparseCategoricalCrossentropy` 関数を使って損失を計算します。この関数は、モデルのクラスごとの予測確率とラベルの取るべき値を使って、サンプル間の平均損失を返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QOsi6b-1CXIn", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tMAT4DcMPwI-", - "colab": {} - }, - "source": [ - "def loss(model, x, y):\n", - " y_ = model(x)\n", - "\n", - " return loss_object(y_true=y, y_pred=y_)\n", - "\n", - "\n", - "l = loss(model, features, labels)\n", - "print(\"Loss test: {}\".format(l))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3IcPqA24QM6B" - }, - "source": [ - "[tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) コンテキストを使って、モデルを最適化する際に使われる [*勾配(gradients)*](https://developers.google.com/machine-learning/crash-course/glossary#gradient) を計算しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x57HcKWhKkei", - "colab": {} - }, - "source": [ - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return loss_value, tape.gradient(loss_value, model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lOxFimtlKruu" - }, - "source": [ - "### オプティマイザの作成\n", - "\n", - "[*オプティマイザ(optimizer)*](https://developers.google.com/machine-learning/crash-course/glossary#optimizer) は、`loss` 関数を最小化するため、計算された勾配をモデルの変数に適用します。損失関数は、曲面として考えることができ（図3 参照）、歩き回ることによって最小となる点をみつけたいのです。勾配は、一番急な上りの方向を示すため、逆方向に進んで丘を下ることになります。バッチごとに損失と勾配を繰り返し計算することにより、訓練中のモデルを調節します。モデルは徐々に、損失を最小化する重みとバイアスの最適な組み合わせをみつけます。損失が小さいほど、モデルの予測がよくなります。\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Optimization\n", - "
    \n", - " 図3. 3次元空間における最適化アルゴリズムの時系列可視化。
    (Source: Stanford class CS231n, MIT License, Image credit: Alec Radford)\n", - "
    \n", - "\n", - "TensorFlow には、訓練に使える [最適化アルゴリズム(optimization algorithms)](https://www.tensorflow.org/api_guides/python/train) がたくさんあります。このモデルでは、[*確率的勾配降下法(stochastic gradient descent)*](https://developers.google.com/machine-learning/crash-course/glossary#gradient_descent) (SGD) アルゴリズムを実装した [tf.train.GradientDescentOptimizer](https://www.tensorflow.org/api_docs/python/tf/train/GradientDescentOptimizer) を使用します。`learning_rate` （学習率）は、イテレーションごとに丘を下る際のステップのサイズを設定します。通常これはよりよい結果を得るために調整する *ハイパーパラメータ(hyperparameter)* です。 " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XkUd6UiZa_dF" - }, - "source": [ - "オプティマイザーを設定しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8xxi2NNGKwG_", - "colab": {} - }, - "source": [ - "optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pJVRZ0hP52ZB" - }, - "source": [ - "これを使って、最適化を１ステップ分計算してみます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rxRNTFVe56RG", - "colab": {} - }, - "source": [ - "loss_value, grads = grad(model, features, labels)\n", - "\n", - "print(\"Step: {}, Initial Loss: {}\".format(optimizer.iterations.numpy(),\n", - " loss_value.numpy()))\n", - "\n", - "optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - "print(\"Step: {}, Loss: {}\".format(optimizer.iterations.numpy(),\n", - " loss(model, features, labels).numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Y2VSELvwAvW" - }, - "source": [ - "### 訓練ループ\n", - "\n", - "すべての部品が揃ったので、モデルの訓練ができるようになりました。訓練ループは、モデルにデータセットのサンプルを供給し、モデルがよりよい予測を行えるようにします。下記のコードブロックは、この訓練のステップを構成します。\n", - "\n", - "1. *epoch（エポック）* をひとつずつ繰り返します。エポックとは、データセットをひととおり処理するということです。\n", - "2. エポック内では、訓練用の `Dataset（データセット）` のサンプルひとつずつから、その *features（特徴量）* (`x`) と *label（ラベル）* (`y`) を取り出して繰り返し処理します。\n", - "3. サンプルの特徴量を使って予測を行い、ラベルと比較します。予測の不正確度を測定し、それを使ってモデルの損失と勾配を計算します。\n", - "4. `optimizer` を使って、モデルの変数を更新します。\n", - "5. 可視化のためにいくつかの統計量を記録します。\n", - "6. これをエポックごとに繰り返します。\n", - "\n", - "`num_epochs` 変数は、データセットのコレクションを何回繰り返すかという数字です。直感には反しますが、モデルを長く訓練すれば必ずよいモデルが得られるというわけではありません。`num_epochs` は、[*ハイパーパラメータ(hyperparameter)*](https://developers.google.com/machine-learning/glossary/#hyperparameter) であり、チューニングできます。適切な数を選択するには、経験と実験の両方が必要です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AIgulGRUhpto", - "colab": {} - }, - "source": [ - "## Note: このセルを再実行すると同じモデル変数が使われます\n", - "\n", - "# 結果をグラフ化のために保存\n", - "train_loss_results = []\n", - "train_accuracy_results = []\n", - "\n", - "num_epochs = 201\n", - "\n", - "for epoch in range(num_epochs):\n", - " epoch_loss_avg = tf.keras.metrics.Mean()\n", - " epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n", - "\n", - " # 訓練ループ - 32個ずつのバッチを使用\n", - " for x, y in train_dataset:\n", - " # モデルの最適化\n", - " loss_value, grads = grad(model, x, y)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " # 進捗の記録\n", - " epoch_loss_avg(loss_value) # Add current batch loss 現在のバッチの損失を加算\n", - " # 予測ラベルと実際のラベルを比較\n", - " epoch_accuracy(y, model(x))\n", - " \n", - " # エポックの終わり\n", - " train_loss_results.append(epoch_loss_avg.result())\n", - " train_accuracy_results.append(epoch_accuracy.result())\n", - "\n", - " if epoch % 50 == 0:\n", - " print(\"Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}\".format(epoch,\n", - " epoch_loss_avg.result(),\n", - " epoch_accuracy.result()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2FQHVUnm_rjw" - }, - "source": [ - "### 時間の経過に対する損失関数の可視化" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j3wdbmtLVTyr" - }, - "source": [ - "モデルの訓練の進み方をプリントするのも役立ちますが、普通はこの進捗を見るほうが*もっと*役に立ちます。[TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) は TensorFlow に付属する優れた可視化ツールですが、\n", - " `matplotlib` モジュールを使って基本的なグラフを描画することもできます。\n", - " \n", - "これらのグラフを解釈するにはある程度の経験が必要ですが、*損失*が低下して、*正解率*が上昇するのをぜひ見たいと思うでしょう。 " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "agjvNd2iUGFn", - "colab": {} - }, - "source": [ - "fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))\n", - "fig.suptitle('Training Metrics')\n", - "\n", - "axes[0].set_ylabel(\"Loss\", fontsize=14)\n", - "axes[0].plot(train_loss_results)\n", - "\n", - "axes[1].set_ylabel(\"Accuracy\", fontsize=14)\n", - "axes[1].set_xlabel(\"Epoch\", fontsize=14)\n", - "axes[1].plot(train_accuracy_results)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zg8GoMZhLpGH" - }, - "source": [ - "## モデルの有効性評価\n", - "\n", - "モデルの訓練が終わったら、その性能の指標を得ることができます。\n", - "\n", - "*評価*とは、モデルの予測がどれだけ効果的であるかどうかを決定することを言います。アヤメの分類でモデルの有効性を見定めるには、モデルに萼片と花弁の測定値をいくつか与え、どのアヤメの品種であるかを予測させます。そして、モデルの予測と実際のラベルを比較します。たとえば、あるモデルが入力サンプルの半分で正解の品種をえらんだとすると、[*正解率(accuracy)*](https://developers.google.com/machine-learning/glossary/#accuracy) は `0.5` ということになります。図４で示すモデルはもう少し効果的であり、5つの予測中4つで正解し、正解率は 80% です。\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    サンプルの特徴量ラベルモデルの予測値
    5.93.04.31.511
    6.93.15.42.122
    5.13.31.70.500
    6.0 3.4 4.5 1.6 12
    5.52.54.01.311
    \n", - " 図4. 正解率 80％ のアヤメ分類器
     \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z-EvK7hGL0d8" - }, - "source": [ - "### テスト用データセットの設定\n", - "\n", - "モデルの評価はモデルの訓練と同様です。もっとも大きな違いは、サンプルが訓練用データセットではなく[*テスト用データセット(test set)*](https://developers.google.com/machine-learning/crash-course/glossary#test_set) からのものであるという点です。モデルの有効性を正しく評価するには、モデルの評価に使うサンプルは訓練用データセットのものとは違うものでなければなりません。\n", - "\n", - "テスト用 `Dataset` の設定は、訓練用 `Dataset` の設定と同様です。CSV ファイルをダウンロードし、値をパースしてからすこしシャッフルを行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ps3_9dJ3Lodk", - "colab": {} - }, - "source": [ - "test_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv\"\n", - "\n", - "test_fp = tf.keras.utils.get_file(fname=os.path.basename(test_url),\n", - " origin=test_url)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SRMWCu30bnxH", - "colab": {} - }, - "source": [ - "test_dataset = tf.data.experimental.make_csv_dataset(\n", - " test_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name='species',\n", - " num_epochs=1,\n", - " shuffle=False)\n", - "\n", - "test_dataset = test_dataset.map(pack_features_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HFuOKXJdMAdm" - }, - "source": [ - "### テスト用データセットでのモデルの評価\n", - "\n", - "訓練段階とは異なり、モデルはテスト用データの 1[エポック(epoch)](https://developers.google.com/machine-learning/glossary/#epoch) だけで行います。下記のコードセルでは、テスト用データセットのサンプルをひとつずつ処理し、モデルの予測と実際のラベルを比較します。この結果を使ってテスト用データセット全体でのモデルの正解率を計算します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tw03-MK1cYId", - "colab": {} - }, - "source": [ - "test_accuracy = tf.keras.metrics.Accuracy()\n", - "\n", - "for (x, y) in test_dataset:\n", - " logits = model(x)\n", - " prediction = tf.argmax(logits, axis=1, output_type=tf.int32)\n", - " test_accuracy(prediction, y)\n", - "\n", - "print(\"Test set accuracy: {:.3%}\".format(test_accuracy.result()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HcKEZMtCOeK-" - }, - "source": [ - "たとえば最後のバッチを見ると、モデルはだいたい正確です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uNwt2eMeOane", - "colab": {} - }, - "source": [ - "tf.stack([y,prediction],axis=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Li2r1tYvW7S" - }, - "source": [ - "## 訓練済みモデルを使った予測\n", - "\n", - "モデルを訓練し、それが、完全ではないにせよアヤメの品種の分類でよい性能を示すことを「証明」しました。次は、この訓練済みモデルを使って [ラベルなしサンプル(unlabeled examples)](https://developers.google.com/machine-learning/glossary/#unlabeled_example) つまり、特徴量だけでラベルのないサンプルについて予測を行ってみましょう。\n", - "\n", - "現実世界では、ラベルなしサンプルはアプリや CSV ファイル、データフィードなどさまざまな異なるソースからやってきます。ここでは、ラベルを予測するため、3つのラベルなしサンプルを手動で与えることにします。ラベルの番号は、下記のように名前を表していることを思い出してください。\n", - "\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kesTS5Lzv-M2", - "colab": {} - }, - "source": [ - "predict_dataset = tf.convert_to_tensor([\n", - " [5.1, 3.3, 1.7, 0.5,],\n", - " [5.9, 3.0, 4.2, 1.5,],\n", - " [6.9, 3.1, 5.4, 2.1]\n", - "])\n", - "\n", - "predictions = model(predict_dataset)\n", - "\n", - "for i, logits in enumerate(predictions):\n", - " class_idx = tf.argmax(logits).numpy()\n", - " p = tf.nn.softmax(logits)[class_idx]\n", - " name = class_names[class_idx]\n", - " print(\"Example {} prediction: {} ({:4.1f}%)\".format(i, name, 100*p))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/customization/performance.ipynb b/site/ja/tutorials/customization/performance.ipynb deleted file mode 100644 index d085797a638..00000000000 --- a/site/ja/tutorials/customization/performance.ipynb +++ /dev/null @@ -1,1310 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "performance.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "ISubpr_SSsiM" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISubpr_SSsiM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "3jTMb1dySr3V", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6DWfyNThSziV" - }, - "source": [ - "# tf.function で性能アップ\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J122XQYG7W6w" - }, - "source": [ - "TensorFlow 2.0 では Eager Execution が既定で有効になっています。ユーザーインターフェイスは直感的で柔軟です（演算を一度だけ行う場合にはずっと簡単に、かつ迅速に実行されます）。しかしながら、それは性能と展開の面での犠牲の上に成り立っています。\n", - "\n", - "最高性能を得ながら、モデルをどこへでも展開できるようにするには、`tf.function` を使ってプログラムから計算グラフを作成します。\n", - "AutoGraph のおかげで、驚くほど多くの Python コードが tf.function でそのまま動作しますが、気をつけなければならない落とし穴も存在します。\n", - "\n", - "ポイントと推奨事項は下記の通りです。\n", - "\n", - "- オブジェクトの変更やリストへの追加のような Python の副作用に依存しないこと\n", - "- tf.functions は NumPy の演算や Python の組み込み演算よりも、TensorFlow の演算に適していること\n", - "- 迷ったときは、`for x in y` というイディオムを使うこと" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "otIdN1TS8N7S", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "D25apou9IOXa", - "colab": {} - }, - "source": [ - "import contextlib\n", - "\n", - "# 遭遇するかもしれないいくつかのエラーをデモするためのヘルパー関数\n", - "@contextlib.contextmanager\n", - "def assert_raises(error_class):\n", - " try:\n", - " yield\n", - " except error_class as e:\n", - " print('Caught expected exception \\n {}: {}'.format(error_class, e))\n", - " except Exception as e:\n", - " print('Got unexpected exception \\n {}: {}'.format(type(e), e))\n", - " else:\n", - " raise Exception('Expected {} to be raised but no error was raised!'.format(\n", - " error_class))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rfayNj-ZIkIB" - }, - "source": [ - "あなたが定義した `tf.function` は TensorFlow Core の演算に似たものです。例えばそれを即時に実行することも、計算グラフで使うこともできますし、勾配を計算することも可能です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SbtT1-Wm70F2", - "colab": {} - }, - "source": [ - "# function は演算のように振る舞う\n", - "\n", - "@tf.function\n", - "def add(a, b):\n", - " return a + b\n", - "\n", - "add(tf.ones([2, 2]), tf.ones([2, 2])) # [[2., 2.], [2., 2.]]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uP-zUelB8DbX", - "colab": {} - }, - "source": [ - "# function は勾配を計算できる\n", - "\n", - "@tf.function\n", - "def add(a, b):\n", - " return a + b\n", - "\n", - "v = tf.Variable(1.0)\n", - "with tf.GradientTape() as tape:\n", - " result = add(v, 1.0)\n", - "tape.gradient(result, v)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "l5qRjdbBVdU6", - "colab": {} - }, - "source": [ - "# function 内で function を使うこともできる\n", - "\n", - "@tf.function\n", - "def dense_layer(x, w, b):\n", - " return add(tf.matmul(x, w), b)\n", - "\n", - "dense_layer(tf.ones([3, 2]), tf.ones([2, 2]), tf.ones([2]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uZ4Do2AV80cO" - }, - "source": [ - "## トレーシングとポリモーフィズム\n", - "\n", - "Python の動的型付けは、関数をさまざまな型の引数で呼び出すことができ、Python がそれぞれのシナリオで異なる動作をするということを意味します。\n", - "\n", - "他方で、TensorFlow の計算グラフでは、dtype と shape の次元が静的であることが必要です。`tf.function` は、正しい計算グラフを生成するために必要なときには関数を再トレースして、このギャップをつなぐ役割を果たします。\n", - "\n", - "異なる型の引数を使って関数を呼び出し、何が起きるか見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kojmJrgq8U9v", - "colab": {} - }, - "source": [ - "# Function はポリモーフィック\n", - "\n", - "@tf.function\n", - "def double(a):\n", - " print(\"Tracing with\", a)\n", - " return a + a\n", - "\n", - "print(double(tf.constant(1)))\n", - "print()\n", - "print(double(tf.constant(1.1)))\n", - "print()\n", - "print(double(tf.constant(\"a\")))\n", - "print()\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4pJqkDR_Q2wz" - }, - "source": [ - "トレースの動作を制御するためには、下記のようなテクニックを使います。\n", - "\n", - "- 新しい `tf.function` を作成する。別々の `tf.function` オブジェクトがトレースを共有することはない。\n", - "- 特定のトレースを得るには `get_concrete_function` メソッドを使用する。\n", - "- 計算グラフの呼び出し時に1回だけトレースを行うには、 `input_signature` を指定して `tf.function` を呼び出す。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mHg2CGtPQ3Hz", - "colab": {} - }, - "source": [ - "print(\"Obtaining concrete trace\")\n", - "double_strings = double.get_concrete_function(tf.TensorSpec(shape=None, dtype=tf.string))\n", - "print(\"Executing traced function\")\n", - "print(double_strings(tf.constant(\"a\")))\n", - "print(double_strings(a=tf.constant(\"b\")))\n", - "print(\"Using a concrete trace with incompatible types will throw an error\")\n", - "with assert_raises(tf.errors.InvalidArgumentError):\n", - " double_strings(tf.constant(1))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_BDMIRmu1RGB", - "colab": {} - }, - "source": [ - "@tf.function(input_signature=(tf.TensorSpec(shape=[None], dtype=tf.int32),))\n", - "def next_collatz(x):\n", - " print(\"Tracing with\", x)\n", - " return tf.where(tf.equal(x % 2, 0), x // 2, 3 * x + 1)\n", - "\n", - "print(next_collatz(tf.constant([1, 2])))\n", - "# 1次元のテンソルを input signature として指定しているので、これは失敗する\n", - "with assert_raises(ValueError):\n", - " next_collatz(tf.constant([[1, 2], [3, 4]]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Es0WZkLIUSdu" - }, - "source": [ - "## いつ再トレースするのか？\n", - "\n", - "ポリモーフィックな `tf.function` はトレーシングによって生成された具象関数のキャッシュを保持しています。キャッシュのキーは、実際にはその関数の引数及びキーワード引数から生成されたキーのタプルです。`tf.Tensor` 引数から生成されるキーは、テンソルの shape と型です。Python の組み込み型引数から生成されるキーはその値です。それ以外の Python の型では、キーはオブジェクトの `id()` に基づいており、メソッドはクラスのインスタンスひとつずつ独立にトレースされます。将来、TensorFlowには、Python オブジェクトについて安全にテンソルに変換できるような、より洗練されたキャッシングが追加されるかもしれません。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AY5oiQN0XIyA" - }, - "source": [ - "## 引数は Python か？ Tensor か？\n", - "\n", - "しばしば、ハイパーパラメータやグラフ構成を制御するために Python の組み込み型の引数が使われます。例えば、`num_layers=10` や `training=True` あるいは `nonlinearity='relu'` のようにです。このため、この Python の組み込み型の引数が変更されると、計算グラフを再びトレースする必要があるということになります。\n", - "\n", - "しかし、グラフの生成を制御するために Python の組み込み型の引数を使用する必要はありません。これらのケースでは、Python引数の値の変更が不必要な再トレースを引き起こす可能性があります。例えば、この訓練ループでは、AutoGraph は動的に展開を行います。複数回トレースを行っていますが、生成される計算グラフは全く変わりません。これは少し非効率です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uydzR5JYUU8H", - "colab": {} - }, - "source": [ - "def train_one_step():\n", - " pass\n", - "\n", - "@tf.function\n", - "def train(num_steps):\n", - " print(\"Tracing with num_steps = {}\".format(num_steps))\n", - " for _ in tf.range(num_steps):\n", - " train_one_step()\n", - "\n", - "train(num_steps=10)\n", - "train(num_steps=20)\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f6pjnylLUW8P" - }, - "source": [ - "ここでの簡単な回避方法は、生成されたグラフの shape が変わらないのであれば、引数をテンソルにキャストすることです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TmL8T-w3UYes", - "colab": {} - }, - "source": [ - "train(num_steps=tf.constant(10))\n", - "train(num_steps=tf.constant(20))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "129-iRsPS-gY" - }, - "source": [ - "## `tf.function` の中の副作用\n", - "\n", - "一般的には、（印字やオブジェクト変更のような）Python の副作用は、トレーシングの最中にだけ発生します。それでは、どうしたら `tf.function` で安定的に副作用を起こすことができるでしょうか？\n", - "\n", - "一般的な原則は、トレースをデバッグする際にだけ Python の副作用を使用するというものです。あるいは、`tf.Variable.assign`、`tf.print`、そして `tf.summary` のような TensorFlow の演算を使うことで、コードがトレースされるときにも、TensorFlowランタイムによって都度呼び出される際にも、確実に実行されるようにできます。一般には、関数型のスタイルを使用することで最も良い結果を得られます。 " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "w2sACuZ9TTRk", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def f(x):\n", - " print(\"Traced with\", x)\n", - " tf.print(\"Executed with\", x)\n", - "\n", - "f(1)\n", - "f(1)\n", - "f(2)\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e1I0dPiqTV8H" - }, - "source": [ - "`tf.function` が呼び出されるたびに Python のコードを実行したいのであれば、`tf.py_function` がぴったりです。`tf.py_function` の欠点は、ポータブルでないこと、それほど性能が高くないこと、（マルチGPU、TPUの）分散環境ではうまく動作しないことなどです。また、`tf.py_function` は計算グラフに組み込まれるため、入出力すべてをテンソルにキャストします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7aJD--9qTWmg", - "colab": {} - }, - "source": [ - "external_list = []\n", - "\n", - "def side_effect(x):\n", - " print('Python side effect')\n", - " external_list.append(x)\n", - "\n", - "@tf.function\n", - "def f(x):\n", - " tf.py_function(side_effect, inp=[x], Tout=[])\n", - "\n", - "f(1)\n", - "f(1)\n", - "f(1)\n", - "assert len(external_list) == 3\n", - "# .numpy() call required because py_function casts 1 to tf.constant(1)\n", - "assert external_list[0].numpy() == 1\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "msTmv-oyUNaf" - }, - "source": [ - "## Python の状態に注意\n", - "\n", - "ジェネレーターやイテレーターなど Python の機能の多くは、状態を追跡するために Python のランタイムに依存しています。これらの仕組みは、一般的には Eager モードでも期待通りに動作しますが、トレーシングの振る舞いにより、`tf.function` の中では予期しないことが起きることがあります。\n", - "\n", - "1例として、イテレーターの状態が進むのは Python の副作用であり、トレーシングの中だけで発生します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FNPD4unZUedH", - "colab": {} - }, - "source": [ - "external_var = tf.Variable(0)\n", - "@tf.function\n", - "def buggy_consume_next(iterator):\n", - " external_var.assign_add(next(iterator))\n", - " tf.print(\"Value of external_var:\", external_var)\n", - "\n", - "iterator = iter([0, 1, 2, 3])\n", - "buggy_consume_next(iterator)\n", - "# 次のコードは、イテレーターの次の値を使うのではなく、最初の値を再利用する\n", - "buggy_consume_next(iterator)\n", - "buggy_consume_next(iterator)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5XMGXMu-Ufjm" - }, - "source": [ - "イテレーターが tf.function の中で生成されすべて使われる場合には、正しく動作するはずです。しかし、イテレーター全体がトレースされることとなり、巨大な計算グラフの生成をまねく可能性があります。これは、望みどおりの動作かもしれません。しかし、もし Python のリストとして表されたメモリー上の巨大なデータセットを使って訓練を行うとすると、これは非常に大きな計算グラフを生成することになり、`tf.function` がスピードアップにはつながらないと考えられます。\n", - "\n", - "Python データを繰り返し使用する場合、もっとも安全な方法は tf.data.Dataset でラップして、`for x in y` というイディオムを使用することです。AutoGraph には、`y` がテンソルあるいは tf.data.Dataset である場合、`for` ループを安全に変換する特別な機能があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ms7f1o_QUiHE", - "colab": {} - }, - "source": [ - "def measure_graph_size(f, *args):\n", - " g = f.get_concrete_function(*args).graph\n", - " print(\"{}({}) contains {} nodes in its graph\".format(\n", - " f.__name__, ', '.join(map(str, args)), len(g.as_graph_def().node)))\n", - "\n", - "@tf.function\n", - "def train(dataset):\n", - " loss = tf.constant(0)\n", - " for x, y in dataset:\n", - " loss += tf.abs(y - x) # ダミー計算\n", - " return loss\n", - "\n", - "small_data = [(1, 1)] * 2\n", - "big_data = [(1, 1)] * 10\n", - "measure_graph_size(train, small_data)\n", - "measure_graph_size(train, big_data)\n", - "\n", - "measure_graph_size(train, tf.data.Dataset.from_generator(\n", - " lambda: small_data, (tf.int32, tf.int32)))\n", - "measure_graph_size(train, tf.data.Dataset.from_generator(\n", - " lambda: big_data, (tf.int32, tf.int32)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGDstsFpWHEI" - }, - "source": [ - "Python/Numpy のデータを Dataset でラップする際には、`tf.data.Dataset.from_generator` と `tf.data.Dataset.from_tensors` の違いに留意しましょう。前者はデータを Python のまま保持し `tf.py_function` を通じて取得するため、性能に影響する場合があります。これに対して後者はデータのコピーを計算グラフの中の、ひとつの大きな `tf.constant()` に結びつけるため、メモリー消費に影響する可能性があります。 \n", - "\n", - "TFRecordDataset/CsvDataset/などを通じてデータをファイルから読み込むことが、データを使用する最も効率的な方法です。TensorFlow 自身が Python とは関係なく非同期のデータ読み込みとプリフェッチを管理することができるからです。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tRdlnCfV_UTn" - }, - "source": [ - "## 自動的な依存関係の制御\n", - "\n", - "プログラミングモデルとしての関数が一般的なデータフローグラフに対して非常に優位である点は、意図したコードの振る舞いがどのようなものであるかということについて、より多くの情報をランタイムに与えられるということにあります。\n", - "\n", - "例えば、同じ変数を何度も読んだり書いたりするコードを書く場合、データフローグラフではもともと意図されていた演算の順番を自然に組み込むわけではありません。`tf.function` の中では、もともとの Python コードの文の実行順序を参照することで、実行順序の曖昧さを解消します。これにより、`tf.function` の中のステートフルな演算の順序が、先行実行モードのセマンティクスを模していることになります。\n", - "\n", - "これは、手動で制御の依存関係を加える必要がないことを意味しています。`tf.function` は十分賢いので、あなたのコードが正しく動作するために必要十分な最小限の制御の依存関係を追加してくれます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SASm0ss8erVX", - "colab": {} - }, - "source": [ - "# 自動的な依存関係の制御\n", - "\n", - "a = tf.Variable(1.0)\n", - "b = tf.Variable(2.0)\n", - "\n", - "@tf.function\n", - "def f(x, y):\n", - " a.assign(y * b)\n", - " b.assign_add(x * a)\n", - " return a + b\n", - "\n", - "f(1.0, 2.0) # 10.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lPr_6mK_AQWL" - }, - "source": [ - "## 変数\n", - "\n", - "`tf.function` の中では、意図したコードの実行順序を活用するという同じアイデアを使って、変数の作成と活用を簡単に行うことができます。しかし、ひとつだけ非常に重要な欠点があります。それは、変数を使った場合、先行実行モードとグラフモードでは動作が変わるコードを書いてしまう可能性があるということです。\n", - "\n", - "特に、呼び出しの都度新しい変数を作成する場合にこれが発生します。トレーシングの意味では、`tf.function` は呼び出しのたびに同じ変数を再利用しますが、Eager モードでは呼び出しごとに新しい変数を生成します。この間違いを防止するため、`tf.function` は危険な変数の生成動作を見つけるとエラーを発生させます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tx0Vvnb_9OB-", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def f(x):\n", - " v = tf.Variable(1.0)\n", - " v.assign_add(x)\n", - " return v\n", - "\n", - "with assert_raises(ValueError):\n", - " f(1.0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DKzNjVg8h4ao", - "colab": {} - }, - "source": [ - "# しかし、曖昧さの無いコードは大丈夫\n", - "\n", - "v = tf.Variable(1.0)\n", - "\n", - "@tf.function\n", - "def f(x):\n", - " return v.assign_add(x)\n", - "\n", - "print(f(1.0)) # 2.0\n", - "print(f(2.0)) # 4.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HQrG5_kOiKl_", - "colab": {} - }, - "source": [ - "# 初めて関数が実行されるときだけ変数が生成されることを保証できれば\n", - "# tf.function 内で変数を作成できる\n", - "\n", - "class C: pass\n", - "obj = C(); obj.v = None\n", - "\n", - "@tf.function\n", - "def g(x):\n", - " if obj.v is None:\n", - " obj.v = tf.Variable(1.0)\n", - " return obj.v.assign_add(x)\n", - "\n", - "print(g(1.0)) # 2.0\n", - "print(g(2.0)) # 4.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_IOVc1eujMH2", - "colab": {} - }, - "source": [ - "# 変数の初期化は、関数の引数や他の変数の値に依存可能\n", - "# 制御の依存関係を生成するのと同じ手法で、正しい初期化の順序を発見可能\n", - "\n", - "state = []\n", - "@tf.function\n", - "def fn(x):\n", - " if not state:\n", - " state.append(tf.Variable(2.0 * x))\n", - " state.append(tf.Variable(state[0] * 3.0))\n", - " return state[0] * x * state[1]\n", - "\n", - "print(fn(tf.constant(1.0)))\n", - "print(fn(tf.constant(3.0)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5f05Vr_YBUCz" - }, - "source": [ - "# AutoGraph の使用\n", - "\n", - "[autograph](https://www.tensorflow.org/guide/function) ライブラリは `tf.function` に完全に統合されており、計算グラフの中で動的に実行される条件文や繰り返しを書くことができます。\n", - "\n", - "`tf.cond` や `tf.while_loop` は `tf.function` でも使えますが、制御フローを含むコードは、命令形式で書いたほうが書きやすいし理解しやすいです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yCQTtTPTW3WF", - "colab": {} - }, - "source": [ - "# 単純な繰り返し\n", - "\n", - "@tf.function\n", - "def f(x):\n", - " while tf.reduce_sum(x) > 1:\n", - " tf.print(x)\n", - " x = tf.tanh(x)\n", - " return x\n", - "\n", - "f(tf.random.uniform([5]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jlQD1ffRXJhl", - "colab": {} - }, - "source": [ - "# 興味があれば AutoGraph が生成するコードを調べることができる\n", - "# ただし、アセンブリ言語を読むような感じがする\n", - "\n", - "def f(x):\n", - " while tf.reduce_sum(x) > 1:\n", - " tf.print(x)\n", - " x = tf.tanh(x)\n", - " return x\n", - "\n", - "print(tf.autograph.to_code(f))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xgKmkrNTZSyz" - }, - "source": [ - "## AutoGraph: 条件分岐\n", - "\n", - "AutoGraph は `if` 文を等価である `tf.cond` の呼び出しに変換します。\n", - "\n", - "この置換は条件がテンソルである場合に行われます。そうでない場合には、条件分岐はトレーシングの中で実行されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "E-7KllizZYsy", - "colab": {} - }, - "source": [ - "def test_tf_cond(f, *args):\n", - " g = f.get_concrete_function(*args).graph\n", - " if any(node.name == 'cond' for node in g.as_graph_def().node):\n", - " print(\"{}({}) uses tf.cond.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n", - " else:\n", - " print(\"{}({}) executes normally.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o86paGR-Zadi", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def hyperparam_cond(x, training=True):\n", - " if training:\n", - " x = tf.nn.dropout(x, rate=0.5)\n", - " return x\n", - "\n", - "@tf.function\n", - "def maybe_tensor_cond(x):\n", - " if x < 0:\n", - " x = -x\n", - " return x\n", - "\n", - "test_tf_cond(hyperparam_cond, tf.ones([1], dtype=tf.float32))\n", - "test_tf_cond(maybe_tensor_cond, tf.constant(-1))\n", - "test_tf_cond(maybe_tensor_cond, -1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5xFLfdApZh8q" - }, - "source": [ - "`tf.cond` には、色々と注意すべき細かな点があります。\n", - "\n", - "- `tf.cond` は条件分岐の両方をトレーシングし、条件に従って実行時に適切な分岐を選択することで機能します。分岐の両方をトレースすることで、Python プログラムを予期せず実行する可能性があります。\n", - "- `tf.cond` では、分岐の一方が後ほど使用されるテンソルを作成する場合、もう一方の分岐もそのテンソルを作成することが必要です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VTMoZEVaZiwk", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def f():\n", - " x = tf.constant(0)\n", - " if tf.constant(True):\n", - " x = x + 1\n", - " print(\"Tracing `then` branch\")\n", - " else:\n", - " x = x - 1\n", - " print(\"Tracing `else` branch\")\n", - " return x\n", - "\n", - "f()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "k_dxWHeFZlaQ", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def f():\n", - " if tf.constant(True):\n", - " x = tf.ones([3, 3])\n", - " return x\n", - "\n", - "# 分岐のどちらの枝でも `x` を定義する必要があるためエラーが発生\n", - "with assert_raises(ValueError):\n", - " f()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yho4J0a0ZkQS" - }, - "source": [ - "## AutoGraph と繰り返し\n", - "\n", - "AutoGraph には繰り返しの変換にいくつかの単純なルールがあります。\n", - "\n", - "- `for`: 反復可能オブジェクトがテンソルである場合に変換する\n", - "- `while`: while 条件がテンソルに依存している場合に変換する\n", - "\n", - "繰り返しが変換される場合、`tf.while_loop` によって動的に展開されます。あるいは、 `for x in tf.data.Dataset` という特別なケースの場合には、 `tf.data.Dataset.reduce` に変換されます。\n", - "\n", - "繰り返しが変換されない場合、それは静的に展開されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OyzGNQAuZsky", - "colab": {} - }, - "source": [ - "def test_dynamically_unrolled(f, *args):\n", - " g = f.get_concrete_function(*args).graph\n", - " if any(node.name == 'while' for node in g.as_graph_def().node):\n", - " print(\"{}({}) uses tf.while_loop.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n", - " elif any(node.name == 'ReduceDataset' for node in g.as_graph_def().node):\n", - " print(\"{}({}) uses tf.data.Dataset.reduce.\".format(\n", - " f.__name__, ', '.join(map(str, args))))\n", - " else:\n", - " print(\"{}({}) gets unrolled.\".format(\n", - " f.__name__, ', '.join(map(str, args))))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Q7tmncQTZt6_", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def for_in_range():\n", - " x = 0\n", - " for i in range(5):\n", - " x += i\n", - " return x\n", - "\n", - "test_dynamically_unrolled(for_in_range)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "H-SM-c-qTuoX", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def for_in_tfrange():\n", - " x = tf.constant(0, dtype=tf.int32)\n", - " for i in tf.range(5):\n", - " x += i\n", - " return x\n", - "\n", - "test_dynamically_unrolled(for_in_tfrange)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "abXQ2iwBTuoZ", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def for_in_tfdataset():\n", - " x = tf.constant(0, dtype=tf.int64)\n", - " for i in tf.data.Dataset.range(5):\n", - " x += i\n", - " return x\n", - "\n", - "test_dynamically_unrolled(for_in_tfdataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "l6s7aU-padY5", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def while_py_cond():\n", - " x = 5\n", - " while x > 0:\n", - " x -= 1\n", - " return x\n", - "\n", - "test_dynamically_unrolled(while_py_cond)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QCLFaZnxTuoc", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def while_tf_cond():\n", - " x = tf.constant(5)\n", - " while x > 0:\n", - " x -= 1\n", - " return x\n", - "\n", - "test_dynamically_unrolled(while_tf_cond)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dSr64Xn6ap-S" - }, - "source": [ - " 繰り返しに、テンソルに依存する `break` や、途中での `return` がある場合、一番外側の条件あるいは反復可能オブジェクトはテンソルである必要があります。 \n", - " \n", - " 比較してみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "s2eaWCe2Tuof", - "colab_type": "code", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def while_py_true_py_break(x):\n", - " while True: # py true\n", - " if x == 0: # py break\n", - " break\n", - " x -= 1\n", - " return x\n", - "\n", - "test_dynamically_unrolled(while_py_true_py_break, 5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "Kx2Z0K_uTuog", - "colab_type": "code", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def buggy_while_py_true_tf_break(x):\n", - " while True: # py true\n", - " if tf.equal(x, 0): # tf break\n", - " break\n", - " x -= 1\n", - " return x\n", - "\n", - "with assert_raises(TypeError):\n", - " test_dynamically_unrolled(buggy_while_py_true_tf_break, 5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "-ozYU9zQTuoj", - "colab_type": "code", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def while_tf_true_tf_break(x):\n", - " while tf.constant(True): # tf true\n", - " if x == 0: # py break\n", - " break\n", - " x -= 1\n", - " return x\n", - "\n", - "test_dynamically_unrolled(while_tf_true_tf_break, 5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "bZSpBq5CTuol", - "colab_type": "code", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def buggy_py_for_tf_break():\n", - " x = 0\n", - " for i in range(5): # py for\n", - " if tf.equal(i, 3): # tf break\n", - " break\n", - " x += i\n", - " return x\n", - "\n", - "with assert_raises(TypeError):\n", - " test_dynamically_unrolled(buggy_py_for_tf_break)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "DT9zyKTwTuon", - "colab_type": "code", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def tf_for_py_break():\n", - " x = 0\n", - " for i in tf.range(5): # tf for\n", - " if i == 3: # py break\n", - " break\n", - " x += i\n", - " return x\n", - "\n", - "test_dynamically_unrolled(tf_for_py_break)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hyksHW9TCukR" - }, - "source": [ - "動的に展開される繰り返しの結果を集計するため、`tf.TensorArray` を使いたくなるかもしれません。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HJ3Vb3dXfefN", - "colab": {} - }, - "source": [ - "batch_size = 2\n", - "seq_len = 3\n", - "feature_size = 4\n", - "\n", - "def rnn_step(inp, state):\n", - " return inp + state\n", - "\n", - "@tf.function\n", - "def dynamic_rnn(rnn_step, input_data, initial_state):\n", - " # [batch, time, features] -> [time, batch, features]\n", - " input_data = tf.transpose(input_data, [1, 0, 2])\n", - " max_seq_len = input_data.shape[0]\n", - "\n", - " states = tf.TensorArray(tf.float32, size=max_seq_len)\n", - " state = initial_state\n", - " for i in tf.range(max_seq_len):\n", - " state = rnn_step(input_data[i], state)\n", - " states = states.write(i, state)\n", - " return tf.transpose(states.stack(), [1, 0, 2])\n", - " \n", - "dynamic_rnn(rnn_step,\n", - " tf.random.uniform([batch_size, seq_len, feature_size]),\n", - " tf.zeros([batch_size, feature_size]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9gmLpHY-bkly" - }, - "source": [ - "`tf.cond` と同様に、`tf.while_loop` にも、色々と注意すべき細かな点があります。\n", - "\n", - "- 繰り返しの実行回数が 0 である可能性があるため、while_loop の後で使用されるテンソルは、繰り返しの前に初期化されなければならない\n", - "- すべての繰り返しの変数は、各繰り返しを通じてその形状と dtype が変わらないことが必要" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CocT5RHwblrQ", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def buggy_loop_var_uninitialized():\n", - " for i in tf.range(3):\n", - " x = i\n", - " return x\n", - "\n", - "with assert_raises(ValueError):\n", - " buggy_loop_var_uninitialized()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BnfgidIhTuow", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def f():\n", - " x = tf.constant(0)\n", - " for i in tf.range(3):\n", - " x = i\n", - " return x\n", - "\n", - "f()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "L_0qnF58Tuoy", - "colab_type": "code", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def buggy_loop_type_changes():\n", - " x = tf.constant(0, dtype=tf.float32)\n", - " for i in tf.range(3): # tf.int32 型のテンソルを1つづつ取り出して…\n", - " x = i\n", - " return x\n", - "\n", - "with assert_raises(tf.errors.InvalidArgumentError):\n", - " buggy_loop_type_changes()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kWF189prbuK0", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def buggy_concat():\n", - " x = tf.ones([0, 10])\n", - " for i in tf.range(5):\n", - " x = tf.concat([x, tf.ones([1, 10])], axis=0)\n", - " return x\n", - "\n", - "with assert_raises(ValueError):\n", - " buggy_concat()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VO6QXZGlTuo4", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def concat_with_padding():\n", - " x = tf.zeros([5, 10])\n", - " for i in tf.range(5):\n", - " x = tf.concat([x[:i], tf.ones([1, 10]), tf.zeros([4-i, 10])], axis=0)\n", - " x.set_shape([5, 10])\n", - " return x\n", - "\n", - "concat_with_padding()" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/images/classification.ipynb b/site/ja/tutorials/images/classification.ipynb deleted file mode 100644 index 05825daf389..00000000000 --- a/site/ja/tutorials/images/classification.ipynb +++ /dev/null @@ -1,1366 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "classification.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TBFXQGKYUc4X" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "1z4xy2gTUc4a", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FE7KNzPPVrVV" - }, - "source": [ - "# Image classification" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KwQtSOz0VrVX" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DjmULc9dAvL-", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gN7G9GFmVrVY" - }, - "source": [ - "このチュートリアルでは、画像から猫または犬を分類する方法を示します。 `tf.keras.Sequential` モデルを使用して画像分類器を構築し、 `tf.keras.preprocessing.image.ImageDataGenerator` を使用してデータをロードします。このチュートリアルでは、以下のコンセプトにしたがって、実践的な経験と感覚を養います。\n", - "\n", - "* `tf.keras.preprocessing.image.ImageDataGenerator` クラスを使用して _データ入力パイプライン_ を構築し、モデルで使用するディスク上のデータを効率的に処理します。\n", - "* _過学習（Overfitting）_ —過学習を識別および防止する方法。\n", - "* _データ拡張（Data Augmentation）_ および _ドロップアウト（dropout）_ —データパイプラインおよび画像分類モデルに組み込むコンピュータービジョンタスクの過学習と戦うための重要なテクニック。\n", - "\n", - "このチュートリアルは、基本的な機械学習のワークフローに従います。\n", - "\n", - "1. データの調査及び理解\n", - "2. 入力パイプラインの構築\n", - "3. モデルの構築\n", - "4. モデルの学習\n", - "5. モデルのテスト\n", - "6. モデルの改善とプロセスの繰り返し" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zF9uvbXNVrVY" - }, - "source": [ - "## パッケージのインポート" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VddxeYBEVrVZ" - }, - "source": [ - "まずは必要なパッケージをインポートすることから始めましょう。 `os`パッケージはファイルとディレクトリ構造を読み込み、 NumPy は python リストの numpy 配列への変換と必要な行列演算の実行、 `matplotlib.pyplot` はグラフの描画や学習データおよび検証データに含まれる画像の表示、に利用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rtPGh2MAVrVa", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Jlchl4x2VrVg" - }, - "source": [ - "モデルの構築に必要な TensorFlow と Keras クラスをインポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "E82grprdYPI0", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version は Colab にのみ存在します。\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "L1WtoaOHVrVh", - "colab": {} - }, - "source": [ - "from tensorflow.keras.models import Sequential\n", - "from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPooling2D\n", - "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", - "\n", - "import os\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UZZI6lNkVrVm" - }, - "source": [ - "## データの読み込み" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DPHx8-t-VrVo" - }, - "source": [ - "データセットのダウンロードから始めます。このチュートリアルでは、 Kaggle の Dogs vs Cats データセットをフィルタリングしたバージョンを使用します。データセットのアーカイブバージョンをダウンロードし、\"/tmp/\"ディレクトリに保存します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "C1nqr-CYY6uw", - "colab": {} - }, - "source": [ - "_URL = 'https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip'\n", - "\n", - "path_to_zip = tf.keras.utils.get_file('cats_and_dogs.zip', origin=_URL, extract=True)\n", - "\n", - "PATH = os.path.join(os.path.dirname(path_to_zip), 'cats_and_dogs_filtered')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Giv0wMQzVrVw" - }, - "source": [ - "データセットのディレクトリ構造は次のとおりです:\n", - "\n", - "
    \n",
-        "cats_and_dogs_filtered\n",
-        "|__ train\n",
-        "    |______ cats: [cat.0.jpg, cat.1.jpg, cat.2.jpg ....]\n",
-        "    |______ dogs: [dog.0.jpg, dog.1.jpg, dog.2.jpg ...]\n",
-        "|__ validation\n",
-        "    |______ cats: [cat.2000.jpg, cat.2001.jpg, cat.2002.jpg ....]\n",
-        "    |______ dogs: [dog.2000.jpg, dog.2001.jpg, dog.2002.jpg ...]\n",
-        "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VpmywIlsVrVx" - }, - "source": [ - "データの内容を抽出した後、学習および検証セットのための適切なファイルパスで変数を設定します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sRucI3QqVrVy", - "colab": {} - }, - "source": [ - "train_dir = os.path.join(PATH, 'train')\n", - "validation_dir = os.path.join(PATH, 'validation')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Utv3nryxVrV0", - "colab": {} - }, - "source": [ - "train_cats_dir = os.path.join(train_dir, 'cats') # 学習用の猫画像のディレクトリ\n", - "train_dogs_dir = os.path.join(train_dir, 'dogs') # 学習用の犬画像のディレクトリ\n", - "validation_cats_dir = os.path.join(validation_dir, 'cats') # 検証用の猫画像のディレクトリ\n", - "validation_dogs_dir = os.path.join(validation_dir, 'dogs') # 検証用の犬画像のディレクトリ" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZdrHHTy2VrV3" - }, - "source": [ - "### データの理解" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LblUYjl-VrV3" - }, - "source": [ - "学習および検証ディレクトリの中にある猫と犬の画像の数を見てみましょう:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vc4u8e9hVrV4", - "colab": {} - }, - "source": [ - "num_cats_tr = len(os.listdir(train_cats_dir))\n", - "num_dogs_tr = len(os.listdir(train_dogs_dir))\n", - "\n", - "num_cats_val = len(os.listdir(validation_cats_dir))\n", - "num_dogs_val = len(os.listdir(validation_dogs_dir))\n", - "\n", - "total_train = num_cats_tr + num_dogs_tr\n", - "total_val = num_cats_val + num_dogs_val" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "g4GGzGt0VrV7", - "colab": {} - }, - "source": [ - "print('total training cat images:', num_cats_tr)\n", - "print('total training dog images:', num_dogs_tr)\n", - "\n", - "print('total validation cat images:', num_cats_val)\n", - "print('total validation dog images:', num_dogs_val)\n", - "print(\"--\")\n", - "print(\"Total training images:\", total_train)\n", - "print(\"Total validation images:\", total_val)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8Lp-0ejxOtP1" - }, - "source": [ - "便宜上、データセットの前処理およびネットワークの学習中に使用する変数を設定します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3NqNselLVrWA", - "colab": {} - }, - "source": [ - "batch_size = 128\n", - "epochs = 15\n", - "IMG_HEIGHT = 150\n", - "IMG_WIDTH = 150" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "INn-cOn1VrWC" - }, - "source": [ - "## データの準備" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5Jfk6aSAVrWD" - }, - "source": [ - "モデルにデータを送る前に、画像を適切に前処理された浮動小数点テンソルにフォーマットします。\n", - "\n", - "1.ディスクから画像を読み取ります。\n", - "2.これらの画像のコンテンツをデコードし、RGB値にしたがって適切なグリッド形式に変換します。\n", - "3.それらを浮動小数点テンソルに変換します。\n", - "4.ニューラルネットワークは小さな入力値を扱う方が適しているため、テンソルを0〜255の値から0〜1の値にリスケーリングします。\n", - "\n", - "幸い、これらすべてのタスクは、 `tf.keras` によって提供される `ImageDataGenerator` クラスで実行できます。この `ImageDataGenerator` はディスクから画像を読み取り、適切なテンソルに前処理を行います。さらに、これらの画像をテンソルのバッチに変換するジェネレータをセットアップします。これは、ネットワーク学習時に便利です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "syDdF_LWVrWE", - "colab": {} - }, - "source": [ - "train_image_generator = ImageDataGenerator(rescale=1./255) # 学習データのジェネレータ\n", - "validation_image_generator = ImageDataGenerator(rescale=1./255) # 検証データのジェネレータ" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RLciCR_FVrWH" - }, - "source": [ - "学習および検証画像のジェネレータを定義したのち、 `flow_from_directory` メソッドはディスクから画像をロードし、リスケーリングを適用し、画像を必要な大きさにリサイズします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Pw94ajOOVrWI", - "colab": {} - }, - "source": [ - "train_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2oUoKUzRVrWM", - "colab": {} - }, - "source": [ - "val_data_gen = validation_image_generator.flow_from_directory(batch_size=batch_size,\n", - " directory=validation_dir,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hyexPJ8CVrWP" - }, - "source": [ - "### 学習用画像の可視化" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "60CnhEL4VrWQ" - }, - "source": [ - "学習用のジェネレータから画像バッチを抽出して可視化します。（この例では32個の画像を抽出し、そのうち5つを `matplotlib` で描画します。）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3f0Z7NZgVrWQ", - "colab": {} - }, - "source": [ - "sample_training_images, _ = next(train_data_gen)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "49weMt5YVrWT" - }, - "source": [ - " `next` 関数はデータセットからバッチを返します。 `next` 関数の返り値は `（x_train、y_train）` の形式で、 `x_train` は学習用の特徴量、 `y_train` はそのラベルです。ラベルを破棄して、学習用画像の可視化のみを行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JMt2RES_VrWU", - "colab": {} - }, - "source": [ - "# この関数は、1行5列のグリッド形式で画像をプロットし、画像は各列に配置されます。\n", - "def plotImages(images_arr):\n", - " fig, axes = plt.subplots(1, 5, figsize=(20,20))\n", - " axes = axes.flatten()\n", - " for img, ax in zip( images_arr, axes):\n", - " ax.imshow(img)\n", - " ax.axis('off')\n", - " plt.tight_layout()\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "d_VVg_gEVrWW", - "colab": {} - }, - "source": [ - "plotImages(sample_training_images[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b5Ej-HLGVrWZ" - }, - "source": [ - "## モデルの構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wEgW4i18VrWZ" - }, - "source": [ - "モデルはmax pooling層を伴う3つの畳み込みブロックからなります。さらに `relu` 活性化関数によるアクティベーションを伴う512ユニットの全結合層があります。モデルは、シグモイド活性化関数による2値分類に基づいてクラスに属する確率を出力します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F15-uwLPVrWa", - "colab": {} - }, - "source": [ - "model = Sequential([\n", - " Conv2D(16, 3, padding='same', activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3)),\n", - " MaxPooling2D(),\n", - " Conv2D(32, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Conv2D(64, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Flatten(),\n", - " Dense(512, activation='relu'),\n", - " Dense(1, activation='sigmoid')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PI5cdkMQVrWc" - }, - "source": [ - "### モデルのコンパイル\n", - "このチュートリアルでは、 *ADAM* オプティマイザーと *binary cross entropy* 損失関数を選択します。各学習エポックの学習と検証の精度を表示するために、`metrics` 引数を渡します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Mg7_TXOVrWd", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2YmQZ3TAVrWg" - }, - "source": [ - "### モデルの概要\n", - "\n", - "すべてのネットワークのレイヤーを見るには、モデルの `summary` メソッドを利用します:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Vtny8hmBVrWh", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N06iqE8VVrWj" - }, - "source": [ - "### モデルの学習" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oub9RtoFVrWk" - }, - "source": [ - "`ImageDataGenerator` クラスの `fit_generator` メソッドを使用して、ネットワークを学習します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KSF2HqhDVrWk", - "colab": {} - }, - "source": [ - "history = model.fit_generator(\n", - " train_data_gen,\n", - " steps_per_epoch=total_train // batch_size,\n", - " epochs=epochs,\n", - " validation_data=val_data_gen,\n", - " validation_steps=total_val // batch_size\n", - ")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ojJNteAGVrWo" - }, - "source": [ - "### 学習結果の可視化" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LZPYT-EmVrWo" - }, - "source": [ - "ネットワークを学習した後、結果を可視化します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K6oA77ADVrWp", - "colab": {} - }, - "source": [ - "acc = history.history['accuracy']\n", - "val_acc = history.history['val_accuracy']\n", - "\n", - "loss = history.history['loss']\n", - "val_loss = history.history['val_loss']\n", - "\n", - "epochs_range = range(epochs)\n", - "\n", - "plt.figure(figsize=(8, 8))\n", - "plt.subplot(1, 2, 1)\n", - "plt.plot(epochs_range, acc, label='Training Accuracy')\n", - "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", - "plt.legend(loc='lower right')\n", - "plt.title('Training and Validation Accuracy')\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.plot(epochs_range, loss, label='Training Loss')\n", - "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", - "plt.legend(loc='upper right')\n", - "plt.title('Training and Validation Loss')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kDnr50l2VrWu" - }, - "source": [ - "プロットからわかるように、学習セットの精度と検証セットの精度は大幅に外れており、モデルは検証セットで約70％の精度しか達成していません。\n", - "\n", - "何がうまくいかなかったかを見て、モデル全体のパフォーマンスを向上してみましょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rLO7yhLlVrWu" - }, - "source": [ - "## 過学習" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hNyx3Lp4VrWv" - }, - "source": [ - "上記のプロットでは、学習セットの精度は時間とともに直線的に向上していますが、検証セットの精度は学習プロセスの中で約70％あたりで頭打ちになっています。そして、学習と検証の精度の違いが顕著です。これは *過学習* のサインです。\n", - "\n", - "学習サンプルが少ない場合、モデルは学習サンプルに含まれるノイズや不要な詳細から学習してしまい、これによって新しいサンプルに対するモデルの性能に悪影響を与えることがあります。この現象は、過学習として知られています。過学習とは、モデルが新しいデータセットに対して汎化するのが難しい状態をいいます。\n", - "\n", - "学習プロセスにおいて過学習に対抗する手段はいくつかあります。このチュートリアルでは、*データ拡張（data Augmentation）* を使用し、さらにモデルに *ドロップアウト（dropout）* を追加します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UOoVpxFwVrWy" - }, - "source": [ - "## データ拡張（Data augmentation）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wn_QLciWVrWy" - }, - "source": [ - "過学習は一般に、学習サンプルが少ない場合に発生します。この問題を解決する方法の1つは、十分な数の学習サンプルが含まれるようにデータセットを拡張することです。データ拡張は、既存の学習サンプルに対してランダムな変換を行い、データセットとして利用できそうな画像を生成するアプローチをとります。このデータ拡張の目的は、学習中にモデルがまったくおなじ画像を2回利用しないようにすることです。これによってモデルをデータのより多くの特徴を利用し、より汎化することができます。\n", - "\n", - "`tf.keras` においては、このデータ拡張を `ImageDataGenerator` クラスを使用して実装します。データセットに対するさまざまな変換を指定することによって、学習プロセス中にそれが適用されます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2uJ1G030VrWz" - }, - "source": [ - "### データの拡張と可視化" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hvX7hHlgVrW0" - }, - "source": [ - "最初に、ランダムな水平反転による拡張をデータセットに適用し、それぞれの画像が変換後にどのように見えるかを確認します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlVj6VqaVrW0" - }, - "source": [ - "### 水平反転の適用" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xcdvx4TVVrW1" - }, - "source": [ - " このデータ拡張を適用するためには、 `ImageDataGenerator` クラスの引数として `horizontal_flip` を渡し、 `True`を設定します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Bi1_vHyBVrW2", - "colab": {} - }, - "source": [ - "image_gen = ImageDataGenerator(rescale=1./255, horizontal_flip=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zvwqmefgVrW3", - "colab": {} - }, - "source": [ - "train_data_gen = image_gen.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zJpRSxJ-VrW7" - }, - "source": [ - "学習サンプルから1つのサンプル画像を取得する作業を5回繰り返して、おなじ画像に5回データ拡張が適用されるようにします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RrKGd_jjVrW7", - "colab": {} - }, - "source": [ - "augmented_images = [train_data_gen[0][0][0] for i in range(5)]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EvBZoQ9xVrW9", - "colab": {} - }, - "source": [ - "# 上で学習用画像の可視化のために定義、使用されたおなじカスタムプロット関数を再利用する\n", - "plotImages(augmented_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "i7n9xcqCVrXB" - }, - "source": [ - "### 画像のランダムな回転" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qXnwkzFuVrXB" - }, - "source": [ - "回転のデータ拡張を利用して学習用サンプルをランダムに左右45度の範囲で回転させてみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1zip35pDVrXB", - "colab": {} - }, - "source": [ - "image_gen = ImageDataGenerator(rescale=1./255, rotation_range=45)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kVoWh4OIVrXD", - "colab": {} - }, - "source": [ - "train_data_gen = image_gen.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH))\n", - "\n", - "augmented_images = [train_data_gen[0][0][0] for i in range(5)]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wmBx8NhrVrXK", - "colab": {} - }, - "source": [ - "plotImages(augmented_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FOqGPL76VrXM" - }, - "source": [ - "### ズームによるデータ拡張の適用" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NvqXaD8BVrXN" - }, - "source": [ - "データセットにズームによるデータ拡張を適用して、画像をランダムに最大50％拡大します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tGNKLa_YVrXR", - "colab": {} - }, - "source": [ - "image_gen = ImageDataGenerator(rescale=1./255, zoom_range=0.5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VOvTs32FVrXU", - "colab": {} - }, - "source": [ - "train_data_gen = image_gen.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH))\n", - "\n", - "augmented_images = [train_data_gen[0][0][0] for i in range(5)]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-KQWw8IZVrXZ", - "colab": {} - }, - "source": [ - "plotImages(augmented_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "usS13KCNVrXd" - }, - "source": [ - "### すべてのデータ拡張を同時に利用する" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OC8fIsalVrXd" - }, - "source": [ - "ここまでで紹介したすべてのデータ拡張機能を適用します。ここでは、学習用画像に対して、リスケール、45度の回転、幅シフト、高さシフト、水平反転、ズームを適用しました。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gnr2xujaVrXe", - "colab": {} - }, - "source": [ - "image_gen_train = ImageDataGenerator(\n", - " rescale=1./255,\n", - " rotation_range=45,\n", - " width_shift_range=.15,\n", - " height_shift_range=.15,\n", - " horizontal_flip=True,\n", - " zoom_range=0.5\n", - " )" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K0Efxy7EVrXh", - "colab": {} - }, - "source": [ - "train_data_gen = image_gen_train.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AW-pV5awVrXl" - }, - "source": [ - "これらのデータ拡張がデータセットにランダムに適用されたときに、一つの画像に対して5回の個別の適用を行った際にそれぞれどのように見えるかを可視化します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "z2m68eMhVrXm", - "colab": {} - }, - "source": [ - "augmented_images = [train_data_gen[0][0][0] for i in range(5)]\n", - "plotImages(augmented_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J8cUd7FXVrXq" - }, - "source": [ - "### 検証データジェネレータの構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "a99fDBt7VrXr" - }, - "source": [ - "一般に、データ拡張は学習サンプルのみに適用します。今回は、 `ImageDataGenerator` を使用して検証画像に対してリスケールのみを実施し、バッチに変換します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "54x0aNbKVrXr", - "colab": {} - }, - "source": [ - "image_gen_val = ImageDataGenerator(rescale=1./255)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1PCHKzI8VrXv", - "colab": {} - }, - "source": [ - "val_data_gen = image_gen_val.flow_from_directory(batch_size=batch_size,\n", - " directory=validation_dir,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yQGhdqHFVrXx" - }, - "source": [ - "## ドロップアウト（dropout）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2Iq5TAH_VrXx" - }, - "source": [ - "過学習を避けるもう一つの方法は、ネットワークに *ドロップアウト* を導入することです。これは、ネットワークにおいて重みを小さくする正則化の方式で、これによって重みの値の分布がより規則的になり、少ない学習データに対する過学習を減らすことができます。ドロップアウトはこのチュートリアルで利用される正則化手法の一つです。\n", - "\n", - "ドロップアウトをレイヤーに適用すると、学習プロセス中に適用されたレイヤーのうちランダムに出力ユニットをドロップアウト（ゼロに設定）します。ドロップアウトは、入力値として0.1、0.2、0.4といった形式の小数をとります。これは、適用されたレイヤーからランダムに出力単位の10％、20％、または40％をドロップアウトすることを意味します。\n", - "\n", - "特定のレイヤーに0.1ドロップアウトを適用すると、各学習エポックにおいて出力ユニットの10％がランダムに0にされます。\n", - "\n", - "この新しいドロップアウト機能を使用したネットワークアーキテクチャを作成し、異なる畳み込みレイヤーや全接続レイヤーに適用してみましょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DyxxXRmVVrXy" - }, - "source": [ - "## ドロップアウトを追加した新しいネットワークの構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1Ba2LjtkVrXy" - }, - "source": [ - "ここでは、ドロップアウトを最初と最後の max pool 層に適用します。ドロップアウトを適用すると、各学習エポック中にニューロンの20％がランダムにゼロに設定されます。これにより、学習データセットに対する過学習を避けることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2fjio8EsVrXz", - "colab": {} - }, - "source": [ - "model_new = Sequential([\n", - " Conv2D(16, 3, padding='same', activation='relu', \n", - " input_shape=(IMG_HEIGHT, IMG_WIDTH ,3)),\n", - " MaxPooling2D(),\n", - " Dropout(0.2),\n", - " Conv2D(32, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Conv2D(64, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Dropout(0.2),\n", - " Flatten(),\n", - " Dense(512, activation='relu'),\n", - " Dense(1, activation='sigmoid')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tpTgIxWAVrX0" - }, - "source": [ - "### モデルのコンパイル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1osvc_iTVrX1" - }, - "source": [ - "ネットワークにドロップアウトを導入した後、モデルをコンパイルし、レイヤーの概要を表示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OkIJhS-WVrX1", - "colab": {} - }, - "source": [ - "model_new.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "model_new.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7KiDshEUVrX6" - }, - "source": [ - "### モデルの学習" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NFj0oVqVVrX6" - }, - "source": [ - "学習サンプルにデータ拡張を導入し、ネットワークにドロップアウトを追加した後、この新しいネットワークを学習します:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GWxHs_luVrX7", - "colab": {} - }, - "source": [ - "history = model_new.fit_generator(\n", - " train_data_gen,\n", - " steps_per_epoch=total_train // batch_size,\n", - " epochs=epochs,\n", - " validation_data=val_data_gen,\n", - " validation_steps=total_val // batch_size\n", - ")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bbdyqZdxVrYA" - }, - "source": [ - "### モデルの可視化" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OgvF2nt7OtR7" - }, - "source": [ - "学習後に新しいモデルを可視化すると、過学習が前回よりも大幅に少ないことがわかります。より多くのエポックでモデルを学習すると、精度はさらに向上するはずです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7BTeMuNAVrYC", - "colab": {} - }, - "source": [ - "acc = history.history['accuracy']\n", - "val_acc = history.history['val_accuracy']\n", - "\n", - "loss = history.history['loss']\n", - "val_loss = history.history['val_loss']\n", - "\n", - "epochs_range = range(epochs)\n", - "\n", - "plt.figure(figsize=(8, 8))\n", - "plt.subplot(1, 2, 1)\n", - "plt.plot(epochs_range, acc, label='Training Accuracy')\n", - "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", - "plt.legend(loc='lower right')\n", - "plt.title('Training and Validation Accuracy')\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.plot(epochs_range, loss, label='Training Loss')\n", - "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", - "plt.legend(loc='upper right')\n", - "plt.title('Training and Validation Loss')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/images/cnn.ipynb b/site/ja/tutorials/images/cnn.ipynb deleted file mode 100644 index 82ec8ef901f..00000000000 --- a/site/ja/tutorials/images/cnn.ipynb +++ /dev/null @@ -1,366 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "cnn.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "x4HI2mpwlrcn", - "colab_type": "text" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "679Lmwt3l1Bk", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DSPCom-KmApV", - "colab_type": "text" - }, - "source": [ - "# 畳み込みニューラルネットワーク (Convolutional Neural Networks)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "klAltGp8ycek" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " View on TensorFlow.org\n", - " \n", - " \n", - " \n", - " Run in Google Colab\n", - " \n", - " \n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qLGkt5qiyz4E" - }, - "source": [ - "このチュートリアルでは、MNIST の数の分類をするための、シンプルな[畳み込みニューラルネットワーク](https://developers.google.com/machine-learning/glossary/#convolutional_neural_network) (CNN: Convolutional Neural Network) の学習について説明します。このシンプルなネットワークは MNIST テストセットにおいて、99%以上の精度を達成します。このチュートリアルでは、[Keras Sequential API](https://www.tensorflow.org/guide/keras)を使用するため、ほんの数行のコードでモデルの作成と学習を行うことができます。\n", - "\n", - "Note: GPU を使うことで CNN をより早く学習させることができます。もし、このノートブックを Colab で実行しているならば、*編集 -> ノートブックの設定 -> ハードウェアアクセラレータ -> GPU* から無料のGPUを有効にすることができます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m7KBpffWzlxH" - }, - "source": [ - "### TensorFlowのインポート" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iAve6DCL4JH4", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras import datasets, layers, models" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jRFxccghyMVo" - }, - "source": [ - "### MNISTデータセットのダウンロードと準備" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JWoEqyMuXFF4", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (test_images, test_labels) = datasets.mnist.load_data()\n", - "\n", - "train_images = train_images.reshape((60000, 28, 28, 1))\n", - "test_images = test_images.reshape((10000, 28, 28, 1))\n", - "\n", - "# ピクセルの値を 0~1 の間に正規化\n", - "train_images, test_images = train_images / 255.0, test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oewp-wYg31t9" - }, - "source": [ - "### 畳み込みの基礎部分の作成\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3hQvqXpNyN3x" - }, - "source": [ - "下記の6行のコードは、一般的なパターンで畳み込みの基礎部分を定義しています: [Conv2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2D) と [MaxPooling2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/MaxPool2D) レイヤーのスタック。\n", - "\n", - "入力として、CNN はバッチサイズを無視して、shape (image_height, image_width, color_channels) のテンソルをとります。color channels について、MNIST は1つ (画像がグレースケールのため) の color channels がありますが、カラー画像には3つ (R, G, B) があります。この例では、MNIST 画像のフォーマットである shape (28, 28, 1) の入力を処理するように CNN を構成します。これを行うには、引数 `input_shape` を最初のレイヤーに渡します。\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "L9YmGQBQPrdn", - "colab": {} - }, - "source": [ - "model = models.Sequential()\n", - "model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n", - "model.add(layers.MaxPooling2D((2, 2)))\n", - "model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n", - "model.add(layers.MaxPooling2D((2, 2)))\n", - "model.add(layers.Conv2D(64, (3, 3), activation='relu'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lvDVFkg-2DPm" - }, - "source": [ - "ここまでのモデルのアーキテクチャを表示してみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8-C4XBg4UTJy", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_j-AXYeZ2GO5" - }, - "source": [ - "上記より、すべての Conv2D と MaxPooling2D レイヤーの出力は shape (height, width, channels) の 3D テンソルであることがわかります。width と height の寸法は、ネットワークが深くなるにつれて縮小する傾向があります。各 Conv2D レイヤーの出力チャネルの数は、第一引数 (例: 32 または 64) によって制御されます。通常、width とheight が縮小すると、各 Conv2D レイヤーにさらに出力チャネルを追加する余裕が (計算上) できます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_v8sVOtG37bT" - }, - "source": [ - "### 上に Dense レイヤーを追加\n", - "モデルを完成するために、(shape (3, 3, 64) の) 畳み込みの基礎部分からの最後の出力テンソルを、1つ以上の Dense レイヤーに入れて分類を実行します。現在の出力は 3D テンソルですが、Dense レイヤーは入力としてベクトル (1D) を取ります。まず、3D 出力を 1D に平滑化 (または展開) してから、最上部に1つ以上の Dense レイヤーを追加します。MNIST は 10 個の出力クラスを持ちます。そのため、我々は最後の Dense レイヤーの出力を 10 にし、softmax関数を使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mRs95d6LUVEi", - "colab": {} - }, - "source": [ - "model.add(layers.Flatten())\n", - "model.add(layers.Dense(64, activation='relu'))\n", - "model.add(layers.Dense(10, activation='softmax'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ipGiQMcR4Gtq" - }, - "source": [ - "これが私たちのモデルの完全なアーキテクチャです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Yu_m-TZUWGX", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xNKXi-Gy3RO-" - }, - "source": [ - "ご覧のとおり、2 つの Dense レイヤーを通過する前に、(3, 3, 64) の出力は shape (576) のベクターに平滑化されました。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P3odqfHP4M67" - }, - "source": [ - "### モデルのコンパイルと学習" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MdDzI75PUXrG", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jKgyC5K_4O0d" - }, - "source": [ - "### モデルの評価" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gtyDF0MKUcM7", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0LvwaKhtUdOo", - "colab": {} - }, - "source": [ - "print(test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8cfJ8AR03gT5" - }, - "source": [ - "ご覧のとおり、我々のシンプルな CNN は 99% 以上のテスト精度を達成しています。数行のコードにしては悪くありません！違うスタイルでの CNN の書き方 (Keras Subclassing API や GradientTape を使ったもの) については[ここ](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/quickstart/advanced.ipynb)を参照してください。" - ] - } - ] -} diff --git a/site/ja/tutorials/keras/classification.ipynb b/site/ja/tutorials/keras/classification.ipynb deleted file mode 100644 index 7f0269f2f20..00000000000 --- a/site/ja/tutorials/keras/classification.ipynb +++ /dev/null @@ -1,1035 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# はじめてのニューラルネットワーク：分類問題の初歩" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EQ4yfFQxW7by", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "このガイドでは、スニーカーやシャツなど、身に着けるものの写真を分類するニューラルネットワークのモデルを訓練します。すべての詳細を理解できなくても問題ありません。TensorFlowの全体を早足で掴むためのもので、詳細についてはあとから見ていくことになります。\n", - "\n", - "このガイドでは、TensorFlowのモデルを構築し訓練するためのハイレベルのAPIである [tf.keras](https://www.tensorflow.org/guide/keras)を使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jL3OqFKZ9dFg", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow と tf.keras のインポート\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# ヘルパーライブラリのインポート\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## ファッションMNISTデータセットのロード" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "このガイドでは、[Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist)を使用します。Fashion MNISTには10カテゴリーの白黒画像70,000枚が含まれています。それぞれは下図のような1枚に付き1種類の衣料品が写っている低解像度（28×28ピクセル）の画像です。\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
     \n", - "
    \n", - "\n", - "Fashion MNISTは、画像処理のための機械学習での\"Hello, World\"としてしばしば登場する[MNIST](http://yann.lecun.com/exdb/mnist/) データセットの代替として開発されたものです。MNISTデータセットは手書きの数字（0, 1, 2 など）から構成されており、そのフォーマットはこれから使うFashion MNISTと全く同じです。\n", - "\n", - "Fashion MNISTを使うのは、目先を変える意味もありますが、普通のMNISTよりも少しだけ手応えがあるからでもあります。どちらのデータセットも比較的小さく、アルゴリズムが期待したとおりに機能するかどうかを確かめるために使われます。プログラムのテストやデバッグのためには、よい出発点になります。\n", - "\n", - "ここでは、60,000枚の画像を訓練に、10,000枚の画像を、ネットワークが学習した画像分類の正確性を評価するのに使います。TensorFlowを使うと、下記のようにFashion MNISTのデータを簡単にインポートし、ロードすることが出来ます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "ロードしたデータセットは、NumPy配列になります。\n", - "\n", - "* `train_images` と `train_labels` の2つの配列は、モデルの訓練に使用される**訓練用データセット**です。\n", - "* 訓練されたモデルは、 `test_images` と `test_labels` 配列からなる**テスト用データセット**を使ってテストします。\n", - "\n", - "画像は28×28のNumPy配列から構成されています。それぞれのピクセルの値は0から255の間の整数です。**ラベル**（label）は、0から9までの整数の配列です。それぞれの数字が下表のように、衣料品の**クラス**（class）に対応しています。\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    LabelClass
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "画像はそれぞれ単一のラベルに分類されます。データセットには上記の**クラス名**が含まれていないため、後ほど画像を出力するときのために、クラス名を保存しておきます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', \n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## データの観察\n", - "\n", - "モデルの訓練を行う前に、データセットのフォーマットを見てみましょう。下記のように、訓練用データセットには28×28ピクセルの画像が60,000枚含まれています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "同様に、訓練用データセットには60,000個のラベルが含まれます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "ラベルはそれぞれ、0から9までの間の整数です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "テスト用データセットには、10,000枚の画像が含まれます。画像は28×28ピクセルで構成されています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "テスト用データセットには10,000個のラベルが含まれます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## データの前処理\n", - "\n", - "ネットワークを訓練する前に、データを前処理する必要があります。最初の画像を調べてみればわかるように、ピクセルの値は0から255の間の数値です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3jCZdQNNCaWv" - }, - "source": [ - "ニューラルネットワークにデータを投入する前に、これらの値を0から1までの範囲にスケールします。そのためには、画素の値を255で割ります。\n", - "\n", - "**訓練用データセット**と**テスト用データセット**は、同じように前処理することが重要です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "**訓練用データセット**の最初の25枚の画像を、クラス名付きで表示してみましょう。ネットワークを構築・訓練する前に、データが正しいフォーマットになっていることを確認します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## モデルの構築\n", - "\n", - "ニューラルネットワークを構築するには、まずモデルの階層を定義し、その後モデルをコンパイルします。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### 層の設定\n", - "\n", - "ニューラルネットワークを形作る基本的な構成要素は**層**（layer）です。層は、入力されたデータから「表現」を抽出します。それらの「表現」は、今取り組もうとしている問題に対して、より「意味のある」ものであることが期待されます。\n", - "\n", - "ディープラーニングモデルのほとんどは、単純な層の積み重ねで構成されています。`tf.keras.layers.Dense` のような層のほとんどには、訓練中に学習されるパラメータが存在します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation='relu'),\n", - " keras.layers.Dense(10, activation='softmax')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "このネットワークの最初の層は、`tf.keras.layers.Flatten` です。この層は、画像を（28×28ピクセルの）2次元配列から、28×28＝784ピクセルの、1次元配列に変換します。この層が、画像の中に積まれているピクセルの行を取り崩し、横に並べると考えてください。この層には学習すべきパラメータはなく、ただデータのフォーマット変換を行うだけです。\n", - "\n", - "ピクセルが１次元化されたあと、ネットワークは2つの `tf.keras.layers.Dense` 層となります。これらの層は、密結合あるいは全結合されたニューロンの層となります。最初の `Dense` 層には、128個のノード（あるはニューロン）があります。最後の層でもある2番めの層は、10ノードの**softmax**層です。この層は、合計が1になる10個の確率の配列を返します。それぞれのノードは、今見ている画像が10個のクラスのひとつひとつに属する確率を出力します。\n", - "\n", - "### モデルのコンパイル\n", - "\n", - "モデルが訓練できるようになるには、いくつかの設定を追加する必要があります。それらの設定は、モデルの**コンパイル**(compile）時に追加されます。\n", - "\n", - "* **損失関数**（loss function） —訓練中にモデルがどれくらい正確かを測定します。この関数の値を最小化することにより、訓練中のモデルを正しい方向に向かわせようというわけです。\n", - "* **オプティマイザ**（optimizer）—モデルが見ているデータと、損失関数の値から、どのようにモデルを更新するかを決定します。\n", - "* **メトリクス**（metrics） —訓練とテストのステップを監視するのに使用します。下記の例では*accuracy* （正解率）、つまり、画像が正しく分類された比率を使用しています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam', \n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## モデルの訓練\n", - "\n", - "ニューラルネットワークの訓練には次のようなステップが必要です。\n", - "\n", - "1. モデルに訓練用データを投入します—この例では `train_images` と `train_labels` の２つの配列です。\n", - "2. モデルは、画像とラベルの対応関係を学習します。\n", - "3. モデルにテスト用データセットの予測（分類）を行わせます—この例では `test_images` 配列です。その後、予測結果と `test_labels` 配列を照合します。 \n", - "\n", - "訓練を開始するには、`model.fit` メソッドを呼び出します。モデルを訓練用データに \"fit\"（適合）させるという意味です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "モデルの訓練の進行とともに、損失値と正解率が表示されます。このモデルの場合、訓練用データでは0.88（すなわち88%）の正解率に達します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## 正解率の評価\n", - "\n", - "次に、テスト用データセットに対するモデルの性能を比較します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('\\nTest accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "ご覧の通り、テスト用データセットでの正解率は、訓練用データセットでの正解率よりも少し低くなります。この訓練時の正解率とテスト時の正解率の差は、**過学習**（over fitting）の一例です。過学習とは、新しいデータに対する機械学習モデルの性能が、訓練時と比較して低下する現象です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## 予測する\n", - "\n", - "モデルの訓練が終わったら、そのモデルを使って画像の分類予測を行うことが出来ます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "これは、モデルがテスト用データセットの画像のひとつひとつを分類予測した結果です。最初の予測を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "予測結果は、10個の数字の配列です。これは、その画像が10の衣料品の種類のそれぞれに該当するかの「確信度」を表しています。どのラベルが一番確信度が高いかを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "というわけで、このモデルは、この画像が、アンクルブーツ、`class_names[9]` である可能性が最も高いと判断したことになります。これが正しいかどうか、テスト用ラベルを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "10チャンネルすべてをグラフ化してみることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VsRq6uZiG7eT", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1]) \n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aZ_jDyLZG7eW" - }, - "source": [ - "0番目の画像と、予測、予測配列を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UH_jgCxEG7eW", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5_7K0ZL7G7eY", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Lduh0pbfG7eb" - }, - "source": [ - "予測の中のいくつかの画像を、予測値とともに表示してみましょう。正しい予測は青で、誤っている予測は赤でラベルを表示します。数字は予測したラベルのパーセント（100分率）を示します。自信があるように見えても間違っていることがあることに注意してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YGBDAiziCaXR", - "colab": {} - }, - "source": [ - "# X個のテスト画像、予測されたラベル、正解ラベルを表示します。\n", - "# 正しい予測は青で、間違った予測は赤で表示しています。\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "最後に、訓練済みモデルを使って1枚の画像に対する予測を行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yRJ7JU7JCaXT", - "colab": {} - }, - "source": [ - "# テスト用データセットから画像を1枚取り出す\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "`tf.keras` モデルは、サンプルの中の**バッチ**（batch）あるいは「集まり」について予測を行うように作られています。そのため、1枚の画像を使う場合でも、リスト化する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# 画像を1枚だけのバッチのメンバーにする\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "そして、予測を行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "scrolled": true, - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6o3nwO-KG7ex", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "`model.predict` メソッドの戻り値は、リストのリストです。リストの要素のそれぞれが、バッチの中の画像に対応します。バッチの中から、（といってもバッチの中身は１つだけですが）予測を取り出します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "np.argmax(predictions_single[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "というわけで、モデルは9というラベルを予測しました。" - ] - } - ] -} diff --git a/site/ja/tutorials/keras/overfit_and_underfit.ipynb b/site/ja/tutorials/keras/overfit_and_underfit.ipynb deleted file mode 100644 index 2e198609961..00000000000 --- a/site/ja/tutorials/keras/overfit_and_underfit.ipynb +++ /dev/null @@ -1,709 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "overfit_and_underfit.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "fTFj8ft5dlbS" - ], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fTFj8ft5dlbS" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "lzyBOpYMdp3F", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m_x4KfSJ7Vt7", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C9HmC2T4ld5B" - }, - "source": [ - "# 過学習と学習不足について知る" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRTxFhXAlnl1" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "a0tw3CshaVrH", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19rPukKZsPG6" - }, - "source": [ - "いつものように、この例のプログラムは`tf.keras` APIを使用します。詳しくはTensorFlowの[Keras guide](https://www.tensorflow.org/guide/keras)を参照してください。\n", - "\n", - "これまでの例、つまり、映画レビューの分類と燃費の推定では、検証用データでのモデルの正解率が、数エポックでピークを迎え、その後低下するという現象が見られました。\n", - "\n", - "言い換えると、モデルが訓練用データを**過学習**したと考えられます。過学習への対処の仕方を学ぶことは重要です。**訓練用データセット**で高い正解率を達成することは難しくありませんが、我々は、（これまで見たこともない）**テスト用データ**に汎化したモデルを開発したいのです。\n", - "\n", - "過学習の反対語は**学習不足**(underfitting)です。学習不足は、モデルがテストデータに対してまだ改善の余地がある場合に発生します。学習不足の原因は様々です。モデルが十分強力でないとか、正則化のしすぎだとか、単に訓練時間が短すぎるといった理由があります。学習不足は、訓練用データの中の関連したパターンを学習しきっていないということを意味します。\n", - "\n", - "モデルの訓練をやりすぎると、モデルは過学習を始め、訓練用データの中のパターンで、テストデータには一般的ではないパターンを学習します。我々は、過学習と学習不足の中間を目指す必要があります。これから見ていくように、ちょうどよいエポック数だけ訓練を行うというのは必要なスキルなのです。\n", - "\n", - "過学習を防止するための、最良の解決策は、より多くの訓練用データを使うことです。多くのデータで訓練を行えば行うほど、モデルは自然により汎化していく様になります。これが不可能な場合、次善の策は正則化のようなテクニックを使うことです。正則化は、モデルに保存される情報の量とタイプに制約を課すものです。ネットワークが少数のパターンしか記憶できなければ、最適化プロセスにより、最も主要なパターンのみを学習することになり、より汎化される可能性が高くなります。\n", - "\n", - "このノートブックでは、重みの正則化とドロップアウトという、よく使われる2つの正則化テクニックをご紹介します。これらを使って、IMDBの映画レビューを分類するノートブックの改善を図ります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5pZ8A2liqvgk", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1cweoTiruj8O" - }, - "source": [ - "## IMDBデータセットのダウンロード\n", - "\n", - "以前のノートブックで使用したエンベディングの代わりに、ここでは文をマルチホットエンコードします。このモデルは、訓練用データセットをすぐに過学習します。このモデルを使って、過学習がいつ起きるかということと、どうやって過学習と戦うかをデモします。\n", - "\n", - "リストをマルチホットエンコードすると言うのは、0と1のベクトルにするということです。具体的にいうと、例えば`[3, 5]`というシーケンスを、インデックス3と5の値が1で、それ以外がすべて0の、10,000次元のベクトルに変換するということを意味します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QpzE4iqZtJly", - "colab": {} - }, - "source": [ - "NUM_WORDS = 10000\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = keras.datasets.imdb.load_data(num_words=NUM_WORDS)\n", - "\n", - "def multi_hot_sequences(sequences, dimension):\n", - " # 形状が (len(sequences), dimension)ですべて0の行列を作る\n", - " results = np.zeros((len(sequences), dimension))\n", - " for i, word_indices in enumerate(sequences):\n", - " results[i, word_indices] = 1.0 # 特定のインデックスに対してresults[i] を１に設定する\n", - " return results\n", - "\n", - "\n", - "train_data = multi_hot_sequences(train_data, dimension=NUM_WORDS)\n", - "test_data = multi_hot_sequences(test_data, dimension=NUM_WORDS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MzWVeXe3NBTn" - }, - "source": [ - "結果として得られるマルチホットベクトルの1つを見てみましょう。単語のインデックスは頻度順にソートされています。このため、インデックスが0に近いほど１が多く出現するはずです。分布を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "71kr5rG4LkGM", - "colab": {} - }, - "source": [ - "plt.plot(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lglk41MwvU5o" - }, - "source": [ - "## 過学習のデモ\n", - "\n", - "過学習を防止するための最も単純な方法は、モデルのサイズ、すなわち、モデル内の学習可能なパラメータの数を小さくすることです（学習パラメータの数は、層の数と層ごとのユニット数で決まります）。ディープラーニングでは、モデルの学習可能なパラメータ数を、しばしばモデルの「キャパシティ」と呼びます。直感的に考えれば、パラメータ数の多いモデルほど「記憶容量」が大きくなり、訓練用のサンプルとその目的変数の間の辞書のようなマッピングをたやすく学習することができます。このマッピングには汎化能力がまったくなく、これまで見たことが無いデータを使って予測をする際には役に立ちません。\n", - "\n", - "ディープラーニングのモデルは訓練用データに適応しやすいけれど、本当のチャレレンジは汎化であって適応ではないということを、肝に銘じておく必要があります。\n", - "\n", - "一方、ネットワークの記憶容量が限られている場合、前述のようなマッピングを簡単に学習することはできません。損失を減らすためには、より予測能力が高い圧縮された表現を学習しなければなりません。同時に、モデルを小さくしすぎると、訓練用データに適応するのが難しくなります。「多すぎる容量」と「容量不足」の間にちょうどよい容量があるのです。\n", - "\n", - "残念ながら、（層の数や、層ごとの大きさといった）モデルの適切なサイズやアーキテクチャを決める魔法の方程式はありません。一連の異なるアーキテクチャを使って実験を行う必要があります。\n", - "\n", - "適切なモデルのサイズを見つけるには、比較的少ない層の数とパラメータから始めるのがベストです。それから、検証用データでの損失値の改善が見られなくなるまで、徐々に層の大きさを増やしたり、新たな層を加えたりします。映画レビューの分類ネットワークでこれを試してみましょう。\n", - "\n", - "比較基準として、```Dense```層だけを使ったシンプルなモデルを構築し、その後、それより小さいバージョンと大きいバージョンを作って比較します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ReKHdC2EgVu" - }, - "source": [ - "### 比較基準を作る" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QKgdXPx9usBa", - "colab": {} - }, - "source": [ - "baseline_model = keras.Sequential([\n", - " # `.summary` を見るために`input_shape`が必要 \n", - " keras.layers.Dense(16, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "baseline_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "baseline_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LqG3MXF5xSjR", - "colab": {} - }, - "source": [ - "baseline_history = baseline_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L-DGRBbGxI6G" - }, - "source": [ - "### より小さいモデルの構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SrfoVQheYSO5" - }, - "source": [ - "今作成したばかりの比較基準となるモデルに比べて隠れユニット数が少ないモデルを作りましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jksi-XtaxDAh", - "colab": {} - }, - "source": [ - "smaller_model = keras.Sequential([\n", - " keras.layers.Dense(4, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(4, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "smaller_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "smaller_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jbngCZliYdma" - }, - "source": [ - "同じデータを使って訓練します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ofn1AwDhx-Fe", - "colab": {} - }, - "source": [ - "smaller_history = smaller_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vIPuf23FFaVn" - }, - "source": [ - "### より大きなモデルの構築\n", - "\n", - "練習として、より大きなモデルを作成し、どれほど急速に過学習が起きるかを見ることもできます。次はこのベンチマークに、この問題が必要とするよりはるかに容量の大きなネットワークを追加しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ghQwwqwqvQM9", - "colab": {} - }, - "source": [ - "bigger_model = keras.models.Sequential([\n", - " keras.layers.Dense(512, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(512, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "bigger_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "bigger_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D-d-i5DaYmr7" - }, - "source": [ - "このモデルもまた同じデータを使って訓練します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "U1A99dhqvepf", - "colab": {} - }, - "source": [ - "bigger_history = bigger_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fy3CMUZpzH3d" - }, - "source": [ - "### 訓練時と検証時の損失をグラフにする\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HSlo1F4xHuuM" - }, - "source": [ - "実線は訓練用データセットの損失、破線は検証用データセットでの損失です（検証用データでの損失が小さい方が良いモデルです）。これをみると、小さいネットワークのほうが比較基準のモデルよりも過学習が始まるのが遅いことがわかります（4エポックではなく6エポック後）。また、過学習が始まっても性能の低下がよりゆっくりしています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0XmKDtOWzOpk", - "colab": {} - }, - "source": [ - "def plot_history(histories, key='binary_crossentropy'):\n", - " plt.figure(figsize=(16,10))\n", - " \n", - " for name, history in histories:\n", - " val = plt.plot(history.epoch, history.history['val_'+key],\n", - " '--', label=name.title()+' Val')\n", - " plt.plot(history.epoch, history.history[key], color=val[0].get_color(),\n", - " label=name.title()+' Train')\n", - "\n", - " plt.xlabel('Epochs')\n", - " plt.ylabel(key.replace('_',' ').title())\n", - " plt.legend()\n", - "\n", - " plt.xlim([0,max(history.epoch)])\n", - "\n", - "\n", - "plot_history([('baseline', baseline_history),\n", - " ('smaller', smaller_history),\n", - " ('bigger', bigger_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bi6hBhdnSfjA" - }, - "source": [ - "より大きなネットワークでは、すぐに、1エポックで過学習が始まり、その度合も強いことに注目してください。ネットワークの容量が大きいほど訓練用データをモデル化するスピードが早くなり（結果として訓練時の損失値が小さくなり）ますが、より過学習しやすく（結果として訓練時の損失値と検証時の損失値が大きく乖離しやすく）なります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ASdv7nsgEFhx" - }, - "source": [ - "## 過学習防止の戦略" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4rHoVWcswFLa" - }, - "source": [ - "### 重みの正則化を加える\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRxWepNawbBK" - }, - "source": [ - "「オッカムの剃刀」の原則をご存知でしょうか。何かの説明が2つあるとすると、最も正しいと考えられる説明は、仮定の数が最も少ない「一番単純な」説明だというものです。この原則は、ニューラルネットワークを使って学習されたモデルにも当てはまります。ある訓練用データとネットワーク構造があって、そのデータを説明できる重みの集合が複数ある時（つまり、複数のモデルがある時）、単純なモデルのほうが複雑なものよりも過学習しにくいのです。\n", - "\n", - "ここで言う「単純なモデル」とは、パラメータ値の分布のエントロピーが小さいもの（あるいは、上記で見たように、そもそもパラメータの数が少ないもの）です。したがって、過学習を緩和するための一般的な手法は、重みが小さい値のみをとることで、重み値の分布がより整然となる（正則）様に制約を与えるものです。これを「重みの正則化」と呼ばれ、ネットワークの損失関数に、重みの大きさに関連するコストを加えることで行われます。このコストには2つの種類があります。\n", - "\n", - "* [L1正則化](https://developers.google.com/machine-learning/glossary/#L1_regularization) 重み係数の絶対値に比例するコストを加える（重みの「L1ノルム」と呼ばれる）。\n", - "\n", - "* [L2正則化](https://developers.google.com/machine-learning/glossary/#L2_regularization) 重み係数の二乗に比例するコストを加える（重み係数の二乗「L2ノルム」と呼ばれる）。L2正則化はニューラルネットワーク用語では重み減衰（Weight Decay）と呼ばれる。呼び方が違うので混乱しないように。重み減衰は数学的にはL2正則化と同義である。\n", - "\n", - "L1正則化は重みパラメータの一部を0にすることでモデルを疎にする効果があります。L2正則化は重みパラメータにペナルティを加えますがモデルを疎にすることはありません。これは、L2正則化のほうが一般的である理由の一つです。\n", - "\n", - "`tf.keras`では、重みの正則化をするために、重み正則化のインスタンスをキーワード引数として層に加えます。ここでは、L2正則化を追加してみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HFGmcwduwVyQ", - "colab": {} - }, - "source": [ - "l2_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "l2_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "l2_model_history = l2_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bUUHoXb7w-_C" - }, - "source": [ - "```l2(0.001)```というのは、層の重み行列の係数全てに対して```0.001 * 重み係数の値 **2```をネットワークの損失値合計に加えることを意味します。このペナルティは訓練時のみに加えられるため、このネットワークの損失値は、訓練時にはテスト時に比べて大きくなることに注意してください。\n", - "\n", - "L2正則化の影響を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7wkfLyxBZdh_", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('l2', l2_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Kx1YHMsVxWjP" - }, - "source": [ - "ご覧のように、L2正則化ありのモデルは比較基準のモデルに比べて過学習しにくくなっています。両方のモデルのパラメータ数は同じであるにもかかわらずです。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HmnBNOOVxiG8" - }, - "source": [ - "### ドロップアウトを追加する\n", - "\n", - "ドロップアウトは、ニューラルネットワークの正則化テクニックとして最もよく使われる手法の一つです。この手法は、トロント大学のヒントンと彼の学生が開発したものです。ドロップアウトは層に適用するもので、訓練時に層から出力された特徴量に対してランダムに「ドロップアウト（つまりゼロ化）」を行うものです。例えば、ある層が訓練時にある入力サンプルに対して、普通は`[0.2, 0.5, 1.3, 0.8, 1.1]` というベクトルを出力するとします。ドロップアウトを適用すると、このベクトルは例えば`[0, 0.5, 1.3, 0, 1.1]`のようにランダムに散らばったいくつかのゼロを含むようになります。「ドロップアウト率」はゼロ化される特徴の割合で、通常は0.2から0.5の間に設定します。テスト時は、どのユニットもドロップアウトされず、代わりに出力値がドロップアウト率と同じ比率でスケールダウンされます。これは、訓練時に比べてたくさんのユニットがアクティブであることに対してバランスをとるためです。\n", - "\n", - "`tf.keras`では、Dropout層を使ってドロップアウトをネットワークに導入できます。ドロップアウト層は、その直前の層の出力に対してドロップアウトを適用します。\n", - "\n", - "それでは、IMDBネットワークに2つのドロップアウト層を追加しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OFEYvtrHxSWS", - "colab": {} - }, - "source": [ - "dpt_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(16, activation='relu'),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "dpt_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "dpt_model_history = dpt_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SPZqwVchx5xp", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('dropout', dpt_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QioQ5e9GN0bM" - }, - "source": [ - "ドロップアウトを追加することで、比較対象モデルより明らかに改善が見られます。\n", - "\n", - "まとめ：ニューラルネットワークにおいて過学習を防ぐ最も一般的な方法は次のとおりです。\n", - "\n", - "* 訓練データを増やす\n", - "* ネットワークの容量をへらす\n", - "* 重みの正則化を行う\n", - "* ドロップアウトを追加する\n", - "\n", - "このガイドで触れていない２つの重要なアプローチがあります。データ拡張とバッチ正規化です。" - ] - } - ] -} diff --git a/site/ja/tutorials/keras/regression.ipynb b/site/ja/tutorials/keras/regression.ipynb deleted file mode 100644 index 69807f891be..00000000000 --- a/site/ja/tutorials/keras/regression.ipynb +++ /dev/null @@ -1,879 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "regression.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# 回帰：燃費を予測する " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tudzcncJXetB" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "回帰問題では、価格や確率といった連続的な値の出力を予測することが目的となります。これは、分類問題の目的が、（たとえば、写真にリンゴが写っているかオレンジが写っているかといった）離散的なラベルを予測することであるのとは対照的です。\n", - "\n", - "このノートブックでは、古典的な[Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg)データセットを使用し、1970年代後半から1980年台初めの自動車の燃費を予測するモデルを構築します。この目的のため、モデルにはこの時期の多数の自動車の仕様を読み込ませます。仕様には、気筒数、排気量、馬力、重量などが含まれています。\n", - "\n", - "このサンプルでは`tf.keras` APIを使用しています。詳細は[このガイド](https://www.tensorflow.org/guide/keras)を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# ペアプロットのためseabornを使用します\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "try:\n", - " # %tensorflow_version はColab でのみ利用可能\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Auto MPG データセット\n", - "\n", - "このデータセットは[UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/)から入手可能です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### データの取得\n", - "\n", - "まず、データセットをダウンロードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"https://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "pandasを使ってデータをインポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin'] \n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### データのクレンジング\n", - "\n", - "このデータには、いくつか欠損値があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "この最初のチュートリアルでは簡単化のためこれらの行を削除します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "`\"Origin\"`の列は数値ではなくカテゴリーです。このため、ワンホットエンコーディングを行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### データを訓練用セットとテスト用セットに分割\n", - "\n", - "データセットを訓練用セットとテスト用セットに分割しましょう。\n", - "\n", - "テスト用データセットは、作成したモデルの最終評価に使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### データの観察\n", - "\n", - "訓練用セットのいくつかの列の組み合わせの同時分布を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "全体の統計値も見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### ラベルと特徴量の分離\n", - "\n", - "ラベル、すなわち目的変数を特徴量から切り離しましょう。このラベルは、モデルに予測させたい数量です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### データの正規化\n", - "\n", - "上の`train_stats`のブロックをもう一度見て、それぞれの特徴量の範囲がどれほど違っているかに注目してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "スケールや値の範囲が異なる特徴量を正規化するのはよい習慣です。特徴量の正規化なしでもモデルは収束する**かもしれませんが**、モデルの訓練はより難しくなり、結果として得られたモデルも入力で使われる単位に依存することになります。\n", - "\n", - "注：（正規化に使用する）統計量は意図的に訓練用データセットだけを使って算出していますが、これらはテスト用データセットの正規化にも使うことになります。テスト用のデータセットを、モデルの訓練に使用した分布とおなじ分布に射影する必要があるのです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "この正規化したデータを使ってモデルを訓練することになります。\n", - "\n", - "注意：ここで入力の正規化に使った統計量（平均と標準偏差）は、さきほど実施したワンホットエンコーディングとともに、モデルに供給するほかのどんなデータにも適用する必要があります。テスト用データセットだけでなく、モデルをプロダクション環境で使用する際の生のデータについても同様です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## モデル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### モデルの構築\n", - "\n", - "それではモデルを構築しましょう。ここでは、2つの全結合の隠れ層と、1つの連続値を返す出力層からなる、`Sequential`モデルを使います。モデルを構築するステップは`build_model`という1つの関数の中に組み込みます。あとから2つ目のモデルを構築するためです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation='relu', input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation='relu'),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mse',\n", - " optimizer=optimizer,\n", - " metrics=['mae', 'mse'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### モデルの検証\n", - "\n", - "`.summary`メソッドを使って、モデルの簡単な説明を表示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "では、モデルを試してみましょう。訓練用データのうち`10`個のサンプルからなるバッチを取り出し、それを使って`model.predict`メソッドを呼び出します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "うまく動作しているようです。予定どおりの型と形状の出力が得られています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### モデルの訓練\n", - "\n", - "モデルを1000エポック訓練し、訓練と検証の正解率を`history`オブジェクトに記録します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# エポックが終わるごとにドットを一つ出力することで進捗を表示\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "`history`オブジェクトに保存された数値を使ってモデルの訓練の様子を可視化します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - " \n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mae'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mae'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - " \n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mse'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mse'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "このグラフを見ると、検証エラーは100エポックを過ぎたあたりで改善が見られなくなり、むしろ悪化しているようです。検証スコアの改善が見られなくなったら自動的に訓練を停止するように、`model.fit`メソッド呼び出しを変更します。ここでは、エポック毎に訓練状態をチェックする*EarlyStopping*コールバックを使用します。設定したエポック数の間に改善が見られない場合、訓練を自動的に停止します。\n", - "\n", - "このコールバックについての詳細は[ここ](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping)を参照ください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# patience は改善が見られるかを監視するエポック数を表すパラメーター\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "検証用データセットでのグラフを見ると、平均誤差は+/- 2 MPG（マイル/ガロン）前後です。これはよい精度でしょうか？その判断はおまかせします。\n", - "\n", - "モデルの訓練に使用していない**テスト用**データセットを使って、モデルがどれくらい汎化できているか見てみましょう。これによって、モデルが実際の現場でどれくらい正確に予測できるかがわかります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "### モデルを使った予測\n", - "\n", - "最後に、テストデータを使ってMPG値を予測します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OrkHGKZcusUo" - }, - "source": [ - "そこそこよい予測ができているように見えます。誤差の分布を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r9_kI6MHu1UU" - }, - "source": [ - "とても正規分布には見えませんが、サンプル数が非常に小さいからだと考えられます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## 結論\n", - "\n", - "このノートブックでは、回帰問題を扱うためのテクニックをいくつか紹介しました。\n", - "\n", - "* 平均二乗誤差（MSE: Mean Squared Error）は回帰問題に使われる一般的な損失関数です（分類問題には異なる損失関数が使われます）。\n", - "* 同様に、回帰問題に使われる評価指標も分類問題とは異なります。回帰問題の一般的な評価指標は平均絶対誤差（MAE: Mean Absolute Error）です。\n", - "* 入力数値特徴量の範囲が異なっている場合、特徴量ごとにおなじ範囲に正規化するべきです。\n", - "* 訓練用データが多くない場合、過学習を避けるために少ない隠れ層をもつ小さいネットワークを使うというのがよい方策の1つです。\n", - "* Early Stoppingは過学習を防止するための便利な手法の一つです。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/keras/save_and_load.ipynb b/site/ja/tutorials/keras/save_and_load.ipynb deleted file mode 100644 index 3f98c265291..00000000000 --- a/site/ja/tutorials/keras/save_and_load.ipynb +++ /dev/null @@ -1,894 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "save_and_load.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pZJ3uY9O17VN" - }, - "source": [ - "# モデルの保存と復元" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M4Ata7_wMul1" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0dYY-BkJaqKb", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBdde4YJeJKF" - }, - "source": [ - "モデルは訓練中にも、訓練が終わったあとも保存できます。このことは、長い訓練時間を掛けなくても、やめたところから再開できるということを意味します。モデルが保存可能であることは、あなたが作ったモデルを他の人と共有できるということでもあります。研究結果であるモデルや手法を公開する際、機械学習の実務家はほとんど次のものを共有します。\n", - "\n", - "* モデルを構築するプログラム\n", - "* 学習済みモデルの重みあるいはパラメータ\n", - "\n", - "このデータを共有することで、他の人がモデルだどの様に動作するかを理解したり、新しいデータに試してみたりすることが容易になります。\n", - "\n", - "注意：信頼できないプログラムには気をつけましょう。TensorFlowのモデルもプログラムです。詳しくは、[Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)を参照してください。\n", - "\n", - "### オプション\n", - "\n", - "TensorFlowのモデルを保存する方法は、使っているAPIによって異なります。このガイドはTensorFlowのモデルを構築し訓練するためのハイレベルなAPIである[tf.keras](https://www.tensorflow.org/guide/keras)を使っています。この他のアプローチについては、TensorFlowの [Save and Restore](https://www.tensorflow.org/guide/saved_model) ガイド、あるいは、[Saving in eager](https://www.tensorflow.org/guide/eager#object-based_saving)を参照してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xCUREq7WXgvg" - }, - "source": [ - "## 設定\n", - "\n", - "### インストールとインポート" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7l0MiTOrXtNv" - }, - "source": [ - "TensorFlowと依存関係のライブラリをインストールし、インポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} - }, - "source": [ - "!pip install h5py pyyaml " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbGsznErXWt6" - }, - "source": [ - "### サンプルデータセットの取得\n", - "\n", - "ここでは、モデルを訓練し重みの保存をデモするために、 [MNIST dataset](http://yann.lecun.com/exdb/mnist/) を使います。デモの実行を速くするため、最初の1,000件のサンプルだけを使います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "tf.__version__" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_labels = train_labels[:1000]\n", - "test_labels = test_labels[:1000]\n", - "\n", - "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", - "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "anG3iVoXyZGI" - }, - "source": [ - "### モデルの定義" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wynsOBfby0Pa" - }, - "source": [ - "重みの保存と読み込みのデモを行うための簡単なモデルを定義しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} - }, - "source": [ - "# 短いシーケンシャルモデルを返す関数\n", - "def create_model():\n", - " model = tf.keras.models.Sequential([\n", - " keras.layers.Dense(512, activation='relu', input_shape=(784,)),\n", - " keras.layers.Dropout(0.2),\n", - " keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - " \n", - " model.compile(optimizer='adam', \n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - " \n", - " return model\n", - "\n", - "\n", - "# 基本的なモデルのインスタンスを作る\n", - "model = create_model()\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "soDE0W_KH8rG" - }, - "source": [ - "## 訓練中にチェックポイントを保存する" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRyd5qQQIXZm" - }, - "source": [ - "主な用途は訓練の**途中**あるいは**終了後**にチェックポイントを自動的に保存することです。こうすることにより、再び訓練を行うことなくモデルを使用することができ、また、訓練が中断された場合に、中止したところから再開できます。\n", - "\n", - "`tf.keras.callbacks.ModelCheckpoint`がこれを行うためのコールバックです。このコールバックにはチェックポイントを構成するためのいくつかの引数があります。\n", - "\n", - "### チェックポイントコールバックの使い方\n", - "\n", - "モデルの訓練時に、`ModelCheckpoint`を渡します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} - }, - "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# チェックポイントコールバックを作る\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, \n", - " save_weights_only=True,\n", - " verbose=1)\n", - "\n", - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs = 10, \n", - " validation_data = (test_images,test_labels),\n", - " callbacks = [cp_callback]) # 訓練にコールバックを渡す\n", - "\n", - "# オプティマイザの状態保存についての警告が表示されるかもしれません。\n", - "# これらの警告は（このノートブックで発生する同様な警告を含めて）\n", - "# 古い用法を非推奨にするためのもので、無視して構いません。" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlM-sgyJO084" - }, - "source": [ - "この結果、エポックごとに更新される一連のTensorFlowチェックポイントファイルが作成されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} - }, - "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wlRN_f56Pqa9" - }, - "source": [ - "訓練していない新しいモデルを作ります。重みだけからモデルを復元する場合には、元のモデルと同じアーキテクチャのモデルが必要です。モデルのアーキテクチャが同じであるため、モデルの異なる**インスタンス**であっても重みを共有することができるのです。\n", - "\n", - "訓練していない全く新しいモデルを作り、テストデータセットで評価します。訓練をしていないモデルは偶然のレベル（正解率10%以下）の性能しか無いはずです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Untrained model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1DTKpZssRSo3" - }, - "source": [ - "次に、チェックポイントから重みをロードし、再び評価します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} - }, - "source": [ - "model.load_weights(checkpoint_path)\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpAbKkAyVPV8" - }, - "source": [ - "### チェックポイントコールバックのオプション\n", - "\n", - "このコールバックには、チェックポイントに一意な名前をつけたり、チェックポイントの頻度を調整するためのオプションがあります。\n", - "\n", - "新しいモデルを訓練し、5エポックごとに一意な名前のチェックポイントを保存します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} - }, - "source": [ - "# ファイル名に(`str.format`を使って)エポック数を埋め込みます\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " checkpoint_path, verbose=1, save_weights_only=True,\n", - " # 重みを5エポックごとに保存します\n", - " period=5)\n", - "\n", - "model = create_model()\n", - "model.fit(train_images, train_labels,\n", - " epochs = 50, callbacks = [cp_callback],\n", - " validation_data = (test_images,test_labels),\n", - " verbose=0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1zFrKTjjavWI" - }, - "source": [ - "次に、出来上がったチェックポイントを確認し、最後のものを選択します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} - }, - "source": [ - "! ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} - }, - "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", - "latest" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zk2ciGbKg561" - }, - "source": [ - "注意：デフォルトのtensorflowフォーマットは、直近の5つのチェックポイントのみを保存します。\n", - "\n", - "テストのため、モデルをリセットし最後のチェックポイントをロードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "model.load_weights(latest)\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c2OxsJOTHxia" - }, - "source": [ - "## これらのファイルは何？" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtdYhvWnH2ib" - }, - "source": [ - "上記のコードでは、重みだけをバイナリで[checkpoint](https://www.tensorflow.org/guide/saved_model#save_and_restore_variables)形式の一連のファイルに保存します。チェックポイントには、次のものが含まれます。\n", - "\n", - "* １つ以上のモデルの重みの断片\n", - "* どの重みがどの断片に保存されているかを示すインデックスファイル\n", - "\n", - "1台のマシンだけでモデルの訓練を行っている場合には、`.data-00000-of-00001`のようなサフィックスのついたファイルが1つだけ作成されます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S_FA-ZvxuXQV" - }, - "source": [ - "## 手動で重みを保存する\n", - "\n", - "上記では重みをモデルにロードする方法を見ました。\n", - "\n", - "手動で重みを保存するのも同じ様に簡単です。`Model.save_weights` メソッドを使います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} - }, - "source": [ - "# 重みの保存\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# 重みの復元\n", - "model = create_model()\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", - "\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kOGlxPRBEvV1" - }, - "source": [ - "## モデル全体の保存\n", - "\n", - "モデルとオプティマイザを、その状態（重みと変数）とモデルの設定の両方を含む1つのファイルに保存することができます。これにより、モデルをオリジナルのPythonコードにアクセスしなくとも使用できるようにエクスポートできます。オプティマイザの状態が復元されるので、中断したところから訓練を再開することも可能です。\n", - "\n", - "完全に機能するモデルを保存できるのは便利です。保存したモデルをTensorFlow.js ([HDF5](https://js.tensorflow.org/tutorials/import-keras.html), [Saved Model](https://js.tensorflow.org/tutorials/import-saved-model.html))でロードし、ブラウザで訓練したり、実行したりすることができるほか、TensorFlow Lite ([HDF5](https://www.tensorflow.org/lite/convert/python_api#exporting_a_tfkeras_file_), [Saved Model](https://www.tensorflow.org/lite/convert/python_api#exporting_a_savedmodel_))\n", - "を使ってモバイルデバイスで実行できるように変換することも可能です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" - }, - "source": [ - "### HDF5ファイルとして\n", - "\n", - "Kerasでは、[HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) 標準を使った基本的なファイルフォーマットが利用できます。ここでの利用目的では、保存されたモデルは単独のバイナリラージオブジェクト（blob）として扱うことができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)\n", - "\n", - "# モデル全体を１つのHDF5ファイルに保存します。\n", - "model.save('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" - }, - "source": [ - "保存したファイルを使ってモデルを再作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} - }, - "source": [ - "# 重みとオプティマイザを含む全く同じモデルを再作成\n", - "new_model = keras.models.load_model('my_model.h5')\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" - }, - "source": [ - "正解率を検査します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} - }, - "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "この方法では、次のすべてが保存されます。\n", - "\n", - "* 重みの値\n", - "* モデルの設定（アーキテクチャ）\n", - "* オプティマイザの設定\n", - "\n", - "Kerasは保存する際にアーキテクチャを調べます。いまのところ、TensorFlowのオプティマイザ（`tf.train`に含まれるもの）を保存することはできません。TensorFlowのオプティマイザを使用している場合には、モデルをロードしたあと再コンパイルする必要があり、オプティマイザの状態は失われます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPyhgcoVzqUB" - }, - "source": [ - "### `saved_model`として" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LtcN4VIb7JkK" - }, - "source": [ - "注意：この手法による`tf.keras`モデルの保存は実験的なもので、将来のバージョンで変更される可能性があります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DSWiSB0Q8c46" - }, - "source": [ - "新しいモデルを作ります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iUvT_3qE8hV5" - }, - "source": [ - "`saved_model`を作成し、タイムスタンプ付きのディレクトリに保存します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "scrolled": true, - "colab": {} - }, - "source": [ - "import time\n", - "saved_model_path = \"./saved_models/{}\".format(int(time.time()))\n", - "\n", - "tf.keras.experimental.export_saved_model(model, saved_model_path)\n", - "saved_model_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MjpmyPfh8-1n" - }, - "source": [ - "作成したsaved_modelsを一覧表示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZtOvxA7V0iTv", - "colab": {} - }, - "source": [ - "!ls saved_models/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B7qfpvpY9HCe" - }, - "source": [ - "保存されたモデル(SavedModel)から新しいKerasモデルをリロードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} - }, - "source": [ - "new_model = tf.keras.experimental.load_from_saved_model(saved_model_path)\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" - }, - "source": [ - "復元されたモデルを実行します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "utcL5bs0aoih", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model.predict(test_images).shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Pc9e6G6w1AWG", - "scrolled": true, - "colab": {} - }, - "source": [ - "# モデルを評価する前にコンパイルする必要があります。\n", - "# モデルをデプロイするだけであればこのステップは不要です。\n", - "\n", - "new_model.compile(optimizer=model.optimizer, # ロードしてあったオプティマイザを保持\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# モデルを評価します。\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eUYTzSz5VxL2" - }, - "source": [ - "## この先は？\n", - "\n", - "`tf.keras`を使った保存とロードのクイックガイドでした。\n", - "\n", - "* [tf.keras guide](https://www.tensorflow.org/guide/keras) には`tf.keras`での保存とロードについて、もう少し記載されています\n", - "\n", - "* Eager Executionでの保存については[Saving in eager](https://www.tensorflow.org/guide/eager#object_based_saving) を参照ください\n", - "\n", - "* [Save and Restore](https://www.tensorflow.org/guide/saved_model)ガイドには、TensorFlowでの保存についてローレベルの詳細が記載されています" - ] - } - ] -} diff --git a/site/ja/tutorials/keras/text_classification.ipynb b/site/ja/tutorials/keras/text_classification.ipynb deleted file mode 100644 index abb700175e3..00000000000 --- a/site/ja/tutorials/keras/text_classification.ipynb +++ /dev/null @@ -1,740 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# 映画レビューのテキスト分類" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FKrWNcH6Xyz5", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "ここでは、映画のレビューをそのテキストを使って**肯定的**か**否定的**かに分類します。これは、二値分類あるいは2クラス分類という問題の例であり、機械学習において重要でいろいろな応用が可能なものです。\n", - "\n", - "ここでは、[Internet Movie Database](https://www.imdb.com/)から抽出した50,000件の映画レビューを含む、 [IMDB dataset](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) を使います。レビューは訓練用とテスト用に25,000件ずつに分割されています。訓練用とテスト用のデータは**均衡**しています。言い換えると、それぞれが同数の肯定的及び否定的なレビューを含んでいます。\n", - "\n", - "ここでは、TensorFlowを使ってモデルを構築・訓練するためのハイレベルなAPIである [tf.keras](https://www.tensorflow.org/guide/keras)を使用します。`tf.keras`を使ったもう少し高度なテキスト分類のチュートリアルについては、 [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## IMDB datasetのダウンロード\n", - "\n", - "IMDBデータセットは、TensorFlowにパッケージ化されています。それは前処理済みのものであり、（単語の連なりである）レビューが、整数の配列に変換されています。そこでは整数が辞書中の特定の単語を表します。\n", - "\n", - "次のコードは、IMDBデータセットをあなたのパソコンにダウンロードします。（すでにダウンロードしていれば、キャッシュされたコピーを使用します）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "`num_words=10000`という引数は、訓練データ中に出てくる単語のうち、最も頻繁に出現する10,000個を保持するためのものです。データサイズを管理可能にするため、稀にしか出現しない単語は破棄されます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## データの観察\n", - "\n", - "データの形式を理解するために少し時間を割いてみましょう。このデータセットは前処理済みで、サンプルそれぞれが、映画レビューの中の単語を表す整数の配列になっています。ラベルはそれぞれ、0または1の整数値で、0が否定的レビュー、1が肯定的なレビューを示しています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} - }, - "source": [ - "print(\"Training entries: {}, labels: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "レビューのテキストは複数の整数に変換されており、それぞれの整数が辞書の中の特定の単語を表します。最初のレビューがどのようなものか見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "映画のレビューはそれぞれ長さが異なっていることでしょう。次のコードで、最初と2つ目のレビューの単語の数を見てみます。ニューラルネットワークへの入力は同じ長さでなければならないため、後ほどその問題を解決する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### 整数を単語に戻してみる\n", - "\n", - "整数をテキストに戻す方法を知っていると便利です。整数を文字列にマッピングする辞書オブジェクトを検索するためのヘルパー関数を定義します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} - }, - "source": [ - "# 単語を整数にマッピングする辞書\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# インデックスの最初の方は予約済み\n", - "word_index = {k:(v+3) for k,v in word_index.items()} \n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # unknown\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "`decode_review`を使うと、最初のレビューのテキストを表示できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} - }, - "source": [ - "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## データの準備\n", - "\n", - "レビュー（整数の配列）は、ニューラルネットワークに投入する前に、テンソルに変換する必要があります。これには2つの方法があります。\n", - "\n", - "* 配列をワンホット（one-hot）エンコーディングと同じように、単語の出現を表す0と1のベクトルに変換します。例えば、[3, 5]という配列は、インデックス3と5を除いてすべてゼロの10,000次元のベクトルになります。そして、これをネットワークの最初の層、すなわち、浮動小数点のベクトルデータを扱うことができるDense（全結合）層とします。ただし、これは単語数×レビュー数の行列が必要なメモリ集約的な方法です。\n", - "* もう一つの方法では、配列をパディングによって同じ長さに揃え、`サンプル数 * 長さの最大値`の形の整数テンソルにします。そして、この形式を扱うことができるEmbedding（埋め込み）層をネットワークの最初の層にします。\n", - "\n", - "このチュートリアルでは、後者を採用することにします。\n", - "\n", - "映画レビューは同じ長さでなければならないので、長さを標準化する [pad_sequences](https://www.tensorflow.org/versions/r1.10/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences) 関数を使うことにします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} - }, - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "サンプルの長さを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "次に、パディング済みの最初のサンプルを確認します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## モデルの構築\n", - "\n", - "ニューラルネットワークは、層を積み重ねることで構成されます。この際、２つの大きな決定が必要です。\n", - "\n", - "* モデルにいくつの**層**を設けるか？\n", - "* 層ごとに何個の**隠れユニット**を使用するか？\n", - "\n", - "この例では、入力データは単語インデックスの配列で構成されています。推定の対象となるラベルは、0または1です。この問題のためのモデルを構築しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "# 入力の形式は映画レビューで使われている語彙数（10,000語）\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation='relu'))\n", - "model.add(keras.layers.Dense(1, activation='sigmoid'))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "これらの層は、分類器を構成するため一列に積み重ねられます。\n", - "\n", - "1. 最初の層は`Embedding`（埋め込み）層です。この層は、整数にエンコードされた語彙を受け取り、それぞれの単語インデックスに対応する埋め込みベクトルを検索します。埋め込みベクトルは、モデルの訓練の中で学習されます。ベクトル化のために、出力行列には次元が１つ追加されます。その結果、次元は、`(batch, sequence, embedding)`となります。\n", - "2. 次は、`GlobalAveragePooling1D`（１次元のグローバル平均プーリング）層です。この層は、それぞれのサンプルについて、シーケンスの次元方向に平均値をもとめ、固定長のベクトルを返します。この結果、モデルは最も単純な形で、可変長の入力を扱うことができるようになります。\n", - "3. この固定長の出力ベクトルは、16個の隠れユニットを持つ全結合（`Dense`）層に受け渡されます。\n", - "4. 最後の層は、1個の出力ノードに全結合されます。シグモイド（`sigmoid`）活性化関数を使うことで、値は確率あるいは確信度を表す0と1の間の浮動小数点数となります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### 隠れユニット\n", - "\n", - "上記のモデルには、入力と出力の間に、2つの中間層あるいは「隠れ」層があります。出力（ユニット、ノード、またはニューロン）は、その層の内部表現の次元数です。言い換えると、このネットワークが学習によって内部表現を獲得する際の自由度ということです。\n", - "\n", - "モデルにより多くの隠れユニットがある場合（内部表現空間の次元数がより大きい場合）、または、より多くの層がある場合、あるいはその両方の場合、ネットワークはより複雑な内部表現を学習することができます。しかしながら、その結果として、ネットワークの計算量が多くなるほか、学習してほしくないパターンを学習するようになります。学習してほしくないパターンとは、訓練データでの性能は向上するものの、テスト用データの性能が向上しないパターンです。この問題を**過学習**（*overfitting*）といいます。この問題は後ほど検証することになります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### 損失関数とオプティマイザ\n", - "\n", - "モデルを訓練するには、損失関数とオプティマイザが必要です。今回の問題は二値分類問題であり、モデルの出力は確率（1ユニットの層とシグモイド活性化関数）であるため、損失関数として`binary_crossentropy`（2値のクロスエントロピー）関数を使用することにします。\n", - "\n", - "損失関数の候補はこれだけではありません。例えば、`mean_squared_error`（平均二乗誤差）を使うこともできます。しかし、一般的には、確率を扱うには`binary_crossentropy`の方が適しています。`binary_crossentropy`は、確率分布の間の「距離」を測定する尺度です。今回の場合には、真の分布と予測値の分布の間の距離ということになります。\n", - "\n", - "後ほど、回帰問題を検証する際には（例えば家屋の値段を推定するとか）、もう一つの損失関数である`mean_squared_error`（平均二乗誤差）の使い方を目にすることになります。\n", - "\n", - "さて、モデルのオプティマイザと損失関数を設定しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "## 検証用データを作る\n", - "\n", - "訓練を行う際、モデルが見ていないデータでの正解率を検証したいと思います。もとの訓練用データから、10,000個のサンプルを取り分けて**検証用データ**（*validation set*）を作ります。（なぜ、ここでテスト用データを使わないのでしょう？ 今回の目的は、訓練用データだけを使って、モデルの開発とチューニングを行うことです。その後、テスト用データを1回だけ使い、正解率を検証するのです。）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} - }, - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## モデルの訓練\n", - "\n", - "512個のサンプルからなるミニバッチを使って、40エポックモデルを訓練します。この結果、`x_train`と`y_train`に含まれるすべてのサンプルを40回繰り返すことになります。訓練中、検証用データの10,000サンプルを用いて、モデルの損失と正解率をモニタリングします。 " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "scrolled": false, - "colab": {} - }, - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## モデルの評価\n", - "\n", - "さて、モデルの性能を見てみましょう。2つの値が返されます。損失（エラーを示す数値であり、小さい方が良い）と正解率です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "この、かなり素朴なアプローチでも87%前後の正解率を達成しました。もっと高度なアプローチを使えば、モデルの正解率は95%に近づけることもできるでしょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## 正解率と損失の時系列グラフを描く\n", - "\n", - "`model.fit()` は、訓練中に発生したすべてのことを記録した辞書を含む`History` オブジェクトを返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} - }, - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "4つのエントリがあります。それぞれが、訓練と検証の際にモニターしていた指標を示します。これを使って、訓練時と検証時の損失を比較するグラフと、訓練時と検証時の正解率を比較するグラフを作成することができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history_dict['accuracy']\n", - "val_acc = history_dict['val_accuracy']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# \"bo\" is for \"blue dot\"\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "# b is for \"solid blue line\"\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} - }, - "source": [ - "plt.clf() # 図のクリア\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "上記のグラフでは、点が訓練時の損失と正解率を、実線が検証時の損失と正解率を表しています。\n", - "\n", - "訓練時の損失がエポックごとに**減少**し、訓練時の正解率がエポックごとに**上昇**していることに気がつくはずです。繰り返すごとに指定された数値指標を最小化する勾配降下法を最適化に使用している場合に期待される動きです。\n", - "\n", - "これは、検証時の損失と正解率には当てはまりません。20エポックを過ぎたあたりから、横ばいになっているようです。これが、過学習の一例です。モデルの性能が、訓練用データでは高い一方で、見たことの無いデータではそれほど高くないというものです。このポイントをすぎると、モデルが最適化しすぎて、訓練用データでは特徴的であるが、テスト用データには一般化できない内部表現を学習しています。\n", - "\n", - "このケースの場合、20エポックを過ぎたあたりで訓練をやめることで、過学習を防止することが出来ます。後ほど、コールバックを使って、これを自動化する方法を紹介します。" - ] - } - ] -} diff --git a/site/ja/tutorials/load_data/csv.ipynb b/site/ja/tutorials/load_data/csv.ipynb deleted file mode 100644 index 8eae982fb50..00000000000 --- a/site/ja/tutorials/load_data/csv.ipynb +++ /dev/null @@ -1,928 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "csv.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DweYe9FcbMK_" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AVV2e0XKbJeX", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sUtoed20cRJJ" - }, - "source": [ - "# tf.data を使って CSV をロードする" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1ap_W4aQcgNT" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RSywPQ2n736s" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C-3Xbt0FfGfs" - }, - "source": [ - "このチュートリアルでは、CSV データを `tf.data.Dataset` にロードする手法の例を示します。\n", - "\n", - "このチュートリアルで使われているデータはタイタニック号の乗客リストから取られたものです。乗客が生き残る可能性を、年齢、性別、チケットの等級、そして乗客が一人で旅行しているか否かといった特性から予測することを試みます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "I4dwMQVQMQWD", - "colab": {} - }, - "source": [ - "try:\n", - " !pip install tf-nightly-2.0-preview\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "baYFZMW_bJHh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import tensorflow as tf\n", - "import tensorflow_datasets as tfds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ncf5t6tgL5ZI", - "colab": {} - }, - "source": [ - "TRAIN_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/train.csv\"\n", - "TEST_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/eval.csv\"\n", - "\n", - "train_file_path = tf.keras.utils.get_file(\"train.csv\", TRAIN_DATA_URL)\n", - "test_file_path = tf.keras.utils.get_file(\"eval.csv\", TEST_DATA_URL)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ONE94qulk6S", - "colab": {} - }, - "source": [ - "# numpy の値を読みやすくする\n", - "np.set_printoptions(precision=3, suppress=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wuqj601Qw0Ml" - }, - "source": [ - "## データのロード\n", - "\n", - "それではいつものように、扱っている CSV ファイルの先頭を見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "54Dv7mCrf9Yw", - "colab": {} - }, - "source": [ - "!head {train_file_path}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YOYKQKmMj3D6" - }, - "source": [ - "ご覧のように、CSV の列にはラベルが付いています。後ほど必要になるので、ファイルから読み出しておきましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "v0sLG216MtwT", - "colab": {} - }, - "source": [ - "# 入力ファイル中の CSV 列\n", - "with open(train_file_path, 'r') as f:\n", - " names_row = f.readline()\n", - "\n", - "\n", - "CSV_COLUMNS = names_row.rstrip('\\n').split(',')\n", - "print(CSV_COLUMNS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZS-bt1LvWn2x" - }, - "source": [ - " データセットコンストラクタはこれらのラベルを自動的にピックアップします。\n", - "\n", - "使用するファイルの1行目に列名がない場合、`make_csv_dataset` 関数の `column_names` 引数に文字列のリストとして渡します。\n", - "\n", - "```python\n", - "\n", - "CSV_COLUMNS = ['survived', 'sex', 'age', 'n_siblings_spouses', 'parch', 'fare', 'class', 'deck', 'embark_town', 'alone']\n", - "\n", - "dataset = tf.data.experimental.make_csv_dataset(\n", - " ...,\n", - " column_names=CSV_COLUMNS,\n", - " ...)\n", - " \n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gZfhoX7bR9u4" - }, - "source": [ - "この例では使用可能な列をすべて使うことになります。データセットから列を除く必要がある場合には、使用したい列だけを含むリストを作り、コンストラクタの（オプションである）`select_columns` 引数として渡します。\n", - "\n", - "\n", - "```python\n", - "\n", - "drop_columns = ['fare', 'embark_town']\n", - "columns_to_use = [col for col in CSV_COLUMNS if col not in drop_columns]\n", - "\n", - "dataset = tf.data.experimental.make_csv_dataset(\n", - " ...,\n", - " select_columns = columns_to_use, \n", - " ...)\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "67mfwr4v-mN_" - }, - "source": [ - "各サンプルのラベルとなる列を特定し、それが何であるかを示す必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iXROZm5f3V4E", - "colab": {} - }, - "source": [ - "LABELS = [0, 1]\n", - "LABEL_COLUMN = 'survived'\n", - "\n", - "FEATURE_COLUMNS = [column for column in CSV_COLUMNS if column != LABEL_COLUMN]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t4N-plO4tDXd" - }, - "source": [ - "コンストラクタの引数の値が揃ったので、ファイルから CSV データを読み込みデータセットを作ることにしましょう。\n", - "\n", - "（完全なドキュメントは、`tf.data.experimental.make_csv_dataset` を参照してください）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Co7UJ7gpNADC", - "colab": {} - }, - "source": [ - "def get_dataset(file_path):\n", - " dataset = tf.data.experimental.make_csv_dataset(\n", - " file_path,\n", - " batch_size=12, # 見やすく表示するために意図して小さく設定しています\n", - " label_name=LABEL_COLUMN,\n", - " na_value=\"?\",\n", - " num_epochs=1,\n", - " ignore_errors=True)\n", - " return dataset\n", - "\n", - "raw_train_data = get_dataset(train_file_path)\n", - "raw_test_data = get_dataset(test_file_path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vHUQFKoQI6G7" - }, - "source": [ - "データセットを構成する要素は、(複数のサンプル, 複数のラベル)の形のタプルとして表されるバッチです。サンプル中のデータは（行ベースのテンソルではなく）列ベースのテンソルとして構成され、それぞれはバッチサイズ（このケースでは12個）の要素が含まれます。\n", - "\n", - "実際に見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qWtFYtwXIeuj", - "colab": {} - }, - "source": [ - "examples, labels = next(iter(raw_train_data)) # 最初のバッチのみ\n", - "print(\"EXAMPLES: \\n\", examples, \"\\n\")\n", - "print(\"LABELS: \\n\", labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9cryz31lxs3e" - }, - "source": [ - "## データの前処理" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tSyrkSQwYHKi" - }, - "source": [ - "### カテゴリデータ\n", - "\n", - "この CSV データ中のいくつかの列はカテゴリ列です。つまり、その中身は、限られた選択肢の中のひとつである必要があります。\n", - "\n", - "この CSV では、これらの選択肢はテキストとして表現されています。このテキストは、モデルの訓練を行えるように、数字に変換する必要があります。これをやりやすくするため、カテゴリ列のリストとその選択肢のリストを作成する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mWDniduKMw-C", - "colab": {} - }, - "source": [ - "CATEGORIES = {\n", - " 'sex': ['male', 'female'],\n", - " 'class' : ['First', 'Second', 'Third'],\n", - " 'deck' : ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'],\n", - " 'embark_town' : ['Cherbourg', 'Southhampton', 'Queenstown'],\n", - " 'alone' : ['y', 'n']\n", - "}\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8Ii0YWsoKBVx" - }, - "source": [ - "カテゴリ値のテンソルを受け取り、それを値の名前のリストとマッチングして、さらにワンホット・エンコーディングを行う関数を書きます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bP02_BflkDbv", - "colab": {} - }, - "source": [ - "def process_categorical_data(data, categories):\n", - " \"\"\"カテゴリ値を表すワンホット・エンコーディングされたテンソルを返す\"\"\"\n", - " \n", - " # 最初の ' ' を取り除く\n", - " data = tf.strings.regex_replace(data, '^ ', '')\n", - " # 最後の '.' を取り除く\n", - " data = tf.strings.regex_replace(data, r'\\.$', '')\n", - " \n", - " # ワンホット・エンコーディング\n", - " # data を1次元（リスト）から2次元（要素が1個のリストのリスト）にリシェープ\n", - " data = tf.reshape(data, [-1, 1])\n", - " # それぞれの要素について、カテゴリ数の長さの真偽値のリストで、\n", - " # 要素とカテゴリのラベルが一致したところが True となるものを作成\n", - " data = categories == data\n", - " # 真偽値を浮動小数点数にキャスト\n", - " data = tf.cast(data, tf.float32)\n", - " \n", - " # エンコーディング全体を次の1行に収めることもできる：\n", - " # data = tf.cast(categories == tf.reshape(data, [-1, 1]), tf.float32)\n", - " return data" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "To2qbBGGMO1D" - }, - "source": [ - "この処理を可視化するため、最初のバッチからカテゴリ列のテンソル1つを取り出し、処理を行い、前後の状態を示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ds7MOLMkK2Gf", - "colab": {} - }, - "source": [ - "class_tensor = examples['class']\n", - "class_tensor" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HdDUSgpoTKfA", - "colab": {} - }, - "source": [ - "class_categories = CATEGORIES['class']\n", - "class_categories" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yHQeR47_ObpT", - "colab": {} - }, - "source": [ - "processed_class = process_categorical_data(class_tensor, class_categories)\n", - "processed_class" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ACkc_cCaTuos" - }, - "source": [ - "2つの入力の長さと、出力の形状の関係に注目してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gvvXM8m0T00O", - "colab": {} - }, - "source": [ - "print(\"Size of batch: \", len(class_tensor.numpy()))\n", - "print(\"Number of category labels: \", len(class_categories))\n", - "print(\"Shape of one-hot encoded tensor: \", processed_class.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9AsbaFmCeJtF" - }, - "source": [ - "### 連続データ" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o2maE8d2ijsq" - }, - "source": [ - "連続データは値が0と1の間にになるように標準化する必要があります。これを行うために、それぞれの値を、1を列値の平均の2倍で割ったものを掛ける関数を書きます。\n", - "\n", - "この関数は、データの2次元のテンソルへのリシェープも行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IwGOy61lkQw-", - "colab": {} - }, - "source": [ - "def process_continuous_data(data, mean):\n", - " # data の標準化\n", - " data = tf.cast(data, tf.float32) * 1/(2*mean)\n", - " return tf.reshape(data, [-1, 1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Yh8R7BujTAu" - }, - "source": [ - "この計算を行うためには、列値の平均が必要です。現実には、この値を計算する必要があるのは明らかですが、この例のために値を示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iNE_mTJqegGQ", - "colab": {} - }, - "source": [ - "MEANS = {\n", - " 'age' : 29.631308,\n", - " 'n_siblings_spouses' : 0.545455,\n", - " 'parch' : 0.379585,\n", - " 'fare' : 34.385399\n", - "}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "raZtRlmaj-A5" - }, - "source": [ - "前と同様に、この関数が実際に何をしているかを見るため、連続値のテンソルを1つ取り、処理前と処理後を見てみます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "G-t_RSBrM2Vm", - "colab": {} - }, - "source": [ - "age_tensor = examples['age']\n", - "age_tensor" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "M9lMLaEsjq3K", - "colab": {} - }, - "source": [ - "process_continuous_data(age_tensor, MEANS['age'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPWkC4_1l3IG" - }, - "source": [ - "### データの前処理" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jIvyqVAXmsN4" - }, - "source": [ - "これらの前処理のタスクを1つの関数にまとめ、データセット内のバッチにマッピングできるようにします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rMxEHN0SNPkC", - "colab": {} - }, - "source": [ - "def preprocess(features, labels):\n", - " \n", - " # カテゴリ特徴量の処理\n", - " for feature in CATEGORIES.keys():\n", - " features[feature] = process_categorical_data(features[feature],\n", - " CATEGORIES[feature])\n", - "\n", - " # 連続特徴量の処理\n", - " for feature in MEANS.keys():\n", - " features[feature] = process_continuous_data(features[feature],\n", - " MEANS[feature])\n", - " \n", - " # 特徴量を1つのテンソルに組み立てる\n", - " features = tf.concat([features[column] for column in FEATURE_COLUMNS], 1)\n", - " \n", - " return features, labels\n", - "\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "34K5ESbYnkg4" - }, - "source": [ - "次に、 `tf.Dataset.map` 関数を使って適用し、過学習を防ぐためにデータセットをシャッフルします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7M5km0f_1pVp", - "colab": {} - }, - "source": [ - "train_data = raw_train_data.map(preprocess).shuffle(500)\n", - "test_data = raw_test_data.map(preprocess)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IQOWatzRr2aF" - }, - "source": [ - "サンプル1個がどうなっているか見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gc1o9ZpCsGGM", - "colab": {} - }, - "source": [ - "examples, labels = next(iter(train_data))\n", - "\n", - "examples, labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aJnOromrse57" - }, - "source": [ - "このサンプルは、（バッチサイズである）12個のアイテムをもつ2次元の配列からできています。アイテムそれぞれは、元の CSV ファイルの1行を表しています。ラベルは12個の値をもつ1次元のテンソルです。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DlF_omQqtnOP" - }, - "source": [ - "## モデルの構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lQoFh16LxtT_" - }, - "source": [ - "この例では、[Keras Functional API](https://www.tensorflow.org/guide/keras/functional) を使用し、単純なモデルを構築するために `get_model` コンストラクタでラッピングしています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JDM3FIgHNCW3", - "colab": {} - }, - "source": [ - "def get_model(input_dim, hidden_units=[100]):\n", - " \"\"\"複数の層を持つ Keras モデルを作成\n", - "\n", - " 引数:\n", - " input_dim: (int) バッチ中のアイテムの形状\n", - " labels_dim: (int) ラベルの形状\n", - " hidden_units: [int] DNN の層のサイズ（入力層が先）\n", - " learning_rate: (float) オプティマイザの学習率\n", - " \n", - " 戻り値:\n", - " Keras モデル\n", - " \"\"\"\n", - "\n", - " inputs = tf.keras.Input(shape=(input_dim,))\n", - " x = inputs\n", - "\n", - " for units in hidden_units:\n", - " x = tf.keras.layers.Dense(units, activation='relu')(x)\n", - " outputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n", - "\n", - " model = tf.keras.Model(inputs, outputs)\n", - " \n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ce9PRb_LzFpm" - }, - "source": [ - "`get_model` コンストラクタは入力データの形状（バッチサイズを除く）を知っている必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qX-DU34ZuKJX", - "colab": {} - }, - "source": [ - "input_shape, output_shape = train_data.output_shapes\n", - "\n", - "input_dimension = input_shape.dims[1] # [0] はバッチサイズ" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hPdtI2ie0lEZ" - }, - "source": [ - "## 訓練、評価、そして予測" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8gvw1RE9zXkD" - }, - "source": [ - "これでモデルをインスタンス化し、訓練することができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Q_nm28IzNDTO", - "colab": {} - }, - "source": [ - "model = get_model(input_dimension)\n", - "model.compile(\n", - " loss='binary_crossentropy',\n", - " optimizer='adam',\n", - " metrics=['accuracy'])\n", - "\n", - "model.fit(train_data, epochs=20)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QyDMgBurzqQo" - }, - "source": [ - "モデルの訓練が終わったら、`test_data` データセットでの正解率をチェックできます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eB3R3ViVONOp", - "colab": {} - }, - "source": [ - "test_loss, test_accuracy = model.evaluate(test_data)\n", - "\n", - "print('\\n\\nTest Loss {}, Test Accuracy {}'.format(test_loss, test_accuracy))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sTrn_pD90gdJ" - }, - "source": [ - "単一のバッチ、または、バッチからなるデータセットのラベルを推論する場合には、`tf.keras.Model.predict` を使います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qwcx74F3ojqe", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_data)\n", - "\n", - "# 結果のいくつかを表示\n", - "for prediction, survived in zip(predictions[:10], list(test_data)[0][1][:10]):\n", - " print(\"Predicted survival: {:.2%}\".format(prediction[0]),\n", - " \" | Actual outcome: \",\n", - " (\"SURVIVED\" if bool(survived) else \"DIED\"))\n" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ja/tutorials/load_data/images.ipynb b/site/ja/tutorials/load_data/images.ipynb deleted file mode 100644 index 4530633621f..00000000000 --- a/site/ja/tutorials/load_data/images.ipynb +++ /dev/null @@ -1,1673 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "images.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mt9dL5dIir8X" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ufPx7EiCiqgR", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ucMoYase6URl" - }, - "source": [ - "# tf.dataを使って画像をロードする" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_Wwu5SXZmEkB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RSywPQ2n736s" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oxw4WahM7DU9" - }, - "source": [ - "このチュートリアルでは、'tf.data' を使って画像データセットをロードする簡単な例を示します。\n", - "\n", - "このチュートリアルで使用するデータセットは、クラスごとに別々のディレクトリに別れた形で配布されています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hoQQiZDB6URn" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DHz3JONNEHlj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KT6CcaqgQewg", - "colab": {} - }, - "source": [ - "AUTOTUNE = tf.data.experimental.AUTOTUNE" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rxndJHNC8YPM" - }, - "source": [ - "## データセットのダウンロードと検査" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wO0InzL66URu" - }, - "source": [ - "### 画像の取得\n", - "\n", - "訓練を始める前に、ネットワークに認識すべき新しいクラスを教えるために画像のセットが必要です。最初に使うためのクリエイティブ・コモンズでライセンスされた花の画像のアーカイブを作成してあります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rN-Pc6Zd6awg", - "colab": {} - }, - "source": [ - "import pathlib\n", - "data_root_orig = tf.keras.utils.get_file(origin='https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", - " fname='flower_photos', untar=True)\n", - "data_root = pathlib.Path(data_root_orig)\n", - "print(data_root)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rFkFK74oO--g" - }, - "source": [ - "218MB をダウンロードすると、花の画像のコピーが使えるようになっているはずです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7onR_lWE7Njj", - "colab": {} - }, - "source": [ - "for item in data_root.iterdir():\n", - " print(item)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4yYX3ZRqGOuq", - "colab": {} - }, - "source": [ - "import random\n", - "all_image_paths = list(data_root.glob('*/*'))\n", - "all_image_paths = [str(path) for path in all_image_paths]\n", - "random.shuffle(all_image_paths)\n", - "\n", - "image_count = len(all_image_paths)\n", - "image_count" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t_BbYnLjbltQ", - "colab": {} - }, - "source": [ - "all_image_paths[:10]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkM-IpB-6URx" - }, - "source": [ - "### 画像の検査\n", - "\n", - "扱っている画像について知るために、画像のいくつかを見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wNGateQJ6UR1", - "colab": {} - }, - "source": [ - "import os\n", - "attributions = (data_root/\"LICENSE.txt\").open(encoding='utf-8').readlines()[4:]\n", - "attributions = [line.split(' CC-BY') for line in attributions]\n", - "attributions = dict(attributions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jgowG2xu88Io", - "colab": {} - }, - "source": [ - "import IPython.display as display\n", - "\n", - "def caption_image(image_path):\n", - " image_rel = pathlib.Path(image_path).relative_to(data_root)\n", - " return \"Image (CC BY 2.0) \" + ' - '.join(attributions[str(image_rel)].split(' - ')[:-1])\n", - " " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YIjLi-nX0txI", - "colab": {} - }, - "source": [ - "for n in range(3):\n", - " image_path = random.choice(all_image_paths)\n", - " display.display(display.Image(image_path))\n", - " print(caption_image(image_path))\n", - " print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OaNOr-co3WKk" - }, - "source": [ - "### 各画像のラベルの決定" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-weOQpDw2Jnu" - }, - "source": [ - "ラベルを一覧してみます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ssUZ7Qh96UR3", - "colab": {} - }, - "source": [ - "label_names = sorted(item.name for item in data_root.glob('*/') if item.is_dir())\n", - "label_names" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9l_JEBql2OzS" - }, - "source": [ - "ラベルにインデックスを割り当てます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y8pCV46CzPlp", - "colab": {} - }, - "source": [ - "label_to_index = dict((name, index) for index,name in enumerate(label_names))\n", - "label_to_index" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VkXsHg162T9F" - }, - "source": [ - "ファイルとラベルのインデックスの一覧を作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q62i1RBP4Q02", - "colab": {} - }, - "source": [ - "all_image_labels = [label_to_index[pathlib.Path(path).parent.name]\n", - " for path in all_image_paths]\n", - "\n", - "print(\"First 10 labels indices: \", all_image_labels[:10])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "i5L09icm9iph" - }, - "source": [ - "### 画像の読み込みと整形" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbqqRUS79ooq" - }, - "source": [ - "TensorFlow には画像を読み込んで処理するために必要なツールが備わっています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jQZdySHvksOu", - "colab": {} - }, - "source": [ - "img_path = all_image_paths[0]\n", - "img_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2t2h2XCcmK1Y" - }, - "source": [ - "以下は生のデータです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LJfkyC_Qkt7A", - "colab": {} - }, - "source": [ - "img_raw = tf.io.read_file(img_path)\n", - "print(repr(img_raw)[:100]+\"...\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "opN8AVc8mSbz" - }, - "source": [ - "画像のテンソルにデコードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tm0tdrlfk0Bb", - "colab": {} - }, - "source": [ - "img_tensor = tf.image.decode_image(img_raw)\n", - "\n", - "print(img_tensor.shape)\n", - "print(img_tensor.dtype)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3k-Of2Tfmbeq" - }, - "source": [ - "モデルに合わせてリサイズします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XFpz-3_vlJgp", - "colab": {} - }, - "source": [ - "img_final = tf.image.resize(img_tensor, [192, 192])\n", - "img_final = img_final/255.0\n", - "print(img_final.shape)\n", - "print(img_final.numpy().min())\n", - "print(img_final.numpy().max())\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aCsAa4Psl4AQ" - }, - "source": [ - "このあと使用するために、簡単な関数にまとめます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HmUiZJNU73vA", - "colab": {} - }, - "source": [ - "def preprocess_image(image):\n", - " image = tf.image.decode_jpeg(image, channels=3)\n", - " image = tf.image.resize(image, [192, 192])\n", - " image /= 255.0 # normalize to [0,1] range\n", - "\n", - " return image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "einETrJnO-em", - "colab": {} - }, - "source": [ - "def load_and_preprocess_image(path):\n", - " image = tf.io.read_file(path)\n", - " return preprocess_image(image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3brWQcdtz78y", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "image_path = all_image_paths[0]\n", - "label = all_image_labels[0]\n", - "\n", - "plt.imshow(load_and_preprocess_image(img_path))\n", - "plt.grid(False)\n", - "plt.xlabel(caption_image(img_path))\n", - "plt.title(label_names[label].title())\n", - "print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n2TCr1TQ8pA3" - }, - "source": [ - "## `tf.data.Dataset`の構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6H9Z5Mq63nSH" - }, - "source": [ - "### 画像のデータセット" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GN-s04s-6Luq" - }, - "source": [ - "`tf.data.Dataset` を構築するもっとも簡単な方法は、`from_tensor_slices` メソッドを使うことです。\n", - "\n", - "文字列の配列をスライスすると、文字列のデータセットが出来上がります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6oRPG3Jz3ie_", - "colab": {} - }, - "source": [ - "path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uML4JeMmIAvO" - }, - "source": [ - "`shapes` と `types` は、データセット中のそれぞれのアイテムの内容を示しています。この場合には、バイナリ文字列のスカラーのセットです。 " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mIsNflFbIK34", - "colab": {} - }, - "source": [ - "print(path_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZjyGcM8OwBJ2" - }, - "source": [ - "`preprocess_image` をファイルパスのデータセットにマップすることで、画像を実行時にロードし整形する新しいデータセットを作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "D1iba6f4khu-", - "colab": {} - }, - "source": [ - "image_ds = path_ds.map(load_and_preprocess_image, num_parallel_calls=AUTOTUNE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JLUPs2a-lEEJ", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.figure(figsize=(8,8))\n", - "for n,image in enumerate(image_ds.take(4)):\n", - " plt.subplot(2,2,n+1)\n", - " plt.imshow(image)\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.xlabel(caption_image(all_image_paths[n]))\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P6FNqPbxkbdx" - }, - "source": [ - "### `(image, label)`のペアのデータセット" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YgvrWLKG67-x" - }, - "source": [ - "おなじ `from_tensor_slices` メソッドを使ってラベルのデータセットを作ることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AgBsAiV06udj", - "colab": {} - }, - "source": [ - "label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(all_image_labels, tf.int64))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HEsk5nN0vyeX", - "colab": {} - }, - "source": [ - "for label in label_ds.take(10):\n", - " print(label_names[label.numpy()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jHjgrEeTxyYz" - }, - "source": [ - "これらのデータセットはおなじ順番なので、zip することで `(image, label)` というペアのデータセットができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AOEWNMdQwsbN", - "colab": {} - }, - "source": [ - "image_label_ds = tf.data.Dataset.zip((image_ds, label_ds))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yA2F09SJLMuM" - }, - "source": [ - "新しいデータセットの `shapes` と `types` は、それぞれのフィールドを示すシェイプと型のタプルです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DuVYNinrLL-N", - "colab": {} - }, - "source": [ - "print(image_label_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2WYMikoPWOQX" - }, - "source": [ - "注： `all_image_labels` や `all_image_paths` のような配列がある場合、 `tf.data.dataset.Dataset.zip` メソッドの代わりとなるのは、配列のペアをスライスすることです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HOFwZI-2WhzV", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.from_tensor_slices((all_image_paths, all_image_labels))\n", - "\n", - "# The tuples are unpacked into the positional arguments of the mapped function\n", - "# タプルは展開され、マップ関数の位置引数に割り当てられます\n", - "def load_and_preprocess_from_path_label(path, label):\n", - " return load_and_preprocess_image(path), label\n", - "\n", - "image_label_ds = ds.map(load_and_preprocess_from_path_label)\n", - "image_label_ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vYGCgJuR_9Qp" - }, - "source": [ - "### 基本的な訓練手法" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wwZavzgsIytz" - }, - "source": [ - "このデータセットを使ってモデルの訓練を行うには、データが\n", - "\n", - "* よくシャッフルされ\n", - "* バッチ化され\n", - "* 限りなく繰り返され\n", - "* バッチが出来るだけ早く利用できる\n", - "\n", - "ことが必要です。\n", - "\n", - "これらの特性は `tf.data` APIを使えば簡単に付け加えることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uZmZJx8ePw_5", - "colab": {} - }, - "source": [ - "BATCH_SIZE = 32\n", - "\n", - "# シャッフルバッファのサイズをデータセットとおなじに設定することで、データが完全にシャッフルされる\n", - "# ようにできます。\n", - "ds = image_label_ds.shuffle(buffer_size=image_count)\n", - "ds = ds.repeat()\n", - "ds = ds.batch(BATCH_SIZE)\n", - "# `prefetch`を使うことで、モデルの訓練中にバックグラウンドでデータセットがバッチを取得できます。\n", - "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6JsM-xHiFCuW" - }, - "source": [ - "注意すべきことがいくつかあります。\n", - "\n", - "1. 順番が重要です。\n", - "\n", - " * `.repeat` の前に `.shuffle` すると、エポックの境界を越えて要素がシャッフルされます。（ほかの要素がすべて出現する前に2回出現する要素があるかもしれません）\n", - " * `.batch` の後に `.shuffle` すると、バッチの順番がシャッフルされますが、要素がバッチを越えてシャッフルされることはありません。\n", - "\n", - "1. 完全なシャッフルのため、 `buffer_size` をデータセットとおなじサイズに設定しています。データセットのサイズ未満の場合、値が大きいほど良くランダム化されますが、より多くのメモリーを使用します。\n", - "\n", - "1. シャッフルバッファがいっぱいになってから要素が取り出されます。そのため、大きな `buffer_size` が `Dataset` を使い始める際の遅延の原因になります。\n", - "\n", - "1. シャッフルされたデータセットは、シャッフルバッファが完全に空になるまでデータセットが終わりであることを伝えません。 `.repeat` によって `Dataset` が再起動されると、シャッフルバッファが一杯になるまでもう一つの待ち時間が発生します。\n", - "\n", - "最後の問題は、 `tf.data.Dataset.apply` メソッドを、融合された `tf.data.experimental.shuffle_and_repeat` 関数と組み合わせることで対処できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ocr6PybXNDoO", - "colab": {} - }, - "source": [ - "ds = image_label_ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE)\n", - "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GBBZMSuAmQVL" - }, - "source": [ - "### データセットをモデルにつなぐ\n", - "\n", - "`tf.keras.applications`からMobileNet v2のコピーを取得します。\n", - "\n", - "これを簡単な転移学習のサンプルに使用します。\n", - "\n", - "MobileNetの重みを訓練不可に設定します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KbJrXn9omO_g", - "colab": {} - }, - "source": [ - "mobile_net = tf.keras.applications.MobileNetV2(input_shape=(192, 192, 3), include_top=False)\n", - "mobile_net.trainable=False" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y7NVWiLF3Vbf" - }, - "source": [ - "このモデルは、入力が `[-1,1]` の範囲に正規化されていることを想定しています。\n", - "\n", - "```\n", - "help(keras_applications.mobilenet_v2.preprocess_input)\n", - "```\n", - "\n", - "
    \n",
-        "...\n",
-        "This function applies the \"Inception\" preprocessing which converts\n",
-        "the RGB values from [0, 255] to [-1, 1] \n",
-        "...\n",
-        "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CboYya6LmdQI" - }, - "source": [ - "このため、データをMobileNetモデルに渡す前に、入力を`[0,1]`の範囲から`[-1,1]`の範囲に変換する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SNOkHUGv3FYq", - "colab": {} - }, - "source": [ - "def change_range(image,label):\n", - " return 2*image-1, label\n", - "\n", - "keras_ds = ds.map(change_range)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QDzZ3Nye5Rpv" - }, - "source": [ - "MobileNetは画像ごとに `6x6` の特徴量の空間を返します。\n", - "\n", - "バッチを１つ渡してみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OzAhGkEK6WuE", - "colab": {} - }, - "source": [ - "# シャッフルバッファがいっぱいになるまで、データセットは何秒かかかります。\n", - "image_batch, label_batch = next(iter(keras_ds))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LcFdiWpO5WbV", - "colab": {} - }, - "source": [ - "feature_map_batch = mobile_net(image_batch)\n", - "print(feature_map_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vrbjEvaC5XmU" - }, - "source": [ - "MobileNet をラップしたモデルを作り、出力層である `tf.keras.layers.Dense` の前に、`tf.keras.layers.GlobalAveragePooling2D` で空間の軸にそって平均値を求めます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X0ooIU9fNjPJ", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " mobile_net,\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(len(label_names))])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "foQYUJs97V4V" - }, - "source": [ - "期待したとおりの形状の出力が得られます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1nwYxvpj7ZEf", - "colab": {} - }, - "source": [ - "logit_batch = model(image_batch).numpy()\n", - "\n", - "print(\"min logit:\", logit_batch.min())\n", - "print(\"max logit:\", logit_batch.max())\n", - "print()\n", - "\n", - "print(\"Shape:\", logit_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pFc4I_J2nNOJ" - }, - "source": [ - "訓練手法を記述するためにモデルをコンパイルします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZWGqLEWYRNvv", - "colab": {} - }, - "source": [ - "model.compile(optimizer=tf.keras.optimizers.Adam(), \n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=[\"accuracy\"])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tF1mO6haBOSd" - }, - "source": [ - "訓練可能な変数は2つ、全結合層の `weights` と `bias` です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pPQ5yqyKBJMm", - "colab": {} - }, - "source": [ - "len(model.trainable_variables) " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kug5Wg66UJjl", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f_glpYZ-nYC_" - }, - "source": [ - "モデルを訓練します。\n", - "\n", - "普通は、エポックごとの本当のステップ数を指定しますが、ここではデモの目的なので3ステップだけとします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AnXPRNWoTypI", - "colab": {} - }, - "source": [ - "steps_per_epoch=tf.math.ceil(len(all_image_paths)/BATCH_SIZE).numpy()\n", - "steps_per_epoch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q_8sabaaSGAp", - "colab": {} - }, - "source": [ - "model.fit(ds, epochs=1, steps_per_epoch=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UMVnoBcG_NlQ" - }, - "source": [ - "## 性能\n", - "\n", - "注：このセクションでは性能の向上に役立ちそうな簡単なトリックをいくつか紹介します。詳しくは、[Input Pipeline Performance](https://www.tensorflow.org/guide/performance/datasets) を参照してください。\n", - "\n", - "上記の単純なパイプラインは、エポックごとにそれぞれのファイルを一つずつ読み込みます。これは、CPU を使ったローカルでの訓練では問題になりませんが、GPU を使った訓練では十分ではなく、いかなる分散訓練でも使うべきではありません。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oNmQqgGhLWie" - }, - "source": [ - "調査のため、まず、データセットの性能をチェックする簡単な関数を定義します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_gFVe1rp_MYr", - "colab": {} - }, - "source": [ - "import time\n", - "default_timeit_steps = 2*steps_per_epoch+1\n", - "\n", - "def timeit(ds, steps=default_timeit_steps):\n", - " overall_start = time.time()\n", - " # Fetch a single batch to prime the pipeline (fill the shuffle buffer),\n", - " # before starting the timer\n", - " it = iter(ds.take(steps+1))\n", - " next(it)\n", - "\n", - " start = time.time()\n", - " for i,(images,labels) in enumerate(it):\n", - " if i%10 == 0:\n", - " print('.',end='')\n", - " print()\n", - " end = time.time()\n", - "\n", - " duration = end-start\n", - " print(\"{} batches: {} s\".format(steps, duration))\n", - " print(\"{:0.5f} Images/s\".format(BATCH_SIZE*steps/duration))\n", - " print(\"Total time: {}s\".format(end-overall_start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TYiOr4vdLcNX" - }, - "source": [ - "現在のデータセットの性能は次のとおりです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZDxLwMJOReVe", - "colab": {} - }, - "source": [ - "ds = image_label_ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjouTJadRxyp", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HsLlXMO7EWBR" - }, - "source": [ - "### キャッシュ" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lV1NOn2zE2lR" - }, - "source": [ - "`tf.data.Dataset.cache` を使うと、エポックを越えて計算結果を簡単にキャッシュできます。特に、データがメモリに収まるときには効果的です。\n", - "\n", - "ここでは、画像が前処理（デコードとリサイズ）された後でキャッシュされます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qj_U09xpDvOg", - "colab": {} - }, - "source": [ - "ds = image_label_ds.cache()\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rdxpvQ7VEo3y", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "usIv7MqqZQps" - }, - "source": [ - "メモリキャッシュを使う際の欠点のひとつは、実行の都度キャッシュを再構築しなければならないことです。このため、データセットがスタートするたびにおなじだけ起動のための遅延が発生します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eKX6ergKb_xd", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jUzpG4lYNkN-" - }, - "source": [ - "データがメモリに収まらない場合には、キャッシュファイルを使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vIvF8K4GMq0g", - "colab": {} - }, - "source": [ - "ds = image_label_ds.cache(filename='./cache.tf-data')\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(1)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eTIj6IOmM4yA", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qqo3dyB0Z4t2" - }, - "source": [ - "キャッシュファイルには、キャッシュを再構築することなくデータセットを再起動できるという利点もあります。2回めがどれほど早いか見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hZhVdR8MbaUj", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WqOVlf8tFrDU" - }, - "source": [ - "### TFRecord ファイル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y1llOTwWFzmR" - }, - "source": [ - "#### 生の画像データ\n", - "\n", - "TFRecord ファイルは、バイナリの大きなオブジェクトのシーケンスを保存するための単純なフォーマットです。複数のサンプルをおなじファイルに詰め込むことで、TensorFlow は複数のサンプルを一度に読み込むことができます。これは、特に GCS のようなリモートストレージサービスを使用する際の性能にとって重要です。\n", - "\n", - "最初に、生の画像データから TFRecord ファイルを構築します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EqtARqKuHQLu", - "colab": {} - }, - "source": [ - "image_ds = tf.data.Dataset.from_tensor_slices(all_image_paths).map(tf.io.read_file)\n", - "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", - "tfrec.write(image_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "flR2GXWFKcO1" - }, - "source": [ - "次に、TFRecord ファイルを読み込み、以前定義した `preprocess_image` 関数を使って画像のデコード/リフォーマットを行うデータセットを構築します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "j9PVUL2SFufn", - "colab": {} - }, - "source": [ - "image_ds = tf.data.TFRecordDataset('images.tfrec').map(preprocess_image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cRp1eZDRKzyN" - }, - "source": [ - "これを、前に定義済みのラベルデータセットと zip し、期待どおりの `(image,label)` のペアを得ます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7XI_nDU2KuhS", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.zip((image_ds, label_ds))\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3ReSapoPK22E", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wb7VyoKNOMms" - }, - "source": [ - "これは、`cache` バージョンよりも低速です。前処理をキャッシュしていないからです。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NF9W-CTKkM-f" - }, - "source": [ - "#### シリアライズしたテンソル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J9HzljSPkxt0" - }, - "source": [ - "前処理を TFRecord ファイルに保存するには、前やったように前処理した画像のデータセットを作ります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OzS0Azukkjyw", - "colab": {} - }, - "source": [ - "paths_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)\n", - "image_ds = paths_ds.map(load_and_preprocess_image)\n", - "image_ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "onWOwLpYlzJQ" - }, - "source": [ - "`.jpeg` 文字列のデータセットではなく、これはテンソルのデータセットです。\n", - "\n", - "これを TFRecord ファイルにシリアライズするには、まず、テンソルのデータセットを文字列のデータセットに変換します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xxZSwnRllyf0", - "colab": {} - }, - "source": [ - "ds = image_ds.map(tf.io.serialize_tensor)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "w9N6hJWAkKPC", - "colab": {} - }, - "source": [ - "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", - "tfrec.write(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OlFc9dJSmcx0" - }, - "source": [ - "前処理をキャッシュしたことにより、データは TFRecord ファイルから非常に効率的にロードできます。テンソルを使用する前にデシリアライズすることを忘れないでください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BsqFyTBFmSCZ", - "colab": {} - }, - "source": [ - "ds = tf.data.TFRecordDataset('images.tfrec')\n", - "\n", - "def parse(x):\n", - " result = tf.io.parse_tensor(x, out_type=tf.float32)\n", - " result = tf.reshape(result, [192, 192, 3])\n", - " return result\n", - "\n", - "ds = ds.map(parse, num_parallel_calls=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OPs_sLV9pQg5" - }, - "source": [ - "次にラベルを追加し、以前とおなじような標準的な処理を適用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XYxBwaLYnGop", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.zip((ds, label_ds))\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W8X6RmGan1-P", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ja/tutorials/load_data/numpy.ipynb b/site/ja/tutorials/load_data/numpy.ipynb deleted file mode 100644 index b103d20fe80..00000000000 --- a/site/ja/tutorials/load_data/numpy.ipynb +++ /dev/null @@ -1,320 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "numpy.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pixDvex9KBqt" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "K16pBM8mKK7a", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TfRdquslKbO3" - }, - "source": [ - "# tf.data を使って NumPy データをロードする" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-uq3F0ggKlZb" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RSywPQ2n736s" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-0tqX8qkXZEj" - }, - "source": [ - "このチュートリアルでは、NumPy 配列から `tf.data.Dataset` にデータを読み込む例を示します。\n", - "\n", - "この例では、MNIST データセットを `.npz` ファイルから読み込みますが、 NumPy 配列がどこに入っているかは重要ではありません。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-Ze5IBx9clLB" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "D1gtCQrnNk6b", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "k6J3JzK5NxQ6", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import tensorflow as tf\n", - "import tensorflow_datasets as tfds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "G0yWiN8-cpDb" - }, - "source": [ - "### `.npz` ファイルからのロード" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GLHNrFM6RWoM", - "colab": {} - }, - "source": [ - "DATA_URL = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz'\n", - "\n", - "path = tf.keras.utils.get_file('mnist.npz', DATA_URL)\n", - "with np.load(path) as data:\n", - " train_examples = data['x_train']\n", - " train_labels = data['y_train']\n", - " test_examples = data['x_test']\n", - " test_labels = data['y_test']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cCeCkvrDgCMM" - }, - "source": [ - "## `tf.data.Dataset` を使って NumPy 配列をロード" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tslB0tJPgB-2" - }, - "source": [ - "サンプルの配列と対応するラベルの配列があるとします。 `tf.data.Dataset.from_tensor_slices` にこれら2つの配列をタプルとして入力し、`tf.data.Dataset` を作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QN_8wwc5R7Qm", - "colab": {} - }, - "source": [ - "train_dataset = tf.data.Dataset.from_tensor_slices((train_examples, train_labels))\n", - "test_dataset = tf.data.Dataset.from_tensor_slices((test_examples, test_labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6Rco85bbkDfN" - }, - "source": [ - "## データセットの使用" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0dvl1uUukc4K" - }, - "source": [ - "### データセットのシャッフルとバッチ化" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GTXdRMPcSXZj", - "colab": {} - }, - "source": [ - "BATCH_SIZE = 64\n", - "SHUFFLE_BUFFER_SIZE = 100\n", - "\n", - "train_dataset = train_dataset.shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE)\n", - "test_dataset = test_dataset.batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w69Jl8k6lilg" - }, - "source": [ - "### モデルの構築と訓練" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Uhxr8py4DkDN", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer=tf.keras.optimizers.RMSprop(),\n", - " loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n", - " metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XLDzlPGgOHBx", - "colab": {} - }, - "source": [ - "model.fit(train_dataset, epochs=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2q82yN8mmKIE", - "colab": {} - }, - "source": [ - "model.evaluate(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ja/tutorials/load_data/pandas_dataframe.ipynb b/site/ja/tutorials/load_data/pandas_dataframe.ipynb deleted file mode 100644 index 4f588f7c41f..00000000000 --- a/site/ja/tutorials/load_data/pandas_dataframe.ipynb +++ /dev/null @@ -1,511 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "pandas_dataframe.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zwBCE43Cv3PH" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "fOad0I2cv569", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YQB7yiF6v9GR" - }, - "source": [ - "# tf.data を使って pandas の DataFrame をロードする" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oqa952X4wQKK" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YRulkZPqFXMn", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BCTbFIJAFEWE", - "colab_type": "text" - }, - "source": [ - "このチュートリアルでは、pandas のDataFrameをロードして、`tf.data.Dataset` にデータを読み込む例を示します。\n", - "\n", - "このチュートリアルは、クリーブランドクリニック財団（the Cleveland Clinic Foundation for Heart Disease）から提供された、小さな [データセット](https://archive.ics.uci.edu/ml/datasets/heart+Disease) を使っています。このデータセット（CSV）には数百行のデータが含まれています。行は各患者を、列はさまざまな属性を表しています。\n", - "\n", - "このデータを使って、患者が心臓病を罹患しているかどうかを判別予測することができます。なお、これは二値分類問題になります。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iiyC7HkqxlUD" - }, - "source": [ - "## pandas を使ってデータを読み込む" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5IoRbCA2n0_V", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import pandas as pd\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hn4XgljZFEWI", - "colab_type": "text" - }, - "source": [ - "heart データセットを含んだCSVをダウンロードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VS4w2LePn9g3", - "colab": {} - }, - "source": [ - "csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/applied-dl/heart.csv')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6BXRPD2-xtQ1" - }, - "source": [ - "pandas を使ってCSVを読み込みます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UEfJ8TcMpe-2", - "colab": {} - }, - "source": [ - "df = pd.read_csv(csv_file)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8FkK6QIRpjd4", - "colab": {} - }, - "source": [ - "df.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_MOAKz654CT5", - "colab": {} - }, - "source": [ - "df.dtypes" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Dg7Fu8FIFEWU", - "colab_type": "text" - }, - "source": [ - "dataframe 内で唯一の `object` 型である `thal` 列を離散値に変換します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LmCl5R5C2IKo", - "colab": {} - }, - "source": [ - "df['thal'] = pd.Categorical(df['thal'])\n", - "df['thal'] = df.thal.cat.codes" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s4XA1SNW2QyI", - "colab": {} - }, - "source": [ - "df.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WWRhH6r4xxQu" - }, - "source": [ - "## `tf.data.Dataset` を使ってデータをロードする" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3yERplFYFEWb", - "colab_type": "text" - }, - "source": [ - "`tf.data.Dataset.from_tensor_slices` メソッドを使って、pandas の dataframeから値を読み込みます。\n", - "\n", - "`tf.data.Dataset` を使う利点は、シンプルに使えて、かつ、大変効率的なデータパイプラインを構築できることです。詳しくは [loading data guide](https://www.tensorflow.org/guide/data) を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2wwhILm1ycSp", - "colab": {} - }, - "source": [ - "target = df.pop('target')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W6Yc-D3aqyBb", - "colab": {} - }, - "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((df.values, target.values))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "chEnp_Swsf0a", - "colab": {} - }, - "source": [ - "for feat, targ in dataset.take(5):\n", - " print ('Features: {}, Target: {}'.format(feat, targ))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "O11hO6nPFEWi", - "colab_type": "text" - }, - "source": [ - "`pd.Series` は `__array__` プロトコルを実装しているため、`np.array` や `tf.Tensor` を使うところでは、だいたいどこでも使うことができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GnpHHkpktl5y", - "colab": {} - }, - "source": [ - "tf.constant(df['thal'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9XLxRHS10Ylp" - }, - "source": [ - "データをシャッフルしてバッチ処理を行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R3dQ-83Ztsgl", - "colab": {} - }, - "source": [ - "train_dataset = dataset.shuffle(len(df)).batch(1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bB9C0XJkyQEk" - }, - "source": [ - "## モデルを作成して訓練する" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FQd9PcPRpkP4", - "colab": {} - }, - "source": [ - "def get_compiled_model():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", - " ])\n", - "\n", - " model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ybDzNUheqxJw", - "colab": {} - }, - "source": [ - "model = get_compiled_model()\n", - "model.fit(train_dataset, epochs=15)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d6V_6F_MBiG9" - }, - "source": [ - "## 特徴列の代替" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vIqjDd0PFEWu", - "colab_type": "text" - }, - "source": [ - "モデルへの入力に辞書型データを渡すことは、 `tf.keras.layers.Input` におなじ型の辞書を作成し、何らかの前処理を適用して、[functional api](../../guide/keras/functional.ipynb) を使ってスタッキングすることと同様に、簡単に行うことができます。これを [特徴列](../keras/feature_columns.ipynb) の替わりに使うことができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FwQ47_WmOBnY", - "colab": {} - }, - "source": [ - "inputs = {key: tf.keras.layers.Input(shape=(), name=key) for key in df.keys()}\n", - "x = tf.stack(list(inputs.values()), axis=-1)\n", - "\n", - "x = tf.keras.layers.Dense(10, activation='relu')(x)\n", - "output = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n", - "\n", - "model_func = tf.keras.Model(inputs=inputs, outputs=output)\n", - "\n", - "model_func.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_a6DP7VnFEWz", - "colab_type": "text" - }, - "source": [ - "`tf.data` を使うときに、pandas の DataFrame の列構造を保持する一番簡単な方法は、DataFrame を辞書型データに変換して、先頭を切り取ることです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wUjRKgEhPZqK", - "colab": {} - }, - "source": [ - "dict_slices = tf.data.Dataset.from_tensor_slices((df.to_dict('list'), target.values)).batch(16)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WWRaiwxeyA9Z", - "colab": {} - }, - "source": [ - "for dict_slice in dict_slices.take(1):\n", - " print (dict_slice)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8nTrfczNyKup", - "colab": {} - }, - "source": [ - "model_func.fit(dict_slices, epochs=15)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/load_data/text.ipynb b/site/ja/tutorials/load_data/text.ipynb deleted file mode 100644 index 44d74df0690..00000000000 --- a/site/ja/tutorials/load_data/text.ipynb +++ /dev/null @@ -1,670 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DweYe9FcbMK_" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AVV2e0XKbJeX", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sUtoed20cRJJ" - }, - "source": [ - "# tf.data を使ったテキストの読み込み" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1ap_W4aQcgNT" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RSywPQ2n736s" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NWeQAo0Ec_BL" - }, - "source": [ - "このチュートリアルでは、`tf.data.TextLineDataset` を使ってテキストファイルからサンプルを読み込む方法を例示します。`TextLineDataset` は、テキストファイルからデータセットを作成するために設計されています。この中では、元のテキストファイルの一行一行がサンプルです。これは、（たとえば、詩やエラーログのような）基本的に行ベースのテキストデータを扱うのに便利でしょう。\n", - "\n", - "このチュートリアルでは、おなじ作品であるホーマーのイリアッドの異なる 3 つの英語翻訳版を使い、テキスト 1 行から翻訳者を特定するモデルを訓練します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "baYFZMW_bJHh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "\n", - "import tensorflow as tf\n", - "\n", - "import tensorflow_datasets as tfds\n", - "import os" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YWVWjyIkffau" - }, - "source": [ - "3 つの翻訳のテキストは次のとおりです。\n", - "\n", - " - [William Cowper](https://en.wikipedia.org/wiki/William_Cowper) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/cowper.txt)\n", - "\n", - " - [Edward, Earl of Derby](https://en.wikipedia.org/wiki/Edward_Smith-Stanley,_14th_Earl_of_Derby) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/derby.txt)\n", - "\n", - "- [Samuel Butler](https://en.wikipedia.org/wiki/Samuel_Butler_%28novelist%29) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/butler.txt)\n", - "\n", - "このチュートリアルで使われているテキストファイルは、ヘッダ、フッタ、行番号、章のタイトルの削除など、いくつかの典型的な前処理を行ったものです。前処理後のファイルをダウンロードしましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4YlKQthEYlFw", - "colab": {} - }, - "source": [ - "DIRECTORY_URL = 'https://storage.googleapis.com/download.tensorflow.org/data/illiad/'\n", - "FILE_NAMES = ['cowper.txt', 'derby.txt', 'butler.txt']\n", - "\n", - "for name in FILE_NAMES:\n", - " text_dir = tf.keras.utils.get_file(name, origin=DIRECTORY_URL+name)\n", - " \n", - "parent_dir = os.path.dirname(text_dir)\n", - "\n", - "parent_dir" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "q3sDy6nuXoNp" - }, - "source": [ - "## テキストをデータセットに読み込む\n", - "\n", - "ファイルをイテレートし、それぞれを別々のデータセットに読み込みます。\n", - "\n", - "サンプルはそれぞれにラベル付けが必要なので、ラベル付け関数を適用するために `tf.data.Dataset.map` を使います。このメソッドは、データセット中のすべてのサンプルをイテレートし、(`example, label`)というペアを返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K0BjCOpOh7Ch", - "colab": {} - }, - "source": [ - "def labeler(example, index):\n", - " return example, tf.cast(index, tf.int64) \n", - "\n", - "labeled_data_sets = []\n", - "\n", - "for i, file_name in enumerate(FILE_NAMES):\n", - " lines_dataset = tf.data.TextLineDataset(os.path.join(parent_dir, file_name))\n", - " labeled_dataset = lines_dataset.map(lambda ex: labeler(ex, i))\n", - " labeled_data_sets.append(labeled_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M8PHK5J_cXE5" - }, - "source": [ - "ラベル付けの終わったデータセットを結合して一つのデータセットにし、シャッフルします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6jAeYkTIi9-2", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 50000\n", - "BATCH_SIZE = 64\n", - "TAKE_SIZE = 5000" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qd544E-Sh63L", - "colab": {} - }, - "source": [ - "all_labeled_data = labeled_data_sets[0]\n", - "for labeled_dataset in labeled_data_sets[1:]:\n", - " all_labeled_data = all_labeled_data.concatenate(labeled_dataset)\n", - " \n", - "all_labeled_data = all_labeled_data.shuffle(\n", - " BUFFER_SIZE, reshuffle_each_iteration=False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r4JEHrJXeG5k" - }, - "source": [ - "`tf.data.Dataset.take` と `print` を使って、`(example, label)` のペアがどのようなものかを見ることができます。`numpy` プロパティがそれぞれのテンソルの値を示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gywKlN0xh6u5", - "colab": {} - }, - "source": [ - "for ex in all_labeled_data.take(5):\n", - " print(ex)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rrpU2_sfDh0" - }, - "source": [ - "## テキスト行を数字にエンコードする\n", - "\n", - "機械学習モデルが扱うのは単語ではなくて数字であるため、文字列は数字のリストに変換する必要があります。このため、一意の単語を一意の数字にマッピングします。\n", - "\n", - "### ボキャブラリーの構築\n", - "\n", - "まず最初に、テキストをトークン化し、個々の一意な単語の集まりとして、ボキャブラリーを構築します。これを行うには、TensorFlow やPython を使ういくつかの方法があります。ここでは次のようにします。\n", - "\n", - "1. 各サンプルの `numpy` 値をイテレートします。\n", - "2. `tfds.features.text.Tokenizer` を使って、それをトークンに分割します。\n", - "3. 重複を排除するため、トークンを Python の集合に集約します。\n", - "4. あとで使用するため、ボキャブラリーのサイズを取得します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YkHtbGnDh6mg", - "colab": {} - }, - "source": [ - "tokenizer = tfds.features.text.Tokenizer()\n", - "\n", - "vocabulary_set = set()\n", - "for text_tensor, _ in all_labeled_data:\n", - " some_tokens = tokenizer.tokenize(text_tensor.numpy())\n", - " vocabulary_set.update(some_tokens)\n", - "\n", - "vocab_size = len(vocabulary_set)\n", - "vocab_size" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0W35VJqAh9zs" - }, - "source": [ - "### サンプルをエンコードする\n", - "\n", - "`vocabulary_set` を `tfds.features.text.TokenTextEncoder` に渡してエンコーダーを作成します。エンコーダーの `encode` メソッドは、テキスト文字列を引数にとり、整数のリストを返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gkxJIVAth6j0", - "colab": {} - }, - "source": [ - "encoder = tfds.features.text.TokenTextEncoder(vocabulary_set)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "v6S5Qyabi-vo" - }, - "source": [ - "1行だけにこれを適用し、出力がどの様になるか確かめることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jgxPZaxUuTbk", - "colab": {} - }, - "source": [ - "example_text = next(iter(all_labeled_data))[0].numpy()\n", - "print(example_text)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XoVpKR3qj5yb", - "colab": {} - }, - "source": [ - "encoded_example = encoder.encode(example_text)\n", - "print(encoded_example)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p9qHM0v8k_Mg" - }, - "source": [ - "次に、このエンコーダーを `tf.py_function` でラッピングして、データセットの `map` メソッドに渡し、データセットに適用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HcIQ7LOTh6eT", - "colab": {} - }, - "source": [ - "def encode(text_tensor, label):\n", - " encoded_text = encoder.encode(text_tensor.numpy())\n", - " return encoded_text, label\n", - "\n", - "def encode_map_fn(text, label):\n", - " return tf.py_function(encode, inp=[text, label], Tout=(tf.int64, tf.int64))\n", - "\n", - "all_encoded_data = all_labeled_data.map(encode_map_fn)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_YZToSXSm0qr" - }, - "source": [ - "## データセットを、テスト用と訓練用のバッチに分割する\n", - "\n", - "`tf.data.Dataset.take`と`tf.data.Dataset.skip`を使って、小さなテスト用データセットと、より大きな訓練用セットを作成します。\n", - "\n", - "モデルに渡す前に、データセットをバッチ化する必要があります。通常、バッチの中のサンプルはおなじサイズと形状である必要があります。しかし、これらのデータセットの中のサンプルはすべておなじサイズではありません。テキストの各行の単語数は異なっています。このため、（`batch`の代わりに）`tf.data.Dataset.padded_batch` メソッドを使ってサンプルをおなじサイズにパディングします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "r-rmbijQh6bf", - "colab": {} - }, - "source": [ - "train_data = all_encoded_data.skip(TAKE_SIZE).shuffle(BUFFER_SIZE)\n", - "train_data = train_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))\n", - "\n", - "test_data = all_encoded_data.take(TAKE_SIZE)\n", - "test_data = test_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xdz7SVwmqi1l" - }, - "source": [ - "もう、`test_data` と `train_data` は、(`example, label`)というペアのコレクションではなく、バッチのコレクションです。それぞれのバッチは、（**たくさんのサンプル**, **たくさんのラベル**）という配列のペアです。\n", - "\n", - "見てみましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kMslWfuwoqpB", - "colab": {} - }, - "source": [ - "sample_text, sample_labels = next(iter(test_data))\n", - "\n", - "sample_text[0], sample_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UI4I6_Sa0vWu" - }, - "source": [ - "（ゼロをパディングに使用した）新しいトークン番号を1つ導入したので、ボキャブラリーサイズは1つ増えています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IlD1Lli91vuc", - "colab": {} - }, - "source": [ - "vocab_size += 1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K8SUhGFNsmRi" - }, - "source": [ - "## モデルを構築する" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QJgI1pow2YR9", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wi0iiKLTKdoF" - }, - "source": [ - "最初の層は、整数表現を密なベクトル埋め込みに変換します。詳細は[単語埋め込み](../../tutorials/sequences/word_embeddings)のチュートリアルを参照ください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DR6-ctbY638P", - "colab": {} - }, - "source": [ - "model.add(tf.keras.layers.Embedding(vocab_size, 64))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_8OJOPohKh1q" - }, - "source": [ - "次の層は[Long Short-Term Memory](http://colah.github.io/posts/2015-08-Understanding-LSTMs/) 層です。この層により、モデルは単語をほかの単語の文脈の中で解釈します。LSTM の Bidirectional ラッパーにより、データポイントを、その前とその後のデータポイントとの関連で学習することができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x6rnq6DN_WUs", - "colab": {} - }, - "source": [ - "model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cdffbMr5LF1g" - }, - "source": [ - "最後に、一つ以上の全結合層があり、最後の層は出力層です。出力層はラベルすべての確率を生成します。もっと複雑なも確率の高いラベルが、モデルが予測するサンプルのラベルです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QTEaNSnLCsv5", - "colab": {} - }, - "source": [ - "# 1 つ以上の Dense 層\n", - "# `for` 行の中のリストを編集して、層のサイズの実験をしてください\n", - "for units in [64, 64]:\n", - " model.add(tf.keras.layers.Dense(units, activation='relu'))\n", - "\n", - "# 出力層 最初の引数はラベルの数\n", - "model.add(tf.keras.layers.Dense(3, activation='softmax'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zLHPU8q5DLi_" - }, - "source": [ - "最後にモデルをコンパイルします。ソフトマックスによるカテゴリー分類モデルでは、損失関数として `sparse_categorical_crossentropy` を使用します。ほかのオプティマイザを使うこともできますが、`adam` がよく使われます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pkTBUVO4h6Y5", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DM-HLo5NDhql" - }, - "source": [ - "## モデルを訓練する\n", - "\n", - "このモデルをこのデータに適用すると（約83%の）まともな結果が得られます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aLtO33tNh6V8", - "colab": {} - }, - "source": [ - "model.fit(train_data, epochs=3, validation_data=test_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KTPCYf_Jh6TH", - "colab": {} - }, - "source": [ - "eval_loss, eval_acc = model.evaluate(test_data)\n", - "\n", - "print('\\nEval loss: {}, Eval accuracy: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ja/tutorials/load_data/tfrecord.ipynb b/site/ja/tutorials/load_data/tfrecord.ipynb deleted file mode 100644 index 689e18c3180..00000000000 --- a/site/ja/tutorials/load_data/tfrecord.ipynb +++ /dev/null @@ -1,1273 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "tfrecord.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "pL--_KGdYoBz" - ], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pL--_KGdYoBz" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "both", - "colab_type": "code", - "id": "uBDvXpYzYnGj", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HQzaEQuJiW_d" - }, - "source": [ - "# TFRecords と `tf.Example` の使用法\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RSywPQ2n736s" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3pkUd_9IZCFO" - }, - "source": [ - "データの読み込みを効率的にするには、データをシリアライズし、連続的に読み込めるファイルのセット（各ファイルは 100-200MB）に保存することが有効です。データをネットワーク経由で流そうとする場合には、特にそうです。また、データの前処理をキャッシングする際にも役立ちます。\n", - "\n", - "TFRecord 形式は、バイナリレコードのシリーズを保存するための単純な形式です。\n", - "\n", - "[プロトコルバッファ](https://developers.google.com/protocol-buffers/) は、構造化データを効率的にシリアライズする、プラットフォームや言語に依存しないライブラリです。\n", - "\n", - "プロトコルメッセージは `.proto` という拡張子のファイルで表されます。メッセージの型を識別するもっとも簡単な方法です。\n", - "\n", - "`tf.Example` メッセージ（あるいはプロトコルバッファ）は、`{\"string\": value}` 形式のマッピングを表現する柔軟なメッセージ型です。これは、TensorFlow 用に設計され、[TFX](https://www.tensorflow.org/tfx/) のような上位レベルの API で共通に使用されています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ac83J0QxjhFt" - }, - "source": [ - "このノートブックでは、`tf.Example` メッセージの作成、解析と使用法をデモし、その後、`tf.Example` メッセージをシリアライズして `.tfrecord` に書き出し、その後読み取る方法を示します。\n", - "\n", - "注：こうした構造は有用ですが必ずそうしなければならなというものではありません。[`tf.data`](https://www.tensorflow.org/guide/datasets) を使っていて、それでもデータの読み込みが訓練のボトルネックであるという場合でなければ、既存のコードを TFRecords を使用するために変更する必要はありません。データセットの性能改善のヒントは、 [Data Input Pipeline Performance](https://www.tensorflow.org/guide/performance/datasets) を参照ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WkRreBf1eDVc" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ja7sezsmnXph", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import numpy as np\n", - "import IPython.display as display" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e5Kq88ccUWQV" - }, - "source": [ - "## `tf.Example`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VrdQHgvNijTi" - }, - "source": [ - "### `tf.Example` 用のデータ型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lZw57Qrn4CTE" - }, - "source": [ - "基本的には `tf.Example` は `{\"string\": tf.train.Feature}` というマッピングです。\n", - "\n", - "`tf.train.Feature` メッセージ型は次の3つの型のうち1つをとることができます。（[.proto file](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto) を参照）このほかの一般的なデータ型のほとんどは、強制的にこれらのうちの1つにすること可能です。\n", - "\n", - "1. `tf.train.BytesList` (次の型のデータを扱うことが可能)\n", - "\n", - " - `string`\n", - " - `byte` \n", - "\n", - "1. `tf.train.FloatList` (次の型のデータを扱うことが可能)\n", - "\n", - " - `float` (`float32`)\n", - " - `double` (`float64`)\n", - "\n", - "1. `tf.train.Int64List` (次の型のデータを扱うことが可能)\n", - "\n", - " - `bool`\n", - " - `enum`\n", - " - `int32`\n", - " - `uint32`\n", - " - `int64`\n", - " - `uint64`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_e3g9ExathXP" - }, - "source": [ - "通常の TensorFlow の型を `tf.Example` 互換の `tf.train.Feature` に変換するには、次のショートカット関数を使うことができます。\n", - "\n", - "どの関数も、1個のスカラー値を入力とし、上記の3つの `list` 型のうちの一つを含む `tf.train.Feature` を返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mbsPOUpVtYxA", - "colab": {} - }, - "source": [ - "# 下記の関数を使うと値を tf.Example と互換性の有る型に変換できる\n", - "\n", - "def _bytes_feature(value):\n", - " \"\"\"string / byte 型から byte_list を返す\"\"\"\n", - " if isinstance(value, type(tf.constant(0))):\n", - " value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n", - " return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n", - "\n", - "def _float_feature(value):\n", - " \"\"\"float / double 型から float_list を返す\"\"\"\n", - " return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n", - "\n", - "def _int64_feature(value):\n", - " \"\"\"bool / enum / int / uint 型から Int64_list を返す\"\"\"\n", - " return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wst0v9O8hgzy" - }, - "source": [ - "注：単純化のため、このサンプルではスカラー値の入力のみを扱っています。スカラー値ではない特徴を扱うもっとも簡単な方法は、`tf.serialize_tensor` を使ってテンソルをバイナリ文字列に変換する方法です。TensorFlow では文字列はスカラー値として扱います。バイナリ文字列をテンソルに戻すには、`tf.parse_tensor` を使用します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vsMbkkC8xxtB" - }, - "source": [ - "上記の関数の使用例を下記に示します。入力がさまざまな型であるのに対して、出力が標準化されていることに注目してください。入力が、強制変換できない型であった場合、例外が発生します。（例： `_int64_feature(1.0)` はエラーとなります。`1.0` が浮動小数点数であるためで、代わりに `_float_feature` 関数を使用すべきです）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hZzyLGr0u73y", - "colab": {} - }, - "source": [ - "print(_bytes_feature(b'test_string'))\n", - "print(_bytes_feature(u'test_bytes'.encode('utf-8')))\n", - "\n", - "print(_float_feature(np.exp(1)))\n", - "\n", - "print(_int64_feature(True))\n", - "print(_int64_feature(1))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nj1qpfQU5qmi" - }, - "source": [ - "主要なメッセージはすべて `.SerializeToString` を使ってバイナリ文字列にシリアライズすることができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5afZkORT5pjm", - "colab": {} - }, - "source": [ - "feature = _float_feature(np.exp(1))\n", - "\n", - "feature.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "laKnw9F3hL-W" - }, - "source": [ - "### `tf.Example` メッセージの作成" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b_MEnhxchQPC" - }, - "source": [ - "既存のデータから `tf.Example` を作成したいとします。実際には、データセットの出処はどこでもよいのですが、1件の観測記録から`tf.Example` メッセージを作る手順はおなじです。\n", - "\n", - "1. 観測記録それぞれにおいて、各値は上記の関数を使って3種類の互換性のある型からなる `tf.train.Feature` に変換する必要があります。\n", - "\n", - "1. 次に、特徴の名前を表す文字列と、#1 で作ったエンコード済みの特徴量を対応させたマップ（ディクショナリ）を作成します。\n", - "\n", - "1. #2 で作成したマップを[特徴量メッセージ](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto#L85)に変換します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4EgFQ2uHtchc" - }, - "source": [ - "このノートブックでは、NumPy を使ってデータセットを作成します。\n", - "\n", - "このデータセットには4つの特徴量があります。\n", - "- `False` または `True` を表す論理値。出現確率は等しいものとします。\n", - "- ランダムなバイト値。全体において一様であるとします。\n", - "- `[-10000, 10000)` の範囲から一様にサンプリングした整数値。\n", - "- 標準正規分布からサンプリングした浮動小数点数。\n", - "\n", - "サンプルは上記の分布から独立しておなじ様に分布した10,000件の観測記録からなるものとします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CnrguFAy3YQv", - "colab": {} - }, - "source": [ - "# データセットに含まれる観測結果の件数\n", - "n_observations = int(1e4)\n", - "\n", - "# ブール特徴量 False または True としてエンコードされている\n", - "feature0 = np.random.choice([False, True], n_observations)\n", - "\n", - "# 整数特徴量 -10000 から 10000 の間の乱数\n", - "feature1 = np.random.randint(0, 5, n_observations)\n", - "\n", - "# バイト特徴量\n", - "strings = np.array([b'cat', b'dog', b'chicken', b'horse', b'goat'])\n", - "feature2 = strings[feature1]\n", - "\n", - "# 浮動小数点数特徴量 標準正規分布から発生\n", - "feature3 = np.random.randn(n_observations)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aGrscehJr7Jd" - }, - "source": [ - "これらの特徴量は、`_bytes_feature`, `_float_feature`, `_int64_feature` のいずれかを使って `tf.Example` 互換の型に強制変換されます。その後、エンコード済みの特徴量から `tf.Example` メッセージを作成できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RTCS49Ij_kUw", - "colab": {} - }, - "source": [ - "def serialize_example(feature0, feature1, feature2, feature3):\n", - " \"\"\"\n", - " ファイル出力可能な tf.Example メッセージを作成する\n", - " \"\"\"\n", - " \n", - " # 特徴量名と tf.Example 互換データ型との対応ディクショナリを作成\n", - " \n", - " feature = {\n", - " 'feature0': _int64_feature(feature0),\n", - " 'feature1': _int64_feature(feature1),\n", - " 'feature2': _bytes_feature(feature2),\n", - " 'feature3': _float_feature(feature3),\n", - " }\n", - " \n", - " # tf.train.Example を用いて特徴メッセージを作成\n", - " \n", - " example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n", - " return example_proto.SerializeToString()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XftzX9CN_uGT" - }, - "source": [ - "たとえば、データセットに `[False, 4, bytes('goat'), 0.9876]` という1つの観測記録があるとします。`create_message()` を使うとこの観測記録から `tf.Example` メッセージを作成し印字できます。上記のように、観測記録一つ一つが `Features` メッセージとして書かれています。`tf.Example` [メッセージ](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88)は、この `Features` メッセージを包むラッパーに過ぎないことに注意してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N8BtSx2RjYcb", - "colab": {} - }, - "source": [ - "# データセットからの観測記録の例\n", - "\n", - "example_observation = []\n", - "\n", - "serialized_example = serialize_example(False, 4, b'goat', 0.9876)\n", - "serialized_example" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_pbGATlG6u-4" - }, - "source": [ - "メッセージをデコードするには、`tf.train.Example.FromString` メソッドを使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dGim-mEm6vit", - "colab": {} - }, - "source": [ - "example_proto = tf.train.Example.FromString(serialized_example)\n", - "example_proto" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y-Hjmee-fbLH" - }, - "source": [ - "## TFRecord フォーマットの詳細\n", - "\n", - "TFRecord ファイルにはレコードのシーケンスが含まれます。このファイルはシーケンシャル読み取りのみが可能です。\n", - "\n", - "それぞれのレコードには、データを格納するためのバイト文字列とデータ長、そして整合性チェックのための CRC32C（Castagnoli 多項式を使った 32 ビットの CRC ）ハッシュ値が含まれます。\n", - "\n", - "各レコードのフォーマットは\n", - "\n", - " uint64 長さ\n", - " uint32 長さのマスク済み crc32 ハッシュ値\n", - " byte data[長さ]\n", - " uint32 データのマスク済み crc32 ハッシュ値\n", - "\n", - "複数のレコードが結合されてファイルを構成します。CRC については[ここ](https://en.wikipedia.org/wiki/Cyclic_redundancy_check)に說明があります。CRC のマスクは下記のとおりです。\n", - "\n", - " masked_crc = ((crc >> 15) | (crc << 17)) + 0xa282ead8ul\n", - " \n", - "注：TFRecord ファイルを作るのに、`tf.Example` を使わなければならないということはありません。tf.Example は、ディクショナリをバイト文字列にシリアライズする方法の1つです。エンコードされた画像データや、（`tf.io.serialize_tensor` を使ってシリアライズされ、`tf.io.parse_tensor` で読み込まれる）シリアライズされたテンソルもあります。そのほかのオプションについては、`tf.io` モジュールを参照してください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LYnQzvAvfchQ" - }, - "source": [ - "## `tf.data` を使用した TFRecord ファイル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GmehkCCT81Ez" - }, - "source": [ - "`tf.data` モジュールには、TensorFlow でデータを読み書きするツールが含まれます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1FISEuz8ubu3" - }, - "source": [ - "### TFRecord ファイルの書き出し\n", - "\n", - "データをデータセットにするもっとも簡単な方法は `from_tensor_slices` メソッドです。\n", - "\n", - "配列に適用すると、このメソッドはスカラー値のデータセットを返します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mXeaukvwu5_-", - "colab": {} - }, - "source": [ - "tf.data.Dataset.from_tensor_slices(feature1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f-q0VKyZvcad" - }, - "source": [ - "配列のタプルに適用すると、タプルのデータセットが返されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "H5sWyu1kxnvg", - "colab": {} - }, - "source": [ - "features_dataset = tf.data.Dataset.from_tensor_slices((feature0, feature1, feature2, feature3))\n", - "features_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m1C-t71Nywze", - "colab": {} - }, - "source": [ - "# データセットから1つのサンプルだけを取り出すには `take(1)` を使います。\n", - "for f0,f1,f2,f3 in features_dataset.take(1):\n", - " print(f0)\n", - " print(f1)\n", - " print(f2)\n", - " print(f3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mhIe63awyZYd" - }, - "source": [ - "`Dataset` のそれぞれの要素に関数を適用するには、`tf.data.Dataset.map` メソッドを使用します。\n", - "\n", - "マップされる関数は TensorFlow のグラフモードで動作する必要があります。関数は `tf.Tensors` を処理し、返す必要があります。`create_example` のような非テンソル関数は、互換性のため `tf.py_func` でラップすることができます。\n", - "\n", - "`tf.py_func` を使用する際には、シェイプと型は取得できないため、指定する必要があります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "apB5KYrJzjPI", - "colab": {} - }, - "source": [ - "def tf_serialize_example(f0,f1,f2,f3):\n", - " tf_string = tf.py_function(\n", - " serialize_example, \n", - " (f0,f1,f2,f3), # 上記の関数にこれらの引数を渡す\n", - " tf.string) # 戻り値の型は tf.string\n", - " return tf.reshape(tf_string, ()) # 結果はスカラー" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "mckRzbHlfchm", - "colab_type": "code", - "colab": {} - }, - "source": [ - "tf_serialize_example(f0,f1,f2,f3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CrFZ9avE3HUF" - }, - "source": [ - "この関数をデータセットのそれぞれの要素に適用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VDeqYVbW3ww9", - "colab": {} - }, - "source": [ - "serialized_features_dataset = features_dataset.map(tf_serialize_example)\n", - "serialized_features_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "CRtx4Cjpfch2", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def generator():\n", - " for features in features_dataset:\n", - " yield serialize_example(*features)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "sDl1JG09fch4", - "colab_type": "code", - "colab": {} - }, - "source": [ - "serialized_features_dataset = tf.data.Dataset.from_generator(\n", - " generator, output_types=tf.string, output_shapes=())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "_ZVqJdH5fch6", - "colab_type": "code", - "colab": {} - }, - "source": [ - "serialized_features_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p6lw5VYpjZZC" - }, - "source": [ - "TFRecord ファイルに書き出します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vP1VgTO44UIE", - "colab": {} - }, - "source": [ - "filename = 'test.tfrecord'\n", - "writer = tf.data.experimental.TFRecordWriter(filename)\n", - "writer.write(serialized_features_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6aV0GQhV8tmp" - }, - "source": [ - "### TFRecord ファイルの読み込み" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o3J5D4gcSy8N" - }, - "source": [ - "`tf.data.TFRecordDataset` クラスを使って TFRecord ファイルを読み込むこともできます。\n", - "\n", - "`tf.data` を使って TFRecord ファイルを取り扱う際の詳細については、[こちら](https://www.tensorflow.org/guide/datasets#consuming_tfrecord_data)を参照ください。 \n", - "\n", - "`TFRecordDataset` を使うことは、入力データを標準化し、パフォーマンスを最適化するのに有用です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6OjX6UZl-bHC", - "colab": {} - }, - "source": [ - "filenames = [filename]\n", - "raw_dataset = tf.data.TFRecordDataset(filenames)\n", - "raw_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6_EQ9i2E_-Fz" - }, - "source": [ - "この時点で、データセットにはシリアライズされた `tf.train.Example` メッセージが含まれています。データセットをイテレートすると、スカラーの文字列テンソルが返ってきます。\n", - "\n", - "`.take` メソッドを使って最初の 10 レコードだけを表示します。\n", - "\n", - "注：`tf.data.Dataset` をイテレートできるのは、Eager Execution が有効になっている場合のみです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hxVXpLz_AJlm", - "colab": {} - }, - "source": [ - "for raw_record in raw_dataset.take(10):\n", - " print(repr(raw_record))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W-6oNzM4luFQ" - }, - "source": [ - "これらのテンソルは下記の関数でパースできます。\n", - "\n", - "注：ここでは、`feature_description` が必要です。データセットはグラフ実行を使用するため、この記述を使ってシェイプと型を構築するのです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zQjbIR1nleiy", - "colab": {} - }, - "source": [ - "# 特徴の記述\n", - "feature_description = {\n", - " 'feature0': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n", - " 'feature1': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n", - " 'feature2': tf.io.FixedLenFeature([], tf.string, default_value=''),\n", - " 'feature3': tf.io.FixedLenFeature([], tf.float32, default_value=0.0),\n", - "}\n", - "\n", - "def _parse_function(example_proto):\n", - " # 上記の記述を使って入力の tf.Example を処理\n", - " return tf.io.parse_single_example(example_proto, feature_description)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gWETjUqhEQZf" - }, - "source": [ - "あるいは、`tf.parse example` を使ってバッチ全体を一度にパースします。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AH73hav6Bnmg" - }, - "source": [ - "`tf.data.Dataset.map` メソッドを使って、データセットの各アイテムにこの関数を適用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ob7D-zmBm1w", - "colab": {} - }, - "source": [ - "parsed_dataset = raw_dataset.map(_parse_function)\n", - "parsed_dataset " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sNV-XclGnOvn" - }, - "source": [ - "Eager Execution を使ってデータセット中の観測記録を表示します。このデータセットには 10,000 件の観測記録がありますが、最初の 10 個だけ表示します。 \n", - "データは特徴量のディクショナリの形で表示されます。それぞれの項目は `tf.Tensor` であり、このテンソルの `numpy` 要素は特徴量を表します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x2LT2JCqhoD_", - "colab": {} - }, - "source": [ - "for parsed_record in parsed_dataset.take(10):\n", - " print(repr(raw_record))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cig9EodTlDmg" - }, - "source": [ - "ここでは、`tf.parse_example` が`tf.Example` のフィールドを通常のテンソルに展開しています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jyg1g3gU7DNn" - }, - "source": [ - "## tf.python_io を使った TFRecord ファイル" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3FXG3miA7Kf1" - }, - "source": [ - "`tf.python_io` モジュールには、TFRecord ファイルの読み書きのための純粋な Python 関数も含まれています。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CKn5uql2lAaN" - }, - "source": [ - "### TFRecord ファイルの書き出し" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LNW_FA-GQWXs" - }, - "source": [ - "次にこの 10,000 件の観測記録を `test.tfrecords` ファイルに出力します。観測記録はそれぞれ `tf.Example` メッセージに変換され、ファイルに出力されます。その後、`test.tfrecords` ファイルが作成されたことを確認することができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MKPHzoGv7q44", - "colab": {} - }, - "source": [ - "# `tf.Example` 観測記録をファイルに出力\n", - "with tf.io.TFRecordWriter(filename) as writer:\n", - " for i in range(n_observations):\n", - " example = serialize_example(feature0[i], feature1[i], feature2[i], feature3[i])\n", - " writer.write(example)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EjdFHHJMpUUo", - "colab": {} - }, - "source": [ - "!du -sh {filename}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wtQ7k0YWQ1cz" - }, - "source": [ - "### TFRecord ファイルの読み込み\n", - "\n", - "これらのシリアライズされたテンソルは、`tf.train.Example.ParseFromString` を使って簡単にパースできます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "36ltP9B8OezA", - "colab": {} - }, - "source": [ - "filenames = [filename]\n", - "raw_dataset = tf.data.TFRecordDataset(filenames)\n", - "raw_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "BpS-R4MLfcic", - "colab_type": "code", - "colab": {} - }, - "source": [ - "for raw_record in raw_dataset.take(1):\n", - " example = tf.train.Example()\n", - " example.ParseFromString(raw_record.numpy())\n", - " print(example)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S0tFDrwdoj3q" - }, - "source": [ - "## ウォークスルー： 画像データの読み書き" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rjN2LFxFpcR9" - }, - "source": [ - "以下は、TFRecord を使って画像データを読み書きする方法の例です。この例の目的は、データ（この場合は画像）を入力し、そのデータを TFRecord ファイルに書き込んで、再びそのファイルを読み込み、画像を表示するという手順を最初から最後まで示すことです。\n", - "\n", - "これは、たとえば、おなじ入力データセットを使って複数のモデルを構築するといった場合に役立ちます。画像データをそのまま保存する代わりに、TFRecord 形式に前処理しておき、その後の処理やモデル構築に使用することができます。\n", - "\n", - "まずは、雪の中の猫の[画像](https://commons.wikimedia.org/wiki/File:Felis_catus-cat_on_snow.jpg)と、ニューヨーク市にあるウイリアムズバーグ橋の [写真](https://upload.wikimedia.org/wikipedia/commons/f/fe/New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg)をダウンロードしましょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5Lk2qrKvN0yu" - }, - "source": [ - "### 画像の取得" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3a0fmwg8lHdF", - "colab": {} - }, - "source": [ - "cat_in_snow = tf.keras.utils.get_file('320px-Felis_catus-cat_on_snow.jpg', 'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg')\n", - "williamsburg_bridge = tf.keras.utils.get_file('194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7aJJh7vENeE4", - "colab": {} - }, - "source": [ - "display.display(display.Image(filename=cat_in_snow))\n", - "display.display(display.HTML('Image cc-by: Von.grzanka'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KkW0uuhcXZqA", - "colab": {} - }, - "source": [ - "display.display(display.Image(filename=williamsburg_bridge))\n", - "display.display(display.HTML('From Wikimedia'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VSOgJSwoN5TQ" - }, - "source": [ - "### TFRecord ファイルの書き出し" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Azx83ryQEU6T" - }, - "source": [ - "上記で行ったように、この特徴量を `tf.Example` と互換のデータ型にエンコードできます。この場合には、生の画像文字列を特徴として保存するだけではなく、縦、横のサイズにチャネル数、更に画像を保存する際に猫の画像と橋の画像を区別するための `label` 特徴量を付け加えます。猫の画像には `0` を、橋の画像には `1` を使うことにしましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kC4TS1ZEONHr", - "colab": {} - }, - "source": [ - "image_labels = {\n", - " cat_in_snow : 0,\n", - " williamsburg_bridge : 1,\n", - "}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c5njMSYNEhNZ", - "colab": {} - }, - "source": [ - "# 猫の画像を使った例\n", - "image_string = open(cat_in_snow, 'rb').read()\n", - "\n", - "label = image_labels[cat_in_snow]\n", - "\n", - "# 関連する特徴量のディクショナリを作成\n", - "def image_example(image_string, label):\n", - " image_shape = tf.image.decode_jpeg(image_string).shape\n", - "\n", - " feature = {\n", - " 'height': _int64_feature(image_shape[0]),\n", - " 'width': _int64_feature(image_shape[1]),\n", - " 'depth': _int64_feature(image_shape[2]),\n", - " 'label': _int64_feature(label),\n", - " 'image_raw': _bytes_feature(image_string),\n", - " }\n", - "\n", - " return tf.train.Example(features=tf.train.Features(feature=feature))\n", - "\n", - "for line in str(image_example(image_string, label)).split('\\n')[:15]:\n", - " print(line)\n", - "print('...')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2G_o3O9MN0Qx" - }, - "source": [ - "ご覧のように、すべての特徴量が `tf.Example` メッセージに保存されました。上記のコードを関数化し、このサンプルメッセージを `images.tfrecords` ファイルに書き込みます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qcw06lQCOCZU", - "colab": {} - }, - "source": [ - "# 生の画像を images.tfrecords ファイルに書き出す\n", - "# まず、2つの画像を tf.Example メッセージに変換し、\n", - "# 次に .tfrecords ファイルに書き出す\n", - "record_file = 'images.tfrecords'\n", - "with tf.io.TFRecordWriter(record_file) as writer:\n", - " for filename, label in image_labels.items():\n", - " image_string = open(filename, 'rb').read()\n", - " tf_example = image_example(image_string, label)\n", - " writer.write(tf_example.SerializeToString())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yJrTe6tHPCfs", - "colab": {} - }, - "source": [ - "!du -sh {record_file}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jJSsCkZLPH6K" - }, - "source": [ - "### TFRecord ファイルの読み込み\n", - "\n", - "これで、`images.tfrecords` ファイルができました。このファイルの中のレコードをイテレートし、書き込んだものを読み出します。このユースケースでは、画像を復元するだけなので、必要なのは生画像の文字列だけです。上記のゲッター、すなわち、`example.features.feature['image_raw'].bytes_list.value[0]` を使って抽出することができます。猫と橋のどちらであるかを決めるため、ラベルも使用します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "M6Cnfd3cTKHN", - "colab": {} - }, - "source": [ - "raw_image_dataset = tf.data.TFRecordDataset('images.tfrecords')\n", - "\n", - "# 特徴量を記述するディクショナリを作成\n", - "image_feature_description = {\n", - " 'height': tf.io.FixedLenFeature([], tf.int64),\n", - " 'width': tf.io.FixedLenFeature([], tf.int64),\n", - " 'depth': tf.io.FixedLenFeature([], tf.int64),\n", - " 'label': tf.io.FixedLenFeature([], tf.int64),\n", - " 'image_raw': tf.io.FixedLenFeature([], tf.string),\n", - "}\n", - "\n", - "def _parse_image_function(example_proto):\n", - " # 入力の tf.Example のプロトコルバッファを上記のディクショナリを使って解釈\n", - " return tf.io.parse_single_example(example_proto, image_feature_description)\n", - "\n", - "parsed_image_dataset = raw_image_dataset.map(_parse_image_function)\n", - "parsed_image_dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0PEEFPk4NEg1" - }, - "source": [ - "TFRecord ファイルから画像を復元しましょう。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yZf8jOyEIjSF", - "colab": {} - }, - "source": [ - "for image_features in parsed_image_dataset:\n", - " image_raw = image_features['image_raw'].numpy()\n", - " display.display(display.Image(data=image_raw))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ja/tutorials/quickstart/advanced.ipynb b/site/ja/tutorials/quickstart/advanced.ipynb deleted file mode 100644 index aafcf2d4432..00000000000 --- a/site/ja/tutorials/quickstart/advanced.ipynb +++ /dev/null @@ -1,427 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "advanced.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# エキスパートのための TensorFlow 2.0 入門" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3iSSiYRWx0ba" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "このファイルは [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) の notebook ファイルです。 Python プログラムはブラウザ上で直接実行されます。TensorFlow を学んだり使ったりするには最良の方法です。Google Colab の notebook の実行方法は以下のとおりです。\n", - "\n", - "1. Python ランタイムへの接続：メニューバーの右上で「接続」を選択します。\n", - "2. ノートブックのコードセルをすべて実行：「ランタイム」メニューから「すべてのセルを実行」を選択します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lqjqwIEawwdN", - "colab_type": "text" - }, - "source": [ - "TensorFlow 2 のパッケージをダウンロードしてインストールします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version は Colab 上でのみ利用可能\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hrIIhL97wwdQ", - "colab_type": "text" - }, - "source": [ - "プログラムに TensorFlow をインポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "vLdp28oMwwdR", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "[MNIST データセット](http://yann.lecun.com/exdb/mnist/)をロードして準備します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JqFRS6K07jJs", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# Add a channels dimension\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "データセットをシャッフルし、バッチ化するために tf.data を使います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Iu_quO024c2", - "colab": {} - }, - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Kerasの [model subclassing API](https://www.tensorflow.org/guide/keras#model_subclassing) を使って`tf.keras`モデルを作ります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "# モデルのインスタンスを作成\n", - "model = MyModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "訓練のためにオプティマイザと損失関数を選びます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u48C9WQ774n4", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "モデルの損失と正解率を計測するためのメトリクスを選択します。これらのメトリクスはエポックごとに値を集計し、最終結果を出力します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N0MqHFb4F_qn", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "`tf.GradientTape`を使ってモデルを訓練します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OZACiVqA8KQV", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z8YT7UmFgpjV" - }, - "source": [ - "次にモデルをテストします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIKdEzHAJGt7", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "i-2pkctU_Ci7", - "colab": {} - }, - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n", - " print (template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))\n", - " \n", - " # 次のエポック用にメトリクスをリセット\n", - " train_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_loss.reset_states()\n", - " test_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "この画像分類器は、今回のデータセットで訓練した場合、最大98%程度の正解率となります。更に学ぶには [TensorFlow tutorials](https://www.tensorflow.org/tutorials/) を読んでください。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/quickstart/beginner.ipynb b/site/ja/tutorials/quickstart/beginner.ipynb deleted file mode 100644 index 4e19b6ad941..00000000000 --- a/site/ja/tutorials/quickstart/beginner.ipynb +++ /dev/null @@ -1,261 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "beginner.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# 初心者のための TensorFlow 2.0 入門" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YRXLphinx2fF" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GgJT2G3OwwlT", - "colab_type": "text" - }, - "source": [ - "この短いイントロダクションでは [Keras](https://www.tensorflow.org/guide/keras/overview) を使って下記のことを行います。\n", - "\n", - "1. 画像を分類するニューラルネットワークを構築する\n", - "2. このニューラルネットワークを訓練する\n", - "3. そして最後に、モデルの正解率を評価する" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "このファイルは [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) の notebook ファイルです。 Python プログラムはブラウザ上で直接実行されます。TensorFlow を学んだり使ったりするには最良の方法です。Google Colab のnotebook の実行方法は以下のとおりです。\n", - "\n", - "1. Pythonランタイムへの接続：メニューバーの右上で「接続」を選択します。\n", - "2. ノートブックのコードセルをすべて実行：「ランタイム」メニューから「すべてのセルを実行」を選択します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kCJXrk_vwwlV", - "colab_type": "text" - }, - "source": [ - "TensorFlow 2 のパッケージをダウンロードしてインストールします。プログラムに TensorFlow をインポートします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow をインストール\n", - "try:\n", - " # %tensorflow_version は Colab 上でのみ利用可能\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "[MNIST データセット](http://yann.lecun.com/exdb/mnist/)をロードして準備します。サンプルを整数から浮動小数点数に変換します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FP5258xjs-v", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "層を積み重ねて`tf.keras.Sequential`モデルを構築します。訓練のためにオプティマイザと損失関数を選びます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "model = tf.keras.models.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "モデルを訓練してから評価します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F7dTAzgHDUh7", - "colab": {} - }, - "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", - "model.evaluate(x_test, y_test, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "この画像分類器は、今回のデータセットで訓練した場合、最大98%程度の正解率となります。更に学ぶには[TensorFlow tutorials](https://www.tensorflow.org/tutorials/)を読んでください。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/structured_data/feature_columns.ipynb b/site/ja/tutorials/structured_data/feature_columns.ipynb deleted file mode 100644 index aa2d0857fb0..00000000000 --- a/site/ja/tutorials/structured_data/feature_columns.ipynb +++ /dev/null @@ -1,745 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "feature_columns.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rNdWfPXCjTjY" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "I1dUQ0GejU8N", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c05P9g5WjizZ" - }, - "source": [ - "# 構造化されたデータの分類" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zofH_gCzgplN" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " View on TensorFlow.org\n", - " \n", - " \n", - " \n", - " Run in Google Colab\n", - " \n", - " \n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lrFrIDaEZAjE", - "colab_type": "text" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K1y4OHpGgss7" - }, - "source": [ - "このチュートリアルでは、（例えばCSVファイルに保存された表形式データのような）構造化されたデータをどうやって分類するかを示します。ここでは、モデルの定義に[Keras](https://www.tensorflow.org/guide/keras)を、[feature columns](https://www.tensorflow.org/guide/feature_columns)をCSVファイルの列をモデルを訓練するための特徴量にマッピングするための橋渡し役として使用します。このチュートリアルには、下記のことを行うコードすべてが含まれています。\n", - "\n", - "* [Pandas](https://pandas.pydata.org/)を使用したCSVファイルの読み込み\n", - "* [tf.data](https://www.tensorflow.org/guide/datasets)を使用して行データをシャッフルし、バッチ化するための入力パイプライン構築\n", - "* feature columnsを使ったCSVの列のモデル訓練用の特徴量へのマッピング\n", - "* Kerasを使ったモデルの構築と、訓練及び評価\n", - "\n", - "## データセット\n", - "\n", - "ここでは、Cleveland Clinic Foundation for Heart Diseaseが提供している小さな[データセット](https://archive.ics.uci.edu/ml/datasets/heart+Disease)を使用します。このCSVファイルには数百行が含まれています。行が患者を、列がその属性を表します。この情報を使用して、患者が心臓疾患を持っているかを予測します。このデータセットの場合には二値分類タスクとなります。\n", - "\n", - "下記はこのデータセットの[說明](https://archive.ics.uci.edu/ml/machine-learning-databases/heart-disease/heart-disease.names)です。数値列とカテゴリー列があることに注目してください。\n", - "\n", - ">列| 說明| 特徴量の型 | データ型\n", - ">------------|--------------------|----------------------|-----------------\n", - ">Age | 年齢 | 数値型 | 整数\n", - ">Sex | (1 = 男性; 0 = 女性) | カテゴリー型 | 整数\n", - ">CP | 胸痛のタイプ (0, 1, 2, 3, 4) | カテゴリー型 | 整数\n", - ">Trestbpd | 安静時血圧 (単位：mm Hg 入院時) | 数値型 | 整数\n", - ">Chol | 血清コレステロール 単位：mg/dl | 数値型 | 整数\n", - ">FBS | (空腹時血糖 > 120 mg/dl) (1 = 真; 0 = 偽) | カテゴリー型 | 整数\n", - ">RestECG | 安静時心電図の診断結果 (0, 1, 2) | カテゴリー型 | 整数\n", - ">Thalach | 最大心拍数 | 数値型 | 整数\n", - ">Exang | 運動誘発狭心症 (1 = はい; 0 = いいえ) | カテゴリー型 | 整数\n", - ">Oldpeak | 安静時と比較した運動時のST低下 | 数値型 | 整数\n", - ">Slope | ピーク運動STセグメントの勾配 | 数値型 | 浮動小数点数\n", - ">CA | 蛍光透視法によって着色された主要血管の数（0−3） | 数値型 | 整数\n", - ">Thal | 3 = 正常; 6 = 固定欠陥; 7 = 可逆的欠陥 | カテゴリー型 | 文字列\n", - ">Target | 心臓疾患の診断 (1 = 真; 0 = 偽) | 分類 | 整数" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VxyBFc_kKazA" - }, - "source": [ - "## TensorFlow他ライブラリのインポート" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LuOWVJBz8a6G", - "colab": {} - }, - "source": [ - "!pip install sklearn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9dEreb4QKizj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import feature_column\n", - "from tensorflow.keras import layers\n", - "from sklearn.model_selection import train_test_split" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KCEhSZcULZ9n" - }, - "source": [ - "## Pandasを使ったデータフレーム作成\n", - "\n", - "[Pandas](https://pandas.pydata.org/)は、構造化データの読み込みや操作のための便利なユーティリティを持つPythonのライブラリです。ここでは、Pandasを使ってURLからデータをダウンロードし、データフレームに読み込みます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "REZ57BXCLdfG", - "colab": {} - }, - "source": [ - "URL = 'https://storage.googleapis.com/applied-dl/heart.csv'\n", - "dataframe = pd.read_csv(URL)\n", - "dataframe.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u0zhLtQqMPem" - }, - "source": [ - "## データフレームを、訓練用、検証用、テスト用に分割\n", - "\n", - "ダウンロードしたデータセットは1つのCSVファイルです。これを、訓練用、検証用、テスト用のデータセットに分割します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YEOpw7LhMYsI", - "colab": {} - }, - "source": [ - "train, test = train_test_split(dataframe, test_size=0.2)\n", - "train, val = train_test_split(train, test_size=0.2)\n", - "print(len(train), 'train examples')\n", - "print(len(val), 'validation examples')\n", - "print(len(test), 'test examples')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "84ef46LXMfvu" - }, - "source": [ - "## tf.dataを使った入力パイプラインの構築\n", - "\n", - "次に、[tf.data](https://www.tensorflow.org/guide/datasets)を使ってデータフレームをラップします。こうすることで、feature columns をPandasデータフレームの列をモデル訓練用の特徴量へのマッピングするための橋渡し役として使うことができます。（メモリに収まらないぐらいの）非常に大きなCSVファイルを扱う場合には、tf.dataを使ってディスクから直接CSVファイルを読み込むことになります。この方法は、このチュートリアルでは扱いません。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NkcaMYP-MsRe", - "colab": {} - }, - "source": [ - "# Pandasデータフレームからtf.dataデータセットを作るためのユーティリティメソッド\n", - "def df_to_dataset(dataframe, shuffle=True, batch_size=32):\n", - " dataframe = dataframe.copy()\n", - " labels = dataframe.pop('target')\n", - " ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n", - " if shuffle:\n", - " ds = ds.shuffle(buffer_size=len(dataframe))\n", - " ds = ds.batch(batch_size)\n", - " return ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CXbbXkJvMy34", - "colab": {} - }, - "source": [ - "batch_size = 5 # デモ用として小さなバッチサイズを使用\n", - "train_ds = df_to_dataset(train, batch_size=batch_size)\n", - "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", - "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qRLGSMDzM-dl" - }, - "source": [ - "## 入力パイプラインを理解する\n", - "\n", - "入力パイプラインを構築したので、それが返すデータのフォーマットを見るために呼び出してみましょう。出力を読みやすくするためにバッチサイズを小さくしてあります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CSBo3dUVNFc9", - "colab": {} - }, - "source": [ - "for feature_batch, label_batch in train_ds.take(1):\n", - " print('Every feature:', list(feature_batch.keys()))\n", - " print('A batch of ages:', feature_batch['age'])\n", - " print('A batch of targets:', label_batch )" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OT5N6Se-NQsC" - }, - "source": [ - "データセットが（データフレームにある）列名からなるディクショナリを返すことがわかります。列名から、データフレームの行に含まれる列の値が得られます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ttIvgLRaNoOQ" - }, - "source": [ - "## feature columnsの様々な型の例\n", - "\n", - "TensorFlowにはたくさんの型のfeature columnがあります。このセクションでは、いくつかの型のfeature columnsを作り、データフレームの列をどのように変換しているかを示します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mxwiHFHuNhmf", - "colab": {} - }, - "source": [ - "# いくつかの型のfeature columnsを例示するためこのバッチを使用する\n", - "example_batch = next(iter(train_ds))[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0wfLB8Q3N3UH", - "colab": {} - }, - "source": [ - "# feature columnsを作りデータのバッチを変換する\n", - "# ユーティリティメソッド\n", - "def demo(feature_column):\n", - " feature_layer = layers.DenseFeatures(feature_column)\n", - " print(feature_layer(example_batch).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q7OEKe82N-Qb" - }, - "source": [ - "### 数値コラム\n", - "\n", - "feature columnsの出力はモデルへの入力になります（上記で定義したdemo関数を使うと、データフレームの列がどのように変換されるかをつぶさに見ることができます）。[数値コラム](https://www.tensorflow.org/api_docs/python/tf/feature_column/numeric_column)は、最も単純な型のコラムです。数値コラムは実数特徴量を表現するのに使われます。このコラムを使う場合、モデルにはデータフレームの列の値がそのまま渡されます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QZTZ0HnHOCxC", - "colab": {} - }, - "source": [ - "age = feature_column.numeric_column(\"age\")\n", - "demo(age)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7a6ddSyzOKpq" - }, - "source": [ - "心臓疾患データセットでは、データフレームのほとんどの列が数値型です。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IcSxUoYgOlA1" - }, - "source": [ - "### バケット化コラム\n", - "\n", - "数値をそのままモデルに入力するのではなく、値の範囲に基づいた異なるカテゴリーに分割したいことがあります。例えば、人の年齢を表す生データを考えてみましょう。[バケット化コラム](https://www.tensorflow.org/api_docs/python/tf/feature_column/bucketized_column)を使うと年齢を数値コラムとして表現するのではなく、年齢をいくつかのバケットに分割できます。下記のワンホット値が、各行がどの年齢範囲にあるかを表していることに注目してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wJ4Wt3SAOpTQ", - "colab": {} - }, - "source": [ - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "demo(age_buckets)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r1tArzewPb-b" - }, - "source": [ - "### カテゴリー型コラム\n", - "\n", - "このデータセットでは、Thalは（'fixed'、'normal'、'reversible'のような）文字列として表現されています。文字列を直接モデルに入力することはできません。まず、文字列を数値にマッピングする必要があります。categorical vocabulary コラムを使うと、（上記で示した年齢バケットのように）文字列をワンホットベクトルとして表現することができます。カテゴリーを表す語彙(vocabulary)は[categorical_column_with_vocabulary_list](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_list)を使ってリストで渡すか、[categorical_column_with_vocabulary_file](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_file)を使ってファイルから読み込むことができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DJ6QnSHkPtOC", - "colab": {} - }, - "source": [ - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "demo(thal_one_hot)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dxQloQ9jOoXL" - }, - "source": [ - "より複雑なデータセットでは、たくさんの列がカテゴリー型（例えば文字列）であることでしょう。feature columns はカテゴリー型データを扱う際に最も役に立ちます。このデータセットでは、カテゴリー型コラムは1つだけですが、他のデータセットを扱う際に使用できるいくつかの重要な型のfeature columnsを紹介するために、この列を使用することにします。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LEFPjUr6QmwS" - }, - "source": [ - "### 埋め込み型コラム\n", - "\n", - "数種類の候補となる文字列ではなく、カテゴリー毎に数千（あるいはそれ以上）の値があるとしましょう。カテゴリーの数が多くなってくると、様々な理由から、ワンホットエンコーディングを使ってニューラルネットワークを訓練することが難しくなります。埋込み型コラムを使うと、こうした制約を克服することが可能です。[埋込み型コラム](https://www.tensorflow.org/api_docs/python/tf/feature_column/embedding_column)は、データを多次元のワンホットベクトルとして表すのではなく、セルの値が0か1かだけではなく、どんな数値でもとれるような密な低次元ベクトルとして表現します。埋め込みのサイズ（下記の例では8）は、チューニングが必要なパラメータです。\n", - "\n", - "キーポイント：カテゴリー型コラムがたくさんの選択肢を持つ場合、埋め込み型コラムを使用することが最善の方法です。ここでは例を一つ示しますので、今後様々なデータセットを扱う際には、この例を参考にしてください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hSlohmr2Q_UU", - "colab": {} - }, - "source": [ - "# この埋込み型コラムの入力は、先程作成したカテゴリ型コラムであることに注意\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "demo(thal_embedding)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "urFCAvTVRMpB" - }, - "source": [ - "### ハッシュ化特徴コラム\n", - "\n", - "値の種類が多いカテゴリー型コラムを表現するもう一つの方法が、[categorical_column_with_hash_bucket](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_hash_bucket)を使う方法です。このfeature columnは文字列をエンコードするために入力のハッシュ値を計算し、`hash_bucket_size`個のバケットの中から1つを選択します。このコラムを使用する場合には、語彙を用意する必要はありません。また、スペースの節約のために、実際のカテゴリー数に比べて極めて少ないバケット数を選択することも可能です。\n", - "\n", - "キーポイント：この手法の重要な欠点の一つは、異なる文字列が同じバケットにマッピングされるというハッシュ値の衝突が起きることです。実務上は、データセットによっては、この問題を無視できることがあります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YHU_Aj2nRRDC", - "colab": {} - }, - "source": [ - "thal_hashed = feature_column.categorical_column_with_hash_bucket(\n", - " 'thal', hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(thal_hashed))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fB94M27DRXtZ" - }, - "source": [ - "### クロスフィーチャーコラム\n", - "\n", - "複数の特徴量をまとめて1つの特徴量にする、[フィーチャークロス](https://developers.google.com/machine-learning/glossary/#feature_cross)として知られている手法は、モデルが特徴量の組み合わせの一つ一つに別々の重みを学習することを可能にします。ここでは年齢とThalをクロスさせて新しい特徴量を作ってみます。交差列(`crossed_column`)が、起こりうるすべての組み合わせ全体のテーブル（これは非常に大きくなる可能性があります）を作るものではないことに注意してください。クロスフィーチャーコラムは、代わりにバックエンドとしてハッシュ化コラムを使用しているため、テーブルの大きさを選択することができます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oaPVERd9Rep6", - "colab": {} - }, - "source": [ - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(crossed_feature))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ypkI9zx6Rj1q" - }, - "source": [ - "## 使用するコラムを選択する\n", - "\n", - "これまで、いくつかのfeature columnの使い方を見てきました。いよいよモデルの訓練にそれらを使用することにします。このチュートリアルの目的は、feature columnsを使うのに必要な完全なコード（いわば力学）を示すことです。以下ではモデルを訓練するための列を適当に選びました。\n", - "\n", - "キーポイント：正確なモデルを構築するのが目的である場合には、できるだけ大きなデータセットを使用して、どの特徴量を含めるのがもっとも意味があるのかや、それらをどう表現したらよいかを、慎重に検討してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4PlLY7fORuzA", - "colab": {} - }, - "source": [ - "feature_columns = []\n", - "\n", - "# 数値コラム\n", - "for header in ['age', 'trestbps', 'chol', 'thalach', 'oldpeak', 'slope', 'ca']:\n", - " feature_columns.append(feature_column.numeric_column(header))\n", - "\n", - "# バケット化コラム\n", - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "feature_columns.append(age_buckets)\n", - "\n", - "# インジケーター（カテゴリー型）コラム\n", - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "feature_columns.append(thal_one_hot)\n", - "\n", - "# 埋め込み型コラム\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "feature_columns.append(thal_embedding)\n", - "\n", - "# クロスフィーチャーコラム\n", - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "crossed_feature = feature_column.indicator_column(crossed_feature)\n", - "feature_columns.append(crossed_feature)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M-nDp8krS_ts" - }, - "source": [ - "### 特徴量層の構築\n", - "\n", - "feature columnsを定義し終わったので、次に[DenseFeatures](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/DenseFeatures)層を使ってKerasモデルへの入力とします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6o-El1R2TGQP", - "colab": {} - }, - "source": [ - "feature_layer = tf.keras.layers.DenseFeatures(feature_columns)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8cf6vKfgTH0U" - }, - "source": [ - "これまでは、feature columnsの働きを見るため、小さなバッチサイズを使ってきました。ここではもう少し大きなバッチサイズの新しい入力パイプラインを作ります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gcemszoGSse_", - "colab": {} - }, - "source": [ - "batch_size = 32\n", - "train_ds = df_to_dataset(train, batch_size=batch_size)\n", - "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", - "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBx4Xu0eTXWq" - }, - "source": [ - "## モデルの構築、コンパイルと訓練" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_YJPPb3xTPeZ", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " feature_layer,\n", - " layers.Dense(128, activation='relu'),\n", - " layers.Dense(128, activation='relu'),\n", - " layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "model.fit(train_ds, \n", - " validation_data=val_ds, \n", - " epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GnFmMOW0Tcaa", - "colab": {} - }, - "source": [ - "loss, accuracy = model.evaluate(test_ds)\n", - "print(\"Accuracy\", accuracy)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3bdfbq20V6zu" - }, - "source": [ - "キーポイント：一般的に、ディープラーニングが最良の結果となるのは、もっと大きくて、もっと複雑なデータセットです。この例のように小さなデータセットを使用する際には、強固なベースラインとして、決定木やランダムフォレストを使うことをおすすめします。このチュートリアルの目的は、訓練により正確なモデルを得ることではなく、構造化データの使い方をデモすることです。今後ご自分のデータセットに取り組まれる際の出発点として、これらのコードをお使いください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SotnhVWuHQCw" - }, - "source": [ - "## 次のステップ\n", - "\n", - "構造化データの分類について更に多くのことを学ぶためには、自分自身で試してみることです。別のデータセットを見つけ、上記と同様のコードを使って、それを分類するモデルを訓練してみてください。正解率を上げるためには、モデルにどの特徴量を含めたらよいかや、その特徴量をどのように表現すべきかをじっくり考えてください。" - ] - } - ] -} diff --git a/site/ja/tutorials/text/text_classification_rnn.ipynb b/site/ja/tutorials/text/text_classification_rnn.ipynb deleted file mode 100644 index 4d97df98073..00000000000 --- a/site/ja/tutorials/text/text_classification_rnn.ipynb +++ /dev/null @@ -1,741 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_classification_rnn.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hX4n9TsbGw-f" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "0nbI5DtDGw-i", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9TnJztDZGw-n" - }, - "source": [ - "# RNN を使ったテキスト分類" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AfN3bMR5Gw-o" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tudzcncJXetB" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lUWearf0Gw-p" - }, - "source": [ - "このテキスト分類チュートリアルでは、感情分析のために [IMDB 映画レビュー大型データセット](http://ai.stanford.edu/~amaas/data/sentiment/) を使って [リカレントニューラルネットワーク](https://developers.google.com/machine-learning/glossary/#recurrent_neural_network) を訓練します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_2VQo4bajwUU" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "z682XYsrjkY9", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version は Colab 上でだけ使用可能\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1rXHa-w9JZhb" - }, - "source": [ - "`matplotlib` をインポートしグラフを描画するためのヘルパー関数を作成します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mp1Z7P9pYRSK", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "def plot_graphs(history, string):\n", - " plt.plot(history.history[string])\n", - " plt.plot(history.history['val_'+string], '')\n", - " plt.xlabel(\"Epochs\")\n", - " plt.ylabel(string)\n", - " plt.legend([string, 'val_'+string])\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pRmMubr0jrE2" - }, - "source": [ - "## 入力パイプラインの設定\n", - "\n", - "IMDB 映画レビュー大型データセットは*二値分類*データセットです。すべてのレビューは、*好意的(positive)* または *非好意的(negative)* のいずれかの感情を含んでいます。\n", - "\n", - "[TFDS](https://www.tensorflow.org/datasets) を使ってこのデータセットをダウンロードします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SHRwRoP2nVHX", - "colab": {} - }, - "source": [ - "dataset, info = tfds.load('imdb_reviews/subwords8k', with_info=True,\n", - " as_supervised=True)\n", - "train_dataset, test_dataset = dataset['train'], dataset['test']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MCorLciXSDJE" - }, - "source": [ - " このデータセットの `info` には、エンコーダー(`tfds.features.text.SubwordTextEncoder`) が含まれています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EplYp5pNnW1S", - "colab": {} - }, - "source": [ - "encoder = info.features['text'].encoder" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "e7ACuHM5hFp3", - "colab": {} - }, - "source": [ - "print ('Vocabulary size: {}'.format(encoder.vocab_size))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tAfGg8YRe6fu" - }, - "source": [ - "このテキストエンコーダーは、任意の文字列を可逆的にエンコードします。必要であればバイトエンコーディングにフォールバックします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Bq6xDmf2SAs-", - "colab": {} - }, - "source": [ - "sample_string = 'Hello TensorFlow.'\n", - "\n", - "encoded_string = encoder.encode(sample_string)\n", - "print ('Encoded string is {}'.format(encoded_string))\n", - "\n", - "original_string = encoder.decode(encoded_string)\n", - "print ('The original string: \"{}\"'.format(original_string))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TN7QbKaM4-5H", - "colab": {} - }, - "source": [ - "assert original_string == sample_string" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MDVc6UGO5Dh6", - "colab": {} - }, - "source": [ - "for index in encoded_string:\n", - " print ('{} ----> {}'.format(index, encoder.decode([index])))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GlYWqhTVlUyQ" - }, - "source": [ - "## 訓練用データの準備" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z2qVJzcEluH_" - }, - "source": [ - "次に、これらのエンコード済み文字列をバッチ化します。`padded_batch` メソッドを使ってバッチ中の一番長い文字列の長さにゼロパディングを行います。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dDsCaZCDYZgm", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 10000\n", - "BATCH_SIZE = 64" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VznrltNOnUc5", - "colab": {} - }, - "source": [ - "train_dataset = train_dataset.shuffle(BUFFER_SIZE)\n", - "train_dataset = train_dataset.padded_batch(BATCH_SIZE, train_dataset.output_shapes)\n", - "\n", - "test_dataset = test_dataset.padded_batch(BATCH_SIZE, test_dataset.output_shapes)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bjUqGVBxGw-t" - }, - "source": [ - "## モデルの作成" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bgs6nnSTGw-t" - }, - "source": [ - "`tf.keras.Sequential` モデルを構築しましょう。最初に Embedding レイヤーから始めます。Embedding レイヤーは単語一つに対して一つのベクトルを収容します。呼び出しを受けると、Embedding レイヤーは単語のインデックスのシーケンスを、ベクトルのシーケンスに変換します。これらのベクトルは訓練可能です。（十分なデータで）訓練されたあとは、おなじような意味をもつ単語は、しばしばおなじようなベクトルになります。\n", - "\n", - "このインデックス参照は、ワンホットベクトルを `tf.keras.layers.Dense` レイヤーを使って行うおなじような演算に比べてずっと効率的です。\n", - "\n", - "リカレントニューラルネットワーク（RNN）は、シーケンスの入力を要素を一つずつ扱うことで処理します。RNN は、あるタイムステップでの出力を次のタイムステップの入力へと、次々に渡していきます。\n", - "\n", - "RNN レイヤーとともに、`tf.keras.layers.Bidirectional` ラッパーを使用することができます。このラッパーは、入力を RNN 層の順方向と逆方向に伝え、その後出力を結合します。これにより、RNN は長期的な依存関係を学習できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LwfoBkmRYcP3", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(encoder.vocab_size, 64),\n", - " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sRI776ZcH3Tf" - }, - "source": [ - "訓練プロセスを定義するため、Keras モデルをコンパイルします。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kj2xei41YZjC", - "colab": {} - }, - "source": [ - "model.compile(loss='binary_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(1e-4),\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zIwH3nto596k" - }, - "source": [ - "## モデルの訓練" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hw86wWS4YgR2", - "colab": {} - }, - "source": [ - "history = model.fit(train_dataset, epochs=10,\n", - " validation_data=test_dataset, \n", - " validation_steps=30)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BaNbXi43YgUT", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_dataset)\n", - "\n", - "print('Test Loss: {}'.format(test_loss))\n", - "print('Test Accuracy: {}'.format(test_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DwSE_386uhxD" - }, - "source": [ - "上記のモデルはシーケンスに適用されたパディングをマスクしていません。パディングされたシーケンスで訓練を行い、パディングをしていないシーケンスでテストするとすれば、このことが結果を歪める可能性があります。理想的にはこれを避けるために、 [マスキング](../../guide/keras/masking_and_padding)を使うべきですが、下記のように出力への影響は小さいものでしかありません。 \n", - "\n", - "予測値が 0.5 以上であればポジティブ、それ以外はネガティブです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8w0dseJMiEUh", - "colab": {} - }, - "source": [ - "def pad_to_size(vec, size):\n", - " zeros = [0] * (size - len(vec))\n", - " vec.extend(zeros)\n", - " return vec" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y-E4cgkIvmVu", - "colab": {} - }, - "source": [ - "def sample_predict(sentence, pad):\n", - " encoded_sample_pred_text = encoder.encode(sample_pred_text)\n", - "\n", - " if pad:\n", - " encoded_sample_pred_text = pad_to_size(encoded_sample_pred_text, 64)\n", - " encoded_sample_pred_text = tf.cast(encoded_sample_pred_text, tf.float32)\n", - " predictions = model.predict(tf.expand_dims(encoded_sample_pred_text, 0))\n", - "\n", - " return (predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "O41gw3KfWHus", - "colab": {} - }, - "source": [ - "# パディングなしのサンプルテキストの推論\n", - "\n", - "sample_pred_text = ('The movie was cool. The animation and the graphics '\n", - " 'were out of this world. I would recommend this movie.')\n", - "predictions = sample_predict(sample_pred_text, pad=False)\n", - "print (predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kFh4xLARucTy", - "colab": {} - }, - "source": [ - "# パディングありのサンプルテキストの推論\n", - "\n", - "sample_pred_text = ('The movie was cool. The animation and the graphics '\n", - " 'were out of this world. I would recommend this movie.')\n", - "predictions = sample_predict(sample_pred_text, pad=True)\n", - "print (predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZfIVoxiNmKBF", - "colab": {} - }, - "source": [ - "plot_graphs(history, 'accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IUzgkqnhmKD2", - "colab": {} - }, - "source": [ - "plot_graphs(history, 'loss')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7g1evcaRpTKm" - }, - "source": [ - "## 2つ以上の LSTM レイヤーを重ねる\n", - "\n", - "Keras のリカレントレイヤーには、コンストラクタの `return_sequences` 引数でコントロールされる2つのモードがあります。\n", - "\n", - "* それぞれのタイムステップの連続した出力のシーケンス全体（shape が `(batch_size, timesteps, output_features)` の3階テンソル）を返す。\n", - "* それぞれの入力シーケンスの最後の出力だけ（shape が `(batch_size, output_features)` の2階テンソル）を返す。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jo1jjO3vn0jo", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(encoder.vocab_size, 64),\n", - " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)),\n", - " tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dropout(0.5),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hEPV5jVGp-is", - "colab": {} - }, - "source": [ - "model.compile(loss='binary_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(1e-4),\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LeSE-YjdqAeN", - "colab": {} - }, - "source": [ - "history = model.fit(train_dataset, epochs=10,\n", - " validation_data=test_dataset,\n", - " validation_steps=30)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_LdwilM1qPM3", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_dataset)\n", - "\n", - "print('Test Loss: {}'.format(test_loss))\n", - "print('Test Accuracy: {}'.format(test_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ykUKnAoqbycW", - "colab": {} - }, - "source": [ - "# パディングなしのサンプルテキストの推論\n", - "\n", - "sample_pred_text = ('The movie was not good. The animation and the graphics '\n", - " 'were terrible. I would not recommend this movie.')\n", - "predictions = sample_predict(sample_pred_text, pad=False)\n", - "print (predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2RiC-94zvdZO", - "colab": {} - }, - "source": [ - "# パディングありのサンプルテキストの推論\n", - "\n", - "sample_pred_text = ('The movie was not good. The animation and the graphics '\n", - " 'were terrible. I would not recommend this movie.')\n", - "predictions = sample_predict(sample_pred_text, pad=True)\n", - "print (predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_YYub0EDtwCu", - "colab": {} - }, - "source": [ - "plot_graphs(history, 'accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DPV3Nn9xtwFM", - "colab": {} - }, - "source": [ - "plot_graphs(history, 'loss')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9xvpE3BaGw_V" - }, - "source": [ - "[GRU レイヤー](https://www.tensorflow.org/api_docs/python/tf/keras/layers/GRU)など既存のほかのレイヤーを調べてみましょう。\n", - "\n", - "カスタム RNN の構築に興味があるのであれば、[Keras RNN ガイド](../../guide/keras/rnn.ipynb) を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UQK2Na3Z_aMx", - "colab": {} - }, - "source": [ - "" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ja/tutorials/text/text_generation.ipynb b/site/ja/tutorials/text/text_generation.ipynb deleted file mode 100644 index f30ebcf636d..00000000000 --- a/site/ja/tutorials/text/text_generation.ipynb +++ /dev/null @@ -1,1256 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "GCCk8_dHpuNf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ovpZyIhNIgoq" - }, - "source": [ - "# RNN によるテキスト生成" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hcD2nPQvPOFM" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RoRi17kuCYFs" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BwpJ5IffzRG6" - }, - "source": [ - "このチュートリアルでは、文字ベースの RNN を使ってテキストを生成する方法を示します。ここでは、Andrej Karpathy の [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/) からのシェイクスピア作品のデータセットを使います。このデータからの文字列（\"Shakespear\"）を入力にして、文字列中の次の文字（\"e\"）を予測するモデルを訓練します。このモデルを繰り返し呼び出すことで、より長い文字列を生成することができます。\n", - "\n", - "Note: このノートブックの実行を速くするために GPU による高速化を有効にしてください。Colab では、*ランタイム　＞　ランタイムのタイプを変更 ＞ ハードウェアアクセラレータ ＞ GPU* を選択します。ローカルで実行する場合には、TensorFlow のバージョンが 1.11 以降であることを確認してください。\n", - "\n", - "このチュートリアルには、[tf.keras](https://www.tensorflow.org/programmers_guide/keras) と [eager execution](https://www.tensorflow.org/programmers_guide/eager) を使ったコードが含まれています。下記は、このチュートリアルのモデルを 30 エポック訓練したものに対して、文字列 \"Q\" を初期値とした場合の出力例です。\n", - "\n", - "
    \n",
-    "QUEENE:\n",
-    "I had thought thou hadst a Roman; for the oracle,\n",
-    "Thus by All bids the man against the word,\n",
-    "Which are so weak of care, by old care done;\n",
-    "Your children were in your holy love,\n",
-    "And the precipitation through the bleeding throne.\n",
-    "\n",
-    "BISHOP OF ELY:\n",
-    "Marry, and will, my lord, to weep in such a one were prettiest;\n",
-    "Yet now I was adopted heir\n",
-    "Of the world's lamentable day,\n",
-    "To watch the next way with his father with his face?\n",
-    "\n",
-    "ESCALUS:\n",
-    "The cause why then we are all resolved more sons.\n",
-    "\n",
-    "VOLUMNIA:\n",
-    "O, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, it is no sin it should be dead,\n",
-    "And love and pale as any will to that word.\n",
-    "\n",
-    "QUEEN ELIZABETH:\n",
-    "But how long have I heard the soul for this world,\n",
-    "And show his hands of life be proved to stand.\n",
-    "\n",
-    "PETRUCHIO:\n",
-    "I say he look'd on, if I must be content\n",
-    "To stay him from the fatal of our country's bliss.\n",
-    "His lordship pluck'd from this sentence then for prey,\n",
-    "And then let us twain, being the moon,\n",
-    "were she such a case as fills m\n",
-    "
    \n", - "\n", - "いくつかは文法にあったものがある一方で、ほとんどは意味をなしていません。このモデルは、単語の意味を学習していませんが、次のことを考えてみてください。\n", - "\n", - "* このモデルは文字ベースです。訓練が始まった時に、モデルは英語の単語のスペルも知りませんし、単語がテキストの単位であることも知らないのです。\n", - "\n", - "* 出力の構造は戯曲に似ています。だいたいのばあい、データセットとおなじ大文字で書かれた話し手の名前で始まっています。\n", - "\n", - "* 以下に示すように、モデルはテキストの小さなバッチ（各100文字）で訓練されていますが、一貫した構造のより長いテキストのシーケンスを生成できます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "srXC6pLGLwS6" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WGyKZj3bzf9p" - }, - "source": [ - "### TensorFlow 等のライブラリインポート" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yG_n40gFzf9s" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version は Colab 上でのみ利用可能\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import numpy as np\n", - "import os\n", - "import time" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EHDoRoc5PKWz" - }, - "source": [ - "### シェイクスピアデータセットのダウンロード\n", - "\n", - "独自のデータで実行するためには下記の行を変更してください。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "pD_55cOxLkAb" - }, - "outputs": [], - "source": [ - "path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UHjdCjDuSvX_" - }, - "source": [ - "### データの読み込み\n", - "\n", - "まずはテキストをのぞいてみましょう。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aavnuByVymwK" - }, - "outputs": [], - "source": [ - "# 読み込んだのち、Python 2 との互換性のためにデコード\n", - "text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n", - "# テキストの長さは含まれる文字数\n", - "print ('Length of text: {} characters'.format(len(text)))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Duhg9NrUymwO" - }, - "outputs": [], - "source": [ - "# テキストの最初の 250文字を参照\n", - "print(text[:250])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IlCgQBRVymwR" - }, - "outputs": [], - "source": [ - "# ファイル中のユニークな文字の数\n", - "vocab = sorted(set(text))\n", - "print ('{} unique characters'.format(len(vocab)))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rNnrKn_lL-IJ" - }, - "source": [ - "## テキストの処理" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LFjSVAlWzf-N" - }, - "source": [ - "### テキストのベクトル化\n", - "\n", - "訓練をする前に、文字列を数値表現に変換する必要があります。2つの参照テーブルを作成します。一つは文字を数字に変換するもの、もう一つは数字を文字に変換するものです。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IalZLbvOzf-F" - }, - "outputs": [], - "source": [ - "# それぞれの文字からインデックスへの対応表を作成\n", - "char2idx = {u:i for i, u in enumerate(vocab)}\n", - "idx2char = np.array(vocab)\n", - "\n", - "text_as_int = np.array([char2idx[c] for c in text])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tZfqhkYCymwX" - }, - "source": [ - "これで、それぞれの文字を整数で表現できました。文字を、0 から`len(unique)` までのインデックスに変換していることに注意してください。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FYyNlCNXymwY" - }, - "outputs": [], - "source": [ - "print('{')\n", - "for char,_ in zip(char2idx, range(20)):\n", - " print(' {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n", - "print(' ...\\n}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l1VKcQHcymwb" - }, - "outputs": [], - "source": [ - "# テキストの最初の 13 文字がどのように整数に変換されるかを見てみる\n", - "print ('{} ---- characters mapped to int ---- > {}'.format(repr(text[:13]), text_as_int[:13]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bbmsf23Bymwe" - }, - "source": [ - "### 予測タスク" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wssHQ1oGymwe" - }, - "source": [ - "ある文字、あるいは文字列が与えられたとき、もっともありそうな次の文字はなにか？これが、モデルを訓練してやらせたいタスクです。モデルへの入力は文字列であり、モデルが出力、つまりそれぞれの時点での次の文字を予測をするようにモデルを訓練します。\n", - "\n", - "RNN はすでに見た要素に基づく内部状態を保持しているため、この時点までに計算されたすべての文字を考えると、次の文字は何でしょうか？" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hgsVvVxnymwf" - }, - "source": [ - "### 訓練用サンプルとターゲットを作成\n", - "\n", - "つぎに、テキストをサンプルシーケンスに分割します。それぞれの入力シーケンスは、元のテキストからの `seq_length` 個の文字を含みます。\n", - "\n", - "入力シーケンスそれぞれに対して、対応するターゲットは同じ長さのテキストを含みますが、1文字ずつ右にシフトしたものです。\n", - "\n", - "そのため、テキストを `seq_length+1` のかたまりに分割します。たとえば、 `seq_length` が 4 で、テキストが \"Hello\" だとします。入力シーケンスは \"Hell\" で、ターゲットシーケンスは \"ello\" となります。\n", - "\n", - "これを行うために、最初に `tf.data.Dataset.from_tensor_slices` 関数を使ってテキストベクトルを文字インデックスの連続に変換します。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0UHJDA39zf-O" - }, - "outputs": [], - "source": [ - "# ひとつの入力としたいシーケンスの文字数としての最大の長さ\n", - "seq_length = 100\n", - "examples_per_epoch = len(text)//(seq_length+1)\n", - "\n", - "# 訓練用サンプルとターゲットを作る\n", - "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n", - "\n", - "for i in char_dataset.take(5):\n", - " print(idx2char[i.numpy()])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ZSYAcQV8OGP" - }, - "source": [ - "`batch` メソッドを使うと、個々の文字を求める長さのシーケンスに簡単に変換できます。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l4hkDU3i7ozi" - }, - "outputs": [], - "source": [ - "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n", - "\n", - "for item in sequences.take(5):\n", - " print(repr(''.join(idx2char[item.numpy()])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UbLcIPBj_mWZ" - }, - "source": [ - "シーケンスそれぞれに対して、`map` メソッドを使って各バッチに単純な関数を適用することで、複製とシフトを行い、入力テキストとターゲットテキストを生成します。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9NGu-FkO_kYU" - }, - "outputs": [], - "source": [ - "def split_input_target(chunk):\n", - " input_text = chunk[:-1]\n", - " target_text = chunk[1:]\n", - " return input_text, target_text\n", - "\n", - "dataset = sequences.map(split_input_target)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiCopyGZymwi" - }, - "source": [ - "最初のサンプルの入力とターゲットを出力します。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GNbw-iR0ymwj" - }, - "outputs": [], - "source": [ - "for input_example, target_example in dataset.take(1):\n", - " print ('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n", - " print ('Target data:', repr(''.join(idx2char[target_example.numpy()])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_33OHL3b84i0" - }, - "source": [ - "これらのベクトルのインデックスそれぞれが一つのタイムステップとして処理されます。タイムステップ 0 の入力として、モデルは \"F\" のインデックスを受け取り、次の文字として \"i\" のインデックスを予測しようとします。次のタイムステップでもおなじことをしますが、`RNN` は現在の入力文字に加えて、過去のステップのコンテキストも考慮します。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0eBu9WZG84i0" - }, - "outputs": [], - "source": [ - "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n", - " print(\"Step {:4d}\".format(i))\n", - " print(\" input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n", - " print(\" expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MJdfPmdqzf-R" - }, - "source": [ - "### 訓練用バッチの作成\n", - "\n", - "`tf.data` を使ってテキストを分割し、扱いやすいシーケンスにします。しかし、このデータをモデルに供給する前に、データをシャッフルしてバッチにまとめる必要があります。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "p2pGotuNzf-S" - }, - "outputs": [], - "source": [ - "# バッチサイズ\n", - "BATCH_SIZE = 64\n", - "\n", - "# データセットをシャッフルするためのバッファサイズ\n", - "# （TF data は可能性として無限長のシーケンスでも使えるように設計されています。\n", - "# このため、シーケンス全体をメモリ内でシャッフルしようとはしません。\n", - "# その代わりに、要素をシャッフルするためのバッファを保持しています）\n", - "BUFFER_SIZE = 10000\n", - "\n", - "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n", - "\n", - "dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r6oUuElIMgVx" - }, - "source": [ - "## モデルの構築" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m8gPwEjRzf-Z" - }, - "source": [ - "`tf.keras.Sequential` を使ってモデルを定義します。この簡単な例では、モデルの定義に3つのレイヤーを使用しています。\n", - "\n", - "* `tf.keras.layers.Embedding`: 入力レイヤー。それぞれの文字を表す数を `embedding_dim`　次元のベクトルに変換する、訓練可能な参照テーブル。\n", - "* `tf.keras.layers.GRU`: サイズが `units=rnn_units` のRNNの一種（ここに LSTM レイヤーを使うこともできる）。\n", - "* `tf.keras.layers.Dense`: `vocab_size` の出力を持つ、出力レイヤー。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zHT8cLh7EAsg" - }, - "outputs": [], - "source": [ - "# 文字数で表されるボキャブラリーの長さ\n", - "vocab_size = len(vocab)\n", - "\n", - "# 埋め込みベクトルの次元\n", - "embedding_dim = 256\n", - "\n", - "# RNN ユニットの数\n", - "rnn_units = 1024" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MtCrdfzEI2N0" - }, - "outputs": [], - "source": [ - "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(vocab_size, embedding_dim,\n", - " batch_input_shape=[batch_size, None]),\n", - " tf.keras.layers.GRU(rnn_units,\n", - " return_sequences=True,\n", - " stateful=True,\n", - " recurrent_initializer='glorot_uniform'),\n", - " tf.keras.layers.Dense(vocab_size)\n", - " ])\n", - " return model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wwsrpOik5zhv" - }, - "outputs": [], - "source": [ - "model = build_model(\n", - " vocab_size = len(vocab),\n", - " embedding_dim=embedding_dim,\n", - " rnn_units=rnn_units,\n", - " batch_size=BATCH_SIZE)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RkA5upJIJ7W7" - }, - "source": [ - "1文字ごとにモデルは埋め込みベクトルを検索し、その埋め込みベクトルを入力として GRU を 1 タイムステップ実行します。そして Dense レイヤーを適用して、次の文字の対数尤度を予測するロジットを生成します。\n", - "\n", - "![A drawing of the data passing through the model](https://github.com/masa-ita/tf-docs/blob/site_ja_tutorials_text_text_generation/site/ja/tutorials/text/images/text_generation_training.png?raw=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ubPo0_9Prjb" - }, - "source": [ - "## モデルを試す\n", - "\n", - "期待通りに動作するかどうかを確認するためモデルを動かしてみましょう。\n", - "\n", - "最初に、出力の shape を確認します。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "C-_70kKAPrPU" - }, - "outputs": [], - "source": [ - "for input_example_batch, target_example_batch in dataset.take(1):\n", - " example_batch_predictions = model(input_example_batch)\n", - " print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q6NzLBi4VM4o" - }, - "source": [ - "上記の例では、入力のシーケンスの長さは `100` ですが、モデルはどのような長さの入力でも実行できます。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vPGmAAXmVLGC" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uwv0gEkURfx1" - }, - "source": [ - "モデルから実際の予測を得るには出力の分布からサンプリングを行う必要があります。この分布は、文字ボキャブラリー全体のロジットで定義されます。\n", - "\n", - "Note: この分布から _サンプリング_ するということが重要です。なぜなら、分布の _argmax_ をとったのでは、モデルは簡単にループしてしまうからです。\n", - "\n", - "バッチ中の最初のサンプルで試してみましょう。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4V4MfFg0RQJg" - }, - "outputs": [], - "source": [ - "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n", - "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QM1Vbxs_URw5" - }, - "source": [ - "これにより、タイムステップそれぞれにおいて、次の文字のインデックスの予測が得られます。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YqFMUQc_UFgM" - }, - "outputs": [], - "source": [ - "sampled_indices" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LfLtsP3mUhCG" - }, - "source": [ - "これらをデコードすることで、この訓練前のモデルによる予測テキストをみることができます。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xWcFwPwLSo05" - }, - "outputs": [], - "source": [ - "print(\"Input: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n", - "print()\n", - "print(\"Next Char Predictions: \\n\", repr(\"\".join(idx2char[sampled_indices ])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LJL0Q0YPY6Ee" - }, - "source": [ - "## モデルの訓練" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YCbHQHiaa4Ic" - }, - "source": [ - "ここまでくれば問題は標準的な分類問題として扱うことができます。これまでの RNN の状態と、いまのタイムステップの入力が与えられ、次の文字のクラスを予測します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "trpqTWyvk0nr" - }, - "source": [ - "### オプティマイザと損失関数の付加" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UAjbjY03eiQ4" - }, - "source": [ - "この場合、標準の `tf.keras.losses.sparse_categorical_crossentropy` 損失関数が使えます。予測の最後の次元に適用されるからです。\n", - "\n", - "このモデルはロジットを返すので、`from_logits` フラグをセットする必要があります。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4HrXTACTdzY-" - }, - "outputs": [], - "source": [ - "def loss(labels, logits):\n", - " return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n", - "\n", - "example_batch_loss = loss(target_example_batch, example_batch_predictions)\n", - "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n", - "print(\"scalar_loss: \", example_batch_loss.numpy().mean())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jeOXriLcymww" - }, - "source": [ - "`tf.keras.Model.compile` を使って、訓練手順を定義します。既定の引数を持った `tf.keras.optimizers.Adam` と、先ほどの loss 関数を使用しましょう。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DDl1_Een6rL0" - }, - "outputs": [], - "source": [ - "model.compile(optimizer='adam', loss=loss)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ieSJdchZggUj" - }, - "source": [ - "### チェックポイントの構成" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C6XBUUavgF56" - }, - "source": [ - "`tf.keras.callbacks.ModelCheckpoint` を使って、訓練中にチェックポイントを保存するようにします。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "W6fWTriUZP-n" - }, - "outputs": [], - "source": [ - "# チェックポイントが保存されるディレクトリ\n", - "checkpoint_dir = './training_checkpoints'\n", - "# チェックポイントファイルの名称\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n", - "\n", - "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n", - " filepath=checkpoint_prefix,\n", - " save_weights_only=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Ky3F_BhgkTW" - }, - "source": [ - "### 訓練の実行" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IxdOA-rgyGvs" - }, - "source": [ - "訓練時間を適切に保つために、10エポックを使用してモデルを訓練します。Google Colab を使用する場合には、訓練を高速化するためにランタイムを GPU に設定します。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7yGBE2zxMMHs" - }, - "outputs": [], - "source": [ - "EPOCHS=10" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UK-hmKjYVoll" - }, - "outputs": [], - "source": [ - "history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKkD5M6eoSiN" - }, - "source": [ - "## テキスト生成" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JIPcXllKjkdr" - }, - "source": [ - "### 最終チェックポイントの復元" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LyeYRiuVjodY" - }, - "source": [ - "予測ステップを単純にするため、バッチサイズ 1 を使用します。\n", - "\n", - "RNN が状態をタイムステップからタイムステップへと渡す仕組みのため、モデルは一度構築されると固定されたバッチサイズしか受け付けられません。\n", - "\n", - "モデルを異なる `batch_size` で実行するためには、モデルを再構築し、チェックポイントから重みを復元する必要があります。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zk2WJ2-XjkGz" - }, - "outputs": [], - "source": [ - "tf.train.latest_checkpoint(checkpoint_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LycQ-ot_jjyu" - }, - "outputs": [], - "source": [ - "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n", - "\n", - "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "model.build(tf.TensorShape([1, None]))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "71xa6jnYVrAN" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DjGz1tDkzf-u" - }, - "source": [ - "### 予測ループ\n", - "\n", - "下記のコードブロックでテキストを生成します。\n", - "\n", - "* 最初に、開始文字列を選択し、RNN の状態を初期化して、生成する文字数を設定します。\n", - "\n", - "* 開始文字列と RNN の状態を使って、次の文字の予測分布を得ます。\n", - "\n", - "* つぎに、カテゴリー分布を使用して、予測された文字のインデックスを計算します。この予測された文字をモデルの次の入力にします。\n", - "\n", - "* モデルによって返された RNN の状態はモデルにフィードバックされるため、1つの文字だけでなく、より多くのコンテキストを持つことになります。つぎの文字を予測した後、更新された RNN の状態が再びモデルにフィードバックされます。こうしてモデルは以前に予測した文字からさらにコンテキストを得ることで学習するのです。\n", - "\n", - "![To generate text the model's output is fed back to the input](https://github.com/masa-ita/tf-docs/blob/site_ja_tutorials_text_text_generation/site/ja/tutorials/text/images/text_generation_sampling.png?raw=1)\n", - "\n", - "生成されたテキストを見ると、モデルがどこを大文字にするかや、段落の区切り方、シェークスピアらしい書き言葉を真似ることを知っていることがわかります。しかし、訓練のエポック数が少ないので、まだ一貫した文章を生成するところまでは学習していません。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WvuwZBX5Ogfd" - }, - "outputs": [], - "source": [ - "def generate_text(model, start_string):\n", - " # 評価ステップ（学習済みモデルを使ったテキスト生成）\n", - "\n", - " # 生成する文字数\n", - " num_generate = 1000\n", - "\n", - " # 開始文字列を数値に変換（ベクトル化）\n", - " input_eval = [char2idx[s] for s in start_string]\n", - " input_eval = tf.expand_dims(input_eval, 0)\n", - "\n", - " # 結果を保存する空文字列\n", - " text_generated = []\n", - "\n", - " # 低い temperature　は、より予測しやすいテキストをもたらし\n", - " # 高い temperature は、より意外なテキストをもたらす\n", - " # 実験により最適な設定を見つけること\n", - " temperature = 1.0\n", - "\n", - " # ここではバッチサイズ　== 1\n", - " model.reset_states()\n", - " for i in range(num_generate):\n", - " predictions = model(input_eval)\n", - " # バッチの次元を削除\n", - " predictions = tf.squeeze(predictions, 0)\n", - "\n", - " # カテゴリー分布をつかってモデルから返された文字を予測 \n", - " predictions = predictions / temperature\n", - " predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n", - "\n", - " # 過去の隠れ状態とともに予測された文字をモデルへのつぎの入力として渡す\n", - " input_eval = tf.expand_dims([predicted_id], 0)\n", - "\n", - " text_generated.append(idx2char[predicted_id])\n", - "\n", - " return (start_string + ''.join(text_generated))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ktovv0RFhrkn" - }, - "outputs": [], - "source": [ - "print(generate_text(model, start_string=u\"ROMEO: \"))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AM2Uma_-yVIq" - }, - "source": [ - "この結果を改善するもっとも簡単な方法は、もっと長く訓練することです（`EPOCHS=30` を試してみましょう）。\n", - "\n", - "また、異なる初期文字列を使ったり、モデルの精度を向上させるためにもうひとつ RNN レイヤーを加えたり、temperature パラメータを調整して、よりランダム性の強い、あるいは、弱い予測を試してみたりすることができます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y4QwTjAM6A2O" - }, - "source": [ - "## 上級編： 訓練のカスタマイズ\n", - "\n", - "上記の訓練手順は単純ですが、制御できるところがそれほどありません。\n", - "\n", - "モデルを手動で実行する方法を見てきたので、訓練ループを展開し、自分で実装してみましょう。このことが、たとえばモデルのオープンループによる出力を安定化するための _カリキュラム学習_ を実装するための出発点になります。\n", - "\n", - "勾配を追跡するために `tf.GradientTape` を使用します。このアプローチについての詳細を学ぶには、 [eager execution guide](https://www.tensorflow.org/guide/eager) をお読みください。\n", - "\n", - "この手順は下記のように動作します。\n", - "\n", - "* 最初に、RNN の状態を初期化する。`tf.keras.Model.reset_states` メソッドを呼び出すことでこれを実行する。\n", - "\n", - "* つぎに、（1バッチずつ）データセットを順番に処理し、それぞれのバッチに対する*予測値*を計算する。\n", - "\n", - "* `tf.GradientTape` をオープンし、そのコンテキストで、予測値と損失を計算する。\n", - "\n", - "* `tf.GradientTape.grads` メソッドを使って、モデルの変数に対する損失の勾配を計算する。\n", - "\n", - "* 最後に、オプティマイザの `tf.train.Optimizer.apply_gradients` メソッドを使って、逆方向の処理を行う。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_XAm7eCoKULT" - }, - "outputs": [], - "source": [ - "model = build_model(\n", - " vocab_size = len(vocab),\n", - " embedding_dim=embedding_dim,\n", - " rnn_units=rnn_units,\n", - " batch_size=BATCH_SIZE)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qUKhnZtMVpoJ" - }, - "outputs": [], - "source": [ - "optimizer = tf.keras.optimizers.Adam()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "b4kH1o0leVIp" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def train_step(inp, target):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(inp)\n", - " loss = tf.reduce_mean(\n", - " tf.keras.losses.sparse_categorical_crossentropy(\n", - " target, predictions, from_logits=True))\n", - " grads = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " return loss" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "d4tSNwymzf-q" - }, - "outputs": [], - "source": [ - "# 訓練ステップ\n", - "EPOCHS = 10\n", - "\n", - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - "\n", - " # 各エポックの最初に、隠れ状態を初期化する\n", - " # 最初は隠れ状態は None\n", - " hidden = model.reset_states()\n", - "\n", - " for (batch_n, (inp, target)) in enumerate(dataset):\n", - " loss = train_step(inp, target)\n", - "\n", - " if batch_n % 100 == 0:\n", - " template = 'Epoch {} Batch {} Loss {}'\n", - " print(template.format(epoch+1, batch_n, loss))\n", - "\n", - " # 5エポックごとにモデル（のチェックポイント）を保存する\n", - " if (epoch + 1) % 5 == 0:\n", - " model.save_weights(checkpoint_prefix.format(epoch=epoch))\n", - "\n", - " print ('Epoch {} Loss {:.4f}'.format(epoch+1, loss))\n", - " print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n", - "\n", - "model.save_weights(checkpoint_prefix.format(epoch=epoch))" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "text_generation.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/site/ja/tutorials/text/word_embeddings.ipynb b/site/ja/tutorials/text/word_embeddings.ipynb deleted file mode 100644 index 80f85b6a4ca..00000000000 --- a/site/ja/tutorials/text/word_embeddings.ipynb +++ /dev/null @@ -1,687 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "word_embeddings.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GE91qWZkm8ZQ" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "YS3NA-i6nAFC", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7SN5USFEIIK3" - }, - "source": [ - "# 単語埋め込み (Word embeddings)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Aojnnc7sXrab" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " View on TensorFlow.org\n", - " \n", - " \n", - " \n", - " Run in Google Colab\n", - " \n", - " \n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tudzcncJXetB" - }, - "source": [ - "Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる 翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の 最新の状態を反映したものであることを保証することはできません。 この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 コミュニティによる翻訳やレビューに参加していただける方は、 [docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q6mJg1g3apaz" - }, - "source": [ - "このチュートリアルでは、単語埋め込みを紹介します。このチュートリアルには、小さいデータセットを使って単語埋め込みを最初から学習させ、その埋め込みベクトルを [Embedding Projector](http://projector.tensorflow.org) （下図参照）を使って可視化するためのプログラムがすべて含まれています。\n", - "\n", - "\"Screenshot\n", - "\n", - "## テキストを数値で表す\n", - "\n", - "機械学習モデルは、ベクトル（数値の配列）を入力として受け取ります。テキストを扱う際、最初に決めなければならないのは、文字列を機械学習モデルに入力する前に、数値に変換する（あるいはテキストを「ベクトル化」する）ための戦略です。このセクションでは、これを行う3つの戦略を見てみます。\n", - "\n", - "### ワンホット・エンコーディング\n", - "\n", - "最初のアイデアとして、ボキャブラリの中の単語それぞれを「ワンホット」エンコードするというのがあります。 \"The cat sat on the mat\" という文を考えてみましょう。この文に含まれるボキャブラリ（ユニークな単語）は、 (cat, mat, on, sat, the) です。それぞれの単語を表現するため、ボキャブラリの長さに等しいゼロベクトルを作り、その単語に対応するインデックスの場所に 1 を立てます。これを下図で示します。 \n", - "\n", - "\"Diagram\n", - "\n", - "文をエンコードしたベクトルを作成するには、その後、それぞれの単語のワンホット・ベクトルをつなげればよいのです。\n", - "\n", - "Key point: この手法は非効率です。ワンホット・エンコードされたベクトルは疎（つまり、ほとんどのインデックスではゼロ）です。ボキャブラリに 10,000 の単語があると考えてみましょう。単語をすべてワンホット・エンコードするということは、要素の 99.99% がゼロであるベクトルを作ることになります。\n", - "\n", - "### それぞれの単語をユニークな数値としてエンコードする\n", - "\n", - "2つ目のアプローチとして、それぞれの単語をユニークな数値でエンコードするというのがあります。上記の例をとれば、\"cat\" に 1、\"mat\" に 2、というふうに番号を割り当てることができます。そうすれば、 \"The cat sat on the mat\" という文は、 [5, 1, 4, 3, 5, 2] という密なベクトルで表すことができます。この手法は効率的です。疎なベクトルの代わりに、密な（すべての要素が入っている）ベクトルが得られます。\n", - "\n", - "しかしながら、このアプローチには 2つの欠点があります。\n", - "\n", - "* 整数エンコーディングは勝手に決めたものです（単語間のいかなる関係性も含んでいません）。\n", - "\n", - "* 整数エンコーディングはモデルにとっては解釈しにくいものです。たとえば、線形分類器はそれぞれの特徴量について単一の重みしか学習しません。したがって、2つの単語が似かよっていることと、それらのエンコーディングが似かよっていることの間には、なんの関係もありません。この特徴と重みの組み合わせには意味がありません。\n", - "\n", - "### 単語埋め込み\n", - "\n", - "単語埋め込みを使うと、似たような単語が似たようにエンコードされる、効率的で密な表現が得られます。重要なのは、このエンコーディングを手動で行う必要がないということです。埋め込みは浮動小数点数の密なベクトルです（そのベクトルの長さはあなたが指定するパラメータです）。埋め込みベクトルの値は指定するものではなく、学習されるパラメータです（モデルが密結合レイヤーの重みを学習するように、訓練をとおしてモデルが学習する重みです）。一般的には、（小さいデータセットの場合の）8次元の埋め込みベクトルから、大きなデータセットを扱う 1024次元のものまで見られます。高次元の埋め込みは単語間の細かな関係を取得できますが、学習にはよりたくさんのデータが必要です。\n", - "\n", - "\"Diagram\n", - "\n", - "上図は単語埋め込みを図示したものです。それぞれの単語が 4次元の浮動小数点数のベクトルで表されています。埋め込みは「参照テーブル」と考えることもできます。重みが学習された後では、テーブルを参照して、それぞれの単語を対応する密ベクトルにエンコードできます。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "u3cb82Ba9BG3", - "colab_type": "text" - }, - "source": [ - "## 設定" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "YmG3iq8d9BG4", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version は Colab 中でのみ使用できます\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "9zEnwkkz9BG6", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "import tensorflow_datasets as tfds\n", - "tfds.disable_progress_bar()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eqBazMiVQkj1" - }, - "source": [ - "## Embedding レイヤーを使う\n", - "\n", - "Keras では単語埋め込みを使うのも簡単です。[Embedding](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Embedding) レイヤーを見てみましょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2dKKV1L2Rk7e" - }, - "source": [ - "Embedding レイヤーは、（特定の単語を示す）整数のインデックスに（その埋め込みである）密なベクトルを対応させる参照テーブルとして理解することができます。埋め込みの次元数（あるいはその幅）は、取り組んでいる問題に適した値を実験して求めるパラメータです。これは、Dense レイヤーの中のニューロンの数を実験で求めるのとまったくおなじです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "BGZ5C1u39BG_", - "colab_type": "code", - "colab": {} - }, - "source": [ - "embedding_layer = layers.Embedding(1000, 5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FTq1as-v9BHC" - }, - "source": [ - "Embedding レイヤーを作成するとき、埋め込みの重みは（ほかのレイヤーとおなじように）ランダムに初期化されます。訓練を通じて、これらの重みはバックプロパゲーションによって徐々に調整されます。いったん訓練が行われると、学習された単語埋め込みは、（モデルを訓練した特定の問題のために学習された結果）単語の間の類似性をおおまかにコード化しています。\n", - "\n", - "Embedding レイヤーに整数を渡すと、結果はそれぞれの整数が埋め込みテーブルのベクトルに置き換えられます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "tjQcSAZW9BHC", - "colab_type": "code", - "colab": {} - }, - "source": [ - "result = embedding_layer(tf.constant([1,2,3]))\n", - "result.numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "G25iWX7p9BHF" - }, - "source": [ - "テキストあるいはシーケンスの問題では、入力として、Embedding レイヤーは shape が `(samples, sequence_length)` の2次元整数テンソルを取ります。ここで、各エントリは整数のシーケンスです。このレイヤーは、可変長のシーケンスを埋め込みベクトルにすることができます。上記のバッチでは、 `(32, 10)` （長さ10のシーケンス32個のバッチ）や、 `(64, 15)` （長さ15のシーケンス64個のバッチ）を埋め込みレイヤーに投入可能です。\n", - "\n", - "返されたテンソルは入力より 1つ軸が多くなっており、埋め込みベクトルはその最後の新しい軸に沿って並べられます。`(2, 3)` の入力バッチを渡すと、出力は `(2, 3, N)` となります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "djzhCGo59BHG", - "colab_type": "code", - "colab": {} - }, - "source": [ - "result = embedding_layer(tf.constant([[0,1,2],[3,4,5]]))\n", - "result.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0wxglamN9BHI" - }, - "source": [ - "シーケンスのバッチを入力されると、Embedding レイヤーは shape が `(samples, sequence_length, embedding_dimensionality)` の3次元浮動小数点数テンソルを返します。この可変長のシーケンスを、固定長の表現に変換するには、さまざまな標準的なアプローチが存在します。Dense レイヤーに渡す前に、RNNやアテンション、プーリングレイヤーを使うことができます。ここでは、一番単純なのでプーリングを使用します。[RNN\n", - " を使ったテキスト分類](https://github.com/tensorflow/docs/blob/master/site/ja/tutorials/text/text_classification_rnn.ipynb) は次のステップとしてよいチュートリアルでしょう。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aGicgV5qT0wh" - }, - "source": [ - "## 埋め込みを最初から学習する" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_Bh8B1TUT6mV" - }, - "source": [ - "IMDB の映画レビューの感情分析器を訓練しようと思います。そのプロセスを通じて、埋め込みを最初から学習します。ここでは、前処理済みのデータセットを使用します。\n", - "\n", - "テキストデータセットを最初からロードする方法については、[テキスト読み込みのチュートリアル](../load_data/text.ipynb)を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cQrYYEvvUuV7", - "colab": {} - }, - "source": [ - "(train_data, test_data), info = tfds.load(\n", - " 'imdb_reviews/subwords8k', \n", - " split = (tfds.Split.TRAIN, tfds.Split.TEST), \n", - " with_info=True, as_supervised=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dF8ORMt2U9lj" - }, - "source": [ - "エンコーダー（`tfds.features.text.SubwordTextEncoder`）を取得し、すこしボキャブラリを見てみましょう。\n", - "\n", - "ボキャブラリ中の \"\\_\" は空白を表しています。ボキャブラリの中にどんなふうに（\"\\_\"）で終わる単語全体と、長い単語を構成する単語の一部が含まれているかに注目してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yueILuntVEhr", - "colab": {} - }, - "source": [ - "encoder = info.features['text'].encoder\n", - "encoder.subwords[:20]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "47oPVxgUVd7g" - }, - "source": [ - "映画のレビューはそれぞれ長さが異なっているはずです。`padded_batch` メソッドを使ってレビューの長さを標準化します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0Ejbvj8mVbIE", - "colab": {} - }, - "source": [ - "padded_shapes = ([None],())\n", - "train_batches = train_data.shuffle(1000).padded_batch(10, padded_shapes = padded_shapes)\n", - "test_batches = test_data.shuffle(1000).padded_batch(10, padded_shapes = padded_shapes)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "meWp84bPV0pu" - }, - "source": [ - "インポートした状態では、レビューのテキストは整数エンコードされています（それぞれの整数がボキャブラリ中の特定の単語あるいは部分単語を表しています）。\n", - "\n", - "あとの方のゼロに注目してください。これは、バッチが一番長いサンプルに合わせてパディングされた結果です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "T7Tef4XhV2u9", - "colab": {} - }, - "source": [ - "train_batch, train_labels = next(iter(train_batches))\n", - "train_batch.numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zI9_wLIiWO8Z" - }, - "source": [ - "### 単純なモデルの構築\n", - "\n", - "[Keras Sequential API](../../guide/keras) を使ってモデルを定義することにします。今回の場合、モデルは「連続した Bag of Words」スタイルのモデルです。\n", - "\n", - "* 次のレイヤーは Embedding レイヤーです。このレイヤーは整数エンコードされた語彙を受け取り、それぞれの単語のインデックスに対応する埋め込みベクトルをみつけて取り出します。これらのベクトルはモデルの訓練により学習されます。このベクトルは出力配列に次元を追加します。その結果次元は `(batch, sequence, embedding)` となります。\n", - "\n", - "* 次に、GlobalAveragePooling1D レイヤーが、それぞれのサンプルについて、シーケンスの次元で平均を取り、固定長の出力ベクトルを返します。これにより、モデルは可変長の入力を最も簡単な方法で扱えるようになります。\n", - "\n", - "* この固定長のベクトルは、16個の隠れユニットを持つ全結合（Dense）レイヤーに接続されます。\n", - "\n", - "* 最後のレイヤーは、1個の出力ノードを持つ Dense レイヤーです。シグモイド活性化関数を使うことで、値は 0 と 1 の間の値を取り、レビューがポジティブ（好意的）であるかどうかの確率（または確信度）を表します。\n", - "\n", - "Caution: このモデルはマスキングを使用していません。このため、ゼロパディングが入力の一部として扱われ、結果としてパディングの長さが出力に影響を与える可能性があります。これを修正するには[マスキングとパディングのガイド](../../guide/keras/masking_and_padding)を参照してください。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pHLcFtn5Wsqj", - "colab": {} - }, - "source": [ - "embedding_dim=16\n", - "\n", - "model = keras.Sequential([\n", - " layers.Embedding(encoder.vocab_size, embedding_dim),\n", - " layers.Dense(16, activation='relu'),\n", - " layers.GlobalAveragePooling1D(),\n", - " layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JjLNgKO7W2fe" - }, - "source": [ - "### モデルのコンパイルと訓練" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lCUgdP69Wzix", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "history = model.fit(\n", - " train_batches,\n", - " epochs=10,\n", - " validation_data=test_batches, validation_steps=20)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LQjpKVYTXU-1" - }, - "source": [ - "このアプローチにより、モデルの評価時の正解率は 88% 前後に達します（モデルは過学習しており、訓練時の正解率の方が際立って高いことに注意してください）。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0D3OTmOT1z1O", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "history_dict = history.history\n", - "\n", - "acc = history_dict['accuracy']\n", - "val_acc = history_dict['val_accuracy']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "plt.figure(figsize=(12,9))\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "plt.show()\n", - "\n", - "plt.figure(figsize=(12,9))\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend(loc='lower right')\n", - "plt.ylim((0.5,1))\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KCoA6qwqP836" - }, - "source": [ - "## 学習した埋め込みの取得\n", - "\n", - "次に、訓練によって学習された単語埋め込みを取得してみます。これは、shape が `(vocab_size, embedding-dimension)` の行列になります。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "j4DOmqtw9BHf", - "colab_type": "code", - "colab": {} - }, - "source": [ - "e = model.layers[0]\n", - "weights = e.get_weights()[0]\n", - "print(weights.shape) # shape: (vocab_size, embedding_dim)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J8MiCA77X8B8" - }, - "source": [ - "この重みをディスクに出力します。[Embedding Projector](http://projector.tensorflow.org) を使うため、タブ区切り形式の2つのファイルをアップロードします。（埋め込みを含む）ベクトルのファイルと、（単語を含む）メタデータファイルです。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GsjempweP9Lq", - "colab": {} - }, - "source": [ - "import io\n", - "\n", - "encoder = info.features['text'].encoder\n", - "\n", - "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n", - "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n", - "\n", - "for num, word in enumerate(encoder.subwords):\n", - " vec = weights[num+1] # 0 はパディングのためスキップ\n", - " out_m.write(word + \"\\n\")\n", - " out_v.write('\\t'.join([str(x) for x in vec]) + \"\\n\")\n", - "out_v.close()\n", - "out_m.close()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JQyMZWyxYjMr" - }, - "source": [ - "このチュートリアルを [Colaboratory](https://colab.research.google.com) で実行している場合には、下記のコードを使ってこれらのファイルをローカルマシンにダウンロードすることができます（あるいは、ファイルブラウザを使います。*表示 -> 目次 -> ファイル* ）。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-gFbbMmvYvhp", - "colab": {} - }, - "source": [ - "try:\n", - " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download('vecs.tsv')\n", - " files.download('meta.tsv')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PXLfFA54Yz-o" - }, - "source": [ - "## 埋め込みを可視化する\n", - "\n", - "埋め込みを可視化するため、これらのファイルを Embedding Projector にアップロードします。\n", - "\n", - "[Embedding Projector](http://projector.tensorflow.org/) を開きます（あるいはローカルの TensorBoard でも実行できます）。\n", - "\n", - "* \"Load data\" をクリックします\n", - "\n", - "* 上記で作成した```vecs.tsv``` と ```meta.tsv``` の 2つのファイルをアップロードします\n", - "\n", - "\n", - "学習させた埋め込みが表示されます。単語を探し、最も近い単語を見つけることができます。たとえば、\"beautiful\" という単語を探してみてください。近くに、 \"wonderful\" のような単語が見つかると思います。\n", - "\n", - "Note: 訓練の前に重みが乱数によってどのように初期化されたかによって、結果は少し異なるかもしれません。\n", - "\n", - "Note: 実験として、もっと単純なモデルを使い、より解釈しやすい埋め込みを作ることもできます。`Dense(16)` レイヤーを削除してみてください。このモデルを再度訓練して、埋め込みの可視化をもう一度行ってみましょう。\n", - "\n", - "\"Screenshot\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iS_uMeMw3Xpj" - }, - "source": [ - "## 次のステップ" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BSgAZpwF5xF_" - }, - "source": [ - "このチュートリアルでは、小さなデータセットを使い、単語埋め込みを最初から訓練し、可視化する方法を見てきました。\n", - "\n", - "* リカレントネットワークについて学ぶには、[Keras RNN ガイド](../../guide/keras/rnn.ipynb) を参照してください。\n", - "\n", - "* テキスト分類について更に学ぶには、（全体のワークフローや、どういうときに埋め込みあるいはワンホットエンコーディングを使うべきかについて興味があれば）この実践的なテキスト分類の [ガイド](https://developers.google.com/machine-learning/guides/text-classification/step-2-5) を推奨します。" - ] - } - ] -} \ No newline at end of file diff --git a/site/ja/xla/architecture.md b/site/ja/xla/architecture.md deleted file mode 100644 index a3d3c571d9f..00000000000 --- a/site/ja/xla/architecture.md +++ /dev/null @@ -1,54 +0,0 @@ -# XLA 概要 - -
    - -
    - -Note: XLAは現在開発中であるため、特定の状況でメモリ使用量の増大や性能の悪化を引き起こす場合があります。 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -XLA(Accelerated Linear Algebra)は線形代数の演算に特化したコンパイラで、XLAを使うことでTensorFlowの演算を最適化し、メモリ使用量、性能、サーバやモバイル環境での移植性の面での改善が期待できます。現在のところ、ほとんどのユーザにとってXLAを使うことによる大きな恩恵は得られないかもしれませんが、[just-in-time (JIT) コンパイル機能](https://www.tensorflow.org/xla/jit) や [ahead-of-time (AOT) コンパイル機能](https://www.tensorflow.org/xla/tfcompile) を通して、実験的にXLAを使っていただくのは歓迎です。ただし、新たなハードウエアアクセラレータの開発者については、XLAを試してみることをおすすめします。 - -XLAは実験的なフレームワークであり、現在活発に開発されています。既存のオペレーションのセマンティクスが変わることはほとんどないと思いますが、重要なユースケースに対応するために新たなオペレーションが追加されることがあります。XLAチームは、GitHubを通したコミュニティへの貢献や、不足している機能に関するフィードバックを歓迎します。 - - -## なぜXLAを開発したか？ - -TensorFlowと連携するXLAを開発した目的はいくつかあります。 - -* **実行速度の向上**: サブグラフをコンパイルし、TensorFlowランタイムのオーバヘッドを削減することで、軽量なオペレーションの実行時間を短縮します。また、複数オペレーションを結合することで、メモリのオーバヘッドを削減します。さらに、既知のテンソルの形状に特化することで、より積極的に定数畳み込みができるようにします。 -* **メモリ使用量の改善**: メモリ使用量の解析とスケジューリングによって、オペレーションの中間データを保持する領域を削減します。 -* **独自オペレーションへの依存度削減**: 自動的に結合された低レベルなオペレーションの性能を向上させ、人手による独自オペレーションと同等の性能を得られるようにすることで、独自オペレーションを作成する必要性が無くなります。 -* **モバイル環境でのディスク占有スペース削減**: サブグラフをAOTコンパイルし、他のアプリケーションに直接リンク可能なオブジェクトとヘッダを出力することで、TensorFlowのランタイムを削除します。これによって、モバイルでの推論時のディスク占有スペースを桁違いに削減できます。 -* **移植性の向上**: TensorFlowのプログラムの大部分を修正することなく、新しいハードウェア向けのバックエンドを書くことが容易になります。これは、TensorFlowのプログラムを書き換えて新しいハードウェア向けのオペレーションを作る方式とは全く異なるものです。 - - -## XLAはどのように動くのか？ - -XLAの入力となる言語は、"HLO IR"または単にHLO(High Level Optimizer)と呼ばれます。HLOのセマンティクスは、[Operation Semantics](https://www.tensorflow.org/xla/operation_semantics) に記載されています。HLOは、コンパイラで扱う [中間表現](https://ja.wikipedia.org/wiki/%E4%B8%AD%E9%96%93%E8%A1%A8%E7%8F%BE) と考えるとわかりやすいかもしれません。 - -XLAはHLOで定義されたグラフ("Computations")を、様々なハードウェアアーキテクチャの実行コードにコンパイルします。[Developing a new backend for XLA](https://www.tensorflow.org/xla/developing_new_backend) に記載されているように、XLAを新たなハードウェアで動作させることが容易であるという点で、XLAはモジュール化されていると言えます。[CPU (x86-64、ARM64) 向けの処理](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/xla/service/cpu) と [NVIDIA GPU向けの処理](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/xla/service/gpu) は、TensorFlowのメインのソースコードツリーから参照することができます。 - -次に示す図は、XLAの内部で行われているコンパイル処理を示しています。 - -![](https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/compiler/xla/g3doc/images/how-does-xla-work.png) - -XLAは、[CSE](https://ja.wikipedia.org/wiki/%E5%85%B1%E9%80%9A%E9%83%A8%E5%88%86%E5%BC%8F%E9%99%A4%E5%8E%BB) やオペレーション結合、計算時のメモリ割り当て解析といった、ターゲットに依存しない最適化や解析を行います。 - -ターゲットに依存しない最適化処理の後、XLAはHLO computationをバックエンドに転送します。バックエンドでは、ターゲット固有の情報を考慮してHLOレベルでの最適化を行います。例えば、バックエンドがXLA GPUである場合、よりGPUのプログラミングモデルに適したオペレーションの結合に加え、computationを複数のストリームへ分割して割り当てる方法も決定します。さらにバックエンドは、オペレーションの集合パターンが、最適化されたライブラリ呼び出しと一致するか確認します。 - -次に、ターゲット固有のコードを生成します。XLAに付随するCPUとGPUのバックエンドは、low-level IRとして [LLVM](http://llvm.org/) を採用し、最適化やコード生成もLLVMを使って行います。これらのバックエンドは、XLAのHLO computationを表現するために効率的なLLVM IRを出力し、LLVMを使ってLLVM IRからネイティブコードを生成します。 - -現在、GPUのバックエンドはLLVM NVPTXのバックエンド経由でNVIDIA GPUをサポートし、CPUのバックエンドは複数のCPUのISAをサポートしています。 - - -## サポートするプラットフォーム - -XLAは、x86-64とNVIDIA GPU向けに [JITコンパイル機能](https://www.tensorflow.org/xla/jit) を、x86-64とARM向けに [AOTコンパイル機能](https://www.tensorflow.org/xla/tfcompile) をサポートしています。 diff --git a/site/ja/xla/custom_call.md b/site/ja/xla/custom_call.md deleted file mode 100644 index c19ec139de6..00000000000 --- a/site/ja/xla/custom_call.md +++ /dev/null @@ -1,195 +0,0 @@ -# XLAのカスタムコール - -このドキュメントでは、XLA「カスタムコール」の書き方と使い方について説明します。 -カスタムコールは、C++やCUDAなどのプログラミング言語で書かれたコードを、XLAプログラムから呼び出すことができます。 - -警告： カスタムコールは、パワーユーザ用の低レベル機能です。 -カスタムコールを使うと、デバッグしにくい（そして気づきにくい）状態の中で、あなたのプログラムが壊れやすくなります。 -何かおかしくなったとき、あなた自身でXLAをデバッグできる準備ができていないなら、カスタムコールを使うべきではありません。 -トラブルに遭遇したとしても、XLA開発者から支援はあまりもらえないと思っているべきです。 - -警告： カスタムコールのAPI/ABIは、現時点では固まっていません。 -きまぐれに変更するつもりはありませんが、変更する可能性はあります。 -将来可能性があるいくつかの変更については以下で説明します。 - -## CPUでのカスタムコール - -XLAクライアントAPI経由で、カスタムコールを表すHLO命令を作ることができます。 -これは、執筆時点ではTensorFlow経由では公開されていません。 - -例えば、以下のコードはCPU上で `A[i] = B[i % 128] + C[i]` をカスタムコールを使用して計算します（もちろん、通常のHLOを使って計算できますし、すべきです！）。 - -```c++ -#include "tensorflow/compiler/xla/client/xla_builder.h" -#include "tensorflow/compiler/xla/service/custom_call_target_registry.h" - -void do_it() { - xla::XlaBuilder b("do_it"); - xla::XlaOp param0 = - xla::Parameter(0, xla::ShapeUtil::CreateShape(F32, {128}), "p0"); - xla::XlaOp param1 = - xla::Parameter(1, xla::ShapeUtil::CreateShape(F32, {2048}), "p1"); - xla::XlaOp custom_call = - xla::CustomCall(&b, "do_custom_call", /*operands=*/{param0, param1}, - /*output_shape=*/ShapeUtil::CreateShape(F32, {2048})); -} - -void do_custom_call(void* out, const void** in) { - float* out_buf = reinterpret_cast(out); - const float* in0 = reinterpret_cast(in[0]); - const float* in1 = reinterpret_cast(in[1]); - for (int i = 0; i < 2048; ++i) { - out_buf[i] = in0[i % 128] + in1[i]; - } -} -XLA_REGISTER_CUSTOM_CALL_TARGET(do_custom_call, "Host"); -``` - -関数 `do_custom_call` は、処理を実行するバッファの次元情報を知っている必要があります。 -この例では、サイズ128と2048を直書きしています。 -もし、これをしたくない場合には、パラメータとして次元情報を関数に渡すことができます。 - -## GPUでのカスタムコール - -GPUのカスタムコールのフレームワークは、CPUのフレームワークと多少異なります。 -ここでは、上記のCPUコードと同じ `A[i] = B[i % 128] + C[i]` の計算を行うCUDAの例をあげます。 - -```c++ -void do_it() { /* 上と同じ実装 */ } - -__global__ custom_call_kernel(const float* in0, const float* in1, float* out) { - size_t idx = threadIdx.x * blockSize.x + gridIdx.x; - out[idx] = in0[idx % 128] + in1[idx]; -} - -void do_custom_call(CUstream stream, void** buffers, - const char* opaque, size_t opaque_len) { - const float* in0 = reinterpret_cast(buffers[0]); - const float* in1 = reinterpret_cast(buffers[1]); - float* out = reinterpret_cast(buffers[2]); - - const int64 block_dim = 64; - const int64 grid_dim = 2048 / block_dim; - custom_call_kernel<<>>(in0, in1, out); -} -XLA_REGISTER_CUSTOM_CALL_TARGET(do_custom_call, "CUDA"); -``` - -最初にGPUカスタムコール関数が、*CPU上で実行される関数である*ことに注意してください。 -CPU用 `do_custom_call` 関数は、GPU上での作業をキューに入れる役割を果たします。 -ここではCUDAカーネルを起動していますが、cublasを呼び出すような他のこともできます。 - -`buffers` はホスト上にあるポインタの配列で、各要素はデバイス（つまりGPU）メモリを指しています。 -パラメータが最初に来て、そのあと出力の値が来ます。 -これは、CPUの呼び出し規則とは大きく異なり、２つのパラメータ、`ins` と `out` があります。 -違う実装をした主な理由は、タプル型の入出力を効率的に処理するためです。 -以下の章をごらんください。 - -CPUの例のように、入出力バッファの大きさをカスタムコールに直書きしました。 -しかし、CPUの場合とは異なり、オペランドとしてバッファの大きさを渡してもうまく動きません。 -通常、CPU上でバッファの大きさが分かっている必要があります。例えば、カーネルを起動するとき、block/gridの次元情報が必要です。 -しかし、カスタムコールにオペランドとしてバッファサイズが渡されると、この値はGPUメモリ上にあります。 -処理の開始時に、この値を読むためのだけに処理が重い同期的なデバイスからホストへのメモリコピーを実行する必要があります。 - -これを回避するために `opaque` パラメータを用意しています。 -カスタムコールをつくるときに、任意のバイト文字列をセットできます。 - -```c++ -std::string opaque = "..."; -xla::CustomCall(&b, "do_custom_call", /*operands=*/{param0, param1}, - /*output_shape=*/ShapeUtil::CreateShape(F32, {2048}), - opaque); -``` - -`xla::Shape` はプロトコルバッファ表現を持つので、 `opaque` の内部にこのシリアライズされた表現を保存してGPUカスタムコールの内部でデシリアライズできます。 -ただし、 `xla::ShapeProto` は頻繁には変更されませんが、 `xla::Shape` は*変更されます*。 -gitログをチェックして、過去にどのような変更が行われたか確認してください。 - -## カスタムコールにタプルを渡す - -以下のカスタムコール呼び出しを考えます。 - -```c++ -using xla::ShapeUtil; -Shape p0_shape = ShapeUtil::MakeTuple({ - ShapeUtil::MakeShape(F32, {32}), - ShapeUtil::MakeTuple({ - ShapeUtil::MakeShape(F32, {64}), - ShapeUtil::MakeShape(F32, {128}), - }), - ShapeUtil::MakeShape(F32, {256}), -}); -xla::XlaOp p0 = xla::Parameter(0, p0_shape, "p0"); - -Shape out_shape = ShapeUtil::MakeTuple({ - ShapeUtil::MakeShape(F32, {512}), - ShapeUtil::MakeShape(F32, {1024}), -}); -xla::CustomCall(&b, "do_custom_call", /*operands=*/{p0}, out_shape); -``` - -CPUとGPUの両方で、タプルはポインタの配列としてメモリ内で表現されます。 -C++擬似コードでは、上記のパラメータ0は以下のように配置されます。 - -```c++ -// 上記のカスタムコールのパラメータ0のメモリ内レイアウト -// CPUとGPUの両方で有効です。 -float* subbuf0 = new float[32]; -float* subbuf1 = new float[64]; -float* subbuf2 = new float[128] -float* subbuf3 = new float[256]; - -void* subtuple = new void*[2]; -(*subtuple)[0] = subbuf1; -(*subtuple)[1] = subbuf2; - -void* p0 = new void*[3]; -(*p0)[0] = subbuf0; -(*p0)[1] = subtuple; -(*p0)[2] = subbuf3; -``` - -CPUとGPUでメモリ内表現は同じですが、CPUとGPUのカスタムコール呼び出し規約では処理方法が異なります。 - -### 一時バッファとしてのタプル出力 - -カスタムコールへのタプル入力は便利ですが、厳密には必須ではありません。 -カスタムコールへのタプル入力がサポートされていないなら、カスタムコールにタプルを渡す前にget-tuple-elementを使ってタプルを分解できます。 - -一方、タプル*出力*は、他の方法ではできないことができます。 - -タプル出力を持つ明確な理由は、それがカスタムコール（または、他のXLA命令）が複数の独立な配列を返す方法だからです。 - -しかし、あまり明確ではないですが、タプル出力はカスタムコールに一時メモリを提供する方法でもあります。 -ええ、*出力*は一時バッファを表現できます。 -出力バッファはオペレーションが書き込めるという性質を持っていて、書き込まれたあとに読み出すことができます。 -これこそが、まさに一時バッファに必要なものです。 - -上の例で、 `F32[1024]` を一時バッファとして使いたいとします。 -上記のようにHLOを記述して、単にカスタムコールのタプルインデックス1を決して読まないようにします。 - -### CPUカスタムコールでのタプル - -CPUコードには、 `do_custom_call(const void** ins, void* out)` 関数があります。 -`ins` は `param0` を指す要素が１つだけの配列です。 -`param0` のサブバッファは、そのポインタをデリファレンスしてアクセスできます。 -`output_tuple` のサブバッファは、`out` をデリファレンスしてアクセスできます。 - -### GPUカスタムコールでのタプル - -GPUコードには、 `do_custom_call(..., void** buffers, ...)` 関数があります。 -この場合 `buffers` は、入出力の各末端のバッファが一要素に対応する、*６つ*のデバイスポインタを持つホストの配列です。 -フラットリストを生成するために、パラメータと出力に対して反復処理をおこない、それぞれについてその形状を行きがけ順に走査します。 -具体的には: - -```c++ -// 上記のカスタムコールのための、GPUカスタムコール関数への -// `buffers` パラメータのレイアウト。 -buffers[0] == subbuf0 -buffers[1] == subbuf1 -buffers[2] == subbuf2 -buffers[3] == subbuf3 -buffers[4] == output_subbuf0 -buffers[5] == output_subbuf1 -``` diff --git a/site/ja/xla/developing_new_backend.md b/site/ja/xla/developing_new_backend.md deleted file mode 100644 index 5864688e427..00000000000 --- a/site/ja/xla/developing_new_backend.md +++ /dev/null @@ -1,59 +0,0 @@ -# 新しいXLAのバックエンド開発 - -Note: これらのドキュメントは私たちTensorFlowコミュニティが翻訳したものです。コミュニティによる -翻訳は**ベストエフォート**であるため、この翻訳が正確であることや[英語の公式ドキュメント](https://www.tensorflow.org/?hl=en)の -最新の状態を反映したものであることを保証することはできません。 -この翻訳の品質を向上させるためのご意見をお持ちの方は、GitHubリポジトリ[tensorflow/docs](https://github.com/tensorflow/docs)にプルリクエストをお送りください。 -\ -コミュニティによる翻訳やレビューに参加していただける方は、 -[docs-ja@tensorflow.org メーリングリスト](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja)にご連絡ください。 - -本予備ガイドは、効率的な方法でTensorFlowをハードウェアに容易に対応させたいと考えている、アーリーアダプターのためのものです。 -本ガイドは1つ1つ丁寧に説明したものではなく、LLVM、Bazel、TensorFlowの知識を前提としています。 - -XLAは、新しいアーキテクチャやアクセラレータが、TensorFlowのグラフを処理するバックエンドを実装するための抽象的なインターフェースを提供します。 -XLAへの対応は、新しいハードウェア向けに既存のすべてのTensorFlowのオペレーションを実装するのと比べて、はるかに簡潔でスケーラブルです。 - - -実装のほとんどは、以下のシナリオのうちの1つに分類されます。 - -1. LLVMのバックエンドが存在するかしないかにかかわらず、公式にXLAでサポートされていない既存のCPUアーキテクチャ -2. LLVMのバックエンドが存在する、CPUではないハードウェア -3. LLVMのバックエンドが存在しない、CPUではないハードウェア - -Note: LLVMのバックエンドとは、公式にリリースされたLLVMのバックエンド、または企業内で開発されたカスタマイズ版LLVMのバックエンドのことを指します。 - - -## シナリオ1: 公式にXLAでサポートされていない既存のCPUアーキテクチャ - -このシナリオの場合、既存の [XLA CPUバックエンド](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/xla/service/cpu) を見ることから始めてください。 -XLAのCPUバックエンド間の主な違いは、LLVMによって生成されるコードであることから、XLAではLLVMを使って異なるCPUをTensorFlowに簡単に対応できます。 -Googleは、x64とARM64のアーキテクチャに対してXLAを試験しています。 - -もしハードウェア企業がハードウェア向けのLLVMのバックエンドをもつ場合、ビルドされたLLVMのバックエンドをXLAに接続することは簡単です。 -JITモードでは、XLAのCPUバックエンドはホスト側のCPUのコードを生成します。 -Ahead-Of-Timeコンパイルでは、[`xla::AotCompilationOptions`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/service/compiler.h) が対象とするアーキテクチャに対して設定するLLVM Tripleを提供します。 - -もし既存のLLVMのバックエンドがなくてもコード生成器が違う形で存在するならば、既存のCPUバックエンドの大部分を再利用できる可能性があります。 - - -## シナリオ2: LLVMのバックエンドが存在する、CPUではないハードウェア - -LLVM IRを出力する既存の [`xla::CPUCompiler`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc) や [`xla::GPUCompiler`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc) クラスをベースとして、新しい [`xla::Compiler`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/service/compiler.h) の実装を作ることが可能です。 -ハードウェアの性質によりLLVM IRの生成方法は異なりますが、多くのコードは既存のバックエンドと共有できるでしょう。 - -よい参考例は、XLAの [GPUバックエンド](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/xla/service/gpu) です。 -GPUのバックエンドはCPUとは異なるISAをターゲットとするため、GPUドメイン固有なコードの生成方法になります。 -ほかの種類のハードウェア、たとえば（アップストリームのLLVMのバックエンドをもつ）HexagonのようなDSPは、LLVM IRの生成のしくみを再利用することができますが、ほかの部分は固有のものになるでしょう - - -## シナリオ3: LLVMのバックエンドが存在しない、CPUではないハードウェア - -LLVMを利用できない場合、対象のハードウェア向けに新しいバックエンドを実装することが最良の選択肢となります。 -この選択肢は、多大な労力を必要とします。 -実装しなければならないクラスは次に示すとおりです。 - -* [StreamExecutor](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/stream_executor/stream_executor.h): 多くのデバイスでは、`StreamExecutor` のすべてのメソッドが必要になることはありません。詳細は既存の `StreamExecutor` の実装を見てください。 -* [xla::Compiler](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/service/compiler.h): 本クラスは、HLO Computationから `xla::Executable` へのコンパイル処理を隠蔽します。 -* [xla::Executable](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/service/executable.h): 本クラスは、コンパイル済みのComputationをプラットフォーム上で実行するために使用されます。 -* [xla::TransferManager](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/compiler/xla/service/transfer_manager.h): 本クラスは、与えられたデバイスメモリのハンドルからXLAのリテラルデータを構築するための、プラットフォーム特有のしくみを提供することを可能にします。言い換えれば、ホストからデバイスまたはその反対のデータ転送処理を隠蔽します。 diff --git a/site/ja/xla/shapes.md b/site/ja/xla/shapes.md deleted file mode 100644 index 435a5e43097..00000000000 --- a/site/ja/xla/shapes.md +++ /dev/null @@ -1,113 +0,0 @@ -# シェイプとレイアウト - -XLA `shape` proto（[xla_data.proto](https://www.tensorflow.org/code/tensorflow/compiler/xla/xla_data.proto)） は、N次元配列（略して配列）のランクとサイズ、データ型について記述します。 - - -## 用語、表記、規約 - -* 配列のランクは次元の数とおなじです。*正しい配列のランク* は、次元数が1以上の配列のことを指します。 - -* 次元は、`N` 次元配列について、`0` から `N-1` で番号づけされます。次元番号は、便宜的につけられたラベルです。次元の順序は、シェイプのレイアウトの優先順序（minor/major order）を意味しているわけではありません。レイアウトは、`Layout` protoによって決まります。 - -* 規約として、次元は次元番号の昇順にリスト化されます。たとえば、サイズが `[A x B x C]` の3次元配列は、次元0のサイズが `A`、次元1のサイズが `B`、次元2のサイズが `C` となります。 - -XLAにあるいくつかのユーティリティは、Pythonと同様に負のインデックスをサポートするものがあります。たとえば、次元-1は、最後の次元（`N` 次元配列では `N-1` とおなじ）を指します。 -上で述べた3次元配列では、次元-1はサイズ `C`、次元-2はサイズ `B` をもつといったように。 - -* 2、3、4次元の配列は、次元と関連付けて特定の文字で明記されることがよくあります。たとえば、2次元配列では以下のようになります。 - * 次元0: `y` - * 次元1: `x` -* 3次元配列では、以下のようになります。 - * 次元0: `z` - * 次元1: `y` - * 次元2: `x` -* 4次元配列では、以下のようになります。 - * 次元0: `p` - * 次元1: `z` - * 次元2: `y` - * 次元3: `x` - -* XLA APIにある次元を受け取る関数は、昇順の次元番号として扱います。これは、`initializer_list` として次元を渡したときとおなじになります。 - -たとえば、`ShapeUtil::MakeShape(F32, {A, B, C, D})` は、`[A, B, C, D]` で構成される配列の次元サイズからなるシェイプを構築します。 - -## レイアウト - -`Layout` protoは、配列がメモリ上でどのように表現されるかを記述します。 -`Layout` protoは、次に示すフィールドを含んでいます。 - -``` -message Layout { - repeated int64 minor_to_major = 1; - repeated int64 padded_dimensions = 2; - optional PaddingValue padding_value = 3; -} -``` - -### 次元の優先順序 - -必要なフィールドは、`minor_to_major` のみです。 -このフィールドは、シェイプ内の次元の優先順序を示しています。 -`minor_to_major` の値は、配列の次元順序（`N` 次元配列なら `0` から `N-1`）で、もっとも優先度の低い次元を示す最初の値から、もっとも優先度の高い次元を示す最後の値になります。 -もっとも優先度の低い次元は、連続なメモリに配置された配列の要素を走査したときに、もっとも頻繁に変化する次元になります。 - -たとえば、サイズが `[2 x 3]` である、次に示す2次元配列を考えてみましょう。 - -``` -a b c -d e f -``` - -ここで次元 `0` はサイズが2で、次元 `1` はサイズが3です。 -レイアウト内の `minor_to_major` フィールドが `[0, 1]` の場合、次元 `0` はもっとも優先度の低い次元となり、次元 `1` はもっとも優先度が高い次元になります。 -これは、連続なメモリでは、以下に示すようなレイアウトと一致します。 - -``` -a d b e c f -``` - -`0` から `N-1` までの次元優先度順のことを、（ランクが2の場合に） *列優先* と呼びます。 -巨大な次元順序を想定した場合、コード上のこのレイアウトにちなんで、別名"次元0が優先度が低い"と呼びます。 - -一方で、レイアウトの `minor_to_major` フィールドが `[1, 0]` の場合は、連続なメモリ内のレイアウトは次のようになります。 - -``` -a b c d e f -``` - -`N-1` から `0` までの次元優先度順のことを、（ランクが2の場合に） *行優先* を呼びます。 -巨大な次元順序を想定した場合、コード上のこのレイアウトにちなんで、別名"次元0の優先度が高い"と呼びます。 - - -#### デフォルトの優先度順 - -新しく作られたシェイプのデフォルトのレイアウトは、"次元順が高いほうから低いほう"（ランク2の場合は行優先）になります。 - - -## パディング - -パディング量は、任意のフィールド `padded_dimensions` と `padding_value` に定義されています。 -フィールド `padded_dimensions` は、それぞれの次元についてパディングするサイズ（幅）を示しています。 -存在する場合は、`padded_dimensions` の要素数がシェイプのランクとおなじである必要があります。 - -たとえば、上で `[2 x 3]` の配列を紹介しましたが、もし `padded_dimensions` が `[3, 5]` ならば、次元0は幅3でパディングされ、次元1は幅5でパディングされます。 -（パディング量0、列優先のレイアウトを想定した場合、）連続なメモリのレイアウトは、次のようになります。 - -``` -a d 0 b e 0 c f 0 0 0 0 0 0 0 -``` - -これは、おなじ次元優先度順で次に示す配列のレイアウトと同様です。 - -``` -a b c 0 0 -d e f 0 0 -0 0 0 0 0 -``` - -## 配列のインデックス化 - -[index_util.h](https://www.tensorflow.org/code/tensorflow/compiler/xla/index_util.h) にある `IndexUtil` クラスは、与えられたシェイプやレイアウトに対して、複数次元のインデックスと連続なインデックスとの間で、変換するためのユーティリティを提供します。 -複数次元のインデックスは、各次元に対して `int64` 型のインデックスを含んでいます。 -連続なインデックスは、単一の `int64` 型の値で、配列を保持するバッファをインデックス化します。 -シェイプとレイアウトを簡単に生成・扱うためのユーティリティのある、`shape_util.h` と `layout_util.h` を見てください。 diff --git a/site/ja/xla/tfcompile.md b/site/ja/xla/tfcompile.md deleted file mode 100644 index 20fa30ece1e..00000000000 --- a/site/ja/xla/tfcompile.md +++ /dev/null @@ -1,252 +0,0 @@ -# AOTコンパイルの利用 - -## tfcompileとは？ - -`tfcompile` は、TensorFlowのグラフを実行コードへ事前に(AOT)コンパイルするための標準ツールです。 -全体のバイナリサイズを減らすことに加えて、いくつかの実行時オーバヘッドを無くすことができます。 -`tfcompile` の典型的な使い方は、推論用の計算グラフをモバイルデバイス向けの実行コードへコンパイルすることです。 - -TensorFlowのグラフは、通常TensorFlowのランタイムによって実行されます。 -これは、グラフの各ノードを実行することになるため、実行時オーバヘッドを招きます。 -TensorFlowのグラフやランタイムのコードが必要になるため、全体のバイナリサイズも大きくなります。 -`tfcompile` よって生成される実行コードは、TensorFlowのランタイムを使わず、実際に計算で使用するカーネルのみに依存します。 - -コンパイラは、XLAフレームワークの上に作られています。 -TensorFlowとXLAフレームワークをつなぐコードは、[tensorflow/compiler](https://www.tensorflow.org/code/tensorflow/compiler/) に存在します。 - - -## tfcompileは何をするか？ - -`tfcompile` は、TensorFlowの概念であるfeedとfetchによって形作られるサブグラフを受け取り、サブグラフを満たすファンクションを生成します。 -`feeds` はファンクションの入力引数、`fetches` はファンクションの出力引数です。 -刈り込んだ結果のサブグラフは、PlaceholderとVariableノードを含めることができないため、すべての入力はfeedによって指定される必要があります。 -すべてのPlacerholderとVariableは、feedとして指定されるのは共通であり、最終的なサブグラフはこれらのノードを一切含みません。 -生成されたファンクションは、ファンクションのシグネチャをエクスポートするヘッダファイルと、実装を含むオブジェクトファイルからなる `cc_library` としてパッケージ化されます。 -ユーザーは、生成されたファンクションを適切に呼び出すコードを書きます。 - - -## tfcompileの利用 - -このセクションでは、`tfcompile` を使ってTensorFlowのサブグラフから実行可能バイナリを生成するための、高レベルのステップを詳しく述べます。 -ステップは、以下からなります。 - -* ステップ1: コンパイルするサブグラフを構成する -* ステップ2: サブグラフをコンパイルするための `tf_library` ビルドマクロを利用する -* ステップ3: サブグラフを呼び出すコードを書く -* ステップ4: 最終的なバイナリを作成する - - -### ステップ1: コンパイルするサブグラフを構成する - -生成されたファンクションの入力および出力引数に相当する、feedとfetchを決めます。 -そして、[`tensorflow.tf2xla.Config`](https://www.tensorflow.org/code/tensorflow/compiler/tf2xla/tf2xla.proto) の `feeds` および `fetches` を設定します。 - -```textproto -# 各feedは、生成されたファンクションにおける入力の位置指定引数です。 -# エントリの順序と入力引数の順序は一致します。 -# ここで "x_hold" と "y_hold" は、グラフ上に定義されたPlaceholderノードの名前を指します。 -feed { - id { node_name: "x_hold" } - shape { - dim { size: 2 } - dim { size: 3 } - } -} -feed { - id { node_name: "y_hold" } - shape { - dim { size: 3 } - dim { size: 2 } - } -} - -# 各fetchは、生成されたファンクションにおける出力の位置指定引数です。 -# エントリの順序と出力引数の順序は一致します。 -# ここで "x_y_prod" は、グラフ上に定義されたMatmulノードの名前を指します。 -fetch { - id { node_name: "x_y_prod" } -} -``` - - -### ステップ2: サブグラフをコンパイルするためのtf_libraryビルドマクロを利用する - -このステップでは、`tf_library` ビルドマクロを利用して、グラフを `cc_library` に変換します。 -`cc_library` は、グラフから生成されたコードを含んだオブジェクトファイルと、生成されたコードにアクセスするためのヘッダファイルから構成されます。 -`tf_library` は、TensorFlowのグラフを実行コードにコンパイルするための `tfcompile` を利用しています。 - -```build -load("//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") - -# グラフを実行コードにコンパイルするために、tf_libraryマクロを使用します。 -tf_library( - # nameは、以下のビルドルールを生成するために使用します。 - # : cc_libraryは、生成されたヘッダとオブジェクトファイルをパッケージ化します。 - # _test : cc_testは、簡単なテストとベンチマークを含みます。 - # _benchmark : cc_binaryは、最小限の依存関係を持つスタンドアロンなベンチマークを含み、 - # モバイルデバイスで実行できます。 - name = "test_graph_tfmatmul", - # cpp_classには、名前空間を含む生成後のC++のクラス名を指定します。 - # クラスは、与えられた名前空間、もし名前空間が与えられていない場合はグローバルな名前空間に生成されます。 - cpp_class = "foo::bar::MatMulComp", - # graphには、入力となるGraphDefを指定しますが、デフォルトではバイナリフォーマットを期待しています。 - # テキストフォーマットを使用する場合、接尾辞 '.pbtex' を使ってください。 - # 入力のfeedと出力のfetchを含んだこの入力グラフから、サブグラフが生成されます。 - # PlaceholderやVariableのOperationは、このサブグラフには存在しません。 - graph = "test_graph_tfmatmul.pb", - # configには、入力となるConfigを指定しますが、デフォルトではバイナリフォーマットを期待しています。 - # テキストフォーマットを使用する場合、接尾辞 '.pbtex' を使ってください。 - # これは前のステップにおいて、feedとfetchを指定したところになります。 - config = "test_graph_tfmatmul.config.pbtxt", -) -``` - -> この例で使用するGraphDef (test_graph_tfmatmul.pb)を生成するためには、--out_dirフラグを使って出力場所を指定した状態で [make_test_graphs.py](https://www.tensorflow.org/code/tensorflow/compiler/aot/tests/make_test_graphs.py) を実行してください。 - -典型的なグラフとして、学習時に学習される重みを表現する [`Variables`](https://www.tensorflow.org/guide/variables) を含んだものがありますが、`tfcompile` は `Variables` を含んだサブグラフをコンパイルできません。 -ツール [freeze_graph.py](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py) は、チェックポイントファイルに保存された値を使って、Variablesを定数に変換します。 -`tf_library` マクロは便宜上、このツールを実行する `freeze_checkpoint` 引数をサポートします。 -[tensorflow/compiler/aot/tests/BUILD](https://www.tensorflow.org/code/tensorflow/compiler/aot/tests/BUILD) には、より多くの例があります。 - -> コンパイルされたサブグラフに現れた定数は、直接生成されるコードへコンパイルされます。生成されたファンクションに定数を渡すためには、これらをコンパイルしてしまうのではなく、単にfeedとして渡します。 - -`tf_library` ビルドマクロに関する詳細は、[tfcompile.bzl](https://www.tensorflow.org/code/tensorflow/compiler/aot/tfcompile.bzl) を参照してください。 - -基本となる `tfcompile` ツールの詳細については、[tfcompile_main.cc](https://www.tensorflow.org/code/tensorflow/compiler/aot/tfcompile_main.cc) を参照してください。 - - -### ステップ3: サブグラフを呼び出すコードを書く - -このステップでは、生成されたコードを呼び出すために、前ステップにおける `tf_library` ビルドマクロによって生成されたヘッダファイル(`test_graph_tfmatmul.h`) を使います。 -ヘッダファイルは、ビルドパッケージとおなじく `bazel-genfiles` に配置され、`tf_library` ビルドマクロに設定されたnameアトリビュートに基づいて命名されます。 -たとえば、`test_graph_tfmatmul` のために生成されるヘッダは、`test_graph_tfmatmul.h` になるでしょう。 -以下は、生成されたものに関する省略版です。 -`bazel-genfiles` に生成されたファイルは、役立つ付加的なコメントを含みます。 - -```c++ -namespace foo { -namespace bar { - -// MatMulCompは、事前にTensorFlowのグラフとして指定された計算を表現し、 -// 実行コードへコンパイルされました。 -class MatMulComp { - public: - // AllocModeは、バッファの割り当てモードを制御します。 - enum class AllocMode { - ARGS_RESULTS_AND_TEMPS, // 引数と結果、そして一時的に使用するバッファを割り当てます。 - RESULTS_AND_TEMPS_ONLY, // 結果と一時的に使用するバッファのみを割り当てます。 - }; - - MatMulComp(AllocMode mode = AllocMode::ARGS_RESULTS_AND_TEMPS); - ~MatMulComp(); - - // 引数のバッファから入力を読み込んで計算を実行し、出力結果をバッファに書き込みます。 - // 成功時にはtrueを返し、失敗時にはfalseを返します。 - bool Run(); - - // 入力バッファを管理するメソッドです。バッファは、行優先のデータ順序となります。 - // それぞれの位置指定引数のためのメソッド群があります。 - void** args(); - - void set_arg0_data(float* data); - float* arg0_data(); - float& arg0(size_t dim0, size_t dim1); - - void set_arg1_data(float* data); - float* arg1_data(); - float& arg1(size_t dim0, size_t dim1); - - // 出力バッファを管理するメソッドです。バッファは、行優先のデータ順序となります。 - // Runの呼び出しが成功したあとに呼ぶ必要があります。 - // それぞれの位置指定結果のためのメソッド群があります。 - void** results(); - - - float* result0_data(); - float& result0(size_t dim0, size_t dim1); -}; - -} // end namespace bar -} // end namespace foo -``` - -生成されたC++クラスは、`tf_library` マクロの `cpp_class` に指定したとおり、名前空間 `foo::bar` における `MatMulComp` になります。 -生成されたすべてのクラスは、引数と結果のためのバッファを扱うメソッドが異なるのみで、おなじAPIを持ちます。 -これらのメソッドは、`tf_library` マクロの引数である `feed` と `fetch` に指定するバッファの数や型によって違いが発生します。 - -生成されたクラスでは、3つのバッファが管理されます。 -`args` は入力、`results` は出力、`temps` は計算を実行するために内部で利用する一時的なバッファを表しています。 -デフォルトでは、生成されたクラスのそれぞれのインスタンスは、これらのすべてのバッファを確保して管理します。 -コンストラクタの引数 `AllocMode` はこの振る舞いを変えるために使うことができます。 -すべてのバッファは、64バイトの境界にアライメントされています。 - -生成されたC++クラスは、XLAによって生成された低レベルのコードを単にラッパするクラスです。 - -`tfcompile_test.cc` をもとに生成されたファンクションを呼び出す例： - -```c++ -#define EIGEN_USE_THREADS -#define EIGEN_USE_CUSTOM_THREAD_POOL - -#include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/compiler/aot/tests/test_graph_tfmatmul.h" // 生成される - -int main(int argc, char** argv) { - Eigen::ThreadPool tp(2); // 必要に応じたスレッドプールのサイズ - Eigen::ThreadPoolDevice device(&tp, tp.NumThreads()); - - - foo::bar::MatMulComp matmul; - matmul.set_thread_pool(&device); - - // 引数を設定し、計算を実行する - const float args[12] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; - std::copy(args + 0, args + 6, matmul.arg0_data()); - std::copy(args + 6, args + 12, matmul.arg1_data()); - matmul.Run(); - - // 結果を確認する - if (matmul.result0(0, 0) == 58) { - std::cout << "Success" << std::endl; - } else { - std::cout << "Failed. Expected value 58 at 0,0. Got:" - << matmul.result0(0, 0) << std::endl; - } - - return 0; -} -``` - - -### ステップ4: 最終的なバイナリを作る - -このステップでは、ステップ2における `tf_library` によって生成されたライブラリと、ステップ3で書いたコードを組み合わせて最終的なバイナリを作ります。 -以下は、`bazel` BUILDファイルの例です。 - -```build -# あなたのバイナリをリンクする例 -# //tensorflow/compiler/aot/tests/BUILDも参照 -load("//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") - -# ステップ2におけるtf_libraryの呼び出しとおなじ -tf_library( - name = "test_graph_tfmatmul", - ... -) - -# tf_libraryによって生成された実行コードは、あなたのコードへリンクすることができます -cc_binary( - name = "my_binary", - srcs = [ - "my_code.cc", # 生成されたヘッダにアクセスするために、test_graph_tfmatmul.hをインクルードします - ], - deps = [ - ":test_graph_tfmatmul", # 生成されたオブジェクトファイルへリンクします - "//third_party/eigen3", - ], - linkopts = [ - "-lpthread", - ] -) -``` diff --git a/site/ja/xla/tutorials/xla_compile.ipynb b/site/ja/xla/tutorials/xla_compile.ipynb deleted file mode 100644 index 280dffe2b82..00000000000 --- a/site/ja/xla/tutorials/xla_compile.ipynb +++ /dev/null @@ -1,419 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "xla_compile.ipynb", - "provenance": [], - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f4TSNCvpENrW" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vamNSA0vEP-m", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e1oSi4lHFt3z" - }, - "source": [ - "# XLAコンパイラAPI" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b7noD9NjFRL-" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "v9YbsuLZaBXy" - }, - "source": [ - "\n", - "\n", - "TensorFlowとXLAライブラリをインポートします。XLAには、一部または全てのモデルを [XLA](https://www.tensorflow.org/extend/xla/) でコンパイルする実験的なAPIである `xla.compile()` が含まれています。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "45kUPj5ZFrRa", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.contrib.compiler import xla" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GZVNiRmTDV-5" - }, - "source": [ - "必要ないくつかの定数を定義し、 MNISTのデータセットを用意します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f37TSEGvGX4_", - "colab": {} - }, - "source": [ - "# それぞれの入力イメージの大きさは、 28 x 28ピクセル\n", - "IMAGE_SIZE = 28 * 28\n", - "# 個別の数字のラベル [0..9] の個数\n", - "NUM_CLASSES = 10\n", - "# それぞれのトレーニングバッチ（ステップ）での標本数\n", - "TRAIN_BATCH_SIZE = 100\n", - "# トレーニングステップを実行する回数\n", - "TRAIN_STEPS = 1000" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TiVXchblG5hK", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 173 - }, - "outputId": "e3cd80ce-3809-4c08-f7ed-3a2d9f385f5d" - }, - "source": [ - "# MNISTデータセットをロードする。\n", - "train, test = tf.keras.datasets.mnist.load_data()\n", - "train_ds = tf.data.Dataset.from_tensor_slices(train).batch(TRAIN_BATCH_SIZE).repeat()\n", - "test_ds = tf.data.Dataset.from_tensor_slices(test).batch(TRAIN_BATCH_SIZE)\n", - "\n", - "iterator = tf.data.Iterator.from_structure(train_ds.output_types, train_ds.output_shapes)\n", - "images, labels = iterator.get_next()\n", - "images = tf.reshape(images, [-1, IMAGE_SIZE])\n", - "images, labels = tf.cast(images, tf.float32), tf.cast(labels, tf.int64)" - ], - "execution_count": 4, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n", - "11493376/11490434 [==============================] - 0s 0us/step\n", - "WARNING:tensorflow:From :5: DatasetV1.output_types (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use `tf.compat.v1.data.get_output_types(dataset)`.\n", - "WARNING:tensorflow:From :5: DatasetV1.output_shapes (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use `tf.compat.v1.data.get_output_shapes(dataset)`.\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x_ZehpZP-SfS" - }, - "source": [ - "# モデルを構築する関数の定義\n", - "\n", - "以下のコードブロックは、順伝搬と逆伝搬の両方を行う、１つのdenseレイヤーを持つ簡単なモデルを構築する関数を含みます。\n", - "\n", - "コードが呼ばれたとき、２つの値を返します。 `y` は、それぞれのターゲットのクラスの予測確率を表す `tf.Tensor` です。 `train_step` は `global_step` の値を増加し、変数の更新を行う `tf.Operation` です。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZbhJl_WvGa3g", - "colab": {} - }, - "source": [ - "def build_mnist_model(x, y_):\n", - " y = tf.keras.layers.Dense(NUM_CLASSES).apply(x)\n", - "\n", - " cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_, logits=y)\n", - " train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n", - "\n", - " return y, train_step" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Jh3lyQHDfM9" - }, - "source": [ - "# XLA の有効化\n", - "\n", - "XLA を有効化するには `build_mnist_model` 関数を `xla.compile` に渡します。以下のコードブロックは、モデルを `xla.compile()` 関数でラップします。これにより、提供された入力を持つターゲット関数をXLAで実行できます。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kYpCXCdRHNuN", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 139 - }, - "outputId": "203aa05d-c54e-404a-eced-4bbfe4a44d5b" - }, - "source": [ - "[y] = xla.compile(build_mnist_model, inputs=[images, labels])" - ], - "execution_count": 6, - "outputs": [ - { - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Call initializer instance with the dtype argument instead of passing it to the constructor\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/losses/losses_impl.py:121: add_dispatch_support..wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use tf.where in 2.0, which has the same broadcast rule as np.where\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4giQh62IrZGF" - }, - "source": [ - "グラフをコンパイルするとき、XLAはターゲット関数によって構築されたグラフの全てのノードを、いくつかのXLAのオペレータで置き換えます。\n", - "\n", - "xla.compileは、生成されたXLAのオペレータから独立して実行できる `tf.Operation` を返しません\n", - "代わりに、ターゲット関数から返された `tf.Operation` ノードは、返された全ての `tf.Tensor` の値との制御依存関係として追加されます。これにより、 返されたテンソルが評価されるときに、 `tf.Operation` ノードの実行をトリガします。\n", - "\n", - " 擬似コードによるxla.compileの実装は、以下のようになります：\n", - "\n", - "---\n", - "```\n", - "# TensorFlowに、XLAが扱いやすい方法でコードを実行するよう依頼する\n", - "\n", - "y, train_step = build_mnist_model(images, labels)\n", - "with tf.control_dependencies([train_step]):\n", - " y = tf.identity(y)\n", - "\n", - "# TensorFlowに、XLAが扱いやすい方法でコードの実行を停止するよう依頼する\n", - "```\n", - "---\n", - "\n", - "xla.compile()は常に `tf.Tensor` のリスト（１要素しか無かったとしても）を返します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TPGas4jjFLZl" - }, - "source": [ - "もしあなたが構築したグラフを今表示したら、通常のTensorFlowのグラフとそれほど変わらないことがわかり、前に述べたXLAのオペレータを見つけることができないでしょう。これは、あなたが `sess.run()` でグラフを実行しようとしても、実際のコンパイルは後ほど発生するからです。後ほど、TensorFlowは実際にXLAオペレータを生成する一連のグラフ書き換えパスをトリガーします。これは、すべての入力がそろったときに、計算をコンパイルして実行します。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EZD1m_n1DxAF" - }, - "source": [ - "# モデルの学習とテスト" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qe28bAHNHUG2", - "colab": {} - }, - "source": [ - "# セッションを作成しすべての変数を初期化。\n", - "# xla.compile()は、Keras model.fit() APIやTF eager modeとはまだ動作しません。\n", - "sess = tf.Session()\n", - "sess.run(tf.global_variables_initializer())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qgsKmz3n2UiW" - }, - "source": [ - "以下のコードブロックはモデルを学習します。 `y` の評価は、制御依存関係がある `train_step` をトリガします。これは、モデル変数を更新します。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_GxF6jTRHVuA", - "outputId": "926558f2-5792-4eab-a9fc-58e19e409aea", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 207 - } - }, - "source": [ - "# 学習用データセットを与える\n", - "sess.run(iterator.make_initializer(train_ds))\n", - "\n", - "# TRAIN_STEPS ステップだけ実行する\n", - "for i in range(TRAIN_STEPS):\n", - " sess.run(y)\n", - "\n", - "print(\"Model trained for %s steps.\" % TRAIN_STEPS)" - ], - "execution_count": 8, - "outputs": [ - { - "output_type": "stream", - "text": [ - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/iterator_ops.py:348: Iterator.output_types (from tensorflow.python.data.ops.iterator_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use `tf.compat.v1.data.get_output_types(iterator)`.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/iterator_ops.py:349: Iterator.output_shapes (from tensorflow.python.data.ops.iterator_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use `tf.compat.v1.data.get_output_shapes(iterator)`.\n", - "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/data/ops/iterator_ops.py:351: Iterator.output_classes (from tensorflow.python.data.ops.iterator_ops) is deprecated and will be removed in a future version.\n", - "Instructions for updating:\n", - "Use `tf.compat.v1.data.get_output_classes(iterator)`.\n", - "Model trained for 1000 steps.\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dHlQlRSRHXD1", - "outputId": "588448d9-93e0-401c-e2de-be754ae140ca", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 34 - } - }, - "source": [ - "# 学習済みモデルをテストする\n", - "\n", - "# テスト用データセットを与える\n", - "sess.run(iterator.make_initializer(test_ds))\n", - "\n", - "# 精度を計算する\n", - "correct_prediction = tf.equal(tf.argmax(y, 1), labels)\n", - "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n", - "print(\"Prediction accuracy after training: %s\" % sess.run(accuracy))" - ], - "execution_count": 9, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Prediction accuracy after training: 0.91\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ynJQIuzjHYOb", - "colab": {} - }, - "source": [ - "# セッションを片付ける\n", - "sess.close()" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ko/README.md b/site/ko/README.md deleted file mode 100644 index a30e3dcf74c..00000000000 --- a/site/ko/README.md +++ /dev/null @@ -1,139 +0,0 @@ -# 커뮤니티 번역 문서들 - -이 문서들은 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 -[공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 -있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 -바랍니다. 문서 번역이나 리뷰에 참여하려면 -[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 -메일을 보내주시기 바랍니다. - -노트: [site/en](https://github.com/tensorflow/docs/tree/master/site/en) -디렉토리에 있는 [텐서플로 2](https://www.tensorflow.org) 문서를 번역하는데 초점을 맞춰 주세요. -2.0 릴리스를 준비하기 위해 [site/ko/r1](https://github.com/tensorflow/docs/tree/master/site/ko/r1) 디렉토리에 있는 TF 1.x 커뮤니티 문서는 더 이상 업데이트되지 않습니다. 이 -[공지](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ)를 -참고하세요. 또한 install 폴더의 내용과 yaml 파일, index.md 파일은 번역하지 않습니다. 이에 대한 자세한 지침은 이 [공지](https://groups.google.com/a/tensorflow.org/forum/#!msg/docs-zh-cn/mhLp-egzNyE/EhGSeIBqAQAJ)를 참고하세요. - -노트: [Swift for TensorFlow](https://www.tensorflow.org/swift)(S4TF)를 위한 번역 -홈 디렉토리는 -[site/ko/swift](https://github.com/tensorflow/docs/tree/master/site/ko/swift) -입니다. 영문 파일의 위치는 tensorflow/swift 레파지토리의 -[docs/site](https://github.com/tensorflow/swift/tree/master/docs/site) -디렉토리입니다. S4TF 노트북은 꼭 출력 결과가 포함되어야 합니다. - -# Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en) and [Tensorflow Docs-Ko Translation](http://bit.ly/tf-docs-translation-status). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Please focus translation efforts on -[TensorFlow 2](https://www.tensorflow.org) in the -[site/en](https://github.com/tensorflow/docs/tree/master/site/en) -directory. TF 1.x community docs in [site/en/r1](https://github.com/tensorflow/docs/tree/master/site/en/r1) will no longer be updated as we prepare for the -2.0 release. See -[the announcement](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). -Also, please do not translate the `install/` section, any `*.yaml` files, or `index.md` files. -See [the announcement](https://groups.google.com/a/tensorflow.org/forum/#!msg/docs-zh-cn/mhLp-egzNyE/EhGSeIBqAQAJ). - -Note: The -[site/ko/swift](https://github.com/tensorflow/docs/tree/master/site/ko/swift) -directory is the home for -[Swift for TensorFlow](https://www.tensorflow.org/swift)(S4TF) translations. -Original files are in the -[docs/site](https://github.com/tensorflow/swift/tree/master/docs/site) directory -of the tensorflow/swift repository. S4TF notebooks must have the outputs saved. - -# 처음 참여하는 사람들에게 - -문서 번역에 참여해 주셔서 감사합니다. -번역에 참여하기 전에 번역된 [문서](https://github.com/tensorflow/docs/tree/master/site/ko)를 -먼저 읽어 보길 권합니다. -번역 문서는 'ㅂ니다'체를 따르며 존칭이나 반말은 쓰지 않습니다. -가능한한 기존 문서의 스타일을 따라야 합니다. - -작업을 시작하려면 [텐서플로 한글 문서 기여자](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko) -메일링 리스트와 [Tensorflow Docs-Ko Translation](http://bit.ly/tf-docs-translation-status) 구글 스프레드 시트에 작업 중임을 알려 주세요. -다른 사람이 작업 중인 파일이 아니라면 en 폴더 안의 파일을 ko 폴더 아래 같은 위치에 복사하여 시작합니다. -site/ko/r1 는 텐서플로 1.x 버전을 위한 파일입니다. -site/ko 는 텐서플로 2.x 버전을 위한 파일입니다. - -마크다운(markdown)과 주석을 모두 번역합니다. 코드 셀(cell)은 실행하지 않습니다. -주피터 노트북은 조금만 수정하더라도 파일 소스 전체가 변경될 수 있습니다. -이런 파일은 깃허브에서 리뷰하기 어렵습니다. -기존 노트북에서 간단한 내용을 수정할 때는 가능하면 텍스트 편집기를 사용하여 직접 노트북 소스를 수정해야 합니다. -노트북 파일의 번역이 완료되면 포크된 자신의 저장소에서 코랩(Colab)에서 실행이 잘되는지 확인해야 합니다. -코랩에서 실행에 문제가 없다면 리뷰를 요청합니다. - -번역과 관련되어 문의 사항이 있다면 언제든지 -[텐서플로 한글 문서 기여자](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko) -메일링 리스트로 메일을 보내 주세요. - -감사합니다! - -# For new contributors - -Thanks for joining the translation effort. -Please read the existing -[KO documents](https://github.com/tensorflow/docs/tree/master/site/ko) -before starting your translation. -You should use 'ㅂ니다' style and not use the honorific or rude words. -Follow the style of existing documents, as possible as you can. - -After your translation is complete, notify the -[Korean TensorFlow Documentation Contributors](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko) -mailing list to coordinate a review. - -Copy a file in `en` folder to same location under `ko` folder if anybody doesn't work on the file, -and get it start. -`site/ko/r1` are for TensorFlow 1.x. -`site/ko` are for TensorFlow 2.x. - -You should translate markdown and comments. You should not run code cells. -Whole file structure can be changed even if you modify only a chunk in the notebook. -It is hard to review such a file in GitHub. -You should use a text editor when you edit a few words of existing notebook. -You should test the notebook in your repository with Colab after you finish the translation. -You can request for review if there is no error. - -If you have any question about translation, feel free to contact -[Korean TensorFlow Documentation Contributors](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko) -mailing list. - -Thanks! - -# Korean translation guide - -Some technical words in English do not have a natural translation. Do *not* -translate the following words: - -* mini-batch -* batch -* label -* class -* helper -* hyperparameter -* optimizer -* one-hot encoding -* epoch -* callback -* sequence -* dictionary (in python) -* embedding -* padding -* unit -* node -* nueron -* target -* checkpoint -* compile -* dropout -* penalty - -If you have any suggestion about translation guide, feel free to contact -[Korean TensorFlow Documentation Translation Glossary](http://bit.ly/tf-docs-translation-glossary) -spreadsheet. diff --git a/site/ko/REVIEWERS b/site/ko/REVIEWERS deleted file mode 100644 index 5da7587e105..00000000000 --- a/site/ko/REVIEWERS +++ /dev/null @@ -1,13 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/ko directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs-ko@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko -# -rickiepark -cre8tor -choiuijin1125 -JKIsaacLee -NoelBird -wckim diff --git a/site/ko/guide/distributed_training.ipynb b/site/ko/guide/distributed_training.ipynb deleted file mode 100644 index 779147e6216..00000000000 --- a/site/ko/guide/distributed_training.ipynb +++ /dev/null @@ -1,969 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "distributed_training.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "Tce3stUlHN0L" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tuOe1ymfHZPu", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# 텐서플로로 분산 훈련하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r6P32iYYV27b" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vqrz9ZBdzQ9C", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/distributed_training.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃허브 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "## 개요\n", - "\n", - "`tf.distribute.Strategy`는 훈련을 여러 GPU 또는 여러 장비, 여러 TPU로 나누어 처리하기 위한 텐서플로 API입니다. 이 API를 사용하면 기존의 모델이나 훈련 코드를 조금만 고쳐서 분산처리를 할 수 있습니다.\n", - "\n", - "`tf.distribute.Strategy`는 다음을 주요 목표로 설계하였습니다.\n", - "\n", - "* 사용하기 쉽고, 연구원, 기계 학습 엔지니어 등 여러 사용자 층을 지원할 것.\n", - "* 그대로 적용하기만 하면 좋은 성능을 보일 것.\n", - "* 전략들을 쉽게 갈아 끼울 수 있을 것.\n", - "\n", - "`tf.distribute.Strategy`는 텐서플로의 고수준 API인 [tf.keras](https://www.tensorflow.org/guide/keras) 및 [tf.estimator](https://www.tensorflow.org/guide/estimator)와 함께 사용할 수 있습니다. 코드 한두 줄만 추가하면 됩니다. 사용자 정의 훈련 루프(그리고 텐서플로를 사용한 모든 계산 작업)에 함께 사용할 수 있는 API도 제공합니다.\n", - "텐서플로 2.0에서는 사용자가 프로그램을 즉시 실행(eager execution)할 수도 있고, [`tf.function`](../tutorials/eager/tf_function.ipynb)을 사용하여 그래프에서 실행할 수도 있습니다. `tf.distribute.Strategy`는 두 가지 실행 방식을 모두 지원하려고 합니다. 이 가이드에서는 대부분의 경우 훈련에 대하여 이야기하겠지만, 이 API 자체는 여러 환경에서 평가나 예측을 분산 처리하기 위하여 사용할 수도 있다는 점을 참고하십시오.\n", - "\n", - "잠시 후 보시겠지만 코드를 약간만 바꾸면 `tf.distribute.Strategy`를 사용할 수 있습니다. 변수, 층, 모델, 옵티마이저, 지표, 서머리(summary), 체크포인트 등 텐서플로를 구성하고 있는 기반 요소들을 전략(Strategy)을 이해하고 처리할 수 있도록 수정했기 때문입니다. \n", - "\n", - "이 가이드에서는 다양한 형식의 전략에 대해서, 그리고 여러 가지 상황에서 이들을 어떻게 사용해야 하는지 알아보겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EVOZFbNgXghB", - "colab": {} - }, - "source": [ - "# 텐서플로 패키지 가져오기\n", - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eQ1QESxxEbCh" - }, - "source": [ - "## 전략의 종류\n", - "`tf.distribute.Strategy`는 서로 다른 다양한 사용 형태를 아우르려고 합니다. 몇 가지 조합은 현재 지원하지만, 추후에 추가될 전략들도 있습니다. 이들 중 몇 가지를 살펴보겠습니다.\n", - "\n", - "* 동기 훈련 대 비동기 훈련: 분산 훈련을 할 때 데이터를 병렬로 처리하는 방법은 크게 두 가지가 있습니다. 동기 훈련을 할 때는 모든 워커(worker)가 입력 데이터를 나누어 갖고 동시에 훈련합니다. 그리고 각 단계마다 그래디언트(gradient)를 모읍니다. 비동기 훈련에서는 모든 워커가 독립적으로 입력 데이터를 사용해 훈련하고 각각 비동기적으로 변수들을 갱신합니다. 일반적으로 동기 훈련은 올 리듀스(all-reduce)방식으로 구현하고, 비동기 훈련은 파라미터 서버 구조를 사용합니다.\n", - "* 하드웨어 플랫폼: 한 장비에 있는 다중 GPU로 나누어 훈련할 수도 있고, 네트워크로 연결된 (GPU가 없거나 여러 개의 GPU를 가진) 여러 장비로 나누어서, 또 혹은 클라우드 TPU에서 훈련할 수도 있습니다.\n", - "\n", - "이런 사용 형태들을 위하여, 현재 5가지 전략을 사용할 수 있습니다. 이후 내용에서 현재 TF 2.0 베타에서 상황마다 어떤 전략을 지원하는지 이야기하겠습니다. 일단 간단한 개요는 다음과 같습니다.\n", - "\n", - "| 훈련 API \t| MirroredStrategy \t| TPUStrategy \t| MultiWorkerMirroredStrategy \t| CentralStorageStrategy \t| ParameterServerStrategy \t|\n", - "|:-----------------------\t|:-------------------\t|:---------------------\t|:---------------------------------\t|:---------------------------------\t|:--------------------------\t|\n", - "| **Keras API** \t| 지원\t| 2.0 RC 지원 예정\t| 실험 기능으로 지원\t| 실험 기능으로 지원\t| 2.0 이후 지원 예정\t|\n", - "| **사용자 정의 훈련 루프** \t| 실험 기능으로 지원\t| 실험 기능으로 지원 \t| 2.0 이후 지원 예정\t| 2.0 RC 지원 예정\t| 아직 미지원\t|\n", - "| **Estimator API** \t| 제한적으로 지원\t| 제한적으로 지원\t| 제한적으로 지원\t| 제한적으로 지원\t| 제한적으로 지원\t|" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DoQKKK8dtfg6" - }, - "source": [ - "### MirroredStrategy\n", - "`tf.distribute.MirroredStrategy`는 장비 하나에서 다중 GPU를 이용한 동기 분산 훈련을 지원합니다. 각각의 GPU 장치마다 복제본이 만들어집니다. 모델의 모든 변수가 복제본마다 미러링 됩니다. 이 미러링된 변수들은 하나의 가상의 변수에 대응되는데, 이를 `MirroredVariable`라고 합니다. 이 변수들은 동일한 변경사항이 함께 적용되므로 모두 같은 값을 유지합니다.\n", - "\n", - "여러 장치에 변수의 변경사항을 전달하기 위하여 효율적인 올 리듀스 알고리즘을 사용합니다. 올 리듀스 알고리즘은 모든 장치에 걸쳐 텐서를 모은 다음, 그 합을 구하여 다시 각 장비에 제공합니다. 이 통합된 알고리즘은 매우 효율적이어서 동기화의 부담을 많이 덜어낼 수 있습니다. 장치 간에 사용 가능한 통신 방법에 따라 다양한 올 리듀스 알고리즘과 구현이 있습니다. 기본값으로는 NVIDIA NCCL을 올 리듀스 구현으로 사용합니다. 또한 제공되는 다른 몇 가지 방법 중에 선택하거나, 직접 만들 수도 있습니다.\n", - "\n", - "`MirroredStrategy`를 만드는 가장 쉬운 방법은 다음과 같습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9Z4FMAY9ADxK", - "colab": {} - }, - "source": [ - "mirrored_strategy = tf.distribute.MirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wldY4aFCAH4r" - }, - "source": [ - "`MirroredStrategy` 인스턴스가 생겼습니다. 텐서플로가 인식한 모든 GPU를 사용하고, 장치 간 통신에는 NCCL을 사용할 것입니다.\n", - "\n", - "장비의 GPU 중 일부만 사용하고 싶다면, 다음과 같이 하면 됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nbGleskCACv_", - "colab": {} - }, - "source": [ - "mirrored_strategy = tf.distribute.MirroredStrategy(devices=[\"/gpu:0\", \"/gpu:1\"])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8-KDnrJLAhav" - }, - "source": [ - "장치 간 통신 방법을 바꾸고 싶다면, `cross_device_ops` 인자에 `tf.distribute.CrossDeviceOps` 타입의 인스턴스를 넘기면 됩니다. 현재 기본값인 `tf.distribute.NcclAllReduce` 이외에 `tf.distribute.HierarchicalCopyAllReduce`와 `tf.distribute.ReductionToOneDevice` 두 가지 추가 옵션을 제공합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6-xIOIpgBItn", - "colab": {} - }, - "source": [ - "mirrored_strategy = tf.distribute.MirroredStrategy(\n", - " cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "45H0Wa8WKI8z" - }, - "source": [ - "### CentralStorageStrategy\n", - "`tf.distribute.experimental.CentralStorageStrategy`도 동기 훈련을 합니다. 하지만 변수를 미러링하지 않고, CPU에서 관리합니다. 작업은 모든 로컬 GPU들로 복제됩니다. 단, 만약 GPU가 하나밖에 없다면 모든 변수와 작업이 그 GPU에 배치됩니다.\n", - "\n", - "다음과 같이 `CentralStorageStrategy` 인스턴스를 만드십시오." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rtjZOyaoMWrP", - "colab": {} - }, - "source": [ - "central_storage_strategy = tf.distribute.experimental.CentralStorageStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KY1nJHNkMl7b" - }, - "source": [ - "`CentralStorageStrategy` 인스턴스가 만들어졌습니다. 인식한 모든 GPU와 CPU를 사용합니다. 각 복제본의 변수 변경사항은 모두 수집된 후 변수에 적용됩니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aAFycYUiNCUb" - }, - "source": [ - "Note: 이 전략은 아직 개선 중이고 더 많은 경우에 쓸 수 있도록 만들고 있기 때문에, [`실험 기능`](https://www.tensorflow.org/guide/versions#what_is_not_covered)으로 지원됩니다. 따라서 다음에 API가 바뀔 수 있음에 유념하십시오." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8Xc3gyo0Bejd" - }, - "source": [ - "### MultiWorkerMirroredStrategy\n", - "\n", - "`tf.distribute.experimental.MultiWorkerMirroredStrategy`은 `MirroredStrategy`와 매우 비슷합니다. 다중 워커를 이용하여 동기 분산 훈련을 합니다. 각 워커는 여러 개의 GPU를 사용할 수 있습니다. `MirroredStrategy`처럼 모델에 있는 모든 변수의 복사본을 모든 워커의 각 장치에 만듭니다.\n", - "\n", - "다중 워커(multi-worker)들 사이에서는 올 리듀스(all-reduce) 통신 방법으로 [CollectiveOps](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/collective_ops.py)를 사용하여 변수들을 같은 값으로 유지합니다. 수집 연산(collective op)은 텐서플로 그래프에 속하는 연산 중 하나입니다. 이 연산은 하드웨어나 네트워크 구성, 텐서 크기에 따라 텐서플로 런타임이 지원하는 올 리듀스 알고리즘을 자동으로 선택합니다.\n", - "\n", - "여기에 추가 성능 최적화도 구현하고 있습니다. 예를 들어 작은 텐서들의 여러 올 리듀스 작업을 큰 텐서들의 더 적은 올 리듀스 작업으로 바꾸는 정적 최적화 기능이 있습니다. 뿐만아니라 플러그인 구조를 갖도록 설계하였습니다. 따라서 추후에는 사용자가 자신의 하드웨어에 더 최적화된 알고리즘을 사용할 수도 있을 것입니다. 참고로 이 수집 연산은 올 리듀스 외에 브로드캐스트(broadcast)나 전체 수집(all-gather)도 구현하고 있습니다.\n", - "\n", - "`MultiWorkerMirroredStrategy`를 만드는 가장 쉬운 방법은 다음과 같습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m3a_6ebbEjre", - "colab": {} - }, - "source": [ - "multiworker_strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bt94JBvhEr4s" - }, - "source": [ - "`MultiWorkerMirroredStrategy`에 사용할 수 있는 수집 연산 구현은 현재 두 가지입니다. `CollectiveCommunication.RING`는 gRPC를 사용한 링 네트워크 기반의 수집 연산입니다. `CollectiveCommunication.NCCL`는 [Nvidia의 NCCL](https://developer.nvidia.com/nccl)을 사용하여 수집 연산을 구현한 것입니다. `CollectiveCommunication.AUTO`로 설정하면 런타임이 알아서 구현을 고릅니다. 최적의 수집 연산 구현은 GPU의 수와 종류, 클러스터의 네트워크 연결 등에 따라 다를 수 있습니다. 예를 들어 다음과 같이 지정할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QGX_QAEtFQSv", - "colab": {} - }, - "source": [ - "multiworker_strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy(\n", - " tf.distribute.experimental.CollectiveCommunication.NCCL)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0JiImlw3F77E" - }, - "source": [ - "다중 GPU를 사용하는 것과 비교해서 다중 워커를 사용하는 것의 가장 큰 차이점은 다중 워커에 대한 설정 부분입니다. 클러스터를 구성하는 각 워커에 \"TF_CONFIG\" 환경변수를 사용하여 클러스터 설정을 하는 것이 텐서플로의 표준적인 방법입니다. [아래쪽 \"TF_CONFIG\"](#TF_CONFIG) 항목에서 어떻게 하는지 자세히 살펴보겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E20tG21LFfv1" - }, - "source": [ - "Note: 이 전략은 아직 개선 중이고 더 많은 경우에 쓸 수 있도록 만들고 있기 때문에, [`실험 기능`](https://www.tensorflow.org/guide/versions#what_is_not_covered)으로 지원됩니다. 따라서 나중에 API가 바뀔 수 있음에 유념하십시오." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPEBCMzsGaO5" - }, - "source": [ - "### TPUStrategy\n", - "`tf.distribute.experimental.TPUStrategy`는 텐서플로 훈련을 텐서처리장치(Tensor Processing Unit, TPU)에서 수행하는 전략입니다. TPU는 구글의 특별한 주문형 반도체(ASIC)로서, 기계 학습 작업을 극적으로 가속하기 위하여 설계되었습니다. TPU는 구글 코랩, [Tensorflow Research Cloud](https://www.tensorflow.org/tfrc), [Google Compute Engine](https://cloud.google.com/tpu)에서 사용할 수 있습니다.\n", - "\n", - "분산 훈련 구조의 측면에서, TPUStrategy는 `MirroredStrategy`와 동일합니다. 동기 분산 훈련 방식을 사용합니다. TPU는 자체적으로 여러 TPU 코어들에 걸친 올 리듀스 및 기타 수집 연산을 효율적으로 구현하고 있습니다. 이 구현이 `TPUStrategy`에 사용됩니다.\n", - "\n", - "`TPUStrategy`를 사용하는 방법은 다음과 같습니다.\n", - "\n", - "Note: 코랩에서 이 코드를 사용하려면, 코랩 런타임으로 TPU를 선택해야 합니다. TPUStrategy를 사용하는 방법에 대한 튜토리얼을 곧 추가하겠습니다.\n", - "\n", - "```\n", - "cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(\n", - " tpu=tpu_address)\n", - "tf.config.experimental_connect_to_host(cluster_resolver.master())\n", - "tf.tpu.experimental.initialize_tpu_system(cluster_resolver)\n", - "tpu_strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oQ7EqjpmK6DU" - }, - "source": [ - "`TPUClusterResolver` 인스턴스는 TPU를 찾도록 도와줍니다. 코랩에서는 아무런 인자를 주지 않아도 됩니다. 클라우드 TPU에서 사용하려면, TPU 자원의 이름을 `tpu` 매개변수에 지정해야 합니다. 또한 TPU는 계산하기 전 초기화(initialize)가 필요합니다. 초기화 중 TPU 메모리가 지워져서 모든 상태 정보가 사라지므로, 프로그램 시작시에 명시적으로 TPU 시스템을 초기화(initialize)해 주어야 합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jARHpraJMbRa" - }, - "source": [ - "Note: 이 전략은 아직 개선 중이고 더 많은 경우에 쓸 수 있도록 만들고 있기 때문에, [`실험 기능`](https://www.tensorflow.org/guide/versions#what_is_not_covered)으로 지원됩니다. 따라서 나중에 API가 바뀔 수 있음에 유념하십시오." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3ZLBhaP9NUNr" - }, - "source": [ - "### ParameterServerStrategy\n", - "`tf.distribute.experimental.ParameterServerStrategy`은 여러 장비에서 훈련할 때 파라미터 서버를 사용합니다. 이 전략을 사용하면 몇 대의 장비는 워커 역할을 하고, 몇 대는 파라미터 서버 역할을 하게 됩니다. 모델의 각 변수는 한 파라미터 서버에 할당됩니다. 계산 작업은 모든 워커의 GPU들에 복사됩니다.\n", - "\n", - "코드만 놓고 보았을 때는 다른 전략들과 비슷합니다.\n", - "```\n", - "ps_strategy = tf.distribute.experimental.ParameterServerStrategy()\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zr1wPHYvOH0N" - }, - "source": [ - "다중 워커 환경에서 훈련하려면, 클러스터에 속한 파라미터 서버와 워커를 \"TF_CONFIG\" 환경변수를 이용하여 설정해야 합니다. 자세한 내용은 [아래쪽 \"TF_CONFIG\"](#TF_CONFIG)에서 설명하겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hQv1lm9UPDFy" - }, - "source": [ - "여기까지 여러 가지 전략들이 어떻게 다르고, 어떻게 사용하는지 살펴보았습니다. 이어지는 절들에서는 훈련을 분산시키기 위하여 이들을 어떻게 사용해야 하는지 살펴보겠습니다. 이 문서에서는 간단한 코드 조각만 보여드리겠지만, 처음부터 끝까지 전체 코드를 실행할 수 있는 더 긴 튜토리얼의 링크도 함께 안내해드리겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_mcuy3UhPcen" - }, - "source": [ - "## 케라스와 함께 `tf.distribute.Strategy` 사용하기\n", - "`tf.distribute.Strategy`는 텐서플로의 [케라스 API 명세](https://keras.io) 구현인 `tf.keras`와 함께 사용할 수 있습니다. `tf.keras`는 모델을 만들고 훈련하는 고수준 API입니다. 분산 전략을 `tf.keras` 백엔드와 함께 쓸 수 있으므로, 케라스 사용자들도 케라스 훈련 프레임워크로 작성한 훈련 작업을 쉽게 분산 처리할 수 있게 되었습니다. 훈련 프로그램에서 고쳐야하는 부분은 거의 없습니다. (1) 적절한 `tf.distribute.Strategy` 인스턴스를 만든 다음 (2) \n", - "케라스 모델의 생성과 컴파일을 `strategy.scope` 안으로 옮겨주기만 하면 됩니다. `Sequential` , 함수형 API, 클래스 상속 등 모든 방식으로 만든 케라스 모델을 다 지원합니다.\n", - "\n", - "다음은 한 개의 밀집 층(dense layer)을 가진 매우 간단한 케라스 모델에 분산 전략을 사용하는 코드의 일부입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gbbcpzRnPZ6V", - "colab": {} - }, - "source": [ - "mirrored_strategy = tf.distribute.MirroredStrategy()\n", - "with mirrored_strategy.scope():\n", - " model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n", - " model.compile(loss='mse', optimizer='sgd')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "773EOxCRVlTg" - }, - "source": [ - "위 예에서는 `MirroredStrategy`를 사용했기 때문에, 하나의 장비가 다중 GPU를 가진 경우에 사용할 수 있습니다. `strategy.scope()`로 분산 처리할 부분을 코드에 지정할 수 있습니다. 이 범위(scope) 안에서 모델을 만들면, 일반적인 변수가 아니라 미러링된 변수가 만들어집니다. 이 범위 안에서 컴파일을 한다는 것은 작성자가 이 전략을 사용하여 모델을 훈련하려고 한다는 의미입니다. 이렇게 구성하고 나서, 일반적으로 실행하는 것처럼 모델의 `fit` 함수를 호출합니다.\n", - "`MirroredStrategy`가 모델의 훈련을 사용 가능한 GPU들로 복제하고, 그래디언트들을 수집하는 것 등을 알아서 처리합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZMmxEFRTEjH5", - "colab": {} - }, - "source": [ - "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100).batch(10)\n", - "model.fit(dataset, epochs=2)\n", - "model.evaluate(dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nofTLwyXWHK8" - }, - "source": [ - "위에서는 훈련과 평가 입력을 위해 `tf.data.Dataset`을 사용했습니다. 넘파이(numpy) 배열도 사용할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lqgd9SdxW5OW", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "inputs, targets = np.ones((100, 1)), np.ones((100, 1))\n", - "model.fit(inputs, targets, epochs=2, batch_size=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IKqaj7QwX0Zb" - }, - "source": [ - "데이터셋이나 넘파이를 사용하는 두 경우 모두 입력 배치가 동일한 크기로 나누어져서 여러 개로 복제된 작업에 전달됩니다. 예를 들어, `MirroredStrategy`를 2개의 GPU에서 사용한다면, 크기가 10개인 배치(batch)가 두 개의 GPU로 배분됩니다. 즉, 각 GPU는 한 단계마다 5개의 입력을 받게 됩니다. 따라서 GPU가 추가될수록 각 에포크(epoch) 당 훈련 시간은 줄어들게 됩니다. 일반적으로는 가속기를 더 추가할 때마다 배치 사이즈도 더 키웁니다. 추가한 컴퓨팅 자원을 더 효과적으로 사용하기 위해서입니다. 모델에 따라서는 학습률(learning rate)을 재조정해야 할 수도 있을 것입니다. 복제본의 수는 `strategy.num_replicas_in_sync`로 얻을 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "quNNTytWdGBf", - "colab": {} - }, - "source": [ - "# 복제본의 수로 전체 배치 크기를 계산.\n", - "BATCH_SIZE_PER_REPLICA = 5\n", - "global_batch_size = (BATCH_SIZE_PER_REPLICA *\n", - " mirrored_strategy.num_replicas_in_sync)\n", - "dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(100)\n", - "dataset = dataset.batch(global_batch_size)\n", - "\n", - "LEARNING_RATES_BY_BATCH_SIZE = {5: 0.1, 10: 0.15}\n", - "learning_rate = LEARNING_RATES_BY_BATCH_SIZE[global_batch_size]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1Muy0gDZwO5" - }, - "source": [ - "### 현재 어떤 것이 지원됩니까?\n", - "\n", - "TF 2.0 베타 버전에서는 케라스와 함께 `MirroredStrategy`와 `CentralStorageStrategy`, `MultiWorkerMirroredStrategy`를 사용하여 훈련할 수 있습니다. `CentralStorageStrategy`와 `MultiWorkerMirroredStrategy`는 아직 실험 기능이므로 추후 바뀔 수 있습니다.\n", - "다른 전략도 조만간 지원될 것입니다. API와 사용 방법은 위에 설명한 것과 동일할 것입니다.\n", - "\n", - "| 훈련 API \t| MirroredStrategy \t| TPUStrategy \t| MultiWorkerMirroredStrategy \t| CentralStorageStrategy \t| ParameterServerStrategy \t|\n", - "|----------------\t|---------------------\t|-----------------------\t|-----------------------------------\t|-----------------------------------\t|---------------------------\t|\n", - "| Keras API \t| 지원 \t| 2.0 RC 지원 예정 \t| 실험 기능으로 지원\t| 실험 기능으로 지원 \t| 2.0 RC 지원 예정\t|\n", - "\n", - "### 예제와 튜토리얼\n", - "\n", - "위에서 설명한 케라스 분산 훈련 방법에 대한 튜토리얼과 예제들의 목록입니다.\n", - "\n", - "1. `MirroredStrategy`를 사용한 [MNIST](../tutorials/distribute/keras.ipynb) 훈련 튜토리얼.\n", - "2. ImageNet 데이터와 `MirroredStrategy`를 사용한 공식 [ResNet50](https://github.com/tensorflow/models/blob/master/official/vision/image_classification/resnet_imagenet_main.py) 훈련.\n", - "3. 클라우드 TPU에서 ImageNet 데이터와 `TPUStrategy`를 사용한 [ResNet50](https://github.com/tensorflow/tpu/blob/master/models/experimental/resnet50_keras/resnet50.py) 훈련. 이 예제는 현재 텐서플로 1.x 버전에서만 동작합니다.\n", - "4. `MultiWorkerMirroredStrategy`를 사용한 [MNIST](../tutorials/distribute/multi_worker_with_keras.ipynb) 훈련 튜토리얼.\n", - "5. `MirroredStrategy`를 사용한 [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) 훈련.\n", - "6. `MirroredStrategy`를 사용한 [Transformer]( https://github.com/tensorflow/models/blob/master/official/transformer/v2/transformer_main.py) 훈련." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IlYVC0goepdk" - }, - "source": [ - "## 사용자 정의 훈련 루프와 함께 `tf.distribute.Strategy` 사용하기\n", - "지금까지 살펴본 것처럼 고수준 API와 함께 `tf.distribute.Strategy`를 사용하려면 코드 몇 줄만 바꾸면 되었습니다. 조금만 더 노력을 들이면 이런 프레임워크를 사용하지 않는 사용자도 `tf.distribute.Strategy`를 사용할 수 있습니다.\n", - "\n", - "텐서플로는 다양한 용도로 사용됩니다. 연구자들 같은 일부 사용자들은 더 높은 자유도와 훈련 루프에 대한 제어를 원합니다. 이 때문에 추정기나 케라스 같은 고수준 API를 사용하기 힘든 경우가 있습니다. 예를 들어, GAN을 사용하는데 매번 생성자(generator)와 판별자(discriminator) 단계의 수를 바꾸고 싶을 수 있습니다. 비슷하게, 고수준 API는 강화 학습(Reinforcement learning)에는 그다지 적절하지 않습니다. 그래서 이런 사용자들은 보통 자신만의 훈련 루프를 작성하게 됩니다.\n", - "\n", - "이 사용자들을 위하여, `tf.distribute.Strategy` 클래스들은 일련의 주요 메서드들을 제공합니다. 이 메서드들을 사용하려면 처음에는 코드를 이리저리 조금 옮겨야 할 수 있겠지만, 한번 작업해 놓으면 전략 인스턴스만 바꿔서 GPU, TPU, 여러 장비로 쉽게 바꿔가며 훈련을 할 수 있습니다.\n", - "\n", - "앞에서 살펴본 케라스 모델을 사용한 훈련 예제를 통하여 사용하는 모습을 간단하게 살펴보겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XNHvSY32nVBi" - }, - "source": [ - "먼저, 전략의 범위(scope) 안에서 모델과 옵티마이저를 만듭니다. 이는 모델이나 옵티마이저로 만들어진 변수가 미러링 되도록 만듭니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W-3Bn-CaiPKD", - "colab": {} - }, - "source": [ - "with mirrored_strategy.scope():\n", - " model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=(1,))])\n", - " optimizer = tf.keras.optimizers.SGD()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mYkAyPeYnlXk" - }, - "source": [ - "다음으로는 입력 데이터셋을 만든 다음, `tf.distribute.Strategy.experimental_distribute_dataset` 메서드를 호출하여 전략에 맞게 데이터셋을 분배합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "94BkvkLInkKd", - "colab": {} - }, - "source": [ - "with mirrored_strategy.scope():\n", - " dataset = tf.data.Dataset.from_tensors(([1.], [1.])).repeat(1000).batch(\n", - " global_batch_size)\n", - " dist_dataset = mirrored_strategy.experimental_distribute_dataset(dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "grzmTlSvn2j8" - }, - "source": [ - "그리고 나서는 한 단계의 훈련을 정의합니다. 그래디언트를 계산하기 위해 `tf.GradientTape`를 사용합니다. 이 그래디언트를 적용하여 우리 모델의 변수를 갱신하기 위해서는 옵티마이저를 사용합니다. 분산 훈련을 위하여 이 훈련 작업을 `step_fn` 함수 안에 구현합니다. 그리고 `step_fn`을 앞에서 만든 `dist_dataset`에서 얻은 입력 데이터와 함께 `tf.distrbute.Strategy.experimental_run_v2`메서드로 전달합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NJxL5YrVniDe", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(dist_inputs):\n", - " def step_fn(inputs):\n", - " features, labels = inputs\n", - "\n", - " with tf.GradientTape() as tape:\n", - " logits = model(features)\n", - " cross_entropy = tf.nn.softmax_cross_entropy_with_logits(\n", - " logits=logits, labels=labels)\n", - " loss = tf.reduce_sum(cross_entropy) * (1.0 / global_batch_size)\n", - "\n", - " grads = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(list(zip(grads, model.trainable_variables)))\n", - " return cross_entropy\n", - "\n", - " per_example_losses = mirrored_strategy.experimental_run_v2(\n", - " step_fn, args=(dist_inputs,))\n", - " mean_loss = mirrored_strategy.reduce(\n", - " tf.distribute.ReduceOp.MEAN, per_example_losses, axis=0)\n", - " return mean_loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yRL5u_NLoTvq" - }, - "source": [ - "위 코드에서 몇 가지 더 짚어볼 점이 있습니다.\n", - "\n", - "1. 손실(loss)을 계산하기 위하여 `tf.nn.softmax_cross_entropy_with_logits`를 사용하였습니다. 그리고 손실의 합을 전체 배치 크기로 나누는 부분이 중요합니다. 이는 모든 복제된 훈련이 동시에 이루어지고 있고, 각 단계에 훈련이 이루어지는 입력의 수는 전체 배치 크기와 같기 때문입니다. 따라서 손실 값은 각 복제된 작업 내의 배치 크기가 아니라 전체 배치 크기로 나누어야 맞습니다.\n", - "2. `tf.distribute.Strategy.experimental_run_v2`에서 반환된 결과를 모으기 위하여 `tf.distribute.Strategy.reduce` API를 사용하였습니다. `tf.distribute.Strategy.experimental_run_v2`는 전략의 각 복제본에서 얻은 결과를 반환합니다. 그리고 이 결과를 사용하는 방법은 여러 가지가 있습니다. 종합한 결과를 얻기 위하여 `reduce` 함수를 사용할 수 있습니다. `tf.distribute.Strategy.experimental_local_results` 메서드로 각 복제본에서 얻은 결과의 값들 목록을 얻을 수도 있습니다.\n", - "3. 분산 전략 범위 안에서 `apply_gradients` 메서드가 호출되면, 평소와는 동작이 다릅니다. 구체적으로는 동기화된 훈련 중 병렬화된 각 작업에서 그래디언트를 적용하기 전에, 모든 복제본의 그래디언트를 더해집니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o9k_6-6vpQ-P" - }, - "source": [ - "훈련 단계를 정의했으므로, 마지막으로는 `dist_dataset`에 대하여 훈련을 반복합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Egq9eufToRf6", - "colab": {} - }, - "source": [ - "with mirrored_strategy.scope():\n", - " for inputs in dist_dataset:\n", - " print(train_step(inputs))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jK8eQXF_q1Zs" - }, - "source": [ - "위 예에서는 `dist_dataset`을 차례대로 처리하며 훈련 입력 데이터를 얻었습니다. `tf.distribute.Strategy.make_experimental_numpy_dataset`를 사용하면 넘파이 입력도 쓸 수 있습니다. `tf.distribute.Strategy.experimental_distribute_dataset` 함수를 호출하기 전에 이 API로 데이터셋을 만들면 됩니다.\n", - "\n", - "데이터를 차례대로 처리하는 또 다른 방법은 명시적으로 반복자(iterator)를 사용하는 것입니다. 전체 데이터를 모두 사용하지 않고, 정해진 횟수만큼만 훈련을 하고 싶을 때 유용합니다. 반복자를 만들고 명시적으로 `next`를 호출하여 다음 입력 데이터를 얻도록 하면 됩니다. 위 루프 코드를 바꿔보면 다음과 같습니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "e5BEvR0-LJAc", - "colab": {} - }, - "source": [ - "with mirrored_strategy.scope():\n", - " iterator = iter(dist_dataset)\n", - " for _ in range(10):\n", - " print(train_step(next(iterator)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJO8mnypqBA" - }, - "source": [ - "`tf.distribute.Strategy` API를 사용하여 사용자 정의 훈련 루프를 분산 처리 하는 가장 단순한 경우를 살펴보았습니다. 현재 API를 개선하는 과정 중에 있습니다. 이 API를 사용하려면 사용자 쪽에서 꽤 많은 작업을 해야 하므로, 나중에 별도의 더 자세한 가이드로 설명하도록 하겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BZjNwCt1qBdw" - }, - "source": [ - "### 현재 어떤 것이 지원됩니까?\n", - "TF 2.0 베타 버전에서는 사용자 정의 훈련 루프와 함께 위에서 설명한 `MirroredStrategy`, 그리고 `TPUStrategy`를 사용할 수 있습니다. 또한 `MultiWorkerMirorredStrategy`도 추후 지원될 것입니다.\n", - "\n", - "| 훈련 API \t| MirroredStrategy \t| TPUStrategy \t| MultiWorkerMirroredStrategy \t| CentralStorageStrategy \t| ParameterServerStrategy \t|\n", - "|:-----------------------\t|:-------------------\t|:-------------------\t|:-----------------------------\t|:------------------------\t|:-------------------------\t|\n", - "| 사용자 정의 훈련 루프 \t| 지원 \t| 지원 \t| 2.0 RC 지원 예정 \t| 2.0 RC 지원 예정 \t| 아직 미지원 \t|\n", - "\n", - "### 예제와 튜토리얼\n", - "사용자 정의 훈련 루프와 함께 분산 전략을 사용하는 예제들입니다.\n", - "\n", - "1. `MirroredStrategy`로 MNIST를 훈련하는 [튜토리얼](../tutorials/distribute/training_loops.ipynb).\n", - "2. `MirroredStrategy`를 사용하는 [DenseNet](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/densenet/distributed_train.py) 예제.\n", - "3. `MirroredStrategy`와 `TPUStrategy`를 사용하여 훈련하는 [BERT](https://github.com/tensorflow/models/blob/master/official/bert/run_classifier.py) 예제.\n", - "이 예제는 분산 훈련 도중 체크포인트로부터 불러오거나 주기적인 체크포인트를 만드는 방법을 이해하는 데 매우 유용합니다.\n", - "4. `keras_use_ctl` 플래그를 켜서 활성화할 수 있는 `MirroredStrategy`로 훈련한 [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) 예제.\n", - "5. `MirroredStrategy`를 사용하여 훈련하는 [NMT](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/nmt_with_attention/distributed_train.py) 예제." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nO0hmFCRoIll" - }, - "source": [ - "## 추정기(Estimator)와 함께 `tf.distribute.Strategy` 사용하기\n", - "`tf.estimator`는 원래부터 비동기 파라미터 서버 방식을 지원했던 분산 훈련 텐서플로 API입니다. 케라스와 마찬가지로 `tf.distribute.Strategy`를 `tf.estimator`와 함께 쓸 수 있습니다. 추정기 사용자는 아주 조금만 코드를 변경하면, 훈련이 분산되는 방식을 쉽게 바꿀 수 있습니다. 따라서 이제는 추정기 사용자들도 다중 GPU나 다중 워커뿐 아니라 다중 TPU에서 동기 방식으로 분산 훈련을 할 수 있습니다. 하지만 추정기는 제한적으로 지원하는 것입니다. 자세한 내용은 아래 [현재 어떤 것이 지원됩니까?](#estimator_support) 부분을 참고하십시오.\n", - "\n", - "추정기와 함께 `tf.distribute.Strategy`를 사용하는 방법은 케라스와는 살짝 다릅니다. `strategy.scope`를 사용하는 대신에, 전략 객체를 추정기의 [`RunConfig`](https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig)(실행 설정)에 넣어서 전달해야합니다.\n", - "\n", - "다음은 기본으로 제공되는 `LinearRegressor`와 `MirroredStrategy`를 함께 사용하는 방법을 보여주는 코드입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oGFY5nW_B3YU", - "colab": {} - }, - "source": [ - "mirrored_strategy = tf.distribute.MirroredStrategy()\n", - "config = tf.estimator.RunConfig(\n", - " train_distribute=mirrored_strategy, eval_distribute=mirrored_strategy)\n", - "regressor = tf.estimator.LinearRegressor(\n", - " feature_columns=[tf.feature_column.numeric_column('feats')],\n", - " optimizer='SGD',\n", - " config=config)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n6eSfLN5RGY8" - }, - "source": [ - "위 예제에서는 기본으로 제공되는 추정기를 사용하였지만, 직접 만든 추정기도 동일한 코드로 사용할 수 있습니다. `train_distribute`가 훈련을 어떻게 분산시킬지를 지정하고, `eval_distribute`가 평가를 어떻게 분산시킬지를 지정합니다. 케라스와 함께 사용할 때 훈련과 평가에 동일한 분산 전략을 사용했던 것과는 차이가 있습니다.\n", - "\n", - "다음과 같이 입력 함수를 지정하면 추정기의 훈련과 평가를 할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ky2ve2PB3YP", - "colab": {} - }, - "source": [ - "def input_fn():\n", - " dataset = tf.data.Dataset.from_tensors(({\"feats\":[1.]}, [1.]))\n", - " return dataset.repeat(1000).batch(10)\n", - "regressor.train(input_fn=input_fn, steps=10)\n", - "regressor.evaluate(input_fn=input_fn, steps=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hgaU9xQSSk2x" - }, - "source": [ - "추정기와 케라스의 또 다른 점인 입력 처리 방식을 살펴봅시다. 케라스에서는 각 배치의 데이터가 여러 개의 복제된 작업으로 나누어진다고 설명했습니다. 하지만 추정기에서는 사용자가 `input_fn` 입력 함수를 제공하고, 데이터를 워커나 장비들에 어떻게 나누어 처리할지를 온전히 제어할 수 있습니다. 텐서플로는 배치의 데이터를 자동으로 나누지도 않고, 각 워커에 자동으로 분배하지도 않습니다. 제공된 `input_fn` 함수는 워커마다 한 번씩 호출됩니다. 따라서 워커마다 데이터셋을 받게 됩니다. 한 데이터셋의 배치 하나가 워커의 복제된 작업 하나에 들어가고, 따라서 워커 하나에 N개의 복제된 작업이 있으면 N개의 배치가 수행됩니다. 다시 말하자면 `input_fn`이 반환하는 데이터셋은 `PER_REPLICA_BATCH_SIZE` 즉 복제 작업 하나가 배치 하나에서 처리할 크기여야 합니다. 한 단계에서 처리하는 전체 배치 크기는 `PER_REPLICA_BATCH_SIZE * strategy.num_replicas_in_sync`가 됩니다. 다중 워커를 사용하여 훈련할 때는 데이터를 워커별로 쪼개거나, 아니면 각자 다른 임의의 순서로 섞는 것이 좋을 수도 있습니다. 이렇게 처리하는 예제는 [추정기로 다중 워커를 써서 훈련하기](../tutorials/distribute/multi_worker_with_estimator.ipynb)에서 볼 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_098zB3vVhuV" - }, - "source": [ - "추정기와 함께 `MirroredStrategy`를 사용하는 예를 보았습니다. `TPUStrategy`도 같은 방법으로 추정기와 함께 사용할 수 있습니다.\n", - "```\n", - "config = tf.estimator.RunConfig(\n", - " train_distribute=tpu_strategy, eval_distribute=tpu_strategy)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "G3ieQKfWZhhL" - }, - "source": [ - "비슷하게 다중 워커나 파라미터 서버를 사용한 전략도 사용할 수 있습니다. 코드는 거의 같지만, `tf.estimator.train_and_evaluate`를 사용해야 합니다. 그리고 클러스터에서 프로그램을 실행할 때 \"TF_CONFIG\" 환경변수를 설정해야 합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "A_lvUsSLZzVg" - }, - "source": [ - "### 현재 어떤 것이 지원됩니까?\n", - "\n", - "TF 2.0 베타 버전에서는 추정기와 함께 모든 전략을 제한적으로 지원합니다. 기본적인 훈련과 평가는 동작합니다. 하지만 스캐폴드(scaffold) 같은 고급 기능은 아직 동작하지 않습니다. 또한 다소 버그가 있을 수 있습니다. 현재로써는 추정기와 함께 사용하는 것을 활발히 개선할 계획은 없습니다. 대신 케라스나 사용자 정의 훈련 루프 지원에 집중할 계획입니다. 만약 가능하다면 `tf.distribute` 사용시 이 API들을 먼저 고려하여 주십시오.\n", - "\n", - "| 훈련 API \t| MirroredStrategy \t| TPUStrategy \t| MultiWorkerMirroredStrategy \t| CentralStorageStrategy \t| ParameterServerStrategy \t|\n", - "|:---------------\t|:------------------\t|:-------------\t|:-----------------------------\t|:------------------------\t|:-------------------------\t|\n", - "| 추정기 API \t| 제한적으로 지원 | 제한적으로 지원 | 제한적으로 지원 | 제한적으로 지원 | 제한적으로 지원 |\n", - "\n", - "### 예제와 튜토리얼\n", - "다음은 추정기와 함께 다양한 전략을 사용하는 방법을 처음부터 끝까지 보여주는 예제들입니다.\n", - "\n", - "1. [추정기로 다중 워커를 써서 훈련하기](../tutorials/distribute/multi_worker_with_estimator.ipynb)에서는 `MultiWorkerMirroredStrategy`로 다중 워커를 써서 MNIST를 훈련합니다.\n", - "2. [처음부터 끝까지 살펴보는 예제](https://github.com/tensorflow/ecosystem/tree/master/distribution_strategy)에서는 tensorflow/ecosystem의 쿠버네티스(Kubernetes) 템플릿을 이용하여 다중 워커를 사용하여 훈련합니다. 이 예제에서는 케라스 모델로 시작해서 `tf.keras.estimator.model_to_estimator` API를 이용하여 추정기 모델로 변환합니다.\n", - "3. `MirroredStrategy`나 `MultiWorkerMirroredStrategy`로 훈련할 수 있는 공식 [ResNet50](https://github.com/tensorflow/models/blob/master/official/r1/resnet/imagenet_main.py) 모델.\n", - "4. `TPUStrategy`를 사용한 [ResNet50](https://github.com/tensorflow/tpu/blob/master/models/experimental/distribution_strategy/resnet_estimator.py) 예제." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xk0JdsTHyUnE" - }, - "source": [ - "## 그 밖의 주제\n", - "이번 절에서는 다양한 사용 방식에 관련한 몇 가지 주제들을 다룹니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cP6BUIBtudRk" - }, - "source": [ - "\n", - "### TF\\_CONFIG 환경변수 설정하기\n", - "\n", - "\n", - "다중 워커를 사용하여 훈련할 때는, 앞서 설명했듯이 클러스터의 각 실행 프로그램마다 \"TF\\_CONFIG\" 환경변수를 설정해야합니다. \"TF\\_CONFIG\" 환경변수는 JSON 형식입니다. 그 안에는 클러스터를 구성하는 작업과 작업의 주소 및 각 작업의 역할을 기술합니다. [tensorflow/ecosystem](https://github.com/tensorflow/ecosystem) 저장소에서 훈련 작업에 맞게 \"TF\\_CONFIG\"를 설정하는 쿠버네티스(Kubernetes) 템플릿을 제공합니다.\n", - "\n", - "\"TF\\_CONFIG\" 예를 하나 보면 다음과 같습니다.\n", - "```\n", - "os.environ[\"TF_CONFIG\"] = json.dumps({\n", - " \"cluster\": {\n", - " \"worker\": [\"host1:port\", \"host2:port\", \"host3:port\"],\n", - " \"ps\": [\"host4:port\", \"host5:port\"]\n", - " },\n", - " \"task\": {\"type\": \"worker\", \"index\": 1}\n", - "})\n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fezd3aF8wj9r" - }, - "source": [ - "이 \"TF\\_CONFIG\"는 세 개의 워커와 두 개의 파라미터 서버(ps) 작업을 각각의 호스트 및 포트와 함께 지정하고 있습니다. \"task\" 부분은 클러스터 내에서 현재 작업이 담당한 역할을 지정합니다. 여기서는 워커(worker) 1번, 즉 두 번째 워커라는 뜻입니다. 클러스터 내에서 가질 수 있는 역할은 \"chief\"(지휘자), \"worker\"(워커), \"ps\"(파라미터 서버), \"evaluator\"(평가자) 중 하나입니다. 단, \"ps\" 역할은 `tf.distribute.experimental.ParameterServerStrategy` 전략을 사용할 때만 쓸 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GXIbqSW-sFVg" - }, - "source": [ - "## 다음으로는...\n", - "\n", - "`tf.distribute.Strategy`는 활발하게 개발 중입니다. 한 번 써보시고 [깃허브 이슈](https://github.com/tensorflow/tensorflow/issues/new)를 통하여 피드백을 주시면 감사하겠습니다." - ] - } - ] -} diff --git a/site/ko/guide/eager.ipynb b/site/ko/guide/eager.ipynb deleted file mode 100644 index e8183c52de8..00000000000 --- a/site/ko/guide/eager.ipynb +++ /dev/null @@ -1,1294 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "eager.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCQY7jpBfMur" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "z6X9omPnfO_h", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2QQJJyDzqGRb" - }, - "source": [ - "# Eager execution\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B1xdylywqUSX" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub)에서 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-agGVYp_4GWZ", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/distribute_strategy.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃허브 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EGjDcGxIqEfX" - }, - "source": [ - "\n", - "\n", - "텐서플로의 즉시 실행은 그래프를 생성하지 않고 함수를 바로 실행하는 명령형 프로그래밍 환경입니다.\n", - "나중에 실행하기 위해 계산가능한 그래프를 생성하는 대신에 계산값을 즉시 알려주는 연산입니다.\n", - "이러한 기능은 텐서플로를 시작하고 모델을 디버깅하는 것을 더욱 쉽게 만들고 불필요한 상용구 코드(boilerplate code) 작성을 줄여줍니다.\n", - "가이드를 따라하려면, 대화형 `파이썬` 해석기(interpreter)를 이용해 아래에 있는 코드 샘플을 실행하세요.\n", - "\n", - "즉시 실행은 연구와 실험을 위한 유연한 기계학습 플랫폼으로 다음과 같은 기능을 제공합니다:\n", - "\n", - "* *직관적인 인터페이스*-코드를 자연스럽게 구조화하고 파이썬의 데이터 구조를 활용. 작은 모델과 작은 데이터를 빠르게 반복\n", - "* *손쉬운 디버깅*-실행중인 모델을 검토하거나 변경 사항을 테스트해보기 위해서 연산을 직접 호출. 에러 확인을 위해서 표준 파이썬 디버깅 툴을 사용\n", - "* *자연스런 흐름 제어*-그래프 제어 흐름 대신에 파이썬 제어 흐름을 사용함으로써 동적인 모델 구조의 단순화\n", - "\n", - "즉시 실행은 대부분의 텐서플로 연산과 GPU 가속을 지원합니다.\n", - "\n", - "Note: 일부 모델은 즉시 실행을 활성화한 경우 추가연산비용(overhead)이 증가한 경우도 있습니다.성능을 향상시키려는 노력은 계속 진행 중이지만 만약에 문제점을 찾거나 관련된 벤치마크를 공유하고 싶다면 [버그를 기록해주세요](https://github.com/tensorflow/tensorflow/issues)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RBAeIwOMrYk8" - }, - "source": [ - "## 설치와 기본 사용법" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ByNsp4VqqEfa", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version은 오직 Colab에서만 사용가능합니다.\n", - " %tensorflow_version 2.x #gpu\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import cProfile" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "48P3-8q4qEfe" - }, - "source": [ - "텐서플로 2.0에서 즉시 실행은 기본으로 활성화되어 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "7aFsD8csqEff", - "colab": {} - }, - "source": [ - "tf.executing_eagerly()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x_G1zZT5qEfh" - }, - "source": [ - "이제부터는 텐서플로 연산을 바로 실행할 수 있고 결과를 즉시 확인할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "9gsI54pbqEfj", - "colab": {} - }, - "source": [ - "x = [[2.]]\n", - "m = tf.matmul(x, x)\n", - "print(\"hello, {}\".format(m))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ajFn6qsdqEfl" - }, - "source": [ - "즉시 실행 활성화는 텐서플로 연산을 바로 평가하고 그 결과를 파이썬에게 알려주는 방식으로 동작을 변경합니다.\n", - "`tf.Tensor` 객체는 계산 그래프에 있는 노드를 가르키는 간접 핸들(symbolic handle) 대신에 구체적인 값을 참조합니다.\n", - "나중에 실행하기 위해서 생성된 계산 그래프가 없기 때문에, `print()`나 디버거를 통해서 결과를 검토하기 쉽습니다.\n", - "텐서값을 평가, 출력하거나 확인하는 것이 그래디언트(gradients)를 계산하는 흐름을 방해하지 않습니다.\n", - "\n", - "즉시 실행은 [NumPy](http://www.numpy.org/)와 같이 잘 작동됩니다.\n", - "NumPy 연산에 `tf.Tensor`를 매개변수로 사용가능합니다.\n", - "텐서플로 [수학 연산](https://www.tensorflow.org/api_guides/python/math_ops)은 파이썬 객체와 NumPy 배열을 `tf.Tensor` 객체로 변환합니다.\n", - "`tf.Tensor.numpy` 메서드는 객체 값을 NumPy `ndarray`로 반환합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sTO0_5TYqz1n", - "colab": {} - }, - "source": [ - "a = tf.constant([[1, 2],\n", - " [3, 4]])\n", - "print(a)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Dp14YT8Gq4r1", - "colab": {} - }, - "source": [ - "# 브로드캐스팅(Broadcasting) 지원\n", - "b = tf.add(a, 1)\n", - "print(b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "69p3waMfq8cQ", - "colab": {} - }, - "source": [ - "# 연산자 오버로딩 지원\n", - "print(a * b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Ui025t1qqEfm", - "colab": {} - }, - "source": [ - "# NumPy값 사용\n", - "import numpy as np\n", - "\n", - "c = np.multiply(a, b)\n", - "print(c)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tq_aFRzWrCua", - "colab": {} - }, - "source": [ - "# 텐서로부터 numpy 값 얻기:\n", - "print(a.numpy())\n", - "# => [[1 2]\n", - "# [3 4]]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H08f9ss9qEft" - }, - "source": [ - "## 동적인 제어 흐름\n", - "\n", - "즉시 실행의 가장 큰 이점은 모델을 실행하는 동안에도 호스트 언어의 모든 기능을 활용할 수 있다는 것입니다.\n", - "그래서 다음과 같이 [fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz)를 손쉽게 작성할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "0fudRMeUqEfu", - "colab": {} - }, - "source": [ - "def fizzbuzz(max_num):\n", - " counter = tf.constant(0)\n", - " max_num = tf.convert_to_tensor(max_num)\n", - " for num in range(1, max_num.numpy()+1):\n", - " num = tf.constant(num)\n", - " if int(num % 3) == 0 and int(num % 5) == 0:\n", - " print('FizzBuzz')\n", - " elif int(num % 3) == 0:\n", - " print('Fizz')\n", - " elif int(num % 5) == 0:\n", - " print('Buzz')\n", - " else:\n", - " print(num.numpy())\n", - " counter += 1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "P2cKknQWrJLB", - "colab": {} - }, - "source": [ - "fizzbuzz(15)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7kA-aC3BqEfy" - }, - "source": [ - "여기에 텐서값에 따른 조건절이 있고 실행중에 그 결과를 출력합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8huKpuuAwICq" - }, - "source": [ - "## 즉시 훈련" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mp2lCCZYrxHd" - }, - "source": [ - "### 그래디언트 계산하기\n", - "\n", - "[자동 미분](https://en.wikipedia.org/wiki/Automatic_differentiation)은 인공 신경망 훈련을 위한\n", - "[역전파](https://en.wikipedia.org/wiki/Backpropagation)와 같은 기계학습 알고리즘을 구현하는데 유용합니다.\n", - "즉시 실행을 사용하는 동안에는, 나중에 그래디언트를 계산하는 연산을 추적하기 위해 `tf.GradientTape`을 사용하세요.\n", - "\n", - "즉시 실행 중에 그래디언트를 계산하고 모델 훈련에 이용하기 위해서 `tf.GradientTape`을 사용할 수 있습니다.\n", - "특히 복잡하고 반복적인 훈련인 경우에 더 유용합니다.\n", - "\n", - "매번 실행될 때 서로 다른 연산이 수행될 수 있기 때문에 모든 정방향(forward-pass) 연산은 \"tape\"에 기록됩니다.\n", - "그다음 tape를 거꾸로 돌려 그래디언트를 계산한 후 tape를 폐기합니다.\n", - "특정한 `tf.GradientTape`는 오직 하나의 그래디언트만을 계산할 수 있고 부가적인 호출은 실행중 에러(runtime error)를 발생시킵니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "7g1yWiSXqEf-", - "colab": {} - }, - "source": [ - "w = tf.Variable([[1.0]])\n", - "with tf.GradientTape() as tape:\n", - " loss = w * w\n", - "\n", - "grad = tape.gradient(loss, w)\n", - "print(grad) # => tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkHs32GqweYS" - }, - "source": [ - "### 모델 훈련\n", - "\n", - "다음 예는 표준 MNIST 손글씨 분류를 위한 다층 모델을 생성합니다.\n", - "즉시 실행 환경에서 훈련가능한 그래프를 생성하기 위한 옵티마이저(optimizer)와 층 API를 보여줍니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "38kymXZowhhz", - "colab": {} - }, - "source": [ - "# mnist 데이터 가져오기 및 포맷 맞추기\n", - "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n", - "\n", - "dataset = tf.data.Dataset.from_tensor_slices(\n", - " (tf.cast(mnist_images[...,tf.newaxis]/255, tf.float32),\n", - " tf.cast(mnist_labels,tf.int64)))\n", - "dataset = dataset.shuffle(1000).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rl1K8rOowmwT", - "colab": {} - }, - "source": [ - "# 모델 생성\n", - "mnist_model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu',\n", - " input_shape=(None, None, 1)),\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(10)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fvyk-HgGwxwl" - }, - "source": [ - "\n", - "즉시 실행에서는 훈련을 하지 않아도 모델을 사용하고 결과를 점검할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BsxystjBwxLS", - "colab": {} - }, - "source": [ - "for images,labels in dataset.take(1):\n", - " print(\"로짓: \", mnist_model(images[0:1]).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y3PGa8G7qEgB" - }, - "source": [ - "케라스 모델은 자체적인 훈련 메서드(fit)을 포함하고 있지만 때로는 좀 더 수정할 필요가 있습니다.\n", - "다음은 즉시 실행을 활용한 반복적인 훈련의 예입니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bzRhM7JDnaEG", - "colab": {} - }, - "source": [ - "optimizer = tf.keras.optimizers.Adam()\n", - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", - "\n", - "loss_history = []" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tXaupYXRI2YM" - }, - "source": [ - "Note: 조건을 만족했는지 확인하기 위해서 `tf.debugging`에 있는 단언문(assert) 함수를 사용하세요. 이것은 즉시 실행과 그래프 실행 모두에서 동작합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DDHrigtiCIA4", - "colab": {} - }, - "source": [ - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " logits = mnist_model(images, training=True)\n", - " \n", - " # 결과의 형태를 확인하기 위해서 단언문 추가\n", - " tf.debugging.assert_equal(logits.shape, (32, 10))\n", - " \n", - " loss_value = loss_object(labels, logits)\n", - "\n", - " loss_history.append(loss_value.numpy().mean())\n", - " grads = tape.gradient(loss_value, mnist_model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, mnist_model.trainable_variables))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "0m1xAXrmqEgJ", - "colab": {} - }, - "source": [ - "def train():\n", - " for epoch in range(3):\n", - " for (batch, (images, labels)) in enumerate(dataset):\n", - " train_step(images, labels)\n", - " print ('에포크 {} 종료'.format(epoch))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "C5dGz0p_nf4W", - "colab": {} - }, - "source": [ - "train()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5vG5ql_2vYB5", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.plot(loss_history)\n", - "plt.xlabel('Batch #')\n", - "plt.ylabel('Loss [entropy]')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKpOlHPLqEgl" - }, - "source": [ - "### 변수와 옵티마이저\n", - "\n", - "`tf.Variable` 객체는 자동 미분을 쉽게 하기 위해서 학습동안 변경된 `tf.Tensor` 값을 저장합니다.\n", - "모델 파라미터는 클래스 인스턴스 변수로 캡슐화될 수 있습니다.\n", - "\n", - "효과적으로 모델 파라미터를 캡슐화하려면 `tf.Variable`을 `tf.GradientTape`과 함께 사용합니다.\n", - "예를 들어, 위의 자동 미분은 다음과 같이 재작성 가능합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "nnQLBYmEqEgm", - "colab": {} - }, - "source": [ - "class Model(tf.keras.Model):\n", - " def __init__(self):\n", - " super(Model, self).__init__()\n", - " self.W = tf.Variable(5., name='weight')\n", - " self.B = tf.Variable(10., name='bias')\n", - " def call(self, inputs):\n", - " return inputs * self.W + self.B\n", - "\n", - "# 약 3 * x + 2개의 점으로 구성된 실험 데이터\n", - "NUM_EXAMPLES = 2000\n", - "training_inputs = tf.random.normal([NUM_EXAMPLES])\n", - "noise = tf.random.normal([NUM_EXAMPLES])\n", - "training_outputs = training_inputs * 3 + 2 + noise\n", - "\n", - "# 최적화할 손실함수\n", - "def loss(model, inputs, targets):\n", - " error = model(inputs) - targets\n", - " return tf.reduce_mean(tf.square(error))\n", - "\n", - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return tape.gradient(loss_value, [model.W, model.B])\n", - "\n", - "# 정의:\n", - "# 1. 모델\n", - "# 2. 모델 파라미터에 대한 손실 함수의 미분\n", - "# 3. 미분에 기초한 변수 업데이트 전략\n", - "model = Model()\n", - "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)\n", - "\n", - "print(\"초기 손실: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", - "\n", - "# 반복 훈련\n", - "for i in range(300):\n", - " grads = grad(model, training_inputs, training_outputs)\n", - " optimizer.apply_gradients(zip(grads, [model.W, model.B]))\n", - " if i % 20 == 0:\n", - " print(\"스텝 {:03d}에서 손실: {:.3f}\".format(i, loss(model, training_inputs, training_outputs)))\n", - "\n", - "print(\"최종 손실: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", - "print(\"W = {}, B = {}\".format(model.W.numpy(), model.B.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rPjb8nRWqEgr" - }, - "source": [ - "## 즉시 실행에서 상태를 위한 객체 사용\n", - "\n", - "텐서플로 1.x 그래프 실행에서, 프로그램 상태(예: 변수)는 전역 컬렉션에 저장되고 그 수명은 `tf.Session` 객체에 의해서 관리됩니다.\n", - "반면에 즉시 실행에서 상태 객체 수명은 그와 관련된 파이썬 객체 수명에 의해서 결정됩니다.\n", - "\n", - "### 변수는 객체입니다\n", - "\n", - "즉시 실행에서 변수는 그 객체의 마지막 참조가 제거될 때까지 유지되고 그 이후 삭제됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "A2boS674qEgs", - "colab": {} - }, - "source": [ - "if tf.config.experimental.list_physical_devices(\"GPU\"):\n", - " with tf.device(\"gpu:0\"):\n", - " print(\"GPU 사용 가능\")\n", - " v = tf.Variable(tf.random.normal([1000, 1000]))\n", - " v = None # v는 더이상 GPU 메모리를 사용하지 않음" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "scMjg6L6qEgv" - }, - "source": [ - "### 객체 기반의 저장\n", - "\n", - "이번 장은 [훈련 체크포인트 가이드](./checkpoint.ipynb) 요약버전입니다.\n", - "\n", - "`tf.train.Checkpoint`는 `tf.Variable`을 체크포인트 파일로 저장하거나 체크포인트 파일에서 복구할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7z5xRfdHzZOQ", - "colab": {} - }, - "source": [ - "x = tf.Variable(10.)\n", - "checkpoint = tf.train.Checkpoint(x=x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IffrUVG7zyVb", - "colab": {} - }, - "source": [ - "x.assign(2.) # 변수에 새로운 값을 할당하고 저장\n", - "checkpoint_path = './ckpt/'\n", - "checkpoint.save('./ckpt/')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "eMT9koCoqEgw", - "colab": {} - }, - "source": [ - "x.assign(11.) # 저장한 후에 변수 변경\n", - "\n", - "# 체크포인트로부터 값을 복구\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_path))\n", - "\n", - "print(x) # => 2.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbFnP-yLqEgx" - }, - "source": [ - "모델을 저장하거나 읽어들이기 위해서, `tf.train.Checkpoint`는 숨겨진 변수를 요구하지 않고 객체 내부 상태를 저장합니다. \n", - "`옵티마이저`와 `모델`, 전역 단계 상태를 기록하려면 `tf.train.Checkpoint`에 전달하면 됩니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "hWZHyAXMqEg0", - "colab": {} - }, - "source": [ - "import os\n", - "\n", - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(10)\n", - "])\n", - "optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\n", - "checkpoint_dir = 'path/to/model_dir'\n", - "if not os.path.exists(checkpoint_dir):\n", - " os.makedirs(checkpoint_dir)\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "root = tf.train.Checkpoint(optimizer=optimizer,\n", - " model=model)\n", - "\n", - "root.save(checkpoint_prefix)\n", - "root.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R-ITwkBCF6GJ" - }, - "source": [ - "Note: 대부분의 반복 훈련 과정에서 변수는 `tf.train.Checkpoint.restore`가 호출된 이후에 생성됩니다.\n", - "이러한 변수는 생성되자마자 복원될 것이므로 단언문을 통해 체크포인트가 완벽히 적재되었다는 것을 보장받을 수 있습니다.\n", - "자세한 내용은 [훈련 체크포인트 가이드](./checkpoint.ipynb)를 참고하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3yoD0VJ7qEg3" - }, - "source": [ - "### 객체 지향형 지표\n", - "\n", - "`tf.keras.metrics`는 객체로 저장됩니다.\n", - "새로운 데이터를 이 객체에 전달하여 지표를 수정하고 `tf.keras.metrics.result` 메서드를 사용해 그 결과를 얻습니다.\n", - "예를 들어:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "9ccu0iAaqEg5", - "colab": {} - }, - "source": [ - "m = tf.keras.metrics.Mean(\"loss\")\n", - "m(0)\n", - "m(5)\n", - "m.result() # => 2.5\n", - "m([8, 9])\n", - "m.result() # => 5.5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aB8qWtT955pI" - }, - "source": [ - "### 서머리(summary)와 텐서보드\n", - "\n", - "[텐서보드](https://tensorflow.org/tensorboard)는 훈련과정에서 모델을 파악하거나 디버깅하고 최적화하기 위해 사용하는 시각화 도구입니다.\n", - "텐서보드는 프로그램이 실행되는 동안 작성된 서머리 이벤트를 사용합니다.\n", - "\n", - "즉시 실행에서 변수의 서머리 정보를 기록하기 위해서 `tf.summary`를 사용합니다.\n", - "예를 들어, 다음은 매 100번째 훈련마다 `loss`의 서머리 정보를 기록합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "z6VInqhA6RH4", - "colab": {} - }, - "source": [ - "logdir = \"./tb/\"\n", - "writer = tf.summary.create_file_writer(logdir)\n", - "\n", - "with writer.as_default(): # 또는 반복 전에 writer.set_as_default()를 호출\n", - " for i in range(1000):\n", - " step = i + 1\n", - " # 실제 훈련 함수로 손실을 계산\n", - " loss = 1 - 0.001 * step\n", - " if step % 100 == 0:\n", - " tf.summary.scalar('손실', loss, step=step)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "08QQD2j36TaI", - "colab": {} - }, - "source": [ - "!ls tb/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xEL4yJe5qEhD" - }, - "source": [ - "## 자동 미분 관련 고급편\n", - "\n", - "### 동적 모델\n", - "\n", - "`tf.GradientTape`는 또한 동적인 모델에서도 사용가능합니다.\n", - "아래 예는 [역추적 길찾기](https://wikipedia.org/wiki/Backtracking_line_search) 알고리즘의 복잡한 제어 흐름에도 불구하고,\n", - "그래디언트가 있으며 미분 가능이 하다는 것을 제외하면 일반적인 NumPy으로 작성한 코드처럼 보입니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "L518n5dkqEhE", - "colab": {} - }, - "source": [ - "def line_search_step(fn, init_x, rate=1.0):\n", - " with tf.GradientTape() as tape:\n", - " # 변수는 자동적으로 기록되지만 텐서는 사용자가 스스로 확인해야 함\n", - " tape.watch(init_x)\n", - " value = fn(init_x)\n", - " grad = tape.gradient(value, init_x)\n", - " grad_norm = tf.reduce_sum(grad * grad)\n", - " init_value = value\n", - " while value > init_value - rate * grad_norm:\n", - " x = init_x - rate * grad\n", - " value = fn(x)\n", - " rate /= 2.0\n", - " return x, value" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gieGOf_DqEhK" - }, - "source": [ - "### 사용자 정의 그래디언트\n", - "\n", - "사용자 정의 그래디언트는 그래디언트를 재정의(override)하는 가장 쉬운 방법입니다.\n", - "정방향 함수안에서 입력값 또는 출력값, 중간값과 관련된 그래디언트를 정의해야 합니다. \n", - "예를 들어 다음은 역전파 과정에서 그래디언트의 놈(norm)을 클리핑(clip)하는 가장 쉬운 방법입니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "-OwwsWUAqEhK", - "colab": {} - }, - "source": [ - "@tf.custom_gradient\n", - "def clip_gradient_by_norm(x, norm):\n", - " y = tf.identity(x)\n", - " def grad_fn(dresult):\n", - " return [tf.clip_by_norm(dresult, norm), None]\n", - " return y, grad_fn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JPLDHkF_qEhN" - }, - "source": [ - "사용자 정의 그래디언트는 일반적으로 연산에 대해 수치적으로(numerically) 안정된 그래디언트를 제공하기 위해 사용됩니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "24WiLROnqEhO", - "colab": {} - }, - "source": [ - "def log1pexp(x):\n", - " return tf.math.log(1 + tf.exp(x))\n", - "\n", - "def grad_log1pexp(x):\n", - " with tf.GradientTape() as tape:\n", - " tape.watch(x)\n", - " value = log1pexp(x)\n", - " return tape.gradient(value, x)\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "n8fq69r9-B-c", - "colab": {} - }, - "source": [ - "# 그래디언트 계산은 x = 0일 때 잘 동작\n", - "grad_log1pexp(tf.constant(0.)).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_VFSU0mG-FSp", - "colab": {} - }, - "source": [ - "# 그러나, x = 100일 때 수치적으로 불안정하기 때문에 실패\n", - "grad_log1pexp(tf.constant(100.)).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-VcTR34rqEhQ" - }, - "source": [ - "여기 `log1pexp` 함수는 이론적으로 사용자 정의 그래디언트를 활용해 간결해 질 수 있습니다.\n", - "아래 구현은 불필요한 계산을 제거함으로써 계산을 좀 더 효율적으로 하기 위해 정방향 경로안에서 계산된 `tf.exp(x)`값을 재사용합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Q7nvfx_-qEhS", - "colab": {} - }, - "source": [ - "@tf.custom_gradient\n", - "def log1pexp(x):\n", - " e = tf.exp(x)\n", - " def grad(dy):\n", - " return dy * (1 - 1 / (1 + e))\n", - " return tf.math.log(1 + e), grad\n", - "\n", - "def grad_log1pexp(x):\n", - " with tf.GradientTape() as tape:\n", - " tape.watch(x)\n", - " value = log1pexp(x)\n", - " return tape.gradient(value, x)\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5gHPKMfl-Kge", - "colab": {} - }, - "source": [ - "# 전처럼, 그래디언트 계산은 x = 0일 때 잘 동작\n", - "grad_log1pexp(tf.constant(0.)).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u38MOfz3-MDE", - "colab": {} - }, - "source": [ - "# 그래디언트 계산은 x = 100일 때 역시 잘 동작\n", - "grad_log1pexp(tf.constant(100.)).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rnZXjfQzqEhV" - }, - "source": [ - "## 성능\n", - "\n", - "즉시 실행에서 계산은 자동으로 GPU로 분배됩니다.\n", - "만약 계산 분배를 사용자가 제어하고 싶다면 그 부분을 `tf.device('/gpu:0')` 블록 (또는 CPU도 동일)으로 감싸서 실행하세요:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Ac9Y64H-qEhX", - "colab": {} - }, - "source": [ - "import time\n", - "\n", - "def measure(x, steps):\n", - " # 텐서플로는 처음 사용할 때 GPU를 초기화, 시간계산에서 제외\n", - " tf.matmul(x, x)\n", - " start = time.time()\n", - " for i in range(steps):\n", - " x = tf.matmul(x, x)\n", - " # tf.matmul는 행렬 곱셈을 완료하기 전에 결과를 반환할 수 있습니다\n", - " # (예, CUDA 스트림 대기열에 연산을 추가한 후에 결과를 반환할 수 있다).\n", - " # 아래 x.numpy() 호출은 대기열에 추가된 모든 연산이 완료될 것임을 보장합니다\n", - " # (그리고 그 결과가 호스트 메모리에 복사될 것이고,\n", - " # 그래서 matnul 연산시간보다는 조금 많은 연산시간이\n", - " # 포함됩니다).\n", - " _ = x.numpy()\n", - " end = time.time()\n", - " return end - start\n", - "\n", - "shape = (1000, 1000)\n", - "steps = 200\n", - "print(\"{} 크기 행렬을 자기 자신과 {}번 곱했을 때 걸리는 시간:\".format(shape, steps))\n", - "\n", - "# CPU에서 실행:\n", - "with tf.device(\"/cpu:0\"):\n", - " print(\"CPU: {} 초\".format(measure(tf.random.normal(shape), steps)))\n", - "\n", - "# GPU에서 실행, 가능하다면:\n", - "if tf.config.experimental.list_physical_devices(\"GPU\"):\n", - " with tf.device(\"/gpu:0\"):\n", - " print(\"GPU: {} 초\".format(measure(tf.random.normal(shape), steps)))\n", - "else:\n", - " print(\"GPU: 없음\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RLw3IS7UqEhe" - }, - "source": [ - "`tf.Tensor` 객체는 실제로 그 연산을 수행할 다른 디바이스로 복사될 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "ny6LX2BVqEhf", - "colab": {} - }, - "source": [ - "if tf.config.experimental.list_physical_devices(\"GPU\"):\n", - " x = tf.random.normal([10, 10])\n", - "\n", - " x_gpu0 = x.gpu()\n", - " x_cpu = x.cpu()\n", - "\n", - " _ = tf.matmul(x_cpu, x_cpu) # CPU에서 실행\n", - " _ = tf.matmul(x_gpu0, x_gpu0) # GPU:0에서 실행" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oA_qaII3-p6c" - }, - "source": [ - "### 벤치마크\n", - "\n", - "GPU에서 학습을 하는 [ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50) 같은 계산량이 많은 모델에서,\n", - "즉시 실행 성능은 `tf.function` 실행과 비교될 수 있습니다.\n", - "그러나 이러한 차이는 계산량이 작은 모델인 경우 더 커지고, 수많은 작은 연산으로 구성된 모델은 자주 반복되는 부분을 최적화하는 사례도 있습니다.\n", - "\n", - "## 함수를 활용\n", - "\n", - "즉시 실행이 개발과 디버깅 과정을 좀 더 대화형(interactive)으로 만들어 주지만\n", - "텐서플로 1.x 형태 그래프 실행은 학습의 분산과 성능, 운영 배포에 장점을 가지고 있습니다.\n", - "이러한 차이를 해소하기 위해서, 텐서플로 2.0에서는 `tf.function` API를 도입했습니다.\n", - "자세한 내용은 [tf.function](./function.ipynb) 가이드를 참고하세요." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/guide/effective_tf2.md b/site/ko/guide/effective_tf2.md deleted file mode 100644 index 9cacd6ad9a1..00000000000 --- a/site/ko/guide/effective_tf2.md +++ /dev/null @@ -1,212 +0,0 @@ -# 이펙티브 텐서플로 2.0 - -Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 -불구하고 -[공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/effective_tf2.md)의 -내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 -[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 -바랍니다. 문서 번역이나 리뷰에 참여하려면 -[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 -메일을 보내주시기 바랍니다. - -텐서플로 2.0은 사용자의 생산성을 향상시키기 위해서 많은 것을 바꾸었습니다. [불필요한 API](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md)를 제거하고 API의 일관성을 높였으며([Unified RNNs](https://github.com/tensorflow/community/blob/master/rfcs/20180920-unify-rnn-interface.md), -[Unified Optimizers](https://github.com/tensorflow/community/blob/master/rfcs/20181016-optimizer-unification.md)) 파이썬 런타임(runtime)과 [즉시 실행](https://www.tensorflow.org/guide/eager)(eager execution)을 통합하였습니다. - -여러 [RFC](https://github.com/tensorflow/community/pulls?utf8=%E2%9C%93&q=is%3Apr) 문서에서 텐서플로 2.0의 변경 내용을 확인할 수 있습니다. 이 가이드에서는 텐서플로 2.0을 사용한 개발 방식을 소개합니다. 여러분이 텐서플로 1.x에 친숙하다고 가정하겠습니다. - -## 주요 변경 사항 요약 - -### API 정리 - -많은 API가 TF 2.0에서 [삭제 또는 이동](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md)되었습니다. 주요한 변화는 다음과 같습니다. `tf.app`, `tf.flags`, `tf.logging`을 삭제하고 [absl-py](https://github.com/abseil/abseil-py) 오픈 소스를 권장합니다. `tf.contrib` 아래에 있던 프로젝트를 이동했습니다. 자주 사용하지 않는 함수를 `tf.math` 같은 서브패키지(subpackage)로 이동하는 식으로 `tf.*` 네임스페이스(namespace)를 정리하였습니다. `tf.summary`, `tf.keras.metrics`, `tf.keras.optimizers`와 같은 일부 API는 2.0 버전으로 교체되었습니다. 교체된 이름을 자동으로 적용하려면 [v2 upgrade script](upgrade.md) 사용하는 것이 가장 편리합니다. - -### 즉시 실행 - -텐서플로 1.x에서는 사용자가 `tf.*` API를 호출해서 [추상 구문 트리](https://ko.wikipedia.org/wiki/%EC%B6%94%EC%83%81_%EA%B5%AC%EB%AC%B8_%ED%8A%B8%EB%A6%AC)를 수동으로 구성했습니다. 그다음 `session.run()`을 호출할 때 출력 텐서와 입력 텐서를 전달하여 추상 구문 트리를 수동으로 컴파일합니다. 텐서플로 2.0은 (보통의 파이썬처럼) 즉시 실행됩니다. 텐서플로 2.0에서 그래프와 세션은 구현 상세(implementation detail)처럼 느껴질 것입니다. - -즉시 실행으로 인한 부수효과 중 하나는 더이상 `tf.control_dependencies()`이 필요하지 않다는 것입니다. 모든 코드는 라인 순서대로 실행됩니다(`tf.function` 안의 코드도 이 효과로 쓰여진 순서대로 실행됩니다). - -### 전역 메커니즘 제거 - -텐서플로 1.x는 겉으로 드러나진 않았지만 전역 이름 공간(namespace)에 크게 의존했습니다. `tf.Variable()`를 호출하면 기본 그래프에 노드(node)를 추가합니다. 노드를 참조하는 파이썬 변수가 삭제되더라도 그래프에 그대로 남아 있습니다. 이 `tf.Variable` 노드를 다시 참조할 수 있지만 생성할 때 지정한 이름을 알아야만 가능합니다. 변수를 직접 만들지 않았다면 어려운 일입니다. 이 때문에 사용자와 프레임워크가 변수를 추적할 수 있도록 여러 종류의 메커니즘이 늘어 났습니다. 변수 범위(variable scope), 전역 컬렉션(global collection), `tf.get_global_step()`이나 `tf.global_variables_initializer()` 같은 헬퍼 메서드 등입니다. 또 옵티마이저(optimizer)는 암묵적으로 훈련 가능한 모든 변수의 그래디언트(graident)를 계산합니다. 텐서플로 2.0은 이런 모든 메커니즘을 삭제했습니다([Variables 2.0 RFC](https://github.com/tensorflow/community/pull/11)). 대신 파이썬 변수를 추적하는 기본 메커니즘을 사용합니다! `tf.Variable`의 참조를 잃어 버렸다면 자동으로 가비지 컬렉션(garbage collection)될 것입니다. - -사용자가 변수를 관리하는 일이 늘어나지만 케라스(Keras)(아래 참조)를 사용하면 최소화할 수 있습니다. - -### 세션 대신 함수 - -`session.run()`은 거의 함수 호출과 비슷합니다. 입력과 함수를 지정하면 일련의 출력을 얻습니다. 텐서플로 2.0에서는 `tf.function()` 데코레이터(decorator)로 파이썬 함수를 감쌀 수 있습니다. 이렇게 하면 텐서플로가 이 함수를 하나의 그래프로 실행하기 위해 JIT 컴파일합니다([Functions 2.0 RFC](https://github.com/tensorflow/community/pull/20)). 이 메커니즘 덕택에 텐서플로 2.0에서 그래프 모드의 장점을 모두 계승할 수 있습니다. - -- 성능: 함수를 최적화할 수 있습니다(노드 가지치기(pruning), 커널 융합(kernel fusion) 등). -- 이식성(portability): 함수를 저장하고 다시 불러올 수 있습니다([SavedModel 2.0 RFC](https://github.com/tensorflow/community/pull/34)). 모듈화된 텐서플로 함수를 재사용하고 공유할 수 있습니다. - -```python -# 텐서플로 1.x -outputs = session.run(f(placeholder), feed_dict={placeholder: input}) -# 텐서플로 2.0 -outputs = f(input) -``` - -파이썬과 텐서플로 코드를 자유롭게 섞어 쓸 수 있기 때문에 파이썬의 장점을 최대한 활용할 수 있습니다. 텐서플로는 파이썬 인터프리터가 없는 모바일, C++, 자바스크립트 같은 환경에서도 실행됩니다. 사용자가 환경에 따라 코드를 재작성하지 않도록 `@tf.function`를 추가하면 [오토그래프](function.ipynb)(AutoGraph)가 파이썬 코드를 동일한 텐서플로 코드로 변경합니다. - -* `for`/`while` -> `tf.while_loop` (`break`과 `continue` 문을 지원합니다.) -* `if` -> `tf.cond` -* `for _ in dataset` -> `dataset.reduce` - -오토그래프는 임의의 중첩된 제어 흐름도 지원합니다. 시퀀스(sequence) 모델, 강화 학습(reinforcement learning), 독자적인 훈련 루프 등 복잡한 머신러닝 프로그램을 간결하면서 높은 성능을 내도록 구현할 수 있습니다. - -## 텐서플로 2.0의 권장 사항 - -### 작은 함수로 코드를 리팩토링하세요. - -텐서플로 1.x의 일반적인 사용 패턴은 "키친 싱크(kitchen sink)" 전략입니다. 먼저 모든 연산을 결합하여 준비한 다음 `session.run()`을 사용해 선택한 텐서를 평가합니다. 텐서플로 2.0에서는 필요할 때 호출할 수 있는 작은 함수로 코드를 리팩토링(refactoring)해야 합니다. 모든 함수에 `tf.function` 데코레이터를 적용할 필요는 없습니다. 모델 훈련의 한 단계(step)나 정방향 연산(forward pass) 같은 고수준 연산에만 `tf.function` 데코레이터를 적용하세요. - -### 케라스 층과 모델을 사용해 변수를 관리하세요. - -케라스 모델과 층(layer)은 재귀적으로 의존하는 모든 변수를 수집하여 `variables`와 `trainable_variables` 속성으로 제공합니다. 따라서 변수를 지역 범위로 관리하기 매우 쉽습니다. - -기본 버전: - -```python -def dense(x, W, b): - return tf.nn.sigmoid(tf.matmul(x, W) + b) - -@tf.function -def multilayer_perceptron(x, w0, b0, w1, b1, w2, b2 ...): - x = dense(x, w0, b0) - x = dense(x, w1, b1) - x = dense(x, w2, b2) - ... - -# 여전히 w_i, b_i 변수를 직접 관리해야 합니다. 이 코드와 떨어져서 크기가 정의됩니다. -``` - -케라스 버전: - -```python -# 각 층은 linear(x)처럼 호출 가능합니다. -layers = [tf.keras.layers.Dense(hidden_size, activation=tf.nn.sigmoid) for _ in range(n)] -perceptron = tf.keras.Sequential(layers) - -# layers[3].trainable_variables => returns [w3, b3] -# perceptron.trainable_variables => returns [w0, b0, ...] -``` - -케라스의 층과 모델은 `tf.train.Checkpointable`을 상속하고 `@tf.function`를 사용하여 통합되어 있습니다. 케라스 객체에서 바로 체크포인트나 SavedModels로 저장할 수 있습니다. 케라스 `.fit()` API를 호출하지 않더라도 이런 기능을 사용할 수 있습니다. - -전이 학습(transfer learning) 예제를 통해서 케라스가 어떻게 관련된 변수를 쉽게 모으는지 알아 보겠습니다. 몸통(trunk)을 공유하는 다중 출력(multi-headed) 모델을 훈련한다고 가정해 보죠. - -```python -trunk = tf.keras.Sequential([...]) -head1 = tf.keras.Sequential([...]) -head2 = tf.keras.Sequential([...]) - -path1 = tf.keras.Sequential([trunk, head1]) -path2 = tf.keras.Sequential([trunk, head2]) - -# 주된 데이터셋에서 훈련합니다. -for x, y in main_dataset: - with tf.GradientTape() as tape: - prediction = path1(x) - loss = loss_fn_head1(prediction, y) - # trunk와 head1 가중치를 동시에 최적화합니다. - gradients = tape.gradient(loss, path1.trainable_variables) - optimizer.apply_gradients(zip(gradients, path1.trainable_variables)) - -# trunk를 재사용하여 head2를 세부 튜닝합니다. -for x, y in small_dataset: - with tf.GradientTape() as tape: - prediction = path2(x) - loss = loss_fn_head2(prediction, y) - # trunk 가중치는 제외하고 head2 가중치만 최적화합니다. - gradients = tape.gradient(loss, head2.trainable_variables) - optimizer.apply_gradients(zip(gradients, head2.trainable_variables)) - -# trunk 연산만 재사용을 위해 저장할 수 있습니다. -tf.saved_model.save(trunk, output_path) -``` - -### tf.data.Datasets과 @tf.function을 연결하세요. - -메모리 크기에 맞는 훈련 데이터를 반복할 때는 보통의 파이썬 반복자를 사용해도 좋습니다. 그렇지 않다면 디스크에서 훈련 데이터를 읽는 가장 좋은 방법은 `tf.data.Dataset`입니다. 데이터셋이 [반복 가능](https://docs.python.org/ko/3/glossary.html#term-iterable)(반복자가 아닙니다)하면 즉시 실행 모드에서는 파이썬의 다른 반복 가능 객체처럼 동작합니다. `tf.function()`으로 코드를 감싸서 비동기 프리페치(prefetch)/스트리밍(streaming) 기능을 모두 사용할 수 있습니다. `tf.function()`은 오토그래프를 사용하여 파이썬 반복문을 동일한 그래프 연산으로 바꾸어 줍니다. - -```python -@tf.function -def train(model, dataset, optimizer): - for x, y in dataset: - with tf.GradientTape() as tape: - prediction = model(x) - loss = loss_fn(prediction, y) - gradients = tape.gradient(loss, model.trainable_variables) - optimizer.apply_gradients(zip(gradients, model.trainable_variables)) -``` - -케라스의 `.fit()` API를 사용하면 데이터셋 반복에 관해 신경 쓸 필요가 없습니다. - -```python -model.compile(optimizer=optimizer, loss=loss_fn) -model.fit(dataset) -``` - -### 파이썬의 제어 흐름과 함께 오토그래프를 사용하세요. - -오토그래프는 데이터에 따라 결정되는 제어 흐름(control flow)을 `tf.cond`와 `tf.while_loop` 같은 그래프 모드 연산으로 변환시켜 줍니다. - -데이터에 의존하는 제어 흐름이 나타나는 대표적인 곳은 시퀀스(sequence) 모델입니다. `tf.keras.layers.RNN`은 RNN 셀(cell)을 감싸서 순환(recurrent) 셀을 정적으로 또는 동적으로 펼칠 수 있습니다. 다음처럼 직접 동적으로 펼치는 구현을 만들어서 확인할 수 있습니다. - -```python -class DynamicRNN(tf.keras.Model): - - def __init__(self, rnn_cell): - super(DynamicRNN, self).__init__(self) - self.cell = rnn_cell - - def call(self, input_data): - # [batch, time, features] -> [time, batch, features] - input_data = tf.transpose(input_data, [1, 0, 2]) - outputs = tf.TensorArray(tf.float32, input_data.shape[0]) - state = self.cell.zero_state(input_data.shape[1], dtype=tf.float32) - for i in tf.range(input_data.shape[0]): - output, state = self.cell(input_data[i], state) - outputs = outputs.write(i, output) - return tf.transpose(outputs.stack(), [1, 0, 2]), state -``` - -오토그래프의 특징에 관한 더 자세한 내용은 이 [가이드](./function.ipynb)를 참고하세요. - -### tf.metrics로 데이터를 수집하고 tf.summary로 기록하세요. - -서머리(summary) 로그를 기록하려면 `tf.summary.(scalar|histogram|...)`를 사용합니다. 컨텍스트 관리자(context manager)를 사용하는 파일 쓰기 객체에 전달해야 합니다. (컨텍스트 관리자를 사용하지 않으면 아무 일도 일어나지 않습니다.) TF 1.x과 달리 서머리는 바로 파일 쓰기 객체에 전달됩니다. 별도의 "머지(merge)" 연산이나 `add_summary()` 호출이 없습니다. 따라서 로그를 기록할 때 `step` 값이 함께 제공되어야 합니다. - -```python -summary_writer = tf.summary.create_file_writer('/tmp/summaries') -with summary_writer.as_default(): - tf.summary.scalar('loss', 0.1, step=42) -``` - -`summary`로 기록할 데이터를 수집하려면 `tf.metrics`를 사용하세요. 측정 정보들은 상태가 있습니다. 이 값들은 누적되어 `.result()`를 호출하면 누적된 결과가 반환됩니다. `.reset_stats()`를 사용하여 누적된 값을 초기화할 수 있습니다. - -```python -def train(model, optimizer, dataset, log_freq=10): - avg_loss = tf.keras.metrics.Mean(name='loss', dtype=tf.float32) - for images, labels in dataset: - loss = train_step(model, optimizer, images, labels) - avg_loss.update_state(loss) - if tf.equal(optimizer.iterations % log_freq, 0): - tf.summary.scalar('loss', avg_loss.result(), step=optimizer.iterations) - avg_loss.reset_states() - -def test(model, test_x, test_y, step_num): - loss = loss_fn(model(test_x), test_y) - tf.summary.scalar('loss', loss, step=step_num) - -train_summary_writer = tf.summary.create_file_writer('/tmp/summaries/train') -test_summary_writer = tf.summary.create_file_writer('/tmp/summaries/test') - -with train_summary_writer.as_default(): - train(model, optimizer, dataset) - -with test_summary_writer.as_default(): - test(model, test_x, test_y, optimizer.iterations) -``` - -텐서보드(TensorBoard)에 로그 디렉토리를 지정하여 생성된 서머리 로그를 시각화해 보세요: `tensorboard --logdir /tmp/summaries` diff --git a/site/ko/guide/function.ipynb b/site/ko/guide/function.ipynb deleted file mode 100644 index 5a18e792fa5..00000000000 --- a/site/ko/guide/function.ipynb +++ /dev/null @@ -1,762 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "function.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "Jxv6goXm7oGF" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Jxv6goXm7oGF" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\");" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "llMNufAK7nfK", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\"); { display-mode: \"form\" }\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8Byow2J6LaPl" - }, - "source": [ - "# 텐서플로 2.0의 tf.function과 오토그래프 (AutoGraph)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kGXS3UWBBNoc" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org 에서 보기\n", - " \n", - " 구글 코랩(Google Colab)에서 실행하기\n", - " \n", - " 깃헙(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GQLxpmF2_AvM", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CydFK2CL7ZHA" - }, - "source": [ - "TF 2.0 버전은 즉시 실행 (eager execution)의 편리함과 TF 1.0의 성능을 합쳤습니다. 이러한 결합의 중심에는 `tf.function` 이 있는데, 이는 파이썬 문법의 일부를 이식 가능하고 높은 성능의 텐서플로 그래프 코드로 변환시켜줍니다. \n", - "\n", - "`tf.function`의 멋지고 새로운 특징은 오토그래프 (AutoGraph)입니다. 이는 자연스러운 파이썬 문법을 활용해서 그래프 코드를 작성할 수 있도록 돕습니다. 오토그래프로 사용할 수 있는 파이썬 특징들의 목록을 보려면 [오토그래프 지원 범위](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/LIMITATIONS.md)를 참고하세요. `tf.function`에 관한 더 자세한 내용을 확인하려면 RFC [TF 2.0: Functions, not Sessions](https://github.com/tensorflow/community/blob/master/rfcs/20180918-functions-not-sessions-20.md)을 참고하세요. 오토그래프에 대한 더 자세한 내용은 `tf.autograph`를 참고하세요.\n", - "\n", - "본 튜토리얼은 `tf.function`와 오토그래프의 기초적인 특징에 대해서 설명할 것입니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n4EKOpw9mObL" - }, - "source": [ - "## 설정\n", - "\n", - "텐서플로 2.0 프리뷰 나이틀리 (Preview Nightly) 버전을 임포트(import)하고, TF 2.0 모드를 설정합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "V9oECvVSI1Kj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "import numpy as np" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mT7meGqrZTz9", - "colab": {} - }, - "source": [ - "!pip install tensorflow==2.0.0-beta1\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "77AsVr1GGtBP" - }, - "source": [ - "## `tf.function` 데코레이터\n", - "\n", - "`tf.function`을 함수에 붙여줄 경우, 여전히 다른 일반 함수들처럼 사용할 수 있습니다. 하지만 그래프 내에서 컴파일 되었을 때는 더 빠르게 실행하고, GPU나 TPU를 사용해서 작동하고, 세이브드모델(SavedModel)로 내보내는 것이 가능해집니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FhIg7-z6HNWj", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def simple_nn_layer(x, y):\n", - " return tf.nn.relu(tf.matmul(x, y))\n", - "\n", - "\n", - "x = tf.random.uniform((3, 3))\n", - "y = tf.random.uniform((3, 3))\n", - "\n", - "simple_nn_layer(x, y)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U-LAE4pMNR9g" - }, - "source": [ - "데코레이터를 붙인 결과를 확인해보면, 텐서플로 런타임시의 모든 상호작용들을 다룰 수 있다는 것을 알 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q4t2iuS7Nqc0", - "colab": {} - }, - "source": [ - "simple_nn_layer" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DqeefLGNXjZQ" - }, - "source": [ - "만일 여러분의 코드가 여러 함수들을 포함하고 있다면, 그것들에 모두 데코레이터를 붙일 필요는 없습니다. 데코레이터가 붙은 함수로부터 호출된 모든 함수들은 그래프 모드에서 동작합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3VGF7tlVXiZY", - "colab": {} - }, - "source": [ - "def linear_layer(x):\n", - " return 2 * x + 1\n", - "\n", - "\n", - "@tf.function\n", - "def deep_net(x):\n", - " return tf.nn.relu(linear_layer(x))\n", - "\n", - "\n", - "deep_net(tf.constant((1, 2, 3)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yQvg6ZSKWyqE" - }, - "source": [ - "작은 연산들을 많이 포함한 그래프의 경우 함수들은 즉시 실행 코드 (eager code) 보다 더 빠르게 동작합니다. 하지만 무거운 연산들을 조금 포함한 그래프의 경우 (컨볼루션 등), 그렇게 빠른 속도 향상은 기대하기 어렵습니다.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0EL6lVwEWuFo", - "colab": {} - }, - "source": [ - "import timeit\n", - "conv_layer = tf.keras.layers.Conv2D(100, 3)\n", - "\n", - "@tf.function\n", - "def conv_fn(image):\n", - " return conv_layer(image)\n", - "\n", - "image = tf.zeros([1, 200, 200, 100])\n", - "# 데이터 준비 (warm up)\n", - "conv_layer(image); conv_fn(image)\n", - "print(\"컨볼루션 즉시 실행:\", timeit.timeit(lambda: conv_layer(image), number=10))\n", - "print(\"컨볼루션 함수:\", timeit.timeit(lambda: conv_fn(image), number=10))\n", - "print(\"컨볼루션의 성능에는 큰 차이가 없다는 것을 확인할 수 있습니다\")\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "L4zj-jpH0jKH", - "colab": {} - }, - "source": [ - "lstm_cell = tf.keras.layers.LSTMCell(10)\n", - "\n", - "@tf.function\n", - "def lstm_fn(input, state):\n", - " return lstm_cell(input, state)\n", - "\n", - "input = tf.zeros([10, 10])\n", - "state = [tf.zeros([10, 10])] * 2\n", - "# 데이터 준비 (warm up)\n", - "lstm_cell(input, state); lstm_fn(input, state)\n", - "print(\"lstm 즉시 실행:\", timeit.timeit(lambda: lstm_cell(input, state), number=10))\n", - "print(\"lstm 함수:\", timeit.timeit(lambda: lstm_fn(input, state), number=10))\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ohbSnA79mcJV" - }, - "source": [ - "## 파이썬의 제어 흐름 사용하기\n", - "\n", - "`tf.function` 내에서 데이터 기반 제어 흐름을 사용할 때, 파이썬의 제어 흐름 문을 사용할 수 있고, 오토그래프 기능은 그것들을 모두 적절한 텐서플로 연산으로 변환할 수 있습니다. 예를 들어, `if` 문은 `Tensor`를 기반으로 작동해야할 때 `tf.cond()` 로 변환될 수 있습니다. \n", - "\n", - "아래 예시에서, `x`는 `Tensor`이지만 `if`문이 예상한대로 정상 작동합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aA3gOodCBkOw", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def square_if_positive(x):\n", - " if x > 0:\n", - " x = x * x\n", - " else:\n", - " x = 0\n", - " return x\n", - "\n", - "\n", - "print('square_if_positive(2) = {}'.format(square_if_positive(tf.constant(2))))\n", - "print('square_if_positive(-2) = {}'.format(square_if_positive(tf.constant(-2))))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GMiCUkdyoq98" - }, - "source": [ - "Note: 위의 예시는 스칼라값으로 간단한 조건절을 사용하였습니다. 하지만 실제 코드에서는 배치(Batching) 가 주로 사용됩니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m-jWmsCmByyw" - }, - "source": [ - "오토그래프는 기본적인 파이썬 문인 `while`, `for`, `if`, `break`, `continue`, `return`과 네스팅(nesting)을 지원합니다. 이는 `Tensor` 표현을 `while`과 `if` 문의 조건 부분에서 사용하거나 `for` 루프에서 `Tensor`를 반복할 수 있다는 것을 의미합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "toxKBOXbB1ro", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def sum_even(items):\n", - " s = 0\n", - " for c in items:\n", - " if c % 2 > 0:\n", - " continue\n", - " s += c\n", - " return s\n", - "\n", - "\n", - "sum_even(tf.constant([10, 12, 15, 20]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AtDaLrbySw4j" - }, - "source": [ - "또한 오토그래프는 고급 사용자를 위해 낮은 수준의 API를 제공합니다. 예를 들어, 여러분은 생성된 코드를 확인하기 위해 다음과 같이 작성할 수 있습니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aRsde3x_SjTQ", - "colab": {} - }, - "source": [ - "print(tf.autograph.to_code(sum_even.python_function))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rvJXCfk8VkLf" - }, - "source": [ - "다음은 더 복잡한 제어 흐름의 예시입니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h-Z87IJqVlKl", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def fizzbuzz(n):\n", - " msg = tf.constant('')\n", - " for i in tf.range(n):\n", - " if tf.equal(i % 3, 0):\n", - " tf.print('Fizz')\n", - " elif tf.equal(i % 5, 0):\n", - " tf.print('Buzz')\n", - " else:\n", - " tf.print(i)\n", - "\n", - "fizzbuzz(tf.constant(15))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "h_Y4uC1R1B55" - }, - "source": [ - "## 케라스와 오토그래프\n", - "\n", - "오토그래프는 기본적으로 비동적(non-dynamic) 케라스 모델에서 사용 가능합니다. 더 자세한 정보를 원한다면, `tf.keras`를 참고하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cR6mpLKP1HLe", - "colab": {} - }, - "source": [ - "class CustomModel(tf.keras.models.Model):\n", - "\n", - " @tf.function\n", - " def call(self, input_data):\n", - " if tf.reduce_mean(input_data) > 0:\n", - " return input_data\n", - " else:\n", - " return input_data // 2\n", - "\n", - "\n", - "model = CustomModel()\n", - "\n", - "model(tf.constant([-2, -4]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NTEvpBK9f8kj" - }, - "source": [ - "## 부수 효과 (Side effects)\n", - "\n", - "즉시 실행 모드 (eager mode)처럼 부수 효과를 사용할 수 있습니다. 예를 들면, `tf.function` 내에 있는 `tf.assign`이나 `tf.print`이 있습니다. 또한 부수 효과들은 작업들이 순서대로 실행된다는 것을 보장하기 위해 필수적인 제어 의존성 (control dependency)을 추가합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-Wd6i8S9gcuC", - "colab": {} - }, - "source": [ - "v = tf.Variable(5)\n", - "\n", - "@tf.function\n", - "def find_next_odd():\n", - " v.assign(v + 1)\n", - " if tf.equal(v % 2, 0):\n", - " v.assign(v + 1)\n", - "\n", - "\n", - "find_next_odd()\n", - "v" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4LfnJjm0Bm0B" - }, - "source": [ - "## 디버깅\n", - "\n", - "`tf.function` 과 오토그래프는 코드를 생성하고 텐서플로 그래프 내에서 해당 코드를 추적함으로써 동작합니다. 이 메커니즘은 아직까지는 `pdb`같은 단계적 (step-by-step) 디버거를 지원하지 않습니다. 하지만 일시적으로 `tf.function` 내에서 즉시 실행 (eager execution)을 가능하게 하는 `tf.config.run_functions_eagerly(True)`을 사용하고 가장 선호하는 디버거를 사용할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Yci8ve6hmgpF", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def f(x):\n", - " if x > 0:\n", - " # 여기에 중단점(breakpoint)을 설정해 보세요!\n", - " # 예시:\n", - " # import pdb\n", - " # pdb.set_trace()\n", - " x = x + 1\n", - " return x\n", - "\n", - "tf.config.experimental_run_functions_eagerly(True)\n", - "\n", - "# 이제 중단점을 설정하고 디버거 내에서 코드를 실행할 수 있습니다.\n", - "f(tf.constant(1))\n", - "\n", - "tf.config.experimental_run_functions_eagerly(False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Em5dzSUOtLRP" - }, - "source": [ - "### 데이터 다운로드" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xqoxumv0ssQW", - "colab": {} - }, - "source": [ - "def prepare_mnist_features_and_labels(x, y):\n", - " x = tf.cast(x, tf.float32) / 255.0\n", - " y = tf.cast(y, tf.int64)\n", - " return x, y\n", - "\n", - "def mnist_dataset():\n", - " (x, y), _ = tf.keras.datasets.mnist.load_data()\n", - " ds = tf.data.Dataset.from_tensor_slices((x, y))\n", - " ds = ds.map(prepare_mnist_features_and_labels)\n", - " ds = ds.take(20000).shuffle(20000).batch(100)\n", - " return ds\n", - "\n", - "train_dataset = mnist_dataset()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "znmy4l8ntMvW" - }, - "source": [ - "### 모델 정의하기" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ltxyJVWTqNAO", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential((\n", - " tf.keras.layers.Reshape(target_shape=(28 * 28,), input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(100, activation='relu'),\n", - " tf.keras.layers.Dense(10)))\n", - "model.build()\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oeYV6mKnJGMr" - }, - "source": [ - "### 훈련 (training) 루프 정의하기" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3xtg_MMhJETd", - "colab": {} - }, - "source": [ - "compute_loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", - "\n", - "compute_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n", - "\n", - "\n", - "def train_one_step(model, optimizer, x, y):\n", - " with tf.GradientTape() as tape:\n", - " logits = model(x)\n", - " loss = compute_loss(y, logits)\n", - "\n", - " grads = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " compute_accuracy(y, logits)\n", - " return loss\n", - "\n", - "\n", - "@tf.function\n", - "def train(model, optimizer):\n", - " train_ds = mnist_dataset()\n", - " step = 0\n", - " loss = 0.0\n", - " accuracy = 0.0\n", - " for x, y in train_ds:\n", - " step += 1\n", - " loss = train_one_step(model, optimizer, x, y)\n", - " if tf.equal(step % 10, 0):\n", - " tf.print('스텝', step, ': 손실', loss, '; 정확도', compute_accuracy.result())\n", - " return step, loss, accuracy\n", - "\n", - "step, loss, accuracy = train(model, optimizer)\n", - "print('최종 스텝', step, ': 손실', loss, '; 정확도', compute_accuracy.result())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SnsumiP6eRYL" - }, - "source": [ - "## 배치 (Batching)\n", - "\n", - "실제 적용시에 배치 (batch) 는 성능을 위해 필수적입니다. 오토그래프로 변환하기 가장 좋은 코드는 제어 흐름이 _배치_ 수준에서 결정되는 코드입니다. 만일 제어 흐름이 개별적인 _예제 (example)_ 수준에서 결정된다면, 성능을 유지하기 위해서 배치 API들을 사용해야합니다.\n", - "\n", - "예를 들어, 파이썬으로 다음과 같은 코드를 작성했다면:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t31QoERiNccJ", - "colab": {} - }, - "source": [ - "def square_if_positive(x):\n", - " return [i ** 2 if i > 0 else i for i in x]\n", - "\n", - "\n", - "square_if_positive(range(-5, 5))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kSeEJ76uNgwD" - }, - "source": [ - "텐서플로에서는 다음과 같이 작성하고 싶을 것입니다. (그리고 다음 코드는 실제로 동작합니다!):\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RqR8WzSzNf87", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def square_if_positive_naive(x):\n", - " result = tf.TensorArray(tf.int32, size=x.shape[0])\n", - " for i in tf.range(x.shape[0]):\n", - " if x[i] > 0:\n", - " result = result.write(i, x[i] ** 2)\n", - " else:\n", - " result = result.write(i, x[i])\n", - " return result.stack()\n", - "\n", - "\n", - "square_if_positive_naive(tf.range(-5, 5))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gTcyWXVGN3gS" - }, - "source": [ - "하지만 이 경우는 아래와 같이 작성할 수도 있습니다:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VO2f6x-lNfVj", - "colab": {} - }, - "source": [ - "def square_if_positive_vectorized(x):\n", - " return tf.where(x > 0, x ** 2, x)\n", - "\n", - "\n", - "square_if_positive_vectorized(tf.range(-5, 5))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ko/guide/gpu.ipynb b/site/ko/guide/gpu.ipynb deleted file mode 100644 index b86d8b1bcf0..00000000000 --- a/site/ko/guide/gpu.ipynb +++ /dev/null @@ -1,485 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "tuOe1ymfHZPu" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# GPU 사용하기\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/gpu\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eTensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/guide/gpu.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/guide/gpu.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/gpu.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUXex9ctTuDB" - }, - "source": [ - "## 설정\n", - "\n", - "최신 버전의 텐서플로가 설치되어있는지 확인하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IqR2PQG4ZaZ0" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UhNtHfuxCGVy" - }, - "source": [ - "## 장치 할당 로깅\n", - "\n", - "연산과 텐서가 어떤 장치에 할당되었는지 확인하려면 `tf.debugging.set_log_device_placement(True)`를 프로그램의 가장 처음에 선언하세요. 장치 할당 로깅을 활성화하면 모든 텐서나 연산 할당이 출력됩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2Dbw0tpEirCd" - }, - "outputs": [], - "source": [ - "tf.debugging.set_log_device_placement(True)\n", - "\n", - "# 텐서 생성\n", - "a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n", - "b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n", - "c = tf.matmul(a, b)\n", - "\n", - "print(c)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKhmFeraTdEI" - }, - "source": [ - "위 코드는 `MatMul` 연산이 `GPU:0`에서 수행되었다고 보여줄 것입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U88FspwGjB7W" - }, - "source": [ - "## 장치 수동 할당\n", - "\n", - "특정 연산을 수행할 장치를 직접 선택하고 싶다면, `with tf.device`로 장치 컨텍스트를 생성할 수 있고 해당 컨텍스트에서의 모든 연산은 지정된 장치에서 수행됩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8wqaQfEhjHit" - }, - "outputs": [], - "source": [ - "tf.debugging.set_log_device_placement(True)\n", - "\n", - "# 텐서를 CPU에 할당\n", - "with tf.device('/CPU:0'):\n", - " a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n", - " b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n", - "\n", - "c = tf.matmul(a, b)\n", - "print(c)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8ixO89gRjJUu" - }, - "source": [ - "`a`와 `b`가 `CPU:0`에 할당되었습니다. `MatMul` 연산은 수행할 장치가 명시적으로 할당되어 있지 않기 때문에 텐서플로 런타임(runtime)은 연산과 가용한 장치들(이 예제에서는 `GPU:0`)을 기반으로 하나를 고를 것이고 필요하다면 장치들간에 텐서를 자동으로 복사할 것입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ARrRhwqijPzN" - }, - "source": [ - "## GPU 메모리 제한하기\n", - "\n", - "기본적으로 텐서플로는 모든 GPU의 거의 모든 메모리를 프로세스가 볼 수 있도록 매핑합니다([`CUDA_VISIBLE_DEVICES`](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars)에 포함되었다고 가정합니다). 이는 메모리 단편화를 줄여서 상대적으로 귀한 GPU 메모리 리소스를 장치에서 보다 효율적으로 사용할 수 있게 합니다. `tf.config.experimental.set_visible_devices` 메서드를 사용하여 텐서플로에서 접근할 수 있는 GPU를 조정할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hPI--n_jhZhv" - }, - "outputs": [], - "source": [ - "gpus = tf.config.experimental.list_physical_devices('GPU')\n", - "if gpus:\n", - " # 텐서플로가 첫 번째 GPU만 사용하도록 제한\n", - " try:\n", - " tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n", - " except RuntimeError as e:\n", - " # 프로그램 시작시에 접근 가능한 장치가 설정되어야만 합니다\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N3x4M55DhYk9" - }, - "source": [ - "어떤 경우에는 프로세스가 가용한 메모리의 일부에만 할당되도록 하거나 프로세스의 요구량만큼 메모리 사용이 가능할 필요가 있습니다. 텐서플로에서는 이를 위해 두 가지 방법을 제공합니다.\n", - "\n", - "첫 번째 방법은 `tf.config.experimental.set_memory_growth`를 호출하여 메모리 증가를 허용하는 것입니다. 이는 런타임에서 할당하는데 필요한 양만큼의 GPU 메모리를 할당합니다: 처음에는 메모리를 조금만 할당하고, 프로그램이 실행되어 더 많은 GPU 메모리가 필요하면, 텐서플로 프로세스에 할당된 GPU 메모리 영역을 확장합니다. 메모리 해제는 메모리 단편화를 악화시키므로 메모리 해제는 하지 않습니다. 특정 GPU의 메모리 증가를 허용하려면 다음 코드를 텐서나 연산 앞에 입력하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jr3Kf1boFnCO" - }, - "outputs": [], - "source": [ - "gpus = tf.config.experimental.list_physical_devices('GPU')\n", - "if gpus:\n", - " try:\n", - " tf.config.experimental.set_memory_growth(gpus[0], True)\n", - " except RuntimeError as e:\n", - " # 프로그램 시작시에 메모리 증가가 설정되어야만 합니다\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "I1o8t51QFnmv" - }, - "source": [ - "또 다른 방법은 `TF_FORCE_GPU_ALLOW_GROWTH` 환경변수를 `true`로 설정하는 것입니다. 이 설정은 플랫폼 종속적입니다.\n", - "\n", - "두 번째 방법은 `tf.config.experimental.set_virtual_device_configuration`으로 가상 GPU 장치를 설정하고 GPU에 할당될 전체 메모리를 제한하는 것입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2qO2cS9QFn42" - }, - "outputs": [], - "source": [ - "gpus = tf.config.experimental.list_physical_devices('GPU')\n", - "if gpus:\n", - " # 텐서플로가 첫 번째 GPU에 1GB 메모리만 할당하도록 제한\n", - " try:\n", - " tf.config.experimental.set_virtual_device_configuration(\n", - " gpus[0],\n", - " [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024)])\n", - " except RuntimeError as e:\n", - " # 프로그램 시작시에 가장 장치가 설정되어야만 합니다\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bsg1iLuHFoLW" - }, - "source": [ - "이는 텐서플로 프로세스에서 사용가능한 GPU 메모리량을 제한하는데 유용합니다. 워크스테이션 GUI같이 GPU가 다른 어플리케이션들에 공유되는 로컬 개발환경에서 보통 사용되는 방법입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B27_-1gyjf-t" - }, - "source": [ - "## 멀티 GPU 시스템에서 하나의 GPU만 사용하기\n", - "\n", - "시스템에 두 개 이상의 GPU가 있다면 낮은 ID의 GPU가 기본으로 선택됩니다. 다른 GPU에서 실행하고 싶으면 명시적으로 표시해야 합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wep4iteljjG1" - }, - "outputs": [], - "source": [ - "tf.debugging.set_log_device_placement(True)\n", - "\n", - "try:\n", - " # 유효하지 않은 GPU 장치를 명시\n", - " with tf.device('/device:GPU:2'):\n", - " a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n", - " b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n", - " c = tf.matmul(a, b)\n", - "except RuntimeError as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jy-4cCO_jn4G" - }, - "source": [ - "명시한 장치가 존재하지 않으면 `RuntimeError`가 나옵니다:\n", - "\n", - "명시한 장치가 존재하지 않을 때 텐서플로가 자동으로 현재 지원하는 장치를 선택하게 하려면 `tf.config.set_soft_device_placement(True)`를 호출하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sut_UHlkjvWd" - }, - "outputs": [], - "source": [ - "tf.config.set_soft_device_placement(True)\n", - "tf.debugging.set_log_device_placement(True)\n", - "\n", - "# 텐서 생성\n", - "a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n", - "b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n", - "c = tf.matmul(a, b)\n", - "\n", - "print(c)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sYTYPrQZj2d9" - }, - "source": [ - "## 멀티 GPU 사용하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDZmEGq4j6kG" - }, - "source": [ - "#### `tf.distribute.Strategy` 사용\n", - "\n", - "멀티 GPU를 사용하는 가장 좋은 방법은 `tf.distriute.Strategy`를 사용하는 것입니다. 간단한 예제를 살펴봅시다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1KgzY8V2AvRv" - }, - "outputs": [], - "source": [ - "strategy = tf.distribute.MirroredStrategy()\n", - "\n", - "with strategy.scope():\n", - " inputs = tf.keras.layers.Input(shape=(1,))\n", - " predictions = tf.keras.layers.Dense(1)(inputs)\n", - " model = tf.keras.models.Model(inputs=inputs, outputs=predictions)\n", - " model.compile(loss='mse',\n", - " optimizer=tf.keras.optimizers.SGD(learning_rate=0.2))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dy7nxlKsAxkK" - }, - "source": [ - "이 프로그램은 입력 데이터를 나누고 모델의 복사본을 각 GPU에서 실행할 것입니다. 이는 \"[데이터 병렬처리](https://en.wikipedia.org/wiki/Data_parallelism)\"라고도 합니다.\n", - "\n", - "병렬화 전략에 대해 더 알고 싶으시면 [가이드](./distributed_training.ipynb)를 참조하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8phxM5TVkAY_" - }, - "source": [ - "#### `tf.distribute.Strategy` 미사용\n", - "\n", - "`tf.distribute.Strategy`는 여러 장치에 걸쳐 계산을 복제해서 동작합니다. 모델을 각 GPU에 구성하여 수동으로 이를 구현할 수 있습니다. 예를 들면:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AqPo9ltUA_EY" - }, - "outputs": [], - "source": [ - "tf.debugging.set_log_device_placement(True)\n", - "\n", - "gpus = tf.config.experimental.list_logical_devices('GPU')\n", - "if gpus:\n", - " # 여러 GPU에 계산을 복제\n", - " c = []\n", - " for gpu in gpus:\n", - " with tf.device(gpu.name):\n", - " a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n", - " b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n", - " c.append(tf.matmul(a, b))\n", - "\n", - " with tf.device('/CPU:0'):\n", - " matmul_sum = tf.add_n(c)\n", - "\n", - " print(matmul_sum)" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "using_tpu.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/guide/keras/index.md b/site/ko/guide/keras/index.md deleted file mode 100644 index 9f36d880b65..00000000000 --- a/site/ko/guide/keras/index.md +++ /dev/null @@ -1,26 +0,0 @@ -# 케라스(Keras) - -Note: 이 문서들은 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 -[공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 -있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 -바랍니다. 문서 번역이나 리뷰에 참여하려면 -[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 -메일을 보내주시기 바랍니다. - -`tf.keras`는 텐서플로의 딥러닝 모델 설계와 훈련을 위한 고수준(high-level) API입니다. 이는 빠른 프로토타이핑, 최첨단 기술의 연구 및 생산에 사용되며, 세 가지 주요 이점이 있습니다: - -- *사용자 친화적*
    케라스는 일반 사용 사례에 최적화된 간단하고 일관적인 인터페이스를 제공합니다. 이는 사용자 오류에 대해 명확하고 실용적인 피드백을 제공합니다. -- *모듈화 및 구성 가능성*
    케라스 모델은 구성 요소의 설정에 의해 연결되는 식으로 거의 제한없이 만들 수 있습니다. -- *쉬운 확장*
    연구를 위한 새로운 아이디어를 표현하기 위해 사용자 정의 설계 블록을 작성하세요. 새로운 층(layers), 지표(metrics), 손실 함수를 생성하고 최첨단 모델을 개발하세요. - -[케라스: 빠른 개요](./overview.ipynb) 가이드는 시작하는 데 도움이 될 것입니다. - -`tf.keras'를 통한 머신 러닝에 입문하는 분들을 위한 안내는 [입문 튜토리얼 세트](https://www.tensorflow.org/tutorials/keras)를 보세요. - -API에 대해 더 자세히 알아보고 싶다면, 텐서플로 케라스의 고급 사용자가 알아야 할 사항을 다루는 다음 가이드 세트를 참조하세요: - -- [케라스의 함수형 API 가이드](./functional.ipynb) -- [훈련 및 평가 가이드](./train_and_evaluate.ipynb) -- [서브 클래싱(subclassing)을 사용한 기초적인 층(layer)과 모델 작성 가이드](./custom_layers_and_models.ipynb) -- [모델 저장 및 직렬화 가이드](./save_and_serialize.ipynb) -- [사용자 정의 콜백 작성 가이드](./custom_callback.ipynb) diff --git a/site/ko/guide/keras/overview.ipynb b/site/ko/guide/keras/overview.ipynb deleted file mode 100644 index 110338c3ceb..00000000000 --- a/site/ko/guide/keras/overview.ipynb +++ /dev/null @@ -1,1174 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCQY7jpBfMur" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "z6X9omPnfO_h" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F1xIRPtY0E1w" - }, - "source": [ - "# 케라스: 빠르게 훑어보기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VyOjQZHhZxaA" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IsK5aF2xZ-40" - }, - "source": [ - "## tf.keras 임포트\n", - "\n", - "`tf.keras`는 [케라스 API 명세](https://keras.io){:.external}의 텐서플로 구현입니다. `tf.keras`는 머신러닝 모델을 만들고 훈련하기 위한 고수준 API로서 텐서플로의 특수 기능을 모두 지원합니다. 여기에는 [즉시 실행](#즉시-실행), `tf.data` 파이프라인(pipeline), [Estimators](../estimators.md)가 포함됩니다. `tf.keras`를 이용하면 유연성과 성능을 손해보지 않고 텐서플로를 쉽게 사용할 수 있습니다.\n", - "\n", - "`tf.keras`를 임포트하여 텐서플로 프로그램을 시작합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Qoc055N3wiUG" - }, - "outputs": [], - "source": [ - "!pip install pyyaml # pyyaml은 선택사항입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TgPcBFru0E1z" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import keras" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lj03RamP0E13" - }, - "source": [ - "`tf.keras`는 케라스 API와 호환되는 어떤 코드라도 실행시킬 수 있지만 다음 사항을 유념하세요:\n", - "\n", - "* 최신 텐서플로 릴리스에 포함된 `tf.keras` 버전은 PyPI에 있는 최신 `keras` 버전과 같지 않을 수 있습니다. `tf.keras.__version__`을 확인해 보세요.\n", - "* [모델의 가중치를 저장](./save_and_serialize.ipynb)할 때 `tf.keras`는 기본적으로 [체크포인트 포맷](../checkpoints.md)을 사용합니다. HDF5를 사용하려면 `save_format='h5'`로 설정하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7e1LPcXx0gR6" - }, - "source": [ - "## 간단한 모델 만들기\n", - "\n", - "### Sequential 모델\n", - "\n", - "케라스에서는 *층(layer)*을 조합하여 *모델(model)*을 만듭니다. 모델은 (일반적으로) 층의 그래프입니다. 가장 흔한 모델 구조는 층을 차례대로 쌓은 `tf.keras.Sequential` 모델입니다.\n", - "\n", - "간단한 완전 연결(fully-connected) 네트워크(즉, 다층 퍼셉트론(multi-layer perceptron))를 만들어 보겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WM-DUVQB0E14" - }, - "outputs": [], - "source": [ - "from tensorflow.keras import layers\n", - "\n", - "model = tf.keras.Sequential()\n", - "# 64개의 유닛을 가진 완전 연결 층을 모델에 추가합니다:\n", - "model.add(layers.Dense(64, activation='relu'))\n", - "# 또 하나를 추가합니다:\n", - "model.add(layers.Dense(64, activation='relu'))\n", - "# 10개의 출력 유닛을 가진 소프트맥스 층을 추가합니다:\n", - "model.add(layers.Dense(10, activation='softmax'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "[여기](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb)에서 `Sequential` 모델을 어떻게 사용하는지 간단하지만 완전한 예제를 볼 수 있습니다.\n", - "\n", - "`Sequential`보다 더 고수준의 모델을 구성하는 방법을 배우려면 다음을 참고하세요:\n", - "- [케라스 함수형 API 가이드](./functional.ipynb)\n", - "- [클래스 상속을 통하여 층과 모델을 밑바닥부터 만드는 방법](./custom_layers_and_models.ipynb)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ztyTipu0E18" - }, - "source": [ - "### 층 설정\n", - "\n", - "`tf.keras.layers` 아래의 클래스들은 일부 생성자 매개변수를 공통으로 가지고 있습니다:\n", - "\n", - "* `activation`: 층의 활성화 함수를 설정합니다. 이 매개변수에는 기본으로 제공되는 함수의 이름을 쓰거나\n", - " 호출 가능한 객체를 지정할 수 있습니다. 기본값은 활성화 함수를 적용하지 않는 것입니다.\n", - "* `kernel_initializer`와 `bias_initializer`: 층의 가중치(weight)(커널(kernel)과 절편(bias))를 초기화하는 방법입니다. 내장 함수나 호출 가능한 객체를 지정합니다. 기본값은 `\"glorot_uniform\"` 초기화입니다.\n", - "* `kernel_regularizer`와 `bias_regularizer`: L1 또는 L2 규제(regularization)와 같이 층의 가중치(커널과 절편)에 적용할 규제 방법을 지정합니다. 기본값은 규제를 적용하지 않는 것입니다.\n", - "\n", - "다음 코드는 여러가지 생성자 매개변수를 사용하여 `tf.keras.layers.Dense` 층의 객체를 만드는 예입니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sJ4AOn090E2A" - }, - "outputs": [], - "source": [ - "# 시그모이드 활성화 층을 만듭니다:\n", - "layers.Dense(64, activation='sigmoid')\n", - "# 또는 다음도 가능합니다:\n", - "layers.Dense(64, activation=tf.keras.activations.sigmoid)\n", - "\n", - "# 커널 행렬에 L1 규제가 적용된 선형 활성화 층. 하이퍼파라미터 0.01은 규제의 양을 조절합니다:\n", - "layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l1(0.01))\n", - "\n", - "# 절편 벡터에 L2 규제가 적용된 선형 활성화 층. 하이퍼파라미터 0.01은 규제의 양을 조절합니다:\n", - "layers.Dense(64, bias_regularizer=tf.keras.regularizers.l2(0.01))\n", - "\n", - "# 커널을 랜덤한 직교 행렬로 초기화한 선형 활성화 층:\n", - "layers.Dense(64, kernel_initializer='orthogonal')\n", - "\n", - "# 절편 벡터를 상수 2.0으로 설정한 선형 활성화 층:\n", - "layers.Dense(64, bias_initializer=tf.keras.initializers.Constant(2.0))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9NR6reyk0E2A" - }, - "source": [ - "## 훈련과 평가\n", - "\n", - "### 훈련 준비\n", - "\n", - "모델을 구성한 후 `compile` 메서드를 호출하여 학습 과정을 설정합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sJ4AOn090E2A" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - "# 64개의 유닛을 가진 완전 연결 층을 모델에 추가합니다:\n", - "layers.Dense(64, activation='relu', input_shape=(32,)),\n", - "# 또 하나를 추가합니다:\n", - "layers.Dense(64, activation='relu'),\n", - "# 10개의 출력 유닛을 가진 소프트맥스 층을 추가합니다:\n", - "layers.Dense(10, activation='softmax')])\n", - "\n", - "model.compile(optimizer=tf.keras.optimizers.Adam(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HG-RAa9F0E2D" - }, - "source": [ - "`tf.keras.Model.compile`에는 세 개의 중요한 매개변수가 있습니다:\n", - "\n", - "* `optimizer`: 훈련 과정을 설정합니다. `tf.keras.optimizers.Adam`이나\n", - " `tf.keras.optimizers.SGD`와 같은 `tf.keras.optimizers` 아래의 옵티마이저 객체를 전달합니다. 기본 매개변수를 사용할 경우 `'adam'`이나 `'sgd'`와 같이 문자열로 지정할 수도 있습니다.\n", - "* `loss`: 최적화 과정에서 최소화될 손실 함수(loss function)를 설정합니다. 평균 제곱 오차(`mse`)와 `categorical_crossentropy`, `binary_crossentropy` 등이 자주 사용됩니다. 손실 함수의 이름을 지정하거나 `tf.keras.losses` 모듈 아래의 호출 가능한 객체를 전달할 수 있습니다.\n", - "* `metrics`: 훈련을 모니터링하기 위해 사용됩니다. 이름이나 `tf.keras.metrics` 모듈 아래의 호출 가능한 객체입니다.\n", - "* 추가적으로 모델의 훈련과 평가를 즉시 실행하려면 `run_eagerly=True` 매개변수를 전달할 수 있습니다.\n", - "\n", - "다음 코드는 모델 훈련을 설정하는 몇 가지 예를 보여줍니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "St4Mgdar0E2E" - }, - "outputs": [], - "source": [ - "# 평균 제곱 오차로 회귀 모델을 설정합니다.\n", - "model.compile(optimizer=tf.keras.optimizers.Adam(0.01),\n", - " loss='mse', # 평균 제곱 오차\n", - " metrics=['mae']) # 평균 절댓값 오차\n", - "\n", - "# 크로스엔트로피 손실 함수로 분류 모델을 설정합니다.\n", - "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.01),\n", - " loss=tf.keras.losses.CategoricalCrossentropy(),\n", - " metrics=[tf.keras.metrics.CategoricalAccuracy()])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yjI5rbi80E2G" - }, - "source": [ - "### 넘파이 데이터를 사용한 훈련\n", - "\n", - "데이터셋이 작은 경우 [넘파이](https://www.numpy.org/){:.external}(NumPy) 배열을 메모리에 적재하여 모델을 훈련하고 평가합니다. 모델은 `fit` 메서드를 통해서 훈련 데이터를 학습합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3CvP6L-m0E2I" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "data = np.random.random((1000, 32))\n", - "labels = np.random.random((1000, 10))\n", - "\n", - "model.fit(data, labels, epochs=10, batch_size=32)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N-pnVaFe0E2N" - }, - "source": [ - "`tf.keras.Model.fit`에는 세 개의 중요한 매개변수가 있습니다:\n", - "\n", - "* `epochs`: 훈련은 *에포크*(epoch)로 구성됩니다. 한 에포크는 전체 입력 데이터를 한번 순회하는 것입니다(작은 배치로 나누어 수행됩니다).\n", - "* `batch_size`: 넘파이 데이터를 전달하면 모델은 데이터를 작은 배치로 나누고 훈련 과정에서 이 배치를 순회합니다. 이 정수 값은 배치의 크기를 지정합니다. 전체 샘플 개수가 배치 크기로 나누어 떨어지지 않으면 마지막 배치의 크기는 더 작을 수 있습니다.\n", - "* `validation_data`: 모델의 프로토타입(prototype)을 만들 때는 검증 데이터(validation data)에서 간편하게 성능을 모니터링해야 합니다. 입력과 레이블(label)의 튜플을 이 매개변수로 전달하면 에포크가 끝날 때마다 추론 모드(inference mode)에서 전달된 데이터의 손실과 측정 지표를 출력합니다.\n", - "\n", - "다음이 `validation_data`를 사용하는 예입니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gFcXzVQa0E2N" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "data = np.random.random((1000, 32))\n", - "labels = np.random.random((1000, 10))\n", - "\n", - "val_data = np.random.random((100, 32))\n", - "val_labels = np.random.random((100, 10))\n", - "\n", - "model.fit(data, labels, epochs=10, batch_size=32,\n", - " validation_data=(val_data, val_labels))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-6ImyXzz0E2Q" - }, - "source": [ - "### tf.data 데이터셋을 사용한 훈련\n", - "\n", - "[데이터셋 API](../datasets.md)를 사용하여 대규모 데이터셋이나 복수의 장치로 확장시킬 수 있습니다. `fit` 메서드에 `tf.data.Dataset` 객체를 전달합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OziqhpIj0E2R" - }, - "outputs": [], - "source": [ - "# 예제 `Dataset` 객체를 만듭니다:\n", - "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n", - "dataset = dataset.batch(32)\n", - "\n", - "# Dataset에서 `fit` 메서드를 호출할 때 `steps_per_epoch` 설정을 잊지 마세요.\n", - "model.fit(dataset, epochs=10, steps_per_epoch=30)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "I7BcMHkB0E2U" - }, - "source": [ - "여기에서 `fit` 메서드는 `steps_per_epoch` 매개변수를 사용합니다. 다음 에포크로 넘어가기 전에 모델이 수행할 훈련 단계 횟수입니다. `Dataset`이 배치 데이터를 생성하기 때문에 `batch_size`가 필요하지 않습니다.\n", - "\n", - "`Dataset`은 검증 데이터에도 사용할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YPMb3A0N0E2V" - }, - "outputs": [], - "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n", - "dataset = dataset.batch(32)\n", - "\n", - "val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_labels))\n", - "val_dataset = val_dataset.batch(32)\n", - "\n", - "model.fit(dataset, epochs=10,\n", - " validation_data=val_dataset)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IgGdlXso0E2X" - }, - "source": [ - "### 평가와 예측\n", - "\n", - "`tf.keras.Model.evaluate`와 `tf.keras.Model.predict` 메서드에는 넘파이 배열이나 `tf.data.Dataset`을 사용할 수 있습니다.\n", - "\n", - "주어진 데이터로 추론 모드의 손실이나 지표를 *평가*합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mhDbOHEK0E2Y" - }, - "outputs": [], - "source": [ - "data = np.random.random((1000, 32))\n", - "labels = np.random.random((1000, 10))\n", - "\n", - "model.evaluate(data, labels, batch_size=32)\n", - "\n", - "model.evaluate(dataset, steps=30)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UXUTmDfb0E2b" - }, - "source": [ - "주어진 데이터로 추론 모드에서 마지막 층의 출력을 *예측*하여 넘파이 배열로 반환합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9e3JsSoQ0E2c" - }, - "outputs": [], - "source": [ - "result = model.predict(data, batch_size=32)\n", - "print(result.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "맞춤형 훈련 반복을 밑바닥부터 작성하는 방법을 포함하여 훈련과 평가에 대한 완전한 설명은 [훈련과 평가 가이드](./train_and_evaluate.ipynb)를 참고하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fzEOW4Cn0E2h" - }, - "source": [ - "## 고급 모델 만들기\n", - "\n", - "### 함수형 API\n", - "\n", - "`tf.keras.Sequential` 모델은 단순히 층을 쌓은 것으로 임의의 구조를 표현할 수 없습니다. [케라스 함수형 API](./functional.ipynb)를 사용하면 다음과 같은 복잡한 모델 구조를 만들 수 있습니다:\n", - "\n", - "* 다중 입력 모델,\n", - "* 다중 출력 모델,\n", - "* 층을 공유하는 모델 (동일한 층을 여러번 호출합니다),\n", - "* 데이터 흐름이 차례대로 진행되지 않는 모델 (예를 들면 잔차 연결(residual connections)).\n", - "\n", - "함수형 API로 모델을 만드는 방식은 다음과 같습니다:\n", - "\n", - "1. 하나의 층 객체는 호출 가능하고 텐서를 반환합니다.\n", - "2. `tf.keras.Model` 객체를 정의하기 위해 입력 텐서와 출력 텐서를 사용합니다.\n", - "3. 이 모델은 `Sequential` 모델과 동일한 방식으로 훈련됩니다.\n", - "\n", - "다음 코드는 함수형 API를 사용하여 간단한 완전 연결 네트워크를 만드는 예입니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mROj832r0E2i" - }, - "outputs": [], - "source": [ - "inputs = tf.keras.Input(shape=(32,)) # 입력 플레이스홀더를 반환합니다.\n", - "\n", - "# 층 객체는 텐서를 사용하여 호출되고 텐서를 반환합니다.\n", - "x = layers.Dense(64, activation='relu')(inputs)\n", - "x = layers.Dense(64, activation='relu')(x)\n", - "predictions = layers.Dense(10, activation='softmax')(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AFmspHeG1_W7" - }, - "source": [ - "입력과 출력을 사용해 모델의 객체를 만듭니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5k5uzlyu16HM" - }, - "outputs": [], - "source": [ - "model = tf.keras.Model(inputs=inputs, outputs=predictions)\n", - "\n", - "# 컴파일 단계는 훈련 과정을 설정합니다.\n", - "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# 5번의 에포크 동안 훈련합니다.\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EcKSLH3i0E2k" - }, - "source": [ - "### 모델 클래스 상속\n", - "\n", - "`tf.keras.Model` 클래스를 상속하고 자신만의 정방향 패스(forward pass)을 정의하여 완전히 커스터마이징된 모델을 만들 수 있습니다. `__init__` 메서드에서 층을 만들어 클래스 객체의 속성으로 지정합니다. 정방향 패스는 `call` 메서드에 정의합니다.\n", - "\n", - "[즉시 실행](../eager.md)이 활성화되어 있을 때 정방향 패스를 명령형 프로그래밍 방식으로 작성할 수 있기 때문에 모델 클래스 상속이 매우 유용합니다.\n", - "\n", - "노트: 정방향 패스를 *항상* 명령형 프로그래밍 방식으로 실행하려면 `super` 객체의 생성자를 호출할 때 `dynamic=True`를 지정하세요.\n", - "\n", - "중요 포인트: 작업에 맞는 API를 사용하세요. 모델 클래스 상속은 유연성을 제공하지만 복잡도가 증가하고 사용자 오류가 발생할 가능성이 높아집니다. 가능한한 함수형 API를 사용하세요.\n", - "\n", - "다음 코드는 `tf.keras.Model`의 클래스를 상속하여 명령형 프로그래밍 방식으로 실행할 필요가 없는 정방향 패스를 구현한 예입니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KLiHWzcn2Fzk" - }, - "outputs": [], - "source": [ - "class MyModel(tf.keras.Model):\n", - "\n", - " def __init__(self, num_classes=10):\n", - " super(MyModel, self).__init__(name='my_model')\n", - " self.num_classes = num_classes\n", - " # 층을 정의합니다.\n", - " self.dense_1 = layers.Dense(32, activation='relu')\n", - " self.dense_2 = layers.Dense(num_classes, activation='sigmoid')\n", - "\n", - " def call(self, inputs):\n", - " # 정방향 패스를 정의합니다.\n", - " # `__init__` 메서드에서 정의한 층을 사용합니다.\n", - " x = self.dense_1(inputs)\n", - " return self.dense_2(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ShDD4fv72KGc" - }, - "source": [ - "새 모델 클래스의 객체를 만듭니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "42C-qQHm0E2l" - }, - "outputs": [], - "source": [ - "model = MyModel(num_classes=10)\n", - "\n", - "# 컴파일 단계는 훈련 과정을 설정합니다.\n", - "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# 5번의 에포크 동안 훈련합니다.\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yqRQiKj20E2o" - }, - "source": [ - "### 맞춤형 층\n", - "\n", - "맞춤형 층(custom layer)을 만들려면 `tf.keras.layers.Layer` 클래스를 상속하고 다음 메서드를 구현합니다:\n", - "\n", - "* `__init__`: 이 층에서 사용되는 하위 층을 정의할 수 있습니다.\n", - "* `build`: 층의 가중치를 만듭니다. `add_weight` 메서드를 사용해 가중치를 추가합니다.\n", - "* `call`: 정방향 패스를 구현합니다.\n", - "\n", - "다음 코드는 입력과 커널 행렬의 `matmul` 계산을 구현한 맞춤형 층의 예입니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l7BFnIHr2WNc" - }, - "outputs": [], - "source": [ - "class MyLayer(layers.Layer):\n", - "\n", - " def __init__(self, output_dim, **kwargs):\n", - " self.output_dim = output_dim\n", - " super(MyLayer, self).__init__(**kwargs)\n", - "\n", - " def build(self, input_shape):\n", - " # 이 층에서 훈련할 가중치 변수를 만듭니다.\n", - " self.kernel = self.add_weight(name='kernel',\n", - " shape=(input_shape[1], self.output_dim),\n", - " initializer='uniform',\n", - " trainable=True)\n", - "\n", - " def call(self, inputs):\n", - " return tf.matmul(inputs, self.kernel)\n", - "\n", - " def get_config(self):\n", - " base_config = super(MyLayer, self).get_config()\n", - " base_config['output_dim'] = self.output_dim\n", - " return base_config\n", - "\n", - " @classmethod\n", - " def from_config(cls, config):\n", - " return cls(**config)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8wXDRgXV2ZrF" - }, - "source": [ - "맞춤형 층을 사용하여 모델을 만듭니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uqH-cY0h0E2p" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - " MyLayer(10),\n", - " layers.Activation('softmax')])\n", - "\n", - "# 컴파일 단계는 훈련 과정을 설정합니다.\n", - "model.compile(optimizer=tf.keras.optimizers.RMSprop(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# 5번의 에포크 동안 훈련합니다.\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "클래스 상속을 통해 맞춤형 층과 모델을 만드는 더 자세한 정보는 [맞춤형 층과 모델을 만드는 방법](./custom_layers_and_models.ipynb)을 참고하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Lu8cc3AJ0E2v" - }, - "source": [ - "## 콜백\n", - "\n", - "콜백(callback)은 훈련하는 동안 모델의 동작을 변경하고 확장하기 위해 전달하는 객체입니다. 자신만의 콜백을 작성하거나 다음과 같은 내장 `tf.keras.callbacks`을 사용할 수 있습니다:\n", - "\n", - "* `tf.keras.callbacks.ModelCheckpoint`: 일정 간격으로 모델의 체크포인트를 저장합니다.\n", - "* `tf.keras.callbacks.LearningRateScheduler`: 학습률(learning rate)을 동적으로 변경합니다.\n", - "* `tf.keras.callbacks.EarlyStopping`: 검증 성능이 향상되지 않으면 훈련을 중지합니다.\n", - "* `tf.keras.callbacks.TensorBoard`: [텐서보드](https://tensorflow.org/tensorboard)를 사용하여 모델을 모니터링합니다.\n", - "\n", - "`tf.keras.callbacks.Callback`을 사용하려면 모델의 `fit` 메서드에 전달합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "rdYwzSYV0E2v" - }, - "outputs": [], - "source": [ - "callbacks = [\n", - " # `val_loss`가 2번의 에포크에 걸쳐 향상되지 않으면 훈련을 멈춥니다.\n", - " tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'),\n", - " # `./logs` 디렉토리에 텐서보드 로그를 기록니다.\n", - " tf.keras.callbacks.TensorBoard(log_dir='./logs')\n", - "]\n", - "model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks,\n", - " validation_data=(val_data, val_labels))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 저장과 복원" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qnl7K-aI0E2z" - }, - "source": [ - "### 가중치\n", - "\n", - "`tf.keras.Model.save_weights`를 사용하여 모델의 가중치를 저장하고 복원합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uQIANjB94fLB" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - "layers.Dense(64, activation='relu', input_shape=(32,)),\n", - "layers.Dense(10, activation='softmax')])\n", - "\n", - "model.compile(optimizer=tf.keras.optimizers.Adam(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4eoHJ-ny0E21" - }, - "outputs": [], - "source": [ - "# 가중치를 텐서플로의 체크포인트 파일로 저장합니다.\n", - "model.save_weights('./weights/my_model')\n", - "\n", - "# 모델의 상태를 복원합니다.\n", - "# 모델의 구조가 동일해야 합니다.\n", - "model.load_weights('./weights/my_model')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u25Id3xe0E25" - }, - "source": [ - "기본적으로 모델의 가중치는 [텐서플로 체크포인트](../checkpoints.md) 파일 포맷으로 저장됩니다. 케라스의 HDF5 포맷으로 가중치를 저장할 수도 있습니다(다양한 백엔드를 지원하는 케라스 구현에서는 HDF5가 기본 설정입니다):" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JSAYoFEd0E26" - }, - "outputs": [], - "source": [ - "# 가중치를 HDF5 파일로 저장합니다.\n", - "model.save_weights('my_model.h5', save_format='h5')\n", - "\n", - "# 모델의 상태를 복원합니다.\n", - "model.load_weights('my_model.h5')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mje_yKL10E29" - }, - "source": [ - "### 설정\n", - "\n", - "모델 설정을 저장하면 가중치는 제외하고 모델의 구조를 직렬화합니다. 원본 모델을 정의한 코드가 없어도 저장된 설정을 사용하여 동일한 구조를 만들고 초기화할 수 있습니다. 케라스는 JSON과 YAML 직렬화 포맷을 지원합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EbET0oJTzGkq" - }, - "outputs": [], - "source": [ - "# 모델을 JSON 포맷으로 직렬화합니다.\n", - "json_string = model.to_json()\n", - "json_string" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "pX_badhH3yWV" - }, - "outputs": [], - "source": [ - "import json\n", - "import pprint\n", - "pprint.pprint(json.loads(json_string))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q7CIa05r4yTb" - }, - "source": [ - "JSON 파일로부터 (완전히 새로 초기화된) 모델을 만듭니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "J9UFv9k00E2_" - }, - "outputs": [], - "source": [ - "fresh_model = tf.keras.models.model_from_json(json_string)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t5NHtICh4uHK" - }, - "source": [ - "YAML 포맷으로 직렬화하려면 *텐서플로를 임포트하기 전에* `pyyaml`을 설치해야 합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aj24KB3Z36S4" - }, - "outputs": [], - "source": [ - "yaml_string = model.to_yaml()\n", - "print(yaml_string)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O53Kerfl43v7" - }, - "source": [ - "YAML 파일로부터 모델을 다시 만듭니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "77yRuwg03_MG" - }, - "outputs": [], - "source": [ - "fresh_model = tf.keras.models.model_from_yaml(yaml_string)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xPvOSSzM0E3B" - }, - "source": [ - "주의: Model 클래스를 상속하여 만든 모델은 `call` 메서드의 본문에 파이썬 코드로 구조가 정의되어 있기 때문에 직렬화되지 않습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iu8qMwld4-71" - }, - "source": [ - "### 전체 모델\n", - "\n", - "가중치와 모델 설정, 심지어 옵티마이저 설정까지 포함된 전체 모델을 파일에 저장할 수 있습니다. 모델의 중간 상태를 저장하고 나중에 원본 코드가 없어도 정확히 동일한 상태에서 훈련을 재개할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "45oNY34Z0E3C" - }, - "outputs": [], - "source": [ - "# 간단한 모델을 만듭니다.\n", - "model = tf.keras.Sequential([\n", - " layers.Dense(10, activation='softmax', input_shape=(32,)),\n", - " layers.Dense(10, activation='softmax')\n", - "])\n", - "model.compile(optimizer='rmsprop',\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "model.fit(data, labels, batch_size=32, epochs=5)\n", - "\n", - "\n", - "# 전체 모델을 HDF5 파일로 저장합니다.\n", - "model.save('my_model.h5')\n", - "\n", - "# 가중치와 옵티마이저를 포함하여 정확히 같은 모델을 다시 만듭니다.\n", - "model = tf.keras.models.load_model('my_model.h5')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "케라스 모델의 저장과 직렬화에 대한 더 자세한 내용은 [모델 저장과 직렬화 가이드](./save_and_serialize.ipynb)를 참고하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PMOWhDOB0E3E" - }, - "source": [ - "## 즉시 실행\n", - "\n", - "[즉시 실행](../eager.md)은 연산을 즉각 평가하는 명령형 프로그래밍(imperative programming) 환경입니다. 케라스에서는 즉시 실행이 필수가 아니지만 `tf.keras`는 이를 지원합니다. 이 기능은 프로그램을 검사하고 디버깅하는데 유용합니다.\n", - "\n", - "모든 `tf.keras` 모델링 API는 즉시 실행과 호환됩니다. `Sequential`이나 함수형 API와 사용할 수 있지만 즉시 실행은 특히 *모델 상속*과 *맞춤형 층*을 만들 때 장점이 나타납니다. 이런 API는 (기존의 층을 조합하여 모델을 만드는 대신) 직접 정방향 패스의 코드를 작성하기 때문입니다.\n", - "\n", - "[즉시 실행 가이드](./eager.ipynb#build_a_model)에서 맞춤형 훈련 반복과 `tf.GradientTape`를 케라스 모델에 같이 사용하는 예를 참고하세요. 또한 간단하지만 완전한 예제를 [여기](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/quickstart/advanced.ipynb)에서 볼 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 분산 처리" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2wG3NVco5B5V" - }, - "source": [ - "### 다중 GPU\n", - "\n", - "`tf.keras` 모델은 `tf.distribute.Strategy`를 사용하여 다중 GPU에서 실행할 수 있습니다. 이 API는 기존 코드를 거의 수정하지 않고 다중 GPU에서 훈련을 분산시킬 수 있습니다.\n", - "\n", - "현재는 `tf.distribute.MirroredStrategy`가 유일하게 지원되는 분산 전략입니다. `MirroredStrategy`는 한 대의 장치에서 계산 결과를 모두 수집하는 방식인 그래프 내 복제(in-graph replication)를 수행합니다. `distribute.Strategy`를 사용하려면 `Strategy`의 `.scope()` 안에 옵티마이저 객체 생성, 모델 구성, 컴파일 단계를 포함시킨 다음 모델을 훈련합니다.\n", - "\n", - "다음 코드는 한 대의 컴퓨터에서 다중 GPU를 사용해 `tf.keras.Model`을 분산 처리하는 예입니다.\n", - "\n", - "먼저, `MirroredStrategy`의 `scope()` 안에서 모델을 정의합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sbaRr7g-0E3I" - }, - "outputs": [], - "source": [ - "strategy = tf.distribute.MirroredStrategy()\n", - "\n", - "with strategy.scope():\n", - " model = tf.keras.Sequential()\n", - " model.add(layers.Dense(16, activation='relu', input_shape=(10,)))\n", - " model.add(layers.Dense(1, activation='sigmoid'))\n", - "\n", - " optimizer = tf.keras.optimizers.SGD(0.2)\n", - "\n", - " model.compile(loss='binary_crossentropy', optimizer=optimizer)\n", - "\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j2hHoMYOgLV-" - }, - "source": [ - "그다음, 보통 때와 같은 데이터로 모델을 훈련합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aTuk02nmbvXe" - }, - "outputs": [], - "source": [ - "x = np.random.random((1024, 10))\n", - "y = np.random.randint(2, size=(1024, 1))\n", - "x = tf.cast(x, tf.float32)\n", - "dataset = tf.data.Dataset.from_tensor_slices((x, y))\n", - "dataset = dataset.shuffle(buffer_size=1024).batch(32)\n", - "\n", - "model.fit(dataset, epochs=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FuV_qqysWcDA" - }, - "source": [ - "더 자세한 정보는 [텐서플로의 분산 훈련 가이드](../distributed_training.ipynb)를 참고하세요." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "keras.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/site/ko/guide/migrate.ipynb b/site/ko/guide/migrate.ipynb deleted file mode 100644 index b083b22a22c..00000000000 --- a/site/ko/guide/migrate.ipynb +++ /dev/null @@ -1,1600 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "migrate.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "metadata": { - "colab_type": "text", - "id": "wJcYs_ERTnnI" - }, - "cell_type": "markdown", - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "HMUDt0CiUJk9", - "colab": {} - }, - "cell_type": "code", - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "77z2OchJTk0l" - }, - "cell_type": "markdown", - "source": [ - "# 기존 코드를 TensorFlow 2.0으로 바꾸기\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " TensorFlow.org에서 보기\n", - " \n", - " \n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " \n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "metadata": { - "id": "Cp6jVAZ6dNtq", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/effective_tf2.md)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "C0V10enS1_WU" - }, - "cell_type": "markdown", - "source": [ - "여전히 텐서플로 1.X 버전의 코드를 수정하지 않고 텐서플로 2.0에서 실행할 수 있습니다(`contrib` 모듈은 제외):\n", - "\n", - "```\n", - "import tensorflow.compat.v1 as tf\n", - "tf.disable_v2_behavior()\n", - "```\n", - "\n", - "하지만 이렇게 하면 텐서플로 2.0에서 제공하는 많은 장점을 활용할 수 없습니다. 이 문서는 성능을 높이면서 코드는 더 간단하고 유지보수하기 쉽도록 업그레이드하는 방법을 안내합니다.\n", - "\n", - "## 자동 변환 스크립트\n", - "\n", - "첫 번째 단계는 [업그레이드 스크립트](./upgrade.md)를 사용해 보는 것입니다.\n", - "\n", - "이는 텐서플로 2.0으로 업그레이드하기 위해 처음에 할 일입니다. 하지만 이 작업이 기존 코드를 텐서플로 2.0 스타일로 바꾸어 주지는 못합니다. 여전히 플레이스홀더(placeholder)나 세션(session), 컬렉션(collection), 그외 1.x 스타일의 기능을 사용하기 위해 `tf.compat.v1` 아래의 모듈을 참조하고 있을 것입니다.\n", - "\n", - "## 2.0에 맞도록 코드 수정하기\n", - "\n", - "텐서플로 1.x 코드를 텐서플로 2.0으로 변환하는 몇 가지 예를 소개하겠습니다. 이 작업을 통해 성능을 최적화하고 간소화된 API의 이점을 사용할 수 있습니다.\n", - "\n", - "각각의 경우에 수정하는 패턴은 다음과 같습니다:" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "uP0O8Pc45LNs" - }, - "cell_type": "markdown", - "source": [ - "### 1. `tf.Session.run` 호출을 바꾸세요.\n", - "\n", - "모든 `tf.Session.run` 호출을 파이썬 함수로 바꾸어야 합니다.\n", - "\n", - "* `feed_dict`와 `tf.placeholder`는 함수의 매개변수가 됩니다.\n", - "* `fetches`는 함수의 반환값이 됩니다.\n", - "\n", - "표준 파이썬 디버거 `pdb`를 사용하여 함수의 코드를 라인 단위로 실행하고 디버깅할 수 있습니다.\n", - "\n", - "작동 결과에 만족하면 그래프 모드에서 효율적으로 실행할 수 있도록 `tf.function` 데코레이터를 추가합니다. 조금 더 자세한 내용은 [오토그래프 가이드](function.ipynb)를 참고하세요." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "jlBOqROL5NmN" - }, - "cell_type": "markdown", - "source": [ - "### 2. 파이썬 객체를 사용하여 변수와 손실을 관리하세요.\n", - "\n", - "`tf.get_variable` 대신에 `tf.Variable`을 사용하세요.\n", - "\n", - "모든 `variable_scope`는 파이썬 객체로 바꿀 수 있습니다. 일반적으로 다음 중 하나가 될 것입니다:\n", - "\n", - "* `tf.keras.layers.Layer`\n", - "* `tf.keras.Model`\n", - "* `tf.Module`\n", - "\n", - "만약 (`tf.Graph.get_collection(tf.GraphKeys.VARIABLES)`처럼) 변수의 리스트가 필요하다면 `Layer`와 `Model` 객체의 `.variables`이나 `.trainable_variables` 속성을 사용하세요.\n", - "\n", - "`Layer`와 `Model` 클래스는 전역 컬렉션이 필요하지 않도록 몇 가지 다른 속성들도 제공합니다. `.losses` 속성은 `tf.GraphKeys.LOSSES` 컬렉션 대신 사용할 수 있습니다.\n", - "\n", - "자세한 내용은 [케라스 가이드](keras.ipynb)를 참고하세요.\n", - "\n", - "경고: `tf.compat.v1`의 상당수 기능은 암묵적으로 전역 컬렉션을 사용합니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "rGFhBzoF5FIq" - }, - "cell_type": "markdown", - "source": [ - "### 3. 훈련 루프를 업그레이드하세요.\n", - "\n", - "풀려는 문제에 맞는 고수준 API를 사용하세요. 훈련 루프(loop)를 직접 만드는 것보다 `tf.keras.Model.fit` 메서드를 사용하는 것이 좋습니다.\n", - "\n", - "고수준 함수는 훈련 루프를 직접 만들 때 놓치기 쉬운 여러 가지 저수준의 세부 사항들을 관리해 줍니다. 예를 들어 자동으로 규제(regularization) 손실을 수집하거나 모델을 호출할 때 `training=True`로 매개변수를 설정해 줍니다.\n", - "\n", - "### 4. 데이터 입력 파이프라인을 업그레이드하세요.\n", - "\n", - "데이터 입력을 위해 `tf.data` 데이터셋을 사용하세요. 이 객체는 효율적이고 간결하며 텐서플로와 잘 통합됩니다.\n", - "\n", - "`tf.keras.Model.fit` 메서드에 바로 전달할 수 있습니다.\n", - "\n", - "```\n", - "model.fit(dataset, epochs=5)\n", - "```\n", - "\n", - "파이썬에서 직접 반복시킬 수 있습니다:\n", - "\n", - "```\n", - "for example_batch, label_batch in dataset:\n", - " break\n", - "```" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "X_ilfTGJ4Yml" - }, - "cell_type": "markdown", - "source": [ - "## 모델 변환하기\n", - "\n", - "### 준비" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "bad2N-Z115W1", - "colab": {} - }, - "cell_type": "code", - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "\n", - "import tensorflow_datasets as tfds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "FB99sqHX2Q5m" - }, - "cell_type": "markdown", - "source": [ - "### 저수준 변수와 연산 실행\n", - "\n", - "저수준 API를 사용하는 예는 다음과 같습니다:\n", - "\n", - "* 재사용을 위해 변수 범위(variable scopes)를 사용하기\n", - "* `tf.get_variable`로 변수를 만들기\n", - "* 명시적으로 컬렉션을 참조하기\n", - "* 다음과 같은 메서드를 사용하여 암묵적으로 컬렉션을 참조하기:\n", - "\n", - " * `tf.global_variables`\n", - " * `tf.losses.get_regularization_loss`\n", - "\n", - "* 그래프 입력을 위해 `tf.placeholder`를 사용하기\n", - "* `session.run`으로 그래프를 실행하기\n", - "* 변수를 수동으로 초기화하기" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "e582IjyF2eje" - }, - "cell_type": "markdown", - "source": [ - "#### 변환 전\n", - "\n", - "다음 코드는 텐서플로 1.x를 사용한 코드에서 볼 수 있는 패턴입니다.\n", - "\n", - "```python\n", - "in_a = tf.placeholder(dtype=tf.float32, shape=(2))\n", - "in_b = tf.placeholder(dtype=tf.float32, shape=(2))\n", - "\n", - "def forward(x):\n", - " with tf.variable_scope(\"matmul\", reuse=tf.AUTO_REUSE):\n", - " W = tf.get_variable(\"W\", initializer=tf.ones(shape=(2,2)),\n", - " regularizer=tf.contrib.layers.l2_regularizer(0.04))\n", - " b = tf.get_variable(\"b\", initializer=tf.zeros(shape=(2)))\n", - " return W * x + b\n", - "\n", - "out_a = forward(in_a)\n", - "out_b = forward(in_b)\n", - "\n", - "reg_loss = tf.losses.get_regularization_loss(scope=\"matmul\")\n", - "\n", - "with tf.Session() as sess:\n", - " sess.run(tf.global_variables_initializer())\n", - " outs = sess.run([out_a, out_b, reg_loss],\n", - " \t feed_dict={in_a: [1, 0], in_b: [0, 1]})\n", - "\n", - "```" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "QARwz4Xd2lc2" - }, - "cell_type": "markdown", - "source": [ - "#### 변환 후" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "x0AVzBFRBPcU" - }, - "cell_type": "markdown", - "source": [ - "변환된 코드의 패턴은 다음과 같습니다:\n", - "\n", - "* 변수는 파이썬 지역 객체입니다.\n", - "* `forward` 함수는 여전히 필요한 계산을 정의합니다.\n", - "* `sess.run` 호출은 `forward` 함수를 호출하는 것으로 바뀝니다.\n", - "* `tf.function` 데코레이터는 선택 사항으로 성능을 위해 추가할 수 있습니다.\n", - "* 어떤 전역 컬렉션도 참조하지 않고 규제를 직접 계산합니다.\n", - "* **세션이나 플레이스홀더를 사용하지 않습니다.**" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "lXEZoLMP2cWJ", - "colab": {} - }, - "cell_type": "code", - "source": [ - "W = tf.Variable(tf.ones(shape=(2,2)), name=\"W\")\n", - "b = tf.Variable(tf.zeros(shape=(2)), name=\"b\")\n", - "\n", - "@tf.function\n", - "def forward(x):\n", - " return W * x + b\n", - "\n", - "out_a = forward([1,0])\n", - "print(out_a)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "YmE96A_1jZTg", - "colab": {} - }, - "cell_type": "code", - "source": [ - "out_b = forward([0,1])\n", - "\n", - "regularizer = tf.keras.regularizers.l2(0.04)\n", - "reg_loss = regularizer(W)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "ycDxY9nL268-" - }, - "cell_type": "markdown", - "source": [ - "### `tf.layers` 기반의 모델" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "K-bIk7wL48U7" - }, - "cell_type": "markdown", - "source": [ - "`tf.layers` 모듈은 변수를 정의하고 재사용하기 위해 `tf.variable_scope`에 의존하는 층 함수를 포함합니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "8I_qKpT73KyM" - }, - "cell_type": "markdown", - "source": [ - "#### 변환 전\n", - "```python\n", - "def model(x, training, scope='model'):\n", - " with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):\n", - " x = tf.layers.conv2d(x, 32, 3, activation=tf.nn.relu,\n", - " kernel_regularizer=tf.contrib.layers.l2_regularizer(0.04))\n", - " x = tf.layers.max_pooling2d(x, (2, 2), 1)\n", - " x = tf.layers.flatten(x)\n", - " x = tf.layers.dropout(x, 0.1, training=training)\n", - " x = tf.layers.dense(x, 64, activation=tf.nn.relu)\n", - " x = tf.layers.batch_normalization(x, training=training)\n", - " x = tf.layers.dense(x, 10, activation=tf.nn.softmax)\n", - " return x\n", - "\n", - "train_out = model(train_data, training=True)\n", - "test_out = model(test_data, training=False)\n", - "```" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "b8_Ii7CQ3fK-" - }, - "cell_type": "markdown", - "source": [ - "#### 변환 후" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "BsAseSMfB9XN" - }, - "cell_type": "markdown", - "source": [ - "* 층을 단순하게 쌓을 경우엔 `tf.keras.Sequential`이 적합합니다. (복잡한 모델인 경우 [맞춤형 층과 모델](keras/custom_layers_and_models.ipynb)이나 [함수형 API](keras/functional.ipynb)를 참고하세요.)\n", - "* 모델이 변수와 규제 손실을 관리합니다.\n", - "* `tf.layers`에서 `tf.keras.layers`로 바로 매핑되기 때문에 일대일로 변환됩니다.\n", - "\n", - "대부분 매개변수는 동일합니다. 다른 부분은 다음과 같습니다:\n", - "\n", - "* 모델이 실행될 때 각 층에 `training` 매개변수가 전달됩니다.\n", - "* 원래 `model` 함수의 첫 번째 매개변수(입력 `x`)는 사라집니다. 층 객체가 모델 구축과 모델 호출을 구분하기 때문입니다.\n", - "\n", - "추가 노트:\n", - "\n", - "* `tf.contrib`에서 규제를 초기화했다면 다른 것보다 매개변수 변화가 많습니다.\n", - "* 더 이상 컬렉션을 사용하지 않기 때문에 `tf.losses.get_regularization_loss`와 같은 함수는 값을 반환하지 않습니다. 이는 훈련 루프를 망가뜨릴 수 있습니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "DLAPORrN3lct", - "colab": {} - }, - "cell_type": "code", - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " kernel_regularizer=tf.keras.regularizers.l2(0.04),\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dropout(0.1),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "train_data = tf.ones(shape=(1, 28, 28, 1))\n", - "test_data = tf.ones(shape=(1, 28, 28, 1))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "6nWh6IXvkMKv", - "colab": {} - }, - "cell_type": "code", - "source": [ - "train_out = model(train_data, training=True)\n", - "print(train_out)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "YnAdIDLlj3go", - "colab": {} - }, - "cell_type": "code", - "source": [ - "test_out = model(test_data, training=False)\n", - "print(test_out)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "sAgqwCJBMx_x", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 훈련되는 전체 변수\n", - "len(model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "uX6knaYMNM8p", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 규제 손실\n", - "model.losses" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "9moqw5E_4Cwl" - }, - "cell_type": "markdown", - "source": [ - "### 변수와 tf.layers의 혼용" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "80DEsImmq6VX" - }, - "cell_type": "markdown", - "source": [ - "기존 코드는 종종 저수준 TF 1.x 변수와 고수준 `tf.layers` 연산을 혼용합니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "oZe9L6RR4OcP" - }, - "cell_type": "markdown", - "source": [ - "#### 변경 전\n", - "```python\n", - "def model(x, training, scope='model'):\n", - " with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):\n", - " W = tf.get_variable(\n", - " \"W\", dtype=tf.float32,\n", - " initializer=tf.ones(shape=x.shape),\n", - " regularizer=tf.contrib.layers.l2_regularizer(0.04),\n", - " trainable=True)\n", - " if training:\n", - " x = x + W\n", - " else:\n", - " x = x + W * 0.5\n", - " x = tf.layers.conv2d(x, 32, 3, activation=tf.nn.relu)\n", - " x = tf.layers.max_pooling2d(x, (2, 2), 1)\n", - " x = tf.layers.flatten(x)\n", - " return x\n", - "\n", - "train_out = model(train_data, training=True)\n", - "test_out = model(test_data, training=False)\n", - "```" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "y6ORX7cD4TkD" - }, - "cell_type": "markdown", - "source": [ - "#### 변경 후" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "2BaRwog5CBpz" - }, - "cell_type": "markdown", - "source": [ - "이런 코드를 변환하려면 이전 예제처럼 층별로 매핑하는 패턴을 사용하세요.\n", - "\n", - "`tf.variable_scope`는 기본적으로 하나의 층입니다. 따라서 `tf.keras.layers.Layer`로 재작성합니다. 자세한 내용은 이 [문서](keras/custom_layers_and_models.ipynb)를 참고하세요.\n", - "\n", - "일반적인 패턴은 다음과 같습니다:\n", - "\n", - "* `__init__`에서 층에 필요한 매개변수를 입력 받습니다.\n", - "* `build` 메서드에서 변수를 만듭니다.\n", - "* `call` 메서드에서 연산을 실행하고 결과를 반환합니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "YcCAjNuP4NVh", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 모델에 추가하기 위해 맞춤형 층을 만듭니다.\n", - "class CustomLayer(tf.keras.layers.Layer):\n", - " def __init__(self, *args, **kwargs):\n", - " super(CustomLayer, self).__init__(*args, **kwargs)\n", - "\n", - " def build(self, input_shape):\n", - " self.w = self.add_weight(\n", - " shape=input_shape[1:],\n", - " dtype=tf.float32,\n", - " initializer=tf.keras.initializers.ones(),\n", - " regularizer=tf.keras.regularizers.l2(0.02),\n", - " trainable=True)\n", - "\n", - " # call 메서드가 그래프 모드에서 사용되면\n", - " # training 변수는 텐서가 됩니다.\n", - " @tf.function\n", - " def call(self, inputs, training=None):\n", - " if training:\n", - " return inputs + self.w\n", - " else:\n", - " return inputs + self.w * 0.5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "dR_QO6_wBgMm", - "colab": {} - }, - "cell_type": "code", - "source": [ - "custom_layer = CustomLayer()\n", - "print(custom_layer([1]).numpy())\n", - "print(custom_layer([1], training=True).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "VzqaIf4E42oY", - "colab": {} - }, - "cell_type": "code", - "source": [ - "train_data = tf.ones(shape=(1, 28, 28, 1))\n", - "test_data = tf.ones(shape=(1, 28, 28, 1))\n", - "\n", - "# 맞춤형 층을 포함한 모델을 만듭니다.\n", - "model = tf.keras.Sequential([\n", - " CustomLayer(input_shape=(28, 28, 1)),\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - "])\n", - "\n", - "train_out = model(train_data, training=True)\n", - "test_out = model(test_data, training=False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "dS5ed_jjOkvh" - }, - "cell_type": "markdown", - "source": [ - "노트:\n", - "\n", - "* 클래스 상속으로 만든 케라스 모델과 층은 v1 그래프(연산간의 의존성이 자동으로 제어되지 않습니다)와 즉시 실행 모드 양쪽에서 실행될 수 있어야 합니다.\n", - " * 오토그래프(autograph)와 의존성 자동 제어(automatic control dependency)를 위해 `tf.function()`으로 `call()` 메서드를 감쌉니다.\n", - "\n", - "* `call` 메서드에 `training` 매개변수를 추가하는 것을 잊지 마세요.\n", - " * 경우에 따라 이 값은 `tf.Tensor`가 됩니다.\n", - " * 경우에 따라 이 값은 파이썬 불리언(boolean)이 됩니다.\n", - "\n", - "* `self.add_weight()`를 사용하여 생성자 메서드나 `def build()` 메서드에서 모델 변수를 만듭니다.\n", - " * `build` 메서드에서 입력 크기를 참조할 수 있으므로 적절한 크기의 가중치를 만들 수 있습니다.\n", - " * `tf.keras.layers.Layer.add_weight`를 사용하면 케라스가 변수와 규제 손실을 관리할 수 있습니다.\n", - "\n", - "* 맞춤형 층 안에 `tf.Tensors` 객체를 포함하지 마세요.\n", - " * `tf.function`이나 즉시 실행 모드에서 모두 텐서가 만들어지지만 이 텐서들의 동작 방식은 다릅니다.\n", - " * 상태를 저장하기 위해서는 `tf.Variable`을 사용하세요. 변수는 양쪽 방식에 모두 사용할 수 있습니다.\n", - " * `tf.Tensors`는 중간 값을 저장하기 위한 용도로만 사용합니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "ulaB1ymO4pw5" - }, - "cell_type": "markdown", - "source": [ - "### Slim & contrib.layers를 위한 노트\n", - "\n", - "예전 텐서플로 1.x 코드는 [Slim](https://ai.googleblog.com/2016/08/tf-slim-high-level-library-to-define.html) 라이브러리를 많이 사용합니다. 이 라이브러리는 텐서플로 1.x의 `tf.contrib.layers`로 패키지되어 있습니다. `contrib` 모듈은 더 이상 텐서플로 2.0에서 지원하지 않고 `tf.compat.v1`에도 포함되지 않습니다. Slim을 사용한 코드를 TF 2.0으로 변환하는 것은 `tf.layers`를 사용한 코드를 변경하는 것보다 더 어렵습니다. 사실 Slim 코드는 `tf.layers`로 먼저 변환하고 그 다음 케라스로 변환하는 것이 좋습니다.\n", - "\n", - "* `arg_scopes`를 삭제하세요. 모든 매개변수는 명시적으로 설정되어야 합니다.\n", - "* `normalizer_fn`과 `activation_fn`를 사용해야 한다면 분리하여 각각 하나의 층으로 만드세요.\n", - "* 분리 합성곱(separable conv) 층은 한 개 이상의 다른 케라스 층으로 매핑합니다(깊이별(depthwise), 점별(pointwise), 분리(separable) 케라스 층).\n", - "* Slim과 `tf.layers`는 매개변수 이름과 기본값이 다릅니다.\n", - "* 일부 매개변수는 다른 스케일(scale)을 가집니다.\n", - "* 사전 훈련된 Slim 모델을 사용한다면 `tf.keras.applications`나 [TFHub](https://tensorflow.orb/hub)를 확인해 보세요.\n", - "\n", - "일부 `tf.contrib` 층은 텐서플로 내부에 포함되지 못했지만 [TF 애드온(add-on) 패키지](https://github.com/tensorflow/addons)로 옮겨졌습니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "1w72KrXm4yZR" - }, - "cell_type": "markdown", - "source": [ - "## 훈련" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "56PQxTgy2bpI" - }, - "cell_type": "markdown", - "source": [ - "여러 가지 방법으로 `tf.keras` 모델에 데이터를 주입할 수 있습니다. 파이썬 제너레이터(generator)와 넘파이 배열을 입력으로 사용할 수 있습니다.\n", - "\n", - "`tf.data` 패키지를 사용하여 모델에 데이터를 주입하는 것이 권장되는 방법입니다. 이 패키지는 데이터 조작을 위한 고성능 클래스들을 포함하고 있습니다.\n", - "\n", - "`tf.queue`는 데이터 구조로만 지원되고 입력 파이프라인으로는 지원되지 않습니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "m6htasZ7iBB4" - }, - "cell_type": "markdown", - "source": [ - "### 데이터셋 사용하기" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "loTPH2Pz4_Oj" - }, - "cell_type": "markdown", - "source": [ - "[텐서플로 데이터셋(Datasets)](https://tensorflow.org/datasets) 패키지(`tfds`)는 `tf.data.Dataset` 객체로 정의된 데이터셋을 적재하기 위한 유틸리티가 포함되어 있습니다.\n", - "\n", - "예를 들어 `tfds`를 사용하여 MNIST 데이터셋을 적재하는 코드는 다음과 같습니다:" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "BMgxaLH74_s-", - "colab": {} - }, - "cell_type": "code", - "source": [ - "datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n", - "mnist_train, mnist_test = datasets['train'], datasets['test']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "hPJhEuvj5VfR" - }, - "cell_type": "markdown", - "source": [ - "그 다음 훈련용 데이터를 준비합니다:\n", - "\n", - " * 각 이미지의 스케일을 조정합니다.\n", - " * 샘플의 순서를 섞습니다.\n", - " * 이미지와 레이블(label)의 배치를 만듭니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "StBRHtJM2S7o", - "colab": {} - }, - "cell_type": "code", - "source": [ - "BUFFER_SIZE = 10 # 실전 코드에서는 더 큰 값을 사용합니다.\n", - "BATCH_SIZE = 64\n", - "NUM_EPOCHS = 5\n", - "\n", - "\n", - "def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - "\n", - " return image, label\n", - "\n", - "train_data = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", - "test_data = mnist_test.map(scale).batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "SKq14zKKFAdv" - }, - "cell_type": "markdown", - "source": [ - "간단한 예제를 위해 5개의 배치만 반환하도록 데이터셋을 자릅니다:" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "_J-o4YjG2mkM", - "colab": {} - }, - "cell_type": "code", - "source": [ - "STEPS_PER_EPOCH = 5\n", - "\n", - "train_data = train_data.take(STEPS_PER_EPOCH)\n", - "test_data = test_data.take(STEPS_PER_EPOCH)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "XEqdkH54VM6c", - "colab": {} - }, - "cell_type": "code", - "source": [ - "image_batch, label_batch = next(iter(train_data))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "mSev7vZC5GJB" - }, - "cell_type": "markdown", - "source": [ - "### 케라스 훈련 루프 사용하기\n", - "\n", - "훈련 과정을 세부적으로 제어할 필요가 없다면 케라스의 내장 메서드인 `fit`, `evaluate`, `predict`를 사용하는 것이 좋습니다. 이 메서드들은 모델 구현(Sequential, 함수형 API, 클래스 상속)에 상관없이 일관된 훈련 인터페이스를 제공합니다.\n", - "\n", - "이 메서드들의 장점은 다음과 같습니다:\n", - "\n", - "* 넘파이 배열, 파이썬 제너레이터, `tf.data.Datasets`을 사용할 수 있습니다.\n", - "* 자동으로 규제와 활성화 손실을 적용합니다.\n", - "* [다중 장치 훈련](distributed_training.ipynb)을 위해 `tf.distribute`을 지원합니다.\n", - "* 임의의 호출 가능한 객체를 손실과 측정 지표로 사용할 수 있습니다.\n", - "* `tf.keras.callbacks.TensorBoard`와 같은 콜백(callback)이나 맞춤형 콜백을 지원합니다.\n", - "* 자동으로 텐서플로 그래프를 사용하므로 성능이 뛰어납니다.\n", - "\n", - "`Dataset`을 사용하여 모델을 훈련하는 예제는 다음과 같습니다. (자세한 작동 방식은 [튜토리얼](../tutorials)을 참고하세요.)" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "uzHFCzd45Rae", - "colab": {} - }, - "cell_type": "code", - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " kernel_regularizer=tf.keras.regularizers.l2(0.02),\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dropout(0.1),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "# 맞춤형 층이 없는 모델입니다.\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "model.fit(train_data, epochs=NUM_EPOCHS)\n", - "loss, acc = model.evaluate(test_data)\n", - "\n", - "print(\"손실 {}, 정확도 {}\".format(loss, acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "akpeOb09YBhq" - }, - "cell_type": "markdown", - "source": [ - "### 맞춤형 훈련 루프 만들기\n", - "\n", - "케라스 모델의 훈련 스텝(step)이 좋지만 그 외 다른 것을 더 제어하려면 자신만의 데이터 반복 루프를 만들고 `tf.keras.model.train_on_batch` 메서드를 사용해 보세요.\n", - "\n", - "기억할 점: 많은 것을 `tf.keras.Callback`으로 구현할 수 있습니다.\n", - "\n", - "이 메서드는 앞에서 언급한 메서드의 장점을 많이 가지고 있고 사용자가 바깥쪽 루프를 제어할 수 있습니다.\n", - "\n", - "훈련하는 동안 성능을 확인하기 위해 `tf.keras.model.test_on_batch`나 `tf.keras.Model.evaluate` 메서드를 사용할 수도 있습니다.\n", - "\n", - "노트: `train_on_batch`와 `test_on_batch`는 기본적으로 하나의 배치에 대한 손실과 측정값을 반환합니다. `reset_metrics=False`를 전달하면 누적된 측정값을 반환합니다. 이 때는 누적된 측정값을 적절하게 초기화해 주어야 합니다. `AUC`와 같은 일부 지표는 `reset_metrics=False`를 설정해야 올바르게 계산됩니다.\n", - "\n", - "앞의 모델을 계속 사용합니다:" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "eXr4CyJMtJJ6", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 맞춤형 층이 없는 모델입니다.\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "metrics_names = model.metrics_names\n", - "\n", - "for epoch in range(NUM_EPOCHS):\n", - " # 누적된 측정값을 초기화합니다.\n", - " model.reset_metrics()\n", - "\n", - " for image_batch, label_batch in train_data:\n", - " result = model.train_on_batch(image_batch, label_batch)\n", - " print(\"훈련: \",\n", - " \"{}: {:.3f}\".format(metrics_names[0], result[0]),\n", - " \"{}: {:.3f}\".format(metrics_names[1], result[1]))\n", - " for image_batch, label_batch in test_data:\n", - " result = model.test_on_batch(image_batch, label_batch,\n", - " # return accumulated metrics\n", - " reset_metrics=False)\n", - " print(\"\\n평가: \",\n", - " \"{}: {:.3f}\".format(metrics_names[0], result[0]),\n", - " \"{}: {:.3f}\".format(metrics_names[1], result[1]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "LQTaHTuK5S5A" - }, - "cell_type": "markdown", - "source": [ - "\n", - "### 훈련 단계 커스터마이징\n", - "\n", - "자유도를 높이고 제어를 더 하려면 다음 세 단계를 사용해 자신만의 훈련 루프를 구현할 수 있습니다:\n", - "\n", - "1. 샘플 배치를 만드는 파이썬 제너레이터나 `tf.data.Dataset`을 반복합니다.\n", - "2. `tf.GradientTape`을 사용하여 그래디언트를 계산합니다.\n", - "3. `tf.keras.optimizer`를 사용하여 모델의 가중치 변수를 업데이트합니다.\n", - "\n", - "기억할 점:\n", - "\n", - "* 클래스 상속으로 만든 층과 모델의 `call` 메서드에는 항상 `training` 매개변수를 포함하세요.\n", - "* 모델을 호출할 때 `training` 매개변수를 올바르게 지정했는지 확인하세요.\n", - "* 사용 방식에 따라 배치 데이터에서 모델이 실행될 때까지 모델 변수가 생성되지 않을 수 있습니다.\n", - "* 모델의 규제 손실 같은 것들을 직접 관리해야 합니다.\n", - "\n", - "v1에 비해 단순해진 것:\n", - "\n", - "* 따로 변수를 초기화할 필요가 없습니다. 변수는 생성될 때 초기화됩니다.\n", - "* 의존성을 수동으로 제어할 필요가 없습니다. `tf.function` 안에서도 연산은 즉시 실행 모드처럼 실행됩니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "gQooejfYlQeF", - "colab": {} - }, - "cell_type": "code", - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " kernel_regularizer=tf.keras.regularizers.l2(0.02),\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dropout(0.1),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "optimizer = tf.keras.optimizers.Adam(0.001)\n", - "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "@tf.function\n", - "def train_step(inputs, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(inputs, training=True)\n", - " regularization_loss = tf.math.add_n(model.losses)\n", - " pred_loss = loss_fn(labels, predictions)\n", - " total_loss = pred_loss + regularization_loss\n", - "\n", - " gradients = tape.gradient(total_loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - "for epoch in range(NUM_EPOCHS):\n", - " for inputs, labels in train_data:\n", - " train_step(inputs, labels)\n", - " print(\"마지막 에포크\", epoch)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "kS7WW5Z75ve3" - }, - "cell_type": "markdown", - "source": [ - "### 새로운 스타일의 측정 지표\n", - "\n", - "텐서플로 2.0에서 측정 지표는 객체입니다. 이 객체는 즉시 실행 모드와 `tf.function`에서 모두 사용할 수 있습니다. 측정 객체는 다음과 같은 메서드를 가집니다:\n", - "\n", - "* `update_state()` — 새로운 측정값을 추가합니다.\n", - "* `result()` — 누적된 측정 결과를 얻습니다.\n", - "* `reset_states()` — 모든 측정 내용을 지웁니다.\n", - "\n", - "이 객체는 호출 가능합니다. `update_state` 메서드처럼 새로운 측정값과 함께 호출하면 상태를 업데이트하고 새로운 측정 결과를 반환합니다.\n", - "\n", - "측정 변수를 수동으로 초기화할 필요가 없습니다. 텐서플로 2.0은 자동으로 의존성을 관리하기 때문에 어떤 경우에도 신경 쓸 필요가 없습니다.\n", - "\n", - "다음은 측정 객체를 사용하여 맞춤형 훈련 루프 안에서 평균 손실을 관리하는 코드입니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "HAbA0fKW58CH", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 측정 객체를 만듭니다.\n", - "loss_metric = tf.keras.metrics.Mean(name='train_loss')\n", - "accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "@tf.function\n", - "def train_step(inputs, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(inputs, training=True)\n", - " regularization_loss = tf.math.add_n(model.losses)\n", - " pred_loss = loss_fn(labels, predictions)\n", - " total_loss = pred_loss + regularization_loss\n", - "\n", - " gradients = tape.gradient(total_loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - " # 측정값을 업데이트합니다.\n", - " loss_metric.update_state(total_loss)\n", - " accuracy_metric.update_state(labels, predictions)\n", - "\n", - "\n", - "for epoch in range(NUM_EPOCHS):\n", - " # 측정값을 초기화합니다.\n", - " loss_metric.reset_states()\n", - " accuracy_metric.reset_states()\n", - "\n", - " for inputs, labels in train_data:\n", - " train_step(inputs, labels)\n", - " # 측정 결과를 얻습니다.\n", - " mean_loss = loss_metric.result()\n", - " mean_accuracy = accuracy_metric.result()\n", - "\n", - " print('에포크: ', epoch)\n", - " print(' 손실: {:.3f}'.format(mean_loss))\n", - " print(' 정확도: {:.3f}'.format(mean_accuracy))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "JmMLBKs66DeA" - }, - "cell_type": "markdown", - "source": [ - "## 저장과 복원" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "5_QKn3Kl6TUu" - }, - "cell_type": "markdown", - "source": [ - "### 체크포인트 호환성\n", - "\n", - "텐서플로 2.0은 [객체 기반의 체크포인트](checkpoint.ipynb)를 사용합니다.\n", - "\n", - "이전 이름 기반 스타일의 체크포인트도 여전히 복원할 수 있지만 주의가 필요합니다.\n", - "코드 변환 과정 때문에 변수 이름이 바뀔 수 있지만 해결 방법이 있습니다.\n", - "\n", - "가장 간단한 방법은 새로운 모델의 이름과 체크포인트에 있는 이름을 나열해 보는 것입니다:\n", - "\n", - "* 여전히 모든 변수는 설정 가능한 `name` 매개변수를 가집니다.\n", - "* 케라스 모델도 `name` 매개변수를 가집니다. 이 값은 변수 이름의 접두어로 사용됩니다.\n", - "* `tf.name_scope` 함수를 변수 이름의 접두어를 지정하는데 사용할 수 있습니다. 이 함수는 `tf.variable_scope`와는 매우 다릅니다. 이름에만 영향을 미치며 변수를 추적하거나 재사용을 관장하지 않습니다.\n", - "\n", - "이것이 주어진 상황에 잘 맞지 않는다면 `tf.compat.v1.train.init_from_checkpoint` 함수를 시도해 보세요. 이 함수는 `assignment_map` 매개변수로 예전 이름과 새로운 이름을 매핑할 수 있습니다.\n", - "\n", - "노트: [지연 적재](checkpoint.ipynb#loading_mechanics)가 되는 객체 기반 체크포인트와는 달리 이름 기반 체크포인트는 함수가 호출될 때 모든 변수가 만들어 집니다. 일부 모델은 `build` 메서드를 호출하거나 배치 데이터에서 모델을 실행할 때까지 변수 생성을 지연합니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "_ONjobDD6Uur" - }, - "cell_type": "markdown", - "source": [ - "### saved_model 호환성\n", - "\n", - "saved_model에는 심각한 호환성 문제가 없습니다.\n", - "\n", - "* 텐서플로 1.x의 saved_model은 텐서플로 2.0와 호환됩니다.\n", - "* 텐서플로 2.0의 saved_model로 저장한 모델도 연산이 지원된다면 TensorFlow 1.x에서 작동됩니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "ewl9P3oZ6ZtR" - }, - "cell_type": "markdown", - "source": [ - "## 추정기" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "YprVP9g3l6eG" - }, - "cell_type": "markdown", - "source": [ - "### 추정기로 훈련하기\n", - "\n", - "텐서플로 2.0은 추정기(estimator)를 지원합니다.\n", - "\n", - "추정기를 사용할 때 텐서플로 1.x의 `input_fn()`, `tf.estimator.TrainSpec`, `tf.estimator.EvalSpec`를 사용할 수 있습니다.\n", - "\n", - "다음은 `input_fn`을 사용하여 훈련과 평가를 수행하는 예입니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "N5kZeJsF8lS2" - }, - "cell_type": "markdown", - "source": [ - "#### input_fn과 훈련/평가 스펙 만들기" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "AOlXGO4J6jDh", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 추정기 input_fn을 정의합니다.\n", - "def input_fn():\n", - " datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n", - " mnist_train, mnist_test = datasets['train'], datasets['test']\n", - "\n", - " BUFFER_SIZE = 10000\n", - " BATCH_SIZE = 64\n", - "\n", - " def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - "\n", - " return image, label[..., tf.newaxis]\n", - "\n", - " train_data = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", - " return train_data.repeat()\n", - "\n", - "# 훈련과 평가 스펙을 정의합니다.\n", - "train_spec = tf.estimator.TrainSpec(input_fn=input_fn,\n", - " max_steps=STEPS_PER_EPOCH * NUM_EPOCHS)\n", - "eval_spec = tf.estimator.EvalSpec(input_fn=input_fn,\n", - " steps=STEPS_PER_EPOCH)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "_o6J48Nj9H5c" - }, - "cell_type": "markdown", - "source": [ - "### 케라스 모델 정의 사용하기" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "IXCQdhGq9SbB" - }, - "cell_type": "markdown", - "source": [ - "텐서플로 2.0에서 추정기를 구성하는 방법은 조금 다릅니다.\n", - "\n", - "케라스를 사용하여 모델을 정의하는 것을 권장합니다. 그 다음 `tf.keras.model_to_estimator` 유틸리티를 사용하여 모델을 추정기로 바꾸세요. 다음 코드는 추정기를 만들고 훈련할 때 이 유틸리티를 사용하는 방법을 보여 줍니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "aelsClm3Cq4I", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def make_model():\n", - " return tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " kernel_regularizer=tf.keras.regularizers.l2(0.02),\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dropout(0.1),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "HJb6f8dtl6rr", - "colab": {} - }, - "cell_type": "code", - "source": [ - "model = make_model()\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "estimator = tf.keras.estimator.model_to_estimator(\n", - " keras_model = model\n", - ")\n", - "\n", - "tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "-ptTxL1q6flL" - }, - "cell_type": "markdown", - "source": [ - "### 맞춤형 `model_fn` 사용하기\n", - "\n", - "기존에 작성한 맞춤형 추정기 `model_fn`을 유지해야 한다면 이 `model_fn`을 케라스 모델로 바꿀 수 있습니다.\n", - "\n", - "그러나 호환성 때문에 맞춤형 `model_fn`은 1.x 스타일의 그래프 모드로 실행될 것입니다. 즉 즉시 실행과 의존성 자동 제어가 없다는 뜻입니다.\n", - "\n", - "맞춤형 `model_fn`에 케라스 모델을 사용하는 것은 맞춤형 훈련 루프에 사용하는 것과 비슷합니다:\n", - "\n", - "* `mode` 매개변수에 기초하여 `training` 상태를 적절하게 지정하세요.\n", - "* 옵티마이저에 모델의 `trainable_variables`를 명시적으로 전달하세요.\n", - "\n", - "[맞춤형 루프](#custom_loop)와 큰 차이점은 다음과 같습니다:\n", - "\n", - "* `model.losses`를 사용하는 대신 `tf.keras.Model.get_losses_for` 사용하여 손실을 추출하세요.\n", - "* `tf.keras.Model.get_updates_for`를 사용하여 모델의 업데이트 값을 추출하세요.\n", - "\n", - "노트: \"업데이트(update)\"는 각 배치가 끝난 후에 모델에 적용해야 할 변화량입니다. 예를 들면 `tf.keras.layers.BatchNormalization` 층에서 평균과 분산의 이동 평균(moving average)이 있습니다.\n", - "\n", - "다음은 맞춤형 `model_fn`으로부터 추정기를 만드는 코드로 이런 개념을 잘 보여 줍니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "iY16eZKW606-", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def my_model_fn(features, labels, mode):\n", - " model = make_model()\n", - "\n", - " optimizer = tf.compat.v1.train.AdamOptimizer()\n", - " loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - " training = (mode == tf.estimator.ModeKeys.TRAIN)\n", - " predictions = model(features, training=training)\n", - "\n", - " reg_losses = model.get_losses_for(None) + model.get_losses_for(features)\n", - " total_loss = loss_fn(labels, predictions) + tf.math.add_n(reg_losses)\n", - "\n", - " accuracy = tf.compat.v1.metrics.accuracy(labels=labels,\n", - " predictions=tf.math.argmax(predictions, axis=1),\n", - " name='acc_op')\n", - "\n", - " update_ops = model.get_updates_for(None) + model.get_updates_for(features)\n", - "\n", - " with tf.control_dependencies(update_ops):\n", - " train_op = optimizer.minimize(\n", - " total_loss,\n", - " var_list=model.trainable_variables,\n", - " global_step=tf.compat.v1.train.get_or_create_global_step())\n", - "\n", - " return tf.estimator.EstimatorSpec(\n", - " mode=mode,\n", - " predictions=predictions,\n", - " loss=total_loss,\n", - " train_op=train_op, eval_metric_ops={'accuracy': accuracy})\n", - "\n", - "# 추정기를 만들고 훈련합니다.\n", - "estimator = tf.estimator.Estimator(model_fn=my_model_fn)\n", - "tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "dt8ct9XCFqls" - }, - "cell_type": "markdown", - "source": [ - "## TensorShape\n", - "\n", - "이 클래스는 `tf.compat.v1.Dimension` 객체 대신에 `int` 값을 가지도록 단순화되었습니다. 따라서 `int` 값을 얻기 위해 `.value()` 메서드를 호출할 필요가 없습니다.\n", - "\n", - "여전히 개별 `tf.compat.v1.Dimension` 객체는 `tf.TensorShape.dims`로 참조할 수 있습니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "x36cWcmM8Eu1" - }, - "cell_type": "markdown", - "source": [ - "다음 코드는 텐서플로 1.x와 텐서플로 2.0의 차이점을 보여줍니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "PbpD-kHOZR4A", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# TensorShape 객체를 만들고 인덱스를 참조합니다.\n", - "i = 0\n", - "shape = tf.TensorShape([16, None, 256])\n", - "shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "kDFck03neNy0" - }, - "cell_type": "markdown", - "source": [ - "TF 1.x에서는 다음과 같이 사용합니다:\n", - "\n", - "```python\n", - "value = shape[i].value\n", - "```\n", - "\n", - "TF 2.0에서는 다음과 같이 사용합니다:" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "KuR73QGEeNdH", - "colab": {} - }, - "cell_type": "code", - "source": [ - "value = shape[i]\n", - "value" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "bPWPNKRiZmkd" - }, - "cell_type": "markdown", - "source": [ - "TF 1.x에서는 다음과 같이 사용합니다:\n", - "\n", - "```python\n", - "for dim in shape:\n", - " value = dim.value\n", - " print(value)\n", - "```\n", - "\n", - "TF 2.0에서는 다음과 같이 사용합니다:" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "y6s0vuuprJfc", - "colab": {} - }, - "cell_type": "code", - "source": [ - "for value in shape:\n", - " print(value)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "YpRgngu3Zw-A" - }, - "cell_type": "markdown", - "source": [ - "TF 1.x에서는 다음과 같이 사용합니다(다른 Dimension 메서드를 사용할 때도):\n", - "\n", - "```python\n", - "dim = shape[i]\n", - "dim.assert_is_compatible_with(other_dim)\n", - "```\n", - "\n", - "TF 2.0에서는 다음과 같이 사용합니다:" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "LpViGEcUZDGX", - "colab": {} - }, - "cell_type": "code", - "source": [ - "other_dim = 16\n", - "Dimension = tf.compat.v1.Dimension\n", - "\n", - "if shape.rank is None:\n", - " dim = Dimension(None)\n", - "else:\n", - " dim = shape.dims[i]\n", - "dim.is_compatible_with(other_dim) # 다른 Dimension 메서드도 동일" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "GaiGe36dOdZ_", - "colab": {} - }, - "cell_type": "code", - "source": [ - "shape = tf.TensorShape(None)\n", - "\n", - "if shape:\n", - " dim = shape.dims[i]\n", - " dim.is_compatible_with(other_dim) # 다른 Dimension 메서드도 동일" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "3kLLY0I3PI-l" - }, - "cell_type": "markdown", - "source": [ - "랭크(rank)를 알 수 있다면 `tf.TensorShape`의 불리언 값은 `True`가 됩니다. 그렇지 않으면 `False`입니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "-Ow1ndKpOnJd", - "colab": {} - }, - "cell_type": "code", - "source": [ - "print(bool(tf.TensorShape([]))) # 스칼라\n", - "print(bool(tf.TensorShape([0]))) # 길이 0인 벡터\n", - "print(bool(tf.TensorShape([1]))) # 길이 1인 벡터\n", - "print(bool(tf.TensorShape([None]))) # 길이를 알 수 없는 벡터\n", - "print(bool(tf.TensorShape([1, 10, 100]))) # 3D 텐서\n", - "print(bool(tf.TensorShape([None, None, None]))) # 크기를 모르는 3D 텐서\n", - "print()\n", - "print(bool(tf.TensorShape(None))) # 랭크를 알 수 없는 텐서" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "lwswSCLT9g63" - }, - "cell_type": "markdown", - "source": [ - "## 그 밖의 동작 방식 변화\n", - "\n", - "텐서플로 2.0에는 몇 가지 동작 방식의 변화가 있습니다.\n", - "\n", - "### ResourceVariables\n", - "\n", - "텐서플로 2.0은 기본적으로 `RefVariables`가 아니라 `ResourceVariables`를 만듭니다.\n", - "\n", - "`ResourceVariables`는 쓰기 금지가 되어 있어서 직관적으로 일관성이 더 잘 보장됩니다.\n", - "\n", - "* 이는 극단적인 경우 동작 방식에 변화를 일으킬 수 있습니다.\n", - "* 경우에 따라서 추가적인 복사를 일으켜 메모리 사용량을 증가시킬 수 있습니다.\n", - "* `tf.Variable` 생성자에 `use_resource=False`를 전달하여 비활성화시킬 수 있습니다.\n", - "\n", - "### 제어 흐름\n", - "\n", - "제어 흐름 연산의 구현이 단순화되었습니다. 따라서 텐서플로 2.0에서는 다른 그래프가 생성됩니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "vKX6AdTAQhB-" - }, - "cell_type": "markdown", - "source": [ - "## 결론\n", - "\n", - "전체적인 과정은 다음과 같습니다:\n", - "\n", - "1. 업그레이드 스크립트를 실행하세요.\n", - "2. `contrib` 모듈을 삭제하세요.\n", - "3. 모델을 객체 지향 스타일(케라스)로 바꾸세요.\n", - "4. 가능한 `tf.keras`나 `tf.estimator`의 훈련과 평가 루프를 사용하세요.\n", - "5. 그렇지 않으면 맞춤형 루프를 사용하세요. 세션과 컬렉션은 사용하지 말아야 합니다.\n", - "\n", - "텐서플로 2.0 스타일로 코드를 바꾸려면 약간의 작업이 필요하지만 다음과 같은 장점을 얻을 수 있습니다:\n", - "\n", - "* 코드 라인이 줄어 듭니다.\n", - "* 명료하고 단순해집니다.\n", - "* 디버깅이 쉬워집니다." - ] - } - ] -} diff --git a/site/ko/guide/saved_model.ipynb b/site/ko/guide/saved_model.ipynb deleted file mode 100644 index 508376e03be..00000000000 --- a/site/ko/guide/saved_model.ipynb +++ /dev/null @@ -1,1176 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6bYaCABobL5q" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "FlUw7tSKbtg4" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xc1srSc51n_4" - }, - "source": [ - "# SavedModel 포맷 사용하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-nBUqG2rchGH" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/guide/saved_model\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " TensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/guide/saved_model.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " 구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/guide/saved_model.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " 깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "collapsed": false, - "id": "eHp7U1y1p51z" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/effective_tf2.md)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CPE-fshLTsXU" - }, - "source": [ - "SavedModel에는 가중치 및 연산을 포함한 완전한 텐서플로 프로그램이 포함됩니다. 기존에 설계했던 모델 코드를 실행할 필요가 없어 공유하거나 ([TFLite](https://tensorflow.org/lite), [TensorFlow.js](https://js.tensorflow.org/), [TensorFlow Serving](https://www.tensorflow.org/tfx/serving/tutorials/Serving_REST_simple), [TFHub](https://tensorflow.org/hub)와 같은 환경으로) 배포하는 데 유용합니다.\n", - "\n", - "파이썬 모델 코드를 가지고 있고 파이썬 내에서 가중치를 불러오고 싶다면, [체크포인트 훈련 가이드](./checkpoint.ipynb)를 참조하세요.\n", - "\n", - "빠른 소개를 위해 이 섹션에서는 미리 훈련된 케라스 모델을 내보내고 그 모델로 이미지 분류 요청을 처리합니다. 나머지 가이드에서는 세부 정보와 SavedModel을 만드는 다른 방법에 대해 설명합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Le5OB-fBHHW7" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from matplotlib import pyplot as plt\n", - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SofdPKo0G8Lb" - }, - "outputs": [], - "source": [ - "file = tf.keras.utils.get_file(\n", - " \"grace_hopper.jpg\",\n", - " \"https://storage.googleapis.com/download.tensorflow.org/example_images/grace_hopper.jpg\")\n", - "img = tf.keras.preprocessing.image.load_img(file, target_size=[224, 224])\n", - "plt.imshow(img)\n", - "plt.axis('off')\n", - "x = tf.keras.preprocessing.image.img_to_array(img)\n", - "x = tf.keras.applications.mobilenet.preprocess_input(\n", - " x[tf.newaxis,...])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sqVcFL10JkF0" - }, - "source": [ - "실행 예제로 그레이스 호퍼(Grace Hopper)의 이미지와 사용이 쉬운 케라스 사전 훈련 이미지 분류 모델을 사용할 것입니다. 사용자 정의 모델도 사용할 수 있는데, 자세한 것은 나중에 설명합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JhVecdzJTsKE" - }, - "outputs": [], - "source": [ - "#tf.keras.applications.vgg19.decode_predictions\n", - "labels_path = tf.keras.utils.get_file('ImageNetLabels.txt','https://storage.googleapis.com/download.tensorflow.org/data/ImageNetLabels.txt')\n", - "imagenet_labels = np.array(open(labels_path).read().splitlines())" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aEHSYjW6JZHV" - }, - "outputs": [], - "source": [ - "pretrained_model = tf.keras.applications.MobileNet()\n", - "result_before_save = pretrained_model(x)\n", - "print()\n", - "\n", - "decoded = imagenet_labels[np.argsort(result_before_save)[0,::-1][:5]+1]\n", - "\n", - "print(\"저장 전 결과:\\n\", decoded)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r4KIsQDZJ5PS" - }, - "source": [ - "이 이미지의 가장 가능성 있는 예측은 \"군복\"입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8nfznDmHCW6F" - }, - "outputs": [], - "source": [ - "tf.saved_model.save(pretrained_model, \"/tmp/mobilenet/1/\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pyX-ETE3wX63" - }, - "source": [ - "저장 경로의 마지막 경로 요소(여기서는 `1/`)는 모델의 버전 번호인 텐서플로 서빙(TensorFlow Serving) 컨벤션을 따릅니다 - 텐서플로 서빙과 같은 도구가 최신 모델을 구분할 수 있게 합니다.\n", - "\n", - "SavedModel은 시그니처(signatures)라 불리는 이름있는 함수를 가집니다. 케라스 모델은 `serving_default` 시그니처 키를 사용하여 정방향 패스(forward pass)를 내보냅니다. [SavedModel 커맨드 라인 인터페이스](#details_of_the_savedmodel_command_line_interface)는 디스크에 저장된 SavedModel을 검사할 때 유용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "djmcTavtIZyT" - }, - "outputs": [], - "source": [ - "!saved_model_cli show --dir /tmp/mobilenet/1 --tag_set serve --signature_def serving_default" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VCZZ8avqLF1g" - }, - "source": [ - "파이썬에서 `tf.saved_model.load`로 SavedModel을 다시 불러오고 해군대장 호퍼(Admiral Hopper)의 이미지가 어떻게 분류되는지 볼 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NP2UpVFRV7N_" - }, - "outputs": [], - "source": [ - "loaded = tf.saved_model.load(\"/tmp/mobilenet/1/\")\n", - "print(list(loaded.signatures.keys())) # [\"serving_default\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K5srGzowfWff" - }, - "source": [ - "가져온 시그니처는 항상 딕셔너리를 반환합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ChFLpegYfQGR" - }, - "outputs": [], - "source": [ - "infer = loaded.signatures[\"serving_default\"]\n", - "print(infer.structured_outputs)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cJYyZnptfuru" - }, - "source": [ - "SavedModel로부터 추론을 실행하면 처음 모델과 같은 결과를 제공합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9WjGEaS3XfX7" - }, - "outputs": [], - "source": [ - "labeling = infer(tf.constant(x))[pretrained_model.output_names[0]]\n", - "\n", - "decoded = imagenet_labels[np.argsort(labeling)[0,::-1][:5]+1]\n", - "\n", - "print(\"저장과 불러오기 이후의 결과:\\n\", decoded)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SJEkdXjTWbtl" - }, - "source": [ - "## 텐서플로 서빙으로 모델 배포하기\n", - "\n", - "SavedModel은 파이썬에서 사용하기에 적합하지만, 일반적으로 프로덕션 환경에서는 추론을 위한 전용 서비스를 사용합니다. 이는 텐서플로 서빙을 사용한 SavedModel로 쉽게 구성할 수 있습니다.\n", - "\n", - "`tensorflow_model_server`를 노트북이나 로컬 머신에 설치하는 방법을 포함한 텐서플로 서빙에 대한 자세한 내용은 [TensorFlow Serving REST 튜토리얼](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/tutorials/Serving_REST_simple.ipynb)을 참조하십시오. 간단한 예를 들면 앞서 내보낸 `mobilenet` 모델을 배포하기 위해 모델 경로를 SavedModel 디렉토리로 설정합니다:\n", - "\n", - "```bash\n", - "nohup tensorflow_model_server \\\n", - " --rest_api_port=8501 \\\n", - " --model_name=mobilenet \\\n", - " --model_base_path=\"/tmp/mobilenet\" \u003eserver.log 2\u003e\u00261\n", - "```\n", - "\n", - " 이제 요청을 보냅니다.\n", - "\n", - "```python\n", - "!pip install requests\n", - "import json\n", - "import numpy\n", - "import requests\n", - "data = json.dumps({\"signature_name\": \"serving_default\",\n", - " \"instances\": x.tolist()})\n", - "headers = {\"content-type\": \"application/json\"}\n", - "json_response = requests.post('http://localhost:8501/v1/models/mobilenet:predict',\n", - " data=data, headers=headers)\n", - "predictions = numpy.array(json.loads(json_response.text)[\"predictions\"])\n", - "```\n", - "\n", - "`predictions`의 결과는 파이썬에서와 같습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bi0ILzu1XdWw" - }, - "source": [ - "### SavedModel 포맷\n", - "\n", - "SavedModel은 변수값과 상수를 포함하고 직렬화된 시그니처와 이를 실행하는 데 필요한 상태를 담은 디렉토리입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6u3YZuYZXyTO" - }, - "outputs": [], - "source": [ - "!ls /tmp/mobilenet/1 # assets\tsaved_model.pb\tvariables" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ple4X5utX8ue" - }, - "source": [ - "`saved_model.pb` 파일은 각각 하나의 함수로 된 이름있는 시그니처 세트를 포함합니다.\n", - "\n", - "SavedModel에는 다중 시그니처 세트(`saved_model_cli`의 `tag_set` 매개변수 값으로 확인된 다중 MetaGraph)를 포함할 수 있지만 이런 경우는 드뭅니다. 다중 시그니처 세트를 작성하는 API에는 [`tf.Estimator.experimental_export_all_saved_models`](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator#experimental_export_all_saved_models) 및 TensorFlow 1.x의 `tf.saved_model.Builder`가 포함됩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Pus0dOYTYXbI" - }, - "outputs": [], - "source": [ - "!saved_model_cli show --dir /tmp/mobilenet/1 --tag_set serve" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eALHpGvRZOhk" - }, - "source": [ - "`variables` 디렉토리에는 일반적인 훈련 체크포인트 파일이 있습니다([훈련 체크포인트 가이드](./checkpoint.ipynb) 참조)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EDYqhDlNZAC2" - }, - "outputs": [], - "source": [ - "!ls /tmp/mobilenet/1/variables" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VKmaZQpHahGh" - }, - "source": [ - "`assets` 디렉토리에는 텐서플로 그래프(TensorFlow graph)에서 사용되는 파일들, 예를 들어 상수 테이블을 초기화하는 데 사용되는 텍스트 파일들이 있습니다. 이번 예제에서는 사용되지 않습니다.\n", - "\n", - "SavedModel은 텐서플로 그래프에서 사용되지 않는 파일을 위해 `assets.extra` 디렉토리를 가질 수 있는데, 예를 들면 사용자가 SavedModel과 함께 사용할 파일입니다. 텐서플로 자체는 이 디렉토리를 사용하지 않습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zIceoF_CYmaF" - }, - "source": [ - "### 사용자 정의 모델 내보내기\n", - "\n", - "첫 번째 섹션에서는, `tf.saved_model.save`가 `tf.keras.Model` 객체에 대한 시그니처를 자동으로 결정했습니다. 이는 케라스의 `Model` 객체가 내보내기 위한 명시적 메서드와 입력 크기를 가지기 때문에 작동했습니다. `tf.saved_model.save`는 저수준(low-level) 모델 설계 API와도 잘 작동하지만, 모델을 텐서플로 서빙에 배포할 계획이라면 시그니처로 사용할 함수를 지정해야 합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6EPvKiqXMm3d" - }, - "outputs": [], - "source": [ - "class CustomModule(tf.Module):\n", - "\n", - " def __init__(self):\n", - " super(CustomModule, self).__init__()\n", - " self.v = tf.Variable(1.)\n", - "\n", - " @tf.function\n", - " def __call__(self, x):\n", - " return x * self.v\n", - "\n", - " @tf.function(input_signature=[tf.TensorSpec([], tf.float32)])\n", - " def mutate(self, new_v):\n", - " self.v.assign(new_v)\n", - "\n", - "module = CustomModule()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fUrCTSK2HV2b" - }, - "source": [ - "이 모듈은 `tf.function` 데코레이터가 적용된 두 메서드를 가지고 있습니다. 이 함수들은 SavedModel에 포함되어 있으므로 `tf.saved_model.load` 함수를 사용하여 파이썬 프로그램에 함께 로드됩니다. 하지만 명시적 선언 없이는 텐서플로 서빙과 같은 시그니처 배포 도구와 `saved_model_cli`가 접근할 수 없습니다.\n", - "\n", - "`module.mutate`는 `input_signature`를 가지고 있어서 계산 그래프를 SavedModel에 저장하기 위한 정보가 이미 충분히 있습니다. `__call__`은 시그니처가 없기에 저장하기 전 이 메서드를 호출해야 합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "85PUO9iWH7xn" - }, - "outputs": [], - "source": [ - "module(tf.constant(0.))\n", - "tf.saved_model.save(module, \"/tmp/module_no_signatures\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eyWD4wr-Ng7m" - }, - "source": [ - "`input_signature`가 없는 함수의 경우, 저장 전에 사용된 입력의 크기는 함수가 불려진 이후에 사용될 것입니다. 스칼라값으로 `__call__`을 호출했으므로 스칼라값만 받아들일 것입니다" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xy7oCex1Ibj1" - }, - "outputs": [], - "source": [ - "imported = tf.saved_model.load(\"/tmp/module_no_signatures\")\n", - "assert 3. == imported(tf.constant(3.)).numpy()\n", - "imported.mutate(tf.constant(2.))\n", - "assert 6. == imported(tf.constant(3.)).numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lbLNVfVJOTfb" - }, - "source": [ - "함수는 벡터와 같은 새로운 형식을 수용하지 않습니다.\n", - "\n", - "```python\n", - "imported(tf.constant([3.]))\n", - "```\n", - "\n", - "\u003cpre\u003e\n", - "ValueError: Could not find matching function to call for canonicalized inputs ((\u003ctf.Tensor 'args_0:0' shape=(1,) dtype=float32\u003e,), {}). Only existing signatures are [((TensorSpec(shape=(), dtype=tf.float32, name=u'x'),), {})].\n", - "\u003c/pre\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WrJqD-epPGnr" - }, - "source": [ - "`get_concrete_function`을 사용해 입력 크기를 함수 호출 없이 추가할 수 있습니다. 이 함수는 매개변수 값으로 `Tensor` 대신 입력 크기와 데이터 타입을 나타내는 `tf.TensorSpec` 객체를 받습니다. 크기가 `None`이면 모든 크기가 수용 가능합니다. 또는 각 축의 크기(axis size)를 담은 리스트일 수도 있습니다. 축 크기가 'None'이면 그 축에 대해 임의의 크기를 사용할 수 있습니다. 또한 `tf.TensorSpecs`는 이름을 가질 수 있는데, 기본값은 함수의 매개변수 키워드(여기서는 \"x\")입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1m9Okb75PFmb" - }, - "outputs": [], - "source": [ - "module.__call__.get_concrete_function(x=tf.TensorSpec([None], tf.float32))\n", - "tf.saved_model.save(module, \"/tmp/module_no_signatures\")\n", - "imported = tf.saved_model.load(\"/tmp/module_no_signatures\")\n", - "assert [3.] == imported(tf.constant([3.])).numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gvy3GFl4IfSW" - }, - "source": [ - "`tf.keras.Model`과 `tf.Module`과 같은 객체에 포함된 함수와 변수는 가져올 때 사용할 수 있지만 많은 파이썬의 타입과 속성은 잃어버립니다. 파이썬 프로그램 자체는 SavedModel에 저장되지 않습니다.\n", - "\n", - "내보낼 함수를 시그니처로 지정하지 못했기에 시그니처는 없습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uNTV6o_TIeRu" - }, - "outputs": [], - "source": [ - "!saved_model_cli show --dir /tmp/module_no_signatures --tag_set serve" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BiNtaMZSI8Tb" - }, - "source": [ - "## 내보낼 시그니처 지정하기\n", - "\n", - "어떤 함수가 시그니처라는 것을 나타내려면 저장할 때 `signatures` 매개변수를 지정합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_pAdgIORR2yH" - }, - "outputs": [], - "source": [ - "call = module.__call__.get_concrete_function(tf.TensorSpec(None, tf.float32))\n", - "tf.saved_model.save(module, \"/tmp/module_with_signature\", signatures=call)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lHiBm-kdKBmG" - }, - "source": [ - "먼저 `tf.function` 객체를 `get_concrete_function` 메서드를 사용해 `ConcreteFunction` 객체로 바꾸었습니다. 이것은 함수가 고정된 `input_signature` 없이 만들어지고 함수와 연관된 명시적인 `Tensor` 입력이 없었으므로 필수적입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nAzRHR0UT4hv" - }, - "outputs": [], - "source": [ - "!saved_model_cli show --dir /tmp/module_with_signature --tag_set serve --signature_def serving_default" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0B25WsscTZoC" - }, - "outputs": [], - "source": [ - "imported = tf.saved_model.load(\"/tmp/module_with_signature\")\n", - "signature = imported.signatures[\"serving_default\"]\n", - "assert [3.] == signature(x=tf.constant([3.]))[\"output_0\"].numpy()\n", - "imported.mutate(tf.constant(2.))\n", - "assert [6.] == signature(x=tf.constant([3.]))[\"output_0\"].numpy()\n", - "assert 2. == imported.v.numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_gH91j1IR4tq" - }, - "source": [ - "하나의 시그니처를 내보냈고 키는 기본값인 \"serving_default\"가 됩니다. 여러 시그니처를 내보내려면 딕셔너리로 전달합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6VYAiQmLUiox" - }, - "outputs": [], - "source": [ - "@tf.function(input_signature=[tf.TensorSpec([], tf.string)])\n", - "def parse_string(string_input):\n", - " return imported(tf.strings.to_number(string_input))\n", - "\n", - "signatures = {\"serving_default\": parse_string,\n", - " \"from_float\": imported.signatures[\"serving_default\"]}\n", - "\n", - "tf.saved_model.save(imported, \"/tmp/module_with_multiple_signatures\", signatures)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8IPx_0RWEx07" - }, - "outputs": [], - "source": [ - "!saved_model_cli show --dir /tmp/module_with_multiple_signatures --tag_set serve" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sRdSPjKFfQpx" - }, - "source": [ - "`saved_model_cli`는 커맨드 라인에서 SavedModel을 직접 실행할 수도 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hcHmgVytfODo" - }, - "outputs": [], - "source": [ - "!saved_model_cli run --dir /tmp/module_with_multiple_signatures --tag_set serve --signature_def serving_default --input_exprs=\"string_input='3.'\"\n", - "!saved_model_cli run --dir /tmp/module_with_multiple_signatures --tag_set serve --signature_def from_float --input_exprs=\"x=3.\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WiNhHa_Ne82K" - }, - "source": [ - "## 가져온 모델 미세 튜닝하기\n", - "\n", - "변수 객체가 사용 가능하므로 imported 함수를 통해 역전파할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mSchcIB2e-n0" - }, - "outputs": [], - "source": [ - "optimizer = tf.optimizers.SGD(0.05)\n", - "\n", - "def train_step():\n", - " with tf.GradientTape() as tape:\n", - " loss = (10. - imported(tf.constant(2.))) ** 2\n", - " variables = tape.watched_variables()\n", - " grads = tape.gradient(loss, variables)\n", - " optimizer.apply_gradients(zip(grads, variables))\n", - " return loss" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Yx9bO2taJJxm" - }, - "outputs": [], - "source": [ - "for _ in range(10):\n", - " # \"v\"는 5로 수렴, \"loss\"는 0으로 수렴\n", - " print(\"loss={:.2f} v={:.2f}\".format(train_step(), imported.v.numpy()))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qyL9tOPrg5Zw" - }, - "source": [ - "## SavedModel의 제어 흐름\n", - "\n", - "`tf.function`에 들어갈 수 있는 것은 모두 SavedModel에 들어갈 수 있습니다. [AutoGraph](./function.ipynb)를 사용하면 Tensor에 의존하는 조건부 논리를 파이썬 제어 흐름으로 표현할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tfbh3uGMgBpH" - }, - "outputs": [], - "source": [ - "@tf.function(input_signature=[tf.TensorSpec([], tf.int32)])\n", - "def control_flow(x):\n", - " if x \u003c 0:\n", - " tf.print(\"유효하지 않음!\")\n", - " else:\n", - " tf.print(x % 3)\n", - "\n", - "to_export = tf.Module()\n", - "to_export.control_flow = control_flow\n", - "tf.saved_model.save(to_export, \"/tmp/control_flow\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bv4EXevIjHch" - }, - "outputs": [], - "source": [ - "imported = tf.saved_model.load(\"/tmp/control_flow\")\n", - "imported.control_flow(tf.constant(-1)) # 유효하지 않음!\n", - "imported.control_flow(tf.constant(2)) # 2\n", - "imported.control_flow(tf.constant(3)) # 0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dk5wWyuMpuHx" - }, - "source": [ - "## 추정기(Estimator)의 SavedModel\n", - "\n", - "추정기는 [`tf.Estimator.export_saved_model`](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator#export_saved_model)을 통해 SavedModel을 내보냅니다. 자세한 내용은 [Estimator 가이드](https://www.tensorflow.org/guide/estimator)를 참조하십시오." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "B9KQq5qzpzbK" - }, - "outputs": [], - "source": [ - "input_column = tf.feature_column.numeric_column(\"x\")\n", - "estimator = tf.estimator.LinearClassifier(feature_columns=[input_column])\n", - "\n", - "def input_fn():\n", - " return tf.data.Dataset.from_tensor_slices(\n", - " ({\"x\": [1., 2., 3., 4.]}, [1, 1, 0, 0])).repeat(200).shuffle(64).batch(16)\n", - "estimator.train(input_fn)\n", - "\n", - "serving_input_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(\n", - " tf.feature_column.make_parse_example_spec([input_column]))\n", - "export_path = estimator.export_saved_model(\n", - " \"/tmp/from_estimator/\", serving_input_fn)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XJ4PJ-Cl4060" - }, - "source": [ - "이 SavedModel은 텐서플로 서빙에 배포하는 데 유용한 직렬화된 `tf.Example` 프로토콜 버퍼를 사용합니다. 그러나 `tf.saved_model.load`로 불러오고 파이썬에서 실행할 수도 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "c_BUBBNB1UH9" - }, - "outputs": [], - "source": [ - "imported = tf.saved_model.load(export_path)\n", - "\n", - "def predict(x):\n", - " example = tf.train.Example()\n", - " example.features.feature[\"x\"].float_list.value.extend([x])\n", - " return imported.signatures[\"predict\"](\n", - " examples=tf.constant([example.SerializeToString()]))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "C1ylWZCQ1ahG" - }, - "outputs": [], - "source": [ - "print(predict(1.5))\n", - "print(predict(3.5))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_IrCCm0-isqA" - }, - "source": [ - "`tf.estimator.export.build_server_input_receiver_fn`를 사용해 `tf.train.Example`이 아닌 원시 텐서를 가지는 입력 함수를 만들 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Co6fDbzw_UnD" - }, - "source": [ - "## C++에서 SavedModel 불러오기\n", - "\n", - "SavedModel의 C++ 버전 [loader](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/loader.h)는 SessionOptions 및 RunOptions을 허용하며 경로에서 SavedModel을 불러오는 API를 제공합니다. 불러 올 그래프와 연관된 태그를 지정해야합니다. 불러온 SavedModel의 버전은 SavedModelBundle이라고 하며 MetaGraphDef와 불러온 세션을 포함합니다.\n", - "\n", - "```C++\n", - "const string export_dir = ...\n", - "SavedModelBundle bundle;\n", - "...\n", - "LoadSavedModel(session_options, run_options, export_dir, {kSavedModelTagTrain},\n", - " \u0026bundle);\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b33KuyEuAO3Z" - }, - "source": [ - "\u003ca id=saved_model_cli/\u003e\n", - "\n", - "## SavedModel 커맨드 라인 인터페이스 세부 사항\n", - "\n", - "SavedModel 커맨드 라인 인터페이스(CLI)를 사용하여 SavedModel을 검사하고 실행할 수 있습니다.\n", - "예를 들어, CLI를 사용하여 모델의 `SignatureDef`를 검사할 수 있습니다.\n", - "CLI를 사용하면 입력 Tensor 크기 및 데이터 타입이 모델과 일치하는지 신속하게 확인할 수 있습니다.\n", - "또한 모델을 테스트하려는 경우 다양한 형식(예를 들어, 파이썬 표현식)의 샘플 입력을\n", - "전달하고 출력을 가져와 CLI를 사용하여 정확성 검사를 수행할 수 있습니다.\n", - "\n", - "### SavedModel CLI 설치하기\n", - "\n", - "대체로 말하자면 다음 두 가지 방법 중 하나로 텐서플로를 설치할 수 있습니다:\n", - "\n", - "* 사전에 빌드된 텐서플로 바이너리로 설치\n", - "* 소스 코드로 텐서플로 빌드\n", - "\n", - "사전에 빌드된 텐서플로 바이너리를 통해 설치한 경우 SavedModel CLI가 이미 \n", - "시스템 경로 `bin\\saved_model_cli`에 설치되어 있습니다.\n", - "\n", - "소스 코드에서 텐서플로를 빌드하는 경우 다음 추가 명령을 실행하여 `saved_model_cli`를 빌드해야 합니다:\n", - "\n", - "```\n", - "$ bazel build tensorflow/python/tools:saved_model_cli\n", - "```\n", - "\n", - "### 명령 개요\n", - "\n", - "SavedModel CLI는 SavedModel의 `MetaGraphDef`에 대해 다음 두 명령어를 지원합니다:\n", - "\n", - "* SavedModel의 `MetaGraphDef`에 대한 계산을 보여주는 `show`\n", - "* `MetaGraphDef`에 대한 계산을 실행하는 `run`\n", - "\n", - "\n", - "### `show` 명령어\n", - "\n", - "SavedModel은 태그 세트로 식별되는 하나 이상의 `MetaGraphDef`를 포함합니다.\n", - "모델을 텐서플로 서빙에 배포하려면, 각 모델에 어떤 종류의 `SignatureDef`가 있는지, 그리고 입력과 출력은 무엇인지 궁금할 수 있습니다.\n", - "`show` 명령은 SavedModel의 내용을 계층적 순서로 검사합니다. 구문은 다음과 같습니다:\n", - "\n", - "```\n", - "usage: saved_model_cli show [-h] --dir DIR [--all]\n", - "[--tag_set TAG_SET] [--signature_def SIGNATURE_DEF_KEY]\n", - "```\n", - "\n", - "예를 들어, 다음 명령은 SavedModel에서 사용 가능한 모든 `MetaGraphDef` 태그 세트를 보여줍니다:\n", - "\n", - "```\n", - "$ saved_model_cli show --dir /tmp/saved_model_dir\n", - "The given SavedModel contains the following tag-sets:\n", - "serve\n", - "serve, gpu\n", - "```\n", - "\n", - "다음 명령은 `MetaGraphDef`에서 사용 가능한 모든 `SignatureDef` 키를 보여줍니다:\n", - "\n", - "```\n", - "$ saved_model_cli show --dir /tmp/saved_model_dir --tag_set serve\n", - "The given SavedModel `MetaGraphDef` contains `SignatureDefs` with the\n", - "following keys:\n", - "SignatureDef key: \"classify_x2_to_y3\"\n", - "SignatureDef key: \"classify_x_to_y\"\n", - "SignatureDef key: \"regress_x2_to_y3\"\n", - "SignatureDef key: \"regress_x_to_y\"\n", - "SignatureDef key: \"regress_x_to_y2\"\n", - "SignatureDef key: \"serving_default\"\n", - "```\n", - "\n", - "`MetaGraphDef`가 태그 세트에 *여러 개의* 태그를 가지고 있는 경우, 모든 태그를 지정해야 하며,\n", - "각 태그는 쉼표로 구분해야 합니다. 예를 들어:\n", - "\n", - "\u003cpre\u003e\n", - "$ saved_model_cli show --dir /tmp/saved_model_dir --tag_set serve,gpu\n", - "\u003c/pre\u003e\n", - "\n", - "특정 `SignatureDef`에 대한 모든 입력 및 출력 텐서 정보(TensorInfo)를 표시하려면 `SignatureDef` 키를\n", - "`signature_def` 옵션으로 전달하십시오. 이것은 나중에 계산 그래프를 실행하기 위해 입력 텐서의 텐서 키 값,\n", - "크기 및 데이터 타입을 알고자 할 때 매우 유용합니다. 예를 들어:\n", - "\n", - "```\n", - "$ saved_model_cli show --dir \\\n", - "/tmp/saved_model_dir --tag_set serve --signature_def serving_default\n", - "The given SavedModel SignatureDef contains the following input(s):\n", - " inputs['x'] tensor_info:\n", - " dtype: DT_FLOAT\n", - " shape: (-1, 1)\n", - " name: x:0\n", - "The given SavedModel SignatureDef contains the following output(s):\n", - " outputs['y'] tensor_info:\n", - " dtype: DT_FLOAT\n", - " shape: (-1, 1)\n", - " name: y:0\n", - "Method name is: tensorflow/serving/predict\n", - "```\n", - "\n", - "SavedModel에 사용 가능한 모든 정보를 표시하려면 `--all` 옵션을 사용하십시오. 예를 들어:\n", - "\n", - "\u003cpre\u003e\n", - "$ saved_model_cli show --dir /tmp/saved_model_dir --all\n", - "MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:\n", - "\n", - "signature_def['classify_x2_to_y3']:\n", - " The given SavedModel SignatureDef contains the following input(s):\n", - " inputs['inputs'] tensor_info:\n", - " dtype: DT_FLOAT\n", - " shape: (-1, 1)\n", - " name: x2:0\n", - " The given SavedModel SignatureDef contains the following output(s):\n", - " outputs['scores'] tensor_info:\n", - " dtype: DT_FLOAT\n", - " shape: (-1, 1)\n", - " name: y3:0\n", - " Method name is: tensorflow/serving/classify\n", - "\n", - "...\n", - "\n", - "signature_def['serving_default']:\n", - " The given SavedModel SignatureDef contains the following input(s):\n", - " inputs['x'] tensor_info:\n", - " dtype: DT_FLOAT\n", - " shape: (-1, 1)\n", - " name: x:0\n", - " The given SavedModel SignatureDef contains the following output(s):\n", - " outputs['y'] tensor_info:\n", - " dtype: DT_FLOAT\n", - " shape: (-1, 1)\n", - " name: y:0\n", - " Method name is: tensorflow/serving/predict\n", - "\u003c/pre\u003e\n", - "\n", - "\n", - "### `run` 명령어\n", - "\n", - "`run` 명령을 호출하여 그래프 계산을 실행하고, 입력을 전달한 다음 출력을 표시(하고 선택적으로 저장)합니다.\n", - "구문은 다음과 같습니다:\n", - "\n", - "```\n", - "usage: saved_model_cli run [-h] --dir DIR --tag_set TAG_SET --signature_def\n", - " SIGNATURE_DEF_KEY [--inputs INPUTS]\n", - " [--input_exprs INPUT_EXPRS]\n", - " [--input_examples INPUT_EXAMPLES] [--outdir OUTDIR]\n", - " [--overwrite] [--tf_debug]\n", - "```\n", - "\n", - "`run` 명령은 입력을 모델에 전달하는 다음 세 가지 방법을 제공합니다:\n", - "\n", - "* `--inputs` 옵션을 사용하여 넘파이(numpy) ndarray를 파일에 전달할 수 있습니다.\n", - "* `--input_exprs` 옵션을 사용하여 파이썬 표현식을 전달할 수 있습니다.\n", - "* `--input_examples` 옵션을 사용하여 `tf.train.Example`을 전달할 수 있습니다.\n", - "\n", - "#### `--inputs`\n", - "\n", - "입력 데이터를 파일에 전달하려면, 다음과 같은 일반적인 형식을 가지는 `--inputs` 옵션을 지정합니다:\n", - "\n", - "```bsh\n", - "--inputs \u003cINPUTS\u003e\n", - "```\n", - "\n", - "여기서 *INPUTS*는 다음 형식 중 하나입니다:\n", - "\n", - "* `\u003cinput_key\u003e=\u003cfilename\u003e`\n", - "* `\u003cinput_key\u003e=\u003cfilename\u003e[\u003cvariable_name\u003e]`\n", - "\n", - "여러 개의 *INPUTS*를 전달할 수 있습니다. 여러 입력을 전달하는 경우 세미콜론을 사용하여 각 *INPUTS*를 구분하십시오.\n", - "\n", - "`saved_model_cli`는 `numpy.load`를 사용하여 *filename*을 불러옵니다.\n", - "*filename*은 다음 형식 중 하나일 수 있습니다:\n", - "\n", - "* `.npy`\n", - "* `.npz`\n", - "* 피클(pickle) 포맷\n", - "\n", - "`.npy` 파일은 항상 넘파이 ndarray를 포함합니다. 그러므로 `.npy` 파일에서 불러올 때,\n", - "배열 내용이 지정된 입력 텐서에 직접 할당될 것입니다. 해당 `.npy` 파일과 함께 *variable_name*을 지정하면\n", - "*variable_name*이 무시되고 경고가 발생합니다.\n", - "\n", - "`.npz`(zip) 파일에서 불러올 때, 입력 텐서 키로 불러올 zip 파일 내의 변수를 *variable_name*으로\n", - "선택적으로 지정할 수 있습니다. *variable_name*을 지정하지 않으면 SavedModel CLI는 zip 파일에 하나의 파일만\n", - "포함되어 있는지 확인하고 지정된 입력 텐서 키로 불러옵니다.\n", - "\n", - "피클 파일에서 불러올 때, 대괄호 안에 `variable_name`이 지정되지 않았다면, 피클 파일 안에 있는\n", - "어떤 것이라도 지정된 입력 텐서 키로 전달될 것입니다. 그렇지 않으면, SavedModel CLI는 피클 파일에\n", - "딕셔너리가 저장되어 있다고 가정하고 *variable_name*에 해당하는 값이 사용됩니다.\n", - "\n", - "#### `--input_exprs`\n", - "\n", - "파이썬 표현식을 통해 입력을 전달하려면 `--input_exprs` 옵션을 지정하십시오. 이는 데이터 파일이 없어도\n", - "모델의 `SignatureDef`의 크기 및 데이터 타입과 일치하는 간단한 입력으로 모델의 정확성 검사를 하려는 경우\n", - "유용할 수 있습니다. 예를 들어:\n", - "\n", - "```bsh\n", - "`\u003cinput_key\u003e=[[1],[2],[3]]`\n", - "```\n", - "\n", - "파이썬 표현식 외에도 넘파이 함수를 전달할 수 있습니다. 예를 들어:\n", - "\n", - "```bsh\n", - "`\u003cinput_key\u003e=np.ones((32,32,3))`\n", - "```\n", - "\n", - "(`numpy` 모듈은 `np`로 이미 사용 가능하다고 가정합니다.)\n", - "\n", - "\n", - "#### `--input_examples`\n", - "\n", - "`tf.train.Example`을 입력으로 전달하려면 `--input_examples` 옵션을 지정하십시오. 입력 키마다 딕셔너리의\n", - "리스트를 받습니다. 각 딕셔너리는 `tf.train.Example`의 인스턴스입니다. 딕셔너리 키는 기능이며 값은 각 기능의\n", - "값 리스트입니다. 예를 들어:\n", - "\n", - "```bsh\n", - "`\u003cinput_key\u003e=[{\"age\":[22,24],\"education\":[\"BS\",\"MS\"]}]`\n", - "```\n", - "\n", - "#### 출력 저장\n", - "\n", - "기본적으로, SavedModel CLI는 출력을 stdout에 기록합니다. `--outdir` 옵션으로 디렉토리를 전달하면,\n", - "지정된 디렉토리 안에 출력 텐서 키의 이름을 따라 .npy 파일로 출력이 저장됩니다.\n", - "\n", - "기존 출력 파일을 덮어 쓰려면 `--overwrite`를 사용하십시오." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "saved_model.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/guide/tensor.md b/site/ko/guide/tensor.md deleted file mode 100644 index f991916aa29..00000000000 --- a/site/ko/guide/tensor.md +++ /dev/null @@ -1,272 +0,0 @@ -# 텐서플로 텐서 - -이름에서 알 수 있듯이, 텐서플로는 텐서를 포함한 계산을 정의하고 실행하는 프레임워크입니다. -*텐서(tensor)*는 벡터와 행렬을 일반화한 것이고 고차원으로 확장 가능합니다. -내부적으로 텐서플로는 기본적으로 제공되는 자료형을 사용해 n-차원 배열로 나타냅니다. - -텐서플로 프로그램을 작성할 때, `tf.Tensor`를 주로 조작하고 전달합니다. -`tf.Tensor` 객체는 부분적으로 정의된 연산으로 표현되고 이것은 결국 값으로 변환됩니다. -텐서플로 프로그램은 `tf.Tensor` 객체 그래프를 만드는 것으로 먼저 시작하고, -각각의 텐서가 다른 텐서를 기반으로 어떤 식으로 계산될 수 있는지 구체화하고, -그 다음 그래프를 실행해서 원하는 결과를 얻게 됩니다. - -`tf.Tensor`는 다음과 같은 속성을 가지고 있습니다: - - * 자료형 (예를 들어, `float32` 또는 `int32`, `string`) - * 형태(shape) - - -텐서안의 각각 원소는 동일한 자료형이고 항상 그 자료형을 알 수 있습니다. -형태(즉, 차원 수와 각 차원마다 길이)는 일부만 알 수 있습니다. -대부분 연산은 입력값 형태를 알 수 있다면 형태가 완전하게 정의된 텐서를 만들지만, -일부 경우에서는 그래프를 실행한 이후에 텐서 형태를 알 수 있기도 합니다. - -일부 특별한 텐서는 텐서플로 가이드문서의 다른 부분에서 다뤄질 것입니다. -핵심인 텐서는 다음과 같습니다: - - * `tf.Variable` - * `tf.constant` - * `tf.placeholder` - * `tf.SparseTensor` - -예외인 `tf.Variable`을 제외한다면, 텐서 값은 변경할 수 없습니다. -즉, 텐서를 한번 실행시킨 경우에는 오직 하나의 값만을 가집니다. -그러나, 동일한 텐서를 다시 실행시킨다면 다른 값을 가질 수 있습니다; -예를 들어 텐서가 디스크로부터 데이터를 읽어들인 결과이거나, -무작위 숫자를 생성하는 경우입니다. - -## 랭크(Rank) - -`tf.Tensor` 객체의 **랭크**는 그 차원의 수입니다. -랭크의 동의어는 **order** 또는 **degree**, **n-dimension**입니다. -텐서플로의 랭크는 수학에서 사용하는 행렬의 랭크와는 다릅니다. -다음 표에서 알 수 있는 것처럼, 텐서플로의 각 랭크는 각각 다른 수학 개체(entity)에 해당됩니다. - -랭크 | 수학 개체 ---- | --- -0 | 스칼라(Scalar) (크기(magnitude)만) -1 | 벡터(Vector) (크기와 방향(direction)) -2 | 행렬(Matrix) (숫자 표) -3 | 3-텐서 (숫자 큐브(cube)) -n | n-텐서 (알 수 있을겁니다(you get the idea)) - - -### 랭크 0 - -다음은 랭크 0 변수 생성 예입니다: - -```python -mammal = tf.Variable("코끼리", tf.string) -ignition = tf.Variable(451, tf.int16) -floating = tf.Variable(3.14159265359, tf.float64) -its_complicated = tf.Variable(12.3 - 4.85j, tf.complex64) -``` - -Note: 문자열은 텐서에서 문자 시퀀스(sequence)가 아니라 단일 객체로 다뤄집니다. -객체는 단일 문자열과 문자열 벡터 등 모두 가능합니다. - -### 랭크 1 - -랭크 1 `tf.Tensor` 객체를 생성하기 위해서 초기값으로 리스트를 사용할 수 있습니다. -예를 들어: - -```python -mystr = tf.Variable(["안녕하세요"], tf.string) -cool_numbers = tf.Variable([3.14159, 2.71828], tf.float32) -first_primes = tf.Variable([2, 3, 5, 7, 11], tf.int32) -its_very_complicated = tf.Variable([12.3 - 4.85j, 7.5 - 6.23j], tf.complex64) -``` - - -### 고차원 랭크 - -랭크 2 `tf.Tensor` 객체는 최소 한 개 이상의 열과 행으로 구성됩니다: - -```python -mymat = tf.Variable([[7],[11]], tf.int16) -myxor = tf.Variable([[False, True],[True, False]], tf.bool) -linear_squares = tf.Variable([[4], [9], [16], [25]], tf.int32) -squarish_squares = tf.Variable([ [4, 9], [16, 25] ], tf.int32) -rank_of_squares = tf.rank(squarish_squares) -mymatC = tf.Variable([[7],[11]], tf.int32) -``` - -유사하게, 고차원 랭크 텐서는 n-차원 배열로 구성됩니다. -예를 들어, 이미지 처리에서는 각각 배치 수와 이미지 높이, 이미지 너비, 색상 채널에 해당하는 4차원 랭크 텐서가 사용됩니다. - -``` python -my_image = tf.zeros([10, 299, 299, 3]) # 배치 x 높이 x 너비 x 색상 -``` - -### `tf.Tensor` 객체 랭크 구하기 - -`tf.Tensor` 객체의 랭크를 알기 위해서는 `tf.rank` 메서드를 호출합니다. -예를 들어, 다음 메서드는 이전 섹션에서 정의된 `tf.Tensor`의 랭크를 알려줍니다. - -```python -r = tf.rank(my_image) -# 이 코드가 실행된 후 r은 4라는 값을 가지게 됩니다. -``` - -### `tf.Tensor` 원소 참조하기 - -`tf.Tensor`는 n-차원 배열로 구성되어 있기 때문에 때문에, -`tf.Tensor`의 원소 하나에 접근하기 위해서는 n개의 인덱스가 필요합니다. - -랭크 0 텐서(스칼라)인 경우 이미 하나의 숫자이기 때문에 인덱스가 필요없습니다. - -랭크 1 텐서(벡터)인 경우 숫자 하나에 접근하기 위해서는 인덱스 한 개를 전달해야 합니다: - -```python -my_scalar = my_vector[2] -``` - -벡터로부터 원소 한 개를 동적으로 선택하기 위해서 -`[]`안에 스칼라형 `tf.Tensor`를 인덱스로 사용할 수 있습니다. - -랭크 2이상의 고차원 텐서인 경우에는 좀 더 흥미롭습니다. -예상한 것처럼 랭크 2인 `tf.Tensor`를 위해 인덱스로 2개를 전달해야 스칼라 한 개를 반환합니다: - - -```python -my_scalar = my_matrix[1, 2] -``` - - -그러나, 한 개만 전달한다면 다음과 같이 행렬의 부분 벡터를 반환합니다: - - -```python -my_row_vector = my_matrix[2] -my_column_vector = my_matrix[:, 3] -``` - -`:` 표기는 "해당 차원을 남겨라"라는 파이썬 슬라이싱(slicing) 문법입니다. -이러한 표기법은 고차원 텐서에서 부분 벡터와 부분 행렬, 다른 부분 텐서들까지도 접근할 수 있도록 만들어 주기 때문에 유용합니다. - - -## 형태 - -텐서의 **형태(shape)**는 각 차원에 있는 원소 개수로 표현됩니다. -텐서플로는 그래프 계산 과정에서 자동으로 텐서 형태를 추론합니다. -이렇게 추론된 형태는 랭크를 알고 있는 경우도 있고 그렇지 않는 경우도 있습니다. -만약에 랭크를 알고 있는 경우라도 각 차원의 원소 개수를 알고 있는 경우도 있고 그렇지 않는 경우도 있습니다. - -텐서플로 문서에서 텐서 차원을 표현하기 위해서 3가지 용어를 사용합니다: 랭크, 형태, 차원. -다음 표는 각 용어가 다른 용어와 어떻게 연관되어 있는지를 보여줍니다. - -랭크 | 형태 | 차원 | 예제 ---- | --- | --- | --- -0 | [] | 0-차원 | 스칼라인 0-차원 텐서. -1 | [D0] | 1-차원 | 형태가 [5]인 1-차원 텐서. -2 | [D0, D1] | 2-차원 | 형태가 [3, 4]인 2-차원 텐서. -3 | [D0, D1, D2] | 3-차원 | 형태가 [1, 4, 3]인 3-차원 텐서. -n | [D0, D1, ... Dn-1] | n-차원 | 형태가 [D0, D1, ... Dn-1]인 텐서. - -형태는 파이썬 리스트 / 정수형 튜플 또는 `tf.TensorShape`으로 표현될 수 있습니다. - -### `tf.Tensor` 객체 형태 얻기 - -`tf.Tensor`의 형태를 알기 위한 2가지 방법이 있습니다. -그래프를 생성하는 동안 텐서의 형태를 알 수 있는 것은 종종 유용합니다. -형태는 `tf.Tensor`객체의 `shape` 속성으로 알 수 있습니다. -이 메서드는 `TensorShape`를 반환하고, 이러한 방식은 완전하게 알지 못하는 형태를 표현하는데 편리한 방법입니다 -(그래프를 생성할 때 모든 형태를 알 수 없기 때문에). - -실행 시점에 다른 `tf.Tensor` 객체의 완전한 형태를 표현하는 `tf.Tensor`를 얻을 수도 있습니다. -이것은 `tf.shape` 연산을 통해서 확인할 수 있습니다. -이를 통해, 입력 `tf.Tensor`의 동적인 형태에 연동된 다른 텐서를 생성함으로써 -텐서 형태를 변경하는 그래프를 생성할 수 있습니다. - -예를 들어 다음은 주어진 행렬의 열 개수와 동일한 크기의 0으로 구성된 벡터를 만드는 예입니다. - -``` python -zeros = tf.zeros(my_matrix.shape[1]) -``` - -### `tf.Tensor` 형태 변경 - -텐서의 **원소 개수**는 모든 형태의 크기를 곱한 것입니다. -스칼라의 원소 개수는 항상 `1`입니다. -원소 개수가 같은 경우에도 형태는 다양할 수 있기 때문에, -그 원소 개수를 유지하면서 `tf.Tensor`의 형태를 변경할 수 있는 것은 유용합니다. -이것은 `tf.reshape`으로 처리할 수 있습니다. - -다음은 텐서 형태를 변경하는 예입니다: - -```python -rank_three_tensor = tf.ones([3, 4, 5]) -matrix = tf.reshape(rank_three_tensor, [6, 10]) # 기존 내용을 6x10 행렬로 - # 형태 변경 -matrixB = tf.reshape(matrix, [3, -1]) # 기존 내용을 3x20 행렬로 형태 변경 - # -1은 차원 크기를 계산하여 - # 자동으로 결정하라는 의미 -matrixAlt = tf.reshape(matrixB, [4, 3, -1]) # 기존 내용을 4x3x5 텐서로 - # 형태 변경 - -# 형태가 변경된 텐서의 원소 개수는 -# 원래 텐서의 원소 개수와 같습니다. -# 그러므로 다음은 원소 개수를 유지하면서 -# 마지막 차원에 사용 가능한 수가 없기 때문에 에러를 발생합니다. -yet_another = tf.reshape(matrixAlt, [13, 2, -1]) # 에러! -``` - -## 자료형 - -차원뿐만 아니라, 텐서는 자료형도 가지고 있습니다. -전체 자료형을 확인하려면 `tf.DType`를 참고하세요. - -`tf.Tensor`가 한 개이상의 자료형을 가지는 것은 불가능합니다. -임의의 데이터 구조를 직렬화한 `string`를 저장한 `tf.Tensor`는 예외입니다. - -`tf.cast`를 이용해서 `tf.Tensor`의 자료형을 다른 것으로 변경하는 것은 가능합니다: - -``` python -# 정수형 텐서를 실수형으로 변환. -float_tensor = tf.cast(tf.constant([1, 2, 3]), dtype=tf.float32) -``` - -`tf.Tensor`의 자료형을 확인하기 위해서는 `Tensor.dtype` 속성을 활용하세요. - -파이썬 객체를 이용해서 `tf.Tensor`를 생성할 때 자료형을 선택적으로 명시할 수 있습니다. -그렇지 않으면 텐서플로가 데이터 표현에 적합한 자료형을 선택합니다. -텐서플로는 파이썬 정수를 `tf.int32`로 변환하고 파이썬 실수는 `tf.float32`으로 변환합니다. -그외에 동일한 규칙을 넘파이(numpy)를 배열로 변경할 때 사용합니다. - -## 텐서 계산하기(evaluate) - -일단 계산 그래프를 만들었다면, 특정 `tf.Tensor`를 생성하거나 텐서에 할당된 값을 가져오는 계산을 실행할 수 있습니다. -이것은 텐서플로 작업의 많은 부분에서 필요할 뿐 아니라 디버깅에 종종 유용합니다. - -텐서를 계산하는 가장 간단한 방법은 `Tensor.eval` 메서드를 사용하는 것입니다. 예를 들어: - -```python -constant = tf.constant([1, 2, 3]) -tensor = constant * constant -print(tensor.eval()) -``` - -`eval` 메서드는 기본 `tf.Session`이 활성화된 경우에만 작동합니다 -(자세한 내용은 [그래프와 세션](./graphs.md)에서 확인하세요). - -`Tensor.eval`은 텐서와 같은 내용을 가지는 넘파이 배열을 반환합니다. - -때때로 현재 사용할 수 없는 동적인 정보에 의존하기 때문에 -컨텍스트(context)가 없는 `tf.Tensor`는 계산할 수 없는 경우가 있습니다. -예를 들어, `placeholder`에 의존하는 텐서는 그 `placeholder`에 해당하는 값이 제공되지 않으면 계산할 수 없습니다. - -``` python -p = tf.placeholder(tf.float32) -t = p + 1.0 -t.eval() # 플레이스홀더(placeholder)가 값을 가지고 있지 않기 때문에, 이것은 실패할 것입니다. -t.eval(feed_dict={p:2.0}) # 플레이스홀더에 해당하는 값을 제공받기 때문에 - # 이것은 성공할 것입니다. -``` - -플레이스홀더뿐만 아니라 어떤 `tf.Tensor`도 값을 제공받는 것이 가능하다는 것에 유의하세요. - -다른 모델 구조는 `tf.Tensor`를 계산하는 것을 복잡하게 만들 수 있습니다. -텐서플로는 함수안이나 제어 흐름안에 정의된 `tf.Tensor`를 직접 계산할 수 없습니다. -만약에 `tf.Tensor`가 큐(queue)에 있는 값을 사용한다면, -무언가가 큐에 들어간 후에 만 `tf.Tensor` 계산을 할 수 있습니다; 그렇지 않으면 계산은 멈출(hang) 것입니다. -큐와 같이 작업할 때, `tf.Tensor`를 계산하기 전 `tf.train.start_queue_runners`를 호출하세요. diff --git a/site/ko/guide/upgrade.md b/site/ko/guide/upgrade.md deleted file mode 100644 index c8a12113335..00000000000 --- a/site/ko/guide/upgrade.md +++ /dev/null @@ -1,98 +0,0 @@ -# 텐서플로 2.0으로 코드 업그레이드 - -Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 -불구하고 -[공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/upgrade.md)의 -내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 -[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 -바랍니다. 문서 번역이나 리뷰에 참여하려면 -[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 -메일을 보내주시기 바랍니다. - -텐서플로 2.0은 매개변수 재배치, 심볼(symbol) 이름 변경, 파라미터 기본값 변경과 같은 많은 API 변화가 있습니다. 이러한 -수정사항들을 일일이 다 반영하는 건 지루하며 실수하기 쉽습니다. 변화들을 간략하게 그리고 TF 2.0에 매끄럽게 옮겨가기 위해 또한 -기존 코드를 새로운 API로 수정하는 것을 쉽게 하기위해 텐서플로 팀은 `tf_upgrade_v2` 유틸리티를 개발하였습니다. - -`tf_upgrade_v2` 유틸리티는 TF 2.0을 `pip install` 하면 자동적으로 설치됩니다. 유틸리티는 기존의 텐서플로 1.x -파이썬 스크립트를 텐서플로 2.0으로 변환하여 업그레이드를 빠르게 합니다. - -변환 스크립트는 자동화되어 있지만 문법적이고 작성 스타일에 관한 변환은 스크립트에 의해 수행되지 않습니다. - -## 호환성 모듈 - -어떤 API 심볼들은 단순히 이름을 변경하는 것만으로는 업그레이드가 안 될 수도 있습니다. 코드를 텐서플로 2.0에서 동작시키기 위해, -`compat.v1` 모듈을 포함하도록 스크립트를 업그레이드 하세요. 이 모듈은 `tf.foo`와 같은 TF 1.x 심볼을 동등한 -`tf.compat.v1.foo`로 대체합니다. 호환성 모듈이 괜찮긴 하지만 직접 수정사항을 확인하고 가능한 빠르게 `tf.compat.v1.*` -네임스페이스(namespace) 대신 `tf.*` 네임스페이스에 있는 새로운 API로 마이그레이션 하기를 추천합니다. - -텐서플로 2.x 모듈에서 사라지는 것들(예를 들면, `tf.flags`와 `tf.contrib`) 때문에, 어떤 변동사항은 -`compat.v1`으로 바꾸는 것만으로는 동작하지 않을 수 있습니다. 이 코드를 업그레이드하려면 다른 라이브러리가 필요하거나(예를 들면, -`absl.flags`) [tensorflow/addons](http://www.github.com/tensorflow/addons)에 있는 -패키지로 바꾸어야 할 수도 있습니다. - -## 업그레이드 스크립트 - -작성한 코드를 텐서플로 1.x에서 텐서플로 2.x로 변경하려면, 다음의 지시에 따르세요: - -### pip 패키지로부터 스크립트 실행 - -첫째로, `pip install`로 `tensorflow` 또는 `tensorflow-gpu` 패키지를 설치합니다. - -Note: `tf_upgrade_v2`는 텐서플로 1.13 그리고 이후 버전에서 자동으로 설치되었습니다. (nightly TF 2.0 빌드를 -포함) - -업그레이드 스크립트는 하나의 파이썬 파일에 대해서 실행됩니다: - -```sh -tf_upgrade_v2 --infile tensorfoo.py --outfile tensorfoo-upgraded.py -``` - -스크립트는 코드에서 대체할 부분을 찾지 못하면 에러 메시지를 표시합니다. 디렉토리 트리에 대해서도 실행가능 합니다: - -``` -# .py 파일을 업그레이드하고 다른 모든 파일들을 outtree에 복사 -tf_upgrade_v2 --intree coolcode --outtree coolcode-upgraded - -# .py 파일만 업그레이드 -tf_upgrade_v2 --intree coolcode --outtree coolcode-upgraded --copyotherfiles False -``` - -## 자세한 리포트 - -스크립트는 자세한 변동사항들의 리스트를 보여줍니다, 예를들면: - -``` -'tensorflow/tools/compatibility/testdata/test_file_v1_12.py' Line 65 --------------------------------------------------------------------------------- - -Added keyword 'input' to reordered function 'tf.argmax' -Renamed keyword argument from 'dimension' to 'axis' - - Old: tf.argmax([[1, 3, 2]], dimension=0)) - ~~~~~~~~~~ - New: tf.argmax(input=[[1, 3, 2]], axis=0)) - -``` - -이 모든 정보는 `report.txt` 파일에 포함되어 있고 이 파일은 현재 디렉토리에 생성될 것입니다. `tf_upgrade_v2`가 실행되고 -업그레이드 스크립트가 만들어지면 각자의 모델을 실행하여 출력이 TF 1.x과 비슷한지 확인할 수 있습니다. -## 주의사항 - -- 이 스크립트를 실행하기 전에 대상코드의 일부를 수동으로 업데이트하지 마세요. 특히, `tf.argmax` 또는 - `tf.batch_to_space`와 같이 매개변수의 순서가 변동된 함수는 스크립트가 부정확한 키워드 매개변수를 추가하여 기존코드를 잘못 - 바꿀 수 있습니다. - -- 스크립트는 `tensorflow`가 `import tensorflow as tf`로 임포트(import) 되어있다고 가정합니다. - -- 이 스크립트는 매개변수의 순서를 바꾸지는 않습니다. 대신에 스크립트는 매개변수의 순서가 바뀐 함수들에 대해서 키워드 매개변수를 - 추가합니다. - -- [tf2up.ml](http://tf2up.ml)을 방문해서 깃헙 저장소의 주피터 노트북과 파이썬 파일들을 업그레이드할 수 있는 편리한 - 툴을 확인하세요. - -업그레이드 스크립트의 버그를 제보하거나 기능 추가 요청을 하려면 -[GitHub](https://github.com/tensorflow/tensorflow/issues). 그리고 텐서플로 2.0을 테스트하고 -있다면, 의견을 듣고 싶습니다! -[TF 2.0 Testing community](https://groups.google.com/a/tensorflow.org/forum/#!forum/testing)에 -가입해 주세요. 질문과 토론은 [testing@tensorflow.org](mailto:testing@tensorflow.org)로 메일 주시기 -바랍니다. diff --git a/site/ko/guide/variable.md b/site/ko/guide/variable.md deleted file mode 100644 index 8340305ee5a..00000000000 --- a/site/ko/guide/variable.md +++ /dev/null @@ -1,100 +0,0 @@ -# 변수 - -Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 -불구하고 -[공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/variable.md)의 -내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 -[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 -바랍니다. 문서 번역이나 리뷰에 참여하려면 -[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 -메일을 보내주시기 바랍니다. - -텐서플로 **변수**는 프로그램에 의해 변화하는 공유된 지속 상태를 표현하는 가장 좋은 방법입니다. - -변수는 `tf.Variable` 클래스에서 처리됩니다. 하나의 `tf.Variable`은 하나의 텐서를 표현하는데, 텐서값은 텐서에 연산을 -수행하여 변경시킬 수 있습니다. 특정한 연산은 이 텐서값을 읽고 수정합니다. `tf.keras` 같은 좀 더 고수준의 라이브러리는 모델 -파라미터를 저장하는데 `tf.Variable`을 사용합니다. 이 가이드는 텐서플로에서 `tf.Variable`의 생성, 업데이트, 관리 방법을 -다룹니다. - -## 변수 생성 - -변수를 생성하려면 단순하게 초기값을 설정하면 됩니다. - -``` python -my_variable = tf.Variable(tf.zeros([1., 2., 3.])) -``` - -이렇게 하면 모양이 `[1, 2, 3]`이고 값은 0으로 채워진 3차원 텐서가 변수로 생성됩니다. 이 변수는 기본적으로 `dtype` -`tf.float32`로 설정될 것입니다. dtype을 지정하지 않은 경우 초기값으로부터 추정됩니다. - -`tf.device` 범위(scope)가 유효하다면, 변수는 해당 장치에 위치합니다; 그렇지 않다면, 변수는 dtype에 호환되는 "가장 빠른" -장치에 위치합니다.(GPU가 가용하다면 대부분의 변수들이 자동적으로 GPU에 위치한다는 의미입니다) 예를들어, 다음의 코드는 `v`라는 변수를 -만들고 두 번째 GPU에 위치시킵니다: - -``` python -with tf.device("/device:GPU:1"): - v = tf.Variable(tf.zeros([10, 10])) -``` - -이상적으로는 `tf.distribute` API를 통해 코드를 한번 작성하고 다양한 분산 설정에서 작동되도록 하세요. - -## 변수의 사용 - -텐서플로 그래프에서 `tf.Variable`의 값을 사용하려면 이를 단순히 `tf.Tensor`로 취급하면 됩니다: - -``` python -v = tf.Variable(0.0) -w = v + 1 # w는 v값 기준으로 계산되는 tf.Tensor 입니다. - # 변수가 수식에 사용될 때, 변수는 자동적으로 - # tf.Tensor로 변환되어 값을 표현합니다. -``` - -값을 변수에 할당하려면 `assign`, `assign_add` 메소드와 `tf.Variable` 클래스에 있는 친구들(friends)을 -사용하세요. 예를들어, 여기에 이 메서드를 호출하는 방법이 있습니다: - -``` python -v = tf.Variable(0.0) -v.assign_add(1) -``` - -대부분의 텐서플로 옵티마이저(optimizer)는 경사 하강법과 같은 알고리즘에 따라 변수값을 효율적으로 업데이트하는 전문적인 연산이 있습니다. -`tf.keras.optimizers.Optimizer`을 보면 옵티마이저 사용방법이 설명되어 있습니다. - -또한, `read_value`를 사용하여 현재 변수값을 명시적으로 읽어올 수 있습니다: - -```python -v = tf.Variable(0.0) -v.assign_add(1) -v.read_value() # 1.0 -``` - -`tf.Variable`에 대한 마지막 참조가 범위(scope)에서 벗어났을때 메모리가 해제됩니다. - -### 변수 추적 - -텐서플로에서 하나의 변수는 하나의 파이썬 객체입니다. 층, 모델, 옵티마이저, 그리고 다른 관련된 도구들을 작성할때 모델 안에 있는 모든 변수의 -리스트가 필요할 수 있습니다. - -자신의 파이썬 코드에서 필요에 따라 변수들을 추적하는 동안 변수를 소유하는 자신의 클래스에 기본 클래스로 `tf.Module` 사용을 -권장합니다. `tf.Module`의 인스턴스는 `variables`와 다른 모듈들을 탐색할 수 있는 모델에서 접근할 수 있는 모든 (훈련 -가능한) 변수을 리턴하는 `trainable_variables` 메서드가 포함되어 있습니다. - -```python -class MyModuleOne(tf.Module): - def __init__(self): - self.v0 = tf.Variable(1.0) - self.vs = [tf.Variable(x) for x in range(10)] - -class MyOtherModule(tf.Module): - def __init__(self): - self.m = MyModuleOne() - self.v = tf.Variable(10.0) - -m = MyOtherModule() -len(m.variables) # 12; 11은 m.m에서 다른 것은 m.v에서 - -``` - -층을 구현하고 있다면 `tf.keras.Layer`가 기본 클래스로 더 좋을 수 있습니다. 인터페이스를 구현하는 것은 층이 케라스에 완전하게 -통합되기 때문에 `model.fit`과 잘 통합된 다른 API들을 사용할 수 있게 됩니다. `tf.keras.Layer`의 변수 추적은 -`tf.Module`의 변수 추적과 동일합니다. diff --git a/site/ko/guide/versions.md b/site/ko/guide/versions.md deleted file mode 100644 index c97146eb4d3..00000000000 --- a/site/ko/guide/versions.md +++ /dev/null @@ -1,296 +0,0 @@ -# 텐서플로 버전 호환성 - -Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 -불구하고 -[공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/versions.md)의 -내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 -[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 -바랍니다. 문서 번역이나 리뷰에 참여하려면 -[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 -메일을 보내주시기 바랍니다. - -이 문서는 텐서플로(코드나 데이터)의 다른 버전들간의 하위 호환성이 필요한 사용자와 텐서플로의 호환성을 유지하면서 텐서플로를 수정하고 싶어하는 -개발자들을 위한 문서입니다. - -## 유의적 버저닝 2.0 - -텐서플로는 공개 API에 유의적 버저닝 2.0([semver](http://semver.org))을 준수합니다. 텐서플로의 각 릴리즈 버전은 -`MAJOR.MINOR.PATCH` 형식입니다. 이를테면 텐서플로 버전 1.2.3은 `MAJOR` 버전 1, `MINOR` 버전 2, -`PATCH` 버전 3을 뜻합니다. 각 숫자의 변화는 다음을 뜻합니다: - -* **MAJOR**: 하위 호환성이 없는 변동일 수 있습니다. 이전 주(major) 버전에서 동작했었던 코드와 데이터는 새로운 버전에서 - 동작하지 않을 수 있습니다. 그러나, 어떤 경우에는 기존 텐서플로 그래프와 체크포인트를 새로운 버전에 마이그레이션할 수도 있습니다; - 자세한 사항은 [그래프와 체크포인트의 호환성](#compatibility_of_graphs_and_checkpoints)을 참고하세요. - -* **MINOR**: 하위 호환되는 특성, 속도 개선 등 입니다. 이전 부(minor) 버전*과* 이전 부 버전의 비실험적인 공개 API를 - 사용했던 코드와 데이터는 정상적으로 동작합니다. 공개 API에 무엇이 포함되고 포함되지 않는지는 - [포함되는 것들](#What_is_covered)를 참조하세요. - -* **PATCH**: 하위 호환되는 버그 픽스들 - -이를테면 릴리즈 1.0.0은 릴리즈 0.12.1에서 하위 호환성이 *없는* 변동사항이 있습니다. 그러나, 릴리즈 1.1.1은 릴리즈 1.0.0과 -하위 호환성이 있습니다. - -## 포함되는 것들 - -텐서플로의 공개 API만이 마이너와 패치 버전에서 하위 호환성을 가집니다. 공개 API는 다음을 포함합니다. - -* 모든 문서화된 [파이썬](https://www.tensorflow.org/api_docs/python) `tensorflow` 모듈과 - 서브모듈에 있는 함수와 클래스, 다음은 제외 - - * 개인 심볼(private symbol): `_`로 시작하는 어떠한 함수나 클래스 등 - * 실험적인 그리고 `tf.contrib` 심볼, 자세한건 [아래](#not_covered)를 참조하세요. - - `examples/`와 `tools/` 경로에 있는 코드는 `tensorflow` 파이썬 모듈을 통해 접근할 수 없고 따라서 호환성을 - 보장할 수 없습니다. - - 한 심볼이 `tensorflow` 파이썬 모듈이나 서브모듈에서 사용가능하지만 문서화되지는 않은 경우, 공개 API의 일부로 간주하지 - **않습니다**. - -* 호환성 API(파이썬의 `tf.compat` 모듈). 주 버전에서 사용자들이 새로운 주 버전으로 옮겨가는 것을 도와주는 유틸리티와 - 추가적인 엔드포인트가 공개될 수도 있습니다. 이러한 API 심볼들은 없어지고 지원되지 않지만(즉, 기능을 추가하지 않고 취약성 이외의 - 버그를 수정하지 않습니다) 호환성은 보장됩니다. - -* [C API](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/c/c_api.h). - -* 다음의 프로토콜 버퍼 파일들: - - * [`attr_value`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto) - * [`config`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/config.proto) - * [`event`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/event.proto) - * [`graph`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/graph.proto) - * [`op_def`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_def.proto) - * [`reader_base`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/reader_base.proto) - * [`summary`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto) - * [`tensor`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor.proto) - * [`tensor_shape`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/tensor_shape.proto) - * [`types`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/types.proto) - - -## 포함되지 *않는* 것들 - -텐서플로의 어떤 부분은 모든 곳에서 하위 호환성이 없도록 변동될 수 있습니다. 이는 다음을 포함합니다: - -* **실험적인 API**: 개발을 용이하게 하기 위해, 어떤 API 심볼들은 실험적인 것으로 규정하고 하위 호환성을 보장하지 않습니다. - 특히 다음은 어떠한 호환성 보장도 하지 않습니다: - - - `tf.contrib` 모듈이나 서브모듈에 있는 모든 심볼들 - - `experimental` 또는 `Experimental`이라는 이름을 포함하는 모든 심볼(모듈, 함수, 매개변수, 속성, - 클래스, 상수); 또는 - - 모듈이나 클래스가 포함하는 절대 표기가 그 자체로 실험적인 모든 심볼들. `experimental`로 분류되는 모든 프로토콜 - 버퍼의 필드나 서브메시지 포함. - -* **다른 언어들:** 파이썬과 C 이외의 텐서플로 API 언어들, 이를테면: - - - [C++](https://www.tensorflow.org/api_guides/cc/guide.md) - ([`tensorflow/cc`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/cc)의 - 헤더파일들을 통해 공개되어 있음). - - [Java](https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/package-summary), - - [Go](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go) - - [JavaScript](https://js.tensorflow.org) - -* **합성 연산 세부사항:** 파이썬의 많은 공개함수가 그래프의 몇몇의 원시 연산에 확장됩니다, 그리고 이러한 세부사항은 - `GraphDef`로 디스크에 저장되는 그래프의 한 부분입니다. 이러한 세부사항은 부(minor) 버전에서 변경될 수 있습니다. 특히, - 그래프간 정확한 매칭이 되는지 확인하는 회귀테스트는 그래프의 행동이 변경되지 않고 기존의 체크포인트가 아직 동작할지라도 서로 다른 부 - 버전에서는 호환되지 않을 가능성이 높습니다. - -* **부동 소수점 세부사항:** 연산을 통해 계산되는 특정 부동 소수점 값은 언제든지 변경될 수 있습니다. 사용자는 계산된 특정 비트에 - 의존하면 안되고, 근사적인 정밀도와 수치적 안정성에 초점을 두어야 합니다. 부 버전과 패치에서 수식의 변화는 상당히 정확도를 높입니다. - (기계학습에서 특정 공식의 향상된 정확도는 전체 시스템에서의 정확도를 낮추는 경우도 있습니다.) - -* **랜덤 숫자:** 특정한 랜덤 숫자가 - [random ops](https://www.tensorflow.org/api_guides/python/constant_op.md#Random_Tensors)를 - 통해 계산되고 언제든지 바뀔 수 있습니다. 사용자는 계산된 특정 비트에 의존하지 말고, 근사적으로 적절한 분포와 통계적 강도에 초점을 - 두어야 합니다. 그러나, 패치 버전에서는 특정한 비트를 거의 바꾸지 않도록 합니다. 당연히 이러한 모든 변동사항은 문서화합니다. - -* **분산 텐서플로에서의 버전 엇갈림:** 하나의 클러스터에서 서로 다른 두 버전의 텐서플로를 실행하는 것은 지원되지 않습니다. 와이어 - 프로토콜(wire protocol)의 하위 호환성을 보장할 수 없습니다. - -* **버그들:** 현재의 구현이 명백하게 문제가 있는 경우, 하위 호환성을 유지하지 않는 변동사항을 만들 수 있습니다. 문서와 구현이 서로 - 모순되는 경우 또는 잘 알려져있고 잘 정의된 의도를 가진 행동이 버그때문에 적절하게 구현되지 않은 경우가 이에 해당됩니다. 이를테면, 잘 - 알려진 최적화 알고리즘이 옵티마이저(optimizer)에 구현되어야 하지만 버그때문에 그 알고리즘과 매치되지 않는다면, - 옵티마이저(optimizer)를 수정할 것입니다. 수정사항은 통합을 위해서 잘못 동작하는 부분에 의존하는 코드를 포함합니다. 릴리즈 - 노트에 그러한 변동사항들이 기록될 것입니다. - -* **에러 동작:** 에러를 에러가 없는 동작으로 수정합니다. 이를테면, 어떤 에러가 문서화되더라도 결과를 계산하거나 에러를 올리는 함수를 - 변경할 수 있습니다. 또한 에러 메시지의 텍스트를 수정할 수 있습니다. 덧붙여, 에러의 타입은 에러가 나는 특정한 조건에 대해 기대하는 - 타입이 문서에 기술되지 않는다면 변경될 수 있습니다. - -## 저장된 모델의 호환성, 그래프와 체크포인트 - -저장된 모델은 텐서플로 프로그램에서 사용하기 위해서 직렬화된 형식(serialization format)이 좋습니다. 저장된 모델은 두 부분으로 -이루어져 있습니다: 하나 또는 더 많은 그래프들이 `GraphDefs`와 체크포인트로 인코드 됩니다. 그래프는 실행할 연산의 데이터 흐름를 -기술하고 체크포인트는 그래프 변수들의 저장된 텐서값을 포함합니다. - -많은 텐서플로 사용자들이 저장된 모델을 만들고 나중에 릴리즈된 텐서플로에서 로드하여 실행합니다. -[semver](https://semver.org)에 따라 어떤 버전의 텐서플로에서 저장된 모델은 같은 주 버전에 속한 나중 버전의 -텐서플로에서는 로드되고 실행될 수 있습니다. - -*지원하는* 저장된 모델에서는 추가적인 보장이 있습니다. 텐서플로 주 버전 `N`에서 **사라지지 않고 실험적이지도 않으며 호환되지 않는 -API**를 사용하여 만든 저장된 모델은 *버전 `N`에서 지원됩니다.* 텐서플로 주 버전 `N`에서 지원하는 모든 저장된 모델은 텐서플로 주 -버전 `N+1`에서도 로드되고 실행될 수 있습니다. 그러나, 그 모델을 만들고 수정하기 위해 필요한 기능들을 더 이상 사용할 수 없는 경우, 이 -보장은 수정하지 않은 저장된 모델에만 적용됩니다. - -가능하면 하위 호환성을 유지하기위해 노력할 것이라서 직렬화된 파일들은 오랫동안 사용가능합니다. - -### GraphDef compatibility - -Graphs are serialized via the `GraphDef` protocol buffer. To facilitate -backwards incompatible changes to graphs, each `GraphDef` has a version number -separate from the TensorFlow version. For example, `GraphDef` version 17 -deprecated the `inv` op in favor of `reciprocal`. The semantics are: - -* Each version of TensorFlow supports an interval of `GraphDef` versions. This - interval will be constant across patch releases, and will only grow across - minor releases. Dropping support for a `GraphDef` version will only occur - for a major release of TensorFlow (and only aligned with the version support - guaranteed for SavedModels). - -* Newly created graphs are assigned the latest `GraphDef` version number. - -* If a given version of TensorFlow supports the `GraphDef` version of a graph, - it will load and evaluate with the same behavior as the TensorFlow version - used to generate it (except for floating point numerical details and random - numbers as outlined above), regardless of the major version of TensorFlow. - In particular, a GraphDef which is compatible with a checkpoint file in one - version of TensorFlow (such as is the case in a SavedModel) will remain - compatible with that checkpoint in subsequent versions, as long as the - GraphDef is supported. - - Note that this applies only to serialized Graphs in GraphDefs (and - SavedModels): *Code* which reads a checkpoint may not be able to read - checkpoints generated by the same code running a different version of - TensorFlow. - -* If the `GraphDef` *upper* bound is increased to X in a (minor) release, there - will be at least six months before the *lower* bound is increased to X. For - example (we're using hypothetical version numbers here): - - * TensorFlow 1.2 might support `GraphDef` versions 4 to 7. - * TensorFlow 1.3 could add `GraphDef` version 8 and support versions 4 to 8. - * At least six months later, TensorFlow 2.0.0 could drop support for - versions 4 to 7, leaving version 8 only. - - Note that because major versions of TensorFlow are usually published more than - 6 months apart, the guarantees for supported SavedModels detailed above are - much stronger than the 6 months guarantee for GraphDefs. - -Finally, when support for a `GraphDef` version is dropped, we will attempt to -provide tools for automatically converting graphs to a newer supported -`GraphDef` version. - -## Graph and checkpoint compatibility when extending TensorFlow - -This section is relevant only when making incompatible changes to the `GraphDef` -format, such as when adding ops, removing ops, or changing the functionality -of existing ops. The previous section should suffice for most users. - - - -### Backward and partial forward compatibility - -버저닝 계획의 세 가지 요건: - -* 이전 버전의 텐서플로에서 만들어진 그래프와 체크포인트 로딩하기 위한 **하위 호환성**. -* 그래프나 체크포인트의 프로듀서(producer)가 컨슈머(consumer)이전에 새 버전의 텐서플로로 업그레이드되는 시나리오를 위한 - **상위 호환성**. -* 텐서플로가 호환하지 않는 방향으로 개선되는 것을 가능하게 함. 이를테면, 연산을 제거하거나 속성을 추가하고 제거하는 것. - -`GraphDef` 버전 메커니즘은 텐서플로 버전과는 분리되어 있지만, `GraphDef` 형식에 하위 호환되지 않는 변동사항은 유의적 -버저닝에서 제한됩니다. 즉, 텐서플로 `주 (MAJOR)` 버전간(이를테면 `1.7`과 `2.0`) 기능이 제거되거나 변할 수 있습니다. 상위 -호환성은 패치 릴리즈(이를테면 `1.x.1`에서 `1.x.2`)안에서 강제됩니다. - -상위 호환성과 하위 호환성을 가능하게 하고 형식의 변동을 언제 강제해야 할지 알기 위해서, 그래프와 체크포인트는 언제 생성되었는지에 대한 -메타데이터를 가집니다. 아래의 섹션은 텐서플로 구현과 `GraphDef` 버전업에 대한 가이드라인이 자세히 나와있습니다. - -### 독립적인 데이터 버전 계획 - -그래프와 체크포인트에는 서로 다른 데이터 버전이 있습니다. 두 데이터 형식은 서로 다른 비율로 버전업되고 또한 텐서플로와도 서로 다른 비율로 -버전업된다. 두 버저닝 시스템 모두 -[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h). -새 버전이 추가될때마다 어떤 것들이 변화하였고 날짜는 어떻게 되는지가 헤더에 추가됩니다. - -### 데이터, 프로듀서, 그리고 컨슈머 - -다음과 같은 데이터 버전 정보를 구분할 것입니다: * **프로듀서**: 데이터를 생성하는 바이너리. 프로듀서는 (`producer`)라는 버전이 -있고 호환되는 최소 컨슈머 버전(`min_consumer`)이 있습니다. * **컨슈머**: 데이터를 소비하는 바이너리. 컨슈머는 -(`consumer`)라는 버전이 있고 호환되는 최소 프로듀서 버전(`min_producer`)이 있습니다. - -데이터 버전은 데이터를 만든 `프로듀서`와 호환이 되는 `min_consumer`, 그리고 허용되지 않은 `bad_consumers` 버전 -리스트를 기록하는 -[`VersionDef versions`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/versions.proto) -필드를 가지고 있습니다. - -기본적으로, 프로듀서가 데이터를 만들면 데이터는 프로듀서의 `producer`와 `min_consumer` 버전을 물려받습니다. 특정한 컨슈머 -버전이 버그를 포함하고 있거나 반드시 피해야 한다면 `bad_consumers`가 설정될 수 있습니다. 컨슈머는 다음이 모두 성립하는 경우 -데이터를 받아들일 수 있습니다: - -* `consumer` >= 데이터의 `min_consumer` -* 데이터의 `producer` >= 컨슈머의 `min_producer` -* 데이터의 `bad_consumers`에 없는 `consumer` - -프로듀서와 컨슈머 모두 같은 텐서플로 코드 베이스로부터 나온 것이기 때문에, -[`core/public/version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/public/version.h)는 -문맥과 `min_consumer`와 `min_producer`(프로듀서와 컨슈머 각각이 필요로 하는)에 따라 `producer`나 -`consumer` 둘 중 하나로 취급되는 메인 데이터 버전을 포함합니다. 구체적으로, - -* `GraphDef` 버전으로 `TF_GRAPH_DEF_VERSION`, `TF_GRAPH_DEF_VERSION_MIN_CONSUMER`, - `TF_GRAPH_DEF_VERSION_MIN_PRODUCER`. -* 체크포인트 버전으로 `TF_CHECKPOINT_VERSION`, `TF_CHECKPOINT_VERSION_MIN_CONSUMER`, - `TF_CHECKPOINT_VERSION_MIN_PRODUCER`. - -### 기존 연산에 새로운 속성을 기본값으로 추가 - -다음의 가이드를 따르면 일련의 연산이 변하지 않았을때만 상위 호환성이 있게 됩니다: - -1. 상위 호환성이 필요하다면, `SavedModelBuilder`클래스의 - `tf.saved_model.SavedModelBuilder.add_meta_graph_and_variables`와 - `tf.saved_model.SavedModelBuilder.add_meta_graph`메소드를 사용하거나 - `tf.estimator.Estimator.export_saved_model`을 사용하는 모델을 내보내는 동안 - `strip_default_attrs`를 `True`로 설정합니다. -2. 이렇게 하면 모델을 생성하고 내보낼 때 기본값 속성을 제거하게 됩니다. -3. 이 컨트롤을 사용하면 오래된 컨슈머(예를들면, 훈련 바이너리에 뒤쳐진 바이너리를 제공하는)가 모델을 불러오기를 계속 할 수 있게 하고 - 모델 서비스 중단을 막을 수 있습니다. - -### GraphDef 버전업 - -이 섹션은 `GraphDef` 형식에 다른 타입의 변동사항을 만들기 위한 버저닝 방법을 설명합니다. - -#### 하나의 연산을 추가하기 - -`GraphDef`버전을 바꾸지 않고 컨슈머와 프로듀서에 동시에 새로운 연산을 추가합니다. 이러한 종류의 변동사항은 자동적으로 하위 호환성이 -있고 기존 프로듀서 스크립트가 갑자기 새로운 기능을 사용하지는 않을 것이기 때문에 상위 호환 계획에 영향을 주지 않습니다. - -#### 연산을 추가하고 이를 사용하기 위해 기존 파이썬 래퍼로 바꾸기 - -1. 새로운 컨슈머 기능을 구현하고 `GraphDef` 버전을 올립니다. -2. 이전에 동작하지 않았던 새로운 기능을 사용하는 래퍼를 만들 수 있습니다, 래퍼는 지금 업데이트 가능합니다. -3. 파이썬 래퍼를 변경하여 새로운 기능을 사용하세요. 이 연산을 사용하지 않는 모델은 고장나지 않아야 하므로 `min_consumer`를 - 올리지 마세요. - -#### 연산의 기능을 제거하거나 제한하기 - -1. 금지된 연산이나 기능을 사용하기 않기 위해 모든 프로듀서 스크립트(텐서플로 자체가 아닌)를 고정합니다. -2. `GraphDef`버전을 올리고 새 버전의 GraphDef나 그 이상에서 제거된 연산이나 기능을 금지하는 새로운 컨슈머 기능을 - 구현하세요. 가능하다면 텐서플로에서 금지된 기능으로 `GraphDefs`를 만들지 않도록 하세요. 그렇게 하려면 - [`REGISTER_OP(...).Deprecated(deprecated_at_version, message)`](https://github.com/tensorflow/tensorflow/blob/b289bc7a50fc0254970c60aaeba01c33de61a728/tensorflow/core/ops/array_ops.cc#L1009)를 - 추가하세요. -3. 하위 호환성을 목적으로 주 릴리즈를 기다리세요. -4. (2)에서의 GraphDef 버전에서 `min_producer`를 올리고 기능을 완전히 제거하세요. - -#### 연산의 기능을 변동시키기 - -1. `SomethingV2`와 같이 비슷한 연산의 이름을 추가하며 이를 위한 절차를 거치고 기존 파이썬 래퍼가 이를 사용할 수 있도록 - 전환하세요. 파이썬 래퍼를 변경할 때 상위 호환성확인을 위해 - [compat.py](https://www.tensorflow.org/code/tensorflow/python/compat/compat.py)에 - 있는 제안사항을 확인하세요. -2. 예전의 연산을 제거하세요(하위 호환성때문에 주 버전이 변경할때만 발생합니다). -3. 예전의 연산을 사용하는 컨슈머를 배제하기 위해 `min_consumer`를 올리고 예전 연산에 `SomethingV2`를 위한 별칭을 - 달아주세요. 그리고 기존 파이썬 래퍼가 사용할 수 있도록 변환하는 절차를 거치세요. -4. `SomethingV2`를 제거하는 절차를 거치세요. - -#### 안전하지 않은 컨슈머 버전을 금지하기 - -1. `GraphDef` 버전을 충돌시키고 나쁜 버전을 모든 새로운 GraphDef의 `bad_consumers`에 추가합니다. 가능하면 - 특정한 연산이나 비슷한 것들을 포함하는 GraphDef에만 `bad_consumers`를 추가합니다. -2. 기존 컨슈머가 나쁜 버전인 경우, 최대한 빠르게 제거합니다. diff --git a/site/ko/hub/README.md b/site/ko/hub/README.md deleted file mode 100644 index 631d98134e5..00000000000 --- a/site/ko/hub/README.md +++ /dev/null @@ -1,5 +0,0 @@ -워프 존(warp zone)에 오신 것을 환영합니다! - -# 텐서플로 허브(hub) - -이 문서는 다음 링크에서 확인할 수 있습니다 : https://github.com/tensorflow/hub/tree/master/docs diff --git a/site/ko/r1/README.md b/site/ko/r1/README.md deleted file mode 100644 index f2f70652ce9..00000000000 --- a/site/ko/r1/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# TensorFlow 1.x - -This archive of the TensorFlow 1.x docs is in *maintenance mode* only. - -For docs contributors, please update the source files in `site/en/` and read the -[TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs). - -For community translations, read the instructions in `site/ko/README.md`. diff --git a/site/ko/r1/tutorials/eager/README.md b/site/ko/r1/tutorials/eager/README.md deleted file mode 100644 index 08eedf973d2..00000000000 --- a/site/ko/r1/tutorials/eager/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# 연구 및 실험 - -Note: 이 문서들은 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 -[공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 -있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 -바랍니다. 문서 번역이나 리뷰에 참여하려면 -[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 -메일을 보내주시기 바랍니다. - -즉시 실행(Eager execution)은 고급 연산을 위한 실행에 의해 정의되는 명령형 인터페이스를 제공합니다. -사용자 정의 층, 정방향 전파, 자동 미분을 사용한 훈련 루프를 작성하세요. 이 노트북으로 보고 난 후에 다음 문서를 읽어보세요. -[즉시 실행 가이드](https://github.com/tensorflow/docs/blob/master/site/en/r1/guide/eager.ipynb). - -1. [즉시 실행](eager_basics.ipynb) -2. [자동 미분과 그래디언트 테이프](automatic_differentiation.ipynb) -3. [사용자 정의 학습 : 기초](custom_training.ipynb) -4. [사용자 정의 층](custom_layers.ipynb) -5. [사용자 정의 학습 : 자세히 둘러보기](custom_training_walkthrough.ipynb) diff --git a/site/ko/r1/tutorials/eager/automatic_differentiation.ipynb b/site/ko/r1/tutorials/eager/automatic_differentiation.ipynb deleted file mode 100644 index 8c5cf3b5c15..00000000000 --- a/site/ko/r1/tutorials/eager/automatic_differentiation.ipynb +++ /dev/null @@ -1,338 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "automatic_differentiation.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "GCCk8_dHpuNf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xh8WkEwWpnm7" - }, - "source": [ - "# 자동 미분과 그래디언트 테이프" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idv0bPeCp325" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Q9_NaXPWxEd8", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJ4XzMqodTy" - }, - "source": [ - "이전 튜토리얼에서는 텐서(tensor)와 텐서의 연산에 대해서 알아보았습니다. 이번 튜토리얼에서는 머신러닝 모델을 최적화할 수 있는 주요 기술 중 하나인 [자동 미분(automatic differentiation)](https://en.wikipedia.org/wiki/Automatic_differentiation)에 대해 알아보겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GQJysDM__Qb0" - }, - "source": [ - "## 설정\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OiMPZStlibBv", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1CLWJl0QliB0" - }, - "source": [ - "## 그래디언트 테이프\n", - "\n", - "텐서플로는 자동 미분(주어진 입력 변수에 대한 연산의 그래디언트(gradient)를 계산하는 것)을 위한 [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) API를 제공합니다. `tf.GradientTape`는 안에서 실행된 모든 연산을 테이프(tape)에 \"기록\"합니다. 그리고 [후진 방식 자동 미분(reverse mode differentiation)](https://en.wikipedia.org/wiki/Automatic_differentiation)을 사용하여 각각의 기록된 연산과 관련된 그래디언트와 테이프를 사용하여 기록된 연산의 그래디언트를 계산합니다. \n", - "\n", - "예를 들면:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bAFeIE8EuVIq", - "colab": {} - }, - "source": [ - "x = tf.ones((2, 2))\n", - " \n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# 입력 텐서 x에 대한 z의 도함수\n", - "dz_dx = t.gradient(z, x)\n", - "for i in [0, 1]:\n", - " for j in [0, 1]:\n", - " assert dz_dx[i][j].numpy() == 8.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N4VlqKFzzGaC" - }, - "source": [ - "또한 `tf.GradientTape` 컨텍스트 안에서 기록되는 동안 계산된 중간 값에 대한 그래디언트도 구할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7XaPRAwUyYms", - "colab": {} - }, - "source": [ - "x = tf.ones((2, 2))\n", - " \n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# 테이프 사용하여 중간 값 y에 대한 도함수를 계산합니다. \n", - "dz_dy = t.gradient(z, y)\n", - "assert dz_dy.numpy() == 8.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISkXuY7YzIcS" - }, - "source": [ - "기본적으로 GradientTape.gradient() 메서드가 호출되면 GradientTape에 포함된 리소스가 해제됩니다. 동일한 연산 대해 여러 그래디언트를 계산하려면, `지속성있는(persistent)` 그래디언트 테이프를 생성하면 됩니다. 이 그래디언트 테이프는 `gradient()` 메서드의 다중 호출을 허용합니다. 테이프 객체가 쓰레기 수집(garbage collection)될때 리소스는 해체됩니다.\n", - "예를 들면 다음과 같습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zZaCm3-9zVCi", - "colab": {} - }, - "source": [ - "x = tf.constant(3.0)\n", - "with tf.GradientTape(persistent=True) as t:\n", - " t.watch(x)\n", - " y = x * x\n", - " z = y * y\n", - "dz_dx = t.gradient(z, x) # 108.0 (4*x^3 at x = 3)\n", - "dy_dx = t.gradient(y, x) # 6.0\n", - "del t # 테이프에 대한 참조를 삭제합니다." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6kADybtQzYj4" - }, - "source": [ - "### 제어 흐름 기록\n", - "\n", - "연산이 실행되는 순서대로 테이프에 기록되기 때문에, 파이썬 제어 흐름(예를 들어 `if` `while`, `for`문 같은)이 자연스럽게 처리됩니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9FViq92UX7P8", - "colab": {} - }, - "source": [ - "def f(x, y):\n", - " output = 1.0\n", - " for i in range(y):\n", - " if i > 1 and i < 5:\n", - " output = tf.multiply(output, x)\n", - " return output\n", - "\n", - "def grad(x, y):\n", - " with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " out = f(x, y)\n", - " return t.gradient(out, x) \n", - "\n", - "x = tf.convert_to_tensor(2.0)\n", - "\n", - "assert grad(x, 6).numpy() == 12.0\n", - "assert grad(x, 5).numpy() == 12.0\n", - "assert grad(x, 4).numpy() == 4.0\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DK05KXrAAld3" - }, - "source": [ - "### 고계도(Higher-order) 그래디언트\n", - "\n", - "`GradientTape` 컨텍스트 매니저안에 있는 연산들은 자동미분을 위해 기록됩니다. 만약 그래디언트가 컨텍스트 안에서 계산되면 그 그래디언트 연산 또한 기록되어집니다. 그 결과 똑같은 API가 고계도 그래디언트에서도 잘 작동합니다. 예를 들면:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cPQgthZ7ugRJ", - "colab": {} - }, - "source": [ - "x = tf.Variable(1.0) # 1.0으로 초기화된 텐서플로 변수를 생성합니다.\n", - "\n", - "with tf.GradientTape() as t:\n", - " with tf.GradientTape() as t2:\n", - " y = x * x * x\n", - " # t 컨텍스트 매니저 안의 그래디언트를 계산합니다.\n", - " # 이것은 또한 그래디언트 연산 자체도 미분가능하다는것을 의미합니다. \n", - " dy_dx = t2.gradient(y, x)\n", - "d2y_dx2 = t.gradient(dy_dx, x)\n", - "\n", - "assert dy_dx.numpy() == 3.0\n", - "assert d2y_dx2.numpy() == 6.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4U1KKzUpNl58" - }, - "source": [ - "## 다음 단계\n", - "\n", - "이번 튜토리얼에서는 텐서플로에서 그래디언트 계산법을 다루었습니다. 이를 통해 신경망(neural network)을 구축하고 훈련시키는데 필요한 많은 기본 요소를 배웠습니다." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/eager/custom_layers.ipynb b/site/ko/r1/tutorials/eager/custom_layers.ipynb deleted file mode 100644 index 3df754708d9..00000000000 --- a/site/ko/r1/tutorials/eager/custom_layers.ipynb +++ /dev/null @@ -1,380 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_layers.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tDnwEv8FtJm7" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "JlknJBWQtKkI", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "60RdWsg1tETW" - }, - "source": [ - "# 사용자 정의 층" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BcJg7Enms86w" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "asCKfd--yH-6", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UEu3q4jmpKVT" - }, - "source": [ - "신경망을 구축하기 위해서 고수준 API인 `tf.keras`를 사용하길 권합니다. 대부분의 텐서플로 API는 즉시 실행(eager execution)과 함께 사용할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pwX7Fii1rwsJ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zSFfVVjkrrsI" - }, - "source": [ - "## 층: 유용한 연산자 집합\n", - "\n", - "머신러닝을 위한 코드를 작성하는 대부분의 경우에 개별적인 연산과 변수를 조작하는 것보다는 높은 수준의 추상화에서 작업할 것입니다.\n", - "\n", - "많은 머신러닝 모델은 비교적 단순한 층(layer)을 조합하고 쌓아서 표현가능합니다. 또한 텐서플로는 여러 표준형 층을 제공하므로 사용자 고유의 응용 프로그램에 관련된 층을 처음부터 작성하거나, 기존 층의 조합으로 쉽게 만들 수 있습니다.\n", - "\n", - "텐서플로는 [전체 케라스](https://keras.io) API를 tf.keras 패키지에 포함하고 있습니다. 케라스 층은 모델을 구축하는데 매우 유용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8PyXlPl-4TzQ", - "colab": {} - }, - "source": [ - "# tf.keras.layers 패키지에서 층은 객체입니다. 층을 구성하려면 간단히 객체를 생성하십시오.\n", - "# 대부분의 layer는 첫번째 인수로 출력 차원(크기) 또는 채널을 취합니다.\n", - "layer = tf.keras.layers.Dense(100)\n", - "# 입력 차원의 수는 층을 처음 실행할 때 유추할 수 있기 때문에 종종 불필요합니다. \n", - "# 일부 복잡한 모델에서는 수동으로 입력 차원의 수를 제공하는것이 유용할 수 있습니다.\n", - "layer = tf.keras.layers.Dense(10, input_shape=(None, 5))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fn69xxPO5Psr" - }, - "source": [ - "미리 구성되어있는 층은 다음 [문서](https://www.tensorflow.org/api_docs/python/tf/keras/layers)에서 확인할 수 있습니다. Dense(완전 연결 층), Conv2D, LSTM, BatchNormalization, Dropout, 등을 포함하고 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "E3XKNknP5Mhb", - "colab": {} - }, - "source": [ - "# 층을 사용하려면, 간단하게 호출합니다.\n", - "layer(tf.zeros([10, 5]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Wt_Nsv-L5t2s", - "colab": {} - }, - "source": [ - "# layer는 유용한 메서드를 많이 가지고 있습니다. 예를 들어, `layer.variables`를 사용하여 층안에 있는 모든 변수를 확인할 수 있으며, \n", - "# `layer.trainable_variables`를 사용하여 훈련가능한 변수를 확인할 수 있습니다. \n", - "# 완전 연결(fully-connected)층은 가중치(weight)와 편향(biases)을 위한 변수를 가집니다. \n", - "layer.variables" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6ilvKjz8_4MQ", - "colab": {} - }, - "source": [ - "# 또한 변수는 객체의 속성을 통해 편리하게 접근가능합니다. \n", - "layer.kernel, layer.bias" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O0kDbE54-5VS" - }, - "source": [ - "## 사용자 정의 층 구현\n", - "사용자 정의 층을 구현하는 가장 좋은 방법은 tf.keras.Layer 클래스를 상속하고 다음과 같이 구현하는 것입니다.\n", - " * `__init__` 에서 층에 필요한 매개변수를 입력 받습니다..\n", - " * `build`, 입력 텐서의 크기를 알고 나머지를 초기화 할 수 있습니다.\n", - " * `call`, 정방향 연산(forward computation)을 진행 할 수 있습니다.\n", - "\n", - "변수를 생성하기 위해 `build`가 호출되길 기다릴 필요가 없다는 것에 주목하세요. 또한 변수를 `__init__`에 생성할 수도 있습니다. 그러나 `build`에 변수를 생성하는 유리한 점은 층이 작동할 입력의 크기를 기준으로 나중에 변수를 만들 수 있다는 것입니다. 반면에, `__init__`에 변수를 생성하는 것은 변수 생성에 필요한 크기가 명시적으로 지정되어야 함을 의미합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5Byl3n1k5kIy", - "colab": {} - }, - "source": [ - "class MyDenseLayer(tf.keras.layers.Layer):\n", - " def __init__(self, num_outputs):\n", - " super(MyDenseLayer, self).__init__()\n", - " self.num_outputs = num_outputs\n", - " \n", - " def build(self, input_shape):\n", - " self.kernel = self.add_variable(\"kernel\", \n", - " shape=[int(input_shape[-1]), \n", - " self.num_outputs])\n", - " \n", - " def call(self, input):\n", - " return tf.matmul(input, self.kernel)\n", - " \n", - "layer = MyDenseLayer(10)\n", - "print(layer(tf.zeros([10, 5])))\n", - "print(layer.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tk8E2vY0-z4Z" - }, - "source": [ - "다른 독자가 표준형 층의 동작을 잘 알고 있기 때문에, 가능한 경우 표준형 층을 사용하는것이 전체 코드를 읽고 유지하는데 더 쉽습니다. 만약 tf.keras.layers 또는 tf.contrib.layers에 없는 층을 사용하기 원하면 [깃허브](http://github.com/tensorflow/tensorflow/issues/new)에 이슈화하거나, 풀 리퀘스트(pull request)를 보내세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Qhg4KlbKrs3G" - }, - "source": [ - "## 모델: 층 구성\n", - "\n", - "머신러닝 모델에서 대부분의 재미있는 많은 것들은 기존의 층을 조합하여 구현됩니다. 예를 들어, 레스넷(resnet)의 각 잔여 블록(residual block)은 합성곱(convolution), 배치 정규화(batch normalization), 쇼트컷(shortcut) 등으로 구성되어 있습니다. \n", - "\n", - "다른층을 포함한 모델을 만들기 위해 사용하는 메인 클래스는 tf.keras.Model입니다. 다음은 tf.keras.Model을 상속(inheritance)하여 구현한 코드입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N30DTXiRASlb", - "colab": {} - }, - "source": [ - "class ResnetIdentityBlock(tf.keras.Model):\n", - " def __init__(self, kernel_size, filters):\n", - " super(ResnetIdentityBlock, self).__init__(name='')\n", - " filters1, filters2, filters3 = filters\n", - "\n", - " self.conv2a = tf.keras.layers.Conv2D(filters1, (1, 1))\n", - " self.bn2a = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2b = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same')\n", - " self.bn2b = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2c = tf.keras.layers.Conv2D(filters3, (1, 1))\n", - " self.bn2c = tf.keras.layers.BatchNormalization()\n", - "\n", - " def call(self, input_tensor, training=False):\n", - " x = self.conv2a(input_tensor)\n", - " x = self.bn2a(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2b(x)\n", - " x = self.bn2b(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2c(x)\n", - " x = self.bn2c(x, training=training)\n", - "\n", - " x += input_tensor\n", - " return tf.nn.relu(x)\n", - "\n", - " \n", - "block = ResnetIdentityBlock(1, [1, 2, 3])\n", - "print(block(tf.zeros([1, 2, 3, 3])))\n", - "print([x.name for x in block.trainable_variables])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yIbK5EiJyH_W", - "colab_type": "text" - }, - "source": [ - "그러나 대부분의 경우에, 많은 층으로 구성된 모델은 간단하게 연이어 하나의 층으로 호출할 수 있습니다. 이는 tf.keras.Sequential 사용하여 간단한 코드로 구현 가능합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "L9frk7Ur4uvJ", - "colab": {} - }, - "source": [ - " my_seq = tf.keras.Sequential([tf.keras.layers.Conv2D(1, (1, 1)),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(2, 1, \n", - " padding='same'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(3, (1, 1)),\n", - " tf.keras.layers.BatchNormalization()])\n", - "my_seq(tf.zeros([1, 2, 3, 3]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c5YwYcnuK-wc" - }, - "source": [ - "# 다음 단계\n", - "\n", - "이제 이전 노트북으로 돌아가서 선형 회귀 예제에 층과 모델을 사용하여 좀 더 나은 구조를 적용할 수 있습니다." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/eager/custom_training.ipynb b/site/ko/r1/tutorials/eager/custom_training.ipynb deleted file mode 100644 index 00e3dc72f98..00000000000 --- a/site/ko/r1/tutorials/eager/custom_training.ipynb +++ /dev/null @@ -1,453 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rmpybwysXGV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m8y3rGtQsYP2", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hrXv0rU9sIma" - }, - "source": [ - "# 사용자 정의 학습: 기초" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7S0BwJ_8sLu7" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Tt0udTybxxwG", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k2o3TTG4TFpt" - }, - "source": [ - "이전 튜토리얼에서는 머신러닝을 위한 기본 구성 요소인 자동 미분(automatic differentiation)을 위한 텐서플로 API를 알아보았습니다. 이번 튜토리얼에서는 이전 튜토리얼에서 소개되었던 텐서플로의 기본 요소를 사용하여 간단한 머신러닝을 수행해보겠습니다. \n", - "\n", - "텐서플로는 반복되는 코드를 줄이기 위해 유용한 추상화를 제공하는 고수준 신경망(neural network) API인 `tf.keras`를 포함하고 있습니다. 신경망을 다룰 때 이러한 고수준의 API을 강하게 추천합니다. 이번 짧은 튜토리얼에서는 탄탄한 기초를 기르기 위해 기본적인 요소만으로 신경망 훈련시켜 보겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3LXMVuV0VhDr" - }, - "source": [ - "## 설정" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "PJ64L90aVir3", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eMAWbDJFVmMk" - }, - "source": [ - "## 변수\n", - "\n", - "텐서플로의 텐서(Tensor)는 상태가 없고, 변경이 불가능한(immutable stateless) 객체입니다. 그러나 머신러닝 모델은 상태가 변경될(stateful) 필요가 있습니다. 예를 들어, 모델 학습에서 예측을 계산하기 위한 동일한 코드는 시간이 지남에 따라 다르게(희망하건대 더 낮은 손실로 가는 방향으로)동작해야 합니다. 이 연산 과정을 통해 변화되어야 하는 상태를 표현하기 위해 명령형 프로그래밍 언어인 파이썬을 사용 할 수 있습니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VkJwtLS_Jbn8", - "colab": {} - }, - "source": [ - "# 파이썬 구문 사용\n", - "x = tf.zeros([10, 10])\n", - "x += 2 # 이것은 x = x + 2와 같으며, x의 초기값을 변경하지 않습니다.\n", - "print(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfneTXy7JcUz" - }, - "source": [ - "텐서플로는 상태를 변경할 수 있는 연산자가 내장되어 있으며, 이러한 연산자는 상태를 표현하기 위한 저수준 파이썬 표현보다 사용하기가 더 좋습니다. 예를 들어, 모델에서 가중치를 나타내기 위해서 텐서플로 변수를 사용하는 것이 편하고 효율적입니다. \n", - "\n", - "텐서플로 변수는 값을 저장하는 객체로 텐서플로 연산에 사용될 때 저장된 이 값을 읽어올 것입니다. `tf.assign_sub`, `tf.scatter_update` 등은 텐서플로 변수에 저장되있는 값을 조작하는 연산자입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "itxmrMil6DQi", - "colab": {} - }, - "source": [ - "v = tf.Variable(1.0)\n", - "assert v.numpy() == 1.0\n", - "\n", - "# 값을 재배열합니다.\n", - "v.assign(3.0)\n", - "assert v.numpy() == 3.0\n", - "\n", - "# tf.square()와 같은 텐서플로 연산에 `v`를 사용하고 재할당합니다. \n", - "v.assign(tf.square(v))\n", - "assert v.numpy() == 9.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-paSaeq1JzwC" - }, - "source": [ - "변수를 사용한 연산은 그래디언트가 계산될 때 자동적으로 추적됩니다. 임베딩(embedding)을 나타내는 변수의 경우 기본적으로 희소 텐서(sparse tensor)를 사용하여 업데이트됩니다. 이는 연산과 메모리에 더욱 효율적입니다. \n", - "\n", - "또한 변수를 사용하는 것은 코드를 읽는 독자에게 상태가 변경될 수 있다는 것을 알려주는 손쉬운 방법입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BMiFcDzE7Qu3" - }, - "source": [ - "## 예: 선형 모델 훈련\n", - "\n", - "지금까지 몇 가지 개념을 설명했습니다. 간단한 모델을 구축하고 학습시키기 위해 ---`Tensor`, `GradientTape`, `Variable` --- 등을 사용하였고, 이는 일반적으로 다음의 과정을 포함합니다.\n", - "\n", - "1. 모델 정의\n", - "2. 손실 함수 정의\n", - "3. 훈련 데이터 가져오기\n", - "4. 훈련 데이터에서 실행, 데이터에 최적화하기 위해 \"옵티마이저(optimizer)\"를 사용한 변수 조정\n", - "\n", - "이번 튜토리얼에서는 선형 모델의 간단한 예제를 살펴보겠습니다. `f(x) = x * W + b`, 모델은 `W` 와 `b` 두 변수를 가지고 있는 선형모델이며, 잘 학습된 모델이 `W = 3.0` and `b = 2.0`의 값을 갖도록 합성 데이터를 만들겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFzH64Jn9PIm" - }, - "source": [ - "### 모델 정의\n", - "\n", - "변수와 연산을 캡슐화하기 위한 간단한 클래스를 정의해봅시다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_WRu7Pze7wk8", - "colab": {} - }, - "source": [ - "class Model(object):\n", - " def __init__(self):\n", - " # 변수를 (5.0, 0.0)으로 초기화 합니다.\n", - " # 실제로는 임의의 값으로 초기화 되어야합니다.\n", - " self.W = tf.Variable(5.0)\n", - " self.b = tf.Variable(0.0)\n", - " \n", - " def __call__(self, x):\n", - " return self.W * x + self.b\n", - " \n", - "model = Model()\n", - "\n", - "assert model(3.0).numpy() == 15.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xa6j_yXa-j79" - }, - "source": [ - "### 손실 함수 정의\n", - "\n", - "손실 함수는 주어진 입력에 대한 모델의 출력이 원하는 출력과 얼마나 잘 일치하는지를 측정합니다. 평균 제곱 오차(mean square error)를 적용한 손실 함수를 사용하겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y0ysUFGY924U", - "colab": {} - }, - "source": [ - "def loss(predicted_y, desired_y):\n", - " return tf.reduce_mean(tf.square(predicted_y - desired_y))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qutT_fkl_CBc" - }, - "source": [ - "### 훈련 데이터 가져오기\n", - "\n", - "약간의 잡음과 훈련 데이터를 합칩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gxPTb-kt_N5m", - "colab": {} - }, - "source": [ - "TRUE_W = 3.0\n", - "TRUE_b = 2.0\n", - "NUM_EXAMPLES = 1000\n", - "\n", - "inputs = tf.random_normal(shape=[NUM_EXAMPLES])\n", - "noise = tf.random_normal(shape=[NUM_EXAMPLES])\n", - "outputs = inputs * TRUE_W + TRUE_b + noise" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-50nq-wPBsAW" - }, - "source": [ - "모델을 훈련시키기 전에, 모델의 현재 상태를 시각화합시다. 모델의 예측을 빨간색으로, 훈련 데이터를 파란색으로 구성합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_eb83LtrB4nt", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(inputs, outputs, c='b')\n", - "plt.scatter(inputs, model(inputs), c='r')\n", - "plt.show()\n", - "\n", - "print('현재 손실: '),\n", - "print(loss(model(inputs), outputs).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sSDP-yeq_4jE" - }, - "source": [ - "### 훈련 루프 정의\n", - "\n", - "이제 네트워크와 훈련 데이터가 준비되었습니다. 모델의 변수(`W` 와 `b`)를 업데이트하기 위해 훈련 데이터를 사용하여 훈련시켜 보죠. 그리고 [경사 하강법(gradient descent)](https://en.wikipedia.org/wiki/Gradient_descent)을 사용하여 손실을 감소시킵니다. 경사 하강법에는 여러가지 방법이 있으며, `tf.train.Optimizer` 에 구현되어있습니다. 이러한 구현을 사용하는것을 강력히 추천드립니다. 그러나 이번 튜토리얼에서는 기본적인 방법을 사용하겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MBIACgdnA55X", - "colab": {} - }, - "source": [ - "def train(model, inputs, outputs, learning_rate):\n", - " with tf.GradientTape() as t:\n", - " current_loss = loss(model(inputs), outputs)\n", - " dW, db = t.gradient(current_loss, [model.W, model.b])\n", - " model.W.assign_sub(learning_rate * dW)\n", - " model.b.assign_sub(learning_rate * db)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RwWPaJryD2aN" - }, - "source": [ - "마지막으로, 훈련 데이터를 반복적으로 실행하고, `W` 와 `b`의 변화 과정을 확인합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XdfkR223D9dW", - "colab": {} - }, - "source": [ - "model = Model()\n", - "\n", - "# 도식화를 위해 W값과 b값의 변화를 저장합니다.\n", - "Ws, bs = [], []\n", - "epochs = range(10)\n", - "for epoch in epochs:\n", - " Ws.append(model.W.numpy())\n", - " bs.append(model.b.numpy())\n", - " current_loss = loss(model(inputs), outputs)\n", - "\n", - " train(model, inputs, outputs, learning_rate=0.1)\n", - " print('에포크 %2d: W=%1.2f b=%1.2f, 손실=%2.5f' %\n", - " (epoch, Ws[-1], bs[-1], current_loss))\n", - "\n", - "# 저장된 값들을 도식화합니다.\n", - "plt.plot(epochs, Ws, 'r',\n", - " epochs, bs, 'b')\n", - "plt.plot([TRUE_W] * len(epochs), 'r--',\n", - " [TRUE_b] * len(epochs), 'b--')\n", - "plt.legend(['W', 'b', 'true W', 'true_b'])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vPnIVuaSJwWz" - }, - "source": [ - "## 다음 단계\n", - "\n", - "이번 튜토리얼에서는 변수를 다루었으며, 지금까지 논의된 텐서플로의 기본 요소를 사용하여 간단한 선형 모델을 구축하고 훈련시켰습니다.\n", - "\n", - "이론적으로, 텐서플로를 머신러닝 연구에 사용하기 위해 알아야 할 것이 매우 많습니다. 실제로 신경망에 있어 `tf.keras`와 같은 고수준 API는 고수준 구성 요소(\"층\"으로 불리는)를 제공하고, 저장 및 복원을 위한 유틸리티, 손실 함수 모음, 최적화 전략 모음 등을 제공하기 때문에 더욱 편리합니다. " - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/eager/custom_training_walkthrough.ipynb b/site/ko/r1/tutorials/eager/custom_training_walkthrough.ipynb deleted file mode 100644 index dac6041cf42..00000000000 --- a/site/ko/r1/tutorials/eager/custom_training_walkthrough.ipynb +++ /dev/null @@ -1,1117 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training_walkthrough", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rwxGnsA92emp" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "CPII1rGR2rF9", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtEZ1pCPn--z" - }, - "source": [ - "# 사용자 정의 학습: 자세히 둘러보기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GV1F7tVTN3Dn" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "64VL9OukxoUc", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LDrzLFXE8T1l" - }, - "source": [ - "이번 튜토리얼은 붓꽃의 품종을 분류하기 위한 머신러닝 모델을 구축할 것입니다. 다음을 위해 즉시 실행[(eager execution)](https://www.tensorflow.org/r1/guide/eager)을 사용합니다.\n", - "1. 모델 구축\n", - "2. 모델 훈련\n", - "3. 예측을 위한 모델 사용\n", - "\n", - "## 텐서플로 프로그래밍\n", - "\n", - "이번 튜토리얼에서는 다음과 같은 고수준 텐서플로의 개념을 사용합니다.\n", - "\n", - "* [즉시 실행(eager execution)](https://www.tensorflow.org/r1/guide/eager) 개발 환경,\n", - "* [데이터셋 API](https://www.tensorflow.org/r1/guide/datasets)를 활용한 데이터 가져오기,\n", - "* [케라스 API](https://keras.io/getting-started/sequential-model-guide/)를 활용한 모델과 층(layer) 구축 .\n", - "\n", - "이번 튜토리얼은 다른 텐서플로 프로그램과 유사하게 구성되어있습니다.\n", - "\n", - "1. 데이터 가져오기 및 분석.\n", - "2. 모델 타입 선정.\n", - "3. 모델 훈련.\n", - "4. 모델 효과 평가.\n", - "5. 예측을 위한 훈련된 모델 사용." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yNr7H-AIoLOR" - }, - "source": [ - "## 프로그램 설정" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1J3AuPBT9gyR" - }, - "source": [ - "### 임포트 및 즉시 실행 구성\n", - "\n", - "텐서플로를 포함하여 필요한 파이썬 모듈을 임포트하고, 즉시 실행을 활성화합니다. 즉시 실행은 텐서플로 연산이 나중에 실행되는 [계산 그래프(computational graph)](https://www.tensorflow.org/r1/guide/graphs)를 만드는 대신에 연산을 즉시 평가하고 구체적인 값을 반환하게 합니다. 만약 파이썬 대화형 창이나 상호작용 콘솔을 사용하시면 더욱 익숙할 겁니다. 즉시 실행은 [Tensorlow >=1.8](https://www.tensorflow.org/install/) 부터 사용 가능합니다.\n", - "\n", - "즉시 실행이 활성화될 때, 동일한 프로그램내에서 비활성화 할 수 없습니다. 더 많은 세부사항은 [즉시 실행 가이드](https://www.tensorflow.org/r1/guide/eager)를 참조하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "g4Wzg69bnwK2", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "import matplotlib.pyplot as plt\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()\n", - "\n", - "print(\"텐서플로 버전: {}\".format(tf.__version__))\n", - "print(\"즉시 실행: {}\".format(tf.executing_eagerly()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zx7wc0LuuxaJ" - }, - "source": [ - "## 붓꽃 분류 문제\n", - "\n", - "당신이 식물학자라고 상상하고, 주어진 붓꽃을 자동적으로 분류하는 방법을 찾고 있다고 가정합시다. 머신러닝은 통계적으로 꽃을 분류할 수 있는 다양한 알고리즘을 제공합니다. 예를 들어, 정교한 머신러닝 프로그램은 사진을 통해 꽃을 분류할 수 있습니다. 이번 튜토리얼의 목적은 좀 더 겸손하게, 측정된 [꽃받침](https://en.wikipedia.org/wiki/Sepal)과 [꽃잎](https://en.wikipedia.org/wiki/Petal)의 길이와 폭을 토대로 붓꽃을 분류하는 것입니다.\n", - "\n", - "이 붓꽃은 약 300종입니다. 하지만 이번 튜토리얼에서는 오직 3가지 품종을 기준으로 분류할 것입니다. \n", - "\n", - "* Iris setosa\n", - "* Iris virginica\n", - "* Iris versicolor\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Petal\n", - "
    \n", - " 그림 1. Iris setosa (by Radomil, CC BY-SA 3.0), Iris versicolor, (by Dlanglois, CC BY-SA 3.0), and Iris virginica (by Frank Mayfield, CC BY-SA 2.0).
     \n", - "
    \n", - "\n", - "다행히도 다른 사람들이 먼저 꽃받침과 꽃잎의 길이와 폭이 측정된 [120개의 붓꽃 데이터](https://en.wikipedia.org/wiki/Iris_flower_data_set)를 만들어 놓았습니다. 이것은 머신러닝 분류 문제에 있어 초보자에게 유명한 고전 데이터셋입니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Px6KAg0Jowz" - }, - "source": [ - "## 훈련 데이터 가져오기 및 파싱\n", - "\n", - "데이터를 불러오고 파이썬 프로그램이 사용할 수 있는 구조로 전환합니다.\n", - "\n", - "### 데이터셋 다운로드\n", - "\n", - "[tf.keras.utils.get_file](https://www.tensorflow.org/api_docs/python/tf/keras/utils/get_file) 함수를 사용하여 데이터셋을 다운로드합니다. 이 함수는 다운로드된 파일의 경로를 반환합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J6c7uEU9rjRM", - "colab": {} - }, - "source": [ - "train_dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv\"\n", - "\n", - "train_dataset_fp = tf.keras.utils.get_file(fname=os.path.basename(train_dataset_url),\n", - " origin=train_dataset_url)\n", - "\n", - "print(\"데이터셋이 복사된 위치: {}\".format(train_dataset_fp))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qnX1-aLors4S" - }, - "source": [ - "### 데이터 탐색\n", - "\n", - "이 데이터셋(`iris_training.csv`)은 콤마 ','로 구분된 CSV 파일입니다. `head -n5` 명령을 사용하여 처음 5개 항목을 확인합니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FQvb_JYdrpPm", - "colab": {} - }, - "source": [ - "!head -n5 {train_dataset_fp}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kQhzD6P-uBoq" - }, - "source": [ - "처음 5개의 데이터로부터 다음을 주목하세요.\n", - "\n", - "1. 첫 번째 줄은 다음과 같은 정보를 포함하고 있는 헤더(header)입니다. \n", - " * 총 120개의 샘플이 있으며, 각 샘플들은 4개의 특성(feature), 3개의 레이블(label)을 가지고 있습니다.\n", - "2. 후속행은 데이터 레코드입니다. 한 줄당 한가지 *[샘플](https://developers.google.com/machine-learning/glossary/#example)*입니다.\n", - " * 처음 4개의 필드는 *[특성](https://developers.google.com/machine-learning/glossary/#feature)*입니다.: 이것들은 샘플의 특징을 나타냅니다. 이 필드들는 붓꽃의 측정값을 부동소수점으로 나타냅니다.\n", - " * 마지막 컬럼(column)은 *[레이블(label)](https://developers.google.com/machine-learning/glossary/#label)*입니다.: 레이블은 예측하고자 하는 값을 나타냅니다. 이 데이터셋에서는 꽃의 이름과 관련된 정수값 0, 1, 2를 나타냅니다.\n", - "\n", - "코드로 표현하면 다음과 같습니다.:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9Edhevw7exl6", - "colab": {} - }, - "source": [ - "# CSV 파일안에서 컬럼의 순서\n", - "column_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']\n", - "\n", - "feature_names = column_names[:-1]\n", - "label_name = column_names[-1]\n", - "\n", - "print(\"특성: {}\".format(feature_names))\n", - "print(\"레이블: {}\".format(label_name))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCtwLoJhhDNc" - }, - "source": [ - "각각의 레이블은 \"setosa\"와 같은 문자형 이름과 연관되어있습니다. 하지만 머신러닝은 전형적으로 숫자형 값에 의존합니다. 레이블을 다음과 같이 맵핑(mapping) 합니다. \n", - "\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica\n", - "\n", - "특성과 레이블에 관한 더 많은 정보를 위해서는 [머신러닝 특강의 전문용어 부분](https://developers.google.com/machine-learning/crash-course/framing/ml-terminology)을 참조하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sVNlJlUOhkoX", - "colab": {} - }, - "source": [ - "class_names = ['Iris setosa', 'Iris versicolor', 'Iris virginica']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dqPkQExM2Pwt" - }, - "source": [ - "### `tf.data.Dataset` 생성\n", - "\n", - "텐서플로의 [Dataset API](https://www.tensorflow.org/r1/guide/datasets)는 데이터를 적재할 때 발생하는 다양한 경우를 다룰 수 있습니다. 이는 훈련에 필요한 형태로 데이터를 읽고 변환하는 고수준 API입니다. 더 많은 정보를 위해서는 [데이터셋 빠른 실행 가이드](https://www.tensorflow.org/get_started/datasets_quickstart)를 참조하세요. \n", - "\n", - "\n", - "데이터셋이 CSV 파일이므로, 적절한 형태로 데이터를 구분하기위해 [make_csv_dataset](https://www.tensorflow.org/api_docs/python/tf/contrib/data/make_csv_dataset) 함수를 사용하겠습니다. 이 함수는 훈련 모델을 위한 데이터를 생성하므로, 초기값은 셔플(`shuffle=True, shuffle_buffer_size=10000`)과 무한반복(`num_epochs=None`)으로 설정되어있습니다. 또한 [배치 사이즈(batch_size)](https://developers.google.com/machine-learning/glossary/#batch_size)를 설정해줍니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WsxHnz1ebJ2S", - "colab": {} - }, - "source": [ - "batch_size = 32\n", - "\n", - "train_dataset = tf.contrib.data.make_csv_dataset(\n", - " train_dataset_fp,\n", - " batch_size, \n", - " column_names=column_names,\n", - " label_name=label_name,\n", - " num_epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gB_RSn62c-3G" - }, - "source": [ - "`make_csv_dataset` 함수는 `(features, label)` 쌍으로 구성된 `tf.data.Dataset`을 반환합니다. `features`는 딕셔너리 객체인: `{'feature_name': value}`로 주어집니다.\n", - "또한 즉시 실행 활성화로 이 `Dataset`은 반복가능합니다. 다음은 특성(feature)을 살펴봅시다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iDuG94H-C122", - "colab": {} - }, - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "features" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E63mArnQaAGz" - }, - "source": [ - "유사한 특성의 값은 같이 그룹 되어있거나, *배치* 돼있다는 사실에 주목하세요. 각 샘플 행의 필드는 해당 특성 배열에 추가됩니다. `batch_size`를 조절하여 이 특성 배열에 저장된 샘플의 수를 설정하세요.\n", - "\n", - "또한 배치(batch)로부터 약간의 특성을 도식화하여 군집돼있는 데이터를 확인할 수 있습니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "me5Wn-9FcyyO", - "colab": {} - }, - "source": [ - "plt.scatter(features['petal_length'].numpy(),\n", - " features['sepal_length'].numpy(),\n", - " c=labels.numpy(),\n", - " cmap='viridis')\n", - "\n", - "plt.xlabel(\"petal length\")\n", - "plt.ylabel(\"sepal length\");" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YlxpSyHlhT6M" - }, - "source": [ - "모델 구축 단계를 단순화하기 위해, 특성 딕셔너리를 `(batch_size, num_features)`의 형태를 가지는 단일 배열로 다시 구성하는 함수를 생성합니다.\n", - "\n", - "이 함수는 텐서의 리스트(list)로부터 값을 취하고 특정한 차원으로 결합된 텐서를 생성하는 [tf.stack](https://www.tensorflow.org/api_docs/python/tf/stack) 메서드(method)를 사용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jm932WINcaGU", - "colab": {} - }, - "source": [ - "def pack_features_vector(features, labels):\n", - " \"\"\"Pack the features into a single array.\"\"\"\n", - " features = tf.stack(list(features.values()), axis=1)\n", - " return features, labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "V1Vuph_eDl8x" - }, - "source": [ - "그 후 각 `(features,label)`쌍의 특성을 훈련 데이터셋에 쌓기위해 [tf.data.Dataset.map](https://www.tensorflow.org/api_docs/python/tf/data/dataset/map) 메서드를 사용합니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZbDkzGZIkpXf", - "colab": {} - }, - "source": [ - "train_dataset = train_dataset.map(pack_features_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NLy0Q1xCldVO" - }, - "source": [ - "데이터셋의 특성 요소는 이제 형태가 `(batch_size, num_features)`인 배열입니다. 첫 5개행의 샘플을 살펴봅시다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kex9ibEek6Tr", - "colab": {} - }, - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "print(features[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LsaVrtNM3Tx5" - }, - "source": [ - "## 모델 타입 선정\n", - "\n", - "### 왜 모델을 사용해야하는가?\n", - "\n", - " *[모델](https://developers.google.com/machine-learning/crash-course/glossary#model)*은 특성(feature)과 레이블(label) 과의 관계입니다. 붓꽃 분류 문제에서 모델은 측정된 꽃받침과 꽃잎 사이의 관계를 정의하고 붓꽃의 품종을 예측합니다. 몇 가지 간단한 모델은 몇 줄의 대수학으로 표현할 수 있으나, 복잡한 머신러닝 모델은 요약하기 힘든 굉장히 많은 수의 매개변수를 가지고 있습니다.\n", - "\n", - "머신러닝을 사용하지 않고 4가지의 특성 사이의 관계를 결정하고 붓꽃을 품종을 예측하실 수 있으신가요? 즉, 특정 품종의 꽃받침과 꽃잎과의 관계를 정의할 수 있을 정도로 데이터셋을 분석했다면, 전통적인 프로그래밍 기술(예를 들어 굉장히 많은 조건문)을 사용하여 모델은 만들 수 있으신가요? 더 복잡한 데이터셋에서 이는 불가능에 가까울 수 있습니다. 잘 구성된 머신러닝은 사용자를 위한 모델을 결정합니다. 만약 충분히 좋은 샘플을 잘 구성된 머신러닝 모델에 제공한다면, 프로그램은 사용자를 위한 특성 간의 관계를 이해하고 제공합니다. \n", - "\n", - "### 모델 선정\n", - "\n", - "이제 학습을 위한 모델의 종류를 선정해야합니다. 여러 종류의 모델이 있고, 이를 선택하는 것은 많은 경험이 필요합니다. 이번 튜토리얼에서는 붓꽃 분류 문제를 해결하기위해 *[신경망(neural network)](https://developers.google.com/machine-learning/glossary/#neural_network)* 모델을 사용하겠습니다. 신경망 모델은 특성과 레이블 사이의 복잡한 관계를 찾을 수 있습니다. 신경망은 하나 또는 그 이상의 *[은닉층(hidden layer)](https://developers.google.com/machine-learning/glossary/#hidden_layer)*으로 구성된 그래프입니다. 각각의 은닉층은 하나 이상의 *[뉴런(neuron)](https://developers.google.com/machine-learning/glossary/#neuron)*으로 구성되어있습니다. 몇가지 신경망의 범주가 있으며, 이번 튜토리얼에서는 *[밀집(dense) 또는 완전 연결 신경망(fully-connected neural network)](https://developers.google.com/machine-learning/glossary/#fully_connected_layer)*를 사용합니다: 완전 연결 신경망(fully-connected neural network)은 하나의 뉴런에 이전층의 모든 뉴런의 입력을 받는 신경망입니다. 예를 들어, `그림 2`는 입력층, 2개의 은닉층, 그리고 출력층으로 구성된 완전 연결 신경망입니다. \n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \n", - "
    \n", - " 그림 2. A neural network with features, hidden layers, and predictions.
     \n", - "
    \n", - "\n", - "그림 2의 모델이 훈련된 다음 레이블 되어있지 않은 데이터를 제공했을때, 모델은 주어진 데이터의 3가지(주어진 레이블의 개수) 예측을 출력합니다. 이러한 예측은 *[추론(inference)](https://developers.google.com/machine-learning/crash-course/glossary#inference)*이라고 불립니다. 이 샘플에서 출력의 합은 1.0입니다. 그림 2에서 예측은 *Iris setosa* `0.02`, *Iris versicolor* `0.95`, *Iris virginica*에 `0.03`로 주어집니다. 이는 모델이 95%의 확률로 주어진 데이터를 *Iris versicolor*로 예측한다는 것을 의미합니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W23DIMVPQEBt" - }, - "source": [ - "### 케라스를 사용한 모델 생성\n", - "\n", - "텐서플로의 [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras) API는 모델과 층을 생성하기 위한 풍부한 라이브러리를 제공합니다. 케라스가 구성 요소를 연결하기 위한 복잡함을 모두 처리해 주기 때문에 모델을 구축하고 실험하는 것이 쉽습니다.\n", - "\n", - "[tf.keras.Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential)은 여러 층을 연이어 쌓은 모델입니다. 이 구조는 층의 인스턴스를 취하며, 아래의 경우 각 층당 10개의 노드(node)를 가지는 2개의 [Dense(완전 연결 층)](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense)과 3개의 예측(레이블의 수) 노드를 가지는 출력 층으로 구성되어있습니다. 첫 번째 층의 `input_shape` 매개변수는 데이터셋의 특성의 수와 관계있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2fZ6oL2ig3ZK", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(4,)), # input shape required\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu),\n", - " tf.keras.layers.Dense(3)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FHcbEzMpxbHL" - }, - "source": [ - "*[활성화 함수(activation function)](https://developers.google.com/machine-learning/crash-course/glossary#activation_function)*는 각 층에서 출력의 크기를 결정합니다. 이러한 비선형성은 중요하며, 활성화 함수가 없는 모델은 하나의 층과 동일하다고 생각할 수 있습니다. 사용 가능한 [활성화 함수](https://www.tensorflow.org/api_docs/python/tf/keras/activations)는 많지만, [ReLU](https://developers.google.com/machine-learning/crash-course/glossary#ReLU)가 은닉층에 주로 사용됩니다. \n", - "\n", - "이상적인 은닉층과 뉴런의 개수는 문제와 데이터셋에 의해 좌우됩니다. 머신러닝의 여러 측면과 마찬가지로, 최적의 신경망 타입을 결정하는 것은 많은 경험과 지식이 필요합니다. 경험을 토대로 보면 은닉층과 뉴런의 증가는 전형적으로 강력한 모델을 생성하므로, 모델을 효과적으로 훈련시키기 위해서 더 많은 데이터를 필요로 합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2wFKnhWCpDSS" - }, - "source": [ - "### 모델 사용\n", - "\n", - "이 모델이 특성의 배치에 대해 수행하는 작업을 간단히 살펴봅시다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xe6SQ5NrpB-I", - "colab": {} - }, - "source": [ - "predictions = model(features)\n", - "predictions[:5]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wxyXOhwVr5S3" - }, - "source": [ - "각 샘플은 각 클래스에 대한 [로짓(logit)](https://developers.google.com/machine-learning/crash-course/glossary#logits)을 반환합니다. \n", - "\n", - "이 로짓(logit)을 각 클래스에 대한 확률로 변환하기 위하서 [소프트맥스(softmax)](https://developers.google.com/machine-learning/crash-course/glossary#softmax) 함수를 사용하겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_tRwHZmTNTX2", - "colab": {} - }, - "source": [ - "tf.nn.softmax(predictions[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uRZmchElo481" - }, - "source": [ - "`tf.argmax`는 예측된 값 중 가장 큰 확률(원하는 클래스)을 반환합니다. 하지만 모델이 아직 훈련되지 않았으므로 이는 좋은 예측이 아닙니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-Jzm_GoErz8B", - "colab": {} - }, - "source": [ - "print(\"예측: {}\".format(tf.argmax(predictions, axis=1)))\n", - "print(\" 레이블: {}\".format(labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vzq2E5J2QMtw" - }, - "source": [ - "## 모델 훈련하기\n", - "\n", - "*[훈련 단계](https://developers.google.com/machine-learning/crash-course/glossary#training)*는 모델이 점진적으로 최적화되거나 데이터셋을 학습하는 머신러닝의 과정입니다. 훈련의 목적은 미지의 데이터를 예측하기 위해, 훈련 데이터셋의 구조에 대해서 충분히 학습하는 것입니다. 만약 모델이 훈련 데이터셋에 대해서 과하게 학습된다면 오직 훈련 데이터셋에 대해서 작동할 것이며, 일반화되기 힘들 것입니다. 이러한 문제를 *[과대적합(overfitting)](https://developers.google.com/machine-learning/crash-course/glossary#overfitting)* 이라고 합니다. 이는 마치 문제를 이해하고 해결한다기보다는 답을 기억하는 것이라고 생각할 수 있습니다. \n", - "\n", - "붓꽃 분류 문제는 *[지도 학습(supervised machine learning)](https://developers.google.com/machine-learning/glossary/#supervised_machine_learning)*의 예시 중 하나입니다.: 지도학습은 모델이 레이블을 포함한 훈련 데이터로부터 학습됩니다. *[비지도 학습(unsupervised machine learning)](https://developers.google.com/machine-learning/glossary/#unsupervised_machine_learning)*에서는 훈련 데이터가 레이블을 포함하고 있지 않습니다. 대신에 모델은 특성 간의 패턴을 찾습니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RaKp8aEjKX6B" - }, - "source": [ - "### 손실 함수와 그래디언트 함수 정의하기\n", - "\n", - "훈련과 평가단계에서 모델의 *[손실(loss)](https://developers.google.com/machine-learning/crash-course/glossary#loss)*을 계산해야 합니다. 손실은 모델의 예측이 원하는 레이블과 얼마나 일치하는지, 또한 모델이 잘 작동하는지에 대한 척도로 사용됩니다. 이 값을 최소화하고, 최적화 해야합니다.\n", - "\n", - "모델의 손실은 [tf.keras.losses.categorical_crossentropy](https://www.tensorflow.org/api_docs/python/tf/losses/sparse_softmax_cross_entropy) 함수를 사용해 계산할 것입니다. 이 함수는 모델의 클래스(레이블)과 예측된 값(로짓)을 입력받아 샘플의 평균 손실을 반환합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tMAT4DcMPwI-", - "colab": {} - }, - "source": [ - "def loss(model, x, y):\n", - " y_ = model(x)\n", - " return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=y_)\n", - "\n", - "\n", - "l = loss(model, features, labels)\n", - "print(\"손실 테스트: {}\".format(l))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3IcPqA24QM6B" - }, - "source": [ - "모델을 최적화하기 위해 사용되는 *[그래디언트(gradient)](https://developers.google.com/machine-learning/crash-course/glossary#gradient)*를 계산하기 위해 [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) 컨텍스트를 사용합니다. 더 자세한 정보는 [즉시 실행 가이드](https://www.tensorflow.org/r1/guide/eager)를 확인하세요. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x57HcKWhKkei", - "colab": {} - }, - "source": [ - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return loss_value, tape.gradient(loss_value, model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lOxFimtlKruu" - }, - "source": [ - "### 옵티마이저 생성 \n", - "\n", - "*[옵티마이저(optimizer)](https://developers.google.com/machine-learning/crash-course/glossary#optimizer)*는 손실 함수를 최소화하기 위해 계산된 그래디언트를 모델의 변수에 적용합니다. 손실 함수를 구부러진 곡선의 표면(그림 3)으로 생각할 수 있으며, 이 함수의 최저점을 찾고자 합니다. 그래디언트는 가장 가파른 상승 방향을 가리키며 따라서 반대 방향으로 이동하는 여행을 합니다. 각 배치마다의 손실과 기울기를 반복적으로 계산하여 훈련과정 동안 모델을 조정합니다. 점진적으로, 모델은 손실을 최소화하기 위해 가중치(weight)와 편향(bias)의 최적의 조합을 찾아냅니다. 손실이 낮을수록 더 좋은 모델의 예측을 기대할 수 있습니다.\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Optimization\n", - "
    \n", - " 그림 3. 3차원 공간에 대한 최적화 알고리즘 시각화.
    (Source: Stanford class CS231n, MIT License, Image credit: Alec Radford)\n", - "
    \n", - "\n", - "텐서플로는 훈련을 위해 사용 가능한 여러종류의 [최적화 알고리즘](https://www.tensorflow.org/api_guides/python/train)을 가지고 있습니다. 이번 모델에서는 *[확률적 경사 하강법(stochastic gradient descent, SGD)](https://developers.google.com/machine-learning/crash-course/glossary#gradient_descent)* 을 구현한 [tf.train.GradientDescentOptimizer](https://www.tensorflow.org/api_docs/python/tf/train/GradientDescentOptimizer)를 사용하겠습니다. `learning_rate`은 경사하강 과정의 크기를 나타내는 매개변수이며, 더 나은 결과를 위해 조절가능한 *하이퍼파라미터(hyperparameter)* 입니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XkUd6UiZa_dF" - }, - "source": [ - "옵티마이저(optimizer)와 `global_step`을 설정합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8xxi2NNGKwG_", - "colab": {} - }, - "source": [ - "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n", - "\n", - "global_step = tf.contrib.eager.Variable(0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pJVRZ0hP52ZB" - }, - "source": [ - "이 값들을 단일 최적화 단계를 계산하기 위해 사용합니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rxRNTFVe56RG", - "colab": {} - }, - "source": [ - "loss_value, grads = grad(model, features, labels)\n", - "\n", - "print(\"단계: {}, 초기 손실: {}\".format(global_step.numpy(),\n", - " loss_value.numpy()))\n", - "\n", - "optimizer.apply_gradients(zip(grads, model.trainable_variables), global_step)\n", - "\n", - "print(\"단계: {}, 손실: {}\".format(global_step.numpy(),\n", - " loss(model, features, labels).numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Y2VSELvwAvW" - }, - "source": [ - "### 훈련 루프\n", - "\n", - "모든 사항이 갖춰졌으므로 모델을 훈련할 준비가 되었습니다! 훈련 루프는 더 좋은 예측을 위해 데이터셋을 모델로 제공합니다. 다음의 코드 블럭은 아래의 훈련 단계를 작성한 것입니다. \n", - "\n", - "1. 각 *에포크(epoch)* 반복. 에포크는 데이터셋을 통과시키는 횟수입니다. \n", - "2. 에포크 내에서, *특성* (`x`)와 *레이블* (`y`)가 포함된 훈련 `데이터셋`에 있는 샘플을 반복합니다.\n", - "3. 샘플의 특성을 사용하여 결과를 예측 하고 레이블과 비교합니다. 예측의 부정확도를 측정하고 모델의 손실과 그래디언트를 계산하기 위해 사용합니다. \n", - "4. 모델의 변수를 업데이트하기 위해 `옵티마이저`를 사용합니다. \n", - "5. 시각화를 위해 몇가지 값들을 저장합니다.\n", - "6. 각 에포크를 반복합니다.\n", - "\n", - "`num_epochs` 변수는 데이터셋의 반복 횟수입니다. 직관과는 반대로, 모델을 길게 학습하는 것이 더 나은 모델이 될 것이라고 보장하지 못합니다. `num_epochs`는 조정가능한 *[하이퍼파라미터(hyperparameter)](https://developers.google.com/machine-learning/glossary/#hyperparameter)* 입니다. 적절한 횟수를 선택하는 것은 많은 경험과 직관을 필요로 합니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AIgulGRUhpto", - "colab": {} - }, - "source": [ - "## Note: 이 셀을 다시 실행하면 동일한 모델의 변수가 사용됩니다.\n", - "\n", - "from tensorflow import contrib\n", - "tfe = contrib.eager\n", - "\n", - "# 도식화를 위해 결과를 저장합니다.\n", - "train_loss_results = []\n", - "train_accuracy_results = []\n", - "\n", - "num_epochs = 201\n", - "\n", - "for epoch in range(num_epochs):\n", - " epoch_loss_avg = tfe.metrics.Mean()\n", - " epoch_accuracy = tfe.metrics.Accuracy()\n", - "\n", - " # 훈련 루프 - 32개의 배치를 사용합니다.\n", - " for x, y in train_dataset:\n", - " # 모델을 최적화합니다.\n", - " loss_value, grads = grad(model, x, y)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables),\n", - " global_step)\n", - "\n", - " # 진행 상황을 추적합니다.\n", - " epoch_loss_avg(loss_value) # 현재 배치 손실을 추가합니다.\n", - " # 예측된 레이블과 실제 레이블 비교합니다.\n", - " epoch_accuracy(tf.argmax(model(x), axis=1, output_type=tf.int32), y)\n", - "\n", - " # epoch 종료\n", - " train_loss_results.append(epoch_loss_avg.result())\n", - " train_accuracy_results.append(epoch_accuracy.result())\n", - " \n", - " if epoch % 50 == 0:\n", - " print(\"에포크 {:03d}: 손실: {:.3f}, 정확도: {:.3%}\".format(epoch, \n", - " epoch_loss_avg.result(), \n", - " epoch_accuracy.result()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2FQHVUnm_rjw" - }, - "source": [ - "### 시간에 따른 손실함수 시각화" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j3wdbmtLVTyr" - }, - "source": [ - "모델의 훈련 과정을 출력하는 것도 도움이 되지만, 훈련 과정을 직접 보는 것이 더 도움이 되곤합니다. [텐서보드(tensorboard)](https://www.tensorflow.org/r1/guide/summaries_and_tensorboard)는 텐서플로에 패키지 되어있는 굉장히 유용한 시각화 툴입니다. 하지만 `matplotlib` 모듈을 사용하여 일반적인 도표를 출력할 수 있습니다.\n", - "\n", - "이 도표를 해석하는 것은 여러 경험이 필요하지만, 결국 모델을 최적화하기 위해 *손실*이 내려가고 *정확도*가 올라가는 것을 원합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "agjvNd2iUGFn", - "colab": {} - }, - "source": [ - "fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))\n", - "fig.suptitle('Training Metrics')\n", - "\n", - "axes[0].set_ylabel(\"loss\", fontsize=14)\n", - "axes[0].plot(train_loss_results)\n", - "\n", - "axes[1].set_ylabel(\"Accuracy\", fontsize=14)\n", - "axes[1].set_xlabel(\"epoch\", fontsize=14)\n", - "axes[1].plot(train_accuracy_results);" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zg8GoMZhLpGH" - }, - "source": [ - "## 모델 유효성 평가\n", - "\n", - "이제 모델은 훈련되었습니다. 모델의 성능에 대한 몇가지 통계를 얻을 수 있습니다. \n", - "\n", - "*평가(Evaluating)*는 모델이 예측을 얼마나 효과적으로 수행하는지 결정하는 것을 의미합니다. 붓꽃 분류 모델의 유효성을 결정하기 위해, 몇가지 꽃잎과 꽃받침 데이터를 통과시키고 어떠한 품종을 예측하는지 확인합니다. 그 후 실제 품종과 비교합니다. 예를 들어, 절반의 데이터를 올바르게 예측한 모델의 *[정확도](https://developers.google.com/machine-learning/glossary/#accuracy)* 는 `0.5`입니다. 그림 4는 조금 더 효과적인 모델입니다. 5개의 예측 중 4개를 올바르게 예측하여 80% 정확도를 냅니다.\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    샘플 특성레이블모델 예측
    5.93.04.31.511
    6.93.15.42.122
    5.13.31.70.500
    6.0 3.4 4.5 1.6 12
    5.52.54.01.311
    \n", - " 그림 4. 80% 정확도 붓꽃 분류기.
     \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z-EvK7hGL0d8" - }, - "source": [ - "### 테스트 데이터 세트 설정\n", - "\n", - "모델을 평가하는 것은 모델을 훈련하는 것과 유사합니다. 가장 큰 차이는 훈련 데이터가 아닌 *[테스트 데이터 세트](https://developers.google.com/machine-learning/crash-course/glossary#test_set)* 를 사용했다는 것입니다. 공정하게 모델의 유효성을 평가하기 위해, 모델을 평가하기 위한 샘플은 반드시 훈련 데이터와 달라야합니다. \n", - "\n", - "테스트 데이터 세트를 설정하는 것은 훈련 데이터 세트를 설정하는 것과 유사합니다. CSV 파일을 다운로드하고 값을 파싱합니다. 그 후 셔플은 적용하지 않습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ps3_9dJ3Lodk", - "colab": {} - }, - "source": [ - "test_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv\"\n", - "\n", - "test_fp = tf.keras.utils.get_file(fname=os.path.basename(test_url),\n", - " origin=test_url)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SRMWCu30bnxH", - "colab": {} - }, - "source": [ - "test_dataset = tf.contrib.data.make_csv_dataset(\n", - " test_fp,\n", - " batch_size, \n", - " column_names=column_names,\n", - " label_name='species',\n", - " num_epochs=1,\n", - " shuffle=False)\n", - "\n", - "test_dataset = test_dataset.map(pack_features_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HFuOKXJdMAdm" - }, - "source": [ - "### 테스트 데이터 세트를 사용한 모델 평가\n", - "\n", - "훈련 단계와는 다르게 모델은 테스트 데이터에 대해서 오직 한 번의 [에포크](https://developers.google.com/machine-learning/glossary/#epoch)를 진행합니다. 다음의 코드 셀은 테스트 셋에 있는 샘플에 대해 실행하고 실제 레이블과 비교합니다. 이는 전체 테스트 데이터 세트에 대한 정확도를 측정하는데 사용됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tw03-MK1cYId", - "colab": {} - }, - "source": [ - "test_accuracy = tfe.metrics.Accuracy()\n", - "\n", - "for (x, y) in test_dataset:\n", - " logits = model(x)\n", - " prediction = tf.argmax(logits, axis=1, output_type=tf.int32)\n", - " test_accuracy(prediction, y)\n", - "\n", - "print(\"테스트 세트 정확도: {:.3%}\".format(test_accuracy.result()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HcKEZMtCOeK-" - }, - "source": [ - "마지막 배치에서 모델이 올바르게 예측한 것을 확인할 수 있습니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uNwt2eMeOane", - "colab": {} - }, - "source": [ - "tf.stack([y,prediction],axis=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Li2r1tYvW7S" - }, - "source": [ - "## 훈련된 모델로 예측하기\n", - "\n", - "이제 붓꽃을 분류하기 위해 완벽하지는 않지만 어느 정도 검증된 모델을 가지고 있습니다. 훈련된 모델을 사용하여 [레이블 되지 않은 데이터](https://developers.google.com/machine-learning/glossary/#unlabeled_example)를 예측해봅시다.\n", - "\n", - "실제로는 레이블 되지 않은 샘플들은 여러 소스(앱, CSV 파일, 직접 제공 등)로부터 제공될 수 있습니다. 지금은 레이블을 예측하기 위해 수동으로 3개의 레이블 되지 않은 샘플을 제공하겠습니다. 레이블은 다음과 같은 붓꽃 이름으로 매핑되어있습니다.\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kesTS5Lzv-M2", - "colab": {} - }, - "source": [ - "predict_dataset = tf.convert_to_tensor([\n", - " [5.1, 3.3, 1.7, 0.5,],\n", - " [5.9, 3.0, 4.2, 1.5,],\n", - " [6.9, 3.1, 5.4, 2.1]\n", - "])\n", - "\n", - "predictions = model(predict_dataset)\n", - "\n", - "for i, logits in enumerate(predictions):\n", - " class_idx = tf.argmax(logits).numpy()\n", - " p = tf.nn.softmax(logits)[class_idx]\n", - " name = class_names[class_idx]\n", - " print(\"예 {} 예측: {} ({:4.1f}%)\".format(i, name, 100*p))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/eager/eager_basics.ipynb b/site/ko/r1/tutorials/eager/eager_basics.ipynb deleted file mode 100644 index f2864e56e25..00000000000 --- a/site/ko/r1/tutorials/eager/eager_basics.ipynb +++ /dev/null @@ -1,480 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "eager_basics.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iPpI7RaYoZuE" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "hro2InpHobKk", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# 즉시 실행 기초" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Hndw-YcxoOJK" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "D3iRBJ-xxfU3", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6sILUVbHoSgH" - }, - "source": [ - "이 노트북은 텐서플로를 사용하기 위한 입문 튜토리얼입니다. 다음 내용을 다룹니다 : \n", - "\n", - "* 필요한 패키지 임포트\n", - "* 텐서(Tensor) 생성 및 사용\n", - "* GPU 가속기 사용\n", - "* 데이터 세트" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1JcS5iBXMRO" - }, - "source": [ - "## 텐서플로 임포트\n", - "\n", - "시작하기 위해서 텐서플로 모듈을 임포트하고 즉시 실행(eager execution)을 활성화합니다. 즉시 실행 활성화로 텐서플로를 조금 더 대화형 프론트엔드(frontend)에 가깝게 만들어 줍니다. 세부사항은 나중에 이야기할 것입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "code", - "colab_type": "code", - "id": "RlIWhyeLoYnG", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H9UySOPLXdaw" - }, - "source": [ - "## 텐서\n", - "\n", - "텐서는 다차원 배열입니다. 넘파이(NumPy) `ndarray` 객체와 비슷하며, `Tensor` 객체는 데이터 타입과 크기를 가지고 있습니다. 또한 텐서는 GPU 같은 가속기 메모리에 상주할 수 있습니다. 텐서플로는 텐서를 생성하고 이용하는 풍부한 연산 라이브러리([tf.add](https://www.tensorflow.org/api_docs/python/tf/add), [tf.matmul](https://www.tensorflow.org/api_docs/python/tf/matmul), [tf.linalg.inv](https://www.tensorflow.org/api_docs/python/tf/linalg/inv) etc.)를 제공합니다. 이러한 연산자는 자동적으로 순수 파이썬 타입을 변환합니다. 예를 들어:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "code", - "colab_type": "code", - "id": "ngUe237Wt48W", - "colab": {} - }, - "source": [ - "print(tf.add(1, 2))\n", - "print(tf.add([1, 2], [3, 4]))\n", - "print(tf.square(5))\n", - "print(tf.reduce_sum([1, 2, 3]))\n", - "print(tf.encode_base64(\"hello world\"))\n", - "\n", - "# 연산자의 오버로딩(overloding) 또한 지원합니다.\n", - "print(tf.square(2) + tf.square(3))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDY4WsYRhP81" - }, - "source": [ - "각각의 텐서는 크기와 데이터 타입을 가지고 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "srYWH1MdJNG7", - "colab": {} - }, - "source": [ - "x = tf.matmul([[1]], [[2, 3]])\n", - "print(x.shape)\n", - "print(x.dtype)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eBPw8e8vrsom" - }, - "source": [ - "넘파이 배열과 텐서플로 텐서의 가장 확연한 차이는 다음과 같습니다:\n", - "\n", - "1. `텐서`는 가속기 메모리(GPU, TPU와 같은)에서 사용할 수 있습니다.\n", - "2. `텐서`는 불변성(immutable)을 가집니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dwi1tdW3JBw6" - }, - "source": [ - "### 넘파이 호환성\n", - "\n", - "텐서와 넘파이 배열 사이의 변환은 다소 간단합니다.\n", - "\n", - "* 텐서플로 연산은 자동적으로 넘파이 배열을 텐서로 변환합니다.\n", - "* 넘파이 연산은 자동적으로 텐서를 넘파이 배열로 변환합니다.\n", - "\n", - "텐서는 `.numpy()` 메서드(method)를 호출하여 넘파이 배열로 변환할 수 있습니다.\n", - "가능한 경우, 텐서와 배열은 메모리 표현을 공유하기 때문에 이러한 변환은 일반적으로 간단(저렴)합니다. 그러나 텐서는 GPU 메모리에 저장될 수 있고, 넘파이 배열은 항상 호스트 메모리에 저장되므로, 이러한 변환이 항상 가능한 것은 아닙니다. 따라서 GPU에서 호스트 메모리로의 복사가 필요합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lCUWzso6mbqR", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "ndarray = np.ones([3, 3])\n", - "\n", - "print(\"텐서플로 연산은 자동적으로 넘파이 배열을 텐서로 변환합니다.\")\n", - "tensor = tf.multiply(ndarray, 42)\n", - "print(tensor)\n", - "\n", - "\n", - "print(\"그리고 넘파이 연산은 자동적으로 텐서를 넘파이 배열로 변환합니다.\")\n", - "print(np.add(tensor, 1))\n", - "\n", - "print(\".numpy() 메서드는 텐서를 넘파이 배열로 변환합니다.\")\n", - "print(tensor.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PBNP8yTRfu_X" - }, - "source": [ - "## GPU 가속기\n", - "\n", - "대부분의 텐서플로 연산은 GPU를 사용하여 가속화할 수 있습니다. 어떠한 주석(annotation)도 없이, 텐서플로는 연산을 위해 자동적으로 CPU 또는 GPU를 사용할 것인지를 정합니다(그리고 필요시 텐서를 CPU 와 GPU에 복사합니다.) 연산에 의해 생성된 텐서는 전형적으로 연산이 실행된 장치의 메모리에 의해 실행됩니다. 예를 들어:" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "code", - "colab_type": "code", - "id": "3Twf_Rw-gQFM", - "colab": {} - }, - "source": [ - "x = tf.random_uniform([3, 3])\n", - "\n", - "print(\"GPU 사용이 가능한가 : \"),\n", - "print(tf.test.is_gpu_available())\n", - "\n", - "print(\"텐서가 GPU #0에 있는가 : \"),\n", - "print(x.device.endswith('GPU:0'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vpgYzgVXW2Ud" - }, - "source": [ - "### 장치 이름\n", - "\n", - "`Tensor.device`는 텐서를 구성하고 있는 호스트 장치의 풀네임을 제공합니다. 이러한 이름은 프로그램이 실행중인 호스트의 네트워크 주소 및 해당 호스트 내의 장치와 같은 많은 세부 정보를 인코딩하며, 이것은 텐서플로 프로그램의 분산 실행에 필요합니다. 텐서가 호스트의 `N`번째 GPU에 놓여지면 문자열은 `GPU:`으로 끝납니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZWZQCimzuqyP" - }, - "source": [ - "### 명시적 장치 배치\n", - "\n", - "텐서플로에서 \"배치(replacement)\"라는 용어는 개별 연산을 실행하기 위해 장치에 할당(배치) 하는 것입니다. 앞서 언급했듯이, 명시적 지침이 없을 경우 텐서플로는 연산을 실행하기 위한 장치를 자동으로 결정하고, 필요시 텐서를 장치에 복사합니다. 그러나 텐서플로 연산은 `tf.device`을 사용하여 특정한 장치에 명시적으로 배치할 수 있습니다. \n", - "예를 들어:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RjkNZTuauy-Q", - "colab": {} - }, - "source": [ - "import time\n", - "\n", - "def time_matmul(x):\n", - " start = time.time()\n", - " for loop in range(10):\n", - " tf.matmul(x, x)\n", - "\n", - " result = time.time()-start\n", - " \n", - " print(\"10 loops: {:0.2f}ms\".format(1000*result))\n", - "\n", - "\n", - "# CPU에서 강제실행합니다.\n", - "print(\"On CPU:\")\n", - "with tf.device(\"CPU:0\"):\n", - " x = tf.random_uniform([1000, 1000])\n", - " assert x.device.endswith(\"CPU:0\")\n", - " time_matmul(x)\n", - "\n", - "# GPU #0가 이용가능시 GPU #0에서 강제실행합니다.\n", - "if tf.test.is_gpu_available():\n", - " with tf.device(\"GPU:0\"): # 또는 GPU:1, GPU:2\n", - " x = tf.random_uniform([1000, 1000])\n", - " assert x.device.endswith(\"GPU:0\")\n", - " time_matmul(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o1K4dlhhHtQj" - }, - "source": [ - "## 데이터셋\n", - "\n", - "이번 섹션에서는 모델에 데이터를 제공하기 위한 파이프라인을 구축하기 위해 [`tf.data.Dataset` API](https://www.tensorflow.org/r1/guide/datasets)를 시연해볼 것입니다. 이는 다음을 포함합니다.\n", - "\n", - "* 데이터셋 생성.\n", - "* 즉시 실행 활성화를 통한 데이터셋 반복\n", - "\n", - "모델을 훈련시키고 평가 루프를 제공할 간단하고 재사용 가능한 모듈로부터, 복잡한 입력 파이프라인을 구축하기위해 데이터셋 API를 사용하기를 권장합니다. \n", - "\n", - "만약 텐서플로 그래프에 익숙하다면 알겠지만, 데이터셋 객체를 생성하기 위한 API는 즉시 실행이 활성화 되어도 동일하게 유지됩니다. 하지만 데이터셋의 요소를 반복하는 프로세스가 약간 더 간단해집니다.\n", - "또한 `tf.data.Dataset` 객체를 통하여 파이썬 반복문을 사용할 수 있으며, 명시적으로 `tf.data.Iterator` 객체를 생성할 필요가 없습니다.\n", - "그 결과, [텐서플로 가이드](https://www.tensorflow.org/r1/guide/datasets)의 반복자(iterator)에 관한 논의는 즉시 실행이 활성화될 때에는 신경 쓰지 않아도 됩니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zI0fmOynH-Ne" - }, - "source": [ - "### 소스 Dataset 생성\n", - "\n", - "굉장히 유용한 함수중 하나인 [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices)와 같은 팩토리(factory) 함수 중 하나를 사용하거나 파일로부터 읽어들이는 객체인 [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) 또는 [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset)를 사용하여 소스 dataset을 생성하세요. 더 많은 정보를 위해서 [텐서플로 가이드](https://www.tensorflow.org/r1/guide/datasets#reading_input_data)를 참조하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F04fVOHQIBiG", - "colab": {} - }, - "source": [ - "ds_tensors = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", - "\n", - "# CSV 파일을 생성합니다.\n", - "import tempfile\n", - "_, filename = tempfile.mkstemp()\n", - "\n", - "with open(filename, 'w') as f:\n", - " f.write(\"\"\"Line 1\n", - "Line 2\n", - "Line 3\n", - " \"\"\")\n", - "\n", - "ds_file = tf.data.TextLineDataset(filename)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbxIhC-5IPdf" - }, - "source": [ - "### 변환 적용\n", - "\n", - "[`맵(map)`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map), [`배치(batch)`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch), [`셔플(shuffle)`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle)과 같은 변환 함수를 사용하여 데이터셋의 레코드에 적용하세요. 세부사항은 [tf.data.Dataset을 위한 API 문서](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)을 참조하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uXSDZWE-ISsd", - "colab": {} - }, - "source": [ - "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", - "\n", - "ds_file = ds_file.batch(2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "A8X1GNfoIZKJ" - }, - "source": [ - "### 반복\n", - "\n", - "즉시 실행이 활성화되면 `Dataset` 객체는 반복이 가능합니다. 만약 텐서플로 그래프에서 데이터셋을 사용하는게 익숙하다면, `Dataset.make_one_shot_iterator()` 또는 `get_next()`와 같은 객체를 호출할 필요가 없는다는 것에 주목하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ws-WKRk5Ic6-", - "colab": {} - }, - "source": [ - "print('ds_tensors 요소:')\n", - "for x in ds_tensors:\n", - " print(x)\n", - "\n", - "print('\\nds_file 요소:')\n", - "for x in ds_file:\n", - " print(x)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/keras/README.md b/site/ko/r1/tutorials/keras/README.md deleted file mode 100644 index 54645f5ea14..00000000000 --- a/site/ko/r1/tutorials/keras/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# 머신러닝을 배우고 사용하기 - -이 문서들은 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 -[공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 -있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 -바랍니다. 문서 번역이나 리뷰에 참여하려면 -[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 -메일을 보내주시기 바랍니다. - -이 노트북 파일들은 -*[Deep Learning with Python](https://books.google.com/books?id=Yo3CAQAACAAJ)* 책을 -바탕으로 작성되었습니다. 이 튜토리얼은 딥러닝 모델을 만들고 훈련하기 위해 텐서플로의 고수준 파이썬 API인 `tf.keras`를 사용합니다. -텐서플로와 케라스(Keras)에 대해 더 알고 싶다면 -[텐서플로 케라스 가이드](https://github.com/tensorflow/docs/blob/master/site/en/r1/guide/keras.ipynb)를 -참고하세요. - -출판사 노트: *Deep Learning with Python*은 딥러닝을 설명하기 위해 파이썬과 케라스 라이브러리를 사용합니다. 이 책은 케라스 창시자이고 구글 AI 연구원인 프랑소와 숄레(François Chollet)가 썼습니다. 이 책에서 직관적인 설명과 실용적인 예제를 통해 딥러닝을 배울 수 있습니다. (이 책의 한글 번역서는 *[케라스 창시자에게 배우는 딥러닝](https://books.google.co.kr/books?id=EJV5DwAAQBAJ)* 입니다) - -머신러닝의 기본 원리와 개념을 배우려면 -[머신러닝 단기집중과정](https://developers.google.com/machine-learning/crash-course/)을 수강해 -보세요. - -1. [기초적인 분류 문제](./basic_classification.ipynb) -2. [텍스트 분류](./basic_text_classification.ipynb) -3. [회귀](./basic_regression.ipynb) -4. [과대적합과 과소적합](./overfit_and_underfit.ipynb) -5. [모델의 저장과 복원](./save_and_restore_models.ipynb) diff --git a/site/ko/r1/tutorials/keras/basic_classification.ipynb b/site/ko/r1/tutorials/keras/basic_classification.ipynb deleted file mode 100644 index 770e23b467f..00000000000 --- a/site/ko/r1/tutorials/keras/basic_classification.ipynb +++ /dev/null @@ -1,1008 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# 첫 번째 신경망 훈련하기: 기초적인 분류 문제" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aJjNjjy3Zbz0", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "이 튜토리얼에서는 운동화나 셔츠 같은 옷 이미지를 분류하는 신경망 모델을 훈련합니다. 상세 내용을 모두 이해하지 못해도 괜찮습니다. 여기서는 완전한 텐서플로(TensorFlow) 프로그램을 빠르게 살펴 보겠습니다. 자세한 내용은 앞으로 배우면서 더 설명합니다.\n", - "\n", - "여기에서는 텐서플로 모델을 만들고 훈련할 수 있는 고수준 API인 [tf.keras](https://www.tensorflow.org/r1/guide/keras)를 사용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "# tensorflow와 tf.keras를 임포트합니다\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# 헬퍼(helper) 라이브러리를 임포트합니다\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## 패션 MNIST 데이터셋 임포트하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "10개의 범주(category)와 70,000개의 흑백 이미지로 구성된 [패션 MNIST](https://github.com/zalandoresearch/fashion-mnist) 데이터셋을 사용하겠습니다. 이미지는 해상도(28x28 픽셀)가 낮고 다음처럼 개별 옷 품목을 나타냅니다:\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " 그림 1. 패션-MNIST 샘플 (Zalando, MIT License).
     \n", - "
    \n", - "\n", - "패션 MNIST는 컴퓨터 비전 분야의 \"Hello, World\" 프로그램격인 고전 [MNIST](http://yann.lecun.com/exdb/mnist/) 데이터셋을 대신해서 자주 사용됩니다. MNIST 데이터셋은 손글씨 숫자(0, 1, 2 등)의 이미지로 이루어져 있습니다. 여기서 사용하려는 옷 이미지와 동일한 포맷입니다.\n", - "\n", - "패션 MNIST는 일반적인 MNIST 보다 조금 더 어려운 문제이고 다양한 예제를 만들기 위해 선택했습니다. 두 데이터셋은 비교적 작기 때문에 알고리즘의 작동 여부를 확인하기 위해 사용되곤 합니다. 코드를 테스트하고 디버깅하는 용도로 좋습니다.\n", - "\n", - "네트워크를 훈련하는데 60,000개의 이미지를 사용합니다. 그다음 네트워크가 얼마나 정확하게 이미지를 분류하는지 10,000개의 이미지로 평가하겠습니다. 패션 MNIST 데이터셋은 텐서플로에서 바로 임포트하여 적재할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "load_data() 함수를 호출하면 네 개의 넘파이(NumPy) 배열이 반환됩니다:\n", - "\n", - "* `train_images`와 `train_labels` 배열은 모델 학습에 사용되는 *훈련 세트*입니다.\n", - "* `test_images`와 `test_labels` 배열은 모델 테스트에 사용되는 *테스트 세트*입니다.\n", - "\n", - "이미지는 28x28 크기의 넘파이 배열이고 픽셀 값은 0과 255 사이입니다. *레이블*(label)은 0에서 9까지의 정수 배열입니다. 이 값은 이미지에 있는 옷의 *클래스*(class)를 나타냅니다:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    레이블클래스
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "각 이미지는 하나의 레이블에 매핑되어 있습니다. 데이터셋에 *클래스 이름*이 들어있지 않기 때문에 나중에 이미지를 출력할 때 사용하기 위해 별도의 변수를 만들어 저장합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## 데이터 탐색\n", - "\n", - "모델을 훈련하기 전에 데이터셋 구조를 살펴보죠. 다음 코드는 훈련 세트에 60,000개의 이미지가 있다는 것을 보여줍니다. 각 이미지는 28x28 픽셀로 표현됩니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "비슷하게 훈련 세트에는 60,000개의 레이블이 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "각 레이블은 0과 9사이의 정수입니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "테스트 세트에는 10,000개의 이미지가 있습니다. 이 이미지도 28x28 픽셀로 표현됩니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "테스트 세트는 10,000개의 이미지에 대한 레이블을 가지고 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## 데이터 전처리\n", - "\n", - "네트워크를 훈련하기 전에 데이터를 전처리해야 합니다. 훈련 세트에 있는 첫 번째 이미지를 보면 픽셀 값의 범위가 0~255 사이라는 것을 알 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "신경망 모델에 주입하기 전에 이 값의 범위를 0~1 사이로 조정하겠습니다. 이렇게 하려면 255로 나누어야 합니다. *훈련 세트*와 *테스트 세트*를 동일한 방식으로 전처리하는 것이 중요합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "*훈련 세트*에서 처음 25개 이미지와 그 아래 클래스 이름을 출력해 보죠. 데이터 포맷이 올바른지 확인하고 네트워크 구성과 훈련할 준비를 마칩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## 모델 구성\n", - "\n", - "신경망 모델을 만들려면 모델의 층을 구성한 다음 모델을 컴파일합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### 층 설정\n", - "\n", - "신경망의 기본 구성 요소는 *층*(layer)입니다. 층은 주입된 데이터에서 표현을 추출합니다. 아마도 문제를 해결하는데 더 의미있는 표현이 추출될 것입니다.\n", - "\n", - "대부분 딥러닝은 간단한 층을 연결하여 구성됩니다. `tf.keras.layers.Dense`와 같은 층들의 가중치(parameter)는 훈련하는 동안 학습됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation=tf.nn.relu),\n", - " keras.layers.Dense(10, activation=tf.nn.softmax)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "이 네트워크의 첫 번째 층인 `tf.keras.layers.Flatten`은 2차원 배열(28 x 28 픽셀)의 이미지 포맷을 28 * 28 = 784 픽셀의 1차원 배열로 변환합니다. 이 층은 이미지에 있는 픽셀의 행을 펼쳐서 일렬로 늘립니다. 이 층에는 학습되는 가중치가 없고 데이터를 변환하기만 합니다.\n", - "\n", - "픽셀을 펼친 후에는 두 개의 `tf.keras.layers.Dense` 층이 연속되어 연결됩니다. 이 층을 밀집 연결(densely-connected) 또는 완전 연결(fully-connected) 층이라고 부릅니다. 첫 번째 `Dense` 층은 128개의 노드(또는 뉴런)를 가집니다. 두 번째 (마지막) 층은 10개의 노드의 *소프트맥스*(softmax) 층입니다. 이 층은 10개의 확률을 반환하고 반환된 값의 전체 합은 1입니다. 각 노드는 현재 이미지가 10개 클래스 중 하나에 속할 확률을 출력합니다.\n", - "\n", - "### 모델 컴파일\n", - "\n", - "모델을 훈련하기 전에 필요한 몇 가지 설정이 모델 *컴파일* 단계에서 추가됩니다:\n", - "\n", - "* *손실 함수*(Loss function)-훈련 하는 동안 모델의 오차를 측정합니다. 모델의 학습이 올바른 방향으로 향하도록 이 함수를 최소화해야 합니다.\n", - "* *옵티마이저*(Optimizer)-데이터와 손실 함수를 바탕으로 모델의 업데이트 방법을 결정합니다.\n", - "* *지표*(Metrics)-훈련 단계와 테스트 단계를 모니터링하기 위해 사용합니다. 다음 예에서는 올바르게 분류된 이미지의 비율인 *정확도*를 사용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## 모델 훈련\n", - "\n", - "신경망 모델을 훈련하는 단계는 다음과 같습니다:\n", - "\n", - "1. 훈련 데이터를 모델에 주입합니다-이 예에서는 `train_images`와 `train_labels` 배열입니다.\n", - "2. 모델이 이미지와 레이블을 매핑하는 방법을 배웁니다.\n", - "3. 테스트 세트에 대한 모델의 예측을 만듭니다-이 예에서는 `test_images` 배열입니다. 이 예측이 `test_labels` 배열의 레이블과 맞는지 확인합니다.\n", - "\n", - "훈련을 시작하기 위해 `model.fit` 메서드를 호출하면 모델이 훈련 데이터를 학습합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "모델이 훈련되면서 손실과 정확도 지표가 출력됩니다. 이 모델은 훈련 세트에서 약 0.88(88%) 정도의 정확도를 달성합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## 정확도 평가\n", - "\n", - "그다음 테스트 세트에서 모델의 성능을 비교합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('테스트 정확도:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "테스트 세트의 정확도가 훈련 세트의 정확도보다 조금 낮습니다. 훈련 세트의 정확도와 테스트 세트의 정확도 사이의 차이는 *과대적합*(overfitting) 때문입니다. 과대적합은 머신러닝 모델이 훈련 데이터보다 새로운 데이터에서 성능이 낮아지는 현상을 말합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## 예측 만들기\n", - "\n", - "훈련된 모델을 사용하여 이미지에 대한 예측을 만들 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "여기서는 테스트 세트에 있는 각 이미지의 레이블을 예측했습니다. 첫 번째 예측을 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "이 예측은 10개의 숫자 배열로 나타납니다. 이 값은 10개의 옷 품목에 상응하는 모델의 신뢰도(confidence)를 나타냅니다. 가장 높은 신뢰도를 가진 레이블을 찾아보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "모델은 이 이미지가 앵클 부츠(`class_name[9]`)라고 가장 확신하고 있습니다. 이 값이 맞는지 테스트 레이블을 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "10개의 신뢰도를 모두 그래프로 표현해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DvYmmrpIy6Y1", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d4Ov9OFDMmOD" - }, - "source": [ - "0번째 원소의 이미지, 예측, 신뢰도 점수 배열을 확인해 보겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HV5jw-5HwSmO", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ko-uzOufSCSe", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "몇 개의 이미지의 예측을 출력해 보죠. 올바르게 예측된 레이블은 파란색이고 잘못 예측된 레이블은 빨강색입니다. 숫자는 예측 레이블의 신뢰도 퍼센트(100점 만점)입니다. 신뢰도 점수가 높을 때도 잘못 예측할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hQlnbqaw2Qu_", - "colab": {} - }, - "source": [ - "# 처음 X 개의 테스트 이미지와 예측 레이블, 진짜 레이블을 출력합니다\n", - "# 올바른 예측은 파랑색으로 잘못된 예측은 빨강색으로 나타냅니다\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "마지막으로 훈련된 모델을 사용하여 한 이미지에 대한 예측을 만듭니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yRJ7JU7JCaXT", - "colab": {} - }, - "source": [ - "# 테스트 세트에서 이미지 하나를 선택합니다\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "`tf.keras` 모델은 한 번에 샘플의 묶음 또는 *배치*(batch)로 예측을 만드는데 최적화되어 있습니다. 하나의 이미지를 사용할 때에도 2차원 배열로 만들어야 합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# 이미지 하나만 사용할 때도 배치에 추가합니다\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "이제 이 이미지의 예측을 만듭니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ai-cpLjO-3A", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "plt.xticks(range(10), class_names, rotation=45)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "`model.predict`는 2차원 넘파이 배열을 반환하므로 첫 번째 이미지의 예측을 선택합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "prediction_result = np.argmax(predictions_single[0])\n", - "print(prediction_result)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "이전과 마찬가지로 모델의 예측은 레이블 9입니다." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/keras/basic_regression.ipynb b/site/ko/r1/tutorials/keras/basic_regression.ipynb deleted file mode 100644 index f3a2494143b..00000000000 --- a/site/ko/r1/tutorials/keras/basic_regression.ipynb +++ /dev/null @@ -1,867 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# 회귀: 자동차 연비 예측하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YYwLLNVaaJU9", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "*회귀*(regression)는 가격이나 확률 같이 연속된 출력 값을 예측하는 것이 목적입니다. 이와는 달리 *분류*(classification)는 여러개의 클래스 중 하나의 클래스를 선택하는 것이 목적입니다(예를 들어, 사진에 사과 또는 오렌지가 포함되어 있을 때 어떤 과일인지 인식하는 것).\n", - "\n", - "이 노트북은 [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) 데이터셋을 사용하여 1970년대 후반과 1980년대 초반의 자동차 연비를 예측하는 모델을 만듭니다. 이 기간에 출시된 자동차 정보를 모델에 제공하겠습니다. 이 정보에는 실린더 수, 배기량, 마력(horsepower), 공차 중량 같은 속성이 포함됩니다.\n", - "\n", - "이 예제는 `tf.keras` API를 사용합니다. 자세한 내용은 [케라스 가이드](https://www.tensorflow.org/r1/guide/keras)를 참고하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# 산점도 행렬을 그리기 위해 seaborn 패키지를 설치합니다\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Auto MPG 데이터셋\n", - "\n", - "이 데이터셋은 [UCI 머신 러닝 저장소](https://archive.ics.uci.edu/ml/)에서 다운로드할 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### 데이터 구하기\n", - "먼저 데이터셋을 다운로드합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "판다스를 사용하여 데이터를 읽습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### 데이터 정제하기\n", - "\n", - "이 데이터셋은 일부 데이터가 누락되어 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "문제를 간단하게 만들기 위해서 누락된 행을 삭제하겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "`\"Origin\"` 열은 수치형이 아니고 범주형이므로 원-핫 인코딩(one-hot encoding)으로 변환하겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### 데이터셋을 훈련 세트와 테스트 세트로 분할하기\n", - "\n", - "이제 데이터를 훈련 세트와 테스트 세트로 분할합니다.\n", - "\n", - "테스트 세트는 모델을 최종적으로 평가할 때 사용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### 데이터 조사하기\n", - "\n", - "훈련 세트에서 몇 개의 열을 선택해 산점도 행렬을 만들어 살펴 보겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "전반적인 통계도 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### 특성과 레이블 분리하기\n", - "\n", - "특성에서 타깃 값 또는 \"레이블\"을 분리합니다. 이 레이블을 예측하기 위해 모델을 훈련시킬 것입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### 데이터 정규화\n", - "\n", - "위 `train_stats` 통계를 다시 살펴보고 각 특성의 범위가 얼마나 다른지 확인해 보죠." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "특성의 스케일과 범위가 다르면 정규화(normalization)하는 것이 권장됩니다. 특성을 정규화하지 않아도 모델이 *수렴할 수 있지만*, 훈련시키기 어렵고 입력 단위에 의존적인 모델이 만들어집니다.\n", - "\n", - "노트: 의도적으로 훈련 세트만 사용하여 통계치를 생성했습니다. 이 통계는 테스트 세트를 정규화할 때에도 사용됩니다. 이는 테스트 세트를 모델이 훈련에 사용했던 것과 동일한 분포로 투영하기 위해서입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "정규화된 데이터를 사용하여 모델을 훈련합니다.\n", - "\n", - "주의: 여기에서 입력 데이터를 정규화하기 위해 사용한 통계치(평균과 표준편차)는 원-핫 인코딩과 마찬가지로 모델에 주입되는 모든 데이터에 적용되어야 합니다. 여기에는 테스트 세트는 물론 모델이 실전에 투입되어 얻은 라이브 데이터도 포함됩니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## 모델" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### 모델 만들기\n", - "\n", - "모델을 구성해 보죠. 여기에서는 두 개의 완전 연결(densely connected) 은닉층으로 `Sequential` 모델을 만들겠습니다. 출력 층은 하나의 연속적인 값을 반환합니다. 나중에 두 번째 모델을 만들기 쉽도록 `build_model` 함수로 모델 구성 단계를 감싸겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation=tf.nn.relu, input_shape=[9]),\n", - " layers.Dense(64, activation=tf.nn.relu),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mean_squared_error',\n", - " optimizer=optimizer,\n", - " metrics=['mean_absolute_error', 'mean_squared_error'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### 모델 확인\n", - "\n", - "`.summary` 메서드를 사용해 모델에 대한 간단한 정보를 출력합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "모델을 한번 실행해 보죠. 훈련 세트에서 `10` 샘플을 하나의 배치로 만들어 `model.predict` 메서드를 호출해 보겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "제대로 작동하는 것 같네요. 결괏값의 크기와 타입이 기대했던 대로입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### 모델 훈련\n", - "\n", - "이 모델을 1,000번의 에포크(epoch) 동안 훈련합니다. 훈련 정확도와 검증 정확도는 `history` 객체에 기록됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# 에포크가 끝날 때마다 점(.)을 출력해 훈련 진행 과정을 표시합니다\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "`history` 객체에 저장된 통계치를 사용해 모델의 훈련 과정을 시각화해 보죠." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure(figsize=(8,12))\n", - "\n", - " plt.subplot(2,1,1)\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mean_absolute_error'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mean_absolute_error'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - "\n", - " plt.subplot(2,1,2)\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mean_squared_error'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mean_squared_error'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "이 그래프를 보면 수 백번 에포크를 진행한 이후에는 모델이 거의 향상되지 않는 것 같습니다. `model.fit` 메서드를 수정하여 검증 점수가 향상되지 않으면 자동으로 훈련을 멈추도록 만들어 보죠. 에포크마다 훈련 상태를 점검하기 위해 *EarlyStopping 콜백(callback)*을 사용하겠습니다. 지정된 에포크 횟수 동안 성능 향상이 없으면 자동으로 훈련이 멈춥니다.\n", - "\n", - "이 콜백에 대해 더 자세한 내용은 [여기](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping)를 참고하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# patience 매개변수는 성능 향상을 체크할 에포크 횟수입니다\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "이 그래프를 보면 검증 세트의 평균 오차가 약 +/- 2 MPG입니다. 좋은 결과인가요? 이에 대한 평가는 여러분에게 맡기겠습니다.\n", - "\n", - "모델을 훈련할 때 사용하지 않았던 **테스트 세트**에서 모델의 성능을 확인해 보죠. 이를 통해 모델이 실전에 투입되었을 때 모델의 성능을 짐작할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"테스트 세트의 평균 절대 오차: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "## 예측\n", - "\n", - "마지막으로 테스트 세트에 있는 샘플을 사용해 MPG 값을 예측해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mU1jBsRLaCeY", - "colab_type": "text" - }, - "source": [ - "모델이 꽤 잘 예측한 것 같습니다. 오차의 분포를 살펴 보죠." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3PkzkjFkaCed", - "colab_type": "text" - }, - "source": [ - "가우시안 분포가 아니지만 아마도 훈련 샘플의 수가 매우 작기 때문일 것입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## 결론\n", - "\n", - "이 노트북은 회귀 문제를 위한 기법을 소개합니다.\n", - "\n", - "* 평균 제곱 오차(MSE)는 회귀 문제에서 자주 사용하는 손실 함수입니다(분류 문제에서 사용하는 손실 함수와 다릅니다).\n", - "* 비슷하게 회귀에서 사용되는 평가 지표도 분류와 다릅니다. 많이 사용하는 회귀 지표는 평균 절댓값 오차(MAE)입니다.\n", - "* 수치 입력 데이터의 특성이 여러 가지 범위를 가질 때 동일한 범위가 되도록 각 특성의 스케일을 독립적으로 조정해야 합니다.\n", - "* 훈련 데이터가 많지 않다면 과대적합을 피하기 위해 은닉층의 개수가 적은 소규모 네트워크를 선택하는 방법이 좋습니다.\n", - "* 조기 종료(Early stopping)은 과대적합을 방지하기 위한 좋은 방법입니다." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/keras/basic_text_classification.ipynb b/site/ko/r1/tutorials/keras/basic_text_classification.ipynb deleted file mode 100644 index c5ee778f842..00000000000 --- a/site/ko/r1/tutorials/keras/basic_text_classification.ipynb +++ /dev/null @@ -1,738 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# 영화 리뷰를 사용한 텍스트 분류" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CMrWLbtWaZWE" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "이 노트북은 영화 리뷰(review) 텍스트를 *긍정*(positive) 또는 *부정*(negative)으로 분류합니다. 이 예제는 *이진*(binary)-또는 클래스(class)가 두 개인- 분류 문제입니다. 이진 분류는 머신러닝에서 중요하고 널리 사용됩니다.\n", - "\n", - "여기에서는 [인터넷 영화 데이터베이스](https://www.imdb.com/)(Internet Movie Database)에서 수집한 50,000개의 영화 리뷰 텍스트를 담은 [IMDB 데이터셋](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb)을 사용하겠습니다. 25,000개 리뷰는 훈련용으로, 25,000개는 테스트용으로 나뉘어져 있습니다. 훈련 세트와 테스트 세트의 클래스는 *균형*이 잡혀 있습니다. 즉 긍정적인 리뷰와 부정적인 리뷰의 개수가 동일합니다.\n", - "\n", - "이 노트북은 모델을 만들고 훈련하기 위해 텐서플로의 고수준 파이썬 API인 [tf.keras](https://www.tensorflow.org/r1/guide/keras)를 사용합니다. `tf.keras`를 사용한 고급 텍스트 분류 튜토리얼은 [MLCC 텍스트 분류 가이드](https://developers.google.com/machine-learning/guides/text-classification/)를 참고하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4h7fwqqEChm", - "colab": {} - }, - "source": [ - "# keras.datasets.imdb is broken in 1.13 and 1.14, by np 1.16.3\n", - "!pip install tf_nightly" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## IMDB 데이터셋 다운로드\n", - "\n", - "IMDB 데이터셋은 텐서플로와 함께 제공됩니다. 리뷰(단어의 시퀀스(sequence))는 미리 전처리해서 정수 시퀀스로 변환되어 있습니다. 각 정수는 어휘 사전에 있는 특정 단어를 의미합니다.\n", - "\n", - "다음 코드는 IMDB 데이터셋을 컴퓨터에 다운로드합니다(또는 이전에 다운로드 받았다면 캐시된 복사본을 사용합니다):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "매개변수 `num_words=10000`은 훈련 데이터에서 가장 많이 등장하는 상위 10,000개의 단어를 선택합니다. 데이터 크기를 적당하게 유지하기 위해 드물에 등장하는 단어는 제외하겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## 데이터 탐색\n", - "\n", - "잠시 데이터 형태를 알아 보죠. 이 데이터셋의 샘플은 전처리된 정수 배열입니다. 이 정수는 영화 리뷰에 나오는 단어를 나타냅니다. 레이블(label)은 정수 0 또는 1입니다. 0은 부정적인 리뷰이고 1은 긍정적인 리뷰입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} - }, - "source": [ - "print(\"훈련 샘플: {}, 레이블: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "리뷰 텍스트는 어휘 사전의 특정 단어를 나타내는 정수로 변환되어 있습니다. 첫 번째 리뷰를 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "영화 리뷰들은 길이가 다릅니다. 다음 코드는 첫 번째 리뷰와 두 번째 리뷰에서 단어의 개수를 출력합니다. 신경망의 입력은 길이가 같아야 하기 때문에 나중에 이 문제를 해결하겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### 정수를 단어로 다시 변환하기\n", - "\n", - "정수를 다시 텍스트로 변환하는 방법이 있다면 유용할 것입니다. 여기에서는 정수와 문자열을 매핑한 딕셔너리(dictionary) 객체에 질의하는\u001c 헬퍼(helper) 함수를 만들겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} - }, - "source": [ - "# 단어와 정수 인덱스를 매핑한 딕셔너리\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# 처음 몇 개 인덱스는 사전에 정의되어 있습니다\n", - "word_index = {k:(v+3) for k,v in word_index.items()}\n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # unknown\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "이제 `decode_review` 함수를 사용해 첫 번째 리뷰 텍스트를 출력할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} - }, - "source": [ - "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## 데이터 준비\n", - "\n", - "리뷰-정수 배열-는 신경망에 주입하기 전에 텐서로 변환되어야 합니다. 변환하는 방법에는 몇 가지가 있습니다:\n", - "\n", - "* 원-핫 인코딩(one-hot encoding)은 정수 배열을 0과 1로 이루어진 벡터로 변환합니다. 예를 들어 배열 [3, 5]을 인덱스 3과 5만 1이고 나머지는 모두 0인 10,000차원 벡터로 변환할 수 있습니다. 그다음 실수 벡터 데이터를 다룰 수 있는 층-Dense 층-을 신경망의 첫 번째 층으로 사용합니다. 이 방법은 `num_words * num_reviews` 크기의 행렬이 필요하기 때문에 메모리를 많이 사용합니다.\n", - "* 다른 방법으로는, 정수 배열의 길이가 모두 같도록 패딩(padding)을 추가해 `max_length * num_reviews` 크기의 정수 텐서를 만듭니다. 이런 형태의 텐서를 다룰 수 있는 임베딩(embedding) 층을 신경망의 첫 번째 층으로 사용할 수 있습니다.\n", - "\n", - "이 튜토리얼에서는 두 번째 방식을 사용하겠습니다.\n", - "\n", - "영화 리뷰의 길이가 같아야 하므로 [pad_sequences](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences) 함수를 사용해 길이를 맞추겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} - }, - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "샘플의 길이를 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "(패딩된) 첫 번째 리뷰 내용을 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## 모델 구성\n", - "\n", - "신경망은 층(layer)을 쌓아서 만듭니다. 이 구조에서는 두 가지를 결정해야 합니다:\n", - "\n", - "* 모델에서 얼마나 많은 층을 사용할 것인가?\n", - "* 각 층에서 얼마나 많은 *은닉 유닛*(hidden unit)을 사용할 것인가?\n", - "\n", - "이 예제의 입력 데이터는 단어 인덱스의 배열입니다. 예측할 레이블은 0 또는 1입니다. 이 문제에 맞는 모델을 구성해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "# 입력 크기는 영화 리뷰 데이터셋에 적용된 어휘 사전의 크기입니다(10,000개의 단어)\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16, input_shape=(None,)))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation=tf.nn.relu))\n", - "model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "층을 순서대로 쌓아 분류기(classifier)를 만듭니다:\n", - "\n", - "1. 첫 번째 층은 `Embedding` 층입니다. 이 층은 정수로 인코딩된 단어를 입력 받고 각 단어 인덱스에 해당하는 임베딩 벡터를 찾습니다. 이 벡터는 모델이 훈련되면서 학습됩니다. 이 벡터는 출력 배열에 새로운 차원으로 추가됩니다. 최종 차원은 `(batch, sequence, embedding)`이 됩니다.\n", - "2. 그다음 `GlobalAveragePooling1D` 층은 `sequence` 차원에 대해 평균을 계산하여 각 샘플에 대해 고정된 길이의 출력 벡터를 반환합니다. 이는 길이가 다른 입력을 다루는 가장 간단한 방법입니다.\n", - "3. 이 고정 길이의 출력 벡터는 16개의 은닉 유닛을 가진 완전 연결(fully-connected) 층(`Dense`)을 거칩니다.\n", - "4. 마지막 층은 하나의 출력 노드(node)를 가진 완전 연결 층입니다. `sigmoid` 활성화 함수를 사용하여 0과 1 사이의 실수를 출력합니다. 이 값은 확률 또는 신뢰도를 나타냅니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### 은닉 유닛\n", - "\n", - "위 모델에는 입력과 출력 사이에 두 개의 중간 또는 \"은닉\" 층이 있습니다. 출력(유닛 또는 노드, 뉴런)의 개수는 층이 가진 표현 공간(representational space)의 차원이 됩니다. 다른 말로 하면, 내부 표현을 학습할 때 허용되는 네트워크 자유도의 양입니다.\n", - "\n", - "모델에 많은 은닉 유닛(고차원의 표현 공간)과 층이 있다면 네트워크는 더 복잡한 표현을 학습할 수 있습니다. 하지만 네트워크의 계산 비용이 많이 들고 원치않는 패턴을 학습할 수도 있습니다. 이런 표현은 훈련 데이터의 성능을 향상시키지만 테스트 데이터에서는 그렇지 못합니다. 이를 *과대적합*(overfitting)이라고 부릅니다. 나중에 이에 대해 알아 보겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### 손실 함수와 옵티마이저\n", - "\n", - "모델이 훈련하려면 손실 함수(loss function)과 옵티마이저(optimizer)가 필요합니다. 이 예제는 이진 분류 문제이고 모델이 확률을 출력하므로(출력층의 유닛이 하나이고 `sigmoid` 활성화 함수를 사용합니다), `binary_crossentropy` 손실 함수를 사용하겠습니다.\n", - "\n", - "다른 손실 함수를 선택할 수 없는 것은 아닙니다. 예를 들어 `mean_squared_error`를 선택할 수 있습니다. 하지만 일반적으로 `binary_crossentropy`가 확률을 다루는데 적합합니다. 이 함수는 확률 분포 간의 거리를 측정합니다. 여기에서는 정답인 타깃 분포와 예측 분포 사이의 거리입니다.\n", - "\n", - "나중에 회귀(regression) 문제(예를 들어 주택 가격을 예측하는 문제)에 대해 살펴 볼 때 평균 제곱 오차(mean squared error) 손실 함수를 어떻게 사용하는지 알아 보겠습니다.\n", - "\n", - "이제 모델이 사용할 옵티마이저와 손실 함수를 설정해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer=tf.train.AdamOptimizer(),\n", - " loss='binary_crossentropy',\n", - " metrics=['acc'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "## 검증 세트 만들기\n", - "\n", - "모델을 훈련할 때 모델이 만난 적 없는 데이터에서 정확도를 확인하는 것이 좋습니다. 원본 훈련 데이터에서 10,000개의 샘플을 떼어내어 *검증 세트*(validation set)를 만들겠습니다. (왜 테스트 세트를 사용하지 않을까요? 훈련 데이터만을 사용하여 모델을 개발하고 튜닝하는 것이 목표입니다. 그다음 테스트 세트를 사용해서 딱 한 번만 정확도를 평가합니다)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} - }, - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## 모델 훈련\n", - "\n", - "이 모델을 512개의 샘플로 이루어진 미니배치(mini-batch)에서 40번의 에포크(epoch) 동안 훈련합니다. `x_train`과 `y_train` 텐서에 있는 모든 샘플에 대해 40번 반복한다는 뜻입니다. 훈련하는 동안 10,000개의 검증 세트에서 모델의 손실과 정확도를 모니터링합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} - }, - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## 모델 평가\n", - "\n", - "모델의 성능을 확인해 보죠. 두 개의 값이 반환됩니다. 손실(오차를 나타내는 숫자이므로 낮을수록 좋습니다)과 정확도입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "이 예제는 매우 단순한 방식을 사용하므로 87% 정도의 정확도를 달성했습니다. 고급 방법을 사용한 모델은 95%에 가까운 정확도를 얻습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## 정확도와 손실 그래프 그리기\n", - "\n", - "`model.fit()`은 `History` 객체를 반환합니다. 여기에는 훈련하는 동안 일어난 모든 정보가 담긴 딕셔너리(dictionary)가 들어 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} - }, - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "네 개의 항목이 있습니다. 훈련과 검증 단계에서 모니터링하는 지표들입니다. 훈련 손실과 검증 손실을 그래프로 그려 보고, 훈련 정확도와 검증 정확도도 그래프로 그려서 비교해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history_dict['acc']\n", - "val_acc = history_dict['val_acc']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# \"bo\"는 \"파란색 점\"입니다\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "# b는 \"파란 실선\"입니다\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} - }, - "source": [ - "plt.clf() # 그림을 초기화합니다\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "이 그래프에서 점선은 훈련 손실과 훈련 정확도를 나타냅니다. 실선은 검증 손실과 검증 정확도입니다.\n", - "\n", - "훈련 손실은 에포크마다 *감소*하고 훈련 정확도는 *증가*한다는 것을 주목하세요. 경사 하강법 최적화를 사용할 때 볼 수 있는 현상입니다. 매 반복마다 최적화 대상의 값을 최소화합니다.\n", - "\n", - "하지만 검증 손실과 검증 정확도에서는 그렇지 못합니다. 약 20번째 에포크 이후가 최적점인 것 같습니다. 이는 과대적합 때문입니다. 이전에 본 적 없는 데이터보다 훈련 데이터에서 더 잘 동작합니다. 이 지점부터는 모델이 과도하게 최적화되어 테스트 데이터에서 *일반화*되기 어려운 훈련 데이터의 특정 표현을 학습합니다.\n", - "\n", - "여기에서는 과대적합을 막기 위해 단순히 20번째 에포크 근처에서 훈련을 멈출 수 있습니다. 나중에 콜백(callback)을 사용하여 자동으로 이렇게 하는 방법을 배워 보겠습니다." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/keras/overfit_and_underfit.ipynb b/site/ko/r1/tutorials/keras/overfit_and_underfit.ipynb deleted file mode 100644 index 7c33cb1e45a..00000000000 --- a/site/ko/r1/tutorials/keras/overfit_and_underfit.ipynb +++ /dev/null @@ -1,692 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "overfit_and_underfit.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "fTFj8ft5dlbS" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fTFj8ft5dlbS" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "lzyBOpYMdp3F", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m_x4KfSJ7Vt7", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C9HmC2T4ld5B" - }, - "source": [ - "# 과대적합과 과소적합" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRTxFhXAlnl1" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1vJpRm6na4Iw", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19rPukKZsPG6" - }, - "source": [ - "지금까지 그랬듯이 이 예제의 코드도 `tf.keras` API를 사용합니다. 텐서플로 [케라스 가이드](https://www.tensorflow.org/r1/guide/keras)에서 `tf.keras` API에 대해 더 많은 정보를 얻을 수 있습니다.\n", - "\n", - "앞서 영화 리뷰 분류와 주택 가격 예측의 두 예제에서 일정 에포크 동안 훈련하면 검증 세트에서 모델 성능이 최고점에 도달한 다음 감소하기 시작한 것을 보았습니다.\n", - "\n", - "다른 말로 하면, 모델이 훈련 세트에 *과대적합*(overfitting)된 것입니다. 과대적합을 다루는 방법은 꼭 배워야 합니다. *훈련 세트*에서 높은 성능을 얻을 수 있지만 진짜 원하는 것은 *테스트 세트*(또는 이전에 본 적 없는 데이터)에 잘 일반화되는 모델입니다.\n", - "\n", - "과대적합의 반대는 *과소적합*(underfitting)입니다. 과소적합은 테스트 세트의 성능이 향상될 여지가 아직 있을 때 일어납니다. 발생하는 원인은 여러가지입니다. 모델이 너무 단순하거나, 규제가 너무 많거나, 그냥 단순히 충분히 오래 훈련하지 않는 경우입니다. 즉 네트워크가 훈련 세트에서 적절한 패턴을 학습하지 못했다는 뜻입니다.\n", - "\n", - "모델을 너무 오래 훈련하면 과대적합되기 시작하고 테스트 세트에서 일반화되지 못하는 패턴을 훈련 세트에서 학습합니다. 과대적합과 과소적합 사이에서 균형을 잡아야 합니다. 이를 위해 적절한 에포크 횟수동안 모델을 훈련하는 방법을 배워보겠습니다.\n", - "\n", - "과대적합을 막는 가장 좋은 방법은 더 많은 훈련 데이터를 사용하는 것입니다. 많은 데이터에서 훈련한 모델은 자연적으로 일반화 성능이 더 좋습니다. 데이터를 더 준비할 수 없을 때 그다음으로 가장 좋은 방법은 규제(regularization)와 같은 기법을 사용하는 것입니다. 모델이 저장할 수 있는 정보의 양과 종류에 제약을 부과하는 방법입니다. 네트워크가 소수의 패턴만 기억할 수 있다면 최적화 과정 동안 일반화 가능성이 높은 가장 중요한 패턴에 촛점을 맞출 것입니다.\n", - "\n", - "이 노트북에서 널리 사용되는 두 가지 규제 기법인 가중치 규제와 드롭아웃(dropout)을 알아 보겠습니다. 이런 기법을 사용하여 IMDB 영화 리뷰 분류 모델의 성능을 향상시켜 보죠." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5pZ8A2liqvgk", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1cweoTiruj8O" - }, - "source": [ - "## IMDB 데이터셋 다운로드\n", - "\n", - "이전 노트북에서처럼 임베딩을 사용하지 않고 여기에서는 문장을 멀티-핫 인코딩(multi-hot encoding)으로 변환하겠습니다. 이 모델은 훈련 세트에 빠르게 과대적합될 것입니다. 과대적합을 발생시키기고 어떻게 해결하는지 보이기 위해 선택했습니다.\n", - "\n", - "멀티-핫 인코딩은 정수 시퀀스를 0과 1로 이루어진 벡터로 변환합니다. 정확하게 말하면 시퀀스 `[3, 5]`를 인덱스 3과 5만 1이고 나머지는 모두 0인 10,000 차원 벡터로 변환한다는 의미입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QpzE4iqZtJly", - "colab": {} - }, - "source": [ - "NUM_WORDS = 10000\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = keras.datasets.imdb.load_data(num_words=NUM_WORDS)\n", - "\n", - "def multi_hot_sequences(sequences, dimension):\n", - " # 0으로 채워진 (len(sequences), dimension) 크기의 행렬을 만듭니다\n", - " results = np.zeros((len(sequences), dimension))\n", - " for i, word_indices in enumerate(sequences):\n", - " results[i, word_indices] = 1.0 # results[i]의 특정 인덱스만 1로 설정합니다\n", - " return results\n", - "\n", - "\n", - "train_data = multi_hot_sequences(train_data, dimension=NUM_WORDS)\n", - "test_data = multi_hot_sequences(test_data, dimension=NUM_WORDS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MzWVeXe3NBTn" - }, - "source": [ - "만들어진 멀티-핫 벡터 중 하나를 살펴 보죠. 단어 인덱스는 빈도 순으로 정렬되어 있습니다. 그래프에서 볼 수 있듯이 인덱스 0에 가까울수록 1이 많이 등장합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "71kr5rG4LkGM", - "colab": {} - }, - "source": [ - "plt.plot(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lglk41MwvU5o" - }, - "source": [ - "## 과대적합 예제\n", - "\n", - "과대적합을 막는 가장 간단한 방법은 모델의 규모를 축소하는 것입니다. 즉, 모델에 있는 학습 가능한 파라미터의 수를 줄입니다(모델 파라미터는 층(layer)의 개수와 층의 유닛(unit) 개수에 의해 결정됩니다). 딥러닝에서는 모델의 학습 가능한 파라미터의 수를 종종 모델의 \"용량\"이라고 말합니다. 직관적으로 생각해 보면 많은 파라미터를 가진 모델이 더 많은 \"기억 용량\"을 가집니다. 이런 모델은 훈련 샘플과 타깃 사이를 일반화 능력이 없는 딕셔너리와 같은 매핑으로 완벽하게 학습할 수 있습니다. 하지만 이전에 본 적 없는 데이터에서 예측을 할 땐 쓸모가 없을 것입니다.\n", - "\n", - "항상 기억해야 할 점은 딥러닝 모델이 훈련 세트에는 학습이 잘 되는 경향이 있지만 진짜 해결할 문제는 학습이 아니라 일반화라는 것입니다.\n", - "\n", - "반면에 네트워크의 기억 용량이 부족하다면 이런 매핑을 쉽게 학습할 수 없을 것입니다. 손실을 최소화하기 위해서는 예측 성능이 더 많은 압축된 표현을 학습해야 합니다. 또한 너무 작은 모델을 만들면 훈련 데이터를 학습하기 어렵울 것입니다. \"너무 많은 용량\"과 \"충분하지 않은 용량\" 사이의 균형을 잡아야 합니다.\n", - "\n", - "안타깝지만 어떤 모델의 (층의 개수나 뉴런 개수에 해당하는) 적절한 크기나 구조를 결정하는 마법같은 공식은 없습니다. 여러 가지 다른 구조를 사용해 실험을 해봐야만 합니다.\n", - "\n", - "알맞은 모델의 크기를 찾으려면 비교적 적은 수의 층과 파라미터로 시작해서 검증 손실이 감소할 때까지 새로운 층을 추가하거나 층의 크기를 늘리는 것이 좋습니다. 영화 리뷰 분류 네트워크를 사용해 이를 실험해 보죠.\n", - "\n", - "```Dense``` 층만 사용하는 간단한 기준 모델을 만들고 작은 규모의 버전와 큰 버전의 모델을 만들어 비교하겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ReKHdC2EgVu" - }, - "source": [ - "### 기준 모델 만들기" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QKgdXPx9usBa", - "colab": {} - }, - "source": [ - "baseline_model = keras.Sequential([\n", - " # `.summary` 메서드 때문에 `input_shape`가 필요합니다\n", - " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "baseline_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "baseline_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LqG3MXF5xSjR", - "colab": {} - }, - "source": [ - "baseline_history = baseline_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L-DGRBbGxI6G" - }, - "source": [ - "### 작은 모델 만들기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SrfoVQheYSO5" - }, - "source": [ - "앞서 만든 기준 모델과 비교하기 위해 적은 수의 은닉 유닛을 가진 모델을 만들어 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jksi-XtaxDAh", - "colab": {} - }, - "source": [ - "smaller_model = keras.Sequential([\n", - " keras.layers.Dense(4, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(4, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "smaller_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "smaller_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jbngCZliYdma" - }, - "source": [ - "같은 데이터를 사용해 이 모델을 훈련합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ofn1AwDhx-Fe", - "colab": {} - }, - "source": [ - "smaller_history = smaller_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vIPuf23FFaVn" - }, - "source": [ - "### 큰 모델 만들기\n", - "\n", - "아주 큰 모델을 만들어 얼마나 빠르게 과대적합이 시작되는지 알아 볼 수 있습니다. 이 문제에 필요한 것보다 훨씬 더 큰 용량을 가진 네트워크를 추가해서 비교해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ghQwwqwqvQM9", - "colab": {} - }, - "source": [ - "bigger_model = keras.models.Sequential([\n", - " keras.layers.Dense(512, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(512, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "bigger_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "bigger_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D-d-i5DaYmr7" - }, - "source": [ - "역시 같은 데이터를 사용해 모델을 훈련합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "U1A99dhqvepf", - "colab": {} - }, - "source": [ - "bigger_history = bigger_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fy3CMUZpzH3d" - }, - "source": [ - "### 훈련 손실과 검증 손실 그래프 그리기\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HSlo1F4xHuuM" - }, - "source": [ - "실선은 훈련 손실이고 점선은 검증 손실입니다(낮은 검증 손실이 더 좋은 모델입니다). 여기서는 작은 네트워크가 기준 모델보다 더 늦게 과대적합이 시작되었습니다(즉 에포크 4가 아니라 6에서 시작됩니다). 또한 과대적합이 시작되고 훨씬 천천히 성능이 감소합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0XmKDtOWzOpk", - "colab": {} - }, - "source": [ - "def plot_history(histories, key='binary_crossentropy'):\n", - " plt.figure(figsize=(16,10))\n", - "\n", - " for name, history in histories:\n", - " val = plt.plot(history.epoch, history.history['val_'+key],\n", - " '--', label=name.title()+' Val')\n", - " plt.plot(history.epoch, history.history[key], color=val[0].get_color(),\n", - " label=name.title()+' Train')\n", - "\n", - " plt.xlabel('Epochs')\n", - " plt.ylabel(key.replace('_',' ').title())\n", - " plt.legend()\n", - "\n", - " plt.xlim([0,max(history.epoch)])\n", - "\n", - "\n", - "plot_history([('baseline', baseline_history),\n", - " ('smaller', smaller_history),\n", - " ('bigger', bigger_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bi6hBhdnSfjA" - }, - "source": [ - "큰 네트워크는 거의 바로 첫 번째 에포크 이후에 과대적합이 시작되고 훨씬 더 심각하게 과대적합됩니다. 네트워크의 용량이 많을수록 훈련 세트를 더 빠르게 모델링할 수 있습니다(훈련 손실이 낮아집니다). 하지만 더 쉽게 과대적합됩니다(훈련 손실과 검증 손실 사이에 큰 차이가 발생합니다)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ASdv7nsgEFhx" - }, - "source": [ - "## 전략" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4rHoVWcswFLa" - }, - "source": [ - "### 가중치를 규제하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRxWepNawbBK" - }, - "source": [ - "아마도 오캄의 면도날(Occam's Razor) 이론을 들어 보았을 것입니다. 어떤 것을 설명하는 두 가지 방법이 있다면 더 정확한 설명은 최소한의 가정이 필요한 가장 \"간단한\" 설명일 것입니다. 이는 신경망으로 학습되는 모델에도 적용됩니다. 훈련 데이터와 네트워크 구조가 주어졌을 때 이 데이터를 설명할 수 있는 가중치의 조합(즉, 가능한 모델)은 많습니다. 간단한 모델은 복잡한 것보다 과대적합되는 경향이 작을 것입니다.\n", - "\n", - "여기서 \"간단한 모델\"은 모델 파라미터의 분포를 봤을 때 엔트로피(entropy)가 작은 모델입니다(또는 앞 절에서 보았듯이 적은 파라미터를 가진 모델입니다). 따라서 과대적합을 완화시키는 일반적인 방법은 가중치가 작은 값을 가지도록 네트워크의 복잡도에 제약을 가하는 것입니다. 이는 가중치 값의 분포를 좀 더 균일하게 만들어 줍니다. 이를 \"가중치 규제\"(weight regularization)라고 부릅니다. 네트워크의 손실 함수에 큰 가중치에 해당하는 비용을 추가합니다. 이 비용은 두 가지 형태가 있습니다:\n", - "\n", - "* [L1 규제](https://developers.google.com/machine-learning/glossary/#L1_regularization)는 가중치의 절댓값에 비례하는 비용이 추가됩니다(즉, 가중치의 \"L1 노름(norm)\"을 추가합니다).\n", - "\n", - "* [L2 규제](https://developers.google.com/machine-learning/glossary/#L2_regularization)는 가중치의 제곱에 비례하는 비용이 추가됩니다(즉, 가중치의 \"L2 노름\"의 제곱을 추가합니다). 신경망에서는 L2 규제를 가중치 감쇠(weight decay)라고도 부릅니다. 이름이 다르지만 혼돈하지 마세요. 가중치 감쇠는 수학적으로 L2 규제와 동일합니다.\n", - "\n", - "`tf.keras`에서는 가중치 규제 객체를 층의 키워드 매개변수에 전달하여 가중치에 규제를 추가합니다. L2 가중치 규제를 추가해 보죠." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HFGmcwduwVyQ", - "colab": {} - }, - "source": [ - "l2_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "l2_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "l2_model_history = l2_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bUUHoXb7w-_C" - }, - "source": [ - "```l2(0.001)```는 네트워크의 전체 손실에 층에 있는 가중치 행렬의 모든 값이 ```0.001 * weight_coefficient_value**2```만큼 더해진다는 의미입니다. 이런 페널티(penalty)는 훈련할 때만 추가됩니다. 따라서 테스트 단계보다 훈련 단계에서 네트워크 손실이 훨씬 더 클 것입니다.\n", - "\n", - "L2 규제의 효과를 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7wkfLyxBZdh_", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('l2', l2_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Kx1YHMsVxWjP" - }, - "source": [ - "결과에서 보듯이 모델 파라미터의 개수는 같지만 L2 규제를 적용한 모델이 기본 모델보다 과대적합에 훨씬 잘 견디고 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HmnBNOOVxiG8" - }, - "source": [ - "### 드롭아웃 추가하기\n", - "\n", - "드롭아웃(dropout)은 신경망에서 가장 효과적이고 널리 사용하는 규제 기법 중 하나입니다. 토론토(Toronto) 대학의 힌튼(Hinton)과 그의 제자들이 개발했습니다. 드롭아웃을 층에 적용하면 훈련하는 동안 층의 출력 특성을 랜덤하게 끕니다(즉, 0으로 만듭니다). 훈련하는 동안 어떤 입력 샘플에 대해 [0.2, 0.5, 1.3, 0.8, 1.1] 벡터를 출력하는 층이 있다고 가정해 보죠. 드롭아웃을 적용하면 이 벡터에서 몇 개의 원소가 랜덤하게 0이 됩니다. 예를 들면, [0, 0.5, 1.3, 0, 1.1]가 됩니다. \"드롭아웃 비율\"은 0이 되는 특성의 비율입니다. 보통 0.2에서 0.5 사이를 사용합니다. 테스트 단계에서는 어떤 유닛도 드롭아웃하지 않습니다. 훈련 단계보다 더 많은 유닛이 활성화되기 때문에 균형을 맞추기 위해 층의 출력 값을 드롭아웃 비율만큼 줄입니다.\n", - "\n", - "`tf.keras`에서는 `Dropout` 층을 이용해 네트워크에 드롭아웃을 추가할 수 있습니다. 이 층은 바로 이전 층의 출력에 드롭아웃을 적용합니다.\n", - "\n", - "IMDB 네트워크에 두 개의 `Dropout` 층을 추가하여 과대적합이 얼마나 감소하는지 알아 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OFEYvtrHxSWS", - "colab": {} - }, - "source": [ - "dpt_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(16, activation=tf.nn.relu),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "dpt_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "dpt_model_history = dpt_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SPZqwVchx5xp", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('dropout', dpt_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gjfnkEeQyAFG" - }, - "source": [ - "드롭아웃을 추가하니 기준 모델보다 확실히 향상되었습니다.\n", - "\n", - "정리하면 신경망에서 과대적합을 방지하기 위해 가장 널리 사용하는 방법은 다음과 같습니다:\n", - "\n", - "* 더 많은 훈련 데이터를 모읍니다.\n", - "* 네트워크의 용량을 줄입니다.\n", - "* 가중치 규제를 추가합니다.\n", - "* 드롭아웃을 추가합니다.\n", - "\n", - "이 문서에서 다루지 않은 중요한 방법 두 가지는 데이터 증식(data-augmentation)과 배치 정규화(batch normalization)입니다." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/r1/tutorials/keras/save_and_restore_models.ipynb b/site/ko/r1/tutorials/keras/save_and_restore_models.ipynb deleted file mode 100644 index 64c5ca8e3a8..00000000000 --- a/site/ko/r1/tutorials/keras/save_and_restore_models.ipynb +++ /dev/null @@ -1,861 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "save_and_restore_models.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pZJ3uY9O17VN" - }, - "source": [ - "# 모델 저장과 복원" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M4Ata7_wMul1" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Uh4aUT57arZH", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBdde4YJeJKF" - }, - "source": [ - "훈련하는 도중이나 훈련이 끝난 후에 모델을 저장할 수 있습니다. 모델을 중지된 지점부터 다시 훈련할 수 있어 한 번에 오랫동안 훈련하지 않아도 됩니다. 또 모델을 저장하면 다른 사람에게 공유할 수 있고 작업을 재현할 수 있습니다. 연구한 모델과 기법을 공개할 때 많은 머신 러닝 기술자들이 다음과 같은 것들을 제공합니다:\n", - "\n", - "* 모델을 만드는 코드\n", - "* 모델의 훈련된 가중치 또는 파라미터\n", - "\n", - "이런 데이터를 공유하면 다른 사람들이 모델의 작동 방식을 이해하고 새로운 데이터로 모델을 실험하는데 도움이 됩니다.\n", - "\n", - "주의: 신뢰할 수 없는 코드는 조심하세요. 텐서플로 모델은 프로그램 코드입니다. 자세한 내용은 [텐서플로를 안전하게 사용하기](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) 문서를 참고하세요.\n", - "\n", - "### 저장 방식\n", - "\n", - "사용하는 API에 따라서 여러가지 방법으로 텐서플로 모델을 저장할 수 있습니다. 이 문서는 텐서플로 모델을 만들고 훈련하기 위한 고수준 API인 [tf.keras](https://www.tensorflow.org/r1/guide/keras)를 사용합니다. 다른 방법들에 대해서는 텐서플로의 [저장과 복원](https://www.tensorflow.org/r1/guide/saved_model) 문서와 즉시 실행(eager execution) 문서의 [저장하기](https://www.tensorflow.org/r1/guide/eager#object-based_saving) 섹션을 참고하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xCUREq7WXgvg" - }, - "source": [ - "## 설정\n", - "\n", - "### 설치와 임포트" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7l0MiTOrXtNv" - }, - "source": [ - "필요한 라이브러리를 설치하고 텐서플로를 임포트(import)합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} - }, - "source": [ - "!pip install h5py pyyaml" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbGsznErXWt6" - }, - "source": [ - "### 예제 데이터셋 받기\n", - "\n", - "[MNIST 데이터셋](http://yann.lecun.com/exdb/mnist/)으로 모델을 훈련하여 가중치를 저장하는 예제를 만들어 보겠습니다. 모델 실행 속도를 빠르게 하기 위해 샘플에서 처음 1,000개만 사용겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import os\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "tf.__version__" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_labels = train_labels[:1000]\n", - "test_labels = test_labels[:1000]\n", - "\n", - "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", - "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "anG3iVoXyZGI" - }, - "source": [ - "### 모델 정의" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wynsOBfby0Pa" - }, - "source": [ - "가중치를 저장하고 불러오는 예제를 위해 간단한 모델을 만들어 보죠." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} - }, - "source": [ - "# 간단한 Sequential 모델을 반환합니다\n", - "def create_model():\n", - " model = tf.keras.models.Sequential([\n", - " keras.layers.Dense(512, activation=tf.keras.activations.relu, input_shape=(784,)),\n", - " keras.layers.Dropout(0.2),\n", - " keras.layers.Dense(10, activation=tf.keras.activations.softmax)\n", - " ])\n", - "\n", - " model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - "\n", - " return model\n", - "\n", - "\n", - "# 모델 객체를 만듭니다\n", - "model = create_model()\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "soDE0W_KH8rG" - }, - "source": [ - "## 훈련하는 동안 체크포인트 저장하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRyd5qQQIXZm" - }, - "source": [ - "*훈련 중간*과 *훈련 마지막*에 체크포인트(checkpoint)를 자동으로 저장하도록 하는 것이 많이 사용하는 방법입니다. 다시 훈련하지 않고 모델을 재사용하거나 훈련 과정이 중지된 경우 이어서 훈련을 진행할 수 있습니다.\n", - "\n", - "`tf.keras.callbacks.ModelCheckpoint`은 이런 작업을 수행하는 콜백(callback)입니다. 이 콜백은 체크포인트 작업을 조정할 수 있도록 여러가지 매개변수를 제공합니다.\n", - "\n", - "### 체크포인트 콜백 사용하기\n", - "\n", - "`ModelCheckpoint` 콜백을 전달하여 모델을 훈련해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} - }, - "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# 체크포인트 콜백 만들기\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,\n", - " save_weights_only=True,\n", - " verbose=1)\n", - "\n", - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs = 10,\n", - " validation_data = (test_images,test_labels),\n", - " callbacks = [cp_callback]) # 훈련 단계에 콜백을 전달합니다\n", - "\n", - "# 옵티마이저의 상태를 저장하는 것과 관련되어 경고가 발생할 수 있습니다.\n", - "# 이 경고는 (그리고 이 노트북의 다른 비슷한 경고는) 이전 사용 방식을 권장하지 않기 위함이며 무시해도 좋습니다." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlM-sgyJO084" - }, - "source": [ - "이 코드는 텐서플로 체크포인트 파일을 만들고 에포크가 종료될 때마다 업데이트합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} - }, - "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wlRN_f56Pqa9" - }, - "source": [ - "훈련하지 않은 새로운 모델을 만들어 보겠습니다. 가중치만 복원할 땐 원본 모델과 동일한 구조로 모델을 만들어야 합니다. 여기서는 동일한 구조로 모델을 만들었으므로 다른 *객체*이지만 가중치를 공유할 수 있습니다.\n", - "\n", - "훈련하지 않은 새 모델을 만들고 테스트 세트에서 평가해 보죠. 훈련되지 않은 모델의 성능은 무작위로 선택하는 정도의 수준입니다(~10% 정확도):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"훈련되지 않은 모델의 정확도: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1DTKpZssRSo3" - }, - "source": [ - "체크포인트에서 가중치를 로드하고 다시 평가해 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} - }, - "source": [ - "model.load_weights(checkpoint_path)\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpAbKkAyVPV8" - }, - "source": [ - "### 체크포인트 콜백 매개변수\n", - "\n", - "이 콜백 함수는 몇 가지 매개변수를 제공합니다. 체크포인트 이름을 고유하게 만들거나 체크포인트 주기를 조정할 수 있습니다.\n", - "\n", - "새로운 모델을 훈련하고 다섯 번의 에포크마다 고유한 이름으로 체크포인트를 저장해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} - }, - "source": [ - "# 파일 이름에 에포크 번호를 포함시킵니다(`str.format` 포맷)\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " checkpoint_path, verbose=1, save_weights_only=True,\n", - " # 다섯 번째 에포크마다 가중치를 저장합니다\n", - " period=5)\n", - "\n", - "model = create_model()\n", - "model.save_weights(checkpoint_path.format(epoch=0))\n", - "model.fit(train_images, train_labels,\n", - " epochs = 50, callbacks = [cp_callback],\n", - " validation_data = (test_images,test_labels),\n", - " verbose=0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1zFrKTjjavWI" - }, - "source": [ - "만들어진 체크포인트를 확인해 보고 마지막 체크포인트를 선택해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} - }, - "source": [ - "! ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} - }, - "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", - "latest" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zk2ciGbKg561" - }, - "source": [ - "노트: 텐서플로는 기본적으로 최근 5개의 체크포인트만 저장합니다.\n", - "\n", - "모델을 초기화하고 최근 체크포인트를 로드하여 테스트해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "model.load_weights(latest)\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c2OxsJOTHxia" - }, - "source": [ - "## 체크포인트 파일" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtdYhvWnH2ib" - }, - "source": [ - "위 코드는 가중치를 일련의 [체크포인트](https://www.tensorflow.org/r1/guide/saved_model#save_and_restore_variables) 포맷의 파일에 저장합니다. 이 파일에 포함되는 것은 훈련된 이진 포맷의 가중치입니다. 체크포인트가 담고 있는 것은 다음과 같습니다:\n", - "\n", - "* 모델의 가중치를 포함하는 하나 이상의 샤드(shard)\n", - "* 가중치가 어느 샤드에 저장되어 있는지를 나타내는 인덱스 파일\n", - "\n", - "단일 머신에서 모델을 훈련한다면 `.data-00000-of-00001` 확장자를 가진 샤드 하나만 만들어 집니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S_FA-ZvxuXQV" - }, - "source": [ - "## 수동으로 가중치 저장하기\n", - "\n", - "앞에서 가중치를 모델에 로드하는 방법을 보았습니다.\n", - "\n", - "수동으로 가중치를 저장하는 것도 쉽습니다. `Model.save_weights` 메서드를 사용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} - }, - "source": [ - "# 가중치를 저장합니다\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# 가중치를 복원합니다\n", - "model = create_model()\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", - "\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kOGlxPRBEvV1" - }, - "source": [ - "## 모델 전체를 저장하기\n", - "\n", - "전체 모델을 파일 하나에 저장할 수 있습니다. 여기에는 가중치, 모델 구성 심지어 옵티마이저에 지정한 설정까지 포함됩니다. 모델의 체크포인트를 저장하므로 원본 코드를 사용하지 않고 나중에 정확히 동일한 상태에서 훈련을 다시 시작할 수 있습니다.\n", - "\n", - "전체 모델을 저장하는 기능은 매우 유용합니다. TensorFlow.js로 모델을 로드한 다음 웹 브라우저에서 모델을 훈련하고 실행할 수 있습니다([HDF5](https://js.tensorflow.org/r1/tutorials/import-keras.html), [Saved Model](https://js.tensorflow.org/r1/tutorials/import-saved-model.html)). 또는 모바일 장치에 맞도록 변환한 다음 TensorFlow Lite를 사용하여 실행할 수 있습니다([HDF5](https://www.tensorflow.org/lite/convert/python_api#exporting_a_tfkeras_file_), [Saved Model](https://www.tensorflow.org/lite/convert/python_api#exporting_a_savedmodel_))." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" - }, - "source": [ - "### HDF5 파일로 저장하기\n", - "\n", - "케라스는 [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) 표준을 따르는 기본 저장 포맷을 제공합니다. 저장된 모델을 하나의 이진 파일(binary blob)처럼 다룰 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)\n", - "\n", - "# 전체 모델을 HDF5 파일로 저장합니다\n", - "model.save('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" - }, - "source": [ - "이제 이 파일로부터 모델을 다시 만들어 보죠:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} - }, - "source": [ - "# 가중치와 옵티마이저를 포함하여 정확히 동일한 모델을 다시 생성합니다\n", - "new_model = keras.models.load_model('my_model.h5')\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" - }, - "source": [ - "정확도를 확인해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} - }, - "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "이 기법은 모든 것을 저장합니다:\n", - "\n", - "* 가중치 값\n", - "* 모델 설정(구조)\n", - "* 옵티마이저 설정\n", - "\n", - "케라스는 모델 구조를 확인하고 저장합니다. 현재는 텐서플로 옵티마이저(`tf.train`)를 저장할 수 없습니다. 이런 경우에는 모델을 로드한 후에 다시 컴파일해야 합니다. 옵티마이저의 상태는 유지되지 않습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPyhgcoVzqUB" - }, - "source": [ - "### `saved_model`을 사용하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LtcN4VIb7JkK" - }, - "source": [ - "주의: `tf.keras` 모델을 저장하는 이 메서드는 실험적이므로 향후 버전에서 변경될 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DSWiSB0Q8c46" - }, - "source": [ - "새로운 모델을 만들어 보겠습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iUvT_3qE8hV5" - }, - "source": [ - "`saved_model`을 만듭니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "colab": {} - }, - "source": [ - "saved_model_path = tf.contrib.saved_model.save_keras_model(model, \"./saved_models\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MjpmyPfh8-1n" - }, - "source": [ - "타임스탬프를 이름으로 가진 디렉토리에 모델이 저장되어 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZtOvxA7V0iTv", - "colab": {} - }, - "source": [ - "!ls saved_models/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B7qfpvpY9HCe" - }, - "source": [ - "저장된 모델로부터 새로운 케라스 모델을 로드합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} - }, - "source": [ - "new_model = tf.contrib.saved_model.load_keras_model(saved_model_path)\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" - }, - "source": [ - "복원된 모델을 실행합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Pc9e6G6w1AWG", - "colab": {} - }, - "source": [ - "# 이 모델을 평가하려면 그전에 컴파일해야 합니다.\n", - "# 단지 저장된 모델의 배포라면 이 단계가 필요하지 않습니다.\n", - "\n", - "new_model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - "\n", - "# 복원된 모델을 평가합니다\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eUYTzSz5VxL2" - }, - "source": [ - "## 그 다음엔\n", - "\n", - "\n", - "이 문서에서는 `tf.keras`를 사용하여 모델을 저장하고 로드하는 방법을 간단하게 안내했습니다.\n", - "\n", - "* [tf.keras 가이드](https://www.tensorflow.org/r1/guide/keras)에서 `tf.keras`로 모델을 저장하고 로드하는 정보를 더 볼 수 있습니다.\n", - "\n", - "* 즉시 실행 모드에서 모델을 저장하려면 [즉시 실행에서 저장하기](https://www.tensorflow.org/r1/guide/eager#object_based_saving) 문서를 참고하세요.\n", - "\n", - "* [저장과 복원](https://www.tensorflow.org/r1/guide/saved_model) 문서는 저수준의 텐서플로 저장 기능에 대해 설명합니다." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/swift/README.md b/site/ko/swift/README.md deleted file mode 100644 index d505dc25e4c..00000000000 --- a/site/ko/swift/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Swift for TensorFlow (S4TF) translations - -To contribute translations for the S4TF docs, please refer to the -[ko/README](https://github.com/tensorflow/docs/tree/master/site/ko/README.md) -and [docs-ko](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko) -group. diff --git a/site/ko/swift/python_interoperability.ipynb b/site/ko/swift/python_interoperability.ipynb deleted file mode 100644 index 61a1bdb1a4f..00000000000 --- a/site/ko/swift/python_interoperability.ipynb +++ /dev/null @@ -1,523 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9TV7IYeqifSv" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors. [Licensed under the Apache License, Version 2.0](#scrollTo=ByZjmtFgB_Y5)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tRIJp_4m_Afz" - }, - "outputs": [], - "source": [ - "// #@title Licensed under the Apache License, Version 2.0 (the \"License\"); { display-mode: \"form\" }\n", - "// Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "// you may not use this file except in compliance with the License.\n", - "// You may obtain a copy of the License at\n", - "//\n", - "// https://www.apache.org/licenses/LICENSE-2.0\n", - "//\n", - "// Unless required by applicable law or agreed to in writing, software\n", - "// distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "// See the License for the specific language governing permissions and\n", - "// limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sI1ZtrdiA4aY" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/swift/tutorials/python_interoperability\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eTensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/swift/python_interoperability.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/swift/python_interoperability.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e깃허브(GitHub)에서 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-agGVYp_4GWZ" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/swift/blob/master/docs/site/tutorials/python_interoperability.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃허브 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8sa42_NblqRE" - }, - "source": [ - "# 파이썬 상호 호환성\n", - "\n", - "텐서플로를 위한 스위프트는 파이썬과 상호 호환됩니다.\n", - "\n", - "스위프트에서 파이썬 모듈을 임포트해서, 스위프트와 파이썬 사이의 값을 바꾸거나 파이썬 함수를 호출할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kZRlD4utdPuX" - }, - "outputs": [], - "source": [ - "import Python\n", - "print(Python.version)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lcsgrcnc490l" - }, - "source": [ - "## 파이썬 버전 설정" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JvvMEn2o490m" - }, - "source": [ - "기본적으로 `import Python`를 하면, 스위프트는 시스템 라이브러리 경로에 따라 설치된 최신 버전의 파이썬을 검색합니다. \n", - "특정한 파이썬을 설치하려면, `PYTHON_LIBRARY` 환경변수에 설치시 제공받은 `libpython` 공유 라이브러리를 설정합니다. 예를 들어: \n", - "\n", - "`export PYTHON_LIBRARY=\"~/anaconda3/lib/libpython3.7m.so\"`\n", - "\n", - "정확한 파일명은 파이썬 환경과 플랫폼마다 다를 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jELTcaDK490n" - }, - "source": [ - "또는 스위프트가 시스템 라이브러리 경로에서 알맞은 파이썬 버전을 찾도록 해서 `PYTHON_VERSION` 환경변수를 설정할 수 있습니다. `PYTHON_LIBRARY`가 `PYTHON_VERSION` 보다 우선한다는 점을 유의해야 합니다.\n", - "\n", - "또한 코드에서 `PYTHON_VERSION` 설정과 동일한 기능을 하는 `PythonLibrary.useVersion` 함수를 호출할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7jEc0ZF8490o" - }, - "outputs": [], - "source": [ - "// PythonLibrary.useVersion(2)\n", - "// PythonLibrary.useVersion(3, 7)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JEucWkEK490q" - }, - "source": [ - "__Note: 파이썬을 임포트한 직후, 파이썬 코드를 호출하기 전에 `PythonLibrary.useVersion`을 실행해야 합니다. 이것은 파이썬 버전을 동적으로 바꾸는 데 사용될 수 없습니다.__" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Meb9Mb9P490q" - }, - "source": [ - "[파이썬 라이브러리 로딩 과정에서 생성되는 디버그 출력](https://github.com/apple/swift/pull/20674#discussion_r235207008)을 확인하기 위해서 `PYTHON_LOADER_LOGGING=1`를 설정하세요. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rU0WY_sJodio" - }, - "source": [ - "## 기초\n", - "\n", - "스위프트에서 `PythonObject`는 파이썬 객체를 나타냅니다.\n", - "모든 파이썬 API는 `PythonObject` 인스턴스를 사용하거나 반환합니다.\n", - "\n", - "스위프트에서 기본형(숫자 및 배열처럼)은 `PythonObject`로 전환할 수 있습니다. 몇몇 경우에 (`PythonConvertible` 인수를 받는 리터럴과 함수의 경우), 변환이 암묵적으로 일어납니다. 스위프트 값을 `PythonObject`에 명시적으로 지정하려면 `PythonObject` 이니셜라이저를 사용합니다.\n", - "\n", - "`PythonObject`는 숫자 연산, 인덱싱, 반복을 포함한 많은 표준 연산을 정의합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kqXILiXhq-iM" - }, - "outputs": [], - "source": [ - "// 표준 스위프트 자료형을 Python으로 변환합니다.\n", - "let pythonInt: PythonObject = 1\n", - "let pythonFloat: PythonObject = 3.0\n", - "let pythonString: PythonObject = \"Hello Python!\"\n", - "let pythonRange: PythonObject = PythonObject(5..\u003c10)\n", - "let pythonArray: PythonObject = [1, 2, 3, 4]\n", - "let pythonDict: PythonObject = [\"foo\": [0], \"bar\": [1, 2, 3]]\n", - "\n", - "// 파이썬 객체에 표준 연산을 수행합니다.\n", - "print(pythonInt + pythonFloat)\n", - "print(pythonString[0..\u003c6])\n", - "print(pythonRange)\n", - "print(pythonArray[2])\n", - "print(pythonDict[\"bar\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fEAEyUExXT3I" - }, - "outputs": [], - "source": [ - "// 파이썬 객체를 다시 스위프트로 변환합니다.\n", - "let int = Int(pythonInt)!\n", - "let float = Float(pythonFloat)!\n", - "let string = String(pythonString)!\n", - "let range = Range\u003cInt\u003e(pythonRange)!\n", - "let array: [Int] = Array(pythonArray)!\n", - "let dict: [String: [Int]] = Dictionary(pythonDict)!\n", - "\n", - "// 표준 연산을 수행합니다.\n", - "// 출력은 파이썬과 동일합니다!\n", - "print(Float(int) + float)\n", - "print(string.prefix(6))\n", - "print(range)\n", - "print(array[2])\n", - "print(dict[\"bar\"]!)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1pMewsl0VgnJ" - }, - "source": [ - "`PythonObject`는 많은 표준 스위프트 프로토콜에 대해 적합하도록 정의합니다:\n", - "* `Equatable`\n", - "* `Comparable`\n", - "* `Hashable`\n", - "* `SignedNumeric`\n", - "* `Strideable`\n", - "* `MutableCollection`\n", - "* 모든 `ExpressibleBy_Literal` 프로토콜\n", - "\n", - "이러한 적합성은 형안전(type-safe)하지 않다는 점에 유의해야 합니다: 호환되지 않는 `PythonObject` 인스턴스에서 프로토콜 기능을 사용하려고 할 때 충돌이 발생할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "W9bUsiOxVf_v" - }, - "outputs": [], - "source": [ - "let one: PythonObject = 1\n", - "print(one == one)\n", - "print(one \u003c one)\n", - "print(one + one)\n", - "\n", - "let array: PythonObject = [1, 2, 3]\n", - "for (i, x) in array.enumerated() {\n", - " print(i, x)\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w3lmTRCWT5sS" - }, - "source": [ - "튜플을 파이썬에서 스위프트로 변환하려면, 정확한 튜플의 길이를 알아야 합니다.\n", - "\n", - "다음 인스턴스 메서드 중 하나를 호출합니다:\n", - "- `PythonObject.tuple2`\n", - "- `PythonObject.tuple3`\n", - "- `PythonObject.tuple4`" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fQ0HEX89T4mW" - }, - "outputs": [], - "source": [ - "let pythonTuple = Python.tuple([1, 2, 3])\n", - "print(pythonTuple, Python.len(pythonTuple))\n", - "\n", - "// 스위프트로 변환합니다.\n", - "let tuple = pythonTuple.tuple3\n", - "print(tuple)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Te7sNNx9c_am" - }, - "source": [ - "## 파이썬 내장 객체\n", - "\n", - "전역 `Python` 인터페이스를 활용해 파이썬 내장 객체에 접근합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jpcOByipc75O" - }, - "outputs": [], - "source": [ - "// `Python.builtins`은 모든 파이썬 내장 객체의 딕셔너리입니다.\n", - "_ = Python.builtins\n", - "\n", - "// 파이썬 내장 객체를 사용합니다.\n", - "print(Python.type(1))\n", - "print(Python.len([1, 2, 3]))\n", - "print(Python.sum([1, 2, 3]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H2wwUL1tY3JX" - }, - "source": [ - "## 파이썬 모듈 임포트\n", - "\n", - "`Python.import`를 사용하여 파이썬 모듈을 임포트합니다. 이것은 `Python`의 `import` 키워드처럼 동작합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "XrZee8n3Y17_" - }, - "outputs": [], - "source": [ - "let np = Python.import(\"numpy\")\n", - "print(np)\n", - "let zeros = np.ones([2, 3])\n", - "print(zeros)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hQvza3dUXlr0" - }, - "source": [ - "안전하게 패키지를 가져오기 위해 예외처리 함수 `Python.attemptImport`를 사용하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QD-uQGuaXhrM" - }, - "outputs": [], - "source": [ - "let maybeModule = try? Python.attemptImport(\"nonexistent_module\")\n", - "print(maybeModule)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Qej_Z6V3mZnG" - }, - "source": [ - "## `numpy.ndarray`로 변환\n", - "\n", - "다음 스위프트 자료형은 `numpy.ndarray`로 변환할 수 있습니다:\n", - "- `Array\u003cElement\u003e`\n", - "- `ShapedArray\u003cScalar\u003e`\n", - "- `Tensor\u003cScalar\u003e`\n", - "\n", - "`Numpy.ndarray`의 `dtype`이 `Element` 또는 `Scalar`의 일반 파라미터 타입과 호환되어야만 변환이 성공합니다.\n", - "\n", - "`Array`의 경우 `numpy.ndarray`가 1-D일 경우에만 `numpy`에서 변환이 성공합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hPvKgZBeDQ1p" - }, - "outputs": [], - "source": [ - "import TensorFlow\n", - "\n", - "let numpyArray = np.ones([4], dtype: np.float32)\n", - "print(\"Swift type:\", type(of: numpyArray))\n", - "print(\"Python type:\", Python.type(numpyArray))\n", - "print(numpyArray.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ZuDgZ5cBS3Uk" - }, - "outputs": [], - "source": [ - "// `numpy.ndarray`에서 스위프트 타입으로 변환하는 예제.\n", - "let array: [Float] = Array(numpy: numpyArray)!\n", - "let shapedArray = ShapedArray\u003cFloat\u003e(numpy: numpyArray)!\n", - "let tensor = Tensor\u003cFloat\u003e(numpy: numpyArray)!\n", - "\n", - "// 스위프트 타입에서 `numpy.ndarray`으로 변환하는 예제.\n", - "print(array.makeNumpyArray())\n", - "print(shapedArray.makeNumpyArray())\n", - "print(tensor.makeNumpyArray())\n", - "\n", - "// dtype이 다른 예제.\n", - "let doubleArray: [Double] = Array(numpy: np.ones([3], dtype: np.float))!\n", - "let intTensor = Tensor\u003cInt32\u003e(numpy: np.ones([2, 3], dtype: np.int32))!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8EQFZZ5iafwh" - }, - "source": [ - "## 이미지 표시\n", - "\n", - "파이썬 노트북에서처럼 `matplotlib`를 이용해 이미지를 결과 창에 표시할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jUzsa2cxafQV" - }, - "outputs": [], - "source": [ - "// 준비.\n", - "%include \"EnableIPythonDisplay.swift\"\n", - "IPythonDisplay.shell.enable_matplotlib(\"inline\")\n", - "\n", - "let np = Python.import(\"numpy\")\n", - "let plt = Python.import(\"matplotlib.pyplot\")\n", - "\n", - "let time = np.arange(0, 10, 0.01)\n", - "let amplitude = np.exp(-0.1 * time)\n", - "let position = amplitude * np.sin(3 * time)\n", - "\n", - "plt.figure(figsize: [15, 10])\n", - "\n", - "plt.plot(time, position)\n", - "plt.plot(time, amplitude)\n", - "plt.plot(time, -amplitude)\n", - "\n", - "plt.xlabel(\"Time (s)\")\n", - "plt.ylabel(\"Position (m)\")\n", - "plt.title(\"Oscillations\")\n", - "\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "Python interoperability.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Swift", - "language": "swift", - "name": "swift" - }, - "language_info": { - "file_extension": ".swift", - "mimetype": "text/x-swift", - "name": "swift", - "version": "" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/customization/autodiff.ipynb b/site/ko/tutorials/customization/autodiff.ipynb deleted file mode 100644 index 74c23107820..00000000000 --- a/site/ko/tutorials/customization/autodiff.ipynb +++ /dev/null @@ -1,360 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "GCCk8_dHpuNf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xh8WkEwWpnm7" - }, - "source": [ - "# 자동 미분과 그래디언트 테이프" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idv0bPeCp325" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/autodiff\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eTensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/customization/autodiff.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/customization/autodiff.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e \n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/ko/tutorials/customization/autodiff.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003e노트북 다운로드\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpc-TaTwBh7l" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 지원하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJ4XzMqodTy" - }, - "source": [ - "이전 튜토리얼에서는 텐서(tensor)와 텐서의 연산에 대해서 알아보았습니다. 이번 튜토리얼에서는 머신러닝 모델을 최적화할 수 있는 주요 기술 중 하나인 [자동 미분(automatic differentiation)](https://en.wikipedia.org/wiki/Automatic_differentiation)에 대해 알아보겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GQJysDM__Qb0" - }, - "source": [ - "## 설정\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cxzaxo6ff2y3" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1CLWJl0QliB0" - }, - "source": [ - "## 그래디언트 테이프\n", - "\n", - "텐서플로는 자동 미분(주어진 입력 변수에 대한 연산의 그래디언트(gradient)를 계산하는 것)을 위한 [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) API를 제공합니다. `tf.GradientTape`는 컨텍스트(context) 안에서 실행된 모든 연산을 테이프(tape)에 \"기록\"합니다. 그 다음 텐서플로는 [후진 방식 자동 미분(reverse mode differentiation)](https://en.wikipedia.org/wiki/Automatic_differentiation)을 사용해 테이프에 \"기록된\" 연산의 그래디언트를 계산합니다.\n", - "\n", - "예를 들면:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bAFeIE8EuVIq" - }, - "outputs": [], - "source": [ - "x = tf.ones((2, 2))\n", - "\n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# 입력 텐서 x에 대한 z의 도함수\n", - "dz_dx = t.gradient(z, x)\n", - "for i in [0, 1]:\n", - " for j in [0, 1]:\n", - " assert dz_dx[i][j].numpy() == 8.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N4VlqKFzzGaC" - }, - "source": [ - "또한 `tf.GradientTape` 컨텍스트 안에서 계산된 중간값에 대한 그래디언트도 구할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7XaPRAwUyYms" - }, - "outputs": [], - "source": [ - "x = tf.ones((2, 2))\n", - "\n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# 테이프 사용하여 중간값 y에 대한 도함수를 계산합니다. \n", - "dz_dy = t.gradient(z, y)\n", - "assert dz_dy.numpy() == 8.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISkXuY7YzIcS" - }, - "source": [ - "기본적으로 GradientTape.gradient() 메서드가 호출되면 GradientTape에 포함된 리소스가 해제됩니다. 동일한 연산에 대해 여러 그래디언트를 계산하려면, `지속성있는`(persistent) 그래디언트 테이프를 생성하면 됩니다. 이 그래디언트 테이프는 `gradient()` 메서드의 다중 호출을 허용합니다. 테이프 객체가 쓰레기 수집(garbage collection)될때 리소스는 해제됩니다.\n", - "예를 들면 다음과 같습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zZaCm3-9zVCi" - }, - "outputs": [], - "source": [ - "x = tf.constant(3.0)\n", - "with tf.GradientTape(persistent=True) as t:\n", - " t.watch(x)\n", - " y = x * x\n", - " z = y * y\n", - "dz_dx = t.gradient(z, x) # 108.0 (4*x^3 at x = 3)\n", - "dy_dx = t.gradient(y, x) # 6.0\n", - "del t # 테이프에 대한 참조를 삭제합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6kADybtQzYj4" - }, - "source": [ - "### 제어 흐름 기록\n", - "\n", - "연산이 실행되는 순서대로 테이프에 기록되기 때문에, 파이썬 제어 흐름(예를 들어 `if` `while`, `for`문 같은)이 자연스럽게 처리됩니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9FViq92UX7P8" - }, - "outputs": [], - "source": [ - "def f(x, y):\n", - " output = 1.0\n", - " for i in range(y):\n", - " if i \u003e 1 and i \u003c 5:\n", - " output = tf.multiply(output, x)\n", - " return output\n", - "\n", - "def grad(x, y):\n", - " with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " out = f(x, y)\n", - " return t.gradient(out, x)\n", - "\n", - "x = tf.convert_to_tensor(2.0)\n", - "\n", - "assert grad(x, 6).numpy() == 12.0\n", - "assert grad(x, 5).numpy() == 12.0\n", - "assert grad(x, 4).numpy() == 4.0\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DK05KXrAAld3" - }, - "source": [ - "### 고계도 그래디언트\n", - "\n", - "`GradientTape` 컨텍스트 매니저안에 있는 연산들은 자동미분을 위해 기록됩니다. 만약 이 컨텍스트 안에서 그래디언트를 계산하면 해당 그래디언트 연산 또한 기록되어집니다. 그 결과 똑같은 API가 고계도(Higher-order) 그래디언트에서도 잘 작동합니다. 예를 들면:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cPQgthZ7ugRJ" - }, - "outputs": [], - "source": [ - "x = tf.Variable(1.0) # 1.0으로 초기화된 텐서플로 변수를 생성합니다.\n", - "\n", - "with tf.GradientTape() as t:\n", - " with tf.GradientTape() as t2:\n", - " y = x * x * x\n", - " # 't' 컨텍스트 매니저 안의 그래디언트를 계산합니다.\n", - " # 이것은 또한 그래디언트 연산 자체도 미분가능하다는 것을 의미합니다. \n", - " dy_dx = t2.gradient(y, x)\n", - "d2y_dx2 = t.gradient(dy_dx, x)\n", - "\n", - "assert dy_dx.numpy() == 3.0\n", - "assert d2y_dx2.numpy() == 6.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4U1KKzUpNl58" - }, - "source": [ - "## 다음 단계\n", - "\n", - "이번 튜토리얼에서는 텐서플로에서 그래디언트를 계산하는 방법을 다루었습니다. 이를 통해 신경망(neural network)을 구축하고 훈련시키는데 필요한 많은 기본 요소를 배웠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "j8BT7T8yf6Rv" - }, - "outputs": [], - "source": [ - "" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "autodiff.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/customization/basics.ipynb b/site/ko/tutorials/customization/basics.ipynb deleted file mode 100644 index ccb4ed9db60..00000000000 --- a/site/ko/tutorials/customization/basics.ipynb +++ /dev/null @@ -1,495 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iPpI7RaYoZuE" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "hro2InpHobKk" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# 텐서와 연산" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Hndw-YcxoOJK" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/basics\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eTensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/customization/basics.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/customization/basics.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e \n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/ko/tutorials/customization/basics.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003e노트북 다운로드\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GqGgwwojCHFS" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 지원하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6sILUVbHoSgH" - }, - "source": [ - "이 노트북은 텐서플로를 사용하기 위한 입문 튜토리얼입니다. 다음 내용을 다룹니다 : \n", - "\n", - "* 필요한 패키지 임포트\n", - "* 텐서(Tensor) 생성 및 사용\n", - "* GPU 가속기 사용\n", - "* `tf.data.Dataset` 시연" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "miTaGiqV9RjO" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1JcS5iBXMRO" - }, - "source": [ - "## 텐서플로 임포트\n", - "\n", - "시작하기 위해서 텐서플로 모듈을 임포트합니다. 텐서플로 2.0에서는 즉시 실행(eager execution)이 기본적으로 실행됩니다. 이는 텐서플로를 조금 더 대화형 프론트엔드(frontend)에 가깝게 만들어 줍니다. 세부사항은 나중에 이야기할 것입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vjBPmYjLdFmk" - }, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H9UySOPLXdaw" - }, - "source": [ - "## 텐서\n", - "\n", - "텐서는 다차원 배열입니다. 넘파이(NumPy) `ndarray` 객체와 비슷하며, `tf.Tensor` 객체는 데이터 타입과 크기를 가지고 있습니다. 또한 `tf.Tensor`는 GPU 같은 가속기 메모리에 상주할 수 있습니다. 텐서플로는 텐서를 생성하고 이용하는 풍부한 연산 라이브러리([tf.add](https://www.tensorflow.org/api_docs/python/tf/add), [tf.matmul](https://www.tensorflow.org/api_docs/python/tf/matmul), [tf.linalg.inv](https://www.tensorflow.org/api_docs/python/tf/linalg/inv) 등.)를 제공합니다. 이러한 연산은 자동으로 텐서를 파이썬 네이티브(native) 타입으로 변환합니다.\n", - "\n", - "예를 들어:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": {}, - "colab_type": "code", - "id": "ngUe237Wt48W" - }, - "outputs": [], - "source": [ - "print(tf.add(1, 2))\n", - "print(tf.add([1, 2], [3, 4]))\n", - "print(tf.square(5))\n", - "print(tf.reduce_sum([1, 2, 3]))\n", - "\n", - "# 연산자 오버로딩(overloading) 또한 지원합니다.\n", - "print(tf.square(2) + tf.square(3))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDY4WsYRhP81" - }, - "source": [ - "각각의 `tf.Tensor`는 크기와 데이터 타입을 가지고 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "srYWH1MdJNG7" - }, - "outputs": [], - "source": [ - "x = tf.matmul([[1]], [[2, 3]])\n", - "print(x)\n", - "print(x.shape)\n", - "print(x.dtype)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eBPw8e8vrsom" - }, - "source": [ - "넘파이 배열과 `tf.Tensor`의 가장 확연한 차이는 다음과 같습니다:\n", - "\n", - "1. `텐서`는 가속기 메모리(GPU, TPU와 같은)에서 사용할 수 있습니다.\n", - "2. `텐서`는 불변성(immutable)을 가집니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dwi1tdW3JBw6" - }, - "source": [ - "### 넘파이 호환성\n", - "\n", - "텐서와 넘파이 배열 사이의 변환은 다소 간단합니다.\n", - "\n", - "* 텐서플로 연산은 자동으로 넘파이 배열을 텐서로 변환합니다.\n", - "* 넘파이 연산은 자동으로 텐서를 넘파이 배열로 변환합니다.\n", - "\n", - "텐서는 `.numpy()` 메서드(method)를 호출하여 넘파이 배열로 변환할 수 있습니다.\n", - "가능한 경우, `tf.Tensor`와 배열은 메모리 표현을 공유하기 때문에 이러한 변환은 일반적으로 간단(저렴)합니다. 그러나 `tf.Tensor`는 GPU 메모리에 저장될 수 있고, 넘파이 배열은 항상 호스트 메모리에 저장되므로, 이러한 변환이 항상 가능한 것은 아닙니다. 따라서 GPU에서 호스트 메모리로 복사가 필요합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lCUWzso6mbqR" - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "ndarray = np.ones([3, 3])\n", - "\n", - "print(\"텐서플로 연산은 자동적으로 넘파이 배열을 텐서로 변환합니다.\")\n", - "tensor = tf.multiply(ndarray, 42)\n", - "print(tensor)\n", - "\n", - "\n", - "print(\"그리고 넘파이 연산은 자동적으로 텐서를 넘파이 배열로 변환합니다.\")\n", - "print(np.add(tensor, 1))\n", - "\n", - "print(\".numpy() 메서드는 텐서를 넘파이 배열로 변환합니다.\")\n", - "print(tensor.numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PBNP8yTRfu_X" - }, - "source": [ - "## GPU 가속\n", - "\n", - "대부분의 텐서플로 연산은 GPU를 사용하여 가속화됩니다. 어떠한 코드를 명시하지 않아도, 텐서플로는 연산을 위해 CPU 또는 GPU를 사용할 것인지를 자동으로 결정합니다. 필요시 텐서를 CPU와 GPU 메모리 사이에서 복사합니다. 연산에 의해 생성된 텐서는 전형적으로 연산이 실행된 장치의 메모리에 의해 실행됩니다. 예를 들어:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "code", - "colab": {}, - "colab_type": "code", - "id": "3Twf_Rw-gQFM" - }, - "outputs": [], - "source": [ - "x = tf.random.uniform([3, 3])\n", - "\n", - "print(\"GPU 사용이 가능한가 : \"),\n", - "print(tf.test.is_gpu_available())\n", - "\n", - "print(\"텐서가 GPU #0에 있는가 : \"),\n", - "print(x.device.endswith('GPU:0'))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vpgYzgVXW2Ud" - }, - "source": [ - "### 장치 이름\n", - "\n", - "`Tensor.device`는 텐서를 구성하고 있는 호스트 장치의 풀네임을 제공합니다. 이러한 이름은 프로그램이 실행중인 호스트의 네트워크 주소 및 해당 호스트 내의 장치와 같은 많은 세부 정보를 인코딩하며, 이것은 텐서플로 프로그램의 분산 실행에 필요합니다. 텐서가 호스트의 `N`번째 GPU에 놓여지면 문자열은 `GPU:\u003cN\u003e`으로 끝납니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZWZQCimzuqyP" - }, - "source": [ - "### 명시적 장치 배치\n", - "\n", - "텐서플로에서 \"배치(replacement)\"는 개별 연산을 실행하기 위해 장치에 할당(배치)하는 것입니다. 앞서 언급했듯이, 명시적 지침이 없을 경우 텐서플로는 연산을 실행하기 위한 장치를 자동으로 결정하고, 필요시 텐서를 장치에 복사합니다. 그러나 텐서플로 연산은 `tf.device`을 사용하여 특정한 장치에 명시적으로 배치할 수 있습니다. \n", - "예를 들어:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RjkNZTuauy-Q" - }, - "outputs": [], - "source": [ - "import time\n", - "\n", - "def time_matmul(x):\n", - " start = time.time()\n", - " for loop in range(10):\n", - " tf.matmul(x, x)\n", - "\n", - " result = time.time()-start\n", - "\n", - " print(\"10 loops: {:0.2f}ms\".format(1000*result))\n", - "\n", - "# CPU에서 강제 실행합니다.\n", - "print(\"On CPU:\")\n", - "with tf.device(\"CPU:0\"):\n", - " x = tf.random.uniform([1000, 1000])\n", - " assert x.device.endswith(\"CPU:0\")\n", - " time_matmul(x)\n", - "\n", - "# GPU #0가 이용가능시 GPU #0에서 강제 실행합니다.\n", - "if tf.test.is_gpu_available():\n", - " print(\"On GPU:\")\n", - " with tf.device(\"GPU:0\"): # Or GPU:1 for the 2nd GPU, GPU:2 for the 3rd etc.\n", - " x = tf.random.uniform([1000, 1000])\n", - " assert x.device.endswith(\"GPU:0\")\n", - " time_matmul(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o1K4dlhhHtQj" - }, - "source": [ - "## 데이터셋\n", - "\n", - "이번에는 모델에 데이터를 제공하기 위한 파이프라인을 구축하기 위해 [`tf.data.Dataset` API](https://www.tensorflow.org/guide/datasets)를 사용해볼 것입니다.\n", - "\n", - "`tf.data.Dataset` API는 모델을 훈련시키고 평가 루프를 제공할, 간단하고 재사용 가능한 모듈로부터 복잡한 입력 파이프라인을 구축하기 위해 사용됩니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zI0fmOynH-Ne" - }, - "source": [ - "### 소스 데이터셋 생성\n", - "\n", - "굉장히 유용한 함수중 하나인 [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices)와 같은 팩토리(factory) 함수 중 하나를 사용하거나 파일로부터 읽어들이는 객체인 [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) 또는 [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset)를 사용하여 *소스* 데이터셋을 생성하세요. 더 많은 정보를 위해서 [텐서플로 데이터셋 가이드](https://www.tensorflow.org/guide/datasets#reading_input_data)를 참조하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "F04fVOHQIBiG" - }, - "outputs": [], - "source": [ - "ds_tensors = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", - "\n", - "# CSV 파일을 생성합니다.\n", - "import tempfile\n", - "_, filename = tempfile.mkstemp()\n", - "\n", - "with open(filename, 'w') as f:\n", - " f.write(\"\"\"Line 1\n", - "Line 2\n", - "Line 3\n", - " \"\"\")\n", - "\n", - "ds_file = tf.data.TextLineDataset(filename)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbxIhC-5IPdf" - }, - "source": [ - "### 변환 적용\n", - "\n", - "[`맵(map)`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map), [`배치(batch)`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch), [`셔플(shuffle)`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle)과 같은 변환 함수를 사용하여 데이터셋의 레코드에 적용하세요. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uXSDZWE-ISsd" - }, - "outputs": [], - "source": [ - "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", - "\n", - "ds_file = ds_file.batch(2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "A8X1GNfoIZKJ" - }, - "source": [ - "### 반복\n", - "`tf.data.Dataset`은 레코드 순회를 지원하는 반복가능한 객체입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ws-WKRk5Ic6-" - }, - "outputs": [], - "source": [ - "print('ds_tensors 요소:')\n", - "for x in ds_tensors:\n", - " print(x)\n", - "\n", - "print('\\nds_file 요소:')\n", - "for x in ds_file:\n", - " print(x)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "last_runtime": { - "build_target": "", - "kind": "local" - }, - "name": "basics.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/customization/custom_layers.ipynb b/site/ko/tutorials/customization/custom_layers.ipynb deleted file mode 100644 index 3991fd12a05..00000000000 --- a/site/ko/tutorials/customization/custom_layers.ipynb +++ /dev/null @@ -1,411 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tDnwEv8FtJm7" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "JlknJBWQtKkI" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "60RdWsg1tETW" - }, - "source": [ - "# 사용자 정의 층" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BcJg7Enms86w" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/custom_layers\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eTensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/customization/custom_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/customization/custom_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e \n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/ko/tutorials/customization/custom_layers.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003e노트북 다운로드\u003c/a\u003e\n", - " \u003c/td\u003e \n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ks0_yvycCJzb" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 지원하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UEu3q4jmpKVT" - }, - "source": [ - "신경망을 구축하기 위해서 고수준 API인 `tf.keras`를 사용하길 권합니다. 대부분의 텐서플로 API는 즉시 실행(eager execution)과 함께 사용할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-sXDg19Q691F" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Py0m-N6VgQFJ" - }, - "outputs": [], - "source": [ - "!pip install tensorflow==2.0.0-alpha0\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zSFfVVjkrrsI" - }, - "source": [ - "## 층: 유용한 연산자 집합\n", - "\n", - "머신러닝을 위한 코드를 작성하는 대부분의 경우에 개별적인 연산과 변수를 조작하는 것보다는 높은 수준의 추상화 도구를 사용할 것입니다.\n", - "\n", - "많은 머신러닝 모델은 비교적 단순한 층(layer)을 조합하고 쌓아서 표현가능합니다. 또한 텐서플로는 여러 표준형 층을 제공하므로 사용자 고유의 응용 프로그램에 특화된 층을 처음부터 작성하거나, 기존 층의 조합으로 쉽게 만들 수 있습니다.\n", - "\n", - "텐서플로는 [케라스](https://keras.io)의 모든 API를 tf.keras 패키지에 포함하고 있습니다. 케라스 층은 모델을 구축하는데 매우 유용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8PyXlPl-4TzQ" - }, - "outputs": [], - "source": [ - "# tf.keras.layers 패키지에서 층은 객체입니다. 층을 구성하려면 간단히 객체를 생성하십시오.\n", - "# 대부분의 layer는 첫번째 인수로 출력 차원(크기) 또는 채널을 취합니다.\n", - "layer = tf.keras.layers.Dense(100)\n", - "# 입력 차원의 수는 층을 처음 실행할 때 유추할 수 있기 때문에 종종 불필요합니다. \n", - "# 일부 복잡한 모델에서는 수동으로 입력 차원의 수를 제공하는것이 유용할 수 있습니다.\n", - "layer = tf.keras.layers.Dense(10, input_shape=(None, 5))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fn69xxPO5Psr" - }, - "source": [ - "미리 구성되어있는 층은 다음 [문서](https://www.tensorflow.org/api_docs/python/tf/keras/layers)에서 확인할 수 있습니다. Dense(완전 연결 층), Conv2D, LSTM, BatchNormalization, Dropout, 등을 포함하고 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "E3XKNknP5Mhb" - }, - "outputs": [], - "source": [ - "# 층을 사용하려면, 간단하게 호출합니다.\n", - "layer(tf.zeros([10, 5]))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Wt_Nsv-L5t2s" - }, - "outputs": [], - "source": [ - "# layer는 유용한 메서드를 많이 가지고 있습니다. 예를 들어, `layer.variables`를 사용하여 층안에 있는 모든 변수를 확인할 수 있으며, \n", - "# `layer.trainable_variables`를 사용하여 훈련 가능한 변수를 확인할 수 있습니다. \n", - "# 완전 연결(fully-connected)층은 가중치(weight)와 편향(biases)을 위한 변수를 가집니다. \n", - "layer.variables" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6ilvKjz8_4MQ" - }, - "outputs": [], - "source": [ - "# 또한 변수는 객체의 속성을 통해 편리하게 접근 가능합니다. \n", - "layer.kernel, layer.bias" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O0kDbE54-5VS" - }, - "source": [ - "## 사용자 정의 층 구현\n", - "사용자 정의 층을 구현하는 가장 좋은 방법은 tf.keras.Layer 클래스를 상속하고 다음과 같이 구현하는 것입니다.\n", - " * `__init__` 에서 층에 필요한 매개변수를 입력 받습니다.\n", - " * `build`, 입력 텐서의 크기를 얻고 남은 초기화를 진행할 수 있습니다\n", - " * `call`, 정방향 연산(forward computation)을 진행 할 수 있습니다.\n", - "\n", - "변수를 생성하기 위해 `build`가 호출되길 기다릴 필요가 없다는 것에 주목하세요. 또한 변수를 `__init__`에 생성할 수도 있습니다. 그러나 `build`에 변수를 생성하는 유리한 점은 층이 작동할 입력의 크기를 기준으로 나중에 변수를 만들 수 있다는 것입니다. 반면에, `__init__`에 변수를 생성하는 것은 변수 생성에 필요한 크기가 명시적으로 지정되어야 함을 의미합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5Byl3n1k5kIy" - }, - "outputs": [], - "source": [ - "class MyDenseLayer(tf.keras.layers.Layer):\n", - " def __init__(self, num_outputs):\n", - " super(MyDenseLayer, self).__init__()\n", - " self.num_outputs = num_outputs\n", - "\n", - " def build(self, input_shape):\n", - " self.kernel = self.add_variable(\"kernel\",\n", - " shape=[int(input_shape[-1]),\n", - " self.num_outputs])\n", - "\n", - " def call(self, input):\n", - " return tf.matmul(input, self.kernel)\n", - "\n", - "layer = MyDenseLayer(10)\n", - "print(layer(tf.zeros([10, 5])))\n", - "print(layer.trainable_variables)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tk8E2vY0-z4Z" - }, - "source": [ - "코드를 읽는 사람이 표준형 층의 동작을 잘 알고 있을 것이므로, 가능한 경우 표준형 층을 사용하는것이 전체 코드를 읽고 유지하는데 더 쉽습니다. 만약 `tf.keras.layers` 에 없는 층을 사용하기 원하면 [깃허브](http://github.com/tensorflow/tensorflow/issues/new)에 이슈화하거나, 풀 리퀘스트(pull request)를 보내세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Qhg4KlbKrs3G" - }, - "source": [ - "## 모델: 층 구성\n", - "\n", - "머신러닝 모델에서 대부분의 재미있는 많은 것들은 기존의 층을 조합하여 구현됩니다. 예를 들어, 레즈넷(resnet)의 각 잔여 블록(residual block)은 합성곱(convolution), 배치 정규화(batch normalization), 쇼트컷(shortcut) 등으로 구성되어 있습니다. \n", - "\n", - "다른 층을 포함한 모델을 만들기 위해 사용하는 메인 클래스는 tf.keras.Model입니다. 다음은 tf.keras.Model을 상속(inheritance)하여 구현한 코드입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "N30DTXiRASlb" - }, - "outputs": [], - "source": [ - "class ResnetIdentityBlock(tf.keras.Model):\n", - " def __init__(self, kernel_size, filters):\n", - " super(ResnetIdentityBlock, self).__init__(name='')\n", - " filters1, filters2, filters3 = filters\n", - "\n", - " self.conv2a = tf.keras.layers.Conv2D(filters1, (1, 1))\n", - " self.bn2a = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2b = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same')\n", - " self.bn2b = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2c = tf.keras.layers.Conv2D(filters3, (1, 1))\n", - " self.bn2c = tf.keras.layers.BatchNormalization()\n", - "\n", - " def call(self, input_tensor, training=False):\n", - " x = self.conv2a(input_tensor)\n", - " x = self.bn2a(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2b(x)\n", - " x = self.bn2b(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2c(x)\n", - " x = self.bn2c(x, training=training)\n", - "\n", - " x += input_tensor\n", - " return tf.nn.relu(x)\n", - "\n", - "\n", - "block = ResnetIdentityBlock(1, [1, 2, 3])\n", - "print(block(tf.zeros([1, 2, 3, 3])))\n", - "print([x.name for x in block.trainable_variables])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wYfucVw65PMj" - }, - "source": [ - "그러나 대부분의 경우에, 많은 층으로 구성된 모델은 단순하게 순서대로 층을 하나씩 호출합니다. 이는 tf.keras.Sequential 사용하여 간단한 코드로 구현 가능합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "L9frk7Ur4uvJ" - }, - "outputs": [], - "source": [ - "my_seq = tf.keras.Sequential([tf.keras.layers.Conv2D(1, (1, 1),\n", - " input_shape=(\n", - " None, None, 3)),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(2, 1,\n", - " padding='same'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(3, (1, 1)),\n", - " tf.keras.layers.BatchNormalization()])\n", - "my_seq(tf.zeros([1, 2, 3, 3]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c5YwYcnuK-wc" - }, - "source": [ - "# 다음 단계\n", - "\n", - "이제 이전 노트북으로 돌아가서 선형 회귀 예제에 층과 모델을 사용하여 좀 더 나은 구조를 적용할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "epDp4j2EgTRs" - }, - "outputs": [], - "source": [ - "" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "custom_layers.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/customization/custom_training.ipynb b/site/ko/tutorials/customization/custom_training.ipynb deleted file mode 100644 index 833ae1fa9d7..00000000000 --- a/site/ko/tutorials/customization/custom_training.ipynb +++ /dev/null @@ -1,476 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rmpybwysXGV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "m8y3rGtQsYP2" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hrXv0rU9sIma" - }, - "source": [ - "# 사용자 정의 학습: 기초" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7S0BwJ_8sLu7" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/custom_training\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eTensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/customization/custom_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/customization/custom_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e \n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/ko/tutorials/customization/custom_training.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003e노트북 다운로드\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jHz2jw4OCM9I" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 지원하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k2o3TTG4TFpt" - }, - "source": [ - "이전 튜토리얼에서는 머신러닝을 위한 기본 구성 요소인 자동 미분(automatic differentiation)을 위한 텐서플로 API를 알아보았습니다. 이번 튜토리얼에서는 이전 튜토리얼에서 소개되었던 텐서플로의 기본 요소를 사용하여 간단한 머신러닝을 수행해보겠습니다. \n", - "\n", - "텐서플로는 반복되는 코드를 줄이기 위해 유용한 추상화를 제공하는 고수준 신경망(neural network) API인 `tf.keras`를 포함하고 있습니다. 신경망을 다룰 때 이러한 고수준의 API을 강하게 추천합니다. 이번 짧은 튜토리얼에서는 탄탄한 기초를 기르기 위해 기본적인 요소만으로 신경망 훈련시켜 보겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3LXMVuV0VhDr" - }, - "source": [ - "## 설정" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NiolgWMPgpwI" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eMAWbDJFVmMk" - }, - "source": [ - "## 변수\n", - "\n", - "텐서플로의 텐서(Tensor)는 상태가 없고, 변경이 불가능한(immutable stateless) 객체입니다. 그러나 머신러닝 모델은 상태가 변경될(stateful) 필요가 있습니다. 예를 들어, 모델 학습에서 예측을 계산하기 위한 동일한 코드는 시간이 지남에 따라 다르게(희망하건대 더 낮은 손실로 가는 방향으로)동작해야 합니다. 이 연산 과정을 통해 변화되어야 하는 상태를 표현하기 위해 명령형 프로그래밍 언어인 파이썬을 사용 할 수 있습니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VkJwtLS_Jbn8" - }, - "outputs": [], - "source": [ - "# 파이썬 구문 사용\n", - "x = tf.zeros([10, 10])\n", - "x += 2 # 이것은 x = x + 2와 같으며, x의 초기값을 변경하지 않습니다.\n", - "print(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfneTXy7JcUz" - }, - "source": [ - "텐서플로는 상태를 변경할 수 있는 연산자가 내장되어 있으며, 이러한 연산자는 상태를 표현하기 위한 저수준 파이썬 표현보다 사용하기가 더 좋습니다. 예를 들어, 모델에서 가중치를 나타내기 위해서 텐서플로 변수를 사용하는 것이 편하고 효율적입니다. \n", - "\n", - "텐서플로 변수는 값을 저장하는 객체로 텐서플로 연산에 사용될 때 저장된 이 값을 읽어올 것입니다. `tf.assign_sub`, `tf.scatter_update` 등은 텐서플로 변수에 저장되있는 값을 조작하는 연산자입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "itxmrMil6DQi" - }, - "outputs": [], - "source": [ - "v = tf.Variable(1.0)\n", - "assert v.numpy() == 1.0\n", - "\n", - "# 값을 재배열합니다.\n", - "v.assign(3.0)\n", - "assert v.numpy() == 3.0\n", - "\n", - "# tf.square()와 같은 텐서플로 연산에 `v`를 사용하고 재할당합니다. \n", - "v.assign(tf.square(v))\n", - "assert v.numpy() == 9.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-paSaeq1JzwC" - }, - "source": [ - "변수를 사용한 연산은 그래디언트가 계산될 때 자동적으로 추적됩니다. 임베딩(embedding)을 나타내는 변수의 경우 기본적으로 희소 텐서(sparse tensor)를 사용하여 업데이트됩니다. 이는 연산과 메모리에 더욱 효율적입니다. \n", - "\n", - "또한 변수를 사용하는 것은 코드를 읽는 독자에게 상태가 변경될 수 있다는 것을 알려주는 손쉬운 방법입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BMiFcDzE7Qu3" - }, - "source": [ - "## 예: 선형 모델 훈련\n", - "\n", - "지금까지 간단한 모델을 구축하고 학습시키기 위해 ---`Tensor`, `GradientTape`, `Variable` --- 와 같은 몇가지 개념을 설명했습니다. 이는 일반적으로 다음의 과정을 포함합니다.\n", - "\n", - "1. 모델 정의\n", - "2. 손실 함수 정의\n", - "3. 훈련 데이터 가져오기\n", - "4. 훈련 데이터에서 실행, 데이터에 최적화하기 위해 \"옵티마이저(optimizer)\"를 사용한 변수 조정\n", - "\n", - "이번 튜토리얼에서는 선형 모델의 간단한 예제를 살펴보겠습니다:\n", - "`f(x) = x * W + b`, 모델은 `W` 와 `b` 두 변수를 가지고 있는 선형모델이며, 잘 학습된 모델이 `W = 3.0` and `b = 2.0`의 값을 갖도록 합성 데이터를 만들겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFzH64Jn9PIm" - }, - "source": [ - "### 모델 정의\n", - "\n", - "변수와 연산을 캡슐화하기 위한 간단한 클래스를 정의해봅시다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_WRu7Pze7wk8" - }, - "outputs": [], - "source": [ - "class Model(object):\n", - " def __init__(self):\n", - " # 변수를 (5.0, 0.0)으로 초기화 합니다.\n", - " # 실제로는 임의의 값으로 초기화 되어야합니다.\n", - " self.W = tf.Variable(5.0)\n", - " self.b = tf.Variable(0.0)\n", - "\n", - " def __call__(self, x):\n", - " return self.W * x + self.b\n", - "\n", - "model = Model()\n", - "\n", - "assert model(3.0).numpy() == 15.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xa6j_yXa-j79" - }, - "source": [ - "### 손실 함수 정의\n", - "\n", - "손실 함수는 주어진 입력에 대한 모델의 출력이 원하는 출력과 얼마나 잘 일치하는지를 측정합니다. 평균 제곱 오차(mean square error)를 적용한 손실 함수를 사용하겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Y0ysUFGY924U" - }, - "outputs": [], - "source": [ - "def loss(predicted_y, desired_y):\n", - " return tf.reduce_mean(tf.square(predicted_y - desired_y))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qutT_fkl_CBc" - }, - "source": [ - "### 훈련 데이터 가져오기\n", - "\n", - "약간의 잡음과 훈련 데이터를 합칩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gxPTb-kt_N5m" - }, - "outputs": [], - "source": [ - "TRUE_W = 3.0\n", - "TRUE_b = 2.0\n", - "NUM_EXAMPLES = 1000\n", - "\n", - "inputs = tf.random.normal(shape=[NUM_EXAMPLES])\n", - "noise = tf.random.normal(shape=[NUM_EXAMPLES])\n", - "outputs = inputs * TRUE_W + TRUE_b + noise" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-50nq-wPBsAW" - }, - "source": [ - "모델을 훈련시키기 전에, 모델의 현재 상태를 시각화합시다. 모델의 예측을 빨간색으로, 훈련 데이터를 파란색으로 구성합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_eb83LtrB4nt" - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(inputs, outputs, c='b')\n", - "plt.scatter(inputs, model(inputs), c='r')\n", - "plt.show()\n", - "\n", - "print('현재 손실: '),\n", - "print(loss(model(inputs), outputs).numpy())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sSDP-yeq_4jE" - }, - "source": [ - "### 훈련 루프 정의\n", - "\n", - "이제 네트워크와 훈련 데이터가 준비되었습니다. 모델의 변수(`W` 와 `b`)를 업데이트하기 위해 훈련 데이터를 사용하여 훈련시켜 보죠. 그리고 [경사 하강법(gradient descent)](https://en.wikipedia.org/wiki/Gradient_descent)을 사용하여 손실을 감소시킵니다. 경사 하강법에는 여러가지 방법이 있으며, `tf.train.Optimizer` 에 구현되어있습니다. 이러한 구현을 사용하는 것을 강력히 추천드립니다. 그러나 이번 튜토리얼에서는 기본적인 방법을 사용하겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MBIACgdnA55X" - }, - "outputs": [], - "source": [ - "def train(model, inputs, outputs, learning_rate):\n", - " with tf.GradientTape() as t:\n", - " current_loss = loss(model(inputs), outputs)\n", - " dW, db = t.gradient(current_loss, [model.W, model.b])\n", - " model.W.assign_sub(learning_rate * dW)\n", - " model.b.assign_sub(learning_rate * db)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RwWPaJryD2aN" - }, - "source": [ - "마지막으로, 훈련 데이터를 반복적으로 실행하고, `W` 와 `b`의 변화 과정을 확인합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "XdfkR223D9dW" - }, - "outputs": [], - "source": [ - "model = Model()\n", - "\n", - "# 도식화를 위해 W값과 b값의 변화를 저장합니다.\n", - "Ws, bs = [], []\n", - "epochs = range(10)\n", - "for epoch in epochs:\n", - " Ws.append(model.W.numpy())\n", - " bs.append(model.b.numpy())\n", - " current_loss = loss(model(inputs), outputs)\n", - "\n", - " train(model, inputs, outputs, learning_rate=0.1)\n", - " print('에포크 %2d: W=%1.2f b=%1.2f, 손실=%2.5f' %\n", - " (epoch, Ws[-1], bs[-1], current_loss))\n", - "\n", - "# 저장된 값들을 도식화합니다.\n", - "plt.plot(epochs, Ws, 'r',\n", - " epochs, bs, 'b')\n", - "plt.plot([TRUE_W] * len(epochs), 'r--',\n", - " [TRUE_b] * len(epochs), 'b--')\n", - "plt.legend(['W', 'b', 'true W', 'true_b'])\n", - "plt.show()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vPnIVuaSJwWz" - }, - "source": [ - "## 다음 단계\n", - "\n", - "이번 튜토리얼에서는 변수를 다루었으며, 지금까지 논의된 텐서플로의 기본 요소를 사용하여 간단한 선형 모델을 구축하고 훈련시켰습니다.\n", - "\n", - "이론적으로, 텐서플로를 머신러닝 연구에 사용하기 위해 알아야 할 것이 매우 많습니다. 실제로 신경망에 있어 `tf.keras`와 같은 고수준 API는 고수준 구성 요소(\"층\"으로 불리는)를 제공하고, 저장 및 복원을 위한 유틸리티, 손실 함수 모음, 최적화 전략 모음 등을 제공하기 때문에 더욱 편리합니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EyUG6XGSgwhk" - }, - "outputs": [], - "source": [ - "" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "custom_training.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/customization/custom_training_walkthrough.ipynb b/site/ko/tutorials/customization/custom_training_walkthrough.ipynb deleted file mode 100644 index c2de2b5d136..00000000000 --- a/site/ko/tutorials/customization/custom_training_walkthrough.ipynb +++ /dev/null @@ -1,1168 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rwxGnsA92emp" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "CPII1rGR2rF9" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtEZ1pCPn--z" - }, - "source": [ - "# 사용자 정의 학습: 자세히 둘러보기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GV1F7tVTN3Dn" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/customization/custom_training_walkthrough\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eTensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/customization/custom_training_walkthrough.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/customization/custom_training_walkthrough.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e \n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/ko/tutorials/customization/custom_training_walkthrough.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003e노트북 다운로드\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iFwQHS09CNxU" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 지원하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LDrzLFXE8T1l" - }, - "source": [ - "이번 튜토리얼은 붓꽃의 품종을 *분류*하기 위한 머신러닝 모델을 구축할 것입니다. 다음을 위해 텐서플로를 사용합니다.\n", - "1. 모델 구축\n", - "2. 모델 훈련\n", - "3. 모델을 사용한 예측\n", - "\n", - "## 텐서플로 프로그래밍\n", - "\n", - "이번 튜토리얼에서는 다음과 같은 고수준 텐서플로의 개념을 사용합니다.\n", - "\n", - "* 텐서플로의 [즉시 실행(eager execution)](https://www.tensorflow.org/guide/eager) 기본 개발 환경 사용,\n", - "* [데이터셋 API](https://www.tensorflow.org/guide/datasets)를 활용한 데이터 가져오기,\n", - "* [케라스 API](https://keras.io/getting-started/sequential-model-guide/)를 활용한 모델과 층(layer) 구축 .\n", - "\n", - "이번 튜토리얼은 다른 텐서플로 프로그램과 유사하게 구성되어있습니다.\n", - "\n", - "1. 데이터 가져오기 및 분석.\n", - "2. 모델 타입 선정.\n", - "3. 모델 훈련.\n", - "4. 모델 효과 평가.\n", - "5. 훈련된 모델을 사용한 예측." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yNr7H-AIoLOR" - }, - "source": [ - "## 프로그램 설정" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1J3AuPBT9gyR" - }, - "source": [ - "### 라이브러리 임포트\n", - "\n", - "텐서플로와 필요한 파이썬 모듈을 임포트합니다. 텐서플로는 연산이 나중에 실행되는 [계산 그래프(computational graph)](https://www.tensorflow.org/guide/graphs)를 만드는 대신에 연산을 즉시 평가하고 구체적인 값을 반환하는 [즉시 실행](https://www.tensorflow.org/guide/eager)을 사용합니다. 만약 파이썬 대화형 창이나 상호작용 콘솔을 사용하면 더욱 익숙할 겁니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jElLULrDhQZR" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bfV2Dai0Ow2o" - }, - "outputs": [], - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "g4Wzg69bnwK2" - }, - "outputs": [], - "source": [ - "print(\"텐서플로 버전: {}\".format(tf.__version__))\n", - "print(\"즉시 실행: {}\".format(tf.executing_eagerly()))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zx7wc0LuuxaJ" - }, - "source": [ - "## 붓꽃 분류 문제\n", - "\n", - "당신이 식물학자라고 상상하고, 주어진 붓꽃을 자동적으로 분류하는 방법을 찾고 있다고 가정합시다. 머신러닝은 통계적으로 꽃을 분류할 수 있는 다양한 알고리즘을 제공합니다. 예를 들어, 정교한 머신러닝 프로그램은 사진을 통해 꽃을 분류할 수 있습니다. 이번 튜토리얼의 목적은 좀 더 겸손하게, 측정된 [꽃받침](https://en.wikipedia.org/wiki/Sepal)과 [꽃잎](https://en.wikipedia.org/wiki/Petal)의 길이와 폭을 토대로 붓꽃을 분류하는 것입니다.\n", - "\n", - "이 붓꽃은 약 300종입니다. 하지만 이번 튜토리얼에서는 오직 3가지 품종을 기준으로 분류할 것입니다. \n", - "\n", - "* Iris setosa\n", - "* Iris virginica\n", - "* Iris versicolor\n", - "\n", - "\u003ctable\u003e\n", - " \u003ctr\u003e\u003ctd\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/iris_three_species.jpg\"\n", - " alt=\"Petal geometry compared for three iris species: Iris setosa, Iris virginica, and Iris versicolor\"\u003e\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\"\u003e\n", - " \u003cb\u003e그림 1.\u003c/b\u003e \u003ca href=\"https://commons.wikimedia.org/w/index.php?curid=170298\"\u003eIris setosa\u003c/a\u003e (by \u003ca href=\"https://commons.wikimedia.org/wiki/User:Radomil\"\u003eRadomil\u003c/a\u003e, CC BY-SA 3.0), \u003ca href=\"https://commons.wikimedia.org/w/index.php?curid=248095\"\u003eIris versicolor\u003c/a\u003e, (by \u003ca href=\"https://commons.wikimedia.org/wiki/User:Dlanglois\"\u003eDlanglois\u003c/a\u003e, CC BY-SA 3.0), and \u003ca href=\"https://www.flickr.com/photos/33397993@N05/3352169862\"\u003eIris virginica\u003c/a\u003e (by \u003ca href=\"https://www.flickr.com/photos/33397993@N05\"\u003eFrank Mayfield\u003c/a\u003e, CC BY-SA 2.0).\u003cbr/\u003e\u0026nbsp;\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e\n", - "\n", - "다행히도 다른 사람들이 먼저 꽃받침과 꽃잎의 길이와 폭이 측정된 [120개의 붓꽃 데이터](https://en.wikipedia.org/wiki/Iris_flower_data_set)를 만들어 놓았습니다. 이것은 머신러닝 분류 문제에 있어 초보자에게 유명한 고전 데이터셋입니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Px6KAg0Jowz" - }, - "source": [ - "## 훈련 데이터 가져오기 및 파싱\n", - "\n", - "데이터를 불러오고 파이썬 프로그램이 사용할 수 있는 구조로 전환합니다.\n", - "\n", - "### 데이터셋 다운로드\n", - "\n", - "[tf.keras.utils.get_file](https://www.tensorflow.org/api_docs/python/tf/keras/utils/get_file) 함수를 사용하여 데이터셋을 다운로드합니다. 이 함수는 다운로드된 파일의 경로를 반환합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "J6c7uEU9rjRM" - }, - "outputs": [], - "source": [ - "train_dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv\"\n", - "\n", - "train_dataset_fp = tf.keras.utils.get_file(fname=os.path.basename(train_dataset_url),\n", - " origin=train_dataset_url)\n", - "\n", - "print(\"데이터셋이 복사된 위치: {}\".format(train_dataset_fp))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qnX1-aLors4S" - }, - "source": [ - "### 데이터 탐색\n", - "\n", - "이 데이터셋(`iris_training.csv`)은 콤마 ','로 구분된 CSV 파일입니다. `head -n5` 명령을 사용하여 처음 5개 항목을 확인합니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FQvb_JYdrpPm" - }, - "outputs": [], - "source": [ - "!head -n5 {train_dataset_fp}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kQhzD6P-uBoq" - }, - "source": [ - "처음 5개의 데이터로부터 다음을 주목하세요.\n", - "\n", - "1. 첫 번째 줄은 다음과 같은 정보를 포함하고 있는 헤더(header)입니다. \n", - " * 총 120개의 샘플이 있으며, 각 샘플들은 4개의 특성(feature), 3개의 레이블(label)을 가지고 있습니다.\n", - "2. 후속행은 데이터 레코드입니다. 한 줄당 한 개의 *[샘플](https://developers.google.com/machine-learning/glossary/#example)* 입니다.\n", - " * 처음 4개의 필드는 *[특성](https://developers.google.com/machine-learning/glossary/#feature)* 입니다.: 이것들은 샘플의 특징을 나타냅니다. 이 필드들는 붓꽃의 측정값을 부동소수점으로 나타냅니다.\n", - " * 마지막 컬럼(column)은 *[레이블(label)](https://developers.google.com/machine-learning/glossary/#label)* 입니다.: 레이블은 예측하고자 하는 값을 나타냅니다. 이 데이터셋에서는 꽃의 이름과 관련된 정수값 0, 1, 2를 나타냅니다.\n", - "\n", - "코드로 표현하면 다음과 같습니다.:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9Edhevw7exl6" - }, - "outputs": [], - "source": [ - "# CSV 파일안에서 컬럼의 순서\n", - "column_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']\n", - "\n", - "feature_names = column_names[:-1]\n", - "label_name = column_names[-1]\n", - "\n", - "print(\"특성: {}\".format(feature_names))\n", - "print(\"레이블: {}\".format(label_name))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCtwLoJhhDNc" - }, - "source": [ - "각각의 레이블은 \"setosa\"와 같은 문자형 이름과 연관되어있습니다. 하지만 머신러닝은 전형적으로 숫자형 값에 의존합니다. 레이블을 다음과 같이 매핑(mapping) 합니다. \n", - "\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica\n", - "\n", - "특성과 레이블에 관한 더 많은 정보를 위해서는 [머신러닝 특강의 전문용어 부분](https://developers.google.com/machine-learning/crash-course/framing/ml-terminology)을 참조하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sVNlJlUOhkoX" - }, - "outputs": [], - "source": [ - "class_names = ['Iris setosa', 'Iris versicolor', 'Iris virginica']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dqPkQExM2Pwt" - }, - "source": [ - "### `tf.data.Dataset` 생성\n", - "\n", - "텐서플로의 [데이터셋 API](https://www.tensorflow.org/guide/datasets)는 데이터를 적재할 때 발생하는 다양한 경우를 다룰 수 있습니다. 이는 훈련에 필요한 형태로 데이터를 읽고 변환하는 고수준 API입니다. 더 많은 정보를 위해서는 [데이터셋 빠른 실행 가이드](https://www.tensorflow.org/get_started/datasets_quickstart)를 참조하세요. \n", - "\n", - "\n", - "데이터셋이 CSV 파일이므로, 적절한 형태로 데이터를 구분하기 위해 [make_csv_dataset](https://www.tensorflow.org/api_docs/python/tf/data/experimental/make_csv_dataset) 함수를 사용하겠습니다. 이 함수는 훈련 모델을 위한 데이터를 생성하므로, 초기값은 셔플(`shuffle=True, shuffle_buffer_size=10000`)과 무한 반복(`num_epochs=None`)으로 설정되어있습니다. 또한 [배치 사이즈(batch_size)](https://developers.google.com/machine-learning/glossary/#batch_size)를 설정해줍니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WsxHnz1ebJ2S" - }, - "outputs": [], - "source": [ - "batch_size = 32\n", - "\n", - "train_dataset = tf.data.experimental.make_csv_dataset(\n", - " train_dataset_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name=label_name,\n", - " num_epochs=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gB_RSn62c-3G" - }, - "source": [ - "`make_csv_dataset` 함수는 `(features, label)` 쌍으로 구성된 `tf.data.Dataset`을 반환합니다. `features`는 딕셔너리 객체인: `{'feature_name': value}`로 주어집니다. 이 데이터셋은 반복가능합니다. 다음은 특성(feature)을 살펴봅시다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iDuG94H-C122" - }, - "outputs": [], - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "print(features)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E63mArnQaAGz" - }, - "source": [ - "유사한 특성의 값은 같이 그룹 되어있거나, *배치* 돼있다는 사실에 주목하세요. 각 샘플 행의 필드는 해당 특성 배열에 추가됩니다. `batch_size`를 조절하여 이 특성 배열에 저장된 샘플의 수를 설정하세요.\n", - "\n", - "또한 배치(batch)로부터 약간의 특성을 도식화하여 군집돼있는 데이터를 확인할 수 있습니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "me5Wn-9FcyyO" - }, - "outputs": [], - "source": [ - "plt.scatter(features['petal_length'],\n", - " features['sepal_length'],\n", - " c=labels,\n", - " cmap='viridis')\n", - "\n", - "plt.xlabel(\"Petal length\")\n", - "plt.ylabel(\"Sepal length\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YlxpSyHlhT6M" - }, - "source": [ - "모델 구축 단계를 단순화하기 위해, 특성 딕셔너리를 `(batch_size, num_features)`의 형태를 가지는 단일 배열로 다시 구성하는 함수를 생성합니다.\n", - "\n", - "이 함수는 텐서의 리스트(list)로부터 값을 취하고 특정한 차원으로 결합된 텐서를 생성하는 [tf.stack](https://www.tensorflow.org/api_docs/python/tf/stack) 메서드(method)를 사용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jm932WINcaGU" - }, - "outputs": [], - "source": [ - "def pack_features_vector(features, labels):\n", - " \"\"\"특성들을 단일 배열로 묶습니다.\"\"\"\n", - " features = tf.stack(list(features.values()), axis=1)\n", - " return features, labels" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "V1Vuph_eDl8x" - }, - "source": [ - "그 후 각 `(features,label)`쌍의 특성을 훈련 데이터 세트에 쌓기위해 [tf.data.Dataset.map](https://www.tensorflow.org/api_docs/python/tf/data/dataset/map) 메서드를 사용합니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ZbDkzGZIkpXf" - }, - "outputs": [], - "source": [ - "train_dataset = train_dataset.map(pack_features_vector)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NLy0Q1xCldVO" - }, - "source": [ - "데이터셋의 특성 요소는 이제 형태가 `(batch_size, num_features)`인 배열입니다. 첫 5개행의 샘플을 살펴봅시다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kex9ibEek6Tr" - }, - "outputs": [], - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "print(features[:5])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LsaVrtNM3Tx5" - }, - "source": [ - "## 모델 타입 선정\n", - "\n", - "### 왜 모델을 사용해야하는가?\n", - "\n", - " *[모델](https://developers.google.com/machine-learning/crash-course/glossary#model)* 은 특성(feature)과 레이블(label) 과의 관계입니다. 붓꽃 분류 문제에서 모델은 측정된 꽃받침과 꽃잎 사이의 관계를 정의하고 붓꽃의 품종을 예측합니다. 몇 가지 간단한 모델은 몇 줄의 대수학으로 표현할 수 있으나, 복잡한 머신러닝 모델은 요약하기 힘든 굉장히 많은 수의 매개변수를 가지고 있습니다.\n", - "\n", - "머신러닝을 사용하지 않고 4가지의 특성 사이의 관계를 결정하고 붓꽃을 품종을 예측하실 수 있나요? 즉, 특정 품종의 꽃받침과 꽃잎과의 관계를 정의할 수 있을 정도로 데이터셋을 분석했다면, 전통적인 프로그래밍 기술(예를 들어 굉장히 많은 조건문)을 사용하여 모델은 만들 수 있으신가요? 더 복잡한 데이터셋에서 이는 불가능에 가까울 수 있습니다. 잘 구성된 머신러닝은 사용자를 위한 모델을 결정합니다. 만약 충분히 좋은 샘플을 잘 구성된 머신러닝 모델에 제공한다면, 프로그램은 사용자를 위한 특성 간의 관계를 이해하고 제공합니다. \n", - "\n", - "### 모델 선정\n", - "\n", - "이제 학습을 위한 모델의 종류를 선정해야합니다. 여러 종류의 모델이 있고, 이를 선택하는 것은 많은 경험이 필요합니다. 이번 튜토리얼에서는 붓꽃 분류 문제를 해결하기위해 *[신경망(neural network)](https://developers.google.com/machine-learning/glossary/#neural_network)* 모델을 사용하겠습니다. 신경망 모델은 특성과 레이블 사이의 복잡한 관계를 찾을 수 있습니다. 신경망은 하나 또는 그 이상의 *[은닉층(hidden layer)](https://developers.google.com/machine-learning/glossary/#hidden_layer)*으로 구성된 그래프입니다. 각각의 은닉층은 하나 이상의 *[뉴런(neuron)](https://developers.google.com/machine-learning/glossary/#neuron)*으로 구성되어있습니다. 몇가지 신경망의 범주가 있으며, 이번 튜토리얼에서는 *[밀집(dense) 또는 완전 연결 신경망(fully-connected neural network)](https://developers.google.com/machine-learning/glossary/#fully_connected_layer)*를 사용합니다: 완전 연결 신경망(fully-connected neural network)은 하나의 뉴런에 이전층의 *모든* 뉴런의 입력을 받는 신경망입니다. 예를 들어, 그림 2는 입력층, 2개의 은닉층, 그리고 출력층으로 구성된 완전 연결 신경망입니다. \n", - "\n", - "\u003ctable\u003e\n", - " \u003ctr\u003e\u003ctd\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/custom_estimators/full_network.png\"\n", - " alt=\"A diagram of the network architecture: Inputs, 2 hidden layers, and outputs\"\u003e\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\"\u003e\n", - " \u003cb\u003e그림 2.\u003c/b\u003e 특성, 은닉층, 예측으로 구성된 신경망\u003cbr/\u003e\u0026nbsp;\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e\n", - "\n", - "그림 2의 모델이 훈련된 다음 레이블 되어있지 않은 데이터를 제공했을때, 모델은 주어진 데이터의 3가지(주어진 레이블의 개수) 예측을 출력합니다. 이러한 예측은 *[추론(inference)](https://developers.google.com/machine-learning/crash-course/glossary#inference)* 이라고 불립니다. 이 샘플에서 출력의 합은 1.0입니다. 그림 2에서 예측은 *Iris setosa* `0.02`, *Iris versicolor* `0.95`, *Iris virginica*에 `0.03`로 주어집니다. 이는 모델이 95%의 확률로 주어진 데이터를 *Iris versicolor*로 예측한다는 것을 의미합니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W23DIMVPQEBt" - }, - "source": [ - "### 케라스를 사용한 모델 생성\n", - "\n", - "텐서플로의 [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras) API는 모델과 층을 생성하기 위한 풍부한 라이브러리를 제공합니다. 케라스가 구성 요소를 연결하기 위한 복잡함을 모두 처리해 주기 때문에 모델을 구축하고 실험하는 것이 쉽습니다.\n", - "\n", - "[tf.keras.Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential)은 여러 층을 연이어 쌓은 모델입니다. 이 구조는 층의 인스턴스를 취하며, 아래의 경우 각 층당 10개의 노드(node)를 가지는 2개의 [완전 연결((Dense) 층](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense)과 3개의 예측(레이블의 수) 노드를 가지는 출력 층으로 구성되어있습니다. 첫 번째 층의 `input_shape` 매개변수는 데이터셋의 특성의 수와 관계있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2fZ6oL2ig3ZK" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(4,)), # 입력의 형태가 필요합니다.\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu),\n", - " tf.keras.layers.Dense(3)\n", - "])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FHcbEzMpxbHL" - }, - "source": [ - "*[활성화 함수(activation function)](https://developers.google.com/machine-learning/crash-course/glossary#activation_function)*는 각 층에서 출력의 형태를 결정합니다. 이러한 비선형성은 중요하며, 활성화 함수가 없는 모델은 하나의 층과 동일하다고 생각할 수 있습니다. 사용 가능한 [활성화 함수](https://www.tensorflow.org/api_docs/python/tf/keras/activations)는 많지만, [ReLU](https://developers.google.com/machine-learning/crash-course/glossary#ReLU)가 은닉층에 주로 사용됩니다. \n", - "\n", - "이상적인 은닉층과 뉴런의 개수는 문제와 데이터셋에 의해 좌우됩니다. 머신러닝의 여러 측면과 마찬가지로, 최적의 신경망 타입을 결정하는 것은 많은 경험과 지식이 필요합니다. 경험을 토대로 보면 은닉층과 뉴런의 증가는 전형적으로 강력한 모델을 생성하므로, 모델을 효과적으로 훈련시키기 위해서 더 많은 데이터를 필요로 합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2wFKnhWCpDSS" - }, - "source": [ - "### 모델 사용\n", - "\n", - "이 모델이 특성의 배치에 대해 수행하는 작업을 간단히 살펴봅시다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xe6SQ5NrpB-I" - }, - "outputs": [], - "source": [ - "predictions = model(features)\n", - "predictions[:5]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wxyXOhwVr5S3" - }, - "source": [ - "각 샘플은 각 클래스에 대한 [로짓(logit)](https://developers.google.com/machine-learning/crash-course/glossary#logits)을 반환합니다. \n", - "\n", - "이 로짓(logit)을 각 클래스에 대한 확률로 변환하기 위하서 [소프트맥스(softmax)](https://developers.google.com/machine-learning/crash-course/glossary#softmax) 함수를 사용하겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_tRwHZmTNTX2" - }, - "outputs": [], - "source": [ - "tf.nn.softmax(predictions[:5])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uRZmchElo481" - }, - "source": [ - "`tf.argmax`는 예측된 값 중 가장 큰 확률(원하는 클래스)을 반환합니다. 하지만 모델이 아직 훈련되지 않았으므로 이는 좋은 예측이 아닙니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-Jzm_GoErz8B" - }, - "outputs": [], - "source": [ - "print(\" 예측: {}\".format(tf.argmax(predictions, axis=1)))\n", - "print(\"레이블: {}\".format(labels))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vzq2E5J2QMtw" - }, - "source": [ - "## 모델 훈련하기\n", - "\n", - "*[훈련 단계](https://developers.google.com/machine-learning/crash-course/glossary#training)* 는 모델이 점진적으로 최적화되거나 데이터셋을 학습하는 머신러닝의 과정입니다. 훈련의 목적은 미지의 데이터를 예측하기 위해, 훈련 데이터 세트의 구조에 대해서 충분히 학습하는 것입니다. 만약 모델이 훈련 데이터 세트에 대해서 과하게 학습된다면 오직 훈련 데이터 세트에 대해서 작동할 것이며, 일반화되기 힘들 것입니다. 이러한 문제를 *[과대적합(overfitting)](https://developers.google.com/machine-learning/crash-course/glossary#overfitting)* 이라고 합니다. 이는 마치 문제를 이해하고 해결한다기보다는 답을 기억하는 것이라고 생각할 수 있습니다. \n", - "\n", - "붓꽃 분류 문제는 *[지도 학습(supervised machine learning)](https://developers.google.com/machine-learning/glossary/#supervised_machine_learning)* 의 예시 중 하나입니다.: 지도학습은 모델이 레이블을 포함한 훈련 데이터로부터 학습됩니다. *[비지도 학습(unsupervised machine learning)](https://developers.google.com/machine-learning/glossary/#unsupervised_machine_learning)* 에서는 훈련 데이터가 레이블을 포함하고 있지 않습니다. 대신에 모델은 특성 간의 패턴을 찾습니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RaKp8aEjKX6B" - }, - "source": [ - "### 손실 함수와 그래디언트 함수 정의하기\n", - "\n", - "훈련과 평가단계에서 모델의 *[손실(loss)](https://developers.google.com/machine-learning/crash-course/glossary#loss)*을 계산해야 합니다. 손실은 모델의 예측이 원하는 레이블과 얼마나 일치하는지, 또한 모델이 잘 작동하는지에 대한 척도로 사용됩니다. 이 값을 최소화하고, 최적화 해야합니다.\n", - "\n", - "모델의 손실은 `tf.keras.losses.categorical_crossentropy` 함수를 사용해 계산할 것입니다. 이 함수는 모델의 클래스(레이블)과 예측된 값(로짓)을 입력받아 샘플의 평균 손실을 반환합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QOsi6b-1CXIn" - }, - "outputs": [], - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tMAT4DcMPwI-" - }, - "outputs": [], - "source": [ - "def loss(model, x, y):\n", - " y_ = model(x)\n", - "\n", - " return loss_object(y_true=y, y_pred=y_)\n", - "\n", - "\n", - "l = loss(model, features, labels)\n", - "print(\"손실 테스트: {}\".format(l))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3IcPqA24QM6B" - }, - "source": [ - "모델을 최적화하기 위해 사용되는 *[그래디언트(gradient)](https://developers.google.com/machine-learning/crash-course/glossary#gradient)*를 계산하기 위해 [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) 컨텍스트를 사용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "x57HcKWhKkei" - }, - "outputs": [], - "source": [ - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return loss_value, tape.gradient(loss_value, model.trainable_variables)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lOxFimtlKruu" - }, - "source": [ - "### 옵티마이저 생성 \n", - "\n", - "*[옵티마이저(optimizer)](https://developers.google.com/machine-learning/crash-course/glossary#optimizer)*는 `손실` 함수를 최소화하기 위해 계산된 그래디언트를 모델의 변수에 적용합니다. 손실 함수를 구부러진 곡선의 표면(그림 3)으로 생각할 수 있으며, 이 함수의 최저점을 찾고자 합니다. 그래디언트는 가장 가파른 상승 방향을 가리키며 따라서 반대 방향으로 이동하는 여행을 합니다. 각 배치마다의 손실과 기울기를 반복적으로 계산하여 훈련과정 동안 모델을 조정합니다. 점진적으로, 모델은 손실을 최소화하기 위해 가중치(weight)와 편향(bias)의 최적의 조합을 찾아냅니다. 손실이 낮을수록 더 좋은 모델의 예측을 기대할 수 있습니다.\n", - "\n", - "\u003ctable\u003e\n", - " \u003ctr\u003e\u003ctd\u003e\n", - " \u003cimg src=\"https://cs231n.github.io/assets/nn3/opt1.gif\" width=\"70%\"\n", - " alt=\"Optimization algorithms visualized over time in 3D space.\"\u003e\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\"\u003e\n", - " \u003cb\u003e 그림 3.\u003c/b\u003e 3차원 공간에 대한 최적화 알고리즘 시각화.\u003cbr/\u003e(Source: \u003ca href=\"http://cs231n.github.io/neural-networks-3/\"\u003eStanford class CS231n\u003c/a\u003e, MIT License, Image credit: \u003ca href=\"https://twitter.com/alecrad\"\u003eAlec Radford\u003c/a\u003e)\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e\n", - "\n", - "텐서플로는 훈련을 위해 사용 가능한 여러종류의 [최적화 알고리즘](https://www.tensorflow.org/api_guides/python/train)을 가지고 있습니다. 이번 모델에서는 *[확률적 경사 하강법(stochastic gradient descent, SGD)](https://developers.google.com/machine-learning/crash-course/glossary#gradient_descent)* 을 구현한 [tf.train.GradientDescentOptimizer](https://www.tensorflow.org/api_docs/python/tf/train/GradientDescentOptimizer)를 사용하겠습니다. `learning_rate`은 경사하강 과정의 크기를 나타내는 매개변수이며, 더 나은 결과를 위해 조절가능한 *하이퍼파라미터(hyperparameter)* 입니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XkUd6UiZa_dF" - }, - "source": [ - "옵티마이저(optimizer)를 설정합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8xxi2NNGKwG_" - }, - "outputs": [], - "source": [ - "optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pJVRZ0hP52ZB" - }, - "source": [ - "이를 사용해 한 번의 최적화 단계를 계산하기 위해 사용합니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "rxRNTFVe56RG" - }, - "outputs": [], - "source": [ - "loss_value, grads = grad(model, features, labels)\n", - "\n", - "print(\"단계: {}, 초기 손실: {}\".format(optimizer.iterations.numpy(),\n", - " loss_value.numpy()))\n", - "\n", - "optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - "print(\"단계: {}, 손실: {}\".format(optimizer.iterations.numpy(),\n", - " loss(model, features, labels).numpy()))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Y2VSELvwAvW" - }, - "source": [ - "### 훈련 루프\n", - "\n", - "모든 사항이 갖춰졌으므로 모델을 훈련할 준비가 되었습니다! 훈련 루프는 더 좋은 예측을 위해 데이터셋을 모델로 제공합니다. 다음의 코드 블럭은 아래의 훈련 단계를 작성한 것입니다. \n", - "\n", - "1. 각 *에포크(epoch)* 반복. 에포크는 데이터셋을 통과시키는 횟수입니다. \n", - "2. 에포크 내에서, *특성* (`x`)와 *레이블* (`y`)가 포함된 훈련 데이터 세트에 있는 샘플을 반복합니다.\n", - "3. 샘플의 특성을 사용하여 결과를 예측 하고 레이블과 비교합니다. 예측의 부정확도를 측정하고 모델의 손실과 그래디언트를 계산하기 위해 사용합니다. \n", - "4. 모델의 변수를 업데이트하기 위해 `옵티마이저`를 사용합니다. \n", - "5. 시각화를 위해 몇가지 값들을 저장합니다.\n", - "6. 각 에포크를 반복합니다.\n", - "\n", - "`num_epochs` 변수는 데이터셋의 반복 횟수입니다. 직관과는 반대로, 모델을 길게 학습하는 것이 더 나은 모델이 될 것이라고 보장하지 못합니다. `num_epochs`는 조정가능한 *[하이퍼파라미터(hyperparameter)](https://developers.google.com/machine-learning/glossary/#hyperparameter)* 입니다. 적절한 횟수를 선택하는 것은 많은 경험과 직관을 필요로 합니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AIgulGRUhpto" - }, - "outputs": [], - "source": [ - "## 노트: 이 셀을 다시 실행하면 동일한 모델의 변수가 사용됩니다.\n", - "\n", - "# 도식화를 위해 결과를 저장합니다.\n", - "train_loss_results = []\n", - "train_accuracy_results = []\n", - "\n", - "num_epochs = 201\n", - "\n", - "for epoch in range(num_epochs):\n", - " epoch_loss_avg = tf.keras.metrics.Mean()\n", - " epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n", - "\n", - " # 훈련 루프 - 32개의 배치를 사용합니다.\n", - " for x, y in train_dataset:\n", - " # 모델을 최적화합니다.\n", - " loss_value, grads = grad(model, x, y)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " # 진행 상황을 추적합니다.\n", - " epoch_loss_avg(loss_value) # 현재 배치 손실을 추가합니다.\n", - " # 예측된 레이블과 실제 레이블 비교합니다.\n", - " epoch_accuracy(y, model(x))\n", - "\n", - " # epoch 종료\n", - " train_loss_results.append(epoch_loss_avg.result())\n", - " train_accuracy_results.append(epoch_accuracy.result())\n", - "\n", - " if epoch % 50 == 0:\n", - " print(\"에포크 {:03d}: 손실: {:.3f}, 정확도: {:.3%}\".format(epoch,\n", - " epoch_loss_avg.result(),\n", - " epoch_accuracy.result()))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2FQHVUnm_rjw" - }, - "source": [ - "### 시간에 따른 손실함수 시각화" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j3wdbmtLVTyr" - }, - "source": [ - "모델의 훈련 과정을 출력하는 것도 도움이 되지만, 훈련 과정을 직접 보는 것이 더 도움이 되곤합니다. [텐서보드(tensorboard)](https://www.tensorflow.org/guide/summaries_and_tensorboard)는 텐서플로에 패키지 되어있는 굉장히 유용한 시각화 툴입니다. 하지만 `matplotlib` 모듈을 사용하여 일반적인 도표를 출력할 수 있습니다.\n", - "\n", - "이 도표를 해석하는 것은 여러 경험이 필요하지만, 결국 모델을 최적화하기 위해 *손실* 이 내려가고 *정확도* 가 올라가는 것을 원합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "agjvNd2iUGFn" - }, - "outputs": [], - "source": [ - "fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))\n", - "fig.suptitle('훈련 지표')\n", - "\n", - "axes[0].set_ylabel(\"손실\", fontsize=14)\n", - "axes[0].plot(train_loss_results)\n", - "\n", - "axes[1].set_ylabel(\"정확도\", fontsize=14)\n", - "axes[1].set_xlabel(\"에포크\", fontsize=14)\n", - "axes[1].plot(train_accuracy_results)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zg8GoMZhLpGH" - }, - "source": [ - "## 모델 유효성 평가\n", - "\n", - "이제 모델은 훈련되었습니다. 모델의 성능에 대한 몇가지 통계를 얻을 수 있습니다. \n", - "\n", - "*평가(Evaluating)*는 모델이 예측을 얼마나 효과적으로 수행하는지 결정하는 것을 의미합니다. 붓꽃 분류 모델의 유효성을 결정하기 위해, 몇가지 꽃잎과 꽃받침 데이터를 통과시키고 어떠한 품종을 예측하는지 확인합니다. 그 후 실제 품종과 비교합니다. 예를 들어, 절반의 데이터를 올바르게 예측한 모델의 *[정확도](https://developers.google.com/machine-learning/glossary/#accuracy)* 는 `0.5`입니다. 그림 4는 조금 더 효과적인 모델입니다. 5개의 예측 중 4개를 올바르게 예측하여 80% 정확도를 냅니다.\n", - "\n", - "\u003ctable cellpadding=\"8\" border=\"0\"\u003e\n", - " \u003ccolgroup\u003e\n", - " \u003ccol span=\"4\" \u003e\n", - " \u003ccol span=\"1\" bgcolor=\"lightblue\"\u003e\n", - " \u003ccol span=\"1\" bgcolor=\"lightgreen\"\u003e\n", - " \u003c/colgroup\u003e\n", - " \u003ctr bgcolor=\"lightgray\"\u003e\n", - " \u003cth colspan=\"4\"\u003e샘플 특성\u003c/th\u003e\n", - " \u003cth colspan=\"1\"\u003e레이블\u003c/th\u003e\n", - " \u003cth colspan=\"1\" \u003e모델 예측\u003c/th\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e5.9\u003c/td\u003e\u003ctd\u003e3.0\u003c/td\u003e\u003ctd\u003e4.3\u003c/td\u003e\u003ctd\u003e1.5\u003c/td\u003e\u003ctd align=\"center\"\u003e1\u003c/td\u003e\u003ctd align=\"center\"\u003e1\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e6.9\u003c/td\u003e\u003ctd\u003e3.1\u003c/td\u003e\u003ctd\u003e5.4\u003c/td\u003e\u003ctd\u003e2.1\u003c/td\u003e\u003ctd align=\"center\"\u003e2\u003c/td\u003e\u003ctd align=\"center\"\u003e2\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e5.1\u003c/td\u003e\u003ctd\u003e3.3\u003c/td\u003e\u003ctd\u003e1.7\u003c/td\u003e\u003ctd\u003e0.5\u003c/td\u003e\u003ctd align=\"center\"\u003e0\u003c/td\u003e\u003ctd align=\"center\"\u003e0\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e6.0\u003c/td\u003e \u003ctd\u003e3.4\u003c/td\u003e \u003ctd\u003e4.5\u003c/td\u003e \u003ctd\u003e1.6\u003c/td\u003e \u003ctd align=\"center\"\u003e1\u003c/td\u003e\u003ctd align=\"center\" bgcolor=\"red\"\u003e2\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\n", - " \u003ctd\u003e5.5\u003c/td\u003e\u003ctd\u003e2.5\u003c/td\u003e\u003ctd\u003e4.0\u003c/td\u003e\u003ctd\u003e1.3\u003c/td\u003e\u003ctd align=\"center\"\u003e1\u003c/td\u003e\u003ctd align=\"center\"\u003e1\u003c/td\u003e\n", - " \u003c/tr\u003e\n", - " \u003ctr\u003e\u003ctd align=\"center\" colspan=\"6\"\u003e\n", - " \u003cb\u003e그림 4.\u003c/b\u003e 80% 정확도 붓꽃 분류기.\u003cbr/\u003e\u0026nbsp;\n", - " \u003c/td\u003e\u003c/tr\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z-EvK7hGL0d8" - }, - "source": [ - "### 테스트 데이터 세트 설정\n", - "\n", - "모델을 평가하는 것은 모델을 훈련하는 것과 유사합니다. 가장 큰 차이는 훈련 데이터가 아닌 *[테스트 데이터 세트](https://developers.google.com/machine-learning/crash-course/glossary#test_set)* 를 사용했다는 것입니다. 공정하게 모델의 유효성을 평가하기 위해, 모델을 평가하기 위한 샘플은 반드시 훈련 데이터와 달라야합니다. \n", - "\n", - "테스트 데이터 세트를 설정하는 것은 훈련 데이터 세트를 설정하는 것과 유사합니다. CSV 파일을 다운로드하고 값을 파싱합니다. 그 후 셔플은 적용하지 않습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Ps3_9dJ3Lodk" - }, - "outputs": [], - "source": [ - "test_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv\"\n", - "\n", - "test_fp = tf.keras.utils.get_file(fname=os.path.basename(test_url),\n", - " origin=test_url)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SRMWCu30bnxH" - }, - "outputs": [], - "source": [ - "test_dataset = tf.data.experimental.make_csv_dataset(\n", - " test_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name='species',\n", - " num_epochs=1,\n", - " shuffle=False)\n", - "\n", - "test_dataset = test_dataset.map(pack_features_vector)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HFuOKXJdMAdm" - }, - "source": [ - "### 테스트 데이터 세트를 사용한 모델 평가\n", - "\n", - "훈련 단계와는 다르게 모델은 테스트 데이터에 대해서 오직 한 번의 [에포크](https://developers.google.com/machine-learning/glossary/#epoch)를 진행합니다. 다음의 코드 셀은 테스트 셋에 있는 샘플에 대해 실행하고 실제 레이블과 비교합니다. 이는 전체 테스트 데이터 세트에 대한 정확도를 측정하는데 사용됩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Tw03-MK1cYId" - }, - "outputs": [], - "source": [ - "test_accuracy = tf.keras.metrics.Accuracy()\n", - "\n", - "for (x, y) in test_dataset:\n", - " logits = model(x)\n", - " prediction = tf.argmax(logits, axis=1, output_type=tf.int32)\n", - " test_accuracy(prediction, y)\n", - "\n", - "print(\"테스트 세트 정확도: {:.3%}\".format(test_accuracy.result()))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HcKEZMtCOeK-" - }, - "source": [ - "마지막 배치에서 모델이 올바르게 예측한 것을 확인할 수 있습니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uNwt2eMeOane" - }, - "outputs": [], - "source": [ - "tf.stack([y,prediction],axis=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Li2r1tYvW7S" - }, - "source": [ - "## 훈련된 모델로 예측하기\n", - "\n", - "이제 붓꽃을 분류하기 위해 완벽하지는 않지만 어느 정도 검증된 모델을 가지고 있습니다. 훈련된 모델을 사용하여 [레이블 되지 않은 데이터](https://developers.google.com/machine-learning/glossary/#unlabeled_example)를 예측해봅시다.\n", - "\n", - "실제로는 레이블 되지 않은 샘플들은 여러 소스(앱, CSV 파일, 직접 제공 등)로부터 제공될 수 있습니다. 지금은 레이블을 예측하기 위해 수동으로 3개의 레이블 되지 않은 샘플을 제공하겠습니다. 레이블은 다음과 같은 붓꽃 이름으로 매핑되어있습니다.\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kesTS5Lzv-M2" - }, - "outputs": [], - "source": [ - "predict_dataset = tf.convert_to_tensor([\n", - " [5.1, 3.3, 1.7, 0.5,],\n", - " [5.9, 3.0, 4.2, 1.5,],\n", - " [6.9, 3.1, 5.4, 2.1]\n", - "])\n", - "\n", - "predictions = model(predict_dataset)\n", - "\n", - "for i, logits in enumerate(predictions):\n", - " class_idx = tf.argmax(logits).numpy()\n", - " p = tf.nn.softmax(logits)[class_idx]\n", - " name = class_names[class_idx]\n", - " print(\"샘플 {} 예측: {} ({:4.1f}%)\".format(i, name, 100*p))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JliO3dfQRcbg" - }, - "outputs": [], - "source": [ - "" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "custom_training_walkthrough.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/distribute/custom_training.ipynb b/site/ko/tutorials/distribute/custom_training.ipynb deleted file mode 100644 index 1da1ff1fea2..00000000000 --- a/site/ko/tutorials/distribute/custom_training.ipynb +++ /dev/null @@ -1,789 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# 훈련 루프와 함께 tf.distribute.Strategy 사용하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tUP8LMdYtWPz" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/gpu.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "이 예제는 사용자 정의 훈련 루프(custom training loops)와 함께 [`tf.distribute.Strategy`](https://www.tensorflow.org/guide/distributed_training)를 사용하는 법을 보여드립니다. 우리는 간단한 CNN 모델을 패션 MNIST 데이터셋에 대해 훈련을 할 것입니다. 패션 MNIST 데이터셋은 60000개의 28 x 28 크기의 훈련 이미지들과 10000개의 28 x 28 크기의 테스트 이미지들을 포함하고 있습니다.\n", - "\n", - "이 예제는 유연성을 높이고, 훈련을 더 잘 제어할 수 있도록 사용자 정의 훈련 루프를 사용합니다. 또한, 사용자 훈련 루프를 사용하는 것은 모델과 훈련 루프를 디버깅하기 쉬워집니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# Import TensorFlow\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "# 헬퍼 라이브러리들\n", - "import numpy as np\n", - "import os\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MM6W__qraV55" - }, - "source": [ - "## 패션 MNIST 데이터셋 다운로드" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = tf.keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\n", - "\n", - "\n", - "# 하나의 차원을 배열에 추가 -> 새로운 shape == (28, 28, 1)\n", - "# 이렇게 하는 이유는 우리의 모델에서 첫 번째 층이 합성곱 층이고\n", - "# 합성곱 층은 4D 입력을 요구하기 때문입니다.\n", - "# (batch_size, height, width, channels).\n", - "# batch_size 차원은 나중에 추가할 것입니다.\n", - "\n", - "train_images = train_images[..., None]\n", - "test_images = test_images[..., None]\n", - "\n", - "# 이미지를 [0, 1] 범위로 변경하기.\n", - "train_images = train_images / np.float32(255)\n", - "test_images = test_images / np.float32(255)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4AXoHhrsbdF3" - }, - "source": [ - "## 변수와 그래프를 분산하는 전략 만들기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5mVuLZhbem8d" - }, - "source": [ - "`tf.distribute.MirroredStrategy` 전략이 어떻게 동작할까요?\n", - "* 모든 변수와 모델 그래프는 장치(replicas, 다른 문서에서는 replica가 분산 훈련에서 장치 등에 복제된 모델을 의미하는 경우가 있으나 이 문서에서는 장치 자체를 의미합니다)에 복제됩니다.\n", - "* 입력은 장치에 고르게 분배되어 들어갑니다.\n", - "* 각 장치는 주어지는 입력에 대해서 손실(loss)과 그래디언트를 계산합니다.\n", - "* 그래디언트들을 전부 더함으로써 모든 장치들 간에 그래디언트들이 동기화됩니다.\n", - "* 동기화된 후에, 동일한 업데이트가 각 장치에 있는 변수의 복사본(copies)에 동일하게 적용됩니다.\n", - "\n", - "노트: 하나의 범위를 지정해서 모든 코드를 집어넣을 수 있습니다. 자, 같이 살펴보시죠!" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F2VeZUWUj5S4", - "colab": {} - }, - "source": [ - "# 만약 장치들의 목록이 `tf.distribute.MirroredStrategy` 생성자 안에 명시되어 있지 않다면,\n", - "# 자동으로 장치를 인식할 것입니다.\n", - "strategy = tf.distribute.MirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZngeM_2o0_JO", - "colab": {} - }, - "source": [ - "print ('장치의 수: {}'.format(strategy.num_replicas_in_sync))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k53F5I_IiGyI" - }, - "source": [ - "## 입력 파이프라인 설정하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Qb6nDgxiN_n" - }, - "source": [ - "그래프와 변수를 플랫폼과 무관한 SavedModel 형식으로 내보냅니다. 모델을 내보냈다면, 모델을 불러올 때 범위(scope)를 지정해도 되고 하지 않아도 됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwJtsCQhHK-E", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = len(train_images)\n", - "\n", - "BATCH_SIZE_PER_REPLICA = 64\n", - "GLOBAL_BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n", - "\n", - "EPOCHS = 10" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J7fj3GskHC8g" - }, - "source": [ - "분산 데이터셋들을 `strategy.scope` 내에 생성합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WYrMNNDhAvVl", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - "\n", - " train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(BUFFER_SIZE).batch(GLOBAL_BATCH_SIZE) \n", - " train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)\n", - " \n", - " test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE) \n", - " test_dist_dataset = strategy.experimental_distribute_dataset(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bAXAo_wWbWSb" - }, - "source": [ - "## 모델 만들기\n", - "\n", - "`tf.keras.Sequential`을 사용해서 모델을 생성합니다. Model Subclassing API로도 모델 생성을 할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "def create_model():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Conv2D(64, 3, activation='relu'),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - "\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9iagoTBfijUz", - "colab": {} - }, - "source": [ - "# 체크포인트들을 저장하기 위해서 체크포인트 디렉토리를 생성합니다.\n", - "checkpoint_dir = './training_checkpoints'\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e-wlFFZbP33n" - }, - "source": [ - "## 손실 함수 정의하기\n", - "\n", - "일반적으로, GPU/CPU 비율이 1인 단일 장치에서 손실은 입력 배치(batch)의 샘플 개수로 나누어집니다.\n", - "\n", - "*그렇다면, `tf.distribute.Strategy`를 사용할 때, 손실은 어떻게 계산되어야 할까요?*\n", - "\n", - "* 예를들면, 4개의 GPU가 있고 입력 배치 크기가 64라고 하죠. 입력 배치 하나가 여러 개의 장치(4개의 GPU)에 분배됩니다. 각 장치는 크기가 16인 입력을 받습니다.\n", - "\n", - "* 각 장치에 있는 모델은 해당 입력에 대해 정방향 계산(forward pass)을 수행하고 손실을 계산합니다. 손실을 장치에 할당된 입력 샘플의 수(BATCH_SIZE_PER_REPLICA = 16)로 나누는 것이 아니라 GLOBAL_BATCH_SIZE(64)로 나누어야 합니다.\n", - "\n", - "*왜 이렇게 할까요?*\n", - "\n", - "* 위와 같이 계산하는 이유는 그래디언트들이 각 장치에서 계산된 다음, 모든 장치를 동기화하기 위해 이 그래디언트 값들을 전부 **더하기** 때문입니다.\n", - "\n", - "*이 것을 텐서플로에서는 어떻게 할까요?*\n", - "\n", - "\n", - "* 만약 이 **예제처럼** 사용자 정의 훈련 루프를 **작성한다면**, **다음과 같이 샘플당** 손실을 더하고 GLOBAL_BATCH_SIZE로 **나누어야** 합니다.\n", - "`scale_loss = tf.reduce_sum(loss) * (1. / GLOBAL_BATCH_SIZE)`\n", - "또는 `tf.nn.compute_average_loss` **함수를** 사용할 수도 있습니다. 이 함수는 **샘플당 손실과 선택적으로 샘플 가중치, GLOBAL_BATCH_SIZE를 매개변수 값으로 받고** 스케일이 조정된 손실을 반환합니다.\n", - "\n", - "* 만약 규제 손실을 사용하는 모델이라면, 장치 개수로 손실 값을 스케일 조정해야 합니다. 이는 `tf.nn_scale_regularization_loss` 함수를 사용하여 처리할 수 있습니다.\n", - "\n", - "* `tf.reduce_mean`을 사용하는 것은 추천하지 않습니다. 이렇게 하는 것은 손실을 실제 장치당 배치 크기로 나눕니다. 이 실제 장치당 배치 크기는 아마 각 단계(step)마다 크기가 다를 수 있습니다.\n", - "\n", - "* 이런 축소(`reduction`)와 스케일 조정은 케라스의 `model.compile`과 `model.fit`에서 자동적으로 수행됩니다.\n", - "\n", - "* 만약 `tf.keras.losses` 클래스(아래의 예제에서처럼)를 사용한다면, reduction 매개변수를 명시적으로 `NONE` 또는 `SUM` 중 하나로 표시해야 합니다. `AUTO`가 허용되지 않는 것은 사용자가 분산 모드에서 어떻게 축소할지 명시적으로 지정하는 것이 바람직하기 때문입니다.\n", - "현재 `SUM_OVER_BATCH_SIZE`가 장치당 배치 크기로만 나누고 장치 개수로 나누는 것은 사용자에게 위임하기 때문입니다. 그래서 이렇게 하는 것 대신에 사용자가 명시적으로 축소를 수행하도록 만드는 것이 좋습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R144Wci782ix", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " # reduction을 `none`으로 설정합니다. 그래서 우리는 축소를 나중에 하고,\n", - " # GLOBAL_BATCH_SIZE로 나눌 수 있습니다.\n", - " loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " reduction=tf.keras.losses.Reduction.NONE)\n", - " # 또는 loss_fn = tf.keras.losses.sparse_categorical_crossentropy를 사용해도 됩니다.\n", - " def compute_loss(labels, predictions):\n", - " per_example_loss = loss_object(labels, predictions)\n", - " return tf.nn.compute_average_loss(per_example_loss, global_batch_size=GLOBAL_BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w8y54-o9T2Ni" - }, - "source": [ - "## 손실과 정확도를 기록하기 위한 지표 정의하기\n", - "\n", - "이 지표(metrics)는 테스트 손실과 훈련 정확도, 테스트 정확도를 기록합니다. `.result()`를 사용해서 누적된 통계값들을 언제나 볼 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zt3AHb46Tr3w", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "\n", - " train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", - " name='train_accuracy')\n", - " test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", - " name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iuKuNXPORfqJ" - }, - "source": [ - "## 훈련 루프" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OrMmakq5EqeQ", - "colab": {} - }, - "source": [ - "# 모델과 옵티마이저는 `strategy.scope`에서 만들어져야 합니다.\n", - "with strategy.scope():\n", - " model = create_model()\n", - "\n", - " optimizer = tf.keras.optimizers.Adam()\n", - "\n", - " checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3UX43wUu04EL", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " def train_step(inputs):\n", - " images, labels = inputs\n", - "\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images, training=True)\n", - " loss = compute_loss(labels, predictions)\n", - "\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_accuracy.update_state(labels, predictions)\n", - " return loss \n", - "\n", - " def test_step(inputs):\n", - " images, labels = inputs\n", - "\n", - " predictions = model(images, training=False)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss.update_state(t_loss)\n", - " test_accuracy.update_state(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gX975dMSNw0e", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " # `experimental_run_v2`는 주어진 계산을 복사하고,\n", - " # 분산된 입력으로 계산을 수행합니다.\n", - " \n", - " @tf.function\n", - " def distributed_train_step(dataset_inputs):\n", - " per_replica_losses = strategy.experimental_run_v2(train_step,\n", - " args=(dataset_inputs,))\n", - " return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n", - " axis=None)\n", - " \n", - " @tf.function\n", - " def distributed_test_step(dataset_inputs):\n", - " return strategy.experimental_run_v2(test_step, args=(dataset_inputs,))\n", - "\n", - " for epoch in range(EPOCHS):\n", - " # 훈련 루프\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " for x in train_dist_dataset:\n", - " total_loss += distributed_train_step(x)\n", - " num_batches += 1\n", - " train_loss = total_loss / num_batches\n", - "\n", - " # 테스트 루프\n", - " for x in test_dist_dataset:\n", - " distributed_test_step(x)\n", - "\n", - " if epoch % 2 == 0:\n", - " checkpoint.save(checkpoint_prefix)\n", - "\n", - " template = (\"에포크 {}, 손실: {}, 정확도: {}, 테스트 손실: {}, \"\n", - " \"테스트 정확도: {}\")\n", - " print (template.format(epoch+1, train_loss,\n", - " train_accuracy.result()*100, test_loss.result(),\n", - " test_accuracy.result()*100))\n", - "\n", - " test_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z1YvXqOpwy08" - }, - "source": [ - "위의 예제에서 주목해야 하는 부분\n", - "* 이 예제는 `train_dist_dataset`과 `test_dist_dataset`을 `for x in ...` 구조를 통해서 반복합니다.\n", - "* 스케일이 조정된 손실은 `distributed_train_step`의 반환값입니다. `tf.distribute.Strategy.reduce` 호출을 사용해서 장치들 간의 스케일이 조정된 손실 값을 전부 합칩니다. 그리고 나서 `tf.distribute.Strategy.reduce` 반환 값을 더하는 식으로 배치 간의 손실을 모읍니다.\n", - "* `tf.keras.Metrics`는 `tf.distribute.Strategy.experimental_run_v2`에 의해서 실행되는 `train_step`과 `test_step` 함수 안에서 업데이트 되어야 합니다.\n", - "* `tf.distribute.Strategy.experimental_run_v2`는 그 전략안에 포함된 각 지역 복제 모델로부터 결과값을 반환해 줍니다. 그리고 이 결과를 사용하는 몇 가지 방법들이 있습니다. `tf.distribute.Strategy.reduce`를 이용하여 값들을 합칠 수 있습니다. `tf.distribute.Strategy.experimental_local_results`를 사용해서 결과값(지역 복제 모델 당 하나의 결과값)에 들어있는 값들의 리스트를 얻을 수도 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-q5qp31IQD8t" - }, - "source": [ - "## 최신 체크포인트를 불러와서 테스트하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WNW2P00bkMGJ" - }, - "source": [ - "`tf.distribute.Strategy`를 사용해서 체크포인트가 만들어진 모델은 전략 사용 여부에 상관없이 불러올 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pg3B-Cw_cn3a", - "colab": {} - }, - "source": [ - "eval_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", - " name='eval_accuracy')\n", - "\n", - "new_model = create_model()\n", - "new_optimizer = tf.keras.optimizers.Adam()\n", - "\n", - "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7qYii7KUYiSM", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def eval_step(images, labels):\n", - " predictions = new_model(images, training=False)\n", - " eval_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LeZ6eeWRoUNq", - "colab": {} - }, - "source": [ - "checkpoint = tf.train.Checkpoint(optimizer=new_optimizer, model=new_model)\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "for images, labels in test_dataset:\n", - " eval_step(images, labels)\n", - "\n", - "print ('전략을 사용하지 않고, 저장된 모델을 복원한 후의 정확도: {}'.format(\n", - " eval_accuracy.result()*100))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EbcI87EEzhzg" - }, - "source": [ - "## 데이터셋에 대해 반복작업을 하는 다른 방법들\n", - "\n", - "### 반복자(iterator)를 사용하기\n", - "\n", - "만약 주어진 스텝의 수에 따라서 반복하기 원하면서 전체 데이터셋을 보는 것을 원치 않는다면, `iter`를 호출하여 반복자를 만들 수 있습니다. 그 다음 명시적으로 `next`를 호출합니다. 또한, `tf.funtion` 내부 또는 외부에서 데이터셋을 반복하도록 설정 할 수 있습니다. 다음은 반복자를 사용하여 `tf.function` 외부에서 데이터셋을 반복하는 코드 예제입니다.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7c73wGC00CzN", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " for _ in range(EPOCHS):\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " train_iter = iter(train_dist_dataset)\n", - "\n", - " for _ in range(10):\n", - " total_loss += distributed_train_step(next(train_iter))\n", - " num_batches += 1\n", - " average_train_loss = total_loss / num_batches\n", - "\n", - " template = (\"에포크 {}, 손실: {}, 정확도: {}\")\n", - " print (template.format(epoch+1, average_train_loss, train_accuracy.result()*100))\n", - " train_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GxVp48Oy0m6y" - }, - "source": [ - "### tf.function 내부에서 반복하기\n", - "전체 입력 `train_dist_dataset`에 대해서 `tf.function` 내부에서 `for x in ...` 생성자를 사용함으로써 반복을 하거나, 위에서 사용했던 것처럼 반복자를 사용함으로써 반복을 할 수 있습니다. 아래의 예제에서는 `tf.function`로 한 훈련의 에포크를 감싸고 그 함수에서 `train_dist_dataset`를 반복하는 것을 보여 줍니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-REzmcXv00qm", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " @tf.function\n", - " def distributed_train_epoch(dataset):\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " for x in dataset:\n", - " per_replica_losses = strategy.experimental_run_v2(train_step,\n", - " args=(x,))\n", - " total_loss += strategy.reduce(\n", - " tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n", - " num_batches += 1\n", - " return total_loss / tf.cast(num_batches, dtype=tf.float32)\n", - "\n", - " for epoch in range(EPOCHS):\n", - " train_loss = distributed_train_epoch(train_dist_dataset)\n", - "\n", - " template = (\"Epoch {}, Loss: {}, Accuracy: {}\")\n", - " print (template.format(epoch+1, train_loss, train_accuracy.result()*100))\n", - "\n", - " train_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MuZGXiyC7ABR" - }, - "source": [ - "### 장치 간의 훈련 손실 기록하기\n", - "\n", - "노트: 일반적인 규칙으로, `tf.keras.Metrics`를 사용하여 샘플당 손실 값을 기록하고 장치 내부에서 값이 합쳐지는 것을 피해야 합니다.\n", - "\n", - "`tf.metrics.Mean`을 사용하여 여러 장치의 훈련 손실을 기록하는 것을 추천하지 *않습니다*. 왜냐하면 손실의 스케일을 조정하는 계산이 수행되기 때문입니다.\n", - "\n", - "예를 들어, 다음과 같은 조건의 훈련을 수행한다고 합시다.\n", - "* 두개의 장치\n", - "* 두개의 샘플들이 각 장치에 의해 처리됩니다.\n", - "* 손실 값을 산출합니다: 각각의 장치에 대해 [2, 3]과 [4, 5]\n", - "* Global batch size = 4\n", - "\n", - "손실의 스케일 조정을 하면, 손실 값을 더하고 GLOBAL_BATCH_SIZE로 나누어 각 장치에 대한 샘플당 손실값을 계산할 수 있습니다. 이 경우에는 (2 + 3) / 4 = 1.24와 (4 + 5) / 4 = 2.25입니다.\n", - "\n", - "만약 `tf.metrics.Mean`을 사용해서 두 개의 장치에 대해 손실값을 계산한다면, 결과값이 다릅니다. 이 예제에서는, 측정 지표의 `result()`가 메서드가 호출될 때 `total`이 3.50이고 `count`가 2입니다. 결과값은 `total/count`가 1.75가 됩니다. `tf.keras.Metrics`를 이용해서 계산한 손실값이 추가적인 요인에 의해서 크기조정되며, 이 추가적인 요인은 동기화되는 장치의 개수입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xisYJaV9KZTN" - }, - "source": [ - "### 예제와 튜토리얼\n", - "사용자 정의 훈련루프를 포함한 분산 전략을 사용하는 몇 가지 예제가 있습니다.\n", - "\n", - "1. `MirroredStrategy`를 사용해서 MNIST를 훈련하는 [Tutorial](../tutorials/distribute/custom_training.ipynb)\n", - "2. `MirroredStrategy`를 사용하는 [DenseNet](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/densenet/distributed_train.py) 예제.\n", - "1. `MirroredStrategy`와 `TPUStrategy`를 사용해서 훈련하는 [BERT](https://github.com/tensorflow/models/blob/master/official/bert/run_classifier.py) 예제. 이 예제는 분산 훈련 중에 어떻게 체크포인트로부터 불러오는지와 어떻게 주기적으로 체크포인트들을 생성해 내는지를 이해하기에 정말 좋습니다.\n", - "2. `keras_use_ctl flag`를 사용해서 활성화 할 수 있는 MirroredStrategy를 이용해서 훈련되는 [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) 예제\n", - "3. `MirroredStrategy`을 사용해서 훈련되는 [NMT](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/nmt_with_attention/distributed_train.py) 예제.\n", - "\n", - "더 많은 예제는 여기에 있습니다. [Distribution strategy guide](../../guide/distributed_training.ipynb#examples_and_tutorials)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6hEJNsokjOKs" - }, - "source": [ - "## 다음단계\n", - "\n", - "새로운 `tf.distribute.Strategy` API를 모델에 적용해 보세요." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/tutorials/distribute/keras.ipynb b/site/ko/tutorials/distribute/keras.ipynb deleted file mode 100644 index abdffc53bc2..00000000000 --- a/site/ko/tutorials/distribute/keras.ipynb +++ /dev/null @@ -1,735 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "keras.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "tuOe1ymfHZPu", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# 케라스를 사용한 분산 훈련" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r6P32iYYV27b" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jp3KSyVr3Erk", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/distribute/keras.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃허브 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "## 개요\n", - "\n", - "`tf.distribute.Strategy` API는 훈련을 여러 처리 장치들로 분산시키는 것을 추상화한 것입니다. 기존의 모델이나 훈련 코드를 조금만 바꾸어 분산 훈련을 할 수 있게 하는 것이 분산 전략 API의 목표입니다.\n", - "\n", - "이 튜토리얼에서는 `tf.distribute.MirroredStrategy`를 사용합니다. 이 전략은 동기화된 훈련 방식을 활용하여 한 장비에 있는 여러 개의 GPU로 그래프 내 복제를 수행합니다. 다시 말하자면, 모델의 모든 변수를 각 프로세서에 복사합니다. 그리고 각 프로세서의 그래디언트(gradient)를 [올 리듀스(all-reduce)](http://mpitutorial.com/tutorials/mpi-reduce-and-allreduce/)를 사용하여 모읍니다. 그다음 모아서 계산한 값을 각 프로세서의 모델 복사본에 적용합니다.\n", - "\n", - "`MirroredStategy`는 텐서플로에서 기본으로 제공하는 몇 가지 분산 전략 중 하나입니다. 다른 전략들에 대해서는 [분산 전략 가이드](../../guide/distributed_training.ipynb)를 참고하십시오." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUXex9ctTuDB" - }, - "source": [ - "### 케라스 API\n", - "\n", - "이 예는 모델과 훈련 루프를 만들기 위해 `tf.keras` API를 사용합니다. 직접 훈련 코드를 작성하는 방법은 [사용자 정의 훈련 루프로 분산 훈련하기](training_loops.ipynb) 튜토리얼을 참고하십시오." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dney9v7BsJij" - }, - "source": [ - "## 필요한 패키지 가져오기" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "r8S3ublR7Ay8", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# 텐서플로와 텐서플로 데이터셋 패키지 가져오기\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "tfds.disable_progress_bar()\n", - "\n", - "import os" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hXhefksNKk2I" - }, - "source": [ - "## 데이터셋 다운로드" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OtnnUwvmB3X5" - }, - "source": [ - "MNIST 데이터셋을 [TensorFlow Datasets](https://www.tensorflow.org/datasets)에서 다운로드받은 후 불러옵니다. 이 함수는 `tf.data` 형식을 반환합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lHAPqG8MtS8M" - }, - "source": [ - "`with_info`를 `True`로 설정하면 전체 데이터에 대한 메타 정보도 함께 불러옵니다. 이 정보는 `info` 변수에 저장됩니다. 여기에는 훈련과 테스트 샘플 수를 비롯한 여러가지 정보들이 들어있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iXMJ3G9NB3X6", - "colab": {} - }, - "source": [ - "datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n", - "\n", - "mnist_train, mnist_test = datasets['train'], datasets['test']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GrjVhv-eKuHD" - }, - "source": [ - "## 분산 전략 정의하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TlH8vx6BB3X9" - }, - "source": [ - "분산과 관련된 처리를 하는 `MirroredStrategy` 객체를 만듭니다. 이 객체가 컨텍스트 관리자(`tf.distribute.MirroredStrategy.scope`)도 제공하는데, 이 안에서 모델을 만들어야 합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4j0tdf4YB3X9", - "colab": {} - }, - "source": [ - "strategy = tf.distribute.MirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cY3KA_h2iVfN", - "colab": {} - }, - "source": [ - "print('장치의 수: {}'.format(strategy.num_replicas_in_sync))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lNbPv0yAleW8" - }, - "source": [ - "## 입력 파이프라인 구성하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "psozqcuptXhK" - }, - "source": [ - "다중 GPU로 모델을 훈련할 때는 배치 크기를 늘려야 컴퓨팅 자원을 효과적으로 사용할 수 있습니다. 기본적으로는 GPU 메모리에 맞추어 가능한 가장 큰 배치 크기를 사용하십시오. 이에 맞게 학습률도 조정해야 합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p1xWxKcnhar9", - "colab": {} - }, - "source": [ - "# 데이터셋 내 샘플의 수는 info.splits.total_num_examples 로도\n", - "# 얻을 수 있습니다.\n", - "\n", - "num_train_examples = info.splits['train'].num_examples\n", - "num_test_examples = info.splits['test'].num_examples\n", - "\n", - "BUFFER_SIZE = 10000\n", - "\n", - "BATCH_SIZE_PER_REPLICA = 64\n", - "BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Wm5rsL2KoDF" - }, - "source": [ - "픽셀의 값은 0~255 사이이므로 [0-1 범위로 정규화](https://en.wikipedia.org/wiki/Feature_scaling)해야 합니다. 정규화 함수를 정의합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Eo9a46ZeJCkm", - "colab": {} - }, - "source": [ - "def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - "\n", - " return image, label" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WZCa5RLc5A91" - }, - "source": [ - "이 함수를 훈련과 테스트 데이터에 적용합니다. 훈련 데이터 순서를 섞고, [훈련을 위해 배치로 묶습니다](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gRZu2maChwdT", - "colab": {} - }, - "source": [ - "train_dataset = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", - "eval_dataset = mnist_test.map(scale).batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4xsComp8Kz5H" - }, - "source": [ - "## 모델 만들기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1BnQYQTpB3YA" - }, - "source": [ - "`strategy.scope` 컨텍스트 안에서 케라스 모델을 만들고 컴파일합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IexhL_vIB3YA", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - "\n", - " model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8i6OU5W9Vy2u" - }, - "source": [ - "## 콜백 정의하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YOXO5nvvK3US" - }, - "source": [ - "여기서 사용하는 콜백은 다음과 같습니다.\n", - "\n", - "* *텐서보드(TensorBoard)*: 이 콜백은 텐서보드용 로그를 남겨서, 텐서보드에서 그래프를 그릴 수 있게 해줍니다.\n", - "* *모델 체크포인트(Checkpoint)*: 이 콜백은 매 에포크(epoch)가 끝난 후 모델을 저장합니다.\n", - "* *학습률 스케줄러*: 이 콜백을 사용하면 매 에포크 혹은 배치가 끝난 후 학습률을 바꿀 수 있습니다.\n", - "\n", - "콜백을 추가하는 방법을 보여드리기 위하여 노트북에 *학습률*을 표시하는 콜백도 추가하겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "A9bwLCcXzSgy", - "colab": {} - }, - "source": [ - "# 체크포인트를 저장할 체크포인트 디렉터리를 지정합니다.\n", - "checkpoint_dir = './training_checkpoints'\n", - "# 체크포인트 파일의 이름\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wpU-BEdzJDbK", - "colab": {} - }, - "source": [ - "# 학습률을 점점 줄이기 위한 함수\n", - "# 필요한 함수를 직접 정의하여 사용할 수 있습니다.\n", - "def decay(epoch):\n", - " if epoch < 3:\n", - " return 1e-3\n", - " elif epoch >= 3 and epoch < 7:\n", - " return 1e-4\n", - " else:\n", - " return 1e-5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jKhiMgXtKq2w", - "colab": {} - }, - "source": [ - "# 에포크가 끝날 때마다 학습률을 출력하는 콜백.\n", - "class PrintLR(tf.keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs=None):\n", - " print('\\n에포크 {}의 학습률은 {}입니다.'.format(epoch + 1,\n", - " model.optimizer.lr.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YVqAbR6YyNQh", - "colab": {} - }, - "source": [ - "callbacks = [\n", - " tf.keras.callbacks.TensorBoard(log_dir='./logs'),\n", - " tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,\n", - " save_weights_only=True),\n", - " tf.keras.callbacks.LearningRateScheduler(decay),\n", - " PrintLR()\n", - "]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "70HXgDQmK46q" - }, - "source": [ - "## 훈련과 평가" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6EophnOAB3YD" - }, - "source": [ - "이제 평소처럼 모델을 학습합시다. 모델의 `fit` 함수를 호출하고 튜토리얼의 시작 부분에서 만든 데이터셋을 넘깁니다. 이 단계는 분산 훈련 여부와 상관없이 동일합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MVw_6CqB3YD", - "colab": {} - }, - "source": [ - "model.fit(train_dataset, epochs=12, callbacks=callbacks)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NUcWAUUupIvG" - }, - "source": [ - "아래에서 볼 수 있듯이 체크포인트가 저장되고 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JQ4zeSTxKEhB", - "colab": {} - }, - "source": [ - "# 체크포인트 디렉터리 확인하기\n", - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qor53h7FpMke" - }, - "source": [ - "모델의 성능이 어떤지 확인하기 위하여, 가장 최근 체크포인트를 불러온 후 테스트 데이터에 대하여 `evaluate`를 호출합니다.\n", - "\n", - "평소와 마찬가지로 적절한 데이터셋과 함께 `evaluate`를 호출하면 됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JtEwxiTgpQoP", - "colab": {} - }, - "source": [ - "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "eval_loss, eval_acc = model.evaluate(eval_dataset)\n", - "\n", - "print('평가 손실: {}, 평가 정확도: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IIeF2RWfYu4N" - }, - "source": [ - "텐서보드 로그를 다운로드받은 후 터미널에서 다음과 같이 텐서보드를 실행하여 훈련 결과를 확인할 수 있습니다.\n", - "\n", - "```\n", - "$ tensorboard --logdir=path/to/log-directory\n", - "```" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LnyscOkvKKBR", - "colab": {} - }, - "source": [ - "!ls -sh ./logs" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kBLlogrDvMgg" - }, - "source": [ - "## SavedModel로 내보내기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xa87y_A0vRma" - }, - "source": [ - "플랫폼에 무관한 SavedModel 형식으로 그래프와 변수들을 내보냅니다. 모델을 내보낸 후에는, 전략 범위(scope) 없이 불러올 수도 있고, 전략 범위와 함께 불러올 수도 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h8Q4MKOLwG7K", - "colab": {} - }, - "source": [ - "path = 'saved_model/'" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4HvcDmVsvQoa", - "colab": {} - }, - "source": [ - "tf.keras.experimental.export_saved_model(model, path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vKJT4w5JwVPI" - }, - "source": [ - "`strategy.scope` 없이 모델 불러오기." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "T_gT0RbRvQ3o", - "colab": {} - }, - "source": [ - "unreplicated_model = tf.keras.experimental.load_from_saved_model(path)\n", - "\n", - "unreplicated_model.compile(\n", - " loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])\n", - "\n", - "eval_loss, eval_acc = unreplicated_model.evaluate(eval_dataset)\n", - "\n", - "print('평가 손실: {}, 평가 정확도: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YBLzcRF0wbDe" - }, - "source": [ - "`strategy.scope`와 함께 모델 불러오기." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BBVo3WGGwd9a", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " replicated_model = tf.keras.experimental.load_from_saved_model(path)\n", - " replicated_model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])\n", - "\n", - " eval_loss, eval_acc = replicated_model.evaluate(eval_dataset)\n", - " print ('평가 손실: {}, 평가 정확도: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUZwaz4AKjtD" - }, - "source": [ - "### 예제와 튜토리얼\n", - "\n", - "케라스 적합/컴파일과 함께 분산 전략을 쓰는 예제들이 더 있습니다.\n", - "\n", - "1. `tf.distribute.MirroredStrategy`를 사용하여 학습한 [Transformer](https://github.com/tensorflow/models/blob/master/official/transformer/v2/transformer_main.py) 예제.\n", - "2. `tf.distribute.MirroredStrategy`를 사용하여 학습한 [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) 예제.\n", - "\n", - "[분산 전략 가이드](../../guide/distributed_training.ipynb#examples_and_tutorials)에 더 많은 예제 목록이 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8uNqWRdDMl5S" - }, - "source": [ - "## 다음 단계\n", - "\n", - "* [분산 전략 가이드](../../guide/distributed_training.ipynb)를 읽어보세요.\n", - "* [사용자 정의 훈련 루프를 사용한 분산 훈련](training_loops.ipynb) 튜토리얼을 읽어보세요.\n", - "\n", - "Note: `tf.distribute.Strategy`은 현재 활발히 개발 중입니다. 근시일내에 예제나 튜토리얼이 더 추가될 수 있습니다. 한 번 사용해 보세요. [깃허브 이슈](https://github.com/tensorflow/tensorflow/issues/new)를 통하여 피드백을 주시면 감사하겠습니다." - ] - } - ] -} diff --git a/site/ko/tutorials/distribute/multi_worker_with_keras.ipynb b/site/ko/tutorials/distribute/multi_worker_with_keras.ipynb deleted file mode 100644 index 0ec32968c27..00000000000 --- a/site/ko/tutorials/distribute/multi_worker_with_keras.ipynb +++ /dev/null @@ -1,484 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "multi_worker_with_keras.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tuOe1ymfHZPu", - "colab": {}, - "cellView": "form" - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# 케라스를 사용한 다중 워커(Multi-worker) 훈련\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FcjoxbKVT3r-", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/distribute/multi_worker_with_keras.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃허브 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "## 개요\n", - "\n", - "이 튜토리얼에서는 `tf.distribute.Strategy` API를 사용하여 케라스 모델을 다중 워커로 분산 훈련하는 방법을 살펴보겠습니다. 다중 워커를 사용하여 훈련할 수 있도록 전략을 디자인했기 때문에, 단일 워커 훈련용으로 만들어진 케라스 모델도 코드를 조금만 바꾸면 다중 워커를 사용하여 훈련할 수 있습니다.\n", - "\n", - "`tf.distribute.Strategy` API에 관심이 있으신 분들은 [텐서플로로 분산 훈련하기](../../guide/distributed_training.ipynb) 가이드에서 텐서플로가 제공하는 분산 훈련 전략들을 훑어보실 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUXex9ctTuDB" - }, - "source": [ - "## 설정\n", - "\n", - "먼저, 텐서플로를 설정하고 필요한 패키지들을 가져옵니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IqR2PQG4ZaZ0", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bnYxvfLD-LW-", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version 기능은 코랩에서만 사용할 수 있습니다.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "tfds.disable_progress_bar()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hPBuZUNSZmrQ" - }, - "source": [ - "## 데이터셋 준비하기\n", - "\n", - "MNIST 데이터셋을 [TensorFlow Datasets](https://www.tensorflow.org/datasets)에서 받아옵시다. [MNIST 데이터셋](http://yann.lecun.com/exdb/mnist/)은 0-9 숫자를 손으로 쓴 28x28 픽셀 흑백 이미지입니다. 6만 개의 훈련 샘플과 만 개의 테스트 샘플이 들어있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dma_wUAxZqo2", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 10000\n", - "BATCH_SIZE = 64\n", - "\n", - "# MNIST 데이터를 (0, 255] 범위에서 (0., 1.] 범위로 조정\n", - "def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - " return image, label\n", - "\n", - "datasets, info = tfds.load(name='mnist',\n", - " with_info=True,\n", - " as_supervised=True)\n", - "\n", - "train_datasets_unbatched = datasets['train'].map(scale).cache().shuffle(BUFFER_SIZE)\n", - "train_datasets = train_datasets_unbatched.batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o87kcvu8GR4-" - }, - "source": [ - "## 케라스 모델 만들기\n", - "`tf.keras.Sequential` API를 사용하여 간단한 합성곱 신경망 케라스 모델을 만들고 컴파일하도록 하겠습니다. 우리 MNIST 데이터셋으로 훈련시킬 모델입니다.\n", - "\n", - "Note: 케라스 모델을 만드는 절차는 [텐서플로 케라스 가이드](https://www.tensorflow.org/guide/keras#sequential_model)에서 더 상세하게 설명하고 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aVPHl0SfG2v1", - "colab": {} - }, - "source": [ - "def build_and_compile_cnn_model():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - " model.compile(\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " optimizer=tf.keras.optimizers.SGD(learning_rate=0.001),\n", - " metrics=['accuracy'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2UL3kisMO90X" - }, - "source": [ - "먼저 단일 워커를 이용하여 적은 수의 에포크만큼만 훈련을 해보고 잘 동작하는지 확인해봅시다. 에포크가 넘어감에 따라 손실(loss)은 줄어들고 정확도는 1.0에 가까워져야 합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Qe6iAf5O8iJ", - "colab": {} - }, - "source": [ - "single_worker_model = build_and_compile_cnn_model()\n", - "single_worker_model.fit(x=train_datasets, epochs=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8YFpxrcsZ2xG" - }, - "source": [ - "## 다중 워커 구성\n", - "\n", - "자 이제 다중 워커 훈련의 세계로 들어가 봅시다. 텐서플로에서 여러 장비를 사용할 때는 `TF_CONFIG` 환경 변수를 설정해야 합니다. 하나의 클러스터를 구성하는 각 장비에 클러스터 구성을 알려주고 각각 다른 역할을 부여하기 위해 `TF_CONFIG`를 사용합니다.\n", - "\n", - "`TF_CONFIG`는 `cluster`와 `task` 두 개의 부분으로 구성됩니다. `cluster`에는 훈련 클러스터에 대한 정보를 지정합니다. `worker` 같은 여러 타입의 작업 이름을 키로 하는 파이썬 딕셔너리를 지정합니다. 다중 워커 훈련에서는 보통 일반적인 워커보다 조금 더 많은 일을 하는 특별한 워커가 하나 필요합니다. 이 워커는 체크포인트를 저장하거나, 서머리(summary)를 쓰는 일 등을 추가로 담당하게 됩니다. 보통 치프('chief') 워커라고 부르고, 관례적으로 `index` 번호가 0인 워커가 치프 워커가 됩니다(사실 `tf.distribute.Strategy`가 이렇게 구현되었습니다). 한편 `task`에는 현재 워커의 작업에 대한 정보를 지정합니다.\n", - "\n", - "이 예에서는 작업(task) `type`을 `\"worker\"`로 지정하고, `index`는 `0`으로 지정하였습니다. 이 말은 이 장비가 첫 번째 워커이고, 따라서 치프 워커이며, 다른 워커보다 더 많은 일을 하게 된다는 뜻입니다. 물론 다른 장비들에도 `TF_CONFIG` 환경변수가 설정되어야 합니다. 다른 장비들에도 `cluster`에는 동일한 딕셔너리를 지정하겠지만, `task`에는 각 장비의 역할에 따라 다른 작업 `type`이나 `index`를 지정해야 합니다.\n", - "\n", - "예시를 위하여, 이 튜토리얼에서는 두 개의 워커를 `localhost`에 띄우는 방법을 보여드리겠습니다. 실제로는 각 워커를 다른 장비에서 띄울텐데, 실제 IP 주소와 포트를 할당하고, 그에 맞게 `TF_CONFIG`를 지정해야 합니다.\n", - "\n", - "주의: 아래 코드를 코랩에서 실행하지 마십시오. 텐서플로 런타임이 주어진 IP와 포트로 gRPC 서버를 띄우려고 할 텐데, 아마도 실패할 것입니다.\n", - "\n", - "```\n", - "os.environ['TF_CONFIG'] = json.dumps({\n", - " 'cluster': {\n", - " 'worker': [\"localhost:12345\", \"localhost:23456\"]\n", - " },\n", - " 'task': {'type': 'worker', 'index': 0}\n", - "})\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P94PrIW_kSCE" - }, - "source": [ - "이 예제에서는 학습률을 바꾸지 않고 그대로 사용한 것에 주의하십시오. 실제로는 전역(global) 배치 크기에 따라 학습률을 조정해야 할 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UhNtHfuxCGVy" - }, - "source": [ - "## 적절한 전략 고르기\n", - "\n", - "텐서플로의 분산 전략은 크게 각 훈련 단계가 워커들이 가진 복제본들끼리 동기화되는 동기 훈련 방식과, 동기화가 엄격하게 이루어지지 않는 비동기 훈련 방식이 있습니다.\n", - "\n", - "이 튜토리얼에서 다루는 `MultiWorkerMirroredStrategy`는 동기 다중 워커 훈련에서 추천하는 전략입니다.\n", - "모델을 훈련하려면 `tf.distribute.experimental.MultiWorkerMirroredStrategy` 인스턴스를 하나 만드십시오.\n", - "`MultiWorkerMirroredStrategy`는 모델의 레이어에 있는 모든 변수의 복사본을 각 워커의 장치마다 만듭니다. 그리고 수집 작업을 위한 텐서플로 연산인 `CollectiveOps`를 사용하여 그래디언트를 모으고, 각 변수의 값을 동기화합니다. [`tf.distribute.Strategy` 가이드](../../guide/distributed_training.ipynb)에 이 전략에 대한 더 자세한 설명이 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1uFSHCJXMrQ-", - "colab": {} - }, - "source": [ - "strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N0iv7SyyAohc" - }, - "source": [ - "Note: `MultiWorkerMirroredStrategy.__init__()`가 호출될 때, `TF_CONFIG`를 파싱하고 텐서플로 gRPC 서버가 구동됩니다. 따라서 `TF_CONFIG` 환경변수는 `tf.distribute.Strategy` 인스턴스를 만들기 전에 설정해야 합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FMy2VM4Akzpr" - }, - "source": [ - "`MultiWorkerMirroredStrategy` 는 [`CollectiveCommunication`](https://github.com/tensorflow/tensorflow/blob/a385a286a930601211d78530734368ccb415bee4/tensorflow/python/distribute/cross_device_ops.py#L928) 매개변수로 선택할 수 있는 여러 가지 구현체를 제공합니다. `RING`(링)은 링 구조 기반의 수집 작업 구현체이고, 장비 간 통신을 위하여 gRPC를 사용합니다. `NCCL`은 [Nvidia의 NCCL](https://developer.nvidia.com/nccl)로 수집 작업을 구현한 것입니다. `AUTO`를 지정하면, 런타임이 알아서 선택합니다. 어떤 수집 작업 구현체가 최적인지는 GPU의 종류와 수, 클러스터 내 네트워크 연결 등 여러 요소에 따라 달라집니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H47DDcOgfzm7" - }, - "source": [ - "## MultiWorkerMirroredStrategy로 모델 훈련하기\n", - "\n", - "다중 워커 분산 훈련을 위하여 `tf.distribute.Strategy` API를 `tf.keras`와 함께 사용하려면, 딱 한 가지만 바꾸면 됩니다. 바로 모델 구성과 `model.compile()` 호출 코드를 `strategy.scope()` 안으로 넣는 것입니다. 분산 전략의 범위(scope)를 써서 변수를 어디에 어떻게 만들지 지정할 수 있습니다. `MultiWorkerMirroredStrategy`의 경우, 만들어지는 변수는 `MirroredVariable`이고, 각 워커에 복제본이 생깁니다.\n", - "\n", - "\n", - "Note: 아래 코드가 예상과 같이 동작하는 것처럼 보이겠지만, 사실은 단일 워커로 동작하는 것입니다. `TF_CONFIG`가 설정되어 있지 않기 때문입니다. 실제로 `TF_CONFIG`를 설정하고 아래 예제를 실행하면, 여러 장비를 활용하여 훈련 속도가 빨라지는 것을 볼 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BcsuBYrpgnlS", - "colab": {} - }, - "source": [ - "NUM_WORKERS = 2\n", - "# 여기서 배치 크기는 워커의 수를 곱한 크기로 늘려야 합니다. `tf.data.Dataset.batch`에는\n", - "# 전역 배치 크기를 지정해야 하기 때문입니다. 전에는 64였지만, 이제 128이 됩니다.\n", - "GLOBAL_BATCH_SIZE = 64 * NUM_WORKERS\n", - "train_datasets = train_datasets_unbatched.batch(GLOBAL_BATCH_SIZE)\n", - "with strategy.scope():\n", - " multi_worker_model = build_and_compile_cnn_model()\n", - "multi_worker_model.fit(x=train_datasets, epochs=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Rr14Vl9GR4zq" - }, - "source": [ - "### 데이터셋 샤딩과 배치 크기\n", - "다중 워커 훈련에서는 수렴과 성능을 위하여 데이터를 여러 부분으로 샤딩(sharding)해야 합니다. 하지만, 위 코드 예에서는 데이터셋을 샤딩하지 않고 바로 `model.fit()`으로 보낸 것을 볼 수 있습니다. 이는 `tf.distribute.Strategy` API가 다중 워커 훈련에 맞게 자동으로 데이터셋을 샤딩해주기 때문입니다.\n", - "\n", - "만약 훈련할 때 샤딩을 직접 하고 싶다면, `tf.data.experimental.DistributeOptions` API를 사용해서 자동 샤딩 기능을 끌 수 있습니다. 다음과 같이 말입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JxEtdh1vH-TF", - "colab": {} - }, - "source": [ - "options = tf.data.Options()\n", - "options.experimental_distribute.auto_shard = False\n", - "train_datasets_no_auto_shard = train_datasets.with_options(options)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NBCtYvmCH-7g" - }, - "source": [ - "또 하나 주목할 점은 `datasets`의 배치 크기입니다. 앞서 코드에서 `GLOBAL_BATCH_SIZE = 64 * NUM_WORKERS`로 지정하였습니다. 단일 워커일 때보다 `NUM_WORKERS` 배만큼 크게 지정한 것입니다. 이는 실제로 각 워커에 전달되는 배치 크기가 `tf.data.Dataset.batch()`에 매개변수로 전달된 전역 배치 크기를 워커의 수로 나눈 것이 되기 때문입니다. 즉, 이렇게 바꾸어야 실제로 워커가 처리하는 배치 크기가 단일 워커일 때와 동일한 값이 됩니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XVk4ftYx6JAO" - }, - "source": [ - "## 성능\n", - "\n", - "이제 케라스 모델이 완성되었습니다. `MultiWorkerMirroredStrategy`를 사용하여 여러 워커를 사용하여 훈련할 수 있습니다. 다중 워커 훈련의 성능을 더 높이려면 다음 기법들을 확인해 보십시오.\n", - "\n", - "\n", - "* `MultiWorkerMirroredStrategy`는 여러 가지 [수집 작업 통신 구현체](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/distribute/cross_device_ops.py)를 제공합니다. `RING`(링)은 링 구조 기반의 수집 작업 구현체이고, 장비 간 통신을 위하여 gRPC를 사용합니다. `NCCL`은 [Nvidia의 NCCL](https://developer.nvidia.com/nccl)로 수집 작업을 구현한 것입니다. `AUTO`를 지정하면, 런타임이 알아서 선택합니다. 어떤 수집 작업 구현체가 최적인지는 GPU의 종류와 수, 클러스터 내 네트워크 연결 등 여러 요소에 따라 달라집니다. 런타임이 알아서 선택한 것을 바꾸려면, `MultiWorkerMirroredStrategy` 생성자의 `communication` 매개변수에 적절한 값을 지정하십시오. 예를 들면 `communication=tf.distribute.experimental.CollectiveCommunication.NCCL`과 같이 지정합니다.\n", - "* 가능하면 변수를 `tf.float` 타입으로 바꾸십시오. 공식 ResNet 모델을 보면 어떻게 바꾸는지 [예제](https://github.com/tensorflow/models/blob/8367cf6dabe11adf7628541706b660821f397dce/official/resnet/resnet_model.py#L466)가 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "97WhAu8uKw3j" - }, - "source": [ - "## 내결함성\n", - "\n", - "동기 훈련 방식에서는, 워커 중 하나가 죽으면 전체 클러스터가 죽어버리고, 복구 메커니즘이 따로 없습니다. 하지만 케라스와 `tf.distribute.Strategy`를 함께 사용하면, 워커가 죽거나 불안정해지는 경우에도 내결함성을 제공합니다. 이는 사용자가 선택한 분산 파일 시스템에 훈련 상태를 저장하는 기능을 제공하기 때문입니다. 기존 인스턴스가 죽거나 정지당해서 재시작되는 경우에도 훈련 상태를 복구할 수 있습니다.\n", - "\n", - "모든 워커가 훈련 에포크 혹은 스텝에 따라 동기화되므로, 다른 워커들은 죽거나 정지당한 워커가 복구될 때까지 기다려야 합니다.\n", - "\n", - "### ModelCheckpoint 콜백\n", - "\n", - "다중 워커 훈련의 내결함 기능을 사용하려면, `tf.keras.Model.fit()`를 호출할 때 `tf.keras.callbacks.ModelCheckpoint`의 인스턴스를 제공해야 합니다. 이 콜백이 체크포인트와 훈련 상태를 `ModelCheckpoint`의 `filepath` 매개변수에 지정한 디렉터리에 저장합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIY9vKnUU82o", - "colab": {} - }, - "source": [ - "# `filepath` 매개변수를 모든 워커가 접근할 수 있는 파일 시스템 경로로 바꾸십시오.\n", - "callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath='/tmp/keras-ckpt')]\n", - "with strategy.scope():\n", - " multi_worker_model = build_and_compile_cnn_model()\n", - "multi_worker_model.fit(x=train_datasets, epochs=3, callbacks=callbacks)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ii6VmEdOjkZr" - }, - "source": [ - "워커가 정지당하면, 정지당한 워커가 다시 살아날 때까지 전체 클러스터가 잠시 멈춥니다. 워커가 클러스터에 다시 들어오면, 다른 워커도 재시작됩니다. 모든 워커가 이전에 저장한 체크포인트 파일을 읽고, 예전 상태를 불러오면 클러스터가 다시 일관된 상태가 됩니다. 그리고서 훈련이 재개됩니다.\n", - "\n", - "`ModelCheckpoint`의 `filepath`가 위치한 디렉터리를 살펴보면, 임시로 생성된 체크포인트 파일들을 확인할 수 있을 것입니다. 이 파일들은 실패한 작업을 복구하는데 필요한 것들로, 다중 워커 훈련 작업을 성공적으로 마치고 나면 `tf.keras.Model.fit()` 함수가 끝날 때 라이브러리가 알아서 삭제할 것입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ega2hdOQEmy_" - }, - "source": [ - "## 참조\n", - "1. [텐서플로로 분산 훈련하기](https://www.tensorflow.org/guide/distributed_training) 가이드는 사용 가능한 분산 전략들을 개괄하고 있습니다.\n", - "2. 공식 [ResNet50](https://github.com/tensorflow/models/blob/master/official/resnet/imagenet_main.py) 모델은 `MirroredStrategy`나 `MultiWorkerMirroredStrategy`로 훈련할 수 있습니다." - ] - } - ] -} \ No newline at end of file diff --git a/site/ko/tutorials/estimator/linear.ipynb b/site/ko/tutorials/estimator/linear.ipynb deleted file mode 100644 index 129f833bf8f..00000000000 --- a/site/ko/tutorials/estimator/linear.ipynb +++ /dev/null @@ -1,683 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "linear.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OoasdhSAp0zJ" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "cIrwotvGqsYh", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C81KT2D_j-xR" - }, - "source": [ - "# 추정기(Estimator)로 선형 모델 만들기\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tUP8LMdYtWPz" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/guide/gpu.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nNgxmCgx8aAM" - }, - "source": [ - "## 개요\n", - "\n", - "이 문서에서는 `tf.estimator` API를 사용하여 로지스틱 회귀 모델(logistic regression model)을 훈련합니다. 이 모델은 다른 더 복잡한 알고리즘의 기초로 사용할 수 있습니다.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkC_j6VpqrDw" - }, - "source": [ - "## 설정" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rutbJGmpqvm3", - "colab": {} - }, - "source": [ - "!pip install sklearn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "54mb4J9PqqDh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "import sys\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "from IPython.display import clear_output\n", - "from six.moves import urllib" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fsjkwfsGOBMT" - }, - "source": [ - "## 타이타닉 데이터셋을 불러오기\n", - "타이타닉 데이터셋을 사용할 것입니다. 성별, 나이, 클래스, 기타 등 주어진 정보를 활용하여 승객이 살아남을 것인지 예측하는 것을 목표로 합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bNiwh-APcRVD", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow.compat.v2.feature_column as fc\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DSeMKcx03d5R", - "colab": {} - }, - "source": [ - "# 데이터셋 불러오기.\n", - "dftrain = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/train.csv')\n", - "dfeval = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/eval.csv')\n", - "y_train = dftrain.pop('survived')\n", - "y_eval = dfeval.pop('survived')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jjm4Qj0u7_cp" - }, - "source": [ - "## 데이터 탐험하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UrQzxKKh4d6u" - }, - "source": [ - "데이터셋은 다음의 특성을 가집니다" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rTjugo3n308g", - "colab": {} - }, - "source": [ - "dftrain.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y86q1fj44lZs", - "colab": {} - }, - "source": [ - "dftrain.describe()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8JSa_duD4tFZ" - }, - "source": [ - "훈련셋은 627개의 샘플로 평가셋은 264개의 샘플로 구성되어 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Fs3Nu5pV4v5J", - "colab": {} - }, - "source": [ - "dftrain.shape[0], dfeval.shape[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RxCA4Nr45AfF" - }, - "source": [ - "대부분의 승객은 20대와 30대 입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RYeCMm7K40ZN", - "colab": {} - }, - "source": [ - "dftrain.age.hist(bins=20)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DItSwJ_B5B0f" - }, - "source": [ - "남자 승객이 여자 승객보다 대략 2배 많습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "b03dVV9q5Dv2", - "colab": {} - }, - "source": [ - "dftrain.sex.value_counts().plot(kind='barh')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rK6WQ29q5Jf5" - }, - "source": [ - "대부분의 승객은 \"삼등석\" 입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dgpJVeCq5Fgd", - "colab": {} - }, - "source": [ - "dftrain['class'].value_counts().plot(kind='barh')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FXJhGGL85TLp" - }, - "source": [ - "여자는 남자보다 살아남을 확률이 훨씬 높습니다. 이는 명확하게 모델에 유용한 특성입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lSZYa7c45Ttt", - "colab": {} - }, - "source": [ - "pd.concat([dftrain, y_train], axis=1).groupby('sex').survived.mean().plot(kind='barh').set_xlabel('% survive')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VqDKQLZn8L-B" - }, - "source": [ - "## 모델을 위한 특성 공학(feature engineering)\n", - "추정기는 [특성 열(feature columns)](https://www.tensorflow.org/guide/feature_columns)이라는 시스템을 사용하여 모델이 각각의 입력 특성을 어떻게 해석할지 설명합니다. 추정기가 숫자 입력 벡터를 요구하면, *특성 열*은 모델이 어떻게 각 특성을 변환해야하는지 설명합니다.\n", - "\n", - "효과적인 모델 학습에서는 적절한 특성 열을 고르고 다듬는 것이 키포인트 입니다. 하나의 특성 열은 특성 딕셔너리(dict)의 원본 입력으로 만들어진 열(*기본 특성 열*)이거나 하나 이상의 기본 열(*얻어진 특성 열*)에 정의된 변환을 이용하여 새로 생성된 열입니다.\n", - "\n", - "선형 추정기는 수치형, 범주형 특성을 모두 사용할 수 있습니다. 특성 열은 모든 텐서플로 추정기와 함께 작동하고 목적은 모델링에 사용되는 특성들을 정의하는 것입니다. 또한 원-핫-인코딩(one-hot-encoding), 정규화(normalization), 버킷화(bucketization)와 같은 특성 공학 방법을 지원합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "puZFOhTDkblt" - }, - "source": [ - "### 기본 특성 열" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GpveXYSsADS6", - "colab": {} - }, - "source": [ - "CATEGORICAL_COLUMNS = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',\n", - " 'embark_town', 'alone']\n", - "NUMERIC_COLUMNS = ['age', 'fare']\n", - "\n", - "feature_columns = []\n", - "for feature_name in CATEGORICAL_COLUMNS:\n", - " vocabulary = dftrain[feature_name].unique()\n", - " feature_columns.append(tf.feature_column.categorical_column_with_vocabulary_list(feature_name, vocabulary))\n", - "\n", - "for feature_name in NUMERIC_COLUMNS:\n", - " feature_columns.append(tf.feature_column.numeric_column(feature_name, dtype=tf.float32))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gt8HMtwOh9lJ" - }, - "source": [ - "`input_function`은 입력 파이프라인을 스트리밍으로 공급하는 `tf.data.Dataset`으로 데이터를 변환하는 방법을 명시합니다. `tf.data.Dataset`은 데이터 프레임, CSV 형식 파일 등과 같은 여러 소스를 사용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qVtrIHFnAe7w", - "colab": {} - }, - "source": [ - "def make_input_fn(data_df, label_df, num_epochs=10, shuffle=True, batch_size=32):\n", - " def input_function():\n", - " ds = tf.data.Dataset.from_tensor_slices((dict(data_df), label_df))\n", - " if shuffle:\n", - " ds = ds.shuffle(1000)\n", - " ds = ds.batch(batch_size).repeat(num_epochs)\n", - " return ds\n", - " return input_function\n", - "\n", - "train_input_fn = make_input_fn(dftrain, y_train)\n", - "eval_input_fn = make_input_fn(dfeval, y_eval, num_epochs=1, shuffle=False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P7UMVkQnkrgb" - }, - "source": [ - "다음과 같이 데이터셋을 점검할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8ZcG_3KiCb1M", - "colab": {} - }, - "source": [ - "ds = make_input_fn(dftrain, y_train, batch_size=10)()\n", - "for feature_batch, label_batch in ds.take(1):\n", - " print('특성 키:', list(feature_batch.keys()))\n", - " print()\n", - " print('클래스 배치:', feature_batch['class'].numpy())\n", - " print()\n", - " print('레이블 배치:', label_batch.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lMNBMyodjlW3" - }, - "source": [ - "또한 `tf.keras.layers.DenseFeatures` 층을 사용하여 특정한 특성 열의 결과를 점검할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IMjlmbPlDmkB", - "colab": {} - }, - "source": [ - "age_column = feature_columns[7]\n", - "tf.keras.layers.DenseFeatures([age_column])(feature_batch).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f4zrAdCIjr3s" - }, - "source": [ - "`DenseFeatures`는 조밀한(dense) 텐서만 허용합니다. 범주형 데이터를 점검하려면 우선 범주형 열에 indicator_column 함수를 적용해야 합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1VXmXFTSFEvv", - "colab": {} - }, - "source": [ - "gender_column = feature_columns[0]\n", - "tf.keras.layers.DenseFeatures([tf.feature_column.indicator_column(gender_column)])(feature_batch).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MEp59g5UkHYY" - }, - "source": [ - "모든 기본 특성을 모델에 추가한 다음에 모델을 훈련해 봅시다. 모델을 훈련하려면 `tf.estimator` API를 이용한 메서드 호출 한번이면 충분합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aGXjdnqqdgIs", - "colab": {} - }, - "source": [ - "linear_est = tf.estimator.LinearClassifier(feature_columns=feature_columns)\n", - "linear_est.train(train_input_fn)\n", - "result = linear_est.evaluate(eval_input_fn)\n", - "\n", - "clear_output()\n", - "print(result)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3tOan4hDsG6d" - }, - "source": [ - "### 도출된 특성 열" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NOG2FSTHlAMu" - }, - "source": [ - "이제 정확도 75%에 도달했습니다. 별도로 각 기본 특성 열을 사용하면 데이터를 설명하기에는 충분치 않을 수 있습니다. 예를 들면, 성별과 레이블간의 상관관계는 성별에 따라 다를 수 있습니다. 따라서 `gender=\"Male\"`과 'gender=\"Female\"`의 단일 모델가중치만 배우면 모든 나이-성별 조합(이를테면 `gender=\"Male\" 그리고 'age=\"30\"` 그리고 `gender=\"Male\"` 그리고 `age=\"40\"`을 구별하는 것)을 포함시킬 수 없습니다.\n", - "\n", - "서로 다른 특성 조합들 간의 차이를 학습하기 위해서 모델에 *교차 특성 열*을 추가할 수 있습니다(또한 교차 열 이전에 나이 열을 버킷화할 수 있습니다):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AM-RsDzNfGlu", - "colab": {} - }, - "source": [ - "age_x_gender = tf.feature_column.crossed_column(['age', 'sex'], hash_bucket_size=100)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DqDFyPKQmGTN" - }, - "source": [ - "조합 특성을 모델에 추가하고 모델을 다시 훈련합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s8FV9oPQfS-g", - "colab": {} - }, - "source": [ - "derived_feature_columns = [age_x_gender]\n", - "linear_est = tf.estimator.LinearClassifier(feature_columns=feature_columns+derived_feature_columns)\n", - "linear_est.train(train_input_fn)\n", - "result = linear_est.evaluate(eval_input_fn)\n", - "\n", - "clear_output()\n", - "print(result)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rwfdZj7ImLwb" - }, - "source": [ - "이제 정확도 77.6%에 도달했습니다. 기본 특성만 이용한 학습보다는 약간 더 좋았습니다. 더 많은 특성과 변환을 사용해서 더 잘할 수 있다는 것을 보여주세요!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8_eyb9d-ncjH" - }, - "source": [ - "이제 훈련 모델을 이용해서 평가셋에서 승객에 대해 예측을 할 수 있습니다. 텐서플로 모델은 한번에 샘플의 배치 또는 일부에 대한 예측을 하도록 최적화되어있습니다. 앞서, `eval_input_fn`은 모든 평가셋을 사용하도록 정의되어 있었습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wiScyBcef6Dq", - "colab": {} - }, - "source": [ - "pred_dicts = list(linear_est.predict(eval_input_fn))\n", - "probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])\n", - "\n", - "probs.plot(kind='hist', bins=20, title='예측 확률')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UEHRCd4sqrLs" - }, - "source": [ - "마지막으로, 수신자 조작 특성(receiver operating characteristic, ROC)을 살펴보면 정탐률(true positive rate)과 오탐률(false positive rate)의 상충관계에 대해 더 잘 이해할 수 있습니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kqEjsezIokIe", - "colab": {} - }, - "source": [ - "from sklearn.metrics import roc_curve\n", - "from matplotlib import pyplot as plt\n", - "\n", - "fpr, tpr, _ = roc_curve(y_eval, probs)\n", - "plt.plot(fpr, tpr)\n", - "plt.title('ROC curve')\n", - "plt.xlabel('오탐률(false positive rate)')\n", - "plt.ylabel('정탐률(true positive rate)')\n", - "plt.xlim(0,)\n", - "plt.ylim(0,)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ko/tutorials/generative/dcgan.ipynb b/site/ko/tutorials/generative/dcgan.ipynb deleted file mode 100644 index 45ea91b1153..00000000000 --- a/site/ko/tutorials/generative/dcgan.ipynb +++ /dev/null @@ -1,917 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_jQ1tEQCxwRx" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "V_sgB_5dx1f1" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rF2x3qooyBTI" - }, - "source": [ - "# 심층 합성곱 생성적 적대 신경망" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TD5ZrvEMbhZ" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/generative/dcgan\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " TensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/generative/dcgan.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " 구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/generative/dcgan.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " 깃허브(GitHub)소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBxnTS5bBk2d" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 공식 영문 문서의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ITZuApL56Mny" - }, - "source": [ - "이 튜토리얼은 [심층 합성곱 생성적 적대 신경망](https://arxiv.org/pdf/1511.06434.pdf) (Deep Convolutional Generative Adversarial Networks, DCGAN)을 이용하여, 손으로 쓴 숫자들을 어떻게 생성할 수 있는지 보여줍니다. 이 코드는 [케라스 Sequential API](https://www.tensorflow.org/guide/keras)와 `tf.GradientTape` 훈련 루프를 사용하여 작성됐습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2MbKJY38Puy9" - }, - "source": [ - "## 생성적 적대 신경망(GANs)은 무엇인가요? \n", - "\n", - "[생성적 적대 신경망](https://arxiv.org/abs/1406.2661) (Generative Adversarial Networks, GANs)은 요즘 컴퓨터 과학에서 가장 흥미로운 아이디어 중 하나입니다. 두개의 모델이 적대적인 과정을 통해 동시에 훈련됩니다. *생성자* (\"예술가\")는 진짜처럼 보이는 이미지를 생성하도록 배우는 와중에, *감별자* (\"예술비평가\")는 가짜의 이미지로부터 진짜를 구별하게 되는 것을 배우게 됩니다.\n", - "\n", - "![생성자와 감별자를 그린 도표](https://tensorflow.org/tutorials/generative/images/gan1.png)\n", - "\n", - "\n", - "\n", - "훈련과정 동안 *생성자*는 점차 실제같은 이미지를 더 잘 생성하게 되고, *감별자*는 점차 진짜와 가짜를 더 잘 구별하게됩니다. 이 과정은 *감별자*가 가짜 이미지에서 진짜 이미지를 더이상 구별하지 못하게 될때, 평형상태에 도달하게 됩니다. \n", - "\n", - "![생성자와 감별자를 그린 두번째 도표](https://tensorflow.org/tutorials/generative/images/gan2.png)\n", - "\n", - "이 노트북은 이 과정을 MNIST 데이터를 이용하여 보여줍니다. 아래의 애니메이션은 50 에포크(epoch)동안 훈련한 *생성자*가 생성해낸 연속된 이미지들을 보여줍니다. 이미지들은 랜덤한 잡음으로 부터 시작되었고, 점차 시간이 지남에 따라 손으로 쓴 숫자들을 닮아가게 됩니다.\n", - "\n", - "![출력 예시](https://tensorflow.org/images/gan/dcgan.gif)\n", - "\n", - "생성적 적대 신경망 (GANs)에 대해 더 배우고 싶으시다면, MIT의 [Intro to Deep Learning](http://introtodeeplearning.com/) 수업을 추천합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e1_Y75QXJS6h" - }, - "source": [ - "### 텐서플로와 다른 라이브러리 불러오기" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "J5oue0oqCkZZ" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "g5RstiiB8V-z" - }, - "outputs": [], - "source": [ - "!pip install tensorflow-gpu==2.0.0-rc1" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WZKbyU2-AiY-" - }, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wx-zNbLqB4K8" - }, - "outputs": [], - "source": [ - "tf.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YzTlj4YdCip_" - }, - "outputs": [], - "source": [ - "# GIF를 만들기위해 설치합니다.\n", - "!pip install imageio" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YfIk2es3hJEd" - }, - "outputs": [], - "source": [ - "import glob\n", - "import imageio\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import os\n", - "import PIL\n", - "from tensorflow.keras import layers\n", - "import time\n", - "\n", - "from IPython import display" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iYn4MdZnKCey" - }, - "source": [ - "### 데이터셋 로딩 및 준비\n", - "생성자와 감별자를 훈련하기위해 MNIST 데이터셋을 사용할것입니다. 생성자는 손글씨 숫자 데이터를 닮은 숫자들을 생성할 것입니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "a4fYMGxGhrna" - }, - "outputs": [], - "source": [ - "(train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NFC2ghIdiZYE" - }, - "outputs": [], - "source": [ - "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n", - "train_images = (train_images - 127.5) / 127.5 # 이미지를 [-1, 1]로 정규화합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "S4PIDhoDLbsZ" - }, - "outputs": [], - "source": [ - "BUFFER_SIZE = 60000\n", - "BATCH_SIZE = 256" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-yKCCQOoJ7cn" - }, - "outputs": [], - "source": [ - "# 데이터 배치를 만들고 섞습니다.\n", - "train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "THY-sZMiQ4UV" - }, - "source": [ - "## 모델 만들기 \n", - "생성자와 감별자는 [케라스 Sequential API](https://www.tensorflow.org/guide/keras#sequential_model)를 이용해 정의됩니다. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-tEyxE-GMC48" - }, - "source": [ - "### 생성자\n", - "\n", - "생성자는 시드값 (seed; 랜덤한 잡음)으로부터 이미지를 생성하기 위해, `tf.keras.layers.Conv2DTranspose` (업샘플링) 층을 이용합니다. 처음 `Dense`층은 이 시드값을 인풋으로 받습니다. 그 다음 원하는 사이즈 28x28x1의 이미지가 나오도록 업샘플링을 여러번 합니다. tanh를 사용하는 마지막 층을 제외한 나머지 각 층마다 활성함수로 `tf.keras.layers.LeakyReLU`을 사용하고 있음을 주목합시다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6bpTcDqoLWjY" - }, - "outputs": [], - "source": [ - "def make_generator_model():\n", - " model = tf.keras.Sequential()\n", - " model.add(layers.Dense(7*7*256, use_bias=False, input_shape=(100,)))\n", - " model.add(layers.BatchNormalization())\n", - " model.add(layers.LeakyReLU())\n", - "\n", - " model.add(layers.Reshape((7, 7, 256)))\n", - " assert model.output_shape == (None, 7, 7, 256) # 주목: 배치사이즈로 None이 주어집니다.\n", - "\n", - " model.add(layers.Conv2DTranspose(128, (5, 5), strides=(1, 1), padding='same', use_bias=False))\n", - " assert model.output_shape == (None, 7, 7, 128)\n", - " model.add(layers.BatchNormalization())\n", - " model.add(layers.LeakyReLU())\n", - "\n", - " model.add(layers.Conv2DTranspose(64, (5, 5), strides=(2, 2), padding='same', use_bias=False))\n", - " assert model.output_shape == (None, 14, 14, 64)\n", - " model.add(layers.BatchNormalization())\n", - " model.add(layers.LeakyReLU())\n", - "\n", - " model.add(layers.Conv2DTranspose(1, (5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))\n", - " assert model.output_shape == (None, 28, 28, 1)\n", - "\n", - " return model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GyWgG09LCSJl" - }, - "source": [ - "(아직 훈련이 되지않은) 생성자를 이용해 이미지를 생성해봅시다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gl7jcC7TdPTG" - }, - "outputs": [], - "source": [ - "generator = make_generator_model()\n", - "\n", - "noise = tf.random.normal([1, 100])\n", - "generated_image = generator(noise, training=False)\n", - "\n", - "plt.imshow(generated_image[0, :, :, 0], cmap='gray')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D0IKnaCtg6WE" - }, - "source": [ - "### 감별자 \n", - "감별자는 합성곱 신경망(Convolutional Neural Network, CNN) 기반의 이미지 분류기입니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dw2tPLmk2pEP" - }, - "outputs": [], - "source": [ - "def make_discriminator_model():\n", - " model = tf.keras.Sequential()\n", - " model.add(layers.Conv2D(64, (5, 5), strides=(2, 2), padding='same',\n", - " input_shape=[28, 28, 1]))\n", - " model.add(layers.LeakyReLU())\n", - " model.add(layers.Dropout(0.3))\n", - "\n", - " model.add(layers.Conv2D(128, (5, 5), strides=(2, 2), padding='same'))\n", - " model.add(layers.LeakyReLU())\n", - " model.add(layers.Dropout(0.3))\n", - "\n", - " model.add(layers.Flatten())\n", - " model.add(layers.Dense(1))\n", - "\n", - " return model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QhPneagzCaQv" - }, - "source": [ - "(아직까지 훈련이 되지 않은) 감별자를 사용하여, 생성된 이미지가 진짜인지 가짜인지 판별합니다. 모델은 진짜 이미지에는 양수의 값 (positive values)을, 가짜 이미지에는 음수의 값 (negative values)을 출력하도록 훈련되어집니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gDkA05NE6QMs" - }, - "outputs": [], - "source": [ - "discriminator = make_discriminator_model()\n", - "decision = discriminator(generated_image)\n", - "print (decision)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0FMYgY_mPfTi" - }, - "source": [ - "## 손실함수와 옵티마이저 정의\n", - "두 모델의 손실함수와 옵티마이저를 정의합니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "psQfmXxYKU3X" - }, - "outputs": [], - "source": [ - "# 이 메서드는 크로스 엔트로피 손실함수 (cross entropy loss)를 계산하기 위해 헬퍼 (helper) 함수를 반환합니다.\n", - "cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PKY_iPSPNWoj" - }, - "source": [ - "### 감별자 손실함수\n", - "\n", - "이 메서드는 감별자가 가짜 이미지에서 얼마나 진짜 이미지를 잘 판별하는지 수치화합니다. 진짜 이미지에 대한 감별자의 예측과 1로 이루어진 행렬을 비교하고, 가짜 (생성된) 이미지에 대한 감별자의 예측과 0으로 이루어진 행렬을 비교합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wkMNfBWlT-PV" - }, - "outputs": [], - "source": [ - "def discriminator_loss(real_output, fake_output):\n", - " real_loss = cross_entropy(tf.ones_like(real_output), real_output)\n", - " fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)\n", - " total_loss = real_loss + fake_loss\n", - " return total_loss" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Jd-3GCUEiKtv" - }, - "source": [ - "### 생성자 손실함수\n", - "\n", - "생성자의 손실함수는 감별자를 얼마나 잘 속였는지에 대해 수치화를 합니다. 직관적으로 생성자가 원활히 수행되고 있다면, 감별자는 가짜 이미지를 진짜 (또는 1)로 분류를 할 것입니다. 여기서 우리는 생성된 이미지에 대한 감별자의 결정을 1로 이루어진 행렬과 비교를 할 것입니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "90BIcCKcDMxz" - }, - "outputs": [], - "source": [ - "def generator_loss(fake_output):\n", - " return cross_entropy(tf.ones_like(fake_output), fake_output)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MgIc7i0th_Iu" - }, - "source": [ - "감별자와 생성자는 따로 훈련되기 때문에, 감별자와 생성자의 옵티마이저는 다릅니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iWCn_PVdEJZ7" - }, - "outputs": [], - "source": [ - "generator_optimizer = tf.keras.optimizers.Adam(1e-4)\n", - "discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mWtinsGDPJlV" - }, - "source": [ - "### 체크포인트 저장\n", - "이 노트북은 오랫동안 진행되는 훈련이 방해되는 경우에 유용하게 쓰일 수 있는 모델의 저장방법과 복구방법을 보여줍니다. " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "CA1w-7s2POEy" - }, - "outputs": [], - "source": [ - "checkpoint_dir = './training_checkpoints'\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,\n", - " discriminator_optimizer=discriminator_optimizer,\n", - " generator=generator,\n", - " discriminator=discriminator)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Rw1fkAczTQYh" - }, - "source": [ - "## 훈련 루프 정의하기" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NS2GWywBbAWo" - }, - "outputs": [], - "source": [ - "EPOCHS = 50\n", - "noise_dim = 100\n", - "num_examples_to_generate = 16\n", - "\n", - "# 이 시드를 시간이 지나도 재활용하겠습니다. \n", - "# (GIF 애니메이션에서 진전 내용을 시각화하는데 쉽기 때문입니다.) \n", - "seed = tf.random.normal([num_examples_to_generate, noise_dim])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jylSonrqSWfi" - }, - "source": [ - "훈련 루프는 생성자가 입력으로 랜덤시드를 받는 것으로부터 시작됩니다. 그 시드값을 사용하여 이미지를 생성합니다. 감별자를 사용하여 (훈련 세트에서 갖고온) 진짜 이미지와 (생성자가 생성해낸) 가짜이미지를 분류합니다. 각 모델의 손실을 계산하고, 그래디언트 (gradients)를 사용해 생성자와 감별자를 업데이트합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3t5ibNo05jCB" - }, - "outputs": [], - "source": [ - "# `tf.function`이 어떻게 사용되는지 주목해 주세요.\n", - "# 이 데코레이터는 함수를 \"컴파일\"합니다.\n", - "@tf.function\n", - "def train_step(images):\n", - " noise = tf.random.normal([BATCH_SIZE, noise_dim])\n", - "\n", - " with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n", - " generated_images = generator(noise, training=True)\n", - "\n", - " real_output = discriminator(images, training=True)\n", - " fake_output = discriminator(generated_images, training=True)\n", - "\n", - " gen_loss = generator_loss(fake_output)\n", - " disc_loss = discriminator_loss(real_output, fake_output)\n", - "\n", - " gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)\n", - " gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)\n", - "\n", - " generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))\n", - " discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2M7LmLtGEMQJ" - }, - "outputs": [], - "source": [ - "def train(dataset, epochs):\n", - " for epoch in range(epochs):\n", - " start = time.time()\n", - "\n", - " for image_batch in dataset:\n", - " train_step(image_batch)\n", - "\n", - " # GIF를 위한 이미지를 바로 생성합니다.\n", - " display.clear_output(wait=True)\n", - " generate_and_save_images(generator,\n", - " epoch + 1,\n", - " seed)\n", - "\n", - " # 15 에포크가 지날 때마다 모델을 저장합니다.\n", - " if (epoch + 1) % 15 == 0:\n", - " checkpoint.save(file_prefix = checkpoint_prefix)\n", - " \n", - " # print (' 에포크 {} 에서 걸린 시간은 {} 초 입니다'.format(epoch +1, time.time()-start))\n", - " print ('Time for epoch {} is {} sec'.format(epoch + 1, time.time()-start))\n", - "\n", - " # 마지막 에포크가 끝난 후 생성합니다.\n", - " display.clear_output(wait=True)\n", - " generate_and_save_images(generator,\n", - " epochs,\n", - " seed)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2aFF7Hk3XdeW" - }, - "source": [ - "**이미지 생성 및 저장**\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RmdVsmvhPxyy" - }, - "outputs": [], - "source": [ - "def generate_and_save_images(model, epoch, test_input):\n", - " # `training`이 False로 맞춰진 것을 주목하세요.\n", - " # 이렇게 하면 (배치정규화를 포함하여) 모든 층들이 추론 모드로 실행됩니다. \n", - " predictions = model(test_input, training=False)\n", - "\n", - " fig = plt.figure(figsize=(4,4))\n", - "\n", - " for i in range(predictions.shape[0]):\n", - " plt.subplot(4, 4, i+1)\n", - " plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')\n", - " plt.axis('off')\n", - "\n", - " plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n", - " plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dZrd4CdjR-Fp" - }, - "source": [ - "## 모델 훈련\n", - "위에 정의된 `train()` 메서드를 생성자와 감별자를 동시에 훈련하기 위해 호출합니다. 생성적 적대 신경망을 학습하는 것은 매우 까다로울 수 있습니다. 생성자와 감별자가 서로를 제압하지 않는 것이 중요합니다. (예를 들어 학습률이 비슷하면 한쪽이 우세해집니다.)\n", - "훈련 초반부에는 생성된 이미지는 랜덤한 노이즈처럼 보입니다. 훈련이 진행될수록, 생성된 숫자는 점차 진짜처럼 보일 것입니다. 약 50 에포크가 지난 후, MNIST 숫자와 닮은 이미지가 생성됩니다. 코랩에서 기본 설정으로 실행하면, 에포크마다 1분정도 소요될 것입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Ly3UN0SLLY2l" - }, - "outputs": [], - "source": [ - "%%time\n", - "train(train_dataset, EPOCHS)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rfM4YcPVPkNO" - }, - "source": [ - "마지막 체크포인트를 복구합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "XhXsd0srPo8c" - }, - "outputs": [], - "source": [ - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P4M_vIbUi7c0" - }, - "source": [ - "## GIF 생성" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WfO5wCdclHGL" - }, - "outputs": [], - "source": [ - "# 에포크 숫자를 사용하여 하나의 이미지를 보여줍니다.\n", - "def display_image(epoch_no):\n", - " return PIL.Image.open('image_at_epoch_{:04d}.png'.format(epoch_no))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5x3q9_Oe5q0A" - }, - "outputs": [], - "source": [ - "display_image(EPOCHS)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NywiH3nL8guF" - }, - "source": [ - "`imageio`로 훈련 중에 저장된 이미지를 사용해 GIF 애니메이션을 만듭니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IGKQgENQ8lEI" - }, - "outputs": [], - "source": [ - "anim_file = 'dcgan.gif'\n", - "\n", - "with imageio.get_writer(anim_file, mode='I') as writer:\n", - " filenames = glob.glob('image*.png')\n", - " filenames = sorted(filenames)\n", - " last = -1\n", - " for i,filename in enumerate(filenames):\n", - " frame = 2*(i**0.5)\n", - " if round(frame) \u003e round(last):\n", - " last = frame\n", - " else:\n", - " continue\n", - " image = imageio.imread(filename)\n", - " writer.append_data(image)\n", - " image = imageio.imread(filename)\n", - " writer.append_data(image)\n", - "\n", - "import IPython\n", - "if IPython.version_info \u003e (6,2,0,''):\n", - " display.Image(filename=anim_file)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cGhC3-fMWSwl" - }, - "source": [ - "코랩에서 작업하고 있다면, 아래의 코드에서 애니메이션을 다운로드 받을 수 있습니다: " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uV0yiKpzNP1b" - }, - "outputs": [], - "source": [ - "try:\n", - " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download(anim_file)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k6qC-SbjK0yW" - }, - "source": [ - "## 다음 단계" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xjjkT9KAK6H7" - }, - "source": [ - "이 튜토리얼은 생성적 적대 신경망을 만들고 훈련하기에 필요한 코드를 보여줍니다. 다음 단계로, 다른 데이터셋을 이용하여 실험해보고 싶을 수도 있습니다. 예를 들면 [캐글에 올라온](https://www.kaggle.com/jessicali9530/celeba-dataset/home) 대규모 연예인 얼굴 데이터셋 (Large-scale Celeb Faces Attributes (CelebA))이 있습니다. 생성적 적대 신경망에 대해 더 배우기 원한다면, [NIPS 2016 튜토리얼: 생성적 적대 신경망](https://arxiv.org/abs/1701.00160)을 추천합니다." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "dcgan.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/generative/style_transfer.ipynb b/site/ko/tutorials/generative/style_transfer.ipynb deleted file mode 100644 index ab673c9e808..00000000000 --- a/site/ko/tutorials/generative/style_transfer.ipynb +++ /dev/null @@ -1,1267 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "style_transfer.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "cell_type": "markdown", - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "cell_type": "code", - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "jo5PziEC4hWs" - }, - "cell_type": "markdown", - "source": [ - "# tf.keras를 사용한 Neural Style Transfer\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "metadata": { - "id": "yJ-t14sot8iX", - "colab_type": "text" - }, - "cell_type": "markdown", - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "aDyGj8DmXCJI" - }, - "cell_type": "markdown", - "source": [ - "## 개요\n", - "\n", - "이번 튜토리얼에서는 딥러닝을 사용하여 원하는 이미지를 다른 스타일의 이미지로 구성하는 법을 배워보겠습니다(피카소나 반 고흐처럼 그리기를 희망하시나요?). 이 기법은 **Neural Style Transfer**로 알려져있으며, Leon A. Gatys의 논문 [A Neural Algorithm of Artistic Style](https://arxiv.org/abs/1508.06576)에 잘 기술되어 있습니다. 꼭 읽어보셔야합니다.\n", - "\n", - "하지만 무엇이 Neural Style Transfer 일까요?\n", - "\n", - "Neural style transfer은 **콘텐츠(Content)** 이미지, **스타일 참조(Style Reference)** 이미지(유명한 작가의 삽화 같은), 그리고 여러분이 변경하기 원하는 **입력(Input)** 이미지 위 3가지의 이미지를 사용하여, 콘텐츠 이미지의 콘텐츠로 변형된 것처럼 보이고, 스타일 참조 이미지로 채색된 것 같은 입력 이미지를 생성하는 최적화 기술입니다.\n", - "\n", - "예를 들어, 아래의 거북이 이미지와 가츠시카 호쿠사이(Katsushika Hokusai) 작가의 *The Great Wave off Kanagawa* 이미지를 사용해봅시다.\n", - "\n", - "\"Drawing\"\n", - "\"Drawing\"\n", - "\n", - "[Image of Green Sea Turtle](https://commons.wikimedia.org/wiki/File:Green_Sea_Turtle_grazing_seagrass.jpg)\n", - "-By P.Lindgren [CC BY-SA 3.0 (https://creativecommons.org/licenses/by-sa/3.0)], from Wikimedia Common\n", - "\n", - "\n", - "만약 가츠시카 호쿠사이 작가가 오직 위 스타일로 거북이를 그린다면 어떻게 생겼을까요? 위와 비슷할까요?\n", - "\n", - "\"Drawing\"\n", - "\n", - "이것은 마법일까요 아니면 그저 딥러닝일까요? 다행스럽게도, 이것은 어떠한 마법도 포함하고있지 않습니다. 스타일 변환은 신경망의 기능과 내부표현을 보여주는 재밌고 흥미로운 기술입니다.\n", - "\n", - "Neural style transfer의 원리는 두 이미지의 콘텐츠의 차이를 표현하는 함수인, $L_{content}$, 두 이미지의 스타일의 차이를 표현하는 함수인, $L_{style}$, 위 두 가지 거리함수를 정의하는 것입니다. 원하는 스타일 이미지, 콘텐츠 이미지 그리고 입력 이미지(콘텐츠 이미지로 초기화 된) 이 3가지의 이미지가 주어졌을 때, 콘텐츠 이미지의 콘텐츠와 스타일 이미지의 스타일 각각의 거리를 최소화하는 입력 이미지의 변형을 시도할 것입니다.\n", - "\n", - "요약하자면, 기본 입력 이미지, 일치시키려는 콘텐츠 및 스타일 이미지를 가져와, 역전파(backpropagation)를 이용하여 콘텐츠와 스타일 거리(손실)함수를 최소화시켜 콘텐츠 이미지의 콘텐츠와 스타일 이미지의 스타일이 매치된 이미지를 생성할 것입니다.\n", - "\n", - "### 적용되는 특별개념:\n", - "이 과정에서, 우리는 실제 경험을 쌓고 아래의 개념을 중심으로 직관력을 개발할 것입니다.\n", - "\n", - "* **즉시 실행(Eager Execution)** - 연산을 즉각적으로 평가하는 Tensorflow의 명령형 프로그래밍 환경 사용\n", - " * [Learn more about eager execution](https://www.tensorflow.org/programmers_guide/eager)\n", - " * [See it in action](https://www.tensorflow.org/get_started/eager)\n", - "* ** 모델 정의를 위한 [함수형 API(Functional API)](https://keras.io/getting-started/functional-api-guide/) 사용** - 함수형 API(Functional API)를 사용하여 필요한 중간 활성화에 대한 접근을 제공할 모델의 부분집합을 구성할 것입니다.\n", - "* **사전학습 모델의 특성맵(Feature map) 활용** - 사전학습된 모델과 그 모델의 특성맵에 대해 배우겠습니다.\n", - "* **사용자 지정 훈련 루프 만들기** - 입력변수와 관련하여 주어진 손실함수를 최소화하기 위해 옵티마이저(optimizer)를 설정하는법을 살펴 보겠습니다.\n", - "\n", - "### 우리는 스타일 변형을 수행하기 위해 아래와 같은 일반적인 단계들을 진행할 것입니다.\n", - "\n", - "1. 데이터 시각화(Visualize data)\n", - "2. 기본 전처리 및 데이터 준비(Basic Preprocessing/preparing our data)\n", - "3. 손실함수 설정(Set up loss functions)\n", - "4. 모델 생성(Create model)\n", - "5. 손실함수 최적화(Optimize for loss function)\n", - "\n", - "**참고:** -\t이 게시글은 기본 머신러닝 개념에 익숙한 중간사용자를 대상으로합니다. 이 게시글을 최대한 활용하려면, 아래 사이트를 참조하세요.\n", - "* [Gatys' paper](https://arxiv.org/abs/1508.06576) - 이번 튜토리얼에서 전체적인 과정에 대해 설명할 것이지만, 위 논문은 이 과제에 대한 보다 나은 이해를 제공할 것입니다.\n", - "* [Understand reducing loss with gradient descent](https://developers.google.com/machine-learning/crash-course/reducing-loss/gradient-descent)\n", - "\n", - "**예상시간**: 30분\n" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "U8ajP_u73s6m" - }, - "cell_type": "markdown", - "source": [ - "## 설정\n", - "\n", - "### 다운로드 이미지" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "riWE_b8k3s6o", - "colab": {} - }, - "cell_type": "code", - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import os\n", - "img_dir = '/tmp/nst'\n", - "if not os.path.exists(img_dir):\n", - " os.makedirs(img_dir)\n", - "!wget --quiet -P /tmp/nst/ https://upload.wikimedia.org/wikipedia/commons/d/d7/Green_Sea_Turtle_grazing_seagrass.jpg\n", - "!wget --quiet -P /tmp/nst/ https://upload.wikimedia.org/wikipedia/commons/0/0a/The_Great_Wave_off_Kanagawa.jpg\n", - "!wget --quiet -P /tmp/nst/ https://upload.wikimedia.org/wikipedia/commons/b/b4/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg\n", - "!wget --quiet -P /tmp/nst/ https://upload.wikimedia.org/wikipedia/commons/0/00/Tuebingen_Neckarfront.jpg\n", - "!wget --quiet -P /tmp/nst/ https://upload.wikimedia.org/wikipedia/commons/6/68/Pillars_of_creation_2014_HST_WFC3-UVIS_full-res_denoised.jpg\n", - "!wget --quiet -P /tmp/nst/ https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1024px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "eqxUicSPUOP6" - }, - "cell_type": "markdown", - "source": [ - "### 모듈 구성 및 임포트" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "sc1OLbOWhPCO", - "colab": {} - }, - "cell_type": "code", - "source": [ - "import matplotlib.pyplot as plt\n", - "import matplotlib as mpl\n", - "mpl.rcParams['figure.figsize'] = (10,10)\n", - "mpl.rcParams['axes.grid'] = False\n", - "\n", - "import numpy as np\n", - "from PIL import Image\n", - "import time\n", - "import functools" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "RYEjlrYk3s6w", - "colab": {} - }, - "cell_type": "code", - "source": [ - "import tensorflow as tf\n", - "import tensorflow.contrib.eager as tfe\n", - "\n", - "from tensorflow.python.keras.preprocessing import image as kp_image\n", - "from tensorflow.python.keras import models\n", - "from tensorflow.python.keras import losses\n", - "from tensorflow.python.keras import layers\n", - "from tensorflow.python.keras import backend as K" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "L7sjDODq67HQ" - }, - "cell_type": "markdown", - "source": [ - "우리는 [즉시 실행(eager execution)](https://www.tensorflow.org/guide/eager)을 활성화하여 시작할 것입니다.\n", - "즉시 실행(eager execution)은 가장 명확하고, 읽기 쉬운 방식으로 작업을 할 수 있게 해줍니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "sfjsSAtNrqQx", - "colab": {} - }, - "cell_type": "code", - "source": [ - "tf.enable_eager_execution()\n", - "print(\"Eager execution: {}\".format(tf.executing_eagerly()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "IOiGrIV1iERH", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 일부 전역변수를 여기에 설정하세요.\n", - "content_path = '/tmp/nst/Green_Sea_Turtle_grazing_seagrass.jpg'\n", - "style_path = '/tmp/nst/The_Great_Wave_off_Kanagawa.jpg'" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "xE4Yt8nArTeR" - }, - "cell_type": "markdown", - "source": [ - "## 입력 시각화" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "3TLljcwv5qZs", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def load_img(path_to_img):\n", - " max_dim = 512\n", - " img = Image.open(path_to_img)\n", - " long = max(img.size)\n", - " scale = max_dim/long\n", - " img = img.resize((round(img.size[0]*scale), round(img.size[1]*scale)), Image.ANTIALIAS)\n", - "\n", - " img = kp_image.img_to_array(img)\n", - "\n", - " # 배치 차원(batch dimension)을 갖기 위해 이미지 배열을 브로드캐스팅(broadcasting)할 필요가 있습니다.\n", - " img = np.expand_dims(img, axis=0)\n", - " return img" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "vupl0CI18aAG", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def imshow(img, title=None):\n", - " # 배치 차원(batch dimension) 제거\n", - " out = np.squeeze(img, axis=0)\n", - " # 디스플레이를 위한 정규화\n", - " out = out.astype('uint8')\n", - " plt.imshow(out)\n", - " if title is not None:\n", - " plt.title(title)\n", - " plt.imshow(out)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "2yAlRzJZrWM3" - }, - "cell_type": "markdown", - "source": [ - "여기 주여진 콘텐츠 이미지와 스타일 이미지가 있습니다. 우리는 콘텐츠 이미지의 콘텐츠와 스타일 이미지의 스타일로 구성된 이미지를 생성하기를 원합니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "_UWQmeEaiKkP", - "colab": {} - }, - "cell_type": "code", - "source": [ - "plt.figure(figsize=(10,10))\n", - "\n", - "content = load_img(content_path).astype('uint8')\n", - "style = load_img(style_path).astype('uint8')\n", - "\n", - "plt.subplot(1, 2, 1)\n", - "imshow(content, 'Content Image')\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "imshow(style, 'Style Image')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "7qMVNvEsK-_D" - }, - "cell_type": "markdown", - "source": [ - "## 데이터 준비\n", - "이미지를 쉽게 전처리하고 적재하기위한 메소드(method)를 만들어봅시다. 이 과정에서는 VGG 학습 프로세스에 따라 예상되는 것과 동일한 전처리 과정을 수행할것입니다. VGG 네트워크들은 각각의 채널이 `평균 = [103.939, 116.779, 123.68]` 및 BGR 채널들로 정규화(normalization)되어 있는 이미지로 훈련됩니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "hGwmTwJNmv2a", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def load_and_process_img(path_to_img):\n", - " img = load_img(path_to_img)\n", - " img = tf.keras.applications.vgg19.preprocess_input(img)\n", - " return img" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "xCgooqs6tAka" - }, - "cell_type": "markdown", - "source": [ - "최적화의 결과들을 보기 위해서는, 역전처리(inverse preprocessing) 과정을 수행 할 필요가 있습니다. 더욱이, 최적화 된 이미지의 값이 $- \\infty$ and $\\infty$사이의 어느값도 가질 수 있기 때문에 0-255사이의 값으로 고정해주여야 합니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "mjzlKRQRs_y2", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def deprocess_img(processed_img):\n", - " x = processed_img.copy()\n", - " if len(x.shape) == 4:\n", - " x = np.squeeze(x, 0)\n", - " assert len(x.shape) == 3, (\"Input to deprocess image must be an image of \"\n", - " \"dimension [1, height, width, channel] or [height, width, channel]\")\n", - " if len(x.shape) != 3:\n", - " raise ValueError(\"Invalid input to deprocessing image\")\n", - "\n", - " # 역전처리 과정 수행\n", - " x[:, :, 0] += 103.939\n", - " x[:, :, 1] += 116.779\n", - " x[:, :, 2] += 123.68\n", - " x = x[:, :, ::-1]\n", - "\n", - " x = np.clip(x, 0, 255).astype('uint8')\n", - " return x" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "GEwZ7FlwrjoZ" - }, - "cell_type": "markdown", - "source": [ - "### 콘텐츠와 스타일 표현(Representation) 정의\n", - "\n", - "이미지의 콘텐츠와 스타일 표현(representation)을 얻기 위해, 모델의 몇가지 중간층(layer)들을 살펴볼 것입니다. 모델을 더 깊게 만들수록, 중간층들은 더 고차원적 특성들을 나타냅니다. 이번 경우, 우리는 사전학습된 이미지 분류 네트워크인 VGG19 네트워크의 구조를 사용할 것입니다. 이 중간층들은 우리 이미지에서 콘텐츠와 스타일의 표현을 정의할 필요가 있습니다. 입력 이미지의 경우, 이 중간층들에서 콘텐츠와 스타일에 해당하는 타깃 표현들을 매치하기 위해 시도할 것입니다.\n", - "\n", - "#### 왜 중간층인가?\n", - "\n", - "사전학습된 이미지 분류 네트워크 속에서 중간 출력들이 스타일과 콘텐츠 표현을 어떻게 정의할 수 있는지 궁금할 수도 있습니다. 고수준에서, 이 현상은 네트워크가 이미지 분류(네트워크가 수행하도록 학습된)를 수행하기 위해서는 반드시 이미지를 이해해야 한다는 사실에 의해서 설명될 수 있습니다. 이것은 미가공 이미지를 입력 픽셀로 사용하고 이미지 내 존재하는 특성 표현(feature present)들의 복합 이해로 전환하는 변형을 통한 내부 표현(internal representation)을 만드는 작업이 포함됩니다. 또한 부분적으로 왜 합성곱(convolutional) 신경망의 일반화(generalize)가 쉽게 가능한지를 나타냅니다. 즉, 합성곱 신경망은 배경잡음(background noise)와 기타잡음(nuisances)에 대해 인지할 수없는 클래스 내에서 불변성(invariance)을 포착할 수 있고 특징을 정의할 수 있습니다.(예 : 고양이 vs 개와 같은). 따라서 입력된 미가공 이미지와 분류 레이블(label)이 출력되는 어딘가에서 모델은 복합 특성(complex feature) 추출기로 사용됩니다. 그러므로, 중간층에 액세스하여 입력 이미지의 콘텐츠와 스타일을 표현할 수 있습니다.\n", - "\n", - "우리는 네트워크에서 이 중간층들을 가져올 것입니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "N4-8eUp_Kc-j", - "colab": {} - }, - "cell_type": "code", - "source": [ - "# 특성맵을 추출 할 Content layer\n", - "content_layers = ['block5_conv2']\n", - "\n", - "# 우리가 관심을 갖는 Style layer\n", - "style_layers = ['block1_conv1',\n", - " 'block2_conv1',\n", - " 'block3_conv1',\n", - " 'block4_conv1',\n", - " 'block5_conv1'\n", - " ]\n", - "\n", - "num_content_layers = len(content_layers)\n", - "num_style_layers = len(style_layers)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "Jt3i3RRrJiOX" - }, - "cell_type": "markdown", - "source": [ - "## 모델 만들기\n", - "이번 경우, 우리는 [VGG19](https://keras.io/applications/#vgg19)모델을 적재하고, 입력 텐서를 모델에 제공할 것입니다. 이것은 콘텐츠, 스타일 그리고 생성된 이미지의 feature maps, 그리고 이어서 콘텐츠 및 스타일 표현을 추출할 수 있습니다.\n", - "\n", - "논문에서 제안한 것처럼 VGG19모델을 사용합니다. 또한 VGG19모델은 ResNet, Inception 등과 같은 모델에 비해 상대적으로 간단한 모델이므로 특징맵들은 이미지 변환에 더 효과적으로 작동할 것입니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "v9AnzEUU6hhx" - }, - "cell_type": "markdown", - "source": [ - "콘텐츠와 스타일 특징맵에 상호작용하는 중간층에 액세스하기 위해서 Keras의 [*함수형 API(Functional API)*](https://keras.io/getting-started/functional-api-guide/)를 사용하여 상호작용되는 출력을 얻고, 원하는 출력 활성화를 사용하여 모델을 정의합니다. 모델을 정의하는 함수형 API는 다음과 같이 간단히 입력 및 출력의 정의를 포함합니다.\n", - "\n", - "`model = Model(inputs, outputs)`" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "nfec6MuMAbPx", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def get_model():\n", - " \"\"\" Creates our model with access to intermediate layers.\n", - "\n", - " This function will load the VGG19 model and access the intermediate layers.\n", - " These layers will then be used to create a new model that will take input image\n", - " and return the outputs from these intermediate layers from the VGG model.\n", - "\n", - " Returns:\n", - " returns a keras model that takes image inputs and outputs the style and\n", - " content intermediate layers.\n", - " \"\"\"\n", - " # Load our model. We load pretrained VGG, trained on imagenet data\n", - " vgg = tf.keras.applications.vgg19.VGG19(include_top=False, weights='imagenet')\n", - " vgg.trainable = False\n", - " # Get output layers corresponding to style and content layers\n", - " style_outputs = [vgg.get_layer(name).output for name in style_layers]\n", - " content_outputs = [vgg.get_layer(name).output for name in content_layers]\n", - " model_outputs = style_outputs + content_outputs\n", - " # Build model\n", - " return models.Model(vgg.input, model_outputs)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "kl6eFGa7-OtV" - }, - "cell_type": "markdown", - "source": [ - "위 코드의 스니펫(snippet)에서, 우리는 사전 학습된 이미지 분류 네트워크를 적재할 것입니다. 그 후 우리가 이전에 정의한데로, 관심을 가지는 층들을 확보합니다. 그런 다음 모델의 입력을 이미지로, 출력을 콘텐츠와 스타일층들의 출력으로 설정하여 모델을 정의합니다. 다시 말하면, 우리는 입력으로 이미지를 사용하고 출력으로 콘텐츠와 스타일의 중간층들을 가지는 모델을 생성합니다.\n" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "vJdYvJTZ4bdS" - }, - "cell_type": "markdown", - "source": [ - "## 손실함수(loss function) 정의 및 생성(콘텐츠, 스타일 거리)" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "F2Hcepii7_qh" - }, - "cell_type": "markdown", - "source": [ - "### 콘텐츠 손실 함수" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "1FvH-gwXi4nq" - }, - "cell_type": "markdown", - "source": [ - "콘텐츠 손실 함수의 정의는 실제로 매우 간단합니다. 먼저 원하는 콘텐츠 이미지와 기본 입력 이미지를 네트워크에 전달합니다. 이렇게하면 모델에서 위에서 정의되었던 중간층들의 출력을 반환할 것입니다. 그 후 우리는 간단히 이 두 이미지의 중간 표현(representation) 사이의 유클리디안 거리(euclidean distance)를 얻을 수 있습니다.\n", - "\n", - "좀 더 공식적으로, 콘텐츠 손실(content loss)은 출력이미지 $x$로 부터 콘텐츠 이미지 $p$까지의 콘텐츠의 거리를 표현한 함수입니다. $C_{nn}$을 사전 학습된 딥 합성곱 신경망 네트워크(deep convolutional neural network)라고 합시다. 다시, 이 경우 [VGG19](https://keras.io/applications/#vgg19)모델을 사용합니다. $X$를 어느 이미지라고 하면, $C_{nn}(X)$은 X로 주어진 네트워크입니다. layer $l$에서 입력값이 $x$와 $p$일때 $F^l_{ij}(x) \\in C_{nn}(x)$ 와 $P^l_{ij}(p) \\in C_{nn}(p)$가 중간 특성 표현(feature representation)을 나타낸다고 하면, 콘텐츠 거리(손실)는 다음과 같이 표현할 수 있습니다.\n", - "$$L^l_{content}(p, x) = \\sum_{i, j} (F^l_{ij}(x) - P^l_{ij}(p))^2$$\n", - "\n", - "우리는 일반적인 방식으로 역전파(backpropagation) 과정을 수행하여 콘텐츠 손실(loss)를 최소화 시킬 것입니다. 따라서 특정층(콘텐츠 레이어로 정의 된)에서 원본 이미지와 비슷한 응답을 가질때까지 초기 이미지를 변화시킬것입니다.\n", - "\n", - "이것은 매우 간단하게 구현될 수 있습니다. 다시, 이것은 x, 입력이미지, p, 콘텐츠 이미지로 주어진 층 l에서의 특성맵을 입력으로 받아들이고 콘텐츠 거리를 반환할 것입니다.\n", - "\n" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "6KsbqPA8J9DY" - }, - "cell_type": "markdown", - "source": [ - "### 콘텐츠 손실 계산\n", - "실제로 우리는 원하는 각 층에 콘텐츠 손실을 추가할 것입니다. 이 방법은, 모델에 입력 이미지가 주어질때마다(단순히 (`model(input_image`)로 작동합니다.) 모델을 통한 모든 콘텐츠 손실이 적절히 계산될 것이며, 즉시 실행(eager excution)이기 때문에 모든 경사(gradients)는 계산될 것입니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "d2mf7JwRMkCd", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def get_content_loss(base_content, target):\n", - " return tf.reduce_mean(tf.square(base_content - target))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "lGUfttK9F8d5" - }, - "cell_type": "markdown", - "source": [ - "## 스타일 손실(loss)" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "I6XtkGK_YGD1" - }, - "cell_type": "markdown", - "source": [ - "스타일 손실을 계산하는것은 약간 더 복잡합니다. 하지만 같은 원리를 통하여, 이번에는 기본 입력 이미지와 스타일 이미지를 네트워크에 제공합니다. 기본 이미지와 스타일 이미지의 미가공 중간 출력을 비교하는 대신 두 출력의 그람 행렬(Gram matrix)를 비교합니다.\n", - "\n", - "수학적으로, 우리는 기본 입력 이미지 $x$와 스타일 이미지 $a$의 스타일 손실을 이들의 스타일 표현(the gram matrix)사이의 거리로 표현했습니다. 또한 layer $l$에서 벡터화된 특성맵 $i$ and $j$ 사이의 내부 산물을 $G^l_{ij}$로 정의할때, 이미지의 스타일 표현은 그람 행렬 $G^l$에 의해 주어진 필터 응답(filter response) 사이의 상관관계로 표현됩니다. 주어진 이미지의 특성맵 위에 생성된 $G^l_{ij}$은 특성맵 $i$ and $j$ 사이의 상관관계를 나타내는 것을 볼 수 있습니다.\n", - "\n", - "기본 입력 이미지의 스타일을 생성하기 위해서 콘텐츠 이미지에서 경사하강(gradient descent)을 수행하여 원본 이미지의 스타일 표현과 일치하는 이미지로 변환합니다. 스타일 이미지의 특성 상관 맵(feature correlation map)과 입력 이미지 간의 평균 제곱 거리(mean squared distance)를 최소화하여 그렇게합니다. 전체 스타일 손실(loss)에 대한 각 층(layer)의 분산은 다음과 같이 표현됩니다.\n", - "$$E_l = \\frac{1}{4N_l^2M_l^2} \\sum_{i,j}(G^l_{ij} - A^l_{ij})^2$$\n", - "\n", - "여기서 $G^l_{ij}$ 와 $A^l_{ij}$은 layer $l$에서 $x$ 와 $a$의 스타일 표현을 나타냅니다. $N_l$은 각각의 크기가 $M_l = height * width$인 특성맵의 개수를 표현합니다. 그러므로 각 층의 총 스타일 손실은 다음과 같이 정의됩니다.\n", - "$$L_{style}(a, x) = \\sum_{l \\in L} w_l E_l$$\n", - "\n", - "각 층의 손실의 분산에 가중치 $w_l$를 곱해줍니다. 이번 케이스의 경우 각 층마다 가중치($w_l =\\frac{1}{|L|}$)를 곱해 줄 것입니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "F21Hm61yLKk5" - }, - "cell_type": "markdown", - "source": [ - "### 스타일 손실 계산\n", - "거리 행렬(distance matrix)로 스타일 손실을 구현합니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "N7MOqwKLLke8", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def gram_matrix(input_tensor):\n", - " # 먼저 이미지 채널을 만듭니다.\n", - " channels = int(input_tensor.shape[-1])\n", - " a = tf.reshape(input_tensor, [-1, channels])\n", - " n = tf.shape(a)[0]\n", - " gram = tf.matmul(a, a, transpose_a=True)\n", - " return gram / tf.cast(n, tf.float32)\n", - "\n", - "def get_style_loss(base_style, gram_target):\n", - " \"\"\"Expects two images of dimension h, w, c\"\"\"\n", - " # 높이, 넓이, 각 층의 필터 개수\n", - " # 손실은 주어진 특성맵의 크기와 필터의 개수에 따라 확장됩니다.\n", - " height, width, channels = base_style.get_shape().as_list()\n", - " gram_style = gram_matrix(base_style)\n", - "\n", - " return tf.reduce_mean(tf.square(gram_style - gram_target))# / (4. * (channels ** 2) * (width * height) ** 2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "pXIUX6czZABh" - }, - "cell_type": "markdown", - "source": [ - "## 이미지에 스타일 변형 적용하기" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "y9r8Lyjb_m0u" - }, - "cell_type": "markdown", - "source": [ - "### 경사하강(Gradient Descent) 실행\n", - "만약 경사하강이나 역전파에 대해 복습이 필요하거나 익숙하지 않을 경우 다음을 확인합니다.[awesome resource](https://developers.google.com/machine-learning/crash-course/reducing-loss/gradient-descent).\n", - "\n", - "이번 과정에서는 손실을 최소화하기 위해 [Adam](https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Adam)* 옵티마이저(optimizer)를 사용할 것입니다. 반복적으로 출력 이미지를 업데이트하여, 손실을 최소화 할 것입니다. 네트워크와 관련된 가중치(weight)를 업데이트하지는 않지만, 손실를 최소화하기 위해 입력 이미지를 학습시킬 것입니다. 이를 위해, 우리는 어떻게 손실과 그래디언트를 계산하는지 알아야합니다.\n", - "\n", - "\\* 참고 만약 위 알고리즘에 익숙하다면 L-BFGS를 사용하는것을 권장합니다. 이번 튜토리얼은 즉시 실행을 사용한 모범사례를 보여주는 것이기 때문에 L-BFGS는 사용하지 않습니다. 또한 Adam을 사용하여 사용자 지정 훈련 루프로 자동미분(autogard)/그래디언트 테이프(gradient tape) 기능을 시연할 수 있습니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "-kGzV6LTp4CU" - }, - "cell_type": "markdown", - "source": [ - "콘텐츠와 스타일 이미지를 적재하고, 네트워크에 피드를 제공하며, 모델에서 콘텐츠와 스타일 특성 표현을 출력하는 헬퍼 함수(helper function)을 정의합니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "O-lj5LxgtmnI", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def get_feature_representations(model, content_path, style_path):\n", - " \"\"\"Helper function to compute our content and style feature representations.\n", - "\n", - " This function will simply load and preprocess both the content and style\n", - " images from their path. Then it will feed them through the network to obtain\n", - " the outputs of the intermediate layers.\n", - "\n", - " Arguments:\n", - " model: The model that we are using.\n", - " content_path: The path to the content image.\n", - " style_path: The path to the style image\n", - "\n", - " Returns:\n", - " returns the style features and the content features.\n", - " \"\"\"\n", - " # 이미지 적재\n", - " content_image = load_and_process_img(content_path)\n", - " style_image = load_and_process_img(style_path)\n", - "\n", - " # 콘텐츠와 스타일 특성 배치 계산\n", - " style_outputs = model(style_image)\n", - " content_outputs = model(content_image)\n", - "\n", - "\n", - " # 콘텐츠와 스타일 표현 얻기\n", - " style_features = [style_layer[0] for style_layer in style_outputs[:num_style_layers]]\n", - " content_features = [content_layer[0] for content_layer in content_outputs[num_style_layers:]]\n", - " return style_features, content_features" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "3DopXw7-lFHa" - }, - "cell_type": "markdown", - "source": [ - "### 손실과 그래디언트 계산하기\n", - "그래디언트를 계산하기위해서 [**tf.GradientTape**](https://www.tensorflow.org/programmers_guide/eager#computing_gradients)를 사용하였습니다. 이는 나중에 그래디언트 계산 과정을 추적하여 사용할 수 있는 automatic differentiation 기능을 활용할 수 있습니다. 이것은 정방향 전파동안의 명령들을 기록하여 역전파시 입력 이미지와 관련하여 손실 함수의 그래디언트를 계산할 수 있게 합니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "oVDhSo8iJunf", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def compute_loss(model, loss_weights, init_image, gram_style_features, content_features):\n", - " \"\"\"This function will compute the loss total loss.\n", - "\n", - " Arguments:\n", - " model: The model that will give us access to the intermediate layers\n", - " loss_weights: The weights of each contribution of each loss function.\n", - " (style weight, content weight, and total variation weight)\n", - " init_image: Our initial base image. This image is what we are updating with\n", - " our optimization process. We apply the gradients wrt the loss we are\n", - " calculating to this image.\n", - " gram_style_features: Precomputed gram matrices corresponding to the\n", - " defined style layers of interest.\n", - " content_features: Precomputed outputs from defined content layers of\n", - " interest.\n", - "\n", - " Returns:\n", - " returns the total loss, style loss, content loss, and total variational loss\n", - " \"\"\"\n", - " style_weight, content_weight = loss_weights\n", - "\n", - " # 초기 이미지를 모델에 제공합니다. 이것은 우리가 원하는 층에서 콘텐츠와 스타일 표현을 제공합니다.\n", - " # 즉시 실행을 사용하고 있기 때문에 모델은 다른 함수와 같이 호출 가능합니다.\n", - " model_outputs = model(init_image)\n", - "\n", - " style_output_features = model_outputs[:num_style_layers]\n", - " content_output_features = model_outputs[num_style_layers:]\n", - "\n", - " style_score = 0\n", - " content_score = 0\n", - "\n", - " # 모든층들에서 스타일 손실을 합산합니다.\n", - " # 각 분산된 손실층에 가중치를 적용합니다.\n", - " weight_per_style_layer = 1.0 / float(num_style_layers)\n", - " for target_style, comb_style in zip(gram_style_features, style_output_features):\n", - " style_score += weight_per_style_layer * get_style_loss(comb_style[0], target_style)\n", - "\n", - " # 모든층들에서 콘텐츠 손실을 합산합니다.\n", - " weight_per_content_layer = 1.0 / float(num_content_layers)\n", - " for target_content, comb_content in zip(content_features, content_output_features):\n", - " content_score += weight_per_content_layer* get_content_loss(comb_content[0], target_content)\n", - "\n", - " style_score *= style_weight\n", - " content_score *= content_weight\n", - "\n", - " # 총 손실을 구합니다.\n", - " loss = style_score + content_score\n", - " return loss, style_score, content_score" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "r5XTvbP6nJQa" - }, - "cell_type": "markdown", - "source": [ - "이렇게 하면 아래와 같이 그래디언트의 계산은 쉽습니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "fwzYeOqOUH9_", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def compute_grads(cfg):\n", - " with tf.GradientTape() as tape:\n", - " all_loss = compute_loss(**cfg)\n", - " # 입력 이미지에 관한 그래디언트 계산\n", - " total_loss = all_loss[0]\n", - " return tape.gradient(total_loss, cfg['init_image']), all_loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "T9yKu2PLlBIE" - }, - "cell_type": "markdown", - "source": [ - "### 루프 최적화" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "pj_enNo6tACQ", - "colab": {} - }, - "cell_type": "code", - "source": [ - "import IPython.display\n", - "\n", - "def run_style_transfer(content_path,\n", - " style_path,\n", - " num_iterations=1000,\n", - " content_weight=1e3,\n", - " style_weight=1e-2):\n", - " # 모델의 어떤층을 훈련시킬 필요가 없기 때문에 trainable에 False 값을 할당합니다.\n", - " model = get_model()\n", - " for layer in model.layers:\n", - " layer.trainable = False\n", - "\n", - " # 특정 중간층에서 스타일과 콘텐츠 표현을 얻습니다.\n", - " style_features, content_features = get_feature_representations(model, content_path, style_path)\n", - " gram_style_features = [gram_matrix(style_feature) for style_feature in style_features]\n", - "\n", - " # 초기 이미지 설정\n", - " init_image = load_and_process_img(content_path)\n", - " init_image = tfe.Variable(init_image, dtype=tf.float32)\n", - " # 옵티마이저 생성\n", - " opt = tf.train.AdamOptimizer(learning_rate=5, beta1=0.99, epsilon=1e-1)\n", - "\n", - " # 중간 이미지 표시\n", - " iter_count = 1\n", - "\n", - " # 최적의 결과 저장\n", - " best_loss, best_img = float('inf'), None\n", - "\n", - " # 최적의 설정 만들기\n", - " loss_weights = (style_weight, content_weight)\n", - " cfg = {\n", - " 'model': model,\n", - " 'loss_weights': loss_weights,\n", - " 'init_image': init_image,\n", - " 'gram_style_features': gram_style_features,\n", - " 'content_features': content_features\n", - " }\n", - "\n", - " # 화면 표시\n", - " num_rows = 2\n", - " num_cols = 5\n", - " display_interval = num_iterations/(num_rows*num_cols)\n", - " start_time = time.time()\n", - " global_start = time.time()\n", - "\n", - " norm_means = np.array([103.939, 116.779, 123.68])\n", - " min_vals = -norm_means\n", - " max_vals = 255 - norm_means\n", - "\n", - " imgs = []\n", - " for i in range(num_iterations):\n", - " grads, all_loss = compute_grads(cfg)\n", - " loss, style_score, content_score = all_loss\n", - " opt.apply_gradients([(grads, init_image)])\n", - " clipped = tf.clip_by_value(init_image, min_vals, max_vals)\n", - " init_image.assign(clipped)\n", - " end_time = time.time()\n", - "\n", - " if loss < best_loss:\n", - " # 총 손실에서 최적의 손실과 이미지 업데이트\n", - " best_loss = loss\n", - " best_img = deprocess_img(init_image.numpy())\n", - "\n", - " if i % display_interval== 0:\n", - " start_time = time.time()\n", - "\n", - " # numpy array 구성을 위한 .numpy() 메소드 사용\n", - " plot_img = init_image.numpy()\n", - " plot_img = deprocess_img(plot_img)\n", - " imgs.append(plot_img)\n", - " IPython.display.clear_output(wait=True)\n", - " IPython.display.display_png(Image.fromarray(plot_img))\n", - " print('Iteration: {}'.format(i))\n", - " print('Total loss: {:.4e}, '\n", - " 'style loss: {:.4e}, '\n", - " 'content loss: {:.4e}, '\n", - " 'time: {:.4f}s'.format(loss, style_score, content_score, time.time() - start_time))\n", - " print('Total time: {:.4f}s'.format(time.time() - global_start))\n", - " IPython.display.clear_output(wait=True)\n", - " plt.figure(figsize=(14,4))\n", - " for i,img in enumerate(imgs):\n", - " plt.subplot(num_rows,num_cols,i+1)\n", - " plt.imshow(img)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " return best_img, best_loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "vSVMx4burydi", - "colab": {} - }, - "cell_type": "code", - "source": [ - "best, best_loss = run_style_transfer(content_path,\n", - " style_path, num_iterations=1000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "dzJTObpsO3TZ", - "colab": {} - }, - "cell_type": "code", - "source": [ - "Image.fromarray(best)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "dCXQ9vSnQbDy" - }, - "cell_type": "markdown", - "source": [ - "Colaboratory에서 이미지를 다운로드하려면 다음 코드의 주석처리를 제거하십시오." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "SSH6OpyyQn7w", - "colab": {} - }, - "cell_type": "code", - "source": [ - "#from google.colab import files\n", - "#files.download('wave_turtle.png')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "LwiZfCW0AZwt" - }, - "cell_type": "markdown", - "source": [ - "## 출력 시각화\n", - "적용되었던 처리 과정을 제거하기 위해 출력 이미지의 역처리 과정을 수행합니다." - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "lqTQN1PjulV9", - "colab": {} - }, - "cell_type": "code", - "source": [ - "def show_results(best_img, content_path, style_path, show_large_final=True):\n", - " plt.figure(figsize=(10, 5))\n", - " content = load_img(content_path)\n", - " style = load_img(style_path)\n", - "\n", - " plt.subplot(1, 2, 1)\n", - " imshow(content, 'Content Image')\n", - "\n", - " plt.subplot(1, 2, 2)\n", - " imshow(style, 'Style Image')\n", - "\n", - " if show_large_final:\n", - " plt.figure(figsize=(10, 10))\n", - "\n", - " plt.imshow(best_img)\n", - " plt.title('Output Image')\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "i6d6O50Yvs6a", - "colab": {} - }, - "cell_type": "code", - "source": [ - "show_results(best, content_path, style_path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "tyGMmWh2Pss8" - }, - "cell_type": "markdown", - "source": [ - "## 다른 이미지 시도\n", - "\n", - "튀빙겐(Tuebingen) 이미지\n", - "\n", - "Photo By: Andreas Praefcke [GFDL (http://www.gnu.org/copyleft/fdl.html) or CC BY 3.0 (https://creativecommons.org/licenses/by/3.0)], from Wikimedia Commons" - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "x2TePU39k9lb" - }, - "cell_type": "markdown", - "source": [ - "### 별이 빛나는 밤(Starry night) + 튀빙겐(Tuebingen)" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "ES9dC6ZyJBD2", - "colab": {} - }, - "cell_type": "code", - "source": [ - "best_starry_night, best_loss = run_style_transfer('/tmp/nst/Tuebingen_Neckarfront.jpg',\n", - " '/tmp/nst/1024px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "X8w8WLkKvzXu", - "colab": {} - }, - "cell_type": "code", - "source": [ - "show_results(best_starry_night, '/tmp/nst/Tuebingen_Neckarfront.jpg',\n", - " '/tmp/nst/1024px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "QcXwvViek4Br" - }, - "cell_type": "markdown", - "source": [ - "### 창조의 기둥(Pillars of Creation) + 튀빙겐(Tuebingen)" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "vJ3u2U-gGmgP", - "colab": {} - }, - "cell_type": "code", - "source": [ - "best_poc_tubingen, best_loss = run_style_transfer('/tmp/nst/Tuebingen_Neckarfront.jpg',\n", - " '/tmp/nst/Pillars_of_creation_2014_HST_WFC3-UVIS_full-res_denoised.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "pQUq3KxpGv2O", - "colab": {} - }, - "cell_type": "code", - "source": [ - "show_results(best_poc_tubingen,\n", - " '/tmp/nst/Tuebingen_Neckarfront.jpg',\n", - " '/tmp/nst/Pillars_of_creation_2014_HST_WFC3-UVIS_full-res_denoised.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "bTZdTOdW3s8H" - }, - "cell_type": "markdown", - "source": [ - "### 칸딘스키(Kandinsky) Composition #7 + 튀빙겐(Tuebingen)" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "bt9mbQfl7exl", - "colab": {} - }, - "cell_type": "code", - "source": [ - "best_kandinsky_tubingen, best_loss = run_style_transfer('/tmp/nst/Tuebingen_Neckarfront.jpg',\n", - " '/tmp/nst/Vassily_Kandinsky,_1913_-_Composition_7.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "Qnz8HeXSXg6P", - "colab": {} - }, - "cell_type": "code", - "source": [ - "show_results(best_kandinsky_tubingen,\n", - " '/tmp/nst/Tuebingen_Neckarfront.jpg',\n", - " '/tmp/nst/Vassily_Kandinsky,_1913_-_Composition_7.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "cg68lW2A3s8N" - }, - "cell_type": "markdown", - "source": [ - "### 창조의 기둥(Pillars of Creation) + 바다 거북(Sea Turtle)" - ] - }, - { - "metadata": { - "colab_type": "code", - "id": "dl0DUot_bFST", - "colab": {} - }, - "cell_type": "code", - "source": [ - "best_poc_turtle, best_loss = run_style_transfer('/tmp/nst/Green_Sea_Turtle_grazing_seagrass.jpg',\n", - " '/tmp/nst/Pillars_of_creation_2014_HST_WFC3-UVIS_full-res_denoised.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "code", - "id": "UzJfE0I1bQn8", - "colab": {} - }, - "cell_type": "code", - "source": [ - "show_results(best_poc_turtle,\n", - " '/tmp/nst/Green_Sea_Turtle_grazing_seagrass.jpg',\n", - " '/tmp/nst/Pillars_of_creation_2014_HST_WFC3-UVIS_full-res_denoised.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "metadata": { - "colab_type": "text", - "id": "sElaeNX-4Vnc" - }, - "cell_type": "markdown", - "source": [ - "## 주요 요점\n", - "\n", - "### 우리가 다룬 내용:\n", - "* 몇가지의 다른 손실(loss)함수를 정의하였고, 손실을 최소화하기 위한 입력 이미지 변형을 위해 역전파(backpropagation)을 사용하였습니다.\n", - " * 이를 위해서 우리는 사전학습된 모델에 적재해야 했으며, 이 모델의 학습된 특성맵을 이미지의 콘텐츠와 스타일 표현을 표현하기위해 사용했습니다.\n", - " * 메인 손실(loss) 함수는 표현 차이(different representations)의 관점에서 주로 거리를 계산하는 것이었습니다.\n", - "* 사용자 지정 모델과 **즉시 실행(eager execution)**을 사용하여 실행하였습니다.\n", - " * Functional API를 사용하여 사용자 지정 모델을 빌드했습니다.\n", - " * 즉시 실행은 파이썬 자연제어흐름을 사용하여 `Tensor`와 함께 동적인 수행을 제공했습니다.\n", - " * `Tensor`를 직접적으로 제어했습니다. 이는 디버깅과 작업 쉽게 만들어줍니다.\n", - "* **tf.gradient**를 사용한 옵티마이저(optimizer) 업데이트 법칙을 적용하여 반복적으로 이미지를 업데이트했습니다. 옵티마이저는 주어진 입력 이미지의 손실과 관련하여 최적화됩니다." - ] - }, - { - "metadata": { - "colab_type": "text", - "id": "U-y02GWonqnD" - }, - "cell_type": "markdown", - "source": [ - "\n", - "**[Image of Tuebingen](https://commons.wikimedia.org/wiki/File:Tuebingen_Neckarfront.jpg)**\n", - "Photo By: Andreas Praefcke [GFDL (http://www.gnu.org/copyleft/fdl.html) or CC BY 3.0 (https://creativecommons.org/licenses/by/3.0)], from Wikimedia Commons\n", - "\n", - "**[Image of Green Sea Turtle](https://commons.wikimedia.org/wiki/File:Green_Sea_Turtle_grazing_seagrass.jpg)**\n", - "By P.Lindgren [CC BY-SA 3.0 (https://creativecommons.org/licenses/by-sa/3.0)], from Wikimedia Commons\n", - "\n" - ] - } - ] -} diff --git a/site/ko/tutorials/images/cnn.ipynb b/site/ko/tutorials/images/cnn.ipynb deleted file mode 100644 index b37d5db1167..00000000000 --- a/site/ko/tutorials/images/cnn.ipynb +++ /dev/null @@ -1,395 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x4HI2mpwlrcn" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "679Lmwt3l1Bk" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DSPCom-KmApV" - }, - "source": [ - "# 합성곱 신경망" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "klAltGp8ycek" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/images/cnn\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " TensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/images/cnn.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " 구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/images/cnn.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " 깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-vSVdjrYPae" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qLGkt5qiyz4E" - }, - "source": [ - "이 튜토리얼은 MNIST 숫자를 분류하기 위해 간단한 [합성곱 신경망](https://developers.google.com/machine-learning/glossary/#convolutional_neural_network)(Convolutional Neural Network, CNN)을 훈련합니다. 간단한 이 네트워크는 MNIST 테스트 세트에서 99% 정확도를 달성할 것입니다. 이 튜토리얼은 [케라스 Sequential API](https://www.tensorflow.org/guide/keras)를 사용하기 때문에 몇 줄의 코드만으로 모델을 만들고 훈련할 수 있습니다.\n", - "\n", - "노트: GPU를 사용하여 CNN의 훈련 속도를 높일 수 있습니다. 코랩에서 이 노트북을 실행한다면 * 수정 -\u003e 노트 설정 -\u003e 하드웨어 가속기* 에서 GPU를 선택하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m7KBpffWzlxH" - }, - "source": [ - "### 텐서플로 임포트하기" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iAve6DCL4JH4" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras import datasets, layers, models" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jRFxccghyMVo" - }, - "source": [ - "### MNIST 데이터셋 다운로드하고 준비하기" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JWoEqyMuXFF4" - }, - "outputs": [], - "source": [ - "(train_images, train_labels), (test_images, test_labels) = datasets.mnist.load_data()\n", - "\n", - "train_images = train_images.reshape((60000, 28, 28, 1))\n", - "test_images = test_images.reshape((10000, 28, 28, 1))\n", - "\n", - "# 픽셀 값을 0~1 사이로 정규화합니다.\n", - "train_images, test_images = train_images / 255.0, test_images / 255.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oewp-wYg31t9" - }, - "source": [ - "### 합성곱 층 만들기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3hQvqXpNyN3x" - }, - "source": [ - "아래 6줄의 코드에서 [Conv2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2D)와 [MaxPooling2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/MaxPool2D) 층을 쌓는 일반적인 패턴으로 합성곱 층을 정의합니다.\n", - "\n", - "CNN은 배치(batch) 크기를 제외하고 (이미지 높이, 이미지 너비, 컬러 채널) 크기의 텐서(tensor)를 입력으로 받습니다. MNIST 데이터는 (흑백 이미지이기 때문에) 컬러 채널(channel)이 하나지만 컬러 이미지는 (R,G,B) 세 개의 채널을 가집니다. 이 예에서는 MNIST 이미지 포맷인 (28, 28, 1) 크기의 입력을 처리하는 CNN을 정의하겠습니다. 이 값을 첫 번째 층의 `input_shape` 매개변수로 전달합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "L9YmGQBQPrdn" - }, - "outputs": [], - "source": [ - "model = models.Sequential()\n", - "model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n", - "model.add(layers.MaxPooling2D((2, 2)))\n", - "model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n", - "model.add(layers.MaxPooling2D((2, 2)))\n", - "model.add(layers.Conv2D(64, (3, 3), activation='relu'))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lvDVFkg-2DPm" - }, - "source": [ - "지금까지 모델의 구조를 출력해 보죠." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8-C4XBg4UTJy" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_j-AXYeZ2GO5" - }, - "source": [ - "위에서 Conv2D와 MaxPooling2D 층의 출력은 (높이, 너비, 채널) 크기의 3D 텐서입니다. 높이와 너비 차원은 네트워크가 깊어질수록 감소하는 경향을 가집니다. Conv2D 층에서 출력 채널의 수는 첫 번째 매개변수에 의해 결정됩니다(예를 들면, 32 또는 64). 일반적으로 높이와 너비가 줄어듦에 따라 (계산 비용 측면에서) Conv2D 층의 출력 채널을 늘릴 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_v8sVOtG37bT" - }, - "source": [ - "### 마지막에 Dense 층 추가하기\n", - "\n", - "모델을 완성하려면 마지막 합성곱 층의 출력 텐서(크기 (3, 3, 64))를 하나 이상의 Dense 층에 주입하여 분류를 수행합니다. Dense 층은 벡터(1D)를 입력으로 받는데 현재 출력은 3D 텐서입니다. 먼저 3D 출력을 1D로 펼치겠습니다. 그다음 하나 이상의 Dense 층을 그 위에 추가하겠습니다. MNIST 데이터는 10개의 클래스가 있으므로 마지막에 Dense 층에 10개의 출력과 소프트맥스 활성화 함수를 사용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mRs95d6LUVEi" - }, - "outputs": [], - "source": [ - "model.add(layers.Flatten())\n", - "model.add(layers.Dense(64, activation='relu'))\n", - "model.add(layers.Dense(10, activation='softmax'))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ipGiQMcR4Gtq" - }, - "source": [ - "최종 모델의 구조를 확인해 보죠." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8Yu_m-TZUWGX" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xNKXi-Gy3RO-" - }, - "source": [ - "여기에서 볼 수 있듯이 두 개의 Dense 층을 통과하기 전에 (3, 3, 64) 출력을 (576) 크기의 벡터로 펼쳤습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P3odqfHP4M67" - }, - "source": [ - "### 모델 컴파일과 훈련하기" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MdDzI75PUXrG" - }, - "outputs": [], - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jKgyC5K_4O0d" - }, - "source": [ - "### 모델 평가" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gtyDF0MKUcM7" - }, - "outputs": [], - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0LvwaKhtUdOo" - }, - "outputs": [], - "source": [ - "print(test_acc)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8cfJ8AR03gT5" - }, - "source": [ - "결과에서 보듯이 간단한 CNN 모델이 99%의 테스트 정확도를 달성합니다. 몇 라인의 코드치고 나쁘지 않네요! (케라스의 서브클래싱 API와 GradientTape를 사용하여) CNN을 만드는 또 다른 방법은 [여기](https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/quickstart/advanced.ipynb)를 참고하세요." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "cnn.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/keras/classification.ipynb b/site/ko/tutorials/keras/classification.ipynb deleted file mode 100644 index cbd13fa06f3..00000000000 --- a/site/ko/tutorials/keras/classification.ipynb +++ /dev/null @@ -1,1038 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "_ckMIh7O7s6D" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "vasWnqRgy1H4" - }, - "outputs": [], - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# 첫 번째 신경망 훈련하기: 기초적인 분류 문제" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aJjNjjy3Zbz0" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "이 튜토리얼에서는 운동화나 셔츠 같은 옷 이미지를 분류하는 신경망 모델을 훈련합니다. 상세 내용을 모두 이해하지 못해도 괜찮습니다. 여기서는 완전한 텐서플로(TensorFlow) 프로그램을 빠르게 살펴 보겠습니다. 자세한 내용은 앞으로 배우면서 더 설명합니다.\n", - "\n", - "여기에서는 텐서플로 모델을 만들고 훈련할 수 있는 고수준 API인 [tf.keras](https://www.tensorflow.org/guide/keras)를 사용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jL3OqFKZ9dFg" - }, - "outputs": [], - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dzLKpmZICaWN" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "# tensorflow와 tf.keras를 임포트합니다\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# 헬퍼(helper) 라이브러리를 임포트합니다\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## 패션 MNIST 데이터셋 임포트하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "10개의 범주(category)와 70,000개의 흑백 이미지로 구성된 [패션 MNIST](https://github.com/zalandoresearch/fashion-mnist) 데이터셋을 사용하겠습니다. 이미지는 해상도(28x28 픽셀)가 낮고 다음처럼 개별 옷 품목을 나타냅니다:\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " 그림 1. 패션-MNIST 샘플 (Zalando, MIT License).
     \n", - "
    \n", - "\n", - "패션 MNIST는 컴퓨터 비전 분야의 \"Hello, World\" 프로그램격인 고전 [MNIST](http://yann.lecun.com/exdb/mnist/) 데이터셋을 대신해서 자주 사용됩니다. MNIST 데이터셋은 손글씨 숫자(0, 1, 2 등)의 이미지로 이루어져 있습니다. 여기서 사용하려는 옷 이미지와 동일한 포맷입니다.\n", - "\n", - "패션 MNIST는 일반적인 MNIST 보다 조금 더 어려운 문제이고 다양한 예제를 만들기 위해 선택했습니다. 두 데이터셋은 비교적 작기 때문에 알고리즘의 작동 여부를 확인하기 위해 사용되곤 합니다. 코드를 테스트하고 디버깅하는 용도로 좋습니다.\n", - "\n", - "네트워크를 훈련하는데 60,000개의 이미지를 사용합니다. 그다음 네트워크가 얼마나 정확하게 이미지를 분류하는지 10,000개의 이미지로 평가하겠습니다. 패션 MNIST 데이터셋은 텐서플로에서 바로 임포트하여 적재할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7MqDQO0KCaWS" - }, - "outputs": [], - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "load_data() 함수를 호출하면 네 개의 넘파이(NumPy) 배열이 반환됩니다:\n", - "\n", - "* `train_images`와 `train_labels` 배열은 모델 학습에 사용되는 *훈련 세트*입니다.\n", - "* `test_images`와 `test_labels` 배열은 모델 테스트에 사용되는 *테스트 세트*입니다.\n", - "\n", - "이미지는 28x28 크기의 넘파이 배열이고 픽셀 값은 0과 255 사이입니다. *레이블*(label)은 0에서 9까지의 정수 배열입니다. 이 값은 이미지에 있는 옷의 *클래스*(class)를 나타냅니다:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    레이블클래스
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "각 이미지는 하나의 레이블에 매핑되어 있습니다. 데이터셋에 *클래스 이름*이 들어있지 않기 때문에 나중에 이미지를 출력할 때 사용하기 위해 별도의 변수를 만들어 저장합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IjnLH5S2CaWx" - }, - "outputs": [], - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## 데이터 탐색\n", - "\n", - "모델을 훈련하기 전에 데이터셋 구조를 살펴보죠. 다음 코드는 훈련 세트에 60,000개의 이미지가 있다는 것을 보여줍니다. 각 이미지는 28x28 픽셀로 표현됩니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zW5k_xz1CaWX" - }, - "outputs": [], - "source": [ - "train_images.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "비슷하게 훈련 세트에는 60,000개의 레이블이 있습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TRFYHB2mCaWb" - }, - "outputs": [], - "source": [ - "len(train_labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "각 레이블은 0과 9사이의 정수입니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "XKnCTHz4CaWg" - }, - "outputs": [], - "source": [ - "train_labels" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "테스트 세트에는 10,000개의 이미지가 있습니다. 이 이미지도 28x28 픽셀로 표현됩니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2KFnYlcwCaWl" - }, - "outputs": [], - "source": [ - "test_images.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "테스트 세트는 10,000개의 이미지에 대한 레이블을 가지고 있습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iJmPr5-ACaWn" - }, - "outputs": [], - "source": [ - "len(test_labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## 데이터 전처리\n", - "\n", - "네트워크를 훈련하기 전에 데이터를 전처리해야 합니다. 훈련 세트에 있는 첫 번째 이미지를 보면 픽셀 값의 범위가 0~255 사이라는 것을 알 수 있습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "m4VEw8Ud9Quh" - }, - "outputs": [], - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "신경망 모델에 주입하기 전에 이 값의 범위를 0~1 사이로 조정하겠습니다. 이렇게 하려면 255로 나누어야 합니다. *훈련 세트*와 *테스트 세트*를 동일한 방식으로 전처리하는 것이 중요합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bW5WzIPlCaWv" - }, - "outputs": [], - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "*훈련 세트*에서 처음 25개 이미지와 그 아래 클래스 이름을 출력해 보죠. 데이터 포맷이 올바른지 확인하고 네트워크 구성과 훈련할 준비를 마칩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "oZTImqg_CaW1" - }, - "outputs": [], - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## 모델 구성\n", - "\n", - "신경망 모델을 만들려면 모델의 층을 구성한 다음 모델을 컴파일합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### 층 설정\n", - "\n", - "신경망의 기본 구성 요소는 *층*(layer)입니다. 층은 주입된 데이터에서 표현을 추출합니다. 아마도 문제를 해결하는데 더 의미있는 표현이 추출될 것입니다.\n", - "\n", - "대부분 딥러닝은 간단한 층을 연결하여 구성됩니다. `tf.keras.layers.Dense`와 같은 층들의 가중치(parameter)는 훈련하는 동안 학습됩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9ODch-OFCaW4" - }, - "outputs": [], - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation='relu'),\n", - " keras.layers.Dense(10, activation='softmax')\n", - "])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "이 네트워크의 첫 번째 층인 `tf.keras.layers.Flatten`은 2차원 배열(28 x 28 픽셀)의 이미지 포맷을 28 * 28 = 784 픽셀의 1차원 배열로 변환합니다. 이 층은 이미지에 있는 픽셀의 행을 펼쳐서 일렬로 늘립니다. 이 층에는 학습되는 가중치가 없고 데이터를 변환하기만 합니다.\n", - "\n", - "픽셀을 펼친 후에는 두 개의 `tf.keras.layers.Dense` 층이 연속되어 연결됩니다. 이 층을 밀집 연결(densely-connected) 또는 완전 연결(fully-connected) 층이라고 부릅니다. 첫 번째 `Dense` 층은 128개의 노드(또는 뉴런)를 가집니다. 두 번째 (마지막) 층은 10개의 노드의 *소프트맥스*(softmax) 층입니다. 이 층은 10개의 확률을 반환하고 반환된 값의 전체 합은 1입니다. 각 노드는 현재 이미지가 10개 클래스 중 하나에 속할 확률을 출력합니다.\n", - "\n", - "### 모델 컴파일\n", - "\n", - "모델을 훈련하기 전에 필요한 몇 가지 설정이 모델 *컴파일* 단계에서 추가됩니다:\n", - "\n", - "* *손실 함수*(Loss function)-훈련 하는 동안 모델의 오차를 측정합니다. 모델의 학습이 올바른 방향으로 향하도록 이 함수를 최소화해야 합니다.\n", - "* *옵티마이저*(Optimizer)-데이터와 손실 함수를 바탕으로 모델의 업데이트 방법을 결정합니다.\n", - "* *지표*(Metrics)-훈련 단계와 테스트 단계를 모니터링하기 위해 사용합니다. 다음 예에서는 올바르게 분류된 이미지의 비율인 *정확도*를 사용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Lhan11blCaW7" - }, - "outputs": [], - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## 모델 훈련\n", - "\n", - "신경망 모델을 훈련하는 단계는 다음과 같습니다:\n", - "\n", - "1. 훈련 데이터를 모델에 주입합니다-이 예에서는 `train_images`와 `train_labels` 배열입니다.\n", - "2. 모델이 이미지와 레이블을 매핑하는 방법을 배웁니다.\n", - "3. 테스트 세트에 대한 모델의 예측을 만듭니다-이 예에서는 `test_images` 배열입니다. 이 예측이 `test_labels` 배열의 레이블과 맞는지 확인합니다.\n", - "\n", - "훈련을 시작하기 위해 `model.fit` 메서드를 호출하면 모델이 훈련 데이터를 학습합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xvwvpA64CaW_" - }, - "outputs": [], - "source": [ - "model.fit(train_images, train_labels, epochs=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "모델이 훈련되면서 손실과 정확도 지표가 출력됩니다. 이 모델은 훈련 세트에서 약 0.88(88%) 정도의 정확도를 달성합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## 정확도 평가\n", - "\n", - "그다음 테스트 세트에서 모델의 성능을 비교합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VflXLEeECaXC" - }, - "outputs": [], - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('\\n테스트 정확도:', test_acc)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "테스트 세트의 정확도가 훈련 세트의 정확도보다 조금 낮습니다. 훈련 세트의 정확도와 테스트 세트의 정확도 사이의 차이는 *과대적합*(overfitting) 때문입니다. 과대적합은 머신러닝 모델이 훈련 데이터보다 새로운 데이터에서 성능이 낮아지는 현상을 말합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## 예측 만들기\n", - "\n", - "훈련된 모델을 사용하여 이미지에 대한 예측을 만들 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Gl91RPhdCaXI" - }, - "outputs": [], - "source": [ - "predictions = model.predict(test_images)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "여기서는 테스트 세트에 있는 각 이미지의 레이블을 예측했습니다. 첫 번째 예측을 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3DmJEUinCaXK" - }, - "outputs": [], - "source": [ - "predictions[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "이 예측은 10개의 숫자 배열로 나타납니다. 이 값은 10개의 옷 품목에 상응하는 모델의 신뢰도(confidence)를 나타냅니다. 가장 높은 신뢰도를 가진 레이블을 찾아보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qsqenuPnCaXO" - }, - "outputs": [], - "source": [ - "np.argmax(predictions[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "모델은 이 이미지가 앵클 부츠(`class_name[9]`)라고 가장 확신하고 있습니다. 이 값이 맞는지 테스트 레이블을 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Sd7Pgsu6CaXP" - }, - "outputs": [], - "source": [ - "test_labels[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "10개 클래스에 대한 예측을 모두 그래프로 표현해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DvYmmrpIy6Y1" - }, - "outputs": [], - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d4Ov9OFDMmOD" - }, - "source": [ - "0번째 원소의 이미지, 예측, 신뢰도 점수 배열을 확인해 보겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HV5jw-5HwSmO" - }, - "outputs": [], - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Ko-uzOufSCSe" - }, - "outputs": [], - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "몇 개의 이미지의 예측을 출력해 보죠. 올바르게 예측된 레이블은 파란색이고 잘못 예측된 레이블은 빨강색입니다. 숫자는 예측 레이블의 신뢰도 퍼센트(100점 만점)입니다. 신뢰도 점수가 높을 때도 잘못 예측할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hQlnbqaw2Qu_" - }, - "outputs": [], - "source": [ - "# 처음 X 개의 테스트 이미지와 예측 레이블, 진짜 레이블을 출력합니다\n", - "# 올바른 예측은 파랑색으로 잘못된 예측은 빨강색으로 나타냅니다\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "마지막으로 훈련된 모델을 사용하여 한 이미지에 대한 예측을 만듭니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yRJ7JU7JCaXT" - }, - "outputs": [], - "source": [ - "# 테스트 세트에서 이미지 하나를 선택합니다\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "`tf.keras` 모델은 한 번에 샘플의 묶음 또는 *배치*(batch)로 예측을 만드는데 최적화되어 있습니다. 하나의 이미지를 사용할 때에도 2차원 배열로 만들어야 합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lDFh5yF_CaXW" - }, - "outputs": [], - "source": [ - "# 이미지 하나만 사용할 때도 배치에 추가합니다\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "이제 이 이미지의 예측을 만듭니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "o_rzNSdrCaXY" - }, - "outputs": [], - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6Ai-cpLjO-3A" - }, - "outputs": [], - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "`model.predict`는 2차원 넘파이 배열을 반환하므로 첫 번째 이미지의 예측을 선택합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2tRmdq_8CaXb" - }, - "outputs": [], - "source": [ - "np.argmax(predictions_single[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "이전과 마찬가지로 모델의 예측은 레이블 9입니다." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "classification.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/site/ko/tutorials/keras/overfit_and_underfit.ipynb b/site/ko/tutorials/keras/overfit_and_underfit.ipynb deleted file mode 100644 index a8f077c0ffd..00000000000 --- a/site/ko/tutorials/keras/overfit_and_underfit.ipynb +++ /dev/null @@ -1,710 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fTFj8ft5dlbS" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "lzyBOpYMdp3F" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "m_x4KfSJ7Vt7" - }, - "outputs": [], - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C9HmC2T4ld5B" - }, - "source": [ - "# 과대적합과 과소적합" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRTxFhXAlnl1" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1vJpRm6na4Iw" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19rPukKZsPG6" - }, - "source": [ - "지금까지 그랬듯이 이 예제의 코드도 `tf.keras` API를 사용합니다. 텐서플로 [케라스 가이드](https://www.tensorflow.org/guide/keras)에서 `tf.keras` API에 대해 더 많은 정보를 얻을 수 있습니다.\n", - "\n", - "앞서 영화 리뷰 분류와 주택 가격 예측의 두 예제에서 일정 에포크 동안 훈련하면 검증 세트에서 모델 성능이 최고점에 도달한 다음 감소하기 시작한 것을 보았습니다.\n", - "\n", - "다른 말로 하면, 모델이 훈련 세트에 *과대적합*(overfitting)된 것입니다. 과대적합을 다루는 방법은 꼭 배워야 합니다. *훈련 세트*에서 높은 성능을 얻을 수 있지만 진짜 원하는 것은 *테스트 세트*(또는 이전에 본 적 없는 데이터)에 잘 일반화되는 모델입니다.\n", - "\n", - "과대적합의 반대는 *과소적합*(underfitting)입니다. 과소적합은 테스트 세트의 성능이 향상될 여지가 아직 있을 때 일어납니다. 발생하는 원인은 여러가지입니다. 모델이 너무 단순하거나, 규제가 너무 많거나, 그냥 단순히 충분히 오래 훈련하지 않는 경우입니다. 즉 네트워크가 훈련 세트에서 적절한 패턴을 학습하지 못했다는 뜻입니다.\n", - "\n", - "모델을 너무 오래 훈련하면 과대적합되기 시작하고 테스트 세트에서 일반화되지 못하는 패턴을 훈련 세트에서 학습합니다. 과대적합과 과소적합 사이에서 균형을 잡아야 합니다. 이를 위해 적절한 에포크 횟수동안 모델을 훈련하는 방법을 배워보겠습니다.\n", - "\n", - "과대적합을 막는 가장 좋은 방법은 더 많은 훈련 데이터를 사용하는 것입니다. 많은 데이터에서 훈련한 모델은 자연적으로 일반화 성능이 더 좋습니다. 데이터를 더 준비할 수 없을 때 그다음으로 가장 좋은 방법은 규제(regularization)와 같은 기법을 사용하는 것입니다. 모델이 저장할 수 있는 정보의 양과 종류에 제약을 부과하는 방법입니다. 네트워크가 소수의 패턴만 기억할 수 있다면 최적화 과정 동안 일반화 가능성이 높은 가장 중요한 패턴에 촛점을 맞출 것입니다.\n", - "\n", - "이 노트북에서 널리 사용되는 두 가지 규제 기법인 가중치 규제와 드롭아웃(dropout)을 알아 보겠습니다. 이런 기법을 사용하여 IMDB 영화 리뷰 분류 모델의 성능을 향상시켜 보죠." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5pZ8A2liqvgk" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "!pip install tf-nightly-2.0-preview\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1cweoTiruj8O" - }, - "source": [ - "## IMDB 데이터셋 다운로드\n", - "\n", - "이전 노트북에서처럼 임베딩을 사용하지 않고 여기에서는 문장을 멀티-핫 인코딩(multi-hot encoding)으로 변환하겠습니다. 이 모델은 훈련 세트에 빠르게 과대적합될 것입니다. 과대적합을 발생시키기고 어떻게 해결하는지 보이기 위해 선택했습니다.\n", - "\n", - "멀티-핫 인코딩은 정수 시퀀스를 0과 1로 이루어진 벡터로 변환합니다. 정확하게 말하면 시퀀스 `[3, 5]`를 인덱스 3과 5만 1이고 나머지는 모두 0인 10,000 차원 벡터로 변환한다는 의미입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QpzE4iqZtJly" - }, - "outputs": [], - "source": [ - "NUM_WORDS = 10000\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = keras.datasets.imdb.load_data(num_words=NUM_WORDS)\n", - "\n", - "def multi_hot_sequences(sequences, dimension):\n", - " # 0으로 채워진 (len(sequences), dimension) 크기의 행렬을 만듭니다\n", - " results = np.zeros((len(sequences), dimension))\n", - " for i, word_indices in enumerate(sequences):\n", - " results[i, word_indices] = 1.0 # results[i]의 특정 인덱스만 1로 설정합니다\n", - " return results\n", - "\n", - "\n", - "train_data = multi_hot_sequences(train_data, dimension=NUM_WORDS)\n", - "test_data = multi_hot_sequences(test_data, dimension=NUM_WORDS)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MzWVeXe3NBTn" - }, - "source": [ - "만들어진 멀티-핫 벡터 중 하나를 살펴 보죠. 단어 인덱스는 빈도 순으로 정렬되어 있습니다. 그래프에서 볼 수 있듯이 인덱스 0에 가까울수록 1이 많이 등장합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "71kr5rG4LkGM" - }, - "outputs": [], - "source": [ - "plt.plot(train_data[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lglk41MwvU5o" - }, - "source": [ - "## 과대적합 예제\n", - "\n", - "과대적합을 막는 가장 간단한 방법은 모델의 규모를 축소하는 것입니다. 즉, 모델에 있는 학습 가능한 파라미터의 수를 줄입니다(모델 파라미터는 층(layer)의 개수와 층의 유닛(unit) 개수에 의해 결정됩니다). 딥러닝에서는 모델의 학습 가능한 파라미터의 수를 종종 모델의 \"용량\"이라고 말합니다. 직관적으로 생각해 보면 많은 파라미터를 가진 모델이 더 많은 \"기억 용량\"을 가집니다. 이런 모델은 훈련 샘플과 타깃 사이를 일반화 능력이 없는 딕셔너리와 같은 매핑으로 완벽하게 학습할 수 있습니다. 하지만 이전에 본 적 없는 데이터에서 예측을 할 땐 쓸모가 없을 것입니다.\n", - "\n", - "항상 기억해야 할 점은 딥러닝 모델이 훈련 세트에는 학습이 잘 되는 경향이 있지만 진짜 해결할 문제는 학습이 아니라 일반화라는 것입니다.\n", - "\n", - "반면에 네트워크의 기억 용량이 부족하다면 이런 매핑을 쉽게 학습할 수 없을 것입니다. 손실을 최소화하기 위해서는 예측 성능이 더 많은 압축된 표현을 학습해야 합니다. 또한 너무 작은 모델을 만들면 훈련 데이터를 학습하기 어렵울 것입니다. \"너무 많은 용량\"과 \"충분하지 않은 용량\" 사이의 균형을 잡아야 합니다.\n", - "\n", - "안타깝지만 어떤 모델의 (층의 개수나 뉴런 개수에 해당하는) 적절한 크기나 구조를 결정하는 마법같은 공식은 없습니다. 여러 가지 다른 구조를 사용해 실험을 해봐야만 합니다.\n", - "\n", - "알맞은 모델의 크기를 찾으려면 비교적 적은 수의 층과 파라미터로 시작해서 검증 손실이 감소할 때까지 새로운 층을 추가하거나 층의 크기를 늘리는 것이 좋습니다. 영화 리뷰 분류 네트워크를 사용해 이를 실험해 보죠.\n", - "\n", - "```Dense``` 층만 사용하는 간단한 기준 모델을 만들고 작은 규모의 버전와 큰 버전의 모델을 만들어 비교하겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ReKHdC2EgVu" - }, - "source": [ - "### 기준 모델 만들기" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QKgdXPx9usBa" - }, - "outputs": [], - "source": [ - "baseline_model = keras.Sequential([\n", - " # `.summary` 메서드 때문에 `input_shape`가 필요합니다\n", - " keras.layers.Dense(16, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "baseline_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "baseline_model.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LqG3MXF5xSjR" - }, - "outputs": [], - "source": [ - "baseline_history = baseline_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L-DGRBbGxI6G" - }, - "source": [ - "### 작은 모델 만들기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SrfoVQheYSO5" - }, - "source": [ - "앞서 만든 기준 모델과 비교하기 위해 적은 수의 은닉 유닛을 가진 모델을 만들어 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jksi-XtaxDAh" - }, - "outputs": [], - "source": [ - "smaller_model = keras.Sequential([\n", - " keras.layers.Dense(4, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(4, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "smaller_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "smaller_model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jbngCZliYdma" - }, - "source": [ - "같은 데이터를 사용해 이 모델을 훈련합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Ofn1AwDhx-Fe" - }, - "outputs": [], - "source": [ - "smaller_history = smaller_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vIPuf23FFaVn" - }, - "source": [ - "### 큰 모델 만들기\n", - "\n", - "아주 큰 모델을 만들어 얼마나 빠르게 과대적합이 시작되는지 알아 볼 수 있습니다. 이 문제에 필요한 것보다 훨씬 더 큰 용량을 가진 네트워크를 추가해서 비교해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ghQwwqwqvQM9" - }, - "outputs": [], - "source": [ - "bigger_model = keras.models.Sequential([\n", - " keras.layers.Dense(512, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(512, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "bigger_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "bigger_model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D-d-i5DaYmr7" - }, - "source": [ - "역시 같은 데이터를 사용해 모델을 훈련합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U1A99dhqvepf" - }, - "outputs": [], - "source": [ - "bigger_history = bigger_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fy3CMUZpzH3d" - }, - "source": [ - "### 훈련 손실과 검증 손실 그래프 그리기\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HSlo1F4xHuuM" - }, - "source": [ - "실선은 훈련 손실이고 점선은 검증 손실입니다(낮은 검증 손실이 더 좋은 모델입니다). 여기서는 작은 네트워크가 기준 모델보다 더 늦게 과대적합이 시작되었습니다(즉 에포크 4가 아니라 6에서 시작됩니다). 또한 과대적합이 시작되고 훨씬 천천히 성능이 감소합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0XmKDtOWzOpk" - }, - "outputs": [], - "source": [ - "def plot_history(histories, key='binary_crossentropy'):\n", - " plt.figure(figsize=(16,10))\n", - "\n", - " for name, history in histories:\n", - " val = plt.plot(history.epoch, history.history['val_'+key],\n", - " '--', label=name.title()+' Val')\n", - " plt.plot(history.epoch, history.history[key], color=val[0].get_color(),\n", - " label=name.title()+' Train')\n", - "\n", - " plt.xlabel('Epochs')\n", - " plt.ylabel(key.replace('_',' ').title())\n", - " plt.legend()\n", - "\n", - " plt.xlim([0,max(history.epoch)])\n", - "\n", - "\n", - "plot_history([('baseline', baseline_history),\n", - " ('smaller', smaller_history),\n", - " ('bigger', bigger_history)])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bi6hBhdnSfjA" - }, - "source": [ - "큰 네트워크는 거의 바로 첫 번째 에포크 이후에 과대적합이 시작되고 훨씬 더 심각하게 과대적합됩니다. 네트워크의 용량이 많을수록 훈련 세트를 더 빠르게 모델링할 수 있습니다(훈련 손실이 낮아집니다). 하지만 더 쉽게 과대적합됩니다(훈련 손실과 검증 손실 사이에 큰 차이가 발생합니다)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ASdv7nsgEFhx" - }, - "source": [ - "## 과대적합을 방지하기 위한 전략" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4rHoVWcswFLa" - }, - "source": [ - "### 가중치를 규제하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRxWepNawbBK" - }, - "source": [ - "아마도 오캄의 면도날(Occam's Razor) 이론을 들어 보았을 것입니다. 어떤 것을 설명하는 두 가지 방법이 있다면 더 정확한 설명은 최소한의 가정이 필요한 가장 \"간단한\" 설명일 것입니다. 이는 신경망으로 학습되는 모델에도 적용됩니다. 훈련 데이터와 네트워크 구조가 주어졌을 때 이 데이터를 설명할 수 있는 가중치의 조합(즉, 가능한 모델)은 많습니다. 간단한 모델은 복잡한 것보다 과대적합되는 경향이 작을 것입니다.\n", - "\n", - "여기서 \"간단한 모델\"은 모델 파라미터의 분포를 봤을 때 엔트로피(entropy)가 작은 모델입니다(또는 앞 절에서 보았듯이 적은 파라미터를 가진 모델입니다). 따라서 과대적합을 완화시키는 일반적인 방법은 가중치가 작은 값을 가지도록 네트워크의 복잡도에 제약을 가하는 것입니다. 이는 가중치 값의 분포를 좀 더 균일하게 만들어 줍니다. 이를 \"가중치 규제\"(weight regularization)라고 부릅니다. 네트워크의 손실 함수에 큰 가중치에 해당하는 비용을 추가합니다. 이 비용은 두 가지 형태가 있습니다:\n", - "\n", - "* [L1 규제](https://developers.google.com/machine-learning/glossary/#L1_regularization)는 가중치의 절댓값에 비례하는 비용이 추가됩니다(즉, 가중치의 \"L1 노름(norm)\"을 추가합니다).\n", - "\n", - "* [L2 규제](https://developers.google.com/machine-learning/glossary/#L2_regularization)는 가중치의 제곱에 비례하는 비용이 추가됩니다(즉, 가중치의 \"L2 노름\"의 제곱을 추가합니다). 신경망에서는 L2 규제를 가중치 감쇠(weight decay)라고도 부릅니다. 이름이 다르지만 혼돈하지 마세요. 가중치 감쇠는 수학적으로 L2 규제와 동일합니다.\n", - "\n", - "L1 규제는 일부 가중치 파라미터를 0으로 만듭니다. L2 규제는 가중치 파라미터를 제한하지만 완전히 0으로 만들지는 않습니다. 이것이 L2 규제를 더 많이 사용하는 이유 중 하나입니다.\n", - "\n", - "`tf.keras`에서는 가중치 규제 객체를 층의 키워드 매개변수에 전달하여 가중치에 규제를 추가합니다. L2 가중치 규제를 추가해 보죠." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HFGmcwduwVyQ" - }, - "outputs": [], - "source": [ - "l2_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "l2_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "l2_model_history = l2_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bUUHoXb7w-_C" - }, - "source": [ - "```l2(0.001)```는 네트워크의 전체 손실에 층에 있는 가중치 행렬의 모든 값이 ```0.001 * weight_coefficient_value**2```만큼 더해진다는 의미입니다. 이런 페널티(penalty)는 훈련할 때만 추가됩니다. 따라서 테스트 단계보다 훈련 단계에서 네트워크 손실이 훨씬 더 클 것입니다.\n", - "\n", - "L2 규제의 효과를 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7wkfLyxBZdh_" - }, - "outputs": [], - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('l2', l2_model_history)])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Kx1YHMsVxWjP" - }, - "source": [ - "결과에서 보듯이 모델 파라미터의 개수는 같지만 L2 규제를 적용한 모델이 기본 모델보다 과대적합에 훨씬 잘 견디고 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HmnBNOOVxiG8" - }, - "source": [ - "### 드롭아웃 추가하기\n", - "\n", - "드롭아웃(dropout)은 신경망에서 가장 효과적이고 널리 사용하는 규제 기법 중 하나입니다. 토론토(Toronto) 대학의 힌튼(Hinton)과 그의 제자들이 개발했습니다. 드롭아웃을 층에 적용하면 훈련하는 동안 층의 출력 특성을 랜덤하게 끕니다(즉, 0으로 만듭니다). 훈련하는 동안 어떤 입력 샘플에 대해 [0.2, 0.5, 1.3, 0.8, 1.1] 벡터를 출력하는 층이 있다고 가정해 보죠. 드롭아웃을 적용하면 이 벡터에서 몇 개의 원소가 랜덤하게 0이 됩니다. 예를 들면, [0, 0.5, 1.3, 0, 1.1]가 됩니다. \"드롭아웃 비율\"은 0이 되는 특성의 비율입니다. 보통 0.2에서 0.5 사이를 사용합니다. 테스트 단계에서는 어떤 유닛도 드롭아웃하지 않습니다. 훈련 단계보다 더 많은 유닛이 활성화되기 때문에 균형을 맞추기 위해 층의 출력 값을 드롭아웃 비율만큼 줄입니다.\n", - "\n", - "`tf.keras`에서는 `Dropout` 층을 이용해 네트워크에 드롭아웃을 추가할 수 있습니다. 이 층은 바로 이전 층의 출력에 드롭아웃을 적용합니다.\n", - "\n", - "IMDB 네트워크에 두 개의 `Dropout` 층을 추가하여 과대적합이 얼마나 감소하는지 알아 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "OFEYvtrHxSWS" - }, - "outputs": [], - "source": [ - "dpt_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(16, activation='relu'),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "dpt_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "dpt_model_history = dpt_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "SPZqwVchx5xp" - }, - "outputs": [], - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('dropout', dpt_model_history)])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gjfnkEeQyAFG" - }, - "source": [ - "드롭아웃을 추가하니 기준 모델보다 확실히 향상되었습니다.\n", - "\n", - "정리하면 신경망에서 과대적합을 방지하기 위해 가장 널리 사용하는 방법은 다음과 같습니다:\n", - "\n", - "* 더 많은 훈련 데이터를 모읍니다.\n", - "* 네트워크의 용량을 줄입니다.\n", - "* 가중치 규제를 추가합니다.\n", - "* 드롭아웃을 추가합니다.\n", - "\n", - "이 문서에서 다루지 않은 중요한 방법 두 가지는 데이터 증식(data-augmentation)과 배치 정규화(batch normalization)입니다." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [ - "fTFj8ft5dlbS" - ], - "name": "overfit_and_underfit.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/site/ko/tutorials/keras/regression.ipynb b/site/ko/tutorials/keras/regression.ipynb deleted file mode 100644 index d789356e28d..00000000000 --- a/site/ko/tutorials/keras/regression.ipynb +++ /dev/null @@ -1,887 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "AwOEIRJC6Une" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "KyPEtTqk6VdG" - }, - "outputs": [], - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# 자동차 연비 예측하기: 회귀" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YYwLLNVaaJU9" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "*회귀*(regression)는 가격이나 확률 같이 연속된 출력 값을 예측하는 것이 목적입니다. 이와는 달리 *분류*(classification)는 여러개의 클래스 중 하나의 클래스를 선택하는 것이 목적입니다(예를 들어, 사진에 사과 또는 오렌지가 포함되어 있을 때 어떤 과일인지 인식하는 것).\n", - "\n", - "이 노트북은 [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) 데이터셋을 사용하여 1970년대 후반과 1980년대 초반의 자동차 연비를 예측하는 모델을 만듭니다. 이 기간에 출시된 자동차 정보를 모델에 제공하겠습니다. 이 정보에는 실린더 수, 배기량, 마력(horsepower), 공차 중량 같은 속성이 포함됩니다.\n", - "\n", - "이 예제는 `tf.keras` API를 사용합니다. 자세한 내용은 [케라스 가이드](https://www.tensorflow.org/guide/keras)를 참고하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "moB4tpEHxKB3" - }, - "outputs": [], - "source": [ - "# 산점도 행렬을 그리기 위해 seaborn 패키지를 설치합니다\n", - "!pip install seaborn" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1rRo8oNqZ-Rj" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Auto MPG 데이터셋\n", - "\n", - "이 데이터셋은 [UCI 머신 러닝 저장소](https://archive.ics.uci.edu/ml/)에서 다운로드할 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### 데이터 구하기\n", - "먼저 데이터셋을 다운로드합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "p9kxxgzvzlyz" - }, - "outputs": [], - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "판다스를 사용하여 데이터를 읽습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "CiX2FI4gZtTt" - }, - "outputs": [], - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### 데이터 정제하기\n", - "\n", - "이 데이터셋은 일부 데이터가 누락되어 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JEJHhN65a2VV" - }, - "outputs": [], - "source": [ - "dataset.isna().sum()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "문제를 간단하게 만들기 위해서 누락된 행을 삭제하겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4ZUDosChC1UN" - }, - "outputs": [], - "source": [ - "dataset = dataset.dropna()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "`\"Origin\"` 열은 수치형이 아니고 범주형이므로 원-핫 인코딩(one-hot encoding)으로 변환하겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gWNTD2QjBWFJ" - }, - "outputs": [], - "source": [ - "origin = dataset.pop('Origin')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ulXz4J7PAUzk" - }, - "outputs": [], - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### 데이터셋을 훈련 세트와 테스트 세트로 분할하기\n", - "\n", - "이제 데이터를 훈련 세트와 테스트 세트로 분할합니다.\n", - "\n", - "테스트 세트는 모델을 최종적으로 평가할 때 사용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qn-IGhUE7_1H" - }, - "outputs": [], - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### 데이터 조사하기\n", - "\n", - "훈련 세트에서 몇 개의 열을 선택해 산점도 행렬을 만들어 살펴 보겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "oRKO_x8gWKv-" - }, - "outputs": [], - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "전반적인 통계도 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yi2FzC3T21jR" - }, - "outputs": [], - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### 특성과 레이블 분리하기\n", - "\n", - "특성에서 타깃 값 또는 \"레이블\"을 분리합니다. 이 레이블을 예측하기 위해 모델을 훈련시킬 것입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "t2sluJdCW7jN" - }, - "outputs": [], - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### 데이터 정규화\n", - "\n", - "위 `train_stats` 통계를 다시 살펴보고 각 특성의 범위가 얼마나 다른지 확인해 보죠." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "특성의 스케일과 범위가 다르면 정규화(normalization)하는 것이 권장됩니다. 특성을 정규화하지 않아도 모델이 *수렴할 수 있지만*, 훈련시키기 어렵고 입력 단위에 의존적인 모델이 만들어집니다.\n", - "\n", - "노트: 의도적으로 훈련 세트만 사용하여 통계치를 생성했습니다. 이 통계는 테스트 세트를 정규화할 때에도 사용됩니다. 이는 테스트 세트를 모델이 훈련에 사용했던 것과 동일한 분포로 투영하기 위해서입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JlC5ooJrgjQF" - }, - "outputs": [], - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "정규화된 데이터를 사용하여 모델을 훈련합니다.\n", - "\n", - "주의: 여기에서 입력 데이터를 정규화하기 위해 사용한 통계치(평균과 표준편차)는 원-핫 인코딩과 마찬가지로 모델에 주입되는 모든 데이터에 적용되어야 합니다. 여기에는 테스트 세트는 물론 모델이 실전에 투입되어 얻은 라이브 데이터도 포함됩니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## 모델" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### 모델 만들기\n", - "\n", - "모델을 구성해 보죠. 여기에서는 두 개의 완전 연결(densely connected) 은닉층으로 `Sequential` 모델을 만들겠습니다. 출력 층은 하나의 연속적인 값을 반환합니다. 나중에 두 번째 모델을 만들기 쉽도록 `build_model` 함수로 모델 구성 단계를 감싸겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "c26juK7ZG8j-" - }, - "outputs": [], - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation='relu', input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation='relu'),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mse',\n", - " optimizer=optimizer,\n", - " metrics=['mae', 'mse'])\n", - " return model" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cGbPb-PHGbhs" - }, - "outputs": [], - "source": [ - "model = build_model()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### 모델 확인\n", - "\n", - "`.summary` 메서드를 사용해 모델에 대한 간단한 정보를 출력합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ReAD0n6MsFK-" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "모델을 한번 실행해 보죠. 훈련 세트에서 `10` 샘플을 하나의 배치로 만들어 `model.predict` 메서드를 호출해 보겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-d-gBaVtGTSC" - }, - "outputs": [], - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "제대로 작동하는 것 같네요. 결괏값의 크기와 타입이 기대했던 대로입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### 모델 훈련\n", - "\n", - "이 모델을 1,000번의 에포크(epoch) 동안 훈련합니다. 훈련 정확도와 검증 정확도는 `history` 객체에 기록됩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sD7qHCmNIOY0" - }, - "outputs": [], - "source": [ - "# 에포크가 끝날 때마다 점(.)을 출력해 훈련 진행 과정을 표시합니다\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "`history` 객체에 저장된 통계치를 사용해 모델의 훈련 과정을 시각화해 보죠." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4Xj91b-dymEy" - }, - "outputs": [], - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "B6XriGbVPh2t" - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure(figsize=(8,12))\n", - "\n", - " plt.subplot(2,1,1)\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mae'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mae'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - "\n", - " plt.subplot(2,1,2)\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mse'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mse'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "plot_history(history)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "이 그래프를 보면 수 백번 에포크를 진행한 이후에는 모델이 거의 향상되지 않는 것 같습니다. `model.fit` 메서드를 수정하여 검증 점수가 향상되지 않으면 자동으로 훈련을 멈추도록 만들어 보죠. 에포크마다 훈련 상태를 점검하기 위해 *EarlyStopping 콜백(callback)*을 사용하겠습니다. 지정된 에포크 횟수 동안 성능 향상이 없으면 자동으로 훈련이 멈춥니다.\n", - "\n", - "이 콜백에 대해 더 자세한 내용은 [여기](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping)를 참고하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "fdMZuhUgzMZ4" - }, - "outputs": [], - "source": [ - "model = build_model()\n", - "\n", - "# patience 매개변수는 성능 향상을 체크할 에포크 횟수입니다\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "이 그래프를 보면 검증 세트의 평균 오차가 약 +/- 2 MPG입니다. 좋은 결과인가요? 이에 대한 평가는 여러분에게 맡기겠습니다.\n", - "\n", - "모델을 훈련할 때 사용하지 않았던 **테스트 세트**에서 모델의 성능을 확인해 보죠. 이를 통해 모델이 실전에 투입되었을 때 모델의 성능을 짐작할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jl_yNr5n1kms" - }, - "outputs": [], - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"테스트 세트의 평균 절대 오차: {:5.2f} MPG\".format(mae))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "## 예측\n", - "\n", - "마지막으로 테스트 세트에 있는 샘플을 사용해 MPG 값을 예측해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Xe7RXH3N3CWU" - }, - "outputs": [], - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mU1jBsRLaCeY" - }, - "source": [ - "모델이 꽤 잘 예측한 것 같습니다. 오차의 분포를 살펴 보죠." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "f-OHX4DiXd8x" - }, - "outputs": [], - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3PkzkjFkaCed" - }, - "source": [ - "가우시안 분포가 아니지만 아마도 훈련 샘플의 수가 매우 작기 때문일 것입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## 결론\n", - "\n", - "이 노트북은 회귀 문제를 위한 기법을 소개합니다.\n", - "\n", - "* 평균 제곱 오차(MSE)는 회귀 문제에서 자주 사용하는 손실 함수입니다(분류 문제에서 사용하는 손실 함수와 다릅니다).\n", - "* 비슷하게 회귀에서 사용되는 평가 지표도 분류와 다릅니다. 많이 사용하는 회귀 지표는 평균 절댓값 오차(MAE)입니다.\n", - "* 수치 입력 데이터의 특성이 여러 가지 범위를 가질 때 동일한 범위가 되도록 각 특성의 스케일을 독립적으로 조정해야 합니다.\n", - "* 훈련 데이터가 많지 않다면 과대적합을 피하기 위해 은닉층의 개수가 적은 소규모 네트워크를 선택하는 방법이 좋습니다.\n", - "* 조기 종료(Early stopping)은 과대적합을 방지하기 위한 좋은 방법입니다." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "regression.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/site/ko/tutorials/keras/save_and_load.ipynb b/site/ko/tutorials/keras/save_and_load.ipynb deleted file mode 100644 index bccfccbd032..00000000000 --- a/site/ko/tutorials/keras/save_and_load.ipynb +++ /dev/null @@ -1,898 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "2pHVBk_seED1" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "N_fMsQ-N8I7j" - }, - "outputs": [], - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pZJ3uY9O17VN" - }, - "source": [ - "# 모델 저장과 복원" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M4Ata7_wMul1" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Uh4aUT57arZH" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBdde4YJeJKF" - }, - "source": [ - "훈련하는 도중이나 훈련이 끝난 후에 모델을 저장할 수 있습니다. 모델을 중지된 지점부터 다시 훈련할 수 있어 한 번에 오랫동안 훈련하지 않아도 됩니다. 또 모델을 저장하면 다른 사람에게 공유할 수 있고 작업을 재현할 수 있습니다. 연구한 모델과 기법을 공개할 때 많은 머신 러닝 기술자들이 다음과 같은 것들을 제공합니다:\n", - "\n", - "* 모델을 만드는 코드\n", - "* 모델의 훈련된 가중치 또는 파라미터\n", - "\n", - "이런 데이터를 공유하면 다른 사람들이 모델의 작동 방식을 이해하고 새로운 데이터로 모델을 실험하는데 도움이 됩니다.\n", - "\n", - "주의: 신뢰할 수 없는 코드는 조심하세요. 텐서플로 모델은 프로그램 코드입니다. 자세한 내용은 [텐서플로를 안전하게 사용하기](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) 문서를 참고하세요.\n", - "\n", - "### 저장 방식\n", - "\n", - "사용하는 API에 따라서 여러가지 방법으로 텐서플로 모델을 저장할 수 있습니다. 이 문서는 텐서플로 모델을 만들고 훈련하기 위한 고수준 API인 [tf.keras](https://www.tensorflow.org/guide/keras)를 사용합니다. 다른 방법들에 대해서는 텐서플로의 [저장과 복원](https://www.tensorflow.org/guide/saved_model) 문서와 즉시 실행(eager execution) 문서의 [저장하기](https://www.tensorflow.org/guide/eager#object-based_saving) 섹션을 참고하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xCUREq7WXgvg" - }, - "source": [ - "## 설정\n", - "\n", - "### 설치와 임포트" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7l0MiTOrXtNv" - }, - "source": [ - "필요한 라이브러리를 설치하고 텐서플로를 임포트(import)합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RzIOVSdnMYyO" - }, - "outputs": [], - "source": [ - "!pip install h5py pyyaml" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbGsznErXWt6" - }, - "source": [ - "### 예제 데이터셋 받기\n", - "\n", - "[MNIST 데이터셋](http://yann.lecun.com/exdb/mnist/)으로 모델을 훈련하여 가중치를 저장하는 예제를 만들어 보겠습니다. 모델 실행 속도를 빠르게 하기 위해 샘플에서 처음 1,000개만 사용겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7Nm7Tyb-gRt-" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import os\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "tf.__version__" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9rGfFwE9XVwz" - }, - "outputs": [], - "source": [ - "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_labels = train_labels[:1000]\n", - "test_labels = test_labels[:1000]\n", - "\n", - "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", - "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "anG3iVoXyZGI" - }, - "source": [ - "### 모델 정의" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wynsOBfby0Pa" - }, - "source": [ - "가중치를 저장하고 불러오는 예제를 위해 간단한 모델을 만들어 보죠." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0HZbJIjxyX1S" - }, - "outputs": [], - "source": [ - "# 간단한 Sequential 모델을 반환합니다\n", - "def create_model():\n", - " model = tf.keras.models.Sequential([\n", - " keras.layers.Dense(512, activation='relu', input_shape=(784,)),\n", - " keras.layers.Dropout(0.2),\n", - " keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - "\n", - " model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - " return model\n", - "\n", - "\n", - "# 모델 객체를 만듭니다\n", - "model = create_model()\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "soDE0W_KH8rG" - }, - "source": [ - "## 훈련하는 동안 체크포인트 저장하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRyd5qQQIXZm" - }, - "source": [ - "*훈련 중간*과 *훈련 마지막*에 체크포인트(checkpoint)를 자동으로 저장하도록 하는 것이 많이 사용하는 방법입니다. 다시 훈련하지 않고 모델을 재사용하거나 훈련 과정이 중지된 경우 이어서 훈련을 진행할 수 있습니다.\n", - "\n", - "`tf.keras.callbacks.ModelCheckpoint`은 이런 작업을 수행하는 콜백(callback)입니다. 이 콜백은 체크포인트 작업을 조정할 수 있도록 여러가지 매개변수를 제공합니다.\n", - "\n", - "### 체크포인트 콜백 사용하기\n", - "\n", - "`ModelCheckpoint` 콜백을 전달하여 모델을 훈련해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IFPuhwntH8VH" - }, - "outputs": [], - "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# 체크포인트 콜백 만들기\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,\n", - " save_weights_only=True,\n", - " verbose=1)\n", - "\n", - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs = 10,\n", - " validation_data = (test_images,test_labels),\n", - " callbacks = [cp_callback]) # 훈련 단계에 콜백을 전달합니다\n", - "\n", - "# 옵티마이저의 상태를 저장하는 것과 관련되어 경고가 발생할 수 있습니다.\n", - "# 이 경고는 (그리고 이 노트북의 다른 비슷한 경고는) 이전 사용 방식을 권장하지 않기 위함이며 무시해도 좋습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlM-sgyJO084" - }, - "source": [ - "이 코드는 텐서플로 체크포인트 파일을 만들고 에포크가 종료될 때마다 업데이트합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gXG5FVKFOVQ3" - }, - "outputs": [], - "source": [ - "!ls {checkpoint_dir}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wlRN_f56Pqa9" - }, - "source": [ - "훈련하지 않은 새로운 모델을 만들어 보겠습니다. 가중치만 복원할 땐 원본 모델과 동일한 구조로 모델을 만들어야 합니다. 여기서는 동일한 구조로 모델을 만들었으므로 다른 *객체*이지만 가중치를 공유할 수 있습니다.\n", - "\n", - "훈련하지 않은 새 모델을 만들고 테스트 세트에서 평가해 보죠. 훈련되지 않은 모델의 성능은 무작위로 선택하는 정도의 수준입니다(~10% 정확도):" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Fp5gbuiaPqCT" - }, - "outputs": [], - "source": [ - "model = create_model()\n", - "\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"훈련되지 않은 모델의 정확도: {:5.2f}%\".format(100*acc))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1DTKpZssRSo3" - }, - "source": [ - "체크포인트에서 가중치를 로드하고 다시 평가해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2IZxbwiRRSD2" - }, - "outputs": [], - "source": [ - "model.load_weights(checkpoint_path)\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpAbKkAyVPV8" - }, - "source": [ - "### 체크포인트 콜백 매개변수\n", - "\n", - "이 콜백 함수는 몇 가지 매개변수를 제공합니다. 체크포인트 이름을 고유하게 만들거나 체크포인트 주기를 조정할 수 있습니다.\n", - "\n", - "새로운 모델을 훈련하고 다섯 번의 에포크마다 고유한 이름으로 체크포인트를 저장해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mQF_dlgIVOvq" - }, - "outputs": [], - "source": [ - "# 파일 이름에 에포크 번호를 포함시킵니다(`str.format` 포맷)\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " checkpoint_path, verbose=1, save_weights_only=True,\n", - " # 다섯 번째 에포크마다 가중치를 저장합니다\n", - " period=5)\n", - "\n", - "model = create_model()\n", - "model.save_weights(checkpoint_path.format(epoch=0))\n", - "model.fit(train_images, train_labels,\n", - " epochs = 50, callbacks = [cp_callback],\n", - " validation_data = (test_images,test_labels),\n", - " verbose=0)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1zFrKTjjavWI" - }, - "source": [ - "만들어진 체크포인트를 확인해 보고 마지막 체크포인트를 선택해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "p64q3-V4sXt0" - }, - "outputs": [], - "source": [ - "! ls {checkpoint_dir}" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1AN_fnuyR41H" - }, - "outputs": [], - "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", - "latest" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zk2ciGbKg561" - }, - "source": [ - "노트: 텐서플로는 기본적으로 최근 5개의 체크포인트만 저장합니다.\n", - "\n", - "모델을 초기화하고 최근 체크포인트를 로드하여 테스트해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3M04jyK-H3QK" - }, - "outputs": [], - "source": [ - "model = create_model()\n", - "model.load_weights(latest)\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c2OxsJOTHxia" - }, - "source": [ - "## 체크포인트 파일" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtdYhvWnH2ib" - }, - "source": [ - "위 코드는 가중치를 일련의 [체크포인트](https://www.tensorflow.org/guide/saved_model#save_and_restore_variables) 포맷의 파일에 저장합니다. 이 파일에 포함되는 것은 훈련된 이진 포맷의 가중치입니다. 체크포인트가 담고 있는 것은 다음과 같습니다:\n", - "\n", - "* 모델의 가중치를 포함하는 하나 이상의 샤드(shard)\n", - "* 가중치가 어느 샤드에 저장되어 있는지를 나타내는 인덱스 파일\n", - "\n", - "단일 머신에서 모델을 훈련한다면 `.data-00000-of-00001` 확장자를 가진 샤드 하나만 만들어 집니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S_FA-ZvxuXQV" - }, - "source": [ - "## 수동으로 가중치 저장하기\n", - "\n", - "앞에서 가중치를 모델에 로드하는 방법을 보았습니다.\n", - "\n", - "수동으로 가중치를 저장하는 것도 쉽습니다. `Model.save_weights` 메서드를 사용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "R7W5plyZ-u9X" - }, - "outputs": [], - "source": [ - "# 가중치를 저장합니다\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# 가중치를 복원합니다\n", - "model = create_model()\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", - "\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kOGlxPRBEvV1" - }, - "source": [ - "## 모델 전체를 저장하기\n", - "\n", - "전체 모델을 파일 하나에 저장할 수 있습니다. 여기에는 가중치, 모델 구성 심지어 옵티마이저에 지정한 설정까지 포함됩니다. 모델의 체크포인트를 저장하므로 원본 코드를 사용하지 않고 나중에 정확히 동일한 상태에서 훈련을 다시 시작할 수 있습니다.\n", - "\n", - "전체 모델을 저장하는 기능은 매우 유용합니다. TensorFlow.js로 모델을 로드한 다음 웹 브라우저에서 모델을 훈련하고 실행할 수 있습니다([HDF5](https://js.tensorflow.org/tutorials/import-keras.html), [Saved Model](https://js.tensorflow.org/tutorials/import-saved-model.html)). 또는 모바일 장치에 맞도록 변환한 다음 TensorFlow Lite를 사용하여 실행할 수 있습니다([HDF5](https://www.tensorflow.org/lite/convert/python_api#exporting_a_tfkeras_file_), [Saved Model](https://www.tensorflow.org/lite/convert/python_api#exporting_a_savedmodel_))." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" - }, - "source": [ - "### HDF5 파일로 저장하기\n", - "\n", - "케라스는 [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) 표준을 따르는 기본 저장 포맷을 제공합니다. 저장된 모델을 하나의 이진 파일(binary blob)처럼 다룰 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "m2dkmJVCGUia" - }, - "outputs": [], - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)\n", - "\n", - "# 전체 모델을 HDF5 파일로 저장합니다\n", - "model.save('my_model.h5')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" - }, - "source": [ - "이제 이 파일로부터 모델을 다시 만들어 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5NDMO_7kS6Do" - }, - "outputs": [], - "source": [ - "# 가중치와 옵티마이저를 포함하여 정확히 동일한 모델을 다시 생성합니다\n", - "new_model = keras.models.load_model('my_model.h5')\n", - "new_model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" - }, - "source": [ - "정확도를 확인해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "jwEaj9DnTCVA" - }, - "outputs": [], - "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "이 기법은 모든 것을 저장합니다:\n", - "\n", - "* 가중치 값\n", - "* 모델 설정(구조)\n", - "* 옵티마이저 설정\n", - "\n", - "케라스는 모델 구조를 확인하고 저장합니다. 현재는 텐서플로 옵티마이저(`tf.train`)를 저장할 수 없습니다. 이런 경우에는 모델을 로드한 후에 다시 컴파일해야 합니다. 옵티마이저의 상태는 유지되지 않습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPyhgcoVzqUB" - }, - "source": [ - "### `saved_model`을 사용하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LtcN4VIb7JkK" - }, - "source": [ - "주의: `tf.keras` 모델을 저장하는 이 메서드는 실험적이므로 향후 버전에서 변경될 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DSWiSB0Q8c46" - }, - "source": [ - "새로운 모델을 만들어 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sI1YvCDFzpl3" - }, - "outputs": [], - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iUvT_3qE8hV5" - }, - "source": [ - "`saved_model`을 만들어 타임스탬프를 이름으로 가진 디렉토리에 저장합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sq8fPglI1RWA" - }, - "outputs": [], - "source": [ - "import time\n", - "saved_model_path = \"./saved_models/{}\".format(int(time.time()))\n", - "\n", - "tf.keras.experimental.export_saved_model(model, saved_model_path)\n", - "saved_model_path" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MjpmyPfh8-1n" - }, - "source": [ - "저장된 모델의 목록을 확인합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ZtOvxA7V0iTv" - }, - "outputs": [], - "source": [ - "!ls saved_models/" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B7qfpvpY9HCe" - }, - "source": [ - "저장된 모델로부터 새로운 케라스 모델을 로드합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0YofwHdN0pxa" - }, - "outputs": [], - "source": [ - "new_model = tf.keras.experimental.load_from_saved_model(saved_model_path)\n", - "new_model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" - }, - "source": [ - "복원된 모델을 실행합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Yh5Mu0yOgE5J" - }, - "outputs": [], - "source": [ - "model.predict(test_images).shape" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Pc9e6G6w1AWG" - }, - "outputs": [], - "source": [ - "# 이 모델을 평가하려면 그전에 컴파일해야 합니다.\n", - "# 단지 저장된 모델의 배포라면 이 단계가 필요하지 않습니다.\n", - "\n", - "new_model.compile(optimizer=model.optimizer, # 복원된 옵티마이저를 사용합니다.\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# 복원된 모델을 평가합니다\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"복원된 모델의 정확도: {:5.2f}%\".format(100*acc))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eUYTzSz5VxL2" - }, - "source": [ - "## 그 다음엔\n", - "\n", - "\n", - "이 문서에서는 `tf.keras`를 사용하여 모델을 저장하고 로드하는 방법을 간단하게 안내했습니다.\n", - "\n", - "* [tf.keras 가이드](https://www.tensorflow.org/guide/keras)에서 `tf.keras`로 모델을 저장하고 로드하는 정보를 더 볼 수 있습니다.\n", - "\n", - "* 즉시 실행 모드에서 모델을 저장하려면 [즉시 실행에서 저장하기](https://www.tensorflow.org/guide/eager#object_based_saving) 문서를 참고하세요.\n", - "\n", - "* [저장과 복원](https://www.tensorflow.org/guide/saved_model) 문서는 저수준의 텐서플로 저장 기능에 대해 설명합니다." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "save_and_load.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/site/ko/tutorials/keras/text_classification.ipynb b/site/ko/tutorials/keras/text_classification.ipynb deleted file mode 100644 index b1106fcb73b..00000000000 --- a/site/ko/tutorials/keras/text_classification.ipynb +++ /dev/null @@ -1,742 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "ioaprt5q5US7" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "yCl0eTNH5RS3" - }, - "outputs": [], - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# 영화 리뷰를 사용한 텍스트 분류" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CMrWLbtWaZWE" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "이 노트북은 영화 리뷰(review) 텍스트를 *긍정*(positive) 또는 *부정*(negative)으로 분류합니다. 이 예제는 *이진*(binary)-또는 클래스(class)가 두 개인- 분류 문제입니다. 이진 분류는 머신러닝에서 중요하고 널리 사용됩니다.\n", - "\n", - "여기에서는 [인터넷 영화 데이터베이스](https://www.imdb.com/)(Internet Movie Database)에서 수집한 50,000개의 영화 리뷰 텍스트를 담은 [IMDB 데이터셋](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb)을 사용하겠습니다. 25,000개 리뷰는 훈련용으로, 25,000개는 테스트용으로 나뉘어져 있습니다. 훈련 세트와 테스트 세트의 클래스는 *균형*이 잡혀 있습니다. 즉 긍정적인 리뷰와 부정적인 리뷰의 개수가 동일합니다.\n", - "\n", - "이 노트북은 모델을 만들고 훈련하기 위해 텐서플로의 고수준 파이썬 API인 [tf.keras](https://www.tensorflow.org/guide/keras)를 사용합니다. `tf.keras`를 사용한 고급 텍스트 분류 튜토리얼은 [MLCC 텍스트 분류 가이드](https://developers.google.com/machine-learning/guides/text-classification/)를 참고하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2ew7HTbPpCJH" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tf-nightly-2.0-preview\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## IMDB 데이터셋 다운로드\n", - "\n", - "IMDB 데이터셋은 텐서플로와 함께 제공됩니다. 리뷰(단어의 시퀀스(sequence))는 미리 전처리해서 정수 시퀀스로 변환되어 있습니다. 각 정수는 어휘 사전에 있는 특정 단어를 의미합니다.\n", - "\n", - "다음 코드는 IMDB 데이터셋을 컴퓨터에 다운로드합니다(또는 이전에 다운로드 받았다면 캐시된 복사본을 사용합니다):" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zXXx5Oc3pOmN" - }, - "outputs": [], - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "매개변수 `num_words=10000`은 훈련 데이터에서 가장 많이 등장하는 상위 10,000개의 단어를 선택합니다. 데이터 크기를 적당하게 유지하기 위해 드물에 등장하는 단어는 제외하겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## 데이터 탐색\n", - "\n", - "잠시 데이터 형태를 알아 보죠. 이 데이터셋의 샘플은 전처리된 정수 배열입니다. 이 정수는 영화 리뷰에 나오는 단어를 나타냅니다. 레이블(label)은 정수 0 또는 1입니다. 0은 부정적인 리뷰이고 1은 긍정적인 리뷰입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "y8qCnve_-lkO" - }, - "outputs": [], - "source": [ - "print(\"훈련 샘플: {}, 레이블: {}\".format(len(train_data), len(train_labels)))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "리뷰 텍스트는 어휘 사전의 특정 단어를 나타내는 정수로 변환되어 있습니다. 첫 번째 리뷰를 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QtTS4kpEpjbi" - }, - "outputs": [], - "source": [ - "print(train_data[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "영화 리뷰들은 길이가 다릅니다. 다음 코드는 첫 번째 리뷰와 두 번째 리뷰에서 단어의 개수를 출력합니다. 신경망의 입력은 길이가 같아야 하기 때문에 나중에 이 문제를 해결하겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr" - }, - "outputs": [], - "source": [ - "len(train_data[0]), len(train_data[1])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### 정수를 단어로 다시 변환하기\n", - "\n", - "정수를 다시 텍스트로 변환하는 방법이 있다면 유용할 것입니다. 여기에서는 정수와 문자열을 매핑한 딕셔너리(dictionary) 객체에 질의하는\u001c", - " 헬퍼(helper) 함수를 만들겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tr5s_1alpzop" - }, - "outputs": [], - "source": [ - "# 단어와 정수 인덱스를 매핑한 딕셔너리\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# 처음 몇 개 인덱스는 사전에 정의되어 있습니다\n", - "word_index = {k:(v+3) for k,v in word_index.items()}\n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # unknown\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "이제 `decode_review` 함수를 사용해 첫 번째 리뷰 텍스트를 출력할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "s_OqxmH6-lkn" - }, - "outputs": [], - "source": [ - "decode_review(train_data[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## 데이터 준비\n", - "\n", - "리뷰-정수 배열-는 신경망에 주입하기 전에 텐서로 변환되어야 합니다. 변환하는 방법에는 몇 가지가 있습니다:\n", - "\n", - "* 원-핫 인코딩(one-hot encoding)은 정수 배열을 0과 1로 이루어진 벡터로 변환합니다. 예를 들어 배열 [3, 5]을 인덱스 3과 5만 1이고 나머지는 모두 0인 10,000차원 벡터로 변환할 수 있습니다. 그다음 실수 벡터 데이터를 다룰 수 있는 층-Dense 층-을 신경망의 첫 번째 층으로 사용합니다. 이 방법은 `num_words * num_reviews` 크기의 행렬이 필요하기 때문에 메모리를 많이 사용합니다.\n", - "* 다른 방법으로는, 정수 배열의 길이가 모두 같도록 패딩(padding)을 추가해 `max_length * num_reviews` 크기의 정수 텐서를 만듭니다. 이런 형태의 텐서를 다룰 수 있는 임베딩(embedding) 층을 신경망의 첫 번째 층으로 사용할 수 있습니다.\n", - "\n", - "이 튜토리얼에서는 두 번째 방식을 사용하겠습니다.\n", - "\n", - "영화 리뷰의 길이가 같아야 하므로 [pad_sequences](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences) 함수를 사용해 길이를 맞추겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2jQv-omsHurp" - }, - "outputs": [], - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "샘플의 길이를 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "USSSBnkE-lky" - }, - "outputs": [], - "source": [ - "len(train_data[0]), len(train_data[1])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "(패딩된) 첫 번째 리뷰 내용을 확인해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "TG8X9cqi-lk9" - }, - "outputs": [], - "source": [ - "print(train_data[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## 모델 구성\n", - "\n", - "신경망은 층(layer)을 쌓아서 만듭니다. 이 구조에서는 두 가지를 결정해야 합니다:\n", - "\n", - "* 모델에서 얼마나 많은 층을 사용할 것인가?\n", - "* 각 층에서 얼마나 많은 *은닉 유닛*(hidden unit)을 사용할 것인가?\n", - "\n", - "이 예제의 입력 데이터는 단어 인덱스의 배열입니다. 예측할 레이블은 0 또는 1입니다. 이 문제에 맞는 모델을 구성해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xpKOoWgu-llD" - }, - "outputs": [], - "source": [ - "# 입력 크기는 영화 리뷰 데이터셋에 적용된 어휘 사전의 크기입니다(10,000개의 단어)\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16, input_shape=(None,)))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation='relu'))\n", - "model.add(keras.layers.Dense(1, activation='sigmoid'))\n", - "\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "층을 순서대로 쌓아 분류기(classifier)를 만듭니다:\n", - "\n", - "1. 첫 번째 층은 `Embedding` 층입니다. 이 층은 정수로 인코딩된 단어를 입력 받고 각 단어 인덱스에 해당하는 임베딩 벡터를 찾습니다. 이 벡터는 모델이 훈련되면서 학습됩니다. 이 벡터는 출력 배열에 새로운 차원으로 추가됩니다. 최종 차원은 `(batch, sequence, embedding)`이 됩니다.\n", - "2. 그다음 `GlobalAveragePooling1D` 층은 `sequence` 차원에 대해 평균을 계산하여 각 샘플에 대해 고정된 길이의 출력 벡터를 반환합니다. 이는 길이가 다른 입력을 다루는 가장 간단한 방법입니다.\n", - "3. 이 고정 길이의 출력 벡터는 16개의 은닉 유닛을 가진 완전 연결(fully-connected) 층(`Dense`)을 거칩니다.\n", - "4. 마지막 층은 하나의 출력 노드(node)를 가진 완전 연결 층입니다. `sigmoid` 활성화 함수를 사용하여 0과 1 사이의 실수를 출력합니다. 이 값은 확률 또는 신뢰도를 나타냅니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### 은닉 유닛\n", - "\n", - "위 모델에는 입력과 출력 사이에 두 개의 중간 또는 \"은닉\" 층이 있습니다. 출력(유닛 또는 노드, 뉴런)의 개수는 층이 가진 표현 공간(representational space)의 차원이 됩니다. 다른 말로 하면, 내부 표현을 학습할 때 허용되는 네트워크 자유도의 양입니다.\n", - "\n", - "모델에 많은 은닉 유닛(고차원의 표현 공간)과 층이 있다면 네트워크는 더 복잡한 표현을 학습할 수 있습니다. 하지만 네트워크의 계산 비용이 많이 들고 원치않는 패턴을 학습할 수도 있습니다. 이런 표현은 훈련 데이터의 성능을 향상시키지만 테스트 데이터에서는 그렇지 못합니다. 이를 *과대적합*(overfitting)이라고 부릅니다. 나중에 이에 대해 알아 보겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### 손실 함수와 옵티마이저\n", - "\n", - "모델이 훈련하려면 손실 함수(loss function)과 옵티마이저(optimizer)가 필요합니다. 이 예제는 이진 분류 문제이고 모델이 확률을 출력하므로(출력층의 유닛이 하나이고 `sigmoid` 활성화 함수를 사용합니다), `binary_crossentropy` 손실 함수를 사용하겠습니다.\n", - "\n", - "다른 손실 함수를 선택할 수 없는 것은 아닙니다. 예를 들어 `mean_squared_error`를 선택할 수 있습니다. 하지만 일반적으로 `binary_crossentropy`가 확률을 다루는데 적합합니다. 이 함수는 확률 분포 간의 거리를 측정합니다. 여기에서는 정답인 타깃 분포와 예측 분포 사이의 거리입니다.\n", - "\n", - "나중에 회귀(regression) 문제(예를 들어 주택 가격을 예측하는 문제)에 대해 살펴 볼 때 평균 제곱 오차(mean squared error) 손실 함수를 어떻게 사용하는지 알아 보겠습니다.\n", - "\n", - "이제 모델이 사용할 옵티마이저와 손실 함수를 설정해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Mr0GP-cQ-llN" - }, - "outputs": [], - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "## 검증 세트 만들기\n", - "\n", - "모델을 훈련할 때 모델이 만난 적 없는 데이터에서 정확도를 확인하는 것이 좋습니다. 원본 훈련 데이터에서 10,000개의 샘플을 떼어내어 *검증 세트*(validation set)를 만들겠습니다. (왜 테스트 세트를 사용하지 않을까요? 훈련 데이터만을 사용하여 모델을 개발하고 튜닝하는 것이 목표입니다. 그다음 테스트 세트를 사용해서 딱 한 번만 정확도를 평가합니다)." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-NpcXY9--llS" - }, - "outputs": [], - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## 모델 훈련\n", - "\n", - "이 모델을 512개의 샘플로 이루어진 미니배치(mini-batch)에서 40번의 에포크(epoch) 동안 훈련합니다. `x_train`과 `y_train` 텐서에 있는 모든 샘플에 대해 40번 반복한다는 뜻입니다. 훈련하는 동안 10,000개의 검증 세트에서 모델의 손실과 정확도를 모니터링합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tXSGrjWZ-llW" - }, - "outputs": [], - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## 모델 평가\n", - "\n", - "모델의 성능을 확인해 보죠. 두 개의 값이 반환됩니다. 손실(오차를 나타내는 숫자이므로 낮을수록 좋습니다)과 정확도입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zOMKywn4zReN" - }, - "outputs": [], - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "이 예제는 매우 단순한 방식을 사용하므로 87% 정도의 정확도를 달성했습니다. 고급 방법을 사용한 모델은 95%에 가까운 정확도를 얻습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## 정확도와 손실 그래프 그리기\n", - "\n", - "`model.fit()`은 `History` 객체를 반환합니다. 여기에는 훈련하는 동안 일어난 모든 정보가 담긴 딕셔너리(dictionary)가 들어 있습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VcvSXvhp-llb" - }, - "outputs": [], - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "네 개의 항목이 있습니다. 훈련과 검증 단계에서 모니터링하는 지표들입니다. 훈련 손실과 검증 손실을 그래프로 그려 보고, 훈련 정확도와 검증 정확도도 그래프로 그려서 비교해 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nGoYf2Js-lle" - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history_dict['accuracy']\n", - "val_acc = history_dict['val_accuracy']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# \"bo\"는 \"파란색 점\"입니다\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "# b는 \"파란 실선\"입니다\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6hXx-xOv-llh" - }, - "outputs": [], - "source": [ - "plt.clf() # 그림을 초기화합니다\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "이 그래프에서 점선은 훈련 손실과 훈련 정확도를 나타냅니다. 실선은 검증 손실과 검증 정확도입니다.\n", - "\n", - "훈련 손실은 에포크마다 *감소*하고 훈련 정확도는 *증가*한다는 것을 주목하세요. 경사 하강법 최적화를 사용할 때 볼 수 있는 현상입니다. 매 반복마다 최적화 대상의 값을 최소화합니다.\n", - "\n", - "하지만 검증 손실과 검증 정확도에서는 그렇지 못합니다. 약 20번째 에포크 이후가 최적점인 것 같습니다. 이는 과대적합 때문입니다. 이전에 본 적 없는 데이터보다 훈련 데이터에서 더 잘 동작합니다. 이 지점부터는 모델이 과도하게 최적화되어 테스트 데이터에서 *일반화*되기 어려운 훈련 데이터의 특정 표현을 학습합니다.\n", - "\n", - "여기에서는 과대적합을 막기 위해 단순히 20번째 에포크 근처에서 훈련을 멈출 수 있습니다. 나중에 콜백(callback)을 사용하여 자동으로 이렇게 하는 방법을 배워 보겠습니다." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "text_classification.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/site/ko/tutorials/keras/text_classification_with_hub.ipynb b/site/ko/tutorials/keras/text_classification_with_hub.ipynb deleted file mode 100644 index e96924383d1..00000000000 --- a/site/ko/tutorials/keras/text_classification_with_hub.ipynb +++ /dev/null @@ -1,487 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "ioaprt5q5US7" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "yCl0eTNH5RS3" - }, - "outputs": [], - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# 케라스와 텐서플로 허브를 사용한 영화 리뷰 텍스트 분류하기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yX7gdv5JgUWp" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/keras/text_classification_with_hub\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eTensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/keras/text_classification_with_hub.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/keras/text_classification_with_hub.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YJ49GhXrgUWq" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "txOb1UvRgUWr" - }, - "source": [ - "이 노트북은 영화 리뷰(review) 텍스트를 *긍정*(positive) 또는 *부정*(negative)으로 분류합니다. 이 예제는 *이진*(binary)-또는 클래스(class)가 두 개인- 분류 문제입니다. 이진 분류는 머신러닝에서 중요하고 널리 사용됩니다.\n", - "\n", - "이 튜토리얼에서는 텐서플로 허브(TensorFlow Hub)와 케라스(Keras)를 사용한 기초적인 전이 학습(transfer learning) 애플리케이션을 보여줍니다.\n", - "\n", - "여기에서는 [인터넷 영화 데이터베이스](https://www.imdb.com/)(Internet Movie Database)에서 수집한 50,000개의 영화 리뷰 텍스트를 담은 [IMDB 데이터셋](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb)을 사용하겠습니다. 25,000개 리뷰는 훈련용으로, 25,000개는 테스트용으로 나뉘어져 있습니다. 훈련 세트와 테스트 세트의 클래스는 *균형*이 잡혀 있습니다. 즉 긍정적인 리뷰와 부정적인 리뷰의 개수가 동일합니다.\n", - "\n", - "이 노트북은 텐서플로에서 모델을 만들고 훈련하기 위한 고수준 파이썬 API인 [tf.keras](https://www.tensorflow.org/guide/keras)와 전이 학습 라이브러리이자 플랫폼인 [텐서플로 허브](https://www.tensorflow.org/hub)를 사용합니다. `tf.keras`를 사용한 고급 텍스트 분류 튜토리얼은 [MLCC 텍스트 분류 가이드](https://developers.google.com/machine-learning/guides/text-classification/)를 참고하세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2ew7HTbPpCJH" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import tensorflow_hub as hub\n", - "import tensorflow_datasets as tfds\n", - "\n", - "print(\"버전: \", tf.__version__)\n", - "print(\"즉시 실행 모드: \", tf.executing_eagerly())\n", - "print(\"허브 버전: \", hub.__version__)\n", - "print(\"GPU \", \"사용 가능\" if tf.config.experimental.list_physical_devices(\"GPU\") else \"사용 불가능\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## IMDB 데이터셋 다운로드\n", - "\n", - "IMDB 데이터셋은 [텐서플로 데이터셋](https://github.com/tensorflow/datasets)(TensorFlow datasets)에 포함되어 있습니다. 다음 코드는 IMDB 데이터셋을 컴퓨터(또는 코랩 런타임)에 다운로드합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zXXx5Oc3pOmN" - }, - "outputs": [], - "source": [ - "# 훈련 세트를 6대 4로 나눕니다.\n", - "# 결국 훈련에 15,000개 샘플, 검증에 10,000개 샘플, 테스트에 25,000개 샘플을 사용하게 됩니다.\n", - "train_validation_split = tfds.Split.TRAIN.subsplit([6, 4])\n", - "\n", - "(train_data, validation_data), test_data = tfds.load(\n", - " name=\"imdb_reviews\", \n", - " split=(train_validation_split, tfds.Split.TEST),\n", - " as_supervised=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## 데이터 탐색\n", - "\n", - "잠시 데이터 형태를 알아 보죠. 이 데이터셋의 샘플은 전처리된 정수 배열입니다. 이 정수는 영화 리뷰에 나오는 단어를 나타냅니다. 레이블(label)은 정수 0 또는 1입니다. 0은 부정적인 리뷰이고 1은 긍정적인 리뷰입니다.\n", - "\n", - "처음 10개의 샘플을 출력해 보겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "QtTS4kpEpjbi" - }, - "outputs": [], - "source": [ - "train_examples_batch, train_labels_batch = next(iter(train_data.batch(10)))\n", - "train_examples_batch" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IFtaCHTdc-GY" - }, - "source": [ - "처음 10개의 레이블도 출력해 보겠습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tvAjVXOWc6Mj" - }, - "outputs": [], - "source": [ - "train_labels_batch" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## 모델 구성\n", - "\n", - "신경망은 층을 쌓아서 만듭니다. 여기에는 세 개의 중요한 구조적 결정이 필요합니다:\n", - "\n", - "* 어떻게 텍스트를 표현할 것인가?\n", - "* 모델에서 얼마나 많은 층을 사용할 것인가?\n", - "* 각 층에서 얼마나 많은 *은닉 유닛*(hidden unit)을 사용할 것인가?\n", - "\n", - "이 예제의 입력 데이터는 문장으로 구성됩니다. 예측할 레이블은 0 또는 1입니다.\n", - "\n", - "텍스트를 표현하는 한 가지 방법은 문장을 임베딩(embedding) 벡터로 바꾸는 것입니다. 그러면 첫 번째 층으로 사전 훈련(pre-trained)된 텍스트 임베딩을 사용할 수 있습니다. 여기에는 두 가지 장점이 있습니다.\n", - "\n", - "* 텍스트 전처리에 대해 신경 쓸 필요가 없습니다.\n", - "* 전이 학습의 장점을 이용합니다.\n", - "* 임베딩은 고정 크기이기 때문에 처리 과정이 단순해집니다.\n", - "\n", - "이 예제는 [텐서플로 허브](https://www.tensorflow.org/hub)에 있는 **사전 훈련된 텍스트 임베딩 모델**인 [google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1)을 사용하겠습니다.\n", - "\n", - "테스트해 볼 수 있는 사전 훈련된 모델이 세 개 더 있습니다:\n", - "\n", - "* [google/tf2-preview/gnews-swivel-20dim-with-oov/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim-with-oov/1) - [google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1)와 동일하지만 어휘 사전(vocabulary)의 2.5%가 OOV 버킷(bucket)으로 변환되었습니다. 이는 해당 문제의 어휘 사전과 모델의 어휘 사전이 완전히 겹치지 않을 때 도움이 됩니다.\n", - "* [google/tf2-preview/nnlm-en-dim50/1](https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1) - 더 큰 모델입니다. 차원 크기는 50이고 어휘 사전의 크기는 1백만 개 이하입니다.\n", - "* [google/tf2-preview/nnlm-en-dim128/1](https://tfhub.dev/google/tf2-preview/nnlm-en-dim128/1) - 훨씬 더 큰 모델입니다. 차원 크기는 128이고 어휘 사전의 크기는 1백만 개 이하입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "In2nDpTLkgKa" - }, - "source": [ - "먼저 문장을 임베딩시키기 위해 텐서플로 허브 모델을 사용하는 케라스 층을 만들어 보죠. 그다음 몇 개의 샘플을 입력하여 테스트해 보겠습니다. 입력 텍스트의 길이에 상관없이 임베딩의 출력 크기는 `(num_examples, embedding_dimension)`가 됩니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_NUbzVeYkgcO" - }, - "outputs": [], - "source": [ - "embedding = \"https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1\"\n", - "hub_layer = hub.KerasLayer(embedding, input_shape=[], \n", - " dtype=tf.string, trainable=True)\n", - "hub_layer(train_examples_batch[:3])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dfSbV6igl1EH" - }, - "source": [ - "이제 전체 모델을 만들어 보겠습니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xpKOoWgu-llD" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential()\n", - "model.add(hub_layer)\n", - "model.add(tf.keras.layers.Dense(16, activation='relu'))\n", - "model.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n", - "\n", - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "순서대로 층을 쌓아 분류기를 만듭니다:\n", - "\n", - "1. 첫 번째 층은 텐서플로 허브 층입니다. 이 층은 사전 훈련된 모델을 사용하여 하나의 문장을 임베딩 벡터로 매핑합니다. 여기서 사용하는 사전 훈련된 텍스트 임베딩 모델([google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1))은 하나의 문장을 토큰(token)으로 나누고 각 토큰의 임베딩을 연결하여 반환합니다. 최종 차원은 `(num_examples, embedding_dimension)`입니다.\n", - "2. 이 고정 크기의 출력 벡터는 16개의 은닉 유닛(hidden unit)을 가진 완전 연결 층(`Dense`)으로 주입됩니다.\n", - "3. 마지막 층은 하나의 출력 노드를 가진 완전 연결 층입니다. `sigmoid` 활성화 함수를 사용하므로 확률 또는 신뢰도 수준을 표현하는 0~1 사이의 실수가 출력됩니다.\n", - "\n", - "이제 모델을 컴파일합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### 손실 함수와 옵티마이저\n", - "\n", - "모델이 훈련하려면 손실 함수(loss function)과 옵티마이저(optimizer)가 필요합니다. 이 예제는 이진 분류 문제이고 모델이 확률을 출력하므로(출력층의 유닛이 하나이고 `sigmoid` 활성화 함수를 사용합니다), `binary_crossentropy` 손실 함수를 사용하겠습니다.\n", - "\n", - "다른 손실 함수를 선택할 수 없는 것은 아닙니다. 예를 들어 `mean_squared_error`를 선택할 수 있습니다. 하지만 일반적으로 `binary_crossentropy`가 확률을 다루는데 적합합니다. 이 함수는 확률 분포 간의 거리를 측정합니다. 여기에서는 정답인 타깃 분포와 예측 분포 사이의 거리입니다.\n", - "\n", - "나중에 회귀(regression) 문제(예를 들어 주택 가격을 예측하는 문제)에 대해 살펴 볼 때 평균 제곱 오차(mean squared error) 손실 함수를 어떻게 사용하는지 알아 보겠습니다.\n", - "\n", - "이제 모델이 사용할 옵티마이저와 손실 함수를 설정해 보죠:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Mr0GP-cQ-llN" - }, - "outputs": [], - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## 모델 훈련\n", - "\n", - "이 모델을 512개의 샘플로 이루어진 미니배치(mini-batch)에서 20번의 에포크(epoch) 동안 훈련합니다. `x_train`과 `y_train` 텐서에 있는 모든 샘플에 대해 20번 반복한다는 뜻입니다. 훈련하는 동안 10,000개의 검증 세트에서 모델의 손실과 정확도를 모니터링합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tXSGrjWZ-llW" - }, - "outputs": [], - "source": [ - "history = model.fit(train_data.shuffle(10000).batch(512),\n", - " epochs=20,\n", - " validation_data=validation_data.batch(512),\n", - " verbose=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## 모델 평가\n", - "\n", - "모델의 성능을 확인해 보죠. 두 개의 값이 반환됩니다. 손실(오차를 나타내는 숫자이므로 낮을수록 좋습니다)과 정확도입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zOMKywn4zReN" - }, - "outputs": [], - "source": [ - "results = model.evaluate(test_data.batch(512), verbose=2)\n", - "for name, value in zip(model.metrics_names, results):\n", - " print(\"%s: %.3f\" % (name, value))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "이 예제는 매우 단순한 방식을 사용하므로 87% 정도의 정확도를 달성했습니다. 고급 방법을 사용한 모델은 95%에 가까운 정확도를 얻습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## 더 읽을거리\n", - "\n", - "문자열 입력을 다루는 조금 더 일반적인 방법과 훈련 과정에서 정확도나 손실 변화에 대한 자세한 분석은 [여기](https://www.tensorflow.org/tutorials/keras/basic_text_classification)를 참고하세요." - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "text_classification_with_hub.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ko/tutorials/load_data/unicode.ipynb b/site/ko/tutorials/load_data/unicode.ipynb deleted file mode 100644 index 6b4030460d2..00000000000 --- a/site/ko/tutorials/load_data/unicode.ipynb +++ /dev/null @@ -1,888 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "unicode.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oL9KopJirB2g" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KHppBIUH0Fao", - "colab": {}, - "cellView": "form" - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ugoVwru20GCg", - "colab": {}, - "cellView": "form" - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AXH1bmUctMld" - }, - "source": [ - "# 유니코드 문자열\n", - "\n", - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    \n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yG0xJEHP3dvn" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 공식 영문 문서의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 tensorflow/docs 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 docs-ko@tensorflow.org로 메일을 보내주시기 바랍니다" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LrHJrKYis06U" - }, - "source": [ - "## 소개\n", - "\n", - "자연어 처리 모델은 종종 다른 문자 집합을 갖는 다양한 언어를 다루게 됩니다. *유니코드*(unicode)는 거의 모든 언어의 문자를 표현할 수 있는 표준 인코딩 시스템입니다. 각 문자는 `0`부터 `0x10FFFF` 사이의 고유한 정수 [코드 포인트](https://en.wikipedia.org/wiki/Code_point)(code point)를 사용해서 인코딩됩니다. *유니코드 문자열*은 0개 또는 그 이상의 코드 포인트로 이루어진 시퀀스(sequence)입니다.\n", - "\n", - "이 튜토리얼에서는 텐서플로(Tensorflow)에서 유니코드 문자열을 표현하고, 표준 문자열 연산의 유니코드 버전을 사용해서 유니코드 문자열을 조작하는 방법에 대해서 소개합니다. 또한 스크립트 감지(script detection)를 활용하여 유니코드 문자열을 토큰으로 분리해 보겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OIKHl5Lvn4gh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n-LkcI-vtWNj" - }, - "source": [ - "## `tf.string` 데이터 타입\n", - "\n", - "\n", - "텐서플로의 기본 `tf.string` `dtype`은 바이트 문자열로 이루어진 텐서를 만듭니다. 유니코드 문자열은 기본적으로 utf-8로 인코딩 됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3yo-Qv6ntaFr", - "colab": {} - }, - "source": [ - "tf.constant(u\"Thanks 😊\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2kA1ziG2tyCT" - }, - "source": [ - "`tf.string` 텐서는 바이트 문자열을 최소 단위로 다루기 때문에 다양한 길이의 바이트 문자열을 다룰 수 있습니다. 문자열 길이는 텐서 차원(dimensions)에 포함되지 않습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eyINCmTztyyS", - "colab": {} - }, - "source": [ - "tf.constant([u\"You're\", u\"welcome!\"]).shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "QCsQJtS2BO0j", - "colab_type": "text" - }, - "source": [ - "노트: 파이썬을 사용해 문자열을 만들 때 버전 2와 버전 3에서 유니코드를 다루는 방식이 다릅니다. 버전 2에서는 위와 같이 \"u\" 접두사를 사용하여 유니코드 문자열을 나타냅니다. 버전 3에서는 유니코드 인코딩된 문자열이 기본값입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hUFZ7B1Lk-uj" - }, - "source": [ - "## 유니코드 표현\n", - "\n", - "텐서플로에서 유니코드 문자열을 표현하기 위한 두 가지 방법이 있습니다:\n", - "\n", - "* `string` 스칼라 — 코드 포인트의 시퀀스가 알려진 [문자 인코딩](https://ko.wikipedia.org/wiki/%EB%AC%B8%EC%9E%90_%EC%9D%B8%EC%BD%94%EB%94%A9)을 사용해 인코딩됩니다.\n", - "* `int32` 벡터 — 위치마다 개별 코드 포인트를 포함합니다.\n", - "\n", - "예를 들어, 아래의 세 가지 값이 모두 유니코드 문자열 `\"语言处理\"`(중국어로 \"언어 처리\"를 의미함)를 표현합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cjQIkfJWvC_u", - "colab": {} - }, - "source": [ - "# UTF-8로 인코딩된 string 스칼라로 표현한 유니코드 문자열입니다.\n", - "text_utf8 = tf.constant(u\"语言处理\")\n", - "text_utf8" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yQqcUECcvF2r", - "colab": {} - }, - "source": [ - "# UTF-16-BE로 인코딩된 string 스칼라로 표현한 유니코드 문자열입니다.\n", - "text_utf16be = tf.constant(u\"语言处理\".encode(\"UTF-16-BE\"))\n", - "text_utf16be" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ExdBr1t7vMuS", - "colab": {} - }, - "source": [ - "# 유니코드 코드 포인트의 벡터로 표현한 유니코드 문자열입니다.\n", - "text_chars = tf.constant([ord(char) for char in u\"语言处理\"])\n", - "text_chars" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B8czv4JNpBnZ" - }, - "source": [ - "### 표현 간의 변환\n", - "\n", - "텐서플로는 다른 표현으로 변환하기 위한 연산을 제공합니다.\n", - "\n", - "* `tf.strings.unicode_decode`: 인코딩된 `string` 스칼라를 코드 포인트의 벡터로 변환합니다.\n", - "* `tf.strings.unicode_encode`: 코드 포인트의 벡터를 인코드된 `string` 스칼라로 변환합니다.\n", - "* `tf.strings.unicode_transcode`: 인코드된 `string` 스칼라를 다른 인코딩으로 변환합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qb-UQ_oLpAJg", - "colab": {} - }, - "source": [ - "tf.strings.unicode_decode(text_utf8,\n", - " input_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kEBUcunnp-9n", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(text_chars,\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0MLhWcLZrph-", - "colab": {} - }, - "source": [ - "tf.strings.unicode_transcode(text_utf8,\n", - " input_encoding='UTF8',\n", - " output_encoding='UTF-16-BE')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QVeLeVohqN7I" - }, - "source": [ - "### 배치(batch) 차원\n", - "\n", - "여러 개의 문자열을 디코딩 할 때 문자열마다 포함된 문자의 개수는 동일하지 않습니다. 반환되는 값은 [`tf.RaggedTensor`](../../guide/ragged_tensor.ipynb)로 가장 안쪽 차원의 크기가 문자열에 포함된 문자의 개수에 따라 결정됩니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N2jVzPymr_Mm", - "colab": {} - }, - "source": [ - "# UTF-8 인코딩된 문자열로 표현한 유니코드 문자열의 배치입니다. \n", - "batch_utf8 = [s.encode('UTF-8') for s in\n", - " [u'hÃllo', u'What is the weather tomorrow', u'Göödnight', u'😊']]\n", - "batch_chars_ragged = tf.strings.unicode_decode(batch_utf8,\n", - " input_encoding='UTF-8')\n", - "for sentence_chars in batch_chars_ragged.to_list():\n", - " print(sentence_chars)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iRh3n1hPsJ9v" - }, - "source": [ - "`tf.RaggedTensor`를 바로 사용하거나, 패딩(padding)을 사용해 `tf.Tensor`로 변환하거나, `tf.RaggedTensor.to_tensor` 와 `tf.RaggedTensor.to_sparse` 메서드를 사용해 `tf.SparseTensor`로 변환할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yz17yeSMsUid", - "colab": {} - }, - "source": [ - "batch_chars_padded = batch_chars_ragged.to_tensor(default_value=-1)\n", - "print(batch_chars_padded.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kBjsPQp3rhfm", - "colab": {} - }, - "source": [ - "batch_chars_sparse = batch_chars_ragged.to_sparse()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GCCkZh-nwlbL" - }, - "source": [ - "길이가 같은 여러 문자열을 인코딩할 때는 `tf.Tensor`를 입력으로 사용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_lP62YUAwjK9", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode([[99, 97, 116], [100, 111, 103], [ 99, 111, 119]],\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w58CMRg9tamW" - }, - "source": [ - "길이가 다른 여러 문자열을 인코딩할 때는 `tf.RaggedTensor`를 입력으로 사용해야 합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "d7GtOtrltaMl", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(batch_chars_ragged, output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T2Nh5Aj9xob3" - }, - "source": [ - "패딩된 텐서나 희소(sparse) 텐서는 `unicode_encode`를 호출하기 전에 `tf.RaggedTensor`로 바꿉니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R2bYCYl0u-Ue", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(\n", - " tf.RaggedTensor.from_sparse(batch_chars_sparse),\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UlV2znh_u_zm", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(\n", - " tf.RaggedTensor.from_tensor(batch_chars_padded, padding=-1),\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hQOOGkscvDpc" - }, - "source": [ - "## 유니코드 연산" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NkmtsA_yvMB0" - }, - "source": [ - "### 길이\n", - "\n", - "`tf.strings.length` 연산은 계산해야 할 길이를 나타내는 `unit` 인자를 가집니다. `unit`의 기본 단위는 `\"BYTE\"`이지만 인코딩된 `string`에 포함된 유니코드 코드 포인트의 수를 파악하기 위해 `\"UTF8_CHAR\"`나 `\"UTF16_CHAR\"`같이 다른 값을 설정할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1ZzMe59mvLHr", - "colab": {} - }, - "source": [ - "# UTF8에서 마지막 문자는 4바이트를 차지합니다.\n", - "thanks = u'Thanks 😊'.encode('UTF-8')\n", - "num_bytes = tf.strings.length(thanks).numpy()\n", - "num_chars = tf.strings.length(thanks, unit='UTF8_CHAR').numpy()\n", - "print('{} 바이트; {}개의 UTF-8 문자'.format(num_bytes, num_chars))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fHG85gxlvVU0" - }, - "source": [ - "### 부분 문자열\n", - "\n", - "이와 유사하게 `tf.strings.substr` 연산은 \"`unit`\" 매개변수 값을 사용해 \"`pos`\"와 \"`len`\" 매개변수로 지정된 문자열의 종류를 결정합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WlWRLV-4xWYq", - "colab": {} - }, - "source": [ - "# 기본: unit='BYTE'. len=1이면 바이트 하나를 반환합니다.\n", - "tf.strings.substr(thanks, pos=7, len=1).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JfNUVDPwxkCS", - "colab": {} - }, - "source": [ - "# unit='UTF8_CHAR'로 지정하면 4 바이트인 문자 하나를 반환합니다.\n", - "print(tf.strings.substr(thanks, pos=7, len=1, unit='UTF8_CHAR').numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zJUEsVSyeIa3" - }, - "source": [ - "### 유니코드 문자열 분리\n", - "\n", - "`tf.strings.unicode_split` 연산은 유니코드 문자열의 개별 문자를 부분 문자열로 분리합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dDjkh5G1ejMt", - "colab": {} - }, - "source": [ - "tf.strings.unicode_split(thanks, 'UTF-8').numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HQqEEZEbdG9O" - }, - "source": [ - "### 문자 바이트 오프셋\n", - "\n", - "`tf.strings.unicode_decode`로 만든 문자 텐서를 원본 문자열과 위치를 맞추려면 각 문자의 시작 위치의 오프셋(offset)을 알아야 합니다. `tf.strings.unicode_decode_with_offsets`은 `unicode_decode`와 비슷하지만 각 문자의 시작 오프셋을 포함한 두 번째 텐서를 반환합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Cug7cmwYdowd", - "colab": {} - }, - "source": [ - "codepoints, offsets = tf.strings.unicode_decode_with_offsets(u\"🎈🎉🎊\", 'UTF-8')\n", - "\n", - "for (codepoint, offset) in zip(codepoints.numpy(), offsets.numpy()):\n", - " print(\"바이트 오프셋 {}: 코드 포인트 {}\".format(offset, codepoint))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2ZnCNxOvx66T" - }, - "source": [ - "## 유니코드 스크립트" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRRHqkqNyGZ6" - }, - "source": [ - "각 유니코드 코드 포인트는 [스크립트](https://en.wikipedia.org/wiki/Script_%28Unicode%29)(script)라 부르는 하나의 코드 포인트의 집합(collection)에 속합니다. 문자의 스크립트는 문자가 어떤 언어인지 결정하는 데 도움이 됩니다. 예를 들어, 'Б'가 키릴(Cyrillic) 스크립트라는 것을 알고 있으면 이 문자가 포함된 텍스트는 아마도 (러시아어나 우크라이나어 같은) 슬라브 언어라는 것을 알 수 있습니다.\n", - "\n", - "텐서플로는 주어진 코드 포인트가 어떤 스크립트를 사용하는지 판별하기 위해 `tf.strings.unicode_script` 연산을 제공합니다. 스크립트 코드는 [International Components for\n", - "Unicode](http://site.icu-project.org/home) (ICU) [`UScriptCode`](http://icu-project.org/apiref/icu4c/uscript_8h.html) 값과 일치하는 `int32` 값입니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K7DeYHrRyFPy", - "colab": {} - }, - "source": [ - "uscript = tf.strings.unicode_script([33464, 1041]) # ['芸', 'Б']\n", - "\n", - "print(uscript.numpy()) # [17, 8] == [USCRIPT_HAN, USCRIPT_CYRILLIC]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2fW992a1lIY6" - }, - "source": [ - "`tf.strings.unicode_script` 연산은 코드 포인트의 다차원 `tf.Tensor`나 `tf.RaggedTensor`에 적용할 수 있습니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uR7b8meLlFnp", - "colab": {} - }, - "source": [ - "print(tf.strings.unicode_script(batch_chars_ragged))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mx7HEFpBzEsB" - }, - "source": [ - "## 예제: 간단한 분할\n", - "\n", - "분할(segmentation)은 텍스트를 단어와 같은 단위로 나누는 작업입니다. 공백 문자가 단어를 나누는 구분자로 사용되는 경우는 쉽지만, (중국어나 일본어 같이) 공백을 사용하지 않는 언어나 (독일어 같이) 단어를 길게 조합하는 언어는 의미를 분석하기 위한 분할 과정이 꼭 필요합니다. 웹 텍스트에는 \"NY株価\"(New York Stock Exchange)와 같이 여러 가지 언어와 스크립트가 섞여 있는 경우가 많습니다.\n", - "\n", - "스크립트의 변화를 단어 경계로 근사하여 (ML 모델 사용 없이) 대략적인 분할을 수행할 수 있습니다. 위에서 언급된 \"NY株価\"의 예와 같은 문자열에 적용됩니다. 다양한 스크립트의 공백 문자를 모두 USCRIPT_COMMON(실제 텍스트의 스크립트 코드와 다른 특별한 스크립트 코드)으로 분류하기 때문에 공백을 사용하는 대부분의 언어들에서도 역시 적용됩니다. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "grsvFiC4BoPb", - "colab": {} - }, - "source": [ - "# dtype: string; shape: [num_sentences]\n", - "#\n", - "# 처리할 문장들 입니다. 이 라인을 수정해서 다른 입력값을 시도해 보세요!\n", - "sentence_texts = [u'Hello, world.', u'世界こんにちは']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CapnbShuGU8i" - }, - "source": [ - "먼저 문장을 문자 코드 포인트로 디코딩하고 각 문자에 대한 스크립트 식별자를 찾습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReQVcDQh1MB8", - "colab": {} - }, - "source": [ - "# dtype: int32; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_codepoint[i, j]는\n", - "# i번째 문장 안에 있는 j번째 문자에 대한 코드 포인트 입니다.\n", - "sentence_char_codepoint = tf.strings.unicode_decode(sentence_texts, 'UTF-8')\n", - "print(sentence_char_codepoint)\n", - "\n", - "# dtype: int32; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_codepoint[i, j]는 \n", - "# i번째 문장 안에 있는 j번째 문자의 유니코드 스크립트 입니다.\n", - "sentence_char_script = tf.strings.unicode_script(sentence_char_codepoint)\n", - "print(sentence_char_script)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O2fapF5UGcUc" - }, - "source": [ - "그다음 스크립트 식별자를 사용하여 단어 경계가 추가될 위치를 결정합니다. 각 문장의 시작과 이전 문자와 스크립트가 다른 문자에 단어 경계를 추가합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7v5W6MOr1Rlc", - "colab": {} - }, - "source": [ - "# dtype: bool; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_starts_word[i, j]는 \n", - "# i번째 문장 안에 있는 j번째 문자가 단어의 시작이면 True 입니다.\n", - "sentence_char_starts_word = tf.concat(\n", - " [tf.fill([sentence_char_script.nrows(), 1], True),\n", - " tf.not_equal(sentence_char_script[:, 1:], sentence_char_script[:, :-1])],\n", - " axis=1)\n", - "\n", - "# dtype: int64; shape: [num_words]\n", - "#\n", - "# word_starts[i]은 (모든 문장의 문자를 일렬로 펼친 리스트에서)\n", - "# i번째 단어가 시작되는 문자의 인덱스 입니다.\n", - "word_starts = tf.squeeze(tf.where(sentence_char_starts_word.values), axis=1)\n", - "print(word_starts)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LAwh-1QkGuC9" - }, - "source": [ - "이 시작 오프셋을 사용하여 전체 배치에 있는 단어 리스트를 담은 `RaggedTensor`를 만듭니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bNiA1O_eBBCL", - "colab": {} - }, - "source": [ - "# dtype: int32; shape: [num_words, (num_chars_per_word)]\n", - "#\n", - "# word_char_codepoint[i, j]은 \n", - "# i번째 단어 안에 있는 j번째 문자에 대한 코드 포인트 입니다.\n", - "word_char_codepoint = tf.RaggedTensor.from_row_starts(\n", - " values=sentence_char_codepoint.values,\n", - " row_starts=word_starts)\n", - "print(word_char_codepoint)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "66a2ZnYmG2ao" - }, - "source": [ - "마지막으로 단어 코드 포인트 `RaggedTensor`를 문장으로 다시 나눕니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NCfwcqLSEjZb", - "colab": {} - }, - "source": [ - "# dtype: int64; shape: [num_sentences]\n", - "#\n", - "# sentence_num_words[i]는 i번째 문장 안에 있는 단어의 수입니다.\n", - "sentence_num_words = tf.reduce_sum(\n", - " tf.cast(sentence_char_starts_word, tf.int64),\n", - " axis=1)\n", - "\n", - "# dtype: int32; shape: [num_sentences, (num_words_per_sentence), (num_chars_per_word)]\n", - "#\n", - "# sentence_word_char_codepoint[i, j, k]는 i번째 문장 안에 있는\n", - "# j번째 단어 안의 k번째 문자에 대한 코드 포인트입니다.\n", - "sentence_word_char_codepoint = tf.RaggedTensor.from_row_lengths(\n", - " values=word_char_codepoint,\n", - " row_lengths=sentence_num_words)\n", - "print(sentence_word_char_codepoint)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xWaX8WcbHyqY" - }, - "source": [ - "최종 결과를 읽기 쉽게 utf-8 문자열로 다시 인코딩합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HSivquOgFr3C", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(sentence_word_char_codepoint, 'UTF-8').to_list()" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/ko/tutorials/quickstart/advanced.ipynb b/site/ko/tutorials/quickstart/advanced.ipynb deleted file mode 100644 index 61de7854be6..00000000000 --- a/site/ko/tutorials/quickstart/advanced.ipynb +++ /dev/null @@ -1,393 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "advanced.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\");" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# 텐서플로 2.0 시작하기: 전문가용" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행 하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7C-jrFz-PkeF", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "이 문서는 [구글 코랩](https://colab.research.google.com/notebooks/welcome.ipynb)(Colaboratory) 노트북 파일입니다. 파이썬 프로그램을 브라우저에서 직접 실행할 수 있기 때문에 텐서플로를 배우고 사용하기 좋은 도구입니다:\n", - "\n", - "1. 파이썬 런타임(runtime)에 연결하세요: 메뉴 막대의 오른쪽 상단에서 *CONNECT*를 선택하세요.\n", - "2. 노트북의 모든 코드 셀(cell)을 실행하세요: *Runtime* > *Run all*을 선택하세요.\n", - "\n", - "더 많은 예제와 자세한 안내는 [텐서플로 튜토리얼](https://www.tensorflow.org/tutorials/)을 참고하세요.\n", - "\n", - "먼저 프로그램에 텐서플로 라이브러리를 임포트합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "[MNIST 데이터셋](http://yann.lecun.com/exdb/mnist/)을 로드하여 준비합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JqFRS6K07jJs", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# 채널 차원을 추가합니다.\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "tf.data를 사용하여 데이터셋을 섞고 배치를 만듭니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Iu_quO024c2", - "colab": {} - }, - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "케라스(Keras)의 [모델 서브클래싱(subclassing) API](https://www.tensorflow.org/guide/keras#model_subclassing)를 사용하여 `tf.keras` 모델을 만듭니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "model = MyModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "훈련에 필요한 옵티마이저(optimizer)와 손실 함수를 선택합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u48C9WQ774n4", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "모델의 손실과 성능을 측정할 지표를 선택합니다. 에포크가 진행되는 동안 수집된 측정 지표를 바탕으로 최종 결과를 출력합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N0MqHFb4F_qn", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "`tf.GradientTape`를 사용하여 모델을 훈련합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OZACiVqA8KQV", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z8YT7UmFgpjV" - }, - "source": [ - "이제 모델을 테스트합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIKdEzHAJGt7", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "i-2pkctU_Ci7", - "colab": {} - }, - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = '에포크: {}, 손실: {}, 정확도: {}, 테스트 손실: {}, 테스트 정확도: {}'\n", - " print (template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "훈련된 이미지 분류기는 이 데이터셋에서 약 98%의 정확도를 달성합니다. 더 자세한 내용은 [TensorFlow 튜토리얼](https://www.tensorflow.org/tutorials/)을 참고하세요." - ] - } - ] -} diff --git a/site/ko/tutorials/quickstart/beginner.ipynb b/site/ko/tutorials/quickstart/beginner.ipynb deleted file mode 100644 index 3d12eb57041..00000000000 --- a/site/ko/tutorials/quickstart/beginner.ipynb +++ /dev/null @@ -1,241 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "beginner.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\");" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# 텐서플로 2.0 시작하기: 초보자용" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " TensorFlow.org에서 보기\n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_dt4BrWOPl-C", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "이 문서는 [구글 코랩](https://colab.research.google.com/notebooks/welcome.ipynb)(Colaboratory) 노트북 파일입니다. 파이썬 프로그램을 브라우저에서 직접 실행할 수 있기 때문에 텐서플로를 배우고 사용하기 좋은 도구입니다:\n", - "\n", - "1. 파이썬 런타임(runtime)에 연결하세요: 메뉴 막대의 오른쪽 상단에서 *CONNECT*를 선택하세요.\n", - "2. 노트북의 모든 코드 셀(cell)을 실행하세요: *Runtime* > *Run all*을 선택하세요.\n", - "\n", - "더 많은 예제와 자세한 안내는 [텐서플로 튜토리얼](https://www.tensorflow.org/tutorials/)을 참고하세요.\n", - "\n", - "먼저 프로그램에 텐서플로 라이브러리를 임포트합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "[MNIST 데이터셋](http://yann.lecun.com/exdb/mnist/)을 로드하여 준비합니다. 샘플 값을 정수에서 부동소수로 변환합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FP5258xjs-v", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "층을 차례대로 쌓아 `tf.keras.Sequential` 모델을 만듭니다. 훈련에 사용할 옵티마이저(optimizer)와 손실 함수를 선택합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "model = tf.keras.models.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "모델을 훈련하고 평가합니다:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F7dTAzgHDUh7", - "colab": {} - }, - "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", - "model.evaluate(x_test, y_test, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "훈련된 이미지 분류기는 이 데이터셋에서 약 98%의 정확도를 달성합니다. 더 자세한 내용은 [TensorFlow 튜토리얼](https://www.tensorflow.org/tutorials/)을 참고하세요." - ] - } - ] -} diff --git a/site/ko/tutorials/structured_data/feature_columns.ipynb b/site/ko/tutorials/structured_data/feature_columns.ipynb deleted file mode 100644 index 57cd8aaef6d..00000000000 --- a/site/ko/tutorials/structured_data/feature_columns.ipynb +++ /dev/null @@ -1,750 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "feature_columns.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rNdWfPXCjTjY" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "I1dUQ0GejU8N", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c05P9g5WjizZ" - }, - "source": [ - "# 정형 데이터 다루기" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zofH_gCzgplN" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " TensorFlow.org에서 보기\n", - " \n", - " \n", - " \n", - " 구글 코랩(Colab)에서 실행하기\n", - " \n", - " \n", - " \n", - " 깃허브(GitHub) 소스 보기\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ldb54WMWcNoM", - "colab_type": "text" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도\n", - "불구하고 [공식 영문 문서](https://www.tensorflow.org/?hl=en)의 내용과 일치하지 않을 수 있습니다.\n", - "이 번역에 개선할 부분이 있다면\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다.\n", - "문서 번역이나 리뷰에 참여하려면\n", - "[docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로\n", - "메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K1y4OHpGgss7" - }, - "source": [ - "이 튜토리얼은 정형 데이터(structured data)를 다루는 방법을 소개합니다(예를 들어 CSV에서 읽은 표 형식의 데이터). [케라스](https://www.tensorflow.org/guide/keras)를 사용하여 모델을 정의하고 [특성 열](https://www.tensorflow.org/guide/feature_columns)(feature column)을 사용하여 CSV의 열을 모델 훈련에 필요한 특성으로 매핑하겠습니다. 이 튜토리얼은 다음 내용을 포함합니다:\n", - "\n", - "* [판다스](https://pandas.pydata.org/)(Pandas)를 사용하여 CSV 파일을 읽기\n", - "* [tf.data](https://www.tensorflow.org/guide/datasets)를 사용하여 행을 섞고 배치로 나누는 입력 파이프라인(pipeline)을 만들기\n", - "* CSV의 열을 feature_column을 사용해 모델 훈련에 필요한 특성으로 매핑하기\n", - "* 케라스를 사용하여 모델 구축, 훈련, 평가하기\n", - "\n", - "## 데이터셋\n", - "\n", - "클리블랜드(Cleveland) 심장병 재단에서 제공한 작은 [데이터셋](https://archive.ics.uci.edu/ml/datasets/heart+Disease)을 사용하겠습니다. 이 CSV 파일은 수백 개의 행으로 이루어져 있습니다. 각 행은 환자 한 명을 나타내고 각 열은 환자에 대한 속성 값입니다. 이 정보를 사용해 환자의 심장병 발병 여부를 예측해 보겠습니다. 즉 이 데이터셋은 이진 분류 문제입니다.\n", - "\n", - "다음은 이 데이터셋에 대한 [설명](https://archive.ics.uci.edu/ml/machine-learning-databases/heart-disease/heart-disease.names)입니다. 수치형과 범주형 열이 모두 있다는 점을 주목하세요.\n", - "\n", - ">열| 설명| 특성 타입 | 데이터 타입\n", - ">------------|--------------------|----------------------|-----------------\n", - ">Age | 나이 | 수치형 | 정수\n", - ">Sex | (1 = 남성; 0 = 여성) | 범주형 | 정수\n", - ">CP | 가슴 통증 유형 (0, 1, 2, 3, 4) | 범주형 | 정수\n", - ">Trestbpd | 안정 혈압 (병원 입원시 mm Hg) | 수치형 | 정수\n", - ">Chol | 혈청 콜레스테롤 (mg/dl) | 수치형 | 정수\n", - ">FBS | (공복 혈당 > 120 mg/dl) (1 = true; 0 = false) | 범주형 | 정수\n", - ">RestECG | 안정 심전도 결과 (0, 1, 2) | 범주형 | 정수\n", - ">Thalach | 최대 심박동수 | 수치형 | 정수\n", - ">Exang | 협심증 유발 운동 (1 = yes; 0 = no) | 범주형 | 정수\n", - ">Oldpeak | 비교적 안정되기까지 운동으로 유발되는 ST depression | 수치형 | 정수\n", - ">Slope | 최대 운동 ST segment의 기울기 | 수치형 | 실수\n", - ">CA | 형광 투시된 주요 혈관의 수 (0-3) | 수치형 | 정수\n", - ">Thal | 3 = 보통; 6 = 해결된 결함; 7 = 해결가능한 결함 | 범주형 | 문자열\n", - ">Target | 심장병 진단 (1 = true; 0 = false) | 분류 | 정수" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VxyBFc_kKazA" - }, - "source": [ - "## 텐서플로와 필요한 라이브러리 임포트하기" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LuOWVJBz8a6G", - "colab": {} - }, - "source": [ - "!pip install sklearn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9dEreb4QKizj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " !pip install tf-nightly-2.0-preview\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import feature_column\n", - "from tensorflow.keras import layers\n", - "from sklearn.model_selection import train_test_split" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KCEhSZcULZ9n" - }, - "source": [ - "## 판다스로 데이터프레임 만들기\n", - "\n", - "[판다스](https://pandas.pydata.org/)는 정형 데이터를 읽고 조작하는데 유용한 유틸리티 함수를 많이 제공하는 파이썬 라이브러리입니다. 판다스를 이용해 URL로부터 데이터를 다운로드하여 읽은 다음 데이터프레임으로 변환하겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "REZ57BXCLdfG", - "colab": {} - }, - "source": [ - "URL = 'https://storage.googleapis.com/applied-dl/heart.csv'\n", - "dataframe = pd.read_csv(URL)\n", - "dataframe.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u0zhLtQqMPem" - }, - "source": [ - "## 데이터프레임을 훈련 세트, 검증 세트, 테스트 세트로 나누기\n", - "\n", - "하나의 CSV 파일에서 데이터셋을 다운로드했습니다. 이를 훈련 세트, 검증 세트, 테스트 세트로 나누겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YEOpw7LhMYsI", - "colab": {} - }, - "source": [ - "train, test = train_test_split(dataframe, test_size=0.2)\n", - "train, val = train_test_split(train, test_size=0.2)\n", - "print(len(train), '훈련 샘플')\n", - "print(len(val), '검증 샘플')\n", - "print(len(test), '테스트 샘플')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "84ef46LXMfvu" - }, - "source": [ - "## tf.data를 사용하여 입력 파이프라인 만들기\n", - "\n", - "그다음 [tf.data](https://www.tensorflow.org/guide/datasets)를 사용하여 데이터프레임을 감싸겠습니다. 이렇게 하면 특성 열을 사용하여 판다스 데이터프레임의 열을 모델 훈련에 필요한 특성으로 매핑할 수 있습니다. 아주 큰 CSV 파일(메모리에 들어갈 수 없을 정도로 큰 파일)을 다룬다면 tf.data로 디스크 디렉토리에서 데이터를 읽을 수 있습니다. 이런 내용은 이 튜토리얼에 포함되어 있지 않습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NkcaMYP-MsRe", - "colab": {} - }, - "source": [ - "# 판다스 데이터프레임으로부터 tf.data 데이터셋을 만들기 위한 함수\n", - "def df_to_dataset(dataframe, shuffle=True, batch_size=32):\n", - " dataframe = dataframe.copy()\n", - " labels = dataframe.pop('target')\n", - " ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n", - " if shuffle:\n", - " ds = ds.shuffle(buffer_size=len(dataframe))\n", - " ds = ds.batch(batch_size)\n", - " return ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CXbbXkJvMy34", - "colab": {} - }, - "source": [ - "batch_size = 5 # 예제를 위해 작은 배치 크기를 사용합니다.\n", - "train_ds = df_to_dataset(train, batch_size=batch_size)\n", - "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", - "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qRLGSMDzM-dl" - }, - "source": [ - "## 입력 파이프라인 이해하기\n", - "\n", - "앞서 만든 입력 파이프라인을 호출하여 반환되는 데이터 포맷을 확인해 보겠습니다. 간단하게 출력하기 위해 작은 배치 크기를 사용합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CSBo3dUVNFc9", - "colab": {} - }, - "source": [ - "for feature_batch, label_batch in train_ds.take(1):\n", - " print('전체 특성:', list(feature_batch.keys()))\n", - " print('나이 특성의 배치:', feature_batch['age'])\n", - " print('타깃의 배치:', label_batch )" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OT5N6Se-NQsC" - }, - "source": [ - "이 데이터셋은 (데이터프레임의) 열 이름을 키로 갖는 딕셔너리를 반환합니다. 데이터프레임 열의 값이 매핑되어 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ttIvgLRaNoOQ" - }, - "source": [ - "## 여러 종류의 특성 열 알아 보기\n", - "\n", - "텐서플로는 여러 종류의 특성 열을 제공합니다. 이 절에서 몇 가지 특성 열을 만들어서 데이터프레임의 열을 변환하는 방법을 알아 보겠습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mxwiHFHuNhmf", - "colab": {} - }, - "source": [ - "# 특성 열을 시험해 보기 위해 샘플 배치를 만듭니다.\n", - "example_batch = next(iter(train_ds))[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0wfLB8Q3N3UH", - "colab": {} - }, - "source": [ - "# 특성 열을 만들고 배치 데이터를 변환하는 함수\n", - "def demo(feature_column):\n", - " feature_layer = layers.DenseFeatures(feature_column)\n", - " print(feature_layer(example_batch).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q7OEKe82N-Qb" - }, - "source": [ - "### 수치형 열\n", - "\n", - "특성 열의 출력은 모델의 입력이 됩니다(앞서 정의한 함수를 사용하여 데이터프레임의 각 열이 어떻게 변환되는지 알아 볼 것입니다). [수치형 열](https://www.tensorflow.org/api_docs/python/tf/feature_column/numeric_column)은 가장 간단한 종류의 열입니다. 이 열은 실수 특성을 표현하는데 사용됩니다. 이 열을 사용하면 모델은 데이터프레임 열의 값을 변형시키지 않고 그대로 전달 받습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QZTZ0HnHOCxC", - "colab": {} - }, - "source": [ - "age = feature_column.numeric_column(\"age\")\n", - "demo(age)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7a6ddSyzOKpq" - }, - "source": [ - "심장병 데이터셋 데이터프레임의 대부분 열은 수치형입니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IcSxUoYgOlA1" - }, - "source": [ - "### 버킷형 열\n", - "\n", - "종종 모델에 수치 값을 바로 주입하기 원치 않을 때가 있습니다. 대신 수치 값의 구간을 나누어 이를 기반으로 범주형으로 변환합니다. 원본 데이터가 사람의 나이를 표현한다고 가정해 보죠. 나이를 수치형 열로 표현하는 대신 [버킷형 열](https://www.tensorflow.org/api_docs/python/tf/feature_column/bucketized_column)(bucketized column)을 사용하여 나이를 몇 개의 버킷(bucket)으로 분할할 수 있습니다. 다음에 원-핫 인코딩(one-hot encoding)된 값은 각 열이 매칭되는 나이 범위를 나타냅니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wJ4Wt3SAOpTQ", - "colab": {} - }, - "source": [ - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "demo(age_buckets)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r1tArzewPb-b" - }, - "source": [ - "### 범주형 열\n", - "\n", - "이 데이터셋에서 thal 열은 문자열입니다(예를 들어 'fixed', 'normal', 'reversible'). 모델에 문자열을 바로 주입할 수 없습니다. 대신 문자열을 먼저 수치형으로 매핑해야 합니다. 범주형 열(categorical column)을 사용하여 문자열을 원-핫 벡터로 표현할 수 있습니다. 문자열 목록은 [categorical_column_with_vocabulary_list](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_list)를 사용하여 리스트로 전달하거나 [categorical_column_with_vocabulary_file](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_file)을 사용하여 파일에서 읽을 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DJ6QnSHkPtOC", - "colab": {} - }, - "source": [ - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "demo(thal_one_hot)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dxQloQ9jOoXL" - }, - "source": [ - "더 복잡한 데이터셋에는 범주형(예를 들면 문자열)인 열이 많을 수 있습니다. 특성 열은 범주형 데이터를 다룰 때 진가가 발휘됩니다. 이 데이터셋에는 범주형 열이 하나 뿐이지만 다른 데이터셋에서 사용할 수 있는 여러 종류의 특성 열을 소개하겠습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LEFPjUr6QmwS" - }, - "source": [ - "### 임베딩 열\n", - "\n", - "가능한 문자열이 몇 개가 있는 것이 아니라 범주마다 수천 개 이상의 값이 있는 경우를 상상해 보겠습니다. 여러 가지 이유로 범주의 개수가 늘어남에 따라 원-핫 인코딩을 사용하여 신경망을 훈련시키는 것이 불가능해집니다. 임베딩 열(embedding column)을 사용하면 이런 제한을 극복할 수 있습니다. 고차원 원-핫 벡터로 데이터를 표현하는 대신 [임베딩 열](https://www.tensorflow.org/api_docs/python/tf/feature_column/embedding_column)을 사용하여 저차원으로 데이터를 표현합니다. 이 벡터는 0 또는 1이 아니라 각 원소에 어떤 숫자도 넣을 수 있는 밀집 벡터(dense vector)입니다. 임베딩의 크기(아래 예제에서는 8입니다)는 튜닝 대상 파라미터입니다.\n", - "\n", - "핵심 포인트: 범주형 열에 가능한 값이 많을 때는 임베딩 열을 사용하는 것이 최선입니다. 여기에서는 예시를 목적으로 하나를 사용하지만 완전한 예제이므로 나중에 다른 데이터셋에 수정하여 적용할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hSlohmr2Q_UU", - "colab": {} - }, - "source": [ - "# 임베딩 열의 입력은 앞서 만든 범주형 열입니다.\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "demo(thal_embedding)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "urFCAvTVRMpB" - }, - "source": [ - "### 해시 특성 열\n", - "\n", - "가능한 값이 많은 범주형 열을 표현하는 또 다른 방법은 [categorical_column_with_hash_bucket](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_hash_bucket)을 사용하는 것입니다. 이 특성 열은 입력의 해시(hash) 값을 계산한 다음 `hash_bucket_size` 크기의 버킷 중 하나를 선택하여 문자열을 인코딩합니다. 이 열을 사용할 때는 어휘 목록을 제공할 필요가 없고 공간을 절약하기 위해 실제 범주의 개수보다 훨씬 작게 해시 버킷(bucket)의 크기를 정할 수 있습니다.\n", - "\n", - "핵심 포인트: 이 기법의 큰 단점은 다른 문자열이 같은 버킷에 매핑될 수 있다는 것입니다. 그럼에도 실전에서는 일부 데이터셋에서 잘 작동합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YHU_Aj2nRRDC", - "colab": {} - }, - "source": [ - "thal_hashed = feature_column.categorical_column_with_hash_bucket(\n", - " 'thal', hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(thal_hashed))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fB94M27DRXtZ" - }, - "source": [ - "### 교차 특성 열\n", - "\n", - "여러 특성을 연결하여 하나의 특성으로 만드는 것을 [교차 특성](https://developers.google.com/machine-learning/glossary/#feature_cross)(feature cross)이라고 합니다. 모델이 특성의 조합에 대한 가중치를 학습할 수 있습니다. 이 예제에서는 age와 thal의 교차 특성을 만들어 보겠습니다. `crossed_column`은 모든 가능한 조합에 대한 해시 테이블을 만들지 않고 `hashed_column` 매개변수를 사용하여 해시 테이블의 크기를 선택합니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oaPVERd9Rep6", - "colab": {} - }, - "source": [ - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(crossed_feature))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ypkI9zx6Rj1q" - }, - "source": [ - "## 사용할 열 선택하기\n", - "\n", - "여러 가지 특성 열을 사용하는 방법을 보았으므로 이제 이를 사용하여 모델을 훈련하겠습니다. 이 튜토리얼의 목적은 특성 열을 사용하는 완전한 코드(예를 들면 작동 방식)를 제시하는 것이므로 임의로 몇 개의 열을 선택하여 모델을 훈련하겠습니다.\n", - "\n", - "핵심 포인트: 제대로 된 모델을 만들어야 한다면 대용량의 데이터셋을 사용하고 어떤 특성을 포함하는 것이 가장 의미있는지, 또 어떻게 표현해야 할지 신중하게 생각하세요." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4PlLY7fORuzA", - "colab": {} - }, - "source": [ - "feature_columns = []\n", - "\n", - "# 수치형 열\n", - "for header in ['age', 'trestbps', 'chol', 'thalach', 'oldpeak', 'slope', 'ca']:\n", - " feature_columns.append(feature_column.numeric_column(header))\n", - "\n", - "# 버킷형 열\n", - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "feature_columns.append(age_buckets)\n", - "\n", - "# 범주형 열\n", - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "feature_columns.append(thal_one_hot)\n", - "\n", - "# 임베딩 열\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "feature_columns.append(thal_embedding)\n", - "\n", - "# 교차 특성 열\n", - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "crossed_feature = feature_column.indicator_column(crossed_feature)\n", - "feature_columns.append(crossed_feature)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M-nDp8krS_ts" - }, - "source": [ - "### 특성 층 만들기\n", - "\n", - "특성 열을 정의하고 나면 [DenseFeatures](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/DenseFeatures) 층을 사용해 케라스 모델에 주입할 수 있습니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6o-El1R2TGQP", - "colab": {} - }, - "source": [ - "feature_layer = tf.keras.layers.DenseFeatures(feature_columns)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8cf6vKfgTH0U" - }, - "source": [ - "앞서 특성 열의 작동 예를 보이기 위해 작은 배치 크기를 사용했습니다. 여기에서는 조금 더 큰 배치 크기로 입력 파이프라인을 만듭니다." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gcemszoGSse_", - "colab": {} - }, - "source": [ - "batch_size = 32\n", - "train_ds = df_to_dataset(train, batch_size=batch_size)\n", - "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", - "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBx4Xu0eTXWq" - }, - "source": [ - "## 모델 생성, 컴파일, 훈련" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_YJPPb3xTPeZ", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " feature_layer,\n", - " layers.Dense(128, activation='relu'),\n", - " layers.Dense(128, activation='relu'),\n", - " layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "model.fit(train_ds,\n", - " validation_data=val_ds,\n", - " epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GnFmMOW0Tcaa", - "colab": {} - }, - "source": [ - "loss, accuracy = model.evaluate(test_ds)\n", - "print(\"정확도\", accuracy)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3bdfbq20V6zu" - }, - "source": [ - "핵심 포인트: 일반적으로 크고 복잡한 데이터셋일 경우 딥러닝 모델에서 최선의 결과를 얻습니다. 이런 작은 데이터셋에서는 기본 모델로 결정 트리(decision tree)나 랜덤 포레스트(random forest)를 사용하는 것이 권장됩니다. 이 튜토리얼의 목적은 정확한 모델을 훈련하는 것이 아니라 정형 데이터를 다루는 방식을 설명하는 것입니다. 실전 데이터셋을 다룰 때 이 코드를 시작점으로 사용하세요." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SotnhVWuHQCw" - }, - "source": [ - "## 그 다음엔\n", - "\n", - "정형 데이터를 사용한 분류 작업에 대해 배우는 가장 좋은 방법은 직접 실습하는 것입니다. 실험해 볼 다른 데이터셋을 찾아서 위와 비슷한 코드를 사용해 모델을 훈련해 보세요. 정확도를 향상시키려면 모델에 포함할 특성과 표현 방법을 신중하게 생각하세요." - ] - } - ] -} diff --git a/site/ko/tutorials/text/text_generation.ipynb b/site/ko/tutorials/text/text_generation.ipynb deleted file mode 100644 index d1a7f1a6b95..00000000000 --- a/site/ko/tutorials/text/text_generation.ipynb +++ /dev/null @@ -1,1250 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "GCCk8_dHpuNf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hcD2nPQvPOFM" - }, - "source": [ - "# 순환 신경망을 활용한 문자열 생성\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/text_generation.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " TensorFlow.org에서 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ko/tutorials/text/text_generation.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " 구글 코랩(Colab)에서 실행하기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ko/tutorials/text/text_generation.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " 깃허브(GitHub) 소스 보기\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tfiUl2tcqKnP" - }, - "source": [ - "Note: 이 문서는 텐서플로 커뮤니티에서 번역했습니다. 커뮤니티 번역 활동의 특성상 정확한 번역과 최신 내용을 반영하기 위해 노력함에도 불구하고 [공식 영문 문서](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/text_generation.ipynb)의 내용과 일치하지 않을 수 있습니다. 이 번역에 개선할 부분이 있다면 [tensorflow/docs](https://github.com/tensorflow/docs) 깃헙 저장소로 풀 리퀘스트를 보내주시기 바랍니다. 문서 번역이나 리뷰에 참여하려면 [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko)로 메일을 보내주시기 바랍니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BwpJ5IffzRG6" - }, - "source": [ - "이 튜토리얼에서는 문자 기반 순환 신경망(RNN, Recurrent Neural Network)을 사용하여 어떻게 텍스트를 생성하는지 설명합니다. Andrej Karpathy의 [순환 신경망의 뛰어난 효율](http://karpathy.github.io/2015/05/21/rnn-effectiveness/)에서 가져온 셰익스피어 데이터셋으로 작업할 것입니다. 이 데이터 셋에서 문자 시퀀스 (\"Shakespear\")가 주어지면, 시퀀스의 다음 문자(\"e\")를 예측하는 모델을 훈련합니다. 모델을 반복하여 호출하면 더 긴 텍스트 시퀀스 생성이 가능합니다.\n", - "\n", - "Note: 이 노트북을 더 빠르게 실행하기 위해 GPU 가속을 활성화합니다. 코랩(Colab)에서: Runtime \u003e Change runtime type \u003e Hardware acclerator \u003e GPU* 탭을 선택합니다. 로컬에서 실행하려면 TensorFlow 버전이 1.11 이상이어야 합니다.\n", - "\n", - "이 튜토리얼은 [tf.keras](https://www.tensorflow.org/programmers_guide/keras)와 [즉시 실행(eager execution)](https://www.tensorflow.org/programmers_guide/eager)을 활용하여 구현된 실행 가능한 코드가 포함되어 있습니다. 다음은 이 튜토리얼의 30번의 에포크(Epoch)로 훈련된 모델에서 \"Q\" 문자열로 시작될 때의 샘플 출력입니다.\n", - "\n", - "\u003cpre\u003e\n", - "QUEENE:\n", - "I had thought thou hadst a Roman; for the oracle,\n", - "Thus by All bids the man against the word,\n", - "Which are so weak of care, by old care done;\n", - "Your children were in your holy love,\n", - "And the precipitation through the bleeding throne.\n", - "\n", - "BISHOP OF ELY:\n", - "Marry, and will, my lord, to weep in such a one were prettiest;\n", - "Yet now I was adopted heir\n", - "Of the world's lamentable day,\n", - "To watch the next way with his father with his face?\n", - "\n", - "ESCALUS:\n", - "The cause why then we are all resolved more sons.\n", - "\n", - "VOLUMNIA:\n", - "O, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, it is no sin it should be dead,\n", - "And love and pale as any will to that word.\n", - "\n", - "QUEEN ELIZABETH:\n", - "But how long have I heard the soul for this world,\n", - "And show his hands of life be proved to stand.\n", - "\n", - "PETRUCHIO:\n", - "I say he look'd on, if I must be content\n", - "To stay him from the fatal of our country's bliss.\n", - "His lordship pluck'd from this sentence then for prey,\n", - "And then let us twain, being the moon,\n", - "were she such a case as fills m\n", - "\u003c/pre\u003e\n", - "\n", - "문장 중 일부는 문법적으로 맞지만 대부분 자연스럽지 않습니다. 이 모델은 단어의 의미를 학습하지는 않았지만, 고려해야 할 점으로:\n", - "\n", - "* 모델은 문자 기반입니다. 훈련이 시작되었을 때, 이 모델은 영어 단어의 철자를 모르거나 심지어 텍스트의 단위가 단어라는 것도 모릅니다.\n", - "\n", - "* 출력의 구조는 대본과 유사합니다. 즉, 텍스트 블록은 대개 화자의 이름으로 시작하고 이 이름들은 모든 데이터셋에서 대문자로 씌여 있습니다.\n", - "\n", - "* 아래에 설명된 것처럼 이 모델은 작은 텍스트 배치(각 100자)로 훈련되었으며 논리적인 구조를 가진 더 긴 텍스트 시퀀스를 생성할 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "srXC6pLGLwS6" - }, - "source": [ - "## 설정" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WGyKZj3bzf9p" - }, - "source": [ - "### 텐서플로와 다른 라이브러리 임포트" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yG_n40gFzf9s" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0\n", - "import tensorflow as tf\n", - "\n", - "import numpy as np\n", - "import os\n", - "import time" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EHDoRoc5PKWz" - }, - "source": [ - "### 셰익스피어 데이터셋 다운로드\n", - "\n", - "다음 코드를 실행하여 데이터를 불러오세요." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "pD_55cOxLkAb" - }, - "outputs": [], - "source": [ - "path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UHjdCjDuSvX_" - }, - "source": [ - "### 데이터 읽기\n", - "\n", - "먼저, 텍스트를 살펴봅시다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aavnuByVymwK" - }, - "outputs": [], - "source": [ - "# 읽은 다음 파이썬 2와 호환되도록 디코딩합니다.\n", - "text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n", - "# 텍스트의 길이는 그 안에 있는 문자의 수입니다.\n", - "print ('텍스트의 길이: {}자'.format(len(text)))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Duhg9NrUymwO" - }, - "outputs": [], - "source": [ - "# 텍스트의 처음 250자를 살펴봅니다\n", - "print(text[:250])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IlCgQBRVymwR" - }, - "outputs": [], - "source": [ - "# 파일의 고유 문자수를 출력합니다.\n", - "vocab = sorted(set(text))\n", - "print ('고유 문자수 {}개'.format(len(vocab)))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rNnrKn_lL-IJ" - }, - "source": [ - "## 텍스트 처리" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LFjSVAlWzf-N" - }, - "source": [ - "### 텍스트 벡터화\n", - "\n", - "훈련 전, 문자들을 수치화할 필요가 있습니다. 두 개의 조회 테이블(lookup table)을 만듭니다: 하나는 문자를 숫자에 매핑하고 다른 하나는 숫자를 문자에 매핑하는 것입니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IalZLbvOzf-F" - }, - "outputs": [], - "source": [ - "# 고유 문자에서 인덱스로 매핑 생성\n", - "char2idx = {u:i for i, u in enumerate(vocab)}\n", - "idx2char = np.array(vocab)\n", - "\n", - "text_as_int = np.array([char2idx[c] for c in text])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tZfqhkYCymwX" - }, - "source": [ - "이제 각 문자에 대한 정수 표현을 만들었습니다. 문자를 0번 인덱스부터 고유 문자 길이까지 매핑한 것을 기억합시다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FYyNlCNXymwY" - }, - "outputs": [], - "source": [ - "print('{')\n", - "for char,_ in zip(char2idx, range(20)):\n", - " print(' {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n", - "print(' ...\\n}')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l1VKcQHcymwb" - }, - "outputs": [], - "source": [ - "# 텍스트에서 처음 13개의 문자가 숫자로 어떻게 매핑되었는지를 보여줍니다\n", - "print ('{} ---- 문자들이 다음의 정수로 매핑되었습니다 ---- \u003e {}'.format(repr(text[:13]), text_as_int[:13]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bbmsf23Bymwe" - }, - "source": [ - "### 예측 과정" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wssHQ1oGymwe" - }, - "source": [ - "주어진 문자나 문자 시퀀스가 주어졌을 때, 다음 문자로 가장 가능성 있는 문자는 무엇일까요? 이는 모델을 훈련하여 수행할 작업입니다. 모델의 입력은 문자열 시퀀스가 될 것이고, 모델을 훈련시켜 출력을 예측합니다. 이 출력은 현재 타임 스텝(time step)의 다음 문자입니다.\n", - "\n", - "RNN은 이전에 본 요소에 의존하는 내부 상태를 유지하고 있으므로, 이 순간까지 계산된 모든 문자를 감안할 때, 다음 문자는 무엇일까요?" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hgsVvVxnymwf" - }, - "source": [ - "### 훈련 샘플과 타깃 만들기\n", - "\n", - "다음으로 텍스트를 샘플 시퀀스로 나눕니다. 각 입력 시퀀스에는 텍스트에서 나온 `seq_length`개의 문자가 포함될 것입니다.\n", - "\n", - "각 입력 시퀀스에서, 해당 타깃은 한 문자를 오른쪽으로 이동한 것을 제외하고는 동일한 길이의 텍스트를 포함합니다.\n", - "\n", - "따라서 텍스트를`seq_length + 1`개의 청크(chunk)로 나눕니다. 예를 들어, `seq_length`는 4이고 텍스트를 \"Hello\"이라고 가정해 봅시다. 입력 시퀀스는 \"Hell\"이고 타깃 시퀀스는 \"ello\"가 됩니다.\n", - "\n", - "이렇게 하기 위해 먼저 `tf.data.Dataset.from_tensor_slices` 함수를 사용해 텍스트 벡터를 문자 인덱스의 스트림으로 변환합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0UHJDA39zf-O" - }, - "outputs": [], - "source": [ - "# 단일 입력에 대해 원하는 문장의 최대 길이\n", - "seq_length = 100\n", - "examples_per_epoch = len(text)//seq_length\n", - "\n", - "# 훈련 샘플/타깃 만들기\n", - "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n", - "\n", - "for i in char_dataset.take(5):\n", - " print(idx2char[i.numpy()])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ZSYAcQV8OGP" - }, - "source": [ - "`batch` 메서드는 이 개별 문자들을 원하는 크기의 시퀀스로 쉽게 변환할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "l4hkDU3i7ozi" - }, - "outputs": [], - "source": [ - "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n", - "\n", - "for item in sequences.take(5):\n", - " print(repr(''.join(idx2char[item.numpy()])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UbLcIPBj_mWZ" - }, - "source": [ - "각 시퀀스에서, `map` 메서드를 사용해 각 배치에 간단한 함수를 적용하고 입력 텍스트와 타깃 텍스트를 복사 및 이동합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9NGu-FkO_kYU" - }, - "outputs": [], - "source": [ - "def split_input_target(chunk):\n", - " input_text = chunk[:-1]\n", - " target_text = chunk[1:]\n", - " return input_text, target_text\n", - "\n", - "dataset = sequences.map(split_input_target)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiCopyGZymwi" - }, - "source": [ - "첫 번째 샘플의 타깃 값을 출력합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GNbw-iR0ymwj" - }, - "outputs": [], - "source": [ - "for input_example, target_example in dataset.take(1):\n", - " print ('입력 데이터: ', repr(''.join(idx2char[input_example.numpy()])))\n", - " print ('타깃 데이터: ', repr(''.join(idx2char[target_example.numpy()])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_33OHL3b84i0" - }, - "source": [ - "이 벡터의 각 인덱스는 하나의 타임 스텝(time step)으로 처리됩니다. 타임 스텝 0의 입력으로 모델은 \"F\"의 인덱스를 받고 다음 문자로 \"i\"의 인덱스를 예측합니다. 다음 타임 스텝에서도 같은 일을 하지만 RNN은 현재 입력 문자 외에 이전 타임 스텝의 컨텍스트**(context)**를 고려합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "0eBu9WZG84i0" - }, - "outputs": [], - "source": [ - "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n", - " print(\"{:4d}단계\".format(i))\n", - " print(\" 입력: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n", - " print(\" 예상 출력: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MJdfPmdqzf-R" - }, - "source": [ - "### 훈련 배치 생성\n", - "\n", - "텍스트를 다루기 쉬운 시퀀스로 분리하기 위해 `tf.data`를 사용했습니다. 그러나 이 데이터를 모델에 넣기 전에 데이터를 섞은 후 배치를 만들어야 합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "p2pGotuNzf-S" - }, - "outputs": [], - "source": [ - "# 배치 크기\n", - "BATCH_SIZE = 64\n", - "\n", - "# 데이터셋을 섞을 버퍼 크기\n", - "# (TF 데이터는 무한한 시퀀스와 함께 작동이 가능하도록 설계되었으며,\n", - "# 따라서 전체 시퀀스를 메모리에 섞지 않습니다. 대신에,\n", - "# 요소를 섞는 버퍼를 유지합니다).\n", - "BUFFER_SIZE = 10000\n", - "\n", - "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n", - "\n", - "dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r6oUuElIMgVx" - }, - "source": [ - "## 모델 설계" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m8gPwEjRzf-Z" - }, - "source": [ - "모델을 정의하려면 `tf.keras.Sequential`을 사용합니다. 이 간단한 예제에서는 3개의 층을 사용하여 모델을 정의합니다:\n", - "\n", - "* `tf.keras.layers.Embedding` : 입력층. `embedding_dim` 차원 벡터에 각 문자의 정수 코드를 매핑하는 훈련 가능한 검색 테이블.\n", - "* `tf.keras.layers.GRU` : 크기가 `units = rnn_units`인 RNN의 유형(여기서 LSTM층을 사용할 수도 있습니다.)\n", - "* `tf.keras.layers.Dense` : 크기가 `vocab_size`인 출력을 생성하는 출력층." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zHT8cLh7EAsg" - }, - "outputs": [], - "source": [ - "# 문자로 된 어휘 사전의 크기\n", - "vocab_size = len(vocab)\n", - "\n", - "# 임베딩 차원\n", - "embedding_dim = 256\n", - "\n", - "# RNN 유닛(unit) 개수\n", - "rnn_units = 1024" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MtCrdfzEI2N0" - }, - "outputs": [], - "source": [ - "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(vocab_size, embedding_dim,\n", - " batch_input_shape=[batch_size, None]),\n", - " tf.keras.layers.LSTM(rnn_units,\n", - " return_sequences=True,\n", - " stateful=True,\n", - " recurrent_initializer='glorot_uniform'),\n", - " tf.keras.layers.Dense(vocab_size)\n", - " ])\n", - " return model" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wwsrpOik5zhv" - }, - "outputs": [], - "source": [ - "model = build_model(\n", - " vocab_size = len(vocab),\n", - " embedding_dim=embedding_dim,\n", - " rnn_units=rnn_units,\n", - " batch_size=BATCH_SIZE)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RkA5upJIJ7W7" - }, - "source": [ - "각 문자에 대해 모델은 임베딩을 검색하고, 임베딩을 입력으로 하여 GRU를 1개의 타임 스텝으로 실행하고, 완전연결층을 적용하여 다음 문자의 로그 가능도(log-likelihood)를 예측하는 로짓을 생성합니다:\n", - "\n", - "![모델을 통과하는 데이터의 사진](https://tensorflow.org/tutorials/text/images/text_generation_training.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ubPo0_9Prjb" - }, - "source": [ - "## 모델 사용\n", - "\n", - "이제 모델을 실행하여 원하는대로 동작하는지 확인합니다.\n", - "\n", - "먼저 출력의 형태를 확인합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "C-_70kKAPrPU" - }, - "outputs": [], - "source": [ - "for input_example_batch, target_example_batch in dataset.take(1):\n", - " example_batch_predictions = model(input_example_batch)\n", - " print(example_batch_predictions.shape, \"# (배치 크기, 시퀀스 길이, 어휘 사전 크기)\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q6NzLBi4VM4o" - }, - "source": [ - "위 예제에서 입력의 시퀀스 길이는 100이지만 모델은 임의 길이의 입력에서 실행될 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vPGmAAXmVLGC" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uwv0gEkURfx1" - }, - "source": [ - "모델로부터 실제 예측을 얻으려면 출력 배열에서 샘플링하여 실제 문자 인덱스를 얻어야 합니다. 이 분포는 문자 어휘에 대한 로짓에 의해 정의됩니다.\n", - "\n", - "Note: 배열에 argmax를 취하면 모델이 쉽게 루프에 걸릴 수 있으므로 배열에서 샘플링하는 것이 중요합니다.\n", - "\n", - "배치의 첫 번째 샘플링을 시도해 봅시다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4V4MfFg0RQJg" - }, - "outputs": [], - "source": [ - "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n", - "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QM1Vbxs_URw5" - }, - "source": [ - "이렇게 하면 각 타임 스텝(time step)에서 다음 문자 인덱스에 대한 예측을 제공합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YqFMUQc_UFgM" - }, - "outputs": [], - "source": [ - "sampled_indices" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LfLtsP3mUhCG" - }, - "source": [ - "훈련되지 않은 모델에 의해 예측된 텍스트를 보기 위해 복호화합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xWcFwPwLSo05" - }, - "outputs": [], - "source": [ - "print(\"입력: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n", - "print()\n", - "print(\"예측된 다음 문자: \\n\", repr(\"\".join(idx2char[sampled_indices ])))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LJL0Q0YPY6Ee" - }, - "source": [ - "## 모델 훈련" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YCbHQHiaa4Ic" - }, - "source": [ - "이 문제는 표준 분류 문제로 취급될 수 있습니다. 이전 RNN 상태와 이번 타임 스텝(time step)의 입력으로 다음 문자의 클래스를 예측합니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "trpqTWyvk0nr" - }, - "source": [ - "### 옵티마이저 붙이기, 그리고 손실 함수" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UAjbjY03eiQ4" - }, - "source": [ - "표준 `tf.keras.losses.sparse_softmax_crossentropy` 손실 함수는 이전 차원의 예측과 교차 적용되기 때문에 이 문제에 적합합니다.\n", - "\n", - "이 모델은 로짓을 반환하기 때문에 `from_logits` 플래그를 설정해야 합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4HrXTACTdzY-" - }, - "outputs": [], - "source": [ - "def loss(labels, logits):\n", - " return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n", - "\n", - "example_batch_loss = loss(target_example_batch, example_batch_predictions)\n", - "print(\"예측 배열 크기(shape): \", example_batch_predictions.shape, \" # (배치 크기, 시퀀스 길이, 어휘 사전 크기\")\n", - "print(\"스칼라 손실: \", example_batch_loss.numpy().mean())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jeOXriLcymww" - }, - "source": [ - "`tf.keras.Model.compile` 메서드를 사용하여 훈련 절차를 설정합니다. 기본 매개변수의 `tf.keras.optimizers.Adam`과 손실 함수를 사용합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DDl1_Een6rL0" - }, - "outputs": [], - "source": [ - "model.compile(optimizer='adam', loss=loss)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ieSJdchZggUj" - }, - "source": [ - "### 체크포인트 구성" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C6XBUUavgF56" - }, - "source": [ - "`tf.keras.callbacks.ModelCheckpoint`를 사용하여 훈련 중 체크포인트(checkpoint)가 저장되도록 합니다:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "W6fWTriUZP-n" - }, - "outputs": [], - "source": [ - "# 체크포인트가 저장될 디렉토리\n", - "checkpoint_dir = './training_checkpoints'\n", - "# 체크포인트 파일 이름\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n", - "\n", - "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n", - " filepath=checkpoint_prefix,\n", - " save_weights_only=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Ky3F_BhgkTW" - }, - "source": [ - "### 훈련 실행" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IxdOA-rgyGvs" - }, - "source": [ - "훈련 시간이 너무 길지 않도록 모델을 훈련하는 데 10개의 에포크(Epoch)를 사용합니다. 코랩(Colab)에서는 런타임을 GPU로 설정하여 보다 빠르게 훈련할 수 있습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7yGBE2zxMMHs" - }, - "outputs": [], - "source": [ - "EPOCHS=10" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UK-hmKjYVoll" - }, - "outputs": [], - "source": [ - "history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKkD5M6eoSiN" - }, - "source": [ - "## 텍스트 생성" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JIPcXllKjkdr" - }, - "source": [ - "### 최근 체크포인트 복원" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LyeYRiuVjodY" - }, - "source": [ - "이 예측 단계를 간단히 유지하기 위해 배치 크기로 1을 사용합니다.\n", - "\n", - "RNN 상태가 타임 스텝에서 타임 스텝으로 전달되는 방식이기 때문에 모델은 한 번 빌드된 고정 배치 크기만 허용합니다.\n", - "\n", - "다른 배치 크기로 모델을 실행하려면 모델을 다시 빌드하고 체크포인트에서 가중치를 복원해야 합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zk2WJ2-XjkGz" - }, - "outputs": [], - "source": [ - "tf.train.latest_checkpoint(checkpoint_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LycQ-ot_jjyu" - }, - "outputs": [], - "source": [ - "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n", - "\n", - "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "model.build(tf.TensorShape([1, None]))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "71xa6jnYVrAN" - }, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DjGz1tDkzf-u" - }, - "source": [ - "### 예측 루프\n", - "\n", - "다음 코드 블록은 텍스트를 생성합니다:\n", - "\n", - "* 시작 문자열 선택과 순환 신경망 상태를 초기화하고 생성할 문자 수를 설정하면 시작됩니다.\n", - "\n", - "* 시작 문자열과 순환 신경망 상태를 사용하여 다음 문자의 예측 배열을 가져옵니다.\n", - "\n", - "* 다음, 범주형 배열을 사용하여 예측된 문자의 인덱스를 계산합니다. 이 예측된 문자를 모델의 다음 입력으로 활용합니다.\n", - "\n", - "* 모델에 의해 리턴된 RNN 상태는 모델로 피드백되어 이제는 하나의 단어가 아닌 더 많은 컨텍스트를 갖추게 됩니다. 다음 단어를 예측한 후 수정된 RNN 상태가 다시 모델로 피드백되어 이전에 예측된 단어에서 더 많은 컨텍스트를 얻으면서 학습하는 방식입니다.\n", - "\n", - "![텍스트를 생성하기 위해 모델의 출력이 입력으로 피드백](https://tensorflow.org/tutorials/text/images/text_generation_sampling.png)\n", - "\n", - "생성된 텍스트를 보면 모델이 언제 대문자로 나타나고, 절을 만들고 셰익스피어와 유사한 어휘를 가져오는지 알 수 있습니다. 훈련 에포크(Epoch)가 적은 관계로 논리적인 문장을 형성하는 것은 아직 훈련되지 않았습니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WvuwZBX5Ogfd" - }, - "outputs": [], - "source": [ - "def generate_text(model, start_string):\n", - " # 평가 단계 (학습된 모델을 사용하여 텍스트 생성)\n", - "\n", - " # 생성할 문자의 수\n", - " num_generate = 1000\n", - "\n", - " # 시작 문자열을 숫자로 변환(벡터화)\n", - " input_eval = [char2idx[s] for s in start_string]\n", - " input_eval = tf.expand_dims(input_eval, 0)\n", - "\n", - " # 결과를 저장할 빈 문자열\n", - " text_generated = []\n", - "\n", - " # 온도가 낮으면 더 예측 가능한 텍스트가 됩니다.\n", - " # 온도가 높으면 더 의외의 텍스트가 됩니다.\n", - " # 최적의 세팅을 찾기 위한 실험\n", - " temperature = 1.0\n", - "\n", - " # 여기에서 배치 크기 == 1\n", - " model.reset_states()\n", - " for i in range(num_generate):\n", - " predictions = model(input_eval)\n", - " # 배치 차원 제거\n", - " predictions = tf.squeeze(predictions, 0)\n", - "\n", - " # 범주형 분포를 사용하여 모델에서 리턴한 단어 예측\n", - " predictions = predictions / temperature\n", - " predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n", - "\n", - " # 예측된 단어를 다음 입력으로 모델에 전달\n", - " # 이전 은닉 상태와 함께\n", - " input_eval = tf.expand_dims([predicted_id], 0)\n", - "\n", - " text_generated.append(idx2char[predicted_id])\n", - "\n", - " return (start_string + ''.join(text_generated))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ktovv0RFhrkn" - }, - "outputs": [], - "source": [ - "print(generate_text(model, start_string=u\"ROMEO: \"))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AM2Uma_-yVIq" - }, - "source": [ - "결과를 개선하는 가장 쉬운 방법은 더 오래 훈련하는 것입니다(`EPOCHS = 30`을 시도해 봅시다).\n", - "\n", - "또한 다른 시작 문자열을 시험해 보거나 모델의 정확도를 높이기 위해 다른 RNN 레이어를 추가하거나 온도 파라미터를 조정하여 많은 혹은 적은 임의의 예측을 생성할 수 있습니다." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y4QwTjAM6A2O" - }, - "source": [ - "## 고급: 맞춤식 훈련\n", - "\n", - "위의 훈련 절차는 간단하지만 많은 권한을 부여하지는 않습니다.\n", - "\n", - "이제 수동으로 모델을 실행하는 방법을 살펴 보았으니 이제 훈련 루프를 해제하고 직접 구현합시다.\n", - "이는 시작점을 제공해 주는데, 예를 들어 커리큘럼 학습(curriculum learning)을 구현하면 모델의 오픈 루프(open-loop) 출력을 안정적으로 하는 데 도움을 줍니다.\n", - "\n", - "기울기 추적을 위해 `tf.GradientTape`을 사용합니다. 이 방법에 대한 자세한 내용은 [즉시 실행 가이드](https://www.tensorflow.org/guide/eager)를 참조하십시오.\n", - "\n", - "절차는 다음과 같이 동작합니다:\n", - "\n", - "* 먼저 RNN 상태를 초기화합니다. 우리는`tf.keras.Model.reset_states` 메서드를 호출하여 이를 수행합니다.\n", - "\n", - "* 다음으로 데이터셋(배치별로)를 반복하고 각각에 연관된 *예측*을 계산합니다.\n", - "\n", - "* `tf.GradientTape`를 열고 그 컨텍스트에서의 예측과 손실을 계산합니다.\n", - "\n", - "* `tf.GradientTape.grads` 메서드를 사용하여 모델 변수에 대한 손실의 기울기를 계산합니다.\n", - "\n", - "* 마지막으로 옵티마이저의 `tf.train.Optimizer.apply_gradients` 메서드를 사용하여 이전 단계로 이동합니다." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_XAm7eCoKULT" - }, - "outputs": [], - "source": [ - "model = build_model(\n", - " vocab_size = len(vocab),\n", - " embedding_dim=embedding_dim,\n", - " rnn_units=rnn_units,\n", - " batch_size=BATCH_SIZE)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qUKhnZtMVpoJ" - }, - "outputs": [], - "source": [ - "optimizer = tf.keras.optimizers.Adam()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "b4kH1o0leVIp" - }, - "outputs": [], - "source": [ - "@tf.function\n", - "def train_step(inp, target):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(inp)\n", - " loss = tf.reduce_mean(\n", - " tf.keras.losses.sparse_categorical_crossentropy(\n", - " target, predictions, from_logits=True))\n", - " grads = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " return loss" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "d4tSNwymzf-q" - }, - "outputs": [], - "source": [ - "# 훈련 횟수\n", - "EPOCHS = 10\n", - "\n", - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - "\n", - " # 모든 에포크(Epoch)의 시작에서 은닉 상태를 초기화\n", - " # 초기 은닉 상태는 None\n", - " hidden = model.reset_states()\n", - "\n", - " for (batch_n, (inp, target)) in enumerate(dataset):\n", - " loss = train_step(inp, target)\n", - "\n", - " if batch_n % 100 == 0:\n", - " template = '에포크 {} 배치 {} 손실 {}'\n", - " print(template.format(epoch+1, batch_n, loss))\n", - "\n", - " # 모든 5 에포크(Epoch)마다(체크포인트) 모델 저장\n", - " if (epoch + 1) % 5 == 0:\n", - " model.save_weights(checkpoint_prefix.format(epoch=epoch))\n", - "\n", - " print ('에포크 {} 손실 {:.4f}'.format(epoch+1, loss))\n", - " print ('1 에포크 당 {}초 소요\\n'.format(time.time() - start))\n", - "\n", - "model.save_weights(checkpoint_prefix.format(epoch=epoch))" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "text_generation.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3.5 (NGC/TensorFlow 18.12) on Backend.AI", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/pt/README.md b/site/pt/README.md deleted file mode 100644 index 55f0b1384fb..00000000000 --- a/site/pt/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Traduções da Comunidade - -A nossa comunidade TensorFlow traduziu estes documentos. Como as traduções da -comunidade são *o melhor esforço*, não há garantias de que sejam uma reflexão -exata e atualizada da [documentação oficial em Inglês](https://www.tensorflow.org/?hl=en). -Se tem alguma sugestão para melhorar esta tradução, por favor envie um pull -request para o repositório do GitHub [tensorflow/docs](https://github.com/tensorflow/docs). -Para se voluntariar para escrever ou rever as traduções da comunidade, contacte a -[lista docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -# Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Please focus translation efforts on -[TensorFlow 2](https://www.tensorflow.org) in the -[site/en/](https://github.com/tensorflow/docs/tree/master/site/en/) -directory. TF 1.x community docs will no longer be updated as we prepare for the -2.0 release. See -[the announcement](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). - -# Portuguese translation guide - -Some technical words in English do not have a natural translation. Do *not* -translate the following words: - -* (mini-) batch -* estimator -* eager execution -* dense diff --git a/site/pt/REVIEWERS b/site/pt/REVIEWERS deleted file mode 100644 index 3a1afdc84b7..00000000000 --- a/site/pt/REVIEWERS +++ /dev/null @@ -1,5 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/pt directory. Remove your username whenever you'd like. -# -diogoarm -ericodex diff --git a/site/pt/r1/README.md b/site/pt/r1/README.md deleted file mode 100644 index c43dac07538..00000000000 --- a/site/pt/r1/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# TensorFlow 1.x - -This archive of the TensorFlow 1.x docs is in *maintenance mode* only. - -For docs contributors, please update the source files in `site/en/` and read the -[TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs). - -For community translations, read the instructions in `site/pt/README.md`. diff --git a/site/pt/r1/tutorials/keras/basic_classification.ipynb b/site/pt/r1/tutorials/keras/basic_classification.ipynb deleted file mode 100644 index 0441796d1fd..00000000000 --- a/site/pt/r1/tutorials/keras/basic_classification.ipynb +++ /dev/null @@ -1,1002 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# Treine a sua primeira rede neural: Classificação Básica" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Execute em Google Colab\n", - " \n", - " Veja a fonte em GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UA1pKuH8wqxz", - "colab_type": "text" - }, - "source": [ - "Note: A nossa comunidade TensorFlow traduziu estes documentos. Como as traduções da comunidade são *o melhor esforço*, não há garantias de que sejam uma reflexão exata e atualizada da [documentação oficial em Inglês](https://www.tensorflow.org/?hl=en). Se tem alguma sugestão para melhorar esta tradução, por favor envie um pull request para o repositório do GitHub [tensorflow/docs](https://github.com/tensorflow/docs). Para se voluntariar para escrever ou rever as traduções da comunidade, contacte a [lista docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "OezgXCpHi4v_", - "colab_type": "text" - }, - "source": [ - "Este guia treina um modelo de uma rede neural para classificar imagens de roupa, como ténis ou camisolas. Não há problema se não perceber os detalhes, isto é uma revisão rápida de um programa completo em TensorFlow com os detalhes explicados à medida que avançamos.\n", - "\n", - "Este guia usa [tf.keras](https://www.tensorflow.org/r1/guide/keras), uma API de alto nivel para construir e treinar modelos em TensorFlow." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "# TensorFlow e tf.keras\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# Bibliotecas de ajuda\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## Importar o dataset Fashion MNIST" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "Este guia usa a base de dados [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) que contém 70.000 imagens a preto e branco de 10 categorias diferentes. As imagens apresentam peças de roupa individuais com pouca resolução (28 por 28 píxeis), como podem ser vistas aqui:\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
     \n", - "
    \n", - "\n", - "Fashion MNIST tem a intenção de substituir a base de dados clássica [MNIST](http://yann.lecun.com/exdb/mnist/), que é usualmente utilizada como \"Olá, Mundo\" de programas de aprendizagem de máquina para visão computacional. A base de dados MNIST contém imagens de dígitos manuscritos (0, 1, 2, etc) num formato idêntico ao dos artigos de roupa que vamos utilizar aqui.\n", - "\n", - "Este guia utiliza Fashion MNIST para variedade, e porque é um problema ligeiramente mais desafiante do que a MNIST. Ambas as bases de dados são relativamente pequenas e são utilizadas para verificar se os algoritmos trabalham como é esperado. São um bom início para testar e depurar o código.\n", - "\n", - "Vamos utilizar 60.000 imagens para treinar a rede e 10.000 imagens para avaliar quão bem a rede aprendeu a classificar as imagens. Você pode acessar a base de dados Fashion MNIST diretamente a partir do TensorFlow, basta importar os dados:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "Carregar a base de dados retorna quatro NumPy arrays:\n", - "\n", - "* As `train_images` e `train_labels` arrays são o *training set*— os dados que o modelo utiliza para aprender\n", - "* O modelo é testado no *test set*, os arrays de `test_images` e os `test_labels`.\n", - "\n", - "As images são 28x28 Numpy arrays, onde os valores de cada pixel variam entre 0 e 255. Os *labels* são um vetor de inteiros, que varia entre 0 e 9. Estes correspondem à *classe* de roupa que a imagem representa:\n", - "\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    LabelClass
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "Cada imagem é mapeada para um único label. Uma vez que os *nomes das classes* não estão incluídos na base de dados, guardamos aqui para mais tarde usar nos gráficos das images:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## Explorar os dados\n", - "\n", - "Vamos explorar o formato da base de dados antes de treinar o modelo. De seguida, mostra-se que existem 60.000 imagens nos dados de treino, com cada imagem representada com 28x28 píxeis:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "De igual forma, existem 60.000 labels no dados de treino:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "Cada label é um inteiro entre 0 e 9:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "Existem 10.000 imagens nos dados de teste. Cada imagem é novamente representada por 28 x 28 píxeis:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "E os dados de teste contém 10.000 labels de image:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## Pré-processamento dos dados\n", - "\n", - "Os dados devem ser pré-processados antes de treinar a rede. Se observar a primeira imagem dos dados de treino, vai ver que o valor de cada píxel vai estar no intervalo entre 0 e 255:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "Nós escalamos estes valores para estarem no intervalo de 0 a 1 antes de entrar no modelo neural. Para isso, dividimos os valores por 255. É importante que os dados de treino e os dados de teste sejam processados da mesma forma:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "Mostre as primeiras 25 imagens dos *dados de treino* e a classe a que pertence por baixo de cada imagem. Verifique que os dados estão no formato correto e estamos prontos para contruir e treinar a rede." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## Construir o modelo\n", - "\n", - "Construir a rede neural requer a configuração das camadas do modelo, e depois a compilação do modelo." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### Configurar as camadas\n", - "\n", - "O bloco básico de uma rede neural é a *layer* (camada). As camadas extraem representações dos dados que as alimentam. E, esperançosamente, estas representações têm mais significado para o problema em mãos.\n", - "\n", - "A maioria do deep learning consiste em encadear camadas simples. A maioria das camadas, como `tf.keras.layers.Dense`, têm parâmetros que são aprendidos durante o treino." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation=tf.nn.relu),\n", - " keras.layers.Dense(10, activation=tf.nn.softmax)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "A primeira camada desta rede, `tf.keras.layers.Flatten`, transforma o formato das imagens de um vector 2D (de 28 por 28 píxeis), num vetor 1D de 28 * 28 = 784 píxeis. Considere esta camada como retirar as linhas de píxeis da imagem e alinhá-las. Esta camada não tem parâmetros para aprender; apenas re-formata os dados.\n", - "\n", - "Depois de os píxeis estarem alinhados, a rede consiste numa sequência de duas camadas `tf.keras.layers.Dense`. Estas são categorizadas como *densely-connected*, ou *fully-connected*. A primeira camada `Dense` tem 128 nós (ou neurónios). A segunda (e última) é uma camada *softmax* com 10 nós - isto retorna um vetor com 10 valores de probabilidade que soma 1. Cada nó contém o valor que indica a probabilidade que a image atual pertence a uma das 10 classes.\n", - "\n", - "### Compila o modelo\n", - "\n", - "Antes de o modelo estar pronto para treinar, este precisa de mais algumas configurações. Estas são adicionadas no passo *compile*:\n", - "\n", - "* *Função de custo - Isto mede quão bem o modelo classifica durante o treino. Nós queremos minimizar esta função para ajustar o modelo na direção correta*\n", - "\n", - "* *Otimizador* - Isto é utilizado para atualizar o modelo com base nos dados e na função de custo.\n", - "\n", - "* *Métricas* - São utilizadas para monitorizar os passos de treino e teste. O próximo exemplo usa *exatidão*, a fração das images que são corretamente classificadas.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## Treinar o modelo\n", - "\n", - "Treinar o modelo da rede neural requer os seguintes passos:\n", - "\n", - "1. Alimentar o modelo com os dados de treino, neste caso os vetores `train_images` e `train_labels`.\n", - "2. O modelo aprende a associar as imagens e os labels.\n", - "3. Nós pedimos ao modelo para fazer uma previsão nos dados de teste - neste exemplo, o vetor `test_images`. Verificamos as previsões com o vetor de labels `test_labels`.\n", - "\n", - "Para começar a treinar, chamamos o método `mode.fit`- o modelo vai ser a aproximado para descrever os dados de treino:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "À medida que o modelo treina, a função de custo e as métricas de exatidão são mostradas. Este modelo atinge uma exatidão de cerca de 0.88 (ou 88%) nos dados de treino." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## Avaliar a exatidão\n", - "\n", - "De seguida, compare como o modelo se comporta nos dados de test:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('Test accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "Ao que parece, a exatidão nos dados de teste é um pouco menor do que a exatidão nos dados de treino. Esta lacuna entre a exatidão de treino e de teste é um exemplo de *overfitting*. Que significa quando um modelo de aprendizagem automática funciona pior em dados novos do que naqueles em que foi treinado." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## Fazer previsões\n", - "\n", - "Com o modelo treinado, podemos utiliza-lo para fazer previsões sobre algumas imagens." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "Aqui, o modelo previu o label para cada imagem nos dados de treino. Vamos observar a primeira previsão:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "A previsão é um vetor de 10 números. Este descrevem a \"confiança\" que o modelo tem de que esta imagem pertence a cada um dos 10 artigos diferentes de roupa. Podemos ver qual o rótulo apresenta a maior confiança:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "Então o modelo está mais confiante de que a image é *ankle boot*, ou `class_names[9]`. E podemos verificar o label do teste para verificar que está correto:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "Podemos criar o gráfico para observar todos os 10 canais." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DvYmmrpIy6Y1", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d4Ov9OFDMmOD" - }, - "source": [ - "Vamos observar a imagem 0, as previsões, e o vector de previsão." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HV5jw-5HwSmO", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ko-uzOufSCSe", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "Vamos criar o gráfico com várias imagens e as suas previsões. As previsões correta encontram-se a azul e as incorretas a vermelho. O número indica a percentagem (em 100) para o label previsto. De notar, que pode estar errado mesmo quando o nível de confiança é elevado." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "dnZo4B9Ci4xZ", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Criar o gráfico das primeiras X imagens, o label previsto e o label correto\n", - "# Colorir as previsões corretas a azul e as incorretas a vermelho\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "Finalmente, use o modelo treinado para fazer previsões sobre uma única imagem." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "dd5X1jYRi4xe", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Escolha uma imagem dos dados de teste\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "Os modelos `tf.keras` estão otimizados para fazer previsões num *batch*, ou coleção, de exemplos de uma vez. Então, ainda que só estejamos a utilizar uma imagem temos de adicioná-la a uma lista:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# Adicione a imagem a batch onde é o único membro.\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "Agora faça uma previsão sobre a imagem:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ai-cpLjO-3A", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "`model.predict` retorna uma lista de listas, uma para cada imagem do batch de dados. Selecione a previsão para a única image do batch:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "np.argmax(predictions_single[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "E tal como antes, o modelo prevê o label 9." - ] - } - ] -} \ No newline at end of file diff --git a/site/pt/r1/tutorials/keras/basic_regression.ipynb b/site/pt/r1/tutorials/keras/basic_regression.ipynb deleted file mode 100644 index d8b2f408b13..00000000000 --- a/site/pt/r1/tutorials/keras/basic_regression.ipynb +++ /dev/null @@ -1,856 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# Regressão: preveja consumo de combustível" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Execute em Google Colab\n", - " \n", - " Veja a fonte em GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UA1pKuH8wqxz", - "colab_type": "text" - }, - "source": [ - "Note: A nossa comunidade TensorFlow traduziu estes documentos. Como as traduções da comunidade são *o melhor esforço*, não há garantias de que sejam uma reflexão exata e atualizada da [documentação oficial em Inglês](https://www.tensorflow.org/?hl=en). Se tem alguma sugestão para melhorar esta tradução, por favor envie um pull request para o repositório do GitHub [tensorflow/docs](https://github.com/tensorflow/docs). Para se voluntariar para escrever ou rever as traduções da comunidade, contacte a [lista docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "OezgXCpHi4v_", - "colab_type": "text" - }, - "source": [ - "Em um problema de regressão, o objetivo é prever as saídas (*outputs*) de um valor contínuo, como um preço ou probabilidade. Em contraste de problemas de classificação, onde temos o propósito de escolher uma classe em uma lista de classificações (por exemplo, se uma imagem contém uma maçã ou laranja, assim reconhecendo qual fruta é representada na imagem).\n", - "\n", - "Este *notebook* usa a clássica base de dados [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) e constrói um modelo para prever a economia de combustíveis de automóveis do final dos anos 1970, início dos anos 1980. Para isso, forneceremos um modelo com descrição de vários automóveis desse período. Essa descrição inclui atributos como: cilindros, deslocamento, potência do motor, e peso.\n", - "\n", - "Este exemplo usa a API `tf.keras`. Veja [este guia](https://www.tensorflow.org/r1/guide/keras) para mais detalhes." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "# Use seaborn para pairplot\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "gOL_C-OkBKva", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## Base de dados Auto MPG\n", - "\n", - "A base de dados está disponível em [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "### Pegando os dados\n", - "Primeiro baixe a base de dados dos automóveis." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "Utilizando o pandas, impoorte os dados:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "### Limpe os dados\n", - "\n", - "Esta base contém alguns valores não conhecidos (*unknown*)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "Para manter esse tutorial básico, remova as linhas com esses valores não conhecidos." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "A coluna \"Origin\" é uma coluna categórica e não numérica. Logo converta para *one-hot* :" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "X_PB-wCUHgxU", - "colab_type": "code", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "### Separando dados de treinamento e teste\n", - "\n", - "Agora separe os dados em um conjunto de treinamento e outro teste.\n", - "\n", - "Iremos utilizar o de conjunto de teste no final da análise do model." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "### Inspecione o dado\n", - "\n", - "Dê uma rápida olhada em como está a distribuição de algumas colunas do conjunto de treinamento." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "Repare na visão geral dos estatísticas:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "### Separe features de labels\n", - "\n", - "Separe o valor alvo (*labels*), das *features*. Essa label é o valor no qual o modelo é treinado para prever." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "### Normalize os dados\n", - "\n", - "Observe novamente o `train_stats` acima e note quão diferente são os intervalos de uma feature e outra." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cVdIpzAdnxew", - "colab_type": "text" - }, - "source": [ - "Uma boa prática é normalizar as *features* que usam diferentes escalas e intervalos. Apesar do modelo poder convergir sem a normalização, isso torna o treinamento mais difícil, e torna o resultado do modelo dependente da escolha das unidades da entrada.\n", - "\n", - "Observação: embora geramos intencionalmente essas estatísticas para os dados de treinamento, essas estatísticas serão usadas também para normalizar o conjunto de teste. Precisamos delinear o conjunto de teste na mesma distribuição que o modelo foi treinado. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JoBpCiVPwwck", - "colab_type": "text" - }, - "source": [ - "Esse dado normalizado é o que usaremos para treinar o modelo.\n", - "\n", - "Atenção: As estatísticas usadas para normalizar as entradas aqui (média e desvio padrão) precisa ser aplicada em qualquer outro dado que alimenta o modelo, junto com o código *one-hot* que fizemos anteriormente. Isso inclui o conjunto de teste e os dados que o modelo usará em produção." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## O Modelo\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### Construindo o modelo\n", - "\n", - "Vamos construir o modelo. Aqui usaremos o modelo `Sequential` com duas camadas *densely connected*, e a camada de saída que retorna um único valor contínuo. Os passos de construção do modelo são agrupados em uma função, build_model, já que criaremos um segundo modelo mais tarde." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation=tf.nn.relu, input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation=tf.nn.relu),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mean_squared_error',\n", - " optimizer=optimizer,\n", - " metrics=['mean_absolute_error', 'mean_squared_error'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## Examine o modelo\n", - "\n", - "Use o método `.summary` para exibir uma descrição simples do modelo." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "Agora teste o modelo. Pegue um batch de de 10 exemplos do conjunto de treinamento e chame `model.predict`nestes." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "PcJrGsO5hZzK", - "colab_type": "code", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "Parece que está funcionando e ele produz o resultado de forma e tipo esperados." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "### Treinando o modelo\n", - "\n", - "Treine o modelo com 1000 *epochs*, e grave a acurácia do treinamento e da validação em um objeto `history`. " - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "a_MWJBuaC8EM", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Mostra o progresso do treinamento imprimindo um único ponto para cada epoch completada\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "OgQZyNaWEALn", - "colab_type": "text" - }, - "source": [ - "Visualize o progresso do modelo de treinamento usando o estados armazenados no objeto `history`" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "dsKd_b-ZEfKe", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mean_absolute_error'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mean_absolute_error'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mean_squared_error'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mean_squared_error'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "Este grafo mostra as pequenas melhoras, ou mesmo a diminuição do `validation error` após 100 *epochs*. Vamos atualizar o `model.fit` para que pare automatixamente o treinamento quando o `validation score` não aumentar mais. Usaremos o `EarlyStopping callback` que testa a condição do treinamento a cada `epoch`. Se um grupo de `epochs` decorre sem mostrar melhoras, o treinamento irá parar automaticamente.\n", - "\n", - "Você pode aprender mais sobre este callback [aqui](https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping)" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "fvV-BxG6FiaZ", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# The patience parameter is the amount of epochs to check for improvement\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "O gráfico mostra que no conjunto de validação, a média de erro é próximo de +/- 2MPG. Isso é bom? Deixaremos essa decisão a você.\n", - "\n", - "Vamos ver quão bem o modelo generaliza usando o conjunto de **teste**, que não usamos para treinar o modelo. Isso diz quão bem podemos esperar que o modelo se saia quando usarmos na vida real." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "prh4jLRTJ_Rc", - "colab_type": "code", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3IfAjv3BJleB", - "colab_type": "text" - }, - "source": [ - "### Make predictions\n", - "Finalmente, prevejamos os valores MPG usando o conjunto de teste." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "Parece que o nosso modelo prediz razoavelmente bem. Vamos dar uma olhada na distribuição dos erros." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "Não é tão gaussiana, porém podemos esperar que por conta do número de exemplo é bem pequeno." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "## Conclusão\n", - "\n", - "Este notebook introduz algumas técnicas para trabalhar com problema de regressão.\n", - "\n", - "* Mean Sqaured Error(MSE), é uma função comum de *loss* usada para problemas de regressão (diferentes funçẽso de *loss* são usadas para problemas de classificação).\n", - "* Similarmente, as métricas de evolução usadas na regressão são diferentes da classificação. Uma métrica comum de regressão é Mean Absolute Error (MAE).\n", - "* Quando o dado de entrada de *features* tem diferentes intervalos, cada *feature* deve ser escalada para o mesmo intervalo.\n", - "* Se não possui muitos dados de treinamento, uma técnica é preferir uma pequena rede com poucas camadas para evitar *overfitting*.\n", - "* *Early stopping* é uma boa técnica para evitar *overfitting*." - ] - } - ] -} \ No newline at end of file diff --git a/site/pt/r1/tutorials/keras/basic_text_classification.ipynb b/site/pt/r1/tutorials/keras/basic_text_classification.ipynb deleted file mode 100644 index 30a88606fe3..00000000000 --- a/site/pt/r1/tutorials/keras/basic_text_classification.ipynb +++ /dev/null @@ -1,724 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# Classificação de texto com avaliações de filmes" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Execute em Google Colab\n", - " \n", - " Veja a fonte em GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "\n", - "Este *notebook* classifica avaliações de filmes como **positiva** ou **negativa** usando o texto da avaliação. Isto é um exemplo de classificação *binária* —ou duas-classes—, um importante e bastante aplicado tipo de problema de aprendizado de máquina.\n", - "\n", - "Usaremos a base de dados [IMDB](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) que contém avaliaçòes de mais de 50000 filmes do bando de dados [Internet Movie Database](https://www.imdb.com/). A base é dividida em 25000 avaliações para treinamento e 25000 para teste. Os conjuntos de treinamentos e testes são *balanceados*, ou seja, eles possuem a mesma quantidade de avaliações positivas e negativas.\n", - "\n", - "O notebook utiliza [tf.keras](https://www.tensorflow.org/r1/guide/keras), uma API alto-nível para construir e treinar modelos com TensorFlow. Para mais tutoriais avançados de classificação de textos usando `tf.keras`, veja em [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JL-LtD0CBSZR", - "colab": {} - }, - "source": [ - "# keras.datasets.imdb está quebrado em 1.13 e 1.14, pelo np 1.16.3\n", - "!pip install tf_nightly" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## Baixe a base de dados IMDB\n", - "\n", - "A base de dados vem empacotada com TensorFlow. Ele já vem pré-processado de forma que as avaliações (sequências de palavras) foi convertida em sequências de inteiros, onde cada inteiro representa uma palavra específica no dicionário.\n", - "\n", - "O código abaixo baixa a base de dados IMDB para a sua máquina (ou usa a cópia em *cache*, caso já tenha baixado):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "O argumento `num_words=10000` mantém as 10000 palavras mais frequentes no conjunto de treinamento. As palavras mais raras são descartadas para preservar o tamanho dos dados de forma maleável." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## Explore os dados\n", - "\n", - "Vamos parar um momento para entender o formato dos dados. O conjunto de dados vem pré-processado: cada exemplo é um *array* de inteiros representando as palavras da avaliação do filme. Cada *label* é um inteiro com valor ou de 0 ou 1, onde 0 é uma avaliação negativa e 1 é uma avaliação positiva." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} - }, - "source": [ - "print(\"Training entries: {}, labels: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "O texto das avaliações foi convertido para inteiros, onde cada inteiro representa uma palavra específica no dicionário. Isso é como se parece a primeira revisão:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "As avaliações dos filmes têm diferentes tamanhos. O código abaixo mostra o número de palavras da primeira e segunda avaliação. Sabendo que o número de entradas da rede neural tem que ser de mesmo também, temos que resolver isto mais tarde." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### Converta os inteiros de volta a palavras\n", - "\n", - "É util saber como converter inteiros de volta a texto. Aqui, criaremos uma função de ajuda para consultar um objeto *dictionary* que contenha inteiros mapeados em strings:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} - }, - "source": [ - "# Um dicionário mapeando palavras em índices inteiros\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# Os primeiros índices são reservados\n", - "word_index = {k:(v+3) for k,v in word_index.items()}\n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # unknown\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "Agora, podemos usar a função `decode_review` para mostrar o texto da primeira avaliação:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} - }, - "source": [ - "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## Prepare os dados\n", - "\n", - "As avaliações—o *arrays* de inteiros— deve ser convertida em tensores (*tensors*) antes de alimentar a rede neural. Essa conversão pode ser feita de duas formas:\n", - "\n", - "* Converter os arrays em vetores de 0s e 1s indicando a ocorrência da palavra, similar com *one-hot encoding*. Por exemplo, a sequência [3, 5] se tornaria um vetor de 10000 dimensões, onde todos seriam 0s, tirando 3 would become a 10,000-dimensional vector that is all zeros except for indices 3 and 5, which are ones. Then, make this the first layer in our network—a Dense layer—that can handle floating point vector data. This approach is memory intensive, though, requiring a `num_words * num_reviews` size matrix.\n", - "\n", - "* Alternatively, we can pad the arrays so they all have the same length, then create an integer tensor of shape `max_length * num_reviews`. We can use an embedding layer capable of handling this shape as the first layer in our network.\n", - "\n", - "In this tutorial, we will use the second approach.\n", - "\n", - "Since the movie reviews must be the same length, we will use the [pad_sequences](https://keras.io/preprocessing/sequence/#pad_sequences) function to standardize the lengths:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} - }, - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "Let's look at the length of the examples now:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "And inspect the (now padded) first review:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## Construindo o modelo\n", - "\n", - "A rede neural é criada por camadas empilhadas —isso necessita duas decisões arquiteturais principais:\n", - "\n", - "* Quantas camadas serão usadas no modelo?\n", - "* Quantas *hidden units* são usadas em cada camada?\n", - "\n", - "Neste exemplo, os dados de entrada são um *array* de palavras-índices. As *labels* para predizer são ou 0 ou 1. Vamos construir um modelo para este problema:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "# O formato de entrada é a contagem vocabulário usados pelas avaliações dos filmes (10000 palavras)\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation=tf.nn.relu))\n", - "model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "As camadas são empilhadas sequencialmente para construir o classificador:\n", - "\n", - "1. A primeira camada é uma camada `Embedding` layer (*`Embedding` layer*). Essa camada pega o vocabulário em inteiros e olha o vetor *embedding* em cada palavra-index. Esses vetores são aprendidos pelo modelo, ao longo do treinamento. Os vetores adicionam a dimensão ao *array* de saída. As dimensões resultantes são: `(batch, sequence, embedding)`.\n", - "2. Depois, uma camada `GlobalAveragePooling1D` retorna um vetor de saída com comprimento fixo para cada exemplo fazendo a média da sequência da dimensão. Isso permite o modelo de lidar com entradas de tamanhos diferentes da maneira mais simples possível.\n", - "3. Esse vetor de saída com tamanho fixo passa por uma camada *fully-connected* (`Dense`) layer com 16 *hidden units*.\n", - "4. A última camada é uma *densely connected* com um único nó de saída. Usando uma função de ativação `sigmoid`, esse valor é um float que varia entre 0 e 1, representando a probabilidade, ou nível de confiança." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### Hidden units\n", - "\n", - "O modelo abaixo tem duas camadas intermediárias ou _\"hidden\"_ (hidden layers), entre a entrada e saída. O número de saídas (unidades— *units*—, nós ou neurônios) é a dimensão do espaço representacional para a camada. Em outras palavras, a quantidade de liberdade que a rede é permitida enquanto aprende uma representação interna.\n", - "\n", - "Se o modelo tem mais *hidden units* (um espaço representacional de maior dimensão), e/ou mais camadas, então a rede pode aprender representações mais complexas. Entretanto, isso faz com que a rede seja computacionamente mais custosa e pode levar o aprendizado de padrões não desejados— padrões que melhoram a performance com os dados de treinamento, mas não com os de teste. Isso se chama *overfitting*, e exploraremos mais tarde." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### Função Loss e otimizadores (optimizer)\n", - "\n", - "O modelo precisa de uma função *loss* e um otimizador (*optimizer*) para treinamento. Já que é um problema de classificação binário e o modelo tem com saída uma probabilidade (uma única camada com ativação sigmoide), usaremos a função loss `binary_crossentropy`.\n", - "\n", - "Essa não é a única escolha de função loss, você poderia escolher, no lugar, a `mean_squared_error`. Mas, geralmente, `binary_crossentropy` é melhor para tratar probabilidades— ela mede a \"distância\" entre as distribuições de probabilidade, ou, no nosso caso, sobre a distribuição real e as previsões.\n", - "\n", - "Mais tarde, quando explorarmos problemas de regressão (como, predizer preço de uma casa), veremos como usar outra função loss chamada *mean squared error*.\n", - "\n", - "Agora, configure o modelo para usar o *optimizer* a função loss:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['acc'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "### Crie um conjunto de validação\n", - "\n", - "Quando treinando. queremos checar a acurácia do modelo com os dados que ele nunca viu. Crie uma conjunto de *validação* tirando 10000 exemplos do conjunto de treinamento original. (Por que não usar o de teste agora? Nosso objetivo é desenvolver e melhorar (tunar) nosso modelo usando somente os dados de treinamento, depois usar o de teste uma única vez para avaliar a acurácia)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} - }, - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## Treine o modelo\n", - "\n", - "Treine o modelo em 40 *epochs* com *mini-batches* de 512 exemplos. Essas 40 iterações sobre todos os exemplos nos tensores `x_train` e `y_train`. Enquanto treina, monitore os valores do loss e da acurácia do modelo nos 10000 exemplos do conjunto de validação:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} - }, - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## Avalie o modelo\n", - "\n", - "E vamos ver como o modelo se saiu. Dois valores serão retornados. Loss (um número que representa o nosso erro, valores mais baixos são melhores), e acurácia." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "Está é uma aproximação ingênua que conseguiu uma acurácia de 87%. Com mais abordagens avançadas, o modelo deve chegar em 95%." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## Crie um gráfico de acurácia e loss por tempo\n", - "\n", - "`model.fit()` retorna um objeto `History` que contém um dicionário de tudo o que aconteceu durante o treinamento:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} - }, - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "Tem 4 entradas: uma para cada métrica monitorada durante a validação e treinamento. Podemos usá-las para plotar a comparação do loss de treinamento e validação, assim como a acurácia de treinamento e validação:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history_dict['acc']\n", - "val_acc = history_dict['val_acc']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# \"bo\" de \"blue dot\" ou \"ponto azul\"\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "# b de \"solid blue line\" \"linha azul\"\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} - }, - "source": [ - "plt.clf() # limpa a figura\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "No gráfico, os pontos representam o loss e acurácia de treinamento, e as linhas são o loss e a acurácia de validação.\n", - "\n", - "Note que o loss de treinamento *diminui* a cada *epoch* e a acurácia *aumenta*. Isso é esperado quando usado um gradient descent optimization—ele deve minimizar a quantidade desejada a cada iteração.\n", - "\n", - "Esse não é o caso do loss e da acurácia de validação— eles parecem ter um pico depois de 20 epochs. Isso é um exemplo de *overfitting*: o modelo desempenha melhor nos dados de treinamento do que quando usado com dados nunca vistos. Depois desse ponto, o modelo otimiza além da conta e aprende uma representação *especifica* para os dados de treinamento e não *generaliza* para os dados de teste.\n", - "\n", - "Para esse caso particular, podemos prevenir o *overfitting* simplesmente parando o treinamento após mais ou menos 20 epochs. Depois, você verá como fazer isso automaticamente com um *callback*." - ] - } - ] -} \ No newline at end of file diff --git a/site/pt/tutorials/keras/classification.ipynb b/site/pt/tutorials/keras/classification.ipynb deleted file mode 100644 index d7f9edcb4a6..00000000000 --- a/site/pt/tutorials/keras/classification.ipynb +++ /dev/null @@ -1,1022 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# Treine sua primeira rede neural: classificação básica" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Veja em TensorFlow.org\n", - " \n", - " Execute em Google Colab\n", - " \n", - " Veja código fonte em GitHub\n", - " \n", - " Baixe o notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "v10Paf9CKcHZ", - "colab_type": "text" - }, - "source": [ - "Note: A nossa comunidade TensorFlow traduziu estes documentos. Como as traduções da comunidade são *o melhor esforço*, não há garantias de que sejam uma reflexão exata e atualizada da [documentação oficial em Inglês](https://www.tensorflow.org/?hl=en). Se tem alguma sugestão para melhorar esta tradução, por favor envie um pull request para o repositório do GitHub [tensorflow/docs](https://github.com/tensorflow/docs). Para se voluntariar para escrever ou rever as traduções da comunidade, contacte a [lista docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "Este tutorial treina um modelo de rede neural para classificação de imagens de roupas, como tênis e camisetas. Tudo bem se você não entender todos os detalhes; este é um visão geral de um programa do TensorFlow com detalhes explicados enquanto progredimos.\n", - "\n", - "O guia usa [tf.keras](https://www.tensorflow.org/guide/keras), uma API alto-nível para construir e treinar modelos no TensorFlow." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jL3OqFKZ9dFg", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow e tf.keras\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# Librariesauxiliares\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## Importe a base de dados Fashion MNIST" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "Esse tutorial usa a base de dados [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) que contém 70,000 imagens em tons de cinza em 10 categorias. As imagens mostram artigos individuais de roupas com baixa resolução (28 por 28 pixels), como vemos aqui:\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Amostras de Fashion-MNIST (por Zalando, MIT License).
     \n", - "
    \n", - "\n", - "Fashion MNIST tem como intenção substituir a clássica base de dados [MNIST](http://yann.lecun.com/exdb/mnist/ )— frequentemente usada como \"Hello, World\" de programas de aprendizado de máquina (*machine learning*) para visão computacional. A base de dados MNIST contém imagens de dígitos escritos à mão (0, 1, 2, etc.) em um formato idêntico ao dos artigos de roupas que usaremos aqui.\n", - "\n", - "Esse tutorial usa a Fashion MNIST para variar, e porque é um problema um pouco mais desafiador que o regular MNIST. Ambas bases são relativamente pequenas e são usadas para verificar se um algoritmo funciona como esperado. Elas são bons pontos de partida para testar e debugar código.\n", - "\n", - "Usaremos 60,000 imagens para treinar nossa rede e 10,000 imagens para avaliar quão precisamente nossa rede aprendeu a classificar as imagens. Você pode acessar a Fashion MNIST directly diretamente do TensorFlow. Importe e carregue a base Fashion MNIST diretamente do TensorFlow:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "Carregando a base de dados que retorna quatro NumPy arrays:\n", - "\n", - "* Os *arrays* `train_images` e `train_labels` são o *conjunto de treinamento*— os dados do modelo usados para aprender.\n", - "* O modelo é testado com o *conjunto de teste*, os *arrays* `test_images` e `test_labels`.\n", - "\n", - "As imagens são arrays NumPy de 28x28, com os valores des pixels entre 0 to 255. As *labels* (alvo da classificação) são um array de inteiros, no intervalo de 0 a 9. Esse corresponde com a classe de roupa que cada imagem representa:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    LabelClasse
    0Camisetas/Top (T-shirt/top)
    1Calça (Trouser)
    2Suéter (Pullover)
    3Vestidos (Dress)
    4Casaco (Coat)
    5Sandálias (Sandal)
    6Camisas (Shirt)
    7Tênis (Sneaker)
    8Bolsa (Bag)
    9Botas (Ankle boot)
    \n", - "\n", - "Cada imagem é mapeada com um só label. Já que o *nome das classes* não são incluídas na base de dados, armazene os dados aqui para usá-los mais tarde quando plotarmos as imagens:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## Explore os dados\n", - "\n", - "Vamos explorar o formato da base de dados antes de treinar o modelo. O próximo comando mostra que existem 60000 imagens no conjunto de treinamento, e cada imagem é representada em 28 x 28 pixels:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "Do mesmo modo, existem 60000 labels no conjunto de treinamento:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "Cada label é um inteiro entre 0 e 9:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "Existem 10000 imagens no conjnto de teste. Novamente, cada imagem é representada por 28 x 28 pixels:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "E um conjunto de teste contendo 10000 labels das imagens :" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## Pré-processe os dados\n", - "\n", - "Os dados precisam ser pré-processados antes de treinar a rede. Se você inspecionar a primeira imagem do conjunto de treinamento, você verá que os valores dos pixels estão entre 0 e 255:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "Escalaremos esses valores no intervalo de 0 e 1 antes antes de alimentar o modelo da rede neural. Para fazer isso, dividimos os valores por 255. É importante que o *conjunto de treinamento* e o *conjunto de teste* podem ser pré-processados do mesmo modo:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "Para verificar que os dados estão no formato correto e que estamos prontos para construir e treinar a rede, vamos mostrar as primeiras 25 imagens do *conjunto de treinamento* e mostrar o nome das classes de cada imagem abaixo." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## Construindo o modelo\n", - "\n", - "Construir a rede neural requer configurar as camadas do modelo, e depois, compilar o modelo." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### Montar as camadas\n", - "\n", - "O principal bloco de construção da rede neural é a camada (*layer*). As camadas (*layers*) extraem representações dos dados inseridos na rede. Com sorte, essas representações são significativas para o problema à mão.\n", - "\n", - "Muito do *deep learning* consiste encadear simples camadas. Muitas camadas, como `tf.keras.layers.Dense`, tem paramêtros que são aprendidos durante o treinamento." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation='relu'),\n", - " keras.layers.Dense(10, activation='softmax')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "A primeira camada da rede, `tf.keras.layers.Flatten`, transforma o formato da imagem de um array de imagens de duas dimensões (of 28 by 28 pixels) para um array de uma dimensão (de 28 * 28 = 784 pixels). Pense nessa camada como camadas não empilhadas de pixels de uma imagem e os emfilere. Essa camada não tem paramêtros para aprender; ela só reformata os dados.\n", - "\n", - "Depois dos pixels serem achatados, a rede consite de uma sequência de duas camadas `tf.keras.layers.Dense`. Essa são camadas neurais *densely connected*, ou *fully connected*. A primeira camada `Dense` tem 128 nós (ou neurônios). A segunda (e última) camda é uma *softmax* de 10 nós que retorna um array de 10 probabilidades, cuja soma resulta em 1. Cada nó contem um valor que indica a probabilidade de que aquela imagem pertence a uma das 10 classes.\n", - "\n", - "### Compile o modelo\n", - "\n", - "Antes do modelo estar pronto para o treinamento, é necessário algumas configurações a mais. Essas serão adicionadas no passo de *compilação*:\n", - "\n", - "* *Função Loss* —Essa mede quão precisa o modelo é durante o treinamento. Queremos minimizar a função para *guiar* o modelo para direção certa.\n", - "* *Optimizer* —Isso é como o modelo se atualiza com base no dado que ele vê e sua função *loss*.\n", - "* *Métricas* —usadas para monitorar os passos de treinamento e teste. O exemplo abaixo usa a *acurácia*, a fração das imagens que foram classificadas corretamente." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## Treine o modelo\n", - "\n", - "Treinar a rede neural requer os seguintes passos:\n", - "\n", - "1. Alimente com os dados de treinamento, o modelo. Neste exemplo, os dados de treinamento são os arrays `train_images` e `train_labels`.\n", - "2. O modelo aprende como associar as imagens as *labels*.\n", - "3. Perguntamos ao modelo para fazer previsões sobre o conjunto de teste — nesse exemplo, o array `test_images`. Verificamos se as previsões combinaram com as *labels* do array `test_labels`.\n", - "\n", - "Para começar a treinar, chame o método `model.fit`— assim chamado, porque ele \"encaixa\" o modelo no conjunto de treinamento:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "À medida que o modelo treina, as métricas loss e acurácia são mostradas. O modelo atinge uma acurácia de 0.88 (ou 88%) com o conjunto de treinamento." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## Avalie a acurácia\n", - "\n", - "Depois, compare como o modelo performou com o conjunto de teste:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('\\nTest accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "Acabou que o a acurácia com o conjunto de teste é um pouco menor do que a acurácia de treinamento. Essa diferença entre as duas acurácias representa um *overfitting*. Overfitting é modelo de aprendizado de máquina performou de maneira pior em um conjunto de entradas novas, e não usadas anteriormente, que usando o conjunto de treinamento." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## Faça predições\n", - "\n", - "Com o modelo treinado, o usaremos para predições de algumas imagens." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "Aqui, o modelo previu que a *label* de cada imagem no conjunto de treinamento. Vamos olhar na primeira predição:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "A predição é um array de 10 números. Eles representam um a *confiança* do modelo que a imagem corresponde a cada um dos diferentes artigos de roupa. Podemos ver cada *label* tem um maior valor de confiança:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "Então, o modelo é confiante de que esse imagem é uma bota (ankle boot) ou `class_names[9]`. Examinando a label do teste, vemos que essa classificação é correta:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "Podemos mostrar graficamente como se parece em um conjunto total de previsão de 10 classes." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DvYmmrpIy6Y1", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d4Ov9OFDMmOD" - }, - "source": [ - "Vamos olhar a previsão imagem na posição 0, do array de predição." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HV5jw-5HwSmO", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ko-uzOufSCSe", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "Vamos plotar algumas da previsão do modelo. Labels preditas corretamente são azuis e as predições erradas são vermelhas. O número dá a porcentagem (de 100) das labels preditas. Note que o modelo pode errar mesmo estão confiante." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hQlnbqaw2Qu_", - "colab": {} - }, - "source": [ - "# Plota o primeiro X test images, e as labels preditas, e as labels verdadeiras.\n", - "# Colore as predições corretas de azul e as incorretas de vermelho.\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "Finamente, use o modelo treinado para fazer a predição de uma única imagem." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yRJ7JU7JCaXT", - "colab": {} - }, - "source": [ - "# Grab an image from the test dataset.\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "Modelos `tf.keras` são otimizados para fazer predições em um *batch*, ou coleções, de exemplos de uma vez. De acordo, mesmo que usemos uma única imagem, precisamos adicionar em uma lista:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# Adiciona a imagem em um batch que possui um só membro.\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "Agora prediremos a label correta para essa imagem:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ai-cpLjO-3A", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "`model.predict` retorna a lista de listas — uma lista para cada imagem em um *batch* de dados. Pegue a predição de nossa (única) imagem no *batch*:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "np.argmax(predictions_single[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "E, como antes, o modelo previu a label como 9." - ] - } - ] -} diff --git a/site/pt/tutorials/keras/regression.ipynb b/site/pt/tutorials/keras/regression.ipynb deleted file mode 100644 index 602de8a1283..00000000000 --- a/site/pt/tutorials/keras/regression.ipynb +++ /dev/null @@ -1,870 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# Regressão: Preveja consumo de combustível" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Veja em TensorFlow.org\n", - " \n", - " Execute em Google Colab\n", - " \n", - " Veja a fonte em GitHub\n", - " \n", - " Baixe o notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JzsKDVTh7UR4", - "colab_type": "text" - }, - "source": [ - "Note: A nossa comunidade TensorFlow traduziu estes documentos. Como as traduções da comunidade são *o melhor esforço*, não há garantias de que sejam uma reflexão exata e atualizada da [documentação oficial em Inglês](https://www.tensorflow.org/?hl=en). Se tem alguma sugestão para melhorar esta tradução, por favor envie um pull request para o repositório do GitHub [tensorflow/docs](https://github.com/tensorflow/docs). Para se voluntariar para escrever ou rever as traduções da comunidade, contacte a [lista docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "Em um problema de regressão, o objetivo é prever as saídas (*outputs*) de um valor contínuo, como um preço ou probabilidade. Em contraste de problemas de classificação, onde temos o próposito de escolher uma classe em uma lista de classificações (por exemplo, se uma imagem contem uma maçã ou laranja, assim reconhecendo qual fruta é representada na imagem).\n", - "\n", - "Este *notebook* usa a clássica base de dados [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) e constrói um modelo para prever a economia de combustiveis de automóveis do final dos anos 1970, inicio dos anos 1980. Para isso, forneceremos um modelo com descrição de vários automóveis desse período. Essa descrição inclui atributos como: cilindros, deslocamento, potência do motor, e peso.\n", - "\n", - "Este exemplo usa a API `tf.keras`. Veja [este guia](https://www.tensorflow.org/guide/keras) para mais detalhes." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# Use seaborn para pairplot\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Base de dados Auto MPG\n", - "\n", - "A base de dados está disponível em [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/).\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### Pegando os dados\n", - "Primeiro baixe a base de dados dos automóveis." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "Utilizando o pandas, impoorte os dados:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### Limpe os dados\n", - "\n", - "Esta base contém alguns valores não conhecidos (*unknown*)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "Para manter esse tutorial básico, remova as linhas com esses valores não conhecidos." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "A coluna \"Origin\" é uma coluna categórica e não numérica. Logo converta para *one-hot* :" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### Separando dados de treinamento e teste\n", - "\n", - "Agora separe os dados em um conjunto de treimento e outro teste.\n", - "\n", - "Iremos utilizar o de conjunto de teste no final da análise do modelo." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### Inspecione o dado\n", - "\n", - "Dê uma rápida olhada em como está a distribuição de algumas colunas do conjunto de treinamento." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "Repare na visão geral das estatísticas:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### Separe features de labels\n", - "\n", - "Separe o valor alvo (*labels*), das *features*. Esta label é o valor no qual o model é treinado para prever." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### Normalize os dados\n", - "\n", - "Olhe novamente para o bloco `train_stats` acima e note quão diferente é a variação de feature." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "Uma boa prática é normalizar as *features* que usam diferentes escalas e intervalos. Apesar do modelo poder convergir sem a normalização, isso torna o treinamento mais difícil, e torna o resultado do modelo dependente da escolha das unidades da entrada.\n", - "\n", - "Note: embora geramos intencionamente essas estatísticas para os dados de treinamento, essas estatísticas serão usadas também para normalizar o conjunto de teste. Precisamos delinear o conjunto de teste na mesma distribuição que o modelo foi treinado. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "O dado normalizado é o que nós usaremos para treinar o modelo.\n", - "\n", - "Caution: A estatística usada para normalizar as entradas aqui (média e desvio padrão) precisa ser aplicada em qualquer outro dado que alimenta o modelo, junto com o codificação one-hot que fizemos mais cedo. Isso inclui o conjunto de teste, assim como dados que o modelo usará em produção." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## O Modelo" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### Construindo o modelo\n", - "\n", - "Vamos construir o modelo. Aqui usaremos o modelo `Sequential` com duas camadas *densely connected*, e a camada de saída que retorna um único valor contínuo. Os passos de construção do modelo são agrupados em uma função, `build_model`, já que criaremos um segundo modelo mais tarde." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation='relu', input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation='relu'),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mse',\n", - " optimizer=optimizer,\n", - " metrics=['mae', 'mse'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### Examine o modelo\n", - "\n", - "Use o método `.summary` para exibir uma descrição simples do modelo." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "Agora teste o modelo. Pegue um batch de de 10 exemplos do conjunto de treinamento e chame `model.predict` nestes." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "Parece que está funcionando e ele produz o resultado de forma e tipo esperados." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### Treinando o modelo\n", - "\n", - "Treine o modelo com 1000 *epochs*, e grave a acurácia do treinamento e da validação em um objeto `history`. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# Mostra o progresso do treinamento imprimindo um único ponto para cada epoch completada\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "Visualize o progresso do modelo de treinamento usando o estados armazenados no objeto `history`" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mae'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mae'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mse'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mse'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "Este grafo mostra as pequenas melhoras, ou mesmo a diminuição do `validation error` após 100 *epochs*. Vamos atualizar o `model.fit` para que pare automaticamente o treinamento quando o `validation score` não aumentar mais. Usaremos o `EarlyStopping callback` que testa a condição do treinamento a cada `epoch`. Se um grupo de `epochs` decorre sem mostrar melhoras, o treinamento irá parar automaticamente.\n", - "\n", - "Você pode aprender mais sobre este callback [aqui](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# O paramêtro patience é o quantidade de epochs para checar as melhoras\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "O gráfico mostra que no conjunto de validação, a média de erro é próximo de +/- 2MPG. Isso é bom? Deixaremos essa decisão a você.\n", - "\n", - "Vamos ver quão bem o modelo generaliza usando o conjunto de **teste**, que não usamos para treinar o modelo. Isso diz quão bem podemos esperar que o modelo se saia quando usarmos na vida real." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "### Faça predições\n", - "\n", - "Finalmente, prevejamos os valores MPG usando o conjunto de teste." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19wyogbOSU5t" - }, - "source": [ - "Parece que o nosso modelo prediz razoavelmente bem. Vamos dar uma olhada na distribuição dos erros." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m0CB5tBjSU5w" - }, - "source": [ - "Não é tão gaussiana, porém podemos esperar que por conta do número de exemplo é bem pequeno." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## Conclusão\n", - "\n", - "Este notebook introduz algumas técnicas para trabalhar com problema de regressão.\n", - "\n", - "* Mean Sqaured Error(MSE), é uma função comum de *loss* usada para problemas de regressão (diferentes funções de *loss* são usadas para problemas de classificação).\n", - "* Similarmente, as métricas de evolução usadas na regressão são diferentes da classificação. Uma métrica comum de regressão é Mean Absolute Error (MAE).\n", - "* Quando o dado de entrada de *features* tem diferentes intervalos, cada *feature* deve ser escalada para o mesmo intervalo.\n", - "* Se não possuir muitos dados de treinamento, uma técnica é preferir uma pequena rede com poucas camadas para evitar *overfitting*.\n", - "* *Early stopping* é uma boa técnica para evitar *overfitting*." - ] - } - ] -} diff --git a/site/pt/tutorials/keras/text_classification.ipynb b/site/pt/tutorials/keras/text_classification.ipynb deleted file mode 100644 index 1149834302d..00000000000 --- a/site/pt/tutorials/keras/text_classification.ipynb +++ /dev/null @@ -1,731 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# Classificação de texto com avaliações de filmes" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Veja em TensorFlow.org\n", - " \n", - " Execute em Google Colab\n", - " \n", - " Veja fonte em GitHub\n", - " \n", - " Baixe o notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "2qJkqPKTnJR1", - "colab_type": "text" - }, - "source": [ - "Note: A nossa comunidade TensorFlow traduziu estes documentos. Como as traduções da comunidade são *o melhor esforço*, não há garantias de que sejam uma reflexão exata e atualizada da [documentação oficial em Inglês](https://www.tensorflow.org/?hl=en). Se tem alguma sugestão para melhorar esta tradução, por favor envie um pull request para o repositório do GitHub [tensorflow/docs](https://github.com/tensorflow/docs). Para se voluntariar para escrever ou rever as traduções da comunidade, contacte a [lista docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "Este *notebook* classifica avaliações de filmes como **positiva** ou **negativa** usando o texto da avaliação. Isto é um exemplo de classificação *binária* —ou duas-classes—, um importante e bastante aplicado tipo de problema de aprendizado de máquina.\n", - "\n", - "Usaremos a base de dados [IMDB](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) que contém avaliaçòes de mais de 50000 filmes do bando de dados [Internet Movie Database](https://www.imdb.com/). A base é dividida em 25000 avaliações para treinamento e 25000 para teste. Os conjuntos de treinamentos e testes são *balanceados*, ou seja, eles possuem a mesma quantidade de avaliações positivas e negativas.\n", - "\n", - "O notebook utiliza [tf.keras](https://www.tensorflow.org/guide/keras), uma API alto-nível para construir e treinar modelos com TensorFlow. Para mais tutoriais avançados de classificação de textos usando `tf.keras`, veja em [MLCC Text Classification Guide](https://developers.google.com/machine-learning/guides/text-classification/)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## Baixe a base de dados IMDB\n", - "\n", - "A base de dados vem empacotada com TensorFlow. Ela já vem pré-processada de forma que as avaliações (sequências de palavras) foram convertidas em sequências de inteiros, onde cada inteiro representa uma palavra específica no dicionário.\n", - "\n", - "O código abaixo baixa a base de dados IMDB para a sua máquina (ou usa a cópia em *cache*, caso já tenha baixado):\"" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "O argumento `num_words=10000` mantém as 10000 palavras mais frequentes no conjunto de treinamento. As palavras mais raras são descartadas para preservar o tamanho dos dados de forma maleável." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## Explore os dados\n", - "\n", - "Vamos parar um momento para entender o formato dos dados. O conjunto de dados vem pré-processado: cada exemplo é um *array* de inteiros representando as palavras da avaliação do filme. Cada *label* é um inteiro com valor ou de 0 ou 1, onde 0 é uma avaliação negativa e 1 é uma avaliação positiva." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} - }, - "source": [ - "print(\"Training entries: {}, labels: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "O texto das avaliações foi convertido para inteiros, onde cada inteiro representa uma palavra específica no dicionário. Isso é como se parece a primeira revisão:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "As avaliações dos filmes têm diferentes tamanhos. O código abaixo mostra o número de palavras da primeira e segunda avaliação. Sabendo que o número de entradas da rede neural tem que ser de mesmo também, temos que resolver isto mais tarde." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### Converta os inteiros de volta a palavras\n", - "\n", - "É util saber como converter inteiros de volta a texto. Aqui, criaremos uma função de ajuda para consultar um objeto *dictionary* que contenha inteiros mapeados em strings:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} - }, - "source": [ - "# Um dicionário mapeando palavras em índices inteiros\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# Os primeiros índices são reservados\n", - "word_index = {k:(v+3) for k,v in word_index.items()}\n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # unknown\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "Agora, podemos usar a função `decode_review` para mostrar o texto da primeira avaliação:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} - }, - "source": [ - "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## Prepare os dados\n", - "\n", - "As avaliações —os *arrays* de inteiros— devem ser convertidas em tensores (*tensors*) antes de alimentar a rede neural. Essa conversão pode ser feita de duas formas:\n", - "\n", - "* Converter os arrays em vetores de 0s e 1s indicando a ocorrência da palavra, similar com one-hot encoding. Por exemplo, a sequência [3, 5] se tornaria um vetor de 10000 dimensões, onde todos seriam 0s, tirando 3 e 5, que são 1s. Depois, faça disso a primeira camada da nossa rede neural — a Dense layer — que pode trabalhar com dados em ponto flutuante. Essa abordagem é intensa em relação a memória, logo requer uma matriz de tamanho `num_words * num_reviews`.\n", - "\n", - "* Alternativamente, podemos preencher o array para que todos tenho o mesmo comprimento, e depois criar um tensor inteiro de formato `max_length * num_reviews`. Podemos usar uma camada *embedding* capaz de lidar com o formato como a primeira camada da nossa rede.\n", - "\n", - "Nesse tutorial, usaremos a segunda abordagem.\n", - "\n", - "Já que as avaliações dos filmes devem ter o mesmo tamanho, usaremos a função [pad_sequences](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences) para padronizar os tamanhos:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} - }, - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "Agora, vamos olhar o tamanho dos exemplos:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "E inspecionar as primeiras avaliações (agora preenchidos):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## Construindo o modelo\n", - "\n", - "A rede neural é criada por camadas empilhadas —isso necessita duas decisões arquiteturais principais:\n", - "\n", - "* Quantas camadas serão usadas no modelo?\n", - "* Quantas *hidden units* são usadas em cada camada?\n", - "\n", - "Neste exemplo, os dados de entrada são um *array* de palavras-índices. As *labels* para predizer são ou 0 ou 1. Vamos construir um modelo para este problema:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "# O formato de entrada é a contagem vocabulário usados pelas avaliações dos filmes (10000 palavras)\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation='relu'))\n", - "model.add(keras.layers.Dense(1, activation='sigmoid'))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "As camadas são empilhadas sequencialmente para construir o classificador:\n", - "\n", - "1. A primeira camada é uma camada `Embedding` (*`Embedding` layer*). Essa camada pega o vocabulário em inteiros e olha o vetor *embedding* em cada palavra-index. Esses vetores são aprendidos pelo modelo, ao longo do treinamento. Os vetores adicionam a dimensão ao *array* de saída. As dimensões resultantes são: `(batch, sequence, embedding)`.\n", - "2. Depois, uma camada `GlobalAveragePooling1D` retorna um vetor de saída com comprimento fixo para cada exemplo fazendo a média da sequência da dimensão. Isso permite o modelo de lidar com entradas de tamanhos diferentes da maneira mais simples possível.\n", - "3. Esse vetor de saída com tamanho fixo passa por uma camada *fully-connected* (`Dense`) layer com 16 *hidden units*.\n", - "4. A última camada é uma *densely connected* com um único nó de saída. Usando uma função de ativação `sigmoid`, esse valor é um float que varia entre 0 e 1, representando a probabilidade, ou nível de confiança." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### Hidden units\n", - "\n", - "O modelo abaixo tem duas camadas intermediárias ou _\\\"hidden\\\"_ (hidden layers), entre a entrada e saída. O número de saídas (unidades— *units*—, nós ou neurônios) é a dimensão do espaço representacional para a camada. Em outras palavras, a quantidade de liberdade que a rede é permitida enquanto aprende uma representação interna.\n", - "\n", - "Se o modelo tem mais *hidden units* (um espaço representacional de maior dimensão), e/ou mais camadas, então a rede pode aprender representações mais complexas. Entretanto, isso faz com que a rede seja computacionalmente mais custosa e pode levar ao aprendizado de padrões não desejados— padrões que melhoram a performance com os dados de treinamento, mas não com os de teste. Isso se chama *overfitting*, e exploraremos mais tarde." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### Função Loss e otimizadores (optimizer)\n", - "\n", - "O modelo precisa de uma função *loss* e um otimizador (*optimizer*) para treinamento. Já que é um problema de classificação binário e o modelo tem como saída uma probabilidade (uma única camada com ativação sigmoide), usaremos a função loss `binary_crossentropy`.\n", - "\n", - "Essa não é a única escolha de função loss, você poderia escolher, no lugar, a `mean_squared_error`. Mas, geralmente, `binary_crossentropy` é melhor para tratar probabilidades— ela mede a \\\"distância\\\" entre as distribuições de probabilidade, ou, no nosso caso, sobre a distribuição real e as previsões.\n", - "\n", - "Mais tarde, quando explorarmos problemas de regressão (como, predizer preço de uma casa), veremos como usar outra função loss chamada *mean squared error*.\n", - "\n", - "Agora, configure o modelo para usar o *optimizer* a função loss:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "## Crie um conjunto de validação\n", - "\n", - "Quando treinando. queremos checar a acurácia do modelo com os dados que ele nunca viu. Crie uma conjunto de *validação* tirando 10000 exemplos do conjunto de treinamento original. (Por que não usar o de teste agora? Nosso objetivo é desenvolver e melhorar (tunar) nosso modelo usando somente os dados de treinamento, depois usar o de teste uma única vez para avaliar a acurácia)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} - }, - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## Treine o modelo\n", - "\n", - "Treine o modelo em 40 *epochs* com *mini-batches* de 512 exemplos. Essas 40 iterações sobre todos os exemplos nos tensores `x_train` e `y_train`. Enquanto treina, monitore os valores do loss e da acurácia do modelo nos 10000 exemplos do conjunto de validação:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} - }, - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## Avalie o modelo\n", - "\n", - "E vamos ver como o modelo se saiu. Dois valores serão retornados. Loss (um número que representa o nosso erro, valores mais baixos são melhores), e acurácia." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "Esta é uma abordagem ingênua que conseguiu uma acurácia de 87%. Com abordagens mais avançadas, o modelo deve chegar em 95%." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## Crie um gráfico de acurácia e loss por tempo\n", - "\n", - "`model.fit()` retorna um objeto `History` que contém um dicionário de tudo o que aconteceu durante o treinamento:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} - }, - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "Tem 4 entradas: uma para cada métrica monitorada durante a validação e treinamento. Podemos usá-las para plotar a comparação do loss de treinamento e validação, assim como a acurácia de treinamento e validação:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history_dict['accuracy']\n", - "val_acc = history_dict['val_accuracy']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# \"bo\" is for \"blue dot\"\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "# b is for \"solid blue line\"\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} - }, - "source": [ - "plt.clf() # clear figure\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "\n", - "No gráfico, os pontos representam o loss e acurácia de treinamento, e as linhas são o loss e a acurácia de validação.\n", - "\n", - "Note: que o loss de treinamento *diminui* a cada *epoch* e a acurácia *aumenta*. Isso é esperado quando usado um gradient descent optimization—ele deve minimizar a quantidade desejada a cada iteração.\n", - "\n", - "Esse não é o caso do loss e da acurácia de validação— eles parecem ter um pico depois de 20 epochs. Isso é um exemplo de *overfitting*: o modelo desempenha melhor nos dados de treinamento do que quando usado com dados nunca vistos. Depois desse ponto, o modelo otimiza além da conta e aprende uma representação *especifica* para os dados de treinamento e não *generaliza* para os dados de teste.\n", - "\n", - "Para esse caso particular, podemos prevenir o *overfitting* simplesmente parando o treinamento após mais ou menos 20 epochs. Depois, você verá como fazer isso automaticamente com um *callback*." - ] - } - ] -} diff --git a/site/ru/README.md b/site/ru/README.md deleted file mode 100644 index ed047f2492d..00000000000 --- a/site/ru/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Неофициальный перевод - -Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow -сообщества на общественных началах. Поскольку этот перевод не является -официальным, мы не гарантируем что он на 100% аккуратен и соответствует -[официальной документации на английском языке](https://www.tensorflow.org/?hl=en). -Если у вас есть предложение как исправить этот перевод, мы будем очень рады -увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) -репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow -лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), -напишите нам на -[docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru). - -# Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Please focus translation efforts on -[TensorFlow 2](https://www.tensorflow.org) in the -[site/en/r2](https://github.com/tensorflow/docs/tree/master/site/en/r2) -directory. TF 1.x community docs will no longer be updated as we prepare for the -2.0 release. See -[the announcement](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). - -# Как добавить свой перевод? - -1. Ознакомьтесь с [уже имеющейся документацией на русском языке](https://github.com/tensorflow/docs/tree/master/site/ru) -2. Внесите правки в уже готовый перевод или [выберите еще не переведенный в папке site/en/](https://github.com/tensorflow/docs/tree/master/site/en) -3. Переводите на русский язык блоки основные текста документации, а также поясняющие комментарии в коде - -Обратите внимание: документы, которые начинаются с символа нижнего подчеркивания (например, `_style_transfer.ipynb`) все еще находятся в процессе работы и пока не импортируются на [tensorflow.org](https://www.tensorflow.org/) - -Смотрите хороший пример переведенного нами [урока по TensorFlow Keras API здесь](https://github.com/tensorflow/docs/blob/master/site/ru/tutorials/keras/basic_classification.ipynb). - -# Советы - -1. Не стесняйтесь добавлять не полностью переведенные документации: таким образом другие участники смогут увидеть ваш Pull Request и помочь если требуется -2. Вы также можете [создать черновик (Draft)](https://help.github.com/en/articles/about-pull-requests#draft-pull-requests) для будущего Pull Request. Таким образом вы сможете добавлять изменения, не уведомляя владельца репозитария: это удобно для работы над крупными текстами -3. Используйте комментарии TODO или IMPROVE в тексте и коде в тех местах, где можно объяснить код или операцию более понятным языком, но вы пока не знаете как (это позволит другим переводчикам помочь вам с толкованием) -4. Проверьте основные моменты [Google Developer Documentation Style Guide](https://developers.google.com/style/highlights): это поможет вам с переводом и проверкой технической документации и уроков - -# Russian translation guide - -Some technical words in English do not have a natural translation. Do *not* -translate the following words: - -* (mini-) batch -* estimator -* eager execution -* dense -* dropout diff --git a/site/ru/REVIEWERS b/site/ru/REVIEWERS deleted file mode 100644 index 663775706c8..00000000000 --- a/site/ru/REVIEWERS +++ /dev/null @@ -1,9 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/ru directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs-ru@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru -# -0101011 -stabuev diff --git a/site/ru/guide/_index.yaml b/site/ru/guide/_index.yaml deleted file mode 100644 index 7f998dea9c2..00000000000 --- a/site/ru/guide/_index.yaml +++ /dev/null @@ -1,200 +0,0 @@ -book_path: /overview/_book.yaml -project_path: /overview/_project.yaml -description: -landing_page: - custom_css_path: /site-assets/css/style.css - nav: left - rows: - - classname: - devsite-landing-row-100 - devsite-landing-row-large-headings - items: - - description: > -

    - TensorFlow 2 фокусируется на простоте и удобстве использования, с такими обновлениями, - как eager execution, интуитивно понятные высокоуровневые API и гибкое построение - моделей на любой платформе. -

    -

    - Многие руководства написаны в виде ноутбуков Jupyter и запускаются непосредственно - в Google Colab - среде для ноутбуков, которая не требует установки. - Используйте в руководствах кнопку «Запустить в Google Colab». -

    - - - heading: Необходимая документация - classname: tfo-landing-page-heading - items: - - classname: tfo-landing-page-card - description: > -

    Установка TensorFlow

     - Установите пакет или соберите из исходников. - Поддержка GPU для карт с поддержкой CUDA®. - path: /install - - classname: tfo-landing-page-card - description: > -

    TensorFlow 2

     - Лучшие практики и инструменты TensorFlow 2 для переноса вашего кода. - path: /guide/effective_tf2 - - classname: tfo-landing-page-card - description: > -

    Keras

     - Keras это более легкий для ML новичков высокоуровневый API - подходящий и для исследователей. - path: /guide/keras/overview - - - items: - - classname: tfo-landing-page-card - description: > -

    Настройка

     - TensorFlow для исследований и экспериментов. Напишите пользовательские слои и модели, прямой проход и обучающие циклы. - path: /guide/eager - - classname: tfo-landing-page-card - description: > -

    Конвейеры ввода данных

     - API tf.data позволяют вам построить сложные конвейеры ввода - данных из простых переиспользуемых частей. - path: /guide/data - - classname: tfo-landing-page-card - description: > -

    Оценщики

     - Высокоуровневый API, представляющий законченную модель, спроектирован - для масштабирования и асинхронного обучения. - path: /guide/estimator - - - items: - - classname: tfo-landing-page-card - description: > -

    Соханение модели

     - Сохраняйте модели TensorFlow используя контрольные точки или формат SavedModel. - path: /guide/checkpoint - - classname: tfo-landing-page-card - description: > -

    Акселераторы

     - Распределите обучение между несколькими GPU, компьютерами или TPU. - path: /guide/distributed_training - - - classname: devsite-landing-row-100 - items: - - description: > -

    Библиотеки и расширения

    - Изучите дополнительные ресурсы для создания сложных - моделей или методов с использованием TensorFlow, получения доступа к специализированным - пакетам приложений, расширяющих функционал TensorFlow. - - - classname: devsite-landing-row-100 - items: - - list: - - description: > -

    TensorBoard

     - Набор инструментов визуализации для понимания, отладки и оптимизации - программ TensorFlow. - path: /tensorboard - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    TensorFlow Hub

     - Библиотека для публикации, изучения и использования переиспользуемых - частей моделей машинного обучения. - path: /hub - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    Model Optimization

     - TensorFlow Model Optimization Toolkit представляет собой набор инструментов - оптимизации моделей ML для использования в продакшне. - path: /model_optimization - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    TensorFlow Federated

     - Фреймворк для машинного обучения и других вычислений на децентрализованных - данных. - path: /federated - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    Нейронное структурированное обучение

     - Учебная парадигма для обучения нейронных сетей путем использования структурированных - сигналов в дополнение к входным параметрам. - path: /neural_structured_learning - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    TensorFlow Graphics

     - Библиотека функций компьютерной графики, начиная от камер, источников света и материалов до рендеров. - path: /graphics - icon: - icon_name: chevron_right - foreground: theme - background: grey - - list: - - description: > -

    Датасеты

     - Коллекция датасетов готовых к использованию с TensorFlow. - path: /datasets - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    Serving

     - Система обслуживания TFX для моделей машинного обучения, предназначенная для достижения высокой - производительности в продакшене. - path: /tfx/guide/serving - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    Probability

     - TensorFlow Probability это библиотека для вероятностных рассуждений и - статистического анализа. - path: /probability - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    MLIR

     - MLIR унифицирует инфраструктуру для высокопроизводительных моделей ML в - TensorFlow. - path: /mlir - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    XLA

     - Компилятор линейной алгебры для различных областей применения, ускоряющий - модели TensorFlow потенциально без изменений исходного кода. - path: /xla - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    SIG Addons

     - Дополнительная функциональность для TensorFlow, поддерживаемая SIG Addons. - path: https://github.com/tensorflow/addons - icon: - icon_name: chevron_right - foreground: theme - background: grey - - description: > -

    SIG IO

     - Расширения баз данных, потоковых передач и файловых систем поддерживаемых SIG IO. - path: https://github.com/tensorflow/io - icon: - icon_name: chevron_right - foreground: theme - background: grey diff --git a/site/ru/guide/effective_tf2.md b/site/ru/guide/effective_tf2.md deleted file mode 100644 index f3aec073532..00000000000 --- a/site/ru/guide/effective_tf2.md +++ /dev/null @@ -1,309 +0,0 @@ -# Эффективный TensorFlow 2.0 - -В TensorFlow 2.0 сделан ряд изменений делающих пользователей TensorFlow более -продуктивными. TensorFlow 2.0 удалил -[избыточные API](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md), -после чего API стали более согласованными -([Unified RNNs](https://github.com/tensorflow/community/blob/master/rfcs/20180920-unify-rnn-interface.md), -[Unified Optimizers](https://github.com/tensorflow/community/blob/master/rfcs/20181016-optimizer-unification.md)), -и лучше интегрировался с Python runtime, с -[Eager execution](https://www.tensorflow.org/guide/eager). - -Многие -[RFCs](https://github.com/tensorflow/community/pulls?utf8=%E2%9C%93&q=is%3Apr) -объяснили изменения которые вошли в TensorFlow 2.0. Это руководство -представляет взгляд на кто как должна выглядеть разработка в TensorFlow 2.0. -Предполагается, что вы знакомы с TensorFlow 1.x. - -## Короткая выдержка основных изменений - -### Очистка API - -Много API либо -[удалены либо перемещены](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md) -в TF 2.0. Некоторые из основных изменений включают удаление `tf.app`, -`tf.flags`, и `tf.logging` в пользу -[absl-py](https://github.com/abseil/abseil-py) который сейчас с открытым -исходным кодом, перенос проектов которые находились в `tf.contrib`, и очистки -основного пространства имен `tf.*` путем перемещения редко используемых функций -в подпакеты наподобие `tf.math`. Неокторые API были замещены своими -эквивалентами 2.0 - `tf.summary`, `tf.keras.metrics`, и `tf.keras.optimizers`. -Наиболее простым способом автоматически применить эти переименования является -использование [скрипта обновления v2](upgrade.md). - -### Eager execution - -В TensorFlow 1.X от пользователей требовалось вручную собирать -[абстрактное синтаксическое дерево](https://ru.wikipedia.org/wiki/%D0%90%D0%B1%D1%81%D1%82%D1%80%D0%B0%D0%BA%D1%82%D0%BD%D0%BE%D0%B5_%D1%81%D0%B8%D0%BD%D1%82%D0%B0%D0%BA%D1%81%D0%B8%D1%87%D0%B5%D1%81%D0%BA%D0%BE%D0%B5_%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE) -(граф) выполняя `tf.*` API запросы. Затем пользователи должны вручную -скомпилировать абстрактное синтаксическое дерево путем передачи множества -выходных и входных тензоров в вызов `session.run()`. TensorFlow 2.0 выполняется -сразу же (как это обычно делает Python) и в 2.0, графы и сессии должны -рассматриваться как детали реализации. - -Одним заметным побочным продуктом eager execution является то, что -`tf.control_dependencies()` более не требуется, так как все строки кода -выполняются по очереди (в пределах `tf.function`, код с побочными эффектами -выполняется в том порядке в котором он написан). - -### Нет больше глобалов - -TensorFlow 1.X значительно зависел от неявных глобальных пространств имен. Когда -вы вызывали `tf.Variable()`, она помещалась в граф по умолчанию, и она -оставалась там даже если вы потеряли track переменной Python указывавшей на -него. Вы можете затем восстановить ту `tf.Variable`, но только если вы знали имя -с которым она была создана. Это было сложно сделать если вы не контролировали -создание переменных. В результате этого, размножались все виды механизмов -пытавшиеся помочь пользователям снова найти их переменные, а для фреймворков - -найти созданные пользователями переменные: Области переменных, глобальные -коллекции, методы помощники такие как `tf.get_global_step()`, -`tf.global_variables_initializer()`, оптимизаторы неявно вычисляющие градиенты -по всем обучаемым переменным, и т.д. TensorFlow 2.0 устраняет все эти механизмы -([Variables 2.0 RFC](https://github.com/tensorflow/community/pull/11)) в пользу -механизма по умолчанию: Отслеживайте свои переменные! Если вы потеряли след -`tf.Variable`, он будет очищен сборщиком мусора. - -Требование отслеживать мусор создает дополнительную работу для пользователя, -но с объектами Keras (см. ниже), нагрузка минимизирована. - -### Функции, не сессии - -Вызов `session.run()` почти похож на вызов функции: Вы определяете вводные -данные, функция вызывается и вы получаете набор результатов. В TensorFlow 2.0, -вы можете декорировать функцию Python используя `tf.function()` чтобы отметить -ее для JIT компиляции так что TensorFlow выполняет его как единый граф -([Functions 2.0 RFC](https://github.com/tensorflow/community/pull/20)). Этот -механизм позволяет TensorFlow 2.0 получить все преимущества режима графа: - -- Производительность: функция может быть оптимизирована (node pruning, kernel - fusion, etc.) -- Портативность: функция может быть экспортирована / реимпортирована - ([RFC SavedModel 2.0](https://github.com/tensorflow/community/pull/34), что - позволяет пользователям повторно использовать и делиться модульными - функциями TensorFlow. - -```python -# TensorFlow 1.X -outputs = session.run(f(placeholder), feed_dict={placeholder: input}) -# TensorFlow 2.0 -outputs = f(input) -``` - -Благодаря возможности свободно перемежать код Python и TensorFlow пользователи -могут воспользоваться преимуществами выразительности Python. Но переносимый -TensorFlow выполняется в контекстах, таких как mobile, C ++ и JavaScript без -интерпретатора Python. Чтобы пользователям не нужно было переписывать свой код -при добавлении `@ tf.function`, [AutoGraph](function.ipynb) преобразует -подмножество Python конструируя его в эквивалентах TensorFlow: - -* `for`/`while` -> `tf.while_loop` (`break` and `continue` are supported) -* `if` -> `tf.cond` -* `for _ in dataset` -> `dataset.reduce` - -AutoGraph поддерживает произвольные вложения control flow, что делает возможным -эффективно и кратко реализовать многие сложные программы машинного обучения, -такие как реккурентные модели, обучение с подкреплением, пользовательские циклы -обучения и многое другое. - -## Рекомендации характерные для TensorFlow 2.0 - -### Рефакторьте ваш код в меньшие функции - -Обычный пользовательский паттерн в TensorFlow 1.X была стратегия "kitchen -sink"(кухонная мойка), где предварительно выкладывалось объединение всех -возможных вычислений, а потом выбранные тензоры оценивались с `session.run()`. В -TensorFlow 2.0, пользователям необходимо отрефакторить свой код в меньшие -функции которые вызываются по мере необходимости. В общем, не обязательно -декорировать каждую из этих функций с `tf.function`; используйте `tf.function` -только для декорирования высокоуровневых вычислений - например, один шаг -обучения или проход вперед в вашей модели. - -### Используйте слои и модели Keras для управления переменными - - -Keras models and layers offer the convenient `variables` and -`trainable_variables` properties, which recursively gather up all dependent -variables. This makes it easy to manage variables locally to where they are -being used. - -Модели и слои Keras предлагают удобные свойства `variables` и -`trainable_variables`, которые рекурсивно собирают все зависимые переменные. Это -облегчает локальное управление переменными в том месте, где они использовались. - -Сравните: - -```python -def dense(x, W, b): - return tf.nn.sigmoid(tf.matmul(x, W) + b) - -@tf.function -def multilayer_perceptron(x, w0, b0, w1, b1, w2, b2 ...): - x = dense(x, w0, b0) - x = dense(x, w1, b1) - x = dense(x, w2, b2) - ... - -# Вам необходимо управлять w_i and b_i, а их размерности определены далеко от кода. -``` - -с версией Keras: - -```python -# Каждый слой может быть вызван с сигнатурой эквивалентной linear(x) -layers = [tf.keras.layers.Dense(hidden_size, activation=tf.nn.sigmoid) for _ in range(n)] -perceptron = tf.keras.Sequential(layers) - -# layers[3].trainable_variables => returns [w3, b3] -# perceptron.trainable_variables => returns [w0, b0, ...] -``` - -Слои/модели Keras наследуются от `tf.train.Checkpointable` и интегрированы -с `@ tf.function`, что позволяет напрямую проверять или экспортировать -SavedModels из объектов Keras. Вам не обязательно использовать Keras -`.fit ()` API чтобы воспользоваться этими интеграциями. - -Вот пример transfer learning, который демонстрирует, как Keras облегчает -сбор подмножества релевантных переменных. Допустим, вы обучаете multi-headed -model with a shared trunk: - -```python -trunk = tf.keras.Sequential([...]) -head1 = tf.keras.Sequential([...]) -head2 = tf.keras.Sequential([...]) - -path1 = tf.keras.Sequential([trunk, head1]) -path2 = tf.keras.Sequential([trunk, head2]) - -# Обучение на первичных данных -for x, y in main_dataset: - with tf.GradientTape() as tape: - prediction = path1(x) - loss = loss_fn_head1(prediction, y) - # Одновременная оптимизация весов trunk и head1. - gradients = tape.gradient(loss, path1.trainable_variables) - optimizer.apply_gradients(zip(gradients, path1.trainable_variables)) - -# Тонкая настройка второй head, переиспользование trunk -for x, y in small_dataset: - with tf.GradientTape() as tape: - prediction = path2(x) - loss = loss_fn_head2(prediction, y) - # Оптимизируются только веса head2, не веса trunk - gradients = tape.gradient(loss, head2.trainable_variables) - optimizer.apply_gradients(zip(gradients, head2.trainable_variables)) - -# Вы можете опубликовать только вычисления trunk чтобы другие люди могли ими воспользоваться. -tf.saved_model.save(trunk, output_path) -``` - -### Комбинируйте tf.data.Datasets и @tf.function - -При итерации по тренировочным данным, которые помещаются в память, свободно -используйте регулярную итерацию Python. Иначе, `tf.data.Dataset` - лучший способ -для передачи тренировочных данных с диска. Данные являются -[iterables (не iterators)](https://docs.python.org/3/glossary.html#term-iterable), -и работают так же, как и другие Python iterables в режиме Eager. Вы можете -полностью использовать свойства dataset async prefetching/streaming упаковав -свой код в `tf.function ()`, которая заменяет итерацию Python эквивалентным -графом операции использующим AutoGraph. - -```python -@tf.function -def train(model, dataset, optimizer): - for x, y in dataset: - with tf.GradientTape() as tape: - prediction = model(x) - loss = loss_fn(prediction, y) - gradients = tape.gradient(loss, model.trainable_variables) - optimizer.apply_gradients(zip(gradients, model.trainable_variables)) -``` - -If you use the Keras `.fit()` API, you won't have to worry about dataset -iteration. - -```python -model.compile(optimizer=optimizer, loss=loss_fn) -model.fit(dataset) -``` - -### Воспользуйтесь преимуществами AutoGraph с Python control flow - -AutoGraph предоставляет способ преобразования зависящего от данных control flow -в эквивалентый режим графа, например `tf.cond` и `tf.while_loop`. - -Одно обычное место, где появляется зависящий от данных control flow находится -sequence models. `tf.keras.layers.RNN` оборачивает ячейку RNN, позволяя вам -статически или динамически развернуть recurrence. Например, вы может -переопределить динамическую развертку следующим образом: - -```python -class DynamicRNN(tf.keras.Model): - - def __init__(self, rnn_cell): - super(DynamicRNN, self).__init__(self) - self.cell = rnn_cell - - def call(self, input_data): - # [batch, time, features] -> [time, batch, features] - input_data = tf.transpose(input_data, [1, 0, 2]) - outputs = tf.TensorArray(tf.float32, input_data.shape[0]) - state = self.cell.zero_state(input_data.shape[1], dtype=tf.float32) - for i in tf.range(input_data.shape[0]): - output, state = self.cell(input_data[i], state) - outputs = outputs.write(i, output) - return tf.transpose(outputs.stack(), [1, 0, 2]), state -``` - -Для более подробного обзора свойств AutoGraph, смотри -[руководство](./function.ipynb). - -### tf.metrics аггрегирует данные and tf.summary ведет их лог - -Для лога summaries используйте `tf.summary. (Scalar | histogram | ...)` и -перенаправьте его на writer используя context manager. (Если вы опустите context -manager, ничего случится.) В отличие от TF 1.x, summaries отправляются -непосредственно writer; там нет отдельной операции "merge" и отдельного вызова -`add_summary()`, что означает, что значение `step` должно быть указано на месте -вызова. - -```python -summary_writer = tf.summary.create_file_writer('/tmp/summaries') -with summary_writer.as_default(): - tf.summary.scalar('loss', 0.1, step=42) -``` - -Чтобы объединить данные перед их записью в виде summaries, используйте -`tf.metrics`. Метрика являются stateful: они накапливают значения и возвращают -совокупный результат, когда вы вызовите `.result()`. Очистите накопленные -значения с помощью `.reset_states ()`. - -```python -def train(model, optimizer, dataset, log_freq=10): - avg_loss = tf.keras.metrics.Mean(name='loss', dtype=tf.float32) - for images, labels in dataset: - loss = train_step(model, optimizer, images, labels) - avg_loss.update_state(loss) - if tf.equal(optimizer.iterations % log_freq, 0): - tf.summary.scalar('loss', avg_loss.result(), step=optimizer.iterations) - avg_loss.reset_states() - -def test(model, test_x, test_y, step_num): - loss = loss_fn(model(test_x), test_y) - tf.summary.scalar('loss', loss, step=step_num) - -train_summary_writer = tf.summary.create_file_writer('/tmp/summaries/train') -test_summary_writer = tf.summary.create_file_writer('/tmp/summaries/test') - -with train_summary_writer.as_default(): - train(model, optimizer, dataset) - -with test_summary_writer.as_default(): - test(model, test_x, test_y, optimizer.iterations) -``` - -Визуализируйте сгенерированные результаты направив TensorBoard в директорий с -summary log: - -``` -tensorboard --logdir /tmp/summaries -``` diff --git a/site/ru/guide/migrate.ipynb b/site/ru/guide/migrate.ipynb deleted file mode 100644 index 78d2017679a..00000000000 --- a/site/ru/guide/migrate.ipynb +++ /dev/null @@ -1,1637 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wJcYs_ERTnnI" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "HMUDt0CiUJk9" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "77z2OchJTk0l" - }, - "source": [ - "# Конвертируйте ваш существующий код в TensorFlow 2.0\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " Смотрите на TensorFlow.org\n", - " \n", - " \n", - " \n", - " Запустите в Google Colab\n", - " \n", - " \n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fj66ZXAzrJC2", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C0V10enS1_WU" - }, - "source": [ - "В TensorFlow 2.0 все еще возможно исполнить 1.X код без изменений (за исключением contrib):\n", - "\n", - "```\n", - "import tensorflow.compat.v1 as tf\n", - "tf.disable_v2_behavior()\n", - "```\n", - "\n", - "Однако, это не дает вам воспользоваться преимуществами многих улучшений сделанных в TensorFlow 2.0. Это руководство поможет вам обновить ваш код, сделав его проще, производительнее и легче в поддержке.\n", - "\n", - "## Скрипт автоматической конвертации\n", - "\n", - "Первым шагом вы можете попробовать запустить [скрипт обновления](./upgrade.md).\n", - "\n", - "Он выполнит начальный этап обновления вашего кода до TensorFlow 2.0. Но это не может сделать ваш код идиоматичным TensorFlowF 2.0. Ваш код все еще может использовать `tf.compat.v1` для доступа к плейсхолдерам, сессиям, коллекциям, и другой функциональности в стиле 1.x.\n", - "\n", - "## Сделайте код 2.0-нативным\n", - "\n", - "\n", - "В этом руководстве рассматриваются несколько примеров преобразования кода TensorFlow 1.x в TensorFlow 2.0. Эти изменения позволят вашему коду воспользоваться преимуществами оптимизации производительности и упрощенных вызовов API.\n", - "\n", - "В каждом случае паттерн следующий:" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uP0O8Pc45LNs" - }, - "source": [ - "### 1. Заменить вызовы `tf.Session.run`\n", - "\n", - "Каждый вызов `tf.Session.run` нужо заменить функцией Python.\n", - "\n", - "* `feed_dict` и `tf.placeholder`s становятся аргументами функции.\n", - "* `fetches` становится возвращаемым значением функции.\n", - "\n", - "Вы можете пройти пошагово и отладить функцию, используя стандартные инструменты Python, такие как `pdb`.\n", - "\n", - "Когда вы убедитесь, что функция работает, добавьте декоратор`tf.function` чтобы она работала эффективно в режиме графа. Смотри [Руководство Autograph](function.ipynb) чтобы узнать больше о том, как это работает." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jlBOqROL5NmN" - }, - "source": [ - "### 2. Используйте объекты python для отслеживания переменных и значений потерь\n", - "\n", - "Используйте `tf.Variable` вместо `tf.get_variable`.\n", - "\n", - "Каждый `variable_scope` может быть сконвертирован в объект Python. Как правило это будет что-то из:\n", - "\n", - "* `tf.keras.layers.Layer`\n", - "* `tf.keras.Model`\n", - "* `tf.Module`\n", - "\n", - "Если вам нужны свести списки переменных (как например `tf.Graph.get_collection(tf.GraphKeys.VARIABLES)`), используйте аттрибуты `.variables` и `.trainable_variables` объектов `Layer` и `Model`.\n", - "\n", - "Эти классы `Layer` и `Model` реализуют несколько других свойств которые устраняют необходимость глобальных коллекций.\n", - "\n", - "Смотри [руководства keras](keras.ipynb) для подробностей.\n", - "\n", - "Предупреждение: Многие символы `tf.compat.v1` неявно используют глобальные коллекции.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rGFhBzoF5FIq" - }, - "source": [ - "### 3. Обновите ваши обучающие циклы\n", - "\n", - "Используйте API наиболее высокого уровня который работает в вашем случае. Предпочтите `tf.keras.Model.fit` построению своего собственного обучающего цикла.\n", - "\n", - "Эти высокоуровневые функции управляют большим количеством низкоуровневых деталей которые могут быть легко упущены если вы пишете собственный обучающий цикл. Например, они автоматически собирают потери регуляризации и устанавливают аргумент `training = True` при вызове модели.\n", - "\n", - "### 4. Обновите ваши конвейеры ввода данных\n", - "\n", - "Используйте наборы данных `tf.data` для входных данных. Эти объекты эффективны, выразительны и хорошо интегрированы с tensorflow.\n", - "\n", - "Их можно передать напрямую в метод `tf.keras.Model.fit`.\n", - "\n", - "```\n", - "model.fit(dataset, epochs=5)\n", - "```\n", - "\n", - "Их можно напрямую итерировать в стандартном Python:\n", - "\n", - "```\n", - "for example_batch, label_batch in dataset:\n", - " break\n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "X_ilfTGJ4Yml" - }, - "source": [ - "## Конвертация моделей\n", - "\n", - "### Установка" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bad2N-Z115W1" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " import tensorflow.compat.v2 as tf\n", - "except Exception:\n", - " pass\n", - "tf.enable_v2_behavior()\n", - "\n", - "\n", - "import tensorflow_datasets as tfds" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FB99sqHX2Q5m" - }, - "source": [ - "### Низкоуровневые переменные и исполнение оператора\n", - "\n", - "Примеры использования низкоуровневого API включают:\n", - "\n", - "* использование областей видимости переменных для управления повторным использованием\n", - "* создание переменных с `tf.get_variable`.\n", - "* явный доступ к коллекциям\n", - "* неявный доступ к коллекциям с такими методами, как:\n", - "\n", - " * `tf.global_variables`\n", - " * `tf.losses.get_regularization_loss`\n", - "\n", - "* использование `tf.placeholder` для установления входных данных графа\n", - "* выполнение графа с `session.run`\n", - "* ручная инициализация переменных\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e582IjyF2eje" - }, - "source": [ - "#### Перед конвертацией\n", - "\n", - "Здесь как могут выглядеть эти паттерны в коде использующем TensorFlow 1.x.\n", - "\n", - "```python\n", - "in_a = tf.placeholder(dtype=tf.float32, shape=(2))\n", - "in_b = tf.placeholder(dtype=tf.float32, shape=(2))\n", - "\n", - "def forward(x):\n", - " with tf.variable_scope(\"matmul\", reuse=tf.AUTO_REUSE):\n", - " W = tf.get_variable(\"W\", initializer=tf.ones(shape=(2,2)),\n", - " regularizer=tf.contrib.layers.l2_regularizer(0.04))\n", - " b = tf.get_variable(\"b\", initializer=tf.zeros(shape=(2)))\n", - " return W * x + b\n", - "\n", - "out_a = forward(in_a)\n", - "out_b = forward(in_b)\n", - "\n", - "reg_loss = tf.losses.get_regularization_loss(scope=\"matmul\")\n", - "\n", - "with tf.Session() as sess:\n", - " sess.run(tf.global_variables_initializer())\n", - " outs = sess.run([out_a, out_b, reg_loss],\n", - " \t feed_dict={in_a: [1, 0], in_b: [0, 1]})\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QARwz4Xd2lc2" - }, - "source": [ - "#### После конвертации" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x0AVzBFRBPcU" - }, - "source": [ - "В сконвертированном коде:\n", - "\n", - "* Переменные являются локальными объектами Python.\n", - "* Функция `forward` все еще определяет вычисления.\n", - "* Вызов `sess.run` заменен вызовом `forward`\n", - "* Опциональный декоратор `tf.function` может быть добавлен для производительности.\n", - "* Регуляризации вычисляются вручную без ссылок на глобальные коллекции.\n", - "* **Нет сессий и плейсхолдеров.**" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lXEZoLMP2cWJ" - }, - "outputs": [], - "source": [ - "W = tf.Variable(tf.ones(shape=(2,2)), name=\"W\")\n", - "b = tf.Variable(tf.zeros(shape=(2)), name=\"b\")\n", - "\n", - "@tf.function\n", - "def forward(x):\n", - " return W * x + b\n", - "\n", - "out_a = forward([1,0])\n", - "print(out_a)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YmE96A_1jZTg" - }, - "outputs": [], - "source": [ - "out_b = forward([0,1])\n", - "\n", - "regularizer = tf.keras.regularizers.l2(0.04)\n", - "reg_loss = regularizer(W)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ycDxY9nL268-" - }, - "source": [ - "### Модели основанные на `tf.layers`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K-bIk7wL48U7" - }, - "source": [ - "Модуль `tf.layers` используется для содержания layer-функций использующих `tf.variable_scope` для определения и переиспользования переменных." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8I_qKpT73KyM" - }, - "source": [ - "#### До конвертации\n", - "```python\n", - "def model(x, training, scope='model'):\n", - " with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):\n", - " x = tf.layers.conv2d(x, 32, 3, activation=tf.nn.relu,\n", - " kernel_regularizer=tf.contrib.layers.l2_regularizer(0.04))\n", - " x = tf.layers.max_pooling2d(x, (2, 2), 1)\n", - " x = tf.layers.flatten(x)\n", - " x = tf.layers.dropout(x, 0.1, training=training)\n", - " x = tf.layers.dense(x, 64, activation=tf.nn.relu)\n", - " x = tf.layers.batch_normalization(x, training=training)\n", - " x = tf.layers.dense(x, 10, activation=tf.nn.softmax)\n", - " return x\n", - "\n", - "train_out = model(train_data, training=True)\n", - "test_out = model(test_data, training=False)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b8_Ii7CQ3fK-" - }, - "source": [ - "#### После конвертации" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BsAseSMfB9XN" - }, - "source": [ - "* Простой стек слоев аккуратно встраивается в `tf.keras.Sequential`. (Для более сложных моделей см. [пользовательские слои и модели](keras/custom_layers_and_models.ipynb), и [функциональный API](keras/functional.ipynb).)\n", - "* Модель отслеживает переменные и потери регуляризации.\n", - "* Преобразование взаимно-однозначно поскольку существует прямое отображение из `tf.layers` в `tf.keras.layers`.\n", - "\n", - "Большинство аргументов остались прежними. Но обратите внимание на различия:\n", - "\n", - "* Аргумент `training` передается моделью каждому слою при его запуске.\n", - "* Первого аргумента исходной функции `model` (вводный `x`) больше нет. Это связано с тем, что слои объекта отделяют построение модели от вызова модели.\n", - "\n", - "\n", - "Также заметьте что:\n", - "\n", - "* Если вы использовали регуляризаторы инициализаторов из `tf.contrib`, у них больше изменений аргументов чем у остальных.\n", - "* Код больше не записывает в коллекции, так что функции наподобие `tf.losses.get_regularization_loss` больше не возращают эти значения, что может нарушить ваши циклы обучения." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DLAPORrN3lct" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " kernel_regularizer=tf.keras.regularizers.l2(0.04),\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dropout(0.1),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "train_data = tf.ones(shape=(1, 28, 28, 1))\n", - "test_data = tf.ones(shape=(1, 28, 28, 1))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6nWh6IXvkMKv" - }, - "outputs": [], - "source": [ - "train_out = model(train_data, training=True)\n", - "print(train_out)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YnAdIDLlj3go" - }, - "outputs": [], - "source": [ - "test_out = model(test_data, training=False)\n", - "print(test_out)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "sAgqwCJBMx_x" - }, - "outputs": [], - "source": [ - "# Здесь все обучаемые переменные.\n", - "len(model.trainable_variables)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uX6knaYMNM8p" - }, - "outputs": [], - "source": [ - "# Здесь потери регуляризации.\n", - "model.losses" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9moqw5E_4Cwl" - }, - "source": [ - "### Смесь переменных и tf.layers\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "80DEsImmq6VX" - }, - "source": [ - "Существующий код часто смешивает низкоуровневые TF 1.x переменные и операции с высокоуровневыми `tf.layers`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oZe9L6RR4OcP" - }, - "source": [ - "#### До конвертации\n", - "```python\n", - "def model(x, training, scope='model'):\n", - " with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):\n", - " W = tf.get_variable(\n", - " \"W\", dtype=tf.float32,\n", - " initializer=tf.ones(shape=x.shape),\n", - " regularizer=tf.contrib.layers.l2_regularizer(0.04),\n", - " trainable=True)\n", - " if training:\n", - " x = x + W\n", - " else:\n", - " x = x + W * 0.5\n", - " x = tf.layers.conv2d(x, 32, 3, activation=tf.nn.relu)\n", - " x = tf.layers.max_pooling2d(x, (2, 2), 1)\n", - " x = tf.layers.flatten(x)\n", - " return x\n", - "\n", - "train_out = model(train_data, training=True)\n", - "test_out = model(test_data, training=False)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y6ORX7cD4TkD" - }, - "source": [ - "#### После конвертации" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2BaRwog5CBpz" - }, - "source": [ - "Для конвертации этого кода следуйте паттерну отображения слоев в слои как и в предыдущем примере.\n", - "\n", - "`tf.variable_scope` фактически является слоем сам по себе. Поэтому перепишите его как `tf.keras.layers.Layer`. См. [руководство](keras/custom_layers_and_models.ipynb) для подробностей.\n", - "\n", - "В общем паттерн следующий:\n", - "\n", - "* Собрать параметры слоев в `__init__`.\n", - "* Создать переменные в `build`.\n", - "* Выполнить вычисления в `call` и вернуть результат.\n", - "\n", - "`tf.variable_scope` по сути является собственным слоем. Поэтому перепишите его как `tf.keras.layers.Layer`. Смотрите [руководство](keras/custom_layers_and_models.ipynb) для деталей." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YcCAjNuP4NVh" - }, - "outputs": [], - "source": [ - "# Создайте пользовательский слой для части модели\n", - "class CustomLayer(tf.keras.layers.Layer):\n", - " def __init__(self, *args, **kwargs):\n", - " super(CustomLayer, self).__init__(*args, **kwargs)\n", - "\n", - " def build(self, input_shape):\n", - " self.w = self.add_weight(\n", - " shape=input_shape[1:],\n", - " dtype=tf.float32,\n", - " initializer=tf.keras.initializers.ones(),\n", - " regularizer=tf.keras.regularizers.l2(0.02),\n", - " trainable=True)\n", - "\n", - " # Метод call будет иногда использоваться в режиме графа,\n", - " # training превратится в тензор\n", - " @tf.function\n", - " def call(self, inputs, training=None):\n", - " if training:\n", - " return inputs + self.w\n", - " else:\n", - " return inputs + self.w * 0.5" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dR_QO6_wBgMm" - }, - "outputs": [], - "source": [ - "custom_layer = CustomLayer()\n", - "print(custom_layer([1]).numpy())\n", - "print(custom_layer([1], training=True).numpy())" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VzqaIf4E42oY" - }, - "outputs": [], - "source": [ - "train_data = tf.ones(shape=(1, 28, 28, 1))\n", - "test_data = tf.ones(shape=(1, 28, 28, 1))\n", - "\n", - "# Build the model including the custom layer\n", - "model = tf.keras.Sequential([\n", - " CustomLayer(input_shape=(28, 28, 1)),\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - "])\n", - "\n", - "train_out = model(train_data, training=True)\n", - "test_out = model(test_data, training=False)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dS5ed_jjOkvh" - }, - "source": [ - "Некоторые вещи на заметку:\n", - "\n", - "* Подклассы моделей и слоев Keras нужно запускать и в v1 графах (без автоматического контроля зависимостей) и в режиме eager mode\n", - " * Оберните `call()` в `tf.function()` чтобы получить autograph и автоматический контроль зависимостей\n", - "\n", - "* Не забудьте принять аргумент `training` в `call`.\n", - " * Иногда это `tf.Tensor`\n", - " * Иногда это булеан Python.\n", - "\n", - "* Создайте переменные модели в конструкторе или `def build()` используя `self.add_weight()`.\n", - " * В `build` у вас есть доступ к размерности входных данных, так что создайте веса с совпадающими размерностями.\n", - " * Использование `tf.keras.layers.Layer.add_weight` позволяет Keras отслеживать переменные и потери регуляризации.\n", - "\n", - "* Не храните `tf.Tensors` в своих объектах.\n", - - " * Они могут быть созданы либо в `tf.function` либо в контексте eager, и эти тензоры ведут себя по-другому.\n", - " * Используйте `tf.Variable`s для состояния, их всегда можно использовать из обеих контекстов\n", - " * `tf.Tensors` это только промежуточные значения." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ulaB1ymO4pw5" - }, - "source": [ - "### Замечание о Slim и contrib.layers\n", - "\n", - "Большое количество старого TensorFlow 1.x кода использует библиотеку [Slim](https://ai.googleblog.com/2016/08/tf-slim-high-level-library-to-define.html) которая входит в пакет TensorFlow 1.x в качестве `tf.contrib.layers`. В качестве модуля `contrib` она более не доступна в TensorFlow 2.0, даже в `tf.compat.v1`. Конвертация кода использовавшего Slim в TF 2.0 запутаннее чем конвертация репозиториев использующих `tf.layers`. Имеет смысл сперва сконвертировать ваш Slim код сперва в`tf.layers`, а затем конвертировать в Keras.\n", - "\n", - "* Уберите `arg_scopes`, все аргументы должны быть явными\n", - "* Если вы используете их, поделите `normalizer_fn` и `activation_fn` каждый в свой собственный слой\n", - "* Separable сверточные слои отображаются в один или более различных слоев Keras (по глубине, поточечно, и separable слои Keras)\n", - "* Slim и `tf.layers` имеют разные имена аргументов и значения по умолчанию\n", - "* Некоторые аргументы имеют разные размерности\n", - "* Если вы используете предобученные модели Slim, попробуйте `tf.keras.applications` или [TFHub](https://tensorflow.orb/hub)\n", - "\n", - "Некоторые слои `tf.contrib` возможно не были перемещены в ядро TensorFlow, а вместо этого были перемещены в пакет [TF add-ons](https://github.com/tensorflow/addons).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1w72KrXm4yZR" - }, - "source": [ - "## Обучение" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "56PQxTgy2bpI" - }, - "source": [ - "Существует много способов подачи данных в модели `tf.keras`. Они допускают генераторы Python и массивы Numpy в качестве входных данных.\n", - "\n", - "Рекомендуемы способ подачи данных в модель это - использовать пакет `tf.data`, который содержит набор высокопроизводительных классов для манипуляций с данными.\n", - "\n", - "Если вы все еще используете `tf.queue`, они поддерживаются только как структуры данных, а не как входные конвейеры." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m6htasZ7iBB4" - }, - "source": [ - "### Использование наборов данных" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "loTPH2Pz4_Oj" - }, - "source": [ - "Пакет [TensorFlow Datasets](https://tensorflow.org/datasets) (`tfds`) содержит утилиты для загрузки предопределенных баз данных как объектов `tf.data.Dataset`.\n", - "\n", - "Например, загрузим MNISTdataset, используя `tfds`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BMgxaLH74_s-" - }, - "outputs": [], - "source": [ - "datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n", - "mnist_train, mnist_test = datasets['train'], datasets['test']" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hPJhEuvj5VfR" - }, - "source": [ - "Затем приготовим данные для обучения:\n", - "\n", - " * Изменим размер каждого изображения.\n", - " * Перемешаем порядок примеров.\n", - " * Соберем batches изображений и меток.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "StBRHtJM2S7o" - }, - "outputs": [], - "source": [ - "BUFFER_SIZE = 10 # Используйте намного большее значение для настоящего кода.\n", - "BATCH_SIZE = 64\n", - "NUM_EPOCHS = 5\n", - "\n", - "\n", - "def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - "\n", - " return image, label" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SKq14zKKFAdv" - }, - "source": [ - " Чтобы пример оставался коротким обрежем данные, чтобы он возвращал только 5 batches:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_J-o4YjG2mkM" - }, - "outputs": [], - "source": [ - "train_data = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE).take(5)\n", - "test_data = mnist_test.map(scale).batch(BATCH_SIZE).take(5)\n", - "\n", - "STEPS_PER_EPOCH = 5\n", - "\n", - "train_data = train_data.take(STEPS_PER_EPOCH)\n", - "test_data = test_data.take(STEPS_PER_EPOCH)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "XEqdkH54VM6c" - }, - "outputs": [], - "source": [ - "image_batch, label_batch = next(iter(train_data))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mSev7vZC5GJB" - }, - "source": [ - "### Использование обучающик циклов Keras\n", - "\n", - "Если вам не нужен низкоуровневый коноль процесса обучения модели, рекомендуется использовать встроенные в Keras методы `fit`, `evaluate` и `predict`. Эти методы обеспечивают единый интерфейс обучения модели независимо от реализации (sequential, functional или sub-classed).\n", - "\n", - "Преимущества этих методов включают:\n", - "\n", - "* Они допускают массивы Numpy, генераторы Python и `tf.data.Datasets`\n", - "* Они применяют регуляризационные и активационные потери автоматически.\n", - "* Они поддерживают `tf.distribute` [для обучения на нескольких устройствах](distributed_training.ipynb).\n", - "* Они поддерживают произвольные вызываемые объекты как потери и метрики.\n", - "* Они поддерживают коллбеки такие как `tf.keras.callbacks.TensorBoard` и пользовательские коллбеки.\n", - "* Они производительны, автоматически используя графы TensorFlow.\n", - "\n", - "Приведем пример обучения модели с ипользованием `Dataset`. (Подробнее о том как это работает смотри [тьюториалы](../tutorials).)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "uzHFCzd45Rae" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " kernel_regularizer=tf.keras.regularizers.l2(0.02),\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dropout(0.1),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "# Model is the full model w/o custom layers\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "model.fit(train_data, epochs=NUM_EPOCHS)\n", - "loss, acc = model.evaluate(test_data)\n", - "\n", - "print(\"Loss {}, Accuracy {}\".format(loss, acc))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "akpeOb09YBhq" - }, - "source": [ - "### Напишите свой собственный цикл\n", - "\n", - "Если обучающий шаг модели Keras подходит вам, но вне шага вам нужет больший контроль, рассмотрите использование `tf.keras.model.train_on_batch` method, в вашем собтвенном цикле итерации данных.\n", - "\n", - "Запомните: Многие вещи могут быть реализованы как `tf.keras.Callback`.\n", - "\n", - "Этот метод имеет много преимуществ перед методами, упомянутыми в предыдущем разделе, но он дает пользователю контроль над внешним циклом.\n", - "\n", - "Вы также можете использовать `tf.keras.model.test_on_batch` или `tf.keras.Model.evaluate` чтобы проверить производительность во время обучения.\n", - "\n", - "Примечание: `train_on_batch` и `test_on_batch` по умолчанию возвращают потерю и метрики для одного batch. Если вы передаете `reset_metrics=False` они возвращают накопленные метрики и вы должны помнить своевременно сбрасывать накопители метрик. Таже помните, что некоторые метрики, такие как `AUC` требуют `reset_metrics=False` для корректного вычисления.\n", - "\n", - "Чтобы продолжить обучение вышеуказанной модели:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "eXr4CyJMtJJ6" - }, - "outputs": [], - "source": [ - "# Model это полная модель без пользовательских слоев\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "metrics_names = model.metrics_names\n", - "\n", - "for epoch in range(NUM_EPOCHS):\n", - " #Reset the metric accumulators\n", - " model.reset_metrics()\n", - "\n", - " for image_batch, label_batch in train_data:\n", - " result = model.train_on_batch(image_batch, label_batch)\n", - " print(\"train: \",\n", - " \"{}: {:.3f}\".format(metrics_names[0], result[0]),\n", - " \"{}: {:.3f}\".format(metrics_names[1], result[1]))\n", - " for image_batch, label_batch in test_data:\n", - " result = model.test_on_batch(image_batch, label_batch,\n", - " # return accumulated metrics\n", - " reset_metrics=False)\n", - " print(\"\\neval: \",\n", - " \"{}: {:.3f}\".format(metrics_names[0], result[0]),\n", - " \"{}: {:.3f}\".format(metrics_names[1], result[1]))\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LQTaHTuK5S5A" - }, - "source": [ - "

    \n", - "\n", - "### Настройте шаг обучения\n", - "\n", - "Если вам нужны большая гибкость и контроль, вы можете получить их реализовав собственный цикл обучения. Есть три шага:\n", - "\n", - "1. Проитерируйте генератор Python или `tf.data.Dataset` чтобы получить пакеты примеров.\n", - "2. Используйте `tf.GradientTape` чтобы собрать градиенты.\n", - "3. Используйте `tf.keras.optimizer` чтобы применить обновления весов к переменным модели.\n", - "\n", - "Помните:\n", - "\n", - "* Всегда включайте аргумент `training` в метод `call` подклассов слоев и моделей.\n", - "* Убедитесь что вызываете модель с корректно установленным аргументом `training`.\n", - "* В зависимости от использования, переменные модели могут не существовать, пока модель не будет запущена на пакете данных.\n", - "* Вам нужно вручную обрабатывать такие вещи, как потери регуляризации для модели.\n", - "\n", - "Обратите внимание на упрощения относительно v1:\n", - "\n", - "* Нет необходимости запускать инициализаторы переменных. Переменные инициализируются при создании.\n", - "* Нет необходимости добавлять зависимости ручного управления. Даже в операциях `tf.function` действующих как в eager mode." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "gQooejfYlQeF" - }, - "outputs": [], - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " kernel_regularizer=tf.keras.regularizers.l2(0.02),\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dropout(0.1),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "optimizer = tf.keras.optimizers.Adam(0.001)\n", - "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "@tf.function\n", - "def train_step(inputs, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(inputs, training=True)\n", - " regularization_loss = tf.math.add_n(model.losses)\n", - " pred_loss = loss_fn(labels, predictions)\n", - " total_loss = pred_loss + regularization_loss\n", - "\n", - " gradients = tape.gradient(total_loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - "for epoch in range(NUM_EPOCHS):\n", - " for inputs, labels in train_data:\n", - " train_step(inputs, labels)\n", - " print(\"Finished epoch\", epoch)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kS7WW5Z75ve3" - }, - "source": [ - "### Метрики в новом стиле\n", - "\n", - "В TensorFlow 2.0, метрики являются объектами. Метрики работают и eagerly и в `tf.function`. Объекты-метрики обладают следующими методами:\n", - "\n", - "* `update_state()` — добавить новые наблюдения\n", - "* `result()` — получить текущий результат метрики при данных наблюдаемых значениях\n", - "* `reset_states()` — очистить все наблюдения.\n", - "\n", - "Объект сам является вызываемым. Вызов обновляет состояние новыми наблюдениями, как и с `update_state`, и возвращает новый результат метрики\n", - "\n", - "Вам не нужно вручную инициализировать переменные метрики, и, поскольку у TensorFlow 2.0 автоматическое управление зависимостями, вам не нужно беспокоиться и об этом.\n", - "\n", - "В приведенном ниже коде используется метрика для отслеживания среднего значения потерь, наблюдаемых в пользовательском цикле обучения." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HAbA0fKW58CH" - }, - "outputs": [], - "source": [ - "# Создайте метрики\n", - "loss_metric = tf.keras.metrics.Mean(name='train_loss')\n", - "accuracy_metric = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "@tf.function\n", - "def train_step(inputs, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(inputs, training=True)\n", - " regularization_loss = tf.math.add_n(model.losses)\n", - " pred_loss = loss_fn(labels, predictions)\n", - " total_loss = pred_loss + regularization_loss\n", - "\n", - " gradients = tape.gradient(total_loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - " # Обновите метрики\n", - " loss_metric.update_state(total_loss)\n", - " accuracy_metric.update_state(labels, predictions)\n", - "\n", - "\n", - "for epoch in range(NUM_EPOCHS):\n", - " # Сбросьте метрики\n", - " loss_metric.reset_states()\n", - " accuracy_metric.reset_states()\n", - "\n", - " for inputs, labels in train_data:\n", - " train_step(inputs, labels)\n", - " # Получите результаты метрики\n", - " mean_loss = loss_metric.result()\n", - " mean_accuracy = accuracy_metric.result()\n", - "\n", - " print('Epoch: ', epoch)\n", - " print(' loss: {:.3f}'.format(mean_loss))\n", - " print(' accuracy: {:.3f}'.format(mean_accuracy))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JmMLBKs66DeA" - }, - "source": [ - "## Сохранение и загрузка\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5_QKn3Kl6TUu" - }, - "source": [ - "### Совместимость контрольных точек\n", - "\n", - "TensorFlow 2.0 использует [контрольные точки основанные на объектах](checkpoint.ipynb).\n", - "\n", - "Контрольные точки в старом стиле основанные на именах по-прежнему могут быть загружены, если вы осторожны с ними.\n", - "В процессе конвертации кода могут измениться имена переменных, но есть обходные пути.\n", - "\n", - "Самый простой подход - согласовать имена новой модели с именами в контрольной точке.:\n", - "\n", - "* У переменных все еще есть аргумент `name` который вы можете установить.\n", - "* Модели Keras также используют аргумент `name`, который они устанавливают в качестве префикса для своих переменных.\n", - "* Функция `tf.name_scope` может использоваться для установки префиксов имен переменных. Это сильно отличается от `tf.variable_scope`. Он влияет только на имена и не отслеживает переменные и их переиспользование.\n", - "\n", - "Если это не работает для вашего случая, попробуйте функцию `tf.compat.v1.train.init_from_checkpoint`. Она принимает аргумент `assignment_map`, который определяет соответствие старых и новых имен.\n", - "\n", - "Примечание: В отличие от основанных на объектах контрольных точек, которые могут [отложить загрузку] (checkpoint.ipynb#loading_mechanics), основанные на именах контрольных точек требуют, чтобы при вызове функции были созданы все переменные. Некоторые модели откладывают создание переменных до тех пор, пока вы не вызовете `build` или не запустите модель на пакете данных." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ONjobDD6Uur" - }, - "source": [ - "### Совместимость сохраненных моделей\n", - "\n", - "У совместимости для сохраненных моделей нет существенных проблем.\n", - "\n", - "* TensorFlow 1.x saved_models работают TensorFlow 2.0.\n", - "* TensorFlow 2.0 saved_models даже загруженные работают в TensorFlow 1.x если все операции поддерживаются." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ewl9P3oZ6ZtR" - }, - "source": [ - "## Estimators" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YprVP9g3l6eG" - }, - "source": [ - "### Обучение с оценщиками\n", - "\n", - "Оценщики поддерживаются TensorFlow 2.0.\n", - "\n", - "Когда вы используете оценщики, вы можете использовать `input_fn()`, `tf.estimator.TrainSpec`, и `tf.estimator.EvalSpec` из TensorFlow 1.x.\n", - "\n", - "Здесь пример использующий `input_fn` с train and evaluate specs." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N5kZeJsF8lS2" - }, - "source": [ - "#### Создание input_fn и train/eval specs" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "AOlXGO4J6jDh" - }, - "outputs": [], - "source": [ - "# Определим input_fn оценщика \n", - "def input_fn():\n", - " datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n", - " mnist_train, mnist_test = datasets['train'], datasets['test']\n", - "\n", - " BUFFER_SIZE = 10000\n", - " BATCH_SIZE = 64\n", - "\n", - " def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - "\n", - " return image, label[..., tf.newaxis]\n", - "\n", - " train_data = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", - " return train_data.repeat()\n", - "\n", - "# Define train \u0026 eval specs\n", - "train_spec = tf.estimator.TrainSpec(input_fn=input_fn,\n", - " max_steps=STEPS_PER_EPOCH * NUM_EPOCHS)\n", - "eval_spec = tf.estimator.EvalSpec(input_fn=input_fn,\n", - " steps=STEPS_PER_EPOCH)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_o6J48Nj9H5c" - }, - "source": [ - "### Использование определения модели Keras" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IXCQdhGq9SbB" - }, - "source": [ - "Есть некоторые отличия в том, как построить ваши оценщики в TensorFlow 2.0.\n", - "\n", - "Мы рекомендуем вам определить модель используя Keras, потом используйте утилиту `tf.keras.model_to_estimator` для преобразования вашей модели в оценщика. Нижеприведенный код показывает как использовать эту утилиту когда создаешь и обучаешь оценщик." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aelsClm3Cq4I" - }, - "outputs": [], - "source": [ - "def make_model():\n", - " return tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " kernel_regularizer=tf.keras.regularizers.l2(0.02),\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dropout(0.1),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "HJb6f8dtl6rr" - }, - "outputs": [], - "source": [ - "model = make_model()\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "estimator = tf.keras.estimator.model_to_estimator(\n", - " keras_model = model\n", - ")\n", - "\n", - "tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ptTxL1q6flL" - }, - "source": [ - "### Использование пользовательской `model_fn`\n", - "\n", - "Если у вас есть существующий пользовательский оценщик `model_fn`, который вам нужно поддерживать, вы можете конвертировать свой` model_fn` чтобы использовать модель Keras.\n", - "\n", - "Однако по соображениям совместимости пользовательский `model_fn` будет по-прежнему работать в стиле 1.x графа. Это означает, что нет будет eager execution и нет автоматического управления зависимостей.\n", - "\n", - "Использование моделей Keras в пользовательском `model_fn` аналогично использованию в пользовательском цикле обучения:\n", - "\n", - "* Установите фазу `training` соответствующе, основываясь на аргументе `mode`.\n", - "* Явно передайте `trainable_variables` модели оптимизатору.\n", - "\n", - "Но есть важные различия отлосящиеся к [пользовательскому циклу](#custom_loop):\n", - "\n", - "* Вместо использования `model.losses` извлеките потери, используя` tf.keras.Model.get_losses_for`.\n", - "* Извлеките обновления модели используя `tf.keras.Model.get_updates_for`\n", - "\n", - "Примечание: \"Updates\" это изменения которые необходимо применить к модели после каждого пакета. Например, скользящие средние среднего и дисперсии в слое `tf.keras.layers.BatchNormalization`.\n", - "\n", - "Следующий код создает оценщик из пользовательского `model_fn`, иллюстрируя все эти проблемы." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iY16eZKW606-" - }, - "outputs": [], - "source": [ - "def my_model_fn(features, labels, mode):\n", - " model = make_model()\n", - "\n", - " optimizer = tf.compat.v1.train.AdamOptimizer()\n", - " loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - " training = (mode == tf.estimator.ModeKeys.TRAIN)\n", - " predictions = model(features, training=training)\n", - "\n", - " reg_losses = model.get_losses_for(None) + model.get_losses_for(features)\n", - " total_loss = loss_fn(labels, predictions) + tf.math.add_n(reg_losses)\n", - "\n", - " accuracy = tf.compat.v1.metrics.accuracy(labels=labels,\n", - " predictions=tf.math.argmax(predictions, axis=1),\n", - " name='acc_op')\n", - "\n", - " update_ops = model.get_updates_for(None) + model.get_updates_for(features)\n", - " minimize_op = optimizer.minimize(\n", - " total_loss,\n", - " var_list=model.trainable_variables,\n", - " global_step=tf.compat.v1.train.get_or_create_global_step())\n", - " train_op = tf.group(minimize_op, update_ops)\n", - "\n", - " return tf.estimator.EstimatorSpec(\n", - " mode=mode,\n", - " predictions=predictions,\n", - " loss=total_loss,\n", - " train_op=train_op, eval_metric_ops={'accuracy': accuracy})\n", - "\n", - "# Создайте оценщик и обучите\n", - "estimator = tf.estimator.Estimator(model_fn=my_model_fn)\n", - "tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g1l6VnOTodfA" - }, - "source": [ - "### Готовые оценщики\n", - "\n", - "[Готовые оценщики](https://www.tensorflow.org/guide/premade_estimators) из семейств `tf.estimator.DNN*`, `tf.estimator.Linear*` и `tf.estimator.DNNLinearCombined*` все еще поддерживаются в TensorFlow 2.0 API, однако, некоторые аргументы изменились:\n", - "\n", - "1. `input_layer_partitioner`: Убрано в 2.0.\n", - "2. `loss_reduction`: Обновлено до `tf.keras.losses.Reduction` вместо `tf.compat.v1.losses.Reduction`. Значение по умолчанию также изменилось и стало `tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE` вместо `tf.compat.v1.losses.Reduction.SUM`.\n", - "3. `optimizer`, `dnn_optimizer` и `linear_optimizer`: эти аргументы обновились до `tf.keras.optimizers` вместо `tf.compat.v1.train.Optimizer`. \n", - "\n", - "Для переноса вышеуказанных изменений:\n", - "1. Для `input_layer_partitioner` миграция не требуется поскольку [`Стратегия распределения`](https://www.tensorflow.org/guide/distributed_training) обработает это автоматически в TF 2.0.\n", - "2. Для `loss_reduction`, проверьте [`tf.keras.losses.Reduction`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/losses/Reduction) для поддерживаемых опций.\n", - "3. Для аргументов `optimizer` args, если вы не передаете аргумента `optimizer`, `dnn_optimizer` или `linear_optimizer`, или если вы укажете в своем коде аргумент `optimizer` как `string`, вам не нужно ничего менять. `tf.keras.optimizers` используются по умолчанию. Иначе, вам нужно обновить его от `tf.compat.v1.train.Optimizer` до соответсвующей [`tf.keras.optimizers`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/optimizers)\n", - "\n", - "#### Конвертер контрольных точек\n", - "Миграция `optimizer` повредит контрольные точки в TF 1.x, так как` tf.keras.optimizer` генерирует другой набор переменных для сохранения в контрольных точках. Чтобы сделать контрольную пригодной к использованию после перехода на TF 2.0, пожалуйста, посмотрите инструмент конвертации контрольных точек для оптимизаторов, чтобы преобразовать контрольные точки из TF 1.x в TF 2.0. Преобразованные контрольные точки можно использовать для восстановления предварительно обученных моделей в TF 2.0." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dt8ct9XCFqls" - }, - "source": [ - "## TensorShape\n", - "\n", - "Этот класс был упрощен для хранения `int` вместо объектов `tf.compat.v1.Dimension`. Так что нет необходимости в вызове `.value()` чтобы получить `int`.\n", - "\n", - "Отдельные объекты `tf.compat.v1.Dimension` по-прежнему доступны из `tf.TensorShape.dims`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x36cWcmM8Eu1" - }, - "source": [ - "\n", - "\n", - "Следующее демонстрирует отличия TensorFlow 1.x и TensorFlow 2.0." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "PbpD-kHOZR4A" - }, - "outputs": [], - "source": [ - "# Создайте shape и выберите index\n", - "i = 0\n", - "shape = tf.TensorShape([16, None, 256])\n", - "shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kDFck03neNy0" - }, - "source": [ - "Если у вас есть это в TF 1.x:\n", - "\n", - "```python\n", - "value = shape[i].value\n", - "```\n", - "\n", - "Сделайте это в TF 2.0:\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KuR73QGEeNdH" - }, - "outputs": [], - "source": [ - "value = shape[i]\n", - "value" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bPWPNKRiZmkd" - }, - "source": [ - "Если у вас есть это в TF 1.x:\n", - "\n", - "```python\n", - "for dim in shape:\n", - " value = dim.value\n", - " print(value)\n", - "```\n", - "\n", - "TСделайте это в TF 2.0:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "y6s0vuuprJfc" - }, - "outputs": [], - "source": [ - "for value in shape:\n", - " print(value)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YpRgngu3Zw-A" - }, - "source": [ - "Если у вас есть это в 1.x (Или используется любой другой метод размерности):\n", - "\n", - "```python\n", - "dim = shape[i]\n", - "dim.assert_is_compatible_with(other_dim)\n", - "```\n", - "\n", - "Сделайте это в TF 2.0:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LpViGEcUZDGX" - }, - "outputs": [], - "source": [ - "other_dim = 16\n", - "Dimension = tf.compat.v1.Dimension\n", - "\n", - "if shape.rank is None:\n", - " dim = Dimension(None)\n", - "else:\n", - " dim = shape.dims[i]\n", - "dim.is_compatible_with(other_dim) # или любой другой метод размерности" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GaiGe36dOdZ_" - }, - "outputs": [], - "source": [ - "shape = tf.TensorShape(None)\n", - "\n", - "if shape:\n", - " dim = shape.dims[i]\n", - " dim.is_compatible_with(other_dim) # или любой другой метод размерности" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3kLLY0I3PI-l" - }, - "source": [ - "Булево значение `tf.TensorShape` является `True` если ранг известен, `False` в противном случае." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-Ow1ndKpOnJd" - }, - "outputs": [], - "source": [ - "print(bool(tf.TensorShape([]))) # Скаляр\n", - "print(bool(tf.TensorShape([0]))) # Вектор длины 0\n", - "print(bool(tf.TensorShape([1]))) # Вектор длины 1\n", - "print(bool(tf.TensorShape([None]))) # Вектор неизвестной длины\n", - "print(bool(tf.TensorShape([1, 10, 100]))) # 3D тензор\n", - "print(bool(tf.TensorShape([None, None, None]))) # 3D тензор с неизвестными размерностями\n", - "print()\n", - "print(bool(tf.TensorShape(None))) # Тензор неизвестного ранга." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lwswSCLT9g63" - }, - "source": [ - "## Другие поведенческие изменения\n", - "\n", - "В TensorFlow 2.0 есть несколько других поведенческих изменений, с которыми вы можете столкнуться.\n", - "\n", - "\n", - "### ResourceVariables\n", - "\n", - "TensorFlow 2.0 создает по умолчанию `ResourceVariables`, а не `RefVariables`.\n", - "\n", - "`ResourceVariables` закрыты для записи, и обеспечивают лучшие гарантии согласовенности.\n", - "\n", - "* Это может изменить поведение в граничных случаях.\n", - "* Это может иногда создавать дополнительные копии и использовать большие объемы памяти\n", - "* Это можно отключить, передав `use_resource = False` конструктору` tf.Variable`.\n", - "\n", - "### Control Flow\n", - "\n", - "Реализация control flow была упрощена, поэтому в TensorFlow 2.0 создаются другие графы." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vKX6AdTAQhB-" - }, - "source": [ - "## Выводы\n", - "\n", - "Общий процесс следующий:\n", - "\n", - "1. Запуститсе upgrade script.\n", - "2. Удалите символы contrib.\n", - "3. Переключите ваши модели в объектно ориентированный стиль (Keras).\n", - "4. Используйте циклы обучения и оценки `tf.keras` или `tf.estimator` там где вы можете.\n", - "5. Иначе используйте пользовательские циклы, но избегайте сессий и коллекций.\n", - "\n", - "\n", - "Для преобразования кода в идиоматический TensorFlow 2.0 требуется небольшая работа, но каждое изменение приводит к:\n", - "\n", - "* Меньшему количеству строк кода.\n", - "* Увеличившейся понятности и простоте.\n", - "* К более легкой отладке.\n", - "\n" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "migrate.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ru/r1/README.md b/site/ru/r1/README.md deleted file mode 100644 index fc0ab81c3ee..00000000000 --- a/site/ru/r1/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# TensorFlow 1.x - -This archive of the TensorFlow 1.x docs is in *maintenance mode* only. - -For docs contributors, please update the source files in `site/en/` and read the -[TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs). - -For community translations, read the instructions in `site/ru/README.md`. diff --git a/site/ru/r1/guide/eager.ipynb b/site/ru/r1/guide/eager.ipynb deleted file mode 100644 index f948e9c836d..00000000000 --- a/site/ru/r1/guide/eager.ipynb +++ /dev/null @@ -1,1750 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "eager.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCQY7jpBfMur" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "z6X9omPnfO_h", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2QQJJyDzqGRb" - }, - "source": [ - "# Eager execution\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B1xdylywqUSX" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Запусти в Google Colab\n", - " \n", - " Изучай код на GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FHHSnZ-0bIaM", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EGjDcGxIqEfX" - }, - "source": [ - "\n", - "\n", - "Eager execution в TensorFlow - это интерактивный режим, в котором все вычисления выполняются мгновенно: операции возвращают конкретные значения, вместо построения вычислительных графов. Это позволяет легко запускать любой код в TensorFlow, упрощает отладку моделей и устраняет необходимость в шаблонном boilerplate коде. Пройди все этапы в этом руководстве, просто запуская примеры кода в интерпретаторе `Python` ниже.\n", - "\n", - "Eager execution является гибкой платформой машинного обучения для проведения\n", - "исследований и экспериментов. Ее основные преимущества:\n", - "\n", - "* *Интуитивный интерфейс* — структурируй свой код и используй стандартные структуры\n", - "данных Python. Быстро проверяй гипотезы с помощью небольших моделей на малых данных\n", - "* *Легкая отладка кода* — Производи любые операции непосредственно на готовых\n", - "моделях и проверяй изменения. Используй стандартные инструменты для отладки\n", - "Python кода для незамедлительного отчета об ошибках\n", - "* *Естественный порядок выполнения* — Используй порядок выполнения Python вместо графа вычислений, упрощая спецификацию динамических моделей\n", - "\n", - "Eager execution поддерживает большинство операций TensoFlow и ускорение при помощи GPU.\n", - "Смотри основные примеры, которые можно\n", - "запускать в eager execution здесь:\n", - "[Примеры в Eager Execution](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).\n", - "\n", - "Обрати внимание: некоторые модели могут испытывать повышенную\n", - "нагрузку при активированном eager execution. Улучшения производительности этого режима находятся в непрерывной разработке, пожалуйста\n", - "[сообщайте о багах](https://github.com/tensorflow/tensorflow/issues) если\n", - "сталкиваетесь с какими-либо проблемами." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RBAeIwOMrYk8" - }, - "source": [ - "## Установка и использование\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "48P3-8q4qEfe" - }, - "source": [ - "Чтобы запустить eager execution, добавь `tf.enable_eager_execution()` в начало\n", - "программы или консольной сессии. Не добавляй эту операцию к другим модулям,\n", - "которые вызывает программа." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "7aFsD8csqEff", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x_G1zZT5qEfh" - }, - "source": [ - "Теперь ты можешь запускать любые операции TensorFlow и получать результаты мгновенно:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5hien2IEqwLQ", - "colab": {} - }, - "source": [ - "tf.executing_eagerly()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "9gsI54pbqEfj", - "colab": {} - }, - "source": [ - "x = [[2.]]\n", - "m = tf.matmul(x, x)\n", - "print(\"привет, {}\".format(m))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ajFn6qsdqEfl" - }, - "source": [ - "Включение режима eager execution изменяет то, как себя ведут операции TensorFlow:\n", - "теперь они выполняются мгновенно. Объекты `tf.Tensor` относятся к конкретным\n", - "значениям, вместо символических имен вычислительных графов. Так как теперь\n", - "нет вычислительного графа, который нужно построить и затем запустить в сессии,\n", - "можно легко инспектировать результаты при помощи функции `print()`\n", - "или отладчика. Вычисления, вывод данных и проверка значений тензоров\n", - "не нарушает порядок выполнения для расчета градиентов.\n", - "\n", - "Eager execution легко работает с [NumPy](http://www.numpy.org/). Операции NumPy\n", - "принимают аргументы `tf.Tensor`. [Математические операции](https://www.tensorflow.org/api_guides/python/math_ops) TensorFlow конвертируют\n", - "объекты Python и массивы NumPy в объекты `tf.Tensor`. Метод\n", - "`tf.Tensor.numpy` возвращает значение объекта как массив NumPy `ndarray`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sTO0_5TYqz1n", - "colab": {} - }, - "source": [ - "a = tf.constant([[1, 2],\n", - " [3, 4]])\n", - "print(a)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Dp14YT8Gq4r1", - "colab": {} - }, - "source": [ - "# Проверяем при помощи функции `print`\n", - "b = tf.add(a, 1)\n", - "print(b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "69p3waMfq8cQ", - "colab": {} - }, - "source": [ - "# Поддерживается перегрузка операторов\n", - "print(a * b)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Ui025t1qqEfm", - "colab": {} - }, - "source": [ - "# Используем значения NumPy\n", - "import numpy as np\n", - "\n", - "c = np.multiply(a, b)\n", - "print(c)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tq_aFRzWrCua", - "colab": {} - }, - "source": [ - "# Получаем значение NumPy из тензора\n", - "print(a.numpy())\n", - "# => [[1 2]\n", - "# [3 4]]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Jdg3nZFwqEfp" - }, - "source": [ - "Модуль `tf.contrib.eager` содержит символы, доступные как в eager, так и в стандартном режиме graph execution, и является весьма полезным при [работе с графами](#work_with_graphs):" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "N-2lSGiHqEfq", - "colab": {} - }, - "source": [ - "tfe = tf.contrib.eager" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H08f9ss9qEft" - }, - "source": [ - "## Динамический порядок выполнения\n", - "\n", - "Большим преимуществом eager execution является то, что весь\n", - "функционал языка хоста доступен в то время, как запущена модель.\n", - "Например, можно легко написать решение задачи [fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz):" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "0fudRMeUqEfu", - "colab": {} - }, - "source": [ - "def fizzbuzz(max_num):\n", - " counter = tf.constant(0)\n", - " max_num = tf.convert_to_tensor(max_num)\n", - " for num in range(1, max_num.numpy()+1):\n", - " num = tf.constant(num)\n", - " if int(num % 3) == 0 and int(num % 5) == 0:\n", - " print('FizzBuzz')\n", - " elif int(num % 3) == 0:\n", - " print('Fizz')\n", - " elif int(num % 5) == 0:\n", - " print('Buzz')\n", - " else:\n", - " print(num.numpy())\n", - " counter += 1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "P2cKknQWrJLB", - "colab": {} - }, - "source": [ - "fizzbuzz(15)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7kA-aC3BqEfy" - }, - "source": [ - "Есть условные операторы, которые зависят от значений тензоров, и эти\n", - "значения выводятся в среде выполнения нашей программы." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zn3mp-j6rjnk" - }, - "source": [ - "## Построение модели\n", - "\n", - "Многие модели машинного обучения состоят из сочетания слоев. Когда\n", - "мы используем TensorFlow с eager execution, ты можешь либо создавать\n", - "свои собственные слои, или использовать уже готовые, которые\n", - "определены в `tf.keras.layers`.\n", - "\n", - "В то время как ты можешь использовать любой объект Python для\n", - "представления слоя, в TensorFlow есть удобный способ использования\n", - "базовых классов слоев из `tf.keras.layers.Layers`. Используй их\n", - "для создания своего собственного слоя:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "kvXiJsmyqEfz", - "colab": {} - }, - "source": [ - "class MySimpleLayer(tf.keras.layers.Layer):\n", - " def __init__(self, output_units):\n", - " super(MySimpleLayer, self).__init__()\n", - " self.output_units = output_units\n", - "\n", - " def build(self, input_shape):\n", - " # Метод `build` вызывается первый раз, когда используется слой.\n", - " # Создание переменных в build() позволяет задать размерность тензоров в зависимости\n", - " # от размерности входных параметров, и таким образом устранить необходимость\n", - " # пользователю уточнять формы полностью. Также возможно создавать переменные\n", - " # во время __init__(), если ты уже знаешь их полные формы.\n", - " self.kernel = self.add_variable(\n", - " \"kernel\", [input_shape[-1], self.output_units])\n", - "\n", - " def call(self, input):\n", - " # Перепишем call() вместо __call__, чтобы мы могли вести счет.\n", - " return tf.matmul(input, self.kernel)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eo3qgrVCqEf2" - }, - "source": [ - "Используй слой `tf.keras.layers.Dense` вместо `MySimpleLayer` выше, так как\n", - "он включает в себя надмножество (также может включать в себя смещение `bias`).\n", - "\n", - "Когда составляешь слои модели, ты можешь использовать `tf.keras.Sequential` для\n", - "создания моделей, которые являются линейный стеком слоев. Это легко использовать\n", - "для стандартных моделей:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "VrfLnhNPqEf3", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, input_shape=(784,)), # укажем размерность входных данных\n", - " tf.keras.layers.Dense(10)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dms3mduTqEf6" - }, - "source": [ - "Альтернативный способ - организовать модели в классы из `tf.keras.Model`.\n", - "Это контейнер слоев, который также сам является слоем, что позволяет\n", - "объектам `tf.keras.Model` содержать в себе другие объекты `tf.keras.Model`." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "MwWxQmNOqEf7", - "colab": {} - }, - "source": [ - "class MNISTModel(tf.keras.Model):\n", - " def __init__(self):\n", - " super(MNISTModel, self).__init__()\n", - " self.dense1 = tf.keras.layers.Dense(units=10)\n", - " self.dense2 = tf.keras.layers.Dense(units=10)\n", - "\n", - " def call(self, input):\n", - " \"\"\"Запускаем модель.\"\"\"\n", - " result = self.dense1(input)\n", - " result = self.dense2(result)\n", - " result = self.dense2(result) # повторно используем переменные из слоя dense2\n", - " return result\n", - "\n", - "model = MNISTModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "a639YaF4qEf-" - }, - "source": [ - "Необязательно устанавливать размерность входных данных для классов `tf.keras.Model`,\n", - "поскольку параметры устанавливаются первый раз и передаются слою.\n", - "\n", - "Классы `tf.keras.layers` создают и содержат собственные переменные\n", - "моделей, время действия которых привязаны к объектам слоев. Чтобы разделить\n", - "переменные слоев, необходимо разделить их объекты." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8huKpuuAwICq" - }, - "source": [ - "## Обучение в Eager" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mp2lCCZYrxHd" - }, - "source": [ - "### Вычисление градиентов\n", - "\n", - "[Автоматическое дифференцирование](https://en.wikipedia.org/wiki/Automatic_differentiation)\n", - "полезна для реализации алгоритмов машинного обучения, например таких ка\n", - "[метод обратного распространения ошибки backpropagation](https://ru.wikipedia.org/wiki/%D0%9C%D0%B5%D1%82%D0%BE%D0%B4_%D0%BE%D0%B1%D1%80%D0%B0%D1%82%D0%BD%D0%BE%D0%B3%D0%BE_%D1%80%D0%B0%D1%81%D0%BF%D1%80%D0%BE%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B5%D0%BD%D0%B8%D1%8F_%D0%BE%D1%88%D0%B8%D0%B1%D0%BA%D0%B8)\n", - "для обучения нейронных сетей. Во время eager execution используй `tf.GradientTape`\n", - "для записи операций и для последующего вычисления градиента.\n", - "\n", - "`tf.GradientTape` - это встроенная возможность обеспечения максимальной производительности модели, когда не записываются операции. Поскольку\n", - "могут возникать разные операции во время каждого вызова, все\n", - "операции прямого прохода записываются на \"пленку\" (\"tape\"). Для\n", - "вычисления градиента, пленка воспроизводится в обратном\n", - "порядке, а затем сбрасывается. Конкретная запись `tf.GradientTape`\n", - "может произвести расчет только одного градиента; все последующие\n", - "вызовы выдадут ошибку рабочей среды." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "7g1yWiSXqEf-", - "colab": {} - }, - "source": [ - "w = tf.Variable([[1.0]])\n", - "with tf.GradientTape() as tape:\n", - " loss = w * w\n", - "\n", - "grad = tape.gradient(loss, w)\n", - "print(grad) # => tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkHs32GqweYS" - }, - "source": [ - "### Обучение модели\n", - "\n", - "В следующем примере мы создадим многослойную модель, которая будет\n", - "классифицировать стандартный датасет MNIST, состоящий из изображений\n", - "рукописных чисел. Данный пример продемонстрирует API оптимизатора\n", - "и слоя для создания обучаемых графов в среде eager execution." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "38kymXZowhhz", - "colab": {} - }, - "source": [ - "# Загружаем и форматируем данные mnist\n", - "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n", - "\n", - "dataset = tf.data.Dataset.from_tensor_slices(\n", - " (tf.cast(mnist_images[...,tf.newaxis]/255, tf.float32),\n", - " tf.cast(mnist_labels,tf.int64)))\n", - "dataset = dataset.shuffle(1000).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rl1K8rOowmwT", - "colab": {} - }, - "source": [ - "# Создаем модель\n", - "mnist_model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu', input_shape=(None,None,1)),\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(10)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fvyk-HgGwxwl" - }, - "source": [ - "Еще не приступив к обучению, попробуй вызвать модель и проинспектировать\n", - "вывод данных в eager execution:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BsxystjBwxLS", - "colab": {} - }, - "source": [ - "for images,labels in dataset.take(1):\n", - " print(\"Логиты: \", mnist_model(images[0:1]).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y3PGa8G7qEgB" - }, - "source": [ - "В то время как модели Keras имеют встроенный тренировочный цикл (используется в методе `fit`), иногда\n", - "требуется более тонкая настройка. Вот пример, когда тренировочный цикл реализован в eager:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bzRhM7JDnaEG", - "colab": {} - }, - "source": [ - "optimizer = tf.train.AdamOptimizer()\n", - "\n", - "loss_history = []" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "0m1xAXrmqEgJ", - "colab": {} - }, - "source": [ - "for (batch, (images, labels)) in enumerate(dataset.take(400)):\n", - " if batch % 80 == 0:\n", - " print()\n", - " print('.', end='')\n", - " with tf.GradientTape() as tape:\n", - " logits = mnist_model(images, training=True)\n", - " loss_value = tf.losses.sparse_softmax_cross_entropy(labels, logits)\n", - "\n", - " loss_history.append(loss_value.numpy())\n", - " grads = tape.gradient(loss_value, mnist_model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, mnist_model.trainable_variables),\n", - " global_step=tf.train.get_or_create_global_step())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5vG5ql_2vYB5", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.plot(loss_history)\n", - "plt.xlabel('Батч #')\n", - "plt.ylabel('Потери [энтропия]')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9ODzXW8xqEgU" - }, - "source": [ - "В этом примере используется\n", - "[модуль dataset.py](https://github.com/tensorflow/models/blob/master/official/mnist/dataset.py)\n", - "из\n", - "[примера TensorFlow MNIST](https://github.com/tensorflow/models/tree/master/official/mnist);\n", - "загрузи этот файл в папку на своем устройстве. Затем запусти код для загрузки\n", - "данных MNIST в рабочую папку и подготовь `tf.data.Dataset`\n", - "к обучению:" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKpOlHPLqEgl" - }, - "source": [ - "### Переменные и оптимизаторы\n", - "\n", - "Объекты `tf.Variable` хранят переменные `tf.Tensor` значения, доступ\n", - "к которым предоставляется во время обучения, чтобы проще производить\n", - "автоматическую дифференцирование. Параметры модели могут быть\n", - "сохранены в классах как переменные.\n", - "\n", - "Лучше сохранять параметры модели, используя `tf.Variable` при помощи\n", - "`tf.GradientTape`. Например таким образом, автоматическая дифференцирование\n", - "примера выше может быть перезаписана:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "nnQLBYmEqEgm", - "colab": {} - }, - "source": [ - "class Model(tf.keras.Model):\n", - " def __init__(self):\n", - " super(Model, self).__init__()\n", - " self.W = tf.Variable(5., name='weight')\n", - " self.B = tf.Variable(10., name='bias')\n", - " def call(self, inputs):\n", - " return inputs * self.W + self.B\n", - "\n", - "# Маленький датасет из точек около 3 * x + 2\n", - "NUM_EXAMPLES = 2000\n", - "training_inputs = tf.random_normal([NUM_EXAMPLES])\n", - "noise = tf.random_normal([NUM_EXAMPLES])\n", - "training_outputs = training_inputs * 3 + 2 + noise\n", - "\n", - "# Функция потерь для оптимизации\n", - "def loss(model, inputs, targets):\n", - " error = model(inputs) - targets\n", - " return tf.reduce_mean(tf.square(error))\n", - "\n", - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return tape.gradient(loss_value, [model.W, model.B])\n", - "\n", - "# Определим:\n", - "# 1. Модель\n", - "# 2. Производные функции потерь с учетом параметров модели\n", - "# 3. Стратегию обновления переменных, основываясь на производных\n", - "model = Model()\n", - "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n", - "\n", - "print(\"Изначальная потеря: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", - "\n", - "# Тренировочный цикл:\n", - "for i in range(300):\n", - " grads = grad(model, training_inputs, training_outputs)\n", - " optimizer.apply_gradients(zip(grads, [model.W, model.B]),\n", - " global_step=tf.train.get_or_create_global_step())\n", - " if i % 20 == 0:\n", - " print(\"Потеря на шаге {:03d}: {:.3f}\".format(i, loss(model, training_inputs, training_outputs)))\n", - "\n", - "print(\"Окончательная потеря: {:.3f}\".format(loss(model, training_inputs, training_outputs)))\n", - "print(\"W = {}, B = {}\".format(model.W.numpy(), model.B.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rPjb8nRWqEgr" - }, - "source": [ - "## Использование объектов для состояния в eager execution\n", - "\n", - "В graph execution состояния программы (такие, как переменные) хранятся\n", - "в глобальных коллекциях и их срок действия определяется объектом\n", - "`tf.Session`. Для сравнения, во время eager execution срок действия\n", - "состояний объектов определяется сроком действия соответствующего\n", - "объекта Python.\n", - "\n", - "### Переменные объекты\n", - "\n", - "Во время eager execution, переменные сохраняются и хранятся до тех пор, пока не\n", - "будет убрана последняя отсылка к объекту, и только тогда он будет удален." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "A2boS674qEgs", - "colab": {} - }, - "source": [ - "if tf.test.is_gpu_available():\n", - " with tf.device(\"gpu:0\"):\n", - " v = tf.Variable(tf.random_normal([1000, 1000]))\n", - " v = None # v больше не занимает место в памяти GPU" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "scMjg6L6qEgv" - }, - "source": [ - "### Объектное сохранение\n", - "\n", - "`tf.train.Checkpoint` может сохранять и загружать `tf.Variable`s как *в*,\n", - "так и *из* контрольных точек:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7z5xRfdHzZOQ", - "colab": {} - }, - "source": [ - "x = tf.Variable(10.)\n", - "checkpoint = tf.train.Checkpoint(x=x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IffrUVG7zyVb", - "colab": {} - }, - "source": [ - "x.assign(2.) # Назначим новое значение переменной и сохраним\n", - "checkpoint_path = './ckpt/'\n", - "checkpoint.save('./ckpt/')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "eMT9koCoqEgw", - "colab": {} - }, - "source": [ - "x.assign(11.) # Изменим переменную после сохранения\n", - "\n", - "# Восстановим значение из контрольной точки\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_path))\n", - "\n", - "print(x) # => 2.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbFnP-yLqEgx" - }, - "source": [ - "Для сохранения и загрузки моделей `tf.train.Checkpoint` сохраняет внутреннее\n", - "состояние объектов без требования скрытых переменных. Чтобы сохранить\n", - "текущее состояние `модели`, выбранный `оптимизатор` и глобальный шаг,\n", - "просто передай их как аргумент к `tf.train.Checkpoint`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "hWZHyAXMqEg0", - "colab": {} - }, - "source": [ - "import os\n", - "import tempfile\n", - "\n", - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(10)\n", - "])\n", - "optimizer = tf.train.AdamOptimizer(learning_rate=0.001)\n", - "checkpoint_dir = tempfile.mkdtemp()\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "root = tf.train.Checkpoint(optimizer=optimizer,\n", - " model=model,\n", - " optimizer_step=tf.train.get_or_create_global_step())\n", - "\n", - "root.save(checkpoint_prefix)\n", - "root.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3yoD0VJ7qEg3" - }, - "source": [ - "### Объектно-ориентированные показатели\n", - "\n", - "`tfe.metrics` сохраняются как объекты. Обнови показатели модели, передав\n", - "новые данные к вызываемому объекту, и получи результат при помощи\n", - "метода `tf.metrics.result` как в этом примере:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "9ccu0iAaqEg5", - "colab": {} - }, - "source": [ - "m = tfe.metrics.Mean(\"loss\")\n", - "m(0)\n", - "m(5)\n", - "m.result() # => 2.5\n", - "m([8, 9])\n", - "m.result() # => 5.5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BE8cXErYqEg8" - }, - "source": [ - "#### Статистика обучения в TensorBoard\n", - "\n", - "[TensorBoard](https://tensorflow.org/tensorboard) - это программа для\n", - "визуализации обучения модели, которая помогает лучше понять процесс\n", - "тренировки, отладить код и оптимизировать модель для достижения\n", - "лучших показателей. TensorBoard записывает все ключевые моменты\n", - "обучения во время тренировки модели.\n", - "\n", - "`tf.contrib.summary` совместим как с режимами eager, так и graph execution.\n", - "Операции для записи итогов обучения, такие как `tf.contrib.summary.scalar`,\n", - "должны быть использованы в коде во время построения модели. Вот пример\n", - "записи показателей модели через каждые 100 глобальных шагов:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "3PXAJB1GqEg9", - "colab": {} - }, - "source": [ - "global_step = tf.train.get_or_create_global_step()\n", - "\n", - "logdir = \"./tb/\"\n", - "writer = tf.contrib.summary.create_file_writer(logdir)\n", - "writer.set_as_default()\n", - "\n", - "for _ in range(10):\n", - " global_step.assign_add(1)\n", - " # Обязательно укажи метод `record_summaries`\n", - " with tf.contrib.summary.record_summaries_every_n_global_steps(100):\n", - " # Основной код для построения модели идет ниже\n", - " tf.contrib.summary.scalar('global_step', global_step)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6TJSs_wG8Spg", - "colab": {} - }, - "source": [ - "ls tb/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZZRIIgFXhLzl" - }, - "source": [ - "## Углубленные темы автоматического дифференцирования" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xEL4yJe5qEhD" - }, - "source": [ - "### Динамические модели\n", - "\n", - "`tf.GradientTape` также может быть использован для динамических\n", - "моделей. Вот пример алгоритма [поиска линии с возвратом](https://ru.wikipedia.org/wiki/%D0%9F%D0%BE%D0%B8%D1%81%D0%BA_%D1%81_%D0%B2%D0%BE%D0%B7%D0%B2%D1%80%D0%B0%D1%82%D0%BE%D0%BC), который выглядит как обычный\n", - "код NumPy code, но с градиентами, и этот код - дифференцируемый,\n", - "несмотря на сложный порядок выполнения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "L518n5dkqEhE", - "colab": {} - }, - "source": [ - "def line_search_step(fn, init_x, rate=1.0):\n", - " with tf.GradientTape() as tape:\n", - " # Переменные записываются автоматически, но следим за тензором вручную.\n", - " tape.watch(init_x)\n", - " value = fn(init_x)\n", - " grad = tape.gradient(value, init_x)\n", - " grad_norm = tf.reduce_sum(grad * grad)\n", - " init_value = value\n", - " while value > init_value - rate * grad_norm:\n", - " x = init_x - rate * grad\n", - " value = fn(x)\n", - " rate /= 2.0\n", - " return x, value" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "03owpCFrqEhH" - }, - "source": [ - "### Дополнительные функции для вычисления градиентов\n", - "\n", - "`tf.GradientTape` хоть и является эффективным интерфейсом для расчета\n", - "градиентов, однако существует также другой API, который работает в стиле\n", - "[Autograd](https://github.com/HIPS/autograd) и используется для автоматического\n", - "дифференцирования. Эти функции весьма полезны при написании математического\n", - "кода только с тензорами и функциями градиентов, но без `tf.variables`:\n", - "\n", - "* `tfe.gradients_function` — Возвращает функцию, которая рассчитывает\n", - "производные ее параметров функции ввода с учетом аргументов. Параметр\n", - "функции ввода должен возвращать скалярное значение. Когда мы вызываем\n", - "возвращенную функцию, она возвращает список объектов `tf.Tensor`: по одному\n", - "элементу для каждого аргумента функции ввода. Поскольку нам могут быть интересны\n", - "любые параметры функции, этот метод может быть весьма неудобным в\n", - "использовании, если есть какие-либо зависимости от множества\n", - "обучаемых параметров\n", - "* `tfe.value_and_gradients_function` — Похожа на `tfe.gradients_function`,\n", - "но когда мы вызываем возвращенную функцию, она возвращает значение функции\n", - "ввода в дополнение к списку производных функции ввода с учетом ее аргументов\n", - "\n", - "В следующем примере `tf.gradients_function` принимает функцию `square`\n", - "как аргумент и возвращает функцию, которая рассчитывает частную производную\n", - "`square` с учетом ее вводов. Для вычисления квадрата производной `square` числа `3`,\n", - "`grad(3.0)` возвращает `6`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QDPFUG-68i-B", - "colab": {} - }, - "source": [ - "def square(x):\n", - " return tf.multiply(x, x)\n", - "\n", - "grad = tfe.gradients_function(square)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_Ow9LHre8k_3", - "colab": {} - }, - "source": [ - "square(3.).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Rg5ea0kC8nEQ", - "colab": {} - }, - "source": [ - "grad(3.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "41D1LzcG87p8", - "colab": {} - }, - "source": [ - "# Квадрат производной второго порядка\n", - "gradgrad = tfe.gradients_function(lambda x: grad(x)[0])\n", - "gradgrad(3.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "DotEGcx5qEhH", - "colab": {} - }, - "source": [ - "# Производная третьего порядка - None\n", - "gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0])\n", - "gradgradgrad(3.)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9DRAews99F4-", - "colab": {} - }, - "source": [ - "# С потоком выполнения\n", - "def abs(x):\n", - " return x if x > 0. else -x\n", - "\n", - "grad = tfe.gradients_function(abs)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hTA1jmu_9gyB", - "colab": {} - }, - "source": [ - "grad(3.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7zs8JpKw9kir", - "colab": {} - }, - "source": [ - "grad(-3.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gieGOf_DqEhK" - }, - "source": [ - "### Собственные градиенты\n", - "\n", - "Создание собственных градиентов - это простой способ переписать\n", - "стандартные градиенты в режимах eager и graph execution. В блоке\n", - "функции прямого прохода определи градиент с учетом вводов, выводов\n", - "и промежуточных результатов. Например, вот легкий способ сжатия\n", - "норм градиентов в обратном проходе `backward pass`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "-OwwsWUAqEhK", - "colab": {} - }, - "source": [ - "@tf.custom_gradient\n", - "def clip_gradient_by_norm(x, norm):\n", - " y = tf.identity(x)\n", - " def grad_fn(dresult):\n", - " return [tf.clip_by_norm(dresult, norm), None]\n", - " return y, grad_fn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JPLDHkF_qEhN" - }, - "source": [ - "Собственные градиенты часто используются для обеспечения численно\n", - "стабильных градиентов для последовательности операций:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "24WiLROnqEhO", - "colab": {} - }, - "source": [ - "def log1pexp(x):\n", - " return tf.log(1 + tf.exp(x))\n", - "grad_log1pexp = tfe.gradients_function(log1pexp)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "n8fq69r9-B-c", - "colab": {} - }, - "source": [ - "# Вычисление градиента работает отлично если x = 0.\n", - "grad_log1pexp(0.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_VFSU0mG-FSp", - "colab": {} - }, - "source": [ - "# Однако, при x = 100 расчет не производится по причине числовой нестабильности.\n", - "grad_log1pexp(100.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-VcTR34rqEhQ" - }, - "source": [ - "Функцию `log1pexp` можно аналитически упростить при помощи\n", - "собственного производного градиента. Реализация этого подхода,\n", - "как в примере ниже, повторно использует значение `tf.exp(x)`,\n", - "которое было рассчитано во время прямого прохода, делает\n", - "данный метод более эффективным за счет устранения\n", - "избыточных вычислений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Q7nvfx_-qEhS", - "colab": {} - }, - "source": [ - "@tf.custom_gradient\n", - "def log1pexp(x):\n", - " e = tf.exp(x)\n", - " def grad(dy):\n", - " return dy * (1 - 1 / (1 + e))\n", - " return tf.log(1 + e), grad\n", - "\n", - "grad_log1pexp = tfe.gradients_function(log1pexp)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5gHPKMfl-Kge", - "colab": {} - }, - "source": [ - "# Как и в предыдущий раз, вычисление градиента работает если x = 0.\n", - "grad_log1pexp(0.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u38MOfz3-MDE", - "colab": {} - }, - "source": [ - "# А также теперь можно рассчитать градиент и при x = 100.\n", - "grad_log1pexp(100.)[0].numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rnZXjfQzqEhV" - }, - "source": [ - "## Улучшение производительности\n", - "\n", - "Вычисления автоматически распределяются на графическом процессоре GPU во время\n", - "работы в режиме eager execution. Если ты хочешь контролировать на каком\n", - "устройстве производить вычисления, то можно просто указать его\n", - "в блоке `tf.device('/gpu:0')` (или другом эквиваленте центрального процессора (далее в тексте и коде - CPU)):" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Ac9Y64H-qEhX", - "colab": {} - }, - "source": [ - "import time\n", - "\n", - "def measure(x, steps):\n", - " # TensorFlow инициализирует GPU во время первого использования,\n", - " # это время исключается из учета производительности.\n", - " tf.matmul(x, x)\n", - " start = time.time()\n", - " for i in range(steps):\n", - " x = tf.matmul(x, x)\n", - " # `tf.matmul` может возвращать значения до завершения процесса умножения\n", - " # матриц (например, эта функция может возвращать значения сразу после\n", - " # постановки операции в очередь в поток CUDA). Вызов x.numpy() ниже\n", - " # удостоверяет, что все операции в очереди были завершены (также копирует\n", - " # результаты в память устройства, с которого производились расчеты,\n", - " # таким образом мы учитываем немного больше времени, чем просто для\n", - " # операции умножения матриц `matmul`).\n", - " _ = x.numpy()\n", - " end = time.time()\n", - " return end - start\n", - "\n", - "shape = (1000, 1000)\n", - "steps = 200\n", - "print(\"Время, требующееся для умножения {} матрицы на себя {} раз:\".format(shape, steps))\n", - "\n", - "# Запускаем на CPU\n", - "with tf.device(\"/cpu:0\"):\n", - " print(\"СPU: {} секунд\".format(measure(tf.random_normal(shape), steps)))\n", - "\n", - "# Если доступен, запускаем на GPU\n", - "if tfe.num_gpus() > 0:\n", - " with tf.device(\"/gpu:0\"):\n", - " print(\"GPU: {} секунд\".format(measure(tf.random_normal(shape), steps)))\n", - "else:\n", - " print(\"GPU: устройство не найдено\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RLw3IS7UqEhe" - }, - "source": [ - "Объект `tf.Tensor` может быть скопирован на другое устройство для\n", - "выполнения своих операций:" - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "ny6LX2BVqEhf", - "colab": {} - }, - "source": [ - "if tf.test.is_gpu_available():\n", - " x = tf.random_normal([10, 10])\n", - "\n", - " x_gpu0 = x.gpu()\n", - " x_cpu = x.cpu()\n", - "\n", - " _ = tf.matmul(x_cpu, x_cpu) # Запускаем на CPU\n", - " _ = tf.matmul(x_gpu0, x_gpu0) # Запускаем на первом GPU:0\n", - "\n", - " if tfe.num_gpus() > 1:\n", - " x_gpu1 = x.gpu(1)\n", - " _ = tf.matmul(x_gpu1, x_gpu1) # Запускаем на втором GPU:1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oA_qaII3-p6c" - }, - "source": [ - "### Тестирование\n", - "\n", - "Для вычисления больших моделей, например таких, как [ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50),\n", - "обучение производится на GPU, а производительность работы в\n", - "eager execution сопоставима с graph execution. Тем не менее этот пробел\n", - "становится еще шире для моделей, где требуется меньше вычислений, и\n", - "где требуется больше работы для оптимизации тех блоков кода, на которые\n", - "приходится основная нагрузка при вычислении. Это особенно актуально\n", - "для моделей с большим количеством малых операций." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TjMTyFIp-sSE" - }, - "source": [ - "## Работаем с графами\n", - "\n", - "В то время как режим eager execution делает разработку и отладку кода\n", - "более интерактивной, graph execution в TensorFlow имеет преимущества\n", - "для распределенного обучения, оптимизации производительности\n", - "и развертывании моделей в продакшене. Однако, при написании кода\n", - "в graph может сильно отличаться от стандартного кода Python, и как\n", - "следствие, может быть более сложным для отладки.\n", - "\n", - "Для построения и обучения моделей в графах, программа Python\n", - "сначала создает граф, который будет представлять расчет, а затем\n", - "вызывает `Session.run` для отправки графа в исполнение в рабочую\n", - "среду на C++. Это включает в себя следующие шаги:\n", - "\n", - "* Автоматическое дифференцирование при помощи статичного autodiff\n", - "* Простое развертывание на платформе независимого сервера\n", - "* Оптимизации на графах (стандартные устранения подвыражений, свертка констант и так далее)\n", - "* Компиляция и слияние ядра\n", - "* Автоматическое распределение и репликация (размещение графов в распределенной системе)\n", - "\n", - "Развертывание кода, написанного для eager execution - более сложная задача:\n", - "либо генерировать граф из модели, либо запустить рабочую среду Python\n", - "и написать код непосредственно на сервер." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hll3ZbE5qEhh" - }, - "source": [ - "### Пишем совместимый код\n", - "\n", - "Один и тот же код, написанный для работы в eager execution также построит\n", - "граф в режиме graph execution. Это можно сделать просто запустив тот же\n", - "код в новой сессии Python, где не активирован eager execution.\n", - "\n", - "Большинство операций TensorFlow работают в eager execution, но есть несколько\n", - "моментов, о которых обязательно нужно помнить:\n", - "\n", - "* Используй `tf.data` для обработки ввода вместо очередей. Это быстрее и проще\n", - "* Используй объектно-ориентированные слои API - например такие, как `tf.keras.layers`\n", - " и `tf.keras.Model` - так как они имеют специальное место для хранения переменных\n", - "* Большинство кода моделей работают одинаково как в eager, так и graph execution,\n", - " однако все-такие есть определенные исключения: например, динамические модели во время\n", - " порядка выполнения Python изменяют вычисления на основе ввода\n", - "* Как только eager execution активирован при помощи `tf.enable_eager_execution`,\n", - " он не может быть выключен. Начни новую сессию Python для возвращения\n", - " к graph execution\n", - "\n", - "Лучше всего писать код сразу для eager и graph execution. Это даст тебе\n", - "возможность для интерактивных экспериментов и возможность отладки кода\n", - "в eager, а также обеспечит лучшую распределенную производительность в режиме\n", - "graph execution.\n", - "\n", - "Пиши код, отлавливай баги, повторяй любые циклы операций в режиме eager\n", - "execution, затем импортируй граф модели для развертывания в продакшене. Используй\n", - "`tf.train.Checkpoint` для сохранения и загрузки переменных моделей, это\n", - "обеспечит легкое передвижение между режимами eager и graph.\n", - "Смотри больше примеров здесь:\n", - "[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sPoSUqmL-ts5" - }, - "source": [ - "### Использование режима eager в graph\n", - "\n", - "Ты можешь выборочно активировать eager execution в режиме graph в TensorFlow\n", - "при помощи `tfe.py_func`. Эту функцию следует использовать когда `tf.enable_eager_execution()`\n", - "*не был вызван*." - ] - }, - { - "cell_type": "code", - "metadata": { - "attributes": { - "classes": [ - "py" - ], - "id": "" - }, - "colab_type": "code", - "id": "Lks-3LB0qEhi", - "colab": {} - }, - "source": [ - "def my_py_func(x):\n", - " x = tf.matmul(x, x) # Ты можешь выполнять операции tf\n", - " print(x) # даже в режиме eager!\n", - " return x\n", - "\n", - "with tf.Session() as sess:\n", - " x = tf.placeholder(dtype=tf.float32)\n", - " # Вызовем функцию eager в режиме graph!\n", - " pf = tfe.py_func(my_py_func, [x], tf.float32)\n", - "\n", - " sess.run(pf, feed_dict={x: [[2.0]]}) # [[4.0]]" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/r1/guide/effective_tf2.md b/site/ru/r1/guide/effective_tf2.md deleted file mode 100644 index f0ac4905790..00000000000 --- a/site/ru/r1/guide/effective_tf2.md +++ /dev/null @@ -1,308 +0,0 @@ -# Как использовать TensorFlow 2.0? - -Note: Вся информация в этом разделе переведена с помощью русскоговорящего -Tensorflow сообщества на общественных началах. Поскольку этот перевод не -является официальным, мы не гарантируем что он на 100% аккуратен и соответствует -[официальной документации на английском языке](https://www.tensorflow.org/?hl=en). -Если у вас есть предложение как исправить этот перевод, мы будем очень рады -увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) -репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow -лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), -напишите нам на -[docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru). - -В TensorFlow 2.0 были сделаны несколько изменений, которые позволят всем пользователям TensorFlow -работать более продуктивно. В TensorFlow 2.0 были удалены -[ненужные API](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md), -существующие и используемые API были проработаны -([объединены RNN](https://github.com/tensorflow/community/blob/master/rfcs/20180920-unify-rnn-interface.md), -[а также оптимизаторы](https://github.com/tensorflow/community/blob/master/rfcs/20181016-optimizer-unification.md)), -улучшена интеграция с рабочей средой Python в режиме -[Eager execution](https://www.tensorflow.org/guide/eager). - -Во многих -[запросах RFC](https://github.com/tensorflow/community/pulls?utf8=%E2%9C%93&q=is%3Apr) -объяснялись основные изменения, которые затронут TensorFlow 2.0. В этом документе -будет показано, как должен выглядеть процесс работы с новым TensorFlow 2.0. -Подразумевается, что вы уже знакомы с TensorFlow 1.x. - -## Краткий список основных изменений - -### Чистка API - -Многие API были либо -[удалены, либо перемещены](https://github.com/tensorflow/community/blob/master/rfcs/20180827-api-names.md) -в TF 2.0. Самыми крупными изменениями являются удаление `tf.app`, `tf.flags`, а также -`tf.logging` в пользу новой библиотеки с открытым исходным кодом -[absl-py](https://github.com/abseil/abseil-py), перемещение проектов, которые были в -`tf.contrib`, а также чистка основного имени `tf.*`: редко используемые функции -были объединены в отдельные модули, например `tf.math`. Некоторые API были заменены -их 2.0 эквивалентами - `tf.summary`, `tf.keras.metrics`, и -`tf.keras.optimizers`. Самый простой способ автоматически переименовать все функции - -это воспользоваться [скриптом для обновления до 2.0](upgrade.md). - -### Активный Eager execution - -В TensorFlow 1.X от пользователя требовалось вручную строить -[абстрактное синтаксическое дерево](https://ru.wikipedia.org/wiki/%D0%90%D0%B1%D1%81%D1%82%D1%80%D0%B0%D0%BA%D1%82%D0%BD%D0%BE%D0%B5_%D1%81%D0%B8%D0%BD%D1%82%D0%B0%D0%BA%D1%81%D0%B8%D1%87%D0%B5%D1%81%D0%BA%D0%BE%D0%B5_%D0%B4%D0%B5%D1%80%D0%B5%D0%B2%D0%BE) (граф) при помощи API вызовов `tf.*`. Для этого было нужно вручную -компилировать дерево, передавая набор получаемых и входящих тензоров к вызову -`session.run()`. TensorFlow 2.0 теперь выполняет все операции мгновенно (точно -так же, как и обычный Python), все графы и сессии теперь будут работать как -стандартное выполнение операций. - -Раньше побочным продуктом eager execution был метод `tf.control_dependencies()`, -который теперь не треубется, и все строки кода будут исполняться в последовательном -порядке, определенном `tf.function`. Сторонний код будет выполняться в -определенном ему порядке. - -### Никаких глобальных переменных - -TensorFlow 1.X внутренне полагался на глобальные переменные. Когда вы вызывали -`tf.Variable()`, то эта переменная помещалась в стандартный граф и оставалась там -даже если уже было изменено имя, присвоенное этой переменной. Вы могли восстановить -эту переменную `tf.Variable`, но только если тебе было известно имя, которое было -присвоено ей при создании. В результате все механизмы были направлены на то, чтобы -помочь пользователю отыскать имя их переменных еще раз: `tf.get_global_step()`, -`tf.global_variables_initializer()`, оптимизаторы, которые рассчитывали -градиенты по всем обучаемым перменным и так далее. В TensorFlow 2.0 эти механизмы были -устранены ([Переменные 2.0 RFC](https://github.com/tensorflow/community/pull/11)) -в пользу нового: следи за состоянием рабочих переменных! Если вы потеряете `tf.Variable` -(например, ей будет присвоено новое имя), то старая будет удалена из памяти в порядке -процесса garbage collection. - -Требование следить за переменными создает дополнительную нагрузку на пользователя, -но в случае с объектами Keras (см. ниже), это нагрузка - минимальная. - -### Функции, а не сессии - -Вызов `session.run()` работал почти как вызов функции: вы определяли вводные данные -и вызывал функцию, получая на выходе результаты. В TensorFlow 2.0, вы можете -декорировать функцию Python при помощи `tf.function()`, отметив её как JIT-компилируемую, -чтобы TensorFlow запустил ее на одном единственном графе -([Функции 2.0 RFC](https://github.com/tensorflow/community/pull/20)). Этот -механизм позволяет TensorFlow 2.0 получить все преимущества режима graph: - -- Производительность: функции могут быть оптимизированы (отсечение узлов графа, - слияние ядра и так далее) -- Портативность: Функции могут быть экспортированы - или импортированы повторно ([Сохранение моделей 2.0 RFC](https://github.com/tensorflow/community/pull/34)), - позволяя пользователям использовать модульные функции TensorFlow и делиться ими. - -```python -# TensorFlow 1.X -outputs = session.run(f(placeholder), feed_dict={placeholder: input}) -# TensorFlow 2.0 -outputs = f(input) -``` - -С новой возможностью просто использовать вместе код Python и TensorFlow, мы ожидаем -что пользователи воспользуются всеми преимуществами выразительности языка Python. -Но портативный TensorFlow выполняет операции в окружении без интерпретатора Python - -на мобильных устройствах, C++ и JavaScript. Чтобы помочь пользователям легко переписать -свой код при использовании новой `@tf.function`, используй [AutoGraph](autograph.ipynb) -для конвертации кода Python в их эквиваленты TensorFlow: - -* `print` -> `tf.print` -* `assert` -> `tf.Assert` -* `for`/`while` -> `tf.while_loop` (поддерживаются `break` и `continue`) -* `if` -> `tf.cond` -* `for _ in dataset` -> `dataset.reduce` - -AutoGraph поддерживает вложенные функции в порядке выполнения программы, что -делает возможным производительно и точно внедрять комплексные программы -машинного обучения, например такие как последовательные модели, обучение с -подкреплением, собственные циклы обучения и многие другие. - -## Рекомендации и идиомы TensorFlow 2.0 - -### Рефакторинг кода на малые функции - -Часто используемый шаблон использования в TensorFlow 1.X работал по принципу -kitchen sink ("кухонной раковины"), когда быа выложена совокупность всех возможных вычислений, -а затем выбранные тензоры вычислялись с помощью `session.run()`. В TensorFlow 2.0 -пользователи должны сами разбивать код на более мелкие функции и вызывать каждую -когда это необходимо. Необязательно декорировать каждую из этих небольших функций -с `tf.function`; используй `tf.function` для декорирования только высокоуровневых -вычислений - например, один шаг обучения или прямого прохода для модели. - -### Используй слои и модели Keras для управления переменными - -Модели и слои Keras предлагают использовать удобное свойство `.variables`, которое -рекурсивно собирает все зависимые переменные. Это значительно облегчает локальное управление -переменными. - -Сравните: - -```python -def dense(x, W, b): - return tf.nn.sigmoid(tf.matmul(x, W) + b) - -@tf.function -def multilayer_perceptron(x, w0, b0, w1, b1, w2, b2 ...): - x = dense(x, w0, b0) - x = dense(x, w1, b1) - x = dense(x, w2, b2) - ... - -# Нам все равно придется управлять w_i и b_i, так как их формы определяются не в коде. -``` - -Версия с использованием Keras: - -```python -# Каждый слой может быть вызван с сигнатурой равной linear(x) -layers = [tf.keras.layers.Dense(hidden_size, activation=tf.nn.sigmoid) for _ in range(n)] -perceptron = tf.keras.Sequential(layers) - -# layers[3].variables => returns [w3, b3] -# perceptron.variables => returns [w0, b0, ...] -``` - -Модели и слои Keras наследуются из `tf.train.Checkpointable` и интегрируются с -`@tf.function`, что делает возможным сохранить или непосредственно экспортировать -сохраненные модели из объектов Keras. Необязательно использовать метод `.fit()` из -Keras API для этих интеграций. - -Вот пример transfer learning (переноса обучения), который демонстрирует как легко -собрать подмножество необходимых переменных с помощью Keras. Предположим, мы обучаем -разветвленную (multi-head) модель с общим корнем (trunk): - -```python -trunk = tf.keras.Sequential([...]) -head1 = tf.keras.Sequential([...]) -head2 = tf.keras.Sequential([...]) - -path1 = tf.keras.Sequential([trunk, head1]) -path2 = tf.keras.Sequential([trunk, head2]) - -# Обучаем на основном датасете: -for x, y in main_dataset: - with tf.GradientTape() as tape: - prediction = path1(x) - loss = loss_fn_head1(prediction, y) - # Одновременно оптимизируем корень и веса первой ветви: - gradients = tape.gradients(loss, path1.variables) - optimizer.apply_gradients(gradients, path1.variables) - -# Настраиваем вторую ветвь, повторно используя корень: -for x, y in small_dataset: - with tf.GradientTape() as tape: - prediction = path2(x) - loss = loss_fn_head2(prediction, y) - # Оптимизируем веса только второй ветви, без весов корня: - gradients = tape.gradients(loss, head2.variables) - optimizer.apply_gradients(gradients, head2.variables) - -# -# Можем сохранить вычисления корня, чтобы другие также могли им воспользоваться. -tf.saved_model.save(trunk, output_path) -``` - -### Объединение tf.data.Datasets и @tf.function - -При обучении модели на данных, которые находятся в памяти, используй стандартный -итератор Python. В остальных случаях `tf.data.Dataset` является лучшим способом -для потока тренировочных данных с диска. Датасеты являются -[итерируемыми (не итераторами)](https://docs.python.org/3/glossary.html#term-iterable), -и работают так же, как и другие итераторы Python в режиме Eager. Вы можете наиболее -полно использовать асинхронные возможности prefetch и stream при помощи -`tf.function()`, которая заменяет итерации Python их эквивалентами операций графов -посредством AutoGraph. - -```python -@tf.function -def train(model, dataset, optimizer): - for x, y in dataset: - with tf.GradientTape() as tape: - prediction = model(x) - loss = loss_fn(prediction, y) - gradients = tape.gradients(loss, model.variables) - optimizer.apply_gradients(gradients, model.variables) -``` - -Если вы используете метод `.fit()` из Keras API, то вам не придется -волноваться об итерации датасета. - -```python -model.compile(optimizer=optimizer, loss=loss_fn) -model.fit(dataset) -``` - -### Воспользуйтесь премиуществами AutoGraph с порядком выполнения Python - -AutoGraph обеспечивает способ конвертации зависимого от данных порядка -выполнения в его эквиваленты в режиме graph, например `tf.cond` и `tf.while_loop`. - -Единственное место, где появляется зависимый от данных порядок выполнения это -последовательные модели. `tf.keras.layers.RNN` использует элемент RNN, позволяя -вам развернуть повтор статически или динамически. Для примера, вы можете -использовать динамическое развертывание: - -```python -class DynamicRNN(tf.keras.Model): - - def __init__(self, rnn_cell): - super(DynamicRNN, self).__init__(self) - self.cell = rnn_cell - - def call(self, input_data): - # [батч, время, параметры] -> [время, батч, параметры] - input_data = tf.transpose(input_data, [1, 0, 2]) - outputs = tf.TensorArray(tf.float32, input_data.shape[0]) - state = self.cell.zero_state(input_data.shape[1], dtype=tf.float32) - for i in tf.range(input_data.shape[0]): - output, state = self.cell(input_data[i], state) - outputs = outputs.write(i, output) - return tf.transpose(outputs.stack(), [1, 0, 2]), state -``` - -Для более детального описание возможностей AutoGraph ознакомьтесь с -[руководством](./autograph.ipynb). - -### Используйте tf.metrics для сбора данных и tf.summary для логов - -Для записи логов, используйте `tf.summary.(scalar|histogram|...)`. Если использовать -данные методы отдельно, то они не будут ничего делать; результаты должны быть -перенаправлены к соответствующему file writer, при помощи контекстного менеджера -(это позволит вам избежать записи логов в file writer). - -```python -summary_writer = tf.summary.create_file_writer('/tmp/summaries') -with summary_writer.as_default(): - tf.summary.scalar('loss', 0.1, step=42) -``` - -Чтобы собрать данные перед записью в лог, используйте `tf.metrics`. Метрики -сохраняют свое состояние; они накапливают значения и возвращают собирательный -результат, когда вы вызываете метод `.result()`. Чтобы очистить все значения -используйте метод `.reset_states()`. - -```python -def train(model, optimizer, dataset, log_freq=10): - avg_loss = tf.keras.metrics.Mean(name='loss', dtype=tf.float32) - for images, labels in dataset: - loss = train_step(model, optimizer, images, labels) - avg_loss.update_state(loss) - if tf.equal(optimizer.iterations % log_freq, 0): - tf.summary.scalar('loss', avg_loss.result(), step=optimizer.iterations) - avg_loss.reset_states() - -def test(model, test_x, test_y, step_num): - loss = loss_fn(model(test_x), test_y) - tf.summary.scalar('loss', loss, step=step_num) - -train_summary_writer = tf.summary.create_file_writer('/tmp/summaries/train') -test_summary_writer = tf.summary.create_file_writer('/tmp/summaries/test') - -with train_summary_writer.as_default(): - train(model, optimizer, dataset) - -with test_summary_writer.as_default(): - test(model, test_x, test_y, optimizer.iterations) -``` - -Указав папку с результами в TensorBoard (`tensorboard --logdir -/tmp/summaries`), вы можете визуализировать полученные в ходе обучения -данные на графиках. - - diff --git a/site/ru/r1/guide/estimators.md b/site/ru/r1/guide/estimators.md deleted file mode 100644 index 7de3c2928de..00000000000 --- a/site/ru/r1/guide/estimators.md +++ /dev/null @@ -1,216 +0,0 @@ -# Estimators - -Note: Вся информация в этом разделе переведена с помощью русскоговорящего -Tensorflow сообщества на общественных началах. Поскольку этот перевод не -является официальным, мы не гарантируем что он на 100% аккуратен и соответствует -[официальной документации на английском языке](https://www.tensorflow.org/?hl=en). -Если у вас есть предложение как исправить этот перевод, мы будем очень рады -увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) -репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow -лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), -напишите нам на -[docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru). - -В этом документе мы познакомимся `tf.estimator`, высокоуровневым API TensorFlow, -который значительно упрощает процесс создания моделей машинного обучения. -Estimators включает в себя следующие операции: - -* обучение -* оценку -* предсказание -* экспорт модели на сервер - -Ты можешь использовать либо уже готовые Estimators или написать свои -собственные для оценки. Все Estimators основаны на классе `tf.estimator.Estimator`. - -Для быстрого ознакомления попробуй запустить [интерактивные уроки по Estimator](../tutorials/estimators/linear.ipynb) -в Google Colab. Чтобы узнать о каждой функции подробнее смотри статью [готовые Estimators](premade_estimators.md). -Для ознакомления с дизайном этого API смотри наш [доклад на arxiv.org](https://arxiv.org/abs/1708.02637). - -Обрати внимание: TensorFlow также включает в себя устаревший класс -`Estimator` в `tf.contrib.learn.Estimator`, который использовать не стоит. - - -## Преимущества Estimators - -Estimators обеспечивают следующие преимущества: - -* Можно запускать модели на основе Estimators локально или на распределенном - сервере без изменений структуры модели. Более того, ты можешь запускать модели - на CPU, GPU и TPU без внесения изменений в код -* С помощью Estimators удобнее делиться своими моделями с другими разработчиками -* Можно разрабатывать современные модели с читаемым высокоуровневым кодом. Проще говоря, - гораздо легче создавать модели с Estimators, чем с низкоуровневым API TensorFlow -* Сами Estimators построены на `tf.keras.layers`, которые упрощают настройку модели - под себя -* Estimators строят граф -* Estimators обеспечивают простой распределенный цикл обучения, который контролирует - как и когда: - - * строить граф - * инициализировать переменные - * загружать данные - * обрабатывать исключения - * создавать контрольные точки и восстанавливаться при неудачных попытках - * сохранять статистику в TensorBoard - -При написании приложения с Estimators ты должен отделять загрузку входных данных -от самой модели. Это разделение упрощает эксперименты с разными наборами данных. - - -## Готовые Estimators - -Готовые Estimators позволяют тебе работать на более высоком уровне, по сравнению -с базовым API TensorFlow. Тебе больше не нужно волноваться о построении вычислительного -графа или сессиях обучения, поскольку Estimators сами делают за тебя всю работу. -Таким образом Estimators сами создают и управляют объектами `tf.Graph` и -`tf.Session`. Более того, готовые Estimators позволяют тебе экспериментировать с -разными архитектурами с минимальными изменениями исходного кода. Например, -`tf.estimator.DNNClassifier` - это готовый класс Estimator, который обучает -классификации модели на основе нейронной сети прямого распространения, которая -состоит из *Dense* слоев. - - -### Структура программ с готовыми Estimators - -Программа TensorFlow на основе готовых Estimators обычно состоит из следующих -четырех шагов: - -1. **Написание одной или более функций для загрузки датасета**. Например, - создадим функцию для импорта тренировочного сета и вторую функцию для - импорта проверочного сета данных. Каждая функция для загрузки датасета - должна возвращать два объекта: - - * словарь, в котором ключи являются именами параметров, а значения - являются тензорами (или *SparseTensors*), содержащие соответствующие - данные параметров - * тензор, содержащий одну или более меток - - Например, в коде ниже показан пример основного скелета для функции ввода - данных: - -```python - def input_fn(dataset): - ... # манипулирует датасетом, извлекая словарь параметров и метки - return feature_dict, label -``` - -Смотри подробнее в статье [Загрузка данных и датасетов](../guide/datasets.md) - -2. **Определение колонок параметров.** Каждая колонка `tf.feature_column` - определяет имя параметра, его тип и любую предварительную обработку - входных данных. Например, в следующем примере кода мы создадим три - колонки параметров, в которых будут храниться данные в формате целых - чисел или чисел с плавающей запятой. Первые две колонки параметров будут - просто идентифицировать имя и тип параметра. Третья колонка параметров указывает - на лямбду-выражение, которое будут вызываться для оценки необработанных - данных: - -```python -# Определим три числовых колонки параметров. -population = tf.feature_column.numeric_column('population') -crime_rate = tf.feature_column.numeric_column('crime_rate') -median_education = tf.feature_column.numeric_column('median_education', - normalizer_fn=lambda x: x - global_education_mean) -``` - -3. **Укажем подходящий готовый Estimator.** Например так мы укажем - готовый Estimator для решения модели `линейного классификатора`: - -```python -# Указываем estimator, передаем колонки параметров. -estimator = tf.estimator.LinearClassifier( - feature_columns=[population, crime_rate, median_education], -) -``` - -4. **Вызов метода обучения, оценки или предсказания** - Например, все Estimators имеют метод `train` для начала обучения модели: - -```python -# `input_fn` - функция, созданная в самом первом шаге -estimator.train(input_fn=my_training_set, steps=2000) -``` - -### Преимущества использования готовых Estimators - -В готовых Estimators используются лучшие практики, которые обеспечивают -следующие преимущества: - -* Лучшие практики для определения какие части вычислительного графа - запускать сначала, а также стратегии для обучения на одном устройстве - или целом кластере -* Стандартизированная практика ведения логов и получения полезной статистики - -Если ты не собираешься использовать готовые Estimators, то тогда тебе -придется указывать все необходимые параметры самому. - - -## Собственные Estimators - -Ядром каждого Estimator, готового или написанного с нуля, является -**функция модели**, которая представляет из себя метод для построения -графа для обучения, оценки и предсказаний. Когда ты используешь готовый -Estimator, кто-то уже написал функцию модели для тебя. В том случае, -когда ты полагаешься на свой собственный Estimator, ты должен сам -написать эту функцию. Более подробно о том, как написать функцию модели -ты можешь узнать в статье [Написание собственных Estimators](../guide/custom_estimators.md) - - -## Рекомендуемый ход работы - -Мы рекомендуем следующий порядок создания модели с помощью Estimators: - -1. Предположим, что есть готовый Estimator, и мы используем его для - построения нашей модели, а также используем результаты оценки для - формирования эталонной модели -2. Создаем и тестируем процесс загрузки данных, проверяем целостность и - надежность наших данных с готовым Estimator -3. Если есть другие подходящие альтернативы, тогда экспериментируем с ними - для поиска Estimator, который покажет лучшие результаты -4. Возможно потребуется улучшить нашу модель при помощи создания нашего - собственного Estimator. - - -## Создание Estimators из моделей Keras - -Ты можешь конвертировать уже имеющиеся у тебя модели Keras в Estimators. Это позволит -тебе использовать все преимущества Estimators для данной модели, например, для распределенного -обучения. Вызови функцию `tf.keras.estimator.model_to_estimator` как в примере ниже: - -```python -# Создаем модель Inception v3 в Keras: -keras_inception_v3 = tf.keras.applications.inception_v3.InceptionV3(weights=None) - -# Компилируем модель с оптимизатором, функцией потерь и метриками обучения по выбору. -keras_inception_v3.compile(optimizer=tf.keras.optimizers.SGD(lr=0.0001, momentum=0.9), - loss='categorical_crossentropy', - metric='accuracy') - -# Создаем Estimator из скомпилированной модели Keras. Обрати внимание, что изначальное -# состояние модели Keras сохраняется при создании Estimator. -est_inception_v3 = tf.keras.estimator.model_to_estimator(keras_model=keras_inception_v3) - -# Теперь мы можем использовать полученный Estimator как любой другой. -# Для начала восстановим вводное имя (или имена) модели Keras, чтобы мы могли использовать их -# как имена колонок параметров функции ввода данных Estimator: -keras_inception_v3.input_names # выводит: ['input_1'] - -# Как только мы получим вводные имена, мы можем создать функцию ввода данных, например, -# для входа данных в формате NumPy ndarray: -train_input_fn = tf.estimator.inputs.numpy_input_fn( - x={"input_1": train_data}, - y=train_labels, - num_epochs=1, - shuffle=False) - -# Для обучения вызываем функцию `train` полученного нами Estimator: -est_inception_v3.train(input_fn=train_input_fn, steps=2000) -``` - -Обрати внимание, что имена колонок параметров и меток Esitmator мы получили -из соответствующей модели Keras. Например, имена вводных ключей для `train_input_fn` -выше могут быть получены из `keras_inception_v3.input_names`, и таким же образом -предсказанные имена могут быть получены из `keras_inception_v3.output_names`. - -Подробнее смотри документацию в статье `tf.keras.estimator.model_to_estimator`. diff --git a/site/ru/r1/guide/keras.ipynb b/site/ru/r1/guide/keras.ipynb deleted file mode 100644 index 6f06c7e21bf..00000000000 --- a/site/ru/r1/guide/keras.ipynb +++ /dev/null @@ -1,1307 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "keras.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCQY7jpBfMur" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "z6X9omPnfO_h", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F1xIRPtY0E1w" - }, - "source": [ - "# Keras" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VyOjQZHhZxaA" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Запусти в Google Colab\n", - " \n", - " Изучай код на GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JWjueNizcxUU", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ViAXWoKlZ8s6" - }, - "source": [ - "Keras - это высокоуровневый API для создания моделей глубокого обучения.\n", - "Он используется для быстрого создания прототипов, сложных исследований,\n", - "а также для создания приложений. Три ключевые примущества Keras API:\n", - "\n", - "- *Простота в использовании*
    \n", - " Keras имеет простой интерфейс, оптимизированный для большинства\n", - " распространенных задач глубокого обучения. Также он дает конкретные подсказки как быстро\n", - " исправить возможные ошибки\n", - "- *Модульность*
    \n", - " Модели Keras строятся при помощи объединения нескольких простых модулей, каждый из которых\n", - " может быть настроен независимо, и не накладывает каких либо значительных ограничений\n", - "- *Легко расширить модель*
    Ты можешь создавать свои собственные модули,\n", - " чтобы свободно выражать свои идеи для исследования. Создавай новые слои,\n", - " функции потерь и разрабатывай современные модели глубокого обучения" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IsK5aF2xZ-40" - }, - "source": [ - "## Импортируем tf.keras\n", - "\n", - "`tf.keras` - это реализация [спецификации Keras API](https://keras.io){:.external} в TensorFlow.\n", - "Это высокоуровневый API для построения\n", - "моделей глубокого обучения с первоклассной поддержкой\n", - "функционала TensorFlow, например [eager execution](#eager_execution),\n", - "конвееры `tf.data` и алгоритмы оценки [Estimators](./estimators.md).\n", - "`tf.keras` делает TensorFlow простым в использовании, не теряя в гибкости и производительности.\n", - "\n", - "Чтобы начать, просто импортируй `tf.keras` после TensorFlow в начале кода:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qoc055N3wiUG", - "colab": {} - }, - "source": [ - "# Используй pyyaml если будешь сохранять в формате YAML\n", - "!pip install pyyaml" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TgPcBFru0E1z", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.VERSION)\n", - "print(tf.keras.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lj03RamP0E13" - }, - "source": [ - "`tf.keras` может запускать любой совместимый с Keras код, но тем не менее помни:\n", - "\n", - "* Версия `tf.keras` в последнем релизе TensorFlow может быть не самой\n", - "последнией версией `keras`, доступной в PyPI. Всегда проверяй версию при помощи `tf.keras.__version__`\n", - "* Когда ты [сохраняешь только веса модели](#weights_only), `tf.keras` по умолчанию\n", - " сохраняет их в [формате контрольной точки](./checkpoints.md). Используй `save_format='h5'`\n", - " чтобы сохранять как HDF5" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_WUw7R_zfpIh" - }, - "source": [ - "## Построим простую модель" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7e1LPcXx0gR6" - }, - "source": [ - "### Последовательная модель\n", - "\n", - "В Keras мы используем *слои* для построения *моделей*. Обычно модель - это граф, состоящий из нескольких слоев. Самый распространенный тип модели это стэк идущих друг за другом слоев - последовательная модель `tf.keras.Sequential`.\n", - "\n", - "Для начала давай построим простую полносвязную сеть (или многослойный перцептрон):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WM-DUVQB0E14", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()\n", - "# Добавим в нашу модель слой Dense из 64 блоков:\n", - "model.add(layers.Dense(64, activation='relu'))\n", - "# И еще один:\n", - "model.add(layers.Dense(64, activation='relu'))\n", - "# Также добавим слой softmax с 10 выводящими блоками:\n", - "model.add(layers.Dense(10, activation='softmax'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ztyTipu0E18" - }, - "source": [ - "### Конфигурация слоев\n", - "\n", - "Существует много разных слоев `tf.keras.layers` с общими параметрами конфигурации:\n", - "\n", - "* `activation`: Устанавливает функцию активации для данного слоя. Этот параметр должен указываться в имени встроенной функции или использоваться как вызываемый объект. По умолчанию активация не используется\n", - "* `kernel_initializer` и `bias_initializer`: Инициализация, которая создает веса данного слоя (ядро kernel и смещение bias). Этот параметр используется как имя или вызываемый объект. По умолчанию используется инициализатор `\"Glorot uniform\"`\n", - "* `kernel_regularizer` и `bias_regularizer`: Регуляризация, которая применяется к весам слоя (ядро kernel и смещение bias), например L1 или L2. По умолчанию не используется\n", - "\n", - "Следующий код используется для построения `tf.keras.layers.Dense` с настройкой конфигурации каждого слоя:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MlL7PBtp0E19", - "colab": {} - }, - "source": [ - "# Создаем сигмоидный слой:\n", - "layers.Dense(64, activation='sigmoid')\n", - "# Или:\n", - "layers.Dense(64, activation=tf.sigmoid)\n", - "\n", - "# Линейный слой с регуляризацией L1 с коэффицентом 0.01 примененная к ядру матрицы:\n", - "layers.Dense(64, kernel_regularizer=tf.keras.regularizers.l1(0.01))\n", - "\n", - "# Линейный слой с регуляризацией L2 с коэффицентом 0.01 примененная к вектору смещения (bias):\n", - "layers.Dense(64, bias_regularizer=tf.keras.regularizers.l2(0.01))\n", - "\n", - "# Линейный слой с ядром, инициализированным в случайной прямоугольный матрице:\n", - "layers.Dense(64, kernel_initializer='orthogonal')\n", - "\n", - "# Линейный слой с вектором смещения, инициализированным с коэффицентом 2:\n", - "layers.Dense(64, bias_initializer=tf.keras.initializers.constant(2.0))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9NR6reyk0E2A" - }, - "source": [ - "## Обучение и оценка\n", - "\n", - "### Настроим конфигурацию обучения\n", - "\n", - "После того как модель построена давай обозначим процесс обучения с помощью метода `compile`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sJ4AOn090E2A", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - "# Добавим в нашу модель слой `Dense` из 64 блоков:\n", - "layers.Dense(64, activation='relu', input_shape=(32,)),\n", - "# И еще один:\n", - "layers.Dense(64, activation='relu'),\n", - "# Также добавим слой softmax с 10 выводящими блоками:\n", - "layers.Dense(10, activation='softmax')])\n", - "\n", - "model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HG-RAa9F0E2D" - }, - "source": [ - "У `tf.keras.Model.compile` есть три важных аргумента:\n", - "\n", - "* `optimizer`: Этот объект определяет процедуру обучения. Мы будем использовать оптимизаторы из модуля `tf.train`, например такие как `tf.train.AdamOptimizer`, `tf.train.RMSPropOptimizer` и `tf.train.GradientDescentOptimizer`\n", - "* `loss`: Эта функция минимизации для решения задач оптимизации. Самыми распространенными выборами этого аргумента являются среднеквадратическое отклонение (`mse`), `categorical_crossentropy`, и `binary_crossentropy`. Функции потерь обозначены по имени или используются как вызываемые объекты из модуля `tf.keras.losses`\n", - "* `metrics`: Используются для наблюдения за процессом обучения. Используются как строки или вызываемые объекты модуля `tf.keras.metrics`\n", - "\n", - "Ниже пример конфигурации модели для обучения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "St4Mgdar0E2E", - "colab": {} - }, - "source": [ - "# Настраиваем регрессию модели, используем среднеквадратическое отклонение\n", - "model.compile(optimizer=tf.train.AdamOptimizer(0.01),\n", - " loss='mse', # среднеквадратическое отклонение\n", - " metrics=['mae']) # средняя абсолютная ошибка\n", - "\n", - "# Настраиваем модель для классификации по категориям\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.01),\n", - " loss=tf.keras.losses.categorical_crossentropy,\n", - " metrics=[tf.keras.metrics.categorical_accuracy])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yjI5rbi80E2G" - }, - "source": [ - "### Используем данные NumPy\n", - "\n", - "При работе с небольшими датасетами мы будем использовать загружаемые в память массивы данных [NumPy](https://www.numpy.org/){:.external} для обучения и оценки моделей. Модель будет \"подстраиваться\" под тренировочные данные при помощи метода `fit`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3CvP6L-m0E2I", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "data = np.random.random((1000, 32))\n", - "labels = np.random.random((1000, 10))\n", - "\n", - "model.fit(data, labels, epochs=10, batch_size=32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N-pnVaFe0E2N" - }, - "source": [ - "У `tf.keras.Model.fit` есть три важных аргумента:\n", - "\n", - "* `epochs`: Процесс обучения структурирован по *эпохам*. Эпоха означает один проход по всему набору входных данных, который происходит небольшими порциями в батчах\n", - "* `batch_size`: Когда мы используем массивы NumPy модель делит данные на небольшие батчи и затем выполняет указанное количество проходов. Это число определяет размер батча. Помни, что последний батч может быть меньше если общее количество образцов данных не делится на размер батча\n", - "* `validation_data`: Когда мы строим прототип модели, мы хотим легко наблюдать за ее точностью на проверочных данных. При помощи данного аргумента - обычно кортеж или метка - модель будет отображать потери и статистику в режиме выводов inference для прошедших через модель данных в конце каждой эпохи\n", - "\n", - "Вот пример использования `validation_data`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gFcXzVQa0E2N", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "data = np.random.random((1000, 32))\n", - "labels = np.random.random((1000, 10))\n", - "\n", - "val_data = np.random.random((100, 32))\n", - "val_labels = np.random.random((100, 10))\n", - "\n", - "model.fit(data, labels, epochs=10, batch_size=32,\n", - " validation_data=(val_data, val_labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-6ImyXzz0E2Q" - }, - "source": [ - "### Используем датасеты tf.data\n", - "\n", - "Чтобы обучать модель на больших датасетах или на нескольких устройствах мы можем воспользоваться [Datasets API](./datasets.md). Просто добавь `tf.data.Dataset` к методу `fit`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OziqhpIj0E2R", - "colab": {} - }, - "source": [ - "# Инициализируем пробную инстанцию датасета:\n", - "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n", - "dataset = dataset.batch(32)\n", - "dataset = dataset.repeat()\n", - "\n", - "# Не забудь указать количество шагов в каждой эпохе `steps_per_epoch` при использовании метода `fit`\n", - "model.fit(dataset, epochs=10, steps_per_epoch=30)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "I7BcMHkB0E2U" - }, - "source": [ - "Таким образом, метод `fit` использует аргумент `steps_per_epoch` - это количество шагов обучения, которые модель должна сделать прежде, чем перейти на следующую эпоху. Поскольку `Dataset` принимает батчи данных, то в этом примере кода нам не требуется указывать размер батча в `batch_size`.\n", - "\n", - "Также `dataset` можно использовать для проверки точности модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YPMb3A0N0E2V", - "colab": {} - }, - "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((data, labels))\n", - "dataset = dataset.batch(32).repeat()\n", - "\n", - "val_dataset = tf.data.Dataset.from_tensor_slices((val_data, val_labels))\n", - "val_dataset = val_dataset.batch(32).repeat()\n", - "\n", - "model.fit(dataset, epochs=10, steps_per_epoch=30,\n", - " validation_data=val_dataset,\n", - " validation_steps=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IgGdlXso0E2X" - }, - "source": [ - "### Оценка и предсказание\n", - "\n", - "Методы `tf.keras.Model.evaluate` и `tf.keras.Model.predict` могут использовать данные NumPy и `tf.data.Dataset`.\n", - "\n", - "Используй следующий пример кода для оценки потерь и других показателей предсказаний на использованных данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mhDbOHEK0E2Y", - "colab": {} - }, - "source": [ - "data = np.random.random((1000, 32))\n", - "labels = np.random.random((1000, 10))\n", - "\n", - "model.evaluate(data, labels, batch_size=32)\n", - "\n", - "model.evaluate(dataset, steps=30)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UXUTmDfb0E2b" - }, - "source": [ - "Для *предсказания* вывода последнего слоя как массив NumPy, используй следующий код:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9e3JsSoQ0E2c", - "colab": {} - }, - "source": [ - "result = model.predict(data, batch_size=32)\n", - "print(result.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dmj899x4hTGT" - }, - "source": [ - "## Построение сложных моделей" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fzEOW4Cn0E2h" - }, - "source": [ - "### Функциональный API\n", - "\n", - "Последовательная модель `tf.keras.Sequential` представляет из себя обычное наложение или стек слоев, которые не могут представлять произвольные модели. Используй\n", - "[функциональный API Keras ](https://keras.io/getting-started/functional-api-guide/){:.external} для построения комплексных топологий моделей, например таких как:\n", - "\n", - "* Модели с множественными входами данных\n", - "* Модели с множественными выводами данных\n", - "* Модели с общими слоями, где один и тот же слой вызывается несколько раз\n", - "* Модели с непоследовательным потоком данных, например где есть остаточные связи\n", - "\n", - "Построение модели при помощи функционального API выглядит следующим образом:\n", - "\n", - "1. Слой вызывается и возвращает тензор\n", - "2. Для определения `tf.keras.Model` используем входные и выводящие тензоры\n", - "3. Модель обучается точно так же, как и `Sequential`\n", - "\n", - "Следующий пример показывает как использовать функциональный API для построения простой полносвязной сети:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mROj832r0E2i", - "colab": {} - }, - "source": [ - "# Возвращает тензор-\"заполнитель\", который мы используем в качестве примера\n", - "inputs = tf.keras.Input(shape=(32,))\n", - "\n", - "# Вызываемый на тензор слой, возвращает тензор\n", - "x = layers.Dense(64, activation='relu')(inputs)\n", - "x = layers.Dense(64, activation='relu')(x)\n", - "predictions = layers.Dense(10, activation='softmax')(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AFmspHeG1_W7" - }, - "source": [ - "Обозначим вход и вывод данных нашей модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5k5uzlyu16HM", - "colab": {} - }, - "source": [ - "model = tf.keras.Model(inputs=inputs, outputs=predictions)\n", - "\n", - "# При помощи метода `compile` настраиваем конфигурацию обучения\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# Тренируем модель в течение 5 эпох\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EcKSLH3i0E2k" - }, - "source": [ - "### Создание подклассов моделей\n", - "\n", - "Также ты можешь создать модель с нуля при помощи подклассов в `tf.keras.Model` и определения собственных прямых проходов. Создавай слои в методе `__init__` и установи их как атрибуты класса. Прямой проход должен быть указан в методе `call`.\n", - "\n", - "Создание подклассов моделей особенно полезно, когда активирован [eager execution](./eager.md), так как прямой проход в этом случае всегда будет записан.\n", - "\n", - "Ключевой момент: всегда используй правильный API для решения конкретной задачи. Поскольку создание подклассов модели предполагает определенную гибкость, эта гибкость осложняет определение структуры и может повлечь больше ошибок у пользователя. Если возможно, то всегда старайся использовать функциональный API.\n", - "\n", - "Смотри следующий пример кода, в котором мы будем использовать подклассы `tf.keras.Model` и собственный прямой проход:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KLiHWzcn2Fzk", - "colab": {} - }, - "source": [ - "class MyModel(tf.keras.Model):\n", - "\n", - " def __init__(self, num_classes=10):\n", - " super(MyModel, self).__init__(name='my_model')\n", - " self.num_classes = num_classes\n", - " # Здесь определим слои\n", - " self.dense_1 = layers.Dense(32, activation='relu')\n", - " self.dense_2 = layers.Dense(num_classes, activation='sigmoid')\n", - "\n", - " def call(self, inputs):\n", - " # А здесь укажем прямой проход,\n", - " # используя указанные выше слои (в `__init__`).\n", - " x = self.dense_1(inputs)\n", - " return self.dense_2(x)\n", - "\n", - " def compute_output_shape(self, input_shape):\n", - " # Если ты хочешь использовать подклассы модели\n", - " # в функциональном стиле, тогда эта функция будет переписана\n", - " # во время запуска кода. В остальных случаях этот метод необязателен.\n", - " shape = tf.TensorShape(input_shape).as_list()\n", - " shape[-1] = self.num_classes\n", - " return tf.TensorShape(shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ShDD4fv72KGc" - }, - "source": [ - "Укажем класс новой модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "42C-qQHm0E2l", - "colab": {} - }, - "source": [ - "model = MyModel(num_classes=10)\n", - "\n", - "# При помощи метода `compile` настраиваем конфигурацию обучения\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# Обучаем в течение 5 эпох\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yqRQiKj20E2o" - }, - "source": [ - "### Создание собственных слоев\n", - "\n", - "Чтобы создать свой собственный слой при помощи подклассов `tf.keras.layers.Layer` нам потребуются следующие методы:\n", - "\n", - "* `build`: Создает веса слоя. Чтобы добавить веса в модель используй метод `add_weight`\n", - "* `call`: Определяет прямой проход\n", - "* `compute_output_shape`: Определяет как вычислить выводящую форму слоя для данной входной формы\n", - "* Также слой можно сериализировать при помощи метода `get_config` и метода класса `from_config`\n", - "\n", - "Вот пример использования нового слоя, в котором мы использовали `matmul` входа с матрицей ядра `kernel`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "l7BFnIHr2WNc", - "colab": {} - }, - "source": [ - "class MyLayer(layers.Layer):\n", - "\n", - " def __init__(self, output_dim, **kwargs):\n", - " self.output_dim = output_dim\n", - " super(MyLayer, self).__init__(**kwargs)\n", - "\n", - " def build(self, input_shape):\n", - " shape = tf.TensorShape((input_shape[1], self.output_dim))\n", - " # Создаем обучаемую переменную весов для этого слоя\n", - " self.kernel = self.add_weight(name='kernel',\n", - " shape=shape,\n", - " initializer='uniform',\n", - " trainable=True)\n", - " # Обязательно вызови метод `build` в конце\n", - " super(MyLayer, self).build(input_shape)\n", - "\n", - " def call(self, inputs):\n", - " return tf.matmul(inputs, self.kernel)\n", - "\n", - " def compute_output_shape(self, input_shape):\n", - " shape = tf.TensorShape(input_shape).as_list()\n", - " shape[-1] = self.output_dim\n", - " return tf.TensorShape(shape)\n", - "\n", - " def get_config(self):\n", - " base_config = super(MyLayer, self).get_config()\n", - " base_config['output_dim'] = self.output_dim\n", - " return base_config\n", - "\n", - " @classmethod\n", - " def from_config(cls, config):\n", - " return cls(**config)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8wXDRgXV2ZrF" - }, - "source": [ - "Теперь создадим модель, используя наш новый слой:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uqH-cY0h0E2p", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " MyLayer(10),\n", - " layers.Activation('softmax')])\n", - "\n", - "# При помощи метода `compile` настраиваем конфигурацию обучения\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "# Обучаем в течение 5 эпох\n", - "model.fit(data, labels, batch_size=32, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Lu8cc3AJ0E2v" - }, - "source": [ - "## Функции обратного вызова\n", - "\n", - "Функция обратного вызова `callback` - это объект, который передается модели для обработки и расширения ее поведения во время обучения. Ты можешь написать свою собственную функцию callback, или использовать готовые `tf.keras.callbacks` в которые входят:\n", - "\n", - "* `tf.keras.callbacks.ModelCheckpoint`: Сохраняет контрольные точки твоей модели через заданный интервал\n", - "* `tf.keras.callbacks.LearningRateScheduler`: Замедляет темп обучения `learning rate` для получения лучших результатов точности\n", - "* `tf.keras.callbacks.EarlyStopping`: Останавливает обучение как только точность перестает увеличиваться\n", - "* `tf.keras.callbacks.TensorBoard`: Следит за поведением модели при помощи [TensorBoard](https://tensorflow.org/tensorboard)\n", - "\n", - "Чтобы использовать `tf.keras.callbacks.Callback` просто передай его в метод `fit` своей модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rdYwzSYV0E2v", - "colab": {} - }, - "source": [ - "callbacks = [\n", - " # Прерывает обучение если потери при проверке `val_loss` перестают\n", - " # уменьшаться после 2 эпох\n", - " tf.keras.callbacks.EarlyStopping(patience=2, monitor='val_loss'),\n", - " # Записываем логи TensorBoard в папку `./logs`\n", - " tf.keras.callbacks.TensorBoard(log_dir='./logs')\n", - "]\n", - "model.fit(data, labels, batch_size=32, epochs=5, callbacks=callbacks,\n", - " validation_data=(val_data, val_labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GK87-Dhv2CCN" - }, - "source": [ - "\n", - "## Сохранение и загрузка" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qnl7K-aI0E2z" - }, - "source": [ - "### Сохраняем веса\n", - "\n", - "Ты можешь сохранять и загружать веса модели при помощи `tf.keras.Model.save_weights`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uQIANjB94fLB", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - "layers.Dense(64, activation='relu'),\n", - "layers.Dense(10, activation='softmax')])\n", - "\n", - "model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4eoHJ-ny0E21", - "colab": {} - }, - "source": [ - "# Сохраняем веса в контрольную точку формата TensorFlow\n", - "model.save_weights('./weights/my_model')\n", - "\n", - "# Восстанавливаем состояние модели,\n", - "# требуется использование модели с точно такой же архитектурой\n", - "model.load_weights('./weights/my_model')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u25Id3xe0E25" - }, - "source": [ - "По умолчанию, веса модели сохраняются в формате [контрольой точки TensorFlow](./checkpoints.md). Веса также могут быть сохранены в формате Keras HDF5, который является стандартным в бэкенд структуре Keras:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JSAYoFEd0E26", - "colab": {} - }, - "source": [ - "# Сохраняем веса в файл HDF5\n", - "model.save_weights('my_model.h5', save_format='h5')\n", - "\n", - "# Загружаем текущее состояние модели\n", - "model.load_weights('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mje_yKL10E29" - }, - "source": [ - "### Сохраняем конфигурацию\n", - "\n", - "Конфигурация модели также может быть сохранена: такой метод сериализирует архитектуру модели без сохранения весов.\n", - "\n", - "Сохраненная конфигурация можеть быть загружена и инициализирована как оригинальная модель, даже без кода изначальной модели.\n", - "\n", - "Keras поддерживает форматы сериализации данных JSON и YAML:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EbET0oJTzGkq", - "colab": {} - }, - "source": [ - "# Сериализация модели в формат JSON\n", - "json_string = model.to_json()\n", - "json_string" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pX_badhH3yWV", - "colab": {} - }, - "source": [ - "import json\n", - "import pprint\n", - "pprint.pprint(json.loads(json_string))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q7CIa05r4yTb" - }, - "source": [ - "Давай воссоздадим только что инициализированную модель из JSON:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J9UFv9k00E2_", - "colab": {} - }, - "source": [ - "fresh_model = tf.keras.models.model_from_json(json_string)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t5NHtICh4uHK" - }, - "source": [ - "Сериализация модели в формат YAML требуется установки `pyyaml` *до импорта TensorFlow*:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aj24KB3Z36S4", - "colab": {} - }, - "source": [ - "yaml_string = model.to_yaml()\n", - "print(yaml_string)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O53Kerfl43v7" - }, - "source": [ - "Восстановим модель из формата YAML:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "77yRuwg03_MG", - "colab": {} - }, - "source": [ - "fresh_model = tf.keras.models.model_from_yaml(yaml_string)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xPvOSSzM0E3B" - }, - "source": [ - "Важно: модели с подклассами не могут быть сериализированы, потому что их архитектура определяется кодом Python в методе `call`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iu8qMwld4-71" - }, - "source": [ - "### Сохраняем модель полностью\n", - "\n", - "Мы можем сохранить модель целиком в один файл, который будет содержать веса, конфигурацию модели и даже настройки оптимизатора. Это позволяет сохранить модель как контрольную точку и продолжить обучение позже - ровно с того же момента и без доступа к исходному коду." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "45oNY34Z0E3C", - "colab": {} - }, - "source": [ - "# Создаем простую модель\n", - "model = tf.keras.Sequential([\n", - " layers.Dense(10, activation='softmax', input_shape=(32,)),\n", - " layers.Dense(10, activation='softmax')\n", - "])\n", - "model.compile(optimizer='rmsprop',\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "model.fit(data, labels, batch_size=32, epochs=5)\n", - "\n", - "# Сохраняем модель целиком в один файл формата HDF5\n", - "model.save('my_model.h5')\n", - "\n", - "# Восстаналиваем ту же самую модель, включая веса и оптимизатор\n", - "model = tf.keras.models.load_model('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PMOWhDOB0E3E" - }, - "source": [ - "## Eager execution\n", - "\n", - "[Eager execution](./eager.md) - это окружение императивного программирования, в котором все операции вычисляются мгновенно. В нем не требуется Keras, однако `tf.keras` поддерживается и оказывается весьма полезен для проверки программы и отладки кода.\n", - "\n", - "Все API `tf.keras` для построения моделей совместимы с eager execution. В то время как функциональный API и `Sequential` могут быть использованы и здесь, наибольшее преимущество получат *модели с подклассами* и *модели с собственными слоями* - это именно те API, в которых тебе необходимо указать прямой проход в виде кода (вместо тех API, которые создают модели посредством сборки существующих слоев).\n", - "\n", - "Смотри [Руководство по eager execution](./eager.md#build_a_model) для ознакомления с примерами использования моделей Keras с уникальными циклами обучения `tf.GradientTape`." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H1jB-t138gQc" - }, - "source": [ - "## Распределенное обучение" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2wG3NVco5B5V" - }, - "source": [ - "### Алгоритмы оценки Estimators\n", - "\n", - "\n", - "API [Estimators](./estimators.md) используется для обучения моделей в распределенном окружении. Он необходим для решения задач распределенного обучения на больших датасетах например для экспорта модели в продакшен.\n", - "\n", - "`tf.keras.Model` может обучаться с API `tf.estimator` посредством конвертации модели в объект `tf.estimator.Estimator` при помощи `tf.keras.estimator.model_to_estimator`. Читай больше в статье [Создание Estimators из моделей Keras](./estimators.md#creating_estimators_from_keras_models)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cVg0vfTO0E3E", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([layers.Dense(10,activation='softmax'),\n", - " layers.Dense(10,activation='softmax')])\n", - "\n", - "model.compile(optimizer=tf.train.RMSPropOptimizer(0.001),\n", - " loss='categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - "estimator = tf.keras.estimator.model_to_estimator(model)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S7FKvikO0E3H" - }, - "source": [ - "Совет: используй [eager execution](./eager.md) для отладки\n", - "[функций входа Estimator](./premade_estimators.md#create_input_functions) и проверки данных." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PJZ6e9J5JHF" - }, - "source": [ - "### Обучение на нескольких GPU\n", - "\n", - "Модели `tf.keras` могут обучаться на нескольких GPU при помощи\n", - "`tf.contrib.distribute.DistributionStrategy`. Этот API обеспечивает распределенное обучение на нескольких GPU почти без изменений основного кода модели.\n", - "\n", - "В настоящее время `tf.contrib.distribute.MirroredStrategy` является единственной поддерживаемой стратегией распределенного обучения. `MirroredStrategy` выполняет внутриграфную репликацию с синхронным обучением, используя функцию all-reduce на одном устройстве. Чтобы использовать `DistributionStrategy` вместе с Keras, конвертируй модель `tf.keras.Model` в `tf.estimator.Estimator` при помощи `tf.keras.estimator.model_to_estimator`, а затем обучи получившийся estimator.\n", - "\n", - "В следующем примере мы посмотрим как распределить `tf.keras.Model` на нескольких GPU на одном устройстве.\n", - "\n", - "Для начала определим простую модель:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sbaRr7g-0E3I", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()\n", - "model.add(layers.Dense(16, activation='relu', input_shape=(10,)))\n", - "model.add(layers.Dense(1, activation='sigmoid'))\n", - "\n", - "optimizer = tf.train.GradientDescentOptimizer(0.2)\n", - "\n", - "model.compile(loss='binary_crossentropy', optimizer=optimizer)\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yw4hSJme0E3L" - }, - "source": [ - "Затем определим функцию *загрузки и обработки данных*. Функция `Input_fn` возвращает объект `tf.data.Dataset`, который используется для распределения данных на нескольких устройствах, где каждый GPU обрабатывает свой входящий батч." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CxJW1QMH0E3L", - "colab": {} - }, - "source": [ - "def input_fn():\n", - " x = np.random.random((1024, 10))\n", - " y = np.random.randint(2, size=(1024, 1))\n", - " x = tf.cast(x, tf.float32)\n", - " dataset = tf.data.Dataset.from_tensor_slices((x, y))\n", - " dataset = dataset.repeat(10)\n", - " dataset = dataset.batch(32)\n", - " return dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rO9MiL6X0E3O" - }, - "source": [ - "Далее, создадим `tf.estimator.RunConfig` и передадим аргумент `train_distribute`\n", - "к `tf.contrib.distribute.MirroredStrategy`. При создании\n", - "`MirroredStrategy` ты можешь определить список устройств или передать аргумент `num_gpus` с заданным количеством GPU для обучения. По умолчанию используются все доступные GPU как в следующем примере:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BEwFq4PM0E3P", - "colab": {} - }, - "source": [ - "strategy = tf.contrib.distribute.MirroredStrategy()\n", - "config = tf.estimator.RunConfig(train_distribute=strategy)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TcnwYVun0E3R" - }, - "source": [ - "Конвертируем модель Keras в `tf.estimator.Estimator`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VSvbuIID0E3S", - "colab": {} - }, - "source": [ - "keras_estimator = tf.keras.estimator.model_to_estimator(\n", - " keras_model=model,\n", - " config=config,\n", - " model_dir='/tmp/model_dir')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N6BXU5F90E3U" - }, - "source": [ - "Наконец, обучаем `Estimator`, передав аргументы `input_fn` и `steps`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKoJ2wUH0E3U", - "colab": {} - }, - "source": [ - "keras_estimator.train(input_fn=input_fn, steps=10)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/r1/guide/tensors.md b/site/ru/r1/guide/tensors.md deleted file mode 100644 index 1377806447a..00000000000 --- a/site/ru/r1/guide/tensors.md +++ /dev/null @@ -1,397 +0,0 @@ -# Тензоры - -Note: Вся информация в этом разделе переведена с помощью русскоговорящего -Tensorflow сообщества на общественных началах. Поскольку этот перевод не -является официальным, мы не гарантируем что он на 100% аккуратен и соответствует -[официальной документации на английском языке](https://www.tensorflow.org/?hl=en). -Если у вас есть предложение как исправить этот перевод, мы будем очень рады -увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) -репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow -лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), -напишите нам на -[docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru). - -Как понятно из названия, TensorFlow ("поток тензоров") - это фреймворк для определения -и вычисления операций с тензорами. Тензор - это обобщенное название векторов и матриц, -вплоть до потенциально высоких размерностей. Во внутренней структуре TensorFlow -векторы представлены как n-размерные массивы примитивных типов данных. - -При написании программ в TensorFlow ключевым объектом всех операций является -`tf.Tensor`. Этот объект представляет собой частично определенное вычисление, -которое в конечном итоге выдаст какое-либо значение. Программы TensorFlow -сначала строят граф объектов `tf.Tensor`, детализирует как каждый тензор будет -вычисляться на других доступных тензорах, а затем запускает построение этого -графа для получения желаемых результатов. - -Объект `tf.Tensor` имеет следующие параметры: - - - * тип данных (например`float32`, `int32`, или `string`) - * форму (a shape) - - -Каждый элемент в тензоре имеет одинаковый тип данных, и этот тип всегда известен. -Размерность, которая определяется количеством размерностей и размером каждого -массива, может быть известна частично. Большинство операций производят -тензоры полностью известных размерностей, если эти входные размерности также -известны, но в некоторых случаях узнать размерность тензора можно только в режиме -graph execution. - -Некоторые типы тензоров являются специализированными, и будут описаны в -других статьях руководства по TensorFlow. Ключевыми являются: - - * `tf.Variable` - * `tf.constant` - * `tf.placeholder` - * `tf.SparseTensor` - - -За исключением `tf.Variable`, значение тензора является неизменным, что означает -в контексте единичного вычисления тензор будет иметь всего одно значение. Однако, -при вычислении одного и того же тензора дважды, он может возвращать разные значения; -например, тензор может быть результатом прочтения данных с диска, или случайно -сгенерированного числа. - - -## Ранг - - -**Ранг** объекта `tf.Tensor` - это количество размерностей массива. Синонимами -ранга являются **порядок**, **степень** или **n-размерность**. -Обрати внимание, что ранг в TensorFlow это не то же самое, что ранг матрицы в -математике. Из следующей таблицы видно, что каждый ранг в TensorFlow соответствует -разным математическим категориям: - - -Ранг | Математическая категория ---- | --- -0 | Скаляр (только величина) -1 | Вектор (величина и направление) -2 | Матрица (таблица чисел) -3 | 3-Тензор (куб чисел) -n | n-Тензор (ты понял идею) - - -### Ранг 0 - -Следующий пример кода демонстрирует создание нескольких переменных -ранга 0: - -```python -mammal = tf.Variable("Elephant", tf.string) -ignition = tf.Variable(451, tf.int16) -floating = tf.Variable(3.14159265359, tf.float64) -its_complicated = tf.Variable(12.3 - 4.85j, tf.complex64) -``` - - -Обрати внимание: строка в TensorFlow является одним объектом, а не последовательностью -символов. Возможно также использовать скалярные строки, вектор строк и так далее. - -### Ранг 1 - - -Чтобы создать объект `tf.Tensor` ранга 1, ты можешь передать список элементов -как первичное значение. Например: - -```python -mystr = tf.Variable(["Hello"], tf.string) -cool_numbers = tf.Variable([3.14159, 2.71828], tf.float32) -first_primes = tf.Variable([2, 3, 5, 7, 11], tf.int32) -its_very_complicated = tf.Variable([12.3 - 4.85j, 7.5 - 6.23j], tf.complex64) -``` - - -### Высшие ранги - - -Ранг 2 объекта `tf.Tensor` состоит из как минимум одного ряда и одного -столбца: - -```python -mymat = tf.Variable([[7],[11]], tf.int16) -myxor = tf.Variable([[False, True],[True, False]], tf.bool) -linear_squares = tf.Variable([[4], [9], [16], [25]], tf.int32) -squarish_squares = tf.Variable([ [4, 9], [16, 25] ], tf.int32) -rank_of_squares = tf.rank(squarish_squares) -mymatC = tf.Variable([[7],[11]], tf.int32) -``` - - -Тензоры высшего ранга подобным образом состоят из n-размерных массивов. Например, -во время обработки изображения используются тензоры ранга 4 с соответствующими -им размерносятми примеров в батче, шириной и высотой изображения, и цветовой модели. - -``` python -my_image = tf.zeros([10, 299, 299, 3]) # batch x height x width x color -``` - -### Получаем ранг объекта `tf.Tensor` - - -Чтобы определить ранг объекта `tf.Tensor`, вызови метод `tf.rank`. Например, -следующий метод программным способом определит ранг `tf.Tensor`, определенного -в предыдущем блоке кода: - -```python -r = tf.rank(my_image) -# После запуска графа, r присвоится значение 4. -``` - -### Работаем с частями `tf.Tensor` - - -Поскольку `tf.Tensor` является n-размерным массивом элементов, то для доступа к -конкретному элементу `tf.Tensor` потребуется уточнить индексы n. - - -Для тензора ранг 0 (скаляра) не требуется никаких индексов, посколько это -и есть одно единственное значение. - - -Для тензора ранга 1 (вектора) потребуется передать всего один индекс, который -предоставит нам доступ к значению: - - -```python -my_scalar = my_vector[2] -``` - - -Обрати внимание, что индекс, передаваемый внутри `[]`, может сам быть скалярным -`tf.Tensor`, если ты хочешь динамически выбирать элементы из вектора. - - -С тензорами ранга 2 и выше ситуация более интересная. Передавая два значения -`tf.Tenosor` ранга 2, он возвращает скаляр, что вполне ожидаемо: - - -```python -my_scalar = my_matrix[1, 2] -``` - - -Однако, передавая одно единственное значение, он возвращает подвектор матрицы как -в этом примере: - - -```python -my_row_vector = my_matrix[2] -my_column_vector = my_matrix[:, 3] -``` - - -Нотация `:` в Python делит синтаксис, оставляя эту размерность "в покое". Этот -прием полезен при работе с тензорами высшего ранга, поскольку предоставляет -доступ к его подвекторам, подматрицам и даже подтензорам. - - -## Форма - - -**Форма** тензора - это количество элементов в каждой размерности. -TensorFlow автоматически назначает формы во время работы в graph execution. -Назначенные формы могут иметь известный или неизвестный ранг. Если ранг -известен, то элементы каждой размерности также могут быть известны или -неизвестны. - - -В документации TensorFlow используются три правила для описания размерности -тензоров: ранг, форма и номер размерности. В следующией таблице видно, как эти -три параметра соотносятся друг с другом: - -Ранг | Форма | Номер размерности | Пример ---- | --- | --- | --- -0 | [] | 0-D | Тензор 0-D . Скаляр. -1 | [D0] | 1-D | Тензор 1-D формы [5]. -2 | [D0, D1] | 2-D | Тензор 2-D формы [3, 4]. -3 | [D0, D1, D2] | 3-D | Тензор 3-D формы [1, 4, 3]. -n | [D0, D1, ... Dn-1] | n-D | Тензор формы [D0, D1, ... Dn-1]. - - -Формы могут быть представлены в Python как списки или кортежи целых чисел, -или как `tf.TensorShape` - -### Получаем форму объекта `tf.Tensor` - - -Есть два способа получить форму `tf.Tensor`. Во время построения графа часто -является полезным узнать, что уже известно о форме тензора. Это можно сделать -прочтя параметр `shape` объека `tf.Tensor`. Этот метод возвращает объект -`TensorShape`, который является весьма удобным способом представления -частично определенных форм, поскольку во время построения графа не все формы -известны полностью. - - -Также возможно получить `tf.Tensor`, который будет представлять полностью -определенную форму другого объекта `tf.Tensor` в рабочей среде. Это достигается -путем вызова операции `tf.shape`. Таким образом ты можешь построить граф, который -манипулирует формами тензоров при помощи создания других тензоров, который зависят -от динамической формы входящего `tf.Tensor`. - - -Например, вот как мы можем сделать вектор из нулей с одинаковым размером и числом -столбцов в матрице: - -``` python -zeros = tf.zeros(my_matrix.shape[1]) -``` - - -### Изменяем форму `tf.Tensor` - - -**Количество элементов** тензора - это продукт размеров всех форм. Количество -элементов - это скаляр, который всегда равен `1`. Посколько часто множесто разных -форм имеют одинаковое количество элементов, то часто удобно позволять менять форму -`tf.Tensor`, зафиксировав его элементы. Это можно сделать с помощью `tf.reshape`. - -В следующем примере показано как изменить форму тензоров: - -```python -rank_three_tensor = tf.ones([3, 4, 5]) -matrix = tf.reshape(rank_three_tensor, [6, 10]) # Изменяем существующую форму на - # матрицу 6x10 -matrixB = tf.reshape(matrix, [3, -1]) # Изменяем форму на матрицу 3х20. - # -1 требует `reshape` рассчитать - # размерность тензора. -matrixAlt = tf.reshape(matrixB, [4, 3, -1]) # Изменяем форму на - # тензор 4х3х5 - -# Обрати внимание, что количество элементов измененных тензоров должно совпадать -# с изначальным количеством элементов. Таким образом, следующий пример выдает -# ошибку, так как нет значения для последней размерности, которое бы совпадало -# с количеством элементов. -yet_another = tf.reshape(matrixAlt, [13, 2, -1]) # Ошибка! -``` - -## Типы данных - - -В дополнение к размерностям, тензоры имеют тип данных. Смотри документацию -`tf.DType` для ознакомления с полным списком типов данных. - - -Невозможно иметь `tf.Tensor` более чем с одним типом данных. Тем не менее, возможно -сериализовать произвольные структуры данных как строки и сохранить их в `tf.Tensor`. - - -Также возможно конвертировать тип данных из одного `tf.Tensor` в другой при помощи -метода `tf.cast`: - - -``` python -# Переводим константу тензора в число с плавающей запятой. -float_tensor = tf.cast(tf.constant([1, 2, 3]), dtype=tf.float32) -``` - - -Для проверки тип данных `tf.Tensor` используй параметр `Tensor.dtype`. - - -При создании `tf.Tensor` из объекта Python, ты можешь также указать тип данных -этого тензора. Если этого не сделать, то TensorFlow сам выберет тип данных, -который будет представлять твои данные. TensorFlow автоматически конвертирует -*целые числа* Python в `tf.int32`, а *числа с плавающей запятой* - в `tf.float32`. -Другими словами, TensorFlow использует те же самые правила, что и NumPy при -конвертации в массивы. - - -## Оценка тензоров - - -Когда вычислительный граф был построен, ты можешь запустить вычисление, которое -будет производить определенный `tf.Tensor` и извлекать присвоенное ему значение. -Это часто является полезным как для отладки, так и необходимо для работы TensorFlow. - - -Самый легкий способ оценить тензор - использовать метод `Tensor.eval`. Вот пример -кода оценки тензора: - - -```python -constant = tf.constant([1, 2, 3]) -tensor = constant * constant -print(tensor.eval()) -``` - - -Метод `eval` работает только когда активирована стандартная `tf.Session` -(смотри [Графы и сессии](https://www.tensorflow.org/guide/graphs) для -более подробной информации). - - -`Tensor.eval` возвращает массив NumPy с тем же самым содержимым, что и -тензор. - - -Иногда невозможно произвести оценку `tf.Tensor` без контекста, потому что -его значение может зависеть от динамичности информации, которая недоступна. -Например, тензоры, которые зависят от `placeholder`' не могут быть оценены -без присвоения значения для `placeholder`. - -``` python -p = tf.placeholder(tf.float32) -t = p + 1.0 -t.eval() # Эта операция не будет выполнена, так как нет значения. -t.eval(feed_dict={p:2.0}) # Эта операция будет произведены успешно, поскольку - # мы присваиваем значение placeholder. -``` - - -Отемтим, что возможно производить оценку любых `tf.Tensor`, а не только -placeholder. - - -Другие структуры модели могут усложнить оценку `tf.Tensor`. TensorFlow -не может напрямую оценить `tf.Tensor`, определенные внутри функций или внутри -порядка выполнения структуры модели. Если `tf.Tensor` зависит от значения из -очереди, то оценка `tf.Tensor` будет работать только если какое-то значение -было поставлено в очередь; в других случаях оценка будет приостановлена. При -работе с очередями, всегда вызывай `tf.train.start_queue_runners` прежде чем -производить оценку каких-либо `tf.Tensor`. - - -## Выводим значения тензоров - - -Чтобы произвести отладку коду возможно понадобится выводить значения `tf.Tensor`. -Несмотря на то, что [tfdbg](../guide/debugger.md) хоть и предлагает углубленную -поддержку отладки, TensorFlow также имеет операцию для неспоредственного вывода -значения `tf.Tensor` на экран. - - -Отметим, что только в редких случаях потребуется использовать следующий -шаблон для вывода значения `tf.Tensor`: - - -``` python -t = <<здесь определена операция tensorflow>> -print(t) # Выводим символический тензор при построении графа. - # Этот тензор не имеет значения в данном контексте. -``` - - -Этот код выведет на экран объект `tf.Tensor` (который представляет из себя -отложенное вычисление), но не его значение. Вместо этого TensorFlow обеспечит -операцию `tf.Print`, которая возвратит его первый неизменный аргумент тензора, -в то время как также выведет набор `tf.Tensor`, который был передан как второй -аргумент. - - -Чтобы правильно использовать `tf.Print`, необходимо использовать его возвращенное -значение как в примере ниже: - - -``` python -t = <<какая-либо операция tensorflow>> -tf.Print(t, [t]) # Не возвращает ничего -t = tf.Print(t, [t]) # Здесь мы используем значение, возвращенное `tf.Print`. -result = t + 1 # Теперь результат оценен, и значение `t` будет выведено. -``` - - -При оценке `result` мы также оцениваем все, от чего зависит `result`. -Поскольку `result` зависит от `t` и оценка `t` окажет эффект на вывод его -значения (предыдущего значения `t`), значение `t` будет выведено на экран. - diff --git a/site/ru/r1/tutorials/keras/README.md b/site/ru/r1/tutorials/keras/README.md deleted file mode 100644 index 3370ccf98bd..00000000000 --- a/site/ru/r1/tutorials/keras/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Изучай и применяй методы машинного обучения - -Note: Вся информация в этом разделе переведена с помощью русскоговорящего -Tensorflow сообщества на общественных началах. Поскольку этот перевод не -является официальным, мы не гарантируем что он на 100% аккуратен и соответствует -[официальной документации на английском языке](https://www.tensorflow.org/?hl=en). -Если у вас есть предложение как исправить этот перевод, мы будем очень рады -увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) -репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow -лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), -напишите нам на -[docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru). - -Создать серию интерактивных руководств нас вдохновила книга -*[Глубокое обучение на Python](https://books.google.com/books?id=Yo3CAQAACAAJ)*. -В наших уроках используется высокоуровневый `tf.keras` API для создания -и обучения моделей глубокого обучения. Узнай больше о том, как использовать Keras -и TensorFlow в нашем [Руководстве по TensorFlow и Keras](../../guide/keras.ipynb). - -Примечание издателя: *Глубокое обучение на Python* представляет из себя -практическое пособие для создания моделей глубокого обучения на Python, -используя все возможности библиотеки Keras. Книга написана Франсуа Шолле, автором -библиотеки и научным сотрудником Google AI. Она поможет тебе понять -основные методы глубокого обучения с помощью наглядных объяснений и практических примеров. - -Чтобы узнать больше об основах машинного обучения, начни -[Ускоренный курс по машинному обучению](https://developers.google.com/machine-learning/crash-course/). - -1. [Простая классификация](./basic_classification.ipynb) -2. [Классификация текста](./basic_text_classification.ipynb) -3. [Регрессия](./basic_regression.ipynb) -4. [Переобучение и недообучение](./overfit_and_underfit.ipynb) -5. [Сохранение и загрузка моделей](./save_and_restore_models.ipynb) diff --git a/site/ru/r1/tutorials/keras/basic_classification.ipynb b/site/ru/r1/tutorials/keras/basic_classification.ipynb deleted file mode 100644 index 83eb429c16e..00000000000 --- a/site/ru/r1/tutorials/keras/basic_classification.ipynb +++ /dev/null @@ -1,990 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# Обучи свою первую нейросеть: простая классификация" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "Это руководство поможет тебе обучить нейросеть, которая классифицирует изображения одежды, например, кроссовки и рубашки. Это нормально, если не все будет понятно сразу: это быстрый, ознакомительный обзор полной программы TensorFlow, где новые детали объясняются по мере их появления.\n", - "\n", - "Руководство использует [tf.keras](https://www.tensorflow.org/r1/guide/keras), высокоуровневый API для построения и обучения моделей в TensorFlow." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow и tf.keras\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# Вспомогательные библиотеки\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## Загружаем датасет Fashion MNIST" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "Это руководство использует датасет [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) который содержит 70,000 монохромных изображений в 10 категориях. На каждом изображении содержится по одному предмету одежды в низком разрешении (28 на 28 пикселей):\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Образцы Fashion-MNIST (Zalando, лицензия MIT).
     \n", - "
    \n", - "\n", - "Fashion MNIST предназначен для замены классического датасета [MNIST](http://yann.lecun.com/exdb/mnist/) который часто используют как \"Hello, World\" программ машинного обучения для компьютерного зрения. Датасет MNIST содержит изображения рукописных цифр (0, 1, 2, и т.д.) в формате идентичном формату изображений одежды которыми мы будем пользоваться здесь.\n", - "\n", - "Это руководство для разнообразия использует Fashion MNIST, и еще потому, что это проблема немного сложнее чем обычный MNIST. Оба датасета относительно малы, и используются для проверки корректности работы алгоритма. Это хорошие отправные точки для тестирования и отладки кода.\n", - "\n", - "Мы используем 60,000 изображений для обучения нейросети и 10,000 изображений чтобы проверить, насколько правильно сеть обучилась их классифицировать. Вы можете получить доступ к Fashion MNIST прямо из TensorFlow. Импортируйте и загрузите данные Fashion MNIST прямо из TensorFlow:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "Загрузка датасета возвращает четыре массива NumPy:\n", - "\n", - "* Массивы `train_images` и `train_labels` являются *тренировочным сетом* — данными, на которых модель будет обучаться.\n", - "* Модель тестируется на *проверочном сете*, а именно массивах `test_images` и `test_labels`.\n", - "\n", - "Изображения являются 28х28 массивами NumPy, где значение пикселей варьируется от 0 до 255. *Метки (labels)* - это массив целых чисел от 0 до 9. Они соответствуют *классам* одежды изображенной на картинках:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    LabelClass
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "Каждому изображению соответствует единственная метка. Так как *названия классов* не включены в датасет, сохраним их тут для дальнейшего использования при построении изображений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## Изучите данные\n", - "\n", - "Давайте посмотрим на формат данных перед обучением модели. Воспользовавшись shape мы видим, что в тренировочном датасете 60,000 изображений, каждое размером 28 x 28 пикселей:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "Соответственно, в тренировочном сете 60,000 меток:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "Каждая метка это целое число от 0 до 9:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "Проверочный сет содержит 10,000 изображений, каждое - также 28 на 28 пикселей:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "И в проверочном сете - ровно 10,000 меток:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## Предобработайте данные\n", - "\n", - "Данные должны быть предобработаны перед обучением нейросети. Если вы посмотрите на первое изображение в тренировочном сете вы увидите, что значения пикселей находятся в диапазоне от 0 до 255:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "Мы масштабируем эти значения к диапазону от 0 до 1 перед тем как скормить их нейросети. Для этого мы поделим значения на 255. Важно, чтобы *тренировочный сет* и *проверочный сет* были предобработаны одинаково:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "Чтобы убедиться, что данные в правильном формате и мы готовы построить и обучить нейросеть, выведем на экран первые 25 изображений из *тренировочного сета* и отобразим под ними наименования их классов." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## Постройте модель\n", - "\n", - "Построение модели нейронной сети требует правильной конфигурации каждого слоя, и последующей компиляции модели." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### Настройте слои\n", - "\n", - "Базовым строительным блоком нейронной сети является *слой*. Слои извлекают образы из данных, которые в них подаются. Надеемся, что эти образы имеют смысл для решаемой задачи.\n", - "\n", - "Большая часть глубокого обучения состоит из соединения в последовательность простых слоев. Большинство слоев, таких как tf.keras.layers.Dense, имеют параметры, которые настраиваются во время обучения." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation='relu'),\n", - " keras.layers.Dense(10, activation='softmax')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "Первый слой этой сети - `tf.keras.layers.Flatten`, пробразует формат изображения из двумерного массива (28 на 28 пикселей) в одномерный (размерностью 28 * 28 = 784 пикселя). Слой извлекает строки пикселей из изображения и выстраивает их в один ряд. Этот слой не имеет параметров для обучения; он только переформатирует данные.\n", - "\n", - "После разложения пикселей, нейросеть содержит два слоя `tf.keras.layers.Dense`. Это полносвязные нейронные слои. Первый `Dense` слой состоит из 128 узлов (или нейронов). Второй (и последний) 10-узловой *softmax* слой возвращает массив из 10 вероятностных оценок дающих в сумме 1. Каждый узел содержит оценку указывающую вероятность принадлежности изображения к одному из 10 классов.\n", - "\n", - "### Скомпилируйте модель\n", - "\n", - "Прежде чем модель будет готова для обучения, нам нужно указать еще несколько параметров. Они добавляются на шаге *compile* модели:\n", - "\n", - "* *Функция потерь (Loss function)* — измеряет точность модели во время обучения. Мы хотим минимизировать эту функцию чтоб \"направить\" модель в верном направлении.\n", - "* *Оптимизатор (Optimizer)* — показывает каким образом обновляется модель на основе входных данных и функции потерь.\n", - "* *Метрики (Metrics)* — используются для мониторинга тренировки и тестирования модели. Наш пример использует метрику *accuracy* равную доле правильно классифицированных изображений." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## Обучите модель\n", - "\n", - "Обучение модели нейронной сети требует выполнения следующих шагов::\n", - "\n", - "1. Подайте тренировочный данные в модель. В этом примере тренировочные данные это массивы `train_images` и `train_labels`.\n", - "2. Модель учится ассоциировать изображения с правильными классами.\n", - "3. Мы просим модель сделать прогнозы для проверочных данных, в этом примере массив test_images. Мы проверяем, соответствуют ли предсказанные классы меткам из массива test_labels.\n", - "\n", - "Для начала обучения, вызовите метод `model.fit`, который называется так, поскольку \"тренирует (fits)\" модель на тренировочных данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "В процессе обучения модели отображаются метрики потери (loss) и точности (accuracy). Эта модель достигает на тренировочных данных точности равной приблизительно 0.88 (88%)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## Оцените точность\n", - "\n", - "Далее, сравните какую точность модель покажет на проверчном датасете:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('\\nТочность на проверочных данных:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "Полученная на проверочном сете точность оказалась немного ниже, чем на тренировочном. Этот разрыв между точностью на тренировке и тесте является примером *переобучения (overfitting)* . Переобучение возникает, когда модель машинного обучения показывает на новых данных худший результат, чем на тех, на которых она обучалась." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## Сделайте предсказания\n", - "\n", - "Теперь, когда модель обучена, мы можем использовать ее чтобы сделать предсказания по поводу нескольких изображений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "Здесь полученная модель предсказала класс одежды для каждого изображения в проверочном датасете. Давайте посмотрим на первое предсказание:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "Прогноз представляет из себя массив из 10 чисел. Они описывают \"уверенность\" (confidence) модели в том, насколько изображение соответствует каждому из 10 разных видов одежды. Мы можем посмотреть какой метке соответствует максимальное значение:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "Модель полагает, что на первой картинке изображен ботинок (ankle boot), или class_names[9]. Проверка показывает, что классификация верна:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "Мы можем построить график, чтобы взглянуть на полный набор из 10 предсказаний классов." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DvYmmrpIy6Y1", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d4Ov9OFDMmOD" - }, - "source": [ - "Давайте посмотрим на нулевое изображение, предсказание и массив предсказаний." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HV5jw-5HwSmO", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ko-uzOufSCSe", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "Давайте посмотрим несколько изображений с их прогнозами. Цвет верных предсказаний синий, а неверных - красный. Число это процент уверенности (от 100) для предсказанной метки. Отметим, что модель может ошибаться даже если она очень уверена." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hQlnbqaw2Qu_", - "colab": {} - }, - "source": [ - "# Отображаем первые X тестовых изображений, их предсказанную и настоящую метки.\n", - "# Корректные предсказания окрашиваем в синий цвет, ошибочные в красный.\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "Наконец, используем обученную модель для предсказания класса на одном изображении." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yRJ7JU7JCaXT", - "colab": {} - }, - "source": [ - "# Берем одну картинку из проверочного сета.\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "Модели tf.keras оптимизированы для предсказаний на *пакетах (batch)* данных, или на множестве примеров сразу. Таким образом, даже если мы используем всего 1 картинку, нам все равно необходимо добавить ее в список:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# Добавляем изображение в пакет данных, состоящий только из одного элемента.\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "Сейчас предскажем правильную метку для изображения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ai-cpLjO-3A", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "Метод `model.predict` возвращает нам список списков, по одному для каждой картинки в пакете данных. Получите прогнозы для нашего (единственного) изображения в пакете:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "np.argmax(predictions_single[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "И, как и ранее, модель предсказывает класс 9." - ] - } - ] -} \ No newline at end of file diff --git a/site/ru/r1/tutorials/keras/basic_regression.ipynb b/site/ru/r1/tutorials/keras/basic_regression.ipynb deleted file mode 100644 index 93bb5f95b4d..00000000000 --- a/site/ru/r1/tutorials/keras/basic_regression.ipynb +++ /dev/null @@ -1,845 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# Предсказывай расход топлива: регрессия" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Запусти в Google Colab\n", - " \n", - " Изучай код на GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TECyBc0udsgK", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "Говоря о задачах *регрессии*, мы преследуем цель дать прогноз какого-либо значения, например цену или вероятность. Это уже совсем другая задача если сравненить с *классификацией*, где нужно предсказать конкретный класс или категорию (например, яблоко или апельсин на картинке).\n", - "\n", - "Мы будем использовать классический датасет [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) и построим модель, которая будет предсказывать эффективность расхода топлива автомобилей конца 70-х и начала 80-х. Для этого мы загрузим описания множества различных автомобилей того времени. Эти описания будут содержать такие параметры как количество цилиндров, лошадиных сил, объем двигателя и вес.\n", - "\n", - "В этом примере используется `tf.keras` API, подробнее [смотри здесь](https://www.tensorflow.org/r1/guide/keras)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# Установим библиотеку seaborn для построения парных графиков\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Датасет Auto MPG\n", - "\n", - "Датасет доступен в [репозитории машинного обучения UCI](https://archive.ics.uci.edu/ml/datasets/auto+mpg).\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### Загружаем данные\n", - "Сперва загрузим наш набор данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "Импортируем его при помощи библиотеки Pandas:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['Расход топлива','Кол-во цилиндров','Объем двигателя','Л.с.','Вес',\n", - " 'Разгон до 100 км/ч', 'Год выпуска', 'Страна выпуска']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### Подготовим данные\n", - "\n", - "Нам нужно почистить наши данные, так как датасет содержит несколько неизвестных значений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "Чтобы было проще давай просто удалим ряды, где отсутствуют значения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "`\"Страна выпуска\"` - это колонка с указанием категории, а не значений. Давай переведем ее в двоичный код:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Страна выпуска')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['США'] = (origin == 1)*1.0\n", - "dataset['Европа'] = (origin == 2)*1.0\n", - "dataset['Япония'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### Разделим на тренировочные и проверочные данные\n", - "\n", - "Теперь нам необходимо разделить данные на две части: для обучения и для проверки.\n", - "\n", - "Точность получившейся модели мы сможем проверить на втором наборе данных." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### Проверим данные\n", - "\n", - "Давай посмотрим на совместное распределение нескольких пар колонок из тренировочного набора данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"Расход топлива\", \"Кол-во цилиндров\", \"Объем двигателя\", \"Вес\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "А также посмотрим как выглядит общая статистика:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"Расход топлива\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### Выделим значения\n", - "\n", - "Теперь давай отделим целевые значения *label* от характеристик.\n", - "\n", - "Мы будем использовать эти значения для обучения нашей модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('Расход топлива')\n", - "test_labels = test_dataset.pop('Расход топлива')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### Нормализуем данные\n", - "\n", - "Давай еще раз взглянем на блок `train_stats` выше. Обрати внимание, что значения каждой характеристики измеряются по разному." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "Нормализация параметров данных, в которых используются разные единицы измерений - один из первых шагов подготовки данных к обучению. Несмотря на то, что модель конечно *может* стремиться к пределу без нормализации, но это излишне усложняет обучение, а также делает получившуюся модель зависимой от выбора единиц измерения входных данных.\n", - "\n", - "Статистику, полученную по данным из обучения, мы будем также использовать для данных из проверки. Так и задумано, так как модель еще не будет иметь никакой информации по проверочным данным." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "Для обучения модели мы будем использовать обновленные, нормализованные данные.\n", - "\n", - "Внимание: статистика, используемая для нормализации входных данных так же важна, как и сами веса модели." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## Модель" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### Построим модель\n", - "\n", - "Теперь давай построим нашу модель. Мы будем использовать `Sequential` (последовательную) модель с двумя полносвязными Dense слоями, а выходящий слой будет возвращать одно постоянно изменяющееся значение. Все этапы построения модели мы опишем в функции `build_model`, так как позже мы создадим еще одну модель." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation=tf.nn.relu, input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation=tf.nn.relu),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.train.RMSPropOptimizer(0.001)\n", - "\n", - " model.compile(loss='mse',\n", - " optimizer=optimizer,\n", - " metrics=['mae', 'mse'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### Проверим модель\n", - "\n", - "Давай воспользуемся методом `.summary` чтобы посмотреть как выглядит наша модель:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "\n", - "Все готово, теперь мы можем опробовать нашу модель. Для начала попробуем небольшой батч из 10 примеров данных из тренировочного набора и попробуем предсказать результат." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "Похоже все работает правильно, модель показывает результат ожидаемой формы и класса." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### Обучим модель\n", - "\n", - "Мы будем обучать модель в течение 1000 эпох и фиксировать точность модели на тренирвочных и проверочных данных в объекте `history`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# Выведем прогресс обучения в виде точек после каждой завершенной эпохи\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "Теперь давай сделаем визуализацию процесса обучения при помощи статистики из объекта `history`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure(figsize=(8,12))\n", - "\n", - " plt.subplot(2,1,1)\n", - " plt.xlabel('Эпоха')\n", - " plt.ylabel('Среднее абсолютное отклонение')\n", - " plt.plot(hist['epoch'], hist['mean_absolute_error'],\n", - " label='Ошибка при обучении')\n", - " plt.plot(hist['epoch'], hist['val_mean_absolute_error'],\n", - " label = 'Ошибка при проверке')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - "\n", - " plt.subplot(2,1,2)\n", - " plt.xlabel('Эпоха')\n", - " plt.ylabel('Среднеквадратическая ошибка')\n", - " plt.plot(hist['epoch'], hist['mean_squared_error'],\n", - " label='Ошибка при обучении')\n", - " plt.plot(hist['epoch'], hist['val_mean_squared_error'],\n", - " label = 'Ошибка при проверке')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "Полученный график показывает, что после нескольких сотен эпох наша модель улучшается совсем незначительно в процессе обучения. Давай обновим метод `model.fit` чтобы автоматически прекращать обучение как только показатель проверки *Val loss* не улучшается. Для этого мы используем функцию обратного вызова *callback*, которая проверяет показатели обучения после каждой эпохи. Если после определенного количество эпох нет никаких улучшений, то функция автоматически остановит его.\n", - "\n", - "Читай больше про функцию *callback* [здесь](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# Параметр patience определяет количество эпох, которые можно пропустить без улучшений\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=50)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "График показывает что среднее значение ошибки на проверочных данных - около 2 галлонов на милю. Хорошо это или плохо? Решать тебе.\n", - "\n", - "Давай посмотрим как наша модель справится на **наборе данных для проверки**, который мы еще не использовали после обучения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Среднее абсолютное отклонение на проверочных данных: {:5.2f} галлон на милю\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "### Делаем предсказания\n", - "\n", - "Наконец давай сделаем предсказания показателей расхода топлива, используя набор наши проверочные данные:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('Истинные значения')\n", - "plt.ylabel('Предсказанные значения')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## Заключение\n", - "\n", - "Это руководство познакомило тебя с несколькими способами решения задач регрессии.\n", - "\n", - "Ключевые моменты урока:\n", - "\n", - "* Mean Squared Error (MSE) (пер. \"Среднеквадратическая ошибка\") - это распространенная функция потерь, используемая для решения задач регрессии (отличная от задач классификации)\n", - "* Показатели оценки модели для регрессии отличаются от используемых в классификации\n", - "* Когда входные данные имеют параметры в разных форматах, каждый параметр должен быть нормализован\n", - "* Если данных для обучения немного, используй небольшую сеть из нескольких скрытых слоев. Это поможет избежать переобучения\n", - "* Используй метод ранней остановки - это очень полезная техника для избежания переобучения" - ] - } - ] -} \ No newline at end of file diff --git a/site/ru/r1/tutorials/keras/basic_text_classification.ipynb b/site/ru/r1/tutorials/keras/basic_text_classification.ipynb deleted file mode 100644 index 36d1f3c97fc..00000000000 --- a/site/ru/r1/tutorials/keras/basic_text_classification.ipynb +++ /dev/null @@ -1,747 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# Классификация текста обзоров фильмов" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Запусти в Google Colab\n", - " \n", - " Изучай код на GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R80081eTd2XD" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "\n", - "В этом интерактивном уроке мы построим модель, которая будет классифицировать обзор фильма как *позитивный* или *негативный* на основе текста. Это пример *бинарной* классификации (по двум классам), важной, и широко применяющейся задачи машинного обучения.\n", - "\n", - "Мы воспользуемся [датасетом IMDB](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb), который содержит тексты 50,000 обзоров фильмов из [Internet Movie Database](https://www.imdb.com/). Они разделены на 25,000 обзоров для обучения, и 25,000 для проверки модели. Тренировочные и проверочные датасеты *сбалансированы*, т.е. содержат одинаковое количество позитивных и негативных обзоров.\n", - "\n", - "Данное руководство использует [tf.keras](https://www.tensorflow.org/r1/guide/keras), высокоуровневый API для создания и обучения моделей в TensorFlow. Чтобы сделать более сложную по структуре классификацую текста при помощи `tf.keras`, читай [Руководство по классификации текстов](https://developers.google.com/machine-learning/guides/text-classification/)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4v3p_h3LBekM", - "colab": {} - }, - "source": [ - "# keras.datasets.imdb is broken in 1.13 and 1.14, by np 1.16.3\n", - "!pip install tf_nightly" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## Загружаем датасет IMDB\n", - "\n", - "Датасет IMDB доступен сразу в TensorFlow при помощи метода `load_data`. Он уже подготовлен таким образом, что обзоры (последовательности слов) были конвертированы в последовательность целых чисел, где каждое целое представляет конкретное слово в массиве.\n", - "\n", - "Давай напишем пару строчек кода чтобы загрузить датасет (или автоматически используем копию из кэша, если ты уже скачал этот набор данных):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "Аргумент `num_words=10000` позволяет нам ограничиться только 10,000 наиболее часто встречающимися словами из тренировочного сета. Все редкие слова исключаются. Это поможет нам держать объем данных в разумных пределах." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## Знакомимся с данными\n", - "\n", - "Давай посмотрим какая информация нам доступна. Данные уже подготовлены: каждый пример - это массив целых чисел, которые представляют слова из обзоров. Каждая метка *label* является целым числом 0 или 1:\n", - "\n", - "0 - негативный обзор, 1 - позитивный." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} - }, - "source": [ - "print(\"Тренировочных записей: {}, меток: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "Текст обзоров уже был конвертирован в целые числа, где каждое целок представляет слово из словаря. Вот пример того, как выглядит первый обзор:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "Разные обзоры фильмов также имеют разное количество слов. Код ниже поможет нам узнать количество слов в первом и втором обзоре. Поскольку нейросеть может принимать только данные одинаковой длины, то нам предстоит как-то решить эту задачу." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### Конвертируем целые обратно в слова\n", - "\n", - "Не будет лишним также знать, как конвертировать целые числа из массива обратно в текст. Напишем вспомогательную функцию, с помощью который мы сможем запрашивать из этого словаря объект, который содержит указанные числа и отображать их в виде слов:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} - }, - "source": [ - "# Назначим словарь, который будет отображать слова из массива данных\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# Зарезервируем первые несколько значений\n", - "word_index = {k:(v+3) for k,v in word_index.items()}\n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # Вместо редких слов, не вошедших в набор из 10,000, будет указано UNK\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "Теперь мы можем легко воспользоваться функцией `decode_review` для отображения текста первого обзора фильма:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} - }, - "source": [ - "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## Подготавливаем данные\n", - "\n", - "Обзоры фильмов из массива целых чисел должны быть конвертированы в тензоры прежде, чем они будут пропущены через нейросеть. Эта конвертация может быть сделана несколькими способами:\n", - "\n", - "* *One-hot encoding* конвертирует массивы в векторы 0 и 1. Например, последовательность [3, 5] станет 10,000-мерным вектором, полностью состоящим из нулей кроме показателей 3 и 5, которые будут представлены единицами. Затем, нам нужно будет создать первый `Dense` слой в нашей сети, который сможет принимать векторые данные с плавающей запятой. Такой подход очень требователен к объему памяти, несмотря на то, что требует указать размеры матрицы `num_words * num_reviews`\n", - "\n", - "* Другой способ - сделать все массивы одинаковыми по длине, а затем создать тензор целых чисел с указанием `max_length * num_reviews`. Мы можем использовать *Embedding* (пер. \"Встроенный\") слой, который может использовать эти параметры в качестве первого слоя нашей сети\n", - "\n", - "В этом руководстве мы попробуем второй способ.\n", - "\n", - "Поскольку все обзоры фильмов должны быть одинаковой длины, то мы используем функцию [pad_sequences](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences), чтобы привести все длины к одному значению:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} - }, - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "Давай теперь посмотрим на длину наших примеров:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "А также проверим как выглядит первый стандартизированный по длине обзор фильма:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## Строим модель\n", - "\n", - "Нейронная сеть создается посредством стека (наложения) слоев - это требует ответов на два вопроса об архитектуре самой модели:\n", - "\n", - "* Сколько слоев будет использовано в модели?\n", - "* Сколько *скрытых блоков* будет использовано для каждого слоя?\n", - "\n", - "В этом примере, входные данные состоят из массива слов (целых чисел). Получаемые предсказания будут в виде меток 0 или 1. Давай построим модель, которая будет решать нашу задачу:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "# Размер входных данных - количество слов, использованных в обзорах фильмов (10,000 слов)\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16, input_shape=(None,)))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation=tf.nn.relu))\n", - "model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "Для создания классификатора все слои проходят процесс стека, или наложения:\n", - "\n", - "1. Первый `Embedding` слой принимает переведенные в целые числа слова и ищет соответствующий вектор для каждой пары слово/число. Модель обучается на этих векторах. Векторы увеличивают размер получаемого массива на 1, в результате чего мы получаем измерения: `(batch, sequence, embedding)`\n", - "\n", - "2. Следующий слой `GlobalAveragePooling1D` возвращает получаемый вектор заданной длины для каждого примера, усредняя размер ряда. Это позволит модели легко принимать данные разной длины\n", - "\n", - "3. Этот вектор пропускается через полносвязный `Dense` слой с 16 скрытыми блоками\n", - "\n", - "4. Последний слой также является полносвязным, но с всего одним выходящим нодом. При помощи функции активации `sigmoid` (Сигмоида) мы будем получать число с плавающей запятой между 0 и 1, которое будет показывать вероятность или уверенность модели" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### Скрытые блоки\n", - "\n", - "Вышеописанная модель имеет 2 промежуточных или *скрытых* слоя, между входом и выходом данных. Количество выходов (блоков, нодов или нейронов) является размером репрезентативного пространства слоя. Другими словами, количество свободы, которая разрешена сети во время обучения.\n", - "\n", - "Если модель имеет больше скрытых блоков, и/или больше слоев, то тогда нейросеть может обучиться более сложным представлениям. Однако в этом случае это будет дороже с точки зрения вычислительных ресурсов и может привести к обучению нежелательных паттернов - паттернов, которые улучшают показатели на тренировочных данных, но не на проверочных. Это называется *переобучением*, и мы обязательно познакомимся с этим явлением далее." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### Функция потерь и оптимизатор\n", - "\n", - "Для модели нам необходимо указать функцию потерь и оптимизатор для обучения. Поскольку наша задача является примером бинарной классификации и модель будет показывать вероятность (слой из единственного блока с сигмоидой в качестве функции активации), то мы воспользуемся функцией потерь `binary_crossentropy` (пер. \"Перекрестная энтропия\").\n", - "\n", - "Это не единственный выбор для нашей функции потерь: ты можешь, например, выбрать `mean_squared_error`. Но обычно `binary_crossentropy` лучше справляется с вероятностями - она измеряет \"дистанцию\" между распределениями вероятностей, или, как в нашем случае, между эталоном и предсказаниями.\n", - "\n", - "Далее, по мере знакомства с задачами регрессии (например, предсказание цен на недвижимость), мы посмотрим как использовать другую функцию потерь, которая называется среднеквадратичская ошибка (MSE).\n", - "\n", - "А сейчас, настроим нашу модель: мы будем использовать *оптимизатор Адама* и *перекрестную энтропию* для потерь:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer=tf.train.AdamOptimizer(),\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "## Создадим проверочный набор данных\n", - "\n", - "Во время обучения мы хотим проверить точность нашей модели на данных, которых она еще не видела. Давай создадим *проверочный сет* данных, выделив 10,000 примеров из оригинального тренировочного сета в отдельный набор.\n", - "\n", - "Почему мы не используем проверочный набор прямо сейчас? Наша цель - разработать и настроить нашу модель, используя только данные для обучения, и только потом использовать проверочный сет всего один раз чтобы оценить точность." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} - }, - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## Обучаем модель\n", - "\n", - "Начнем тренировку нашей модели с 40 эпох при помощи мини-батчей по 512 образцов (*батч* - набор, пакет данных). Это означает, что мы сделаем 40 итераций (или проходов) по всем образцам данных в тензорах `x_train` и `y_train`. После обучения мы узнаем потери и точность нашей модели, показав ей 10,000 образцов из проверочного набора данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} - }, - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## Оценим точность модели\n", - "\n", - "Теперь когда обучение прошло успешно, давай посмотрим какие результаты показывает модель.\n", - "\n", - "Она будет возвращать 2 значения: потери *loss* (число, которое показывает ошибку, чем оно ниже, тем лучше), и точность *accuracy*." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "Как мы видим, этот достаточно наивный подход достиг точности около 87%. Если бы мы использовали более сложные методы, то модель приблизилась бы к отметке в 95%." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## Построим временной график точности и потерь\n", - "\n", - "Метод `model.fit()` возвращает объект `History`, который содержит все показатели, которые были записаны в лог во время обучения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} - }, - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "Здесь всего четыре показателя, по одному для каждой отслеживаемой метрики во время обучения и проверки. Мы можем использовать их, чтобы построить графики потерь и точности обоих стадий для сравнения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history.history['acc']\n", - "val_acc = history.history['val_acc']\n", - "loss = history.history['loss']\n", - "val_loss = history.history['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# \"bo\" означает \"blue dot\", синяя точка\n", - "plt.plot(epochs, loss, 'bo', label='Потери обучения')\n", - "# \"b\" означает \"solid blue line\", непрерывная синяя линия\n", - "plt.plot(epochs, val_loss, 'b', label='Потери проверки')\n", - "plt.title('Потери во время обучения и проверки')\n", - "plt.xlabel('Эпохи')\n", - "plt.ylabel('Потери')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} - }, - "source": [ - "plt.clf() # Очистим график\n", - "acc_values = history_dict['acc']\n", - "val_acc_values = history_dict['val_acc']\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Точность обучения')\n", - "plt.plot(epochs, val_acc, 'b', label='Точность проверки')\n", - "plt.title('Точность во время обучения и проверки')\n", - "plt.xlabel('Эпохи')\n", - "plt.ylabel('Точность')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "\n", - "На графиках точками отмечены потери и точность модели во время обучения, а линией - во время проверки.\n", - "\n", - "Обрати внимание что потери во время обучения *уменьшаются*, а точность - *увеличивается* с каждой следующей эпохой. Это вполне ожидаемо, поскольку мы используем градиентный спуск *gradient descent* - он минимизирует показатели потерь с каждой итерацией настолько быстро, насколько это возможно.\n", - "\n", - "Но это совсем не тот случай если мы посмотрим на потери и точность во время проверки модели: после приблизительно 20 эпох они находятся на пике. Это явный пример переобучения: модель показывает более лучшие показатели на данных для обучения, нежели на новых, которых она еще не видела. После этого момента модель начинает переоптимизироваться и обучается представлениям, которые *являются свойственны* только данным для обучения. Таким образом, модель не учится *обобщать* новые, проверочные данные.\n", - "\n", - "Именно здесь мы можем предотвратить переобучение просто прекратив тренировку сразу после 20 эпох обучения. Далее мы посмотрим, как это можно сделать автоматически при помощи *callback*, функции обратного вызова." - ] - } - ] -} \ No newline at end of file diff --git a/site/ru/r1/tutorials/keras/overfit_and_underfit.ipynb b/site/ru/r1/tutorials/keras/overfit_and_underfit.ipynb deleted file mode 100644 index b78306c9198..00000000000 --- a/site/ru/r1/tutorials/keras/overfit_and_underfit.ipynb +++ /dev/null @@ -1,694 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "overfit_and_underfit.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "fTFj8ft5dlbS" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fTFj8ft5dlbS" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "lzyBOpYMdp3F", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m_x4KfSJ7Vt7", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C9HmC2T4ld5B" - }, - "source": [ - "# Знакомимся с переобучением и недообучением" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRTxFhXAlnl1" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Запусти в Google Colab\n", - " \n", - " Изучай код на GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ChkcoSVZeD9G", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19rPukKZsPG6" - }, - "source": [ - "\n", - "Как и в предыдущий раз мы будем использовать `tf.keras` API, подробнее о котором ты можешь прочитать в нашем [руководстве по Keras](https://www.tensorflow.org/r1/guide/keras).\n", - "\n", - "В обоих предыдщих примерах с классификацией обзоров фильмов и предсказанием цен на жилье, мы увидели, что точность нашей модели на проверочных данных достигает пика после определенного количества эпох, а затем начинает снижаться.\n", - "\n", - "Другими словами, наша модель учится на одних и тех же данных слишком долго - это называется *переобучение*. Очень важно знать способы как можно предотвратить это. Несмотря на то, что при помощи переобучения можно достичь более высоких показателей точности, но только на *тренировочных данных*, нашей целью всегда является обучить нейросеть обобщать их и узнавать паттерны на проверочных, новых данных.\n", - "\n", - "Обратным случаем переобучения является *недообучение*: оно возникает когда все еще есть возможность улучшить показатели модели на проверочном наборе данных. Недообучение может произойти по разным причинам: например, если модель недостаточно сильная, или слишком сложная, или просто недостаточно тренировалась на данных. В любом случае это будет означать, что не были выучены основные паттерны из проверочного сета.\n", - "\n", - "Если ты будешь тренировать модель слишком долго, то модель начнет обучаться шаблонам, которые свойственны *только* тренировочным данным, и не научится узнавать паттерны в новых данных. Нам нужно найти золотую середину. Понимание того как долго тренировать модель, сколько эпох выбрать - это очень полезный навык, которому мы сейчас научимся.\n", - "\n", - "Чтобы избежать переобучения, наиболее оптимальным решением будет использовать больше тренировочных данных. Модели, обученные на большем количестве данных, естественным образом обобщают их лучше. Когда увеличить точность более не представляется возможным, то тогда мы начинаем использовать методы *регуляризации*. Они ограничивают количество и тип инофрмации, которые модель может хранить в себе. Если нейросеть может запомнить только небольшое количество паттернов, то тогда процесс оптимизации заставит ее сфокусироваться на самых важных, наиболее заметных шаблонах, которые будут иметь более высокий шанс обобщения.\n", - "\n", - "В этом уроке мы познакомимся с двумя распространенными методами регуляризации: *регуляризация весов* и *исключение* (*dropout*). Мы используем их чтобы улучшить показатели нашей модели из урока по классификации обзоров фильмов из IMDB." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5pZ8A2liqvgk", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1cweoTiruj8O" - }, - "source": [ - "## Загружаем датасет IMDB\n", - "\n", - "Вместо того, чтобы использовать *embedding* слой, как мы делали это в предыдущем уроке, здесь мы попробуем *multi-hot-encoding*. Наша модель быстро начнет переобучаться на тренировочных данных. Мы посмотрим как это произойдет и рассмотрим способы предотвращения этого.\n", - "\n", - "Использование multi-hot-encoding на нашем массиве конвертирует его в векторы 0 и 1. Говоря конкретнее, это означает что например последовательность `[3, 5]` будет конвертирована в 10,000-размерный вектор, который будет состоять полностью из нулей за исключением 3 и 5, которые будут представлены в виде единиц." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QpzE4iqZtJly", - "colab": {} - }, - "source": [ - "NUM_WORDS = 10000\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = keras.datasets.imdb.load_data(num_words=NUM_WORDS)\n", - "\n", - "def multi_hot_sequences(sequences, dimension):\n", - " # Создаем матрицу формы (len(sequences), dimension), состоящую из нулей\n", - " results = np.zeros((len(sequences), dimension))\n", - " for i, word_indices in enumerate(sequences):\n", - " results[i, word_indices] = 1.0 # назначаем единицу на конкретные показатели results[i]\n", - " return results\n", - "\n", - "\n", - "train_data = multi_hot_sequences(train_data, dimension=NUM_WORDS)\n", - "test_data = multi_hot_sequences(test_data, dimension=NUM_WORDS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MzWVeXe3NBTn" - }, - "source": [ - "Давай посмотрим на один из получившихся multi-hot векторов. Номера слов были отсортированы по частоте, и вполне ожидаемо, что многие значения единицы будут около нуля. Проверим это на графике:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "71kr5rG4LkGM", - "colab": {} - }, - "source": [ - "plt.plot(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lglk41MwvU5o" - }, - "source": [ - "## Продемонстрируем переобучение\n", - "\n", - "Самый простой способ предотвратить переобучение, это уменьшить размер модели, или количество обучаемых параметров, которые определяются количеством слоев и блоков на каждый слой. В глубоком обучении количество обучаемых параметров часто называют *емкостью модели*. Понятно, что модель с большим количество параметров будет иметь больший запас для обучения, и следовательно легче сможет выучить взаимосвязи между тренировочными образцами данных и целевыми проверочными. Обучение же без возможности обобщения окажется бесполезным, особенно если мы попытаемся получить предсказания на новых, ранее не виденных данных.\n", - "\n", - "Всегда помни об этом: модели глубокого обучения всегда хорошо справляются с подстраиванием под тренировочные данные, но наша конечная цель - обучение обощению.\n", - "\n", - "С другой стороны, если нейросеть имеет ограниченные ресурсы для запоминания шаблонов, то тогда она не сможет так же легко находить паттерны в данных. Чтобы сократить потери, такая модель будет вынуждена обучаться сжатым представлениям, которые имеют больше предсказательной силы. В то же самое время, если мы сделаем нашу модель слишком маленькой, тогда ей будет трудно подстроиться под тренировочный сет данных. Всегда нужно искать баланс между *слишком большой емкостью* и *недостаточной емкостью*.\n", - "\n", - "К сожалению, не существует магической формулы, чтобы определить правильный размер или архитектуру модели, говоря о количестве слоев или размере каждого слоя. Тебе необходимо попробовать использовать разные архитектуры модели, прежде чем найти подходящую.\n", - "\n", - "Чтобы найди подходящий размер модели лучше начать с относительно небольших слоев и параметров, затем начать увеличивать размер слоев или добавлять новые до тех пор, пока ты показатели не начнут ухудшаться на проверочных данных. Давай попробуем разобраться на примере нашей сети для классификации обзоров.\n", - "\n", - "Для начала мы построим простую модель используя только слои ```Dense``` в качестве основы, а затем сделаем маленькую и большую версию этой модели для сравнения." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ReKHdC2EgVu" - }, - "source": [ - "### Строим основу для модели" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QKgdXPx9usBa", - "colab": {} - }, - "source": [ - "baseline_model = keras.Sequential([\n", - " # Параметр `input_shape` нужен только для того, чтобы заработал `.summary`\n", - " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "baseline_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "baseline_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LqG3MXF5xSjR", - "colab": {} - }, - "source": [ - "baseline_history = baseline_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L-DGRBbGxI6G" - }, - "source": [ - "### Создаем малый вариант" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SrfoVQheYSO5" - }, - "source": [ - "Давай построим модель с меньшим количесвом скрытых блоков и сравним ее с первой моделью:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jksi-XtaxDAh", - "colab": {} - }, - "source": [ - "smaller_model = keras.Sequential([\n", - " keras.layers.Dense(4, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(4, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "smaller_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "smaller_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jbngCZliYdma" - }, - "source": [ - "И обучим модель используя те же данные:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ofn1AwDhx-Fe", - "colab": {} - }, - "source": [ - "smaller_history = smaller_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vIPuf23FFaVn" - }, - "source": [ - "### Создаем большую модель\n", - "\n", - "В качестве упражнения ты можешь создать модель даже еще больше, и посмотреть как быстро она начнет переобучаться. Затем протестируем эту модель, которая будет иметь гораздо бóльшую емкость, чем требуется для решения нашей задачи:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ghQwwqwqvQM9", - "colab": {} - }, - "source": [ - "bigger_model = keras.models.Sequential([\n", - " keras.layers.Dense(512, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(512, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "bigger_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "bigger_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D-d-i5DaYmr7" - }, - "source": [ - "И опять потренируем уже новую модель используя те же данные:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "U1A99dhqvepf", - "colab": {} - }, - "source": [ - "bigger_history = bigger_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fy3CMUZpzH3d" - }, - "source": [ - "### Построим графики потерь\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HSlo1F4xHuuM" - }, - "source": [ - "Непрерывные линии показывают потери во время обучения, а прерывистые - во время проверки (помни - чем меньше потери на проверочных данных, тем точнее модель). В нашем случае самая маленькая модель начинает переобучаться позже, чем основная (после 6 эпох вместо 4) и ее показатели ухудшаются гораздо медленее после переобучения." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0XmKDtOWzOpk", - "colab": {} - }, - "source": [ - "def plot_history(histories, key='binary_crossentropy'):\n", - " plt.figure(figsize=(16,10))\n", - "\n", - " for name, history in histories:\n", - " val = plt.plot(history.epoch, history.history['val_'+key],\n", - " '--', label=name.title()+' Val')\n", - " plt.plot(history.epoch, history.history[key], color=val[0].get_color(),\n", - " label=name.title()+' Train')\n", - "\n", - " plt.xlabel('Epochs')\n", - " plt.ylabel(key.replace('_',' ').title())\n", - " plt.legend()\n", - "\n", - " plt.xlim([0,max(history.epoch)])\n", - "\n", - "\n", - "plot_history([('baseline', baseline_history),\n", - " ('smaller', smaller_history),\n", - " ('bigger', bigger_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bi6hBhdnSfjA" - }, - "source": [ - "Обрати внимание, что большая сеть начинает переобучаться почти сразу же после первой эпохи, и ее метрики ухудшаются гораздо быстрее. Чем больше емкость модели, тем легче она сможет вместить тренировочный сет данных, что повлечет за собой низкие потери при обучении. Но в таком случае она будет более чувствительна к переобучению: разница в потерях между обучением и проверкой будет очень велика." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ASdv7nsgEFhx" - }, - "source": [ - "## Как решить проблему переобучения?" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4rHoVWcswFLa" - }, - "source": [ - "### Добавить регуляризацию весов\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRxWepNawbBK" - }, - "source": [ - "Тебе может быть знаком принцип *бритвы Оккама*: если есть 2 толкования явления, то правильным является самое \"простое\" - то, которое содержит меньше всего предположений. Этот принцип также применим к моделям, обучемым при помощи нейронных сетей: для одних и той же сети и данных существует несколько весовых значений, или моделей, которые могут быть обучены. Простые модели переобучиваются гораздо реже, чем сложные.\n", - "\n", - "В этом контексте \"простая модель\" - та, в которой распределение значений параметров имеет меньшую энтропию. Другими словами, модель с меньшим количеством параметров, которую мы строили выше является простой. Таким образом, для предотвращение переобучения часто используется ограничение сложности сети путем уменьшения ее коэфицентов, что делает распределение более равномерным или *регулярным*. Этот метод называется *регуляризация весов*: к функции потерь нашей сети мы добавляем штраф (или *cost*, стоимость) за использование больших весов.\n", - "\n", - "Штраф имеет 2 вида:\n", - "\n", - "* Регуляризация L1 - штраф прямо пропорционален абсолютному значению коэффицентов весов (сокращенно мы называем его \"норма L1\")\n", - "\n", - "* Регуляризация L2 - штраф добавляется пропорционально квадрату значения коэффицента весов. Норму L2 также называют *угасанием весов*. Это два одинаковых названия для одной и той же математической формулы\n", - "\n", - "Чтобы осуществить регуляризацию в `tf.keras` мы добавим новый регулятор в блок со слоями как аргумент. Давай попробуем добавить L2 и посмотреть что получится:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HFGmcwduwVyQ", - "colab": {} - }, - "source": [ - "l2_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "l2_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "l2_model_history = l2_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bUUHoXb7w-_C" - }, - "source": [ - "Значение ```l2(0.001)``` означает, что каждый коэффицент матрицы весов слоя будет добавлять ```0.001 * weight_coefficient_value**2``` к общей потери сети. Обрати внимание, что штраф добавляется только во время обучения, потери во время этой стадии будут гораздо выше, чем во время проверки.\n", - "\n", - "Вот так выглядит влияние регуляризации L2:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7wkfLyxBZdh_", - "colab": {} - }, - "source": [ - "plot_history([('Базовая модель', baseline_history),\n", - " ('Регуляризация L2', l2_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Kx1YHMsVxWjP" - }, - "source": [ - "Как видишь, прошедшая L2 регуляризцию модель стала более устойчива к переобучению, чем наша изначальная, несмотря на то, что обе модели имели равное количество параметров." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HmnBNOOVxiG8" - }, - "source": [ - "### Добавить исключение Dropout\n", - "\n", - "Метод исключения (или выпадения) *Dropout* - один из самых эффективных и часто используемых приемов регуляризации нейронных сетей. Он был разработан Джеффом Хинтоном совместно с его студентами в Университете Торонто. Применяемый к слою Dropout состоит из случайно выпадающих (или равных нулю) признаков этого слоя.\n", - "\n", - "Допустим, что наш слой обычно возвращает вектор [0.2, 0.5, 1.3, 0.8, 1.1] на входной образец данных. После применения Dropout этот вектор будет случайным образом приравнивать к нулю какие-то его значения, например так - [0, 0.5, 1.3, 0, 1.1].\n", - "\n", - "Ту часть признаков, которые \"выпадут\" или обнуляться называют *коэффицентом исключения dropout*. Обычно его устанавливают между 0.2 и 0.5. Во время проверки dropout не используется, и вместо этого все выходные значения уменьшаются на соотвествующий коэффиент (скажем, 0.5). Это поможет нам сбалансировать тот факт, что во время проверки было активировано больше блоков, чем во время обучения.\n", - "\n", - "В `tf.keras` ты можешь использовать метод исключения в своей сети при помощи слоя Dropout, который применяется к выводу данных из предшествующего слоя.\n", - "\n", - "Давай добавим два слоя Dropout в нашу сеть на данных IMDB и посмотрим насколько хорошо она справится с переобучением:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OFEYvtrHxSWS", - "colab": {} - }, - "source": [ - "dpt_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(16, activation=tf.nn.relu),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "dpt_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "dpt_model_history = dpt_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SPZqwVchx5xp", - "colab": {} - }, - "source": [ - "plot_history([('Базовая модель', baseline_history),\n", - " ('Метод Dropout', dpt_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gjfnkEeQyAFG" - }, - "source": [ - "Метод Dropout имеет явные преимущества по сравнению с нашей изначальной, базовой моделью.\n", - "\n", - "Подведем итоги - вот самые основные способы предотвращения переобучения нейросетей:\n", - "\n", - "* Использовать больше данных для обучения\n", - "* Уменьшить емкость сети\n", - "* Использовать регуляризацию весов\n", - "* Или dropout\n", - "\n", - "Также существуют еще два важных подхода, которые не были продемонстрированы в этом уроке: увеличение или *аугментация данных* и *нормализация батча*." - ] - } - ] -} \ No newline at end of file diff --git a/site/ru/r1/tutorials/keras/save_and_restore_models.ipynb b/site/ru/r1/tutorials/keras/save_and_restore_models.ipynb deleted file mode 100644 index aef29e7eb5b..00000000000 --- a/site/ru/r1/tutorials/keras/save_and_restore_models.ipynb +++ /dev/null @@ -1,851 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "save_and_restore_models.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pZJ3uY9O17VN" - }, - "source": [ - "# Сохранение и загрузка моделей" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M4Ata7_wMul1" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Запусти в Google Colab\n", - " \n", - " Изучай код на GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yWu9XXGieSIJ", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBdde4YJeJKF" - }, - "source": [ - "Прогресс обучения моделей можно сохранять во время и после обучения: тренировку можно возобновить с того места, где ты остановился. Это обычно помогает избежать долгих бесперервыных сессий обучения. Сохраняя модель, ты также можешь поделиться ею с другими, чтобы они могли воспроизвести результаты ее работы. Большинство практиков машинного обучения помимо самой модели и использованных техник также публикуют:\n", - "\n", - "* Код, при помощи которого обучалась модель\n", - "* Тренировочные веса, или параметры модели\n", - "\n", - "Публикация этих данных помогает другим понять как работает модель, а также они смогут проверить как она ведет себя с новыми данными.\n", - "\n", - "Внимание! Будь осторожен с кодом, которому ты не доверяешь. Обязательно прочти [Как использовать TensorFlow безопасно?](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)\n", - "\n", - "### Варианты\n", - "\n", - "Существуют разные способы сохранять модели TensorFlow - все зависит от API, которые ты использовал в своей модели. В этом уроке используется [tf.keras](https://www.tensorflow.org/r1/guide/keras), высокоуровневый API для построения и обучения моделей в TensorFlow. Для всех остальных подходов читай руководство по TensorFlow [Сохраняй и загружай модели](https://www.tensorflow.org/r1/guide/saved_model) или [Сохранение в Eager](https://www.tensorflow.org/r1/guide/eager#object-based_saving)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xCUREq7WXgvg" - }, - "source": [ - "## Настройка\n", - "\n", - "### Настроим и импортируем зависимости" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7l0MiTOrXtNv" - }, - "source": [ - "Установим и импортируем TensorFlow и все зависимые библиотеки:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} - }, - "source": [ - "!pip install h5py pyyaml" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbGsznErXWt6" - }, - "source": [ - "### Загрузим датасет\n", - "\n", - "Мы воспользуемся [датасетом MNIST](http://yann.lecun.com/exdb/mnist/) для обучения нашей модели, чтобы показать как сохранять веса. Ускорим процесс, используя только первые 1000 образцов данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals, unicode_literals\n", - "\n", - "import os\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "tf.__version__" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_labels = train_labels[:1000]\n", - "test_labels = test_labels[:1000]\n", - "\n", - "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", - "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "anG3iVoXyZGI" - }, - "source": [ - "### Построим модель" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wynsOBfby0Pa" - }, - "source": [ - "Давай построим простую модель, на которой мы продемонстрируем как сохранять и загружать веса моделей:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} - }, - "source": [ - "# Возвращает короткую последовательную модель\n", - "def create_model():\n", - " model = tf.keras.models.Sequential([\n", - " keras.layers.Dense(512, activation=tf.nn.relu, input_shape=(784,)),\n", - " keras.layers.Dropout(0.2),\n", - " keras.layers.Dense(10, activation=tf.nn.softmax)\n", - " ])\n", - "\n", - " model.compile(optimizer=tf.train.AdamOptimizer(),\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - "\n", - " return model\n", - "\n", - "\n", - "# Создадим модель\n", - "model = create_model()\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "soDE0W_KH8rG" - }, - "source": [ - "## Сохраняем контрольные точки" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRyd5qQQIXZm" - }, - "source": [ - "Основная задача заключается в том, чтобы автоматически сохранять модель как *во время*, так и *по окончании* обучения. Таким образом ты сможешь снова использовать модель без необходимости обучать ее заново, или просто продолжить с места, на котором обучение было приостановлено.\n", - "\n", - "Эту задачу выполняет функция обратного вызова `tf.keras.callbacks.ModelCheckpoint`. Эта функция также может быть настроена при помощи нескольких аргументов.\n", - "\n", - "### Использование функции\n", - "\n", - "Обучим нашу модель и передадим ей функцию `ModelCheckpoint`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} - }, - "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# Создадим контрольную точку при помощи callback функции\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path,\n", - " save_weights_only=True,\n", - " verbose=1)\n", - "\n", - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs = 10,\n", - " validation_data = (test_images,test_labels),\n", - " callbacks = [cp_callback]) # передаем callback обучению" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlM-sgyJO084" - }, - "source": [ - "Это создаст одну совокупность файлов контрольных точек TensorFlow, которые обновлялись в конце каждой эпохи:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} - }, - "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wlRN_f56Pqa9" - }, - "source": [ - "Теперь создадим новую необученную модель. Когда мы восстанавливаем модель только из весов, новая модель должна быть точно такой же структуры, как и старая. Поскольку архитектура модели точно такая же, мы можем опубликовать веса из другой *инстанции* модели.\n", - "\n", - "Также мы оценим точность новой модели на проверочных данных. Необученная модель будет лишь изредка угадывать правильную категорию обзоров фильмов (точность будет около 10%):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Необученная модель, точность: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1DTKpZssRSo3" - }, - "source": [ - "А теперь загрузим веса из контрольной точки и проверим еще раз:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} - }, - "source": [ - "model.load_weights(checkpoint_path)\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Восстановленная модель, точность: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpAbKkAyVPV8" - }, - "source": [ - "### Параметры вызова контрольной точки\n", - "\n", - "У callback функции есть несколько параметров, которые дают контрольным точкам уникальные имена, а также корректируют частоту сохранения.\n", - "\n", - "Обучим новую модель и укажем параметр чтобы сохранять контрольные точки через каждые 5 эпох:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} - }, - "source": [ - "# Укажем эпоху в имени файла (переведем ее в строки при помощи `str.format`)\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " checkpoint_path, verbose=1, save_weights_only=True,\n", - " # Сохраняем веса через каждые 5 эпох\n", - " period=5)\n", - "\n", - "model = create_model()\n", - "model.fit(train_images, train_labels,\n", - " epochs = 50, callbacks = [cp_callback],\n", - " validation_data = (test_images,test_labels),\n", - " verbose=0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1zFrKTjjavWI" - }, - "source": [ - "Теперь посмотрим на получившиеся контрольные точки и выберем последнюю:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} - }, - "source": [ - "! ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} - }, - "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", - "latest" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zk2ciGbKg561" - }, - "source": [ - "Помни: по умолчанию TensorFlow сохраняет только 5 последних контрольных точек.\n", - "\n", - "Для проверки восстановим модель по умолчанию и загрузим последнюю контрольную точку:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "model.load_weights(latest)\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Восстановленная модель, точность: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c2OxsJOTHxia" - }, - "source": [ - "## Как выглядят эти файлы?" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtdYhvWnH2ib" - }, - "source": [ - "Код выше сохраняет веса модели как совокупность [контрольных точек](https://www.tensorflow.org/r1/guide/saved_model#save_and_restore_variables) - форматированных файлов, которые содержат только обученные веса в двоичном формате. Они включают в себя:\n", - "* Один или несколько шардов (shard, пер. \"Часть данных\"), в которых хранятся веса твоей модели\n", - "* Индекс, который указывает какие веса хранятся в каждом шарде\n", - "\n", - "Если ты обучаешь модель на одном компьютере, то тогда у тебя будет всего один шард, оканчивающийся на `.data-00000-of-00001`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S_FA-ZvxuXQV" - }, - "source": [ - "## Сохраняем веса вручную\n", - "\n", - "Выше мы посмотрели как загружать веса в модель.\n", - "\n", - "Сохранять веса вручную так же просто, просто воспользуйся методом `Model.save_weights`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} - }, - "source": [ - "# Сохраняем веса\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# Восстанавливаем веса\n", - "model = create_model()\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", - "\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Восстановленная модель, точность: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kOGlxPRBEvV1" - }, - "source": [ - "## Сохраняем модель целиком\n", - "\n", - "Ты также можешь сохранить модель целиком в единый файл, который будет содержать все веса, конфигурацию модели и даже оптимизатор конфигурации (однако это зависит от выбранных параметров). Это позволит тебе восстановить модель и продолжить обучение позже, ровно с того момента, где ты остановился, и без правки изначального кода.\n", - "\n", - "Сохранять рабочую модель полностью весьма полезно. Например, ты можешь потом восстановить ее в TensorFlow.js ([HDF5](https://js.tensorflow.org/r1/tutorials/import-keras.html), [Сохраненные модели](https://js.tensorflow.org/r1/tutorials/import-saved-model.html)) и затем обучать и запускать ее в веб-браузерах, или конвертировать ее в формат для мобильных устройств, используя TensorFlow Lite ([HDF5](https://www.tensorflow.org/lite/convert/python_api#exporting_a_tfkeras_file_), [Сохраненные модели](https://www.tensorflow.org/lite/convert/python_api#exporting_a_savedmodel_))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" - }, - "source": [ - "### Сохраняем в формате HDF5\n", - "\n", - "В Keras есть встроенный формат для сохранения модель при помощи стандарта [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format). Для наших целей сохраненная модель будет использована как единый двоичный объект *blob*." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "# Используй keras.optimizer чтобы восстановить оптимизатор из файла HDF5\n", - "model.compile(optimizer=keras.optimizers.Adam(),\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)\n", - "\n", - "# Сохраним модель полностью в единый HDF5 файл\n", - "model.save('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" - }, - "source": [ - "Теперь воссоздадим модель из этого файла:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} - }, - "source": [ - "# Воссоздадим точно такую же модель, включая веса и оптимизатор:\n", - "new_model = keras.models.load_model('my_model.h5')\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" - }, - "source": [ - "Проверим ее точность:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} - }, - "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Восстановленная модель, точность: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "Данная техника сохраняет все:\n", - "\n", - "* Веса модели\n", - "* Конфигурацию (ее структуру)\n", - "* Параметры оптимизатора\n", - "\n", - "Keras сохраняет модель путем исследования ее архитектуры. В настоящее время он не может сохранять оптимизаторы TensorFlow из `tf.train`. В случае их использования нужно скомпилировать модель еще раз после загрузки. Таким образом ты получишь параметры оптимизатора.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPyhgcoVzqUB" - }, - "source": [ - "### Сохраняем как `saved_model`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LtcN4VIb7JkK" - }, - "source": [ - "Обрати внимание: этот метод сохранения моделей `tf.keras` является экспериментальным и может измениться в будущих версиях." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DSWiSB0Q8c46" - }, - "source": [ - "Построим новую модель:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iUvT_3qE8hV5" - }, - "source": [ - "Создадим `saved_model`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "colab": {} - }, - "source": [ - "saved_model_path = tf.contrib.saved_model.save_keras_model(model, \"./saved_models\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MjpmyPfh8-1n" - }, - "source": [ - "Сохраненные модели будут помещены в папку и отмечены текущей датой и временем в названии:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZtOvxA7V0iTv", - "colab": {} - }, - "source": [ - "!ls saved_models/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B7qfpvpY9HCe" - }, - "source": [ - "Загрузим новую модель Keras из уже сохраненной:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} - }, - "source": [ - "new_model = tf.contrib.saved_model.load_keras_model(saved_model_path)\n", - "new_model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" - }, - "source": [ - "Запустим загруженную модель:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Pc9e6G6w1AWG", - "colab": {} - }, - "source": [ - "# Оптимизатор не был восстановлен, поэтому мы укажим новый\n", - "new_model.compile(optimizer=tf.train.AdamOptimizer(),\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - "\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Загруженная модель, точность: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eUYTzSz5VxL2" - }, - "source": [ - "## Что дальше?\n", - "\n", - "Это был короткий урок по сохранению и загрузке своих моделей при помощи `tf.kers`.\n", - "\n", - "* В [руководстве по tf.keras](https://www.tensorflow.org/r1/guide/keras) рассказывается подробнее о том, как можно сохранять и загружать модели при помощи `tf.keras`\n", - "\n", - "* Статья [Сохраняй в Eager](https://www.tensorflow.org/r1/guide/eager#object_based_saving) рассказывает как сохранять модель во время Eager Execution\n", - "\n", - "* В руководстве [Сохраняй и загружай модели](https://www.tensorflow.org/r1/guide/saved_model) содержится подробный урок обо всех низкоуровневых деталях сохранения моделей TensorFlow" - ] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/keras/classification.ipynb b/site/ru/tutorials/keras/classification.ipynb deleted file mode 100644 index 8c76667c7ee..00000000000 --- a/site/ru/tutorials/keras/classification.ipynb +++ /dev/null @@ -1,1022 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# Обучи свою первую нейросеть: простая классификация" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fj66ZXAzrJC2", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "Это руководство поможет тебе обучить нейросеть, которая классифицирует изображения одежды, например, кроссовки и рубашки. Это нормально, если не все будет понятно сразу: это быстрый, ознакомительный обзор полной программы TensorFlow, где новые детали объясняются по мере их появления.\n", - "\n", - "Руководство использует [tf.keras](https://www.tensorflow.org/guide/keras), высокоуровневый API для построения и обучения моделей в TensorFlow." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jL3OqFKZ9dFg", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow и tf.keras\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# Вспомогательные библиотеки\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## Загружаем датасет Fashion MNIST" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "Это руководство использует датасет [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) который содержит 70,000 монохромных изображений в 10 категориях. На каждом изображении содержится по одному предмету одежды в низком разрешении (28 на 28 пикселей):\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Образцы Fashion-MNIST (Zalando, лицензия MIT).
     \n", - "
    \n", - "\n", - "Fashion MNIST предназначен для замены классического датасета [MNIST](http://yann.lecun.com/exdb/mnist/) который часто используют как \"Hello, World\" программ машинного обучения для компьютерного зрения. Датасет MNIST содержит изображения рукописных цифр (0, 1, 2, и т.д.) в формате идентичном формату изображений одежды которыми мы будем пользоваться здесь.\n", - "\n", - "Это руководство для разнообразия использует Fashion MNIST, и еще потому, что это проблема немного сложнее чем обычный MNIST. Оба датасета относительно малы, и используются для проверки корректности работы алгоритма. Это хорошие отправные точки для тестирования и отладки кода.\n", - "\n", - "Мы используем 60,000 изображений для обучения нейросети и 10,000 изображений чтобы проверить, насколько правильно сеть обучилась их классифицировать. Вы можете получить доступ к Fashion MNIST прямо из TensorFlow. Импортируйте и загрузите данные Fashion MNIST прямо из TensorFlow:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "Загрузка датасета возвращает четыре массива NumPy:\n", - "\n", - "* Массивы `train_images` и `train_labels` являются *тренировочным сетом* — данными, на которых модель будет обучаться.\n", - "* Модель тестируется на *проверочном сете*, а именно массивах `test_images` и `test_labels`.\n", - "\n", - "Изображения являются 28х28 массивами NumPy, где значение пикселей варьируется от 0 до 255. *Метки (labels)* - это массив целых чисел от 0 до 9. Они соответствуют *классам* одежды изображенной на картинках:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    LabelClass
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "Каждому изображению соответствует единственная метка. Так как *названия классов* не включены в датасет, сохраним их тут для дальнейшего использования при построении изображений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## Изучите данные\n", - "\n", - "Давайте посмотрим на формат данных перед обучением модели. Воспользовавшись shape мы видим, что в тренировочном датасете 60,000 изображений, каждое размером 28 x 28 пикселей:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "Соответственно, в тренировочном сете 60,000 меток:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "Каждая метка это целое число от 0 до 9:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "Проверочный сет содержит 10,000 изображений, каждое - также 28 на 28 пикселей:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "И в проверочном сете - ровно 10,000 меток:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## Предобработайте данные\n", - "\n", - "Данные должны быть предобработаны перед обучением нейросети. Если вы посмотрите на первое изображение в тренировочном сете вы увидите, что значения пикселей находятся в диапазоне от 0 до 255:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "Мы масштабируем эти значения к диапазону от 0 до 1 перед тем как скормить их нейросети. Для этого мы поделим значения на 255. Важно, чтобы *тренировочный сет* и *проверочный сет* были предобработаны одинаково:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "Чтобы убедиться, что данные в правильном формате и мы готовы построить и обучить нейросеть, выведем на экран первые 25 изображений из *тренировочного сета* и отобразим под ними наименования их классов." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## Постройте модель\n", - "\n", - "Построение модели нейронной сети требует правильной конфигурации каждого слоя, и последующей компиляции модели." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### Настройте слои\n", - "\n", - "Базовым строительным блоком нейронной сети является *слой*. Слои извлекают образы из данных, которые в них подаются. Надеемся, что эти образы имеют смысл для решаемой задачи.\n", - "\n", - "Большая часть глубокого обучения состоит из соединения в последовательность простых слоев. Большинство слоев, таких как tf.keras.layers.Dense, имеют параметры, которые настраиваются во время обучения." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation='relu'),\n", - " keras.layers.Dense(10, activation='softmax')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "Первый слой этой сети - `tf.keras.layers.Flatten`, преобразует формат изображения из двумерного массива (28 на 28 пикселей) в одномерный (размерностью 28 * 28 = 784 пикселя). Слой извлекает строки пикселей из изображения и выстраивает их в один ряд. Этот слой не имеет параметров для обучения; он только переформатирует данные.\n", - "\n", - "После разложения пикселей, нейросеть содержит два слоя `tf.keras.layers.Dense`. Это полносвязные нейронные слои. Первый `Dense` слой состоит из 128 узлов (или нейронов). Второй (и последний) 10-узловой *softmax* слой возвращает массив из 10 вероятностных оценок дающих в сумме 1. Каждый узел содержит оценку указывающую вероятность принадлежности изображения к одному из 10 классов.\n", - "\n", - "### Скомпилируйте модель\n", - "\n", - "Прежде чем модель будет готова для обучения, нам нужно указать еще несколько параметров. Они добавляются на шаге *compile* модели:\n", - "\n", - "* *Функция потерь (Loss function)* — измеряет точность модели во время обучения. Мы хотим минимизировать эту функцию чтоб \"направить\" модель в верном направлении.\n", - "* *Оптимизатор (Optimizer)* — показывает каким образом обновляется модель на основе входных данных и функции потерь.\n", - "* *Метрики (Metrics)* — используются для мониторинга тренировки и тестирования модели. Наш пример использует метрику *accuracy* равную доле правильно классифицированных изображений." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## Обучите модель\n", - "\n", - "Обучение модели нейронной сети требует выполнения следующих шагов::\n", - "\n", - "1. Подайте тренировочный данные в модель. В этом примере тренировочные данные это массивы `train_images` и `train_labels`.\n", - "2. Модель учится ассоциировать изображения с правильными классами.\n", - "3. Мы просим модель сделать прогнозы для проверочных данных, в этом примере массив test_images. Мы проверяем, соответствуют ли предсказанные классы меткам из массива test_labels.\n", - "\n", - "Для начала обучения, вызовите метод `model.fit`, который называется так, поскольку \"тренирует (fits)\" модель на тренировочных данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "В процессе обучения модели отображаются метрики потери (loss) и точности (accuracy). Эта модель достигает на тренировочных данных точности равной приблизительно 0.88 (88%)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## Оцените точность\n", - "\n", - "Далее, сравните какую точность модель покажет на проверчном датасете:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('\\nТочность на проверочных данных:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "Полученная на проверочном сете точность оказалась немного ниже, чем на тренировочном. Этот разрыв между точностью на тренировке и тесте является примером *переобучения (overfitting)* . Переобучение возникает, когда модель машинного обучения показывает на новых данных худший результат, чем на тех, на которых она обучалась." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## Сделайте предсказания\n", - "\n", - "Теперь, когда модель обучена, мы можем использовать ее чтобы сделать предсказания по поводу нескольких изображений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "Здесь полученная модель предсказала класс одежды для каждого изображения в проверочном датасете. Давайте посмотрим на первое предсказание:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "Прогноз представляет из себя массив из 10 чисел. Они описывают \"уверенность\" (confidence) модели в том, насколько изображение соответствует каждому из 10 разных видов одежды. Мы можем посмотреть какой метке соответствует максимальное значение:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "Модель полагает, что на первой картинке изображен ботинок (ankle boot), или class_names[9]. Проверка показывает, что классификация верна:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "Мы можем построить график, чтобы взглянуть на полный набор из 10 предсказаний классов." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DvYmmrpIy6Y1", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d4Ov9OFDMmOD" - }, - "source": [ - "Давайте посмотрим на нулевое изображение, предсказание и массив предсказаний." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HV5jw-5HwSmO", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ko-uzOufSCSe", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "Давайте посмотрим несколько изображений с их прогнозами. Цвет верных предсказаний синий, а неверных - красный. Число это процент уверенности (от 100) для предсказанной метки. Отметим, что модель может ошибаться даже если она очень уверена." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hQlnbqaw2Qu_", - "colab": {} - }, - "source": [ - "# Отображаем первые X тестовых изображений, их предсказанную и настоящую метки.\n", - "# Корректные предсказания окрашиваем в синий цвет, ошибочные в красный.\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "Наконец, используем обученную модель для предсказания класса на одном изображении." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yRJ7JU7JCaXT", - "colab": {} - }, - "source": [ - "# Берем одну картинку из проверочного сета.\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "Модели tf.keras оптимизированы для предсказаний на *пакетах (batch)* данных, или на множестве примеров сразу. Таким образом, даже если мы используем всего 1 картинку, нам все равно необходимо добавить ее в список:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# Добавляем изображение в пакет данных, состоящий только из одного элемента.\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "Сейчас предскажем правильную метку для изображения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ai-cpLjO-3A", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "Метод `model.predict` возвращает нам список списков, по одному для каждой картинки в пакете данных. Получите прогнозы для нашего (единственного) изображения в пакете:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "np.argmax(predictions_single[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "И, как и ранее, модель предсказывает класс 9." - ] - } - ] -} diff --git a/site/ru/tutorials/keras/overfit_and_underfit.ipynb b/site/ru/tutorials/keras/overfit_and_underfit.ipynb deleted file mode 100644 index 4764089d0b8..00000000000 --- a/site/ru/tutorials/keras/overfit_and_underfit.ipynb +++ /dev/null @@ -1,699 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "overfit_and_underfit.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "fTFj8ft5dlbS" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fTFj8ft5dlbS" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "lzyBOpYMdp3F", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m_x4KfSJ7Vt7", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C9HmC2T4ld5B" - }, - "source": [ - "# Explore overfit and underfit" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRTxFhXAlnl1" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jZwaVqMxhh9_", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19rPukKZsPG6" - }, - "source": [ - "Как всегда, код в этом примере будет использовать API `tf.keras`, о котором вы можете узнать больше в [руководстве TensorFlow Keras](https://www.tensorflow.org/guide/keras).\n", - "\n", - "В обоих предыдщих примерах с классификацией обзоров фильмов и предсказанием эффективности расхода топлива, мы увидели, что точность нашей модели на проверочных данных достигает пика после обучения за некоторое количество эпох, а затем начинает снижаться.\n", - "\n", - "Другими словами, наша модель *переобучилась* на тренировочных данных. Важно научиться работать с переобученностью. Хотя часто возможно достичь высокой точности на *обучающей выборке*, на самом деле мы хотим построить модель которая хорошо обобщается на *тестовой выборке* (данных которые модель не видела ранее).\n", - "\n", - "Обратным случаем переобучения является *недообучение*. Недообучение возникает когда еще есть возможность для улучшения модели на тестовых данных. Это может случитья по ряду причин: модель недостаточно сильная, с избыточной регуляризацией или просто недостаточно долго обучалась. Это значит, что сеть не изучила релевантные паттерны в обучающей выборке.\n", - "\n", - "Если ты будешь обучать модель слишком долго, модель начнет переобучаться и настроится на паттерны тренировочных данных которые не обобщаются на тестовые данные. Нам нужно найти баланс. Понимание того, как обучать модель за подходящее количество эпох, как мы выясним ниже - очень полезный навык.\n", - "\n", - "Лучшее решение для предотвращения переобученности - использовать больше тренировочных данных. Модель обученная на большем количестве данных естественным образом обобщает лучше. Когда это более невозможно, следующее решение - использовать техники наподобие регуляризации. Они накладывают ограничения на количество и тип информации которую ваша модель может хранить. Если нейросеть может запомнить только небольшое число паттернов, то процесс оптимизации заставит ее сфокусироваться на наиболее заметных паттернах, у которых более высокий шанс обобщения.\n", - "\n", - "В этом уроке мы познакомимся с двумя распространенными техниками регуляризации - *регуляризацией весов* и *исключением (dropout)* и используем их для того, чтобы улучшить нашу модель классификации обзоров фильмов из IMDB." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5pZ8A2liqvgk", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1cweoTiruj8O" - }, - "source": [ - "## Загрузите датасет IMDB\n", - "\n", - "Вместо использования вложения (embedding) как в предыдущем уроке, здесь мы используем multi-hot encode предложений. Эта модель быстро переобучится на тренировочных данных. Мы посмотрим как произойдет переобучение и как его предотвратить.\n", - "\n", - "Multi-hot-encoding наших списков означет их преобразование в вектора из 0 и 1. Конкретнее это значит что например последовательность `[3, 5]` преобразуется в 10 000-мерный вектор, который будет состоять полностью из нулей за исключением индексов 3 и 5 которые будут единицами." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QpzE4iqZtJly", - "colab": {} - }, - "source": [ - "NUM_WORDS = 10000\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = keras.datasets.imdb.load_data(num_words=NUM_WORDS)\n", - "\n", - "def multi_hot_sequences(sequences, dimension):\n", - " # Создадим нулевую матрицу размерности (len(sequences), dimension)\n", - " results = np.zeros((len(sequences), dimension))\n", - " for i, word_indices in enumerate(sequences):\n", - " results[i, word_indices] = 1.0 # приравняем требуемые индексы results[i] к 1\n", - " return results\n", - "\n", - "\n", - "train_data = multi_hot_sequences(train_data, dimension=NUM_WORDS)\n", - "test_data = multi_hot_sequences(test_data, dimension=NUM_WORDS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MzWVeXe3NBTn" - }, - "source": [ - "Давайте посмотрим на один из получившихся multi-hot векторов. Индексы слов были отсортированы по частоте поэтому ожидаемо много значений 1 возле нулевого индекса, что мы и видим на этом графике:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "71kr5rG4LkGM", - "colab": {} - }, - "source": [ - "plt.plot(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lglk41MwvU5o" - }, - "source": [ - "## Продемонстрируем переобучение\n", - "\n", - "Простейший способ предотвратить переобучение это сократить размер модели, т.е. количество обучаемых параметров модели (которые определяются числом слоев и элементов в каждом слое). В глубоком обучении количество обучаемых параметров модели часто называют \"емкостью\" модели. Интуитивно, модель с большим количеством параметров будет иметь большую \"запоминающую емкость\" и поэтому легко сможет выучить идеальный словарь - как отображение между обучающими примерами и их целевыми значениями, отображение безо всякой обобщающей силы. Но это будет бесполезно при прогнозировании на новых, ранее не виденных данных.\n", - "\n", - "Всегда имейте это ввиду: модели глубокого обучения хорошо настраиваются на тренировочных данных, но настоящим вызовом является обобщение, не обучение.\n", - "\n", - "С другой стороны, если нейросеть имеет ограниченные ресурсы ззапоминания, то она не сможет выучить отображение так легко. Для минимизации функции потерь модель вынуждена выучить только сжатые представления у которых больше предсказательной силы. В то же время, если вы сделаете вашу модель слишком маленькой, она с трудом подстроится под тренировочный сет. Существует баланс между \"слишком большой емкостью\" и \"недостаточной емкостью\".\n", - "\n", - "К сожалению, не существует магической формулы, чтобы определить правильный размер или архитектуру модели, говоря о количестве слоев или размере каждого слоя. Вам необходимо поэкспериментировать с использованием разных архитектур модели.\n", - "\n", - "Чтобы найди подходящий размер модели лучше начать с относительно небольшого количества слоев и параметров, затем начать увеличивать размер слоев или добавлять новые до тех пор, пока вы не увидите уменьшение отдачи функции ошибок на проверочных данных. Давай попробуем это на примере нашей сети для классификации обзоров фильмов.\n", - "\n", - "Мы построим простую модель используя только ```Dense``` слои в качестве базовой, затем создадим меньшую и большую версии модели и сравним их." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ReKHdC2EgVu" - }, - "source": [ - "### Создайте базовую модель" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QKgdXPx9usBa", - "colab": {} - }, - "source": [ - "baseline_model = keras.Sequential([\n", - " # Параметр `input_shape` нужен только для того, чтобы заработал `.summary`.\n", - " keras.layers.Dense(16, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "baseline_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "baseline_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LqG3MXF5xSjR", - "colab": {} - }, - "source": [ - "baseline_history = baseline_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L-DGRBbGxI6G" - }, - "source": [ - "### Создайте меньшую модель" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SrfoVQheYSO5" - }, - "source": [ - "Давайте построим модель с меньшим количеством скрытых нейронов чтобы сравнить ее с базовой моделью, которую мы только создали:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jksi-XtaxDAh", - "colab": {} - }, - "source": [ - "smaller_model = keras.Sequential([\n", - " keras.layers.Dense(4, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(4, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "smaller_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "smaller_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jbngCZliYdma" - }, - "source": [ - "И обучим модель используя те же данные:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ofn1AwDhx-Fe", - "colab": {} - }, - "source": [ - "smaller_history = smaller_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vIPuf23FFaVn" - }, - "source": [ - "### Создайте большую модель\n", - "\n", - "В качестве упражнения вы можете построить еще большую модель и увидеть как быстро она начнет переобучаться. Далее давайте сравним с эталоном нейросеть которая имеет намного большую емкость чем того требует задача:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ghQwwqwqvQM9", - "colab": {} - }, - "source": [ - "bigger_model = keras.models.Sequential([\n", - " keras.layers.Dense(512, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(512, activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "bigger_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "bigger_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D-d-i5DaYmr7" - }, - "source": [ - "И опять обучим модель используя те же данные:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "U1A99dhqvepf", - "colab": {} - }, - "source": [ - "bigger_history = bigger_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fy3CMUZpzH3d" - }, - "source": [ - "### Постройте графики потерь на тренировочных и проверочных данных\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HSlo1F4xHuuM" - }, - "source": [ - "Непрерывные линии показывают потери во время обучения, а прерывистые - во время проверки (помни - меньшие потери на проверочных данных указывают на лучшую модель). В нашем случае самая маленькая модель начинает переобучаться позже, чем основная (после 6 эпох вместо 4) и ее показатели ухудшаются гораздо медленее после переобучения." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0XmKDtOWzOpk", - "colab": {} - }, - "source": [ - "def plot_history(histories, key='binary_crossentropy'):\n", - " plt.figure(figsize=(16,10))\n", - "\n", - " for name, history in histories:\n", - " val = plt.plot(history.epoch, history.history['val_'+key],\n", - " '--', label=name.title()+' Val')\n", - " plt.plot(history.epoch, history.history[key], color=val[0].get_color(),\n", - " label=name.title()+' Train')\n", - "\n", - " plt.xlabel('Epochs')\n", - " plt.ylabel(key.replace('_',' ').title())\n", - " plt.legend()\n", - "\n", - " plt.xlim([0,max(history.epoch)])\n", - "\n", - "\n", - "plot_history([('baseline', baseline_history),\n", - " ('smaller', smaller_history),\n", - " ('bigger', bigger_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bi6hBhdnSfjA" - }, - "source": [ - "Обратите внимание, что большая сеть начинает переобучаться почти сразу же после первой эпохи, и переобучение происходит гораздо быстрее. Чем больше емкость модели, тем легче она смоделирует тренировочные данные (и мы получим низкое значение потерь на тренировочных данных). Но в таком случае она будет более чувствительна к переобучению: разница потерь между обучением и проверкой будет очень велика." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ASdv7nsgEFhx" - }, - "source": [ - "## Стратегии предотвращения переобучения" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4rHoVWcswFLa" - }, - "source": [ - "### Добавить регуляризацию весов\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRxWepNawbBK" - }, - "source": [ - "Вам может быть знаком принцип бритвы Оккама: из двух толкований некоторого явления, правильным скорее всего является самое \"простое\" - то, которое содержит меньше всего предположений. Этот принцип также применим к моделям, обучемым при помощи нейронных сетей: если у наших данных и сетевой архитектуры существует несколько наборов значений весов (несколько моделей) которые могут объяснить данные и более простые модели переобучаются реже, чем сложные.\n", - "\n", - "В этом контексте \"простая модель\" это модель в которой распределение значений параметров имеет меньшую энтропию (или модель с меньшим количеством параметров, как та которую мы строили выше). Таким образом, для предотвращение переобучения часто используется ограничение сложности сети путем принуждения ее коэфицентов принимать только небольшие значения, что делает распределение весов более \"регулярным\". Этот метод называется \"регуляризация весов\": к функции потерь нашей сети мы добавляем штраф (или cost, стоимость) за использование больших весов. Регуляризация бывает двух видов:\n", - "\n", - "* [L1 регуляризация](https://developers.google.com/machine-learning/glossary/#L1_regularization), где добавляемый штраф пропорционален абсолютным значениям коэффициентов весов (т.е. то что называется \"L1 нормой\" весов).\n", - "\n", - "* [L2 регуляризация](https://developers.google.com/machine-learning/glossary/#L2_regularization), где добавляемый штраф пропорционален квадрату значений коэффициентов весов (т.е. то, что называется квадратом \"L2 нормы\" весов). L2 регуляризацию также называют сокращением весов в контексте нейросетей. Не дайте разным названиям запутать себя: сокращение весов математически ровно то же самое что и L2 регуляризация.\n", - "\n", - "L1 регуляризация вводит разреженность обнуляя некотороые из ваших весовых параметров. L2 регуляризация оштрафует весовые параметры не делая их разреженными - это одна из причин, почему L2 более распространена.\n", - "\n", - "В `tf.keras` регуляризация весов добавляется передачей экземпляров регуляризатора слоям в качестве аргумента. Добавим сейчас L2 регуляризатор весов." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HFGmcwduwVyQ", - "colab": {} - }, - "source": [ - "l2_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation='relu'),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "l2_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "l2_model_history = l2_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bUUHoXb7w-_C" - }, - "source": [ - "```l2(0.001)``` значит что каждый коэффициент в матрице весов слоя добавит ```0.001 * weight_coefficient_value**2``` к значению потерь нейросети. Заметьте, что поскольку этот штраф доавляется только во время обучения, потери сети во время этой стадии будут гораздо выше чем во время теста.\n", - "\n", - "Так выглядит влияние регуляризации L2:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7wkfLyxBZdh_", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('l2', l2_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Kx1YHMsVxWjP" - }, - "source": [ - "Как вы можете видеть L2 регуляризованная модель стала более устойчивой к переобучению чем базовая модель, несмотря на то что обе модели имеют одинаковое количество параметров." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HmnBNOOVxiG8" - }, - "source": [ - "### Добавьте дропаут\n", - "\n", - "Дропаут(исключение) одна из наиболее эффективных и часто используемых техник регуляризации нейросетей разработанная Джеффом Хинтоном и его студентами в университете Торонто. Примененный к слою Dropout состоит из некоторого количества случайно \"исключенных\" (т.е. приравненных к нулю) во время обучения выходных параметров слоя. Допустим что наш слой возвращает вектор [0.2, 0.5, 1.3, 0.8, 1.1] для некоторых входных данных при обучении; после применения дропаута, в этом векторе появится несколько нулевых значений распределенных случайным образом, например [0, 0.5,\n", - "1.3, 0, 1.1]. \"Коэффициент дропаута(dropout rate)\" это доля признаков которые будут обнулены; его обычно устанавливают между 0.2 и 0.5. Во время теста дропаут не используется, вместо этого выходные данные слоев масштабируются на коэффициент равный коэффициенту дропаута, чтобы сбалансировать тот факт, что во время проверки активно больше нейронов чем во время обучения.\n", - "\n", - "В tf.keras можно ввести дропаут с помощью слоя Dropout, который применяется к выходным данным предыдущего слоя.\n", - "\n", - "Давайте применим два слоя Dropout к нашей нейросети IMDB и посмотрим насколько хорошо она сократит переобучение:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OFEYvtrHxSWS", - "colab": {} - }, - "source": [ - "dpt_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, activation='relu', input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(16, activation='relu'),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "dpt_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "dpt_model_history = dpt_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SPZqwVchx5xp", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('dropout', dpt_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gjfnkEeQyAFG" - }, - "source": [ - "Добавление дропаута явно улучает базовую модель.\n", - "\n", - "Подведем итоги - вот самые основные способы предотвращения переобучения нейросетей:\n", - "\n", - "* Использовать больше данных для обучения.\n", - "* Уменьшить емкость сети.\n", - "* Использовать регуляризацию весов.\n", - "* Добавить дропаут.\n", - "\n", - "Два важных подхода которые не были рассмотрены в данном руководстве это аугментация данных (data-augmentation) и батч-нормализация (batch normalization)." - ] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/keras/regression.ipynb b/site/ru/tutorials/keras/regression.ipynb deleted file mode 100644 index d9534dd6e0b..00000000000 --- a/site/ru/tutorials/keras/regression.ipynb +++ /dev/null @@ -1,871 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# Регрессия: Спрогнозируй эффективность расхода топлива" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fj66ZXAzrJC2", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "В задаче *регрессии*, мы хотим дать прогноз какого-либо непрерывного значения, например цену или вероятность. Сравните это с задачей *классификации*, где нужно выбрать конкретную категорию из ограниченного списка (например, есть ли на картинке яблоко или апельсин, распознайте какой фрукт на изображении).\n", - "\n", - "Этот урок использует классический датасет [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) и строит модель, предсказывающую эффективность расхода топлива автомобилей конца 70-х и начала 80-х. Чтобы сделать это, мы предоставим модели описания множества различных автомобилей того времени. Эти описания будут содержать такие параметры как количество цилиндров, лошадиных сил, объем двигателя и вес.\n", - "\n", - "В этом примере используется tf.keras API, подробнее [смотри здесь](https://www.tensorflow.org/guide/keras)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# Установим библиотеку seaborn для построения парных графиков\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Датасет Auto MPG\n", - "\n", - "Датасет доступен в [репозитории машинного обучения UCI](https://archive.ics.uci.edu/ml/).\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### Получите данные\n", - "Сперва загрузим датасет." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "Импортируем его при помощи библиотеки Pandas:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### Подготовьте данные\n", - "\n", - "Датасет содержит несколько неизвестных значений." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "Чтобы урок оставался простым, удалим эти строки." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "Столбец `\"Origin\"` на самом деле категорийный, а не числовой. Поэтому конвертируем его в one-hot" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### Разделите данные на обучающую и тестовую выборки\n", - "\n", - "Сейчас разделим датасет на обучающую и тестовую выборки.\n", - "\n", - "Тестовую выборку будем использовать для итоговой оценки нашей модели" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### Проверьте данные\n", - "\n", - "Посмотрите на совместное распределение нескольких пар колонок из тренировочного набора данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "Также посмотрите на общую статистику:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### Отделите признаки от меток\n", - "\n", - "Отделите целевые значения или \"метки\" от признаков. Обучите модель для предсказания значений." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### Нормализуйте данные\n", - "\n", - "Взгляните еще раз на блок train_stats приведенный выше. Обратите внимание на то, как отличаются диапазоны каждого из признаков." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "Это хорошая практика - нормализовать признаки у которых различные масштабы и диапазон изменений. Хотя модель *может* сходиться и без нормализации признаков, обучение при этом усложняется и итоговая модель становится зависимой от выбранных единиц измерения входных данных..\n", - "\n", - "Примечание. Мы намеренно генерируем эти статистические данные только из обучающей выборки, они же будут использоваться для нормализации тестовой выборки. Мы должны сделать это, чтобы тестовая выборка была из того распределения на которой обучалась модель." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "Для обучения модели мы будем использовать эти нормализованные данные.\n", - "\n", - "Внимание: статистики использованные для нормализации входных данных (среднее и стандартное отклонение) должны быть применены к любым другим данным, которые используются в модели. Это же касается one-hot кодирования которое мы делали ранее. Преобразования необходимо применять как к тестовым данным, так и к данным с которыми модель используется в работе." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## Модель" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### Постройте модель\n", - "\n", - "Давайте построим нашу модель. Мы будем использовать `Sequential` (последовательную) модель с двумя полносвязными скрытыми слоями, а выходной слой будет возвращать одно непрерывное значение. Этапы построения модели мы опишем в функции build_model, так как позже мы создадим еще одну модель." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation='relu', input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation='relu'),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mse',\n", - " optimizer=optimizer,\n", - " metrics=['mae', 'mse'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### Проверьте модель\n", - "\n", - "Используйте метод `.summary` чтобы напечатать простое описание модели." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "\n", - "Сейчас попробуем нашу модель. Возьмем пакет из`10` примеров из обучающей выборки и вызовем `model.predict` на них." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "Похоже все работает правильно, модель показывает результат ожидаемой размерности и типа." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### Обучите модель\n", - "\n", - "Обучите модель за 1000 эпох и запишите точность модели на тренировочных и проверочных данных в объекте `history`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# Выведем прогресс обучения в виде точек после каждой завершенной эпохи\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "Визуализируйте процесс обучения модели используя статистику содержащуюся в объекте `history`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mae'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mae'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mse'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mse'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "Полученный график показывает, небольшое улучшение или даже деградацию ошибки валидации после примерно 100 эпох обучения. Давай обновим метод model.fit чтобы автоматически прекращать обучение когда ошибка валидации Val loss прекращает улучшаться. Для этого используем функцию *EarlyStopping callback* которая проверяет показатели обучения после каждой эпохи. Если после определенного количество эпох нет никаких улучшений, то функция автоматически остановит обучение.\n", - "\n", - "Вы можете больше узнать про этот коллбек [здесь](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# Параметр patience определяет количество эпох, проверяемых на улучшение\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "График показывает что среднее значение ошибки на проверочных данных - около 2 галлонов на милю. Хорошо это или плохо? Решать тебе.\n", - "\n", - "Давай посмотрим как наша модель справится на **тестовой** выборке, которую мы еще не использовали при обучении модели. Эта проверка покажет нам какого результата ожидать от модели, когда мы будем ее использовать в реальном мире" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "### Сделайте прогноз\n", - "\n", - "Наконец, спрогнозируйте значения миль-на-галлон используя данные из тестовой выборки:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19wyogbOSU5t" - }, - "source": [ - "Вроде наша модель дает хорошие предсказания. Давайте посмотрим распределение ошибки." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m0CB5tBjSU5w" - }, - "source": [ - "Она не достаточно гауссова, но мы могли это предполагать потому что количество примеров очень мало." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## Заключение\n", - "\n", - "Это руководство познакомило тебя с несколькими способами решения задач регрессии.\n", - "\n", - "* Среднеквадратичная ошибка (MSE) это распространенная функция потерь используемая для задач регрессии (для классификации используются другие функции).\n", - "* Аналогично, показатели оценки модели для регрессии отличаются от используемых в классификации. Обычной метрикой для регрессии является средняя абсолютная ошибка (MAE).\n", - "* Когда значения числовых входных данных из разных диапазонов, каждый признак должен быть незавизимо масштабирован до одного и того же диапазона.\n", - "* Если данных для обучения немного, используй небольшую сеть из нескольких скрытых слоев. Это поможет избежать переобучения.\n", - "* Метод ранней остановки очень полезная техника для избежания переобучения." - ] - } - ] -} diff --git a/site/ru/tutorials/keras/save_and_load.ipynb b/site/ru/tutorials/keras/save_and_load.ipynb deleted file mode 100644 index 646a6e8af54..00000000000 --- a/site/ru/tutorials/keras/save_and_load.ipynb +++ /dev/null @@ -1,881 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "save_and_load.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pZJ3uY9O17VN" - }, - "source": [ - "# Сохранение и загрузка моделей" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M4Ata7_wMul1" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aM8euLg4l1bQ", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBdde4YJeJKF" - }, - "source": [ - "Прогресс обучения модели может быть сохранен во время и после обучения. Это значит, что можно продолжить обучение с того места, где оно было остановлено, что позволит избежать длительных безостановочных сессий. Сохранение также значит то, что вы можете поделиться своей моделью с другими и они смогут воспроизвести вашу работу. При публикации исследовательских моделей и техник большинство практиков машинного обучения делятся:\n", - "\n", - "* кодом необходимым для создания модели, и\n", - "* обученными весами, или параметрами модели\n", - "\n", - "Публикация этих данных помогает другим понять как работает модель и попробовать ее самостоятельно с новыми данными.\n", - "\n", - "Предупреждение: Будьте осторожны с ненадежным кодом — модели TensorFlow являются кодом. См. [Безопасное использование TensorFlow](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) для подробностей.\n", - "\n", - "### Варианты\n", - "\n", - "Существуют различные способы сохранения моделей Tensorflow - зависит от API которое ты используешь. Это руководство использует [tf.keras](https://www.tensorflow.org/guide/keras) высокоуровневый API для построения и обучения моделей в TensorFlow. Для остальных подходов см. руководство TensorFlow [сохранение и восстановление](https://www.tensorflow.org/guide/saved_model) или [Сохранение в eager](https://www.tensorflow.org/guide/eager#object-based_saving)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xCUREq7WXgvg" - }, - "source": [ - "## Установка\n", - "\n", - "### Инсталляция и импорт" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7l0MiTOrXtNv" - }, - "source": [ - "Установим и импортируем TensorFlow и зависимые библиотеки:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "!pip install pyyaml h5py # Требуется для сохранения модели в формате HDF5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "print(tf.version.VERSION)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbGsznErXWt6" - }, - "source": [ - "### Получите набор данных\n", - "\n", - "Для демонстрации того как сохранять и загружать веса вы используете [датасет MNIST](http://yann.lecun.com/exdb/mnist/). Для ускорения запусков используйте только первые 1000 примеров:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_labels = train_labels[:1000]\n", - "test_labels = test_labels[:1000]\n", - "\n", - "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", - "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "anG3iVoXyZGI" - }, - "source": [ - "### Определите модель" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wynsOBfby0Pa" - }, - "source": [ - "Начните с построения простой последовательной модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} - }, - "source": [ - "# Определим простую последовательную модель\n", - "def create_model():\n", - " model = tf.keras.models.Sequential([\n", - " keras.layers.Dense(512, activation='relu', input_shape=(784,)),\n", - " keras.layers.Dropout(0.2),\n", - " keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - "\n", - " model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - " return model\n", - "\n", - "# Создадим экземпляр базовой модели\n", - "model = create_model()\n", - "\n", - "# Распечатаем архитектуру модели\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "soDE0W_KH8rG" - }, - "source": [ - "## Сохраните контрольные точки во время обучения" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRyd5qQQIXZm" - }, - "source": [ - "Вы можете использовать обученную модель без повторного ее обучения, или продолжить обучение с того места где вы остановились, в случае если процесс обучения был прерван. Коллбек `tf.keras.callbacks.ModelCheckpoint` позволяет непрерывно сохранять модель как *во время* так и *по окончанию* обучения\n", - "\n", - "### Использование коллбека контрольной точки (checkpoint callback)\n", - "\n", - "Создайте коллбек `tf.keras.callbacks.ModelCheckpoint` который сохраняет веса только во время обучения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} - }, - "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# Создаем коллбек сохраняющий веса модели\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n", - " save_weights_only=True,\n", - " verbose=1)\n", - "\n", - "# Обучаем модель с новым коллбеком\n", - "model.fit(train_images, \n", - " train_labels, \n", - " epochs=10,\n", - " validation_data=(test_images,test_labels),\n", - " callbacks=[cp_callback]) # Pass callback to training\n", - "\n", - "# Это может генерировать предупреждения относящиеся к сохранению состояния оптимизатора.\n", - "# Эти предупреждения (и подобные предупреждения в этом уроке)\n", - "# используются для предотвращения устаревшего использования и могут быть проигнорированы." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlM-sgyJO084" - }, - "source": [ - "Это создаст единый набор файлов контрольных точек TensorFlow который обновляется в конце каждой эпохи:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} - }, - "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wlRN_f56Pqa9" - }, - "source": [ - "Создайте новую, необученную модель. При восстановлении модели только из весов, у вас должна быть модель с такой же архитектурой, как и восстанавливаемая. Так как архитектура модели такая же вы можете поделиться весами несмотря на то что это другой *экземпляр* модели.\n", - "\n", - "Сейчас восстановите свежую, необученную модель и оцените ее на тестовом наборе. Неподготовленная модель будет работать на уровне угадывания (точность ~ 10%):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} - }, - "source": [ - "# Создайте экземпляр базовой модели\n", - "model = create_model()\n", - "\n", - "# Оцените модель\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Untrained model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1DTKpZssRSo3" - }, - "source": [ - "Затем загрузите веса из чекпоинта и оцените снова:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} - }, - "source": [ - "# Загрузим веса\n", - "model.load_weights(checkpoint_path)\n", - "\n", - "# Оценим модель снова\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpAbKkAyVPV8" - }, - "source": [ - "### Параметры коллбека контрольной точки\n", - "\n", - "У коллбека есть несколько параметров которые задают контрольным точкам уникальные имена, а также корректируют частоту создания контрольных точек.\n", - "\n", - "Обучите новую модель и сохраните по разному названные чекпоинты каждые пять эпох:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} - }, - "source": [ - "# Добавим эпоху в имя файла (uses `str.format`)\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# Создадим коллбек сохраняющий веса модели каждые 5 эпох\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " filepath=checkpoint_path, \n", - " verbose=1, \n", - " save_weights_only=True,\n", - " period=5)\n", - "\n", - "# Создадим новый экземпляр модели\n", - "model = create_model()\n", - "\n", - "# Сохраним веса используя формат `checkpoint_path` format\n", - "model.save_weights(checkpoint_path.format(epoch=0))\n", - "\n", - "# Обучим модель с новым коллбеком\n", - "model.fit(train_images, \n", - " train_labels,\n", - " epochs=50, \n", - " callbacks=[cp_callback],\n", - " validation_data=(test_images,test_labels),\n", - " verbose=0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1zFrKTjjavWI" - }, - "source": [ - "Сейчас посмотрите на получившиеся чекпоинты и выберите последний:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} - }, - "source": [ - "! ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} - }, - "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", - "latest" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zk2ciGbKg561" - }, - "source": [ - "Замечание: базовый формат tensorflow сохраняет только 5 последних контрольных точек.\n", - "\n", - "Для проверки сбросим модель и загрузим последний чекпоинт:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} - }, - "source": [ - "# Создадим новый экземпляр модели\n", - "model = create_model()\n", - "\n", - "# Загрузим предварительно сохраненные веса\n", - "model.load_weights(latest)\n", - "\n", - "# Заново оценим модель\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c2OxsJOTHxia" - }, - "source": [ - "## Что это за файлы?" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtdYhvWnH2ib" - }, - "source": [ - "Приведенный выше код хранит веса в коллекции [checkpoint](https://www.tensorflow.org/guide/saved_model#save_and_restore_variables)-отформатированных файлов, которые содержат только обученные веса в двоичном формате. Чекпоинты содержат:\n", - "* Один или несколько разделов (shards) которые содержат веса вашей модели.\n", - "* Индексный файл который указывает какие веса содрержатся в каждом из разделов.\n", - "\n", - "Если Вы обучаете модель только на одной машине, у Вас будет один раздел с суффиксом: `.data-00000-of-00001`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S_FA-ZvxuXQV" - }, - "source": [ - "## Сохраните веса вручную\n", - "\n", - "Вы уже видели как загрузить веса в модель. Ручное их сохранение так же просто с использованием метода `Model.save_weights`. По умолчанию `tf.keras` и `save_weights` в частности используют формат [чекпоинтов](../../guide/keras/checkpoints) TensorFlow с расширением `.ckpt` (сохранение в [HDF5](https://js.tensorflow.org/tutorials/import-keras.html) с раширением `.h5` рассмотренно в материале [Сохранение и сериализация моделей](../../guide/keras/save_and_serialize#weights-only_saving_in_savedmodel_format) guide):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} - }, - "source": [ - "# Сохраним веса\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# Создадим новый экземпляр модели\n", - "model = create_model()\n", - "\n", - "# Восстановим веса\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# Оценим модель\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kOGlxPRBEvV1" - }, - "source": [ - "## Сохраните всю модель\n", - "\n", - "Модель и оптимизатор могут быть сохранены в файл который содержит и их состояние (веса и переменные) и конфигурацию модели. Это позволит Вам экспортировать модель так, чтобы она могла быть использована без доступа к исходному коду Python. Поскольку состояние оптимизатора восстановлено, вы можете продолжить обучение в точности с того места где остановились.\n", - "\n", - "Сохранение полностью функциональной модели очень полезно - вы можете загрузить его в TensorFlow.js ([HDF5](https://js.tensorflow.org/tutorials/import-keras.html), [Сохраненная модель](https://js.tensorflow.org/tutorials/import-saved-model.html)) и затем обучить и запустить ее в веб-браузере, или сконвертировать для исполнения в мобильных устройствах с использованием TensorFlow Lite ([HDF5](https://www.tensorflow.org/lite/convert/python_api#exporting_a_tfkeras_file_), [Сохраненная модель](https://www.tensorflow.org/lite/convert/python_api#exporting_a_savedmodel_))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" - }, - "source": [ - "### Сохраните модель в файл HDF5\n", - "\n", - "Keras также обеспечивает базовый формат сохранения с использованием стандарта [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format). Для наших целей сохраненная модель может быть рассмотрена как единый двоичный blob:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} - }, - "source": [ - "# Создадим новый экземпляр модели\n", - "model = create_model()\n", - "\n", - "# Обучим модель\n", - "model.fit(train_images, train_labels, epochs=5)\n", - "\n", - "# Сохраним всю модель в HDF5 файл\n", - "model.save('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" - }, - "source": [ - "Сейчас восстановим модель с того файла:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} - }, - "source": [ - "# Восстановим в точности ту же модель, включая веса и оптимизатор\n", - "new_model = keras.models.load_model('my_model.h5')\n", - "\n", - "# Покажем архитектуру модели\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" - }, - "source": [ - "Проверим ее точность:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} - }, - "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Точность восстановленной модели: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "Эта техника сохраняет все:\n", - "\n", - "* Значения весов\n", - "* Конфигурацию модели (архитектуру)\n", - "* Конфигурацию оптимизатора\n", - "\n", - "Keras сохраняет модели путем проверки архитектуры. Сейчас Keras не может сохранять оптимизаторы TensorFlow (из `tf.train`). При их использовании вам нужно перекомпилировать модель после загрузки и вы потеряете текущее состояние оптимизатора.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPyhgcoVzqUB" - }, - "source": [ - "### Как и `saved_model`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LtcN4VIb7JkK" - }, - "source": [ - "Предупреждение: Этот метод сохранения модели `tf.keras` экспериментальный и может измениться в будущих версиях.\n", - "\n", - "Постройте новую модель и затем обучите ее:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iUvT_3qE8hV5" - }, - "source": [ - "Создайте `saved_model`, и поместите ее в папку с временной меткой с `tf.keras.experimental.export_saved_model`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "colab": {} - }, - "source": [ - "import time\n", - "saved_model_path = \"./saved_models/{}\".format(int(time.time()))\n", - "\n", - "tf.keras.experimental.export_saved_model(model, saved_model_path)\n", - "saved_model_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MjpmyPfh8-1n" - }, - "source": [ - "Выведите на экран ваши сохраненные модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZtOvxA7V0iTv", - "colab": {} - }, - "source": [ - "!ls saved_models/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B7qfpvpY9HCe" - }, - "source": [ - "Загрузите свежую модель Keras из сохраненной модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} - }, - "source": [ - "new_model = tf.keras.experimental.load_from_saved_model(saved_model_path)\n", - "\n", - "# Проверим ее архитектуру\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" - }, - "source": [ - "Запустите предсказания с восстановленной моделью:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Yh5Mu0yOgE5J", - "colab": {} - }, - "source": [ - "model.predict(test_images).shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Pc9e6G6w1AWG", - "colab": {} - }, - "source": [ - "# Модель должна быть скомпилирована перед использованием.\n", - "# Этот шаг не требуется если сохраненная модель только разворачивается.\n", - "\n", - "new_model.compile(optimizer=model.optimizer, #Оставляем загруженный оптимизатор\n", - " loss='sparse_categorical_crossentropy', \n", - " metrics=['accuracy'])\n", - "\n", - "# Evaluate the restored model\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Точность восстановленной модели: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/keras/text_classification_with_hub.ipynb b/site/ru/tutorials/keras/text_classification_with_hub.ipynb deleted file mode 100644 index 6a3b63a5858..00000000000 --- a/site/ru/tutorials/keras/text_classification_with_hub.ipynb +++ /dev/null @@ -1,439 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_classification_with_hub.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# Классификация текстов обзоров фильмов с Keras и TensorFlow Hub" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fj66ZXAzrJC2", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "\n", - "В этом уроке мы будем классифицировать обзоры фильмов как *позитивные* или *негативные*, используя текст обзора. Это пример *бинарной* или двуклассовой классификации, важный и широко применяющийся тип задач машинного обучения.\n", - "\n", - "Учебное руководство демонстрирует применение переноса обучения (transfer learning) с использованием TensorFlow Hub и Keras.\n", - "\n", - "Мы будем использовать [набор данных IMDB](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) содержащий тексты 50,000 обзоров фильмов из [Internet Movie Database](https://www.imdb.com/). Они разделены на 25,000 обзоров для обучения, и 25,000 обзоров для проверки модели. Обучающая и тестовая выборка *сбалансированы*, т.е. содержат одинаковое количество позитивных и негативных обзоров. \n", - "\n", - "Здесь мы будем использовать [tf.keras](https://www.tensorflow.org/guide/keras), высокоуровневый API для построения и обучения моделей в TensorFlow и [TensorFlow Hub](https://www.tensorflow.org/hub), библиотека и платформа для переноса обучения. Для более продвинутого руководства по классификации текстов с использованием `tf.keras`, см. [Руководство по классификации текстов MLCC](https://developers.google.com/machine-learning/guides/text-classification/)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import tensorflow_hub as hub\n", - "import tensorflow_datasets as tfds\n", - "\n", - "print(\"Version: \", tf.__version__)\n", - "print(\"Eager mode: \", tf.executing_eagerly())\n", - "print(\"Hub version: \", hub.__version__)\n", - "print(\"GPU is\", \"available\" if tf.config.experimental.list_physical_devices(\"GPU\") else \"NOT AVAILABLE\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## Скачайте датасет IMDB\n", - "\n", - "Датасет IMDB доступен в [датасетах TensorFlow](https://github.com/tensorflow/datasets). Следующий код скачивает датасет IMDB на ваш компьютер (или в среду выполнения Colab):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "# Разобьем обучающую выборку в пропорции 60% на 40%, и у нас будет 15,000 примеров\n", - "# для обучения, 10,000 примеров для валидации и 25,000 примеров для проверки.\n", - "train_validation_split = tfds.Split.TRAIN.subsplit([6, 4])\n", - "\n", - "(train_data, validation_data), test_data = tfds.load(\n", - " name=\"imdb_reviews\", \n", - " split=(train_validation_split, tfds.Split.TEST),\n", - " as_supervised=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## Исследуйте данные \n", - "\n", - "Давайте уделим немного времени, чтобы разобраться в формате данных. Каждый пример представляет собой предложение являющееся обзором фильма и соответствующую метку. Предложение никак не предобработано. Метка является целым числом, 0 или 1, где 0 - это отрицательный отзыв, а 1 - положительный.\n", - "\n", - "Выведем первые 10 примеров." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "train_examples_batch, train_labels_batch = next(iter(train_data.batch(10)))\n", - "train_examples_batch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IFtaCHTdc-GY" - }, - "source": [ - "Также напечатаем первые 10 меток." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tvAjVXOWc6Mj", - "colab": {} - }, - "source": [ - "train_labels_batch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## Постройте модель\n", - "\n", - "Нейронная сеть создается послойно - это требует три основных архитектурных решения:\n", - "\n", - "* Как представить текст?\n", - "* Сколько слоев использовать в модели?\n", - "* Сколько *скрытых нейронов* использовать в каждом слое?\n", - "\n", - "В этом примере, входные данные состоят из предложений. Метки которые нужно предсказать являются 0 либо 1.\n", - "\n", - "Одним из способов представления текста является преобразование предложений в векторные представления слов. Мы можем использовать предварительно обученное векторное представление текста в качестве первого слоя. Это имеет три преимущества:\n", - "\n", - "* нам не нужно беспокоиться о препроцессинге текста,\n", - "* мы можем извлечь выгоду из переноса обучения,\n", - "* векторное представление фиксированного размера, поэтому его проще обрабатывать.\n", - "\n", - "Для этого примера мы используем **предобученную модель векторного представления текста** из [TensorFlow Hub](https://www.tensorflow.org/hub) называемую [google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1).\n", - "\n", - "Есть еще три другие предварительно обученных модели подходящие для этого руководства:\n", - "\n", - "* [google/tf2-preview/gnews-swivel-20dim-with-oov/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim-with-oov/1) - аналогичная [google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1), но с 2.5% словаря конвертированного в OOV buckets. Это может помочь, если словарь задачи и словарь модели не полностью совпадают.\n", - "* [google/tf2-preview/nnlm-en-dim50/1](https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1) - Намного большая модель с размером словаря ~1M и размерностью вложения 50.\n", - "* [google/tf2-preview/nnlm-en-dim128/1](https://tfhub.dev/google/tf2-preview/nnlm-en-dim128/1) - Еще большая модель с размером словаря ~1M и размерностью вложения 128." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "In2nDpTLkgKa" - }, - "source": [ - "Давайте сначала создадим слой Keras, который использует модель TensorFlow Hub для векторного представления предложений, и опробуем его на нескольких входных примерах. Обратите внимание, что независимо от длины входного текста, размерность векторного представления будет следующей: `(num_examples, embedding_dimension)`" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_NUbzVeYkgcO", - "colab": {} - }, - "source": [ - "embedding = \"https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1\"\n", - "hub_layer = hub.KerasLayer(embedding, input_shape=[], \n", - " dtype=tf.string, trainable=True)\n", - "hub_layer(train_examples_batch[:3])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dfSbV6igl1EH" - }, - "source": [ - "Давайте построим полную модель:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()\n", - "model.add(hub_layer)\n", - "model.add(tf.keras.layers.Dense(16, activation='relu'))\n", - "model.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "Для построения классификатора зададим слои последовательно:\n", - "\n", - "1. Первый слой это слой TensorFlow Hub. Этот слой использует предобученную Saved Model, отображающую предложения в векторные представления. Предобученная модель векторного представления слов которую мы используем ([google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1)) разбивает предложение на токены, встраивает каждый токен и затем объединяет вложения. В результате получаются размерности: `(num_examples, embedding_dimension)`.\n", - "2. Получившийся в результате вектор фиксированной длины пропускается сквозь полносвязный (`Dense`) слой состоящий из 16 скрытых нейронов.\n", - "3. Последний слой плотно связан с единственным выходным нейроном. С использованием функции активации `сигмоида`, значение получается между 0 и 1, представляя вероятность или уровень доверия.\n", - "\n", - "Давайте скомпилируем модель." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### Функция потерь и оптимизатор\n", - "\n", - "Для модели нам необходимо указать функцию потерь и оптимизатор для обучения. Поскольку мы решаем задачу бинарной классификации и на выходе модели будут вероятности (слой из единственного элемента с сигмоидой в качестве функции активации), то мы воспользуемся функцией потерь binary_crossentropy (пер. \"Перекрестная энтропия\").\n", - "\n", - "Это не единственный выбор для функции потерь: Вы можете, например, выбрать `mean_squared_error`. Но обычно `binary_crossentropy` лучше справляется с вероятностями - она измеряет \"дистанцию\" между распределениями вероятностей, или, как в нашем случае, между истинным распределением и предсказаниями.\n", - "\n", - "Далее, когда мы исследуем задачи регрессии (например, предсказание цен на недвижимость), мы посмотрим как использовать другую функцию потерь, которая называется среднеквадратическая ошибка (MSE).\n", - "\n", - "А сейчас настроим модель с использованием оптимизатора и функции потерь:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## Обучите модель\n", - "\n", - "Обучите модель за 20 эпох в мини-батчах по 512 образцов. Это 20 итераций по всем образцам данных в тензорах `x_train` и `y_train`. Во время обучениЯ следите за функцией потерь и точностью на 10 000 примеров из проверочного набора данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} - }, - "source": [ - "history = model.fit(train_data.shuffle(10000).batch(512),\n", - " epochs=20,\n", - " validation_data=validation_data.batch(512),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## Оцените модель\n", - "\n", - "Давайте посмотрим как работает модель. Она будет возвращать два значения. Потери (число, показывающее нашу ошибку, меньшие значения - лучше) и точность (accuracy)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data.batch(512), verbose=2)\n", - "for name, value in zip(model.metrics_names, results):\n", - " print(\"%s: %.3f\" % (name, value))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "Этот довольно наивный подход достиг точности около 87%. С более продвинутыми методами модель бы приблизилась к 95%." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## Для дальнейшего чтения\n", - "\n", - "Для более общего способа работы со строковыми входными данными и более подробного анализа прогресса точности и потерь во время обучения, взгляните [сюда](https://www.tensorflow.org/tutorials/keras/basic_text_classification)." - ] - } - ] -} diff --git a/site/ru/tutorials/load_data/csv.ipynb b/site/ru/tutorials/load_data/csv.ipynb deleted file mode 100644 index 6cc57e96816..00000000000 --- a/site/ru/tutorials/load_data/csv.ipynb +++ /dev/null @@ -1,1025 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "csv.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DweYe9FcbMK_" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AVV2e0XKbJeX", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sUtoed20cRJJ" - }, - "source": [ - "# Загрузите данные в формате CSV" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1ap_W4aQcgNT" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bv99y3uVom_t", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C-3Xbt0FfGfs" - }, - "source": [ - "Этот учебник приводит пример того как выгрузить данные в формате CSV из файла в `tf.data.Dataset`\n", - "\n", - "Данные использованные в этом примере взяты из списка пассажиров Титаника. Модель предскажет вероятность спасения пассажира основываясь на таких характеристиках, как возраст, пол, класс билета и путешествовал ли пассажир один." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" - }, - "source": [ - "## Установка" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "I4dwMQVQMQWD", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "baYFZMW_bJHh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "import functools\n", - "\n", - "import numpy as np\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ncf5t6tgL5ZI", - "colab": {} - }, - "source": [ - "TRAIN_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/train.csv\"\n", - "TEST_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/eval.csv\"\n", - "\n", - "train_file_path = tf.keras.utils.get_file(\"train.csv\", TRAIN_DATA_URL)\n", - "test_file_path = tf.keras.utils.get_file(\"eval.csv\", TEST_DATA_URL)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ONE94qulk6S", - "colab": {} - }, - "source": [ - "# Сделаем значения numpy читабельнее.\n", - "np.set_printoptions(precision=3, suppress=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wuqj601Qw0Ml" - }, - "source": [ - "## Загрузка данных\n", - "\n", - "Для начала давайте посмотрим начало CSV файла, чтобы увидеть как он отформатирован." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "54Dv7mCrf9Yw", - "colab": {} - }, - "source": [ - "!head {train_file_path}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jC9lRhV-q_R3" - }, - "source": [ - "Вы можете [загрузить данные используя pandas](pandas.ipynb), и передать массивы NumPy в TensorFlow. Если Вам нужно масштабироваться до большого количества файлов или нужен загрузчик который совместим с [TensorFlow и tf.data](../../guide/data,ipynb) то используйте функцию `tf.data.experimental.make_csv_dataset`:" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "67mfwr4v-mN_" - }, - "source": [ - "Единственная колонка которую нужно указать явно - это та значение которой вы собиратесь предсказывать с помощью модели." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iXROZm5f3V4E", - "colab": {} - }, - "source": [ - "LABEL_COLUMN = 'survived'\n", - "LABELS = [0, 1]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t4N-plO4tDXd" - }, - "source": [ - "Сейчас прочитайте данные CSV из файла и создайте датасет. \n", - "\n", - "(Для полной документации см. `tf.data.experimental.make_csv_dataset`)\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yIbUscB9sqha", - "colab": {} - }, - "source": [ - "def get_dataset(file_path, **kwargs):\n", - " dataset = tf.data.experimental.make_csv_dataset(\n", - " file_path,\n", - " batch_size=5, # Значение искусственно занижено для удобства восприятия.\n", - " label_name=LABEL_COLUMN,\n", - " na_value=\"?\",\n", - " num_epochs=1,\n", - " ignore_errors=True, \n", - " **kwargs)\n", - " return dataset\n", - "\n", - "raw_train_data = get_dataset(train_file_path)\n", - "raw_test_data = get_dataset(test_file_path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "v4oMO9MIxgTG", - "colab": {} - }, - "source": [ - "def show_batch(dataset):\n", - " for batch, label in dataset.take(1):\n", - " for key, value in batch.items():\n", - " print(\"{:20s}: {}\".format(key,value.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vHUQFKoQI6G7" - }, - "source": [ - "Каждый элемент в датасете это пакет представленный в виде кортежа (*много примеров*, *много меток*). Данные из примеров организованы в тензоры столбцы (а не тензоры строки), каждый с таким количеством элементов каков размер пакета (12 в этом случае).\n", - "\n", - "Будет лучше увидеть это вам самим." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HjrkJROoxoll", - "colab": {} - }, - "source": [ - "show_batch(raw_train_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YOYKQKmMj3D6" - }, - "source": [ - "Как вы видите столбцы в CVS с именами. Конструктор датасета использует эти имена автоматически. Если файл с которым вы работаете не содержит имен столбцов в первой строке передайте их списком строк в аргумент `column_names` функции `make_csv_dataset`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2Av8_9L3tUg1", - "colab": {} - }, - "source": [ - "CSV_COLUMNS = ['survived', 'sex', 'age', 'n_siblings_spouses', 'parch', 'fare', 'class', 'deck', 'embark_town', 'alone']\n", - "\n", - "temp_dataset = get_dataset(train_file_path, column_names=CSV_COLUMNS)\n", - "\n", - "show_batch(temp_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gZfhoX7bR9u4" - }, - "source": [ - "Этот пример будет использовать все возможные столбцы. Если не нужны некоторые столбцы в датасете, создайте список только из тех колонок, которые вы планируете использовать и передайте его в (опциональный) аргумент `select_columns` конструктора.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "S1TzSkUKwsNP", - "colab": {} - }, - "source": [ - "SELECT_COLUMNS = ['survived', 'age', 'n_siblings_spouses', 'class', 'deck', 'alone']\n", - "\n", - "temp_dataset = get_dataset(train_file_path, select_columns=SELECT_COLUMNS)\n", - "\n", - "show_batch(temp_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9cryz31lxs3e" - }, - "source": [ - "## Препроцессинг данных\n", - "\n", - "CSV файл может содержать множество видов данных. Обычно, перед тем как передать данные в вашу модель, вы хотите преобразовать эти смешанные типы в вектор фиксированной длины.\n", - "\n", - "У TensorFlow есть встроенная система для описания распространенных входных преобразований: `tf.feature_column`, см. [этот учебник](../keras/feature_columns) для подробностей.\n", - "\n", - "\n", - "Вы можете преобработать ваши данные используя любой инструмент который вам нравится (например [nltk](https://www.nltk.org/) или [sklearn](https://scikit-learn.org/stable/)), и просто передать обработанные данные в TensorFlow. \n", - "\n", - "\n", - "Главное преимущество предобработки данных внутри вашей модели это то, что когда вы экспортируете модель она включает препроцессинг. В этом случае вы можете передавать необработанные данные прямо в свою модель." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9AsbaFmCeJtF" - }, - "source": [ - "### Непрерывные данные" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xl0Q0DcfA_rt" - }, - "source": [ - "Если ваши данные уже имеют подходящий числовой формат, вы можете упаковать данные в вектор, прежде чем передать их в модель:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Yfji3J5BMxz", - "colab": {} - }, - "source": [ - "SELECT_COLUMNS = ['survived', 'age', 'n_siblings_spouses', 'parch', 'fare']\n", - "DEFAULTS = [0, 0.0, 0.0, 0.0, 0.0]\n", - "temp_dataset = get_dataset(train_file_path, \n", - " select_columns=SELECT_COLUMNS,\n", - " column_defaults = DEFAULTS)\n", - "\n", - "show_batch(temp_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zEUhI8kZCfq8", - "colab": {} - }, - "source": [ - "example_batch, labels_batch = next(iter(temp_dataset)) " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IP45_2FbEKzn" - }, - "source": [ - "Вот простая функция которая упакует вместе все колонки:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JQ0hNSL8CC3a", - "colab": {} - }, - "source": [ - "def pack(features, label):\n", - " return tf.stack(list(features.values()), axis=-1), label" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "75LA9DisEIoE" - }, - "source": [ - "Примените это к каждому элементу датасета:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VnP2Z2lwCTRl", - "colab": {} - }, - "source": [ - "packed_dataset = temp_dataset.map(pack)\n", - "\n", - "for features, labels in packed_dataset.take(1):\n", - " print(features.numpy())\n", - " print()\n", - " print(labels.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1VBvmaFrFU6J" - }, - "source": [ - "Если у вас смешанные типы данных, то вам может захотеться выделить эти простые числовые поля. API `tf.feature_column` может с этим справиться, но это повлечет за собой накладные расходы, и это стоит делать только если действительно необходимо. Вернитесь к смешанному датасету:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ad-IQ_JPFQge", - "colab": {} - }, - "source": [ - "show_batch(raw_train_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HSrYNKKcIdav", - "colab": {} - }, - "source": [ - "example_batch, labels_batch = next(iter(temp_dataset)) " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p5VtThKfGPaQ" - }, - "source": [ - "Так что выберите более общий препроцессор который выбирает список числовых свойств и упаковывает их в одну колонку:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5DRishYYGS-m", - "colab": {} - }, - "source": [ - "class PackNumericFeatures(object):\n", - " def __init__(self, names):\n", - " self.names = names\n", - "\n", - " def __call__(self, features, labels):\n", - " numeric_freatures = [features.pop(name) for name in self.names]\n", - " numeric_features = [tf.cast(feat, tf.float32) for feat in numeric_freatures]\n", - " numeric_features = tf.stack(numeric_features, axis=-1)\n", - " features['numeric'] = numeric_features\n", - "\n", - " return features, labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1SeZka9AHfqD", - "colab": {} - }, - "source": [ - "NUMERIC_FEATURES = ['age','n_siblings_spouses','parch', 'fare']\n", - "\n", - "packed_train_data = raw_train_data.map(\n", - " PackNumericFeatures(NUMERIC_FEATURES))\n", - "\n", - "packed_test_data = raw_test_data.map(\n", - " PackNumericFeatures(NUMERIC_FEATURES))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wFrw0YobIbUB", - "colab": {} - }, - "source": [ - "show_batch(packed_train_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_EPUS8fPLUb1", - "colab": {} - }, - "source": [ - "example_batch, labels_batch = next(iter(packed_train_data)) " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o2maE8d2ijsq" - }, - "source": [ - "#### Нормализация данных\n", - "\n", - "Непрерывные данные должны быть всегда нормализованы." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WKT1ASWpwH46", - "colab": {} - }, - "source": [ - "import pandas as pd\n", - "desc = pd.read_csv(train_file_path)[NUMERIC_FEATURES].describe()\n", - "desc" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cHHstcKPsMXM", - "colab": {} - }, - "source": [ - "MEAN = np.array(desc.T['mean'])\n", - "STD = np.array(desc.T['std'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "REKqO_xHPNx0", - "colab": {} - }, - "source": [ - "def normalize_numeric_data(data, mean, std):\n", - " # Центрируем данные\n", - " return (data-mean)/std\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VPsoMUgRCpUM" - }, - "source": [ - "Сейчас создайте числовой столбец. В API `tf.feature_columns.numeric_column` можно использовать аргумент `normalizer_fn` который выполнится на каждом пакете.\n", - "\n", - "Добавьте `MEAN` и `STD` к normalizer fn с помощью [`functools.partial`](https://docs.python.org/3/library/functools.html#functools.partial)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Bw0I35xRS57V", - "colab": {} - }, - "source": [ - "# See what you just created.\n", - "normalizer = functools.partial(normalize_numeric_data, mean=MEAN, std=STD)\n", - "\n", - "numeric_column = tf.feature_column.numeric_column('numeric', normalizer_fn=normalizer, shape=[len(NUMERIC_FEATURES)])\n", - "numeric_columns = [numeric_column]\n", - "numeric_column" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HZxcHXc6LCa7" - }, - "source": [ - "Когда вы обучаете модель добавьте этот столбец признаков чтобы выбрать и центрировать блок числовых данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "b61NM76Ot_kb", - "colab": {} - }, - "source": [ - "example_batch['numeric']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "j-r_4EAJAZoI", - "colab": {} - }, - "source": [ - "numeric_layer = tf.keras.layers.DenseFeatures(numeric_columns)\n", - "numeric_layer(example_batch).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M37oD2VcCO4R" - }, - "source": [ - "Использованная здесь нормализация на основе среднего требует предварительного знания средних значений каждого столбца." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tSyrkSQwYHKi" - }, - "source": [ - "### Категорийные данные\n", - "\n", - "Некоторые из столбцов в данных CSV являются категорийными. Это значит, что содержимое является одним из ограниченного числа вариантов.\n", - "\n", - "Используйте `tf.feature_column` API чтобы создать коллекцию с `tf.feature_column.indicator_column` для каждого категорийного столбца.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mWDniduKMw-C", - "colab": {} - }, - "source": [ - "CATEGORIES = {\n", - " 'sex': ['male', 'female'],\n", - " 'class' : ['First', 'Second', 'Third'],\n", - " 'deck' : ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'],\n", - " 'embark_town' : ['Cherbourg', 'Southhampton', 'Queenstown'],\n", - " 'alone' : ['y', 'n']\n", - "}\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kkxLdrsLwHPT", - "colab": {} - }, - "source": [ - "categorical_columns = []\n", - "for feature, vocab in CATEGORIES.items():\n", - " cat_col = tf.feature_column.categorical_column_with_vocabulary_list(\n", - " key=feature, vocabulary_list=vocab)\n", - " categorical_columns.append(tf.feature_column.indicator_column(cat_col))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "H18CxpHY_Nma", - "colab": {} - }, - "source": [ - "# Посмотрите что вы только что создали.\n", - "categorical_columns" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p7mACuOsArUH", - "colab": {} - }, - "source": [ - "categorical_layer = tf.keras.layers.DenseFeatures(categorical_columns)\n", - "print(categorical_layer(example_batch).numpy()[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R7-1QG99_1sN" - }, - "source": [ - "Позже, когда вы создадите модель это станет частью обработки входных данных." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPWkC4_1l3IG" - }, - "source": [ - "### Комбинированный слой предобработки" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R3QAjo1qD4p9" - }, - "source": [ - "Добавьте две коллекции столбцов признаков и передайте их в `tf.keras.layers.DenseFeatures` чтобы создать входной слой который извлечет и предобработает оба входных типа:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3-OYK7GnaH0r", - "colab": {} - }, - "source": [ - "preprocessing_layer = tf.keras.layers.DenseFeatures(categorical_columns+numeric_columns)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m7_U_K0UMSVS", - "colab": {} - }, - "source": [ - "print(preprocessing_layer(example_batch).numpy()[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DlF_omQqtnOP" - }, - "source": [ - "## Постройте модель" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lQoFh16LxtT_" - }, - "source": [ - "Постройте `tf.keras.Sequential` начиная с препроцессингового слоя `preprocessing_layer`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3mSGsHTFPvFo", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " preprocessing_layer,\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid'),\n", - "])\n", - "\n", - "model.compile(\n", - " loss='binary_crossentropy',\n", - " optimizer='adam',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hPdtI2ie0lEZ" - }, - "source": [ - "## Обучите, оцените и предскажите" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8gvw1RE9zXkD" - }, - "source": [ - "Теперь модель может быть реализована и обучена." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sW-4XlLeEQ2B", - "colab": {} - }, - "source": [ - "train_data = packed_train_data.shuffle(500)\n", - "test_data = packed_test_data" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Q_nm28IzNDTO", - "colab": {} - }, - "source": [ - "model.fit(train_data, epochs=20)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QyDMgBurzqQo" - }, - "source": [ - "После того как модель обучена вы можете проверить ее точность на множестве `test_data`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eB3R3ViVONOp", - "colab": {} - }, - "source": [ - "test_loss, test_accuracy = model.evaluate(test_data)\n", - "\n", - "print('\\n\\nTest Loss {}, Test Accuracy {}'.format(test_loss, test_accuracy))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sTrn_pD90gdJ" - }, - "source": [ - "Используйте `tf.keras.Model.predict` чтобы выводить метки на пакет или датасет пакетов." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qwcx74F3ojqe", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_data)\n", - "\n", - "# Покажем некоторые результаты\n", - "for prediction, survived in zip(predictions[:10], list(test_data)[0][1][:10]):\n", - " print(\"Predicted survival: {:.2%}\".format(prediction[0]),\n", - " \" | Actual outcome: \",\n", - " (\"SURVIVED\" if bool(survived) else \"DIED\"))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/load_data/images.ipynb b/site/ru/tutorials/load_data/images.ipynb deleted file mode 100644 index a072a3ffa57..00000000000 --- a/site/ru/tutorials/load_data/images.ipynb +++ /dev/null @@ -1,1663 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "images.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mt9dL5dIir8X" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ufPx7EiCiqgR", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "deletable": true, - "editable": true, - "id": "ucMoYase6URl" - }, - "source": [ - "# Загрузка изображений" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_Wwu5SXZmEkB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3W5HaXTbpf4H", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oxw4WahM7DU9" - }, - "source": [ - "В этом руководстве приведен простой пример загрузки датасета изображений с использованием `tf.data`.\n", - "\n", - "Набор данных, используемый в этом примере, распространяется в виде папок изображений, с одним классом изображений в каждой папке." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "deletable": true, - "editable": true, - "id": "hoQQiZDB6URn" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QGXxBuPyKJw1", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KT6CcaqgQewg", - "colab": {} - }, - "source": [ - "AUTOTUNE = tf.data.experimental.AUTOTUNE" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rxndJHNC8YPM" - }, - "source": [ - "## Скачайте и проверьте набор данных" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "deletable": true, - "editable": true, - "id": "wO0InzL66URu" - }, - "source": [ - "### Получите изображения\n", - "\n", - "Перед тем как начать любое обучение вам нужен набор изображений для обучения нейронной сети новым классам которые вы хотите распознавать. Вы уже создали архив распространяемых по свободной лицензии фото цветов для первоначального использования:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rN-Pc6Zd6awg", - "colab": {} - }, - "source": [ - "import pathlib\n", - "data_root_orig = tf.keras.utils.get_file(origin='https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", - " fname='flower_photos', untar=True)\n", - "data_root = pathlib.Path(data_root_orig)\n", - "print(data_root)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rFkFK74oO--g" - }, - "source": [ - "После скачивания 218MB, вам теперь доступна копия изображений цветов:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7onR_lWE7Njj", - "colab": {} - }, - "source": [ - "for item in data_root.iterdir():\n", - " print(item)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4yYX3ZRqGOuq", - "colab": {} - }, - "source": [ - "import random\n", - "all_image_paths = list(data_root.glob('*/*'))\n", - "all_image_paths = [str(path) for path in all_image_paths]\n", - "random.shuffle(all_image_paths)\n", - "\n", - "image_count = len(all_image_paths)\n", - "image_count" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t_BbYnLjbltQ", - "colab": {} - }, - "source": [ - "all_image_paths[:10]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkM-IpB-6URx" - }, - "source": [ - "### Просмотрите изображения\n", - "Сейчас давайте быстро просмотрим парочку изображений, чтобы вы знали с чем имеете дело:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wNGateQJ6UR1", - "colab": {} - }, - "source": [ - "import os\n", - "attributions = (data_root/\"LICENSE.txt\").open(encoding='utf-8').readlines()[4:]\n", - "attributions = [line.split(' CC-BY') for line in attributions]\n", - "attributions = dict(attributions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jgowG2xu88Io", - "colab": {} - }, - "source": [ - "import IPython.display as display\n", - "\n", - "def caption_image(image_path):\n", - " image_rel = pathlib.Path(image_path).relative_to(data_root)\n", - " return \"Image (CC BY 2.0) \" + ' - '.join(attributions[str(image_rel)].split(' - ')[:-1])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YIjLi-nX0txI", - "colab": {} - }, - "source": [ - "for n in range(3):\n", - " image_path = random.choice(all_image_paths)\n", - " display.display(display.Image(image_path))\n", - " print(caption_image(image_path))\n", - " print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OaNOr-co3WKk" - }, - "source": [ - "### Определите метку для каждого изображения" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-weOQpDw2Jnu" - }, - "source": [ - "Выведите на экран все доступные метки:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ssUZ7Qh96UR3", - "colab": {} - }, - "source": [ - "label_names = sorted(item.name for item in data_root.glob('*/') if item.is_dir())\n", - "label_names" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9l_JEBql2OzS" - }, - "source": [ - "Присвойте индекс каждой метке:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y8pCV46CzPlp", - "colab": {} - }, - "source": [ - "label_to_index = dict((name, index) for index, name in enumerate(label_names))\n", - "label_to_index" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VkXsHg162T9F" - }, - "source": [ - "Создайте список всех файлов и индексов их меток:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q62i1RBP4Q02", - "colab": {} - }, - "source": [ - "all_image_labels = [label_to_index[pathlib.Path(path).parent.name]\n", - " for path in all_image_paths]\n", - "\n", - "print(\"First 10 labels indices: \", all_image_labels[:10])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "i5L09icm9iph" - }, - "source": [ - "### Загрузите и отформатируйте изображения" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbqqRUS79ooq" - }, - "source": [ - "TensorFlow включает все инструменты которые вам могут понадобиться для загрузки и обработки изображений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jQZdySHvksOu", - "colab": {} - }, - "source": [ - "img_path = all_image_paths[0]\n", - "img_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2t2h2XCcmK1Y" - }, - "source": [ - "Вот сырые данные:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LJfkyC_Qkt7A", - "colab": {} - }, - "source": [ - "img_raw = tf.io.read_file(img_path)\n", - "print(repr(img_raw)[:100]+\"...\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "opN8AVc8mSbz" - }, - "source": [ - "Преобразуйте ее в тензор изображения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tm0tdrlfk0Bb", - "colab": {} - }, - "source": [ - "img_tensor = tf.image.decode_image(img_raw)\n", - "\n", - "print(img_tensor.shape)\n", - "print(img_tensor.dtype)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3k-Of2Tfmbeq" - }, - "source": [ - "Измените ее размер для вашей модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XFpz-3_vlJgp", - "colab": {} - }, - "source": [ - "img_final = tf.image.resize(img_tensor, [192, 192])\n", - "img_final = img_final/255.0\n", - "print(img_final.shape)\n", - "print(img_final.numpy().min())\n", - "print(img_final.numpy().max())\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aCsAa4Psl4AQ" - }, - "source": [ - "Оберните их в простые функции для будущего использования." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HmUiZJNU73vA", - "colab": {} - }, - "source": [ - "def preprocess_image(image):\n", - " image = tf.image.decode_jpeg(image, channels=3)\n", - " image = tf.image.resize(image, [192, 192])\n", - " image /= 255.0 # normalize to [0,1] range\n", - "\n", - " return image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "einETrJnO-em", - "colab": {} - }, - "source": [ - "def load_and_preprocess_image(path):\n", - " image = tf.io.read_file(path)\n", - " return preprocess_image(image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3brWQcdtz78y", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "image_path = all_image_paths[0]\n", - "label = all_image_labels[0]\n", - "\n", - "plt.imshow(load_and_preprocess_image(img_path))\n", - "plt.grid(False)\n", - "plt.xlabel(caption_image(img_path))\n", - "plt.title(label_names[label].title())\n", - "print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n2TCr1TQ8pA3" - }, - "source": [ - "## Постройте `tf.data.Dataset`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6H9Z5Mq63nSH" - }, - "source": [ - "### Датасет изображений" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GN-s04s-6Luq" - }, - "source": [ - "Простейший способ построения `tf.data.Dataset` это использование метода `from_tensor_slices`.\n", - "\n", - "Нарезка массива строк, приводит к датасету строк:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6oRPG3Jz3ie_", - "colab": {} - }, - "source": [ - "path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uML4JeMmIAvO" - }, - "source": [ - "Параметры `shapes` и `types` описывают содержимое каждого элемента датасета. В этом случае у нас множество скалярных двоичных строк" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mIsNflFbIK34", - "colab": {} - }, - "source": [ - "print(path_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZjyGcM8OwBJ2" - }, - "source": [ - "Сейчас создадим новый датасет который загружает и форматирует изображения на лету пройдясь с `preprocess_image` по датасету с путями к файлам." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "D1iba6f4khu-", - "colab": {} - }, - "source": [ - "image_ds = path_ds.map(load_and_preprocess_image, num_parallel_calls=AUTOTUNE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JLUPs2a-lEEJ", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.figure(figsize=(8,8))\n", - "for n, image in enumerate(image_ds.take(4)):\n", - " plt.subplot(2,2,n+1)\n", - " plt.imshow(image)\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.xlabel(caption_image(all_image_paths[n]))\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P6FNqPbxkbdx" - }, - "source": [ - "### Датасет пар `(image, label)`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YgvrWLKG67-x" - }, - "source": [ - "Используя тот же метод `from_tensor_slices` вы можете собрать датасет меток:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AgBsAiV06udj", - "colab": {} - }, - "source": [ - "label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(all_image_labels, tf.int64))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HEsk5nN0vyeX", - "colab": {} - }, - "source": [ - "for label in label_ds.take(10):\n", - " print(label_names[label.numpy()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jHjgrEeTxyYz" - }, - "source": [ - "Поскольку датасеты следуют в одном и том же порядке, вы можете просто собрать их вместе при помощи функции zip в набор данных пары `(image, label)`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AOEWNMdQwsbN", - "colab": {} - }, - "source": [ - "image_label_ds = tf.data.Dataset.zip((image_ds, label_ds))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yA2F09SJLMuM" - }, - "source": [ - "`Shapes` и `types` нового датасета это кортежи размерностей и типов описывающие каждое поле:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DuVYNinrLL-N", - "colab": {} - }, - "source": [ - "print(image_label_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2WYMikoPWOQX" - }, - "source": [ - "Примечание. Если у вас есть такие массивы, как «all_image_labels» и «all_image_paths», альтернативой «tf.data.dataset.Dataset.zip» является срез (slice) пары массивов." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HOFwZI-2WhzV", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.from_tensor_slices((all_image_paths, all_image_labels))\n", - "\n", - "# Кортежи распаковываются в позиционные аргументы отображаемой функции\n", - "def load_and_preprocess_from_path_label(path, label):\n", - " return load_and_preprocess_image(path), label\n", - "\n", - "image_label_ds = ds.map(load_and_preprocess_from_path_label)\n", - "image_label_ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vYGCgJuR_9Qp" - }, - "source": [ - "### Базовые способы обучения" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wwZavzgsIytz" - }, - "source": [ - "Для обучения модели на этом датасете, вам необходимо, чтобы данные:\n", - "\n", - "* Были хорошо перемешаны.\n", - "* Были упакованы.\n", - "* Повторялись вечно.\n", - "* Пакеты должны быть доступны как можно скорее.\n", - "\n", - "Эти свойства могут быть легко добавлены с помощью `tf.data` api." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uZmZJx8ePw_5", - "colab": {} - }, - "source": [ - "BATCH_SIZE = 32\n", - "\n", - "# Установка размера буфера перемешивания, равного набору данных, гарантирует\n", - "# полное перемешивание данных.\n", - "ds = image_label_ds.shuffle(buffer_size=image_count)\n", - "ds = ds.repeat()\n", - "ds = ds.batch(BATCH_SIZE)\n", - "# `prefetch` позволяет датасету извлекать пакеты в фоновом режиме, во время обучения модели.\n", - "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6JsM-xHiFCuW" - }, - "source": [ - "Здесь необходимо отметить несколько вещей:\n", - "\n", - "1. Важна последовательность действий.\n", - "\n", - " * `.shuffle` после `.repeat` перемешает элементы вне границ эпох (некоторые элементы будут увидены дважды в то время как другие ни разу).\n", - " * `.shuffle` после `.batch` перемешает порядок пакетов, но не перемешает элементы внутри пакета.\n", - "\n", - "1. Используйте `buffer_size` такого же размера как и датасет для полного перемешивания. Вплоть до размера набора данных, большие значения обеспечивают лучшую рандомизацию, но используют больше памяти.\n", - "\n", - "1. Из буфера перемешивания не выбрасываются элементы пока он не заполнится. Поэтому большой размер `buffer_size` может стать причиной задержки при запуске `Dataset`.\n", - "\n", - "1. Перемешанный датасет не сообщает об окончании, пока буфер перемешивания полностью не опустеет. `Dataset` перезапускается с помощью` .repeat`, вызывая еще одно ожидание заполнения буфера перемешивания.\n", - "\n", - "Последний пункт может быть решен использованием метода `tf.data.Dataset.apply` вместе со слитой функцией `tf.data.experimental.shuffle_and_repeat`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ocr6PybXNDoO", - "colab": {} - }, - "source": [ - "ds = image_label_ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE)\n", - "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GBBZMSuAmQVL" - }, - "source": [ - "### Передайте датасет в модель\n", - "\n", - "Получите копию MobileNet v2 из `tf.keras.applications`.\n", - "\n", - "Она будет использована для простого примера передачи обучения (transfer learning).\n", - "\n", - "Установите веса MobileNet необучаемыми:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KbJrXn9omO_g", - "colab": {} - }, - "source": [ - "mobile_net = tf.keras.applications.MobileNetV2(input_shape=(192, 192, 3), include_top=False)\n", - "mobile_net.trainable=False" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y7NVWiLF3Vbf" - }, - "source": [ - "Эта модель предполагает нормализацию входных данных в диапазоне `[-1,1]`:\n", - "\n", - "```\n", - "help(keras_applications.mobilenet_v2.preprocess_input)\n", - "```\n", - "\n", - "

    \n",
-        "...\n",
-        "Эта функция применяет препроцессинг \"Inception\" который преобразует\n",
-        "RGB значения из [0, 255] в [-1, 1]\n",
-        "...\n",
-        "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CboYya6LmdQI" - }, - "source": [ - "Перед передачей входных данных в модель MobilNet, вам нужно конвертировать их из диапазона `[0,1]` в `[-1,1]`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SNOkHUGv3FYq", - "colab": {} - }, - "source": [ - "def change_range(image,label):\n", - " return 2*image-1, label\n", - "\n", - "keras_ds = ds.map(change_range)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QDzZ3Nye5Rpv" - }, - "source": [ - "MobileNet возвращает `6x6` сетку признаков для каждого изображения.\n", - "\n", - "Передайте ей пакет изображений чтобы увидеть:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OzAhGkEK6WuE", - "colab": {} - }, - "source": [ - "# Датасету может понадобиться несколько секунд для старта пока заполняется буфер перемешивания.\n", - "image_batch, label_batch = next(iter(keras_ds))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LcFdiWpO5WbV", - "colab": {} - }, - "source": [ - "feature_map_batch = mobile_net(image_batch)\n", - "print(feature_map_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vrbjEvaC5XmU" - }, - "source": [ - "Постройте модель обернутую вокруг MobileNet и используйте `tf.keras.layers.GlobalAveragePooling2D` для усреднения по этим размерностям пространства перед выходным слоем `tf.keras.layers.Dense`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X0ooIU9fNjPJ", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " mobile_net,\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(len(label_names), activation = 'softmax')])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "foQYUJs97V4V" - }, - "source": [ - "Сейчас он производит выходные данные ожидаемых размеров:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1nwYxvpj7ZEf", - "colab": {} - }, - "source": [ - "logit_batch = model(image_batch).numpy()\n", - "\n", - "print(\"min logit:\", logit_batch.min())\n", - "print(\"max logit:\", logit_batch.max())\n", - "print()\n", - "\n", - "print(\"Shape:\", logit_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pFc4I_J2nNOJ" - }, - "source": [ - "Скомпилируйте модель чтобы описать процесс обучения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZWGqLEWYRNvv", - "colab": {} - }, - "source": [ - "model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=[\"accuracy\"])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tF1mO6haBOSd" - }, - "source": [ - "Есть две переменные для обучения - Dense `weights` и `bias`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pPQ5yqyKBJMm", - "colab": {} - }, - "source": [ - "len(model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kug5Wg66UJjl", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f_glpYZ-nYC_" - }, - "source": [ - "Вы готовы обучать модель.\n", - "\n", - "Отметим, что для демонстрационных целей вы запустите только 3 шага на эпоху, но обычно вам нужно указать действительное число шагов, как определено ниже, перед передачей их в `model.fit()`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AnXPRNWoTypI", - "colab": {} - }, - "source": [ - "steps_per_epoch=tf.math.ceil(len(all_image_paths)/BATCH_SIZE).numpy()\n", - "steps_per_epoch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q_8sabaaSGAp", - "colab": {} - }, - "source": [ - "model.fit(ds, epochs=1, steps_per_epoch=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UMVnoBcG_NlQ" - }, - "source": [ - "## Производительность\n", - "\n", - "Примечание: Этот раздел лишь показывает пару простых приемов которые могут помочь производительности. Для более глубокого изучения см. [Производительность входного конвейера](https://www.tensorflow.org/guide/performance/datasets).\n", - "\n", - "Простые конвейеры, использованные выше, прочитывают каждый файл отдельно во время каждой эпохи. Это подходит для локального обучения на CPU, может быть недостаточно для обучения на GPU и абсолютно неприемлемо для любого вида распределенного обучения." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oNmQqgGhLWie" - }, - "source": [ - "Чтобы исследовать производительность наших датасетов, сперва постройте простую функцию:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_gFVe1rp_MYr", - "colab": {} - }, - "source": [ - "import time\n", - "default_timeit_steps = 2*steps_per_epoch+1\n", - "\n", - "def timeit(ds, steps=default_timeit_steps):\n", - " overall_start = time.time()\n", - " # Выберем один пакет для передачи в пайплайн (заполнение буфера перемешивания),\n", - " # перед запуском таймера\n", - " it = iter(ds.take(steps+1))\n", - " next(it)\n", - "\n", - " start = time.time()\n", - " for i,(images,labels) in enumerate(it):\n", - " if i%10 == 0:\n", - " print('.',end='')\n", - " print()\n", - " end = time.time()\n", - "\n", - " duration = end-start\n", - " print(\"{} batches: {} s\".format(steps, duration))\n", - " print(\"{:0.5f} Images/s\".format(BATCH_SIZE*steps/duration))\n", - " print(\"Total time: {}s\".format(end-overall_start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TYiOr4vdLcNX" - }, - "source": [ - "Производительность данного датасета равна:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZDxLwMJOReVe", - "colab": {} - }, - "source": [ - "ds = image_label_ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjouTJadRxyp", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HsLlXMO7EWBR" - }, - "source": [ - "### Кеш" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lV1NOn2zE2lR" - }, - "source": [ - "Используйте tf.data.Dataset.cache, чтобы с легкостью кэшировать вычисления от эпохи к эпохе. Это особенно эффективно если данные помещаются в память.\n", - "\n", - "Здесь изображения кэшируются после предварительной обработки (перекодирования и изменения размера):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qj_U09xpDvOg", - "colab": {} - }, - "source": [ - "ds = image_label_ds.cache()\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rdxpvQ7VEo3y", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "usIv7MqqZQps" - }, - "source": [ - "Одним из недостатков использования кэша памяти является то, что кэш должен перестраиваться при каждом запуске, давая одинаковую начальную задержку при каждом запуске датасета:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eKX6ergKb_xd", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jUzpG4lYNkN-" - }, - "source": [ - "Если данные не помещаются в памяти, используйте файл кэша:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vIvF8K4GMq0g", - "colab": {} - }, - "source": [ - "ds = image_label_ds.cache(filename='./cache.tf-data')\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(1)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eTIj6IOmM4yA", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qqo3dyB0Z4t2" - }, - "source": [ - "Также у файла кеша есть преимущество использования быстрого перезапуска датасета без перестраивания кеша. Обратите внимание, насколько быстрее это происходит во второй раз:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hZhVdR8MbaUj", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WqOVlf8tFrDU" - }, - "source": [ - "### Файл TFRecord" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y1llOTwWFzmR" - }, - "source": [ - "#### Необработанные данные - изображения\n", - "\n", - "TFRecord файлы - это простой формат для хранения двоичных блобов (blob). Упаковывая несколько примеров в один файл, TensorFlow может читать несколько элементов за раз, что особенно важно для производительности особенно при использовании удаленного сервиса хранения, такого как GCS.\n", - "\n", - "Сперва построим файл TFRecord из необработанных данных изображений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EqtARqKuHQLu", - "colab": {} - }, - "source": [ - "image_ds = tf.data.Dataset.from_tensor_slices(all_image_paths).map(tf.io.read_file)\n", - "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", - "tfrec.write(image_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "flR2GXWFKcO1" - }, - "source": [ - "Затем построим датасет, который прочитывает файл TFRecord и обрабатывает изображения с использованием функции `preprocess_image`, которую вы задали ранее:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "j9PVUL2SFufn", - "colab": {} - }, - "source": [ - "image_ds = tf.data.TFRecordDataset('images.tfrec').map(preprocess_image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cRp1eZDRKzyN" - }, - "source": [ - "Объедините этот датасет с датасетом меток, который вы определили ранее, чтобы получить пару из `(image,label)`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7XI_nDU2KuhS", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.zip((image_ds, label_ds))\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3ReSapoPK22E", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wb7VyoKNOMms" - }, - "source": [ - "Это медленнее `cache` версии, поскольку обработанные изображения не кешируются." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NF9W-CTKkM-f" - }, - "source": [ - "#### Сериализованные тензоры" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J9HzljSPkxt0" - }, - "source": [ - "Чтобы сохранить некоторый препроцессинг в файл TFRecord сперва, как и ранее, создайте датасет обработанных изображений:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OzS0Azukkjyw", - "colab": {} - }, - "source": [ - "paths_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)\n", - "image_ds = paths_ds.map(load_and_preprocess_image)\n", - "image_ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "onWOwLpYlzJQ" - }, - "source": [ - "Сейчас вместо датасета строк `.jpeg`, у вас датасет тензоров.\n", - "\n", - "Чтобы сериализовать это в файл TFRecord сперва сконвертируйте датасет тензоров в датасет строк:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xxZSwnRllyf0", - "colab": {} - }, - "source": [ - "ds = image_ds.map(tf.io.serialize_tensor)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "w9N6hJWAkKPC", - "colab": {} - }, - "source": [ - "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", - "tfrec.write(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OlFc9dJSmcx0" - }, - "source": [ - "С кешированным препроцессингом данные могут быть выгружены из TFrecord файла очень эффективно - не забудьте только десериализовать тензор перед использованием:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BsqFyTBFmSCZ", - "colab": {} - }, - "source": [ - "ds = tf.data.TFRecordDataset('images.tfrec')\n", - "\n", - "def parse(x):\n", - " result = tf.io.parse_tensor(x, out_type=tf.float32)\n", - " result = tf.reshape(result, [192, 192, 3])\n", - " return result\n", - "\n", - "ds = ds.map(parse, num_parallel_calls=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OPs_sLV9pQg5" - }, - "source": [ - "Сейчас добавьте метки и примените те же стандартные операции, что и ранее:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XYxBwaLYnGop", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.zip((ds, label_ds))\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W8X6RmGan1-P", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/load_data/numpy.ipynb b/site/ru/tutorials/load_data/numpy.ipynb deleted file mode 100644 index df2ef59de70..00000000000 --- a/site/ru/tutorials/load_data/numpy.ipynb +++ /dev/null @@ -1,310 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "numpy.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pixDvex9KBqt" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "K16pBM8mKK7a", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TfRdquslKbO3" - }, - "source": [ - "# Загрузка данных NumPy" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-uq3F0ggKlZb" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fj66ZXAzrJC2", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-0tqX8qkXZEj" - }, - "source": [ - "В этом руководстве приведен пример загрузки данных из массивов NumPy в `tf.data.Dataset`.\n", - "\n", - "Этот пример загружает датасет MNIST из файла `.npz`. Однако источник массивов NumPy не важен.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-Ze5IBx9clLB" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "D1gtCQrnNk6b", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "k6J3JzK5NxQ6", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - " \n", - "import numpy as np\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "G0yWiN8-cpDb" - }, - "source": [ - "### Load from `.npz` file" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GLHNrFM6RWoM", - "colab": {} - }, - "source": [ - "DATA_URL = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz'\n", - "\n", - "path = tf.keras.utils.get_file('mnist.npz', DATA_URL)\n", - "with np.load(path) as data:\n", - " train_examples = data['x_train']\n", - " train_labels = data['y_train']\n", - " test_examples = data['x_test']\n", - " test_labels = data['y_test']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cCeCkvrDgCMM" - }, - "source": [ - "## Загрузите массивы NumPy с `tf.data.Dataset`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tslB0tJPgB-2" - }, - "source": [ - "Представьте что у вас есть массив примеров и соответствующий массив меток, передайте эти два массива кортежом в `tf.data.Dataset.from_tensor_slices` чтобы создать `tf.data.Dataset`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QN_8wwc5R7Qm", - "colab": {} - }, - "source": [ - "train_dataset = tf.data.Dataset.from_tensor_slices((train_examples, train_labels))\n", - "test_dataset = tf.data.Dataset.from_tensor_slices((test_examples, test_labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6Rco85bbkDfN" - }, - "source": [ - "## Используйте датасеты" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0dvl1uUukc4K" - }, - "source": [ - "### Перемешайте датасеты и разбейте их на пакеты" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GTXdRMPcSXZj", - "colab": {} - }, - "source": [ - "BATCH_SIZE = 64\n", - "SHUFFLE_BUFFER_SIZE = 100\n", - "\n", - "train_dataset = train_dataset.shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE)\n", - "test_dataset = test_dataset.batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w69Jl8k6lilg" - }, - "source": [ - "### Постройте и обучите модель" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Uhxr8py4DkDN", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer=tf.keras.optimizers.RMSprop(),\n", - " loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n", - " metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XLDzlPGgOHBx", - "colab": {} - }, - "source": [ - "model.fit(train_dataset, epochs=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2q82yN8mmKIE", - "colab": {} - }, - "source": [ - "model.evaluate(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/load_data/pandas_dataframe.ipynb b/site/ru/tutorials/load_data/pandas_dataframe.ipynb deleted file mode 100644 index 6e7604e4c10..00000000000 --- a/site/ru/tutorials/load_data/pandas_dataframe.ipynb +++ /dev/null @@ -1,509 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "pandas_dataframe.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zwBCE43Cv3PH" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\");" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "fOad0I2cv569", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YQB7yiF6v9GR" - }, - "source": [ - "# Загрузка pandas.DataFrame" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oqa952X4wQKK" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fj66ZXAzrJC2", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UmyEaf4Awl2v" - }, - "source": [ - "Из этого руководства вы узнаете, как загрузить датафрейм Pandas в `tf.data.Dataset`.\n", - "\n", - "Это руководство использует небольшой [датасет](https://archive.ics.uci.edu/ml/datasets/heart+Disease) предоставленный Фондом сердечных заболеваний кливлендской клиники. В ней несколько сотен строк в формате CSV. Каждая строка описывает пациента, а каждый столбец описывает свойство. Мы используем эту информацию чтобы предсказать имеет ли пациент сердечное заболевание, что в этом датасете является задачей двоичной классификации." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iiyC7HkqxlUD" - }, - "source": [ - "## Прочитайте данные используя pandas" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5IoRbCA2n0_V", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import pandas as pd\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-2kBGy_pxn47" - }, - "source": [ - "Скачайте файл csv содержащий датасет с пациентами." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VS4w2LePn9g3", - "colab": {} - }, - "source": [ - "csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/applied-dl/heart.csv')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6BXRPD2-xtQ1" - }, - "source": [ - "Прочитайте csv файл используя pandas." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UEfJ8TcMpe-2", - "colab": {} - }, - "source": [ - "df = pd.read_csv(csv_file)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8FkK6QIRpjd4", - "colab": {} - }, - "source": [ - "df.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_MOAKz654CT5", - "colab": {} - }, - "source": [ - "df.dtypes" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ww4lRDCS3qPh" - }, - "source": [ - "Конвертируйте столбец `thal` являющийся `object` в датафрейме в дискретные числовые значения." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LmCl5R5C2IKo", - "colab": {} - }, - "source": [ - "df['thal'] = pd.Categorical(df['thal'])\n", - "df['thal'] = df.thal.cat.codes" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s4XA1SNW2QyI", - "colab": {} - }, - "source": [ - "df.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WWRhH6r4xxQu" - }, - "source": [ - "## Загрузите данные используя `tf.data.Dataset`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GuqmVVH_yApQ" - }, - "source": [ - "Используйте `tf.data.Dataset.from_tensor_slices` чтобы прочитать значения из датафрейма pandas. \n", - "\n", - "Одним из преимуществ использования `tf.data.Dataset` является то, что он позволяет вам писать простые высокоэффективные конвейеры данных. Прочитайте [руководство по загрузке данных](https://www.tensorflow.org/guide/data) для подробностей." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2wwhILm1ycSp", - "colab": {} - }, - "source": [ - "target = df.pop('target')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W6Yc-D3aqyBb", - "colab": {} - }, - "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((df.values, target.values))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "chEnp_Swsf0a", - "colab": {} - }, - "source": [ - "for feat, targ in dataset.take(5):\n", - " print ('Признаки: {}, Цель: {}'.format(feat, targ))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GzwlAhX6xH9Q" - }, - "source": [ - "Поскольку `pd.Series` реализует протокол `__array__` он может быть использован практически везде где вы бы использовали `np.array` или `tf.Tensor`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GnpHHkpktl5y", - "colab": {} - }, - "source": [ - "tf.constant(df['thal'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9XLxRHS10Ylp" - }, - "source": [ - "Перемешайте датасет и разбейте его на пакеты." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R3dQ-83Ztsgl", - "colab": {} - }, - "source": [ - "train_dataset = dataset.shuffle(len(df)).batch(1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bB9C0XJkyQEk" - }, - "source": [ - "## Создайте и обучите модель" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hvFUYZr7OLcL" - }, - "source": [ - "" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FQd9PcPRpkP4", - "colab": {} - }, - "source": [ - "def get_compiled_model():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", - " ])\n", - "\n", - " model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ybDzNUheqxJw", - "colab": {} - }, - "source": [ - "model = get_compiled_model()\n", - "model.fit(train_dataset, epochs=15)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d6V_6F_MBiG9" - }, - "source": [ - "## Альтернатива столбцам признаков" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "X63B9vDsD8Ly" - }, - "source": [ - "Передать словарь в качестве входных данных для модели так же просто, как и создать словарь соответствия слоев `tf.keras.layers.Input`, применяя любой препроцессинг и складывая их, используя [функциональный api](../../guide/keras/functional.ipynb). Вы можете использовать это в качестве альтернативы [столбцов признаков](../keras/feature_columns.ipynb)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FwQ47_WmOBnY", - "colab": {} - }, - "source": [ - "inputs = {key: tf.keras.layers.Input(shape=(), name=key) for key in df.keys()}\n", - "x = tf.stack(list(inputs.values()), axis=-1)\n", - "\n", - "x = tf.keras.layers.Dense(10, activation='relu')(x)\n", - "output = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n", - "\n", - "model_func = tf.keras.Model(inputs=inputs, outputs=output)\n", - "\n", - "model_func.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qSCN5f_vUURE" - }, - "source": [ - "Простейший способ сохранения структуры столбцов в `pd.DataFrame` при использовании с `tf.data` это конвертация `pd.DataFrame` в `dict` и сделав срезы этого словаря." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wUjRKgEhPZqK", - "colab": {} - }, - "source": [ - "dict_slices = tf.data.Dataset.from_tensor_slices((df.to_dict('list'), target.values)).batch(16)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WWRaiwxeyA9Z", - "colab": {} - }, - "source": [ - "for dict_slice in dict_slices.take(1):\n", - " print (dict_slice)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8nTrfczNyKup", - "colab": {} - }, - "source": [ - "model_func.fit(dict_slices, epochs=15)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/load_data/text.ipynb b/site/ru/tutorials/load_data/text.ipynb deleted file mode 100644 index 898cfe2ea24..00000000000 --- a/site/ru/tutorials/load_data/text.ipynb +++ /dev/null @@ -1,663 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DweYe9FcbMK_" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AVV2e0XKbJeX", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sUtoed20cRJJ" - }, - "source": [ - "# Загрузка текста" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1ap_W4aQcgNT" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fj66ZXAzrJC2", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NWeQAo0Ec_BL" - }, - "source": [ - "Это руководство предлагает пример того как использовать `tf.data.TextLineDataset` для загрузки примеров из текстовых файлов. `TextLineDataset` предназначен для создания набора данных из текстового файла, в которых каждый пример это строка текста в исходном файле. Это потенциально полезно для любой текстовых данных которые изначально строковые (например, поэзия или логи ошибок).\n", - "\n", - "В этом руководстве мы будем использовать три разных английских перевода одного и того же текста - \"Илиады\" Гомера, и обучим модель определять переводчика на основе одной строки из текста." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" - }, - "source": [ - "## Установка" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "baYFZMW_bJHh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import tensorflow_datasets as tfds\n", - "import os" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YWVWjyIkffau" - }, - "source": [ - "Тексты трех переводов выполнили:\n", - "\n", - " - [Вильям Купер](https://en.wikipedia.org/wiki/William_Cowper) — [текст](https://storage.googleapis.com/download.tensorflow.org/data/illiad/cowper.txt)\n", - "\n", - " - [Эрл Эдвард](https://en.wikipedia.org/wiki/Edward_Smith-Stanley,_14th_Earl_of_Derby) — [текст](https://storage.googleapis.com/download.tensorflow.org/data/illiad/derby.txt)\n", - "\n", - "- [Сэмюель Батлер](https://en.wikipedia.org/wiki/Samuel_Butler_%28novelist%29) — [текст](https://storage.googleapis.com/download.tensorflow.org/data/illiad/butler.txt)\n", - "\n", - "Текстовые файлы использованные в этом руководстве были подвергнуты некоторым типичным задачам предварительной обработки, в основном удаление материала - шапка и футер документа, нумерация строк, заголовки глав. Скачайте эти файлы локально." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4YlKQthEYlFw", - "colab": {} - }, - "source": [ - "DIRECTORY_URL = 'https://storage.googleapis.com/download.tensorflow.org/data/illiad/'\n", - "FILE_NAMES = ['cowper.txt', 'derby.txt', 'butler.txt']\n", - "\n", - "for name in FILE_NAMES:\n", - " text_dir = tf.keras.utils.get_file(name, origin=DIRECTORY_URL+name)\n", - " \n", - "parent_dir = os.path.dirname(text_dir)\n", - "\n", - "parent_dir" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "q3sDy6nuXoNp" - }, - "source": [ - "## Загрузите текст в датасеты\n", - "\n", - "Переберите файлы, загружая каждый в свой датасет.\n", - "\n", - "Каждый пример нужно пометить индивидуально, так что используйте `tf.data.Dataset.map` чтобы применить функцию расставляющую метки каждому элементу. Она переберет каждую запись в датасете возвращая пару (`example, label`)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K0BjCOpOh7Ch", - "colab": {} - }, - "source": [ - "def labeler(example, index):\n", - " return example, tf.cast(index, tf.int64) \n", - "\n", - "labeled_data_sets = []\n", - "\n", - "for i, file_name in enumerate(FILE_NAMES):\n", - " lines_dataset = tf.data.TextLineDataset(os.path.join(parent_dir, file_name))\n", - " labeled_dataset = lines_dataset.map(lambda ex: labeler(ex, i))\n", - " labeled_data_sets.append(labeled_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M8PHK5J_cXE5" - }, - "source": [ - "Объедините эти размеченные наборы данных в один и перемешайте его.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6jAeYkTIi9-2", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 50000\n", - "BATCH_SIZE = 64\n", - "TAKE_SIZE = 5000" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qd544E-Sh63L", - "colab": {} - }, - "source": [ - "all_labeled_data = labeled_data_sets[0]\n", - "for labeled_dataset in labeled_data_sets[1:]:\n", - " all_labeled_data = all_labeled_data.concatenate(labeled_dataset)\n", - " \n", - "all_labeled_data = all_labeled_data.shuffle(\n", - " BUFFER_SIZE, reshuffle_each_iteration=False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r4JEHrJXeG5k" - }, - "source": [ - "Вы можете использовать `tf.data.Dataset.take` и `print`, чтобы посмотреть как выглядят пары `(example, label)`. Свойство `numpy` показывает каждое значение тензора." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gywKlN0xh6u5", - "colab": {} - }, - "source": [ - "for ex in all_labeled_data.take(5):\n", - " print(ex)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rrpU2_sfDh0" - }, - "source": [ - "## Закодируйте текстовые строки числами\n", - "\n", - "Модели машинного обучения работают с числами, не словами, так что строковые значения необходимо конвертировать в списки с числами. Чтобы сделать это поставьте в соответствие каждому слову свое число.\n", - "\n", - "### Создайте словарь\n", - "\n", - "Сперва создайте словарь токенизировав текст в коллекцию отдельных отличающихся слов. Есть несколько способов сделать это и в TensorFlow и в Python. В этом учебнике:\n", - "\n", - "1. Переберите `numpy` значения всех примеров.\n", - "2. Используйте `tfds.features.text.Tokenizer` чтобы разбить их на токены.\n", - "3. Соберите эти токены в множество Python чтобы избавиться от дубликатов\n", - "4. Получите размер словаря для последующего использования." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YkHtbGnDh6mg", - "colab": {} - }, - "source": [ - "tokenizer = tfds.features.text.Tokenizer()\n", - "\n", - "vocabulary_set = set()\n", - "for text_tensor, _ in all_labeled_data:\n", - " some_tokens = tokenizer.tokenize(text_tensor.numpy())\n", - " vocabulary_set.update(some_tokens)\n", - "\n", - "vocab_size = len(vocabulary_set)\n", - "vocab_size" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0W35VJqAh9zs" - }, - "source": [ - "### Закодируйте примеры\n", - "\n", - "Создайте кодировщик передав `vocabulary_set` в `tfds.features.text.TokenTextEncoder`. Метод `encode` кодировщика берет строку текста и возвращает список чисел." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gkxJIVAth6j0", - "colab": {} - }, - "source": [ - "encoder = tfds.features.text.TokenTextEncoder(vocabulary_set)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "v6S5Qyabi-vo" - }, - "source": [ - "Вы можете посмотреть одну строку чтобы увидеть как выглядит результат работы кодировщика." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jgxPZaxUuTbk", - "colab": {} - }, - "source": [ - "example_text = next(iter(all_labeled_data))[0].numpy()\n", - "print(example_text)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XoVpKR3qj5yb", - "colab": {} - }, - "source": [ - "encoded_example = encoder.encode(example_text)\n", - "print(encoded_example)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p9qHM0v8k_Mg" - }, - "source": [ - "Теперь запустите кодировщик на датасете обернув его в `tf.py_function` и передав в метод `map` датасета." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HcIQ7LOTh6eT", - "colab": {} - }, - "source": [ - "def encode(text_tensor, label):\n", - " encoded_text = encoder.encode(text_tensor.numpy())\n", - " return encoded_text, label\n", - "\n", - "def encode_map_fn(text, label):\n", - " return tf.py_function(encode, inp=[text, label], Tout=(tf.int64, tf.int64))\n", - "\n", - "all_encoded_data = all_labeled_data.map(encode_map_fn)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_YZToSXSm0qr" - }, - "source": [ - "## Разбейте датасет на тестовую и обучающую выборки\n", - "\n", - "Используйте `tf.data.Dataset.take` и `tf.data.Dataset.skip` чтобы создать небольшой тестовый и большой обучающий датасеты.\n", - "\n", - "Перед передачей в модель датасет должны быть разбиты на пакеты. Обычно количество записей в пакете и их размерность должно быть одинаковые." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "r-rmbijQh6bf", - "colab": {} - }, - "source": [ - "train_data = all_encoded_data.skip(TAKE_SIZE).shuffle(BUFFER_SIZE)\n", - "train_data = train_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))\n", - "\n", - "test_data = all_encoded_data.take(TAKE_SIZE)\n", - "test_data = test_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xdz7SVwmqi1l" - }, - "source": [ - "Сейчас, `test_data` и `train_data` являются не коллекциями пар (`example, label`), а коллекциями пакетов. Каждый пакет это пара вида (*много примеров*, *много меток*) представленная в виде массивов.\n", - "\n", - "Чтобы проиллюстрировать:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kMslWfuwoqpB", - "colab": {} - }, - "source": [ - "sample_text, sample_labels = next(iter(test_data))\n", - "\n", - "sample_text[0], sample_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UI4I6_Sa0vWu" - }, - "source": [ - "Так как мы ввели новую кодировку токенов (нуль использовался для заполнения), размер словаря увеличился на единицу." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IlD1Lli91vuc", - "colab": {} - }, - "source": [ - "vocab_size += 1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K8SUhGFNsmRi" - }, - "source": [ - "## Постройте модель\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QJgI1pow2YR9", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wi0iiKLTKdoF" - }, - "source": [ - "Первый слой конвертирует целочисленные представления в плотные векторные вложения. См. руководство [Вложения слов] (../../tutorials/sequences/word_embeddings) для подробностей." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DR6-ctbY638P", - "colab": {} - }, - "source": [ - "model.add(tf.keras.layers.Embedding(vocab_size, 64))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_8OJOPohKh1q" - }, - "source": [ - "Следующий слой является [Long Short-Term Memory](http://colah.github.io/posts/2015-08-Understanding-LSTMs/) слоем, который позволяет моедли понять слова в контексте других слов. Двунаправленная обертка LSTM позволяет ей выучить взаимодействие элементов как с предыдущими так и с последующими элементами." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x6rnq6DN_WUs", - "colab": {} - }, - "source": [ - "model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cdffbMr5LF1g" - }, - "source": [ - "И наконец у нас есть серии из одной и более плотно связанных слоев, последний из которых - выходной слой. Выходной слой генерирует вероятность для всех меток. Та у которой большая вероятность и является предсказанием модели для этого примера." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QTEaNSnLCsv5", - "colab": {} - }, - "source": [ - "# Один или более плотных слоев.\n", - "# Отредактируйте список в строке `for` чтобы поэкспериментировать с размером слоев.\n", - "for units in [64, 64]:\n", - " model.add(tf.keras.layers.Dense(units, activation='relu'))\n", - "\n", - "# Выходной слой. Первый аргумент - число меток.\n", - "model.add(tf.keras.layers.Dense(3, activation='softmax'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zLHPU8q5DLi_" - }, - "source": [ - "Наконец скомпилируйте модель. Для softmax категоризационной модели используйте `sparse_categorical_crossentropy` в виде функции потерь. Вы можете попробовать другие оптимизаторы, но `adam` очень часто используемая." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pkTBUVO4h6Y5", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DM-HLo5NDhql" - }, - "source": [ - "## Обучите модель\n", - "\n", - "Наша модель, работающая на этих данных, дает достойные результаты (около 83%)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aLtO33tNh6V8", - "colab": {} - }, - "source": [ - "model.fit(train_data, epochs=3, validation_data=test_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KTPCYf_Jh6TH", - "colab": {} - }, - "source": [ - "eval_loss, eval_acc = model.evaluate(test_data)\n", - "\n", - "print('\\nEval loss: {:.3f}, Eval accuracy: {:.3f}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/load_data/tfrecord.ipynb b/site/ru/tutorials/load_data/tfrecord.ipynb deleted file mode 100644 index 9c284d1c1f7..00000000000 --- a/site/ru/tutorials/load_data/tfrecord.ipynb +++ /dev/null @@ -1,1265 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pL--_KGdYoBz" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "uBDvXpYzYnGj" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HQzaEQuJiW_d" - }, - "source": [ - "# TFRecord и tf.Example\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/load_data/tfrecord\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eСмотрите на TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/ru/tutorials/load_data/tfrecord.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eЗапустите в Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/ru/tutorials/load_data/tfrecord.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eИзучайте код на GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/ru/tutorials/load_data/tfrecord.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eСкачайте ноутбук\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gVN4BFQ5Dg4v", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3pkUd_9IZCFO" - }, - "source": [ - "Чтобы эффективно читать данные будет полезно сериализовать ваши данные и держать их в наборе файлов (по 100-200MB каждый) каждый из которых может быть прочитан построчно. Это особенно верно если данные передаются по сети. Также это может быть полезно для кеширования и предобработки данных.\n", - "\n", - "Формат TFRecord это простой формат для хранения последовательности двоичных записей.\n", - "\n", - "[Protocol buffers](https://developers.google.com/protocol-buffers/) это кросс-платформенная, кросс-языковая библиотека для эффективной сериализации структурированных данных.\n", - "\n", - "Сообщения протокола обычно определяются файлами `.proto`. Это часто простейший способ понять тип сообщения.\n", - "\n", - "Сообщение `tf.Example` (или protobuf) гибкий тип сообщений, который преедставляет сопоставление `{\"string\": value}`. Он разработан для использования с TensorFlow и используется в высокоуровневых APIs таких как [TFX](https://www.tensorflow.org/tfx/)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ac83J0QxjhFt" - }, - "source": [ - "\n", - "Этот урок покажет как создавать, парсить и использовать сообщение `tf.Example`, а затем сериализовать читать и писать сообщения `tf.Example` в/из файлов `.tfrecord`.\n", - "\n", - "Замечание: Хотя эти структуры полезны, они необязательны. Нет необходимости конвертировать существующий код для использования TFRecords если вы не используете [`tf.data`](https://www.tensorflow.org/guide/datasets) и чтение данных все еще узкое место обучения. См. [Производительность конвейера входных данных](https://www.tensorflow.org/guide/performance/datasets) для советов по производительности датасета." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WkRreBf1eDVc" - }, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Ja7sezsmnXph" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import numpy as np\n", - "import IPython.display as display" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e5Kq88ccUWQV" - }, - "source": [ - "## `tf.Example`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VrdQHgvNijTi" - }, - "source": [ - "### Типы данных для `tf.Example`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lZw57Qrn4CTE" - }, - "source": [ - "Фундаментально `tf.Example` это соответствие `{\"string\": tf.train.Feature}`.\n", - "\n", - "Вид сообщений `tf.train.Feature` допускает один из следующих трех типов (См. [файл `.proto`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto) для справки). Большинство других общих типов может быть сведено к одному из этих трех:\n", - "\n", - "1. `tf.train.BytesList` (можно привести следующие типы)\n", - "\n", - " - `string`\n", - " - `byte`\n", - "\n", - "1. `tf.train.FloatList` (можно привести следующие типы)\n", - "\n", - " - `float` (`float32`)\n", - " - `double` (`float64`)\n", - "\n", - "1. `tf.train.Int64List` (можно привести следующие типы)\n", - "\n", - " - `bool`\n", - " - `enum`\n", - " - `int32`\n", - " - `uint32`\n", - " - `int64`\n", - " - `uint64`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_e3g9ExathXP" - }, - "source": [ - "Чтобы преобразовать стандартный тип TensorFlow в `tf.Example`-совместимый` tf.train.Feature`, вы можете использовать приведенные ниже функции. Обратите внимание, что каждая функция принимает на вход скалярное значение и возвращает `tf.train.Feature` содержащий один из трех вышеприведенных `list` типов:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mbsPOUpVtYxA" - }, - "outputs": [], - "source": [ - "# Следующая функция может быть использована чтобы преобразовать значение в тип совместимый с\n", - "# с tf.Example.\n", - "\n", - "def _bytes_feature(value):\n", - " \"\"\"Преобразует string / byte в bytes_list.\"\"\"\n", - " if isinstance(value, type(tf.constant(0))):\n", - " value = value.numpy() # BytesList не будет распаковывать строку из EagerTensor.\n", - " return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n", - "\n", - "def _float_feature(value):\n", - " \"\"\"Преобразует float / double в float_list.\"\"\"\n", - " return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n", - "\n", - "def _int64_feature(value):\n", - " \"\"\"Преобразует bool / enum / int / uint в int64_list.\"\"\"\n", - " return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wst0v9O8hgzy" - }, - "source": [ - "Замечание: Для простоты этот пример использует только скалярные входные данные. Простейший способ обработки нескалярных признаков - использование `tf.serialize_tensor` для конвертации тензоров в двоичнеые строки. Стоки являются скалярами в тензорфлоу. Используйте `tf.parse_tensor` для обратной конвертации двоичных сток в тензор." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vsMbkkC8xxtB" - }, - "source": [ - "Ниже приведены несколько примеров того как работают эти функции. Обратите внимание на различные типы ввода и стандартизированные типы вывода. Если входной тип функции не совпадает с одним из приводимых типов указанных выше, функция вызовет исключение (например `_int64_feature(1.0)` выдаст ошибку поскольку `1.0` это значение с плавающей точкой и должно быть использовано с функцией `_float_feature`):" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hZzyLGr0u73y" - }, - "outputs": [], - "source": [ - "print(_bytes_feature(b'test_string'))\n", - "print(_bytes_feature(u'test_bytes'.encode('utf-8')))\n", - "\n", - "print(_float_feature(np.exp(1)))\n", - "\n", - "print(_int64_feature(True))\n", - "print(_int64_feature(1))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nj1qpfQU5qmi" - }, - "source": [ - "Все proto сообщения могут быть сериализованы в двоичную строку с использованием метода `.SerializeToString`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5afZkORT5pjm" - }, - "outputs": [], - "source": [ - "feature = _float_feature(np.exp(1))\n", - "\n", - "feature.SerializeToString()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "laKnw9F3hL-W" - }, - "source": [ - "### Создание сообщения `tf.Example`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "b_MEnhxchQPC" - }, - "source": [ - "Допустим вы хотите создать сообщение `tf.Example` из существующих данных. На практике данные могут прийти откуда угодно, но процедура создания сообщения `tf.Example` из одного наблюдения будет той же:\n", - "\n", - "1. В рамках каждого наблюдения каждое значение должно быть преобразовано в `tf.train.Feature` содержащее одно из 3 совместимых типов, с использованием одной из вышеприведенных функций.\n", - "\n", - "2. Вы создаете отображение (словарь) из строки названий признаков в закодированное значение признака выполненное на шаге #1.\n", - "\n", - "3. Отображение (map) созданное на шаге 2 конвертируется в [`Features` message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto#L85)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4EgFQ2uHtchc" - }, - "source": [ - "В этом уроке вы создадите датасет с использованием NumPy.\n", - "\n", - "У этого датасета будет 4 признака:\n", - "* булев признак, `False` или `True` с равной вероятностью\n", - "* целочисленный признак - равномерно случайно выбранный из `[0, 5]`\n", - "* строковый признак сгенерированный из табицы строк с использованием целочисленного признака в качестве индекса\n", - "* признак с плавающей точкой из стандартного нормального распределения\n", - "\n", - "Рассмотрим выборку состающую из 10 000 независимых, одинаково распределенных наблюдений из каждого вышеприведенного распределения:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "CnrguFAy3YQv" - }, - "outputs": [], - "source": [ - "# Число наблюдений в датасете.\n", - "n_observations = int(1e4)\n", - "\n", - "# Булев признак, принимающий значения False или True.\n", - "feature0 = np.random.choice([False, True], n_observations)\n", - "\n", - "# Целочисленный признак, случайное число от 0 до 4.\n", - "feature1 = np.random.randint(0, 5, n_observations)\n", - "\n", - "# Строковый признак\n", - "strings = np.array([b'cat', b'dog', b'chicken', b'horse', b'goat'])\n", - "feature2 = strings[feature1]\n", - "\n", - "# Признак с плавающей точкой, из стандартного нормального распределения\n", - "feature3 = np.random.randn(n_observations)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aGrscehJr7Jd" - }, - "source": [ - "Каждый из этих признаков может быть приведен к `tf.Example`-совместимому типу с использованием одного из `_bytes_feature`, `_float_feature`, `_int64_feature`. Вы можете затем создать `tf.Example`-сообщение из этих закодированных признаков:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RTCS49Ij_kUw" - }, - "outputs": [], - "source": [ - "def serialize_example(feature0, feature1, feature2, feature3):\n", - " \"\"\"\n", - " Создает tf.Example-сообщение готовое к записи в файл.\n", - " \"\"\"\n", - " # Создает словарь отображение имен признаков в tf.Example-совместимые\n", - " # типы данных.\n", - " feature = {\n", - " 'feature0': _int64_feature(feature0),\n", - " 'feature1': _int64_feature(feature1),\n", - " 'feature2': _bytes_feature(feature2),\n", - " 'feature3': _float_feature(feature3),\n", - " }\n", - "\n", - " # Создает Features message с использованием tf.train.Example.\n", - "\n", - " example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n", - " return example_proto.SerializeToString()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XftzX9CN_uGT" - }, - "source": [ - "Возьмем, например, одно наблюдение из датасета, `[False, 4, bytes('goat'), 0.9876]`. Вы можете создать и распечатать `tf.Example`-сообщение для этого наблюдения с использованием `create_message()`. Каждое наблюдение может быть записано в виде `Features`-сообщения как указано выше. Note that the `tf.Example`-[сообщение](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88) это всего лишь обертка вокруг `Features`-сообщения:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "N8BtSx2RjYcb" - }, - "outputs": [], - "source": [ - "# Это пример наблюдения из набора данных.\n", - "\n", - "example_observation = []\n", - "\n", - "serialized_example = serialize_example(False, 4, b'goat', 0.9876)\n", - "serialized_example" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_pbGATlG6u-4" - }, - "source": [ - "Для декодирования сообщения используйте метод `tf.train.Example.FromString`." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dGim-mEm6vit" - }, - "outputs": [], - "source": [ - "example_proto = tf.train.Example.FromString(serialized_example)\n", - "example_proto" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o6qxofy89obI" - }, - "source": [ - "## Детали формата TFRecords\n", - "\n", - "Файл TFRecord содержит последовательность записей. Файл может быть прочитан только линейно.\n", - "\n", - "Каждая запись содержит строку байтов для данных плюс длину данных и CRC32C (32-bit CRC использующий полином Кастаньоли) хеши для проверки целостности.\n", - "\n", - "Каждая запись хранится в следующих форматах:\n", - "\n", - " uint64 length\n", - " uint32 masked_crc32_of_length\n", - " byte data[length]\n", - " uint32 masked_crc32_of_data\n", - "\n", - "Записи сцеплены друг с другом и организуют файл.. CRCs\n", - "[описаны тут](https://en.wikipedia.org/wiki/Cyclic_redundancy_check), и\n", - "маска CRC выглядит так:\n", - "\n", - " masked_crc = ((crc \u003e\u003e 15) | (crc \u003c\u003c 17)) + 0xa282ead8ul\n", - "\n", - "Замечание: Не обязательно использовать `tf.Example` в файлах TFRecord. `tf.Example` это всего лишь метод сериализации словарей в байтовые строки. Строки текста, закодированные данные изображений, или сериализованные тензоры (с использованием `tf.io.serialize_tensor`, и\n", - "`tf.io.parse_tensor` при загрузке). См. модуль `tf.io` для дополнительных возможностей." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y-Hjmee-fbLH" - }, - "source": [ - "## Файлы TFRecord с использованием `tf.data`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GmehkCCT81Ez" - }, - "source": [ - "Модуль `tf.data` также предоставляет инструменты для чтения и записи данных в TensorFlow." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1FISEuz8ubu3" - }, - "source": [ - "### Запись файла TFRecord\n", - "\n", - "Простейший способ помещения данных в датасет это использование метода `from_tensor_slices`.\n", - "\n", - "Примененный к массиву он возвращает датасет скаляров:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "mXeaukvwu5_-" - }, - "outputs": [], - "source": [ - "tf.data.Dataset.from_tensor_slices(feature1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f-q0VKyZvcad" - }, - "source": [ - "Примененный к кортежу массивов он возвращает датасет кортежей:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "H5sWyu1kxnvg" - }, - "outputs": [], - "source": [ - "features_dataset = tf.data.Dataset.from_tensor_slices((feature0, feature1, feature2, feature3))\n", - "features_dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "m1C-t71Nywze" - }, - "outputs": [], - "source": [ - "# Используйте `take(1)` чтобы взять только один пример из датасета.\n", - "for f0,f1,f2,f3 in features_dataset.take(1):\n", - " print(f0)\n", - " print(f1)\n", - " print(f2)\n", - " print(f3)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mhIe63awyZYd" - }, - "source": [ - "Используйте метод `tf.data.Dataset.map` чтобы применить функцию к каждому элементу `Dataset`.\n", - "\n", - "«Функция отображения должна работать в графовом режиме TensorFlow - она должна принимать и возвращать` tf.Tensors`. Не тензорная функция, такая как `create_example`, может быть заключена в` tf.py_function`, для совместимости.\n", - "\n", - "Использование `tf.py_function` требует указания размерности и информации о типе, которая в противном случае недоступна:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "apB5KYrJzjPI" - }, - "outputs": [], - "source": [ - "def tf_serialize_example(f0,f1,f2,f3):\n", - " tf_string = tf.py_function(\n", - " serialize_example,\n", - " (f0,f1,f2,f3), # передайте эти аргументы в верхнюю функцию.\n", - " tf.string) # возвращаемый тип `tf.string`.\n", - " return tf.reshape(tf_string, ()) # Результатом является скаляр" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "lHFjW4u4Npz9" - }, - "outputs": [], - "source": [ - "tf_serialize_example(f0,f1,f2,f3)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CrFZ9avE3HUF" - }, - "source": [ - "Примените эту функцию к каждому элементу датасета:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VDeqYVbW3ww9" - }, - "outputs": [], - "source": [ - "serialized_features_dataset = features_dataset.map(tf_serialize_example)\n", - "serialized_features_dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "DlDfuh46bRf6" - }, - "outputs": [], - "source": [ - "def generator():\n", - " for features in features_dataset:\n", - " yield serialize_example(*features)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iv9oXKrcbhvX" - }, - "outputs": [], - "source": [ - "serialized_features_dataset = tf.data.Dataset.from_generator(\n", - " generator, output_types=tf.string, output_shapes=())" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "Dqz8C4D5cIj9" - }, - "outputs": [], - "source": [ - "serialized_features_dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p6lw5VYpjZZC" - }, - "source": [ - "И запишите их в файл TFRecord:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vP1VgTO44UIE" - }, - "outputs": [], - "source": [ - "filename = 'test.tfrecord'\n", - "writer = tf.data.experimental.TFRecordWriter(filename)\n", - "writer.write(serialized_features_dataset)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6aV0GQhV8tmp" - }, - "source": [ - "### Чтение TFRecord файла" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o3J5D4gcSy8N" - }, - "source": [ - "Вы можете также прочитать TFRecord файл используя класс `tf.data.TFRecordDataset`.\n", - "\n", - "Больше информации об использовании TFRecord файлов с использованием `tf.data` может быть найдено [тут](https://www.tensorflow.org/guide/datasets#consuming_tfrecord_data)..\n", - "\n", - "Использование `TFRecordDataset`-ов может быть полезно для стандартизации входных данных и оптимизации производительности." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6OjX6UZl-bHC" - }, - "outputs": [], - "source": [ - "filenames = [filename]\n", - "raw_dataset = tf.data.TFRecordDataset(filenames)\n", - "raw_dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6_EQ9i2E_-Fz" - }, - "source": [ - "На этом этапе датасет содержит сериализованные сообщения `tf.train.Example`. При их итерации возвращаются скалярные строки тензоров.\n", - "\n", - "Используйте метод `.take` чтобы показать только первые 10 записей.\n", - "\n", - "Замечание: итерация по `tf.data.Dataset` работает только при включенном eager execution." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "hxVXpLz_AJlm" - }, - "outputs": [], - "source": [ - "for raw_record in raw_dataset.take(10):\n", - " print(repr(raw_record))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W-6oNzM4luFQ" - }, - "source": [ - "Эти тензоры может распарсить используя нижеприведенную функцию. Заметьте что `feature_description` обязателен тут поскольку датасеты используют графовое исполнение и нуждаются в этом описании для построения своей размерностной и типовой сигнатуры:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zQjbIR1nleiy" - }, - "outputs": [], - "source": [ - "# Создайте описание этих признаков\n", - "feature_description = {\n", - " 'feature0': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n", - " 'feature1': tf.io.FixedLenFeature([], tf.int64, default_value=0),\n", - " 'feature2': tf.io.FixedLenFeature([], tf.string, default_value=''),\n", - " 'feature3': tf.io.FixedLenFeature([], tf.float32, default_value=0.0),\n", - "}\n", - "\n", - "def _parse_function(example_proto):\n", - " # Разберите `tf.Example` proto используя вышеприведенный словарь.\n", - " return tf.io.parse_single_example(example_proto, feature_description)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gWETjUqhEQZf" - }, - "source": [ - "Альтернативно, используйте `tf.parse example` чтобы распарсить весь пакет за раз. Примените эту функцию к кажому элементу датасета используя метод `tf.data.Dataset.map`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6Ob7D-zmBm1w" - }, - "outputs": [], - "source": [ - "parsed_dataset = raw_dataset.map(_parse_function)\n", - "parsed_dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sNV-XclGnOvn" - }, - "source": [ - "Используйте eager execution чтобы показывать наблюдения в датасете. В этом наборе данных 10,000 наблюдений, но вы выведете только первые 10. Данные показываются как словарь признаков. Каждое наблюдение это `tf.Tensor`, и элемент `numpy`этого тензора показывает значение признака:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "x2LT2JCqhoD_" - }, - "outputs": [], - "source": [ - "for parsed_record in parsed_dataset.take(10):\n", - " print(repr(parsed_record))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cig9EodTlDmg" - }, - "source": [ - "Здесь функция `tf.parse_example` распаковывает поля `tf.Example` в стандартные тензоры." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jyg1g3gU7DNn" - }, - "source": [ - "## TFRecord файлы в Python" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3FXG3miA7Kf1" - }, - "source": [ - "Модуль `tf.io` также содержит чисто Python функции для чтения и записи файлов TFRecord." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CKn5uql2lAaN" - }, - "source": [ - "### Запись TFRecord файла" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LNW_FA-GQWXs" - }, - "source": [ - "Далее запишем эти 10 000 наблюдений в файл `test.tfrecord`. Каждое наблюдения конвертируется в `tf.Example`-сообщение и затем пишется в файл. Вы можете после проверить, что файл `test.tfrecord` был создан:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "MKPHzoGv7q44" - }, - "outputs": [], - "source": [ - "# Запишем наблюдения `tf.Example` в файл.\n", - "with tf.io.TFRecordWriter(filename) as writer:\n", - " for i in range(n_observations):\n", - " example = serialize_example(feature0[i], feature1[i], feature2[i], feature3[i])\n", - " writer.write(example)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "EjdFHHJMpUUo" - }, - "outputs": [], - "source": [ - "!du -sh {filename}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2osVRnYNni-E" - }, - "source": [ - "### Чтение TFRecord файла\n", - "\n", - "Эти сериализованные тензоры могут быть легко распарсены с использование `tf.train.Example.ParseFromString`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "U3tnd3LerOtV" - }, - "outputs": [], - "source": [ - "filenames = [filename]\n", - "raw_dataset = tf.data.TFRecordDataset(filenames)\n", - "raw_dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "nsEAACHcnm3f" - }, - "outputs": [], - "source": [ - "for raw_record in raw_dataset.take(1):\n", - " example = tf.train.Example()\n", - " example.ParseFromString(raw_record.numpy())\n", - " print(example)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S0tFDrwdoj3q" - }, - "source": [ - "## Упражнение: Чтение и запись данных изображений" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rjN2LFxFpcR9" - }, - "source": [ - "Это пример того как читать и писать данные изображений используя TFRecords. Цель этого показать как, от начала до конца, ввести данные (в этом случае изображение) и записать данные в TFRecord файл, затем прочитать файл и показать изображение.\n", - "\n", - "Это будет полезно если, например, вы хотите использовать несколько моделей на одних и тех же входных данных. Вместо хранения сырых данных изображений, они могут быть предобработанны в формат TFRecords, и затем могут быть использованы во всех дальнейших обработках и моделированиях.\n", - "\n", - "Сперва давайте скачаем [это изображение](https://commons.wikimedia.org/wiki/File:Felis_catus-cat_on_snow.jpg) кота и покажем [это фото](https://upload.wikimedia.org/wikipedia/commons/f/fe/New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg) строительства моста Williamsburg, NYC." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5Lk2qrKvN0yu" - }, - "source": [ - "### Получите изображения" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3a0fmwg8lHdF" - }, - "outputs": [], - "source": [ - "cat_in_snow = tf.keras.utils.get_file('320px-Felis_catus-cat_on_snow.jpg', 'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg')\n", - "williamsburg_bridge = tf.keras.utils.get_file('194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7aJJh7vENeE4" - }, - "outputs": [], - "source": [ - "display.display(display.Image(filename=cat_in_snow))\n", - "display.display(display.HTML('Image cc-by: \u003ca \"href=https://commons.wikimedia.org/wiki/File:Felis_catus-cat_on_snow.jpg\"\u003eVon.grzanka\u003c/a\u003e'))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "KkW0uuhcXZqA" - }, - "outputs": [], - "source": [ - "display.display(display.Image(filename=williamsburg_bridge))\n", - "display.display(display.HTML('\u003ca \"href=https://commons.wikimedia.org/wiki/File:New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg\"\u003eFrom Wikimedia\u003c/a\u003e'))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VSOgJSwoN5TQ" - }, - "source": [ - "### Write the TFRecord file" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Azx83ryQEU6T" - }, - "source": [ - "Как и ранее закодируйте признаки как типы совместимые с `tf.Example`. Здесь хранится необработанные данные изображения в формате string, так же как и высота, ширина, глубина и произвольный признак `label`. Последнее используется когда вы пишете файл чтобы различать изображение кота и моста. Используйте `0` изображения кота, и `1` для моста:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "kC4TS1ZEONHr" - }, - "outputs": [], - "source": [ - "image_labels = {\n", - " cat_in_snow : 0,\n", - " williamsburg_bridge : 1,\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "c5njMSYNEhNZ" - }, - "outputs": [], - "source": [ - "# Это пример использования только изображения кота.\n", - "image_string = open(cat_in_snow, 'rb').read()\n", - "\n", - "label = image_labels[cat_in_snow]\n", - "\n", - "# Создайте библиотеку с признаками которые могут быть релевантны.\n", - "def image_example(image_string, label):\n", - " image_shape = tf.image.decode_jpeg(image_string).shape\n", - "\n", - " feature = {\n", - " 'height': _int64_feature(image_shape[0]),\n", - " 'width': _int64_feature(image_shape[1]),\n", - " 'depth': _int64_feature(image_shape[2]),\n", - " 'label': _int64_feature(label),\n", - " 'image_raw': _bytes_feature(image_string),\n", - " }\n", - "\n", - " return tf.train.Example(features=tf.train.Features(feature=feature))\n", - "\n", - "for line in str(image_example(image_string, label)).split('\\n')[:15]:\n", - " print(line)\n", - "print('...')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2G_o3O9MN0Qx" - }, - "source": [ - "Заметьте что все признаки сейчас содержатся в `tf.Example`-сообщении. Далее функционализируйте вышеприведенный код и запишите пример сообщений в файл с именем `images.tfrecords`:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "qcw06lQCOCZU" - }, - "outputs": [], - "source": [ - "# Запишем файлы изображений в `images.tfrecords`.\n", - "# Сперва, преобразуем два изображения в `tf.Example`-сообщения.\n", - "# Затем запишем их в `.tfrecords` файл.\n", - "record_file = 'images.tfrecords'\n", - "with tf.io.TFRecordWriter(record_file) as writer:\n", - " for filename, label in image_labels.items():\n", - " image_string = open(filename, 'rb').read()\n", - " tf_example = image_example(image_string, label)\n", - " writer.write(tf_example.SerializeToString())" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yJrTe6tHPCfs" - }, - "outputs": [], - "source": [ - "!du -sh {record_file}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jJSsCkZLPH6K" - }, - "source": [ - "### Чтение TFRecord файла\n", - "\n", - "У вас сейчас есть файл `images.tfrecords` и вы можете проитерировать записи в нем чтобы прочитать то что вы в него записали. Поскольку этот пример содержит только изображение единственное свойство которое вам нужно это необработанная строка изображения. Извлеките ее используя геттеры описанные выше, а именно `example.features.feature['image_raw'].bytes_list.value[0]`. Вы можете также использовать метки чтобы определить, которая запись является котом, и которая мостом:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "M6Cnfd3cTKHN" - }, - "outputs": [], - "source": [ - "raw_image_dataset = tf.data.TFRecordDataset('images.tfrecords')\n", - "\n", - "# Создадим словарь описывающий свойства.\n", - "image_feature_description = {\n", - " 'height': tf.io.FixedLenFeature([], tf.int64),\n", - " 'width': tf.io.FixedLenFeature([], tf.int64),\n", - " 'depth': tf.io.FixedLenFeature([], tf.int64),\n", - " 'label': tf.io.FixedLenFeature([], tf.int64),\n", - " 'image_raw': tf.io.FixedLenFeature([], tf.string),\n", - "}\n", - "\n", - "def _parse_image_function(example_proto):\n", - " # Распарсим входной tf.Example proto используя вышесозданный словарь.\n", - " return tf.io.parse_single_example(example_proto, image_feature_description)\n", - "\n", - "parsed_image_dataset = raw_image_dataset.map(_parse_image_function)\n", - "parsed_image_dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0PEEFPk4NEg1" - }, - "source": [ - "Восстановим изображение из TFRecord файла:" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "yZf8jOyEIjSF" - }, - "outputs": [], - "source": [ - "for image_features in parsed_image_dataset:\n", - " image_raw = image_features['image_raw'].numpy()\n", - " display.display(display.Image(data=image_raw))" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [ - "pL--_KGdYoBz" - ], - "name": "tfrecord.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/ru/tutorials/load_data/unicode.ipynb b/site/ru/tutorials/load_data/unicode.ipynb deleted file mode 100644 index d91b2c8536c..00000000000 --- a/site/ru/tutorials/load_data/unicode.ipynb +++ /dev/null @@ -1,846 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "unicode.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "oL9KopJirB2g" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oL9KopJirB2g" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "SKaX3Hd3ra6C", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AXH1bmUctMld" - }, - "source": [ - "# Юникод-строки\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fj66ZXAzrJC2", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LrHJrKYis06U" - }, - "source": [ - "## Введение\n", - "\n", - "Модели обрабатывающие естественные языки, часто имеют дело с разными языками и разными наборами символов. * Unicode * - это стандартная система кодирования, которая используется для представления символов практически всех языков. Каждый символ кодируется с использованием уникального целого числа [кодовой точки](https://en.wikipedia.org/wiki/Code_point) между `0` и` 0x10FFFF`. *Юникод-строка* - это последовательность из нуля или более таких кодовых точек.\n", - "\n", - "Это руководство показывает как представлять юникод-строки в Tensorflow и манипулировать ими используя юникодовские эквиваленты стандартной строковой. Она выделяет юникод-строки в токены на основе обнаружения скрипта." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OIKHl5Lvn4gh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version существует только в Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n-LkcI-vtWNj" - }, - "source": [ - "## Тип данных `tf.string`\n", - "\n", - "Базовый TensorFlow `tf.string` `dtype` позволяет вам строить тензоры байт-строк.\n", - "Юникод-строки по умолчанию в кодировке utf-8." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3yo-Qv6ntaFr", - "colab": {} - }, - "source": [ - "tf.constant(u\"Спасибо 😊\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2kA1ziG2tyCT" - }, - "source": [ - "Тензор `tf.string` может содержать байт-строки различной длины поскольку байт-строки обрабатываются как отдельные единицы. Длина строки не включена в размерность тензора.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eyINCmTztyyS", - "colab": {} - }, - "source": [ - "tf.constant([u\"Добро\", u\"пожаловать!\"]).shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jsMPnjb6UDJ1" - }, - "source": [ - "Замечание: При использовании python при конструировании строк обработка юникода отличается между v2 и v3. В v2, юникод-строки отмечены префиксом \"u\", как и выше. В v3, строки закодированы в юникоде по умолчанию." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hUFZ7B1Lk-uj" - }, - "source": [ - "## Представление Юникода\n", - "\n", - "Есть два стандартных способа представления юникод-строк в TensorFlow:\n", - "\n", - "* `string` скаляр — где последовательность кодовых точек закодирована с использованием [набора символов](https://ru.wikipedia.org/wiki/%D0%9D%D0%B0%D0%B1%D0%BE%D1%80_%D1%81%D0%B8%D0%BC%D0%B2%D0%BE%D0%BB%D0%BE%D0%B2).\n", - "* `int32` вектор — где каждая позиция содержит единственную кодовую точку.\n", - "\n", - "Например, следующие три значения все представляют юникод-строку `\"语言处理\"` (что значит \"обработка языка\" на китайском):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cjQIkfJWvC_u", - "colab": {} - }, - "source": [ - "# Юникод-строки, представленные как UTF-8 закодированные строки скаляры.\n", - "text_utf8 = tf.constant(u\"语言处理\")\n", - "text_utf8" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yQqcUECcvF2r", - "colab": {} - }, - "source": [ - "# Юникод-строки представленные как UTF-16-BE закодированные строки скаляры.\n", - "text_utf16be = tf.constant(u\"语言处理\".encode(\"UTF-16-BE\"))\n", - "text_utf16be" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ExdBr1t7vMuS", - "colab": {} - }, - "source": [ - "# Юникод строки представленные как векторы юникодовских кодовых точек.\n", - "text_chars = tf.constant([ord(char) for char in u\"语言处理\"])\n", - "text_chars" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B8czv4JNpBnZ" - }, - "source": [ - "### Конвертация между представлениями\n", - "\n", - "TensorFlow предоставляет операции для конвертации между этими различными представлениями:\n", - "\n", - "* `tf.strings.unicode_decode`: Конвертирует закодированную строку скаляр в вектор кодовых точек.\n", - "* `tf.strings.unicode_encode`: Конвертирует вектор кодовых точек в закодированную строку скаляр.\n", - "* `tf.strings.unicode_transcode`: Конвертирует строку скаляр в другую кодировку." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qb-UQ_oLpAJg", - "colab": {} - }, - "source": [ - "tf.strings.unicode_decode(text_utf8,\n", - " input_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kEBUcunnp-9n", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(text_chars,\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0MLhWcLZrph-", - "colab": {} - }, - "source": [ - "tf.strings.unicode_transcode(text_utf8,\n", - " input_encoding='UTF8',\n", - " output_encoding='UTF-16-BE')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QVeLeVohqN7I" - }, - "source": [ - "### Размерности пакета\n", - "\n", - "При декодировании нескольких строк размер символов в каждой строке может не совпадать, Возвращаемый результат это [`tf.RaggedTensor`](../../guide/ragged_tensor.ipynb), где длина самого внутреннего измерения меняется в зависимости от количества символов в каждой строке:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N2jVzPymr_Mm", - "colab": {} - }, - "source": [ - "# Пакет юнокод-строк каждая из которых представлена в виде строки в юникод-кодировке.\n", - "batch_utf8 = [s.encode('UTF-8') for s in\n", - " [u'hÃllo', u'What is the weather tomorrow', u'Göödnight', u'😊']]\n", - "batch_chars_ragged = tf.strings.unicode_decode(batch_utf8,\n", - " input_encoding='UTF-8')\n", - "for sentence_chars in batch_chars_ragged.to_list():\n", - " print(sentence_chars)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iRh3n1hPsJ9v" - }, - "source": [ - "Вы можете использовать `tf.RaggedTensor` напрямую, или конвертировать его в плотный `tf.Tensor` с паддингом или в `tf.SparseTensor` используя методы `tf.RaggedTensor.to_tensor` и `tf.RaggedTensor.to_sparse`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yz17yeSMsUid", - "colab": {} - }, - "source": [ - "batch_chars_padded = batch_chars_ragged.to_tensor(default_value=-1)\n", - "print(batch_chars_padded.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kBjsPQp3rhfm", - "colab": {} - }, - "source": [ - "batch_chars_sparse = batch_chars_ragged.to_sparse()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GCCkZh-nwlbL" - }, - "source": [ - "При кодировании нескольких строк одинаковой длины `tf.Tensor` может быть использован в качестве входных данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_lP62YUAwjK9", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode([[99, 97, 116], [100, 111, 103], [ 99, 111, 119]],\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w58CMRg9tamW" - }, - "source": [ - "При кодировании нескольких строк различной длины нужно использовать `tf.RaggedTensor` в качестве входных данных:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "d7GtOtrltaMl", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(batch_chars_ragged, output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T2Nh5Aj9xob3" - }, - "source": [ - "Если у вас тензор с несколькими строками с паддингом или в разреженном формате, то ковертируйте его в `tf.RaggedTensor` перед вызовом `unicode_encode`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R2bYCYl0u-Ue", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(\n", - " tf.RaggedTensor.from_sparse(batch_chars_sparse),\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UlV2znh_u_zm", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(\n", - " tf.RaggedTensor.from_tensor(batch_chars_padded, padding=-1),\n", - " output_encoding='UTF-8')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hQOOGkscvDpc" - }, - "source": [ - "## Операции юникода" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NkmtsA_yvMB0" - }, - "source": [ - "### Длина символа\n", - "\n", - "Операция `tf.strings.length` имеет параметр `unit`, который показывает как должна быть посчитана длина. По умолчанию размер `unit` равен `\"BYTE\"`, но он может быть установлен с другим значением, таким как `\"UTF8_CHAR\"` или `\"UTF16_CHAR\"`, чтобы определить число кодовых точек в каждой закодированой `string`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1ZzMe59mvLHr", - "colab": {} - }, - "source": [ - "# Заметьте что последний символ занимает до 4 байтов в UTF8.\n", - "thanks = u'Thanks 😊'.encode('UTF-8')\n", - "num_bytes = tf.strings.length(thanks).numpy()\n", - "num_chars = tf.strings.length(thanks, unit='UTF8_CHAR').numpy()\n", - "print('{} bytes; {} UTF-8 characters'.format(num_bytes, num_chars))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fHG85gxlvVU0" - }, - "source": [ - "### Подстроки символов\n", - "\n", - "Аналогично у операции `tf.strings.substr` есть параметр \"`unit`\", который используется для определения смещений параметров \"`pos`\" и \"`len`\"." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WlWRLV-4xWYq", - "colab": {} - }, - "source": [ - "# по умолчанию: unit='BYTE'. С len=1, мы возвращаем один байт.\n", - "tf.strings.substr(thanks, pos=7, len=1).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JfNUVDPwxkCS", - "colab": {} - }, - "source": [ - "# Установив unit='UTF8_CHAR', мы возвратим один символ, размер которого в этом случае\n", - "# 4 байта.\n", - "print(tf.strings.substr(thanks, pos=7, len=1, unit='UTF8_CHAR').numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zJUEsVSyeIa3" - }, - "source": [ - "### Разбиение юникод-строки\n", - "\n", - "Операция `tf.strings.unicode_split` разбивает юникод-строки в подстроки отдельных символов:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dDjkh5G1ejMt", - "colab": {} - }, - "source": [ - "tf.strings.unicode_split(thanks, 'UTF-8').numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HQqEEZEbdG9O" - }, - "source": [ - "### Смещения байтов для символов\n", - "\n", - "Чтобы выровнять тензор символа порожденный `tf.strings.unicode_decode` с оригинальной строкой, полезно знать смещение начала каждого символа. Метод `tf.strings.unicode_decode_with_offsets` аналогичен `unicode_decode`, за исключением того, что он возвращает второй тензор содержащий размер отступа от начала для каждого символа." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Cug7cmwYdowd", - "colab": {} - }, - "source": [ - "codepoints, offsets = tf.strings.unicode_decode_with_offsets(u\"🎈🎉🎊\", 'UTF-8')\n", - "\n", - "for (codepoint, offset) in zip(codepoints.numpy(), offsets.numpy()):\n", - " print(\"At byte offset {}: codepoint {}\".format(offset, codepoint))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2ZnCNxOvx66T" - }, - "source": [ - "## Юникод скрипты" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRRHqkqNyGZ6" - }, - "source": [ - "Каждая кодовая точка принадлежит коллекции символов известной как [система письма](https://en.wikipedia.org/wiki/Script_%28Unicode%29) . Система письма полезна для определения того, какому языку может принадлежать символ. Например, зная что 'Б' из кириллицы указывает на то, что современный текст содержащий этот символ скорее всего из славянского языка, такого как русский или украинский.\n", - "\n", - "В TensorFlow есть операция `tf.strings.unicode_script` для определения какой системе письма принадлежит данная кодовая точка. Коды систем письма это `int32` числа соответствующие [Международным компонентам для\n", - "юникода](http://site.icu-project.org/home) (ICU) [`UScriptCode`](http://icu-project.org/apiref/icu4c/uscript_8h.html) значения.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K7DeYHrRyFPy", - "colab": {} - }, - "source": [ - "uscript = tf.strings.unicode_script([33464, 1041]) # ['芸', 'Б']\n", - "\n", - "print(uscript.numpy()) # [17, 8] == [USCRIPT_HAN, USCRIPT_CYRILLIC]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2fW992a1lIY6" - }, - "source": [ - "Операция `tf.strings.unicode_script` может быть также применена к многомерному `tf.Tensor`s или кодовым точкам `tf.RaggedTensor`:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uR7b8meLlFnp", - "colab": {} - }, - "source": [ - "print(tf.strings.unicode_script(batch_chars_ragged))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mx7HEFpBzEsB" - }, - "source": [ - "## Пример: Простая сегментация\n", - "\n", - "Сегментация это задача разбиения текста на словоподобные юниты. Часто это легко когда испольуются символы пробела для отделения слов, но некоторые языки (например китайский и японский) не используют пробелы, а некоторые языки (например немецкий) содержат длинные соединения, которые должны быть разделены для анализа их значений. В веб текстах, различные языки и скрипты часто перемешаны между собой , как например в \"NY株価\" (New York Stock Exchange).\n", - "\n", - "Мы можем выполнить грубую сегментацию (без реализации каких-либо моделей ML), используя изменения систем письма для приблизительного определения границ слов. Это будет работать для строк наподобие вышеприведенного примера \"NY株価\". Это также будет работать для всех языков, которые используют пробелы, так как символы пробела в различных системах письма все классифицируются как USCRIPT_COMMON, специальный код, который отличается от кода любого актуального текста." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "grsvFiC4BoPb", - "colab": {} - }, - "source": [ - "# dtype: string; shape: [num_sentences]\n", - "#\n", - "# Предложения для обработки. Поменяйте эту строку чтобы попробовать разные входные данные!\n", - "sentence_texts = [u'Hello, world.', u'世界こんにちは']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CapnbShuGU8i" - }, - "source": [ - "Сперва мы декодируем предложения в кодовые точки, и определим идентификатор системы письма для каждого символа." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReQVcDQh1MB8", - "colab": {} - }, - "source": [ - "# dtype: int32; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_codepoint[i, j] кусок кода для j-го символа\n", - "# в i-м предложении.\n", - "sentence_char_codepoint = tf.strings.unicode_decode(sentence_texts, 'UTF-8')\n", - "print(sentence_char_codepoint)\n", - "\n", - "# dtype: int32; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_scripts[i, j] код системы письма для j-го символа в\n", - "# i-м предложении.\n", - "sentence_char_script = tf.strings.unicode_script(sentence_char_codepoint)\n", - "print(sentence_char_script)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O2fapF5UGcUc" - }, - "source": [ - "Далее мы используем эти идентификаторы систем письма чтобы определить куда должны быть добавлены границы слов. Мы добавим границу слова в начало каждого предложения и для каждого символа чья система письма отличается от предыдущего символа:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7v5W6MOr1Rlc", - "colab": {} - }, - "source": [ - "# dtype: bool; shape: [num_sentences, (num_chars_per_sentence)]\n", - "#\n", - "# sentence_char_starts_word[i, j] является True если j'th символ в i'th\n", - "# предложении является началом слова.\n", - "sentence_char_starts_word = tf.concat(\n", - " [tf.fill([sentence_char_script.nrows(), 1], True),\n", - " tf.not_equal(sentence_char_script[:, 1:], sentence_char_script[:, :-1])],\n", - " axis=1)\n", - "\n", - "# dtype: int64; shape: [num_words]\n", - "#\n", - "# word_starts[i] это индекс символа начинающего i-е слово (в\n", - "# выпрямленном списке символов всех предложений).\n", - "word_starts = tf.squeeze(tf.where(sentence_char_starts_word.values), axis=1)\n", - "print(word_starts)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LAwh-1QkGuC9" - }, - "source": [ - "Затем мы можем использовать эти сдвиги от начала для построения `RaggedTensor` содержащего список слов из всех пакетов:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bNiA1O_eBBCL", - "colab": {} - }, - "source": [ - "# dtype: int32; shape: [num_words, (num_chars_per_word)]\n", - "#\n", - "# word_char_codepoint[i, j] is the кодовая точка для j-го символа в\n", - "# i-м слове.\n", - "word_char_codepoint = tf.RaggedTensor.from_row_starts(\n", - " values=sentence_char_codepoint.values,\n", - " row_starts=word_starts)\n", - "print(word_char_codepoint)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "66a2ZnYmG2ao" - }, - "source": [ - "И наконец, мы можем сегментировать кодовые точки слов `RaggedTensor` обратно в предложения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NCfwcqLSEjZb", - "colab": {} - }, - "source": [ - "# dtype: int64; shape: [num_sentences]\n", - "#\n", - "# sentence_num_words[i] число слов в i'th предложении.\n", - "sentence_num_words = tf.reduce_sum(\n", - " tf.cast(sentence_char_starts_word, tf.int64),\n", - " axis=1)\n", - "\n", - "# dtype: int32; shape: [num_sentences, (num_words_per_sentence), (num_chars_per_word)]\n", - "#\n", - "# sentence_word_char_codepoint[i, j, k] это кодовая точка для k-го символа\n", - "# в j-м слове i-го предложения.\n", - "sentence_word_char_codepoint = tf.RaggedTensor.from_row_lengths(\n", - " values=word_char_codepoint,\n", - " row_lengths=sentence_num_words)\n", - "print(sentence_word_char_codepoint)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xWaX8WcbHyqY" - }, - "source": [ - "Чтобы сделать итоговый результат проще для чтения, мы можем закодировать его обратно в UTF-8 строки:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HSivquOgFr3C", - "colab": {} - }, - "source": [ - "tf.strings.unicode_encode(sentence_word_char_codepoint, 'UTF-8').to_list()" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/ru/tutorials/quickstart/advanced.ipynb b/site/ru/tutorials/quickstart/advanced.ipynb deleted file mode 100644 index 078da242b07..00000000000 --- a/site/ru/tutorials/quickstart/advanced.ipynb +++ /dev/null @@ -1,407 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "advanced.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# Начните с TensorFlow 2.0 для экспертов" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gVN4BFQ5Dg4v", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Это файл [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) notebook. Программы Python programs запускающиеся прямо в браузере - хороший способ учить и использовать TensorFlow. Чтобы следовать этому руководству, запустите notebook в Google Colab нажав на соответствующую кнопку вверху этой страницы.\n", - "\n", - "1. В Colab подключитесь к среде исполнения Python: в правом верхнем углу строки меню выберите *CONNECT*.\n", - "2. Запустите код во всех ячейках notebook: Выберите *Runtime* и *Run all*." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "eOsVdx6GGHmU", - "colab_type": "text" - }, - "source": [ - "Скачайте и установите пакет TensorFlow 2.0:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "ioLbtB3uGKPX", - "colab_type": "code", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "QS7DDTiZGRTo", - "colab_type": "text" - }, - "source": [ - "Импортируйте TensorFlow в свою программу:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Загрузите и приготовьте [набор данных MNIST](http://yann.lecun.com/exdb/mnist/)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JqFRS6K07jJs", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# Add a channels dimension\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "Используйте `tf.data` чтобы разбить на пакеты и перемешать данные:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Iu_quO024c2", - "colab": {} - }, - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Постройте модель `tf.keras` используя Keras [model subclassing API](https://www.tensorflow.org/guide/keras#model_subclassing):" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "model = MyModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "Выберите оптимизатор и функцию потерь для обучения: " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u48C9WQ774n4", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "Выберите метрики для измерения потерь и доли правильных ответов (accuracy) модели. Эти метрики аккумулируют значения за эпохи и потом выводят общий результат." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N0MqHFb4F_qn", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Используйте `tf.GradientTape` для обучения модели:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OZACiVqA8KQV", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z8YT7UmFgpjV" - }, - "source": [ - "Протестируйте модель:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIKdEzHAJGt7", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "i-2pkctU_Ci7", - "colab": {} - }, - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n", - " print (template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "Классификатор изображений теперь обучен на этих данных с точностью ~ 98%. Чтобы узнать больше, прочитайте [учебники по TensorFlow](https://www.tensorflow.org/tutorials/)." - ] - } - ] -} diff --git a/site/ru/tutorials/quickstart/beginner.ipynb b/site/ru/tutorials/quickstart/beginner.ipynb deleted file mode 100644 index 5cf024f75d0..00000000000 --- a/site/ru/tutorials/quickstart/beginner.ipynb +++ /dev/null @@ -1,235 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "beginner.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# Начните с TensorFlow 2.0 для новичков" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Смотрите на TensorFlow.org\n", - " \n", - " Запустите в Google Colab\n", - " \n", - " Изучайте код на GitHub\n", - " \n", - " Скачайте ноутбук\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "10jPgwiVOBbc", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Это файл [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) notebook. Программы Python programs запускающиеся прямо в браузере - хороший способ учить и использовать TensorFlow. Чтобы следовать этому руководству, запустите notebook в Google Colab нажав на соответствующую кнопку вверху этой страницы.\n", - "\n", - "1. В Colab подключитесь к среде исполнения Python: в правом верхнем углу строки меню выберите *CONNECT*.\n", - "2. Запустите код во всех ячейках notebook: Выберите *Runtime* и *Run all*." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nnrWf3PCEzXL" - }, - "source": [ - "Скачайте и установите пакет TensorFlow 2.0 Beta. Импортируйте в вашу программу TensorFlow:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# Install TensorFlow\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Загрузите и установите [набор данных MNIST](http://yann.lecun.com/exdb/mnist/). Преобразуйте примеры из целых чисел в числа с плавающей запятой:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FP5258xjs-v", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Создайте модель `tf.keras.Sequential`, сложив слои. Выберите оптимизатор и функцию потерь для обучения:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "model = tf.keras.models.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Обучите и оцените модель:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F7dTAzgHDUh7", - "colab": {} - }, - "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", - "model.evaluate(x_test, y_test, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "Классификатор изображений теперь обучен на этих данных с точностью ~ 98%. Чтобы узнать больше, прочитайте [учебники по TensorFlow](https://www.tensorflow.org/tutorials/)." - ] - } - ] -} diff --git a/site/ru/tutorials/structured_data/feature_columns.ipynb b/site/ru/tutorials/structured_data/feature_columns.ipynb deleted file mode 100644 index 56ea647e7ca..00000000000 --- a/site/ru/tutorials/structured_data/feature_columns.ipynb +++ /dev/null @@ -1,728 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "feature_columns.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rNdWfPXCjTjY" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "I1dUQ0GejU8N", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c05P9g5WjizZ" - }, - "source": [ - "# Классифицируй структурированные данные" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zofH_gCzgplN" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " View on TensorFlow.org\n", - " \n", - " \n", - " \n", - " Run in Google Colab\n", - " \n", - " \n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9YD0qEZdzDeN", - "colab_type": "text" - }, - "source": [ - "Note: Вся информация в этом разделе переведена с помощью русскоговорящего Tensorflow сообщества на общественных началах. Поскольку этот перевод не является официальным, мы не гарантируем что он на 100% аккуратен и соответствует [официальной документации на английском языке](https://www.tensorflow.org/?hl=en). Если у вас есть предложение как исправить этот перевод, мы будем очень рады увидеть pull request в [tensorflow/docs](https://github.com/tensorflow/docs) репозиторий GitHub. Если вы хотите помочь сделать документацию по Tensorflow лучше (сделать сам перевод или проверить перевод подготовленный кем-то другим), напишите нам на [docs-ru@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K1y4OHpGgss7" - }, - "source": [ - "Этот учебник показывает, как классифицировать структурированные данные (например, табличные данные в CSV). Мы будем использовать [Keras](https://www.tensorflow.org/guide/keras) чтобы определить модель и [feature columns](https://www.tensorflow.org/guide/feature_columns) в для отображения столбцов в CSV в признаки, используемыми для обучения модели. Этот учебник содержит полный код с помощью которого Вы сможете:\n", - "\n", - "* Загрузить CSV файл с использованием [Pandas](https://pandas.pydata.org/).\n", - "* Создать входной пайплайн для пакетной обработки и перемешивания строк, используя [tf.data](https://www.tensorflow.org/guide/datasets).\n", - "* Отобразить колонки CSV в признаки используемые для обучения модели используя feature columns.\n", - "* Построить, обучить и оценить модель используя Keras.\n", - "\n", - "## Набор данных\n", - "\n", - "Мы будем использовать небольшой [датасет](https://archive.ics.uci.edu/ml/datasets/heart+Disease) предоставленный Кливлендской клиникой (Cleveland Clinic Foundation for Heart Disease). Датасет содержит несколько сотен строк в формате CSV. Каждая строка описывает пациента, а каждая колонка характеризует свойство. Мы будем использовать эту информацию чтобы предсказать, есть ли у пациента сердечное заболевание, что в этом наборе данных является задачей бинарной классификации.\n", - "\n", - "По ссылке [описание](https://archive.ics.uci.edu/ml/machine-learning-databases/heart-disease/heart-disease.names) этого датасета. Обратите внимание что в нем есть и числовые и категорийные столбцы.\n", - "\n", - ">Column| Description| Feature Type | Data Type\n", - ">------------|--------------------|----------------------|-----------------\n", - ">Age | Age in years | Numerical | integer\n", - ">Sex | (1 = male; 0 = female) | Categorical | integer\n", - ">CP | Chest pain type (0, 1, 2, 3, 4) | Categorical | integer\n", - ">Trestbpd | Resting blood pressure (in mm Hg on admission to the hospital) | Numerical | integer\n", - ">Chol | Serum cholestoral in mg/dl | Numerical | integer\n", - ">FBS | (fasting blood sugar > 120 mg/dl) (1 = true; 0 = false) | Categorical | integer\n", - ">RestECG | Resting electrocardiographic results (0, 1, 2) | Categorical | integer\n", - ">Thalach | Maximum heart rate achieved | Numerical | integer\n", - ">Exang | Exercise induced angina (1 = yes; 0 = no) | Categorical | integer\n", - ">Oldpeak | ST depression induced by exercise relative to rest | Numerical | integer\n", - ">Slope | The slope of the peak exercise ST segment | Numerical | float\n", - ">CA | Number of major vessels (0-3) colored by flourosopy | Numerical | integer\n", - ">Thal | 3 = normal; 6 = fixed defect; 7 = reversable defect | Categorical | string\n", - ">Target | Diagnosis of heart disease (1 = true; 0 = false) | Classification | integer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VxyBFc_kKazA" - }, - "source": [ - "## Импортируйте TensorFlow и прочие библиотеки" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LuOWVJBz8a6G", - "colab": {} - }, - "source": [ - "!pip install sklearn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9dEreb4QKizj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import feature_column\n", - "from tensorflow.keras import layers\n", - "from sklearn.model_selection import train_test_split" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KCEhSZcULZ9n" - }, - "source": [ - "## Используйте Pandas чтобы создать датафрейм\n", - "\n", - "[Pandas](https://pandas.pydata.org/) это библиотека Python множеством полезных утилит для загрузки и работы со структурированными данными. Мы будем использовать Pandas для скачивания данных по ссылке и выгрузки их в датафрейм." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "REZ57BXCLdfG", - "colab": {} - }, - "source": [ - "URL = 'https://storage.googleapis.com/applied-dl/heart.csv'\n", - "dataframe = pd.read_csv(URL)\n", - "dataframe.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u0zhLtQqMPem" - }, - "source": [ - "## Разбейте датафрейм на обучающую, проверочную и тестовую выборки\n", - "\n", - "Датасет который мы скачали был одним CSV файлом. Мы разделим его на тренировочную, проверочную и тестовую выборки." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YEOpw7LhMYsI", - "colab": {} - }, - "source": [ - "train, test = train_test_split(dataframe, test_size=0.2)\n", - "train, val = train_test_split(train, test_size=0.2)\n", - "print(len(train), 'train examples')\n", - "print(len(val), 'validation examples')\n", - "print(len(test), 'test examples')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "84ef46LXMfvu" - }, - "source": [ - "## Создайте входной пайплайн с помощью tf.data\n", - "\n", - "Далее мы обернем датафреймы в [tf.data](https://www.tensorflow.org/guide/datasets). Это позволит нам использовать feature columns в качестве моста для отображения столбцов датафрейма Pandas в признаки используемые для обучения модели. Если бы мы работали с очень большим CSV файлом (таким большим, что он не помещается в память), нам нужно было использовать tf.data чтобы прочитать его напрямую с диска. Подобный случай не рассматривается в этом уроке." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NkcaMYP-MsRe", - "colab": {} - }, - "source": [ - "# Вспомогательный метод для создания tf.data dataset из датафрейма Pandas\n", - "def df_to_dataset(dataframe, shuffle=True, batch_size=32):\n", - " dataframe = dataframe.copy()\n", - " labels = dataframe.pop('target')\n", - " ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n", - " if shuffle:\n", - " ds = ds.shuffle(buffer_size=len(dataframe))\n", - " ds = ds.batch(batch_size)\n", - " return ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CXbbXkJvMy34", - "colab": {} - }, - "source": [ - "batch_size = 5 # Небольшой размер пакета используется для демонстрационных целей\n", - "train_ds = df_to_dataset(train, batch_size=batch_size)\n", - "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", - "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qRLGSMDzM-dl" - }, - "source": [ - "## Поймите входной пайплайн\n", - "\n", - "Сейчас когда мы создали входной пайплайн, давайте вызовем его чтобы увидеть формат данных который он возвращает. Мы использовали небольшой размер пакета чтобы результат был читабельный." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CSBo3dUVNFc9", - "colab": {} - }, - "source": [ - "for feature_batch, label_batch in train_ds.take(1):\n", - " print('Every feature:', list(feature_batch.keys()))\n", - " print('A batch of ages:', feature_batch['age'])\n", - " print('A batch of targets:', label_batch )" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OT5N6Se-NQsC" - }, - "source": [ - "Мы видим что датасет возвращает словарь имен столбцов (из датафрейма) который сопоставляет их значениям столбцов взятых из строк датафрейма." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ttIvgLRaNoOQ" - }, - "source": [ - "## Продемонстрируем несколько видов столбцов признаков (feature columns)\n", - "TensorFlow предоставляет множество типов столбцов признаков. В этом разделе мы создадим несколько видов столбцов признаков и покажем как они преобразуют столбцы из датафрейма." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mxwiHFHuNhmf", - "colab": {} - }, - "source": [ - "# Мы используем этот пакте для демонстрации нескольких видов столбцов признаков\n", - "example_batch = next(iter(train_ds))[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0wfLB8Q3N3UH", - "colab": {} - }, - "source": [ - "# Служебный метод для создания столбца признаков\n", - "# и преобразования пакета данных\n", - "def demo(feature_column):\n", - " feature_layer = layers.DenseFeatures(feature_column)\n", - " print(feature_layer(example_batch).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q7OEKe82N-Qb" - }, - "source": [ - "### Численные столбцы (numeric columns)\n", - "Выходные данные столбцов признаков становятся входными данными модели (используя демо функцию определенную выше мы сможем посмотреть как конкретно преобразуется каждый столбец датафрейма). Числовой столбец [(numeric column)](https://www.tensorflow.org/api_docs/python/tf/feature_column/numeric_column) простейший вид столбца. Он используется для представления числовых признаков. При использовании этого столбца модель получает столбец значений из датафрейма без изменений." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QZTZ0HnHOCxC", - "colab": {} - }, - "source": [ - "age = feature_column.numeric_column(\"age\")\n", - "demo(age)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7a6ddSyzOKpq" - }, - "source": [ - "В наборе данных о сердечных заболеваниях большинство столбцов из датафрейма - числовые." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IcSxUoYgOlA1" - }, - "source": [ - "### Сгруппированные столбцы (bucketized columns)\n", - "Часто вы не хотите передавать числа непосредственно в модель, а вместо этого делите их на несколько категорий на основе числовых диапазонов. Рассмотрим данные представляющие возраст человека. Вместо представления возраста как числового столбца мы можем разбить возраст на несколько категорий использовав [сгруппированный столбец](https://www.tensorflow.org/api_docs/python/tf/feature_column/bucketized_column). Обратите внимание, что one-hot значения приведенные ниже описывают к которому возрастному диапазону относится каждая из строк." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wJ4Wt3SAOpTQ", - "colab": {} - }, - "source": [ - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "demo(age_buckets)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r1tArzewPb-b" - }, - "source": [ - "### Категориальные столбцы (categorical columns)\n", - "В этом датасете thal представлен в виде строк (например 'fixed', 'normal', или 'reversible'). Мы не можем передать строки напрямую в модель. Вместо этого мы должны сперва поставить им в соответствие численные значения. Словарь категориальных столбцов (categorical vocabulary columns) обеспечивает способ представления строк в виде one-hot векторов (как вы видели выше для возраста разбитого на категории). Справочник может быть передан как список с использованием [categorical_column_with_vocabulary_list](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_list), или загружен из файла с использованием [categorical_column_with_vocabulary_file](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_file)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DJ6QnSHkPtOC", - "colab": {} - }, - "source": [ - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "demo(thal_one_hot)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dxQloQ9jOoXL" - }, - "source": [ - "В более сложных датасетах многие столбцы бывают категориальными (т.е. строками). Столбцы признаков наиболее полезны при работе с категориальными данными. Хотя в этом наборе данных есть только один категориальный столбец, мы будем использовать его для демонстрации нескольких важных видов столбцов признаков, которые вы можете использовать при работе с другими наборами данных." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LEFPjUr6QmwS" - }, - "source": [ - "### Столбцы векторных представлений (embedding column)\n", - "Предположим, что вместо нескольких возможных строковых значений мы имеем тысячи и более значений для категорий. По ряду причин когда число категорий сильно вырастает, становится невозможным обучение нейронной сети с использованием one-hot кодирования. Мы можем использовать столбец векторных представлений для преодоления этого ограничения. Вместо представления данных в виде многомерных one-hot векторов [столбец векторных представлений](https://www.tensorflow.org/api_docs/python/tf/feature_column/embedding_column) представляет эти данные в виде плотных векторов меньшей размерности, в которых в каждой ячейке может содержаться любое число, а не только О или 1. Размерность векторного представления (8, в нижеприведенном при мере) это параметр который необходимо настраивать.\n", - "\n", - "Ключевой момент: использование столбца векторных представлений лучше всего, когда у категориального столбца много возможных значений. Здесь мы используем его только для демонстрационных целей, чтобы у вас был полный пример, который вы можете использовать в будущем для другого набора данных." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hSlohmr2Q_UU", - "colab": {} - }, - "source": [ - "# Обратите внимание, что входными данными для столбца векторных представлений является категориальный столбец\n", - "# который мы создали до этого\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "demo(thal_embedding)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "urFCAvTVRMpB" - }, - "source": [ - "### Хэшированные столбцы признаков\n", - "\n", - "Другим способом представления категориального столбца с большим количеством значений является использование [categorical_column_with_hash_bucket](https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_hash_bucket). This feature column calculates a hash value of the input, then selects one of the `hash_bucket_size` buckets to encode a string. When using this column, you do not need to provide the vocabulary, and you can choose to make the number of hash_buckets significantly smaller than the number of actual categories to save space.\n", - "\n", - "Ключевой момент: Важным недостатком этого метода является то, что возможны коллизии, при которых разные строки отображаются в одну и ту же категорию. На практике метод хорошо работает для некоторых наборов данных." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YHU_Aj2nRRDC", - "colab": {} - }, - "source": [ - "thal_hashed = feature_column.categorical_column_with_hash_bucket(\n", - " 'thal', hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(thal_hashed))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fB94M27DRXtZ" - }, - "source": [ - "### Пересеченные столбцы признаков (crossed feature columns)\n", - "Комбинирование признаков в один больше известное как [пересечение признаков](https://developers.google.com/machine-learning/glossary/#feature_cross), позволяет модели изучать отдельные веча для каждой комбинации свойств. Здесь мы создадим новый признак являющийся пересечением возраста и thal. Обратите внимание на то, что `crossed_column` не строит полную таблицу комбинаций значений признаков (которая может быть очень большой). Вместо этого он поддерживает `hashed_column` так что Вы можете сами выбирать размер таблицф." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oaPVERd9Rep6", - "colab": {} - }, - "source": [ - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(crossed_feature))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ypkI9zx6Rj1q" - }, - "source": [ - "## Выберите которые столбцы использовать\n", - "Мы увидели как использовать несколько видов столбцов признаков. Сейчас мы их используем для обучения модели. Данное руководство покажет Вам полный код (т.е. механизм) необходимый для работы со столбцами признаков. Мы ниже случайно выбрали несколько столбцов для обучения нашей модели.\n", - "\n", - "Ключевой момент: если вы собиратесь построить точную модель, попробуйте сами больший набор данных и тщательно подумайте о том, какие признаки являются наиболее значимыми для включения и как они должны быть представлены." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4PlLY7fORuzA", - "colab": {} - }, - "source": [ - "feature_columns = []\n", - "\n", - "# численные столбцы\n", - "for header in ['age', 'trestbps', 'chol', 'thalach', 'oldpeak', 'slope', 'ca']:\n", - " feature_columns.append(feature_column.numeric_column(header))\n", - "\n", - "# группировка значений столбцов\n", - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "feature_columns.append(age_buckets)\n", - "\n", - "# столбцы индикаторы\n", - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "feature_columns.append(thal_one_hot)\n", - "\n", - "# столбцы векторных представлений\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "feature_columns.append(thal_embedding)\n", - "\n", - "# столбцы пересечений свойств\n", - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "crossed_feature = feature_column.indicator_column(crossed_feature)\n", - "feature_columns.append(crossed_feature)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M-nDp8krS_ts" - }, - "source": [ - "### Создайте слой признаков\n", - "Сейчас, после того как мы определили колонки признаков, используем слой [DenseFeatures](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/DenseFeatures) чтобы передать их в модель Keras." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6o-El1R2TGQP", - "colab": {} - }, - "source": [ - "feature_layer = tf.keras.layers.DenseFeatures(feature_columns)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8cf6vKfgTH0U" - }, - "source": [ - "Ранее мы использовали пакеты маленького размера, чтобы показать вам как работают колонки признаков. Сейчас мы создали новый входной пайплайн с большим размером пакетов." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gcemszoGSse_", - "colab": {} - }, - "source": [ - "batch_size = 32\n", - "train_ds = df_to_dataset(train, batch_size=batch_size)\n", - "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", - "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBx4Xu0eTXWq" - }, - "source": [ - "## Создайте, скомпилируйте и обучите модель" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_YJPPb3xTPeZ", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " feature_layer,\n", - " layers.Dense(128, activation='relu'),\n", - " layers.Dense(128, activation='relu'),\n", - " layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'],\n", - " run_eagerly=True)\n", - "\n", - "model.fit(train_ds,\n", - " validation_data=val_ds,\n", - " epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GnFmMOW0Tcaa", - "colab": {} - }, - "source": [ - "loss, accuracy = model.evaluate(test_ds)\n", - "print(\"Accuracy\", accuracy)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3bdfbq20V6zu" - }, - "source": [ - "Ключевой момент: вы, как правило, получите лучшие результаты при использовании глубокого обучения с гораздо большими и более сложными датасетами. При работе с небольшими наборами данных, подобным этому, мы рекомендуем использовать дерево решений или случайный лес в качестве надежной базовой модели. Цель этого руководства - не обучить точную модель, а продемонстрировать механику работы со структурированными данными, дать вам код, который можно использовать в качестве старта при работе с вашими собственными наборами данных в будущем." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SotnhVWuHQCw" - }, - "source": [ - "## Следующие шаги\n", - "Лучший способ узнать больше о классификации структурированных данных - попробовать самостоятельно. Мы предлагаем взять другой набор данных и обучить модель его классифицировать с использованием кода, аналогичного приведенному выше. Чтобы повысить точность, тщательно продумайте, какие признаки включить в вашу модель и как они должны быть представлены." - ] - } - ] -} diff --git a/site/tr/README.md b/site/tr/README.md deleted file mode 100644 index 59c426e3fd4..00000000000 --- a/site/tr/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Topluluk Çevirileri - -Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir. -Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince -güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en) -ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek -için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs) -havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için -[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr) -listesi ile iletişime geçebilirsiniz. - -# Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Please focus translation efforts on -[TensorFlow 2](https://www.tensorflow.org) in the -[site/en/](https://github.com/tensorflow/docs/tree/master/site/en/) -directory. TF 1.x community docs will no longer be updated as we prepare for the -2.0 release. See -[the announcement](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). - -# Turkish translation guide - -Some technical words in English do not have a natural translation. Do *not* -translate the following words: - -* estimator -* pull request diff --git a/site/tr/REVIEWERS b/site/tr/REVIEWERS deleted file mode 100644 index 71b6bc7266a..00000000000 --- a/site/tr/REVIEWERS +++ /dev/null @@ -1,7 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/tr directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs-tr@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr -# diff --git a/site/tr/r1/README.md b/site/tr/r1/README.md deleted file mode 100644 index 850cc4bbd7f..00000000000 --- a/site/tr/r1/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# TensorFlow 1.x - -This archive of the TensorFlow 1.x docs is in *maintenance mode* only. - -For docs contributors, please update the source files in `site/en/` and read the -[TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs). - -For community translations, read the instructions in `site/tr/README.md`. diff --git a/site/tr/r1/tutorials/distribute/keras.ipynb b/site/tr/r1/tutorials/distribute/keras.ipynb deleted file mode 100644 index 38e7f5161c3..00000000000 --- a/site/tr/r1/tutorials/distribute/keras.ipynb +++ /dev/null @@ -1,710 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "keras.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "Tce3stUlHN0L" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.0" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "tuOe1ymfHZPu", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# TensorFlow ile egitimin dagitimi" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r6P32iYYV27b" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SNsdT6sJihFa" - }, - "source": [ - "Note: Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir.\n", - "Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince\n", - "güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en)\n", - "ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek\n", - "için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için\n", - "[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr)\n", - "listesi ile iletişime geçebilirsiniz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "## Genel Bakis\n", - "`tf.distribute.Strategy` API'si TF programinizi birden cok bilgisayar/GPU'ya dagitmak icin bir taslak saglar. Buradaki amac kullanicilarin var olan modellerini ve kodlarini olabildigince ayni tutarak dagitim yapmalarini saglamaktir.\n", - "\n", - "Bu rehber `tf.distribute.MirroredStrategy` taktigini kullanarak ayni makinedeki birden fazla GPU'da ayni \"graph\"i kullanarak eszamanli egitim yapmamizi saglar. Isin ozunde, bu taktik modelinizin degiskenlerini butun islemcilere kopyalar. Daha sonra, [all-reduce](http://mpitutorial.com/tutorials/mpi-reduce-and-allreduce/) yontemi ile butun islemcilerden gelen degisimleri birlestirir ve elde edilen bu degeri varolan butun kopyalara uygular.\n", - "\n", - "`MirroredStrategy` taktigi TensorFlow'un icindeki dagitim taktiklerinden sadece biridir. Diger taktikler hakkinda bilgi edinmek icin bu rehbere goz atabilirsiniz: [dagitim taktikleri rehberi](../../guide/distribute_strategy.ipynb).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUXex9ctTuDB" - }, - "source": [ - "### Keras API\n", - "Bu ornekteki modeller ve egitim donguleri `tf.keras` API'si kullanilarak olusturulmustur. Eger, kendinize ozgu bir egitim dongusu olusturmak istiyorsaniz, [buraya](training_loops.ipynb) goz atiniz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dney9v7BsJij" - }, - "source": [ - "## Gereksinimleri indirelim" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "r8S3ublR7Ay8", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "74rHkS_DB3X2", - "colab": {} - }, - "source": [ - "# TensorFlow kutuphanesini getirelim\n", - "import tensorflow as tf #gpu\n", - "import tensorflow_datasets as tfds\n", - "\n", - "import os" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hXhefksNKk2I" - }, - "source": [ - "## Veri setini indirelim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OtnnUwvmB3X5" - }, - "source": [ - "MNIST veri setini [TensorFlow Datasets](https://www.tensorflow.org/datasets) kullanarak indirip yukleyelim." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lHAPqG8MtS8M" - }, - "source": [ - "`with_info` degiskenini `True` olarak belirtmek metadata'nin butun veri setini kapsadigini belirtir. Bu `metadata` `ds_info` degiskeninde saklanmaktadir. \n", - "Bu metadata nesnesi egitim ve test orneklerinin sayisini da icinde barindirir.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iXMJ3G9NB3X6", - "colab": {} - }, - "source": [ - "datasets, ds_info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n", - "mnist_train, mnist_test = datasets['train'], datasets['test']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GrjVhv-eKuHD" - }, - "source": [ - "## Dagitim taktigimizi tanimlayalim\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TlH8vx6BB3X9" - }, - "source": [ - "Bir `MirroredStrategy` nesnesi olusturalim. Bu sayede dagitimi kontrol edebilir ve modelimizi icinde olusturabilecegimiz bir ortam yonetmeni (`tf.distribute.MirroredStrategy.scope`) saglamis oluruz. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4j0tdf4YB3X9", - "colab": {} - }, - "source": [ - "strategy = tf.distribute.MirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cY3KA_h2iVfN", - "colab": {} - }, - "source": [ - "print ('Number of devices: {}'.format(strategy.num_replicas_in_sync))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lNbPv0yAleW8" - }, - "source": [ - "## Input pipeline kuralim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "psozqcuptXhK" - }, - "source": [ - "Eger bir model birden fazla GPU ile egitiliyorsa, grup boyutleri ayni oranda arttirilmalidir ki islemci gucunu verimli sekilde kullanabilelim. Ayni zamanda ogrenme hizinin da GPU miktarina gore ayarlanmasi gerekir. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p1xWxKcnhar9", - "colab": {} - }, - "source": [ - "# 'ds_info.splits.total_num_examples' metodunu kullanarak da orneklerin sayisini bulabilirsiniz.\n", - "\n", - "num_train_examples = ds_info.splits['train'].num_examples\n", - "num_test_examples = ds_info.splits['test'].num_examples\n", - "\n", - "BUFFER_SIZE = 10000\n", - "\n", - "BATCH_SIZE_PER_REPLICA = 64\n", - "BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Wm5rsL2KoDF" - }, - "source": [ - "Piksel degerleri, ki bunlar aslinda 0-255 arasindadir, [normallestirilerek 0 ile 1 arasinda bir degere indirgenmelidir](https://en.wikipedia.org/wiki/Feature_scaling). Bu ingirdenme olcegini bir fonksiyon ile tanimlayalim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Eo9a46ZeJCkm", - "colab": {} - }, - "source": [ - "def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - "\n", - " return image, label" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WZCa5RLc5A91" - }, - "source": [ - "Simdi bu fonksiyonu egitim ve test verisine uygulayalim. Sonrasinda egitim verisini karistiralim ve [egitim icin gruplayalim](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch).\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gRZu2maChwdT", - "colab": {} - }, - "source": [ - "train_dataset = mnist_train.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", - "eval_dataset = mnist_test.map(scale).batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4xsComp8Kz5H" - }, - "source": [ - "## Modeli olusturalim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1BnQYQTpB3YA" - }, - "source": [ - "Keras modelimizi `strategy.scope` ortaminda olusturalim ve toparlayalim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IexhL_vIB3YA", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - "\n", - " model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8i6OU5W9Vy2u" - }, - "source": [ - "## Bildirim fonksiyonlarini tanimlayalim\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YOXO5nvvK3US" - }, - "source": [ - "Burada kullanilan bildirim fonksiyonlari sunlardir:\n", - "* *Tensorboard*: Bu geri arama Tensorboard'lar icin kayit tutar. Bu kayitlar ile bir grafik olusturabiliriz.\n", - "* *Model Checkpoint*: Bu geri arama her devir sonunda modeli kaydeder.\n", - "* *Learning Rate Scheduler*: Bu geri aramayi kullanarak, ogrenme hizinin her devir ya da gruptan sonra degismesini programlayabilirsiniz.\n", - "\n", - "Daha aciklayici olmasi icin, 'ogrenme hizini' bir geri arama ile kitapciga yazdirabilirsiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "A9bwLCcXzSgy", - "colab": {} - }, - "source": [ - "# Kontrol noktalarini kaydetmek icin bir kontrol noktasi dosyasi olusturalim.\n", - "\n", - "checkpoint_dir = './training_checkpoints'\n", - "# Kontrol noktasi dosyalarinin ismi\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wpU-BEdzJDbK", - "colab": {} - }, - "source": [ - "# Azalan ogrenme hizi icin bir fonksiyon\n", - "# Ihtiyaciniz olan butun azalma fonksiyonlarini kullanabilirsiniz.\n", - "def decay(epoch):\n", - " if epoch < 3:\n", - " return 1e-3\n", - " elif epoch >= 3 and epoch < 7:\n", - " return 1e-4\n", - " else:\n", - " return 1e-5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jKhiMgXtKq2w", - "colab": {} - }, - "source": [ - "# Her devrin sonunda LR'i (ogrenme hizini) yazdirmak icin bir geri arama fonksiyonu.\n", - "class PrintLR(tf.keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs=None):\n", - " print ('\\nLearning rate for epoch {} is {}'.format(\n", - " epoch + 1, tf.keras.backend.get_value(model.optimizer.lr)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YVqAbR6YyNQh", - "colab": {} - }, - "source": [ - "callbacks = [\n", - " tf.keras.callbacks.TensorBoard(log_dir='./logs'),\n", - " tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,\n", - " save_weights_only=True),\n", - " tf.keras.callbacks.LearningRateScheduler(decay),\n", - " PrintLR()\n", - "]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "70HXgDQmK46q" - }, - "source": [ - "## Egitim ve yorumlama" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6EophnOAB3YD" - }, - "source": [ - "Simdi, modelimizi her zamanki gibi egitelim: `fit` yontemini model uzerinde cagirip bu rehberin basinda olusturdugumuz veri setini modele verelim. Bu asama dagitim yapsaniz da yapmasaniz da hep ayni kalacaktir.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MVw_6CqB3YD", - "colab": {} - }, - "source": [ - "model.fit(train_dataset, epochs=10, callbacks=callbacks)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NUcWAUUupIvG" - }, - "source": [ - "Asagida gordugunuz gibi, kontrol noktalari hafizaya kaydedilmektedir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JQ4zeSTxKEhB", - "colab": {} - }, - "source": [ - "# Kontrol noktalari dosyasina bakalim.\n", - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qor53h7FpMke" - }, - "source": [ - "Modelin nasil calistigini gormek icin, en son kaydedilen kontrol noktasini yukleyip 'evaluate' yontemini test verisinde cagirabilirsiniz.\n", - "\n", - "'evaluate' yontemini daha once yaptiginiz gibi uygun veri setlerinde kullanmalisiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JtEwxiTgpQoP", - "colab": {} - }, - "source": [ - "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "eval_loss, eval_acc = model.evaluate(eval_dataset)\n", - "print('Eval loss: {}, Eval Accuracy: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IIeF2RWfYu4N" - }, - "source": [ - "Programin sonuclarini, TensorBoard kayitlarini indirerek terminalde gorebilirsiniz. \n", - "\n", - "```\n", - "$ tensorboard --logdir=path/to/log-directory\n", - "```" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LnyscOkvKKBR", - "colab": {} - }, - "source": [ - "!ls -sh ./logs" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kBLlogrDvMgg" - }, - "source": [ - "## Kaydedilen modelin _'SavedModel'_ cikartilmasi" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xa87y_A0vRma" - }, - "source": [ - "Eger \"graph\"larin ve degiskenlerin program disinda kullanilmasini istiyorsaniz, `SavedModel` sizin icin ideal yontem. Bu yontem herhangi bir kapsama bagli olmadan yuklenebir ve herhangi bir platforma da bagli degildir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h8Q4MKOLwG7K", - "colab": {} - }, - "source": [ - "path = 'saved_model/'" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4HvcDmVsvQoa", - "colab": {} - }, - "source": [ - "tf.keras.experimental.export_saved_model(model, path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vKJT4w5JwVPI" - }, - "source": [ - "Modeli 'strategy.scope' olmadan yukleyelim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "T_gT0RbRvQ3o", - "colab": {} - }, - "source": [ - "unreplicated_model = tf.keras.experimental.load_from_saved_model(path)\n", - "\n", - "unreplicated_model.compile(\n", - " loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])\n", - "\n", - "eval_loss, eval_acc = unreplicated_model.evaluate(eval_dataset)\n", - "print('Eval loss: {}, Eval Accuracy: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8uNqWRdDMl5S" - }, - "source": [ - "## Sirada ne var?\n", - "\n", - "Dagitim taktikleri rehberini [distribution strategy guide](../../guide/distribute_strategy_tf1.ipynb) okuyunuz.\n", - "\n", - "Note: `tf.distribute.Strategy` surekli gelistirilmektedir ve yakin zamanda yeni ornekler ve rehberler buraya eklenecektir. Lutfen bu yontemleri deneyiniz. Sizden gelen yorumlara her zaman acigiz. Bu yorumlari buraya yazabilirsiniz [issues on GitHub](https://github.com/tensorflow/tensorflow/issues/new)." - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/distribute/training_loops.ipynb b/site/tr/r1/tutorials/distribute/training_loops.ipynb deleted file mode 100644 index 60aa3243e02..00000000000 --- a/site/tr/r1/tutorials/distribute/training_loops.ipynb +++ /dev/null @@ -1,460 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "training_loops.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "MhoQ0WE77laV" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.0" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# Egitim donguleri ile tf.distribute.Strategy" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-adnPCDYkI8e" - }, - "source": [ - "Note: Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir.\n", - "Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince\n", - "güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en)\n", - "ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek\n", - "için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için\n", - "[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr)\n", - "listesi ile iletişime geçebilirsiniz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "Bu rehber egitim donguleri ile [`tf.distribute.Strategy`](https://www.tensorflow.org/r1/guide/distribute_strategy)'nin nasil kullanildigini gosteriyor. Basit bir CNN modelini Fashion MNIST veri seti ile egitecegiz. Bu veri seti icinde 28X28 boyutunda 60000 egitim resmini ve 28X28 boyutunda 10000 test resmini barindirir.\n", - "\n", - "Burada bize esneklik ve daha cok kontrol kazandirmasi icin ozellestirilmis egitim donguleri kullanacagiz. Ustelik, bu ozel donguler modeli ve egitim dongulerindeki hatalari ayiklamamizi da kolaylastiracaktir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow'u yukleyelim\n", - "import tensorflow as tf\n", - "\n", - "# Yardimci kutuphaneler\n", - "import numpy as np\n", - "import os\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MM6W__qraV55" - }, - "source": [ - "## Fashion MNIST veri setini indirelim" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = tf.keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\n", - "\n", - "# Diziye yeni bir boyut ekleyelim-> new shape == (28, 28, 1)\n", - "# Bunu yapmamizin sebebi ise modelimizin ilk katmaninin katlamali olmasi\n", - "# ve 4D bir girdiye ihtiyac duyar (batch_size, height, width, channels).\n", - "# batch_size boyutunu daha sonra ekleyecegiz.\n", - "train_images = train_images[..., None]\n", - "test_images = test_images[..., None]\n", - "\n", - "# Resimleri [0, 1] araligina indirgeyelim.\n", - "train_images = train_images / np.float32(255)\n", - "test_images = test_images / np.float32(255)\n", - "\n", - "train_labels = train_labels.astype('int64')\n", - "test_labels = test_labels.astype('int64')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4AXoHhrsbdF3" - }, - "source": [ - "## Degiskenleri ve grafigi dagitmak icin bir taktik olusturalim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5mVuLZhbem8d" - }, - "source": [ - "`tf.distribute.MirroredStrategy` nasil calisir?\n", - "\n", - "* Butun degiskenler ve model grafigi birkac kere kopyalanir.\n", - "* Girdi bu kopyalara esit olarak dagitilir.\n", - "* Her kopya verilen girdiye gore bir kayip ve degisim tablosu hesaplar.\n", - "* Butun degisim verileri toplanir ve kopyalardaki degerler bu toplama gore guncellenir.\n", - "* Bu islemden sonra, ayni guncelleme degiskenlerin kopyalarina da uygulanir. \n", - "\n", - "Note: Butun kodu tek bir kapsam icine koyabilirsiniz, fakat biz burada daha aciklayici olmasi icin kodu boluyoruz.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F2VeZUWUj5S4", - "colab": {} - }, - "source": [ - "# Eger kullanilacak cihazlar `tf.distribute.MirroredStrategy` yapicisinda belirtilmediyse\n", - "# otomatik olarak bulunacaktir.\n", - "strategy = tf.distribute.MirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZngeM_2o0_JO", - "colab": {} - }, - "source": [ - "print ('Number of devices: {}'.format(strategy.num_replicas_in_sync))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k53F5I_IiGyI" - }, - "source": [ - "## Girdi hattinin kurulmasi" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Qb6nDgxiN_n" - }, - "source": [ - "Eger bir model birden fazla GPU'da egitiliyorsa, grup boyutu buna orantili olarak arttirilmalidir ki fazla bilgisayar gucunu verimli bir sekilde kullanabilelim. Ayrica, egitim hizi da orantili olarak ayarlanmaidir. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwJtsCQhHK-E", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = len(train_images)\n", - "\n", - "BATCH_SIZE_PER_REPLICA = 64\n", - "BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n", - "\n", - "EPOCHS = 10" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J7fj3GskHC8g" - }, - "source": [ - "`strategy.make_dataset_iterator`, veriyi kopyalara esit olarak dagitan bir iterator olusturur. \n", - "\n", - "\n", - "Note: Bu API yakin zamanda degisecektir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WYrMNNDhAvVl", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " train_dataset = tf.data.Dataset.from_tensor_slices(\n", - " (train_images, train_labels)).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", - " train_iterator = strategy.make_dataset_iterator(train_dataset)\n", - "\n", - " test_dataset = tf.data.Dataset.from_tensor_slices(\n", - " (test_images, test_labels)).batch(BATCH_SIZE)\n", - " test_iterator = strategy.make_dataset_iterator(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bAXAo_wWbWSb" - }, - "source": [ - "## Modelin olusturulmasi\n", - "\n", - "`tf.keras.Sequential` ile modelimizi olusturalim. Model Subclassing API'yini da kullanarak bu islemi yapabiliriz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu',\n", - " input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Conv2D(64, 3, activation='relu'),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - " optimizer = tf.train.GradientDescentOptimizer(0.001)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e-wlFFZbP33n" - }, - "source": [ - "## Kayip fonksiyonunu tanimlayalim\n", - "\n", - "Normalde, eger 1 GPU/CPU'lu bir makine kullaniyorsak, kayip girdi grubundaki ornek sayisina bolunur.\n", - "\n", - "*Peki `tf.distribute.Strategy` ile kayip nasil hesaplanir?*\n", - "\n", - "> Ornegin, 4 GPU'muz ve boyutu 64 olan girdimiz oldugunu varsayalim. Bu girdiler esit olarak 4 GPU (4 kopya) ustune bolunur, yani her kopyaya giden girdi grub boyutu 16 idir.\n", - "\n", - "> Her kopyadaki model icindeki girdinin ustunden gecerek kayip degerini hesaplar. Simdi, bu kayip degerini icindeki girdi sayisina (16) bolmek yerine, en bastaki evrensel girdi miktarina (64) boler. \n", - "\n", - "*Neden bu islem boyle yaplir?*\n", - "\n", - "> Cunku degisim degerleri her kopyada hesaplandiktan sonra, butun kopyalardaki degerler butun degisim degerlerinin toplamina esitlenir.\n", - "\n", - "*Bunu TensorFlow'da nasil yapabiliriz?*\n", - "\n", - "Eger ozellestirilmis bir egitim dongusu yaziyorsaniz, her ornekteki kayiplari toplayip butun orneklerin toplamina bolmelisiniz:\n", - "\n", - "```\n", - "GLOBAL_BATCH_SIZE:`scale_loss = tf.reduce_sum(loss) * (1. / GLOBAL_BATCH_SIZE)`\n", - "```\n", - "\n", - "* `tf.reduce_mean` metodunu kullanmanizi tavsiye etmiyoruz. Bu metod kayip degerini kopyalardaki ornek sayisina boler ki bu her adimda degisebilir.\n", - "\n", - "* Bu indirgeme ve olcekleme keras'ta otomatok olarak yapilir: model.fit ve model.compile ile\n", - "* Eger `tf.keras.losses` siniflarini kullaniyorsaniz, kayip indirgemesinin ozellikle `NONE` ya da `SUM` olarak belirtilmesi gerekmektedir. `AUTO` ve `SUM_OVER_BATCH_SIZE` ise `tf.distribute.Strategy` ile birlikte kullanilamaz. Cunku kullanicilarin `AUTO` kullanmadan once yaptiklari indirgemenin o anki dagitim ornegindeki dogrulugundan emin olmalari gerekir. `SUM_OVER_BATCH_SIZE` kullanilamaz cunku su anki haliyle sadece kopyadaki ornek sayisina bolum yapip asil toplam ornek sayisina bolme islemini kullaniciya birakir, ki bu cok kolay gozden kacabilecek bir noktadir. Onun yerine kullanicinin indirgemeyi kendilerinin yapmalarini istiyoruz.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iuKuNXPORfqJ" - }, - "source": [ - "## Egitim dongusu" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "47BLVkRkVQDO", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " def train_step():\n", - " def step_fn(inputs):\n", - " images, labels = inputs\n", - " logits = model(images)\n", - " cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(\n", - " logits=logits, labels=labels)\n", - " loss = tf.reduce_sum(cross_entropy) * (1.0 / BATCH_SIZE)\n", - " train_op = optimizer.minimize(loss)\n", - " with tf.control_dependencies([train_op]):\n", - " return tf.identity(loss)\n", - "\n", - " per_replica_losses = strategy.experimental_run(\n", - " step_fn, train_iterator)\n", - " mean_loss = strategy.reduce(\n", - " tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n", - " return mean_loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7x7s5iYAYSGD", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " iterator_init = train_iterator.initialize()\n", - " var_init = tf.global_variables_initializer()\n", - " loss = train_step()\n", - " with tf.Session() as sess:\n", - " sess.run([var_init])\n", - " for epoch in range(EPOCHS):\n", - " sess.run([iterator_init])\n", - " for step in range(10000):\n", - " if step % 1000 == 0:\n", - " print('Epoch {} Step {} Loss {:.4f}'.format(epoch+1,\n", - " step,\n", - " sess.run(loss)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6hEJNsokjOKs" - }, - "source": [ - "## Sirada ne var?\n", - "\n", - "Simdi `tf.distribute.Strategy` API'yini kendi modellerinizde deneyin. " - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/eager/README.md b/site/tr/r1/tutorials/eager/README.md deleted file mode 100644 index b004e8ebd1a..00000000000 --- a/site/tr/r1/tutorials/eager/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# Arastirma ve denemeler - -Eager uygulamasi daha gelismis islemler icin zorunlu ve her calistirmada yeniden -tanimlanabilir bir arayuz saglar. Otomatik ayrimlasma sayesinde kendinize ozel -katman, ileri gecisler ve egitim donguleri yazabilirsiniz. Bu egitim -kitapciklari ile baslayiniz, daha sonra -[eager uygulama kilavuzu](https://github.com/tensorflow/docs/blob/master/site/en/r1/guide/eager.ipynb) -'na bakabilirsiniz. - -1. [Eager uygulamasi](eager_basics.ipynb) -2. [Otomatik ayrimlasma ve degisim bandi](automatic_differentiation.ipynb) -3. [Kisisel egitim: ana hatlari](custom_training.ipynb) -4. [Kisisel katmanlar](custom_layers.ipynb) -5. [Kisisel egitim: gozden gecirme](custom_training_walkthrough.ipynb) diff --git a/site/tr/r1/tutorials/eager/automatic_differentiation.ipynb b/site/tr/r1/tutorials/eager/automatic_differentiation.ipynb deleted file mode 100644 index 5e65a651946..00000000000 --- a/site/tr/r1/tutorials/eager/automatic_differentiation.ipynb +++ /dev/null @@ -1,309 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "automatic_differentiation.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "GCCk8_dHpuNf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xh8WkEwWpnm7" - }, - "source": [ - "# Otomatik degisim ve egim banti" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idv0bPeCp325" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJ4XzMqodTy" - }, - "source": [ - "Bir onceki egitim kitapciginda 'Tensor'lari ve onlar ustunde kullanabileceginiz operasyonlari tanittik. Bu kitapcikta ise makine ogrenmesi modellerinin eniyilenmesinde onemli bir teknik olan [otomatik degisimi](https://en.wikipedia.org/wiki/Automatic_differentiation) ogrenecegiz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GQJysDM__Qb0" - }, - "source": [ - "## Kurulum\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OiMPZStlibBv", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1CLWJl0QliB0" - }, - "source": [ - "## Egim bantlari\n", - "\n", - "TensorFlow'un [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) API'si otomatik degisim yani girdi degiskenlerine bagli olarak hesaplanan egimin hesaplanisini hali hazirda bize saglar. Tensorflow `tf.GradientTape` kapsaminda yapilan butun operasyonlari bir \"tape(bant)\"e \"kaydeder\". Tensorflow daha sonra \"kaydedilmis\" egimleri, bu bant ve her bir kayitla iliskili egim verilerini [ters mod degisimi](https://en.wikipedia.org/wiki/Automatic_differentiation) kullanarak hesaplar.\n", - "\n", - "Ornegin:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bAFeIE8EuVIq", - "colab": {} - }, - "source": [ - "x = tf.ones((2, 2))\n", - "\n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# Orjinal girdi tensoru x'e gore z'nin turevi\n", - "dz_dx = t.gradient(z, x)\n", - "for i in [0, 1]:\n", - " for j in [0, 1]:\n", - " assert dz_dx[i][j].numpy() == 8.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N4VlqKFzzGaC" - }, - "source": [ - "Ayrica \"kaydedilmis\" 'tf.GradientTape' kapsaminda hesaplanan ara degerlere gore ciktilari egimini de isteyebilirsiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7XaPRAwUyYms", - "colab": {} - }, - "source": [ - "x = tf.ones((2, 2))\n", - "\n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# Banti kullanarak ara deger y'ye gore z'nin turevini hesaplayabiliriz.\n", - "dz_dy = t.gradient(z, y)\n", - "assert dz_dy.numpy() == 8.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISkXuY7YzIcS" - }, - "source": [ - "GradientTape.gradient() yontemini cagirdimizda GradientTape tarafindan tutulan kaynaklar serbest birakilir. Ayni degerleri kullanarak birden fazla egim hesaplamak istiyorsaniz 'persistent(kalici)' egim banti olusturmalisiniz. Bu sayede bant nesnesi cop toplayicisi tarafindan toplanip kaynaklar serbest birakildikca 'gradient()' yontemini bircok kere cagirmamiza izin verir. Ornegin:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zZaCm3-9zVCi", - "colab": {} - }, - "source": [ - "x = tf.constant(3.0)\n", - "with tf.GradientTape(persistent=True) as t:\n", - " t.watch(x)\n", - " y = x * x\n", - " z = y * y\n", - "dz_dx = t.gradient(z, x) # 108.0 (4*x^3 at x = 3)\n", - "dy_dx = t.gradient(y, x) # 6.0\n", - "del t # Referansi banta indirgeyelim" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6kADybtQzYj4" - }, - "source": [ - "### Kontrol akimini kaydedelim\n", - "\n", - "Bantlar operasyonlar yurutuldukce kaydettigi icin, Python kontrol akimlari (`if`ler ve `while`lar gibi) dogal olarak islenir:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9FViq92UX7P8", - "colab": {} - }, - "source": [ - "def f(x, y):\n", - " output = 1.0\n", - " for i in range(y):\n", - " if i > 1 and i < 5:\n", - " output = tf.multiply(output, x)\n", - " return output\n", - "\n", - "def grad(x, y):\n", - " with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " out = f(x, y)\n", - " return t.gradient(out, x)\n", - "\n", - "x = tf.convert_to_tensor(2.0)\n", - "\n", - "assert grad(x, 6).numpy() == 12.0\n", - "assert grad(x, 5).numpy() == 12.0\n", - "assert grad(x, 4).numpy() == 4.0\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DK05KXrAAld3" - }, - "source": [ - "### Yuksek-sirali egimler\n", - "\n", - "`GradientTape` kapsam yoneticisindeki operasyonlar otomatik degisim icin kaydedilir. Eger egimler bu kapsamda hesaplandiysa onlar da ayni sekilde kaydedilir. Sonuc olarak, ayni API'yi kullanarak yuksek-sirali egimleri hesaplayabiliriz. Ornegin:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cPQgthZ7ugRJ", - "colab": {} - }, - "source": [ - "x = tf.Variable(1.0) # 1.0 degerine ilklenmis bir Tensorflow degiskeni olusturalim\n", - "\n", - "with tf.GradientTape() as t:\n", - " with tf.GradientTape() as t2:\n", - " y = x * x * x\n", - " # 't' kapsam yoneticisi icerisinde egimi hesaplayalim\n", - " # ki bu egim hesaplanmasinin turevlenebilir oldugu anlamina gelir.\n", - " dy_dx = t2.gradient(y, x)\n", - "d2y_dx2 = t.gradient(dy_dx, x)\n", - "\n", - "assert dy_dx.numpy() == 3.0\n", - "assert d2y_dx2.numpy() == 6.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4U1KKzUpNl58" - }, - "source": [ - "## Bir sonraki adimlar\n", - "\n", - "Bu kitapcikta egim hesaplanmasinin TensorFlow'da nasil yapildigini gorduk. Simdi sinir agimizi olusturmak ve egitmek icin gerekli ilkellerin hepsine sahibiz." - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/eager/custom_layers.ipynb b/site/tr/r1/tutorials/eager/custom_layers.ipynb deleted file mode 100644 index 79a07a3eb32..00000000000 --- a/site/tr/r1/tutorials/eager/custom_layers.ipynb +++ /dev/null @@ -1,354 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_layers.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tDnwEv8FtJm7" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "JlknJBWQtKkI", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "60RdWsg1tETW" - }, - "source": [ - "# Ozel Katmanlar" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BcJg7Enms86w" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UEu3q4jmpKVT" - }, - "source": [ - "Sinir aglarini olustururken yuksek seviye API olan 'tf.keras'i kullanmanizi tavsiye ederiz, ama bircok TensorFlow APIsi eager modunda rahatlikla kullanilabilir.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pwX7Fii1rwsJ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zSFfVVjkrrsI" - }, - "source": [ - "## Katmanlar: kullanisli operasyonlarin bir araya geldigi setler\n", - "\n", - "Makine ogrenmesi modelleri icin kod yazarken degiskenleri tek tek kontrol etmek ya da operasyonlari tek tek yonetmek yerine yuksek seviyede soyut dusunmeyi tercih ederiz.\n", - "\n", - "Bircok makine ogrenmesi modeli basit katmanlarin ust uste binmesi ve birlestirilmesi olarak aciklanabilir. TensorFlow hem cok sayida genel katmanlar sunar hem de kendi uygulamaniza ozel katmanlari ya sifirdan ya da var olan katmanlara eklemeler yaparak kolayca olusturmanizi saglar.\n", - "\n", - "[Keras](https://keras.io) API'si TensorFlow icindeki tf.keras paketinden bulunabilir ve Keras katmanlari kendi modellerinizi olustururken kullanabilirsiniz.\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8PyXlPl-4TzQ", - "colab": {} - }, - "source": [ - "# tf.keras.layers paketinde katmanlar birer nesnedir. Katmanlari olusturmak icin,\n", - "# nesneyi olusturmaniz yeterlidir. Bircok katman cikti boyutlarinin ya da kanallarin sayisini \n", - "# ilk bagimsiz degisken olarak alir.\n", - "layer = tf.keras.layers.Dense(100)\n", - "# Girdi boyutlarinin sayisi ise katman ilk kullanildiginda anlasilabilecegi icin cogunlukla gereksizdir,\n", - "# fakat isterseniz bunu ozellikle belirtebilirsiniz ki bu kompleks modellerde faydali olabilir\n", - "layer = tf.keras.layers.Dense(10, input_shape=(None, 5))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fn69xxPO5Psr" - }, - "source": [ - "Var olan katmanlarin tam listesi icin [buraya bakiniz](https://www.tensorflow.org/api_docs/python/tf/keras/layers). Bunlardan bazilari Dense: (tam-bagli katman),\n", - "Conv2D, LSTM, BatchNormalization, Dropout." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "E3XKNknP5Mhb", - "colab": {} - }, - "source": [ - "# Katmani kullanmak icin cagirmaniz yeterlidir.\n", - "layer(tf.zeros([10, 5]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Wt_Nsv-L5t2s", - "colab": {} - }, - "source": [ - "# Katmanlarin bircok kullanisli yontemleri vardir. Ornegin, butun katmandaki degiskenleri \n", - "# `layer.variables` yontemi, egitebilir degiskenleri ise `layer.trainable_variables` \n", - "# yontemi ile inceleyebilirsiniz. Burada ise tam-bagli bir katman agirlik ve sapma degiskenlerine sahiptir.\n", - "layer.variables" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6ilvKjz8_4MQ", - "colab": {} - }, - "source": [ - "# Degiskenlere ayni zamanda 'nice accessors' ile erisebiliriz\n", - "layer.kernel, layer.bias" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "O0kDbE54-5VS" - }, - "source": [ - "## Ozel katmanlari gerceklestirelim\n", - "tf.keras.Layer sinifini kullanip gelistirmek kendi katmanlarinizi olusturmanin en iyi yoludur:\n", - " * `__init__` , girdiye bagli olamayan ilklendirmeleri burda yapabilirsiniz\n", - " * `build`, burada girdi tensorlarinin seklini bildiginiz icin geri kalan ilkelendirmeleri yapabilirsiniz\n", - " * `call`, ileri hesaplamalari yapabildigimiz yer\n", - "\n", - "Degiskenlerinizi olusturmak icin `build` asamasini beklemenize gerek yoktur, onlari `__init__` asamasinda da olusturabilirsiniz. Fakat, `build` asamasinda olusturulan degiskenler katmanlarin uzerinde calisacagi girdilerin seklini bilme avantajina sahiptir. Degiskenlerin `__init__` evresinde olusturulmasi ise bu sekillerin ozellikle belirtilmesini gerektirir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5Byl3n1k5kIy", - "colab": {} - }, - "source": [ - "class MyDenseLayer(tf.keras.layers.Layer):\n", - " def __init__(self, num_outputs):\n", - " super(MyDenseLayer, self).__init__()\n", - " self.num_outputs = num_outputs\n", - "\n", - " def build(self, input_shape):\n", - " self.kernel = self.add_variable(\"kernel\",\n", - " shape=[int(input_shape[-1]),\n", - " self.num_outputs])\n", - "\n", - " def call(self, input):\n", - " return tf.matmul(input, self.kernel)\n", - "\n", - "layer = MyDenseLayer(10)\n", - "print(layer(tf.zeros([10, 5])))\n", - "print(layer.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tk8E2vY0-z4Z" - }, - "source": [ - "Kullanicilar standart katmanlarin davranisina asina oldugu icin mumkun oldugunca bunlari kullanmak kodunuzun okunabilirligini ve bakimini kolaylastiracaktir. tf.keras.layers ya da tf.contrib.layers icerisinde bulunmayan bir katman kullanmak istiyorsaniz [buraya](http://github.com/tensorflow/tensorflow/issues/new) basvurmanizi ya da daha iyisi bize bir 'pull request' gondermenizi tavsiye ederiz!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Qhg4KlbKrs3G" - }, - "source": [ - "## Modeller: katmanlarin birlestirilmesi\n", - "\n", - "Makine ogrenmesi modellerindeki ilginc katman benzeri seyler var olan katmanlarin birlestirilmesi ile olusturulur. Ornegin, 'resnet'teki her obek kivrimlarin, toplu normallestirmelerin ve kisa yollarin birlesmesi ile olusmustur.\n", - "\n", - "tf.keras.Model sinif diger katmnalari iceren katman benzeri seylerin olusturulmasinda kullanilir. tf.keras.Model sinifindan kalit alarak bunu yapabilirsiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N30DTXiRASlb", - "colab": {} - }, - "source": [ - "class ResnetIdentityBlock(tf.keras.Model):\n", - " def __init__(self, kernel_size, filters):\n", - " super(ResnetIdentityBlock, self).__init__(name='')\n", - " filters1, filters2, filters3 = filters\n", - "\n", - " self.conv2a = tf.keras.layers.Conv2D(filters1, (1, 1))\n", - " self.bn2a = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2b = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same')\n", - " self.bn2b = tf.keras.layers.BatchNormalization()\n", - "\n", - " self.conv2c = tf.keras.layers.Conv2D(filters3, (1, 1))\n", - " self.bn2c = tf.keras.layers.BatchNormalization()\n", - "\n", - " def call(self, input_tensor, training=False):\n", - " x = self.conv2a(input_tensor)\n", - " x = self.bn2a(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2b(x)\n", - " x = self.bn2b(x, training=training)\n", - " x = tf.nn.relu(x)\n", - "\n", - " x = self.conv2c(x)\n", - " x = self.bn2c(x, training=training)\n", - "\n", - " x += input_tensor\n", - " return tf.nn.relu(x)\n", - "\n", - "\n", - "block = ResnetIdentityBlock(1, [1, 2, 3])\n", - "print(block(tf.zeros([1, 2, 3, 3])))\n", - "print([x.name for x in block.trainable_variables])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wYfucVw65PMj" - }, - "source": [ - "Cogunlukla bircok katmandan olusan modeller sadece katmanlari belli bir sirayla cagirirlar. Bu islem tf.keras.Sequential kullanilarak cok az bir kod ile yapilabilir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "L9frk7Ur4uvJ", - "colab": {} - }, - "source": [ - " my_seq = tf.keras.Sequential([tf.keras.layers.Conv2D(1, (1, 1)),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(2, 1,\n", - " padding='same'),\n", - " tf.keras.layers.BatchNormalization(),\n", - " tf.keras.layers.Conv2D(3, (1, 1)),\n", - " tf.keras.layers.BatchNormalization()])\n", - "my_seq(tf.zeros([1, 2, 3, 3]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c5YwYcnuK-wc" - }, - "source": [ - "# Sirada ne var\n", - "\n", - "Simdi bir onceki kitapciga donerek dogrusal gerileme ornegini katmanlari ve modelleri kullanarak daha iyi yapilandirabilirsiniz." - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/eager/custom_training.ipynb b/site/tr/r1/tutorials/eager/custom_training.ipynb deleted file mode 100644 index 1870710fb2f..00000000000 --- a/site/tr/r1/tutorials/eager/custom_training.ipynb +++ /dev/null @@ -1,427 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rmpybwysXGV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m8y3rGtQsYP2", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hrXv0rU9sIma" - }, - "source": [ - "# Kisisellestirilmis egitim: Temelleri" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7S0BwJ_8sLu7" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k2o3TTG4TFpt" - }, - "source": [ - "Bir onceki egitim kitapciginda TensorFLow API'sinin makine ogrenmesinin temel yapi taslarindan biri olan otomatik degisimlerde kullanimini gorduk.\n", - "Bu kitapcikta ise daha onceki kitapciklarda ogrendigimiz TensorFlow ilkellerini basit makine ogrenmesinde kullanacagiz.\n", - "\n", - "Bunun disinda TensorFlow icinde soyutlamayi kolaylastiran yuksek seviye sinir agi API'si 'tf.keras'i da bulundurmaktadir. Sinir aglari ile calisanlarin bu yuksek seviye API'larini kullanmalarini siddetle tavsiye ederiz. Fakat, bu kisa kitapcikta saglam temeller olusturmak icin sinir aglari egitiminin ilk ilkelerinden bahsedecegiz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3LXMVuV0VhDr" - }, - "source": [ - "## Kurulum" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "PJ64L90aVir3", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eMAWbDJFVmMk" - }, - "source": [ - "## Degiskenler\n", - "\n", - "TensorFlow'daki Tensorlar sabit(immutable), durumsuz(stateless) nesnelerdir. Fakat makine ogrenmesi modellerinin durumu degiskendir: modeliniz egitiminde ilerledikce ayni kodun yaptigi tahminler farkli davranislar sergilemelidir (bunu dusuk kayipla yapmasini umut ediyoruz!). Yaptiginiz islemler suresunce degisen bu durumu Python'un durumsal bir yazilim dili olmasi ile gosterebiliriz:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VkJwtLS_Jbn8", - "colab": {} - }, - "source": [ - "# Python durumunu kullanmak\n", - "x = tf.zeros([10, 10])\n", - "x += 2 # This is equivalent to x = x + 2, which does not mutate the original\n", - " # value of x\n", - "print(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfneTXy7JcUz" - }, - "source": [ - "TensorFlow icerisinde durumsal operasyonlar bulundurmaktadir ve cogu zaman durumun gosterimi icin bunlari kullanmak dusuk-seviye Python gosterimlerinden daha iyidir. Ornegin, modeldeki agirliklari gostermek icin TensorFlow degiskenlerini kullanmak hem kolay hem de verimlidir.\n", - "\n", - "TensorFlow hesaplamalarinda kullanildiginda kaydedilen degeri okuyacak olan, Degisken icinde bir deger bulunduran bir nesnedir. TensorFlow degiskenininde kaydedilen degeri degistirmek icin `tf.assign_sub` ve `tf.scatter_update` gibi cesitli operasyonlar bulunmaktadir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "itxmrMil6DQi", - "colab": {} - }, - "source": [ - "v = tf.Variable(1.0)\n", - "assert v.numpy() == 1.0\n", - "\n", - "# Degeri yeniden esleyelim\n", - "v.assign(3.0)\n", - "assert v.numpy() == 3.0\n", - "\n", - "# tf.square() gibi bir TensorFlow operasyonunda `v`yi kullanmalim ve yeniden esleyelim\n", - "v.assign(tf.square(v))\n", - "assert v.numpy() == 9.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-paSaeq1JzwC" - }, - "source": [ - "Degiskenler kullanilarak yapilan hesaplamalarda egimler otomatik olarak takip edelir. Katistirmalari gosteren Degiskenlerde ise TensorFlow donem donem guncellestirme yapar, ki bunlar hesaplama gucunde ve hafizada verimlidir.\n", - "\n", - "Degiskenlerin kullanilmasi ayni zamanda kodunuzu okuyan kisiye bu durumun sabit olmadigini(mutable) belirtir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BMiFcDzE7Qu3" - }, - "source": [ - "## Ornek: Dogrusal modeli uydurmak\n", - "\n", - "Simdi 'Tensor', 'GradientTape', ve 'Variable' gibi ogrendigimiz kavramlari bir araya getirip basit bir modeli olusturalim ve egitelim. Bu genel olarak birkac asamadan olusur:\n", - "\n", - "1. Modeli tanimla.\n", - "2. Kayip fonksiyonunu tanimla.\n", - "3. Egitim verilerini topla.\n", - "4. Egitim verilerini incele ve \"optimizer\" kullanarak degiskenleri veriye uydur.\n", - "\n", - "Bu egitimde, onemsiz bir ornek olan `f(x) = x * W + b` dogrusal modelinin ustunden gececegiz. Bu modelde iki degisken bulunmaktadir - `W` ve `b`. Dahasi, veriyi oyle olusturacagiz ki iyi egitilmis bir model `W = 3.0` ve `b = 2.0` degerlerine sahip olacak." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFzH64Jn9PIm" - }, - "source": [ - "### Modeli tanimlayalim\n", - "\n", - "Degiskenleri ve hesaplamalari kapsayacak basit bir sinif tanimlayalim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_WRu7Pze7wk8", - "colab": {} - }, - "source": [ - "class Model(object):\n", - " def __init__(self):\n", - " # Degiskeni (5.0, 0.0) degerlerine ilkle\n", - " # Normalde bu degerler rasgele verilir.\n", - " self.W = tf.Variable(5.0)\n", - " self.b = tf.Variable(0.0)\n", - "\n", - " def __call__(self, x):\n", - " return self.W * x + self.b\n", - "\n", - "model = Model()\n", - "\n", - "assert model(3.0).numpy() == 15.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xa6j_yXa-j79" - }, - "source": [ - "### Kayip fonksiyonunu tanimlayalim\n", - "\n", - "Kayip fonksiyonu belirli bir girdiye gore beklenen cikti degeri ile modelin sonuclarinin ne kadar benzedigini hesaplar. Burada standart L2 kaybini kullanalim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y0ysUFGY924U", - "colab": {} - }, - "source": [ - "def loss(predicted_y, desired_y):\n", - " return tf.reduce_mean(tf.square(predicted_y - desired_y))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qutT_fkl_CBc" - }, - "source": [ - "### Egitim verisini toplayalim\n", - "\n", - "Biraz gurultulu bir egitim verisi sentezleyelim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gxPTb-kt_N5m", - "colab": {} - }, - "source": [ - "TRUE_W = 3.0\n", - "TRUE_b = 2.0\n", - "NUM_EXAMPLES = 1000\n", - "\n", - "inputs = tf.random_normal(shape=[NUM_EXAMPLES])\n", - "noise = tf.random_normal(shape=[NUM_EXAMPLES])\n", - "outputs = inputs * TRUE_W + TRUE_b + noise" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-50nq-wPBsAW" - }, - "source": [ - "Modelimizi egitmeye baslamadan once modelimizin durumuna bir bakalim. Modelin tahminlarini kirmizi ile egittim verisini mavi ile cizecegiz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_eb83LtrB4nt", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(inputs, outputs, c='b')\n", - "plt.scatter(inputs, model(inputs), c='r')\n", - "plt.show()\n", - "\n", - "print('Current loss: '),\n", - "print(loss(model(inputs), outputs).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sSDP-yeq_4jE" - }, - "source": [ - "### Egitim dongusunu tanimlayalim\n", - "\n", - "Simdi agimiz ve egitim verimiz hazir. Haydi modelimizi egitelim. Egitim verilerini kullanarak modelin degiskenlerini (`W` ve `b`) guncelleyelim ki kayip fonksiyonumuzun degeri [alcalan egim](https://en.wikipedia.org/wiki/Gradient_descent) ile azalsin. `tf.train.Optimizer` gerceklemesi alcalan egimin bircok degisimin yakalar. Bu nedenle bu gerceklemeleri kullanmanizi siddetle tavsiye ederiz. Fakat burada ilk ilkeleri kullanarak olusturdugumuz model icin bu basit matematik islemlerini kendimiz gercekleyecegiz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MBIACgdnA55X", - "colab": {} - }, - "source": [ - "def train(model, inputs, outputs, learning_rate):\n", - " with tf.GradientTape() as t:\n", - " current_loss = loss(model(inputs), outputs)\n", - " dW, db = t.gradient(current_loss, [model.W, model.b])\n", - " model.W.assign_sub(learning_rate * dW)\n", - " model.b.assign_sub(learning_rate * db)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RwWPaJryD2aN" - }, - "source": [ - "Son olarak, egitim verisinden tekrar tekrar gecerek 'W' ve 'b' degerlerinin nasil evrimlestigini gorelim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XdfkR223D9dW", - "colab": {} - }, - "source": [ - "model = Model()\n", - "\n", - "# W ve b degerlerini daha sonra bir grafikte gostermek icin toplayalim\n", - "Ws, bs = [], []\n", - "epochs = range(10)\n", - "for epoch in epochs:\n", - " Ws.append(model.W.numpy())\n", - " bs.append(model.b.numpy())\n", - " current_loss = loss(model(inputs), outputs)\n", - "\n", - " train(model, inputs, outputs, learning_rate=0.1)\n", - " print('Epoch %2d: W=%1.2f b=%1.2f, loss=%2.5f' %\n", - " (epoch, Ws[-1], bs[-1], current_loss))\n", - "\n", - "# Simdi bunlari cizelim\n", - "plt.plot(epochs, Ws, 'r',\n", - " epochs, bs, 'b')\n", - "plt.plot([TRUE_W] * len(epochs), 'r--',\n", - " [TRUE_b] * len(epochs), 'b--')\n", - "plt.legend(['W', 'b', 'true W', 'true_b'])\n", - "plt.show()\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vPnIVuaSJwWz" - }, - "source": [ - "## Siradaki adimlar\n", - "\n", - "Bu egitim kitapciginda `Variable(Degisken)`leri gozden gecirdik ve TensorFlow ilkellerini kullanarak basit dogrusal bir model egittik.\n", - "\n", - "Teoride bunlar TensorFlow'u makine ogrenmesi arastirmalarinda kullanmanizda yeterlidir.\n", - "Pratikte, ozellikle sinir aglari icin, yuksek seviye yapi taslari (\"layers(katmanlar)\"), durumu(state) kaydetme ve geri yukleme imkanlari, cok sayida kayip fonksiyonu ve eniyileme taktikleri sundugu icin `tf.keras` gibi yuksek seviye API kullanimi cok daha kolay olacaktir.\n" - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/eager/custom_training_walkthrough.ipynb b/site/tr/r1/tutorials/eager/custom_training_walkthrough.ipynb deleted file mode 100644 index 784cfdb8166..00000000000 --- a/site/tr/r1/tutorials/eager/custom_training_walkthrough.ipynb +++ /dev/null @@ -1,1092 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training_walkthrough", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rwxGnsA92emp" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "CPII1rGR2rF9", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtEZ1pCPn--z" - }, - "source": [ - "# Kisisellestirilmis egitim: adim adim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GV1F7tVTN3Dn" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LDrzLFXE8T1l" - }, - "source": [ - "Bu egitim kitapciginda makine ogrenmesini kullanarak zambak ciceklerini turlerine gore *siniflandirmayi* ogrenecegiz. Bunun icin TensorFlow'un [eager modunu](https://www.tensorflow.org/r1/guide/eager) kullanacagiz:\n", - "1. Modeli olustur,\n", - "2. Ornek veri ustunde modeli egit ve\n", - "3. Bilinmeyen veri hakkinda tahmin yurutmek icin modeli kullan.\n", - "\n", - "## TensorFlow yazilimi\n", - "\n", - "Burada yuksek-seviye TensorFlow kapsamlarini kullanacagiz:\n", - "\n", - "* [Eager modu](https://www.tensorflow.org/r1/guide/eager) gelistirme ortamini aktive etmek,\n", - "* [Datasets API](https://www.tensorflow.org/r1/guide/datasets)'si ile verileri iceri almak,\n", - "* TensorFlow [Keras API](https://keras.io/getting-started/sequential-model-guide/)'si ile modelleri ve katmanlari olusturmak.\n", - "\n", - "Bu kitapcigi bircok TensorFlow programi gibi planladik:\n", - "\n", - "1. Veri setlerini iceri al ve ayristir.\n", - "2. Modeli sec.\n", - "3. Modeli egit.\n", - "4. Modelin verimliligini degerlendir\n", - "5. Egitilen modeli tahmin yurutmek icin kullan." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yNr7H-AIoLOR" - }, - "source": [ - "## Programi kuralim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1J3AuPBT9gyR" - }, - "source": [ - "### Iceri alimlari ve eager modunu yapilandiralim\n", - "\n", - "TensorFlow'da dahil olmak uzere gerekli Python birimlerini iceri alalim ve eager modunu aktiflestirelim. Eager modu sayesinde TensorFlow operasyonlari aninda degerlendirir ve daha sonra yurutulecek bir [computational graph(Hesap grafigi)](https://www.tensorflow.org/r1/guide/graphs) olusturmak yerine somut degerler cikartir. Bu REPL ya da 'python'un etkilesimli konsoluna cok benzer. Eager modu [Tensorlow >=1.8](https://www.tensorflow.org/install/)'da bulunmaktadir.\n", - "\n", - "Eager modu bir kere aktive edildiginde ayni program icinde kapatilamaz. Detaylar icin [eager modu rehberine](https://www.tensorflow.org/r1/guide/eager) bakiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "g4Wzg69bnwK2", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "import matplotlib.pyplot as plt\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()\n", - "\n", - "print(\"TensorFlow version: {}\".format(tf.__version__))\n", - "print(\"Eager execution: {}\".format(tf.executing_eagerly()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zx7wc0LuuxaJ" - }, - "source": [ - "## Zambak siniflandirmasi problemi\n", - "\n", - "Buldugunuz her zambak cicegini siniflandirmayi amaclayan bir botanikci oldugunuzu hayal edin. Makine ogrenmesi bunu istatistiksel olarak yapmanizi saglayan algoritmalar saglar. Ornegin, iyi bir makine ogrenmesi programi cicekleri resimlerine bakarak siniflandirabilir. Biz burada daha mutevazi bir yol izleyip zambaklari [canak yapraklarinin](https://en.wikipedia.org/wiki/Sepal) ve [tac yapraklarinin](https://en.wikipedia.org/wiki/Petal) uzunluguna ve capina gore siniflandiracagiz.\n", - "\n", - "300'den fazla zambak turu bulunmasina ragmen programimiz bunlardan ucunu siniflandiracak:\n", - "\n", - "* Iris setosa\n", - "* Iris virginica\n", - "* Iris versicolor\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Petal\n", - "
    \n", - " Figure 1. Iris setosa (by Radomil, CC BY-SA 3.0), Iris versicolor, (by Dlanglois, CC BY-SA 3.0), and Iris virginica (by Frank Mayfield, CC BY-SA 2.0).
     \n", - "
    \n", - "\n", - "Sansimiza bir [120 zambak cicegi veri seti](https://en.wikipedia.org/wiki/Iris_flower_data_set) tac ve canak yapragi olculeri ile olusturulmus. Bu makine ogrenmesi siniflandirma problemine acemi olanlar icin populer bir veri setidir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Px6KAg0Jowz" - }, - "source": [ - "## Egitim veri setini iceri al ve ayristir\n", - "\n", - "Veri seti dosyasini indirelim ve bu Python programi tarafindan kullanilabilecek yapiya cevirelim.\n", - "\n", - "### Veri setini indirelim\n", - "\n", - "[tf.keras.utils.get_file](https://www.tensorflow.org/api_docs/python/tf/keras/utils/get_file) fonksiyonunu kullanarak egitim veri setini indirelim. Bu yontem indirilen dosyanin dosya yolunu geri dondurur." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J6c7uEU9rjRM", - "colab": {} - }, - "source": [ - "train_dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv\"\n", - "\n", - "train_dataset_fp = tf.keras.utils.get_file(fname=os.path.basename(train_dataset_url),\n", - " origin=train_dataset_url)\n", - "\n", - "print(\"Local copy of the dataset file: {}\".format(train_dataset_fp))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qnX1-aLors4S" - }, - "source": [ - "### Veriyi inceleyelim\n", - "\n", - "Bu veri seti, `iris_training.csv`, virgulle ayrilmis degerlerin (CSV) bulundugu basit bir metin dosyasidir. `head -n5` komutunu kullanarak ilk bes girise goz atabiliriz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FQvb_JYdrpPm", - "colab": {} - }, - "source": [ - "!head -n5 {train_dataset_fp}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kQhzD6P-uBoq" - }, - "source": [ - "Veri setine bu acidan baktigimizda sunlara dikkat edelim:\n", - "\n", - "1. Ilk satir veri seti ile ilgili bilgi iceren basliktir:\n", - " * Toplamda 120 ornek bulunur. Her ornekte dort ozellik ve olasi uc etiket isminden birisi vardir.\n", - "2. Sonra gelen satirlar veri kayitlarini gosterir, *[ornegin](https://developers.google.com/machine-learning/glossary/#example)* burada:\n", - " * Ilk dort alan *[ozelliklerdir](https://developers.google.com/machine-learning/glossary/#feature)*. Burada, alanlarda bulunan reel sayilar cicegin olculerini temsil eder.\n", - " * Son sutun ise *[etiket](https://developers.google.com/machine-learning/glossary/#label)* yani tahmin etmek istedigimiz degerdir. Bu veri setinde 0, 1 ve 2 sayilari bir cicek adina karsilik gelir.\n", - "\n", - "Simdi bu kodu yazalim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9Edhevw7exl6", - "colab": {} - }, - "source": [ - "# CSV dosyasindaki sutun sirasi\n", - "column_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']\n", - "\n", - "feature_names = column_names[:-1]\n", - "label_name = column_names[-1]\n", - "\n", - "print(\"Features: {}\".format(feature_names))\n", - "print(\"Label: {}\".format(label_name))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCtwLoJhhDNc" - }, - "source": [ - "Her etiket bir dizgi adiyla baglantilidir (ornegin, \"setosa\"), ama makine ogrenmesi genellikler sayilarla calisir. Etiket sayilari su sekilde isimlerle eslestirilebilir:\n", - "\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica\n", - "\n", - "Ozellikler ve etiketlerle ilgili daha fazla bilgi almak istiyorsaniz [buraya bakiniz](https://developers.google.com/machine-learning/crash-course/framing/ml-terminology)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sVNlJlUOhkoX", - "colab": {} - }, - "source": [ - "class_names = ['Iris setosa', 'Iris versicolor', 'Iris virginica']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dqPkQExM2Pwt" - }, - "source": [ - "### `tf.data.Dataset`i olusturalim\n", - "\n", - "TensorFlow'un [Dataset API](https://www.tensorflow.org/r1/guide/datasets)'si veriyi modele yuklerken gorulen bircok durumu halledebilir. Veriyi okumak ve egitim icin uygun sekle sokmak icin yuksek-seviye bir API'dir. Detayli bilgi icin [buraya bakiniz](https://www.tensorflow.org/get_started/datasets_quickstart).\n", - "\n", - "\n", - "CSV bicimli metin dosyasi olarak elimizde olan veri setini [make_csv_dataset](https://www.tensorflow.org/api_docs/python/tf/contrib/data/make_csv_dataset) fonksiyonunu kullanarak veriyi uygun sekle sokabiliriz. Bu fonksiyon egitim modelleri icin veri urettigi icin, varsayilan davranis verinin karistirilip (`shuffle=True, shuffle_buffer_size=10000`), veri setinin sonsuza kadar tekrarlanmasidir (`num_epochs=None`). Burada ayrica [batch_size](https://developers.google.com/machine-learning/glossary/#batch_size) degistirgesini de belirtiyoruz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WsxHnz1ebJ2S", - "colab": {} - }, - "source": [ - "batch_size = 32\n", - "\n", - "train_dataset = tf.contrib.data.make_csv_dataset(\n", - " train_dataset_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name=label_name,\n", - " num_epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gB_RSn62c-3G" - }, - "source": [ - "`make_csv_dataset` fonksiyonu `(features, label)` ikilisinden olusan bir `tf.data.Dataset` dondurur. Burada `features` bir sozluktur: `{'feature_name': deger}`\n", - "\n", - "Eager modu aktive edildiginde bu 'Dataset' nesnesi yinelenebilir. Simdi ozelliklere goz atalim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iDuG94H-C122", - "colab": {} - }, - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "features" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E63mArnQaAGz" - }, - "source": [ - "Benzer ozelliklerin bir araya getirildigine yani *batched* edildigine dikkat edelim. Her ornegin sutunlari eslestigi ozellik dizisine eklenir. `batch_size` degerini bu ozellik dizilerinde bulunan ornek sayisina esleyelim.\n", - "\n", - "Bu 'batch'ten birkac ozelligi grafik halinde gosterirseniz gruplasmayi gorebilirsiniz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "me5Wn-9FcyyO", - "colab": {} - }, - "source": [ - "plt.scatter(features['petal_length'].numpy(),\n", - " features['sepal_length'].numpy(),\n", - " c=labels.numpy(),\n", - " cmap='viridis')\n", - "\n", - "plt.xlabel(\"Petal length\")\n", - "plt.ylabel(\"Sepal length\")\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YlxpSyHlhT6M" - }, - "source": [ - "Model olustuma basamagini kolaylastirmak icin, ozellik sozlugunu `(batch_size, num_features)` sekli olan bir diziye cevirecek bir fonksiyon yazalim.\n", - "\n", - "Bu fonksiyon tensor dizisindeki degerleri alarak belirli boyuttaki birlestirilmis tensoru olusturan [tf.stack](https://www.tensorflow.org/api_docs/python/tf/stack) yontemini kullanir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jm932WINcaGU", - "colab": {} - }, - "source": [ - "def pack_features_vector(features, labels):\n", - " \"\"\"Ozellikleri tek bir diziye paketleyelimPack the features into a single array.\"\"\"\n", - " features = tf.stack(list(features.values()), axis=1)\n", - " return features, labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "V1Vuph_eDl8x" - }, - "source": [ - "Sonrasinda [tf.data.Dataset.map](https://www.tensorflow.org/api_docs/python/tf/data/dataset/map) yontemi ile her `(features,label)` ikisindeki `features`i egitim veri setine paketleyelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZbDkzGZIkpXf", - "colab": {} - }, - "source": [ - "train_dataset = train_dataset.map(pack_features_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NLy0Q1xCldVO" - }, - "source": [ - "`Dataset` icindeki ozellik elemani simdi `(batch_size, num_features)` sekli olan dizilerdir. Ilk birkac ornege bakalim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kex9ibEek6Tr", - "colab": {} - }, - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "print(features[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LsaVrtNM3Tx5" - }, - "source": [ - "## Modelimizi secelim\n", - "\n", - "### Neden Model?\n", - "\n", - "*[Model](https://developers.google.com/machine-learning/crash-course/glossary#model)* ozelliklerle etiketler arasindaki etkilesimdir. Zambak siniflandirmasi probleminde ise modelimiz tac yaprak ve canak yaprak olculeri ile tahmin edilen Zambak turleri arasindaki iliskiyi belirler. Bazi basit modeller birkac satir matematiksel islemle aciklanabilirken kompleks makine ogrenmesi modellerini aciklamak cok sayida degistirgenin bulunmasi sebebiyle cok zordur. \n", - "\n", - "Makine ogrenmesini kullanmadan Zambak turleri ve ozellikleri arasindaki iliskiyi kurabilir miydiniz? Yani kosullu deyimler gibi geleneksel programlama tekniklerini kullanarak bir model olusturabilir miydiniz? Eger verileri yeteri kadar inceleyip elimizdeki olculerle Zambak turleri arasindaki iliskiyi bulabilirseniz belki modeli olusturabilirsiniz. Bu karisik veri setlerinde neredeyse imkansiz hale gelir. Iyi bir makine ogrenmesi *modelinizi sizin yerinize olusturur*. Eger dogru makine ogrenmesi modeline dogru ornekleri verebilirseniz program sizin yerinize bu iliskiyi belirler.\n", - "\n", - "### Modelimizi secelim\n", - "\n", - "Simdi egitmek icin modelimizi secmeliyiz. Bircok model cesidi bulunur ve dogru modeli secmek deneyim gerektirir. Bu egitim sinir aglarini kullanarak Zambak siniflandirmasi problemini cozer. *[Sinir aglari](https://developers.google.com/machine-learning/glossary/#neural_network)* ozelliklerle etiketler arasindaki kompleks iliskiyi bulabilir. Bir ya da birden fazla *[sakli katmanlar (hidden layers)](https://developers.google.com/machine-learning/glossary/#hidden_layer)* seklinde organize olmus iyi yapilandirilmis bir grafiktir. Her sakli katman bir ya da birden fazla *[sinirden](https://developers.google.com/machine-learning/glossary/#neuron)* olusmustur. Bircok sinir agi kategorisi vardir ve bu program yogun (dense), ya da *[tamamen birbirine bagli sinir agini (fully-connected neural network)](https://developers.google.com/machine-learning/glossary/#fully_connected_layer)* kullanir: bir katmandaki sinirler bir onceki katmandaki *butun* sinirlerden girdi baglantilarini alir. Asagidaki Sekil 2'de bir girdi katmani, iki sakli katman ve bir cikti katmanindan olusan yogun sinir aginin bir ornegini gorebilirsiniz:\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \n", - "
    \n", - " Sekil 2. Ozellikleri, sakli katmanlari ve tahminleri olan bir sinir agi.
     \n", - "
    \n", - "\n", - "Sekildeki modeli egitip etiketlenmemis bir ornek verdigimizde bize bu ornegin Zambak turlerinden birisi olup olmadigina yonelik bir tahmin yurutur. Bu tahmine *[cikarsama (inference)](https://developers.google.com/machine-learning/crash-course/glossary#inference)* adi verilir. Bu ornekteki ciktilarin degerlerinin toplami 1.0 idir. Sekildeki tahminler ise soyledir: *Iris setosa* icin `0.02`, *Iris versicolor* icin `0.95` ve *Iris virginica* icin `0.03`. Bu modelimizin yuzde 95 olasikla etiketsiz ornegi *Iris versicolor* olarak tahmin ettigi anlamina gelir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W23DIMVPQEBt" - }, - "source": [ - "### Keras kullanarak modelimizi olusturalim \n", - "\n", - "TensorFlow [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras) API'si modelleri ve katmanlari olusturmak icin tavsiye edilen yontemdir. Keras her seyin birbirine baglanmasini hallederken sizin modeli olusturmaniz ve test etmeniz kolaylasir. \n", - "\n", - "[tf.keras.Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential) modeli dogrusal katman yiginidir. Yapicisi katman ornegi listesini alir. Ornegimizde bu liste her biri 10 dugumden olusan 2 [Yogun (Dense)](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense) katman ve etiket tahminlerini temsil eden 3 dugumden olusan bir cikti katmanini kapsar. Ilk katmanin gereken degistirgesi `input_shape`, veri setindeki ozelliklere denk gelir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2fZ6oL2ig3ZK", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(4,)), # girdi sekli gereklidir \n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu),\n", - " tf.keras.layers.Dense(3)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FHcbEzMpxbHL" - }, - "source": [ - "*[Aktiflestirme fonksiyonu (activation function)](https://developers.google.com/machine-learning/crash-course/glossary#activation_function)* katamnada bulunan her dugumun cikti seklini belirler. Bu tur dogrusal olmayan iliskiler onemlidir cunku onlar olmadan modelimiz tek katmanli gibi davranirdi. Bircok [aktiflestirme turu](https://www.tensorflow.org/api_docs/python/tf/keras/activations) bulunur, fakat gizli tabakalar icin [ReLU](https://developers.google.com/machine-learning/crash-course/glossary#ReLU) yaygin olarak kullanilir.\n", - "\n", - "Sakli katmanlarin ve sinirlerin sayisi problemin turune ve veri setine gore degisir. Makine ogrenmesinin butun alanlarinda oldugu gibi bunu yapmak biraz deneyim, biraz bilgi ve biraz test etme gerektirir. Genel bir kural koymamiz gerekirse sakli katman ve sinir sayisi arttikca modelimiz genelde daha guclenir, fakat boyle bir modeli verimli sekilde egitmek icin daha cok veriye ihtiyacimiz vardir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2wFKnhWCpDSS" - }, - "source": [ - "### Modeli kullanalim\n", - "\n", - "Simdi modelimizin elimizdeki ozelliklere ne yaptigina hizlica goz atalim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xe6SQ5NrpB-I", - "colab": {} - }, - "source": [ - "predictions = model(features)\n", - "predictions[:5]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wxyXOhwVr5S3" - }, - "source": [ - "Buradaki her ornek her sinif icin bir [logit](https://developers.google.com/machine-learning/crash-course/glossary#logits) dondurur.\n", - "\n", - "Her sinif icin bu logitleri olasiliga donusturmek icin [softmax](https://developers.google.com/machine-learning/crash-course/glossary#softmax) fonksiyonunu kullanalim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_tRwHZmTNTX2", - "colab": {} - }, - "source": [ - "tf.nn.softmax(predictions[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uRZmchElo481" - }, - "source": [ - "`tf.argmax`i siniflar arasindan gecirerek tahmin edilen sinifin dizinini bulabiliriz. Fakat, modelimiz daha egitilmedigi icin tahminler iyi degildir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-Jzm_GoErz8B", - "colab": {} - }, - "source": [ - "print(\"Prediction: {}\".format(tf.argmax(predictions, axis=1)))\n", - "print(\" Labels: {}\".format(labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vzq2E5J2QMtw" - }, - "source": [ - "## Modeli egitelim\n", - "\n", - "*[Egitim](https://developers.google.com/machine-learning/crash-course/glossary#training)* makine ogrenmesi modelinin derece derece eniyilestirilmesi ya da modelin veri setini *ogrenmesi* asamasidir. Hedef egitim veri setinin yapisini iyice ogrenerek bilinmeyen veri hakkinda tahmin yapabilmektir. Eger egitim setini *cok iyi* ogrenirse tahminler sadece bu veri setinde calisacak ve genelestirilemez olacaktir. Bu probleme *[asiri uyum (overfitting)](https://developers.google.com/machine-learning/crash-course/glossary#overfitting)* adi verilir—bu tipki sorunun cozumunu anlamak yerine cevaplari ezberlemeye benzer.\n", - "\n", - "Zambak siniflandirmasi problemi *[gozetimli makine ogrenmesi](https://developers.google.com/machine-learning/glossary/#supervised_machine_learning)* ornegidir: model etiketleri olan orneklerle egitilir. *[Gozetimsiz makine ogrenmesinde](https://developers.google.com/machine-learning/glossary/#unsupervised_machine_learning)* ise, orneklerin etiketleri yoktur. Etiket yerine model ozellikler arasindaki deseni bulmaya calisir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RaKp8aEjKX6B" - }, - "source": [ - "### Kayip ve egim fonksiyonlarini tanimlayalim\n", - "\n", - "Egitim ve degerlendirme asamalari modelin *[kaybini](https://developers.google.com/machine-learning/crash-course/glossary#loss)* hesaplamalidir. Bu modelin tahminlerinin istenen etiketlerle farkini yani modelin ne kadar kotu calistigini hesaplar. Bu degeri kucultmeyi ve eniyilestirmeyi isteriz.\n", - "\n", - "Modelimiz [tf.keras.losses.categorical_crossentropy](https://www.tensorflow.org/api_docs/python/tf/losses/sparse_softmax_cross_entropy) fonksiyonu ile kaybi hesaplar. Bu fonksiyon modelin sinif tahminlerini ve istenen etiketleri alir ve butun orneklerin kayip degerlerinin ortalamasini dondurur." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tMAT4DcMPwI-", - "colab": {} - }, - "source": [ - "def loss(model, x, y):\n", - " y_ = model(x)\n", - " return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=y_)\n", - "\n", - "\n", - "l = loss(model, features, labels)\n", - "print(\"Loss test: {}\".format(l))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3IcPqA24QM6B" - }, - "source": [ - "[tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) kapsamini kullanarak modelimizi eniyilestirecek olan *[egimleri](https://developers.google.com/machine-learning/crash-course/glossary#gradient)* hesaplayabiliriz. Bu konuda daha cok ornek icin eager modu kitapcigina [buradan bakabilirsiniz](https://www.tensorflow.org/r1/guide/eager)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x57HcKWhKkei", - "colab": {} - }, - "source": [ - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return loss_value, tape.gradient(loss_value, model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lOxFimtlKruu" - }, - "source": [ - "### Eniyilestirici (optimizer) olusturalim\n", - "\n", - "*[Eniyilestirici](https://developers.google.com/machine-learning/crash-course/glossary#optimizer)* kayip fonkiyonunu en aza indirgemek icin hesaplanan egim degerlerini modelin degiskenlerine uygular. Kayip fonksiyonunu kivrimli bir yuzey olarak dusunebilirsini (Sekil 3) ve biz bu seklin en dusuk noktasini uzerinden gecerek bulmak istiyoruz. Egimler bizi en dik tirmanislara yonlendirir, bu yuzden biz ters yone dogru giderek tepeden asagiya hareket edecegiz. Her grup (batch) icin kayip ve egim degerlerini etkilesimli olarak hesaplayip egitim sirasinda modelimizi ayarlayacagiz. Modelimiz kaybi dusurmek icin yavas yavas agirliklar ve sapmalarin en iyi birlesimini bulacaktir. Kayip degeri ne kadar dusuk olursa modelimizin tahminleri o kadar iyi olur.\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Optimization\n", - "
    \n", - " Sekil 3. Eniyilestirme algoritmalarinin zamanla 3D duzlemde goruntulenmesi.
    (Kaynak: Stanford class CS231n, MIT License, Goruntu: Alec Radford)\n", - "
    \n", - "\n", - "TensorFlow'un egitim icin bircok [eniyilestirme algoritmasi](https://www.tensorflow.org/api_guides/python/train) bulunmaktadir. Burada *[stochastic gradient descent](https://developers.google.com/machine-learning/crash-course/glossary#gradient_descent)* (SGD) algoritmasini gerceklestiren [tf.train.GradientDescentOptimizer](https://www.tensorflow.org/api_docs/python/tf/train/GradientDescentOptimizer) kullaniyoruz. `learning_rate` degeri her yinelemede basamak degerini asagi ceker. Bu daha iyi sonuc almak icin degistireceginiz bir *hyperparameter* idir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XkUd6UiZa_dF" - }, - "source": [ - "Eniyilestiriciyi (optimizer) ve `global_step` sayacini kuralim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8xxi2NNGKwG_", - "colab": {} - }, - "source": [ - "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)\n", - "\n", - "global_step = tf.Variable(0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pJVRZ0hP52ZB" - }, - "source": [ - "Bunu bir eniyilestirme basamagini hesaplamak icin kullanacagiz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rxRNTFVe56RG", - "colab": {} - }, - "source": [ - "loss_value, grads = grad(model, features, labels)\n", - "\n", - "print(\"Step: {}, Initial Loss: {}\".format(global_step.numpy(),\n", - " loss_value.numpy()))\n", - "\n", - "optimizer.apply_gradients(zip(grads, model.trainable_variables), global_step)\n", - "\n", - "print(\"Step: {}, Loss: {}\".format(global_step.numpy(),\n", - " loss(model, features, labels).numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Y2VSELvwAvW" - }, - "source": [ - "### Egitim dongusu\n", - "\n", - "Butun parcalar yerine geldigine gore modelimizi egitebiliriz! Egitim dongusu veri setindeki ornekleri modelimize vererek daha iyi tahminde bulunmasini saglar. Asagida egitim basamaklarini gorebilirsiniz:\n", - "\n", - "1. Her *devri(epoch)* yineleyelim. Bir devir veri setinin ustunden bir kere gecmek anlamina gelir.\n", - "2. Her devirde egitim veri setindeki (Dataset) ornekleri tek tek yineleyerek *ozelliklerini* (`x`) ve *etiketlerini* (`y`) yakalayalim.\n", - "3. Ornegin ozelliklerini kullanarak bir tahminde bulunalim ve bunu etiket ile karsilastiralim. Tahminin hatasini hesaplayalim ve bu degeri modelin kaybini ve egimini bulmak icin kullanalim.\n", - "4. Bir `eniyiseltirici (optimizer)` kullanarak modelin degiskenlerini guncelleyelim.\n", - "5. Bazi istatiksel degerlerini daha sonra goruntulemek icin takip edelim.\n", - "6. Her devir icin tekrarlayalim.\n", - "\n", - "`num_epochs` degiskeni veri setinin ustunden kac dongu gececegini belirler. Bu icgudulerimize ters gelse de modeli daha uzun egitmek daha iyi bir model elde edecegimiz anlamina gelmez. `num_epochs` ayarlacabileceginiz bir *[hyperparameter](https://developers.google.com/machine-learning/glossary/#hyperparameter)*'dir. Dogru devir sayisini secmek hem deneyim hem de test etmeyi gerektirir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AIgulGRUhpto", - "colab": {} - }, - "source": [ - "## Not: Bu hucreyi tekrar calistirinca ayni model degiskenleri kullanilir \n", - "\n", - "from tensorflow import contrib\n", - "tfe = contrib.eager\n", - "\n", - "# sonuclari ileride goruntulemek icin tutalim\n", - "train_loss_results = []\n", - "train_accuracy_results = []\n", - "\n", - "num_epochs = 201\n", - "\n", - "for epoch in range(num_epochs):\n", - " epoch_loss_avg = tfe.metrics.Mean()\n", - " epoch_accuracy = tfe.metrics.Accuracy()\n", - "\n", - " # Egitim dongusu - 32ser topluluk kullanarak \n", - " for x, y in train_dataset:\n", - " # Modeli eniyilestireliml\n", - " loss_value, grads = grad(model, x, y)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables),\n", - " global_step)\n", - "\n", - " # Ilerlemeyi takip edelim \n", - " epoch_loss_avg(loss_value) # su anki topluluk kaybini ekleyelim\n", - " # tahmin edilen etiketle asil etiketi karsilastir\n", - " epoch_accuracy(tf.argmax(model(x), axis=1, output_type=tf.int32), y)\n", - "\n", - " # devri bitirelim\n", - " train_loss_results.append(epoch_loss_avg.result())\n", - " train_accuracy_results.append(epoch_accuracy.result())\n", - "\n", - " if epoch % 50 == 0:\n", - " print(\"Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}\".format(epoch,\n", - " epoch_loss_avg.result(),\n", - " epoch_accuracy.result()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2FQHVUnm_rjw" - }, - "source": [ - "### Kayip fonksiyonunun zaman icinde degisimi goruntuleyelim " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j3wdbmtLVTyr" - }, - "source": [ - "Modelin egitim surecini yazdirmak yardimci olsa da bu sureci gorebilmek *daha* da faydalidir. [TensorBoard](https://www.tensorflow.org/r1/guide/summaries_and_tensorboard), TensorFlow ile gelen guzel bir goruntuleme aracidir, fakat basit cizimleri `matplotlib` birimi ile de yapabiliriz.\n", - "\n", - "Bu cizimleri yorumlamak biraz deneyim gerektirir, fakat asil gormek istedigimiz sey *kayip* degerinin dusmesi ve *dogruluk* degerinin yukselmesidir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "agjvNd2iUGFn", - "colab": {} - }, - "source": [ - "fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))\n", - "fig.suptitle('Training Metrics')\n", - "\n", - "axes[0].set_ylabel(\"Loss\", fontsize=14)\n", - "axes[0].plot(train_loss_results)\n", - "\n", - "axes[1].set_ylabel(\"Accuracy\", fontsize=14)\n", - "axes[1].set_xlabel(\"Epoch\", fontsize=14)\n", - "axes[1].plot(train_accuracy_results)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zg8GoMZhLpGH" - }, - "source": [ - "## Modelin ne kadar etkili oldugunu degerlendirelim\n", - "\n", - "Modelimizi egittigimize gore basarimi ustune istatistikleri alabiliriz.\n", - "\n", - "*Degerlendirme* modelin tahminlerini ne kadar etkili yaptiginin belirlenmesi anlamina gelir. Zambak siniflandirmasinda modelin etkililigini belirlemek icin birkac canak yaprak ve tac yaprak olculerini modelimize verelim ve modelimize bunlarin hangi ture ait oldugunu tahmin etmesini soralim. Sonra modelin tahmini ile asil etiketi karsilastiralim. Ornegin, girdinin yarisinda dogru tahminde bulunan bir modelin *[dogruluk degeri](https://developers.google.com/machine-learning/glossary/#accuracy)* `0.5` idir. Sekil 4'te 5 tahminden 4'unu tutturarak yuzde 80 dogruluk degerine sahip biraz daha etkili bir modeli gorebilirsiniz:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    Example featuresLabelModel prediction
    5.93.04.31.511
    6.93.15.42.122
    5.13.31.70.500
    6.0 3.4 4.5 1.6 12
    5.52.54.01.311
    \n", - " Sekil 4. Yuzde 80 dogruluk degeri olan bir Zambak siniflandiricisi.
     \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z-EvK7hGL0d8" - }, - "source": [ - "### Test veri setini olusturalim\n", - "\n", - "Modelin degerlendirilmesi modelin egitilmesine benzer. En buyuk fark orneklerin egitim setinden farkli bir *[test setinden](https://developers.google.com/machine-learning/crash-course/glossary#test_set)* gelmesidir. Modelin etkililigini adil sekilde degerlendirebilmek icin test orneklerinin egitim orneklerinden farkli olmasi gerekmektedir.\n", - "\n", - "Test `Dataset` ile egitim `Dataset`nin kurulumu birbirine benzer. CSV metin dosyasini indirelim ve degerleri ayristiralim, sonra da siralarini karistiralim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ps3_9dJ3Lodk", - "colab": {} - }, - "source": [ - "test_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv\"\n", - "\n", - "test_fp = tf.keras.utils.get_file(fname=os.path.basename(test_url),\n", - " origin=test_url)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SRMWCu30bnxH", - "colab": {} - }, - "source": [ - "test_dataset = tf.contrib.data.make_csv_dataset(\n", - " test_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name='species',\n", - " num_epochs=1,\n", - " shuffle=False)\n", - "\n", - "test_dataset = test_dataset.map(pack_features_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HFuOKXJdMAdm" - }, - "source": [ - "### Modeli test veri setinde degerlendirelim\n", - "\n", - "Egitim asamasindan farkli olarak, model test verisinin sadece tek bir [devrini](https://developers.google.com/machine-learning/glossary/#epoch) degerlendirir. Asagidaki kod hucresinde test setindeki her ornegi yineleyerek modelin tahmini ile asil etiketi karsilastiriyoruz. Boylece modelin dogrulugunu butun test setinde olcmus olacagiz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tw03-MK1cYId", - "colab": {} - }, - "source": [ - "test_accuracy = tfe.metrics.Accuracy()\n", - "\n", - "for (x, y) in test_dataset:\n", - " logits = model(x)\n", - " prediction = tf.argmax(logits, axis=1, output_type=tf.int32)\n", - " test_accuracy(prediction, y)\n", - "\n", - "print(\"Test set accuracy: {:.3%}\".format(test_accuracy.result()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HcKEZMtCOeK-" - }, - "source": [ - "Ornegin son topluluktaki degere bakarak modelin genelde dogru oldugunu gorebiliriz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uNwt2eMeOane", - "colab": {} - }, - "source": [ - "tf.stack([y,prediction],axis=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Li2r1tYvW7S" - }, - "source": [ - "## Egitilen modeli tahminde bulunmak icin kullanalim\n", - "\n", - "Bir modeli egittik ve Zambak turlerini siniflandirma da mukemmel olmasa da iyi oldugunu \"kanitladik\". Simdi bu modeli [etiketsiz ornekler](https://developers.google.com/machine-learning/glossary/#unlabeled_example), yani ozellikleri bulunan ama etiketleri olmayan ornekler, ustunde tahmin yurutmek icin kullanalim.\n", - "\n", - "Gercek hayatta, etiketsiz ornekler aplikasyonlar, CSV metinleri, ve veri besleyiciler gibi bircok farkli kaynaktan gelebilir. Simdilik biz tahmin yurutmek icin etiketsiz ornekleri kendimiz verecegiz. Hatirlarsaniz, etiket rakamlarini su sekilde isimlerle eslemistik:\n", - "\n", - "* `0`: Iris setosa\n", - "* `1`: Iris versicolor\n", - "* `2`: Iris virginica" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kesTS5Lzv-M2", - "colab": {} - }, - "source": [ - "predict_dataset = tf.convert_to_tensor([\n", - " [5.1, 3.3, 1.7, 0.5,],\n", - " [5.9, 3.0, 4.2, 1.5,],\n", - " [6.9, 3.1, 5.4, 2.1]\n", - "])\n", - "\n", - "predictions = model(predict_dataset)\n", - "\n", - "for i, logits in enumerate(predictions):\n", - " class_idx = tf.argmax(logits).numpy()\n", - " p = tf.nn.softmax(logits)[class_idx]\n", - " name = class_names[class_idx]\n", - " print(\"Example {} prediction: {} ({:4.1f}%)\".format(i, name, 100*p))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/eager/eager_basics.ipynb b/site/tr/r1/tutorials/eager/eager_basics.ipynb deleted file mode 100644 index 5fa8c8771f9..00000000000 --- a/site/tr/r1/tutorials/eager/eager_basics.ipynb +++ /dev/null @@ -1,454 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "eager_basics.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iPpI7RaYoZuE" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "hro2InpHobKk", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U9i2Dsh-ziXr" - }, - "source": [ - "# Eager modunun ana hatlari" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Hndw-YcxoOJK" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6sILUVbHoSgH" - }, - "source": [ - "Bu kitapcikta TensorFlow kullanarak konuya giris yapacagiz. Asagidaki konulari isleyecegiz:\n", - "\n", - "* Gerekli paketleri iceri aktarma\n", - "* Tensorlari olusturma ve kullanma\n", - "* GPU hizlandirmayi kullanmak\n", - "* Veri setleri" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1JcS5iBXMRO" - }, - "source": [ - "## TensorFlow'u iceri alalim\n", - "\n", - "'tensorflow' modulunu iceri alalim ver eager modunu secelim.\n", - "Eager modu, TensorFlow'a detaylarini daha sonra aciklayacagimiz etkilesimli bir arayuz saglar." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "code", - "colab_type": "code", - "id": "RlIWhyeLoYnG", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function\n", - "\n", - "import tensorflow as tf\n", - "\n", - "tf.enable_eager_execution()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H9UySOPLXdaw" - }, - "source": [ - "## Tensorlar\n", - "\n", - "Tensor kisaca cok boyutlu bir dizidir. NumPy'deki 'ndarray' nesneleri gibi, `Tensor` nesnesinin de bir veri turu ve sekli vardir. Ayrica Tensorlar GPU gibi hizlandirilmis hafizada bulunabilirler. TensorFlow, Tensorlari olusturmak ve kullanmak icin zengin islemlere sahiptir ([tf.add](https://www.tensorflow.org/api_docs/python/tf/add), [tf.matmul](https://www.tensorflow.org/api_docs/python/tf/matmul), [tf.linalg.inv](https://www.tensorflow.org/api_docs/python/tf/linalg/inv) etc.). Bu islemler Python tiplerini otomatik olarak degistirirler. Ornegin:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "code", - "colab_type": "code", - "id": "ngUe237Wt48W", - "colab": {} - }, - "source": [ - "print(tf.add(1, 2))\n", - "print(tf.add([1, 2], [3, 4]))\n", - "print(tf.square(5))\n", - "print(tf.reduce_sum([1, 2, 3]))\n", - "print(tf.encode_base64(\"hello world\"))\n", - "\n", - "# Islec asiri yuklenmesi de desteklenir\n", - "print(tf.square(2) + tf.square(3))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IDY4WsYRhP81" - }, - "source": [ - "Her Tensor'un bir sekli ve veri turu vardir" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "srYWH1MdJNG7", - "colab": {} - }, - "source": [ - "x = tf.matmul([[1]], [[2, 3]])\n", - "print(x.shape)\n", - "print(x.dtype)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eBPw8e8vrsom" - }, - "source": [ - "NumPy dizileri ve TensorFlow Tensorlari arasindaki en belirgin farklar sunlardir:\n", - "\n", - "1. Tensorlar hizlandirilmis hafizalar tarafindan desteklenebilr (GPU, TPU gibi).\n", - "2. Tensorlar degistirilemez." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dwi1tdW3JBw6" - }, - "source": [ - "### NumPy Uyumlulugu\n", - "\n", - "TensorFlow Tensorlari ile NumPy 'ndarray'leri arasindaki donusum cok basittir:\n", - "\n", - "* TensorFlow islemleri otomatik olarak NumPy ndarray'lerini Tensorlara donusturur.\n", - "* NumPy islemleri de otomatik olarak Tensorlari NumPy ndarray'lerine donusturur.\n", - "\n", - "'.numpy()' metodunu kullanarak Tensorlari belirgin sekilde NumPy ndarray'lerine donusturebilirsiniz.\n", - "Tensorlar ve 'ndarray'ler temelde mumkun oldugunca ayni sekilde tanimlandigi icin bu donusturmeler ucuzdur. Fakat, NumPy dizileri her zaman ana hafizada calisirken Tensorlar GPU hafizasini da kullanabildigi icin her zaman benzer sekilde tanimlanamazlar ve donusturme isleminde GPU'dan ana hafizaya kopyalama da bulunur." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lCUWzso6mbqR", - "colab": {} - }, - "source": [ - "import numpy as np\n", - "\n", - "ndarray = np.ones([3, 3])\n", - "\n", - "print(\"TensorFlow operations convert numpy arrays to Tensors automatically\")\n", - "tensor = tf.multiply(ndarray, 42)\n", - "print(tensor)\n", - "\n", - "\n", - "print(\"And NumPy operations convert Tensors to numpy arrays automatically\")\n", - "print(np.add(tensor, 1))\n", - "\n", - "print(\"The .numpy() method explicitly converts a Tensor to a numpy array\")\n", - "print(tensor.numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PBNP8yTRfu_X" - }, - "source": [ - "## GPU hizlandirmasi\n", - "\n", - "Hesaplamalar icin GPU kullanarak bircok TensorFlow islemleri hizlandirilabilir. TensorFlow bir islem icin, ek aciklamaya gerek duymadan, otomatik olarak GPU ya da CPU kullanimina karar verir (ve gerektiginde tensorlari GPU ve CPU hafizalarina kopyalar). Bir islem sonucu olusan tensorlar o islem hangi hafizada yurutulduyse o hafizaya kopyalanir. Ornegin:" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "code", - "colab_type": "code", - "id": "3Twf_Rw-gQFM", - "colab": {} - }, - "source": [ - "x = tf.random_uniform([3, 3])\n", - "\n", - "print(\"Is there a GPU available: \"),\n", - "print(tf.test.is_gpu_available())\n", - "\n", - "print(\"Is the Tensor on GPU #0: \"),\n", - "print(x.device.endswith('GPU:0'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vpgYzgVXW2Ud" - }, - "source": [ - "### Aygit Isimleri\n", - "\n", - "`Tensor.device` ozelligi tensorlarin bulundugu aygitin tam adini dizgi olarak temin eder. Bu dizgide bircok detay bulunmaktadir: programin calistigi anasistemin bulundugu agin taniticisi ve anasistemdeki aygit. Bunlar TensorFlow programlarinin dagitiminda gerekli olan bilgilerdir. Eger tensor sistemdeki 'N'inci GPU'ya yerlestirilmisse bu dizgi `GPU:` ile biter." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZWZQCimzuqyP" - }, - "source": [ - "\n", - "\n", - "### Belirtilmis Aygit Yerlestirilmesi\n", - "\n", - "TensorFlow'da \"yerlestirme\" terimi islemlerin uygulama sirasinda sistemde tek tek nasil atandigi (yerlestirildigi) anlaminda kullanilmistir. Yukarida da bahsettigimiz gibi, eger ozellikle belirtilmemisse TensorFlow bir islemi nerde calistiracagina otomatik olarak karar verir ve gerekirse tensorlari oraya kopyalar. Fakat, TensorFlow islemleri 'tf.device' baglam yoneticisi kullanilarak belirli aygitlara yerlestirilebilir. Ornegin:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RjkNZTuauy-Q", - "colab": {} - }, - "source": [ - "import time\n", - "\n", - "def time_matmul(x):\n", - " start = time.time()\n", - " for loop in range(10):\n", - " tf.matmul(x, x)\n", - "\n", - " result = time.time()-start\n", - "\n", - " print(\"10 loops: {:0.2f}ms\".format(1000*result))\n", - "\n", - "\n", - "# CPU ustunde zorla calistirma\n", - "print(\"On CPU:\")\n", - "with tf.device(\"CPU:0\"):\n", - " x = tf.random_uniform([1000, 1000])\n", - " assert x.device.endswith(\"CPU:0\")\n", - " time_matmul(x)\n", - "\n", - "# Eger mumkunse GPU ustunde zorla calistirma #0\n", - "if tf.test.is_gpu_available():\n", - " with tf.device(\"GPU:0\"): # Or GPU:1 for the 2nd GPU, GPU:2 for the 3rd etc.\n", - " x = tf.random_uniform([1000, 1000])\n", - " assert x.device.endswith(\"GPU:0\")\n", - " time_matmul(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o1K4dlhhHtQj" - }, - "source": [ - "## Veri setleri\n", - "\n", - "Simdi modelimize veri akimini saglamak icin [`tf.data.Dataset` API](https://www.tensorflow.org/r1/guide/datasets)'sini nasil kullanacagimizi gorecegiz:\n", - "\n", - "* `Dataset`i olusturalim.\n", - "* Eager modunda `Dataset`in yinelenmesi.\n", - "\n", - "Modelimizin egitim ve degerlendirme dongulerine verilen kompleks girdi hatlarini 'Dataset' API'si ile basit ve tekrar kullanilabilir parcalardan olusturmanizi tavsiye ederiz.\n", - "\n", - "'Dataset' nesnesi olusturma API'si eager modunda iken TensorFlow graph'taki ile aynidir, fakat veri setindeki elemanlarin yinelenmesi islemi biraz daha basittir.\n", - "'tf.data.Dataset' nesnesi ustunde direk olarak Python yinelemesi yapabildiginiz icin `tf.data.Iterator` nesnesi olusturmaniza gerek yoktur.\n", - "Sonuc olarak, eger eager modunu kullaniyorsaniz, [TensorFlow Rehberi](https://www.tensorflow.org/r1/guide/datasets)'nde anlatilan yineleme gereksizdir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zI0fmOynH-Ne" - }, - "source": [ - "### `Dataset` kaynagi olusturalim\n", - "\n", - "Buradaki fonksiyonlardan birini [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#from_tensor_slices) ya da kutuklerden okunan nesneleri [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TextLineDataset) veya [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset) kullanarak _source_ dataset olusturabiliriz. [TensorFlow Rehberi](https://www.tensorflow.org/r1/guide/datasets#reading_input_data)'nde daha detayli bilgi bulabilirsiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F04fVOHQIBiG", - "colab": {} - }, - "source": [ - "ds_tensors = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", - "\n", - "# CSV kutugunu olusturalim\n", - "import tempfile\n", - "_, filename = tempfile.mkstemp()\n", - "\n", - "with open(filename, 'w') as f:\n", - " f.write(\"\"\"Line 1\n", - "Line 2\n", - "Line 3\n", - " \"\"\")\n", - "\n", - "ds_file = tf.data.TextLineDataset(filename)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbxIhC-5IPdf" - }, - "source": [ - "### Transformations (donusumler) uygulayalim\n", - "\n", - "Veri seti kayitlarini donusturmek icin transformations (donusumler) fonksiyonlarini kullanabiliriz: ornegin [`map`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle). `tf.data.Dataset` API dokumanlari hakkinda daha fazla bilgi icin [buraya bakiniz](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uXSDZWE-ISsd", - "colab": {} - }, - "source": [ - "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", - "\n", - "ds_file = ds_file.batch(2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "A8X1GNfoIZKJ" - }, - "source": [ - "### Yineleme\n", - "\n", - "Eager modunda 'Dataset' nesneleri yinelemeleri destekler.\n", - "Eger TensorFlow 'graphs'taki 'Dataset' kullanimina asina iseniz, `Dataset.make_one_shot_iterator()` ya da `get_next()` kullanimina gerek olmadigina lutfen dikkat ediniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ws-WKRk5Ic6-", - "colab": {} - }, - "source": [ - "print('Elements of ds_tensors:')\n", - "for x in ds_tensors:\n", - " print(x)\n", - "\n", - "print('\\nElements in ds_file:')\n", - "for x in ds_file:\n", - " print(x)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/keras/README.md b/site/tr/r1/tutorials/keras/README.md deleted file mode 100644 index 3dbd732b8dc..00000000000 --- a/site/tr/r1/tutorials/keras/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Makine öğrenmesini anlayalım ve kullanalım - -Note: Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir. -Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince -güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en) -ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek -için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs) -havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için -[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr) -listesi ile iletişime geçebilirsiniz. - -Aşağıda linkleri verilmiş olan Juypter Notebook'ları, -*[Deep Learning with Python](https://books.google.com/books?id=Yo3CAQAACAAJ)* -kitabından esinlenerek hazırlanmıştır. Bu yardımcı eğitim dökümanları, -TensorFlow derin öğrenme modellerinin oluşturulması ve eğitiminde kullanılan -yüksek-seviye Python API si `tf.keras`'ı kullanmaktadır. Keras ile ilgili daha -fazla bildi edinmek için, -[TensorFlow Keras Guide](https://github.com/tensorflow/docs/blob/master/site/en/r1/guide/keras.ipynb) -'a göz atabilirsiniz. - -Yayıncı notu: *Deep Learning with Python* kitabı, Python dili ve Keras -kütüphanesini kullanarak derin öğrenme konusunu tanıtmaktadır. Keras'ın -yaratıcısı ve Google AI araştırmacısı François Chollet tarafından yazılan bu -kitap, kolay anlaşılabilir açıklamalar ve pratikte geçerliliği olan örnekler ile -konuyu anlamanızı sağlamaktadır. - -Makine Öğrenmesinin temellerini ve ilgili kavramları öğrenmek için -[Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/) -' a katılmayı düşünebilirsiniz. - -1. [Temel sınıflandırma](basic_classification.ipynb) -2. [Metin sınıflandırma](basic_text_classification.ipynb) -3. [Regrasyon](basic_regression.ipynb) -4. [Aşırı uyum ve yetersiz uyum](overfit_and_underfit.ipynb) -5. [Modellerin kaydedilmesi ve yeniden yüklenmesi](save_and_restore_models.ipynb) diff --git a/site/tr/r1/tutorials/keras/basic_classification.ipynb b/site/tr/r1/tutorials/keras/basic_classification.ipynb deleted file mode 100644 index 8d2e89f543c..00000000000 --- a/site/tr/r1/tutorials/keras/basic_classification.ipynb +++ /dev/null @@ -1,1008 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "MhoQ0WE77laV", - "colab_type": "text" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "_ckMIh7O7s6D", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "vasWnqRgy1H4", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jYysdyb-CaWM", - "colab_type": "text" - }, - "source": [ - "# İlk sinir ağınızı eğitin: temel sınıflandırma" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "S5Uhzt6vVIB2", - "colab_type": "text" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Google Colab’da Çalıştır\n", - " \n", - " GitHub'da Kaynağı Görüntüle\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EVlm-3_0Q4zZ", - "colab_type": "text" - }, - "source": [ - "Note: Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir.\n", - "Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince\n", - "güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en)\n", - "ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek\n", - "için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için\n", - "[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr)\n", - "listesi ile iletişime geçebilirsiniz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FbVhjPpzn6BM", - "colab_type": "text" - }, - "source": [ - "Bu yardımcı döküman, spor ayakkabısı ve gömlek gibi çeşitli giysi görüntülerini sınıflandırmak için bir sinir ağı modelini eğitir. Örnekte yer alan tüm detayları anlayamadıysanız bu sorun değil, aşağıda ilerledikçe ayrıntıları açıklanacak olan Tensorflow'a hızlı bir genel bakış yapılmaktadır.\n", - "\n", - "Bu örnekte, Tensorflow'da yapay zeka modellerinin oluşturulması ve eğitilmesinde kullanılan yüksek-seviye API olan, [tf.keras](https://www.tensorflow.org/r1/guide/keras) kullanmaktadır." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "dzLKpmZICaWN", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow ve tf.keras\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# Yardimci kutuphaneler\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yR0EdgrLCaWR", - "colab_type": "text" - }, - "source": [ - "## Fashion MNIST veri kümesini yükleyelim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DLdCchMdCaWQ", - "colab_type": "text" - }, - "source": [ - "Bu örnek uygulama, 10 kategoride 70,000 siyah-beyaz görüntü içeren [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) veri kümesini kullanmaktadır. Aşağıda görüldüğü gibi bu veri kümesi, çeşitli giyim eşyalarının düşük çüzünürlükteki (28 x 28 piksel) görüntülerini içermektedir :\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figür 1. Fashion-MNIST örnekleri (Zalando tarafından, MIT lisansı ile).
     \n", - "
    \n", - "\n", - "Fashion MNIST, klasik [MNIST](http://yann.lecun.com/exdb/mnist/) veri kümesinin yerine kolayca kullanılabilecek şekilde geliştirilmiştir. Klasik MNIST veri kümesi, yukarıda yer alan giysi görüntüleri ile aynı formatta, el yazısı rakam (0, 1, 2, vb) görüntülerini içermektedir.\n", - "\n", - "Fashion MNIST, klasik MNIST'e göre biraz daha zorlayıcı olduğu için ve çeşitliliğin arttırılması amacıyla kullanılmıştır. İki veri kümesi de nispeten küçüktür ve algoritmaların beklendiği gibi çalışıp çalışmadığının doğrulanmasında kullanılırlar. Ayrıca, yazdığımız kodun test edilmesi ve hataların tespit edilmesinde oldukça iyi bir başlangıç noktası oluştururlar.\n", - "\n", - "Oluşturacağımız sinir ağının eğitilmesinde 60,000 görüntü, eğitilmiş sinir ağının görüntü sınıflandırma doğruluğunu değerlendirmek içinse 10,000 görüntü kullanacağız. Fashion MNIST'e TensorFlow içerisinde doğrudan ulaşabilirsiniz, bunun için yapmanız gereken sadece veriyi içeri almak ve yüklemek:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "7MqDQO0KCaWS", - "colab_type": "code", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "t9FDsUlxCaWW", - "colab_type": "text" - }, - "source": [ - "Veri kümesinin yüklenmesi sonucunda 4 NumPy dizisi oluşur:\n", - "\n", - "* `train_images` ve `train_labels` dizileri *eğitim veri setidir* - modelin eğitilmesinde kullanılır.\n", - "* `test_images` ve `test_labels` dizileri *test veri setidir* - modelin test edilmesinde kullanılır.\n", - "\n", - "*train_images, test_images* 28x28 boyutunda ve piksel değerleri 0 ile 255 arasında değişen NumPy dizileridir. *train_labels, test_labels* ise 0 ile 9 arasında değişen ve her biri bir giyim eşyası sınıfı ile eşleşen tam sayı dizisidir:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    EtiketSınıf
    0Tişört/Üst
    1Pantolon
    2Kazak
    3Elbise
    4Mont
    5Sandal
    6Gömlek
    7Spor Ayakkabı
    8Çanta
    9Yarım Bot
    \n", - "\n", - "Veri kümesi içerisindeki her bir görüntü tek bir etiket ile eşleştirilmiştir. *Sınıf isimleri* veri kümesi içerisinde yer almadığı için, daha sonra görüntüleri ekrana yazdırmak için bunları aşağıdaki gibi bir dizi içerisinde saklayalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "IjnLH5S2CaWx", - "colab_type": "code", - "colab": {} - }, - "source": [ - "class_names = ['Tişört/Üst', 'Pantolon', 'Kazak', 'Elbise', 'Mont', \n", - " 'Sandal', 'Gömlek', 'Spor Ayakkabı', 'Çanta', 'Yarım Bot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Brm0b_KACaWX", - "colab_type": "text" - }, - "source": [ - "## Veriyi inceleyelim\n", - "\n", - "Modeli eğitmeye başlamadan önce, veri kümesi yapısını birlikte inceleyelim. Aşağıda, modelin eğitilmesinde kullanılan veri setinin 60,000 görüntüden oluştuğu ve her birinin 28 x 28 piksel olduğunu görmektesiniz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "zW5k_xz1CaWX", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cIAcvQqMCaWf", - "colab_type": "text" - }, - "source": [ - "Benzer şekilde, eğitim veri setinde 60,000 adet etiket bilgisi yer almaktadır:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "TRFYHB2mCaWb", - "colab_type": "code", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YSlYxFuRCaWk", - "colab_type": "text" - }, - "source": [ - "Her bir etiket 0 ile 9 arasında bir tam sayıdır:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "XKnCTHz4CaWg", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TMPI88iZpO2T", - "colab_type": "text" - }, - "source": [ - "Test veri kümesinde 10,000 görüntü mevcuttur. Her bir görüntü, benzer şekilde 28 x 28 piksel den oluşmaktadır:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "2KFnYlcwCaWl", - "colab_type": "code", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rd0A0Iu0CaWq", - "colab_type": "text" - }, - "source": [ - "Ve test veri seti 10,000 etiket bilgisini kapsamaktadır:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "iJmPr5-ACaWn", - "colab_type": "code", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ES6uQoLKCaWr", - "colab_type": "text" - }, - "source": [ - "## Verileri Ön İşleme\n", - "\n", - "Sinir ağının eğitilmesinden önce verinin bir ön işleme tabi tutulması gerekmektedir. Eğitim veri setindeki ilk görüntüyü inceleyecek olursanız, piksel değerlerinin 0 ile 255 arasında olduğunu göreceksiniz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "m4VEw8Ud9Quh", - "colab_type": "code", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Wz7l27Lz9S1P", - "colab_type": "text" - }, - "source": [ - "Bu görüntüler ile sinir ağını beslemeden önce, görüntülerin piksel değerlerini 0 ile 1 aralığına ölçekleyeceğiz. Bunun için, piksel değerlerini 255'e böleceğiz. Bu noktada *eğitim veri seti* ile *test veri seti*'nin aynı şekilde ön işlemden geçirilmesi, modelimizin doğru sonuç vermesi açısından önem taşımaktadır:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "bW5WzIPlCaWv", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ee638AlnCaWz", - "colab_type": "text" - }, - "source": [ - "*eğitim veri seti*'nin ilk 25 görüntüsünü, her bir görüntünün altında sınıf etiketi yazacak şekilde ekranda gösterelim. Verinin doğru formatta olduğunu doğruladıktan sonra artık modeli oluşturup eğitmeye hazırız." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "oZTImqg_CaW1", - "colab_type": "code", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "59veuiEZCaW4", - "colab_type": "text" - }, - "source": [ - "## Modelin oluşturulması\n", - "\n", - "Sinir ağının oluşturulması için öncelikle model katmanlarının yapılandırılması ve sonrasında modelin derlenmesi gerekmektedir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Gxg1XGm0eOBy", - "colab_type": "text" - }, - "source": [ - "### Katmanların hazırlanması\n", - "\n", - "Sinir ağını oluşturan temel yapı taşları *katman*'lardır. Katmanlar, kendilerine beslenen verileri kullanarak bu verilere ait çıkarımlar oluştururlar. Bu çıkarımların, bu örnekte görüntülerin sınıflandırılması olarak karşımıza çıkan problemin çözümüne yardımcı olması beklenir.\n", - "\n", - "Çoğu derin öğrenme modeli, birbirlerine bağlanmış birçok basit katman içermektedir. Çoğu katman, `tf.keras.layers.Dense` gibi, eğitme sürecinde öğrenilen parametrelere sahiptir." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "9ODch-OFCaW4", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation=tf.nn.relu),\n", - " keras.layers.Dense(10, activation=tf.nn.softmax)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gut8A_7rCaW6", - "colab_type": "text" - }, - "source": [ - "Ağımızın ilk katmanı olan `tf.keras.layers.Flatten`, görüntülerin formatını 2 boyutlu sayı dizisinden (28 x 28 piksel), 28 * 28 = 784 piksel değerinden oluşan tek boyutlu bir sayı dizisine çevirir. Bu katmanın, görüntüleri oluşturan piksel satırlarının çıkarılarak, art arda birleştirilmesi ile oluştuğunu düşünebilirsiniz. Bu katmanda öğrenilecek parametre olmayıp, sadece görüntünün formatını düzenler.\n", - "\n", - "Görüntüyü oluşturan pikselleri tek boyutlu sayı dizisine düzleştirdikten sonra, ağımız ardaşık iki `tf.keras.layers.Dense` katmanını içerir. Bunlara, yoğun-bağlı veya tam-bağlı ağ katmanları denir. İlk 'Yoğun' katman 128 neron'a (düğüm) sahiptir. İkinci katman is 10 neronlu 'softmax' katmanıdır. Bu son katmanın çıktısı, toplam değerleri 1' eşit olan ve 10 farklı olasılık sonucunu içeren sayı dizisidir. Her bir düğüm, mevcut görüntünün hangi sınıfa ait olduğu belirten olasılık değerini içerir.\n", - "\n", - "### Modelin derlenmesi\n", - "\n", - "Modelin eğitilmeye tamamıyla hazır olması öncesinde bir kaç düzenleme daha yapılması gerekmektedir. Bu düzenlemeler modelin 'derlenme' adımında eklenmektedir:\n", - "\n", - "* *Kayıp Fonksiyonu - Loss Function* — Bu fonksiyon modelin eğitim sürecinde ne kadar doğru sonuç verdiğini ölçer. Bu fonksiyonun değerini en aza indirgeyerek, modelin doğru istikamete \"yönlendirmek\" isteriz.\n", - "* *Eniyileme - Optimizer* — Beslenen veriler ve kayıp fonksiyonu ile modelin nasıl güncellediğini belirler\n", - "* *Metrikler - Metrics* — Eğitim ve test adımlarını gözlemlemek için kullanılır. Aşağıdaki örnekte, *doğruluk-accuracy*, modelin doğru sınıfladığı görüntü oranı, kullanılmaktadır." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Lhan11blCaW7", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam', \n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qKF6uW-BCaW-", - "colab_type": "text" - }, - "source": [ - "## Modelin eğitilmesi\n", - "\n", - "Sinir ağının eğitilmesi aşağıdaki adımları gerektirir:\n", - "\n", - "1. Eğitim veri setinin modele beslenmesi - bu örnekte veri setimiz yukarıda açıklanan 'train_images' ve 'train_labels' dizileridir.\n", - "2. Model etiketleri ve görüntüleri kullanarak çıkarımlar yapar ve öğrenir.\n", - "3. Modelden test veri setini - bu örnekte 'test_images' dizisidir, kullanarak tahminleme yapmasını isteriz. Sonucu 'test_labels' dizisindeki etiket ile eşleştirerek, bu kestirimlerin doğruluğunu teyid edebiliriz.\n", - "\n", - "Eğitimi başlatmak için 'model.fit' methodu çalıştırılır:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "xvwvpA64CaW_", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "W3ZVOhugCaXA", - "colab_type": "text" - }, - "source": [ - "Model eğitimi süresince, kayıp ve doğruluk metrikleri ekranda gösterilir. Örneğimizdeki model, eğitim veri setiyle 0.88 (or 88%) doğruluk eğerine ulaşır." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oEw4bZgGCaXB", - "colab_type": "text" - }, - "source": [ - "## Model doğruluğunun değerlendirlmesi\n", - "\n", - "Sonrasında, modelin test veri seti ile nasıl bir performans gösterdiğini karşılaştıralım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "VflXLEeECaXC", - "colab_type": "code", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('Test accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yWfgsmVXCaXG", - "colab_type": "text" - }, - "source": [ - "Test veri seti ile aldığımız model doğruluk değerinin, eğitim veri seti ile aldığımız model doğruluk değerinden biraz daha düşük olduğunu görmekteyiz. Eğitim doğruluk değeri ile test doğruluk değeri arasındaki bu farka *aşırı uyum-overfitting* denir. Aşırı uyum, modelin yeni veriler ile tahminleme yaparken, eğitim veri setine göre daha kötü performans göstermesi durumudur. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xsoS7CPDCaXH", - "colab_type": "text" - }, - "source": [ - "## Modeli kullanarak tahminleme yapalım\n", - "\n", - "Eğitilmiş modelimizi kullanarak, bir kaç görüntü için tahminleme yapabiliriz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Gl91RPhdCaXI", - "colab_type": "code", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "x9Kk1voUCaXJ", - "colab_type": "text" - }, - "source": [ - "Aşağıda, test veri setinde yer alan her bir görüntü için, modelimiz etiket sınıflandırması yapmaktadır. İlk tahminlemeye birlikte bakalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "3DmJEUinCaXK", - "colab_type": "code", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-hw1hgeSCaXN", - "colab_type": "text" - }, - "source": [ - "Tahminleme sonucu, 10 sayıdan oluşan bir dizi elde ederiz. Bu sayı dizisi bize, görüntünün 10 farklı sınıftan hangi giysi türüne ait olduğuna dair modelin \"güvenini\" tanımlamaktadır. Bu değerlere bakarak, hangi etiket sınıfının en yüksek güven değerine sahip olduğunu görebiliriz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "qsqenuPnCaXO", - "colab_type": "code", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "E51yS7iCCaXO", - "colab_type": "text" - }, - "source": [ - "Modelimiz yarım bot etiketi, (`veya class_names[9]`) için en yüksek kestirim güven değeri vermektedir. Ve test veri setindeki etikete bakarak sonucun doğru olduğunu görebiliriz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Sd7Pgsu6CaXP", - "colab_type": "code", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ygh2yYC972ne", - "colab_type": "text" - }, - "source": [ - "10 farklı sınıfın tümüne bakabilmek için sonucun grafiğini oluşturabiliriz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "DvYmmrpIy6Y1", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " \n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - " \n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1]) \n", - " predicted_label = np.argmax(predictions_array)\n", - " \n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "d4Ov9OFDMmOD", - "colab_type": "text" - }, - "source": [ - "0'ıncı görüntüye, tahminlere ve tahmin dizisine bakalım: " - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "HV5jw-5HwSmO", - "colab_type": "code", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "Ko-uzOufSCSe", - "colab_type": "code", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kgdvGD52CaXR", - "colab_type": "text" - }, - "source": [ - "Çeşitli görüntüleri, tahminlemeleri ile ekranda gösterelim. Doğru tahminleme sonuçları mavi, yanlış olanlar ise kırmızı ile ekranda gösterilecektir. Rakamlar ise, 100 üzerinden, yapılan tahminlemenin güven değerini vermektedir. Güven değeri yüksek olsa bile, sonucun yanlış olabileceğini görebilirsiniz. " - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "hQlnbqaw2Qu_", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Ilk X resmi, tahmin edilen etiketini ve asil etiketlerini cizelim.\n", - "# Dogru tahminler mavi, yanlis tahminler ise kirmizi renktedir.\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "R32zteKHCaXT", - "colab_type": "text" - }, - "source": [ - "Son olarak, eğitilmiş modeli kullanarak tek bir görüntü üzerinden tahminleme yapalım: " - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "yRJ7JU7JCaXT", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Test veri setinden bir resim secelim.\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vz3bVp21CaXV", - "colab_type": "text" - }, - "source": [ - "`tf.keras` modelleri, *veri yığınları* içerisindeki örnekler üzerinden tahminleme yapmak üzere optimize edilmiştirler. Tek bir görüntü kullanmamıza rağmen, bu nedenle görüntüyü bir listeye aktarmamız gerekmektedir:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "lDFh5yF_CaXW", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# Resmi tek ogesi kendisi olacagi bir listeye aktaralim.\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EQ5wLTkcCaXY", - "colab_type": "text" - }, - "source": [ - "Şimdi görüntüyü tahminleyelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "o_rzNSdrCaXY", - "colab_type": "code", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "6Ai-cpLjO-3A", - "colab_type": "code", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "plt.xticks(range(10), class_names, rotation=45)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cU1Y2OAMCaXb", - "colab_type": "text" - }, - "source": [ - "`model.predict` çalıştırıldığında, veri yığını içerisindeki her bir görüntüye ait bir liste verir. Yığın içerisinden görüntümüze (örneğimizdeki tek görüntü) ait tahminleme sonuçlarını alalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "2tRmdq_8CaXb", - "colab_type": "code", - "colab": {} - }, - "source": [ - "prediction_result = np.argmax(predictions_single[0])\n", - "print(prediction_result)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YFc2HbEVCaXd", - "colab_type": "text" - }, - "source": [ - "Daha önceden olduğu gibi, modelimiz etiket değeri olarak 9'u vermektedir." - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/keras/basic_regression.ipynb b/site/tr/r1/tutorials/keras/basic_regression.ipynb deleted file mode 100644 index 654f08486c3..00000000000 --- a/site/tr/r1/tutorials/keras/basic_regression.ipynb +++ /dev/null @@ -1,866 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# Regrasyon: yakıt verimliliğini tahmin edelim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NzLRKeK3iaH_", - "colab_type": "text" - }, - "source": [ - "Note: Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir.\n", - "Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince\n", - "güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en)\n", - "ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek\n", - "için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için\n", - "[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr)\n", - "listesi ile iletişime geçebilirsiniz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "*Regrasyon* problemlerinde, olasılık veya fiyat gibi sürekli değişken olan çıktıyı tahmin etmeyi amaçlarız. Aksine, *sınıflandırma* problemlerinde ise amacımız, belirli bir sınıf listesi içerisinden en uygun sınıfı seçebilmektir (örneğin bir resimde elma veya portakal olsun, resimde elma mı yoksa portakal mı olduğunu belirlemek isteriz).\n", - "\n", - "Bu çalışma kitabı, 1970'lerin sonları ve 1980'lerin başlarında üretilmiş olan araçların yakıt verimliliğini (MPG) tahmin edebilecek bir model geliştirmek için klasik [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) veri setini kullanmaktadır. Bunu yapabilmek için, belirtilen zaman aralığında üretilmiş olan araç bilgilerini modelimize besleyeceğiz. Modelimize besleyeceğimiz bu bilgiler değişik araç özelliklerini kapsamaktadır: motorun kaç silindirli olduğu, beygir günü, hacmi ve araç ağırlığı.\n", - "\n", - "Bu örnekte `tf.keras` API kullanılmıştır, detaylar için bu [yardımcı döküman](https://www.tensorflow.org/r1/guide/keras)'a göz atabilirsiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# pairplot icin seaborn kullanalim\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Auto MPG veri seti\n", - "\n", - "Bu veri seti [UCI Machine Learning Repository](https://archive.ics.uci.edu/ml/) sitesinde bulunmaktadır.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### Veriyi alalım\n", - "İlk olarak veri setini indirelim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"https://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "pandas kütüphanesini kullanarak verileri içeri alalım" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin'] \n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### Veriyi temizleyelim\n", - "\n", - "Veri seti bilinmeyen bir kaç değer içermektedir. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "Eğitim dökümanını kısa tutabilmek adına, bu satırları veri setinden çıkaralım. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "`\"Origin\"` kolonu sayısal değil, kategorik değer içermektedir. Dolayısı ile, bu kolonu one-hot forma çevirelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### Veri setini eğitim ve test olarak ikiye ayıralım\n", - "\n", - "Şimdi, veri setini eğitim seti ve test setine ayıralım.\n", - "\n", - "Modelimizin tamamlandıktan sonraki son değerlendirmesinde test veri setini kullanacağız." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### Veriyi inceleyelim \n", - "\n", - "Eğitim veri setindeki birkaç kolonun değer dağılımlarına birlikte hızlıca göz atalım." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "Genel istatistiklere bakalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### Etiketlerin özelliklerden ayırılması\n", - "\n", - "Hedef verisini, veya \"etiketi\", özelliklerden ayıralım. Modeli, bu etiket değerini tahminleyebilmek için eğitiyoruz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### Veriyi normalize edelim\n", - "\n", - "Yukarıda yer alan 'train_stats' değerlerine baktığımızda, her bir özelliğin birbirinden ne kadar farklı değer aralığına sahip olduğunu görmekteyiz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "Birbirinden farklı büyüklükteki ve aralıklardaki özellik değerlerinin normalize edilmesi her zaman işimizi kolaylaştırır. Modelin mevcut verilerle sonuca yakınsaması mümkün olsa bile, bu modelin eğitilmesini güçleştirir, ayrıca modelin seçilen girdinin birimine bağlı sonuçlar vermesine neden olur. \n", - "\n", - "Not: İstatistikleri bilinçli olarak sadece eğitim veri setine göre oluşturmuş olsakta, bu bilgiler test veri setinin normalizasyonu için de kullanılacaktır. Test veri setini, modelin eğitilmiş olduğu veri setinin sahip olduğu dağılıma yansitabilmek için, aynı şekilde normalize etmemiz gerekmektedir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "Bu normalize edilmiş veri, modelin eğitimi için kullanacağımız veridir.\n", - "\n", - "Dikkat: Değerleri normalize etmek için kullandığımız istatiksel değerler (ortalama ve standart sapma), one-hot şifreleme kullanılarak modele beslenecek diğer tüm verilere de uygulanmalıdır. Bu normalizasyon işlemi test verisinde, ayrıca modelimizin canlı kullanımında modele beslenen verilerde de aynı şekilde uygulanmalıdır." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## Model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### Modeli oluşturalım\n", - "\n", - "Modelimizi birlikte oluşturalım. İki 'yoğun bağlı (densely connected)' gizli katman, ayrıca tek bir sürekli değer üreten çıktı katmanı kullanacağız. Sonradan ikinci bir model oluşturacağımız için, kolaylık olması açısından model oluşturma adımlar 'build_model' fonsiyonu içerisinde yer almaktadır." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation=tf.nn.relu, input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation=tf.nn.relu),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mean_squared_error',\n", - " optimizer=optimizer,\n", - " metrics=['mean_absolute_error', 'mean_squared_error'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### Modeli inceleyelim\n", - "\n", - "Modelin basit bir açıklamasını ekrana yazdırabilmek için '.summary' methodunu kullanalım." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "\n", - "Şimdi modeli kullanalım. Eğitim veri setinden 10 özelliği içeren veri grubunu alalım ve 'model.predict' metodunu bu veri grubu ile çalıştıralım." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "Model beklenen form ve tipte sonuçlar üretiyor ve beklendiği şekliyle çalışıyor gözüküyor." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### Modeli eğitelim\n", - "\n", - "Modeli 1000 epoch döngüsü içerisinde egitelim ve doğrulama ve eğitim doğruluk sonuçlarını 'history' objesi içerisinde kayit edelim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# Tamamlanan her epoch dongusu icin bir nokta yazdirarak egitimin gelisimini gosterelim\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "Model eğitim sürecini, 'history' objesi içerisine kaydetmiş olduğumuz değerleri kullanarak görselleştirelim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - " \n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mean_absolute_error'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mean_absolute_error'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - " \n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mean_squared_error'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mean_squared_error'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "100 epoch sonrasında doğrulama hata değerinin iyileşmediği aksine bir miktar kötüleştiği görülmektedir. 'model.fit' metodunu, doğrulama değerinin iyileşmemesi durumunda model eğitimini otomatik olarak durduracak şekilde güncelleyelim. *EarlyStopping callback* kullanarak eğitim durumunu her epoch sonunda kontrol ediyor olacağız. Belirli bir adet epoch süresince model iyileşme göstermezse, model eğitimini otomatik olarak durduracağız.\n", - "\n", - " Bu callback ile ilgili daha fazla bilgiyi [burada](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping) bulabilirsiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# 'patience parameter' kac adet epoch dongusunun iyilesme icin kontrol edilecegini belirler\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "Grafik, doğrulama seti ile ortalama hatanın +/- 2 MPG aralığında olduğunu göstermektedir. Bu hata değerinin iyi mi, yoksa kötü mü olduğu kararını size bırakıyoruz? \n", - "\n", - "Modelin daha önce görmediği **test** verileri ile nasıl performans gösterdiğine, yani genelleme yeteneğinin ne kadar iyi olduğuna birlikte bakalım. Bu bize modelimizin gerçek dünyada, kullanıcı verileri ile çalıştırıldığında ne kadar iyi tahminleme yapacağını gösterecektir." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "### Tahminleme yapalım\n", - "\n", - "Son olarak, test veri setini kullanarak MPG değerlerini tahminleyelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OrkHGKZcusUo" - }, - "source": [ - "Modelimizin epey iyi sonuç verdiği görülmektedir. Hata dağılımına birlikte bakalım." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r9_kI6MHu1UU" - }, - "source": [ - "Hataların dağılımı gauss dağılımına çok benzer değil, veri adedimiz oldukça az olduğu için bu beklenen bir durumdur. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## Sonuç ve kapanış\n", - "\n", - "Bu notebook ile, regrasyon problemlerinin çözümünde kullanılan bazı teknikler açıklanmıştır: \n", - "\n", - "* Ortalama karekök hatası - Mean Squared Error (MSE), regrasyon problemlerinin çözümünde sıklıkla kullanılan kayıp fonksiyonudur (sınıflandırma problemlerinde daha farklı kayıp fonksiyonları kullanılır). \n", - "* Benzer şekilde, regrasyon ve sınıflandırma modellerinin değerlendirme metrikleri de farklılık gösterir. Regrasyon modellerinde genel olarak kullanılan metrik, Ortalama Mutlak Hata - Mean Absolute Error (MAE)'dır. \n", - "* Farklı özelliklere ait sayısal verilerinin değer aralıklarının farklı olması durumunda, her bir özelliğın bağımsız olarak aynı değer aralığına indirgenmesi gerekmektedir. \n", - "* Eğitim veri seti için elimizde fazla veri yoksa, aşırı uyum (overfitting) gözlemlenmemesi için, az sayıda gizli katman içeren daha küçük sinir ağı modellerini tercih etmemiz gerekmektedir. \n", - "* Model eğitiminin erken durdurulması, aşırı uyumun oluşmasını engelleyen kullanışlı bir tekniktir." - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/keras/basic_text_classification.ipynb b/site/tr/r1/tutorials/keras/basic_text_classification.ipynb deleted file mode 100644 index 46ea05eaf1b..00000000000 --- a/site/tr/r1/tutorials/keras/basic_text_classification.ipynb +++ /dev/null @@ -1,742 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# Film yorumları ile metin sınıflandırma" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Google Colab’da Çalıştır\n", - " \n", - " GitHub'da Kaynağı Görüntüle\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pHRsNlyRQ9h9" - }, - "source": [ - "Note: Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir.\n", - "Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince\n", - "güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en)\n", - "ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek\n", - "için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için\n", - "[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr)\n", - "listesi ile iletişime geçebilirsiniz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "\n", - "Bu yardımcı döküman, yorum metinlerini kullanarak film yorumlarını *olumlu* veya *olumsuz* olarak sınıflandırmaktadır. Bu örnek, yoğun olarak kullanılan ve önemli bir makina öğrenmesi uygulaması olan *ikili* veya *iki kategorili sınıflandırma*' yı kapsamaktadır. \n", - "\n", - "Bu örnekte, [Internet Film Veritabanı](https://www.imdb.com/) sitesinde yer alan 50,000 film değerlendirme metnini içeren [IMDB veri seti](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb) 'ni kullancağız. Bu veri seti içerisindeki 25,000 yorum modelin eğitimi için, 25,000 yorum ise modelin testi için ayrılmıştır. Eğitim ve test veri setleri eşit miktarda olumlu ve olumsuz yorum içerecek şekilde dengelenmiştir. \n", - "\n", - "Bu yardımcı döküman, Tensorflow'da modellerin oluşturulması ve eğitilmesinde kullanına yüksek-seviye API [tf.keras](https://www.tensorflow.org/r1/guide/keras) 'ı kullanır. `tf.keras` ile ileri seviye metin sınıflandımayı öğrenmek için [MLCC Metin Sınıflandırma ](https://developers.google.com/machine-learning/guides/text-classification/)'a göz atabilirsiniz." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uubldcNwCAV-", - "colab": {} - }, - "source": [ - "# keras.datasets.imdb is broken in 1.13 and 1.14, by np 1.16.3\n", - "!pip install tf_nightly" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## IMDB veri setini indirelim\n", - "\n", - "IMDB veri seti TensorFlow ile birlikte bütünleşik olarak gelmektedir. Yorumların kelime diziliş sıraları, her bir sayının bir kelimeyi temsil ettiği sıralı bir tam sayı dizisine çevrilerek veri seti ön işlemden geçirilmiştir. \n", - "\n", - "Aşağıdaki kodlar, IMDB veri setini bilgisayarınıza indirir (eğer daha önceden indirme yapmışsanız, önbellekteki veri kullanılır) :" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "`num_words=10000` değişkeni eğitim veri setinde en sık kullanılan 10,000 kelimeyi tutar, az kullanılan kelimeleri veri boyutunun yönetilebilir olması için ihmal eder." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## Veriyi inceleyelim \n", - "\n", - "Veri formatını aşağıdaki kodlar yardımı ile birlikte inceleyelim. Veri seti, ön işlem uygulanmış şekilde gelmektedir: tüm film yorum örnekleri, her bir sayının yorumundaki bir kelimeye denk geldiği tam sayı dizisi olarak gelmektedir. Tüm etiketler 0 veya 1 değerine sahiptir (0 olumsuz değerlendirme, 1 olumlu değerlendirme)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} - }, - "source": [ - "print(\"Training entries: {}, labels: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "Yorum metinleri, her bir sayının sözlükte yer alan bir kelimeye denk geldiği sayı dizisine çevrilmiştir. İlk yorum metni, aşağıdaki gibidir:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "Farklı film yorumlarının uzunlukları farklı olabilir. Aşağıdaki kod, ilk ve ikinci yorumda yer alan kelime sayılarını göstermektedir. Sinir ağlarında girdi boyutlarının aynı olması gerekmektedir, bu problemi daha sonra çözeceğiz. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### Tam sayıları kelimelere geri çevirelerim\n", - "\n", - "Tam sayıları metin'e çevirme işlemini bilmemiz, bazı durumlarda işimize yarayabilir. Bunun için bir yardımcı fonksiyon oluşturacağız. Bu fonksiyon, tam sayı-karakter eşleştirmesi içeren bir sözlük nesnesini sorguyabilmemizi sağlayacak:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} - }, - "source": [ - "# A dictionary mapping words to an integer index\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# İlk indisler rezervedir\n", - "word_index = {k:(v+3) for k,v in word_index.items()} \n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # unknown\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "'decode_review' fonksiyonunu kullanarak ilk yorum metnini şimdi ekranda gösterebiliriz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} - }, - "source": [ - "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## Veriyi hazırlayalım\n", - "\n", - "Yorumlar -tam sayı dizileri- sinir ağına beslenmeden önce ilk olarak tensor yapısına çevrilmelidir. Bu çevirme işlemi birkaç farklı şekilde yapabilir: \n", - "\n", - "* Bu ilk yöntemde, one-hot encoding işlemine benzer şekilde, tam sayı dizileri kelimelerin mevcut olup olmamasına göre 0 ve 1 ler içeren, vektörlere çevrilir. Örnek olarak, [3, 5] dizisini vektör'e dönüştürdüğümüzde, bu dizi 3üncü ve 5inci indeksleri dışında tüm değerleri 0 olan 10,000 boyutlu bir vektor'e dönüşür. Sonrasında, ağımızın ilk katmanını floating point vektor verisini işleyebilen yoğun katman (dense layer) olarak oluşturabiliriz. Bu yöntem, 'num_words * num_reviews' boyutlu bir matris oluşturduğumuz için, yoğun hafıza kullanımına ihtiyaç duyar.\n", - "\n", - "* Alternatif olarak, tüm dizileri aynı boyutta olacak şekilde doldurabiliriz. Sonrasında 'max_length * max_review' boyutlu bir tam sayı vektorü oluşturabiliriz. Son olarak, bu boyuttaki vektörleri işleyebilen gömülü katmanı, ağımızın ilk katmanı olarak oluşturabiliriz.\n", - "\n", - "Bu örnekte ikinci yöntem ile ilerleyeceğiz. \n", - "\n", - "Film yorumlarımızın aynı boyutta olması gerektiği için, yorum boyutlarını standart uzunluğa dönüştüren [pad_sequences](https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences) fonksiyonunu kullanacağız:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} - }, - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "Şimdi, ilk yorum örneklerinin uzunluklarına birlikte bakalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "Ve ilk yorumu (doldurulmuş şekliyle) inceleyelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## Modeli oluşturalım\n", - "\n", - "Sinir ağları, katmanların birleştirilmesiyle oluşturulur. Bu noktada, modelin yapısıyla ilgili iki temel karar vermemiz gerekmektedir:\n", - "\n", - "* Modeli oluşturuken kaç adet katman kullanacağız?\n", - "* Her bir katmanda kaç adet *gizli birim* (hidden units) kullanacağız?\n", - "\n", - "Bu örnekte modelimizin girdi verisi, kelime indekslerini kapsayan bir tam sayı dizisidir. Tahmin edilecek etiket değerleri 0 ve 1'dir. Problemimiz için modelimizi oluşturalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "# Girdiler film yorumları için kullanılan kelime sayısıdır (10,000 kelime)\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation=tf.nn.relu))\n", - "model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "Sınıflandırıcı modelimizi oluşturmak için katmanlar sıralı bir şekilde birleştirilmiştir:\n", - "\n", - "1. İlk katmanımız 'gömülü-embedding' katmandır. Bu katman tam sayı olarak şifrelenmiş sözcük grubu içerisinden kelime değerlerini alıp, her bir kelime indeksi için bu değeri gömülü vektör içerisinde arar. Bu vektörler modelin eğitimi sırasında öğrenilirler ve çıktı dizisine bir boyut eklerler. Sonuç olarak boyutlar '(batch, sequence, embedding)' şeklinde oluşur:\n", - "2. Sonrasında, `GlobalAveragePooling1D` katmanı, her bir yorum örneği için, ardaşık boyutların ortalamasını alarak sabit uzunlukta bir çıktı vektörü oluştur. Bu işlem, en basit şekliyle, modelimizin faklı boyutlardaki girdileri işleyebilmesini sağlar.\n", - "3. Bu sabit boyutlu çıktı vektörü, 16 gizli birim (hidden units) içeren tam-bağlı (fully-connected) yoğun katman'a beslenir.\n", - "4. Son katman, tek bir çıktı düğümü içeren yoğun bağlı bir katmandır. 'sigmoid' aktivasyon fonksiyonunu kullanarak, bu düğümün çıktısı 0 ile 1 arasında, olasılık veya güven değerini temsil eden bir değer alır." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### Gizli birimler (Hidden units)\n", - "\n", - "Yukarıdaki model, girdi ve çıktı arasında iki adet ara veya \"gizli\" katman içerir. Çıktıların sayısı (birimler, düğümler veya neronlar), mevcut katman içerisinde yapılan çıkarımların boyutudur. Başka bir ifade ile, ağın öğrenirken yapabileceği ara çıkarım miktarını, katmanın çıktı boyutu belirler.\n", - "\n", - "Eğer model fazla gizli birim (daha fazla boyutta çıkarım alanı) veya fazla katmana sahipse, model daha kompleks çıkarımlar yapabilir. Bu durumda daha yoğun hesaplama gücüne ihtiyaç duyulur. Bununla birlikte, modelimiz problemin çözümü için gerekli olmayacak derecede çıkarımlar yaparak eğitim verisi ile çok iyi sonuçlar verse de, test verisinde aynı oranda başarılı olmayabilir. Buna *aşırı uyum - overfitting* denir, bu kavramı daha sonra tekrar inceleyeceğiz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### Kayıp fonksiyonu ve optimize edici\n", - "\n", - "Modelimizin eğitilmesi için bir kayıp fonksiyonuna ve optimize ediciye ihitiyacımız vardır. Problemimiz, film yorumlarını olumlu ve olumsuz olarak sınıflandırmak (yani ikili siniflandirma problemi) olduğu için, 'binary_crossentropy' kayıp fonksiyonunu kullanacağız. \n", - "\n", - "Bu kayıp fonksiyonu tek seçeneğimiz olmasa da, örneğin 'mean_squared_error' kayıp fonksiyonunu da kullanabilirdik, 'binary_crossentropy' kayıp fonksiyonu, olasılık dağılımları (kesin referans ile tahmin edilen olaralık dağılımı) arasındaki farkı ölçerek, olasılık hesaplamaları için daha iyi sonuç verir.\n", - "\n", - "Daha sonra, regrasyon problemlerini incelediğimizde (yani bir evin fiyatını tahmin etmek için), 'mean squared error' gibi diğer kayıp fonksiyonlarını nasıl kullanabileceğimizi göreceğiz.\n", - "\n", - "Şimdi, kayıp fonksiyonu ve optimize ediciyi kullanarak modelimizi yapılandıralım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['acc'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "## Doğrulama veri setini oluşturalım\n", - "\n", - "Eğitim sürecinde, daha önce görmediği veriler ile modelin doğrulunu kontrol etmek isteriz. *Doğrulama veri seti* oluşturmak için eğitim veri seti içerisinden 10,000 yorum ayıralım. (Doğrulama için neden test veri setini şimdi kullanmıyoruz? Bunun nedeni modeli oluşturulması ve düzenlenmesi için sadece eğitim veri setini kullanmak istememizdir. Modelimiz oluşup, eğitildikten sonra, test verisini modelimizin doğruluğunu değerlendirmek için kullanacağız)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} - }, - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## Modelin eğitilmesi\n", - "\n", - "Modeli, her bir mini-batch te 512 yorum örneği olacak şekilde 40 epoch döngüsü ile eğitelim. 'x_train' ve 'y_train' tensorlarını kullanarak tüm yorumları bu 40 iterasyon ile kapsıyoruz. Eğitim süresince, doğrulama veri setini kullanarak modelin kayıp fonksiyon değerini ve doğruluğunu gözlemleyelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} - }, - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## Modeli değerlendirelim\n", - "\n", - "Ve modelin nasıl performans gösterdiğini görelim. Bunun için iki değer kullanacağız. Kayıp (hatayı temsil eden sayı, düşük değerler daha iyi anlamına gelmektedir) ve doğruluk değeri." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "Bu oldukça basit yöntem ile %87 gibi bir doğruluk değeri elde ediyoruz. Daha ileri yöntemler ile modelimiz %95'e kadar çıkan doğruluk sonuçları verebilir." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## Doğruluk ve kayıp değerlerinin zamana göre değişimini veren bir grafik oluşturalım\n", - "\n", - "`model.fit()` methodu eğitim sürecinde olan biten herşeyi görebileceğimiz 'History' sözlük nesnesi oluşturur:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} - }, - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "Grafiğimiz için 4 adet girdimiz mevcut: eğitim ve doğrulama olmak üzere, gözlemlenen metrikler (kayıp ve doğruluk değeri) için birer değer mevcuttur. Bu değerleri, eğitim ve doğrulama kayıplarını, aynı şekilde doğruluk değerlerini karşılaştırmak için grafik üzerine çizdireceğiz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history_dict['acc']\n", - "val_acc = history_dict['val_acc']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# \"bo\", \"mavi nokta\"'nın kısaltmasıdır\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "# b, \"düz mavi çizgi\"'nin kısaltmasıdır\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} - }, - "source": [ - "plt.clf() # grafiğin görüntüsünü temizleyelim\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "\n", - "Grafikte noktalı çizgiler eğitim kayıp ve doğruluk değerlerini temsil etmektedir. Aynı şekilde, düz çizgiler doğrulama kayıp ve doğruluk değerlerini temsil etmektedir.\n", - "\n", - "Eğitim kayıp değerleri her bir epoch iterasyonuyla *düşerken*, eğitim doğruluk değerlerinin *arttığını* görebilirsiniz. Gradient descent optimizasyonu, her bir iterasyonda belirli bir oranda değerleri minimize ettiği için, bu beklenen bir durumdur.\n", - "\n", - "Aynı durum doğrulama kayıp ve doğruluk değerleri için geçerli değildir. Görüldüğü gibi doğrulama değerleri, 20nci epoch iterasyonunda en iyi değerlere ulaşmaktadır. Bu durum aşırı uyuma bir örnektir: modelin eğitim veri kümesiyle, daha önceden hiç görmediği verilere göre daha iyi sonuç vermesi durumu. Bu noktadan sonra model gereğinden fazla optimize edilir ve eğitim veri setine özgü, test verisine genellenemeyen çıkarımları öğrenir.\n", - "\n", - "Örneğimizdeki bu özel durum nedeniyle, gözlemlemiş olduğumuz fazla uyumu giderebilmek için, eğitim işlemini 20nci epoch iterasyonu sonrası durdurabiliriz. Bunu otomatik olarak nasıl yapabileceğimizi daha sonra göreceğiz." - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/keras/overfit_and_underfit.ipynb b/site/tr/r1/tutorials/keras/overfit_and_underfit.ipynb deleted file mode 100644 index 02b686e3c7e..00000000000 --- a/site/tr/r1/tutorials/keras/overfit_and_underfit.ipynb +++ /dev/null @@ -1,709 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "overfit_and_underfit.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "fTFj8ft5dlbS" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fTFj8ft5dlbS" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "lzyBOpYMdp3F", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m_x4KfSJ7Vt7", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C9HmC2T4ld5B" - }, - "source": [ - "# Aşırı uyum (overfitting) ve yetersiz uyum (underfitting) nedir ?" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRTxFhXAlnl1" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4hj2R9kBjERd" - }, - "source": [ - "Note: Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir.\n", - "Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince\n", - "güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en)\n", - "ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek\n", - "için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için\n", - "[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr)\n", - "listesi ile iletişime geçebilirsiniz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19rPukKZsPG6" - }, - "source": [ - "Bu örneğimizde, 'tf.keras' API'sini kullanacağız. Tensorflow 'tf.keras' API'si ile ilgili daha fazla bilgiye [Keras guide](https://www.tensorflow.org/r1/guide/keras) linki üzerinden ulaşabilirsiniz.\n", - "\n", - "Önceki örneklerimizde (film yorumlarının sınıflandırılması, yakıt verimliliğinin tahminlenmesi) gördüğünüz gibi, doğrulama seti üzerinden ölçtüğümüz model doğruluğu, belirli bir epoch sayısında tepe noktasına ulaşıp, sonrasında azalmaya başlamaktadır. \n", - "\n", - "Diğer bir deyişle, modelimiz eğitim veri setine *aşırı uyumlu (overfit)* hale gelmektedir. Aşırı uyum ile nasıl baş edeceğimiz önemlidir. *Eğitim veri seti* ile çok yüksek doğruluk değerine sahip olmamız mümkün olsa bile, asıl amacımız modelin daha önce görmediği veriler ile yani *test veri seti* ile yüksek doğrulukta sonuç üretebilmesi, bu şekilde genelleştirilmesidir.\n", - "\n", - "Aşırı uyumun tersi *yetersiz uyum (underfittig)* 'dur. Yetersiz uyum, test verisi ile değerlendirdiğimizde modelde hala belirli bir iyileşme alanının olması durumudur. Yetersiz uyum belirli nedenler sonucunda oluşur: Eğer model yeterince güçlü değil ise, çok fazla regülarize edildi ise, yeterince uzun süre eğitilmedi ise. Bu, ağın eğitim verisinden ilgili çıkarımları gerektiği kadar yapamadığı anlamına gelmektedir. \n", - "\n", - "Bunun yanında eğer modeli çok uzun süre eğitirseniz, moldel eğitim veri setine aşırı uyum sağlayarak, yaptığı çıkarımlar ile test veri setinde iyi şekilde çalışabilmesi için gerekli olan genelleme yeteneğini kaybedecektir. Modelimizin daha iyi performans göstermesi için dengeyi sağlamalıyız. Bu nedenle, aşağıda inceleyeceğimiz gibi, modelin uygun sayıda epoch ile nasıl eğitileceğini anlamak önemli bir beceridir. \n", - "\n", - "Aşırı uyumu engellemek için en iyi çözüm daha fazla eğitim verisinin kullanılmasıdır. Daha fazla veri ile eğitilen modeller doğal olarak daha iyi genelleme yeteneğine sahip olurlar. Bunun mümkün olmadığı durumlarda, ikinci en iyi çözüm *regularization* tekniklerini kullanmak olur. Bu teknikleri kullandığımızda, modelimizin sahip olacağı bilgi tipi ve miktarını sınırlarız. Model ağımız daha az miktarda çıkarımı hafızasına alırsa, optimizasyon süreci en önemli çıkarımları yapması için modeli zorlayacaktır. Bunun sonucunda modelin genelleme yeteneği artacaktır. \n", - "\n", - "Bu notebook'ta, IMBD film yorumlarının sınıflandırılması modelimizin iyileştirilmesi için en çok kullanılan iki regularizasyon tekniği olan ağırlık regularizasyonu (weigth regularization) ve dropout teknikleri kullanılacaktır." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OP5sztkoETj0", - "colab": {} - }, - "source": [ - "# keras.datasets.imdb is broken in 1.13 and 1.14, by np 1.16.3\n", - "!pip install tf_nightly" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5pZ8A2liqvgk", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1cweoTiruj8O" - }, - "source": [ - "## IMDB veri setini indirelim\n", - "\n", - "Önceki notebook'ta yer alan `emmbeding` işlemi yerine, bu örnekte cümlelere `multi-hot encoding` işlemi uygulayacağız. Modelimiz hızlı bir şekilde eğitim verisine aşırı uyum gösterecektir. Bu şekilde, aşırı uyumun nasıl oluştuğunu ve nasıl baş edeceğimizi göstermiş olacağız. \n", - "\n", - "`Multi-hot-encoding` ile listemizi 0'lar ve 1'ler içeren vektöre dönüştürelim. Örnek olarak bu işlem ile `[3, 5]` dizisi, 3 ve 5 indisleri 1 olan bunun dışındaki tüm değerleri 0 olan 10,000 boyutlu bir vektöre çevrilir. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QpzE4iqZtJly", - "colab": {} - }, - "source": [ - "NUM_WORDS = 10000\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = keras.datasets.imdb.load_data(num_words=NUM_WORDS)\n", - "\n", - "def multi_hot_sequences(sequences, dimension):\n", - " # Create an all-zero matrix of shape (len(sequences), dimension)\n", - " results = np.zeros((len(sequences), dimension))\n", - " for i, word_indices in enumerate(sequences):\n", - " results[i, word_indices] = 1.0 # set specific indices of results[i] to 1s\n", - " return results\n", - "\n", - "\n", - "train_data = multi_hot_sequences(train_data, dimension=NUM_WORDS)\n", - "test_data = multi_hot_sequences(test_data, dimension=NUM_WORDS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MzWVeXe3NBTn" - }, - "source": [ - "Sonuçta oluşan `multi-hot` vektörlerden birine bakalım. Kelime indisleri frekanslarına göre sıralanmıştır, yani aşağıdaki grafikte görüldüğü gibi 0 indeksine yaklaştıkça daha fazla 1 değeri görürüz: " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "71kr5rG4LkGM", - "colab": {} - }, - "source": [ - "plt.plot(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lglk41MwvU5o" - }, - "source": [ - "## Aşırı uyumu açıklayalım\n", - "\n", - "Aşırı uyumu engellemenin en basit yolu model boyutunun küçültülmesidir. Model boyutunu, modelin katman sayısına ve katmanları içersinde yere alan birim sayısına bağlı olan öğrenebilir parametrelerin sayısı belirler. Derin öğrenmede, modelde yer alan öğrenebilir parametrelerin sayısı modelin \"kapasitesi\" belirlemektedir. Daha basit haliyle, daha fazla parametreye sahip model, daha fazla \"hafıza kapasitesine\" sahiptir. Bu nedenle, model kusursuz bir şekilde eğitim verisine uyum sağlayacaktır. Genelleme yeteneğine sahip olmayan bu modelin, daha önce görmediği veriler ile başarılı bir şekilde tahminleme yapması mümkün değildir. \n", - "\n", - "Her zaman aklımızda tutmamız gerekir: derin öğrenme modelleri eğitim verisine uyum sağlama eğilimindedirler, asıl başarılması gereken bu uyum değil modele genelleme yeteneği kazandırabilmektir. \n", - "\n", - "Buna karşın, eğer ağın hafızaya alma kaynakları kısıtlı ise, öğrenmesi kolay olmayacaktır. Eğitim sürecinde kayıp değerini küçültmek ve daha fazla tahminleme gücü kazanmak için, modelin sıkıştırılmış çıkarımlar oluşturması gerekecektir. Bununla birlikte, modeli küçük oluşturusak eğitim veri setine uyum sağlamakta zorluk çekecektir. \"Çok fazla kapasite\" ile \"olması gerektiğinden az kapasite\" arasında bir denge oluşturulması gerekmektedir.\n", - "\n", - "Ne yazık ki, doğru büyüklükte bir model yapısını (modeldeki katman sayısı ve katmanların büyüklüğü) belirleyebileceğimiz sihirli bir formül bulunmamaktadır. Farklı model yapılarını deneyerek, problemimizin çözümü için en uygun modeli belirlememiz gerekmektedir. \n", - "\n", - "Uygun model boyutunu belirlemek için, göreceli olarak az katman ve parametreye sahip bir yapıyla başlamak en iyisidir. Doğrulama kayıp değerlerinde azalma görmemeye başladığımız zamana kadar, modelin katman sayısını veya katmanlar içerisindeki birimleri artırabiliriz. Film yorumlarının sınıflandırılmasını sağlayan ağımızda bunu deneyelim: \n", - "\n", - "Referans olarak, sadece ```Yoğun-Dense``` katmanlar içeren basit bir modelle başlayacağız. Sonrasında bu modelin daha küçük ve büyük versiyonlarını oluşturarak sonuçları karşılaştıracağız." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ReKHdC2EgVu" - }, - "source": [ - "### Referans modeli oluşturalım" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QKgdXPx9usBa", - "colab": {} - }, - "source": [ - "baseline_model = keras.Sequential([\n", - " # `input_shape` is only required here so that `.summary` works. \n", - " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "baseline_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "baseline_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LqG3MXF5xSjR", - "colab": {} - }, - "source": [ - "baseline_history = baseline_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L-DGRBbGxI6G" - }, - "source": [ - "### Daha küçük bir model oluşturalım" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SrfoVQheYSO5" - }, - "source": [ - "Yukarıda oluşturduğumuz referans modele göre daha az gizli birim içeren modelimizi oluşturalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jksi-XtaxDAh", - "colab": {} - }, - "source": [ - "smaller_model = keras.Sequential([\n", - " keras.layers.Dense(4, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(4, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "smaller_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "smaller_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jbngCZliYdma" - }, - "source": [ - "Ve modelimizi aynı veriler ile eğitelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ofn1AwDhx-Fe", - "colab": {} - }, - "source": [ - "smaller_history = smaller_model.fit(train_data,\n", - " train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vIPuf23FFaVn" - }, - "source": [ - "### Daha büyük bir model oluşturalım\n", - "\n", - "Daha büyük bir model oluşturabilir ve nasıl hızlı bir şekilde aşırı uyum gösterdiğini görebiliriz. Sonrasında, problem için gereğinden fazla kapasiteye sahip bu modeli, karşılaştıma grubumuza ekleyelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ghQwwqwqvQM9", - "colab": {} - }, - "source": [ - "bigger_model = keras.models.Sequential([\n", - " keras.layers.Dense(512, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(512, activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "bigger_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "bigger_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D-d-i5DaYmr7" - }, - "source": [ - "Ve yeniden, aynı veriler ile modeli eğitelim: " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "U1A99dhqvepf", - "colab": {} - }, - "source": [ - "bigger_history = bigger_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fy3CMUZpzH3d" - }, - "source": [ - "### Eğitim ve doğrulama kayıp değerlerini grafikte gösterelim\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HSlo1F4xHuuM" - }, - "source": [ - "Grafikteki düz çizgiler eğitim kayıp değerini, kesikli çizgiler ise doğrulama kayıp değerini (düşük kayıp değerlerinin daha iyi modelin göstergesi olduğunu hatırlayalım) göstermektedir. Küçük modelimizin referans modele göre daha geç (4ncü epoch yerine 6ncı epochta) aşırı uyum gösterdiğini görebiliriz. Bunun yanında küçük modelimizin performansı, aşırı uyum sonrasında daha yavaş düşmektedir. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0XmKDtOWzOpk", - "colab": {} - }, - "source": [ - "def plot_history(histories, key='binary_crossentropy'):\n", - " plt.figure(figsize=(16,10))\n", - " \n", - " for name, history in histories:\n", - " val = plt.plot(history.epoch, history.history['val_'+key],\n", - " '--', label=name.title()+' Val')\n", - " plt.plot(history.epoch, history.history[key], color=val[0].get_color(),\n", - " label=name.title()+' Train')\n", - "\n", - " plt.xlabel('Epochs')\n", - " plt.ylabel(key.replace('_',' ').title())\n", - " plt.legend()\n", - "\n", - " plt.xlim([0,max(history.epoch)])\n", - "\n", - "\n", - "plot_history([('baseline', baseline_history),\n", - " ('smaller', smaller_history),\n", - " ('bigger', bigger_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Bi6hBhdnSfjA" - }, - "source": [ - "Büyük modelimizin neredeyse hemen, ilk eopch sonrası, aşırı uyum gösterdiğini görüyoruz. Ağımızın kapasitesi arttıkça, modelin uyum gösterme hızı artıyor, bunun sonucunda eğitim kayıpları hızlıca azalıyor. Bunun yanında modelimiz eğitim verisine aşırı uyum göstermeye daha yatkın hale geliyor, bundan dolayı eğitim ve doğrulama kayıpları arasında büyük fark görüyoruz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ASdv7nsgEFhx" - }, - "source": [ - "## Stratejiler" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4rHoVWcswFLa" - }, - "source": [ - "### Modelimize ağırlık regülarizasyonu (weigth regularization) ekleyelim\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kRxWepNawbBK" - }, - "source": [ - "Occam's Razor prensibini biliyor olabilirsiniz: bu prensibe göre bir konuyla ilgili iki farklı açıklamadan doğru olması en olası seçenek, \"en basit\" açıklamadır yani konuyla ilgili en az varsayım içeren açıklamadır. Bu prensip, sinir ağ modelleri içinde geçerlidir: çeşitli eğitim veri setleri ve model yapıları göz önüne alındığında, veriyi açıklayan farklı ağırlık değer setleri (farklı modeller) mevcuttur ve bunların arasından en basit olanlar kompleks modellere göre daha az aşırı uyum olasılığı taşımaktadır.\n", - "\n", - "Bu bağlamda \"basit model\", parametre değer dağılımının düzensizliğinin (entropy) az olduğu modeldir (veya yukarıdaki bölümde gördüğümüz gibi tümüyle daha az parametreye sahip modeldir). Bu nedenle aşırı uyumu engellemenin bir yoluda, ağırlıklarının küçük değerler almasına zorlayarak modelin karmaşıklığına sınırlama getirmektir. Bunun sonucunda ağırlık değerlerinin dağılımı daha \"düzenli\" olur. Buna \"ağırlık regülarizasyonu\" denir ve uygulamada büyük ağırlık değerleri için modelin kayıp fonksiyonuna ilave bir ceza değeri eklenmesiyle gerçekleştirilir. Bu ceza değerini iki farklı şekilde oluşturabiliriz: \n", - "\n", - "* [L1 regülarizasyonu](https://developers.google.com/machine-learning/glossary/#L1_regularization), bu yöntemde ceza değerleri, ağırlıkların mutlak değerleriyle orantılı olarak eklenir (buna ağırlıkların \"L1 normu\" denir).\n", - "\n", - "* [L2 regülarizasyonu](https://developers.google.com/machine-learning/glossary/#L2_regularization), bu yöntemde ceza değerleri, ağırlık katsayı değerlerinin karesiyle orantılı şekilde eklenir (buna ağırlıkların \"L2 normu\" denir). L2 regüarizasyonu aynı zamanda weigth decay olarakta adlandırılır. Farklı adlandırmalar karışıklığa neden olabilir: weigth decay ile L2 regularization aslında matematiksel olarak aynı işlemlerdir. \n", - "\n", - "`tf.keras`'ta bu işlem, katmanlara ağırlık regülarizasyon parametre değerlerinin girilmesi ile uygulanır. Şimdi, modelimize L2 ağırlık regülarizasyonunu birlikte ekleyelim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HFGmcwduwVyQ", - "colab": {} - }, - "source": [ - "l2_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dense(16, kernel_regularizer=keras.regularizers.l2(0.001),\n", - " activation=tf.nn.relu),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "l2_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy', 'binary_crossentropy'])\n", - "\n", - "l2_model_history = l2_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bUUHoXb7w-_C" - }, - "source": [ - "```l2(0.001)``` 'nin anlamı şudur; katmanın ağırlık matrisi içerisindeki her bir katsayı, ağın toplam kayıp değerine ```0.001 * weight_coefficient_value**2``` ceza değerini ekleyecektir. Bu ceza değeri sadece eğitim sürecinde ekleneceği için, kayıp değerleri eğitim sürecinde test sürecine göre çok daha büyük olacaktır. \n", - "\n", - "L2 regülarizasyon ceza değerinin sonuçlara etkisini aşağıda görebiliriz:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7wkfLyxBZdh_", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('l2', l2_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Kx1YHMsVxWjP" - }, - "source": [ - "Göreceğiniz gibi, L2 regülarizasyonunun uygulandığı model, referans modelimiz ile aynı sayıda parametreye sahip olmasına karşın, aşırı uyuma çok daha dirençli bir hale geldi." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HmnBNOOVxiG8" - }, - "source": [ - "### Modelimize 'dropout' ekleyelim\n", - "\n", - "Dropout, Toronto Üniversitesi'nden Hilton ve öğrencileri tarafından geliştirilmiş olan bir yöntemdir. Sinir ağları için en etkin ve en yaygın kullanılan regülarizasyon yöntemlerinden biridir. Dropout, eğitim sürecinde katmanlara ait rastgele seçilen belirli sayıdaki çıktı özelliğinin \"bırakılması\" (değerinin sıfıra eşitlenmesiyle) anlamına gelmektedir. Örneğin, ele aldığımız katmanın eğitim sürecinde beslenen girdi değeri ile çıktı olarak [0.2, 0.5, 1.3, 0.8, 1.1] vektörü oluşturduğunu varsayalım; dropout uygulandıktan sonra bu vektör gösterildiği şekliyle rastgele dağılmış olarak çeşitli sıfır değerleri içerecektir; [0, 0.5, \n", - "1.3, 0, 1.1]. \"Dropout oranı\" sıfırlanacak özelliklerin oranıdır; genelde 0.2 ile 0.5 arasında bir değer alır. Test sürecinde, daha fazla aktif birime sahip olmak için hiç bir birim sıfırlanmaz, bunun yerine dengeyi korumak için katmanların çıktı değerleri droput oranı ile aynı oranda küçültülür.\n", - "\n", - "tf.keras API'si Dropout katmanıyla, modelimize dropout işlemini uygulayabiliriz. \n", - "\n", - "Aşırı uyumu ne oranda azalttığını değerlendirebilmek için, IMDB ağına iki adet Dropout katmanı ekleyelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OFEYvtrHxSWS", - "colab": {} - }, - "source": [ - "dpt_model = keras.models.Sequential([\n", - " keras.layers.Dense(16, activation=tf.nn.relu, input_shape=(NUM_WORDS,)),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(16, activation=tf.nn.relu),\n", - " keras.layers.Dropout(0.5),\n", - " keras.layers.Dense(1, activation=tf.nn.sigmoid)\n", - "])\n", - "\n", - "dpt_model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy','binary_crossentropy'])\n", - "\n", - "dpt_model_history = dpt_model.fit(train_data, train_labels,\n", - " epochs=20,\n", - " batch_size=512,\n", - " validation_data=(test_data, test_labels),\n", - " verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SPZqwVchx5xp", - "colab": {} - }, - "source": [ - "plot_history([('baseline', baseline_history),\n", - " ('dropout', dpt_model_history)])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gjfnkEeQyAFG" - }, - "source": [ - "Droput eklenmesi, referans modele göre açık bir iyileşme sunmaktadır.\n", - "\n", - "\n", - "Tekrardan özetleyelim: sinir ağlarında aşırı uyumu engellemek için yaygın olarak kullanılan yöntemler aşağıda listelenmiştir:\n", - "\n", - "* Daha fazla eğitim verisi kullanarak.\n", - "* Ağın kapasitesini düşürerek.\n", - "* Ağırlık regülarizasyonu ekleyerek.\n", - "* Droput ekleyerek.\n", - "\n", - "Ve bu notebook'ta yer almayan diğer iki önemli yöntem ise: data-augmentation ve batch normalization yöntemleridir." - ] - } - ] -} \ No newline at end of file diff --git a/site/tr/r1/tutorials/keras/save_and_restore_models.ipynb b/site/tr/r1/tutorials/keras/save_and_restore_models.ipynb deleted file mode 100644 index f122478d6da..00000000000 --- a/site/tr/r1/tutorials/keras/save_and_restore_models.ipynb +++ /dev/null @@ -1,860 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "save_and_restore_models.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pZJ3uY9O17VN" - }, - "source": [ - "# Modelleri kaydedelim ve tekrar yükleyelim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M4Ata7_wMul1" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UVRUNY8NjYMK", - "colab_type": "text" - }, - "source": [ - "Note: Bu dökümanlar TensorFlow gönüllü kullanıcıları tarafından çevirilmiştir.\n", - "Topluluk tarafından sağlananan çeviriler gönüllülerin ellerinden geldiğince\n", - "güncellendiği için [Resmi İngilizce dökümanlar](https://www.tensorflow.org/?hl=en)\n", - "ile bire bir aynı olmasını garantileyemeyiz. Eğer bu tercümeleri iyileştirmek\n", - "için önerileriniz var ise lütfen [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "havuzuna pull request gönderin. Gönüllü olarak çevirilere katkıda bulunmak için\n", - "[docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr)\n", - "listesi ile iletişime geçebilirsiniz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBdde4YJeJKF" - }, - "source": [ - "Eğitim sırasında veya sonrasında modelin ilerleyişi kaydedilebilir. Bunun anlamı, modelin kaldığı yerden ilerlemeye devam edebilir olması, ayrıca uzun eğitim sürelerinin önüne geçilebilir olmasıdır. Modellerin kaydedilmesi, aynı zamanda modellerin paylaşılabilmesi ve diğerlerinin yaptığımız çalışmaları tekrardan oluşturabilmeleri anlamına gelmektedir. Çoğu uygulayıcı araştırma modellerini ve tekniklerini yayınladıklarında, aşağıdaki bilgileri paylaşırlar: \n", - "\n", - "* modeli oluşturan kodu\n", - "* modele ait eğitilmiş ağırlık değerlerini veya parametrelerini\n", - "\n", - "Bu verilerin paylaşılması, diğerlerinin modelimizin nasıl çalıştığını anlamasına ve yeni veriler ile modeli denemelerine yardımcı olur. \n", - "\n", - "Dikkat: Güvenilmeyen kodlar ile ilgili dikkatli olunuz-Tensorflow modelleri kodlardan oluşmaktadır. Detaylar için [TensorFlow'un Güvenli Kullanımı](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) linkine göz atınız.\n", - "\n", - "### Seçenekler\n", - "\n", - "Kullandığınız API'ye bağlı olarak, Tensorflow modellerini kaydetmenin farklı yolları vardır. Bu eğitim dökümanı, Tensorflowda yapay zeka modellerinin oluşturulması ve eğitilmesinde kullanılan [tf.keras](https://www.tensorflow.org/r1/guide/keras) 'ı kullanmaktadır. Farklı yöntemler için TensorFlow [Kaydet ve Geri Yükle](https://www.tensorflow.org/r1/guide/saved_model) eğitim dökümanına veya [eager'da kaydedelim](https://www.tensorflow.org/r1/guide/eager#object-based_saving) linkine göz atailirsiniz.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xCUREq7WXgvg" - }, - "source": [ - "## Kurulum\n", - "\n", - "### Kuralım ve İçeri Alalım" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7l0MiTOrXtNv" - }, - "source": [ - "Tensorflow ve bağlı kütüphanelerini kuralım ve içeri alalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} - }, - "source": [ - "!pip install h5py pyyaml " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbGsznErXWt6" - }, - "source": [ - "### Örnek veri setini alalım\n", - "\n", - "Ağırlıkların nasıl kaydedildiğini gösterebilmek için, modelimizi [MNIST dataset](http://yann.lecun.com/exdb/mnist/) verisi ile eğiteceğiz. Süreci hızlandırmak için, sadece ilk 1000 örneği kullanacağız:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "tf.__version__" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_labels = train_labels[:1000]\n", - "test_labels = test_labels[:1000]\n", - "\n", - "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", - "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "anG3iVoXyZGI" - }, - "source": [ - "### Modeli tanımlayalım" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wynsOBfby0Pa" - }, - "source": [ - "Ağırlıkların nasıl kaydedileceğini ve yükleneceğini gösterebilmek için basit bir model oluşturalım." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} - }, - "source": [ - "# Returns a short sequential model\n", - "def create_model():\n", - " model = tf.keras.models.Sequential([\n", - " keras.layers.Dense(512, activation=tf.keras.activations.relu, input_shape=(784,)),\n", - " keras.layers.Dropout(0.2),\n", - " keras.layers.Dense(10, activation=tf.keras.activations.softmax)\n", - " ])\n", - " \n", - " model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - " \n", - " return model\n", - "\n", - "\n", - "# Create a basic model instance\n", - "model = create_model()\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "soDE0W_KH8rG" - }, - "source": [ - "## Eğitim sırasında kontrol noktalarını (checkpoints) kaydedelim" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRyd5qQQIXZm" - }, - "source": [ - "Ana kullanım şekli, eğitim sırasında ve sonunda kontrol noktalarının otomatik olarak kaydedilmesidir. Bu şekilde eğitilmiş modeli tekrar eğitmeye gerek kalmadan kullanabiliriz veya eğitim süreci yarıda kalmışsa kaldığı yerden eğitime devam edebiliriz. \n", - "\n", - "`tf.keras.callbacks.ModelCheckpoint` bu işlemi yapan callback fonksiyonudur. Bu fonksiyon, kontrol noktalarını yapılandırmak için birkaç parametre değeri alır. \n", - "\n", - "### Kontrol noktası callback fonksiyonu kullanımı\n", - "\n", - "Modeli eğitelim ve `ModelCheckpoint` callback fonksiyonunu modele aktaralım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} - }, - "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# Create checkpoint callback\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(checkpoint_path, \n", - " save_weights_only=True,\n", - " verbose=1)\n", - "\n", - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs = 10, \n", - " validation_data = (test_images,test_labels),\n", - " callbacks = [cp_callback]) # pass callback to training\n", - "\n", - "# This may generate warnings related to saving the state of the optimizer.\n", - "# These warnings (and similar warnings throughout this notebook)\n", - "# are in place to discourage outdated usage, and can be ignored." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlM-sgyJO084" - }, - "source": [ - "Bu kod, her bir ephoc sonunda güncellenen, bir grup TensorFlow kontrol noktası dosyası oluşturur:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} - }, - "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wlRN_f56Pqa9" - }, - "source": [ - "Eğitilmemiş yeni bir model oluşturalım. Sadece ağırlık değerleri ile bir modeli geri yüklemek istediğimizde, elimizde orjinal model ile aynı yapıyı sahip bir modelimiz olmalıdır. Aynı model yapısına sahip olduğumuz için, farklı zamanlarda oluşmuş model örnekleri arasında ağırlık değerlerini paylaşabiliriz. \n", - "\n", - "Şimdi yeni bir eğitilmemiş model oluşturalım ve bu modeli test veri seti ile değerlendirelim. Eğitilmemiş bir model, şans yüzdesi kadar (~10% doğruluk) doğruluğa sahip olacaktır:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Untrained model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1DTKpZssRSo3" - }, - "source": [ - "Sonrasında ağırlık değerlerini kaydettiğimiz kontrol noktasından model geri yükleyelim ve modeli tekrardan değerlendirelim: " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} - }, - "source": [ - "model.load_weights(checkpoint_path)\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpAbKkAyVPV8" - }, - "source": [ - "### Kontrol noktası callback seçenekleri:\n", - "\n", - "Callback fonksiyonu, kontrol noktalarının isimlendirilmesi ve frekanslarının ayarlanması için çeşitli seçenekler sunar. \n", - "\n", - "Yeni bir modeli eğitelim ve her 5 epoch'ta bir farklı isimler ile kontrol noktalarını isimlendirelim:\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} - }, - "source": [ - "# include the epoch in the file name. (uses `str.format`)\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " checkpoint_path, verbose=1, save_weights_only=True,\n", - " # Save weights, every 5-epochs.\n", - " period=5)\n", - "\n", - "model = create_model()\n", - "model.save_weights(checkpoint_path.format(epoch=0))\n", - "model.fit(train_images, train_labels,\n", - " epochs = 50, callbacks = [cp_callback],\n", - " validation_data = (test_images,test_labels),\n", - " verbose=0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1zFrKTjjavWI" - }, - "source": [ - "Şimdi, oluşan kontrol noktalarına bakalım ve en güncel olanını seçelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} - }, - "source": [ - "! ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} - }, - "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", - "latest" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zk2ciGbKg561" - }, - "source": [ - "Not: Tensorflow varsayılan formatı, sadece en güncel 5 kontrol noktasını kaydeder.\n", - "\n", - "Test için, modeli resetleyelim ve en güncel kontrol noktasını yükleyelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "model.load_weights(latest)\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c2OxsJOTHxia" - }, - "source": [ - "## Bu dosyalar nedir?" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtdYhvWnH2ib" - }, - "source": [ - "Yukardaki kod, ağırlık değerlerini bir grup [checkpoint](https://www.tensorflow.org/r1/guide/saved_model#save_and_restore_variables)- dosyaya binary formatta kaydeder. Kontrol noktası aşağıdakileri kapsar: \n", - "* Modele ait ağırlık değerlerini içeren, bir veya daha fazla dosya parçası (shards). \n", - "* Hangi ağırlık değerinin hangi dosya parçasında olduğunu gösteren bir index dosyası. \n", - "\n", - "Eğer modelinizi tek bir bilgisayarda eğitiyorsanız, son takısı `.data-00000-of-00001` olan tek bir dosya parçası oluşacaktır." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S_FA-ZvxuXQV" - }, - "source": [ - "## Ağırlıkların manuel kaydedilmesi\n", - "\n", - "Yukarıda, ağırlıkların modele nasıl yüklendiğini gördünüz.\n", - "\n", - "`Model.save_weights` methodunu kullanarak, ağırlıkların manuel olarak kaydedilmeside aynı şeklide kolaydır." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} - }, - "source": [ - "# Save the weights\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# Restore the weights\n", - "model = create_model()\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", - "\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kOGlxPRBEvV1" - }, - "source": [ - "## Tüm modelin kaydedilmesi\n", - "\n", - "Ağırlık değerleri, model yapısı ve hatta optimize edici parametrelerini (yapılandırmaya bağlı olarak) kapsayan tek bir dosya ile tüm model kaydedilebilir. Bu, orjinal koda ulaşmaya gerek kalmadan, modele ait kontrol noktasını kaydetmeyi ve sonrasında eğitime kalındığı yerden tekrardan başlanmasını sağlar. \n", - "\n", - "Tam-fonksiyonel modelin kaydedilmesi çok faydalıdır, bu modeli TensorFlow.js ye ([HDF5](https://js.tensorflow.org/r1/tutorials/import-keras.html), [Saved Model](https://js.tensorflow.org/r1/tutorials/import-saved-model.html)) yükleyip sonrasında eğitebilir ve web browserda çalıştırabiliriz. Veya modeli TensorFlow Lite formatına dönüştürerek ([HDF5](https://www.tensorflow.org/lite/convert/python_api#exporting_a_tfkeras_file_), [Saved Model](https://www.tensorflow.org/lite/convert/python_api#exporting_a_savedmodel_)) mobil cihazlarda çalıştırabiliriz." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" - }, - "source": [ - "### Modeli HDF5 dosyası olarak kaydedelim\n", - "\n", - "Keras, [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) standartını kullanarak, temel bir kayıt formatı sunar. Bizim kullanım amacımıza göre, kayıtlı model tek bir binary blob olarak değerlendirilebilir. " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)\n", - "\n", - "# Save entire model to a HDF5 file\n", - "model.save('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" - }, - "source": [ - "Şimdi, bu dosya ile modelimizi tekrardan oluşturalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} - }, - "source": [ - "# Recreate the exact same model, including weights and optimizer.\n", - "new_model = keras.models.load_model('my_model.h5')\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" - }, - "source": [ - "Doğruluğunu kontrol edelim:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} - }, - "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "Bu yöntem modelle ilgili herşeyi kaydeder:\n", - "\n", - "* Ağırlık değerlerini\n", - "* Model yapısını\n", - "* Optimizer parametrelerini\n", - "\n", - "Keras, modelleri kaydederken model yapılarını inceler ve TensorFlow optimizer'ları (`tf.train`'dan) kaydetmesi halihazırda mümkün değildir. Bunu kullanırken, modeli yükledikten sonra tekrar derlememiz gerekecektir ve optimizer'ın son durum bilgisini kaybetmiş oluruz. \n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPyhgcoVzqUB" - }, - "source": [ - "### Modelin `saved_model` olarak kaydedilmesi" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LtcN4VIb7JkK" - }, - "source": [ - "Dikkat: `tf.keras` modellerinin bu yöntemle kaydedilmesi deneysel olup, gelecek versiyonlarda değişiklik gösterebilir. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DSWiSB0Q8c46" - }, - "source": [ - "Yeni ir model oluşturalım:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iUvT_3qE8hV5" - }, - "source": [ - "`saved_model` oluşturalım: " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "colab": {} - }, - "source": [ - "saved_model_path = tf.contrib.saved_model.save_keras_model(model, \"./saved_models\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MjpmyPfh8-1n" - }, - "source": [ - "Kaydedilmiş modeller zaman bilgisini içeren klasörler içerisine oluşturulur:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZtOvxA7V0iTv", - "colab": {} - }, - "source": [ - "!ls saved_models/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B7qfpvpY9HCe" - }, - "source": [ - "Kaydedilmiş modeli kullanarak yeni bir keras modelini yükleyelim." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} - }, - "source": [ - "new_model = tf.contrib.saved_model.load_keras_model(saved_model_path)\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" - }, - "source": [ - "Yeniden yüklenmiş modeli çalıştıralım." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Pc9e6G6w1AWG", - "colab": {} - }, - "source": [ - "# The model has to be compiled before evaluating.\n", - "# This step is not required if the saved model is only being deployed.\n", - "\n", - "new_model.compile(optimizer=tf.keras.optimizers.Adam(), \n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " metrics=['accuracy'])\n", - "\n", - "# Evaluate the restored model.\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eUYTzSz5VxL2" - }, - "source": [ - "## Sırada ne var? \n", - "\n", - "Bu yardımcı döküman, `tf.keras` ile modellerin kaydedilmesi ve yüklenmesini içeren temel bilgiler sunar. \n", - "\n", - "* [tf.keras guide](https://www.tensorflow.org/r1/guide/keras) modellerin `tf.keras` ile kaydedilip yüklenmesiyle ilgili daha fazla bilgi içerir. \n", - "\n", - "* Eager çalışma sırasında modellerin kaydedilmesiyle ilgi bilgi için [Saving in eager](https://www.tensorflow.org/r1/guide/eager#object_based_saving) linkine göz atabilirsiniz.\n", - "\n", - "* [Save and Restore](https://www.tensorflow.org/r1/guide/saved_model) ise TensorFlow'da modellerin kaydedilmesiyle ilgili düşük-seviye detay bilgileri içerir." - ] - } - ] -} \ No newline at end of file diff --git a/site/vi/README.md b/site/vi/README.md deleted file mode 100644 index 3f3e4d8f63c..00000000000 --- a/site/vi/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Tài liệu TensorFlow - -## Bản dịch cộng đồng - -Cộng đồng TensorFlow Việt Nam đã dịch các tài liệu này từ nguyên bản tiếng Anh. -Vì bản dịch này dựa trên sự cố gắng từ các tình nguyện viên, -nên không thể đám bảo luôn bám sát -[Tài liệu chính thức bằng tiếng Anh](https://www.tensorflow.org/?hl=en). -Nếu bạn có đề xuất để cải thiện bản dịch này, vui lòng tạo PR đến repository trên GitHub của -[tensorflow/docs](https://github.com/tensorflow/docs) - -Để đăng ký dịch hoặc duyệt lại nội dung bản dịch, các bạn hãy liên hệ -[docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -## Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. - -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -# Tracking spreadsheet - -[This spreadsheet](https://docs.google.com/spreadsheets/d/1Qr104JJEJluj2Hg0kFuY74MhAXMYjDWVyVztZstwdzU/edit?usp=sharing) -is used for tracking the progress of VI translation, -and a the glossary to standardize the usage of the translated -(or keeping the original) of jargons in ML and Tensorflow. diff --git a/site/vi/REVIEWERS b/site/vi/REVIEWERS deleted file mode 100644 index b96c1f5f107..00000000000 --- a/site/vi/REVIEWERS +++ /dev/null @@ -1,9 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/vi directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs -truongsinh -truongnh1992 -anniehiehie diff --git a/site/vi/tutorials/keras/classification.ipynb b/site/vi/tutorials/keras/classification.ipynb deleted file mode 100644 index 2cc2dd0af66..00000000000 --- a/site/vi/tutorials/keras/classification.ipynb +++ /dev/null @@ -1,1044 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "classification.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "vasWnqRgy1H4", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# Phân loại cơ bản: Dự đoán ảnh quần áo giày dép" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Xem trên TensorFlow.org\n", - " \n", - " Chạy trên Google Colab\n", - " \n", - " Xem mã nguồn trên GitHub\n", - " \n", - " Tải notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "38FdzGHfR4WQ", - "colab_type": "text" - }, - "source": [ - "Note: Cộng đồng TensorFlow Việt Nam đã dịch các tài liệu này từ nguyên bản tiếng Anh.\n", - "Vì bản dịch này dựa trên sự cố gắng từ các tình nguyện viên, nên không thể đám bảo luôn bám sát\n", - "[Tài liệu chính thức bằng tiếng Anh](https://www.tensorflow.org/?hl=en).\n", - "Nếu bạn có đề xuất để cải thiện bản dịch này, vui lòng tạo PR đến repository trên GitHub của [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "\n", - "Để đăng ký dịch hoặc duyệt lại nội dung bản dịch, các bạn hãy liên hệ \n", - "[docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "Trong hướng dẫn này, chúng ta sẽ huấn luyện một mô hình mạng neuron để phân loại các hình ảnh về quần áo và giày dép.\n", - "\n", - "Đừng ngại nếu bạn không hiểu hết mọi chi tiết, vì chương trình trong hướng dẫn này là một chương trình TensorFlow hoàn chỉnh, và các chi tiết sẽ dần được giải thích ở những phần sau.\n", - "\n", - "Hướng dẫn này dùng [tf.keras](https://www.tensorflow.org/guide/keras), một API cấp cao để xây dựng và huấn luyện các mô hình trong TensorFlow." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jL3OqFKZ9dFg", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# TensorFlow and tf.keras\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# Helper libraries\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yR0EdgrLCaWR" - }, - "source": [ - "## Import tập dữ liệu về quần áo và giày dép từ Fashion MNIST" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DLdCchMdCaWQ" - }, - "source": [ - "Chúng ta sẽ dùng tập dữ liệu về quần áo và giày dép từ [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist), chứa khoảng 70,000 ảnh đen trắng phân thành 10 loại. Mỗi một ảnh là một loại quần áo hoặc giày dép với độ phân giải thấp (28 by 28 pixel), như hình minh hoạ bên dưới:\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
     \n", - "
    \n", - "\n", - "Fashion MNIST là tập dữ liệu được dùng để thay thế cho tập dữ liệu [MNIST](http://yann.lecun.com/exdb/mnist/) kinh điển thường dùng cho các chương trình \"Hello, World\" của machine learning trong lĩnh vực thị giác máy tính. Tập dữ liệu kinh điển vừa đề cập gồm ảnh của các con số (ví dụ 0, 1, 2) được viết tay. Các ảnh này có cùng định dạng tệp và độ phân giải với các ảnh về quần áo và giầy dép chúng ta sắp dùng.\n", - "\n", - "Hướng dẫn này sử dụng tập dữ liệu Fashion MNIST, vì đây là một bài toán tương đối phức tạp hơn so với bài toán nhận diện chữ số viết tay. Cả 2 tập dữ liệu (Fashion MNIST và MNIST kinh điển) đều tương đối nhỏ và thường dùng để đảm bảo một giải thuật chạy đúng, phù hợp cho việc kiểm thử và debug.\n", - "\n", - "Với tập dữ liệu này, 60.000 ảnh sẽ được dùng để huấn luyện và 10.000 ảnh sẽ đường dùng để đánh giá khả năng phân loại nhận diện ảnh của mạng neuron. Chúng ta có dùng tập dữ liệu Fashion MNIST trực tiếp từ TensorFlow như sau:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t9FDsUlxCaWW" - }, - "source": [ - "Tập dữ liệu sau khi được tải sẽ trả về 4 mảng NumPy:\n", - "\n", - "* 2 mảng `train_images` và `train_labels` là *tập huấn luyện*. Mô hình sẽ học từ dữ liệu của 2 mảng này.\n", - "* 2 mảng `test_images` vả `test_labels` là *tập kiểm thử*. Sau khi mô hình được huấn luyện xong, chúng ta sẽ chạy thử mô hình với dữ liệu đầu vào từ `test_images` để lấy kết quả, và so sánh kết quả đó với dữ liệu đối ứng từ `test_labels` để đánh giá chất lượng của mạng neuron.\n", - "\n", - "Mỗi ảnh là một mảng NumPy 2 chiều, 28x28, với mỗi pixel có giá trị từ 0 đến 255. *Nhãn* là một mảng của các số nguyên từ 0 đến 9, tương ứng với mỗi *lớp* quần áo giày dép:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    NhãnLớp
    0Áo thun
    1Quần dài
    2Áo liền quần
    3Đầm
    4Áo khoác
    5Sandal
    6Áo sơ mi
    7Giày
    8Túi xách
    9Ủng
    \n", - "\n", - "Mỗi ảnh sẽ được gán với một nhãn duy nhất. Vì tên của mỗi lớp không có trong tập dữ liệu, nên chúng ta có thể định nghĩa ở đây để dùng về sau:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjnLH5S2CaWx", - "colab": {} - }, - "source": [ - "class_names = ['Áo thun', 'Quần dài', 'Áo liền quần', 'Đầm', 'Áo khoác',\n", - " 'Sandal', 'Áo sơ mi', 'Giày', 'Túi xách', 'Ủng']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Brm0b_KACaWX" - }, - "source": [ - "## Khám phá dữ liệu\n", - "\n", - "Chúng ta có thể khám phá dữ liệu một chút trước khi huấn luyện mô hình. Câu lệnh sau sẽ cho ta thấy có 60.000 ảnh trong tập huấn luyện, với mỗi ảnh được biểu diễn theo dạng 28x28 pixel:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zW5k_xz1CaWX", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cIAcvQqMCaWf" - }, - "source": [ - "Tương tự, tập huấn luyện cũng có 60.000 nhãn đối ứng:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TRFYHB2mCaWb", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YSlYxFuRCaWk" - }, - "source": [ - "Mỗi nhãn là một số nguyên từ 0 đến 9:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XKnCTHz4CaWg", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TMPI88iZpO2T" - }, - "source": [ - "Có 10.000 ảnh trong tập kiểm thử, mỗi ảnh cũng được biểu diễn ở dãng 28 x 28 pixel:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2KFnYlcwCaWl", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rd0A0Iu0CaWq" - }, - "source": [ - "Và tập kiểm thử cũng chứa 10,000 nhãn:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJmPr5-ACaWn", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ES6uQoLKCaWr" - }, - "source": [ - "## Tiền xử lý dữ liệu\n", - "\n", - "Dữ liệu cần được tiền xử lý trước khi được dùng để huấn luyện mạng neuron. Phân tích ảnh đầu tiên trong tập dữ liệu, chúng ta sẽ thấy các pixel có giá trị từ 0 đến 255:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m4VEw8Ud9Quh", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wz7l27Lz9S1P" - }, - "source": [ - "Chúng ta cần tiền xử lý để mỗi một điểm ảnh có giá trị từ 0 đến 1 (có thể hiểu là 0% đến 100%). Để làm điều này, chúng ta chỉ cần lấy giá trị của pixel chia cho 255. Cần lưu ý rằng việc tiền xử lý này phải được áp dụng đồng thời cho cả *tập huấn luyện* và *tập kiểm thử*:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bW5WzIPlCaWv", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ee638AlnCaWz" - }, - "source": [ - "Để chắc chắn việc tiền xử lý diễn ra chính xác, chúng ta có thể in ra 25 ảnh đầu trong *tập huấn luyện* cùng với tên lớp dưới mỗi ảnh." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oZTImqg_CaW1", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "59veuiEZCaW4" - }, - "source": [ - "## Xây dựng mô hình\n", - "\n", - "Để xây dựng mạng neuron, chúng tay cần cấu hình các layer của mô hình, và sau đó biên dịch mô hình." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Gxg1XGm0eOBy" - }, - "source": [ - "### Thiết lập các layers\n", - "\n", - "Thành phần cơ bản của một mạng neuron là các *layer*. Các layer trích xuất các điểm đặc biệt từ dữ liệu mà chúng đón nhận. Khi thực hiện tốt, những điểm đặc biệt này mang nhiều ý nghĩa và phục vụ cho toán của chúng ta.\n", - "\n", - "Đa số các mô hình deep learning đều chứa các layer đơn gian được xâu chuỗi lại với nhau. Đa số các layer, ví dụ `tf.keras.layers.Dense`, đều có các trọng số sẽ được học trong quá trình huấn luyện." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation='relu'),\n", - " keras.layers.Dense(10, activation='softmax')\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gut8A_7rCaW6" - }, - "source": [ - "Trong mạng neuron trên, lớp đầu tiên, `tf.keras.layers.Flatten`, chuyển đổi định dạng của hình ảnh từ mảng hai chiều (28x28) thành mảng một chiều (28x28 = 784). Tưởng tương công việc của layer này là cắt từng dòng của anh, và ghép nối lại thành một dòng duy nhất nhưng dài gấp 28 lần. Lớp này không có trọng số để học, nó chỉ định dạng lại dữ liệu.\n", - "\n", - "Sau layer làm phẳng ảnh (từ 2 chiều thành 1 chiều), phần mạng neuron còn lại gồm một chuỗi hai layer `tf.keras.layers.Dense`. Đây là các layer neuron được kết nối hoàn toàn (mỗi một neuron của layer này kết nối đến tất cả các neuron của lớp trước và sau nó). Layer `Dense` đầu tiên có 128 nút (hoặc neuron). Layer thứ hai (và cuối cùng) là lớp *softmax* có 10 nút, với mỗi nút tương đương với điểm xác suất, và tổng các giá trị của 10 nút này là 1 (tương đương 100%). Mỗi nút chứa một giá trị cho biết xác suất hình ảnh hiện tại thuộc về một trong 10 lớp.\n", - "\n", - "### Biên dịch mô hình\n", - "\n", - "Trước khi mô hình có thể được huấn luyện, chúng ta cần thêm vài chỉnh sửa. Các chỉnh sửa này được thêm vào trong bước *biên dịch* của mô hình:\n", - "\n", - "* *Hàm thiệt hại* — dùng để đo lường mức độ chính xác của mô hình trong quá trình huấn luyện. Chúng ta cần giảm thiểu giá trị của hạm này \"điều khiển\" mô hình đi đúng hướng (thiệt hại càng ít, chính xác càng cao).\n", - "* *Trình tối ưu hoá* — Đây là cách mô hình được cập nhật dựa trên dữ liệu huấn luyện được cung cấp và hàm thiệt hại.\n", - "* *Số liệu* — dùng để theo dõi các bước huấn luyện và kiểm thử. Ví dụ sau dùng *accuracy*, tỉ lệ ảnh được phân loại chính xác." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Lhan11blCaW7", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qKF6uW-BCaW-" - }, - "source": [ - "## Huấn luyện mô hình\n", - "\n", - "Huấn luyện mô hình mạng neuron cần các bước sau:\n", - "\n", - "1. Cung cấp dữ liệu huấn luyện cho mô hình. Trong ví dụ này, dữ liệu huấn luyện năm trong 2 mảng `train_images` và `train_labels`\n", - "2. Mô hình sẽ học cách liên kết ảnh với nhãn.\n", - "3. Chúng ta sẽ yêu cầu mô hình đưa ra dự đoán từ dữ liệu của tập kiểm thử, trong ví dụ này là mảng `test_images`, sau đó lấy kết quả dự đoán đối chiếu với nhãn trong mảng `test_labels`.\n", - "\n", - "Để bắt đầu huấn luyện, gọi hàm `model.fit`. Hàm này được đặt tên `fit` vì nó sẽ \"fit\" (\"khớp\") mô hình với dữ liệu huấn luyện:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xvwvpA64CaW_", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W3ZVOhugCaXA" - }, - "source": [ - "Trong quá trình huấn luyện, các số liệu như thiệt hại và hay độ chính xác được hiển thị. Với dữ liệu huấn luyện này, mô hình đạt đến độ accuracy vào khoảng 0.88 (88%)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oEw4bZgGCaXB" - }, - "source": [ - "## Đánh giá mô hình\n", - "\n", - "Tiếp theo, chúng đánh giá các chất lượng của mô hình bằng tập kiểm thử:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VflXLEeECaXC", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('\\nTest accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yWfgsmVXCaXG" - }, - "source": [ - "Đến thời điểm này, chúng ta thấy rằng độ accuracy của mô hình, khi đánh giá bằng tập kiểm thử, hơi thấp hơn so với số liệu trong quá trình huấn luyện. Khoảng cách giữa hai độ accuracy khi huấn luyện và khi kiểm thử thể hiện sự *overfitting*. Overfitting xảy ra khi một mô hình ML hoạt động kém hơn khi được cung cấp các đầu vào mới, mà mô hình chưa từng thấy trước đây trong quá trình đào tạo." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xsoS7CPDCaXH" - }, - "source": [ - "## Đưa ra dự đoán\n", - "\n", - "Với một mô hình đã được đào tạo, chúng ta có thể dùng nó để đưa ra dự đoán với một số ảnh." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Gl91RPhdCaXI", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "x9Kk1voUCaXJ" - }, - "source": [ - "Ở đây, mô hình sẽ dự đoán nhãn cho từng hình ảnh trong bộ thử nghiệm. Hãy xem dự đoán đầu tiên:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3DmJEUinCaXK", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-hw1hgeSCaXN" - }, - "source": [ - "Trong ví dụ này, dự đoán là một mảng 10 số thực, mỗi số tương ứng với \"độ tự tin\" của mô hình rằng ảnh đó thuộc về nhãn đó. Chúng ta có thể thấy nhãn nào có độ tư tin cao nhất:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qsqenuPnCaXO", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E51yS7iCCaXO" - }, - "source": [ - "Vậy là mô hình tự tin nhất rằng ảnh này là một loại ủng, hoặc `class_names[9]`. Đối chiếu với nhãn trong tập kiểm thử, ta thấy dự đoán này là đúng:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sd7Pgsu6CaXP", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ygh2yYC972ne" - }, - "source": [ - "Ta thử vẽ biểu đồ để xem các dự đoán trên cả 10 lớp của mô hình." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DvYmmrpIy6Y1", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array, true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - "\n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - "\n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - "\n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array, true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks(range(10))\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - "\n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d4Ov9OFDMmOD" - }, - "source": [ - "Chúng ta có thể nhìn vào ảnh 0th, các dự đoán, và mảng dự đoán.\n", - "Nhãn dự đoán đúng màu xanh và nhãn sai màu đỏ. Con số là số phần trăm của các nhãn được dự đoán." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HV5jw-5HwSmO", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions[i], test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions[i], test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ko-uzOufSCSe", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions[i], test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions[i], test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kgdvGD52CaXR" - }, - "source": [ - "Thử vẽ biểu đồ với vài ảnh và dự đoán đi kèm. Chú ý thấy rằng mô hình đôi khi dự đoán sai dù điểm tự tin rất cao." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hQlnbqaw2Qu_", - "colab": {} - }, - "source": [ - "# Plot the first X test images, their predicted labels, and the true labels.\n", - "# Color correct predictions in blue and incorrect predictions in red.\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions[i], test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions[i], test_labels)\n", - "plt.tight_layout()\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R32zteKHCaXT" - }, - "source": [ - "Cuối cùng, dùng mô hình để đưa ra dự đoán về một ảnh duy nhất." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yRJ7JU7JCaXT", - "colab": {} - }, - "source": [ - "# Grab an image from the test dataset.\n", - "img = test_images[1]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vz3bVp21CaXV" - }, - "source": [ - "Các mô hình `tf.keras` được tối ưu hóa để đưa ra dự đoán về một *lô* hoặc bộ sưu tập các ví dụ cùng một lúc. Theo đó, mặc dù bạn đang sử dụng một ảnh duy nhất, bạn cần thêm nó vào list:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lDFh5yF_CaXW", - "colab": {} - }, - "source": [ - "# Add the image to a batch where it's the only member.\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EQ5wLTkcCaXY" - }, - "source": [ - "Dự đoán nhãn cho ảnh này:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o_rzNSdrCaXY", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6Ai-cpLjO-3A", - "colab": {} - }, - "source": [ - "plot_value_array(1, predictions_single[0], test_labels)\n", - "_ = plt.xticks(range(10), class_names, rotation=45)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cU1Y2OAMCaXb" - }, - "source": [ - "`model.predict` trả về một list của lists — mỗi list cho mỗi ảnh trong lô dữ liệu. Lấy dự đoán cho hình ảnh trong lô:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2tRmdq_8CaXb", - "colab": {} - }, - "source": [ - "np.argmax(predictions_single[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YFc2HbEVCaXd" - }, - "source": [ - "Mô hình dự đoán ảnh này có nhãn là đúng như mong muốn." - ] - } - ] -} \ No newline at end of file diff --git a/site/vi/tutorials/quickstart/advanced.ipynb b/site/vi/tutorials/quickstart/advanced.ipynb deleted file mode 100644 index dd3c535aa14..00000000000 --- a/site/vi/tutorials/quickstart/advanced.ipynb +++ /dev/null @@ -1,420 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "advanced.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# Lướt nhanh nâng cao TensorFlow 2.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Xem trên TensorFlow.org\n", - " \n", - " Chạy trên Google Colab\n", - " \n", - " Xem mã nguồn trên GitHub\n", - " \n", - " Tải notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kbYGPLQdTdNp", - "colab_type": "text" - }, - "source": [ - "Note: Cộng đồng TensorFlow Việt Nam đã dịch các tài liệu này từ nguyên bản tiếng Anh.\n", - "Vì bản dịch này dựa trên sự cố gắng từ các tình nguyện viên, nên không thể đám bảo luôn bám sát\n", - "[Tài liệu chính thức bằng tiếng Anh](https://www.tensorflow.org/?hl=en).\n", - "Nếu bạn có đề xuất để cải thiện bản dịch này, vui lòng tạo PR đến repository trên GitHub của [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "\n", - "Để đăng ký dịch hoặc duyệt lại nội dung bản dịch, các bạn hãy liên hệ \n", - "[docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Đây là một tệp notebook [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb). Các chương trình Python sẽ chạy trực tiếp trong trình duyệt, giúp bạn dễ dàng tìm hiểu và sử dụng TensorFlow. Để làm theo giáo trình này, chạy notebook trên Google Colab bằng cách nhấp vào nút ở đầu trang.\n", - "\n", - "1. Trong Colab, kết nối đến Python runtime: Ở phía trên cùng bên phải của thanh menu, chọn *CONNECT*.\n", - "2. Chạy tất cả các ô chứa mã trong notebook: Chọn *Runtime* > *Run all*." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eOsVdx6GGHmU" - }, - "source": [ - "Tải và cài đặt TensorFlow 2.0 RC." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ioLbtB3uGKPX", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QS7DDTiZGRTo" - }, - "source": [ - "Import TensorFlow vào chương trình:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Load và chuẩn bị [tập dữ liệu MNIST](http://yann.lecun.com/exdb/mnist/)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JqFRS6K07jJs", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# Add a channels dimension\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "Dùng `tf.data` để xáo trộn tập dữ liệu:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Iu_quO024c2", - "colab": {} - }, - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Xây dựng mô hình `tf.keras` với [subclassing API](https://www.tensorflow.org/guide/keras#model_subclassing) của Keras:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "# Create an instance of the model\n", - "model = MyModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "Chọn trình tối ưu hoá (optimizer) và hàm thiệt hại (loss) để huấn luyện:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u48C9WQ774n4", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "Chọn số liệu để đo lường thiệt hại (loss) và độ chính xác (accuracy) của mô hình. Các số liệu này tích lũy các giá trị qua các epoch và sau đó in kết quả tổng thể." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N0MqHFb4F_qn", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Dùng `tf.GradientTape` để huấn luyện:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OZACiVqA8KQV", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z8YT7UmFgpjV" - }, - "source": [ - "Kiểm thử mô hình:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIKdEzHAJGt7", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "i-2pkctU_Ci7", - "colab": {} - }, - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n", - " print(template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))\n", - "\n", - " # Reset the metrics for the next epoch\n", - " train_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_loss.reset_states()\n", - " test_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "Mô hình phân loại ảnh này, sau khi được huấn luyện bằng tập dữ liệu trên, đạt độ chính xác (accuracy) ~98%. Để tìm hiểu thêm, bạn có thể đọc [Giáo trình TensorFlow](https://www.tensorflow.org/tutorials/)." - ] - } - ] -} \ No newline at end of file diff --git a/site/vi/tutorials/quickstart/beginner.ipynb b/site/vi/tutorials/quickstart/beginner.ipynb deleted file mode 100644 index 14ab8e2bdb6..00000000000 --- a/site/vi/tutorials/quickstart/beginner.ipynb +++ /dev/null @@ -1,240 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "beginner.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# Lướt nhanh cơ bản TensorFlow 2.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " Xem trên TensorFlow.org\n", - " \n", - " Chạy trên Google Colab\n", - " \n", - " Xem mã nguồn trên GitHub\n", - " \n", - " Tải notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BbU6UaFdS-WR", - "colab_type": "text" - }, - "source": [ - "Note: Cộng đồng TensorFlow Việt Nam đã dịch các tài liệu này từ nguyên bản tiếng Anh.\n", - "Vì bản dịch này dựa trên sự cố gắng từ các tình nguyện viên, nên không thể đám bảo luôn bám sát\n", - "[Tài liệu chính thức bằng tiếng Anh](https://www.tensorflow.org/?hl=en).\n", - "Nếu bạn có đề xuất để cải thiện bản dịch này, vui lòng tạo PR đến repository trên GitHub của [tensorflow/docs](https://github.com/tensorflow/docs)\n", - "\n", - "Để đăng ký dịch hoặc duyệt lại nội dung bản dịch, các bạn hãy liên hệ \n", - "[docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "Đây là một tệp notebook [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb). Các chương trình Python sẽ chạy trực tiếp trong trình duyệt, giúp bạn dễ dàng tìm hiểu và sử dụng TensorFlow. Để làm theo giáo trình này, chạy notebook trên Google Colab bằng cách nhấp vào nút ở đầu trang.\n", - "\n", - "1. Trong Colab, kết nối đến Python runtime: Ở phía trên cùng bên phải của thanh menu, chọn *CONNECT*.\n", - "2. Chạy tất cả các ô chứa mã trong notebook: Chọn *Runtime* > *Run all*." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nnrWf3PCEzXL" - }, - "source": [ - "Tải và cài đặt TensorFlow 2.0 RC. Import TensorFlow vào chương trình:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# Install TensorFlow\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "Load và chuẩn bị [tập dữ liệu MNIST](http://yann.lecun.com/exdb/mnist/). Chuyển kiểu dữ liệu của các mẫu từ số nguyên sang số thực dấu phẩy động:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FP5258xjs-v", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "Xây dựng mô hình `tf.keras.Sequential` bằng cách xếp chồng các layers. Chọn trình tối ưu hoá (optimizer) và hàm thiệt hại (loss) để huấn luyện:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "model = tf.keras.models.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "Huấn luyện và đánh giá mô hình:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F7dTAzgHDUh7", - "colab": {} - }, - "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", - "model.evaluate(x_test, y_test, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "Mô hình phân loại ảnh này, sau khi được huấn luyện bằng tập dữ liệu trên, đạt độ chính xác (accuracy) ~98%. Để tìm hiểu thêm, bạn có thể đọc [Giáo trình TensorFlow](https://www.tensorflow.org/tutorials/)." - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/README.md b/site/zh-cn/README.md deleted file mode 100644 index 2bad200f988..00000000000 --- a/site/zh-cn/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# 非官方翻译 - -我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的 -[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到 -[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入 -[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。 - -# Community translations - -Our TensorFlow community has translated these documents. Because community -translations are *best-effort*, there is no guarantee that this is an accurate -and up-to-date reflection of the -[official English documentation](https://www.tensorflow.org/?hl=en). -If you have suggestions to improve this translation, please send a pull request -to the [tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository. -To volunteer to write or review community translations, contact the -[docs@tensorflow.org list](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). - -Note: Please focus translation efforts on -[TensorFlow 2](https://www.tensorflow.org) in the -[site/en/](https://github.com/tensorflow/docs/tree/master/site/en/) -directory. TF 1.x community docs will no longer be updated as we prepare for the -2.0 release. See -[the announcement](https://groups.google.com/a/tensorflow.org/d/msg/docs/vO0gQnEXcSM/YK_ybv7tBQAJ). - -# Simplified Chinese translation guide - -Some technical words in English do not have a natural translation. Do *not* -translate the following words: - -* (mini-) batch -* estimator -* eager execution -* dense diff --git a/site/zh-cn/REVIEWERS b/site/zh-cn/REVIEWERS deleted file mode 100644 index 361a72477dd..00000000000 --- a/site/zh-cn/REVIEWERS +++ /dev/null @@ -1,18 +0,0 @@ -# Add your GitHub username to get comment-tagged on all pull requests to -# the site/zh-cn directory. Remove your username whenever you'd like. -# -# New pull request notifications are also sent to the -# docs-zh-cn@tensorflow.org list: -# https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn -# -kuri-leo -JayYip -yantaozhao -loveunk -wind2esg -tigerneil -MofiiTech -gaoljhy -Mr-Linus -flopsySong -echosun1996 diff --git a/site/zh-cn/community/contribute/docs.md b/site/zh-cn/community/contribute/docs.md deleted file mode 100644 index 09fe3814968..00000000000 --- a/site/zh-cn/community/contribute/docs.md +++ /dev/null @@ -1,289 +0,0 @@ -# 参与TensorFlow文档编写 - -TensorFlow欢迎文档贡献 - 如果您改进了文档，等同于改进TensorFlow库本身。 tensorflow.org上的文档分为以下几类： - -* *API 文档* —[API 文档](https://www.tensorflow.org/api_docs/) - 经由 - [TensorFlow 源代码](https://github.com/tensorflow/tensorflow)中的文档字符串(docstring)生成. -* *叙述文档* —这部分内容为[教程](https://www.tensorflow.org/tutorials)、 - [指南](https://www.tensorflow.org/guide)以及其他不属于TensorFlow代码的内容. 这部分代码位于GitHub的 - [tensorflow/docs](https://github.com/tensorflow/docs) 仓库(repository)中. -* *社区翻译* —这些是经由社区翻译的指南和教程。他们都被存放在 - [tensorflow/docs](https://github.com/tensorflow/docs/tree/master/site) 仓库(repository)中. - -一些 [TensorFlow 项目](https://github.com/tensorflow) 将文档源文件保存在单独的存储库中，通常位于`docs/`目录中。 请参阅项目的`CONTRIBUTING.md`文件或联系维护者以参与。 - -参与到TensorFlow文档社区的方式有: - -* 关注GitHub中的 [tensorflow/docs](https://github.com/tensorflow/docs) 仓库(repository). -* 订阅 [docs@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs). -* 加入 [Gitter 聊天室](https://gitter.im/tensorflow/docs). - -## API 文档 - -如果想更新API文档，找到其对应的 -[源文件](https://www.tensorflow.org/code/tensorflow/python/) -并编辑相应的 -文档字符串(docstring). -tensorflow.org上的许多API 引用的页面都包含了指向源文件定义位置的链接。 文档字符串支持 -Markdown格式 -并且（绝大多数时候）都能使用 -Markdown 预览器 -进行浏览. - -有关参考文档质量以及如何参与文档冲刺和社区，请参阅 -[TensorFlow 2.0 API文档建议](https://docs.google.com/document/d/1e20k9CuaZ_-hp25-sSd8E8qldxKPKQR-SkwojYr_r-U/preview)。 - -### 版本(Versions) 和 分支(Branches) - -本网站的 [API 文档](https://www.tensorflow.org/api_docs/python/tf) -版本默认为最新的稳定二进制文件—即与通过`pip install tensorflow`安装的版本所匹配. - -默认的TensorFlow 包是根据tensorflow/tensorflow仓库(repository)中的稳定分支`rX.x`所构建的。文档则是由 -Python、 -C++与 -Java代码中的注释与文档字符串所生成。 - -以前版本的TensorFlow文档在TensorFlow Docs 仓库(repository)中以[rX.x 分支](https://github.com/tensorflow/docs/branches) 的形式提供。在发布新版本时会添加这些分支。 - -### 构建API文档 - -注意：编辑或预览API文档字符串不需要此步骤，只需生成tensorflow.org上使用的HTML。 - -#### Python 文档 - -`tensorflow_docs`包中包含[Python API 文档](https://www.tensorflow.org/api_docs/python/tf)的生成器。 -安装方式： - -
    -pip install git+https://github.com/tensorflow/docs
-
    - -要生成TensorFlow 2.0文档，使用 -`tensorflow/tools/docs/generate2.py` 脚本: - -
    -git clone https://github.com/tensorflow/tensorflow tensorflow
-cd tensorflow/tensorflow/tools/docs
-pip install tensorflow
-python generate2.py --output_dir=/tmp/out
-
    - -注意：此脚本使用*已安装*的TensorFlow包生成文档并且仅适用于TensorFlow 2.x. - -## 叙述文档 - -TensorFlow [指南](https://www.tensorflow.org/guide) 和 -[教程](https://www.tensorflow.org/tutorials) 是通过 -Markdown -文件和交互式的 -Jupyter 笔记本所编写。 可以使用 -Google Colaboratory -在您的浏览器中运行笔记本。 -[tensorflow.org](https://www.tensorflow.org)中的叙述文档是根据 -tensorflow/docs的 -`master` 分支构建. 旧版本存储在在GitHub 仓库(repository)下的`rX.x`发行版分支中。 - -### 简单更改 - -进行简单文档更新和修复的最简单方法是使用GitHub的 -Web文件编辑器。 -浏览[tensorflow/docs](https://github.com/tensorflow/docs/tree/master/site/en) -仓库(repository) 以寻找与 -tensorflow.org -中的URL 结构相对应的Markdown或notebook文件。 在文件视图的右上角，单击铅笔图标 - -来打开文件编辑器。 编辑文件，然后提交新的拉取请求(pull request)。 - -### 设置本地Git仓库(repository) - -对于多文件编辑或更复杂的更新，最好使用本地Git工作流来创建拉取请求(pull request)。 - -注意：Git 是用于跟踪源代码更改的开源版本控制系统（VCS）。 -GitHub是一种在线服务， -提供与Git配合使用的协作工具。请参阅GitHub Help以设置您的GitHub帐户并开始使用。 - -只有在第一次设置本地项目时才需要以下Git步骤。 - -#### 复制(fork) tensorflow/docs 仓库(repository) - -在 -tensorflow/docs -的Github页码中，点击*Fork*按钮 - -在您的GitHub帐户下创建您自己的仓库副本。复制(fork) 完成，您需要保持您的仓库副本副本与上游TensorFlow仓库的同步。 - -#### 克隆您的仓库(repository) - -下载一份您 username/docs 仓库的副本到本地计算机。这是您之后进行操作的工作目录： - -
    -git clone git@github.com:username/docs
-cd ./docs
-
    - -#### 添加上游仓库(upstream repo)以保持最新（可选） - -要使本地存储库与`tensorflow/docs`保持同步，需要添加一个*上游(upstream)* -仓库来下载最新的更改。 - -注意：确保在开始撰稿*之前*更新您的本地仓库。定期向上游同步会降低您在提交拉取请求(pull request)时产生合并冲突(merge conflict)的可能性。 - -添加远程仓库: - -
    -git remote add upstream git@github.com:tensorflow/docs.git
-
-# 浏览远程仓库
-git remote -v
-origin    git@github.com:username/docs.git (fetch)
-origin    git@github.com:username/docs.git (push)
-upstream  git@github.com:tensorflow/docs.git (fetch)
-upstream  git@github.com:tensorflow/docs.git (push)
-
    - -更新: - -
    -git checkout master
-git pull upstream master
-
-git push  # Push changes to your GitHub account (defaults to origin)
-
    - -### GitHub 工作流 - -#### 1. 创建一个新分支 - -从`tensorflow / docs`更新您的仓库后，从本地*master*分支中创建一个新的分支: - -
    -git checkout -b feature-name
-
-git branch  # 列出本地分支
-  master
-* feature-name
-
    - -#### 2. 做更改 - -在您喜欢的编辑器中编辑文件，并请遵守 -[TensorFlow文档样式指南](./docs_style.md)。 - -提交文件更改： - -
    -# 查看更改
-git status  # 查看哪些文件被修改
-git diff    # 查看文件中的更改内容
-
-git add path/to/file.md
-git commit -m "Your meaningful commit message for the change."
-
    - -根据需要添加更多提交。 - -#### 3. 创建一个拉取请求(pull request) - -将您的本地分支上传到您的远程GitHub仓库 -(github.com/username/docs): - -
    -git push
-
    - -推送完成后，消息可能会显示一个URL，以自动向上游存储库提交拉取请求。如果没有，请转到 -tensorflow/docs -仓库—或者您自己的仓库—GitHub将提示您创建拉取请求(pull request)。 - -#### 4. 审校 - -维护者和其他贡献者将审核您的拉取请求(pull request)。请参与讨论并根据要求进行修改。当您的请求获得批准后，它将合并到上游TensorFlow文档仓库中。 - -成功后：您的更改会被TensorFlow文档接受。 - -从GitHub仓库更新 -[tensorflow.org](https://www.tensorflow.org)是一个单独的步骤。通常情况下，多个更改将被一并处理，并定期上传至网站中。 - -## 交互式笔记本（notebook） - -虽然可以使用GitHub的web文本编辑器来编辑笔记本JSON文件，但不推荐使用它，因为格式错误的JSON可能会损坏文件。 确保在提交拉取请求(pull request)之前测试笔记本。 - -Google Colaboratory -是一个托管笔记本环境，可以轻松编辑和运行笔记本文档。 GitHub中的笔记本通过将路径传递给Colab URL（例如，位于GitHub中的笔记本）在Google Colab中加载： -https://github.com/tensorflow/docs/blob/master/site/en/tutorials/keras/basic_classification.ipynb
    -可以通过以下URL链接在Google Colab中加载: -https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/keras/basic_classification.ipynb - -有一个 -Open in Colab -扩展程序，可以在GitHub上浏览笔记本时执行此URL替换。 这在您复制的仓库中中打开笔记本时非常有用，因为顶部按钮始终链接到TensorFlow Docs的`master`分支。 - -### 在Colab编辑 - -在Google Colab环境中，双击单元格以编辑文本和代码块。文本单元格使用Markdown格式，请遵循 -[TensorFlow文档样式指南](./docs_style.md). - -通过点击 *File > Download .pynb* 可以从Colab中下载笔记本文件。 将此文件提交到您的[本地Git仓库](###设置本地Git仓库(repository))后再提交拉取请求。 - -如需要创建新笔记本，请复制和编辑 -TensorFlow 笔记本模板. - -### Colab-GitHub工作流 - -您可以直接从Google Colab编辑和更新复制的GitHub仓库，而不是下载笔记本文件并使用本地Git工作流： - -1. 在您复制(fork)的 username/docs 仓库中，使用GitHub Web界面 - 创建新分支。 -2. 导航到要编辑的笔记本文件。 -3. 在Google Colab中打开笔记本：使用URL替换或*Open in Colab* Chrome扩展程序。 -4. 在Colab中编辑笔记本。 -5. 通过点击 - *File > Save a copy in GitHub...*从Colab中向GitHub提交更改。保存对话框中选择到相应的仓库与分支。并添加一条有意义的提交消息。 -6. 保存之后，浏览您的仓库或者 - tensorflow/docs - 仓库，GitHub会提示您创建一个pull请求。 -7. 仓库维护者会审核您的拉取请求(pull request)。 - -成功后：您的更改会被TensorFlow文档接受。 - -## 社区翻译 - -社区翻译是让TensorFlow在全世界都可以访问的好方法。如需更新或添加翻译，在[语言目录](https://github.com/tensorflow/docs/tree/master/site)中按照`en/`相同的目录结构找到或添加一个新文件。英语文档是*最基础*的来源，翻译应尽可能地遵循这些指南。也就是说，翻译应尽量保持原汁原味。如果英语术语，短语，风格或语气不能翻译成其他语言，请采用适合读者的翻译。 - -注意：*请勿翻译* tensorflow.org中的API引用. - -有特定于语言的文档组，使翻译贡献者可以更轻松地进行组织。 如果您是作者，评论者或只是想为社区构建TensorFlow.org内容，请加入： - -* 简体中文: [docs-zh-cn@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn) -* 日语: [docs-ja@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ja) -* 韩语: [docs-ko@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ko) -* 俄文: [docs-ru@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-ru) -* 土耳其语: [docs-tr@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-tr) - -### 审校须知 - -所有文档更新都需要审核。 为了更有效地与TensorFlow翻译社区进行协作，以下是一些保持语言特定活动的方法： - -* 加入上面列出的语言组，以接收任何涉及该语言site/lang目录的*已创建的* 拉取请求。 -* 将您的GitHub用户名添加至`site//REVIEWERS`文件在拉取请求中能被自动注释标记。在被标记后，GitHub会向您发送该拉取请求中所有更改和讨论的通知。 - -### 在翻译中让代码保持最新 - -对于像TensorFlow这样的开源项目，保持文档最新是一项挑战。在与社区交谈之后，翻译内容的读者能容忍有点过时的文本，但过时的代码会让人抓狂。为了更容易保持代码同步，请为翻译的笔记本使用 -[nb-code-sync](https://github.com/tensorflow/docs/blob/master/tools/nb_code_sync.py)工具： - -
    -./tools/nb_code_sync.py [--lang=en] site/lang/notebook.ipynb
-
    - -此脚本读取语言笔记本的代码单元格，并根据英语版本进行检查。 剥离注释后，它会比较代码块并更新语言笔记本（如果它们不同）。 此工具对于交互式git工作流特别有用，可以选择性地将文件添加至更改中: `git add --patch site/lang/notebook.ipynb` - -## Docs sprint - -参加您附近的 -[TensorFlow 2.0 Global Docs Sprint](https://www.google.com/maps/d/viewer?mid=1FmxIWZBXi4cvSy6gJUW9WRPfvVRbievf) -活动，或远程加入。 请关注此 -[博客文章](https://medium.com/tensorflow/https-medium-com-margaretmz-tf-docs-sprint-cheatsheet-7cb1dfd3e8b5?linkId=68384164)。这些事件是开始为TensorFlow文档做出贡献的好方法。 diff --git a/site/zh-cn/js/guide/conversion.md b/site/zh-cn/js/guide/conversion.md deleted file mode 100644 index 243620f3d67..00000000000 --- a/site/zh-cn/js/guide/conversion.md +++ /dev/null @@ -1,51 +0,0 @@ -# 模型转换 - -TensorFlow.js 配备了各种预训练模型，这些模型可以在浏览器中使用，[模型仓库](https://github.com/tensorflow/tfjs-models) 中有相关介绍。但是，您可能已经在其他地方找到或创建了一个 TensorFlow 模型，并希望在 web 应用程序中使用该模型。为此，TensorFlow.js 提供了一个 [模型转换器](https://github.com/tensorflow/tfjs-converter) 。TensorFlow.js 转换器有两个组件: - -1. 一个命令行程序，用于转换 Keras 和 TensorFlow 模型以在 TensorFlow.js 中使用。 -2. 一个 API ，用于在浏览器中使用 TensorFlow.js 加载和执行模型。 - -## 转换您的模型 - -TensorFlow.js 转换器可以转换以下几种格式的模型: - -**SavedModel**: 保存 TensorFlow 模型的默认格式。SavedModel 的格式细节请 [查阅此处](https://www.tensorflow.org/guide/saved_model)。 - -**Keras model**: Keras 模型通常保存为 HDF5 文件。有关保存 Keras 模型的更多信息，请访问 [此处](https://keras.io/getting-started/faq/#savingloading-whole-models-architecture-weights-optimizer-state)。 - -**TensorFlow Hub module**: 这些是打包后在 TensorFlow Hub 中进行分发的模型，TensorFlow Hub 是一个共享和发现模型的平台。模型库见 [此处](tfhub.dev)。 - -取决于尝试转换的模型的格式，您需要将不同的参数传递给转换器。比如，假设您保存了一个名为 `model.h5` 的 Keras 模型到 `tmp/` 目录。为了使用 TensorFlow.js 转换器转换模型，您可以运行以下命令: - - $ tensorflowjs_converter --input_format=keras /tmp/model.h5 /tmp/tfjs_model - -这会将路径为 `/tmp/model.h5` 的模型转换并输出 `model.json` 文件及其二进制权重文件到目录 `tmp/tfjs_model/` 中。 - -有关不同格式的模型相对应的命令行参数的更多信息，请参阅 TensorFlow.js 转换器 [自述文件](https://github.com/tensorflow/tfjs-converter)。 - -在转换过程中，我们会遍历模型图形并确认 TensorFlow.js 是否支持每个操作。如果是支持的，我们将图形转换成浏览器可以使用的格式。我们尝试通过将权重分成 4MB 的文件来优化模型以便在 web 上使用 - 这样它们就可以被浏览器缓存。我们也尝试使用开源工程 [Grappler](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/grappler) 简化模型图形。图形的简化包括相邻操作的折叠，消除常见子图像等。这些更改对模型的输出没有影响。为了进一步优化，用户可以输入参数以指示转换器将模型量化到特定的字节大小。量化是一种减少模型大小的技术，它是通过用更少的比特表示权重实现的。用户应务必确保量化后模型的准确度保持在可接受范围内。 -如果在转换过程中遇到了不支持的操作，则该过程失败，我们将为用户打印出该操作的名称。请将此提交到我们的 [GitHub](https://github.com/tensorflow/tfjs/issues) - 我们会尝试根据用户的需求实现更多新的操作。 - -### 最佳做法 - -虽然在转换过程中我们尽力优化您的模型，但通常确保您的模型顺利运行的最佳方式是在考虑资源受限的环境下构建。这意味着避免过于复杂的建构和尽可能减少参数（权重）的数目。 - -## 运行您的模型 - -成功转换模型之后，您将得到一组权重文件和一个模型拓扑文件。TensorFlow.js 提供模型加载 APIs ，您可以使用这些接口获取模型并且在浏览器中运行推断。 - -以下是加载转换后的 TensorFlow SavedModel 或 TensorFlow Hub 模块的 API : - -```js -const model = await tf.loadGraphModel(‘path/to/model.json’); -``` - -以下是转换后的 Keras 模型的 API : - -```js -const model = await tf.loadLayersModel(‘path/to/model.json’); -``` - -`tf.loadGraphModel` API 返回 `tf.FrozenModel`，这意味着各项参数是固定的并且您不能使用新数据对模型进行微调。`tf.loadLayersModel` API 返回可训练的 tf.Model。有关训练 tf.Model 的相关信息，请参阅[训练模型指南](train_models.md)。 - -在转换之后，我们建议您进行几次推断并且对模型的速度进行基准测试。基于这个目的，我们有一个独立的基准测试页面: https://github.com/tensorflow/tfjs-core/blob/master/integration_tests/benchmarks/benchmark.html。 您可能注意到我们丢弃了初始预热运行中的测量值 - 这是因为（通常情况）下，由于创建纹理和编译着色器的资源消耗，您的模型的第一次的推断将比后续推断慢几倍。 diff --git a/site/zh-cn/js/guide/index.md b/site/zh-cn/js/guide/index.md deleted file mode 100644 index 229ef73358f..00000000000 --- a/site/zh-cn/js/guide/index.md +++ /dev/null @@ -1,16 +0,0 @@ -这篇教程将会涵盖一下章节： - -* [Tensors(张量)和Operations(操作)](tensors_operations.md) — - 此章节将介绍Tensorflow.js的组成部分，如tensors(张量)，数据，形状以及数据类型等。 -* [Platform(平台) & Environment(环境)](platform_environment.md) — - 此章节为Tensorflow.js在不同平台以及环境概述，并简述了Tensorflow.js在不同平台上的取舍。 -* [Models(模型) and Layers(层)](models_and_layers.md) — - 此章节介绍了如何利用Layers以及Core的API在Tensorflow.js中建立模型。 -* [训练模型](train_models.md) — - 此章节介绍了如何训练：models（模型），optimizers（优化器），losses（损失函数），metrics（评价标准）以及variables（变量）。 -* [保存和加载模型](save_load.md) — 此章节将会指导你如何在Tensorflow.js中保存以及加载模型 -* [模型转换](conversion.md) — 此章节主要介绍在Tensorflow.js中可用的模型类型以及其转换细节 -* [与Python tf.keras的差异](layers_for_keras_users.md) — - 此章节将会阐述tensorflow.js和python'tf.keras'主要的区别和功能，并了解JavaScript中的API规范。 -* [在Node.js 中使用Tensorflow.js](nodejs.md) — - 此章节将会主要介绍Tensorflow.js在Node.js中的三种可用模式以及所需要的系统要求。 diff --git a/site/zh-cn/js/guide/layers_for_keras_users.md b/site/zh-cn/js/guide/layers_for_keras_users.md deleted file mode 100644 index dd102248216..00000000000 --- a/site/zh-cn/js/guide/layers_for_keras_users.md +++ /dev/null @@ -1,201 +0,0 @@ -# 用于 Keras 用户使用的 TensorFlow.js layers API - -TensorFlow.js 的Layers API以Keras为模型。考虑到 JavaScript 和 Python 之间的差异，我们努力使[Layers API](https://js.tensorflow.org/api/latest/) 与Keras 类似。这让具有使用Python开发Keras模型经验的用户可以更轻松地将项目迁移到 JavaScript中的TensorFlow.js Layers。例如，以下 Keras 代码转换为 JavaScript： - -```python -# Python: -import keras -import numpy as np - -# 建立并编译模型. -model = keras.Sequential() -model.add(keras.layers.Dense(units=1, input_shape=[1])) -model.compile(optimizer='sgd', loss='mean_squared_error') - -# 生成一些用于训练的数据. -xs = np.array([[1], [2], [3], [4]]) -ys = np.array([[1], [3], [5], [7]]) - -# 用 fit() 训练模型. -model.fit(xs, ys, epochs=1000) - -# 用 predict() 推理. -print(model.predict(np.array([[5]]))) -``` - -```js -// JavaScript: -import * as tf from '@tensorlowjs/tfjs'; - -// 建立并编译模型. -const model = tf.sequential(); -model.add(tf.layers.dense({units: 1, inputShape: [1]})); -model.compile({optimizer: 'sgd', loss: 'meanSquaredError'}); - -// 生成一些用于训练的数据. -const xs = tf.tensor2d([[1], [2], [3], [4]], [4, 1]); -const ys = tf.tensor2d([[1], [3], [5], [7]], [4, 1]); - -// 用 fit() 训练模型. -await model.fit(xs, ys, {epochs: 1000}); - -// 用 predict() 推理. -model.predict(tf.tensor2d([[5]], [1, 1])).print(); -``` - -但是，我们希望在本文档中说明并解释一些差异。一旦理解了这些差异及其背后的基本原理，将您的程序从Python 迁移到JavaScript（或反向迁移）应该会是一种相对平稳的体验。 - -## 构造函数将 JavaScript 对象作为配置 - -比较上面示例中的以下 Python 和 JavaScript 代码：它们都创建了一个[全连接层](https://keras.io/layers/core/#dense)。 - -```python -# Python: -keras.layers.Dense(units=1, inputShape=[1]) -``` - -```js -// JavaScript: -tf.layers.dense({units: 1, inputShape: [1]}); -``` - -JavaScript函数在Python 函数中没有等效的关键字参数。我们希望避免在 JavaScript 中实现构造函数选项作为位置参数，这对于记忆和使用具有大量关键字参数的构造函数（如[LSTM](https://keras.io/layers/recurrent/#lstm)尤其麻烦 。这就是我们使用JavaScript 配置对象的原因。这些对象提供与Python关键字参数相同的位置不变性和灵活性。 - -Model 类的一些方法（例如，[`Model.compile()`](https://keras.io/models/model/#model-class-api)）也将 JavaScript 配置对象作为输入。但是，请记住 Model.fit()、Model.evaluate() 和 Model.predict() 略有不同。因为这些方法将强制 x（feature 特征）和 y（label 标签或 target 目标）数据作为输入；x 和 y 是与后续配置对象分开的位置参数，属于关键字参数。例如： - - -## Model.fit()是异步的 - -`Model.fit()` 是用户在Tensorflow.js中执行模型训练的主要方法。这个方法往往是长时间运行的（持续数秒或数分钟）。因此，我们利用了JavaScript语言的“异步”特性。所以在浏览器中运行时，这样使用此函数就不会阻塞主UI线程。这和JavaScript中其他可能长期运行的函数类似，例如`async`[获取](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)。需要注意`async`是一个在python中不存在的构造。当[`fit()`](https://keras.io/models/model/#model-class-api)方法在keras中返回一个历史对象, 在JavaScript中`fit()`方法的对应项返回一个包含训练历史的[Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise)这个应答可以[await(等待)](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await)，也可以与then()方法一起使用。 - - -## TensorFlow.js 中没有 NumPy - -Python Keras 用户经常使用[NumPy](http://www.numpy.org/)来执行基本的数值和数组的操作，例如在上面的示例中生成 2D 张量。 - -```python -# Python: -xs = np.array([[1], [2], [3], [4]]) -``` - -在 TensorFlow.js 中，这种基本的数字的操作是使用包本身完成的。例如： - -```js -// JavaScript: -const xs = tf.tensor2d([[1], [2], [3], [4]], [4, 1]); -``` - -该 tf.* 命名空间还提供数组和线性代数的operations（操作），如矩阵乘法。有关更多信息，请参阅 [TensorFlow.js核心文档](https://js.tensorflow.org/api/latest/)。 - -## 使用factory(工厂)方法，而不是构造函数 - -Python 中的这一行（来自上面的例子）是一个构造函数调用： - -```python -# Python: -model = keras.Sequential() -``` - -如果严格转换为 JavaScript，则等效构造函数调用将如下所示： - -```js -// JavaScript: -const model = new tf.Sequential(); // 不！ 要！ 这！ 样！ 做！ -``` - -然而，我们决定不使用“new”构造函数，因为 1)“new”关键字会使代码更加膨胀；2)“new”构造函数被视为 JavaScript 的“bad part”：一个潜在的陷阱，如在[*JavaScript: the Good Parts*](http://archive.oreilly.com/pub/a/javascript/excerpts/javascript-good-parts/bad-parts.html).中的争论。要在 TensorFlow.js 中创建模型和 Layer ，可以调用被称为 lowerCamelCase（小驼峰命名）的工厂方法，例如： - -```js -// JavaScript: -const model = tf.sequential(); - -const layer = tf.layers.batchNormalization({axis: 1}); -``` - -## 选项字符串值为小驼峰命名，而不是 snake_case - -在 JavaScript 中，与 Python 相比，更常见的是使用小驼峰作为符号名称（例如，[Google JavaScript Style Guide](https://google.github.io/styleguide/jsguide.html#naming-camel-case-defined)），而 Python 中 snake_case 很常见（例如，在 Keras 中）。因此，我们决定使用小驼峰命名作为选项的字符串值，包括以下内容： - -* DataFormat，例如，channelsFirst 而不是 channels_first -* Initializer，例如，glorotNormal 而不是 glorot_normal -* Loss and metrics，例如，meanSquaredError 而不是 mean_squared_error，categoricalCrossentropy 而不是 categorical_crossentropy。 - -例如，如上例所示： - -```js -// JavaScript: -model.compile({optimizer: 'sgd', loss: 'meanSquaredError'}); -``` - -对于模型序列化和反序列化，请放心。请放心。TensorFlow.js 的内部机制确保正确处理 JSON 对象中的 snake_case ，例如，从 Python Keras 加载预训练模型时。 - - -## 使用 apply() 运行 Layer 对象，而不是将其作为函数调用 - -在 Keras 中，Layer 对象定义了`__call__`方法。因此，用户可以通过将对象作为函数调用来调用 Layer 的逻辑，例如: - -```python -# Python: -my_input = keras.Input(shape=[2, 4]) -flatten = keras.layers.Flatten() - -print(flatten(my_input).shape) -``` - -这个 Python 语法糖在 TensorFlow.js 中以 apply() 方法实现： - -```js -// JavaScript: -const myInput = tf.input{shape: [2, 4]}); -const flatten = tf.layers.flatten(); - -console.log(flatten.apply(myInput).shape); -``` - -## Layer.apply() 支持对具体 Tensor（张量）的命令式（Eager）执行 - -目前，在 Keras 中，`__call__`方法只能对（Python）TensorFlow 的 `tf.Tensor` 对象进行操作（假设 TensorFlow 是后端），这些对象是符号化的并且不包含实际的数值。这就是上一节中的示例中所显示的内容。但是，在 TensorFlow.js 中，Layer 的 `apply()` 方法可以在符号和命令模式下运行。如果用 SymbolicTensor 调用 `apply()`（类似于 tf.Tensor）调用，则返回值将为 SymbolicTensor。这通常发生在模型构建期间。但是如果用实际的具体 Tensor（张量）值调用 `apply()`，将返回一个具体的 Tensor（张量）。例如： - -```js -// JavaScript: -const flatten = tf.layers.flatten(); - -flatten.apply(tf.ones([2, 3, 4])).print(); -``` - -这个特性让人联想到（Python）TensorFlow 的[Eager Execution](https://www.tensorflow.org/guide/eager)。它在模型开发期间提供了更大的交互性和可调试性，并且为组成动态神经网络打开了大门。 - -## Optimizers（优化器）在 train.* 下，而不是 optimizers.* - -在 Keras 中，Optimizer（优化器）对象的构造函数位于 keras.optimizers.* 命名空间下。在 TensorFlow.js Layer 中，Optimizer（优化器）的工厂方法位于 tf.train.* 命名空间下。例如： - -```python -# Python: -my_sgd = keras.optimizers.sgd(lr=0.2) -``` - -```js -// JavaScript: -const mySGD = tf.train.sgd({lr: 0.2}); -``` - -## loadLayersModel() 从 URL 加载，而不是 HDF5 文件 - -在 Keras 中，模型通常[保存](https://keras.io/getting-started/faq/#how-can-i-save-a-keras-model)为 HDF5（.h5）文件，然后可以使用 `keras.models.load_model()`方法加载 。该方法采用 .h5 文件的路径。TensorFlow.js 中的 load_model() 对应的是[`tf.loadLayersModel()`](https://js.tensorflow.org/api/latest/#loadLayersModel)。由于 HDF5 文件格式对浏览器并不友好，因此 tf.loadLayersModel() 采用 TensorFlow.js 特定的格式。tf.lloadLayersModel() 将 model.json 文件作为其输入参数。可以使用 tensorflowjs 的 pip 包从 Keras HDF5 文件转换 model.json。 - -```js -// JavaScript: -const model = await tf.loadLayersModel('https://foo.bar/model.json'); -``` - -还要注意的是`tf.loadLayersModel()`返回的是[`tf.Model`](https://js.tensorflow.org/api/latest/#class:Model)的[应答`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise)。 - -通常，tf.Model在 TensorFlow.js中保存和加载分别使用`tf.Model.save`和`tf.loadLayersModel`方法。我们将这些 API 设计为类似于Keras[the save and load_model API](https://keras.io/getting-started/faq/#how-can-i-save-a-keras-model)。但是浏览器环境与 Keras 等主要深度学习框架运行的后端环境完全不同，特别是用于持久化和传输数据的路由数组中。因此，TensorFlow.js 和 Keras 中的 save/load API 之间存在一些有趣的差异。有关更多详细信息，请参阅我们关于 [保存和加载tf.Model](./save_load.md)的教程。 - -## 用`fitDataset()`训练模型使用`tf.data.Dataset`对象 - -在python版本的tensorflow keras中， 一个模型可以使用[Dataset](https://www.tensorflow.org/guide/datasets)对象进行训练。模型的`fit()`方法直接接受这样的对象。一个Tensorflow.js方法可以使用相当于Dataset对象的Javascript进行训练，详见[TensorFlow.js的tf.data API文档](https://js.tensorflow.org/api/latest/#Data)。然而，与python不同， 基于Dataset的训练是通过一个专门的方法来完成的这个方法称之为[fitDataset](https://js.tensorflow.org/api/0.15.1/#tf.Model.fitDataset)。[fit()](https://js.tensorflow.org/api/latest/#tf.Model.fitDataset) 只针对基于Tensor(张量)的模型训练。 - -## Layer(层)对象和Model(模型)对象的内存管理 - -TensorFlow.js在浏览器中的WebGL上运行，其中层和模型对象的权重由WebGL纹理支持。然而WebGL并不支持内置的垃圾收集。在推理和训练的过程中，Layer(层)和Model(模型)对象为用户在内部管理Tensor(张量)内存。但是它们也允许用户清理它们以释放它们占用的WebGL内存。这对于在单页加载过程中创建和释放许多模型实例的情况很有用。想要清理一个Layer(层)和Model(模型)对象，使用`dispose()` 方法。 diff --git a/site/zh-cn/js/guide/models_and_layers.md b/site/zh-cn/js/guide/models_and_layers.md deleted file mode 100644 index 9f052c0e1c5..00000000000 --- a/site/zh-cn/js/guide/models_and_layers.md +++ /dev/null @@ -1,193 +0,0 @@ -# 模型和层 - -机器学习中，一个 _model_ 是一个带有可训练[参数](https://developers.google.com/machine-learning/glossary/#parameter)的函数。这个函数将输入转化为输出。通俗的来说，这个函数表达了输入和输出之间的变换关系。我们通过在数据集上训练模型来获得最佳参数。训练好的模型可以精确的将输入数据转换为我们想得到的输出。 - -TensorFlow.js有两种创建机器学习的方法： - -1. 用 Layers API（用 _layers_ 来创建模型） -2. 用 Core API（底端算子，例如 `tf.matMul()`或`tf.add()`等）来建立模型 - -我们首先会用高层API：Layers API来建立模型。然后，我们会展示如何用Core API来搭建相同的模型。 - -## 用Layers API创建模型 - -Layers API有两种方式创建模型：第一种是创建 _sequential_ 模型，第二种是创建 _functional_ 模型。下面两段会分别解释这两种模型创建方式。 - -### 使用sequential model - -最常见的模型是[Sequential](https://js.tensorflow.org/api/0.15.1/#class:Sequential)模型。Sequential模型将网络的每一层简单的叠在一起。您可以将需要的层按顺序写在一个列表里，然后将列表作为[sequential()](https://js.tensorflow.org/api/0.15.1/#sequential) 函数的输入： - -```js -const model = tf.sequential({ - layers: [ - tf.layers.dense({inputShape: [784], units: 32, activation: 'relu'}), - tf.layers.dense({units: 10, activation: 'softmax'}), - ] -}); -``` - -或用 `add()` 方法： - -```js -const model = tf.sequential(); -model.add(tf.layers.dense({inputShape: [784], units: 32, activation: 'relu'})); -model.add(tf.layers.dense({units: 10, activation: 'softmax'})); -``` - -> 注意：模型的第一层需要“输入形状”参数（`inputShape`）。不要在“输入型状”中包含batch size（批次大小）。假设您要向模型输入一个形状为`[B, 784]`的张量（`B`是任意batch size），您只需要将“输入型状”设为`[784]`。 - -您可以通过`model.layers`来使用模型中的每一层。例如，您可以用`model.inputLayers`和`model.outputLayers`来调用输入层和输出层。 - -### 使用functional model - -我们也可以通过`tf.model()`来创建`LayersModel`。`tf.model()`和`tf.sequential()`的主要区别为，您可以用`tf.model()`来创建任何非闭环的计算图。 - -以下是一段如何用`tf.model()` API 建立和上文相同模型的列子： - -```js -// 用apply()方法创建任意计算图 -const input = tf.input({shape: [784]}); -const dense1 = tf.layers.dense({units: 32, activation: 'relu'}).apply(input); -const dense2 = tf.layers.dense({units: 10, activation: 'softmax'}).apply(dense1); -const model = tf.model({inputs: input, outputs: dense2}); -``` - -我们在每一层用`apply()`将上一层的输出作为本层的输入。`apply()`返回一个`SymbolicTensor`（类似于张量，但不包含任何数值） - -不同于sequential model使用`inputShape`来定义第一层的输入，我们用`tf.input()`创建的`SymbolicTensor`作为第一层的输入 - -如果您向`apply()`输入一个数值张量，它会进行计算并返还一个数值张量： - -```js -const t = tf.tensor([-2, 1, 0, 5]); -const o = tf.layers.activation({activation: 'relu'}).apply(t); -o.print(); // [0, 1, 0, 5] -``` - -这个方式适用于单独测试每一层并检查它们的输出。 - -和sequential model一样，您可以通过`model.layers`来使用模型中的每一层。例如，您可以用`model.inputLayers`和`model.outputLayers`来调用输入层和输出层。 - -## 验证 - -Sequential model和functional model都属于`LayersModel`类。使用`LayersModels`让验证更方便：它要求您定义输入形状，并用您定义的形状来验证您对模型的输入。`LayersModel`会自动计算模型中所有张量的形状。知道张量的形状后，模型就可以自动创建它所需要的参数。您也可以用形状信息来判断两层相邻的层是否相互兼容。 - -## 模型总览 - -使用`model.summary()`可以显示很多模型的重要信息，包括： - -* 每一层的名字和类型 -* 每一层的输出形状 -* 每一层的权重数量 -* 每一层的输入 -* 一个模型拥有的可训练参数总量，和不可训练参数总量 - -用前面定义的模型来做例子，我们可以在命令行中得到以下信息： - - - - - - - - - - - - - - - - - - - - -
    Layer (type) - Output shape - Param # -
    dense_Dense1 (Dense) - [null,32] - 25120 -
    dense_Dense2 (Dense) - [null,10] - 330 -
    Total params: 25450
    Trainable params: 25450
    Non-trainable params: 0 -
    - -注意：每一层的输出形状中都含有`null`值。模型的输入形状包含了批次大小，而批次大小是可以灵活更变的，所以批次的值在张量形状中以`null`显示。 - -## 序列化 - -相对于底端API而言，使用`LayersModel`的另一个好处是方便存储、加载模型。`LayersModel`包含如下信息： - -* 可用于重建模型的模型架构信息 -* 模型的权重 -* 训练配置（例如损失函数，优化器和评估方式） -* 优化器的状态（可用于继续训练模型） - -存储和加载模型只需要一行代码： - -```js -const saveResult = await model.save('localstorage://my-model-1'); -const model = await tf.loadLayersModel('localstorage://my-model-1'); -``` - -在这个例子中，模型被存储在浏览器的本地存储里。请访问[model.save()](https://js.tensorflow.org/api/latest/#tf.Model.save)和[save and load](save_load.md)了解如何把模型保存在不同的媒介中（例如 file storage, IndexedDB, 触发下载到浏览器等等）。 - -## 自定义层 - -层是创建模型的基础。如果您的模型需要定制化计算模块，您可以写一个自定义层并插入模型中。下面的例子是一个计算平方和的自定义层： - -```js -class SquaredSumLayer extends tf.layers.Layer { - constructor() { - super({}); - } - // In this case, the output is a scalar. - computeOutputShape(inputShape) { return []; } - - // call() is where we do the computation. - call(input, kwargs) { return input.square().sum();} - - // Every layer needs a unique name. - getClassName() { return 'SquaredSum'; } -} -``` - -可以用`apply()`方法在一个张量上测试这个自定义层 - -```js -const t = tf.tensor([-2, 1, 0, 5]); -const o = new SquaredSumLayer().apply(t); -o.print(); // prints 30 -``` - -> 注意：如果您在模型中包含了自定义层，模型将不能序列化 - -## 用Core API创建模型 - -本文开头提到了两种在TensorFlow.js中建立模型的方法。最常用的方式是使用 Layers API，因为它的模式是基于广泛应用的Keras API（详情见 [best practices and reduces cognitive load](https://keras.io/why-use-keras/)）。Layers API提供了大量方便的工具，例如权重初始化，模型序列化，训练监测，可迁移性和安全检查。 - -当您遇到如下情况时，可能会需要使用Core API： - -* 您需要更多灵活性和控制 -* 您不需要序列化或可以创造自己的序列化方法 - -用Core API写的模型包含了一系列的函数。这些函数以一个或多个张量作为输入，并输出另一个张量。我们可以用Core API来重写之前定义的模型： - -```js -// The weights and biases for the two dense layers. -const w1 = tf.variable(tf.randomNormal([784, 32])); -const b1 = tf.variable(tf.randomNormal([32])); -const w2 = tf.variable(tf.randomNormal([32, 10])); -const b2 = tf.variable(tf.randomNormal([10])); - -function model(x) { - return x.matMul(w1).add(b1).relu().matMul(w2).add(b2).softmax(); -} -``` - -在Core API中，我们需要自己创建和初始化权重。每个权重都是一个`Variable`，TensorFlow.js会把`Variable`权重设为可训练张量。您可以用[tf.variable()](https://js.tensorflow.org/api/latest/#variable)创建`Variable`或把一个已存在的张量放到`Variable`中。 - -本文介绍了如何用Layers和Core API创建模型。接下来，请看[training models](train_models.md)学习如何训练模型。 diff --git a/site/zh-cn/js/guide/nodejs.md b/site/zh-cn/js/guide/nodejs.md deleted file mode 100644 index 376e003b668..00000000000 --- a/site/zh-cn/js/guide/nodejs.md +++ /dev/null @@ -1,102 +0,0 @@ -# Node 中的 TensorFlow.js - -## TensorFlow CPU - -TensorFlow CPU 包，可以按如下方式导入： - - -```js -import * as tf from '@tensorflow/tfjs-node' -``` - - -当从这个包导入 TensorFlow.js 时，您导入的模块将由 TensorFlow C 二进制文件加速并在 CPU 上运行。CPU 上的 TensorFlow 使用硬件加速来加速内部的线性代数运算。 - -此软件包适用于支持 TensorFlow 的 Linux，Windows 和 Mac 平台。 - -> 注意：您没有必要导入'@tensorflow/tfjs'或者将其添加到您的 package.json 文件中，这是由 Node 库间接导入的。 - - -## TensorFlow GPU - -TensorFlow GPU 包，可以按如下方式导入： - - -```js -import * as tf from '@tensorflow/tfjs-node-gpu' -``` - -与 CPU 包一样，您导入的模块将由 TensorFlow C 二进制文件加速，但是它将使用 CUDA 在 GPU 上运行张量运算，因此只能运行在 Linux 平台。该绑定比其他可选绑定可以快至少一个数量级。 - -> 注意：此软件包目前仅适用于 CUDA。在选择本方案之前，您需要在带有 NVIDIA 显卡的的机器上安装 CUDA。 - -> 注意：您没有必要导入'@tensorflow/tfjs'或将其添加到您的 package.json 文件中，这是由 Node 库间接导入的。 - - -## 普通 CPU - -使用普通 CPU 运行 TensorFlow.js 版本，可以按如下方式导入： - - -```js -import * as tf from '@tensorflow/tfjs' -``` - -这个包与您在浏览器中使用的包类似。在这个包中，这些调用是在 CPU 上以原生 JavaScript 运行。这个包比其他包小得多，因为它不需要 TensorFlow 二进制文件，但是速度要慢得多。 - -由于这个软件包不依赖于 TensorFlow，因此它可用于支持 Node.js 的更多设备，而不仅仅是 Linux，Windows 和 Mac平台。 - - -## 生产环境考虑因素 - -Node.js Bindings 为 TensorFlow.js 提供了一个同步地执行操作的后端。这意味着当您调用一个操作时，例如 `tf.matMul(a, b)`，它将阻塞主线程直到这个操作完成。 - -因此，当前这种 Bindings 非常适合脚本和离线任务。如果您要在实际应用程序（如：Web 服务器）中使用Node.js Bindings，则应设置一个工作队列或设置一些工作线程，以便您的 TensorFlow.js 代码不会阻止主线程。 - - -## APIs - -一旦您在上面的任何选项中将包导入为 tf 后，所有普通的 TensorFlow.js 符号都将出现在导入的模块上。 - -### tf.browser - -在普通的 TensorFlow.js 包中，`tf.browser.*` 命名空间中的符号将在 Node.js 中不可用，因为它们使用特定浏览器的 API。 - -目前，有如下 API： - -* tf.browser.fromPixels -* tf.browser.toPixels - -### tf.node - -有两个 Node.js 包还提供了一个名称为 `tf.node` 的命名空间，其中包含了特定 Node 的 API。 - -TensorBoard 是一个特定 Node.js API 的重要例子。 - -这是一个将训练的总结(summaries)导出至Node.js的TensorBoard中的案例 - -```js -const model = tf.sequential(); -model.add(tf.layers.dense({units: 1})); -model.compile({ - loss: 'meanSquaredError', - optimizer: 'sgd', - metrics: ['MAE'] -}); - - -// 为了演示目的生成一些随机假数据。 -const xs = tf.randomUniform([10000, 200]); -const ys = tf.randomUniform([10000, 1]); -const valXs = tf.randomUniform([1000, 200]); -const valYs = tf.randomUniform([1000, 1]); - - -// 开始模型训练过程。 -await model.fit(xs, ys, { - epochs: 100, - validationData: [valXs, valYs], - // 在这里添加 tensorBoard 回调。 - callbacks: tf.node.tensorBoard('/tmp/fit_logs_1') -}); -``` diff --git a/site/zh-cn/js/guide/platform_environment.md b/site/zh-cn/js/guide/platform_environment.md deleted file mode 100644 index f80d84cb018..00000000000 --- a/site/zh-cn/js/guide/platform_environment.md +++ /dev/null @@ -1,104 +0,0 @@ -# 平台和环境 - -TensorFlow.js有两种工作平台：浏览器和Node.js。不同平台有很多不同的配置，平台间的差异影响着基于平台的应用开发。 - -在浏览器平台上，TensorFlow.js既支持移动设备，也支持台式设备。虽然设备之间有很多差异，TensorFlow.js提供的WebGL API能够自动检测并做相应的优化配置。 - -在Node.js平台上，TensorFlow.js既支持直接使用TensorFlow API，也支持更慢的CPU环境。 - -## [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#environments)环境 - -当一个用TensorFlow.js开发的程序运行时，所有的配置被统称为环境。它包含一个全局的backend，以及一些可以精确控制TensorFlow.js特性的标记。 - -### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#backends)Backends - -TensorFlow.js支持多个不同的backend，用来实现张量的存储和数学操作。任何时候都只有一个backend生效。大部分时间，TensorFlow.js会根据当前环境自动选择使用最佳的backend。即使这样，你仍然需要知道，如何得知当前正在使用的是哪个backend，以及如何在不同backend之间切换。 - -下面命令用来获取当前正使用的backend -```js -console.log(tf.getBackend()); -``` - -下面命令用来手动切换backend -```js -tf.setBackend(‘cpu’); -console.log(tf.getBackend()); -``` - -#### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#webgl-backend)WebGL backend - -WebGL backend，简称“webgl”，是在浏览器平台上最强大的一个backend。它比CPU backend要快100倍。部分原因是，Tensor是作为WebGL纹理保存的，数学运算操作实现在WebGL shader里面。 - -下面是在使用这个backend时需要了解的一些知识。 - -##### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#avoid-blocking-the-ui-thread)避免阻塞UI线程 -当调用一个操作，如tf.matMul(a,b)时，返回值tf.Tensor会同步返回，然而这时矩阵乘法运算还不一定完成。这意味着返回值tf.Tensor只是一个指向运算的句柄。当调用`x.data()`或`x.array()`时，只有当运算完成时才能取到实际值。因此在运算过程中，为避免阻塞UI线程，需要使用异步版本的`x.data()`和`x.array()`，而不是同步版本的`x.dataSync()`和`x.arraySync()`。 -##### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#memory-management)内存管理 - -强调一下，在使用WebGL backend时，需要显式管理内存。因为存储Tensor的WebGL纹理，不会被浏览器的垃圾收集机制自动清理。 - -调用dispose()清理tf.Tensor占用的内存 - -```js -const a = tf.tensor([[1,2], [3,4]]); -a.dispose(); -``` - -在应用中，经常需要把多个操作组合起来。维持一个对所有中间变量的引用，然后清理其占用的内存，这种方法使代码可读性变差。TensorFlow.js提供tf.tidy()方法清理函数返回时不再需要的tf.Tensor，这就好像函数执行后，本地变量都会被清理一样。 - -```js -const a = tf.tensor([[1, 2], [3, 4]]); -const y = tf.tidy(() => { - const result = a.square().log().neg(); - return result; -}); -``` - ->注意：其他非WebGL环境（如Node.js TensorFlow backend或CPU backend）有自动垃圾回收机制，在这些环境下使用dispose()或tidy()没有副作用。实际上，主动调用通常会比垃圾回收的清理带来更好的性能。 - -##### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#precision)精度 - -在移动设备，WebGL只支持16位浮点纹理操作。然而，大部分机器学习模型都用32位浮点的weight和activation训练的。由于16位浮点数字只能表示[0.000000059605， 65504]这个范围，当把模型移植到移动设备时，它会产生精度问题。你需要保证自己模型中的weight和activation不要超出这个范围。 -##### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#shader-compilation--texture-uploads)编译Shader& texture 上传 -TensorFlow.js在GPU里执行WebGL的shader程序。然而这些shader只有在被调用时才会被编译，即lazy-compile。编译过程在CPU上的主线程完成，这导致程序变慢。TensorFlow.js会自动缓存编译好的shader，下次再调用有同样shape，同样输入输出的tensor时能快很多。TensorFlow.js开发的应用一般会多次使用同样的操作，因此第二次运行会快很多。 - -TensorFlow.js还会把tf.Tensor数据存储为WebGL纹理。当一个tf.Tensor被创建后，不会被立即上传到GPU，而是当其被用到时才这么做。如果这个tf.Tensor被第二次使用，由于已经在GPU里，因此省掉了上传开销。在一个典型的机器学习模型中，这意味着weight在第一次预测时被上传，第二次就会快很多。 - -如果希望加快第一次预测的性能，我们推荐对模型进行预热，即传递一个有同样shape的输入Tensor。 -例如: -```js -const model = await tf.loadLayersModel(modelUrl); -// 使用真实数据来预热模型 -const warmupResult = model.predict(tf.zeros(inputShape)); -warmupResult.dataSync(); -warmupResult.dispose(); - -// 第二次执行 predict() 的时候将会更加快速 -const result = model.predict(userData); -``` - -#### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#nodejs-tensorflow-backend)Node.js TensorFlow backend - -在Node.js TensorFlow backend中，所谓“node”,即TensorFlow的C语言API被用来加速操作。它会尽可能使用机器的硬件加速模块，如CUDA。 - -在这个backend中，和WebGL backend一样，函数会同步返回`tf.Tensor`。然而，与WebGL backend不同的是，当你获得这个tensor返回值时，运算已经完成。这意味着`tf.matMul(a,b)`调用会阻塞UI线程。 - -因此，如果你在生产环境下使用这个方法，你需要在工作线程中调用，而不是主线程。 - -更多关于Node.js的信息，请查看相关文档。 -#### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#cpu-backend)CPU backend - -这个backend是性能最差的backend，然而是最简单的。所有操作都实现在vanilla JavaScript中，因此很少有并行化，并且会阻塞UI线程。 - -这个backend对测试有用，或者是用于WebGL不能使用的设备。 - -### [](https://github.com/tensorflow/tfjs-website/blob/master/docs/guide/platform_environment.md#flags)Flags - -TensorFlow.js有一套环境标记，能够自动评估和检测，保证是当前平台上的最佳配置。这些标记大部分是内部使用，其中有一些全局标记可以被API控制。 - -- `tf.enableProdMode():` 启用生产模式。它会去掉模型验证，NaN检查，以及其他错误校验操作，从而提高性能。 -- `tf.enableDebugMode()`: 启用调试模式。它会记录每种操作的日志并输出到到调试台，还记录运行性能信息，如内存footprint和内核执行时间。注意这将极大降低应用运行时间，不可在生产环境中使用。 - -注：这两种方法应该在程序的最前面调用，因为它们影响所有的其他标记。基于同样的原因，没有相应的disable方法。 - -注：所有标记在控制台都记录为tf.ENV.features。尽管没有对应的公开API（不需要考虑版本兼容），你可以使用tf.ENV.set来改变这些标记，从而对程序做微调或诊断。 diff --git a/site/zh-cn/js/guide/save_load.md b/site/zh-cn/js/guide/save_load.md deleted file mode 100644 index ab6f65d9abf..00000000000 --- a/site/zh-cn/js/guide/save_load.md +++ /dev/null @@ -1,202 +0,0 @@ -# 保存并加载 tf.Model - -TensorFlow.js提供了保存和加载模型的功能，这些模型可以是使用[`Layers`](https://js.tensorflow.org/api/0.14.2/#Models)API创建的或从现有TensorFlow模型转换来的。可能是您自己训练过的模型，也可能是别人训练的模型。使用Layers API的一个主要好处是使用它创建的模型是可序列化的，这就是我们将在本教程中探讨的内容。 - -本教程将会介绍如何在 TensorFlow.js 中保存和加载模型(可通过JSON文件识别)。我们同样可以导入Tensorflow Python模型。 - -以下两个教程介绍了加载这些模型： - -- [导入Keras模型](../tutorials/conversion/import_keras.md) -- [导入Graphdef模型](../tutorials/conversion/import_saved_model.md) - - -## 保存 tf.Model - -[`tf.Model`](https://js.tensorflow.org/api/0.14.2/#class:Model) 和 [`tf.Sequential`](https://js.tensorflow.org/api/0.14.2/#class:Model) -同时提供了函数 [`model.save`](https://js.tensorflow.org/api/0.14.2/#tf.Model.save) 允许您保存一个模型的 -_拓扑结构(topology)_ 和 _权重(weights)_ 。 - -- 拓扑结构(Topology): 这是一个描述模型结构的文件（例如它使用的了哪些操作）。它包含对存储在外部的模型权重的引用。 - -- 权重(Weights): 这些是以有效格式存储给定模型权重的二进制文件。它们通常存储在与拓扑结构相同的文件夹中。 - -让我们看看保存模型的代码是什么样子的 - -```js -const saveResult = await model.save('localstorage://my-model-1'); -``` - -一些需要注意的地方: - -- `save` 方法采用以 scheme 字符串开头的类 URL 字符串参数（下文简称 scheme）。它描述了我们想保存模型的地址的类型。 在本例中我们使用 localstorage:// scheme 将模型保存到本地存储。 -- 在 scheme 之后是 **路径(path)**。 在上面的例子中，路径是'my-model-1'。 -- `save` 方法是异步的。 -- `model.save` 的返回值是一个 JSON 对象，它包含一些可能有用的信息，例如模型的拓扑结构和权重的大小。 -- 用于保存模型的环境不会影响那些可以加载模型的环境。在 node.js 中保存模型时并不会阻碍模型在浏览器中被加载。 - - -下面我们将介绍以下不同方案。 - -### 本地存储 (仅限浏览器) - -**Scheme:** `localstorage://` - -```js -await model.save('localstorage://my-model'); -``` -这样可以在浏览器的[本地存储](https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage)中以名称 `my-model` 来保存模型。这样存储能够在浏览器刷新后保持不变，而当存储空间成为问题时，用户或浏览器本身可以清除本地存储。 每个浏览器还可以对给定域在本地的存储空间设定限额。 - -### IndexedDB (仅限浏览器) - -**Scheme:** `indexeddb://` - -```js -await model.save('indexeddb://my-model'); -``` - -这样会将模型保存到浏览器的[IndexedDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API)存储中。 -与本地存储一样，它在刷新后仍然存在，同时它往往也对存储的对象的大小有较大的限制。 - -### 文件下载 (仅限浏览器) - -**Scheme:** `downloads://` - -```js -await model.save('downloads://my-model'); -``` -这会让浏览器下载模型文件至用户的机器上，并生成两个文件： - 1. 一个名为 `[my-model].json` 的 JSON 文件，它包含了模型的拓扑结构和下面将要介绍的权重文件的引用。 - 2. 一个二进制文件，其中包含名为 `[my-model].weights.bin` 的权重值。 - -您可以更换 `[my-model]` 的名称以获得一个不同的名称的文件。 - -由于`.json`使用相对路径指向 `.bin`，所以两个文件需要被安放在同一个文件夹中。 - -> 注意: 某些浏览器要求用户在同时下载多个文件之前授予权限。 - - - -### HTTP(S) Request - -**Scheme:** `http://` or `https://` - -```js -await model.save('http://model-server.domain/upload') -``` - -这将创建一个Web请求，以将模型保存到远程服务器。 您应该控制该远程服务器，以便确保它能够处理该请求。 -模型将通过[POST](https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST) 请求发送到指定的 HTTP 服务器。 -POST 请求的 body 遵守称为`multipart/form-data`的格式。它由以下两个文件组成 - - 1. 一个名为 `model.json` 的 JSON 文件，其中包含拓扑结构和对下面描述的权重文件的引用。 - 2. 一个二进制文件，其中包含名为 `[my-model].weights.bin` 的权重值。 - -请注意，这两个文件的名称需要与上述介绍中的保持完全相同（因为名称内置于函数中，无法更改）。 此[ api 文档](https://js.tensorflow.org/api/latest/#tf.io.browserHTTPRequest)包含一个 Python 代码片段，演示了如何使用 [flask](http://flask.pocoo.org/) web 框架来处理源自 `save` 的请求。 - -通常，您必须向 HTTP 服务器传递更多参数或请求头（例如，用于身份验证，或者如果要指定应保存模型的文件夹）。您可以通过替换 `tf.io.browserHTTPRequest` 函数中的 URL字符串参数来获得对来自 `save` 函数的请求在这些方面的细粒度控制。这个API在控制 HTTP 请求方面提供了更大的灵活性。 - -例如： - -```js -await model.save(tf.io.browserHTTPRequest( - 'http://model-server.domain/upload', - {method: 'PUT', headers: {'header_key_1': 'header_value_1'}})); -``` - - -### 本机文件系统 (仅限于Node.js) - -**Scheme:** `file://` - -```js -await model.save('file:///path/to/my-model'); -``` - -当运行Node.js后我们当然可以直接访问文件系统并且保存模型。这个命令将会保存两个文件在`scheme`之后指定的`path`中。 - - 1. 一个名为 `model.json` 的 JSON 文件，其中包含拓扑结构和对下面描述的权重文件的引用。 -1. 一个二进制文件，其中包含名为`model.weights.bin`. 的权重值。 - -请注意，这两个文件的名称将始终与上面指定的完全相同（该名称内置于函数中）。 - - -## 加载 tf.Model - -给定一个使用上述方法之一保存的模型，我们可以使用 `tf.loadLayersModel` API来加载它。 - -让我们看一下加载模型的代码是什么样子的 - -```js -const model = await tf.loadLayersModel('localstorage://my-model-1'); -``` - -一些事情值得注意: -- 类似于`model.save()`, `loadLayersModel`函数使用以 **scheme**开头的类似URL的字符串参数。它描述了我们试图从中加载模型的目标类型。 -- scheme 由**path**指定。在上述例子中路径为`my-model-1`。 -- URL字符串可以被替换为一个符合IOHandler接口的对象。 -- `tf.loadLayersModel()`函数是异步的。 -- `tf.loadLayersModel`返回的值是 `tf.Model` - -下面我们将介绍可用的不同方案。 - - -### 本地存储 (仅限浏览器) - -**Scheme:** `localstorage://` - -```js -const model = await tf.loadLayersModel('localstorage://my-model'); -``` - -这将从浏览器的[本地存储](https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage). -加载一个名为`my-model`模型。 - -### IndexedDB (仅限浏览器) - -**Scheme:** `indexeddb://` - -```js -const model = await tf.loadLayersModel('indexeddb://my-model'); -``` -这将从浏览器的[IndexedDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API). -加载一个模型。 - - -### HTTP(S) - -**Scheme:** `http://` or `https://` - -```js -const model = await tf.loadLayersModel('http://model-server.domain/download/model.json'); -``` -这将从HTTP端点加载模型。加载`json` 文件后，函数将请求对应的`json` 文件引用的`.bin`文件。 - -> 注意：这个工具依赖于[`fetch`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch)方法。如果您的环境没有提供原生的fetch方法，您可以提供全局方法名称[`fetch`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch)从而满足接口要求或是使用类似于(`node-fetch`)[https://www.npmjs.com/package/node-fetch]的库。 - - -### 本机文件系统 (仅限于Node.js) - -**Scheme:** `file://` - -```js -const model = await tf.loadLayersModel('file://path/to/my-model/model.json'); -``` - -当运行在Node.js上时，我们可以直接访问文件系统并且从那里加载模型。注意，在上面的函数调用中，我们引用model.json文件本身（而在保存时，我们指定一个文件夹）。相应的`.bin`文件需要和`json` 文件在同一个文件夹中。 - -## 使用 IOHandlers 加载模型 - -如果上述方案没有满足您的需求，您可以使用`IOHandler`执行自定义的加载行为。Tensorflow.js的`IOHandler`提供了[`tf.io.browserFiles`](https://js.tensorflow.org/api/latest/#io.browserFiles) ，运行浏览器用户在浏览器中上传文件。您可以在 [文档](https://js.tensorflow.org/api/latest/#io.browserFiles)中查看更多信息。 - -# 使用自定义的 IOHandlers 保存或加载模型 - -如果上述方案没有满足您的保存和加载模型的需求，您可以通过执行`IOHandler`以执行自定义的序列化行为。 - -`IOHandler`是一个含有`save` 和 `load`方法的对象。 - -`save`函数接受一个与[ModelArtifacts](https://github.com/tensorflow/tfjs-core/blob/master/src/io/types.ts#L165)接口匹配的参数并且会返回一个解析为[SaveResult](https://github.com/tensorflow/tfjs-core/blob/master/src/io/types.ts#L107)的对象。 - -`load`函数没有接受参数而回返回一个解析为[ModelArtifacts](https://github.com/tensorflow/tfjs-core/blob/master/src/io/types.ts#L165)的对象。这和传递给`save`的相同对象。 - - -查看[BrowserHTTPRequest](https://github.com/tensorflow/tfjs-core/blob/master/src/io/browser_http.ts)获取如何执行IOHandler的例子。 diff --git a/site/zh-cn/js/guide/tensors_operations.md b/site/zh-cn/js/guide/tensors_operations.md deleted file mode 100644 index eab95fe087f..00000000000 --- a/site/zh-cn/js/guide/tensors_operations.md +++ /dev/null @@ -1,138 +0,0 @@ -# 张量(Tensors) 和 操作(operations) - -TensorFlow.js是一个在JavaScript中使用张量来定义并运行计算的框架。张量是向量和矩阵向更高维度的推广。 - -## 张量(Tensors) - -`tf.Tensor`是TensorFlow.js中的最重要的数据单元，它是一个形状为一维或多维数组组成的数值的集合。`tf.Tensor`和多维数组其实非常的相似。 - -一个`tf.Tensor`还包含如下属性: - -* `rank`: 张量的维度 -* `shape`: 每个维度的数据大小 -* `dtype`: 张量中的数据类型 - ->注释：在后文中，我们将用术语“维度（dimension）”表示`rank（秩）`。在机器学习中，张量的“维数（dimensionality）”有时也指特定维度的大小。（例如，一个形状为[10, 5]的矩阵是一个rank-2 的张量，或者可以说成一个2-维的张量。第一个维度的维数是10。所以在这里用注释的方式，描述一下这个术语的双重用法，避免之后的理解错误。） - -我们可以用`tf.tensor()`方法将一个数组(array)创建为一个`tf.Tensor`： - -```js -// 从一个多维数组创建一个rank-2的张量矩阵 -const a = tf.tensor([[1, 2], [3, 4]]); -console.log('shape:', a.shape); -a.print(); -// 或者您可以用一个一维数组并指定特定的形状来创建一个张量 -const shape = [2, 2]; -const b = tf.tensor([1, 2, 3, 4], shape); -console.log('shape:', b.shape); -b.print(); -``` - -在默认的情况下，`tf.Tensor`的数据类型也就是 -`dtype`为32位浮点型(`float32`)。当然`tf.Tensor`也可以被创建为以下数据类型：布尔(`bool`), 32位整型(`int32`), -64位复数(`complex64`), 和字符串(`string`)： - -```js -const a = tf.tensor([[1, 2], [3, 4]], [2, 2], 'int32'); -console.log('shape:', a.shape); -console.log('dtype', a.dtype); -a.print(); -``` - -TensorFlow.js同样也提供了一系列方便的模式用作创建随机张量，比如将张量填入特定的数值或是从`HTMLImageElement`中获取张量。当然您还可以在文档中找到更多的[方法](https://js.tensorflow.org/api/latest/#Tensors-Creation)。 - -#### 修改张量的形状 - -`tf.Tensor`中的元素数量是这个张量的形状的乘积(例如一个形状为[2,3]的张量所含有的元素个数为2*3=6个)。所以说在大部分时候不同形状的张量的大小却是相同的,那么将一个`tf.Tensor`改变形状(reshape)成为另外一个形状通常是有用且有效的。上述操作可以用`reshape()` -方法实现: - -```js -const a = tf.tensor([[1, 2], [3, 4]]); -console.log('a shape:', a.shape); -a.print(); - -const b = a.reshape([4, 1]); -console.log('b shape:', b.shape); -b.print(); -``` - -#### 获取张量的值 - -如果您想要获取一个`tf.Tensor`的值，可以使用`Tensor.array()` or `Tensor.data()`这两个方法: - -```js - const a = tf.tensor([[1, 2], [3, 4]]); - //返回多维数组的值 - a.array().then(array => console.log(array)); - // 返回张量所包含的所有值的一维数组 - a.data().then(data => console.log(data)); -``` - -我们同时也提供了这些方法能够更简单运用的同步执行版本，但是这些方法可能会导致您的应用程序遇到一些性能瓶颈。在生产环境的应用程序中，您应该始终优先使用异步方法。 - -```js -const a = tf.tensor([[1, 2], [3, 4]]); - //返回多维数组的值 -console.log(a.arraySync()); -// 返回张量所包含的所有值的一维数组 -console.log(a.dataSync()); -``` - -## 操作 - -您可以使用张量存储数据，而操作(operation)可以让您操作这些数据。TensorFlow.js还提供了多种能在张量上执行，适用于线性代数和机器学习的操作。 - -例1: 给`tf.Tensor`中所有的元素执行x2函数: - -```js -const x = tf.tensor([1, 2, 3, 4]); -const y = x.square(); // 相当于 tf.square(x) -y.print(); -``` - -例2: 将两个 `tf.Tensor`中的元素对应相加: - -```js -const a = tf.tensor([1, 2, 3, 4]); -const b = tf.tensor([10, 20, 30, 40]); -const y = a.add(b); // 相当于 tf.add(a, b) -y.print(); -``` - -因为张量是不可变的，所以这些运算并不会更改他们的值。相应的这些操作总会返回一个新的`tf.Tensor`。 - -> 注释: 大部分的操作会同步返回 `tf.Tensor`, -> 然而结果可能不会立刻被计算出来。这意味着您得到的`tf.Tensor`实际上是计算的一个句柄。当您调用`Tensor.data()`或是`Tensor.array()`时，这些方法将会等待计算完成之后才将数值解析出来。这意味着您应该始终优先选择这些方法的异步版本而不是同步版本，以避免在计算的过程中阻塞UI线程。 - -您可以在这里找到更多关于Tensorflow.js中[操作](https://js.tensorflow.org/api/latest/#Operations)的技术支持。 - -## 内存 - -当使用WebGL后端时, `tf.Tensor`的内存必须以显式管理。这是因为WebGL不足以让`tf.Tensor`超出生命周期后内存被自动释放。 - -您可以使用`dispose() `方法或是`tf.dispose()`方法用以释放`tf.Tensor`所占用的内存: - -```js -const a = tf.tensor([[1, 2], [3, 4]]); -a.dispose(); // 相当于 tf.dispose(a) -``` - -在一个应用程序中，将多个操作链接在一起是非常常见的。保存对所有中间变量的引用以释放它们所占用的空间会降低代码的可读性。为了解决这个问题，TensorFlow.js提供了`tf.tidy()`方法。这个方法可以清除所有在执行函数后没有返回的`tf.Tensor`,这和执行函数时清除一些局部变量的方法有些类似: - -```js -const a = tf.tensor([[1, 2], [3, 4]]); -const y = tf.tidy(() => { - const result = a.square().log().neg(); - return result; -}); -``` - -在这个例子中，`square()`和`log()`这两个函数因为没有返回任何值，所以它们的结果会自动的被释放。而`neg()`是`tf.tidy()`的返回值，所以它的结果不会被释放。 - -当然您还可以获取TensorFlow.js程序中张量的数量。 - -```js -console.log(tf.memory()); -``` - -`tf.memory()`将会打印出有关当前分配了多少内存的信息。在[这里](https://js.tensorflow.org/api/latest/#memory)您可以获得更多的资料。 diff --git a/site/zh-cn/js/tutorials/conversion/import_keras.md b/site/zh-cn/js/tutorials/conversion/import_keras.md deleted file mode 100644 index ab16aae2851..00000000000 --- a/site/zh-cn/js/tutorials/conversion/import_keras.md +++ /dev/null @@ -1,75 +0,0 @@ -# 将Keras模型导入Tensorflow.js - -Keras模型（通常通过Python API创建）可能被保存成[多种格式之一](https://keras.io/getting-started/faq/#how-can-i-save-a-keras-model). 整个模型格式可以被转换为Tensorflow.js的层(Layer)格式，这个格式可以被加载并直接用作Tensorflow.js的推断或是进一步的训练。 - -转换后的TensorFlow.js图层(Layer)格式是一个包含model.json文件和一组二进制格式的分片权重文件的目录。 model.json文件包含模型拓扑结构（又名“架构(architecture)”或“图形(graph)”：它是对图层(Layer)及其连接方式的描述）和权重文件的清单。 - -## 要求 - -转换过程要求Python的编程环境，您可能需要独立的使用[pipenv](https://github.com/pypa/pipenv)或是[virtualenv](https://virtualenv.pypa.io)。并使用 `pip install tensorflowjs`安装转换器 - -将Keras模型导入Tensorflow.js需要两步过程。首先，将已有Keras模型转换成TF.js层(Layer)格式，然后将其加载进Tensorflow.js。 - -## Step 1. 将已有Keras模型转换成TF.js层(Layer)格式 - -Keras模型通常通过 `model.save(filepath)`进行保存，这样做会产生一个同时含有模型拓扑结构以及权重的HDF5(.h5)文件。如需要转换这样一个文件成为TF.js层格式，则可以运行以下代码。这里的`path/to/my_model.h5`为Keras .h5文件地址，而`path/to/tfjs_target_dir`则是对应输出的TF.js目录。 - -```sh -# bash - -tensorflowjs_converter --input_format keras \ - path/to/my_model.h5 \ - path/to/tfjs_target_dir -``` - -## 另一种方式: 使用 Python API 直接导出为 TF.js 图层(Layer)格式 - -如果您有一个Python的Keras模型，您可以用以下方法直接输出一个Tensoflow.js图层(Layers)格式: - - -```py -# Python - -import tensorflowjs as tfjs - -def train(...): - model = keras.models.Sequential() # for example - ... - model.compile(...) - model.fit(...) - tfjs.converters.save_keras_model(model, tfjs_target_dir) -``` - -## Step 2: 将模型加载进Tensorflow.js - -使用一个web服务器为您在步骤1中生成的转换后的模型文件提供服务。注意，您可能需要将您的服务器配置为[允许跨源资源共享(CORS)](https://enable-cors.org/), 以允许在 JavaScript 中提取文件。 - -然后通过提供model.json文件的URL将模型加载到TensorFlow.js中： - - -```js -// JavaScript - -import * as tf from '@tensorflow/tfjs'; - -const model = await tf.loadLayersModel('https://foo.bar/tfjs_artifacts/model.json'); -``` - -现在，该模型已准备好进行推理(inference)，评估(evaluation)或重新训练(re-training)。例如，模型完成加载后可以立即进行预测(predict)： - - -```js -// JavaScript - -const example = tf.fromPixels(webcamElement); // for example -const prediction = model.predict(example); -``` - -很多[Tensorflow.js样例](https://github.com/tensorflow/tfjs-examples)采用这种方法，使用已在 Google 云存储上转换和托管的预训练模型。 - -注意，您使用`model.json`文件名引用整个模型。`loadModel(...)` 获取 `model.json`，并且通过额外的HTTP(S)请求以获取`model.json`权重清单中引用的分片权重文件。 此方法允许浏览器将这些文件全部缓存(可还能被缓存在互联网上其他缓存服务器中)。这是因为 `model.json`和权重分块都小于典型的缓存文件大小限制。因此这个模型可能在随后的场景中加载地更快。 - - -## 已支持的特性 - -TensorFlow.js的图层(Layers)目前仅支持基于标准Keras结构的Keras模型。 使用不支持的操作(ops)或层(layers)的模型 - 例如 自定义图层，Lambda图层，自定义损失(loss)或自定义指标(metrics)无法自动导入，因为它们依赖于无法被可靠地转换为JavaScript的Python代码。 diff --git a/site/zh-cn/js/tutorials/conversion/import_saved_model.md b/site/zh-cn/js/tutorials/conversion/import_saved_model.md deleted file mode 100644 index 6e5ceb06f4b..00000000000 --- a/site/zh-cn/js/tutorials/conversion/import_saved_model.md +++ /dev/null @@ -1,131 +0,0 @@ -# 在TensorFlow.js中引入TensorFlow GraphDef模型 - -TensorFlow GraphDef模型（一般是通过Python API创建的）可以保存成以下几种格式： -1. TensorFlow [SavedModel](https://www.tensorflow.org/programmers_guide/saved_model#overview_of_saving_and_restoring_models) -2. [Frozen Model](https://www.tensorflow.org/mobile/prepare_models#how_do_you_get_a_model_you_can_use_on_mobile) -3. [Session Bundle](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/session_bundle/README.md) -4. [Tensorflow Hub module](https://www.tensorflow.org/hub/) - -以上所有格式都可以被[TensorFlow.js converter](https://github.com/tensorflow/tfjs-converter)转换成TensorFlow.js可读取的模型格式，并用于推算（inference）。 - -（注意：TensorFlow已经淘汰了session bundle格式，请将您的模型转换成SavedModel格式。） - -## 必要条件 - -模型转换的工作需要用到Python环境；你可以用[pipenv](https://github.com/pypa/pipenv) 或 [virtualenv](https://virtualenv.pypa.io)创建一个隔离的环境。用这条命令安装模型转换器： - -```bash - pip install tensorflowjs -``` - -将TensorFlow模型引入到TensorFlow.js需要两个步骤。首先，将您的模型转换为TensorFlow.js可用的web格式，然后载入到TensorFlow.js中。 - -## 第一步：将TensorFlow模型转换至TensorFlow.js可用的 web 格式模型 - -运行转换器提供的转换脚本： - -用法：以SavedModel为例： - -```bash -tensorflowjs_converter \ - --input_format=tf_saved_model \ - --output_node_names='MobilenetV1/Predictions/Reshape_1' \ - --saved_model_tags=serve \ - /mobilenet/saved_model \ - /mobilenet/web_model -``` - -Frozen model 为例: - -```bash -tensorflowjs_converter \ - --input_format=tf_frozen_model \ - --output_node_names='MobilenetV1/Predictions/Reshape_1' \ - /mobilenet/frozen_model.pb \ - /mobilenet/web_model -``` - -Tensorflow Hub module 为例: - -```bash -tensorflowjs_converter \ - --input_format=tf_hub \ - 'https://tfhub.dev/google/imagenet/mobilenet_v1_100_224/classification/1' \ - /mobilenet/web_model -``` - -|脚本参数 | 描述 | -|---|---| -|`input_path` | saved model, session bundle 或 frozen model的完整的路径，或TensorFlow Hub模块的路径。| -|`output_path` | 输出文件的保存路径。| - -| 选项 | 描述 -|---|---| -|`--input_format` | 要转换的模型的格式。SavedModel 为 tf_saved_model, frozen model 为 tf_frozen_model, session bundle 为 tf_session_bundle, TensorFlow Hub module 为 tf_hub，Keras HDF5 为 keras。 | -|`--output_node_names`| 输出节点的名字，每个名字用逗号分离。| -|`--saved_model_tags` | 只对SavedModel转换用的选项：输入需要加载的MetaGraphDef相对应的tag，多个tag请用逗号分隔。默认为 `serve`。| -|`--signature_name` | 只对TensorFlow Hub module转换用的选项：对应要加载的签名，默认为`default`。请参考 https://www.tensorflow.org/hub/common_signatures/.| - -用以下命令查看帮助信息： - -```bash -tensorflowjs_converter --help -``` - -### 转换器产生的文件 - -转换脚本会产生两种文件： - -* `model.json` （数据流图和权重清单） -* `group1-shard\*of\*` （二进制权重文件） - -这里举例Mobilenet v2模型转换后输出的文件： - -```html - output_directory/model.json - output_directory/group1-shard1of5 - ... - output_directory/group1-shard5of5 -``` - -## 第二步：在浏览器加载和运行模型 - -1. 安装tfjs-convert npm包： - -`yarn add @tensorflow/tfjs` 或 `npm install @tensorflow/tfjs` - -2. 创建 [FrozenModel class](https://github.com/tensorflow/tfjs-converter/blob/master/src/executor/frozen_model.ts) 并开始推算： - -```js -import * as tf from '@tensorflow/tfjs'; -import {loadGraphModel} from '@tensorflow/tfjs-converter'; - -const MODEL_URL = 'model_directory/model.json'; - -const model = await loadGraphModel(MODEL_URL); -const cat = document.getElementById('cat'); -model.execute(tf.fromPixels(cat)); -``` - -具体代码请参考 [MobileNet 演示](https://github.com/tensorflow/tfjs-converter/tree/master/demo/mobilenet). - -`loadGraphModel` API中的`LoadOptions`参数可以用来发送密钥或者自定义请求中的头文件。更多信息请参考 [loadGraphModel() 文档](https://js.tensorflow.org/api/1.0.0/#loadGraphModel)。 - -## 支持的操作 - -目前，TensorFlow.js只支持部分TensorFlow算子。若您的模型包含了不被支持的算子，`tensorflowjs_converter`脚本会报错并列出您的模型中不被支持的算子。请在github上发起 [issue](https://github.com/tensorflow/tfjs/issues)让我们知道您需要支持的算子。 - -## 加载模型权重 - -若您只需要加载模型的权重，请参考以下代码： - -```js -import * as tf from '@tensorflow/tfjs'; - -const weightManifestUrl = "https://example.org/model/weights_manifest.json"; - -const manifest = await fetch(weightManifestUrl); -this.weightManifest = await manifest.json(); -const weightMap = await tf.io.loadWeights( - this.weightManifest, "https://example.org/model"); -``` diff --git a/site/zh-cn/js/tutorials/index.md b/site/zh-cn/js/tutorials/index.md deleted file mode 100644 index fd9acba877e..00000000000 --- a/site/zh-cn/js/tutorials/index.md +++ /dev/null @@ -1,50 +0,0 @@ -# 开始 - -TensorFlow.js 是一个 JavaScript 库，用于在浏览器和 Node.js 训练和部署机器学习模型。 - -了解开始的更多方式，请参阅下面的部分。 - - -## 在不直接处理张量的情况下编写 ML 程序 - -想要开始机器学习，同时不用担心任何类似张量或优化器的低级细节吗？ - -ml5.js 库构建在 TensorFlow.js 之上，通过简洁的、可利用的 API，可以在浏览器中访问机器学习算法和模型。 - -Check out ml5.js - - -## 安装 TensorFlow.js - -TensorFlow.js 与 Tensors (张量)、Layers (图层)、Optimizers (优化器) 和损失函数等概念兼容（或希望与它们兼容）吗？TensorFlow.js 为 JavaScript 中神经网络编程提供了灵活的构建块。 - -请参阅如何在浏览器或 Node.js 中启动和运行 TensorFlow.js 代码。 - -Get Setup - -### 将预训练模型转换到 TensorFlow.js - -学习如何将预训练模型从 python 转换为 TensorFlow.js - -Keras Model -GraphDef Model - -## 从现有 TensorFlow.js 代码中学习 - -tfjs-examples 提供了使用 TensorFlow.js 实现各种 ML 任务的简短代码示例。 - -See it on GitHub - -## 可视化您的 TensorFlow.js 模型的状态 - -tfjs-vis 是一个用于在浏览器内实现可视化的小型库，用于TensorFlow.js。 - -See it on GitHub -See Demo - - -## 准备好用 TensorFlow.js 处理数据 - -TensorFlow.js 支持使用 ML 最佳实践处理数据。 - -See docs diff --git a/site/zh-cn/js/tutorials/learning/ml.md b/site/zh-cn/js/tutorials/learning/ml.md deleted file mode 100644 index f8526f5f925..00000000000 --- a/site/zh-cn/js/tutorials/learning/ml.md +++ /dev/null @@ -1,43 +0,0 @@ -# 机器学习资源 - -使用Tensorflow.js进行机器学习需要一些列不同的只是,尤其是机器学习,神经网络,Javascript,Node.js以及浏览器开发这些方面。取决于你的背景你可能对其中的一个或一些知识领域并不熟悉。这里有很多优秀的在线学习资源，本页将重点介绍一些资源，以帮助用神经网络引导您的机器学习知识。 - -## 高水平介绍 - -我们推荐您可以从下列视频获取高水平的深度学习以及Tensorflow.js的介绍。 - -- [但是什么是神经网络?](https://www.youtube.com/watch?v=aircAruvnKk) by 3blue1brown -- [深度学习中的 JS](https://www.youtube.com/watch?v=SV-cgdobtTA) by Ashi Krishnan - - -## TensorFlow.js 介绍 - -下列资源诸着重介绍了Tensorflow.js同时也涉及了机器学习的基本知识。 - -- [TensorFlow.js](https://www.youtube.com/playlist?list=PLs6AluHXaQnjeI6jzDkpKXvbPj31i4GgF) by TensorFlow -- [TensorFlow.js: 智能与学习系列](https://www.youtube.com/playlist?list=PLRqwX-V7Uu6YIeVA3dNxbR9PYj4wV31oQ) by Coding Train -- [TensorFlow.js 用Javascript进行机器学习](https://www.youtube.com/playlist?list=PLZbbT5o_s2xr83l8w44N_g3pygvajLrJ-) by Deeplizard - -## 综合课程 - -下列是综合的深度学习在线课程。大部分课程使用python作为主要语言。然而，即使语法没有涉及但是这些概念也可以被转换在Tensorflow.js上。 - -- [机器学习速成课程](https://developers.google.com/machine-learning/crash-course/ml-intro) by Google -- [专业化的深度学习](https://www.coursera.org/specializations/deep-learning) by Coursera - -## 综合书籍 - -- [神经网络与深度学习](http://neuralnetworksanddeeplearning.com/) by Michael Nielsen -- [用python进行深度学习](https://www.manning.com/books/deep-learning-with-python) by Francois Chollet - - -## 数学概念 - -机器学习是一门需要一定数学基础的学科，如果你只是需要使用机器学习模型，那么理解模型背后的数学原理是没有必要的。如果你计划调整机器学习模型或是从头建立一个模型，那么熟悉基本的数学概念对您是有帮助的。 -你不必预先学习所有的数学知识，而是可以在遇到不熟悉的概念时查找并学习它们。 - - -- [Essence of Linear Algebra](https://www.youtube.com/playlist?list=PLZHQObOWTQDPD3MizzM2xVFitgF8hE_ab) by 3blue1brown -- [Essence of Calculus](https://www.youtube.com/playlist?list=PLZHQObOWTQDMsr9K-rj53DwVRMYO3t5Yr) by 3blue1brown -- [Linear Algebra](https://www.khanacademy.org/math/linear-algebra) by Khan Academy -- [Calculus](https://www.khanacademy.org/math/calculus-home) by Khan Academy \ No newline at end of file diff --git a/site/zh-cn/js/tutorials/setup.md b/site/zh-cn/js/tutorials/setup.md deleted file mode 100644 index 92b99ceb48e..00000000000 --- a/site/zh-cn/js/tutorials/setup.md +++ /dev/null @@ -1,161 +0,0 @@ -# 安装 - -## 浏览器安装 - -在您基于浏览器的项目中获取TensorFlow.js有以下两种主要方法 - -- 使用 - [脚本标签(script tags)](https://developer.mozilla.org/en-US/docs/Learn/HTML/Howto/Use_JavaScript_within_a_webpage)。 -- 从[NPM](https://www.npmjs.com)安装并且使用[Parcel](https://parceljs.org/), - [WebPack](https://webpack.js.org/)或是 - [Rollup](https://rollupjs.org/guide/en)这样的构建工具。 - -如果您不熟悉Web开发，或者从未听说过webpack或parcel等工具，_我们建议您使用脚本标签(script -tags)_。如果您经验丰富或想要编写更大的程序，那么使用构建工具进行探索可能更加合适。 - -### 使用脚本标签(script tags) - -将以下脚本标签添加到您的主HTML文件中： - - -```html - -``` - -有关脚本标签的设置，请参阅代码示例： - - - -### 从NPM安装 - -您可以使用 -[npm cli](https://docs.npmjs.com/cli/npm)工具或是[yarn](https://yarnpkg.com/en/)安装TensorFlow.js。 - -``` -yarn add @tensorflow/tfjs -``` - -_或者_ - -``` -npm install @tensorflow/tfjs -``` - - - - -## Node.js 安装 - -您可以使用 -[npm cli](https://docs.npmjs.com/cli/npm)工具或是[yarn](https://yarnpkg.com/en/)安装TensorFlow.js。 - -**选项1:** 安装带有原生C++绑定的TensorFlow.js。 - -``` -yarn add @tensorflow/tfjs-node -``` - -_或者_ - -``` -npm install @tensorflow/tfjs-node -``` - -**选项2:** -（仅限Linux）如果您的系统具有[支持CUDA](https://www.tensorflow.org/install/install_linux#NVIDIARequirements)的NVIDIA®GPU，请使用GPU包以获得更高的性能。 - -``` -yarn add @tensorflow/tfjs-node-gpu -``` - -_or_ - -``` -npm install @tensorflow/tfjs-node-gpu -``` - -**选项3:** 安装纯JavaScript版本，这是性能方面最慢的选项。 - -``` -yarn add @tensorflow/tfjs -``` - -_or_ - -``` -npm install @tensorflow/tfjs -``` - - - -### TypeScript - -当使用TypeScript时，如果您的项目使用严格的空值检查，或者在编译过程中遇到错误，则您可能需要在您的`tsconfig.json`文件中设置`skipLibCheck：true`。 diff --git a/site/zh-cn/js/tutorials/training/handwritten_digit_cnn.md b/site/zh-cn/js/tutorials/training/handwritten_digit_cnn.md deleted file mode 100644 index 5ee7b8a4ce9..00000000000 --- a/site/zh-cn/js/tutorials/training/handwritten_digit_cnn.md +++ /dev/null @@ -1,5 +0,0 @@ -# 使用卷积神经网络进行手写数字识别 - -在这个教程中，我们将会使用Tensorflow.js建立手写数字识别卷积神经网络模型。首先，我们训练分类器，让它查看数千个图像以及其标签。然后我们将使用模型从未见过的测试数据来评估分类器的准确性。 - -在 CodeLab 中打开 \ No newline at end of file diff --git a/site/zh-cn/js/tutorials/training/linear_regression.md b/site/zh-cn/js/tutorials/training/linear_regression.md deleted file mode 100644 index a9a4767a120..00000000000 --- a/site/zh-cn/js/tutorials/training/linear_regression.md +++ /dev/null @@ -1,9 +0,0 @@ -# 从二维数据进行预测 - -在这个教程中你将使用一组汽车的数字数据集训练一个模型并且预测。 - -本练习将演示训练许多不同类型模型的常用步骤，但将使用小数据集和简单（浅）模型。 -主要目的是帮助您熟悉TensorFlow.js训练模型的基本术语、概念和语法 -并为进一步的探索和学习提供一个垫脚石。 - -打开Codelab diff --git a/site/zh-cn/js/tutorials/transfer/audio_recognizer.md b/site/zh-cn/js/tutorials/transfer/audio_recognizer.md deleted file mode 100644 index ddf9d1196ed..00000000000 --- a/site/zh-cn/js/tutorials/transfer/audio_recognizer.md +++ /dev/null @@ -1,7 +0,0 @@ -# 语音识别的迁移学习 - -在这个教程中，您将会学习到如何建立一个客制化的语音分类器。您将使用Tensorflow.js在浏览器中进行训练。您将使用它通过发出声音来控制浏览器中的滑块。 - -您将会使用迁移学习建立一个模型，这个模型使用较少的训练数据对短音频进行分类。您将会使用预训练模型[语音命令识别](https://github.com/tensorflow/tfjs-models/tree/master/speech-commands)。您将会在预训练模型的基础上训练一个新的模型，以识别您自己定义的声音类。 - -这个教程在codelab上已经为您准备好了 [从这里打开codelab吧！](https://codelabs.developers.google.com/codelabs/tensorflowjs-audio-codelab/index.html) diff --git a/site/zh-cn/js/tutorials/transfer/image_classification.md b/site/zh-cn/js/tutorials/transfer/image_classification.md deleted file mode 100644 index 9fa026b52d5..00000000000 --- a/site/zh-cn/js/tutorials/transfer/image_classification.md +++ /dev/null @@ -1,7 +0,0 @@ -# 图像分类迁移学习 - -本指南中，你将学习如何使用TensorFlow.js创建一个自定义的图像分类器，并在浏览器中对它进行训练。 - -通过使用迁移学习，你将用少量的训练数据创建一个高精度的模型。你将使用一个名为MobileNet的预训练模型来做图像分类。在此基于上，你将训练一个模型以自定义其所能识别的图像分类。 - -本指南通过一个 codelab 展示。 [点击这个链接打开codelab](https://codelabs.developers.google.com/codelabs/tensorflowjs-teachablemachine-codelab/index.html) diff --git a/site/zh-cn/js/tutorials/transfer/what_is_transfer_learning.md b/site/zh-cn/js/tutorials/transfer/what_is_transfer_learning.md deleted file mode 100644 index e83dd2d788d..00000000000 --- a/site/zh-cn/js/tutorials/transfer/what_is_transfer_learning.md +++ /dev/null @@ -1,13 +0,0 @@ -# 什么是迁移学习 - -复杂的深度学习模型通常有数百万的参数（weights），从头训练模型需要大量计算资源。迁移学习将已经在相关任务中训练过的模型的一部分复用到新模型中，从而很大程度地降低对大量计算资源的需要。 - -例如，本节的下一个指南将向你展示如何利用一个的已训练好的模型来构建你自己的图像识别器，此图像识别器能够识别图像中 1000 种不同类型的物体。使用预训练模型的已有知识来识别你的图像，会比原始模型所需的训练数据少得多。 - -在快速开发新模型时这点很有用，尤其是在浏览器和移动设备这类受限环境下。 - -通常在实现迁移学习时，我们不调整原始模型的参数，而是删除原始模型的最后一层，并基于此截断模型的输出训练一个新的（通常相当浅的）模型。这就是你将在本节指南中看到的技术。 - - -- [构建一个基于迁移学习的图像分类器](image_classification) -- [构建一个基于迁移学习的语音识别器](audio_recognizer) diff --git a/site/zh-cn/lite/convert/cmdline.md b/site/zh-cn/lite/convert/cmdline.md deleted file mode 100644 index 4b1dbde519a..00000000000 --- a/site/zh-cn/lite/convert/cmdline.md +++ /dev/null @@ -1,35 +0,0 @@ -# 模型转换器命令参考 - -本页介绍如何使用 TensorFlow 2.0 命令行工具中的[TensorFlow Lite 模型转换器](index.md)。首选的转换方法是使用 [Python API](python_api.md)。 - -## 综述 - -TensorFlow Lite 转换器命令行工具 `tflite_convert` 它支持基础模型转换。使用 `TFLiteConverter` [Python API](python_api.md) 支持任何涉及量化或其他参数的转换(例如：SavedModels 签名，或者在 Keras 模型上自定义对象). - -## 使用 - -下列命令参数用于输入和输出文件。 - -* `--output_file`. 类型: string. 指定输出文件的绝对路径。 -* --saved_model_dir. 类型: string. 指定含有 TensorFlow 1.x 或者 2.0 使用 SavedModel 生成文件的绝对路径目录。 -* --keras_model_file. Type: string. 指定含有 TensorFlow 1.x 或者 2.0 使用 tf.keras model 生成 HDF5 文件的绝对路径目录。 - -例如： - -``` -tflite_convert \ - --saved_model_dir=/tmp/mobilenet_saved_model \ - --output_file=/tmp/mobilenet.tflite -``` - -## 附加说明 - -### 从源代码构建 - -想要运行最新版本的 TensorFlow Lite 模型转换器可以通过 [pip](https://www.tensorflow.org/install/pip) 安装 TensorFlow 2.0 测试版或者[克隆 TensorFlow 代码库](https://www.tensorflow.org/install/source)然后使用 `bazel` 从源代码编译 TensorFlow 。下面是一个从源代码编译 TensorFlow 的例子。 - -``` -bazel run //third_party/tensorflow/lite/python:tflite_convert -- \ - --saved_model_dir=/tmp/mobilenet_saved_model \ - --output_file=/tmp/mobilenet.tflite -``` diff --git a/site/zh-cn/lite/convert/concrete_function.md b/site/zh-cn/lite/convert/concrete_function.md deleted file mode 100644 index 1af95840853..00000000000 --- a/site/zh-cn/lite/convert/concrete_function.md +++ /dev/null @@ -1,178 +0,0 @@ -# 生成一个具体函数 - -为了将 TensorFlow 2.0 的模型转换成 TensorFlow Lite，模型需要转换成有具体的函数。这篇文档将会阐述什么是具体的函数以及如何根据现有的模型生成函数。 - -[TOC] - -## 背景 - -在 TensorFlow 2.0 中，在一般情况下是优先执行的。TensorFlow 的优先执行是一个命令式的编程环境，可以立即执行评估操作，并且无需建立图形。评估操作会返回具体值，而不是在后续运行所构建的计算图形。一份关于优先执行的详细指南[此处获取](https://github.com/tensorflow/docs/blob/master/site/en/r2/guide/eager.ipynb)。 - -虽然通过执行优先执行将会使得开发环境和调试更加具有互动性，但是它却不允许部署在设备上。`tf.function` API接口可以将模型作为图片保存，这需要在 TensorFlow Lite 2.0 版本中运行。包含在 `tf.function` 装饰器中的所有操作都可以导出为图形，然后可以将其转换成 TensorFlow Lite FlatBuffer 的格式。 - -## 术语 - -本文档后续中用到的一些术语。 - -* **签名** - 输入和输出的一组操作。 -* **具体函数** - 具有一个签名的图表。 -* **多态函数** - Python 可以调用封装了几个具体函数图的API。 - -## 方法 - -这一部分描述了如何生成一个具体函数。 - -### 使用 `tf.function` 注释函数 - -使用 `tf.function` 注释函数会生成包含这些操作的*多态函数*。所有未使用 `tf.function` 注释的操作都将通过优先执行进行评估。以下示例显示了如何使用 `tf.function`。 - -```python -@tf.function -def pow(x): - return x ** 2 -``` - -```python -tf.function(lambda x : x ** 2) -``` - -### 创建要保存的对象 - -`tf.function` 可以选择存储为 `tf.Module` 对象的一部分。变量只能在 `tf.Module` 中定义一次。下面的示例显示了创建 `Checkpoint` 的继承类的两种不同方法。 - -```python -class BasicModel(tf.Module): - - def __init__(self): - self.const = None - - @tf.function - def pow(self, x): - if self.const is None: - self.const = tf.Variable(2.) - return x ** self.const - -root = BasicModel() -``` - -```python -root = tf.Module() -root.const = tf.Variable(2.) -root.pow = tf.function(lambda x : x ** root.const) -``` - -### 生成具体函数 - -具体函数定义的图表，可以被转化成 TensorFlow Lite 的模型或者导出到 SavedModel。为了从多态函数中导出具体函数，首先需要定义签名。签名可以根据以下方式定义： - -* 在 `tf.function` 中定义 `input_signature` 参数。 -* 将 `tf.TensorSpec` 传递给 `get_concrete_function`：例如 `tf.TensorSpec（shape = [1]，dtype = tf.float32）`。 -* 将样本输入量传递给 `get_concrete_function`：例如 `tf.constant（1。，shape = [1]）`。 - -下面的例子展示了如何在 `tf.function` 中`input_signature` 参数。 - -```python -class BasicModel(tf.Module): - - def __init__(self): - self.const = None - - @tf.function(input_signature=[tf.TensorSpec(shape=[1], dtype=tf.float32)]) - def pow(self, x): - if self.const is None: - self.const = tf.Variable(2.) - return x ** self.const - -# 创建 tf.Module 对象。 -root = BasicModel() - -# 获取具体函数。 -concrete_func = root.pow.get_concrete_function() -``` - -下面的示例展示了将样本输入量传递给 `get_concrete_function`。 - -```python -# 创建 tf.Module 对象。 -root = tf.Module() -root.const = tf.Variable(2.) -root.pow = tf.function(lambda x : x ** root.const) - -# 获取具体函数。 -input_data = tf.constant(1., shape=[1]) -concrete_func = root.pow.get_concrete_function(input_data) -``` - -## 示例代码 - -```python -import tensorflow as tf - -# 初始化 tf.Module 对象。 -root = tf.Module() - -# 初始化变量一次。 -root.var = None - -# 定义一个函数，可以不用预先进行计算操作。 -@tf.function -def exported_function(x): - # 每个变量只能被定义一次。变量可以在函数中定义。 - # 但是该函数需要包含有函数外部的引用。 - if root.var is None: - root.var = tf.Variable(tf.random.uniform([2, 2])) - root.const = tf.constant([[37.0, -23.0], [1.0, 4.0]]) - root.mult = tf.matmul(root.const, root.var) - return root.mult * x - -# 将函数保存为 tf.function 对象的一部分。 -root.func = exported_function - -# 获取具体函数。 -concrete_func = root.func.get_concrete_function( - tf.TensorSpec([1, 1], tf.float32)) -``` - -## 常见问题 - -### 怎么才能将一个具体函数保存为 SavedModel ？ - -想要在 TensorFlow 模型转换成 TensorFlow Lite 之前保存的用户应该将模型保存为 SavedModel。获取具体函数之后，调用 `tf.saved_model.save` 来保存模型。可以使用下面的指令来保存上面的例子。 - -```python -tf.saved_model.save(root, export_dir, concrete_func) -``` - -参考 -[指南](https://github.com/tensorflow/docs/blob/master/site/en/r2/guide/saved_model.ipynb) -来获取保存模型的详细介绍。 - -### 怎么才能从 SavedModel 中获得一个具体函数？ - -SavedModel 中的每个具体功能都可以通过签名密钥进行标识。 默认签名密钥是`tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY`。 下面的示例显示了如何从模型中获取具体功能。 - -```python -model = tf.saved_model.load(export_dir) -concrete_func = model.signatures[ - tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY] -``` - -### 如何获得 `tf.Keras` 模型的具体功能？ - -有下面两种办法你可以使用： - -1. 将模型保存为 SavedModel。在保存过程中会生成一个具体函数，可以在加载模型时访问其功能。 -2. 使用 `tf.function` 注释模型，如下所示。 - -```python -model = tf.keras.Sequential([tf.keras.layers.Dense(units=1, input_shape=[1])]) -model.compile(optimizer='sgd', loss='mean_squared_error') -model.fit(x=[-1, 0, 1, 2, 3, 4], y=[-3, -1, 1, 3, 5, 7], epochs=50) - -# 从 Keras 模型中得到一个具体函数。 -run_model = tf.function(lambda x : model(x)) - -# 保存函数。 -concrete_func = run_model.get_concrete_function( - tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype)) -``` diff --git a/site/zh-cn/lite/convert/python_api.md b/site/zh-cn/lite/convert/python_api.md deleted file mode 100644 index 3de4532fa84..00000000000 --- a/site/zh-cn/lite/convert/python_api.md +++ /dev/null @@ -1,262 +0,0 @@ -# 模型转换器（Converter）的 Python API 指南 - -此页面提供了一个关于在 TensorFlow 2.0 中如何使用 -[TensorFlow Lite 转换器（TensorFlow Lite converter）](index.md) Python API 的示例。 - -[TOC] - -## Python API - -在 TensorFlow 2.0 中，用来将原始的 TensorFlow 模型格式转换为 TensorFlow Lite 的 Python API 是 `tf.lite.TFLiteConverter`。在 `TFLiteConverter` 中有以下的类方法（classmethod）： - -* `TFLiteConverter.from_saved_model()`：用来转换 - [SavedModel 格式模型](https://www.tensorflow.org/guide/saved_model)。 -* `TFLiteConverter.from_keras_model()`：用来转换 - [`tf.keras` 模型](https://www.tensorflow.org/guide/keras/overview)。 -* `TFLiteConverter.from_concrete_functions()`：用来转换 - [concrete functions](https://tensorflow.org/guide/concrete_function)。 - -注意: 在 TensorFlow Lite 2.0 中有一个不同版本的 -`TFLiteConverter` API， 该 API 只包含了 -[`from_concrete_function`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/lite/TFLiteConverter#from_concrete_function)。 -本文中用到的的新版本 API 可以通过 pip 安装 -[`tf-nightly-2.0-preview`](#2.0-nightly)。 - -本文展示了 API 的 [示例用法](#examples)，不同 TensorFlow 版本的 API 详细列表请看 [1.X 版本到 2.0 版本 API 的改变](#differences)，和 -[安装 TensorFlow](#versioning) 来安装和使用。 - -## 示例 - -### 转换 SavedModel 格式模型 - -以下示例展示了如何将一个 -[SavedModel](https://www.tensorflow.org/guide/saved_model) 转换为 -TensorFlow Lite 中的 [`FlatBuffer`](https://google.github.io/flatbuffers/)格式。 - -```python -import tensorflow as tf - -# 建立一个简单的模型。 -root = tf.train.Checkpoint() -root.v1 = tf.Variable(3.) -root.v2 = tf.Variable(2.) -root.f = tf.function(lambda x: root.v1 * root.v2 * x) - -# 保存模型。 -export_dir = "/tmp/test_saved_model" -input_data = tf.constant(1., shape=[1, 1]) -to_save = root.f.get_concrete_function(input_data) -tf.saved_model.save(root, export_dir, to_save) - -# 转换模型。 -converter = tf.lite.TFLiteConverter.from_saved_model(export_dir) -tflite_model = converter.convert() -``` - -此 API 不支持指定输入向量的维度。 如果您的模型需要指定输入向量的维度，请使用 -[`from_concrete_functions`](#concrete_function) 来完成。 示例： - -```python -model = tf.saved_model.load(export_dir) -concrete_func = model.signatures[ - tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY] -concrete_func.inputs[0].set_shape([1, 256, 256, 3]) -converter = TFLiteConverter.from_concrete_functions([concrete_func]) -``` - -### 转换 Keras 模型 - -以下示例展示了如何将一个 -[tf.keras 模型](https://www.tensorflow.org/guide/keras/overview) 转换为 -TensorFlow Lite 中的 [`FlatBuffer`](https://google.github.io/flatbuffers/) 格式。 - -```python -import tensorflow as tf - -# 创建一个简单的 Keras 模型。 -x = [-1, 0, 1, 2, 3, 4] -y = [-3, -1, 1, 3, 5, 7] - -model = tf.keras.models.Sequential( - [tf.keras.layers.Dense(units=1, input_shape=[1])]) -model.compile(optimizer='sgd', loss='mean_squared_error') -model.fit(x, y, epochs=50) - -# 转换模型。 -converter = tf.lite.TFLiteConverter.from_keras_model(model) -tflite_model = converter.convert() -``` - -### 转换 concrete function - -以下示例展示了如何将 TensorFlow 中的 -[concrete function](https://tensorflow.org/guide/concrete_function) -转换为TensorFlow Lite 中的 [`FlatBuffer`](https://google.github.io/flatbuffers/) 格式。 - -```python -import tensorflow as tf - -# 建立一个模型。 -root = tf.train.Checkpoint() -root.v1 = tf.Variable(3.) -root.v2 = tf.Variable(2.) -root.f = tf.function(lambda x: root.v1 * root.v2 * x) - -# 生成 concrete function。 -input_data = tf.constant(1., shape=[1, 1]) -concrete_func = root.f.get_concrete_function(input_data) - -# 转换模型。 -# -# `from_concrete_function` 的传入参数被设计为一个一个 concrete function 的列表，然而 -# 现阶段仅支持每次调用时仅接受一个concrete function。 -# 同时转换多个concrete function的功能正在开发中。 -converter = tf.lite.TFLiteConverter.from_concrete_functions([concrete_func]) -tflite_model = converter.convert() -``` - -### 端到端 MobileNet 转换 - -以下示例展示了如何将将一个提前训练好的 -`tf.keras` MobileNet 模型转换为 TensorFlow Lite 支持的类型并运行推断 （inference）。 随机数据分别在 -TensorFlow 和 TensorFlow Lite 模型中运行的结果将被比较。如果是从文件加载模型，请使用 `model_path` 来代替 `model_content`。 - -```python -import numpy as np -import tensorflow as tf - -# 加载 MobileNet tf.keras 模型。 -model = tf.keras.applications.MobileNetV2( - weights="imagenet", input_shape=(224, 224, 3)) - -# 转换模型。 -converter = tf.lite.TFLiteConverter.from_keras_model(model) -tflite_model = converter.convert() - -# 加载 TFLite 模型并分配张量（tensor）。 -interpreter = tf.lite.Interpreter(model_content=tflite_model) -interpreter.allocate_tensors() - -# 获取输入和输出张量。 -input_details = interpreter.get_input_details() -output_details = interpreter.get_output_details() - -# 使用随机数据作为输入测试 TensorFlow Lite 模型。 -input_shape = input_details[0]['shape'] -input_data = np.array(np.random.random_sample(input_shape), dtype=np.float32) -interpreter.set_tensor(input_details[0]['index'], input_data) - -interpreter.invoke() - -# 函数 `get_tensor()` 会返回一份张量的拷贝。 -# 使用 `tensor()` 获取指向张量的指针。 -tflite_results = interpreter.get_tensor(output_details[0]['index']) - -# 使用随机数据作为输入测试 TensorFlow 模型。 -tf_results = model(tf.constant(input_data)) - -# 对比结果。 -for tf_result, tflite_result in zip(tf_results, tflite_results): - np.testing.assert_almost_equal(tf_result, tflite_result, decimal=5) -``` - -## 总结 1.X 版本到 2.0 版本 API 的改变 - -本节总结了从 1.X 到 2.0 版本 Python API 的改变。 -如果对某些改动有异议，请提交 -[GitHub issue](https://github.com/tensorflow/tensorflow/issues)。 - -### `TFLite转换器` 支持的格式类型 - -`TFLite转换器` 在 2.0 版本中支持由 1.X 版本和 2.0 版本生成的 SavedModels 和 Keras 模型。但是，转换过程不再支持由 -1.X 版本冻结的 `GraphDefs`。 开发者可通过调用 `tf.compat.v1.lite.TFLiteConverter` 来把冻结的 -`GraphDefs` 转换到 TensorFlow Lite 版本。 - -### 量化感知训练（Quantization-aware training） - -以下与 -[量化感知训练（Quantization-aware training）](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/quantize) -有关的属性和方法在 TensorFlow 2.0 中从`TFLiteConverter` 中被移除。 - -* `inference_type` -* `inference_input_type` -* `quantized_input_stats` -* `default_ranges_stats` -* `reorder_across_fake_quant` -* `change_concat_input_ranges` -* `post_training_quantize` - 在 1.X API 中被弃用 -* `get_input_arrays()` - -支持量化感知训练的重写器（rewriter）函数不支持由 TensorFlow 2.0 生成的模型。此外，TensorFlow Lite 的量化 API -已按支持 Keras 中量化感知训练 API 的思路重新设计和精简。 在新的量化 API 部署前，这些属性将不会出现在 2.0 的 API 中。开发者可以使用 -`tf.compat.v1.lite.TFLiteConverter` 来转换由重写器函数生成的模型。 - -### 关于 `TFLiteConverter` 中属性的改变 - -属性 `target_ops` 已成为 `TargetSpec` 中的属性且作为未来对优化框架的补充被重命名为 `supported_ops`。 - -此外，以下属性被移除: - -* `drop_control_dependency` (default: `True`) - TFLite 现不支持控制流（control flow），所以此属性将恒为 `True`。 -* _Graph visualization_ - 在 TensorFlow 2.0 中，推荐使用 - [visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/visualize.py) 实现对 TensorFlow Lite 图（graph）的可视化。 - 不同于 GraphViz, 它支持开发者对已进行过 post training 量化的图（graph）可视化。以下与图可视化的属性将被移除： - * `output_format` - * `dump_graphviz_dir` - * `dump_graphviz_video` - -### 通用 API 的改变 - -#### 转换方法 - -以下在 1.X 中被弃用的方法不会在 2.0 中出现： - -* `lite.toco_convert` -* `lite.TocoConverter` - -#### `lite.constants` - -在 2.0 中，为了减少 TensorFlow 和 TensorFlow Lite 之间的重复移除了 `lite.constants` API。以下的列表展示了 -`lite.constant` 中的类型在 TensorFlow 中对应的类型： - -* `lite.constants.FLOAT`: `tf.float32` -* `lite.constants.INT8`: `tf.int8` -* `lite.constants.INT32`: `tf.int32` -* `lite.constants.INT64`: `tf.int64` -* `lite.constants.STRING`: `tf.string` -* `lite.constants.QUANTIZED_UINT8`: `tf.uint8` - -此外，`lite.constants.TFLITE` 和 `lite.constants.GRAPHVIZ_DOT` 被移除（由于 `TFLiteConverter` 中的 flage `output_format`被移除）。 - -#### `lite.OpHint` - -由于 API `OpHint` 与 2.0 的 API 不兼容，故暂不可用。 此 API可用于转换基于 LSTM 的模型。 在 2.0 中对 -LSTMs 的支持正在被探究。所有与 `lite.experimental` 有关的 API 都因此被移除。 - -## 安装 TensorFlow - -### 安装 TensorFlow 2.0 nightly - -可用以下命令安装 TensorFlow 2.0 nightly： - -``` -pip install tf-nightly-2.0-preview -``` - -### 在已安装的 1.X 中使用 TensorFlow 2.0 - -可通过以下代码片段从最近安装的 1.X 中使用 TensorFlow 2.0。 - -```python -import tensorflow.compat.v2 as tf - -tf.enable_v2_behavior() -``` - -### 从源代码安装 - -为使用最新版本的 TensorFlow Lite 转换器 Python API， -您可通过以下方式安装 nightly build： -[pip](https://www.tensorflow.org/install/pip) (推荐方式) 或 -[Docker](https://www.tensorflow.org/install/docker), 或 -[从源代码构建 pip 包](https://www.tensorflow.org/install/source). diff --git a/site/zh-cn/lite/convert/quantization.md b/site/zh-cn/lite/convert/quantization.md deleted file mode 100644 index a083e5248da..00000000000 --- a/site/zh-cn/lite/convert/quantization.md +++ /dev/null @@ -1,29 +0,0 @@ -# 转换量化模型 -本文提供有关如何转换量化 TensorFlow Lite 模型的信息。详细信息，请参阅[模型优化](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/performance/model_optimization.md)。 - -# 训练后：针对特定 CPU 型号的量化模型 -创建小模型的最简单方法是在推理期间将权重量化为 8 位并“在运行中”量化输入/激活。这具有延迟优势，但优先考虑减小尺寸。 - -在转换期间，将 optimizations 标志设置为针对大小进行优化： -``` -converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) -converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE] -tflite_quant_model = converter.convert() -``` - -# 训练过程中：仅用于整数执行的量化模型 -仅用于整数执行的量化模型获得具有更低延迟，更小尺寸和仅针对整数加速器兼容模型的模型。目前，这需要训练具有["假量化"节点](https://github.com/tensorflow/tensorflow/tree/r1.13/tensorflow/contrib/quantize)的模型 。 - -转换图表： -``` -converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) -converter.inference_type = tf.lite.constants.QUANTIZED_UINT8 -input_arrays = converter.get_input_arrays() -converter.quantized_input_stats = {input_arrays[0] : (0., 1.)} # mean, std_dev -tflite_model = converter.convert() -``` -对于全整数模型，输入为 uint8。mean 和 std_dev values 指定在训练模型时这些 UINT8 的值是如何值映射到输入的浮点值。 - -mean 是 0 到 255 之间的整数值，映射到浮点数 0.0f。std_dev = 255 /（float_max - float_min） - -对于大多数用户，我们建议使用训练后量化。我们正在研究用于后期训练和训练量化的新工具，我们希望这将简化生成量化模型。 diff --git a/site/zh-cn/lite/convert/rnn.md b/site/zh-cn/lite/convert/rnn.md deleted file mode 100644 index 3d059171c88..00000000000 --- a/site/zh-cn/lite/convert/rnn.md +++ /dev/null @@ -1,72 +0,0 @@ -# 转换 RNN 模型 - -TensorFlow Lite 解释器目前实现了部分 TensorFlow 操作。由于那些缺失的操作，部分模型架构不能立刻被转换。 - -一些基于 RNN 的架构将会受到这个情况的影响。下列文档概述了目前的现状，并提供了转换 RNN 模型的策略。 - -## 目前支持的转换 - -目前，只要没有指定 `sequence_length`，就可以成功转换使用[`tf.nn.static_rnn`](https://www.tensorflow.org/api_docs/python/tf/nn/static_rnn) 的 RNN 模型。 - -下列 `tf.nn.rnn_cell` 操作使用 `tf.nn.static_rnn`: - -* [tf.nn.rnn_cell.LSTMCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/LSTMCell) -* [tf.nn.rnn_cell.RNNCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/RNNCell) -* [tf.nn.rnn_cell.GRUCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/GRUCell) -* [tf.nn.rnn_cell.BasicLSTMCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/BasicLSTMCell) -* [tf.nn.rnn_cell.BasicRNNCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/BasicRNNCell) - -另外，TensorFlow Lite 提供了一些 RNN 操作的替代方法。这些方法使你可以在 TensorFlow Lite 中使用动态 RNN 架构。 - -可用的替代方法如下： - -* [tf.nn.dynamic_rnn](https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn) -* [tf.nn.bidirectional_dynamic_rnn](https://www.tensorflow.org/api_docs/python/tf/nn/bidirectional_dynamic_rnn) -* [tf.nn.rnn_cell.RNNCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/RNNCell) -* [tf.nn.rnn_cell.LSTMCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/LSTMCell) - -## 暂不支持的转换 - -TensorFlow Lite 目前暂不支持 [Control Flow](https://www.tensorflow.org/api_docs/cc/group/control-flow-ops) 操作。这表示，除非使用下文中提到的转换策略，否则使用下列 TensorFlow 函数的模型将不能被成功转换： - -* [tf.nn.static_rnn](https://www.tensorflow.org/api_docs/python/tf/nn/static_rnn) 同时指明了 `sequence_length` -* [tf.nn.dynamic_rnn](https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn) -* [tf.nn.bidirectional_dynamic_rnn](https://www.tensorflow.org/api_docs/python/tf/nn/bidirectional_dynamic_rnn) - -注意：TensorFlow Lite 计划在 2019 年底前实现所有必须的 Control Flow 操作。届时，所有的 RNN 架构将可以被成功转换。 - -## 转换策略 - -为了成功转换使用了上述函数的 RNN 模型，你需要修改它的架构并且重新训练。下列策略是可行的： - -### 1. 重构 - -如果可能，最简单的策略是重构模型架构，使用不带有 `sequence_length` 的 [tf.nn.static_rnn](https://www.tensorflow.org/api_docs/python/tf/nn/static_rnn)。 - -### 2. 使用操作提示和融合操作的替代方法 - -TensorFlow Lite 为 RNN 操作提供了一些替代方法，使得在 TensorFlow Lite 中可以使用动态 RNN 架构。使用 [OpHints](https://www.tensorflow.org/lite/guide/ops_custom#converting_tensorflow_models_to_convert_graphs)，这些方法在训练时可以正常运行，但在 TensorFlow Lite 解释器中运行时，它们被替换为特殊的融合操作。 - -下列是可用的替代方法： - -* [tf.lite.experimental.nn.dynamic_rnn](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/examples/lstm/rnn.py#L41) - * 替代 tf.nn.dynamic_rnn -* [tf.lite.experimental.nn.bidirectional_dynamic_rnn](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/examples/lstm/rnn.py#L279) - * 替代 tf.nn.bidirectional_dynamic_rnn -* [tf.lite.experimental.nn.TfLiteRNNCell](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/examples/lstm/rnn_cell.py#L39) - * 替代 tf.nn.rnn_cell.RNNCell -* [tf.lite.experimental.nn.TfLiteLSTMCell](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/examples/lstm/rnn_cell.py#L159) - * 替代 tf.nn.rnn_cell.LSTMCell - - -注意：这些替代方法必须一起使用。例如，如果你正在使用 `tf.lite.experimental.nn.dynamic_rnn`，你必须将它和 `tf.lite.experimental.nn.TfLiteRNNCell` 配合使用，而不是使用 `tf.nn.rnn_cell.RNNCell`。 - - -使用[tf.keras.layers.StackedRNNCells](https://www.tensorflow.org/api_docs/python/tf/keras/layers/StackedRNNCells) 来替换 [tf.nn.rnn_cell.MultiRNNCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/MultiRNNCell)。 - - -[TensorFlow Lite LSTM ops API](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/examples/lstm/g3doc/README.md) 提供了使用这些替代方法的教程。 - -有关的 Colab 教程，可以参阅 [TensorFlowLite_LSTM_Keras_Tutorial](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/examples/lstm/TensorFlowLite_LSTM_Keras_Tutorial.ipynb)。 - -注意：对于 [tf.nn.rnn_cell.GRUCell](https://www.tensorflow.org/api_docs/python/tf/nn/rnn_cell/GRUCell)，没有可替代的方法。 \ No newline at end of file diff --git a/site/zh-cn/lite/guide/android.md b/site/zh-cn/lite/guide/android.md deleted file mode 100644 index 5e817e62f38..00000000000 --- a/site/zh-cn/lite/guide/android.md +++ /dev/null @@ -1,74 +0,0 @@ -# Android 快速上手 - -要在Android上使用TensorFlow Lite，我们推荐您探索下面的例子。 - -Android -图像分类示例 - -有关源代码的说明，您还应该阅读 -[TensorFlow Lite Android 图像分类](https://www.tensorflow.org/lite/models/image_classification/android). - -该示例应用程序使用 -[图像分类](https://www.tensorflow.org/lite/models/image_classification/overview) -来连续地对设备的后置摄像头所看到的内容进行分类。 -该应用程序可以运行在真实设备或者模拟器上。 - -使用 TensorFlow Lite Java API 来执行推理。该演示应用程序实时地对图像帧进行分类，显示最可能的分类结果。它允许用户选择浮点或 -[量化](https://www.tensorflow.org/lite/performance/post_training_quantization) -模型，选择线程数，并决定运行在CPU，GPU上，亦或是通过 -[NNAPI](https://developer.android.com/ndk/guides/neuralnetworks)运行。 - -注意: 这些[示例](https://www.tensorflow.org/lite/examples)提供了其他的在多种用例中演示使用TensorFlow Lite的应用程序。 - -## 在Android Studio中构建 - -如果您要在Android Studio 构建例子，请遵循 -[README.md](https://github.com/tensorflow/examples/blob/master/lite/examples/image_classification/android/README.md)中的说明。 - -## 创建您自己的Android应用程序 - -如果您想快速编写您的Android代码, 我们推荐使用 -[Android 图像分类代码例子](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android) -作为起始点。 - -下面的部分包含了一些有关如何在Android上使用TensorFlow Lite的有用信息。 - -### 使用JCenter中的TensorFlow Lite AAR - -如果您要在您的Android应用程序中使用TensorFlow Lite，我们推荐使用 -[在JCenter中托管的TensorFlow Lite AAR](https://bintray.com/google/tensorflow/tensorflow-lite)。 - -您可以像下面这样在您的`build.gradle`依赖中指定它: - -```build -dependencies { - implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly' -} -``` - -这个AAR包含了 -[Android ABIs](https://developer.android.com/ndk/guides/abis)中的所有的二进制文件。您可以通过只包含您需要支持的ABIs来减少您应用程序的二进制文件大小。 - -我们推荐大部分的开发者删简 `x86`，`x86_64`，和`arm32` 的ABIs。您可以通过如下的Gradle配置实现，这个配置只包括了 `armeabi-v7a`和`arm64-v8a`，该配置能涵盖住大部分的现代Android设备。 - -```build -android { - defaultConfig { - ndk { - abiFilters 'armeabi-v7a', 'arm64-v8a' - } - } -} -``` - -想要了解更多有关 `abiFilters`的信息, 请查看Android Gradle文档中的 -[`NdkOptions`](https://google.github.io/android-gradle-dsl/current/com.android.build.gradle.internal.dsl.NdkOptions.html)。 - -### 在本地构建TensorFlow Lite - -在某些情况下，您可能希望使用一个本地构建的TensorFlow Lite. 比如说，您可能正在构建一个自定义的包含了 -[从TensorFlow中选择的操作](https://www.tensorflow.org/lite/guide/ops_select)的二进制文件。 - -在这种情况下，请参照 -[自定义 AAR 构建说明](https://www.tensorflow.org/lite/guide/ops_select#android_aar) -来创建你自己的AAR并将其包含在您的APP中. diff --git a/site/zh-cn/lite/guide/build_arm64.md b/site/zh-cn/lite/guide/build_arm64.md deleted file mode 100644 index 836facfecbb..00000000000 --- a/site/zh-cn/lite/guide/build_arm64.md +++ /dev/null @@ -1,61 +0,0 @@ -# 在ARM64板上构建Tensorflow Lite - -## 交叉编译 - -### 安装工具链 - -```bash -sudo apt-get update -sudo apt-get install crossbuild-essential-arm64 -``` - -> 如果你使用docker，可能不需要加上`sudo` - -### 构建 - -复制Tensorflow代码仓库。在代码仓库根目录下运行下面的脚本来下载依赖： - -> 你也可以尝试使用docker镜像`tensorflow/tensorflow:nightly-devel`， -> tensorflow代码仓库在`/tensorflow` - -```bash -./tensorflow/lite/tools/make/download_dependencies.sh -``` - -注意你只需要做一次这个操作 - -编译: - -```bash -./tensorflow/lite/tools/make/build_aarch64_lib.sh -``` - -这会编译出一个静态库在： -`tensorflow/lite/tools/make/gen/aarch64_armv8-a/lib/libtensorflow-lite.a`. - -## 原生编译 - -以下步骤在 HardKernel Odroid C2 和gcc 5.4.0版本上测试过. - -登录你的开发板，安装工具链 - -```bash -sudo apt-get install build-essential -``` - -首先，复制Tensorflow代码仓库。在代码仓库根目录下运行： - -```bash -./tensorflow/lite/tools/make/download_dependencies.sh -``` - -注意你只需要做一次这个操作 - -编译: - -```bash -./tensorflow/lite/tools/make/build_aarch64_lib.sh -``` - -这会编译出一个静态库在： -`tensorflow/lite/tools/make/gen/aarch64_armv8-a/lib/libtensorflow-lite.a`. diff --git a/site/zh-cn/lite/guide/build_ios.md b/site/zh-cn/lite/guide/build_ios.md deleted file mode 100644 index 62daf0c0b7b..00000000000 --- a/site/zh-cn/lite/guide/build_ios.md +++ /dev/null @@ -1,67 +0,0 @@ -# 在 iOS 上构建 TensorFlow Lite - -本文档描述了如何构建 TensorFlow Lite iOS 库。如果仅需使用，可以直接使用 TensorFlow Lite CocoaPod 版本。参阅 [TensorFlow Lite iOS Demo](ios.md) 获取示例。 - -## 构建 - -TensorFlow Lite 的通用 iOS 库需要在 MacOS 机器上，通过 Xcode 的命令行工具来构建。 -如果你还没有配置好环境，可以通过 `xcode-select` 来安装 Xcode 8(或更高版本) 和工具: - -```bash -xcode-select --install -``` - -如果这是第一次安装，你需要先运行一次 XCode 并同意它的许可。 - -(你也需要安装好 [Homebrew](http://brew.sh/)) - -下面安装 [automake](https://en.wikipedia.org/wiki/Automake)/[libtool](https://en.wikipedia.org/wiki/GNU_Libtool): - -```bash -brew install automake -brew install libtool -``` - -如果你遇到了 automake 和 libtool 已经安装但未正确链接的错误，首先输入以下命令: -```bash -sudo chown -R $(whoami) /usr/local/* -``` -然后使用下面的命令来使链接生效: -```bash -brew link automake -brew link libtool -``` - -接着你需用通过 shell 脚本来下载所需的依赖: -```bash -tensorflow/lite/tools/make/download_dependencies.sh -``` - -这会从网上获取库和数据的拷贝，并安装在`tensorflow/lite/downloads`目录 - -所有的依赖都已经创建完毕，你现在可以在 iOS 上为五个支持的体系架构构建库: - -```bash -tensorflow/lite/tools/make/build_ios_universal_lib.sh -``` - -它使用 `tensorflow/lite` 中的 makefile 来构建不同版本的库，然后调用 `lipo` 将它们捆绑到包含 armv7, armv7s, arm64, i386, 和 x86_64 架构的通用文件中。生成的库在: `tensorflow/lite/tools/make/gen/lib/libtensorflow-lite.a` - -如果你在运行 `build_ios_universal_lib.sh` 时，遇到了如 `no such file or directory: 'x86_64'` 的错误: -打开 Xcode > Preferences > Locations，确保在"Command Line Tools"下拉菜单中有一个选中值。 - -## 在应用中使用 - -你需要更新一些你的应用设置来链接 TensorFlow Lite。你可以在示例项目 -`tensorflow/lite/examples/ios/simple/simple.xcodeproj` 查看这些设置， -但下面提供了一个完整的纲要: - -- 你需要将库 `tensorflow/lite/gen/lib/libtensorflow-lite.a` 加入你的链接构建阶段，并且在 Search Paths 的 Library Search Paths 设置中添加 `tensorflow/lite/gen/lib` - -- _Header Search_ 路径需要包含: - - - tensorflow 的根目录, - - `tensorflow/lite/downloads` - - `tensorflow/lite/downloads/flatbuffers/include` - -- 设置 `C++ Language Dialect` 为 `GNU++11` (或 `GNU++14`), 同时设置 `C++ Standard Library` 为 `libc++` 来启用 C++11 支持 (或更高版本) diff --git a/site/zh-cn/lite/guide/build_rpi.md b/site/zh-cn/lite/guide/build_rpi.md deleted file mode 100644 index ed0f511b318..00000000000 --- a/site/zh-cn/lite/guide/build_rpi.md +++ /dev/null @@ -1,40 +0,0 @@ -# 在树莓派上构建 TensorFlow Lite -## 交叉编译 -### 安装工具链 -此功能已在64位的 ubuntu 16.04.3 和 Tensorflow devel docker image [tensorflow/tensorflow:nightly-devel](https://hub.docker.com/r/tensorflow/tensorflow/tags/) 上测试。 -要使用 TensorFlow Lite 交叉编译功能，应先安装工具链和相关的库。 -```bash -sudo apt-get update -sudo apt-get install crossbuild-essential-armhf -``` -如果您使用 Docker，您可能无法使用 `sudo`。 -### 构建 -克隆此Tensorflow库，在库的根目录下运行此脚本以下载所有依赖项： -> tensorflow库在 /tensorflow 下。如果您使用的是 docker 镜像 tensorflow/tensorflow:nightly-develimage，则会使用Tensorflow存储库，尝试使用以下命令即可。 -```bash -./tensorflow/lite/tools/make/download_dependencies.sh -``` -请注意，您只需要执行一次此操作。 -然后您便能够编译： -```bash -./tensorflow/lite/tools/make/build_rpi_lib.sh -``` -这将编译一个静态库,它位于： -tensorflow/lite/tools/make/gen/rpi_armv7l/lib/libtensorflow-lite.a. -## 本地编译 -这已经在Raspberry Pi 3b，Raspbian GNU / Linux 9.1（stretch），gcc版本6.3.0 20170516（Raspbian 6.3.0-18 + rpi1）上进行了测试。 -登录Raspberry Pi，安装工具链。 -```bash -sudo apt-get install build-essential -``` -首先，克隆TensorFlow库。在库的根目录运行： -```bash -./tensorflow/lite/tools/make/download_dependencies.sh -``` -请注意，您只需要执行一次此操作。 -然后您便能够编译： -```bash -./tensorflow/lite/tools/make/build_rpi_lib.sh -``` -这将编译一个静态库,它位于： -tensorflow/lite/tools/make/gen/lib/rpi_armv7/libtensorflow-lite.a 。 diff --git a/site/zh-cn/lite/guide/get_started.md b/site/zh-cn/lite/guide/get_started.md deleted file mode 100644 index d6cd289a73f..00000000000 --- a/site/zh-cn/lite/guide/get_started.md +++ /dev/null @@ -1,200 +0,0 @@ -# 开始使用 TensorFlow Lite - -TensorFlow Lite 提供了转换 TensorFlow 模型，并在移动端（mobile）、嵌入式（embeded）和物联网（IoT）设备上运行 TensorFlow 模型所需的所有工具。以下指南介绍了开发人员工作流程的每个步骤，并提供了进一步说明的链接。 - -## 1. 选择一个模型 - - - -TensorFlow Lite 允许您在多种设备上运行 TensorFlow 模型。TensorFlow 模型是一种数据结构，这种数据结构包含了在解决一个特定问题时，训练得到的机器学习网络的逻辑和知识。 - -有多种方式可以获得 TensorFlow 模型，从使用预训练模型（pre-trained models）到训练自己的模型。为了在 TensorFlow Lite 中使用模型，模型必须转换成一种特殊格式。这将在第二节[转换模型](#2_convert_the_model_format)中解释。 - -Note: 不是所有的 TensorFlow 模型都能在 TensorFlow Lite 中运行，因为解释器（interpreter）只支持部分（a limited subset）TensorFlow 运算符（operations）。参考第二节[转换模型](#2_convert_the_model_format)来了解兼容性。 - -### 使用预训练模型 - -TensorFlow Lite 团队提供了一系列预训练模型（pre-trained models），用于解决各种机器学习问题。这些模型已经转换为能与 TensorFlow Lite 一起使用，且可以在您的应用程序中使用的模型。 - -这些预训练模型包括： - -* [图像分类（Image classification）](../models/image_classification/overview.md) -* [物体检测（Object detection）](../models/object_detection/overview.md) -* [智能回复（Smart reply）](../models/smart_reply/overview.md) -* [姿态估计（Pose estimation）](../models/pose_estimation/overview.md) -* [语义分割（Segmentation）](../models/segmentation/overview.md) - -在[模型列表（Models）](../models)中查看预训练模型的完整列表。 - -#### 来自其他来源的模型 - -您还可以在许多其他地方得到预训练的 TensorFlow 模型，包括 [TensorFlow Hub](https://www.tensorflow.org/hub)。在大多数情况下，这些模型不会以 TensorFlow Lite 格式提供，您必须在使用前[转换（convert）](#2_convert_the_model_format)这些模型。 - -### 重新训练模型（迁移学习） - -迁移学习（transfer learning）允许您采用训练好的模型并重新（re-train）训练，以执行其他任务。例如，一个[图像分类](../models/image_classification/overview.md)模型可以重新训练以识别新的图像类别。与从头开始训练模型相比，重新训练花费的时间更少，所需的数据更少。 - -您可以使用迁移学习，根据您的应用程序定制预训练模型。在用 TensorFlow 识别花朵的 codelab 中，您可以学习如何进行迁移学习。 - -### 训练自定义模型 - -如果您设计并训练了您自己的 TensorFlow 模型，或者您训练了从其他来源得到的模型，在使用前，您需要将此模型转换成 TensorFlow Lite 的格式。 - -## 2. 转换模型 - - - -TensorFlow Lite 的设计旨在在各种设备上高效执行模型。这种高效部分源于在存储模型时，采用了一种特殊的格式。TensorFlow 模型在能被 TensorFlow Lite 使用前，必须转换成这种格式。 - -转换模型减小了模型文件大小，并引入了不影响准确性（accuracy）的优化措施（optimizations）。开发人员可以在进行一些取舍的情况下，选择进一步减小模型文件大小，并提高执行速度。您可以使用 TensorFlow Lite 转换器（converter）选择要执行的优化措施。 - -因为 TensorFlow Lite 支持部分 TensorFlow 运算符（operations），所以并非所有模型都能转换。参看[ Ops 兼容性](#Ops兼容性)获得更多信息。 - -### TensorFlow Lite 转换器 - -[TensorFlow Lite 转换器（converter）](../convert)是一个将训练好的 TensorFlow 模型转换成 TensorFlow Lite 格式的工具。它还能引入优化措施（optimizations），这将在第四节[优化您的模型](#4_optimize_your_model_optional)中介绍。 - -转换器以 Python API 的形式提供。下面的例子说明了将一个 TensorFlow `SavedModel` 转换成 TensorFlow Lite 格式的过程： - -```python -import tensorflow as tf - -converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) -tflite_model = converter.convert() -open("converted_model.tflite", "wb").write(tflite_model) -``` - -您可以用类似的方法[转换 TensorFlow 2.0 模型](../convert) - -虽然也能从[命令行](../convert/cmdline_examples)使用转换器，但是推荐用 Python API 进行转换。 - -### 选项 - -转换器可以从各种输入类型转换模型。 - -当转换[ TensorFlow 1.x 模型](../convert/python_api.md)时，这些输入类型有： - -* [SavedModel 文件夹](https://www.tensorflow.org/guide/saved_model) -* Frozen GraphDef (通过[ freeze_graph.py ](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py)生成的模型) -* [Keras](https://keras.io) HDF5 模型 -* 从 `tf.Session` 得到的模型 - -当转换[ TensorFlow 2.x 模型](../convert/python_api.md)时，这些输入类型有： - -* [SavedModel 文件夹](https://www.tensorflow.org/guide/saved_model) -* [`tf.keras` 模型](https://www.tensorflow.org/guide/keras/overview) -* [具体函数（Concrete functions）](https://tensorflow.org/guide/concrete_function) - -转换器可以配置为应用各种优化措施（optimizations），这些优化措施可以提高性能，减少文件大小。这将在第四节[优化您的模型](#4_optimize_your_model_optional)中介绍。 - -### Ops 兼容性 - -TensorFlow Lite 当前支持[一部分（limited subset）](ops_compatibility.md) TensorFlow 运算符（operations）。长期目标是将来能支持全部的 TensorFlow 运算符。 - -如果您期望转换的模型中含有不受支持的运算符，您可以使用[ TensorFlow Select](ops_select.md) 包含来自 TensorFlow 的运算符。这会使得部署到设备上的二进制文件更大。 - - -## 3. 使用模型进行推理 - - - -*推理（Inference）* 是通过模型（model）运行数据（data）以获得预测（predictions）的过程。这个过程需要模型（model）、解释器（interpreter）和输入数据（input data）。 - -### TensorFlow Lite 解释器 - -[TensorFlow Lite 解释器（interpreter）](inference.md)是一个库（library），它接收一个模型文件（model file），执行模型文件在输入数据（input data）上定义的运算符（operations），并提供对输出（output）的访问。 - -该解释器（interpreter）适用于多个平台，提供了一个简单的 API，用于从 Java、Swift、Objective-C、C++ 和 Python 运行 TensorFlow Lite 模型。 - -下面的代码显示了从 Java 调用解释器的方式: - -```java -try (Interpreter interpreter = new Interpreter(tensorflow_lite_model_file)) { - interpreter.run(input, output); -} -``` - -### GPU 加速和委托 - -一些设备为机器学习运算符提供硬件加速（hardware acceleration）。例如，大多数手机有 GPU，这些 GPU 可以比 CPU 执行更快的浮点矩阵运算（floating point matrix operations）。 - -速度提升（speed-up）能有显著（substantial）效果。例如，当使用 GPU 加速时，MobileNet v1 图像分类模型在 Pixel 3 手机上的运行速度提高了 5.5 倍。 - -TensorFlow Lite 解释器可以配置[委托（Delegates）](../performance/delegates.md)以在不同设备上使用硬件加速。[GPU 委托（GPU Delegates）](../performance/gpu.md)允许解释器在设备的 GPU 上运行适当的运算符。 - -下面的代码显示了从 Java 中使用 GPU 委托的方式: - -```java -GpuDelegate delegate = new GpuDelegate(); -Interpreter.Options options = (new Interpreter.Options()).addDelegate(delegate); -Interpreter interpreter = new Interpreter(tensorflow_lite_model_file, options); -try { - interpreter.run(input, output); -} -``` - -要添加对新硬件加速器的支持，您可以[定义您自己的委托](../performance/delegates.md#how_to_add_a_delegate)。 - -### Android 和 iOS - -TensorFlow Lite 解释器很容易在两个主要移动平台上使用。要入门，请浏览[ Android 快速入门](android.md)和[ iOS 快速入门](iOS.md)指南。对这两个平台，都有[示例应用程序](https://www.tensorflow.org/lite/examples)。 - -要获得所需的库（libraries），Android 开发人员应该使用[ TensorFlow Lite AAR](android.md#use_the_tensorflow_lite_aar_from_jcenter)。iOS 开发人员应该使用[ CocoaPods for Swift or Objective-C](ios.md#add_tensorflow_lite_to_your_swift_or_objective-c_project)。 - -### Linux - -嵌入式 Linux 是一个部署机器学习的重要平台。我们为[ Raspberry Pi ](build_rpi.md)和[基于 Arm64 的主板](build_arm64.md)，如 Odroid C2、Pine64 和 NanoPi，提供了构建说明。 - -### 微控制器 - -[TensorFlow Lite 微控制器（Microcontrollers）版](../microcontrollers/overview.md)是一个 TensorFlow Lite 的实验端口，该端口针对只有几千字节（kilobytes）内存（memory）的微控制器和其他设备。 - -### 运算符 - -如果您的模型需要 TensorFlow Lite 中尚未实现的 TensorFlow 运算符（operations），您可以使用[ TensorFlow Select ](ops_select.md)在模型中使用它们。您需要构建一个包含 TensorFlow 运算符的自定义版本解释器。 - -您可以用[自定义运算符（Custom operators）](ops_custom.md)编写您自己的运算符（operations），或将新运算符移植（port）到 TensorFlow Lite 中。 - -[运算符版本（Operator versions）](ops_version.md)让您能为已有的运算符添加新的功能和参数。 - -## 4. 优化您的模型 - - - -TensorFlow Lite 提供了优化模型大小（size）和性能（performance）的工具，通常对准确性（accuracy）影响甚微。优化模型可能需要稍微复杂的训练（training），转换（conversion）或集成（integration）。 - -机器学习优化是一个不断发展的领域，TensorFlow Lite 的[模型优化工具包（Model Optimization Toolkit）](#模型优化工具包)随着新技术的发展而不断发展。 - -### 性能 - -模型优化的目标是在给定设备上，实现性能（performance）、模型大小（model size）和准确性（accuracy）的理想平衡。 -[性能最佳实践（Performance best practices）](../performance/best_practices.md)可以帮助指导您完成这个过程。 - -### 量化 - -通过降低模型中数值（values）和运算符（operations）的精度（precision），量化（quantization）可以减小模型的大小和推理所需的时间。对很多模型，只有极小的准确性（accuracy）损失。 - -TensorFlow Lite 转换器让量化 TensorFlow 模型变得简单。下面的 Python 代码量化了一个 `SavedModel` 并将其保存在硬盘中： - -```python -import tensorflow as tf - -converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) -converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE] -tflite_quant_model = converter.convert() -open("converted_model.tflite", "wb").write(tflite_quantized_model) -``` - -要了解有关量化的更多信息，请参阅[训练后量化（Post-training quantization）](../performance/post_training_quantization.md)。 - -### 模型优化工具包 - -[模型优化工具包（Model Optimization Toolkit）](../performance/model_optimization.md)是一套工具和技术，旨在使开发人员可以轻松优化它们的模型。虽然其中的许多技术可以应用于所有 TensorFlow 模型，并非特定于 TensorFlow Lite，但在资源有限的设备上进行推理（inference）时，它们特别有价值。 - -## 下一步 - -既然您已经熟悉了 TensorFlow Lite，请探索以下一些资源： - -* 如果您是移动开发人员，请访问[ Android 快速入门](android.md)或[ iOS 快速入门](ios.md)。 -* 探索我们的[预训练模型](../models)。 -* 尝试我们的[示例应用程序](https://www.tensorflow.org/lite/examples)。 diff --git a/site/zh-cn/lite/guide/ops_select.md b/site/zh-cn/lite/guide/ops_select.md deleted file mode 100644 index c07fd8d5248..00000000000 --- a/site/zh-cn/lite/guide/ops_select.md +++ /dev/null @@ -1,179 +0,0 @@ -# 从 TensorFlow 中选择运算符 - -注意：该功能是实验性的。 - -TensorFlow Lite 已经内置了很多运算符，并且还在不断扩展，但是仍然还有一部分 TensorFlow 运算符没有被 TensorFlow Lite 原生支持。这些不被支持的运算符会给 TensorFlow Lite 的模型转换带来一些阻力。为了减少模型转换的阻力，TensorFlow Lite 开发团队最近一直致力于一个实验性功能的开发。 - -这篇文档简单介绍了怎样在 TensorFlow Lite 使用 TensorFlow 运算符。*注意，这只是一个实验性的功能，并且还在开发中。* 在使用该功能的时候，请记住这些[已知的局限性](#已知的局限性)，并且请将使用中遇到的问题反馈至 tflite@tensorflow.org。 - -TensorFlow Lite 会继续为移动设备和嵌入式设备优化[内置的运算符](ops_compatibility.md)。但是现在，当 TensorFlow Lite 内置的运算符不够的时候，TensorFlow Lite 模型可以使用部分 TensorFlow 的运算符。 - -TensorFlow Lite 解释器在处理转换后的包含 TensorFlow 运算符的模型的时候，会比处理只包含 TensorFlow Lite 内置运算符的模型占用更多的空间。并且，TensorFlow Lite 模型中包含的任何 TensorFlow 运算符，性能都不会被优化。 - -这篇文档简单介绍了怎样针对不同的平台[转换](#转换模型)和[运行](#运行模型)包含 TensorFlow 运算符的 TensorFlow Lite 模型。并且讨论了一些[已知的局限性](#已知的局限性)、为此功能制定的[未来的计划](#未来的计划)以及基本的[性能和空间指标](#性能和空间指标)。 - -## 转换模型 - -为了能够转换包含 TensorFlow 运算符的 TensorFlow Lite 模型，可使用位于 [TensorFlow Lite 转换器](../convert/) 中的 `target_spec.supported_ops` 参数。`target_spec.supported_ops` 的可选值如下： - -* `TFLITE_BUILTINS` - 使用 TensorFlow Lite 内置运算符转换模型。 -* `SELECT_TF_OPS` - 使用 TensorFlow 运算符转换模型。已经支持的 TensorFlow 运算符的完整列表可以在白名单 - `lite/delegates/flex/whitelisted_flex_ops.cc` 中查看。 - -注意：`target_spec.supported_ops` 是之前 Python API 中的 `target_ops`。 - -我们优先推荐使用 `TFLITE_BUILTINS` 转换模型，然后是同时使用 `TFLITE_BUILTINS,SELECT_TF_OPS` ，最后是只使用 `SELECT_TF_OPS`。同时使用两个选项（也就是 `TFLITE_BUILTINS,SELECT_TF_OPS`）会用 TensorFlow Lite 内置的运算符去转换支持的运算符。有些 TensorFlow 运算符 TensorFlow Lite 只支持部分用法，这时可以使用 `SELECT_TF_OPS` 选项来避免这种局限性。 - -下面的示例展示了通过 Python API 中的 [`TFLiteConverter`](./convert/python_api.md) 来使用该功能。 - -``` -import tensorflow as tf - -converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) -converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, - tf.lite.OpsSet.SELECT_TF_OPS] -tflite_model = converter.convert() -open("converted_model.tflite", "wb").write(tflite_model) -``` - -下面的示例展示了在命令行工具 [`tflite_convert`](../convert/cmdline_examples.md) 中通过 `target_ops` 标记来使用该功能。 - -``` -tflite_convert \ - --output_file=/tmp/foo.tflite \ - --graph_def_file=/tmp/foo.pb \ - --input_arrays=input \ - --output_arrays=MobilenetV1/Predictions/Reshape_1 \ - --target_ops=TFLITE_BUILTINS,SELECT_TF_OPS -``` - -如果直接使用 `bazel` 编译和运行 `tflite_convert`，请传入参数 `--define=with_select_tf_ops=true`。 - -``` -bazel run --define=with_select_tf_ops=true tflite_convert -- \ - --output_file=/tmp/foo.tflite \ - --graph_def_file=/tmp/foo.pb \ - --input_arrays=input \ - --output_arrays=MobilenetV1/Predictions/Reshape_1 \ - --target_ops=TFLITE_BUILTINS,SELECT_TF_OPS -``` - -## 运行模型 - -如果 TensorFlow Lite 模型在转换的时候支持 TensorFlow select 运算符，那么在使用的时候 Tensorflow Lite 运行时必须包含 TensorFlow 运算符的库。 - -### Android AAR - -为了便于使用，新增了一个支持 TensorFlow select 运算符的Android AAR。如果已经有了可用的 TensorFlow Lite -编译环境，可以按照下面的方式编译支持使用 TensorFlow select 运算符的 Android AAR： - -```sh -bazel build --cxxopt='--std=c++11' -c opt \ - --config=android_arm --config=monolithic \ - //tensorflow/lite/java:tensorflow-lite-with-select-tf-ops -``` - -上面的命令会在 `bazel-genfiles/tensorflow/lite/java/` 目录下生成一个 AAR 文件。你可以直接将这个 AAR 文件导入到项目中，也可以将其发布到本地的 Maven 仓库： - -```sh -mvn install:install-file \ - -Dfile=bazel-genfiles/tensorflow/lite/java/tensorflow-lite-with-select-tf-ops.aar \ - -DgroupId=org.tensorflow \ - -DartifactId=tensorflow-lite-with-select-tf-ops -Dversion=0.1.100 -Dpackaging=aar -``` - -最后，在应用的 `build.gradle` 文件中需要保证有 `mavenLocal()` 依赖，并且需要用支持 TensorFlow select 运算符的 TensorFlow Lite 依赖去替换标准的 TensorFlow Lite 依赖： - -``` -allprojects { - repositories { - jcenter() - mavenLocal() - } -} - -dependencies { - implementation 'org.tensorflow:tensorflow-lite-with-select-tf-ops:0.1.100' -} -``` - -### iOS - -如果安装了 XCode 命令行工具，可以用下面的命令编译支持 TensorFlow select 运算符的 TensorFlow Lite： - -```sh -tensorflow/contrib/makefile/build_all_ios_with_tflite.sh -``` - -这条命令会在 `tensorflow/contrib/makefile/gen/lib/` 目录下生成所需要的静态链接库。 - -TensorFlow Lite 的相机示例应用可以用来进行测试。一个新的支持 TensorFlow select 运算符的 TensorFlow Lite XCode 项目已经添加在 `tensorflow/lite/examples/ios/camera/tflite_camera_example_with_select_tf_ops.xcodeproj` 中。 - -如果想要在自己的项目中使用这个功能，你可以克隆示例项目，也可以按照下面的方式对项目进行设置： - -* 在 Build Phases -> Link Binary With Libraries 中，添加 `tensorflow/contrib/makefile/gen/lib/` 目录中的静态库： - * `libtensorflow-lite.a` - * `libprotobuf.a` - * `nsync.a` -* 在 Build Settings -> Header Search Paths 中，添加下面的路径： - * `tensorflow/lite/` - * `tensorflow/contrib/makefile/downloads/flatbuffer/include` - * `tensorflow/contrib/makefile/downloads/eigen` -* 在 Build Settings -> Other Linker Flags 中，添加 `-force_load - tensorflow/contrib/makefile/gen/lib/libtensorflow-lite.a`。 - -未来还会发布支持 TensorFlow select 运算符的 CocoaPod 。 - -### C++ - -如果使用 bazel 编译 TensorFlow Lite 库，可以按照下面的方式添加和支持额外的 TensorFlow 运算符的库。 - -* 如果需要单体编译，可以添加 `--config=monolithic` 编译标记。 -* 从下面的方案中选择一个： - * 在用 `bazel build` 命令编译 TensorFlow Lite 时添加 `--define=with_select_tf_ops=true` 编译标记。 - * 在编译依赖中添加 TensorFlow 运算符库依赖 `tensorflow/lite/delegates/flex:delegate`。 - -注意，只要委托链接到了客户端库，在运行时创建解释器的时候就会自动安装所需的 `TfLiteDelegate`，而不需要像其他委托类型去显式安装委托实例。 - -### Python pip Package - -对 Python 的支持还在开发当中。 - -## 性能和空间指标 - -### 性能 - -如果 TensorFlow Lite 模型是同时混合使用内置运算符和 TensorFlow select 运算符进行转换的，那么模型依然可以使用针对 TensorFlow Lite 的优化以及内置的优化内核。 - -下表列出了在 Pixel 2 上 MobileNet 的平均推断时间。表中的时间是 100 次运行的平均时间。在对 Android 平台编译的时候添加了 `--config=android_arm64 -c opt` 标记。 - -编译 | 推断时间 (milliseconds) ------------------------------------- | ------------------- -Only built-in ops (`TFLITE_BUILTIN`) | 260.7 -Using only TF ops (`SELECT_TF_OPS`) | 264.5 - -### 二进制文件大小 - -下表列出了不同编译方式生成的 TensorFlow Lite 二进制文件的大小。在对 Android 平台编译的时候添加了 `--config=android_arm -c opt` 标记。 - -编译 | C++ 二进制文件大小 | Android APK 大小 ---------------------- | --------------- | ---------------- -Only built-in ops | 796 KB | 561 KB -Built-in ops + TF ops | 23.0 MB | 8.0 MB - -## 已知的局限性 - -下面列出了一些已知的局限性： - -* 目前还不支持控制流运算符。 -* 目前还不支持 TensorFlow 运算符的 [`post_training_quantization`](https://www.tensorflow.org/performance/post_training_quantization) 标记，所以不会对任何 TensorFlow 运算符进行权重量化。如果模型中既包含 TensorFlow Lite 运算符又包含 TensorFlow 运算符，那么 TensorFlow Lite 内置的运算符的权重是可以被量化的。 -* 目前还不支持像 HashTableV2 这种需要显式用资源进行初始化的运算符。 -* 某些 TensorFlow 操作可能不支持 TensorFlow 库中整套常规可用输入/输出操作。 - -## 未来的计划 - -下面列出了正在开发中的针对该功能的一些改进： - -* *选择性注册* - 有一项正在完成的工作是，让生成只包含特定模型集合所需的 Tensorflow 运算符的 TensorFlow Lite 二进制文件变得更简单。 -* *提升可用性* - 模型转换的过程将被简化，只需要一次性完成转换。 并且还会提供预编译的 Android AAR 和 iOS CocoaPod 二进制文件。 -* *提升性能* - 有一项正在完成的工作是，让使用 TensorFlow 运算符的 TensorFlow Lite 具有与 TensorFlow Mobile 同等的性能。 diff --git a/site/zh-cn/lite/guide/ops_version.md b/site/zh-cn/lite/guide/ops_version.md deleted file mode 100644 index ef11479da75..00000000000 --- a/site/zh-cn/lite/guide/ops_version.md +++ /dev/null @@ -1,172 +0,0 @@ -TensorFlow Lite 操作(operator)的版本 - -本文档描述了TensorFlow Lite的操作(operator)版本架构。 操作(operator)的版本使开发人员能够将新功能和参数添加到现有操作中。 此外，它保证以下内容： - -* 向后兼容性：新版本的 TensorFlow Lite 实现方式可以处理旧的模型文件。 -* 向前兼容性：只要没有使用新功能，旧版本的 TensorFlow Lite 实现方式可以处理由新版 TOCO 生成的新版本的模型文件。 -* 前向兼容性检测：如果旧的 TensorFlow Lite 实现读取包含不支持的新版本的模型，则应报告错误。 - -##示例：将膨胀(Dilation)添加到卷积操作中 -本文档的其余部分通过展示如何在卷积操作中添加膨胀系数来解释 TFLite 中操作(operator)的版本。 - -了解本文档内容并不需要了解卷积核膨胀的知识。需要注意的是： - -* 将添加2个新的整数参数：'dilation_width_factor' 和 'dilation_height_factor'。 -* 不支持膨胀的旧卷积核相当于将扩张因子膨胀系数设置为1。 - -### 更改 FlatBuffer 架构(Schema) - -要将新参数添加到操作(operator)中，请更改`lite/schema/schema.fbs`中的选项表 。 - -例如，卷积的选项表如下所示： - -``` -table Conv2DOptions { - padding:Padding; - stride_w:int; - stride_h:int; - fused_activation_function:ActivationFunctionType; -} -``` - -在添加新参数时： - -* 添加注释，指明哪个版本支持哪些参数。 -* 当新的实现获取新添加的参数的默认值时，它应该与旧实现完全相同。 - -添加新参数后，参数表如下所示： - -``` -table Conv2DOptions { - // 版本1支持的参数： - padding:Padding; - stride_w:int; - stride_h:int; - fused_activation_function:ActivationFunctionType; - - // 版本2支持的参数： - dilation_width_factor:int = 1; - dilation_height_factor:int = 1; -} -``` - -### 更改C中的结构体和内核实现 - -在TensorFlow Lite中，内核实现与FlatBuffer定义是分离发。 内核从`lite/builtin_op_data.h`中定义的C的结构体中读取参数。 - -原始卷积参数如下： - -``` -typedef struct { - TfLitePadding padding; - int stride_width; - int stride_height; - TfLiteFusedActivation activation; -} TfLiteConvParams; -``` - -与FlatBuffer架构(Schema)一样，通过添加注释，指明从哪个版本开始支持哪些参数。结果如下： - -``` -typedef struct { - // 版本1支持的参数： - TfLitePadding padding; - int stride_width; - int stride_height; - TfLiteFusedActivation activation; - - // 版本2支持的参数： - int dilation_width_factor; - int dilation_height_factor; -} TfLiteConvParams; -``` - -另外，请更改内核实现从C结构体中读取新添加的参数。 细节在此不再赘述。 - -### 更改 FlatBuffer 代码以获取新参数 - -负责读取 FlatBuffer 并生成 C 结构体的逻辑是由 `lite/model.cc` 实现的。 - -更新该文件以处理新参数，如下所示： - -``` -case BuiltinOperator_CONV_2D: { - TfLiteConvParams* params = MallocPOD(); - if (auto* conv_params = op->builtin_options_as_Conv2DOptions()) { - params->padding = parse_padding(conv_params->padding()); - params->stride_width = conv_params->stride_w(); - params->stride_height = conv_params->stride_h(); - params->activation = - parse_activation(conv_params->fused_activation_function()); - params->dilation_width_factor = conv_params->dilation_width_factor(); - params->dilation_height_factor = conv_params->dilation_height_factor(); - } - *builtin_data = reinterpret_cast(params); - break; -} -``` - -这里不需要检查操作版本。 当新实现读取缺少扩张因子的旧模型文件时，它将使用1作为默认值，并且新内核将与旧内核一致地工作。 - -### 更改内核注册 -MutableOpResolver（在`lite/op_resolver.h`中定义）提供了一些注册操作(operator)内核的函数。默认情况下，最小和最大版本都为1： -``` -void AddBuiltin(tflite::BuiltinOperator op, TfLiteRegistration* registration, - int min_version = 1, int max_version = 1); -void AddCustom(const char* name, TfLiteRegistration* registration, - int min_version = 1, int max_version = 1); -``` - -内置的操作在 `lite/kernels/register.cc` 中注册。 在这个例子中，我们实现了一个新的操作内核，它可以处理 `Conv2D` 的版本1和版本2，所以我们需要将下面这行： - -``` -AddBuiltin(BuiltinOperator_CONV_2D, Register_CONV_2D()); -``` - -修改为： - -``` -AddBuiltin(BuiltinOperator_CONV_2D, Register_CONV_2D(), 1, 2); -``` - -### 改变 TOCO TFLite 的导出 - -最后一步是让 TOCO 填充(populate)执行操作(operator)所需的最低版本。在这个例子中，它意味着： - -* 当膨胀系数均为1时，填充 版本=1。 -* 除此之外，填充 版本=2。 - -为此，您需要在`lite/toco/tflite/operator.cc`中重写定义操作(operator)的类(class)中的`GetVersion`函数。 - -对于只有一个版本的操作，它的 `GetVersion` 函数被定义为： -``` -int GetVersion(const Operator& op) const override { return 1; } -``` - -当支持多个版本时，请检查参数并确定op的版本，如以下示例所示： - -``` -int GetVersion(const Operator& op) const override { - const auto& conv_op = static_cast(op); - if (conv_op.dilation_width_factor != 1 || - conv_op.dilation_height_factor != 1) { - return 2; - } - return 1; -} -``` - -### 委托实现 - -TensorFlow Lite 提供了一个委托 API，可以将操作委派给硬件后端。在 Delegate 的 Prepare 函数中，检查该版本是否支持委派代码中的每个节点。 -``` -const int kMinVersion = 1; -TfLiteNode* node; -TfLiteRegistration; -context->GetNodeAndRegistration(context, node_index, &node, ®istration); - -if (registration->version > kMinVersion) { - // 如果不支持该版本，则拒绝该节点。 -} -``` -即使委派仅支持版本1的操作，这也是必需的，这使委派可以在获得更高版本操作时检测到不兼容性。 diff --git a/site/zh-cn/lite/microcontrollers/build_convert.md b/site/zh-cn/lite/microcontrollers/build_convert.md deleted file mode 100644 index 5dee9cf8743..00000000000 --- a/site/zh-cn/lite/microcontrollers/build_convert.md +++ /dev/null @@ -1,73 +0,0 @@ -# 建立与转换模型 - -微控制器具有有限的 RAM 和存储空间，这限制了机器学习模型的规模。此外，面向微控制器的 TensorFlow Lite 目前只支持有限的一部分运算，因此并非所有的模型结构都是可行的。 - -本文档解释了转换一个 TensorFlow 模型以使其可在微控制器上运行的过程。本文档也概述了可支持的运算，并对于设计与训练一个模型以使其符合内存限制给出了一些指导。 - -一个端到端的、可运行的建立与转换模型的示例，见于如下的 Jupyter notebook 中： -create_sine_model.ipynb - -## 模型转换 - -为了转换一个已训练好的 TensorFlow 模型以使其可在微控制器上运行，你应该使用 [TensorFlow Lite 转换器 Python API](https://tensorflow.google.cn/lite/convert/python_api) 。它能够将模型转换成 [`FlatBuffer`](https://google.github.io/flatbuffers/) 格式，减小模型规模，并修改模型以使用 TensorFlow Lite 支持的运算。 - -### 量化 -为了获得尽可能小的模型规模，你应该考虑使用[训练后量化](https://tensorflow.google.cn/lite/performance/post_training_quantization)。它会降低你模型中数字的精度，从而减小模型规模。不过，这种操作可能会导致模型准确性的下降，对于小规模模型来说尤为如此。在量化前后分析你模型的准确性以确保这种损失在可接受范围内是非常重要的。 - -以下的 Python 代码片段展示了如何使用预训练量化进行模型转换： - -```python -import tensorflow as tf -converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) -converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE] -tflite_quant_model = converter.convert() -open("converted_model.tflite", "wb").write(tflite_quant_model) -``` - -### 转换为一个 C 数组 -许多微控制器平台没有本地文件系统的支持。从程序中使用一个模型最简单的方式是将其以一个 C 数组的形式包含并编译进你的程序。 - -以下的 unix 命令会生成一个以 `char` 数组形式包含 TensorFlow Lite 模型的 C 源文件： - -```bash -xxd -i converted_model.tflite > model_data.cc -``` - -其输出类似如下： - -```c -unsigned char converted_model_tflite[] = { - 0x18, 0x00, 0x00, 0x00, 0x54, 0x46, 0x4c, 0x33, 0x00, 0x00, 0x0e, 0x00, - // -}; -unsigned int converted_model_tflite_len = 18200; -``` - -一旦你已经生成了此文件，你可以将它包含入你的程序。在嵌入式平台上，将数组声明改变为 `const` 类型以获得更好的内存效率是重要的。 - -一个如何在你的程序中包含及使用模型的例子，请见微型语音示例中的 -[`tiny_conv_micro_features_model_data.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/micro_speech/micro_features/tiny_conv_micro_features_model_data.h) -。 - -## 模型结构与训练 - -在设计一个面向微控制器的模型时，考虑模型的规模、工作负载，以及用到的运算是非常重要的。 - -### 模型规模 - -一个模型必须在二进制和运行时方面都足够小，以使其可以和你程序的其他部分一起符合你目标设备的内存限制。 - -为了创建一个更小的模型，你可以在你的结构里使用更少和更小的层。然而，小规模的模型更易面临欠拟合问题。这意味着对于许多问题，尝试并使用符合内存限制的尽可能大规模的模型是有意义的。但是，使用更大规模的模型也会导致处理器工作负载的增加。 - -注：在一个 Cortex M3 上，面向微控制器的 TensorFlow Lite 的核心运行时占 16 KB。 - -### 工作负载 - -工作负载受到模型规模与复杂度的影响。大规模、复杂的模型可能会导致更高的占空比，即导致你所用设备处理器的工作时间增长、空闲时间缩短。视你的应用，这种情况所带来的电力消耗与热量输出的增加可能会成为一个问题。 - -### 运算支持 -面向微控制器的 TensorFlow Lite 目前仅支持有限的部分 TensorFlow 运算，这影响了可以运行的模型结构。我们正致力于在参考实现和针对特定结构的优化方面扩展运算支持。 - -已支持的运算可以在文件 -[`all_ops_resolver.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/kernels/all_ops_resolver.cc) -中看到。 diff --git a/site/zh-cn/lite/microcontrollers/get_started.md b/site/zh-cn/lite/microcontrollers/get_started.md deleted file mode 100644 index d1c974b5fa2..00000000000 --- a/site/zh-cn/lite/microcontrollers/get_started.md +++ /dev/null @@ -1,274 +0,0 @@ -# 微控制器入门 - -本文档将帮助您开始使用用于微控制器的 Tensorflow Lite。 - -首先请阅读并运行我们的[示例](#示例) - -注意：如果您需要一个入门设备，我们建议使用 -[由 Tensorflow 提供技术支持的 SparkFun Edge](https://www.sparkfun.com/products/15170)。 -它是与 Tensorflow Lite 团队合作设计的，为在微控制器上进行深度学习实验提供了灵活的平台。 - -有关运行推断所需代码的介绍，请参阅下文的*运行推断*部分 - -## 示例 - -下面几个示例演示了如何使用 Tensorflow Lite 构建嵌入式机器学习应用程序： - -### Hello World 示例 - -本示例旨在演示将 Tensorflow Lite 用于微控制器的绝对基础知识。它包括了训练模型、将模型转换以供 Tensorflow Lite -使用以及在微控制器上进运行推断的完整端到端工作流程。 - -在这个示例中，一个模型被训练用来模拟正弦函数。部署到微控制器上时，其预测可用来闪烁 LED 或者控制动画。 - -Hello -World 示例 - -示例代码包含一个演示如何训练和转换模型的 Jupyter notebook： - -create_sine_model.ipynb - -指南[“构建与转换模型”](build_convert.md)中也介绍了构建和转换模型的流程。 - -要了解推断是如何执行的，请查看 -[hello_world_test.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/hello_world/hello_world_test.cc)。 - -该示例在以下平台上进行了测试： - -- [由 Tensorflow 提供技术支持的 SparkFun Edge(Apollo3 Blue)](https://www.sparkfun.com/products/15170) -- [Arduino MKRZERO](https://store.arduino.cc/usa/arduino-mkrzero) -- [STM32F746G 探索板（Discovery Board）](https://www.st.com/en/evaluation-tools/32f746gdiscovery.html) -- Mac OS X - -### 微语音示例 - -此示例使用一个简单的 -[音频识别模型](https://www.tensorflow.org/tutorials/sequences/audio_recognition) -来识别语音中的关键字。示例代码从设备的麦克风中捕获音频。模型通过对该音频进行实时分类来确定是否说过“是”或“否一词。 - -微语音示例 - -[“运行推断”](#运行推断) 部分将纵览微语音示例的代码并解释其工作原理。 - -该示例在以下平台上进行了测试： - -- [由 Tensorflow 提供技术支持的 SparkFun Edge(Apollo3 Blue)](https://www.sparkfun.com/products/15170) -- [STM32F746G 探索板（Discovery Board）](https://www.st.com/en/evaluation-tools/32f746gdiscovery.html) -- Mac OS X - -注意：若要开始使用 SparkFun Edge -板，我们建议遵循[“在使用 SparkFun Tensorflow 的微控制器上进行机器学习”](https://codelabs.developers.google.com/codelabs/sparkfun-tensorflow)中所描述的流程,这是一个向您介绍开发工作流程的代码实验室（codelab）。 - -### 微视觉示例 - -本示例展示了如何使用 Tensorflow Lite 运行一个 25 -万字节的神经网络来识别由摄像机拍摄的图像中的人。该示例被设计成可以在具有少量内存的系统上运行，如微控制器和 DSP。 - -微视觉示例 - -该示例在以下平台上进行了测试： - -- [由 Tensorflow 提供技术支持的 SparkFun Edge(Apollo3 Blue)](https://www.sparkfun.com/products/15170) -- [STM32F746G 探索板（Discovery Board）](https://www.st.com/en/evaluation-tools/32f746gdiscovery.html) -- Mac OS X - -## 运行推断 - -以下部分将介绍[微语音](#微语音示例)示例中的 -[main.cc](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/micro_speech/main.cc) -文件并解释了它如何使用用于微控制器的 Tensorflow Lite 来运行推断。 - -### 包含项 - -要使用库，必须包含以下头文件： - -```C++ -#include "tensorflow/lite/micro/kernels/all_ops_resolver.h" -#include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/micro_interpreter.h" -#include "tensorflow/lite/schema/schema_generated.h" -#include "tensorflow/lite/version.h" -``` - -- [`all_ops_resolver.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/kernels/all_ops_resolver.h) - 提供给解释器（interpreter）用于运行模型的操作。 -- [`micro_error_reporter.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/micro_error_reporter.h) - 输出调试信息。 -- [`micro_interpreter.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/micro_interpreter.h) - 包含处理和运行模型的代码。 -- [`schema_generated.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/schema/schema_generated.h) - 包含 TensorFlow Lite [`FlatBuffer`](https://google.github.io/flatbuffers/) - 模型文件格式的模式。 -- [`version.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/version.h) - 提供 Tensorflow Lite 架构的版本信息。 - -示例还包括其他一些文件。以下这些是最重要的： - -```C++ -#include "tensorflow/lite/micro/examples/micro_speech/feature_provider.h" -#include "tensorflow/lite/micro/examples/micro_speech/micro_features/micro_model_settings.h" -#include "tensorflow/lite/micro/examples/micro_speech/micro_features/tiny_conv_micro_features_model_data.h" -``` - -- [`feature_provider.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/micro_speech/micro_features/feature_provider.h) - 包含从音频流中提取要输入到模型中的特征的代码。 -- [`tiny_conv_micro_features_model_data.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/micro_speech/micro_features/tiny_conv_micro_features_model_data.h) - 包含存储为 `char` 数组的模型。阅读 [“构建与转换模型”](build_convert.md) 来了解如何将 Tensorflow Lite - 模型转换为该格式。 -- [`micro_model_settings.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/micro_speech/micro_features/micro_model_settings.h) - 定义与模型相关的各种常量。 - -### 设置日志记录 - -要设置日志记录，需要使用一个指向 `tflite::MicroErrorReporter` 实例的指针来创建一个 `tflite::ErrorReporter` -指针： - -```C++ -tflite::MicroErrorReporter micro_error_reporter; -tflite::ErrorReporter* error_reporter = µ_error_reporter; -``` - -该变量被传递到解释器（interpreter）中，解释器允许它写日志。由于微控制器通常具有多种日志记录机制，`tflite::MicroErrorReporter` -的实现是为您的特定设备所定制的。 - -### 加载模型 - -在以下代码中，模型是从一个 `char` 数组中实例化的，`g_tiny_conv_micro_features_model_data` -（要了解其是如何构建的，请参见[“构建与转换模型”](build_convert.md)）。 随后我们检查模型来确保其架构版本与我们使用的版本所兼容： - -```C++ -const tflite::Model* model = - ::tflite::GetModel(g_tiny_conv_micro_features_model_data); -if (model->version() != TFLITE_SCHEMA_VERSION) { - error_reporter->Report( - "Model provided is schema version %d not equal " - "to supported version %d.\n", - model->version(), TFLITE_SCHEMA_VERSION); - return 1; -} -``` - -### 实例化操作解析器 - -解释器（interpreter）需要一个 -[`AllOpsResolver`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/kernels/all_ops_resolver.h) -实例来访问 Tensorflow 操作。可以扩展此类以向您的项目添加自定义操作： - -```C++ -tflite::ops::micro::AllOpsResolver resolver; -``` - -### 分配内存 - -我们需要预先为输入、输出以及中间数组分配一定的内存。该预分配的内存是一个大小为 `tensor_arena_size` 的 `uint8_t` 数组，它被传递给 -`tflite::SimpleTensorAllocator` 实例： - -```C++ -const int tensor_arena_size = 10 * 1024; -uint8_t tensor_arena[tensor_arena_size]; -tflite::SimpleTensorAllocator tensor_allocator(tensor_arena, - tensor_arena_size); -``` - -注意：所需内存大小取决于您使用的模型，可能需要通过实验来确定。 - -### 实例化解释器（Interpreter） - -我们创建一个 `tflite::MicroInterpreter` 实例，传递给之前创建的变量： - -```C++ -tflite::MicroInterpreter interpreter(model, resolver, &tensor_allocator, - error_reporter); -``` - -### 验证输入维度 - -`MicroInterpreter` 实例可以通过调用 `.input(0)` 为我们提供一个指向模型输入张量的指针，其中 `0` -代表第一个（也是唯一一个）输入张量。我们检查这个张量以确认它的维度与类型是我们所期望的： - -```C++ -TfLiteTensor* model_input = interpreter.input(0); -if ((model_input->dims->size != 4) || (model_input->dims->data[0] != 1) || - (model_input->dims->data[1] != kFeatureSliceCount) || - (model_input->dims->data[2] != kFeatureSliceSize) || - (model_input->type != kTfLiteUInt8)) { - error_reporter->Report("Bad input tensor parameters in model"); - return 1; -} -``` - -在这个代码段中，变量 `kFeatureSliceCount` 和 `kFeatureSliceSize` 与输入的属性相关，它们定义在 -[`micro_model_settings.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/micro_speech/micro_features/micro_model_settings.h) -中。枚举值 `kTfLiteUInt8` 是对 Tensorflow Lite 某一数据类型的引用，它定义在 -[`c_api_internal.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/c_api_internal.h) -中。 - -### 生成特征 - -我们输入到模型中的数据必须由微控制器的音频输入生成。[`feature_provider.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/micro_speech/micro_features/feature_provider.h) -中定义的 `FeatureProvider` 类捕获音频并将其转换为一组将被传入模型的特征集合。当该类被实例化时，我们用之前获取的 `TfLiteTensor` -来传入一个指向输入数组的指针。`FeatureProvider` 使用它来填充将传递给模型的输入数据： - -```C++ - FeatureProvider feature_provider(kFeatureElementCount, - model_input->data.uint8); -``` - -以下代码使 `FeatureProvider` 从最近一秒的音频生成一组特征并填充进输入张量： - -```C++ -TfLiteStatus feature_status = feature_provider.PopulateFeatureData( - error_reporter, previous_time, current_time, &how_many_new_slices); -``` - -在此例子中，特征生成和推断是在一个循环中发生的，因此设备能够不断地捕捉和处理新的音频。 - -当在编写自己的程序时，您可能会以其它的方式生成特征，但您总需要在运行模型之前就用数据填充输入张量。 - -### 运行模型 - -要运行模型，我们可以在 `tflite::MicroInterpreter` 实例上调用 `Invoke()`： - -```C++ -TfLiteStatus invoke_status = interpreter.Invoke(); -if (invoke_status != kTfLiteOk) { - error_reporter->Report("Invoke failed"); - return 1; -} -``` - -我们可以检查返回值 `TfLiteStatus` 以确定运行是否成功。在 -[`c_api_internal.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/c_api_internal.h) -中定义的 `TfLiteStatus` 的可能值有 `kTfLiteOk` 和 `kTfLiteError`。 - -### 获取输出 - -模型的输出张量可以通过在 `tflite::MicroIntepreter` 上调用 `output(0)` 获得，其中 `0` -代表第一个（也是唯一一个）输出张量。 - -在示例中，输出是一个数组，表示输入属于不同类别（“是”（yes）、“否”（no）、“未知”（unknown）以及“静默”（silence））的概率。由于它们是按照集合顺序排列的，我们可以使用简单的逻辑来确定概率最高的类别： - -```C++ - TfLiteTensor* output = interpreter.output(0); - uint8_t top_category_score = 0; - int top_category_index; - for (int category_index = 0; category_index < kCategoryCount; - ++category_index) { - const uint8_t category_score = output->data.uint8[category_index]; - if (category_score > top_category_score) { - top_category_score = category_score; - top_category_index = category_index; - } - } -``` - -在示例其他部分中，使用了一个更加复杂的算法来平滑多帧的识别结果。该部分在 -[recognize_commands.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/micro_speech/recognize_commands.h) -中有所定义。在处理任何连续的数据流时，也可以使用相同的技术来提高可靠性。 - -## 下一步 - -创建并运行示例后，请阅读以下文档： - -* 在[“构建与转换模型”](build_convert.md)中了解如何使用模型。 -* 在[“了解C++库”](library.md)中了解更多关于 C++ 库的内容。 diff --git a/site/zh-cn/lite/microcontrollers/library.md b/site/zh-cn/lite/microcontrollers/library.md deleted file mode 100644 index 0c0ee5b3342..00000000000 --- a/site/zh-cn/lite/microcontrollers/library.md +++ /dev/null @@ -1,92 +0,0 @@ -# 理解 C++ 库 - -微控制器版 TensorFlow Lite C++ 库是 -[TensorFlow 仓库](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro) -的一部分。它可读、易修改，测试结果良好，易整合，并且与标准 TensorFlow Lite 兼容。 - -下面的文档列出了 C++ 库的基本结构，提供了编译所需的命令，并给出了将程序写入新设备的概览。 - -在 -[README.md](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/README.md#如何将用于微控制器的TensorFlow-Lite写入一个新的平台) -中包含更多关于所有这些话题的更多深入信息。 - -## 文件结构 - -在 -[`micro`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro) -根目录中有一个相对简单的结构。然而，因为它位于巨大的 TensorFlow 仓库中，所以我们创建了一些脚本和预生成的项目文件，为多种嵌入式开发环境（如 Arduino, Keil, Make 和 Mbed）提供分离的相关源文件。 - -### 关键文件 - -使用微控制器版 TensorFlow Lite 解释器时最重要的文件在项目的根目录中，并附带测试： - -- [`all_ops_resolver.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/kernels/all_ops_resolver.h) - 提供解释器运行模型的运算符。 -- [`micro_error_reporter.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/micro_error_reporter.h) - 输出调试信息。 -- [`micro_interpreter.h`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/micro_interpreter.h) - 包含控制和运行模型的代码。 - -在 [开始使用微控制器](get_started.md) 可以找到典型的用途的展示。 - -构建系统提供某些文件在特定平台的实现。它们在以平台名称命名的目录下，例如： -[`sparkfun_edge`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/micro/sparkfun_edge)。 - -还有许多其他目录，包括： - -- [`kernel`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/micro/kernels), - 包含运算符的实现和相关代码。 -- [`tools`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/micro/tools), - 包含构建工具和它们的输出。 -- [`examples`](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/micro/examples), - 包含示例代码。 - -### 生成项目文件 - -项目中的 `Makefile` 能够生成包含所有必需源文件的独立项目，这些源文件可以被导入嵌入式开发环境。目前被支持的环境是 Arduino, Keil, Make 和 Mbed。 - -注意：我们为其中一些环境托管预建项目。参阅 -[支持的平台](overview.md#supported-platforms) -以下载。 - -要在 Make 中生成项目，请使用如下指令： - -```bash -make -f tensorflow/lite/micro/tools/make/Makefile generate_projects -``` - -这需要几分钟，因为它需要下载一些大型工具链依赖。结束后，你应看到像这样的路径中，创建了一些文件夹： -`tensorflow/lite/micro/tools/make/gen/linux_x86_64/prj/` -（确切的路径取决于您的主机操作系统）。这些文件夹包含生成的项目和源文件。例如： -`tensorflow/lite/micro/tools/make/gen/linux_x86_64/prj/keil` 包含了 Keil uVision -目标。 - -## 构建库 - -如果您在使用一个已生成的项目，参阅它内含的 README 以获取构建指南。 - -要构建库并从主 TensorFlow 存储库中运行测试，请执行以下命令： - -1. 从 GitHub 中把 TensorFlow 存储库克隆到方便的地方。 - - ```bash - git clone --depth 1 https://github.com/tensorflow/tensorflow.git - ``` - -2. 进入上一步创建的目录。 - - ```bash - cd tensorflow - ``` - -3. 调用 `Makefile` 来构建项目并运行测试。 注意这将会下载所有需要的依赖： - - ```bash - make -f tensorflow/lite/micro/tools/make/Makefile test - ``` - -## 写入新设备 - -把微控制器版 TensorFlow Lite 写入新平台和设备的指南，可在 -[README.md](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro#如何将用于微控制器的TensorFlow-Lite写入一个新的平台) -中查看。 diff --git a/site/zh-cn/lite/microcontrollers/overview.md b/site/zh-cn/lite/microcontrollers/overview.md deleted file mode 100644 index 28540ccfd3f..00000000000 --- a/site/zh-cn/lite/microcontrollers/overview.md +++ /dev/null @@ -1,90 +0,0 @@ -# TensorFlow Lite for Microcontrollers - -TensorFlow Lite for Microcontrollers 是 TensorFlow Lite 的一个实验性移植版本，它适用于微控制器和其他一些仅有数千字节内存的设备。 - -它可以直接在“裸机”上运行，不需要操作系统支持、任何标准 C/C++ 库和动态内存分配。核心运行时(core runtime)在 Cortex M3 上运行时仅需 16KB，加上足以用来运行语音关键字检测模型的操作，也只需 22KB 的空间。 - -## 开始 - -要快速入门并运行 TensorFlow Lite for Microcontrollers，请阅读[微控制器入门](get_started.md)。 - -## 为什么微控制器很重要 - -微控制器通常是小型、低能耗的计算设备，经常嵌入在只需要进行基本运算的硬件中，包括家用电器和物联网设备等。每年都有数十亿个微控制器被生产出来。 - -微控制器通常针对低能耗和小尺寸进行优化，但代价是降低了处理能力、内存和存储。一些微控制器具有用来优化机器学习任务性能的功能。 - -通过在微控制器上运行机器学习推断，开发人员可以在不依赖于网络连接的情况下将 AI 添加到各种各样的硬件设备中，这经常用来克服带宽、功率以及由它们所导致的高延迟而造成的约束。在设备上运行推断也可以帮助保护隐私，因为没有数据从设备中发送出去。 - -## 功能和组件 - -* C++ API，其运行时(runtime)在 Cortex M3 上仅需 16KB -* 使用标准的 TensorFlow Lite [FlatBuffer](https://google.github.io/flatbuffers/) - 架构(schema) -* 为 Arduino、Keil 和 Mbed 等较为流行的嵌入式开发平台预生成的项目文件 -* 针对多个嵌入式平台优化 -* 演示口语热词检测的[示例代码](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/micro/examples/micro_speech) - -## 开发工作流程 - -这是将 TensorFlow 模型部署到微控制器的过程： - -1. **创建或获取 TensorFlow 模型** - - 该模型必须非常小，以便在转换后适合您的目标设备。它只能使用[支持的操作](build_convert.md#支持的操作)。如果要使用当前不被支持的操作，可以提供自己的实现。 - -2. **将模型转换为 TensorFlow Lite FlatBuffer** - - 您将使用 [TensorFlow Lite 转换器](build_convert.md#转换模型)来将模型转换为标准 TensorFlow Lite 格式。您可能希望输出量化模型，因为它们的尺寸更小、执行效率更高。 - -3. **将 FlatBuffer 转换为 C byte 数组** - - 模型保存在只读程序存储器中，并以简单的 C 文件的形式提供。标准工具可用于[将 FlatBuffer 转换为 C 数组](build_convert.md#转换为-C-数组)。 - -4. **集成 TensorFlow Lite for Microcontrollers 的 C++ 库** - - 编写微控制器代码以使用 [C++ 库](library.md)执行推断。 - -5. **部署到您的设备** - - 构建程序并将其部署到您的设备。 - -## 支持的平台 - -嵌入式软件开发的挑战之一是存在许多不同的体系结构、设备、操作系统和构建系统。我们的目标是尽可能多地支持流行的组合，并尽可能地让给其他设备添加支持变得简单。 - -如果您是产品开发人员，您可以下载我们提供的以下平台的构建说明或预生成的项目文件： - -设备 | Mbed | Keil | Make/GCC -------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------------------------------------------ | -------- -[STM32F746G Discovery Board](https://www.st.com/en/evaluation-tools/32f746gdiscovery.html) | [下载](https://drive.google.com/open?id=1OtgVkytQBrEYIpJPsE8F6GUKHPBS3Xeb) | - | [下载](https://drive.google.com/open?id=1u46mTtAMZ7Y1aD-He1u3R8AE4ZyEpnOl) -["Blue Pill" STM32F103 兼容开发板](https://github.com/google/stm32_bare_lib) | - | - | [说明](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/README.md#building-for-the-blue-pill-stm32f103-using-make) -[Ambiq Micro Apollo3Blue EVB（使用 Make）](https://ambiqmicro.com/apollo-ultra-low-power-mcus/) | - | - | [说明](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/README.md#building-for-ambiq-micro-apollo3blue-evb-using-make) -[Generic Keil uVision Projects](http://www2.keil.com/mdk5/uvision/) | - | [下载](https://drive.google.com/open?id=1Lw9rsdquNKObozClLPoE5CTJLuhfh5mV) | - -[Eta Compute ECM3531 EVB](https://etacompute.com/) | - | - | [说明](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/README.md#Building-for-the-Eta-Compute-ECM3531-EVB-using-Make) - -如果您的设备尚未被支持，添加支持也许并不困难。您可以在 -[README.md](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/README.md#how-to-port-tensorflow-lite-micro-to-a-new-platform) -中了解该过程。 - -### 可移植参考代码 - -如果您还没有考虑具体的的微控制器平台，或者只想在开始移植之前试用代码，最简单的方法是[下载与平台无关的参考代码](https://drive.google.com/open?id=1cawEQAkqquK_SO4crReDYqf_v7yAwOY8)。 - -归档中有很多文件夹，每个文件夹只包含构建一个二进制文件所需的源文件。每个文件夹都有一个简单的 Makefile 文件，您应该能够将文件加载到几乎任何 IDE 中并构建它们。我们还提供了已经设置好的 [Visual Studio Code](https://code.visualstudio.com/) 项目文件，因此您可以轻松地在跨平台 IDE 中浏览代码。 - -## 目标 - -我们的设计目标是使框架可读、易于修改、经过良好测试、易于集成，并通过一致的文件架构、解释器、API 和内核接口与 TensorFlow Lite 完全兼容。 - -您可以阅读更多在[目标和权衡](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro#goals)方面有关设计的信息。 - -## 限制 - -TensorFlow Lite for Microcontrollers 专为微控制器开发中的特殊限制而设计。如果您正在使用更强大的设备（例如像 Raspberry Pi 这样的嵌入式 Linux 设备），标准的 TensorFlow Lite 框架可能更容易集成。 - -应考虑以下限制： - -* 仅支持 TensorFlow 操作的[有限子集](build_convert.md#支持的操作) -* 仅支持有限的一些设备 -* 低级 C++ API 需要手动内存管理 diff --git a/site/zh-cn/lite/models/image_classification/overview.md b/site/zh-cn/lite/models/image_classification/overview.md deleted file mode 100644 index 11c7c8bcd34..00000000000 --- a/site/zh-cn/lite/models/image_classification/overview.md +++ /dev/null @@ -1,209 +0,0 @@ -# 图像分类 - - - -使用优化的预训练模型来识别上百种对象，包括人、活动、动物、植物和地点。 - -## 开始 - -如果你对图像分类的概念不熟悉，你应该先阅读 什么是图像分类？ - -关于如何在移动应用中使用图像分类，推荐查看我们提供的 示例应用和指导。 - -如果你使用 Android 和 iOS 之外的平台，或者你已经熟悉了 TensorFlow Lite 接口，你可以直接下载我们的新手图像分类模型及其附带的标签。 - -下载新手图像分类及标签 - -当新手模型在你的目标设备运行起来之后，你可以尝试其他模型，在性能、准确率以及模型体积间找到最佳的平衡点。详见 选择不同模型。 - -### 示例应用和指导 - -我们在 Android 和 iOS 平台上都有图像分类的示例应用，并解释了它们的工作原理。 - -#### Android - -查看Android示例 - -阅读 [Android example guide](android.md) 以了解应用工作原理。 - -#### iOS - -查看iOS示例 - -阅读 [iOS example guide](ios.md) 以了解应用工作原理。 - -#### 截屏 - -下面的截屏为 Android 图像分类示例应用。 - -Screenshot of Android example - -## 什么是图像分类？ - -机器学习的一个常见应用是图像识别。比如，我们可能想要知道下图中出现了哪类动物。 - -dog - -预测图像类别的任务被称为 _图像分类_ 。训练图像分类模型的目的是识别各类图像。比如，一个模型可能被训练用于识别三种动物的特征：兔子、仓鼠和狗。 - -当我们提供一张新的图片给模型时，它会输出这张图片含有这三种动物的概率。以下是一个输出示例： - - - - - - - - - - - - - - - - - - - - - - -
    动物种类概率
    兔子0.07
    仓鼠0.02
    狗0.91
    - -基于输出，我们能够看到分类模型预测出，这张图片有很大概率表示的是一条狗。 - -注意：图像分类只能告诉你图片里出现的类别及其概率，并且只能是被训练过的类别。它不能告诉你图片里对象的位置或者名称。 -如果你需要识别图片里对象的名称及位置，你应该使用 物体检测 模型。 - -### 训练、标签和推断 - -在训练中，用图像和其对应的 _标签_ 投喂一个图像分类模型。每个标签是一个概念或种类的名字。这个模型就要学会去识别这些标签。 - -给予足够多的训练数据（通常一个标签对应数以百计的图片），这个图像分类模型就能够学习去预测新的图片是否属于训练数据中的某些种类。这个预测的过程被称为 _推断_ 。 - -为了执行推断，一张图片被输入进模型中。接着，模型将输出一串代表概率的数组，元素大小介于 0 和 1 之间。结合我们的示例模型，这个过程可能如下所示： - - - - - - -
    dog→[0.07, 0.02, 0.91]
    - -输出中的每个数字都对应训练数据中的一个标签。将我们的输出和这三个训练标签关联，我们能够看出，这个模型预测了这张图片中的对象有很大概率是一条狗。 - - - - - - - - - - - - - - - - - - - - - - -
    标签概率
    兔子0.07
    仓鼠0.02
    狗0.91
    - -你可能注意到这些概率的总和（兔子，仓鼠和狗的概率）是 1。这是多分类模型的常见输出。（详见：Softmax） - -### 模糊不清的结果 - -既然概率的总和总是等于 1，那么如果这张图片没有被模型识别出来，也就是不属于被训练的种类，你可能会发现它的几个标签都没有特别大的概率。 - -比如，下表可能表示了一个模糊不清的结果： - - - - - - - - - - - - - - - - - - - - - - -
    标签概率
    兔子0.31
    仓鼠0.35
    狗0.34
    - -### 使用和限制 - -我们提供的这些图形分类模型对单标签分类很有用。单标签分类是指预测图像最有可能表示的某一个标签。这些模型被训练用于识别 1000 类图像。完整的标签列表：模型压缩包 - -如果你想要训练模型识别新的类别：自定义模型. - -针对以下使用案例，你应该采用不同的模型： - -
      -
    • 预测图片里的一个或多个对象的种类和位置（详见：物体检测）
      • -
    • 预测图像的组成，比如主体与背景（详见：分割）
      • -
    - - -当新手模型在你的目标设备运行起来之后，你可以尝试其他模型，在性能、准确率以及模型体积间找到最佳的平衡点。详见：选择不同模型。 - -## 选择不同模型 - -我们的 模型列表 中有许多图像分类模型供你选择。 -你应该在它们的性能、准确率和模型体积之间进行权衡，以选择对你来说最优的模型。 - -### 性能 - -我们根据在同样的硬件条件下，一个模型执行推断所花费的时间来衡量性能。时间越短，模型越快。 - -你需要的性能取决于你的应用。对实时视频这类应用来说，性能可能非常重要。因为需要在下一帧绘制完之前及时分析每一帧（例如：推断用时必须少于 33 ms 才能实时推断 30 fps 的视频流）。 - -我们经过量化的MobileNet 模型的性能范围为 3.7 ms 至 80.3 ms。 - -### 准确率 - -我们根据模型正确分类图像的频率来衡量准确度。比如，一个准确率为 60% 的模型平均有 60% 的时间能正确分类一张图片。 - -我们的 模型列表 提供 Top-1 和 Top-5 准确率数据。Top-1 是指模型输出正确标签的概率为最高的频率。Top-5 是指模型输出正确标签的概率在前五的频率。 - -我们经过量化的 MobileNet 模型的准确率范围为 64.4% 至 89.9%。 - -### 体积 - -磁盘上模型的体积因其性能和准确性而异。体积可能对移动开发（可能影响应用的下载体积）或者硬件开发（可用存储可能是有限的）很重要。 - -我们经过量化的 MobileNet 模型的准确率范围为 0.5 Mb 至 3.4 Mb。 - -### 模型结构 - -模型列表 中的模型有不同的结构，从模型名可以看出，比如，你可以选择 MobileNet、Inception 或者其他的结构。 - -模型的结构影响它的性能、准确率和体积。我们提供的模型都是用同样的数据训练的，意味着你可以通过我们提供的统计数据对比这些模型，来选择最适合你的应用的。 - -注意：我们提供的图像分类模型接受的输入尺寸不同。有些模型将其标注在文件名上。比如，Mobilenet_V1_1.0_224 模型接受 224x224 像素的输入。

    -所有模型都要求每个像素有三个颜色通道（红、绿、蓝）。经过量化的模型中每个通道需要 1 个字节，浮点模型中每个通道需要 4 个字节。

    -我们的 Android 和 iOS 代码样本展示了如何将全尺寸相机图像处理为每个模型需要的格式。 - -## 自定义模型 - -我们提供的预训练模型被训练用于识别 1000 类图像。完整的标签列表：模型压缩包。 - -你能使用 _迁移学习_ 技术来再训练(re-train)一个模型，以识别新的类别。比如你能再训练一个模型来区分不同品种的树，尽管原始训练数据中并没有树。为了达到这个目的，你的每个新标签都需要一组训练图片。 - -学习如何实现迁移学习：用 TensorFlow 识别花卉 codelab。 \ No newline at end of file diff --git a/site/zh-cn/lite/models/object_detection/overview.md b/site/zh-cn/lite/models/object_detection/overview.md deleted file mode 100644 index af2856a3619..00000000000 --- a/site/zh-cn/lite/models/object_detection/overview.md +++ /dev/null @@ -1,207 +0,0 @@ -# 物体检测 - - - -使用矩形框识别图片中的多个对象。可以辨别出80多种不同种类的物体。 - -## 开始使用 - -如果你是 TensorFlow Lite 新手并且使用 Android 或 iOS 进行工作，我们推进您使用如下的应用实例帮助您进行探索。 - - Android -示例 - iOS -示例 - -如果您使用的不是 Android 或 iOS 平台，或者您对于 TensorFlow Lite APIs 较为熟悉, 您可以下载我们的初始对象检测模块以及对应的标签。 - -下载初始模块和标签 - -更多关于初始模块的内容，请参考 -初始模块. - -## 何为物体检测? - -对于给定的图片或者视频流，对象检测模块可以识别出已知的物体和该物体在图片中的位置。例如，如下的示例应用屏幕截图中展示了如何辨识物体以及标注对应的坐标: - -Screenshot of Android example - -物体检测模块被训练用于检测多种物体的存在以及他们的位置。例如，模型可使用包含多个水果的图片和水果所分别代表（如，苹果，香蕉，草莓）的 _label_ 进行训练，返回的数据指明了图像中对象所出现的位置。 - -随后，当我们为模型提供图片，模型将会返回一个列表，其中包含检测到的对象，包含对象矩形框的坐标和代表检测可信度的分数。 -### 模块输出 - -想象一下一个模块被训练用于检测苹果，香蕉和草莓。当我们输入一幅图片后，模块将会返回给我们一组本示例的检测结果。 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    类别分数坐标
    苹果0.92[18, 21, 57, 63]
    香蕉0.88[100, 30, 180, 150]
    草莓0.87[7, 82, 89, 163]
    香蕉0.23[42, 66, 57, 83]
    苹果0.11[6, 42, 31, 58]
    - -### 信任分数 - -我们使用信任分数和所检测到对象的坐标来表示检测结果。分数反应了被检测到物体的可信度，范围在 0 和 1 之间。最大值为1，数值越大可信度越高。 - -您可以检测到裁切的极限值并据此放弃检测结果，这取决于您的应用。 在我们的示例中，我们检测到裁切的极限值为 0.5 （这意味50%的检测是可信的）。在此示例中，我们将会忽略数组中最后两个对象，因为他们的信任分数低于了 0.5 。 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    类型分数坐标
    苹果0.92[18, 21, 57, 63]
    香蕉0.88[100, 30, 180, 150]
    草莓0.87[7, 82, 89, 163]
    香蕉0.23[42, 66, 57, 83]
    苹果0.11[6, 42, 31, 58]
    - -使用裁切功能必须基于误判位置（物体识别错误或物体位置识别有误）或错误（由于可信度过低导致的物体未被捕捉）。 -如下图所示，梨子（未被模块训练检测的物体）被误判为“人”。实例中的误判可以通过适当的图片裁切来忽略。在此示例中，裁剪 0.6 （或 60% ）可以适当的排除误判。 - -Screenshot of Android example showing a false positive - -### 坐标 - -针对每个被检测的物体，模块将会返回一个由四个数字组成的数组，该四个数字代表了围绕物体的一个矩形框。在我们提供的示例模块中，返回的数组中的元素按照如下顺序： - - - - - - - - - - - -
    [top,left,bottom,right]
    - -top 的值代表了矩形框的顶部距离图片上部的距离，单位为像素。 left 的值代表了矩形框的左边距离图片左边的距离。bottom 和 right 值的表示方法同理。 -注意：对象检测模块接受特定尺寸的模型作为输入。这很有可能与您的图像设备生成的原始图片尺寸不同，所以您需要编写代码将原始图片缩放至模型可接受的尺寸(我们提供了 示范程序)。

    模块输出的像素值表示在缩放后的图片中的位置，所以您需要调整调整原始图片等尺寸来保证正确。 - -## 初始模型 - -我们推荐使用预训练的量化 COCO SSD MobileNet v1 模型来入门。 - -下载初始模型和标签 - -### 用法和限制 - -物体检测模块最多能够在一张图中识别和定位10个物体。目前支持80种物体的识别，详细列表如下： -model -zip. - -如果您需为识别新类型而训练模型，请参考 -自定义模块. - -接下来的使用案例中，您可能用到不同种类的模块： - -
      -
    • 预测所表达内容 (参考 图片分类)
      • -
    • 预测图片的构成，如主题与背景 (参考 分割)
      • -
    - -### 输入 - -模块使用单个图片作为输入。理想的图片尺寸是 300x300 像素，每个像素有3个通道（红，蓝，和绿）。这将反馈给模块一个 27000 字节（ 300 x 300 x 3 ）的扁平化缓存。由于该模块经过标准化处理，每一个字节代表了 0 到 255 之间的一个值。 -### 输出 - -该模型输出四个数组，分别对应索引的 0-4。前三个数组描述10个被检测到的物体，每个数组的最后一个元素匹配每个对象。检测到的物体数量总是10。 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    索引名称描述
    0坐标 [10][4] 多维数组，每一个元素由 0 到1 之间的浮点数，内部数组表示了矩形边框的 [top, left, bottom, right]
    1类型10个整型元素组成的数组（输出为浮点型值），每一个元素代表标签文件中的索引。
    2分数10个整型元素组成的数组，元素值为 0 至 1 之间的浮点数，代表检测到的类型
    3检测到的物体和数量长度为1的数组，元素为检测到的总数
    - -## 自定义模块 - -我们提供预训练模块可被用于检测80多种物体。详细的列表如下： -model -zip. - -您可以使用转移学习等技术来重新训练模型从而能够辨识初始设置之外的物品种类。例如，您可以重新训练模型来辨识各种蔬菜，哪怕原始训练数据中只有一种蔬菜。为达成此目标，您需要为每一个需要训练的标签准备一系列训练图片。 -了解如何实时转移学习 -30分钟内训练和运行实时移动物体检测器. diff --git a/site/zh-cn/lite/models/pose_estimation/overview.md b/site/zh-cn/lite/models/pose_estimation/overview.md deleted file mode 100644 index 153cb42bd30..00000000000 --- a/site/zh-cn/lite/models/pose_estimation/overview.md +++ /dev/null @@ -1,135 +0,0 @@ -# 姿势预测 - - - -## 开始使用 - -_PoseNet_ 能够通过预测图像或视频中人体的关键位置进行姿势的预测。 - -下载此模块 - -Android 和 iOS 设备上的一对一课程即将面世. 与此同时，如果您想要在 web 浏览器中体验此模块，可以访问 -TensorFlow.js -GitHub 代码仓库. - -## 工作原理 - -姿势检测通过使用计算机图形技术来对图片和视频中的人进行检测和判断，如图片中的人露出了肘臂。 - -为了达到清晰的目的，该算法只是对图像中的人简单的预测身体关键位置所在，而不会去辨别此人是谁。 - -关键点检测使用“编号 部位”的格式进行索引，并对部位的探测结果伴随一个信任值。信任值取值范围在 0.0 至 1.0，1.0 为最高信任值。 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    编号部位
    0鼻子
    1左眼
    2右眼
    3左耳
    4右耳
    5左肩
    6右肩
    7左肘
    8右肘
    9左腕
    10右腕
    11左髋
    12右髋
    13左膝
    14右膝
    15左踝
    16右踝
    - -## 示例输出 - -Animation showing pose estimation - -## 模块性能 - -性能很大程度取决于您的设备性能以及输出的幅度(热点图和偏移向量)。PoseNet 对于不同尺寸的图片是不变式，也就是说在原始图像和缩小后图像中预测姿势位置是一样的。这也意味着 PostNet 能精确配置性能消耗。 - -输出幅度决定了缩小后的和输入的图片尺寸的相关程度。输出幅度同样影响到了图层的尺寸和输出的模型。更高的输出幅度决定了更小的网络和输出的图层分辨率，和更小的可信度。 - -在此示例中，输出幅度可以为 8、16 或 32。换句话说，当输出幅度为 32，则会拥有最高性能和最差的可信度；当输出幅度为 8，则会有用最高的可信度和最低的性能。我们给出的建议是 16。 - -下图展示了输出幅度的程度决定缩放后的输出和输入的图像的相关度。更高的输出幅度速度更快，但也会导致更低的可信度。 - -Output stride and heatmap resolution - -## 关于此模块的更多内容 - - - -### 使用案例 - - diff --git a/site/zh-cn/lite/models/segmentation/overview.md b/site/zh-cn/lite/models/segmentation/overview.md deleted file mode 100644 index 390600e66c6..00000000000 --- a/site/zh-cn/lite/models/segmentation/overview.md +++ /dev/null @@ -1,31 +0,0 @@ -# Segmentation - -![segmentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/models/images/segmentation.png) - -## Get started - -DeepLab 是用于语义图像分割的最先进的深度学习模型，其目标是为图像中的每个像素分配语义标签(例如人，狗，猫)。 - -[Download starter model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/deeplabv3_257_mv_gpu.tflite) - -## How it works - -语义图像分割预测图像的每个像素是否与某个类相关联。这与检测矩形区域中目标[目标检测]的任务(https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/models/object_detection/overview.md)和对整个图像进行分类[图像分类]的任务(https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/models/image_classification/overview.md)形成对照。 - -当前的实现包括以下功能： - -1. DeepLabv1 :我们使用 atrous convolution 来显示地控制在深度卷积神经网络中计算特征响应部分的分辨率。 -2. DeepLabv2 :我们应用 atrous spatial pyramid pooling(ASPP) ,使用多个采样率和有效视野的过滤器,在多种尺度上稳健地分割目标对象。 -3. DeepLabv3 :我们使用图像级特征[5,6]来扩展ASPP模块以捕获更长距离的信息。我们还增加了批标准化[7]参数以加快训练。特别的，在训练和评估期间我们应用 atrous convolution 来提取不同输出步幅的输出特征，这在输出步幅等于16时有效的促进了批标准化训练,并在输出步幅为8时得到了更高的评估效果。 -4. DeepLabv3+ :我们扩展了 DeepLabv3 ,增加了一个简单但有效的解码器模块，以优化细分结果，尤其是沿着对象边界。此外，在这种编码器-解码器结构中，可以通过 atrous convolution 任意地控制所提取的编码器特征的分辨率，以折衷精度和运行时间。 - -## Example output - -该模型将以很高的精度在目标对象上创建掩膜。 -![segmentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/models/segmentation/images/segmentation.gif) - -## Read more about segmentation - -* [Semantic Image Segmentation with DeepLab in TensorFlow](https://ai.googleblog.com/2018/03/semantic-image-segmentation-with.html) -* [TensorFlow Lite Now Faster with Mobile GPUs (Developer Preview)](https://medium.com/tensorflow/tensorflow-lite-now-faster-with-mobile-gpus-developer-preview-e15797e6dee7) -* [DeepLab: Deep Labelling for Semantic Image Segmentation](https://github.com/tensorflow/models/tree/master/research/deeplab) diff --git a/site/zh-cn/lite/models/smart_reply/overview.md b/site/zh-cn/lite/models/smart_reply/overview.md deleted file mode 100644 index eb61404b0de..00000000000 --- a/site/zh-cn/lite/models/smart_reply/overview.md +++ /dev/null @@ -1,51 +0,0 @@ -# 智能回复 - - - -## 开始 - -我们的智能回复模型基于聊天消息生成回复建议。该建议是主要是依据上下文的相关内容，一触即发的响应帮助用户轻松回复传入的消息。 - -下载入门模型和标签 - -### 应用示例 - -在 Android 上，演示 TensorFlow Lite 智能回复模型的示例 - -查看 -Android 示例应用 - -阅读该 -[GitHub 页面](https://github.com/tensorflow/examples/tree/master/lite/examples/smart_reply/android/) -以了解该应用的工作原理。其中，你还会学到如何定制化 C++ ops。 - -## 怎么运行 - -该模型为会话聊天消息生成回复建议。 - -该设备内置的模型可以带来以下优势： -
      -
    • 运行快速：该模型内置在设备中和无需网络连接。因此，模型推理快速，同时平均延迟只有几毫秒。
      • -
    • 资源高效：该模型在设备中占用的内存很小。
      • -
    • 隐私保护：用户数据从不离开设备。
      • -
    - -## 例子展示 - -Animation showing smart reply - -## 了解更多 - - - -## 落地场景 - - diff --git a/site/zh-cn/lite/performance/benchmarks.md b/site/zh-cn/lite/performance/benchmarks.md deleted file mode 100644 index 4d555daa79e..00000000000 --- a/site/zh-cn/lite/performance/benchmarks.md +++ /dev/null @@ -1,167 +0,0 @@ -# 性能跑分 - -本文档列出了在一些 Android 和 iOS 设备上运行常见模型时 TensorFlow Lite 的跑分。 - -这些跑分数据由 [Android TFLite benchmark binary](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark) 及 [iOS benchmark app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/ios) 产生。 - -# 安卓环境的跑分 - -对于安卓环境的跑分，为了减少设备间的差异性，CPU 亲和性被设置为使用大核跑分。 (查看[详情](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark#reducing-variance-between-runs-on-android)) - -假设模型被下载并解压缩到 `/data/local/tmp/tflite_models` 路径。用于跑分的二进制文件 - -使用 [这些命令构建](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark#on-android)。 -此外，我们假设该文件位于 `/data/local/tmp` 目录。 - -使用以下语句运行跑分: - -``` -adb shell taskset ${CPU_MASK} /data/local/tmp/benchmark_model \ - --num_threads=1 \ - --graph=/data/local/tmp/tflite_models/${GRAPH} \ - --warmup_runs=1 \ - --num_runs=50 \ - --use_nnapi=false -``` - -在这里， `${GRAPH}`是模型的名字， `${CPU_MASK}` 是CPU亲和度设置。 -请从下表中选择: - -Device | CPU_MASK --------| ---------- -Pixel 2 | f0 -Pixel xl | 0c - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# iOS 跑分 - -要运行 iOS 跑分， [跑分应用程序](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/ios)应含有合适的模型，且 benchmark_params.json` 中的` `num_threads` 被设置为 1。 - -
    模型名称设备平均推理时间
    - Mobilenet_1.0_224(float) - Pixel 2 123.3 ms
    Pixel XL 113.3 ms
    - Mobilenet_1.0_224 (quant) - Pixel 2 65.4 ms
    Pixel XL 74.6 ms
    - NASNet mobile - Pixel 2 273.8 ms
    Pixel XL 210.8 ms
    - SqueezeNet - Pixel 2 234.0 ms
    Pixel XL 158.0 ms
    - Inception_ResNet_V2 - Pixel 2 2846.0 ms
    Pixel XL 1973.0 ms
    - Inception_V4 - Pixel 2 3180.0 ms
    Pixel XL 2262.0 ms
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    模型名称设备平均推理时间
    - Mobilenet_1.0_224(float) - iPhone 8 32.2 ms
    - Mobilenet_1.0_224 (quant) - iPhone 8 24.4 ms
    - NASNet mobile - iPhone 8 60.3 ms
    - SqueezeNet - iPhone 8 44.3
    - Inception_ResNet_V2 - iPhone 8562.4 ms
    - Inception_V4 - iPhone 8 661.0 ms
    - - \ No newline at end of file diff --git a/site/zh-cn/lite/performance/best_practices.md b/site/zh-cn/lite/performance/best_practices.md deleted file mode 100644 index 3497a21363f..00000000000 --- a/site/zh-cn/lite/performance/best_practices.md +++ /dev/null @@ -1,61 +0,0 @@ -# 性能的最佳实践 - -由于移动和嵌入式设备的运算能力有限，所以保持应用的资源被高效利用是非常重要的。我们已经编写了一份最佳实践和策略的清单，你能用它来优化你的 TensorFlow Lite 模型和应用。 - -## 为任务选择最佳的模型 - -根据任务的不同，你会需要在模型复杂度和大小之间做取舍。如果你的任务需要高准确率，那么你可能需要一个大而复杂的模型。对于精确度不高的任务，就最好使用小一点的模型，因为小的模型不仅占用更少的磁盘和内存，也一般更快更高效。比如，下图展示了常见的图像分类模型中准确率和延迟对模型大小的影响。 - -![模型大小和准确度的关系图](../images/performance/model_size_vs_accuracy.png "模型大小和准确度") - -![准确度和延迟时间的关系图](../images/performance/accuracy_vs_latency.png "准确度和延迟时间") - -一个针对移动设备优化的示例模型就是 [MobileNets](https://arxiv.org/abs/1704.04861)，该模型是为了移动端视觉应用而优化的。我们的 [模型列表](../models/hosted.md) 列出了另外几种专为移动和嵌入式设备优化的模型。 - -你可以用你自己的数据通过迁移学习再训练这些模型。查看我们的迁移学习教程：[图像分类](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/#0) 和 [物体检测](https://medium.com/tensorflow/training-and-serving-a-realtime-mobile-object-detector-in-30-minutes-with-cloud-tpus-b78971cf1193)。 - -## 测试你的模型 - -在选择了一个适合你的任务的模型之后，测试该模型和设立基准很好的行为。TensorFlow Lite [测试工具](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark) 有内置的测试器，可展示每一个运算符的测试数据。这能帮助理解性能瓶颈和哪些运算符主导了运算时间。 - -## 测试和优化图（graph）中的运算符 - -如果某个特定的运算符频繁出现在模型中，并且基于测试你发现这个运算符消耗了大部分时间，那么你可以研究如何优化这个运算符。这种情况应该非常少见，因为 TensorFlow Lite 中的大部分运算符都是优化过的版本。然而，如果你了解该运算符的运行限制，你或许可以写一个自定义的更快的版本。查看我们的 [自定义运算符文档](../custom_operators.md)。 - -## 优化你的模型 - -模型压缩旨在创建更小的模型，并且通常更快、更高效节能。因此它们能被部署到移动设备上。 - -### 量化 - -如果你的模型使用浮点权重或者激励函数，那么模型大小或许可以通过量化减少75%，该方法有效地将浮点权重从32字节转化为8字节。量化分为：[训练后量化](post_training_quantization.md) 和 [量化训练](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/quantize/README.md){:.external}。前者不需要再训练模型，但是在极少情况下会有精度损失。当精度损失超过了可接受范围，则应该使用量化训练。 - -我们强烈推荐设立基准以确保模型压缩期间准确率没有被影响。查看详细信息：[模型优化文档](model_optimization.md)。 - -## 调整线程数 - -TensorFlow Lite 支持针对多运算符使用多线程内核。你可以增加线程数以提高运算符运行速度。然而，增加线程数会使你的模型使用更多的资源和能源。 - -对有些应用来说，延迟或许比能源效率更重要。你可以通过设定 [解释器](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/interpreter.h#L346) 的数量来增加线程数。然而，根据同时运行的其他操作不同，多线程运行会增加性能的可变性。比如，隔离测试可能显示多线程的速度是单线程的两倍，但如果同时有另一个应用在运行的话，性能测试结果可能比单线程更差。 - -## 清除冗余副本 - -如果你的应用没有被很好地设计，在输入模型和读取模型输出时可能会有冗余副本。应确保清除冗余副本。如果你在使用高层 API，如 Java，请确保仔细阅读性能注意事项。比如，如果使用 ByteBuffers 作为[输入](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java#L175)，Java API 会快得多。 - -## 用平台特定工具测试你的应用 - -平台特定工具，如 [Android profiler](https://developer.android.com/studio/profile/android-profiler) 和 [Instruments](https://help.apple.com/instruments/mac/current/)，提供了丰富的可被用于调试应用的测试信息。有时性能问题可能不出自于模型，而是与模型交互的应用代码。确保熟悉平台特定测试工具和对该平台最好的测试方法。 - -## 评估你的模型是否受益于使用设备上可用的硬件加速器 - -TensorFlow Lite 增加了新的方法来配合更快的硬件加速模型，比如 GPU、DSP 和神经加速器。一般来说，这些加速器通过 [代理](delegates.md) 子模块暴露，这些子模块接管部分解释器执行。TensorFlow Lite 能通过以下方法使用代理： - -* 使用 Android 的 [神经网络 API](https://developer.android.com/ndk/guides/neuralnetworks/)。你可以利用这些硬件加速器后台来提升模型速度和效率。要启用神经网络 API，在解释器实例内调用 [UseNNAPI](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/interpreter.h#L343)。 -* 我们已经发布了一个仅限二进制的 GPU 代理，Android 和 iOS 分别使用 OpenGL 和 Metal。要试用它们，查看 [GPU 代理教程](gpu.md) 和 [文档](gpu_advanced.md)。 -* 如果你能访问非标准硬件，也可以创建你自己的代理。更多信息，查看 [TensorFlow Lite 代理](delegates.md)。 - -请注意，有的加速器在某些模型效果更好。为每个代理设立基准以测试出最优的选择是很重要的。比如，如果你有一个非常小的模型，那可能没必要将模型委托给 NN API 或 GPU。相反，对于具有高算术强度的大模型来说，加速器就是一个很好的选择。 - -## 需要更多帮助？ - -TensorFlow 团队非常乐意帮助你诊断和定位具体的性能问题。请在 [GitHub](https://github.com/tensorflow/tensorflow/issues) 提出问题并描述细节。 diff --git a/site/zh-cn/lite/performance/delegates.md b/site/zh-cn/lite/performance/delegates.md deleted file mode 100644 index 54276d6e05a..00000000000 --- a/site/zh-cn/lite/performance/delegates.md +++ /dev/null @@ -1,193 +0,0 @@ -# TensorFlow Lite 代理 - -_说明：Delegate API 仍处于试验阶段并将随时进行调整。_ - -## 什么是 TensorFlow Lite 代理？ - -TensorFlow Lite 代理是一种将部分或全部的图形运算委托予另一线程执行的方法。 - -## 你为什么应该使用代理？ - -由于移动设备的处理能力不足以及电量受限，在其之上进行高算力的机器学习模型的演算是不可行的。 - -为了避免加重 CPU（中央处理器）的负担，一些设备具有诸如 GPU（图形处理器）或 DSP（数字信号处理器）等的硬件加速器以求获取更佳的性能与更高的能效。 - -## 使用 GPU 代理 - -TensorFlow Lite 为具备 GPU 的设备提供了一个 GPU 代理用以模型计算的加速。 - -有关 GPU 代理的概述，请查看 -[TensorFlow Lite 在 GPU 环境下](https://www.tensorflow.org/lite/performance/gpu_advanced) 。 -有关在 Android 和 iOS 设备上使用 GPU 代理的步骤教程，请查看 -[TensorFlow Lite GPU 代理](https://www.tensorflow.org/lite/performance/gpu) 。 - -## 代理是如何运作的？ - -假设我们将一个简化的图形样本进行如下图所示的操作： - -![原生图形样本](../images/performance/tflite_delegate_graph_1.png "原生图形样本") - -如果把一个代理用于进行具体操作，那么 TensorFlow Lite 会将图形分割为多个交由代理进行处理的子图块。 - -若使用一个拥有高效处理 Conv2D（卷积层）和计算 Mean（平均值）操作的能力且名为“MyDelegate”的代理，那么它将导致主图变更为进行如下图所示的操作。 - -![使用代理的图形样本](../images/performance/tflite_delegate_graph_2.png "使用代理的图形样本") - -在返回值中，每个交由代理进行处理的子图将会被更替为评估该子图的节点。 - -根据不同的模型，末图可以一个节点终结，这意味着所有的图将被代理或以多个节点的子图进行处理。一般而言，当你每次从代理切换至主图而不希望采用由代理处理的混合子图时，将会造成由子图转换为主图的损耗。毕竟，内存交换并非总是安全的。 - -## 如何添置一个代理 - -_请注意以下所采用的 API 仍处于试验阶段并将随时进行调整。_ - -基于上节所述，添置一个代理需要完成以下步骤： - -1. 定义一个用于负责评估代理子图的核心节点 -2. 创建一个用于负责注册该核心节点以及说明代理可用节点的实例 [TensorFlow Lite 代理](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/c_api_internal.h#L545) - -为了使用代码进行说明，让我们定义一个可快速执行 Conv2D 和计算 Mean 操作的代理并将其命名为“MyDelegate”。 - -``` -// 这是执行操作或整个图形的开始。 -// 该类具有一个空实现，仅作结构体的声明。 -class MyDelegate { - public: - // 如果代理可以处理此类操作，则返回“true”。 - static bool SupportedOp(const TfLiteRegistration* registration) { - switch (registration->builtin_code) { - case kTfLiteBuiltinConv2d: - case kTfLiteBuiltinMean: - return true; - default: - return false; - } - } - - // 代码初始化 - bool Init() {} - // 初始工作分配（例如：分配缓冲区） - bool Prepare(TfLiteContext* context, TfLiteNode* node) {} - // 代理子图开始运行。 - bool Invoke(TfLiteContext* context, TfLiteNode* node) {} - // ... 添加其他所需的方法 -}; - -// 为核心节点创建一个替代主 TfLite 图中的子图的 TfLiteRegistration。 -TfLiteRegistration GetMyDelegateNodeRegistration() { - // 这是为了获取被添加至 TFLite 图而非替换它的子图的代理节点的初始化 - // 它被视为一个操作节点。 - // 但在此，Init 函数将用于初始化代理，而 Invoke 函数将用于运行代理图。 - // 预缓冲。 - // 释放内存。 - TfLiteRegistration kernel_registration; - kernel_registration.builtin_code = kTfLiteBuiltinDelegate; - kernel_registration.custom_name = "MyDelegate"; - kernel_registration.free = [](TfLiteContext* context, void* buffer) -> void { - delete reinterpret_cast(buffer); - }; - kernel_registration.init = [](TfLiteContext* context, const char* buffer, - size_t) -> void* { - // 在节点的初始化阶段中，初始化“MyDelegate”实例。 - const TfLiteDelegateParams* delegate_params = - reinterpret_cast(buffer); - MyDelegate* my_delegate = new MyDelegate; - if (!my_delegate->Init(context, params)) { - return nullptr; - } - return my_delegate; - }; - kernel_registration.invoke = [](TfLiteContext* context, - TfLiteNode* node) -> TfLiteStatus { - MyDelegate* kernel = reinterpret_cast(node->user_data); - return kernel->Invoke(context, node); - }; - kernel_registration.prepare = [](TfLiteContext* context, - TfLiteNode* node) -> TfLiteStatus { - MyDelegate* kernel = reinterpret_cast(node->user_data); - return kernel->Prepare(context, node); - }; - - return kernel_registration; -} - -// 实现 TfLiteDelegate 方法 - -TfLiteStatus DelegatePrepare(TfLiteContext* context, TfLiteDelegate* delegate) { - // 说明所有可被代理评估的节点以及请求框架使用代理核心替换图。 - // 当我们需要获取头节点的大小时，保留一个节点。 - std::vector supported_nodes(1); - TfLiteIntArray* plan; - TF_LITE_ENSURE_STATUS(context->GetExecutionPlan(context, &plan)); - TfLiteNode* node; - TfLiteRegistration* registration; - for (int node_index : TfLiteIntArrayView(plan)) { - TF_LITE_ENSURE_STATUS(context->GetNodeAndRegistration( - context, node_index, &node, ®istration)); - if (MyDelegate::SupportedOp(registration)) { - supported_nodes.push_back(node_index); - } - } - // 设置替换所有节点的头节点。 - supported_nodes[0] = supported_nodes.size() - 1; - TfLiteRegistration my_delegate_kernel_registration = - GetMyDelegateNodeRegistration(); - - // 该返回值将图分割为子图块，对于子图，它将被代理视为一个 - // ‘my_delegate_kernel_registration’进行处理。 - return context->ReplaceNodeSubsetsWithDelegateKernels( - context, my_delegate_kernel_registration, - reinterpret_cast(supported_nodes.data()), delegate); -} - -void FreeBufferHandle(TfLiteContext* context, TfLiteDelegate* delegate, - TfLiteBufferHandle* handle) { - // 用于实现释放内存的方法。 -} - -TfLiteStatus CopyToBufferHandle(TfLiteContext* context, - TfLiteDelegate* delegate, - TfLiteBufferHandle buffer_handle, - TfLiteTensor* tensor) { - // 若有所需，复制 tensor（张量）的数据至代理的缓冲区。 - return kTfLiteOk; -} - -TfLiteStatus CopyFromBufferHandle(TfLiteContext* context, - TfLiteDelegate* delegate, - TfLiteBufferHandle buffer_handle, - TfLiteTensor* tensor) { - // 从代理的缓冲区存入数据至 tensor 的原始内存区域。 - return kTfLiteOk; -} - -// 回调函数获取返回指针的所有权。 -TfLiteDelegate* CreateMyDelegate() { - TfLiteDelegate* delegate = new TfLiteDelegate; - - delegate->data_ = nullptr; - delegate->flags = kTfLiteDelegateFlagsNone; - delegate->Prepare = &DelegatePrepare; - // 该项不可为空。 - delegate->CopyFromBufferHandle = &CopyFromBufferHandle; - // 该项可为空。 - delegate->CopyToBufferHandle = &CopyToBufferHandle; - // 该项可为空。 - delegate->FreeBufferHandle = &FreeBufferHandle; - - return delegate; -} - -// 添加你所需调用的代理 - -auto* my_delegate = CreateMyDelegate(); -if (interpreter->ModifyGraphWithDelegate(my_delegate) != - kTfLiteOk) { - // 用于实现解决异常的方法 -} else { - interpreter->Invoke(); -} -... -// 最后千万要记住注销代理。 -delete my_delegate; -``` diff --git a/site/zh-cn/lite/performance/gpu.md b/site/zh-cn/lite/performance/gpu.md deleted file mode 100644 index 6d2130da8fe..00000000000 --- a/site/zh-cn/lite/performance/gpu.md +++ /dev/null @@ -1,186 +0,0 @@ -# TensorFlow Lite GPU 代理 - -[TensorFlow Lite](https://www.tensorflow.org/lite) 支持多种硬件加速器。本文档描述了如何在 Android 和 iOS 设备上使用 TensorFlow Lite 的代理 APIs 来预览实验性的 GPU 后端功能。 - -GPU 是设计用来完成高吞吐量的大规模并行工作的。因此，它们非常适合用在包含大量运算符的神经网络上，一些输入张量可以容易的被划分为更小的工作负载且可以同时执行，通常这会导致更低的延迟。在最佳情况下，用 GPU 在实时应用程序上做推理运算已经可以运行的足够快，而这在以前是不可能的。 - -不同于 CPU 的是，GPU 可以计算 16 位浮点数或者 32 位浮点数并且 GPU 不需要量化来获得最佳的系统性能。 - -使用 GPU 做推理运算还有一个好处就是它的能源效率。GPU 可以以非常高效和优化的方式下进行计算，所以 GPU 在完成和 CPU 一样的任务时可以消耗更少的电力和产生更少的热量。 - -## 演示应用程序教程 - -最简单的尝试实验 GPU 代理的方法就是跟着下面的教程，教程将贯串我们整个使用 GPU 构建的分类演示应用程序。GPU 代码现在只有二进制的形式，但是很快就会开源。一旦你理解了如何把我们的演示程序运行起来，你就可以在你自己的模型上尝试。 - -### Android（使用 Android Studio） - -如果需要一个分步教程, 请观看 -[适用于 Android 的实验性 GPU 代理](https://youtu.be/Xkhgre8r5G0) 的视频。 - -注意：这需要 OpenGL ES 3.1或者更高版本 - -#### 第一步 克隆 TensorFlow 的源代码并在 Android Studio 中打开 - -``` -git clone https://github.com/tensorflow/tensorflow -``` - -#### 第二步 编辑 `app/build.gradle` 文件来使用 nightly 版本的 GPU AAR - -在现有的 `dependencies` 模块已有的 `tensorflow-lite` 包的位置下添加 `tensorflow-lite-gpu` 包。 - -``` -dependencies { - ... - implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly' - implementation 'org.tensorflow:tensorflow-lite-gpu:0.0.0-nightly' -} -``` - -#### 第三步. 编译和运行 - -点击 Run 按钮来运行应用程序。当你运行应用程序的时候你会看到一个启用 GPU 的按钮。将应用程序从量化模式改为浮点模式后点击 GPU 按钮后，程序将在 GPU 上运行。 - -![运行 Android gpu 演示应用程序和切换到 GPU](images/android_gpu_demo.gif) - -### iOS (使用 XCode) - -如果需要一个分步教程, 请观看 -[适用于 iOS 的实验性 GPU 代理](https://youtu.be/Xkhgre8r5G0) 的视频。 - -注意：这需要 XCode 10.1 或者更高版本 - -#### 第一步. 获取演示应用程序的源码并确保它已被编译 - -遵照我们的 iOS 演示应用程序[教程](https://www.tensorflow.org/lite/demo_ios)。这会告诉你没有修改的iOS相机应用程序是如何在我们的手机上运行的。 - -#### 第二部. 修改 Podfile 文件来使用 TensorFlow Lite GPU CocoaPod - -我们构建了一个包含 GPU 代理的二进制 CocoaPod 文件。如果需要切换到工程并使用它，修改 -`tensorflow/tensorflow/lite/examples/ios/camera/Podfile` 文件来使用 `TensorFlowLiteGpuExperimental` 的 pod 替代 `TensorFlowLite`。 - -``` -target 'YourProjectName' - # pod 'TensorFlowLite', '1.12.0' - pod 'TensorFlowLiteGpuExperimental' -``` - -#### 第三步. 启用 GPU 代理 - -为了确保代码会使用 GPU 代理，你需要将 `CameraExampleViewController.h` 的 -`TFLITE_USE_GPU_DELEGATE` 从 0 修改为 1 。 - -```c -#define TFLITE_USE_GPU_DELEGATE 1 -``` - -#### 第四步. 编译和运行演示应用程序 - -如果你完成了上面的步骤，你应该已经可以运行这个应用程序了。 - -#### 第五步. 发布模式 - -你在第四步是在调试模式下运行的应用程序，为了获得更好的性能表现，你应该使用适当的最佳 Metal 设置将应用程序改为发布版本。特别需要注意的是，需要修改这些设置 `Product > Scheme > Edit -Scheme...`，选择 ` Run `，在 ` Info ` 一栏，修改 ` Build Configuration `，从 `Debug ` 改为 ` Release `，取消选择 ` Debug executable`。 - -![设置发布](images/iosdebug.png) - -然后点击 `Options` 栏然后将 `GPU Frame Capture` 修改成 `Disabled`，并将 `Metal API Validation` 修改成 `Disabled`。 - -![设置 metal 选项](images/iosmetal.png) - -最后需要确保发布版本只能在 64 位系统上构建。在 `Project -navigator -> tflite_camera_example -> PROJECT -> tflite_camera_example -> Build -Settings` 上将 `Build Active Architecture Only > Release`选择为 Yes。 - -![设置发布选项](images/iosrelease.png) - -## 在你自己的模型上使用GPU代理 - -### Android - -查看演示应用程序来了解如何添加代理。在你的应用程序中，像上面一样添加 AAR ，导入`org.tensorflow.lite.gpu.GpuDelegate` 模块，并使用 `addDelegate` 功能将GPU代理注册到解释器中。 - -```java -import org.tensorflow.lite.Interpreter; -import org.tensorflow.lite.gpu.GpuDelegate; - -// 初始化使用 GPU 代理的解释器 -GpuDelegate delegate = new GpuDelegate(); -Interpreter.Options options = (new Interpreter.Options()).addDelegate(delegate); -Interpreter interpreter = new Interpreter(model, options); - -// 进行推理 -while (true) { - writeToInput(input); - interpreter.run(input, output); - readFromOutput(output); -} - -// 清理 -delegate.close(); -``` - -### iOS - -在你的应用程序代码中，引入 GPU 代理头文件来让`Interpreter::ModifyGraphWithDelegate` 功能将 GPU 代理注册到解释器中。 - -```cpp -#import "tensorflow/lite/delegates/gpu/metal_delegate.h" - -// 初始化使用 GPU 代理的解释器 -std::unique_ptr interpreter; -InterpreterBuilder(*model, resolver)(&interpreter); -auto* delegate = NewGpuDelegate(nullptr); // default config -if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false; - -// 进行推理 -while (true) { - WriteToInputTensor(interpreter->typed_input_tensor(0)); - if (interpreter->Invoke() != kTfLiteOk) return false; - ReadFromOutputTensor(interpreter->typed_output_tensor(0)); -} - -// 清理 -interpreter = nullptr; -DeleteGpuDelegate(delegate); -``` - -## 支持的模型和 Ops - -在 GPU 代理发布后，我们提供了少数可以在后端运行的模型： - -* [MobileNet v1 (224x224)图像分类](https://ai.googleblog.com/2017/06/mobilenets-open-source-models-for.html) [[下载]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/mobilenet_v1_1.0_224.tflite) -
    (为移动和嵌入式视觉应用设计的图像分类模型) -* [DeepLab 分割 (257x257)](https://ai.googleblog.com/2018/03/semantic-image-segmentation-with.html) [[下载]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/deeplabv3_257_mv_gpu.tflite) -
    (将输入图像的每个像素指定语义标签（例如，狗，猫。汽车的图像分割模型) -* [MobileNet SSD 物体检测](https://ai.googleblog.com/2018/07/accelerated-training-and-inference-with.html) [[下载]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/mobile_ssd_v2_float_coco.tflite) -
    (用于检测多个带有边框的对象的图像分类模型) -* [PoseNet用于姿势估计](https://github.com/tensorflow/tfjs-models/tree/master/posenet) [[下载]](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/multi_person_mobilenet_v1_075_float.tflite) -
    (用于估计图像或视频中人物的姿势的视觉模型) - -如果需要完整的支持的 Ops 的列表，请看[进阶文档](gpu_advanced.md)。 - -## 不支持的模型和 ops - -如果一些 ops 并不支持 GPU 代理，框架只会在 GPU 上运行图形的一部分，剩下的部分会在 CPU 上运行。因为这会导致 CPU/GPU 同时出现很高的使用率，像这样的分开执行模式会导致运行起来比整个网络在 CPU 上运行要慢。在这种情况下，用户会收到一个像这样的警告： - -``` -WARNING: op code #42 cannot be handled by this delegate. -``` - -``` -警告：此代理无法处理#42操作码 -``` - -我们没有为这种失败提供回调，因为这不是真的运行错误，但是这个错误是开发者可以注意到的，他们可以尝试将整个网络在代理上运行。 - -## 优化建议 - -一些在 CPU 上的琐碎的的操作可能在 GPU 上会有很高的占用。其中的一种操作就是很多形式的 reshape 操作，像 `BATCH_TO_SPACE`, `SPACE_TO_BATCH`, `SPACE_TO_DEPTH` 等等。如果这些 ops 只是为了方便网络构架师的逻辑思考而放置在网络中，为了更好的性能将他们在网络中移除是值得的。 - -在 GPU 上，张量数据被分成4个通道。因此，计算一个 `[B,H,W,5]` 的张量和计算 `[B,H,W,8]`的效果是一样的，但是它们都比运行 `[B,H,W,4]` 的性能要差的多。 - -从这个意义上讲，如果相机硬件支持 RGBA 形式图像帧，4 通道输入明显更快因为可以避免内存复制(从 3 通道 RGB 转变到 4 通道 RGBX）。 - -为了获得最佳性能，请不要犹豫，使用移动优化的网络架构来重新训练您的分类器。这是优化设备推断性能的重要部分。 diff --git a/site/zh-cn/lite/performance/gpu_advanced.md b/site/zh-cn/lite/performance/gpu_advanced.md deleted file mode 100644 index dd5c1b18bee..00000000000 --- a/site/zh-cn/lite/performance/gpu_advanced.md +++ /dev/null @@ -1,274 +0,0 @@ -# TensorFlow Lite 在GPU环境下 - -[TensorFlow Lite](https://www.tensorflow.org/mobile/tflite/)支持多种硬件加速器。本文档介绍如何在安卓系统（要求OpenGL ES 3.1或更高版本）和iOS（要求iOS 8 或更高版本）的GPU后端（backend）使用TensorFLow Lite delegate APIs。 - -## 使用GPU加速的优势 - -### 速度 - -GPUs 设计为具有高吞吐量、可大规模并行化的工作负载（workloads）。因此，它们非常适合于一个由大量运算符组成的深度神经网络，其中每一个GPU都可以处理一些输入张量（tensor）并且容易划分为较小的工作负载（workloads），然后并行执行。这样并行性通常能够有较低的延迟。在最好的情况下，在GPU上推断（inference）可以运行得足够快，以适应实时程序，这在以前是不可能的。 - -### 精度 - -GPU使用16位或32位浮点数进行运算，并且（与CPU不同）不需要量化（quantization）以获得最佳的性能。如果精度降低使得模型的量化（quantization）无法达到要求，那么在GPU上运行神经网络可能可以消除这种担忧。 - -### 能效 - -使用GPU进行推断（inference）的另一个好处在于它的能效。GPU能以非常有效和优化方法来进行运算，比在CPU上运行相同任务消耗更少的能源并产生更少的发热量。 - -### 支持的Ops - -TensorFlow Lite 在GPU上支持16位和32位浮点精度中的以下操作： - -* `ADD v1` -* `AVERAGE_POOL_2D v1` -* `CONCATENATION v1` -* `CONV_2D v1` -* `DEPTHWISE_CONV_2D v1-2` -* `FULLY_CONNECTED v1` -* `LOGISTIC v1` -* `MAX_POOL_2D v1` -* `MUL v1` -* `PAD v1` -* `PRELU v1` -* `RELU v1` -* `RELU6 v1` -* `RESHAPE v1` -* `RESIZE_BILINEAR v1` -* `SOFTMAX v1` -* `STRIDED_SLICE v1` -* `SUB v1` -* `TRANSPOSE_CONV v1` - -## 基本用法 - -### Android (Java) - -使用`TfLiteDelegate`在GPU上运行TensorFlow Lite，在Java中，您可以通过`Interpreter.Options`来指定GpuDelegate。 - -```java -// NEW: Prepare GPU delegate. -GpuDelegate delegate = new GpuDelegate(); -Interpreter.Options options = (new Interpreter.Options()).addDelegate(delegate); - -// Set up interpreter. -Interpreter interpreter = new Interpreter(model, options); - -// Run inference. -writeToInputTensor(inputTensor); -interpreter.run(inputTensor, outputTensor); -readFromOutputTensor(outputTensor); - -// Clean up. -delegate.close(); -``` - -### Android (C/C++) - -在Android GPU上使用C/C++语言的TensorFlow Lite，可以使用`TfLiteGpuDelegateCreate()`创建，并使用`TfLiteGpuDelegateDelete()`销毁。 - -```c++ -// Set up interpreter. -auto model = FlatBufferModel::BuildFromFile(model_path); -if (!model) return false; -ops::builtin::BuiltinOpResolver op_resolver; -std::unique_ptr interpreter; -InterpreterBuilder(*model, op_resolver)(&interpreter); - -// NEW: Prepare GPU delegate. -const TfLiteGpuDelegateOptions options = { - .metadata = NULL, - .compile_options = { - .precision_loss_allowed = 1, // FP16 - .preferred_gl_object_type = TFLITE_GL_OBJECT_TYPE_FASTEST, - .dynamic_batch_enabled = 0, // Not fully functional yet - }, -}; -auto* delegate = TfLiteGpuDelegateCreate(&options); -if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false; - -// Run inference. -WriteToInputTensor(interpreter->typed_input_tensor(0)); -if (interpreter->Invoke() != kTfLiteOk) return false; -ReadFromOutputTensor(interpreter->typed_output_tensor(0)); - -// NEW: Clean up. -TfLiteGpuDelegateDelete(delegate); -``` - -适用于Android C / C ++的TFLite GPU使用[Bazel](https://bazel.io)构建系统。例如，可以使用以下命令构建委托（delegate）： - -```sh -bazel build -c opt --config android_arm64 tensorflow/lite/delegates/gpu:gl_delegate # for static library -bazel build -c opt --config android_arm64 tensorflow/lite/delegates/gpu:libtensorflowlite_gpu_gl.so # for dynamic library -``` - -### iOS(ObjC++) - -要在GPU上运行TensorFlow Lite，需要通过`NewGpuDelegate()`对GPU委托（delegate），然后将其传递给`Interpreter::ModifyGraphWithDelegate()`（而不是调用`Interpreter::AllocateTensors()`） - -```c++ -// Set up interpreter. -auto model = FlatBufferModel::BuildFromFile(model_path); -if (!model) return false; -tflite::ops::builtin::BuiltinOpResolver op_resolver; -std::unique_ptr interpreter; -InterpreterBuilder(*model, op_resolver)(&interpreter); - -// NEW: Prepare GPU delegate. - -const GpuDelegateOptions options = { - .allow_precision_loss = false, - .wait_type = kGpuDelegateOptions::WaitType::Passive, -}; - -auto* delegate = NewGpuDelegate(options); -if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false; - -// Run inference. -WriteToInputTensor(interpreter->typed_input_tensor(0)); -if (interpreter->Invoke() != kTfLiteOk) return false; -ReadFromOutputTensor(interpreter->typed_output_tensor(0)); - -// Clean up. -DeleteGpuDelegate(delegate); -``` - -注意：调用`Interpreter::ModifyGraphWithDelegate()`或`Interpreter::Invoke()`时，调用者必须在当前线程中有一个`EGLContext`，并且从同一个`EGLContext`中调用`Interpreter::Invoke()`。如果`EGLContext`不存在，委托（delegate）将在内部创建一个，但是开发人员必须确保始终从调用`Interpreter::Invoke()`的同一个线程调用`Interpreter::ModifyGraphWithDelegate()`。 - -## 高级用法 - -### 委托（Delegate）iOS 选项 - -`NewGpuDelegate()`接受一个 `struct` 选项。 - -```c++ -struct GpuDelegateOptions { - // Allows to quantify tensors, downcast values, process in float16 etc. - bool allow_precision_loss; - - enum class WaitType { - // waitUntilCompleted - kPassive, - // Minimize latency. It uses active spinning instead of mutex and consumes - // additional CPU resources. - kActive, - // Useful when the output is used with GPU pipeline then or if external - // command encoder is set - kDoNotWait, - }; - WaitType wait_type; -}; -``` - -将`nullptr`传递给`NewGpuDelegate()`，并设置默认选项（即在上面的基本用法示例中阐述）。 - -```c++ -// THIS: -const GpuDelegateOptions options = { - .allow_precision_loss = false, - .wait_type = kGpuDelegateOptions::WaitType::Passive, -}; - -auto* delegate = NewGpuDelegate(options); - -// IS THE SAME AS THIS: -auto* delegate = NewGpuDelegate(nullptr); -``` - -虽然使用`nullptr`很方便，但我们建议您指定设置选项，以避免在以后更改默认值时出现任何异常情况。 - -### 输入/输出缓冲器 - -要想在GPU上进行计算，数据必须能够让GPU可见。这通常需要进行内存复制。如果可以的话，最好不要交叉CPU / GPU内存边界，因为这会占用大量时间。通常来说，这种交叉是不可避免的，但在某些特殊情况下，可以忽略其中一个。 - -如果网络的输入是已经加载到GPU内存中的图像（例如，包含相机传输的GPU纹理），那么可以直接保留在GPU内存中而无需进入到CPU内存。同样，如果网络的输出采用可渲染图像的格式（例如， [image style transfer](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf)_)，那么它可以直接显示在屏幕上。 - -为了获得最佳性能，TensorFlow Lite让用户可以直接读取和写入TensorFlow硬件缓冲区并绕过可避免的内存副本。 - -#### Android - -假设图像送入在GPU存储器中，则必须首先将其转换为OpenGL着色器存储缓冲区对象（SSBO）。您可以使用`Interpreter.bindGlBufferToTensor()`将TfLiteTensor与用户准备的SSBO相关联。注意：`Interpreter.bindGlBufferToTensor()`必须在`Interpreter.modifyGraphWithDelegate()`之前调用。 - -```java -// Ensure a valid EGL rendering context. -EGLContext eglContext = eglGetCurrentContext(); -if (eglContext.equals(EGL_NO_CONTEXT)) return false; - -// Create an SSBO. -int[] id = new int[1]; -glGenBuffers(id.length, id, 0); -glBindBuffer(GL_SHADER_STORAGE_BUFFER, id[0]); -glBufferData(GL_SHADER_STORAGE_BUFFER, inputSize, null, GL_STREAM_COPY); -glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0); // unbind -int inputSsboId = id[0]; - -// Create interpreter. -Interpreter interpreter = new Interpreter(tfliteModel); -Tensor inputTensor = interpreter.getInputTensor(0); -GpuDelegate gpuDelegate = new GpuDelegate(); -// The buffer must be bound before the delegate is installed. -gpuDelegate.bindGlBufferToTensor(inputTensor, inputSsboId); -interpreter.modifyGraphWithDelegate(gpuDelegate); - -// Run inference; the null input argument indicates use of the bound buffer for input. -fillSsboWithCameraImageTexture(inputSsboId); -float[] outputArray = new float[outputSize]; -interpreter.runInference(null, outputArray); -``` - -类似的方法可以应用于输出张量(tensor)。在这种情况下，`Interpreter.Options.setAllowBufferHandleOutput(true)`应该被用来传递，来禁用从GPU内存到CPU内存的网络输出复制的默认操作。 - -```java -// Ensure a valid EGL rendering context. -EGLContext eglContext = eglGetCurrentContext(); -if (eglContext.equals(EGL_NO_CONTEXT)) return false; - -// Create a SSBO. -int[] id = new int[1]; -glGenBuffers(id.length, id, 0); -glBindBuffer(GL_SHADER_STORAGE_BUFFER, id[0]); -glBufferData(GL_SHADER_STORAGE_BUFFER, outputSize, null, GL_STREAM_COPY); -glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0); // unbind -int outputSsboId = id[0]; - -// Create interpreter. -Interpreter.Options options = (new Interpreter.Options()).setAllowBufferHandleOutput(true); -Interpreter interpreter = new Interpreter(tfliteModel, options); -Tensor outputTensor = interpreter.getOutputTensor(0); -GpuDelegate gpuDelegate = new GpuDelegate(); -// The buffer must be bound before the delegate is installed. -gpuDelegate.bindGlBufferToTensor(outputTensor, outputSsboId); -interpreter.modifyGraphWithDelegate(gpuDelegate); - -// Run inference; the null output argument indicates use of the bound buffer for output. -ByteBuffer input = getCameraImageByteBuffer(); -interpreter.runInference(input, null); -renderOutputSsbo(outputSsboId); -``` - -#### iOS - -假设图像送入在GPU存储器中，则必须首先将其转换为Metal的`MTLBuffer`对象。您可以将TfLiteTensor与用户准备的`MTLBuffer`和`BindMetalBufferToTensor()`相关联。注意：必须在`Interpreter::ModifyGraphWithDelegate()`之前调用`BindMetalBufferToTensor()`。此外，默认情况下，推断（inference）结果的输出，会从GPU内存复制到CPU内存。在初始化期间调用`Interpreter::SetAllowBufferHandleOutput(true)`可以关闭该操作。 - -```c++ -// Prepare GPU delegate. -auto* delegate = NewGpuDelegate(nullptr); -interpreter->SetAllowBufferHandleOutput(true); // disable default gpu->cpu copy -if (!BindMetalBufferToTensor(delegate, interpreter->inputs()[0], user_provided_input_buffer)) return false; -if (!BindMetalBufferToTensor(delegate, interpreter->outputs()[0], user_provided_output_buffer)) return false; -if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) return false; - -// Run inference. -if (interpreter->Invoke() != kTfLiteOk) return false; -``` - -注意：一旦关闭从GPU内存复制到CPU内存的操作后，将推断（inference）结果输出从GPU内存复制到CPU内存需要为每个输出张量显式调用`Interpreter::EnsureTensorDataIsReadable()`。 - -## 提示与技巧 - -* 在CPU上执行一些微不足道的操作可能需要非常高的代价，譬如各种形式的reshape操作（包括`BATCH_TO_SPACE`，`SPACE_TO_BATCH`，`SPACE_TO_DEPTH`和其他类似的操作）。如果不需要这些操作（比如使用这些操作是为了帮助理解网络架构和了解整个系统但不会影响输出），那么值得删除它们以提高性能。 -* 在GPU上，张量（tensor）数据被划分为4个通道（channel）。因此对形状为`[B, H, W, 5]` 的张量（tensor）的计算量大致与`[B, H, W, 8]`相同，但明显比`[B, H, W, 4]`要大。 - * 比如：如果相机的硬件支持RGBA，那么传输4通道（channel）数据的速度要快得多，因为可以避免内存复制（从3通道RGB到4通道RGBX）。 -* 为了获得最佳性能，请不要犹豫使用移动优化过（mobile-optimized）的网络架构重新训练您的分类器。 这是设备推断（inference）优化的重要部分。 - diff --git a/site/zh-cn/lite/performance/model_optimization.md b/site/zh-cn/lite/performance/model_optimization.md deleted file mode 100644 index adbde4d1559..00000000000 --- a/site/zh-cn/lite/performance/model_optimization.md +++ /dev/null @@ -1,57 +0,0 @@ -# 模型优化 - -Tensorflow Lite 和 [Tensorflow Model Optimization Toolkit](https://www.tensorflow.org/model_optimization) (Tensorflow模型优化工具包)提供了最小优化推理复杂性的工具。 - -对于移动和物联网 (IoT) 等边缘设备,推理效率尤其重要。这些设备在处理，内存，能耗和模型存储方面有许多限制。 -此外，模型优化解锁了定点硬件 (fixed-point hardware) 和下一代硬件加速器的处理能力。 - -## 模型量化 - -深度神经网络的量化使用了一些技术，这些技术可以降低权重的精确表示，并且可选的降低存储和计算的激活值。量化的好处有: - -* 对现有 CPU 平台的支持。 -* 激活值得的量化降低了用于读取和存储中间激活值的存储器访问成本。 -* 许多 CPU 和硬件加速器实现提供 SIMD 指令功能，这对量化特别有益。 - -TensorFlow Lite 对量化提供了多种级别的对量化支持。 - -* Tensorflow Lite [post-training quantization](post_training_quantization.md) 量化使权重和激活值的 Post training 更简单。 -* [Quantization-aware training](https://github.com/tensorflow/tensorflow/tree/r1.13/tensorflow/contrib/quantize){:.external} 可以以最小精度下降来训练网络；这仅适用于卷积神经网络的一个子集。 - -### 延时和准确性结果 - -以下是一些模型经过 post-training quantization 和 quantization-aware training 后的延迟和准确性结果。所有延迟数都是在使用单个大内核的 Pixel 2 设备上测量的。随着工具包的改进，这些数字也会随之提高: - -
    - - - - - - - - - - - - - - - - - - - -
    模型Top-1 精确性(初始) Top-1 精确性(Post Training量化) Top-1 精确性 (Quantization Aware Training) 延迟 (初始) (ms) 延迟 (Post Training量化) (ms) 延迟 (Quantization Aware) (ms) 大小 (初始) (MB) 大小 (优化后) (MB)
    Mobilenet-v1-1-2240.7090.6570.701241126416.94.3
    Mobilenet-v2-1-2240.7190.6370.709899854143.6
    Inception_v30.780.7720.775113084554395.723.9
    Resnet_v2_1010.7700.768N/A39732868N/A178.344.9
    -
    - Table 1 模型量化对选择CNN模型的好处 -
    -
    - -## 工具选择 - -首先，检查 [hosted models](../guide/hosted_models.md) 中的模型是否适合您的应用程序。如果没有，我们建议用户从 [post-training quantization tool](post_training_quantization.md) 开始，因为它广泛适用的，且无需训练数据。 - -对于精度和延迟目标没有达到，或者需要硬件加速器支持情况， [quantization-aware training](https://github.com/tensorflow/tensorflow/tree/r1.13/tensorflow/contrib/quantize) {:.external} 是更好的选择。参见 Tensorflow 模型优化工具包[Tensorflow Model Optimization Toolkit](https://www.tensorflow.org/model_optimization) 中的的其他优化技术。 - -注意: Quantization-aware training 支持卷积神经网络体系结构的子集。 diff --git a/site/zh-cn/lite/r1/convert/cmdline_examples.md b/site/zh-cn/lite/r1/convert/cmdline_examples.md deleted file mode 100644 index ed8a286729d..00000000000 --- a/site/zh-cn/lite/r1/convert/cmdline_examples.md +++ /dev/null @@ -1,303 +0,0 @@ -# 转换器的命令行实例 - -这个页面展示如何在命令行中使用 TensorFlow Lite 转换器 - -[TOC] - -## 在命令行中使用的命令 - -以下是在命令行中使用转换器的两种方法： - -* `tflite_convert`: 从 TensorFlow 1.9 起开始支持 - `tflite_convert` 作为 Python 包的一部分被安装。简便起见，以下所有示例使用 `tflite_convert` 指代。 - * 示例: `tflite_convert --output_file=...` -* `bazel`: 为了使用最新版本的 TensorFlow Lite Converter，你可以使用 - [pip](https://www.tensorflow.org/install/pip) 或[克隆 TensorFlow 仓库](https://www.tensorflow.org/install/source) 来安装并使用 nightly 版本的的 `bazel`。 - * 示例: `bazel run //tensorflow/lite/python:tflite_convert ----output_file=...` - -### 在低于 1.9 版本的 TensorFlow 中转换模型 - -如果你安装有低于 1.9 版本的 Tensorflow，并想转换模型，我们推荐你使用 -[Python API](python_api.md#pre_tensorflow_1.9)。 如果你想要使用命令行转换模型, 在 Tensorflow 1.7 中，你可使用 toco。 - -你可以通过在终端中键入`toco —help`来获取更多关于命令行参数的细节信息。 - -在 TensorFlow 1.8 中没有可用的命令行工具。 - -## 基础示例 - -以下部分向你展示怎样将各种数据从支持的类型转换到 TensorFlow Lite FlatBuffers。 - -### 转换 TensorFlow GraphDef - -以下部分向你展示如何将基本的 TensorFlow GraphDef (使用 [freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py) 冻结)转换为 TensorFlow Lite FlatBuffer 来进行浮点数推理。被冻结的图包含存储在检查点文件中的变量，这些变量被作为 Const ops 保存。 - -``` -curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \ - | tar xzv -C /tmp -tflite_convert \ - --output_file=/tmp/foo.tflite \ - --graph_def_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \ - --input_arrays=input \ - --output_arrays=MobilenetV1/Predictions/Reshape_1 -``` - -`input_shapes` 的取值在可能时被自动确定。 - -### 转换 TensorFlow SavedModel - -以下部分向你展示如何将基本的 TensorFlow SavedModel 转换为 Tensorflow Lite FlatBuffer 来进行浮点数推理。 - -``` -tflite_convert \ - --output_file=/tmp/foo.tflite \ - --saved_model_dir=/tmp/saved_model -``` - -[SavedModel](https://www.tensorflow.org/guide/saved_model#using_savedmodel_with_estimators) -与冻结后的图比较，它需要更少的参数，这是由于保存在 SavedModel 中的附加数据所致。 - `--input_arrays`和 `--output_arrays` 所需要的值是 [MetaGraphDef](https://www.tensorflow.org/saved_model#apis_to_build_and_load_a_savedmodel) 中 [SignatureDefs](../../serving/signature_defs.md) 里的一个聚合起来的，按照字母顺序排列的输入输出列表，它由`—saved_model_tag_set`指定。 - 和 GraphDef 一样, `input_shapes` 的值也在可能时被自动定义。 - -现阶段暂不提供对不带 SignatureDef 的 MetaGraphDefs 或是 -使用[`assets/`directory](https://www.tensorflow.org/guide/saved_model#structure_of_a_savedmodel_directory) 的 MetaGraphDefs 的支持。 - -### 转换 tf.Keras 模型 - -以下部分展示如何将一个 `tf.keras` 模型转换为一个 TensorFlow Lite Flatbuffer。 - - `tf.keras` 文件必须同时包含模型和权重。 - -``` -tflite_convert \ - --output_file=/tmp/foo.tflite \ - --keras_model_file=/tmp/keras_model.h5 -``` - -## 量化 - -### 将一个TensorFlow GraphDef 转换为量化的推理 - -TensorFlow Lite Converter 兼容定点量化模型，详情见[这里](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/quantize/README.md)。 -浮点模型中有 `FakeQuant*` ops ，它们被插入在混合层的边界来记录最大最小值的范围信息。 - -这产生一个量化的推理工作流，它复现了训练期间被使用的量化行为。 - -下列命令从"量化的" TensorFlow GraphDef 中产生量化的 TensorFlow Lite FlatBuffer。 - - -``` -tflite_convert \ - --output_file=/tmp/foo.tflite \ - --graph_def_file=/tmp/some_quantized_graph.pb \ - --inference_type=QUANTIZED_UINT8 \ - --input_arrays=input \ - --output_arrays=MobilenetV1/Predictions/Reshape_1 \ - --mean_values=128 \ - --std_dev_values=127 -``` - -### 使用 "dummy-quantization\" 在浮点数图上进行量化推理 - -为了评估生成量化图的可能的好处，转换器允许在浮点图上进行 "dummy-quantization"。参数 -`--default_ranges_min` 和 `--default_ranges_max` 在所有不含有最大最小值信息的 array 中指定最大最小值范围。"Dummy-quantization" 的精度低一些，但也近似于一个精确量化模型。 - -下方的例子展示了一个带有 Relu6 激活函数的模型。由此，我们可以得出一个合理的猜测，大部分的激活函数的范围应该在[0, 6]。 - -``` -curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \ - | tar xzv -C /tmp -tflite_convert \ - --output_file=/tmp/foo.cc \ - --graph_def_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \ - --inference_type=QUANTIZED_UINT8 \ - --input_arrays=input \ - --output_arrays=MobilenetV1/Predictions/Reshape_1 \ - --default_ranges_min=0 \ - --default_ranges_max=6 \ - --mean_values=128 \ - --std_dev_values=127 -``` - -## 确定输入和输出的数组 - -### 多输入数组 - -如下方的例子所示，参数 `input_arrays` 接受一个用逗号分隔的列表作为输入数组。 - -这对于有多输入的模型或子图来说是很有用的。 - -``` -curl https://storage.googleapis.com/download.tensorflow.org/models/inception_v1_2016_08_28_frozen.pb.tar.gz \ - | tar xzv -C /tmp -tflite_convert \ - --graph_def_file=/tmp/inception_v1_2016_08_28_frozen.pb \ - --output_file=/tmp/foo.tflite \ - --input_shapes=1,28,28,96:1,28,28,16:1,28,28,192:1,28,28,64 \ - --input_arrays=InceptionV1/InceptionV1/Mixed_3b/Branch_1/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_2/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_3/MaxPool_0a_3x3/MaxPool,InceptionV1/InceptionV1/Mixed_3b/Branch_0/Conv2d_0a_1x1/Relu \ - --output_arrays=InceptionV1/Logits/Predictions/Reshape_1 -``` - -需要注意的是， `input_shapes` 是用冒号分割的列表。其中， 每个输入形状对应于各自数组中相同位置的输入数组。 - -### 多输出数组 - -如下方的例子所示，参数 `output_arrays` 接收一个用逗号分隔的列表作为输出数组。 - -这对于有多输出的模型或子图来说是很有用的。 - -``` -curl https://storage.googleapis.com/download.tensorflow.org/models/inception_v1_2016_08_28_frozen.pb.tar.gz \ - | tar xzv -C /tmp -tflite_convert \ - --graph_def_file=/tmp/inception_v1_2016_08_28_frozen.pb \ - --output_file=/tmp/foo.tflite \ - --input_arrays=input \ - --output_arrays=InceptionV1/InceptionV1/Mixed_3b/Branch_1/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_2/Conv2d_0a_1x1/Relu -``` - -### 指定子图 - -输入文件中的任何数组都可以被指定为输入或输出数组，以便从输入的图文件中提取子图。TensorFlow Lite -Converter 忽略指定子图范围之外的该图的其他部分。 可使用 [graph visualizations](#graph_visualizations) 来识别组成所需子图的输入和输出数组。 - -下列命令展示怎样从一个 TensorFlow GraphDef 中提取单个混合层。 - -``` -curl https://storage.googleapis.com/download.tensorflow.org/models/inception_v1_2016_08_28_frozen.pb.tar.gz \ - | tar xzv -C /tmp -tflite_convert \ - --graph_def_file=/tmp/inception_v1_2016_08_28_frozen.pb \ - --output_file=/tmp/foo.pb \ - --input_shapes=1,28,28,96:1,28,28,16:1,28,28,192:1,28,28,64 \ - --input_arrays=InceptionV1/InceptionV1/Mixed_3b/Branch_1/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_2/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_3/MaxPool_0a_3x3/MaxPool,InceptionV1/InceptionV1/Mixed_3b/Branch_0/Conv2d_0a_1x1/Relu \ - --output_arrays=InceptionV1/InceptionV1/Mixed_3b/concat_v2 -``` - -注意，TensorFlow Lite FlatBuffers 中的最终表示的粒度往往比 TensorFlow GraphDef 非常细的表示粒度更粗。例如，虽然在 TensorFlow GraphDef 中，一个全连接层通常被表示为至少四个单独的 op (变形，矩阵乘法，偏置项目加，Relu…)，但在转换器的最优表示和最终设备上的表示中，它通常被表示为单个“混合的op”。 - -由于粒度变粗，一些中间的数组 (例如 TensorFlow GraphDef 中矩阵乘和偏置项加之间的数组)将被丢弃。 - -当使用`--input_arrays` 和 `--output_arrays`指定中间数组时，推荐（有时是必须）指定在混合后生成的最终形式的图中保留的数组。它们通常是激活函数的输出（因为在每一层中，所有在激活函数前出现的部分都倾向于被混合）。 - -## 记录日志 - - -## 可视化图 - -转换器可将图输出为 Graphviz Dot 格式，可使用`--output_format` 参数或是 -`--dump_graphviz_dir`参数轻松地进行可视化。下面的小节概述了多个用例。 - -### 使用 `--output_format=GRAPHVIZ_DOT` - -第一种渲染 Graphviz 的方式是将 GRAPHVIZ_DOT` 参数传入 -`、`—output_format`。这将生成可视化图。此操作降低了在 TensorFlow GraphDef 和 TensorFlow Lite FlatBuffer 间转换的要求。当到 TFLite 的转换失败时，此操作是很有用的。 - -``` -curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \ - | tar xzv -C /tmp -tflite_convert \ - --graph_def_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \ - --output_file=/tmp/foo.dot \ - --output_format=GRAPHVIZ_DOT \ - --input_shape=1,128,128,3 \ - --input_arrays=input \ - --output_arrays=MobilenetV1/Predictions/Reshape_1 -``` - -生成的`.dot` 文件可以使用以下命令渲染为pdf文件： - -``` -dot -Tpdf -O /tmp/foo.dot -``` - -生成的 `.dot.pdf` 文件可以在任何 PDF 阅读器上查看，但我们建议使用一个能够在大页面上缩放自如的查看工具，例如 Google Chrome ： - -``` -google-chrome /tmp/foo.dot.pdf -``` - -可在下一节中在线查看示例的 PDF。 - -### 使用 `--dump_graphviz_dir` - -第二种渲染 Graphviz 的办法是传入 `—dump_graphviz_dir`参数，并指定保存渲染结果文件的目标目录。 - -和前一个方法不同的是，此方法保留了原始输出格式。它提供了由特定图生成实际图的可视化的转换过程。 - -``` -curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \ - | tar xzv -C /tmp -tflite_convert \ - --graph_def_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \ - --output_file=/tmp/foo.tflite \ - --input_arrays=input \ - --output_arrays=MobilenetV1/Predictions/Reshape_1 \ - --dump_graphviz_dir=/tmp -``` - -此操作将在目标文件夹中生成一些文件。 其中，两个最重要的文件是 `toco_AT_IMPORT.dot` 和`/tmp/toco_AFTER_TRANSFORMATIONS.dot`。 -`toco_AT_IMPORT.dot` 文件只包含转换的原始图，此操作在载入时就被完成。由于每个节点的信息有限， - -这导致生成的可视化结果不好理解。此操作在转换命令失败时十分有用。 - -`toco_AFTER_TRANSFORMATIONS.dot` 含有模型在被输出之前，且在进行了所有的转换之后的信息。 - -通常，这个图文件比较小，且包含每个节点更多的细节信息。 - -和之前一样，这些文件可以被渲染为PDF文件： - -``` -dot -Tpdf -O /tmp/toco_*.dot -``` - -示例输出文件如下所示。需要注意的是，它们展示的都是图片右上角的同一个 -`AveragePool` 节点。 - - - - - - -
    - - - - - - - -
    beforeafter
    - -### 像“拍视频”一样记录日志 - -当使用 `--dump_graphviz_dir` 命令时，通常会再传入一个 -`—dump_graphviz_video`命令。这个命令使得每次图转换后，都会保存一个图可视化“快照”。这可能导致需要存储非常多的图可视化文件。 -通常，人们通过查看这些文件来了解图的变化过程。 - -### 图形可视化的图例 - -* “操作”为红色方块: - * 大部分的操作看起来像是这样 - bright - red。 - * 一些重量级操作 (比如卷积)看起来像是这样 - darker - red。 -* 数组看起来像是这样: - * 常量数组 - blue。 - * 激活数组 - * 内部 (中间) 激活数组 - light - gray。 - * 被指定为 `--input_arrays` 或`--output_arrays` 的激活数组 - dark - gray。 - * RNN 的状态数组是绿色的。 由于转换器显式地表示RNN的回边，每个RNN 状态被表示为两个绿色数组: - * 作为RNN回边输入的激活数组 (例如，当它的内容在被计算后复制到RNN的状态数组），此时它看起来像是这样 - light - green。 - * 实际的 RNN 状态数组看起来像这样 - dark - green。它是RNN回边更新的目标。 \ No newline at end of file diff --git a/site/zh-cn/lite/r1/convert/cmdline_reference.md b/site/zh-cn/lite/r1/convert/cmdline_reference.md deleted file mode 100644 index 37d6ea24e7a..00000000000 --- a/site/zh-cn/lite/r1/convert/cmdline_reference.md +++ /dev/null @@ -1,88 +0,0 @@ -# 转换器命令行参考 - -本页为 TensorFlow 1.9 至 TensorFlow 最新版本中 TensorFlow Lite 转换器命令行使用的命令行参数提供全面参考。 - -## 高级命令行参数 - -下列高级命令行参数指定输入文件和输出文件的细节。命令行参数 `--output_file` 总是需要指定。此外，`--graph_def_file`，`--saved_model_dir` 和 `--keras_model_file` 至少需要指定一个。 - -* `--output_file`。类型：字符串。指定输出文件的全路径。 - -* `--graph_def_file`。类型：字符串。指定输入 GraphDef 文件（使用 [freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py)冻结）的全路径。 - -* `--saved_model_dir`。类型：字符串。指定包含 SavedModel 的目录的全路径。 - -* `--keras_model_file`。类型：字符串。指定包含 tf.keras 模型的 HDF5 文件的全路径。 - -* `--output_format`。类型：字符串。缺省值：`TFLITE`。指定输出文件的格式。允许下列值： - - * `TFLITE`：TensorFlow Lite FlatBuffer 格式。 - * `GRAPHVIZ_DOT`：GraphViz `.dot` 格式包含图变换后生成一个图可视化。 - -* 请注意，将 `--output_format` 设为 `GRAPHVIZ_DOT` 会对 TFLite 特定变换造成损失。因此，所得的可视化可能无法反映最终的图变换。如果想获得反映所有图变换的最终可视化，请使用 `--dump_graphviz_dir`。 - -以下命令行参数指定使用 SavedModels 时的可选函数参数。 - -* `--saved_model_tag_set`。类型：字符串。缺省值： [kSavedModelTagServe](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/tag_constants.h)。指定一组以逗号分隔的标签，用于识别要分析的 SavedModel 内的 MetaGraphDef。标签组中的所有标签都必须指定。 - -* `--saved_model_signature_key`：类型: 字符串。缺省值：`tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY`。指定识别包含输入和输出的 SignatureDef 的键。 - -## 模型命令行参数 - -模型命令行参数提供有关储存在输入文件中的模型的额外信息。 - -* `--input_arrays`。类型：以逗号分隔的字符串列表。指定一个包含输入激活张量名称的列表。 - -* `--output_arrays`。类型：以逗号分隔的字符串列表。 指定一个包含输出激活张量名称的列表。 - -以下命令行参数定义输入张量的属性。命令行参数 `--input_arrays` 中的每一项应该根据索引对应以下命令行参数中的每一项。 - -* `--input_shapes`。类型：以冒号分隔的列表，列表由以逗号分隔的整数组成的子列表组成。每个子列表指定一个输入数组的形状，形状的格式参见 [TensorFlow 惯例](https://www.tensorflow.org/guide/tensor#shape)。 - -* 例： `--input_shapes=1,60,80,3` 对于一个典型的视觉模型，表示批量大小为 1 ， 输入图像的高为 60 ， 输入图像的宽为 80 ， 输入图像的深为 3 （代表红绿蓝通道）。 - -* 例： `--input_arrays=foo,bar --input_shapes=2,3:4,5,6` 表示 "foo" 的形状为 [2, 3]， "bar" 的形状为 [4, 5, 6]。 - -* `--std_dev_values`， `--mean_values`。 类型：以逗号分隔的浮点列表。它们指定当输入数组量化时，输入数组的量化（或反量化）函数参数。只有当 `inference_input_type` 被指定为 `QUANTIZED_UINT8` 时，才需要设定它们。 - - * `mean_values` 与 `std_dev_values` 的意义如下：量化输入数组中的每个量化值将根据如下公式被解读为一个数学实数（即一个输入激活值）： - - * `real_value = (quantized_input_value - mean_value) / std_dev_value`。 - -* 当对一个量化输入进行浮点推断 （`--inference_type=FLOAT`） 时，在进行浮点推断之前，推断代码将立即根据上述公式对量化输入进行反量化。 - -* 当进行量化推断 （`--inference_type=QUANTIZED_UINT8`） 时，推断代码不会进行反量化。然而，所有数组的量化函数参数，包括输入数组通过 `mean_value` 和 `std_dev_value` 指定的量化函数参数，决定了量化推断代码中使用的不动点乘数。`mean_value` 在进行量化推断时必须是整数。 - -## 变换命令行参数 - -变换命令行参数指定应用在图上的可选变换，即它们指定输出文件应具有哪些属性。 - -* `--inference_type`。类型：字符串。缺省值：`FLOAT`。输出文件中所有实数数组的数据类型，输入数组 （用 `--inference_input_type` 指定）除外。必须是 `{FLOAT, QUANTIZED_UINT8}`。 - - 这个命令行参数只影响实数数组，包括浮点数组和量化数组。这不包括其他所有数据类型，包括通常整数（plain integer）数组和字符串数组。具体如下： - - * 如果指定为 `FLOAT`，那么输出文件中的实数数组将是浮点型。如果它们在输入文件中被量化，则它们将被反量化。 - - * 如果指定为 `QUANTIZED_UINT8`，那么输出文件中的实数数组将被量化为 uint8。如果它们在输入文件中是浮点型，则它们将被量化。 - -* `--inference_input_type`。类型：字符串。输出文件中的一个实数输入数组的数据类型。所有输入数组的数据类型的缺省值是与 `--inference_type`的指定相同。这个命令行参数的主要目的是生成一个具有量化输入数组的浮点图。在输入数组之后紧接着添加一个反量化算子。必须是 `{FLOAT, QUANTIZED_UINT8}`。 - - 这个命令行参数主要用于这样的视觉模型：输入是位图，但是要求浮点推断。对于这样的图像模型，其 uint8 输入将被量化，并且这样的输入数组使用的量化函数参数是它们的 `mean_value` 和 `std_dev_value` 函数参数。 - -* `--default_ranges_min`， `--default_ranges_max`。类型：浮点型。指定缺省（最小，最大）区间值，用于所有没有指定区间的数组。允许用户对未量化的输入文件或者错误量化的输入文件进行量化。这些命令行参数导致模型准确率降低。它们的目的在于通过“虚拟量化”来简单试验一下量化。 - -* `--drop_control_dependency`。类型：布尔型。 缺省值：True。指定是否静默丢弃控制依赖。这是由于 TensorFlow Lite 不支持控制依赖。 - -* `--reorder_across_fake_quant`。类型：布尔型。 缺省值：False。指定是否对预料之外的位置上的 FakeQuant 节点进行重新排序。用于 FakeQuant 节点的位置阻碍图变换，以至于影响转换图的情况。它会导致生成的图与量化训练图不同，有可能会造成不同的算术行为。 - -* `--allow_custom_ops`。类型：字符串。 缺省值：False。指定是否允许自定义操作。当设定为 false 时，所有未知操作都会报错。当定义为 true 时，所有未知操作会生成自定义操作。开发者需通过在 TensorFlow Lite runtime 配置自定义解析器来提供这些信息。 - -* `--post_training_quantize`。类型：布尔型。 缺省值：False。指定是否量化被转换的浮点模型的权重。模型将变小，延迟将改善（以准确率降低为代价）。 - -## 日志命令行参数 - -下列命令行参数在图变换过程中的多个时间点生成 [GraphViz](https://www.graphviz.org/) `.dot` 文件的图可视化。 - -- `--dump_graphviz_dir`。类型：字符串。指定 GraphViz `.dot` 文件输出到的目录的全路径。在读入图之后，以及所有变换完成之后，会输出图。 - -- `--dump_graphviz_video`。类型：布尔型。指定是否在每次图变换之后输出 GraphViz 文件。它要求 `--dump_graphviz_dir` 有指定值。 diff --git a/site/zh-cn/lite/r1/convert/index.md b/site/zh-cn/lite/r1/convert/index.md deleted file mode 100644 index 2570a781cd3..00000000000 --- a/site/zh-cn/lite/r1/convert/index.md +++ /dev/null @@ -1,25 +0,0 @@ -# TensorFlow Lite converter -TensorFlow Lite converter是用于将TensorFlow模型转化为优化过的[FlatBuffer](https://google.github.io/flatbuffers/)格式，以便让TensorFlow Lite解释器调用。 - -注意：此页面包含TensorFlow 1.x的converter API文档，[TensorFlow 2.0的API请点击此链接](https://www.tensorflow.org/lite/convert/) - -## FlatBuffers -FlatBuffers是一个高效的开源跨平台序列化库。它类似于[protocol buffers](https://developers.google.com/protocol-buffers)，区别在于FlatBuffers在访问数据之前不需要对其次要表达进行解析/解压，从而避免对每个对象进行内存分配。FlatBuffers的代码占用空间比protocol buffers小一个数量级。 - -## 从模型培训到设备部署 -TensorFlow Lite converter可以从TensorFlow模型中生成TensorFlow Lite [FlatBuffers](https://google.github.io/flatbuffers/)文件(.tflite)。 - -converter支持以下输入格式： -- [SavedModels](https://www.tensorflow.org/guide/saved_model#using_savedmodel_with_estimators) -- 变量固定为常数(Frozen)的`GraphDef`:由[freeze_graph.py](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py)生成的模型 -- `tf.keras` HDF5模型 -- 任何从 `tf.Session`获取的模型（仅限Python API） - -然后，将TensorFlow Lite FlatBuffer文件部署到客户端设备，TensorFlow Lite 解释器会使用压缩模型在设备上进行推断(inference)。该会话过程如下图所示： -![TFLite converter workflow](https://github.com/tensorflow/tensorflow/raw/master/tensorflow/lite/g3doc/images/convert/workflow.svg?sanitize=true) - -## 选项 - -TensorFlow Lite Converter 可以通过以下两种方式使用： -- [Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/convert/python_api.md)（**首选方式**）：使用Python API可以更轻松地将模型转换为模型开发流(model development pipeline)的一部分，并有助于在早期开发过程中缓解[兼容性](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/tf_ops_compatibility.md)问题 -- [命令行](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/convert/cmdline_examples.md) diff --git a/site/zh-cn/lite/r1/convert/python_api.md b/site/zh-cn/lite/r1/convert/python_api.md deleted file mode 100644 index a652eed3078..00000000000 --- a/site/zh-cn/lite/r1/convert/python_api.md +++ /dev/null @@ -1,230 +0,0 @@ -# Python API 指南 - -本页提供了一些示例来说明如何通过 Python API 调用 TensorFlow Lite 转换器，以及解释器。 - -注意 : 本文介绍的是 Tensorflow nightly 版本的转换器， 运行 `pip install tf-nightly` 安装此版本。 -旧版文档请参考[“转换 TensorFlow 1.12 及之前版本的模型”](#pre_tensorflow_1.12)。 - -[TOC] - - -## 概述 - -虽然也可以在命令行中调用 TensorFlow Lite 转换器，但用 Python 脚本调用 API 的方式可以作为模型开发流水线 (model development pipeline) 的一环，通常会更加便捷；可以让你更早的了解正在设计的模型是否针对移动设备 - -## API - -`tf.lite.TFLiteConverter`：用于将 TensorFlow 模型转换为 TensorFlow Lite 的 API。 -`tf.lite.Interpreter`：用于调用 Python 解释器的 API。 - -针对不同的模型原始格式，`TFLiteConverter` 提供了多种用于转换的类方法。 -`TFLiteConverter.from_session()` 用于 GraphDefs。 -`TFLiteConverter.from_saved_model()` 用于 SavedModels。 -`TFLiteConverter.from_keras_model_file()` 用于 `tf.Keras` 文件。 -[基本示例](#basic) 展示简单浮点模型的用法。[复杂示例](#complex) 展示更复杂的模型用法。 - -## 基本示例 - -以下部分显示了如何把基本浮点模型从各种原始数据格式转换成 TensorFlow Lite FlatBuffers。 - -### 使用 tf.Session 导出 GraphDef - -以下示例展示了如何从 `tf.Session` 对象转换一个 TensorFlow GraphDef 成 TensorFlow Lite FlatBuffer。 - -```python -import tensorflow as tf - -img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3)) -var = tf.get_variable("weights", dtype=tf.float32, shape=(1, 64, 64, 3)) -val = img + var -out = tf.identity(val, name="out") - -with tf.Session() as sess: - sess.run(tf.global_variables_initializer()) - converter = tf.lite.TFLiteConverter.from_session(sess, [img], [out]) - tflite_model = converter.convert() - open("converted_model.tflite", "wb").write(tflite_model) -``` - -### 使用文件导出 GraphDef - -以下示例展示了当 GraphDef 被存成文件时，是怎样转换一个 TensorFlow GraphDef 到 TensorFlow Lite FlatBuffer。支持文件后缀为 .pb 和 .pbtxt。 - -示例中用到的文件下载包：[Mobilenet_1.0_224](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)。 -该函数只支持用 [freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py) 冻结的 GraphDef。 - -```python -import tensorflow as tf - -graph_def_file = "/path/to/Downloads/mobilenet_v1_1.0_224/frozen_graph.pb" -input_arrays = ["input"] -output_arrays = ["MobilenetV1/Predictions/Softmax"] - -converter = tf.lite.TFLiteConverter.from_frozen_graph( - graph_def_file, input_arrays, output_arrays) -tflite_model = converter.convert() -open("converted_model.tflite", "wb").write(tflite_model) -``` - -### 导出 SavedModel - -以下示例展示了如何将 SavedModel 转换成 TensorFlow Lite FlatBuffer。 - -```python -import tensorflow as tf - -converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir) -tflite_model = converter.convert() -open("converted_model.tflite", "wb").write(tflite_model) -``` - -对于更复杂的 SavedModel, 可以给 `TFLiteConverter.from_saved_model()` 函数传递可选参数： -`input_arrays`，`input_shapes`，`output_arrays`，`tag_set`，`signature_key`。 -运行 `help(tf.lite.TFLiteConverter)` 查看参数详情。 - -### 导出 tf.keras 文件 - -以下示例展示如何将 `tf.keras` 模型转换成 TensorFlow Lite FlatBuffer。示例需要先安装[`h5py`](http://docs.h5py.org/en/latest/build.html) - -```python -import tensorflow as tf - -converter = tf.lite.TFLiteConverter.from_keras_model_file("keras_model.h5") -tflite_model = converter.convert() -open("converted_model.tflite", "wb").write(tflite_model) -``` - -`tf.keras` 文件必须包含模型和权重。一个全面的包括模型构造在内的示例如下所示： - -```python -import numpy as np -import tensorflow as tf - -# Generate tf.keras model. -model = tf.keras.models.Sequential() -model.add(tf.keras.layers.Dense(2, input_shape=(3,))) -model.add(tf.keras.layers.RepeatVector(3)) -model.add(tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(3))) -model.compile(loss=tf.keras.losses.MSE, - optimizer=tf.keras.optimizers.RMSprop(lr=0.0001), - metrics=[tf.keras.metrics.categorical_accuracy], - sample_weight_mode='temporal') - -x = np.random.random((1, 3)) -y = np.random.random((1, 3, 3)) -model.train_on_batch(x, y) -model.predict(x) - -# Save tf.keras model in HDF5 format. -keras_file = "keras_model.h5" -tf.keras.models.save_model(model, keras_file) - -# Convert to TensorFlow Lite model. -converter = tf.lite.TFLiteConverter.from_keras_model_file(keras_file) -tflite_model = converter.convert() -open("converted_model.tflite", "wb").write(tflite_model) -``` - -## 复杂示例 - -对于属性默认值不足的模型，在调用 `convert()` 之前应该设置属性值。 -例如设置任何常量都需要使用 `tf.lite.constants.`，以下示例中使用了常量 `QUANTIZED_UINT8`。 -您可以在 Python 终端中运行 `help(tf.lite.TFLiteConverter)` 获取有关属性的详细文档。 - -尽管示例中只演示了包含常量的 GraphDefs，同样的逻辑可以应用于每一种输入数据格式。 - -### 导出量化 GraphDef - -以下示例展示了如何把量化模型转换成 TensorFlow Lite FlatBuffer。 - -```python -import tensorflow as tf - -img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3)) -const = tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.]) -val = img + const -out = tf.fake_quant_with_min_max_args(val, min=0., max=1., name="output") - -with tf.Session() as sess: - converter = tf.lite.TFLiteConverter.from_session(sess, [img], [out]) - converter.inference_type = tf.lite.constants.QUANTIZED_UINT8 - input_arrays = converter.get_input_arrays() - converter.quantized_input_stats = {input_arrays[0] : (0., 1.)} # mean, std_dev - tflite_model = converter.convert() - open("converted_model.tflite", "wb").write(tflite_model) -``` - -## TensorFlow Lite Python 解释器 - -### 从模型文件调用解释器 - -以下示例展示了获得 TensorFlow Lite FlatBuffer 文件后，如何使用 TensorFlow Lite Python 解释器。 -此代码还演示了如何对随机输入数据进行推理。您可以在 Python 终端中运行 `help(tf.lite.Interpreter)` 获取解释器的详细文档。 - -```python -import numpy as np -import tensorflow as tf - -# Load TFLite model and allocate tensors. -interpreter = tf.lite.Interpreter(model_path="converted_model.tflite") -interpreter.allocate_tensors() - -# Get input and output tensors. -input_details = interpreter.get_input_details() -output_details = interpreter.get_output_details() - -# Test model on random input data. -input_shape = input_details[0]['shape'] -input_data = np.array(np.random.random_sample(input_shape), dtype=np.float32) -interpreter.set_tensor(input_details[0]['index'], input_data) - -interpreter.invoke() - -# The function `get_tensor()` returns a copy of the tensor data. -# Use `tensor()` in order to get a pointer to the tensor. -output_data = interpreter.get_tensor(output_details[0]['index']) -print(output_data) -``` - -### 从模型数据调用解释器 - -以下示例展示了如何从之前加载好的 TensorFlow Lite Flatbuffer 模型，调用 TensorFlow Lite Python 解释器。 -此代码显示了一个从构建 TensorFlow 模型开始的端到端案例。 - -```python -import numpy as np -import tensorflow as tf - -img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3)) -const = tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.]) -val = img + const -out = tf.identity(val, name="out") - -with tf.Session() as sess: - converter = tf.lite.TFLiteConverter.from_session(sess, [img], [out]) - tflite_model = converter.convert() - -# Load TFLite model and allocate tensors. -interpreter = tf.lite.Interpreter(model_content=tflite_model) -interpreter.allocate_tensors() -``` - -## 附加说明 - -### 源码构建 - -为了运行最新版本的 TensorFlow Lite Converter Python API，您可以选择一种方式安装 nightly 版本： -[pip](https://www.tensorflow.org/install/pip)（推荐）， -[Docker](https://www.tensorflow.org/install/docker)， -[从源代码构建 pip 包](https://www.tensorflow.org/install/source)。 - -### 转换 TensorFlow 1.12 及之前版本的模型 - -参考下表在 TensorFlow 1.12 之前的版本中转换 TensorFlow 模型到 TensorFlow Lite -运行 `help()` 获取每种 API 的详情。 - -TensorFlow 版本 | Python API ------------------- | --------------------------------- -1.12 | `tf.contrib.lite.TFLiteConverter` -1.9-1.11 | `tf.contrib.lite.TocoConverter` -1.7-1.8 | `tf.contrib.lite.toco_convert` \ No newline at end of file diff --git a/site/zh-cn/r1/README.md b/site/zh-cn/r1/README.md deleted file mode 100644 index 2091eb4af9b..00000000000 --- a/site/zh-cn/r1/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# TensorFlow 1.x - -This archive of the TensorFlow 1.x docs is in *maintenance mode* only. - -For docs contributors, please update the source files in `site/en/` and read the -[TensorFlow docs contributor guide](https://www.tensorflow.org/community/contribute/docs). - -For community translations, read the instructions in `site/zh-cn/README.md`. diff --git a/site/zh-cn/r1/tutorials/keras/basic_classification.ipynb b/site/zh-cn/r1/tutorials/keras/basic_classification.ipynb deleted file mode 100644 index 77b3d81e7a5..00000000000 --- a/site/zh-cn/r1/tutorials/keras/basic_classification.ipynb +++ /dev/null @@ -1,991 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "basic_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "MhoQ0WE77laV", - "colab_type": "text" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "_ckMIh7O7s6D", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "vasWnqRgy1H4", - "colab_type": "code", - "cellView": "form", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jYysdyb-CaWM", - "colab_type": "text" - }, - "source": [ - "# 训练您的第一个神经网络: 基本分类" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "S5Uhzt6vVIB2", - "colab_type": "text" - }, - "source": [ - "\n", - " \n", - " \n", - "
    \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FbVhjPpzn6BM", - "colab_type": "text" - }, - "source": [ - "本指南训练了一个神经网络模型，来对服装图像进行分类，例如运动鞋和衬衫。如果您不了解所有细节也不需要担心，这是一个对完整TensorFlow项目的简要概述，相关的细节会在需要时进行解释\n", - "\n", - "本指南使用[tf.keras](https://www.tensorflow.org/r1/guide/keras)，这是一个用于在TensorFlow中构建和训练模型的高级API。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "dzLKpmZICaWN", - "colab_type": "code", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# 导入TensorFlow和tf.keras\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "# 导入辅助库\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yR0EdgrLCaWR", - "colab_type": "text" - }, - "source": [ - "## 导入Fashion MNIST数据集" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DLdCchMdCaWQ", - "colab_type": "text" - }, - "source": [ - "本指南使用[Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) 数据集，其中包含了10个类别中共70,000张灰度图像。图像包含了低分辨率（28 x 28像素）的单个服装物品，如下所示:\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Fashion\n", - "
    \n", - " Figure 1. Fashion-MNIST samples (by Zalando, MIT License).
     \n", - "
    \n", - "\n", - "Fashion MNIST 旨在替代传统的[MNIST](http://yann.lecun.com/exdb/mnist/)数据集 — 它经常被作为机器学习在计算机视觉方向的\"Hello, World\"。MNIST数据集包含手写数字（0,1,2等）的图像，其格式与我们在此处使用的服装相同。\n", - "\n", - "本指南使用Fashion MNIST进行多样化，因为它比普通的MNIST更具挑战性。两个数据集都相对较小，用于验证算法是否按预期工作。它们是测试和调试代码的良好起点。\n", - "\n", - "我们将使用60,000张图像来训练网络和10,000张图像来评估网络模型学习图像分类任务的准确程度。您可以直接从TensorFlow使用Fashion MNIST，只需导入并加载数据" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "7MqDQO0KCaWS", - "colab_type": "code", - "colab": {} - }, - "source": [ - "fashion_mnist = keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "t9FDsUlxCaWW", - "colab_type": "text" - }, - "source": [ - "加载数据集并返回四个NumPy数组:\n", - "\n", - "* `train_images`和`train_labels`数组是*训练集*—这是模型用来学习的数据。\n", - "* 模型通过*测试集*进行测试, 即`test_images`与 `test_labels`两个数组。\n", - "\n", - "图像是28x28 NumPy数组，像素值介于0到255之间。*labels*是一个整数数组，数值介于0到9之间。这对应了图像所代表的服装的*类别*:\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    标签类别
    0T-shirt/top
    1Trouser
    2Pullover
    3Dress
    4Coat
    5Sandal
    6Shirt
    7Sneaker
    8Bag
    9Ankle boot
    \n", - "\n", - "每个图像都映射到一个标签。由于*类别名称*不包含在数据集中,因此把他们存储在这里以便在绘制图像时使用:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "IjnLH5S2CaWx", - "colab_type": "code", - "colab": {} - }, - "source": [ - "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n", - " 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Brm0b_KACaWX", - "colab_type": "text" - }, - "source": [ - "## 探索数据\n", - "\n", - "让我们在训练模型之前探索数据集的格式。以下显示训练集中有60,000个图像，每个图像表示为28 x 28像素:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "zW5k_xz1CaWX", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cIAcvQqMCaWf", - "colab_type": "text" - }, - "source": [ - "同样，训练集中有60,000个标签:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "TRFYHB2mCaWb", - "colab_type": "code", - "colab": {} - }, - "source": [ - "len(train_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YSlYxFuRCaWk", - "colab_type": "text" - }, - "source": [ - "每个标签都是0到9之间的整数:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "XKnCTHz4CaWg", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TMPI88iZpO2T", - "colab_type": "text" - }, - "source": [ - "测试集中有10,000个图像。 同样，每个图像表示为28×28像素:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "2KFnYlcwCaWl", - "colab_type": "code", - "colab": {} - }, - "source": [ - "test_images.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "rd0A0Iu0CaWq", - "colab_type": "text" - }, - "source": [ - "测试集包含10,000个图像标签:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "iJmPr5-ACaWn", - "colab_type": "code", - "colab": {} - }, - "source": [ - "len(test_labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ES6uQoLKCaWr", - "colab_type": "text" - }, - "source": [ - "## 数据预处理\n", - "\n", - "在训练网络之前必须对数据进行预处理。 如果您检查训练集中的第一个图像，您将看到像素值落在0到255的范围内:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "m4VEw8Ud9Quh", - "colab_type": "code", - "colab": {} - }, - "source": [ - "plt.figure()\n", - "plt.imshow(train_images[0])\n", - "plt.colorbar()\n", - "plt.grid(False)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Wz7l27Lz9S1P", - "colab_type": "text" - }, - "source": [ - "在馈送到神经网络模型之前，我们将这些值缩放到0到1的范围。为此，我们将像素值值除以255。重要的是，对*训练集*和*测试集*要以相同的方式进行预处理:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "bW5WzIPlCaWv", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_images = train_images / 255.0\n", - "\n", - "test_images = test_images / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ee638AlnCaWz", - "colab_type": "text" - }, - "source": [ - "显示*训练集*中的前25个图像，并在每个图像下方显示类名。验证数据格式是否正确，我们是否已准备好构建和训练网络。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "oZTImqg_CaW1", - "colab_type": "code", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(10,10))\n", - "for i in range(25):\n", - " plt.subplot(5,5,i+1)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.grid(False)\n", - " plt.imshow(train_images[i], cmap=plt.cm.binary)\n", - " plt.xlabel(class_names[train_labels[i]])\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "59veuiEZCaW4", - "colab_type": "text" - }, - "source": [ - "## 构建模型\n", - "\n", - "构建神经网络需要配置模型的层，然后编译模型。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Gxg1XGm0eOBy", - "colab_type": "text" - }, - "source": [ - "### 设置网络层\n", - "\n", - "一个神经网络最基本的组成部分便是*网络层*。网络层从提供给他们的数据中提取表示，并期望这些表示对当前的问题更加有意义\n", - "\n", - "大多数深度学习是由串连在一起的网络层所组成。大多数网络层，例如`tf.keras.layers.Dense`，具有在训练期间学习的参数。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "9ODch-OFCaW4", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model = keras.Sequential([\n", - " keras.layers.Flatten(input_shape=(28, 28)),\n", - " keras.layers.Dense(128, activation=tf.nn.relu),\n", - " keras.layers.Dense(10, activation=tf.nn.softmax)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gut8A_7rCaW6", - "colab_type": "text" - }, - "source": [ - "网络中的第一层, `tf.keras.layers.Flatten`, 将图像格式从一个二维数组(包含着28x28个像素)转换成为一个包含着28 * 28 = 784个像素的一维数组。可以将这个网络层视为它将图像中未堆叠的像素排列在一起。这个网络层没有需要学习的参数;它仅仅对数据进行格式化。\n", - "\n", - "在像素被展平之后，网络由一个包含有两个`tf.keras.layers.Dense`网络层的序列组成。他们被称作稠密链接层或全连接层。 第一个`Dense`网络层包含有128个节点(或被称为神经元)。第二个(也是最后一个)网络层是一个包含10个节点的*softmax*层—它将返回包含10个概率分数的数组，总和为1。每个节点包含一个分数，表示当前图像属于10个类别之一的概率。\n", - "\n", - "### 编译模型\n", - "\n", - "在模型准备好进行训练之前，它还需要一些配置。这些是在模型的*编译(compile)*步骤中添加的:\n", - "\n", - "* *损失函数* —这可以衡量模型在培训过程中的准确程度。 我们希望将此函数最小化以\"驱使\"模型朝正确的方向拟合。\n", - "* *优化器* —这就是模型根据它看到的数据及其损失函数进行更新的方式。\n", - "* *评价方式* —用于监控训练和测试步骤。以下示例使用*准确率(accuracy)*，即正确分类的图像的分数。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Lhan11blCaW7", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qKF6uW-BCaW-", - "colab_type": "text" - }, - "source": [ - "## 训练模型\n", - "\n", - "训练神经网络模型需要以下步骤:\n", - "\n", - "1. 将训练数据提供给模型 - 在本案例中，他们是`train_images`和`train_labels`数组。\n", - "2. 模型学习如何将图像与其标签关联\n", - "3. 我们使用模型对测试集进行预测, 在本案例中为`test_images`数组。我们验证预测结果是否匹配`test_labels`数组中保存的标签。\n", - "\n", - "通过调用`model.fit`方法来训练模型 — 模型对训练数据进行\"拟合\"。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "xvwvpA64CaW_", - "colab_type": "code", - "colab": {} - }, - "source": [ - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "W3ZVOhugCaXA", - "colab_type": "text" - }, - "source": [ - "随着模型训练，将显示损失和准确率等指标。该模型在训练数据上达到约0.88(或88％)的准确度。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oEw4bZgGCaXB", - "colab_type": "text" - }, - "source": [ - "## 评估准确率\n", - "\n", - "接下来，比较模型在测试数据集上的执行情况:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "VflXLEeECaXC", - "colab_type": "code", - "colab": {} - }, - "source": [ - "test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "\n", - "print('Test accuracy:', test_acc)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yWfgsmVXCaXG", - "colab_type": "text" - }, - "source": [ - "事实证明，测试数据集的准确性略低于训练数据集的准确性。训练精度和测试精度之间的差距是*过拟合*的一个例子。过拟合是指机器学习模型在新数据上的表现比在训练数据上表现更差。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xsoS7CPDCaXH", - "colab_type": "text" - }, - "source": [ - "## 进行预测\n", - "\n", - "通过训练模型，我们可以使用它来预测某些图像。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Gl91RPhdCaXI", - "colab_type": "code", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_images)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "x9Kk1voUCaXJ", - "colab_type": "text" - }, - "source": [ - "在此，模型已经预测了测试集中每个图像的标签。我们来看看第一个预测:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "3DmJEUinCaXK", - "colab_type": "code", - "colab": {} - }, - "source": [ - "predictions[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-hw1hgeSCaXN", - "colab_type": "text" - }, - "source": [ - "预测是10个数字的数组。这些描述了模型的\"信心\"，即图像对应于10种不同服装中的每一种。我们可以看到哪个标签具有最高的置信度值：" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "qsqenuPnCaXO", - "colab_type": "code", - "colab": {} - }, - "source": [ - "np.argmax(predictions[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "E51yS7iCCaXO", - "colab_type": "text" - }, - "source": [ - "因此，模型最有信心的是这个图像是ankle boot，或者 `class_names[9]`。 我们可以检查测试标签，看看这是否正确:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "Sd7Pgsu6CaXP", - "colab_type": "code", - "colab": {} - }, - "source": [ - "test_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ygh2yYC972ne", - "colab_type": "text" - }, - "source": [ - "我们可以用图表来查看全部10个类别" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "DvYmmrpIy6Y1", - "colab_type": "code", - "colab": {} - }, - "source": [ - "def plot_image(i, predictions_array, true_label, img):\n", - " predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " \n", - " plt.imshow(img, cmap=plt.cm.binary)\n", - " \n", - " predicted_label = np.argmax(predictions_array)\n", - " if predicted_label == true_label:\n", - " color = 'blue'\n", - " else:\n", - " color = 'red'\n", - " \n", - " plt.xlabel(\"{} {:2.0f}% ({})\".format(class_names[predicted_label],\n", - " 100*np.max(predictions_array),\n", - " class_names[true_label]),\n", - " color=color)\n", - "\n", - "def plot_value_array(i, predictions_array, true_label):\n", - " predictions_array, true_label = predictions_array[i], true_label[i]\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n", - " plt.ylim([0, 1])\n", - " predicted_label = np.argmax(predictions_array)\n", - " \n", - " thisplot[predicted_label].set_color('red')\n", - " thisplot[true_label].set_color('blue')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "d4Ov9OFDMmOD", - "colab_type": "text" - }, - "source": [ - "让我们看看第0个图像，预测和预测数组。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "HV5jw-5HwSmO", - "colab_type": "code", - "colab": {} - }, - "source": [ - "i = 0\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "Ko-uzOufSCSe", - "colab_type": "code", - "colab": {} - }, - "source": [ - "i = 12\n", - "plt.figure(figsize=(6,3))\n", - "plt.subplot(1,2,1)\n", - "plot_image(i, predictions, test_labels, test_images)\n", - "plt.subplot(1,2,2)\n", - "plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kgdvGD52CaXR", - "colab_type": "text" - }, - "source": [ - "让我们绘制几个图像及其预测结果。正确的预测标签是蓝色的，不正确的预测标签是红色的。该数字给出了预测标签的百分比(满分100)。请注意，即使非常自信，也可能出错。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "hQlnbqaw2Qu_", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# 绘制前X个测试图像，预测标签和真实标签\n", - "# 以蓝色显示正确的预测，红色显示不正确的预测\n", - "num_rows = 5\n", - "num_cols = 3\n", - "num_images = num_rows*num_cols\n", - "plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n", - "for i in range(num_images):\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+1)\n", - " plot_image(i, predictions, test_labels, test_images)\n", - " plt.subplot(num_rows, 2*num_cols, 2*i+2)\n", - " plot_value_array(i, predictions, test_labels)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "R32zteKHCaXT", - "colab_type": "text" - }, - "source": [ - "最后，使用训练的模型对单个图像进行预测。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "yRJ7JU7JCaXT", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# 从测试数据集中获取图像\n", - "img = test_images[0]\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vz3bVp21CaXV", - "colab_type": "text" - }, - "source": [ - "`tf.keras`模型经过优化，可以一次性对*批量*,或者一个集合的数据进行预测。因此，即使我们使用单个图像，我们也需要将其添加到列表中:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "lDFh5yF_CaXW", - "colab_type": "code", - "colab": {} - }, - "source": [ - "# 将图像添加到批次中，即使它是唯一的成员。\n", - "img = (np.expand_dims(img,0))\n", - "\n", - "print(img.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EQ5wLTkcCaXY", - "colab_type": "text" - }, - "source": [ - "现在来预测图像:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "o_rzNSdrCaXY", - "colab_type": "code", - "colab": {} - }, - "source": [ - "predictions_single = model.predict(img)\n", - "\n", - "print(predictions_single)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "id": "6Ai-cpLjO-3A", - "colab_type": "code", - "colab": {} - }, - "source": [ - "plot_value_array(0, predictions_single, test_labels)\n", - "plt.xticks(range(10), class_names, rotation=45)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cU1Y2OAMCaXb", - "colab_type": "text" - }, - "source": [ - "`model.predict`返回一个包含列表的列表，每个图像对应一个列表的数据。获取批次中我们(仅有的)图像的预测:" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "2tRmdq_8CaXb", - "colab_type": "code", - "colab": {} - }, - "source": [ - "prediction_result = np.argmax(predictions_single[0])\n", - "print(prediction_result)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YFc2HbEVCaXd", - "colab_type": "text" - }, - "source": [ - "而且，和之前一样，模型预测标签为9。" - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/customization/autodiff.ipynb b/site/zh-cn/tutorials/customization/autodiff.ipynb deleted file mode 100644 index ba0f556c2b3..00000000000 --- a/site/zh-cn/tutorials/customization/autodiff.ipynb +++ /dev/null @@ -1,338 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "GCCk8_dHpuNf" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xh8WkEwWpnm7" - }, - "source": [ - "# 自动微分和梯度带" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "idv0bPeCp325" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 TensorFlow.google.cn 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vDJ4XzMqodTy" - }, - "source": [ - "在上一个教程中，我们介绍了 \"张量\"（Tensor）及其操作。本教程涉及[自动微分（automatic differentitation）](https://en.wikipedia.org/wiki/Automatic_differentiation)，它是优化机器学习模型的关键技巧之一。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GQJysDM__Qb0" - }, - "source": [ - "## 创建 \n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cxzaxo6ff2y3" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow==2.0.0-beta1\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1CLWJl0QliB0" - }, - "source": [ - "## 梯度带\n", - "\n", - "TensorFlow 为自动微分提供了 [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) API ，根据某个函数的输入变量来计算它的导数。Tensorflow 会把 'tf.GradientTape' 上下文中执行的所有操作都记录在一个磁带上 (\"tape\")。 然后基于这个磁带和每次操作产生的导数，用反向微分法（\"reverse mode differentiation\"）来计算这些被“记录在案”的函数的导数。\n", - "\n", - "例如：" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bAFeIE8EuVIq" - }, - "outputs": [], - "source": [ - "x = tf.ones((2, 2))\n", - "\n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# Derivative of z with respect to the original input tensor x\n", - "dz_dx = t.gradient(z, x)\n", - "for i in [0, 1]:\n", - " for j in [0, 1]:\n", - " assert dz_dx[i][j].numpy() == 8.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N4VlqKFzzGaC" - }, - "source": [ - "你也可以使用 `tf.GradientTape` 上下文计算过程产生的中间结果来求取导数。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "7XaPRAwUyYms" - }, - "outputs": [], - "source": [ - "x = tf.ones((2, 2))\n", - "\n", - "with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " y = tf.reduce_sum(x)\n", - " z = tf.multiply(y, y)\n", - "\n", - "# Use the tape to compute the derivative of z with respect to the\n", - "# intermediate value y.\n", - "dz_dy = t.gradient(z, y)\n", - "assert dz_dy.numpy() == 8.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ISkXuY7YzIcS" - }, - "source": [ - "默认情况下，调用 GradientTape.gradient() 方法时， GradientTape 占用的资源会立即得到释放。通过创建一个持久的梯度带，可以计算同个函数的多个导数。这样在磁带对象被垃圾回收时，就可以多次调用 'gradient()' 方法。例如：" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "zZaCm3-9zVCi" - }, - "outputs": [], - "source": [ - "x = tf.constant(3.0)\n", - "with tf.GradientTape(persistent=True) as t:\n", - " t.watch(x)\n", - " y = x * x\n", - " z = y * y\n", - "dz_dx = t.gradient(z, x) # 108.0 (4*x^3 at x = 3)\n", - "dy_dx = t.gradient(y, x) # 6.0\n", - "del t # Drop the reference to the tape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6kADybtQzYj4" - }, - "source": [ - "### 记录控制流\n", - "\n", - "由于磁带会记录所有执行的操作，Python 控制流（如使用 if 和 while 的代码段）自然得到了处理。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "9FViq92UX7P8" - }, - "outputs": [], - "source": [ - "def f(x, y):\n", - " output = 1.0\n", - " for i in range(y):\n", - " if i > 1 and i < 5:\n", - " output = tf.multiply(output, x)\n", - " return output\n", - "\n", - "def grad(x, y):\n", - " with tf.GradientTape() as t:\n", - " t.watch(x)\n", - " out = f(x, y)\n", - " return t.gradient(out, x)\n", - "\n", - "x = tf.convert_to_tensor(2.0)\n", - "\n", - "assert grad(x, 6).numpy() == 12.0\n", - "assert grad(x, 5).numpy() == 12.0\n", - "assert grad(x, 4).numpy() == 4.0\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DK05KXrAAld3" - }, - "source": [ - "### 高阶导数\n", - "\n", - "在 'GradientTape' 上下文管理器中记录的操作会用于自动微分。如果导数是在上下文中计算的，导数的函数也会被记录下来。因此，同个 API 可以用于高阶导数。例如：" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "cPQgthZ7ugRJ" - }, - "outputs": [], - "source": [ - "x = tf.Variable(1.0) # Create a Tensorflow variable initialized to 1.0\n", - "\n", - "with tf.GradientTape() as t:\n", - " with tf.GradientTape() as t2:\n", - " y = x * x * x\n", - " # Compute the gradient inside the 't' context manager\n", - " # which means the gradient computation is differentiable as well.\n", - " dy_dx = t2.gradient(y, x)\n", - "d2y_dx2 = t.gradient(dy_dx, x)\n", - "\n", - "assert dy_dx.numpy() == 3.0\n", - "assert d2y_dx2.numpy() == 6.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4U1KKzUpNl58" - }, - "source": [ - "## 下一步\n", - "\n", - "本教程涉及 TensorFlow 中的导数计算。以此为基础，我们具备了足够的先修知识来构建和训练神经网络。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "j8BT7T8yf6Rv" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "autodiff.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/site/zh-cn/tutorials/customization/custom_training.ipynb b/site/zh-cn/tutorials/customization/custom_training.ipynb deleted file mode 100644 index d7df7fdc3dc..00000000000 --- a/site/zh-cn/tutorials/customization/custom_training.ipynb +++ /dev/null @@ -1,447 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rmpybwysXGV" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "m8y3rGtQsYP2", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hrXv0rU9sIma" - }, - "source": [ - "# 定制化训练：基础" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7S0BwJ_8sLu7" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 Tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k2o3TTG4TFpt" - }, - "source": [ - "在之前的教程中，我们讨论了自动差分，一个基本的机器学习模块。在这个教程中，我们将使用在之前介绍的 Tensorflow 基础语句实现简单的机器学习模型。\n", - "\n", - "Tensorflow 也提供了高级神经网络 API（`tf.keras`），可以精简范例代码。我们强烈建议在神经网络方面的工作使用高级的 API。在这篇教程中，我们使用基本规则来训练神经网络，为以后打下牢固基础。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3LXMVuV0VhDr" - }, - "source": [ - "## 创建" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NiolgWMPgpwI", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eMAWbDJFVmMk" - }, - "source": [ - "## 变量\n", - "\n", - "Tensorflow 中的 tensor 是不可变无状态对象。机器学习模型需要可改变状态，比如模型训练和模型预测的代码是相同的，但变量值随着时间而不同（希望尽量小的 loss）。为了应对随着计算而改变的状态，可以利用 Python 的状态可变性。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VkJwtLS_Jbn8", - "colab": {} - }, - "source": [ - "# 使用 python 状态\n", - "x = tf.zeros([10, 10])\n", - "# 等价于 x = x + 2, 不改变原本 x 的值\n", - "x += 2 \n", - "print(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfneTXy7JcUz" - }, - "source": [ - "TensorFlow，拥有内建可变状态操作，比使用底层 Python 状态表示更常见的。比如，表示模型的权重，使用 TensorFlow 变量更方便高效。\n", - "\n", - "变量是一个对象，这个对象存储着数值，当在 TensorFlow 计算中使用时，会隐式地读取这个存储的数值。有一些操作（`tf.assign_sub`, `tf.scatter_update` 等）会复制 TensorFlow 变量存储的数值。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "itxmrMil6DQi", - "colab": {} - }, - "source": [ - "v = tf.Variable(1.0)\n", - "assert v.numpy() == 1.0\n", - "\n", - "# 重新赋值\n", - "v.assign(3.0)\n", - "assert v.numpy() == 3.0\n", - "\n", - "# 在 TensorFlow 操作中使用 `v`，比如 tf.square() 和重新赋值\n", - "v.assign(tf.square(v))\n", - "assert v.numpy() == 9.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-paSaeq1JzwC" - }, - "source": [ - "当计算梯度时，会自动跟踪使用变量的计算过程。用变量来表示向量时，TensorFlow 会默认使用稀疏更新，这样可以带来计算和存储高效性。\n", - "\n", - "使用变量也是一种更快的提醒方式，就是代码的这部分是状态可变的。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BMiFcDzE7Qu3" - }, - "source": [ - "## 示例：尝试一个线性模型\n", - "\n", - "让我们来使用目前为止学到的概念---`Tensor`，`Variable`，和 `GradientTape`---来创建和训练一个简单的模型。一般需要下面这些步骤：\n", - "\n", - "1. 定义模型\n", - "2. 定义损失函数\n", - "3. 获取训练数据\n", - "4. 通过训练数据运行模型，使用 \"optimizer\" 来调整变量以满足数据\n", - "\n", - "在这个教程中，我们使用一个简单线性模型作为示例：`f(x) = x * W + b`，有2个变量- `W` 和 `b`。另外，我们会生成数据让训练好的模型满足 `W = 3.0` 和 `b = 2.0`。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFzH64Jn9PIm" - }, - "source": [ - "### 定义模型\n", - "\n", - "定义一个简单的类封装变量和计算" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_WRu7Pze7wk8", - "colab": {} - }, - "source": [ - "class Model(object):\n", - " def __init__(self):\n", - " # 初始化变量值为(5.0, 0.0)\n", - " # 实际上，这些变量应该初始化为随机值\n", - " self.W = tf.Variable(5.0)\n", - " self.b = tf.Variable(0.0)\n", - "\n", - " def __call__(self, x):\n", - " return self.W * x + self.b\n", - "\n", - "model = Model()\n", - "\n", - "assert model(3.0).numpy() == 15.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xa6j_yXa-j79" - }, - "source": [ - "### 定义损失函数\n", - "\n", - "损失函数用来衡量在给定输入的情况下，模型的预测输出与实际输出的偏差。我们这里使用标准 L2 损失函数。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y0ysUFGY924U", - "colab": {} - }, - "source": [ - "def loss(predicted_y, desired_y):\n", - " return tf.reduce_mean(tf.square(predicted_y - desired_y))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qutT_fkl_CBc" - }, - "source": [ - "### 获取训练数据\n", - "\n", - "我们来生成带噪声的训练数据。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gxPTb-kt_N5m", - "colab": {} - }, - "source": [ - "TRUE_W = 3.0\n", - "TRUE_b = 2.0\n", - "NUM_EXAMPLES = 1000\n", - "\n", - "inputs = tf.random.normal(shape=[NUM_EXAMPLES])\n", - "noise = tf.random.normal(shape=[NUM_EXAMPLES])\n", - "outputs = inputs * TRUE_W + TRUE_b + noise" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-50nq-wPBsAW" - }, - "source": [ - "在训练模型之前，我们来看看当前的模型表现。我们绘制模型的预测结果和训练数据，预测结果用红色表示，训练数据用蓝色表示。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_eb83LtrB4nt", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.scatter(inputs, outputs, c='b')\n", - "plt.scatter(inputs, model(inputs), c='r')\n", - "plt.show()\n", - "\n", - "print('Current loss: '),\n", - "print(loss(model(inputs), outputs).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sSDP-yeq_4jE" - }, - "source": [ - "### 定义训练循环\n", - "\n", - "我们已经定义了网络模型，并且获得了训练数据。现在对模型进行训练，采用[梯度下降](https://en.wikipedia.org/wiki/Gradient_descent)的方式，通过训练数据更新模型的变量（`W` 和 `b`）使得损失量变小。梯度下降中有很多参数，通过 `tf.train.Optimizer` 实现。我们强烈建议使用这些实现方式，但基于通过基本规则创建模型的精神，在这个特别示例中，我们自己实现基本的数学运算。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MBIACgdnA55X", - "colab": {} - }, - "source": [ - "def train(model, inputs, outputs, learning_rate):\n", - " with tf.GradientTape() as t:\n", - " current_loss = loss(model(inputs), outputs)\n", - " dW, db = t.gradient(current_loss, [model.W, model.b])\n", - " model.W.assign_sub(learning_rate * dW)\n", - " model.b.assign_sub(learning_rate * db)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RwWPaJryD2aN" - }, - "source": [ - "最后，我们对训练数据重复地训练，观察 `W` 和 `b` 是怎么变化的。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XdfkR223D9dW", - "colab": {} - }, - "source": [ - "model = Model()\n", - "\n", - "# 收集 W 和 b 的历史数值，用于显示\n", - "Ws, bs = [], []\n", - "epochs = range(10)\n", - "for epoch in epochs:\n", - " Ws.append(model.W.numpy())\n", - " bs.append(model.b.numpy())\n", - " current_loss = loss(model(inputs), outputs)\n", - "\n", - " train(model, inputs, outputs, learning_rate=0.1)\n", - " print('Epoch %2d: W=%1.2f b=%1.2f, loss=%2.5f' %\n", - " (epoch, Ws[-1], bs[-1], current_loss))\n", - "\n", - "# 显示所有\n", - "plt.plot(epochs, Ws, 'r',\n", - " epochs, bs, 'b')\n", - "plt.plot([TRUE_W] * len(epochs), 'r--',\n", - " [TRUE_b] * len(epochs), 'b--')\n", - "plt.legend(['W', 'b', 'true W', 'true_b'])\n", - "plt.show()\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vPnIVuaSJwWz" - }, - "source": [ - "## 下一步\n", - "\n", - "在这个教程中，我们讨论了 `Variable`，而且创建和训练了一个简单的线性模型，使用了在此之前所学习的 TensorFlow 知识点。\n", - "\n", - "理论上，掌握了 TensorFlow 这些知识点即可用于机器学习研究。实际上，采用高级的 API 比如 `tf.keras` 是更方便的，特别是神经网络，因为它提供了更高级别的内建模块（命名为 \"layers\"），可以保存和恢复状态，还有配套的损失函数和优化策略等。\n" - ] - } - ] -} diff --git a/site/zh-cn/tutorials/customization/custom_training_walkthrough.ipynb b/site/zh-cn/tutorials/customization/custom_training_walkthrough.ipynb deleted file mode 100644 index c03f72d12ae..00000000000 --- a/site/zh-cn/tutorials/customization/custom_training_walkthrough.ipynb +++ /dev/null @@ -1,1147 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "celltoolbar": "编辑元数据", - "colab": { - "name": "custom_training_walkthrough.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rwxGnsA92emp" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "CPII1rGR2rF9", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtEZ1pCPn--z" - }, - "source": [ - "# 自定义训练: 演示" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GV1F7tVTN3Dn" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " View on TensorFlow.org\n", - " \n", - " Run in Google Colab\n", - " \n", - " View source on GitHub\n", - " \n", - " Download notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LDrzLFXE8T1l" - }, - "source": [ - "这个教程将利用机器学习的手段来对鸢尾花按照物种进行分类。本教程将利用 TensorFlow 来进行以下操作：\n", - "1. 构建一个模型，\n", - "2. 用样例数据集对模型进行训练，以及\n", - "3. 利用该模型对未知数据进行预测。\n", - "\n", - "## TensorFlow 编程\n", - "\n", - "本指南采用了以下高级 TensorFlow 概念:\n", - "\n", - "* 使用 TensorFlow 默认的 [eager execution](https://www.tensorflow.org/guide/eager) 开发环境,\n", - "* 使用 [Datasets API](https://www.tensorflow.org/guide/datasets) 导入数据,\n", - "* 使用 TensorFlow 的 [Keras API](https://keras.io/getting-started/sequential-model-guide/) 来构建各层以及整个模型。\n", - "\n", - "本教程的结构同很多 TensorFlow 程序相似：\n", - "\n", - "1. 数据集的导入与解析\n", - "2. 选择模型类型\n", - "3. 对模型进行训练\n", - "4. 评估模型效果\n", - "5. 使用训练过的模型进行预测" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yNr7H-AIoLOR" - }, - "source": [ - "## 环境的搭建" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1J3AuPBT9gyR" - }, - "source": [ - "### 配置导入\n", - "\n", - "导入 TensorFlow 以及其他需要的 Python 库。 默认情况下，TensorFlow 用 [eager execution](https://www.tensorflow.org/guide/eager) 来实时评估操作, 返回具体值而不是建立一个稍后执行的[计算图](https://www.tensorflow.org/guide/graphs)。 如果您习惯使用 REPL 或 python 交互控制台, 对此您会感觉得心应手。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jElLULrDhQZR", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "import matplotlib.pyplot as plt" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bfV2Dai0Ow2o", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "g4Wzg69bnwK2", - "colab": {} - }, - "source": [ - "print(\"TensorFlow version: {}\".format(tf.__version__))\n", - "print(\"Eager execution: {}\".format(tf.executing_eagerly()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zx7wc0LuuxaJ" - }, - "source": [ - "## 鸢尾花分类问题\n", - "\n", - " 想象一下，您是一名植物学家，正在寻找一种能够对所发现的每株鸢尾花进行自动归类的方法。机器学习可提供多种从统计学上分类花卉的算法。例如，一个复杂的机器学习程序可以根据照片对花卉进行分类。我们的要求并不高 - 我们将根据鸢尾花花萼和花瓣的长度和宽度对其进行分类。\n", - "\n", - "鸢尾属约有 300 个品种，但我们的程序将仅对下列三个品种进行分类：\n", - "\n", - "* 山鸢尾\n", - "* 维吉尼亚鸢尾\n", - "* 变色鸢尾\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Petal\n", - "
    \n", - " Figure 1. 山鸢尾 (by Radomil, CC BY-SA 3.0), 变色鸢尾, (by Dlanglois, CC BY-SA 3.0), and 维吉尼亚鸢尾 (by Frank Mayfield, CC BY-SA 2.0).
     \n", - "
    \n", - "\n", - "幸运的是，有人已经创建了一个包含有花萼和花瓣的测量值的[120 株鸢尾花的数据集](https://en.wikipedia.org/wiki/Iris_flower_data_set)。这是一个在入门级机器学习分类问题中经常使用的经典数据集。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Px6KAg0Jowz" - }, - "source": [ - "## 导入和解析训练数据集\n", - "\n", - "下载数据集文件并将其转换为可供此 Python 程序使用的结构。\n", - "\n", - "### 下载数据集\n", - "\n", - "使用 [tf.keras.utils.get_file](https://www.tensorflow.org/api_docs/python/tf/keras/utils/get_file) 函数下载训练数据集文件。该函数会返回下载文件的文件路径：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J6c7uEU9rjRM", - "colab": {} - }, - "source": [ - "train_dataset_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv\"\n", - "\n", - "train_dataset_fp = tf.keras.utils.get_file(fname=os.path.basename(train_dataset_url),\n", - " origin=train_dataset_url)\n", - "\n", - "print(\"Local copy of the dataset file: {}\".format(train_dataset_fp))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qnX1-aLors4S" - }, - "source": [ - "### 检查数据\n", - "\n", - "数据集 `iris_training.csv` 是一个纯文本文件，其中存储了逗号分隔值 (CSV) 格式的表格式数据.请使用 `head -n5` 命令查看前 5 个条目：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FQvb_JYdrpPm", - "colab": {} - }, - "source": [ - "!head -n5 {train_dataset_fp}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kQhzD6P-uBoq" - }, - "source": [ - "我们可以从该数据集视图中注意到以下信息：\n", - "1. 第一行是表头，其中包含数据集信息：\n", - "\n", - "* 共有 120 个样本。每个样本都有四个特征和一个标签名称，标签名称有三种可能。\n", - "2. 后面的行是数据记录，每个[样本](https://developers.google.com/machine-learning/glossary/#example)各占一行，其中：\n", - " * 前四个字段是[特征](https://developers.google.com/machine-learning/glossary/#feature): 这四个字段代表的是样本的特点。在此数据集中，这些字段存储的是代表花卉测量值的浮点数。\n", - " * 最后一列是[标签](https://developers.google.com/machine-learning/glossary/#label):即我们想要预测的值。对于此数据集，该值为 0、1 或 2 中的某个整数值（每个值分别对应一个花卉名称）。\n", - "\n", - "我们用代码表示出来：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9Edhevw7exl6", - "colab": {} - }, - "source": [ - "# CSV文件中列的顺序\n", - "column_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']\n", - "\n", - "feature_names = column_names[:-1]\n", - "label_name = column_names[-1]\n", - "\n", - "print(\"Features: {}\".format(feature_names))\n", - "print(\"Label: {}\".format(label_name))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CCtwLoJhhDNc" - }, - "source": [ - "每个标签都分别与一个字符串名称（例如 “setosa” ）相关联，但机器学习通常依赖于数字值。标签编号会映射到一个指定的表示法，例如：\n", - "\n", - "* `0` : 山鸢尾\n", - "* `1` : 变色鸢尾\n", - "* `2` : 维吉尼亚鸢尾\n", - "\n", - "如需详细了解特征和标签，请参阅 [《机器学习速成课程》的“机器学习术语”部分](https://developers.google.com/machine-learning/crash-course/framing/ml-terminology)." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sVNlJlUOhkoX", - "colab": {} - }, - "source": [ - "class_names = ['Iris setosa', 'Iris versicolor', 'Iris virginica']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dqPkQExM2Pwt" - }, - "source": [ - "### 创建一个 `tf.data.Dataset`\n", - "\n", - "TensorFlow的 [Dataset API](https://www.tensorflow.org/guide/datasets) 可处理在向模型加载数据时遇到的许多常见情况。这是一种高阶 API ，用于读取数据并将其转换为可供训练使用的格式。如需了解详情，请参阅[数据集快速入门指南](https://www.tensorflow.org/get_started/datasets_quickstart) \n", - "\n", - "\n", - "由于数据集是 CSV 格式的文本文件，请使用 [make_csv_dataset](https://www.tensorflow.org/api_docs/python/tf/data/experimental/make_csv_dataset) 函数将数据解析为合适的格式。由于此函数为训练模型生成数据，默认行为是对数据进行随机处理 （`shuffle=True, shuffle_buffer_size=10000`），并且无限期重复数据集（`num_epochs=None`）。 我们还设置了 [batch_size](https://developers.google.com/machine-learning/glossary/#batch_size) 参数:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WsxHnz1ebJ2S", - "colab": {} - }, - "source": [ - "batch_size = 32\n", - "\n", - "train_dataset = tf.data.experimental.make_csv_dataset(\n", - " train_dataset_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name=label_name,\n", - " num_epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gB_RSn62c-3G" - }, - "source": [ - " `make_csv_dataset` 返回一个`(features, label)` 对构建的 `tf.data.Dataset` ，其中 `features` 是一个字典: `{'feature_name': value}`\n", - "\n", - "这些 `Dataset` 对象是可迭代的。 我们来看看下面的一些特征:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iDuG94H-C122", - "colab": {} - }, - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "print(features)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "E63mArnQaAGz" - }, - "source": [ - "注意到具有相似特征的样本会归为一组，即分为一批。更改 `batch_size` 可以设置存储在这些特征数组中的样本数。\n", - "\n", - "绘制该批次中的几个特征后，就会开始看到一些集群现象：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "me5Wn-9FcyyO", - "colab": {} - }, - "source": [ - "plt.scatter(features['petal_length'],\n", - " features['sepal_length'],\n", - " c=labels,\n", - " cmap='viridis')\n", - "\n", - "plt.xlabel(\"Petal length\")\n", - "plt.ylabel(\"Sepal length\")\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YlxpSyHlhT6M" - }, - "source": [ - "要简化模型构建步骤，请创建一个函数以将特征字典重新打包为形状为 `(batch_size, num_features)` 的单个数组。\n", - "\n", - "此函数使用 [tf.stack](https://www.tensorflow.org/api_docs/python/tf/stack) 方法，该方法从张量列表中获取值，并创建指定维度的组合张量:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jm932WINcaGU", - "colab": {} - }, - "source": [ - "def pack_features_vector(features, labels):\n", - " \"\"\"将特征打包到一个数组中\"\"\"\n", - " features = tf.stack(list(features.values()), axis=1)\n", - " return features, labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "V1Vuph_eDl8x" - }, - "source": [ - "然后使用 [tf.data.Dataset.map](https://www.tensorflow.org/api_docs/python/tf/data/dataset/map) 方法将每个 `(features,label)` 对中的 `features` 打包到训练数据集中：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZbDkzGZIkpXf", - "colab": {} - }, - "source": [ - "train_dataset = train_dataset.map(pack_features_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NLy0Q1xCldVO" - }, - "source": [ - " `Dataset` 的特征元素被构成了形如 `(batch_size, num_features)` 的数组。我们来看看前几个样本:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kex9ibEek6Tr", - "colab": {} - }, - "source": [ - "features, labels = next(iter(train_dataset))\n", - "\n", - "print(features[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LsaVrtNM3Tx5" - }, - "source": [ - "## 选择模型类型\n", - "\n", - "### 为何要使用模型?\n", - "\n", - "[模型](https://developers.google.com/machine-learning/crash-course/glossary#model)是指特征与标签之间的关系。对于鸢尾花分类问题，模型定义了花萼和花瓣测量值与预测的鸢尾花品种之间的关系。一些简单的模型可以用几行代数进行描述，但复杂的机器学习模型拥有大量难以汇总的参数。\n", - "\n", - "\n", - "您能否在不使用机器学习的情况下确定四个特征与鸢尾花品种之间的关系？也就是说，您能否使用传统编程技巧（例如大量条件语句）创建模型？也许能，前提是反复分析该数据集，并最终确定花瓣和花萼测量值与特定品种的关系。对于更复杂的数据集来说，这会变得非常困难，或许根本就做不到。一个好的机器学习方法可为您确定模型。如果您将足够多的代表性样本馈送到正确类型的机器学习模型中，该程序便会为您找出相应的关系。\n", - "\n", - "### 选择模型\n", - "\n", - "我们需要选择要进行训练的模型类型。模型具有许多类型，挑选合适的类型需要一定的经验。本教程使用神经网络来解决鸢尾花分类问题。[神经网络](https://developers.google.com/machine-learning/glossary/#neural_network)可以发现特征与标签之间的复杂关系。神经网络是一个高度结构化的图，其中包含一个或多个[隐含层](https://developers.google.com/machine-learning/glossary/#hidden_layer)。每个隐含层都包含一个或多个[神经元](https://developers.google.com/machine-learning/glossary/#neuron)。 神经网络有多种类别，该程序使用的是密集型神经网络，也称为[全连接神经网络](https://developers.google.com/machine-learning/glossary/#fully_connected_layer) : 一个层中的神经元将从上一层中的每个神经元获取输入连接。例如，图 2 显示了一个密集型神经网络，其中包含 1 个输入层、2 个隐藏层以及 1 个输出层：\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"网络结构示意图:\n", - "
    \n", - " 图 2. 包含特征、隐藏层和预测的神经网络
     \n", - "
    \n", - "\n", - "当图 2 中的模型经过训练并获得无标签样本后，它会产生 3 个预测结果：相应鸢尾花属于指定品种的可能性。这种预测称为[推理](https://developers.google.com/machine-learning/crash-course/glossary#inference)。对于该示例，输出预测结果的总和是 1.0。在图 2 中，该预测结果分解如下：山鸢尾为 0.02，变色鸢尾为 0.95，维吉尼亚鸢尾为 0.03。这意味着该模型预测某个无标签鸢尾花样本是变色鸢尾的概率为 95％。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W23DIMVPQEBt" - }, - "source": [ - "### 使用 Keras 创建模型\n", - "\n", - "TensorFlow [tf.keras](https://www.tensorflow.org/api_docs/python/tf/keras) API 是创建模型和层的首选方式。通过该 API，您可以轻松地构建模型并进行实验，而将所有部分连接在一起的复杂工作则由 Keras 处理。\n", - "\n", - "[tf.keras.Sequential](https://www.tensorflow.org/api_docs/python/tf/keras/Sequential) 模型是层的线性堆叠。该模型的构造函数会采用一系列层实例；在本示例中，采用的是 2 个[密集层](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense)（各自包含10个节点）,以及 1 个输出层（包含 3 个代表标签预测的节点。第一个层的 `input_shape` 参数对应该数据集中的特征数量，它是一项必需参数：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2fZ6oL2ig3ZK", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(4,)), # 需要给出输入的形式\n", - " tf.keras.layers.Dense(10, activation=tf.nn.relu),\n", - " tf.keras.layers.Dense(3)\n", - "])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FHcbEzMpxbHL" - }, - "source": [ - "[激活函数](https://developers.google.com/machine-learning/crash-course/glossary#activation_function)可决定层中每个节点的输出形式。 这些非线性关系很重要，如果没有它们，模型将等同于单个层。[激活函数](https://www.tensorflow.org/api_docs/python/tf/keras/activations)有很多种，但隐藏层通常使用 [ReLU](https://developers.google.com/machine-learning/crash-course/glossary#ReLU)。\n", - "\n", - "隐藏层和神经元的理想数量取决于问题和数据集。与机器学习的多个方面一样，选择最佳的神经网络形状需要一定的知识水平和实验基础。一般来说，增加隐藏层和神经元的数量通常会产生更强大的模型，而这需要更多数据才能有效地进行训练。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2wFKnhWCpDSS" - }, - "source": [ - "### 使用模型\n", - "\n", - "我们快速了解一下此模型如何处理一批特征：\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xe6SQ5NrpB-I", - "colab": {} - }, - "source": [ - "predictions = model(features)\n", - "predictions[:5]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wxyXOhwVr5S3" - }, - "source": [ - "在此示例中，每个样本针对每个类别返回一个 [logit](https://developers.google.com/machine-learning/crash-course/glossary#logits)。\n", - "\n", - "要将这些对数转换为每个类别的概率，请使用 [softmax](https://developers.google.com/machine-learning/crash-course/glossary#softmax) 函数:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_tRwHZmTNTX2", - "colab": {} - }, - "source": [ - "tf.nn.softmax(predictions[:5])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uRZmchElo481" - }, - "source": [ - "对每个类别执行 `tf.argmax` 运算可得出预测的类别索引。不过，该模型尚未接受训练，因此这些预测并不理想。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-Jzm_GoErz8B", - "colab": {} - }, - "source": [ - "print(\"Prediction: {}\".format(tf.argmax(predictions, axis=1)))\n", - "print(\" Labels: {}\".format(labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vzq2E5J2QMtw" - }, - "source": [ - "## 训练模型\n", - "\n", - "[训练](https://developers.google.com/machine-learning/crash-course/glossary#training) 是一个机器学习阶段，在此阶段中，模型会逐渐得到优化，也就是说，模型会了解数据集。目标是充分了解训练数据集的结构，以便对未见过的数据进行预测。如果您从训练数据集中获得了过多的信息，预测便会仅适用于模型见过的数据，但是无法泛化。此问题被称之为[过拟合](https://developers.google.com/machine-learning/crash-course/glossary#overfitting)—就好比将答案死记硬背下来，而不去理解问题的解决方式。\n", - "\n", - "鸢尾花分类问题是[监督式机器学习](https://developers.google.com/machine-learning/glossary/#supervised_machine_learning)的一个示例: 模型通过包含标签的样本加以训练。 而在[非监督式机器学习](https://developers.google.com/machine-learning/glossary/#unsupervised_machine_learning)中，样本不包含标签。相反，模型通常会在特征中发现一些规律。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RaKp8aEjKX6B" - }, - "source": [ - "### 定义损失和梯度函数\n", - "\n", - "在训练和评估阶段，我们都需要计算模型的[损失](https://developers.google.com/machine-learning/crash-course/glossary#loss)。 这样可以衡量模型的预测结果与预期标签有多大偏差，也就是说，模型的效果有多差。我们希望尽可能减小或优化这个值。\n", - "\n", - "我们的模型会使用 `tf.keras.losses.SparseCategoricalCrossentropy` 函数计算其损失，此函数会接受模型的类别概率预测结果和预期标签，然后返回样本的平均损失。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QOsi6b-1CXIn", - "scrolled": true, - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tMAT4DcMPwI-", - "colab": {} - }, - "source": [ - "def loss(model, x, y):\n", - " y_ = model(x)\n", - "\n", - " return loss_object(y_true=y, y_pred=y_)\n", - "\n", - "\n", - "l = loss(model, features, labels)\n", - "print(\"Loss test: {}\".format(l))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3IcPqA24QM6B" - }, - "source": [ - "使用 [tf.GradientTape](https://www.tensorflow.org/api_docs/python/tf/GradientTape) 的前后关系来计算[梯度](https://developers.google.com/machine-learning/crash-course/glossary#gradient)以优化你的模型:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x57HcKWhKkei", - "colab": {} - }, - "source": [ - "def grad(model, inputs, targets):\n", - " with tf.GradientTape() as tape:\n", - " loss_value = loss(model, inputs, targets)\n", - " return loss_value, tape.gradient(loss_value, model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lOxFimtlKruu" - }, - "source": [ - "### 创建优化器\n", - "\n", - "[优化器](https://developers.google.com/machine-learning/crash-course/glossary#optimizer) 会将计算出的梯度应用于模型的变量，以使 `loss` 函数最小化。您可以将损失函数想象为一个曲面（见图 3），我们希望通过到处走动找到该曲面的最低点。梯度指向最高速上升的方向，因此我们将沿相反的方向向下移动。我们以迭代方式计算每个批次的损失和梯度，以在训练过程中调整模型。模型会逐渐找到权重和偏差的最佳组合，从而将损失降至最低。损失越低，模型的预测效果就越好。\n", - "\n", - "\n", - " \n", - " \n", - "
    \n", - " \"Optimization\n", - "
    \n", - " 图 3. 优化算法在三维空间中随时间推移而变化的可视化效果。
    (来源: 斯坦福大学 CS231n 课程，MIT 许可证，Image credit: Alec Radford)\n", - "
    \n", - "\n", - "TensorFlow有许多可用于训练的[优化算法](https://www.tensorflow.org/api_guides/python/train)。此模型使用的是 [tf.train.GradientDescentOptimizer](https://www.tensorflow.org/api_docs/python/tf/train/GradientDescentOptimizer) ， 它可以实现[随机梯度下降法](https://developers.google.com/machine-learning/crash-course/glossary#gradient_descent)（SGD）。`learning_rate` 被用于设置每次迭代（向下行走）的步长。 这是一个 *超参数* ，您通常需要调整此参数以获得更好的结果。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XkUd6UiZa_dF" - }, - "source": [ - "我们来设置优化器：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8xxi2NNGKwG_", - "colab": {} - }, - "source": [ - "optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYgezhPMhaNE" - }, - "source": [ - "我们将使用它来计算单个优化步骤：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rxRNTFVe56RG", - "colab": {} - }, - "source": [ - "loss_value, grads = grad(model, features, labels)\n", - "\n", - "print(\"Step: {}, Initial Loss: {}\".format(optimizer.iterations.numpy(),\n", - " loss_value.numpy()))\n", - "\n", - "optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - "print(\"Step: {}, Loss: {}\".format(optimizer.iterations.numpy(),\n", - " loss(model, features, labels).numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Y2VSELvwAvW" - }, - "source": [ - "### 训练循环\n", - "\n", - "\n", - "一切准备就绪后，就可以开始训练模型了！训练循环会将数据集样本馈送到模型中，以帮助模型做出更好的预测。以下代码块可设置这些训练步骤：\n", - "\n", - "1. 迭代每个周期。通过一次数据集即为一个周期。\n", - "2. 在一个周期中，遍历训练 `Dataset` 中的每个样本，并获取样本的*特征*（`x`）和*标签*（`y`）。\n", - "3. 根据样本的特征进行预测，并比较预测结果和标签。衡量预测结果的不准确性，并使用所得的值计算模型的损失和梯度。\n", - "4. 使用 `optimizer` 更新模型的变量。\n", - "5. 跟踪一些统计信息以进行可视化。\n", - "6. 对每个周期重复执行以上步骤。\n", - "\n", - "\n", - "`num_epochs` 变量是遍历数据集集合的次数。与直觉恰恰相反的是，训练模型的时间越长，并不能保证模型就越好。`num_epochs` 是一个可以调整的[超参数](https://developers.google.com/machine-learning/glossary/#hyperparameter)。选择正确的次数通常需要一定的经验和实验基础。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AIgulGRUhpto", - "colab": {} - }, - "source": [ - "## Note: 使用相同的模型变量重新运行此单元\n", - "\n", - "# 保留结果用于绘制\n", - "train_loss_results = []\n", - "train_accuracy_results = []\n", - "\n", - "num_epochs = 201\n", - "\n", - "for epoch in range(num_epochs):\n", - " epoch_loss_avg = tf.keras.metrics.Mean()\n", - " epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n", - "\n", - " # Training loop - using batches of 32\n", - " for x, y in train_dataset:\n", - " # 优化模型\n", - " loss_value, grads = grad(model, x, y)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " # 追踪进度\n", - " epoch_loss_avg(loss_value) # 添加当前的 batch loss\n", - " # 比较预测标签与真实标签\n", - " epoch_accuracy(y, model(x))\n", - "\n", - " # 循环结束\n", - " train_loss_results.append(epoch_loss_avg.result())\n", - " train_accuracy_results.append(epoch_accuracy.result())\n", - "\n", - " if epoch % 50 == 0:\n", - " print(\"Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}\".format(epoch,\n", - " epoch_loss_avg.result(),\n", - " epoch_accuracy.result()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2FQHVUnm_rjw" - }, - "source": [ - "### 可视化损失函数随时间推移而变化的情况" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j3wdbmtLVTyr" - }, - "source": [ - "虽然输出模型的训练过程有帮助，但查看这一过程往往*更有帮助*。 [TensorBoard](https://www.tensorflow.org/guide/summaries_and_tensorboard) 是与 TensorFlow 封装在一起的出色可视化工具，不过我们可以使用 `matplotlib` 模块创建基本图表。\n", - "\n", - "解读这些图表需要一定的经验，不过您确实希望看到*损失*下降且*准确率*上升。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "agjvNd2iUGFn", - "colab": {} - }, - "source": [ - "fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))\n", - "fig.suptitle('Training Metrics')\n", - "\n", - "axes[0].set_ylabel(\"Loss\", fontsize=14)\n", - "axes[0].plot(train_loss_results)\n", - "\n", - "axes[1].set_ylabel(\"Accuracy\", fontsize=14)\n", - "axes[1].set_xlabel(\"Epoch\", fontsize=14)\n", - "axes[1].plot(train_accuracy_results)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zg8GoMZhLpGH" - }, - "source": [ - "## 评估模型的效果\n", - "\n", - "模型已经过训练，现在我们可以获取一些关于其效果的统计信息了。\n", - "\n", - "*评估* 指的是确定模型做出预测的效果。要确定模型在鸢尾花分类方面的效果，请将一些花萼和花瓣测量值传递给模型，并要求模型预测它们所代表的鸢尾花品种。然后，将模型的预测结果与实际标签进行比较。例如，如果模型对一半输入样本的品种预测正确，则 [准确率](https://developers.google.com/machine-learning/glossary/#accuracy) 为 `0.5` 。 图 4 显示的是一个效果更好一些的模型，该模型做出 5 次预测，其中有 4 次正确，准确率为 80%：\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    样本特征标签模型预测
    5.93.04.31.511
    6.93.15.42.122
    5.13.31.70.500
    6.0 3.4 4.5 1.6 12
    5.52.54.01.311
    \n", - " 图 4. 准确率为 80% 的鸢尾花分类器
     \n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z-EvK7hGL0d8" - }, - "source": [ - "### 建立测试数据集\n", - "\n", - "评估模型与训练模型相似。最大的区别在于，样本来自一个单独的[测试集](https://developers.google.com/machine-learning/crash-course/glossary#test_set)，而不是训练集。为了公正地评估模型的效果，用于评估模型的样本务必与用于训练模型的样本不同。\n", - "\n", - "测试 `Dataset` 的建立与训练 `Dataset` 相似。下载 CSV 文本文件并解析相应的值，然后对数据稍加随机化处理：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ps3_9dJ3Lodk", - "colab": {} - }, - "source": [ - "test_url = \"https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv\"\n", - "\n", - "test_fp = tf.keras.utils.get_file(fname=os.path.basename(test_url),\n", - " origin=test_url)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SRMWCu30bnxH", - "colab": {} - }, - "source": [ - "test_dataset = tf.data.experimental.make_csv_dataset(\n", - " test_fp,\n", - " batch_size,\n", - " column_names=column_names,\n", - " label_name='species',\n", - " num_epochs=1,\n", - " shuffle=False)\n", - "\n", - "test_dataset = test_dataset.map(pack_features_vector)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HFuOKXJdMAdm" - }, - "source": [ - "### 根据测试数据集评估模型\n", - "\n", - "与训练阶段不同，模型仅评估测试数据的一个[周期](https://developers.google.com/machine-learning/glossary/#epoch)。在以下代码单元格中，我们会遍历测试集中的每个样本，然后将模型的预测结果与实际标签进行比较。这是为了衡量模型在整个测试集中的准确率。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tw03-MK1cYId", - "colab": {} - }, - "source": [ - "test_accuracy = tf.keras.metrics.Accuracy()\n", - "\n", - "for (x, y) in test_dataset:\n", - " logits = model(x)\n", - " prediction = tf.argmax(logits, axis=1, output_type=tf.int32)\n", - " test_accuracy(prediction, y)\n", - "\n", - "print(\"Test set accuracy: {:.3%}\".format(test_accuracy.result()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HcKEZMtCOeK-" - }, - "source": [ - "例如，我们可以看到对于最后一批数据，该模型通常预测正确：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uNwt2eMeOane", - "colab": {} - }, - "source": [ - "tf.stack([y,prediction],axis=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7Li2r1tYvW7S" - }, - "source": [ - "## 使用经过训练的模型进行预测\n", - "\n", - "我们已经训练了一个模型并“证明”它是有效的，但在对鸢尾花品种进行分类方面，这还不够。现在，我们使用经过训练的模型对 [无标签样本](https://developers.google.com/machine-learning/glossary/#unlabeled_example)（即包含特征但不包含标签的样本）进行一些预测。\n", - "\n", - "在现实生活中，无标签样本可能来自很多不同的来源，包括应用、CSV 文件和数据 Feed。暂时我们将手动提供三个无标签样本以预测其标签。回想一下，标签编号会映射到一个指定的表示法：\n", - "\n", - "* `0`: 山鸢尾\n", - "* `1`: 变色鸢尾\n", - "* `2`: 维吉尼亚鸢尾" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kesTS5Lzv-M2", - "colab": {} - }, - "source": [ - "predict_dataset = tf.convert_to_tensor([\n", - " [5.1, 3.3, 1.7, 0.5,],\n", - " [5.9, 3.0, 4.2, 1.5,],\n", - " [6.9, 3.1, 5.4, 2.1]\n", - "])\n", - "\n", - "predictions = model(predict_dataset)\n", - "\n", - "for i, logits in enumerate(predictions):\n", - " class_idx = tf.argmax(logits).numpy()\n", - " p = tf.nn.softmax(logits)[class_idx]\n", - " name = class_names[class_idx]\n", - " print(\"Example {} prediction: {} ({:4.1f}%)\".format(i, name, 100*p))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/zh-cn/tutorials/distribute/custom_training.ipynb b/site/zh-cn/tutorials/distribute/custom_training.ipynb deleted file mode 100644 index ab6927f5a41..00000000000 --- a/site/zh-cn/tutorials/distribute/custom_training.ipynb +++ /dev/null @@ -1,774 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "custom_training.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MhoQ0WE77laV" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "_ckMIh7O7s6D", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jYysdyb-CaWM" - }, - "source": [ - "# 使用 tf.distribute.Strategy 进行自定义训练" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S5Uhzt6vVIB2" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 TensorFlow.org 上查看\n", - " \n", - " 在 Google Colab 上运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载该 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FbVhjPpzn6BM" - }, - "source": [ - "本教程演示了如何使用 [`tf.distribute.Strategy`](https://www.tensorflow.org/guide/distribute_strategy) 来进行自定义训练循环。 我们将在流行的 MNIST 数据集上训练一个简单的 CNN 模型。 流行的 MNIST 数据集包含了 60000 张尺寸为 28 x 28 的训练图像和 10000 张尺寸为 28 x 28 的测试图像。 \n", - "\n", - "我们用自定义训练循环来训练我们的模型是因为它们在训练的过程中为我们提供了灵活性和在训练过程中更好的控制。而且，使它们调试模型和训练循环的时候更容易。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dzLKpmZICaWN", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# 导入 TensorFlow\n", - "try:\n", - " # %tensorflow_version 仅存在于 Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "# 帮助库\n", - "import numpy as np\n", - "import os\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MM6W__qraV55" - }, - "source": [ - "## 下载流行的 MNIST 数据集" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MqDQO0KCaWS", - "scrolled": false, - "colab": {} - }, - "source": [ - "fashion_mnist = tf.keras.datasets.fashion_mnist\n", - "\n", - "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\n", - "\n", - "# 向数组添加维度 -> 新的维度 == (28, 28, 1)\n", - "# 我们这样做是因为我们模型中的第一层是卷积层\n", - "# 而且它需要一个四维的输入 (批大小, 高, 宽, 通道).\n", - "# 批大小维度稍后将添加。\n", - "train_images = train_images[..., None]\n", - "test_images = test_images[..., None]\n", - "\n", - "# 获取[0,1]范围内的图像。\n", - "train_images = train_images / np.float32(255)\n", - "test_images = test_images / np.float32(255)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4AXoHhrsbdF3" - }, - "source": [ - "## 创建一个分发变量和图形的策略" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5mVuLZhbem8d" - }, - "source": [ - "`tf.distribute.MirroredStrategy` 策略是如何运作的？\n", - "\n", - "* 所有变量和模型图都复制在副本上。\n", - "* 输入都均匀分布在副本中。\n", - "* 每个副本在收到输入后计算输入的损失和梯度。\n", - "* 通过求和，每一个副本上的梯度都能同步。\n", - "* 同步后，每个副本上的复制的变量都可以同样更新。\n", - "\n", - "注意：您可以将下面的所有代码放在一个单独单元内。 我们将它分成几个代码单元用于说明目的。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F2VeZUWUj5S4", - "colab": {} - }, - "source": [ - "# 如果设备未在 `tf.distribute.MirroredStrategy` 的指定列表中，它会被自动检测到。\n", - "strategy = tf.distribute.MirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZngeM_2o0_JO", - "colab": {} - }, - "source": [ - "print ('Number of devices: {}'.format(strategy.num_replicas_in_sync))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k53F5I_IiGyI" - }, - "source": [ - "## 设置输入流水线" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Qb6nDgxiN_n" - }, - "source": [ - "将图形和变量导出成平台不可识别的 SavedModel 格式。在你的模型保存后，你可以在有或没有范围的情况下载入它。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwJtsCQhHK-E", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = len(train_images)\n", - "\n", - "BATCH_SIZE_PER_REPLICA = 64\n", - "GLOBAL_BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n", - "\n", - "EPOCHS = 10" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J7fj3GskHC8g" - }, - "source": [ - "创建数据集并分发它们：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WYrMNNDhAvVl", - "colab": {} - }, - "source": [ - "train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(BUFFER_SIZE).batch(GLOBAL_BATCH_SIZE) \n", - "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE) \n", - "\n", - "train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)\n", - "test_dist_dataset = strategy.experimental_distribute_dataset(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bAXAo_wWbWSb" - }, - "source": [ - "## 创建模型\n", - "\n", - "使用 `tf.keras.Sequential` 创建一个模型。你也可以使用模型子类化 API 来完成这个。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9ODch-OFCaW4", - "colab": {} - }, - "source": [ - "def create_model():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu'),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Conv2D(64, 3, activation='relu'),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - "\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9iagoTBfijUz", - "colab": {} - }, - "source": [ - "# 创建检查点目录以存储检查点。\n", - "checkpoint_dir = './training_checkpoints'\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e-wlFFZbP33n" - }, - "source": [ - "## 定义损失函数\n", - "\n", - "通常，在一台只有一个 GPU / CPU 的机器上，损失需要除去输入批量中的示例数。\n", - "\n", - "*那么，使用 `tf.distribute.Strategy` 时应该如何计算损失？*\n", - "\n", - "* 举一个例子，假设您有4个 GPU，批量大小为64. 输入的一个批次分布在各个副本（4个 GPU）上，每个副本获得的输入大小为16。\n", - "\n", - "* 每个副本上的模型使用其各自的输入执行正向传递并计算损失。 现在，相较于将损耗除以其各自输入中的示例数（BATCH_SIZE_PER_REPLICA = 16），应将损失除以GLOBAL_BATCH_SIZE（64）。\n", - "\n", - "*为什么这样做？*\n", - "\n", - "* 需要这样做是因为在每个副本上计算梯度之后，它们通过 **summing** 来使得在自身在各个副本之间同步。\n", - "\n", - "*如何在TensorFlow中执行此操作？*\n", - "* 如果您正在编写自定义训练循环，如本教程中所示，您应该将每个示例损失相加并将总和除以 GLOBAL_BATCH_SIZE ：\n", - "`scale_loss = tf.reduce_sum(loss) * (1. / GLOBAL_BATCH_SIZE)` 或者你可以使用`tf.nn.compute_average_loss` 来获取每个示例的损失，可选的样本权重，将GLOBAL_BATCH_SIZE作为参数，并返回缩放的损失。\n", - "\n", - "* 如果您在模型中使用正则化损失，则需要进行缩放多个副本的损失。 您可以使用`tf.nn.scale_regularization_loss`函数执行此操作。\n", - "\n", - "* 建议不要使用`tf.reduce_mean`。 这样做会将损失除以实际的每个副本中每一步都会改变的批次大小。\n", - "\n", - "* 这种缩小和缩放是在 keras中 `modelcompile`和`model.fit`中自动完成的\n", - "\n", - "* 如果使用`tf.keras.losses`类（如下面这个例子所示），则需要将损失减少明确指定为“NONE”或者“SUM”。 使用 `tf.distribute.Strategy` 时，`AUTO`和`SUM_OVER_BATCH_SIZE` 是不能使用的。 不能使用 `AUTO` 是因为用户应明确考虑到在分布式情况下他们想做的哪些减少是正确的。不能使用`SUM_OVER_BATCH_SIZE`是因为目前它只按每个副本批次大小进行划分，并按照用户的副本数进行划分，这导致了它们很容易丢失。 因此，我们要求用户要明确这些减少。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R144Wci782ix", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " # 将减少设置为“无”，以便我们可以在之后进行这个减少并除以全局批量大小。\n", - " loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " reduction=tf.keras.losses.Reduction.NONE)\n", - " # 或者使用 loss_fn = tf.keras.losses.sparse_categorical_crossentropy\n", - " def compute_loss(labels, predictions):\n", - " per_example_loss = loss_object(labels, predictions)\n", - " return tf.nn.compute_average_loss(per_example_loss, global_batch_size=GLOBAL_BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w8y54-o9T2Ni" - }, - "source": [ - "## 定义衡量指标以跟踪损失和准确性\n", - "\n", - "这些指标可以跟踪测试的损失，训练和测试的准确性。 您可以使用`.result（）`随时获取累积的统计信息。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zt3AHb46Tr3w", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "\n", - " train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", - " name='train_accuracy')\n", - " test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", - " name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iuKuNXPORfqJ" - }, - "source": [ - "## 训练循环" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OrMmakq5EqeQ", - "colab": {} - }, - "source": [ - "# 必须在`strategy.scope`下创建模型和优化器。\n", - "with strategy.scope():\n", - " model = create_model()\n", - "\n", - " optimizer = tf.keras.optimizers.Adam()\n", - "\n", - " checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=model)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3UX43wUu04EL", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " def train_step(inputs):\n", - " images, labels = inputs\n", - "\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images, training=True)\n", - " loss = compute_loss(labels, predictions)\n", - "\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_accuracy.update_state(labels, predictions)\n", - " return loss \n", - "\n", - " def test_step(inputs):\n", - " images, labels = inputs\n", - "\n", - " predictions = model(images, training=False)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss.update_state(t_loss)\n", - " test_accuracy.update_state(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gX975dMSNw0e", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " # `experimental_run_v2`将复制提供的计算并使用分布式输入运行它。\n", - " @tf.function\n", - " def distributed_train_step(dataset_inputs):\n", - " per_replica_losses = strategy.experimental_run_v2(train_step,\n", - " args=(dataset_inputs,))\n", - " return strategy.reduce(tf.distribute.ReduceOp.SUM, per_replica_losses,\n", - " axis=None)\n", - " \n", - " @tf.function\n", - " def distributed_test_step(dataset_inputs):\n", - " return strategy.experimental_run_v2(test_step, args=(dataset_inputs,))\n", - "\n", - " for epoch in range(EPOCHS):\n", - " # 训练循环\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " for x in train_dist_dataset:\n", - " total_loss += distributed_train_step(x)\n", - " num_batches += 1\n", - " train_loss = total_loss / num_batches\n", - "\n", - " # 测试循环\n", - " for x in test_dist_dataset:\n", - " distributed_test_step(x)\n", - "\n", - " if epoch % 2 == 0:\n", - " checkpoint.save(checkpoint_prefix)\n", - "\n", - " template = (\"Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, \"\n", - " \"Test Accuracy: {}\")\n", - " print (template.format(epoch+1, train_loss,\n", - " train_accuracy.result()*100, test_loss.result(),\n", - " test_accuracy.result()*100))\n", - "\n", - " test_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " test_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z1YvXqOpwy08" - }, - "source": [ - "以上示例中需要注意的事项：\n", - "\n", - "* 我们使用`for x in ...`迭代构造`train_dist_dataset`和`test_dist_dataset`。\n", - "* 缩放损失是`distributed_train_step`的返回值。 这个值会在各个副本使用`tf.distribute.Strategy.reduce`的时候合并，然后通过`tf.distribute.Strategy.reduce`叠加各个返回值来跨批次。\n", - "* 在执行`tf.distribute.Strategy.experimental_run_v2`时，`tf.keras.Metrics`应在`train_step`和`test_step`中更新。\n", - "* `tf.distribute.Strategy.experimental_run_v2`返回策略中每个本地副本的结果，并且有多种方法可以处理此结果。 您可以执行`tf.distribute.Strategy.reduce`来获取汇总值。 您还可以执行`tf.distribute.Strategy.experimental_local_results`来获取每个本地副本中结果中包含的值列表。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-q5qp31IQD8t" - }, - "source": [ - "## 恢复最新的检查点并进行测试" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WNW2P00bkMGJ" - }, - "source": [ - "一个模型使用了`tf.distribute.Strategy`的检查点可以使用策略或者不使用策略进行恢复。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pg3B-Cw_cn3a", - "colab": {} - }, - "source": [ - "eval_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", - " name='eval_accuracy')\n", - "\n", - "new_model = create_model()\n", - "new_optimizer = tf.keras.optimizers.Adam()\n", - "\n", - "test_dataset = tf.data.Dataset.from_tensor_slices((test_images, test_labels)).batch(GLOBAL_BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7qYii7KUYiSM", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def eval_step(images, labels):\n", - " predictions = new_model(images, training=False)\n", - " eval_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LeZ6eeWRoUNq", - "colab": {} - }, - "source": [ - "checkpoint = tf.train.Checkpoint(optimizer=new_optimizer, model=new_model)\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "for images, labels in test_dataset:\n", - " eval_step(images, labels)\n", - "\n", - "print ('Accuracy after restoring the saved model without strategy: {}'.format(\n", - " eval_accuracy.result()*100))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EbcI87EEzhzg" - }, - "source": [ - "## 迭代一个数据集的替代方法\n", - "\n", - "### 使用迭代器\n", - "\n", - "如果你想要迭代一个已经给定步骤数量而不需要整个遍历的数据集，你可以创建一个迭代器并在迭代器上调用`iter`和显式调用`next`。 您可以选择在 tf.function 内部和外部迭代数据集。 这是一个小片段，演示了使用迭代器在 tf.function 外部迭代数据集。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7c73wGC00CzN", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " for _ in range(EPOCHS):\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " train_iter = iter(train_dist_dataset)\n", - "\n", - " for _ in range(10):\n", - " total_loss += distributed_train_step(next(train_iter))\n", - " num_batches += 1\n", - " average_train_loss = total_loss / num_batches\n", - "\n", - " template = (\"Epoch {}, Loss: {}, Accuracy: {}\")\n", - " print (template.format(epoch+1, average_train_loss, train_accuracy.result()*100))\n", - " train_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GxVp48Oy0m6y" - }, - "source": [ - "### 在 tf.function 中迭代\n", - "\n", - "您还可以使用`for x in ...`构造在 tf.function 内部迭代整个输入`train_dist_dataset`，或者像上面那样创建迭代器。下面的例子演示了在 tf.function 中包装一个 epoch 并在功能内迭代`train_dist_dataset`。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-REzmcXv00qm", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " @tf.function\n", - " def distributed_train_epoch(dataset):\n", - " total_loss = 0.0\n", - " num_batches = 0\n", - " for x in dataset:\n", - " per_replica_losses = strategy.experimental_run_v2(train_step,\n", - " args=(x,))\n", - " total_loss += strategy.reduce(\n", - " tf.distribute.ReduceOp.SUM, per_replica_losses, axis=None)\n", - " num_batches += 1\n", - " return total_loss / tf.cast(num_batches, dtype=tf.float32)\n", - "\n", - " for epoch in range(EPOCHS):\n", - " train_loss = distributed_train_epoch(train_dist_dataset)\n", - "\n", - " template = (\"Epoch {}, Loss: {}, Accuracy: {}\")\n", - " print (template.format(epoch+1, train_loss, train_accuracy.result()*100))\n", - "\n", - " train_accuracy.reset_states()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MuZGXiyC7ABR" - }, - "source": [ - "### 跟踪副本中的训练的损失\n", - "\n", - "注意：作为通用的规则，您应该使用`tf.keras.Metrics`来跟踪每个样本的值以避免它们在副本中合并。\n", - "\n", - "我们 *不* 建议使用`tf.metrics.Mean` 来跟踪不同副本的训练损失，因为在执行过程中会进行损失缩放计算。\n", - "\n", - "例如，如果您运行具有以下特点的训练作业：\n", - "* 两个副本\n", - "* 在每个副本上处理两个例子\n", - "* 产生的损失值：每个副本为[2,3]和[4,5]\n", - "* 全局批次大小 = 4\n", - "\n", - "通过损失缩放，您可以通过添加损失值来计算每个副本上的每个样本的损失值，然后除以全局批量大小。 在这种情况下：`（2 + 3）/ 4 = 1.25`和`（4 + 5）/ 4 = 2.25`。\n", - "\n", - "如果您使用 `tf.metrics.Mean` 来跟踪两个副本的损失，结果会有所不同。 在这个例子中，你最终得到一个`total`为 3.50 和`count`为 2 的结果，当调用`result（）`时，你将得到`total` /`count` = 1.75。 使用`tf.keras.Metrics`计算损失时会通过一个等于同步副本数量的额外因子来缩放。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xisYJaV9KZTN" - }, - "source": [ - "### 例子和教程\n", - "以下是一些使用自定义训练循环来分发策略的示例：\n", - "\n", - "1. [教程](../tutorials/distribute/training_loops.ipynb) 使用 `MirroredStrategy` 来训练 MNIST 。\n", - "2. [DenseNet](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/densenet/distributed_train.py) 使用 `MirroredStrategy`的例子。\n", - "1. [BERT](https://github.com/tensorflow/models/blob/master/official/bert/run_classifier.py) 使用 `MirroredStrategy` 和`TPUStrategy`来训练的例子。\n", - "此示例对于了解如何在分发训练过程中如何载入一个检测点和定期生成检查点特别有帮助。\n", - "2. [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) 使用 `MirroredStrategy` 来启用 `keras_use_ctl` 标记。\n", - "3. [NMT](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/nmt_with_attention/distributed_train.py) 使用 `MirroredStrategy`来训练的例子。\n", - "\n", - "更多的例子列在 [分发策略指南](../../guide/distribute_strategy.ipynb#examples_and_tutorials)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6hEJNsokjOKs" - }, - "source": [ - "## 下一步\n", - "\n", - "在你的模型上尝试新的`tf.distribute.Strategy` API。" - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/distribute/keras.ipynb b/site/zh-cn/tutorials/distribute/keras.ipynb deleted file mode 100644 index fc6e44406f2..00000000000 --- a/site/zh-cn/tutorials/distribute/keras.ipynb +++ /dev/null @@ -1,756 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "keras.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "tuOe1ymfHZPu", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# Keras 的分布式训练" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r6P32iYYV27b" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GEe3i16tQPjo", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "## 概述\n", - "\n", - "`tf.distribute.Strategy` API 提供了一个抽象的 API ，用于跨多个处理单元（processing units）分布式训练。它的目的是允许用户使用现有模型和训练代码，只需要很少的修改，就可以启用分布式训练。\n", - "\n", - "本教程使用 `tf.distribute.MirroredStrategy`，这是在一台计算机上的多 GPU（单机多卡）进行同时训练的图形内复制（in-graph replication）。事实上，它会将所有模型的变量复制到每个处理器上，然后，通过使用 [all-reduce](http://mpitutorial.com/tutorials/mpi-reduce-and-allreduce/) 去整合所有处理器的梯度（gradients），并将整合的结果应用于所有副本之中。\n", - "\n", - "`MirroredStategy` 是 tensorflow 中可用的几种分发策略之一。 您可以在 [分发策略指南](../../guide/distribute_strategy.ipynb) 中阅读更多分发策略。\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUXex9ctTuDB" - }, - "source": [ - "### Keras API\n", - "\n", - "这个例子使用 `tf.keras` API 去构建和训练模型。 关于自定义训练模型，请参阅 [tf.distribute.Strategy with training loops](training_loops.ipynb) 教程。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Dney9v7BsJij" - }, - "source": [ - "## 导入依赖" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "r8S3ublR7Ay8", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# 导入 TensorFlow 和 TensorFlow 数据集\n", - "try:\n", - " # %tensorflow_version 只存在于 Colab。\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "tfds.disable_progress_bar()\n", - "\n", - "import os" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hXhefksNKk2I" - }, - "source": [ - "## 下载数据集" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OtnnUwvmB3X5" - }, - "source": [ - "下载 MNIST 数据集并从 [TensorFlow Datasets](https://tensorflow.google.cn/datasets) 加载。 这会返回 `tf.data` 格式的数据集。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lHAPqG8MtS8M" - }, - "source": [ - "将 `with_info` 设置为 `True` 会包含整个数据集的元数据,其中这些数据集将保存在 `info` 中。 除此之外，该元数据对象包括训练和测试示例的数量。 \n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iXMJ3G9NB3X6", - "colab": {} - }, - "source": [ - "datasets, info = tfds.load(name='mnist', with_info=True, as_supervised=True)\n", - "\n", - "mnist_train, mnist_test = datasets['train'], datasets['test']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GrjVhv-eKuHD" - }, - "source": [ - "## 定义分配策略" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TlH8vx6BB3X9" - }, - "source": [ - "创建一个 `MirroredStrategy` 对象。这将处理分配策略，并提供一个上下文管理器（`tf.distribute.MirroredStrategy.scope`）来构建你的模型。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4j0tdf4YB3X9", - "colab": {} - }, - "source": [ - "strategy = tf.distribute.MirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cY3KA_h2iVfN", - "colab": {} - }, - "source": [ - "print('Number of devices: {}'.format(strategy.num_replicas_in_sync))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lNbPv0yAleW8" - }, - "source": [ - "## 设置输入管道（pipeline）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "psozqcuptXhK" - }, - "source": [ - "在训练具有多个 GPU 的模型时，您可以通过增加批量大小（batch size）来有效地使用额外的计算能力。通常来说，使用适合 GPU 内存的最大批量大小（batch size），并相应地调整学习速率。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p1xWxKcnhar9", - "colab": {} - }, - "source": [ - "# 您还可以执行 info.splits.total_num_examples 来获取总数\n", - "# 数据集中的样例数量。\n", - "\n", - "num_train_examples = info.splits['train'].num_examples\n", - "num_test_examples = info.splits['test'].num_examples\n", - "\n", - "BUFFER_SIZE = 10000\n", - "\n", - "BATCH_SIZE_PER_REPLICA = 64\n", - "BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Wm5rsL2KoDF" - }, - "source": [ - "0-255 的像素值， [必须标准化到 0-1 范围](https://en.wikipedia.org/wiki/Feature_scaling)。在函数中定义标准化。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Eo9a46ZeJCkm", - "colab": {} - }, - "source": [ - "def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - "\n", - " return image, label" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WZCa5RLc5A91" - }, - "source": [ - "将此功能应用于训练和测试数据，随机打乱训练数据，并[批量训练](https://tensorflow.google.cn/api_docs/python/tf/data/Dataset#batch)。 请注意，我们还保留了训练数据的内存缓存以提高性能。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gRZu2maChwdT", - "colab": {} - }, - "source": [ - "train_dataset = mnist_train.map(scale).cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n", - "eval_dataset = mnist_test.map(scale).batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4xsComp8Kz5H" - }, - "source": [ - "## 生成模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1BnQYQTpB3YA" - }, - "source": [ - "在 `strategy.scope` 的上下文中创建和编译 Keras 模型。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IexhL_vIB3YA", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - "\n", - " model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8i6OU5W9Vy2u" - }, - "source": [ - "## 定义回调（callback）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YOXO5nvvK3US" - }, - "source": [ - "这里使用的回调（callbacks）是：\n", - "\n", - "* *TensorBoard*: 此回调（callbacks）为 TensorBoard 写入日志，允许您可视化图形。\n", - "* *Model Checkpoint*: 此回调（callbacks）在每个 epoch 后保存模型。\n", - "* *Learning Rate Scheduler*: 使用此回调（callbacks），您可以安排学习率在每个 epoch/batch 之后更改。\n", - "\n", - "为了便于说明，添加打印回调（callbacks）以在笔记本中显示*学习率*。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "A9bwLCcXzSgy", - "colab": {} - }, - "source": [ - "# 定义检查点（checkpoint）目录以存储检查点（checkpoints）\n", - "\n", - "checkpoint_dir = './training_checkpoints'\n", - "# 检查点（checkpoint）文件的名称\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wpU-BEdzJDbK", - "colab": {} - }, - "source": [ - "# 衰减学习率的函数。\n", - "# 您可以定义所需的任何衰减函数。\n", - "def decay(epoch):\n", - " if epoch < 3:\n", - " return 1e-3\n", - " elif epoch >= 3 and epoch < 7:\n", - " return 1e-4\n", - " else:\n", - " return 1e-5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jKhiMgXtKq2w", - "colab": {} - }, - "source": [ - "# 在每个 epoch 结束时打印LR的回调（callbacks）。\n", - "class PrintLR(tf.keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs=None):\n", - " print('\\nLearning rate for epoch {} is {}'.format(epoch + 1,\n", - " model.optimizer.lr.numpy()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YVqAbR6YyNQh", - "colab": {} - }, - "source": [ - "callbacks = [\n", - " tf.keras.callbacks.TensorBoard(log_dir='./logs'),\n", - " tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,\n", - " save_weights_only=True),\n", - " tf.keras.callbacks.LearningRateScheduler(decay),\n", - " PrintLR()\n", - "]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "70HXgDQmK46q" - }, - "source": [ - "## 训练和评估" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6EophnOAB3YD" - }, - "source": [ - "在该部分，以普通的方式训练模型，在模型上调用 `fit` 并传入在教程开始时创建的数据集。 无论您是否分布式训练，此步骤都是相同的。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7MVw_6CqB3YD", - "colab": {} - }, - "source": [ - "model.fit(train_dataset, epochs=12, callbacks=callbacks)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NUcWAUUupIvG" - }, - "source": [ - "如下所示，检查点（checkpoint）将被保存。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JQ4zeSTxKEhB", - "colab": {} - }, - "source": [ - "# 检查检查点（checkpoint）目录\n", - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qor53h7FpMke" - }, - "source": [ - "要查看模型的执行方式，请加载最新的检查点（checkpoint）并在测试数据上调用 `evaluate` 。\n", - "\n", - "使用适当的数据集调用 `evaluate` 。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JtEwxiTgpQoP", - "colab": {} - }, - "source": [ - "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "eval_loss, eval_acc = model.evaluate(eval_dataset)\n", - "\n", - "print('Eval loss: {}, Eval Accuracy: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IIeF2RWfYu4N" - }, - "source": [ - "要查看输出，您可以在终端下载并查看 TensorBoard 日志。\n", - "\n", - "```\n", - "$ tensorboard --logdir=path/to/log-directory\n", - "```" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LnyscOkvKKBR", - "colab": {} - }, - "source": [ - "!ls -sh ./logs" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kBLlogrDvMgg" - }, - "source": [ - "## 导出到 SavedModel" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xa87y_A0vRma" - }, - "source": [ - "将图形和变量导出为与平台无关的 SavedModel 格式。 保存模型后，可以在有或没有 scope 的情况下加载模型。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h8Q4MKOLwG7K", - "colab": {} - }, - "source": [ - "path = 'saved_model/'" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4HvcDmVsvQoa", - "colab": {} - }, - "source": [ - "tf.keras.experimental.export_saved_model(model, path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vKJT4w5JwVPI" - }, - "source": [ - "在无需 `strategy.scope` 加载模型。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "T_gT0RbRvQ3o", - "colab": {} - }, - "source": [ - "unreplicated_model = tf.keras.experimental.load_from_saved_model(path)\n", - "\n", - "unreplicated_model.compile(\n", - " loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])\n", - "\n", - "eval_loss, eval_acc = unreplicated_model.evaluate(eval_dataset)\n", - "\n", - "print('Eval loss: {}, Eval Accuracy: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YBLzcRF0wbDe" - }, - "source": [ - "在含 `strategy.scope` 加载模型。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BBVo3WGGwd9a", - "colab": {} - }, - "source": [ - "with strategy.scope():\n", - " replicated_model = tf.keras.experimental.load_from_saved_model(path)\n", - " replicated_model.compile(loss='sparse_categorical_crossentropy',\n", - " optimizer=tf.keras.optimizers.Adam(),\n", - " metrics=['accuracy'])\n", - "\n", - " eval_loss, eval_acc = replicated_model.evaluate(eval_dataset)\n", - " print ('Eval loss: {}, Eval Accuracy: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUZwaz4AKjtD" - }, - "source": [ - "### 示例和教程\n", - "以下是使用 keras fit/compile 分布式策略的一些示例：\n", - "1. 使用`tf.distribute.MirroredStrategy` 训练 [Transformer](https://github.com/tensorflow/models/blob/master/official/transformer/v2/transformer_main.py) 的示例。\n", - "2. 使用`tf.distribute.MirroredStrategy` 训练 [NCF](https://github.com/tensorflow/models/blob/master/official/recommendation/ncf_keras_main.py) 的示例。\n", - "\n", - "[分布式策略指南](../../guide/distribute_strategy.ipynb#examples_and_tutorials)中列出的更多示例 " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8uNqWRdDMl5S" - }, - "source": [ - "## 下一步\n", - "\n", - "* 阅读[分布式策略指南](../../guide/distribute_strategy.ipynb)。\n", - "* 阅读[自定义训练的分布式训练](training_loops.ipynb)教程。\n", - "\n", - "注意：`tf.distribute.Strategy` 正在积极开发中，我们将在不久的将来添加更多示例和教程。欢迎您进行尝试。我们欢迎您通过[ GitHub 上的 issue ](https://github.com/tensorflow/tensorflow/issues/new) 提供反馈。" - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/distribute/multi_worker_with_estimator.ipynb b/site/zh-cn/tutorials/distribute/multi_worker_with_estimator.ipynb deleted file mode 100644 index 444d5e6834d..00000000000 --- a/site/zh-cn/tutorials/distribute/multi_worker_with_estimator.ipynb +++ /dev/null @@ -1,398 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "multi_worker_with_estimator_copy.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "Tce3stUlHN0L" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tuOe1ymfHZPu", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# 利用 Estimator 进行多工作器训练\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    \n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GEe3i16tQPjo", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "## 概述\n", - "本教程展示了在训练分布式多工作器（worker）时，如何使用 `tf.distribute.Strategy`。如果你的代码使用了 `tf.estimator`，而且你也对拓展单机以获取高性能有兴趣，那么这个教程就是为你准备的。\n", - "\n", - "在开始之前，请先阅读 [`tf.distribute.Strategy` 指南](../../guide/distribute_strategy.ipynb)。同样相关的还有 [使用多 GPU 训练教程](./keras.ipynb)，因为在这个教程里也使用了相同的模型。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUXex9ctTuDB" - }, - "source": [ - "## 创建\n", - "\n", - "首先，设置好 TensorFlow 以及将会用到的输入模块。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IqR2PQG4ZaZ0", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals " - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bnYxvfLD-LW-", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "tfds.disable_progress_bar()\n", - "\n", - "import os, json" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hPBuZUNSZmrQ" - }, - "source": [ - "## 输入函数\n", - "\n", - "本教程里我们使用的是 [TensorFlow 数据集（TensorFlow Datasets）](https://www.tensorflow.org/datasets)里的 MNIST 数据集。本教程里的代码和 [使用多 GPU 训练教程](./keras.ipynb) 类似，但有一个主要区别：当我们使用 Estimator 进行多工作器训练时，需要根据工作器的数量对数据集进行拆分，以确保模型收敛。输入的数据根据工作器其自身的索引来拆分，因此每个工作器各自负责处理该数据集 `1/num_workers` 个不同部分。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dma_wUAxZqo2", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 10000\n", - "BATCH_SIZE = 64\n", - "\n", - "def input_fn(mode, input_context=None):\n", - " datasets, info = tfds.load(name='mnist',\n", - " with_info=True,\n", - " as_supervised=True)\n", - " mnist_dataset = (datasets['train'] if mode == tf.estimator.ModeKeys.TRAIN else\n", - " datasets['test'])\n", - "\n", - " def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - " return image, label\n", - "\n", - " if input_context:\n", - " mnist_dataset = mnist_dataset.shard(input_context.num_input_pipelines,\n", - " input_context.input_pipeline_id)\n", - " return mnist_dataset.map(scale).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4BlcVXMhB59T" - }, - "source": [ - "使模型收敛的另一种合理方式是在每个工作器上设置不同的随机种子，然后对数据集进行随机重排。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8YFpxrcsZ2xG" - }, - "source": [ - "## 多工作器配置\n", - "\n", - "本教程主要的不同（区别于[使用多 GPU 训练教程](./keras.ipynb)）在于多工作器的创建。明确集群中每个工作器的配置的标准方式是设置环境变量 `TF_CONFIG` 。\n", - "\n", - "`TF_CONFIG` 里包括了两个部分：`cluster` 和 `task`。`cluster` 提供了关于整个集群的信息，也就是集群中的工作器和参数服务器（parameter server）。`task` 提供了关于当前任务的信息。在本例中，任务的类型（type）是 worker 且该任务的索引（index）是 0。\n", - "\n", - "出于演示的目的，本教程展示了怎么将 `TF_CONFIG` 设置成两个本地的工作器。在实践中，你可以在外部的IP地址和端口上创建多个工作器，并为每个工作器正确地配置好 `TF_CONFIG` 变量，也就是更改任务的索引。\n", - "\n", - "警告：不要在 Colab 里执行以下代码。TensorFlow 的运行程序会试图在指定的 IP 地址和端口创建 gRPC 服务器，这会导致创建失败。\n", - "\n", - "```\n", - "os.environ['TF_CONFIG'] = json.dumps({\n", - " 'cluster': {\n", - " 'worker': [\"localhost:12345\", \"localhost:23456\"]\n", - " },\n", - " 'task': {'type': 'worker', 'index': 0}\n", - "})\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qDreJzTffAP5" - }, - "source": [ - "## 定义模型\n", - "\n", - "定义训练中用到的层，优化器和损失函数。本教程使用 Keras layers 定义模型，同[使用多 GPU 训练教程](./keras.ipynb)类似。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WNvOn_OeiUYC", - "colab": {} - }, - "source": [ - "LEARNING_RATE = 1e-4\n", - "def model_fn(features, labels, mode):\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - " logits = model(features, training=False)\n", - "\n", - " if mode == tf.estimator.ModeKeys.PREDICT:\n", - " predictions = {'logits': logits}\n", - " return tf.estimator.EstimatorSpec(labels=labels, predictions=predictions)\n", - "\n", - " optimizer = tf.compat.v1.train.GradientDescentOptimizer(\n", - " learning_rate=LEARNING_RATE)\n", - " loss = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " from_logits=True, reduction=tf.keras.losses.Reduction.NONE)(labels, logits)\n", - " loss = tf.reduce_sum(loss) * (1. / BATCH_SIZE)\n", - " if mode == tf.estimator.ModeKeys.EVAL:\n", - " return tf.estimator.EstimatorSpec(mode, loss=loss)\n", - "\n", - " return tf.estimator.EstimatorSpec(\n", - " mode=mode,\n", - " loss=loss,\n", - " train_op=optimizer.minimize(\n", - " loss, tf.compat.v1.train.get_or_create_global_step()))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P94PrIW_kSCE" - }, - "source": [ - "注意：尽管在本例中学习率是固定的，但是通常情况下可能有必要基于全局的批次大小对学习率进行调整。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UhNtHfuxCGVy" - }, - "source": [ - "## MultiWorkerMirroredStrategy\n", - "\n", - "为训练模型，需要使用 `tf.distribute.experimental.MultiWorkerMirroredStrategy` 实例。`MultiWorkerMirroredStrategy` 创建了每个设备中模型层里所有变量的拷贝，且是跨工作器的。其用到了 `CollectiveOps`，这是 TensorFlow 里的一种操作，用来整合梯度以及确保变量同步。该策略的更多细节可以在 [`tf.distribute.Strategy` 指南](../../guide/distribute_strategy.ipynb)中找到。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1uFSHCJXMrQ-", - "colab": {} - }, - "source": [ - "strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H47DDcOgfzm7" - }, - "source": [ - "## 训练和评估模型\n", - "接下来，在 `RunConfig` 中为 estimator 指明分布式策略，同时通过调用 `tf.estimator.train_and_evaluate` 训练和评估模型。本教程只通过指明 `train_distribute` 进行分布式训练。但是也同样也可以通过指明 `eval_distribute` 来进行分布式评估。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BcsuBYrpgnlS", - "colab": {} - }, - "source": [ - "config = tf.estimator.RunConfig(train_distribute=strategy)\n", - "\n", - "classifier = tf.estimator.Estimator(\n", - " model_fn=model_fn, model_dir='/tmp/multiworker', config=config)\n", - "tf.estimator.train_and_evaluate(\n", - " classifier,\n", - " train_spec=tf.estimator.TrainSpec(input_fn=input_fn),\n", - " eval_spec=tf.estimator.EvalSpec(input_fn=input_fn)\n", - ")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XVk4ftYx6JAO" - }, - "source": [ - "# 优化训练后的模型性能\n", - "\n", - "现在你已经有了由 `tf.distribute.Strategy` 的模型和能支持多工作器的 Estimator。你可以尝试使用下列技巧来优化多工作器训练的性能。\n", - "\n", - "* *增加单批次的大小：* 此处的批次大小指的是每个 GPU 上的批次大小。通常来说，最大的批次大小应该适应 GPU 的内存大小。\n", - "* *变量转换：* 尽可能将变量转换成 `tf.float`。官方的 ResNet 模型包括了如何完成的[样例](https://github.com/tensorflow/models/blob/8367cf6dabe11adf7628541706b660821f397dce/official/resnet/resnet_model.py#L466)。\n", - "* *使用集群通信：* `MultiWorkerMirroredStrategy` 提供了好几种[集群通信的实现](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/distribute/cross_device_ops.py). \n", - " * `RING` 实现了基于环状的集群，使用了 gRPC 作为跨主机通讯层。 \n", - " * `NCCL` 使用了 [英伟达的 NCCL](https://developer.nvidia.com/nccl) 来实现集群。\n", - " * `AUTO` 将选择延后至运行时。\n", - "\n", - "集群实现的最优选择不仅基于 GPU 的数量和种类，也基于集群间的通信网络。想要覆盖自动的选项，需要指明 `MultiWorkerMirroredStrategy` 的构造器里的 `communication` 参数，例如让 `communication=tf.distribute.experimental.CollectiveCommunication.NCCL` 。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AW0Hb2xM6EGX" - }, - "source": [ - "## 更多的代码示例\n", - "\n", - "1. [端到端的示例](https://github.com/tensorflow/ecosystem/tree/master/distribution_strategy)里使用了 Kubernetes 模板。在这个例子里我们一开始使用了 Keras 模型，并使用了 `tf.keras.estimator.model_to_estimator` API 将其转换成了 Estimator。\n", - "2. 官方的 [ResNet50](https://github.com/tensorflow/models/blob/master/official/resnet/imagenet_main.py) 模型，我们可以使用 `MirroredStrategy` 或 `MultiWorkerMirroredStrategy` 来训练它。" - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/distribute/multi_worker_with_keras.ipynb b/site/zh-cn/tutorials/distribute/multi_worker_with_keras.ipynb deleted file mode 100644 index 78a3feb88f9..00000000000 --- a/site/zh-cn/tutorials/distribute/multi_worker_with_keras.ipynb +++ /dev/null @@ -1,507 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tce3stUlHN0L" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "tuOe1ymfHZPu" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfBg1C5NB3X0" - }, - "source": [ - "# 利用 Keras 来训练多工作器（worker）\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GEe3i16tQPjo" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xHxb-dlhMIzW" - }, - "source": [ - "## 概述\n", - "\n", - "本教程使用 `tf.distribute.Strategy` API 演示了使用 Keras 模型的多工作器（worker）分布式培训。借助专为多工作器（worker）训练而设计的策略，设计在单一工作器（worker）上运行的 Keras 模型可以在最少的代码更改的情况下无缝地处理多个工作器。\n", - "\n", - "[TensorFlow 中的分布式培训](../../guide/distribute_strategy.ipynb)指南可用于概述TensorFlow支持的分布式策略，并想要更深入理解`tf.distribute.Strategy` API 感兴趣的人。\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MUXex9ctTuDB" - }, - "source": [ - "## 配置\n", - "\n", - "首先，设置 TensorFlow 和必要的导入。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IqR2PQG4ZaZ0" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "bnYxvfLD-LW-" - }, - "outputs": [], - "source": [ - "try:\n", - " # % tensorflow_version仅存在于Colab中。\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "tfds.disable_progress_bar()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hPBuZUNSZmrQ" - }, - "source": [ - "## 准备数据集\n", - "\n", - "现在，让我们从 [TensorFlow 数据集](https://tensorflow.google.cn/datasets) 中准备MNIST数据集。 [MNIST 数据集](http://yann.lecun.com/exdb/mnist/) 包括60,000个训练样本和10,000个手写数字0-9的测试示例，格式为28x28像素单色图像。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "dma_wUAxZqo2" - }, - "outputs": [], - "source": [ - "BUFFER_SIZE = 10000\n", - "BATCH_SIZE = 64\n", - "\n", - "# 将 MNIST 数据从 (0, 255] 缩放到 (0., 1.]\n", - "def scale(image, label):\n", - " image = tf.cast(image, tf.float32)\n", - " image /= 255\n", - " return image, label\n", - "\n", - "datasets, info = tfds.load(name='mnist',\n", - " with_info=True,\n", - " as_supervised=True)\n", - "\n", - "train_datasets_unbatched = datasets['train'].map(scale).cache().shuffle(BUFFER_SIZE)\n", - "train_datasets = train_datasets_unbatched.batch(BATCH_SIZE)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o87kcvu8GR4-" - }, - "source": [ - "## 构建 Keras 模型\n", - "在这里，我们使用`tf.keras.Sequential` API来构建和编译一个简单的卷积神经网络 Keras 模型，用我们的 MNIST 数据集进行训练。\n", - "\n", - "注意：有关构建 Keras 模型的详细训练说明，请参阅[TensorFlow Keras 指南](https://tensorflow.google.cn/guide/keras#sequential_model)。\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "aVPHl0SfG2v1" - }, - "outputs": [], - "source": [ - "def build_and_compile_cnn_model():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Conv2D(32, 3, activation='relu', input_shape=(28, 28, 1)),\n", - " tf.keras.layers.MaxPooling2D(),\n", - " tf.keras.layers.Flatten(),\n", - " tf.keras.layers.Dense(64, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - " model.compile(\n", - " loss=tf.keras.losses.sparse_categorical_crossentropy,\n", - " optimizer=tf.keras.optimizers.SGD(learning_rate=0.001),\n", - " metrics=['accuracy'])\n", - " return model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2UL3kisMO90X" - }, - "source": [ - "让我们首先尝试用少量的 epoch 来训练模型，并在单个工作器（worker）中观察结果，以确保一切正常。 随着训练的迭代，您应该会看到损失（loss）下降和准确度（accuracy）接近1.0。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "6Qe6iAf5O8iJ" - }, - "outputs": [], - "source": [ - "single_worker_model = build_and_compile_cnn_model()\n", - "single_worker_model.fit(x=train_datasets, epochs=3)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8YFpxrcsZ2xG" - }, - "source": [ - "## 多工作器（worker）配置\n", - "\n", - "现在让我们进入多工作器（worker)训练的世界。在 TensorFlow 中，需要 `TF_CONFIG` 环境变量来训练多台机器，每台机器可能具有不同的角色。 `TF_CONFIG`用于指定作为集群一部分的每个 worker 的集群配置。\n", - "\n", - "`TF_CONFIG` 有两个组件：`cluster` 和 `task` 。 `cluster` 提供有关训练集群的信息，这是一个由不同类型的工作组成的字典，例如 `worker` 。在多工作器（worker）培训中，除了常规的“工作器”之外，通常还有一个“工人”承担更多责任，比如保存检查点和为 TensorBoard 编写摘要文件。这样的工作器（worker）被称为“主要”工作者，习惯上`worker` 中 `index` 0被指定为主要的 `worker`（事实上这就是`tf.distribute.Strategy`的实现方式）。 另一方面，`task` 提供当前任务的信息。\n", - "\n", - "在这个例子中，我们将任务 `type` 设置为 `\"worker\"` 并将任务 `index` 设置为 `0` 。这意味着具有这种设置的机器是第一个工作器，它将被指定为主要工作器并且要比其他工作器做更多的工作。请注意，其他机器也需要设置 `TF_CONFIG` 环境变量，它应该具有相同的 `cluster` 字典，但是不同的任务`type` 或 `index` 取决于这些机器的角色。\n", - "\n", - "为了便于说明，本教程展示了如何在 `localhost` 上设置一个带有2个工作器的`TF_CONFIG`。 实际上，用户会在外部IP地址/端口上创建多个工作器，并在每个工作器上适当地设置`TF_CONFIG`。\n", - "\n", - "警告：不要在 Colab 中执行以下代码。TensorFlow 的运行时将尝试在指定的IP地址和端口创建 gRPC 服务器，这可能会失败。\n", - "\n", - "```\n", - "os.environ['TF_CONFIG'] = json.dumps({\n", - " 'cluster': {\n", - " 'worker': [\"localhost:12345\", \"localhost:23456\"]\n", - " },\n", - " 'task': {'type': 'worker', 'index': 0}\n", - "})\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P94PrIW_kSCE" - }, - "source": [ - "注意，虽然在该示例中学习速率是固定的，但是通常可能需要基于全局批量大小来调整学习速率。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UhNtHfuxCGVy" - }, - "source": [ - "## 选择正确的策略\n", - "\n", - "在 TensorFlow 中，分布式训练包括同步训练（其中训练步骤跨工作器和副本同步）、异步训练（训练步骤未严格同步）。\n", - "\n", - "`MultiWorkerMirroredStrategy` 是同步多工作器训练的推荐策略，将在本指南中进行演示。\n", - "\n", - "要训练模型，请使用 `tf.distribute.experimental.MultiWorkerMirroredStrategy` 的实例。 `MultiWorkerMirroredStrategy` 在所有工作器的每台设备上创建模型层中所有变量的副本。 它使用 `CollectiveOps` ，一个用于集体通信的 TensorFlow 操作，来聚合梯度并使变量保持同步。 [`tf.distribute.Strategy`指南](../../guide/distribute_strategy.ipynb)有关于此策略的更多详细信息。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "1uFSHCJXMrQ-" - }, - "outputs": [], - "source": [ - "strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "N0iv7SyyAohc" - }, - "source": [ - "注意：解析 `TF_CONFIG` 并且在调用 `MultiWorkerMirroredStrategy.__init__()` 时启动 TensorFlow 的 GRPC 服务器，因此必须在创建`tf.distribute.Strategy`实例之前设置 `TF_CONFIG` 环境变量。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FMy2VM4Akzpr" - }, - "source": [ - "`MultiWorkerMirroredStrategy` 通过[`CollectiveCommunication`](https://github.com/tensorflow/tensorflow/blob/a385a286a930601211d78530734368ccb415bee4/tensorflow/python/distribute/cross_device_ops.py#L928)参数提供多个实现。`RING` 使用 gRPC 作为跨主机通信层实现基于环的集合。`NCCL` 使用[Nvidia 的 NCCL](https://developer.nvidia.com/nccl)来实现集体。 `AUTO` 将选择推迟到运行时。 集体实现的最佳选择取决于GPU的数量和种类以及群集中的网络互连。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H47DDcOgfzm7" - }, - "source": [ - "## 使用 MultiWorkerMirroredStrategy 训练模型\n", - "\n", - "通过将 `tf.distribute.Strategy` API集成到 `tf.keras` 中，将训练分发给多人的唯一更改就是将模型进行构建和 `model.compile()` 调用封装在 `strategy.scope()` 内部。 分发策略的范围决定了如何创建变量以及在何处创建变量，对于 MultiWorkerMirroredStrategy 而言，创建的变量为 MirroredVariable ，并且将它们复制到每个工作器上。\n", - "\n", - "注意：在此Colab中，以下代码可以按预期结果运行，但是由于未设置`TF_CONFIG`，因此这实际上是单机训练。 在您自己的示例中设置了 `TF_CONFIG` 后，您应该期望在多台机器上进行培训可以提高速度。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "BcsuBYrpgnlS" - }, - "outputs": [], - "source": [ - "NUM_WORKERS = 2\n", - "# 由于`tf.data.Dataset.batch`需要全局的批处理大小， \n", - "# 因此此处的批处理大小按工作器数量增加。 \n", - "# 以前我们使用64，现在变成128。\n", - "GLOBAL_BATCH_SIZE = 64 * NUM_WORKERS\n", - "train_datasets = train_datasets_unbatched.batch(GLOBAL_BATCH_SIZE)\n", - "with strategy.scope():\n", - " multi_worker_model = build_and_compile_cnn_model()\n", - "multi_worker_model.fit(x=train_datasets, epochs=3)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Rr14Vl9GR4zq" - }, - "source": [ - "### 数据集分片和批（batch）大小\n", - "\n", - "在多工作器训练中，需要将数据分片为多个部分，以确保融合和性能。 但是，请注意，在上面的代码片段中，数据集直接发送到`model.fit（）`，而无需分片； 这是因为`tf.distribute.Strategy` API在多工作器训练中会自动处理数据集分片。\n", - "\n", - "如果您喜欢手动分片进行训练，则可以通过`tf.data.experimental.DistributeOptions` API关闭自动分片。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "JxEtdh1vH-TF" - }, - "outputs": [], - "source": [ - "options = tf.data.Options()\n", - "options.experimental_distribute.auto_shard = False\n", - "train_datasets_no_auto_shard = train_datasets.with_options(options)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NBCtYvmCH-7g" - }, - "source": [ - "要注意的另一件事是 `datasets` 的批处理大小。 在上面的代码片段中，我们使用 `GLOBAL_BATCH_SIZE = 64 * NUM_WORKERS` ，这是单个工作器的大小的 `NUM_WORKERS` 倍，因为每个工作器的有效批量大小是全局批量大小（参数从 `tf.data.Dataset.batch()` 传入）除以工作器的数量，通过此更改，我们使每个工作器的批处理大小与以前相同。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "XVk4ftYx6JAO" - }, - "source": [ - "## 性能\n", - "\n", - "现在，您已经有了一个Keras模型，该模型全部通过 `MultiWorkerMirroredStrategy` 运行在多个工作器中。 您可以尝试以下技术来调整多工作器训练的效果。\n", - "\n", - "* `MultiWorkerMirroredStrategy` 提供了多个[集体通信实现][collective communication implementations](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/distribute/cross_device_ops.py). `RING` 使用gRPC作为跨主机通信层实现基于环的集合。 `NCCL` 使用 [Nvidia's NCCL](https://developer.nvidia.com/nccl) 来实现集合。 `AUTO` 将推迟到运行时选择。集体实施的最佳选择取决于GPU的数量和种类以及集群中的网络互连。 要覆盖自动选择，请为 `MultiWorkerMirroredStrategy` 的构造函数的 `communication` 参数指定一个有效值，例如： `communication=tf.distribute.experimental.CollectiveCommunication.NCCL`.\n", - "* 如果可能的话，将变量强制转换为 `tf.float`。ResNet 的官方模型包括如何完成此操作的[示例](https://github.com/tensorflow/models/blob/8367cf6dabe11adf7628541706b660821f397dce/official/resnet/resnet_model.py#L466)。\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "97WhAu8uKw3j" - }, - "source": [ - "## 容错能力\n", - "\n", - "在同步训练中，如果其中一个工作器出现故障并且不存在故障恢复机制，则集群将失败。 在工作器退出或不稳定的情况下，将 Keras 与 `tf.distribute.Strategy` 一起使用会具有容错的优势。 我们通过在您选择的分布式文件系统中保留训练状态来做到这一点，以便在重新启动先前失败或被抢占的实例后，将恢复训练状态。\n", - "\n", - "由于所有工作器在训练 epochs 和 steps 方面保持同步，因此其他工作器将需要等待失败或被抢占的工作器重新启动才能继续。\n", - "\n", - "### ModelCheckpoint 回调\n", - "\n", - "要在多工作器训练中利用容错功能，请在调用 `tf.keras.Model.fit()` 时提供一个 `tf.keras.callbacks.ModelCheckpoint` 实例。 回调会将检查点和训练状态存储在与 `ModelCheckpoint` 的 `filepath` 参数相对应的目录中。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "xIY9vKnUU82o" - }, - "outputs": [], - "source": [ - "# 将`filepath`参数替换为文件系统中的路径\n", - "# 使所有工作器能够访问到\n", - "callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath='/tmp/keras-ckpt')]\n", - "with strategy.scope():\n", - " multi_worker_model = build_and_compile_cnn_model()\n", - "multi_worker_model.fit(x=train_datasets, epochs=3, callbacks=callbacks)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ii6VmEdOjkZr" - }, - "source": [ - "如果某个工作线程被抢占，则整个集群将暂停，直到重新启动被抢占的工作线程为止。工作器重新加入集群后，其他工作器也将重新启动。 现在，每个工作器都将读取先前保存的检查点文件，并获取其以前的状态，从而使群集能够恢复同步，然后继续训练。\n", - "\n", - "如果检查包含在`ModelCheckpoint` 中指定的 `filepath` 的目录，则可能会注意到一些临时生成的检查点文件。 这些文件是恢复以前丢失的实例所必需的，并且在成功退出多工作器训练后，这些文件将在 `tf.keras.Model.fit()` 的末尾被库删除。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ega2hdOQEmy_" - }, - "source": [ - "## 您可以查阅\n", - "1. [Distributed Training in TensorFlow](https://www.tensorflow.org/guide/distribute_strategy) 该指南概述了可用的分布式策略。\n", - "2. [ResNet50](https://github.com/tensorflow/models/blob/master/official/resnet/imagenet_main.py) 官方模型，该模型可以使用 `MirroredStrategy` 或 `MultiWorkerMirroredStrategy` 进行训练" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "multi_worker_with_keras.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/estimator/boosted_trees.ipynb b/site/zh-cn/tutorials/estimator/boosted_trees.ipynb deleted file mode 100644 index 21fbb2eaf63..00000000000 --- a/site/zh-cn/tutorials/estimator/boosted_trees.ipynb +++ /dev/null @@ -1,1115 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "boosted_trees.ipynb", - "provenance": [], - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7765UFHoyGx6" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KVtTDrUNyL7x", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xPYxZMrWyA0N" - }, - "source": [ - "# 在 Tensorflow 中训练提升树（Boosted Trees）模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p_vOREjRx-Y0" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorFlow.google.cn 上查看\n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ucwnDr3gXTmT", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dW3r7qVxzqN5" - }, - "source": [ - "本教程是使用基于 `tf.estimator` API的决策树来训练梯度提升模型的端到端演示。提升树（Boosted Trees）模型是回归和分类问题中最受欢迎并最有效的机器学习方法之一。这是一种融合技术，它结合了几个（10 个，100 个或者甚至 1000 个）树模型的预测值。\n", - "\n", - "提升树（Boosted Trees）模型受到许多机器学习从业者的欢迎，因为它们可以通过最小化的超参数调整获得令人印象深刻的性能。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eylrTPAN3rJV" - }, - "source": [ - "## 加载泰坦尼克数据集\n", - "\n", - "您将使用泰坦尼克数据集，该数据集的目标是在给出性别、年龄、阶级等特征的条件下预测乘客幸存与否。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KuhAiPfZ3rJW", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from IPython.display import clear_output\n", - "from matplotlib import pyplot as plt\n", - "\n", - "# 加载数据集。\n", - "dftrain = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/train.csv')\n", - "dfeval = pd.read_csv('https://storage.googleapis.com/tf-datasets/titanic/eval.csv')\n", - "y_train = dftrain.pop('survived')\n", - "y_eval = dfeval.pop('survived')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NFtnFm1T0kMf", - "colab": {} - }, - "source": [ - "!pip install tf-nightly-2.0-preview\n", - "import tensorflow as tf\n", - "tf.random.set_seed(123)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3ioodHdVJVdA" - }, - "source": [ - "数据集由训练集和验证集组成：\n", - "\n", - "* `dftrain` 与 `y_train` 是*训练集*——模型用来学习的数据。\n", - "* 模型根据*评估集*，`dfeval` 和 `y_eval` 进行测试。\n", - "\n", - "您将使用以下特征来进行训练：\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    特征名称描述
    sex乘客的性别
    age乘客的年龄
    n_siblings_spouses船上的兄弟姐妹与伙伴
    parch船上的父母与孩子
    fare乘客所支付的票价
    class乘客在船上的舱室等级
    deck哪个甲板上的乘客
    embark_town乘客是从哪个城镇上船的
    alone是否乘客独自一人
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AoPiWsJALr-k" - }, - "source": [ - "## 探索数据" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "slcat1yzmzw5" - }, - "source": [ - "让我们首先预览一些数据，并在训练集上创建摘要统计。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "15PLelXBlxEW", - "outputId": "b300d1e6-b3c2-476e-c7a2-a5a00d3ef03f", - "colab": { - "height": 204 - } - }, - "source": [ - "dftrain.head()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    sexagen_siblings_spousesparchfareclassdeckembark_townalone
    0male22.0107.2500ThirdunknownSouthamptonn
    1female38.01071.2833FirstCCherbourgn
    2female26.0007.9250ThirdunknownSouthamptony
    3female35.01053.1000FirstCSouthamptonn
    4male28.0008.4583ThirdunknownQueenstowny
    \n", - "
    " - ], - "text/plain": [ - " sex age n_siblings_spouses parch ... class deck embark_town alone\n", - "0 male 22.0 1 0 ... Third unknown Southampton n\n", - "1 female 38.0 1 0 ... First C Cherbourg n\n", - "2 female 26.0 0 0 ... Third unknown Southampton y\n", - "3 female 35.0 1 0 ... First C Southampton n\n", - "4 male 28.0 0 0 ... Third unknown Queenstown y\n", - "\n", - "[5 rows x 9 columns]" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 0 - } - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "j2hiM4ETmqP0", - "outputId": "e8b8cc92-0a70-44bd-b7a6-54b16bb0ed63", - "colab": { - "height": 297 - } - }, - "source": [ - "dftrain.describe()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/html": [ - "
    \n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
    agen_siblings_spousesparchfare
    count627.000000627.000000627.000000627.000000
    mean29.6313080.5454550.37958534.385399
    std12.5118181.1510900.79299954.597730
    min0.7500000.0000000.0000000.000000
    25%23.0000000.0000000.0000007.895800
    50%28.0000000.0000000.00000015.045800
    75%35.0000001.0000000.00000031.387500
    max80.0000008.0000005.000000512.329200
    \n", - "
    " - ], - "text/plain": [ - " age n_siblings_spouses parch fare\n", - "count 627.000000 627.000000 627.000000 627.000000\n", - "mean 29.631308 0.545455 0.379585 34.385399\n", - "std 12.511818 1.151090 0.792999 54.597730\n", - "min 0.750000 0.000000 0.000000 0.000000\n", - "25% 23.000000 0.000000 0.000000 7.895800\n", - "50% 28.000000 0.000000 0.000000 15.045800\n", - "75% 35.000000 1.000000 0.000000 31.387500\n", - "max 80.000000 8.000000 5.000000 512.329200" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 0 - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-IR0e8V-LyJ4" - }, - "source": [ - "训练集和评估集分别有 627 和 264 个样本。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_1NwYqGwDjFf", - "outputId": "b9901780-2cbc-4731-c040-21b79a245c3f", - "colab": { - "height": 34 - } - }, - "source": [ - "dftrain.shape[0], dfeval.shape[0]" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "(627, 264)" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 0 - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "28UFJ4KSMK3V" - }, - "source": [ - "大多数乘客在 20 岁或 30 岁。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CaVDmZtuDfux", - "outputId": "df4752dc-9976-4db1-9f09-6a057c1b6f46", - "colab": { - "height": 279 - } - }, - "source": [ - "dftrain.age.hist(bins=20)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEGCAYAAACHGfl5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFddJREFUeJzt3W1sU+fBxvHLcRKHEQxOljqpw4agH8KLphCCmMQeaU2E\nBisUoX5oeFmzql2l0YSNLmK0S0jKS6cQOqYmgoGmSWiiQdVWYAbBYE2l6ZkmQAtoUWHa2KhgwQHm\nEMxLmjD7fj4g/GDyHp/gY/L/SZWa+/g+vjgnyZVzfHzsMMYYAQDGtZREBwAAJB5lAACgDAAAlAEA\nQJQBAEA2LYNQKKTGxkaFQqFERxkUOa1FTuskQ0aJnFaLJ+eQZdDV1aU33nhDS5Ys0fLly7Vu3Trd\nvHlTknTu3DktX75cixcv1muvvabOzs7ovMGWDecf1NTUlBQbnpzWIad1kiGjRE6rxZNzyDJwOBz6\n3ve+p2PHjunw4cPKz8/X+++/L0nasGGD6urqdPz4cRUXF2vHjh3ReYMtAwDYy5BlMHnyZM2fPz/6\ndWFhoa5evaq2tja5XC7NnTtXklRWVqZjx45J0qDLAAD2M6LXDIwxam5uVklJiQKBgHw+X3SZx+OR\n9OAwZbBlAAD7SR3Jgzdv3qyJEydqzZo1OnHiRJ/lg93ZYqBloVCoT0l0dHSoqKhITqdzJPGeOKfT\nKZ/PR06LkNM6yZBRIqfVnE6nioqK1NHR0WeZ2+2W2+0ecK5juPcmqq+v19///nft2bNHqampamtr\n0zvvvCO/3y9J6uzsVGlpqc6ePTvossc1NjaqqakpZqyoqEjNzc3DiQUAeMzKlSvV2toaM1ZRUaHK\nysoB5wzryGDnzp06f/689u7dq9TUB1PmzJmjnp4etba2qqioSAcOHNCSJUuGXPa48vJyrVixImbs\nYfvevHlXkYi976OXnZ2pYPBOomMMiZzWSoacyZBRIqeVUlIc8ngm6mc/+5nC4XDMssGOCqRhlMHF\nixe1d+9eTZs2TS+//LIkaerUqWpsbFR9fb02bdqk3t5e5efnq6GhQdKDK5C2b9+umpqaPsseN9ih\nSyRibF8GkpIio0ROqyVDzmTIKJHTanl5eSOeM+zTRIkQDN6x/cbPyZmkGzduJzrGkMhprWTImQwZ\nJXJaKSXFoezszNHNtTgLACAJUQYAgJFdWgpI0iT3BGW4Rv+t80XPf3U71G1hIgDxogwwYhmuVC37\n0eFRz/e/v1z2PvMKjD+cJgIAUAYAAMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAA\nIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAg\nygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDK\nAAAgygAAIMoAACDKAAAgKXU4D6qvr9eJEyfU3t6uI0eO6LnnnpMklZSUKCMjQ+np6XI4HKqqqtLC\nhQslSefOnVNtba16enrk8/nU0NCgrKyssfuXAABGbVhHBosWLdKHH34on88XM+5wONTY2KhDhw7p\n4MGD0SKQpA0bNqiurk7Hjx9XcXGxduzYYW1yAIBlhlUGRUVF8nq9MsbEjBtj+oxJUltbm1wul+bO\nnStJKisr07FjxyyICwAYC8M6TTSYqqoqGWM0b948vfXWW8rMzFQgEIg5ivB4PJKkUCgkt9sdMz8U\nCikUCsWMOZ1O5eXlxRsNAMalQCCgcDgcM+Z2u/v8/n1UXGXQ3Nwsr9er+/fva9u2bXr33XfV0NDQ\n72P7O4KQpH379qmpqSlmzOfzqaWlRdnZmfHEe2JyciYlOsKw2CnnYFnslHMwyZAzGTJK5LTa6tWr\n1d7eHjNWUVGhysrKAefEVQZer1eSlJaWplWrVmnt2rWSpLy8vJggnZ2dcjgc/bZSeXm5VqxYETPm\ndDolScHgHUUi/ZeIXeTkTNKNG7cTHWNIVua04gdioCzjcXuOlWTIKJHTSikpDmVnZ2r//v39HhkM\nZtRl0N3drXA4rMzMB3+9Hz16VDNnzpQkzZkzRz09PWptbVVRUZEOHDigJUuW9LueoQ5dAAAjM5rT\n7MMqg61bt+rkyZMKBoP67ne/K4/Ho927d6uyslKRSESRSEQzZsxQbW2tpAdXGW3fvl01NTXq7e1V\nfn7+gKePAACJN6wyqK6uVnV1dZ/xgwcPDjinsLBQfr9/9MkAAE8M70AGAFAGAADKAAAgygAAIMoA\nACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAA\nIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAg\nygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoAAKBh\nlEF9fb1KS0tVUFCgixcvRsc///xzlZWVafHixSorK9Ply5eHtQwAYD9DlsGiRYv04YcfyufzxYzX\n1tZqzZo1On78uFatWqWampphLQMA2M+QZVBUVCSv1ytjTHSss7NTFy5c0AsvvCBJWrp0qc6fP6+b\nN28OugwAYE+po5kUCATk9XrlcDgkSSkpKXrmmWfU0dGhSCQy4DKPx9NnXaFQSKFQKGbM6XQqLy9v\nNNEAYNwLBAIKh8MxY263W263e8A5oyoDK+3bt09NTU0xYz6fTy0tLcrOzkxQqpHJyZmU6AjDYqec\ng2WxU87BJEPOZMgokdNqq1evVnt7e8xYRUWFKisrB5wzqjLIy8vTtWvXZIyRw+FQJBLR9evXlZub\nK2PMgMv6U15erhUrVsSMOZ1OSVIweEeRiOlvmm3k5EzSjRu3Ex1jSFbmtOIHYqAs43F7jpVkyCiR\n00opKQ5lZ2dq//79/R4ZDGZEZfDwdYOsrCwVFBTI7/frxRdflN/v16xZs6KngQZb9rihDl0AACMz\nmtPsQ5bB1q1bdfLkSQWDQb366qvyeDzy+/2qq6vTxo0btWvXLk2ePFn19fXROYMtAwDYz5BlUF1d\nrerq6j7j06dP10cffdTvnMGWAQDsh3cgAwAoAwAAZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAA\nEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQ\nZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAABBlAAAQZQAAEGUAAJCUmugAeLImuScow8Vu\nBxCL3wrjTIYrVct+dDiudfjfX25RGgB2wWkiAABlAACgDAAAogwAAKIMAADiaiIkQO/9sHJyJg24\nfLBlD/X0huVKd8aV44ue/+p2qDuudQBPC8oAT1x6mtOSy1utWMftuNYAPD04TQQAoAwAAJQBAECU\nAQBAFryAXFJSooyMDKWnp8vhcKiqqkoLFy7UuXPnVFtbq56eHvl8PjU0NCgrK8uKzONaPDeaG85V\nOgDGp7jLwOFwqLGxUTNmzIgZ37Bhg+rr6zV37lzt3r1bO3bs0HvvvRfv04178d5ojpvMAehP3KeJ\njDEyxsSMtbW1yeVyae7cuZKksrIyHTt2LN6nAgCMEUveZ1BVVSVjjObNm6f169crEAjI5/NFl3s8\nHklSKBSS2+2OmRsKhRQKhWLGnE6n8vLyrIgGAONOIBBQOByOGXO73X1+/z4q7jJobm6W1+vV/fv3\ntW3bNm3evFmLFi3q87jHjx4e2rdvn5qammLGfD6fWlpalJ2dGW+8J4Jz8ckrnn2XDPs9GTJK5LTa\n6tWr1d7eHjNWUVGhysrKAefEXQZer1eSlJaWplWrVmnt2rUqLy+PCdLZ2SmHw9FvK5WXl2vFihUx\nY07ng9sMBIN3FIn0XyJ2kZMzSTduPLn3sSbLN2OyGO2+e9L7fTSSIaNETiulpDiUnZ2p/fv393tk\nMJi4yqC7u1vhcFiZmQ/+gj969KhmzZql2bNnq6enR62trSoqKtKBAwe0ZMmSftcx1KELAGBkRnOa\nPa4y+M9//qN169YpEokoEoloxowZ2rRpkxwOh7Zv366amhr19vYqPz9fDQ0N8TwVAGAMxVUGU6dO\n1cGDB/tdVlhYKL/fH8/qAQBPCO9ABgBQBgAAygAAIMoAACDKAAAgygAAIMoAACDKAAAgygAAIMoA\nACDKAAAgygAAIMoAACDKAAAgygAAIAs+9hJIVr33w3F9jOgk9wTdDnVbmAhIHMoA41Z6mlPLfnR4\n1PP97y+XvT8RFxg+ThMBACgDAABlAAAQrxkACTXJPUEZrtH/GH7R819exIYlKAMggTJcqbyIDVvg\nNBEAIDmODCZMSJPD4YhrHd3d92WMsSgRADxdbF8GTmeKUlKdOvK/l0a9juKZXuW4Xfrii/sWJgOA\np4fty0CS7nbf16+PXRj1/MmZ6cqZnWthIsAehnoX9XDeYc2L0JCSpAwA9C/ed1FLvAiNByiDJyje\nywhhL/He2wiwE34zPUHxXkYoPfgrDvZg1V/lgB1waSkAgDIAAFAGAABRBgAAUQYAAFEGAABRBgAA\nUQYAAFEGAABRBgAAcTuKERno3kLcnwbjXbz33ZrknsCdUxOMMhgBKz6iEHga8fGdyY/TRAAAygAA\nMI5OE6W7UjVpUkaiYwCALY2bMnCl81kCgF1Z8UFBfHxnfMZNGQCwLz6+M/HGtAw+//xzbdy4UV1d\nXZoyZYq2b9+ur3zlK2P5lABGiI/vfGA4l8cOtZ2S+ehkTMugtrZWa9as0dKlS/W73/1ONTU12rdv\n31g+JYARelo+vtOKUhvPRydjVgadnZ26cOGCXnjhBUnS0qVLtWXLFt28eVMej2esnhbAOBVvqdmh\n0BJpzC4tDQQC8nq9cjgcD54oJUXPPPOMOjo6xuopAQCjlPAXkEOhkEKhUMyY0+lUXl6eUlIccjik\nSZkuNVT+z6ifIzd7oiTpGc+EuLJasQ4y2CeDFesgg3XreFoypKQ4Rj03MzNDrjhu6/FQIBBQOByO\nGXO73XK73QPOcRhjTNzP3I/Ozk4tXrxYp06dksPhUCQS0YIFC3TixImY00SNjY1qamqKmVtUVKTm\n5uaxiAUAT72VK1eqtbU1ZqyiokKVlZUDTzJj6Dvf+Y45fPiwMcaYQ4cOmVdeeaXPY27dumWuXLkS\n89+ZM2dMWVmZuXr16ljGi9vVq1fN888/T06LkNM6yZDRGHJa7erVq6asrMycOXOmz+/VW7duDTp3\nTE8T1dXVaePGjdq1a5cmT56s+vr6Po8Z6NCltbW1z2GO3YTDYbW3t5PTIuS0TjJklMhptXA4rNbW\nVuXm5io/P39Ec8e0DKZPn66PPvpoLJ8CAGABblQHAKAMAACSs66uri7RIfrjcrm0YMECuVyuREcZ\nFDmtRU7rJENGiZxWG23OMbu0FACQPDhNBACgDAAANrgdxePsetvr+vp6nThxQu3t7Tpy5Iiee+45\nSfbL29XVpQ0bNujKlStKT0/XV7/6Vb377rvyeDw6d+6camtr1dPTI5/Pp4aGBmVlZSUs65tvvqn2\n9nY5HA5NnDhR1dXVKigosN02laSmpiY1NTVF973dtmVJSYkyMjKUnp4uh8OhqqoqLVy40HY5e3t7\n9d577+nPf/6zXC6XCgsLtXnzZlvt8/b2dr355pvR+6rdunVLd+/e1alTp3Tp0iW9/fbbtsgpSZ9+\n+qk++OADGWNkjFFFRYUWLVo0uu35BN4UNyKvvPKK8fv9xhhjDh8+3O+7lhPhL3/5i+no6DAlJSXm\nH//4R3Tcbnm7urrM6dOno1/X19ebn/zkJ8YYYxYtWmRaW1uNMcbs2rXLvP322wnJ+NDt27ej//+H\nP/zBrFixwhhjv2362Wefmddff908//zz0X1vt21ZUlJiLl682Gfcbjm3bNlifvrTn0a/DgaDxhj7\n7fNHbdu2zWzZssUYY7+c8+fPj+73v/3tb2bu3LnGmNHltFUZBINBM3/+fBOJRIwxxoTDYVNcXGw6\nOzsTnOz/PfoLIRny/v73vzevvvqq+etf/2qWLl0aHe/s7DSFhYUJTBbr4MGD5qWXXrLdNu3p6TEv\nv/yy+fe//x3d93bclo9+Xz5kt5x37941xcXF5t69ezHjdtvnj+rt7TVf//rXzYULF2yZc8GCBdGy\nP336tPnWt75lgsGgKS4uHnFOW50mGuy213b8DAS75zXGqLm5WaWlpQoEAvL5fNFlD/OFQqFB72Q4\n1qqrq/WnP/1JkvTLX/7Sdtv0gw8+0PLly2O2nV23ZVVVlYwxmjdvntavX2+7nJcvX9aUKVPU2Nio\nU6dOaeLEifrBD36gjIwMW+3zR33yySfKzc1VQUGBPvvsM9vl3Llzp77//e/rS1/6ku7evau9e/cq\nEAgoNzd3xDl5AfkptnnzZk2cOFFr1qzpd7mxwVXFW7du1aeffqr169dH711lh1ySdO7cObW1tWnl\nypXRsYGyJTpzc3OzDh06pN/85jeKRCLavHlzv49LZM5wOKwrV65ozpw5+u1vf6uqqipVVlbq3r17\nCd9+A/n444/10ksvJTpGv8LhsPbu3atf/OIXamlp0e7du/XDH/5Q9+7dG9X6bFUGeXl5unbtWvQb\nIxKJ6Pr168rNzU1wsv7ZOW99fb0uX76sn//855IeZG1vb48u7+zslMPhSOhfso968cUXderUKVtt\n09OnT+vSpUsqLS1VSUmJrl27ptdff12XL1+23bb0er2SpLS0NK1atUpnz57Vs88+a6uczz77rFJT\nU/Xtb39bkvS1r31NWVlZcrlcun79ui32+aOuX7+uM2fOaNmyZZLs9/N+4cIF3bhxQ4WFhZIe3Pp/\nwoQJcrlco8ppqzLIyspSQUGB/H6/JMnv92vWrFkJP1R83MONbNe8O3fu1Pnz57Vr1y6lpj44Ezhn\nzhz19PRE73F+4MABLVmyJGEZ7927F/Opdy0tLZoyZYqysrI0c+ZMW2zTN954Q3/84x/1ySefqKWl\nRV6vV7/61a/02muv2Wpbdnd3686dO9Gvjx49qlmzZmn27Nm2yunxeLRgwYLoacFLly4pGAxq+vTp\ntvw5+vjjj/XNb35TkydPlmS/n/fc3Fx1dHTo0qVLkqR//vOfCgaDmjZt2qhy2u4dyP/617+0ceNG\nhUKh6G2vp02bluhY2rp1q06ePKlgMKgpU6bI4/HI7/fbLu/Fixe1bNkyTZs2Lfp29KlTp6qxsVFn\nz57Vpk2b1Nvbq/z8/IReZhgMBrV27Vp1d3crJSVFU6ZM0Y9//GPNnDnTdtv0odLSUu3Zsyd6aWlN\nTY0ttuWVK1e0bt06RSIRRSIRzZgxQ9XV1fryl79sq5wPs77zzjvq6upSWlqa3nrrLX3jG9+w5T5f\nvHixampqtHDhwuiY3XIeOXJEe/bskdPplCStW7dOJSUlo8ppuzIAADx5tjpNBABIDMoAAEAZAAAo\nAwCAKAMAgCgDAIAoAwCAKAMAgKT/Az96X9CTJZCZAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1pifWiCoMbR5" - }, - "source": [ - "男乘客大约是女乘客的两倍。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-WazAq30MO5J", - "outputId": "77a2208b-a25f-4ee8-a31e-8cb3fb1a5f41", - "colab": { - "height": 274 - } - }, - "source": [ - "dftrain.sex.value_counts().plot(kind='barh')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEBCAYAAAC0WehTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEpVJREFUeJzt3HtsU/X/x/FXW8YkG+Uy5xggMWh0gCaCGGMUjAN/gLc5\nBAOCaDIkUbkIGhVFbi7BAV5ZECPeIggSvC5GRZzGazQIGBSJTMXpHDAHs9ucm1vP9w+z/izrtuLe\n7anl+UhIOGen5/PqZ5/1tXMo9TiO4wgAAENetwPEUiAQ0Jo1axQIBNyOEkKm6JApeomYi0zRSeZM\nSV8uxcXFCfeNI1PnyBS9RMxFpugkc6akLhcAgDsoFwCAOcoFAGAu6ctlxIgR8vl8bscI8fl8GjBg\nAJk6QaboJWIuMkUnUTONGDGiy+fxJPNbkevq6pSenu52DAD4z+nq62dSl0uro0frFQwmztPMyEhX\ndXWd2zHCkCk6iZhJSsxcZIpOomXyej3q0yety+fpZpAl4QWDTkKVi6SEyyORKVqJmElKzFxkik4i\nZuqqpP83FwBA/FEuAABzlAsAwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDA\nHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDA\nHOUCADBHuQAAzHVzO0A8ZGSkux2hjczMnm5HaINM0TneTH82Nqs20BCjNEBiOiHKpaBwmw4f5Ycb\n7ih5KE+1bocA4ozbYgAAc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzlAsA\nwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzUZXL\n9u3bdfnll2vixIk6cOBATAMtXLhQGzdujOkYAIDY6hbNQS+99JLmzZuncePGxToPACAJdFouK1as\n0I4dO3TgwAG9+OKLuuOOO7R69WrV19dLkubOnatLLrlEFRUVuvbaa3Xdddfpo48+UmNjo1atWqXN\nmzfrq6++Uo8ePbR27VplZGTou+++07Jly9TQ0KCmpiZdd911mjFjRpux//rrLz3yyCPasWOH/vrr\nL5155plaunSpevToYT8TAAAznZbLwoULtXfvXs2cOVMjRozQjBkz9NRTT+nkk09WVVWVJk2apDff\nfFOSVFNTo5EjR2rBggV6+umnddNNN2nDhg164IEHtGzZMm3YsEHz5s3TwIED9dxzzyklJUV//PGH\nJk+erIsvvliDBw8OG3v9+vXy+/3asmWLJGn16tVat26d5s+f3yZnIBBQIBAI2+fz+ZSdnf2vJwcA\nTlSVlZVqaWkJ2+f3++X3+6N6fFS3xVrt3LlTv/zyi26++WY5jiPp7xfwn376Sb1791ZaWppGjx4t\nSRo6dKj69euns846S5I0bNgwffbZZ5KkhoYGLVmyRPv27ZPX61VVVZX27dvXplxKS0tVX1+vt99+\nW9LfVzI5OTkRsz3//PMqLi4O2zdgwACVlpYez1MEYiIzs2dSjHG8yBSdRMw0bdo0VVRUhO2bPXu2\n5syZE9Xjj6tcJCknJ0cvvPBCm/0VFRXq3r17aNvn8yk1NTVsu7m5WZL08MMPKzMzUytXrpTH41FB\nQYGampranNNxHC1ZskQXXHBBp7luvPFG5efnh+3z+XxRPy8glqqqamN6/szMnjEf43iRKTqJlsnr\n9SgjI10bN26MeOUS9XmOZ9Dhw4frwIED+vzzz0P79uzZE/p769VMZ2pra5WdnS2Px6PvvvtOO3bs\niHhcbm6unn32WTU2NkqS6uvr9f3330c81u/3a+DAgWF/uCUGAP9OdnZ2m9fU4ymXqK5cPB6PpL9f\nwJ944gkVFRVpxYoVampq0qBBg7Ru3bqw4zpzyy236K677tIbb7yhQYMG6fzzz4943KxZs7RmzRpN\nmjRJHo9HXq9Xs2fP1umnnx7VOAAAd3icaC83/sMKCrfp8NEGt2PgBFXyUB63xRIEmTrXelusy+cx\nyAIAQBjKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmOvmdoB4\neHrR/7kdASewPxub3Y4AxN0JUS7V1XUKBh23Y4RkZvZUVVWt2zHCkCk6iZgJSETcFgMAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADm\nKBcAgDnKBQBgjnIBAJijXAAA5jyO4zhuhwAARPZnY7NqAw1xG8/r9SgjI73L5+lmkCXhFRRu0+Gj\n8fvmAICVkofyVOt2iH+B22IAAHOUCwDAHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAA\nc5QLAMAc5QIAMEe5AADMUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAA\nc5QLAMBcwpVLcXGxVq5c6XYMAEAXJFy5AAD++7pZniwnJ0e33367tm/frt9//13Lly/XZ599po8+\n+kjNzc167LHHNHjwYP32229asGCB6uvr1dTUpEsuuUR33nlnxHOuX79e27ZtU3Nzs7KyslRYWKiM\njAzL2AAAY+ZXLr169dLWrVt1xx136NZbb9XIkSP16quvKi8vT+vWrZMk+f1+Pfnkk3r55Zf16quv\nas+ePfr444/bnOuNN95QeXm5tmzZoldeeUWjR4/WihUrIo4bCAT0yy+/hP2prKy0fnoAcEKorKxs\n85oaCASifrzplYskTZgwQZI0bNgweb1ejR49OrS9fft2SVJLS4uKioq0a9cuOY6j6upqffvtt7r4\n4ovDzlVaWqpvvvlG11xzTehxfr8/4rjPP/+8iouLw/YNGDBApaWlps8PAOItM7Nn3MecNm2aKioq\nwvbNnj1bc+bMierxpuXi8XiUmpoqSfJ6verevXvoaz6fT83NzZKkZ599VrW1tdq6datSUlK0ePFi\nNTY2tjmf4zi65ZZbNHHixE7HvvHGG5Wfnx+2z+fzdeXpAEBCqKqqjdtYXq9HGRnp2rhxo1paWsK+\n1t4v95GYlovjOB1ut6qtrVVmZqZSUlJ06NAhvffee5o6dWqb43Jzc/XCCy9o7Nix8vv9ampq0g8/\n/KCcnJw2x/r9/uN64gCA9mVnZ3fp8eZXLh1tt7rhhhs0b948TZw4Uf369dOFF14Y8bi8vDzV1NRo\n+vTp8ng8CgaDuv766yOWCwAgcXic9i4vkkhB4TYdPtrgdgwAOG4lD+W5clusy+cxyAIAQBjKBQBg\njnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADmKBcAgDnKBQBg\njnIBAJijXAAA5igXAIA5ygUAYI5yAQCYo1wAAOYoFwCAOcoFAGCOcgEAmPM4juO4HQIAENmfjc2q\nDTTEbTyv16OMjPQun6ebQZaEV11dp2AwcTo0M7Onqqpq3Y4RhkzRScRMUmLmIlN0EjGTBW6LAQDM\nUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADM\nUS4AAHOUCwDAHOUCADBHuQAAzFEuAABzlAsAwBzlAgAwR7kAAMxRLgAAc5QLAMAc5QIAMEe5AADM\ndXM7QDx4vR63I7RBpuiQKXqJmItM0UmkTFZZPI7jOCZnSkB1dXVKT093OwYA/Od09fUzqW+L1dTU\naOrUqaqsrHQ7SkhlZaVyc3PJ1AkyRS8Rc5EpOomaaerUqaqpqenSeZK6XCRp586damlpcTtGSEtL\niyoqKsjUCTJFLxFzkSk6iZpp586dXT5P0pcLACD+KBcAgDnKBQBgzrd06dKlboeIpdTUVF1wwQVK\nTU11O0oImaJDpuglYi4yRSdZMyX1W5EBAO7gthgAwBzlAgAwl7TlcuDAAU2ZMkXjx4/XlClTVF5e\n7kqO3NxcXX755brmmmuUn5+vTz75RJK0e/du5eXlafz48SooKNCRI0dilqGoqEhjxoxRTk6OysrK\nQvs7mqNYz197mdqbLyn2c1ZTU6NZs2ZpwoQJysvL09y5c3X06NFOx45lro4y5eTkKC8vLzRX+/fv\nDz2utLRUEyZM0Lhx47RgwQI1NjaaZZKk2267LTTu9OnTtW/fPknurqn2Mrm5ploVFxeHrXW31lNn\nuUzXlJOkZsyY4ZSUlDiO4zivv/66M2PGDFdy5ObmOmVlZW32X3bZZc7OnTsdx3GctWvXOgsXLoxZ\nhi+//NI5ePCgk5ub6+zfvz+0v6M5ivX8tZepvflynNjPWU1NjfPFF1+EtouKipz77ruv07Fjmauj\nTDk5OU5DQ0Obx9TX1zsXXXSRU15e7jiO49x3331OcXGxWSbHcZza2trQ37dv3+7k5+c7juPummov\n06WXXuramnIcx/nmm2+cmTNnOpdeemlorbu1njrLZbmmkvLK5ciRI/r22291xRVXSJKuvPJK7d27\nN/QbXzw5jiPnmPdM7NmzR6mpqRo+fLgkacqUKXrrrbdilmHEiBHKysoKy9HRHMVj/iJlkiLPlxSf\nOevVq5fOP//80Pa5556rX3/9tcOxY52rvUxS+3P14Ycf6uyzz9app54ak0ySwj5zqra2Vl6v1/U1\ndWwmn88X2nZrTTU1NWn58uX655ty3VxPHeWSbNdUUn4qcmVlpbKysuTx/P3pnl6vV6eccooOHjyo\nPn36xD3PnXfeKcdxdN5552n+/PmqrKzUgAEDQl9vzRQIBOT3++OSqaM5CgaDrs7fP+drwYIFSk9P\nj/ucOY6jTZs2acyYMR2OHc9crZnGjh0rSfJ4PLrhhhvU0tKiUaNGac6cOUpJSWmTqX///jp48KBp\nFklatGhR6BbT+vXrE2JNHZuplVtr6vHHH1deXl7YOImwniLlkmzXVFJeuSSSTZs26bXXXtPWrVsV\nDAa1fPnyiMdF+m3hRHTsfC1btqzdY2M5Z8uXL1daWpqmT59+3GPHKldrpmnTpkmSPvjgA23dulUb\nNmxQWVmZ1q5dG5Nx21NYWKj3339f8+fPV1FRkST313GkTG6tqd27d2vPnj2aOnVqp+eP53qKlKuV\n5ZpKynLJzs7WoUOHQt+UYDCow4cPq1+/fnHPkpWVJUlKSUnR9ddfr127dql///6qqKgIHXPkyBF5\nPJ64XbVIHc+Rm/MXab5a88ZrzoqKilReXq5HH32007HjlevYTNL/z1VaWpomT54c+rDBYzP9+uuv\nys7ONs3zT1dffbU+//zzhFpTrZl+//1319bUF198oR9//FFjxoxRbm6uDh06pJkzZ6q8vNzV9RQp\nV0FBgT799FPTNZWU5dK3b1/l5OSopKREklRSUqKhQ4fG/ZZYQ0OD6urqQttvvvmmhg4dqmHDhqmx\nsTH0jdu8ebMmTJgQl0ytP9wdzZFb8xdpvoYMGSJJOvvss+MyZ4888oj27t2rtWvXqlu3bp2OHY9c\nkTIFAoHQu3Wam5v1zjvvhOZq1KhR+vrrr0Pvxtq8ebPGjx9vluePP/4IuyVSWlqq3r17q2/fvhoy\nZIgra6q9TKmpqa6tqVmzZunDDz/Ue++9p9LSUmVlZemZZ55RQUGBq+upvVznnHOO6ZpK2v+h/8MP\nP+iee+5RIBBQr169VFRUpNNOOy2uGX7++WfNnTtXwWBQwWBQp59+uhYtWqSTTz5Zu3fv1v3336+m\npiYNHDhQq1atUt++fWOSo7CwUO+++66qq6vVu3dv9enTRyUlJR3OUaznL1KmJ554QnPmzIk4X5Ji\nPmdlZWW66qqrdNppp4U+9uLUU0/VmjVrtGvXLi1evDji2LHMdWwmj8ejgQMHqqCgQIsXL5bX61Vz\nc7OGDx+ue++9Vz169JD094vrypUr5TiOhgwZogcffFAnnXSSSabq6mrdeuutamhokNfrVe/evXX3\n3XdryJAhrq2pSJnuuecepaWltfszKMV+Tf3TmDFj9OSTT+qMM87ocNx4Zvpnrrq6OtM1lbTlAgBw\nT1LeFgMAuItyAQCYo1wAAOYoFwCAOcoFAGCOcgEAmKNcAADmKBcAgLn/AQvkwWROxmRCAAAAAElF\nTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7_XkxrpmmVU_" - }, - "source": [ - "大多数乘客都在“三等”舱。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zZ3PvVy4l4gI", - "outputId": "147730b6-c79a-4864-9a40-b40e97383e8c", - "colab": { - "height": 274 - } - }, - "source": [ - "dftrain['class'].value_counts().plot(kind='barh')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAEBCAYAAABBp2PjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFHBJREFUeJzt3V1wVPX9x/HP7pIHSUhIIqYxBkWjTRAq4LSxFZ/S2kFF\nMQN9kLRwQb1oJUypDCIGlDRTSaLFGVbCTPWCsUjrZIAaHEbKROpM2yEdMVMd7UwzFkNDIikkbgIh\nkN3zv2DI38DmAZJvzp7k/brKnt1z9rPnd9gP5+zO/nyO4zgCAMCI3+0AYyEUCmnr1q0KhUJuR7kq\n5HePl7NL5Hcb+S+YMEUTDAY9Pdjkd4eXs0vkdxv5L5gQRQMAcA9FAwAwRdEAAExNmKKZN2+eAoGA\n2zGuSiAQUHZ2Nvld4OXsEvndNh7yz5s3b8Tb8U2Erzd3dXUpOTnZ7RgA4EkjfQ+dEEVzUXv7aUUi\n3ny5GRnJOnmyy+0YV83L+b2cXSK/27yc3+/3KS0tacTbmTQKWTwjEnE8WzSSPJ1d8nZ+L2eXyO82\nr+cfqQnzGQ0AwB0UDQDAFEUDADBF0QAATFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMA\nMEXRAABMUTQAAFMUDQDAFEUDADA1oeajycjw9iyb06ZNcTvCiFjnP9vTq85Qt+lzALhyE6poVpQf\n0Il23ojGq9qXF6nT7RAALsOlMwCAKYoGAGCKogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApiga\nAIApigYAYIqiAQCYomgAAKaGVTT79+9XUVGRioqK9PDDD2vNmjXWuQZVX1+vxYsXu5oBADA8Q/56\nc1tbm8rKyrR3715lZmZKkv71r3+ZBxuKz+dzOwIAYBiGLJr//e9/iouLU2pqat+yvLw8SdI///lP\nvfTSSzp9+rQkadWqVbrvvvskSe+9956CwaB6e3sVCAS0efNm3XbbbXr//fe1ZcsWRSIRpaenq6ys\nTDk5Oaqvr9dvfvMbfeMb31BDQ4P8fr9++9vf6uabb5YkbdmyRfv371dmZqZmz5496jsCAGBjyKLJ\ny8vT7Nmzdf/99+tb3/qW7rzzTi1atEiBQEDPP/+8fve73+naa69VW1ublixZonfeeUdtbW3asGGD\ndu3apZycHJ0/f17nz5/XqVOn9Mwzz2jnzp26+eabVVNTo6efflpvvfWWJKmxsVGbN29WWVmZtm/f\nrurqalVVVamurk6HDh3S22+/rYSEBP385z833zEAgNExZNH4fD69+uqramxsVH19vQ4ePKjXX39d\na9eu1X//+189+eSTchxHkhQIBPT555+roaFB9913n3JyciRJcXFxiouL0+HDh5Wfn993lrJ48WJt\n2rRJZ86ckSTNmDGj72zpjjvu0KFDhyRd+Ezm4YcfVmJioiRpyZIl2r59e9S8oVBIoVCo37JAIKCs\nrKwr3TcAAEktLS0Kh8P9lqWkpCglJWVY6w97hs3c3Fzl5uZq6dKleuSRRyRdONt54403LntsQ0ND\n1G04jnPZZytfvZ2QkND3dyAQUG9vb996w7Vjxw4Fg8F+y7Kzs1VXVzfsbcC7rKaLZhptd5HfXcXF\nxWpubu63bOXKlSopKRnW+kMWzRdffKGWlhbNmTNHktTa2qr29nbl5ubq6NGjOnz4sAoKCiRJH330\nkWbPnq358+erurpaTU1Nmj59us6dO6fz589rzpw5Ki0t1X/+8x/NmDFDu3fv1syZMzV58uRBM3z7\n29/WK6+8omXLlik+Pl67d+8e8LHLly9XUVFRv2WBQGDIHYHxoa1t9CdznjZtisl2xwr53eXl/H6/\nTxkZydq5c2fUM5rhGrJowuGwtm7dquPHjyshIUGO42j16tXKy8tTdXW1Kioq9OKLL+rcuXOaPn26\ntm/frhtvvFHl5eX65S9/qXA4rEAgoIqKCt16662qrKzU008/rXA4rPT0dFVVVQ0Z8v7771dDQ4Me\nf/xxXXfddSooKNCJEyeiPvZKTucAAEMb6UcPPudKrkt53IryAzrR3u12DBipfXkRZzRRkN9dXs5/\n8YxmxNsZhSwAAAyIogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgA\nAKYoGgCAKYoGAGCKogEAmBr2DJvjweul33c7Agyd7el1OwKAKCZU0Zw82aVIxJvT73h5TgvJ+/kB\nXD0unQEATFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUD\nADBF0QAATFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUD\nADBF0QAATFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUD\nADA1ye0AYykjI9ntCCMybdoUtyOMiJfzD5X9bE+vOkPdY5QG8JYJVTQryg/oRDtvBhh9tS8vUqfb\nIYAYxaUzAIApigYAYIqiAQCYomgAAKYoGgCAKYoGAGCKogEAmKJoAACmKBoAgCmKBgBgiqIBAJii\naAAApsbkRzULCwuVmJio+Ph4+Xw+FRQUKCkpSbm5uXrooYeGvZ3Ozk798Y9/1M9+9jPDtACA0TRm\nv968detW3XLLLUM+znEc+Xy+qPd9+eWXeu211ygaAPCQMSsax3H63X722Wc1a9YsFRcXKxgM6vPP\nP9eZM2d07NgxvfHGG9qyZYvq6+sVHx+vyZMn680339Svf/1rdXV1qaioSImJidq1a9dYxQcAXKUx\nK5pVq1b1XTpbs2bNZfd/8MEH2rNnj1JTU/Xpp5/q73//u959911JFy6ZSdLGjRu1ZMkS7dmzZ8Dn\nCYVCCoVC/ZYFAgFlZWWN4qsBgImjpaVF4XC437KUlBSlpKQMa33XLp3t27ev3/333nuvUlNTJUk5\nOTlyHEfr169XQUGBHnjggWE/z44dOxQMBvsty87OVl1d3QjSA0OL5RlEYznbcJDfXcXFxWpubu63\nbOXKlSopKRnW+q5dOrvU5MmT+/5OTk7Wvn37VF9fr7/97W966aWXtHfv3mE9z/Lly1VUVNRvWSAQ\nuPLAwBVqa4vNOTanTZsSs9mGg/zu8ft9yshI1s6dO6Oe0QxXTE7lfOrUKU2aNEnz58/Xd77zHf3l\nL3/RsWPHNGPGDJ09e1aRSER+f/RvZl/J6RwAYGgj/ehhTIpmoG+RDaS1tVWlpaWKRCIKh8O69957\nNWfOHEnSo48+qoULFyo1NZUvAwCAB/icoa5pjSMryg/oRHu32zEwDtW+vChmL494+dKNRH43Xbx0\nNuLtjEIWAAAGRNEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUDADBF0QAATFE0AABT\nFA0AwBRFAwAwRdEAAEzF5MRnVl4v/b7bETBOne3pdTsCELMmVNGcPNmlSMSb0+94eU4Lydv5vZwd\niAVcOgMAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgAAKYoGgCAKYoG\nAGCKogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgAAKYoGgCAKYoG\nAGCKogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgAAKYoGgCAKZ/j\nOI7bIQAAY+9sT686Q90D3u/3+5SRkTzi55k04i14yIryAzrRPvBOBYCJpPblReocg+fh0hkAwBRF\nAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMUTQAAFMUDQDAFEUDADBF0QAATFE0AABTo/qjmj/8\n4Q91/vx5nTt3TkePHtVtt90mx3HU2dmptLQ01dTUXLbOxx9/rB07dqiqqmpYzxEMBnXmzBmtXbt2\nNKMDAIyMatG89dZbkqTm5mYtWbJEe/bskSTV19ersrIy6jqzZs0asGTC4bACgcBoRgQAjLExu3TW\n29urjRs36rHHHtPjjz+uzz77TNKFElq8eLGkCwV11113KRgMaunSpaqpqVFXV5dWrVqlhQsX6skn\nn1RTU9NYRQYAjIIxK5rGxkYtXbpUb7/9thYsWKDq6uq++3w+X9/fHR0dys3N1Ztvvqkf/ehHevXV\nVzVlyhTt27dPlZWV+sc//jFWkQEAo2DMJj6bMWOG8vLyJEl33HGHDh06FPVxiYmJWrBgQd/tw4cP\na8OGDZKktLQ0Pfjgg4M+TygUUigU6rcsEAgoKytrBOkBYOJqaWlROBzutywlJUUpKSnDWn/MiiYh\nIaHv70AgoN7e3qiPu+aaa/rdvtKZpnfs2KFgMNhvWXZ2turq6q5oOwAwEUybNmXIxxQXF6u5ubnf\nspUrV6qkpGRYz2FWNFdaEAOtd9ddd2n37t2aO3eu2tvbdfDgwX5nPJdavny5ioqK+i3jCwUAEF1b\n28CTOfv9PmVkJGvnzp1Rz2iGy6xovvq5y0jWe+qpp7R+/XotXLhQ2dnZmj9//qDrX8npHABgaCP9\n6MHnXO2phwetKD+gE+3dbscAgJhQ+/KiYZ3RjBS/DAAAMEXRAABMUTQAAFMUDQDAFEUDADBF0QAA\nTFE0AABTFA0AwBRFAwAwRdEAAExRNAAAUxQNAMAURQMAMEXRAABMTahpAgAA/+9sT686QwNPnTJa\n0wSM2VTOseDkyS5FIt7s1WnTpgw6b0Ss83J+L2eXyO82r+cfDVw6AwCYomgAAKYoGgCAKYoGAGCK\nogEAmKJoAACmKBoAgCmKBgBgiqIBAJiiaAAApigaAIApigYAYIqiAQCYomgAAKYoGgCAKYoGAGBq\nQk185vf73I4wIuR3j5ezS+R3m1fzj1buCTGVc1dXl5KTRz4dKQBMRCN9D50Ql846Ojr0xBNPqKWl\nxe0oV6WlpUWFhYXkd4GXs0vkd9t4yP/EE0+oo6NjRNuZEEUjSUeOHFE4HHY7xlUJh8Nqbm4mvwu8\nnF0iv9vGQ/4jR46MeDsTpmgAAO6gaAAApigaAICpwAsvvPCC2yHGQkJCggoKCpSQkOB2lKtCfvd4\nObtEfreRf4J8vRkA4B4unQEATFE0AABT4/4naI4ePap169apo6NDU6dOVWVlpaZPn+52rEEVFhYq\nMTFR8fHx8vl8WrNmje6++241NDTo+eefV09Pj7Kzs1VVVaX09HRXs1ZUVOjAgQNqbm7Wvn37lJub\nK2nw/R5LYzJQ/oHGQFJMjUNHR4fWrl2rY8eOKT4+XjfeeKM2bdqktLS0QXPGymsYLH9eXp6+/vWv\ny+fzyefzqbKyUrfeeqskqa6uTlVVVYpEIrr99tv14osvuvIZyFNPPaXm5mb5fD4lJSWptLRUeXl5\nnjn+B8o/6se/M84tW7bMqa2tdRzHcf70pz85y5YtcznR0AoLC53GxsbLlj/44IPOkSNHHMdxnG3b\ntjnPPvvsWEe7zAcffOC0trY6hYWFzr///e++5YPt91gak4HyDzQGjhNb49DR0eHU19f33a6oqHCe\ne+45x3EGzxkrr2Gw/Hl5eU53d/dl65w+fdq5++67naamJsdxHOe5555zgsHg2AS+RGdnZ9/fBw8e\ndIqKihzH8c7xP1D+Bx54YFSP/3F96ezUqVP69NNP9cgjj0iSFi5cqE8++UTt7e0uJxuc4zhyLvmO\nxkcffaSEhATNnTtXkvTjH/9Y+/fvdyNeP/PmzVNmZma/vIPt91gbk2j5pehjIMXeOKSmpuqb3/xm\n3+05c+bo+PHjg+aMpdcwUH5p4DF4//33NWvWLOXk5EhyN/9Xf/+rs7NTfr/fU8f/pfkDgUDf7dE8\n/sf1pbOWlhZlZmbK57vwC6R+v1/XXXedWltblZaW5nK6wa1Zs0aO4+jOO+/U6tWr1dLSouzs7L77\nL+YPhUJKSUlxK2ZUg+33SCTimTH56hj86le/UnJyckyPg+M42rVrl7773e8OmjNWX8PF/N/73vck\nST6fTz/96U8VDod1zz33qKSkRHFxcZflv/7669Xa2upWbJWWluqvf/2rJOm1117z3PF/af6LRvP4\nH9dnNF61a9cu7d27VzU1NYpEIiorK4v6uGj/48DouHQMNm3aNOBjY2UcysrKlJSUpJ/85CdR7x8s\nZyy8hov5i4uLJUmHDh1STU2Nfv/736uxsVHbtm1zOWF05eXleu+997R69WpVVFRIio39OVzR8o/2\n8T+uiyYrK0tffPFF346IRCI6ceKEvva1r7mcbHCZmZmSpLi4OC1dulQffvihrr/+ejU3N/c95tSp\nU/L5fK7/Lzqawfa7V8Yk2hhIF15bLI5DRUWFmpqa9Morr0gaPGcsvoZL80v/PwZJSUn6wQ9+0Pfj\njpfmP378uLKyssY2cBSPPfaYDh8+7Nnj/2L+L7/8ctSP/3FdNOnp6crLy1Ntba0kqba2VjNnzoy5\nSzRf1d3dra6urr7b77zzjmbOnKnbb79dPT09ff/Y/vCHP+ihhx5yK2ZUF//xDLbfvTAm0cYgPz9f\nkjRr1qyYG4ctW7bok08+0bZt2zRp0oWr4YPljLXXEC1/KBRST0+PJKm3t1fvvvtu3xjcc889+vjj\nj9XU1CTpQv4FCxaMee4zZ870u2RXV1enqVOnKj09Xfn5+TF//A+UPyEhYdSP/3H/ywCfffaZ1q1b\np1AopNTUVFVUVOimm25yO9aAjh07plWrVikSiSgSieiWW25RaWmprr32WjU0NGjDhg06d+6cbrjh\nhpj4enN5ebn+/Oc/6+TJk5o6darS0tJUW1s76H6PpTGJlr+6ulolJSVRx0BSTI1DY2OjHn30Ud10\n0019X+/NycnR1q1b9eGHH2rjxo1Rc8bKa7g0v8/n0w033KAVK1Zo48aN8vv96u3t1dy5c7V+/Xpd\nc801ki68KVZWVspxHOXn52vz5s1KTEwc0+wnT57UL37xC3V3d8vv92vq1Kl65plnlJ+f74njP1r+\ndevWKSkpacD3IOnqjp1xXzQAAHeN60tnAAD3UTQAAFMUDQDAFEUDADBF0QAATFE0AABTFA0AwBRF\nAwAw9X+gI1LNFj+FVgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HM5SlwlxmZMT" - }, - "source": [ - "大多数乘客从南安普顿出发。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RVTSrdr4mZaC", - "outputId": "3512101a-0913-48d3-d4b0-6e0fdc2ed34d", - "colab": { - "height": 274 - } - }, - "source": [ - "dftrain['embark_town'].value_counts().plot(kind='barh')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAEBCAYAAAAD5BB0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtUVPXi/vH3zKCkAl7IiMArZnhLMTtKLf0a5clLpXhs\nLRPTymp1EY+30tJMyUw0syNUy45Wapa3xNSysshjJ1FWmZmZhZ0ML2CE2gAqCLN/f/hjjiiXUYGZ\nzud5reWK2TN774cPn+Zh7xn22CzLshAREfkfZ/d2AJM4nU6SkpJwOp3ejuKmTJ7xxUzgm7mUyTPK\n5LnqyqXCq0VOp5Pk5GSfmkzK5BlfzAS+mUuZPKNMnquuXCo8ERExggpPRESMoMITEREjqPBqWdeu\nXXE4HN6O4eZwOAgLC1OmKvhiJvDNXMrkGWXynMPhoGvXrpe9HZv+LKH25OfnExAQ4O0YIiJ/Spf7\nHKrC84LjxwtwuXxn2IODA8jNzfd2jDKUyXO+mEuZPKNMnrHbbTRu3OCyt+NXDVnkIrlclk8VHuBz\neUCZLoYv5lImzyhT7dFreCIiYgQVnoiIGEGFJyIiRlDhiYiIEVR4IiJiBBWeiIgYQYUnIiJGUOGJ\niIgRVHgiImIEFZ6IiBhBhSciIkZQ4XlBQMAV3o4gImIcFZ4X+Pvrmt0iIrVNhSciIkZQ4YmIiBFU\neCIiYgQVnoiIGEGFJyIiRlDhiYiIEVR4IiJihD9t4R0+fJgePXp4O4aIiPxJ/GkLD8Bms3k7goiI\n/En4ROGdf7RWerv0v/Pnzyc2NpZ+/fqxc+fOC9YvKipi7NixJCYmAnDvvfcyZ84chg0bRp8+fZg3\nb577sZmZmdx3333cddddDB48mC+++AKAlStXkpCQAMDu3buJjIxkz549AMyYMYPVq1cDEBkZycKF\nCxkyZAh9+vRh8+bNNTMoIiJSrXyi8ODCo7XS2ydOnKBr166kpKTw2GOPMXfu3DKP++OPPxg1ahTd\nunVj0qRJ7uXZ2dm88847pKSksHr1ajIzMwGYOHEid911F+vXr2fu3Lk88cQTHD9+nOjoaLZv3w7A\n9u3biYqKIi0tDYC0tDSio6Pd2w4MDGTNmjUkJiby3HPPVf9giIhItfOZwqtIgwYN+L//+z8AunTp\nwsGDB933nT59mri4OIYPH87w4cPLrNe3b18AAgICiIiIIDMzk4KCAn744QcGDx4MQEREBO3atePb\nb7+lefPmnD59mqNHj5KWlsaECRNIS0sjOzubM2fOEB4e7t52//793XlycnIoKiq6ILfT6eTQoUNl\n/mVlZVXv4IiIGCQrK+uC51Wn0+nx+j5xFWM/Pz9cLpf7dmFhofvrunXrur+22+2UlJSUua9z586k\npqbSp08f7Pb/9re/v/8F61mWVe7rfqXLevTowZYtW8jNzaVbt27k5OSwZcuWMqdbbTabe9ul+zs3\nU6klS5aQnJxcZllYWBipqakANG0aWNmQ1DpfywPKdDF8MZcyeUaZPBcXF8fhw4fLLBs9ejTx8fEe\nre8ThXfllVdSXFzMwYMHadasGRs2bHDfZ1lWmceee9tut/P888+TkJDA2LFjeemll/Dzq/hbCggI\noF27dqSkpBAbG8vPP//Mjz/+yPXXXw+cLbyXX36ZXr16ARAVFcXrr7/O+PHjPcpzrpEjRxIbG1tm\nmcPhcH+dk5NXYc7a1rRpoE/lAWW6GL6YS5k8o0yesdttBAcHsHz58gsOMIKCgjzejk8UnsPhYMqU\nKdx3332EhYXRvXt3930VvbZ3rmnTppGYmMjo0aNZsGBBpevMnTuXadOm8eabb+Ln58fcuXNp3Lgx\ncLbwsrKyuOmmmwCIjo5m9erVFxzhVZUHzv4QLuYHISIilQsNDb2s9W1WRYcoUqN86TcoX/yNTpk8\n54u5lMkzyuSZ0iO8y95ONWQRERHxeSo8ERExggpPRESMoMITEREjqPBERMQIKjwRETGCCk9ERIyg\nwhMRESOo8ERExAgqPC8oLCz2dgQREeOo8LwgP/+0tyOIiBhHhSciIkZQ4YmIiBFUeCIiYgQVnoiI\nGEGFJyIiRlDhiYiIEVR4IiJiBBWeiIgYQYUnIiJGUOGJiIgRVHgiImIEFZ6IiBhBhSciIkZQ4YmI\niBFUeCIiYgQVnoiIGEGFJyIiRlDhiYiIEVR4IiJiBBWeiIgYQYUnIiJGUOGJiIgRVHgiImIEFZ6I\niBhBhSciIkZQ4YmIiBH8vB3ARMHBAd6OcIGmTQM5XVhMnvOUt6OIiNQIFZ4XjJr5Cb8d971i2TBv\nIHneDiEiUkN0SlNERIygwhMRESOo8ERExAgqPBERMYIKT0REjKDCExERI6jwRETECCo8ERExwiUX\nXlFREbNnz6ZPnz7079+f2NhYPvroo+rMdtmSk5MpLi72dgwREfEBl3yllenTp3P69Gk+/PBD6tSp\nQ0ZGBqNGjaJRo0b06NGjOjNesuTkZEaNGoWfny4oIyJiuktqgiNHjvDRRx/xr3/9izp16gBw7bXX\n8uijj5KcnMxXX31FQUEBkyZNAs4Wz8mTJ3nyySc5c+YM8+fP56uvvuLMmTO0bduW6dOnU69ePfLz\n85k9ezY//fQThYWFdO/enaeeegqbzca9995Lp06d2LVrFzk5OfTt25cJEya4t//hhx/i7++PzWZj\n6dKlvPTSS9hsNoYOHYrdbmfZsmUUFhby7LPPkpmZCcADDzzAoEGD+Pe//82yZctYuHAhubm53Hzz\nzfzjH//g9ttvZ9GiReTl5TFu3DhiYmIYNGgQ27ZtIycnhwceeIC4uLjq+DmIiEgNu6RTmj/99BMt\nWrQgMDCwzPIuXbrw008/AWCz2cpdd9GiRQQFBbFq1SpSUlJo2rQpCxcuBGD27Nn85S9/YdWqVaxb\nt47c3FzWrFnjXjc7O5t33nmHlJQUVq9eTWZmJk6nkzfeeIN169aRkpLC22+/Tf369Zk2bRqWZbFy\n5UpSUlIICAhg5syZtG3blvXr17N48WJefPFF9u/fT7du3fj2228pKSlh+/btREVFkZaWBkBaWho3\n3XSTO8Pp06dZsWIFS5cu5cUXX+TUqfKviel0Ojl06FCZf1lZWZcy3CIiAmRlZV3wvOp0Oj1e/5KO\n8CzLqvC+ioquVGpqKgUFBe7X+86cOUNkZKT7vu+++4433ngDOFsuoaGh7nX79u0LQEBAABEREWRm\nZhIeHk7r1q2ZOHEiPXv2pHfv3tSvX7/crNu2bWPy5MkANG3alN69e7Njxw7atGnDtddey65du9i2\nbRuPP/44c+bM4cyZM+zZs4eoqCj3NgYMGABAWFgYjRo1Ijs7m1atWl3wfS5ZsoTk5OQyy8LCwkhN\nTa10fLytadPAqh9US3wpSylfzAS+mUuZPKNMnouLi+Pw4cNllo0ePZr4+HiP1r+kwmvbti2//vor\nTqeToKAg9/JvvvmGqKgo/Pz8yhRNYWGh+2vLsnj22Wfp3r17udt+5ZVXCA8PL/c+f39/99d2u52S\nkhLsdjurVq1i586dpKWlMXjwYBYvXkzbtm0vWN9ms1VYyN27dyctLY3du3czY8YMgoOD2bhxI+3a\ntaNu3brlZrDZbJSUlJS7vZEjRxIbG1tmmcPhKPexviQnxzc+L6Fp00CfyVLKFzOBb+ZSJs8ok2fs\ndhvBwQEsX778gufcczuoyu1cys7DwsLo27cv06dPp6ioCDh7mnPp0qWMHTuWZs2asWfPHizLIj8/\nny1btrjXjYmJ4c0333SXYEFBAT///LP7vtdffx2XywXA8ePHOXToUKVZCgoKyM3NpVu3bsTHx9O2\nbVsyMjKAs0eCeXn//cHddNNNrFq1CoCcnBy2bt3qfoNNdHQ0a9eu5eqrr8bPz4/o6GiSkpKIjo6+\nlCEiKCiI8PDwMv/OPVoVEZGLExoaesHz6sUU3mW9S3PevHn0798fm83Gb7/9xsqVK4mMjCQiIoJN\nmzZxxx130Lx5czp27Ohe7+GHHyYpKYkhQ4Zgs9mw2+2MHj2aiIgInnrqKebOncvAgQOBs0dTTz/9\nNOHh4RccmZXezs/PJz4+nsLCQlwuFx06dKBPnz4A3H///YwYMYJ69eqxbNkypkyZwrRp07jrrrsA\nmDhxIhEREQB07tyZEydOuF+v69GjB/Pnzy/zjtOKMoiIiO+zWZW9IOeh4uJinn32WbKzs3nttdfK\nnAKUC/nyB8D6yqkMXzyt4ouZwDdzKZNnlMkzpac0L1e1/IGan58fzz//fHVsSkREpEbo0mIiImIE\nFZ6IiBhBhSciIkZQ4YmIiBFUeCIiYgQVnoiIGEGFJyIiRlDhiYiIEfTJqF6weOpfvR2hXKcL9enw\nIvK/S4XnBbm5+bhcl31Ft2rji5cSEhGpbjqlKSIiRlDhiYiIEVR4IiJiBBWeiIgYQYUnIiJGUOGJ\niIgRVHgiImIEFZ6IiBhBhSciIkZQ4YmIiBFUeCIiYgQVnoiIGEGFJyIiRlDhiYiIEVR4IiJiBBWe\niIgYQYUnIiJGUOGJiIgRVHgiImIEFZ6IiBhBhSciIkZQ4YmIiBFUeCIiYgQVnoiIGEGFJyIiRlDh\niYiIEfy8HcBEwcEB3o5wgaZNA70d4QKXm+l0YTF5zlPVlEZE/uxUeF4wauYn/HZcT8Q1bcO8geR5\nO4SI+Ayd0hQRESOo8ERExAgqPBERMYIKT0REjKDCExERI6jwRETECCo8ERExggpPRESMUCt/eF5c\nXMwrr7zChx9+SN26dXG5XPTq1YvWrVvzxRdfsGDBgsveR0xMDK+//jpt2rSphsQiIvK/plYKb/Lk\nyRQVFbFu3Trq1atHSUkJa9eupaioCJvNdlnbtizrsrfhCZfLhd2uA2IRkT+rGi+8X3/9lc8++4wv\nvviCevXqAeBwOLj77rtJSUkhPz+fcePGkZGRQVBQEElJSQQHBwOwaNEiPvnkE4qLiwkJCWHmzJkE\nBweTnJzMr7/+ysmTJzl48CBvv/02AOvXr2fnzp3k5OQwYsQI4uLiANi9ezezZs3i1KlT1KtXjylT\nptCpUyfS09NJTEzkvffeAyhzOz09nVmzZtGtWzf27NnDo48+SmRkJJMmTSI3N5fw8HAsy6Jnz57u\n/YiIiO+q8cLbu3cvLVu2JCCg/Asm79mzh/Xr1xMSEsIzzzzDsmXLGDt2LOvXryczM5NVq1YB8O67\n7/LCCy/w4osvAvD111+TkpJCw4YN3dvKzc3l7bffJjc3l0GDBnHjjTfSqlUr/v73v/PCCy/Qo0cP\n0tLSGDNmDJs3bwa44Ojw3NsZGRkkJCQwdepUAMaMGUOPHj145JFHOHLkCHfeeSc9e/Ys9/tyOp04\nnc4yyxwOB6GhoRczfCIi8v9lZWVRUlJSZllQUBBBQUEerV/jhWdZVqX3R0VFERISAkDnzp1JS0sD\nIDU1le+//55BgwYBUFJSUuab6tWrV5myAxgyZAgAwcHB9O7dm/T0dADq1q1Ljx49AIiOjqZu3br8\n8ssvVWZv0aIF119/vfv2jh073OV3zTXXEB0dXeG6S5YsITk5ucyysLAwUlNTq9yvVJ/q/hQIX/xU\nCfDNXMrkGWXyXFxcHIcPHy6zbPTo0cTHx3u0fo0XXocOHThw4AB5eXkEBl44iP7+/u6vHQ4HxcXF\nwNmifPTRRxk8eHC5261fv36l+y19ba+8wi29z+Fw4HK53MsLCwur3IenrxeOHDmS2NjYMsscDodH\n60r1ycmpvs9LaNo0sFq3V118MZcyeUaZPGO32wgODmD58uXlHuF5vJ3qDna+Fi1aEBMTw7Rp0ygo\nKADOvgFk6dKlnDx5ssL1YmJieOedd9ynBYuKiti3b1+l+0pJSQHg2LFjbN26le7du9O6dWvOnDnj\nPtrbvn07xcXFtGzZkvDwcA4dOkReXh6WZfHBBx9Uuv3u3buzdu1a4Oyh9fbt2yt8bFBQEOHh4WX+\n6XSmiMilCw0NveB59WIKr1bepZmYmEhSUhKDBw+mbt26WJZFr169aNWqVYXrDBw4kBMnTjB8+HBs\nNhsul4thw4YRGRlZ7uNtNhuhoaHExcXx+++/88gjj7j/RGHBggXMnDnT/aaVpKQk/Pz8CAkJ4f77\n7yc2NpZmzZrRqVMn9u/fX2Gmp59+mkmTJrFp0yZat27NDTfcUO5Rq4iI+B6bVdWLbOJWWFiIn58f\nDoeDnJwc7r77bt566y1atmx5UdvRB8DWjg3zBuqUppcok2eUyTOlpzQvlz7x/CIcOHCASZMmYVkW\nJSUljB49+qLLTkREvEOFdxGuu+461q1b5+0YIiJyCXTpEBERMYIKT0REjKDCExERI6jwRETECCo8\nERExggpPRESMoMITEREj6O/wvGDx1L96O4IRThcWezuCiPgQFZ4X5Obm43L5zhXdfPFSQr6YSUT+\n3HRKU0REjKDCExERI6jwRETECCo8ERExggpPRESMoMITEREjqPBERMQIKjwRETGCCk9ERIygwhMR\nESOo8ERExAgqPBERMYIKT0REjKDCExERI6jwRETECCo8ERExggpPRESMoMITEREjqPBERMQIKjwR\nETGCCk9ERIygwhMRESOo8ERExAgqPBERMYIKT0REjKDCExERI9gsy7K8HUJERKSm+Xk7gIlGzfyE\n346f8nYMEZE/hasa12Px1L9e9nZ0SlNERIygwhMRESOo8ERExAgqPBERMYIKT0REjKDCExERI6jw\nRETECB4V3qZNm4iNjSU2Npb+/fszceLES97hvn372LRpU5llkZGRnDrlnb9LS09P58svv/TKvkVE\npPZU+YfnOTk5JCQksG7dOkJCQoCzpXWp9u7dy5YtW+jXr597mc1mu+TtXa709HQKCgq4+eabvZZB\nRERqXpWF9/vvv1OnTh0aNmzoXhYZGQnA1q1bmT9/Pi6XiyZNmpCQkECzZs1ISUnh888/Z8GCBQDu\n2wkJCSQlJVFQUEBsbCzdunVjypQpWJbF0qVL2bx5M3/88QdPPPEEf/3r2b+qnzhxIgcOHKCoqIgW\nLVowa9YsAgMDSU9P5/nnn+f6669n165d1KlThzlz5pCcnExGRgahoaEkJydzxRVXkJyczP79+zl5\n8iRHjhyhdevWzJo1iyNHjrBixQosy2L79u3079+fhx56iHXr1rF48WLsdjvNmzdnxowZNGnShJSU\nFDZu3EhQUBAZGRkEBQWRlJREcHBwTfxsRESkGlV5SjMyMpJOnTrRu3dvxowZw5IlSzhx4gTHjh1j\n0qRJzJs3j/fff58BAwYwYcIE93rnH7XZbDYaNWrEmDFjiI6OJiUlhSlTprjvDwwMZM2aNSQmJjJz\n5kz38qlTp7JmzRrWr19PREQE//znP933/fzzzwwfPpwNGzbQpUsXHnzwQZ5++mk++OAD7HY7Gzdu\ndD/266+/Zvbs2WzcuJGAgABeffVV2rZty9ChQxk4cCApKSk89NBDZGRkMG/ePN566y3ef/992rRp\nw3PPPefezp49e5g8eTIbN24kIiKCZcuWXeSQi4iIN1R5hGez2XjllVfYv38/6enpfPrppyxevJjx\n48fTrl07WrduDcDf/vY3EhISOHny5CUF6d+/PwBdunQhJyeHoqIi6tatS0pKChs2bODMmTOcPn2a\nli1butdp1aoV1113HQDt27fnyJEjXHXVVQB06NCBzMxM92NvueUWmjRpAsCQIUPKlOq5duzYQe/e\nvd1HbaWFWCoqKsp9ardz586kpaWVux2n04nT6SyzzOFwEBoa6vGYiIjIf2VlZVFSUlJmWVBQEEFB\nQR6t7/HFo9u0aUObNm0YNmwYAwYMqPR1N4fDwbkfwlBYWFjptm02G/7+/gDY7WcPOktKSvjqq69Y\nsWIFK1eupFGjRmzcuJFVq1a51ytdp3Sf59+uaL+WZVWY//z7zr99/j6Ki4vL3c6SJUtITk4usyws\nLIzU1NRyHy8iIpWLi4vj8OHDZZaNHj2a+Ph4j9avsvCOHj1KVlYWXbp0ASA7O5vjx48TERHBvn37\n+OWXX2jVqhVr166lffv21K9fn+bNm/Pjjz9y5swZLMvi448/djdwQEAA+fn5ZfZx/icUld7Oy8sj\nMDCQhg0bUlRUxHvvvefRN1WeLVu2cPz4cRo3bkxKSgrdu3d35/ntt9/cj4uOjmbRokXk5uYSHBzM\nqlWruOmmmy56fyNHjiQ2NrbMMofDccn5RURMt3z58nKP8DxVZeGVlJSQlJTEkSNH8Pf3x7Isxo0b\nR8eOHZkzZw4TJkygpKSEJk2aMHfuXODsacno6GjuuOMOwsPDiYiIICcnBzhbKIsXL2bQoEHceOON\nTJkypdzX+wB69erF+vXr6devH1dffTUdO3Zk9+7dHn9z54qOjuapp57i0KFDtG7dmsmTJwNw2223\nER8f7/6Ti4ceeojx48dz3333YbfbadasGQkJCRe9v4s5zBYRkapd7ktCRnwAbHJyMidPnuTJJ5/0\ndhRAn4cnInIx9Hl4IiIiF8GITzwfPXq0tyOIiIiX6QhPRESMoMITEREjqPBERMQIKjwRETGCCk9E\nRIygwhMRESOo8ERExAgqPBERMYIRlxYTEREx4korviY3Nx+Xy3d+z2jaNJCcnDxvxyhDmTzni7mU\nyTPK5Bm73UZwcMDlb6casoiIiPg8FZ6IiBhBhSciIkZQ4YmIiBFUeCIiYgQVnoiIGEGFJyIiRlDh\niYiIEVR4IiJiBBWeiIgYQYUnIiJG0LU0vcBut3k7wgWUyTO+mAl8M5cyeUaZqlZdefRpCbUoPz+f\ngIDLvwCqiIiJLvc5VKc0a9GJEye45557yMrK8nYUt6ysLGJiYpSpCr6YCXwzlzJ5Rpk8l5WVxT33\n3MOJEycuazsqvFq2c+dOSkpKvB3DraSkhMOHDytTFXwxE/hmLmXyjDJ5rqSkhJ07d172dlR4IiJi\nBBWeiIgYQYUnIiJGcEyfPn26t0OYxN/fn+7du+Pv7+/tKG7K5BlfzAS+mUuZPKNMnquOXPqzBBER\nMYJOaYqIiBFUeCIiYgQVXi05cOAAQ4cOpW/fvgwdOpTMzEyv5IiJiaF///4MGjSI2NhYvvzySwB2\n7drFwIED6du3L6NGjeLYsWM1liExMZFbb72VyMhI9u/f715e2RjV9PhVlKmi8YKaH7MTJ07w8MMP\n069fPwYOHMiYMWM4fvx4lfuuyVyVZYqMjGTgwIHuscrIyHCvl5qaSr9+/bj99tsZP348hYWF1ZYJ\n4PHHH3fvd/jw4ezbtw/w7pyqLJc35xVAcnJymbnurflUVa5qn1OW1IoRI0ZYGzZssCzLst5//31r\nxIgRXskRExNj7d+//4Llffr0sXbu3GlZlmW9+uqr1lNPPVVjGb7++msrOzvbiomJsTIyMtzLKxuj\nmh6/ijJVNF6WVfNjduLECSs9Pd19OzEx0ZoyZUqV+67JXJVlioyMtE6dOnXBOgUFBdbNN99sZWZm\nWpZlWVOmTLGSk5OrLZNlWVZeXp77608//dSKjY21LMu7c6qyXLfccovX5tX3339vPfjgg9Ytt9zi\nnuvemk9V5aruOaUjvFpw7NgxfvjhBwYMGADAHXfcwd69e92/Gdcmy7Kwznuf0nfffYe/vz9RUVEA\nDB06lE2bNtVYhq5duxISElImR2VjVBvjV14mKH+8oHbGrGHDhtx4443u2126dOHIkSOV7rumc1WU\nCSoeq61bt9KxY0eaNWtWI5mAMtdXzMvLw263e31OlZfL4XC4b3tjXhUVFZGQkMC5b8735nyqLBdU\n/5zSpyXUgqysLEJCQrDZzl7x2263c9VVV5GdnU3jxo1rPc/EiROxLIsbbriBcePGkZWVRVhYmPv+\n0kxOp5OgoKBayVTZGLlcLq+O37njNX78eAICAmp9zCzL4t133+XWW2+tdN+1mas002233QaAzWbj\n3nvvpaSkhJ49exIfH0+dOnUuyHTNNdeQnZ1drVkApk6d6j41uGjRIp+ZU+fnKuWNebVgwQIGDhxY\nZh++MJ/KywXVP6d0hGeYd999l3Xr1rFmzRpcLhcJCQnlPq6836pMdP54zZgxo8LH1uSYJSQk0KBB\nA4YPH37R+66pXKWZ4uLiANiyZQtr1qzh7bffZv/+/bz66qs1st+KzJw5k88//5xx48aRmJgI+MY8\nLi+XN+bVrl27+O6777jnnnuq3HZtzqfycpWq7jmlwqsFoaGhHD161D1RXC4Xv/32G1dffXWtZwkJ\nCQGgTp06DBs2jG+++YZrrrmGw4cPux9z7NgxbDZbrR3dQeVj5M3xK2+8SvPW1pglJiaSmZnJyy+/\nXOW+ayvX+Zngv2PVoEED7r77bvfFfs/PdOTIEUJDQ6s1z7nuuusuduzY4XNzqjTXH3/84ZV5lZ6e\nzi+//MKtt95KTEwMR48e5cEHHyQzM9Or86m8XKNGjWLbtm3VPqdUeLWgSZMmREZGsmHDBgA2bNhA\n+/bta/105qlTp8jPz3ff/uCDD2jfvj0dOnSgsLDQPZlWrFhBv379aiVT6RNOZWPkrfErb7zatWsH\nQMeOHWtlzObPn8/evXt59dVX8fPzq3LftZGrvExOp9P9Lrni4mI+/vhj91j17NmTPXv2uN8FuWLF\nCvr27VtteU6ePFnmdFZqaiqNGjWiSZMmtGvXzmtzqqJc/v7+XplXDz/8MFu3buWzzz4jNTWVkJAQ\n3njjDUaNGuXV+VRRrk6dOlX7nNKVVmrJf/7zHyZPnozT6aRhw4YkJibSsmXLWs1w8OBBxowZg8vl\nwuVyERERwdSpU7nyyivZtWsXzzzzDEVFRYSHhzN37lyaNGlSIzlmzpzJ5s2byc3NpVGjRjRu3JgN\nGzZUOkY1PX7lZXrttdeIj48vd7yAGh+z/fv3c+edd9KyZUv35ZSaNWtGUlIS33zzDdOmTSt33zWZ\n6/xMNpuN8PBwRo0axbRp07Db7RQXFxMVFcXTTz9NvXr1gLNP9nPmzMGyLNq1a8fs2bO54oorqiVT\nbm4ujz32GKdOncJut9OoUSMmTZpEu3btvDqnyss1efJkGjRoUOH/h1Dz86rUrbfeysKFC2nTpk2l\n+6zN54ZfhbmSAAAAQ0lEQVRzc+Xn51f7nFLhiYiIEXRKU0REjKDCExERI6jwRETECCo8ERExggpP\nRESMoMITEREjqPBERMQIKjwRETHC/wNAPMBcj8HpnQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aTn1niLPob3x" - }, - "source": [ - "与男性相比，女性存活的几率要高得多。这显然是该模型的预测特征。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Eh3KW5oYkaNS", - "outputId": "830984b0-0fbc-4d69-b107-9d4097b0549f", - "colab": { - "height": 292 - } - }, - "source": [ - "pd.concat([dftrain, y_train], axis=1).groupby('sex').survived.mean().plot(kind='barh').set_xlabel('% survive')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAETCAYAAABjv5J2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGStJREFUeJzt3XtQlXXix/HP4YiMC/IzWVK8bVkm5tqkaWar0oLkjZvX\n0TK1UHdyTVvdzdE2766pmaWMl7ZWWbTZRVcHwcuq4H3Q8tKqG66psaSS4i0uKnDg+f3hykRgHOPA\n+SLv10wzci7P+YAO784jnmOzLMsSAACG8XD3gJogOztbS5cuVXZ2trun3FNN2Cix09XY6VrsdB1X\nbCRQTsjOzlZMTIzxfxhM3yix09XY6VrsdB1XbCRQAAAjESgAgJEIFADASATKSR06dJDdbnf3jHuy\n2+1q2rSp0RsldroaO12Lna5jt9vVoUOHSh3Dxo+ZVyw3N1c+Pj7ungEANU5lvn8SqPtw/XqeiovN\n/XL5+fno6tVcd8+oEDtdi52uxU7X8PCw6aGHvCt1jDou2lIrFBdbRgdKkvH77mKna7HTtdhpBv4O\nCgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICRCBQAwEgECgBgJAIFADASgQIAGIlAAQCM\nRKAAAEYiUAAAIxEoAICRCBQAwEgECgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICRCBQA\nwEgECgBgpDruHlCT+Pn5uHtChfz967t7gm7nO5STfcvdMwDUcATqPkTP2a7L1/nGW5HERZHKcfcI\nADUep/gAAEYiUAAAIxEoAICRCBQAwEgECgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICR\nCBQAwEgECgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICRCBQAwEgECgBgpAc2UDExMVqw\nYIG7ZwAAfqIHNlAAgJqtjrsHlCcwMFBvvvmmdu7cqe+++06zZs1Samqq9u3bJ4fDoQ8//FAtW7bU\nlStXNHHiROXl5amgoEBBQUH6/e9/X+4xP/74Y23fvl0Oh0ONGjXSnDlz5OfnV82fGQDAWcY+g/q/\n//s/rV+/XpMmTdLYsWPVsWNHbdy4UZGRkVqxYoUkydfXVytXrtQ//vEPbdy4USdOnND+/fvLHGvT\npk3KyMhQfHy8NmzYoO7du2vevHnlPm52drbOnz9f6r/MzMwq/VwB4EGVmZlZ5ntqdna2U/c18hmU\nJPXu3VuS1LZtW3l4eKh79+4lH+/cuVOSVFRUpPnz5+vYsWOyLEtXr15VWlqaunbtWupYKSkp+ve/\n/62oqKiS+/n6+pb7uLGxsYqJiSl1WdOmTZWSkuLSz+9B5+9fv1LXm4KdrsVO16oJO19++WVduHCh\n1GXjxo3TG2+8UeF9jQyUzWaTl5eXJMnDw0N169Ytuc5ut8vhcEiSVq1apZycHK1fv16enp6aNm2a\n8vPzyxzPsiy9/vrr6t+/f4WPPWLECPXr16/UZXa7vTKfTq2UlZVzz+v8/ev/6PWmYKdrsdO1TN/p\n4WGTn5+P1q5dq6KiolLX3esJwg8ZGSjLsn7047tycnLk7+8vT09PXbp0ScnJyRo6dGiZ2wUHBysu\nLk49evSQr6+vCgoKdO7cOQUGBpa5ra+vr9NfPADAjwsICPjJ9zUyUDab7Uc/vuuVV17RhAkT1L9/\nfzVu3FhdunQp93aRkZG6ceOGhg0bJpvNpuLiYr300kvlBgoAYAabda+nJygjes52Xb5+y90zjJe4\nKJJTfNWIna7FTte4e4qvUsdw0RYAAFyKQAEAjESgAABGIlAAACMRKACAkQgUAMBIBAoAYCQCBQAw\nEoECABiJQAEAjESgAABGIlAAACMRKACAkQgUAMBIBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAA\nACMRKACAkQgUAMBIBAoAYKQ67h5Qk3zyxxfdPaFGuJ3vcPcEAA8AAnUfrl7NVXGx5e4Z9+TvX19Z\nWTnungEALsEpPgCAkQgUAMBIBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAAACMRKACAkQgUAMBI\nBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAAACMRKACAkQgUAMBITgfq+vXr5V6ekZHhsjEAANzl\ndKDCw8O1Z8+eUpd9+umnGjRokMtHAQBQx9kbzp07V3/84x8VEhKiV199VbNnz9bly5cVGxtblfsA\nALWU08+ggoKClJiYqCNHjqhXr15q0KCB1q9fr8DAwKrcBwCopZwOVF5enubPn6/c3FyNHDlSe/fu\n1caNG6tyGwCgFnM6UJGRkXI4HNq0aZMmT56s2NhYrVmzRmPGjKnKfQCAWsrpQE2cOFELFy5U/fr1\nJUlt2rTR+vXr9eijj1bZOABA7eV0oPr06SNJKi4u1uXLlyVJXl5emjJlStUsAwDUak4HKjs7W5Mm\nTdJTTz2lF198UZKUnJysxYsXV9k4AEDt5XSgpk+fLh8fH6WkpMjT01OS1L59e23durXKxgEAai+n\n/x1Uamqq9u3bJ09PT9lsNklSw4YNdfXq1SobBwCovZx+BlW/fv0yL3d08eJF+fv7u3wUAABOB2rQ\noEEaP368Dh48qOLiYh07dkyTJ0/WkCFDqnIfAKCWcvoU3+jRo1W3bl3NmjVLDodDU6dO1ZAhQzR8\n+PCq3AcAqKWcfgZ16NAhhYSEaMuWLdq+fbvatWunU6dO6cqVK1W5DwBQSzkdqJkzZ8put0uS5s+f\nr6KiItlsNr3zzjtVNg4AUHs5fYrv0qVLatKkiRwOh/bt26ddu3bJ09NT3bp1q8p9AIBayulA+fj4\n6MqVK/rqq6/0+OOPy9vbWwUFBXI4HFW5DwBQSzkdqGHDhmngwIEqLCzU1KlTJUlHjx5Vy5Ytq2wc\nAKD2cjpQY8aMUWhoqOx2u1q0aCFJatSokebMmVNl4wAAtZfTgZJU5pXLeSVzAEBVcfqn+AAAqE4E\nCgBgJAIFADASgQIAGIlAAQCMRKAAAEYiUAAAIxEoAICRCBQAwEgECgBgJAIFADASgQIAGIlAAQCM\nRKAAAEYiUAAAI9ksy7LcPQIAUPPczncoJ/tWudd5eNjk5+dTqePf1xsW1nbRc7br8vXyfzMAoLZJ\nXBSpnCo8Pqf4AABGIlAAACMRKACAkQgUAMBIBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAAACMR\nKACAkQgUAMBIBAoAYCQCBQAwEoECABiJQAEAjESgAABGIlAAACMRKACAkQgUAMBIBAoAYCQCBQAw\nUrUGaufOnerTp4/69++v9PT0Kn2sKVOmaO3atVX6GACAqlOnOh/s73//uyZMmKCePXtW58MCAGqg\nagvUvHnzdPjwYaWnp+vTTz/VpEmT9N577ykvL0+SNH78eAUFBenChQsaMGCABg8erH379ik/P18L\nFy7U3/72N/3rX/9SvXr1tGzZMvn5+en06dOaOXOmbt26pYKCAg0ePFjDhw8v89iFhYVavHixDh8+\nrMLCQj3xxBOaMWOG6tWrV12fPgDgPlVboKZMmaIvv/xSo0aNUocOHTR8+HD9+c9/1s9//nNlZWVp\n4MCB2rx5syTpxo0b6tixoyZOnKhPPvlEI0eO1Jo1azR79mzNnDlTa9as0YQJE9SsWTOtXr1anp6e\nunnzpgYNGqSuXbuqZcuWpR77448/lq+vr+Lj4yVJ7733nlasWKHf/e53ZXZmZ2crOzu71GV2u10B\nAQFV9JUBgAdXZmamioqKSl3m6+srX1/fCu9braf47jp69KjOnz+v0aNHy7IsSXci8N///lcNGjSQ\nt7e3unfvLkl68skn1bhxY7Vu3VqS1LZtW6WmpkqSbt26penTp+vUqVPy8PBQVlaWTp06VSZQKSkp\nysvL07Zt2yTdeUYVGBhY7rbY2FjFxMSUuqxp06ZKSUlx3RcAAB4Q/v71f/T6l19+WRcuXCh12bhx\n4/TGG29UeGy3BEqSAgMDFRcXV+byCxcuqG7duiUf2+12eXl5lfrY4XBIkt5//335+/trwYIFstls\nio6OVkFBQZljWpal6dOnq3PnzhXuGjFihPr161fqMrvd7vTnBQC1SVZWTrmXe3jY5Ofno7Vr15b7\nDMoZbvkx8/bt2ys9PV2HDh0quezEiRMlv777rKoiOTk5CggIkM1m0+nTp3X48OFybxccHKxVq1Yp\nPz9fkpSXl6ezZ8+We1tfX181a9as1H+c3gOAnyYgIKDM91RnA1Wtz6BsNpukOxFYvny55s+fr3nz\n5qmgoEAtWrTQihUrSt2uIq+//rreeustbdq0SS1atFCnTp3Kvd2YMWO0dOlSDRw4UDabTR4eHho3\nbpwee+wx13xiAACXs1nOPl2Bouds1+Xrt9w9AwCMkLgossJTfJXBK0kAAIxEoAAARiJQAAAjESgA\ngJEIFADASAQKAGAkAgUAMBKBAgAYiUABAIxEoAAARiJQAAAjESgAgJEIFADASAQKAGAkAgUAMBKB\nAgAYiUABAIxEoAAARiJQAAAjESgAgJEIFADASAQKAGAkAgUAMJLNsizL3SMAADXP7XyHcrJvlXud\nh4dNfn4+lTp+nUrdu5a5ejVXxcXm9tzfv76ysnLcPaNC7HQtdroWO83BKT4AgJEIFADASAQKAGAk\nAgUAMBKBAgAYiUABAIxEoAAARiJQAAAjESgAgJEIFADASAQKAGAkAgUAMBKBAgAYiUABAIxEoAAA\nRiJQAAAjESgAgJEIFADASAQKAGAkAgUAMBKBAgAYiUABAIxEoAAARiJQAAAjESgAgJHquHtATeLh\nYXP3hArVhI0SO12Nna7FzspzxTabZVmWC7Y80HJzc+Xj4+PuGQBQ41Tm+yen+Jxw48YNDR06VJmZ\nme6eck+ZmZkKDg42eqPETldjp2ux03UyMzM1dOhQ3bhx4ycfg0A56ejRoyoqKnL3jHsqKirShQsX\njN4osdPV2Ola7HSdoqIiHT16tFLHIFAAACMRKACAkQgUAMBI9hkzZsxw94iawMvLS507d5aXl5e7\np9xTTdgosdPV2Ola7HSdym7kx8wBAEbiFB8AwEgECgBgJAL1Penp6RoyZIh69eqlIUOGKCMjo8xt\niouLNXPmTIWGhqpnz55at26dcRsPHDigAQMGqF27dlqwYEG17rvLmZ3Lli1TWFiYoqKiNGDAAO3f\nv9/InRs2bFBERISioqIUERGhuLg4I3fede7cOT399NNu+b13ZmdMTIyef/559evXT/369dPs2bON\n3ClJW7ZsUXh4uMLDwxUREaFr164Zt3Py5MmKiopSv379FBUVpTZt2mjXrl1Gbbx27Zp+85vfKCIi\nQn369NGsWbNUXFxc8cEtlBg+fLiVmJhoWZZlJSQkWMOHDy9zm40bN1rR0dGWZVnW1atXre7du1sX\nLlwwamNGRoaVlpZmffDBB9b8+fOrbdv3ObNz//791u3bty3Lsqy0tDSrY8eOVn5+vnE7c3NzS36d\nl5dn/frXv7b+85//VNtGy3Jup2VZVlFRkTVs2DBr0qRJbvm9d2bn0qVL3fbn8i5ndh4/ftzq27ev\ndfXqVcuyLCsnJ8fIP5/fl5aWZnXu3NkqKCiojnmWZTm3ce7cuSW/5w6Hwxo0aJC1devWCo/NM6j/\nuXbtmtLS0tS3b19JUlhYmL788ktdv3691O22bt2qwYMHS5IaNmyoHj16aNu2bUZtbN68uQIDA2W3\n26tl1w85u/NXv/pVyU/3BAYGSlKZ25iw09vbu+TXN2/elMPhkM1WfS/S6exOSfroo48UHBysRx55\npNr23XU/Oy03/myWsztjY2P12muvqWHDhpIkHx8f1a1b17id37d+/XqFh4fL09PTqI02m015eXmy\nLEu3b9+Ww+FQo0aNKjw+gfqfzMxMNWrUqOQbj4eHhx5++GF9++23pW538eJFNWnSpOTjgICAans9\nLGc3uttP2blx40Y1b97cqT+0rnI/O1NSUhQWFqaQkBBFR0erVatWxu08deqUDhw4oJEjR1bbtu+7\nn6/n1q1bFRkZqejoaH3xxRdG7jx79qwyMjI0bNgw9e/fX8uXLzdy512FhYVKSkrSgAEDjNs4duxY\nff311+ratau6deumrl27qn379hUen0DB7T777DMtXbpUixcvdveUewoODlZSUpL++c9/KiEhQenp\n6e6eVIrD4dC0adM0Y8aMan1291MMHTpUycnJSkhIUHR0tMaOHavvvvvO3bPKcDgcOn36tFavXq24\nuDjt3btXCQkJ7p51Tzt27FCTJk1KzkaYZNu2bQoMDNSBAwe0d+9effbZZ9q+fXuF9yNQ/xMQEKBL\nly6VnHooLi7W5cuX1bhx41K3a9KkiS5evFjycWZmpgICAoza6G73s/PYsWOaPHmyli1bpl/84hfG\n7ryrcePGateunXbv3l1NK53bmZWVpW+++UZjxoxRcHCwYmNjtW7dOk2bNs2onZLk5+dXcvr5+eef\nV+PGjfXVV18Zt7Np06bq2bOn6tSpI29vb4WEhOjEiRPG7bxrw4YN1frsSXJ+45o1axQeHi7pzqnS\nkJAQHTp0qMLjE6j/adiwoQIDA5WYmChJSkxM1JNPPqmHHnqo1O169eql+Ph4WZala9euKTk5WS++\n+KJRG7/PHef6nd15/PhxTZw4UR9++KFb/q/P2Z3nzp0r+fW1a9d06NAhPfHEE0btDAgIUGpqqpKT\nk5WSkqIRI0Zo0KBBmjVrllE7JenSpUslv05LS9PFixf16KOPGrczLCxMBw4ckHTn9Flqaqpat25t\n3E5J+vbbb3XkyJGSCJi2sVmzZtq3b58kqaCgQKmpqc6dJq/0j3A8QM6ePWsNGjTI6tmzpzV48GAr\nPT3dsizLGj16tHXy5EnLsu78lNT06dOtHj16WKGhoVZ8fLxxGw8fPmx1797deuaZZ6wOHTpYQUFB\n1v79+43bOWDAAKtLly5WVFSUFRkZaUVFRVmnT582buef/vQnq2/fviU716xZU60bnd35fe76STln\ndk6ePNkKCwuzIiIirIEDB1p79+41cmdxcbE1b948q3fv3lZYWJj17rvvGrnTsixr+fLl1sSJE6t9\nn7MbMzIyrFdffdUKDw+3+vbta82ePdsqKiqq8Ni81BEAwEic4gMAGIlAAQCMRKAAAEYiUAAAIxEo\nAICRCBQAwEgECqhlVq5cqXfeecfdM4AK8e+gABeaO3euEhIS1LJlSy1ZskQPP/ywJGnTpk06efKk\npk6d6uaFQM3BMyjARY4fP660tDQdOHBAHTp00EcffSRJysnJ0erVqzV+/Phq2eHUG8EBNQCBAlzk\n/PnzeuaZZ+Tp6akuXbrom2++kSR98MEHGjVqlHx8fH70/nv27FHfvn3VoUMHBQUFadWqVZLuvBXJ\nSy+9VOq2gYGBJcefMmWKZsyYoTFjxqh9+/ZauXKlunbtWup1GHfs2KHIyEhJd97R9q233pIkjRo1\nSmvXri117MjISO3cuVPSnbeceO2119S5c2f17t1bW7du/alfHuC+ESjARVq1aqXDhw8rPz9fqamp\nevzxx3Xy5Emlp6erT58+Fd7/7bff1uzZs3X06FElJSXpueeeK7nuh2+h8cOPN2/erLFjx+rYsWOK\njo7Wz372Mx08eLDk+qSkpHJfSDQsLExJSUklH585c0aZmZl64YUXdOvWLUVHRysiIkIHDx7U+++/\nr1mzZuns2bNOf02AyiBQgIu0atVKoaGhGjx4sC5duqRRo0Zp7ty5evvtt/XXv/5Vw4YN0x/+8Afl\n5uaWe/+6devqzJkzys3NVf369dWmTZt7PtYP/+o4JCRETz/9dMlx+vTpU/IK07m5udq7d6/CwsLK\nHCc0NFSnTp0qedPNxMREhYaGqk6dOtq1a5eaNWumqKgo2Ww2tWnTRqGhodX2DtIAgQJcaOTIkUpI\nSNCiRYu0ZcsWderUScXFxVq3bp1iY2PVsmVLrVy5stz7LlmyRLt371ZwcLBeeeWV+3qn2R++/054\neLh27typwsJC7dixQ23bti33fYS8vb0VFBSkzZs3S5K2bNmiiIgISXfePfqLL77Qs88+q2effVad\nOnVSUlKSrly54vQuoDLquHsA8CC6cuWK4uPjFR8fr+TkZLVu3Vp2u13t2rVTXFxcuff55S9/qWXL\nlqmoqEhxcXF68803tXv3btWrV0+3bt0quV1WVlaZ+/7wlN9jjz2mJk2aaM+ePUpKSir32dNdYWFh\niomJUceOHZWfn6/OnTtLuvMeU507d9Ynn3zyU74EQKXxDAqoAu+++67Gjx8vLy8vNWvWTCdOnNDN\nmzd16NAhNW/evMztCwsLlZiYqNzcXNntdnl7e5e862xgYKDOnDmjU6dOqaCgQDExMU69rXtYWJji\n4uJ05MgR9erV65636969uy5evKglS5aU+ruyF154QV9//bUSEhLkcDhUWFioEydO8HdQqDYECnCx\nQ4cOKTc3VyEhIZKkp556SkFBQXrhhRf0+eefa/To0eXeLyEhQSEhIerYsaPi4+O1cOFCSdIjjzyi\n3/72txo5cqR69uypjh07OrWjb9+++vzzz/Xcc8+pQYMG97xd3bp1FRoaqtTU1FLPtLy9vfWXv/xF\nW7ZsUbdu3dStWzctWrRIhYWFzn4pgErhH+oCAIzEMygAgJEIFADASAQKAGAkAgUAMBKBAgAYiUAB\nAIxEoAAARiJQAAAjESgAgJH+HwQ8SRjqGoVPAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "krkRHuMp3rJn" - }, - "source": [ - "## 创建特征列与输入函数\n", - "\n", - "梯度提升（Gradient Boosting） Estimator 可以利用数值和分类特征。特征列适用于所有的 Tensorflow estimator，其目的是定义用于建模的特征。此外，它们还提供一些特征工程功能，如独热编码（one-hot-encoding）、标准化（normalization）和桶化（bucketization）。在本教程中，`CATEGORICAL_COLUMNS` 中的字段从分类列转换为独热编码列([指标列](https://tensorflow.google.cn/api_docs/python/tf/feature_column/indicator_column))：\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "upaNWxcF3rJn", - "colab": {} - }, - "source": [ - "fc = tf.feature_column\n", - "CATEGORICAL_COLUMNS = ['sex', 'n_siblings_spouses', 'parch', 'class', 'deck',\n", - " 'embark_town', 'alone']\n", - "NUMERIC_COLUMNS = ['age', 'fare']\n", - "\n", - "def one_hot_cat_column(feature_name, vocab):\n", - " return tf.feature_column.indicator_column(\n", - " tf.feature_column.categorical_column_with_vocabulary_list(feature_name,\n", - " vocab))\n", - "feature_columns = []\n", - "for feature_name in CATEGORICAL_COLUMNS:\n", - " # Need to one-hot encode categorical features.\n", - " vocabulary = dftrain[feature_name].unique()\n", - " feature_columns.append(one_hot_cat_column(feature_name, vocabulary))\n", - "\n", - "for feature_name in NUMERIC_COLUMNS:\n", - " feature_columns.append(tf.feature_column.numeric_column(feature_name,\n", - " dtype=tf.float32))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "74GNtFpStSAz" - }, - "source": [ - "您可以查看特征列生成的转换。例如，以下是在单个样本中使用 `indicator_column` 的输出：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Eaq79D9FtmF8", - "outputId": "d6db9dc1-a533-46c7-b78a-8ce367056a59", - "colab": { - "height": 51 - } - }, - "source": [ - "example = dict(dftrain.head(1))\n", - "class_fc = tf.feature_column.indicator_column(tf.feature_column.categorical_column_with_vocabulary_list('class', ('First', 'Second', 'Third')))\n", - "print('Feature value: \"{}\"'.format(example['class'].iloc[0]))\n", - "print('One-hot encoded: ', tf.keras.layers.DenseFeatures([class_fc])(example).numpy())" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "Feature value: \"Third\"\n", - "One-hot encoded: [[ 0. 0. 1.]]\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YbCUn3nCusC3" - }, - "source": [ - "此外，您还可以一起查看所有特征列的转换：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "omIYcsVws3g0", - "outputId": "165d15ee-4e7f-481d-f623-106f5dbc9d23", - "colab": { - "height": 102 - } - }, - "source": [ - "tf.keras.layers.DenseFeatures(feature_columns)(example).numpy()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "array([[ 22. , 1. , 0. , 1. , 0. , 0. , 1. , 0. ,\n", - " 0. , 0. , 0. , 0. , 0. , 0. , 1. , 0. ,\n", - " 0. , 0. , 7.25, 1. , 0. , 0. , 0. , 0. ,\n", - " 0. , 0. , 1. , 0. , 0. , 0. , 0. , 0. ,\n", - " 1. , 0. ]], dtype=float32)" - ] - }, - "metadata": { - "tags": [] - }, - "execution_count": 0 - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-UOlROp33rJo" - }, - "source": [ - "接下来，您需要创建输入函数。这些将指定如何将数据读入到我们的模型中以供训练与推理。您将使用 [`tf.data`](https://tensorflow.google.cn/api_docs/python/tf/data)API 中的 `from_tensor_slices` 方法直接从 Pandas 中读取数据。这适用于较小的内存数据集。对于较大的数据集，tf.data API 支持各种文件格式（包括 [csv](https://tensorflow.google.cn/api_docs/python/tf/data/experimental/make_csv_dataset)）,以便您能处理那些不适合放入内存中的数据集。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9dquwCQB3rJp", - "colab": {} - }, - "source": [ - "# 使用大小为全部数据的 batch ，因为数据规模非常小.\n", - "NUM_EXAMPLES = len(y_train)\n", - "\n", - "def make_input_fn(X, y, n_epochs=None, shuffle=True):\n", - " def input_fn():\n", - " dataset = tf.data.Dataset.from_tensor_slices((dict(X), y))\n", - " if shuffle:\n", - " dataset = dataset.shuffle(NUM_EXAMPLES)\n", - " # 对于训练，可以按需多次循环数据集（n_epochs=None）。\n", - " dataset = dataset.repeat(n_epochs)\n", - " # 在内存中训练不使用 batch。\n", - " dataset = dataset.batch(NUM_EXAMPLES)\n", - " return dataset\n", - " return input_fn\n", - "\n", - "# 训练与评估的输入函数。\n", - "train_input_fn = make_input_fn(dftrain, y_train)\n", - "eval_input_fn = make_input_fn(dfeval, y_eval, shuffle=False, n_epochs=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HttfNNlN3rJr" - }, - "source": [ - "## 训练与评估模型\n", - "\n", - "您将执行以下步骤：\n", - "\n", - "1. 初始化模型，指定特征和超参数。\n", - "2. 使用 `train_input_fn` 将训练数据输入模型，使用 `train` 函数训练模型。\n", - "3. 您将使用此示例中的评估集评估模型性能，即 `dfeval` DataFrame。您将验证预测是否与 `y_eval` 数组中的标签匹配。\n", - "\n", - "在训练提升树（Boosted Trees）模型之前，让我们先训练一个线性分类器（逻辑回归模型）。最好的做法是从更简单的模型开始建立基准。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JPOGpmmq3rJr", - "outputId": "0b6bd66c-90fb-40b6-ed15-7130ff63627e", - "colab": { - "height": 221 - } - }, - "source": [ - "linear_est = tf.estimator.LinearClassifier(feature_columns)\n", - "\n", - "# 训练模型。\n", - "linear_est.train(train_input_fn, max_steps=100)\n", - "\n", - "# 评估。\n", - "result = linear_est.evaluate(eval_input_fn)\n", - "clear_output()\n", - "print(pd.Series(result))" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "accuracy 0.765152\n", - "accuracy_baseline 0.625000\n", - "auc 0.832844\n", - "auc_precision_recall 0.789631\n", - "average_loss 0.478908\n", - "global_step 100.000000\n", - "label/mean 0.375000\n", - "loss 0.478908\n", - "precision 0.703297\n", - "prediction/mean 0.350790\n", - "recall 0.646465\n", - "dtype: float64\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BarkNXwA3rJu" - }, - "source": [ - "下面让我们训练提升树（Boosted Trees）模型。提升树（Boosted Trees）是支持回归（`BoostedTreesRegressor`）和分类（`BoostedTreesClassifier`）的。由于目标是预测一个生存与否的标签，您将使用 `BoostedTreesClassifier`。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tgEzMtlw3rJu", - "outputId": "2ac2d0f9-a7e9-46aa-d8f5-569a65c32251", - "colab": { - "height": 221 - } - }, - "source": [ - "# 由于数据存入内存中，在每层使用全部数据会更快。\n", - "# 上面一个 batch 定义为整个数据集。\n", - "n_batches = 1\n", - "est = tf.estimator.BoostedTreesClassifier(feature_columns,\n", - " n_batches_per_layer=n_batches)\n", - "\n", - "# 一旦建立了指定数量的树，模型将停止训练，\n", - "# 而不是基于训练步数。\n", - "est.train(train_input_fn, max_steps=100)\n", - "\n", - "# 评估。\n", - "result = est.evaluate(eval_input_fn)\n", - "clear_output()\n", - "print(pd.Series(result))" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "stream", - "text": [ - "accuracy 0.829545\n", - "accuracy_baseline 0.625000\n", - "auc 0.872788\n", - "auc_precision_recall 0.857807\n", - "average_loss 0.411839\n", - "global_step 100.000000\n", - "label/mean 0.375000\n", - "loss 0.411839\n", - "precision 0.793478\n", - "prediction/mean 0.381942\n", - "recall 0.737374\n", - "dtype: float64\n" - ], - "name": "stdout" - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hEflwznXvuMP" - }, - "source": [ - "现在您可以使用训练的模型从评估集上对乘客进行预测了。Tensorflow 模型经过优化，可以同时在一个 batch 或一个集合的样本上进行预测。之前，`eval_inout_fn` 是使用整个评估集定义的。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6zmIjTr73rJ4", - "outputId": "f934b588-cc13-4b5b-e9d1-5253606fa40d", - "colab": { - "height": 289 - } - }, - "source": [ - "pred_dicts = list(est.predict(eval_input_fn))\n", - "probs = pd.Series([pred['probabilities'][1] for pred in pred_dicts])\n", - "\n", - "probs.plot(kind='hist', bins=20, title='predicted probabilities')\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEQCAYAAABIqvhxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtUVOX+BvBnGG4mjAIHcQCttJLMo0imcTTNuybe8qCk\nKJrXVHTVMTMTITUN8ZJB6tFc65B5NPVkOhXkOdDFW2p5wZZaaSoGAxhgY3KTmff3hz8GJ1D2Zmb2\nDPh81nIt5519+c43nKf97gsqIYQAERGRDC6OLoCIiBoehgcREcnG8CAiItkYHkREJBvDg4iIZGN4\nEBGRbAwPapBef/11rFu3DgDw3XffYfDgwYrsNyQkBFevXlVkX3369MGRI0fqte748eOxe/fuWt/T\n6/UICwtD1VX6dy6r0+kwefLku25XyV6Tc2N4UIPXpUsXpKWl1bncnj17MHbsWKv2pVKprFrfGWi1\nWpw4caLWzzJ06FBs2bLF/PrPYSm119T4MTzI4YxGoyL7EUJY/eVvq3tqlfrM1moMYUn2wfAgu+jT\npw82bdqEIUOGoFu3bli4cCEqKioAAMeOHUOvXr2wefNm9OjRAwsXLgQAfPnllxgxYgSeeuopvPDC\nC/jxxx/N2zt79iyef/55PPnkk3j55ZdRXl5ufq9qe1Xy8vIQGxuL8PBwPP3001i2bBkuXryIhIQE\nnDp1Cp07d0bXrl0BABUVFUhMTETv3r3Ro0cPJCQkmOsEgPfffx89evRAz5498Z///OeeX6bjx4/H\nmjVrEBkZiS5dumDWrFkwGAwAgJycHISEhGD37t3o3bs3Jk6cCADIyMhAREQEunbtigkTJuDixYsW\n2zxz5kytPTQYDJgxYwbCw8PRrVs3zJgxA/n5+RbrZmdn37MWk8lU4zPceXQWHR0NIQSGDRuGsLAw\npKWl1eh1QUEB5syZg/DwcPTr1w9bt241v5eVlYVRo0bhySefRI8ePZCYmHjX3lEDJIjsoHfv3iIi\nIkLk5eWJ33//XURFRYl33nlHCCHE0aNHRfv27cXq1atFRUWFKC8vFz/88IMIDw8XWVlZwmQyiT17\n9ojevXuLiooKUVFRIXr37i1SU1NFZWWlSE9PF0888YTF9nr16iWEEMJoNIphw4aJt99+W5SVlYny\n8nLx/fffCyGE+Pjjj8XYsWMt6ly2bJl46aWXhMFgEDdv3hQzZswQa9asEUII8fXXX4vu3buLCxcu\niNLSUvHKK6+IkJAQkZ2dXetnjo6OFj179jQvHxsbK+bNmyeEEOLXX38V7dq1E6+99pooLS0V5eXl\n4tKlSyI0NFQcPnxYVFZWis2bN4v+/fuLW7du1dnD4uJisX//flFeXi5u3rwp5s6dK2bOnCm5lpCQ\nEGE0Gs3L7tq1q9YetWvXzuLz3tlrk8kkRo4cKdavXy8qKyvF1atXRb9+/cTBgweFEEKMGTNG7N27\nVwghRElJiTh9+rSEnxxqKHjkQXYzfvx4BAQEQKPRYMaMGfjss8/M77m4uCA2NhZubm5wd3fHrl27\nEBUVhb/+9a9QqVQYMWIE3N3dcfr0aZw+fRqVlZWYMGEC1Go1Bg4ciA4dOtS6z9OnT+PatWt49dVX\n4eHhAXd3d4SFhd21xt27d+P111+Ht7c3HnjgAUybNg2ffvopACA9PR3PP/882rZtC09PT8TGxtb5\nmYcPH25efu7cuUhLSzNPdalUKsTGxsLT0xPu7u74/PPP8eyzzyI8PBxqtRqTJ09GWVkZTp48WWcP\nmzdvjv79+8Pd3R0PPPAApk+fju+++05yLbaQlZWF69ev46WXXoJarUZwcDAiIyPNNbq6uiI7OxvF\nxcVo0qQJOnbsaLN9k+O5OroAarwCAgLMfw8KCkJBQYH5ta+vL9zc3Myvc3NzsXfvXnz44YcAbp9b\nqKysNK9z57aqtlebvLw8BAYGwsWl7v8vKioqQmlpKUaNGmUeM5lM5i/YgoICi5AKDAys88u3ZcuW\nFstXVlaiuLi41vcLCgoQGBhofq1SqaDVai2mn+7Ww7KyMixfvhwHDx6EwWCAEAIlJSUW53XqqsVa\nubm5yM/PN08BCiFgMpnw1FNPAQCWL1+OdevWYfDgwWjVqhVmzZqFZ5991mb7J8dieJDd5OXlmf+e\nk5ODFi1amF//+dxBy5YtMWPGDEyfPr3Gdo4fP15jPj83NxetW7eusaxWq4Ver4fJZKoRIH/ep4+P\nD5o0aYJPP/3UorYq/v7+Fp8hNze3zhPIf17ezc0NPj4+KC0trVFDixYt8PPPP1usr9frLb7079bD\nLVu24PLly9i9ezd8fX1x/vx5jBw50iI86qrFWlqtFsHBwfjiiy9qfb9169ZYvXo1AOCLL77AnDlz\ncOzYMXh6etpk/+RYnLYiu9m2bRvy8/Nx/fp1bNq0Cc8999xdlx09ejR27NiBrKwsAEBJSQm+/vpr\nlJSUIDQ0FK6urti6dSuMRiP279+PM2fO1Lqdjh07wt/fH6tWrUJpaSkqKipw4sQJAICfnx/y8vJw\n69YtALe/yCMjI7F8+XIUFRUBAPLz83Hw4EEAwODBg/Hxxx/j4sWLKC0txXvvvVfnZ963b595+Xff\nfReDBg0yf5n/+ahl8ODB+Oqrr/Dtt9+isrISW7ZsgYeHB0JDQ+vsYUlJCTw9PeHl5YXr168jOTnZ\nqlru5i9/+ctd72vp2LEjvLy8sHnzZpSXl8NoNOLnn382/7fZt2+fua/e3t5QqVSSjgipYWjw/yUN\nBgOSk5PNV5Lcz5ytFxEREXjxxRcxYMAAtG7dGi+99NJdl+3QoQOWLl2KJUuWoGvXrhg4cCD27NkD\nAHBzc0NycjI+/vhjdO3aFenp6RgwYECt23FxccHGjRtx8eJFPP300+jZs6f5voSnn34ajz76KHr0\n6IHw8HAAwLx58/Dggw9i9OjR6NKlC1588UVcvnwZANCzZ0/ExMQgJiYGAwcONK9zL8OHD8eCBQvw\nzDPP4NatW3jjjTfM7/35qOXhhx9GUlISli5divDwcHz11VfYuHEjXF1dzcvfrYcxMTEoLS1Ft27d\nEBUVZXEFVNW6VbX06NEDP/74I+bMmVNrLfc6moqNjcX8+fPNfa+t1+fPn0ffvn3xt7/9DXFxcfjj\njz8AAAcOHEBERATCwsKwYsUKrF27Fu7u7nX20J6c7d+II1ndCyXPzpeXl4v4+HgxYMAAMXToUBEX\nFyeEEOLSpUtizJgxYuDAgWLMmDHiypUrkrd59epV8dhjj4mrV6/aq+wGw5l60bt3b3H48GGH7d8R\nvbjzqiVn4kw/F47GXlSztheKnvNYuXIlPD09zXOkVYe08fHxiI6ORkREBPbt24e4uDikpqYqWRoR\nEcmg2LRVSUkJ9u7di7lz55rHfH19UVRUhHPnzmHIkCEAbk91nD171qZXhZDy7sc7k+/Hz0z3L8WO\nPLKzs9G8eXMkJyfj6NGjaNq0KebOnQtPT08EBASY/+G5uLigRYsWyMvLg4+Pj1LlkY1lZGQ4ugTF\nffDBB44ugUgxioWH0WjE1atX0aFDB8yfPx9ZWVmYMWMG1q1bJ/nKD4PBUOPkTl5eHsLCwqBWq+1R\ndoOiVqsRFBTEXoC9uBN7UY29qKZWqxEWFmZxSXcVjUYDjUZzz/VVQuo3t5WKi4vxzDPP4IcffjCP\nRUREYPny5ZgyZQqOHj0KlUoFk8mEbt26Yf/+/TWOPJKTk5GSkmIxFhYWhu3btyvxEYiIGp0XXnjB\nfDl7ldmzZ9f5RAXFjjx8fHzQrVs3HDp0CN27d8elS5dQWFiINm3aICQkBDqdDsOGDYNOp0P79u1r\nnbKKiYnByJEjLcaq/g+iuPgmTCbb5aCfnxcmL9tf7/W3LBqAwsI/bFaPVH5+Xg7ZrzNiL6qxF9XY\ni9tcXFTw8WmKNWvW1HjKc11HHYDCd5gnJCRg4cKFePvtt+Hm5oakpCR4eXkhISEBCxYswPr169Gs\nWbO7Pn3zXodSJpOwaXgAQEGxdXfi2roeZ9+vM2IvqrEX1diLalqttl7rKRoerVq1snhkc5U2bdpg\n586dSpZCRERWaPB3mBMRkfIYHkREJBvDg4iIZGN4EBGRbAwPIiKSjeFBRESyMTyIiEg2hgcREcnG\n8CAiItkYHkREJBvDg4iIZGN4EBGRbAwPIiKSjeFBRESyMTyIiEg2hgcREcnG8CAiItkYHkREJBvD\ng4iIZGN4EBGRbAwPIiKSjeFBRESyMTyIiEg2hgcREcnG8CAiItkYHkREJBvDg4iIZGN4EBGRbK5K\n7qxPnz7w9PSEu7s7VCoV5s2bh+7du+PUqVOIj49HeXk5goKCkJSUBF9fXyVLIyIiGRQND5VKheTk\nZLRt29ZifP78+UhMTETnzp2xYcMGrFq1CsuXL1eyNCIikkHRaSshBIQQFmNnzpyBh4cHOnfuDACI\niopCWlqakmUREZFMih55AMC8efMghMCTTz6Jl19+GXq9HkFBQeb3fXx8AAAGgwEajUbp8oiISAJF\nw2P79u0ICAjArVu38NZbb2HJkiXo379/jeX+fHRSxWAwwGAwWIyp1WpotVq71EtE1Njp9XoYjUaL\nMY1GU+f/vCsaHgEBAQAANzc3jB07FjNnzkRMTAxycnLMyxQVFUGlUtVaeGpqKlJSUizGgoKCkJmZ\nCT8/L/sWXw/+/t731X6dEXtRjb2oxl5UGzdunMV3MADMnj0bsbGx91xPsfAoLS2F0WiEl9ftL/nP\nPvsM7du3xxNPPIHy8nKcOHECYWFh2LFjBwYPHlzrNmJiYjBy5EiLMbVaDQAoLPwDJlPtRyz1YYsf\nrmvXbtigEnn8/b0dsl9nxF5UYy+qsRe3ubio4OfnhW3bttV65FEXxcLjt99+w5w5c2AymWAymdC2\nbVssXrwYKpUKK1euRFxcHCoqKhAcHIykpKRatyHlUIqIiKSr77S/YuHRqlUr7Nmzp9b3QkNDodPp\nlCqFiIisxDvMiYhINoYHERHJxvAgIiLZGB5ERCQbw4OIiGRjeBARkWwMDyIiko3hQUREsjE8iIhI\nNoYHERHJxvAgIiLZGB5ERCQbw4OIiGRjeBARkWwMDyIiko3hQUREsjE8iIhINoYHERHJxvAgIiLZ\nGB5ERCQbw4OIiGRjeBARkWwMDyIiko3hQUREsjE8iIhINoYHERHJxvAgIiLZGB5ERCQbw4OIiGRz\nSHikpKQgJCQEFy5cAACcOnUKw4cPx6BBgzB58mQUFRU5oiwiIpJI8fA4e/YsTp8+jcDAQPPY/Pnz\nkZCQgPT0dHTp0gWrVq1SuiwiIpJB0fCoqKjAkiVLkJCQYB47c+YMPDw80LlzZwBAVFQU0tLSlCyL\niIhkclVyZ++++y6GDx+OoKAg85her7d47ePjAwAwGAzQaDQW6xsMBhgMBosxtVoNrVZrx6qJiBov\nvV4Po9FoMabRaGp8//6ZYuFx6tQpnDlzBvPmzTOPCSFqXfZu46mpqUhJSbEYCwoKQmZmJvz8vGxX\nrI34+3vfV/t1RuxFNfaiGntRbdy4ccjJybEYmz17NmJjY++5nmLhcezYMVy6dAl9+/aFEAL5+fmY\nMmUKxo8fb1F4UVERVCpVrakXExODkSNHWoyp1WoAQGHhHzCZag+d+rDFD9e1azdsUIk8/v7eDtmv\nM2IvqrEX1diL21xcVPDz88K2bdtqPfKoi2LhMW3aNEybNs38uk+fPti8eTPatGmDnTt34sSJEwgL\nC8OOHTswePDgWrch5VCKiIikq++0v6LnPO6kUqkghIBKpcLKlSsRFxeHiooKBAcHIykpyVFlERGR\nBA4Lj4yMDPPfQ0NDodPpHFUKERHJxDvMiYhINoYHERHJxvAgIiLZGB5ERCSb5PD44IMP+MBCIiIC\nICM8Dh8+jL59+2L69On4/PPPUVFRYc+6iIjIiUkOj40bNyIzMxM9e/ZEamoqunfvjjfeeAPHjx+3\nZ31EROSEZJ3z8PHxwbhx4/DRRx9h69atOHPmDCZMmIA+ffpgw4YNuHnzpr3qJCIiJyL7JsEjR45g\n3759yMjIQIcOHTBlyhQEBgbigw8+wNSpU/Hvf//bHnUSEZETkRweiYmJ+Oyzz+Dt7Y3hw4dDp9Mh\nICDA/H6nTp3QtWtXuxRJRETORXJ4lJeXIyUlBR07dqz1fTc3N+zevdtmhRERkfOSHB7Tp0+Hp6en\nxdjvv/+OsrIy8xFI27ZtbVsdERE5JcknzGfOnIm8vDyLsby8PMyePdvmRRERkXOTHB6XLl1Cu3bt\nLMbatWuHX375xeZFERGRc5McHn5+frhy5YrF2JUrV9C8eXObF0VERM5NcniMGjUKsbGx+PLLL3Hh\nwgVkZmZizpw5iIyMtGd9RETkhCSfMJ82bRpcXV2RmJiIvLw8tGzZEpGRkZg0aZI96yMiIickOTxc\nXFwwZcoUTJkyxZ71EBFRAyDrDvNffvkF58+fR0lJicX43//+d5sWRUREzk1yeGzcuBHvvfceQkJC\nLO73UKlUDA8iovuM5PBITU3Frl27EBISYs96iIioAZB8tZWnpyfatGljz1qIiKiBkBwec+fOxbJl\ny1BQUACTyWTxh4iI7i+Sp60WLFgAANi1a5d5TAgBlUqFc+fO2b4yIiJyWpLDIyMjw5512IyLiwqu\n7q4Qov7bUKlsVw8RUWMkOTyCgoIAACaTCb/99htatGhht6KsoVa74EbpLWz4OKve25gTGYpmNqyJ\niKixkRweBoMBb775Jr744gu4urri1KlTyMjIQFZWFl5++WV71ihbSVklsn7+rd7rl5ZX2rAaIqLG\nR/IJ8/j4eHh5eSEzMxNubm4AgM6dOyMtLc1uxRERkXOSfORx5MgRHDhwAG5ublD9/0kBX19fFBYW\nSt7ZrFmzkJOTA5VKhaZNm2LRokUICQnB5cuXsWDBAly/fh3NmzfHypUr0bp1a/mfhoiIFCH5yMPb\n2xvFxcUWY7m5ufD395e8s8TERHzyySfYs2cPJk2ahIULFwK4fVQTHR2N9PR0jB07FnFxcZK3SURE\nypMcHpGRkZgzZw6+/fZbmEwmnDx5Eq+99hqioqIk78zLy8v89xs3bsDFxQVFRUU4d+4chgwZAgCI\niIjA2bNnawQVERE5D8nTVlOnToW7uzuWLFmCyspKLFy4EGPGjEFMTIysHS5atAiHDh0CALz//vvQ\n6/UICAgwT4W5uLigRYsWyMvLg4+Pj8W6BoMBBoPBYkytVkOr1cqqgYiIbtPr9TAajRZjGo0GGo3m\nnutJDg+VSoWJEydi4sSJ9SqwyrJlywAA+/btQ2JiIubOnQsh8aaM1NRUpKSkWIwFBQUhMzMTfn7V\nRzW/3aiwqkaVi21u9PD397bJdhrKfp0Re1GNvajGXlQbN24ccnJyLMZmz56N2NjYe64n64T53YSH\nh0vdjNmwYcMQFxcHrVaL/Px8893qJpMJBQUFaNmyZY11YmJiMHLkSIsxtVoNACgs/AMmk4Cbmxom\na+4QBCBM1q1f5dq1GzbZjhz+/t4O2a8zYi+qsRfV2IvbXFxU8PPzwrZt22o98qiL5PB44403LF4X\nFxfj1q1bCAgIkHT3eUlJCQwGgzkUMjMz0bx5c/j6+uLxxx+HTqfDsGHDoNPp0L59+xpTVoC0Qyki\nIpKuvtP+ksMjMzPT4rXRaMSGDRvQtGlTSeuXlpZi7ty5KC0thYuLC5o3b46NGzcCABISErBgwQKs\nX78ezZo1Q2JiooyPQERESpP1mwTvpFarMWPGDPTq1UvS7zH38/PDRx99VOt7bdq0wc6dO+tbChER\nKUzypbq1OXTokPkqKSIiun9IPvLo1auXRVCUlpaioqIC8fHxdimMiIicl+TwSEpKsnjdpEkTPPzw\nwxY3/hER0f1Bcnh07drVnnU0OhW3jFZfS15WXokbhlIbVUREZDuSw+PVV1+VdH5j5cqVVhXUWLi7\nqTH0H3ut2oZu9XDwanQickaST5hrNBr873//g9FoRMuWLWEymZCRkQGNRoPWrVub/xARUeMn+cjj\n8uXL2LRpE7p06WIe++6777BhwwZs2bLFLsUREVHtvDVN4OlR77stUG7lL72TvOdTp06hU6dOFmOd\nOnXCyZMnrSqAiIjk8/RwtWpqXLd6uFX7lzxt1b59e6xZswZlZWUAgLKyMqxduxaPP/64VQUQEVHD\nI/nIY8WKFZg3bx66dOkCjUYDg8GADh061LiEl4iIGj/J4REcHIwdO3ZAr9ejoKAA/v7+CAwMtGdt\nRETkpGQ9nqS4uBhHjx7FsWPHEBgYiPz8fOTl5dmrNiIiclKSw+PYsWMYNGgQdDod1q9fDwC4cuUK\nEhIS7FUbERE5KcnhsXz5crzzzjvYsmULXF1vz3Z16tQJWVlZdiuOiIick+TwyMnJMf/GwKo7zd3c\n3Gr8BioiImr8JIdH27ZtceDAAYuxw4cP47HHHrN5UURE5NwkX221YMECTJ8+Hc8++yzKysqwePFi\nZGZmms9/EBHR/UPykUdoaCj27duHRx55BKNGjUJwcDB2796Njh072rM+IiJyQpKOPIxGIyZOnIgt\nW7Zg6tSp9q6JiIicnKQjD7VajV9//RUmk8ne9RARUQMgedpq1qxZSEhIQE5ODoxGI0wmk/kPERHd\nXySfMF+0aBEA4JNPPjFfqiuEgEqlwrlz5+xTHREROaU6w+PatWvw9/dHRkaGEvUQEVEDUOe01cCB\nAwEAQUFBCAoKwooVK8x/r/pDRET3lzrDQwhh8frYsWN2K4aIiBqGOsOj6vwGERFRlTrPeRiNRnz7\n7bfmI5DKykqL1wDMz7wiIqL7Q53h4efnh4ULF5pfN2/e3OK1SqXiyXQiovtMneGRmZlpkx1dv34d\n8+fPx9WrV+Hu7o4HH3wQb775Jnx8fHDq1CnEx8ejvLwcQUFBSEpKgq+vr032S0REtifrNwlaQ6VS\nYerUqUhLS8PevXsRHByM1atXAwDmz5+PhIQEpKeno0uXLli1apVSZRERUT0oFh7NmjXDU089ZX4d\nGhqK3NxcnDlzBh4eHujcuTMAICoqCmlpaUqVRURE9SD5DnNbEkJg+/bt6Nu3L/R6vcW9Ij4+PgAA\ng8EAjUZjsZ7BYIDBYLAYU6vV0Gq19i+aiKgR0uv1NX6pn0ajqfH9+2cOCY8lS5agadOmiI6Oxv79\n+2u8/+d7S6qkpqYiJSXFYiwoKAiZmZnw8/Myj/12o8Kq+lQuznN5sr+/tyLrNFbsRTX2ohp7UW3c\nuHHIycmxGJs9ezZiY2PvuZ7i4ZGYmIjs7Gz885//BABotVqLwouKiqBSqWpNvZiYGIwcOdJiTK1W\nAwAKC/+AySTg5qaG6S7hI5UwWbe+LV27dkPW8v7+3rLXaazYi2rsRbXG0gtbBeC2bdtqPfKoi6Lh\nsXbtWpw9exabNm2Cq+vtXXfo0AHl5eU4ceIEwsLCsGPHDgwePLjW9aUcShERkXT1nfZXLDwuXLiA\nTZs24aGHHsKYMWMAAK1atUJycjISExOxePFiVFRUIDg4GElJSUqVRURE9aBYeDzyyCN3fXR7586d\nodPplCqFiIispNilukRE1HgwPIiISDaGBxERycbwICIi2RgeREQkG8ODiIhkY3gQEZFsDA8iIpKN\n4UFERLIxPIiISDaGBxERycbwICIi2RgeREQkG8ODiIhkY3gQEZFsDA8iIpKN4UFERLIxPIiISDaG\nBxERycbwICIi2RgeREQkm6ujC6C7q7hlhL+/t+z1qtYpK6/EDUOprcsiImJ4ODN3NzWG/mNvvdfX\nrR6OGzash4ioCqetiIhINoYHERHJxvAgIiLZGB5ERCQbw4OIiGRTLDwSExPRt29fhISE4MKFC+bx\ny5cvIyoqCoMGDUJUVBSys7OVKomIiOpJsUt1+/fvj4kTJ2Ls2LEW4/Hx8YiOjkZERAT27duHuLg4\npKamKlUWEZHivDVN4OnRsO+UUKz6sLAwAIAQwjxWVFSEc+fOYciQIQCAiIgILF26FMXFxfDx8VGq\nNCIiRXl6uFp1Dxdw+z4uR3LoOQ+9Xo+AgACoVKrbxbi4oEWLFsjLy3NkWUREVIcGddxkMBhgMBgs\nxtRqNbRarYMqIiJq2PR6PYxGo8WYRqOBRqO553oODQ+tVov8/HwIIaBSqWAymVBQUICWLVvWunxq\naipSUlIsxoKCgpCZmQk/Py/z2G83KqyqS+Wismp9Z1KfZ2PdqeKWEe5uaoetby1rP39jwl5UYy+q\njRs3Djk5ORZjs2fPRmxs7D3Xc0h4VJ338PX1RUhICHQ6HYYNGwadTof27dvf9XxHTEwMRo4caTGm\nVt/+Yios/AMmk4CbmxqmO86r1Ks+k3XrO5Nr16x7upW/v7fVz9eytob68vf3dti+nQ17Uc0ZeuFM\n4bVt27Zajzzqolh4LFu2DP/9739RWFiISZMmwcfHBzqdDgkJCViwYAHWr1+PZs2aITEx8a7bkHIo\nRURE0tV32l+x8Fi0aBEWLVpUY7xNmzbYuXOnUmXcV+r7SHeyPVtcmslH7N9mbS8rbhnrXsjONTQG\n9/enb+SsfaQ74PjLARsLW12ayYkn63tpi59pZ6jB0fh4EiIiko3hQUREsnHaiogks3aun+dtGg+G\nBxFJZou5fp63aRw4bUVERLIxPIiISDZOW5HTs2ae3d/fm/Ps/4/3JpAt8SeJnB7n2W2jMTwGnJwH\np62IiEg2HnkQScApn8aDj+2xDf5rIJKAj6NoPPjYHtvgtBUREcnG8CAiItkYHkREJBvDg4iIZGN4\nEBGRbAwPIiKSjZfqUqPXWK7rbyyfgxoHhgc1eo3lun5rP4czfAZqPDhtRUREsjE8iIhINk5bkV1x\nnp6ocWJ4kF01lvMNRGSJ01ZERCQbjzyISDGcxmw8GB5EpBhOYzYenLYiIiLZnCY8Ll++jKioKAwa\nNAhRUVHIzs52dElERHQXThMe8fHxiI6ORnp6OsaOHYu4uDhHl0RERHfhFOFRVFSEc+fOYciQIQCA\niIgInD17FsXFxQ6ujIiIauMU4aHX6xEQEACVSgUAcHFxQYsWLZCXl+fgyoiIqDYN6morg8EAg8Fg\nMaZWq6FIH2abAAAGG0lEQVTVauHiUhU8KrQK8EJS7DP13o/WrykAoIVPk/oXa4P1WQNrYA2swd7b\n0Ov1MBqNFmMajQYajeae66mEEMLqvVupqKgIgwYNwtGjR6FSqWAymdCtWzfs378fPj4+5uWSk5OR\nkpJisW5YWBi2b9+udMlERI3CCy+8gBMnTliMzZ49G7GxsfdeUTiJ8ePHi7179wohhPjkk0/EhAkT\naizz+++/i6tXr1r8OX78uIiKihK5ublKl+x0cnNzRe/evdkLwV7cib2oxl5Uy83NFVFRUeL48eM1\nvld///33Otd3mmmrhIQELFiwAOvXr0ezZs2QmJhYY5m7HUqdOHGixmHX/choNCInJ4e9AHtxJ/ai\nGntRzWg04sSJE2jZsiWCg4Nlr+804dGmTRvs3LnT0WUQEZEETnG1FRERNSwMDyIikk2dkJCQ4Ogi\nrOXh4YFu3brBw8PD0aU4HHtRjb2oxl5UYy+qWdMLp7hUl4iIGhZOWxERkWwMDyIikq3BhIeUR7ab\nTCa8+eab6N+/PwYOHIhdu3Y5oFL7k9KL9evXIyIiAiNGjMCoUaNw8OBBB1Rqf3Ie5f/LL78gNDQU\nK1euVLBC5Ujtxeeff46hQ4di6NChGDZsGIqKihSu1P6k9KKoqAjTp0/HsGHD8Nxzz2HJkiUwmUwO\nqNa+EhMT0bdvX4SEhODChQu1LlOv706738ZoIxMmTBA6nU4IIcTevXtrvQN9z549YvLkyUIIIQoL\nC0XPnj1FTk6OonUqQUovDh48KMrKyoQQQpw7d0506dJFlJeXK1qnEqT0QgghjEajiI6OFv/4xz9E\nYmKikiUqRkovsrKyxJAhQ0RhYaEQQogbN27ctz8Xb731lvlnobKyUkRGRoq0tDRF61TC999/L/Ly\n8kSfPn3Ezz//XOsy9fnubBBHHlIf2Z6WlobRo0cDAHx9fdGvXz+kp6crXq89Se1F9+7dzVdQhISE\nAECje8S9nEf5b9q0CX369MFDDz2kcJXKkNqL1NRUvPjii/D19QUAeHl5wd3dXfF67UlqL1QqFW7e\nvAkhBMrKylBZWYmAgABHlGxXYWFhCAgIgLjHtVH1+e5sEOEh9ZHtubm5CAwMNL/WarXQ6/WK1mpv\n9Xl8/Z49e9CqVatG9w9Dai/Onz+PQ4cOYeLEiQ6oUhlSe3Hx4kVkZ2cjOjoazz//PDZs2OCIcu1K\nai9mzpyJS5cuoUePHnjmmWfQo0cPdO7c2RElO1x9vjsbRHhQ/R07dgzJyclYu3ato0txiMrKSixe\nvBgJCQnmL5P7WWVlJX766Sf861//wtatW/HNN99g7969ji7LIdLT0xESEoJDhw7hm2++wbFjx7B/\n/35Hl9VgNIjw0Gq1yM/PNx92mUwmFBQUoGXLlhbLBQYGIjc31/xar9dDq9UqWqu9Se0FAJw8eRKv\nvfYa1q9fjwcffFDpUu1OSi+uXbuGq1evYtq0aejTpw9SU1Oxa9cuLF682FFl24XUn4ugoCAMHDgQ\nrq6uaNq0Kfr27YszZ844omS7kdqLDz/8EEOHDgVwe/qub9++OHr0qOL1OoP6fHc2iPDw9fVFSEgI\ndDodAECn06F9+/YWv+sDAAYNGoSdO3dCCIGioiJkZGRgwIABjijZbqT2IisrC6+88grWrVtnPufR\n2EjphVarxZEjR5CRkYHMzEzExMQgMjISS5YscVTZdiH15yIiIgKHDh0CANy6dQtHjhxBu3btFK/X\nnqT2Ijg4GAcOHAAAVFRU4MiRI3j00UcVr9cZ1Ou701Zn9O3t4sWLIjIyUgwcOFCMHj1aXL58WQgh\nxNSpU8UPP/wghLh9RU18fLzo16+f6N+/v9i5c6cjS7YbKb0YNWqUCA8PFyNGjBDDhw8XI0aMED/9\n9JMjy7YLKb24U3JycqO92kpKL0wmk1ixYoUYPHiwiIiIEG+//bYjS7YbKb3Izs4WkyZNEkOHDhVD\nhgwRS5cuFUaj0ZFl28XSpUtFz549xRNPPCG6d+8uIiIihBDWf3fy8SRERCRbg5i2IiIi58LwICIi\n2RgeREQkG8ODiIhkY3gQEZFsDA8iIpKN4UFERLIxPIiISLb/A+ehpDT73FBfAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - } - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBUaNN1BzJHG" - }, - "source": [ - "最后，您还可以查看结果的受试者工作特征曲线（ROC），这将使我们更好地了解真阳性率与假阴性率之间的权衡。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NzxghvVz3rJ6", - "outputId": "98bcd7d1-35b8-4593-b22f-ca020c2323fa", - "colab": { - "height": 307 - } - }, - "source": [ - "from sklearn.metrics import roc_curve\n", - "\n", - "fpr, tpr, _ = roc_curve(y_eval, probs)\n", - "plt.plot(fpr, tpr)\n", - "plt.title('ROC curve')\n", - "plt.xlabel('false positive rate')\n", - "plt.ylabel('true positive rate')\n", - "plt.xlim(0,)\n", - "plt.ylim(0,)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [ - { - "output_type": "display_data", - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEiCAYAAAA8ij+xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xlc1HX+B/DXzCAIAnIIOOCRdohHHngveCEKCopaXkVh\npqSp66b91DSF1dXSrTVXwrRcw2zdtDTF1NV0KzMSTVdtEzWPQG45HAE5Zubz+4OYaeLwC8PMMPB6\nPh4+iu98Z+Y974fOi8/3+HxkQggBIiIiI8gtXQAREVk/hgkRERmNYUJEREZjmBARkdEYJkREZDSG\nCRERGY1hQkRERrOxdAFE5hYYGIjc3FwoFAo4ODhgyJAhWLVqFezt7XX7nD9/Hps2bcLly5ehUCjQ\nr18/vPrqq3j00Ud1+xQWFmLTpk348ssvce/ePXh4eGD48OGYO3cuXFxcLPHRiCyGIxNqlrZu3Yrz\n58/jwIED+Omnn7B161bdYxcuXMCLL76IUaNG4dtvv8WJEyfQpUsXTJ8+HXfu3AEAlJeXIzIyEjdv\n3sT27dtx/vx57N69Gy4uLrh06ZLJ6tZoNCZ7bSJjMEyoWaqc+MHd3R0BAQFITk7WPfbWW29h0qRJ\niIiIgIODA5ydnfGnP/0JvXr1QmxsLADg888/R1ZWFt5991107twZAODm5oa5c+di6NCh1b7n9evX\nMXPmTAwcOBABAQHYtm0bAOC1117Dpk2bdPslJSVh2LBhup8DAwPx/vvvY/z48ejTpw+2bNmCP/7x\njwav/Ze//AVr164FUDFiWrFiBQICAjBs2DC888474EQXZGoME2rWMjMz8c0336Bjx44AgJKSEly4\ncAHBwcFV9h0zZgxOnz4NAEhMTMSQIUPQsmVLSe9TVFSEF154AUOHDsWpU6dw7NgxDBo0qMb9ZTKZ\nwc+HDx/G+++/j3PnziE8PBynTp1CUVERAECr1eLo0aMYN24cAGDJkiVo0aIFTpw4gf379+O7777D\n3r17JdVJVF8ME2qW5s2bBz8/PwwfPhxt2rTBggULAAD37t2DVquFh4dHled4eHggPz8fAFBQUFDt\nPjX56quv4OnpiRkzZsDW1hYODg7o2bOn5Oc///zz8PLygq2tLby9vdGtWzd8+eWXACqCrfL17t69\ni1OnTmH58uWws7ODm5sbIiMjcejQIcnvRVQfPAFPzVJcXBwGDRqEc+fOYfHixcjPz4ejoyOcnZ0h\nl8uRk5ODTp06GTwnJycHrq6uAAAXFxfk5ORIfr+MjAy0b9++3vW2bdvW4OfQ0FB88cUXCA8Px6FD\nhxAWFgYASE9Ph1qtRkBAAICKw3lCCCiVynq/N5EUHJlQs1R5DqFfv36YMGEC3nzzTQCAvb09evfu\njaNHj1Z5zpEjR/CHP/wBADB48GCcOnUKJSUlkt5PqVQiJSWl2sfs7e0NXkdKSIWEhCApKQlZWVn4\n8ssvdWGiVCphZ2eHM2fOICkpCWfPnsW5c+eQkJAgqU6i+mKYULMXGRmJ7777TncSfvHixdi/fz92\n7dqFoqIi3Lt3Dxs3bsTFixcxb948AEB4eDiUSiUWLFiAmzdvQgiB/Px8bN26Fd98802V9xg+fDju\n3r2LnTt3oqysDEVFRbqrvrp27Yqvv/4a9+7dQ05ODnbu3PnQmt3c3NC/f3+89tpraN++ve4iAA8P\nD/j7+2PdunUoLCyEEAKpqak4e/ZsQ7WLqFpNLkxUKhU2b94MlUpl6VIsjr2o8Ps+/P7ktpubGyZM\nmIC4uDgAQN++fbF9+3b8+9//RkBAAEaOHImrV69i9+7d6NChAwDA1tYWO3bsQOfOnTFz5kz07dsX\nU6dORUFBAXr16lWlhlatWmHHjh04efIkAgICEBwcjDNnzgCoCKYuXbogMDAQs2bNwtixYw2e+/t6\nK4WFhSExMVF34r3S+vXrUV5ejtDQUAwYMAALFy7UjXb4d0KPvdBrkF4IM3nzzTdFYGCg6NKli7h+\n/Xq1+2g0GhETEyOCgoLE6NGjxZ49e+r8PqmpqeKJJ54QqampxpZs9diLCuyDHnuhx17oNUQvzDYy\nGTVqFP75z3/Cx8enxn0OHjyI1NRUHD9+HLt370ZsbCzS09PNVSIREdWT2cLEz88PXl5etd48deTI\nEUyZMgVAxaGHoKCgak+EEhFR49Kozpmkp6fD29tb97NSqURGRoYFKyIiIims+j4TlUpV5YRRZmYm\n/Pz8oFAoLFRV46FQKODj49Pse8E+6LEXeuyFnkKhgJ+fHzIzM6s85uzsDGdn54e+RqMKE29vb6Sn\np6NHjx4AKm70qu0cS3x8vG6upEp+fn7YvXu3Seu0FkqlEidPnrR0GRbHPuixF3rshZ5SqcTu3bsx\nffp0nD9/3uCx+fPn62aIqE2jCpOQkBDs2bMHo0aNQn5+Pk6cOIFdu3bVuH9kZCQmTpxosK3yt4z8\n/CJotZzczt3dEbm5hZYuw+LYBz32Qq8uvcjMK8Y/v7yG4hK1iasyndIyDUrLNfjL7EGwtdGf5ZDL\nZXB1bYW//e1vVWamljIqAcwYJn/5y19w/Phx5ObmYsaMGXB1dUVCQgKioqKwcOFCdO/eHeHh4bh4\n8SJGjx4NmUyGefPmoV27djW+Zm3DL61WMEx+xT5UYB/02As9Kb3IvVeC9R+fh1Yr0OsxdzNUZToe\nLvawkcuq/dzGTLsjE7VdXmXFcnML+Q8GgIeHE3Jy7lu6DItjH/TYC73f9qKopByffnUDpeVV14y5\nkXYPhQ/UWPasH9p7Opq7TJOTy2VwdzfuczWqw1xERJZyK12Fr/+bDhdHW9jaGJ6Ut2thg5ljuzbJ\nIGkoDBMiot94eeKTeMyntaXLsDqN6j4TIiKyThyZEJFZlJVrUFBUZukyDGjkcuQWPAAAFBQ2rtqs\nDcOEiEzu7r0HeGPXeeTfL7V0KQ/VQsEDNvXBMCEik7pfXIa/fXIRJWUaPB/SpVF9WTs7t4RKpV+Y\nzN7OBu29eJK9PhgmRE3IueRsfPr1DWg02hr3kSvk0NbyeEN7UKpBmVqLxVN7oUsHV7O9rxS8TLrh\nMEyImoifbudh68H/wbtNK3RoV/PVSC1btkBJSbnZ6pJBhkHdvRpdkFDDYpgQmcDpyxlITsk33xsK\n4IdrOVC6O2DpM33g0LJFjbvyt3EyBYYJkQkcSvwF+aoSODnU/KXe0Np5OGLuhB61BgmRqTBMiOpB\nKwRyCh7gTnYRSsqqTvz3oFSN3o+3wZzwHhaojsj8GCZED1FarsGdnEKkZhciNevX/+YUorSs6hxO\nv+XsYGumCoksj2FC9BuqojLczlQhNbsQKb8GR1ZeMSqnDLW3U6C9hyMCeijR3ssR7T0d0cq++sNK\nbZxbmq9wIgtjmBD96sdbufj7p5eg1lRER5vWLdHe0xEDunqig5cT2ns6ok3rlpDJZBaulKjxYZgQ\nAbiVocK7+35EW7dWeHbU42jv6QSHlvznQSQV/7VQs1dapsGmvRfhaN8Cr0zpBVcnO0uXRGR1Gs+8\nBkQWcq+oFKricowPeIRBQlRPHJlQs6PWaLHv65soLq24C/xBacVVWQo5z4UQ1RfDhJqdtJwiHE1K\nQauWNmhhUzE493BpCZ82nOCPqL4YJtRszQztij6Pe1i6DKImgedMiIjIaAwTIiIyGsOEiIiMxjAh\nIiKj8QQ8NVl37z3Au/t/RFm5BgqFXLf6YFm5+VYZJGouGCbUZKXfLcYvmffRtaMr3F3sUVqqnyre\nt6MLHvOpeTVCIqobhgk1eZOGdcagXu24uiCRCfGcCRERGY0jE7J65WotfryVq5s6vlJKFkciRObC\nMCGrd+F6Dt478L8aH2/FNdGJTI5hQlatuESN+8UVEzYuqmb6eHs7G7hxxUMik2OYkNX678938fdP\nL+l+9m7TisFBZCEME7JaBfdLAQATh3aGl6s9g4TIghgm1Gj8cDUHB769CfHwXQEAhQ8qDm8N6amE\niyMXtSKyJIYJNRrJv+QjI7cYvR9rI+0JroCLox2cW9matjAieiiGCZndz2n3kPRTVpXtV1ML0NJW\ngXmTnrRAVURkDLOGye3bt7Fs2TIUFBTAxcUFGzZsQIcOHQz2ycvLw2uvvYaMjAyo1WoMGjQIr7/+\nOuRy3l/ZVBw/m4pzydmwt6v61+/xdpzihMgamTVMoqOjERERgbCwMBw8eBArV65EfHy8wT7vvfce\nHn30UWzduhUajQbTp0/HsWPHEBISYs5S6TfuZBciu+BBg71e/v1StHV3wNrZgxrsNYnIsswWJnl5\nebhy5QpCQ0MBAGFhYVizZg3y8/Ph6uqq208mk6GoqAhCCJSUlECtVsPLy8tcZVI1/vqvC7p7ORoK\nJ1kkalrMFiYZGRnw8vKCTCYDAMjlcnh6eiIzM9MgTF5++WUsWLAAAQEBePDgASIiItCnT59qX1Ol\nUkGlUhlsUygUUCqVpvsgTZhao612evbScg0Gd2+L4AHtG+y92rS2b7DXIqKGkZGRAY1GY7DN2dkZ\nzs7OD31uozsBf/ToUfj6+mLnzp0oLCzErFmzcOzYMYwePbrKvvHx8YiNjTXY5uPjg5MnT8Ld3dFc\nJTd6Hh5OD91Ho9HihTXHkP/rvRu/5+PlhL49vBu6NLOS0ofmgr3QYy/0nn32WaSlpRlsmz9/PhYs\nWPDQ55otTJRKJbKysiCEgEwmg1arRXZ2Ntq2bWuw365du7Bu3ToAgKOjI0aOHIkzZ85UGyaRkZGY\nOHGiwTaFQgEAyM0thFYr9Y6FpsvDw0nS1Oul5Rrk3y9Fr0fd0bWjq8FjMpkM/Xw9rXoKd6l9aA7Y\nCz32ooJcLoO7uyM+/vjjakcmUpgtTNzc3ODr64uEhASMHz8eCQkJ6Natm8EhLgBo164dTp06hSef\nfBJlZWVITEysNkgA6cOvpuzuvQfYlvATyso11T5uY6OAWl39Y78lfs3dJ9q7YPSADrXvTERNkjGn\nCGRCCLP9+n7z5k0sW7YMKpUKrVu3xoYNG9CxY0dERUVh4cKF6N69O1JTUxEdHY27d+9Cq9Vi0KBB\nWL58eZ0vDW4uI5P/Xr+Lv392CU+0aw2HambHtbW1QVmZuppnViWXyzAhoBPaeTa9Q4T8DVSPvdBj\nLypUjkyMYdYwMaemFiaqojIcO5sKtcbwBHlOwQNcuH4Xq2b0wyNtq47S+I+lAvugx17osRcVGiJM\nGt0JeKrexZ/v4vD3v8C2hVx3RVwlVyc7uDpxkkMishyGiYU8KFXjWmqB5EkNU7IKAQDrZg/i7LhE\n1OgwTCzk8Pe/4IvEX+r0HLlMBjtbhYkqIiKqP4aJhZSWa2DXQoElz1R/Q2Z1nOxbcAlaImqUGCYW\nJJfL0EnZvC9tJqKmgVPxEhGR0TgyMZNzydk4ciZF93OuqsSC1RARNSyGiZlcvHEXqdmF8O3oAgBo\nZe+IzjzERURNBMPExH7JvI+k5CzczryP1q1aYNGU3pYuiYiowTFMTOzfSSn4/qcs2CjkeLKzm6XL\nISIyCYaJiWmFgJebA96I4qqCRNR08WouIiIymuSRyenTp/HFF18gLy8P7733Hi5fvozCwkIMHjzY\nlPUREZEVkDQy+eijjxATE4NHHnkEZ8+eBQC0bNkSmzZtMmlxRERkHSSFSXx8PHbs2IGoqCjduiKd\nO3fGrVu3TFocERFZB0mHuYqKinQrcFVOf65Wq9GiBeeJqknij5n4z4U0ZOYVo5U9+0RETZukkUn/\n/v2xbds2g207d+7EwIEDTVJUU/DDtRykZheig5cjhvaq/1KYRETWQNLI5PXXX8ecOXOwd+9eFBUV\nITg4GI6OjnjvvfdMXZ9V83BpiVenSZ8VmIjIWkkKE09PT3z22We4fPky0tLSoFQq0bNnzzqvy05E\nRE2TpDSYO3cuZDIZevbsiTFjxqB3796Qy+WYP3++qeuzGveLy3Dldp7uj6q4zNIlERGZjaSRyZkz\nZ6rdnpSU1KDFWLMdh5Px35/vGmx7zKe1haohIjKvWsOk8j6S8vLyKveUpKamwtvb23SVWZnScg3a\nebTCs6Oe0G1r697KghUREZlPrWGSmZkJABBC6P6/klKpxIIFC0xXmRWyt7NBlw6uli6DiMjsag2T\nN954AwDQp08fTJkyxSwFERGR9ZF0zqQySAoLC5Gfn2/wWPv27Ru+KiIisiqSwuTGjRtYvHgxkpOT\nIZPJIITQ3Ql/5coVkxZIRESNn6RLg2NiYjBw4EAkJSXB0dERZ8+exdSpU/Hmm2+auj4iIrICksIk\nOTkZr776KpydnSGEgJOTE5YsWcJZg4mICIDEMLGzs4NarQYAuLq6Ij09HVqtFgUFBSYtzhoUPijH\n1ZR8FJWUW7oUIiKLkXTOpG/fvjhy5AgmTZqE4OBgzJ49G7a2thg0iEvRbjv4P/x4Kw8A0INrvBNR\nMyUpTH57OGvRokV4/PHHUVRUhAkTJpisMGtRUqZBRy8nTBnxKHw8HC1dDhGRRTw0TDQaDWbMmIHt\n27fD1tYWcrkc4eHh5qitUdt+6Cd8979MCAH06OSGro9wVEJEzddDw0ShUODOnTvQarXmqMdq3Llb\nBA8Xewzo6oWej7pbuhwiIouSdAJ+3rx5iImJQVpaGjQaDbRare5Pc9bWzQGThnbmhI5E1OxJXhwL\nAA4cOKDbVnnjIm9aJCIiSWFy4sSJBnmz27dvY9myZSgoKICLiws2bNiADh06VNnv8OHD2LJlC4CK\nNec//PBDuLlZ9pzET7fzcDNdpfv5XmEpWreytWBFRESNh6Qw8fHxaZA3i46ORkREBMLCwnDw4EGs\nXLkS8fHxBvtcvnwZcXFx2LlzJ9zc3FBYWAhbW8t/ae86dg2ZecUG2/r7OlioGiKixkVSmDSEvLw8\nXLlyBaGhoQCAsLAwrFmzBvn5+XB11U/bHh8fj5kzZ+pGIo6ODXu5bU7BA9wrqvsqiKXlGgzo6olZ\nYd1022wUXLaYiAgwY5hkZGTAy8tLN0GkXC6Hp6cnMjMzDcLkxo0baNeuHSIiIlBcXIxRo0Zh7ty5\nDVJDWbkGK97/HmqNqNfzW9raMECIiKphtjCRSq1W49q1a/jwww9RWlqKWbNmwdvbu9p7W1QqFVQq\nlcE2hUIBpVJZ/WtrBNQagWG9vdH3CY8619bZ27nOzyEishYZGRnQaDQG25ydneHs/PDvvjqFSUZG\nBrKystC7d++6VYiKlRmzsrJ0V4FptVpkZ2ejbdu2Bvv5+PggODgYNjY2sLGxwciRI3H58uVqwyQ+\nPh6xsbFVnn/y5Em4u1c9PFb0oGL+rMc6uGHEwEfq/BmslYeHk6VLaBTYBz32Qo+90Hv22WeRlpZm\nsG3+/PmSVtWVFCbp6elYtGiRbj2TCxcu4OjRozh16hTWrl0rqUg3Nzf4+voiISEB48ePR0JCArp1\n62ZwiAuoOJfyzTffIDw8HOXl5UhMTERISEi1rxkZGYmJEycabFMoFACA3NxCaLWGh7OKSyomqywq\nLEFOzn1JdVs7Dw+nZvNZa8M+6LEXeuxFBblcBnd3R3z88cfVjkykkBQmq1atwvDhw/HPf/4TAwcO\nBAD4+/tj/fr1dSo4JiYGy5YtQ1xcHFq3bo0NGzYAAKKiorBw4UJ0794doaGh+PHHHzF27FgoFAoE\nBARg8uTJ1b6e1OGXVgjsPJqMnIKSOtVLRNSc1HSKQAqZEOKhZ6MHDhyIxMREyOVyDBgwAElJSQCA\nfv364dy5c/V+c1P67chEVVSGP23+Fu7OLeHqbIfpIx9HJ2XzOP/B37wqsA967IUee1GhcmRiDEkj\nE3d3d/zyyy/o1KmTbtvPP/9sVIpZwphBHRDo187SZRARNTmSrnOdOXMm5syZg88++wxqtRqHDh3C\nK6+8gtmzZ5u6PiIisgKSRiZPP/00XFxc8Mknn0CpVOLzzz/HwoULERQUZOr6iIjICkgKE41Gg6Cg\nIIYHERFVS9JhLn9/f8TExOCHH34wdT1ERGSFJIXJP/7xDzg4OGDx4sUIDAzE22+/jatXr5q6NiIi\nshKSDnN169YN3bp1w5IlS5CUlIRDhw5hxowZaNOmDRISEkxdIxERNXJ1nrWwU6dOePTRR6FUKqvc\ndk9ERM2TpJGJSqXCv//9bxw6dAgXL16Ev78/Zs2ahZEjR5q6PiIisgKSwmTIkCHo06cPwsLCEBsb\nCycnToxGRER6ksLk+PHj8PT0NHUtRERkpWoMk7Nnz6J///4AKhasunHjRrX7DR482DSVERGR1agx\nTP785z/j0KFDAIAVK1ZUu49MJsOJEydMUxkREVmNGsOkMkgA4OTJk2YphoiIrJOkS4NrWoN9/vz5\nDVpMQyv9dc3317YlAgBkFq6HiKipknQC/syZM9Vur1zXpLEqLC5HRm4xunZ0RUcvJ/R+vO7rvhMR\n0cPVGiabNm0CAJSXl+v+v1Jqaiq8vb1NV1kDGtTNC0N6WUetRETWqNYwyczMBAAIIXT/X0mpVEpa\nZJ6IiJq+WsPkjTfeAAD06dMHU6ZMMUtBRERkfWoMkzt37qBdu4olbgcPHozU1NRq92vfvr1pKiMi\nIqtRY5iMGzcOFy5cAACMGjUKMpkMQgiDfWQyGa5cuWLaComIqNGrMUwqgwQAkpOTzVIMERFZpzpP\nQQ9UXMnF6eeJiKiSpDBZtGgRzp8/DwD47LPPEBoaitDQUOzdu9ekxRnjrX9dQMyOX++D4d2KREQm\nJSlMEhMT0aNHDwDAhx9+iB07dmDv3r14//33TVqcMbLzH6CDlxNG9WuPnp3dLV0OEVGTJukO+PLy\nctja2iIrKwsFBQXo27cvAODu3bsmLc5YAT2VGNy9raXLICJq8iSFSdeuXbF161akpaVh+PDhAICs\nrCw4OjqasjYiIrISkg5zrV27FteuXUNpaSkWLlwIoOJqr3Hjxpm0OCIisg6SRiYdOnTA22+/bbAt\nJCQEISEhJimKiIisi6QwASqu4jpw4ACysrLg5eWF8PBwPPXUU6asjYiIrISkMNmyZQs+//xzzJw5\nE97e3khPT8cHH3yA7OzsGtc6ISKi5kNSmOzduxcfffQRfHx8dNsCAgIQERHBMCEiImkn4B88eAA3\nNzeDbS4uLigpKTFJUQ2F9yoSEZmHpDAZMmQIXn31Vdy8eRMlJSW4ceMGli1bhoCAAFPXV2/+PZTo\n+ojbw3ckIiKjSQqTVatWoVWrVggPD0fv3r0RHh4Oe3t7rFy50tT11Vv4kE5o3crW0mUQETULMvH7\neeVrodVqkZ+fD1dXV8jl9Zoj0mxycwuh1Ur+aE2Wh4cTcnLuW7oMi2Mf9NgLPfaiglwug7u7cTeh\nS06E27dvY+vWrYiNjcXWrVtx+/btOr/Z7du3MW3aNISEhGDatGlISUmpcd+bN2+id+/e2LBhQ53f\nh4iIzEtSmCQkJGDixIm4evUq7O3tce3aNUycOBEJCQl1erPo6GhERETg6NGjeOaZZ2o8TKbVahEd\nHY2goKA6vT4REVmGpEuD33nnHWzbtg39+/fXbTt37hyWLFkieUqVvLw8XLlyBaGhoQCAsLAwrFmz\nRnfY7Le2bduGwMBAFBUVobi4WOpnISIiC5E0MikqKkLv3r0NtvXq1atOX/QZGRnw8vKCTFZxwa5c\nLoenpycyMzMN9ktOTsbp06cxY8YMya9NRESWJWlk8sILL+Bvf/sb/vSnP8HOzg4lJSX4+9//jhde\neKFBi1Gr1Vi1ahXeeOMNXejURqVSQaVSGWxTKBRQKpUNWhcRUXOQkZEBjUZjsM3Z2RnOzs4Pfa6k\nq7mGDRuGu3fvQiaTwdnZGSqVCkIIeHh4GOz31Vdf1fgaeXl5CAkJwZkzZyCTyaDVajFw4EAcO3ZM\nd5grIyMDkyZNgoODA4QQuH+/4iqLMWPGYPXq1VVec/PmzYiNjTXY5uPjg5MnTz70gxMRkaHAwMAq\nS7LPnz8fCxYseOhzJYVJUlKSpEIGDBhQ6+PPP/88nn76aYwfPx4HDhzAvn37EB8fX+P+sbGxKC4u\nxpIlS6p9vLaRCS8NrsBLHyuwD3rshR57UaHy0mBjRiaSDnM9LCSkiomJwbJlyxAXF4fWrVvrLvuN\niorCwoUL0b179zq9ntQPSURED2fMKYI63bRoTTgyqcDfvCqwD3rshR57UcGsNy0SERHVhGFCRERG\nq1OYaLVaZGdnm6oWIiKyUpLCRKVSYfHixejZsydGjx4NADhx4gQ2btxo0uKIiMg6SAqT6OhoODo6\n4uTJk2jRogUAoE+fPjhy5IhJiyMiIusg6dLgxMREnDp1Ci1atNDdme7m5obc3FyTFkdERNZB0sjE\nyckJ+fn5BtvS09Or3AFPRETNk6QwmTx5Mv74xz/i+++/h1arxYULF7B06VJMmzbN1PUREZEVkHSY\na/bs2bC1tcXq1auhVquxfPlyTJ06FZGRkaauj4iIrADvgG/ieIdvBfZBj73QYy8qNMQd8JJPwNdk\n8ODBRhVARETWT1KYrFixwuDn/Px8lJeXw8vLCydOnDBJYUREZD0khcnv1wfRaDTYsmULWrVqZZKi\niIjIutRrbi6FQoE5c+bggw8+aOh6iIjICtV7osfTp09LWlqXiIiaPkmHuYYNG2YQHA8ePEBZWRmi\no6NNVhgREVkPSWHy17/+1eBne3t7dOrUCY6Oxl1KRkRETcNDw0Sj0WDz5s3Yvn07bG1tzVETERFZ\nmYeeM1EoFLhz5w60Wq056iEiIisk6QT8vHnzEBMTg7S0NGg0Gmi1Wt0fIiIiSedMXn/9dQDAgQMH\ndNuEEJDJZLhy5YppKiMiIqshKUx4lzsREdVG0mGuo0ePwsfHp8qfY8eOmbo+IiKyApLC5N133612\n+5YtWxq0GCIisk61HuaqnC1Yq9Xi+++/x29nq79z5w7n5iIiIgAPCZPK2YJLS0uxfPly3XaZTAYP\nDw/diXkiImreag2TytmClyxZgg0bNpilICIisj6SzpkwSIiIqDb1njWYiIioEsOEiIiMxjAhIiKj\nMUyIiMhoDBMiIjIaw4SIiIzGMCEiIqMxTIiIyGiSpqBvKLdv38ayZctQUFAAFxcXbNiwAR06dDDY\nJy4uDoebDyV/AAARKUlEQVQPH4aNjQ0UCgVeeeUVBAQEmLNMIiKqI7OGSXR0NCIiIhAWFoaDBw9i\n5cqViI+PN9inV69eePHFF2FnZ4fk5GQ899xzOH36NNefJyJqxMx2mCsvLw9XrlxBaGgoACAsLAw/\n/fQT8vPzDfbz9/eHnZ0dAMDX1xcAquxDRESNi9lGJhkZGfDy8oJMJgMAyOVyeHp6IjMzE66urtU+\nZ//+/Wjfvj28vLyqfVylUkGlUhlsUygUUCqVDVs8EVEzkJGRAY1GY7DN2dkZzs7OD32uWQ9z1UVS\nUhI2b96MHTt21LhPfHw8YmNjDbb5+Pjg5MmTcHd3NHWJVsPDw8nSJTQK7IMee6HHXug9++yzSEtL\nM9g2f/58LFiw4KHPNVuYKJVKZGVlQQgBmUwGrVaL7OxstG3btsq+Fy5cwNKlS7FlyxZ07NixxteM\njIzExIkTDbYpFAoAQG5uIbRaUd3TmhUPDyfk5Ny3dBkWxz7osRd67EUFuVwGd3dHfPzxx9WOTKQw\nW5i4ubnB19cXCQkJGD9+PBISEtCtW7cqh7guXbqERYsWYdOmTbpzJjWROvwiIqKHM+YUgUz8di1e\nE7t58yaWLVsGlUqF1q1bY8OGDejYsSOioqKwcOFCdO/eHU8//TTS09Ph5eWlG8Vs2LABjz/+eJ3e\niyOTCvzNqwL7oMde6LEXFSpHJsYwa5iYE8OkAv+xVGAf9NgLPfaiQkOECe+AJyIiozFMiIjIaAwT\nIiIyGsOEiIiMxjAhIiKjMUyIiMhoDBMiIjIaw4SIiIzGMCEiIqMxTIiIyGgMEyIiMhrDhIiIjMYw\nISIiozFMiIjIaAwTIiIyGsOEiIiMxjAhIiKjMUyIiMhoDBMiIjIaw4SIiIzGMCEiIqMxTIiIyGgM\nEyIiMhrDhIiIjMYwISIiozFMiIjIaAwTIiIyGsOEiIiMxjAhIiKjMUyIiMhoDBMiIjIaw4SIiIzG\nMCEiIqMxTIiIyGgMEyIiMppZw+T27duYNm0aQkJCMG3aNKSkpFTZR6vV4s9//jNGjRqF4OBg7N27\n15wlEhFRPZg1TKKjoxEREYGjR4/imWeewcqVK6vsc/DgQaSmpuL48ePYvXs3YmNjkZ6ebs4yiYio\njswWJnl5ebhy5QpCQ0MBAGFhYfjpp5+Qn59vsN+RI0cwZcoUAICbmxuCgoJw9OhRc5VJRET1YLYw\nycjIgJeXF2QyWcUby+Xw9PREZmamwX7p6enw9vbW/axUKpGRkWGuMomIqB5sLF2AMVQqFVQqlcE2\nhUIBpVIJuVxmoaoaH/aiAvugx17osRf6HmRkZECj0Rg85uzsDGdn54e+htnCRKlUIisrC0IIyGQy\naLVaZGdno23btgb7eXt7Iz09HT169ABQ8eF8fHyqfc34+HjExsYabPPz88Pu3bvh6trKNB/ECrm7\nO1q6hEaBfdBjL/TYC71Fixbh/PnzBtvmz5+PBQsWPPzJwoyee+45ceDAASGEEJ9//rl4/vnnq+yz\nb98+8eKLLwqtVityc3PFsGHDRGpqarWvd+/ePZGammrw5+zZs2LatGkiPT3dpJ/FGqSnp4sRI0Y0\n+16wD3rshR57oZeeni6mTZsmzp49W+U79d69e5Jew6yHuWJiYrBs2TLExcWhdevW2LBhAwAgKioK\nCxcuRPfu3REeHo6LFy9i9OjRkMlkmDdvHtq1a1ft69U0/Dp//nyVoVpzpNFokJaW1ux7wT7osRd6\n7IWeRqPB+fPn0bZt2xq/bx/GrGHSuXNn7Nmzp8r2bdu26f5fLpcjJibGjFUREZGxeAc8EREZjWFC\nRERGU8Q0wWNKdnZ2GDhwIOzs7CxdisWxFxXYBz32Qo+90DO2FzIhhGjgmoiIqJnhYS4iIjIaw4SI\niIxmtWHC6ez1pPQiLi4OYWFhmDBhAp566il8++23FqjU9KT0otLNmzfRu3dv3f1OTY3UXhw+fBjj\nxo3DuHHjMH78eOTl5Zm5UtOS0oe8vDy89NJLGD9+PMaOHYvVq1dDq9VaoFrTWr9+PUaOHAlfX1/8\n/PPP1e5T7+9Nk95WaULPP/+8SEhIEEIIceDAgWrvpt+/f7948cUXhRBC5ObmiqFDh4q0tDSz1mkO\nUnrx7bffipKSEiGEEFeuXBH9+vUTpaWlZq3THKT0QgghNBqNiIiIEIsXLxbr1683Z4lmI6UXly5d\nEqGhoSI3N1cIIcT9+/eb3N8LKX1Yu3at7u+BWq0WkydPFkeOHDFrnebwww8/iMzMTBEYGCiuX79e\n7T71/d60ypEJp7PXk9oLf39/3VUavr6+AFBlH2sntRdAxY2ygYGBeOSRR8xcpXlI7UV8fDxmzpwJ\nNzc3AICjoyNsbW3NXq+pSO2DTCZDUVERhBAoKSmBWq2Gl5eXJUo2KT8/P3h5eUHUct1Vfb83rTJM\nOJ29ntRe/Nb+/fvRvn37JvePRWovkpOTcfr0acyYMcMCVZqH1F7cuHEDKSkpiIiIwKRJk7BlyxZL\nlGsyUvvw8ssv49atWwgICMCQIUMQEBCAPn36WKJki6vv96ZVhgnVX1JSEjZv3oyNGzdauhSLUKvV\nWLVqFWJiYnRfMM2ZWq3GtWvX8OGHH+Kjjz7CN998gwMHDli6LLM7evQofH19cfr0aXzzzTdISkrC\nsWPHLF2WVbHKMPntdPYAHjqdfaWMjAwolUqz1mpqUnsBABcuXMDSpUsRFxeHjh07mrtUk5PSi5yc\nHKSmpiIqKgqBgYGIj4/H3r17sWrVKkuVbRJS/174+PggODgYNjY2aNWqFUaOHInLly9bomSTkNqH\nXbt2Ydy4cQAqDvWNHDkSZ86cMXu9jUF9vzetMkzc3Nzg6+uLhIQEAEBCQgK6desGV1dXg/1CQkKw\nZ88eCCGQl5eHEydOYPTo0ZYo2WSk9uLSpUtYtGgRNm3apDtn0tRI6YVSqURiYiJOnDiBkydPIjIy\nEpMnT8bq1astVbZJSP17ERYWhtOnTwMAysvLkZiYiC5dupi9XlOR2od27drh1KlTAICysjIkJibi\n8ccfN3u9jUG9vzcb4goBS7hx44aYPHmyCA4OFlOmTBG3b98WQggxe/Zs8eOPPwohKq7YiY6OFkFB\nQWLUqFFiz549lizZZKT04qmnnhKDBw8WEyZMEOHh4WLChAni2rVrlizbJKT04rc2b97cZK/mktIL\nrVYr3njjDTFmzBgRFhYm3nzzTUuWbBJS+pCSkiJeeOEFMW7cOBEaGirWrFkjNBqNJcs2iTVr1oih\nQ4eK7t27C39/fxEWFiaEaJjvTU6nQkRERrPKw1xERNS4MEyIiMhoDBMiIjIaw4SIiIzGMCEiIqMx\nTIiIyGgME2q0bt26hYkTJ6Jv377YtWtXrfumpaXB19fXKqYNDwsLw9mzZ2t8fPbs2fj888/NWBGR\n8XifCTVaK1asgJOTE5YtW/bQfdPS0hAUFIT//e9/kMut53ek2NhYpKSkNMo1VRpzbdT4WM+/Omp2\n0tPT8dhjj1m6jCZJo9FYugRqYhgm1ChFRkbizJkzWL16Nfz8/PDLL7/g66+/1h32GjFiBGJjY2t8\n/r59+xAUFAQ/Pz8EBQXh0KFDusc+/fRTjB07FgMHDsSsWbMMJrX7rcpDZ3v27MGQIUMwZMgQ7Nix\nQ/d4WVkZ1q5diyFDhmDo0KFYt24dysvLAVSsFTNnzhz0798fAwcOREREhO55gYGBSExMxKlTp/De\ne+/h8OHD6NOnDyZMmAAAeO655/Dpp5+irKwM/fv3N1gRLy8vD7169dKthvif//wHEyZMQP/+/TF9\n+nRcvXq1xp74+vri448/RnBwMIKDgwEAa9euxfDhw9G3b1889dRTOHfuHADUWFthYSFWrFiBgIAA\nDBs2DO+8806ta2NQM2KK+V+IGkJERITYu3ev7uekpCTdfGJXr14V/v7+4ssvvxRCCHHnzh3h6+sr\nNBqNKC4uFn5+fro5mHJycsTPP/8shBDi+PHjYvTo0eLmzZtCo9GILVu2iKlTp1b7/nfu3BFdunQR\nixYtEiUlJeLq1ati0KBB4rvvvhNCCPHOO++IqVOniry8PJGXlyemTp0qNm3aJIQQ4u233xbR0dFC\no9EItVotzp07p3vdESNG6F5j8+bN4v/+7/9q/NzLly8XGzdu1D22a9cuMWvWLCGEED/++KMYPHiw\nuHTpktBqtWL//v1ixIgRoqysrNrP06VLFzFz5kyhUql0qykePHhQ3Lt3T2g0GrFjxw7h7++ve6y6\n2ubOnSuio6NFSUmJyM3NFZMnTxaffPJJte9HzQtHJmQ1+vfvr5vJ9YknnsDYsWNrPJGtUChw7do1\nlJaWok2bNnj00UcBAJ988gmioqLQqVMnyOVyREVFITk5udbFfxYsWAA7Ozs88cQTmDRpEr744gsA\nwKFDhzBv3jy4urrC1dUV8+fP160FYmNjg5ycHNy5cwcKhQJ9+/at12cOCwszGFUdOnRIN1X63r17\nMW3aNDz55JOQyWSYMGECbG1tcfHixRpf76WXXoKTk5NuNcVx48bB2dkZcrkcM2bMQFlZGW7dulXt\nc3Nzc3Hq1CksX74cdnZ2cHNzQ2RkpEF91HzZWLoAIqkuXbqEt956C9evX0d5eTnKy8sREhJSZT97\ne3ts3LgR27dvx/Lly9G3b18sXboUnTp1Qnp6OtauXYv169cDAIQQkMlkyMrKqnbNBplMZrD2hbe3\nN65fvw4AyM7ONliRztvbG9nZ2QCAF198EbGxsZg5cyZkMhkmT56MqKioOn/mQYMGobS0FJcuXUKb\nNm2QnJyMoKAgABXnlA4cOKC70k0IAbVarauhOr9fx+Mf//gHPv30U+Tk5AAAioqKalzOOS0tDWq1\nGgEBAbr3E0I0uTWCqH4YJmQ1Fi9ejOeeew7bt29HixYtsG7dOhQUFFS7r7+/P/z9/VFWVoaNGzdi\n5cqV2LVrF9q2bYu5c+ciLCxM0nsKIZCRkYFOnToBqFgoyNPTEwDg6emJtLQ03agnPT1d91irVq2w\ndOlSLF26FDdu3MBzzz2Hnj17YtCgQXX6zDKZDGPGjMGhQ4fQpk0bjBgxAg4ODgAqgmHOnDl46aWX\n6vR6lc6dO4cPPvgAO3fu1F3oMGDAgBrPgSiVStjZ2eHMmTNcpZKq4GEushrFxcVwdnZGixYtcOnS\npSqHVyq/BHNzc3Hy5Ek8ePAANjY2cHBw0F0uPH36dGzdulV3Uvv+/fs4evRore8bFxeHkpISXL9+\nHfv27UNoaCgAIDQ0FFu2bEFeXh7y8vIQFxeH8PBwAMBXX32FlJQUAICDgwMUCgVsbKr+7tamTRuk\npaXVehI7LCwMhw8fRkJCgkEITpkyBf/6179w6dIlXX++/vprFBcX1/p5KhUVFcHGxgYuLi4oKytD\nbGwsioqKaqzNw8MD/v7+WLduHQoLCyGEQGpqaq33zFDzwZEJNVq//+03Ojoab775JtasWYP+/ftj\n7NixUKlUVfbXarXYsWMHli5dCplMBl9fX0RHRwMAgoKCUFxcjFdeeQUZGRlwcnLCH/7wh2oPl1Ua\nMGAARo0aBSEEZs2ahcGDBwMAXn75ZRQVFWH8+PG6EcScOXMAALdv38bq1auRn5+P1q1b49lnn0W/\nfv2qfK6QkBAcPHgQAwcORLt27bBv374qn7tnz55wcHBATk4Ohg4dqtveo0cPrFmzBqtXr0ZKSgrs\n7OzQt29f9O/fX1I/K69QCw4OhoODA2bMmGFwGKy62tavX4+33noLoaGhKC4uRvv27TFr1qwae0fN\nB29aJKqBtd4ISWQJ/BdCVAv+rkUkDcOEqBY80UwkDQ9zERGR0TgyISIiozFMiIjIaAwTIiIyGsOE\niIiMxjAhIiKjMUyIiMho/w/TWJJiO+X3vQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": { - "tags": [] - } - } - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/estimator/premade.ipynb b/site/zh-cn/tutorials/estimator/premade.ipynb deleted file mode 100644 index 1963c50cf13..00000000000 --- a/site/zh-cn/tutorials/estimator/premade.ipynb +++ /dev/null @@ -1,614 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "premade.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1Z6Wtb_jisbA" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "QUyRGn9riopB", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "H1yCdGFW4j_F" - }, - "source": [ - "# 预创建的 Estimators" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PS6_yKSoyLAl" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorFlow.google.cn 上查看\n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FgdA9XE5ZCS3", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R4YZ_ievcY7p" - }, - "source": [ - "\n", - "本教程将向您展示如何使用 Estimators 解决 Tensorflow 中的鸢尾花（Iris）分类问题。Estimator 是 Tensorflow 完整模型的高级表示，它被设计用于轻松扩展和异步训练。更多细节请参阅 [Estimators](https://tensorflow.google.cn/guide/estimator)。\n", - "\n", - "请注意，在 Tensorflow 2.0 中，[Keras API](https://tensorflow.google.cn/guide/keras) 可以完成许多相同的任务，而且被认为是一个更易学习的API。如果您刚刚开始入门，我们建议您从 Keras 开始。有关 Tensorflow 2.0 中可用高级API的更多信息，请参阅 [Keras标准化](https://medium.com/tensorflow/standardizing-on-keras-guidance-on-high-level-apis-in-tensorflow-2-0-bad2b04c819a)。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8IFct0yedsTy" - }, - "source": [ - "## 首先要做的事\n", - "\n", - "为了开始，您将首先导入 Tensorflow 和一系列您需要的库。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jPo5bQwndr9P", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf\n", - "\n", - "import pandas as pd" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c5w4m5gncnGh" - }, - "source": [ - "## 数据集\n", - "\n", - "本文档中的示例程序构建并测试了一个模型，该模型根据[花萼](https://en.wikipedia.org/wiki/Sepal)和[花瓣](https://en.wikipedia.org/wiki/Petal)的大小将鸢尾花分成三种物种。\n", - "\n", - "您将使用鸢尾花数据集训练模型。该数据集包括四个特征和一个[标签](https://developers.google.com/machine-learning/glossary/#label)。这四个特征确定了单个鸢尾花的以下植物学特征：\n", - "\n", - "* 花萼长度\n", - "* 花萼宽度\n", - "* 花瓣长度\n", - "* 花瓣宽度\n", - "\n", - "根据这些信息，您可以定义一些有用的常量来解析数据：\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lSyrXp_He_UE", - "colab": {} - }, - "source": [ - "CSV_COLUMN_NAMES = ['SepalLength', 'SepalWidth', 'PetalLength', 'PetalWidth', 'Species']\n", - "SPECIES = ['Setosa', 'Versicolor', 'Virginica']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j6mTfIQzfC9w" - }, - "source": [ - "接下来，使用 Keras 与 Pandas 下载并解析鸢尾花数据集。注意为训练和测试保留不同的数据集。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "PumyCN8VdGGc", - "colab": {} - }, - "source": [ - "train_path = tf.keras.utils.get_file(\n", - " \"iris_training.csv\", \"https://storage.googleapis.com/download.tensorflow.org/data/iris_training.csv\")\n", - "test_path = tf.keras.utils.get_file(\n", - " \"iris_test.csv\", \"https://storage.googleapis.com/download.tensorflow.org/data/iris_test.csv\")\n", - "\n", - "train = pd.read_csv(train_path, names=CSV_COLUMN_NAMES, header=0)\n", - "test = pd.read_csv(test_path, names=CSV_COLUMN_NAMES, header=0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wHFxNLszhQjz" - }, - "source": [ - "通过检查数据您可以发现有四列浮点型特征和一列 int32 型标签。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WOJt-ML4hAwI", - "colab": {} - }, - "source": [ - "train.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jQJEYfVvfznP" - }, - "source": [ - "对于每个数据集都分割出标签，模型将被训练来预测这些标签。" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "sSaJNGeaZCTG", - "colab_type": "code", - "colab": {} - }, - "source": [ - "train_y = train.pop('Species')\n", - "test_y = test.pop('Species')\n", - "\n", - "# 标签列现已从数据中删除\n", - "train.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jZx1L_1Vcmxv" - }, - "source": [ - "## Estimator 编程概述\n", - "\n", - "现在您已经设定好了数据，您可以使用 Tensorflow Estimator 定义模型。Estimator 是从 `tf.estimator.Estimator` 中派生的任何类。Tensorflow提供了一组`tf.estimator`(例如，`LinearRegressor`)来实现常见的机器学习算法。此外，您可以编写您自己的[自定义 Estimator](https://tensorflow.google.cn/guide/custom_estimators)。入门阶段我们建议使用预创建的 Estimator。\n", - "\n", - "为了编写基于预创建的 Estimator 的 Tensorflow 项目，您必须完成以下工作：\n", - "\n", - "* 创建一个或多个输入函数\n", - "* 定义模型的特征列\n", - "* 实例化一个 Estimator，指定特征列和各种超参数。\n", - "* 在 Estimator 对象上调用一个或多个方法，传递合适的输入函数以作为数据源。\n", - "\n", - "我们来看看这些任务是如何在鸢尾花分类中实现的。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2OcguDfBcmmg" - }, - "source": [ - "## 创建输入函数\n", - "\n", - "您必须创建输入函数来提供用于训练、评估和预测的数据。\n", - "\n", - "**输入函数**是一个返回 `tf.data.Dataset` 对象的函数，此对象会输出下列含两个元素的元组：\n", - "\n", - "* [`features`](https://developers.google.com/machine-learning/glossary/#feature)——Python字典，其中：\n", - " * 每个键都是特征名称\n", - " * 每个值都是包含此特征所有值的数组\n", - "* `label` 包含每个样本的[标签](https://developers.google.com/machine-learning/glossary/#label)的值的数组。\n", - "\n", - "为了向您展示输入函数的格式，请查看下面这个简单的实现：\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nzr5vRr5caGF", - "colab": {} - }, - "source": [ - "def input_evaluation_set():\n", - " features = {'SepalLength': np.array([6.4, 5.0]),\n", - " 'SepalWidth': np.array([2.8, 2.3]),\n", - " 'PetalLength': np.array([5.6, 3.3]),\n", - " 'PetalWidth': np.array([2.2, 1.0])}\n", - " labels = np.array([2, 1])\n", - " return features, labels" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NpXvGjfnjHgY" - }, - "source": [ - "\n", - "您的输入函数可以以您喜欢的方式生成 `features` 字典与 `label` 列表。但是，我们建议使用 Tensorflow 的 [Dataset API](https://tensorflow.google.cn/guide/datasets)，该 API 可以用来解析各种类型的数据。\n", - "\n", - "Dataset API 可以为您处理很多常见情况。例如，使用 Dataset API，您可以轻松地从大量文件中并行读取记录，并将它们合并为单个数据流。\n", - "\n", - "为了简化此示例，我们将使用 [pandas](https://pandas.pydata.org/) 加载数据，并利用此内存数据构建输入管道。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "T20u1anCi8NP", - "colab": {} - }, - "source": [ - "def input_fn(features, labels, training=True, batch_size=256):\n", - " \"\"\"An input function for training or evaluating\"\"\"\n", - " # 将输入转换为数据集。\n", - " dataset = tf.data.Dataset.from_tensor_slices((dict(features), labels))\n", - "\n", - " # 如果在训练模式下混淆并重复数据。\n", - " if training:\n", - " dataset = dataset.shuffle(1000).repeat()\n", - " \n", - " return dataset.batch(batch_size)\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xIwcFT4MlZEi" - }, - "source": [ - "## 定义特征列（feature columns）\n", - "\n", - "[**特征列（feature columns）**](https://developers.google.com/machine-learning/glossary/#feature_columns)是一个对象，用于描述模型应该如何使用特征字典中的原始输入数据。当您构建一个 Estimator 模型的时候，您会向其传递一个特征列的列表，其中包含您希望模型使用的每个特征。`tf.feature_column` 模块提供了许多为模型表示数据的选项。\n", - "\n", - "对于鸢尾花问题，4 个原始特征是数值，因此我们将构建一个特征列的列表，以告知 Estimator 模型将 4 个特征都表示为 32 位浮点值。故创建特征列的代码如下所示：\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZTTriO8FlSML", - "colab": {} - }, - "source": [ - "# 特征列描述了如何使用输入。\n", - "my_feature_columns = []\n", - "for key in train.keys():\n", - " my_feature_columns.append(tf.feature_column.numeric_column(key=key))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jpKkhMoZljco" - }, - "source": [ - "特征列可能比上述示例复杂得多。您可以从[指南](https://tensorflow.google.cn/guide/feature_columns)获取更多关于特征列的信息。\n", - "\n", - "我们已经介绍了如何使模型表示原始特征，现在您可以构建 Estimator 了。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kuE59XHEl22K" - }, - "source": [ - "## 实例化 Estimator\n", - "\n", - "鸢尾花为题是一个经典的分类问题。幸运的是，Tensorflow 提供了几个预创建的 Estimator 分类器，其中包括：\n", - "\n", - "* `tf.estimator.DNNClassifier` 用于多类别分类的深度模型\n", - "* `tf.estimator.DNNLinearCombinedClassifier` 用于广度与深度模型\n", - "* `tf.estimator.LinearClassifier` 用于基于线性模型的分类器\n", - "\n", - "对于鸢尾花问题，`tf.estimator.LinearClassifier` 似乎是最好的选择。您可以这样实例化该 Estimator：\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qnf4o2V5lcPn", - "colab": {} - }, - "source": [ - "# 构建一个拥有两个隐层，隐藏节点分别为 30 和 10 的深度神经网络。\n", - "classifier = tf.estimator.DNNClassifier(\n", - " feature_columns=my_feature_columns,\n", - " # 隐层所含结点数量分别为 30 和 10.\n", - " hidden_units=[30, 10],\n", - " # 模型必须从三个类别中做出选择。\n", - " n_classes=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tzzt5nUpmEe3" - }, - "source": [ - " ## 训练、评估和预测\n", - "\n", - "我们已经有一个 Estimator 对象，现在可以调用方法来执行下列操作：\n", - "\n", - "* 训练模型。\n", - "* 评估经过训练的模型。\n", - "* 使用经过训练的模型进行预测。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rnihuLdWmE75" - }, - "source": [ - "### 训练模型\n", - "\n", - "通过调用 Estimator 的 `Train` 方法来训练模型，如下所示：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4jW08YtPl1iS", - "colab": {} - }, - "source": [ - "# 训练模型。\n", - "classifier.train(\n", - " input_fn=lambda: input_fn(train, train_y, training=True),\n", - " steps=5000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ybiTFDmlmes8" - }, - "source": [ - "注意将 ` input_fn` 调用封装在 [`lambda`](https://docs.python.org/3/tutorial/controlflow.html) 中以获取参数，同时提供不带参数的输入函数，如 Estimator 所预期的那样。`step` 参数告知该方法在训练多少步后停止训练。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HNvJLH8hmsdf" - }, - "source": [ - "### 评估经过训练的模型\n", - "\n", - "现在模型已经经过训练，您可以获取一些关于模型性能的统计信息。代码块将在测试数据上对经过训练的模型的准确率（accuracy）进行评估：\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "A169XuO4mKxF", - "colab": {} - }, - "source": [ - "eval_result = classifier.evaluate(\n", - " input_fn=lambda: input_fn(test, test_y, training=False))\n", - "\n", - "print('\\nTest set accuracy: {accuracy:0.3f}\\n'.format(**eval_result))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VnPMP5EHph17" - }, - "source": [ - "与对 `train` 方法的调用不同，我们没有传递 `steps` 参数来进行评估。用于评估的 `input_fn` 只生成一个 [epoch](https://developers.google.com/machine-learning/glossary/#epoch) 的数据。\n", - "\n", - "`eval_result` 字典亦包含 `average_loss`（每个样本的平均误差），`loss`（每个 mini-batch 的平均误差）与 Estimator 的 `global_step`（经历的训练迭代次数）值。 " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ur624ibpp52X" - }, - "source": [ - "### 利用经过训练的模型进行预测（推理）\n", - "\n", - "我们已经有一个经过训练的模型，可以生成准确的评估结果。我们现在可以使用经过训练的模型，根据一些无标签测量结果预测鸢尾花的品种。与训练和评估一样，我们使用单个函数调用进行预测：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wltc0jpgng38", - "colab": {} - }, - "source": [ - "# 由模型生成预测\n", - "expected = ['Setosa', 'Versicolor', 'Virginica']\n", - "predict_x = {\n", - " 'SepalLength': [5.1, 5.9, 6.9],\n", - " 'SepalWidth': [3.3, 3.0, 3.1],\n", - " 'PetalLength': [1.7, 4.2, 5.4],\n", - " 'PetalWidth': [0.5, 1.5, 2.1],\n", - "}\n", - "\n", - "def input_fn(features, batch_size=256):\n", - " \"\"\"An input function for prediction.\"\"\"\n", - " # 将输入转换为无标签数据集。\n", - " return tf.data.Dataset.from_tensor_slices(dict(features)).batch(batch_size)\n", - "\n", - "predictions = classifier.predict(\n", - " input_fn=lambda: input_fn(predict_x))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JsETKQo0rHvi" - }, - "source": [ - "`predict` 方法返回一个 Python 可迭代对象，为每个样本生成一个预测结果字典。以下代码输出了一些预测及其概率：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Efm4mLzkrCxp", - "colab": {} - }, - "source": [ - "for pred_dict, expec in zip(predictions, expected):\n", - " class_id = pred_dict['class_ids'][0]\n", - " probability = pred_dict['probabilities'][class_id]\n", - "\n", - " print('Prediction is \"{}\" ({:.1f}%), expected \"{}\"'.format(\n", - " SPECIES[class_id], 100 * probability, expec))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/generative/cvae.ipynb b/site/zh-cn/tutorials/generative/cvae.ipynb deleted file mode 100644 index 94d2ca20929..00000000000 --- a/site/zh-cn/tutorials/generative/cvae.ipynb +++ /dev/null @@ -1,644 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ndo4ERqnwQOU" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "MTKwbguKwT4R" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xfNT-mlFwxVM" - }, - "source": [ - "# 卷积变分自编码器" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TD5ZrvEMbhZ" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://tensorflow.google.cn/tutorials/generative/cvae\"\u003e\n", - " \u003cimg src=\"https://tensorflow.google.cn/images/tf_logo_32px.png\" /\u003e\n", - " 在 tensorFlow.google.cn 上查看\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/zh-cn/tutorials/generative/cvae.ipynb\"\u003e\n", - " \u003cimg src=\"https://tensorflow.google.cn/images/colab_logo_32px.png\" /\u003e\n", - " 在 Google Colab 中运行\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/zh-cn/tutorials/generative/cvae.ipynb\"\u003e\n", - " \u003cimg src=\"https://tensorflow.google.cn/images/GitHub-Mark-32px.png\" /\u003e\n", - " 在 GitHub 上查看源代码\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/zh-cn/tutorials/generative/cvae.ipynb\"\u003e\u003cimg src=\"https://tensorflow.google.cn/images/download_logo_32px.png\" /\u003e下载 notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fKsm6LhC7TAw" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ITZuApL56Mny" - }, - "source": [ - "![训练过程中输出的演变](https://tensorflow.google.cn/images/autoencoders/cvae.gif)\n", - "\n", - "本笔记演示了如何通过训练变分自编码器（[1](https://arxiv.org/abs/1312.6114), [2](https://arxiv.org/abs/1401.4082)）来生成手写数字图片。\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "P-JuIu2N_SQf" - }, - "outputs": [], - "source": [ - "# 用于生成 gif\n", - "!pip install imageio" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e1_Y75QXJS6h" - }, - "source": [ - "## 导入 Tensorflow 与其他库" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "YfIk2es3hJEd" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version 仅应用于 Colab。\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import os\n", - "import time\n", - "import numpy as np\n", - "import glob\n", - "import matplotlib.pyplot as plt\n", - "import PIL\n", - "import imageio\n", - "\n", - "from IPython import display" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iYn4MdZnKCey" - }, - "source": [ - "## 加载 MNIST 数据集\n", - "\n", - "每个 MNIST 图片最初都是包含 784 个整数的向量，每个整数取值都在 0-255 之间，表示像素的强度。我们在模型中使用伯努利分布对每个像素进行建模，并对数据集进行静态二值化。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "a4fYMGxGhrna" - }, - "outputs": [], - "source": [ - "(train_images, _), (test_images, _) = tf.keras.datasets.mnist.load_data()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NFC2ghIdiZYE" - }, - "outputs": [], - "source": [ - "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n", - "test_images = test_images.reshape(test_images.shape[0], 28, 28, 1).astype('float32')\n", - "\n", - "# 标准化图片到区间 [0., 1.] 内\n", - "train_images /= 255.\n", - "test_images /= 255.\n", - "\n", - "# 二值化\n", - "train_images[train_images \u003e= .5] = 1.\n", - "train_images[train_images \u003c .5] = 0.\n", - "test_images[test_images \u003e= .5] = 1.\n", - "test_images[test_images \u003c .5] = 0." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "S4PIDhoDLbsZ" - }, - "outputs": [], - "source": [ - "TRAIN_BUF = 60000\n", - "BATCH_SIZE = 100\n", - "\n", - "TEST_BUF = 10000" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PIGN6ouoQxt3" - }, - "source": [ - "## 使用 *tf.data* 来将数据分批和打乱" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "-yKCCQOoJ7cn" - }, - "outputs": [], - "source": [ - "train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(TRAIN_BUF).batch(BATCH_SIZE)\n", - "test_dataset = tf.data.Dataset.from_tensor_slices(test_images).shuffle(TEST_BUF).batch(BATCH_SIZE)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "THY-sZMiQ4UV" - }, - "source": [ - "## 通过 *tf.keras.Sequential* 连接生成网络与推理网络\n", - "\n", - "\n", - "在我们的 VAE 示例中，我们将两个小型的 ConvNet 用于生成和推断网络。由于这些神经网络较小，我们使用 `tf.keras.Sequential` 来简化代码。在下面的描述中，令 $x$ 和 $z$ 分别表示观测值和潜在变量\n", - "\n", - "### 生成网络\n", - "\n", - "\n", - "这里定义了生成模型，该模型将潜在编码作为输入，并输出用于观测条件分布的参数，即 $p(x|z)$。另外，我们对潜在变量使用单位高斯先验 $p(z)$。\n", - "\n", - "### 推理网络\n", - "\n", - "\n", - "这里定义了近似后验分布 $q(z|x)$，该后验分布以观测值作为输入，并输出用于潜在表示的条件分布的一组参数。在本示例中，我们仅将此分布建模为对角高斯模型。在这种情况下，推断网络将输出因式分解的高斯均值和对数方差参数（为了数值稳定性使用对数方差而不是直接使用方差）。\n", - "\n", - "### 重参数化技巧\n", - "\n", - "在优化过程中，我们可以从 $q(z|x)$ 中采样，方法是首先从单位高斯采样，然后乘以标准偏差并加平均值。这样可以确保梯度能够通过样本传递到推理网络参数。\n", - "\n", - "### 网络架构\n", - "\n", - "对于推理网络，我们使用两个卷积层，后接一个全连接层。在生成网络中，我们通过使用全连接层，后接三个卷积转置层（在某些情况下也称为反卷积层）来镜像词体系结构。请注意，在训练 VAE 时避免使用批归一化（batch normalization）是一种常见的做法，因为使用小批量处理会导致额外的随机性，从而加剧随机抽样的不稳定性。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VGLbvBEmjK0a" - }, - "outputs": [], - "source": [ - "class CVAE(tf.keras.Model):\n", - " def __init__(self, latent_dim):\n", - " super(CVAE, self).__init__()\n", - " self.latent_dim = latent_dim\n", - " self.inference_net = tf.keras.Sequential(\n", - " [\n", - " tf.keras.layers.InputLayer(input_shape=(28, 28, 1)),\n", - " tf.keras.layers.Conv2D(\n", - " filters=32, kernel_size=3, strides=(2, 2), activation='relu'),\n", - " tf.keras.layers.Conv2D(\n", - " filters=64, kernel_size=3, strides=(2, 2), activation='relu'),\n", - " tf.keras.layers.Flatten(),\n", - " # No activation\n", - " tf.keras.layers.Dense(latent_dim + latent_dim),\n", - " ]\n", - " )\n", - "\n", - " self.generative_net = tf.keras.Sequential(\n", - " [\n", - " tf.keras.layers.InputLayer(input_shape=(latent_dim,)),\n", - " tf.keras.layers.Dense(units=7*7*32, activation=tf.nn.relu),\n", - " tf.keras.layers.Reshape(target_shape=(7, 7, 32)),\n", - " tf.keras.layers.Conv2DTranspose(\n", - " filters=64,\n", - " kernel_size=3,\n", - " strides=(2, 2),\n", - " padding=\"SAME\",\n", - " activation='relu'),\n", - " tf.keras.layers.Conv2DTranspose(\n", - " filters=32,\n", - " kernel_size=3,\n", - " strides=(2, 2),\n", - " padding=\"SAME\",\n", - " activation='relu'),\n", - " # No activation\n", - " tf.keras.layers.Conv2DTranspose(\n", - " filters=1, kernel_size=3, strides=(1, 1), padding=\"SAME\"),\n", - " ]\n", - " )\n", - "\n", - " @tf.function\n", - " def sample(self, eps=None):\n", - " if eps is None:\n", - " eps = tf.random.normal(shape=(100, self.latent_dim))\n", - " return self.decode(eps, apply_sigmoid=True)\n", - "\n", - " def encode(self, x):\n", - " mean, logvar = tf.split(self.inference_net(x), num_or_size_splits=2, axis=1)\n", - " return mean, logvar\n", - "\n", - " def reparameterize(self, mean, logvar):\n", - " eps = tf.random.normal(shape=mean.shape)\n", - " return eps * tf.exp(logvar * .5) + mean\n", - "\n", - " def decode(self, z, apply_sigmoid=False):\n", - " logits = self.generative_net(z)\n", - " if apply_sigmoid:\n", - " probs = tf.sigmoid(logits)\n", - " return probs\n", - "\n", - " return logits" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0FMYgY_mPfTi" - }, - "source": [ - "## 定义损失函数和优化器\n", - "\n", - "VAE 通过最大化边际对数似然的证据下界（ELBO）进行训练：\n", - "\n", - "$$\\log p(x) \\ge \\text{ELBO} = \\mathbb{E}_{q(z|x)}\\left[\\log \\frac{p(x, z)}{q(z|x)}\\right].$$\n", - "\n", - "实际上，我们优化了此期望的单样本蒙卡特罗估计：\n", - "\n", - "$$\\log p(x| z) + \\log p(z) - \\log q(z|x),$$\n", - "其中 $z$ 从 $q(z|x)$ 中采样。\n", - "\n", - "**注意**：我们也可以分析性地计算 KL 项，但简单起见，这里我们将所有三个项合并到蒙卡特罗估计器中。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "iWCn_PVdEJZ7" - }, - "outputs": [], - "source": [ - "optimizer = tf.keras.optimizers.Adam(1e-4)\n", - "\n", - "def log_normal_pdf(sample, mean, logvar, raxis=1):\n", - " log2pi = tf.math.log(2. * np.pi)\n", - " return tf.reduce_sum(\n", - " -.5 * ((sample - mean) ** 2. * tf.exp(-logvar) + logvar + log2pi),\n", - " axis=raxis)\n", - "\n", - "@tf.function\n", - "def compute_loss(model, x):\n", - " mean, logvar = model.encode(x)\n", - " z = model.reparameterize(mean, logvar)\n", - " x_logit = model.decode(z)\n", - "\n", - " cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, labels=x)\n", - " logpx_z = -tf.reduce_sum(cross_ent, axis=[1, 2, 3])\n", - " logpz = log_normal_pdf(z, 0., 0.)\n", - " logqz_x = log_normal_pdf(z, mean, logvar)\n", - " return -tf.reduce_mean(logpx_z + logpz - logqz_x)\n", - "\n", - "@tf.function\n", - "def compute_apply_gradients(model, x, optimizer):\n", - " with tf.GradientTape() as tape:\n", - " loss = compute_loss(model, x)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Rw1fkAczTQYh" - }, - "source": [ - "## 训练\n", - "\n", - "* 我们从迭代数据集开始\n", - "* 在每次迭代期间，我们将图像传递给编码器，以获得近似后验 $q(z|x)$ 的一组均值和对数方差参数\n", - "* 然后，我们应用 *重参数化技巧* 从 $q(z|x)$ 中采样\n", - "* 最后，我们将重新参数化的样本传递给解码器，以获取生成分布 $p(x|z)$ 的 logit\n", - "* **注意：**由于我们使用的是由 keras 加载的数据集，其中训练集中有 6 万个数据点，测试集中有 1 万个数据点，因此我们在测试集上的最终 ELBO 略高于对 Larochelle 版 MNIST 使用动态二值化的文献中的报告结果。\n", - "\n", - "## 生成图片\n", - "\n", - "* 进行训练后，可以生成一些图片了\n", - "* 我们首先从单位高斯先验分布 $p(z)$ 中采样一组潜在向量\n", - "* 随后生成器将潜在样本 $z$ 转换为观测值的 logit，得到分布 $p(x|z)$\n", - "* 这里我们画出伯努利分布的概率" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "NS2GWywBbAWo" - }, - "outputs": [], - "source": [ - "epochs = 100\n", - "latent_dim = 50\n", - "num_examples_to_generate = 16\n", - "\n", - "# 保持随机向量恒定以进行生成（预测），以便更易于看到改进。\n", - "random_vector_for_generation = tf.random.normal(\n", - " shape=[num_examples_to_generate, latent_dim])\n", - "model = CVAE(latent_dim)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "RmdVsmvhPxyy" - }, - "outputs": [], - "source": [ - "def generate_and_save_images(model, epoch, test_input):\n", - " predictions = model.sample(test_input)\n", - " fig = plt.figure(figsize=(4,4))\n", - "\n", - " for i in range(predictions.shape[0]):\n", - " plt.subplot(4, 4, i+1)\n", - " plt.imshow(predictions[i, :, :, 0], cmap='gray')\n", - " plt.axis('off')\n", - "\n", - " # tight_layout 最小化两个子图之间的重叠\n", - " plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2M7LmLtGEMQJ" - }, - "outputs": [], - "source": [ - "generate_and_save_images(model, 0, random_vector_for_generation)\n", - "\n", - "for epoch in range(1, epochs + 1):\n", - " start_time = time.time()\n", - " for train_x in train_dataset:\n", - " compute_apply_gradients(model, train_x, optimizer)\n", - " end_time = time.time()\n", - "\n", - " if epoch % 1 == 0:\n", - " loss = tf.keras.metrics.Mean()\n", - " for test_x in test_dataset:\n", - " loss(compute_loss(model, test_x))\n", - " elbo = -loss.result()\n", - " display.clear_output(wait=False)\n", - " print('Epoch: {}, Test set ELBO: {}, '\n", - " 'time elapse for current epoch {}'.format(epoch,\n", - " elbo,\n", - " end_time - start_time))\n", - " generate_and_save_images(\n", - " model, epoch, random_vector_for_generation)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P4M_vIbUi7c0" - }, - "source": [ - "### 使用 epoch 编号显示图片" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WfO5wCdclHGL" - }, - "outputs": [], - "source": [ - "def display_image(epoch_no):\n", - " return PIL.Image.open('image_at_epoch_{:04d}.png'.format(epoch_no))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5x3q9_Oe5q0A" - }, - "outputs": [], - "source": [ - "plt.imshow(display_image(epochs))\n", - "plt.axis('off')# 显示图片" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NywiH3nL8guF" - }, - "source": [ - "### 生成所有保存图片的 GIF" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "IGKQgENQ8lEI" - }, - "outputs": [], - "source": [ - "anim_file = 'cvae.gif'\n", - "\n", - "with imageio.get_writer(anim_file, mode='I') as writer:\n", - " filenames = glob.glob('image*.png')\n", - " filenames = sorted(filenames)\n", - " last = -1\n", - " for i,filename in enumerate(filenames):\n", - " frame = 2*(i**0.5)\n", - " if round(frame) \u003e round(last):\n", - " last = frame\n", - " else:\n", - " continue\n", - " image = imageio.imread(filename)\n", - " writer.append_data(image)\n", - " image = imageio.imread(filename)\n", - " writer.append_data(image)\n", - "\n", - "import IPython\n", - "if IPython.version_info \u003e= (6,2,0,''):\n", - " display.Image(filename=anim_file)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yQXO_dlXkKsT" - }, - "source": [ - "如果您正使用 Colab，您可以使用以下代码下载动画。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "4fSJS3m5HLFM" - }, - "outputs": [], - "source": [ - "try:\n", - " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download(anim_file)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [], - "name": "cvae.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/zh-cn/tutorials/generative/cyclegan.ipynb b/site/zh-cn/tutorials/generative/cyclegan.ipynb deleted file mode 100644 index 9901b856e1c..00000000000 --- a/site/zh-cn/tutorials/generative/cyclegan.ipynb +++ /dev/null @@ -1,958 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "file_extension": ".py", - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - }, - "mimetype": "text/x-python", - "name": "python", - "npconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": 3, - "colab": { - "name": "cyclegan.ipynb", - "provenance": [], - "private_outputs": true, - "toc_visible": true - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "v1CUZ0dkOo_F" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "qmkj-80IHxnd", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_xnMOsbqHz61" - }, - "source": [ - "# CycleGAN" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ds4o1h4WHz9U" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorFlow.google.cn 上查看\n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TE8yKcX672WZ", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ITZuApL56Mny" - }, - "source": [ - "本笔记演示了使用条件 GAN 进行的未配对图像到图像转换，如[使用循环一致的对抗网络进行未配对图像到图像转换](https://arxiv.org/abs/1703.10593) 中所述，也称之为 CycleGAN。论文提出了一种可以捕捉图像域特征并找出如何将这些特征转换为另一个图像域的方法，而无需任何成对的训练样本。\n", - "\n", - "本笔记假定您熟悉 Pix2Pix，您可以在 [Pix2Pix 教程](https://tensorflow.google.cn/tutorials/generative/pix2pix)中了解有关它的信息。CycleGAN 的代码与其相似，主要区别在于额外的损失函数，以及非配对训练数据的使用。\n", - "\n", - "CycleGAN 使用循环一致损失来使训练过程无需配对数据。换句话说，它可以从一个域转换到另一个域，而不需要在源域与目标域之间进行一对一映射。\n", - "\n", - "这为完成许多有趣的任务开辟了可能性，例如照片增强、图片着色、风格迁移等。您所需要的只是源数据集和目标数据集（仅仅是图片目录）\n", - "\n", - "![输出图像 1](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/images/horse2zebra_1.png?raw=1)\n", - "![输出图像 2](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/images/horse2zebra_2.png?raw=1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e1_Y75QXJS6h" - }, - "source": [ - "## 设定输入管线" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5fGHWOKPX4ta" - }, - "source": [ - "安装 [tensorflow_examples](https://github.com/tensorflow/examples) 包，以导入生成器和判别器。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bJ1ROiQxJ-vY", - "colab": {} - }, - "source": [ - "!pip install git+https://github.com/tensorflow/examples.git" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lhSsUx9Nyb3t", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version 只在 Colab 中使用。\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YfIk2es3hJEd", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow_datasets as tfds\n", - "from tensorflow_examples.models.pix2pix import pix2pix\n", - "\n", - "import os\n", - "import time\n", - "import matplotlib.pyplot as plt\n", - "from IPython.display import clear_output\n", - "\n", - "tfds.disable_progress_bar()\n", - "AUTOTUNE = tf.data.experimental.AUTOTUNE" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iYn4MdZnKCey" - }, - "source": [ - "## 输入管线\n", - "\n", - "本教程训练一个模型，以将普通马图片转换为斑马图片。您可以在[此处](https://tensorflow.google.cn/datasets/datasets#cycle_gan)获取该数据集以及类似数据集。\n", - "\n", - "如[论文](https://arxiv.org/abs/1703.10593)所述，将随机抖动和镜像应用到训练集。这是一些避免过拟合的图像增强技术。\n", - "\n", - "这类似于 [pix2pix](https://tensorflow.google.cn/tutorials/generative/pix2pix#load_the_dataset) 中所做的工作。\n", - "\n", - "* 在随机抖动中，图片大小调整为 `286 x 286`，随后被随机裁剪为 `256 x 256`。\n", - "* 在随机镜像中，图片会从左到右随机翻转。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iuGVPOo7Cce0", - "colab": {} - }, - "source": [ - "dataset, metadata = tfds.load('cycle_gan/horse2zebra',\n", - " with_info=True, as_supervised=True)\n", - "\n", - "train_horses, train_zebras = dataset['trainA'], dataset['trainB']\n", - "test_horses, test_zebras = dataset['testA'], dataset['testB']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2CbTEt448b4R", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 1000\n", - "BATCH_SIZE = 1\n", - "IMG_WIDTH = 256\n", - "IMG_HEIGHT = 256" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Yn3IwqhiIszt", - "colab": {} - }, - "source": [ - "def random_crop(image):\n", - " cropped_image = tf.image.random_crop(\n", - " image, size=[IMG_HEIGHT, IMG_WIDTH, 3])\n", - "\n", - " return cropped_image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "muhR2cgbLKWW", - "colab": {} - }, - "source": [ - "# 将图像归一化到区间 [-1, 1] 内。\n", - "def normalize(image):\n", - " image = tf.cast(image, tf.float32)\n", - " image = (image / 127.5) - 1\n", - " return image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fVQOjcPVLrUc", - "colab": {} - }, - "source": [ - "def random_jitter(image):\n", - " # 调整大小为 286 x 286 x 3\n", - " image = tf.image.resize(image, [286, 286],\n", - " method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)\n", - "\n", - " # 随机裁剪到 256 x 256 x 3\n", - " image = random_crop(image)\n", - "\n", - " # 随机镜像\n", - " image = tf.image.random_flip_left_right(image)\n", - "\n", - " return image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tyaP4hLJ8b4W", - "colab": {} - }, - "source": [ - "def preprocess_image_train(image, label):\n", - " image = random_jitter(image)\n", - " image = normalize(image)\n", - " return image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VB3Z6D_zKSru", - "colab": {} - }, - "source": [ - "def preprocess_image_test(image, label):\n", - " image = normalize(image)\n", - " return image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RsajGXxd5JkZ", - "colab": {} - }, - "source": [ - "train_horses = train_horses.map(\n", - " preprocess_image_train, num_parallel_calls=AUTOTUNE).cache().shuffle(\n", - " BUFFER_SIZE).batch(1)\n", - "\n", - "train_zebras = train_zebras.map(\n", - " preprocess_image_train, num_parallel_calls=AUTOTUNE).cache().shuffle(\n", - " BUFFER_SIZE).batch(1)\n", - "\n", - "test_horses = test_horses.map(\n", - " preprocess_image_test, num_parallel_calls=AUTOTUNE).cache().shuffle(\n", - " BUFFER_SIZE).batch(1)\n", - "\n", - "test_zebras = test_zebras.map(\n", - " preprocess_image_test, num_parallel_calls=AUTOTUNE).cache().shuffle(\n", - " BUFFER_SIZE).batch(1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "e3MhJ3zVLPan", - "colab": {} - }, - "source": [ - "sample_horse = next(iter(train_horses))\n", - "sample_zebra = next(iter(train_zebras))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4pOYjMk_KfIB", - "colab": {} - }, - "source": [ - "plt.subplot(121)\n", - "plt.title('Horse')\n", - "plt.imshow(sample_horse[0] * 0.5 + 0.5)\n", - "\n", - "plt.subplot(122)\n", - "plt.title('Horse with random jitter')\n", - "plt.imshow(random_jitter(sample_horse[0]) * 0.5 + 0.5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0KJyB9ENLb2y", - "colab": {} - }, - "source": [ - "plt.subplot(121)\n", - "plt.title('Zebra')\n", - "plt.imshow(sample_zebra[0] * 0.5 + 0.5)\n", - "\n", - "plt.subplot(122)\n", - "plt.title('Zebra with random jitter')\n", - "plt.imshow(random_jitter(sample_zebra[0]) * 0.5 + 0.5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hvX8sKsfMaio" - }, - "source": [ - "## 导入并重用 Pix2Pix 模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cGrL73uCd-_M" - }, - "source": [ - "通过安装的 [tensorflow_examples](https://github.com/tensorflow/examples) 包导入 [Pix2Pix](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/pix2pix/pix2pix.py) 中的生成器和判别器。\n", - "\n", - "本教程中使用模型体系结构与 [pix2pix](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/pix2pix/pix2pix.py) 中所使用的非常相似。一些区别在于：\n", - "\n", - "* Cyclegan 使用 [instance normalization（实例归一化）](https://arxiv.org/abs/1607.08022)而不是 [batch normalization （批归一化）](https://arxiv.org/abs/1502.03167)。\n", - "* [CycleGAN 论文](https://arxiv.org/abs/1703.10593)使用一种基于 `resnet` 的改进生成器。简单起见，本教程使用的是改进的 `unet` 生成器。\n", - "\n", - "这里训练了两个生成器（G 和 F）以及两个判别器（X 和 Y）。\n", - "\n", - "* 生成器 `G` 学习将图片 `X` 转换为 `Y`。 $(G: X -> Y)$\n", - "* 生成器 `F` 学习将图片 `Y` 转换为 `X`。 $(F: Y -> X)$\n", - "* 判别器 `D_X` 学习区分图片 `X` 与生成的图片 `X` (`F(Y)`)。\n", - "* 判别器 `D_Y` 学习区分图片 `Y` 与生成的图片 `Y` (`G(X)`)。\n", - "\n", - "![Cyclegan 模型](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/images/cyclegan_model.png?raw=1)" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8ju9Wyw87MRW", - "colab": {} - }, - "source": [ - "OUTPUT_CHANNELS = 3\n", - "\n", - "generator_g = pix2pix.unet_generator(OUTPUT_CHANNELS, norm_type='instancenorm')\n", - "generator_f = pix2pix.unet_generator(OUTPUT_CHANNELS, norm_type='instancenorm')\n", - "\n", - "discriminator_x = pix2pix.discriminator(norm_type='instancenorm', target=False)\n", - "discriminator_y = pix2pix.discriminator(norm_type='instancenorm', target=False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wDaGZ3WpZUyw", - "colab": {} - }, - "source": [ - "to_zebra = generator_g(sample_horse)\n", - "to_horse = generator_f(sample_zebra)\n", - "plt.figure(figsize=(8, 8))\n", - "contrast = 8\n", - "\n", - "imgs = [sample_horse, to_zebra, sample_zebra, to_horse]\n", - "title = ['Horse', 'To Zebra', 'Zebra', 'To Horse']\n", - "\n", - "for i in range(len(imgs)):\n", - " plt.subplot(2, 2, i+1)\n", - " plt.title(title[i])\n", - " if i % 2 == 0:\n", - " plt.imshow(imgs[i][0] * 0.5 + 0.5)\n", - " else:\n", - " plt.imshow(imgs[i][0] * 0.5 * contrast + 0.5)\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "O5MhJmxyZiy9", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(8, 8))\n", - "\n", - "plt.subplot(121)\n", - "plt.title('Is a real zebra?')\n", - "plt.imshow(discriminator_y(sample_zebra)[0, ..., -1], cmap='RdBu_r')\n", - "\n", - "plt.subplot(122)\n", - "plt.title('Is a real horse?')\n", - "plt.imshow(discriminator_x(sample_horse)[0, ..., -1], cmap='RdBu_r')\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0FMYgY_mPfTi" - }, - "source": [ - "## 损失函数" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JRqt02lupRn8" - }, - "source": [ - "在 CycleGAN 中，没有可训练的成对数据，因此无法保证输入 `x` 和 目标 `y` 数据对在训练期间是有意义的。所以为了强制网络学习正确的映射，作者提出了循环一致损失。\n", - "\n", - "判别器损失和生成器损失和 [pix2pix](https://google.tensorflow.cn/tutorials/generative/pix2pix#define_the_loss_functions_and_the_optimizer) 中所使用的类似。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cyhxTuvJyIHV", - "colab": {} - }, - "source": [ - "LAMBDA = 10" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Q1Xbz5OaLj5C", - "colab": {} - }, - "source": [ - "loss_obj = tf.keras.losses.BinaryCrossentropy(from_logits=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wkMNfBWlT-PV", - "colab": {} - }, - "source": [ - "def discriminator_loss(real, generated):\n", - " real_loss = loss_obj(tf.ones_like(real), real)\n", - "\n", - " generated_loss = loss_obj(tf.zeros_like(generated), generated)\n", - "\n", - " total_disc_loss = real_loss + generated_loss\n", - "\n", - " return total_disc_loss * 0.5" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "90BIcCKcDMxz", - "colab": {} - }, - "source": [ - "def generator_loss(generated):\n", - " return loss_obj(tf.ones_like(generated), generated)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5iIWQzVF7f9e" - }, - "source": [ - "循环一致意味着结果应接近原始输出。例如，将一句英文译为法文，随后再从法文翻译回英文，最终的结果句应与原始句输入相同。\n", - "\n", - "在循环一致损失中，\n", - "\n", - "* 图片 $X$ 通过生成器 $G$ 传递，该生成器生成图片 $\\hat{Y}$。\n", - "* 生成的图片 $\\hat{Y}$ 通过生成器 $F$ 传递，循环生成图片 $\\hat{X}$。\n", - "* 在 $X$ 和 $\\hat{X}$ 之间计算平均绝对误差。\n", - "\n", - "$$forward\\ cycle\\ consistency\\ loss: X -> G(X) -> F(G(X)) \\sim \\hat{X}$$\n", - "\n", - "$$backward\\ cycle\\ consistency\\ loss: Y -> F(Y) -> G(F(Y)) \\sim \\hat{Y}$$\n", - "\n", - "\n", - "![循环损失](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/images/cycle_loss.png?raw=1)" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NMpVGj_sW6Vo", - "colab": {} - }, - "source": [ - "def calc_cycle_loss(real_image, cycled_image):\n", - " loss1 = tf.reduce_mean(tf.abs(real_image - cycled_image))\n", - " \n", - " return LAMBDA * loss1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U-tJL-fX0Mq7" - }, - "source": [ - "如上所示，生成器 $G$ 负责将图片 $X$ 转换为 $Y$。一致性损失表明，如果您将图片 $Y$ 馈送给生成器 $G$，它应当生成真实图片 $Y$ 或接近于 $Y$ 的图片。\n", - "\n", - "$$Identity\\ loss = |G(Y) - Y| + |F(X) - X|$$" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "05ywEH680Aud", - "colab": {} - }, - "source": [ - "def identity_loss(real_image, same_image):\n", - " loss = tf.reduce_mean(tf.abs(real_image - same_image))\n", - " return LAMBDA * 0.5 * loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "G-vjRM7IffTT" - }, - "source": [ - "为所有生成器和判别器初始化优化器。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iWCn_PVdEJZ7", - "colab": {} - }, - "source": [ - "generator_g_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\n", - "generator_f_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\n", - "\n", - "discriminator_x_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)\n", - "discriminator_y_optimizer = tf.keras.optimizers.Adam(2e-4, beta_1=0.5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aKUZnDiqQrAh" - }, - "source": [ - "## Checkpoints" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WJnftd5sQsv6", - "colab": {} - }, - "source": [ - "checkpoint_path = \"./checkpoints/train\"\n", - "\n", - "ckpt = tf.train.Checkpoint(generator_g=generator_g,\n", - " generator_f=generator_f,\n", - " discriminator_x=discriminator_x,\n", - " discriminator_y=discriminator_y,\n", - " generator_g_optimizer=generator_g_optimizer,\n", - " generator_f_optimizer=generator_f_optimizer,\n", - " discriminator_x_optimizer=discriminator_x_optimizer,\n", - " discriminator_y_optimizer=discriminator_y_optimizer)\n", - "\n", - "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n", - "\n", - "# 如果存在检查点，恢复最新版本检查点\n", - "if ckpt_manager.latest_checkpoint:\n", - " ckpt.restore(ckpt_manager.latest_checkpoint)\n", - " print ('Latest checkpoint restored!!')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Rw1fkAczTQYh" - }, - "source": [ - "## 训练\n", - "\n", - "注意：本示例模型比论文中训练了更少的 epoch（本示例为 40 epoch，论文中为 200 epoch），以使训练时间相对于本教程是合理的。预测的准确率可能会低一些。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NS2GWywBbAWo", - "colab": {} - }, - "source": [ - "EPOCHS = 40" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RmdVsmvhPxyy", - "colab": {} - }, - "source": [ - "def generate_images(model, test_input):\n", - " prediction = model(test_input)\n", - " \n", - " plt.figure(figsize=(12, 12))\n", - "\n", - " display_list = [test_input[0], prediction[0]]\n", - " title = ['Input Image', 'Predicted Image']\n", - "\n", - " for i in range(2):\n", - " plt.subplot(1, 2, i+1)\n", - " plt.title(title[i])\n", - " # 获取范围在 [0, 1] 之间的像素值以绘制它。\n", - " plt.imshow(display_list[i] * 0.5 + 0.5)\n", - " plt.axis('off')\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kE47ERn5fyLC" - }, - "source": [ - "尽管训练循环看起来很复杂，其实包含四个基本步骤：\n", - "\n", - "* 获取预测。\n", - "* 计算损失值。\n", - "* 使用反向传播计算损失值。\n", - "* 将梯度应用于优化器。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KBKUV2sKXDbY", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(real_x, real_y):\n", - " # persistent 设置为 Ture，因为 GradientTape 被多次应用于计算梯度。\n", - " with tf.GradientTape(persistent=True) as tape:\n", - " # 生成器 G 转换 X -> Y。\n", - " # 生成器 F 转换 Y -> X。\n", - " \n", - " fake_y = generator_g(real_x, training=True)\n", - " cycled_x = generator_f(fake_y, training=True)\n", - "\n", - " fake_x = generator_f(real_y, training=True)\n", - " cycled_y = generator_g(fake_x, training=True)\n", - "\n", - " # same_x 和 same_y 用于一致性损失。\n", - " same_x = generator_f(real_x, training=True)\n", - " same_y = generator_g(real_y, training=True)\n", - "\n", - " disc_real_x = discriminator_x(real_x, training=True)\n", - " disc_real_y = discriminator_y(real_y, training=True)\n", - "\n", - " disc_fake_x = discriminator_x(fake_x, training=True)\n", - " disc_fake_y = discriminator_y(fake_y, training=True)\n", - "\n", - " # 计算损失。\n", - " gen_g_loss = generator_loss(disc_fake_y)\n", - " gen_f_loss = generator_loss(disc_fake_x)\n", - " \n", - " total_cycle_loss = calc_cycle_loss(real_x, cycled_x) + calc_cycle_loss(real_y, cycled_y)\n", - " \n", - " # 总生成器损失 = 对抗性损失 + 循环损失。\n", - " total_gen_g_loss = gen_g_loss + total_cycle_loss + identity_loss(real_y, same_y)\n", - " total_gen_f_loss = gen_f_loss + total_cycle_loss + identity_loss(real_x, same_x)\n", - "\n", - " disc_x_loss = discriminator_loss(disc_real_x, disc_fake_x)\n", - " disc_y_loss = discriminator_loss(disc_real_y, disc_fake_y)\n", - " \n", - " # 计算生成器和判别器损失。\n", - " generator_g_gradients = tape.gradient(total_gen_g_loss, \n", - " generator_g.trainable_variables)\n", - " generator_f_gradients = tape.gradient(total_gen_f_loss, \n", - " generator_f.trainable_variables)\n", - " \n", - " discriminator_x_gradients = tape.gradient(disc_x_loss, \n", - " discriminator_x.trainable_variables)\n", - " discriminator_y_gradients = tape.gradient(disc_y_loss, \n", - " discriminator_y.trainable_variables)\n", - " \n", - " # 将梯度应用于优化器。\n", - " generator_g_optimizer.apply_gradients(zip(generator_g_gradients, \n", - " generator_g.trainable_variables))\n", - "\n", - " generator_f_optimizer.apply_gradients(zip(generator_f_gradients, \n", - " generator_f.trainable_variables))\n", - " \n", - " discriminator_x_optimizer.apply_gradients(zip(discriminator_x_gradients,\n", - " discriminator_x.trainable_variables))\n", - " \n", - " discriminator_y_optimizer.apply_gradients(zip(discriminator_y_gradients,\n", - " discriminator_y.trainable_variables))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2M7LmLtGEMQJ", - "colab": {} - }, - "source": [ - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - "\n", - " n = 0\n", - " for image_x, image_y in tf.data.Dataset.zip((train_horses, train_zebras)):\n", - " train_step(image_x, image_y)\n", - " if n % 10 == 0:\n", - " print ('.', end='')\n", - " n+=1\n", - "\n", - " clear_output(wait=True)\n", - " # 使用一致的图像（sample_horse），以便模型的进度清晰可见。\n", - " generate_images(generator_g, sample_horse)\n", - "\n", - " if (epoch + 1) % 5 == 0:\n", - " ckpt_save_path = ckpt_manager.save()\n", - " print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,\n", - " ckpt_save_path))\n", - "\n", - " print ('Time taken for epoch {} is {} sec\\n'.format(epoch + 1,\n", - " time.time()-start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1RGysMU_BZhx" - }, - "source": [ - "## 使用测试数据集进行生成" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KUgSnmy2nqSP", - "colab": {} - }, - "source": [ - "# 在测试数据集上运行训练的模型。\n", - "for inp in test_horses.take(5):\n", - " generate_images(generator_g, inp)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ABGiHY6fE02b" - }, - "source": [ - "## 下一步\n", - "本教程展示了如何从 [Pix2Pix](https://tensorflow.google.cn/tutorials/generative/pix2pix) 教程实现的生成器和判别器开始实现 CycleGAN。 下一步，您可以尝试使用一个来源于 [TensorFlow 数据集](https://tensorflow.google.cn/datasets/datasets#cycle_gan)的不同的数据集。\n", - "\n", - "您也可以训练更多的 epoch 以改进结果，或者可以实现[论文](https://arxiv.org/abs/1703.10593)中所使用的改进 ResNet 生成器来代替这里使用的 U-Net 生成器。" - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/generative/dcgan.ipynb b/site/zh-cn/tutorials/generative/dcgan.ipynb deleted file mode 100644 index 54755dda8fc..00000000000 --- a/site/zh-cn/tutorials/generative/dcgan.ipynb +++ /dev/null @@ -1,931 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "dcgan.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_jQ1tEQCxwRx" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "V_sgB_5dx1f1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rF2x3qooyBTI" - }, - "source": [ - "# 深度卷积生成对抗网络" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0TD5ZrvEMbhZ" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " 在 tensorFlow.google.cn 上查看\n", - " \n", - " \n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " \n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M0gHG-LEgLZx" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ITZuApL56Mny" - }, - "source": [ - "本教程演示了如何使用[深度卷积生成对抗网络](https://arxiv.org/pdf/1511.06434.pdf)（DCGAN）生成手写数字图片。该代码是使用 [Keras Sequential API](https://tensorflow.google.cn/guide/keras) 与 `tf.GradientTape` 训练循环编写的。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2MbKJY38Puy9" - }, - "source": [ - "## 什么是生成对抗网络？\n", - "\n", - "[生成对抗网络](https://arxiv.org/abs/1406.2661)（GANs）是当今计算机科学领域最有趣的想法之一。两个模型通过对抗过程同时训练。一个*生成器*（“艺术家”）学习创造看起来真实的图像，而*判别器*（“艺术评论家”）学习区分真假图像。\n", - "\n", - "![生成器和判别器图示](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/images/gan1.png?raw=1)\n", - "\n", - "训练过程中，*生成器*在生成逼真图像方面逐渐变强，而*判别器*在辨别这些图像的能力上逐渐变强。当*判别器*不再能够区分真实图片和伪造图片时，训练过程达到平衡。\n", - "\n", - "![生成器和判别器图示二](https://github.com/tensorflow/docs/blob/master/site/en/tutorials/generative/images/gan2.png?raw=1)\n", - "\n", - "本笔记在 MNIST 数据集上演示了该过程。下方动画展示了当训练了 50 个epoch （全部数据集迭代50次） 时*生成器*所生成的一系列图片。图片从随机噪声开始，随着时间的推移越来越像手写数字。\n", - "\n", - "![输出样本](https://tensorflow.google.cn/images/gan/dcgan.gif)\n", - "\n", - "要了解关于 GANs 的更多信息，我们建议参阅 MIT的 [深度学习入门](http://introtodeeplearning.com/) 课程。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "e1_Y75QXJS6h" - }, - "source": [ - "### Import TensorFlow and other libraries" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J5oue0oqCkZZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "g5RstiiB8V-z", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version 只在 Colab 中使用。\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WZKbyU2-AiY-", - "colab": {} - }, - "source": [ - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "57FFuKn4gLZ9", - "colab": {} - }, - "source": [ - "tf.__version__" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YzTlj4YdCip_", - "colab": {} - }, - "source": [ - "# 用于生成 GIF 图片\n", - "!pip install imageio" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YfIk2es3hJEd", - "colab": {} - }, - "source": [ - "import glob\n", - "import imageio\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import os\n", - "import PIL\n", - "from tensorflow.keras import layers\n", - "import time\n", - "\n", - "from IPython import display" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iYn4MdZnKCey" - }, - "source": [ - "### 加载和准备数据集\n", - "\n", - "您将使用 MNIST 数据集来训练生成器和判别器。生成器将生成类似于 MNIST 数据集的手写数字。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "a4fYMGxGhrna", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (_, _) = tf.keras.datasets.mnist.load_data()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NFC2ghIdiZYE", - "colab": {} - }, - "source": [ - "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n", - "train_images = (train_images - 127.5) / 127.5 # 将图片标准化到 [-1, 1] 区间内" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "S4PIDhoDLbsZ", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 60000\n", - "BATCH_SIZE = 256" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-yKCCQOoJ7cn", - "colab": {} - }, - "source": [ - "# 批量化和打乱数据\n", - "train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "THY-sZMiQ4UV" - }, - "source": [ - "## 创建模型\n", - "\n", - "生成器和判别器均使用 [Keras Sequential API](https://tensorflow.google.cn/guide/keras#sequential_model) 定义。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-tEyxE-GMC48" - }, - "source": [ - "### 生成器\n", - "\n", - "生成器使用 `tf.keras.layers.Conv2DTranspose` （上采样）层来从种子（随机噪声）中产生图片。以一个使用该种子作为输入的 `Dense` 层开始，然后多次上采样直到达到所期望的 28x28x1 的图片尺寸。注意除了输出层使用 tanh 之外，其他每层均使用 `tf.keras.layers.LeakyReLU` 作为激活函数。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6bpTcDqoLWjY", - "colab": {} - }, - "source": [ - "def make_generator_model():\n", - " model = tf.keras.Sequential()\n", - " model.add(layers.Dense(7*7*256, use_bias=False, input_shape=(100,)))\n", - " model.add(layers.BatchNormalization())\n", - " model.add(layers.LeakyReLU())\n", - "\n", - " model.add(layers.Reshape((7, 7, 256)))\n", - " assert model.output_shape == (None, 7, 7, 256) # 注意：batch size 没有限制\n", - "\n", - " model.add(layers.Conv2DTranspose(128, (5, 5), strides=(1, 1), padding='same', use_bias=False))\n", - " assert model.output_shape == (None, 7, 7, 128)\n", - " model.add(layers.BatchNormalization())\n", - " model.add(layers.LeakyReLU())\n", - "\n", - " model.add(layers.Conv2DTranspose(64, (5, 5), strides=(2, 2), padding='same', use_bias=False))\n", - " assert model.output_shape == (None, 14, 14, 64)\n", - " model.add(layers.BatchNormalization())\n", - " model.add(layers.LeakyReLU())\n", - "\n", - " model.add(layers.Conv2DTranspose(1, (5, 5), strides=(2, 2), padding='same', use_bias=False, activation='tanh'))\n", - " assert model.output_shape == (None, 28, 28, 1)\n", - "\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GyWgG09LCSJl" - }, - "source": [ - "使用（尚未训练的）生成器创建一张图片。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "o6VvUbMqgLaS", - "colab": {} - }, - "source": [ - "generator = make_generator_model()\n", - "\n", - "noise = tf.random.normal([1, 100])\n", - "generated_image = generator(noise, training=False)\n", - "\n", - "plt.imshow(generated_image[0, :, :, 0], cmap='gray')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "D0IKnaCtg6WE" - }, - "source": [ - "### 判别器\n", - "\n", - "判别器是一个基于 CNN 的图片分类器。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dw2tPLmk2pEP", - "colab": {} - }, - "source": [ - "def make_discriminator_model():\n", - " model = tf.keras.Sequential()\n", - " model.add(layers.Conv2D(64, (5, 5), strides=(2, 2), padding='same',\n", - " input_shape=[28, 28, 1]))\n", - " model.add(layers.LeakyReLU())\n", - " model.add(layers.Dropout(0.3))\n", - "\n", - " model.add(layers.Conv2D(128, (5, 5), strides=(2, 2), padding='same'))\n", - " model.add(layers.LeakyReLU())\n", - " model.add(layers.Dropout(0.3))\n", - "\n", - " model.add(layers.Flatten())\n", - " model.add(layers.Dense(1))\n", - "\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QhPneagzCaQv" - }, - "source": [ - "使用（尚未训练的）判别器来对图片的真伪进行判断。模型将被训练为为真实图片输出正值，为伪造图片输出负值。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-nnSVbzhgLaX", - "colab": {} - }, - "source": [ - "discriminator = make_discriminator_model()\n", - "decision = discriminator(generated_image)\n", - "print (decision)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0FMYgY_mPfTi" - }, - "source": [ - "## 定义损失函数和优化器\n", - "\n", - "为两个模型定义损失函数和优化器。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "psQfmXxYKU3X", - "colab": {} - }, - "source": [ - "# 该方法返回计算交叉熵损失的辅助函数\n", - "cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PKY_iPSPNWoj" - }, - "source": [ - "### 判别器损失\n", - "\n", - "该方法量化判别器从判断真伪图片的能力。它将判别器对真实图片的预测值与值全为 1 的数组进行对比，将判别器对伪造（生成的）图片的预测值与值全为 0 的数组进行对比。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wkMNfBWlT-PV", - "colab": {} - }, - "source": [ - "def discriminator_loss(real_output, fake_output):\n", - " real_loss = cross_entropy(tf.ones_like(real_output), real_output)\n", - " fake_loss = cross_entropy(tf.zeros_like(fake_output), fake_output)\n", - " total_loss = real_loss + fake_loss\n", - " return total_loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Jd-3GCUEiKtv" - }, - "source": [ - "### 生成器损失\n", - "\n", - "生成器损失量化其欺骗判别器的能力。直观来讲，如果生成器表现良好，判别器将会把伪造图片判断为真实图片（或 1）。这里我们将把判别器在生成图片上的判断结果与一个值全为 1 的数组进行对比。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "90BIcCKcDMxz", - "colab": {} - }, - "source": [ - "def generator_loss(fake_output):\n", - " return cross_entropy(tf.ones_like(fake_output), fake_output)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MgIc7i0th_Iu" - }, - "source": [ - "由于我们需要分别训练两个网络，判别器和生成器的优化器是不同的。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iWCn_PVdEJZ7", - "colab": {} - }, - "source": [ - "generator_optimizer = tf.keras.optimizers.Adam(1e-4)\n", - "discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mWtinsGDPJlV" - }, - "source": [ - "### 保存检查点\n", - "\n", - "本笔记还演示了如何保存和恢复模型，这在长时间训练任务被中断的情况下比较有帮助。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CA1w-7s2POEy", - "colab": {} - }, - "source": [ - "checkpoint_dir = './training_checkpoints'\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,\n", - " discriminator_optimizer=discriminator_optimizer,\n", - " generator=generator,\n", - " discriminator=discriminator)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Rw1fkAczTQYh" - }, - "source": [ - "## 定义训练循环\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NS2GWywBbAWo", - "colab": {} - }, - "source": [ - "EPOCHS = 50\n", - "noise_dim = 100\n", - "num_examples_to_generate = 16\n", - "\n", - "\n", - "# 我们将重复使用该种子（因此在动画 GIF 中更容易可视化进度）\n", - "seed = tf.random.normal([num_examples_to_generate, noise_dim])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jylSonrqSWfi" - }, - "source": [ - "训练循环在生成器接收到一个随机种子作为输入时开始。该种子用于生产一张图片。判别器随后被用于区分真实图片（选自训练集）和伪造图片（由生成器生成）。针对这里的每一个模型都计算损失函数，并且计算梯度用于更新生成器与判别器。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3t5ibNo05jCB", - "colab": {} - }, - "source": [ - "# 注意 `tf.function` 的使用\n", - "# 该注解使函数被“编译”\n", - "@tf.function\n", - "def train_step(images):\n", - " noise = tf.random.normal([BATCH_SIZE, noise_dim])\n", - "\n", - " with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:\n", - " generated_images = generator(noise, training=True)\n", - "\n", - " real_output = discriminator(images, training=True)\n", - " fake_output = discriminator(generated_images, training=True)\n", - "\n", - " gen_loss = generator_loss(fake_output)\n", - " disc_loss = discriminator_loss(real_output, fake_output)\n", - "\n", - " gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)\n", - " gradients_of_discriminator = disc_tape.gradient(disc_loss, discriminator.trainable_variables)\n", - "\n", - " generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))\n", - " discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator, discriminator.trainable_variables))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2M7LmLtGEMQJ", - "colab": {} - }, - "source": [ - "def train(dataset, epochs):\n", - " for epoch in range(epochs):\n", - " start = time.time()\n", - "\n", - " for image_batch in dataset:\n", - " train_step(image_batch)\n", - "\n", - " # 继续进行时为 GIF 生成图像\n", - " display.clear_output(wait=True)\n", - " generate_and_save_images(generator,\n", - " epoch + 1,\n", - " seed)\n", - "\n", - " # 每 15 个 epoch 保存一次模型\n", - " if (epoch + 1) % 15 == 0:\n", - " checkpoint.save(file_prefix = checkpoint_prefix)\n", - "\n", - " print ('Time for epoch {} is {} sec'.format(epoch + 1, time.time()-start))\n", - "\n", - " # 最后一个 epoch 结束后生成图片\n", - " display.clear_output(wait=True)\n", - " generate_and_save_images(generator,\n", - " epochs,\n", - " seed)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2aFF7Hk3XdeW" - }, - "source": [ - "**生成与保存图片**\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RmdVsmvhPxyy", - "colab": {} - }, - "source": [ - "def generate_and_save_images(model, epoch, test_input):\n", - " # 注意 training` 设定为 False\n", - " # 因此，所有层都在推理模式下运行（batchnorm）。\n", - " predictions = model(test_input, training=False)\n", - "\n", - " fig = plt.figure(figsize=(4,4))\n", - "\n", - " for i in range(predictions.shape[0]):\n", - " plt.subplot(4, 4, i+1)\n", - " plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')\n", - " plt.axis('off')\n", - "\n", - " plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dZrd4CdjR-Fp" - }, - "source": [ - "## 训练模型\n", - "调用上面定义的 `train()` 方法来同时训练生成器和判别器。注意，训练 GANs 可能是棘手的。重要的是，生成器和判别器不能够互相压制对方（例如，他们以相似的学习率训练）。\n", - "\n", - "在训练之初，生成的图片看起来像是随机噪声。随着训练过程的进行，生成的数字将越来越真实。在大概 50 个 epoch 之后，这些图片看起来像是 MNIST 数字。使用 Colab 中的默认设置可能需要大约 1 分钟每 epoch。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ly3UN0SLLY2l", - "colab": {} - }, - "source": [ - "%%time\n", - "train(train_dataset, EPOCHS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rfM4YcPVPkNO" - }, - "source": [ - "恢复最新的检查点。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SKeqh9GlgLa7", - "colab": {} - }, - "source": [ - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P4M_vIbUi7c0" - }, - "source": [ - "## 创建 GIF\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WfO5wCdclHGL", - "colab": {} - }, - "source": [ - "# 使用 epoch 数生成单张图片\n", - "def display_image(epoch_no):\n", - " return PIL.Image.open('image_at_epoch_{:04d}.png'.format(epoch_no))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TsUudzLCgLbA", - "colab": {} - }, - "source": [ - "display_image(EPOCHS)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NywiH3nL8guF" - }, - "source": [ - "使用训练过程中生成的图片通过 `imageio` 生成动态 gif " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IGKQgENQ8lEI", - "colab": {} - }, - "source": [ - "anim_file = 'dcgan.gif'\n", - "\n", - "with imageio.get_writer(anim_file, mode='I') as writer:\n", - " filenames = glob.glob('image*.png')\n", - " filenames = sorted(filenames)\n", - " last = -1\n", - " for i,filename in enumerate(filenames):\n", - " frame = 2*(i**0.5)\n", - " if round(frame) > round(last):\n", - " last = frame\n", - " else:\n", - " continue\n", - " image = imageio.imread(filename)\n", - " writer.append_data(image)\n", - " image = imageio.imread(filename)\n", - " writer.append_data(image)\n", - "\n", - "import IPython\n", - "if IPython.version_info > (6,2,0,''):\n", - " display.Image(filename=anim_file)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cGhC3-fMWSwl" - }, - "source": [ - "如果您正在使用 Colab，您可以通过如下代码下载动画：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uV0yiKpzNP1b", - "colab": {} - }, - "source": [ - "try:\n", - " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download(anim_file)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k6qC-SbjK0yW" - }, - "source": [ - "## 下一步\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xjjkT9KAK6H7" - }, - "source": [ - "本教程展示了实现和训练 GAN 模型所需的全部必要代码。接下来，您可能想尝试其他数据集，例如大规模名人面部属性（CelebA）数据集 [在 Kaggle 上获取](https://www.kaggle.com/jessicali9530/celeba-dataset/home)。要了解更多关于 GANs 的信息，我们推荐参阅 [NIPS 2016 教程： 生成对抗网络](https://arxiv.org/abs/1701.00160)。" - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/generative/style_transfer.ipynb b/site/zh-cn/tutorials/generative/style_transfer.ipynb deleted file mode 100644 index bc9352cc492..00000000000 --- a/site/zh-cn/tutorials/generative/style_transfer.ipynb +++ /dev/null @@ -1,1173 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "style_transfer.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6msVLevwcRhm" - }, - "source": [ - "# 神经风格迁移" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ds4o1h4WHz9U" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 上运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GEe3i16tQPjo", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aDyGj8DmXCJI" - }, - "source": [ - "本教程使用深度学习来用其他图像的风格创造一个图像（曾经你是否希望可以像毕加索或梵高一样绘画？）。 这被称为*神经风格转移*，该技术概述于 A Neural Algorithm of Artistic Style (Gatys et al.). \n", - "\n", - "神经风格迁移是一种优化技术，用于将两个图像——一个*内容*图像和一个*风格参考*图像（如著名画家的一个作品）——混合在一起，使输出的图像看起来像内容图像， 但是用了风格参考图像的风格。\n", - "\n", - "这是通过优化输出图像以匹配内容图像的内容统计数据和风格参考图像的风格统计数据来实现的。 这些统计数据可以使用卷积网络从图像中提取。\n", - "\n", - "例如，我们选取这张乌龟的照片和 Wassily Kandinsky 的作品 7：\n", - "\n", - "\n", - "\n", - "[绿海龟照片](https://commons.wikimedia.org/wiki/File:Green_Sea_Turtle_grazing_seagrass.jpg) -摄影者： P.Lindgren [CC BY-SA 3.0](https://creativecommons.org/licenses/by-sa/3.0), 来自 Wikimedia Common\n", - "\n", - "\n", - "\n", - "\n", - "如果 Kandinsky 决定用这种风格来专门描绘这只海龟会是什么样子？ 是否如下图一样？\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U8ajP_u73s6m" - }, - "source": [ - "## 配置\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eqxUicSPUOP6" - }, - "source": [ - "### 导入和配置模块" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2Mdpou0qzCm6", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NyftRTSMuwue", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sc1OLbOWhPCO", - "colab": {} - }, - "source": [ - "import IPython.display as display\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib as mpl\n", - "mpl.rcParams['figure.figsize'] = (12,12)\n", - "mpl.rcParams['axes.grid'] = False\n", - "\n", - "import numpy as np\n", - "import time\n", - "import functools" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oeXebYusyHwC" - }, - "source": [ - "下载图像并选择风格图像和内容图像：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wqc0OJHwyFAk", - "colab": {} - }, - "source": [ - "content_path = tf.keras.utils.get_file('turtle.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/Green_Sea_Turtle_grazing_seagrass.jpg')\n", - "style_path = tf.keras.utils.get_file('kandinsky.jpg','https://storage.googleapis.com/download.tensorflow.org/example_images/Vassily_Kandinsky%2C_1913_-_Composition_7.jpg')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xE4Yt8nArTeR" - }, - "source": [ - "## 将输入可视化" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "klh6ObK2t_vH" - }, - "source": [ - "定义一个加载图像的函数，并将其最大尺寸限制为 512 像素。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3TLljcwv5qZs", - "colab": {} - }, - "source": [ - "def load_img(path_to_img):\n", - " max_dim = 512\n", - " img = tf.io.read_file(path_to_img)\n", - " img = tf.image.decode_image(img, channels=3)\n", - " img = tf.image.convert_image_dtype(img, tf.float32)\n", - "\n", - " shape = tf.cast(tf.shape(img)[:-1], tf.float32)\n", - " long_dim = max(shape)\n", - " scale = max_dim / long_dim\n", - "\n", - " new_shape = tf.cast(shape * scale, tf.int32)\n", - "\n", - " img = tf.image.resize(img, new_shape)\n", - " img = img[tf.newaxis, :]\n", - " return img" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2yAlRzJZrWM3" - }, - "source": [ - "创建一个简单的函数来显示图像：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cBX-eNT8PAK_", - "colab": {} - }, - "source": [ - "def imshow(image, title=None):\n", - " if len(image.shape) > 3:\n", - " image = tf.squeeze(image, axis=0)\n", - "\n", - " plt.imshow(image)\n", - " if title:\n", - " plt.title(title)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_UWQmeEaiKkP", - "colab": {} - }, - "source": [ - "content_image = load_img(content_path)\n", - "style_image = load_img(style_path)\n", - "\n", - "plt.subplot(1, 2, 1)\n", - "imshow(content_image, 'Content Image')\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "imshow(style_image, 'Style Image')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GEwZ7FlwrjoZ" - }, - "source": [ - "## 定义内容和风格的表示\n", - "\n", - "使用模型的中间层来获取图像的*内容*和*风格*表示。 从网络的输入层开始，前几个层的激励响应表示边缘和纹理等低级 feature (特征)。 随着层数加深，最后几层代表更高级的 feature (特征)——实体的部分，如*轮子*或*眼睛*。 在此教程中，我们使用的是 VGG19 网络结构，这是一个已经预训练好的图像分类网络。 这些中间层是从图像中定义内容和风格的表示所必需的。 对于一个输入图像，我们尝试匹配这些中间层的相应风格和内容目标的表示。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LP_7zrziuiJk" - }, - "source": [ - "加载 [VGG19](https://keras.io/applications/#vgg19) 并在我们的图像上测试它以确保正常运行：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fMbzrr7BCTq0", - "colab": {} - }, - "source": [ - "x = tf.keras.applications.vgg19.preprocess_input(content_image*255)\n", - "x = tf.image.resize(x, (224, 224))\n", - "vgg = tf.keras.applications.VGG19(include_top=True, weights='imagenet')\n", - "prediction_probabilities = vgg(x)\n", - "prediction_probabilities.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1_FyCm0dYnvl", - "colab": {} - }, - "source": [ - "predicted_top_5 = tf.keras.applications.vgg19.decode_predictions(prediction_probabilities.numpy())[0]\n", - "[(class_name, prob) for (number, class_name, prob) in predicted_top_5]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ljpoYk-0f6HS" - }, - "source": [ - "现在，加载没有分类部分的 `VGG19` ，并列出各层的名称：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Yh_AV6220ebD", - "colab": {} - }, - "source": [ - "vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\n", - "\n", - "print()\n", - "for layer in vgg.layers:\n", - " print(layer.name)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wt-tASys0eJv" - }, - "source": [ - "从网络中选择中间层的输出以表示图像的风格和内容：\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ArfX_6iA0WAX", - "colab": {} - }, - "source": [ - "# 内容层将提取出我们的 feature maps （特征图）\n", - "content_layers = ['block5_conv2'] \n", - "\n", - "# 我们感兴趣的风格层\n", - "style_layers = ['block1_conv1',\n", - " 'block2_conv1',\n", - " 'block3_conv1', \n", - " 'block4_conv1', \n", - " 'block5_conv1']\n", - "\n", - "num_content_layers = len(content_layers)\n", - "num_style_layers = len(style_layers)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2o4nSwuN0U3X" - }, - "source": [ - "#### 用于表示风格和内容的中间层\n", - "\n", - "那么,为什么我们预训练的图像分类网络中的这些中间层的输出允许我们定义风格和内容的表示？\n", - "\n", - "从高层理解，为了使网络能够实现图像分类（该网络已被训练过），它必须理解图像。 这需要将原始图像作为输入像素并构建内部表示，这个内部表示将原始图像像素转换为对图像中存在的 feature (特征)的复杂理解。\n", - "\n", - "这也是卷积神经网络能够很好地推广的一个原因：它们能够捕获不变性并定义类别（例如猫与狗）之间的 feature (特征)，这些 feature (特征)与背景噪声和其他干扰无关。 因此，将原始图像传递到模型输入和分类标签输出之间的某处的这一过程，可以视作复杂的 feature (特征)提取器。通过这些模型的中间层，我们就可以描述输入图像的内容和风格。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Jt3i3RRrJiOX" - }, - "source": [ - "## 建立模型 \n", - "\n", - "使用`tf.keras.applications`中的网络可以让我们非常方便的利用Keras的功能接口提取中间层的值。\n", - "\n", - "在使用功能接口定义模型时，我们需要指定输入和输出：\n", - "\n", - "`model = Model(inputs, outputs)`\n", - "\n", - "以下函数构建了一个 VGG19 模型，该模型返回一个中间层输出的列表：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nfec6MuMAbPx", - "colab": {} - }, - "source": [ - "def vgg_layers(layer_names):\n", - " \"\"\" Creates a vgg model that returns a list of intermediate output values.\"\"\"\n", - " # 加载我们的模型。 加载已经在 imagenet 数据上预训练的 VGG \n", - " vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')\n", - " vgg.trainable = False\n", - " \n", - " outputs = [vgg.get_layer(name).output for name in layer_names]\n", - "\n", - " model = tf.keras.Model([vgg.input], outputs)\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jbaIvZf5wWn_" - }, - "source": [ - "然后建立模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LkyvPpBHSfVi", - "colab": {} - }, - "source": [ - "style_extractor = vgg_layers(style_layers)\n", - "style_outputs = style_extractor(style_image*255)\n", - "\n", - "#查看每层输出的统计信息\n", - "for name, output in zip(style_layers, style_outputs):\n", - " print(name)\n", - " print(\" shape: \", output.numpy().shape)\n", - " print(\" min: \", output.numpy().min())\n", - " print(\" max: \", output.numpy().max())\n", - " print(\" mean: \", output.numpy().mean())\n", - " print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lGUfttK9F8d5" - }, - "source": [ - "## 风格计算\n", - "\n", - "图像的内容由中间 feature maps (特征图)的值表示。\n", - "\n", - "事实证明，图像的风格可以通过不同 feature maps (特征图)上的平均值和相关性来描述。 通过在每个位置计算 feature (特征)向量的外积，并在所有位置对该外积进行平均,可以计算出包含此信息的 Gram 矩阵。 对于特定层的 Gram 矩阵，具体计算方法如下所示：\n", - "\n", - "$$G^l_{cd} = \\frac{\\sum_{ij} F^l_{ijc}(x)F^l_{ijd}(x)}{IJ}$$\n", - "\n", - "这可以使用`tf.linalg.einsum`函数来实现：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HAy1iGPdoEpZ", - "colab": {} - }, - "source": [ - "def gram_matrix(input_tensor):\n", - " result = tf.linalg.einsum('bijc,bijd->bcd', input_tensor, input_tensor)\n", - " input_shape = tf.shape(input_tensor)\n", - " num_locations = tf.cast(input_shape[1]*input_shape[2], tf.float32)\n", - " return result/(num_locations)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pXIUX6czZABh" - }, - "source": [ - "## 提取风格和内容\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1HGHvwlJ1nkn" - }, - "source": [ - "构建一个返回风格和内容张量的模型。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Sr6QALY-I1ja", - "colab": {} - }, - "source": [ - "class StyleContentModel(tf.keras.models.Model):\n", - " def __init__(self, style_layers, content_layers):\n", - " super(StyleContentModel, self).__init__()\n", - " self.vgg = vgg_layers(style_layers + content_layers)\n", - " self.style_layers = style_layers\n", - " self.content_layers = content_layers\n", - " self.num_style_layers = len(style_layers)\n", - " self.vgg.trainable = False\n", - "\n", - " def call(self, inputs):\n", - " \"Expects float input in [0,1]\"\n", - " inputs = inputs*255.0\n", - " preprocessed_input = tf.keras.applications.vgg19.preprocess_input(inputs)\n", - " outputs = self.vgg(preprocessed_input)\n", - " style_outputs, content_outputs = (outputs[:self.num_style_layers], \n", - " outputs[self.num_style_layers:])\n", - "\n", - " style_outputs = [gram_matrix(style_output)\n", - " for style_output in style_outputs]\n", - "\n", - " content_dict = {content_name:value \n", - " for content_name, value \n", - " in zip(self.content_layers, content_outputs)}\n", - "\n", - " style_dict = {style_name:value\n", - " for style_name, value\n", - " in zip(self.style_layers, style_outputs)}\n", - " \n", - " return {'content':content_dict, 'style':style_dict}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xuj1o33t1edl" - }, - "source": [ - "在图像上调用此模型，可以返回 style_layers 的 gram 矩阵（风格）和 content_layers 的内容：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rkjO-DoNDU0A", - "colab": {} - }, - "source": [ - "extractor = StyleContentModel(style_layers, content_layers)\n", - "\n", - "results = extractor(tf.constant(content_image))\n", - "\n", - "style_results = results['style']\n", - "\n", - "print('Styles:')\n", - "for name, output in sorted(results['style'].items()):\n", - " print(\" \", name)\n", - " print(\" shape: \", output.numpy().shape)\n", - " print(\" min: \", output.numpy().min())\n", - " print(\" max: \", output.numpy().max())\n", - " print(\" mean: \", output.numpy().mean())\n", - " print()\n", - "\n", - "print(\"Contents:\")\n", - "for name, output in sorted(results['content'].items()):\n", - " print(\" \", name)\n", - " print(\" shape: \", output.numpy().shape)\n", - " print(\" min: \", output.numpy().min())\n", - " print(\" max: \", output.numpy().max())\n", - " print(\" mean: \", output.numpy().mean())\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y9r8Lyjb_m0u" - }, - "source": [ - "## 梯度下降\n", - "\n", - "使用此风格和内容提取器，我们现在可以实现风格传输算法。我们通过计算每个图像的输出和目标的均方误差来做到这一点，然后取这些损失值的加权和。\n", - "\n", - "设置风格和内容的目标值：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "PgkNOnGUFcKa", - "colab": {} - }, - "source": [ - "style_targets = extractor(style_image)['style']\n", - "content_targets = extractor(content_image)['content']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CNPrpl-e_w9A" - }, - "source": [ - "定义一个 `tf.Variable` 来表示要优化的图像。 为了快速实现这一点，使用内容图像对其进行初始化（ `tf.Variable` 必须与内容图像的形状相同）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J0vKxF8ZO6G8", - "colab": {} - }, - "source": [ - "image = tf.Variable(content_image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M6L8ojmn_6rH" - }, - "source": [ - "由于这是一个浮点图像，因此我们定义一个函数来保持像素值在 0 和 1 之间：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kdgpTJwL_vE2", - "colab": {} - }, - "source": [ - "def clip_0_1(image):\n", - " return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MBU5RFpcAo7W" - }, - "source": [ - "创建一个 optimizer 。 本教程推荐 LBFGS，但 `Adam` 也可以正常工作：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "r4XZjqUk_5Eu", - "colab": {} - }, - "source": [ - "opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "As-evbBiA2qT" - }, - "source": [ - "为了优化它，我们使用两个损失的加权组合来获得总损失：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Dt4pxarvA4I4", - "colab": {} - }, - "source": [ - "style_weight=1e-2\n", - "content_weight=1e4" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0ggx2Na8oROH", - "colab": {} - }, - "source": [ - "def style_content_loss(outputs):\n", - " style_outputs = outputs['style']\n", - " content_outputs = outputs['content']\n", - " style_loss = tf.add_n([tf.reduce_mean((style_outputs[name]-style_targets[name])**2) \n", - " for name in style_outputs.keys()])\n", - " style_loss *= style_weight / num_style_layers\n", - "\n", - " content_loss = tf.add_n([tf.reduce_mean((content_outputs[name]-content_targets[name])**2) \n", - " for name in content_outputs.keys()])\n", - " content_loss *= content_weight / num_content_layers\n", - " loss = style_loss + content_loss\n", - " return loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vbF2WnP9BI5M" - }, - "source": [ - "使用 `tf.GradientTape` 来更新图像。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0t0umkajFIuh", - "colab": {} - }, - "source": [ - "@tf.function()\n", - "def train_step(image):\n", - " with tf.GradientTape() as tape:\n", - " outputs = extractor(image)\n", - " loss = style_content_loss(outputs)\n", - "\n", - " grad = tape.gradient(loss, image)\n", - " opt.apply_gradients([(grad, image)])\n", - " image.assign(clip_0_1(image))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5FHMJq4UBRIQ" - }, - "source": [ - "现在，我们运行几个步来测试一下：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y542mxi-O2a2", - "colab": {} - }, - "source": [ - "train_step(image)\n", - "train_step(image)\n", - "train_step(image)\n", - "plt.imshow(image.read_value()[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mNzE-mTbBVgY" - }, - "source": [ - "运行正常，我们来执行一个更长的优化：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rQW1tXYoLbUS", - "colab": {} - }, - "source": [ - "import time\n", - "start = time.time()\n", - "\n", - "epochs = 10\n", - "steps_per_epoch = 100\n", - "\n", - "step = 0\n", - "for n in range(epochs):\n", - " for m in range(steps_per_epoch):\n", - " step += 1\n", - " train_step(image)\n", - " print(\".\", end='')\n", - " display.clear_output(wait=True)\n", - " imshow(image.read_value())\n", - " plt.title(\"Train step: {}\".format(step))\n", - " plt.show()\n", - "\n", - "end = time.time()\n", - "print(\"Total time: {:.1f}\".format(end-start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWVB3anJMY2v" - }, - "source": [ - "## 总变分损失\n", - "\n", - "此实现只是一个基础版本，它的一个缺点是它会产生大量的高频误差。 我们可以直接通过正则化图像的高频分量来减少这些高频误差。 在风格转移中，这通常被称为*总变分损失*：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7szUUybCQMB3", - "colab": {} - }, - "source": [ - "def high_pass_x_y(image):\n", - " x_var = image[:,:,1:,:] - image[:,:,:-1,:]\n", - " y_var = image[:,1:,:,:] - image[:,:-1,:,:]\n", - "\n", - " return x_var, y_var" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Atc2oL29PXu_", - "colab": {} - }, - "source": [ - "x_deltas, y_deltas = high_pass_x_y(content_image)\n", - "\n", - "plt.figure(figsize=(14,10))\n", - "plt.subplot(2,2,1)\n", - "imshow(clip_0_1(2*y_deltas+0.5), \"Horizontal Deltas: Original\")\n", - "\n", - "plt.subplot(2,2,2)\n", - "imshow(clip_0_1(2*x_deltas+0.5), \"Vertical Deltas: Original\")\n", - "\n", - "x_deltas, y_deltas = high_pass_x_y(image)\n", - "\n", - "plt.subplot(2,2,3)\n", - "imshow(clip_0_1(2*y_deltas+0.5), \"Horizontal Deltas: Styled\")\n", - "\n", - "plt.subplot(2,2,4)\n", - "imshow(clip_0_1(2*x_deltas+0.5), \"Vertical Deltas: Styled\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lqHElVgBkgkz" - }, - "source": [ - "这显示了高频分量如何增加。\n", - "\n", - "而且，本质上高频分量是一个边缘检测器。 我们可以从 Sobel 边缘检测器获得类似的输出，例如：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HyvqCiywiUfL", - "colab": {} - }, - "source": [ - "plt.figure(figsize=(14,10))\n", - "\n", - "sobel = tf.image.sobel_edges(content_image)\n", - "plt.subplot(1,2,1)\n", - "imshow(clip_0_1(sobel[...,0]/4+0.5), \"Horizontal Sobel-edges\")\n", - "plt.subplot(1,2,2)\n", - "imshow(clip_0_1(sobel[...,1]/4+0.5), \"Vertical Sobel-edges\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vv5bKlSDnPP7" - }, - "source": [ - "与此相关的正则化损失是这些值的平方和：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mP-92lXMIYPn", - "colab": {} - }, - "source": [ - "def total_variation_loss(image):\n", - " x_deltas, y_deltas = high_pass_x_y(image)\n", - " return tf.reduce_mean(x_deltas**2) + tf.reduce_mean(y_deltas**2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nTessd-DCdcC" - }, - "source": [ - "## 重新进行优化\n", - "\n", - "选择 `total_variation_loss` 的权重：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tGeRLD4GoAd4", - "colab": {} - }, - "source": [ - "total_variation_weight=1e8" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kG1-T4kJsoAv" - }, - "source": [ - "现在，将它加入 `train_step` 函数中：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BzmfcyyYUyWq", - "colab": {} - }, - "source": [ - "@tf.function()\n", - "def train_step(image):\n", - " with tf.GradientTape() as tape:\n", - " outputs = extractor(image)\n", - " loss = style_content_loss(outputs)\n", - " loss += total_variation_weight*total_variation_loss(image)\n", - "\n", - " grad = tape.gradient(loss, image)\n", - " opt.apply_gradients([(grad, image)])\n", - " image.assign(clip_0_1(image))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lcLWBQChsutQ" - }, - "source": [ - "重新初始化优化的变量：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "a-dPRr8BqexB", - "colab": {} - }, - "source": [ - "image = tf.Variable(content_image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BEflRstmtGBu" - }, - "source": [ - "并进行优化：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q3Cc3bLtoOWy", - "colab": {} - }, - "source": [ - "import time\n", - "start = time.time()\n", - "\n", - "epochs = 10\n", - "steps_per_epoch = 100\n", - "\n", - "step = 0\n", - "for n in range(epochs):\n", - " for m in range(steps_per_epoch):\n", - " step += 1\n", - " train_step(image)\n", - " print(\".\", end='')\n", - " display.clear_output(wait=True)\n", - " imshow(image.read_value())\n", - " plt.title(\"Train step: {}\".format(step))\n", - " plt.show()\n", - "\n", - "end = time.time()\n", - "print(\"Total time: {:.1f}\".format(end-start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KKox7K46tKxy" - }, - "source": [ - "最后，保存结果：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SSH6OpyyQn7w", - "colab": {} - }, - "source": [ - "file_name = 'kadinsky-turtle.png'\n", - "mpl.image.imsave(file_name, image[0])\n", - "\n", - "try:\n", - " from google.colab import files\n", - "except ImportError:\n", - " pass\n", - "else:\n", - " files.download(file_name)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/images/segmentation.ipynb b/site/zh-cn/tutorials/images/segmentation.ipynb deleted file mode 100644 index f67660f6206..00000000000 --- a/site/zh-cn/tutorials/images/segmentation.ipynb +++ /dev/null @@ -1,708 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "segmentation.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cZCM65CBt1CJ" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\");" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "JOgMcEajtkmg", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rCSP-dbMw88x" - }, - "source": [ - "# 图像分割" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NEWs8JXRuGex" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " 在 tensorFlow.google.cn 上查看\n", - " \n", - " \n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " \n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sMP7mglMuGT2" - }, - "source": [ - "这篇教程将重点讨论图像分割任务，使用的是改进版的 [U-Net](https://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/)。\n", - "\n", - "## 什么是图像分割？\n", - "目前你已经了解在图像分类中，神经网络的任务是给每张输入图像分配一个标签或者类别。但是，有时你想知道一个物体在一张图像中的位置、这个物体的形状、以及哪个像素属于哪个物体等等。这种情况下你会希望分割图像，也就是给图像中的每个像素各分配一个标签。因此，图像分割的任务是训练一个神经网络来输出该图像对每一个像素的掩码。这对从更底层，即像素层级，来理解图像很有帮助。图像分割在例如医疗图像、自动驾驶车辆以及卫星图像等领域有很多应用。\n", - "\n", - "本教程将使用的数据集是 [Oxford-IIIT Pet 数据集](https://www.robots.ox.ac.uk/~vgg/data/pets/)，由 Parkhi *et al.* 创建。该数据集由图像、图像所对应的标签、以及对像素逐一标记的掩码组成。掩码其实就是给每个像素的标签。每个像素分别属于以下三个类别中的一个：\n", - "* 类别 1：像素是宠物的一部分。\n", - "* 类别 2：像素是宠物的轮廓。\n", - "* 类别 3：以上都不是/外围像素。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MQmKthrSBCld", - "colab": {} - }, - "source": [ - "!pip install git+https://github.com/tensorflow/examples.git" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YQX7R4bhZy5h", - "colab": {} - }, - "source": [ - "try:\n", - " # %tensorflow_version 只存在于 Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "g87--n2AtyO_", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "from tensorflow_examples.models.pix2pix import pix2pix\n", - "\n", - "import tensorflow_datasets as tfds\n", - "tfds.disable_progress_bar()\n", - "\n", - "from IPython.display import clear_output\n", - "import matplotlib.pyplot as plt" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oWe0_rQM4JbC" - }, - "source": [ - "## 下载 Oxford-IIIT Pets 数据集\n", - "\n", - "这个数据集已经集成在 Tensorflow datasets 中，只需下载即可。图像分割掩码在版本 3.0.0 中才被加入，因此我们特别选用这个版本。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "40ITeStwDwZb", - "colab": {} - }, - "source": [ - "dataset, info = tfds.load('oxford_iiit_pet:3.0.0', with_info=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rJcVdj_U4vzf" - }, - "source": [ - "下面的代码进行了一个简单的图像翻转扩充。然后，将图像标准化到 [0,1]。最后，如上文提到的，像素点在图像分割掩码中被标记为 {1, 2, 3} 中的一个。为了方便起见，我们将分割掩码都减 1，得到了以下的标签：{0, 1, 2}。 " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FD60EbcAQqov", - "colab": {} - }, - "source": [ - "def normalize(input_image, input_mask):\n", - " input_image = tf.cast(input_image, tf.float32)/255.0\n", - " input_mask -= 1\n", - " return input_image, input_mask" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2NPlCnBXQwb1", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def load_image_train(datapoint):\n", - " input_image = tf.image.resize(datapoint['image'], (128, 128))\n", - " input_mask = tf.image.resize(datapoint['segmentation_mask'], (128, 128))\n", - "\n", - " if tf.random.uniform(()) > 0.5:\n", - " input_image = tf.image.flip_left_right(input_image)\n", - " input_mask = tf.image.flip_left_right(input_mask)\n", - "\n", - " input_image, input_mask = normalize(input_image, input_mask)\n", - "\n", - " return input_image, input_mask" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Zf0S67hJRp3D", - "colab": {} - }, - "source": [ - "def load_image_test(datapoint):\n", - " input_image = tf.image.resize(datapoint['image'], (128, 128))\n", - " input_mask = tf.image.resize(datapoint['segmentation_mask'], (128, 128))\n", - "\n", - " input_image, input_mask = normalize(input_image, input_mask)\n", - "\n", - " return input_image, input_mask" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "65-qHTjX5VZh" - }, - "source": [ - "数据集已经包含了所需的测试集和训练集划分，所以我们也延续使用相同的划分。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yHwj2-8SaQli", - "colab": {} - }, - "source": [ - "TRAIN_LENGTH = info.splits['train'].num_examples\n", - "BATCH_SIZE = 64\n", - "BUFFER_SIZE = 1000\n", - "STEPS_PER_EPOCH = TRAIN_LENGTH // BATCH_SIZE" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "39fYScNz9lmo", - "colab": {} - }, - "source": [ - "train = dataset['train'].map(load_image_train, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n", - "test = dataset['test'].map(load_image_test)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DeFwFDN6EVoI", - "colab": {} - }, - "source": [ - "train_dataset = train.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()\n", - "train_dataset = train_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n", - "test_dataset = test.batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xa3gMAE_9qNa" - }, - "source": [ - "我们来看一下数据集中的一例图像以及它所对应的掩码。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3N2RPAAW9q4W", - "colab": {} - }, - "source": [ - "def display(display_list):\n", - " plt.figure(figsize=(15, 15))\n", - "\n", - " title = ['Input Image', 'True Mask', 'Predicted Mask']\n", - "\n", - " for i in range(len(display_list)):\n", - " plt.subplot(1, len(display_list), i+1)\n", - " plt.title(title[i])\n", - " plt.imshow(tf.keras.preprocessing.image.array_to_img(display_list[i]))\n", - " plt.axis('off')\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "a6u_Rblkteqb", - "colab": {} - }, - "source": [ - "for image, mask in train.take(1):\n", - " sample_image, sample_mask = image, mask\n", - "display([sample_image, sample_mask])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FAOe93FRMk3w" - }, - "source": [ - "## 定义模型\n", - "这里用到的模型是一个改版的 U-Net。U-Net 由一个编码器（下采样器（downsampler））和一个解码器（上采样器（upsampler））组成。为了学习到鲁棒的特征，同时减少可训练参数的数量，这里可以使用一个预训练模型作为编码器。因此，这项任务中的编码器将使用一个预训练的 MobileNetV2 模型，它的中间输出值将被使用。解码器将使用在 TensorFlow Examples 中的 [Pix2pix tutorial](https://github.com/tensorflow/examples/blob/master/tensorflow_examples/models/pix2pix/pix2pix.py) 里实施过的升频取样模块。 \n", - "\n", - "输出信道数量为 3 是因为每个像素有三种可能的标签。把这想象成一个多类别分类，每个像素都将被分到三个类别当中。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c6iB4iMvMkX9", - "colab": {} - }, - "source": [ - "OUTPUT_CHANNELS = 3" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "W4mQle3lthit" - }, - "source": [ - "如之前提到的，编码器是一个预训练的 MobileNetV2 模型，它在 [tf.keras.applications](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/applications) 中已被准备好并可以直接使用。编码器中包含模型中间层的一些特定输出。注意编码器在模型的训练过程中是不会被训练的。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "liCeLH0ctjq7", - "colab": {} - }, - "source": [ - "base_model = tf.keras.applications.MobileNetV2(input_shape=[128, 128, 3], include_top=False)\n", - "\n", - "# 使用这些层的激活设置\n", - "layer_names = [\n", - " 'block_1_expand_relu', # 64x64\n", - " 'block_3_expand_relu', # 32x32\n", - " 'block_6_expand_relu', # 16x16\n", - " 'block_13_expand_relu', # 8x8\n", - " 'block_16_project', # 4x4\n", - "]\n", - "layers = [base_model.get_layer(name).output for name in layer_names]\n", - "\n", - "# 创建特征提取模型\n", - "down_stack = tf.keras.Model(inputs=base_model.input, outputs=layers)\n", - "\n", - "down_stack.trainable = False" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KPw8Lzra5_T9" - }, - "source": [ - "解码器/升频取样器是简单的一系列升频取样模块，在 TensorFlow examples 中曾被实施过。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p0ZbfywEbZpJ", - "colab": {} - }, - "source": [ - "up_stack = [\n", - " pix2pix.upsample(512, 3), # 4x4 -> 8x8\n", - " pix2pix.upsample(256, 3), # 8x8 -> 16x16\n", - " pix2pix.upsample(128, 3), # 16x16 -> 32x32\n", - " pix2pix.upsample(64, 3), # 32x32 -> 64x64\n", - "]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "45HByxpVtrPF", - "colab": {} - }, - "source": [ - "def unet_model(output_channels):\n", - "\n", - " # 这是模型的最后一层\n", - " last = tf.keras.layers.Conv2DTranspose(\n", - " output_channels, 3, strides=2,\n", - " padding='same', activation='softmax') #64x64 -> 128x128\n", - "\n", - " inputs = tf.keras.layers.Input(shape=[128, 128, 3])\n", - " x = inputs\n", - "\n", - " # 在模型中降频取样\n", - " skips = down_stack(x)\n", - " x = skips[-1]\n", - " skips = reversed(skips[:-1])\n", - "\n", - " # 升频取样然后建立跳跃连接\n", - " for up, skip in zip(up_stack, skips):\n", - " x = up(x)\n", - " concat = tf.keras.layers.Concatenate()\n", - " x = concat([x, skip])\n", - "\n", - " x = last(x)\n", - "\n", - " return tf.keras.Model(inputs=inputs, outputs=x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "j0DGH_4T0VYn" - }, - "source": [ - "## 训练模型\n", - "现在，要做的只剩下编译和训练模型了。这里用到的损失函数是 losses.sparse_categorical_crossentropy。使用这个损失函数是因为神经网络试图给每一个像素分配一个标签，和多类别预测是一样的。在正确的分割掩码中，每个像素点的值是 {0,1,2} 中的一个。同时神经网络也输出三个信道。本质上，每个信道都在尝试学习预测一个类别，而 losses.sparse_categorical_crossentropy 正是这一情形下推荐使用的损失函数。根据神经网络的输出值，分配给每个像素的标签为输出值最高的信道所表示的那一类。这就是 create_mask 函数所做的工作。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6he36HK5uKAc", - "colab": {} - }, - "source": [ - "model = unet_model(OUTPUT_CHANNELS)\n", - "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tc3MiEO2twLS" - }, - "source": [ - "我们试着运行一下模型，看看它在训练之前给出的预测值。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UwvIKLZPtxV_", - "colab": {} - }, - "source": [ - "def create_mask(pred_mask):\n", - " pred_mask = tf.argmax(pred_mask, axis=-1)\n", - " pred_mask = pred_mask[..., tf.newaxis]\n", - " return pred_mask[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YLNsrynNtx4d", - "colab": {} - }, - "source": [ - "def show_predictions(dataset=None, num=1):\n", - " if dataset:\n", - " for image, mask in dataset.take(num):\n", - " pred_mask = model.predict(image)\n", - " display([image[0], mask[0], create_mask(pred_mask)])\n", - " else:\n", - " display([sample_image, sample_mask,\n", - " create_mask(model.predict(sample_image[tf.newaxis, ...]))])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X_1CC0T4dho3", - "colab": {} - }, - "source": [ - "show_predictions()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "22AyVYWQdkgk" - }, - "source": [ - "我们来观察模型是怎样随着训练而改善的。为达成这一目的，下面将定义一个 callback 函数。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wHrHsqijdmL6", - "colab": {} - }, - "source": [ - "class DisplayCallback(tf.keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs=None):\n", - " clear_output(wait=True)\n", - " show_predictions()\n", - " print ('\\nSample Prediction after epoch {}\\n'.format(epoch+1))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "StKDH_B9t4SD", - "colab": {} - }, - "source": [ - "EPOCHS = 20\n", - "VAL_SUBSPLITS = 5\n", - "VALIDATION_STEPS = info.splits['test'].num_examples//BATCH_SIZE//VAL_SUBSPLITS\n", - "\n", - "model_history = model.fit(train_dataset, epochs=EPOCHS,\n", - " steps_per_epoch=STEPS_PER_EPOCH,\n", - " validation_steps=VALIDATION_STEPS,\n", - " validation_data=test_dataset,\n", - " callbacks=[DisplayCallback()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "P_mu0SAbt40Q", - "colab": {} - }, - "source": [ - "loss = model_history.history['loss']\n", - "val_loss = model_history.history['val_loss']\n", - "\n", - "epochs = range(EPOCHS)\n", - "\n", - "plt.figure()\n", - "plt.plot(epochs, loss, 'r', label='Training loss')\n", - "plt.plot(epochs, val_loss, 'bo', label='Validation loss')\n", - "plt.title('Training and Validation Loss')\n", - "plt.xlabel('Epoch')\n", - "plt.ylabel('Loss Value')\n", - "plt.ylim([0, 1])\n", - "plt.legend()\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "unP3cnxo_N72" - }, - "source": [ - "## 做出预测" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7BVXldSo-0mW" - }, - "source": [ - "我们来做几个预测。为了节省时间，这里只使用很少的周期（epoch）数，但是你可以设置更多的数量以获得更准确的结果。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ikrzoG24qwf5", - "colab": {} - }, - "source": [ - "show_predictions(test_dataset, 3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R24tahEqmSCk" - }, - "source": [ - "## 接下来\n", - "现在你已经对图像分割是什么以及它的工作原理有所了解。你可以在本教程里尝试使用不同的中间层输出值，或者甚至使用不同的预训练模型。你也可以去 Kaggle 举办的 [Carvana](https://www.kaggle.com/c/carvana-image-masking-challenge/overview) 图像分割挑战赛上挑战自己。\n", - "\n", - "你也可以看看 [Tensorflow Object Detection API](https://github.com/tensorflow/models/tree/master/research/object_detection) 上面其他的你可以使用自己数据进行再训练的模型。" - ] - } - ] -} diff --git a/site/zh-cn/tutorials/keras/regression.ipynb b/site/zh-cn/tutorials/keras/regression.ipynb deleted file mode 100644 index c87e5f661a5..00000000000 --- a/site/zh-cn/tutorials/keras/regression.ipynb +++ /dev/null @@ -1,867 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "regression.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FhGuhbZ6M5tl" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AwOEIRJC6Une", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "KyPEtTqk6VdG", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EIdT9iu_Z4Rb" - }, - "source": [ - "# Basic regression: Predict fuel efficiency" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBIlTPscrIT9" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorFlow.google.cn 上查看\n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AHp3M9ZmrIxj" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。\n", - "\n", - "在 *回归 (regression)* 问题中，我们的目的是预测出如价格或概率这样连续值的输出。相对于*分类(classification)* 问题，*分类(classification)* 的目的是从一系列的分类出选择出一个分类 （如，给出一张包含苹果或橘子的图片，识别出图片中是哪种水果）。\n", - "\n", - "本 notebook 使用经典的 [Auto MPG](https://archive.ics.uci.edu/ml/datasets/auto+mpg) 数据集，构建了一个用来预测70年代末到80年代初汽车燃油效率的模型。为了做到这一点，我们将为该模型提供许多那个时期的汽车描述。这个描述包含：气缸数，排量，马力以及重量。\n", - "\n", - "本示例使用 `tf.keras` API，相关细节请参阅 [本指南](https://tensorflow.google.cn/guide/keras)。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "moB4tpEHxKB3", - "colab": {} - }, - "source": [ - "# 使用 seaborn 绘制矩阵图 (pairplot)\n", - "!pip install seaborn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1rRo8oNqZ-Rj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import pathlib\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "import seaborn as sns\n", - "\n", - "try:\n", - " # %tensorflow_version only exists in Colab.\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F_72b0LCNbjx" - }, - "source": [ - "## Auto MPG 数据集\n", - "\n", - "该数据集可以从 [UCI机器学习库](https://archive.ics.uci.edu/ml/) 中获取.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gFh9ne3FZ-On" - }, - "source": [ - "### 获取数据\n", - "首先下载数据集。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p9kxxgzvzlyz", - "colab": {} - }, - "source": [ - "dataset_path = keras.utils.get_file(\"auto-mpg.data\", \"http://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data\")\n", - "dataset_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nslsRLh7Zss4" - }, - "source": [ - "使用 pandas 导入数据集。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CiX2FI4gZtTt", - "colab": {} - }, - "source": [ - "column_names = ['MPG','Cylinders','Displacement','Horsepower','Weight',\n", - " 'Acceleration', 'Model Year', 'Origin']\n", - "raw_dataset = pd.read_csv(dataset_path, names=column_names,\n", - " na_values = \"?\", comment='\\t',\n", - " sep=\" \", skipinitialspace=True)\n", - "\n", - "dataset = raw_dataset.copy()\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3MWuJTKEDM-f" - }, - "source": [ - "### 数据清洗\n", - "\n", - "数据集中包括一些未知值。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JEJHhN65a2VV", - "colab": {} - }, - "source": [ - "dataset.isna().sum()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9UPN0KBHa_WI" - }, - "source": [ - "为了保证这个初始示例的简单性，删除这些行。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ZUDosChC1UN", - "colab": {} - }, - "source": [ - "dataset = dataset.dropna()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8XKitwaH4v8h" - }, - "source": [ - "`\"Origin\"` 列实际上代表分类，而不仅仅是一个数字。所以把它转换为独热码 （one-hot）:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gWNTD2QjBWFJ", - "colab": {} - }, - "source": [ - "origin = dataset.pop('Origin')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ulXz4J7PAUzk", - "colab": {} - }, - "source": [ - "dataset['USA'] = (origin == 1)*1.0\n", - "dataset['Europe'] = (origin == 2)*1.0\n", - "dataset['Japan'] = (origin == 3)*1.0\n", - "dataset.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Cuym4yvk76vU" - }, - "source": [ - "### 拆分训练数据集和测试数据集\n", - "\n", - "现在需要将数据集拆分为一个训练数据集和一个测试数据集。\n", - "\n", - "我们最后将使用测试数据集对模型进行评估。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qn-IGhUE7_1H", - "colab": {} - }, - "source": [ - "train_dataset = dataset.sample(frac=0.8,random_state=0)\n", - "test_dataset = dataset.drop(train_dataset.index)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J4ubs136WLNp" - }, - "source": [ - "### 数据检查\n", - "\n", - "快速查看训练集中几对列的联合分布。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oRKO_x8gWKv-", - "colab": {} - }, - "source": [ - "sns.pairplot(train_dataset[[\"MPG\", \"Cylinders\", \"Displacement\", \"Weight\"]], diag_kind=\"kde\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gavKO_6DWRMP" - }, - "source": [ - "也可以查看总体的数据统计:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yi2FzC3T21jR", - "colab": {} - }, - "source": [ - "train_stats = train_dataset.describe()\n", - "train_stats.pop(\"MPG\")\n", - "train_stats = train_stats.transpose()\n", - "train_stats" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Db7Auq1yXUvh" - }, - "source": [ - "### 从标签中分离特征\n", - "\n", - "将特征值从目标值或者\"标签\"中分离。 这个标签是你使用训练模型进行预测的值。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t2sluJdCW7jN", - "colab": {} - }, - "source": [ - "train_labels = train_dataset.pop('MPG')\n", - "test_labels = test_dataset.pop('MPG')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRklxK5s388r" - }, - "source": [ - "### 数据规范化\n", - "\n", - "再次审视下上面的 `train_stats` 部分，并注意每个特征的范围有什么不同。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ywmerQ6dSox" - }, - "source": [ - "使用不同的尺度和范围对特征归一化是好的实践。尽管模型*可能* 在没有特征归一化的情况下收敛，它会使得模型训练更加复杂，并会造成生成的模型依赖输入所使用的单位选择。\n", - "\n", - "注意：尽管我们仅仅从训练集中有意生成这些统计数据，但是这些统计信息也会用于归一化的测试数据集。我们需要这样做，将测试数据集放入到与已经训练过的模型相同的分布中。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JlC5ooJrgjQF", - "colab": {} - }, - "source": [ - "def norm(x):\n", - " return (x - train_stats['mean']) / train_stats['std']\n", - "normed_train_data = norm(train_dataset)\n", - "normed_test_data = norm(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BuiClDk45eS4" - }, - "source": [ - "我们将会使用这个已经归一化的数据来训练模型。\n", - "\n", - "警告: 用于归一化输入的数据统计（均值和标准差）需要反馈给模型从而应用于任何其他数据，以及我们之前所获得独热码。这些数据包含测试数据集以及生产环境中所使用的实时数据。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SmjdzxKzEu1-" - }, - "source": [ - "## 模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6SWtkIjhrZwa" - }, - "source": [ - "### 构建模型\n", - "\n", - "让我们来构建我们自己的模型。这里，我们将会使用一个“顺序”模型，其中包含两个紧密相连的隐藏层，以及返回单个、连续值得输出层。模型的构建步骤包含于一个名叫 'build_model' 的函数中，稍后我们将会创建第二个模型。 两个密集连接的隐藏层。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c26juK7ZG8j-", - "colab": {} - }, - "source": [ - "def build_model():\n", - " model = keras.Sequential([\n", - " layers.Dense(64, activation='relu', input_shape=[len(train_dataset.keys())]),\n", - " layers.Dense(64, activation='relu'),\n", - " layers.Dense(1)\n", - " ])\n", - "\n", - " optimizer = tf.keras.optimizers.RMSprop(0.001)\n", - "\n", - " model.compile(loss='mse',\n", - " optimizer=optimizer,\n", - " metrics=['mae', 'mse'])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cGbPb-PHGbhs", - "colab": {} - }, - "source": [ - "model = build_model()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Sj49Og4YGULr" - }, - "source": [ - "### 检查模型\n", - "\n", - "使用 `.summary` 方法来打印该模型的简单描述。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ReAD0n6MsFK-", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Vt6W50qGsJAL" - }, - "source": [ - "\n", - "现在试用下这个模型。从训练数据中批量获取‘10’条例子并对这些例子调用 `model.predict` 。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-d-gBaVtGTSC", - "colab": {} - }, - "source": [ - "example_batch = normed_train_data[:10]\n", - "example_result = model.predict(example_batch)\n", - "example_result" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QlM8KrSOsaYo" - }, - "source": [ - "它似乎在工作，并产生了预期的形状和类型的结果" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0-qWCsh6DlyH" - }, - "source": [ - "### 训练模型\n", - "\n", - "对模型进行1000个周期的训练，并在 `history` 对象中记录训练和验证的准确性。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sD7qHCmNIOY0", - "colab": {} - }, - "source": [ - "# 通过为每个完成的时期打印一个点来显示训练进度\n", - "class PrintDot(keras.callbacks.Callback):\n", - " def on_epoch_end(self, epoch, logs):\n", - " if epoch % 100 == 0: print('')\n", - " print('.', end='')\n", - "\n", - "EPOCHS = 1000\n", - "\n", - "history = model.fit(\n", - " normed_train_data, train_labels,\n", - " epochs=EPOCHS, validation_split = 0.2, verbose=0,\n", - " callbacks=[PrintDot()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tQm3pc0FYPQB" - }, - "source": [ - "使用 `history` 对象中存储的统计信息可视化模型的训练进度。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4Xj91b-dymEy", - "colab": {} - }, - "source": [ - "hist = pd.DataFrame(history.history)\n", - "hist['epoch'] = history.epoch\n", - "hist.tail()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "B6XriGbVPh2t", - "colab": {} - }, - "source": [ - "def plot_history(history):\n", - " hist = pd.DataFrame(history.history)\n", - " hist['epoch'] = history.epoch\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Abs Error [MPG]')\n", - " plt.plot(hist['epoch'], hist['mae'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mae'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,5])\n", - " plt.legend()\n", - "\n", - " plt.figure()\n", - " plt.xlabel('Epoch')\n", - " plt.ylabel('Mean Square Error [$MPG^2$]')\n", - " plt.plot(hist['epoch'], hist['mse'],\n", - " label='Train Error')\n", - " plt.plot(hist['epoch'], hist['val_mse'],\n", - " label = 'Val Error')\n", - " plt.ylim([0,20])\n", - " plt.legend()\n", - " plt.show()\n", - "\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AqsuANc11FYv" - }, - "source": [ - "该图表显示在约100个 epochs 之后误差非但没有改进，反而出现恶化。 让我们更新 `model.fit` 调用，当验证值没有提高上是自动停止训练。\n", - "我们将使用一个 *EarlyStopping callback* 来测试每个 epoch 的训练条件。如果经过一定数量的 epochs 后没有改进，则自动停止训练。\n", - "\n", - "你可以从[这里](https://tensorflow.google.cn/versions/master/api_docs/python/tf/keras/callbacks/EarlyStopping)学习到更多的回调。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "fdMZuhUgzMZ4", - "colab": {} - }, - "source": [ - "model = build_model()\n", - "\n", - "# patience 值用来检查改进 epochs 的数量\n", - "early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=10)\n", - "\n", - "history = model.fit(normed_train_data, train_labels, epochs=EPOCHS,\n", - " validation_split = 0.2, verbose=0, callbacks=[early_stop, PrintDot()])\n", - "\n", - "plot_history(history)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3St8-DmrX8P4" - }, - "source": [ - "如图所示，验证集中的平均的误差通常在 +/- 2 MPG左右。 这个结果好么？ 我们将决定权留给你。\n", - "\n", - "让我们看看通过使用 **测试集** 来泛化模型的效果如何，我们在训练模型时没有使用测试集。这告诉我们，当我们在现实世界中使用这个模型时，我们可以期望它预测得有多好。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jl_yNr5n1kms", - "colab": {} - }, - "source": [ - "loss, mae, mse = model.evaluate(normed_test_data, test_labels, verbose=2)\n", - "\n", - "print(\"Testing set Mean Abs Error: {:5.2f} MPG\".format(mae))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ft603OzXuEZC" - }, - "source": [ - "### 做预测\n", - " \n", - "最后，使用测试集中的数据预测 MPG 值:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Xe7RXH3N3CWU", - "colab": {} - }, - "source": [ - "test_predictions = model.predict(normed_test_data).flatten()\n", - "\n", - "plt.scatter(test_labels, test_predictions)\n", - "plt.xlabel('True Values [MPG]')\n", - "plt.ylabel('Predictions [MPG]')\n", - "plt.axis('equal')\n", - "plt.axis('square')\n", - "plt.xlim([0,plt.xlim()[1]])\n", - "plt.ylim([0,plt.ylim()[1]])\n", - "_ = plt.plot([-100, 100], [-100, 100])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "19wyogbOSU5t" - }, - "source": [ - "这看起来我们的模型预测得相当好。我们来看下误差分布。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f-OHX4DiXd8x", - "colab": {} - }, - "source": [ - "error = test_predictions - test_labels\n", - "plt.hist(error, bins = 25)\n", - "plt.xlabel(\"Prediction Error [MPG]\")\n", - "_ = plt.ylabel(\"Count\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m0CB5tBjSU5w" - }, - "source": [ - "它不是完全的高斯分布，但我们可以推断出，这是因为样本的数量很小所导致的。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vgGQuV-yqYZH" - }, - "source": [ - "## 结论\n", - "\n", - "本笔记本 (notebook) 介绍了一些处理回归问题的技术。\n", - "\n", - "* 均方误差（MSE）是用于回归问题的常见损失函数（分类问题中使用不同的损失函数）。\n", - "* 类似的，用于回归的评估指标与分类不同。 常见的回归指标是平均绝对误差（MAE）。\n", - "* 当数字输入数据特征的值存在不同范围时，每个特征应独立缩放到相同范围。\n", - "* 如果训练数据不多，一种方法是选择隐藏层较少的小网络，以避免过度拟合。\n", - "* 早期停止是一种防止过度拟合的有效技术。" - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/keras/save_and_load.ipynb b/site/zh-cn/tutorials/keras/save_and_load.ipynb deleted file mode 100644 index f47c18155c4..00000000000 --- a/site/zh-cn/tutorials/keras/save_and_load.ipynb +++ /dev/null @@ -1,896 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "save_and_load.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "g_nWetWWd_ns" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "2pHVBk_seED1", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "N_fMsQ-N8I7j", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pZJ3uY9O17VN" - }, - "source": [ - "# 保存和恢复模型 " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M4Ata7_wMul1" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GEe3i16tQPjo", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mBdde4YJeJKF" - }, - "source": [ - "模型可以在训练期间和训练完成后进行保存。这意味着模型可以从任意中断中恢复，并避免耗费比较长的时间在训练上。保存也意味着您可以共享您的模型，而其他人可以通过您的模型来重新创建工作。在发布研究模型和技术时，大多数机器学习从业者分享：\n", - "\n", - "* 用于创建模型的代码\n", - "* 模型训练的权重 (weight) 和参数 (parameters) 。\n", - "\n", - "共享数据有助于其他人了解模型的工作原理，并使用新数据自行尝试。\n", - "\n", - "注意：小心不受信任的代码——Tensorflow 模型是代码。有关详细信息，请参阅 [安全使用Tensorflow](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md)。\n", - "\n", - "### 选项\n", - "\n", - "保存 Tensorflow 的模型有许多方法——具体取决于您使用的 API。本指南使用 [tf.keras](https://tensorflow.google.cn/guide/keras)， 一个高级 API 用于在 Tensorflow 中构建和训练模型。有关其他方法的实现，请参阅 TensorFlow [保存和恢复](https://tensorflow.google.cn/guide/saved_model)指南或[保存到 eager](https://tensorflow.google.cn/guide/eager#object-based_saving)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xCUREq7WXgvg" - }, - "source": [ - "## 配置\n", - "\n", - "### 安装并导入" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7l0MiTOrXtNv" - }, - "source": [ - "安装并导入Tensorflow和依赖项：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "RzIOVSdnMYyO", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "!pip install pyyaml h5py # 需要以 HDF5 格式保存模型" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7Nm7Tyb-gRt-", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import os\n", - "\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "print(tf.version.VERSION)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbGsznErXWt6" - }, - "source": [ - "### 获取示例数据集\n", - "\n", - "要演示如何保存和加载权重，您将使用 [MNIST 数据集](http://yann.lecun.com/exdb/mnist/). 要加快运行速度，请使用前1000个示例：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9rGfFwE9XVwz", - "colab": {} - }, - "source": [ - "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n", - "\n", - "train_labels = train_labels[:1000]\n", - "test_labels = test_labels[:1000]\n", - "\n", - "train_images = train_images[:1000].reshape(-1, 28 * 28) / 255.0\n", - "test_images = test_images[:1000].reshape(-1, 28 * 28) / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "anG3iVoXyZGI" - }, - "source": [ - "### 定义模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wynsOBfby0Pa" - }, - "source": [ - "首先构建一个简单的序列（sequential）模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0HZbJIjxyX1S", - "colab": {} - }, - "source": [ - "# 定义一个简单的序列模型\n", - "def create_model():\n", - " model = tf.keras.models.Sequential([\n", - " keras.layers.Dense(512, activation='relu', input_shape=(784,)),\n", - " keras.layers.Dropout(0.2),\n", - " keras.layers.Dense(10, activation='softmax')\n", - " ])\n", - "\n", - " model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])\n", - "\n", - " return model\n", - "\n", - "# 创建一个基本的模型实例\n", - "model = create_model()\n", - "\n", - "# 显示模型的结构\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "soDE0W_KH8rG" - }, - "source": [ - "## 在训练期间保存模型（以 checkpoints 形式保存）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mRyd5qQQIXZm" - }, - "source": [ - "您可以使用训练好的模型而无需从头开始重新训练，或在您打断的地方开始训练，以防止训练过程没有保存。 `tf.keras.callbacks.ModelCheckpoint` 允许在训练的*过程中*和*结束时*回调保存的模型。\n", - "\n", - "### Checkpoint 回调用法\n", - "\n", - "创建一个只在训练期间保存权重的 `tf.keras.callbacks.ModelCheckpoint` 回调：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IFPuhwntH8VH", - "colab": {} - }, - "source": [ - "checkpoint_path = \"training_1/cp.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# 创建一个保存模型权重的回调\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n", - " save_weights_only=True,\n", - " verbose=1)\n", - "\n", - "# 使用新的回调训练模型\n", - "model.fit(train_images, \n", - " train_labels, \n", - " epochs=10,\n", - " validation_data=(test_images,test_labels),\n", - " callbacks=[cp_callback]) # 通过回调训练\n", - "\n", - "# 这可能会生成与保存优化程序状态相关的警告。\n", - "# 这些警告（以及整个笔记本中的类似警告）是防止过时使用，可以忽略。" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rlM-sgyJO084" - }, - "source": [ - "这将创建一个 TensorFlow checkpoint 文件集合，这些文件在每个 epoch 结束时更新：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gXG5FVKFOVQ3", - "colab": {} - }, - "source": [ - "!ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wlRN_f56Pqa9" - }, - "source": [ - "创建一个新的未经训练的模型。仅恢复模型的权重时，必须具有与原始模型具有相同网络结构的模型。由于模型具有相同的结构，您可以共享权重，尽管它是模型的不同*实例*。\n", - "现在重建一个新的未经训练的模型，并在测试集上进行评估。未经训练的模型将在机会水平（chance levels）上执行（准确度约为10％）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Fp5gbuiaPqCT", - "colab": {} - }, - "source": [ - "# 创建一个基本模型实例\n", - "model = create_model()\n", - "\n", - "# 评估模型\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Untrained model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1DTKpZssRSo3" - }, - "source": [ - "然后从 checkpoint 加载权重并重新评估：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2IZxbwiRRSD2", - "colab": {} - }, - "source": [ - "# 加载权重\n", - "model.load_weights(checkpoint_path)\n", - "\n", - "# 重新评估模型\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bpAbKkAyVPV8" - }, - "source": [ - "### checkpoint 回调选项\n", - "\n", - "回调提供了几个选项，为 checkpoint 提供唯一名称并调整 checkpoint 频率。\n", - "\n", - "训练一个新模型，每五个 epochs 保存一次唯一命名的 checkpoint ：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mQF_dlgIVOvq", - "colab": {} - }, - "source": [ - "# 在文件名中包含 epoch (使用 `str.format`)\n", - "checkpoint_path = \"training_2/cp-{epoch:04d}.ckpt\"\n", - "checkpoint_dir = os.path.dirname(checkpoint_path)\n", - "\n", - "# 创建一个回调，每 5 个 epochs 保存模型的权重\n", - "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n", - " filepath=checkpoint_path, \n", - " verbose=1, \n", - " save_weights_only=True,\n", - " period=5)\n", - "\n", - "# 创建一个新的模型实例\n", - "model = create_model()\n", - "\n", - "# 使用 `checkpoint_path` 格式保存权重\n", - "model.save_weights(checkpoint_path.format(epoch=0))\n", - "\n", - "# 使用新的回调*训练*模型\n", - "model.fit(train_images, \n", - " train_labels,\n", - " epochs=50, \n", - " callbacks=[cp_callback],\n", - " validation_data=(test_images,test_labels),\n", - " verbose=0)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1zFrKTjjavWI" - }, - "source": [ - "现在查看生成的 checkpoint 并选择最新的 checkpoint ：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p64q3-V4sXt0", - "colab": {} - }, - "source": [ - "! ls {checkpoint_dir}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1AN_fnuyR41H", - "colab": {} - }, - "source": [ - "latest = tf.train.latest_checkpoint(checkpoint_dir)\n", - "latest" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Zk2ciGbKg561" - }, - "source": [ - "注意: 默认的 tensorflow 格式仅保存最近的5个 checkpoint 。\n", - "\n", - "如果要进行测试，请重置模型并加载最新的 checkpoint ：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3M04jyK-H3QK", - "colab": {} - }, - "source": [ - "# 创建一个新的模型实例\n", - "model = create_model()\n", - "\n", - "# 加载以前保存的权重\n", - "model.load_weights(latest)\n", - "\n", - "# 重新评估模型\n", - "loss, acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c2OxsJOTHxia" - }, - "source": [ - "## 这些文件是什么？" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JtdYhvWnH2ib" - }, - "source": [ - "上述代码将权重存储到 [checkpoint](https://tensorflow.google.cn/guide/saved_model#save_and_restore_variables)—— 格式化文件的集合中，这些文件仅包含二进制格式的训练权重。 Checkpoints 包含：\n", - "* 一个或多个包含模型权重的分片。\n", - "* 索引文件，指示哪些权重存储在哪个分片中。\n", - "\n", - "如果你只在一台机器上训练一个模型，你将有一个带有后缀的碎片： `.data-00000-of-00001`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "S_FA-ZvxuXQV" - }, - "source": [ - "## 手动保存权重\n", - "\n", - "您将了解如何将权重加载到模型中。使用 `Model.save_weights` 方法手动保存它们同样简单。默认情况下， `tf.keras` 和 `save_weights` 特别使用 TensorFlow [checkpoints](../../guide/keras/checkpoints) 格式 `.ckpt` 扩展名和 ( 保存在 [HDF5](https://js.tensorflow.org/tutorials/import-keras.html) 扩展名为 `.h5` [保存并序列化模型](../../guide/keras/save_and_serialize#weights-only_saving_in_savedmodel_format) )：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "R7W5plyZ-u9X", - "colab": {} - }, - "source": [ - "# 保存权重\n", - "model.save_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# 创建模型实例\n", - "model = create_model()\n", - "\n", - "# Restore the weights\n", - "model.load_weights('./checkpoints/my_checkpoint')\n", - "\n", - "# Evaluate the model\n", - "loss,acc = model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kOGlxPRBEvV1" - }, - "source": [ - "## 保存整个模型\n", - "\n", - "模型和优化器可以保存到包含其状态（权重和变量）和模型参数的文件中。这可以让您导出模型，以便在不访问原始 python 代码的情况下使用它。而且您可以通过恢复优化器状态的方式，从中断的位置恢复训练。\n", - "\n", - "保存完整模型会非常有用——您可以在 TensorFlow.js ([HDF5](https://js.tensorflow.org/tutorials/import-keras.html), [Saved Model](https://js.tensorflow.org/tutorials/import-saved-model.html)) 加载他们，然后在 web 浏览器中训练和运行它们，或者使用 TensorFlow Lite 将它们转换为在移动设备上运行([HDF5](https://tensorflow.google.cn/lite/convert/python_api#exporting_a_tfkeras_file_), [Saved Model](https://tensorflow.google.cn/lite/convert/python_api#exporting_a_savedmodel_))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SkGwf-50zLNn" - }, - "source": [ - "### 将模型保存为HDF5文件\n", - "\n", - "Keras 可以使用 [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) 标准提供基本保存格式。出于我们的目的，可以将保存的模型视为单个二进制blob：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "m2dkmJVCGUia", - "colab": {} - }, - "source": [ - "# 创建一个新的模型实例\n", - "model = create_model()\n", - "\n", - "# 训练模型\n", - "model.fit(train_images, train_labels, epochs=5)\n", - "\n", - "# 将整个模型保存为HDF5文件\n", - "model.save('my_model.h5')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GWmttMOqS68S" - }, - "source": [ - "现在，从该文件重新创建模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5NDMO_7kS6Do", - "colab": {} - }, - "source": [ - "# 重新创建完全相同的模型，包括其权重和优化程序\n", - "new_model = keras.models.load_model('my_model.h5')\n", - "\n", - "# 显示网络结构\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JXQpbTicTBwt" - }, - "source": [ - "检查其准确率（accuracy）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jwEaj9DnTCVA", - "colab": {} - }, - "source": [ - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dGXqd4wWJl8O" - }, - "source": [ - "这项技术可以保存一切:\n", - "\n", - "* 权重\n", - "* 模型配置(结构)\n", - "* 优化器配置\n", - "\n", - "Keras 通过检查网络结构来保存模型。目前，它无法保存 Tensorflow 优化器（调用自 `tf.train`）。使用这些的时候，您需要在加载后重新编译模型，否则您将失去优化器的状态。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPyhgcoVzqUB" - }, - "source": [ - "### 通过 `saved_model` 保存" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LtcN4VIb7JkK" - }, - "source": [ - "注意：这种保存 `tf.keras` 模型的方法是实验性的，在将来的版本中可能有所改变。\n", - "\n", - "建立一个新模型，然后训练它：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sI1YvCDFzpl3", - "colab": {} - }, - "source": [ - "model = create_model()\n", - "\n", - "model.fit(train_images, train_labels, epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iUvT_3qE8hV5" - }, - "source": [ - "创建一个 `saved_model`，并将其放在带有 `tf.keras.experimental.export_saved_model` 的带时间戳的目录中：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sq8fPglI1RWA", - "colab": {} - }, - "source": [ - "import time\n", - "saved_model_path = \"./saved_models/{}\".format(int(time.time()))\n", - "\n", - "tf.keras.experimental.export_saved_model(model, saved_model_path)\n", - "saved_model_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MjpmyPfh8-1n" - }, - "source": [ - "列出您保存的模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZtOvxA7V0iTv", - "colab": {} - }, - "source": [ - "!ls saved_models/" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "B7qfpvpY9HCe" - }, - "source": [ - "从保存的模型重新加载新的 Keras 模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0YofwHdN0pxa", - "colab": {} - }, - "source": [ - "new_model = tf.keras.experimental.load_from_saved_model(saved_model_path)\n", - "\n", - "# 显示网络结构\n", - "new_model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uWwgNaz19TH2" - }, - "source": [ - "使用恢复的模型运行预测：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Yh5Mu0yOgE5J", - "colab": {} - }, - "source": [ - "model.predict(test_images).shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Pc9e6G6w1AWG", - "colab": {} - }, - "source": [ - "# 必须在评估之前编译模型。\n", - "# 如果仅部署已保存的模型，则不需要此步骤。\n", - "\n", - "new_model.compile(optimizer=model.optimizer, # 保留已加载的优化程序\n", - " loss='sparse_categorical_crossentropy', \n", - " metrics=['accuracy'])\n", - "\n", - "# 评估已恢复的模型\n", - "loss, acc = new_model.evaluate(test_images, test_labels, verbose=2)\n", - "print(\"Restored model, accuracy: {:5.2f}%\".format(100*acc))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/zh-cn/tutorials/keras/text_classification.ipynb b/site/zh-cn/tutorials/keras/text_classification.ipynb deleted file mode 100644 index 70418d703fe..00000000000 --- a/site/zh-cn/tutorials/keras/text_classification.ipynb +++ /dev/null @@ -1,748 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_classification.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "yCl0eTNH5RS3", - "colab": {} - }, - "source": [ - "#@title MIT License\n", - "#\n", - "# Copyright (c) 2017 François Chollet\n", - "#\n", - "# Permission is hereby granted, free of charge, to any person obtaining a\n", - "# copy of this software and associated documentation files (the \"Software\"),\n", - "# to deal in the Software without restriction, including without limitation\n", - "# the rights to use, copy, modify, merge, publish, distribute, sublicense,\n", - "# and/or sell copies of the Software, and to permit persons to whom the\n", - "# Software is furnished to do so, subject to the following conditions:\n", - "#\n", - "# The above copyright notice and this permission notice shall be included in\n", - "# all copies or substantial portions of the Software.\n", - "#\n", - "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n", - "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n", - "# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n", - "# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n", - "# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\n", - "# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\n", - "# DEALINGS IN THE SOFTWARE." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# 电影评论文本分类" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorFlow.google.cn 上查看\n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GEe3i16tQPjo", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "\n", - "此笔记本（notebook）使用评论文本将影评分为*积极（positive）*或*消极（nagetive）*两类。这是一个*二元（binary）*或者二分类问题，一种重要且应用广泛的机器学习问题。\n", - "\n", - "我们将使用来源于[网络电影数据库（Internet Movie Database）](https://www.imdb.com/)的 [IMDB 数据集（IMDB dataset）](https://tensorflow.google.cn/api_docs/python/tf/keras/datasets/imdb)，其包含 50,000 条影评文本。从该数据集切割出的25,000条评论用作训练，另外 25,000 条用作测试。训练集与测试集是*平衡的（balanced）*，意味着它们包含相等数量的积极和消极评论。\n", - "\n", - "此笔记本（notebook）使用了 [tf.keras](https://tensorflow.google.cn/guide/keras)，它是一个 Tensorflow 中用于构建和训练模型的高级API。有关使用 `tf.keras` 进行文本分类的更高级教程，请参阅 [MLCC文本分类指南（MLCC Text Classification Guide）](https://developers.google.com/machine-learning/guides/text-classification/)。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "\n", - "import numpy as np\n", - "\n", - "print(tf.__version__)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## 下载 IMDB 数据集\n", - "\n", - "IMDB 数据集已经打包在 Tensorflow 中。该数据集已经经过预处理，评论（单词序列）已经被转换为整数序列，其中每个整数表示字典中的特定单词。\n", - "\n", - "以下代码将下载 IMDB 数据集到您的机器上（如果您已经下载过将从缓存中复制）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "imdb = keras.datasets.imdb\n", - "\n", - "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "odr-KlzO-lkL" - }, - "source": [ - "参数 `num_words=10000` 保留了训练数据中最常出现的 10,000 个单词。为了保持数据规模的可管理性，低频词将被丢弃。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## 探索数据\n", - "\n", - "让我们花一点时间来了解数据格式。该数据集是经过预处理的：每个样本都是一个表示影评中词汇的整数数组。每个标签都是一个值为 0 或 1 的整数值，其中 0 代表消极评论，1 代表积极评论。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y8qCnve_-lkO", - "colab": {} - }, - "source": [ - "print(\"Training entries: {}, labels: {}\".format(len(train_data), len(train_labels)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RnKvHWW4-lkW" - }, - "source": [ - "评论文本被转换为整数值，其中每个整数代表词典中的一个单词。首条评论是这样的：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hIE4l_72x7DP" - }, - "source": [ - "电影评论可能具有不同的长度。以下代码显示了第一条和第二条评论的中单词数量。由于神经网络的输入必须是统一的长度，我们稍后需要解决这个问题。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X-6Ii9Pfx6Nr", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "4wJg2FiYpuoX" - }, - "source": [ - "### 将整数转换回单词\n", - "\n", - "了解如何将整数转换回文本对您可能是有帮助的。这里我们将创建一个辅助函数来查询一个包含了整数到字符串映射的字典对象：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tr5s_1alpzop", - "colab": {} - }, - "source": [ - "# 一个映射单词到整数索引的词典\n", - "word_index = imdb.get_word_index()\n", - "\n", - "# 保留第一个索引\n", - "word_index = {k:(v+3) for k,v in word_index.items()}\n", - "word_index[\"\"] = 0\n", - "word_index[\"\"] = 1\n", - "word_index[\"\"] = 2 # unknown\n", - "word_index[\"\"] = 3\n", - "\n", - "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n", - "\n", - "def decode_review(text):\n", - " return ' '.join([reverse_word_index.get(i, '?') for i in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "U3CNRvEZVppl" - }, - "source": [ - "现在我们可以使用 `decode_review` 函数来显示首条评论的文本：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s_OqxmH6-lkn", - "colab": {} - }, - "source": [ - "decode_review(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lFP_XKVRp4_S" - }, - "source": [ - "## 准备数据\n", - "\n", - "影评——即整数数组必须在输入神经网络之前转换为张量。这种转换可以通过以下两种方式来完成：\n", - "\n", - "* 将数组转换为表示单词出现与否的由 0 和 1 组成的向量，类似于 one-hot 编码。例如，序列[3, 5]将转换为一个 10,000 维的向量，该向量除了索引为 3 和 5 的位置是 1 以外，其他都为 0。然后，将其作为网络的首层——一个可以处理浮点型向量数据的稠密层。不过，这种方法需要大量的内存，需要一个大小为 `num_words * num_reviews` 的矩阵。\n", - "\n", - "* 或者，我们可以填充数组来保证输入数据具有相同的长度，然后创建一个大小为 `max_length * num_reviews` 的整型张量。我们可以使用能够处理此形状数据的嵌入层作为网络中的第一层。\n", - "\n", - "在本教程中，我们将使用第二种方法。\n", - "\n", - "由于电影评论长度必须相同，我们将使用 [pad_sequences](https://tensorflow.google.cn/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences) 函数来使长度标准化：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2jQv-omsHurp", - "colab": {} - }, - "source": [ - "train_data = keras.preprocessing.sequence.pad_sequences(train_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)\n", - "\n", - "test_data = keras.preprocessing.sequence.pad_sequences(test_data,\n", - " value=word_index[\"\"],\n", - " padding='post',\n", - " maxlen=256)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VO5MBpyQdipD" - }, - "source": [ - "现在让我们看下样本的长度：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "USSSBnkE-lky", - "colab": {} - }, - "source": [ - "len(train_data[0]), len(train_data[1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QJoxZGyfjT5V" - }, - "source": [ - "并检查一下首条评论（当前已经填充）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TG8X9cqi-lk9", - "colab": {} - }, - "source": [ - "print(train_data[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## 构建模型\n", - "\n", - "神经网络由堆叠的层来构建，这需要从两个主要方面来进行体系结构决策：\n", - "\n", - "* 模型里有多少层？\n", - "* 每个层里有多少*隐层单元（hidden units）*？\n", - "\n", - "在此样本中，输入数据包含一个单词索引的数组。要预测的标签为 0 或 1。让我们来为该问题构建一个模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "# 输入形状是用于电影评论的词汇数目（10,000 词）\n", - "vocab_size = 10000\n", - "\n", - "model = keras.Sequential()\n", - "model.add(keras.layers.Embedding(vocab_size, 16))\n", - "model.add(keras.layers.GlobalAveragePooling1D())\n", - "model.add(keras.layers.Dense(16, activation='relu'))\n", - "model.add(keras.layers.Dense(1, activation='sigmoid'))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "层按顺序堆叠以构建分类器：\n", - "\n", - "1. 第一层是`嵌入（Embedding）`层。该层采用整数编码的词汇表，并查找每个词索引的嵌入向量（embedding vector）。这些向量是通过模型训练学习到的。向量向输出数组增加了一个维度。得到的维度为：`(batch, sequence, embedding)`。\n", - "2. 接下来，`GlobalAveragePooling1D` 将通过对序列维度求平均值来为每个样本返回一个定长输出向量。这允许模型以尽可能最简单的方式处理变长输入。\n", - "3. 该定长输出向量通过一个有 16 个隐层单元的全连接（`Dense`）层传输。\n", - "4. 最后一层与单个输出结点密集连接。使用 `Sigmoid` 激活函数，其函数值为介于 0 与 1 之间的浮点数，表示概率或置信度。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0XMwnDOp-llH" - }, - "source": [ - "### 隐层单元\n", - "\n", - "上述模型在输入输出之间有两个中间层或“隐藏层”。输出（单元，结点或神经元）的数量即为层表示空间的维度。换句话说，是学习内部表示时网络所允许的自由度。\n", - "\n", - "如果模型具有更多的隐层单元（更高维度的表示空间）和/或更多层，则可以学习到更复杂的表示。但是，这会使网络的计算成本更高，并且可能导致学习到不需要的模式——一些能够在训练数据上而不是测试数据上改善性能的模式。这被称为*过拟合（overfitting）*，我们稍后会对此进行探究。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### 损失函数与优化器\n", - "\n", - "一个模型需要损失函数和优化器来进行训练。由于这是一个二分类问题且模型输出概率值（一个使用 sigmoid 激活函数的单一单元层），我们将使用 `binary_crossentropy` 损失函数。\n", - "\n", - "这不是损失函数的唯一选择，例如，您可以选择 `mean_squared_error` 。但是，一般来说 `binary_crossentropy` 更适合处理概率——它能够度量概率分布之间的“距离”，或者在我们的示例中，指的是度量 ground-truth 分布与预测值之间的“距离”。\n", - "\n", - "稍后，当我们研究回归问题（例如，预测房价）时，我们将介绍如何使用另一种叫做均方误差的损失函数。\n", - "\n", - "现在，配置模型来使用优化器和损失函数：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hCWYwkug-llQ" - }, - "source": [ - "## 创建一个验证集\n", - "\n", - "在训练时，我们想要检查模型在未见过的数据上的准确率（accuracy）。通过从原始训练数据中分离 10,000 个样本来创建一个*验证集*。（为什么现在不使用测试集？我们的目标是只使用训练数据来开发和调整模型，然后只使用一次测试数据来评估准确率（accuracy））。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "-NpcXY9--llS", - "colab": {} - }, - "source": [ - "x_val = train_data[:10000]\n", - "partial_x_train = train_data[10000:]\n", - "\n", - "y_val = train_labels[:10000]\n", - "partial_y_train = train_labels[10000:]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## 训练模型\n", - "\n", - "以 512 个样本的 mini-batch 大小迭代 40 个 epoch 来训练模型。这是指对 `x_train` 和 `y_train` 张量中所有样本的的 40 次迭代。在训练过程中，监测来自验证集的 10,000 个样本上的损失值（loss）和准确率（accuracy）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "id": "D6G9oqEV-Se-", - "colab_type": "code", - "colab": {} - }, - "source": [ - "history = model.fit(partial_x_train,\n", - " partial_y_train,\n", - " epochs=40,\n", - " batch_size=512,\n", - " validation_data=(x_val, y_val),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## 评估模型\n", - "\n", - "我们来看一下模型的性能如何。将返回两个值。损失值（loss）（一个表示误差的数字，值越低越好）与准确率（accuracy）。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data, test_labels, verbose=2)\n", - "\n", - "print(results)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "这种十分朴素的方法得到了约 87% 的准确率（accuracy）。若采用更好的方法，模型的准确率应当接近 95%。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## 创建一个准确率（accuracy）和损失值（loss）随时间变化的图表\n", - "\n", - "`model.fit()` 返回一个 `History` 对象，该对象包含一个字典，其中包含训练阶段所发生的一切事件：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VcvSXvhp-llb", - "colab": {} - }, - "source": [ - "history_dict = history.history\n", - "history_dict.keys()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nRKsqL40-lle" - }, - "source": [ - "有四个条目：在训练和验证期间，每个条目对应一个监控指标。我们可以使用这些条目来绘制训练与验证过程的损失值（loss）和准确率（accuracy），以便进行比较。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nGoYf2Js-lle", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "acc = history_dict['accuracy']\n", - "val_acc = history_dict['val_accuracy']\n", - "loss = history_dict['loss']\n", - "val_loss = history_dict['val_loss']\n", - "\n", - "epochs = range(1, len(acc) + 1)\n", - "\n", - "# “bo”代表 \"蓝点\"\n", - "plt.plot(epochs, loss, 'bo', label='Training loss')\n", - "# b代表“蓝色实线”\n", - "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", - "plt.title('Training and validation loss')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Loss')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6hXx-xOv-llh", - "colab": {} - }, - "source": [ - "plt.clf() # 清除数字\n", - "\n", - "plt.plot(epochs, acc, 'bo', label='Training acc')\n", - "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n", - "plt.title('Training and validation accuracy')\n", - "plt.xlabel('Epochs')\n", - "plt.ylabel('Accuracy')\n", - "plt.legend()\n", - "\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oFEmZ5zq-llk" - }, - "source": [ - "\n", - "在该图中，点代表训练损失值（loss）与准确率（accuracy），实线代表验证损失值（loss）与准确率（accuracy）。\n", - "\n", - "注意训练损失值随每一个 epoch *下降*而训练准确率（accuracy）随每一个 epoch *上升*。这在使用梯度下降优化时是可预期的——理应在每次迭代中最小化期望值。\n", - "\n", - "验证过程的损失值（loss）与准确率（accuracy）的情况却并非如此——它们似乎在 20 个 epoch 后达到峰值。这是过拟合的一个实例：模型在训练数据上的表现比在以前从未见过的数据上的表现要更好。在此之后，模型过度优化并学习*特定*于训练数据的表示，而不能够*泛化*到测试数据。\n", - "\n", - "对于这种特殊情况，我们可以通过在 20 个左右的 epoch 后停止训练来避免过拟合。稍后，您将看到如何通过回调自动执行此操作。" - ] - } - ] -} diff --git a/site/zh-cn/tutorials/keras/text_classification_with_hub.ipynb b/site/zh-cn/tutorials/keras/text_classification_with_hub.ipynb deleted file mode 100644 index 55144b8b891..00000000000 --- a/site/zh-cn/tutorials/keras/text_classification_with_hub.ipynb +++ /dev/null @@ -1,454 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text_classification_with_hub.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ic4_occAAiAT" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ioaprt5q5US7", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ItXfxkxvosLH" - }, - "source": [ - "# 使用 Keras 和 Tensorflow Hub 对电影评论进行文本分类" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hKY4XMc9o8iB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorFlow.google.cn 上查看\n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "eDesG2pcNY-E", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Eg62Pmz3o83v" - }, - "source": [ - "\n", - "此笔记本（notebook）使用评论文本将影评分为*积极（positive）*或*消极（nagetive）*两类。这是一个*二元（binary）*或者二分类问题，一种重要且应用广泛的机器学习问题。\n", - "\n", - "本教程演示了使用 Tensorflow Hub 和 Keras 进行迁移学习的基本应用。\n", - "\n", - "我们将使用来源于[网络电影数据库（Internet Movie Database）](https://www.imdb.com/)的 [IMDB 数据集（IMDB dataset）](https://www.tensorflow.org/api_docs/python/tf/keras/datasets/imdb)，其包含 50,000 条影评文本。从该数据集切割出的 25,000 条评论用作训练，另外 25,000 条用作测试。训练集与测试集是*平衡的（balanced）*，意味着它们包含相等数量的积极和消极评论。\n", - "\n", - "此笔记本（notebook）使用了 [tf.keras](https://www.tensorflow.org/guide/keras)，它是一个 Tensorflow 中用于构建和训练模型的高级API，此外还使用了 [TensorFlow Hub](https://www.tensorflow.org/hub)，一个用于迁移学习的库和平台。有关使用 `tf.keras` 进行文本分类的更高级教程，请参阅 [MLCC文本分类指南（MLCC Text Classification Guide）](https://developers.google.com/machine-learning/guides/text-classification/)。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2ew7HTbPpCJH", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import tensorflow_hub as hub\n", - "import tensorflow_datasets as tfds\n", - "\n", - "print(\"Version: \", tf.__version__)\n", - "print(\"Eager mode: \", tf.executing_eagerly())\n", - "print(\"Hub version: \", hub.__version__)\n", - "print(\"GPU is\", \"available\" if tf.config.experimental.list_physical_devices(\"GPU\") else \"NOT AVAILABLE\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iAsKG535pHep" - }, - "source": [ - "## 下载 IMDB 数据集\n", - "IMDB数据集可以在 [Tensorflow 数据集](https://github.com/tensorflow/datasets)处获取。以下代码将 IMDB 数据集下载至您的机器（或 colab 运行时环境）中：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zXXx5Oc3pOmN", - "colab": {} - }, - "source": [ - "# 将训练集按照 6:4 的比例进行切割，从而最终我们将得到 15,000\n", - "# 个训练样本, 10,000 个验证样本以及 25,000 个测试样本\n", - "train_validation_split = tfds.Split.TRAIN.subsplit([6, 4])\n", - "\n", - "(train_data, validation_data), test_data = tfds.load(\n", - " name=\"imdb_reviews\", \n", - " split=(train_validation_split, tfds.Split.TEST),\n", - " as_supervised=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "l50X3GfjpU4r" - }, - "source": [ - "## 探索数据\n", - "\n", - "让我们花一点时间来了解数据的格式。每一个样本都是一个表示电影评论和相应标签的句子。该句子不以任何方式进行预处理。标签是一个值为 0 或 1 的整数，其中 0 代表消极评论，1 代表积极评论。\n", - "\n", - "我们来打印下前十个样本。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QtTS4kpEpjbi", - "colab": {} - }, - "source": [ - "train_examples_batch, train_labels_batch = next(iter(train_data.batch(10)))\n", - "train_examples_batch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IFtaCHTdc-GY" - }, - "source": [ - "我们再打印下前十个标签。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tvAjVXOWc6Mj", - "colab": {} - }, - "source": [ - "train_labels_batch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LLC02j2g-llC" - }, - "source": [ - "## 构建模型\n", - "\n", - "神经网络由堆叠的层来构建，这需要从三个主要方面来进行体系结构决策：\n", - "\n", - "* 如何表示文本？\n", - "* 模型里有多少层？\n", - "* 每个层里有多少*隐层单元（hidden units）*？\n", - "\n", - "本示例中，输入数据由句子组成。预测的标签为 0 或 1。\n", - "\n", - "表示文本的一种方式是将句子转换为嵌入向量（embeddings vectors）。我们可以使用一个预先训练好的文本嵌入（text embedding）作为首层，这将具有三个优点：\n", - "\n", - "* 我们不必担心文本预处理\n", - "* 我们可以从迁移学习中受益\n", - "* 嵌入具有固定长度，更易于处理\n", - "\n", - "针对此示例我们将使用 [TensorFlow Hub](https://www.tensorflow.org/hub) 中名为 [google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1) 的一种**预训练文本嵌入（text embedding）模型** 。\n", - "\n", - "为了达到本教程的目的还有其他三种预训练模型可供测试：\n", - "\n", - "* [google/tf2-preview/gnews-swivel-20dim-with-oov/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim-with-oov/1) ——类似 [google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1)，但 2.5%的词汇转换为未登录词桶（OOV buckets）。如果任务的词汇与模型的词汇没有完全重叠，这将会有所帮助。\n", - "* [google/tf2-preview/nnlm-en-dim50/1](https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1) ——一个拥有约 1M 词汇量且维度为 50 的更大的模型。\n", - "* [google/tf2-preview/nnlm-en-dim128/1](https://tfhub.dev/google/tf2-preview/nnlm-en-dim128/1) ——拥有约 1M 词汇量且维度为128的更大的模型。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "In2nDpTLkgKa" - }, - "source": [ - "让我们首先创建一个使用 Tensorflow Hub 模型嵌入（embed）语句的Keras层，并在几个输入样本中进行尝试。请注意无论输入文本的长度如何，嵌入（embeddings）输出的形状都是：`(num_examples, embedding_dimension)`。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_NUbzVeYkgcO", - "colab": {} - }, - "source": [ - "embedding = \"https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1\"\n", - "hub_layer = hub.KerasLayer(embedding, input_shape=[], \n", - " dtype=tf.string, trainable=True)\n", - "hub_layer(train_examples_batch[:3])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dfSbV6igl1EH" - }, - "source": [ - "现在让我们构建完整模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xpKOoWgu-llD", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()\n", - "model.add(hub_layer)\n", - "model.add(tf.keras.layers.Dense(16, activation='relu'))\n", - "model.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n", - "\n", - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6PbKQ6mucuKL" - }, - "source": [ - "层按顺序堆叠以构建分类器：\n", - "\n", - "1. 第一层是 Tensorflow Hub 层。这一层使用一个预训练的保存好的模型来将句子映射为嵌入向量（embedding vector）。我们所使用的预训练文本嵌入（embedding）模型([google/tf2-preview/gnews-swivel-20dim/1](https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1))将句子切割为符号，嵌入（embed）每个符号然后进行合并。最终得到的维度是：`(num_examples, embedding_dimension)`。\n", - "2. 该定长输出向量通过一个有 16 个隐层单元的全连接层（`Dense`）进行管道传输。\n", - "3. 最后一层与单个输出结点紧密相连。使用 `Sigmoid` 激活函数，其函数值为介于 0 与 1 之间的浮点数，表示概率或置信水平。\n", - "\n", - "让我们编译模型。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "L4EqVWg4-llM" - }, - "source": [ - "### 损失函数与优化器\n", - "\n", - "一个模型需要损失函数和优化器来进行训练。由于这是一个二分类问题且模型输出概率值（一个使用 sigmoid 激活函数的单一单元层），我们将使用 `binary_crossentropy` 损失函数。\n", - "\n", - "这不是损失函数的唯一选择，例如，您可以选择 `mean_squared_error` 。但是，一般来说 `binary_crossentropy` 更适合处理概率——它能够度量概率分布之间的“距离”，或者在我们的示例中，指的是度量 ground-truth 分布与预测值之间的“距离”。\n", - "\n", - "稍后，当我们研究回归问题（例如，预测房价）时，我们将介绍如何使用另一种叫做均方误差的损失函数。\n", - "\n", - "现在，配置模型来使用优化器和损失函数：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mr0GP-cQ-llN", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "35jv_fzP-llU" - }, - "source": [ - "## 训练模型\n", - "\n", - "以 512 个样本的 mini-batch 大小迭代 20 个 epoch 来训练模型。 这是指对 `x_train` 和 `y_train` 张量中所有样本的的 20 次迭代。在训练过程中，监测来自验证集的 10,000 个样本上的损失值（loss）和准确率（accuracy）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tXSGrjWZ-llW", - "colab": {} - }, - "source": [ - "history = model.fit(train_data.shuffle(10000).batch(512),\n", - " epochs=20,\n", - " validation_data=validation_data.batch(512),\n", - " verbose=1)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9EEGuDVuzb5r" - }, - "source": [ - "## 评估模型\n", - "\n", - "我们来看下模型的表现如何。将返回两个值。损失值（loss）（一个表示误差的数字，值越低越好）与准确率（accuracy）。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zOMKywn4zReN", - "colab": {} - }, - "source": [ - "results = model.evaluate(test_data.batch(512), verbose=2)\n", - "for name, value in zip(model.metrics_names, results):\n", - " print(\"%s: %.3f\" % (name, value))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "z1iEXVTR0Z2t" - }, - "source": [ - "这种十分朴素的方法得到了约 87% 的准确率（accuracy）。若采用更好的方法，模型的准确率应当接近 95%。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5KggXVeL-llZ" - }, - "source": [ - "## 进一步阅读\n", - "\n", - "有关使用字符串输入的更一般方法，以及对训练期间准确率（accuracy）和损失值（loss）更详细的分析，请参阅[此处](https://www.tensorflow.org/tutorials/keras/basic_text_classification)。" - ] - } - ] -} diff --git a/site/zh-cn/tutorials/load_data/csv.ipynb b/site/zh-cn/tutorials/load_data/csv.ipynb deleted file mode 100644 index 4affc86b14c..00000000000 --- a/site/zh-cn/tutorials/load_data/csv.ipynb +++ /dev/null @@ -1,711 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "csv.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DweYe9FcbMK_" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AVV2e0XKbJeX", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sUtoed20cRJJ" - }, - "source": [ - "# 用 tf.data 加载 CSV 数据" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1ap_W4aQcgNT" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 Tensorflow.org 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Z4x9SYyev8QZ", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C-3Xbt0FfGfs" - }, - "source": [ - "这篇教程通过一个示例展示了怎样将 CSV 格式的数据加载进 `tf.data.Dataset`。\n", - "\n", - "这篇教程使用的是泰坦尼克号乘客的数据。模型会根据乘客的年龄、性别、票务舱和是否独自旅行等特征来预测乘客生还的可能性。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" - }, - "source": [ - "## 设置" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "I4dwMQVQMQWD", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "baYFZMW_bJHh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "import functools\n", - "\n", - "import numpy as np\n", - "import tensorflow as tf\n", - "import tensorflow_datasets as tfds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ncf5t6tgL5ZI", - "colab": {} - }, - "source": [ - "TRAIN_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/train.csv\"\n", - "TEST_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/eval.csv\"\n", - "\n", - "train_file_path = tf.keras.utils.get_file(\"train.csv\", TRAIN_DATA_URL)\n", - "test_file_path = tf.keras.utils.get_file(\"eval.csv\", TEST_DATA_URL)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4ONE94qulk6S", - "colab": {} - }, - "source": [ - "# 让 numpy 数据更易读。\n", - "np.set_printoptions(precision=3, suppress=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Wuqj601Qw0Ml" - }, - "source": [ - "## 加载数据\n", - "\n", - "开始的时候，我们通过打印 CSV 文件的前几行来了解文件的格式。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "54Dv7mCrf9Yw", - "colab": {} - }, - "source": [ - "!head {train_file_path}" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YOYKQKmMj3D6" - }, - "source": [ - "正如你看到的那样，CSV 文件的每列都会有一个列名。dataset 的构造函数会自动识别这些列名。如果你使用的文件的第一行不包含列名，那么需要将列名通过字符串列表传给 `make_csv_dataset` 函数的 `column_names` 参数。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZS-bt1LvWn2x" - }, - "source": [ - " \n", - "\n", - "\n", - "\n", - "```python\n", - "\n", - "CSV_COLUMNS = ['survived', 'sex', 'age', 'n_siblings_spouses', 'parch', 'fare', 'class', 'deck', 'embark_town', 'alone']\n", - "\n", - "dataset = tf.data.experimental.make_csv_dataset(\n", - " ...,\n", - " column_names=CSV_COLUMNS,\n", - " ...)\n", - " \n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gZfhoX7bR9u4" - }, - "source": [ - "这个示例使用了所有的列。如果你需要忽略数据集中的某些列，创建一个包含你需要使用的列的列表，然后传给构造器的（可选）参数 `select_columns`。\n", - "\n", - "```python\n", - "\n", - "dataset = tf.data.experimental.make_csv_dataset(\n", - " ...,\n", - " select_columns = columns_to_use, \n", - " ...)\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "67mfwr4v-mN_" - }, - "source": [ - "对于包含模型需要预测的值的列是你需要显式指定的。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iXROZm5f3V4E", - "colab": {} - }, - "source": [ - "LABEL_COLUMN = 'survived'\n", - "LABELS = [0, 1]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t4N-plO4tDXd" - }, - "source": [ - "现在从文件中读取 CSV 数据并且创建 dataset。\n", - "\n", - "(完整的文档，参考 `tf.data.experimental.make_csv_dataset`)\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Co7UJ7gpNADC", - "colab": {} - }, - "source": [ - "def get_dataset(file_path):\n", - " dataset = tf.data.experimental.make_csv_dataset(\n", - " file_path,\n", - " batch_size=12, # 为了示例更容易展示，手动设置较小的值\n", - " label_name=LABEL_COLUMN,\n", - " na_value=\"?\",\n", - " num_epochs=1,\n", - " ignore_errors=True)\n", - " return dataset\n", - "\n", - "raw_train_data = get_dataset(train_file_path)\n", - "raw_test_data = get_dataset(test_file_path)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vHUQFKoQI6G7" - }, - "source": [ - "dataset 中的每个条目都是一个批次，用一个元组（*多个样本*，*多个标签*）表示。样本中的数据组织形式是以列为主的张量（而不是以行为主的张量），每条数据中包含的元素个数就是批次大小（这个示例中是 12）。\n", - "\n", - "阅读下面的示例有助于你的理解。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qWtFYtwXIeuj", - "colab": {} - }, - "source": [ - "examples, labels = next(iter(raw_train_data)) # 第一个批次\n", - "print(\"EXAMPLES: \\n\", examples, \"\\n\")\n", - "print(\"LABELS: \\n\", labels)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9cryz31lxs3e" - }, - "source": [ - "## 数据预处理" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tSyrkSQwYHKi" - }, - "source": [ - "### 分类数据\n", - "\n", - "CSV 数据中的有些列是分类的列。也就是说，这些列只能在有限的集合中取值。\n", - "\n", - "使用 `tf.feature_column` API 创建一个 `tf.feature_column.indicator_column` 集合，每个 `tf.feature_column.indicator_column` 对应一个分类的列。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mWDniduKMw-C", - "colab": {} - }, - "source": [ - "CATEGORIES = {\n", - " 'sex': ['male', 'female'],\n", - " 'class' : ['First', 'Second', 'Third'],\n", - " 'deck' : ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'],\n", - " 'embark_town' : ['Cherbourg', 'Southhampton', 'Queenstown'],\n", - " 'alone' : ['y', 'n']\n", - "}\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kkxLdrsLwHPT", - "colab": {} - }, - "source": [ - "categorical_columns = []\n", - "for feature, vocab in CATEGORIES.items():\n", - " cat_col = tf.feature_column.categorical_column_with_vocabulary_list(\n", - " key=feature, vocabulary_list=vocab)\n", - " categorical_columns.append(tf.feature_column.indicator_column(cat_col))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "H18CxpHY_Nma", - "colab": {} - }, - "source": [ - "# 你刚才创建的内容\n", - "categorical_columns" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R7-1QG99_1sN" - }, - "source": [ - "这将是后续构建模型时处理输入数据的一部分。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9AsbaFmCeJtF" - }, - "source": [ - "### 连续数据" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o2maE8d2ijsq" - }, - "source": [ - "连续数据需要标准化。\n", - "\n", - "写一个函数标准化这些值，然后将这些值改造成 2 维的张量。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "REKqO_xHPNx0", - "colab": {} - }, - "source": [ - "def process_continuous_data(mean, data):\n", - " # 标准化数据\n", - " data = tf.cast(data, tf.float32) * 1/(2*mean)\n", - " return tf.reshape(data, [-1, 1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VPsoMUgRCpUM" - }, - "source": [ - "现在创建一个数值列的集合。`tf.feature_columns.numeric_column` API 会使用 `normalizer_fn` 参数。在传参的时候使用 [`functools.partial`](https://docs.python.org/3/library/functools.html#functools.partial)，`functools.partial` 由使用每个列的均值进行标准化的函数构成。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WKT1ASWpwH46", - "colab": {} - }, - "source": [ - "MEANS = {\n", - " 'age' : 29.631308,\n", - " 'n_siblings_spouses' : 0.545455,\n", - " 'parch' : 0.379585,\n", - " 'fare' : 34.385399\n", - "}\n", - "\n", - "numerical_columns = []\n", - "\n", - "for feature in MEANS.keys():\n", - " num_col = tf.feature_column.numeric_column(feature, normalizer_fn=functools.partial(process_continuous_data, MEANS[feature]))\n", - " numerical_columns.append(num_col)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Bw0I35xRS57V", - "colab": {} - }, - "source": [ - "# 你刚才创建的内容。\n", - "numerical_columns" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M37oD2VcCO4R" - }, - "source": [ - "这里使用标准化的方法需要提前知道每列的均值。如果需要计算连续的数据流的标准化的值可以使用 [TensorFlow Transform](https://www.tensorflow.org/tfx/transform/get_started)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kPWkC4_1l3IG" - }, - "source": [ - "### 创建预处理层" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "R3QAjo1qD4p9" - }, - "source": [ - "将这两个特征列的集合相加，并且传给 `tf.keras.layers.DenseFeatures` 从而创建一个进行预处理的输入层。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3-OYK7GnaH0r", - "colab": {} - }, - "source": [ - "preprocessing_layer = tf.keras.layers.DenseFeatures(categorical_columns+numerical_columns)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DlF_omQqtnOP" - }, - "source": [ - "## 构建模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lQoFh16LxtT_" - }, - "source": [ - "从 `preprocessing_layer` 开始构建 `tf.keras.Sequential`。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3mSGsHTFPvFo", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " preprocessing_layer,\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid'),\n", - "])\n", - "\n", - "model.compile(\n", - " loss='binary_crossentropy',\n", - " optimizer='adam',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hPdtI2ie0lEZ" - }, - "source": [ - "## 训练、评估和预测" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8gvw1RE9zXkD" - }, - "source": [ - "现在可以实例化和训练模型。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sW-4XlLeEQ2B", - "colab": {} - }, - "source": [ - "train_data = raw_train_data.shuffle(500)\n", - "test_data = raw_test_data" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Q_nm28IzNDTO", - "colab": {} - }, - "source": [ - "model.fit(train_data, epochs=20)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QyDMgBurzqQo" - }, - "source": [ - "当模型训练完成的时候，你可以在测试集 `test_data` 上检查准确性。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eB3R3ViVONOp", - "colab": {} - }, - "source": [ - "test_loss, test_accuracy = model.evaluate(test_data)\n", - "\n", - "print('\\n\\nTest Loss {}, Test Accuracy {}'.format(test_loss, test_accuracy))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sTrn_pD90gdJ" - }, - "source": [ - "使用 `tf.keras.Model.predict` 推断一个批次或多个批次的标签。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qwcx74F3ojqe", - "colab": {} - }, - "source": [ - "predictions = model.predict(test_data)\n", - "\n", - "# 显示部分结果\n", - "for prediction, survived in zip(predictions[:10], list(test_data)[0][1][:10]):\n", - " print(\"Predicted survival: {:.2%}\".format(prediction[0]),\n", - " \" | Actual outcome: \",\n", - " (\"SURVIVED\" if bool(survived) else \"DIED\"))\n", - "\n" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/zh-cn/tutorials/load_data/images.ipynb b/site/zh-cn/tutorials/load_data/images.ipynb deleted file mode 100644 index e195ee27580..00000000000 --- a/site/zh-cn/tutorials/load_data/images.ipynb +++ /dev/null @@ -1,1676 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "images.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.6" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mt9dL5dIir8X" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "ufPx7EiCiqgR", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ucMoYase6URl" - }, - "source": [ - "# 用 tf.data 加载图片" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_Wwu5SXZmEkB" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GEe3i16tQPjo", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oxw4WahM7DU9" - }, - "source": [ - "本教程提供一个如何使用 `tf.data` 加载图片的简单例子。\n", - "\n", - "本例中使用的数据集分布在图片文件夹中，一个文件夹含有一类图片。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hoQQiZDB6URn" - }, - "source": [ - "## 配置" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QGXxBuPyKJw1", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KT6CcaqgQewg", - "colab": {} - }, - "source": [ - "AUTOTUNE = tf.data.experimental.AUTOTUNE" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rxndJHNC8YPM" - }, - "source": [ - "## 下载并检查数据集" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wO0InzL66URu" - }, - "source": [ - "### 检索图片\n", - "\n", - "在你开始任何训练之前，你将需要一组图片来教会网络你想要训练的新类别。你已经创建了一个文件夹，存储了最初使用的拥有创作共用许可的花卉照片。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rN-Pc6Zd6awg", - "colab": {} - }, - "source": [ - "import pathlib\n", - "data_root_orig = tf.keras.utils.get_file(origin='https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz',\n", - " fname='flower_photos', untar=True)\n", - "data_root = pathlib.Path(data_root_orig)\n", - "print(data_root)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rFkFK74oO--g" - }, - "source": [ - "下载了 218 MB 之后，你现在应该有花卉照片副本：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7onR_lWE7Njj", - "colab": {} - }, - "source": [ - "for item in data_root.iterdir():\n", - " print(item)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4yYX3ZRqGOuq", - "colab": {} - }, - "source": [ - "import random\n", - "all_image_paths = list(data_root.glob('*/*'))\n", - "all_image_paths = [str(path) for path in all_image_paths]\n", - "random.shuffle(all_image_paths)\n", - "\n", - "image_count = len(all_image_paths)\n", - "image_count" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t_BbYnLjbltQ", - "colab": {} - }, - "source": [ - "all_image_paths[:10]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vkM-IpB-6URx" - }, - "source": [ - "### 检查图片\n", - "现在让我们快速浏览几张图片，这样你知道你在处理什么：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wNGateQJ6UR1", - "colab": {} - }, - "source": [ - "import os\n", - "attributions = (data_root/\"LICENSE.txt\").open(encoding='utf-8').readlines()[4:]\n", - "attributions = [line.split(' CC-BY') for line in attributions]\n", - "attributions = dict(attributions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jgowG2xu88Io", - "colab": {} - }, - "source": [ - "import IPython.display as display\n", - "\n", - "def caption_image(image_path):\n", - " image_rel = pathlib.Path(image_path).relative_to(data_root)\n", - " return \"Image (CC BY 2.0) \" + ' - '.join(attributions[str(image_rel)].split(' - ')[:-1])\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YIjLi-nX0txI", - "colab": {} - }, - "source": [ - "for n in range(3):\n", - " image_path = random.choice(all_image_paths)\n", - " display.display(display.Image(image_path))\n", - " print(caption_image(image_path))\n", - " print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OaNOr-co3WKk" - }, - "source": [ - "### 确定每张图片的标签" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-weOQpDw2Jnu" - }, - "source": [ - "列出可用的标签：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ssUZ7Qh96UR3", - "colab": {} - }, - "source": [ - "label_names = sorted(item.name for item in data_root.glob('*/') if item.is_dir())\n", - "label_names" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9l_JEBql2OzS" - }, - "source": [ - "为每个标签分配索引：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Y8pCV46CzPlp", - "colab": {} - }, - "source": [ - "label_to_index = dict((name, index) for index, name in enumerate(label_names))\n", - "label_to_index" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VkXsHg162T9F" - }, - "source": [ - "创建一个列表，包含每个文件的标签索引：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q62i1RBP4Q02", - "colab": {} - }, - "source": [ - "all_image_labels = [label_to_index[pathlib.Path(path).parent.name]\n", - " for path in all_image_paths]\n", - "\n", - "print(\"First 10 labels indices: \", all_image_labels[:10])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "i5L09icm9iph" - }, - "source": [ - "### 加载和格式化图片" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SbqqRUS79ooq" - }, - "source": [ - "TensorFlow 包含加载和处理图片时你需要的所有工具：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jQZdySHvksOu", - "colab": {} - }, - "source": [ - "img_path = all_image_paths[0]\n", - "img_path" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2t2h2XCcmK1Y" - }, - "source": [ - "以下是原始数据：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LJfkyC_Qkt7A", - "colab": {} - }, - "source": [ - "img_raw = tf.io.read_file(img_path)\n", - "print(repr(img_raw)[:100]+\"...\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "opN8AVc8mSbz" - }, - "source": [ - "将它解码为图像 tensor（张量）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Tm0tdrlfk0Bb", - "colab": {} - }, - "source": [ - "img_tensor = tf.image.decode_image(img_raw)\n", - "\n", - "print(img_tensor.shape)\n", - "print(img_tensor.dtype)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3k-Of2Tfmbeq" - }, - "source": [ - "根据你的模型调整其大小：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XFpz-3_vlJgp", - "colab": {} - }, - "source": [ - "img_final = tf.image.resize(img_tensor, [192, 192])\n", - "img_final = img_final/255.0\n", - "print(img_final.shape)\n", - "print(img_final.numpy().min())\n", - "print(img_final.numpy().max())\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aCsAa4Psl4AQ" - }, - "source": [ - "将这些包装在一个简单的函数里，以备后用。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HmUiZJNU73vA", - "colab": {} - }, - "source": [ - "def preprocess_image(image):\n", - " image = tf.image.decode_jpeg(image, channels=3)\n", - " image = tf.image.resize(image, [192, 192])\n", - " image /= 255.0 # normalize to [0,1] range\n", - "\n", - " return image" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "einETrJnO-em", - "colab": {} - }, - "source": [ - "def load_and_preprocess_image(path):\n", - " image = tf.io.read_file(path)\n", - " return preprocess_image(image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3brWQcdtz78y", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "image_path = all_image_paths[0]\n", - "label = all_image_labels[0]\n", - "\n", - "plt.imshow(load_and_preprocess_image(img_path))\n", - "plt.grid(False)\n", - "plt.xlabel(caption_image(img_path))\n", - "plt.title(label_names[label].title())\n", - "print()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n2TCr1TQ8pA3" - }, - "source": [ - "## 构建一个 `tf.data.Dataset`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6H9Z5Mq63nSH" - }, - "source": [ - "### 一个图片数据集" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GN-s04s-6Luq" - }, - "source": [ - "构建 `tf.data.Dataset` 最简单的方法就是使用 `from_tensor_slices` 方法。\n", - "\n", - "将字符串数组切片，得到一个字符串数据集：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6oRPG3Jz3ie_", - "colab": {} - }, - "source": [ - "path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uML4JeMmIAvO" - }, - "source": [ - "`shapes（维数）` 和 `types（类型）` 描述数据集里每个数据项的内容。在这里是一组标量二进制字符串。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mIsNflFbIK34", - "colab": {} - }, - "source": [ - "print(path_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZjyGcM8OwBJ2" - }, - "source": [ - "现在创建一个新的数据集，通过在路径数据集上映射 `preprocess_image` 来动态加载和格式化图片。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "D1iba6f4khu-", - "colab": {} - }, - "source": [ - "image_ds = path_ds.map(load_and_preprocess_image, num_parallel_calls=AUTOTUNE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JLUPs2a-lEEJ", - "colab": {} - }, - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "plt.figure(figsize=(8,8))\n", - "for n, image in enumerate(image_ds.take(4)):\n", - " plt.subplot(2,2,n+1)\n", - " plt.imshow(image)\n", - " plt.grid(False)\n", - " plt.xticks([])\n", - " plt.yticks([])\n", - " plt.xlabel(caption_image(all_image_paths[n]))\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "P6FNqPbxkbdx" - }, - "source": [ - "### 一个`(图片, 标签)`对数据集" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YgvrWLKG67-x" - }, - "source": [ - "使用同样的 `from_tensor_slices` 方法你可以创建一个标签数据集：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AgBsAiV06udj", - "colab": {} - }, - "source": [ - "label_ds = tf.data.Dataset.from_tensor_slices(tf.cast(all_image_labels, tf.int64))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HEsk5nN0vyeX", - "colab": {} - }, - "source": [ - "for label in label_ds.take(10):\n", - " print(label_names[label.numpy()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jHjgrEeTxyYz" - }, - "source": [ - "由于这些数据集顺序相同，你可以将他们打包在一起得到一个`(图片, 标签)`对数据集：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AOEWNMdQwsbN", - "colab": {} - }, - "source": [ - "image_label_ds = tf.data.Dataset.zip((image_ds, label_ds))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yA2F09SJLMuM" - }, - "source": [ - "这个新数据集的 `shapes（维数）` 和 `types（类型）` 也是维数和类型的元组，用来描述每个字段：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DuVYNinrLL-N", - "colab": {} - }, - "source": [ - "print(image_label_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2WYMikoPWOQX" - }, - "source": [ - "注意：当你拥有形似 `all_image_labels` 和 `all_image_paths` 的数组，`tf.data.dataset.Dataset.zip` 的替代方法是将这对数组切片。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HOFwZI-2WhzV", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.from_tensor_slices((all_image_paths, all_image_labels))\n", - "\n", - "# 元组被解压缩到映射函数的位置参数中\n", - "def load_and_preprocess_from_path_label(path, label):\n", - " return load_and_preprocess_image(path), label\n", - "\n", - "image_label_ds = ds.map(load_and_preprocess_from_path_label)\n", - "image_label_ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vYGCgJuR_9Qp" - }, - "source": [ - "### 训练的基本方法" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wwZavzgsIytz" - }, - "source": [ - "要使用此数据集训练模型，你将会想要数据：\n", - "\n", - "* 被充分打乱。\n", - "* 被分割为 batch。\n", - "* 永远重复。\n", - "* 尽快提供 batch。\n", - "\n", - "使用 `tf.data` api 可以轻松添加这些功能。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "uZmZJx8ePw_5", - "colab": {} - }, - "source": [ - "BATCH_SIZE = 32\n", - "\n", - "# 设置一个和数据集大小一致的 shuffle buffer size（随机缓冲区大小）以保证数据\n", - "# 被充分打乱。\n", - "ds = image_label_ds.shuffle(buffer_size=image_count)\n", - "ds = ds.repeat()\n", - "ds = ds.batch(BATCH_SIZE)\n", - "# 当模型在训练的时候，`prefetch` 使数据集在后台取得 batch。\n", - "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6JsM-xHiFCuW" - }, - "source": [ - "这里有一些注意事项：\n", - "\n", - "1. 顺序很重要。\n", - "\n", - " * 在 `.repeat` 之后 `.shuffle`，会在 epoch 之间打乱数据（当有些数据出现两次的时候，其他数据还没有出现过）。\n", - " \n", - " * 在 `.batch` 之后 `.shuffle`，会打乱 batch 的顺序，但是不会在 batch 之间打乱数据。\n", - "\n", - "1. 你在完全打乱中使用和数据集大小一样的 `buffer_size（缓冲区大小）`。较大的缓冲区大小提供更好的随机化，但使用更多的内存，直到超过数据集大小。\n", - "\n", - "1. 在从随机缓冲区中拉取任何元素前，要先填满它。所以当你的 `Dataset（数据集）`启动的时候一个大的 `buffer_size（缓冲区大小）`可能会引起延迟。\n", - "\n", - "1. 在随机缓冲区完全为空之前，被打乱的数据集不会报告数据集的结尾。`Dataset（数据集）`由 `.repeat` 重新启动，导致需要再次等待随机缓冲区被填满。\n", - "\n", - "\n", - "最后一点可以通过使用 `tf.data.Dataset.apply` 方法和融合过的 `tf.data.experimental.shuffle_and_repeat` 函数来解决:" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ocr6PybXNDoO", - "colab": {} - }, - "source": [ - "ds = image_label_ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE)\n", - "ds = ds.prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GBBZMSuAmQVL" - }, - "source": [ - "### 传递数据集至模型\n", - "\n", - "从 `tf.keras.applications` 取得 MobileNet v2 副本。\n", - "\n", - "该模型副本会被用于一个简单的迁移学习例子。\n", - "\n", - "设置 MobileNet 的权重为不可训练：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KbJrXn9omO_g", - "colab": {} - }, - "source": [ - "mobile_net = tf.keras.applications.MobileNetV2(input_shape=(192, 192, 3), include_top=False)\n", - "mobile_net.trainable=False" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y7NVWiLF3Vbf" - }, - "source": [ - "该模型期望它的输出被标准化至 `[-1,1]` 范围内：\n", - "\n", - "```\n", - "help(keras_applications.mobilenet_v2.preprocess_input)\n", - "```\n", - "\n", - "
    \n",
-        "……\n",
-        "该函数使用“Inception”预处理，将\n",
-        "RGB 值从 [0, 255] 转化为 [-1, 1]\n",
-        "……\n",
-        "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CboYya6LmdQI" - }, - "source": [ - "在你将输出传递给 MobilNet 模型之前，你需要将其范围从 `[0,1]` 转化为 `[-1,1]`：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "SNOkHUGv3FYq", - "colab": {} - }, - "source": [ - "def change_range(image,label):\n", - " return 2*image-1, label\n", - "\n", - "keras_ds = ds.map(change_range)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QDzZ3Nye5Rpv" - }, - "source": [ - "MobileNet 为每张图片的特征返回一个 `6x6` 的空间网格。\n", - "\n", - "传递一个 batch 的图片给它，查看结果：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OzAhGkEK6WuE", - "colab": {} - }, - "source": [ - "# 数据集可能需要几秒来启动，因为要填满其随机缓冲区。\n", - "image_batch, label_batch = next(iter(keras_ds))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LcFdiWpO5WbV", - "colab": {} - }, - "source": [ - "feature_map_batch = mobile_net(image_batch)\n", - "print(feature_map_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vrbjEvaC5XmU" - }, - "source": [ - "构建一个包装了 MobileNet 的模型并在 `tf.keras.layers.Dense` 输出层之前使用 `tf.keras.layers.GlobalAveragePooling2D` 来平均那些空间向量：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "X0ooIU9fNjPJ", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " mobile_net,\n", - " tf.keras.layers.GlobalAveragePooling2D(),\n", - " tf.keras.layers.Dense(len(label_names), activation = 'softmax')])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "foQYUJs97V4V" - }, - "source": [ - "现在它产出符合预期 shape(维数)的输出：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1nwYxvpj7ZEf", - "colab": {} - }, - "source": [ - "logit_batch = model(image_batch).numpy()\n", - "\n", - "print(\"min logit:\", logit_batch.min())\n", - "print(\"max logit:\", logit_batch.max())\n", - "print()\n", - "\n", - "print(\"Shape:\", logit_batch.shape)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pFc4I_J2nNOJ" - }, - "source": [ - "编译模型以描述训练过程：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZWGqLEWYRNvv", - "colab": {} - }, - "source": [ - "model.compile(optimizer=tf.keras.optimizers.Adam(),\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=[\"accuracy\"])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tF1mO6haBOSd" - }, - "source": [ - "此处有两个可训练的变量 —— Dense 层中的 `weights（权重）` 和 `bias（偏差）`：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pPQ5yqyKBJMm", - "colab": {} - }, - "source": [ - "len(model.trainable_variables)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kug5Wg66UJjl", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "f_glpYZ-nYC_" - }, - "source": [ - "你已经准备好来训练模型了。\n", - "\n", - "注意，出于演示目的每一个 epoch 中你将只运行 3 step，但一般来说在传递给 `model.fit()` 之前你会指定 step 的真实数量，如下所示：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AnXPRNWoTypI", - "colab": {} - }, - "source": [ - "steps_per_epoch=tf.math.ceil(len(all_image_paths)/BATCH_SIZE).numpy()\n", - "steps_per_epoch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "q_8sabaaSGAp", - "colab": {} - }, - "source": [ - "model.fit(ds, epochs=1, steps_per_epoch=3)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UMVnoBcG_NlQ" - }, - "source": [ - "## 性能\n", - "\n", - "注意：这部分只是展示一些可能帮助提升性能的简单技巧。深入指南，请看：[输入 pipeline（管道）的性能](https://tensorflow.google.cn/guide/performance/datasets)。\n", - "\n", - "上面使用的简单 pipeline（管道）在每个 epoch 中单独读取每个文件。在本地使用 CPU 训练时这个方法是可行的，但是可能不足以进行 GPU 训练并且完全不适合任何形式的分布式训练。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oNmQqgGhLWie" - }, - "source": [ - "要研究这点，首先构建一个简单的函数来检查数据集的性能：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_gFVe1rp_MYr", - "colab": {} - }, - "source": [ - "import time\n", - "default_timeit_steps = 2*steps_per_epoch+1\n", - "\n", - "def timeit(ds, steps=default_timeit_steps):\n", - " overall_start = time.time()\n", - " # 在开始计时之前\n", - " # 取得单个 batch 来填充 pipeline（管道）（填充随机缓冲区）\n", - " it = iter(ds.take(steps+1))\n", - " next(it)\n", - "\n", - " start = time.time()\n", - " for i,(images,labels) in enumerate(it):\n", - " if i%10 == 0:\n", - " print('.',end='')\n", - " print()\n", - " end = time.time()\n", - "\n", - " duration = end-start\n", - " print(\"{} batches: {} s\".format(steps, duration))\n", - " print(\"{:0.5f} Images/s\".format(BATCH_SIZE*steps/duration))\n", - " print(\"Total time: {}s\".format(end-overall_start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TYiOr4vdLcNX" - }, - "source": [ - "当前数据集的性能是：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZDxLwMJOReVe", - "colab": {} - }, - "source": [ - "ds = image_label_ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IjouTJadRxyp", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "HsLlXMO7EWBR" - }, - "source": [ - "### 缓存" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "lV1NOn2zE2lR" - }, - "source": [ - "使用 `tf.data.Dataset.cache` 在 epoch 之间轻松缓存计算结果。这是非常高效的，特别是当内存能容纳全部数据时。\n", - "\n", - "在被预处理之后（解码和调整大小），图片在此被缓存了：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qj_U09xpDvOg", - "colab": {} - }, - "source": [ - "ds = image_label_ds.cache()\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(buffer_size=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rdxpvQ7VEo3y", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "usIv7MqqZQps" - }, - "source": [ - "使用内存缓存的一个缺点是必须在每次运行时重建缓存，这使得每次启动数据集时有相同的启动延迟：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eKX6ergKb_xd", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jUzpG4lYNkN-" - }, - "source": [ - "如果内存不够容纳数据，使用一个缓存文件：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vIvF8K4GMq0g", - "colab": {} - }, - "source": [ - "ds = image_label_ds.cache(filename='./cache.tf-data')\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds = ds.batch(BATCH_SIZE).prefetch(1)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eTIj6IOmM4yA", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qqo3dyB0Z4t2" - }, - "source": [ - "这个缓存文件也有可快速重启数据集而无需重建缓存的优点。注意第二次快了多少：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hZhVdR8MbaUj", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WqOVlf8tFrDU" - }, - "source": [ - "### TFRecord 文件" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y1llOTwWFzmR" - }, - "source": [ - "#### 原始图片数据\n", - "\n", - "TFRecord 文件是一种用来存储一串二进制 blob 的简单格式。通过将多个示例打包进同一个文件内，TensorFlow 能够一次性读取多个示例，当使用一个远程存储服务，如 GCS 时，这对性能来说尤其重要。\n", - "\n", - "首先，从原始图片数据中构建出一个 TFRecord 文件：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EqtARqKuHQLu", - "colab": {} - }, - "source": [ - "image_ds = tf.data.Dataset.from_tensor_slices(all_image_paths).map(tf.io.read_file)\n", - "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", - "tfrec.write(image_ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "flR2GXWFKcO1" - }, - "source": [ - "接着，构建一个从 TFRecord 文件读取的数据集，并使用你之前定义的 `preprocess_image` 函数对图像进行解码/重新格式化：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "j9PVUL2SFufn", - "colab": {} - }, - "source": [ - "image_ds = tf.data.TFRecordDataset('images.tfrec').map(preprocess_image)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cRp1eZDRKzyN" - }, - "source": [ - "压缩该数据集和你之前定义的标签数据集以得到期望的 `(图片,标签)` 对：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7XI_nDU2KuhS", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.zip((image_ds, label_ds))\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "3ReSapoPK22E", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wb7VyoKNOMms" - }, - "source": [ - "这比 `缓存` 版本慢，因为你还没有缓存预处理。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NF9W-CTKkM-f" - }, - "source": [ - "#### 序列化的 Tensor（张量）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J9HzljSPkxt0" - }, - "source": [ - "要为 TFRecord 文件省去一些预处理过程，首先像之前一样制作一个处理过的图片数据集：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OzS0Azukkjyw", - "colab": {} - }, - "source": [ - "paths_ds = tf.data.Dataset.from_tensor_slices(all_image_paths)\n", - "image_ds = paths_ds.map(load_and_preprocess_image)\n", - "image_ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "onWOwLpYlzJQ" - }, - "source": [ - "现在你有一个 tensor（张量）数据集，而不是一个 `.jpeg` 字符串数据集。\n", - "\n", - "要将此序列化至一个 TFRecord 文件你首先将该 tensor（张量）数据集转化为一个字符串数据集：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xxZSwnRllyf0", - "colab": {} - }, - "source": [ - "ds = image_ds.map(tf.io.serialize_tensor)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "w9N6hJWAkKPC", - "colab": {} - }, - "source": [ - "tfrec = tf.data.experimental.TFRecordWriter('images.tfrec')\n", - "tfrec.write(ds)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OlFc9dJSmcx0" - }, - "source": [ - "有了被缓存的预处理，就能从 TFrecord 文件高效地加载数据——只需记得在使用它之前反序列化：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BsqFyTBFmSCZ", - "colab": {} - }, - "source": [ - "ds = tf.data.TFRecordDataset('images.tfrec')\n", - "\n", - "def parse(x):\n", - " result = tf.io.parse_tensor(x, out_type=tf.float32)\n", - " result = tf.reshape(result, [192, 192, 3])\n", - " return result\n", - "\n", - "ds = ds.map(parse, num_parallel_calls=AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OPs_sLV9pQg5" - }, - "source": [ - "现在，像之前一样添加标签和进行相同的标准操作：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XYxBwaLYnGop", - "colab": {} - }, - "source": [ - "ds = tf.data.Dataset.zip((ds, label_ds))\n", - "ds = ds.apply(\n", - " tf.data.experimental.shuffle_and_repeat(buffer_size=image_count))\n", - "ds=ds.batch(BATCH_SIZE).prefetch(AUTOTUNE)\n", - "ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W8X6RmGan1-P", - "colab": {} - }, - "source": [ - "timeit(ds)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/zh-cn/tutorials/load_data/numpy.ipynb b/site/zh-cn/tutorials/load_data/numpy.ipynb deleted file mode 100644 index 80d9062e0e5..00000000000 --- a/site/zh-cn/tutorials/load_data/numpy.ipynb +++ /dev/null @@ -1,326 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "numpy.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "pixDvex9KBqt" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "K16pBM8mKK7a", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TfRdquslKbO3" - }, - "source": [ - "# 使用 tf.data 加载 NumPy 数据" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-uq3F0ggKlZb" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 Tensorflow.org 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GEe3i16tQPjo", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-0tqX8qkXZEj" - }, - "source": [ - "本教程提供了将数据从 NumPy 数组加载到 `tf.data.Dataset` 的示例\n", - "本示例从一个 `.npz` 文件中加载 MNIST 数据集。但是，本实例中 NumPy 数据的来源并不重要。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-Ze5IBx9clLB" - }, - "source": [ - "## 安装" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "D1gtCQrnNk6b", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "k6J3JzK5NxQ6", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - " \n", - "import numpy as np\n", - "import tensorflow as tf\n", - "import tensorflow_datasets as tfds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "G0yWiN8-cpDb" - }, - "source": [ - "### 从 `.npz` 文件中加载" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GLHNrFM6RWoM", - "colab": {} - }, - "source": [ - "DATA_URL = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz'\n", - "\n", - "path = tf.keras.utils.get_file('mnist.npz', DATA_URL)\n", - "with np.load(path) as data:\n", - " train_examples = data['x_train']\n", - " train_labels = data['y_train']\n", - " test_examples = data['x_test']\n", - " test_labels = data['y_test']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cCeCkvrDgCMM" - }, - "source": [ - "## 使用 `tf.data.Dataset` 加载 NumPy 数组" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tslB0tJPgB-2" - }, - "source": [ - "假设您有一个示例数组和相应的标签数组，请将两个数组作为元组传递给 `tf.data.Dataset.from_tensor_slices` 以创建 `tf.data.Dataset` 。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QN_8wwc5R7Qm", - "colab": {} - }, - "source": [ - "train_dataset = tf.data.Dataset.from_tensor_slices((train_examples, train_labels))\n", - "test_dataset = tf.data.Dataset.from_tensor_slices((test_examples, test_labels))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6Rco85bbkDfN" - }, - "source": [ - "## 使用该数据集" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0dvl1uUukc4K" - }, - "source": [ - "### 打乱和批次化数据集" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GTXdRMPcSXZj", - "colab": {} - }, - "source": [ - "BATCH_SIZE = 64\n", - "SHUFFLE_BUFFER_SIZE = 100\n", - "\n", - "train_dataset = train_dataset.shuffle(SHUFFLE_BUFFER_SIZE).batch(BATCH_SIZE)\n", - "test_dataset = test_dataset.batch(BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "w69Jl8k6lilg" - }, - "source": [ - "### 建立和训练模型" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Uhxr8py4DkDN", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer=tf.keras.optimizers.RMSprop(),\n", - " loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n", - " metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XLDzlPGgOHBx", - "colab": {} - }, - "source": [ - "model.fit(train_dataset, epochs=10)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2q82yN8mmKIE", - "colab": {} - }, - "source": [ - "model.evaluate(test_dataset)" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/zh-cn/tutorials/load_data/pandas_dataframe.ipynb b/site/zh-cn/tutorials/load_data/pandas_dataframe.ipynb deleted file mode 100644 index 5bb8648a775..00000000000 --- a/site/zh-cn/tutorials/load_data/pandas_dataframe.ipynb +++ /dev/null @@ -1,497 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zwBCE43Cv3PH" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors.\n", - "\n", - "Licensed under the Apache License, Version 2.0 (the \"License\");" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "cellView": "form", - "colab": {}, - "colab_type": "code", - "id": "fOad0I2cv569" - }, - "outputs": [], - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YQB7yiF6v9GR" - }, - "source": [ - "# 使用 tf.data 加载 pandas dataframes" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Oqa952X4wQKK" - }, - "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://tensorflow.google.cn/tutorials/load_data/pandas_dataframe\"\u003e\u003cimg src=\"https://tensorflow.google.cn/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/zh-cn/tutorials/load_data/pandas_dataframe.ipynb\"\u003e\u003cimg src=\"https://tensorflow.google.cn/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/zh-cn/tutorials/load_data/pandas_dataframe.ipynb\"\u003e\u003cimg src=\"https://tensorflow.google.cn/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/zh-cn/tutorials/load_data/pandas_dataframe.ipynb\"\u003e\u003cimg src=\"https://tensorflow.google.cn/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yvUZdtkue89v" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UmyEaf4Awl2v" - }, - "source": [ - "本教程提供了如何将 pandas dataframes 加载到 `tf.data.Dataset`。\n", - "\n", - "本教程使用了一个小型[数据集](https://archive.ics.uci.edu/ml/datasets/heart+Disease)，由克利夫兰诊所心脏病基金会（Cleveland Clinic Foundation for Heart Disease）提供. 此数据集中有几百行CSV。每行表示一个患者，每列表示一个属性（describe）。我们将使用这些信息来预测患者是否患有心脏病，这是一个二分类问题。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "iiyC7HkqxlUD" - }, - "source": [ - "## 使用 pandas 读取数据" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "5IoRbCA2n0_V" - }, - "outputs": [], - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import pandas as pd\n", - "import tensorflow as tf" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-2kBGy_pxn47" - }, - "source": [ - "下载包含心脏数据集的 csv 文件。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "VS4w2LePn9g3" - }, - "outputs": [], - "source": [ - "csv_file = tf.keras.utils.get_file('heart.csv', 'https://storage.googleapis.com/applied-dl/heart.csv')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6BXRPD2-xtQ1" - }, - "source": [ - "使用 pandas 读取 csv 文件。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "UEfJ8TcMpe-2" - }, - "outputs": [], - "source": [ - "df = pd.read_csv(csv_file)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8FkK6QIRpjd4" - }, - "outputs": [], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "_MOAKz654CT5" - }, - "outputs": [], - "source": [ - "df.dtypes" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ww4lRDCS3qPh" - }, - "source": [ - "将 `thal` 列（数据帧（dataframe）中的 `object` ）转换为离散数值。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "LmCl5R5C2IKo" - }, - "outputs": [], - "source": [ - "df['thal'] = pd.Categorical(df['thal'])\n", - "df['thal'] = df.thal.cat.codes" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "s4XA1SNW2QyI" - }, - "outputs": [], - "source": [ - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WWRhH6r4xxQu" - }, - "source": [ - "## 使用 `tf.data.Dataset` 读取数据" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GuqmVVH_yApQ" - }, - "source": [ - "使用 `tf.data.Dataset.from_tensor_slices` 从 pandas dataframe 中读取数值。\n", - "\n", - "使用 `tf.data.Dataset` 的其中一个优势是可以允许您写一些简单而又高效的数据管道（data pipelines)。从 [loading data guide](https://www.tensorflow.org/guide/data) 可以了解更多。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "2wwhILm1ycSp" - }, - "outputs": [], - "source": [ - "target = df.pop('target')" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "W6Yc-D3aqyBb" - }, - "outputs": [], - "source": [ - "dataset = tf.data.Dataset.from_tensor_slices((df.values, target.values))" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "chEnp_Swsf0a" - }, - "outputs": [], - "source": [ - "for feat, targ in dataset.take(5):\n", - " print ('Features: {}, Target: {}'.format(feat, targ))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GzwlAhX6xH9Q" - }, - "source": [ - "由于 `pd.Series` 实现了 `__array__` 协议，因此几乎可以在任何使用 `np.array` 或 `tf.Tensor` 的地方透明地使用它。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "GnpHHkpktl5y" - }, - "outputs": [], - "source": [ - "tf.constant(df['thal'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "9XLxRHS10Ylp" - }, - "source": [ - "随机读取（shuffle）并批量处理数据集。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "R3dQ-83Ztsgl" - }, - "outputs": [], - "source": [ - "train_dataset = dataset.shuffle(len(df)).batch(1)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bB9C0XJkyQEk" - }, - "source": [ - "## 创建并训练模型" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FQd9PcPRpkP4" - }, - "outputs": [], - "source": [ - "def get_compiled_model():\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Dense(10, activation='relu'),\n", - " tf.keras.layers.Dense(10, activation='relu'),\n", - " tf.keras.layers.Dense(1, activation='sigmoid')\n", - " ])\n", - "\n", - " model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])\n", - " return model" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "ybDzNUheqxJw" - }, - "outputs": [], - "source": [ - "model = get_compiled_model()\n", - "model.fit(train_dataset, epochs=15)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "d6V_6F_MBiG9" - }, - "source": [ - "## 代替特征列" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "X63B9vDsD8Ly" - }, - "source": [ - "将字典作为输入传输给模型就像创建 `tf.keras.layers.Input` 层的匹配字典一样简单，应用任何预处理并使用 [functional api](../../guide/keras/functional.ipynb)。 您可以使用它作为 [feature columns](../keras/feature_columns.ipynb) 的替代方法。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "FwQ47_WmOBnY" - }, - "outputs": [], - "source": [ - "inputs = {key: tf.keras.layers.Input(shape=(), name=key) for key in df.keys()}\n", - "x = tf.stack(list(inputs.values()), axis=-1)\n", - "\n", - "x = tf.keras.layers.Dense(10, activation='relu')(x)\n", - "output = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n", - "\n", - "model_func = tf.keras.Model(inputs=inputs, outputs=output)\n", - "\n", - "model_func.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qSCN5f_vUURE" - }, - "source": [ - "与 `tf.data` 一起使用时，保存 `pd.DataFrame` 列结构的最简单方法是将 `pd.DataFrame` 转换为 `dict` ，并对该字典进行切片。" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "wUjRKgEhPZqK" - }, - "outputs": [], - "source": [ - "dict_slices = tf.data.Dataset.from_tensor_slices((df.to_dict('list'), target.values)).batch(16)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "WWRaiwxeyA9Z" - }, - "outputs": [], - "source": [ - "for dict_slice in dict_slices.take(1):\n", - " print (dict_slice)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "8nTrfczNyKup" - }, - "outputs": [], - "source": [ - "model_func.fit(dict_slices, epochs=15)" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "pandas_dataframe.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/site/zh-cn/tutorials/load_data/text.ipynb b/site/zh-cn/tutorials/load_data/text.ipynb deleted file mode 100644 index bf64931b32e..00000000000 --- a/site/zh-cn/tutorials/load_data/text.ipynb +++ /dev/null @@ -1,678 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "text.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DweYe9FcbMK_" - }, - "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "AVV2e0XKbJeX", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "sUtoed20cRJJ" - }, - "source": [ - "# 使用 tf.data 加载文本数据" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1ap_W4aQcgNT" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 TensorFlow.org 上查看\n", - " \n", - " 在 Google Colab 上运行\n", - " \n", - " 查看 GitHub 上的资源\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GEe3i16tQPjo" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://tensorflow.google.cn/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "NWeQAo0Ec_BL" - }, - "source": [ - "本教程为你提供了一个如何使用 `tf.data.TextLineDataset` 来加载文本文件的示例。`TextLineDataset` 通常被用来以文本文件构建数据集（原文件中的一行为一个样本) 。这适用于大多数的基于行的文本数据（例如，诗歌或错误日志) 。下面我们将使用相同作品（荷马的伊利亚特）三个不同版本的英文翻译，然后训练一个模型来通过单行文本确定译者。\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fgZ9gjmPfSnK" - }, - "source": [ - "## 环境搭建" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "baYFZMW_bJHh", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "!pip install tensorflow-gpu==2.0.0-rc1\n", - "import tensorflow as tf\n", - "\n", - "import tensorflow_datasets as tfds\n", - "import os" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YWVWjyIkffau" - }, - "source": [ - "三个版本的翻译分别来自于:\n", - "\n", - " - [William Cowper](https://en.wikipedia.org/wiki/William_Cowper) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/cowper.txt)\n", - "\n", - " - [Edward, Earl of Derby](https://en.wikipedia.org/wiki/Edward_Smith-Stanley,_14th_Earl_of_Derby) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/derby.txt)\n", - "\n", - "- [Samuel Butler](https://en.wikipedia.org/wiki/Samuel_Butler_%28novelist%29) — [text](https://storage.googleapis.com/download.tensorflow.org/data/illiad/butler.txt)\n", - "\n", - "本教程中使用的文本文件已经进行过一些典型的预处理，主要包括删除了文档页眉和页脚，行号，章节标题。请下载这些已经被局部改动过的文件。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4YlKQthEYlFw", - "colab": {} - }, - "source": [ - "DIRECTORY_URL = 'https://storage.googleapis.com/download.tensorflow.org/data/illiad/'\n", - "FILE_NAMES = ['cowper.txt', 'derby.txt', 'butler.txt']\n", - "\n", - "for name in FILE_NAMES:\n", - " text_dir = tf.keras.utils.get_file(name, origin=DIRECTORY_URL+name)\n", - " \n", - "parent_dir = os.path.dirname(text_dir)\n", - "\n", - "parent_dir" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "q3sDy6nuXoNp" - }, - "source": [ - "## 将文本加载到数据集中\n", - "\n", - "迭代整个文件，将整个文件加载到自己的数据集中。\n", - "\n", - "每个样本都需要单独标记，所以请使用 `tf.data.Dataset.map` 来为每个样本设定标签。这将迭代数据集中的每一个样本并且返回（ `example, label` ）对。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "K0BjCOpOh7Ch", - "colab": {} - }, - "source": [ - "def labeler(example, index):\n", - " return example, tf.cast(index, tf.int64) \n", - "\n", - "labeled_data_sets = []\n", - "\n", - "for i, file_name in enumerate(FILE_NAMES):\n", - " lines_dataset = tf.data.TextLineDataset(os.path.join(parent_dir, file_name))\n", - " labeled_dataset = lines_dataset.map(lambda ex: labeler(ex, i))\n", - " labeled_data_sets.append(labeled_dataset)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M8PHK5J_cXE5" - }, - "source": [ - "将这些标记的数据集合并到一个数据集中，然后对其进行随机化操作。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6jAeYkTIi9-2", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 50000\n", - "BATCH_SIZE = 64\n", - "TAKE_SIZE = 5000" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Qd544E-Sh63L", - "colab": {} - }, - "source": [ - "all_labeled_data = labeled_data_sets[0]\n", - "for labeled_dataset in labeled_data_sets[1:]:\n", - " all_labeled_data = all_labeled_data.concatenate(labeled_dataset)\n", - " \n", - "all_labeled_data = all_labeled_data.shuffle(\n", - " BUFFER_SIZE, reshuffle_each_iteration=False)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r4JEHrJXeG5k" - }, - "source": [ - "你可以使用 `tf.data.Dataset.take` 与 `print` 来查看 `(example, label)` 对的外观。`numpy` 属性显示每个 Tensor 的值。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gywKlN0xh6u5", - "colab": {} - }, - "source": [ - "for ex in all_labeled_data.take(5):\n", - " print(ex)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "5rrpU2_sfDh0" - }, - "source": [ - "## 将文本编码成数字\n", - "\n", - "机器学习基于的是数字而非文本，所以字符串需要被转化成数字列表。\n", - "为了达到此目的，我们需要构建文本与整数的一一映射。\n", - "\n", - "### 建立词汇表\n", - "\n", - "\n", - "首先，通过将文本标记为单独的单词集合来构建词汇表。在 TensorFlow 和 Python 中均有很多方法来达成这一目的。在本教程中:\n", - "\n", - "1. 迭代每个样本的 `numpy` 值。\n", - "2. 使用 `tfds.features.text.Tokenizer` 来将其分割成 `token`。\n", - "3. 将这些 `token` 放入一个 Python 集合中，借此来清除重复项。\n", - "4. 获取该词汇表的大小以便于以后使用。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YkHtbGnDh6mg", - "colab": {} - }, - "source": [ - "tokenizer = tfds.features.text.Tokenizer()\n", - "\n", - "vocabulary_set = set()\n", - "for text_tensor, _ in all_labeled_data:\n", - " some_tokens = tokenizer.tokenize(text_tensor.numpy())\n", - " vocabulary_set.update(some_tokens)\n", - "\n", - "vocab_size = len(vocabulary_set)\n", - "vocab_size" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0W35VJqAh9zs" - }, - "source": [ - "### 样本编码\n", - "\n", - "通过传递 `vocabulary_set` 到 `tfds.features.text.TokenTextEncoder` 来构建一个编码器。编码器的 `encode` 方法传入一行文本，返回一个整数列表。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gkxJIVAth6j0", - "colab": {} - }, - "source": [ - "encoder = tfds.features.text.TokenTextEncoder(vocabulary_set)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "v6S5Qyabi-vo" - }, - "source": [ - "你可以尝试运行这一行代码并查看输出的样式。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jgxPZaxUuTbk", - "colab": {} - }, - "source": [ - "example_text = next(iter(all_labeled_data))[0].numpy()\n", - "print(example_text)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "XoVpKR3qj5yb", - "colab": {} - }, - "source": [ - "encoded_example = encoder.encode(example_text)\n", - "print(encoded_example)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p9qHM0v8k_Mg" - }, - "source": [ - "现在，在数据集上运行编码器（通过将编码器打包到 `tf.py_function` 并且传参至数据集的 `map` 方法的方式来运行）。 " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "HcIQ7LOTh6eT", - "colab": {} - }, - "source": [ - "def encode(text_tensor, label):\n", - " encoded_text = encoder.encode(text_tensor.numpy())\n", - " return encoded_text, label\n", - "\n", - "def encode_map_fn(text, label):\n", - " return tf.py_function(encode, inp=[text, label], Tout=(tf.int64, tf.int64))\n", - "\n", - "all_encoded_data = all_labeled_data.map(encode_map_fn)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_YZToSXSm0qr" - }, - "source": [ - "## 将数据集分割为测试集和训练集且进行分支\n", - "\n", - "使用 `tf.data.Dataset.take` 和 `tf.data.Dataset.skip` 来建立一个小一些的测试数据集和稍大一些的训练数据集。\n", - "\n", - "在数据集被传入模型之前，数据集需要被分批。最典型的是，每个分支中的样本大小与格式需要一致。但是数据集中样本并不全是相同大小的（每行文本字数并不相同）。因此，使用 `tf.data.Dataset.padded_batch`（而不是 `batch` ）将样本填充到相同的大小。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "r-rmbijQh6bf", - "colab": {} - }, - "source": [ - "train_data = all_encoded_data.skip(TAKE_SIZE).shuffle(BUFFER_SIZE)\n", - "train_data = train_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))\n", - "\n", - "test_data = all_encoded_data.take(TAKE_SIZE)\n", - "test_data = test_data.padded_batch(BATCH_SIZE, padded_shapes=([-1],[]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Xdz7SVwmqi1l" - }, - "source": [ - "\n", - "现在，test_data 和 train_data 不是（ `example, label` ）对的集合，而是批次的集合。每个批次都是一对（*多样本*, *多标签* ），表示为数组。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kMslWfuwoqpB", - "colab": {} - }, - "source": [ - "sample_text, sample_labels = next(iter(test_data))\n", - "\n", - "sample_text[0], sample_labels[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UI4I6_Sa0vWu" - }, - "source": [ - "\n", - "由于我们引入了一个新的 token 来编码（填充零），因此词汇表大小增加了一个。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IlD1Lli91vuc", - "colab": {} - }, - "source": [ - "vocab_size += 1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K8SUhGFNsmRi" - }, - "source": [ - "## 建立模型\n", - "\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QJgI1pow2YR9", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wi0iiKLTKdoF" - }, - "source": [ - "第一层将整数表示转换为密集矢量嵌入。更多内容请查阅 [Word Embeddings](../../tutorials/sequences/word_embeddings) 教程。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DR6-ctbY638P", - "colab": {} - }, - "source": [ - "model.add(tf.keras.layers.Embedding(vocab_size, 64))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_8OJOPohKh1q" - }, - "source": [ - "\n", - "下一层是 [LSTM](http://colah.github.io/posts/2015-08-Understanding-LSTMs/) 层，它允许模型利用上下文中理解单词含义。 LSTM 上的双向包装器有助于模型理解当前数据点与其之前和之后的数据点的关系。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "x6rnq6DN_WUs", - "colab": {} - }, - "source": [ - "model.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "cdffbMr5LF1g" - }, - "source": [ - "\n", - "最后，我们将获得一个或多个紧密连接的层，其中最后一层是输出层。输出层输出样本属于各个标签的概率，最后具有最高概率的分类标签即为最终预测结果。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QTEaNSnLCsv5", - "colab": {} - }, - "source": [ - "# 一个或多个紧密连接的层\n", - "# 编辑 `for` 行的列表去检测层的大小\n", - "for units in [64, 64]:\n", - " model.add(tf.keras.layers.Dense(units, activation='relu'))\n", - "\n", - "# 输出层。第一个参数是标签个数。\n", - "model.add(tf.keras.layers.Dense(3, activation='softmax'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zLHPU8q5DLi_" - }, - "source": [ - "最后，编译这个模型。对于一个 softmax 分类模型来说，通常使用 `sparse_categorical_crossentropy` 作为其损失函数。你可以尝试其他的优化器，但是 `adam` 是最常用的。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pkTBUVO4h6Y5", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DM-HLo5NDhql" - }, - "source": [ - "## 训练模型\n", - "\n", - "利用提供的数据训练出的模型有着不错的精度（大约 83% ）。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aLtO33tNh6V8", - "colab": {} - }, - "source": [ - "model.fit(train_data, epochs=3, validation_data=test_data)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KTPCYf_Jh6TH", - "colab": {} - }, - "source": [ - "eval_loss, eval_acc = model.evaluate(test_data)\n", - "\n", - "print('\\nEval loss: {}, Eval accuracy: {}'.format(eval_loss, eval_acc))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} diff --git a/site/zh-cn/tutorials/quickstart/advanced.ipynb b/site/zh-cn/tutorials/quickstart/advanced.ipynb deleted file mode 100644 index 62135d9b0be..00000000000 --- a/site/zh-cn/tutorials/quickstart/advanced.ipynb +++ /dev/null @@ -1,423 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "advanced.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.2" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# 针对专业人员的 TensorFlow 2.0 入门" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 中运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XBr8cc2zy4NQ", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "这是一个 [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) 笔记本（notebook）文件。Python 程序直接在浏览器中运行——这是一种学习和使用 Tensorflow 的好方法。要学习本教程，请单击本页顶部按钮,在 Google Colab 中运行笔记本（notebook）.\n", - "\n", - "1. 在 Colab 中，连接到 Python 运行时：在菜单栏右上角，选择*连接（CONNECT）*。\n", - "2. 运行所有笔记本（notebook）代码单元格：选择*运行时（Runtime）* > *运行所有（Run all）*。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "eOsVdx6GGHmU" - }, - "source": [ - "下载并安装 TensorFlow 2.0 Beta 软件包：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ioLbtB3uGKPX", - "colab": {} - }, - "source": [ - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QS7DDTiZGRTo" - }, - "source": [ - "将 Tensorflow 导入您的程序：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow.keras.layers import Dense, Flatten, Conv2D\n", - "from tensorflow.keras import Model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "加载并准备 [MNIST 数据集](http://yann.lecun.com/exdb/mnist/)。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JqFRS6K07jJs", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0\n", - "\n", - "# Add a channels dimension\n", - "x_train = x_train[..., tf.newaxis]\n", - "x_test = x_test[..., tf.newaxis]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "k1Evqx0S22r_" - }, - "source": [ - "使用 `tf.data` 来将数据集切分为 batch 以及混淆数据集：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8Iu_quO024c2", - "colab": {} - }, - "source": [ - "train_ds = tf.data.Dataset.from_tensor_slices(\n", - " (x_train, y_train)).shuffle(10000).batch(32)\n", - "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "使用 Keras [模型子类化（model subclassing） API](https://www.tensorflow.org/guide/keras#model_subclassing) 构建 `tf.keras` 模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "class MyModel(Model):\n", - " def __init__(self):\n", - " super(MyModel, self).__init__()\n", - " self.conv1 = Conv2D(32, 3, activation='relu')\n", - " self.flatten = Flatten()\n", - " self.d1 = Dense(128, activation='relu')\n", - " self.d2 = Dense(10, activation='softmax')\n", - "\n", - " def call(self, x):\n", - " x = self.conv1(x)\n", - " x = self.flatten(x)\n", - " x = self.d1(x)\n", - " return self.d2(x)\n", - "\n", - "model = MyModel()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uGih-c2LgbJu" - }, - "source": [ - "为训练选择优化器与损失函数：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "u48C9WQ774n4", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n", - "\n", - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JB6A1vcigsIe" - }, - "source": [ - "选择衡量指标来度量模型的损失值（loss）和准确率（accuracy）。这些指标在 epoch 上累积值，然后打印出整体结果。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "N0MqHFb4F_qn", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n", - "\n", - "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", - "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "使用 `tf.GradientTape` 来训练模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OZACiVqA8KQV", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(images, labels):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(images)\n", - " loss = loss_object(labels, predictions)\n", - " gradients = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n", - "\n", - " train_loss(loss)\n", - " train_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z8YT7UmFgpjV" - }, - "source": [ - "测试模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xIKdEzHAJGt7", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def test_step(images, labels):\n", - " predictions = model(images)\n", - " t_loss = loss_object(labels, predictions)\n", - "\n", - " test_loss(t_loss)\n", - " test_accuracy(labels, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "i-2pkctU_Ci7", - "colab": {} - }, - "source": [ - "EPOCHS = 5\n", - "\n", - "for epoch in range(EPOCHS):\n", - " for images, labels in train_ds:\n", - " train_step(images, labels)\n", - "\n", - " for test_images, test_labels in test_ds:\n", - " test_step(test_images, test_labels)\n", - "\n", - " template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'\n", - " print (template.format(epoch+1,\n", - " train_loss.result(),\n", - " train_accuracy.result()*100,\n", - " test_loss.result(),\n", - " test_accuracy.result()*100))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "该图片分类器现在在此数据集上训练得到了接近 98% 的准确率（accuracy）。要了解更多信息，请阅读 [TensorFlow 教程](https://www.tensorflow.org/tutorials/keras)。" - ] - } - ] -} diff --git a/site/zh-cn/tutorials/quickstart/beginner.ipynb b/site/zh-cn/tutorials/quickstart/beginner.ipynb deleted file mode 100644 index 33779391417..00000000000 --- a/site/zh-cn/tutorials/quickstart/beginner.ipynb +++ /dev/null @@ -1,238 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "beginner.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rX8mhOLljYeM" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "BZSlp3DAjdYf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3wF5wszaj97Y" - }, - "source": [ - "# 初学者的 TensorFlow 2.0 教程" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DUNzJc4jTj6G" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 TensorFlow.org 观看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 GitHub 查看源代码\n", - " \n", - " 下载笔记本\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GEe3i16tQPjo", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiH7AC-NTniF" - }, - "source": [ - "这是一个 [Google Colaboratory](https://colab.research.google.com/notebooks/welcome.ipynb) 笔记本文件。 Python程序可以直接在浏览器中运行，这是学习 Tensorflow 的绝佳方式。想要学习该教程，请点击此页面顶部的按钮，在Google Colab中运行笔记本。\n", - "\n", - "1. 在 Colab中, 连接到Python运行环境： 在菜单条的右上方, 选择 *CONNECT*。\n", - "2. 运行所有的代码块: 选择 *Runtime* > *Run all*。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nnrWf3PCEzXL" - }, - "source": [ - "下载并安装 TensorFlow 2.0 测试版包。将 TensorFlow 载入你的程序：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0trJmd6DjqBZ", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "# 安装 TensorFlow\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "\n", - "import tensorflow as tf" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7NAbSZiaoJ4z" - }, - "source": [ - "载入并准备好 [MNIST 数据集](http://yann.lecun.com/exdb/mnist/)。将样本从整数转换为浮点数：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7FP5258xjs-v", - "colab": {} - }, - "source": [ - "mnist = tf.keras.datasets.mnist\n", - "\n", - "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n", - "x_train, x_test = x_train / 255.0, x_test / 255.0" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BPZ68wASog_I" - }, - "source": [ - "将模型的各层堆叠起来，以搭建 `tf.keras.Sequential` 模型。为训练选择优化器和损失函数：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "h3IKyzTCDNGo", - "colab": {} - }, - "source": [ - "model = tf.keras.models.Sequential([\n", - " tf.keras.layers.Flatten(input_shape=(28, 28)),\n", - " tf.keras.layers.Dense(128, activation='relu'),\n", - " tf.keras.layers.Dropout(0.2),\n", - " tf.keras.layers.Dense(10, activation='softmax')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='sparse_categorical_crossentropy',\n", - " metrics=['accuracy'])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ix4mEL65on-w" - }, - "source": [ - "训练并验证模型：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "F7dTAzgHDUh7", - "colab": {} - }, - "source": [ - "model.fit(x_train, y_train, epochs=5)\n", - "\n", - "model.evaluate(x_test, y_test, verbose=2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "T4JfEh7kvx6m" - }, - "source": [ - "现在，这个照片分类器的准确度已经达到 98%。想要了解更多，请阅读 [TensorFlow 教程](https://www.tensorflow.org/tutorials/)。" - ] - } - ] -} diff --git a/site/zh-cn/tutorials/structured_data/feature_columns.ipynb b/site/zh-cn/tutorials/structured_data/feature_columns.ipynb deleted file mode 100644 index ae1d4bd85b7..00000000000 --- a/site/zh-cn/tutorials/structured_data/feature_columns.ipynb +++ /dev/null @@ -1,730 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "feature_columns.ipynb", - "version": "0.3.2", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rNdWfPXCjTjY" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "I1dUQ0GejU8N", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "c05P9g5WjizZ" - }, - "source": [ - "# 对结构化数据进行分类" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zofH_gCzgplN" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " \n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " \n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Zz7U1iGCwp7T", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "K1y4OHpGgss7" - }, - "source": [ - "本教程演示了如何对结构化数据进行分类（例如，CSV 中的表格数据）。我们将使用 [Keras](https://tensorflow.google.cn/guide/keras) 来定义模型，将[特征列（feature columns）](https://tensorflow.google.cn/guide/feature_columns) 作为从 CSV 中的列（columns）映射到用于训练模型的特征（features）的桥梁。本教程包括了以下内容的完整代码：\n", - "\n", - "* 用 [Pandas](https://pandas.pydata.org/) 导入 CSV 文件。\n", - "* 用 [tf.data](https://tensorflow.google.cn/guide/datasets) 建立了一个输入流水线（pipeline），用于对行进行分批（batch）和随机排序（shuffle）。\n", - "* 用特征列将 CSV 中的列映射到用于训练模型的特征。\n", - "* 用 Keras 构建，训练并评估模型。\n", - "\n", - "## 数据集\n", - "\n", - "我们将使用一个小型 [数据集](https://archive.ics.uci.edu/ml/datasets/heart+Disease)，该数据集由克利夫兰心脏病诊所基金会（Cleveland Clinic Foundation for Heart Disease）提供。CSV 中有几百行数据。每行描述了一个病人（patient），每列描述了一个属性（attribute）。我们将使用这些信息来预测一位病人是否患有心脏病，这是在该数据集上的二分类任务。\n", - "\n", - "下面是该数据集的[描述](https://archive.ics.uci.edu/ml/machine-learning-databases/heart-disease/heart-disease.names)。 请注意，有数值（numeric）和类别（categorical）类型的列。\n", - "\n", - ">列| 描述| 特征类型 | 数据类型\n", - ">------------|--------------------|----------------------|-----------------\n", - ">Age | 年龄以年为单位 | Numerical | integer\n", - ">Sex | （1 = 男；0 = 女） | Categorical | integer\n", - ">CP | 胸痛类型（0，1，2，3，4）| Categorical | integer\n", - ">Trestbpd | 静息血压（入院时，以mm Hg计） | Numerical | integer\n", - ">Chol | 血清胆固醇（mg/dl） | Numerical | integer\n", - ">FBS |（空腹血糖> 120 mg/dl）（1 = true；0 = false）| Categorical | integer\n", - ">RestECG | 静息心电图结果（0，1，2）| Categorical | integer\n", - ">Thalach | 达到的最大心率 | Numerical | integer\n", - ">Exang | 运动诱发心绞痛（1 =是；0 =否）| Categorical | integer\n", - ">Oldpeak | 与休息时相比由运动引起的 ST 节段下降|Numerical | integer\n", - ">Slope | 在运动高峰 ST 段的斜率 | Numerical | float\n", - ">CA | 荧光透视法染色的大血管动脉（0-3）的数量 | Numerical | integer\n", - ">Thal | 3 =正常；6 =固定缺陷；7 =可逆缺陷 | Categorical | string\n", - ">Target | 心脏病诊断（1 = true；0 = false） | Classification | integer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "VxyBFc_kKazA" - }, - "source": [ - "## 导入 TensorFlow 和其他库" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LuOWVJBz8a6G", - "colab": {} - }, - "source": [ - "!pip install sklearn" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9dEreb4QKizj", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "try:\n", - " # Colab only\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "from tensorflow import feature_column\n", - "from tensorflow.keras import layers\n", - "from sklearn.model_selection import train_test_split" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "KCEhSZcULZ9n" - }, - "source": [ - "## 使用 Pandas 创建一个 dataframe\n", - "\n", - "[Pandas](https://pandas.pydata.org/) 是一个 Python 库，它有许多有用的实用程序，用于加载和处理结构化数据。我们将使用 Pandas 从 URL下载数据集，并将其加载到 dataframe 中。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "REZ57BXCLdfG", - "colab": {} - }, - "source": [ - "URL = 'https://storage.googleapis.com/applied-dl/heart.csv'\n", - "dataframe = pd.read_csv(URL)\n", - "dataframe.head()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "u0zhLtQqMPem" - }, - "source": [ - "## 将 dataframe 拆分为训练、验证和测试集\n", - "\n", - "我们下载的数据集是一个 CSV 文件。 我们将其拆分为训练、验证和测试集。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YEOpw7LhMYsI", - "colab": {} - }, - "source": [ - "train, test = train_test_split(dataframe, test_size=0.2)\n", - "train, val = train_test_split(train, test_size=0.2)\n", - "print(len(train), 'train examples')\n", - "print(len(val), 'validation examples')\n", - "print(len(test), 'test examples')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "84ef46LXMfvu" - }, - "source": [ - "## 用 tf.data 创建输入流水线\n", - "\n", - "接下来，我们将使用 [tf.data](https://tensorflow.google.cn/guide/datasets) 包装 dataframe。这让我们能将特征列作为一座桥梁，该桥梁将 Pandas dataframe 中的列映射到用于训练模型的特征。如果我们使用一个非常大的 CSV 文件（非常大以至于它不能放入内存），我们将使用 tf.data 直接从磁盘读取它。本教程不涉及这一点。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "NkcaMYP-MsRe", - "colab": {} - }, - "source": [ - "# 一种从 Pandas Dataframe 创建 tf.data 数据集的实用程序方法（utility method）\n", - "def df_to_dataset(dataframe, shuffle=True, batch_size=32):\n", - " dataframe = dataframe.copy()\n", - " labels = dataframe.pop('target')\n", - " ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n", - " if shuffle:\n", - " ds = ds.shuffle(buffer_size=len(dataframe))\n", - " ds = ds.batch(batch_size)\n", - " return ds" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CXbbXkJvMy34", - "colab": {} - }, - "source": [ - "batch_size = 5 # 小批量大小用于演示\n", - "train_ds = df_to_dataset(train, batch_size=batch_size)\n", - "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", - "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qRLGSMDzM-dl" - }, - "source": [ - "## 理解输入流水线\n", - "\n", - "现在我们已经创建了输入流水线，让我们调用它来查看它返回的数据的格式。 我们使用了一小批量大小来保持输出的可读性。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CSBo3dUVNFc9", - "colab": {} - }, - "source": [ - "for feature_batch, label_batch in train_ds.take(1):\n", - " print('Every feature:', list(feature_batch.keys()))\n", - " print('A batch of ages:', feature_batch['age'])\n", - " print('A batch of targets:', label_batch )" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "OT5N6Se-NQsC" - }, - "source": [ - "我们可以看到数据集返回了一个字典，该字典从列名称（来自 dataframe）映射到 dataframe 中行的列值。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ttIvgLRaNoOQ" - }, - "source": [ - "## 演示几种特征列\n", - "TensorFlow 提供了多种特征列。本节中，我们将创建几类特征列，并演示特征列如何转换 dataframe 中的列。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mxwiHFHuNhmf", - "colab": {} - }, - "source": [ - "# 我们将使用该批数据演示几种特征列\n", - "example_batch = next(iter(train_ds))[0]" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0wfLB8Q3N3UH", - "colab": {} - }, - "source": [ - "# 用于创建一个特征列\n", - "# 并转换一批次数据的一个实用程序方法\n", - "def demo(feature_column):\n", - " feature_layer = layers.DenseFeatures(feature_column)\n", - " print(feature_layer(example_batch).numpy())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q7OEKe82N-Qb" - }, - "source": [ - "### 数值列\n", - "一个特征列的输出将成为模型的输入（使用上面定义的 demo 函数，我们将能准确地看到 dataframe 中的每列的转换方式）。 [数值列（numeric column）](https://tensorflow.google.cn/api_docs/python/tf/feature_column/numeric_column) 是最简单的列类型。它用于表示实数特征。使用此列时，模型将从 dataframe 中接收未更改的列值。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "QZTZ0HnHOCxC", - "colab": {} - }, - "source": [ - "age = feature_column.numeric_column(\"age\")\n", - "demo(age)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7a6ddSyzOKpq" - }, - "source": [ - "在这个心脏病数据集中，dataframe 中的大多数列都是数值列。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IcSxUoYgOlA1" - }, - "source": [ - "### 分桶列\n", - "通常，您不希望将数字直接输入模型，而是根据数值范围将其值分成不同的类别。考虑代表一个人年龄的原始数据。我们可以用 [分桶列（bucketized column）](https://tensorflow.google.cn/api_docs/python/tf/feature_column/bucketized_column)将年龄分成几个分桶（buckets），而不是将年龄表示成数值列。请注意下面的 one-hot 数值表示每行匹配的年龄范围。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wJ4Wt3SAOpTQ", - "colab": {} - }, - "source": [ - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "demo(age_buckets)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r1tArzewPb-b" - }, - "source": [ - "### 分类列\n", - "在此数据集中，thal 用字符串表示（如 'fixed'，'normal'，或 'reversible'）。我们无法直接将字符串提供给模型。相反，我们必须首先将它们映射到数值。分类词汇列（categorical vocabulary columns）提供了一种用 one-hot 向量表示字符串的方法（就像您在上面看到的年龄分桶一样）。词汇表可以用 [categorical_column_with_vocabulary_list](https://tensorflow.google.cn/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_list) 作为 list 传递，或者用 [categorical_column_with_vocabulary_file](https://tensorflow.google.cn/api_docs/python/tf/feature_column/categorical_column_with_vocabulary_file) 从文件中加载。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DJ6QnSHkPtOC", - "colab": {} - }, - "source": [ - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "demo(thal_one_hot)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "dxQloQ9jOoXL" - }, - "source": [ - "在更复杂的数据集中，许多列都是分类列（如 strings）。在处理分类数据时，特征列最有价值。尽管在该数据集中只有一列分类列，但我们将使用它来演示在处理其他数据集时，可以使用的几种重要的特征列。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LEFPjUr6QmwS" - }, - "source": [ - "### 嵌入列\n", - "假设我们不是只有几个可能的字符串，而是每个类别有数千（或更多）值。 由于多种原因，随着类别数量的增加，使用 one-hot 编码训练神经网络变得不可行。我们可以使用嵌入列来克服此限制。[嵌入列（embedding column）](https://tensorflow.google.cn/api_docs/python/tf/feature_column/embedding_column)将数据表示为一个低维度密集向量，而非多维的 one-hot 向量，该低维度密集向量可以包含任何数，而不仅仅是 0 或 1。嵌入的大小（在下面的示例中为 8）是必须调整的参数。\n", - "\n", - "关键点：当分类列具有许多可能的值时，最好使用嵌入列。我们在这里使用嵌入列用于演示目的，为此您有一个完整的示例，以在将来可以修改用于其他数据集。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hSlohmr2Q_UU", - "colab": {} - }, - "source": [ - "# 注意到嵌入列的输入是我们之前创建的类别列\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "demo(thal_embedding)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "urFCAvTVRMpB" - }, - "source": [ - "### 经过哈希处理的特征列\n", - "\n", - "表示具有大量数值的分类列的另一种方法是使用 [categorical_column_with_hash_bucket](https://tensorflow.google.cn/api_docs/python/tf/feature_column/categorical_column_with_hash_bucket)。该特征列计算输入的一个哈希值，然后选择一个 `hash_bucket_size` 分桶来编码字符串。使用此列时，您不需要提供词汇表，并且可以选择使 hash_buckets 的数量远远小于实际类别的数量以节省空间。\n", - "\n", - "关键点：该技术的一个重要缺点是可能存在冲突，不同的字符串被映射到同一个范围。实际上，无论如何，经过哈希处理的特征列对某些数据集都有效。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YHU_Aj2nRRDC", - "colab": {} - }, - "source": [ - "thal_hashed = feature_column.categorical_column_with_hash_bucket(\n", - " 'thal', hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(thal_hashed))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fB94M27DRXtZ" - }, - "source": [ - "### 组合的特征列\n", - "将多种特征组合到一个特征中，称为[特征组合（feature crosses）](https://developers.google.com/machine-learning/glossary/#feature_cross)，它让模型能够为每种特征组合学习单独的权重。此处，我们将创建一个 age 和 thal 组合的新特征。请注意，`crossed_column` 不会构建所有可能组合的完整列表（可能非常大）。相反，它由 `hashed_column` 支持，因此您可以选择表的大小。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "oaPVERd9Rep6", - "colab": {} - }, - "source": [ - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "demo(feature_column.indicator_column(crossed_feature))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ypkI9zx6Rj1q" - }, - "source": [ - "## 选择要使用的列\n", - "我们已经了解了如何使用几种类型的特征列。 现在我们将使用它们来训练模型。本教程的目标是向您展示使用特征列所需的完整代码（例如，机制）。我们任意地选择了几列来训练我们的模型。\n", - "\n", - "关键点：如果您的目标是建立一个准确的模型，请尝试使用您自己的更大的数据集，并仔细考虑哪些特征最有意义，以及如何表示它们。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4PlLY7fORuzA", - "colab": {} - }, - "source": [ - "feature_columns = []\n", - "\n", - "# 数值列\n", - "for header in ['age', 'trestbps', 'chol', 'thalach', 'oldpeak', 'slope', 'ca']:\n", - " feature_columns.append(feature_column.numeric_column(header))\n", - "\n", - "# 分桶列\n", - "age_buckets = feature_column.bucketized_column(age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])\n", - "feature_columns.append(age_buckets)\n", - "\n", - "# 分类列\n", - "thal = feature_column.categorical_column_with_vocabulary_list(\n", - " 'thal', ['fixed', 'normal', 'reversible'])\n", - "thal_one_hot = feature_column.indicator_column(thal)\n", - "feature_columns.append(thal_one_hot)\n", - "\n", - "# 嵌入列\n", - "thal_embedding = feature_column.embedding_column(thal, dimension=8)\n", - "feature_columns.append(thal_embedding)\n", - "\n", - "# 组合列\n", - "crossed_feature = feature_column.crossed_column([age_buckets, thal], hash_bucket_size=1000)\n", - "crossed_feature = feature_column.indicator_column(crossed_feature)\n", - "feature_columns.append(crossed_feature)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M-nDp8krS_ts" - }, - "source": [ - "### 建立一个新的特征层\n", - "现在我们已经定义了我们的特征列，我们将使用[密集特征（DenseFeatures）](https://tensorflow.google.cn/versions/r2.0/api_docs/python/tf/keras/layers/DenseFeatures)层将特征列输入到我们的 Keras 模型中。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6o-El1R2TGQP", - "colab": {} - }, - "source": [ - "feature_layer = tf.keras.layers.DenseFeatures(feature_columns)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "8cf6vKfgTH0U" - }, - "source": [ - "之前，我们使用一个小批量大小来演示特征列如何运转。我们将创建一个新的更大批量的输入流水线。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "gcemszoGSse_", - "colab": {} - }, - "source": [ - "batch_size = 32\n", - "train_ds = df_to_dataset(train, batch_size=batch_size)\n", - "val_ds = df_to_dataset(val, shuffle=False, batch_size=batch_size)\n", - "test_ds = df_to_dataset(test, shuffle=False, batch_size=batch_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bBx4Xu0eTXWq" - }, - "source": [ - "## 创建，编译和训练模型" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_YJPPb3xTPeZ", - "colab": {} - }, - "source": [ - "model = tf.keras.Sequential([\n", - " feature_layer,\n", - " layers.Dense(128, activation='relu'),\n", - " layers.Dense(128, activation='relu'),\n", - " layers.Dense(1, activation='sigmoid')\n", - "])\n", - "\n", - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'],\n", - " run_eagerly=True)\n", - "\n", - "model.fit(train_ds,\n", - " validation_data=val_ds,\n", - " epochs=5)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GnFmMOW0Tcaa", - "colab": {} - }, - "source": [ - "loss, accuracy = model.evaluate(test_ds)\n", - "print(\"Accuracy\", accuracy)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3bdfbq20V6zu" - }, - "source": [ - "关键点：通常使用更大更复杂的数据集进行深度学习，您将看到最佳结果。使用像这样的小数据集时，我们建议使用决策树或随机森林作为强有力的基准。本教程的目的不是训练一个准确的模型，而是演示处理结构化数据的机制，这样，在将来使用自己的数据集时，您有可以使用的代码作为起点。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SotnhVWuHQCw" - }, - "source": [ - "### 下一步\n", - "了解有关分类结构化数据的更多信息的最佳方法是亲自尝试。我们建议寻找另一个可以使用的数据集，并使用和上面相似的代码，训练一个模型，对其分类。要提高准确率，请仔细考虑模型中包含哪些特征，以及如何表示这些特征。" - ] - } - ] -} diff --git a/site/zh-cn/tutorials/text/nmt_with_attention.ipynb b/site/zh-cn/tutorials/text/nmt_with_attention.ipynb deleted file mode 100644 index a3d838535fa..00000000000 --- a/site/zh-cn/tutorials/text/nmt_with_attention.ipynb +++ /dev/null @@ -1,1049 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "nmt_with_attention.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "s_qNSzzyaCbD" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "jmjh290raIky", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J0Qjg6vuaHNt" - }, - "source": [ - "# 基于注意力的神经机器翻译" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AOpGoE2T-YXS" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " 在 TensorFlow.org 上查看\n", - " \n", - " \n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " \n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8dEwzVWg0f-E", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "CiwtNgENbx2g" - }, - "source": [ - "此笔记本训练一个将西班牙语翻译为英语的序列到序列（sequence to sequence，简写为 seq2seq）模型。此例子难度较高，需要对序列到序列模型的知识有一定了解。\n", - "\n", - "训练完此笔记本中的模型后，你将能够输入一个西班牙语句子，例如 *\"¿todavia estan en casa?\"*，并返回其英语翻译 *\"are you still at home?\"*\n", - "\n", - "对于一个简单的例子来说，翻译质量令人满意。但是更有趣的可能是生成的注意力图：它显示在翻译过程中，输入句子的哪些部分受到了模型的注意。\n", - "\n", - "\"spanish-english\n", - "\n", - "请注意：运行这个例子用一个 P100 GPU 需要花大约 10 分钟。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tnxXKDjq3jEL", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version 仅存在于 Colab\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.ticker as ticker\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "import unicodedata\n", - "import re\n", - "import numpy as np\n", - "import os\n", - "import io\n", - "import time" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wfodePkj3jEa" - }, - "source": [ - "## 下载和准备数据集\n", - "\n", - "我们将使用 http://www.manythings.org/anki/ 提供的一个语言数据集。这个数据集包含如下格式的语言翻译对：\n", - "\n", - "```\n", - "May I borrow this book?\t¿Puedo tomar prestado este libro?\n", - "```\n", - "\n", - "这个数据集中有很多种语言可供选择。我们将使用英语 - 西班牙语数据集。为方便使用，我们在谷歌云上提供了此数据集的一份副本。但是你也可以自己下载副本。下载完数据集后，我们将采取下列步骤准备数据：\n", - "\n", - "1. 给每个句子添加一个 *开始* 和一个 *结束* 标记（token）。\n", - "2. 删除特殊字符以清理句子。\n", - "3. 创建一个单词索引和一个反向单词索引（即一个从单词映射至 id 的词典和一个从 id 映射至单词的词典）。\n", - "4. 将每个句子填充（pad）到最大长度。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "kRVATYOgJs1b", - "colab": {} - }, - "source": [ - "# 下载文件\n", - "path_to_zip = tf.keras.utils.get_file(\n", - " 'spa-eng.zip', origin='http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip',\n", - " extract=True)\n", - "\n", - "path_to_file = os.path.dirname(path_to_zip)+\"/spa-eng/spa.txt\"" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "rd0jw-eC3jEh", - "colab": {} - }, - "source": [ - "# 将 unicode 文件转换为 ascii\n", - "def unicode_to_ascii(s):\n", - " return ''.join(c for c in unicodedata.normalize('NFD', s)\n", - " if unicodedata.category(c) != 'Mn')\n", - "\n", - "\n", - "def preprocess_sentence(w):\n", - " w = unicode_to_ascii(w.lower().strip())\n", - "\n", - " # 在单词与跟在其后的标点符号之间插入一个空格\n", - " # 例如： \"he is a boy.\" => \"he is a boy .\"\n", - " # 参考：https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n", - " w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n", - " w = re.sub(r'[\" \"]+', \" \", w)\n", - "\n", - " # 除了 (a-z, A-Z, \".\", \"?\", \"!\", \",\")，将所有字符替换为空格\n", - " w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n", - "\n", - " w = w.rstrip().strip()\n", - "\n", - " # 给句子加上开始和结束标记\n", - " # 以便模型知道何时开始和结束预测\n", - " w = ' ' + w + ' '\n", - " return w" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "opI2GzOt479E", - "colab": {} - }, - "source": [ - "en_sentence = u\"May I borrow this book?\"\n", - "sp_sentence = u\"¿Puedo tomar prestado este libro?\"\n", - "print(preprocess_sentence(en_sentence))\n", - "print(preprocess_sentence(sp_sentence).encode('utf-8'))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OHn4Dct23jEm", - "colab": {} - }, - "source": [ - "# 1. 去除重音符号\n", - "# 2. 清理句子\n", - "# 3. 返回这样格式的单词对：[ENGLISH, SPANISH]\n", - "def create_dataset(path, num_examples):\n", - " lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n", - "\n", - " word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')] for l in lines[:num_examples]]\n", - "\n", - " return zip(*word_pairs)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cTbSbBz55QtF", - "colab": {} - }, - "source": [ - "en, sp = create_dataset(path_to_file, None)\n", - "print(en[-1])\n", - "print(sp[-1])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "OmMZQpdO60dt", - "colab": {} - }, - "source": [ - "def max_length(tensor):\n", - " return max(len(t) for t in tensor)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bIOn8RCNDJXG", - "colab": {} - }, - "source": [ - "def tokenize(lang):\n", - " lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(\n", - " filters='')\n", - " lang_tokenizer.fit_on_texts(lang)\n", - "\n", - " tensor = lang_tokenizer.texts_to_sequences(lang)\n", - "\n", - " tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n", - " padding='post')\n", - "\n", - " return tensor, lang_tokenizer" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "eAY9k49G3jE_", - "colab": {} - }, - "source": [ - "def load_dataset(path, num_examples=None):\n", - " # 创建清理过的输入输出对\n", - " targ_lang, inp_lang = create_dataset(path, num_examples)\n", - "\n", - " input_tensor, inp_lang_tokenizer = tokenize(inp_lang)\n", - " target_tensor, targ_lang_tokenizer = tokenize(targ_lang)\n", - "\n", - " return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GOi42V79Ydlr" - }, - "source": [ - "### 限制数据集的大小以加快实验速度（可选）\n", - "\n", - "在超过 10 万个句子的完整数据集上训练需要很长时间。为了更快地训练，我们可以将数据集的大小限制为 3 万个句子（当然，翻译质量也会随着数据的减少而降低）：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "cnxC7q-j3jFD", - "colab": {} - }, - "source": [ - "# 尝试实验不同大小的数据集\n", - "num_examples = 30000\n", - "input_tensor, target_tensor, inp_lang, targ_lang = load_dataset(path_to_file, num_examples)\n", - "\n", - "# 计算目标张量的最大长度 （max_length）\n", - "max_length_targ, max_length_inp = max_length(target_tensor), max_length(input_tensor)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4QILQkOs3jFG", - "colab": {} - }, - "source": [ - "# 采用 80 - 20 的比例切分训练集和验证集\n", - "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n", - "\n", - "# 显示长度\n", - "print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lJPmLZGMeD5q", - "colab": {} - }, - "source": [ - "def convert(lang, tensor):\n", - " for t in tensor:\n", - " if t!=0:\n", - " print (\"%d ----> %s\" % (t, lang.index_word[t]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "VXukARTDd7MT", - "colab": {} - }, - "source": [ - "print (\"Input Language; index to word mapping\")\n", - "convert(inp_lang, input_tensor_train[0])\n", - "print ()\n", - "print (\"Target Language; index to word mapping\")\n", - "convert(targ_lang, target_tensor_train[0])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rgCLkfv5uO3d" - }, - "source": [ - "### 创建一个 tf.data 数据集" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "TqHsArVZ3jFS", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = len(input_tensor_train)\n", - "BATCH_SIZE = 64\n", - "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n", - "embedding_dim = 256\n", - "units = 1024\n", - "vocab_inp_size = len(inp_lang.word_index)+1\n", - "vocab_tar_size = len(targ_lang.word_index)+1\n", - "\n", - "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n", - "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qc6-NK1GtWQt", - "colab": {} - }, - "source": [ - "example_input_batch, example_target_batch = next(iter(dataset))\n", - "example_input_batch.shape, example_target_batch.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "TNfHIF71ulLu" - }, - "source": [ - "## 编写编码器 （encoder） 和解码器 （decoder） 模型\n", - "\n", - "实现一个基于注意力的编码器 - 解码器模型。关于这种模型，你可以阅读 TensorFlow 的 [神经机器翻译 (序列到序列) 教程](https://github.com/tensorflow/nmt)。本示例采用一组更新的 API。此笔记本实现了上述序列到序列教程中的 [注意力方程式](https://github.com/tensorflow/nmt#background-on-the-attention-mechanism)。下图显示了注意力机制为每个输入单词分配一个权重，然后解码器将这个权重用于预测句子中的下一个单词。下图和公式是 [Luong 的论文](https://arxiv.org/abs/1508.04025v5)中注意力机制的一个例子。\n", - "\n", - "\"attention\n", - "\n", - "输入经过编码器模型，编码器模型为我们提供形状为 *(批大小，最大长度，隐藏层大小)* 的编码器输出和形状为 *(批大小，隐藏层大小)* 的编码器隐藏层状态。\n", - "\n", - "下面是所实现的方程式：\n", - "\n", - "\"attention\n", - "\"attention\n", - "\n", - "本教程的编码器采用 [Bahdanau 注意力](https://arxiv.org/pdf/1409.0473.pdf)。在用简化形式编写之前，让我们先决定符号：\n", - "\n", - "* FC = 完全连接（密集）层\n", - "* EO = 编码器输出\n", - "* H = 隐藏层状态\n", - "* X = 解码器输入\n", - "\n", - "以及伪代码：\n", - "\n", - "* `score = FC(tanh(FC(EO) + FC(H)))`\n", - "* `attention weights = softmax(score, axis = 1)`。 Softmax 默认被应用于最后一个轴，但是这里我们想将它应用于 *第一个轴*, 因为分数 （score） 的形状是 *(批大小，最大长度，隐藏层大小)*。最大长度 （`max_length`） 是我们的输入的长度。因为我们想为每个输入分配一个权重，所以 softmax 应该用在这个轴上。\n", - "* `context vector = sum(attention weights * EO, axis = 1)`。选择第一个轴的原因同上。\n", - "* `embedding output` = 解码器输入 X 通过一个嵌入层。\n", - "* `merged vector = concat(embedding output, context vector)`\n", - "* 此合并后的向量随后被传送到 GRU\n", - "\n", - "每个步骤中所有向量的形状已在代码的注释中阐明：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "nZ2rI24i3jFg", - "colab": {} - }, - "source": [ - "class Encoder(tf.keras.Model):\n", - " def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n", - " super(Encoder, self).__init__()\n", - " self.batch_sz = batch_sz\n", - " self.enc_units = enc_units\n", - " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", - " self.gru = tf.keras.layers.GRU(self.enc_units,\n", - " return_sequences=True,\n", - " return_state=True,\n", - " recurrent_initializer='glorot_uniform')\n", - "\n", - " def call(self, x, hidden):\n", - " x = self.embedding(x)\n", - " output, state = self.gru(x, initial_state = hidden)\n", - " return output, state\n", - "\n", - " def initialize_hidden_state(self):\n", - " return tf.zeros((self.batch_sz, self.enc_units))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "60gSVh05Jl6l", - "colab": {} - }, - "source": [ - "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n", - "\n", - "# 样本输入\n", - "sample_hidden = encoder.initialize_hidden_state()\n", - "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n", - "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n", - "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "umohpBN2OM94", - "colab": {} - }, - "source": [ - "class BahdanauAttention(tf.keras.layers.Layer):\n", - " def __init__(self, units):\n", - " super(BahdanauAttention, self).__init__()\n", - " self.W1 = tf.keras.layers.Dense(units)\n", - " self.W2 = tf.keras.layers.Dense(units)\n", - " self.V = tf.keras.layers.Dense(1)\n", - "\n", - " def call(self, query, values):\n", - " # 隐藏层的形状 == （批大小，隐藏层大小）\n", - " # hidden_with_time_axis 的形状 == （批大小，1，隐藏层大小）\n", - " # 这样做是为了执行加法以计算分数 \n", - " hidden_with_time_axis = tf.expand_dims(query, 1)\n", - "\n", - " # 分数的形状 == （批大小，最大长度，1）\n", - " # 我们在最后一个轴上得到 1， 因为我们把分数应用于 self.V\n", - " # 在应用 self.V 之前，张量的形状是（批大小，最大长度，单位）\n", - " score = self.V(tf.nn.tanh(\n", - " self.W1(values) + self.W2(hidden_with_time_axis)))\n", - "\n", - " # 注意力权重 （attention_weights） 的形状 == （批大小，最大长度，1）\n", - " attention_weights = tf.nn.softmax(score, axis=1)\n", - "\n", - " # 上下文向量 （context_vector） 求和之后的形状 == （批大小，隐藏层大小）\n", - " context_vector = attention_weights * values\n", - " context_vector = tf.reduce_sum(context_vector, axis=1)\n", - "\n", - " return context_vector, attention_weights" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "k534zTHiDjQU", - "colab": {} - }, - "source": [ - "attention_layer = BahdanauAttention(10)\n", - "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n", - "\n", - "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n", - "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yJ_B3mhW3jFk", - "colab": {} - }, - "source": [ - "class Decoder(tf.keras.Model):\n", - " def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n", - " super(Decoder, self).__init__()\n", - " self.batch_sz = batch_sz\n", - " self.dec_units = dec_units\n", - " self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n", - " self.gru = tf.keras.layers.GRU(self.dec_units,\n", - " return_sequences=True,\n", - " return_state=True,\n", - " recurrent_initializer='glorot_uniform')\n", - " self.fc = tf.keras.layers.Dense(vocab_size)\n", - "\n", - " # 用于注意力\n", - " self.attention = BahdanauAttention(self.dec_units)\n", - "\n", - " def call(self, x, hidden, enc_output):\n", - " # 编码器输出 （enc_output） 的形状 == （批大小，最大长度，隐藏层大小）\n", - " context_vector, attention_weights = self.attention(hidden, enc_output)\n", - "\n", - " # x 在通过嵌入层后的形状 == （批大小，1，嵌入维度）\n", - " x = self.embedding(x)\n", - "\n", - " # x 在拼接 （concatenation） 后的形状 == （批大小，1，嵌入维度 + 隐藏层大小）\n", - " x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n", - "\n", - " # 将合并后的向量传送到 GRU\n", - " output, state = self.gru(x)\n", - "\n", - " # 输出的形状 == （批大小 * 1，隐藏层大小）\n", - " output = tf.reshape(output, (-1, output.shape[2]))\n", - "\n", - " # 输出的形状 == （批大小，vocab）\n", - " x = self.fc(output)\n", - "\n", - " return x, state, attention_weights" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "P5UY8wko3jFp", - "colab": {} - }, - "source": [ - "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n", - "\n", - "sample_decoder_output, _, _ = decoder(tf.random.uniform((64, 1)),\n", - " sample_hidden, sample_output)\n", - "\n", - "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_ch_71VbIRfK" - }, - "source": [ - "## 定义优化器和损失函数" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WmTHr5iV3jFr", - "colab": {} - }, - "source": [ - "optimizer = tf.keras.optimizers.Adam()\n", - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " from_logits=True, reduction='none')\n", - "\n", - "def loss_function(real, pred):\n", - " mask = tf.math.logical_not(tf.math.equal(real, 0))\n", - " loss_ = loss_object(real, pred)\n", - "\n", - " mask = tf.cast(mask, dtype=loss_.dtype)\n", - " loss_ *= mask\n", - "\n", - " return tf.reduce_mean(loss_)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DMVWzzsfNl4e" - }, - "source": [ - "## 检查点（基于对象保存）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Zj8bXQTgNwrF", - "colab": {} - }, - "source": [ - "checkpoint_dir = './training_checkpoints'\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n", - "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n", - " encoder=encoder,\n", - " decoder=decoder)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hpObfY22IddU" - }, - "source": [ - "## 训练\n", - "\n", - "1. 将 *输入* 传送至 *编码器*，编码器返回 *编码器输出* 和 *编码器隐藏层状态*。\n", - "2. 将编码器输出、编码器隐藏层状态和解码器输入（即 *开始标记*）传送至解码器。\n", - "3. 解码器返回 *预测* 和 *解码器隐藏层状态*。\n", - "4. 解码器隐藏层状态被传送回模型，预测被用于计算损失。\n", - "5. 使用 *教师强制 （teacher forcing）* 决定解码器的下一个输入。\n", - "6. *教师强制* 是将 *目标词* 作为 *下一个输入* 传送至解码器的技术。\n", - "7. 最后一步是计算梯度，并将其应用于优化器和反向传播。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sC9ArXSsVfqn", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(inp, targ, enc_hidden):\n", - " loss = 0\n", - "\n", - " with tf.GradientTape() as tape:\n", - " enc_output, enc_hidden = encoder(inp, enc_hidden)\n", - "\n", - " dec_hidden = enc_hidden\n", - "\n", - " dec_input = tf.expand_dims([targ_lang.word_index['']] * BATCH_SIZE, 1)\n", - "\n", - " # 教师强制 - 将目标词作为下一个输入\n", - " for t in range(1, targ.shape[1]):\n", - " # 将编码器输出 （enc_output） 传送至解码器\n", - " predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n", - "\n", - " loss += loss_function(targ[:, t], predictions)\n", - "\n", - " # 使用教师强制\n", - " dec_input = tf.expand_dims(targ[:, t], 1)\n", - "\n", - " batch_loss = (loss / int(targ.shape[1]))\n", - "\n", - " variables = encoder.trainable_variables + decoder.trainable_variables\n", - "\n", - " gradients = tape.gradient(loss, variables)\n", - "\n", - " optimizer.apply_gradients(zip(gradients, variables))\n", - "\n", - " return batch_loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ddefjBMa3jF0", - "colab": {} - }, - "source": [ - "EPOCHS = 10\n", - "\n", - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - "\n", - " enc_hidden = encoder.initialize_hidden_state()\n", - " total_loss = 0\n", - "\n", - " for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n", - " batch_loss = train_step(inp, targ, enc_hidden)\n", - " total_loss += batch_loss\n", - "\n", - " if batch % 100 == 0:\n", - " print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n", - " batch,\n", - " batch_loss.numpy()))\n", - " # 每 2 个周期（epoch），保存（检查点）一次模型\n", - " if (epoch + 1) % 2 == 0:\n", - " checkpoint.save(file_prefix = checkpoint_prefix)\n", - "\n", - " print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n", - " total_loss / steps_per_epoch))\n", - " print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "mU3Ce8M6I3rz" - }, - "source": [ - "## 翻译\n", - "\n", - "* 评估函数类似于训练循环，不同之处在于在这里我们不使用 *教师强制*。每个时间步的解码器输入是其先前的预测、隐藏层状态和编码器输出。\n", - "* 当模型预测 *结束标记* 时停止预测。\n", - "* 存储 *每个时间步的注意力权重*。\n", - "\n", - "请注意：对于一个输入，编码器输出仅计算一次。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "EbQpyYs13jF_", - "colab": {} - }, - "source": [ - "def evaluate(sentence):\n", - " attention_plot = np.zeros((max_length_targ, max_length_inp))\n", - "\n", - " sentence = preprocess_sentence(sentence)\n", - "\n", - " inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n", - " inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n", - " maxlen=max_length_inp,\n", - " padding='post')\n", - " inputs = tf.convert_to_tensor(inputs)\n", - "\n", - " result = ''\n", - "\n", - " hidden = [tf.zeros((1, units))]\n", - " enc_out, enc_hidden = encoder(inputs, hidden)\n", - "\n", - " dec_hidden = enc_hidden\n", - " dec_input = tf.expand_dims([targ_lang.word_index['']], 0)\n", - "\n", - " for t in range(max_length_targ):\n", - " predictions, dec_hidden, attention_weights = decoder(dec_input,\n", - " dec_hidden,\n", - " enc_out)\n", - "\n", - " # 存储注意力权重以便后面制图\n", - " attention_weights = tf.reshape(attention_weights, (-1, ))\n", - " attention_plot[t] = attention_weights.numpy()\n", - "\n", - " predicted_id = tf.argmax(predictions[0]).numpy()\n", - "\n", - " result += targ_lang.index_word[predicted_id] + ' '\n", - "\n", - " if targ_lang.index_word[predicted_id] == '':\n", - " return result, sentence, attention_plot\n", - "\n", - " # 预测的 ID 被输送回模型\n", - " dec_input = tf.expand_dims([predicted_id], 0)\n", - "\n", - " return result, sentence, attention_plot" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "s5hQWlbN3jGF", - "colab": {} - }, - "source": [ - "# 注意力权重制图函数\n", - "def plot_attention(attention, sentence, predicted_sentence):\n", - " fig = plt.figure(figsize=(10,10))\n", - " ax = fig.add_subplot(1, 1, 1)\n", - " ax.matshow(attention, cmap='viridis')\n", - "\n", - " fontdict = {'fontsize': 14}\n", - "\n", - " ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n", - " ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n", - "\n", - " ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n", - " ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n", - "\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "sl9zUHzg3jGI", - "colab": {} - }, - "source": [ - "def translate(sentence):\n", - " result, sentence, attention_plot = evaluate(sentence)\n", - "\n", - " print('Input: %s' % (sentence))\n", - " print('Predicted translation: {}'.format(result))\n", - "\n", - " attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n", - " plot_attention(attention_plot, sentence.split(' '), result.split(' '))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "n250XbnjOaqP" - }, - "source": [ - "## 恢复最新的检查点并验证" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UJpT9D5_OgP6", - "colab": {} - }, - "source": [ - "# 恢复检查点目录 （checkpoint_dir） 中最新的检查点\n", - "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WrAM0FDomq3E", - "colab": {} - }, - "source": [ - "translate(u'hace mucho frio aqui.')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zSx2iM36EZQZ", - "colab": {} - }, - "source": [ - "translate(u'esta es mi vida.')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "A3LLCx3ZE0Ls", - "colab": {} - }, - "source": [ - "translate(u'¿todavia estan en casa?')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DUQVLVqUE1YW", - "colab": {} - }, - "source": [ - "# 错误的翻译\n", - "translate(u'trata de averiguarlo.')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RTe5P5ioMJwN" - }, - "source": [ - "## 下一步\n", - "\n", - "* [下载一个不同的数据集](http://www.manythings.org/anki/)实验翻译，例如英语到德语或者英语到法语。\n", - "* 实验在更大的数据集上训练，或者增加训练周期。" - ] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/text/text_generation.ipynb b/site/zh-cn/tutorials/text/text_generation.ipynb deleted file mode 100644 index 647c1381dd7..00000000000 --- a/site/zh-cn/tutorials/text/text_generation.ipynb +++ /dev/null @@ -1,1248 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "text_generation.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t09eeeR5prIJ" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "GCCk8_dHpuNf", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ovpZyIhNIgoq" - }, - "source": [ - "# 循环神经网络（RNN）文本生成" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hcD2nPQvPOFM" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " 在 GitHub 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "BwpJ5IffzRG6" - }, - "source": [ - "本教程演示如何使用基于字符的 RNN 生成文本。我们将使用 Andrej Karpathy 在[《循环神经网络不合理的有效性》](http://karpathy.github.io/2015/05/21/rnn-effectiveness/)一文中提供的莎士比亚作品数据集。给定此数据中的一个字符序列 （“Shakespear”），训练一个模型以预测该序列的下一个字符（“e”）。通过重复调用该模型，可以生成更长的文本序列。\n", - "\n", - "请注意：启用 GPU 加速可以更快地执行此笔记本。在 Colab 中依次选择：*运行时 > 更改运行时类型 > 硬件加速器 > GPU*。如果在本地运行，请确保 TensorFlow 的版本为 1.11 或更高。\n", - "\n", - "本教程包含使用 [tf.keras](https://www.tensorflow.org/programmers_guide/keras) 和 [eager execution](https://www.tensorflow.org/programmers_guide/eager) 实现的可运行代码。以下是当本教程中的模型训练 30 个周期 （epoch），并以字符串 “Q” 开头时的示例输出：\n", - "\n", - "
    \n",
-        "QUEENE:\n",
-        "I had thought thou hadst a Roman; for the oracle,\n",
-        "Thus by All bids the man against the word,\n",
-        "Which are so weak of care, by old care done;\n",
-        "Your children were in your holy love,\n",
-        "And the precipitation through the bleeding throne.\n",
-        "\n",
-        "BISHOP OF ELY:\n",
-        "Marry, and will, my lord, to weep in such a one were prettiest;\n",
-        "Yet now I was adopted heir\n",
-        "Of the world's lamentable day,\n",
-        "To watch the next way with his father with his face?\n",
-        "\n",
-        "ESCALUS:\n",
-        "The cause why then we are all resolved more sons.\n",
-        "\n",
-        "VOLUMNIA:\n",
-        "O, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, no, it is no sin it should be dead,\n",
-        "And love and pale as any will to that word.\n",
-        "\n",
-        "QUEEN ELIZABETH:\n",
-        "But how long have I heard the soul for this world,\n",
-        "And show his hands of life be proved to stand.\n",
-        "\n",
-        "PETRUCHIO:\n",
-        "I say he look'd on, if I must be content\n",
-        "To stay him from the fatal of our country's bliss.\n",
-        "His lordship pluck'd from this sentence then for prey,\n",
-        "And then let us twain, being the moon,\n",
-        "were she such a case as fills m\n",
-        "
    \n", - "\n", - "虽然有些句子符合语法规则，但是大多数句子没有意义。这个模型尚未学习到单词的含义，但请考虑以下几点：\n", - "\n", - "* 此模型是基于字符的。训练开始时，模型不知道如何拼写一个英文单词，甚至不知道单词是文本的一个单位。\n", - "\n", - "* 输出文本的结构类似于剧本 -- 文本块通常以讲话者的名字开始；而且与数据集类似，讲话者的名字采用全大写字母。\n", - "\n", - "* 如下文所示，此模型由小批次 （batch） 文本训练而成（每批 100 个字符）。即便如此，此模型仍然能生成更长的文本序列，并且结构连贯。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "srXC6pLGLwS6" - }, - "source": [ - "## 设置" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "WGyKZj3bzf9p" - }, - "source": [ - "### 导入 TensorFlow 和其他库" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yG_n40gFzf9s", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " # %tensorflow_version 仅存在于 Colab\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow as tf\n", - "\n", - "import numpy as np\n", - "import os\n", - "import time" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "EHDoRoc5PKWz" - }, - "source": [ - "### 下载莎士比亚数据集\n", - "\n", - "修改下面一行代码，在你自己的数据上运行此代码。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "pD_55cOxLkAb", - "colab": {} - }, - "source": [ - "path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UHjdCjDuSvX_" - }, - "source": [ - "### 读取数据\n", - "\n", - "首先，看一看文本：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "aavnuByVymwK", - "colab": {} - }, - "source": [ - "# 读取并为 py2 compat 解码\n", - "text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n", - "\n", - "# 文本长度是指文本中的字符个数\n", - "print ('Length of text: {} characters'.format(len(text)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Duhg9NrUymwO", - "colab": {} - }, - "source": [ - "# 看一看文本中的前 250 个字符\n", - "print(text[:250])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "IlCgQBRVymwR", - "colab": {} - }, - "source": [ - "# 文本中的非重复字符\n", - "vocab = sorted(set(text))\n", - "print ('{} unique characters'.format(len(vocab)))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "rNnrKn_lL-IJ" - }, - "source": [ - "## 处理文本" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LFjSVAlWzf-N" - }, - "source": [ - "### 向量化文本\n", - "\n", - "在训练之前，我们需要将字符串映射到数字表示值。创建两个查找表格：一个将字符映射到数字，另一个将数字映射到字符。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "y0FUEx8jjmLk", - "colab": {} - }, - "source": [ - "# 创建从非重复字符到索引的映射\n", - "char2idx = {u:i for i, u in enumerate(vocab)}\n", - "idx2char = np.array(vocab)\n", - "\n", - "text_as_int = np.array([char2idx[c] for c in text])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "tZfqhkYCymwX" - }, - "source": [ - "现在，每个字符都有一个整数表示值。请注意，我们将字符映射至索引 0 至 `len(unique)`." - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "FYyNlCNXymwY", - "colab": {} - }, - "source": [ - "print('{')\n", - "for char,_ in zip(char2idx, range(20)):\n", - " print(' {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n", - "print(' ...\\n}')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "l1VKcQHcymwb", - "colab": {} - }, - "source": [ - "# 显示文本首 13 个字符的整数映射\n", - "print ('{} ---- characters mapped to int ---- > {}'.format(repr(text[:13]), text_as_int[:13]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "bbmsf23Bymwe" - }, - "source": [ - "### 预测任务" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wssHQ1oGymwe" - }, - "source": [ - "给定一个字符或者一个字符序列，下一个最可能出现的字符是什么？这就是我们训练模型要执行的任务。输入进模型的是一个字符序列，我们训练这个模型来预测输出 -- 每个时间步（time step）预测下一个字符是什么。\n", - "\n", - "由于 RNN 是根据前面看到的元素维持内部状态，那么，给定此时计算出的所有字符，下一个字符是什么？" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hgsVvVxnymwf" - }, - "source": [ - "### 创建训练样本和目标\n", - "\n", - "接下来，将文本划分为样本序列。每个输入序列包含文本中的 `seq_length` 个字符。\n", - "\n", - "对于每个输入序列，其对应的目标包含相同长度的文本，但是向右顺移一个字符。\n", - "\n", - "将文本拆分为长度为 `seq_length+1` 的文本块。例如，假设 `seq_length` 为 4 而且文本为 “Hello”， 那么输入序列将为 “Hell”，目标序列将为 “ello”。\n", - "\n", - "为此，首先使用 `tf.data.Dataset.from_tensor_slices` 函数把文本向量转换为字符索引流。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mQ1jrAoVjmLu", - "colab": {} - }, - "source": [ - "# 设定每个输入句子长度的最大值\n", - "seq_length = 100\n", - "examples_per_epoch = len(text)//seq_length\n", - "\n", - "# 创建训练样本 / 目标\n", - "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n", - "\n", - "for i in char_dataset.take(5):\n", - " print(idx2char[i.numpy()])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ZSYAcQV8OGP" - }, - "source": [ - "`batch` 方法使我们能轻松把单个字符转换为所需长度的序列。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "l4hkDU3i7ozi", - "colab": {} - }, - "source": [ - "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n", - "\n", - "for item in sequences.take(5):\n", - " print(repr(''.join(idx2char[item.numpy()])))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UbLcIPBj_mWZ" - }, - "source": [ - "对于每个序列，使用 `map` 方法先复制再顺移，以创建输入文本和目标文本。`map` 方法可以将一个简单的函数应用到每一个批次 （batch）。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9NGu-FkO_kYU", - "colab": {} - }, - "source": [ - "def split_input_target(chunk):\n", - " input_text = chunk[:-1]\n", - " target_text = chunk[1:]\n", - " return input_text, target_text\n", - "\n", - "dataset = sequences.map(split_input_target)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "hiCopyGZymwi" - }, - "source": [ - "打印第一批样本的输入与目标值：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "GNbw-iR0ymwj", - "colab": {} - }, - "source": [ - "for input_example, target_example in dataset.take(1):\n", - " print ('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n", - " print ('Target data:', repr(''.join(idx2char[target_example.numpy()])))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_33OHL3b84i0" - }, - "source": [ - "这些向量的每个索引均作为一个时间步来处理。作为时间步 0 的输入，模型接收到 “F” 的索引，并尝试预测 “i” 的索引为下一个字符。在下一个时间步，模型执行相同的操作，但是 `RNN` 不仅考虑当前的输入字符，还会考虑上一步的信息。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "0eBu9WZG84i0", - "colab": {} - }, - "source": [ - "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n", - " print(\"Step {:4d}\".format(i))\n", - " print(\" input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n", - " print(\" expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MJdfPmdqzf-R" - }, - "source": [ - "### 创建训练批次\n", - "\n", - "前面我们使用 `tf.data` 将文本拆分为可管理的序列。但是在把这些数据输送至模型之前，我们需要将数据重新排列 （shuffle） 并打包为批次。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "p2pGotuNzf-S", - "colab": {} - }, - "source": [ - "# 批大小\n", - "BATCH_SIZE = 64\n", - "\n", - "# 设定缓冲区大小，以重新排列数据集\n", - "# （TF 数据被设计为可以处理可能是无限的序列，\n", - "# 所以它不会试图在内存中重新排列整个序列。相反，\n", - "# 它维持一个缓冲区，在缓冲区重新排列元素。） \n", - "BUFFER_SIZE = 10000\n", - "\n", - "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n", - "\n", - "dataset" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "r6oUuElIMgVx" - }, - "source": [ - "## 创建模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "m8gPwEjRzf-Z" - }, - "source": [ - "使用 `tf.keras.Sequential` 定义模型。在这个简单的例子中，我们使用了三个层来定义模型：\n", - "\n", - "* `tf.keras.layers.Embedding`：输入层。一个可训练的对照表，它会将每个字符的数字映射到一个 `embedding_dim` 维度的向量。 \n", - "* `tf.keras.layers.GRU`：一种 RNN 类型，其大小由 `units=rnn_units` 指定（这里你也可以使用一个 LSTM 层）。\n", - "* `tf.keras.layers.Dense`：输出层，带有 `vocab_size` 个输出。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zHT8cLh7EAsg", - "colab": {} - }, - "source": [ - "# 词集的长度\n", - "vocab_size = len(vocab)\n", - "\n", - "# 嵌入的维度\n", - "embedding_dim = 256\n", - "\n", - "# RNN 的单元数量\n", - "rnn_units = 1024" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MtCrdfzEI2N0", - "colab": {} - }, - "source": [ - "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n", - " model = tf.keras.Sequential([\n", - " tf.keras.layers.Embedding(vocab_size, embedding_dim,\n", - " batch_input_shape=[batch_size, None]),\n", - " tf.keras.layers.GRU(rnn_units,\n", - " return_sequences=True,\n", - " stateful=True,\n", - " recurrent_initializer='glorot_uniform'),\n", - " tf.keras.layers.Dense(vocab_size)\n", - " ])\n", - " return model" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "wwsrpOik5zhv", - "colab": {} - }, - "source": [ - "model = build_model(\n", - " vocab_size = len(vocab),\n", - " embedding_dim=embedding_dim,\n", - " rnn_units=rnn_units,\n", - " batch_size=BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RkA5upJIJ7W7" - }, - "source": [ - "对于每个字符，模型会查找嵌入，把嵌入当作输入运行 GRU 一个时间步，并用密集层生成逻辑回归 （logits），预测下一个字符的对数可能性。\n", - "![数据在模型中传输的示意图](https://github.com/littlebeanbean7/docs/blob/master/site/en/tutorials/text/images/text_generation_training.png?raw=1)\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "-ubPo0_9Prjb" - }, - "source": [ - "## 试试这个模型\n", - "\n", - "现在运行这个模型，看看它是否按预期运行。\n", - "\n", - "首先检查输出的形状：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "C-_70kKAPrPU", - "colab": {} - }, - "source": [ - "for input_example_batch, target_example_batch in dataset.take(1):\n", - " example_batch_predictions = model(input_example_batch)\n", - " print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Q6NzLBi4VM4o" - }, - "source": [ - "在上面的例子中，输入的序列长度为 `100`， 但是这个模型可以在任何长度的输入上运行：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "vPGmAAXmVLGC", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uwv0gEkURfx1" - }, - "source": [ - "为了获得模型的实际预测，我们需要从输出分布中抽样，以获得实际的字符索引。这个分布是根据对字符集的逻辑回归定义的。\n", - "\n", - "请注意：从这个分布中 _抽样_ 很重要，因为取分布的 _最大值自变量点集（argmax）_ 很容易使模型卡在循环中。\n", - "\n", - "试试这个批次中的第一个样本：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4V4MfFg0RQJg", - "colab": {} - }, - "source": [ - "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n", - "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QM1Vbxs_URw5" - }, - "source": [ - "这使我们得到每个时间步预测的下一个字符的索引。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YqFMUQc_UFgM", - "colab": {} - }, - "source": [ - "sampled_indices" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LfLtsP3mUhCG" - }, - "source": [ - "解码它们，以查看此未经训练的模型预测的文本：" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "xWcFwPwLSo05", - "colab": {} - }, - "source": [ - "print(\"Input: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n", - "print()\n", - "print(\"Next Char Predictions: \\n\", repr(\"\".join(idx2char[sampled_indices ])))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LJL0Q0YPY6Ee" - }, - "source": [ - "## 训练模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YCbHQHiaa4Ic" - }, - "source": [ - "此时，这个问题可以被视为一个标准的分类问题：给定先前的 RNN 状态和这一时间步的输入，预测下一个字符的类别。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "trpqTWyvk0nr" - }, - "source": [ - "### 添加优化器和损失函数" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "UAjbjY03eiQ4" - }, - "source": [ - "标准的 `tf.keras.losses.sparse_categorical_crossentropy` 损失函数在这里适用，因为它被应用于预测的最后一个维度。\n", - "\n", - "因为我们的模型返回逻辑回归，所以我们需要设定命令行参数 `from_logits`。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4HrXTACTdzY-", - "colab": {} - }, - "source": [ - "def loss(labels, logits):\n", - " return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n", - "\n", - "example_batch_loss = loss(target_example_batch, example_batch_predictions)\n", - "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n", - "print(\"scalar_loss: \", example_batch_loss.numpy().mean())" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "jeOXriLcymww" - }, - "source": [ - "使用 `tf.keras.Model.compile` 方法配置训练步骤。我们将使用 `tf.keras.optimizers.Adam` 并采用默认参数，以及损失函数。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "DDl1_Een6rL0", - "colab": {} - }, - "source": [ - "model.compile(optimizer='adam', loss=loss)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ieSJdchZggUj" - }, - "source": [ - "### 配置检查点" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "C6XBUUavgF56" - }, - "source": [ - "使用 `tf.keras.callbacks.ModelCheckpoint` 来确保训练过程中保存检查点。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "W6fWTriUZP-n", - "colab": {} - }, - "source": [ - "# 检查点保存至的目录\n", - "checkpoint_dir = './training_checkpoints'\n", - "\n", - "# 检查点的文件名\n", - "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n", - "\n", - "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n", - " filepath=checkpoint_prefix,\n", - " save_weights_only=True)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "3Ky3F_BhgkTW" - }, - "source": [ - "### 执行训练" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "IxdOA-rgyGvs" - }, - "source": [ - "为保持训练时间合理，使用 10 个周期来训练模型。在 Colab 中，将运行时设置为 GPU 以加速训练。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7yGBE2zxMMHs", - "colab": {} - }, - "source": [ - "EPOCHS=10" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UK-hmKjYVoll", - "colab": {} - }, - "source": [ - "history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kKkD5M6eoSiN" - }, - "source": [ - "## 生成文本" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JIPcXllKjkdr" - }, - "source": [ - "### 恢复最新的检查点" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "LyeYRiuVjodY" - }, - "source": [ - "为保持此次预测步骤简单，将批大小设定为 1。\n", - "\n", - "由于 RNN 状态从时间步传递到时间步的方式，模型建立好之后只接受固定的批大小。\n", - "\n", - "若要使用不同的 `batch_size` 来运行模型，我们需要重建模型并从检查点中恢复权重。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zk2WJ2-XjkGz", - "colab": {} - }, - "source": [ - "tf.train.latest_checkpoint(checkpoint_dir)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LycQ-ot_jjyu", - "colab": {} - }, - "source": [ - "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n", - "\n", - "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n", - "\n", - "model.build(tf.TensorShape([1, None]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "71xa6jnYVrAN", - "colab": {} - }, - "source": [ - "model.summary()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "DjGz1tDkzf-u" - }, - "source": [ - "### 预测循环\n", - "\n", - "下面的代码块生成文本：\n", - "\n", - "* 首先设置起始字符串，初始化 RNN 状态并设置要生成的字符个数。\n", - "\n", - "* 用起始字符串和 RNN 状态，获取下一个字符的预测分布。\n", - "\n", - "* 然后，用分类分布计算预测字符的索引。把这个预测字符当作模型的下一个输入。\n", - "\n", - "* 模型返回的 RNN 状态被输送回模型。现在，模型有更多上下文可以学习，而非只有一个字符。在预测出下一个字符后，更改过的 RNN 状态被再次输送回模型。模型就是这样，通过不断从前面预测的字符获得更多上下文，进行学习。\n", - "\n", - "![为生成文本，模型的输出被输送回模型作为输入](https://github.com/littlebeanbean7/docs/blob/master/site/en/tutorials/text/images/text_generation_sampling.png?raw=1)\n", - "\n", - "查看生成的文本，你会发现这个模型知道什么时候使用大写字母，什么时候分段，而且模仿出了莎士比亚式的词汇。由于训练的周期小，模型尚未学会生成连贯的句子。\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WvuwZBX5Ogfd", - "colab": {} - }, - "source": [ - "def generate_text(model, start_string):\n", - " # 评估步骤（用学习过的模型生成文本）\n", - "\n", - " # 要生成的字符个数\n", - " num_generate = 1000\n", - "\n", - " # 将起始字符串转换为数字（向量化）\n", - " input_eval = [char2idx[s] for s in start_string]\n", - " input_eval = tf.expand_dims(input_eval, 0)\n", - "\n", - " # 空字符串用于存储结果\n", - " text_generated = []\n", - "\n", - " # 低温度会生成更可预测的文本\n", - " # 较高温度会生成更令人惊讶的文本\n", - " # 可以通过试验以找到最好的设定\n", - " temperature = 1.0\n", - "\n", - " # 这里批大小为 1\n", - " model.reset_states()\n", - " for i in range(num_generate):\n", - " predictions = model(input_eval)\n", - " # 删除批次的维度\n", - " predictions = tf.squeeze(predictions, 0)\n", - "\n", - " # 用分类分布预测模型返回的字符\n", - " predictions = predictions / temperature\n", - " predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n", - "\n", - " # 把预测字符和前面的隐藏状态一起传递给模型作为下一个输入\n", - " input_eval = tf.expand_dims([predicted_id], 0)\n", - "\n", - " text_generated.append(idx2char[predicted_id])\n", - "\n", - " return (start_string + ''.join(text_generated))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ktovv0RFhrkn", - "colab": {} - }, - "source": [ - "print(generate_text(model, start_string=u\"ROMEO: \"))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AM2Uma_-yVIq" - }, - "source": [ - "若想改进结果，最简单的方式是延长训练时间 （试试 `EPOCHS=30`）。\n", - "\n", - "你还可以试验使用不同的起始字符串，或者尝试增加另一个 RNN 层以提高模型的准确率，亦或调整温度参数以生成更多或者更少的随机预测。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Y4QwTjAM6A2O" - }, - "source": [ - "## 高级：自定义训练\n", - "\n", - "上面的训练步骤简单，但是能控制的地方不多。\n", - "\n", - "至此，你已经知道如何手动运行模型。现在，让我们打开训练循环，并自己实现它。这是一些任务的起点，例如实现 _课程学习_ 以帮助稳定模型的开环输出。\n", - "\n", - "你将使用 `tf.GradientTape` 跟踪梯度。关于此方法的更多信息请参阅 [eager execution 指南](https://www.tensorflow.org/guide/eager)。\n", - "\n", - "步骤如下：\n", - "\n", - "* 首先，初始化 RNN 状态，使用 `tf.keras.Model.reset_states` 方法。\n", - "\n", - "* 然后，迭代数据集（逐批次）并计算每次迭代对应的 *预测*。\n", - "\n", - "* 打开一个 `tf.GradientTape` 并计算该上下文时的预测和损失。\n", - "\n", - "* 使用 `tf.GradientTape.grads` 方法，计算当前模型变量情况下的损失梯度。\n", - "\n", - "* 最后，使用优化器的 `tf.train.Optimizer.apply_gradients` 方法向下迈出一步。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_XAm7eCoKULT", - "colab": {} - }, - "source": [ - "model = build_model(\n", - " vocab_size = len(vocab),\n", - " embedding_dim=embedding_dim,\n", - " rnn_units=rnn_units,\n", - " batch_size=BATCH_SIZE)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "qUKhnZtMVpoJ", - "colab": {} - }, - "source": [ - "optimizer = tf.keras.optimizers.Adam()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "b4kH1o0leVIp", - "colab": {} - }, - "source": [ - "@tf.function\n", - "def train_step(inp, target):\n", - " with tf.GradientTape() as tape:\n", - " predictions = model(inp)\n", - " loss = tf.reduce_mean(\n", - " tf.keras.losses.sparse_categorical_crossentropy(\n", - " target, predictions, from_logits=True))\n", - " grads = tape.gradient(loss, model.trainable_variables)\n", - " optimizer.apply_gradients(zip(grads, model.trainable_variables))\n", - "\n", - " return loss" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "d4tSNwymzf-q", - "colab": {} - }, - "source": [ - "# 训练步骤\n", - "EPOCHS = 10\n", - "\n", - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - "\n", - " # 在每个训练周期开始时，初始化隐藏状态\n", - " # 隐藏状态最初为 None\n", - " hidden = model.reset_states()\n", - "\n", - " for (batch_n, (inp, target)) in enumerate(dataset):\n", - " loss = train_step(inp, target)\n", - "\n", - " if batch_n % 100 == 0:\n", - " template = 'Epoch {} Batch {} Loss {}'\n", - " print(template.format(epoch+1, batch_n, loss))\n", - "\n", - " # 每 5 个训练周期，保存（检查点）1 次模型\n", - " if (epoch + 1) % 5 == 0:\n", - " model.save_weights(checkpoint_prefix.format(epoch=epoch))\n", - "\n", - " print ('Epoch {} Loss {:.4f}'.format(epoch+1, loss))\n", - " print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n", - "\n", - "model.save_weights(checkpoint_prefix.format(epoch=epoch))" - ], - "execution_count": 0, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/site/zh-cn/tutorials/text/transformer.ipynb b/site/zh-cn/tutorials/text/transformer.ipynb deleted file mode 100644 index e8cf4f333a1..00000000000 --- a/site/zh-cn/tutorials/text/transformer.ipynb +++ /dev/null @@ -1,2058 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "transformer.ipynb", - "provenance": [], - "private_outputs": true, - "collapsed_sections": [ - "s_qNSzzyaCbD" - ], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "s_qNSzzyaCbD" - }, - "source": [ - "##### Copyright 2019 The TensorFlow Authors." - ] - }, - { - "cell_type": "code", - "metadata": { - "cellView": "form", - "colab_type": "code", - "id": "jmjh290raIky", - "colab": {} - }, - "source": [ - "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "J0Qjg6vuaHNt" - }, - "source": [ - "# 理解语言的 Transformer 模型" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "AOpGoE2T-YXS" - }, - "source": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
    \n", - " \n", - " \n", - " 在 tensorflow.google.cn 上查看\n", - " \n", - " \n", - " \n", - " 在 Google Colab 运行\n", - " \n", - " \n", - " \n", - " 在 Github 上查看源代码\n", - " \n", - " 下载此 notebook\n", - "
    " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7Saq5g1mnE5Y", - "colab_type": "text" - }, - "source": [ - "Note: 我们的 TensorFlow 社区翻译了这些文档。因为社区翻译是尽力而为， 所以无法保证它们是最准确的，并且反映了最新的\n", - "[官方英文文档](https://www.tensorflow.org/?hl=en)。如果您有改进此翻译的建议， 请提交 pull request 到\n", - "[tensorflow/docs](https://github.com/tensorflow/docs) GitHub 仓库。要志愿地撰写或者审核译文，请加入\n", - "[docs-zh-cn@tensorflow.org Google Group](https://groups.google.com/a/tensorflow.org/forum/#!forum/docs-zh-cn)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "M-f8TnGpE_ex" - }, - "source": [ - "本教程训练了一个 Transformer 模型 用于将葡萄牙语翻译成英语。这是一个高级示例，假定您具备[文本生成（text generation）](text_generation.ipynb)和 [注意力机制（attention）](nmt_with_attention.ipynb) 的知识。\n", - "\n", - "Transformer 模型的核心思想是*自注意力机制（self-attention）*——能注意输入序列的不同位置以计算该序列的表示的能力。Transformer 创建了多层自注意力层（self-attetion layers）组成的堆栈，下文的*按比缩放的点积注意力（Scaled dot product attention）*和*多头注意力（Multi-head attention）*部分对此进行了说明。\n", - "\n", - "一个 transformer 模型用自注意力层而非 [RNNs](text_classification_rnn.ipynb) 或 [CNNs](../images/intro_to_cnns.ipynb) 来处理变长的输入。这种通用架构有一系列的优势：\n", - "\n", - "* 它不对数据间的时间/空间关系做任何假设。这是处理一组对象（objects）的理想选择（例如，[星际争霸单位（StarCraft units）](https://deepmind.com/blog/alphastar-mastering-real-time-strategy-game-starcraft-ii/#block-8)）。\n", - "* 层输出可以并行计算，而非像 RNN 这样的序列计算。\n", - "* 远距离项可以影响彼此的输出，而无需经过许多 RNN 步骤或卷积层（例如，参见[场景记忆 Transformer（Scene Memory Transformer）](https://arxiv.org/pdf/1903.03878.pdf)）\n", - "* 它能学习长距离的依赖。在许多序列任务中，这是一项挑战。\n", - "\n", - "该架构的缺点是：\n", - "\n", - "* 对于时间序列，一个单位时间的输出是从*整个历史记录*计算的，而非仅从输入和当前的隐含状态计算得到。这*可能*效率较低。 \n", - "* 如果输入*确实*有时间/空间的关系，像文本，则必须加入一些位置编码，否则模型将有效地看到一堆单词。\n", - "\n", - "在此 notebook 中训练完模型后，您将能输入葡萄牙语句子，得到其英文翻译。\n", - "\n", - "\"Attention" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "JjJJyJTZYebt", - "colab": {} - }, - "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "try:\n", - " %tensorflow_version 2.x\n", - "except Exception:\n", - " pass\n", - "import tensorflow_datasets as tfds\n", - "import tensorflow as tf\n", - "\n", - "import time\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fd1NWMxjfsDd" - }, - "source": [ - "## 设置输入流水线（input pipeline）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "t4_Qt8W1hJE_" - }, - "source": [ - "使用 [TFDS](https://tensorflow.google.cn/datasets) 来导入 [葡萄牙语-英语翻译数据集](https://github.com/neulab/word-embeddings-for-nmt)，该数据集来自于 [TED 演讲开放翻译项目](https://www.ted.com/participate/translate).\n", - "\n", - "该数据集包含来约 50000 条训练样本，1100 条验证样本，以及 2000 条测试样本。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8q9t4FmN96eN", - "colab": {} - }, - "source": [ - "examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True,\n", - " as_supervised=True)\n", - "train_examples, val_examples = examples['train'], examples['validation']" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RCEKotqosGfq" - }, - "source": [ - "从训练数据集创建自定义子词分词器（subwords tokenizer）。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "KVBg5Q8tBk5z", - "colab": {} - }, - "source": [ - "tokenizer_en = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n", - " (en.numpy() for pt, en in train_examples), target_vocab_size=2**13)\n", - "\n", - "tokenizer_pt = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n", - " (pt.numpy() for pt, en in train_examples), target_vocab_size=2**13)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "4DYWukNFkGQN", - "colab": {} - }, - "source": [ - "sample_string = 'Transformer is awesome.'\n", - "\n", - "tokenized_string = tokenizer_en.encode(sample_string)\n", - "print ('Tokenized string is {}'.format(tokenized_string))\n", - "\n", - "original_string = tokenizer_en.decode(tokenized_string)\n", - "print ('The original string: {}'.format(original_string))\n", - "\n", - "assert original_string == sample_string" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "o9KJWJjrsZ4Y" - }, - "source": [ - "如果单词不在词典中，则分词器（tokenizer）通过将单词分解为子词来对字符串进行编码。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bf2ntBxjkqK6", - "colab": {} - }, - "source": [ - "for ts in tokenized_string:\n", - " print ('{} ----> {}'.format(ts, tokenizer_en.decode([ts])))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bcRp7VcQ5m6g", - "colab": {} - }, - "source": [ - "BUFFER_SIZE = 20000\n", - "BATCH_SIZE = 64" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kGi4PoVakxdc" - }, - "source": [ - "将开始和结束标记（token）添加到输入和目标。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UZwnPr4R055s", - "colab": {} - }, - "source": [ - "def encode(lang1, lang2):\n", - " lang1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode(\n", - " lang1.numpy()) + [tokenizer_pt.vocab_size+1]\n", - "\n", - " lang2 = [tokenizer_en.vocab_size] + tokenizer_en.encode(\n", - " lang2.numpy()) + [tokenizer_en.vocab_size+1]\n", - " \n", - " return lang1, lang2" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6JrGp5Gek6Ql" - }, - "source": [ - "Note：为了使本示例较小且相对较快，删除长度大于40个标记的样本。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "2QEgbjntk6Yf", - "colab": {} - }, - "source": [ - "MAX_LENGTH = 40" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "c081xPGv1CPI", - "colab": {} - }, - "source": [ - "def filter_max_length(x, y, max_length=MAX_LENGTH):\n", - " return tf.logical_and(tf.size(x) <= max_length,\n", - " tf.size(y) <= max_length)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Tx1sFbR-9fRs" - }, - "source": [ - "`.map()` 内部的操作以图模式（graph mode）运行，`.map()` 接收一个不具有 numpy 属性的图张量（graph tensor）。该`分词器（tokenizer）`需要将一个字符串或 Unicode 符号，编码成整数。因此，您需要在 `tf.py_function` 内部运行编码过程，`tf.py_function` 接收一个 eager 张量，该 eager 张量有一个包含字符串值的 numpy 属性。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Mah1cS-P70Iz", - "colab": {} - }, - "source": [ - "def tf_encode(pt, en):\n", - " return tf.py_function(encode, [pt, en], [tf.int64, tf.int64])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9mk9AZdZ5bcS", - "colab": {} - }, - "source": [ - "train_dataset = train_examples.map(tf_encode)\n", - "train_dataset = train_dataset.filter(filter_max_length)\n", - "# 将数据集缓存到内存中以加快读取速度。\n", - "train_dataset = train_dataset.cache()\n", - "train_dataset = train_dataset.shuffle(BUFFER_SIZE).padded_batch(\n", - " BATCH_SIZE, padded_shapes=([-1], [-1]))\n", - "train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)\n", - "\n", - "\n", - "val_dataset = val_examples.map(tf_encode)\n", - "val_dataset = val_dataset.filter(filter_max_length).padded_batch(\n", - " BATCH_SIZE, padded_shapes=([-1], [-1]))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "_fXvfYVfQr2n", - "colab": {} - }, - "source": [ - "pt_batch, en_batch = next(iter(val_dataset))\n", - "pt_batch, en_batch" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "nBQuibYA4n0n" - }, - "source": [ - "## 位置编码（Positional encoding）\n", - "\n", - "因为该模型并不包括任何的循环（recurrence）或卷积，所以模型添加了位置编码，为模型提供一些关于单词在句子中相对位置的信息。\n", - "\n", - "位置编码向量被加到嵌入（embedding）向量中。嵌入表示一个 d 维空间的标记，在 d 维空间中有着相似含义的标记会离彼此更近。但是，嵌入并没有对在一句话中的词的相对位置进行编码。因此，当加上位置编码后，词将基于*它们含义的相似度以及它们在句子中的位置*，在 d 维空间中离彼此更近。\n", - "\n", - "参看 [位置编码](https://github.com/tensorflow/examples/blob/master/community/en/position_encoding.ipynb) 的 notebook 了解更多信息。计算位置编码的公式如下：\n", - "\n", - "$$\\Large{PE_{(pos, 2i)} = sin(pos / 10000^{2i / d_{model}})} $$\n", - "$$\\Large{PE_{(pos, 2i+1)} = cos(pos / 10000^{2i / d_{model}})} $$" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "WhIOZjMNKujn", - "colab": {} - }, - "source": [ - "def get_angles(pos, i, d_model):\n", - " angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n", - " return pos * angle_rates" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1Rz82wEs5biZ", - "colab": {} - }, - "source": [ - "def positional_encoding(position, d_model):\n", - " angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n", - " np.arange(d_model)[np.newaxis, :],\n", - " d_model)\n", - " \n", - " # 将 sin 应用于数组中的偶数索引（indices）；2i\n", - " angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n", - " \n", - " # 将 cos 应用于数组中的奇数索引；2i+1\n", - " angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n", - " \n", - " pos_encoding = angle_rads[np.newaxis, ...]\n", - " \n", - " return tf.cast(pos_encoding, dtype=tf.float32)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "1kLCla68EloE", - "colab": {} - }, - "source": [ - "pos_encoding = positional_encoding(50, 512)\n", - "print (pos_encoding.shape)\n", - "\n", - "plt.pcolormesh(pos_encoding[0], cmap='RdBu')\n", - "plt.xlabel('Depth')\n", - "plt.xlim((0, 512))\n", - "plt.ylabel('Position')\n", - "plt.colorbar()\n", - "plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "a_b4ou4TYqUN" - }, - "source": [ - "## 遮挡（Masking）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "s42Uydjkv0hF" - }, - "source": [ - "遮挡一批序列中所有的填充标记（pad tokens）。这确保了模型不会将填充作为输入。该 mask 表明填充值 `0` 出现的位置：在这些位置 mask 输出 `1`，否则输出 `0`。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "U2i8-e1s8ti9", - "colab": {} - }, - "source": [ - "def create_padding_mask(seq):\n", - " seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n", - " \n", - " # 添加额外的维度来将填充加到\n", - " # 注意力对数（logits）。\n", - " return seq[:, tf.newaxis, tf.newaxis, :] # (batch_size, 1, 1, seq_len)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "A7BYeBCNvi7n", - "colab": {} - }, - "source": [ - "x = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])\n", - "create_padding_mask(x)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Z0hzukDBgVom" - }, - "source": [ - "前瞻遮挡（look-ahead mask）用于遮挡一个序列中的后续标记（future tokens）。换句话说，该 mask 表明了不应该使用的条目。\n", - "\n", - "这意味着要预测第三个词，将仅使用第一个和第二个词。与此类似，预测第四个词，仅使用第一个，第二个和第三个词，依此类推。 " - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "dVxS8OPI9uI0", - "colab": {} - }, - "source": [ - "def create_look_ahead_mask(size):\n", - " mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n", - " return mask # (seq_len, seq_len)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yxKGuXxaBeeE", - "colab": {} - }, - "source": [ - "x = tf.random.uniform((1, 3))\n", - "temp = create_look_ahead_mask(x.shape[1])\n", - "temp" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xluDl5cXYy4y" - }, - "source": [ - "## 按比缩放的点积注意力（Scaled dot product attention）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "vsxEE_-Wa1gF" - }, - "source": [ - "\"scaled_dot_product_attention\"\n", - "\n", - "Transformer 使用的注意力函数有三个输入：Q（请求（query））、K（主键（key））、V（数值（value））。用于计算注意力权重的等式为：\n", - "\n", - "$$\\Large{Attention(Q, K, V) = softmax_k(\\frac{QK^T}{\\sqrt{d_k}}) V} $$\n", - "\n", - "点积注意力被缩小了深度的平方根倍。这样做是因为对于较大的深度值，点积的大小会增大，从而推动 softmax 函数往仅有很小的梯度的方向靠拢，导致了一种很硬的（hard）softmax。\n", - "\n", - "例如，假设 `Q` 和 `K` 的均值为0，方差为1。它们的矩阵乘积将有均值为0，方差为 `dk`。因此，*`dk` 的平方根*被用于缩放（而非其他数值），因为，`Q` 和 `K` 的矩阵乘积的均值本应该为 0，方差本应该为1，这样会获得一个更平缓的 softmax。\n", - "\n", - "遮挡（mask）与 -1e9（接近于负无穷）相乘。这样做是因为遮挡与缩放的 Q 和 K 的矩阵乘积相加，并在 softmax 之前立即应用。目标是将这些单元归零，因为 softmax 的较大负数输入在输出中接近于零。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LazzUq3bJ5SH", - "colab": {} - }, - "source": [ - "def scaled_dot_product_attention(q, k, v, mask):\n", - " \"\"\"计算注意力权重。\n", - " q, k, v 必须具有匹配的前置维度。\n", - " k, v 必须有匹配的倒数第二个维度，例如：seq_len_k = seq_len_v。\n", - " 虽然 mask 根据其类型（填充或前瞻）有不同的形状，\n", - " 但是 mask 必须能进行广播转换以便求和。\n", - " \n", - " 参数:\n", - " q: 请求的形状 == (..., seq_len_q, depth)\n", - " k: 主键的形状 == (..., seq_len_k, depth)\n", - " v: 数值的形状 == (..., seq_len_v, depth_v)\n", - " mask: Float 张量，其形状能转换成\n", - " (..., seq_len_q, seq_len_k)。默认为None。\n", - " \n", - " 返回值:\n", - " 输出，注意力权重\n", - " \"\"\"\n", - "\n", - " matmul_qk = tf.matmul(q, k, transpose_b=True) # (..., seq_len_q, seq_len_k)\n", - " \n", - " # 缩放 matmul_qk\n", - " dk = tf.cast(tf.shape(k)[-1], tf.float32)\n", - " scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n", - "\n", - " # 将 mask 加入到缩放的张量上。\n", - " if mask is not None:\n", - " scaled_attention_logits += (mask * -1e9) \n", - "\n", - " # softmax 在最后一个轴（seq_len_k）上归一化，因此分数\n", - " # 相加等于1。\n", - " attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) # (..., seq_len_q, seq_len_k)\n", - "\n", - " output = tf.matmul(attention_weights, v) # (..., seq_len_q, depth_v)\n", - "\n", - " return output, attention_weights" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "FiqETnhCkoXh" - }, - "source": [ - "当 softmax 在 K 上进行归一化后，它的值决定了分配到 Q 的重要程度。\n", - "\n", - "输出表示注意力权重和 V（数值）向量的乘积。这确保了要关注的词保持原样，而无关的词将被清除掉。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "n90YjClyInFy", - "colab": {} - }, - "source": [ - "def print_out(q, k, v):\n", - " temp_out, temp_attn = scaled_dot_product_attention(\n", - " q, k, v, None)\n", - " print ('Attention weights are:')\n", - " print (temp_attn)\n", - " print ('Output is:')\n", - " print (temp_out)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "yAzUAf2DPlNt", - "colab": {} - }, - "source": [ - "np.set_printoptions(suppress=True)\n", - "\n", - "temp_k = tf.constant([[10,0,0],\n", - " [0,10,0],\n", - " [0,0,10],\n", - " [0,0,10]], dtype=tf.float32) # (4, 3)\n", - "\n", - "temp_v = tf.constant([[ 1,0],\n", - " [ 10,0],\n", - " [ 100,5],\n", - " [1000,6]], dtype=tf.float32) # (4, 2)\n", - "\n", - "# 这条 `请求（query）符合第二个`主键（key）`，\n", - "# 因此返回了第二个`数值（value）`。\n", - "temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32) # (1, 3)\n", - "print_out(temp_q, temp_k, temp_v)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "zg6k-fGhgXra", - "colab": {} - }, - "source": [ - "# 这条请求符合重复出现的主键（第三第四个），\n", - "# 因此，对所有的相关数值取了平均。\n", - "temp_q = tf.constant([[0, 0, 10]], dtype=tf.float32) # (1, 3)\n", - "print_out(temp_q, temp_k, temp_v)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UAq3YOzUgXhb", - "colab": {} - }, - "source": [ - "# 这条请求符合第一和第二条主键，\n", - "# 因此，对它们的数值去了平均。\n", - "temp_q = tf.constant([[10, 10, 0]], dtype=tf.float32) # (1, 3)\n", - "print_out(temp_q, temp_k, temp_v)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aOz-4_XIhaTP" - }, - "source": [ - "将所有请求一起*传递*。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "6dlU8Tm-hYrF", - "colab": {} - }, - "source": [ - "temp_q = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32) # (3, 3)\n", - "print_out(temp_q, temp_k, temp_v)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "kmzGPEy64qmA" - }, - "source": [ - "## 多头注意力（Multi-head attention）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fz5BMC8Kaoqo" - }, - "source": [ - "\"multi-head\n", - "\n", - "\n", - "多头注意力由四部分组成：\n", - "* 线性层并分拆成多头。\n", - "* 按比缩放的点积注意力。\n", - "* 多头及联。\n", - "* 最后一层线性层。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "JPmbr6F1C-v_" - }, - "source": [ - "每个多头注意力块有三个输入：Q（请求）、K（主键）、V（数值）。这些输入经过线性（Dense）层，并分拆成多头。 \n", - "\n", - "将上面定义的 `scaled_dot_product_attention` 函数应用于每个头（进行了广播（broadcasted）以提高效率）。注意力这步必须使用一个恰当的 mask。然后将每个头的注意力输出连接起来（用`tf.transpose` 和 `tf.reshape`），并放入最后的 `Dense` 层。\n", - "\n", - "Q、K、和 V 被拆分到了多个头，而非单个的注意力头，因为多头允许模型共同注意来自不同表示空间的不同位置的信息。在分拆后，每个头部的维度减少，因此总的计算成本与有着全部维度的单个注意力头相同。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "BSV3PPKsYecw", - "colab": {} - }, - "source": [ - "class MultiHeadAttention(tf.keras.layers.Layer):\n", - " def __init__(self, d_model, num_heads):\n", - " super(MultiHeadAttention, self).__init__()\n", - " self.num_heads = num_heads\n", - " self.d_model = d_model\n", - " \n", - " assert d_model % self.num_heads == 0\n", - " \n", - " self.depth = d_model // self.num_heads\n", - " \n", - " self.wq = tf.keras.layers.Dense(d_model)\n", - " self.wk = tf.keras.layers.Dense(d_model)\n", - " self.wv = tf.keras.layers.Dense(d_model)\n", - " \n", - " self.dense = tf.keras.layers.Dense(d_model)\n", - " \n", - " def split_heads(self, x, batch_size):\n", - " \"\"\"分拆最后一个维度到 (num_heads, depth).\n", - " 转置结果使得形状为 (batch_size, num_heads, seq_len, depth)\n", - " \"\"\"\n", - " x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n", - " return tf.transpose(x, perm=[0, 2, 1, 3])\n", - " \n", - " def call(self, v, k, q, mask):\n", - " batch_size = tf.shape(q)[0]\n", - " \n", - " q = self.wq(q) # (batch_size, seq_len, d_model)\n", - " k = self.wk(k) # (batch_size, seq_len, d_model)\n", - " v = self.wv(v) # (batch_size, seq_len, d_model)\n", - " \n", - " q = self.split_heads(q, batch_size) # (batch_size, num_heads, seq_len_q, depth)\n", - " k = self.split_heads(k, batch_size) # (batch_size, num_heads, seq_len_k, depth)\n", - " v = self.split_heads(v, batch_size) # (batch_size, num_heads, seq_len_v, depth)\n", - " \n", - " # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n", - " # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n", - " scaled_attention, attention_weights = scaled_dot_product_attention(\n", - " q, k, v, mask)\n", - " \n", - " scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3]) # (batch_size, seq_len_q, num_heads, depth)\n", - "\n", - " concat_attention = tf.reshape(scaled_attention, \n", - " (batch_size, -1, self.d_model)) # (batch_size, seq_len_q, d_model)\n", - "\n", - " output = self.dense(concat_attention) # (batch_size, seq_len_q, d_model)\n", - " \n", - " return output, attention_weights" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0D8FJue5lDyZ" - }, - "source": [ - "创建一个 `MultiHeadAttention` 层进行尝试。在序列中的每个位置 `y`，`MultiHeadAttention` 在序列中的所有其他位置运行所有8个注意力头，在每个位置y，返回一个新的同样长度的向量。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Hu94p-_-2_BX", - "colab": {} - }, - "source": [ - "temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\n", - "y = tf.random.uniform((1, 60, 512)) # (batch_size, encoder_sequence, d_model)\n", - "out, attn = temp_mha(y, k=y, q=y, mask=None)\n", - "out.shape, attn.shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RdDqGayx67vv" - }, - "source": [ - "## 点式前馈网络（Point wise feed forward network）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "gBqzJXGfHK3X" - }, - "source": [ - "点式前馈网络由两层全联接层组成，两层之间有一个 ReLU 激活函数。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ET7xLt0yCT6Z", - "colab": {} - }, - "source": [ - "def point_wise_feed_forward_network(d_model, dff):\n", - " return tf.keras.Sequential([\n", - " tf.keras.layers.Dense(dff, activation='relu'), # (batch_size, seq_len, dff)\n", - " tf.keras.layers.Dense(d_model) # (batch_size, seq_len, d_model)\n", - " ])" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "mytb1lPyOHLB", - "colab": {} - }, - "source": [ - "sample_ffn = point_wise_feed_forward_network(512, 2048)\n", - "sample_ffn(tf.random.uniform((64, 50, 512))).shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "7e7hKcxn6-zd" - }, - "source": [ - "## 编码与解码（Encoder and decoder）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "yScbC0MUH8dS" - }, - "source": [ - "\"transformer\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "MfYJG-Kvgwy2" - }, - "source": [ - "Transformer 模型与标准的[具有注意力机制的序列到序列模型（sequence to sequence with attention model）](nmt_with_attention.ipynb)，遵循相同的一般模式。\n", - "\n", - "* 输入语句经过 `N` 个编码器层，为序列中的每个词/标记生成一个输出。\n", - "* 解码器关注编码器的输出以及它自身的输入（自注意力）来预测下一个词。" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QFv-FNYUmvpn" - }, - "source": [ - "### 编码器层（Encoder layer）\n", - "\n", - "每个编码器层包括以下子层：\n", - "\n", - "1. 多头注意力（有填充遮挡）\n", - "2. 点式前馈网络（Point wise feed forward networks）。\n", - "\n", - "每个子层在其周围有一个残差连接，然后进行层归一化。残差连接有助于避免深度网络中的梯度消失问题。\n", - "\n", - "每个子层的输出是 `LayerNorm(x + Sublayer(x))`。归一化是在 `d_model`（最后一个）维度完成的。Transformer 中有 N 个编码器层。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ncyS-Ms3i2x_", - "colab": {} - }, - "source": [ - "class EncoderLayer(tf.keras.layers.Layer):\n", - " def __init__(self, d_model, num_heads, dff, rate=0.1):\n", - " super(EncoderLayer, self).__init__()\n", - "\n", - " self.mha = MultiHeadAttention(d_model, num_heads)\n", - " self.ffn = point_wise_feed_forward_network(d_model, dff)\n", - "\n", - " self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " \n", - " self.dropout1 = tf.keras.layers.Dropout(rate)\n", - " self.dropout2 = tf.keras.layers.Dropout(rate)\n", - " \n", - " def call(self, x, training, mask):\n", - "\n", - " attn_output, _ = self.mha(x, x, x, mask) # (batch_size, input_seq_len, d_model)\n", - " attn_output = self.dropout1(attn_output, training=training)\n", - " out1 = self.layernorm1(x + attn_output) # (batch_size, input_seq_len, d_model)\n", - " \n", - " ffn_output = self.ffn(out1) # (batch_size, input_seq_len, d_model)\n", - " ffn_output = self.dropout2(ffn_output, training=training)\n", - " out2 = self.layernorm2(out1 + ffn_output) # (batch_size, input_seq_len, d_model)\n", - " \n", - " return out2" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "AzZRXdO0mI48", - "colab": {} - }, - "source": [ - "sample_encoder_layer = EncoderLayer(512, 8, 2048)\n", - "\n", - "sample_encoder_layer_output = sample_encoder_layer(\n", - " tf.random.uniform((64, 43, 512)), False, None)\n", - "\n", - "sample_encoder_layer_output.shape # (batch_size, input_seq_len, d_model)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "6LO_48Owmx_o" - }, - "source": [ - "### 解码器层（Decoder layer）\n", - "\n", - "每个解码器层包括以下子层：\n", - "\n", - "1. 遮挡的多头注意力（前瞻遮挡和填充遮挡）\n", - "2. 多头注意力（用填充遮挡）。V（数值）和 K（主键）接收*编码器输出*作为输入。Q（请求）接收*遮挡的多头注意力子层的输出*。\n", - "3. 点式前馈网络\n", - "\n", - "每个子层在其周围有一个残差连接，然后进行层归一化。每个子层的输出是 `LayerNorm(x + Sublayer(x))`。归一化是在 `d_model`（最后一个）维度完成的。\n", - "\n", - "Transformer 中共有 N 个解码器层。\n", - "\n", - "当 Q 接收到解码器的第一个注意力块的输出，并且 K 接收到编码器的输出时，注意力权重表示根据编码器的输出赋予解码器输入的重要性。换一种说法，解码器通过查看编码器输出和对其自身输出的自注意力，预测下一个词。参看按比缩放的点积注意力部分的演示。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "9SoX0-vd1hue", - "colab": {} - }, - "source": [ - "class DecoderLayer(tf.keras.layers.Layer):\n", - " def __init__(self, d_model, num_heads, dff, rate=0.1):\n", - " super(DecoderLayer, self).__init__()\n", - "\n", - " self.mha1 = MultiHeadAttention(d_model, num_heads)\n", - " self.mha2 = MultiHeadAttention(d_model, num_heads)\n", - "\n", - " self.ffn = point_wise_feed_forward_network(d_model, dff)\n", - " \n", - " self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n", - " \n", - " self.dropout1 = tf.keras.layers.Dropout(rate)\n", - " self.dropout2 = tf.keras.layers.Dropout(rate)\n", - " self.dropout3 = tf.keras.layers.Dropout(rate)\n", - " \n", - " \n", - " def call(self, x, enc_output, training, \n", - " look_ahead_mask, padding_mask):\n", - " # enc_output.shape == (batch_size, input_seq_len, d_model)\n", - "\n", - " attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask) # (batch_size, target_seq_len, d_model)\n", - " attn1 = self.dropout1(attn1, training=training)\n", - " out1 = self.layernorm1(attn1 + x)\n", - " \n", - " attn2, attn_weights_block2 = self.mha2(\n", - " enc_output, enc_output, out1, padding_mask) # (batch_size, target_seq_len, d_model)\n", - " attn2 = self.dropout2(attn2, training=training)\n", - " out2 = self.layernorm2(attn2 + out1) # (batch_size, target_seq_len, d_model)\n", - " \n", - " ffn_output = self.ffn(out2) # (batch_size, target_seq_len, d_model)\n", - " ffn_output = self.dropout3(ffn_output, training=training)\n", - " out3 = self.layernorm3(ffn_output + out2) # (batch_size, target_seq_len, d_model)\n", - " \n", - " return out3, attn_weights_block1, attn_weights_block2" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "Ne2Bqx8k71l0", - "colab": {} - }, - "source": [ - "sample_decoder_layer = DecoderLayer(512, 8, 2048)\n", - "\n", - "sample_decoder_layer_output, _, _ = sample_decoder_layer(\n", - " tf.random.uniform((64, 50, 512)), sample_encoder_layer_output, \n", - " False, None, None)\n", - "\n", - "sample_decoder_layer_output.shape # (batch_size, target_seq_len, d_model)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "SE1H51Ajm0q1" - }, - "source": [ - "### 编码器（Encoder）\n", - "\n", - "`编码器` 包括：\n", - "1. 输入嵌入（Input Embedding）\n", - "2. 位置编码（Positional Encoding）\n", - "3. N 个编码器层（encoder layers）\n", - "\n", - "输入经过嵌入（embedding）后，该嵌入与位置编码相加。该加法结果的输出是编码器层的输入。编码器的输出是解码器的输入。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "jpEox7gJ8FCI", - "colab": {} - }, - "source": [ - "class Encoder(tf.keras.layers.Layer):\n", - " def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n", - " maximum_position_encoding, rate=0.1):\n", - " super(Encoder, self).__init__()\n", - "\n", - " self.d_model = d_model\n", - " self.num_layers = num_layers\n", - " \n", - " self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n", - " self.pos_encoding = positional_encoding(maximum_position_encoding, \n", - " self.d_model)\n", - " \n", - " \n", - " self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n", - " for _ in range(num_layers)]\n", - " \n", - " self.dropout = tf.keras.layers.Dropout(rate)\n", - " \n", - " def call(self, x, training, mask):\n", - "\n", - " seq_len = tf.shape(x)[1]\n", - " \n", - " # 将嵌入和位置编码相加。\n", - " x = self.embedding(x) # (batch_size, input_seq_len, d_model)\n", - " x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n", - " x += self.pos_encoding[:, :seq_len, :]\n", - "\n", - " x = self.dropout(x, training=training)\n", - " \n", - " for i in range(self.num_layers):\n", - " x = self.enc_layers[i](x, training, mask)\n", - " \n", - " return x # (batch_size, input_seq_len, d_model)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "8QG9nueFQKXx", - "colab": {} - }, - "source": [ - "sample_encoder = Encoder(num_layers=2, d_model=512, num_heads=8, \n", - " dff=2048, input_vocab_size=8500,\n", - " maximum_position_encoding=10000)\n", - "\n", - "sample_encoder_output = sample_encoder(tf.random.uniform((64, 62)), \n", - " training=False, mask=None)\n", - "\n", - "print (sample_encoder_output.shape) # (batch_size, input_seq_len, d_model)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "p-uO6ls8m2O5" - }, - "source": [ - "### 解码器（Decoder）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "ZtT7PKzrXkNr" - }, - "source": [ - "`解码器`包括：\n", - "1. 输出嵌入（Output Embedding）\n", - "2. 位置编码（Positional Encoding）\n", - "3. N 个解码器层（decoder layers）\n", - "\n", - "目标（target）经过一个嵌入后，该嵌入和位置编码相加。该加法结果是解码器层的输入。解码器的输出是最后的线性层的输入。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "d5_d5-PLQXwY", - "colab": {} - }, - "source": [ - "class Decoder(tf.keras.layers.Layer):\n", - " def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n", - " maximum_position_encoding, rate=0.1):\n", - " super(Decoder, self).__init__()\n", - "\n", - " self.d_model = d_model\n", - " self.num_layers = num_layers\n", - " \n", - " self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n", - " self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n", - " \n", - " self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n", - " for _ in range(num_layers)]\n", - " self.dropout = tf.keras.layers.Dropout(rate)\n", - " \n", - " def call(self, x, enc_output, training, \n", - " look_ahead_mask, padding_mask):\n", - "\n", - " seq_len = tf.shape(x)[1]\n", - " attention_weights = {}\n", - " \n", - " x = self.embedding(x) # (batch_size, target_seq_len, d_model)\n", - " x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n", - " x += self.pos_encoding[:, :seq_len, :]\n", - " \n", - " x = self.dropout(x, training=training)\n", - "\n", - " for i in range(self.num_layers):\n", - " x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n", - " look_ahead_mask, padding_mask)\n", - " \n", - " attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n", - " attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n", - " \n", - " # x.shape == (batch_size, target_seq_len, d_model)\n", - " return x, attention_weights" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "a1jXoAMRZyvu", - "colab": {} - }, - "source": [ - "sample_decoder = Decoder(num_layers=2, d_model=512, num_heads=8, \n", - " dff=2048, target_vocab_size=8000,\n", - " maximum_position_encoding=5000)\n", - "\n", - "output, attn = sample_decoder(tf.random.uniform((64, 26)), \n", - " enc_output=sample_encoder_output, \n", - " training=False, look_ahead_mask=None, \n", - " padding_mask=None)\n", - "\n", - "output.shape, attn['decoder_layer2_block2'].shape" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y54xnJnuYgJ7" - }, - "source": [ - "## 创建 Transformer" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "uERO1y54cOKq" - }, - "source": [ - "Transformer 包括编码器，解码器和最后的线性层。解码器的输出是线性层的输入，返回线性层的输出。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "PED3bIpOYkBu", - "colab": {} - }, - "source": [ - "class Transformer(tf.keras.Model):\n", - " def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n", - " target_vocab_size, pe_input, pe_target, rate=0.1):\n", - " super(Transformer, self).__init__()\n", - "\n", - " self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n", - " input_vocab_size, pe_input, rate)\n", - "\n", - " self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n", - " target_vocab_size, pe_target, rate)\n", - "\n", - " self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n", - " \n", - " def call(self, inp, tar, training, enc_padding_mask, \n", - " look_ahead_mask, dec_padding_mask):\n", - "\n", - " enc_output = self.encoder(inp, training, enc_padding_mask) # (batch_size, inp_seq_len, d_model)\n", - " \n", - " # dec_output.shape == (batch_size, tar_seq_len, d_model)\n", - " dec_output, attention_weights = self.decoder(\n", - " tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n", - " \n", - " final_output = self.final_layer(dec_output) # (batch_size, tar_seq_len, target_vocab_size)\n", - " \n", - " return final_output, attention_weights" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "tJ4fbQcIkHW1", - "colab": {} - }, - "source": [ - "sample_transformer = Transformer(\n", - " num_layers=2, d_model=512, num_heads=8, dff=2048, \n", - " input_vocab_size=8500, target_vocab_size=8000, \n", - " pe_input=10000, pe_target=6000)\n", - "\n", - "temp_input = tf.random.uniform((64, 62))\n", - "temp_target = tf.random.uniform((64, 26))\n", - "\n", - "fn_out, _ = sample_transformer(temp_input, temp_target, training=False, \n", - " enc_padding_mask=None, \n", - " look_ahead_mask=None,\n", - " dec_padding_mask=None)\n", - "\n", - "fn_out.shape # (batch_size, tar_seq_len, target_vocab_size)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "wsINyf1VEQLC" - }, - "source": [ - "## 配置超参数（hyperparameters）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "zVjWCxFNcgbt" - }, - "source": [ - "为了让本示例小且相对较快，已经减小了*num_layers、 d_model 和 dff* 的值。 \n", - "\n", - "Transformer 的基础模型使用的数值为：*num_layers=6*，*d_model = 512*，*dff = 2048*。关于所有其他版本的 Transformer，请查阅[论文](https://arxiv.org/abs/1706.03762)。\n", - "\n", - "Note：通过改变以下数值，您可以获得在许多任务上达到最先进水平的模型。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lnJn5SLA2ahP", - "colab": {} - }, - "source": [ - "num_layers = 4\n", - "d_model = 128\n", - "dff = 512\n", - "num_heads = 8\n", - "\n", - "input_vocab_size = tokenizer_pt.vocab_size + 2\n", - "target_vocab_size = tokenizer_en.vocab_size + 2\n", - "dropout_rate = 0.1" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xYEGhEOtzn5W" - }, - "source": [ - "## 优化器（Optimizer）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "GOmWW--yP3zx" - }, - "source": [ - "根据[论文](https://arxiv.org/abs/1706.03762)中的公式，将 Adam 优化器与自定义的学习速率调度程序（scheduler）配合使用。\n", - "\n", - "$$\\Large{lrate = d_{model}^{-0.5} * min(step{\\_}num^{-0.5}, step{\\_}num * warmup{\\_}steps^{-1.5})}$$\n" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iYQdOO1axwEI", - "colab": {} - }, - "source": [ - "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n", - " def __init__(self, d_model, warmup_steps=4000):\n", - " super(CustomSchedule, self).__init__()\n", - " \n", - " self.d_model = d_model\n", - " self.d_model = tf.cast(self.d_model, tf.float32)\n", - "\n", - " self.warmup_steps = warmup_steps\n", - " \n", - " def __call__(self, step):\n", - " arg1 = tf.math.rsqrt(step)\n", - " arg2 = step * (self.warmup_steps ** -1.5)\n", - " \n", - " return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7r4scdulztRx", - "colab": {} - }, - "source": [ - "learning_rate = CustomSchedule(d_model)\n", - "\n", - "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n", - " epsilon=1e-9)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "f33ZCgvHpPdG", - "colab": {} - }, - "source": [ - "temp_learning_rate_schedule = CustomSchedule(d_model)\n", - "\n", - "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n", - "plt.ylabel(\"Learning Rate\")\n", - "plt.xlabel(\"Train Step\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "YgkDE7hzo8r5" - }, - "source": [ - "## 损失函数与指标（Loss and metrics）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "oxGJtoDuYIHL" - }, - "source": [ - "由于目标序列是填充（padded）过的，因此在计算损失函数时，应用填充遮挡非常重要。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "MlhsJMm0TW_B", - "colab": {} - }, - "source": [ - "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n", - " from_logits=True, reduction='none')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "67oqVHiT0Eiu", - "colab": {} - }, - "source": [ - "def loss_function(real, pred):\n", - " mask = tf.math.logical_not(tf.math.equal(real, 0))\n", - " loss_ = loss_object(real, pred)\n", - "\n", - " mask = tf.cast(mask, dtype=loss_.dtype)\n", - " loss_ *= mask\n", - " \n", - " return tf.reduce_mean(loss_)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "phlyxMnm-Tpx", - "colab": {} - }, - "source": [ - "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", - "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n", - " name='train_accuracy')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "aeHumfr7zmMa" - }, - "source": [ - "## 训练与检查点（Training and checkpointing）" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "UiysUa--4tOU", - "colab": {} - }, - "source": [ - "transformer = Transformer(num_layers, d_model, num_heads, dff,\n", - " input_vocab_size, target_vocab_size, \n", - " pe_input=input_vocab_size, \n", - " pe_target=target_vocab_size,\n", - " rate=dropout_rate)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "ZOJUSB1T8GjM", - "colab": {} - }, - "source": [ - "def create_masks(inp, tar):\n", - " # 编码器填充遮挡\n", - " enc_padding_mask = create_padding_mask(inp)\n", - " \n", - " # 在解码器的第二个注意力模块使用。\n", - " # 该填充遮挡用于遮挡编码器的输出。\n", - " dec_padding_mask = create_padding_mask(inp)\n", - " \n", - " # 在解码器的第一个注意力模块使用。\n", - " # 用于填充（pad）和遮挡（mask）解码器获取到的输入的后续标记（future tokens）。\n", - " look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n", - " dec_target_padding_mask = create_padding_mask(tar)\n", - " combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n", - " \n", - " return enc_padding_mask, combined_mask, dec_padding_mask" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Fzuf06YZp66w" - }, - "source": [ - "创建检查点的路径和检查点管理器（manager）。这将用于在每 `n` 个周期（epochs）保存检查点。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "hNhuYfllndLZ", - "colab": {} - }, - "source": [ - "checkpoint_path = \"./checkpoints/train\"\n", - "\n", - "ckpt = tf.train.Checkpoint(transformer=transformer,\n", - " optimizer=optimizer)\n", - "\n", - "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n", - "\n", - "# 如果检查点存在，则恢复最新的检查点。\n", - "if ckpt_manager.latest_checkpoint:\n", - " ckpt.restore(ckpt_manager.latest_checkpoint)\n", - " print ('Latest checkpoint restored!!')" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "0Di_Yaa1gf9r" - }, - "source": [ - "目标（target）被分成了 tar_inp 和 tar_real。tar_inp 作为输入传递到解码器。`tar_real` 是位移了 1 的同一个输入：在 `tar_inp` 中的每个位置，`tar_real` 包含了应该被预测到的下一个标记（token）。\n", - "\n", - "例如，`sentence` = \"SOS A lion in the jungle is sleeping EOS\"\n", - "\n", - "`tar_inp` = \"SOS A lion in the jungle is sleeping\"\n", - "\n", - "`tar_real` = \"A lion in the jungle is sleeping EOS\"\n", - "\n", - "Transformer 是一个自回归（auto-regressive）模型：它一次作一个部分的预测，然后使用到目前为止的自身的输出来决定下一步要做什么。\n", - "\n", - "在训练过程中，本示例使用了 teacher-forcing 的方法（就像[文本生成教程](./text_generation.ipynb)中一样）。无论模型在当前时间步骤下预测出什么，teacher-forcing 方法都会将真实的输出传递到下一个时间步骤上。\n", - "\n", - "当 transformer 预测每个词时，*自注意力（self-attention）*功能使它能够查看输入序列中前面的单词，从而更好地预测下一个单词。\n", - "\n", - "为了防止模型在期望的输出上达到峰值，模型使用了前瞻遮挡（look-ahead mask）。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "LKpoA6q1sJFj", - "colab": {} - }, - "source": [ - "EPOCHS = 20" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "iJwmp9OE29oj", - "colab": {} - }, - "source": [ - "# 该 @tf.function 将追踪-编译 train_step 到 TF 图中，以便更快地\n", - "# 执行。该函数专用于参数张量的精确形状。为了避免由于可变序列长度或可变\n", - "# 批次大小（最后一批次较小）导致的再追踪，使用 input_signature 指定\n", - "# 更多的通用形状。\n", - "\n", - "train_step_signature = [\n", - " tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n", - " tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n", - "]\n", - "\n", - "@tf.function(input_signature=train_step_signature)\n", - "def train_step(inp, tar):\n", - " tar_inp = tar[:, :-1]\n", - " tar_real = tar[:, 1:]\n", - " \n", - " enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n", - " \n", - " with tf.GradientTape() as tape:\n", - " predictions, _ = transformer(inp, tar_inp, \n", - " True, \n", - " enc_padding_mask, \n", - " combined_mask, \n", - " dec_padding_mask)\n", - " loss = loss_function(tar_real, predictions)\n", - "\n", - " gradients = tape.gradient(loss, transformer.trainable_variables) \n", - " optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n", - " \n", - " train_loss(loss)\n", - " train_accuracy(tar_real, predictions)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "qM2PDWGDJ_8V" - }, - "source": [ - "葡萄牙语作为输入语言，英语为目标语言。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "bbvmaKNiznHZ", - "colab": {} - }, - "source": [ - "for epoch in range(EPOCHS):\n", - " start = time.time()\n", - " \n", - " train_loss.reset_states()\n", - " train_accuracy.reset_states()\n", - " \n", - " # inp -> portuguese, tar -> english\n", - " for (batch, (inp, tar)) in enumerate(train_dataset):\n", - " train_step(inp, tar)\n", - " \n", - " if batch % 50 == 0:\n", - " print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n", - " epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n", - " \n", - " if (epoch + 1) % 5 == 0:\n", - " ckpt_save_path = ckpt_manager.save()\n", - " print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,\n", - " ckpt_save_path))\n", - " \n", - " print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, \n", - " train_loss.result(), \n", - " train_accuracy.result()))\n", - "\n", - " print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "QfcsSWswSdGV" - }, - "source": [ - "## 评估（Evaluate）" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "y6APsFrgImLW" - }, - "source": [ - "以下步骤用于评估：\n", - "\n", - "* 用葡萄牙语分词器（`tokenizer_pt`）编码输入语句。此外，添加开始和结束标记，这样输入就与模型训练的内容相同。这是编码器输入。\n", - "* 解码器输入为 `start token == tokenizer_en.vocab_size`。\n", - "* 计算填充遮挡和前瞻遮挡。\n", - "* `解码器`通过查看`编码器输出`和它自身的输出（自注意力）给出预测。\n", - "* 选择最后一个词并计算它的 argmax。\n", - "* 将预测的词连接到解码器输入，然后传递给解码器。\n", - "* 在这种方法中，解码器根据它预测的之前的词预测下一个。\n", - "\n", - "Note：这里使用的模型具有较小的能力以保持相对较快，因此预测可能不太正确。要复现论文中的结果，请使用全部数据集，并通过修改上述超参数来使用基础 transformer 模型或者 transformer XL。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "5buvMlnvyrFm", - "colab": {} - }, - "source": [ - "def evaluate(inp_sentence):\n", - " start_token = [tokenizer_pt.vocab_size]\n", - " end_token = [tokenizer_pt.vocab_size + 1]\n", - " \n", - " # 输入语句是葡萄牙语，增加开始和结束标记\n", - " inp_sentence = start_token + tokenizer_pt.encode(inp_sentence) + end_token\n", - " encoder_input = tf.expand_dims(inp_sentence, 0)\n", - " \n", - " # 因为目标是英语，输入 transformer 的第一个词应该是\n", - " # 英语的开始标记。\n", - " decoder_input = [tokenizer_en.vocab_size]\n", - " output = tf.expand_dims(decoder_input, 0)\n", - " \n", - " for i in range(MAX_LENGTH):\n", - " enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n", - " encoder_input, output)\n", - " \n", - " # predictions.shape == (batch_size, seq_len, vocab_size)\n", - " predictions, attention_weights = transformer(encoder_input, \n", - " output,\n", - " False,\n", - " enc_padding_mask,\n", - " combined_mask,\n", - " dec_padding_mask)\n", - " \n", - " # 从 seq_len 维度选择最后一个词\n", - " predictions = predictions[: ,-1:, :] # (batch_size, 1, vocab_size)\n", - "\n", - " predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n", - " \n", - " # 如果 predicted_id 等于结束标记，就返回结果\n", - " if predicted_id == tokenizer_en.vocab_size+1:\n", - " return tf.squeeze(output, axis=0), attention_weights\n", - " \n", - " # 连接 predicted_id 与输出，作为解码器的输入传递到解码器。\n", - " output = tf.concat([output, predicted_id], axis=-1)\n", - "\n", - " return tf.squeeze(output, axis=0), attention_weights" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "CN-BV43FMBej", - "colab": {} - }, - "source": [ - "def plot_attention_weights(attention, sentence, result, layer):\n", - " fig = plt.figure(figsize=(16, 8))\n", - " \n", - " sentence = tokenizer_pt.encode(sentence)\n", - " \n", - " attention = tf.squeeze(attention[layer], axis=0)\n", - " \n", - " for head in range(attention.shape[0]):\n", - " ax = fig.add_subplot(2, 4, head+1)\n", - " \n", - " # 画出注意力权重\n", - " ax.matshow(attention[head][:-1, :], cmap='viridis')\n", - "\n", - " fontdict = {'fontsize': 10}\n", - " \n", - " ax.set_xticks(range(len(sentence)+2))\n", - " ax.set_yticks(range(len(result)))\n", - " \n", - " ax.set_ylim(len(result)-1.5, -0.5)\n", - " \n", - " ax.set_xticklabels(\n", - " ['']+[tokenizer_pt.decode([i]) for i in sentence]+[''], \n", - " fontdict=fontdict, rotation=90)\n", - " \n", - " ax.set_yticklabels([tokenizer_en.decode([i]) for i in result \n", - " if i < tokenizer_en.vocab_size], \n", - " fontdict=fontdict)\n", - " \n", - " ax.set_xlabel('Head {}'.format(head+1))\n", - " \n", - " plt.tight_layout()\n", - " plt.show()" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "lU2_yG_vBGza", - "colab": {} - }, - "source": [ - "def translate(sentence, plot=''):\n", - " result, attention_weights = evaluate(sentence)\n", - " \n", - " predicted_sentence = tokenizer_en.decode([i for i in result \n", - " if i < tokenizer_en.vocab_size]) \n", - "\n", - " print('Input: {}'.format(sentence))\n", - " print('Predicted translation: {}'.format(predicted_sentence))\n", - " \n", - " if plot:\n", - " plot_attention_weights(attention_weights, sentence, result, plot)" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "YsxrAlvFG8SZ", - "colab": {} - }, - "source": [ - "translate(\"este é um problema que temos que resolver.\")\n", - "print (\"Real translation: this is a problem we have to solve .\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "7EH5y_aqI4t1", - "colab": {} - }, - "source": [ - "translate(\"os meus vizinhos ouviram sobre esta ideia.\")\n", - "print (\"Real translation: and my neighboring homes heard about this idea .\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "J-hVCTSUMlkb", - "colab": {} - }, - "source": [ - "translate(\"vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.\")\n", - "print (\"Real translation: so i 'll just share with you some stories very quickly of some magical things that have happened .\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "_1MxkSZvz0jX" - }, - "source": [ - "您可以为 `plot` 参数传递不同的层和解码器的注意力模块。" - ] - }, - { - "cell_type": "code", - "metadata": { - "colab_type": "code", - "id": "t-kFyiOLH0xg", - "colab": {} - }, - "source": [ - "translate(\"este é o primeiro livro que eu fiz.\", plot='decoder_layer4_block2')\n", - "print (\"Real translation: this is the first book i've ever done.\")" - ], - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "RqQ1fIsLwkGE" - }, - "source": [ - "## 总结\n", - "\n", - "在本教程中，您已经学习了位置编码，多头注意力，遮挡的重要性以及如何创建一个 transformer。\n", - "\n", - "尝试使用一个不同的数据集来训练 transformer。您可也可以通过修改上述的超参数来创建基础 transformer 或者 transformer XL。您也可以使用这里定义的层来创建 [BERT](https://arxiv.org/abs/1810.04805) 并训练最先进的模型。此外，您可以实现 beam search 得到更好的预测。" - ] - } - ] -} \ No newline at end of file diff --git a/tools/linkcheckerrc b/tools/linkcheckerrc new file mode 100644 index 00000000000..e5f2423f9eb --- /dev/null +++ b/tools/linkcheckerrc @@ -0,0 +1,18 @@ +# linkchecker config for tensorflow.org +# Usage (include trailing slash): +# $ linkchecker -f tools/linkcheckerrc https://www.tensorflow.org/{subsite}/ +# Docs: +# - https://wummel.github.io/linkchecker/man5/linkcheckerrc.5.html +# - https://github.com/wummel/linkchecker/blob/master/config/linkcheckerrc + +[checking] +#pause=1 + +[filtering] + +# Check external URLs +checkextern=1 + +ignore = + .*?(fonts|gstatic|medium).* + http://developers.google.com/ReferenceObject diff --git a/tools/nb_code_sync.py b/tools/nb_code_sync.py deleted file mode 100755 index 3a074381db9..00000000000 --- a/tools/nb_code_sync.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Keep translated notebook code in sync with the source-of-truth notebook. - -This tool attempts to make it easier to keep the community translation *code* -in sync with the en/ source-or-truth notebooks. It intentionally ignores -Markdown cells and only compares code cells. There must be the same amount of -code cells in source notebook and translation notebook. - -Usage: nb_code_sync.py [--lang=en] site/lang/notebook.ipynb [...] - -Useful when used with interactive git workflow to selectively add hunks: -git add --patch site/lang/notebook.ipynb -Commands: - y: stage this hunk - n: do not stage this hunk - s: split this hunk - e: edit this hunk -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import errno -import json -import os -import pathlib -import re -import sys -from absl import app -from absl import flags - -flags.DEFINE_enum("lang", "en", ["en", "js", "ko", "pt", "ru", "tr", "zh-cn"], - "Language directory to import from.") -flags.DEFINE_string("src", None, "Source file or parent directory of source.") -flags.DEFINE_boolean("stdout", False, "Write to stdout instead of file.") -flags.DEFINE_string("site_root", None, "Root directory of site docs.") - - -class Notebook(object): - """Represents a parsed .ipynb notebook file. - - Attributes: - path: Path to the notebook file. - data: All cells parsed from notebook. - code_cells: Only code cells parsed from notebook. - """ - - path = None - - def __init__(self, data): - """Inits Notebook from parsed .ipynb notebook data.""" - self.data = data - self.code_cells = self._load_code_cells(self.data) - - @classmethod - def from_path(cls, path): - """Inits Notebook using path to .pynb file.""" - pth = Notebook._check_path(path) - with open(pth) as json_data: - data = json.load(json_data) - nb = Notebook(data) - nb.path = pth - return nb - - @staticmethod - def is_notebook(path): - """Test of a file is an .ipynb file based on extension.""" - if not os.path.isfile(path): - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) - return os.path.splitext(path)[-1].lower() == ".ipynb" - - @staticmethod - def _check_path(pth): - if not Notebook.is_notebook(pth): - raise Exception("Notebook must be an .ipynb file: {}".format(pth)) - path = pathlib.Path(pth) - if not path.exists(): - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) - return path - - def _load_code_cells(self, data): - # parse code cells - code_cells = [c for c in data["cells"] if c["cell_type"] == "code"] - # Discard last cell if empty - cell_source = code_cells[-1]["source"] - # remove empty strings, then test if anything is left - if not any(cell_source): - del code_cells[-1] - return code_cells - - @staticmethod - def _strip_line(line): - """Remove comments and any trailing whitespace.""" - line = re.sub(r"^(.*?)#(.*)$", r"\1", line) - return line.rstrip() - - @staticmethod - def _is_source_code_equal(x_list, y_list): - """Scrub lines of comments, remove empty lines, then compare.""" - x_list = [Notebook._strip_line(line) for line in x_list if line] - y_list = [Notebook._strip_line(line) for line in y_list if line] - return x_list == y_list - - def _set_cell_source(self, cell_id, source): - for i, cell in enumerate(self.data["cells"]): - if cell["metadata"]["id"] == cell_id: - self.data["cells"][i]["source"] = source - break - else: - # for-loop exhausted - raise Exception("Did not find cell id '{}' in notebook.".format(cell_id)) - - def update(self, notebook): - """Update code cells that differ from the provided notebook.""" - if len(self.code_cells) != len(notebook.code_cells): - raise Exception("Notebooks must have same amount of code cells.") - # Iterate all cells for destination reference - for i, src_cell in enumerate(notebook.code_cells): - dest_cell = self.code_cells[i] - # Compare source code after scrubbing comments. - # Ensures translated comments are preserved until the code changes. - if not Notebook._is_source_code_equal(src_cell["source"], - dest_cell["source"]): - self._set_cell_source(dest_cell["metadata"]["id"], src_cell["source"]) - - def write(self, use_stdout=False): - """Write notebook to file or print to screen.""" - def print_file(outfile): - json.dump(self.data, outfile, indent=2, ensure_ascii=False) - outfile.write("\n") # add trailing newline - - if use_stdout: - print_file(sys.stdout) - else: - with open(self.path, "w") as outfile: - print_file(outfile) - print("Wrote: {}".format(self.path)) - - -def get_src_path(user_flags, notebook): - """Get path of source notebook based on user flags or the destination file. - - Args: - user_flags: Command-line arguments - notebook: Destination notebook used to select source notebook. - Returns: - A Path of the source-of-truth notebook. - Raises: - FileNotFoundError: If user args for site_root or src are invalid locations. - """ - if user_flags.site_root: - site_root = pathlib.Path(user_flags.site_root) - else: - site_root = pathlib.Path(__file__).parent.parent.joinpath("site") - if not site_root.is_dir(): - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), site_root) - - if not user_flags.src: - # Determine path from previous notebook and source language - fp_relpath = notebook.path.relative_to(site_root) # relative path - fp_relpath = pathlib.Path(*fp_relpath.parts[1:]) - return site_root.joinpath(user_flags.lang, fp_relpath) - elif os.path.isdir(user_flags.src): - return pathlib.Path(user_flags.src) / notebook.path.name - elif os.path.isfile(user_flags.src): - return pathlib.Path(user_flags.src) - else: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), - user_flags.src) - - -def main(argv): - if len(argv) < 2: - raise app.UsageError("Missing command-line arguments.") - - for dest_path in argv[1:]: - if not Notebook.is_notebook(dest_path): - print("Not a notebook file, skipping: {}".format(dest_path), - file=sys.stderr) - continue - - dest_notebook = Notebook.from_path(dest_path) - - src_path = get_src_path(flags.FLAGS, dest_notebook) - src_notebook = Notebook.from_path(src_path) - - dest_notebook.update(src_notebook) - dest_notebook.write(flags.FLAGS.stdout) - - -if __name__ == "__main__": - app.run(main) diff --git a/tools/release_tools/push_version_to_github.sh b/tools/release_tools/push_version_to_github.sh new file mode 100644 index 00000000000..074fedcfdea --- /dev/null +++ b/tools/release_tools/push_version_to_github.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Run this from your github clone. +# It assumes you have tensorflow_docs as the "upstream" remote. + +OLD_BRANCH=$1 +NEW_BRANCH=$2 + +if [ -n "$OLD_BRANCH" ]; then + echo "OLD_BRANCH=$OLD_BRANCH" +else + echo '$OLD_BRANCH is unset' && exit; +fi + +if [ -n "$NEW_BRANCH" ]; then + echo "NEW_BRANCH=$NEW_BRANCH" +else + echo '$NEW_BRANCH is unset' && exit +fi + +# Print commands while running, and fail if a command fails +set -e +set -x + +# Merge the previous releases's docs into the current. +git checkout master +git pull upstream master + +git fetch upstream +git branch -D $NEW_BRANCH || echo "failed -> branch doesn't exist -> that\'s ok" +# Checkout the upstream branch, if it doesn't exist create it from master +git checkout --track upstream/"$NEW_BRANCH" || (git checkout -b "$NEW_BRANCH") +# Merge the previous version. If that gives a merge-conflict, auto-resolve and commit. +git merge upstream/$OLD_BRANCH --message "merge $OLD_BRANCH" || (grep -l "<<<<<<<" -r | xargs git checkout --ours && git commit -am "merge $OLD_BRANCH") + +# Overwrite the ditectory with the contents from tensorflow/en/versions +rm -rf site/en/api_docs/python +python ../tensorflow/tensorflow/tools/docs/generate2.py --output_dir=site/en/api_docs/python --code_url_prefix="https://github.com/tensorflow/tensorflow/blob/${NEW_BRANCH}/tensorflow" + +# Commit and push to your github. +git add site/en/api_docs/ +git commit -am "Update docs to $NEW_BRANCH" +git push -f origin $NEW_BRANCH diff --git a/tools/spelltest/spelltest.sh b/tools/spelltest/spelltest.sh index c5653e41890..ff6125695f2 100755 --- a/tools/spelltest/spelltest.sh +++ b/tools/spelltest/spelltest.sh @@ -14,7 +14,7 @@ ## Count also also includes code cells: ## $ spelltest -c -C notebook.ipynb [...] ## -## Print notebook as Markdown to stdout and save to clipoard (OSX): +## Print notebook as Markdown to stdout and save to clipboard (OSX): ## $ spelltest -p [-C] notebook.ipynb | pbcopy ## set -e @@ -56,7 +56,7 @@ done shift $((OPTIND - 1)) -## Check requirmeents: aspell and nbconvert +## Check requirements: aspell and nbconvert if [[ ! -x "$(which aspell)" ]]; then echo "${LOG_NAME} Error: Requires the 'aspell' command" >&2 @@ -105,6 +105,12 @@ read_file_contents() { fi contents="$($NBCONVERT_BIN $opts --stdout $fp 2>/dev/null)" + # Check that nbconvert ran. Virtualenvs get hosed on Python upgrades, etc. + if [[ $? != 0 ]]; then + echo "[$(basename ${NBCONVERT_BIN})] Error: " >&2 + $NBCONVERT_BIN $opts --stdout "$fp" >&2 + exit 1 + fi else echo "${LOG_NAME} Error: File format not supported: ${fp}" >&2 diff --git a/tools/spelltest/wordlist.txt b/tools/spelltest/wordlist.txt index 8c5abb22b9f..c81cf71c3a7 100644 --- a/tools/spelltest/wordlist.txt +++ b/tools/spelltest/wordlist.txt @@ -1,43 +1,73 @@ +ansatz APIs autoencoder backend backpropagation +bazel bfloat bytecode CIFAR +Cirq Colab Colaboratory +convolutional CPUs CSV CUDA +datapoints dataset datasets +differentiator +differentiators dtype +eigensolver +eigenstate +eigenstates +excitations +GHZ GPU GPUs ImageNet initializer JSON Keras +learnable LSTM +manylinux +MERA MNIST NaNs +NISQ NumPy +observables optimizer perceptron +prepend +preprocess pseudocode PyPI +QCNN +QNN +QNNs +qubit +qubits quickstart ResNet RNN +runnable runtime SavedModel sigmoid softmax +subsampling TensorBoard TensorFlow +TFQ TPU TPUs +translationally VAE +variational VGG +VQE XLA diff --git a/tools/templates/build_docs.py b/tools/templates/build_docs.py index d53e569d1c1..f0d24a1386f 100644 --- a/tools/templates/build_docs.py +++ b/tools/templates/build_docs.py @@ -20,44 +20,73 @@ $> python build_docs.py """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +import os from absl import app from absl import flags -import tensorflow_docs +import tensorflow_docs.api_generator from tensorflow_docs.api_generator import generate_lib from tensorflow_docs.api_generator import public_api PROJECT_SHORT_NAME = 'tfdocs' PROJECT_FULL_NAME = 'TensorFlow Docs' -CODE_URL_PREFIX = 'https://github.com/tensorflow/docs/tree/master/tools/tensorflow_docs' -FLAGS = flags.FLAGS - -flags.DEFINE_string( +_OUTPUT_DIR = flags.DEFINE_string( 'output_dir', default='/tmp/generated_docs', help='Where to write the resulting docs to.') +_URL_PREFIX = flags.DEFINE_string( + 'code_url_prefix', 'https://github.com/tensorflow/docs/tree/master/tools/tensorflow_docs', + 'The url prefix for links to code.') + +_SEARCH_HINTS = flags.DEFINE_bool( + 'search_hints', True, + 'Include metadata search hints in the generated files') + +_SITE_PATH = flags.DEFINE_string('site_path', '/api_docs/python', + 'Path prefix in the _toc.yaml') + -def main(argv): - if argv[1:]: - raise ValueError('Unrecognized arguments: {}'.format(argv[1:])) +def gen_api_docs(): + """Generates api docs for the tensorflow docs package.""" + + # The below `del`'s are to avoid the api_gen_test to not document these. + # Please remove these lines from your build_docs.py files when you create + # them. + try: + del tensorflow_docs.google + except AttributeError: + pass + + try: + del tensorflow_docs.api_generator.report.schema + except AttributeError: + pass doc_generator = generate_lib.DocGenerator( root_title=PROJECT_FULL_NAME, - # Replace `tensorflow_docs` with your module, here. - py_modules=[(PROJECT_SHORT_NAME, tensorflow_docs)], - code_url_prefix=CODE_URL_PREFIX, - # This callback cleans up a lot of aliases caused by internal imports. - callbacks=[public_api.local_definitions_filter]) + # Replace `tensorflow_docs.api_generator` with your module, here. + py_modules=[(PROJECT_SHORT_NAME, tensorflow_docs.api_generator)], + # Replace `tensorflow_docs.api_generator` with your module, here. + base_dir=os.path.dirname(tensorflow_docs.api_generator.__file__), + code_url_prefix=_URL_PREFIX.value, + search_hints=_SEARCH_HINTS.value, + site_path=_SITE_PATH.value, + # This callback ensures that docs are only generated for objects that + # are explicitly imported in your __init__.py files. There are other + # options but this is a good starting point. + callbacks=[public_api.explicit_package_contents_filter], + ) + + doc_generator.build(_OUTPUT_DIR.value) + + print('Output docs to: ', _OUTPUT_DIR.value) - doc_generator.build(FLAGS.output_dir) - print('Output docs to: ', FLAGS.output_dir) +def main(_): + gen_api_docs() if __name__ == '__main__': diff --git a/tools/templates/notebook.ipynb b/tools/templates/notebook.ipynb index 1f6bc782f66..7a68b3f7c50 100644 --- a/tools/templates/notebook.ipynb +++ b/tools/templates/notebook.ipynb @@ -3,20 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Tce3stUlHN0L" }, "source": [ - "##### Copyright 2019 The TensorFlow Authors." + "##### Copyright 2020 The TensorFlow Authors." ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "tuOe1ymfHZPu" }, "outputs": [], @@ -37,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "qFdPvlXBOdUN" }, "source": [ @@ -47,40 +43,42 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MfBg1C5NB3X0" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/not_a_real_link\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/tools/templates/notebook.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/tools/templates/notebook.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/docs/tools/templates/notebook.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View on GitHub\n", + " \n", + " Download notebook\n", + " \n", + " See TF Hub model\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "r6P32iYYV27b" }, "source": [ - "[Update button links]" + "[Update button links]\n", + "\n", + "*See model on TFHub* is only required if the notebook uses a model from [tfhub.dev](https://tfhub.dev)" ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xHxb-dlhMIzW" }, "source": [ @@ -92,7 +90,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "MUXex9ctTuDB" }, "source": [ @@ -102,7 +99,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "1Eh-iCRVBm0p" }, "source": [ @@ -111,18 +107,13 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "IqR2PQG4ZaZ0" }, "outputs": [], "source": [ - "from __future__ import absolute_import, division, print_function, unicode_literals\n", - "\n", - "import tensorflow.compat.v2 as tf\n", - "tf.enable_v2_behavior()\n", + "import tensorflow as tf\n", "\n", "import numpy as np" ] @@ -130,7 +121,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "UhNtHfuxCGVy" }, "source": [ @@ -144,7 +134,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "2V22fKegUtF9" }, "source": [ @@ -154,19 +143,19 @@ "* Save the notebook with the table of contents open.\n", "* Use one `H1` header for the title.\n", "* Include the button-bar immediately after the `H1`.\n", - "* Avoid using `H1` headers for section titles. Use `H2` and `H3` instead.\n", + "* Headers that are`H4` and below are not visible in the navigation\n", + "bar of [tensorflow.org](http://www.tensorflow.org).\n", "* Include an overview section before any code.\n", "* Put all your installs and imports in a setup section.\n", - "* Always include the `__future__` imports.\n", - "* Write Python 3 compatible code.\n", "* Keep code and text cells as brief as possible.\n", - "* Avoid leaving an empty cell at the end of the notebook." + "* Break text cells at headings\n", + "* Break code cells between \"building\" and \"running\", and between \"printing one result\" and \"printing another result\".\n", + "* Necessary but uninteresting code (like plotting logic) should be hidden in a toggleable code cell by putting `#@title` as the first line." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "QKp40qS-DGEZ" }, "source": [ @@ -176,15 +165,14 @@ "* Use the [Google Python Style Guide](http://google.github.io/styleguide/pyguide.html), where applicable.\n", "* tensorflow.org doesn't support interactive plots.\n", "* Keep examples quick. Use small datasets, or small slices of datasets. Don't train to convergence, train until it's obvious it's making progress.\n", + "* If you define a function, run it and show us what it does before using it in another function.\n", "* Demonstrate small parts before combining them into something more complex, like this:" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "KtylpxOmceaC" }, "outputs": [], @@ -199,7 +187,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "pwdM2pl3RSPb" }, "source": [ @@ -208,10 +195,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "mMOeXVmbdilM" }, "outputs": [], @@ -227,7 +212,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "uabQmjMtRtzs" }, "source": [ @@ -236,10 +220,8 @@ }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { - "colab": {}, - "colab_type": "code", "id": "U82B_tH2d294" }, "outputs": [], @@ -251,24 +233,22 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "TJdqBNBbS78n" }, "source": [ "### Code content\n", "\n", "* Use the highest level API that gets the job done (unless the goal is to demonstrate the low level API).\n", - "* Use `keras.Sequential` \u003e keras functional api \u003e keras model subclassing \u003e ...\n", - "* Use `model.fit` \u003e `model.train_on_batch` \u003e manual `GradientTapes`.\n", - "* Use eager-style code.\n", + "* Use `keras.Sequential` > keras functional api > keras model subclassing > ...\n", + "* Use `model.fit` > `model.train_step` > manual `GradientTapes`.\n", "* Use `tensorflow_datasets` and `tf.data` where possible.\n", + "* When using pre-trained models, prefer models from [tfhub.dev](https://tfhub.dev) where possible.\n", "* Avoid `compat.v1`." ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "78HBT9cQXJko" }, "source": [ @@ -283,7 +263,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "YrsKXcPRUvK9" }, "source": [ @@ -293,7 +272,7 @@ "* This notebook has the \"Omit code cell output when saving this notebook\" option set. GitHub refuses to diff notebooks with large diffs (inline images).\n", "* [ReviewNB.com](http://reviewnb.com) can help with diffs. This is linked in a comment on a notebook pull request.\n", "* Use the [Open in Colab](https://chrome.google.com/webstore/detail/open-in-colab/iogfkhleblhcpcekbiedikdehleodpjo) extension to open a GitHub notebook in Colab.\n", - "* The easiest way to edit a notebook in GitHub is to open it with Colab from the branch you want to edit. Then use File --\u003e Save a copy in GitHub, which will save it back to the branch you opened it from.\n", + "* The easiest way to edit a notebook in GitHub is to open it with Colab from the branch you want to edit. Then use File --> Save a copy in GitHub, which will save it back to the branch you opened it from.\n", "* For PRs it's helpful to post a direct Colab link to the PR head: https://colab.research.google.com/github/{USER}/{REPO}/blob/{BRANCH}/{PATH}.ipynb" ] } @@ -304,10 +283,7 @@ "Tce3stUlHN0L" ], "name": "notebook.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/tools/templates/subsite/g3doc/_book.yaml b/tools/templates/subsite/g3doc/_book.yaml index 9c4bac877a2..8125466f078 100644 --- a/tools/templates/subsite/g3doc/_book.yaml +++ b/tools/templates/subsite/g3doc/_book.yaml @@ -14,26 +14,28 @@ upper_tabs: - name: Tutorials contents: - title: Tutorials overview - path: /PROJECT_NAME/tutorials/ + path: /{PROJECT_NAME}/tutorials/ - title: Foo tutorial - path: /PROJECT_NAME/tutorials/foo_tutorial + path: /{PROJECT_NAME}/tutorials/foo_tutorial # - title: Notebook tutorial -# path: /PROJECT_NAME/tutorials/notebook +# path: /{PROJECT_NAME}/tutorials/notebook - name: Guide contents: - title: Guide overview - path: /PROJECT_NAME/overview + path: /{PROJECT_NAME}/overview - title: Bar guide - path: /PROJECT_NAME/bar_guide + path: /{PROJECT_NAME}/bar_guide - name: Examples contents: - title: Examples overview - path: /PROJECT_NAME/examples/ + path: /{PROJECT_NAME}/examples/ - title: Baz example - path: /PROJECT_NAME/examples/baz_example + path: /{PROJECT_NAME}/examples/baz_example - name: API skip_translation: true contents: - - include: /PROJECT_NAME/api_docs/python/_toc.yaml + - title: All Symbols + path: /{PROJECT_NAME}/api_docs/python/{SHORT_NAME}/all_symbols + - include: /{PROJECT_NAME}/api_docs/python/{SHORT_NAME}/_toc.yaml - include: /_upper_tabs_right.yaml diff --git a/tools/templates/subsite/g3doc/_index.yaml b/tools/templates/subsite/g3doc/_index.yaml index 4f7d9bb4301..401d2cb77c2 100644 --- a/tools/templates/subsite/g3doc/_index.yaml +++ b/tools/templates/subsite/g3doc/_index.yaml @@ -1,6 +1,7 @@ book_path: /PROJECT_NAME/_book.yaml project_path: /PROJECT_NAME/_project.yaml -description: +description: > + Page description used for search and social. landing_page: custom_css_path: /site-assets/css/style.css rows: diff --git a/tools/templates/subsite/g3doc/tutorials/notebook.ipynb b/tools/templates/subsite/g3doc/tutorials/notebook.ipynb index f7cf245e2c6..9a8c484a0b6 100644 --- a/tools/templates/subsite/g3doc/tutorials/notebook.ipynb +++ b/tools/templates/subsite/g3doc/tutorials/notebook.ipynb @@ -3,21 +3,17 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Tce3stUlHN0L" }, "source": [ - "##### Copyright 2018 The TensorFlow Authors.\n", - "\n" + "##### Copyright 2018 The TensorFlow Authors.\n" ] }, { "cell_type": "code", - "execution_count": 0, + "execution_count": null, "metadata": { "cellView": "form", - "colab": {}, - "colab_type": "code", "id": "tuOe1ymfHZPu" }, "outputs": [], @@ -38,7 +34,6 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "s7Bo2MipUnXX" }, "source": [ @@ -48,33 +43,28 @@ { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "Birwb-khUOIq" }, "source": [ - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/{PATH}\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003e\n", - " View on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/tools/templates/subsite/g3doc/tutorials/notebook.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003e\n", - " Run in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/tools/templates/subsite/g3doc/tutorials/notebook.ipynb\"\u003e\n", - " \u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003e\n", - " View source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "\n", + " \n", + " \n", + " \n", + " \n", + "
    \n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
    " ] }, { "cell_type": "markdown", "metadata": { - "colab_type": "text", "id": "xHxb-dlhMIzW" }, "source": [ @@ -90,10 +80,7 @@ "Tce3stUlHN0L" ], "name": "notebook.ipynb", - "private_outputs": true, - "provenance": [], - "toc_visible": true, - "version": "0.3.2" + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/tools/templates/tflite_model_page_template.md b/tools/templates/tflite_model_page_template.md new file mode 100644 index 00000000000..af1486018a1 --- /dev/null +++ b/tools/templates/tflite_model_page_template.md @@ -0,0 +1,61 @@ +# Title (ML Task) + +Short description of ML Task. Link off to relevant content on the TF docs for +background info. + +## Get started + +Image of model output (preferably GIFs) + + + +If you are new to TensorFlow Lite and are working with Android or iOS, we +recommend exploring the following example applications that can help you get +started. + +Download +Android model +Download +iOS model + +If you are using a platform other than Android/iOS, or if you are already +familiar with the +[TensorFlow Lite APIs](https://www.tensorflow.org/api_docs/python/tf/lite), +download the starter model and supporting files (if applicable). + +Download +starter model + +## Model description + +In this section, include content about: + +### How it works + +* How does the model work? Provide usage instructions with examples as appropriate. + + +### Inputs + +* Which format does the model expect inputs in? +* How can the user convert inputs of a certain type into a compatible format? + +### Outputs + +* How does the user interpret the model results? + +### Limitations + +* What can the user *not* do with the model? + +## Model customization + +* How can the user customize this model to work with their datasets? + +## Performance benchmarks + +## Further reading and resources + +* Academic paper (if applicable) +* Use cases +* Other resources diff --git a/tools/tensorflow_docs/__init__.py b/tools/tensorflow_docs/__init__.py index 99f39c10a2a..b51b74966b5 100644 --- a/tools/tensorflow_docs/__init__.py +++ b/tools/tensorflow_docs/__init__.py @@ -13,9 +13,3 @@ # limitations under the License. # ============================================================================== """tensorflow_docs is a package for generating python api-reference docs.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow_docs import api_generator diff --git a/tools/tensorflow_docs/api_generator/__init__.py b/tools/tensorflow_docs/api_generator/__init__.py index 8a6d9c2ec31..325a9eb075e 100644 --- a/tools/tensorflow_docs/api_generator/__init__.py +++ b/tools/tensorflow_docs/api_generator/__init__.py @@ -13,16 +13,12 @@ # limitations under the License. # ============================================================================== """Tools for building tensorflow api reference docs.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from tensorflow_docs.api_generator import doc_controls from tensorflow_docs.api_generator import doc_generator_visitor from tensorflow_docs.api_generator import generate_lib from tensorflow_docs.api_generator import parser -from tensorflow_docs.api_generator import pretty_docs +from tensorflow_docs.api_generator import toc_processing from tensorflow_docs.api_generator import public_api -from tensorflow_docs.api_generator import tf_inspect from tensorflow_docs.api_generator import traverse from tensorflow_docs.api_generator import utils diff --git a/tools/tensorflow_docs/api_generator/compat_test/__init__.py b/tools/tensorflow_docs/api_generator/compat_test/__init__.py new file mode 100644 index 00000000000..dba383f4c29 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/compat_test/__init__.py @@ -0,0 +1,88 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This is a test module. + +@compatibility(TF2) +test +@end_compatibility + +Hello +""" + +from tensorflow_docs.api_generator.compat_test import estimator + + +def a_function(x, y): + """This is a function. + + @compatibility(TF2) + test + @end_compatibility + + @compatibility(numpy) + test + @end_compatibility + + It does things. + + Args: + x: x + y: y + + Returns: + None + """ + del x + del y + return None + + +class AClass: + """This is a class. + + @compatibility(TF2) + test + @end_compatibility + + It does things too. + + Attributes: + x: x + y: x + """ + + def __init__(self, x, y): + self.x = x + self.y = y + + def a_method(self, x, y): + """Methods can have compatibility notes too. + + @compatibility(TF2) + test + @end_compatibility + + It does things too. + + Args: + x: x + y: y + + Returns: + None + """ + del x + del y + return None diff --git a/tools/tensorflow_docs/api_generator/compat_test/estimator.py b/tools/tensorflow_docs/api_generator/compat_test/estimator.py new file mode 100644 index 00000000000..0b54303ca39 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/compat_test/estimator.py @@ -0,0 +1,86 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This is a test module. + +@compatibility(TF2) +test +@end_compatibility + +Hello +""" + + +def a_function(x, y): + """This is a function. + + @compatibility(TF2) + test + @end_compatibility + + @compatibility(numpy) + test + @end_compatibility + + It does things. + + Args: + x: x + y: y + + Returns: + None + """ + del x + del y + return None + + +class AClass: + """This is a class. + + @compatibility(TF2) + test + @end_compatibility + + It does things too. + + Attributes: + x: x + y: x + """ + + def __init__(self, x, y): + self.x = x + self.y = y + + def a_method(self, x, y): + """Methods can have compatibility notes too. + + @compatibility(TF2) + test + @end_compatibility + + It does things too. + + Args: + x: x + y: y + + Returns: + None + """ + del x + del y + return None diff --git a/tools/tensorflow_docs/api_generator/config.py b/tools/tensorflow_docs/api_generator/config.py new file mode 100644 index 00000000000..f2c3a3daf60 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/config.py @@ -0,0 +1,70 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The `ParserConfig` contains the information extracted by walking the API.""" + +class ParserConfig(object): + """Stores all indexes required to parse the docs.""" + + def __init__( + self, + *, + reference_resolver, + duplicates, + duplicate_of, + tree, + index, + reverse_index, + path_tree, + api_tree, + base_dir, + code_url_prefix, + self_link_base + ): + """Object with the common config for docs_for_object() calls. + + Args: + reference_resolver: An instance of ReferenceResolver. + duplicates: A `dict` mapping fully qualified names to a set of all aliases + of this name. This is used to automatically generate a list of all + aliases for each name. + duplicate_of: A map from duplicate names to preferred names of API + symbols. + tree: A `dict` mapping a fully qualified name to the names of all its + members. Used to populate the members section of a class or module page. + index: A `dict` mapping full names to objects. + reverse_index: A `dict` mapping object ids to full names. + path_tree: A PathTree datastructure to manage all the API paths. + api_tree: A PathTree datastructure to manage all the API objects. + base_dir: A base path that is stripped from file locations written to the + docs. + code_url_prefix: A Url to pre-pend to the links to file locations. + self_link_base: A Url to pre-pend to self-links to the generated docs + pages. + """ + self.reference_resolver = reference_resolver + self.duplicates = duplicates + self.duplicate_of = duplicate_of + self.tree = tree + self.reverse_index = reverse_index + self.index = index + self.path_tree = path_tree + self.api_tree = api_tree + self.base_dir = base_dir + self.code_url_prefix = code_url_prefix + self.self_link_base = self_link_base + + def py_name_to_object(self, full_name): + """Return the Python object for a Python symbol name.""" + return self.index[full_name] diff --git a/tools/tensorflow_docs/api_generator/doc_controls.py b/tools/tensorflow_docs/api_generator/doc_controls.py index aba41c3aa9b..ec8e0e862fd 100644 --- a/tools/tensorflow_docs/api_generator/doc_controls.py +++ b/tools/tensorflow_docs/api_generator/doc_controls.py @@ -13,15 +13,87 @@ # limitations under the License. # ============================================================================== """Documentation control decorators.""" +from typing import Iterable, Optional, TypeVar + +T = TypeVar("T") + +_DEPRECATED = "_tf_docs_deprecated" + + +def set_deprecated(obj: T) -> T: + """Explicitly tag an object as deprecated for the doc generator.""" + setattr(obj, _DEPRECATED, None) + return obj + + +def is_deprecated(obj) -> bool: + return hasattr(obj, _DEPRECATED) + + +_INHERITABLE_HEADER = "_tf_docs_inheritable_header" + + +def inheritable_header(text: str): + + def _wrapped(obj): + setattr(obj, _INHERITABLE_HEADER, text) + return obj + + return _wrapped + + +def get_inheritable_header(obj) -> Optional[str]: + return getattr(obj, _INHERITABLE_HEADER, None) + + +header = inheritable_header +get_header = get_inheritable_header + + +_NO_SEARCH_HINTS = "_tf_docs_no_search_hints" + + + +def hide_from_search(obj: T) -> T: + """Marks an object so metadata search hints will not be included on it's page. + + The page is set to "noindex" to hide it from search. + + Note: This only makes sense to apply to functions, classes and modules. + Constants, and methods do not get their own pages. + + Args: + obj: the object to hide. + + Returns: + The object. + """ + setattr(obj, _NO_SEARCH_HINTS, None) + return obj + + +def should_hide_from_search(obj) -> bool: + """Returns true if metadata search hints should not be included.""" + return hasattr(obj, _NO_SEARCH_HINTS) + + +_CUSTOM_PAGE_BUILDER_CLS = "_tf_docs_custom_page_builder_cls" + + +def set_custom_page_builder_cls(obj, cls): + """Replace most of the generated page with custom content.""" + setattr(obj, _CUSTOM_PAGE_BUILDER_CLS, cls) + + +def get_custom_page_builder_cls(obj): + """Gets custom page content if available.""" + return getattr(obj, _CUSTOM_PAGE_BUILDER_CLS, None) -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function _DO_NOT_DOC = "_tf_docs_do_not_document" -def do_not_generate_docs(obj): +def do_not_generate_docs(obj: T) -> T: """A decorator: Do not generate docs for this object. For example the following classes: @@ -102,7 +174,7 @@ def x(self): _DO_NOT_DOC_INHERITABLE = "_tf_docs_do_not_doc_inheritable" -def do_not_doc_inheritable(obj): +def do_not_doc_inheritable(obj: T) -> T: """A decorator: Do not generate docs for this method. This version of the decorator is "inherited" by subclasses. No docs will be @@ -165,7 +237,7 @@ def x(self): _FOR_SUBCLASS_IMPLEMENTERS = "_tf_docs_tools_for_subclass_implementers" -def for_subclass_implementers(obj): +def for_subclass_implementers(obj: T) -> T: """A decorator: Only generate docs for this method in the defining class. Also group this method's docs with and `@abstractmethod` in the class's docs. @@ -242,11 +314,111 @@ def x(self): do_not_doc_in_subclasses = for_subclass_implementers +_DOC_PRIVATE = "_tf_docs_doc_private" + + +def doc_private(obj: T) -> T: + """A decorator: Generates docs for private methods/functions. + + For example: + + ``` + class Try: + + @doc_controls.doc_private + def _private(self): + ... + ``` + + As a rule of thumb, private(beginning with `_`) methods/functions are + not documented. + + This decorator allows to force document a private method/function. + + Args: + obj: The class-attribute to hide from the generated docs. + + Returns: + obj + """ + + setattr(obj, _DOC_PRIVATE, None) + return obj + + +def should_doc_private(obj) -> bool: + return hasattr(obj, _DOC_PRIVATE) + -def should_skip(obj): +_DOC_IN_CURRENT_AND_SUBCLASSES = "_tf_docs_doc_in_current_and_subclasses" + + +def doc_in_current_and_subclasses(obj: T) -> T: + """Overrides `do_not_doc_in_subclasses` decorator. + + If this decorator is set on a child class's method whose parent's method + contains `do_not_doc_in_subclasses`, then that will be overridden and the + child method will get documented. All classes inheriting from the child will + also document that method. + + For example: + + ``` + class Parent: + @do_not_doc_in_subclasses + def method1(self): + pass + def method2(self): + pass + + class Child1(Parent): + @doc_in_current_and_subclasses + def method1(self): + pass + def method2(self): + pass + + class Child2(Parent): + def method1(self): + pass + def method2(self): + pass + + class Child11(Child1): + pass + ``` + + This will produce the following docs: + + ``` + /Parent.md + # method1 + # method2 + /Child1.md + # method1 + # method2 + /Child2.md + # method2 + /Child11.md + # method1 + # method2 + ``` + + Args: + obj: The class-attribute to hide from the generated docs. + + Returns: + obj + """ + + setattr(obj, _DOC_IN_CURRENT_AND_SUBCLASSES, None) + return obj + + +def should_skip(obj) -> bool: """Returns true if docs generation should be skipped for this object. - checks for the `do_not_generate_docs` or `do_not_doc_inheritable` decorators. + Checks for the `do_not_generate_docs` or `do_not_doc_inheritable` decorators. Args: obj: The object to document, or skip. @@ -267,6 +439,7 @@ def should_skip(obj): return hasattr(obj, _DO_NOT_DOC) or hasattr(obj, _DO_NOT_DOC_INHERITABLE) + def _unwrap_func(obj): # Unwrap fget if the object is a property or static method or classmethod. if isinstance(obj, property): @@ -277,6 +450,26 @@ def _unwrap_func(obj): return obj + +def _cls_attr_has_tag(cls, attr, tag): + """Check if a class attribute `attr` is decorated with `dec`.""" + # Use __dict__, it doesn't go up the __mro__ like getattr. + obj = cls.__dict__.get(attr, None) + if obj is None: + return False + obj = _unwrap_func(obj) + + if isinstance(obj, type): + # The attribute is a class. Check __dict__ to see if the attribute is set + # on _this_ class, not its parents. + if tag in obj.__dict__: + return True + else: + return False + + return hasattr(obj, tag) + + def should_skip_class_attr(cls, name): """Returns true if docs should be skipped for this class attribute. @@ -292,10 +485,8 @@ def should_skip_class_attr(cls, name): try: obj = getattr(cls, name) except AttributeError: - # Avoid error caused by enum metaclasses in python3 - if name in ("name", "value"): - return True - raise + # This can fail for a variety of reasons. Always skip if `getattr` fails. + return True # Unwrap fget if the object is a property obj = _unwrap_func(obj) @@ -305,36 +496,61 @@ def should_skip_class_attr(cls, name): if should_skip(obj): return True - # Use __dict__ lookup to get the version defined in *this* class. - obj = cls.__dict__.get(name, None) - obj = _unwrap_func(obj) + classes = getattr(cls, "__mro__", [cls]) + + # Find where all the decorators turn docs on and off. + # All these lists contain `(level, skip)` pairs. + for_subclass_levels = [ + # The [1:] is because `for_subclass_implementers` turns off docs + # one level down (and you don't want to consider level -1). + (i, True) + for (i, mro_cls) in enumerate(classes[1:]) + if _cls_attr_has_tag(mro_cls, name, _FOR_SUBCLASS_IMPLEMENTERS) + ] + not_below_levels = [ + (i, True) + for (i, mro_cls) in enumerate(classes) + if _cls_attr_has_tag(mro_cls, name, _DO_NOT_DOC_INHERITABLE) + ] + doc_below_levels = [ + (i, False) + for (i, mro_cls) in enumerate(classes) + if _cls_attr_has_tag(mro_cls, name, _DOC_IN_CURRENT_AND_SUBCLASSES) + ] + + all_levels = not_below_levels + for_subclass_levels + doc_below_levels + if all_levels: + # Find the lowest `(level, skip)` pair, and return `skip` + return min(all_levels)[1] + + # No decorators --> don't skip + return False - if obj is not None: - # If not none, the object is defined in *this* class. - # Do not skip if decorated with `for_subclass_implementers`. - if hasattr(obj, _FOR_SUBCLASS_IMPLEMENTERS): - return False - # for each parent class - for parent in getattr(cls, "__mro__", [])[1:]: - # if the class should be skipped, don't doc this object. - if should_skip(parent): - return True +def decorate_all_class_attributes(decorator, cls, skip: Iterable[str]): + """Applies `decorator` to every attribute defined in `cls`. - obj = getattr(parent, name, None) + Args: + decorator: The decorator to apply. + cls: The class to apply the decorator to. + skip: A collection of attribute names that the decorator should not be + aplied to. + """ + skip = frozenset(skip) + class_contents = list(cls.__dict__.items()) - if obj is None: + for name, obj in class_contents: + if name in skip: continue - obj = _unwrap_func(obj) - - # Skip if the parent's definition is decorated with `do_not_doc_inheritable` - # or `for_subclass_implementers` - if hasattr(obj, _DO_NOT_DOC_INHERITABLE): - return True + # Otherwise, exclude from documentation. + if isinstance(obj, property): + obj = obj.fget - if hasattr(obj, _FOR_SUBCLASS_IMPLEMENTERS): - return True + if isinstance(obj, (staticmethod, classmethod)): + obj = obj.__func__ - # No blockng decorators --> don't skip - return False + try: + decorator(obj) + except AttributeError: + pass diff --git a/tools/tensorflow_docs/api_generator/doc_controls_test.py b/tools/tensorflow_docs/api_generator/doc_controls_test.py index 62c70e23804..9aeadc21915 100644 --- a/tools/tensorflow_docs/api_generator/doc_controls_test.py +++ b/tools/tensorflow_docs/api_generator/doc_controls_test.py @@ -175,7 +175,7 @@ class Grand2Child(Child): self.assertTrue( doc_controls.should_skip_class_attr(Grand2Child, 'my_method')) - def test_for_subclass_implementers_short_circuit(self): + def test_lowest_decorator_wins(self): class GrandParent(object): @@ -190,7 +190,7 @@ def my_method(self): class Child(Parent): - @doc_controls.do_not_doc_inheritable + @doc_controls.doc_in_current_and_subclasses def my_method(self): pass @@ -200,18 +200,59 @@ class GrandChild(Child): def my_method(self): pass - class Grand2Child(Child): + class Grand2Child(GrandChild): pass self.assertFalse( doc_controls.should_skip_class_attr(GrandParent, 'my_method')) self.assertTrue(doc_controls.should_skip_class_attr(Parent, 'my_method')) - self.assertTrue(doc_controls.should_skip_class_attr(Child, 'my_method')) + self.assertFalse(doc_controls.should_skip_class_attr(Child, 'my_method')) self.assertFalse( doc_controls.should_skip_class_attr(GrandChild, 'my_method')) self.assertTrue( doc_controls.should_skip_class_attr(Grand2Child, 'my_method')) + def test_nested_class_do_not_document(self): + + class Outer1: + + @doc_controls.do_not_generate_docs + class Inner: + pass + + class Outer2: + Inner = Outer1.Inner + + class Outer3: + + class Inner(Outer1.Inner): + pass + + self.assertTrue(doc_controls.should_skip_class_attr(Outer1, 'Inner')) + self.assertTrue(doc_controls.should_skip_class_attr(Outer2, 'Inner')) + self.assertFalse(doc_controls.should_skip_class_attr(Outer3, 'Inner')) + + def test_nested_class_inheritable_decorators(self): + + class Outer1: + + @doc_controls.do_not_doc_inheritable + class Inner: + pass + + class Outer2(Outer1): + pass + + class Outer3(Outer2): + + @doc_controls.doc_in_current_and_subclasses + class Inner(Outer2.Inner): + pass + + self.assertTrue(doc_controls.should_skip_class_attr(Outer1, 'Inner')) + self.assertTrue(doc_controls.should_skip_class_attr(Outer2, 'Inner')) + self.assertFalse(doc_controls.should_skip_class_attr(Outer3, 'Inner')) + def test_skip_class_short_circuit(self): class GrandParent(object): @@ -233,7 +274,32 @@ class Child(Parent): self.assertFalse( doc_controls.should_skip_class_attr(GrandParent, 'my_method')) self.assertFalse(doc_controls.should_skip_class_attr(Parent, 'my_method')) - self.assertTrue(doc_controls.should_skip_class_attr(Child, 'my_method')) + self.assertFalse(doc_controls.should_skip_class_attr(Child, 'my_method')) + + def test_doc_in_current_and_subclasses(self): + + class Parent: + + @doc_controls.do_not_doc_in_subclasses + def my_method(self): + pass + + class Child1(Parent): + + @doc_controls.doc_in_current_and_subclasses + def my_method(self): + pass + + class Child11(Child1): + pass + + class Child2(Parent): + pass + + self.assertFalse(doc_controls.should_skip_class_attr(Parent, 'my_method')) + self.assertFalse(doc_controls.should_skip_class_attr(Child1, 'my_method')) + self.assertFalse(doc_controls.should_skip_class_attr(Child11, 'my_method')) + self.assertTrue(doc_controls.should_skip_class_attr(Child2, 'my_method')) if __name__ == '__main__': diff --git a/tools/tensorflow_docs/api_generator/doc_generator_visitor.py b/tools/tensorflow_docs/api_generator/doc_generator_visitor.py index c5997201488..1c6f31eb3da 100644 --- a/tools/tensorflow_docs/api_generator/doc_generator_visitor.py +++ b/tools/tensorflow_docs/api_generator/doc_generator_visitor.py @@ -13,19 +13,27 @@ # limitations under the License. # ============================================================================== """A `traverse` visitor for processing documentation.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import annotations import collections +import dataclasses +import enum +import inspect +import logging + + +from typing import Any, Dict, List, Optional, NamedTuple, Sequence, Tuple + +from tensorflow_docs.api_generator import obj_type as obj_type_lib + -import six +# To see the logs pass: --logger_levels=tensorflow_docs:DEBUG --alsologtostderr +_LOGGER = logging.getLogger(__name__) -from tensorflow_docs.api_generator import tf_inspect +ApiPath = Tuple[str, ...] -def maybe_singleton(py_object): +def maybe_singleton(py_object: Any) -> bool: """Returns `True` if `py_object` might be a singleton value . Many immutable values in python act like singletons: small ints, some strings, @@ -43,106 +51,111 @@ def maybe_singleton(py_object): A bool, True if the object might be a singleton. """ # isinstance accepts nested tuples of types. - is_immutable_type = isinstance( - py_object, (six.integer_types, six.string_types, six.binary_type, - six.text_type, float, complex, bool, type(None))) + immutable_types = (int, str, bytes, float, complex, bool, type(None)) + is_immutable_type = isinstance(py_object, immutable_types) # Check if the object is the empty tuple. - return is_immutable_type or py_object is () # pylint: disable=literal-comparison + return is_immutable_type or (isinstance(py_object, tuple) and py_object == ()) # pylint: disable=g-explicit-bool-comparison -class ApiTreeNode(object): - """Represents a single API end-point. +@dataclasses.dataclass +class PathTreeNode(object): + """Represents a path to an object in the API, an object can have many paths. Attributes: path: A tuple of strings containing the path to the object from the root like `('tf', 'losses', 'hinge')` - obj: The python object. - children: A dictionary from short name to `ApiTreeNode`, including the - children nodes. - parent: The parent node. + py_object: The python object. + children: A dictionary from short name to `PathTreeNode`, of this node's + children. + parent: This node's parent. This is a tree, there can only be one. short_name: The last path component full_name: All path components joined with "." """ + path: ApiPath + py_object: Any + parent: Optional[PathTreeNode] = None + children: Dict[str, PathTreeNode] = dataclasses.field(default_factory=dict) - def __init__(self, path, obj, parent): - self.path = path - self.obj = obj - self.children = {} - self.parent = parent + def __hash__(self): + return id(self) + + def __repr__(self): + return f'{type(self).__name__}({self.full_name})' + + __str__ = __repr__ + + def __eq__(self, other): + raise ValueError("Don't try to compare these") @property - def short_name(self): + def short_name(self) -> str: return self.path[-1] @property - def full_name(self): + def full_name(self) -> str: return '.'.join(self.path) -class ApiTree(object): - """Represents all api end-points as a tree. +class PathTree(Dict[ApiPath, PathTreeNode]): + """An index/tree of all object-paths in the API. Items must be inserted in order, from root to leaf. + Attributes: - index: A dict, mapping from path tuples to `ApiTreeNode`. - aliases: A dict, mapping from object ids to a list of all `ApiTreeNode` that - refer to the object. - root: The root `ApiTreeNode` + root: The root `PathTreeNode` """ def __init__(self): - root = ApiTreeNode((), None, None) - self.index = {(): root} - self.aliases = collections.defaultdict(list) - self.root = root + root = PathTreeNode(path=(), py_object=None, parent=None, children={}) + super().__setitem__((), root) - def __contains__(self, path): - """Returns `True` if path exists in the tree. + self.root: PathTreeNode = root + self._nodes_for_id: Dict[int, List[PathTreeNode]] = ( + collections.defaultdict(list)) - Args: - path: A tuple of strings, the api path to the object. + def __eq__(self, other): + raise ValueError("Don't try to compare these") - Returns: - True if `path` exists in the tree. - """ - return path in self.index + def iter_nodes(self): + """Iterate over the nodes in BFS order.""" + stack = collections.deque([self.root]) + while stack: + children = list(stack.popleft().children.values()) + yield from children + stack.extend(children) - def __getitem__(self, path): - """Fetch an item from the tree. - - Args: - path: A tuple of strings, the api path to the object. + def __contains__(self, path: ApiPath) -> bool: # pylint: disable=useless-super-delegation + # TODO(b/184563451): remove + return super().__contains__(path) - Returns: - An `ApiTreeNode`. - - Raises: - KeyError: If no node can be found at that path. - """ - return self.index[path] - - def __setitem__(self, path, obj): + def __setitem__(self, path: ApiPath, obj: Any): """Add an object to the tree. Args: path: A tuple of strings. obj: The python object. """ + assert path not in self + parent_path = path[:-1] - parent = self.index[parent_path] + parent = self[parent_path] - node = ApiTreeNode(path=path, obj=obj, parent=parent) + node = PathTreeNode(path=path, py_object=obj, parent=parent) - self.index[path] = node + super().__setitem__(path, node) if not maybe_singleton(obj): # We cannot use the duplicate mechanism for some constants, since e.g., # id(c1) == id(c2) with c1=1, c2=1. This isn't problematic since constants # have no usable docstring and won't be documented automatically. - self.aliases[id(obj)].append(node) + nodes = self.nodes_for_obj(obj) + nodes.append(node) parent.children[node.short_name] = node + def nodes_for_obj(self, py_object) -> List[PathTreeNode]: + return self._nodes_for_id[id(py_object)] + class DocGeneratorVisitor(object): """A visitor that generates docs for a python object when __call__ed.""" @@ -157,26 +170,27 @@ def __init__(self): This object accumulates the various data-structures necessary to build the docs, including (see the property definitions for details.): - In the decsription below "master name" is the object's preferred fully + In the description below "main name" is the object's preferred fully qualified name. Params: - index: A mapping from master names to python python objects. - tree: A mapping from master names to a list if attribute names. - reverse_index: Mapping from python object ids to master names. + index: A mapping from main names to python python objects. + tree: A mapping from main names to a list if attribute names. + reverse_index: Mapping from python object ids to main names. Note that this doesn't work for python numbers, strings or tuples. duplicate_of: A mapping from a fully qualified names to the object's - master name. The master names are not included as keys. - duplicates: A mapping from master names to lists of other fully qualified + main name. The main names are not included as keys. + duplicates: A mapping from main names to lists of other fully qualified names for the object. """ - self._index = {} - self._tree = {} - self._reverse_index = None - self._duplicates = None - self._duplicate_of = None + self._index: Dict[str, Any] = {} + self._tree: Dict[str, List[str]] = {} + self._reverse_index: Dict[int, str] = None + self._duplicates: Dict[str, List[str]] = None + self._duplicate_of: Dict[str, str] = None - self._api_tree = ApiTree() + self.path_tree = PathTree() + self.api_tree = None @property def index(self): @@ -213,7 +227,6 @@ def reverse_index(self): Returns: The `id(object)` to full name map. """ - self._maybe_find_duplicates() return self._reverse_index @property @@ -227,24 +240,22 @@ def duplicate_of(self): Returns: The map from duplicate name to preferred name. """ - self._maybe_find_duplicates() return self._duplicate_of @property def duplicates(self): """A map from preferred full names to a list of all names for this symbol. - This function returns a map from preferred (master) name for a symbol to a - lexicographically sorted list of all aliases for that name (incl. the master + This function returns a map from preferred (main) name for a symbol to a + lexicographically sorted list of all aliases for that name (incl. the main name). Symbols without duplicate names do not appear in this map. It is computed when it, `reverse_index`, or `duplicate_of` are first accessed. Returns: - The map from master name to list of all duplicate names. + The map from main name to list of all duplicate names. """ - self._maybe_find_duplicates() return self._duplicates def __call__(self, parent_path, parent, children): @@ -259,8 +270,8 @@ def __call__(self, parent_path, parent, children): found during traversal. parent: The Python object referenced by `parent_name`. children: A list of `(name, py_object)` pairs enumerating, in alphabetical - order, the children (as determined by `tf_inspect.getmembers`) of - `parent`. `name` is the local name of `py_object` in `parent`. + order, the children (as determined by `inspect.getmembers`) of + `parent`. `name` is the local name of `py_object` in `parent`. Returns: The list of children, with any __metaclass__ removed. @@ -272,15 +283,16 @@ class or module. parent_name = '.'.join(parent_path) self._index[parent_name] = parent self._tree[parent_name] = [] - if parent_path not in self._api_tree: - self._api_tree[parent_path] = parent + if parent_path not in self.path_tree: + self.path_tree[parent_path] = parent - if not (tf_inspect.ismodule(parent) or tf_inspect.isclass(parent)): - raise RuntimeError('Unexpected type in visitor -- %s: %r' % (parent_name, - parent)) + if not (inspect.ismodule(parent) or inspect.isclass(parent)): + raise TypeError('Unexpected type in visitor -- ' + f'{parent_name}: {parent!r}') - for (name, child) in children: - self._api_tree[parent_path + (name,)] = child + for name, child in children: + child_path = parent_path + (name,) + self.path_tree[child_path] = child full_name = '.'.join([parent_name, name]) if parent_name else name self._index[full_name] = child @@ -288,7 +300,14 @@ class or module. return children - def _score_name(self, name): + class NameScore(NamedTuple): + defining_class_score: int + experimental_score: int + keras_score: int + module_length_score: int + path: ApiPath + + def _score_name(self, path: ApiPath) -> NameScore: """Return a tuple of scores indicating how to sort for the best name. This function is meant to be used as the `key` to the `sorted` function. @@ -304,76 +323,96 @@ def _score_name(self, name): name: Fallback, sorts lexicographically on the full_name. Args: - name: the full name to score, for example `tf.estimator.Estimator` + path: APiPath to score, for example `('tf','estimator','Estimator')` Returns: A tuple of scores. When sorted the preferred name will have the lowest value. """ - parts = name.split('.') - short_name = parts[-1] - if len(parts) == 1: - return (-99, -99, -99, -99, short_name) + py_object = self.path_tree[path].py_object + if len(path) == 1: + return self.NameScore(-99, -99, -99, -99, path) - container = self._index['.'.join(parts[:-1])] + short_name = path[-1] + container = self.path_tree[path[:-1]].py_object - defining_class_score = 1 - if tf_inspect.isclass(container): + # Prefer the reference that is not in a class. + defining_class_score = -1 + container_type = obj_type_lib.ObjType.get(container) + if container_type is obj_type_lib.ObjType.CLASS: if short_name in container.__dict__: - # prefer the defining class - defining_class_score = -1 + # If a alias points into a class, prefer the defining class + defining_class_score = 0 + else: + defining_class_score = 1 experimental_score = -1 - if 'contrib' in parts or any('experimental' in part for part in parts): + if 'contrib' in path or any('experimental' in part for part in path): experimental_score = 1 keras_score = 1 - if 'keras' in parts: + if 'keras' in path: keras_score = -1 - while parts: - container = self._index['.'.join(parts)] - if tf_inspect.ismodule(container): + if inspect.ismodule(py_object): + # prefer short paths for modules + module_length_score = len(path) + else: + module_length_score = self._get_module_length_score(path) + + return self.NameScore( + defining_class_score=defining_class_score, + experimental_score=experimental_score, + keras_score=keras_score, + module_length_score=module_length_score, + path=path) + + def _get_module_length_score(self, path): + partial_path = list(path) + while partial_path: + container = self.path_tree[tuple(partial_path[:-1])].py_object + partial_path.pop() + if inspect.ismodule(container): break - parts.pop() - module_length = len(parts) + module_length = len(partial_path) - if len(parts) == 2: + if module_length == 2: # `tf.submodule.thing` is better than `tf.thing` module_length_score = -1 else: # shorter is better module_length_score = module_length - return (defining_class_score, experimental_score, keras_score, - module_length_score, name) + return module_length_score - def _maybe_find_duplicates(self): + def build(self): """Compute data structures containing information about duplicates. - Find duplicates in `index` and decide on one to be the "master" name. + Find duplicates in `index` and decide on one to be the "main" name. - Computes a reverse_index mapping each object id to its master name. + Computes a reverse_index mapping each object id to its main name. - Also computes a map `duplicate_of` from aliases to their master name (the - master name itself has no entry in this map), and a map `duplicates` from - master names to a lexicographically sorted list of all aliases for that name - (incl. the master name). + Also computes a map `duplicate_of` from aliases to their main name (the + main name itself has no entry in this map), and a map `duplicates` from + main names to a lexicographically sorted list of all aliases for that name + (incl. the main name). All these are computed and set as fields if they haven't already. """ if self._reverse_index is not None: return + self.api_tree = ApiTree.from_path_tree(self.path_tree, self._score_name) + # Maps the id of a symbol to its fully qualified name. For symbols that have # several aliases, this map contains the first one found. # We use id(py_object) to get a hashable value for py_object. Note all # objects in _index are in memory at the same time so this is safe. reverse_index = {} - # Decide on master names, rewire duplicates and make a duplicate_of map - # mapping all non-master duplicates to the master name. The master symbol + # Decide on main names, rewire duplicates and make a duplicate_of map + # mapping all non-main duplicates to the main name. The main symbol # does not have an entry in this map. duplicate_of = {} @@ -381,37 +420,256 @@ def _maybe_find_duplicates(self): # symbol (incl. itself). duplicates = {} - for path, node in six.iteritems(self._api_tree.index): + for path, node in self.path_tree.items(): + _LOGGER.debug('DocGeneratorVisitor.build') + _LOGGER.debug(' path: %s', path) + if not path: continue full_name = node.full_name - py_object = node.obj + py_object = node.py_object object_id = id(py_object) if full_name in duplicates: continue - aliases = self._api_tree.aliases[object_id] + aliases = self.path_tree.nodes_for_obj(py_object) + # maybe_singleton types can't be looked up by object. if not aliases: aliases = [node] - names = [alias.full_name for alias in aliases] + name_tuples = [alias.path for alias in aliases] - names = sorted(names) - # Choose the master name with a lexical sort on the tuples returned by + # Choose the main name with a lexical sort on the tuples returned by # by _score_name. - master_name = min(names, key=self._score_name) + main_name_tuple = min(name_tuples, key=self._score_name) + main_name = '.'.join(main_name_tuple) - if names: - duplicates[master_name] = list(names) + names = ['.'.join(name_tuple) for name_tuple in name_tuples] + if name_tuples: + duplicates[main_name] = sorted(names) - names.remove(master_name) for name in names: - duplicate_of[name] = master_name + if name != main_name: + duplicate_of[name] = main_name # Set the reverse index to the canonical name. if not maybe_singleton(py_object): - reverse_index[object_id] = master_name + reverse_index[object_id] = main_name self._duplicate_of = duplicate_of self._duplicates = duplicates self._reverse_index = reverse_index + + +@dataclasses.dataclass(repr=False) +class ApiTreeNode(PathTreeNode): + """A node in the ApiTree.""" + aliases: List[ApiPath] = dataclasses.field(default_factory=list) + physical_path: Optional[ApiPath] = None + + @property + def obj_type(self) -> obj_type_lib.ObjType: + return obj_type_lib.ObjType.get(self.py_object) + + class OutputType(enum.Enum): + PAGE = 'page' + FRAGMENT = 'fragment' + + def output_type(self) -> OutputType: + obj_type = obj_type_lib.ObjType.get(self.py_object) + + if obj_type in (obj_type_lib.ObjType.CLASS, obj_type_lib.ObjType.MODULE): + return self.OutputType.PAGE + elif obj_type in (obj_type_lib.ObjType.CALLABLE, + obj_type_lib.ObjType.TYPE_ALIAS): + assert self.parent is not None + parent_type = obj_type_lib.ObjType.get(self.parent.py_object) + if parent_type is obj_type_lib.ObjType.CLASS: + return self.OutputType.FRAGMENT + else: + return self.OutputType.PAGE + else: + return self.OutputType.FRAGMENT + + +class ApiTree(Dict[ApiPath, ApiTreeNode]): + """Public API index. + + Items must be inserted in order from root to leaves. + + Lookup a path-tuple to fetch a node: + + ``` + node = index[path] + ``` + + Use the `node_from_obj` method to lookup the node for a python object: + + ``` + node = index.node_from_obj(obj) + ``` + + Remember that `maybe_singleton` (numbers, strings, tuples) classes can't be + looked up this way. + + To build a tree, nodes must be inserted in tree order starting from the root. + + + Attributes: + root: The root `ApiFileNode` of the tree. + """ + + def __init__(self): + root = ApiTreeNode( + path=(), py_object=None, parent=None, aliases=[()]) # type: ignore + self.root = root + super().__setitem__((), root) + self._nodes = [] + self._node_for_object = {} + + def __eq__(self, other): + raise ValueError("Don't try to compare these") + + def node_for_object(self, obj: Any) -> Optional[ApiTreeNode]: + if maybe_singleton(obj): + return None + return self._node_for_object.get(id(obj), None) + + def __contains__(self, path: ApiPath) -> bool: # pylint: disable=useless-super-delegation + # TODO(b/184563451): remove + return super().__contains__(path) + + def iter_nodes(self): + """Iterate over the nodes in BFS order.""" + stack = collections.deque([self.root]) + while stack: + children = list(stack.popleft().children.values()) + yield from children + stack.extend(children) + + def __setitem__(self, *args, **kwargs): + raise TypeError('Use .insert instead of setitem []') + + def insert(self, path: ApiPath, py_object: Any, aliases: List[ApiPath]): + """Add an object to the index.""" + _LOGGER.debug('ApiTree.insert') + _LOGGER.debug(' path: %s', path) + _LOGGER.debug(' py_object: %s', py_object) + _LOGGER.debug(' aliases: %s', aliases) + assert path not in self, 'A path was inserted twice.' + + parent_path = path[:-1] + parent = self[parent_path] + + node = ApiTreeNode( + path=path, + py_object=py_object, + aliases=aliases, + parent=parent, + physical_path=self._get_physical_path(py_object)) + + super().__setitem__(path, node) + self._nodes.append(node) + for alias in aliases: + if alias == path: + continue + assert alias not in self + super().__setitem__(alias, node) + + self._node_for_object[id(node.py_object)] = node + + parent.children[node.short_name] = node + + def _get_physical_path(self, py_object): + physical_path = None + obj_type = obj_type_lib.ObjType.get(py_object) + if obj_type in [obj_type.CLASS, obj_type.CALLABLE]: + try: + physical_path = tuple( + py_object.__module__.split('.') + py_object.__qualname__.split('.')) + except AttributeError: + pass + elif obj_type is obj_type.MODULE: + physical_path = tuple(py_object.__name__.split('.')) + + return physical_path + + @classmethod + def from_path_tree(cls, path_tree: PathTree, score_name_fn) -> ApiTree: + """Create an ApiTree from a PathTree. + + Args: + path_tree: The `PathTree` to convert. + score_name_fn: The name scoring function. + + Returns: + an `ApiIndex`, created from `path_tree`. + """ + self = cls() + + active_nodes = collections.deque(path_tree.root.children.values()) + while active_nodes: + current_node = active_nodes.popleft() + if current_node.path in self: + continue + + duplicate_nodes = set( + path_tree.nodes_for_obj(current_node.py_object)) + + if not duplicate_nodes: + # Singleton objects will return `[]`. So look up the parent object's + # duplicate nodes and collect their children. + assert current_node.parent is not None + parent_nodes = path_tree.nodes_for_obj(current_node.parent.py_object) + duplicate_nodes = [ + parent_node.children[current_node.short_name] + for parent_node in parent_nodes + ] + + parents = [ + node.parent for node in duplicate_nodes if node.parent is not None + ] + + # Choose the priority name with a lexical sort on the tuples returned by + # _score_name. + if not all(parent.path in self for parent in parents): + # rewind + active_nodes.appendleft(current_node) + # do each duplicate's immediate parents first. + for parent in parents: + if parent.path in self: + continue + active_nodes.appendleft(parent) + continue + # If we've made it here, the immediate parents of each of the paths have + # been processed, so now we can choose its priority name. + aliases = [node.path for node in duplicate_nodes] + + priority_path = self._choose_priority_path(aliases, score_name_fn) + + if priority_path is None: + # How did this happen? + # No parents in the public api -> you are not in the public API. + continue + + self.insert(priority_path, current_node.py_object, aliases) + + active_nodes.extend(current_node.children.values()) + + return self + + def _choose_priority_path(self, aliases: Sequence[ApiPath], + score_name_fn) -> Optional[ApiPath]: + # Only consider a path an option for the priority_path if its parent-path + # is the priority_path for that object. + priority_path_options = [] + for alias in aliases: + parent_path = alias[:-1] + + if self[parent_path].path == parent_path: + priority_path_options.append(alias) + + try: + return min(priority_path_options, key=score_name_fn) + except ValueError: + return None diff --git a/tools/tensorflow_docs/api_generator/doc_generator_visitor_test.py b/tools/tensorflow_docs/api_generator/doc_generator_visitor_test.py index 48142403193..87a181698b9 100644 --- a/tools/tensorflow_docs/api_generator/doc_generator_visitor_test.py +++ b/tools/tensorflow_docs/api_generator/doc_generator_visitor_test.py @@ -14,18 +14,17 @@ # ============================================================================== """Tests for tools.docs.doc_generator_visitor.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse +import dataclasses +import io import os +import textwrap import types from absl.testing import absltest from tensorflow_docs.api_generator import doc_generator_visitor from tensorflow_docs.api_generator import generate_lib +from tensorflow_docs.api_generator import toc as toc_lib class NoDunderVisitor(doc_generator_visitor.DocGeneratorVisitor): @@ -38,6 +37,17 @@ def __call__(self, parent_name, parent, children): return super(NoDunderVisitor, self).__call__(parent_name, parent, children) +class TestDocGenerator(generate_lib.DocGenerator): + + def __init__(self, py_modules): + kwargs = {} + kwargs['py_modules'] = py_modules + kwargs['root_title'] = 'TensorFlow' + kwargs['visitor_cls'] = NoDunderVisitor + kwargs['code_url_prefix'] = '/' + super().__init__(**kwargs) + + class DocGeneratorVisitorTest(absltest.TestCase): def test_call_module(self): @@ -48,6 +58,7 @@ def test_call_module(self): self.assertEqual({'doc_generator_visitor': ['DocGeneratorVisitor']}, visitor.tree) + self.assertEqual({ 'doc_generator_visitor': doc_generator_visitor, 'doc_generator_visitor.DocGeneratorVisitor': @@ -55,23 +66,28 @@ def test_call_module(self): }, visitor.index) def test_call_class(self): + + class ExampleClass: + + def example_method(self): + pass + visitor = doc_generator_visitor.DocGeneratorVisitor() visitor( - ('DocGeneratorVisitor',), doc_generator_visitor.DocGeneratorVisitor, - [('index', doc_generator_visitor.DocGeneratorVisitor.reverse_index)]) + parent_path=('ExampleClass',), + parent=ExampleClass, + children=[('example_method', ExampleClass.example_method)]) - self.assertEqual({'DocGeneratorVisitor': ['index']}, - visitor.tree) - self.assertEqual({ - 'DocGeneratorVisitor': - doc_generator_visitor.DocGeneratorVisitor, - 'DocGeneratorVisitor.index': - doc_generator_visitor.DocGeneratorVisitor.reverse_index - }, visitor.index) + self.assertEqual({'ExampleClass': ['example_method']}, visitor.tree) + self.assertEqual( + { + 'ExampleClass': ExampleClass, + 'ExampleClass.example_method': ExampleClass.example_method, + }, visitor.index) def test_call_raises(self): visitor = doc_generator_visitor.DocGeneratorVisitor() - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): visitor(('non_class_or_module',), 'non_class_or_module_object', []) def test_duplicates_module_class_depth(self): @@ -87,40 +103,41 @@ class Nested(object): tf.submodule = types.ModuleType('submodule') tf.submodule.Parent = Parent - visitor = generate_lib.extract( - [('tf', tf)], - base_dir=os.path.dirname(tf.__file__), - private_map={}, - do_not_descend_map={}, - visitor_cls=NoDunderVisitor) + config = TestDocGenerator([('tf', tf)]).run_extraction() self.assertEqual( { - 'tf.submodule.Parent': - sorted([ - 'tf.Parent', - 'tf.submodule.Parent', - ]), - 'tf.submodule.Parent.Nested': - sorted([ - 'tf.Parent.Nested', - 'tf.submodule.Parent.Nested', - ]), + 'tf.submodule.Parent': sorted([ + 'tf.Parent', + 'tf.submodule.Parent', + ]), + 'tf.submodule.Parent.Nested': sorted([ + 'tf.Parent.Nested', + 'tf.submodule.Parent.Nested', + ]), 'tf': ['tf'], - 'tf.submodule': ['tf.submodule'] - }, visitor.duplicates) + 'tf.submodule': ['tf.submodule'], + }, + config.duplicates, + ) - self.assertEqual({ - 'tf.Parent.Nested': 'tf.submodule.Parent.Nested', - 'tf.Parent': 'tf.submodule.Parent', - }, visitor.duplicate_of) + self.assertEqual( + { + 'tf.Parent.Nested': 'tf.submodule.Parent.Nested', + 'tf.Parent': 'tf.submodule.Parent', + }, + config.duplicate_of, + ) - self.assertEqual({ - id(Parent): 'tf.submodule.Parent', - id(Parent.Nested): 'tf.submodule.Parent.Nested', - id(tf): 'tf', - id(tf.submodule): 'tf.submodule', - }, visitor.reverse_index) + self.assertEqual( + { + id(Parent): 'tf.submodule.Parent', + id(Parent.Nested): 'tf.submodule.Parent.Nested', + id(tf): 'tf', + id(tf.submodule): 'tf.submodule', + }, + config.reverse_index, + ) def test_duplicates_contrib(self): @@ -134,27 +151,29 @@ class Parent(object): tf.contrib.Parent = Parent tf.submodule.Parent = Parent - visitor = generate_lib.extract( - [('tf', tf)], - base_dir=os.path.dirname(tf.__file__), - private_map={}, - do_not_descend_map={}, - visitor_cls=NoDunderVisitor) + config = TestDocGenerator([('tf', tf)]).run_extraction() - self.assertEqual( - sorted(['tf.contrib.Parent', 'tf.submodule.Parent']), - visitor.duplicates['tf.submodule.Parent']) + self.assertCountEqual( + ['tf.contrib.Parent', 'tf.submodule.Parent'], + config.duplicates['tf.submodule.Parent'], + ) - self.assertEqual({ - 'tf.contrib.Parent': 'tf.submodule.Parent', - }, visitor.duplicate_of) + self.assertEqual( + { + 'tf.contrib.Parent': 'tf.submodule.Parent', + }, + config.duplicate_of, + ) - self.assertEqual({ - id(tf): 'tf', - id(tf.submodule): 'tf.submodule', - id(Parent): 'tf.submodule.Parent', - id(tf.contrib): 'tf.contrib', - }, visitor.reverse_index) + self.assertEqual( + { + id(tf): 'tf', + id(tf.submodule): 'tf.submodule', + id(Parent): 'tf.submodule.Parent', + id(tf.contrib): 'tf.contrib', + }, + config.reverse_index, + ) def test_duplicates_defining_class(self): @@ -169,29 +188,28 @@ class Child(Parent): tf.Parent = Parent tf.Child = Child - visitor = generate_lib.extract( - [('tf', tf)], - base_dir=os.path.dirname(tf.__file__), - private_map={}, - do_not_descend_map={}, - visitor_cls=NoDunderVisitor) + config = TestDocGenerator([('tf', tf)]).run_extraction() - self.assertEqual( - sorted([ - 'tf.Parent.obj1', - 'tf.Child.obj1', - ]), visitor.duplicates['tf.Parent.obj1']) + self.assertCountEqual( + ['tf.Parent.obj1', 'tf.Child.obj1'], config.duplicates['tf.Parent.obj1'] + ) - self.assertEqual({ - 'tf.Child.obj1': 'tf.Parent.obj1', - }, visitor.duplicate_of) + self.assertEqual( + { + 'tf.Child.obj1': 'tf.Parent.obj1', + }, + config.duplicate_of, + ) - self.assertEqual({ - id(tf): 'tf', - id(Parent): 'tf.Parent', - id(Child): 'tf.Child', - id(Parent.obj1): 'tf.Parent.obj1', - }, visitor.reverse_index) + self.assertEqual( + { + id(tf): 'tf', + id(Parent): 'tf.Parent', + id(Child): 'tf.Child', + id(Parent.obj1): 'tf.Parent.obj1', + }, + config.reverse_index, + ) def test_duplicates_module_depth(self): @@ -205,27 +223,26 @@ class Parent(object): tf.Parent = Parent tf.submodule.submodule2.Parent = Parent - visitor = generate_lib.extract( - [('tf', tf)], - base_dir=os.path.dirname(tf.__file__), - private_map={}, - do_not_descend_map={}, - visitor_cls=NoDunderVisitor) + config = TestDocGenerator([('tf', tf)]).run_extraction() - self.assertEqual( - sorted(['tf.Parent', 'tf.submodule.submodule2.Parent']), - visitor.duplicates['tf.Parent']) + self.assertCountEqual( + ['tf.Parent', 'tf.submodule.submodule2.Parent'], + config.duplicates['tf.Parent'], + ) - self.assertEqual({ - 'tf.submodule.submodule2.Parent': 'tf.Parent' - }, visitor.duplicate_of) + self.assertEqual( + {'tf.submodule.submodule2.Parent': 'tf.Parent'}, config.duplicate_of + ) - self.assertEqual({ - id(tf): 'tf', - id(tf.submodule): 'tf.submodule', - id(tf.submodule.submodule2): 'tf.submodule.submodule2', - id(Parent): 'tf.Parent', - }, visitor.reverse_index) + self.assertEqual( + { + id(tf): 'tf', + id(tf.submodule): 'tf.submodule', + id(tf.submodule.submodule2): 'tf.submodule.submodule2', + id(Parent): 'tf.Parent', + }, + config.reverse_index, + ) def test_duplicates_name(self): @@ -239,37 +256,135 @@ class Parent(object): tf.submodule = types.ModuleType('submodule') tf.submodule.Parent = Parent - visitor = generate_lib.extract( - [('tf', tf)], - base_dir=os.path.dirname(tf.__file__), - private_map={}, - do_not_descend_map={}, - visitor_cls=NoDunderVisitor) + config = TestDocGenerator([('tf', tf)]).run_extraction() + self.assertEqual( sorted([ 'tf.submodule.Parent.obj1', 'tf.submodule.Parent.obj2', - ]), visitor.duplicates['tf.submodule.Parent.obj1']) + ]), + config.duplicates['tf.submodule.Parent.obj1'], + ) - self.assertEqual({ - 'tf.submodule.Parent.obj2': 'tf.submodule.Parent.obj1', - }, visitor.duplicate_of) + self.assertEqual( + { + 'tf.submodule.Parent.obj2': 'tf.submodule.Parent.obj1', + }, + config.duplicate_of, + ) - self.assertEqual({ - id(tf): 'tf', - id(tf.submodule): 'tf.submodule', - id(Parent): 'tf.submodule.Parent', - id(Parent.obj1): 'tf.submodule.Parent.obj1', - }, visitor.reverse_index) + self.assertEqual( + { + id(tf): 'tf', + id(tf.submodule): 'tf.submodule', + id(Parent): 'tf.submodule.Parent', + id(Parent.obj1): 'tf.submodule.Parent.obj1', + }, + config.reverse_index, + ) + def test_handles_duplicate_classmethods(self): -class ApiTreeTest(absltest.TestCase): + class MyClass: + + @classmethod + def from_value(cls, value): + pass + + tf = types.ModuleType('fake_tf') + tf.__file__ = '/tmp/tf/__init__.py' + tf.MyClass = MyClass + tf.sub = types.ModuleType('sub') + tf.sub.MyClass = MyClass + + config = TestDocGenerator([('tf', tf)]).run_extraction() + + paths = ['.'.join(p) for p in config.path_tree.keys()] + + expected = [ + '', + 'tf', + 'tf.MyClass', + 'tf.MyClass.from_value', + 'tf.sub', + 'tf.sub.MyClass', + 'tf.sub.MyClass.from_value', + ] + self.assertCountEqual(expected, paths) + + apis = [node.full_name for node in config.api_tree.iter_nodes()] + expected = [ + 'tf', + 'tf.sub', + 'tf.sub.MyClass', + 'tf.sub.MyClass.from_value', + ] + self.assertCountEqual(expected, apis) + + self.assertIs( + config.api_tree[('tf', 'MyClass')], + config.api_tree[('tf', 'sub', 'MyClass')], + ) + self.assertIs( + config.api_tree[('tf', 'MyClass', 'from_value')], + config.api_tree[('tf', 'sub', 'MyClass', 'from_value')], + ) + + def test_handles_duplicate_singleton_attributes(self): + + class MyClass: + simple = 1 + + tf = types.ModuleType('fake_tf') + tf.__file__ = '/tmp/tf/__init__.py' + tf.MyClass = MyClass + tf.sub = types.ModuleType('sub') + tf.sub.MyClass = MyClass + + config = TestDocGenerator([('tf', tf)]).run_extraction() + + paths = ['.'.join(p) for p in config.path_tree.keys()] + + expected = [ + '', + 'tf', + 'tf.MyClass', + 'tf.MyClass.simple', + 'tf.sub', + 'tf.sub.MyClass', + 'tf.sub.MyClass.simple', + ] + self.assertCountEqual(expected, paths) + + apis = ['.'.join(p) for p in config.api_tree.keys()] + expected = [ + '', + 'tf', + 'tf.MyClass', + 'tf.MyClass.simple', + 'tf.sub', + 'tf.sub.MyClass', + 'tf.sub.MyClass.simple', + ] + self.assertCountEqual(expected, apis) + + self.assertIs( + config.api_tree[('tf', 'MyClass')], + config.api_tree[('tf', 'sub', 'MyClass')], + ) + self.assertIs( + config.api_tree[('tf', 'MyClass', 'simple')], + config.api_tree[('tf', 'sub', 'MyClass', 'simple')], + ) + + +class PathTreeTest(absltest.TestCase): def test_contains(self): - tf = argparse.Namespace() - tf.sub = argparse.Namespace() + tf = types.ModuleType('tf') + tf.sub = types.ModuleType('sub') - tree = doc_generator_visitor.ApiTree() + tree = doc_generator_visitor.PathTree() tree[('tf',)] = tf tree[('tf', 'sub')] = tf.sub @@ -277,30 +392,30 @@ def test_contains(self): self.assertIn(('tf', 'sub'), tree) def test_node_insertion(self): - tf = argparse.Namespace() - tf.sub = argparse.Namespace() + tf = types.ModuleType('tf') + tf.sub = types.ModuleType('sub') tf.sub.object = object() - tree = doc_generator_visitor.ApiTree() + tree = doc_generator_visitor.PathTree() tree[('tf',)] = tf tree[('tf', 'sub')] = tf.sub tree[('tf', 'sub', 'thing')] = tf.sub.object node = tree[('tf', 'sub')] self.assertEqual(node.full_name, 'tf.sub') - self.assertIs(node.obj, tf.sub) + self.assertIs(node.py_object, tf.sub) self.assertIs(node.parent, tree[('tf',)]) self.assertLen(node.children, 1) self.assertIs(node.children['thing'], tree[('tf', 'sub', 'thing')]) def test_duplicate(self): - tf = argparse.Namespace() - tf.sub = argparse.Namespace() + tf = types.ModuleType('tf') + tf.sub = types.ModuleType('sub') tf.sub.thing = object() - tf.sub2 = argparse.Namespace() + tf.sub2 = types.ModuleType('sub2') tf.sub2.thing = tf.sub.thing - tree = doc_generator_visitor.ApiTree() + tree = doc_generator_visitor.PathTree() tree[('tf',)] = tf tree[('tf', 'sub')] = tf.sub tree[('tf', 'sub', 'thing')] = tf.sub.thing @@ -308,25 +423,232 @@ def test_duplicate(self): tree[('tf', 'sub2', 'thing')] = tf.sub2.thing self.assertCountEqual( - tree.aliases[id(tf.sub.thing)], + tree.nodes_for_obj(tf.sub.thing), [tree[('tf', 'sub', 'thing')], tree[('tf', 'sub2', 'thing')]]) def test_duplicate_singleton(self): - tf = argparse.Namespace() - tf.sub = argparse.Namespace() + tf = types.ModuleType('tf') + tf.sub = types.ModuleType('sub') tf.sub.thing = 999 - tf.sub2 = argparse.Namespace() + tf.sub2 = types.ModuleType('sub2') tf.sub2.thing = tf.sub.thing - tree = doc_generator_visitor.ApiTree() + tree = doc_generator_visitor.PathTree() tree[('tf',)] = tf tree[('tf', 'sub')] = tf.sub tree[('tf', 'sub', 'thing')] = tf.sub.thing tree[('tf', 'sub2')] = tf.sub2 tree[('tf', 'sub2', 'thing')] = tf.sub2.thing - self.assertEmpty(tree.aliases[tf.sub.thing], []) + found = tree.nodes_for_obj(tf.sub.thing) + self.assertEqual([], found) + + +class ApiTreeTest(absltest.TestCase): + + def _make_fake_module(self) -> types.ModuleType: + + class Parent: + + def method1(self): + pass + + def method2(self): + pass + + class Child(Parent): + + def method2(self): + pass + + def method3(self): + pass + + class Outer(object): + attribute = object() + + class Nested(object): + pass + + fun1 = lambda x: x + fun2 = lambda x: x + + tf = types.ModuleType('tf') + tf.__file__ = __file__ + tf.seven = 7 + tf.Parent = Parent + tf.Outer = Outer + tf.fun1 = fun1 + tf.sub1 = types.ModuleType('sub1') + tf.sub1.Parent = Parent + tf.sub2 = types.ModuleType('sub2') + tf.sub2.Child = Child + tf.sub2.fun2 = fun2 + tf.sub1.sub2 = tf.sub2 + + return tf + + def test_physical_path(self): + tf = self._make_fake_module() + + api_tree = doc_generator_visitor.ApiTree() + api_tree.insert(path=('tf',), py_object=tf, aliases=[('tf',)]) + api_tree.insert( + path=('tf', 'sub2'), py_object=tf.sub2, aliases=[('tf', 'sub2')]) + api_tree.insert( + path=('tf', 'seven'), py_object=tf.seven, aliases=[('tf', 'seven')]) + api_tree.insert( + path=('tf', 'fun1'), py_object=tf.fun1, aliases=[('tf', 'fun1')]) + api_tree.insert( + path=('tf', 'sub2', 'Child'), + py_object=tf.sub2.Child, + aliases=[('tf', 'sub2', 'Child')]) + + self.assertEqual(('sub2',), api_tree[('tf', 'sub2')].physical_path) + self.assertIsNone(api_tree[('tf', 'seven')].physical_path) + self.assertEqual(('__main__', 'ApiTreeTest', '_make_fake_module', + '', ''), + api_tree[('tf', 'fun1')].physical_path) + self.assertEqual( + ('__main__', 'ApiTreeTest', '_make_fake_module', '', 'Child'), + api_tree[('tf', 'sub2', 'Child')].physical_path) + + def test_api_tree(self): + tf = self._make_fake_module() + + api_tree = doc_generator_visitor.ApiTree() + api_tree.insert(path=('tf',), py_object=tf, aliases=[('tf',)]) + api_tree.insert( + path=('tf', 'Parent'), + py_object=tf.Parent, + aliases=[('tf', 'Parent'), ('tf', 'Parent2')]) + api_tree.insert( + path=('tf', 'seven'), py_object=tf.seven, aliases=[('tf', 'seven')]) + + # A node can be looked up by any alias + self.assertIs(api_tree[('tf', 'Parent')], api_tree[('tf', 'Parent2')]) + # Nodes only show up once when iterating + self.assertEqual([ + api_tree[('tf',)], api_tree[('tf', 'Parent')], api_tree[('tf', 'seven')] + ], list(api_tree.iter_nodes())) + # Test lookup by object. + self.assertIs(api_tree[('tf', 'Parent')], + api_tree.node_for_object(tf.Parent)) + # You can't lookup things that maybe singletons. + self.assertIs(api_tree[('tf', 'seven')].py_object, tf.seven) + self.assertIsNone(api_tree.node_for_object(tf.seven)) + + def test_from_path_tree(self): + tf = self._make_fake_module() + + path_tree = doc_generator_visitor.PathTree() + path_tree[('tf',)] = tf + path_tree[('tf', 'Parent')] = tf.Parent + path_tree[('tf', 'Parent2')] = tf.Parent + + result = doc_generator_visitor.ApiTree.from_path_tree( + path_tree, score_name_fn=lambda name: name) + + expected = doc_generator_visitor.ApiTree() + expected.insert(path=('tf',), py_object=tf, aliases=[('tf',)]) + expected.insert( + path=('tf', 'Parent'), + py_object=tf.Parent, + aliases=[('tf', 'Parent'), ('tf', 'Parent2')]) + + result = sorted(result.iter_nodes(), key=lambda node: node.path) + expected = sorted(expected.iter_nodes(), key=lambda node: node.path) + + # Circular references make it hard to compare trees or nodes. + for e, r in zip(result, expected): + self.assertEqual(e.path, r.path) + self.assertIs(e.py_object, r.py_object) + self.assertCountEqual(e.aliases, r.aliases) + self.assertCountEqual(e.children.keys(), r.children.keys()) + + def test_api_tree_toc_integration(self): + tf = self._make_fake_module() + + gen = TestDocGenerator([('tf', tf)]) + filters = gen.make_default_filters() + visitor = generate_lib.extract( + [('tf', tf)], filters=filters, visitor_cls=NoDunderVisitor + ) + + api_tree = doc_generator_visitor.ApiTree.from_path_tree( + visitor.path_tree, visitor._score_name) + + toc = toc_lib.TocBuilder(site_path='/').build(api_tree) + + stream = io.StringIO() + toc.write(stream) + + expected = textwrap.dedent("""\ + toc: + - title: tf + section: + - title: Overview + path: /tf + - title: Outer + path: /tf/Outer + - title: Outer.Nested + path: /tf/Outer/Nested + - title: fun1 + path: /tf/fun1 + - title: sub1 + section: + - title: Overview + path: /tf/sub1 + - title: Parent + path: /tf/sub1/Parent + - title: sub2 + section: + - title: Overview + path: /tf/sub2 + - title: Child + path: /tf/sub2/Child + - title: fun2 + path: /tf/sub2/fun2 + """) + + self.assertEqual(expected, stream.getvalue()) + + def test_non_priority_name(self): + + class Class1: + pass + mod = types.ModuleType('mod') + mod.a = types.ModuleType('sub') + mod.a.Class1 = Class1 + mod.b = mod.a + + path_tree = doc_generator_visitor.PathTree() + path_tree[('mod',)] = mod + path_tree[('mod', 'a')] = mod.a + path_tree[('mod', 'a', 'Class1')] = mod.a.Class1 + path_tree[('mod', 'b')] = mod.b + path_tree[('mod', 'b', 'Class1')] = mod.b.Class1 + + def inconsistent_name_score(path): + # `mod.a` is preferred over `mod.b`, but `b.Class1` is preferred over + # `a.Class1`! + scores = { + ('mod',): 0, + ('mod', 'a'): 0, # prefer 'a' + ('mod', 'b'): 1, + ('mod', 'a', 'Class1'): 1, + ('mod', 'b', 'Class1'): 0, # prefer 'b.Class1' + } + return scores[path] + + api_tree = doc_generator_visitor.ApiTree.from_path_tree( + path_tree, inconsistent_name_score) + node = api_tree.node_for_object(Class1) + + # `Class1` can't choose `b.Class1` as its priority_path because + # `a` is the priority_path for `sub`. + self.assertEqual('mod.a.Class1', node.full_name) if __name__ == '__main__': absltest.main() diff --git a/tools/tensorflow_docs/api_generator/gen_java/__init__.py b/tools/tensorflow_docs/api_generator/gen_java/__init__.py new file mode 100644 index 00000000000..a8b1bd19b7f --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/__init__.py @@ -0,0 +1,83 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Generate javadoc-doclava reference docs for tensorflow.org.""" + +import os +import pathlib +import subprocess +from typing import Iterable, Mapping, Optional, Union + +from tensorflow_docs.api_generator import toc_processing + +import yaml + + +class Formatter(yaml.dumper.Dumper): + pass + + +def _dict_representer(dumper, data): + """Force yaml to output dictionaries in order, not alphabetically.""" + return dumper.represent_dict(data.items()) + + +Formatter.add_representer(dict, _dict_representer) + +# __file__ is the path to this file +GEN_JAVA_DIR = pathlib.Path(__file__).resolve().parent + +TEMPLATES = GEN_JAVA_DIR / 'templates' +DOCLAVA_FOR_TF = GEN_JAVA_DIR / 'run-javadoc-for-tf.sh' + + +def gen_java_docs( + package: Union[str, Iterable[str]], + source_path: pathlib.Path, + output_dir: pathlib.Path, + site_path: pathlib.Path, + script_path: pathlib.Path = DOCLAVA_FOR_TF, + section_labels: Optional[Mapping[str, str]] = None, + federated_docs: Optional[Mapping[str, pathlib.Path]] = None, +) -> None: + """Generate tensorflow.org java-docs for `package`.""" + fed_paths = list(federated_docs.values()) if federated_docs else [] + for path in [source_path, output_dir, script_path, TEMPLATES] + fed_paths: + assert path.is_absolute(), 'All paths used in doc-gen must be absolute' + + if federated_docs: + os.environ['FEDERATED_DOCS'] = ' '.join( + [f'{url},{file}' for url, file in federated_docs.items()]) + + if isinstance(package, str): + os.environ['PACKAGE'] = package + root_pkgs = [package] + else: + os.environ['PACKAGE'] = ' '.join(package) + root_pkgs = package + + os.environ['SOURCE_PATH'] = str(source_path) + os.environ['OUTPUT_DIR'] = str(output_dir) + os.environ['SITE_PATH'] = str(pathlib.Path('/') / site_path) + os.environ['TEMPLATES'] = str(TEMPLATES) + subprocess.check_call(['bash', script_path]) + + yaml_path = pathlib.Path(output_dir) / '_toc.yaml' + yaml_content = yaml_path.read_text() + yaml_data = yaml.safe_load(yaml_content) + if section_labels: + yaml_data = toc_processing.add_package_headings(yaml_data, root_pkgs, + section_labels) + yaml_content = yaml.dump(yaml_data, Dumper=Formatter) + yaml_path.write_text(yaml_content) diff --git a/tools/tensorflow_docs/api_generator/gen_java/run-javadoc-for-tf.sh b/tools/tensorflow_docs/api_generator/gen_java/run-javadoc-for-tf.sh new file mode 100755 index 00000000000..03bbf8ee036 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/run-javadoc-for-tf.sh @@ -0,0 +1,96 @@ +#!/bin/bash +set -ex + +# https://android.googlesource.com/platform/external/doclava/ +# There's a debian package: +# https://packages.debian.org/unstable/doclava-aosp +# Install with: +# +# $ sudo apt install doclava-aosp #v 6.0.1+r55-1+build1 +# +# https://unix.stackexchange.com/questions/594841/how-do-i-assign-a-value-to-a-bash-variable-iff-that-variable-is-null-unassigned +DOCLAVA_JAR=${DOCLAVA_JAR:-'/usr/share/java/doclava.jar'} + +# Install java clear silver templates with: +# +# $ sudo apt install libjsilver-aosp-java #v 6.0.1+r55-1+build1 +JSILVER_JAR=${JSILVER_JAR:-'/usr/share/java/jsilver.jar'} + + +######### DELETE OUTPUT_DIR ################# + +# Empty the output directory in case a class has been deleted +rm -rf "${OUTPUT_DIR:?}"/* +############ RUN DOCLAVA ################### + +# $FEDERATED_DOCS is a space-separated string of url,file pairs. +read -a api_pairs <<< "${FEDERATED_DOCS}" +FEDERATED_PARAMS="" +for i in "${!api_pairs[@]}"; do + api_pair_str="${api_pairs[$i]}" # "url,api.txt" + read -a api_pair <<< "${api_pair_str//,/ }" + # Using the index as the API "name", build the federation params. Note that + # using 0 as an API name will evaluate to false and cause rendering bugs, + # so we preface with "api_". + FEDERATED_PARAMS+=" -federate api_${i} ${api_pair[0]}" + FEDERATED_PARAMS+=" -federationapi api_${i} ${api_pair[1]}" +done + +# To install javadoc, for example, use +# +# sudo apt install openjdk-11-jdk +# +# doclava doesn't work with openjdk-13 +# ``` +# javadoc: error - Class com.google.doclava.Doclava is not a valid doclet. +# Note: As of JDK 13, the com.sun.javadoc API is no longer supported. +# ``` +# It's used here: https://android.googlesource.com/platform/external/doclava/+/refs/heads/master/src/com/google/doclava/Doclava.java + +# Each package in $PACKAGE needs to prefaced with -subpackages, so do that. +SUBPACKAGES="" +read -r -a packages <<< "${PACKAGE}" +for pkg in "${packages[@]}"; do + SUBPACKAGES+=" -subpackages ${pkg}" +done +( # Capture the return code. it may be non-zero for minor errors. + javadoc \ + -sourcepath "${SOURCE_PATH}" \ + -docletpath "${DOCLAVA_JAR}:${JSILVER_JAR}" \ + -doclet com.google.doclava.Doclava \ + -d "${OUTPUT_DIR}" \ + -toroot "${SITE_PATH}"/ \ + -yaml _toc.yaml \ + -templatedir "${TEMPLATES}" \ + -public \ + -devsite \ + ${FEDERATED_PARAMS} \ + ${SUBPACKAGES} +) + + +mv "${OUTPUT_DIR}"/reference/* "${OUTPUT_DIR}" + +################################################################### +################### START OF POST-PROCESSING ###################### +################################################################### +rm "${OUTPUT_DIR}/navtree_data.js" || true +rm "${OUTPUT_DIR}/hierarchy.html" || true + +find ${OUTPUT_DIR} -name "*.html" | xargs sed --in-place "s|${SITE_PATH}/reference|${SITE_PATH}|g" +find ${OUTPUT_DIR} -name "*.yaml" | xargs sed --in-place "s|${SITE_PATH}/reference|${SITE_PATH}|g" +find ${OUTPUT_DIR} -name "*.html" | xargs sed --in-place "s|a href=\"reference/org/tensorflow|a href=\"${SITE_PATH}/org/tensorflow|g" +find ${OUTPUT_DIR} -name "*.html" | xargs sed --in-place "s|a href=\"reference/com/google|a href=\"${SITE_PATH}/com/google|g" + +JAVA_LANG=https://docs.oracle.com/javase/8/docs/api +find ${OUTPUT_DIR} -name "*.html" | xargs sed --in-place "s|a href=\"reference/java/lang|a href=\"${JAVA_LANG}/java/lang|g" + +find ${OUTPUT_DIR} -name "*.html" | xargs sed --in-place 's|
    |
    |g'
+
+rm ${OUTPUT_DIR}/timestamp.js || true
+rm ${OUTPUT_DIR}/lists.js || true
+rm ${OUTPUT_DIR}/index.html || true
+
+cp ${TEMPLATES}/screen.css ${OUTPUT_DIR}/
+
+
diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/class.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/class.cs
new file mode 100644
index 00000000000..1190481db0c
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/gen_java/templates/class.cs
@@ -0,0 +1,591 @@
+
+
+
+
+  
+  
    
+    
    
+
+    
+    
+      
+       
+      
+      
+       
+      
+      
+       
+      
+      
+       
+      
+    
+
+    
    
+      
+      
+    
    
+    
    
+
+    
+    
+
+
+    
+    
    
+           
+      
+    
    
+    
+
+    
    
+
+    
    
+
+    
+
+    
+    
    
+    Known Direct Subclasses
+    
+    
    
+    
+
+    
+    
    
+    Known Indirect Subclasses
+    
+    
    
+    
+
+    
    
+    
+    
+    
    
+    
+    
+    
    
+
+
+    
+
+    
+    
+    
+        
+
    + + + + + + + +
    + () + +
    + +
    +
    + + + +
    Attribute Name
    + + + + +  
    + +
    + + + + + + +

    XML Attributes

    + +
    + + + + + + +
    +

    Inherited XML Attributes

    + + + + + +
    + + + + + +
    +

    Enum Values

    + + + + + + + + + +
     
    +
    + + + + + +

    Constants

    + + +
    +
    + + + + + + +
    +

    Inherited Constants

    + + + + + +
    + + + + + +

    Fields

    + + +
    +
    + + + + + + +
    +

    Inherited Fields

    + + + + + +
    + + + + + +
    +

    Public Constructors

    + + +
    +
    + + + + + +
    +

    Protected Constructors

    + + +
    +
    + + + + + + +
    +

    Public Methods

    + + +
    +
    + + + + + +
    +

    Protected Methods

    + + +
    +
    + + + + + + +
    +

    Inherited Methods

    + + + + + +
    + + + + + + + + + + + + +     + +
    +

    + + + + + + + + + + + + +

    +
    + + +
    +
    + + +
    + Constant Value: + + + + + + + +
    + +
    +
    + + + + + + + + +
    +

    + + + + + + + + + + () +

    +
    +
    + +
    +
    + +
    +
    + + + + + + + + +
    +

    +

    +
    + +
    +
    + + +
    +
    Related Methods
    +
      + +
      • + +
    +
    +
    +
    + + + + + + + + +
    +

    XML Attributes

    + +
    + + + + + + +
    +

    Enum Values

    + +
    + + + + + + +
    +

    Constants

    + +
    + + + + + + +
    +

    Fields

    + +
    + + + + + + +
    +

    Public Constructors

    + +
    + + + + + + +
    +

    Protected Constructors

    + +
    + + + + + + + +
    +

    Public Methods

    + +
    + + + + + +
    +

    Protected Methods

    + +
    + + + + + + + + + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/classes.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/classes.cs new file mode 100644 index 00000000000..f466d868c85 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/classes.cs @@ -0,0 +1,27 @@ + + + + + + +
    + +
    + + + +

    + + + + + + + + + +
     
    + + + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/head_tag.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/head_tag.cs new file mode 100644 index 00000000000..997903dea9d --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/head_tag.cs @@ -0,0 +1,6 @@ + + <?cs if:page.title ?><?cs var:page.title ?><?cs /if ?> + + + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/header.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/header.cs new file mode 100644 index 00000000000..db8a11d630e --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/header.cs @@ -0,0 +1,2 @@ + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/left-nav-android.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/left-nav-android.cs new file mode 100644 index 00000000000..0c0d9f10bd4 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/left-nav-android.cs @@ -0,0 +1,105 @@ + + + +
    +
    +
    +
    +
    + +
    + +
    +
    +
    Android APIs
    +
    + + +
    + + +
    + +
    +
    + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/macros.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/macros.cs new file mode 100644 index 00000000000..f70b3071c4f --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/macros.cs @@ -0,0 +1,305 @@ + + + +) +?> extends  &  super  & <, > + + + + + + + + , + +

    { } + + + This was deprecated + in API level . + This is deprecated. + +

    +

    + + This was deprecated + in API level + This is deprecated.
    +

    + + +
    +
    See Also
    +
    • [ERROR: Unknown @see kind]
      • +
    +
    + + + + Added in API level + + +
    + Also: + + , + +
    + + + +

    +
    +
    Related XML Attributes
    +
      +
      • +
    +
    +
    +
    Parameters
    + + + + + +
    <>
    +
    +
    +
    Returns
    +
    +
    +
    +
    Throws
    + + + + + +
    +
    + + + + + + + +
     
    + + +

  • +
      +
      • +
    +
    • + + +

  • +
      +
      • +
    +
    • + + +
    • + + + + + +
    + +
    style="display: none;" + > +
    +
    + + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/navtree_data.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/navtree_data.cs new file mode 100644 index 00000000000..02556707a7d --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/navtree_data.cs @@ -0,0 +1,2 @@ +var NAVTREE_DATA = + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/package-descr.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/package-descr.cs new file mode 100644 index 00000000000..36d5415f357 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/package-descr.cs @@ -0,0 +1,39 @@ + + + + + + + +
    + +
    +
    + + +
    +
    + +
    + package +

    +
    + Classes | Description +
    +
    + +
    + +
    +
    +

    +
    + + +
    +
    + + + + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/package-list.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/package-list.cs new file mode 100644 index 00000000000..7f0f889805d --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/package-list.cs @@ -0,0 +1,2 @@ + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/package.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/package.cs new file mode 100644 index 00000000000..532e1c5b059 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/package.cs @@ -0,0 +1,55 @@ + + + + + + + +
    +
    + +
    +
    + + +
    +
    + +
    + +
    + +
    + +
    + + +
    + +
    + + + + +

    +
    + +
    + + + + + + + + + + +
    +
    + + +
    + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/packages.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/packages.cs new file mode 100644 index 00000000000..57aa8b94bee --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/packages.cs @@ -0,0 +1,23 @@ + + + + + + +
    +

    +
    + + + + + + + + + + +
    + + + diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/screen.css b/tools/tensorflow_docs/api_generator/gen_java/templates/screen.css new file mode 100644 index 00000000000..d6b07a41c64 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/screen.css @@ -0,0 +1,38 @@ +/** Heading for "Parameters", "See Also", etc., in details sections */ +body.docs h4.jd-details-title { + font-size: 1.15em; + margin: 1.5em 0 0.6em -0.5em; + padding: 0.2em 8em 0.4em 2.5em; + text-indent: -2em; + background-color: #E9E9E9; /** #DEEAFA */ +} + +#jd-header { + margin-bottom: 1em; +} + +/* Hide empty first cell of constructors */ +#pubctors td.jd-typecol { + display: none; +} + +p { + margin-bottom: 0.7em; +} + +body.docs table th { + border: 1px solid #ddd; +} + +.main tr th { + color: #222; + background-color: #fff; +} + +body.docs h2 { + margin-left: -0.3em; +} + +td.jd-typecol { + min-width: 100px; +} diff --git a/tools/tensorflow_docs/api_generator/gen_java/templates/yaml_navtree.cs b/tools/tensorflow_docs/api_generator/gen_java/templates/yaml_navtree.cs new file mode 100644 index 00000000000..8eec1140263 --- /dev/null +++ b/tools/tensorflow_docs/api_generator/gen_java/templates/yaml_navtree.cs @@ -0,0 +1,47 @@ + +- title: "" + path: + section: + - title: "" + path: +toc: + \ No newline at end of file diff --git a/tools/tensorflow_docs/api_generator/generate_lib.py b/tools/tensorflow_docs/api_generator/generate_lib.py index cd9673e413a..fdeb0f60601 100644 --- a/tools/tensorflow_docs/api_generator/generate_lib.py +++ b/tools/tensorflow_docs/api_generator/generate_lib.py @@ -12,31 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Generate docs for the TensorFlow Python API.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +"""Generate tensorflow.org style API Reference docs for a Python module.""" import collections -import fnmatch -import operator +import logging import os +import pathlib import shutil -import subprocess import tempfile +from typing import Any, Optional, Sequence, Type, Union -import pathlib2 as pathlib -import six - +from tensorflow_docs.api_generator import config from tensorflow_docs.api_generator import doc_generator_visitor from tensorflow_docs.api_generator import parser -from tensorflow_docs.api_generator import pretty_docs from tensorflow_docs.api_generator import public_api -from tensorflow_docs.api_generator import py_guide_parser -from tensorflow_docs.api_generator import tf_inspect +from tensorflow_docs.api_generator import reference_resolver as reference_resolver_lib +from tensorflow_docs.api_generator import toc as toc_lib from tensorflow_docs.api_generator import traverse - +from tensorflow_docs.api_generator.pretty_docs import docs_for_object +from tensorflow_docs.api_generator.report import utils import yaml # Used to add a collections.OrderedDict representer to yaml so that the @@ -45,9 +39,12 @@ # Using a normal dict doesn't preserve the order of the input dictionary. _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG +# To see the logs pass: --logger_levels=tensorflow_docs:DEBUG --alsologtostderr +_LOGGER = logging.getLogger(__name__) + def dict_representer(dumper, data): - return dumper.represent_dict(six.iteritems(data)) + return dumper.represent_dict(data.items()) def dict_constructor(loader, node): @@ -58,369 +55,20 @@ def dict_constructor(loader, node): yaml.add_constructor(_mapping_tag, dict_constructor) -class TocNode(object): - """Represents a node in the TOC. - - Attributes: - full_name: Name of the module. - short_name: The last path component. - py_object: Python object of the module. - path: Path to the module's page on tensorflow.org relative to - tensorflow.org. - experimental: Whether the module is experimental or not. - deprecated: Whether the module is deprecated or not. - """ - - def __init__(self, module, py_object, path): - self._module = module - self._py_object = py_object - self._path = path - - @property - def full_name(self): - return self._module - - @property - def short_name(self): - return self.full_name.split('.')[-1] - - @property - def py_object(self): - return self._py_object - - @property - def path(self): - return self._path - - @property - def experimental(self): - return 'experimental' in self.short_name - - _DEPRECATED_STRING = 'THIS FUNCTION IS DEPRECATED' - - @property - def deprecated(self): - """Checks if the module is deprecated or not. - - Special case is `tf.contrib`. It doesn't have the _tf_decorator attribute - but that module should be marked as deprecated. - - Each deprecated function has a `_tf_decorator.decorator_name` attribute. - Check the docstring of that function to confirm if the function was - indeed deprecated. If a different deprecation setting was used on the - function, then "THIS FUNCTION IS DEPRECATED" substring won't be inserted - into the docstring of that function by the decorator. - - Returns: - True if depreacted else False. - """ - - if 'tf.contrib' in self.full_name: - return True - - try: - # Instead of only checking the docstring, checking for the decorator - # provides an additional level of certainty about the correctness of the - # the application of `status: deprecated`. - decorator = operator.attrgetter('_tf_decorator.decorator_name')( - self.py_object) - if decorator == 'deprecated': - return self._check_docstring() - except AttributeError: - pass - - return False - - def _check_docstring(self): - # Only add the deprecated status if the function is deprecated. There are - # other settings that should be ignored like deprecate_args, etc. - docstring = self.py_object.__doc__ - return docstring is not None and self._DEPRECATED_STRING in docstring - - -class Module(TocNode): - """Represents a single module and its children and submodules. - - Attributes: - full_name: Name of the module. - short_name: The last path component. - py_object: Python object of the module. - title: Title of the module in _toc.yaml - path: Path to the module's page on tensorflow.org relative to - tensorflow.org. - children: List of attributes on the module. - submodules: List of submodules in the module. - experimental: Whether the module is experimental or not. - deprecated: Whether the module is deprecated or not. - """ - - def __init__(self, module, py_object, path): - super(Module, self).__init__(module, py_object, path) - - self._children = [] - self._submodules = [] - - @property - def title(self): - if self.full_name.count('.') > 1: - title = self.full_name.split('.')[-1] - else: - title = self.full_name - return title - - @property - def children(self): - return sorted( - self._children, key=lambda x: (x.full_name.upper(), x.full_name)) - - @property - def submodules(self): - return self._submodules - - def add_children(self, children): - if not isinstance(children, list): - children = [children] - - self._children.extend(children) - - def add_submodule(self, sub_mod): - self._submodules.append(sub_mod) - - -class ModuleChild(TocNode): - """Represents a child of a module. - - Attributes: - full_name: Name of the child. - short_name: The last path component. - py_object: Python object of the child. - title: Title of the module in _toc.yaml - path: Path to the module's page on tensorflow.org relative to - tensorflow.org. - experimental: Whether the module child is experimental or not. - deprecated: Whether the module is deprecated or not. - """ - - def __init__(self, name, py_object, parent, path): - self._parent = parent - super(ModuleChild, self).__init__(name, py_object, path) - - @property - def title(self): - return self.full_name[len(self._parent) + 1:] - - -class GenerateToc(object): - """Generates a data structure that defines the structure of _toc.yaml.""" - - def __init__(self, modules): - self._modules = modules - - def _create_graph(self): - """Creates a graph to allow a dfs traversal on it to generate the toc. - - Each graph key contains a module and its value is an object of `Module` - class. That module object contains a list of submodules. - - Example low-level structure of the graph is as follows: - - { - 'module1': [submodule1, submodule2], - 'submodule1': [sub1-submodule1], - 'sub1-submodule1': [], - 'submodule2': [], - 'module2': [], - 'module3': [submodule4], - 'submodule4': [sub1-submodule4], - 'sub1-submodule4': [sub1-sub1-submodule4], - 'sub1-sub1-submodule4': [] - } - - Returns: - A tuple of (graph, base_modules). Base modules is returned because while - creating a nested list of dictionaries, the top level should only contain - the base modules. - """ - - # Sort the modules in case-insensitive alphabetical order. - sorted_modules = sorted(self._modules.keys(), key=lambda a: a.lower()) - toc_base_modules = [] - - toc_graph = {} - for module in sorted_modules: - mod = self._modules[module] - - # Add the module to the graph. - toc_graph[module] = mod - - # If the module's name contains more than one dot, it is not a base level - # module. Hence, add it to its parents submodules list. - if module.count('.') > 1: - # For example, if module is `tf.keras.applications.densenet` then its - # parent is `tf.keras.applications`. - parent_module = '.'.join(module.split('.')[:-1]) - parent_mod_obj = toc_graph.get(parent_module, None) - if parent_mod_obj is not None: - parent_mod_obj.add_submodule(mod) - else: - toc_base_modules.append(module) - - return toc_graph, toc_base_modules - - def _generate_children(self, mod, is_parent_deprecated): - """Creates a list of dictionaries containing child's title and path. - - For example: The dictionary created will look this this in _toc.yaml. - - ``` - children_list = [{'title': 'Overview', 'path': '/tf/app'}, - {'title': 'run', 'path': '/tf/app/run'}] - ``` - - The above list will get converted to the following yaml syntax. - - ``` - - title: Overview - path: /tf/app - - title: run - path: /tf/app/run - ``` - - Args: - mod: A module object. - is_parent_deprecated: Bool, Whether the parent is deprecated or not. - - Returns: - A list of dictionaries containing child's title and path. - """ - - children_list = [] - children_list.append( - collections.OrderedDict([('title', 'Overview'), ('path', mod.path)])) - - for child in mod.children: - child_yaml_content = [('title', child.title), ('path', child.path)] - - # Set `status: deprecated` only if the parent's status is not - # deprecated. - if child.deprecated and not is_parent_deprecated: - child_yaml_content.insert(1, ('status', 'deprecated')) - elif child.experimental: - child_yaml_content.insert(1, ('status', 'experimental')) - - children_list.append(collections.OrderedDict(child_yaml_content)) - - return children_list - - def _dfs(self, mod, visited, is_parent_deprecated): - """Does a dfs traversal on the graph generated. - - This creates a nested dictionary structure which is then dumped as .yaml - file. Each submodule's dictionary of title and path is nested under its - parent module. - - For example, `tf.keras.app.net` will be nested under `tf.keras.app` which - will be nested under `tf.keras`. Here's how the nested dictionaries will - look when its dumped as .yaml. - - ``` - - title: tf.keras - section: - - title: Overview - path: /tf/keras - - title: app - section: - - title: Overview - path: /tf/keras/app - - title: net - section: - - title: Overview - path: /tf/keras/app/net - ``` - - The above nested structure is what the dfs traversal will create in form - of lists of dictionaries. - - Args: - mod: A module object. - visited: A dictionary of modules visited by the dfs traversal. - is_parent_deprecated: Bool, Whether any parent is deprecated or not. - - Returns: - A dictionary containing the nested data structure. - """ - - visited[mod.full_name] = True - - # parent_exp is set to the current module because the current module is - # the parent for its children. - children_list = self._generate_children( - mod, is_parent_deprecated or mod.deprecated) - - # generate for submodules within the submodule. - for submod in mod.submodules: - if not visited[submod.full_name]: - sub_mod_dict = self._dfs(submod, visited, is_parent_deprecated or - mod.deprecated) - children_list.append(sub_mod_dict) - - # If the parent module is not experimental, then add the experimental - # status to the submodule. - submod_yaml_content = [('title', mod.title), ('section', children_list)] - - # If the parent module is not deprecated, then add the deprecated - # status to the submodule. If the parent is deprecated, then setting its - # status to deprecated in _toc.yaml propagates to all its children and - # submodules. - if mod.deprecated and not is_parent_deprecated: - submod_yaml_content.insert(1, ('status', 'deprecated')) - elif mod.experimental: - submod_yaml_content.insert(1, ('status', 'experimental')) - - return collections.OrderedDict(submod_yaml_content) - - def generate(self): - """Generates the final toc. - - Returns: - A list of dictionaries which will be dumped into .yaml file. - """ - - toc = [] - toc_graph, toc_base_modules = self._create_graph() - visited = {node: False for node in toc_graph.keys()} - - # Sort in alphabetical case-insensitive order. - toc_base_modules = sorted(toc_base_modules, key=lambda a: a.lower()) - for module in toc_base_modules: - module_obj = toc_graph[module] - # Generate children of the base module. - section = self._generate_children( - module_obj, is_parent_deprecated=module_obj.deprecated) - - # DFS traversal on the submodules. - for sub_mod in module_obj.submodules: - sub_mod_list = self._dfs( - sub_mod, visited, is_parent_deprecated=module_obj.deprecated) - section.append(sub_mod_list) - - module_yaml_content = [('title', module_obj.title), ('section', section)] - if module_obj.deprecated: - module_yaml_content.insert(1, ('status', 'deprecated')) - elif module_obj.experimental: - module_yaml_content.insert(1, ('status', 'experimental')) - - toc.append(collections.OrderedDict(module_yaml_content)) - - return {'toc': toc} - - -def write_docs(output_dir, - parser_config, - yaml_toc, - root_title='TensorFlow', - search_hints=True, - site_path='api_docs/python'): +def write_docs( + *, + output_dir: Union[str, pathlib.Path], + parser_config: config.ParserConfig, + yaml_toc: Union[bool, Type[toc_lib.TocBuilder]], + root_module_name: str, + root_title: str = 'TensorFlow', + search_hints: bool = True, + site_path: str = 'api_docs/python', + gen_redirects: bool = True, + gen_report: bool = True, + extra_docs: Optional[dict[int, str]] = None, + page_builder_classes: Optional[docs_for_object.PageBuilderDict] = None, +): """Write previously extracted docs to disk. Write a docs page for each symbol included in the indices of parser_config to @@ -432,100 +80,81 @@ def write_docs(output_dir, Args: output_dir: Directory to write documentation markdown files to. Will be created if it doesn't exist. - parser_config: A `parser.ParserConfig` object, containing all the necessary + parser_config: A `config.ParserConfig` object, containing all the necessary indices. yaml_toc: Set to `True` to generate a "_toc.yaml" file. + root_module_name: (str) the name of the root module (`tf` for tensorflow). root_title: The title name for the root level index.md. search_hints: (bool) include meta-data search hints at the top of each output file. site_path: The output path relative to the site root. Used in the `_toc.yaml` and `_redirects.yaml` files. + gen_redirects: Bool which decides whether to generate _redirects.yaml file + or not. + gen_report: If True, a report for the library is generated by linting the + docstrings of its public API symbols. + extra_docs: To add docs for a particular object instance set it's __doc__ + attribute. For some classes (list, tuple, etc) __doc__ is not writable. + Pass those docs like: `extra_docs={id(obj): "docs"}` + page_builder_classes: A optional dict of `{ObjectType:Type[PageInfo]}` for + overriding the default page builder classes. Raises: ValueError: if `output_dir` is not an absolute path """ + output_dir = pathlib.Path(output_dir) + site_path = pathlib.Path('/', site_path) + # Make output_dir. - if not os.path.isabs(output_dir): + if not output_dir.is_absolute(): raise ValueError("'output_dir' must be an absolute path.\n" - " output_dir='%s'" % output_dir) - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - # These dictionaries are used for table-of-contents generation below - # They will contain, after the for-loop below:: - # - module name(string):classes and functions the module contains(list) - module_children = {} + f" output_dir='{output_dir}'") + output_dir.mkdir(parents=True, exist_ok=True) # Collect redirects for an api _redirects.yaml file. redirects = [] + api_report = None + if gen_report: + api_report = utils.ApiReport() + # Parse and write Markdown pages, resolving cross-links (`tf.symbol`). - for full_name in sorted(parser_config.index.keys(), key=lambda k: k.lower()): - py_object = parser_config.index[full_name] + num_docs_output = 0 + for api_node in parser_config.api_tree.iter_nodes(): + _LOGGER.debug('generate_lib.write_docs') + _LOGGER.debug(' full_name: %s', api_node.full_name) - if full_name in parser_config.duplicate_of: - continue + full_name = api_node.full_name - # Methods and some routines are documented only as part of their class. - if not (tf_inspect.ismodule(py_object) or tf_inspect.isclass(py_object) or - parser.is_free_function(py_object, full_name, parser_config.index)): + if api_node.output_type() is api_node.OutputType.FRAGMENT: continue - # Remove the extension from the path. - docpath, _ = os.path.splitext(parser.documentation_path(full_name)) - - # For a module, remember the module for the table-of-contents - if tf_inspect.ismodule(py_object): - if full_name in parser_config.tree: - mod_obj = Module( - module=full_name, - py_object=py_object, - path=os.path.join('/', site_path, docpath)) - module_children[full_name] = mod_obj - # For something else that's documented, - # figure out what module it lives in - else: - subname = str(full_name) - while True: - subname = subname[:subname.rindex('.')] - if tf_inspect.ismodule(parser_config.index[subname]): - module_name = parser_config.duplicate_of.get(subname, subname) - child_mod = ModuleChild( - name=full_name, - py_object=py_object, - parent=module_name, - path=os.path.join('/', site_path, docpath)) - module_children[module_name].add_children(child_mod) - break - # Generate docs for `py_object`, resolving references. try: - page_info = parser.docs_for_object(full_name, py_object, parser_config) - except: + page_info = docs_for_object.docs_for_object( + api_node=api_node, + parser_config=parser_config, + extra_docs=extra_docs, + search_hints=search_hints, + page_builder_classes=page_builder_classes) + + if api_report is not None and not full_name.startswith( + ('tf.compat.v', 'tf.keras.backend', 'tf.numpy', + 'tf.experimental.numpy')): + api_report.fill_metrics(page_info) + except Exception as e: raise ValueError( - 'Failed to generate docs for symbol: `{}`'.format(full_name)) + f'Failed to generate docs for symbol: `{full_name}`') from e + + path = output_dir / parser.documentation_path(full_name) - path = os.path.join(output_dir, parser.documentation_path(full_name)) - directory = os.path.dirname(path) try: - if not os.path.exists(directory): - os.makedirs(directory) - # This function returns raw bytes in PY2 or unicode in PY3. - if search_hints: - content = [page_info.get_metadata_html()] - else: - content = [''] - - content.append(pretty_docs.build_md_page(page_info)) - text = '\n'.join(content) - if six.PY3: - text = text.encode('utf-8') - with open(path, 'wb') as f: - f.write(text) - except OSError: - raise OSError('Cannot write documentation for %s to %s' % - (full_name, directory)) + path.parent.mkdir(exist_ok=True, parents=True) + path.write_text(page_info.page_text, encoding='utf-8') + num_docs_output += 1 + except OSError as e: + raise OSError('Cannot write documentation for ' + f'{full_name} to {path.parent}') from e duplicates = parser_config.duplicates.get(full_name, []) if not duplicates: @@ -533,50 +162,47 @@ def write_docs(output_dir, duplicates = [item for item in duplicates if item != full_name] - for dup in duplicates: - from_path = os.path.join(site_path, dup.replace('.', '/')) - to_path = os.path.join(site_path, full_name.replace('.', '/')) - redirects.append({ - 'from': os.path.join('/', from_path), - 'to': os.path.join('/', to_path) - }) - - if redirects: - if yaml_toc: - redirects.append({ - 'from': os.path.join('/', site_path, 'tf_overview'), - 'to': os.path.join('/', site_path, 'tf') - }) + if gen_redirects: + for dup in duplicates: + from_path = site_path / dup.replace('.', '/') + to_path = site_path / full_name.replace('.', '/') + redirects.append({'from': str(from_path), 'to': str(to_path)}) + + if api_report is not None: + api_report.write(output_dir / root_module_name / 'api_report.pb') + + if num_docs_output <= 1: + raise ValueError( + 'The `DocGenerator` failed to generate any docs. Verify ' + 'your arguments (`base_dir` and `callbacks`). ' + 'Everything you want documented should be within ' + '`base_dir`.' + ) + + if yaml_toc: + if isinstance(yaml_toc, bool): + yaml_toc = toc_lib.FlatModulesTocBuilder + toc = yaml_toc(site_path).build(parser_config.api_tree) + + toc_path = output_dir / root_module_name / '_toc.yaml' + toc.write(toc_path) + if redirects and gen_redirects: redirects_dict = { 'redirects': sorted(redirects, key=lambda redirect: redirect['from']) } - api_redirects_path = os.path.join(output_dir, '_redirects.yaml') + api_redirects_path = output_dir / root_module_name / '_redirects.yaml' with open(api_redirects_path, 'w') as redirect_file: yaml.dump(redirects_dict, redirect_file, default_flow_style=False) - if yaml_toc: - toc_gen = GenerateToc(module_children) - toc_dict = toc_gen.generate() - - # Replace the overview path *only* for 'TensorFlow' to - # `/api_docs/python/tf_overview`. This will be redirected to - # `/api_docs/python/tf`. - toc_values = toc_dict['toc'][0] - if toc_values['title'] == 'tf': - section = toc_values['section'][0] - section['path'] = os.path.join('/', site_path, 'tf_overview') - - leftnav_toc = os.path.join(output_dir, '_toc.yaml') - with open(leftnav_toc, 'w') as toc_file: - yaml.dump(toc_dict, toc_file, default_flow_style=False) - # Write a global index containing all full names with links. - with open(os.path.join(output_dir, 'index.md'), 'w') as f: - f.write( - parser.generate_global_index(root_title, parser_config.index, - parser_config.reference_resolver)) + with open(output_dir / root_module_name / 'all_symbols.md', 'w') as f: + global_index = parser.generate_global_index( + root_title, parser_config.index, parser_config.reference_resolver) + if not search_hints: + global_index = 'robots: noindex\n' + global_index + f.write(global_index) def add_dict_to_dict(add_from, add_to): @@ -587,12 +213,13 @@ def add_dict_to_dict(add_from, add_to): add_to[key] = add_from[key] -def extract(py_modules, - base_dir, - private_map, - do_not_descend_map, - visitor_cls=doc_generator_visitor.DocGeneratorVisitor, - callbacks=None): +def extract( + py_modules, + visitor_cls: Type[ + doc_generator_visitor.DocGeneratorVisitor + ] = doc_generator_visitor.DocGeneratorVisitor, + filters: Optional[public_api.ApiFilter] = None, +): """Walks the module contents, returns an index of all visited objects. The return value is an instance of `self._visitor_cls`, usually: @@ -601,162 +228,96 @@ def extract(py_modules, Args: py_modules: A list containing a single (short_name, module_object) pair. like `[('tf',tf)]`. - base_dir: The package root directory. Nothing defined outside of this - directory is documented. - private_map: A {'path':["name"]} dictionary listing particular object - locations that should be ignored in the doc generator. - do_not_descend_map: A {'path':["name"]} dictionary listing particular object - locations where the children should not be listed. visitor_cls: A class, typically a subclass of `doc_generator_visitor.DocGeneratorVisitor` that acumulates the indexes of - obejcts to document. - callbacks: Additional callbacks passed to `traverse`. Executed between the - `PublicApiFilter` and the accumulator (`DocGeneratorVisitor`). The - primary use case for these is to filter the listy of children (see: - `public_api.local_definitions_filter`) + objects to document. + filters: Filters passed to `traverse`. Executed before the accumulator + (`DocGeneratorVisitor`). These filter the list of children. Returns: The accumulator (`DocGeneratorVisitor`) """ - if callbacks is None: - callbacks = [] + if filters is None: + filters = [] if len(py_modules) != 1: raise ValueError("only pass one [('name',module)] pair in py_modules") short_name, py_module = py_modules[0] - api_filter = public_api.PublicAPIFilter( - base_dir=base_dir, - do_not_descend_map=do_not_descend_map, - private_map=private_map) - accumulator = visitor_cls() + traverse.traverse(py_module, filters, accumulator, root_name=short_name) - # The objects found during traversal, and their children are passed to each - # of these visitors in sequence. Each visitor returns the list of children - # to be passed to the next visitor. - visitors = [api_filter, public_api.ignore_typing] + callbacks + [accumulator] - - traverse.traverse(py_module, visitors, short_name) - + accumulator.build() return accumulator -class _GetMarkdownTitle(py_guide_parser.PyGuideParser): - """Extract the title from a .md file.""" - - def __init__(self): - self.title = None - py_guide_parser.PyGuideParser.__init__(self) - - def process_title(self, _, title): - if self.title is None: # only use the first title - self.title = title - - EXCLUDED = set(['__init__.py', 'OWNERS', 'README.txt']) -def replace_refs(src_dir, - output_dir, - reference_resolver, - file_pattern='*.md', - api_docs_relpath='api_docs'): - """Link `tf.symbol` references found in files matching `file_pattern`. - - A matching directory structure, with the modified files is - written to `output_dir`. - - `{"__init__.py","OWNERS","README.txt"}` are skipped. - - Files not matching `file_pattern` (using `fnmatch`) are copied with no change. - - Also, files in the `api_guides/python` directory get explicit ids set on all - heading-2s to ensure back-links work. - - Args: - src_dir: The directory to convert files from. - output_dir: The root directory to write the resulting files to. - reference_resolver: A `parser.ReferenceResolver` to make the replacements. - file_pattern: Only replace references in files matching file_patters, using - `fnmatch`. Non-matching files are copied unchanged. - api_docs_relpath: Relative-path string to the api_docs, from the src_dir. - """ - # Iterate through all the source files and process them. - for dirpath, _, filenames in os.walk(src_dir): - depth = os.path.relpath(src_dir, start=dirpath) - # How to get from `dirpath` to api_docs/python/ - relative_path_to_root = os.path.join(depth, api_docs_relpath, 'python') - - # Make the directory under output_dir. - new_dir = os.path.join(output_dir, - os.path.relpath(path=dirpath, start=src_dir)) - if not os.path.exists(new_dir): - os.makedirs(new_dir) - - for base_name in filenames: - if base_name in EXCLUDED: - continue - full_in_path = os.path.join(dirpath, base_name) - - suffix = os.path.relpath(path=full_in_path, start=src_dir) - full_out_path = os.path.join(output_dir, suffix) - # Copy files that do not match the file_pattern, unmodified. - if not fnmatch.fnmatch(base_name, file_pattern): - if full_in_path != full_out_path: - shutil.copyfile(full_in_path, full_out_path) - continue - - with open(full_in_path, 'rb') as f: - content = f.read().decode('utf-8') - - content = reference_resolver.replace_references(content, - relative_path_to_root) - with open(full_out_path, 'wb') as f: - f.write(content.encode('utf-8')) - - -class DocGenerator(object): +class DocGenerator: """Main entry point for generating docs.""" - def __init__(self, - root_title, - py_modules, - base_dir=None, - code_url_prefix=(), - search_hints=True, - site_path='api_docs/python', - private_map=None, - do_not_descend_map=None, - visitor_cls=doc_generator_visitor.DocGeneratorVisitor, - api_cache=True, - callbacks=None): + def __init__( + self, + *, + root_title: str, + py_modules: Sequence[tuple[str, Any]], + base_dir: Optional[Sequence[Union[str, pathlib.Path]]] = None, + code_url_prefix: Union[Optional[str], Sequence[Optional[str]]] = (), + self_link_base: Optional[str] = None, + search_hints: bool = True, + site_path: str = 'api_docs/python', + private_map: Optional[dict[str, str]] = None, + visitor_cls: Type[doc_generator_visitor.DocGeneratorVisitor] = ( + doc_generator_visitor.DocGeneratorVisitor + ), + api_cache: bool = True, + callbacks: Optional[list[public_api.ApiFilter]] = None, + yaml_toc: Union[bool, Type[toc_lib.TocBuilder]] = True, + gen_redirects: bool = True, + gen_report: bool = True, + extra_docs: Optional[dict[int, str]] = None, + page_builder_classes: Optional[docs_for_object.PageBuilderDict] = None, + ): """Creates a doc-generator. Args: root_title: A string. The main title for the project. Like "TensorFlow" py_modules: The python module to document. base_dir: String or tuple of strings. Directories that "Defined in" links - are generated relative to. Modules outside one of these directories are - not documented. No `base_dir` should be inside another. + are generated relative to. **Modules outside one of these directories + are not documented**. No `base_dir` should be inside another. code_url_prefix: String or tuple of strings. The prefix to add to "Defined in" paths. These are zipped with `base-dir`, to set the `defined_in` path for each file. The defined in link for `{base_dir}/path/to/file` is set to `{code_url_prefix}/path/to/file`. + self_link_base: A string. A URL prefix pre-pend to self-links to the + generated docs pages. Optional, if no `self_link_base` is supplied, no + self-link will be added. search_hints: Bool. Include metadata search hints at the top of each file. site_path: Path prefix in the "_toc.yaml" - private_map: A {"module.path.to.object": ["names"]} dictionary. Specific - aliases that should not be shown in the resulting docs. - do_not_descend_map: A {"module.path.to.object": ["names"]} dictionary. - Specific aliases that will be shown, but not expanded. + private_map: DEPRECATED. Use `api_generator.doc_controls`, or pass a + filter to the `callbacks` argument. A `{"module.path.to.object": + ["names"]}` dictionary. Specific aliases that should not be shown in the + resulting docs. visitor_cls: An option to override the default visitor class `doc_generator_visitor.DocGeneratorVisitor`. api_cache: Bool. Generate an api_cache file. This is used to easily add api links for backticked symbols (like `tf.add`) in other docs. callbacks: Additional callbacks passed to `traverse`. Executed between the `PublicApiFilter` and the accumulator (`DocGeneratorVisitor`). The - primary use case for these is to filter the listy of children (see: - `public_api.local_definitions_filter`) + primary use case for these is to filter the list of children (see: + `public_api.ApiFilter` for the required signature) + yaml_toc: Bool which decides whether to generate _toc.yaml file or not. + gen_redirects: Bool which decides whether to generate _redirects.yaml file + or not. + gen_report: If True, a report for the library is generated by linting the + docstrings of its public API symbols. + extra_docs: To add docs for a particular object instance set it's __doc__ + attribute. For some classes (list, tuple, etc) __doc__ is not writable. + Pass those docs like: `extra_docs={id(obj): "docs"}` + page_builder_classes: An optional dict of `{ObjectType:Type[PageInfo]}` + for overriding the default page builder classes. """ self._root_title = root_title self._py_modules = py_modules @@ -764,63 +325,98 @@ def __init__(self, self._py_module = py_modules[0][1] if base_dir is None: - base_dir = os.path.dirname(self._py_module.__file__) - if isinstance(base_dir, str): - base_dir = (base_dir,) - self._base_dir = tuple(base_dir) - assert self._base_dir, '`base_dir` cannot be empty' + # Determine the base_dir for the module + base_dir = public_api.get_module_base_dirs(self._py_module) + else: + if isinstance(base_dir, (str, pathlib.Path)): + base_dir = (base_dir,) + base_dir = tuple(pathlib.Path(d) for d in base_dir) + self._base_dir = base_dir + + if not self._base_dir: + raise ValueError('`base_dir` cannot be empty') - if isinstance(code_url_prefix, str): + if isinstance(code_url_prefix, str) or code_url_prefix is None: code_url_prefix = (code_url_prefix,) self._code_url_prefix = tuple(code_url_prefix) if not self._code_url_prefix: raise ValueError('`code_url_prefix` cannot be empty') if len(self._code_url_prefix) != len(base_dir): - raise ValueError('The `base_dir` list should have the same number of ' - 'elements as the `code_url_prefix` list (they get ' - 'zipped together).') + raise ValueError( + 'The `base_dir` list should have the same number of ' + 'elements as the `code_url_prefix` list (they get ' + 'zipped together).' + ) + self._self_link_base = self_link_base self._search_hints = search_hints self._site_path = site_path self._private_map = private_map or {} - self._do_not_descend_map = do_not_descend_map or {} self._visitor_cls = visitor_cls self.api_cache = api_cache if callbacks is None: callbacks = [] self._callbacks = callbacks + self._yaml_toc = yaml_toc + self._gen_redirects = gen_redirects + self._gen_report = gen_report + self._extra_docs = extra_docs + self._page_builder_classes = page_builder_classes def make_reference_resolver(self, visitor): - return parser.ReferenceResolver.from_visitor( - visitor, py_module_names=[self._short_name]) + return reference_resolver_lib.ReferenceResolver.from_visitor( + visitor, py_module_names={self._short_name: self._py_module.__name__}) - def make_parser_config(self, visitor, reference_resolver): - return parser.ParserConfig( + def make_parser_config(self, + visitor: doc_generator_visitor.DocGeneratorVisitor): + reference_resolver = self.make_reference_resolver(visitor) + return config.ParserConfig( reference_resolver=reference_resolver, duplicates=visitor.duplicates, duplicate_of=visitor.duplicate_of, tree=visitor.tree, index=visitor.index, reverse_index=visitor.reverse_index, + path_tree=visitor.path_tree, + api_tree=visitor.api_tree, base_dir=self._base_dir, - code_url_prefix=self._code_url_prefix) - - def run_extraction(self): + code_url_prefix=self._code_url_prefix, + self_link_base=self._self_link_base, + ) + + def make_default_filters(self) -> list[public_api.ApiFilter]: + # The objects found during traversal, and their children are passed to each + # of these filters in sequence. Each visitor returns the list of children + # to be passed to the next visitor. + return [ + # filter the api. + public_api.FailIfNestedTooDeep(10), + public_api.filter_module_all, + public_api.add_proto_fields, + public_api.filter_private_symbols, + public_api.FilterBaseDirs(self._base_dir), + public_api.FilterPrivateMap(self._private_map), + public_api.filter_doc_controls_skip, + public_api.ignore_typing, + ] + + def run_extraction(self) -> config.ParserConfig: """Walks the module contents, returns an index of all visited objects. - The return value is an instance of `self._visitor_cls`, usually: - `doc_generator_visitor.DocGeneratorVisitor` - Returns: + An instance of `parser_config.ParserConfig`. """ - return extract( + default_filters = self.make_default_filters() + visitor = extract( py_modules=self._py_modules, - base_dir=self._base_dir, - private_map=self._private_map, - do_not_descend_map=self._do_not_descend_map, visitor_cls=self._visitor_cls, - callbacks=self._callbacks) + filters=default_filters + self._callbacks, + ) + + # Write the api docs. + parser_config = self.make_parser_config(visitor) + return parser_config def build(self, output_dir): """Build all the docs. @@ -835,34 +431,48 @@ def build(self, output_dir): workdir = pathlib.Path(tempfile.mkdtemp()) # Extract the python api from the _py_modules - visitor = self.run_extraction() - reference_resolver = self.make_reference_resolver(visitor) - # Replace all the `tf.symbol` references in the workdir. - replace_refs( - str(workdir), str(workdir), reference_resolver, file_pattern='*.md') - - # Write the api docs. - parser_config = self.make_parser_config(visitor, reference_resolver) + parser_config = self.run_extraction() work_py_dir = workdir / 'api_docs/python' write_docs( output_dir=str(work_py_dir), parser_config=parser_config, - yaml_toc=True, + yaml_toc=self._yaml_toc, root_title=self._root_title, + root_module_name=self._short_name.replace('.', '/'), search_hints=self._search_hints, - site_path=self._site_path) + site_path=self._site_path, + gen_redirects=self._gen_redirects, + gen_report=self._gen_report, + extra_docs=self._extra_docs, + page_builder_classes=self._page_builder_classes, + ) if self.api_cache: - reference_resolver.to_json_file( - str(work_py_dir / self._short_name / '_api_cache.json')) - - try: - os.makedirs(output_dir) - except OSError as e: - if e.strerror != 'File exists': - raise - - subprocess.check_call([ - 'rsync', '--recursive', '--quiet', '--delete', - '{}/'.format(work_py_dir), output_dir - ]) + parser_config.reference_resolver.to_json_file( + str(work_py_dir / self._short_name.replace('.', '/') / + '_api_cache.json')) + + os.makedirs(output_dir, exist_ok=True) + + # Typical results are something like: + # + # out_dir/ + # {short_name}/ + # _redirects.yaml + # _toc.yaml + # api_report.pb + # index.md + # {short_name}.md + # + # Copy the top level files to the `{output_dir}/`, delete and replace the + # `{output_dir}/{short_name}/` directory. + + for work_path in work_py_dir.glob('*'): + out_path = pathlib.Path(output_dir) / work_path.name + out_path.parent.mkdir(exist_ok=True, parents=True) + + if work_path.is_file(): + shutil.copy2(work_path, out_path) + elif work_path.is_dir(): + shutil.rmtree(out_path, ignore_errors=True) + shutil.copytree(work_path, out_path) diff --git a/tools/tensorflow_docs/api_generator/generate_lib_test.py b/tools/tensorflow_docs/api_generator/generate_lib_test.py index 182986078fb..5673cab3f42 100644 --- a/tools/tensorflow_docs/api_generator/generate_lib_test.py +++ b/tools/tensorflow_docs/api_generator/generate_lib_test.py @@ -14,28 +14,33 @@ # ============================================================================== """Tests for doc generator traversal.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - import os +import pathlib import sys import tempfile import textwrap +import types from absl import flags from absl.testing import absltest +from tensorflow_docs.api_generator import config +from tensorflow_docs.api_generator import doc_controls from tensorflow_docs.api_generator import generate_lib from tensorflow_docs.api_generator import parser +from tensorflow_docs.api_generator import reference_resolver as reference_resolver_lib +from tensorflow_docs.api_generator.pretty_docs import function_page + +import yaml FLAGS = flags.FLAGS -def _tf_decorator(): - return True +def deprecated(func): + return func +@deprecated def test_function(): """Docstring for test_function. @@ -43,11 +48,6 @@ def test_function(): """ pass -# Set the _tf_decorator.decorator_name attributes on test_function -# so as to mark it as deprecated and activate that part of the logic. -setattr(_tf_decorator, 'decorator_name', 'deprecated') -setattr(test_function, '_tf_decorator', _tf_decorator) - class TestClass(object): """Docstring for TestClass itself.""" @@ -78,187 +78,73 @@ def setUp(self): def get_test_objects(self): # These are all mutable objects, so rebuild them for each test. # Don't cache the objects. - module = sys.modules[__name__] - - index = { - 'tf': sys, # Can be any module, this test doesn't care about content. - 'tf.TestModule': module, - 'tf.test_function': test_function, - 'tf.TestModule.test_function': test_function, - 'tf.TestModule.TestClass': TestClass, - 'tf.TestModule.TestClass.ChildClass': TestClass.ChildClass, - 'tf.TestModule.TestClass.ChildClass.GrandChildClass': - TestClass.ChildClass.GrandChildClass, - } - - tree = { - 'tf': ['TestModule', 'test_function'], - 'tf.TestModule': ['test_function', 'TestClass'], - 'tf.TestModule.TestClass': ['ChildClass'], - 'tf.TestModule.TestClass.ChildClass': ['GrandChildClass'], - 'tf.TestModule.TestClass.ChildClass.GrandChildClass': [] - } - - duplicate_of = {'tf.test_function': 'tf.TestModule.test_function'} - - duplicates = { - 'tf.TestModule.test_function': [ - 'tf.test_function', 'tf.TestModule.test_function' - ] - } - - base_dir = os.path.dirname(__file__) - - visitor = DummyVisitor(index, duplicate_of) - - reference_resolver = parser.ReferenceResolver.from_visitor( - visitor=visitor, py_module_names=['tf']) - - parser_config = parser.ParserConfig( - reference_resolver=reference_resolver, - duplicates=duplicates, - duplicate_of=duplicate_of, - tree=tree, - index=index, - reverse_index={}, - base_dir=base_dir, - code_url_prefix='/') - - return reference_resolver, parser_config + tf = types.ModuleType('tf') + tf.__file__ = __file__ + tf.TestModule = types.ModuleType('module') + tf.test_function = test_function + tf.TestModule.test_function = test_function + tf.TestModule.TestClass = TestClass + + generator = generate_lib.DocGenerator( + root_title='TensorFlow', + py_modules=[('tf', tf)], + code_url_prefix='https://tensorflow.org/') + + parser_config = generator.run_extraction() + + return parser_config.reference_resolver, parser_config def test_write(self): _, parser_config = self.get_test_objects() - output_dir = self.workdir + output_dir = pathlib.Path(self.workdir) - generate_lib.write_docs(output_dir, parser_config, yaml_toc=True) + generate_lib.write_docs( + output_dir=output_dir, + parser_config=parser_config, + root_module_name='tf', + yaml_toc=True) # Check redirects - redirects_file = os.path.join(output_dir, '_redirects.yaml') - self.assertTrue(os.path.exists(redirects_file)) - with open(redirects_file) as f: - redirects = f.read() - self.assertEqual(redirects.split(), [ - 'redirects:', '-', 'from:', '/api_docs/python/tf/test_function', 'to:', - '/api_docs/python/tf/TestModule/test_function', '-', 'from:', - '/api_docs/python/tf_overview', 'to:', '/api_docs/python/tf' - ]) - - toc_file = os.path.join(output_dir, '_toc.yaml') - self.assertTrue(os.path.exists(toc_file)) - with open(toc_file) as f: - toc = f.read() - toc_list = toc.split() + redirects_file = output_dir / 'tf/_redirects.yaml' + self.assertTrue(redirects_file.exists()) + redirects = yaml.safe_load(redirects_file.read_text()) + self.assertEqual( + redirects, { + 'redirects': [{ + 'from': '/api_docs/python/tf/test_function', + 'to': '/api_docs/python/tf/TestModule/test_function' + }] + }) + + toc_file = output_dir / 'tf/_toc.yaml' + self.assertTrue(toc_file.exists()) + toc_list = yaml.safe_load(toc_file.read_text())['toc'] # Number of sections in the toc should be 2. - self.assertEqual(toc_list.count('section:'), 2) - - # TOC should always begin with `toc:` - self.assertEqual(toc_list[0], 'toc:') + self.assertLen([item for item in toc_list if 'section' in item], 2) # The last path in the TOC must be the ground truth below. # This will check if the symbols are being sorted in case-insensitive # alphabetical order too, spanning across submodules and children. - self.assertEqual(toc_list[-1], + test_function_toc = toc_list[1]['section'][-1] + self.assertEqual(test_function_toc['path'], '/api_docs/python/tf/TestModule/test_function') - - # The last module (`test_function`) should have its status marked as - # deprecated. - toc_line_split = toc.splitlines() - output = textwrap.dedent('\n'.join(toc_line_split[-3:])) - expected_output = textwrap.dedent("""\ - - title: test_function - status: deprecated - path: /api_docs/python/tf/TestModule/test_function""") - self.assertEqual(output, expected_output) + self.assertEqual(test_function_toc['status'], 'deprecated') # Make sure that the right files are written to disk. - self.assertTrue(os.path.exists(os.path.join(output_dir, 'index.md'))) - self.assertTrue(os.path.exists(os.path.join(output_dir, 'tf.md'))) - self.assertTrue(os.path.exists(os.path.join(output_dir, '_toc.yaml'))) - self.assertTrue( - os.path.exists(os.path.join(output_dir, 'tf/TestModule.md'))) - self.assertFalse( - os.path.exists(os.path.join(output_dir, 'tf/test_function.md'))) + self.assertTrue((output_dir / 'tf/all_symbols.md').exists()) + self.assertTrue((output_dir / 'tf.md').exists()) + self.assertTrue((output_dir / 'tf/TestModule.md').exists()) + self.assertFalse((output_dir / 'tf/test_function.md').exists()) + self.assertTrue((output_dir / 'tf/TestModule/TestClass.md').exists()) self.assertTrue( - os.path.exists( - os.path.join(output_dir, 'tf/TestModule/TestClass.md'))) + (output_dir / 'tf/TestModule/TestClass/ChildClass.md').exists()) self.assertTrue( - os.path.exists( - os.path.join(output_dir, - 'tf/TestModule/TestClass/ChildClass.md'))) - self.assertTrue( - os.path.exists( - os.path.join( - output_dir, - 'tf/TestModule/TestClass/ChildClass/GrandChildClass.md'))) + (output_dir / + 'tf/TestModule/TestClass/ChildClass/GrandChildClass.md').exists()) # Make sure that duplicates are not written - self.assertTrue( - os.path.exists( - os.path.join(output_dir, 'tf/TestModule/test_function.md'))) - - def test_replace_refes(self): - test_dir = self.workdir - test_in_dir = os.path.join(test_dir, 'in') - test_in_dir_a = os.path.join(test_dir, 'in/a') - test_in_dir_b = os.path.join(test_dir, 'in/b') - os.makedirs(test_in_dir) - os.makedirs(test_in_dir_a) - os.makedirs(test_in_dir_b) - - test_out_dir = os.path.join(test_dir, 'out') - os.makedirs(test_out_dir) - - test_path1 = os.path.join(test_in_dir_a, 'file1.md') - test_path2 = os.path.join(test_in_dir_b, 'file2.md') - test_path3 = os.path.join(test_in_dir_b, 'file3.notmd') - test_path4 = os.path.join(test_in_dir_b, 'OWNERS') - - with open(test_path1, 'w') as f: - f.write('Use `tf.test_function` to test things.') - - with open(test_path2, 'w') as f: - f.write('Use `tf.TestModule.TestClass.ChildClass` to test things.\n' - "`tf.whatever` doesn't exist") - - with open(test_path3, 'w') as f: - file3_content = ( - 'Not a .md file. Should be copied unchanged:' - '`tf.TestModule.TestClass.ChildClass`, `tf.test_function`') - f.write(file3_content) - - with open(test_path4, 'w') as f: - f.write('') - - reference_resolver, _ = self.get_test_objects() - generate_lib.replace_refs(test_in_dir, test_out_dir, reference_resolver, - '*.md') - - with open(os.path.join(test_out_dir, 'a/file1.md')) as f: - content = f.read() - self.assertEqual( - content, - 'Use ' - 'tf.test_function to test things.') - - with open(os.path.join(test_out_dir, 'b/file2.md')) as f: - content = f.read() - self.assertEqual( - content, - 'Use ' - '' - 'tf.TestModule.TestClass.ChildClass ' - 'to test things.\n' - '`tf.whatever` doesn\'t exist') - - with open(os.path.join(test_out_dir, 'b/file3.notmd')) as f: - content = f.read() - self.assertEqual(content, file3_content) - - with self.assertRaises(IOError): - # This should fail. The OWNERS file should not be copied - with open(os.path.join(test_out_dir, 'b/OWNERS')) as f: - content = f.read() + self.assertTrue((output_dir / 'tf/TestModule/test_function.md').exists()) if __name__ == '__main__': diff --git a/tools/tensorflow_docs/api_generator/get_source.py b/tools/tensorflow_docs/api_generator/get_source.py new file mode 100644 index 00000000000..8ea0b02014a --- /dev/null +++ b/tools/tensorflow_docs/api_generator/get_source.py @@ -0,0 +1,55 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Simple get_source.""" +import ast +import inspect +import textwrap + +from typing import Any, Optional, Sequence, Tuple + + +def get_ast(py_object) -> Optional[ast.AST]: + if isinstance(py_object, str): + source = textwrap.dedent(py_object) + else: + source = get_source(py_object) + if source is None: + return None + + try: + return ast.parse(source) + except Exception: # pylint: disable=broad-except + return None + + +def get_source(py_object: Any) -> Optional[str]: + if py_object is not None: + try: + return textwrap.dedent(inspect.getsource(py_object)) + except Exception: # pylint: disable=broad-except + # A wide-variety of errors can be thrown here. + pass + return None + + +def get_source_lines( + py_object: Any) -> Tuple[Optional[Sequence[str]], Optional[int]]: + if py_object is not None: + try: + return inspect.getsourcelines(py_object) + except Exception: # pylint: disable=broad-except + # A wide-variety of errors can be thrown here. + pass + return None, None diff --git a/tools/tensorflow_docs/api_generator/obj_type.py b/tools/tensorflow_docs/api_generator/obj_type.py new file mode 100644 index 00000000000..fb1b36a05ad --- /dev/null +++ b/tools/tensorflow_docs/api_generator/obj_type.py @@ -0,0 +1,51 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Turn Python docstrings into Markdown for TensorFlow documentation.""" + + +import enum +import inspect + +from typing import Any + + +class ObjType(enum.Enum): + """Enum to standardize object type checks.""" + TYPE_ALIAS = 'type_alias' + MODULE = 'module' + CLASS = 'class' + CALLABLE = 'callable' + # properties or any `descriptor` + PROPERTY = 'property' + OTHER = 'other' + + @classmethod + def get(cls, py_obj: Any) -> 'ObjType': + """Get the `ObjType` for the `py_object`.""" + if (getattr(py_obj, '__args__', None) and + getattr(py_obj, '__origin__', None)): + return cls.TYPE_ALIAS + elif inspect.ismodule(py_obj): + return cls.MODULE + elif inspect.isclass(py_obj): + return cls.CLASS + elif callable(py_obj): + return cls.CALLABLE + elif hasattr(py_obj, '__get__'): + # This handles any descriptor not only properties. + # https://docs.python.org/3/howto/descriptor.html + return cls.PROPERTY + else: + return cls.OTHER diff --git a/tools/tensorflow_docs/api_generator/parser.py b/tools/tensorflow_docs/api_generator/parser.py index fcf45abfe61..f3d087bc6fc 100644 --- a/tools/tensorflow_docs/api_generator/parser.py +++ b/tools/tensorflow_docs/api_generator/parser.py @@ -14,58 +14,40 @@ # ============================================================================== """Turn Python docstrings into Markdown for TensorFlow documentation.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import ast -import collections - -import itertools -import json -import os +import dataclasses +import enum +import inspect +import pathlib +import posixpath +import pprint import re import textwrap +import typing -from absl import logging - -import astor -import six - -from tensorflow_docs.api_generator import doc_controls -from tensorflow_docs.api_generator import tf_inspect +from typing import Any, Dict, List, Tuple, Iterable, Optional, Union -from google.protobuf.message import Message as ProtoMessage +from tensorflow_docs.api_generator import config +from tensorflow_docs.api_generator import doc_generator_visitor +from tensorflow_docs.api_generator import get_source +from tensorflow_docs.api_generator import obj_type as obj_type_lib +from tensorflow_docs.api_generator import signature as signature_lib -def is_free_function(py_object, full_name, index): - """Check if input is a free function (and not a class- or static method). +def is_class_attr(full_name, index): + """Check if the object's parent is a class. Args: - py_object: The object in question. full_name: The full name of the object, like `tf.module.symbol`. index: The {full_name:py_object} dictionary for the public API. Returns: - True if the object is a stand-alone function, and not part of a class - definition. + True if the object is a class attribute. """ - if not tf_inspect.isfunction(py_object): - return False - parent_name = full_name.rsplit('.', 1)[0] - if tf_inspect.isclass(index[parent_name]): - return False - - return True - - -# A regular expression capturing a python identifier. -IDENTIFIER_RE = r'[a-zA-Z_]\w*' - + if inspect.isclass(index[parent_name]): # pytype: disable=not-supported-yet + return True -class TFDocsError(Exception): - pass + return False def documentation_path(full_name, is_fragment=False): @@ -78,9 +60,10 @@ def documentation_path(full_name, is_fragment=False): Args: full_name: Fully qualified name of a library symbol. - is_fragment: If `False` produce a direct markdown link (`tf.a.b.c` --> + is_fragment: If `False` produce a page link (`tf.a.b.c` --> `tf/a/b/c.md`). If `True` produce fragment link, `tf.a.b.c` --> `tf/a/b.md#c` + Returns: The file path to which to write the documentation for `full_name`. """ @@ -88,7 +71,7 @@ def documentation_path(full_name, is_fragment=False): if is_fragment: parts, fragment = parts[:-1], parts[-1] - result = os.path.join(*parts) + '.md' + result = posixpath.join(*parts) + '.md' if is_fragment: result = result + '#' + fragment @@ -106,30 +89,65 @@ def _get_raw_docstring(py_object): Returns: The docstring, or the empty string if no docstring was found. """ - # For object instances, tf_inspect.getdoc does give us the docstring of their - # type, which is not what we want. Only return the docstring if it is useful. - if (tf_inspect.isclass(py_object) or tf_inspect.ismethod(py_object) or - tf_inspect.isfunction(py_object) or tf_inspect.ismodule(py_object) or - isinstance(py_object, property)): - result = tf_inspect.getdoc(py_object) or '' + obj_type = obj_type_lib.ObjType.get(py_object) + + if obj_type is obj_type_lib.ObjType.TYPE_ALIAS: + result = inspect.getdoc(py_object) + if result == inspect.getdoc(py_object.__origin__): + result = '' + elif obj_type is obj_type_lib.ObjType.CLASS: + if dataclasses.is_dataclass(py_object): + result = _get_dataclass_docstring(py_object) + else: + result = inspect.getdoc(py_object) or '' + if ( + result == inspect.getdoc(dict) + or result == inspect.getdoc(list) + or result == inspect.getdoc(tuple) + ): + result = '' + elif obj_type is obj_type_lib.ObjType.OTHER: + result = '' else: + result = inspect.getdoc(py_object) or '' + + if result is None: result = '' - return _AddDoctestFences()(result + '\n') + lines = result.splitlines() + if lines and 'GENERATED' in lines[0]: + result = '' + + result = _StripTODOs()(result) + result = _StripPylintAndPyformat()(result) + result = _AddDoctestFences()(result + '\n') + result = _DowngradeH1Keywords()(result) + return result + + +def _get_dataclass_docstring(py_object): + result = inspect.getdoc(py_object) or '' + + if (result.startswith(f'{py_object.__name__}(') and result.endswith(')') and + '\n' not in result): + # This is probably an autogenerated dataclass docstring. + # We don't want it. These are not formatted and can be huge and unreadable. + result = '' + return result class _AddDoctestFences(object): """Adds ``` fences around doctest caret blocks >>> that don't have them.""" CARET_BLOCK_RE = re.compile( r""" - (?<=\n)\s*?\n # After a blank line. - (?P\s*)(?P\>\>\>.*?) # Whitespace and a triple caret. - \n\s*?(?=\n|$) # Followed by a blank line""", + \n # After a blank line. + (?P\ *)(?P\>\>\>.*?) # Whitespace and a triple caret. + \n\s*?(?=\n|$) # Followed by a blank line""", re.VERBOSE | re.DOTALL) def _sub(self, match): groups = match.groupdict() - fence = '\n{}```\n'.format(groups['indent']) + fence = f"\n{groups['indent']}```\n" content = groups['indent'] + groups['content'] return ''.join([fence, content, fence]) @@ -138,389 +156,64 @@ def __call__(self, content): return self.CARET_BLOCK_RE.sub(self._sub, content) -class IgnoreLineInBlock(object): - """Ignores the lines in a block. - - Attributes: - block_start: Contains the start string of a block to ignore. - block_end: Contains the end string of a block to ignore. - """ - - def __init__(self, block_start, block_end): - self._block_start = block_start - self._block_end = block_end - self._in_block = False - - self._start_end_regex = re.escape(self._block_start) + r'.*?' + re.escape( - self._block_end) - - def __call__(self, line): - # If start and end block are on the same line, return True. - if re.match(self._start_end_regex, line): - return True - - if not self._in_block: - if self._block_start in line: - self._in_block = True - - elif self._block_end in line: - self._in_block = False - # True is being returned here because the last line in the block should - # also be ignored. - return True - - return self._in_block - -# ?P<...> helps to find the match by entering the group name instead of the -# index. For example, instead of doing match.group(1) we can do -# match.group('brackets') -AUTO_REFERENCE_RE = re.compile( - r""" - (?P\[.*?\]) # find characters inside '[]' - | - `(?P[\w\(\[\)\]\{\}.,=\s]+?)` # or find characters inside '``' - """, - flags=re.VERBOSE) - - -class ReferenceResolver(object): - """Class for replacing `tf.symbol` references with Markdown links.""" - - def __init__(self, duplicate_of, is_fragment, py_module_names): - """Initializes a Reference Resolver. - - Args: - duplicate_of: A map from duplicate names to preferred names of API - symbols. - is_fragment: A map from full names to bool for each symbol. If True the - object lives at a page fragment `tf.a.b.c` --> `tf/a/b#c`. If False - object has a page to itself: `tf.a.b.c` --> `tf/a/b/c`. - py_module_names: A list of string names of Python modules. - """ - self._duplicate_of = duplicate_of - self._is_fragment = is_fragment - self._all_names = set(is_fragment.keys()) - self._py_module_names = py_module_names - self._partial_symbols_dict = self._create_partial_symbols_dict() - - @classmethod - def from_visitor(cls, visitor, **kwargs): - """A factory function for building a ReferenceResolver from a visitor. - - Args: - visitor: an instance of `DocGeneratorVisitor` - **kwargs: all remaining args are passed to the constructor - Returns: - an instance of `ReferenceResolver` () - """ - is_fragment = {} - for name, obj in visitor.index.items(): - has_page = ( - tf_inspect.isclass(obj) or tf_inspect.ismodule(obj) or - is_free_function(obj, name, visitor.index)) - - is_fragment[name] = not has_page - - return cls( - duplicate_of=visitor.duplicate_of, - is_fragment=is_fragment, - **kwargs) - - @classmethod - def from_json_file(cls, filepath): - """Initialize the reference resolver via _api_cache.json.""" - with open(filepath) as f: - json_dict = json.load(f) - - return cls(**json_dict) - - def _partial_symbols(self, symbol): - """Finds the partial symbols given the true symbol. - - For example, symbol: `tf.keras.layers.Conv2D`, then the partial dictionary - returned will be: - - partials = ["tf.keras.layers.Conv2D","keras.layers.Conv2D","layers.Conv2D"] - - There should at least be one '.' in the partial symbol generated so as to - avoid guessing for the true symbol. - - Args: - symbol: String, representing the true symbol. - - Returns: - A list of partial symbol names - """ - - split_symbol = symbol.split('.') - partials = [ - '.'.join(split_symbol[i:]) - for i in range(1, len(split_symbol) - 1) - ] - return partials - - def _create_partial_symbols_dict(self): - """Creates a partial symbols dictionary for all the symbols in TensorFlow. - - Returns: - A dictionary mapping {partial_symbol: real_symbol} - """ - partial_symbols_dict = collections.defaultdict(list) - - for name in sorted(self._all_names): - if 'tf.compat.v' in name or 'tf.contrib' in name: - continue - partials = self._partial_symbols(name) - for partial in partials: - partial_symbols_dict[partial].append(name) - - new_partial_dict = {} - for partial, full_names in partial_symbols_dict.items(): - if not full_names: - continue - - full_names = [ - self._duplicate_of.get(full_name, full_name) - for full_name in full_names - ] - - new_partial_dict[partial] = full_names[0] - - return new_partial_dict - - def to_json_file(self, filepath): - """Converts the RefenceResolver to json and writes it to the specified file. - - Args: - filepath: The file path to write the json to. - """ - - try: - os.makedirs(os.path.dirname(filepath)) - except OSError: - pass - - json_dict = {} - for key, value in self.__dict__.items(): - # Drop these fields, they are generated by the constructor. - if key == '_all_names' or key == '_partial_symbols_dict': - continue - - # Strip off any leading underscores on field names as these are not - # recognized by the constructor. - json_dict[key.lstrip('_')] = value - - with open(filepath, 'w') as f: - json.dump(json_dict, f, indent=2, sort_keys=True) - f.write('\n') - - def replace_references(self, string, relative_path_to_root, full_name=None): - """Replace `tf.symbol` references with links to symbol's documentation page. - - This function finds all occurrences of "`tf.symbol`" in `string` - and replaces them with markdown links to the documentation page - for "symbol". - - `relative_path_to_root` is the relative path from the document - that contains the "`tf.symbol`" reference to the root of the API - documentation that is linked to. If the containing page is part of - the same API docset, `relative_path_to_root` can be set to - `os.path.dirname(documentation_path(name))`, where `name` is the - python name of the object whose documentation page the reference - lives on. - - Args: - string: A string in which "`tf.symbol`" references should be replaced. - relative_path_to_root: The relative path from the containing document to - the root of the API documentation that is being linked to. - full_name: (optional) The full name of current object, so replacements - can depend on context. - - Returns: - `string`, with "`tf.symbol`" references replaced by Markdown links. - """ - - def one_ref(match): - return self._one_ref(match, relative_path_to_root, full_name) - - fixed_lines = [] - - filters = [ - IgnoreLineInBlock('
    ',
-                          '{% endhtmlescape %}
    '), - IgnoreLineInBlock('```', '```') - ] - - for line in string.splitlines(): - if not any(filter_block(line) for filter_block in filters): - line = re.sub(AUTO_REFERENCE_RE, one_ref, line) - fixed_lines.append(line) - - return '\n'.join(fixed_lines) - - def python_link(self, link_text, ref_full_name, relative_path_to_root, - code_ref=True): - """Resolve a "`tf.symbol`" reference to a Markdown link. - - This will pick the canonical location for duplicate symbols. The - input to this function should already be stripped of the '@' and - '{}'. This function returns a Markdown link. If `code_ref` is - true, it is assumed that this is a code reference, so the link - text will be rendered as code (using backticks). - `link_text` should refer to a library symbol, starting with 'tf.'. - - Args: - link_text: The text of the Markdown link. - ref_full_name: The fully qualified name of the symbol to link to. - relative_path_to_root: The relative path from the location of the current - document to the root of the API documentation. - code_ref: If true (the default), put `link_text` in `...`. - - Returns: - A markdown link to the documentation page of `ref_full_name`. - """ - url = self.reference_to_url(ref_full_name, relative_path_to_root) - - if code_ref: - link_text = link_text.join(['', '']) - else: - link_text = self._link_text_to_html(link_text) - - return '{}'.format(url, link_text) - - @staticmethod - def _link_text_to_html(link_text): - code_re = '`(.*?)`' - return re.sub(code_re, r'\1', link_text) - - def py_master_name(self, full_name): - """Return the master name for a Python symbol name.""" - return self._duplicate_of.get(full_name, full_name) - - def reference_to_url(self, ref_full_name, relative_path_to_root): - """Resolve a "`tf.symbol`" reference to a relative path. - - The input to this function should already be stripped of the '@' - and '{}', and its output is only the link, not the full Markdown. +class _StripTODOs(object): + TODO_RE = re.compile('#? *TODO.*') - If `ref_full_name` is the name of a class member, method, or property, the - link will point to the page of the containing class, and it will include the - method name as an anchor. For example, `tf.module.MyClass.my_method` will be - translated into a link to - `os.join.path(relative_path_to_root, 'tf/module/MyClass.md#my_method')`. + def __call__(self, content: str) -> str: + return self.TODO_RE.sub('', content) - Args: - ref_full_name: The fully qualified name of the symbol to link to. - relative_path_to_root: The relative path from the location of the current - document to the root of the API documentation. - - Returns: - A relative path that links from the documentation page of `from_full_name` - to the documentation page of `ref_full_name`. - - Raises: - TFDocsError: If the symbol is not found. - """ - if self._is_fragment.get(ref_full_name, False): - # methods and constants get duplicated. And that's okay. - # Use the master name of their parent. - parent_name, short_name = ref_full_name.rsplit('.', 1) - parent_master_name = self._duplicate_of.get(parent_name, parent_name) - master_name = '.'.join([parent_master_name, short_name]) - else: - master_name = self._duplicate_of.get(ref_full_name, ref_full_name) - - # Check whether this link exists - if master_name not in self._all_names: - raise TFDocsError( - 'Cannot make link to "%s": Not in index.' % master_name) - ref_path = documentation_path(master_name, self._is_fragment[master_name]) - return os.path.join(relative_path_to_root, ref_path) +class _StripPylintAndPyformat(object): + STRIP_RE = re.compile('# *?(pylint|pyformat):.*', re.I) - def _one_ref(self, match, relative_path_to_root, full_name=None): - """Return a link for a single "`tf.symbol`" reference.""" + def __call__(self, content: str) -> str: + return self.STRIP_RE.sub('', content) - if match.group(1): - # Found a '[]' group, return it unmodified. - return match.group('brackets') - # Found a '``' group. - string = match.group('backticks') - - link_text = string - - string = re.sub(r'(.*)[\(\[].*', r'\1', string) - - if string.startswith('compat.v1') or string.startswith('compat.v2'): - string = 'tf.' + string - elif string.startswith('v1') or string.startswith('v2'): - string = 'tf.compat.' + string - - elif full_name is None or ('tf.compat.v' not in full_name and - 'tf.contrib' not in full_name): - string = self._partial_symbols_dict.get(string, string) - - try: - if string.startswith('tensorflow::'): - # C++ symbol - return self._cc_link(string, link_text, relative_path_to_root) - - is_python = False - for py_module_name in self._py_module_names: - if string == py_module_name or string.startswith(py_module_name + '.'): - is_python = True - break - - if is_python: # Python symbol - return self.python_link( - link_text, string, relative_path_to_root, code_ref=True) - except TFDocsError: - pass - - return match.group(0) - - def _cc_link(self, string, link_text, relative_path_to_root): - """Generate a link for a `tensorflow::...` reference.""" - # TODO(joshl): Fix this hard-coding of paths. - if string == 'tensorflow::ClientSession': - ret = 'class/tensorflow/client-session.md' - elif string == 'tensorflow::Scope': - ret = 'class/tensorflow/scope.md' - elif string == 'tensorflow::Status': - ret = 'class/tensorflow/status.md' - elif string == 'tensorflow::Tensor': - ret = 'class/tensorflow/tensor.md' - elif string == 'tensorflow::ops::Const': - ret = 'namespace/tensorflow/ops.md#const' - else: - raise TFDocsError('C++ reference not understood: "%s"' % string) +class _DowngradeH1Keywords(): + """Convert keras docstring keyword format to google format.""" - # relative_path_to_root gets you to api_docs/python, we go from there - # to api_docs/cc, and then add ret. - cc_relative_path = os.path.normpath(os.path.join( - relative_path_to_root, '../cc', ret)) - - return '{}'.format(cc_relative_path, - link_text) - - -# TODO(aselle): Collect these into a big list for all modules and functions -# and make a rosetta stone page. -def _handle_compatibility(doc): + KEYWORD_H1_RE = re.compile( + r""" + ^ # Start of line + (?P\s*) # Capture leading whitespace as + (?PArgs|Arguments|Returns|Raises|Yields|Examples?|Notes?) + \s*:? # Optional whitespace and optional ":" + """, re.VERBOSE) + + def __call__(self, docstring): + lines = docstring.splitlines() + + new_lines = [] + is_code = False + for line in lines: + if line.strip().startswith('```'): + is_code = not is_code + elif not is_code: + line = self.KEYWORD_H1_RE.sub(r'\g\g:', line) + new_lines.append(line) + + docstring = '\n'.join(new_lines) + return docstring + + +def _handle_compatibility(doc) -> Tuple[str, Dict[str, str]]: """Parse and remove compatibility blocks from the main docstring. Args: - doc: The docstring that contains compatibility notes" + doc: The docstring that contains compatibility notes. Returns: - a tuple of the modified doc string and a hash that maps from compatibility + A tuple of the modified doc string and a hash that maps from compatibility note type to the text of the note. """ compatibility_notes = {} - match_compatibility = re.compile(r'[ \t]*@compatibility\((\w+)\)\s*\n' - r'((?:[^@\n]*\n)+)' - r'\s*@end_compatibility') + match_compatibility = re.compile( + r'[ \t]*@compatibility\(([^\n]+?)\)\s*\n' + r'(.*?)' + r'[ \t]*@end_compatibility', re.DOTALL) for f in match_compatibility.finditer(doc): compatibility_notes[f.group(1)] = f.group(2) return match_compatibility.subn(r'', doc)[0], compatibility_notes @@ -539,6 +232,36 @@ def _pairs(items): return list(zip(items[::2], items[1::2])) +# Don't change the width="214px" without consulting with the devsite-team. +TABLE_TEMPLATE = textwrap.dedent(""" + + + + + {text} + {items} +
    {title}
    + """) + +ITEMS_TEMPLATE = textwrap.dedent("""\ + + + {name}{anchor} + + + {description} + + """) + +TEXT_TEMPLATE = textwrap.dedent("""\ + + + {text} + + """) + + +@dataclasses.dataclass class TitleBlock(object): """A class to parse title blocks (like `Args:`) and convert them to markdown. @@ -578,19 +301,78 @@ class TitleBlock(object): indentation. """ - def __init__(self, title, text, items): - self.title = title - self.text = text - self.items = items + _INDENTATION_REMOVAL_RE = re.compile(r'( *)(.+)') + + title: Optional[str] + text: str + items: Iterable[Tuple[str, str]] + + def _dedent_after_first_line(self, text): + if '\n' not in text: + return text + + first, remainder = text.split('\n', 1) + remainder = textwrap.dedent(remainder) + result = '\n'.join([first, remainder]) + return result + + def table_view(self, + title_template: Optional[str] = None, + anchors: bool = True) -> str: + """Returns the TitleBlock as an HTML table. + + Args: + title_template: Template for title detailing how to display it. + + Returns: + Table containing the content to display. + """ + + if title_template is not None: + title = title_template.format(title=self.title) + else: + title = self.title + + text = self.text.strip() + if text: + text = self._dedent_after_first_line(text) + text = TEXT_TEMPLATE.format(text=text) + + items = [] + for name, description in self.items: + if not description: + description = '' + else: + description = description.strip('\n') + description = self._dedent_after_first_line(description) + + if anchors: + anchor = f'' + else: + anchor = '' + + item_table = ITEMS_TEMPLATE.format( + name=f'`{name}`', anchor=anchor, description=description) + items.append(item_table) + + return '\n' + TABLE_TEMPLATE.format( + title=title, text=text, items=''.join(items)) + '\n' + + def __str__(self) -> str: + """Returns a non-tempated version of the TitleBlock.""" - def __str__(self): - """Returns a markdown compatible version of the TitleBlock.""" sub = [] - sub.append('\n\n#### ' + self.title + ':\n') + sub.append(f'\n\n#### {self.title}:\n') sub.append(textwrap.dedent(self.text)) sub.append('\n') + for name, description in self.items: - sub.append('* `{}`: {}'.format(name, description)) + description = description.strip() + if not description: + sub.append(f'* `{name}`\n') + else: + sub.append(f'* `{name}`: {description}\n') + return ''.join(sub) # This regex matches an entire title-block. @@ -604,17 +386,21 @@ def __str__(self): # (a new-line followed by non-whitespace) """, re.VERBOSE | re.DOTALL) - # This ITEM_RE = re.compile( r""" - ^(\*?\*? # Capture optional *s to allow *args, **kwargs. - \w[\w.]*? # Capture a word character followed by word characters - # or "."s. - )\s*:\s # Allow any whitespace around the colon.""", - re.MULTILINE | re.VERBOSE) + ^(\*?\*?'?"? # Optional * to allow *args, **kwargs and quotes + \w[\w.'"]*? # words, dots and closing quotes + (?: # non capturing + [ ]? # maybe a space + \( # a `(` + [\ \(\[\w.,\)\]]*? # word chars, dots or more open/close or space + )? # all optional + )\s*:\s # Allow any whitespace around the colon.""", + re.MULTILINE | re.VERBOSE, + ) @classmethod - def split_string(cls, docstring): + def split_string(cls, docstring: str): r"""Given a docstring split it into a list of `str` or `TitleBlock` chunks. For example the docstring of `tf.nn.relu`: @@ -674,1000 +460,317 @@ def split_string(cls, docstring): text = split.pop(0) items = _pairs(split) - title_block = cls(title, text, items) + items = list(cls._split_items(items)) + + title_block = cls(title=title, text=text, items=items) parts.append(title_block) return parts + @classmethod + def _split_items( + cls, items: Iterable[Tuple[str, str]] + ) -> Iterable[Tuple[str, str]]: + """If there's a type in the name, move it to the top of the description.""" + for name, value in items: + if '(' in name: + name, type_str = re.split(r' ?[\(]', name, maxsplit=1) + type_str = f"`{type_str.rstrip(')')}`" + value = f'{type_str}\n\n{value}' -_DocstringInfo = collections.namedtuple( - '_DocstringInfo', ['brief', 'docstring_parts', 'compatibility']) - - -def _parse_md_docstring(py_object, relative_path_to_root, full_name, - reference_resolver): - """Parse the object's docstring and return a `_DocstringInfo`. - - This function clears @@'s from the docstring, and replaces `` references - with markdown links. - - For links within the same set of docs, the `relative_path_to_root` for a - docstring on the page for `full_name` can be set to: - - ```python - relative_path_to_root = os.path.relpath( - path='.', start=os.path.dirname(documentation_path(full_name)) or '.') - ``` - - Args: - py_object: A python object to retrieve the docs for (class, function/method, - or module). - relative_path_to_root: The relative path from the location of the current - document to the root of the Python API documentation. This is used to - compute links for "`tf.symbol`" references. - full_name: (optional) The api path to the current object, so replacements - can depend on context. - reference_resolver: An instance of ReferenceResolver. + yield name, value - Returns: - A _DocstringInfo object, all fields will be empty if no docstring was found. - """ - # TODO(wicke): If this is a partial, use the .func docstring and add a note. - raw_docstring = _get_raw_docstring(py_object) - raw_docstring = reference_resolver.replace_references(raw_docstring, - relative_path_to_root, - full_name) +class DocstringInfo(typing.NamedTuple): + brief: str + docstring_parts: List[Union[TitleBlock, str]] + compatibility: Dict[str, str] - atat_re = re.compile(r' *@@[a-zA-Z_.0-9]+ *$') - raw_docstring = '\n'.join( - line for line in raw_docstring.split('\n') if not atat_re.match(line)) - docstring, compatibility = _handle_compatibility(raw_docstring) +def _get_other_member_doc( + obj: Any, + parser_config: config.ParserConfig, + extra_docs: Optional[Dict[int, str]], +) -> str: + """Returns the docs for other members of a module.""" - if 'Generated by: tensorflow/tools/api/generator' in docstring: - docstring = '' + # An object's __doc__ attribute will mask the class'. + my_doc = inspect.getdoc(obj) + class_doc = inspect.getdoc(type(obj)) - # Remove the first-line "brief" docstring. - lines = docstring.split('\n') - brief = lines.pop(0) + description = None + if my_doc != class_doc: + # If they're different it's because __doc__ is set on the instance. + if my_doc is not None: + description = my_doc - docstring = '\n'.join(lines) + if description is None and extra_docs is not None: + description = extra_docs.get(id(obj), None) - docstring_parts = TitleBlock.split_string(docstring) + value_repr = _tfrepr(obj, parser_config) - return _DocstringInfo(brief, docstring_parts, compatibility) + parts = [value_repr, description] + parts = [item for item in parts if item is not None] + return '\n\n'.join(parts) -def _remove_first_line_indent(string): - indent = len(re.match(r'^\s*', string).group(0)) - return '\n'.join([line[indent:] for line in string.split('\n')]) +def _tfrepr(obj, parser_config): + """Convert an object to a string for display.""" + info = None -PAREN_NUMBER_RE = re.compile(r'^\(([0-9.e-]+)\)') -OBJECT_MEMORY_ADDRESS_RE = re.compile(r'<(?P.+) object at 0x[\da-f]+>') + if isinstance(obj, dict): + # pprint.pformat (next block) doesn't sort dicts until python 3.8 + items = [ + f' {name!r}: {value!r}' + for name, value in sorted(obj.items(), key=repr) + ] + items = ',\n'.join(items) + info = f'```\n{{\n{items}\n}}\n```' + + elif isinstance(obj, (set, frozenset)): + # pprint.pformat (next block) doesn't sort dicts until python 3.8 + items = [f' {value!r}' for value in sorted(obj, key=repr)] + items = ',\n'.join(items) + info = f'```\n{{\n{items}\n}}\n```' + elif (doc_generator_visitor.maybe_singleton(obj) or + isinstance(obj, (list, tuple, enum.Enum))): + # * Use pformat instead of repr so dicts and sets are sorted (deterministic) + # * Escape ` so it doesn't break code formatting. You can't use "`" + # here since it will diaplay as a literal. I'm not sure why 
    
+    #   breaks on the site.
+    info = pprint.pformat(obj).replace('`', r'\`')
+    info = f'`{info}`'
+  elif obj_type_lib.ObjType.get(obj) is obj_type_lib.ObjType.PROPERTY:
+    info = None
+  else:
+    class_full_name = parser_config.reverse_index.get(id(type(obj)), None)
+    if class_full_name is None:
+      module = getattr(type(obj), '__module__', None)
+      class_name = type(obj).__name__
+      if module is None or module == 'builtins':
+        class_full_name = class_name
+      else:
+        class_full_name = f'{module}.{class_name}'
+    info = f'Instance of `{class_full_name}`'
 
+  if info is not None:
+    info = signature_lib.strip_obj_addresses(info)
 
-def _generate_signature(func, reverse_index):
-  """Given a function, returns a list of strings representing its args.
+  return info
 
-  This function produces a list of strings representing the arguments to a
-  python function. It uses tf_inspect.getfullargspec, which
-  does not generalize well to Python 3.x, which is more flexible in how *args
-  and **kwargs are handled. This is not a problem in TF, since we have to remain
-  compatible to Python 2.7 anyway.
 
-  This function uses `__name__` for callables if it is available. This can lead
-  to poor results for functools.partial and other callable objects.
+def parse_md_docstring(
+    py_object: Any,
+    full_name: str,
+    parser_config: config.ParserConfig,
+    extra_docs: Optional[Dict[int, str]] = None,
+) -> DocstringInfo:
+  """Parse the object's docstring and return a `DocstringInfo`.
 
-  The returned string is Python code, so if it is included in a Markdown
-  document, it should be typeset as code (using backticks), or escaped.
+  This function clears @@'s from the docstring, and replaces `` references
+  with links.
 
   Args:
-    func: A function, method, or functools.partial to extract the signature for.
-    reverse_index: A map from object ids to canonical full names to use.
+    py_object: A python object to retrieve the docs for (class, function/method,
+      or module).
+    full_name: (optional) The api path to the current object, so replacements
+      can depend on context.
+    parser_config: An instance of `config.ParserConfig`.
+    extra_docs: Extra docs for symbols like public constants(list, tuple, etc)
+      that need to be added to the markdown pages created.
 
   Returns:
-    A list of strings representing the argument signature of `func` as python
-    code.
+    A DocstringInfo object, all fields will be empty if no docstring was found.
   """
 
-  args_list = []
+  if obj_type_lib.ObjType.get(py_object) is obj_type_lib.ObjType.OTHER:
+    raw_docstring = _get_other_member_doc(
+        obj=py_object, parser_config=parser_config, extra_docs=extra_docs)
+  else:
+    raw_docstring = _get_raw_docstring(py_object)
 
-  argspec = tf_inspect.getfullargspec(func)
-  first_arg_with_default = (
-      len(argspec.args or []) - len(argspec.defaults or []))
+  atat_re = re.compile(r' *@@[a-zA-Z_.0-9]+ *$')
+  raw_docstring = '\n'.join(
+      line for line in raw_docstring.split('\n') if not atat_re.match(line))
 
-  # Python documentation skips `self` when printing method signatures.
-  # Note we cannot test for ismethod here since unbound methods do not register
-  # as methods (in Python 3).
-  first_arg = 1 if 'self' in argspec.args[:1] else 0
+  docstring, compatibility = _handle_compatibility(raw_docstring)
+  compatibility = {
+      key: parser_config.reference_resolver.replace_references(value, full_name)
+      for key, value in compatibility.items()
+  }
 
-  # Add all args without defaults.
-  for arg in argspec.args[first_arg:first_arg_with_default]:
-    args_list.append(arg)
+  if 'Generated by: tensorflow/tools/api/generator' in docstring:
+    docstring = ''
 
-  # Add all args with defaults.
-  if argspec.defaults:
-    try:
-      source = _remove_first_line_indent(tf_inspect.getsource(func))
-      func_ast = ast.parse(source)
-      ast_defaults = func_ast.body[0].args.defaults
-    except IOError:  # If this is a builtin, getsource fails with IOError
-      # If we cannot get the source, assume the AST would be equal to the repr
-      # of the defaults.
-      ast_defaults = [None] * len(argspec.defaults)
-    except SyntaxError:
-      # You may get a SyntaxError using pytype in python 2.
-      ast_defaults = [None] * len(argspec.defaults)
-    except IndexError:
-      # Some python3 signatures fail in tf_inspect.getsource with IndexError
-      ast_defaults = [None] * len(argspec.defaults)
-    except AttributeError:
-      # Some objects in tfp throw attribute errors here.
-      ast_defaults = [None] * len(argspec.defaults)
-
-    for arg, default, ast_default in zip(
-        argspec.args[first_arg_with_default:], argspec.defaults, ast_defaults):
-      if id(default) in reverse_index:
-        default_text = reverse_index[id(default)]
-      elif ast_default is not None:
-        default_text = (
-            astor.to_source(ast_default).rstrip('\n').replace('\t', '\\t')
-            .replace('\n', '\\n').replace('"""', "'"))
-        default_text = PAREN_NUMBER_RE.sub('\\1', default_text)
-
-        if default_text != repr(default):
-          # This may be an internal name. If so, handle the ones we know about.
-          # TODO(wicke): This should be replaced with a lookup in the index.
-          # TODO(wicke): (replace first ident with tf., check if in index)
-          internal_names = {
-              'ops.GraphKeys': 'tf.GraphKeys',
-              '_ops.GraphKeys': 'tf.GraphKeys',
-              'init_ops.zeros_initializer': 'tf.zeros_initializer',
-              'init_ops.ones_initializer': 'tf.ones_initializer',
-              'saver_pb2.SaverDef': 'tf.train.SaverDef',
-          }
-          full_name_re = '^%s(.%s)+' % (IDENTIFIER_RE, IDENTIFIER_RE)
-          match = re.match(full_name_re, default_text)
-          if match:
-            lookup_text = default_text
-            for internal_name, public_name in six.iteritems(internal_names):
-              if match.group(0).startswith(internal_name):
-                lookup_text = public_name + default_text[len(internal_name):]
-                break
-            if default_text is lookup_text:
-              logging.warn(
-                  'WARNING: Using default arg, failed lookup: %s, repr: %r',
-                  default_text, default)
-            else:
-              default_text = lookup_text
-      else:
-        default_text = repr(default)
-        # argspec.defaults can contain object memory addresses, i.e.
-        # containers.MutableMapping.pop. Strip these out to avoid
-        # unnecessary doc churn between invocations.
-        default_text = OBJECT_MEMORY_ADDRESS_RE.sub(r'<\g>', default_text)
+  lines = docstring.split('\n')
+  first_line = lines[0].strip()
+
+  good_first_line = (
+      first_line.endswith(('.', '!', '?', ')')) or first_line.isupper()
+  )
+
+  def escape(match):
+    return (
+        match.group(0)
+        .replace('.', '.')
+        .replace('!', '!')
+        .replace('?', '?')
+    )
+
+  def unescape(s):
+    return (
+        s.replace('.', '.')
+        .replace('!', '!')
+        .replace('?', '?')
+    )
+
+  escaped = re.sub('`(.|\n)*?`', escape, docstring)
+  match = re.match(
+      r"""
+      (?P
+          .*?    # Take as little as possible
+          (
+              [.!?](?!)($|(?=\s))\n?|   # stop at the end of a sentence
+              (?=\n\n)              # or before a blank line
+          )
+      )
+      (?P.*)         # collect the rest of the docstring
+      """,
+      escaped,
+      re.VERBOSE | re.DOTALL,
+  )
+  if not good_first_line and match:
+    groupdict = match.groupdict()
+    brief = unescape(re.sub('\s+', ' ', groupdict['first_sentence']))
+    docstring = unescape(groupdict['remainder'])
+  else:
+    # Use the first line
+    brief = lines.pop(0)
+    docstring = '\n'.join(lines)
 
-      args_list.append('%s=%s' % (arg, default_text))
+  brief = brief.strip()
+  brief = parser_config.reference_resolver.replace_references(brief, full_name)
+  docstring = parser_config.reference_resolver.replace_references(
+      docstring, full_name
+  )
 
-  # Add *args and *kwargs.
-  if argspec.varargs:
-    args_list.append('*' + argspec.varargs)
-  if argspec.varkw:
-    args_list.append('**' + argspec.varkw)
+  docstring_parts = TitleBlock.split_string(docstring)
 
-  return args_list
+  return DocstringInfo(brief, docstring_parts, compatibility)
 
 
-def _get_defining_class(py_class, name):
-  for cls in tf_inspect.getmro(py_class):
+def get_defining_class(py_class, name):
+  for cls in inspect.getmro(py_class):
     if name in cls.__dict__:
       return cls
   return None
 
 
-class _LinkInfo(
-    collections.namedtuple(
-        '_LinkInfo', ['short_name', 'full_name', 'obj', 'doc', 'url'])):
-
-  __slots__ = []
-
-  def is_link(self):
-    return True
-
-
-class _OtherMemberInfo(
-    collections.namedtuple('_OtherMemberInfo',
-                           ['short_name', 'full_name', 'obj', 'doc'])):
-
-  __slots__ = []
-
-  def is_link(self):
-    return False
-
-
-_PropertyInfo = collections.namedtuple(
-    '_PropertyInfo', ['short_name', 'full_name', 'obj', 'doc'])
-
-_MethodInfo = collections.namedtuple('_MethodInfo', [
-    'short_name',
-    'full_name',
-    'obj',
-    'doc',
-    'signature',
-    'decorators',
-    'defined_in',
-])
-
-
-class _FunctionPageInfo(object):
-  """Collects docs For a function Page."""
-
-  def __init__(self, full_name):
-    self._full_name = full_name
-    self._defined_in = None
-    self._aliases = None
-    self._doc = None
-
-    self._signature = None
-    self._decorators = []
-
-  def for_function(self):
-    return True
-
-  def for_class(self):
-    return False
-
-  def for_module(self):
-    return False
-
-  @property
-  def full_name(self):
-    return self._full_name
-
-  @property
-  def short_name(self):
-    return self._full_name.split('.')[-1]
-
-  @property
-  def defined_in(self):
-    return self._defined_in
-
-  def set_defined_in(self, defined_in):
-    assert self.defined_in is None
-    self._defined_in = defined_in
-
-  @property
-  def aliases(self):
-    return self._aliases
-
-  def set_aliases(self, aliases):
-    assert self.aliases is None
-    self._aliases = aliases
-
-  @property
-  def doc(self):
-    return self._doc
-
-  def set_doc(self, doc):
-    assert self.doc is None
-    self._doc = doc
-
-  @property
-  def signature(self):
-    return self._signature
-
-  def set_signature(self, function, reverse_index):
-    """Attach the function's signature.
-
-    Args:
-      function: The python function being documented.
-      reverse_index: A map from object ids in the index to full names.
-    """
-
-    assert self.signature is None
-    self._signature = _generate_signature(function, reverse_index)
-
-  @property
-  def decorators(self):
-    return list(self._decorators)
-
-  def add_decorator(self, dec):
-    self._decorators.append(dec)
-
-  def get_metadata_html(self):
-    return Metadata(self.full_name).build_html()
-
-
-class _ClassPageInfo(object):
-  """Collects docs for a class page.
-
-  Attributes:
-    full_name: The fully qualified name of the object at the master
-      location. Aka `master_name`. For example: `tf.nn.sigmoid`.
-    short_name: The last component of the `full_name`. For example: `sigmoid`.
-    defined_in: The path to the file where this object is defined.
-    aliases: The list of all fully qualified names for the locations where the
-      object is visible in the public api. This includes the master location.
-    doc: A `_DocstringInfo` object representing the object's docstring (can be
-      created with `_parse_md_docstring`).
-   bases: A list of `_LinkInfo` objects pointing to the docs for the parent
-      classes.
-    properties: A list of `_PropertyInfo` objects documenting the class'
-      properties (attributes that use `@property`).
-    methods: A list of `_MethodInfo` objects documenting the class' methods.
-    classes: A list of `_LinkInfo` objects pointing to docs for any nested
-      classes.
-    other_members: A list of `_OtherMemberInfo` objects documenting any other
-      object's defined inside the class object (mostly enum style fields).
-    namedtuplefields: a list of the namedtuple fields in the class.
-  """
-
-  def __init__(self, full_name):
-    self._full_name = full_name
-    self._defined_in = None
-    self._aliases = None
-    self._doc = None
-    self._namedtuplefields = None
-
-    self._bases = None
-    self._properties = []
-    self._methods = []
-    self._classes = []
-    self._other_members = []
-
-  def for_function(self):
-    """Returns true if this object documents a function."""
-    return False
-
-  def for_class(self):
-    """Returns true if this object documents a class."""
-    return True
-
-  def for_module(self):
-    """Returns true if this object documents a module."""
-    return False
-
-  @property
-  def full_name(self):
-    """Returns the documented object's fully qualified name."""
-    return self._full_name
-
-  @property
-  def short_name(self):
-    """Returns the documented object's short name."""
-    return self._full_name.split('.')[-1]
-
-  @property
-  def defined_in(self):
-    """Returns the path to the file where the documented object is defined."""
-    return self._defined_in
-
-  def set_defined_in(self, defined_in):
-    """Sets the `defined_in` path."""
-    assert self.defined_in is None
-    self._defined_in = defined_in
-
-  @property
-  def aliases(self):
-    """Returns a list of all full names for the documented object."""
-    return self._aliases
-
-  def set_aliases(self, aliases):
-    """Sets the `aliases` list.
-
-    Args:
-      aliases: A list of strings. Containing all the object's full names.
-    """
-    assert self.aliases is None
-    self._aliases = aliases
-
-  @property
-  def doc(self):
-    """Returns a `_DocstringInfo` created from the object's docstring."""
-    return self._doc
-
-  def set_doc(self, doc):
-    """Sets the `doc` field.
-
-    Args:
-      doc: An instance of `_DocstringInfo`.
-    """
-    assert self.doc is None
-    self._doc = doc
-
-  @property
-  def namedtuplefields(self):
-    return self._namedtuplefields
-
-  def set_namedtuplefields(self, py_class):
-    if issubclass(py_class, tuple):
-      if all(
-          hasattr(py_class, attr)
-          for attr in ('_asdict', '_fields', '_make', '_replace')):
-        self._namedtuplefields = py_class._fields
-
-  @property
-  def bases(self):
-    """Returns a list of `_LinkInfo` objects pointing to the class' parents."""
-    return self._bases
-
-  def _set_bases(self, relative_path, parser_config):
-    """Builds the `bases` attribute, to document this class' parent-classes.
-
-    This method sets the `bases` to a list of `_LinkInfo` objects point to the
-    doc pages for the class' parents.
-
-    Args:
-      relative_path: The relative path from the doc this object describes to
-        the documentation root.
-      parser_config: An instance of `ParserConfig`.
-    """
-    bases = []
-    obj = parser_config.py_name_to_object(self.full_name)
-    for base in obj.__bases__:
-      base_full_name = parser_config.reverse_index.get(id(base), None)
-      if base_full_name is None:
-        continue
-      base_doc = _parse_md_docstring(base, relative_path, self.full_name,
-                                     parser_config.reference_resolver)
-      base_url = parser_config.reference_resolver.reference_to_url(
-          base_full_name, relative_path)
-
-      link_info = _LinkInfo(short_name=base_full_name.split('.')[-1],
-                            full_name=base_full_name, obj=base,
-                            doc=base_doc, url=base_url)
-      bases.append(link_info)
-
-    self._bases = bases
-
-  @property
-  def properties(self):
-    """Returns a list of `_PropertyInfo` describing the class' properties."""
-    props_dict = {prop.short_name: prop for prop in self._properties}
-    props = []
-    if self.namedtuplefields:
-      for field in self.namedtuplefields:
-        field_prop = props_dict.pop(field, None)
-        if field_prop is not None:
-          props.append(field_prop)
-
-    props.extend(sorted(props_dict.values()))
-
-    return props
-
-  def _add_property(self, short_name, full_name, obj, doc):
-    """Adds a `_PropertyInfo` entry to the `properties` list.
-
-    Args:
-      short_name: The property's short name.
-      full_name: The property's fully qualified name.
-      obj: The property object itself
-      doc: The property's parsed docstring, a `_DocstringInfo`.
-    """
-    # Hide useless namedtuple docs-trings.
-    if re.match('Alias for field number [0-9]+', doc.brief):
-      doc = doc._replace(docstring_parts=[], brief='')
-    property_info = _PropertyInfo(short_name, full_name, obj, doc)
-    self._properties.append(property_info)
-
-  @property
-  def methods(self):
-    """Returns a list of `_MethodInfo` describing the class' methods."""
-    return self._methods
-
-  def _add_method(self, short_name, full_name, obj, doc, signature, decorators,
-                  defined_in):
-    """Adds a `_MethodInfo` entry to the `methods` list.
-
-    Args:
-      short_name: The method's short name.
-      full_name: The method's fully qualified name.
-      obj: The method object itself
-      doc: The method's parsed docstring, a `_DocstringInfo`
-      signature: The method's parsed signature (see: `_generate_signature`)
-      decorators: A list of strings describing the decorators that should be
-        mentioned on the object's docs page.
-      defined_in: A `_FileLocation` object pointing to the object source.
-    """
-    method_info = _MethodInfo(short_name, full_name, obj, doc, signature,
-                              decorators, defined_in)
-    self._methods.append(method_info)
-
-  @property
-  def classes(self):
-    """Returns a list of `_LinkInfo` pointing to any nested classes."""
-    return self._classes
-
-  def get_metadata_html(self):
-    meta_data = Metadata(self.full_name)
-    for item in itertools.chain(self.classes, self.properties, self.methods,
-                                self.other_members):
-      meta_data.append(item)
-
-    return meta_data.build_html()
-
-  def _add_class(self, short_name, full_name, obj, doc, url):
-    """Adds a `_LinkInfo` for a nested class to `classes` list.
-
-    Args:
-      short_name: The class' short name.
-      full_name: The class' fully qualified name.
-      obj: The class object itself
-      doc: The class' parsed docstring, a `_DocstringInfo`
-      url: A url pointing to where the nested class is documented.
-    """
-    page_info = _LinkInfo(short_name, full_name, obj, doc, url)
-
-    self._classes.append(page_info)
-
-  @property
-  def other_members(self):
-    """Returns a list of `_OtherMemberInfo` describing any other contents."""
-    return self._other_members
-
-  def _add_other_member(self, short_name, full_name, obj, doc):
-    """Adds an `_OtherMemberInfo` entry to the `other_members` list.
-
-    Args:
-      short_name: The class' short name.
-      full_name: The class' fully qualified name.
-      obj: The class object itself
-      doc: The class' parsed docstring, a `_DocstringInfo`
-    """
-    other_member_info = _OtherMemberInfo(short_name, full_name, obj, doc)
-    self._other_members.append(other_member_info)
-
-  def collect_docs_for_class(self, py_class, parser_config):
-    """Collects information necessary specifically for a class's doc page.
-
-    Mainly, this is details about the class's members.
-
-    Args:
-      py_class: The class object being documented
-      parser_config: An instance of ParserConfig.
-    """
-    self.set_namedtuplefields(py_class)
-    doc_path = documentation_path(self.full_name)
-    relative_path = os.path.relpath(
-        path='.', start=os.path.dirname(doc_path) or '.')
-
-    self._set_bases(relative_path, parser_config)
-
-    for short_name in parser_config.tree[self.full_name]:
-      child_name = '.'.join([self.full_name, short_name])
-      child = parser_config.py_name_to_object(child_name)
-
-      # Don't document anything that is defined in object or by protobuf.
-      defining_class = _get_defining_class(py_class, short_name)
-      if defining_class in [object, type, tuple, BaseException, Exception]:
-        continue
-
-      # The following condition excludes most protobuf-defined symbols.
-      if (defining_class and
-          defining_class.__name__ in ['CMessage', 'Message', 'MessageMeta']):
-        continue
-
-      if doc_controls.should_skip_class_attr(py_class, short_name):
-        continue
-
-      child_doc = _parse_md_docstring(child, relative_path, self.full_name,
-                                      parser_config.reference_resolver)
-
-      if isinstance(child, property):
-        self._add_property(short_name, child_name, child, child_doc)
-
-      elif tf_inspect.isclass(child):
-        if defining_class is None:
-          continue
-        url = parser_config.reference_resolver.reference_to_url(
-            child_name, relative_path)
-        self._add_class(short_name, child_name, child, child_doc, url)
-
-      elif (tf_inspect.ismethod(child) or tf_inspect.isfunction(child) or
-            tf_inspect.isroutine(child)):
-        if defining_class is None:
-          continue
-
-        # Omit methods defined by namedtuple.
-        original_method = defining_class.__dict__[short_name]
-        if (hasattr(original_method, '__module__') and
-            (original_method.__module__ or '').startswith('namedtuple')):
-          continue
-
-        # Some methods are often overridden without documentation. Because it's
-        # obvious what they do, don't include them in the docs if there's no
-        # docstring.
-        if not child_doc.brief.strip() and short_name in [
-            '__del__', '__copy__'
-        ]:
-          continue
-
-        try:
-          child_signature = _generate_signature(child,
-                                                parser_config.reverse_index)
-        except TypeError:
-          # If this is a (dynamically created) slot wrapper, tf_inspect will
-          # raise typeerror when trying to get to the code. Ignore such
-          # functions.
-          continue
-
-        child_decorators = []
-        try:
-          if isinstance(py_class.__dict__[short_name], classmethod):
-            child_decorators.append('classmethod')
-        except KeyError:
-          pass
-
-        try:
-          if isinstance(py_class.__dict__[short_name], staticmethod):
-            child_decorators.append('staticmethod')
-        except KeyError:
-          pass
-
-        defined_in = _get_defined_in(child, parser_config)
-        self._add_method(short_name, child_name, child, child_doc,
-                         child_signature, child_decorators, defined_in)
-      else:
-        # Exclude members defined by protobuf that are useless
-        if issubclass(py_class, ProtoMessage):
-          if (short_name.endswith('_FIELD_NUMBER') or
-              short_name in ['__slots__', 'DESCRIPTOR']):
-            continue
-
-        # TODO(wicke): We may want to also remember the object itself.
-        self._add_other_member(short_name, child_name, child, child_doc)
-
-
-class _ModulePageInfo(object):
-  """Collects docs for a module page."""
-
-  def __init__(self, full_name):
-    self._full_name = full_name
-    self._defined_in = None
-    self._aliases = None
-    self._doc = None
-
-    self._modules = []
-    self._classes = []
-    self._functions = []
-    self._other_members = []
-
-  def for_function(self):
-    return False
-
-  def for_class(self):
-    return False
-
-  def for_module(self):
-    return True
-
-  @property
-  def full_name(self):
-    return self._full_name
-
-  @property
-  def short_name(self):
-    return self._full_name.split('.')[-1]
-
-  @property
-  def defined_in(self):
-    return self._defined_in
-
-  def set_defined_in(self, defined_in):
-    assert self.defined_in is None
-    self._defined_in = defined_in
-
-  @property
-  def aliases(self):
-    return self._aliases
-
-  def set_aliases(self, aliases):
-    assert self.aliases is None
-    self._aliases = aliases
-
-  @property
-  def doc(self):
-    return self._doc
-
-  def set_doc(self, doc):
-    assert self.doc is None
-    self._doc = doc
-
-  @property
-  def modules(self):
-    return self._modules
-
-  def _add_module(self, short_name, full_name, obj, doc, url):
-    self._modules.append(_LinkInfo(short_name, full_name, obj, doc, url))
-
-  @property
-  def classes(self):
-    return self._classes
-
-  def _add_class(self, short_name, full_name, obj, doc, url):
-    self._classes.append(_LinkInfo(short_name, full_name, obj, doc, url))
-
-  @property
-  def functions(self):
-    return self._functions
-
-  def _add_function(self, short_name, full_name, obj, doc, url):
-    self._functions.append(_LinkInfo(short_name, full_name, obj, doc, url))
-
-  @property
-  def other_members(self):
-    return self._other_members
-
-  def _add_other_member(self, short_name, full_name, obj, doc):
-    self._other_members.append(
-        _OtherMemberInfo(short_name, full_name, obj, doc))
-
-  def get_metadata_html(self):
-    meta_data = Metadata(self.full_name)
-
-    # Objects with their own pages are not added to the metadata list for the
-    # module, the module only has a link to the object page. No docs.
-    for item in self.other_members:
-      meta_data.append(item)
-
-    return meta_data.build_html()
-
-  def collect_docs_for_module(self, parser_config):
-    """Collect information necessary specifically for a module's doc page.
-
-    Mainly this is information about the members of the module.
-
-    Args:
-      parser_config: An instance of ParserConfig.
-    """
-    relative_path = os.path.relpath(
-        path='.',
-        start=os.path.dirname(documentation_path(self.full_name)) or '.')
-
-    member_names = parser_config.tree.get(self.full_name, [])
-    for name in member_names:
-
-      if name in ['__builtins__', '__doc__', '__file__',
-                  '__name__', '__path__', '__package__',
-                  '__cached__', '__loader__', '__spec__', 'absolute_import',
-                  'division', 'print_function', 'unicode_literals']:
-        continue
-
-      member_full_name = self.full_name + '.' + name if self.full_name else name
-      member = parser_config.py_name_to_object(member_full_name)
-
-      member_doc = _parse_md_docstring(member, relative_path, self.full_name,
-                                       parser_config.reference_resolver)
-
-      url = parser_config.reference_resolver.reference_to_url(
-          member_full_name, relative_path)
-
-      if tf_inspect.ismodule(member):
-        self._add_module(name, member_full_name, member, member_doc, url)
-
-      elif tf_inspect.isclass(member):
-        self._add_class(name, member_full_name, member, member_doc, url)
-
-      elif tf_inspect.isfunction(member):
-        self._add_function(name, member_full_name, member, member_doc, url)
-
-      else:
-        self._add_other_member(name, member_full_name, member, member_doc)
-
-
-class ParserConfig(object):
-  """Stores all indexes required to parse the docs."""
-
-  def __init__(self, reference_resolver, duplicates, duplicate_of, tree, index,
-               reverse_index, base_dir, code_url_prefix):
-    """Object with the common config for docs_for_object() calls.
-
-    Args:
-      reference_resolver: An instance of ReferenceResolver.
-      duplicates: A `dict` mapping fully qualified names to a set of all
-        aliases of this name. This is used to automatically generate a list of
-        all aliases for each name.
-      duplicate_of: A map from duplicate names to preferred names of API
-        symbols.
-      tree: A `dict` mapping a fully qualified name to the names of all its
-        members. Used to populate the members section of a class or module page.
-      index: A `dict` mapping full names to objects.
-      reverse_index: A `dict` mapping object ids to full names.
-      base_dir: A base path that is stripped from file locations written to the
-        docs.
-      code_url_prefix: A Url to pre-pend to the links to file locations.
-    """
-    self.reference_resolver = reference_resolver
-    self.duplicates = duplicates
-    self.duplicate_of = duplicate_of
-    self.tree = tree
-    self.reverse_index = reverse_index
-    self.index = index
-    self.base_dir = base_dir
-    self.code_url_prefix = code_url_prefix
-
-  def py_name_to_object(self, full_name):
-    """Return the Python object for a Python symbol name."""
-    return self.index[full_name]
-
-
-def docs_for_object(full_name, py_object, parser_config):
-  """Return a PageInfo object describing a given object from the TF API.
-
-  This function uses _parse_md_docstring to parse the docs pertaining to
-  `object`.
-
-  This function resolves '`tf.symbol`' references in the docstrings into links
-  to the appropriate location. It also adds a list of alternative names for the
-  symbol automatically.
-
-  It assumes that the docs for each object live in a file given by
-  `documentation_path`, and that relative links to files within the
-  documentation are resolvable.
-
-  Args:
-    full_name: The fully qualified name of the symbol to be
-      documented.
-    py_object: The Python object to be documented. Its documentation is sourced
-      from `py_object`'s docstring.
-    parser_config: A ParserConfig object.
-
-  Returns:
-    Either a `_FunctionPageInfo`, `_ClassPageInfo`, or a `_ModulePageInfo`
-    depending on the type of the python object being documented.
-
-  Raises:
-    RuntimeError: If an object is encountered for which we don't know how
-      to make docs.
-  """
-
-  # Which other aliases exist for the object referenced by full_name?
-  master_name = parser_config.reference_resolver.py_master_name(full_name)
-  duplicate_names = parser_config.duplicates.get(master_name, [])
-  if master_name in duplicate_names:
-    duplicate_names.remove(master_name)
-
-  # TODO(wicke): Once other pieces are ready, enable this also for partials.
-  if (tf_inspect.ismethod(py_object) or tf_inspect.isfunction(py_object) or
-      # Some methods in classes from extensions come in as routines.
-      tf_inspect.isroutine(py_object)):
-    page_info = _FunctionPageInfo(master_name)
-    page_info.set_signature(py_object, parser_config.reverse_index)
-
-  elif tf_inspect.isclass(py_object):
-    page_info = _ClassPageInfo(master_name)
-    page_info.collect_docs_for_class(py_object, parser_config)
-
-  elif tf_inspect.ismodule(py_object):
-    page_info = _ModulePageInfo(master_name)
-    page_info.collect_docs_for_module(parser_config)
-
-  else:
-    raise RuntimeError('Cannot make docs for object %s: %r' % (full_name,
-                                                               py_object))
-
-  relative_path = os.path.relpath(
-      path='.', start=os.path.dirname(documentation_path(full_name)) or '.')
-
-  page_info.set_doc(
-      _parse_md_docstring(py_object, relative_path, full_name,
-                          parser_config.reference_resolver))
-
-  page_info.set_aliases(duplicate_names)
-
-  page_info.set_defined_in(_get_defined_in(py_object, parser_config))
-
-  return page_info
+def _unwrap_obj(obj):
+  while True:
+    unwrapped_obj = getattr(obj, '__wrapped__', None)
+    if unwrapped_obj is None:
+      break
+    obj = unwrapped_obj
+  return obj
 
 
-class _FileLocation(object):
+@dataclasses.dataclass
+class FileLocation(object):
   """This class indicates that the object is defined in a regular file.
 
   This can be used for the `defined_in` slot of the `PageInfo` objects.
   """
-  GITHUB_LINE_NUMBER_TEMPLATE = '#L{start_line:d}-L{end_line:d}'
-
-  def __init__(self, rel_path, url=None, start_line=None, end_line=None):
-    self.rel_path = rel_path
-    self.url = url
-    self.start_line = start_line
-    self.end_line = end_line
-
-    github_master_re = 'github.com.*?(blob|tree)/master'
-    suffix = ''
-    # Only attach a line number for github URLs that are not using "master"
-    if self.start_line and not re.search(github_master_re, self.url):
-      if 'github.com' in self.url:
-        suffix = self.GITHUB_LINE_NUMBER_TEMPLATE.format(
-            start_line=self.start_line, end_line=self.end_line)
 
-        self.url = self.url + suffix
+  base_url: Optional[str] = None
+  start_line: Optional[int] = None
+  end_line: Optional[int] = None
 
-
-def _get_defined_in(py_object, parser_config):
+  @property
+  def url(self) -> Optional[str]:
+    if self.start_line and self.end_line:
+      if self.base_url and self.base_url.startswith('https://github.com/'):
+        return f'{self.base_url}#L{self.start_line}-L{self.end_line}'
+    return self.base_url
+
+def get_defined_in(
+    py_object: Any,
+    parser_config: config.ParserConfig) -> Optional[FileLocation]:
   """Returns a description of where the passed in python object was defined.
 
   Args:
     py_object: The Python object.
-    parser_config: A ParserConfig object.
+    parser_config: A config.ParserConfig object.
 
   Returns:
-    A `_FileLocation`
+    A `FileLocation`
   """
   # Every page gets a note about where this object is defined
   base_dirs_and_prefixes = zip(parser_config.base_dir,
                                parser_config.code_url_prefix)
+  try:
+    obj_path = pathlib.Path(inspect.getfile(_unwrap_obj(py_object)))
+  except TypeError:  # getfile throws TypeError if py_object is a builtin.
+    return None
+
+  if obj_path.suffix not in ('.py', '.pyc'):
+    return None
+
   code_url_prefix = None
   for base_dir, temp_prefix in base_dirs_and_prefixes:
     try:
-      obj_path = tf_inspect.getfile(py_object)
-    except TypeError:  # getfile throws TypeError if py_object is a builtin.
+      rel_path = obj_path.relative_to(base_dir)
+    except ValueError:
       continue
 
-    rel_path = os.path.relpath(
-        path=obj_path, start=base_dir)
-    # A leading ".." indicates that the file is not inside `base_dir`, and
-    # the search should continue.
-    if rel_path.startswith('..'):
-      continue
-    else:
-      code_url_prefix = temp_prefix
-      break
-
-  try:
-    lines, start_line = tf_inspect.getsourcelines(py_object)
-    end_line = start_line + len(lines) - 1
-  except IOError:
-    # The source is not available.
-    start_line = None
-    end_line = None
-  except TypeError:
-    # This is a builtin, with no python-source.
-    start_line = None
-    end_line = None
-  except IndexError:
-    start_line = None
-    end_line = None
+    code_url_prefix = temp_prefix
+    # rel_path is currently a platform-specific path, so we need to convert
+    # it to a posix path (for lack of a URL path).
+    posix_rel_path_str = str(pathlib.PurePosixPath(rel_path))
+    break
 
   # No link if the file was not found in a `base_dir`, or the prefix is None.
   if code_url_prefix is None:
     return None
 
-  # TODO(wicke): If this is a generated file, link to the source instead.
-  # TODO(wicke): Move all generated files to a generated/ directory.
-  # TODO(wicke): And make their source file predictable from the file name.
+  lines, start_line = get_source.get_source_lines(py_object)
+  if start_line is None:
+    end_line = None
+  else:
+    end_line = start_line + len(lines) - 1
+    if 'GENERATED' in lines[0]:
+      # don't link to files generated by tf_export
+      return None
 
   # In case this is compiled, point to the original
-  if rel_path.endswith('.pyc'):
-    rel_path = rel_path[:-1]
-
-  # Never include links outside this code base.
-  if re.search(r'\b_api\b', rel_path):
-    return None
-  if re.search(r'\bapi/(_v2|_v1)\b', rel_path):
-    return None
-  if re.search(r'<[\w\s]+>', rel_path):
+  if posix_rel_path_str.endswith('.pyc'):
+    # If a PY3 __pycache__/ subdir is being used, omit it.
+    posix_rel_path_str = posix_rel_path_str.replace('__pycache__/', '')
+    # Strip everything after the first . so that variants such as .pyc and
+    # .cpython-3x.pyc or similar are all handled.
+    posix_rel_path_str = posix_rel_path_str.partition('.')[0] + '.py'
+
+  if re.search(r'<[\w\s]+>', posix_rel_path_str):
     # Built-ins emit paths like , , etc.
     return None
+  if '\n')
 
   # Sort all the symbols once, so that the ordering is preserved when its broken
-  # up into master symbols and compat symbols and sorting the sublists is not
+  # up into main symbols and compat symbols and sorting the sublists is not
   # required.
   symbol_links = sorted(symbol_links, key=lambda x: x[0])
 
@@ -1714,7 +815,8 @@ def generate_global_index(library_name, index, reference_resolver):
 
   for symbol, link in symbol_links:
     if symbol.startswith('tf.compat.v1'):
-      compat_v1_symbol_links.append(link)
+      if 'raw_ops' not in symbol:
+        compat_v1_symbol_links.append(link)
     elif symbol.startswith('tf.compat.v2'):
       compat_v2_symbol_links.append(link)
     else:
@@ -1722,17 +824,17 @@ def generate_global_index(library_name, index, reference_resolver):
 
   lines.append('## Primary symbols')
   for link in primary_symbol_links:
-    lines.append('*  %s' % link)
+    lines.append(f'*  {link}')
 
   if compat_v2_symbol_links:
     lines.append('\n## Compat v2 symbols\n')
     for link in compat_v2_symbol_links:
-      lines.append('*  %s' % link)
+      lines.append(f'*  {link}')
 
   if compat_v1_symbol_links:
     lines.append('\n## Compat v1 symbols\n')
     for link in compat_v1_symbol_links:
-      lines.append('*  %s' % link)
+      lines.append(f'*  {link}')
 
   # TODO(markdaoust): use a _ModulePageInfo -> prety_docs.build_md_page()
   return '\n'.join(lines)
@@ -1775,13 +877,14 @@ def append(self, item):
 
   def build_html(self):
     """Returns the Metadata block as an Html string."""
+    # Note: A schema is not a URL. It is defined with http: but doesn't resolve.
     schema = 'http://developers.google.com/ReferenceObject'
-    parts = ['
    ' % schema]
+    parts = [f'
    ']
 
-    parts.append('' % self.name)
-    parts.append('' % self.version)
+    parts.append(f'')
+    parts.append(f'')
     for item in self._content:
-      parts.append('' % item)
+      parts.append(f'')
 
     parts.extend(['
    ', ''])
 
diff --git a/tools/tensorflow_docs/api_generator/parser_test.py b/tools/tensorflow_docs/api_generator/parser_test.py
index 3f1b6c6196e..0bfffeded92 100644
--- a/tools/tensorflow_docs/api_generator/parser_test.py
+++ b/tools/tensorflow_docs/api_generator/parser_test.py
@@ -14,24 +14,26 @@
 # ==============================================================================
 """Tests for documentation parser."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import collections
-import functools
-import os
-import tempfile
+from collections import abc
+import dataclasses
+import inspect
 import textwrap
-import unittest
+import types
+
+from typing import List, Union
 
 from absl.testing import absltest
 from absl.testing import parameterized
-import six
+import attr
 
+from tensorflow_docs.api_generator import config
 from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import doc_generator_visitor
+from tensorflow_docs.api_generator import generate_lib
 from tensorflow_docs.api_generator import parser
-from tensorflow_docs.api_generator import tf_inspect
+from tensorflow_docs.api_generator import reference_resolver as reference_resolver_lib
+from tensorflow_docs.api_generator.pretty_docs import docs_for_object
 
 # The test needs a real module. `types.ModuleType()` doesn't work, as the result
 # is a `builtin` module. Using "parser" here is arbitraty. The tests don't
@@ -58,7 +60,11 @@ def hidden_method(self):
 
 
 class TestClass(ParentClass):
-  """Docstring for TestClass itself."""
+  """Docstring for TestClass itself.
+
+  Attributes:
+    hello: hello
+  """
 
   def a_method(self, arg='default'):
     """Docstring for a method."""
@@ -80,18 +86,19 @@ def a_property(self):
     """Docstring for a property."""
     pass
 
-  CLASS_MEMBER = 'a class member'
-
+  @staticmethod
+  def static_method(arg):
+    pass
 
-class DummyVisitor(object):
+  @classmethod
+  def class_method(cls):
+    pass
 
-  def __init__(self, index, duplicate_of):
-    self.index = index
-    self.duplicate_of = duplicate_of
+  CLASS_MEMBER = 'a class member'
 
 
-class ConcreteMutableMapping(collections.MutableMapping):
-  """MutableMapping subclass to repro tf_inspect.getsource() IndexError."""
+class ConcreteMutableMapping(abc.MutableMapping):
+  """MutableMapping subclass to repro getsource() IndexError."""
 
   def __init__(self):
     self._map = {}
@@ -115,6 +122,25 @@ def __len__(self):
 ConcreteNamedTuple = collections.namedtuple('ConcreteNamedTuple', ['a', 'b'])
 
 
+@attr.s
+class ClassUsingAttrs(object):
+  member = attr.ib(type=int)
+
+
+@dataclasses.dataclass
+class ExampleDataclass:
+  x: List[str]
+  z: int
+  c: List[int] = dataclasses.field(default_factory=list)
+  a: Union[List[str], str, int] = None
+  b: str = 'test'
+  y: bool = False
+
+  def add(self, x: int, y: int) -> int:
+    q: int = x + y
+    return q
+
+
 class ParserTest(parameterized.TestCase):
 
   def test_documentation_path(self):
@@ -122,131 +148,146 @@ def test_documentation_path(self):
     self.assertEqual('test/module.md', parser.documentation_path('test.module'))
 
   def test_replace_references(self):
+
     class HasOneMember(object):
 
       def foo(self):
         pass
 
-    string = (
-        'A `tf.reference`, a member `tf.reference.foo`, and a `tf.third`. '
-        'This is `not a symbol`, and this is `tf.not.a.real.symbol`')
+    class Other:
+      pass
 
-    duplicate_of = {'tf.third': 'tf.fourth'}
-    index = {'tf.reference': HasOneMember,
-             'tf.reference.foo': HasOneMember.foo,
-             'tf.third': HasOneMember,
-             'tf.fourth': HasOneMember}
+    tf = types.ModuleType('tf')
+    tf.__file__ = __file__
+    tf.reference = HasOneMember
+    tf.third = Other
+    tf.fourth = Other
 
-    visitor = DummyVisitor(index, duplicate_of)
+    string = ('A `@tf.reference`, a member `tf.reference.foo`, and a '
+              '`tf.third(what)`. '
+              'This is `not a symbol`, and this is `tf.not.a.real.symbol`')
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('tf', tf)],
+        code_url_prefix='https://tensorflow.org')
 
-    result = reference_resolver.replace_references(string, '../..')
-    self.assertEqual('A '
-                     'tf.reference, '
-                     'a member '
-                     'tf.reference.foo, '
-                     'and a '
-                     'tf.third. '
-                     'This is `not a symbol`, and this is '
-                     '`tf.not.a.real.symbol`',
-                     result)
+    parser_config = generator.run_extraction()
 
-  def test_docs_for_class(self):
+    result = (
+        parser_config.reference_resolver.with_prefix(
+            '../..').replace_references(string))
 
-    index = {
-        'TestClass': TestClass,
-        'TestClass.a_method': TestClass.a_method,
-        'TestClass.a_property': TestClass.a_property,
-        'TestClass.ChildClass': TestClass.ChildClass,
-        'TestClass.CLASS_MEMBER': TestClass.CLASS_MEMBER
-    }
+    self.assertEqual(
+        'A '
+        '@tf.reference, '
+        'a member '
+        'tf.reference.foo, '
+        'and a '
+        'tf.third(what). '
+        'This is `not a symbol`, and this is '
+        '`tf.not.a.real.symbol`', result)
 
-    visitor = DummyVisitor(index=index, duplicate_of={})
+  def test_docs_for_class(self):
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.TestClass = TestClass
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
 
-    tree = {
-        'TestClass': ['a_method', 'a_property', 'ChildClass', 'CLASS_MEMBER']
-    }
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='TestClass', py_object=TestClass, parser_config=parser_config)
+    parser_config = generator.run_extraction()
+
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=(
+            'm',
+            'TestClass',
+        ), py_object=TestClass)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
 
     # Make sure the brief docstring is present
     self.assertEqual(
-        tf_inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief)
+        inspect.getdoc(TestClass).split('\n')[0], page_info.doc.brief)
 
     # Make sure the method is present
-    self.assertEqual(TestClass.a_method, page_info.methods[0].obj)
+    method_infos = {
+        method_info.short_name: method_info for method_info in page_info.methods
+    }
+
+    self.assertIs(method_infos['a_method'].py_object, TestClass.a_method)
 
     # Make sure that the signature is extracted properly and omits self.
-    self.assertEqual(["arg='default'"], page_info.methods[0].signature)
+    self.assertEqual('(\n    arg='default'\n)',
+                     str(method_infos['a_method'].signature))
+
+    self.assertEqual(method_infos['static_method'].decorators, ['staticmethod'])
+    self.assertEqual(method_infos['class_method'].decorators, ['classmethod'])
 
     # Make sure the property is present
-    self.assertIs(TestClass.a_property, page_info.properties[0].obj)
+    attrs = page_info.attr_block
+    self.assertIsInstance(attrs, parser.TitleBlock)
+    self.assertIn('a_property', [name for name, desc in attrs.items])
 
     # Make sure there is a link to the child class and it points the right way.
-    self.assertIs(TestClass.ChildClass, page_info.classes[0].obj)
+    self.assertIs(TestClass.ChildClass, page_info.classes[0].py_object)
 
-    # Make sure this file is contained as the definition location.
-    self.assertEqual(
-        os.path.relpath(__file__, '/'), page_info.defined_in.rel_path)
-
-  def test_namedtuple_field_order(self):
-    namedtupleclass = collections.namedtuple('namedtupleclass',
-                                             {'z', 'y', 'x', 'w', 'v', 'u'})
-
-    index = {
-        'namedtupleclass': namedtupleclass,
-        'namedtupleclass.u': namedtupleclass.u,
-        'namedtupleclass.v': namedtupleclass.v,
-        'namedtupleclass.w': namedtupleclass.w,
-        'namedtupleclass.x': namedtupleclass.x,
-        'namedtupleclass.y': namedtupleclass.y,
-        'namedtupleclass.z': namedtupleclass.z,
-    }
+  def test_dataclass_attributes_table(self):
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.ExampleDataclass = ExampleDataclass
 
-    visitor = DummyVisitor(index=index, duplicate_of={})
-
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
-
-    tree = {'namedtupleclass': {'u', 'v', 'w', 'x', 'y', 'z'}}
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='namedtupleclass',
-        py_object=namedtupleclass,
-        parser_config=parser_config)
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
+
+    parser_config = generator.run_extraction()
+
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('m', 'ExampleDataclass'), py_object=ExampleDataclass)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
+
+    self.assertCountEqual(['a', 'b', 'c', 'x', 'y', 'z'],
+                          [name for name, value in page_info.attr_block.items])
+
+  def test_namedtuple_field_order_respects_hidden(self):
+    namedtupleclass = collections.namedtuple(
+        'namedtupleclass', ['z', 'y', 'x', 'hidden', 'w', 'v', 'u'])
+
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.namedtupleclass = namedtupleclass
+
+    def hide(path, parent, children):
+      return [(name, value) for name, value in children if name != 'hidden']
+
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org',
+        callbacks=[hide])
+
+    parser_config = generator.run_extraction()
+
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('m', 'namedtupleclass'), py_object=namedtupleclass)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
+
+    self.assertIsNone(page_info._namedtuplefields['hidden'])
 
     # Each namedtiple field has a docstring of the form:
     #   'Alias for field number ##'. These props are returned sorted.
+    def field_number(desc):
+      return int(desc.split(' ')[-1])
 
-    def sort_key(prop_info):
-      return int(prop_info.obj.__doc__.split(' ')[-1])
-
-    self.assertSequenceEqual(page_info.properties,
-                             sorted(page_info.properties, key=sort_key))
+    self.assertSequenceEqual(
+        [0, 1, 2, 4, 5, 6],
+        [field_number(desc) for name, desc in page_info.attr_block.items])
 
   def test_docs_for_class_should_skip(self):
 
@@ -261,32 +302,24 @@ class Child(Parent):
       def a_method(self, arg='default'):
         pass
 
-    index = {
-        'Child': Child,
-        'Child.a_method': Child.a_method,
-    }
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.Child = Child
 
-    visitor = DummyVisitor(index=index, duplicate_of={})
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    parser_config = generator.run_extraction()
 
-    tree = {
-        'Child': ['a_method'],
-    }
-
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='Child', py_object=Child, parser_config=parser_config)
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=(
+            'm',
+            'Child',
+        ), py_object=Child)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
 
     # Make sure the `a_method` is not present
     self.assertEmpty(page_info.methods)
@@ -313,168 +346,106 @@ class ChildMessage(CMessage, Message, MessageMeta):
       def my_method(self):
         pass
 
-    index = {
-        'ChildMessage': ChildMessage,
-        'ChildMessage.hidden': ChildMessage.hidden,
-        'ChildMessage.hidden2': ChildMessage.hidden2,
-        'ChildMessage.hidden3': ChildMessage.hidden3,
-        'ChildMessage.my_method': ChildMessage.my_method,
-    }
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.ChildMessage = ChildMessage
 
-    visitor = DummyVisitor(index=index, duplicate_of={})
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    parser_config = generator.run_extraction()
 
-    tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']}
-
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='ChildMessage',
-        py_object=ChildMessage,
-        parser_config=parser_config)
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('m', 'ChildMessage'), py_object=ChildMessage)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
 
     self.assertLen(page_info.methods, 1)
     self.assertEqual('my_method', page_info.methods[0].short_name)
 
   def test_docs_for_module(self):
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.test_function = test_function
+    m.test_function_with_args_kwargs = test_function_with_args_kwargs
+    m.TestClass = TestClass
 
-    index = {
-        'TestModule':
-            test_module,
-        'TestModule.test_function':
-            test_function,
-        'TestModule.test_function_with_args_kwargs':
-            test_function_with_args_kwargs,
-        'TestModule.TestClass':
-            TestClass,
-    }
-
-    visitor = DummyVisitor(index=index, duplicate_of={})
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    parser_config = generator.run_extraction()
 
-    tree = {
-        'TestModule': ['TestClass', 'test_function',
-                       'test_function_with_args_kwargs']
-    }
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='TestModule',
-        py_object=test_module,
-        parser_config=parser_config)
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('m',), py_object=test_module)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
 
     # Make sure the brief docstring is present
     self.assertEqual(
-        tf_inspect.getdoc(test_module).split('\n')[0], page_info.doc.brief)
+        inspect.getdoc(test_module).split('\n')[0], page_info.doc.brief)
 
     # Make sure that the members are there
-    funcs = {f_info.obj for f_info in page_info.functions}
+    funcs = {f_info.py_object for f_info in page_info.functions}
     self.assertEqual({test_function, test_function_with_args_kwargs}, funcs)
 
-    classes = {cls_info.obj for cls_info in page_info.classes}
+    classes = {cls_info.py_object for cls_info in page_info.classes}
     self.assertEqual({TestClass}, classes)
 
-    # Make sure the module's file is contained as the definition location.
-    self.assertEqual(
-        os.path.relpath(test_module.__file__.rstrip('c'), '/'),
-        page_info.defined_in.rel_path)
-
   def test_docs_for_function(self):
-    index = {
-        'test_function': test_function
-    }
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.test_function = test_function
 
-    visitor = DummyVisitor(index=index, duplicate_of={})
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    parser_config = generator.run_extraction()
 
-    tree = {
-        '': ['test_function']
-    }
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='test_function',
-        py_object=test_function,
-        parser_config=parser_config)
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('test_function',), py_object=test_function)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
 
     # Make sure the brief docstring is present
     self.assertEqual(
-        tf_inspect.getdoc(test_function).split('\n')[0], page_info.doc.brief)
+        inspect.getdoc(test_function).split('\n')[0], page_info.doc.brief)
 
     # Make sure the extracted signature is good.
-    self.assertEqual(['unused_arg', "unused_kwarg='default'"],
-                     page_info.signature)
-
-    # Make sure this file is contained as the definition location.
-    self.assertEqual(
-        os.path.relpath(__file__, '/'), page_info.defined_in.rel_path)
+    self.assertEqual('(\n    unused_arg, unused_kwarg='default'\n)',
+                     str(page_info.signature))
 
   def test_docs_for_function_with_kwargs(self):
-    index = {
-        'test_function_with_args_kwargs': test_function_with_args_kwargs
-    }
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.test_function_with_args_kwargs = test_function_with_args_kwargs
 
-    visitor = DummyVisitor(index=index, duplicate_of={})
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    parser_config = generator.run_extraction()
 
-    tree = {
-        '': ['test_function_with_args_kwargs']
-    }
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='test_function_with_args_kwargs',
-        py_object=test_function_with_args_kwargs,
-        parser_config=parser_config)
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('test_function_with_args_kwargs',),
+        py_object=test_function_with_args_kwargs)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
 
     # Make sure the brief docstring is present
     self.assertEqual(
-        tf_inspect.getdoc(test_function_with_args_kwargs).split('\n')[0],
+        inspect.getdoc(test_function_with_args_kwargs).split('\n')[0],
         page_info.doc.brief)
 
     # Make sure the extracted signature is good.
-    self.assertEqual(['unused_arg', '*unused_args', '**unused_kwargs'],
-                     page_info.signature)
+    self.assertEqual('(\n    unused_arg, *unused_args, **unused_kwargs\n)',
+                     str(page_info.signature))
 
   def test_parse_md_docstring(self):
 
@@ -498,9 +469,11 @@ def test_function_with_fancy_docstring(arg):
       NumPy has nothing as awesome as this function.
       @end_compatibility
 
-      @compatibility(theano)
+      @compatibility(two words!)
       Theano has nothing as awesome as this function.
 
+      @tf.function
+
       Check it out.
       @end_compatibility
 
@@ -512,26 +485,31 @@ class HasOneMember(object):
       def foo(self):
         pass
 
-    duplicate_of = {'tf.third': 'tf.fourth'}
-    index = {
-        'tf': test_module,
-        'tf.fancy': test_function_with_fancy_docstring,
-        'tf.reference': HasOneMember,
-        'tf.reference.foo': HasOneMember.foo,
-        'tf.third': HasOneMember,
-        'tf.fourth': HasOneMember
-    }
+    class HasOneMember2(object):
+
+      def foo(self):
+        pass
 
-    visitor = DummyVisitor(index=index, duplicate_of=duplicate_of)
+    tf = types.ModuleType('tf')
+    tf.__file__ = __file__
+    tf.fancy = test_function_with_fancy_docstring
+    tf.reference = HasOneMember
+    tf.third = HasOneMember2
+    tf.fourth = HasOneMember2
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('tf', tf)],
+        code_url_prefix='https://tensorflow.org')
 
-    doc_info = parser._parse_md_docstring(
+    parser_config = generator.run_extraction()
+    parser_config.reference_resolver = (
+        parser_config.reference_resolver.with_prefix('/'))
+
+    doc_info = parser.parse_md_docstring(
         test_function_with_fancy_docstring,
-        relative_path_to_root='../..',
         full_name=None,
-        reference_resolver=reference_resolver)
+        parser_config=parser_config)
 
     freeform_docstring = '\n'.join(
         part for part in doc_info.docstring_parts if isinstance(part, str))
@@ -545,233 +523,100 @@ def foo(self):
 
     self.assertLen(title_blocks, 3)
 
-    self.assertCountEqual(doc_info.compatibility.keys(), {'numpy', 'theano'})
+    self.assertCountEqual(doc_info.compatibility.keys(),
+                          {'numpy', 'two words!'})
 
-    self.assertEqual(doc_info.compatibility['numpy'],
-                     'NumPy has nothing as awesome as this function.\n')
+    self.assertEqual(
+        doc_info.compatibility['numpy'],
+        'NumPy has nothing as awesome as this function.',
+    )
 
-  def test_generate_index(self):
+  def test_downgrade_h1_docstrings(self):
+    h1_docstring = textwrap.dedent("""\
+      Hello.
 
-    index = {
-        'tf': test_module,
-        'tf.TestModule': test_module,
-        'tf.test_function': test_function,
-        'tf.TestModule.test_function': test_function,
-        'tf.TestModule.TestClass': TestClass,
-        'tf.TestModule.TestClass.a_method': TestClass.a_method,
-        'tf.TestModule.TestClass.a_property': TestClass.a_property,
-        'tf.TestModule.TestClass.ChildClass': TestClass.ChildClass,
-    }
-    duplicate_of = {'tf.TestModule.test_function': 'tf.test_function'}
+      Some keras functions have docstrings like this.
+
+      # Arguments
+        a: a
+        b: b
+        c: c
+
+      # Example
 
-    visitor = DummyVisitor(index=index, duplicate_of=duplicate_of)
+        ```
+        # comment
+        ```
+
+      # Returns
+        a+b+c
+
+      # Raises
+        ValueError: always
+      """)
+    downgrader = parser._DowngradeH1Keywords()
+    doc = downgrader(h1_docstring)
+    self.assertIn('\n  ```\n  # comment\n  ```', doc)
+    self.assertIn('\nArguments:', doc)
+    self.assertIn('\nExample:', doc)
+    self.assertIn('\nReturns:', doc)
+    self.assertIn('\nRaises:', doc)
+
+  def test_generate_index(self):
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.TestClass = TestClass
+    m.test_function = test_function
+    m.submodule = types.ModuleType('submodule')
+    m.submodule.test_function = test_function
 
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
 
-    docs = parser.generate_global_index('TestLibrary', index=index,
-                                        reference_resolver=reference_resolver)
+    parser_config = generator.run_extraction()
+
+    docs = parser.generate_global_index(
+        'TestLibrary',
+        index=parser_config.index,
+        reference_resolver=parser_config.reference_resolver)
 
     # Make sure duplicates and non-top-level symbols are in the index, but
     # methods and properties are not.
     self.assertNotIn('a_method', docs)
     self.assertNotIn('a_property', docs)
-    self.assertIn('TestModule.TestClass', docs)
-    self.assertIn('TestModule.TestClass.ChildClass', docs)
-    self.assertIn('TestModule.test_function', docs)
-    # Leading backtick to make sure it's included top-level.
-    # This depends on formatting, but should be stable.
-    self.assertIn('tf.test_function', docs)
-
-  def test_argspec_for_functools_partial(self):
-    # pylint: disable=unused-argument
-    def test_function_for_partial1(arg1, arg2, kwarg1=1, kwarg2=2):
-      pass
-    # pylint: enable=unused-argument
-
-    # pylint: disable=protected-access
-    # Make sure everything works for regular functions.
-    expected = tf_inspect.FullArgSpec(
-        args=['arg1', 'arg2', 'kwarg1', 'kwarg2'],
-        varargs=None,
-        varkw=None,
-        defaults=(1, 2),
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    self.assertEqual(expected,
-                     tf_inspect.getfullargspec(test_function_for_partial1))
-
-    # Make sure doing nothing works.
-    expected = tf_inspect.FullArgSpec(
-        args=['arg1', 'arg2', 'kwarg1', 'kwarg2'],
-        varargs=None,
-        varkw=None,
-        defaults=(1, 2),
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    partial = functools.partial(test_function_for_partial1)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
-
-    # Make sure setting args from the front works.
-    expected = tf_inspect.FullArgSpec(
-        args=['arg2', 'kwarg1', 'kwarg2'],
-        varargs=None,
-        varkw=None,
-        defaults=(1, 2),
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    partial = functools.partial(test_function_for_partial1, 1)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
-
-    expected = tf_inspect.FullArgSpec(
-        args=['kwarg2'],
-        varargs=None,
-        varkw=None,
-        defaults=(2,),
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    partial = functools.partial(test_function_for_partial1, 1, 2, 3)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
-
-    # Make sure setting kwargs works.
-    expected = tf_inspect.FullArgSpec(
-        args=['arg1', 'arg2'],
-        varargs=None,
-        varkw=None,
-        defaults=None,
-        kwonlyargs=['kwarg1', 'kwarg2'],
-        kwonlydefaults={
-            'kwarg1': 0,
-            'kwarg2': 2
-        },
-        annotations={})
-    partial = functools.partial(test_function_for_partial1, kwarg1=0)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
-
-    expected = tf_inspect.FullArgSpec(
-        args=['arg1', 'arg2', 'kwarg1'],
-        varargs=None,
-        varkw=None,
-        defaults=(1,),
-        kwonlyargs=['kwarg2'],
-        kwonlydefaults={'kwarg2': 0},
-        annotations={})
-    partial = functools.partial(test_function_for_partial1, kwarg2=0)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
-
-    expected = tf_inspect.FullArgSpec(
-        args=['arg1'],
-        varargs=None,
-        varkw=None,
-        defaults=None,
-        kwonlyargs=['arg2', 'kwarg1', 'kwarg2'],
-        kwonlydefaults={
-            'arg2': 0,
-            'kwarg1': 0,
-            'kwarg2': 0
-        },
-        annotations={})
-    partial = functools.partial(test_function_for_partial1,
-                                arg2=0, kwarg1=0, kwarg2=0)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
-
-  def test_argspec_for_functools_partial_starargs(self):
-    # pylint: disable=unused-argument
-    def test_function_for_partial2(arg1, arg2, *my_args, **my_kwargs):
-      pass
-    # pylint: enable=unused-argument
-    # Make sure *args, *kwargs is accounted for.
-    expected = tf_inspect.FullArgSpec(
-        args=[],
-        varargs='my_args',
-        varkw='my_kwargs',
-        defaults=None,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    partial = functools.partial(test_function_for_partial2, 0, 1)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
-
-    # Make sure *args, *kwargs is accounted for.
-    expected = tf_inspect.FullArgSpec(
-        args=[],
-        varargs='my_args',
-        varkw='my_kwargs',
-        defaults=None,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    partial = functools.partial(test_function_for_partial2, 0, 1, 2, 3, 4, 5)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
-
-    # Make sure *args, *kwargs is accounted for.
-    expected = tf_inspect.FullArgSpec(
-        args=['arg1', 'arg2'],
-        varargs='my_args',
-        varkw='my_kwargs',
-        defaults=None,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    partial = functools.partial(test_function_for_partial2, a=1, b=2, c=3)
-    self.assertEqual(expected, tf_inspect.getfullargspec(partial))
+    self.assertIn('m.TestClass', docs)
+    self.assertIn('m.TestClass.ChildClass', docs)
+    self.assertIn('m.submodule.test_function', docs)
+    self.assertIn('m.submodule.test_function', docs)
 
   def test_getsource_indexerror_resilience(self):
     """Validates that parser gracefully handles IndexErrors.
 
-    tf_inspect.getsource() can raise an IndexError in some cases. It's unclear
+    getsource() can raise an IndexError in some cases. It's unclear
     why this happens, but it consistently repros on the `get` method of
     collections.MutableMapping subclasses.
     """
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.ConcreteMutableMapping = ConcreteMutableMapping
 
-    # This isn't the full set of APIs from MutableMapping, but sufficient for
-    # testing.
-    index = {
-        'ConcreteMutableMapping':
-            ConcreteMutableMapping,
-        'ConcreteMutableMapping.__init__':
-            ConcreteMutableMapping.__init__,
-        'ConcreteMutableMapping.__getitem__':
-            ConcreteMutableMapping.__getitem__,
-        'ConcreteMutableMapping.__setitem__':
-            ConcreteMutableMapping.__setitem__,
-        'ConcreteMutableMapping.values':
-            ConcreteMutableMapping.values,
-        'ConcreteMutableMapping.get':
-            ConcreteMutableMapping.get
-    }
-    visitor = DummyVisitor(index=index, duplicate_of={})
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
-
-    tree = {
-        'ConcreteMutableMapping': [
-            '__init__', '__getitem__', '__setitem__', 'values', 'get'
-        ]
-    }
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='ConcreteMutableMapping',
-        py_object=ConcreteMutableMapping,
-        parser_config=parser_config)
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
+
+    parser_config = generator.run_extraction()
+
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('m', 'ConcreteMutableMapping'), py_object=ConcreteMutableMapping)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
 
     self.assertIn(ConcreteMutableMapping.get,
-                  [m.obj for m in page_info.methods])
+                  [m.py_object for m in page_info.methods])
 
-  @unittest.skipIf(six.PY2, "Haven't found a repro for this under PY2.")
   def test_strips_default_arg_memory_address(self):
     """Validates that parser strips memory addresses out out default argspecs.
 
@@ -780,165 +625,196 @@ def test_strips_default_arg_memory_address(self):
 
      See: `help(collections.MutableMapping.pop)`
     """
-    index = {
-        'ConcreteMutableMapping': ConcreteMutableMapping,
-        'ConcreteMutableMapping.pop': ConcreteMutableMapping.pop
-    }
-    visitor = DummyVisitor(index=index, duplicate_of={})
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
-
-    tree = {'ConcreteMutableMapping': ['pop']}
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index=index,
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    page_info = parser.docs_for_object(
-        full_name='ConcreteMutableMapping',
-        py_object=ConcreteMutableMapping,
-        parser_config=parser_config)
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.fun = lambda x=object(): x
+
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
+
+    parser_config = generator.run_extraction()
 
-    pop_default_arg = page_info.methods[0].signature[1]
-    self.assertNotIn('object at 0x', pop_default_arg)
-    self.assertIn('', pop_default_arg)
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('m', 'fun'), py_object=m.fun)
+    page_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
+
+    output = str(page_info.signature)
+    self.assertNotIn('object at 0x', output)
+    self.assertIn('<object object>', output)
 
   @parameterized.named_parameters(
       ('mutable_mapping', 'ConcreteMutableMapping', '__contains__',
        ConcreteMutableMapping.__contains__),
       ('namedtuple', 'ConcreteNamedTuple', '__new__',
        ConcreteNamedTuple.__new__),
+      ('ClassUsingAttrs_eq', 'ClassUsingAttrs', '__eq__',
+       ClassUsingAttrs.__eq__),
+      ('ClassUsingAttrs_init', 'ClassUsingAttrs', '__init__',
+       ClassUsingAttrs.__init__),
   )
-  def test_builtins_defined_in(self, cls, method, py_object):
-    """Validates that the parser omits the defined_in location for built-ins.
+  def test_empty_defined_in(self, cls, method, py_object):
+    """Validates that the parser omits the defined_in location properly.
+
+    This test covers two cases where the parser should omit the defined_in
+    location:
+      1. built-ins.
+      2. methods automatically generated by python attr library.
 
-    Without special handling, the defined-in URL ends up like:
+    For built-ins, if without special handling, the defined-in URL ends up like:
       http://prefix//_collections_abc.py
 
+    For methods automatically generated by python attr library, if without
+    special handling, the defined-in URL ends up like:
+      http://prefix/
+
     Args:
       cls: The class name to generate docs for.
       method: The class method name to generate docs for.
       py_object: The python object for the specified cls.method.
     """
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.ConcreteMutableMapping = ConcreteMutableMapping
 
-    visitor = DummyVisitor(index={}, duplicate_of={})
-    reference_resolver = parser.ReferenceResolver.from_visitor(
-        visitor=visitor, py_module_names=['tf'])
-
-    tree = {cls: [method]}
-    parser_config = parser.ParserConfig(
-        reference_resolver=reference_resolver,
-        duplicates={},
-        duplicate_of={},
-        tree=tree,
-        index={},
-        reverse_index={},
-        base_dir='/',
-        code_url_prefix='/')
-
-    function_info = parser.docs_for_object(
-        full_name='%s.%s' % (cls, method),
-        py_object=py_object,
-        parser_config=parser_config)
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
+
+    parser_config = generator.run_extraction()
+
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=(cls, method), py_object=py_object)
+    function_info = docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
 
     self.assertIsNone(function_info.defined_in)
 
+  def test_get_other_member_doc_object_doc_attr(self):
 
-class TestReferenceResolver(absltest.TestCase):
-  _BASE_DIR = tempfile.mkdtemp()
+    class A():
+      """Class docs."""
+      pass
 
-  def setUp(self):
-    super(TestReferenceResolver, self).setUp()
-    self.workdir = os.path.join(self._BASE_DIR, self.id())
-    os.makedirs(self.workdir)
+    a = A()
+    a.__doc__ = 'Object doc'
 
-  def testSaveReferenceResolver(self):
-    duplicate_of = {'AClass': ['AClass2']}
-    is_fragment = {
-        'tf': False,
-        'tf.VERSION': True,
-        'tf.AClass': False,
-        'tf.AClass.method': True,
-        'tf.AClass2': False,
-        'tf.function': False
-    }
-    py_module_names = ['tf', 'tfdbg']
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.a = a
 
-    resolver = parser.ReferenceResolver(duplicate_of, is_fragment,
-                                        py_module_names)
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
 
-    outdir = self.workdir
+    parser_config = generator.run_extraction()
 
-    filepath = os.path.join(outdir, 'resolver.json')
+    result = parser._get_other_member_doc(a, parser_config, {})
 
-    resolver.to_json_file(filepath)
-    resolver2 = parser.ReferenceResolver.from_json_file(filepath)
+    expected = textwrap.dedent("""\
+      Instance of `__main__.A`
 
-    # There are no __slots__, so all fields are visible in __dict__.
-    self.assertEqual(resolver.__dict__, resolver2.__dict__)
+      Object doc""")
 
-  def testIsFreeFunction(self):
+    self.assertEqual(expected, result)
 
-    result = parser.is_free_function(test_function, 'test_module.test_function',
-                                     {'test_module': test_module})
-    self.assertTrue(result)
+  def test_get_other_member_doc_extra_doc(self):
+    # This will get sorted.
+    a = {4, 2, 1, 3}
+    # You can't set __doc__ on a list or a set so use extra_docs
+    doc = 'Object doc'
+    extra_docs = {id(a): doc}
 
-    result = parser.is_free_function(test_function, 'TestClass.test_function',
-                                     {'TestClass': TestClass})
-    self.assertFalse(result)
+    result = parser._get_other_member_doc(a, None, extra_docs)
 
-    result = parser.is_free_function(TestClass, 'TestClass', {})
-    self.assertFalse(result)
+    expected = textwrap.dedent("""\
+      ```
+      {
+       1,
+       2,
+       3,
+       4
+      }
+      ```
+
+      Object doc""")
+    self.assertEqual(expected, result)
 
-    result = parser.is_free_function(test_module, 'test_module', {})
+  def test_get_other_member_basic_type(self):
+    a = 5
+    result = parser._get_other_member_doc(a, None, {})
+
+    self.assertEqual('`5`', result)
+
+  def test_get_other_member_doc_unknown_class(self):
+
+    class A():
+      """Class docs."""
+      pass
+
+    a = A()
+
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.a = a
+
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
+
+    parser_config = generator.run_extraction()
+
+    result = parser._get_other_member_doc(a, parser_config, {})
+    expected = textwrap.dedent("""\
+      Instance of `__main__.A`""")
+
+    self.assertEqual(expected, result)
+
+  def test_get_other_member_doc_known_class(self):
+
+    class A():
+      """Class docs."""
+      pass
+
+    a = A()
+
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.A = A
+    m.a = a
+
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
+
+    parser_config = generator.run_extraction()
+
+    result = parser._get_other_member_doc(a, parser_config, {})
+
+    self.assertEqual('Instance of `m.A`', result)
+
+  def testIsClassAttr(self):
+    result = parser.is_class_attr('test_module.test_function',
+                                  {'test_module': test_module})
     self.assertFalse(result)
 
-  def test_duplicate_fragment(self):
-    duplicate_of = {
-        'tf.Class2.method': 'tf.Class1.method',
-        'tf.sub.Class2.method': 'tf.Class1.method',
-        'tf.sub.Class2': 'tf.Class2'
-    }
-    is_fragment = {
-        'tf.Class1.method': True,
-        'tf.Class2.method': True,
-        'tf.sub.Class2.method': True,
-        'tf.Class1': False,
-        'tf.Class2': False,
-        'tf.sub.Class2': False
-    }
-    py_module_names = ['tf']
-
-    reference_resolver = parser.ReferenceResolver(duplicate_of, is_fragment,
-                                                  py_module_names)
-
-    # Method references point to the method, in the canonical class alias.
-    result = reference_resolver.reference_to_url('tf.Class1.method', '')
-    self.assertEqual('tf/Class1.md#method', result)
-    result = reference_resolver.reference_to_url('tf.Class2.method', '')
-    self.assertEqual('tf/Class2.md#method', result)
-    result = reference_resolver.reference_to_url('tf.sub.Class2.method', '')
-    self.assertEqual('tf/Class2.md#method', result)
-
-    # Class references point to the canonical class alias
-    result = reference_resolver.reference_to_url('tf.Class1', '')
-    self.assertEqual('tf/Class1.md', result)
-    result = reference_resolver.reference_to_url('tf.Class2', '')
-    self.assertEqual('tf/Class2.md', result)
-    result = reference_resolver.reference_to_url('tf.sub.Class2', '')
-    self.assertEqual('tf/Class2.md', result)
+    result = parser.is_class_attr('TestClass.test_function',
+                                  {'TestClass': TestClass})
+    self.assertTrue(result)
+
 
 RELU_DOC = """Computes rectified linear: `max(features, 0)`
 
 RELU is an activation
 
 Args:
-  features: A `Tensor`. Must be one of the following types: `float32`,
+  'features': A `Tensor`. Must be one of the following types: `float32`,
     `float64`, `int32`, `int64`, `uint8`, `int16`, `int8`, `uint16`,
     `half`.
   name: A name for the operation (optional)
@@ -968,7 +844,7 @@ def test_split_title_blocks(self):
     self.assertEqual(args.title, 'Args')
     self.assertEqual(args.text, '\n')
     self.assertLen(args.items, 2)
-    self.assertEqual(args.items[0][0], 'features')
+    self.assertEqual(args.items[0][0], "'features'")
     self.assertEqual(
         args.items[0][1],
         'A `Tensor`. Must be one of the following types: `float32`,\n'
@@ -986,166 +862,99 @@ def test_split_title_blocks(self):
                      '\nSome tensors, with the same type as the input.\n')
     self.assertLen(returns.items, 2)
 
+  def test_title_block(self):
+    docstring = textwrap.dedent("""\
+      hello
+ 
+      Attributes:
+        extra paragraph?
+        item: description
+          describe describe
+        item2 (int): is a number
+        this is not an item: really not 
+        this either: nope
 
-class TestPartialSymbolAutoRef(parameterized.TestCase):
-  REF_TEMPLATE = '{text}'
+      goodbye
+    """)
+    docstring_parts = parser.TitleBlock.split_string(docstring)
+    print(docstring_parts)
+    self.assertEqual('hello', docstring_parts[0])
+    self.assertIsInstance(docstring_parts[1], parser.TitleBlock)
+    self.assertEqual('\ngoodbye\n', docstring_parts[2])
 
-  @parameterized.named_parameters(
-      ('basic1', 'keras.Model.fit', '../tf/keras/Model.md#fit'),
-      ('duplicate_object', 'layers.Conv2D', '../tf/keras/layers/Conv2D.md'),
-      ('parens', 'Model.fit(x, y, epochs=5)', '../tf/keras/Model.md#fit'),
-      ('duplicate_name', 'tf.matmul', '../tf/linalg/matmul.md'),
-      ('full_name', 'tf.concat', '../tf/concat.md'),
-      ('normal_and_compat', 'linalg.matmul', '../tf/linalg/matmul.md'),
-      ('compat_only', 'math.deprecated', None),
-      ('contrib_only', 'y.z', None),
-  )
-  def test_partial_symbol_references(self, string, link):
-    duplicate_of = {
-        'tf.matmul': 'tf.linalg.matmul',
-        'tf.layers.Conv2d': 'tf.keras.layers.Conv2D',
-    }
+    block = docstring_parts[1]
+    self.assertEqual('\nextra paragraph?\n', block.text)
+    self.assertEqual('item', block.items[0][0])
+    self.assertEqual('item2', block.items[1][0])
+    self.assertStartsWith(block.items[1][1], '`int`')
+    self.assertLen(block.items, 2)
 
-    is_fragment = {
-        'tf.keras.Model.fit': True,
-        'tf.concat': False,
-        'tf.keras.layers.Conv2D': False,
-        'tf.linalg.matmul': False,
-        'tf.compat.v1.math.deprecated': False,
-        'tf.compat.v1.linalg.matmul': False,
-        'tf.contrib.y.z': False,
-    }
+  def test_strip_todos(self):
+    input_str = ("""#  TODO(blah) blah
 
-    py_module_names = ['tf']
+        hello    TODO: more stuff
+        middle
+        goodbye  TODO
+        """)
 
-    resolver = parser.ReferenceResolver(duplicate_of, is_fragment,
-                                        py_module_names)
-    input_string = string.join('``')
-    ref_string = resolver.replace_references(input_string, '..')
+    expected = ("""
 
-    if link is None:
-      expected = input_string
-    else:
-      expected = self.REF_TEMPLATE.format(link=link, text=string)
+        hello
+        middle
+        goodbye
+        """)
+    strip_todos = parser._StripTODOs()
+    self.assertEqual(expected, strip_todos(input_str))
 
-    self.assertEqual(expected, ref_string)
+  def test_strip_pylintandpyformat(self):
+    input_str = textwrap.dedent("""
+        hello  #  pyformat: disable
+        middle  # pyformat: enable
+        goodbye  TODO  # pylint: disable=g-top-imports
 
+        # pyformat: disable
+        xyz
+        # pyformat: enable
 
-class TestIgnoreLineInBlock(parameterized.TestCase):
+        # pylint: disable=g-top-imports
+        abc
+        # pylint: enable=g-top-imports
+        """)
 
-  @parameterized.named_parameters(
-      ('ignore_backticks',
-       ['```'],
-       ['```'],
-       '```\nFiller\n```\n```Same line```\n```python\nDowner\n```'),
-
-      ('ignore_code_cell_output',
-       ['
    {% html %}'],
-       ['{% endhtml %}
    '],
-       '
    {% html %}\nOutput\nmultiline{% endhtml %}
    '),
-
-      ('ignore_backticks_and_cell_output',
-       ['
    {% html %}', '```'],
-       ['{% endhtml %}
    ', '```'],
-       ('```\nFiller\n```\n```Same line```\n
    {% html %}\nOutput\nmultiline'
-        '{% endhtml %}
    \n```python\nDowner\n```'))
-      )
-  def test_ignore_lines(self, block_start, block_end, expected_ignored_lines):
-
-    text = textwrap.dedent('''\
-    ```
-    Filler
-    ```
-
-    ```Same line```
-
-    
    {% html %}
-    Output
-    multiline{% endhtml %}
    
-
-    ```python
-    Downer
-    ```
-    ''')
-
-    filters = [parser.IgnoreLineInBlock(start, end)
-               for start, end in zip(block_start, block_end)]
-
-    ignored_lines = []
-    for line in text.splitlines():
-      if any(filter_block(line) for filter_block in filters):
-        ignored_lines.append(line)
-
-    self.assertEqual('\n'.join(ignored_lines), expected_ignored_lines)
-
-  def test_clean_text(self):
-    text = textwrap.dedent('''\
-    ```
-    Ignore lines here.
-    ```
-    Useful information.
-    Don't ignore.
-    ```python
-    Ignore here too.
-    ```
-    Stuff.
-    ```Not useful.```
-    ''')
-
-    filters = [parser.IgnoreLineInBlock('```', '```')]
-
-    clean_text = []
-    for line in text.splitlines():
-      if not any(filter_block(line) for filter_block in filters):
-        clean_text.append(line)
-
-    expected_clean_text = 'Useful information.\nDon\'t ignore.\nStuff.'
-
-    self.assertEqual('\n'.join(clean_text), expected_clean_text)
-
-
-class TestGenerateSignature(absltest.TestCase):
-
-  def test_known_object(self):
-    known_object = object()
-    reverse_index = {id(known_object): 'location.of.object.in.api'}
-
-    def example_fun(arg=known_object):  # pylint: disable=unused-argument
-      pass
+    expected = textwrap.dedent("""
+        hello  
+        middle  
+        goodbye  TODO  
 
-    sig = parser._generate_signature(example_fun, reverse_index)
-    self.assertEqual(sig, ['arg=location.of.object.in.api'])
 
-  def test_literals(self):
-    def example_fun(a=5, b=5.0, c=None, d=True, e='hello', f=(1, (2, 3))):  # pylint: disable=g-bad-name, unused-argument
-      pass
+        xyz
 
-    sig = parser._generate_signature(example_fun, reverse_index={})
-    self.assertEqual(
-        sig, ['a=5', 'b=5.0', 'c=None', 'd=True', "e='hello'", 'f=(1, (2, 3))'])
 
-  def test_dotted_name(self):
-    # pylint: disable=g-bad-name
 
-    class a(object):
+        abc
 
-      class b(object):
+        """)
+    strip_todos = parser._StripPylintAndPyformat()
+    self.assertEqual(expected, strip_todos(input_str))
 
-        class c(object):
+  def test_get_dataclass_docstring(self):
 
-          class d(object):
+    @dataclasses.dataclass
+    class MyClass():
+      """Docstring!"""
+      a: int
+      b: float
 
-            def __init__(self, *args):
-              pass
-    # pylint: enable=g-bad-name
+    self.assertEqual(parser._get_raw_docstring(MyClass), 'Docstring!')
 
-    e = {'f': 1}
+  def test_get_dataclass_docstring_no_autogen_docstring(self):
 
-    def example_fun(arg1=a.b.c.d, arg2=a.b.c.d(1, 2), arg3=e['f']):  # pylint: disable=unused-argument
-      pass
+    @dataclasses.dataclass
+    class MyClass():
+      a: int
+      b: float
 
-    sig = parser._generate_signature(example_fun, reverse_index={})
-    self.assertEqual(sig, ['arg1=a.b.c.d', 'arg2=a.b.c.d(1, 2)', "arg3=e['f']"])
+    self.assertEmpty(parser._get_raw_docstring(MyClass))
 
 if __name__ == '__main__':
   absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs.py b/tools/tensorflow_docs/api_generator/pretty_docs.py
deleted file mode 100644
index 046076019a2..00000000000
--- a/tools/tensorflow_docs/api_generator/pretty_docs.py
+++ /dev/null
@@ -1,406 +0,0 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""A module for converting parsed doc content into markdown pages.
-
-The adjacent `parser` module creates `PageInfo` objects, containing all data
-necessary to document an element of the TensorFlow API.
-
-This module contains one public function, which handels the conversion of these
-`PageInfo` objects into a markdown string:
-
-    md_page = build_md_page(page_info)
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import textwrap
-
-from tensorflow_docs.api_generator import doc_generator_visitor
-
-
-def build_md_page(page_info):
-  """Given a PageInfo object, return markdown for the page.
-
-  Args:
-    page_info: must be a `parser.FunctionPageInfo`, `parser.ClassPageInfo`, or
-      `parser.ModulePageInfo`
-
-  Returns:
-    Markdown for the page
-
-  Raises:
-    ValueError: if `page_info` is an instance of an unrecognized class
-  """
-  if page_info.for_function():
-    return _build_function_page(page_info)
-
-  if page_info.for_class():
-    return _build_class_page(page_info)
-
-  if page_info.for_module():
-    return _build_module_page(page_info)
-
-  raise ValueError('Unknown Page Info Type: %s' % type(page_info))
-
-
-def _build_function_page(page_info):
-  """Given a FunctionPageInfo object Return the page as an md string."""
-  parts = ['# %s\n\n' % page_info.full_name]
-
-  parts.append('\n')
-
-  parts.append(_top_source_link(page_info.defined_in))
-  parts.append('\n\n')
-
-  parts.append(page_info.doc.brief + '\n\n')
-
-  parts.append(_build_main_aliases(page_info.aliases))
-
-  if page_info.signature is not None:
-    parts.append(_build_signature(page_info))
-    parts.append('\n\n')
-
-  # This will be replaced by the "Used in: " whenever it is run.
-  parts.append('\n')
-
-  parts.extend(str(item) for item in page_info.doc.docstring_parts)
-  parts.append(_build_compatibility(page_info.doc.compatibility))
-
-  parts.append('\n\n')
-  parts.append(_build_compat_aliases(page_info.aliases))
-
-  return ''.join(parts)
-
-
-def _build_class_page(page_info):
-  """Given a ClassPageInfo object Return the page as an md string."""
-  parts = ['# {page_info.full_name}\n\n'.format(page_info=page_info)]
-
-  parts.append('\n')
-
-  parts.append(_top_source_link(page_info.defined_in))
-  parts.append('\n\n')
-
-  parts.append('## Class `%s`\n\n' % page_info.full_name.split('.')[-1])
-
-  parts.append(page_info.doc.brief + '\n\n')
-
-  if page_info.bases:
-    parts.append('Inherits From: ')
-
-    link_template = '[`{short_name}`]({url})'
-    parts.append(', '.join(
-        link_template.format(**base._asdict()) for base in page_info.bases))
-
-  parts.append('\n\n')
-
-  parts.append(_build_main_aliases(page_info.aliases))
-
-  # This will be replaced by the "Used in: " whenever it is run.
-  parts.append('\n')
-
-  parts.extend(str(item) for item in page_info.doc.docstring_parts)
-  parts.append(_build_compatibility(page_info.doc.compatibility))
-
-  parts.append('\n\n')
-
-  # Sort the methods list, but make sure constructors come first.
-  constructor_names = ['__init__', '__new__']
-  constructors = sorted(method for method in page_info.methods
-                        if method.short_name in constructor_names)
-  other_methods = sorted(method for method in page_info.methods
-                         if method.short_name not in constructor_names)
-
-  if constructors:
-    for method_info in constructors:
-      parts.append(_build_method_section(method_info, heading_level=2))
-    parts.append('\n\n')
-
-  if page_info.classes:
-    parts.append('## Child Classes\n')
-
-    link_template = ('[`class {class_info.short_name}`]'
-                     '({class_info.url})\n\n')
-    class_links = sorted(
-        link_template.format(class_info=class_info)
-        for class_info in page_info.classes)
-
-    parts.extend(class_links)
-
-  if page_info.properties:
-    parts.append('## Properties\n\n')
-    for prop_info in page_info.properties:
-      h3 = '
    {short_name}
    \n\n'
-      parts.append(h3.format(short_name=prop_info.short_name))
-
-      parts.append(prop_info.doc.brief + '\n')
-      parts.extend(str(item) for item in prop_info.doc.docstring_parts)
-      parts.append(_build_compatibility(prop_info.doc.compatibility))
-
-      parts.append('\n\n')
-
-    parts.append('\n\n')
-
-  if other_methods:
-    parts.append('## Methods\n\n')
-
-    for method_info in other_methods:
-      parts.append(_build_method_section(method_info))
-    parts.append('\n\n')
-
-  if page_info.other_members:
-    parts.append('## Class Members\n\n')
-
-    parts.append(_other_members(page_info.other_members))
-
-  parts.append('\n\n')
-  parts.append(_build_compat_aliases(page_info.aliases))
-
-  return ''.join(parts)
-
-
-def _other_members(other_members):
-  """Returns "other_members" rendered to markdown.
-
-  `other_members` is used for anything that is not a class, function, module,
-  or method.
-
-  Args:
-    other_members: a list of (name, object) pairs.
-
-  Returns:
-    A markdown string
-  """
-  parts = []
-  list_item = '* `{short_name}` \n'
-  list_item_with_value = ('* `{short_name} = {obj!r}` '
-                          '\n')
-  for other_member in other_members:
-    if doc_generator_visitor.maybe_singleton(other_member.obj):
-      part = list_item_with_value.format(**other_member._asdict())
-    else:
-      part = list_item.format(**other_member._asdict())
-    parts.append(part)
-
-  return ''.join(parts)
-
-
-def _build_method_section(method_info, heading_level=3):
-  """Generates a markdown section for a method.
-
-  Args:
-    method_info: A `MethodInfo` object.
-    heading_level: An Int, which HTML heading level to use.
-
-  Returns:
-    A markdown string.
-  """
-  parts = []
-  heading = (''
-             '{short_name}'
-             '\n\n')
-  parts.append(
-      heading.format(heading_level=heading_level, **method_info._asdict()))
-
-  if method_info.defined_in:
-    parts.append(_small_source_link(method_info.defined_in))
-
-  if method_info.signature is not None:
-    parts.append(_build_signature(method_info, use_full_name=False))
-
-  parts.append(method_info.doc.brief + '\n')
-  parts.extend(str(item) for item in method_info.doc.docstring_parts)
-  parts.append(_build_compatibility(method_info.doc.compatibility))
-  parts.append('\n\n')
-  return ''.join(parts)
-
-
-def _build_module_page(page_info):
-  """Given a ClassPageInfo object Return the page as an md string."""
-  parts = ['# Module: {full_name}\n\n'.format(full_name=page_info.full_name)]
-
-  parts.append(_top_source_link(page_info.defined_in))
-  parts.append('\n\n')
-
-  # First line of the docstring i.e. a brief introduction about the symbol.
-  parts.append(page_info.doc.brief + '\n\n')
-
-  parts.append(_build_main_aliases(page_info.aliases))
-
-  # All lines in the docstring, expect the brief introduction.
-  parts.extend(str(item) for item in page_info.doc.docstring_parts)
-  parts.append(_build_compatibility(page_info.doc.compatibility))
-
-  parts.append('\n\n')
-
-  if page_info.modules:
-    parts.append('## Modules\n\n')
-    template = '[`{short_name}`]({url}) module'
-
-    for item in page_info.modules:
-      parts.append(template.format(**item._asdict()))
-
-      if item.doc.brief:
-        parts.append(': ' + item.doc.brief)
-
-      parts.append('\n\n')
-
-  if page_info.classes:
-    parts.append('## Classes\n\n')
-    template = '[`class {short_name}`]({url})'
-
-    for item in page_info.classes:
-      parts.append(template.format(**item._asdict()))
-
-      if item.doc.brief:
-        parts.append(': ' + item.doc.brief)
-
-      parts.append('\n\n')
-
-  if page_info.functions:
-    parts.append('## Functions\n\n')
-    template = '[`{short_name}(...)`]({url})'
-
-    for item in page_info.functions:
-      parts.append(template.format(**item._asdict()))
-
-      if item.doc.brief:
-        parts.append(': ' + item.doc.brief)
-
-      parts.append('\n\n')
-
-  if page_info.other_members:
-    # TODO(markdaoust): Document the value of the members,
-    #                   at least for basic types.
-    parts.append('## Other Members\n\n')
-
-    parts.append(_other_members(page_info.other_members))
-
-  parts.append('\n\n')
-  parts.append(_build_compat_aliases(page_info.aliases))
-
-  return ''.join(parts)
-
-
-def _build_signature(obj_info, use_full_name=True):
-  """Returns a md code block showing the function signature."""
-  # Special case tf.range, since it has an optional first argument
-  if obj_info.full_name == 'tf.range':
-    return ('``` python\n'
-            "tf.range(limit, delta=1, dtype=None, name='range')\n"
-            "tf.range(start, limit, delta=1, dtype=None, name='range')\n"
-            '```\n\n')
-
-  parts = ['``` python']
-  parts.extend(['@' + dec for dec in obj_info.decorators])
-  signature_template = '{name}({sig})'
-
-  if not obj_info.signature:
-    sig = ''
-  elif len(obj_info.signature) == 1:
-    sig = obj_info.signature[0]
-  else:
-    sig = ',\n'.join('    %s' % sig_item for sig_item in obj_info.signature)
-    sig = '\n' + sig + '\n'
-
-  if use_full_name:
-    obj_name = obj_info.full_name
-  else:
-    obj_name = obj_info.short_name
-  parts.append(signature_template.format(name=obj_name, sig=sig))
-  parts.append('```\n\n')
-
-  return '\n'.join(parts)
-
-
-def _build_compatibility(compatibility):
-  """Return the compatibility section as an md string."""
-  parts = []
-  sorted_keys = sorted(compatibility.keys())
-  for key in sorted_keys:
-
-    value = compatibility[key]
-    # Dedent so that it does not trigger markdown code formatting.
-    value = textwrap.dedent(value)
-    parts.append('\n\n#### %s Compatibility\n%s\n' % (key.title(), value))
-
-  return ''.join(parts)
-
-
-def _top_source_link(location):
-  """Retrns a source link with Github image, like the notebook butons."""
-  table_template = textwrap.dedent("""
-    
-    {}
    
-
-    """)
-
-  link_template = textwrap.dedent("""
-    
-      
-        
-        View source on GitHub
-      
-    """)
-
-  if location is None or not location.url:
-    return table_template.format('')
-
-  if 'github.com' not in location.url:
-    return table_template.format('') + _small_source_link(location)
-
-  link = link_template.format(url=location.url)
-  table = table_template.format(link)
-  return table
-
-
-def _small_source_link(location):
-  """Returns a small source link."""
-  template = 'View source\n\n'
-
-  if not location.url:
-    return ''
-
-  return template.format(url=location.url)
-
-
-def _build_main_aliases(aliases):
-  """Returns the top "Aliases" line."""
-  aliases = [name for name in aliases if '__' not in name]
-  aliases = [name for name in aliases if 'compat.v' not in name]
-
-  parts = []
-  if aliases:
-    parts.append('**Aliases**: ')
-    parts.append(', '.join('`{}`'.format(name) for name in aliases))
-    parts.append('\n\n')
-
-  return ''.join(parts)
-
-
-def _build_compat_aliases(aliases):
-  """Returns the "Compat Aliases" block."""
-  aliases = [name for name in aliases if '__' not in name]
-  aliases = [name for name in aliases if 'compat.v' in name]
-
-  parts = []
-  if aliases:
-    parts.append('## Compat aliases\n\n')
-    parts.extend(['* `{}`\n'.format(name) for name in aliases])
-    parts.append('\n')
-
-  return ''.join(parts)
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/__init__.py b/tools/tensorflow_docs/api_generator/pretty_docs/__init__.py
new file mode 100644
index 00000000000..b98e8d8ce8c
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/__init__.py
@@ -0,0 +1,30 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from tensorflow_docs.api_generator.pretty_docs.base_page import PageBuilder
+from tensorflow_docs.api_generator.pretty_docs.base_page import PageInfo
+from tensorflow_docs.api_generator.pretty_docs.base_page import TemplatePageBuilder
+
+from tensorflow_docs.api_generator.pretty_docs.class_page import ClassPageBuilder
+from tensorflow_docs.api_generator.pretty_docs.class_page import ClassPageInfo
+
+from tensorflow_docs.api_generator.pretty_docs.function_page import FunctionPageBuilder
+from tensorflow_docs.api_generator.pretty_docs.function_page import FunctionPageInfo
+
+from tensorflow_docs.api_generator.pretty_docs.module_page import ModulePageBuilder
+from tensorflow_docs.api_generator.pretty_docs.module_page import ModulePageInfo
+
+from tensorflow_docs.api_generator.pretty_docs.type_alias_page import TypeAliasPageBuilder
+from tensorflow_docs.api_generator.pretty_docs.type_alias_page import TypeAliasPageInfo
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/base_page.py b/tools/tensorflow_docs/api_generator/pretty_docs/base_page.py
new file mode 100644
index 00000000000..59c20006a5e
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/base_page.py
@@ -0,0 +1,560 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Base classes for page construction."""
+import abc
+import os
+import pathlib
+import posixpath
+import textwrap
+from typing import Any, ClassVar, Dict, List, NamedTuple, Optional, Sequence, Tuple, Type
+
+from tensorflow_docs.api_generator import config
+from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator import signature as signature_lib
+
+import jinja2
+
+
+class PageBuilder(abc.ABC):
+
+  def __init__(self, page_info):
+    self.page_info = page_info
+
+  @abc.abstractmethod
+  def build(self) -> str:
+    pass
+
+
+class TemplatePageBuilder(PageBuilder):
+  """A Page builder implemented on a jinja template."""
+
+  TEMPLATE = 'templates/page.jinja'
+  TEMPLATE_SEARCH_PATH = tuple([str(pathlib.Path(__file__).parent)])
+  JINJA_ENV = jinja2.Environment(
+      trim_blocks=True,
+      lstrip_blocks=True,
+      loader=jinja2.FileSystemLoader(TEMPLATE_SEARCH_PATH))
+
+  def build(self) -> str:
+    template = self.JINJA_ENV.get_template(self.TEMPLATE)
+    content = template.render(builder=self, page_info=self.page_info)
+    return content
+
+  def top_source_link(self):
+    return top_source_link(self.page_info.defined_in)
+
+  def build_collapsable_aliases(self):
+    return build_collapsable_aliases(sorted(self.page_info.aliases))
+
+  def top_compat(self):
+    return build_top_compat(self.page_info, h_level=2)
+
+  def bottom_compat(self):
+    return build_bottom_compat(self.page_info, h_level=2)
+
+  def format_docstring_part(self, part):
+    return str(part)
+
+  def get_devsite_headers(self):
+    """Returns the list of header lines for this page."""
+    hidden = doc_controls.should_hide_from_search(self.page_info.py_object)
+    brief_no_backticks = self.page_info.doc.brief.replace('`', '').strip()
+    headers = []
+    if brief_no_backticks:
+      headers.append(f'description: {brief_no_backticks}')
+
+    if self.page_info.search_hints and not hidden:
+      if headers:
+        headers.append('')
+      headers.append(self.page_info.get_metadata_html())
+    else:
+      headers.append('robots: noindex')
+      headers.append('')
+
+    result = '\n'.join(headers)
+    return result
+
+
+class PageInfo:
+  """Base-class for api_pages objects.
+
+  Converted to markdown by pretty_docs.py.
+
+  Attributes:
+    full_name: The full, main name, of the object being documented.
+    short_name: The last part of the full name.
+    py_object: The object being documented.
+    defined_in: A _FileLocation describing where the object was defined.
+    aliases: A list of full-name for all aliases for this object.
+    doc: A list of objects representing the docstring. These can all be
+      converted to markdown using str().
+    search_hints: If true include metadata search hints, else include a
+      "robots: noindex"
+    text: The resulting page text.
+    page_text: The cached result.
+  """
+  DEFAULT_BUILDER_CLASS: ClassVar[Type[PageBuilder]] = TemplatePageBuilder
+
+  def __init__(
+      self,
+      api_node,
+      extra_docs: Optional[Dict[int, str]] = None,
+      search_hints: bool = True,
+      parser_config=None,
+  ):
+    """Initialize a PageInfo.
+
+    Args:
+      full_name: The full, main name, of the object being documented.
+      py_object: The object being documented.
+      extra_docs: Extra docs for symbols like public constants(list, tuple, etc)
+        that need to be added to the markdown pages created.
+      search_hints: If true include metadata search hints, else include a
+        "robots: noindex"
+
+    """
+    self.api_node = api_node
+    self.full_name = api_node.full_name
+    self.py_object = api_node.py_object
+    self._extra_docs = extra_docs
+    self.search_hints = search_hints
+    self.parser_config = parser_config
+
+    self._defined_in = None
+    self._aliases = None
+    self._doc = None
+    self._page_text = None
+
+  def collect_docs(self):
+    """Collects additional information from the `config.ParserConfig`."""
+    pass
+
+  def docs_for_object(self):
+    relative_path = os.path.relpath(
+        path='.',
+        start=os.path.dirname(parser.documentation_path(self.full_name)) or '.')
+
+    # Convert from OS-specific path to URL/POSIX path.
+    relative_path = posixpath.join(*relative_path.split(os.path.sep))
+
+    with self.parser_config.reference_resolver.temp_prefix(relative_path):
+      self.set_doc(
+          parser.parse_md_docstring(
+              self.py_object,
+              self.full_name,
+              self.parser_config,
+              self._extra_docs,
+          ))
+
+      self.collect_docs()
+
+      aliases = ['.'.join(alias) for alias in self.api_node.aliases]
+      if self.full_name in aliases:
+        aliases.remove(self.full_name)
+      self.set_aliases(aliases)
+
+      self.set_defined_in(
+          parser.get_defined_in(self.py_object, self.parser_config))
+
+      self._page_text = self.build()
+
+    return self._page_text
+
+  def build(self) -> str:
+    """Builds the documentation."""
+    cls = self.DEFAULT_BUILDER_CLASS
+    return cls(self).build()
+
+  @property
+  def page_text(self):
+    if self._page_text is None:
+      self._page_text = self.build()
+    return self._page_text
+
+  def __eq__(self, other):
+    if isinstance(other, PageInfo):
+      return self.__dict__ == other.__dict__
+    else:
+      return NotImplemented
+
+  @property
+  def short_name(self):
+    """Returns the documented object's short name."""
+    return self.full_name.split('.')[-1]
+
+  @property
+  def defined_in(self):
+    """Returns the path to the file where the documented object is defined."""
+    return self._defined_in
+
+  def set_defined_in(self, defined_in):
+    """Sets the `defined_in` path."""
+    assert self.defined_in is None
+    self._defined_in = defined_in
+
+  @property
+  def self_link(self):
+    if not self.parser_config.self_link_base:
+      return None
+    rel_path = parser.documentation_path(self.full_name)
+    rel_path = rel_path[: -1 * len('.md')]  # strip suffix
+    return f'{self.parser_config.self_link_base}/{rel_path}'
+
+  @property
+  def aliases(self):
+    """Returns a list of all full names for the documented object."""
+    return self._aliases
+
+  def set_aliases(self, aliases):
+    """Sets the `aliases` list.
+
+    Args:
+      aliases: A list of strings. Containing all the object's full names.
+    """
+    assert self.aliases is None
+    self._aliases = aliases
+
+  @property
+  def doc(self) -> parser.DocstringInfo:
+    """Returns a `parser.DocstringInfo` created from the object's docstring."""
+    return self._doc
+
+  def set_doc(self, doc: parser.DocstringInfo):
+    """Sets the `doc` field.
+
+    Args:
+      doc: An instance of `parser.DocstringInfo`.
+    """
+    assert self.doc is None
+    self._doc = doc
+
+
+class MemberInfo(NamedTuple):
+  """Describes an attribute of a class or module."""
+  short_name: str
+  full_name: str
+  py_object: Any
+  doc: parser.DocstringInfo
+  url: str
+
+
+_ALWAYS_TABLE_ITEMS = ('arg', 'return', 'raise', 'attr', 'yield')
+
+
+def format_docstring(item,
+                     *,
+                     table_title_template: Optional[str] = None,
+                     anchors: bool = True) -> str:
+  """Formats a docstring part into a string.
+
+  Args:
+    item: A TitleBlock instance or a normal string.
+    table_title_template: Template for title detailing how to display it in the
+      table.
+
+  Returns:
+    A formatted docstring.
+  """
+
+  if isinstance(item, parser.TitleBlock):
+    if (item.items or  # A colon-list like under args
+        item.text.strip() or  # An indented block
+        item.title.lower().startswith(_ALWAYS_TABLE_ITEMS)):
+      return item.table_view(
+          title_template=table_title_template, anchors=anchors)
+    else:
+      return str(item)
+  else:
+    return str(item)
+
+
+def build_other_members(other_members: List[MemberInfo], title: str):
+  """Returns "other_members" rendered to markdown.
+
+  `other_members` is used for anything that is not a class, function, module,
+  or method.
+
+  Args:
+    other_members: A list of `base_page.MemberInfo` objects.
+    title: Title of the table.
+
+  Returns:
+    A markdown string
+  """
+
+  items = []
+
+  for other_member in other_members:
+    description = [other_member.doc.brief]
+    for doc_part in other_member.doc.docstring_parts:
+      if isinstance(doc_part, parser.TitleBlock):
+        # Use list_view here because description will be part of a table.
+        description.append(str(doc_part))
+      else:
+        description.append(doc_part)
+
+    items.append(
+        parser.ITEMS_TEMPLATE.format(
+            name=other_member.short_name,
+            anchor=f'',
+            description='\n'.join(description),
+        ))
+  return '\n' + parser.TABLE_TEMPLATE.format(
+      title=title, text='', items=''.join(items))
+
+
+DECORATOR_ALLOWLIST = frozenset({
+    'classmethod',
+    'staticmethod',
+    'tf_contextlib.contextmanager',
+    'contextlib.contextmanager',
+    'tf.function',
+    'types.method',
+    'abc.abstractmethod',
+})
+
+
+def build_signature(name: str,
+                    signature: signature_lib.TfSignature,
+                    decorators: Optional[Sequence[str]],
+                    type_alias: bool = False) -> str:
+  """Returns a markdown code block containing the function signature.
+
+  Wraps the signature and limits it to 80 characters.
+
+  Args:
+    name: the name to put in the template.
+    signature: the signature object.
+    decorators: a list of decorators to apply.
+    type_alias: If True, uses an `=` instead of `()` for the signature.
+      For example: `TensorLike = (Union[str, tf.Tensor, int])`. Defaults to
+        `False`.
+
+  Returns:
+    The signature of the object.
+  """
+  if name == 'tf.range':
+    # Special case tf.range, since it has an optional first argument
+    return textwrap.dedent("""
+      ```python
+      tf.range(limit, delta=1, dtype=None, name='range')
+      tf.range(start, limit, delta=1, dtype=None, name='range')
+      ```
+      """)
+
+  full_signature = str(signature)
+
+  parts = [
+      '
    '
+  ]
+
+  if decorators:
+    parts.extend([
+        f'@{dec}' for dec in decorators
+        if dec in DECORATOR_ALLOWLIST
+    ])
+
+  if type_alias:
+    parts.append(f'{name} = {full_signature}')
+  else:
+    parts.append(f'{name}{full_signature}')
+  parts.append('
    \n\n')
+
+  return '\n'.join(parts)
+
+
+def _split_compat_top_bottom(page_info) -> Tuple[Optional[str], Dict[str, str]]:
+  """Split the compatibility dict between the top and bottom sections."""
+  compat: Dict[str, str] = page_info.doc.compatibility
+  top_compat = None
+
+  if ('compat.v1' in page_info.full_name or 'estimator' in page_info.full_name):
+    bottom_compat = {}
+    for key, value in compat.items():
+      if key == 'TF2':
+        top_compat = value
+      else:
+        bottom_compat[key] = value
+  else:
+    bottom_compat = compat
+
+  return top_compat, bottom_compat
+
+
+_TOP_COMPAT_TEMPLATE = """
+
+ 
    
+ Migrate to TF2
+
+Caution: This API was designed for TensorFlow v1.
+Continue reading for details on how to migrate from this API to a native
+TensorFlow v2 equivalent. See the
+[TensorFlow v1 to TensorFlow v2 migration guide](https://www.tensorflow.org/guide/migrate)
+for instructions on how to migrate the rest of your code.
+
+{value}
+
+ 
    
+
+Description
+
+"""
+
+
+def build_top_compat(page_info: PageInfo, h_level: int) -> str:
+  """Add the top section compatibility blocks."""
+  compat, _ = _split_compat_top_bottom(page_info)
+  if compat:
+    value = textwrap.dedent(compat)
+    return _TOP_COMPAT_TEMPLATE.format(value=value, h_level=h_level)
+  else:
+    return ''
+
+
+_BOTTOM_COMPAT_TEMPLATE = """
+
+ 
    
+ {title}
+
+{value}
+
+ 
    
+
+"""
+
+
+def build_bottom_compat(page_info: PageInfo, h_level: int) -> str:
+  """Add the bottom section compatibility blocks."""
+  _, compat = _split_compat_top_bottom(page_info)
+
+  def _tf2_key_tuple(key):
+    # False sorts before True.
+    return (key == 'TF2', key)
+
+  parts = []
+  for key in sorted(compat, key=_tf2_key_tuple):
+    value = textwrap.dedent(compat[key])
+    if key == 'TF2':
+      expanded = ''
+      title = 'Migrate to TF2'
+    else:
+      expanded = 'expanded'
+      title = key + ' compatibility'
+    parts.append(
+        _BOTTOM_COMPAT_TEMPLATE.format(
+            title=title, value=value, h_level=h_level, expanded=expanded))
+
+  return ''.join(parts)
+
+
+TABLE_HEADER = (
+    '')
+
+_TABLE_TEMPLATE = textwrap.dedent("""
+    {table_header}
+    {table_content}
+    
+
+    {table_footer}""")
+
+_TABLE_LINK_TEMPLATE = textwrap.dedent("""\
+    
+      
+        
+        View source on GitHub
+      
+    """)
+
+
+def top_source_link(location):
+  """Returns a source link with Github image, like the notebook butons."""
+
+  table_content = ''
+  table_footer = ''
+
+  if location and location.url:
+    if 'github.com' not in location.url:
+      table_footer = small_source_link(location)
+    else:
+      table_content = _TABLE_LINK_TEMPLATE.format(url=location.url)
+
+  table = _TABLE_TEMPLATE.format(
+      table_header=TABLE_HEADER,
+      table_content=table_content,
+      table_footer=table_footer)
+
+  return table
+
+
+def small_source_link(location, text='View source'):
+  """Returns a small source link."""
+  if location.url:
+    return ('{text}\n\n')
+  else:
+    return ''
+
+
+def build_collapsable_aliases(aliases: List[str]) -> str:
+  """Returns the top "Aliases" line."""
+
+  def join_aliases(aliases: List[str]) -> str:
+    return ', '.join('`{}`'.format(name) for name in aliases)
+
+  collapsable_template = textwrap.dedent("""\
+    
+    """)
+
+  main_alias_template = textwrap.dedent("""
+    Main aliases
+    
    {content}
    
+    """)
+
+  compat_alias_template = textwrap.dedent("""
+    Compat aliases for migration
+    
    See
+    Migration guide for
+    more details.
    
+    
    {content}
    
+    """)
+
+  main_aliases = []
+  compat_aliases = []
+
+  for alias in aliases:
+    if '__' in alias:
+      continue
+    elif 'compat.v' in alias:
+      compat_aliases.append(alias)
+    else:
+      main_aliases.append(alias)
+
+  alias_content = ''
+  if main_aliases:
+    alias_content += main_alias_template.format(
+        content=join_aliases(main_aliases))
+  if compat_aliases:
+    alias_content += compat_alias_template.format(
+        content=join_aliases(compat_aliases))
+
+  if alias_content:
+    return collapsable_template.format(content=alias_content) + '\n'
+
+  return alias_content
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/class_page.py b/tools/tensorflow_docs/api_generator/pretty_docs/class_page.py
new file mode 100644
index 00000000000..b14bfcee606
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/class_page.py
@@ -0,0 +1,664 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Page builder classes for class pages."""
+import collections
+import dataclasses
+import itertools
+import re
+import textwrap
+from typing import Any, Dict, List, NamedTuple, Optional
+
+from tensorflow_docs.api_generator import config
+from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import obj_type as obj_type_lib
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator import signature as signature_lib
+from tensorflow_docs.api_generator.pretty_docs import base_page
+
+from google.protobuf.message import Message as ProtoMessage
+
+
+class ClassPageBuilder(base_page.TemplatePageBuilder):
+  """Builds a markdown page from a `ClassPageInfo` instance."""
+  TEMPLATE = 'templates/class.jinja'
+
+  def __init__(self, page_info):
+    super().__init__(page_info)
+    # Split the methods into constructor and other methods.
+    self.methods = split_methods(page_info.methods)
+
+  def build_bases(self):
+    page_info = self.page_info
+    # If a class is a child class, add which classes it inherits from.
+    parts = []
+    if self.page_info.bases:
+      parts.append('\nInherits From: ')
+
+      link_template = '[`{short_name}`]({url})'
+      parts.append(', '.join(
+          link_template.format(**base._asdict()) for base in page_info.bases))
+      parts.append('\n')
+
+    return ''.join(parts)
+
+  def build_constructor(self):
+    page_info = self.page_info
+
+    # If the class has a constructor, build its signature.
+    # The signature will contain the class name followed by the arguments it
+    # takes.
+    parts = []
+    if self.methods.constructor is not None:
+      parts.append(
+          base_page.build_signature(
+              name=page_info.full_name,
+              signature=self.methods.constructor.signature,
+              decorators=self.methods.constructor.decorators))
+      parts.append('\n\n')
+
+    return ''.join(parts)
+
+  def build_class_docstring(self):
+
+    parts = merge_class_and_constructor_docstring(self.page_info,
+                                                  self.methods.constructor)
+
+    parts.append('\n\n')
+
+    return ''.join(parts)
+
+  def build_attr_block(self):
+    parts = []
+    if self.page_info.attr_block is not None:
+      parts.append(
+          base_page.format_docstring(
+              self.page_info.attr_block,
+              table_title_template='
    {title}
    '))
+      parts.append('\n\n')
+    return ''.join(parts)
+
+  def build_method_section(self, method):
+    return _build_method_section(method)
+
+  def build_other_member_section(self):
+    if self.page_info.other_members:
+      return base_page.build_other_members(
+          self.page_info.other_members,
+          title='
    Class Variables
    ',
+      )
+    else:
+      return ''
+
+
+class ClassPageInfo(base_page.PageInfo):
+  """Collects docs for a class page.
+
+  Attributes:
+    full_name: The full, main name, of the object being documented.
+    short_name: The last part of the full name.
+    py_object: The object being documented.
+    defined_in: A _FileLocation describing where the object was defined.
+    aliases: A list of full-name for all aliases for this object.
+    doc: A list of objects representing the docstring. These can all be
+      converted to markdown using str().
+    attributes: A dict mapping from "name" to a docstring
+    bases: A list of `base_page.MemberInfo` objects pointing to the docs for the
+      parent classes.
+    methods: A list of `MethodInfo` objects documenting the class' methods.
+    classes: A list of `base_page.MemberInfo` objects pointing to docs for any
+      nested classes.
+    other_members: A list of `base_page.MemberInfo` objects documenting any
+      other object's defined inside the class object (mostly enum style fields).
+    attr_block: A `TitleBlock` containing information about the Attributes of
+      the class.
+    inheritable_header: A header that may be placed on a base-class.
+  """
+  DEFAULT_BUILDER_CLASS = ClassPageBuilder
+
+  def __init__(self, *, api_node, **kwargs):
+    """Initialize a ClassPageInfo.
+
+    Args:
+      full_name: The full, main name, of the object being documented.
+      py_object: The object being documented.
+      **kwargs: Extra arguments.
+    """
+    super().__init__(api_node, **kwargs)
+
+    self._namedtuplefields = collections.OrderedDict()
+    if issubclass(api_node.py_object, tuple):
+      namedtuple_attrs = ('_asdict', '_fields', '_make', '_replace')
+      if all(hasattr(api_node.py_object, attr) for attr in namedtuple_attrs):
+        for name in api_node.py_object._fields:
+          self._namedtuplefields[name] = None
+
+    self._properties = collections.OrderedDict()
+    self._bases = None
+    self._methods = []
+    self._classes = []
+    self._other_members = []
+    self.attr_block = None
+
+  @property
+  def bases(self):
+    """Returns a list of `base_page.MemberInfo` objects pointing to the class' parents."""
+    return self._bases
+
+  @property
+  def inheritable_header(self) -> Optional[str]:
+    header = doc_controls.get_inheritable_header(self.py_object)
+    if header is not None:
+      header = textwrap.dedent(header)
+    return header
+
+  def set_attr_block(self, attr_block):
+    assert self.attr_block is None
+    self.attr_block = attr_block
+
+  def _set_bases(self):
+    """Builds the `bases` attribute, to document this class' parent-classes.
+
+    This method sets the `bases` to a list of `base_page.MemberInfo` objects
+    point to the
+    doc pages for the class' parents.
+    """
+    bases = []
+    for base in self.py_object.__mro__[1:]:
+      base_api_node = self.parser_config.api_tree.node_for_object(base)
+      if base_api_node is None:
+        continue
+      base_full_name = base_api_node.full_name
+      base_doc = parser.parse_md_docstring(base, base_full_name,
+                                           self.parser_config, self._extra_docs)
+      base_url = self.parser_config.reference_resolver.reference_to_url(
+          base_full_name)
+
+      link_info = base_page.MemberInfo(
+          short_name=base_full_name.split('.')[-1],
+          full_name=base_full_name,
+          py_object=base,
+          doc=base_doc,
+          url=base_url)
+      bases.append(link_info)
+
+    self._bases = bases
+
+  def _add_property(self, member_info: base_page.MemberInfo):
+    """Adds an entry to the `properties` list.
+
+    Args:
+      member_info: a `base_page.MemberInfo` describing the property.
+    """
+    doc = member_info.doc
+    # Clarify the default namedtuple docs-strings.
+    if re.match('Alias for field number [0-9]+', doc.brief):
+      new_brief = f'A `namedtuple` {doc.brief.lower()}'
+      doc = doc._replace(docstring_parts=[], brief=new_brief)
+
+    new_parts = [doc.brief]
+    # Strip args/returns/raises from property
+    new_parts.extend([
+        str(part)
+        for part in doc.docstring_parts
+        if not isinstance(part, parser.TitleBlock)
+    ])
+    new_parts.append('')
+    desc = '\n'.join(new_parts)
+
+    if member_info.short_name in self._namedtuplefields:
+      self._namedtuplefields[member_info.short_name] = desc
+    else:
+      self._properties[member_info.short_name] = desc
+
+  @property
+  def methods(self):
+    """Returns a list of `MethodInfo` describing the class' methods."""
+    return self._methods
+
+  def _add_method(
+      self,
+      member_info: base_page.MemberInfo,
+      defining_class: Optional[type],  # pylint: disable=g-bare-generic
+  ) -> None:
+    """Adds a `MethodInfo` entry to the `methods` list.
+
+    Args:
+      member_info: a `base_page.MemberInfo` describing the method.
+      defining_class: The `type` object where this method is defined.
+    """
+    if defining_class is None:
+      return
+
+    # Omit methods defined by namedtuple.
+    original_method = defining_class.__dict__[member_info.short_name]
+    if (hasattr(original_method, '__module__') and
+        (original_method.__module__ or '').startswith('namedtuple')):
+      return
+
+    # Some methods are often overridden without documentation. Because it's
+    # obvious what they do, don't include them in the docs if there's no
+    # docstring.
+    if (not member_info.doc.brief.strip() and
+        member_info.short_name in ['__del__', '__copy__']):
+      return
+
+    # If the current class py_object is a dataclass then use the class object
+    # instead of the __init__ method object because __init__ is a
+    # generated method on dataclasses (unless the definition used init=False)
+    # and `api_generator.get_source.get_source` doesn't work on generated
+    # methods (as the source file doesn't exist) which is required for
+    # signature generation.
+    if (dataclasses.is_dataclass(self.py_object) and
+        member_info.short_name == '__init__' and
+        self.py_object.__dataclass_params__.init):
+      is_dataclass = True
+      py_obj = self.py_object
+    else:
+      is_dataclass = False
+      py_obj = member_info.py_object
+
+    func_type = signature_lib.get_method_type(original_method,
+                                              member_info.short_name,
+                                              is_dataclass)
+    signature = signature_lib.generate_signature(
+        py_obj, self.parser_config, func_type=func_type)
+
+    decorators = signature_lib.extract_decorators(member_info.py_object)
+
+    defined_in = parser.get_defined_in(member_info.py_object,
+                                       self.parser_config)
+
+    method_info = MethodInfo.from_member_info(member_info, signature,
+                                              decorators, defined_in)
+    self._methods.append(method_info)
+
+  @property
+  def classes(self):
+    """Returns a list of `base_page.MemberInfo` pointing to any nested classes."""
+    return sorted(self._classes, key=lambda x: x.short_name)
+
+  def get_metadata_html(self) -> str:
+    meta_data = parser.Metadata(self.full_name)
+    for item in itertools.chain(self.classes, self.methods, self.other_members):
+      meta_data.append(item)
+
+    return meta_data.build_html()
+
+  def _add_class(self, member_info):
+    """Adds a `base_page.MemberInfo` for a nested class to `classes` list.
+
+    Args:
+      member_info: a `base_page.MemberInfo` describing the class.
+    """
+    self._classes.append(member_info)
+
+  @property
+  def other_members(self):
+    """Returns a list of `base_page.MemberInfo` describing any other contents."""
+    return self._other_members
+
+  def _add_other_member(self, member_info: base_page.MemberInfo):
+    """Adds an `base_page.MemberInfo` entry to the `other_members` list.
+
+    Args:
+      member_info: a `base_page.MemberInfo` describing the object.
+    """
+    self.other_members.append(member_info)
+
+  def _add_member(
+      self,
+      member_info: base_page.MemberInfo,
+      defining_class: Optional[type],  # pylint: disable=g-bare-generic
+  ) -> None:
+    """Adds a member to the class page."""
+    obj_type = obj_type_lib.ObjType.get(member_info.py_object)
+
+    if obj_type is obj_type_lib.ObjType.PROPERTY:
+      self._add_property(member_info)
+    elif obj_type is obj_type_lib.ObjType.CLASS:
+      if defining_class is None:
+        return
+      self._add_class(member_info)
+    elif obj_type is obj_type_lib.ObjType.CALLABLE:
+      self._add_method(member_info, defining_class)
+    elif obj_type is obj_type_lib.ObjType.OTHER:
+      # Exclude members defined by protobuf that are useless
+      if issubclass(self.py_object, ProtoMessage):
+        if (member_info.short_name.endswith('_FIELD_NUMBER') or
+            member_info.short_name in ['__slots__', 'DESCRIPTOR']):
+          return
+
+      self._add_other_member(member_info)
+
+  def collect_docs(self):
+    """Collects information necessary specifically for a class's doc page.
+
+    Mainly, this is details about the class's members.
+    """
+    py_class = self.py_object
+
+    self._set_bases()
+
+    class_path_node = self.parser_config.path_tree[self.api_node.path]
+    for _, path_node in sorted(class_path_node.children.items()):
+      # TODO(b/284321463): This should go in the `traverse` function.
+      # Don't document anything that is defined in common builtin types.
+      defining_class = parser.get_defining_class(py_class, path_node.short_name)
+      if defining_class in [
+          object,
+          type,
+          tuple,
+          dict,
+          list,
+          BaseException,
+          Exception,
+      ]:
+        continue
+
+      # The following condition excludes most protobuf-defined symbols.
+      if (defining_class and
+          defining_class.__name__ in ['CMessage', 'Message', 'MessageMeta']):
+        continue
+
+      if doc_controls.should_skip_class_attr(py_class, path_node.short_name):
+        continue
+
+      child_doc = parser.parse_md_docstring(path_node.py_object, self.full_name,
+                                            self.parser_config,
+                                            self._extra_docs)
+
+      child_url = self.parser_config.reference_resolver.reference_to_url(
+          path_node.full_name)
+
+      member_info = base_page.MemberInfo(path_node.short_name,
+                                         path_node.full_name,
+                                         path_node.py_object, child_doc,
+                                         child_url)
+      self._add_member(member_info, defining_class)
+
+    self.set_attr_block(self._augment_attributes(self.doc.docstring_parts))
+
+  def _augment_attributes(
+      self, docstring_parts: List[Any]) -> Optional[parser.TitleBlock]:
+    """Augments and deletes the "Attr" block of the docstring.
+
+    The augmented block is returned and then added to the markdown page by
+    pretty_docs.py. The existing Attribute block is deleted from the docstring.
+
+    Merges `namedtuple` fields and properties into the attrs block.
+
+    + `namedtuple` fields first, in order.
+    + Then the docstring `Attr:` block.
+    + Then any `properties` or `dataclass` fields not mentioned above.
+
+    Args:
+      docstring_parts: A list of docstring parts.
+
+    Returns:
+      Augmented "Attr" block.
+    """
+    attribute_block = None
+
+    for attr_block_index, part in enumerate(docstring_parts):
+      if isinstance(part, parser.TitleBlock) and part.title.startswith('Attr'):
+        raw_attrs = collections.OrderedDict(part.items)
+        old_block = part
+        break
+    else:
+      # Didn't find the attributes block, there may still be attributes so
+      # add a placeholder for them at the end.
+      raw_attrs = collections.OrderedDict()
+      old_block = None
+      attr_block_index = len(docstring_parts)
+      docstring_parts.append(None)
+
+    attrs = collections.OrderedDict()
+    # namedtuple fields first, in order.
+    for name, desc in self._namedtuplefields.items():
+      # If a namedtuple field has been filtered out, it's description will
+      # not have been set in loop in `collect_docs`, so skip fields with `None`
+      # as the description.
+      if desc is not None:
+        attrs[name] = desc
+    # the contents of the `Attrs:` block from the docstring
+    attrs.update(raw_attrs)
+
+    # properties and dataclass fields last.
+    for name, desc in self._properties.items():
+      # Don't overwrite existing items
+      attrs.setdefault(name, desc)
+
+    if dataclasses.is_dataclass(self.py_object):
+      for name, desc in self._dataclass_fields().items():
+        # Don't overwrite existing items
+        attrs.setdefault(name, desc)
+
+    if attrs:
+      if old_block is not None:
+        text = old_block.text
+      else:
+        text = ''
+      attribute_block = parser.TitleBlock(
+          title='Attributes', text=text, items=list(attrs.items())
+      )
+
+    # Delete the Attrs block if it exists or delete the placeholder.
+    del docstring_parts[attr_block_index]
+
+    return attribute_block
+
+  def _dataclass_fields(self):
+    fields = {
+        name: 'Dataclass field'
+        for name in self.py_object.__dataclass_fields__.keys()
+        if not name.startswith('_')
+    }
+
+    return fields
+
+
+class MethodInfo(NamedTuple):
+  """Described a method."""
+  short_name: str
+  full_name: str
+  py_object: Any
+  doc: parser.DocstringInfo
+  url: str
+  signature: signature_lib.TfSignature
+  decorators: List[str]
+  defined_in: Optional[parser.FileLocation]
+
+  @classmethod
+  def from_member_info(cls, method_info: base_page.MemberInfo,
+                       signature: signature_lib.TfSignature,
+                       decorators: List[str],
+                       defined_in: Optional[parser.FileLocation]):
+    """Upgrades a `base_page.MemberInfo` to a `MethodInfo`."""
+    return cls(
+        **method_info._asdict(),
+        signature=signature,
+        decorators=decorators,
+        defined_in=defined_in)
+
+
+class Methods(NamedTuple):
+  info_dict: Dict[str, MethodInfo]
+  constructor: MethodInfo
+
+
+def split_methods(methods: List[MethodInfo]) -> Methods:
+  """Splits the given methods list into constructors and the remaining methods.
+
+  If both `__init__` and `__new__` exist on the class, then prefer `__init__`
+  as the constructor over `__new__` to document.
+
+  Args:
+    methods: List of all the methods on the `ClassPageInfo` object.
+
+  Returns:
+    A `DocumentMethods` object containing a {method_name: method object}
+    dictionary and a constructor object.
+  """
+
+  # Create a method_name to methods object dictionary.
+  methods = sorted(methods, key=_method_sort)
+  method_info_dict = {method.short_name: method for method in methods}
+
+  # Pop the constructors from the dictionary.
+  init_constructor = method_info_dict.pop('__init__', None)
+  new_constructor = method_info_dict.pop('__new__', None)
+
+  constructor = None
+  # Prefers `__init__` over `__new__` as the constructor to document.
+  if init_constructor is not None:
+    constructor = init_constructor
+  elif new_constructor is not None:
+    constructor = new_constructor
+
+  return Methods(info_dict=method_info_dict, constructor=constructor)
+
+
+def merge_blocks(class_page_info: ClassPageInfo, ctor_info: MethodInfo):
+  """Helper function to merge TitleBlock in constructor and class docstring."""
+
+  # Get the class docstring. `.doc.docstring_parts` contain the entire
+  # docstring except for the one-line docstring that's compulsory.
+  class_doc = list(class_page_info.doc.docstring_parts)
+
+  # If constructor doesn't exist, return the class docstring as it is.
+  if ctor_info is None:
+    return class_doc
+
+  # Get the constructor's docstring parts.
+  constructor_doc = ctor_info.doc.docstring_parts
+
+  # If `Args`/`Arguments` and `Raises` already exist in the class docstring,
+  # then record them and don't lift those sections from the constructor.
+  existing_items_in_class = []
+  for item in class_doc:
+    if isinstance(item, parser.TitleBlock):
+      title = item.title
+      if title.startswith(('Args', 'Arguments')):
+        title = 'Arg'
+      existing_items_in_class.append(title)
+
+  # Extract the `Arguments`/`Args` from the constructor's docstring.
+  # A constructor won't contain `Args` and `Arguments` section at once.
+  # It can contain either one of these so check for both.
+  for block in constructor_doc:
+    if isinstance(block, parser.TitleBlock):
+      # If the block doesn't exist in class docstring, then lift the block.
+      if (block.title.startswith(('Args', 'Arguments', 'Raises')) and
+          not block.title.startswith(tuple(existing_items_in_class))):
+        class_doc.append(block)
+  return class_doc
+
+
+def merge_class_and_constructor_docstring(
+    class_page_info: ClassPageInfo,
+    ctor_info: MethodInfo,
+) -> List[str]:
+  """Merges the class and the constructor docstrings.
+
+  While merging, the following rules are followed:
+
+  * Only `Arguments` and `Raises` blocks from constructor are uplifted to the
+    class docstring. Rest of the stuff is ignored since it doesn't add much
+    value and in some cases the information is repeated.
+
+  * The `Raises` block is added to the end of the classes docstring.
+
+  * The `Arguments` or `Args` block is inserted before the `Attributes` section.
+    If `Attributes` section does not exist in the class docstring then add it
+    to the end.
+
+  * If the constructor does not exist on the class, then the class docstring
+    is returned as it is.
+
+  Args:
+    class_page_info: Object containing information about the class.
+    ctor_info: Object containing information about the constructor of the class.
+
+  Returns:
+    A list of strings containing the merged docstring.
+  """
+
+  def _create_class_doc(doc):
+    updated_doc = []
+    for item in doc:
+      updated_doc.append(
+          base_page.format_docstring(
+              item, table_title_template='
    {title}
    '))
+    return updated_doc
+
+  class_doc = merge_blocks(class_page_info, ctor_info)
+
+  return _create_class_doc(class_doc)
+
+
+def _method_sort(method):
+  """Create a sort-key tuple for a method based on its name."""
+  # All private methods will be at the end of the list in an alphabetically
+  # sorted order. All dunder methods will be above private methods and below
+  # public methods. Public methods will be at the top in an alphabetically
+  # sorted order.
+  method_name = method.short_name
+  if method_name.startswith('__'):
+    return (1, method_name)
+  if method_name.startswith('_'):
+    return (2, method_name)
+  return (-1, method_name)
+
+
+def _build_method_section(method_info, heading_level=3):
+  """Generates a markdown section for a method.
+
+  Args:
+    method_info: A `MethodInfo` object.
+    heading_level: An Int, which HTML heading level to use.
+
+  Returns:
+    A markdown string.
+  """
+  parts = []
+  heading = (''
+             '{short_name}'
+             '\n\n')
+  parts.append(
+      heading.format(heading_level=heading_level, **method_info._asdict()))
+
+  if method_info.defined_in:
+    parts.append(base_page.small_source_link(method_info.defined_in))
+
+  if method_info.signature is not None:
+    parts.append(
+        base_page.build_signature(
+            name=method_info.short_name,
+            signature=method_info.signature,
+            decorators=method_info.decorators))
+
+  parts.append(method_info.doc.brief + '\n')
+
+  parts.append(base_page.build_top_compat(method_info, h_level=4))
+
+  for item in method_info.doc.docstring_parts:
+    parts.append(
+        base_page.format_docstring(
+            item, table_title_template=None, anchors=False))
+
+  parts.append(base_page.build_bottom_compat(method_info, h_level=4))
+
+  parts.append('\n\n')
+  return ''.join(parts)
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/docs_for_object.py b/tools/tensorflow_docs/api_generator/pretty_docs/docs_for_object.py
new file mode 100644
index 00000000000..91f5af47292
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/docs_for_object.py
@@ -0,0 +1,85 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Create a `pretty_docs.base_page.PageInfo` from a python object."""
+from typing import Any, Dict, Optional, Type
+
+from tensorflow_docs.api_generator import config
+from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import doc_generator_visitor
+from tensorflow_docs.api_generator import obj_type as obj_type_lib
+from tensorflow_docs.api_generator.pretty_docs import base_page
+from tensorflow_docs.api_generator.pretty_docs import class_page
+from tensorflow_docs.api_generator.pretty_docs import function_page
+from tensorflow_docs.api_generator.pretty_docs import module_page
+from tensorflow_docs.api_generator.pretty_docs import type_alias_page
+
+_DEFAULT_PAGE_BUILDER_CLASSES = {
+    obj_type_lib.ObjType.CLASS: class_page.ClassPageInfo,
+    obj_type_lib.ObjType.CALLABLE: function_page.FunctionPageInfo,
+    obj_type_lib.ObjType.MODULE: module_page.ModulePageInfo,
+    obj_type_lib.ObjType.TYPE_ALIAS: type_alias_page.TypeAliasPageInfo,
+}
+
+PageBuilderDict = Dict[obj_type_lib.ObjType, Type[base_page.PageInfo]]
+
+
+def docs_for_object(
+    *,
+    api_node: doc_generator_visitor.ApiTreeNode,
+    parser_config: config.ParserConfig,
+    extra_docs: Optional[Dict[int, str]] = None,
+    search_hints: bool = True,
+    page_builder_classes: Optional[PageBuilderDict] = None,
+) -> base_page.PageInfo:
+  """Return a PageInfo object describing a given object from the TF API.
+
+  This function resolves `tf.symbol` references in the docstrings into links
+  to the appropriate location.
+
+  Args:
+    api_node: The ApiTreeNode for the object.
+    parser_config: A `config.ParserConfig` object.
+    extra_docs: Extra docs for symbols like public constants(list, tuple, etc)
+      that need to be added to the markdown pages created.
+    search_hints: If true include metadata search hints, else include a
+        "robots: noindex".
+    page_builder_classes: An optional dict of `{ObjectType:Type[PageInfo]}` for
+        overriding the default page builder classes.
+
+  Returns:
+    Either a subclass of `pretty_docs.base_page.PageInfo` depending on the type
+    of the python object being documented.
+
+  Raises:
+    RuntimeError: If an object is encountered for which we don't know how
+      to make docs.
+  """
+  if page_builder_classes is None:
+    page_builder_classes = _DEFAULT_PAGE_BUILDER_CLASSES
+
+  page_info_class = doc_controls.get_custom_page_builder_cls(api_node.py_object)
+  if page_info_class is None:
+    obj_type = obj_type_lib.ObjType.get(api_node.py_object)
+    page_info_class = page_builder_classes[obj_type]
+
+  page_info = page_info_class(
+      api_node=api_node,
+      search_hints=search_hints,
+      extra_docs=extra_docs,
+      parser_config=parser_config)
+
+  page_info.docs_for_object()
+
+  return page_info
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/function_page.py b/tools/tensorflow_docs/api_generator/pretty_docs/function_page.py
new file mode 100644
index 00000000000..54b184b1c1e
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/function_page.py
@@ -0,0 +1,99 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import textwrap
+from typing import Any, Optional
+
+from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator import signature as signature_lib
+from tensorflow_docs.api_generator.pretty_docs import base_page
+
+
+class FunctionPageBuilder(base_page.TemplatePageBuilder):
+  """Builds a markdown page from a `FunctionPageInfo` object."""
+  TEMPLATE = 'templates/function.jinja'
+
+  def format_docstring_part(self, part):
+    ttt = '
    {title}
    '
+    return base_page.format_docstring(part, table_title_template=ttt)
+
+  def build_signature(self):
+    return base_page.build_signature(
+        name=self.page_info.full_name,
+        signature=self.page_info.signature,
+        decorators=self.page_info.decorators)
+
+
+class FunctionPageInfo(base_page.PageInfo):
+  """Collects docs For a function Page.
+
+  Attributes:
+    full_name: The full, main name, of the object being documented.
+    short_name: The last part of the full name.
+    py_object: The object being documented.
+    defined_in: A _FileLocation describing where the object was defined.
+    aliases: A list of full-name for all aliases for this object.
+    doc: A list of objects representing the docstring. These can all be
+      converted to markdown using str().
+    signature: the parsed signature (see: generate_signature)
+    decorators: A list of decorator names.
+  """
+  DEFAULT_BUILDER_CLASS = FunctionPageBuilder
+
+  def __init__(self, *, api_node, **kwargs):
+    """Initialize a FunctionPageInfo.
+
+    Args:
+      api_node: the api tree node.
+      **kwargs: Extra arguments.
+    """
+    super().__init__(api_node, **kwargs)
+
+    self._signature = None
+    self._decorators = []
+
+  @property
+  def signature(self):
+    return self._signature
+
+  def collect_docs(self):
+    """Collect all information necessary to genertate the function page.
+
+    Mainly this is details about the function signature.
+    """
+    assert self.signature is None
+    self._signature = signature_lib.generate_signature(
+        self.py_object,
+        self.parser_config,
+        func_type=signature_lib.FuncType.FUNCTION,
+    )
+    self._decorators = signature_lib.extract_decorators(self.py_object)
+
+  @property
+  def decorators(self):
+    return list(self._decorators)
+
+  def add_decorator(self, dec):
+    self._decorators.append(dec)
+
+  @property
+  def header(self) -> Optional[str]:
+    header = doc_controls.get_header(self.py_object)
+    if header is not None:
+      header = textwrap.dedent(header)
+    return header
+
+  def get_metadata_html(self):
+    return parser.Metadata(self.full_name).build_html()
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/module_page.py b/tools/tensorflow_docs/api_generator/pretty_docs/module_page.py
new file mode 100644
index 00000000000..4154eb7a456
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/module_page.py
@@ -0,0 +1,169 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from typing import List
+
+from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import obj_type as obj_type_lib
+from tensorflow_docs.api_generator import parser
+
+from tensorflow_docs.api_generator.pretty_docs import base_page
+
+
+class ModulePageBuilder(base_page.TemplatePageBuilder):
+  """Builds a markdown page from a `ModulePageInfo` instance."""
+  TEMPLATE = 'templates/module.jinja'
+
+  def __init__(self, page_info: 'ModulePageInfo'):
+    super().__init__(page_info)
+
+  def build_other_member_section(self):
+    if self.page_info.other_members:
+      return base_page.build_other_members(
+          self.page_info.other_members,
+          title='
    Other Members
    ',
+      )
+    else:
+      return ''
+
+
+class ModulePageInfo(base_page.PageInfo):
+  """Collects docs for a module page.
+
+  Attributes:
+    full_name: The full, main name, of the object being documented.
+    short_name: The last part of the full name.
+    py_object: The object being documented.
+    defined_in: A _FileLocation describing where the object was defined.
+    aliases: A list of full-name for all aliases for this object.
+    doc: A list of objects representing the docstring. These can all be
+      converted to markdown using str().
+    classes: A list of `base_page.MemberInfo` objects pointing to docs for the
+      classes in this module.
+    functions: A list of `base_page.MemberInfo` objects pointing to docs for the
+      functions in this module
+    modules: A list of `base_page.MemberInfo` objects pointing to docs for the
+      modules in this module.
+    type_alias: A list of `base_page.MemberInfo` objects pointing to docs for
+      the type aliases in this module.
+    other_members: A list of `base_page.MemberInfo` objects documenting any
+      other object's defined on the module object (mostly enum style fields).
+  """
+  DEFAULT_BUILDER_CLASS = ModulePageBuilder
+
+  def __init__(self, *, api_node, **kwargs):
+    """Initialize a `ModulePageInfo`.
+
+    Args:
+      full_name: The full, main name, of the object being documented.
+      py_object: The object being documented.
+      **kwargs: Extra arguments.
+    """
+    super().__init__(api_node, **kwargs)
+
+    self._modules = []
+    self._classes = []
+    self._functions = []
+    self._other_members = []
+    self._type_alias = []
+
+  @property
+  def modules(self):
+    return self._modules
+
+  @property
+  def functions(self):
+    return self._functions
+
+  @property
+  def classes(self):
+    return self._classes
+
+  @property
+  def type_alias(self):
+    return self._type_alias
+
+  @property
+  def other_members(self):
+    return self._other_members
+
+  def _add_module(self, member_info: base_page.MemberInfo):
+    self._modules.append(member_info)
+
+  def _add_class(self, member_info: base_page.MemberInfo):
+    self._classes.append(member_info)
+
+  def _add_function(self, member_info: base_page.MemberInfo):
+    self._functions.append(member_info)
+
+  def _add_type_alias(self, member_info: base_page.MemberInfo):
+    self._type_alias.append(member_info)
+
+  def _add_other_member(self, member_info: base_page.MemberInfo):
+    self.other_members.append(member_info)
+
+  def get_metadata_html(self):
+    meta_data = parser.Metadata(self.full_name)
+
+    # Objects with their own pages are not added to the metadata list for the
+    # module, the module only has a link to the object page. No docs.
+    for item in self.other_members:
+      meta_data.append(item)
+
+    return meta_data.build_html()
+
+  def _add_member(self, member_info: base_page.MemberInfo) -> None:
+    """Adds members of the modules to the respective lists."""
+    obj_type = obj_type_lib.ObjType.get(member_info.py_object)
+    if obj_type is obj_type_lib.ObjType.MODULE:
+      self._add_module(member_info)
+    elif obj_type is obj_type_lib.ObjType.CLASS:
+      self._add_class(member_info)
+    elif obj_type is obj_type_lib.ObjType.CALLABLE:
+      self._add_function(member_info)
+    elif obj_type is obj_type_lib.ObjType.TYPE_ALIAS:
+      self._add_type_alias(member_info)
+    elif obj_type is obj_type_lib.ObjType.OTHER:
+      self._add_other_member(member_info)
+
+  def collect_docs(self):
+    """Collect information necessary specifically for a module's doc page.
+
+    Mainly this is information about the members of the module.
+    """
+    # the path_tree has nodes for all api-paths, not just the preferred paths.
+    module_path_node = self.parser_config.path_tree[self.api_node.path]
+    for (_, path_node) in sorted(module_path_node.children.items()):
+      member_doc = parser.parse_md_docstring(path_node.py_object,
+                                             self.full_name, self.parser_config,
+                                             self._extra_docs)
+
+      url = self.parser_config.reference_resolver.reference_to_url(
+          path_node.full_name)
+
+      member_info = base_page.MemberInfo(path_node.short_name,
+                                         path_node.full_name,
+                                         path_node.py_object, member_doc, url)
+      self._add_member(member_info)
+
+
+def _build_module_parts(module_parts: List[base_page.MemberInfo],
+                        template: str) -> List[str]:
+  mod_str_parts = []
+  for item in module_parts:
+    mod_str_parts.append(template.format(**item._asdict()))
+    if item.doc.brief:
+      mod_str_parts.append(': ' + item.doc.brief)
+    mod_str_parts.append('\n\n')
+  return mod_str_parts
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/pretty_docs_test.py b/tools/tensorflow_docs/api_generator/pretty_docs/pretty_docs_test.py
new file mode 100644
index 00000000000..27292a0306a
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/pretty_docs_test.py
@@ -0,0 +1,140 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for MD page generator."""
+
+import textwrap
+
+from absl.testing import absltest
+
+from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator import doc_generator_visitor
+
+from tensorflow_docs.api_generator.pretty_docs import base_page
+from tensorflow_docs.api_generator.pretty_docs import function_page
+
+
+class ParserTest(absltest.TestCase):
+
+  def test_github_source_link_in_table(self):
+    url = "https://github.com/tensorflow/docs/blob/master/path/to/file"
+    location = parser.FileLocation(base_url=url)
+    table = base_page.top_source_link(location)
+
+    expected = textwrap.dedent(f"""
+        
+        
+        
+
+        """)
+    self.assertEqual(expected, table)
+
+  def test_other_source_link_after_table(self):
+    url = "somewhere/else"
+    location = parser.FileLocation(base_url=url)
+    table = base_page.top_source_link(location)
+
+    expected = textwrap.dedent(f"""
+       
+
+       
+
+       View source
+
+       """)
+    self.assertEqual(expected, table)
+
+  def test_no_source_link(self):
+    location = parser.FileLocation()
+    table = base_page.top_source_link(location)
+
+    expected = textwrap.dedent("""
+       
+
+       
+
+       """)
+    self.assertEqual(expected, table)
+
+  def _get_test_page_builder(self, search_hints):
+
+    def test_function():
+      pass
+
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('abc',), py_object=test_function, children={})
+    page_info = function_page.FunctionPageInfo(
+        api_node=api_node, search_hints=search_hints)
+    docstring_info = parser.DocstringInfo(
+        brief='hello `tensorflow`',
+        docstring_parts=['line1', 'line2'],
+        compatibility={})
+    page_info.set_doc(docstring_info)
+    page_builder = function_page.FunctionPageBuilder(page_info)
+    return page_builder
+
+  def test_get_headers_global_hints(self):
+    page_builder = self._get_test_page_builder(search_hints=True)
+    result = page_builder.get_devsite_headers()
+
+    expected = textwrap.dedent("""\
+      description: hello tensorflow
+
+      
    
+      
+      
+      
    
+      """)
+
+    self.assertEqual(expected, result)
+
+  def test_get_headers_global_no_hints(self):
+    page_builder = self._get_test_page_builder(search_hints=False)
+    result = page_builder.get_devsite_headers()
+
+    expected = textwrap.dedent("""\
+      description: hello tensorflow
+      robots: noindex
+      """)
+
+    self.assertEqual(expected, result)
+
+  def test_get_headers_local_no_hints(self):
+    page_builder = self._get_test_page_builder(search_hints=True)
+    result = page_builder.get_devsite_headers()
+
+    @doc_controls.hide_from_search
+    def py_object():
+      pass
+
+    page_builder.page_info.py_object = py_object
+
+    result = page_builder.get_devsite_headers()
+
+    expected = textwrap.dedent("""\
+      description: hello tensorflow
+      robots: noindex
+      """)
+
+    self.assertEqual(expected, result)
+
+
+if __name__ == "__main__":
+  absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/templates/class.jinja b/tools/tensorflow_docs/api_generator/pretty_docs/templates/class.jinja
new file mode 100644
index 00000000000..d80a44ae740
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/templates/class.jinja
@@ -0,0 +1,69 @@
+{% extends "templates/page.jinja" %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block metadata %}
+{{ builder.top_source_link() }}
+
+{{ page_info.doc.brief -}}
+
+{% if page_info.inheritable_header %}
+
+
+{{ page_info.inheritable_header -}}
+{% endif %}
+
+{{ builder.build_bases() }}
+{{ builder.build_collapsable_aliases() -}}
+{{ builder.build_constructor() -}}
+{% endblock metadata %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block body %}
+{{- builder.top_compat() -}}
+
+
+{{ builder.build_class_docstring() -}}
+{{ builder.build_attr_block() }}
+{{- self.child_classes() -}}
+{{- self.methods() -}}
+
+{{- self.other_members() -}}
+
+{{- builder.bottom_compat() -}}
+{% endblock body %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block child_classes %}
+{%- if page_info.classes %}
+## Child Classes
+  {% for child_class in page_info.classes %}
+[`class {{child_class.short_name}}`]({{child_class.url}})
+
+  {% endfor %}
+{% endif -%}
+{% endblock child_classes%}
+
+{#----------------------------------------------------------------------------#}
+
+{% block methods %}
+{%- if builder.methods.info_dict -%}
+## Methods
+
+  {% for name, method in builder.methods.info_dict.items() %}
+    {{- builder.build_method_section(method) -}}
+  {% endfor %}
+
+
+{% endif -%}
+{% endblock methods%}
+
+{#----------------------------------------------------------------------------#}
+
+{% block other_members %}
+{%- if page_info.other_members -%}
+{{ builder.build_other_member_section() }}
+{% endif %}
+{% endblock other_members%}
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/templates/function.jinja b/tools/tensorflow_docs/api_generator/pretty_docs/templates/function.jinja
new file mode 100644
index 00000000000..3d445435d2c
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/templates/function.jinja
@@ -0,0 +1,26 @@
+{% extends "templates/page.jinja" %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block metadata %}
+{{ builder.top_source_link() }}
+
+{{ page_info.doc.brief }}
+
+{% if page_info.header %}
+
+
+{{ page_info.header -}}
+{% endif %}
+
+{{ builder.build_collapsable_aliases() -}}
+{% endblock metadata %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block body %}
+{% if page_info.signature %}
+{{ builder.build_signature() }}
+{% endif %}
+{{ super() -}}
+{% endblock %}
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/templates/module.jinja b/tools/tensorflow_docs/api_generator/pretty_docs/templates/module.jinja
new file mode 100644
index 00000000000..c31e2f3b6e9
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/templates/module.jinja
@@ -0,0 +1,100 @@
+{% extends "templates/page.jinja" %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block header %}
+{{ builder.get_devsite_headers() }}
+# Module: {{page_info.full_name}}
+
+
+{% endblock header%}
+
+{#----------------------------------------------------------------------------#}
+
+{% block body %}
+{{ builder.top_compat() -}}
+
+  {% for part in page_info.doc.docstring_parts %}
+    {{- builder.format_docstring_part(part) -}}
+  {% endfor %}
+{{ builder.bottom_compat() }}
+
+{{ self.modules() }}
+{{- self.classes() }}
+{{- self.functions() }}
+{{- self.type_aliases() }}
+{{- self.other_members() -}}
+{% endblock body %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block modules %}
+{%- if page_info.modules %}
+## Modules
+
+  {% for module in page_info.modules %}
+    {%if module.doc.brief%}
+[`{{module.short_name}}`]({{module.url}}) module: {{module.doc.brief}}
+    {%else%}
+[`{{module.short_name}}`]({{module.url}}) module
+    {%endif%}
+
+  {% endfor %}
+{% endif -%}
+{% endblock modules  %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block classes %}
+{%- if page_info.classes %}
+## Classes
+
+  {% for cls in page_info.classes %}
+    {%if cls.doc.brief%}
+[`class {{cls.short_name}}`]({{cls.url}}): {{cls.doc.brief}}
+    {%else%}
+[`class {{cls.short_name}}`]({{cls.url}})
+    {%endif%}
+
+  {% endfor %}
+{% endif -%}
+{% endblock classes%}
+
+{#----------------------------------------------------------------------------#}
+
+{% block functions %}
+{%- if page_info.functions -%}
+## Functions
+
+  {% for fun in page_info.functions %}
+    {%if fun.doc.brief%}
+[`{{fun.short_name}}(...)`]({{fun.url}}): {{fun.doc.brief}}
+    {%else%}
+[`{{fun.short_name}}(...)`]({{fun.url}})
+    {%endif%}
+
+  {% endfor %}
+{% endif -%}
+{% endblock functions%}
+
+{#----------------------------------------------------------------------------#}
+
+{% block type_aliases %}
+{%- if page_info.type_alias -%}
+## Type Aliases
+
+  {% for alias in page_info.type_alias %}
+[`{{alias.short_name}}`]({{alias.url}})
+
+  {% endfor %}
+{% endif -%}
+{% endblock type_aliases%}
+
+{#----------------------------------------------------------------------------#}
+
+{% block other_members %}
+{%- if page_info.other_members -%}
+{{ builder.build_other_member_section() }}
+{% endif %}
+{% endblock other_members%}
+
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/templates/page.jinja b/tools/tensorflow_docs/api_generator/pretty_docs/templates/page.jinja
new file mode 100644
index 00000000000..30073a1ec46
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/templates/page.jinja
@@ -0,0 +1,43 @@
+{%- extends 'templates/root.jinja' -%}
+
+{#----------------------------------------------------------------------------#}
+
+{% block content %}
+  {{- self.header() -}}
+  {{- self.metadata() -}}
+  {{- self.body() -}}
+{% endblock content %}}
+
+{#----------------------------------------------------------------------------#}
+
+{% block header %}
+{{ builder.get_devsite_headers() }}
+# {{page_info.full_name}}
+
+
+{% endblock header%}
+
+{#----------------------------------------------------------------------------#}
+
+{% block metadata %}
+{{ builder.top_source_link() }}
+
+{{ page_info.doc.brief }}
+
+{{ builder.build_collapsable_aliases() -}}
+{% endblock metadata %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block body %}
+
+{{ builder.top_compat() -}}
+
+  {% for part in page_info.doc.docstring_parts %}
+    {{- builder.format_docstring_part(part) -}}
+  {% endfor %}
+{{ builder.bottom_compat() -}}
+{% endblock body %}
+
+
+
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/templates/root.jinja b/tools/tensorflow_docs/api_generator/pretty_docs/templates/root.jinja
new file mode 100644
index 00000000000..a4d77de6349
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/templates/root.jinja
@@ -0,0 +1 @@
+{{- self.content() -}}
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/templates/type_alias.jinja b/tools/tensorflow_docs/api_generator/pretty_docs/templates/type_alias.jinja
new file mode 100644
index 00000000000..d4550f9f608
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/templates/type_alias.jinja
@@ -0,0 +1,20 @@
+{% extends "templates/page.jinja" %}
+
+
+{% block metadata %}
+This symbol is a **type alias**.
+
+{{ page_info.doc.brief }}
+
+{% endblock metadata %}
+
+{#----------------------------------------------------------------------------#}
+
+{% block body %}
+{% if page_info.signature %}
+#### Source:
+
+{{ builder.build_signature() }}
+{% endif %}
+{{ super() -}}
+{% endblock %}
diff --git a/tools/tensorflow_docs/api_generator/pretty_docs/type_alias_page.py b/tools/tensorflow_docs/api_generator/pretty_docs/type_alias_page.py
new file mode 100644
index 00000000000..90441eadcf0
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/pretty_docs/type_alias_page.py
@@ -0,0 +1,157 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Bage builder classes for type alias pages."""
+import textwrap
+import types
+import typing
+from typing import Any, Dict, List
+
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator import signature as signature_lib
+from tensorflow_docs.api_generator.pretty_docs import base_page
+
+
+class TypeAliasPageBuilder(base_page.TemplatePageBuilder):
+  """Builds a markdown page from a `TypeAliasPageBuilder` object."""
+  TEMPLATE = 'templates/type_alias.jinja'
+
+  def build_signature(self):
+    return base_page.build_signature(
+        name=self.page_info.short_name,
+        signature=self.page_info.signature,
+        decorators=None,
+        type_alias=True)
+
+
+class TypeAliasPageInfo(base_page.PageInfo):
+  """Collects docs For a type alias page.
+
+  Attributes:
+    full_name: The full, main name, of the object being documented.
+    short_name: The last part of the full name.
+    py_object: The object being documented.
+    defined_in: A _FileLocation describing where the object was defined.
+    aliases: A list of full-name for all aliases for this object.
+    doc: A list of objects representing the docstring. These can all be
+      converted to markdown using str().
+    signature: the parsed signature (see: generate_signature)
+    decorators: A list of decorator names.
+  """
+  DEFAULT_BUILDER_CLASS = TypeAliasPageBuilder
+
+  def __init__(self, *, api_node, **kwargs) -> None:
+    """Initialize a `TypeAliasPageInfo`.
+
+    Args:
+      full_name: The full, main name, of the object being documented.
+      py_object: The object being documented.
+      **kwargs: Extra arguments.
+    """
+
+    super().__init__(api_node, **kwargs)
+    self._signature = None
+
+  @property
+  def signature(self):
+    return self._signature
+
+  def _custom_join(self, args: List[str], origin: str) -> str:
+    """Custom join for Callable and other type hints.
+
+    Args:
+      args: Args of a type annotation object returned by `__args__`.
+      origin: Origin of a type annotation object returned by `__origin__`.
+
+    Returns:
+      A joined string containing the representation of a type annotation.
+    """
+    if 'Callable' in origin:
+      if args[0] == '...':
+        return 'Callable[%s]' % ', '.join(args)
+      else:
+        return 'Callable[[%s], %s]' % (', '.join(args[:-1]), args[-1])
+    elif 'UnionType' in origin:
+      return ' | '.join(args)
+
+    return '%s[%s]' % (origin, ', '.join(args))
+
+  def _link_type_args(self, obj: Any, reverse_index: Dict[int, str],
+                      linker: signature_lib.FormatArguments) -> str:
+    """Recurses into typehint object and links known objects to their pages."""
+    arg_full_name = reverse_index.get(id(obj), None)
+    if arg_full_name is not None:
+      return linker.get_link(arg_full_name)
+
+    result = []
+    if getattr(obj, '__args__', None):
+      for arg in obj.__args__:
+        result.append(self._link_type_args(arg, reverse_index, linker))
+      origin_str = typing._type_repr(typing.get_origin(obj))  # pylint: disable=protected-access # pytype: disable=module-attr
+      return self._custom_join(result, origin_str)
+    else:
+      return typing._type_repr(obj)  # pylint: disable=protected-access # pytype: disable=module-attr
+
+  def collect_docs(self) -> None:
+    """Collect all information necessary to genertate the function page.
+
+    Mainly this is details about the function signature.
+
+    For the type alias signature, the args are extracted and replaced with the
+    full_name if the object is present in `parser_config.reverse_index`. They
+    are also linkified to point to that symbol's page.
+
+    For example (If generating docs for symbols in TF library):
+
+    ```
+    X = Union[int, str, bool, tf.Tensor, np.ndarray]
+    ```
+
+    In this case `tf.Tensor` will get linked to that symbol's page.
+    Note: In the signature `tf.Tensor` is an object, so it will show up as
+    `tensorflow.python.framework.ops.Tensor`. That's why we need to query
+    `parser_config.reverse_index` to get the full_name of the object which will
+    be `tf.Tensor`. Hence the signature will be:
+
+    ```
+    X = Union[int, str, bool, tf.Tensor, np.ndarray]
+    ```
+    """
+    assert self.signature is None
+
+    linker = signature_lib.FormatArguments(parser_config=self.parser_config)
+
+    sig_args = []
+    if typing.get_origin(self.py_object):
+      for arg_obj in self.py_object.__args__:
+        sig_args.append(
+            self._link_type_args(arg_obj, self.parser_config.reverse_index,
+                                 linker))
+
+    sig_args_str = textwrap.indent(',\n'.join(sig_args), '    ')
+    if typing.get_origin(self.py_object):
+      origin_str = typing._type_repr(typing.get_origin(self.py_object))  # pylint: disable=protected-access # pytype: disable=module-attr
+      sig = f'{origin_str}[\n{sig_args_str}\n]'
+    else:
+      sig = repr(self.py_object)
+
+    # pytype: enable=module-attr
+
+    # Starting in Python 3.7, the __origin__ attribute of typing constructs
+    # contains the equivalent runtime class rather than the construct itself
+    # (e.g., typing.Callable.__origin__ is collections.abc.Callable).
+    self._signature = sig.replace('typing.', '').replace('collections.abc.', '')
+
+  def get_metadata_html(self) -> str:
+    return parser.Metadata(self.full_name).build_html()
diff --git a/tools/tensorflow_docs/api_generator/public_api.py b/tools/tensorflow_docs/api_generator/public_api.py
index ee03ea8000f..e6a994bff5b 100644
--- a/tools/tensorflow_docs/api_generator/public_api.py
+++ b/tools/tensorflow_docs/api_generator/public_api.py
@@ -14,30 +14,76 @@
 # ==============================================================================
 """Visitor restricting traversal to only the public tensorflow API."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
+import ast
+import dataclasses
 import inspect
-
+import os
+import pathlib
+import sys
+import types
+import typing
+from typing import Any, Callable, Dict, Iterable, List, Sequence, Tuple, Union
 
 from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import doc_generator_visitor
+from tensorflow_docs.api_generator import get_source
+
+from google.protobuf.message import Message as ProtoMessage
 
 try:
-  import typing  # pylint: disable=g-import-not-at-top
-  _TYPING = {id(value) for value in typing.__dict__.values()}
-  del typing
+  import proto  # pylint: disable=g-import-not-at-top  # pytype: disable=import-error
 except ImportError:
-  _TYPING = {}
+  proto = None
+
+
+_TYPING_IDS = frozenset(
+    id(obj)
+    for obj in typing.__dict__.values()
+    if not doc_generator_visitor.maybe_singleton(obj))
+
 
+Children = Iterable[Tuple[str, Any]]
+ApiFilter = Callable[[Tuple[str, ...], Any, Children], Children]
 
-def ignore_typing(path, parent, children):
+
+def get_module_base_dirs(module) -> Tuple[pathlib.Path, ...]:
+  """Returns the list of base_dirs.
+
+  Args:
+    module: A python module object.
+
+  Returns:
+    A tuple of paths. Usually 1 unless the module is a namespace package.
+  """
+  if not hasattr(module, '__file__'):
+    return ()
+
+  mod_file = module.__file__
+  if mod_file is None:
+    # namespace packages have `__file__=None` but the directory *paths* are
+    # available in `__path__._path`.
+    # https://www.python.org/dev/peps/pep-0451/
+    # This is a **list of paths**.
+    base_dirs = module.__path__._path  # pylint: disable=protected-access  # pytype: disable=attribute-error
+  elif mod_file.endswith('__init__.py'):
+    # A package directory will have an `__init__.py`,
+    # accept anything in that directory.
+    base_dirs = [os.path.dirname(mod_file)]
+  else:
+    # This is a single file module. The file is the base_dir.
+    base_dirs = [mod_file]
+
+  return tuple(pathlib.Path(b) for b in base_dirs)
+
+
+def ignore_typing(path: Sequence[str], parent: Any,
+                  children: Children) -> Children:
   """Removes all children that are members of the typing module.
 
   Arguments:
-    path: A tuple or name parts forming the attribute-lookup path to this
+    path: A tuple of name parts forming the attribute-lookup path to this
       object. For `tf.keras.layers.Dense` path is:
-        ("tf","keras","layers","Dense")
+      ("tf","keras","layers","Dense")
     parent: The parent object.
     children: A list of (name, value) pairs. The attributes of the patent.
 
@@ -46,13 +92,16 @@ def ignore_typing(path, parent, children):
   """
   del path
   del parent
-  children = [
-      (name, value) for (name, value) in children if id(value) not in _TYPING
-  ]
+
+  children = [(name, child_obj)
+              for (name, child_obj) in children
+              if id(child_obj) not in _TYPING_IDS]
+
   return children
 
 
-def local_definitions_filter(path, parent, children):
+def local_definitions_filter(path: Sequence[str], parent: Any,
+                             children: Children) -> Children:
   """Filters children recursively.
 
   Only returns those defined in this package.
@@ -129,79 +178,314 @@ def util_2
   return filtered_children
 
 
-class PublicAPIFilter(object):
-  """Visitor to use with `traverse` to filter just the public API."""
-
-  def __init__(self,
-               base_dir,
-               do_not_descend_map=None,
-               private_map=None):
-    """Constructor.
-
-    Args:
-      base_dir: The directory to take source file paths relative to.
-      do_not_descend_map: A mapping from dotted path like "tf.symbol" to a list
-        of names. Included names will have no children listed.
-      private_map: A mapping from dotted path like "tf.symbol" to a list
-        of names. Included names will not be listed at that location.
-    """
-    self._base_dir = base_dir
-    self._do_not_descend_map = do_not_descend_map or {}
-    self._private_map = private_map or {}
-
-  ALLOWED_PRIVATES = frozenset([
-      '__abs__', '__add__', '__and__', '__bool__', '__call__', '__concat__',
-      '__contains__', '__div__', '__enter__', '__eq__', '__exit__',
-      '__floordiv__', '__ge__', '__getitem__', '__gt__', '__init__',
-      '__invert__', '__iter__', '__le__', '__len__', '__lt__', '__matmul__',
-      '__mod__', '__mul__', '__new__', '__ne__', '__neg__', '__pos__',
-      '__nonzero__', '__or__', '__pow__', '__radd__', '__rand__', '__rdiv__',
-      '__rfloordiv__', '__rmatmul__', '__rmod__', '__rmul__', '__ror__',
-      '__rpow__', '__rsub__', '__rtruediv__', '__rxor__', '__sub__',
-      '__truediv__', '__xor__', '__version__'
-  ])
-
-  def _is_private(self, path, name, obj):
-    """Return whether a name is private."""
-    # Skip objects blocked by doc_controls.
-    if doc_controls.should_skip(obj):
-      return True
-
-    # Skip modules outside of the package root.
+def _get_imported_symbols(obj: Union[str, types.ModuleType]):
+  """Returns a list of symbol names imported by the given `obj`."""
+
+  class ImportNodeVisitor(ast.NodeVisitor):
+    """An `ast.Visitor` that collects the names of imported symbols."""
+
+    def __init__(self):
+      self.imported_symbols = []
+
+    def _add_imported_symbol(self, node):
+      for alias in node.names:
+        name = alias.asname or alias.name
+        if name == '*':
+          continue
+        if '.' in name:
+          continue
+        self.imported_symbols.append(name)
+
+    def visit_Import(self, node):  # pylint: disable=invalid-name
+      self._add_imported_symbol(node)
+
+    def visit_ImportFrom(self, node):  # pylint: disable=invalid-name
+      self._add_imported_symbol(node)
+
+  tree = get_source.get_ast(obj)
+  if tree is None:
+    return []
+
+  visitor = ImportNodeVisitor()
+  visitor.visit(tree)
+  return visitor.imported_symbols
+
+
+def explicit_package_contents_filter(path: Sequence[str], parent: Any,
+                                     children: Children) -> Children:
+  """Filter submodules, only keep what's explicitly included.
+
+  This filter only affects the visibility of **modules**. Other objects are not
+  affected.
+
+  This filter is useful if you explicitly define your API in the packages of
+  your library (the __init__.py files), but do not expliticly define that API
+  in the `__all__` variable of each module. The purpose is to make it easier to
+  maintain that API.
+
+  **This filter makes it so that modules are only documented where they are
+  explicitly imported in an __init__.py**
+
+  ### Packages
+
+  Lots of imports **indirectly** inject modules into package namespaces, this
+  filter helps you ignore those. Anywhere you `import pkg.sub1` it will inject
+  `sub1` into the `pkg` namsspace.
+
+  When filtering a package it only keeps modules that are **directly**
+  impotrted in the package. This code, injects `[sub0, sub1, sub2, sub3, sub4,
+  sub_sub1, *]` into the pkg namespace:
+
+  pkg/__init__.py
+
+  ```
+  import sub0
+  import pkg.sub1
+  from pkg import sub2
+  from pkg.sub3 import sub_sub1
+  from pkg.sub4 import *
+  ```
+
+  But this filter will only keep the modules `[sub0, sub2, sub_sub1]` in the
+  docs for `pkg`.
+
+  ### Regular modules
+
+  For regular modules all modules in the namespace are assumed to be
+  implementation details and/or documented in their source location. For example
+  in this package:
+
+  ```
+  pkg/
+    __init__.py
+    sub1.py
+    sub2.py
+  ```
+
+  If you `import sub2` in `__init__.py` `sub2` will documented in `pkg`
+  But if you `import sub2` in `sub1.py` `sub2` will not be documented in `sub1`
+
+  Args:
+    path: A tuple of names forming the path to the object.
+    parent: The parent object.
+    children: A list of (name, value) tuples describing the attributes of the
+      patent.
+
+  Returns:
+    A filtered list of children `(name, value)` pairs.
+  """
+  del path  # Unused
+  is_parent_module = inspect.ismodule(parent)
+  is_parent_package = is_parent_module and hasattr(parent, '__path__')
+  if is_parent_package:
+    imported_symbols = _get_imported_symbols(parent)
+  filtered_children = []
+  for child in children:
+    name, obj = child
     if inspect.ismodule(obj):
-      if hasattr(obj, '__file__'):
-        # `startswith` will match any item in a tuple.
-        if not obj.__file__.startswith(self._base_dir):
-          return True
+      # Do not include modules in a package not explicitly imported by the
+      # package.
+      if is_parent_package and name not in imported_symbols:
+        continue
+      # Do not include modules imported by a module that is not a package.
+      if is_parent_module and not is_parent_package:
+        continue
+    filtered_children.append(child)
+  return filtered_children
 
-    # Skip objects blocked by the private_map
-    if name in self._private_map.get('.'.join(path), []):
-      return True
 
-    # Skip "_" hidden attributes
-    if name.startswith('_') and name not in self.ALLOWED_PRIVATES:
-      return True
+ALLOWED_DUNDER_METHODS = frozenset([
+    '__abs__', '__add__', '__and__', '__array__', '__bool__', '__call__',
+    '__concat__', '__contains__', '__div__', '__enter__', '__eq__', '__exit__',
+    '__floordiv__', '__ge__', '__getitem__', '__gt__', '__init__', '__invert__',
+    '__iter__', '__le__', '__len__', '__lshift__', '__lt__', '__matmul__',
+    '__mod__', '__mul__', '__new__', '__ne__', '__neg__', '__pos__',
+    '__nonzero__', '__or__', '__pow__', '__radd__', '__rand__', '__rdiv__',
+    '__rfloordiv__', '__rlshift__', '__rmatmul__', '__rmod__', '__rmul__',
+    '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
+    '__rtruediv__', '__rxor__', '__sub__', '__truediv__', '__xor__',
+    '__version__'
+])
 
-    return False
 
-  def __call__(self, path, parent, children):
-    """Visitor interface, see `traverse` for details."""
+@dataclasses.dataclass
+class FailIfNestedTooDeep:
+  max_depth: int
 
-    # Avoid long waits in cases of pretty unambiguous failure.
+  def __call__(self, path: Sequence[str], parent: Any,
+               children: Children) -> Children:
     if inspect.ismodule(parent) and len(path) > 10:
-      raise RuntimeError(
-          'Modules nested too deep:\n\n%s\n\nThis is likely a '
-          'problem with an accidental public import.' % ('.'.join(path)))
+      raise RuntimeError('Modules nested too deep:\n\n{}\n\nThis is likely a '
+                         'problem with an accidental public import.'.format(
+                             '.'.join(path)))
+    return children
+
+
+@dataclasses.dataclass
+class FilterBaseDirs:
+  """A class for filtering based on a list of allowed parent directories."""
+  base_dirs: Sequence[pathlib.Path]
+
+  def __call__(self, path: Sequence[str], parent: Any,
+               children: Children) -> Children:
+    for name, child in children:
+      if not inspect.ismodule(child):
+        yield name, child
+        continue
+      mod_base_dirs = get_module_base_dirs(child)
+      # This check only handles normal packages/modules. Namespace-package
+      # contents will get filtered when the submodules are checked.
+      if len(mod_base_dirs) == 1:
+        mod_base_dir = mod_base_dirs[0]
+        # Check that module is, or is in one of the `self._base_dir`s
+        if not (any(base in mod_base_dir.parents for base in self.base_dirs) or
+                mod_base_dir in self.base_dirs):
+          continue
+      yield name, child
+
+
+@dataclasses.dataclass
+class FilterPrivateMap:
+  private_map: Dict[str, List[str]]
+
+  def __call__(self, path: Sequence[str], parent: Any,
+               children: Children) -> Children:
+    if self.private_map is None:
+      yield from children
+
+    for name, child in children:
+      if name in self.private_map.get('.'.join(path), []):
+        continue
+      yield (name, child)
+
+
+def filter_private_symbols(path: Sequence[str], parent: Any,
+                           children: Children) -> Children:
+  del path
+  del parent
+  for name, child in children:
+    # Skip "_" hidden attributes
+    if name.startswith('_') and name not in ALLOWED_DUNDER_METHODS:
+      if not doc_controls.should_doc_private(child):
+        continue
+    yield (name, child)
+
+
+def filter_doc_controls_skip(path: Sequence[str], parent: Any,
+                             children: Children) -> Children:
+  del path
+  for name, child in children:
+    if doc_controls.should_skip(child):
+      continue
+    if isinstance(parent, type):
+      if doc_controls.should_skip_class_attr(parent, name):
+        continue
+    yield (name, child)
+
+
+def filter_module_all(path: Sequence[str], parent: Any,
+                      children: Children) -> Children:
+  """Filters module children based on the "__all__" arrtibute.
+
+  Args:
+    path: API to this symbol
+    parent: The object
+    children: A list of (name, object) pairs.
+
+  Returns:
+    `children` filtered to respect __all__
+  """
+  del path
+  if not (inspect.ismodule(parent) and hasattr(parent, '__all__')):
+    return children
+  module_all = set(parent.__all__)
+  children = [(name, value) for (name, value) in children if name in module_all]
+
+  return children
 
-    # No children if "do_not_descend" is set.
-    parent_path = '.'.join(path[:-1])
-    name = path[-1]
-    if name in self._do_not_descend_map.get(parent_path, []):
-      del children[:]
 
-    # Remove things that are not visible.
-    for child_name, child in list(children):
-      if self._is_private(path, child_name, child):
-        children.remove((child_name, child))
+def add_proto_fields(path: Sequence[str], parent: Any,
+                     children: Children) -> Children:
+  """Add properties to Proto classes, so they can be documented.
 
+  Warning: This inserts the Properties into the class so the rest of the system
+  is unaffected. This patching is acceptable because there is never a reason to
+  run other tensorflow code in the same process as the doc generator.
+
+  Args:
+    path: API to this symbol
+    parent: The object
+    children: A list of (name, object) pairs.
+
+  Returns:
+    `children` with proto fields added as properties.
+  """
+  del path
+  if not inspect.isclass(parent):
     return children
+
+  real_parent = parent
+  if not issubclass(parent, ProtoMessage):
+    if proto is not None:
+      if issubclass(parent, proto.message.Message):
+        parent = parent.pb()
+        children = [
+            (name, value) for (name, value) in children if name != 'meta'
+        ]
+      else:
+        return children
+
+  descriptor = getattr(parent, 'DESCRIPTOR', None)
+  if descriptor is None:
+    return children
+  fields = descriptor.fields
+  if not fields:
+    return children
+
+  field = fields[0]
+  # Make the dictionaries mapping from int types and labels to type and
+  # label names.
+  field_types = {
+      getattr(field, name): name
+      for name in dir(field)
+      if name.startswith('TYPE')
+  }
+
+  labels = {
+      getattr(field, name): name
+      for name in dir(field)
+      if name.startswith('LABEL')
+  }
+
+  field_properties = {}
+
+  for field in fields:
+    name = field.name
+    doc_parts = []
+
+    label = labels[field.label].lower().replace('label_', '')
+    if label != 'optional':
+      doc_parts.append(label)
+
+    type_name = field_types[field.type]
+    if type_name == 'TYPE_MESSAGE':
+      type_name = field.message_type.name
+    elif type_name == 'TYPE_ENUM':
+      type_name = field.enum_type.name
+    else:
+      type_name = type_name.lower().replace('type_', '')
+
+    doc_parts.append(type_name)
+    doc_parts.append(name)
+    doc = '`{}`'.format(' '.join(doc_parts))
+    prop = property(fget=lambda x: x, doc=doc)
+    field_properties[name] = prop
+
+  for name, prop in field_properties.items():
+    setattr(real_parent, name, prop)
+
+  children = dict(children)
+  children.update(field_properties)
+  children = sorted(children.items(), key=lambda item: item[0])
+
+  return children
diff --git a/tools/tensorflow_docs/api_generator/public_api_test.py b/tools/tensorflow_docs/api_generator/public_api_test.py
index db9e5285021..4130099d0b8 100644
--- a/tools/tensorflow_docs/api_generator/public_api_test.py
+++ b/tools/tensorflow_docs/api_generator/public_api_test.py
@@ -14,16 +14,16 @@
 # ==============================================================================
 """Tests for tensorflow.tools.common.public_api."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import inspect
+import pathlib
 import types
 
 import typing
 
 from absl.testing import absltest
+# This import is using to test
+from tensorflow_docs import api_generator
+from tensorflow_docs.api_generator import doc_controls
 from tensorflow_docs.api_generator import public_api
 
 
@@ -42,79 +42,24 @@ def __call__(self, path, parent, children):
       self.last_children = list(children)  # Make a copy to preserve state.
       return children
 
-  def test_call_forward(self):
-    visitor = self.TestVisitor()
-
-    api_visitors = [public_api.PublicAPIFilter(base_dir='/'), visitor]
-
-    path = ('tf', 'test')
-    parent = 'dummy'
-    children = [('name1', 'thing1'), ('name2', 'thing2')]
-
-    for api_visitor in api_visitors:
-      children = api_visitor(path, parent, children)
-
-    self.assertEqual(set([(
-        'tf',
-        'test',
-    )]), visitor.symbols)
-    self.assertEqual('dummy', visitor.last_parent)
-    self.assertEqual([('name1', 'thing1'), ('name2', 'thing2')],
-                     visitor.last_children)
-
-  def test_private_child_removal(self):
-    visitor = self.TestVisitor()
-    api_visitors = [
-        public_api.PublicAPIFilter(base_dir='/'),
-        visitor,
-    ]
-
-    children = [('name1', 'thing1'), ('_name2', 'thing2')]
-    path = ('tf', 'test')
-    parent = 'dummy'
-    for api_visitor in api_visitors:
-      children = api_visitor(path, parent, children)
-
-    # Make sure the private symbols are removed before the visitor is called.
-    self.assertEqual([('name1', 'thing1')], visitor.last_children)
-    self.assertEqual([('name1', 'thing1')], children)
+  def test_filter_private_symbols(self):
+    module = types.ModuleType('module')
+    module.a = 1
+    module._b = 2
 
-  def test_no_descent_child_removal(self):
-    visitor = self.TestVisitor()
+    result = public_api.filter_private_symbols(('module'), module,
+                                               [('a', module.a),
+                                                ('_b', module._b)])
+    self.assertEqual([('a', module.a)], list(result))
 
-    api_visitors = [
-        public_api.PublicAPIFilter(
-            base_dir='/', do_not_descend_map={'tf.test': ['mock']}), visitor
-    ]
+  def test_private_map_filter(self):
+    private_map_filter = public_api.FilterPrivateMap({'tf.test': ['mock']})
+    result = private_map_filter(
+        path=('tf', 'test'),
+        parent='dummy',
+        children=[('name1', 'thing1'), ('mock', 'thing2')])
 
-    children = [('name1', 'thing1'), ('name2', 'thing2')]
-    path = ('tf', 'test', 'mock')
-    parent = 'dummy'
-
-    for api_visitor in api_visitors:
-      children = api_visitor(path, parent, children)
-
-    # Make sure not-to-be-descended-into symbols's children are removed.
-    self.assertEqual([], visitor.last_children)
-    self.assertEqual([], children)
-
-  def test_private_map_child_removal(self):
-    visitor = self.TestVisitor()
-
-    api_visitors = [
-        public_api.PublicAPIFilter(
-            base_dir='/', private_map={'tf.test': ['mock']}), visitor
-    ]
-
-    children = [('name1', 'thing1'), ('mock', 'thing2')]
-    path = ('tf', 'test')
-    parent = 'dummy'
-
-    for api_visitor in api_visitors:
-      children = api_visitor(path, parent, children)
-    # Make sure private aliases are removed.
-    self.assertEqual([('name1', 'thing1')], visitor.last_children)
-    self.assertEqual([('name1', 'thing1')], children)
+    self.assertEqual([('name1', 'thing1')], list(result))
 
   def test_local_definitions_filter(self):
     tf = types.ModuleType('tf')
@@ -153,12 +98,133 @@ def public_members(obj):
 
     self.assertCountEqual([], filtered_names)
 
+  def test_explicit_package_contents_filter_removes_modules_not_explicitly_imported(
+      self):
+    path = ('tensorflow_docs', 'api_generator')
+    parent = api_generator
+    members = inspect.getmembers(parent)
+    members.append(('inspect', inspect))
+
+    # Assert that parent is a module and is a package, and that the members of
+    # parent include a module named `inspect`.
+    self.assertTrue(inspect.ismodule(parent))
+    self.assertTrue(hasattr(parent, '__path__'))
+    self.assertIn('inspect', [name for name, _ in members])
+    self.assertTrue(inspect.ismodule(inspect))
+
+    filtered_members = public_api.explicit_package_contents_filter(
+        path, parent, members)
+
+    # Assert that the filtered_members do not include a module named `inspect`.
+    self.assertNotIn('inspect', [name for name, _ in filtered_members])
+
+  def test_explicit_package_contents_filter_removes_modules_imported_by_modules(
+      self):
+    path = ('tensorflow_docs', 'api_generator', 'public_api')
+    parent = public_api
+    members = inspect.getmembers(parent)
+
+    # Assert that parent is a module and not a package, and that the members of
+    # parent include a module named `inspect`.
+    self.assertTrue(inspect.ismodule(parent))
+    self.assertFalse(hasattr(parent, '__path__'))
+    self.assertIn('inspect', [name for name, _ in members])
+    self.assertTrue(inspect.ismodule(inspect))
+
+    filtered_members = public_api.explicit_package_contents_filter(
+        path, parent, members)
+
+    # Assert that the filtered_members do not include a module named `inspect`.
+    self.assertNotIn('inspect', [name for name, _ in filtered_members])
+
+  def test_get_imported_symbols(self):
+    source = """
+        import sub0
+        import pkg.sub1
+        from pkg import sub2
+        from pkg.sub3 import sub_sub1
+        from pkg.sub4 import *
+        from pkg import sub5 as r1
+        from pkg import sub6 as r2, sub7, sub8 as r3
+
+        """
+    imported = public_api._get_imported_symbols(source)
+    self.assertCountEqual(
+        ['sub0', 'sub2', 'sub_sub1', 'r1', 'r2', 'sub7', 'r3'], imported)
+
   def test_ignore_typing(self):
     children_before = [('a', 1), ('b', 3), ('c', typing.List)]
     children_after = public_api.ignore_typing('ignored', 'ignored',
                                               children_before)
     self.assertEqual(children_after, children_before[:-1])
 
+  def test_ignore_class_attr(self):
+
+    class MyClass:
+
+      def method(self):
+        pass
+
+      @doc_controls.do_not_doc_inheritable
+      def hidden_method(self):
+        pass
+
+    class SubClass(MyClass):
+
+      def hidden_method(self):
+        'still hidden'
+
+    result = public_api.filter_doc_controls_skip(
+        path=('a', 'b'),
+        parent=SubClass,
+        children=[('method', SubClass.method),
+                  ('hidden_method', SubClass.hidden_method)])
+
+    self.assertEqual([('method', MyClass.method)], list(result))
+
+  def test_filter_all(self):
+    module = types.ModuleType('module')
+    module.__all__ = ['a']
+    module.a = 1
+    module.b = 2
+
+    result = public_api.filter_module_all(('module'), module, [('a', module.a),
+                                                               ('b', module.b)])
+    self.assertEqual([('a', module.a)], list(result))
+
+  def test_filter_base_dirs(self):
+    module = types.ModuleType('module')
+    module.__file__ = '/1/2/3/module'
+    module.a = 1
+    module.sub1 = types.ModuleType('sub1')
+    module.sub1.__file__ = '/1/2/3/4/sub1'
+    module.sub2 = types.ModuleType('sub2')
+    module.sub2.__file__ = '/1/2/bad/sub2'
+
+    my_filter = public_api.FilterBaseDirs(base_dirs=[pathlib.Path('/1/2/3/')])
+
+    result = my_filter(
+        path=('module',),
+        parent=module,
+        children=[('a', module.a), ('sub1', module.sub1),
+                  ('sub2', module.sub2)])
+    self.assertEqual([('a', module.a), ('sub1', module.sub1)], list(result))
+
+  def test_filter_base_dir_pointing_to_submodule_dir(self):
+    module = types.ModuleType('module')
+    module.__file__ = '/1/2/3/module'
+    module.submodule = types.ModuleType('submodule')
+    module.submodule.__file__ = '/1/2/3/submodule/__init__.py'
+
+    test_filter = public_api.FilterBaseDirs(
+        base_dirs=[pathlib.Path('/1/2/3/submodule')])
+    result = test_filter(
+        path=('module',),
+        parent=module,
+        children=[('submodule', module.submodule)])
+
+    self.assertEqual([('submodule', module.submodule)], list(result))
+
 
 if __name__ == '__main__':
   absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/py_guide_parser.py b/tools/tensorflow_docs/api_generator/py_guide_parser.py
deleted file mode 100644
index b00694dc403..00000000000
--- a/tools/tensorflow_docs/api_generator/py_guide_parser.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""Library for operating on Python API Guide files."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import re
-
-
-def md_files_in_dir(py_guide_src_dir):
-  """Returns a list of filename (full_path, base) pairs for guide files."""
-  all_in_dir = [(os.path.join(py_guide_src_dir, f), f)
-                for f in os.listdir(py_guide_src_dir)]
-  return [(full, f) for full, f in all_in_dir
-          if os.path.isfile(full) and f.endswith('.md')]
-
-
-class PyGuideParser(object):
-  """Simple parsing of a guide .md file.
-
-  Descendants can override the process_*() functions (called by process())
-  to either record information from the guide, or call replace_line()
-  to affect the return value of process().
-  """
-
-  def __init__(self):
-    self._lines = None
-
-  def process(self, full_path):
-    """Read and process the file at `full_path`."""
-    with open(full_path, 'rb') as f:
-      md_string = f.read().decode('utf-8')
-    self._lines = md_string.split('\n')
-    seen = set()
-
-    in_blockquote = False
-    for i, line in enumerate(self._lines):
-      if '```' in line:
-        in_blockquote = not in_blockquote
-
-      if not in_blockquote and line.startswith('# '):
-        self.process_title(i, line[2:])
-      elif not in_blockquote and line.startswith('## '):
-        section_title = line.strip()[3:]
-        existing_tag = re.search(' {([^}]+)} *$', line)
-        if existing_tag:
-          tag = existing_tag.group(1)
-        else:
-          tag = re.sub('[^a-zA-Z0-9]+', '_', section_title)
-          if tag in seen:
-            suffix = 0
-            while True:
-              candidate = '%s_%d' % (tag, suffix)
-              if candidate not in seen:
-                tag = candidate
-                break
-        seen.add(tag)
-        self.process_section(i, section_title, tag)
-
-      elif in_blockquote:
-        self.process_in_blockquote(i, line)
-      else:
-        self.process_line(i, line)
-
-    ret = '\n'.join(self._lines)
-    self._lines = None
-    return ret
-
-  def replace_line(self, line_number, line):
-    """Replace the contents of line numbered `line_number` with `line`."""
-    self._lines[line_number] = line
-
-  def process_title(self, line_number, title):
-    pass
-
-  def process_section(self, line_number, section_title, tag):
-    pass
-
-  def process_in_blockquote(self, line_number, line):
-    pass
-
-  def process_line(self, line_number, line):
-    pass
diff --git a/tools/tensorflow_docs/api_generator/py_guide_parser_test.py b/tools/tensorflow_docs/api_generator/py_guide_parser_test.py
deleted file mode 100644
index 3924d53d919..00000000000
--- a/tools/tensorflow_docs/api_generator/py_guide_parser_test.py
+++ /dev/null
@@ -1,93 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for py_guide_parser."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import tempfile
-import textwrap
-
-from absl.testing import absltest
-
-from tensorflow_docs.api_generator import py_guide_parser
-
-
-class TestPyGuideParser(py_guide_parser.PyGuideParser):
-
-  def __init__(self):
-    self.calls = []
-    py_guide_parser.PyGuideParser.__init__(self)
-
-  def process_title(self, line_number, title):
-    self.calls.append((line_number, 'title', title))
-
-  def process_section(self, line_number, section_title, tag):
-    self.calls.append((line_number, 'section',
-                       '%s : %s' % (section_title, tag)))
-
-  def process_in_blockquote(self, line_number, line):
-    self.calls.append((line_number, 'blockquote', line))
-    self.replace_line(line_number, line + ' BQ')
-
-  def process_line(self, line_number, line):
-    self.calls.append((line_number, 'line', line))
-
-
-class PyGuideParserTest(absltest.TestCase):
-  _BASE_DIR = tempfile.mkdtemp()
-
-  def setUp(self):
-    self.workdir = os.path.join(self._BASE_DIR, self.id())
-    os.makedirs(self.workdir)
-
-  def testBasics(self):
-    tmp = os.path.join(self.workdir, 'py_guide_parser_test.md')
-    f = open(tmp, 'w')
-    f.write(
-        textwrap.dedent("""
-        # a title
-        a line
-        ## a section
-        ```shell
-        in a blockquote
-        ```
-        out of blockquote
-        """)[1:])
-    f.close()
-    parser = TestPyGuideParser()
-    result = parser.process(tmp)
-    expected = textwrap.dedent("""
-        # a title
-        a line
-        ## a section
-        ```shell BQ
-        in a blockquote BQ
-        ```
-        out of blockquote
-        """)[1:]
-    self.assertEqual(expected, result)
-    expected = [(0, 'title', 'a title'), (1, 'line', 'a line'),
-                (2, 'section', 'a section : a_section'),
-                (3, 'blockquote', '```shell'),
-                (4, 'blockquote', 'in a blockquote'), (5, 'line', '```'),
-                (6, 'line', 'out of blockquote'), (7, 'line', '')]
-    self.assertEqual(expected, parser.calls)
-
-
-if __name__ == '__main__':
-  absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/reference_resolver.py b/tools/tensorflow_docs/api_generator/reference_resolver.py
new file mode 100644
index 00000000000..2e6c0c91a7c
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/reference_resolver.py
@@ -0,0 +1,441 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Turn Python docstrings into Markdown for TensorFlow documentation."""
+
+from __future__ import annotations
+
+import collections
+import contextlib
+import html
+import json
+import os
+import posixpath
+import re
+
+from typing import Optional, Union
+
+from tensorflow_docs.api_generator import parser
+
+
+class TFDocsError(Exception):
+  pass
+
+
+class IgnoreLineInBlock:
+  """Ignores the lines in a block.
+
+  Attributes:
+    block_start: Contains the start string of a block to ignore.
+    block_end: Contains the end string of a block to ignore.
+  """
+
+  def __init__(self, block_start, block_end):
+    self._block_start = block_start
+    self._block_end = block_end
+    self._in_block = False
+
+    self._start_end_regex = re.escape(self._block_start) + r'.*?' + re.escape(
+        self._block_end)
+
+  def __call__(self, line):
+    # If start and end block are on the same line, return True.
+    if re.match(self._start_end_regex, line):
+      return True
+
+    if not self._in_block:
+      if self._block_start in line:
+        self._in_block = True
+
+    elif self._block_end in line:
+      self._in_block = False
+      # True is being returned here because the last line in the block should
+      # also be ignored.
+      return True
+
+    return self._in_block
+
+
+class ReferenceResolver:
+  """Class for replacing `tf.symbol` references with links."""
+
+  # ?P<...> helps to find the match by entering the group name instead of the
+  # index. For example, instead of doing match.group(1) we can do
+  # match.group('brackets')
+  AUTO_REFERENCE_RE = re.compile(
+      r"""
+      (?P\[.*?\])|                      # match a '[]' span
+      ``?(?P@?[\w\(\[\)\]\{\}.,=\s]+?)``?  # or a `` span
+      """,
+      flags=re.VERBOSE,
+  )
+
+  def __init__(
+      self,
+      *,
+      duplicate_of: dict[str, str],
+      is_fragment: dict[str, bool],
+      py_module_names: Union[list[str], dict[str, str]],
+      link_prefix: Optional[str] = None,
+      physical_path: Optional[dict[str, str]] = None,
+  ):
+    """Initializes a Reference Resolver.
+
+    Args:
+      duplicate_of: A map from duplicate names to preferred names of API
+        symbols.
+      is_fragment: A map from full names to bool for each symbol. If True the
+        object lives at a page fragment `tf.a.b.c` --> `tf/a/b#c`. If False
+        object has a page to itself: `tf.a.b.c` --> `tf/a/b/c`.
+      py_module_names: A dict from short name to module name Like
+        `{'tf': 'tensorflow'}`. Or [deprecated] a list of short-names like
+        `['tf']`.
+      link_prefix: The website to which these symbols should link to. A prefix
+        is added before the links to enable cross-site linking if `link_prefix`
+        is not None.
+      physical_path: A mapping from the preferred full_name to the object's
+        physical path.
+    """
+    self._duplicate_of = duplicate_of
+    self._is_fragment = is_fragment
+    self._physical_path = physical_path
+    if isinstance(py_module_names, list):
+      py_module_names = {short: short for short in py_module_names}
+    self._py_module_names = py_module_names
+
+    self._link_prefix = link_prefix
+
+    self._all_names = set(is_fragment.keys())
+    self._partial_symbols_dict = self._create_partial_symbols_dict()
+
+  def get_main_name(self, name: str) -> Optional[str]:
+    full_name = self._partial_symbols_dict.get(name, name)
+    main_name = self._duplicate_of.get(full_name, full_name)
+    if main_name in self._all_names:
+      return main_name
+    else:
+      return None
+
+  @classmethod
+  def from_visitor(cls, visitor, **kwargs):
+    """A factory function for building a ReferenceResolver from a visitor.
+
+    Args:
+      visitor: an instance of `DocGeneratorVisitor`
+      **kwargs: all remaining args are passed to the constructor
+
+    Returns:
+      an instance of `ReferenceResolver` ()
+    """
+    api_tree = visitor.api_tree
+    all_is_fragment = {}
+    duplicate_of = {}
+    physical_path = {}
+    for node in api_tree.iter_nodes():
+      full_name = node.full_name
+      is_fragment = node.output_type() is node.OutputType.FRAGMENT
+      if node.physical_path:
+        physical_path[node.full_name] = '.'.join(node.physical_path)
+      for alias in node.aliases:
+        alias_name = '.'.join(alias)
+        duplicate_of[alias_name] = full_name
+        all_is_fragment[alias_name] = is_fragment
+
+    return cls(
+        duplicate_of=visitor.duplicate_of,
+        is_fragment=all_is_fragment,
+        physical_path=physical_path,
+        **kwargs)
+
+  def with_prefix(self, prefix):
+    return type(self)(
+        duplicate_of=self._duplicate_of,
+        is_fragment=self._is_fragment,
+        py_module_names=self._py_module_names,
+        link_prefix=prefix,
+    )
+
+  @contextlib.contextmanager
+  def temp_prefix(self, link_prefix):
+    old_prefix = self._link_prefix
+    self._link_prefix = link_prefix
+    try:
+      yield
+    finally:
+      self._link_prefix = old_prefix
+
+  def is_fragment(self, full_name: str):
+    """Returns True if the object's doc is a subsection of another page."""
+    return self._is_fragment[full_name]
+
+  @classmethod
+  def from_json_file(cls, filepath):
+    """Initialize the reference resolver via _api_cache.json."""
+    with open(filepath) as f:
+      json_dict = json.load(f)
+
+    return cls(**json_dict)
+
+  def _partial_symbols(self, symbol):
+    """Finds the partial symbols given the true symbol.
+
+    For example, symbol: `tf.keras.layers.Conv2D`, then the partial dictionary
+    returned will be:
+
+    partials = ["tf.keras.layers.Conv2D","keras.layers.Conv2D","layers.Conv2D"]
+
+    There should at least be one '.' in the partial symbol generated so as to
+    avoid guessing for the true symbol.
+
+    Args:
+      symbol: String, representing the true symbol.
+
+    Returns:
+      A list of partial symbol names
+    """
+
+    split_symbol = symbol.split('.')
+    partials = [
+        '.'.join(split_symbol[i:]) for i in range(1,
+                                                  len(split_symbol) - 1)
+    ]
+    return partials
+
+  def _create_partial_symbols_dict(self):
+    """Creates a partial symbols dictionary for all the symbols in TensorFlow.
+
+    Returns:
+      A dictionary mapping {partial_symbol: real_symbol}
+    """
+    partial_symbols_dict = collections.defaultdict(list)
+
+    for name in sorted(self._all_names):
+      if 'tf.compat.v' in name or 'tf.contrib' in name:
+        continue
+      # TODO(yashkatariya): Remove `tf.experimental.numpy` after `tf.numpy` is
+      # in not in experimental namespace.
+      if 'tf.experimental.numpy' in name or 'tf.numpy' in name:
+        continue
+      partials = self._partial_symbols(name)
+      for partial in partials:
+        partial_symbols_dict[partial].append(name)
+
+    new_partial_dict = {}
+    for partial, full_names in partial_symbols_dict.items():
+      if not full_names:
+        continue
+
+      full_names = [
+          self._duplicate_of.get(full_name, full_name)
+          for full_name in full_names
+      ]
+
+      new_partial_dict[partial] = full_names[0]
+
+    return new_partial_dict
+
+  def to_json_file(self, filepath):
+    """Converts the ReferenceResolver to json and writes it to the specified file.
+
+    Args:
+      filepath: The file path to write the json to.
+    """
+
+    try:
+      os.makedirs(os.path.dirname(filepath))
+    except OSError:
+      pass
+
+    json_dict = {}
+    for key, value in self.__dict__.items():
+      # Drop these fields, they are generated by the constructor.
+      if key == '_all_names' or key == '_partial_symbols_dict':
+        continue
+
+      # Strip off any leading underscores on field names as these are not
+      # recognized by the constructor.
+      json_dict[key.lstrip('_')] = value
+
+    with open(filepath, 'w') as f:
+      json.dump(json_dict, f, indent=2, sort_keys=True)
+      f.write('\n')
+
+  def replace_references(self, string, full_name=None):
+    """Replace `tf.symbol` references with links to symbol's documentation page.
+
+    This function finds all occurrences of "`tf.symbol`" in `string`
+    and replaces them with links to the documentation page
+    for "symbol".
+
+
+    Args:
+      string: A string in which "`tf.symbol`" references should be replaced.
+      full_name: (optional) The full name of current object, so replacements can
+        depend on context.
+
+    Returns:
+      `string`, with "`tf.symbol`" references replaced by links.
+    """
+
+    def one_ref(match):
+      return self._one_ref(match, full_name)
+
+    fixed_lines = []
+
+    filters = [
+        IgnoreLineInBlock('
    ',
+                          '
    '),
+        IgnoreLineInBlock('```', '```'),
+        IgnoreLineInBlock(
+            '
    ',
+            '
    '),
+        IgnoreLineInBlock('![', ')'),  # Don't replace within image's caption
+    ]
+
+    for line in string.splitlines():
+      if not any(filter_block(line) for filter_block in filters):
+        line = re.sub(self.AUTO_REFERENCE_RE, one_ref, line)
+      fixed_lines.append(line)
+
+    return '\n'.join(fixed_lines)
+
+  def python_link(self, link_text: str, ref_full_name: Optional[str] = None):
+    """Resolve a "`tf.symbol`" reference to a link.
+
+    This will pick the canonical location for duplicate symbols.
+
+    Args:
+      link_text: The text of the link.
+      ref_full_name: The fully qualified name of the symbol to link to.
+
+    Returns:
+      A link to the documentation page of `ref_full_name`.
+    """
+    if ref_full_name is None:
+      ref_full_name = link_text
+    link_text = html.escape(link_text, quote=True)
+
+    url = self.reference_to_url(ref_full_name)
+    url = html.escape(url, quote=True)
+    return f'{link_text}'
+
+  def py_main_name(self, full_name):
+    """Return the main name for a Python symbol name."""
+    return self._duplicate_of.get(full_name, full_name)
+
+  def reference_to_url(self, ref_full_name):
+    """Resolve a "`tf.symbol`" reference to a relative path.
+
+    The input to this function should already be stripped of the '@'
+    and '{}', and its output is only the link, not the full Markdown.
+
+    If `ref_full_name` is the name of a class member, method, or property, the
+    link will point to the page of the containing class, and it will include the
+    method name as an anchor. For example, `tf.module.MyClass.my_method`.
+
+    Args:
+      ref_full_name: The fully qualified name of the symbol to link to.
+
+    Returns:
+      A relative path that links from the documentation page of `from_full_name`
+      to the documentation page of `ref_full_name`.
+
+    Raises:
+      TFDocsError: If the symbol is not found.
+    """
+    if self._is_fragment.get(ref_full_name, False):
+      # methods and constants get duplicated. And that's okay.
+      # Use the main name of their parent.
+      parent_name, short_name = ref_full_name.rsplit('.', 1)
+      parent_main_name = self._duplicate_of.get(parent_name, parent_name)
+      main_name = '.'.join([parent_main_name, short_name])
+    else:
+      main_name = self._duplicate_of.get(ref_full_name, ref_full_name)
+
+    # Check whether this link exists
+    if main_name not in self._all_names:
+      raise TFDocsError(f'Cannot make link to {main_name!r}: Not in index.')
+
+    rel_path = parser.documentation_path(main_name,
+                                         self._is_fragment[main_name])
+
+    if self._link_prefix is None:
+      raise ValueError('you must set the `link_prefix`')
+    url = posixpath.join(self._link_prefix, rel_path)
+    return url
+
+  def _one_ref(self, match, full_name=None):
+    """Return a link for a single "`tf.symbol`" reference."""
+
+    if match.group(1):
+      # Found a '[]' group, return it unmodified.
+      return match.group('brackets')
+
+    # Found a '``' group.
+    string = match.group('backticks')
+
+    link_text = string
+
+    # Drop everything after the *last* ( or [ to get the
+    # symbol name. The last is used so complex nested or chained calls are not
+    # recognized as valid links.
+    string = re.sub(r'^(.*)[\(\[].*', r'\1', string)
+    # Drop the optional leading `@`.
+    string = re.sub(r'^@', r'', string)
+
+    if string.startswith('compat.v1') or string.startswith('compat.v2'):
+      string = 'tf.' + string
+    elif string.startswith('v1') or string.startswith('v2'):
+      string = 'tf.compat.' + string
+
+    elif full_name is None or ('tf.compat.v' not in full_name and
+                               'tf.contrib' not in full_name):
+      string = self._partial_symbols_dict.get(string, string)
+
+    if not string:
+      return match.group(0)
+
+    try:
+      if string.startswith('tensorflow::'):
+        # C++ symbol
+        return self._cc_link(string, link_text)
+
+      return self.python_link(link_text, string)
+    except TFDocsError:
+      pass
+
+    return match.group(0)
+
+  def _cc_link(self, string, link_text):
+    """Generate a link for a `tensorflow::...` reference."""
+    # TODO(joshl): Fix this hard-coding of paths.
+    if string == 'tensorflow::ClientSession':
+      ret = 'class/tensorflow/client-session.md'
+    elif string == 'tensorflow::Scope':
+      ret = 'class/tensorflow/scope.md'
+    elif string == 'tensorflow::Status':
+      ret = 'class/tensorflow/status.md'
+    elif string == 'tensorflow::Tensor':
+      ret = 'class/tensorflow/tensor.md'
+    elif string == 'tensorflow::ops::Const':
+      ret = 'namespace/tensorflow/ops.md#const'
+    else:
+      raise TFDocsError(f'C++ reference not understood: "{string}"')
+
+    # relative_path_to_root gets you to api_docs/python, we go from there
+    # to api_docs/cc, and then add ret.
+    cc_relative_path = os.path.normpath(
+        posixpath.join(self._link_prefix, '../cc', ret))
+
+    return f'{link_text}'
diff --git a/tools/tensorflow_docs/api_generator/reference_resolver_test.py b/tools/tensorflow_docs/api_generator/reference_resolver_test.py
new file mode 100644
index 00000000000..31ca192baf9
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/reference_resolver_test.py
@@ -0,0 +1,222 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for documentation parser."""
+
+import os
+import tempfile
+import textwrap
+
+from typing import Dict, List, Optional, Union
+
+from absl.testing import absltest
+from absl.testing import parameterized
+
+
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator import reference_resolver as reference_resolver_lib
+
+
+class TestReferenceResolver(absltest.TestCase):
+  _BASE_DIR = tempfile.mkdtemp()
+
+  def setUp(self):
+    super(TestReferenceResolver, self).setUp()
+    self.workdir = os.path.join(self._BASE_DIR, self.id())
+    os.makedirs(self.workdir)
+
+  def testSaveReferenceResolver(self):
+    duplicate_of = {'AClass': ['AClass2']}
+    is_fragment = {
+        'tf': False,
+        'tf.VERSION': True,
+        'tf.AClass': False,
+        'tf.AClass.method': True,
+        'tf.AClass2': False,
+        'tf.function': False
+    }
+    py_module_names = {'tf': 'tensorflow'}
+
+    resolver = reference_resolver_lib.ReferenceResolver(
+        duplicate_of=duplicate_of,
+        is_fragment=is_fragment,
+        py_module_names=py_module_names)
+
+    outdir = self.workdir
+
+    filepath = os.path.join(outdir, 'resolver.json')
+
+    resolver.to_json_file(filepath)
+    resolver2 = reference_resolver_lib.ReferenceResolver.from_json_file(
+        filepath)
+
+    # There are no __slots__, so all fields are visible in __dict__.
+    self.assertEqual(resolver.__dict__, resolver2.__dict__)
+
+  def test_duplicate_fragment(self):
+    duplicate_of = {
+        'tf.Class2.method': 'tf.Class1.method',
+        'tf.sub.Class2.method': 'tf.Class1.method',
+        'tf.sub.Class2': 'tf.Class2'
+    }
+    is_fragment = {
+        'tf.Class1.method': True,
+        'tf.Class2.method': True,
+        'tf.sub.Class2.method': True,
+        'tf.Class1': False,
+        'tf.Class2': False,
+        'tf.sub.Class2': False
+    }
+    py_module_names = {'tf': 'tensorflow'}
+
+    reference_resolver = reference_resolver_lib.ReferenceResolver(
+        duplicate_of=duplicate_of,
+        is_fragment=is_fragment,
+        py_module_names=py_module_names,
+        link_prefix='')
+
+    # Method references point to the method, in the canonical class alias.
+    result = reference_resolver.reference_to_url('tf.Class1.method')
+    self.assertEqual('tf/Class1.md#method', result)
+    result = reference_resolver.reference_to_url('tf.Class2.method')
+    self.assertEqual('tf/Class2.md#method', result)
+    result = reference_resolver.reference_to_url('tf.sub.Class2.method')
+    self.assertEqual('tf/Class2.md#method', result)
+
+    # Class references point to the canonical class alias
+    result = reference_resolver.reference_to_url('tf.Class1')
+    self.assertEqual('tf/Class1.md', result)
+    result = reference_resolver.reference_to_url('tf.Class2')
+    self.assertEqual('tf/Class2.md', result)
+    result = reference_resolver.reference_to_url('tf.sub.Class2')
+    self.assertEqual('tf/Class2.md', result)
+
+
+class TestPartialSymbolAutoRef(parameterized.TestCase):
+  REF_TEMPLATE = '{text}'
+
+  @parameterized.named_parameters(
+      ('basic1', 'keras.Model.fit', '../tf/keras/Model.md#fit'),
+      ('duplicate_object', 'layers.Conv2D', '../tf/keras/layers/Conv2D.md'),
+      ('parens', 'Model.fit(x, y, epochs=5)', '../tf/keras/Model.md#fit'),
+      ('duplicate_name', 'tf.matmul', '../tf/linalg/matmul.md'),
+      ('full_name', 'tf.concat', '../tf/concat.md'),
+      ('extra_backticks', '`tf.concat`', '../tf/concat.md'),
+      ('normal_and_compat', 'linalg.matmul', '../tf/linalg/matmul.md'),
+      ('compat_only', 'math.deprecated', None),
+      ('contrib_only', 'y.z', None),
+  )
+  def test_partial_symbol_references(self, string, link):
+    duplicate_of = {
+        'tf.matmul': 'tf.linalg.matmul',
+        'tf.layers.Conv2d': 'tf.keras.layers.Conv2D',
+    }
+
+    is_fragment = {
+        'tf.keras.Model.fit': True,
+        'tf.concat': False,
+        'tf.keras.layers.Conv2D': False,
+        'tf.linalg.matmul': False,
+        'tf.compat.v1.math.deprecated': False,
+        'tf.compat.v1.linalg.matmul': False,
+        'tf.contrib.y.z': False,
+    }
+
+    py_module_names = {'tf': 'tensorflow'}
+
+    resolver = reference_resolver_lib.ReferenceResolver(
+        duplicate_of=duplicate_of,
+        is_fragment=is_fragment,
+        py_module_names=py_module_names,
+        link_prefix='..')
+    input_string = string.join('``')
+    ref_string = resolver.replace_references(input_string)
+
+    if link is None:
+      expected = input_string
+    else:
+      expected = self.REF_TEMPLATE.format(link=link, text=string.strip('`'))
+
+    self.assertEqual(expected, ref_string)
+
+
+class TestIgnoreLineInBlock(parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('ignore_backticks', ['```'], ['```'],
+       '```\nFiller\n```\n```Same line```\n```python\nDowner\n```'),
+      ('ignore_code_cell_output', ['
    {% html %}'], ['{% endhtml %}
    '],
+       '
    {% html %}\nOutput\nmultiline{% endhtml %}
    '),
+      ('ignore_backticks_and_cell_output', ['
    {% html %}', '```'
+                                           ], ['{% endhtml %}
    ', '```'],
+       ('```\nFiller\n```\n```Same line```\n
    {% html %}\nOutput\nmultiline'
+        '{% endhtml %}
    \n```python\nDowner\n```')))
+  def test_ignore_lines(self, block_start, block_end, expected_ignored_lines):
+
+    text = textwrap.dedent("""\
+    ```
+    Filler
+    ```
+
+    ```Same line```
+
+    
    {% html %}
+    Output
+    multiline{% endhtml %}
    
+
+    ```python
+    Downer
+    ```
+    """)
+
+    filters = [
+        reference_resolver_lib.IgnoreLineInBlock(start, end)
+        for start, end in zip(block_start, block_end)
+    ]
+
+    ignored_lines = []
+    for line in text.splitlines():
+      if any(filter_block(line) for filter_block in filters):
+        ignored_lines.append(line)
+
+    self.assertEqual('\n'.join(ignored_lines), expected_ignored_lines)
+
+  def test_clean_text(self):
+    text = textwrap.dedent("""\
+    ```
+    Ignore lines here.
+    ```
+    Useful information.
+    Don't ignore.
+    ```python
+    Ignore here too.
+    ```
+    Stuff.
+    ```Not useful.```
+    """)
+
+    filters = [reference_resolver_lib.IgnoreLineInBlock('```', '```')]
+
+    clean_text = []
+    for line in text.splitlines():
+      if not any(filter_block(line) for filter_block in filters):
+        clean_text.append(line)
+
+    expected_clean_text = 'Useful information.\nDon\'t ignore.\nStuff.'
+
+    self.assertEqual('\n'.join(clean_text), expected_clean_text)
+
+
+if __name__ == '__main__':
+  absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/test_module2.py b/tools/tensorflow_docs/api_generator/report/__init__.py
similarity index 61%
rename from tools/tensorflow_docs/api_generator/test_module2.py
rename to tools/tensorflow_docs/api_generator/report/__init__.py
index bd09d443c89..efaab0c5923 100644
--- a/tools/tensorflow_docs/api_generator/test_module2.py
+++ b/tools/tensorflow_docs/api_generator/report/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,24 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""A module target for TraverseTest.test_module."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-class ModuleClass2(object):
-
-  def __init__(self):
-    pass
-
-  def __model_class1_method__(self):
-    pass
-
-
-class Hidden(object):
-  pass
-
-
-__all__ = ['ModuleClass2']
+"""Tools to generate the API report."""
diff --git a/tools/tensorflow_docs/api_generator/report/linter.py b/tools/tensorflow_docs/api_generator/report/linter.py
new file mode 100644
index 00000000000..0dab205eba2
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/report/linter.py
@@ -0,0 +1,261 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Lints the docstring of an API symbol."""
+
+import ast
+import inspect
+import re
+import textwrap
+
+from typing import Optional, Any, List, Tuple
+
+import astor
+
+from tensorflow_docs.api_generator import get_source
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator.pretty_docs import base_page
+from tensorflow_docs.api_generator.report.schema import api_report_pb2
+
+
+
+
+def _count_empty_param(items: List[Tuple[str, Optional[str]]]) -> int:
+  count = 0
+  for name, description in items:
+    del name
+    if description is None or description.strip() == '':
+      count += 1
+  return count
+
+
+def lint_params(page_info: base_page.PageInfo) -> api_report_pb2.ParameterLint:
+  """Lints the parameters of a docstring.
+
+  Args:
+    page_info: A `PageInfo` object containing the information of a page
+      generated via the api generation.
+
+  Returns:
+    A filled `DescriptionLint` proto object.
+  """
+  param_lint = api_report_pb2.ParameterLint()
+
+  reserved_keywords = frozenset(['self', 'cls', '_cls'])
+
+  if page_info.py_object is not None:
+    try:
+      sig = inspect.signature(page_info.py_object)
+      args_in_code = sig.parameters.keys()
+      num_args_in_code = len(args_in_code)
+      for arg in args_in_code:
+        if arg in reserved_keywords:
+          num_args_in_code -= 1
+          break
+      param_lint.num_args_in_code = num_args_in_code
+    except (ValueError, TypeError):
+      param_lint.num_args_in_code = 0
+  else:
+    param_lint.num_args_in_code = 0
+
+  for part in page_info.doc.docstring_parts:
+    if isinstance(part, parser.TitleBlock):
+      if part.title.lower().startswith('arg'):
+        param_lint.num_args_in_doc = len(part.items)
+        param_lint.num_empty_param_desc_args = _count_empty_param(part.items)
+
+      if part.title.lower().startswith('attr'):
+        param_lint.total_attr_param = len(part.items)
+        param_lint.num_empty_param_desc_attr = _count_empty_param(part.items)
+
+  return param_lint
+
+
+def lint_description(
+    page_info: base_page.PageInfo) -> api_report_pb2.DescriptionLint:
+  """Lints the description of a docstring.
+
+  If a field in the proto is assigned 0, then it means that that field doesn't
+  exist.
+
+  Args:
+    page_info: A `PageInfo` object containing the information of a page
+      generated via the api generation.
+
+  Returns:
+    A filled `DescriptionLint` proto object.
+  """
+
+  len_brief = 0
+  if page_info.doc.brief:
+    len_brief = len(page_info.doc.brief.split())
+
+  len_long_desc = 0
+  for part in page_info.doc.docstring_parts:
+    if not isinstance(part, parser.TitleBlock):
+      len_long_desc += len(part.split())
+
+  return api_report_pb2.DescriptionLint(
+      len_brief=len_brief, len_long_desc=len_long_desc)
+
+
+_EXAMPLE_RE = re.compile(
+    r"""
+    (?P\ *)(?P```.*?\n\s*?```)
+    """, re.VERBOSE | re.DOTALL)
+
+
+def lint_usage_example(
+    page_info: base_page.PageInfo) -> api_report_pb2.UsageExampleLint:
+  """Counts the number of doctests and untested examples in a docstring.
+
+  Args:
+    page_info: A `PageInfo` object containing the information of a page
+      generated via the api generation.
+
+  Returns:
+    A filled `UsageExampleLint` proto object.
+  """
+
+  description = []
+  for part in page_info.doc.docstring_parts:
+    if isinstance(part, parser.TitleBlock):
+      description.append(str(part))
+    else:
+      description.append(part)
+  desc_str = ''.join(description)
+
+  num_doctest = 0
+  num_untested_examples = 0
+  # The doctests are wrapped in backticks (```).
+  for match in _EXAMPLE_RE.finditer(desc_str):
+    if '>>>' in match.groupdict()['content']:
+      num_doctest += 1
+    else:
+      num_untested_examples += 1
+
+  return api_report_pb2.UsageExampleLint(
+      num_doctest=num_doctest, num_untested_examples=num_untested_examples)
+
+
+class ReturnVisitor(ast.NodeVisitor):
+  """Visits the Returns node in an AST."""
+
+  def __init__(self) -> None:
+    self.total_returns = []
+
+  def visit_Return(self, node) -> None:  # pylint: disable=invalid-name
+    if node.value is None:
+      self.total_returns.append('None')
+    else:
+      self.total_returns.append(astor.to_source(node.value))
+
+
+def lint_returns(
+    page_info: base_page.PageInfo) -> Optional[api_report_pb2.ReturnLint]:
+  """"Lints the returns/yields block in the docstring.
+
+  This linter only checks if a `Returns`/`Yields` block exists in the docstring
+  if it finds `return`/`yield` keyword in the source code.
+
+  Args:
+    page_info: A `PageInfo` object containing the information of a page
+      generated via the api generation.
+
+  Returns:
+    A filled `ReturnLint` proto object.
+  """
+  return_visitor = ReturnVisitor()
+
+  source = get_source.get_source(page_info.py_object)
+  obj_ast = get_source.get_ast(page_info.py_object)
+  if obj_ast is not None:
+    try:
+      return_visitor.visit(obj_ast)
+    except Exception:  # pylint: disable=broad-except
+      pass
+
+  keywords = ('return', 'yield')
+
+  if source is not None and any(word in source for word in keywords):
+    for item in page_info.doc.docstring_parts:
+      if isinstance(item, parser.TitleBlock):
+        if item.title.lower().startswith(keywords):
+          return api_report_pb2.ReturnLint(returns_defined=True)
+    # If "Returns"/"Yields" word is present in the brief docstring then having
+    # a separate `Returns`/`Yields` section is not needed.
+    if page_info.doc.brief.lower().startswith(keywords):
+      return api_report_pb2.ReturnLint(returns_defined=True)
+    # If the code only returns None then `Returns` section in the docstring is
+    # not required.
+    if all(return_val == 'None' for return_val in return_visitor.total_returns):
+      return None
+    return api_report_pb2.ReturnLint(returns_defined=False)
+
+  return None
+
+
+class RaiseVisitor(ast.NodeVisitor):
+  """Visits the Raises node in an AST."""
+
+  def __init__(self) -> None:
+    self.total_raises = []
+
+  def visit_Raise(self, node) -> None:  # pylint: disable=invalid-name
+    # This `if` block means that there is a bare raise in the code.
+    if node.exc is None:
+      return
+    self.total_raises.append(astor.to_source(node.exc.func).strip())
+
+
+def lint_raises(page_info: base_page.PageInfo) -> api_report_pb2.RaisesLint:
+  """Lints the raises block in the docstring.
+
+  The total raises in code are extracted via an AST and compared against those
+  extracted from the docstring.
+
+  Args:
+    page_info: A `PageInfo` object containing the information of a page
+      generated via the api generation.
+
+  Returns:
+    A filled `RaisesLint` proto object.
+  """
+
+  raises_lint = api_report_pb2.RaisesLint()
+
+  # Extract the raises from the source code.
+  raise_visitor = RaiseVisitor()
+  obj_ast = get_source.get_ast(page_info.py_object)
+  if obj_ast is not None:
+    try:
+      raise_visitor.visit(obj_ast)
+    except Exception:  # pylint: disable=broad-except
+      pass
+
+  raises_lint.total_raises_in_code = len(raise_visitor.total_raises)
+
+  # Extract the raises defined in the docstring.
+  raises_defined_in_doc = []
+  for part in page_info.doc.docstring_parts:
+    if isinstance(part, parser.TitleBlock):
+      if part.title.lower().startswith('raises'):
+        raises_lint.num_raises_defined = len(part.items)
+        if part.items:
+          raises_defined_in_doc.extend(list(zip(*part.items))[0])
+        break
+  else:
+    raises_lint.num_raises_defined = 0
+
+  return raises_lint
diff --git a/tools/tensorflow_docs/api_generator/report/linter_test.py b/tools/tensorflow_docs/api_generator/report/linter_test.py
new file mode 100644
index 00000000000..1b0367a5d9c
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/report/linter_test.py
@@ -0,0 +1,236 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tensorflow_docs.api_generator.report.linter."""
+
+import copy
+
+import types
+from typing import Optional
+
+from absl.testing import absltest
+
+from tensorflow_docs.api_generator import config
+from tensorflow_docs.api_generator import doc_generator_visitor
+from tensorflow_docs.api_generator import generate_lib
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator import reference_resolver as reference_resolver_lib
+from tensorflow_docs.api_generator.pretty_docs import docs_for_object
+from tensorflow_docs.api_generator.report import utils
+from tensorflow_docs.api_generator.report.schema import api_report_pb2
+
+
+class DummyVisitor(object):
+
+  def __init__(self, index, duplicate_of):
+    self.index = index
+    self.duplicate_of = duplicate_of
+
+
+class TestClass:
+  """Class docstring.
+
+  Some words here.
+
+  Another paragraph.
+
+  >>> x = 1
+  >>> print(x)
+  1
+  >>> x += 2
+  >>> print(x)
+  3
+
+  >>> z = 'api'
+  >>> z += ' report'
+  >>> print(z)
+  api report
+
+  Attributes:
+    temp_a: Temporary variable a.
+
+  A example usage here.
+
+  ```
+  y = 2
+  z = y + 3
+  assert z == 5
+  ```
+  """
+
+  def __init__(self, temp_a, temp_b, temp_c):  # pylint: disable=g-doc-args
+    """Initializes the class.
+
+    Args:
+      temp_a: Temporary variable a.
+      temp_b: Temporary variable b.
+      temp_c:
+      temp_d:
+
+    Raises:
+      ValueError: Temp_a value not allowed.
+      TypeError: Type not allowed.
+    """
+    self.temp_a = temp_a
+    self._temp_c = temp_c
+
+    if self.temp_a:
+      raise ValueError('temp_a value not allowed.')
+    else:
+      raise TypeError('Type not allowed.')
+
+  @property
+  def temp_c(self):  # pylint: disable=g-missing-from-attributes
+    return self._temp_c
+
+  def method_one(self, x: str) -> Optional[str]:
+    """Does some nice things.
+
+    ```
+    x = 'api'
+    method_one(x)  # output == apitemp
+    ```
+
+    Args:
+      x: A variable.
+
+    Returns:
+      Returns the modified variable.
+    """
+
+    if x:
+      return x + 'temp'
+    return None
+
+
+class LinterTest(absltest.TestCase):
+
+  def _build_page_info(self):
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.TestClass = TestClass
+
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
+
+    parser_config = generator.run_extraction()
+
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=('m', 'TestClass'), py_object=TestClass)
+    return docs_for_object.docs_for_object(
+        api_node=api_node, parser_config=parser_config)
+
+  def _make_report(self):
+    page_info = self._build_page_info()
+
+    test_api_report = utils.ApiReport()
+    test_api_report.fill_metrics(page_info)
+    return test_api_report
+
+  def test_fill_report_doesnt_edit_page(self):
+    page1 = self._build_page_info()
+    page2 = self._build_page_info()
+
+    test_api_report = utils.ApiReport()
+    test_api_report.fill_metrics(page2)
+
+    page1.api_node = None
+    page1.parser_config = None
+
+    page2.api_node = None
+    page2.parser_config = None
+
+    self.assertEqual(page1, page2)
+
+  def test_class_raises_lint(self):
+    test_api_report = self._make_report()
+
+    for test_report in test_api_report.api_report.symbol_metric:
+      if (test_report.symbol_name == 'm.TestClass' and
+          test_report.object_type == api_report_pb2.ObjectType.CLASS):
+        self.assertEqual(test_report.raises_lint.num_raises_defined, 2)
+        self.assertEqual(test_report.raises_lint.total_raises_in_code, 2)
+
+  def test_method_return_lint(self):
+    test_api_report = self._make_report()
+
+    for test_report in test_api_report.api_report.symbol_metric:
+      if (test_report.symbol_name == 'm.TestClass.method_one' and
+          test_report.object_type == api_report_pb2.ObjectType.METHOD):
+        self.assertTrue(test_report.return_lint.returns_defined)
+
+  def test_description_lint(self):
+    test_api_report = self._make_report()
+
+    for test_report in test_api_report.api_report.symbol_metric:
+      if (test_report.symbol_name == 'm.TestClass' and
+          test_report.object_type == api_report_pb2.ObjectType.CLASS):
+        self.assertEqual(test_report.desc_lint.len_brief, 2)
+        self.assertEqual(test_report.desc_lint.len_long_desc, 54)
+
+      if (test_report.symbol_name == 'm.TestClass.method_one' and
+          test_report.object_type == api_report_pb2.ObjectType.METHOD):
+        self.assertEqual(test_report.desc_lint.len_brief, 4)
+        self.assertEqual(test_report.desc_lint.len_long_desc, 10)
+
+  def test_parameter_lint(self):
+    test_api_report = self._make_report()
+
+    for test_report in test_api_report.api_report.symbol_metric:
+      if (test_report.symbol_name == 'm.TestClass' and
+          test_report.object_type == api_report_pb2.ObjectType.CLASS):
+        self.assertEqual(test_report.parameter_lint.num_empty_param_desc_args,
+                         2)
+        self.assertEqual(test_report.parameter_lint.num_args_in_doc, 4)
+        self.assertEqual(test_report.parameter_lint.num_args_in_code, 3)
+        self.assertEqual(test_report.parameter_lint.num_empty_param_desc_attr,
+                         1)
+        self.assertEqual(test_report.parameter_lint.total_attr_param, 2)
+
+      if (test_report.symbol_name == 'm.TestClass.method_one' and
+          test_report.object_type == api_report_pb2.ObjectType.METHOD):
+        self.assertEqual(test_report.parameter_lint.num_empty_param_desc_args,
+                         0)
+        self.assertEqual(test_report.parameter_lint.num_args_in_doc, 1)
+        self.assertEqual(test_report.parameter_lint.num_args_in_code, 1)
+        self.assertEqual(test_report.parameter_lint.num_empty_param_desc_attr,
+                         0)
+        self.assertEqual(test_report.parameter_lint.total_attr_param, 0)
+
+  def test_example_lint(self):
+    test_api_report = self._make_report()
+
+    for test_report in test_api_report.api_report.symbol_metric:
+      if (test_report.symbol_name == 'm.TestClass' and
+          test_report.object_type == api_report_pb2.ObjectType.CLASS):
+        self.assertEqual(test_report.usage_example_lint.num_doctest, 2)
+        self.assertEqual(test_report.usage_example_lint.num_untested_examples,
+                         1)
+        self.assertEqual(
+            'm',
+            test_report.package_group,
+        )
+
+      if (test_report.symbol_name == 'm.TestClass.method_one' and
+          test_report.object_type == api_report_pb2.ObjectType.METHOD):
+        self.assertEqual(test_report.usage_example_lint.num_doctest, 0)
+        self.assertEqual(test_report.usage_example_lint.num_untested_examples,
+                         1)
+        self.assertEqual('m', test_report.package_group)
+
+
+if __name__ == '__main__':
+  absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/report/schema/__init__.py b/tools/tensorflow_docs/api_generator/report/schema/__init__.py
new file mode 100644
index 00000000000..16f5b669125
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/report/schema/__init__.py
@@ -0,0 +1,31 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Public API for api report proto."""
+
+import sys
+
+from google import protobuf
+
+_version_parts = protobuf.__version__.split('.')
+_version = (int(_version_parts[0]), int(_version_parts[1]))
+
+if _version >= (3, 20):
+  from tensorflow_docs.api_generator.report.schema import api_report_generated_pb2 as api_report_pb2  # pylint: disable=g-import-not-at-top
+else:
+  from tensorflow_docs.api_generator.report.schema import api_report_generated_319_pb2 as api_report_pb2  # pylint: disable=g-import-not-at-top
+
+sys.modules['tensorflow_docs.api_generator.report.schema.api_report_pb2'] = (
+    api_report_pb2
+)
diff --git a/tools/tensorflow_docs/api_generator/report/schema/api_report.proto b/tools/tensorflow_docs/api_generator/report/schema/api_report.proto
new file mode 100644
index 00000000000..65c103f893b
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/report/schema/api_report.proto
@@ -0,0 +1,117 @@
+// LINT.IfChange
+syntax = "proto2";
+
+package tensorflow_docs.api_report.schema;
+
+// Available externally too.
+import "google/protobuf/timestamp.proto";
+
+// This message is used to track the parameters in a docstring and lint it. This
+// includes `Args` and `Attributes`.
+message ParameterLint {
+  // The number of empty parameter descriptions that are present in the args
+  // section of a docstring.
+  optional float num_empty_param_desc_args = 1;
+
+  // The total number of parameters in the signature of an method/function.
+  optional float num_args_in_code = 2;
+
+  // The number of empty parameter descriptions that are present in the
+  // attributes section of a docstring.
+  optional float num_empty_param_desc_attr = 3;
+
+  // The total number of parameters in the attributes section of a docstring.
+  optional float total_attr_param = 4;
+
+  // The total number of parameters in the args section of a docstring.
+  optional float num_args_in_doc = 5;
+}
+
+// This message is used to track the description of a docstring. It tracks the
+// one-line brief and the multi-line description.
+message DescriptionLint {
+  // Length of the one line brief description of a docstring. If 0, it means
+  // that the one line brief doesn't exist.
+  optional float len_brief = 1;
+
+  // Length of the multi-line description of a docstring. If 0, it means
+  // that the multi-line description doesn't exist.
+  optional float len_long_desc = 3;
+}
+
+// This message is used to track the examples in a docstring that demonstrate
+// how to use that API.
+message UsageExampleLint {
+  // Number of testable examples in a docstring. go/testable-docstrings
+  // describes how to write a testable example.
+  optional float num_doctest = 1;
+
+  // Number of untested examples in a docstring. These examples are usually
+  // wrapped in triple backticks (```).
+  optional float num_untested_examples = 2;
+}
+
+// This message is used to track if `Returns` is defined in a docstring if
+// `return` keyword is used in the code.
+message ReturnLint {
+  // Whether a `Returns` block exists in the docstring.
+  optional bool returns_defined = 1;
+}
+
+// This message is used to track if `Raises` is defined in a docstring if
+// `raise` keyword is used in the code.
+message RaisesLint {
+  // Number of raises that are defined in the docstring. If two raises
+  // are of the same name (`ValueError`) but the error raised is different then
+  // both of them should be documented.
+  optional float num_raises_defined = 1;
+
+  // Total number of unique raises that are present in the code.
+  optional float total_raises_in_code = 2;
+}
+
+// This Enum is used to describe the type of object. `api_generator` traverses
+// and extracts docstrings from the objects in the pip package of a library.
+enum ObjectType {
+  CLASS = 0;
+  METHOD = 1;
+  FUNCTION = 2;
+  MODULE = 3;
+  TYPE_ALIAS = 4;
+}
+
+// This message tracks the symbol and the statistics of the lints that are run
+// on the docstring of the symbol.
+message ApiSymbolMetric {
+  // The full name of the symbol being linted.
+  optional string symbol_name = 1;
+  // The type of object the symbol is.
+  optional ObjectType object_type = 2;
+  // Statistics after linting the parameters of the docstring of the symbol.
+  optional ParameterLint parameter_lint = 3;
+  // Statistics after linting the description of the docstring of the symbol.
+  optional DescriptionLint desc_lint = 4;
+  // Statistics after linting the usage examples in the docstring of the symbol.
+  optional UsageExampleLint usage_example_lint = 5;
+  // Statistics after linting the return blocks in the docstring of the symbol.
+  optional ReturnLint return_lint = 6;
+  // Statistics after linting the raises blocks in the docstring of the symbol.
+  optional RaisesLint raises_lint = 7;
+  // The package group a symbol belongs too.
+  // For example:
+  //   tf.keras.layers.Conv2D -> tf.keras
+  //   tf.Variable -> tf
+  //   tf.Variable.assign (method) -> tf
+  //   tf.data.tensor_slices -> tf.data
+  optional string package_group = 8;
+}
+
+message ApiReport {
+  optional .google.protobuf.Timestamp timestamp = 1;
+  // Will be filled in via Timestamp().ToJsonString().
+  optional string date = 2;
+  // Repeated field to lint each API symbol in the package as traversed by the
+  // api_generator.
+  repeated ApiSymbolMetric symbol_metric = 3;
+}
+// LINT.ThenChange(docs/tools/api_generator/report/schema/api_report_generated_pb2.py)
diff --git a/tools/tensorflow_docs/api_generator/report/schema/api_report_generated_319_pb2.py b/tools/tensorflow_docs/api_generator/report/schema/api_report_generated_319_pb2.py
new file mode 100644
index 00000000000..8cf6e7c2ddb
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/report/schema/api_report_generated_319_pb2.py
@@ -0,0 +1,496 @@
+# THIS IS A GENERATED FILE. DO NOT EDIT!
+# pylint: skip-file
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: api_report.proto
+
+from google.protobuf.internal import enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='api_report.proto',
+  package='tensorflow_docs.api_report.schema',
+  syntax='proto2',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x10\x61pi_report.proto\x12!tensorflow_docs.api_report.schema\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa2\x01\n\rParameterLint\x12!\n\x19num_empty_param_desc_args\x18\x01 \x01(\x02\x12\x18\n\x10num_args_in_code\x18\x02 \x01(\x02\x12!\n\x19num_empty_param_desc_attr\x18\x03 \x01(\x02\x12\x18\n\x10total_attr_param\x18\x04 \x01(\x02\x12\x17\n\x0fnum_args_in_doc\x18\x05 \x01(\x02\";\n\x0f\x44\x65scriptionLint\x12\x11\n\tlen_brief\x18\x01 \x01(\x02\x12\x15\n\rlen_long_desc\x18\x03 \x01(\x02\"F\n\x10UsageExampleLint\x12\x13\n\x0bnum_doctest\x18\x01 \x01(\x02\x12\x1d\n\x15num_untested_examples\x18\x02 \x01(\x02\"%\n\nReturnLint\x12\x17\n\x0freturns_defined\x18\x01 \x01(\x08\"F\n\nRaisesLint\x12\x1a\n\x12num_raises_defined\x18\x01 \x01(\x02\x12\x1c\n\x14total_raises_in_code\x18\x02 \x01(\x02\"\xeb\x03\n\x0f\x41piSymbolMetric\x12\x13\n\x0bsymbol_name\x18\x01 \x01(\t\x12\x42\n\x0bobject_type\x18\x02 \x01(\x0e\x32-.tensorflow_docs.api_report.schema.ObjectType\x12H\n\x0eparameter_lint\x18\x03 \x01(\x0b\x32\x30.tensorflow_docs.api_report.schema.ParameterLint\x12\x45\n\tdesc_lint\x18\x04 \x01(\x0b\x32\x32.tensorflow_docs.api_report.schema.DescriptionLint\x12O\n\x12usage_example_lint\x18\x05 \x01(\x0b\x32\x33.tensorflow_docs.api_report.schema.UsageExampleLint\x12\x42\n\x0breturn_lint\x18\x06 \x01(\x0b\x32-.tensorflow_docs.api_report.schema.ReturnLint\x12\x42\n\x0braises_lint\x18\x07 \x01(\x0b\x32-.tensorflow_docs.api_report.schema.RaisesLint\x12\x15\n\rpackage_group\x18\x08 \x01(\t\"\x93\x01\n\tApiReport\x12-\n\ttimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04\x64\x61te\x18\x02 \x01(\t\x12I\n\rsymbol_metric\x18\x03 \x03(\x0b\x32\x32.tensorflow_docs.api_report.schema.ApiSymbolMetric*M\n\nObjectType\x12\t\n\x05\x43LASS\x10\x00\x12\n\n\x06METHOD\x10\x01\x12\x0c\n\x08\x46UNCTION\x10\x02\x12\n\n\x06MODULE\x10\x03\x12\x0e\n\nTYPE_ALIAS\x10\x04'
+  ,
+  dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,])
+
+_OBJECTTYPE = _descriptor.EnumDescriptor(
+  name='ObjectType',
+  full_name='tensorflow_docs.api_report.schema.ObjectType',
+  filename=None,
+  file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='CLASS', index=0, number=0,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='METHOD', index=1, number=1,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='FUNCTION', index=2, number=2,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='MODULE', index=3, number=3,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='TYPE_ALIAS', index=4, number=4,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=1141,
+  serialized_end=1218,
+)
+_sym_db.RegisterEnumDescriptor(_OBJECTTYPE)
+
+ObjectType = enum_type_wrapper.EnumTypeWrapper(_OBJECTTYPE)
+CLASS = 0
+METHOD = 1
+FUNCTION = 2
+MODULE = 3
+TYPE_ALIAS = 4
+
+
+
+_PARAMETERLINT = _descriptor.Descriptor(
+  name='ParameterLint',
+  full_name='tensorflow_docs.api_report.schema.ParameterLint',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='num_empty_param_desc_args', full_name='tensorflow_docs.api_report.schema.ParameterLint.num_empty_param_desc_args', index=0,
+      number=1, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='num_args_in_code', full_name='tensorflow_docs.api_report.schema.ParameterLint.num_args_in_code', index=1,
+      number=2, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='num_empty_param_desc_attr', full_name='tensorflow_docs.api_report.schema.ParameterLint.num_empty_param_desc_attr', index=2,
+      number=3, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='total_attr_param', full_name='tensorflow_docs.api_report.schema.ParameterLint.total_attr_param', index=3,
+      number=4, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='num_args_in_doc', full_name='tensorflow_docs.api_report.schema.ParameterLint.num_args_in_doc', index=4,
+      number=5, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=89,
+  serialized_end=251,
+)
+
+
+_DESCRIPTIONLINT = _descriptor.Descriptor(
+  name='DescriptionLint',
+  full_name='tensorflow_docs.api_report.schema.DescriptionLint',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='len_brief', full_name='tensorflow_docs.api_report.schema.DescriptionLint.len_brief', index=0,
+      number=1, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='len_long_desc', full_name='tensorflow_docs.api_report.schema.DescriptionLint.len_long_desc', index=1,
+      number=3, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=253,
+  serialized_end=312,
+)
+
+
+_USAGEEXAMPLELINT = _descriptor.Descriptor(
+  name='UsageExampleLint',
+  full_name='tensorflow_docs.api_report.schema.UsageExampleLint',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='num_doctest', full_name='tensorflow_docs.api_report.schema.UsageExampleLint.num_doctest', index=0,
+      number=1, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='num_untested_examples', full_name='tensorflow_docs.api_report.schema.UsageExampleLint.num_untested_examples', index=1,
+      number=2, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=314,
+  serialized_end=384,
+)
+
+
+_RETURNLINT = _descriptor.Descriptor(
+  name='ReturnLint',
+  full_name='tensorflow_docs.api_report.schema.ReturnLint',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='returns_defined', full_name='tensorflow_docs.api_report.schema.ReturnLint.returns_defined', index=0,
+      number=1, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=386,
+  serialized_end=423,
+)
+
+
+_RAISESLINT = _descriptor.Descriptor(
+  name='RaisesLint',
+  full_name='tensorflow_docs.api_report.schema.RaisesLint',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='num_raises_defined', full_name='tensorflow_docs.api_report.schema.RaisesLint.num_raises_defined', index=0,
+      number=1, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='total_raises_in_code', full_name='tensorflow_docs.api_report.schema.RaisesLint.total_raises_in_code', index=1,
+      number=2, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=425,
+  serialized_end=495,
+)
+
+
+_APISYMBOLMETRIC = _descriptor.Descriptor(
+  name='ApiSymbolMetric',
+  full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='symbol_name', full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric.symbol_name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='object_type', full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric.object_type', index=1,
+      number=2, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='parameter_lint', full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric.parameter_lint', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='desc_lint', full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric.desc_lint', index=3,
+      number=4, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='usage_example_lint', full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric.usage_example_lint', index=4,
+      number=5, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='return_lint', full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric.return_lint', index=5,
+      number=6, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='raises_lint', full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric.raises_lint', index=6,
+      number=7, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='package_group', full_name='tensorflow_docs.api_report.schema.ApiSymbolMetric.package_group', index=7,
+      number=8, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=498,
+  serialized_end=989,
+)
+
+
+_APIREPORT = _descriptor.Descriptor(
+  name='ApiReport',
+  full_name='tensorflow_docs.api_report.schema.ApiReport',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='timestamp', full_name='tensorflow_docs.api_report.schema.ApiReport.timestamp', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='date', full_name='tensorflow_docs.api_report.schema.ApiReport.date', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='symbol_metric', full_name='tensorflow_docs.api_report.schema.ApiReport.symbol_metric', index=2,
+      number=3, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto2',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=992,
+  serialized_end=1139,
+)
+
+_APISYMBOLMETRIC.fields_by_name['object_type'].enum_type = _OBJECTTYPE
+_APISYMBOLMETRIC.fields_by_name['parameter_lint'].message_type = _PARAMETERLINT
+_APISYMBOLMETRIC.fields_by_name['desc_lint'].message_type = _DESCRIPTIONLINT
+_APISYMBOLMETRIC.fields_by_name['usage_example_lint'].message_type = _USAGEEXAMPLELINT
+_APISYMBOLMETRIC.fields_by_name['return_lint'].message_type = _RETURNLINT
+_APISYMBOLMETRIC.fields_by_name['raises_lint'].message_type = _RAISESLINT
+_APIREPORT.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP
+_APIREPORT.fields_by_name['symbol_metric'].message_type = _APISYMBOLMETRIC
+DESCRIPTOR.message_types_by_name['ParameterLint'] = _PARAMETERLINT
+DESCRIPTOR.message_types_by_name['DescriptionLint'] = _DESCRIPTIONLINT
+DESCRIPTOR.message_types_by_name['UsageExampleLint'] = _USAGEEXAMPLELINT
+DESCRIPTOR.message_types_by_name['ReturnLint'] = _RETURNLINT
+DESCRIPTOR.message_types_by_name['RaisesLint'] = _RAISESLINT
+DESCRIPTOR.message_types_by_name['ApiSymbolMetric'] = _APISYMBOLMETRIC
+DESCRIPTOR.message_types_by_name['ApiReport'] = _APIREPORT
+DESCRIPTOR.enum_types_by_name['ObjectType'] = _OBJECTTYPE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+ParameterLint = _reflection.GeneratedProtocolMessageType('ParameterLint', (_message.Message,), {
+  'DESCRIPTOR' : _PARAMETERLINT,
+  '__module__' : 'api_report_pb2'
+  # @@protoc_insertion_point(class_scope:tensorflow_docs.api_report.schema.ParameterLint)
+  })
+_sym_db.RegisterMessage(ParameterLint)
+
+DescriptionLint = _reflection.GeneratedProtocolMessageType('DescriptionLint', (_message.Message,), {
+  'DESCRIPTOR' : _DESCRIPTIONLINT,
+  '__module__' : 'api_report_pb2'
+  # @@protoc_insertion_point(class_scope:tensorflow_docs.api_report.schema.DescriptionLint)
+  })
+_sym_db.RegisterMessage(DescriptionLint)
+
+UsageExampleLint = _reflection.GeneratedProtocolMessageType('UsageExampleLint', (_message.Message,), {
+  'DESCRIPTOR' : _USAGEEXAMPLELINT,
+  '__module__' : 'api_report_pb2'
+  # @@protoc_insertion_point(class_scope:tensorflow_docs.api_report.schema.UsageExampleLint)
+  })
+_sym_db.RegisterMessage(UsageExampleLint)
+
+ReturnLint = _reflection.GeneratedProtocolMessageType('ReturnLint', (_message.Message,), {
+  'DESCRIPTOR' : _RETURNLINT,
+  '__module__' : 'api_report_pb2'
+  # @@protoc_insertion_point(class_scope:tensorflow_docs.api_report.schema.ReturnLint)
+  })
+_sym_db.RegisterMessage(ReturnLint)
+
+RaisesLint = _reflection.GeneratedProtocolMessageType('RaisesLint', (_message.Message,), {
+  'DESCRIPTOR' : _RAISESLINT,
+  '__module__' : 'api_report_pb2'
+  # @@protoc_insertion_point(class_scope:tensorflow_docs.api_report.schema.RaisesLint)
+  })
+_sym_db.RegisterMessage(RaisesLint)
+
+ApiSymbolMetric = _reflection.GeneratedProtocolMessageType('ApiSymbolMetric', (_message.Message,), {
+  'DESCRIPTOR' : _APISYMBOLMETRIC,
+  '__module__' : 'api_report_pb2'
+  # @@protoc_insertion_point(class_scope:tensorflow_docs.api_report.schema.ApiSymbolMetric)
+  })
+_sym_db.RegisterMessage(ApiSymbolMetric)
+
+ApiReport = _reflection.GeneratedProtocolMessageType('ApiReport', (_message.Message,), {
+  'DESCRIPTOR' : _APIREPORT,
+  '__module__' : 'api_report_pb2'
+  # @@protoc_insertion_point(class_scope:tensorflow_docs.api_report.schema.ApiReport)
+  })
+_sym_db.RegisterMessage(ApiReport)
+
+
+# @@protoc_insertion_point(module_scope)
\ No newline at end of file
diff --git a/tools/tensorflow_docs/api_generator/report/schema/api_report_generated_pb2.py b/tools/tensorflow_docs/api_generator/report/schema/api_report_generated_pb2.py
new file mode 100644
index 00000000000..fa046d85011
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/report/schema/api_report_generated_pb2.py
@@ -0,0 +1,56 @@
+# THIS IS A GENERATED FILE. DO NOT EDIT!
+# pylint: skip-file
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: api_report.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10\x61pi_report.proto\x12!tensorflow_docs.api_report.schema\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa2\x01\n\rParameterLint\x12!\n\x19num_empty_param_desc_args\x18\x01 \x01(\x02\x12\x18\n\x10num_args_in_code\x18\x02 \x01(\x02\x12!\n\x19num_empty_param_desc_attr\x18\x03 \x01(\x02\x12\x18\n\x10total_attr_param\x18\x04 \x01(\x02\x12\x17\n\x0fnum_args_in_doc\x18\x05 \x01(\x02\";\n\x0f\x44\x65scriptionLint\x12\x11\n\tlen_brief\x18\x01 \x01(\x02\x12\x15\n\rlen_long_desc\x18\x03 \x01(\x02\"F\n\x10UsageExampleLint\x12\x13\n\x0bnum_doctest\x18\x01 \x01(\x02\x12\x1d\n\x15num_untested_examples\x18\x02 \x01(\x02\"%\n\nReturnLint\x12\x17\n\x0freturns_defined\x18\x01 \x01(\x08\"F\n\nRaisesLint\x12\x1a\n\x12num_raises_defined\x18\x01 \x01(\x02\x12\x1c\n\x14total_raises_in_code\x18\x02 \x01(\x02\"\xeb\x03\n\x0f\x41piSymbolMetric\x12\x13\n\x0bsymbol_name\x18\x01 \x01(\t\x12\x42\n\x0bobject_type\x18\x02 \x01(\x0e\x32-.tensorflow_docs.api_report.schema.ObjectType\x12H\n\x0eparameter_lint\x18\x03 \x01(\x0b\x32\x30.tensorflow_docs.api_report.schema.ParameterLint\x12\x45\n\tdesc_lint\x18\x04 \x01(\x0b\x32\x32.tensorflow_docs.api_report.schema.DescriptionLint\x12O\n\x12usage_example_lint\x18\x05 \x01(\x0b\x32\x33.tensorflow_docs.api_report.schema.UsageExampleLint\x12\x42\n\x0breturn_lint\x18\x06 \x01(\x0b\x32-.tensorflow_docs.api_report.schema.ReturnLint\x12\x42\n\x0braises_lint\x18\x07 \x01(\x0b\x32-.tensorflow_docs.api_report.schema.RaisesLint\x12\x15\n\rpackage_group\x18\x08 \x01(\t\"\x93\x01\n\tApiReport\x12-\n\ttimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04\x64\x61te\x18\x02 \x01(\t\x12I\n\rsymbol_metric\x18\x03 \x03(\x0b\x32\x32.tensorflow_docs.api_report.schema.ApiSymbolMetric*M\n\nObjectType\x12\t\n\x05\x43LASS\x10\x00\x12\n\n\x06METHOD\x10\x01\x12\x0c\n\x08\x46UNCTION\x10\x02\x12\n\n\x06MODULE\x10\x03\x12\x0e\n\nTYPE_ALIAS\x10\x04')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'api_report_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  _OBJECTTYPE._serialized_start=1141
+  _OBJECTTYPE._serialized_end=1218
+  _PARAMETERLINT._serialized_start=89
+  _PARAMETERLINT._serialized_end=251
+  _DESCRIPTIONLINT._serialized_start=253
+  _DESCRIPTIONLINT._serialized_end=312
+  _USAGEEXAMPLELINT._serialized_start=314
+  _USAGEEXAMPLELINT._serialized_end=384
+  _RETURNLINT._serialized_start=386
+  _RETURNLINT._serialized_end=423
+  _RAISESLINT._serialized_start=425
+  _RAISESLINT._serialized_end=495
+  _APISYMBOLMETRIC._serialized_start=498
+  _APISYMBOLMETRIC._serialized_end=989
+  _APIREPORT._serialized_start=992
+  _APIREPORT._serialized_end=1139
+# @@protoc_insertion_point(module_scope)
\ No newline at end of file
diff --git a/tools/tensorflow_docs/api_generator/report/utils.py b/tools/tensorflow_docs/api_generator/report/utils.py
new file mode 100644
index 00000000000..6db22cf52e0
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/report/utils.py
@@ -0,0 +1,160 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for generating report for a package."""
+
+from tensorflow_docs.api_generator import public_api
+from tensorflow_docs.api_generator.pretty_docs import base_page
+from tensorflow_docs.api_generator.pretty_docs import class_page
+from tensorflow_docs.api_generator.pretty_docs import function_page
+from tensorflow_docs.api_generator.report import linter
+
+from tensorflow_docs.api_generator.report.schema import api_report_pb2
+from google.protobuf import timestamp_pb2
+
+
+class ApiReport:
+  """Generates the API report for a package."""
+
+  def __init__(self):
+    self.api_report = api_report_pb2.ApiReport()
+
+    invocation_timestamp = timestamp_pb2.Timestamp()
+    invocation_timestamp.GetCurrentTime()
+    self.api_report.timestamp.CopyFrom(invocation_timestamp)
+    self.api_report.date = invocation_timestamp.ToJsonString()
+
+  def write(self, path):
+    api_report = api_report_pb2.ApiReport(
+        timestamp=self.api_report.timestamp,
+        date=self.api_report.date,
+        symbol_metric=sorted(
+            self.api_report.symbol_metric, key=lambda sm: sm.symbol_name))
+
+    path.write_bytes(api_report.SerializeToString())
+
+  def _lint(
+      self,
+      *,
+      name: str,
+      object_type: api_report_pb2.ObjectType,
+      package_group: str,
+      page_info: base_page.PageInfo,
+  ) -> None:
+    self.api_report.symbol_metric.add(
+        symbol_name=name,
+        object_type=object_type,
+        package_group=package_group,
+        parameter_lint=linter.lint_params(page_info),
+        desc_lint=linter.lint_description(page_info),
+        usage_example_lint=linter.lint_usage_example(page_info),
+        return_lint=linter.lint_returns(page_info),
+        raises_lint=linter.lint_raises(page_info),
+    )
+
+  def _find_pkg_group(self, name: str) -> str:
+    name_list = name.split('.')
+    # name = 'tf.keras.layers'; name_list = ['tf', 'keras', 'layers']
+    # Number of dots in name == 2 == len(name_list) - 1
+    dot_count = len(name_list) - 1
+    if dot_count == 1:
+      return name_list[0]
+    return '.'.join(name_list[:2])
+
+  def _make_constructor_info(
+      self, class_page_info: class_page.ClassPageInfo) -> base_page.PageInfo:
+    """Convert a class description into a description of the constructor."""
+    methods = class_page.split_methods(class_page_info.methods)
+
+    constructor_info = base_page.PageInfo(api_node=class_page_info.api_node)
+
+    # Replace the class py_object with constructors py_object. This is done
+    # because each method is linted separately and class py_object contains the
+    # source code of all its methods too.
+    if methods.constructor is not None:
+      constructor_info.py_object = methods.constructor.py_object
+    else:
+      constructor_info.py_object = None
+
+    # Merge the constructor and class docstrings.
+    class_blocks = class_page.merge_blocks(class_page_info, methods.constructor)
+    # Add the `Attributes` sections (if it exists) to the merged class blocks.
+    if class_page_info.attr_block is not None:
+      class_blocks.append(class_page_info.attr_block)
+
+    new_doc = class_page_info.doc._replace(docstring_parts=class_blocks)
+
+    constructor_info.set_doc(new_doc)
+
+    return constructor_info
+
+  def _fill_class_metric(self,
+                         class_page_info: class_page.ClassPageInfo) -> None:
+    """Fills in the lint metrics for a class and its methods.
+
+    The constructor and class's docstring is merged for linting. Class's
+    `py_object` is replaced with that class's constructor's `py_object`.
+
+    Every other method except `__init__` or `__new__` is linted separately.
+
+    Args:
+      class_page_info: A `ClassPageInfo` object containing information that's
+        used to calculate metrics for the class and its methods.
+    """
+    constructor_info = self._make_constructor_info(class_page_info)
+    package_group = self._find_pkg_group(class_page_info.full_name)
+
+    self._lint(
+        name=class_page_info.full_name,
+        object_type=api_report_pb2.ObjectType.CLASS,
+        package_group=package_group,
+        page_info=constructor_info,
+    )
+
+    methods: class_page.Methods = class_page.split_methods(
+        class_page_info.methods)
+    # Lint each method separately and add its metrics to the proto object.
+    for method in methods.info_dict.values():
+      # Skip the dunder methods from being in the report.
+      if method.short_name not in public_api.ALLOWED_DUNDER_METHODS:
+        self._lint(
+            name=method.full_name,
+            object_type=api_report_pb2.ObjectType.METHOD,
+            # Since methods are part of a class, each method belongs to the
+            # package group of the respective class.
+            package_group=package_group,
+            page_info=method,
+        )
+
+  def _fill_function_metric(self,
+                            function_page_info: function_page.FunctionPageInfo):
+    """Fills in the lint metrics for a function.
+
+    Args:
+      function_page_info: A `FunctionPageInfo` object containing information
+        that's used to calculate metrics for the function.
+    """
+    self._lint(
+        name=function_page_info.full_name,
+        object_type=api_report_pb2.ObjectType.FUNCTION,
+        package_group=self._find_pkg_group(function_page_info.full_name),
+        page_info=function_page_info,
+    )
+
+  def fill_metrics(self, page_info: base_page.PageInfo) -> None:
+    if isinstance(page_info, class_page.ClassPageInfo):
+      self._fill_class_metric(page_info)
+
+    if isinstance(page_info, function_page.FunctionPageInfo):
+      self._fill_function_metric(page_info)
diff --git a/tools/tensorflow_docs/api_generator/signature.py b/tools/tensorflow_docs/api_generator/signature.py
new file mode 100644
index 00000000000..dacf5d2bada
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/signature.py
@@ -0,0 +1,682 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Turn Python docstrings into Markdown for TensorFlow documentation."""
+
+import ast
+import copy
+import dataclasses
+import enum
+import functools
+import html
+import inspect
+import re
+import textwrap
+import typing
+
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type
+
+import astor
+
+from tensorflow_docs.api_generator import config
+from tensorflow_docs.api_generator import get_source
+from tensorflow_docs.api_generator import public_api
+
+EMPTY = inspect.Signature.empty
+
+
+def _source_from_ast(node: ast.AST) -> str:
+  return astor.to_source(node).strip().replace('"""', "'")
+
+
+class _BaseDefaultAndAnnotationExtractor(ast.NodeVisitor):
+  """A base class for extracting annotations and defaults from the AST."""
+  _PAREN_NUMBER_RE = re.compile(r'^\((True|False|[0-9.e-]+)\)')
+
+  def __init__(self):
+    self.annotations = {}
+    self.defaults = {}
+    self.return_annotation = EMPTY
+
+  def _preprocess_default(self, val: ast.AST) -> str:
+    text_default_val = (
+        _source_from_ast(val).replace('\t', '\\t').replace('\n', '\\n'))
+    text_default_val = self._PAREN_NUMBER_RE.sub('\\1', text_default_val)
+    return text_default_val
+
+  def extract(self, obj: Any):
+    obj_ast = get_source.get_ast(obj)
+    if obj_ast is not None:
+      self.visit(obj_ast)
+
+
+class _ArgDefaultAndAnnotationExtractor(_BaseDefaultAndAnnotationExtractor):
+  """Extracts the type annotations by parsing the AST of a function."""
+
+  def visit_FunctionDef(self, node) -> None:  # pylint: disable=invalid-name
+    """Visits the `FunctionDef` node in AST tree and extracts the typehints."""
+
+    # Capture the return type annotation.
+    if node.returns:
+      self.return_annotation = _source_from_ast(node.returns)
+
+    # Capture the args type annotation.
+    for arg in node.args.args:
+      if arg.annotation:
+        self.annotations[arg.arg] = _source_from_ast(arg.annotation)
+        self.arguments_typehint_exists = True
+
+    # Capture the kwarg only args type annotation.
+    for kwarg in node.args.kwonlyargs:
+      if kwarg.annotation:
+        self.annotations[kwarg.arg] = _source_from_ast(kwarg.annotation)
+        self.arguments_typehint_exists = True
+
+    # From https://docs.python.org/3/library/ast.html#ast.arguments:
+    #   `defaults` is a list of default values for arguments that can be passed
+    #   positionally. If there are fewer defaults, they correspond to the last
+    #   n arguments.
+
+    last_n_pos_args = node.args.args[-1 * len(node.args.defaults):]
+    for arg, default_val in zip(last_n_pos_args, node.args.defaults):
+      if default_val is not None:
+        text_default_val = self._preprocess_default(default_val)
+        self.defaults[arg.arg] = text_default_val
+
+    for kwarg, default_val in zip(node.args.kwonlyargs, node.args.kw_defaults):
+      if default_val is not None:
+        text_default_val = self._preprocess_default(default_val)
+        self.defaults[kwarg.arg] = text_default_val
+
+
+class _ClassDefaultAndAnnotationExtractor(_BaseDefaultAndAnnotationExtractor):
+  """Extracts the type annotations by parsing the AST of a dataclass."""
+
+  def __init__(self):
+    super().__init__()
+    self.annotations = {}
+    self.defaults = {}
+    self.return_annotation = EMPTY
+
+  def visit_ClassDef(self, node) -> None:  # pylint: disable=invalid-name
+    # Don't visit all nodes. Only visit top-level AnnAssign nodes so that
+    # If there's an AnnAssign in a method it doesn't get picked up.
+    for sub in node.body:
+      if isinstance(sub, ast.AnnAssign):
+        self.visit_AnnAssign(sub)
+      elif isinstance(sub, ast.Assign):
+        self.visit_Assign(sub)
+
+  def visit_AnnAssign(self, node) -> None:  # pylint: disable=invalid-name
+    """Vists an assignment with a type annotation. Dataclasses is an example."""
+
+    arg = _source_from_ast(node.target)
+    self.annotations[arg] = _source_from_ast(node.annotation)
+    if node.value is not None:
+      self.defaults[arg] = self._preprocess_default(node.value)
+
+  def visit_Assign(self, node) -> None:  # pylint: disable=invalid-name
+    """Vists an assignment with a type annotation. Dataclasses is an example."""
+    names = [_source_from_ast(t) for t in node.targets]
+    if node.value is not None:
+      val = self._preprocess_default(node.value)
+      for name in names:
+        self.defaults[name] = val
+
+  def extract(self, cls):
+    # Iterate over the classes in reverse order so each class overwrites it's
+    # parents. Skip `object`.
+    for cls in reversed(cls.__mro__[:-1]):
+      super().extract(cls)
+
+
+_OBJECT_MEMORY_ADDRESS_RE = re.compile(r'<(?P.+?) at 0x[\da-f]+>')
+
+
+def strip_obj_addresses(text):
+  return _OBJECT_MEMORY_ADDRESS_RE.sub(r'<\g>', text)
+
+
+class FormatArguments(object):
+  """Formats the arguments and adds type annotations if they exist."""
+
+  # A regular expression capturing a python identifier.
+  _IDENTIFIER_RE = r'[a-zA-Z_]\w*'
+
+  _INDIVIDUAL_TYPES_RE = re.compile(
+      r"""
+        (?P
+          ([\w.]*)
+          (?=$|,| |\]|\[)
+        )
+      """, re.IGNORECASE | re.VERBOSE)
+
+  _TYPING = frozenset(
+      list(typing.__dict__.keys()) +
+      ['int', 'str', 'bytes', 'float', 'complex', 'bool', 'None'])
+
+  _IMMUTABLE_TYPES = frozenset([
+      int, str, bytes, float, complex, bool, Ellipsis,
+      type(None), tuple, frozenset
+  ])
+
+  def __init__(
+      self,
+      parser_config: config.ParserConfig,
+  ) -> None:
+    self._reverse_index = parser_config.reverse_index
+    self._reference_resolver = parser_config.reference_resolver
+
+  def get_link(self,
+               link_text: str,
+               obj_full_name: Optional[str] = None) -> str:
+    return self._reference_resolver.python_link(
+        link_text=link_text, ref_full_name=obj_full_name)
+
+  def _extract_non_builtin_types(self, arg_obj: Any,
+                                 non_builtin_types: List[Any]) -> List[Any]:
+    """Extracts the non-builtin types from a type annotations object.
+
+    Recurses if an object contains `__args__` attribute. If an object is
+    an inbuilt object or an `Ellipsis` then its skipped.
+
+    Args:
+      arg_obj: Type annotation object.
+      non_builtin_types: List to keep track of the non-builtin types extracted.
+
+    Returns:
+      List of non-builtin types.
+    """
+
+    annotations = getattr(arg_obj, '__args__', [arg_obj])
+    if annotations is None:
+      annotations = [arg_obj]
+
+    for anno in annotations:
+      if self._reverse_index.get(id(anno), None):
+        non_builtin_types.append(anno)
+      elif (anno in self._IMMUTABLE_TYPES or
+            id(type(anno)) in public_api._TYPING_IDS):  # pylint: disable=protected-access
+        continue
+      elif hasattr(anno, '__args__'):
+        self._extract_non_builtin_types(anno, non_builtin_types)
+      else:
+        non_builtin_types.append(anno)
+    return non_builtin_types
+
+  def _get_non_builtin_ast_types(self, ast_typehint: str) -> List[str]:
+    """Extracts non-builtin types from a string AST type annotation.
+
+    If the type is an inbuilt type or an `...`(Ellipsis) then its skipped.
+
+    Args:
+      ast_typehint: AST extracted type annotation.
+
+    Returns:
+      List of non-builtin ast types.
+    """
+    non_builtin_ast_types = []
+    for single_type, _ in self._INDIVIDUAL_TYPES_RE.findall(ast_typehint):
+      if (not single_type or single_type in self._TYPING or
+          single_type == '...'):
+        continue
+      non_builtin_ast_types.append(single_type)
+    return non_builtin_ast_types
+
+  def _linkify(self, non_builtin_map: Dict[str, Any], match) -> str:
+    """Links off to types that can be linked.
+
+    Args:
+      non_builtin_map: Dictionary mapping non-builtin_ast_types to
+        non_builtin_type_objs
+      match: Match object returned by `re.sub`.
+
+    Returns:
+      Linked type annotation if the type annotation object exists.
+    """
+
+    group = match.groupdict()
+    ast_single_typehint = group['single_type']
+
+    # If the AST type hint is a built-in type hint or an `Ellipsis`,
+    # return it as is.
+    if ast_single_typehint not in non_builtin_map:
+      return ast_single_typehint
+
+    if not non_builtin_map:
+      return ast_single_typehint
+
+    # Get the type object from the ast_single_typehint and lookup the object
+    # in reverse_index to get its full name.
+    obj_full_name = self._reverse_index.get(
+        id(non_builtin_map[ast_single_typehint]), None)
+    if obj_full_name is None:
+      return ast_single_typehint
+
+    return self.get_link(obj_full_name)
+
+  def maybe_add_link(self, source: str, value: Any) -> str:
+    """Return a link to an object's api page if found.
+
+    Args:
+      source: The source string from the code.
+      value: The value of the object.
+
+    Returns:
+      The original string with maybe an HTML link added.
+    """
+    cls = type(value)
+
+    value_name = self._reverse_index.get(id(value), None)
+    cls_name = self._reverse_index.get(id(cls), None)
+
+    if cls_name is not None:
+      # It's much more common for the class to be documented than the instance.
+      # and the class page will provide better docs.
+      before = source.split('(')[0]
+      cls_short_name = cls_name.split('.')[-1]
+      if before.endswith(cls_short_name):
+        # Yes, this is a guess but it will usually be right.
+        return self.get_link(source, cls_name)
+
+    if value_name is not None:
+      return self.get_link(value_name, value_name)
+
+    return source
+
+  def preprocess(self, string: str, value: Any) -> str:
+    """Links type annotations to its page if it exists.
+
+    Args:
+      string: AST extracted type annotation.
+      value: Type annotation object.
+
+    Returns:
+      Linked type annotation if the type annotation object exists.
+    """
+    # If the object annotations exists in the reverse_index, get the link
+    # directly for the entire annotation.
+    obj_anno_full_name = self._reverse_index.get(id(value), None)
+    if obj_anno_full_name is not None:
+      return self.get_link(obj_anno_full_name)
+
+    non_builtin_ast_types = self._get_non_builtin_ast_types(string)
+    try:
+      non_builtin_type_objs = self._extract_non_builtin_types(value, [])
+    except RecursionError:
+      non_builtin_type_objs = {}
+
+    # If the length doesn't match then don't linkify any type annotation. This
+    # is done to avoid linking to wrong pages instead of guessing.
+    if len(non_builtin_type_objs) != len(non_builtin_ast_types):
+      non_builtin_map = {}
+    else:
+      non_builtin_map = dict(zip(non_builtin_ast_types, non_builtin_type_objs))
+
+    partial_func = functools.partial(self._linkify, non_builtin_map)
+    return self._INDIVIDUAL_TYPES_RE.sub(partial_func, string)
+
+  def format_return(self, return_anno: Tuple[Any, str]) -> str:
+    value, source = return_anno
+    return self.preprocess(source, value)
+
+  def format_args(self, args: List[inspect.Parameter]) -> List[str]:
+    """Creates a text representation of the args in a method/function.
+
+    Args:
+      args: List of args to format.
+
+    Returns:
+      Formatted args with type annotations if they exist.
+    """
+
+    args_text_repr = []
+
+    for arg in args:
+      typeanno = None
+      if arg.annotation is not EMPTY:
+        value, source = arg.annotation
+        if source is not None:
+          typeanno = self.preprocess(source, value)
+
+      if typeanno:
+        args_text_repr.append(f'{arg.name}: {typeanno}')
+      else:
+        args_text_repr.append(f'{arg.name}')
+
+    return args_text_repr
+
+  def format_kwargs(self, kwargs: List[inspect.Parameter]) -> List[str]:
+    """Creates a text representation of the kwargs in a method/function.
+
+    Args:
+      kwargs: List of kwargs to format.
+
+    Returns:
+      Formatted kwargs with type annotations if they exist.
+    """
+
+    kwargs_text_repr = []
+
+    for kwarg in kwargs:
+      default_text = None
+      if kwarg.default is not EMPTY:
+        default_val, default_source = kwarg.default
+        if default_source is None:
+          default_source = strip_obj_addresses(repr(default_val))
+        default_source = html.escape(default_source)
+
+        default_text = self.maybe_add_link(default_source, default_val)
+
+      # Format the kwargs to add the type annotation and default values.
+      typeanno = None
+      if kwarg.annotation is not EMPTY:
+        anno_value, anno_source = kwarg.annotation
+        if anno_source is not None:
+          typeanno = self.preprocess(anno_source, anno_value)
+
+      if typeanno is not None and default_text is not None:
+        kwargs_text_repr.append(f'{kwarg.name}: {typeanno} = {default_text}')
+      elif default_text is not None:
+        kwargs_text_repr.append(f'{kwarg.name}={default_text}')
+      elif typeanno is not None:
+        kwargs_text_repr.append(f'{kwarg.name}: {typeanno}')
+      else:
+        kwargs_text_repr.append(kwarg.name)
+
+    return kwargs_text_repr
+
+
+class TfSignature(inspect.Signature):
+  """A custom version of `inspect.Signature`."""
+
+  def __init__(self, parameters, *, return_annotation, parser_config):
+    super().__init__(parameters, return_annotation=return_annotation)  # pytype: disable=wrong-arg-types  # mapping-is-not-sequence
+    self.parser_config = parser_config
+
+  def replace(self, **kwargs):
+    attrs = {
+        'parameters': self.parameters,
+        'return_annotation': self.return_annotation,
+        'parser_config': self.parser_config,
+    }
+    attrs.update(kwargs)
+    return type(self)(**attrs)
+
+  def __copy__(self):
+    return TfSignature(
+        list(self.parameters.values()),
+        return_annotation=self.return_annotation,
+        parser_config=self.parser_config)
+
+  def __deepcopy__(self, memo):
+    return TfSignature(
+        copy.deepcopy(list(self.parameters.values()), memo),
+        return_annotation=copy.deepcopy(self.return_annotation, memo),
+        parser_config=copy.deepcopy(self.parser_config, memo))
+
+  def __str__(self):
+    # separate the args by type
+    pos_only_args = []
+    args = []
+    kwargs = []
+    only_kwargs = []
+    varargs = None
+    varkwargs = None
+
+    for index, param in enumerate(self.parameters.values()):
+      kind = param.kind
+      default = param.default
+
+      if kind == param.POSITIONAL_ONLY:
+        pos_only_args.append(param)
+      elif default is EMPTY and kind == param.POSITIONAL_OR_KEYWORD:
+        args.append(param)
+      elif default is not EMPTY and kind == param.POSITIONAL_OR_KEYWORD:
+        kwargs.append(param)
+      elif kind == param.VAR_POSITIONAL:
+        varargs = (index, param)
+      elif kind == param.KEYWORD_ONLY:
+        only_kwargs.append(param)
+      elif kind == param.VAR_KEYWORD:
+        varkwargs = param
+
+    # Build the text representation.
+    all_args_list = []
+
+    formatter = FormatArguments(parser_config=self.parser_config)
+
+    if pos_only_args:
+      all_args_list.extend(formatter.format_args(pos_only_args))
+      all_args_list.append('/')
+
+    if args:
+      all_args_list.extend(formatter.format_args(args))
+
+    if kwargs:
+      all_args_list.extend(formatter.format_kwargs(kwargs))
+
+    if only_kwargs:
+      if varargs is None:
+        all_args_list.append('*')
+      all_args_list.extend(formatter.format_kwargs(only_kwargs))
+
+    if varargs is not None:
+      all_args_list.insert(varargs[0], '*' + varargs[1].name)
+
+    if varkwargs is not None:
+      all_args_list.append('**' + varkwargs.name)
+
+    return_annotation_text = ''
+    if self.return_annotation is not EMPTY:
+      if EMPTY not in self.return_annotation:
+        return_annotation_text = formatter.format_return(self.return_annotation)
+
+    arguments_signature = ''
+    has_any_annotations = any(
+        v.annotation is not EMPTY for v in self.parameters.values())
+    if all_args_list:
+      str_signature = ',\n'.join(all_args_list)
+      # If it fits on one line flatten it.
+      if len(str_signature) + 4 < 80:
+        str_signature = textwrap.fill(str_signature, width=80)
+
+      arguments_signature = '\n' + textwrap.indent(
+          str_signature, prefix='    ') + '\n'
+
+    full_signature = f'({arguments_signature})'
+    if return_annotation_text:
+      full_signature = f'({arguments_signature}) -> {return_annotation_text}'
+    else:
+      full_signature = f'({arguments_signature})'
+    return full_signature
+
+
+class FuncType(enum.Enum):
+  """Enum to recognize type of function passed to `generate_signature`."""
+  FUNCTION = 'function'
+  METHOD = 'method'
+  CLASSMETHOD = 'classmethod'
+
+
+def get_method_type(method, name, is_dataclass):
+  """Determine the type of callable."""
+  if isinstance(method, classmethod):
+    func_type = FuncType.CLASSMETHOD
+  elif name == '__new__':
+    # __new__ acts like a regular method for this.
+    # - At this point all args are visible in the signature.
+    # - When used the first argument gets boound (like self).
+    # - Sometimes users wrap it with a `staticmethod` but that gets ignored.
+    func_type = FuncType.METHOD
+  elif isinstance(method, staticmethod):
+    func_type = FuncType.FUNCTION
+  elif is_dataclass:
+    # When building the init signature directly from a dataclass-class (for
+    # the auto-generated __init__) `self` is already removed from the
+    # signature.
+    func_type = FuncType.FUNCTION
+  else:
+    func_type = FuncType.METHOD
+  return func_type
+
+
+def generate_signature(
+    func: Any,
+    parser_config: config.ParserConfig,
+    func_type: FuncType = FuncType.FUNCTION,
+) -> TfSignature:
+  """Given a function, returns a list of strings representing its args.
+
+  This function uses `__name__` for callables if it is available. This can lead
+  to poor results for functools.partial and other callable objects.
+
+  The returned string is Python code, so if it is included in a Markdown
+  document, it should be typeset as code (using backticks), or escaped.
+
+  Args:
+    func: A function, method, or functools.partial to extract the signature for.
+    parser_config: `config.ParserConfig` for the method/function whose signature
+      is generated.
+    func_type: Type of the current `func`. This is required because there isn't
+      a clear distinction between function and method being passed to
+      `generate_signature`. Sometimes methods are detected as function by
+      `inspect`. Since we know the type of `func` when generate_signature is
+      called, use that to pass the type of `func`.
+
+  Returns:
+    A `SignatureComponents` NamedTuple.
+  """
+  try:
+    sig = inspect.signature(func)
+  except (ValueError, TypeError):
+    sig = inspect.signature(lambda: None)
+
+  params = list(sig.parameters.values())
+
+  # Drop `self`
+  if params:
+    first = params[0]
+    if first.kind != first.VAR_POSITIONAL:
+      if func_type == FuncType.METHOD:
+        # - Skip the first arg for regular methods.
+        # - Some wrapper methods forget `self` and just use `(*args, **kwargs)`.
+        #   That's still valid, don't drop `*args`.
+        # - For classmethods the `cls` arg already bound here (it's not in
+        #   `params`).
+        # - For regular functions (or staticmethods) you never need to skip.
+        params.pop(0)
+
+  sig = sig.replace(parameters=params)
+
+  if dataclasses.is_dataclass(func) and inspect.isclass(func):
+    sig = sig.replace(return_annotation=EMPTY)
+    extract_fn = _extract_class_defaults_and_annotations
+  else:
+    extract_fn = _extract_arg_defaults_and_annotations
+
+  (annotation_source_dict, defaults_source_dict,
+   return_annotation_source) = extract_fn(func)
+
+  # Replace everything with either `EMPTY` or (value, source) pairs.
+  new_params = []
+  for name, param in sig.parameters.items():
+    default = param.default
+    if default is not EMPTY:
+      default = (default, defaults_source_dict.get(name, None))
+
+    annotation = param.annotation
+    if annotation is not EMPTY:
+      annotation = (annotation, annotation_source_dict.get(name, None))
+
+    param = param.replace(default=default, annotation=annotation)
+    new_params.append(param)
+
+  return_annotation = sig.return_annotation
+  if return_annotation is not EMPTY:
+    return_annotation = (return_annotation, return_annotation_source)
+
+  sig = TfSignature(
+      parameters=new_params,
+      return_annotation=return_annotation,
+      parser_config=parser_config)
+
+  return sig
+
+
+AnnotsDefaultsReturns = Tuple[Dict[str, str], Dict[str, str], Any]
+
+
+def _extract_class_defaults_and_annotations(
+    cls: Type[object]) -> AnnotsDefaultsReturns:
+  """Extract ast defaults and annotations form a dataclass."""
+  ast_visitor = _ClassDefaultAndAnnotationExtractor()
+  ast_visitor.extract(cls)
+
+  return (ast_visitor.annotations, ast_visitor.defaults,
+          ast_visitor.return_annotation)
+
+
+def _extract_arg_defaults_and_annotations(
+    func: Callable[..., Any]) -> AnnotsDefaultsReturns:
+  """Extract ast defaults and annotations form a standard callable."""
+
+  ast_visitor = _ArgDefaultAndAnnotationExtractor()
+
+  annotation_source_dict = {}
+  defaults_source_dict = {}
+  return_annotation_source = EMPTY
+
+  try:
+    # Extract the type annotation from the parsed ast.
+    ast_visitor.extract(func)
+  except Exception:  # pylint: disable=broad-except
+    # A wide-variety of errors can be thrown here.
+    pass
+  else:
+    annotation_source_dict = ast_visitor.annotations
+    defaults_source_dict = ast_visitor.defaults
+    return_annotation_source = ast_visitor.return_annotation
+
+  return annotation_source_dict, defaults_source_dict, return_annotation_source
+
+
+def extract_decorators(func: Any) -> List[str]:
+  """Extracts the decorators on top of functions/methods.
+
+  Args:
+    func: The function to extract the decorators from.
+
+  Returns:
+    A List of decorators.
+  """
+
+  class ASTDecoratorExtractor(ast.NodeVisitor):
+
+    def __init__(self):
+      self.decorator_list = []
+
+    def visit_FunctionDef(self, node):  # pylint: disable=invalid-name
+      for dec in node.decorator_list:
+        self.decorator_list.append(_source_from_ast(dec))
+
+  visitor = ASTDecoratorExtractor()
+
+  # Note: get_source doesn't include the decorator lines on classes,
+  # this won't work for classes until that's fixed.
+  func_ast = get_source.get_ast(func)
+  if func_ast is not None:
+    visitor.visit(func_ast)
+
+  return visitor.decorator_list
diff --git a/tools/tensorflow_docs/api_generator/signature_test.py b/tools/tensorflow_docs/api_generator/signature_test.py
new file mode 100644
index 00000000000..e988da28cc0
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/signature_test.py
@@ -0,0 +1,471 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for documentation parser."""
+
+import dataclasses
+import textwrap
+import types
+
+from typing import Callable, Dict, List, Optional, Union
+
+from absl.testing import absltest
+from absl.testing import parameterized
+
+from tensorflow_docs.api_generator import config
+from tensorflow_docs.api_generator import doc_generator_visitor
+from tensorflow_docs.api_generator import generate_lib
+from tensorflow_docs.api_generator import parser
+from tensorflow_docs.api_generator import reference_resolver as reference_resolver_lib
+from tensorflow_docs.api_generator import signature
+from tensorflow_docs.api_generator.pretty_docs import class_page
+from tensorflow_docs.api_generator.pretty_docs import type_alias_page
+
+
+@dataclasses.dataclass
+class ExampleDataclass:
+  x: List[str]
+  z: int
+  c: List[int] = dataclasses.field(default_factory=list)
+  a: Union[List[str], str, int] = None
+  b: str = 'test'
+  y: bool = False
+
+  def add(self, x: int, y: int) -> int:
+    q: int = x + y
+    return q
+
+
+class TestGenerateSignature(parameterized.TestCase, absltest.TestCase):
+
+  def setUp(self):
+    super().setUp()
+    self.known_object = object()
+
+    m = types.ModuleType('m')
+    m.__file__ = __file__
+    m.extract_decorators = signature.extract_decorators
+    m.submodule = types.ModuleType('submodule')
+    m.submodule.known = self.known_object
+
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('m', m)],
+        code_url_prefix='https://tensorflow.org')
+
+    self.parser_config = generator.run_extraction()
+
+
+  def test_known_object(self):
+
+    def example_fun(arg=self.known_object):  # pylint: disable=unused-argument
+      pass
+
+    self.parser_config.reference_resolver = (
+        self.parser_config.reference_resolver.with_prefix('/'))
+
+    sig = signature.generate_signature(
+        example_fun,
+        parser_config=self.parser_config,
+        func_type=signature.FuncType.FUNCTION)
+
+    expected = textwrap.dedent("""\
+        (
+            arg=m.submodule.known
+        )""")
+
+    self.assertEqual(expected, str(sig))
+
+  def test_literals(self):
+
+    def example_fun(
+        self,
+        cls,
+        a=5,
+        b=5.0,
+        c=None,
+        d=True,
+        e='hello',
+        f=(1, (2, 3)),
+    ):  # pylint: disable=g-bad-name, unused-argument
+      pass
+
+    sig = signature.generate_signature(
+        example_fun,
+        parser_config=self.parser_config,
+        func_type=signature.FuncType.FUNCTION)
+
+    expected = textwrap.dedent("""\
+        (
+            self, cls, a=5, b=5.0, c=None, d=True, e='hello', f=(1, (2, 3))
+        )""")
+    self.assertEqual(expected, str(sig))
+
+  def test_dotted_name(self):
+    # pylint: disable=g-bad-name
+
+    class a:
+
+      class b:
+
+        class c:
+
+          class d:
+
+            def __init__(self, *args):
+              pass
+
+    # pylint: enable=g-bad-name
+
+    e = {'f': 1}
+
+    def example_fun(arg1=a.b.c.d, arg2=a.b.c.d(1, 2), arg3=e['f']):  # pylint: disable=unused-argument
+      pass
+
+    sig = signature.generate_signature(
+        example_fun,
+        parser_config=self.parser_config,
+        func_type=signature.FuncType.FUNCTION)
+    expected = ('(\n    arg1=a.b.c.d, arg2=a.b.c.d(1, 2), '
+                'arg3=e['f']\n)')
+    self.assertEqual(expected, str(sig))
+
+  def test_compulsory_kwargs_without_defaults(self):
+
+    def example_fun(x, z, a=True, b='test', *, c, y=None, d, **kwargs) -> bool:  # pylint: disable=unused-argument
+      return True
+
+    sig = signature.generate_signature(
+        example_fun,
+        parser_config=self.parser_config,
+        func_type=signature.FuncType.FUNCTION)
+    self.assertEqual(
+        list(sig.parameters.keys()),
+        ['x', 'z', 'a', 'b', 'c', 'y', 'd', 'kwargs'])
+    expected = textwrap.dedent("""\
+    (
+        x, z, a=True, b='test', *, c, y=None, d, **kwargs
+    ) -> bool""")
+    self.assertEqual(expected, str(sig))
+
+  def test_compulsory_kwargs_without_defaults_with_args(self):
+
+    def example_fun(x, z, cls, *args, a=True, b='test', y=None, c, **kwargs):  # pylint: disable=unused-argument
+      return True
+
+    sig = signature.generate_signature(
+        example_fun,
+        parser_config=self.parser_config,
+        func_type=signature.FuncType.FUNCTION)
+    self.assertEqual(
+        list(sig.parameters.keys()),
+        ['x', 'z', 'cls', 'args', 'a', 'b', 'y', 'c', 'kwargs'])
+    self.assertEqual(
+        str(sig),
+        '(\n    x, z, cls, *args, a=True, b='test', y=None, c, **kwargs\n)'
+    )
+
+  def test_type_annotations(self):
+    # pylint: disable=unused-argument
+
+    class TestMethodSig:
+
+      def example_fun(self,
+                      x: List[str],
+                      z: int,
+                      a: Union[List[str], str, int] = None,
+                      b: str = 'test',
+                      *,
+                      y: bool = False,
+                      c: Callable[..., int],
+                      **kwargs) -> None:
+        pass
+
+    # pylint: enable=unused-argument
+
+    sig = signature.generate_signature(
+        TestMethodSig.example_fun,
+        parser_config=self.parser_config,
+        func_type=signature.FuncType.METHOD,
+    )
+    expected = textwrap.dedent("""\
+        (
+            x: List[str],
+            z: int,
+            a: Union[List[str], str, int] = None,
+            b: str = 'test',
+            *,
+            y: bool = False,
+            c: Callable[..., int],
+            **kwargs
+        ) -> None""")
+    self.assertEqual(expected, str(sig))
+
+  def test_dataclasses_type_annotations(self):
+
+    sig = signature.generate_signature(
+        ExampleDataclass,
+        parser_config=self.parser_config,
+        func_type=signature.FuncType.FUNCTION)
+
+    expected = textwrap.dedent("""\
+      (
+          x: List[str],
+          z: int,
+          c: List[int] = dataclasses.field(default_factory=list),
+          a: Union[List[str], str, int] = None,
+          b: str = 'test',
+          y: bool = False
+      )""")
+    self.assertEqual(expected, str(sig))
+
+  @parameterized.named_parameters(
+      ('deep_objects', Union[Dict[str, Dict[bool, signature.extract_decorators]],
+                             int, bool, signature.extract_decorators,
+                             List[Dict[int, signature.extract_decorators]]],
+       textwrap.dedent("""\
+        Union[
+            dict[str, dict[bool, m.extract_decorators]],
+            int,
+            bool,
+            m.extract_decorators,
+            list[dict[int, m.extract_decorators]]
+        ]""")),
+      ('callable_ellipsis_sig', Union[Callable[..., int], str],
+       textwrap.dedent("""\
+        Union[
+            Callable[..., int],
+            str
+        ]""")),
+      ('callable_args_sig', Union[Callable[[bool, signature.extract_decorators],
+                                           float], int],
+       textwrap.dedent("""\
+        Union[
+            Callable[[bool, m.extract_decorators], float],
+            int
+        ]""")),
+      ('callable_without_args', Union[None, dict, str, Callable],
+       textwrap.dedent("""\
+        Union[
+            NoneType,
+            dict,
+            str,
+            Callable
+        ]""")),
+  )  # pyformat: disable
+  def test_type_alias_signature(self, alias, expected_sig):
+    api_node = doc_generator_visitor.ApiTreeNode(
+        path=tuple('tfdocs.api_generator.generate_lib.DocGenerator'.split('.')),
+        py_object=alias)
+    info_obj = type_alias_page.TypeAliasPageInfo(
+        api_node=api_node, parser_config=self.parser_config)
+    with self.parser_config.reference_resolver.temp_prefix('../../..'):
+      info_obj.collect_docs()
+      self.assertEqual(info_obj.signature, expected_sig)
+
+  def _setup_class_info(self, cls):
+    self.known_object = object()
+
+    x = types.ModuleType('x')
+    x.__file__ = __file__
+    x.Cls = cls
+
+    generator = generate_lib.DocGenerator(
+        root_title='test',
+        py_modules=[('x', x)],
+        code_url_prefix='https://tensorflow.org')
+
+    parser_config = generator.run_extraction()
+    parser_config.reference_resolver = (
+        parser_config.reference_resolver.with_prefix('/'))
+
+    api_node = parser_config.api_tree['x', 'Cls']
+    info = class_page.ClassPageInfo(
+        api_node=api_node, parser_config=parser_config)
+    info._doc = parser.DocstringInfo('doc', ['doc'], {})
+    info.collect_docs()
+
+    return info
+
+  def test_signature_method_wrong_self_name(self):
+
+    # Calling these classes all `Cls` confuses get_source, you need to
+    # use unique names.
+    class Cls1:
+
+      def method(x):  # pylint: disable=no-self-argument
+        pass
+
+    info = self._setup_class_info(Cls1)
+    self.assertEqual('()', str(info.methods[0].signature))
+
+  def test_signature_method_star_args(self):
+
+    class Cls2:
+
+      def method(*args):  # pylint: disable=no-method-argument
+        pass
+
+    info = self._setup_class_info(Cls2)
+    self.assertEqual('(\n    *args\n)', str(info.methods[0].signature))
+
+  def test_signature_classmethod_wrong_cls_name(self):
+
+    class Cls3:
+
+      @classmethod
+      def method(x):  # pylint: disable=bad-classmethod-argument
+        pass
+
+    info = self._setup_class_info(Cls3)
+    self.assertEqual('()', str(info.methods[0].signature))
+
+  def test_signature_staticmethod(self):
+
+    class Cls4:
+
+      @staticmethod
+      def method(x):
+        pass
+
+    info = self._setup_class_info(Cls4)
+    self.assertEqual('(\n    x\n)', str(info.methods[0].signature))
+
+  def test_signature_new(self):
+
+    class Cls5:
+
+      def __new__(x):  # pylint: disable=bad-classmethod-argument
+        pass
+
+    info = self._setup_class_info(Cls5)
+    self.assertEqual('()', str(info.methods[0].signature))
+
+  def test_signature_dataclass_auto_init(self):
+
+    @dataclasses.dataclass
+    class Cls6:
+      a: Optional[int]
+      b: Optional[str]
+
+    info = self._setup_class_info(Cls6)
+    builder = info.DEFAULT_BUILDER_CLASS(info)
+
+    self.assertEqual('(\n    a: Optional[int], b: Optional[str]\n)',
+                     str(builder.methods.constructor.signature))
+
+  def test_signature_dataclass_custom_init(self):
+
+    @dataclasses.dataclass(init=False)
+    class Cls7:
+      a: Optional[int]
+      b: Optional[str]
+
+      def __init__(self, x: Optional[Union[int, str]]):
+        self.a = int(x)
+        self.b = str(x)
+
+    info = self._setup_class_info(Cls7)
+    builder = info.DEFAULT_BUILDER_CLASS(info)
+    self.assertEqual('(\n    x: Optional[Union[int, str]]\n)',
+                     str(builder.methods.constructor.signature))
+
+  def test_dataclass_default_uses_ast_repr(self):
+
+    @dataclasses.dataclass
+    class MyClass:
+      a: float = 1 / 9
+
+    sig = signature.generate_signature(
+        MyClass, parser_config=self.parser_config)
+
+    expected = '(\n    a: float = (1 / 9)\n)'
+    self.assertEqual(expected, str(sig))
+
+  def test_dataclass_inheritance_sees_parent(self):
+    const = 3.14159
+
+    @dataclasses.dataclass
+    class Parent:
+      a: int = 60 * 60
+      b: float = 1 / 9
+
+    @dataclasses.dataclass
+    class Child(Parent):
+      b: float = 2 / 9
+      c: float = const
+
+    sig = signature.generate_signature(Child, parser_config=self.parser_config)
+    expected = textwrap.dedent("""\
+        (
+            a: int = (60 * 60), b: float = (2 / 9), c: float = const
+        )""")
+    self.assertEqual(expected, str(sig))
+
+  def test_extract_non_annotated(self):
+
+    const = 1234
+
+    class A:
+      a = 60 * 60
+      b = 1 / 9
+
+    class B(A):
+      b = 2 / 9
+      c = const
+
+    ast_extractor = signature._ClassDefaultAndAnnotationExtractor()
+    ast_extractor.extract(B)
+
+    self.assertEqual({
+        'a': '(60 * 60)',
+        'b': '(2 / 9)',
+        'c': 'const'
+    }, ast_extractor.defaults)
+
+
+  def test_vararg_before_kwargonly_consistent_order(self):
+
+    def my_fun(*args, a=1, **kwargs):  # pylint: disable=unused-argument
+      pass
+
+    sig = signature.generate_signature(my_fun, parser_config=self.parser_config)
+    expected = '(\n    *args, a=1, **kwargs\n)'
+    self.assertEqual(expected, str(sig))
+
+  def test_class_vararg_before_kwargonly_consistent_order(self):
+
+    class MyClass:
+
+      def __init__(*args, a=1, **kwargs):  # pylint: disable=no-method-argument
+        pass
+
+    sig = signature.generate_signature(
+        MyClass, parser_config=self.parser_config)
+    expected = '(\n    *args, a=1, **kwargs\n)'
+    self.assertEqual(expected, str(sig))
+
+  def test_strip_address(self):
+
+    class What:
+      pass
+
+    w = What()
+
+    expected = ('<__main__.TestGenerateSignature.test_strip_address.'
+                '.What object>')
+    self.assertEqual(expected, signature.strip_obj_addresses(str(w)))
+
+if __name__ == '__main__':
+  absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/tf_inspect.py b/tools/tensorflow_docs/api_generator/tf_inspect.py
deleted file mode 100644
index d770ec624b1..00000000000
--- a/tools/tensorflow_docs/api_generator/tf_inspect.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""TFDecorator-aware replacements for the inspect module."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from collections import namedtuple
-import functools
-import inspect
-from inspect import *  # pylint: disable=wildcard-import
-import sys
-
-import six
-
-if sys.version_info.major < 3:
-  FullArgSpec = namedtuple('FullArgSpec', [
-      'args', 'varargs', 'varkw', 'defaults', 'kwonlyargs', 'kwonlydefaults',
-      'annotations'
-  ])
-
-  def getfullargspec(target):
-    """A python2 version of `inspect.getfullargspec`.
-
-    Args:
-      target: the target object to inspect.
-
-    Returns:
-      A `FullArgSpec`.
-    """
-    if isinstance(target, functools.partial):
-      return _get_fullargspec_for_partial(target)
-    argspecs = getargspec(target)
-    fullargspecs = FullArgSpec(
-        args=argspecs.args,
-        varargs=argspecs.varargs,
-        varkw=argspecs.keywords,
-        defaults=argspecs.defaults,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-    return fullargspecs
-
-  def getargspec(obj):
-    """A py2 version of `inspect.getargspec`, more compatible with py3.
-
-    Note: `getfullargspec` is recommended as the python 2/3 compatible
-    replacement for this function.
-
-    Args:
-      obj: A function, partial function, or callable object.
-
-    Returns:
-      an `ArgSpec` that describes the callable's signature.
-
-    Raises:
-      ValueError: When callable's signature can not be expressed with
-        ArgSpec.
-      TypeError: For objects of unsupported types.
-    """
-    if isinstance(obj, functools.partial):
-      full = _get_fullargspec_for_partial(obj)
-      if full.kwonlyargs or full.annotations:
-        raise ValueError('Function has keyword-only arguments or annotations, '
-                         'use getfullargspec() API which can support them')
-
-      return inspect.ArgSpec(
-          args=full.args,
-          varargs=full.varargs,
-          keywords=full.varkw,
-          defaults=full.defaults)
-
-    try:
-      return inspect.getargspec(obj)
-    except TypeError:
-      pass
-
-    if isinstance(obj, type):
-      try:
-        return inspect.getargspec(obj.__init__)
-      except TypeError:
-        pass
-
-      try:
-        return inspect.getargspec(obj.__new__)
-      except TypeError:
-        pass
-
-    # The `type(obj)` ensures that if a class is received we don't return
-    # the signature of it's __call__ method.
-    return inspect.getargspec(type(obj).__call__)
-
-  def _get_fullargspec_for_partial(obj):
-    """Implements `getargspec` for `functools.partial` objects.
-
-    Args:
-      obj: The `functools.partial` obeject
-
-    Returns:
-      An `inspect.ArgSpec`
-    Raises:
-      ValueError: When callable's signature can not be expressed with
-        ArgSpec.
-    """
-    n_prune_args = len(obj.args)
-    partial_keywords = obj.keywords or {}
-
-    args, varargs, keywords, defaults = getargspec(obj.func)
-
-    # Partial function may give default value to any argument, therefore length
-    # of default value list must be len(args) to allow each argument to
-    # potentially be given a default value.
-    all_defaults = [None] * len(args)
-
-    if defaults:
-      all_defaults[-len(defaults):] = defaults
-
-    # Prune the args that have a value set by partial.
-    args = args[n_prune_args:]
-    all_defaults = all_defaults[n_prune_args:]
-
-    # Fill in keyword defaults provided by partial.
-    for kw, default in six.iteritems(partial_keywords):
-      if kw in args:
-        idx = args.index(kw)
-        all_defaults[idx] = default
-
-    # Split key-word only args and defaults from others.
-    # Everything from the first partial_keyword is now key-word only.
-    known_kws = [kw for kw in partial_keywords if kw in args]
-    if known_kws:
-      stop = min([args.index(kw) for kw in known_kws])
-      args, kwonlyargs = args[:stop], args[stop:]
-      all_defaults, kwonlydefaults = all_defaults[:stop], all_defaults[stop:]
-      kwonlydefaults = dict(zip(kwonlyargs, kwonlydefaults))
-    else:
-      kwonlyargs = []
-      kwonlydefaults = None
-
-    # Find first argument with default value set.
-    first_default = next((idx for idx, x in enumerate(all_defaults) if x), None)
-    if first_default is None:
-      result_defaults = None
-    else:
-      result_defaults = tuple(all_defaults[first_default:])
-
-    return FullArgSpec(
-        args,
-        varargs,
-        keywords,
-        result_defaults,
-        kwonlyargs=kwonlyargs,
-        kwonlydefaults=kwonlydefaults,
-        annotations={})
diff --git a/tools/tensorflow_docs/api_generator/tf_inspect_test.py b/tools/tensorflow_docs/api_generator/tf_inspect_test.py
deleted file mode 100644
index 4f430355bf6..00000000000
--- a/tools/tensorflow_docs/api_generator/tf_inspect_test.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Unit tests for tf_inspect."""
-
-# pylint: disable=unused-import
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import functools
-
-from absl.testing import absltest
-
-from tensorflow_docs.api_generator import tf_inspect
-
-
-class TfInspectTest(absltest.TestCase):
-
-  def testGetArgSpecOnPartialPositionalArgumentOnly(self):
-    """Tests getargspec on partial function with only positional arguments."""
-
-    def func(m, n):
-      return 2 * m + n
-
-    partial_func = functools.partial(func, 7)
-    argspec = tf_inspect.ArgSpec(
-        args=['n'], varargs=None, keywords=None, defaults=None)
-
-    self.assertEqual(argspec, tf_inspect.getargspec(partial_func))
-
-  def testGetArgSpecOnPartialInvalidArgspec(self):
-    """Tests getargspec on partial function that doesn't have valid argspec."""
-
-    def func(m, n, l, k=4):
-      return 2 * m + l + n * k
-
-    partial_func = functools.partial(func, n=7)
-
-    with self.assertRaisesRegexp(ValueError, 'Function has keyword-only.*'):
-      tf_inspect.getargspec(partial_func)
-
-  def testGetArgSpecOnPartialValidArgspec(self):
-    """Tests getargspec on partial function with valid argspec."""
-
-    def func(m, n, l, k=4):
-      return 2 * m + l + n * k
-
-    partial_func = functools.partial(func, n=7, l=2)
-    argspec = tf_inspect.FullArgSpec(
-        args=['m'],
-        varargs=None,
-        varkw=None,
-        defaults=None,
-        kwonlyargs=['n', 'l', 'k'],
-        kwonlydefaults={
-            'n': 7,
-            'l': 2,
-            'k': 4
-        },
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func))
-    with self.assertRaisesRegexp(ValueError, 'Function has keyword-only.*'):
-      tf_inspect.getargspec(partial_func)
-
-  def testGetArgSpecOnPartialNoArgumentsLeft(self):
-    """Tests getargspec on partial function that prunes all arguments."""
-
-    def func(m, n):
-      return 2 * m + n
-
-    partial_func = functools.partial(func, 7, 10)
-    argspec = tf_inspect.ArgSpec(
-        args=[], varargs=None, keywords=None, defaults=None)
-
-    self.assertEqual(argspec, tf_inspect.getargspec(partial_func))
-
-  def testGetArgSpecOnPartialKeywordArgument(self):
-    """Tests getargspec on partial function that prunes some arguments."""
-
-    def func(m, n):
-      return 2 * m + n
-
-    partial_func = functools.partial(func, n=7)
-    argspec = tf_inspect.FullArgSpec(
-        args=['m'],
-        varargs=None,
-        varkw=None,
-        defaults=None,
-        kwonlyargs=['n'],
-        kwonlydefaults={'n': 7},
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func))
-
-    with self.assertRaisesRegexp(ValueError, 'Function has keyword-only.*'):
-      tf_inspect.getargspec(partial_func)
-
-  def testGetArgSpecOnPartialKeywordArgumentWithDefaultValue(self):
-    """Tests getargspec on partial function that prunes argument by keyword."""
-
-    def func(m=1, n=2):
-      return 2 * m + n
-
-    partial_func = functools.partial(func, n=7)
-    argspec = tf_inspect.FullArgSpec(
-        args=['m'],
-        varargs=None,
-        varkw=None,
-        defaults=(1,),
-        kwonlyargs=['n'],
-        kwonlydefaults={'n': 7},
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func))
-
-    with self.assertRaisesRegexp(ValueError, 'Function has keyword-only.*'):
-      tf_inspect.getargspec(partial_func)
-
-  def testGetArgSpecOnPartialWithVarargs(self):
-    """Tests getargspec on partial function with variable arguments."""
-
-    def func(m, *arg):
-      return m + len(arg)
-
-    partial_func = functools.partial(func, 7, 8)
-    argspec = tf_inspect.ArgSpec(
-        args=[], varargs='arg', keywords=None, defaults=None)
-
-    self.assertEqual(argspec, tf_inspect.getargspec(partial_func))
-
-  def testGetArgSpecOnPartialWithVarkwargs(self):
-    """Tests getargspec on partial function with variable keyword arguments."""
-
-    def func(m, n, **kwarg):
-      return m * n + len(kwarg)
-
-    partial_func = functools.partial(func, 7)
-    argspec = tf_inspect.ArgSpec(
-        args=['n'], varargs=None, keywords='kwarg', defaults=None)
-
-    self.assertEqual(argspec, tf_inspect.getargspec(partial_func))
-
-  def testGetArgSpecOnCallableObject(self):
-
-    class Callable(object):
-
-      def __call__(self, a, b=1, c='hello'):
-        pass
-
-    argspec = tf_inspect.ArgSpec(
-        args=['self', 'a', 'b', 'c'],
-        varargs=None,
-        keywords=None,
-        defaults=(1, 'hello'))
-
-    test_obj = Callable()
-    self.assertEqual(argspec, tf_inspect.getargspec(test_obj))
-
-  def testGetArgSpecOnInitClass(self):
-
-    class InitClass(object):
-
-      def __init__(self, a, b=1, c='hello'):
-        pass
-
-    argspec = tf_inspect.ArgSpec(
-        args=['self', 'a', 'b', 'c'],
-        varargs=None,
-        keywords=None,
-        defaults=(1, 'hello'))
-
-    self.assertEqual(argspec, tf_inspect.getargspec(InitClass))
-
-  def testGetArgSpecOnNewClass(self):
-
-    class NewClass(object):
-
-      def __new__(cls, a, b=1, c='hello'):
-        pass
-
-    argspec = tf_inspect.ArgSpec(
-        args=['cls', 'a', 'b', 'c'],
-        varargs=None,
-        keywords=None,
-        defaults=(1, 'hello'))
-
-    self.assertEqual(argspec, tf_inspect.getargspec(NewClass))
-
-  def testGetFullArgsSpecForPartial(self):
-
-    def func(a, b):
-      del a, b
-
-    partial_function = functools.partial(func, 1)
-    argspec = tf_inspect.FullArgSpec(
-        args=['b'],
-        varargs=None,
-        varkw=None,
-        defaults=None,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(partial_function))
-
-  def testGetFullArgSpecOnPartialNoArgumentsLeft(self):
-    """Tests getfullargspec on partial function that prunes all arguments."""
-
-    def func(m, n):
-      return 2 * m + n
-
-    partial_func = functools.partial(func, 7, 10)
-    argspec = tf_inspect.FullArgSpec(
-        args=[],
-        varargs=None,
-        varkw=None,
-        defaults=None,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func))
-
-  def testGetFullArgSpecOnPartialWithVarargs(self):
-    """Tests getfullargspec on partial function with variable arguments."""
-
-    def func(m, *arg):
-      return m + len(arg)
-
-    partial_func = functools.partial(func, 7, 8)
-    argspec = tf_inspect.FullArgSpec(
-        args=[],
-        varargs='arg',
-        varkw=None,
-        defaults=None,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func))
-
-  def testGetFullArgSpecOnPartialWithVarkwargs(self):
-    """Tests getfullargspec.
-
-    Tests on partial function with variable keyword arguments.
-    """
-
-    def func(m, n, **kwarg):
-      return m * n + len(kwarg)
-
-    partial_func = functools.partial(func, 7)
-    argspec = tf_inspect.FullArgSpec(
-        args=['n'],
-        varargs=None,
-        varkw='kwarg',
-        defaults=None,
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(partial_func))
-
-  def testGetFullArgSpecOnCallableObject(self):
-
-    class Callable(object):
-
-      def __call__(self, a, b=1, c='hello'):
-        pass
-
-    argspec = tf_inspect.FullArgSpec(
-        args=['self', 'a', 'b', 'c'],
-        varargs=None,
-        varkw=None,
-        defaults=(1, 'hello'),
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-
-    test_obj = Callable()
-    self.assertEqual(argspec, tf_inspect.getfullargspec(test_obj))
-
-  def testGetFullArgSpecOnInitClass(self):
-
-    class InitClass(object):
-
-      def __init__(self, a, b=1, c='hello'):
-        pass
-
-    argspec = tf_inspect.FullArgSpec(
-        args=['self', 'a', 'b', 'c'],
-        varargs=None,
-        varkw=None,
-        defaults=(1, 'hello'),
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(InitClass))
-
-  def testGetFullArgSpecOnNewClass(self):
-
-    class NewClass(object):
-
-      def __new__(cls, a, b=1, c='hello'):
-        pass
-
-    argspec = tf_inspect.FullArgSpec(
-        args=['cls', 'a', 'b', 'c'],
-        varargs=None,
-        varkw=None,
-        defaults=(1, 'hello'),
-        kwonlyargs=[],
-        kwonlydefaults=None,
-        annotations={})
-
-    self.assertEqual(argspec, tf_inspect.getfullargspec(NewClass))
-
-
-if __name__ == '__main__':
-  absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/toc.py b/tools/tensorflow_docs/api_generator/toc.py
new file mode 100644
index 00000000000..feaa15b8bda
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/toc.py
@@ -0,0 +1,350 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Classes for generating the TOC."""
+
+import contextlib
+import dataclasses
+import enum
+import os
+import pathlib
+
+from typing import Any, IO, Iterator, List, Optional, Tuple, Union
+
+from tensorflow_docs.api_generator import doc_controls
+from tensorflow_docs.api_generator import doc_generator_visitor
+from tensorflow_docs.api_generator import obj_type as obj_type_lib
+from tensorflow_docs.api_generator import signature
+
+import yaml
+
+
+class _TocDumper(yaml.SafeDumper):
+
+  def ignore_aliases(self, data):
+    """Don't output references for duplicated objects (usually strings)."""
+    return True
+
+
+def _dict_representer(dumper: yaml.SafeDumper, data: Any):
+  """Represent the object as a dict created of (key, value) pairs."""
+  return dumper.represent_dict(iter(data))
+
+
+def _use_yaml_dict_representer(cls):
+  """Register the class's as using a `_dict_representer`."""
+  yaml.add_representer(cls, representer=_dict_representer, Dumper=_TocDumper)
+  return cls
+
+
+def _str_enum_representer(dumper: yaml.SafeDumper, data: Any):
+  """Represent a str-Enum as a string."""
+  return dumper.represent_str(data.value)
+
+
+def _use_yaml_str_enum_representer(cls):
+  """Register the class as using `_str_enum_representer`."""
+  yaml.add_representer(
+      cls, representer=_str_enum_representer, Dumper=_TocDumper)
+  return cls
+
+
+@_use_yaml_str_enum_representer
+class Status(enum.Enum):
+  """Represents a page status."""
+  ALPHA = 'alpha'
+  BETA = 'beta'
+  DEPRECATED = 'deprecated'
+  EXPERIMENTAL = 'experimental'
+  EXTERNAL = 'external'
+  LIMITED = 'limited'
+  NEW = 'new'
+  NIGHTLY = 'nightly'
+  PREVIEW = 'preview'
+  UNSUPPORTED = 'unsupported'
+
+
+@_use_yaml_str_enum_representer
+class HeadingStyle(enum.Enum):
+  """Represents a Heading Style."""
+  ACCORDION = 'accordion'
+  DIVIDER = 'divider'
+
+
+class Entry:
+  """Base class for toc entries."""
+
+  def replace(self, **kwargs):
+    new_kwargs = dict(self)
+    new_kwargs.update(kwargs)
+    return type(self)(**new_kwargs)
+
+  def __iter__(self) -> Iterator[Tuple[str, Any]]:
+    """Support `dict(entry)` for yaml output."""
+    for key, value in self.__dict__.items():
+      if value is not None:
+        yield (key, value)
+
+
+@_use_yaml_dict_representer
+@dataclasses.dataclass
+class Heading(Entry):
+  """A toc heading."""
+  heading: str
+  style: Optional[HeadingStyle] = None
+
+
+@_use_yaml_dict_representer
+@dataclasses.dataclass
+class Section(Entry):
+  """A toc section."""
+  title: str
+  section: List[Entry]
+  status: Optional[Status] = None
+
+  def __iter__(self) -> Iterator[Tuple[str, Any]]:
+    """Support `dict(entry)` for yaml output."""
+    yield 'title', self.title
+    if self.status is not None:
+      yield 'status', self.status
+    yield 'section', self.section
+
+
+@_use_yaml_dict_representer
+@dataclasses.dataclass
+class Link(Entry):
+  """Represents toc page-link."""
+  title: str
+  path: str
+  status: Optional[Status] = None
+
+  def __iter__(self) -> Iterator[Tuple[str, Any]]:
+    """Support `dict(entry)` for yaml output."""
+    yield 'title', self.title
+    if self.status is not None:
+      yield 'status', self.status
+    yield 'path', self.path
+
+
+@_use_yaml_dict_representer
+class Break(Entry):
+  """Represents a toc whitesoace break."""
+
+  def __init__(self):
+    self.__dict__['break'] = True
+
+
+@_use_yaml_dict_representer
+@dataclasses.dataclass
+class Toc(Entry):
+  """Represents the top-level `toc` element in included files."""
+  toc: List[Entry]
+
+  @contextlib.contextmanager
+  def _maybe_open(self, file: Union[os.PathLike, IO[str]]) -> Iterator[IO[str]]:
+    if isinstance(file, os.PathLike):
+      with open(file, 'w') as stream:
+        yield stream
+    else:
+      stream = file
+      yield stream
+
+  def write(self, file: Union[os.PathLike, IO[str]]) -> None:
+    with self._maybe_open(file) as stream:
+      yaml.dump(
+          self, stream=stream, default_flow_style=False, Dumper=_TocDumper)
+
+
+class TocBuilder:
+  """A class to build a Toc from an ApiTree."""
+
+  def __init__(self, site_path):
+    self.site_path = pathlib.Path(site_path)
+
+  def build(self, api_tree: doc_generator_visitor.ApiTree) -> Toc:
+    """Create a `Toc` from an `ApiTree`."""
+    entries = []
+    for child in api_tree.root.children.values():
+      entries.extend(self._entries_from_api_node(child))
+    return Toc(toc=entries)
+
+  def _entries_from_api_node(
+      self, api_node: doc_generator_visitor.ApiTreeNode) -> List[Entry]:
+
+    """Converts an ApiTreeNode to a list of toc entries."""
+    obj_type = api_node.obj_type
+
+    if obj_type is obj_type_lib.ObjType.MODULE:
+      return [self._make_section(api_node)]
+    if obj_type is obj_type_lib.ObjType.CLASS:
+      return self._flat_class_entries(api_node)
+    if obj_type in [
+        obj_type_lib.ObjType.CALLABLE, obj_type_lib.ObjType.TYPE_ALIAS
+    ]:
+      return [self._make_link(api_node)]
+    else:
+      return []
+
+  def _get_docpath(self, api_path) -> pathlib.Path:
+    api_path = (p.replace('.', '/') for p in api_path)
+    return pathlib.Path(self.site_path, *api_path)
+
+  def _make_link(self,
+                 api_node: doc_generator_visitor.ApiTreeNode,
+                 title: Optional[str] = None) -> Link:
+
+    docpath = self._get_docpath(api_path=api_node.path)
+    title = title or api_node.short_name
+    return Link(
+        title=title, path=str(docpath), status=self._make_status(api_node))
+
+  def _make_section(self,
+                    api_node: doc_generator_visitor.ApiTreeNode,
+                    title: Optional[str] = None) -> Section:
+    """Create a `toc.Section` from a module's ApiTreeNode."""
+    overview = self._make_overview(api_node)
+    entries = []
+    for child in api_node.children.values():
+      entries.extend(self._entries_from_api_node(child))
+    entries = sorted(entries, key=self._section_order_key)
+    entries = [overview] + entries
+
+    status = self._make_status(api_node)
+    return Section(
+        title=title or api_node.short_name, section=entries, status=status)
+
+  def _make_overview(self, api_node: doc_generator_visitor.ApiTreeNode):
+    docpath = self._get_docpath(api_path=api_node.path)
+    return Link(title='Overview', path=str(docpath))
+
+  def _section_order_key(self, entry: Entry) -> Tuple[bool, str]:
+    title = getattr(entry, 'title', None)
+    is_section = isinstance(entry, Section)
+
+    return (is_section, title)
+
+  def _flat_class_entries(self,
+                          api_node: doc_generator_visitor.ApiTreeNode,
+                          title: Optional[str] = None) -> List[Entry]:
+    """Returns entries for both `Class` and `Class.Nested`."""
+    title = title or api_node.short_name
+    entries = [self._make_link(api_node, title=title)]
+    for name, child_node in api_node.children.items():
+      if child_node.obj_type in [
+          obj_type_lib.ObjType.CLASS, obj_type_lib.ObjType.MODULE
+      ]:
+        subtitle = f'{title}.{name}'
+        entries.extend(self._flat_class_entries(child_node, title=subtitle))
+
+    return entries
+
+  def _make_status(self, api_node: doc_generator_visitor.ApiTreeNode):
+    """Returns the toc.Status of an ApiTreeNode."""
+    if self._is_deprecated(api_node):
+      return Status.DEPRECATED
+    if self._is_experimental(api_node):
+      return Status.EXPERIMENTAL
+    return None
+
+  def _is_experimental(self, api_node: doc_generator_visitor.ApiTreeNode):
+    return 'experimental' in api_node.short_name.lower()
+
+  def _is_deprecated(self, api_node: doc_generator_visitor.ApiTreeNode):
+    """Checks if an object is deprecated or not.
+
+    Each deprecated function has a `_tf_decorator.decorator_name` attribute.
+    Check the docstring of that function to confirm if the function was
+    indeed deprecated. If a different deprecation setting was used on the
+    function, then "THIS FUNCTION IS DEPRECATED" substring won't be inserted
+    into the docstring of that function by the decorator.
+
+    Args:
+      api_node: The node to evaluate.
+
+    Returns:
+      True if deprecated else False.
+    """
+    if doc_controls.is_deprecated(api_node.py_object):
+      return True
+
+    decorator_list = signature.extract_decorators(api_node.py_object)
+    if any('deprecat' in dec for dec in decorator_list):
+      docstring = getattr(api_node.py_object, '__doc__') or ''
+      return 'THIS FUNCTION IS DEPRECATED' in docstring
+
+    return False
+
+
+class FlatModulesTocBuilder(TocBuilder):
+  """Builds a toc where the top level submodules are peers (not children).
+
+  The base TocBuilder does this:
+
+  ```
+  module:
+    thing1
+    sub1:
+      thing2
+    sub2:
+      thing3
+  ```
+
+  This one outputs:
+
+  ```
+  module:
+    thing1
+  module.sub1:
+    thing2
+  module.sub2:
+    thing3
+  ```
+  """
+
+  def build(self, api_tree: doc_generator_visitor.ApiTree) -> Toc:
+    entries = []
+    for module_node in api_tree.root.children.values():
+      if '.' in module_node.short_name:
+        entries.extend(self._entries_from_api_node(module_node))
+      else:
+        assert module_node.obj_type is obj_type_lib.ObjType.MODULE
+        entries.extend(self._flat_module_entries(module_node))
+
+    return Toc(toc=entries)
+
+  def _flat_module_entries(self,
+                           api_node: doc_generator_visitor.ApiTreeNode,
+                           title: Optional[str] = None) -> List[Section]:
+    """For top-level modules, place the submodules as peers."""
+    title = title or api_node.short_name
+
+    overview = self._make_link(api_node, title='Overview')
+    entries = []
+    submodule_sections = []
+    for name, child_node in api_node.children.items():
+      if child_node.obj_type is obj_type_lib.ObjType.MODULE:
+        subtitle = f'{title}.{name}'
+        submodule_sections.append(
+            self._make_section(child_node, title=subtitle))
+      else:
+        entries.extend(self._entries_from_api_node(child_node))
+
+    entries = sorted(entries, key=self._section_order_key)
+    entries.insert(0, overview)
+
+    submodule_sections = sorted(submodule_sections, key=self._section_order_key)
+
+    status = self._make_status(api_node)
+    module_section = Section(title=title, section=entries, status=status)
+    return [module_section] + submodule_sections
diff --git a/tools/tensorflow_docs/api_generator/toc_processing.py b/tools/tensorflow_docs/api_generator/toc_processing.py
new file mode 100644
index 00000000000..fbe0c3cf842
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/toc_processing.py
@@ -0,0 +1,116 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tools for processing generated Java documentation."""
+import itertools
+from typing import Any, Iterable, Mapping, MutableMapping, MutableSequence
+
+# TODO(b/193033225): If possible, this should be a TypedDict. If not, using the
+#   real protos might make things a little cleaner.
+TocEntry = MutableMapping[str, Any]
+Section = MutableSequence[TocEntry]
+Toc = Mapping[str, Section]
+
+
+def add_package_headings(toc: Toc, root_pkgs: Iterable[str],
+                         labels: Mapping[str, str]) -> Toc:
+  """Breaks up a flat structure with headings for each 1st-level package."""
+  new_toc = []
+  current_section = None
+  for entry in sort_toc(toc, labels.keys())['toc']:
+    new_entry = dict(entry)
+    for root_pkg in root_pkgs:
+      if new_entry.get('title', '').startswith(root_pkg):
+        # Strip the common root_pkg from the title.
+        new_title = new_entry['title'][len(root_pkg):].lstrip('.')
+        # a, a.b, a.b.c from the remainder of the package name
+        all_sections = itertools.accumulate(
+            new_title.split('.'), lambda a, b: f'{a}.{b}')
+        # Filter out any packages that aren't defined as labelled sections.
+        candidate_sections = filter(lambda s: f'{root_pkg}.{s}' in labels,
+                                    all_sections)
+        # If there are more than one, pick the most specific (longest). If there
+        # are none, use the bare trailing package.
+        section = max(candidate_sections, key=len, default=new_title)
+
+        if section != current_section:
+          # We've hit a new section, add a label if one was supplied.
+          section_pkg = f'{root_pkg}.{section}' if section else root_pkg
+          new_toc.append({'heading': labels.get(section_pkg, section)})
+          current_section = section
+
+        new_entry['title'] = new_title or root_pkg
+    new_toc.append(new_entry)
+  return {'toc': new_toc}
+
+
+def nest_toc(toc: Toc) -> Toc:
+  """Nests a flat TOC into a tree structure based on common packages."""
+  new_toc = []
+
+  # We only look at the first level for flat package names.
+  entries_by_title = {e['title']: e for e in toc['toc']}
+  for title, entry in entries_by_title.items():
+    target_entry = _nest_toc_entry(title, new_toc)
+
+    # Populate the target entry with the original entry, sans title.
+    # (pytype suppressed due to inferring .keys() as a List)
+    fields = entry.keys() - {'title'}  # pytype: disable=unsupported-operands
+    target_entry.update({f: entry[f] for f in fields})
+
+    # Clean up empty sections
+    if not target_entry.get('section'):
+      target_entry.pop('section', None)
+
+  return {'toc': new_toc}
+
+
+def _nest_toc_entry(title: str, section: Section) -> TocEntry:
+  """Nest the title (split by .) into the TOC. Creating hierarchy as needed."""
+  pkg, *maybe_rest = title.split('.', 1)
+
+  for entry in section:
+    if entry.get('title') == pkg:
+      target_entry = entry
+      if 'section' not in target_entry:
+        target_entry['section'] = []
+      break
+  else:
+    target_entry = {'title': pkg, 'section': []}
+    section.append(target_entry)
+
+  if not maybe_rest:
+    return target_entry
+  else:
+    rest = maybe_rest[0]
+    return _nest_toc_entry(rest, target_entry['section'])
+
+
+def sort_toc(toc: Toc, labels: Iterable[str]) -> Toc:
+  """Pre-sort the TOC entries by `labels`."""
+  new_toc = []
+  remaining_entries = list(toc.get('toc', []))
+  for label in labels:
+    more_specific_labels = [l for l in labels if len(l) > len(label)]
+    for entry in remaining_entries[:]:  # copy so we can remove() later
+      title = entry.get('title', '')
+      better_match_exists = any(
+          [title.startswith(l) for l in more_specific_labels])
+      if title.startswith(label) and not better_match_exists:
+        new_toc.append(entry)
+        # Remove the matched entry so it doesn't duplicate, and so we can track
+        # any un-matched entries.
+        remaining_entries.remove(entry)
+
+  return {'toc': new_toc + remaining_entries}
diff --git a/tools/tensorflow_docs/api_generator/toc_processing_test.py b/tools/tensorflow_docs/api_generator/toc_processing_test.py
new file mode 100644
index 00000000000..8b49c0b3728
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/toc_processing_test.py
@@ -0,0 +1,267 @@
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Java doc processing code."""
+
+from absl.testing import absltest
+
+from tensorflow_docs.api_generator import toc_processing
+
+
+class ProcessingTest(absltest.TestCase):
+
+  def test_toc_package_sections(self):
+    toc_in = {
+        'toc': [{
+            'title': 'org.tf',
+            'path': '/tf/docs.html',
+            'section': [{
+                'title': 'SymbolOne',
+                'path': '/tf/symbol.html',
+            }],
+        }, {
+            'title':
+                'org.tf.foo.baa',
+            'path':
+                '/tf/foo/baa/docs.html',
+            'section': [{
+                'title': 'FooBaaOne',
+                'path': '/tf/foo/baa/FooOne.html',
+            }],
+        }, {
+            'title':
+                'org.tf.foo.bar',
+            'path':
+                '/tf/foo/bar/docs.html',
+            'section': [{
+                'title': 'FooBarBuilder',
+                'path': '/tf/foo/bar/FooBarBuilder.html',
+            }],
+        }]
+    }
+    labels = {
+        'org.tf': 'RootLabel',
+        'org.tf.foo': 'FooPackage',
+    }
+    actual_toc = toc_processing.add_package_headings(toc_in, ['org.tf'], labels)
+    expected_toc = {
+        'toc': [
+            {
+                # New heading inserted, label from labels dict.
+                'heading': 'RootLabel',
+            },
+            {
+                'title': 'org.tf',
+                'path': '/tf/docs.html',
+                'section': [{
+                    'title': 'SymbolOne',
+                    'path': '/tf/symbol.html',
+                }],
+            },
+            {
+                # New heading inserted, label from labels dict.
+                'heading': 'FooPackage',
+            },
+            {
+                # Title has package prefix trimmed.
+                'title':
+                    'foo.baa',
+                'path':
+                    '/tf/foo/baa/docs.html',
+                'section': [{
+                    'title': 'FooBaaOne',
+                    'path': '/tf/foo/baa/FooOne.html',
+                }],
+            },
+            {
+                # Title has package prefix trimmed. Under `foo` heading.
+                'title':
+                    'foo.bar',
+                'path':
+                    '/tf/foo/bar/docs.html',
+                'section': [{
+                    'title': 'FooBarBuilder',
+                    'path': '/tf/foo/bar/FooBarBuilder.html',
+                }],
+            }
+        ]
+    }
+    self.assertDictEqual(actual_toc, expected_toc)
+
+  def test_multiple_roots(self):
+    toc_in = {
+        'toc': [{
+            'title': 'org.tf.one',
+        }, {
+            'title': 'com.google.two',
+        }]
+    }
+    roots = ['org.tf', 'com.google']
+    actual_toc = toc_processing.add_package_headings(toc_in, roots, {})
+    expected_toc = {
+        'toc': [
+            {
+                'heading': 'one'
+            },
+            {
+                'title': 'one'
+            },
+            {
+                'heading': 'two'
+            },
+            {
+                'title': 'two'
+            },
+        ]
+    }
+    self.assertEqual(actual_toc['toc'], expected_toc['toc'])
+
+  def test_overlapping_packages(self):
+    toc_in = {
+        'toc': [{
+            'title': 'org.tf.one',
+            'path': '/tf/one/docs.html',
+            'section': [{
+                'title': 'SymbolOne',
+                'path': '/tf/one/symbol.html',
+            }],
+        }, {
+            'title':
+                'org.tf.one.two',
+            'path':
+                '/tf/one/two/docs.html',
+            'section': [{
+                'title': 'SymbolOneTwo',
+                'path': '/tf/one/two/SymbolOneTwo.html',
+            }],
+        }]
+    }
+    labels = {
+        'org.tf.one': 'Section One',
+        'org.tf.one.two': 'Section Two',
+    }
+    actual_toc = toc_processing.add_package_headings(toc_in, ['org.tf'], labels)
+    expected_toc = {
+        'toc': [{
+            'heading': 'Section One',
+        }, {
+            'title': 'one',
+            'path': '/tf/one/docs.html',
+            'section': [{
+                'title': 'SymbolOne',
+                'path': '/tf/one/symbol.html',
+            }],
+        }, {
+            'heading': 'Section Two',
+        }, {
+            'title':
+                'one.two',
+            'path':
+                '/tf/one/two/docs.html',
+            'section': [{
+                'title': 'SymbolOneTwo',
+                'path': '/tf/one/two/SymbolOneTwo.html',
+            }],
+        }]
+    }
+    self.assertEqual(actual_toc['toc'], expected_toc['toc'])
+
+  def test_nesting_toc(self):
+    toc_in = {
+        'toc': [{
+            'title': 'tf_lite.support',
+            'path': '/tflite/support.html',
+        }, {
+            'title': 'tf_lite.support.cls',
+            'path': '/tflite/support/cls.html',
+        }, {
+            'title': 'tf_lite.task.things',
+            'path': '/tflite/task/things.html',
+        }, {
+            'title': 'tf_other.widgets',
+            'path': '/tfother/widgets.html',
+        }]
+    }
+    actual_toc = toc_processing.nest_toc(toc_in)
+    expected_toc = {
+        'toc': [{
+            'title':
+                'tf_lite',
+            'section': [{
+                'title':
+                    'support',
+                'path':
+                    '/tflite/support.html',
+                'section': [{
+                    'title': 'cls',
+                    'path': '/tflite/support/cls.html'
+                }]
+            }, {
+                'title':
+                    'task',
+                'section': [{
+                    'title': 'things',
+                    'path': '/tflite/task/things.html'
+                }]
+            }]
+        }, {
+            'title': 'tf_other',
+            'section': [{
+                'title': 'widgets',
+                'path': '/tfother/widgets.html'
+            }]
+        }]
+    }
+    self.assertEqual(actual_toc['toc'], expected_toc['toc'])
+
+  def test_ordering(self):
+    toc_in = {
+        'toc': [
+            {
+                'title': 'org.tf.a.third'
+            },
+            {
+                'title': 'org.tf.b.first'
+            },
+            {
+                'title': 'com.google.c.second'
+            },
+            {
+                'title': 'ai.google.d.unspecified'
+            },
+        ]
+    }
+    labels = ['org.tf.b', 'com.google.c', 'org.tf']
+    actual_toc = toc_processing.sort_toc(toc_in, labels)
+    expected_toc = {
+        'toc': [
+            {
+                'title': 'org.tf.b.first'
+            },
+            {
+                'title': 'com.google.c.second'
+            },
+            {
+                'title': 'org.tf.a.third'
+            },
+            {
+                'title': 'ai.google.d.unspecified'
+            },
+        ]
+    }
+    self.assertDictEqual(actual_toc, expected_toc)
+
+
+if __name__ == '__main__':
+  absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/toc_test.py b/tools/tensorflow_docs/api_generator/toc_test.py
new file mode 100644
index 00000000000..beb7a452ec0
--- /dev/null
+++ b/tools/tensorflow_docs/api_generator/toc_test.py
@@ -0,0 +1,175 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+import io
+import textwrap
+import types
+
+from absl.testing import absltest
+
+from tensorflow_docs.api_generator import doc_generator_visitor
+from tensorflow_docs.api_generator import toc as toc_lib
+
+
+class TestToc(absltest.TestCase):
+
+  def test_toc_write(self):
+    link = toc_lib.Link(title='A link', path='/path/to/link')
+    link2 = toc_lib.Link(
+        title='Another link',
+        path='/path/to/link2',
+        status=toc_lib.Status.EXTERNAL)
+
+    subsection = toc_lib.Section(
+        title='A subsection',
+        section=[link],
+        status=toc_lib.Status.EXPERIMENTAL)
+
+    toc = toc_lib.Toc([
+        # pyformat: disable
+        toc_lib.Heading('Hello'),
+        link,
+        link2,
+        toc_lib.Break(),
+        subsection
+    ])
+
+    stream = io.StringIO()
+    toc.write(stream)
+
+    expected = textwrap.dedent("""\
+        toc:
+        - heading: Hello
+        - title: A link
+          path: /path/to/link
+        - title: Another link
+          status: external
+          path: /path/to/link2
+        - break: true
+        - title: A subsection
+          status: experimental
+          section:
+          - title: A link
+            path: /path/to/link
+        """)
+
+    self.assertEqual(expected, stream.getvalue())
+
+  def test_replace(self):
+    link = toc_lib.Link(title='A link', path='/path/to/link')
+    new_link = link.replace(status=toc_lib.Status.NEW, title='New title.')
+
+    expected = toc_lib.Link(
+        title='New title.', path='/path/to/link', status=toc_lib.Status.NEW)
+    self.assertEqual(expected, new_link)
+
+  def _make_tree(self) -> doc_generator_visitor.ApiTree:
+    api_tree = doc_generator_visitor.ApiTree()
+    api_tree.insert(
+        path=('module',), py_object=types.ModuleType('module'), aliases=[])
+    api_tree.insert(path=('module', 'func1'), py_object=lambda x: x, aliases=[])
+    api_tree.insert(
+        path=('module', 'Class'),
+        py_object=types.new_class('Class'),
+        aliases=[])
+    api_tree.insert(
+        path=('module', 'Class', 'method'), py_object=lambda x: x, aliases=[])
+    api_tree.insert(
+        path=('module', 'Class', 'NestedClass'),
+        py_object=types.new_class('NestedClass'),
+        aliases=[])
+    api_tree.insert(
+        path=('module', 'Class', 'NestedClass', 'method2'),
+        py_object=lambda x: x,
+        aliases=[])
+    api_tree.insert(
+        path=('module', 'Class', 'constant'),
+        py_object='Just a string.',
+        aliases=[])
+    api_tree.insert(
+        path=('module', 'submodule'),
+        py_object=types.ModuleType('submodule'),
+        aliases=[])
+    api_tree.insert(
+        path=('module', 'submodule', 'func2'),
+        py_object=lambda x: x,
+        aliases=[])
+    api_tree.insert(
+        path=('module', 'submodule', 'constant'),
+        py_object='Another string.',
+        aliases=[])
+    return api_tree
+
+  def test_toc_builder(self):
+    api_tree = self._make_tree()
+    builder = toc_lib.TocBuilder('/path/in/site')
+    toc = builder.build(api_tree)
+    stream = io.StringIO()
+    toc.write(stream)
+
+    expected = textwrap.dedent("""\
+        toc:
+        - title: module
+          section:
+          - title: Overview
+            path: /path/in/site/module
+          - title: Class
+            path: /path/in/site/module/Class
+          - title: Class.NestedClass
+            path: /path/in/site/module/Class/NestedClass
+          - title: func1
+            path: /path/in/site/module/func1
+          - title: submodule
+            section:
+            - title: Overview
+              path: /path/in/site/module/submodule
+            - title: func2
+              path: /path/in/site/module/submodule/func2
+        """)
+
+    self.assertEqual(expected, stream.getvalue())
+
+  def test_flat_modules_builder(self):
+    api_tree = self._make_tree()
+    builder = toc_lib.FlatModulesTocBuilder('/path/in/site')
+    toc = builder.build(api_tree)
+    stream = io.StringIO()
+    toc.write(stream)
+
+    expected = textwrap.dedent("""\
+        toc:
+        - title: module
+          section:
+          - title: Overview
+            path: /path/in/site/module
+          - title: Class
+            path: /path/in/site/module/Class
+          - title: Class.NestedClass
+            path: /path/in/site/module/Class/NestedClass
+          - title: func1
+            path: /path/in/site/module/func1
+        - title: module.submodule
+          section:
+          - title: Overview
+            path: /path/in/site/module/submodule
+          - title: func2
+            path: /path/in/site/module/submodule/func2
+        """)
+
+    self.assertEqual(expected, stream.getvalue())
+
+
+if __name__ == '__main__':
+  absltest.main()
diff --git a/tools/tensorflow_docs/api_generator/traverse.py b/tools/tensorflow_docs/api_generator/traverse.py
index 5c59656ac18..3d76422735d 100644
--- a/tools/tensorflow_docs/api_generator/traverse.py
+++ b/tools/tensorflow_docs/api_generator/traverse.py
@@ -13,169 +13,85 @@
 # limitations under the License.
 # ==============================================================================
 """Traversing Python modules and classes."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import inspect
-import sys
-
-from google.protobuf.message import Message as ProtoMessage
+import logging
 
-__all__ = ['traverse']
+from typing import Any, Dict, List, Sequence, Tuple
 
+from tensorflow_docs.api_generator import doc_generator_visitor
+from tensorflow_docs.api_generator import public_api
 
-def _filter_module_all(path, root, children):
-  """Filters module children based on the "__all__" arrtibute.
 
-  Args:
-    path: API to this symbol
-    root: The object
-    children: A list of (name, object) pairs.
-
-  Returns:
-    `children` filtered to respect __all__
-  """
-  del path
-  if not (inspect.ismodule(root) and hasattr(root, '__all__')):
-    return children
-  module_all = set(root.__all__)
-  children = [(name, value) for (name, value) in children if name in module_all]
-
-  return children
+# To see the logs pass: --logger_levels=tensorflow_docs:DEBUG --alsologtostderr
+_LOGGER = logging.getLogger(__name__)
 
+__all__ = ['traverse']
 
-def _add_proto_fields(path, root, children):
-  """Add properties to Proto classes, so they can be documented.
 
-  Warning: This inserts the Properties into the class so the rest of the system
-  is unaffected. This patching is acceptable because there is never a reason to
-  run other tensorflow code in the same process as the doc generator.
+class _Traverser:
+  """Crawls the public API."""
 
-  Args:
-    path: API to this symbol
-    root: The object
-    children: A list of (name, object) pairs.
+  def __init__(self, filters: Sequence[public_api.ApiFilter],
+               accumulator: doc_generator_visitor.DocGeneratorVisitor):
+    self.filters = list(filters)
+    self.accumulator = accumulator
+    self.children_cache: Dict[int, List[Tuple[str, Any]]] = {}
 
-  Returns:
-    `children` with proto fields added as properties.
-  """
-  del path
-  if not inspect.isclass(root) or not issubclass(root, ProtoMessage):
-    return children
+  def traverse(self, root, stack, path):
+    """Execute the traversal."""
+    new_stack = stack + [root]
 
-  fields = root.DESCRIPTOR.fields
-  if not fields:
-    return children
+    # Only traverse modules and classes
+    if not inspect.isclass(root) and not inspect.ismodule(root):
+      return
 
-  field = fields[0]
-  # Make the dictionaries mapping from int types and labels to type and
-  # label names.
-  types = {
-      getattr(field, name): name
-      for name in dir(field)
-      if name.startswith('TYPE')
-  }
-
-  labels = {
-      getattr(field, name): name
-      for name in dir(field)
-      if name.startswith('LABEL')
-  }
-
-  field_properties = {}
-
-  for field in fields:
-    name = field.name
-    doc_parts = []
-
-    label = labels[field.label].lower().replace('label_', '')
-    if label != 'optional':
-      doc_parts.append(label)
-
-    type_name = types[field.type]
-    if type_name == 'TYPE_MESSAGE':
-      type_name = field.message_type.name
-    elif type_name == 'TYPE_ENUM':
-      type_name = field.enum_type.name
+    _LOGGER.debug('path: %s', path)
+    children = self.children_cache.get(id(root), None)
+    if children is None:
+      children = self.get_children(root, new_stack, path)
+      self.children_cache[id(root)] = children
     else:
-      type_name = type_name.lower().replace('type_', '')
-
-    doc_parts.append(type_name)
-    doc_parts.append(name)
-    doc = '`{}`'.format(' '.join(doc_parts))
-    prop = property(fget=lambda x: x, doc=doc)
-    field_properties[name] = prop
-
-  for name, prop in field_properties.items():
-    setattr(root, name, prop)
-
-  children = dict(children)
-  children.update(field_properties)
-  children = sorted(children.items(), key=lambda item: item[0])
-
-  return children
+      _LOGGER.debug('    children (cached): %s', [n for n, c in children])
+
+    self.accumulator(path, root, children)
+
+    for name, child in children:
+      child_path = path + (name,)
+      self.traverse(child, new_stack, child_path)
+
+  def get_children(self, root, new_stack, path) -> public_api.Children:
+    """Return the children for an object."""
+    try:
+      children = inspect.getmembers(root)
+    except ImportError:
+      # On some Python installations, some modules do not support enumerating
+      # members (six in particular), leading to import errors.
+      children = []
+
+    # Break cycles.
+    filtered_children = []
+    for name, child in children:
+      if any(child is item for item in new_stack):  # `in`, but using `is`
+        continue
+      filtered_children.append((name, child))
+    children = filtered_children
+
+    _LOGGER.debug('    children: %s', [n for n, c in children])
+    # Apply all callbacks, allowing each to filter the children
+    for fil in self.filters:
+      old_names = [n for n, c in children]
+      children = fil(path, root, children)
+      children = list(children)
+      new_names = [n for n, c in children]
+
+      if old_names != new_names:
+        _LOGGER.debug('  filter: %s', fil)
+        _LOGGER.debug('    children: %s', new_names)
 
+    return children
 
-def _filter_builtin_modules(path, root, children):
-  """Filters module children to remove builtin modules.
 
-  Args:
-    path: API to this symbol
-    root: The object
-    children: A list of (name, object) pairs.
-
-  Returns:
-    `children` with all builtin modules removed.
-  """
-  del path
-  del root
-  # filter out 'builtin' modules
-  filtered_children = []
-  for name, child in children:
-    # Do not descend into built-in modules
-    if inspect.ismodule(child) and child.__name__ in sys.builtin_module_names:
-      continue
-    filtered_children.append((name, child))
-  return filtered_children
-
-
-def _traverse_internal(root, visitors, stack, path):
-  """Internal helper for traverse."""
-  new_stack = stack + [root]
-
-  # Only traverse modules and classes
-  if not inspect.isclass(root) and not inspect.ismodule(root):
-    return
-
-  try:
-    children = inspect.getmembers(root)
-  except ImportError:
-    # On some Python installations, some modules do not support enumerating
-    # members (six in particular), leading to import errors.
-    children = []
-
-  # Break cycles.
-  filtered_children = []
-  for name, child in children:
-    if any(child is item for item in new_stack):  # `in`, but using `is`
-      continue
-
-    filtered_children.append((name, child))
-  children = filtered_children
-
-  # Apply all callbacks, allowing each to filter the children
-  for visitor in visitors:
-    children = visitor(path, root, list(children))
-
-  for name, child in children:
-    # Break cycles
-    child_path = path + (name,)
-    _traverse_internal(child, visitors, new_stack, child_path)
-
-
-def traverse(root, visitors, root_name):
+def traverse(root, filters, accumulator, root_name) -> None:
   """Recursively enumerate all members of `root`.
 
   Similar to the Python library function `os.path.walk`.
@@ -202,17 +118,13 @@ def traverse(root, visitors, root_name):
   is already in the stack.
 
   Traversing system modules can take a long time, it is advisable to pass a
-  `visit` callable which blacklists such modules.
+  `visit` callable which denylists such modules.
 
   Args:
     root: A python object with which to start the traversal.
-    visitors: A list of callables. Each taking `(path, parent, children)` as
+    filters: A list of callables. Each taking `(path, parent, children)` as
       arguments, and returns a list of accepted children.
+    accumulator: a DocGenerator to accumulate the results.
     root_name: The short-name of the root module.
   """
-  base_visitors = [
-      _filter_module_all,
-      _add_proto_fields,
-      _filter_builtin_modules
-  ]
-  _traverse_internal(root, base_visitors+visitors, [], (root_name,))
+  _Traverser(filters, accumulator).traverse(root, [], (root_name,))
diff --git a/tools/tensorflow_docs/api_generator/traverse_test.py b/tools/tensorflow_docs/api_generator/traverse_test.py
index 1279884b68a..7ea780b10f8 100644
--- a/tools/tensorflow_docs/api_generator/traverse_test.py
+++ b/tools/tensorflow_docs/api_generator/traverse_test.py
@@ -13,15 +13,8 @@
 # limitations under the License.
 # ==============================================================================
 """Tests for Python module traversal."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 from absl.testing import absltest
 
-from tensorflow_docs.api_generator import test_module1
-from tensorflow_docs.api_generator import test_module2
 
 from tensorflow_docs.api_generator import traverse
 
@@ -45,22 +38,12 @@ class Cyclist(object):
     Cyclist.cycle = Cyclist
 
     visitor = TestVisitor()
-    traverse.traverse(Cyclist, [visitor], root_name='root_name')
+    traverse.traverse(Cyclist, [], visitor, root_name='root_name')
     # We simply want to make sure we terminate.
 
-  def test_module(self):
-    visitor = TestVisitor()
-    traverse.traverse(test_module1, [visitor], root_name='root_name')
-
-    called = [parent for _, parent, _ in visitor.call_log]
-
-    self.assertIn(test_module1.ModuleClass1, called)
-    self.assertIn(test_module2.ModuleClass2, called)
-    self.assertNotIn(test_module2.Hidden, called)
-
   def test_class(self):
     visitor = TestVisitor()
-    traverse.traverse(TestVisitor, [visitor], root_name='root_name')
+    traverse.traverse(TestVisitor, [], visitor, root_name='root_name')
     self.assertEqual(TestVisitor,
                      visitor.call_log[0][1])
     # There are a bunch of other members, but make sure that the ones we know
@@ -75,7 +58,7 @@ def test_class(self):
   def test_non_class(self):
     integer = 5
     visitor = TestVisitor()
-    traverse.traverse(integer, [visitor], root_name='root_name')
+    traverse.traverse(integer, [], visitor, root_name='root_name')
     self.assertEqual([], visitor.call_log)
 
 
diff --git a/tools/tensorflow_docs/api_generator/utils.py b/tools/tensorflow_docs/api_generator/utils.py
index a2211113a3a..f90ae9e6c28 100644
--- a/tools/tensorflow_docs/api_generator/utils.py
+++ b/tools/tensorflow_docs/api_generator/utils.py
@@ -17,17 +17,11 @@
 import importlib
 import logging
 import pkgutil
-import sys
-
-_ALLOWED_EXCEPTIONS = (ImportError, AttributeError, NotImplementedError)
 
 
 def _onerror(name):
   logging.exception('Failed to load package: %r', name)
-  errortype, error, _ = sys.exc_info()
-
-  if not issubclass(errortype, _ALLOWED_EXCEPTIONS):
-    raise error
+  logging.error('Continuing')
 
 
 def recursive_import(root, strict=False):
@@ -54,10 +48,10 @@ def recursive_import(root, strict=False):
       modules.append(importlib.import_module(name))
     # And ignore the same set of errors if import_module fails, these are not
     # triggered by walk_packages.
-    except _ALLOWED_EXCEPTIONS:
+    except Exception:  # pylint: disable=broad-except
       if strict:
         raise
       else:
-        logging.exception('Failed to load module: %r', name)
+        _onerror(name)
 
   return modules
diff --git a/tools/tensorflow_docs/modeling/__init__.py b/tools/tensorflow_docs/modeling/__init__.py
index 0c54b6cde62..58d7903d5bc 100644
--- a/tools/tensorflow_docs/modeling/__init__.py
+++ b/tools/tensorflow_docs/modeling/__init__.py
@@ -36,4 +36,4 @@ def on_epoch_end(self, epoch, logs):
       print()
 
     if epoch % self.dot_every == 0:
-      print('.', end='')
+      print('.', end='', flush=True)
diff --git a/tools/tensorflow_docs/plots/__init__.py b/tools/tensorflow_docs/plots/__init__.py
index b855da583da..584087bdf60 100644
--- a/tools/tensorflow_docs/plots/__init__.py
+++ b/tools/tensorflow_docs/plots/__init__.py
@@ -25,19 +25,20 @@
 
 
 def _smooth(values, std):
-  """Smooths a list of values by convolving with a gussian.
+  """Smooths a list of values by convolving with a Gaussian distribution.
 
   Assumes equal spacing.
 
   Args:
     values: A 1D array of values to smooth.
-    std: The standard devistion of the gussian. The units are array elements.
+    std: The standard deviation of the Gaussian distribution. The units are
+      array elements.
 
   Returns:
     The smoothed array.
   """
   width = std * 4
-  x = np.linspace(-width, width, 2 * width + 1)
+  x = np.linspace(-width, width, min(2 * width + 1, len(values)))
   kernel = np.exp(-(x / 5)**2)
 
   values = np.array(values)
@@ -50,7 +51,7 @@ def _smooth(values, std):
 
 
 class HistoryPlotter(object):
-  """A class for plotting named set of keras-histories.
+  """A class for plotting a named set of Keras-histories.
 
   The class maintains colors for each key from plot to plot.
   """
@@ -61,15 +62,15 @@ def __init__(self, metric=None, smoothing_std=None):
     self.smoothing_std = smoothing_std
 
   def plot(self, histories, metric=None, smoothing_std=None):
-    """Plots a {name: history} dictionary of keras histories.
+    """Plots a {name: history} dictionary of Keras histories.
 
     Colors are assigned to the name-key, and maintained from call to call.
     Training metrics are shown as a solid line, validation metrics dashed.
 
     Args:
-      histories: {name: history} dictionary of keras histories.
+      histories: {name: history} a dictionary of Keras histories.
       metric: which metric to plot from all the histories.
-      smoothing_std: the standard-deviaation of the smoothing kernel applied
+      smoothing_std: the standard deviation of the smoothing kernel applied
         before plotting. The units are in array-indices.
     """
     if metric is None:
@@ -78,7 +79,7 @@ def plot(self, histories, metric=None, smoothing_std=None):
       smoothing_std = self.smoothing_std
 
     for name, history in histories.items():
-      # Remember name->color asociations.
+      # Remember name->color associations.
       if name in self.color_table:
         color = self.color_table[name]
       else:
diff --git a/tools/tensorflow_docs/tools/.pre-commit-hooks.yaml b/tools/tensorflow_docs/tools/.pre-commit-hooks.yaml
new file mode 100644
index 00000000000..1124461cfa4
--- /dev/null
+++ b/tools/tensorflow_docs/tools/.pre-commit-hooks.yaml
@@ -0,0 +1,11 @@
+# Example using the https://pre-commit.com/ framework.
+# See: https://github.com/tensorflow/docs/blob/master/tools/tensorflow_docs/tools/README.md#pre-commit
+- id: nbformat
+  name: nbformat
+  description: "Run 'nbformat' on a Jupyter Notebook"
+  entry: python -m tensorflow_docs.tools.nbfmt
+  language: python
+  language_version: python3
+  require_serial: true
+  types: [jupyter]
+  additional_dependencies: []
diff --git a/tools/tensorflow_docs/tools/README.md b/tools/tensorflow_docs/tools/README.md
new file mode 100644
index 00000000000..548fbd60a86
--- /dev/null
+++ b/tools/tensorflow_docs/tools/README.md
@@ -0,0 +1,134 @@
+# TensorFlow Docs notebook tools
+
+The `tensorflow-docs` package contains a collection of notebook tools designed
+for open source documentation workflows.
+
+[Jupyter notebooks](https://nbformat.readthedocs.io/en/latest/) are the
+preferred documentation format for TensorFlow
+[guides](https://www.tensorflow.org/guide) and
+[tutorials](https://www.tensorflow.org/tutorials). These docs integrate with
+[Google Colab](https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb)
+for reproducible environments, regularly tested code examples, and more
+maintainable documentation.
+
+
+## Install
+
+Use `pip` to install the latest `tensorflow-docs` package directly from the
+[tensorflow/docs](https://github.com/tensorflow/docs) GitHub repository:
+
+```shell
+$ python3 -m pip install -U --user git+https://github.com/tensorflow/docs
+```
+
+
+## nbfmt
+
+A notebook formatting tool that makes Jupyter notebook source diffs consistent
+and easier to review. Since notebook authoring environments differ with regard
+to file output, indentation, metadata and other non-specified fields; `nbfmt`
+uses opinionated defaults with a preference for the TensorFlow docs Colab
+workflow. To format a notebook, install the `tensorflow-docs` package and run
+the `nbfmt` tool:
+
+```shell
+$ python3 -m tensorflow_docs.tools.nbfmt [options] notebook.ipynb [...]
+
+$ python3 -m tensorflow_docs.tools.nbfmt --help
+```
+
+`nbfmt` accepts directory arguments that format all child notebooks (skipping
+non-notebook files).
+
+For TensorFlow docs projects, notebooks *without* output cells are executed and
+tested; notebooks *with* saved output cells are published as-is. `nbfmt`
+respects the notebook state and uses the `--remove_outputs` option to explicitly
+remove output cells.
+
+The `--test` flag is for continuous integration tests. This does not format the
+notebook, rather it exits with an error code if the notebook is not in an
+up-to-date formatted state. See the tensorflow/docs
+[GitHub Actions workflow](https://github.com/tensorflow/docs/blob/master/.github/workflows/ci.yaml)
+for an example.
+
+### Pre-commit
+
+You can set up the `nbfmt` tool as a pre-commit check in other repos. To do
+this, use a standard Git hook or use the
+[https://pre-commit.com/](https://pre-commit.com/) framework to create the hook
+for you.
+
+If you want to use pre-commit to handle the hook installation for you, include
+the [.pre-commit-hooks.yaml](./.pre-commit-hooks.yaml) file in your repo with
+the following contents:
+
+```
+repos:
+- repo: https://github.com/tensorflow/docs
+  rev: pre-commit
+```
+
+Someone who clones that repo for development would then install the hook with:
+
+```shell
+# Install pre-commit framework
+$ pip3 install pre-commit
+
+# Install hooks
+$ pre-commit install
+```
+
+## nblint
+
+A notebook linting tool that checks documentation style rules. Used to catch
+common errors and is useful for CI tests. To lint a notebook, install the
+`tensorflow-docs` package and run the `nblint` tool:
+
+```shell
+$ python3 -m tensorflow_docs.tools.nblint [options] notebook.ipynb [...]
+
+$ python3 -m tensorflow_docs.tools.nblint --fix [options] notebook.ipynb [...]
+
+$ python3 -m tensorflow_docs.tools.nblint --help
+```
+
+Some styles require a user-defined argument passed at the command-line. For
+example, the `tensorflow` style (default) uses the `repo` argument to check links:
+
+```shell
+$ python3 -m tensorflow_docs.tools.nblint --arg=repo:tensorflow/docs notebook.ipynb
+```
+
+Lints are assertions that test specific sections of the notebook. These lints
+are collected into
+[style modules](https://github.com/tensorflow/docs/tree/master/tools/tensorflow_docs/tools/nblint/style).
+`nblint` tests the `google` and `tensorflow` styles by default, and different
+styles can be set with the `--styles` option:
+
+```shell
+$ python3 -m tensorflow_docs.tools.nblint \
+    --styles=tensorflow,tensorflow_docs_l10n --arg=repo:tensorflow/docs-1l0n \
+    notebook.ipynb
+```
+
+A style module may contain some lint checks that do not fit your project. You
+can exclude specific lint checks with the `--exclude_lint` option:
+
+```shell
+$ python3 -m tensorflow_docs.tools.nblint \
+    --styles=tensorflow --arg=repo:community/repo-name \
+    --exclude_lint=tensorflow::copyright_check \
+    --exclude_lint=tensorflow::button_website \
+    ./community/notebook.ipynb
+```
+
+Some lint errors can be automatically fixed in the notebook file:
+
+```shell
+$ python3 -m tensorflow_docs.tools.nblint --fix \
+    --arg=repo:tensorflow/docs notebook.ipynb
+```
+
+This applies the lint fixes to the notebook and overwrites the file. Not all
+lint errors have an associated fix. Fixes are applied from the loaded style
+modules.
diff --git a/tools/tensorflow_docs/api_generator/test_module1.py b/tools/tensorflow_docs/tools/__init__.py
similarity index 59%
rename from tools/tensorflow_docs/api_generator/test_module1.py
rename to tools/tensorflow_docs/tools/__init__.py
index 0f4171fa596..a9fdb31abc2 100644
--- a/tools/tensorflow_docs/api_generator/test_module1.py
+++ b/tools/tensorflow_docs/tools/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,19 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""A module target for TraverseTest.test_module."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow_docs.api_generator import test_module2
-
-
-class ModuleClass1(object):
-
-  def __init__(self):
-    self._m2 = test_module2.ModuleClass2()
-
-  def __model_class1_method__(self):
-    pass
diff --git a/tools/tensorflow_docs/tools/nbcp/__init__.py b/tools/tensorflow_docs/tools/nbcp/__init__.py
new file mode 100644
index 00000000000..78cb171abba
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nbcp/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
diff --git a/tools/tensorflow_docs/tools/nbcp/__main__.py b/tools/tensorflow_docs/tools/nbcp/__main__.py
new file mode 100644
index 00000000000..c85b22129d1
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nbcp/__main__.py
@@ -0,0 +1,92 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility for copying cells between notebooks."""
+import pathlib
+import sys
+import textwrap
+
+from absl import app
+import nbformat
+
+from tensorflow_docs.tools.nbfmt import __main__ as nbfmt
+from tensorflow_docs.tools.nbfmt import notebook_utils
+
+
+def process_stats(stats: notebook_utils.CellCopyStats) -> bool:
+  """Displays summary stats to the user. Returns True if any warnings."""
+  print(
+      textwrap.dedent(f"""
+      Notebook copy complete.
+        - Total code cells processed: {stats.processed_cells}
+        - Cells updated: {stats.updated_cells}
+      """))
+
+  has_warnings = any((
+      stats.unmatched_target_cells,
+      stats.unmatched_source_cells,
+      stats.out_of_order_target_cells,
+  ))
+  if has_warnings:
+    print('Warnings:')
+
+    if stats.unmatched_target_cells:
+      notebook_utils.warn(
+          '- Cells in source notebook that are not in the destination: '
+          f'{" ".join(stats.unmatched_target_cells)}')
+
+    if stats.unmatched_source_cells:
+      notebook_utils.warn(
+          '- Cells in destination notebook that are not in the source: '
+          f'{" ".join(stats.unmatched_source_cells)}')
+
+    if stats.out_of_order_target_cells:
+      notebook_utils.warn(
+          '- Cells found earlier in destination notebook than source: '
+          f'{" ".join(stats.out_of_order_target_cells)}')
+
+    print()
+
+  return has_warnings
+
+
+def main(args: list[str]) -> int:
+  if len(args) != 3:
+    notebook_utils.warn('nbcp requires 2 notebooks as arguments:')
+    notebook_utils.warn('  $ ...nbcp src_notebook.ipynb dest_notebook.ipynb'
+                        ' [--nbfmt --args --supported]')
+    sys.exit(1)
+
+  src = pathlib.Path(args[1])
+  dest = pathlib.Path(args[2])
+
+  # Open files and copy cells.
+  with src.open('rt') as src_fh, dest.open('rt') as dest_fh:
+    dest_nb = nbformat.read(dest_fh, nbformat.NO_CONVERT)
+    stats = notebook_utils.copy_code_cells(
+        nbformat.read(src_fh, nbformat.NO_CONVERT), dest_nb)
+
+  # Write over destination file.
+  with dest.open('wt') as dest_fh:
+    nbformat.write(dest_nb, dest_fh)
+
+  warnings = process_stats(stats)
+
+  # Format the notebook.
+  nbfmt.main(['', str(dest)])
+
+  return int(warnings)
+
+
+if __name__ == '__main__':
+  app.run(main)
diff --git a/tools/tensorflow_docs/tools/nbfmt/__init__.py b/tools/tensorflow_docs/tools/nbfmt/__init__.py
new file mode 100644
index 00000000000..a9fdb31abc2
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nbfmt/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
diff --git a/tools/tensorflow_docs/tools/nbfmt/__main__.py b/tools/tensorflow_docs/tools/nbfmt/__main__.py
new file mode 100644
index 00000000000..b806d093a25
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nbfmt/__main__.py
@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Format notebooks using the TensorFlow docs style.
+
+Install the tensorflow-docs package:
+$ python3 -m pip install -U [--user] git+https://github.com/tensorflow/docs
+
+Usage:
+$ python3 -m tensorflow_docs.tools.nbfmt [options] notebook.ipynb [...]
+
+See the TensorFlow notebook template:
+https://github.com/tensorflow/docs/blob/master/tools/templates/notebook.ipynb
+And the TensorFlow docs contributor guide:
+https://www.tensorflow.org/community/contribute/docs
+"""
+
+import enum
+import json
+import os
+import pathlib
+import re
+import sys
+import textwrap
+
+from typing import Any, Dict, List
+
+from absl import app
+from absl import flags
+
+from tensorflow_docs.tools.nbfmt import notebook_utils
+
+OSS = True
+
+flags.DEFINE_integer(
+    "indent", 2, "Indention level for pretty-printed JSON.", lower_bound=0)
+flags.DEFINE_bool("oss", None, "Use OSS formatting.")
+flags.DEFINE_bool("remove_outputs", False,
+                  "Remove output cells from the notebook")
+flags.DEFINE_bool("test", False,
+                  "Test if the notebook is formatted (useful for CI).")
+
+FLAGS = flags.FLAGS
+
+
+def clean_notebook(data: Dict[str, Any], nb_source: str, filepath: pathlib.Path,
+                   remove_outputs: bool, indent: int) -> bytes:
+  """The main notebook formatting logic.
+
+  Args:
+    data: object representing a parsed JSON notebook.
+    nb_source: JSON string of entire notebook contents.
+    filepath: String of notebook filepath passed to the command-line.
+    remove_outputs: Boolean to clear cell output.
+    indent: Integer indicating the number of spaces to indent the JSON.
+
+  Returns:
+    A byte string for the JSON formatted notebook.
+  """
+  clean_root(data, filepath)  # Top-level notebook fields.
+  clean_cells(data, nb_source, remove_outputs)
+  update_license_cells(data)
+
+  nbjson = json.dumps(data, sort_keys=True, ensure_ascii=False, indent=indent)
+
+  if not OSS:
+    # Serialization differences in environments.
+    str_replaces = {"<": r"\u003c", ">": r"\u003e", "&": r"\u0026"}
+    for str_from, str_to in str_replaces.items():
+      nbjson = nbjson.replace(str_from, str_to)
+
+  return (nbjson + "\n").encode("utf-8")
+
+
+def clean_root(data: Dict[str, Any], filepath: pathlib.Path) -> None:
+  """Deletes extra top-level notebook fields and metadata.
+
+  Jupyter format spec:
+  https://nbformat.readthedocs.io/en/latest/format_description.html
+
+  Args:
+    data: object representing a parsed JSON notebook.
+    filepath: String of notebook filepath passed to the command-line.
+  """
+  # These top-level fields are required:
+  notebook_utils.del_entries_except(
+      data, keep=["cells", "metadata", "nbformat_minor", "nbformat"])
+  # All metadata is optional according to spec, but we use some of it.
+  notebook_utils.del_entries_except(
+      data["metadata"], keep=["accelerator", "colab", "kernelspec", "google"]
+  )
+
+  metadata = data.get("metadata", {})
+
+  # Set top-level notebook defaults.
+  data["nbformat"] = 4
+  data["nbformat_minor"] = 0
+
+  # Colab metadata
+  colab = metadata.get("colab", {})
+  notebook_utils.del_entries_except(
+      colab, keep=["collapsed_sections", "name", "toc_visible"])
+  colab["name"] = os.path.basename(filepath)
+  colab["toc_visible"] = True
+  metadata["colab"] = colab
+
+  # Kernelspec metadata
+  kernelspec = metadata.get("kernelspec", {})
+  notebook_utils.del_entries_except(kernelspec, keep=["display_name", "name"])
+
+  supported_kernels = {"python3": "Python 3", "swift": "Swift"}
+  kernel_name = kernelspec.get("name")
+  if kernel_name not in supported_kernels:
+    kernel_name = "python3"  # Notebook defaults to Python3 (same as Colab).
+
+  kernelspec["name"] = kernel_name
+  kernelspec["display_name"] = supported_kernels[kernel_name]
+  metadata["kernelspec"] = kernelspec
+
+  # Google metadata
+  google = metadata.get("google", {})
+  notebook_utils.del_entries_except(google, keep=["keywords", "image_path"])
+  # Don't add the field if it's empty.
+  if google:
+    metadata["google"] = google
+  else:
+    metadata.pop("google", None)
+
+  data["metadata"] = metadata
+
+
+def _clean_code_cell(cell_data: Dict[str, Any], remove_outputs: bool) -> None:
+  """Clean an individual code cell and optionally remove outputs.
+
+  Args:
+    cell_data: object representing a parsed JSON cell.
+    remove_outputs: Boolean to clear cell output.
+  """
+  if remove_outputs:
+    cell_data["outputs"] = []
+    cell_data["execution_count"] = None
+
+  # Ensure outputs field exists since part of the nbformat spec.
+  if cell_data.get("outputs", None) is None:
+    cell_data["outputs"] = []
+
+  # Spec allows null or int (null is Colab default).
+  if cell_data.get("execution_count") == 0:
+    cell_data["execution_count"] = None
+
+
+def _clean_metadata_colab(cell_metadata: Dict[str, Any],
+                          remove_outputs: bool) -> None:
+  """Clean up a cell's `metadata.colab` field.
+
+  Remove all `metadata.colab` contents except for `metadata.colab.resources`, if
+  present. The Colab resources are used to embed data within the notebook and
+  can be treated like output cells (kept unless explicitly removed).
+
+  Args:
+    cell_metadata: object representing the parsed JSON metadata from a cell.
+    remove_outputs: Boolean to clear cell output.
+  """
+  colab = cell_metadata.pop("colab", {})
+  # If no outputs, just clear out `metadata.colab`.
+  if remove_outputs:
+    return
+
+  # Clear around `resources` if not empty. Otherwise, clear out `metata.colab`.
+  if colab.get("resources"):
+    notebook_utils.del_entries_except(colab, keep=["resources"])
+    cell_metadata["colab"] = colab
+
+
+def clean_cells(data: Dict[str, Any], nb_source: str,
+                remove_outputs: bool) -> None:
+  """Remove empty cells and clean code cells.
+
+  Args:
+    data: Object representing a parsed JSON notebook.
+    nb_source: JSON string of entire notebook contents.
+    remove_outputs: Boolean True to remove code cell outputs, False to keep.
+  """
+  # Clear leading and trailing newlines.
+  for cell in data["cells"]:
+    cell_source = cell["source"]
+    while cell_source and cell_source[0] == "\n":
+      cell_source.pop(0)
+    while cell_source and cell_source[-1] == "\n":
+      cell_source.pop()
+    cell["source"] = cell_source
+
+  # Remove empty cells.
+  data["cells"] = [cell for cell in data["cells"] if any(cell["source"])]
+
+  # Clean cell metadata.
+  cell_count = 0
+  for cell in data["cells"]:
+    cell_count += 1
+    cell_metadata = cell.get("metadata", {})
+    if "id" not in cell_metadata:
+      cell_metadata["id"] = notebook_utils.generate_cell_id(
+          cell["source"], cell_count)
+    notebook_utils.del_entries_except(
+        cell_metadata, keep=["id", "cellView", "colab"])
+    _clean_metadata_colab(cell_metadata, remove_outputs)
+
+    cell["metadata"] = cell_metadata
+
+  # The presence of this field indicates that ouputs are already saved.
+  has_outputs = True if '"output_type"' in nb_source else False
+
+  for cell in data["cells"]:
+    if cell["cell_type"] == "code":
+      _clean_code_cell(cell, remove_outputs)
+
+  if has_outputs and remove_outputs:
+    notebook_utils.warn("Removed the existing output cells.")
+
+
+def update_license_cells(data: Dict[str, Any]) -> None:
+  """Format license cell to hide code pane from the Colab form.
+
+  Args:
+    data: object representing a parsed JSON notebook.
+  """
+  # This pattern in Apache and MIT license boilerplate.
+  license_re = re.compile(r"#\s?@title.*License")
+
+  for idx, cell in enumerate(data["cells"]):
+    src_text = "".join(cell["source"])
+
+    if license_re.search(src_text):
+      # Hide code pane from license form
+      metadata = cell.get("metadata", {})
+      metadata["cellView"] = "form"
+      data["cells"][idx]["metadata"] = metadata
+
+
+class Status(enum.Enum):
+  PASS = 0
+  FAIL = 1
+
+
+def format_nb(
+    *,
+    notebooks: List[str],
+    remove_outputs: bool = False,
+    indent: int = 2,
+    test: bool = False,
+) -> Status:
+  """Formats a notebook."""
+  found_error = False  # Track errors for final return code.
+  test_fail_notebooks = []
+  paths, err_paths = notebook_utils.collect_notebook_paths(notebooks)
+
+  if err_paths:
+    found_error = True
+    test_fail_notebooks.extend(err_paths)
+
+  for path in paths:
+    print(f"Format notebook: {path}", file=sys.stderr)
+
+    data, source = notebook_utils.load_notebook(path)
+
+    if not data:
+      found_error = True
+      test_fail_notebooks.append(path)
+      continue
+
+    # Returns formatted JSON byte string.
+    expected_output = clean_notebook(data, source, path, remove_outputs, indent)
+
+    if test:
+      # Compare formatted contents with original file contents.
+      src_bytes = path.read_bytes()
+      if expected_output != src_bytes:
+        test_fail_notebooks.append(path)
+    else:
+      path.write_bytes(expected_output)
+
+  if test:
+    if test_fail_notebooks:
+      error_template = textwrap.dedent("""
+      [test] The following notebooks are not formatted:
+      {notebooks}
+      Please install `nbfmt` and format:
+      $ python3 -m pip install -U --user git+https://github.com/tensorflow/docs
+      $ python3 -m tensorflow_docs.tools.nbfmt notebook.ipynb
+      """)
+      notebooks = "\n".join([f"- {str(fp)}" for fp in test_fail_notebooks])
+      print(error_template.format(notebooks=notebooks), file=sys.stderr)
+      return Status.FAIL
+    else:
+      print("[test] Notebooks are formatted", file=sys.stderr)
+      return Status.PASS
+
+  if found_error:
+    return Status.FAIL
+
+  return Status.PASS
+
+
+def main(argv):
+  if len(argv) <= 1:
+    raise app.UsageError("Missing arguments.")
+
+  if FLAGS.oss is not None:
+    global OSS
+    OSS = FLAGS.oss
+
+  exit_code = format_nb(
+      notebooks=argv[1:],
+      remove_outputs=FLAGS.remove_outputs,
+      indent=FLAGS.indent,
+      test=FLAGS.test)
+  if exit_code == Status.FAIL:
+    sys.exit(1)
+  else:
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/tools/tensorflow_docs/tools/nbfmt/nbfmtmain_test.py b/tools/tensorflow_docs/tools/nbfmt/nbfmtmain_test.py
new file mode 100644
index 00000000000..5f07c103cab
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nbfmt/nbfmtmain_test.py
@@ -0,0 +1,74 @@
+# Copyright 2024 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for nbfmt."""
+import pathlib
+import unittest
+from nbformat import notebooknode
+from tensorflow_docs.tools.nbfmt import __main__ as nbfmt
+
+
+class NotebookFormatTest(unittest.TestCase):
+
+  def test_metadata_cleansing(self):
+    subject_notebook = notebooknode.NotebookNode({
+        "cells": [],
+        "metadata": {
+            "unknown": ["delete", "me"],
+            "accelerator": "GPU",
+            "colab": {
+                "name": "/this/is/clobbered.ipynb",
+                "collapsed_sections": [],
+                "deleteme": "pls",
+            },
+            "kernelspec": {
+                "display_name": "Python 2 foreverrrr",
+                "name": "python2",
+                "deleteme": "deldeldel",
+            },
+            "google": {
+                "keywords": ["one", "two"],
+                "image_path": "/foo/img.png",
+                "more_stuff": "delete me",
+            },
+        },
+    })
+
+    expected_notebook = notebooknode.NotebookNode({
+        "cells": [],
+        "metadata": {
+            "accelerator": "GPU",
+            "colab": {
+                "name": "test.ipynb",
+                "collapsed_sections": [],
+                "toc_visible": True,
+            },
+            "kernelspec": {
+                "display_name": "Python 3",
+                "name": "python3",
+            },
+            "google": {
+                "keywords": ["one", "two"],
+                "image_path": "/foo/img.png",
+            },
+        },
+        "nbformat": 4,
+        "nbformat_minor": 0,
+    })
+
+    nbfmt.clean_root(subject_notebook, pathlib.Path("/path/test.ipynb"))
+    self.assertEqual(subject_notebook, expected_notebook)
+
+
+if __name__ == "__main__":
+  unittest.main()
diff --git a/tools/tensorflow_docs/tools/nbfmt/notebook_utils.py b/tools/tensorflow_docs/tools/nbfmt/notebook_utils.py
new file mode 100644
index 00000000000..6e5e8a36553
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nbfmt/notebook_utils.py
@@ -0,0 +1,172 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A collection of utilities for working with notebook files."""
+import dataclasses
+import hashlib
+import json
+import logging
+import pathlib
+import sys
+import textwrap
+
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from nbformat import notebooknode
+
+
+def collect_notebook_paths(
+    filepaths: List[Union[str, pathlib.Path]]
+) -> Tuple[List[pathlib.Path], List[pathlib.Path]]:
+  """Return list of `pathlib.Path`s for (recursive) notebook filepaths.
+
+  Skips any file that's not an .ipynb notebook or invalid file.
+
+  Args:
+    filepaths: List file path strings passed at command-line.
+
+  Returns:
+    A list of Path objects for all notebook files.
+    A list of Path objects that returned an error.
+  """
+  nb_paths = []
+  err_nb_paths = []
+  for fp in filepaths:
+    path = pathlib.Path(fp)
+    if path.is_dir():
+      nb_paths.extend(path.rglob("*.ipynb"))
+    elif path.is_file():
+      if path.suffix == ".ipynb":
+        nb_paths.append(path)
+      else:
+        print(f"Not an '.ipynb' file, skipping: {path}", file=sys.stderr)
+        err_nb_paths.append(path)
+    else:
+      print(f"Invalid file, skipping: {path}", file=sys.stderr)
+      err_nb_paths.append(path)
+  return nb_paths, err_nb_paths
+
+
+def load_notebook(path: pathlib.Path) -> Tuple[Optional[Dict[str, Any]], str]:
+  """Load and parse JSON data from a notebook file.
+
+  Args:
+    path: A `pathlib.Path` of a Jupyter notebook.
+
+  Returns:
+    Dict: Contains data of the parsed JSON notebook, or null if can't read.
+    String: The entire JSON source code of the notebook.
+  """
+  source = path.read_text(encoding="utf-8")
+  try:
+    data = json.loads(source)
+    if not isinstance(data.get("cells"), list):
+      data = None
+      print("Invalid notebook, unable to find list of cells.", file=sys.stderr)
+  except (json.JSONDecodeError, ValueError) as err:
+    print(
+        textwrap.dedent(f"""
+      Unable to load JSON:
+      {err.__class__.__name__}: {err}
+      """),
+        file=sys.stderr)
+    data = None
+
+  return data, source
+
+
+def warn(msg: str) -> None:
+  """Print highlighted warning message to stderr.
+
+  Args:
+    msg: String to print to console.
+  """
+  # Use terminal codes to print color output to console.
+  print(f" \033[33m {msg}\033[00m", file=sys.stderr)
+
+
+def generate_cell_id(source: str, cell_count: int) -> str:
+  """Generate a new cell ID unique to the notebook."""
+  str_to_hash = f"{cell_count} {source}"
+  cell_id = hashlib.sha256(str_to_hash.encode("utf-8")).hexdigest()
+  return cell_id[:12]
+
+
+def del_entries_except(data: Dict[str, Any], keep: List[str]) -> None:
+  """Modifies `data` to remove any entry not specified in the `keep` list.
+
+  Args:
+    data: Dict representing a parsed JSON object.
+    keep: List of key entries to not deleted from `data`.
+  """
+  to_delete = set(data.keys()) - frozenset(keep)
+  for key in to_delete:
+    del data[key]
+
+
+@dataclasses.dataclass
+class CellCopyStats:
+  processed_cells: int = 0
+  updated_cells: int = 0
+  unmatched_target_cells: list[str] = dataclasses.field(default_factory=list)
+  unmatched_source_cells: list[str] = dataclasses.field(default_factory=list)
+  out_of_order_target_cells: list[str] = dataclasses.field(default_factory=list)
+
+
+def copy_code_cells(source: notebooknode.NotebookNode,
+                    target: notebooknode.NotebookNode) -> CellCopyStats:
+  """Copies code cell source and outputs from source to target."""
+  stats = CellCopyStats()
+  if len(source.cells) != len(target.cells):
+    logging.warning('Source and target notebook have unequal cell counts.')
+
+  target_indices = {c['metadata']['id']: i for i, c in enumerate(target.cells)}
+
+  last_target_idx = -1
+  for cell in source.cells:
+    cell_id = cell['metadata']['id']
+
+    if cell.get('cell_type') != 'code':
+      target_indices.pop(cell_id, None)
+      continue
+
+    if cell_id not in target_indices:
+      logging.warning('Cell %s is not present in the target notebook.', cell_id)
+      stats.unmatched_target_cells.append(cell_id)
+      continue
+
+    stats.processed_cells += 1
+
+    if last_target_idx > (target_idx := target_indices.pop(cell_id)):
+      logging.warning(
+          'Cell %s has been moved earlier in the notebook than expected.',
+          cell_id)
+      stats.out_of_order_target_cells.append(cell_id)
+
+    target_cell = target.cells[target_idx]
+    modified = False
+    for field in 'source', 'outputs':
+      new_value = cell.get(field)
+      if target_cell.get(field) != new_value:
+        target_cell[field] = new_value
+        modified = True
+
+    stats.updated_cells += modified
+    last_target_idx = target_idx
+
+  stats.unmatched_source_cells = [
+      c for c, i in target_indices.items()
+      if target.cells[i].get('cell_type') == 'code'
+  ]
+  return stats
diff --git a/tools/tensorflow_docs/tools/nbfmt/notebook_utils_test.py b/tools/tensorflow_docs/tools/nbfmt/notebook_utils_test.py
new file mode 100644
index 00000000000..4866dcb21b8
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nbfmt/notebook_utils_test.py
@@ -0,0 +1,169 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for notebook_utils."""
+
+from absl.testing import absltest
+from nbformat import notebooknode
+
+from tensorflow_docs.tools.nbfmt import notebook_utils
+
+
+class NotebookUtilsTest(absltest.TestCase):
+
+  def test_copy_code_cells(self):
+    source_notebook = notebooknode.NotebookNode({
+        "cells": [{
+            "cell_type": "markdown",
+            "metadata": {
+                "id": "id1"
+            },
+            "source": ["## Some text"]
+        }, {
+            "cell_type": "code",
+            "metadata": {
+                "id": "id2"
+            },
+            "source": ["# some python\n", "print('hi')"]
+        }]
+    })
+    target_notebook = notebooknode.NotebookNode({
+        "cells": [{
+            "cell_type": "markdown",
+            "metadata": {
+                "id": "id1"
+            },
+            "source": ["## Different text"]
+        }, {
+            "cell_type": "code",
+            "metadata": {
+                "id": "id2"
+            },
+            "source": ["# some old python\n", "print 'hi'"]
+        }]
+    })
+
+    stat = notebook_utils.copy_code_cells(source_notebook, target_notebook)
+
+    # Ensure we have the expected contents (markdown untouched, code copied)
+    self.assertIn("## Different text", target_notebook.cells[0]["source"])
+    self.assertIn("print('hi')", target_notebook.cells[1]["source"])
+
+    # Ensure only the code cell was updated
+    self.assertEqual(1, stat.updated_cells)
+    self.assertEqual(1, stat.processed_cells)
+
+  def test_missing_target_cell(self):
+    source_notebook = notebooknode.NotebookNode({
+        "cells": [{
+            "cell_type": "code",
+            "metadata": {
+                "id": "cell1"
+            },
+            "source": ["# some python\n", "print('hi')"]
+        }, {
+            "cell_type": "markdown",
+            "metadata": {
+                "id": "md1"
+            },
+            "source": ["## text"]
+        }]
+    })
+    target_notebook = notebooknode.NotebookNode({
+        "cells": [{
+            "cell_type": "code",
+            "metadata": {
+                "id": "cell2"
+            },
+            "source": ["# some old python\n", "print 'hi'"]
+        }]
+    })
+
+    stat = notebook_utils.copy_code_cells(source_notebook, target_notebook)
+
+    self.assertEqual(0, stat.updated_cells)
+    self.assertEqual(0, stat.processed_cells)
+    self.assertEqual(["cell1"], stat.unmatched_target_cells)
+
+  def test_missing_source_cell(self):
+    source_notebook = notebooknode.NotebookNode({
+        "cells": [{
+            "cell_type": "code",
+            "metadata": {
+                "id": "cell1"
+            },
+            "source": ["# some python\n", "print('hi')"]
+        }]
+    })
+    target_notebook = notebooknode.NotebookNode({
+        "cells": [{
+            "cell_type": "code",
+            "metadata": {
+                "id": "cell2"
+            },
+            "source": ["# some old python\n", "print 'hi'"]
+        }, {
+            "cell_type": "markdown",
+            "metadata": {
+                "id": "text1"
+            },
+            "source": ["## texty texty"]
+        }]
+    })
+
+    stat = notebook_utils.copy_code_cells(source_notebook, target_notebook)
+
+    self.assertEqual(0, stat.updated_cells)
+    self.assertEqual(0, stat.processed_cells)
+    self.assertEqual(["cell2"], stat.unmatched_source_cells)
+
+  def test_cell_ordering(self):
+    source_notebook = notebooknode.NotebookNode({
+        "cells": [{
+            "cell_type": "code",
+            "metadata": {
+                "id": "cell1"
+            },
+            "source": ["# first code\n"]
+        }, {
+            "cell_type": "code",
+            "metadata": {
+                "id": "cell2"
+            },
+            "source": ["# second code\n"]
+        }]
+    })
+    target_notebook = notebooknode.NotebookNode({
+        "cells": [{
+            "cell_type": "code",
+            "metadata": {
+                "id": "cell2"
+            },
+            "source": ["# update me\n"]
+        }, {
+            "cell_type": "code",
+            "metadata": {
+                "id": "cell1"
+            },
+            "source": ["# update me\n"]
+        }]
+    })
+
+    stat = notebook_utils.copy_code_cells(source_notebook, target_notebook)
+
+    self.assertEqual(2, stat.updated_cells)
+    self.assertIn("cell2", stat.out_of_order_target_cells)
+
+
+if __name__ == "__main__":
+  absltest.main()
diff --git a/tools/tensorflow_docs/tools/nblint/__init__.py b/tools/tensorflow_docs/tools/nblint/__init__.py
new file mode 100644
index 00000000000..a9fdb31abc2
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
diff --git a/tools/tensorflow_docs/tools/nblint/__main__.py b/tools/tensorflow_docs/tools/nblint/__main__.py
new file mode 100644
index 00000000000..1411fbf4c7b
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/__main__.py
@@ -0,0 +1,222 @@
+# pylint: disable=invalid-name
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Check notebook conformance with documentation styles.
+
+Install the tensorflow-docs package:
+$ python3 -m pip install -U [--user] git+https://github.com/tensorflow/docs
+
+Usage:
+$ python3 -m tensorflow_docs.tools.nblint [options] notebook.ipynb [...]
+$ python3 -m tensorflow_docs.tools.nblint --fix [options] notebook.ipynb [...]
+
+$ python3 -m tensorflow_docs.tools.nblint --verbose \
+    [--styles=google,tensorflow] notebook.ipynb [...]
+$ python3 -m tensorflow_docs.tools.nblint --arg=x:foo --arg=y:bar notebook.ipynb
+$ python3 -m tensorflow_docs.tools.nblint [options] \
+    --exclude_lint=tensorflow::copyright_check [...] notebook.ipynb
+
+See the TensorFlow notebook template:
+https://github.com/tensorflow/docs/blob/master/tools/templates/notebook.ipynb
+And the TensorFlow docs contributor guide:
+https://www.tensorflow.org/community/contribute/docs
+"""
+
+import importlib
+import inspect
+import sys
+import textwrap
+
+from absl import app
+from absl import flags
+
+from tensorflow_docs.tools.nbfmt import notebook_utils
+from tensorflow_docs.tools.nblint import decorator
+from tensorflow_docs.tools.nblint import linter
+
+
+flags.DEFINE_multi_string("arg", [], "User arguments to pass to lint callback.")
+flags.DEFINE_multi_string(
+    "exclude_lint", [],
+    "Do not check a specific lint within a style. Format: 'style::function'")
+flags.DEFINE_boolean("fix", False, "Fix lint errors, if possible.")
+flags.DEFINE_list("styles", ["google", "tensorflow"],
+                  "Lint style modules to include.")
+flags.DEFINE_boolean("verbose", False, "Display verbose output.")
+
+FLAGS = flags.FLAGS
+
+
+def _print_fails(path_list):
+  """Format notebooks that failed lint and print to console.
+
+  Args:
+    path_list: A list of Path objects.
+  """
+  template = textwrap.dedent("""\
+    The following notebook{plural} failed lint:
+    {filepaths}""")
+  paths = "\n".join([f" {str(fp)}" for fp in path_list])
+  plural = "s" if len(paths) > 1 else ""
+  print(template.format(filepaths=paths, plural=plural), file=sys.stderr)
+
+
+def _is_user_defined_lint(mod_name):
+  """Return a function that tests if a module member is a user-defined lint.
+
+  Args:
+    mod_name: THe string name of the module file containing the lint.
+
+  Returns:
+    Function: This returns a Boolean: True if the module member is a lint.
+  """
+
+  def is_lint(member):
+    return (inspect.isfunction(member) and member.__module__ == mod_name and
+            hasattr(member, "_lint"))
+
+  return is_lint
+
+
+def add_styles(styles, excluded_lints, verbose):
+  """Import lint assertions from style modules.
+
+  Style modules must exist in the `style/` directory of this package.
+
+  Args:
+    styles: A list of short names for style modules to import.
+    excluded_lints: List of lint functions to skip ('style::function').
+    verbose: Bool, to print more details to console. Default is False.
+
+  Returns:
+    A dictionary containing all the lint styles.
+  """
+
+  lint_dict = {
+      decorator.Options.Scope.CODE: {
+          decorator.Options.Cond.ALL: [],
+          decorator.Options.Cond.ANY: []
+      },
+      decorator.Options.Scope.TEXT: {
+          decorator.Options.Cond.ALL: [],
+          decorator.Options.Cond.ANY: []
+      },
+      decorator.Options.Scope.CELLS: {
+          decorator.Options.Cond.ALL: [],
+          decorator.Options.Cond.ANY: []
+      },
+      decorator.Options.Scope.FILE: {
+          decorator.Options.Cond.ANY: []  # Only one queue is relevant.
+      }
+  }
+
+  for style in styles:
+    mod_name = f"tensorflow_docs.tools.nblint.style.{style}"
+    mod = importlib.import_module(mod_name)
+    is_lint = _is_user_defined_lint(mod_name)
+
+    # Extract Lint instance attached to function object by decorator.
+    lints = [
+        getattr(mem[1], "_lint") for mem in inspect.getmembers(mod, is_lint)
+    ]
+
+    # Remove lints that have been explicitly excluded at the command-line.
+    lints = [
+        lint for lint in lints if f"{style}::{lint.name}" not in excluded_lints
+    ]
+
+    if verbose:
+      lint_names = ", ".join([lint.name for lint in lints])
+      print(f"From style '{mod_name}' import lints: {lint_names}\n")
+
+    for lint in lints:
+      lint.style = style
+      lint_dict[lint.scope][lint.cond].append(lint)
+
+  return lint_dict
+
+
+def _parse_user_args(args_list):
+  """Parse user-defined arguments passed at command-line.
+
+  Args:
+    args_list: List of strings in "key:value" format.
+
+  Returns:
+    A dictionary containing user-defined keys and values.
+  """
+  args_dict = {}
+  for arg_str in args_list:
+    parts = arg_str.split(":", 1)
+    key = parts[0]
+    # Command-line args are strings. If no value provided, use "True".
+    val = parts[1] if len(parts) == 2 else "True"
+    # Add basic string parsing into Python types.
+    if val.isdigit():
+      # Non-exhaustive numeric parsing, for convenience.
+      val = int(val)
+    elif val.lower() == "true":
+      val = True
+    elif val.lower() == "false":
+      val = False
+    args_dict[key] = val
+  return args_dict
+
+
+# Linter is run in both lint and fix modes. When linting, the lint status list
+# is displayed and the program exits with an error on any lint failure.
+# When fixing, no status is displayed and it runs through the queue of fix
+# callbacks, exiting without error reguardless if everything is fixed.
+def main(argv):
+  if len(argv) <= 1:
+    raise app.UsageError("Missing arguments.", 1)
+  if not FLAGS.styles:
+    raise app.UsageError("Missing styles.", 1)
+
+  user_args = _parse_user_args(FLAGS.arg)
+
+  nb_linter = linter.Linter(verbose=FLAGS.verbose)
+  lint_dict = add_styles(FLAGS.styles, FLAGS.exclude_lint, FLAGS.verbose)
+
+  linter_fails = []  # Track failed notebooks for final return code.
+
+  paths, _ = notebook_utils.collect_notebook_paths(argv[1:])
+
+  for path in paths:
+    mode = "Fix" if FLAGS.fix else "Lint"
+    print(f"{mode} notebook: {path}")
+
+    status = nb_linter.run(path, lint_dict, user_args)
+    if not status.is_success:
+      linter_fails.append(path)
+
+    if FLAGS.fix:
+      status.fix_lints()
+    else:
+      print(status)
+
+  # Fix mode always exits as success.
+  if FLAGS.fix:
+    sys.exit(0)
+
+  if linter_fails:
+    _print_fails(linter_fails)
+    sys.exit(1)
+  else:
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/tools/tensorflow_docs/tools/nblint/decorator.py b/tools/tensorflow_docs/tools/nblint/decorator.py
new file mode 100644
index 00000000000..d74045c7ca7
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/decorator.py
@@ -0,0 +1,185 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Defines the @lint API used in style files to create lint tests.
+
+Users define lints by adding the @lint decorator to test functions. A collection
+of lint functions are grouped in a style module. Style modules can be
+enabled/disabled at the command-line.
+
+Lint functions must return a Boolean value: True for pass, False for fail.
+Negative assertions must follow this logic, always pass True if the lint test is
+considered a success.
+
+Whether a lint assertion passes or fails depends on its scope and condition
+parameters:
+
+- The *condition* determines if a test is considered a success if it passes for
+  *all* cells in the notebook, or *any* cells (just one).
+- The *scope* determines where the assertion should be executed: for all
+notebook
+  cells, only text cells, only code, cells, or just run once per notebook.
+
+Implementation-wise, the @lint decorator creates a `Lint` instance that is
+attached to the underlying function object within the style module. When the
+module is imported into the `Linter`, the `Lint` object is extracted.
+
+"""
+import enum
+import functools
+
+from typing import Any, Callable, List, Optional
+
+
+class Options:
+  """Options to define the condition and scope of a @lint defined assertion."""
+
+  class Cond(enum.Enum):
+    """Determines if a test is considered a success by which cells it passes.
+
+    Attributes:
+      ALL: Success if all cells pass.
+      ANY: Success if any cells pass (just one). [Default]
+    """
+    ALL = enum.auto()
+    ANY = enum.auto()
+
+  class Scope(enum.Enum):
+    """Determines where a function test is executed.
+
+    Attributes:
+      CELLS: Code and text cells. [Default]
+      CODE: Code cells only.
+      TEXT: Text cells only.
+      FILE: Run assertion function once per file.
+    """
+    CELLS = enum.auto()
+    CODE = enum.auto()
+    TEXT = enum.auto()
+    FILE = enum.auto()
+
+
+class Lint:
+  """Contains the function and properties defined by the @lint decorator.
+
+  Attributes:
+    run: User-defined assertion callback that returns a Boolean.
+    scope: `Options.Scope` to determine where an assertion is executed.
+    cond: `Options.Cond` to determine if an assertion is considered a success.
+    name: Optional string name for assertion function in reports.
+    message: String message to include in status report.
+    style: String name of style module that defines the Lint. (Added on load.)
+  """
+
+  def __init__(self, fn, scope, cond, message=None, name=None):
+    self.run = fn
+    self.scope = scope
+    self.cond = cond
+    self.name = name if name else fn.__name__
+    self.message = message.strip() if message else ""
+    self.style = None  # Added on style load.
+
+
+# Define a decorator with optional arguments.
+def lint(fn=None, *, message=None, scope=None, cond=None):
+  """Function decorator for user-defined lint assertions.
+
+  Args:
+    fn: User-defined assertion callback that returns a Boolean. See `Linter.run`
+      for args passed to callback, depending on scope:
+        * For cell-scope: callback(source, cell_data, path).
+        * For file-scope: callback(source, all_data, path)
+    message: Optional string message to include in status report.
+    scope: Determines where the function should be executed, options in
+      `Options.Scope`.
+    cond: Determines how an assertion is considered a success, if it passes all
+      cells or any cells. Options available in `Options.Cond`.
+
+  Returns:
+    Function: A wrapper around the user-defined `fn` function.
+  """
+  if fn is None:
+    return functools.partial(lint, message=message, scope=scope, cond=cond)
+
+  scope = scope if scope else Options.Scope.CELLS
+  cond = cond if cond else Options.Cond.ANY
+
+  @functools.wraps(fn)
+  def wrapper(*args, **kwargs):
+    return fn(*args, **kwargs)
+
+  # Attach to function object to access when importing the style module.
+  setattr(wrapper, "_lint", Lint(fn, scope, cond, message))
+  return wrapper
+
+
+class LintFailError(Exception):
+  """Exception raised for lint failure with optional message.
+
+    Attributes:
+      message: String message to add to status log.
+      always_show: Boolean if failure message should display in status,
+        regardless if larger conditional met.
+      fix_fn: Optional Callable to run that fixes the lint failure.
+      fix_args: List of arguments passed to the `fix_fn` Callable.
+  """
+
+  def __init__(self,
+               message: str = "Lint failure",
+               always_show: bool = False,
+               fix_fn: Optional[Callable[[], None]] = None,
+               fix_args: Optional[List[Any]] = None):
+    self.message: str = message
+    self.always_show: bool = always_show
+    self.fix_fn: Optional[Callable[[], None]] = fix_fn
+    if fix_args is None:
+      fix_args = []
+    self.fix_args: List[Any] = fix_args
+    super().__init__(self.message)
+
+
+def fail(message: Optional[str] = None,
+         always_show: bool = False,
+         fix: Optional[Callable[[], None]] = None,
+         fix_args: Optional[List[Any]] = None) -> None:
+  """Signal within a @lint function that the test fails.
+
+  While sufficient to simply return False from a failing @lint function, this
+  function can add a message to the status log to provide the user additional
+  context. Stack trace available with `--verbose` flag.
+
+  Failure messages come in two flavors:
+  - conditional: (Default) While this test may fail here, it may succeed
+    elsewhere, and thus, the larger condition passes and do not display this
+    message.
+  - non-conditional (always show): Regardless if the larger condition is met,
+    display this error message in the status report. For example, a
+    configuration error should always display, even if the test succeeds
+    elsewhere.
+
+  A `fix` function/Callable is executed with the --fix command-line option. It
+  is run as `fn(lint_args, *fix_args)` and executed once for each unique fn-args
+  pair. Callbacks passed the same arg are deduplicated.
+
+  Args:
+    message: String message to add to status log.
+    always_show: Boolean if failure message should display in status, reguardles
+      if larger conditional met.
+    fix: Optional Callable to run that fixes the lint failure.
+    fix_args: Optional list of arguments passed to the `fix` Callable.
+
+  Raises:
+    LintFailError: Lint failure with optional message.
+  """
+  raise LintFailError(message, always_show, fix, fix_args)
diff --git a/tools/tensorflow_docs/tools/nblint/fix.py b/tools/tensorflow_docs/tools/nblint/fix.py
new file mode 100644
index 00000000000..34c603c1ca7
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/fix.py
@@ -0,0 +1,59 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Utlities for fixing lints.
+
+Convenience functions for writing general fix functions for style lints. Opening
+and closing files, modifying contents, all should happen in the fix function
+callback.
+
+Fix functions are run as `fn(lint_args, *fix_args)` and executed once for each
+unique fn-args pair. Callbacks passed the same arg are deduplicated. But it's
+not the end of the world if the same function/args are run multiple times since
+the file changes should be the same.
+"""
+import re
+from typing import Any, Dict, Pattern
+
+
+def regex_replace_all(args: Dict[str, Any], pattern: Pattern[str],
+                      repl: str) -> None:
+  """Replace regex matched content in a file.
+
+  Args:
+    args: Dict of args passed to the lint function.
+    pattern: Regex pattern containing two groups to match.
+    repl: Replacement text to insert between the two matched groups.
+  """
+  fp = args["path"]
+  contents = fp.read_text(encoding="utf-8")
+  contents_new = re.sub(
+      pattern, rf"{repl}", contents, flags=re.MULTILINE | re.DOTALL)
+  if contents_new != contents:
+    fp.write_text(contents_new, encoding="utf-8")
+
+
+def regex_between_groups_replace_all(args: Dict[str, Any],
+                                     pattern: Pattern[str], repl: str) -> None:
+  """Replace content between two matched groups in a file.
+
+  Regex pattern must contain two groups: r'(foo).*(bar)'
+  and the replacement text is inserted between these matches.
+
+  Args:
+    args: Dict of args passed to the lint function.
+    pattern: Regex pattern containing two groups to match.
+    repl: Replacement text to insert between the two matched groups.
+  """
+  regex_replace_all(args, pattern, rf"\g<1>{repl}\g<2>")
diff --git a/tools/tensorflow_docs/tools/nblint/linter.py b/tools/tensorflow_docs/tools/nblint/linter.py
new file mode 100644
index 00000000000..68be9d7b549
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/linter.py
@@ -0,0 +1,473 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Import lint tests, run lints, and report status.
+
+A `Linter` instance imports lint tests from a style file into a structured queue
+that are then run on notebook files. Depending on condition and scope options,
+these are executed for the entire notebook or for each text and code cell.
+
+A `LinterStatus` instance is returned when lint tests are run on a file. While
+a single `Linter` instance can be run on multiple files, a `LinterStatus` is
+associated with a single notebook file. It maintains the pass/fail state for
+each lint test on the file. Additionally, `LinterStatus` implements the
+formatting required to print the report that the console.
+"""
+import collections
+import io
+import pathlib
+import sys
+import traceback
+
+from typing import Any, Callable, Dict, List, NamedTuple, Optional, Set, Tuple
+
+from tensorflow_docs.tools.nbfmt import notebook_utils
+from tensorflow_docs.tools.nblint import decorator
+
+
+# Custom type for a dictionary containing the lint styles.
+LintDict = Dict[decorator.Options.Scope, Dict[decorator.Options.Cond,
+                                              List[decorator.Lint]]]
+
+
+class Linter:
+  """Manages the collection of lints to execute on a notebook.
+
+  Lint assertions are imported by style modules and dispatched by condition and
+  scope. A Linter can be run on multiple notebooks.
+
+  Attributes:
+    verbose: Boolean to print more details to console. Default is False.
+  """
+
+  class RunLintStatus(NamedTuple):
+    """The return status and metadata from an executed lint function.
+
+    Attributes:
+      is_success: Boolean
+      lint: `decorator.Lint` associated with this status.
+      lint_args: Dict of args passed to lint function.
+      cond_fail_msg: Optional string to print in failure message.
+      fix_fn: Optional Callable to run that fixes the lint failure.
+      fix_args: List of arguments passed to the `fix_fn` Callable.
+    """
+    is_success: bool
+    lint: decorator.Lint
+    lint_args: Dict[str, Any]
+    cond_fail_msg: Optional[str] = None
+    fix_fn: Optional[Callable[[], None]] = None
+    fix_args: List[Any] = []
+
+  def __init__(self, verbose: bool = False):
+    self.verbose: bool = verbose
+
+  def _load_notebook(self, path: pathlib.Path) -> Tuple[Dict[str, Any], str]:
+    """Load and parse JSON data from a notebook file.
+
+    Args:
+      path: A `pathlib.Path` of a Jupyter notebook.
+
+    Returns:
+      Dict: Contains data of the parsed JSON notebook.
+      String: The entire JSON source code of the notebook.
+    """
+    data, source = notebook_utils.load_notebook(path)
+
+    if not data:
+      sys.exit(1)  # load_notebook prints warning.
+
+    return data, source
+
+  def _run_lint(self, lint: decorator.Lint, lint_args: Dict[str, Any],
+                status: "LinterStatus") -> "Linter.RunLintStatus":
+    """Run lint and capture any stderr output for the status display.
+
+    Args:
+      lint: `decorator.Lint` containing the assertion, scope, and condition.
+      lint_args: Nested dictionary of args to pass the lint callback function.
+      status: The `LinterStatus` to add individual entries for group members.
+
+    Returns:
+      Linter.RunLintStatus: Return status and metadata for this lint function.
+    """
+
+    cond_fail_msg = None
+    fix_fn = None
+    fix_args = []
+    try:
+      is_success = lint.run(lint_args)
+    except decorator.LintFailError as err:
+      is_success = False
+      fix_fn = err.fix_fn
+      fix_args = err.fix_args
+      if err.always_show:
+        # Grab stack trace and carry on.
+        f = io.StringIO()
+        traceback.print_exc(file=f)
+        trace = f.getvalue()
+        # Add any non-conditional failure messages to queue. Will de-dup.
+        status.log_lint_message(err.message, lint, verbose_msg=trace)
+      else:
+        # Defer logging a conditional message until the group status is known.
+        cond_fail_msg = err.message
+
+    return self.RunLintStatus(is_success, lint, lint_args, cond_fail_msg,
+                              fix_fn, fix_args)
+
+  def _run_lint_group(self, lint: decorator.Lint, lint_args: Dict[str, Any],
+                      data: Dict[str, Any],
+                      status: "LinterStatus") -> Tuple[bool, Set[str]]:
+    """Run lint over all cells with scope and return cumulative pass/fail.
+
+    Args:
+      lint: `decorator.Lint` containing the assertion, scope, and condition.
+      lint_args: Nested dictionary of args to pass the lint callback function.
+      data: `dict` containing data of entire parse notebook.
+      status: The `LinterStatus` to add individual entries for group members.
+
+    Returns:
+      Boolean: True if lint passes for all/any cells, otherwise False.
+      Set: Deduplicated list of failure message strings.
+
+    Raises:
+      Exception: Unsupported lint condition in `decorator.Options.Cond`.
+    """
+    scope: decorator.Options.Scope = lint.scope
+    # Return value of each (scoped) cell in notebook.
+    is_success_list: List[bool] = []
+    # All conditional failure messages from lint function (deduplicated).
+    cond_fail_message_list: Set[str] = set()
+
+    for cell_idx, cell in enumerate(data.get("cells")):
+      # Evict notebook cells outside of scope.
+      cell_type = cell.get("cell_type")
+      if scope is decorator.Options.Scope.TEXT and cell_type != "markdown":
+        continue
+      elif scope is decorator.Options.Scope.CODE and cell_type != "code":
+        continue
+
+      # Add cell-specific data to args passed to lint callback.
+      lint_args["cell_data"] = cell
+      lint_args["cell_source"] = "".join(cell["source"])
+
+      # Execute lint on cell and collect result.
+      run_status = self._run_lint(lint, lint_args, status)
+      is_success_list.append(run_status.is_success)
+      if run_status.cond_fail_msg:
+        cond_fail_message_list.add(run_status.cond_fail_msg)
+
+      # All lint runs get a status entry. Group success is a separate entry.
+      name = f"{lint.name}__cell_{cell_idx}"
+      status.add_entry(
+          lint, run_status, name=name, group=lint.name, is_group_entry=True)
+
+    # Return True/False success for entire cell group.
+    if lint.cond is decorator.Options.Cond.ANY:
+      return any(is_success_list), cond_fail_message_list
+    elif lint.cond is decorator.Options.Cond.ALL:
+      return all(is_success_list), cond_fail_message_list
+    else:
+      raise Exception("Unsupported lint condition.")
+
+  def run(self, path: pathlib.Path, lint_dict: LintDict,
+          user_args_dict: Dict[str, Any]) -> "LinterStatus":
+    """Multiple hooks provided to run tests at specific points.
+
+    Args:
+      path: `pathlib.Path` of notebook to run lints against.
+      lint_dict: A dictionary containing the lint styles.
+      user_args_dict: Dictionary of user-defined args passed to lint callback.
+
+    Returns:
+      LinterStatus: Provides status and reporting of lint tests for a notebook.
+    """
+    data, source = self._load_notebook(path)
+    if not data:
+      return False
+
+    # Args passed to lint callback function.
+    lint_args = {
+        "cell_data": None,  # Added per-cell in _run_lint_group.
+        "cell_source": None,  # Added per-cell in _run_lint_group.
+        "file_data": data,
+        "file_source": source,
+        "path": path,
+        "user": user_args_dict
+    }
+
+    status = LinterStatus(path, verbose=self.verbose)
+
+    # File-level scope.
+    # Lint run once for the file.
+    for lint in lint_dict[decorator.Options.Scope.FILE][
+        decorator.Options.Cond.ANY]:
+      run_status = self._run_lint(lint, lint_args, status)
+      status.add_entry(lint, run_status)
+
+    # Cell-level scope.
+    # These lints run on each cell, then return a cumulative result.
+    for scope in [
+        decorator.Options.Scope.CELLS, decorator.Options.Scope.CODE,
+        decorator.Options.Scope.TEXT
+    ]:
+      for cond in decorator.Options.Cond:
+        lints = lint_dict[scope][cond]
+        for lint in lints:
+          # Run lint group and create a separate group status.
+          is_success, cond_fail_msgs = self._run_lint_group(
+              lint, lint_args, data, status)
+          run_status = Linter.RunLintStatus(is_success, lint, lint_args)
+          status.add_entry(lint, run_status, group=lint.name)
+          if not is_success:
+            # Once group status is known, log any conditional messages.
+            for msg in cond_fail_msgs:
+              # Grab stack trace and carry on.
+              f = io.StringIO()
+              traceback.print_exc(file=f)
+              trace = f.getvalue()
+              # Add any non-conditional failure messages to queue. Will de-dup.
+              status.log_lint_message(msg, lint, verbose_msg=trace)
+
+    return status
+
+
+class LinterStatus:
+  """Provides status and reporting of lint tests for a notebook.
+
+  A new `LinterStatus` object is returned when `Linter.run` is executed on a
+  given notebook. A `LinterStatus` object represents a run of all lints for a
+  single notebook file. Multiple notebook files require multiple `LinterStatus`
+  objects. Though multiple status objects can be created by the same `Linter`.
+
+  The `LinterStatus` instance manages `LintStatusEntry` objects. These are added
+  in the `Linter.run` for each lint test. Some entries may be a part of a larger
+  lint group that represents a collective pass/fail status.
+
+  A `LinterStatus` instance is also reponsible for printing status reports for
+  entries to the console to display to the user.
+
+  Attributes:
+    path: `pathlib.Path` of notebook that lints were run against.
+    verbose: Boolean to print more details to console. Default is False.
+    is_success: Boolean status of entire lint report: True if all tests pass,
+      otherwise False.
+  """
+
+  class LintStatusEntry(NamedTuple):
+    """Represents the status of a lint tested against a single section.
+
+    Depending on the scope of the lint, one lint can create multiple
+    `LintStatusEntry` objects. For example, if tested against all notebook
+    cells, one status entry would be created for each cell it is run on. This
+    would also create a group entry representing the cumulative conditional
+    test: any/all.
+
+    Groups are determined by a shared a group name. If an entry is designed with
+    True for `is_group_entry`, that means it's a member (child) of the group.
+    The cumulative status is the one member of the group that is set to False
+    for `is_group_entry`.
+
+    Attributes:
+      lint: `decorator.Lint` associated with this status.
+      run_status: `Linter.RunLintStatus` return status for lint.
+      name: Optional name of the status entry for reports. Default to lint name.
+      group: Optional string of shared group name for multiple entries.
+      is_group_entry: Boolean. If in group, True if entry is memmber/child of
+        group, and Falsw if it represents the collective status of a group.
+    """
+    lint: decorator.Lint
+    run_status: Linter.RunLintStatus
+    name: str
+    group: Optional[str]
+    is_group_entry: bool
+
+  def __init__(self, path: pathlib.Path, verbose: bool = False) -> None:
+    self.path: pathlib.Path = path
+    self.verbose: bool = verbose
+    # Contains all status entries.
+    self._status_list: List[self.LintStatusEntry] = []
+    # Deduplicated stderr messages printed within lint functions.
+    self._lint_messages: Set[str] = set()
+
+  def add_entry(self,
+                lint,
+                run_status,
+                name=None,
+                group=None,
+                is_group_entry=False):
+    """Add a new `LintStatusEntry` to report.
+
+    Args:
+      lint: `decorator.Lint` associated with this status.
+      run_status: `Linter.RunLintStatus`
+      name: Optional name of the status entry for reports. Default to lint name.
+      group: Optional string of shared group name for multiple entries.
+      is_group_entry: Boolean. If in group, True if entry is memmber/child of
+        group, and Falsw if it represents the collective status of a group.
+    """
+    if not isinstance(run_status.is_success, bool):
+      raise TypeError(
+          f"Lint status must return Boolean, got: {run_status.is_success}")
+    name = name if name else lint.name
+    entry = self.LintStatusEntry(lint, run_status, name, group, is_group_entry)
+    self._status_list.append(entry)
+
+  @property
+  def is_success(self):
+    """Represents the status of entire lint report.
+
+    Returns:
+      Boolean: True if all top-level status entries pass, otherwise False.
+    """
+    status = True
+    for entry in self._status_list:
+      if not entry.is_group_entry and not entry.run_status.is_success:
+        status = False
+        break
+    return status
+
+  def log_lint_message(self,
+                       msg: str,
+                       lint: decorator.Lint,
+                       verbose_msg: Optional[str] = None) -> None:
+    """Add message to lint message queue.
+
+    Args:
+      msg: String message captured from stderr.
+      lint: `decorator.Lint` associated with this message.
+      verbose_msg: String to add to next line, displayed with `--verbose` flag.
+    """
+    prefix = f"\033[33m[{lint.style}::{lint.name}]\033[00m"  # Yellow
+    log_line = f"{prefix} {msg}"
+    if self.verbose and verbose_msg:
+      log_line += f"\n{verbose_msg}"
+    self._lint_messages.add(log_line)
+
+  def _format_lint_messages(self):
+    """Pretty-print stderr messages logged from within lint functions.
+
+    Returns:
+      String: Contains list of stderr messages.
+    """
+    output_str = ""
+    if self._lint_messages:
+      output_str += "\nLint log:\n"
+      for msg in self._lint_messages:
+        output_str += (msg + "\n")
+    return output_str
+
+  def _format_status(self, entry: "LinterStatus.LintStatusEntry") -> str:
+    """Pretty-print an entry status for console (with color).
+
+    Args:
+      entry: `LintStatusEntry` with status.
+
+    Returns:
+      String: 'Pass' or 'Fail' with terminal color codes.
+    """
+    if entry.run_status.is_success:
+      msg = "\033[32mPass\033[00m"  # Green
+    else:
+      if entry.is_group_entry:
+        msg = "\033[33mFail\033[00m"  # Yellow: group entry
+      else:
+        msg = "\033[91mFail\033[00m"  # Light red: root entry
+    return msg
+
+  def __str__(self):
+    """Print the entire status report of all entries to console.
+
+    Arrange and format entries for reporting to console. If
+    `LinterStatus.verbose` is True, display group member entries in addition to
+    the cumulative group status. Called as: `print(linter_status)`.
+
+    Returns:
+      String containing the entire lint report.
+    """
+    # Sort group entries to display nested underneath parent.
+    groups = {}
+    # Can skip if not displaying.
+    if self.verbose:
+      for entry in self._status_list:
+        if entry.is_group_entry:
+          if entry.group in groups:
+            groups[entry.group].append(entry)
+          else:
+            groups[entry.group] = [entry]
+
+    # Filter top-level entries.
+    root_entries = [obj for obj in self._status_list if not obj.is_group_entry]
+    output = ""
+
+    for entry in root_entries:
+      # Print top-level entry.
+      status = self._format_status(entry)
+      name = f"{entry.lint.style}::{entry.name}"
+      msg = f" | {entry.lint.message}" if entry.lint.message else ""
+      output += f"{status} | {name}{msg}\n"
+
+      # Print child entries, if applicable.
+      if self.verbose and entry.group in groups:
+        output += "[All results]\n"
+        for child in groups[entry.group]:
+          output += f"- {self._format_status(child)} | {child.name}\n"
+
+        output += "\n"
+
+    # Print any stderror messages from within lint functions.
+    output += self._format_lint_messages()
+
+    return output
+
+  def fix_lints(self):
+    """Fix lint errors, if possible.
+
+    This executes all the fix callbacks passed to the `fail` function in a lint
+    test. Any file changes and writes must be added to the callback since this
+    is only a runner.
+
+    Fix functions are run as `fn(lint_args, *fix_args)` and executed once for
+    each unique fn-args pair. Callbacks passed the same arg are deduplicated.
+    But it's not the end of the world if the same function/args are run multiple
+    times since the file changes should be the same.
+    """
+
+    # Custom type for: {function: [(display_name, args), ...]}
+    FixFnQueue = Dict[Callable[[], None], List[Tuple[str, List[Any]]]]
+    fix_fns: FixFnQueue = collections.defaultdict(list)
+
+    for entry in self._status_list:
+      if not entry.run_status.fix_fn:
+        continue
+
+      fn = entry.run_status.fix_fn
+      args = [entry.run_status.lint_args] + entry.run_status.fix_args
+      display_name = f"{entry.lint.style}::{entry.lint.name}::{entry.run_status.fix_fn.__name__}"
+      arg_group = (display_name, args)
+
+      if arg_group not in fix_fns[fn]:
+        fix_fns[fn].append(arg_group)
+
+    # Execute each fix function with different arg lists.
+    fn_count = 0
+    for fix_fn, arg_groups in fix_fns.items():
+      for arg_group in arg_groups:
+        display_name, args = arg_group
+        if self.verbose:
+          print(f" {display_name}")
+        fn_count += 1
+        fix_fn(*args)
+
+    plural = "" if fn_count == 1 else "es"
+    print(f"Ran {fn_count} lint fix{plural}. (For details, use --verbose)")
diff --git a/tools/tensorflow_docs/tools/nblint/style/__init__.py b/tools/tensorflow_docs/tools/nblint/style/__init__.py
new file mode 100644
index 00000000000..a9fdb31abc2
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/style/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
diff --git a/tools/tensorflow_docs/tools/nblint/style/gemini_cookbook.py b/tools/tensorflow_docs/tools/nblint/style/gemini_cookbook.py
new file mode 100644
index 00000000000..1390de65c21
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/style/gemini_cookbook.py
@@ -0,0 +1,431 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Lint assertions for notebooks published on tensorflow.org.
+
+These lints are a non-exhaustive implementation of style rules found in the
+TensorFlow documentation and style guides. See:
+
+- https://www.tensorflow.org/community/contribute/docs
+- https://www.tensorflow.org/community/contribute/docs_style
+
+When adding lints, link to the URL of the relevant style rule, if applicable.
+
+Lint functions return a boolean: True to pass, False to fail.
+For @lint options, see the docstrings in `decorator.py`.
+
+Lint callback functions are passed an `args` dict with the following entries:
+  cell_data: Dict of parsed cell (cell-scope only)
+  cell_source: String of cell content (cell-scope only)
+  file_data: Dict of parsed notebook
+  file_source: String of notebook content
+  path: Filepath of notebook
+  user: Dict of args passed at the command-line
+"""
+import pathlib
+import re
+import urllib
+
+from tensorflow_docs.tools.nblint import fix
+from tensorflow_docs.tools.nblint.decorator import fail
+from tensorflow_docs.tools.nblint.decorator import lint
+from tensorflow_docs.tools.nblint.decorator import Options
+
+
+def search_wordlist(wordlist, src_str):
+  """Search for wordlist entries in text and return set of found items.
+
+  Args:
+    wordlist: Dict of word entries and recommendations to search in string.
+    src_str: String to search for word entries.
+
+  Returns:
+    A dict that is a subset of entries from `wordlist` found in `src_str`.
+  """
+  found_words = {}
+  for word in wordlist:
+    # Word-boundary and ignore between path separator '/'.
+    if re.search(rf"[^/]\b{word}\b[^/]", src_str, re.IGNORECASE):
+      alt_word = wordlist[word]
+      if not alt_word:
+        alt_word = "n/a"
+      found_words[word] = alt_word
+  return found_words
+
+
+# Acceptable copyright heading for notebooks following this style.
+copyrights_re = [
+    r"Copyright 20[1-9][0-9] The TensorFlow\s.*?\s?Authors",
+    r"Copyright 20[1-9][0-9] Google",
+    r"Copyright 20[1-9][0-9] The AI Edge Authors",
+]
+
+
+@lint(message="Copyright required", scope=Options.Scope.TEXT)
+def copyright_check(args):
+  cell_source = args["cell_source"]
+  return any(re.search(pattern, cell_source) for pattern in copyrights_re)
+
+
+license_re = re.compile("#\s?@title Licensed under the Apache License")
+
+
+@lint(
+    message="Apache license cell is required",
+    scope=Options.Scope.CODE,
+    cond=Options.Cond.ANY,
+)
+def license_check(args):
+  if license_re.search(args["cell_source"]):
+    return True
+  else:
+    template_url = "https://github.com/tensorflow/docs/blob/master/tools/templates/notebook.ipynb"
+    fail(f"License cell missing or doesn't follow template: {template_url}")
+
+
+@lint(scope=Options.Scope.FILE)
+def not_translation(args):
+  if "site" not in args["path"].parents:
+    return True
+  else:
+    return "site/en" in args["path"].parents
+
+
+# Button checks
+
+# Look for class="tfo-notebook-buttons" (CSS used on website versions) or the
+# run-in-colab logo (for notebooks that stick to GitHub/Colab).
+is_button_cell_re = re.compile(
+    r"class.*tfo-notebook-buttons|colab_logo_32px\.png|colab-badge\.svg"
+)
+
+
+def get_arg_or_fail(user_args, arg_name, arg_fmt):
+  """Get value of the user-defined arg passed at the command-line.
+
+  Args:
+    user_args: Dict containing user-defined args passed at command-line.
+    arg_name: String name of user-defined arg.
+    arg_fmt: String format of expected user-defined arg.
+
+  Returns:
+    Value of arg passed to command-line. If the arg does not exist, raise a
+    failure, log a message, and skip the lint function.
+  """
+  if arg_name in user_args:
+    return user_args.get(arg_name)
+  else:
+    fail(
+        f"Requires user-argument '{arg_name}': nblint"
+        f" --arg={arg_name}:{arg_fmt} ...",
+        always_show=True,
+    )
+
+
+def split_doc_path(filepath):
+  """Return paths for docs root prefix directory and the relative path to file.
+
+  Given a full path to notebook file, standalone or within an established
+  documentation directory layout, split the provided path into two:
+  1. a path reprsenting the prefix directory to the docs root (if it exists),
+  2. the relative path to the file from the docs root directory.
+  If in an unknown docs directory layout, return an empty prefix path and the
+  full path of the original argument.
+
+  For example:
+  "site/en/notebook.ipynb" => ("site/en", "notebook.ipynb")
+  "tensorflow/docs/notebook.ipynb" => ("docs", "notebook.ipynb")
+  "unknown/path/notebook.ipynb" => ("", "unknown/path/notebook.ipynb")
+
+  Args:
+    filepath: `pathlib.Path` to a documentation notebook.
+
+  Returns:
+    pathlib.Path: The path of the doc root prefix directory., if applicable.
+    pathlib.Path: The relative path to notebook from the prefix directory.
+  """
+  fp_full = filepath.resolve()  # Check full path for sub-elements.
+
+  def split_path_on_dir(fp, dirname, offset=1):
+    parts = fp.parts
+    idx = parts.index(dirname)
+    docs_dir = pathlib.Path(*parts[idx : idx + offset])
+    rel_path = fp.relative_to(*parts[: idx + offset])
+    return docs_dir, rel_path
+
+  if "site" in fp_full.parts:
+    return split_path_on_dir(fp_full, "site", offset=2)  # site//
+  elif "docs" in fp_full.parts:
+    return split_path_on_dir(fp_full, "docs")
+  elif "g3doc" in fp_full.parts:
+    idx = fp_full.parts.index("g3doc")
+    if fp_full.parts[idx + 1] == "en":
+      offset = 2
+    else:
+      offset = 1
+    return split_path_on_dir(fp_full, "g3doc", offset=offset)
+  else:
+    # Unknown setup. Return empty root and unsplit path.
+    return pathlib.Path(), filepath
+
+
+@lint(
+    message="Missing or malformed URL in Colab button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY,
+)
+def button_colab(args):
+  """Test that the URL in the Colab button matches the file path."""
+  cell_source = args["cell_source"]
+  repo = get_arg_or_fail(args["user"], "repo", "")
+  branch = args["user"].get("branch", "master")
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  # Buttons use OSS URLs.
+  if str(docs_dir) == "g3doc/en":
+    docs_dir = pathlib.Path("site/en")
+
+  base_url = f"colab.research.google.com/github/{repo}/blob/{branch}"
+  this_url = "https://" + str(base_url / docs_dir / rel_path)
+
+  if is_button_cell_re.search(cell_source) and cell_source.find(this_url) != -1:
+    return True
+  else:
+    fail(
+        f"Colab button URL doesn't match: {this_url}",
+        fix=fix.regex_between_groups_replace_all,
+        fix_args=[r"(href.*)http.*?(\\\".*colab_logo_32px.png)", this_url],
+    )
+
+
+@lint(
+    message="Missing or malformed URL in Download button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY,
+)
+def button_download(args):
+  """Test that the URL in the Download button matches the file path."""
+  cell_source = args["cell_source"]
+  repo = get_arg_or_fail(args["user"], "repo", "")
+  repo_name = pathlib.Path(repo.split("/")[1])
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  if "r1" in rel_path.parts:
+    return True  # No download button for TF 1.x docs.
+
+  # Buttons use OSS URLs.
+  if str(docs_dir) == "g3doc/en":
+    docs_dir = pathlib.Path("site/en")
+
+  this_url = urllib.parse.urljoin(
+      "https://storage.googleapis.com",
+      str(f"tensorflow_docs/{repo_name}" / docs_dir / rel_path),
+  )
+
+  if is_button_cell_re.search(cell_source) and cell_source.find(this_url) != -1:
+    return True
+  else:
+    fail(
+        f"Download button URL doesn't match: {this_url}",
+        fix=fix.regex_between_groups_replace_all,
+        fix_args=[r"(href.*)http.*?(\\\".*download_logo_32px.png)", this_url],
+    )
+
+
+@lint(
+    message="Missing or malformed URL in GitHub button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY,
+)
+def button_github(args):
+  """Test that the URL in the GitHub button matches the file path."""
+  cell_source = args["cell_source"]
+  repo = get_arg_or_fail(args["user"], "repo", "")
+  branch = args["user"].get("branch", "master")
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  # Buttons use OSS URLs.
+  if str(docs_dir) == "g3doc/en":
+    docs_dir = pathlib.Path("site/en")
+
+  base_url = f"github.com/{repo}/blob/{branch}"
+  this_url = "https://" + str(base_url / docs_dir / rel_path)
+
+  if is_button_cell_re.search(cell_source) and cell_source.find(this_url) != -1:
+    return True
+  else:
+    fail(
+        f"GitHub button URL doesn't match: {this_url}",
+        fix=fix.regex_between_groups_replace_all,
+        fix_args=[r"(href.*)http.*?(\\\".*GitHub-Mark-32px.png)", this_url],
+    )
+
+
+@lint(
+    message="Missing or malformed URL in 'View on' button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY,
+)
+def button_website(args):
+  """Test that the website URL in the 'View on' button matches the file path.
+
+  Because of subsites and different output directories, the exact website path
+  can't be known from the file alone. But can check that the URL matches a
+  correct pattern.
+
+  Args:
+    args: Nested dict of runtime arguments.
+
+  Returns:
+    Boolean: True if lint test passes, False if not.
+  """
+  cell_source = args["cell_source"]
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  if "r1" in rel_path.parts:
+    return True  # No website button for TF 1.x docs.
+
+  user_url = args["user"].get("base_url")
+  if user_url:
+    base_url = user_url
+  elif str(docs_dir) == "site/zh-cn" or str(docs_dir) == "site/zh-tw":
+    base_url = "https://tensorflow.google.cn/"
+  else:
+    base_url = "https://www.tensorflow.org/"
+
+  # Construct website URL pattern based on location of this file in repo.
+  url_path = rel_path.with_suffix("")
+  # If run in source repo, we don't know for certain the published subsite URL.
+  # Match: base//notebook-path
+  this_url = rf"{base_url}[\w\-/]*{url_path}"
+
+  if is_button_cell_re.search(cell_source) and re.search(this_url, cell_source):
+    return True
+  else:
+    # If included verbatim, bracket will fail lint. That's desired.
+    url_format = f"{base_url}/{url_path}"
+    fail(f"'View on' button URL doesn't match pattern: {url_format}")
+
+
+@lint(
+    message="Missing or malformed URL in 'TFHub' button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY,
+)
+def button_hub(args):
+  """Notebooks that mention tfhub.dev should have a TFHub button."""
+  cell_source = args["cell_source"]
+  file_source = args["file_source"]
+
+  hub_url = "https://tfhub.dev/"
+
+  # Only check files that mention TFHub.
+  if file_source.find(hub_url) == -1:
+    return True
+
+  if is_button_cell_re.search(cell_source) and cell_source.find(hub_url) != -1:
+    return True
+  else:
+    # If included verbatim, bracket will fail lint. That's desired.
+    url_format = f"{hub_url}"
+    fail(f"'TFHub' button URL doesn't match pattern: {url_format}")
+
+
+@lint(
+    message="Remove extra buttons from TF 1.x docs.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ALL,
+)
+def button_r1_extra(args):
+  """The r1/ docs should not have website or download buttons."""
+  cell_source = args["cell_source"]
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  # Only test r1/ notebooks.
+  if "r1" not in rel_path.parts:
+    return True
+  # Only check text cells that contain the button nav bar.
+  if not is_button_cell_re.search(cell_source):
+    return True
+
+  download_url = "https://storage.googleapis.com/tensorflow_docs/"
+  if str(docs_dir) == "site/zh-cn" or str(docs_dir) == "site/zh-tw":
+    base_url = "https://tensorflow.google.cn/"
+  else:
+    base_url = "https://www.tensorflow.org/"
+
+  # Look for button URLs that shouldn't be there..
+  if (
+      re.search(f"{base_url}/(?!images)", cell_source)
+      or cell_source.find(download_url) != -1
+  ):
+    fail(
+        "Remove the 'View on' and 'Download notebook' buttons since r1/ docs"
+        " are not published."
+    )
+  else:
+    return True
+
+
+# Non-exhaustive list: {word: alt-word} (Use False if alt not provided.)
+_SECOND_PERSON_WORDLIST = {"we": "you", "we're": "you are"}
+
+
+@lint(
+    message=(
+        "Prefer second person instead of first person:"
+        " https://developers.google.com/style/person"
+    ),
+    cond=Options.Cond.ALL,
+)
+def second_person(args):
+  """Test for first person usage in doc and recommend second person."""
+  found_words = search_wordlist(_SECOND_PERSON_WORDLIST, args["cell_source"])
+  if found_words:
+    words = ", ".join([f"{word} => {alt}" for word, alt in found_words.items()])
+    fail(
+        f"Prefer second person instead of first person. Found: {words} in"
+        f" {args['cell_source']}"
+    )
+  else:
+    return True
+
+
+# Non-exhaustive list: {word: alt-word} (Use False if alt not provided.)
+_INCLUSIVE_WORDLIST = {
+    "blacklist": "blocked",
+    "whitelist": "allowed",
+    "master": "primary",
+    "slave": "replica",
+}
+
+
+@lint(
+    message=(
+        "Use inclusive language:"
+        " https://developers.google.com/style/inclusive-documentation"
+    ),
+    cond=Options.Cond.ALL,
+)
+def inclusive_language(args):
+  """Test for words found in inclusive wordlist and recommend alternatives."""
+  found_words = search_wordlist(_INCLUSIVE_WORDLIST, args["cell_source"])
+  if found_words:
+    words = ", ".join([f"{word} => {alt}" for word, alt in found_words.items()])
+    fail(
+        f"Use inclusive language where possible and accurate. Found: {words} in"
+        f" {args['cell_source']}"
+    )
+  else:
+    return True
diff --git a/tools/tensorflow_docs/tools/nblint/style/google.py b/tools/tensorflow_docs/tools/nblint/style/google.py
new file mode 100644
index 00000000000..9bd1a47376a
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/style/google.py
@@ -0,0 +1,93 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Lint assertions that adhere to the Google dev docs style guide.
+
+This style module is a non-exhaustive implementation of style rules found in the
+Google developer documentation style guide: https://developers.google.com/style
+
+When adding lints, please link to the URL of the relevant style rule.
+"""
+import re
+
+from tensorflow_docs.tools.nblint.decorator import fail
+from tensorflow_docs.tools.nblint.decorator import lint
+from tensorflow_docs.tools.nblint.decorator import Options
+
+
+def search_wordlist(wordlist, src_str):
+  """Search for wordlist entries in text and return set of found items.
+
+  Args:
+    wordlist: Dict of word entries and recommendations to search in string.
+    src_str: String to search for word entries.
+
+  Returns:
+    A dict that is a subset of entries from `wordlist` found in `src_str`.
+  """
+  found_words = {}
+  for word in wordlist:
+    # Word-boundary and ignore between path separator '/'.
+    if re.search(rf"[^/]\b{word}\b[^/]", src_str, re.IGNORECASE):
+      alt_word = wordlist[word]
+      if not alt_word:
+        alt_word = "n/a"
+      found_words[word] = alt_word
+  return found_words
+
+
+# Non-exhaustive list: {word: alt-word} (Use False if alt not provided.)
+_INCLUSIVE_WORDLIST = {
+    "blacklist": "blocked",
+    "whitelist": "allowed",
+    "master": "primary",
+    "slave": "replica",
+    "native": "built-in"
+}
+
+
+@lint(
+    message="Use inclusive language: https://developers.google.com/style/inclusive-documentation",
+    cond=Options.Cond.ALL)
+def inclusive_language(args):
+  """Test for words found in inclusive wordlist and recommend alternatives."""
+  found_words = search_wordlist(_INCLUSIVE_WORDLIST, args["cell_source"])
+  if found_words:
+    words = ", ".join([f"{word} => {alt}" for word, alt in found_words.items()])
+    fail(
+        f"Use inclusive language where possible and accurate. Found: {words} in"
+        f" {args['cell_source']}"
+    )
+  else:
+    return True
+
+
+# Non-exhaustive list: {word: alt-word} (Use False if alt not provided.)
+_SECOND_PERSON_WORDLIST = {"we": "you", "we're": "you are"}
+
+
+@lint(
+    message="Prefer second person instead of first person: https://developers.google.com/style/person",
+    cond=Options.Cond.ALL)
+def second_person(args):
+  """Test for first person usage in doc and recommend second person."""
+  found_words = search_wordlist(_SECOND_PERSON_WORDLIST, args["cell_source"])
+  if found_words:
+    words = ", ".join([f"{word} => {alt}" for word, alt in found_words.items()])
+    fail(
+        f"Prefer second person instead of first person. Found: {words} in"
+        f" {args['cell_source']}"
+    )
+  else:
+    return True
diff --git a/tools/tensorflow_docs/tools/nblint/style/tensorflow.py b/tools/tensorflow_docs/tools/nblint/style/tensorflow.py
new file mode 100644
index 00000000000..611562c43fa
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/style/tensorflow.py
@@ -0,0 +1,341 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Lint assertions for notebooks published on tensorflow.org.
+
+These lints are a non-exhaustive implementation of style rules found in the
+TensorFlow documentation and style guides. See:
+
+- https://www.tensorflow.org/community/contribute/docs
+- https://www.tensorflow.org/community/contribute/docs_style
+
+When adding lints, link to the URL of the relevant style rule, if applicable.
+
+Lint functions return a boolean: True to pass, False to fail.
+For @lint options, see the docstrings in `decorator.py`.
+
+Lint callback functions are passed an `args` dict with the following entries:
+  cell_data: Dict of parsed cell (cell-scope only)
+  cell_source: String of cell content (cell-scope only)
+  file_data: Dict of parsed notebook
+  file_source: String of notebook content
+  path: Filepath of notebook
+  user: Dict of args passed at the command-line
+"""
+import pathlib
+import re
+import urllib
+
+from tensorflow_docs.tools.nblint import fix
+from tensorflow_docs.tools.nblint.decorator import fail
+from tensorflow_docs.tools.nblint.decorator import lint
+from tensorflow_docs.tools.nblint.decorator import Options
+
+
+# Acceptable copyright heading for notebooks following this style.
+copyrights_re = [
+    r"Copyright 20[1-9][0-9] The TensorFlow\s.*?\s?Authors",
+    r"Copyright 20[1-9][0-9] Google",
+    r"Copyright 20[1-9][0-9] The AI Edge Authors",
+]
+
+
+@lint(message="Copyright required", scope=Options.Scope.TEXT)
+def copyright_check(args):
+  cell_source = args["cell_source"]
+  return any(re.search(pattern, cell_source) for pattern in copyrights_re)
+
+
+license_re = re.compile("#\s?@title Licensed under the Apache License")
+
+
+@lint(
+    message="Apache license cell is required",
+    scope=Options.Scope.CODE,
+    cond=Options.Cond.ANY)
+def license_check(args):
+  if license_re.search(args["cell_source"]):
+    return True
+  else:
+    template_url = "https://github.com/tensorflow/docs/blob/master/tools/templates/notebook.ipynb"
+    fail(f"License cell missing or doesn't follow template: {template_url}")
+
+
+@lint(scope=Options.Scope.FILE)
+def not_translation(args):
+  if "site" not in args["path"].parents:
+    return True
+  else:
+    return "site/en" in args["path"].parents
+
+
+# Button checks
+
+# Look for class="tfo-notebook-buttons" (CSS used on website versions) or the
+# run-in-colab logo (for notebooks that stick to GitHub/Colab).
+is_button_cell_re = re.compile(
+    r"class.*tfo-notebook-buttons|colab_logo_32px\.png"
+)
+
+
+def get_arg_or_fail(user_args, arg_name, arg_fmt):
+  """Get value of the user-defined arg passed at the command-line.
+
+  Args:
+    user_args: Dict containing user-defined args passed at command-line.
+    arg_name: String name of user-defined arg.
+    arg_fmt: String format of expected user-defined arg.
+
+  Returns:
+    Value of arg passed to command-line. If the arg does not exist, raise a
+    failure, log a message, and skip the lint function.
+  """
+  if arg_name in user_args:
+    return user_args.get(arg_name)
+  else:
+    fail(
+        f"Requires user-argument '{arg_name}': nblint --arg={arg_name}:{arg_fmt} ...",
+        always_show=True)
+
+
+def split_doc_path(filepath):
+  """Return paths for docs root prefix directory and the relative path to file.
+
+  Given a full path to notebook file, standalone or within an established
+  documentation directory layout, split the provided path into two:
+  1. a path reprsenting the prefix directory to the docs root (if it exists),
+  2. the relative path to the file from the docs root directory.
+  If in an unknown docs directory layout, return an empty prefix path and the
+  full path of the original argument.
+
+  For example:
+  "site/en/notebook.ipynb" => ("site/en", "notebook.ipynb")
+  "tensorflow/docs/notebook.ipynb" => ("docs", "notebook.ipynb")
+  "unknown/path/notebook.ipynb" => ("", "unknown/path/notebook.ipynb")
+
+  Args:
+    filepath: `pathlib.Path` to a documentation notebook.
+
+  Returns:
+    pathlib.Path: The path of the doc root prefix directory., if applicable.
+    pathlib.Path: The relative path to notebook from the prefix directory.
+  """
+  fp_full = filepath.resolve()  # Check full path for sub-elements.
+
+  def split_path_on_dir(fp, dirname, offset=1):
+    parts = fp.parts
+    idx = parts.index(dirname)
+    docs_dir = pathlib.Path(*parts[idx:idx + offset])
+    rel_path = fp.relative_to(*parts[:idx + offset])
+    return docs_dir, rel_path
+
+  if "site" in fp_full.parts:
+    return split_path_on_dir(fp_full, "site", offset=2)  # site//
+  elif "docs" in fp_full.parts:
+    return split_path_on_dir(fp_full, "docs")
+  elif "g3doc" in fp_full.parts:
+    idx = fp_full.parts.index("g3doc")
+    if fp_full.parts[idx + 1] == "en":
+      offset = 2
+    else:
+      offset = 1
+    return split_path_on_dir(fp_full, "g3doc", offset=offset)
+  else:
+    # Unknown setup. Return empty root and unsplit path.
+    return pathlib.Path(), filepath
+
+
+@lint(
+    message="Missing or malformed URL in Colab button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY)
+def button_colab(args):
+  """Test that the URL in the Colab button matches the file path."""
+  cell_source = args["cell_source"]
+  repo = get_arg_or_fail(args["user"], "repo", "")
+  branch = args["user"].get("branch", "master")
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  # Buttons use OSS URLs.
+  if str(docs_dir) == "g3doc/en":
+    docs_dir = pathlib.Path("site/en")
+
+  base_url = f"colab.research.google.com/github/{repo}/blob/{branch}"
+  this_url = "https://" + str(base_url / docs_dir / rel_path)
+
+  if is_button_cell_re.search(cell_source) and cell_source.find(this_url) != -1:
+    return True
+  else:
+    fail(
+        f"Colab button URL doesn't match: {this_url}",
+        fix=fix.regex_between_groups_replace_all,
+        fix_args=[r"(href.*)http.*?(\\\".*colab_logo_32px.png)", this_url])
+
+
+@lint(
+    message="Missing or malformed URL in Download button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY)
+def button_download(args):
+  """Test that the URL in the Download button matches the file path."""
+  cell_source = args["cell_source"]
+  repo = get_arg_or_fail(args["user"], "repo", "")
+  repo_name = pathlib.Path(repo.split("/")[1])
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  if "r1" in rel_path.parts:
+    return True  # No download button for TF 1.x docs.
+
+  # Buttons use OSS URLs.
+  if str(docs_dir) == "g3doc/en":
+    docs_dir = pathlib.Path("site/en")
+
+  this_url = urllib.parse.urljoin(
+      "https://storage.googleapis.com",
+      str(f"tensorflow_docs/{repo_name}" / docs_dir / rel_path))
+
+  if is_button_cell_re.search(cell_source) and cell_source.find(this_url) != -1:
+    return True
+  else:
+    fail(
+        f"Download button URL doesn't match: {this_url}",
+        fix=fix.regex_between_groups_replace_all,
+        fix_args=[r"(href.*)http.*?(\\\".*download_logo_32px.png)", this_url])
+
+
+@lint(
+    message="Missing or malformed URL in GitHub button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY)
+def button_github(args):
+  """Test that the URL in the GitHub button matches the file path."""
+  cell_source = args["cell_source"]
+  repo = get_arg_or_fail(args["user"], "repo", "")
+  branch = args["user"].get("branch", "master")
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  # Buttons use OSS URLs.
+  if str(docs_dir) == "g3doc/en":
+    docs_dir = pathlib.Path("site/en")
+
+  base_url = f"github.com/{repo}/blob/{branch}"
+  this_url = "https://" + str(base_url / docs_dir / rel_path)
+
+  if is_button_cell_re.search(cell_source) and cell_source.find(this_url) != -1:
+    return True
+  else:
+    fail(
+        f"GitHub button URL doesn't match: {this_url}",
+        fix=fix.regex_between_groups_replace_all,
+        fix_args=[r"(href.*)http.*?(\\\".*GitHub-Mark-32px.png)", this_url])
+
+
+@lint(
+    message="Missing or malformed URL in 'View on' button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY)
+def button_website(args):
+  """Test that the website URL in the 'View on' button matches the file path.
+
+  Because of subsites and different output directories, the exact website path
+  can't be known from the file alone. But can check that the URL matches a
+  correct pattern.
+
+  Args:
+    args: Nested dict of runtime arguments.
+
+  Returns:
+    Boolean: True if lint test passes, False if not.
+  """
+  cell_source = args["cell_source"]
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  if "r1" in rel_path.parts:
+    return True  # No website button for TF 1.x docs.
+
+  user_url = args["user"].get("base_url")
+  if user_url:
+    base_url = user_url
+  elif str(docs_dir) == "site/zh-cn" or str(docs_dir) == "site/zh-tw":
+    base_url = "https://tensorflow.google.cn/"
+  else:
+    base_url = "https://www.tensorflow.org/"
+
+  # Construct website URL pattern based on location of this file in repo.
+  url_path = rel_path.with_suffix("")
+  # If run in source repo, we don't know for certain the published subsite URL.
+  # Match: base//notebook-path
+  this_url = rf"{base_url}[\w\-/]*{url_path}"
+
+  if is_button_cell_re.search(cell_source) and re.search(this_url, cell_source):
+    return True
+  else:
+    # If included verbatim, bracket will fail lint. That's desired.
+    url_format = f"{base_url}/{url_path}"
+    fail(f"'View on' button URL doesn't match pattern: {url_format}")
+
+
+@lint(
+    message="Missing or malformed URL in 'TFHub' button.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ANY)
+def button_hub(args):
+  """Notebooks that mention tfhub.dev should have a TFHub button."""
+  cell_source = args["cell_source"]
+  file_source = args["file_source"]
+
+  hub_url = "https://tfhub.dev/"
+
+  # Only check files that mention TFHub.
+  if file_source.find(hub_url) == -1:
+    return True
+
+  if is_button_cell_re.search(cell_source) and cell_source.find(hub_url) != -1:
+    return True
+  else:
+    # If included verbatim, bracket will fail lint. That's desired.
+    url_format = f"{hub_url}"
+    fail(f"'TFHub' button URL doesn't match pattern: {url_format}")
+
+
+@lint(
+    message="Remove extra buttons from TF 1.x docs.",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ALL)
+def button_r1_extra(args):
+  """The r1/ docs should not have website or download buttons."""
+  cell_source = args["cell_source"]
+  docs_dir, rel_path = split_doc_path(args["path"])
+
+  # Only test r1/ notebooks.
+  if "r1" not in rel_path.parts:
+    return True
+  # Only check text cells that contain the button nav bar.
+  if not is_button_cell_re.search(cell_source):
+    return True
+
+  download_url = "https://storage.googleapis.com/tensorflow_docs/"
+  if str(docs_dir) == "site/zh-cn" or str(docs_dir) == "site/zh-tw":
+    base_url = "https://tensorflow.google.cn/"
+  else:
+    base_url = "https://www.tensorflow.org/"
+
+  # Look for button URLs that shouldn't be there..
+  if (re.search(f"{base_url}/(?!images)", cell_source) or
+      cell_source.find(download_url) != -1):
+    fail(
+        "Remove the 'View on' and 'Download notebook' buttons since r1/ docs are not published."
+    )
+  else:
+    return True
diff --git a/tools/tensorflow_docs/tools/nblint/style/tensorflow_docs_l10n.py b/tools/tensorflow_docs/tools/nblint/style/tensorflow_docs_l10n.py
new file mode 100644
index 00000000000..84ed779b795
--- /dev/null
+++ b/tools/tensorflow_docs/tools/nblint/style/tensorflow_docs_l10n.py
@@ -0,0 +1,132 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Lint assertions specific to the tensorflow/docs-l10n repo.
+
+When adding lints, link to the URL of the relevant style rule, if applicable.
+
+Lint functions return a boolean: True to pass, False to fail.
+For @lint options, see the docstrings in `decorator.py`.
+
+Lint callback functions are passed an `args` dict with the following entries:
+  cell_data: Dict of parsed cell (cell-scope only)
+  cell_source: String of cell content (cell-scope only)
+  file_data: Dict of parsed notebook
+  file_source: String of notebook content
+  path: Filepath of notebook
+  user: Dict of args passed at the command-line
+"""
+import re
+
+from tensorflow_docs.tools.nblint import fix
+from tensorflow_docs.tools.nblint.decorator import fail
+from tensorflow_docs.tools.nblint.decorator import lint
+from tensorflow_docs.tools.nblint.decorator import Options
+from tensorflow_docs.tools.nblint.style.tensorflow import is_button_cell_re
+from tensorflow_docs.tools.nblint.style.tensorflow import split_doc_path
+
+
+@lint(
+    message="Only edit translated files. Source files are here: https://github.com/tensorflow/docs",
+    scope=Options.Scope.FILE)
+def is_translation(args):
+  """Translations live in the site// directory of the docs-l10n repo."""
+  path_str = str(args["path"].resolve())
+
+  if "site/" not in path_str:
+    return False
+  elif "site/en/" in path_str:
+    return False
+  elif "site/en-snapshot/" in path_str:
+    return False
+  else:
+    return True
+
+
+# Catch tensorflow.org hostname usage in Chinese docs. Ignore false positives
+# for the Google Group (../a/tensorflow.org/..), email (docs*@tensorflow.org),
+# and subdomains like (blog|download|js).tensorflow.org.
+has_tf_hostname_re = re.compile(
+    r"(?", re.IGNORECASE)
+has_copyright_re = re.compile(r"Copyright 20[1-9][0-9]")
+
+
+@lint(
+    message="RTL languages must wrap all text cell elements with: 
    ...
    ",
+    scope=Options.Scope.TEXT,
+    cond=Options.Cond.ALL)
+def rtl_language_wrap(args):
+  """Check that RTL languages wrap text elemenst in a directional div.
+
+  Required for languages like Arabic to render correctly in Colab. Some care
+  must be taken or any Markdown syntax within the div will break.
+
+  Args:
+    args: Nested dict of runtime arguments.
+
+  Returns:
+    Boolean: True if lint test passes, False if not.
+  """
+  docs_dir, _ = split_doc_path(args["path"])
+
+  # Only applicable for RTL languages.
+  if str(docs_dir) != "site/ar":
+    return True
+
+  cell_source = args["cell_source"]
+
+  # Ignore the text cells for copyright and buttons.
+  if (has_copyright_re.search(cell_source) or
+      is_button_cell_re.search(cell_source)):
+    return True
+
+  if has_rtl_div_re.search(cell_source):
+    return True
+  else:
+    fail(
+        "Wrap all text elements in `
    ...
    ` for Colab. But check this doesn't break any Markdown syntax within."
+    )
diff --git a/tools/tensorflow_docs/vis/__init__.py b/tools/tensorflow_docs/vis/__init__.py
index 058f45dc625..9043810d541 100644
--- a/tools/tensorflow_docs/vis/__init__.py
+++ b/tools/tensorflow_docs/vis/__init__.py
@@ -17,4 +17,4 @@
 Use this module for plotting and visualization code that is too long to inline
 into a notebook.
 """
-from tensorflow_docs.vis.webp_animation import Webp
+
diff --git a/tools/tensorflow_docs/vis/embed.py b/tools/tensorflow_docs/vis/embed.py
new file mode 100644
index 00000000000..720440cd7c4
--- /dev/null
+++ b/tools/tensorflow_docs/vis/embed.py
@@ -0,0 +1,49 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Simple functions for embedding data in a notebook file."""
+
+import base64
+import mimetypes
+import os
+import pathlib
+import textwrap
+
+import IPython.display
+
+
+def embed_data(mime: str, data: bytes) -> IPython.display.HTML:
+  """Embeds data as an html tag with a data-url."""
+  b64 = base64.b64encode(data).decode()
+  if mime.startswith('image'):
+    tag = f''
+  elif mime.startswith('video'):
+    tag = textwrap.dedent(f"""
+        
+        """)
+  else:
+    raise ValueError('Images and Video only.')
+  return IPython.display.HTML(tag)
+
+
+def embed_file(path: os.PathLike) -> IPython.display.HTML:
+  """Embeds a file in the notebook as an html tag with a data-url."""
+  path = pathlib.Path(path)
+  mime, unused_encoding = mimetypes.guess_type(str(path))
+  data = path.read_bytes()
+
+  return embed_data(mime, data)
diff --git a/tools/tensorflow_docs/vis/webp_animation.py b/tools/tensorflow_docs/vis/webp_animation.py
deleted file mode 100644
index 3fc92cda950..00000000000
--- a/tools/tensorflow_docs/vis/webp_animation.py
+++ /dev/null
@@ -1,160 +0,0 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Easy notebook embedded webp animations.
-
-```
-import tensorflow_docs.vis.webp_animation as webp_animation
-
-env = gym.make('SpaceInvaders-v0')
-obs = env.reset()
-done = False
-n = 0
-
-anim = webp_animation.Webp()
-
-while not done:
-  img = env.render(mode = 'rgb_array')
-  anim.append(img)
-  act = env.action_space.sample() # take a random action
-  obs, reward, done, info = env.step(act)
-  n += 1
-
-anim.save("test.webp")
-anim
-```
-"""
-
-import base64
-
-import numpy as np
-import PIL.Image
-import webp
-
-
-class Webp(object):
-  """Builds a webp animation.
-
-  Attributes:
-    frame_rate: The default frame rate for appended images.
-    shape: The shape of the animation frames. Will default to the size of the
-      first image if not set.
-    result: The binary image data string. Once the animation has been used, it
-      can no longer updated. And the result field contains the webp encoded
-      data.
-  """
-
-  def __init__(self, shape=None, frame_rate=60.0, **options):
-    """A notebook-embedable webp animation.
-
-    Args:
-      shape: Optional. The image_shape of the animation. Defaults to the shape
-        of the first image if unset.
-      frame_rate: The default frame rate for the animation.
-      **options: Additional arguments passed to `WebPAnimEncoderOptions.new`.
-    """
-    self.frame_rate = frame_rate
-    self._timestamp_ms = 0
-    self._empty = True
-
-    if options is None:
-      options = {}
-
-    self._options = webp.WebPAnimEncoderOptions.new(**options)
-    self._encoder = None
-    self._shape = shape
-    self._result = None
-
-  def append(self, img, dt_ms=None):
-    """Append an image to the animation.
-
-    Args:
-      img: The image to add.
-      dt_ms: override the animation frame rate for this frame with a frame
-        length in ms.
-
-    Raises:
-      ValueError:
-        * if the video has already been "assembled" (used).
-        * if `img` does not match the shape of the animation.
-    """
-    if self._result is not None:
-      raise ValueError(
-          "Can't append to an animation after it has been \"assembled\" (used)."
-      )
-    self._empty = False
-
-    if not isinstance(img, PIL.Image.Image):
-      img = np.asarray(img)
-      img = PIL.Image.fromarray(img)
-
-    if self._shape is None:
-      self._shape = img.size
-
-    if self._encoder is None:
-      self._encoder = webp.WebPAnimEncoder.new(self.shape[0], self.shape[1],
-                                               self._options)
-
-    if img.size != self.shape:
-      raise ValueError("Image shape does not match video shape")
-
-    img = webp.WebPPicture.from_pil(img)
-
-    self._encoder.encode_frame(img, int(self._timestamp_ms))
-
-    if dt_ms is None:
-      self._timestamp_ms += 1000 * (1.0 / self.frame_rate)
-    else:
-      self._timestamp_ms += dt_ms
-
-  def extend(self, imgs, dt_ms=None):
-    """Extend tha animation with an iterable if images.
-
-    Args:
-      imgs: An iterable of images, to pass to `.append`.
-      dt_ms: Override the animation frame rate for these frames with a frame
-        length in ms.
-    """
-    for img in imgs:
-      self.append(img, dt_ms=dt_ms)
-
-  @property
-  def result(self):
-    result = self._result
-    if result is None:
-      anim_data = self._encoder.assemble(int(self._timestamp_ms))
-      result = anim_data.buffer()
-      self._result = result
-    return result
-
-  @property
-  def shape(self):
-    """The shape of the animation. Read only once set."""
-    return self._shape
-
-  def _repr_html_(self):
-    """Notebook display hook, embed the image in an  tag."""
-    if self._empty:
-      return "Empty Animation"
-
-    # convert raw binary data to base64
-    img_data = base64.b64encode(self.result).decode("utf-8")
-
-    img_html = "" % img_data
-    return img_html
-
-  def save(self, filename):
-    """Write the webp data to a file."""
-    with open(filename, "wb") as f:
-      f.write(self.result)
diff --git a/tools/tensorflow_docs/vis/webp_test.py b/tools/tensorflow_docs/vis/webp_test.py
deleted file mode 100644
index a2f2b9b22c5..00000000000
--- a/tools/tensorflow_docs/vis/webp_test.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for tensorflow_docs.vis.webp."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-
-from absl.testing import absltest
-
-import numpy as np
-import PIL.Image
-
-from tensorflow_docs.vis import webp_animation
-
-
-class WebpTest(absltest.TestCase):
-
-  def test_smoke(self):
-    workdir = self.create_tempdir().full_path
-
-    img = PIL.Image.fromarray(np.zeros([10, 12, 3], dtype=np.uint8))
-    anim = webp_animation.Webp()
-
-    anim.append(img)
-    anim.extend([img])
-    anim.save(os.path.join(workdir, 'test.webp'))
-
-
-if __name__ == '__main__':
-  absltest.main()